From aed289fe7a9923b6c8a98fa08048d56aa97c71c4 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Wed, 28 Jan 2015 17:14:16 +0100 Subject: [PATCH 209/323] mm/memcontrol: Replace local_irq_disable with local locks Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/5.10/older/patches-5.10.204-rt100.tar.xz There are a few local_irq_disable() which then take sleeping locks. This patch converts them local locks. [bigeasy: Move unlock after memcg_check_events() in mem_cgroup_swapout(), pointed out by Matt Fleming ] Signed-off-by: Sebastian Andrzej Siewior --- mm/memcontrol.c | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 53522a52ff15..5bf696318643 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -63,6 +63,7 @@ #include #include #include "slab.h" +#include #include @@ -93,6 +94,13 @@ bool cgroup_memory_noswap __read_mostly; static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq); #endif +struct event_lock { + local_lock_t l; +}; +static DEFINE_PER_CPU(struct event_lock, event_lock) = { + .l = INIT_LOCAL_LOCK(l), +}; + /* Whether legacy memory+swap accounting is active */ static bool do_memsw_account(void) { @@ -5747,12 +5755,12 @@ static int mem_cgroup_move_account(struct page *page, ret = 0; - local_irq_disable(); + local_lock_irq(&event_lock.l); mem_cgroup_charge_statistics(to, page, nr_pages); memcg_check_events(to, page); mem_cgroup_charge_statistics(from, page, -nr_pages); memcg_check_events(from, page); - local_irq_enable(); + local_unlock_irq(&event_lock.l); out_unlock: unlock_page(page); out: @@ -6822,10 +6830,10 @@ int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask) css_get(&memcg->css); commit_charge(page, memcg); - local_irq_disable(); + local_lock_irq(&event_lock.l); mem_cgroup_charge_statistics(memcg, page, nr_pages); memcg_check_events(memcg, page); - local_irq_enable(); + local_unlock_irq(&event_lock.l); /* * Cgroup1's unified memory+swap counter has been charged with the @@ -6881,11 +6889,11 @@ static void uncharge_batch(const struct uncharge_gather *ug) memcg_oom_recover(ug->memcg); } - local_irq_save(flags); + local_lock_irqsave(&event_lock.l, flags); __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout); __this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_pages); memcg_check_events(ug->memcg, ug->dummy_page); - local_irq_restore(flags); + local_unlock_irqrestore(&event_lock.l, flags); /* drop reference from uncharge_page */ css_put(&ug->memcg->css); @@ -7039,10 +7047,10 @@ void mem_cgroup_migrate(struct page *oldpage, struct page *newpage) css_get(&memcg->css); commit_charge(newpage, memcg); - local_irq_save(flags); + local_lock_irqsave(&event_lock.l, flags); mem_cgroup_charge_statistics(memcg, newpage, nr_pages); memcg_check_events(memcg, newpage); - local_irq_restore(flags); + local_unlock_irqrestore(&event_lock.l, flags); } DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key); @@ -7217,6 +7225,7 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry) struct mem_cgroup *memcg, *swap_memcg; unsigned int nr_entries; unsigned short oldid; + unsigned long flags; VM_BUG_ON_PAGE(PageLRU(page), page); VM_BUG_ON_PAGE(page_count(page), page); @@ -7262,9 +7271,13 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry) * important here to have the interrupts disabled because it is the * only synchronisation we have for updating the per-CPU variables. */ + local_lock_irqsave(&event_lock.l, flags); +#ifndef CONFIG_PREEMPT_RT VM_BUG_ON(!irqs_disabled()); +#endif mem_cgroup_charge_statistics(memcg, page, -nr_entries); memcg_check_events(memcg, page); + local_unlock_irqrestore(&event_lock.l, flags); css_put(&memcg->css); } -- 2.43.0