From bf07bc52ad387c3cec5179aac1a8a08bfe0e24f7 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 15 Jun 2011 12:36:06 +0200 Subject: [PATCH 115/347] hotplug: Lightweight get online cpus Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/4.19/older/patches-4.19.246-rt110.tar.xz get_online_cpus() is a heavy weight function which involves a global mutex. migrate_disable() wants a simpler construct which prevents only a CPU from going doing while a task is in a migrate disabled section. Implement a per cpu lockless mechanism, which serializes only in the real unplug case on a global mutex. That serialization affects only tasks on the cpu which should be brought down. Signed-off-by: Thomas Gleixner --- include/linux/cpu.h | 5 +++++ kernel/cpu.c | 15 +++++++++++++++ kernel/sched/core.c | 4 ++++ 3 files changed, 24 insertions(+) diff --git a/include/linux/cpu.h b/include/linux/cpu.h index aab4273810e3..e67645924404 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -118,6 +118,8 @@ extern void cpu_hotplug_disable(void); extern void cpu_hotplug_enable(void); void clear_tasks_mm_cpumask(int cpu); int cpu_down(unsigned int cpu); +extern void pin_current_cpu(void); +extern void unpin_current_cpu(void); #else /* CONFIG_HOTPLUG_CPU */ @@ -129,6 +131,9 @@ static inline int cpus_read_trylock(void) { return true; } static inline void lockdep_assert_cpus_held(void) { } static inline void cpu_hotplug_disable(void) { } static inline void cpu_hotplug_enable(void) { } +static inline void pin_current_cpu(void) { } +static inline void unpin_current_cpu(void) { } + #endif /* !CONFIG_HOTPLUG_CPU */ /* Wrappers which go away once all code is converted */ diff --git a/kernel/cpu.c b/kernel/cpu.c index d6fd362afc81..89a96cc14b39 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -283,6 +283,21 @@ static int cpu_hotplug_disabled; #ifdef CONFIG_HOTPLUG_CPU +/** + * pin_current_cpu - Prevent the current cpu from being unplugged + */ +void pin_current_cpu(void) +{ + +} + +/** + * unpin_current_cpu - Allow unplug of current cpu + */ +void unpin_current_cpu(void) +{ +} + DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock); void cpus_read_lock(void) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index ba6f445ac9e8..ed79f921a84e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7243,6 +7243,7 @@ void migrate_disable(void) } preempt_disable(); + pin_current_cpu(); migrate_disable_update_cpus_allowed(p); p->migrate_disable = 1; @@ -7308,12 +7309,15 @@ void migrate_enable(void) arg.task = p; arg.dest_cpu = dest_cpu; + unpin_current_cpu(); preempt_enable(); stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg); tlb_migrate_finish(p->mm); + return; } } + unpin_current_cpu(); preempt_enable(); } EXPORT_SYMBOL(migrate_enable); -- 2.36.1