From 784e4a9806842dc64aa0902210fe4e11984d868a Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 19 Jul 2017 17:31:20 +0200 Subject: [PATCH 228/347] cpu/hotplug: Implement CPU pinning Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/4.19/older/patches-4.19.246-rt110.tar.xz Signed-off-by: Thomas Gleixner --- include/linux/sched.h | 1 + kernel/cpu.c | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/include/linux/sched.h b/include/linux/sched.h index 577045ae8693..7523c0786a63 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -671,6 +671,7 @@ struct task_struct { #if defined(CONFIG_SMP) && defined(CONFIG_PREEMPT_RT_BASE) int migrate_disable; int migrate_disable_update; + int pinned_on_cpu; # ifdef CONFIG_SCHED_DEBUG int migrate_disable_atomic; # endif diff --git a/kernel/cpu.c b/kernel/cpu.c index 89a96cc14b39..797216a0b4c6 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -77,6 +77,11 @@ static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = { .fail = CPUHP_INVALID, }; +#ifdef CONFIG_HOTPLUG_CPU +static DEFINE_PER_CPU(struct rt_rw_lock, cpuhp_pin_lock) = \ + __RWLOCK_RT_INITIALIZER(cpuhp_pin_lock); +#endif + #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP) static struct lockdep_map cpuhp_state_up_map = STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map); @@ -288,7 +293,28 @@ static int cpu_hotplug_disabled; */ void pin_current_cpu(void) { + struct rt_rw_lock *cpuhp_pin; + unsigned int cpu; + int ret; + +again: + cpuhp_pin = this_cpu_ptr(&cpuhp_pin_lock); + ret = __read_rt_trylock(cpuhp_pin); + if (ret) { + current->pinned_on_cpu = smp_processor_id(); + return; + } + cpu = smp_processor_id(); + preempt_enable(); + + __read_rt_lock(cpuhp_pin); + preempt_disable(); + if (cpu != smp_processor_id()) { + __read_rt_unlock(cpuhp_pin); + goto again; + } + current->pinned_on_cpu = cpu; } /** @@ -296,6 +322,13 @@ void pin_current_cpu(void) */ void unpin_current_cpu(void) { + struct rt_rw_lock *cpuhp_pin = this_cpu_ptr(&cpuhp_pin_lock); + + if (WARN_ON(current->pinned_on_cpu != smp_processor_id())) + cpuhp_pin = per_cpu_ptr(&cpuhp_pin_lock, current->pinned_on_cpu); + + current->pinned_on_cpu = -1; + __read_rt_unlock(cpuhp_pin); } DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock); @@ -920,6 +953,7 @@ static int take_cpu_down(void *_param) static int takedown_cpu(unsigned int cpu) { + struct rt_rw_lock *cpuhp_pin = per_cpu_ptr(&cpuhp_pin_lock, cpu); struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); int err; @@ -932,11 +966,14 @@ static int takedown_cpu(unsigned int cpu) */ irq_lock_sparse(); + __write_rt_lock(cpuhp_pin); + /* * So now all preempt/rcu users must observe !cpu_active(). */ err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu)); if (err) { + __write_rt_unlock(cpuhp_pin); /* CPU refused to die */ irq_unlock_sparse(); /* Unpark the hotplug thread so we can rollback there */ @@ -955,6 +992,7 @@ static int takedown_cpu(unsigned int cpu) wait_for_ap_thread(st, false); BUG_ON(st->state != CPUHP_AP_IDLE_DEAD); + __write_rt_unlock(cpuhp_pin); /* Interrupts are moved away from the dying cpu, reenable alloc/free */ irq_unlock_sparse(); -- 2.36.1