From: Thomas Gleixner <tglx@linutronix.de>
Date: Wed, 15 Jun 2011 12:36:06 +0200
Subject: [PATCH 115/354] hotplug: Lightweight get online cpus
Origin: https://git.kernel.org/cgit/linux/kernel/git/rt/linux-stable-rt.git/commit?id=3ba8eb74c6b86a06075162358d936bfe7e1c0dfa

get_online_cpus() is a heavy weight function which involves a global
mutex. migrate_disable() wants a simpler construct which prevents only
a CPU from going doing while a task is in a migrate disabled section.

Implement a per cpu lockless mechanism, which serializes only in the
real unplug case on a global mutex. That serialization affects only
tasks on the cpu which should be brought down.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 include/linux/cpu.h |  5 +++++
 kernel/cpu.c        | 15 +++++++++++++++
 kernel/sched/core.c |  4 ++++
 3 files changed, 24 insertions(+)

diff --git a/include/linux/cpu.h b/include/linux/cpu.h
index c376a59a3e42..90985dbce8f4 100644
--- a/include/linux/cpu.h
+++ b/include/linux/cpu.h
@@ -124,6 +124,8 @@ extern void cpu_hotplug_disable(void);
 extern void cpu_hotplug_enable(void);
 void clear_tasks_mm_cpumask(int cpu);
 int cpu_down(unsigned int cpu);
+extern void pin_current_cpu(void);
+extern void unpin_current_cpu(void);
 
 #else /* CONFIG_HOTPLUG_CPU */
 
@@ -135,6 +137,9 @@ static inline int  cpus_read_trylock(void) { return true; }
 static inline void lockdep_assert_cpus_held(void) { }
 static inline void cpu_hotplug_disable(void) { }
 static inline void cpu_hotplug_enable(void) { }
+static inline void pin_current_cpu(void) { }
+static inline void unpin_current_cpu(void) { }
+
 #endif	/* !CONFIG_HOTPLUG_CPU */
 
 /* Wrappers which go away once all code is converted */
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 34c09c3d37bc..b25993345ea6 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -284,6 +284,21 @@ static int cpu_hotplug_disabled;
 
 #ifdef CONFIG_HOTPLUG_CPU
 
+/**
+ * pin_current_cpu - Prevent the current cpu from being unplugged
+ */
+void pin_current_cpu(void)
+{
+
+}
+
+/**
+ * unpin_current_cpu - Allow unplug of current cpu
+ */
+void unpin_current_cpu(void)
+{
+}
+
 DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
 
 void cpus_read_lock(void)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 3c1f897b8b70..be29571fa5dc 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -7246,6 +7246,7 @@ void migrate_disable(void)
 	}
 
 	preempt_disable();
+	pin_current_cpu();
 
 	migrate_disable_update_cpus_allowed(p);
 	p->migrate_disable = 1;
@@ -7311,12 +7312,15 @@ void migrate_enable(void)
 			arg.task = p;
 			arg.dest_cpu = dest_cpu;
 
+			unpin_current_cpu();
 			preempt_enable();
 			stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
 			tlb_migrate_finish(p->mm);
+
 			return;
 		}
 	}
+	unpin_current_cpu();
 	preempt_enable();
 }
 EXPORT_SYMBOL(migrate_enable);