1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
|
From: Thomas Gleixner <tglx@linutronix.de>
Date: Wed, 15 Jun 2011 12:36:06 +0200
Subject: [PATCH 115/351] hotplug: Lightweight get online cpus
Origin: https://git.kernel.org/cgit/linux/kernel/git/rt/linux-stable-rt.git/commit?id=e035a899f179507ef0a952663aaa96e83989be51
get_online_cpus() is a heavy weight function which involves a global
mutex. migrate_disable() wants a simpler construct which prevents only
a CPU from going doing while a task is in a migrate disabled section.
Implement a per cpu lockless mechanism, which serializes only in the
real unplug case on a global mutex. That serialization affects only
tasks on the cpu which should be brought down.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
include/linux/cpu.h | 5 +++++
kernel/cpu.c | 15 +++++++++++++++
kernel/sched/core.c | 4 ++++
3 files changed, 24 insertions(+)
diff --git a/include/linux/cpu.h b/include/linux/cpu.h
index 8cc06e1d4fc2..a3c246214d28 100644
--- a/include/linux/cpu.h
+++ b/include/linux/cpu.h
@@ -122,6 +122,8 @@ extern void cpu_hotplug_disable(void);
extern void cpu_hotplug_enable(void);
void clear_tasks_mm_cpumask(int cpu);
int cpu_down(unsigned int cpu);
+extern void pin_current_cpu(void);
+extern void unpin_current_cpu(void);
#else /* CONFIG_HOTPLUG_CPU */
@@ -133,6 +135,9 @@ static inline int cpus_read_trylock(void) { return true; }
static inline void lockdep_assert_cpus_held(void) { }
static inline void cpu_hotplug_disable(void) { }
static inline void cpu_hotplug_enable(void) { }
+static inline void pin_current_cpu(void) { }
+static inline void unpin_current_cpu(void) { }
+
#endif /* !CONFIG_HOTPLUG_CPU */
/* Wrappers which go away once all code is converted */
diff --git a/kernel/cpu.c b/kernel/cpu.c
index c9ca190ec034..f4c49bbc3fa3 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -284,6 +284,21 @@ static int cpu_hotplug_disabled;
#ifdef CONFIG_HOTPLUG_CPU
+/**
+ * pin_current_cpu - Prevent the current cpu from being unplugged
+ */
+void pin_current_cpu(void)
+{
+
+}
+
+/**
+ * unpin_current_cpu - Allow unplug of current cpu
+ */
+void unpin_current_cpu(void)
+{
+}
+
DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
void cpus_read_lock(void)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 7b4752fa34d1..b1771f3de84a 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -7244,6 +7244,7 @@ void migrate_disable(void)
}
preempt_disable();
+ pin_current_cpu();
migrate_disable_update_cpus_allowed(p);
p->migrate_disable = 1;
@@ -7309,12 +7310,15 @@ void migrate_enable(void)
arg.task = p;
arg.dest_cpu = dest_cpu;
+ unpin_current_cpu();
preempt_enable();
stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
tlb_migrate_finish(p->mm);
+
return;
}
}
+ unpin_current_cpu();
preempt_enable();
}
EXPORT_SYMBOL(migrate_enable);
|
