1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
|
From: Daniel Wagner <wagi@monom.org>
Date: Mon, 24 Oct 2022 10:58:29 +0200
Subject: [PATCH 346/351] rcu: Update rcuwait
Origin: https://git.kernel.org/cgit/linux/kernel/git/rt/linux-stable-rt.git/commit?id=6f1be9e81d653b56aeefbff1883d1f762e8b594e
This is an all in one commit backporting updates for rcuwait:
- 03f4b48edae7 ("rcuwait: Annotate task_struct with __rcu")
- 191a43be61d6 ("rcuwait: Introduce rcuwait_active()")
- 5c21f7b322cb ("rcuwait: Introduce prepare_to and finish_rcuwait")
- 80fbaf1c3f29 ("rcuwait: Add @state argument to rcuwait_wait_event()")
- 9d9a6ebfea32 ("rcuwait: Let rcuwait_wake_up() return whether or not a task was awoken")
- 58d4292bd037 ("rcu: Uninline multi-use function: finish_rcuwait()")
Signed-off-by: Daniel Wagner <wagi@monom.org>
---
include/linux/rcuwait.h | 42 +++++++++++++++++++++++++++--------
kernel/exit.c | 7 ++++--
kernel/locking/percpu-rwsem.c | 2 +-
kernel/rcu/update.c | 8 +++++++
4 files changed, 47 insertions(+), 12 deletions(-)
diff --git a/include/linux/rcuwait.h b/include/linux/rcuwait.h
index 90bfa3279a01..4fe9ecd56aac 100644
--- a/include/linux/rcuwait.h
+++ b/include/linux/rcuwait.h
@@ -3,6 +3,7 @@
#define _LINUX_RCUWAIT_H_
#include <linux/rcupdate.h>
+#include <linux/sched/signal.h>
/*
* rcuwait provides a way of blocking and waking up a single
@@ -18,7 +19,7 @@
* awoken.
*/
struct rcuwait {
- struct task_struct *task;
+ struct task_struct __rcu *task;
};
#define __RCUWAIT_INITIALIZER(name) \
@@ -29,14 +30,33 @@ static inline void rcuwait_init(struct rcuwait *w)
w->task = NULL;
}
-extern void rcuwait_wake_up(struct rcuwait *w);
+extern int rcuwait_wake_up(struct rcuwait *w);
+
+/*
+ * Note: this provides no serialization and, just as with waitqueues,
+ * requires care to estimate as to whether or not the wait is active.
+ */
+static inline int rcuwait_active(struct rcuwait *w)
+{
+ return !!rcu_access_pointer(w->task);
+}
/*
* The caller is responsible for locking around rcuwait_wait_event(),
- * such that writes to @task are properly serialized.
+ * and [prepare_to/finish]_rcuwait() such that writes to @task are
+ * properly serialized.
*/
-#define rcuwait_wait_event(w, condition) \
+
+static inline void prepare_to_rcuwait(struct rcuwait *w)
+{
+ rcu_assign_pointer(w->task, current);
+}
+
+extern void finish_rcuwait(struct rcuwait *w);
+
+#define rcuwait_wait_event(w, condition, state) \
({ \
+ int __ret = 0; \
/* \
* Complain if we are called after do_exit()/exit_notify(), \
* as we cannot rely on the rcu critical region for the \
@@ -44,21 +64,25 @@ extern void rcuwait_wake_up(struct rcuwait *w);
*/ \
WARN_ON(current->exit_state); \
\
- rcu_assign_pointer((w)->task, current); \
+ prepare_to_rcuwait(w); \
for (;;) { \
/* \
* Implicit barrier (A) pairs with (B) in \
* rcuwait_wake_up(). \
*/ \
- set_current_state(TASK_UNINTERRUPTIBLE); \
+ set_current_state(state); \
if (condition) \
break; \
\
+ if (signal_pending_state(state, current)) { \
+ __ret = -EINTR; \
+ break; \
+ } \
+ \
schedule(); \
} \
- \
- WRITE_ONCE((w)->task, NULL); \
- __set_current_state(TASK_RUNNING); \
+ finish_rcuwait(w); \
+ __ret; \
})
#endif /* _LINUX_RCUWAIT_H_ */
diff --git a/kernel/exit.c b/kernel/exit.c
index 2a414fc71b87..cf68896a94fa 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -291,8 +291,9 @@ struct task_struct *task_rcu_dereference(struct task_struct **ptask)
return task;
}
-void rcuwait_wake_up(struct rcuwait *w)
+int rcuwait_wake_up(struct rcuwait *w)
{
+ int ret = 0;
struct task_struct *task;
rcu_read_lock();
@@ -316,8 +317,10 @@ void rcuwait_wake_up(struct rcuwait *w)
*/
task = rcu_dereference(w->task);
if (task)
- wake_up_process(task);
+ ret = wake_up_process(task);
rcu_read_unlock();
+
+ return ret;
}
/*
diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c
index 883cf1b92d90..41787e80dbde 100644
--- a/kernel/locking/percpu-rwsem.c
+++ b/kernel/locking/percpu-rwsem.c
@@ -159,7 +159,7 @@ void percpu_down_write(struct percpu_rw_semaphore *sem)
*/
/* Wait for all now active readers to complete. */
- rcuwait_wait_event(&sem->writer, readers_active_check(sem));
+ rcuwait_wait_event(&sem->writer, readers_active_check(sem), TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL_GPL(percpu_down_write);
diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c
index ed75addd3ccd..4b2ce6bb94a4 100644
--- a/kernel/rcu/update.c
+++ b/kernel/rcu/update.c
@@ -53,6 +53,7 @@
#include <linux/rcupdate_wait.h>
#include <linux/sched/isolation.h>
#include <linux/kprobes.h>
+#include <linux/rcuwait.h>
#define CREATE_TRACE_POINTS
@@ -375,6 +376,13 @@ void __wait_rcu_gp(bool checktiny, int n, call_rcu_func_t *crcu_array,
}
EXPORT_SYMBOL_GPL(__wait_rcu_gp);
+void finish_rcuwait(struct rcuwait *w)
+{
+ rcu_assign_pointer(w->task, NULL);
+ __set_current_state(TASK_RUNNING);
+}
+EXPORT_SYMBOL_GPL(finish_rcuwait);
+
#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
void init_rcu_head(struct rcu_head *head)
{
|
