1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
|
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_PERCPU_COUNTER_H
#define _LINUX_PERCPU_COUNTER_H
/*
* A simple "approximate counter" for use in ext2 and ext3 superblocks.
*
* WARNING: these things are HUGE. 4 kbytes per counter on 32-way P4.
*/
#include <linux/spinlock.h>
#include <linux/smp.h>
#include <linux/list.h>
#include <linux/threads.h>
#include <linux/percpu.h>
#include <linux/types.h>
/* percpu_counter batch for local add or sub */
#define PERCPU_COUNTER_LOCAL_BATCH INT_MAX
#ifdef CONFIG_SMP
struct percpu_counter {
raw_spinlock_t lock;
s64 count;
#ifdef CONFIG_HOTPLUG_CPU
struct list_head list; /* All percpu_counters are on a list */
#endif
s32 __percpu *counters;
};
extern int percpu_counter_batch;
int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount,
gfp_t gfp, u32 nr_counters,
struct lock_class_key *key);
#define percpu_counter_init_many(fbc, value, gfp, nr_counters) \
({ \
static struct lock_class_key __key; \
\
__percpu_counter_init_many(fbc, value, gfp, nr_counters,\
&__key); \
})
#define percpu_counter_init(fbc, value, gfp) \
percpu_counter_init_many(fbc, value, gfp, 1)
void percpu_counter_destroy_many(struct percpu_counter *fbc, u32 nr_counters);
static inline void percpu_counter_destroy(struct percpu_counter *fbc)
{
percpu_counter_destroy_many(fbc, 1);
}
void percpu_counter_set(struct percpu_counter *fbc, s64 amount);
void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount,
s32 batch);
s64 __percpu_counter_sum(struct percpu_counter *fbc);
int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch);
void percpu_counter_sync(struct percpu_counter *fbc);
static inline int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs)
{
return __percpu_counter_compare(fbc, rhs, percpu_counter_batch);
}
static inline void percpu_counter_add(struct percpu_counter *fbc, s64 amount)
{
percpu_counter_add_batch(fbc, amount, percpu_counter_batch);
}
/*
* With percpu_counter_add_local() and percpu_counter_sub_local(), counts
* are accumulated in local per cpu counter and not in fbc->count until
* local count overflows PERCPU_COUNTER_LOCAL_BATCH. This makes counter
* write efficient.
* But percpu_counter_sum(), instead of percpu_counter_read(), needs to be
* used to add up the counts from each CPU to account for all the local
* counts. So percpu_counter_add_local() and percpu_counter_sub_local()
* should be used when a counter is updated frequently and read rarely.
*/
static inline void
percpu_counter_add_local(struct percpu_counter *fbc, s64 amount)
{
percpu_counter_add_batch(fbc, amount, PERCPU_COUNTER_LOCAL_BATCH);
}
static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc)
{
s64 ret = __percpu_counter_sum(fbc);
return ret < 0 ? 0 : ret;
}
static inline s64 percpu_counter_sum(struct percpu_counter *fbc)
{
return __percpu_counter_sum(fbc);
}
static inline s64 percpu_counter_read(struct percpu_counter *fbc)
{
return fbc->count;
}
/*
* It is possible for the percpu_counter_read() to return a small negative
* number for some counter which should never be negative.
*
*/
static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc)
{
/* Prevent reloads of fbc->count */
s64 ret = READ_ONCE(fbc->count);
if (ret >= 0)
return ret;
return 0;
}
static inline bool percpu_counter_initialized(struct percpu_counter *fbc)
{
return (fbc->counters != NULL);
}
#else /* !CONFIG_SMP */
struct percpu_counter {
s64 count;
};
static inline int percpu_counter_init_many(struct percpu_counter *fbc,
s64 amount, gfp_t gfp,
u32 nr_counters)
{
u32 i;
for (i = 0; i < nr_counters; i++)
fbc[i].count = amount;
return 0;
}
static inline int percpu_counter_init(struct percpu_counter *fbc, s64 amount,
gfp_t gfp)
{
return percpu_counter_init_many(fbc, amount, gfp, 1);
}
static inline void percpu_counter_destroy_many(struct percpu_counter *fbc,
u32 nr_counters)
{
}
static inline void percpu_counter_destroy(struct percpu_counter *fbc)
{
}
static inline void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
{
fbc->count = amount;
}
static inline int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs)
{
if (fbc->count > rhs)
return 1;
else if (fbc->count < rhs)
return -1;
else
return 0;
}
static inline int
__percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch)
{
return percpu_counter_compare(fbc, rhs);
}
static inline void
percpu_counter_add(struct percpu_counter *fbc, s64 amount)
{
unsigned long flags;
local_irq_save(flags);
fbc->count += amount;
local_irq_restore(flags);
}
/* non-SMP percpu_counter_add_local is the same with percpu_counter_add */
static inline void
percpu_counter_add_local(struct percpu_counter *fbc, s64 amount)
{
percpu_counter_add(fbc, amount);
}
static inline void
percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)
{
percpu_counter_add(fbc, amount);
}
static inline s64 percpu_counter_read(struct percpu_counter *fbc)
{
return fbc->count;
}
/*
* percpu_counter is intended to track positive numbers. In the UP case the
* number should never be negative.
*/
static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc)
{
return fbc->count;
}
static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc)
{
return percpu_counter_read_positive(fbc);
}
static inline s64 percpu_counter_sum(struct percpu_counter *fbc)
{
return percpu_counter_read(fbc);
}
static inline bool percpu_counter_initialized(struct percpu_counter *fbc)
{
return true;
}
static inline void percpu_counter_sync(struct percpu_counter *fbc)
{
}
#endif /* CONFIG_SMP */
static inline void percpu_counter_inc(struct percpu_counter *fbc)
{
percpu_counter_add(fbc, 1);
}
static inline void percpu_counter_dec(struct percpu_counter *fbc)
{
percpu_counter_add(fbc, -1);
}
static inline void percpu_counter_sub(struct percpu_counter *fbc, s64 amount)
{
percpu_counter_add(fbc, -amount);
}
static inline void
percpu_counter_sub_local(struct percpu_counter *fbc, s64 amount)
{
percpu_counter_add_local(fbc, -amount);
}
#endif /* _LINUX_PERCPU_COUNTER_H */
|
