1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
|
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright 2023 Red Hat
*/
#ifndef VDO_FUNNEL_QUEUE_H
#define VDO_FUNNEL_QUEUE_H
#include <linux/atomic.h>
#include <linux/cache.h>
/*
* A funnel queue is a simple (almost) lock-free queue that accepts entries from multiple threads
* (multi-producer) and delivers them to a single thread (single-consumer). "Funnel" is an attempt
* to evoke the image of requests from more than one producer being "funneled down" to a single
* consumer.
*
* This is an unsynchronized but thread-safe data structure when used as intended. There is no
* mechanism to ensure that only one thread is consuming from the queue. If more than one thread
* attempts to consume from the queue, the resulting behavior is undefined. Clients must not
* directly access or manipulate the internals of the queue, which are only exposed for the purpose
* of allowing the very simple enqueue operation to be inlined.
*
* The implementation requires that a funnel_queue_entry structure (a link pointer) is embedded in
* the queue entries, and pointers to those structures are used exclusively by the queue. No macros
* are defined to template the queue, so the offset of the funnel_queue_entry in the records placed
* in the queue must all be the same so the client can derive their structure pointer from the
* entry pointer returned by vdo_funnel_queue_poll().
*
* Callers are wholly responsible for allocating and freeing the entries. Entries may be freed as
* soon as they are returned since this queue is not susceptible to the "ABA problem" present in
* many lock-free data structures. The queue is dynamically allocated to ensure cache-line
* alignment, but no other dynamic allocation is used.
*
* The algorithm is not actually 100% lock-free. There is a single point in vdo_funnel_queue_put()
* at which a preempted producer will prevent the consumers from seeing items added to the queue by
* later producers, and only if the queue is short enough or the consumer fast enough for it to
* reach what was the end of the queue at the time of the preemption.
*
* The consumer function, vdo_funnel_queue_poll(), will return NULL when the queue is empty. To
* wait for data to consume, spin (if safe) or combine the queue with a struct event_count to
* signal the presence of new entries.
*/
/* This queue link structure must be embedded in client entries. */
struct funnel_queue_entry {
/* The next (newer) entry in the queue. */
struct funnel_queue_entry *next;
};
/*
* The dynamically allocated queue structure, which is allocated on a cache line boundary so the
* producer and consumer fields in the structure will land on separate cache lines. This should be
* consider opaque but it is exposed here so vdo_funnel_queue_put() can be inlined.
*/
struct __aligned(L1_CACHE_BYTES) funnel_queue {
/*
* The producers' end of the queue, an atomically exchanged pointer that will never be
* NULL.
*/
struct funnel_queue_entry *newest;
/* The consumer's end of the queue, which is owned by the consumer and never NULL. */
struct funnel_queue_entry *oldest __aligned(L1_CACHE_BYTES);
/* A dummy entry used to provide the non-NULL invariants above. */
struct funnel_queue_entry stub;
};
int __must_check vdo_make_funnel_queue(struct funnel_queue **queue_ptr);
void vdo_free_funnel_queue(struct funnel_queue *queue);
/*
* Put an entry on the end of the queue.
*
* The entry pointer must be to the struct funnel_queue_entry embedded in the caller's data
* structure. The caller must be able to derive the address of the start of their data structure
* from the pointer that passed in here, so every entry in the queue must have the struct
* funnel_queue_entry at the same offset within the client's structure.
*/
static inline void vdo_funnel_queue_put(struct funnel_queue *queue,
struct funnel_queue_entry *entry)
{
struct funnel_queue_entry *previous;
/*
* Barrier requirements: All stores relating to the entry ("next" pointer, containing data
* structure fields) must happen before the previous->next store making it visible to the
* consumer. Also, the entry's "next" field initialization to NULL must happen before any
* other producer threads can see the entry (the xchg) and try to update the "next" field.
*
* xchg implements a full barrier.
*/
WRITE_ONCE(entry->next, NULL);
previous = xchg(&queue->newest, entry);
/*
* Preemptions between these two statements hide the rest of the queue from the consumer,
* preventing consumption until the following assignment runs.
*/
WRITE_ONCE(previous->next, entry);
}
struct funnel_queue_entry *__must_check vdo_funnel_queue_poll(struct funnel_queue *queue);
bool __must_check vdo_is_funnel_queue_empty(struct funnel_queue *queue);
bool __must_check vdo_is_funnel_queue_idle(struct funnel_queue *queue);
#endif /* VDO_FUNNEL_QUEUE_H */
|
