diff options
Diffstat (limited to 'debian/patches-rt/0088-printk-ringbuffer-Do-not-skip-non-finalized-records-.patch')
| -rw-r--r-- | debian/patches-rt/0088-printk-ringbuffer-Do-not-skip-non-finalized-records-.patch | 304 |
1 files changed, 304 insertions, 0 deletions
diff --git a/debian/patches-rt/0088-printk-ringbuffer-Do-not-skip-non-finalized-records-.patch b/debian/patches-rt/0088-printk-ringbuffer-Do-not-skip-non-finalized-records-.patch new file mode 100644 index 000000000..917f777f1 --- /dev/null +++ b/debian/patches-rt/0088-printk-ringbuffer-Do-not-skip-non-finalized-records-.patch @@ -0,0 +1,304 @@ +From: John Ogness <john.ogness@linutronix.de> +Date: Thu, 19 Oct 2023 10:32:05 +0000 +Subject: [PATCH 088/134] printk: ringbuffer: Do not skip non-finalized records + with prb_next_seq() +Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/6.6/older/patches-6.6.7-rt18.tar.xz + +Commit f244b4dc53e5 ("printk: ringbuffer: Improve +prb_next_seq() performance") introduced an optimization for +prb_next_seq() by using best-effort to track recently finalized +records. However, the order of finalization does not +necessarily match the order of the records. The optimization +changed prb_next_seq() to return inconsistent results, possibly +yielding sequence numbers that are not available to readers +because they are preceded by non-finalized records or they are +not yet visible to the reader CPU. + +Rather than simply best-effort tracking recently finalized +records, force the committing writer to read records and +increment the last "contiguous block" of finalized records. In +order to do this, the sequence number instead of ID must be +stored because ID's cannot be directly compared. + +A new memory barrier pair is introduced to guarantee that a +reader can always read the records up until the sequence number +returned by prb_next_seq() (unless the records have since +been overwritten in the ringbuffer). + +This restores the original functionality of prb_next_seq() +while also keeping the optimization. + +For 32bit systems, only the lower 32 bits of the sequence +number are stored. When reading the value, it is expanded to +the full 64bit sequence number using the 32bit seq macros, +which fold in the value returned by prb_first_seq(). + +Fixes: f244b4dc53e5 ("printk: ringbuffer: Improve prb_next_seq() performance") +Signed-off-by: John Ogness <john.ogness@linutronix.de> +Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> +--- + kernel/printk/printk_ringbuffer.c | 164 ++++++++++++++++++++++++++++---------- + kernel/printk/printk_ringbuffer.h | 4 + 2 files changed, 127 insertions(+), 41 deletions(-) + +--- a/kernel/printk/printk_ringbuffer.c ++++ b/kernel/printk/printk_ringbuffer.c +@@ -6,6 +6,7 @@ + #include <linux/errno.h> + #include <linux/bug.h> + #include "printk_ringbuffer.h" ++#include "internal.h" + + /** + * DOC: printk_ringbuffer overview +@@ -303,6 +304,9 @@ + * + * desc_push_tail:B / desc_reserve:D + * set descriptor reusable (state), then push descriptor tail (id) ++ * ++ * desc_update_last_finalized:A / desc_last_finalized_seq:A ++ * store finalized record, then set new highest finalized sequence number + */ + + #define DATA_SIZE(data_ring) _DATA_SIZE((data_ring)->size_bits) +@@ -1442,19 +1446,117 @@ bool prb_reserve_in_last(struct prb_rese + } + + /* ++ * @last_finalized_seq value guarantees that all records up to and including ++ * this sequence number are finalized and can be read. The only exception are ++ * too old records which have already been overwritten. ++ * ++ * It is also guaranteed that @last_finalized_seq only increases. ++ * ++ * Be aware that finalized records following non-finalized records are not ++ * reported because they are not yet available to the reader. For example, ++ * a new record stored via printk() will not be available to a printer if ++ * it follows a record that has not been finalized yet. However, once that ++ * non-finalized record becomes finalized, @last_finalized_seq will be ++ * appropriately updated and the full set of finalized records will be ++ * available to the printer. And since each printk() caller will either ++ * directly print or trigger deferred printing of all available unprinted ++ * records, all printk() messages will get printed. ++ */ ++static u64 desc_last_finalized_seq(struct printk_ringbuffer *rb) ++{ ++ struct prb_desc_ring *desc_ring = &rb->desc_ring; ++ unsigned long ulseq; ++ ++ /* ++ * Guarantee the sequence number is loaded before loading the ++ * associated record in order to guarantee that the record can be ++ * seen by this CPU. This pairs with desc_update_last_finalized:A. ++ */ ++ ulseq = atomic_long_read_acquire(&desc_ring->last_finalized_seq ++ ); /* LMM(desc_last_finalized_seq:A) */ ++ ++ return __ulseq_to_u64seq(rb, ulseq); ++} ++ ++static bool _prb_read_valid(struct printk_ringbuffer *rb, u64 *seq, ++ struct printk_record *r, unsigned int *line_count); ++ ++/* ++ * Check if there are records directly following @last_finalized_seq that are ++ * finalized. If so, update @last_finalized_seq to the latest of these ++ * records. It is not allowed to skip over records that are not yet finalized. ++ */ ++static void desc_update_last_finalized(struct printk_ringbuffer *rb) ++{ ++ struct prb_desc_ring *desc_ring = &rb->desc_ring; ++ u64 old_seq = desc_last_finalized_seq(rb); ++ unsigned long oldval; ++ unsigned long newval; ++ u64 finalized_seq; ++ u64 try_seq; ++ ++try_again: ++ finalized_seq = old_seq; ++ try_seq = finalized_seq + 1; ++ ++ /* Try to find later finalized records. */ ++ while (_prb_read_valid(rb, &try_seq, NULL, NULL)) { ++ finalized_seq = try_seq; ++ try_seq++; ++ } ++ ++ /* No update needed if no later finalized record was found. */ ++ if (finalized_seq == old_seq) ++ return; ++ ++ oldval = __u64seq_to_ulseq(old_seq); ++ newval = __u64seq_to_ulseq(finalized_seq); ++ ++ /* ++ * Set the sequence number of a later finalized record that has been ++ * seen. ++ * ++ * Guarantee the record data is visible to other CPUs before storing ++ * its sequence number. This pairs with desc_last_finalized_seq:A. ++ * ++ * Memory barrier involvement: ++ * ++ * If desc_last_finalized_seq:A reads from ++ * desc_update_last_finalized:A, then desc_read:A reads from ++ * _prb_commit:B. ++ * ++ * Relies on: ++ * ++ * RELEASE from _prb_commit:B to desc_update_last_finalized:A ++ * matching ++ * ACQUIRE from desc_last_finalized_seq:A to desc_read:A ++ * ++ * Note: _prb_commit:B and desc_update_last_finalized:A can be ++ * different CPUs. However, the desc_update_last_finalized:A ++ * CPU (which performs the release) must have previously seen ++ * _prb_commit:B. ++ */ ++ if (!atomic_long_try_cmpxchg_release(&desc_ring->last_finalized_seq, ++ &oldval, newval)) { /* LMM(desc_update_last_finalized:A) */ ++ old_seq = __ulseq_to_u64seq(rb, oldval); ++ goto try_again; ++ } ++} ++ ++/* + * Attempt to finalize a specified descriptor. If this fails, the descriptor + * is either already final or it will finalize itself when the writer commits. + */ +-static void desc_make_final(struct prb_desc_ring *desc_ring, unsigned long id) ++static void desc_make_final(struct printk_ringbuffer *rb, unsigned long id) + { ++ struct prb_desc_ring *desc_ring = &rb->desc_ring; + unsigned long prev_state_val = DESC_SV(id, desc_committed); + struct prb_desc *d = to_desc(desc_ring, id); + +- atomic_long_cmpxchg_relaxed(&d->state_var, prev_state_val, +- DESC_SV(id, desc_finalized)); /* LMM(desc_make_final:A) */ +- +- /* Best effort to remember the last finalized @id. */ +- atomic_long_set(&desc_ring->last_finalized_id, id); ++ if (atomic_long_try_cmpxchg_relaxed(&d->state_var, &prev_state_val, ++ DESC_SV(id, desc_finalized))) { /* LMM(desc_make_final:A) */ ++ desc_update_last_finalized(rb); ++ } + } + + /** +@@ -1550,7 +1652,7 @@ bool prb_reserve(struct prb_reserved_ent + * readers. (For seq==0 there is no previous descriptor.) + */ + if (info->seq > 0) +- desc_make_final(desc_ring, DESC_ID(id - 1)); ++ desc_make_final(rb, DESC_ID(id - 1)); + + r->text_buf = data_alloc(rb, r->text_buf_size, &d->text_blk_lpos, id); + /* If text data allocation fails, a data-less record is committed. */ +@@ -1643,7 +1745,7 @@ void prb_commit(struct prb_reserved_entr + */ + head_id = atomic_long_read(&desc_ring->head_id); /* LMM(prb_commit:A) */ + if (head_id != e->id) +- desc_make_final(desc_ring, e->id); ++ desc_make_final(e->rb, e->id); + } + + /** +@@ -1663,12 +1765,9 @@ void prb_commit(struct prb_reserved_entr + */ + void prb_final_commit(struct prb_reserved_entry *e) + { +- struct prb_desc_ring *desc_ring = &e->rb->desc_ring; +- + _prb_commit(e, desc_finalized); + +- /* Best effort to remember the last finalized @id. */ +- atomic_long_set(&desc_ring->last_finalized_id, e->id); ++ desc_update_last_finalized(e->rb); + } + + /* +@@ -2008,7 +2107,9 @@ u64 prb_first_valid_seq(struct printk_ri + * newest sequence number available to readers will be. + * + * This provides readers a sequence number to jump to if all currently +- * available records should be skipped. ++ * available records should be skipped. It is guaranteed that all records ++ * previous to the returned value have been finalized and are (or were) ++ * available to the reader. + * + * Context: Any context. + * Return: The sequence number of the next newest (not yet available) record +@@ -2016,34 +2117,19 @@ u64 prb_first_valid_seq(struct printk_ri + */ + u64 prb_next_seq(struct printk_ringbuffer *rb) + { +- struct prb_desc_ring *desc_ring = &rb->desc_ring; +- enum desc_state d_state; +- unsigned long id; + u64 seq; + +- /* Check if the cached @id still points to a valid @seq. */ +- id = atomic_long_read(&desc_ring->last_finalized_id); +- d_state = desc_read(desc_ring, id, NULL, &seq, NULL); ++ seq = desc_last_finalized_seq(rb); + +- if (d_state == desc_finalized || d_state == desc_reusable) { +- /* +- * Begin searching after the last finalized record. +- * +- * On 0, the search must begin at 0 because of hack#2 +- * of the bootstrapping phase it is not known if a +- * record at index 0 exists. +- */ +- if (seq != 0) +- seq++; +- } else { +- /* +- * The information about the last finalized sequence number +- * has gone. It should happen only when there is a flood of +- * new messages and the ringbuffer is rapidly recycled. +- * Give up and start from the beginning. +- */ +- seq = 0; +- } ++ /* ++ * Begin searching after the last finalized record. ++ * ++ * On 0, the search must begin at 0 because of hack#2 ++ * of the bootstrapping phase it is not known if a ++ * record at index 0 exists. ++ */ ++ if (seq != 0) ++ seq++; + + /* + * The information about the last finalized @seq might be inaccurate. +@@ -2085,7 +2171,7 @@ void prb_init(struct printk_ringbuffer * + rb->desc_ring.infos = infos; + atomic_long_set(&rb->desc_ring.head_id, DESC0_ID(descbits)); + atomic_long_set(&rb->desc_ring.tail_id, DESC0_ID(descbits)); +- atomic_long_set(&rb->desc_ring.last_finalized_id, DESC0_ID(descbits)); ++ atomic_long_set(&rb->desc_ring.last_finalized_seq, 0); + + rb->text_data_ring.size_bits = textbits; + rb->text_data_ring.data = text_buf; +--- a/kernel/printk/printk_ringbuffer.h ++++ b/kernel/printk/printk_ringbuffer.h +@@ -75,7 +75,7 @@ struct prb_desc_ring { + struct printk_info *infos; + atomic_long_t head_id; + atomic_long_t tail_id; +- atomic_long_t last_finalized_id; ++ atomic_long_t last_finalized_seq; + }; + + /* +@@ -259,7 +259,7 @@ static struct printk_ringbuffer name = { + .infos = &_##name##_infos[0], \ + .head_id = ATOMIC_INIT(DESC0_ID(descbits)), \ + .tail_id = ATOMIC_INIT(DESC0_ID(descbits)), \ +- .last_finalized_id = ATOMIC_INIT(DESC0_ID(descbits)), \ ++ .last_finalized_seq = ATOMIC_INIT(0), \ + }, \ + .text_data_ring = { \ + .size_bits = (avgtextbits) + (descbits), \ |