From: Thomas Gleixner Date: Tue, 26 Sep 2023 13:03:52 +0000 Subject: [PATCH 33/46] printk: nbcon: Add printer thread wakeups Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/6.9/older/patches-6.9-rt5.tar.xz Add a function to wakeup the printer threads. Use the new function when: - records are added to the printk ringbuffer - consoles are resumed - triggered via printk_trigger_flush() The actual waking is performed via irq_work so that the wakeup can be triggered from any context. Co-developed-by: John Ogness Signed-off-by: John Ogness Signed-off-by: Thomas Gleixner (Intel) Signed-off-by: Sebastian Andrzej Siewior --- include/linux/console.h | 3 ++ kernel/printk/internal.h | 1 kernel/printk/nbcon.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++ kernel/printk/printk.c | 7 +++++ 4 files changed, 67 insertions(+) --- a/include/linux/console.h +++ b/include/linux/console.h @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -327,6 +328,7 @@ struct nbcon_write_context { * @pbufs: Pointer to nbcon private buffer * @kthread: Printer kthread for this console * @rcuwait: RCU-safe wait object for @kthread waking + * @irq_work: Defer @kthread waking to IRQ work context */ struct console { char name[16]; @@ -446,6 +448,7 @@ struct console { struct printk_buffers *pbufs; struct task_struct *kthread; struct rcuwait rcuwait; + struct irq_work irq_work; }; #ifdef CONFIG_LOCKDEP --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -91,6 +91,7 @@ void nbcon_atomic_flush_pending(void); bool nbcon_legacy_emit_next_record(struct console *con, bool *handover, int cookie); void nbcon_kthread_create(struct console *con); +void nbcon_wake_threads(void); /* * Check if the given console is currently capable and allowed to print --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -1056,6 +1056,61 @@ static int nbcon_kthread_func(void *__co goto wait_for_event; } +/** + * nbcon_irq_work - irq work to wake printk thread + * @irq_work: The irq work to operate on + */ +static void nbcon_irq_work(struct irq_work *irq_work) +{ + struct console *con = container_of(irq_work, struct console, irq_work); + + nbcon_kthread_wake(con); +} + +static inline bool rcuwait_has_sleeper(struct rcuwait *w) +{ + bool has_sleeper; + + rcu_read_lock(); + /* + * Guarantee any new records can be seen by tasks preparing to wait + * before this context checks if the rcuwait is empty. + * + * This full memory barrier pairs with the full memory barrier within + * set_current_state() of ___rcuwait_wait_event(), which is called + * after prepare_to_rcuwait() adds the waiter but before it has + * checked the wait condition. + * + * This pairs with nbcon_kthread_func:A. + */ + smp_mb(); /* LMM(rcuwait_has_sleeper:A) */ + has_sleeper = !!rcu_dereference(w->task); + rcu_read_unlock(); + + return has_sleeper; +} + +/** + * nbcon_wake_threads - Wake up printing threads using irq_work + */ +void nbcon_wake_threads(void) +{ + struct console *con; + int cookie; + + cookie = console_srcu_read_lock(); + for_each_console_srcu(con) { + /* + * Only schedule irq_work if the printing thread is + * actively waiting. If not waiting, the thread will + * notice by itself that it has work to do. + */ + if (con->kthread && rcuwait_has_sleeper(&con->rcuwait)) + irq_work_queue(&con->irq_work); + } + console_srcu_read_unlock(cookie); +} + /* Track the nbcon emergency nesting per CPU. */ static DEFINE_PER_CPU(unsigned int, nbcon_pcpu_emergency_nesting); static unsigned int early_nbcon_pcpu_emergency_nesting __initdata; @@ -1538,6 +1593,7 @@ void nbcon_init(struct console *con, u64 BUG_ON(!con->pbufs); rcuwait_init(&con->rcuwait); + init_irq_work(&con->irq_work, nbcon_irq_work); nbcon_seq_force(con, init_seq); nbcon_state_set(con, &state); } --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -2402,6 +2402,8 @@ asmlinkage int vprintk_emit(int facility } } + nbcon_wake_threads(); + if (do_trylock_unlock) { /* * The caller may be holding system-critical or @@ -2708,6 +2710,10 @@ void resume_console(void) */ synchronize_srcu(&console_srcu); + /* + * Since this runs in task context, wake the threaded printers + * directly rather than scheduling irq_work to do it. + */ cookie = console_srcu_read_lock(); for_each_console_srcu(con) { flags = console_srcu_read_flags(con); @@ -4161,6 +4167,7 @@ void defer_console_output(void) void printk_trigger_flush(void) { + nbcon_wake_threads(); defer_console_output(); }