diff options
Diffstat (limited to '')
| -rw-r--r-- | debian/patches-rt/0012-softirq-Use-a-dedicated-thread-for-timer-wakeups.patch | 234 |
1 files changed, 234 insertions, 0 deletions
diff --git a/debian/patches-rt/0012-softirq-Use-a-dedicated-thread-for-timer-wakeups.patch b/debian/patches-rt/0012-softirq-Use-a-dedicated-thread-for-timer-wakeups.patch new file mode 100644 index 000000000..720116187 --- /dev/null +++ b/debian/patches-rt/0012-softirq-Use-a-dedicated-thread-for-timer-wakeups.patch @@ -0,0 +1,234 @@ +From 454343a4f08e5de772024588aec2bd396177ee89 Mon Sep 17 00:00:00 2001 +From: Sebastian Andrzej Siewior <bigeasy@linutronix.de> +Date: Wed, 1 Dec 2021 17:41:09 +0100 +Subject: [PATCH 12/62] softirq: Use a dedicated thread for timer wakeups. +Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/6.1/older/patches-6.1.69-rt21.tar.xz + +A timer/hrtimer softirq is raised in-IRQ context. With threaded +interrupts enabled or on PREEMPT_RT this leads to waking the ksoftirqd +for the processing of the softirq. +Once the ksoftirqd is marked as pending (or is running) it will collect +all raised softirqs. This in turn means that a softirq which would have +been processed at the end of the threaded interrupt, which runs at an +elevated priority, is now moved to ksoftirqd which runs at SCHED_OTHER +priority and competes with every regular task for CPU resources. +This introduces long delays on heavy loaded systems and is not desired +especially if the system is not overloaded by the softirqs. + +Split the TIMER_SOFTIRQ and HRTIMER_SOFTIRQ processing into a dedicated +timers thread and let it run at the lowest SCHED_FIFO priority. +RT tasks are are woken up from hardirq context so only timer_list timers +and hrtimers for "regular" tasks are processed here. The higher priority +ensures that wakeups are performed before scheduling SCHED_OTHER tasks. + +Using a dedicated variable to store the pending softirq bits values +ensure that the timer are not accidentally picked up by ksoftirqd and +other threaded interrupts. +It shouldn't be picked up by ksoftirqd since it runs at lower priority. +However if the timer bits are ORed while a threaded interrupt is +running, then the timer softirq would be performed at higher priority. +The new timer thread will block on the softirq lock before it starts +softirq work. This "race window" isn't closed because while timer thread +is performing the softirq it can get PI-boosted via the softirq lock by +a random force-threaded thread. +The timer thread can pick up pending softirqs from ksoftirqd but only +if the softirq load is high. It is not be desired that the picked up +softirqs are processed at SCHED_FIFO priority under high softirq load +but this can already happen by a PI-boost by a force-threaded interrupt. + +Reported-by: kernel test robot <lkp@intel.com> [ static timer_threads ] +Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> +--- + include/linux/interrupt.h | 16 +++++++ + kernel/softirq.c | 92 +++++++++++++++++++++++++++++++++++++-- + kernel/time/hrtimer.c | 4 +- + kernel/time/timer.c | 2 +- + 4 files changed, 108 insertions(+), 6 deletions(-) + +diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h +index 4a1dc88ddbff..0efba74a835c 100644 +--- a/include/linux/interrupt.h ++++ b/include/linux/interrupt.h +@@ -609,6 +609,22 @@ extern void __raise_softirq_irqoff(unsigned int nr); + extern void raise_softirq_irqoff(unsigned int nr); + extern void raise_softirq(unsigned int nr); + ++#ifdef CONFIG_PREEMPT_RT ++extern void raise_timer_softirq(void); ++extern void raise_hrtimer_softirq(void); ++ ++#else ++static inline void raise_timer_softirq(void) ++{ ++ raise_softirq(TIMER_SOFTIRQ); ++} ++ ++static inline void raise_hrtimer_softirq(void) ++{ ++ raise_softirq_irqoff(HRTIMER_SOFTIRQ); ++} ++#endif ++ + DECLARE_PER_CPU(struct task_struct *, ksoftirqd); + + static inline struct task_struct *this_cpu_ksoftirqd(void) +diff --git a/kernel/softirq.c b/kernel/softirq.c +index c8a6913c067d..ed6d7c41aa17 100644 +--- a/kernel/softirq.c ++++ b/kernel/softirq.c +@@ -637,6 +637,29 @@ static inline void tick_irq_exit(void) + #endif + } + ++#ifdef CONFIG_PREEMPT_RT ++static DEFINE_PER_CPU(struct task_struct *, timersd); ++static DEFINE_PER_CPU(unsigned long, pending_timer_softirq); ++ ++static unsigned int local_pending_timers(void) ++{ ++ return __this_cpu_read(pending_timer_softirq); ++} ++ ++static void wake_timersd(void) ++{ ++ struct task_struct *tsk = __this_cpu_read(timersd); ++ ++ if (tsk) ++ wake_up_process(tsk); ++} ++ ++#else ++ ++static inline void wake_timersd(void) { } ++ ++#endif ++ + static inline void __irq_exit_rcu(void) + { + #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED +@@ -646,8 +669,13 @@ static inline void __irq_exit_rcu(void) + #endif + account_hardirq_exit(current); + preempt_count_sub(HARDIRQ_OFFSET); +- if (!in_interrupt() && local_softirq_pending()) +- invoke_softirq(); ++ if (!in_interrupt()) { ++ if (local_softirq_pending()) ++ invoke_softirq(); ++ ++ if (IS_ENABLED(CONFIG_PREEMPT_RT) && local_pending_timers()) ++ wake_timersd(); ++ } + + tick_irq_exit(); + } +@@ -976,12 +1004,70 @@ static struct smp_hotplug_thread softirq_threads = { + .thread_comm = "ksoftirqd/%u", + }; + ++#ifdef CONFIG_PREEMPT_RT ++static void timersd_setup(unsigned int cpu) ++{ ++ sched_set_fifo_low(current); ++} ++ ++static int timersd_should_run(unsigned int cpu) ++{ ++ return local_pending_timers(); ++} ++ ++static void run_timersd(unsigned int cpu) ++{ ++ unsigned int timer_si; ++ ++ ksoftirqd_run_begin(); ++ ++ timer_si = local_pending_timers(); ++ __this_cpu_write(pending_timer_softirq, 0); ++ or_softirq_pending(timer_si); ++ ++ __do_softirq(); ++ ++ ksoftirqd_run_end(); ++} ++ ++static void raise_ktimers_thread(unsigned int nr) ++{ ++ trace_softirq_raise(nr); ++ __this_cpu_or(pending_timer_softirq, 1 << nr); ++} ++ ++void raise_hrtimer_softirq(void) ++{ ++ raise_ktimers_thread(HRTIMER_SOFTIRQ); ++} ++ ++void raise_timer_softirq(void) ++{ ++ unsigned long flags; ++ ++ local_irq_save(flags); ++ raise_ktimers_thread(TIMER_SOFTIRQ); ++ wake_timersd(); ++ local_irq_restore(flags); ++} ++ ++static struct smp_hotplug_thread timer_threads = { ++ .store = &timersd, ++ .setup = timersd_setup, ++ .thread_should_run = timersd_should_run, ++ .thread_fn = run_timersd, ++ .thread_comm = "ktimers/%u", ++}; ++#endif ++ + static __init int spawn_ksoftirqd(void) + { + cpuhp_setup_state_nocalls(CPUHP_SOFTIRQ_DEAD, "softirq:dead", NULL, + takeover_tasklets); + BUG_ON(smpboot_register_percpu_thread(&softirq_threads)); +- ++#ifdef CONFIG_PREEMPT_RT ++ BUG_ON(smpboot_register_percpu_thread(&timer_threads)); ++#endif + return 0; + } + early_initcall(spawn_ksoftirqd); +diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c +index 5561dabc9b22..c5d480d5da15 100644 +--- a/kernel/time/hrtimer.c ++++ b/kernel/time/hrtimer.c +@@ -1805,7 +1805,7 @@ void hrtimer_interrupt(struct clock_event_device *dev) + if (!ktime_before(now, cpu_base->softirq_expires_next)) { + cpu_base->softirq_expires_next = KTIME_MAX; + cpu_base->softirq_activated = 1; +- raise_softirq_irqoff(HRTIMER_SOFTIRQ); ++ raise_hrtimer_softirq(); + } + + __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD); +@@ -1918,7 +1918,7 @@ void hrtimer_run_queues(void) + if (!ktime_before(now, cpu_base->softirq_expires_next)) { + cpu_base->softirq_expires_next = KTIME_MAX; + cpu_base->softirq_activated = 1; +- raise_softirq_irqoff(HRTIMER_SOFTIRQ); ++ raise_hrtimer_softirq(); + } + + __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD); +diff --git a/kernel/time/timer.c b/kernel/time/timer.c +index 717fcb9fb14a..e6219da89933 100644 +--- a/kernel/time/timer.c ++++ b/kernel/time/timer.c +@@ -1822,7 +1822,7 @@ static void run_local_timers(void) + if (time_before(jiffies, base->next_expiry)) + return; + } +- raise_softirq(TIMER_SOFTIRQ); ++ raise_timer_softirq(); + } + + /* +-- +2.43.0 + |