diff options
Diffstat (limited to 'debian/patches-rt/0239-irqwork-push-most-work-into-softirq-context.patch')
| -rw-r--r-- | debian/patches-rt/0239-irqwork-push-most-work-into-softirq-context.patch | 189 |
1 files changed, 189 insertions, 0 deletions
diff --git a/debian/patches-rt/0239-irqwork-push-most-work-into-softirq-context.patch b/debian/patches-rt/0239-irqwork-push-most-work-into-softirq-context.patch new file mode 100644 index 000000000..1ac41992f --- /dev/null +++ b/debian/patches-rt/0239-irqwork-push-most-work-into-softirq-context.patch @@ -0,0 +1,189 @@ +From 57453ffdf0bcba3fa27ca0b19f622ea371c2c686 Mon Sep 17 00:00:00 2001 +From: Sebastian Andrzej Siewior <bigeasy@linutronix.de> +Date: Tue, 23 Jun 2015 15:32:51 +0200 +Subject: [PATCH 239/323] irqwork: push most work into softirq context +Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/5.10/older/patches-5.10.204-rt100.tar.xz + +Initially we defered all irqwork into softirq because we didn't want the +latency spikes if perf or another user was busy and delayed the RT task. +The NOHZ trigger (nohz_full_kick_work) was the first user that did not work +as expected if it did not run in the original irqwork context so we had to +bring it back somehow for it. push_irq_work_func is the second one that +requires this. + +This patch adds the IRQ_WORK_HARD_IRQ which makes sure the callback runs +in raw-irq context. Everything else is defered into softirq context. Without +-RT we have the orignal behavior. + +This patch incorporates tglx orignal work which revoked a little bringing back +the arch_irq_work_raise() if possible and a few fixes from Steven Rostedt and +Mike Galbraith, + +[bigeasy: melt tglx's irq_work_tick_soft() which splits irq_work_tick() into a + hard and soft variant] +Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> +--- + include/linux/irq_work.h | 6 +++++ + kernel/irq_work.c | 58 +++++++++++++++++++++++++++++++--------- + kernel/sched/topology.c | 1 + + kernel/time/timer.c | 2 ++ + 4 files changed, 55 insertions(+), 12 deletions(-) + +diff --git a/include/linux/irq_work.h b/include/linux/irq_work.h +index 30823780c192..f941f2d7d71c 100644 +--- a/include/linux/irq_work.h ++++ b/include/linux/irq_work.h +@@ -55,4 +55,10 @@ static inline void irq_work_run(void) { } + static inline void irq_work_single(void *arg) { } + #endif + ++#if defined(CONFIG_IRQ_WORK) && defined(CONFIG_PREEMPT_RT) ++void irq_work_tick_soft(void); ++#else ++static inline void irq_work_tick_soft(void) { } ++#endif ++ + #endif /* _LINUX_IRQ_WORK_H */ +diff --git a/kernel/irq_work.c b/kernel/irq_work.c +index eca83965b631..8183d30e1bb1 100644 +--- a/kernel/irq_work.c ++++ b/kernel/irq_work.c +@@ -18,6 +18,7 @@ + #include <linux/cpu.h> + #include <linux/notifier.h> + #include <linux/smp.h> ++#include <linux/interrupt.h> + #include <asm/processor.h> + + +@@ -52,13 +53,19 @@ void __weak arch_irq_work_raise(void) + /* Enqueue on current CPU, work must already be claimed and preempt disabled */ + static void __irq_work_queue_local(struct irq_work *work) + { ++ struct llist_head *list; ++ bool lazy_work, realtime = IS_ENABLED(CONFIG_PREEMPT_RT); ++ ++ lazy_work = atomic_read(&work->flags) & IRQ_WORK_LAZY; ++ + /* If the work is "lazy", handle it from next tick if any */ +- if (atomic_read(&work->flags) & IRQ_WORK_LAZY) { +- if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)) && +- tick_nohz_tick_stopped()) +- arch_irq_work_raise(); +- } else { +- if (llist_add(&work->llnode, this_cpu_ptr(&raised_list))) ++ if (lazy_work || (realtime && !(atomic_read(&work->flags) & IRQ_WORK_HARD_IRQ))) ++ list = this_cpu_ptr(&lazy_list); ++ else ++ list = this_cpu_ptr(&raised_list); ++ ++ if (llist_add(&work->llnode, list)) { ++ if (!lazy_work || tick_nohz_tick_stopped()) + arch_irq_work_raise(); + } + } +@@ -102,7 +109,13 @@ bool irq_work_queue_on(struct irq_work *work, int cpu) + if (cpu != smp_processor_id()) { + /* Arch remote IPI send/receive backend aren't NMI safe */ + WARN_ON_ONCE(in_nmi()); +- __smp_call_single_queue(cpu, &work->llnode); ++ ++ if (IS_ENABLED(CONFIG_PREEMPT_RT) && !(atomic_read(&work->flags) & IRQ_WORK_HARD_IRQ)) { ++ if (llist_add(&work->llnode, &per_cpu(lazy_list, cpu))) ++ arch_send_call_function_single_ipi(cpu); ++ } else { ++ __smp_call_single_queue(cpu, &work->llnode); ++ } + } else { + __irq_work_queue_local(work); + } +@@ -120,9 +133,8 @@ bool irq_work_needs_cpu(void) + raised = this_cpu_ptr(&raised_list); + lazy = this_cpu_ptr(&lazy_list); + +- if (llist_empty(raised) || arch_irq_work_has_interrupt()) +- if (llist_empty(lazy)) +- return false; ++ if (llist_empty(raised) && llist_empty(lazy)) ++ return false; + + /* All work should have been flushed before going offline */ + WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); +@@ -160,8 +172,12 @@ static void irq_work_run_list(struct llist_head *list) + struct irq_work *work, *tmp; + struct llist_node *llnode; + ++#ifndef CONFIG_PREEMPT_RT ++ /* ++ * nort: On RT IRQ-work may run in SOFTIRQ context. ++ */ + BUG_ON(!irqs_disabled()); +- ++#endif + if (llist_empty(list)) + return; + +@@ -177,7 +193,16 @@ static void irq_work_run_list(struct llist_head *list) + void irq_work_run(void) + { + irq_work_run_list(this_cpu_ptr(&raised_list)); +- irq_work_run_list(this_cpu_ptr(&lazy_list)); ++ if (IS_ENABLED(CONFIG_PREEMPT_RT)) { ++ /* ++ * NOTE: we raise softirq via IPI for safety, ++ * and execute in irq_work_tick() to move the ++ * overhead from hard to soft irq context. ++ */ ++ if (!llist_empty(this_cpu_ptr(&lazy_list))) ++ raise_softirq(TIMER_SOFTIRQ); ++ } else ++ irq_work_run_list(this_cpu_ptr(&lazy_list)); + } + EXPORT_SYMBOL_GPL(irq_work_run); + +@@ -187,8 +212,17 @@ void irq_work_tick(void) + + if (!llist_empty(raised) && !arch_irq_work_has_interrupt()) + irq_work_run_list(raised); ++ ++ if (!IS_ENABLED(CONFIG_PREEMPT_RT)) ++ irq_work_run_list(this_cpu_ptr(&lazy_list)); ++} ++ ++#if defined(CONFIG_IRQ_WORK) && defined(CONFIG_PREEMPT_RT) ++void irq_work_tick_soft(void) ++{ + irq_work_run_list(this_cpu_ptr(&lazy_list)); + } ++#endif + + /* + * Synchronize against the irq_work @entry, ensures the entry is not +diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c +index ff2c6d3ba6c7..2f1bea4f7dba 100644 +--- a/kernel/sched/topology.c ++++ b/kernel/sched/topology.c +@@ -514,6 +514,7 @@ static int init_rootdomain(struct root_domain *rd) + rd->rto_cpu = -1; + raw_spin_lock_init(&rd->rto_lock); + init_irq_work(&rd->rto_push_work, rto_push_irq_work_func); ++ atomic_or(IRQ_WORK_HARD_IRQ, &rd->rto_push_work.flags); + #endif + + init_dl_bw(&rd->dl_bw); +diff --git a/kernel/time/timer.c b/kernel/time/timer.c +index a4fdc7cfb723..1cad0efd635c 100644 +--- a/kernel/time/timer.c ++++ b/kernel/time/timer.c +@@ -1770,6 +1770,8 @@ static __latent_entropy void run_timer_softirq(struct softirq_action *h) + { + struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]); + ++ irq_work_tick_soft(); ++ + __run_timers(base); + if (IS_ENABLED(CONFIG_NO_HZ_COMMON)) + __run_timers(this_cpu_ptr(&timer_bases[BASE_DEF])); +-- +2.43.0 + |