diff options
Diffstat (limited to 'debian/patches-rt/0065-blk-mq-Use-llist_head-for-blk_cpu_done.patch')
| -rw-r--r-- | debian/patches-rt/0065-blk-mq-Use-llist_head-for-blk_cpu_done.patch | 201 |
1 files changed, 201 insertions, 0 deletions
diff --git a/debian/patches-rt/0065-blk-mq-Use-llist_head-for-blk_cpu_done.patch b/debian/patches-rt/0065-blk-mq-Use-llist_head-for-blk_cpu_done.patch new file mode 100644 index 000000000..b7faea536 --- /dev/null +++ b/debian/patches-rt/0065-blk-mq-Use-llist_head-for-blk_cpu_done.patch @@ -0,0 +1,201 @@ +From a958b1b96b5419f3d9372de0c7851daae941bf47 Mon Sep 17 00:00:00 2001 +From: Sebastian Andrzej Siewior <bigeasy@linutronix.de> +Date: Sat, 23 Jan 2021 21:10:27 +0100 +Subject: [PATCH 065/323] blk-mq: Use llist_head for blk_cpu_done +Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/5.10/older/patches-5.10.204-rt100.tar.xz + +With llist_head it is possible to avoid the locking (the irq-off region) +when items are added. This makes it possible to add items on a remote +CPU without additional locking. +llist_add() returns true if the list was previously empty. This can be +used to invoke the SMP function call / raise sofirq only if the first +item was added (otherwise it is already pending). +This simplifies the code a little and reduces the IRQ-off regions. + +blk_mq_raise_softirq() needs a preempt-disable section to ensure the +request is enqueued on the same CPU as the softirq is raised. +Some callers (USB-storage) invoke this path in preemptible context. + +Reviewed-by: Christoph Hellwig <hch@lst.de> +Reviewed-by: Daniel Wagner <dwagner@suse.de> +Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> +Signed-off-by: Jens Axboe <axboe@kernel.dk> +Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> +--- + block/blk-mq.c | 101 ++++++++++++++++++----------------------- + include/linux/blkdev.h | 2 +- + 2 files changed, 44 insertions(+), 59 deletions(-) + +diff --git a/block/blk-mq.c b/block/blk-mq.c +index 87b8377a74fc..a6618bf45992 100644 +--- a/block/blk-mq.c ++++ b/block/blk-mq.c +@@ -41,7 +41,7 @@ + #include "blk-mq-sched.h" + #include "blk-rq-qos.h" + +-static DEFINE_PER_CPU(struct list_head, blk_cpu_done); ++static DEFINE_PER_CPU(struct llist_head, blk_cpu_done); + + static void blk_mq_poll_stats_start(struct request_queue *q); + static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb); +@@ -569,68 +569,29 @@ void blk_mq_end_request(struct request *rq, blk_status_t error) + } + EXPORT_SYMBOL(blk_mq_end_request); + +-/* +- * Softirq action handler - move entries to local list and loop over them +- * while passing them to the queue registered handler. +- */ +-static __latent_entropy void blk_done_softirq(struct softirq_action *h) ++static void blk_complete_reqs(struct llist_head *list) + { +- struct list_head *cpu_list, local_list; +- +- local_irq_disable(); +- cpu_list = this_cpu_ptr(&blk_cpu_done); +- list_replace_init(cpu_list, &local_list); +- local_irq_enable(); +- +- while (!list_empty(&local_list)) { +- struct request *rq; ++ struct llist_node *entry = llist_reverse_order(llist_del_all(list)); ++ struct request *rq, *next; + +- rq = list_entry(local_list.next, struct request, ipi_list); +- list_del_init(&rq->ipi_list); ++ llist_for_each_entry_safe(rq, next, entry, ipi_list) + rq->q->mq_ops->complete(rq); +- } + } + +-static void blk_mq_trigger_softirq(struct request *rq) ++static __latent_entropy void blk_done_softirq(struct softirq_action *h) + { +- struct list_head *list; +- unsigned long flags; +- +- local_irq_save(flags); +- list = this_cpu_ptr(&blk_cpu_done); +- list_add_tail(&rq->ipi_list, list); +- +- /* +- * If the list only contains our just added request, signal a raise of +- * the softirq. If there are already entries there, someone already +- * raised the irq but it hasn't run yet. +- */ +- if (list->next == &rq->ipi_list) +- raise_softirq_irqoff(BLOCK_SOFTIRQ); +- local_irq_restore(flags); ++ blk_complete_reqs(this_cpu_ptr(&blk_cpu_done)); + } + + static int blk_softirq_cpu_dead(unsigned int cpu) + { +- /* +- * If a CPU goes away, splice its entries to the current CPU +- * and trigger a run of the softirq +- */ +- local_irq_disable(); +- list_splice_init(&per_cpu(blk_cpu_done, cpu), +- this_cpu_ptr(&blk_cpu_done)); +- raise_softirq_irqoff(BLOCK_SOFTIRQ); +- local_irq_enable(); +- ++ blk_complete_reqs(&per_cpu(blk_cpu_done, cpu)); + return 0; + } + +- + static void __blk_mq_complete_request_remote(void *data) + { +- struct request *rq = data; +- +- blk_mq_trigger_softirq(rq); ++ __raise_softirq_irqoff(BLOCK_SOFTIRQ); + } + + static inline bool blk_mq_complete_need_ipi(struct request *rq) +@@ -659,6 +620,32 @@ static inline bool blk_mq_complete_need_ipi(struct request *rq) + return cpu_online(rq->mq_ctx->cpu); + } + ++static void blk_mq_complete_send_ipi(struct request *rq) ++{ ++ struct llist_head *list; ++ unsigned int cpu; ++ ++ cpu = rq->mq_ctx->cpu; ++ list = &per_cpu(blk_cpu_done, cpu); ++ if (llist_add(&rq->ipi_list, list)) { ++ rq->csd.func = __blk_mq_complete_request_remote; ++ rq->csd.info = rq; ++ rq->csd.flags = 0; ++ smp_call_function_single_async(cpu, &rq->csd); ++ } ++} ++ ++static void blk_mq_raise_softirq(struct request *rq) ++{ ++ struct llist_head *list; ++ ++ preempt_disable(); ++ list = this_cpu_ptr(&blk_cpu_done); ++ if (llist_add(&rq->ipi_list, list)) ++ raise_softirq(BLOCK_SOFTIRQ); ++ preempt_enable(); ++} ++ + bool blk_mq_complete_request_remote(struct request *rq) + { + WRITE_ONCE(rq->state, MQ_RQ_COMPLETE); +@@ -671,17 +658,15 @@ bool blk_mq_complete_request_remote(struct request *rq) + return false; + + if (blk_mq_complete_need_ipi(rq)) { +- rq->csd.func = __blk_mq_complete_request_remote; +- rq->csd.info = rq; +- rq->csd.flags = 0; +- smp_call_function_single_async(rq->mq_ctx->cpu, &rq->csd); +- } else { +- if (rq->q->nr_hw_queues > 1) +- return false; +- blk_mq_trigger_softirq(rq); ++ blk_mq_complete_send_ipi(rq); ++ return true; + } + +- return true; ++ if (rq->q->nr_hw_queues == 1) { ++ blk_mq_raise_softirq(rq); ++ return true; ++ } ++ return false; + } + EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote); + +@@ -3980,7 +3965,7 @@ static int __init blk_mq_init(void) + int i; + + for_each_possible_cpu(i) +- INIT_LIST_HEAD(&per_cpu(blk_cpu_done, i)); ++ init_llist_head(&per_cpu(blk_cpu_done, i)); + open_softirq(BLOCK_SOFTIRQ, blk_done_softirq); + + cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD, +diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h +index 98fdf5a31fd6..b35193aec5e3 100644 +--- a/include/linux/blkdev.h ++++ b/include/linux/blkdev.h +@@ -161,7 +161,7 @@ struct request { + */ + union { + struct hlist_node hash; /* merge hash */ +- struct list_head ipi_list; ++ struct llist_node ipi_list; + }; + + /* +-- +2.43.0 + |