From: Thomas Gleixner Date: Mon, 1 Jul 2013 11:02:42 +0200 Subject: [PATCH 192/342] workqueue: Prevent workqueue versus ata-piix livelock Origin: https://git.kernel.org/cgit/linux/kernel/git/rt/linux-stable-rt.git/commit?id=7e44200efbc637bb4e2d7632e624fae8d13d050f An Intel i7 system regularly detected rcu_preempt stalls after the kernel was upgraded from 3.6-rt to 3.8-rt. When the stall happened, disk I/O was no longer possible, unless the system was restarted. The kernel message was: INFO: rcu_preempt self-detected stall on CPU { 6} [..] NMI backtrace for cpu 6 CPU 6 Pid: 119, comm: irq/19-ata_piix Not tainted 3.8.13-rt13 #11 Shuttle Inc. SX58/SX58 RIP: 0010:[] [] ip_compute_csum+0x30/0x30 RSP: 0018:ffff880333303cb0 EFLAGS: 00000002 RAX: 0000000000000006 RBX: 00000000000003e9 RCX: 0000000000000034 RDX: 0000000000000000 RSI: ffffffff81aa16d0 RDI: 0000000000000001 RBP: ffff880333303ce8 R08: ffffffff81aa16d0 R09: ffffffff81c1b8cc R10: 0000000000000000 R11: 0000000000000000 R12: 000000000005161f R13: 0000000000000006 R14: ffffffff81aa16d0 R15: 0000000000000002 FS: 0000000000000000(0000) GS:ffff880333300000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b CR2: 0000003c1b2bb420 CR3: 0000000001a0f000 CR4: 00000000000007e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 Process irq/19-ata_piix (pid: 119, threadinfo ffff88032d88a000, task ffff88032df80000) Stack: ffffffff8124cb32 000000000005161e 00000000000003e9 0000000000001000 0000000000009022 ffffffff81aa16d0 0000000000000002 ffff880333303cf8 ffffffff8124caa9 ffff880333303d08 ffffffff8124cad2 ffff880333303d28 Call Trace: [] ? delay_tsc+0x33/0xe3 [] __delay+0xf/0x11 [] __const_udelay+0x27/0x29 [] native_safe_apic_wait_icr_idle+0x39/0x45 [] __default_send_IPI_dest_field.constprop.0+0x1e/0x58 [] default_send_IPI_mask_sequence_phys+0x49/0x7d [] physflat_send_IPI_all+0x17/0x19 [] arch_trigger_all_cpu_backtrace+0x50/0x79 [] rcu_check_callbacks+0x1cb/0x568 [] ? raise_softirq+0x2e/0x35 [] ? tick_sched_do_timer+0x38/0x38 [] update_process_times+0x44/0x55 [] tick_sched_handle+0x4a/0x59 [] tick_sched_timer+0x3c/0x5b [] __run_hrtimer+0x9b/0x158 [] hrtimer_interrupt+0x172/0x2aa [] smp_apic_timer_interrupt+0x76/0x89 [] apic_timer_interrupt+0x6d/0x80 [] ? __local_lock_irqsave+0x17/0x4a [] try_to_grab_pending+0x42/0x17e [] mod_delayed_work_on+0x32/0x88 [] mod_delayed_work+0x1c/0x1e [] blk_run_queue_async+0x37/0x39 [] flush_end_io+0xf1/0x107 [] blk_finish_request+0x21e/0x264 [] blk_end_bidi_request+0x42/0x60 [] blk_end_request+0x10/0x12 [] scsi_io_completion+0x1bf/0x492 [] ? sd_done+0x298/0x2ef [] scsi_finish_command+0xe9/0xf2 [] scsi_softirq_done+0x106/0x10f [] blk_done_softirq+0x77/0x87 [] do_current_softirqs+0x172/0x2e1 [] ? irq_thread_fn+0x3a/0x3a [] local_bh_enable+0x43/0x72 [] irq_forced_thread_fn+0x46/0x52 [] irq_thread+0x8c/0x17c [] ? irq_thread+0x17c/0x17c [] ? wake_threads_waitq+0x44/0x44 [] kthread+0x8d/0x95 [] ? __kthread_parkme+0x65/0x65 [] ret_from_fork+0x7c/0xb0 [] ? __kthread_parkme+0x65/0x65 The state of softirqd of this CPU at the time of the crash was: ksoftirqd/6 R running task 0 53 2 0x00000000 ffff88032fc39d18 0000000000000046 ffff88033330c4c0 ffff8803303f4710 ffff88032fc39fd8 ffff88032fc39fd8 0000000000000000 0000000000062500 ffff88032df88000 ffff8803303f4710 0000000000000000 ffff88032fc38000 Call Trace: [] ? __queue_work+0x27c/0x27c [] preempt_schedule+0x61/0x76 [] migrate_enable+0xe5/0x1df [] ? __queue_work+0x27c/0x27c [] run_timer_softirq+0x161/0x1d6 [] do_current_softirqs+0x172/0x2e1 [] run_ksoftirqd+0x2d/0x45 [] smpboot_thread_fn+0x2ea/0x308 [] ? test_ti_thread_flag+0xc/0xc [] ? test_ti_thread_flag+0xc/0xc [] kthread+0x8d/0x95 [] ? __kthread_parkme+0x65/0x65 [] ret_from_fork+0x7c/0xb0 [] ? __kthread_parkme+0x65/0x65 Apparently, the softirq demon and the ata_piix IRQ handler were waiting for each other to finish ending up in a livelock. After the below patch was applied, the system no longer crashes. Reported-by: Carsten Emde Proposed-by: Thomas Gleixner Tested by: Carsten Emde Signed-off-by: Carsten Emde Signed-off-by: Thomas Gleixner Signed-off-by: Sebastian Andrzej Siewior --- kernel/workqueue.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 2b4c052c51b3..6e73e3eaca5e 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -51,6 +51,7 @@ #include #include #include +#include #include "workqueue_internal.h" @@ -1282,7 +1283,7 @@ static int try_to_grab_pending(struct work_struct *work, bool is_dwork, local_unlock_irqrestore(pendingb_lock, *flags); if (work_is_canceling(work)) return -ENOENT; - cpu_relax(); + cpu_chill(); return -EAGAIN; }