FYI, we noticed the below changes on git://internal_merge_and_test_tree revert-8ba848287b7a408b280e30beea9a29d1297662a5-8ba848287b7a408b280e30beea9a29d1297662a5 commit 8ba848287b7a408b280e30beea9a29d1297662a5 ("x86/dumpstack: Show top of special stack if we OOPS on a special stack") +------------------------------------------------------------------------------+------------+------------+ | | 0de0ad4a15 | 8ba848287b | +------------------------------------------------------------------------------+------------+------------+ | boot_successes | 11 | 11 | | boot_failures | 20 | 19 | | INFO:task_blocked_for_more_than#seconds | 18 | 18 | | RIP:delay_tsc | 2 | 1 | | RIP:flat_send_IPI_mask | 18 | 5 | | Kernel_panic-not_syncing:hung_task:blocked_tasks | 18 | 4 | | backtrace:vfs_write | 16 | 15 | | backtrace:SyS_write | 16 | 15 | | backtrace:vfs_read | 4 | 1 | | backtrace:SyS_read | 4 | 1 | | backtrace:watchdog | 18 | 4 | | RIP:update_cfs_shares | 1 | | | backtrace:cpu_startup_entry | 8 | 7 | | RIP:__schedule | 1 | | | RIP:native_safe_halt | 5 | 2 | | RIP:irq_exit | 1 | | | backtrace:do_sys_open | 2 | 1 | | backtrace:SyS_open | 2 | 1 | | RIP:rcu_read_lock_held | 1 | | | RIP:___might_sleep | 1 | | | RIP:lock_is_held | 1 | | | RIP:arch_cpu_idle | 1 | | | RIP:sched_clock_local | 1 | 1 | | invoked_oom-killer:gfp_mask=0x | 2 | 1 | | Mem-Info | 2 | 1 | | Out_of_memory:Kill_process | 1 | 1 | | RIP:set_next_entity | 1 | | | RIP:raise_softirq | 1 | | | RIP:get_next_timer_interrupt | 1 | | | BUG:scheduling_while_atomic | 0 | 14 | | BUG:sleeping_function_called_from_invalid_context_at_include/linux/pagemap.h | 0 | 1 | | BUG:sleeping_function_called_from_invalid_context_at_mm/slub.c | 0 | 2 | | backtrace:do_execve | 0 | 1 | | backtrace:SyS_execve | 0 | 1 | | RIP:trace_hardirqs_on | 0 | 1 | | backtrace:x86_64_start_kernel | 0 | 3 | | backtrace:do_dup2 | 0 | 2 | | backtrace:SyS_dup3 | 0 | 2 | | backtrace:SyS_dup2 | 0 | 2 | | RIP:trace_hardirqs_on_caller | 0 | 1 | +------------------------------------------------------------------------------+------------+------------+ The first softlockup seems not related to the commit. But the BUG following appears related. [ 360.697040] BUG: scheduling while atomic: swapper/0/0/0x00000002 [ 120.359549] XFS (sdc): Unmounting Filesystem [ 120.378720] XFS (sdd): Unmounting Filesystem 2015-09-19 18:08:35 export TEST_DIR=/fs/sda [ 360.534202] INFO: task cat-vmstat:4009 blocked for more than 120 seconds. [ 360.536125] Not tainted 4.3.0-rc1-00050-g8ba8482 #1 [ 360.537786] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 360.540392] cat-vmstat D ffff8800383d6880 13568 4009 1 0x00000000 [ 360.549607] ffff88003d66fa48 0000000000000086 000000177cbca02e ffff8800383d6898 [ 360.552534] ffff8800361e0000 ffff88003b81d400 ffff88003d670000 7fffffffffffffff [ 360.555425] 7fffffffffffffff ffffffff81c2eae2 0000000000000002 ffff88003d66fa60 [ 360.567558] Call Trace: [ 360.577645] [] ? bit_wait+0x4b/0x4b [ 360.579203] [] schedule+0x7a/0x8f [ 360.580749] [] schedule_timeout+0x3f/0x29d [ 360.582410] [] ? sched_clock_cpu+0x99/0xad [ 360.584026] [] ? mark_held_locks+0x52/0x68 [ 360.585627] [] ? ktime_get+0x8e/0x117 [ 360.605356] [] ? bit_wait+0x4b/0x4b [ 360.607036] [] ? kvm_clock_read+0x25/0x2e [ 360.608770] [] ? bit_wait+0x4b/0x4b [ 360.610315] [] io_schedule_timeout+0xb7/0x12b [ 360.611859] [] ? io_schedule_timeout+0xb7/0x12b [ 360.613416] [] bit_wait_io+0x51/0x55 [ 360.614845] [] __wait_on_bit+0x4b/0x7d [ 360.616290] [] out_of_line_wait_on_bit+0x71/0x7c [ 360.630974] [] ? bit_wait+0x4b/0x4b [ 360.632642] [] ? autoremove_wake_function+0x3a/0x3a [ 360.634526] [] nfs_wait_on_request+0x47/0x4a [ 360.636301] [] nfs_updatepage+0x637/0x7b9 [ 360.638030] [] nfs_write_end+0x12c/0x2fe [ 360.639744] [] generic_perform_write+0x142/0x1fd [ 360.641567] [] __generic_file_write_iter+0xce/0x174 [ 360.643441] [] generic_file_write_iter+0x11b/0x188 [ 360.658129] [] nfs_file_write+0xa1/0x11e [ 360.659622] [] __vfs_write+0x95/0xbe [ 360.661046] [] vfs_write+0xbc/0x163 [ 360.671479] [] SyS_write+0x51/0x92 [ 360.673127] [] entry_SYSCALL_64_fastpath+0x12/0x76 [ 360.674915] 2 locks held by cat-vmstat/4009: [ 360.676182] #0: (sb_writers#12){.+.+.+}, at: [] __sb_start_write+0x5f/0xb0 [ 360.679201] #1: (&sb->s_type->i_mutex_key#14){+.+.+.}, at: [] generic_file_write_iter+0x38/0x188 [ 360.682508] Sending NMI to all CPUs: [ 360.697029] NMI backtrace for cpu 0 [ 360.697040] BUG: scheduling while atomic: swapper/0/0/0x00000002 [ 360.697042] no locks held by swapper/0/0. [ 360.697057] Modules linked in: snd_pcsp [ 360.697060] CPU: 0 PID: 0 Comm: swapper/0 Not tainted 4.3.0-rc1-00050-g8ba8482 #1 [ 360.697061] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.7.5-20140531_083030-gandalf 04/01/2014 [ 360.697064] 0000000000000000 ffffffff8222be68 ffffffff814fdfac ffffffff82238580 [ 360.697066] ffffffff8222be80 ffffffff810b7bd4 ffff8800381d6880 ffffffff8222bed8 [ 360.697069] ffffffff81c2d5a8 ffffffff82238580 ffffffff81110149 ffffffff82228000 [ 360.697069] Call Trace: [ 360.697072] [] dump_stack+0x4b/0x63 [ 360.697076] [] __schedule_bug+0x64/0x73 [ 360.697078] [] __schedule+0x6d/0xe9a [ 360.697081] [] ? tick_nohz_idle_exit+0xf6/0x102 [ 360.697083] [] schedule+0x7a/0x8f [ 360.697084] [] schedule_preempt_disabled+0x15/0x1e [ 360.697087] [] cpu_startup_entry+0x30b/0x3a3 [ 360.697089] [] rest_init+0x13a/0x140 [ 360.697091] [] start_kernel+0x43b/0x448 [ 360.697093] [] ? early_idt_handler_array+0x120/0x120 [ 360.697096] [] x86_64_start_reservations+0x2a/0x2c [ 360.697098] [] x86_64_start_kernel+0x138/0x145 [ 360.701035] BUG: scheduling while atomic: swapper/0/0/0x00000002 [ 360.701036] no locks held by swapper/0/0. [ 360.701039] Modules linked in: snd_pcsp [ 360.701041] CPU: 0 PID: 0 Comm: swapper/0 Tainted: G W 4.3.0-rc1-00050-g8ba8482 #1 [ 360.701042] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.7.5-20140531_083030-gandalf 04/01/2014 [ 360.701045] 0000000000000000 ffffffff8222be68 ffffffff814fdfac ffffffff82238580 [ 360.701047] ffffffff8222be80 ffffffff810b7bd4 ffff8800381d6880 ffffffff8222bed8 [ 360.701050] ffffffff81c2d5a8 ffffffff82238580 ffffffff81110149 ffffffff82228000 [ 360.701050] Call Trace: [ 360.701054] [] dump_stack+0x4b/0x63 [ 360.701057] [] __schedule_bug+0x64/0x73 [ 360.701060] [] __schedule+0x6d/0xe9a [ 360.701063] [] ? tick_nohz_idle_exit+0xf6/0x102 [ 360.701065] [] schedule+0x7a/0x8f [ 360.701067] [] schedule_preempt_disabled+0x15/0x1e [ 360.701069] [] cpu_startup_entry+0x30b/0x3a3 [ 360.701071] [] rest_init+0x13a/0x140 [ 360.701075] [] start_kernel+0x43b/0x448 [ 360.701078] [] ? early_idt_handler_array+0x120/0x120 [ 360.701080] [] x86_64_start_reservations+0x2a/0x2c [ 360.701082] [] x86_64_start_kernel+0x138/0x145 [ 360.706048] BUG: scheduling while atomic: swapper/0/0/0x00000002 [ 360.706049] no locks held by swapper/0/0. [ 360.706052] Modules linked in: snd_pcsp [ 360.706054] CPU: 0 PID: 0 Comm: swapper/0 Tainted: G W 4.3.0-rc1-00050-g8ba8482 #1 [ 360.706055] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.7.5-20140531_083030-gandalf 04/01/2014 [ 360.706059] 0000000000000000 ffffffff8222be68 ffffffff814fdfac ffffffff82238580 [ 360.706061] ffffffff8222be80 ffffffff810b7bd4 ffff8800381d6880 ffffffff8222bed8 [ 360.706063] ffffffff81c2d5a8 ffffffff82238580 ffffffff81110149 ffffffff82228000 [ 360.706064] Call Trace: [ 360.706070] [] dump_stack+0x4b/0x63 [ 360.706074] [] __schedule_bug+0x64/0x73 [ 360.706076] [] __schedule+0x6d/0xe9a [ 360.706081] [] ? tick_nohz_idle_exit+0xf6/0x102 [ 360.706083] [] schedule+0x7a/0x8f [ 360.706085] [] schedule_preempt_disabled+0x15/0x1e [ 360.706088] [] cpu_startup_entry+0x30b/0x3a3 [ 360.706091] [] rest_init+0x13a/0x140 [ 360.706096] [] start_kernel+0x43b/0x448 [ 360.706098] [] ? early_idt_handler_array+0x120/0x120 [ 360.706101] [] x86_64_start_reservations+0x2a/0x2c [ 360.706103] [] x86_64_start_kernel+0x138/0x145 [ 360.709045] BUG: scheduling while atomic: swapper/0/0/0x00000002 [ 360.709047] no locks held by swapper/0/0. [ 360.709049] Modules linked in: snd_pcsp [ 360.709052] CPU: 0 PID: 0 Comm: swapper/0 Tainted: G W 4.3.0-rc1-00050-g8ba8482 #1 [ 360.709053] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.7.5-20140531_083030-gandalf 04/01/2014 [ 360.709056] 0000000000000000 ffffffff8222be68 ffffffff814fdfac ffffffff82238580 [ 360.709058] ffffffff8222be80 ffffffff810b7bd4 ffff8800381d6880 ffffffff8222bed8 [ 360.709060] ffffffff81c2d5a8 ffffffff82238580 ffffffff81110149 ffffffff82228000 [ 360.709061] Call Trace: [ 360.709066] [] dump_stack+0x4b/0x63 [ 360.709069] [] __schedule_bug+0x64/0x73 [ 360.709071] [] __schedule+0x6d/0xe9a [ 360.709074] [] ? tick_nohz_idle_exit+0xf6/0x102 [ 360.709076] [] schedule+0x7a/0x8f [ 360.709078] [] schedule_preempt_disabled+0x15/0x1e [ 360.709081] [] cpu_startup_entry+0x30b/0x3a3 [ 360.709083] [] rest_init+0x13a/0x140 [ 360.709086] [] start_kernel+0x43b/0x448 [ 360.709088] [] ? early_idt_handler_array+0x120/0x120 [ 360.709090] [] x86_64_start_reservations+0x2a/0x2c [ 360.709092] [] x86_64_start_kernel+0x138/0x145 [ 360.717047] BUG: scheduling while atomic: swapper/0/0/0x00000002 [ 360.717048] no locks held by swapper/0/0. [ 360.717051] Modules linked in: snd_pcsp [ 360.717054] CPU: 0 PID: 0 Comm: swapper/0 Tainted: G W 4.3.0-rc1-00050-g8ba8482 #1 [ 360.717055] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.7.5-20140531_083030-gandalf 04/01/2014 [ 360.717058] 0000000000000000 ffffffff8222be68 ffffffff814fdfac ffffffff82238580 [ 360.717060] ffffffff8222be80 ffffffff810b7bd4 ffff8800381d6880 ffffffff8222bed8 [ 360.717062] ffffffff81c2d5a8 ffffffff82238580 ffffffff81110149 ffffffff82228000 [ 360.717063] Call Trace: [ 360.717069] [] dump_stack+0x4b/0x63 [ 360.717074] [] __schedule_bug+0x64/0x73 [ 360.717077] [] __schedule+0x6d/0xe9a [ 360.717081] [] ? tick_nohz_idle_exit+0xf6/0x102 [ 360.717083] [] schedule+0x7a/0x8f [ 360.717085] [] schedule_preempt_disabled+0x15/0x1e [ 360.717089] [] cpu_startup_entry+0x30b/0x3a3 [ 360.717092] [] rest_init+0x13a/0x140 [ 360.717096] [] start_kernel+0x43b/0x448 [ 360.717099] [] ? early_idt_handler_array+0x120/0x120 [ 360.717102] [] x86_64_start_reservations+0x2a/0x2c [ 360.717104] [] x86_64_start_kernel+0x138/0x145 [ 360.722112] BUG: scheduling while atomic: swapper/0/0/0x00000002 [ 360.722113] no locks held by swapper/0/0. [ 360.722116] Modules linked in: snd_pcsp [ 360.722119] CPU: 0 PID: 0 Comm: swapper/0 Tainted: G W 4.3.0-rc1-00050-g8ba8482 #1 [ 360.722120] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.7.5-20140531_083030-gandalf 04/01/2014 [ 360.722123] 0000000000000000 ffffffff8222be68 ffffffff814fdfac ffffffff82238580 [ 360.722125] ffffffff8222be80 ffffffff810b7bd4 ffff8800381d6880 ffffffff8222bed8 [ 360.722128] ffffffff81c2d5a8 ffffffff82238580 ffffffff81110149 ffffffff82228000 [ 360.722128] Call Trace: [ 360.722134] [] dump_stack+0x4b/0x63 [ 360.722138] [] __schedule_bug+0x64/0x73 [ 360.722140] [] __schedule+0x6d/0xe9a [ 360.722144] [] ? tick_nohz_idle_exit+0xf6/0x102 [ 360.722146] [] schedule+0x7a/0x8f [ 360.722148] [] schedule_preempt_disabled+0x15/0x1e [ 360.722150] [] cpu_startup_entry+0x30b/0x3a3 [ 360.722152] [] rest_init+0x13a/0x140 [ 360.722156] [] start_kernel+0x43b/0x448 [ 360.722158] [] ? early_idt_handler_array+0x120/0x120 [ 360.722160] [] x86_64_start_reservations+0x2a/0x2c [ 360.722162] [] x86_64_start_kernel+0x138/0x145 [ 360.725037] BUG: scheduling while atomic: swapper/0/0/0x00000002 [ 360.725038] no locks held by swapper/0/0. [ 360.725040] Modules linked in: snd_pcsp [ 360.725042] CPU: 0 PID: 0 Comm: swapper/0 Tainted: G W 4.3.0-rc1-00050-g8ba8482 #1 [ 360.725043] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.7.5-20140531_083030-gandalf 04/01/2014 [ 360.725046] 0000000000000000 ffffffff8222be68 ffffffff814fdfac ffffffff82238580 [ 360.725048] ffffffff8222be80 ffffffff810b7bd4 ffff8800381d6880 ffffffff8222bed8 [ 360.725050] ffffffff81c2d5a8 ffffffff82238580 ffffffff81110149 ffffffff82228000 [ 360.725050] Call Trace: [ 360.725055] [] dump_stack+0x4b/0x63 [ 360.725059] [] __schedule_bug+0x64/0x73 [ 360.725061] [] __schedule+0x6d/0xe9a [ 360.725064] [] ? tick_nohz_idle_exit+0xf6/0x102 [ 360.725067] [] schedule+0x7a/0x8f [ 360.725068] [] schedule_preempt_disabled+0x15/0x1e [ 360.725071] [] cpu_startup_entry+0x30b/0x3a3 [ 360.725074] [] rest_init+0x13a/0x140 [ 360.725077] [] start_kernel+0x43b/0x448 [ 360.725079] [] ? early_idt_handler_array+0x120/0x120 [ 360.725081] [] x86_64_start_reservations+0x2a/0x2c [ 360.725083] [] x86_64_start_kernel+0x138/0x145 [ 360.728046] BUG: scheduling while atomic: swapper/0/0/0x00000002 [ 360.728047] no locks held by swapper/0/0. [ 360.728049] Modules linked in: snd_pcsp [ 360.728052] CPU: 0 PID: 0 Comm: swapper/0 Tainted: G W 4.3.0-rc1-00050-g8ba8482 #1 [ 360.728053] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.7.5-20140531_083030-gandalf 04/01/2014 [ 360.728056] 0000000000000000 ffffffff8222be68 ffffffff814fdfac ffffffff82238580 [ 360.728058] ffffffff8222be80 ffffffff810b7bd4 ffff8800381d6880 ffffffff8222bed8 [ 360.728060] ffffffff81c2d5a8 ffffffff82238580 ffffffff81110149 ffffffff82228000 [ 360.728061] Call Trace: [ 360.728066] [] dump_stack+0x4b/0x63 [ 360.728069] [] __schedule_bug+0x64/0x73 To reproduce: git clone git://git.kernel.org/pub/scm/linux/kernel/git/wfg/lkp-tests.git cd lkp-tests bin/lkp install job.yaml # job file is attached in this email bin/lkp run job.yaml Thanks, Ying Huang