FYI, we noticed the below changes on commit 2b75869bba676c248d8d25ae6d2bd9221dfffdb6 ("sysfs/kernfs: allow attributes to request write buffer be pre-allocated.") +----------------------------------------------------+------------+------------+ | | 0936896056 | 2b75869bba | +----------------------------------------------------+------------+------------+ | boot_successes | 64 | 22 | | boot_failures | 6 | 48 | | Initramfs_unpacking_failed | 6 | 12 | | BUG:unable_to_handle_kernel | 6 | 12 | | Oops | 4 | 9 | | EIP_is_at_native_set_pte | 4 | 7 | | Kernel_panic-not_syncing:Fatal_exception | 4 | 7 | | backtrace:set_memory_np | 4 | 7 | | backtrace:free_init_pages | 4 | 7 | | backtrace:populate_rootfs | 4 | 7 | | backtrace:kernel_init_freeable | 4 | 43 | | INFO:possible_circular_locking_dependency_detected | 0 | 36 | | Out_of_memory:Kill_process | 0 | 36 | | backtrace:vfs_write | 0 | 36 | | backtrace:SyS_write | 0 | 36 | | backtrace:cpu_up | 0 | 36 | | backtrace:smp_init | 0 | 36 | | backtrace:process_vm_rw | 0 | 17 | | backtrace:SyS_process_vm_writev | 0 | 7 | | page_allocation_failure:order:#,mode | 0 | 11 | | backtrace:SyS_process_vm_readv | 0 | 10 | | backtrace:do_fork | 0 | 7 | | backtrace:SyS_clone | 0 | 6 | | backtrace:vfs_writev | 0 | 6 | | backtrace:SyS_writev | 0 | 6 | | backtrace:pgd_alloc | 0 | 3 | | backtrace:mm_init | 0 | 3 | | BUG:unable_to | 0 | 1 | | backtrace:lock_torture_stats | 0 | 1 | | backtrace:wait_for_helper | 0 | 2 | | backtrace:__mm_populate | 0 | 1 | | backtrace:SyS_remap_file_pages | 0 | 1 | | backtrace:path_listxattr | 0 | 1 | | backtrace:SyS_listxattr | 0 | 1 | +----------------------------------------------------+------------+------------+ [ 61.776179] Writes: Total: 4 Max/Min: 0/0 Fail: 0 [ 69.817190] [ 69.817551] ====================================================== [ 69.818586] [ INFO: possible circular locking dependency detected ] [ 69.819634] 3.19.0-rc1 #1 Not tainted [ 69.820091] ------------------------------------------------------- [ 69.820091] trinity-c0/3694 is trying to acquire lock: [ 69.820091] (cpu_hotplug.lock){++++++}, at: [<84e69793>] get_online_cpus+0x42/0xcc [ 69.820091] [ 69.820091] but task is already holding lock: [ 69.820091] ((oom_notify_list).rwsem){.+.+..}, at: [<84e94e5d>] __blocking_notifier_call_chain+0x36/0x87 [ 69.820091] [ 69.820091] which lock already depends on the new lock. [ 69.820091] [ 69.820091] [ 69.820091] the existing dependency chain (in reverse order) is: [ 69.820091] -> #4 ((oom_notify_list).rwsem){.+.+..}: [ 69.820091] [<84ec66bd>] lock_acquire+0x9c/0xda [ 69.820091] [<858c6d5a>] down_read+0x56/0x81 [ 69.820091] [<84e94e5d>] __blocking_notifier_call_chain+0x36/0x87 [ 69.820091] [<84e94ec8>] blocking_notifier_call_chain+0x1a/0x2a [ 69.820091] [<84f49773>] out_of_memory+0x39/0x55a [ 69.820091] [<84f50213>] __alloc_pages_nodemask+0x967/0xb2e [ 69.820091] [<84f62bb4>] shmem_getpage_gfp+0x477/0xa83 [ 69.820091] [<84f63281>] shmem_getpage+0x2d/0x40 [ 69.820091] [<84f641a9>] shmem_fault+0x1e8/0x233 [ 69.820091] [<84f78ae2>] __do_fault+0x37/0x95 [ 69.820091] [<84f7afea>] do_read_fault+0x2bf/0x410 [ 69.820091] [<84f7ba04>] handle_mm_fault+0x23c/0x1556 [ 69.820091] [<84e5c345>] __do_page_fault+0x792/0xbea [ 69.820091] [<84e5c7ef>] do_page_fault+0x52/0x64 [ 69.820091] [<84e57088>] do_async_page_fault+0x3f/0xe6 [ 69.820091] [<858cadf6>] error_code+0x42/0x48 [ 69.820091] [<84f73dd9>] copy_page_from_iter+0xef/0x34d [ 69.820091] [<84f92954>] process_vm_rw_core+0x5da/0x7d0 [ 69.820091] [<84f92c81>] process_vm_rw+0x137/0x1c7 [ 69.820091] [<84f92dad>] SyS_process_vm_writev+0x38/0x64 [ 69.820091] [<858ca505>] syscall_after_call+0x0/0x4 [ 69.820091] -> #3 (&mm->mmap_sem){++++++}: [ 69.820091] [<84ec66bd>] lock_acquire+0x9c/0xda [ 69.820091] [<84f786bb>] might_fault+0xb2/0xf0 [ 69.820091] [<850279e2>] kernfs_fop_write+0x13b/0x260 [ 69.820091] [<84fa98bb>] vfs_write+0x101/0x1c4 [ 69.820091] [<84fa9cb9>] SyS_write+0x82/0xf5 [ 69.820091] [<858ca505>] syscall_after_call+0x0/0x4 [ 69.820091] -> #2 (s_active#7){++++.+}: [ 69.820091] [<84ec66bd>] lock_acquire+0x9c/0xda [ 69.820091] [<850248e1>] __kernfs_remove+0x2d4/0x5e0 [ 69.820091] [<85025df9>] kernfs_remove_by_name_ns+0xa5/0xeb [ 69.820091] [<850287a8>] sysfs_remove_file_ns+0x1b/0x2b [ 69.820091] [<8561e9b3>] device_remove_file+0x3b/0x59 [ 69.820091] [<8561f9de>] device_del+0x256/0x37b [ 69.820091] [<8561fb7f>] device_unregister+0x7c/0xaa [ 69.820091] [<8561fc11>] device_destroy+0x64/0x77 [ 69.820091] [<84e42a76>] msr_device_destroy+0x22/0x32 [ 69.820091] [<84e42ae5>] msr_class_cpu_callback+0x5f/0xab [ 69.820091] [<84e947ef>] notifier_call_chain+0x91/0xee [ 69.820091] [<84e94892>] __raw_notifier_call_chain+0x1c/0x2c [ 69.820091] [<84e699fa>] __cpu_notify+0x23/0x72 [ 69.820091] [<84e69a64>] cpu_notify+0x1b/0x2b [ 69.820091] [<84e69a8a>] cpu_notify_nofail+0x16/0x48 [ 69.820091] [<858b1296>] _cpu_down+0x37c/0x520 [ 69.820091] [<858b1486>] cpu_down+0x4c/0x75 [ 69.820091] [<85629270>] cpu_subsys_offline+0x1c/0x2c [ 69.820091] [<8562223d>] device_offline+0xc2/0x129 [ 69.820091] [<8562241b>] online_store+0xa0/0xe5 [ 69.820091] [<8561e528>] dev_attr_store+0x2f/0x4a [ 69.820091] [<850285ea>] sysfs_kf_write+0x57/0x72 [ 69.820091] [<85027a75>] kernfs_fop_write+0x1ce/0x260 [ 69.820091] [<84fa98bb>] vfs_write+0x101/0x1c4 [ 69.820091] [<84fa9cb9>] SyS_write+0x82/0xf5 [ 69.820091] [<858ca505>] syscall_after_call+0x0/0x4 [ 69.820091] -> #1 (cpu_hotplug.lock#2){+.+.+.}: [ 69.820091] [<84ec66bd>] lock_acquire+0x9c/0xda [ 69.820091] [<858c2327>] mutex_lock_nested+0x82/0x9c7 [ 69.820091] [<84e69cd8>] cpu_hotplug_begin+0x62/0xe3 [ 69.820091] [<84e69e83>] cpu_up+0xd3/0x35f [ 69.820091] [<85feddfe>] smp_init+0x11a/0x12d [ 69.820091] [<85fc68f4>] kernel_init_freeable+0x108/0x3f6 [ 69.820091] [<858b0125>] kernel_init+0x16/0x1e7 [ 69.820091] [<858ca341>] ret_from_kernel_thread+0x21/0x30 [ 69.820091] -> #0 (cpu_hotplug.lock){++++++}: [ 69.820091] [<84ec5bf0>] __lock_acquire+0x14e7/0x1f18 [ 69.820091] [<84ec66bd>] lock_acquire+0x9c/0xda [ 69.820091] [<84e697ba>] get_online_cpus+0x69/0xcc [ 69.820091] [<84ee11c0>] rcu_oom_notify+0xea/0x218 [ 69.820091] [<84e947ef>] notifier_call_chain+0x91/0xee [ 69.820091] [<84e94e7d>] __blocking_notifier_call_chain+0x56/0x87 [ 69.820091] [<84e94ec8>] blocking_notifier_call_chain+0x1a/0x2a [ 69.820091] [<84f49773>] out_of_memory+0x39/0x55a [ 69.820091] [<84f50213>] __alloc_pages_nodemask+0x967/0xb2e [ 69.820091] [<84f62bb4>] shmem_getpage_gfp+0x477/0xa83 [ 69.820091] [<84f63281>] shmem_getpage+0x2d/0x40 [ 69.820091] [<84f641a9>] shmem_fault+0x1e8/0x233 [ 69.820091] [<84f78ae2>] __do_fault+0x37/0x95 [ 69.820091] [<84f7afea>] do_read_fault+0x2bf/0x410 [ 69.820091] [<84f7ba04>] handle_mm_fault+0x23c/0x1556 [ 69.820091] [<84e5c345>] __do_page_fault+0x792/0xbea [ 69.820091] [<84e5c7ef>] do_page_fault+0x52/0x64 [ 69.820091] [<84e57088>] do_async_page_fault+0x3f/0xe6 [ 69.820091] [<858cadf6>] error_code+0x42/0x48 [ 69.820091] [<84f73dd9>] copy_page_from_iter+0xef/0x34d [ 69.820091] [<84f92954>] process_vm_rw_core+0x5da/0x7d0 [ 69.820091] [<84f92c81>] process_vm_rw+0x137/0x1c7 [ 69.820091] [<84f92dad>] SyS_process_vm_writev+0x38/0x64 [ 69.820091] [<858ca505>] syscall_after_call+0x0/0x4 [ 69.820091] [ 69.820091] other info that might help us debug this: [ 69.820091] [ 69.820091] Chain exists of: cpu_hotplug.lock --> &mm->mmap_sem --> (oom_notify_list).rwsem [ 69.820091] Possible unsafe locking scenario: [ 69.820091] [ 69.820091] CPU0 CPU1 [ 69.820091] ---- ---- [ 69.820091] lock((oom_notify_list).rwsem); [ 69.820091] lock(&mm->mmap_sem); [ 69.820091] lock((oom_notify_list).rwsem); [ 69.820091] lock(cpu_hotplug.lock); [ 69.820091] [ 69.820091] *** DEADLOCK *** [ 69.820091] [ 69.820091] 2 locks held by trinity-c0/3694: [ 69.820091] #0: (&mm->mmap_sem){++++++}, at: [<84e5c252>] __do_page_fault+0x69f/0xbea [ 69.820091] #1: ((oom_notify_list).rwsem){.+.+..}, at: [<84e94e5d>] __blocking_notifier_call_chain+0x36/0x87 [ 69.820091] [ 69.820091] stack backtrace: [ 69.820091] CPU: 0 PID: 3694 Comm: trinity-c0 Not tainted 3.19.0-rc1 #1 [ 69.820091] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.7.5-20140531_083030-gandalf 04/01/2014 [ 69.820091] 00000000 817ef9b4 858b7132 86511480 817ef9e4 84ec0717 85cb3759 85cb3531 [ 69.820091] 85cb34f9 85cb351a 85cb34f9 817efa38 816a09c0 00000002 816a0e60 816a09c0 [ 69.820091] 817efa68 84ec5bf0 816a0e48 fffffff4 ffffffff 86511480 000004b0 00000000 [ 69.820091] Call Trace: [ 69.820091] [<858b7132>] dump_stack+0xf3/0x192 [ 69.820091] [<84ec0717>] print_circular_bug+0x3cd/0x3e8 [ 69.820091] [<84ec5bf0>] __lock_acquire+0x14e7/0x1f18 [ 69.820091] [<84ec54dc>] ? __lock_acquire+0xdd3/0x1f18 [ 69.820091] [<84ec66bd>] lock_acquire+0x9c/0xda [ 69.820091] [<84e69793>] ? get_online_cpus+0x42/0xcc [ 69.820091] [<84e697ba>] get_online_cpus+0x69/0xcc [ 69.820091] [<84e69793>] ? get_online_cpus+0x42/0xcc [ 69.820091] [<84ee11c0>] rcu_oom_notify+0xea/0x218 [ 69.820091] [<84ec66e2>] ? lock_acquire+0xc1/0xda [ 69.820091] [<84e947ef>] notifier_call_chain+0x91/0xee [ 69.820091] [<84e94e7d>] __blocking_notifier_call_chain+0x56/0x87 [ 69.820091] [<84e94ec8>] blocking_notifier_call_chain+0x1a/0x2a [ 69.820091] [<84f49773>] out_of_memory+0x39/0x55a [ 69.820091] [<84f4963c>] ? oom_zonelist_trylock+0x140/0x158 [ 69.820091] [<84f50213>] __alloc_pages_nodemask+0x967/0xb2e [ 69.820091] [<84f62bb4>] shmem_getpage_gfp+0x477/0xa83 [ 69.820091] [<84f63281>] shmem_getpage+0x2d/0x40 [ 69.820091] [<84f641a9>] shmem_fault+0x1e8/0x233 [ 69.820091] [<84f78ae2>] __do_fault+0x37/0x95 [ 69.820091] [<84f7afea>] do_read_fault+0x2bf/0x410 [ 69.820091] [<84f7ba04>] handle_mm_fault+0x23c/0x1556 [ 69.820091] [<84e5c345>] __do_page_fault+0x792/0xbea [ 69.820091] [<84f031fe>] ? tick_program_event+0x2d/0x40 [ 69.820091] [<84eecf95>] ? hrtimer_interrupt+0x1fc/0x343 [ 69.820091] [<84ebe651>] ? trace_hardirqs_off_caller+0x1df/0x257 [ 69.820091] [<84e57049>] ? kvm_async_pf_task_wake+0x236/0x236 [ 69.820091] [<84e5c7ef>] do_page_fault+0x52/0x64 [ 69.820091] [<84e57088>] do_async_page_fault+0x3f/0xe6 [ 69.820091] [<858cadf6>] error_code+0x42/0x48 [ 69.820091] [<84e900d8>] ? alloc_pid+0x298/0x535 [ 69.820091] [<84f729c0>] ? fault_in_pages_readable+0x62/0x92 [ 69.820091] [<84f73dd9>] copy_page_from_iter+0xef/0x34d [ 69.820091] [<84f431b0>] ? wake_up_page+0x33/0x45 [ 69.820091] [<84f44539>] ? unlock_page+0x62/0x72 [ 69.820091] [<84f92954>] process_vm_rw_core+0x5da/0x7d0 [ 69.820091] [<84fa8d2f>] ? copy_from_user+0x3e/0x4e [ 69.820091] [<84fa9fe6>] ? rw_copy_check_uvector+0xc8/0x1bf [ 69.820091] [<84f92c81>] process_vm_rw+0x137/0x1c7 [ 69.820091] [<84ec6e1e>] ? lock_release+0x394/0x417 [ 69.820091] [<84fa8cb3>] ? file_end_write+0x3b/0x4b [ 69.820091] [<84fa8aa1>] ? do_iter_readv_writev+0xbe/0xbe [ 69.820091] [<84facbce>] ? __sb_end_write+0xcf/0xe4 [ 69.820091] [<84fa8cb3>] ? file_end_write+0x3b/0x4b [ 69.820091] [<84fa9963>] ? vfs_write+0x1a9/0x1c4 [ 69.820091] [<858ca53e>] ? restore_all+0xf/0xf [ 69.820091] [<84f92dad>] SyS_process_vm_writev+0x38/0x64 [ 69.820091] [<858ca505>] syscall_call+0x7/0x7 [ 69.820091] [<858c0000>] ? __schedule+0xa8/0x9b4 [ 69.980353] trinity-c0 invoked oom-killer: gfp_mask=0x200da, order=0, oom_score_adj=0 [ 69.981664] CPU: 0 PID: 3694 Comm: trinity-c0 Not tainted 3.19.0-rc1 #1 [ 69.982763] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.7.5-20140531_083030-gandalf 04/01/2014 [ 69.984531] 00000000 817efaa4 858b7132 816a09c0 817efaec 858b5ac3 85cbaffb 816a0c9c [ 69.986092] 000200da 00000000 00000000 00000246 817efad4 858c901d 85e046ac 00000001 [ 69.987650] 817efaec 00000000 858e56e0 000200da 816a09c0 00000000 817efb10 84f48fec [ 69.989203] Call Trace: [ 69.989641] [<858b7132>] dump_stack+0xf3/0x192 [ 69.990405] [<858b5ac3>] dump_header+0x9e/0x2da [ 69.991301] [<858c901d>] ? _raw_spin_unlock_irqrestore+0xbd/0x141 [ 69.992394] [<84f48fec>] oom_kill_process+0xac/0x521 [ 69.993255] [<84ec6e1e>] ? lock_release+0x394/0x417 [ 69.994098] [<84f49c45>] out_of_memory+0x50b/0x55a Thanks, Huang, Ying