Unknown NMI after S4 resume

Unknown NMI after S4 resume

[Tony W Wang-oc]

Hi,

When running S4 test on Zhaoxin platform with Ubuntu22.04 kernel-6.10 we 
got unknown NMI messages after S4 resumed:

[  115.792224] Uhhuh. NMI received for unknown reason 2d on CPU 0.
[  115.792226] Do you have a strange power saving mode enabled?
[  115.792228] Dazed and confused, but trying to continue

And reproduced on Intel platform.

After tracing, we find that the reason for this Unknown NMI occurs is as 
follows:
a, 1st kernel starts normally and NMI watchdog is enabled on all cores;
b, NMI watchdog is disabled on all cores through the sys interface, then 
variable active_events goto zero;
c, Start hibernate, create & save hibernation image, then go hibernated;
d, S4 resume event happened, 2nd kernel starts normally and NMI watchdog 
is enabled on All cores;
e, 2nd kernel find S4 image and try to restore S4 image;
f, 2nd kernel disable non-boot CPUs, which would disable NMI watchdog 
for APs;
g, Restore S4 image saved at step c;
h, 1st-hibernated kernel restore, re-enable non-boot CPUs, as NMI 
watchdog is disabled in step b, this which would keep APs NMI watchdog 
disabled; Besides, the variable active_events is restored to zero;

But BSP NMI watchdog is still enabled, and the PMC will trigger NMI 
interrupt periodically.
If PMC NMI triggered, perf_event_nmi_handler will be called, but it 
would see active_events is zero, so it goes out directly and return 
NMI_DONE;
This then leads to unknown NMI messages.

static int
perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
{
        u64 start_clock;
        u64 finish_clock;
        int ret;

        /*
        * All PMUs/events that share this PMI handler should make sure to
        * increment active_events for their events.
        */
        if (!atomic_read(&active_events))
               return NMI_DONE;
......

It seems that the BSP does not refer to the settings of the NMI watchdog 
sys interface previously saved to the S4 image to configure the NMI 
watchdog when doing S4 resume.
Should consider this situation and patch it?

---
Zhaoxin platform /proc/cpuinfo
processor	: 0
vendor_id	: CentaurHauls
cpu family	: 7
model		: 59
model name	: ZHAOXIN KaiXian KX-U6780A@2.7GHz
stepping	: 0
cpu MHz		: 2694.594
cache size	: 4096 KB
physical id	: 0
siblings	: 8
core id		: 0
cpu cores	: 8
apicid		: 0
initial apicid	: 0
fpu		: yes
fpu_exception	: yes
cpuid level	: 13
wp		: yes
flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov 
pat pse36 clflush acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb 
rdtscp lm constant_tsc arch_perfmon rep_good nopl nonstop_tsc cpuid 
aperfmperf pni pclmulqdq monitor vmx smx est tm2 ssse3 cx16 xtpr pcid 
sse4_1 sse4_2 movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand 
rng rng_en ace ace_en ace2 phe phe_en pmm pmm_en lahf_lm abm 
3dnowprefetch ibrs ibpb tpr_shadow ept vpid fsgsbase tsc_adjust bmi1 
smep bmi2 invpcid rdseed adx smap sha_ni xsaveopt dtherm vnmi umip pku 
ospke rdpid arch_capabilities
vmx flags	: vnmi invvpid ept_x_only ept_1gb tsc_offset vtpr mtf ept vpid 
unrestricted_guest ple
bugs		: spectre_v1
bogomips	: 5389.21
clflush size	: 64
cache_alignment	: 64
address sizes	: 40 bits physical, 48 bits virtual
power management:

Intel platform /proc/cpuinfo
processor	: 0
vendor_id	: GenuineIntel
cpu family	: 6
model		: 151
model name	: 12th Gen Intel(R) Core(TM) i7-12700K
stepping	: 2
microcode	: 0x35
cpu MHz		: 4653.842
cache size	: 25600 KB
physical id	: 0
siblings	: 20
core id		: 0
cpu cores	: 12
apicid		: 0
initial apicid	: 0
fpu		: yes
fpu_exception	: yes
cpuid level	: 32
wp		: yes
flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov 
pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx 
pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl 
xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq 
dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm 
sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c 
rdrand lahf_lm abm 3dnowprefetch cpuid_fault cat_l2 cdp_l2 ssbd ibrs 
ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad 
fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid rdt_a rdseed adx 
smap clflushopt clwb intel_pt sha_ni xsaveopt xsavec xgetbv1 xsaves 
split_lock_detect avx_vnni dtherm ida arat pln pts hwp hwp_notify 
hwp_act_window hwp_epp hwp_pkg_req hfi vnmi umip pku ospke waitpkg gfni 
vaes vpclmulqdq rdpid movdiri movdir64b fsrm md_clear serialize pconfig 
arch_lbr ibt flush_l1d arch_capabilities
vmx flags	: vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad 
ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid 
unrestricted_guest vapic_reg vid ple shadow_vmcs ept_violation_ve 
ept_mode_based_exec tsc_scaling usr_wait_pause
bugs		: spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb rfds bhi
bogomips	: 7219.20
clflush size	: 64
cache_alignment	: 64
address sizes	: 46 bits physical, 48 bits virtual
power management:

BRs!
TonyWWang-oc

Re: Unknown NMI after S4 resume

[Pavel Machek]

[-- Attachment #1: Type: text/plain, Size: 2400 bytes --]

Hi!

> When running S4 test on Zhaoxin platform with Ubuntu22.04 kernel-6.10 we got
> unknown NMI messages after S4 resumed:
> 
> [  115.792224] Uhhuh. NMI received for unknown reason 2d on CPU 0.
> [  115.792226] Do you have a strange power saving mode enabled?
> [  115.792228] Dazed and confused, but trying to continue
> 
> And reproduced on Intel platform.
> 
> After tracing, we find that the reason for this Unknown NMI occurs is as
> follows:
> a, 1st kernel starts normally and NMI watchdog is enabled on all cores;
> b, NMI watchdog is disabled on all cores through the sys interface, then
> variable active_events goto zero;
> c, Start hibernate, create & save hibernation image, then go hibernated;
> d, S4 resume event happened, 2nd kernel starts normally and NMI watchdog is
> enabled on All cores;
> e, 2nd kernel find S4 image and try to restore S4 image;
> f, 2nd kernel disable non-boot CPUs, which would disable NMI watchdog for
> APs;
> g, Restore S4 image saved at step c;
> h, 1st-hibernated kernel restore, re-enable non-boot CPUs, as NMI watchdog
> is disabled in step b, this which would keep APs NMI watchdog disabled;
> Besides, the variable active_events is restored to zero;
> 
> But BSP NMI watchdog is still enabled, and the PMC will trigger NMI
> interrupt periodically.
> If PMC NMI triggered, perf_event_nmi_handler will be called, but it would
> see active_events is zero, so it goes out directly and return NMI_DONE;
> This then leads to unknown NMI messages.
> 
> static int
> perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
> {
>        u64 start_clock;
>        u64 finish_clock;
>        int ret;
> 
>        /*
>        * All PMUs/events that share this PMI handler should make sure to
>        * increment active_events for their events.
>        */
>        if (!atomic_read(&active_events))
>               return NMI_DONE;
> ......
> 
> It seems that the BSP does not refer to the settings of the NMI watchdog sys
> interface previously saved to the S4 image to configure the NMI watchdog
> when doing S4 resume.
> Should consider this situation and patch it?

Yes, please.

The watchdog driver should get suspend/resume hooks, and probably do
same init on boot and on resume.

Best regards,

								Pavel
-- 
People of Russia, stop Putin before his war on Ukraine escalates.

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 195 bytes --]

Re: Unknown NMI after S4 resume

[Tony W Wang-oc]

On 2024/7/25 16:27, Pavel Machek wrote:
> 
> Yes, please.
> 
> The watchdog driver should get suspend/resume hooks, and probably do
> same init on boot and on resume.
> 

Two patch methods were tried and the results are as follows:

Patch 1:
diff -aNur linux-6.10/kernel/watchdog.c linux-6.10_test/kernel/watchdog.c
--- linux-6.10/kernel/watchdog.c	2024-07-15 06:43:32.000000000 +0800
+++ linux-6.10_test/kernel/watchdog.c	2024-07-26 18:38:16.442600567 +0800
@@ -25,6 +25,7 @@
  #include <linux/stop_machine.h>
  #include <linux/sysctl.h>
  #include <linux/tick.h>
+#include <linux/syscore_ops.h>

  #include <linux/sched/clock.h>
  #include <linux/sched/debug.h>
@@ -1264,3 +1265,35 @@

  	lockup_detector_setup();
  }
+
+static int watchdog_suspend(void)
+{
+        if (cpumask_test_cpu(0, &watchdog_allowed_mask))
+                watchdog_hardlockup_disable(0);
+
+        lockup_detector_cleanup();
+
+        return 0;
+}
+
+static void watchdog_resume(void)
+{
+        if (cpumask_test_cpu(0, &watchdog_allowed_mask)
+        && (watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED))
+                watchdog_hardlockup_enable(0);
+}
+
+static struct syscore_ops watchdog_syscore_ops = {
+        .resume = watchdog_resume,
+        .suspend = watchdog_suspend,
+};
+
+static int __init init_watchdog_sysfs(void)
+{
+        register_syscore_ops(&watchdog_syscore_ops);
+
+        return 0;
+}
+
+late_initcall(init_watchdog_sysfs);
+

Patch 1 result is: There is no unknown NMI after S4 resume, but this 
patch introduces the following warning:

[    8.994619] ------------[ cut here ]------------
[    8.999205] Interrupts enabled after watchdog_suspend+0x0/0x50
[    9.005011] WARNING: CPU: 0 PID: 219 at drivers/base/syscore.c:68 
syscore_suspend+0x17c/0x290
[    9.013463] Modules linked in: uas usb_storage i2c_hid_acpi i2c_hid 
r8169 via_rng crc32_pclmul ahci drm xhci_pci psmouse libahci 
xhci_pci_renesas realtek hid video wmi
[    9.028289] CPU: 0 PID: 219 Comm: sh Not tainted 6.10.0 #2
[    9.033726] Hardware name: Shanghai Zhaoxin Semiconductor Co., Ltd. 
ZXE CRB/HX005EZ, BIOS 47.00 06/19/2024
[    9.043289] RIP: 0010:syscore_suspend+0x17c/0x290
[    9.047953] Code: 80 fc 01 0f 87 0f 06 5f 00 41 83 e4 01 0f 85 1a ff 
ff ff 48 8b 73 10 48 c7 c7 0e c8 81 b0 c6 05 06 d4 48 01 01 e8 a4 de 5a 
ff <0f> 0b e9 fc fe ff ff 65 8b 05 f2 5e 6e 50 48 0f a3 05 7e 04 4e 01
[    9.066546] RSP: 0018:ffffbabf8073fbe8 EFLAGS: 00010246
[    9.071723] RAX: 0000000000000000 RBX: ffffffffb0bf29e0 RCX: 
0000000000000000
[    9.078792] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 
0000000000000000
[    9.085864] RBP: ffffbabf8073fbf8 R08: 0000000000000000 R09: 
0000000000000000
[    9.092933] R10: 0000000000000000 R11: 0000000000000000 R12: 
0000000000000000
[    9.100000] R13: fffffffffffffff2 R14: ffff8edd5106e000 R15: 
0000000000000004
[    9.107070] FS:  000070a4a2ac3740(0000) GS:ffff8ee06fa00000(0000) 
knlGS:0000000000000000
[    9.115086] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[    9.120779] CR2: 00005d4095519bd8 CR3: 00000001030e8002 CR4: 
0000000000770ef0
[    9.127849] PKRU: 55555554
[    9.130534] Call Trace:
[    9.132961]  <TASK>
[    9.135042]  ? show_regs+0x71/0x90
[    9.138416]  ? __warn+0x88/0x140
[    9.141617]  ? syscore_suspend+0x17c/0x290
[    9.145678]  ? report_bug+0x1ab/0x1c0
[    9.149313]  ? console_unlock+0x77/0x120
[    9.153204]  ? handle_bug+0x46/0x90
[    9.156665]  ? exc_invalid_op+0x19/0x80
[    9.160464]  ? asm_exc_invalid_op+0x1b/0x20
[    9.164614]  ? syscore_suspend+0x17c/0x290
[    9.168674]  ? syscore_suspend+0x17c/0x290
......

Patch 2:
diff -aNur linux-6.10/include/linux/nmi.h 
linux-6.10_test/include/linux/nmi.h
--- linux-6.10/include/linux/nmi.h	2024-07-15 06:43:32.000000000 +0800
+++ linux-6.10_test/include/linux/nmi.h	2024-07-25 18:57:38.886105105 +0800
@@ -19,6 +19,9 @@
  void lockup_detector_soft_poweroff(void);
  void lockup_detector_cleanup(void);

+extern int nmi_watchdog_bsp_suspend(void);
+extern void nmi_watchdog_bsp_resume(void);
+
  extern int watchdog_user_enabled;
  extern int watchdog_thresh;
  extern unsigned long watchdog_enabled;
diff -aNur linux-6.10/kernel/power/hibernate.c 
linux-6.10_test/kernel/power/hibernate.c
--- linux-6.10/kernel/power/hibernate.c	2024-07-15 06:43:32.000000000 +0800
+++ linux-6.10_test/kernel/power/hibernate.c	2024-07-25 
19:07:16.470095318 +0800
@@ -313,6 +313,8 @@
  	if (error || hibernation_test(TEST_PLATFORM))
  		goto Platform_finish;

+	nmi_watchdog_bsp_suspend();
+
  	error = pm_sleep_disable_secondary_cpus();
  	if (error || hibernation_test(TEST_CPUS))
  		goto Enable_cpus;
@@ -356,7 +358,9 @@

   Enable_cpus:
  	pm_sleep_enable_secondary_cpus();
-
+	
+	nmi_watchdog_bsp_resume();
+	
  	/* Allow architectures to do nosmt-specific post-resume dances */
  	if (!in_suspend)
  		error = arch_resume_nosmt();
@@ -481,6 +485,8 @@

  	cpuidle_pause();

+	nmi_watchdog_bsp_suspend();
+
  	error = hibernate_resume_nonboot_cpu_disable();
  	if (error)
  		goto Enable_cpus;
@@ -526,6 +532,8 @@
   Enable_cpus:
  	pm_sleep_enable_secondary_cpus();

+	nmi_watchdog_bsp_resume();
+
   Cleanup:
  	platform_restore_cleanup(platform_mode);

diff -aNur linux-6.10/kernel/watchdog.c linux-6.10_test/kernel/watchdog.c
--- linux-6.10/kernel/watchdog.c	2024-07-15 06:43:32.000000000 +0800
+++ linux-6.10_test/kernel/watchdog.c	2024-07-26 09:53:04.579979589 +0800
@@ -1249,6 +1249,36 @@
  }
  late_initcall_sync(lockup_detector_check);

+int bsp_watchdog_enable(void *data)
+{
+        if (cpumask_test_cpu(smp_processor_id(), &watchdog_allowed_mask))
+                watchdog_hardlockup_enable(smp_processor_id());
+        return 0;
+}
+int bsp_watchdog_disable(void *data)
+{
+        if (cpumask_test_cpu(smp_processor_id(), &watchdog_allowed_mask))
+                watchdog_hardlockup_disable(smp_processor_id());
+        return 0;
+}
+
+int nmi_watchdog_bsp_suspend(void)
+{
+        smp_call_on_cpu(0,bsp_watchdog_disable,NULL,false);
+        return 0;
+}
+EXPORT_SYMBOL(nmi_watchdog_bsp_suspend);
+
+void nmi_watchdog_bsp_resume(void)
+{
+        /* Enable the perf event */
+        if (watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED){
+               smp_call_on_cpu(0,bsp_watchdog_enable,NULL,false);
+        }
+        return;
+}
+EXPORT_SYMBOL(nmi_watchdog_bsp_resume);
+
  void __init lockup_detector_init(void)
  {
  	if (tick_nohz_full_enabled())

Patch 2 result is: There is no unknown NMI after S4 resume, and the 
results of multiple tests are ok.

Can patch 2 be used to fix this problem?

BRs!
TonyWWang-oc