[PATCH v2] x86/nmi: Enable panic() when NMI handler runs too long

["Paul E. McKenney" <paulmck@kernel.org>]

Firmware, device-driver, and configuration issues can result in
long-running NMI handlers, for example, nmi_cpu_backtrace_handler()
running for more than two seconds, and with no console issues.  In this
case, this is not the fault of nmi_cpu_backtrace_handler(), but it is
the first symptom of the underlying problem.  These NMI handlers result
in various other problems, including CSD-lock warnings, clocksource
watchdog false positives, scheduler frequency invariance wobbliness,
systemd excessive-CPU complaints, and even the occasional RCU CPU
stall warning.  Eventually, the automation will take the system out of
service, but with quite a bit of noise obscuring the underlying issue,
and thus confusing both automation and human beings.

Therefore, provide a toolong_nmi_panic __setup() parameter that makes
nmi_check_duration() invoke nmi_panic() if the just-completed NMI handler
ran for more than the specified number of milliseconds.  This parameter
defaults to zero, which disables the nmi_panic().  As in, by default
you get the same behavior that you get today.

[ paulmck: Apply kernel test robot feedback, add documentation. ]

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Cc: Thomas Gleixner <tglx@kernel.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: <x86@kernel.org>

---
 Documentation/admin-guide/kernel-parameters.txt |    6 ++++++
 arch/x86/kernel/nmi.c                           |   21 +++++++++++++++++++--
 2 files changed, 25 insertions(+), 2 deletions(-)

Changes since v1:

o	Add documentation of new toolong_nmi_panic kernel boot parameter.

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 68647ff4bdd24..e7dbd5bf48fc9 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -7642,6 +7642,12 @@ Kernel parameters
 			Format: {ipi}
 			ipi: switch to IPI-based TLB flushing
 
+	toolong_nmi_panic= [X86]
+			Cause a panic upon return from an NMI handler
+			that took longer than the specified number
+			of milliseconds.  The default value of zero
+			suppresses these panics.
+
 	topology=	[S390,EARLY]
 			Format: {off | on}
 			Specify if the kernel should make use of the cpu
diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
index 3c9f60d6ca5a7..aae7254774680 100644
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -85,6 +85,7 @@ static DEFINE_PER_CPU(struct nmi_stats, nmi_stats);
 static int ignore_nmis __read_mostly;
 
 int unknown_nmi_panic;
+static unsigned long toolong_nmi_panic;
 int panic_on_unrecovered_nmi;
 int panic_on_io_nmi;
 
@@ -101,6 +102,20 @@ static int __init setup_unknown_nmi_panic(char *str)
 }
 __setup("unknown_nmi_panic", setup_unknown_nmi_panic);
 
+static int __init setup_toolong_nmi_panic(char *str)
+{
+	int ret;
+
+	ret = kstrtoul(str, 0, &toolong_nmi_panic);
+	if (ret) {
+		pr_alert("%s: %s on '%s', disabling to zero.\n",
+			 __func__, ret == -ERANGE ? "Overflow" : "Parse error", str);
+		toolong_nmi_panic = 0;
+	}
+	return 1;
+}
+__setup("toolong_nmi_panic=", setup_toolong_nmi_panic);
+
 static u64 nmi_longest_ns = 1 * NSEC_PER_MSEC;
 
 static int __init nmi_warning_debugfs(void)
@@ -111,7 +126,7 @@ static int __init nmi_warning_debugfs(void)
 }
 fs_initcall(nmi_warning_debugfs);
 
-static void nmi_check_duration(struct nmiaction *action, u64 duration)
+static void nmi_check_duration(struct nmiaction *action, u64 duration, struct pt_regs *regs)
 {
 	int remainder_ns, decimal_msecs;
 
@@ -126,6 +141,8 @@ static void nmi_check_duration(struct nmiaction *action, u64 duration)
 
 	pr_info_ratelimited("INFO: NMI handler (%ps) took too long to run: %lld.%03d msecs\n",
 			    action->handler, duration, decimal_msecs);
+	if (toolong_nmi_panic && duration >= toolong_nmi_panic)
+		nmi_panic(regs, "Panic per toolong_nmi_panic");
 }
 
 static int nmi_handle(unsigned int type, struct pt_regs *regs)
@@ -165,7 +182,7 @@ static int nmi_handle(unsigned int type, struct pt_regs *regs)
 		delta = sched_clock() - delta;
 		trace_nmi_handler(a->handler, (int)delta, thishandled);
 
-		nmi_check_duration(a, delta);
+		nmi_check_duration(a, delta, regs);
 	}
 
 	rcu_read_unlock();