[PATCH] x86/resctrl: Require force for L3 CAT on early Skylake-X

[PATCH] x86/resctrl: Require force for L3 CAT on early Skylake-X

[Alexander Graf]

When L3 CAT gets enabled with rdt=l3cat on Skylake-X stepping 3,
consecutive kexec boots hang for me during secondary socket bringup.
Cold boots on the same machine work.

We already disable L3 CAT by default on Skylake-X and Cascade Lake and
enable it only when the rdt=l3cat option is passed on the kernel command
line. I want to enable l3cat on Cascade Lake, but preserve a single kernel
command line across both Skylake and Cascade Lake systems.

To enable this, guard l3cat on Skylake-X steppings below 4 behind a force
option. That way, safe (but potentially imprecise) options continue to be
guarded by the normal rdt command line, but combinations that impact
system stability also require force in addition. With this, I can have a
single, safe kernel command line across both environments.

Signed-off-by: Alexander Graf <graf@amazon.com>
---
 .../admin-guide/kernel-parameters.txt         |  6 ++++
 arch/x86/kernel/cpu/resctrl/core.c            | 31 ++++++++++++++++++-
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index e75344f4e0cd..54f8a9ea665a 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -6458,6 +6458,12 @@ Kernel parameters
 			perf telemetry monitoring associated with guid 0x12345
 			is enabled use:
 				rdt=!energy,perf:0x12345
+			On Skylake-X stepping < 4, rdt=l3cat is ignored
+			unless the bare token "force" is also given, e.g.:
+				rdt=force,l3cat
+			because L3 CAT there can hang a later kexec boot.
+			The token may appear anywhere in any rdt= parameter.
+			"!force" is ignored.
 
 	reboot=		[KNL]
 			Format (x86 or x86_64):
diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c
index 55214d6fdc49..c4eb157b8b72 100644
--- a/arch/x86/kernel/cpu/resctrl/core.c
+++ b/arch/x86/kernel/cpu/resctrl/core.c
@@ -833,6 +833,8 @@ static struct rdt_options rdt_options[]  __ro_after_init = {
 };
 #define NUM_RDT_OPTIONS ARRAY_SIZE(rdt_options)
 
+static bool rdt_force_raw __initdata;
+
 static int __init set_rdt_options(char *str)
 {
 	struct rdt_options *o;
@@ -842,6 +844,10 @@ static int __init set_rdt_options(char *str)
 	if (*str == '=')
 		str++;
 	while ((tok = strsep(&str, ",")) != NULL) {
+		if (!strcmp(tok, "force")) {
+			rdt_force_raw = true;
+			continue;
+		}
 		force_off = *tok == '!';
 		if (force_off)
 			tok++;
@@ -989,6 +995,26 @@ static __init bool get_rdt_mon_resources(void)
 	return !rdt_get_l3_mon_config(r);
 }
 
+/*
+ * Disable a feature that plain rdt= enable requests must not re-enable.
+ * Only rdt=force lets an explicit enable request override this.
+ */
+static __init void rdt_force_off_hard(int idx)
+{
+	struct rdt_options *o = &rdt_options[idx];
+
+	o->force_off = true;
+	if (!o->force_on)
+		return;
+
+	if (rdt_force_raw) {
+		pr_info("rdt=force: keeping %s enabled on this CPU\n", o->name);
+	} else {
+		pr_info("Ignoring rdt=%s: unsafe on this CPU, see rdt=force\n", o->name);
+		o->force_on = false;
+	}
+}
+
 static __init void __check_quirks_intel(void)
 {
 	switch (boot_cpu_data.x86_vfm) {
@@ -998,7 +1024,10 @@ static __init void __check_quirks_intel(void)
 		break;
 	case INTEL_SKYLAKE_X:
 		if (boot_cpu_data.x86_stepping <= 4)
-			set_rdt_options("!cmt,!mbmtotal,!mbmlocal,!l3cat");
+			set_rdt_options("!cmt,!mbmtotal,!mbmlocal");
+		/* L3 CAT on these steppings can hang a later kexec boot. */
+		if (boot_cpu_data.x86_stepping < 4)
+			rdt_force_off_hard(RDT_FLAG_L3_CAT);
 		else
 			set_rdt_options("!l3cat");
 		fallthrough;

base-commit: e767a4ea70a3992c37ed604157d32f0dfbf9b1e3
-- 
2.47.1

RE: [PATCH] x86/resctrl: Require force for L3 CAT on early Skylake-X

[Luck, Tony]

> When L3 CAT gets enabled with rdt=l3cat on Skylake-X stepping 3,
> consecutive kexec boots hang for me during secondary socket bringup.
> Cold boots on the same machine work.

This is troubling. I thought that the problems on Skylake were simply that in some circumstances
the CLOSID cache mask would not be honored so that data would spill into cache ways that
were supposed to be reserved for other users. I did not expect that the system could hang
with L3 cat enabled.

Your fix seems quite complex.  Perhaps something like this?

	case INTEL_SKYLAKE_X:
		if (boot_cpu_data.x86_stepping == 3)
			setup_clear_cpu_cap(X86_FEATURE_CAT_L3);
		if (boot_cpu_data.x86_stepping <= 4)
			set_rdt_options("!cmt,!mbmtotal,!mbmlocal,!l3cat");
		else
			set_rdt_options("!l3cat");
		fallthrough;

-Tony