[PATCH v7 13/19] x86,fs/resctrl: Add CPU hotplug support for kernel mode

[Babu Moger <babu.moger@amd.com>]

The RESCTRL_GLOBAL_ENABLE_PER_CPU kernel mode allows the CPU's CLOSID
and/or RMID association to change when transitioning from user mode to
kernel mode. The mode is enabled or disabled on a per-CPU basis through a
backing resource group. CPUs that come online are brought into a state
consistent with the active kernel mode, while CPUs that go offline are
removed from its CPU mask.

Integrate RESCTRL_GLOBAL_ENABLE_PER_CPU mode with CPU hotplug handling so
that CPUs automatically join the active kernel mode when they come online
and are removed from it when they go offline. Also, ensure the register
state remains in a consistent reset state across kexec boot transitions.
This ensures that kernel mode behavior remains consistent across CPU
hotplug events and kexec transitions.

Introduce resctrl_kmode_global_online_cpu() and
resctrl_kmode_global_offline_cpu(), and hook them into the existing
resctrl_online_cpu() and resctrl_offline_cpu() paths. Also update
clear_closid_rmid() to reset MSR_IA32_PQR_PLZA_ASSOC register across kexec
transitions.

Signed-off-by: Babu Moger <babu.moger@amd.com>
---
v7: Changelog update.
    Code comment update.
    Added changes to reset the PLZA register during the bootup.

v6: Update the changelog in non-bugfix form.
    Update the code to remove rdtgroup->kmode.
    Introduce new wrapper resctrl_configure_kmode_global() to consolidate
    architecture calls.

v5: Added resctrl_kmode_offline_cpu() to disable the kernel mode
    association and update kmode_cpu_mask when an associated CPU goes
    offline.
    This addresses sashiko comments:
    https://sashiko.dev/#/patchset/cover.1783461016.git.babu.moger%40amd.com?part=11
    https://sashiko.dev/#/patchset/cover.1783461016.git.babu.moger%40amd.com?part=10
    Also moved the patch earlier than patch 10.
    [RESEND PATCH v4 10/15] fs/resctrl: Reset the kernel mode binding when an rdtgroup is removed
    That seemed much more clear.

v4: New patch in the series. Patch taken from:
    https://lore.kernel.org/lkml/20260611111706.1981788-5-qinyuntan@linux.alibaba.com/
    Updated the code to enable kernel mode by default whenever a CPU comes
    online when one of global-assign mode is enabled.
---
 arch/x86/kernel/cpu/resctrl/core.c |  3 ++
 fs/resctrl/rdtgroup.c              | 56 ++++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+)

diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c
index 14530c11bbd7..d7faf30c2885 100644
--- a/arch/x86/kernel/cpu/resctrl/core.c
+++ b/arch/x86/kernel/cpu/resctrl/core.c
@@ -704,6 +704,9 @@ static void clear_closid_rmid(int cpu)
 	state->cur_closid = RESCTRL_RESERVED_CLOSID;
 	state->cur_rmid = RESCTRL_RESERVED_RMID;
 	wrmsrq(MSR_IA32_PQR_ASSOC, val.q);
+
+	if (boot_cpu_has(X86_FEATURE_PLZA))
+		wrmsrq(MSR_IA32_PQR_PLZA_ASSOC, 0);
 }
 
 static int resctrl_arch_online_cpu(unsigned int cpu)
diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c
index 16f68f4317ef..af4c929d7ed4 100644
--- a/fs/resctrl/rdtgroup.c
+++ b/fs/resctrl/rdtgroup.c
@@ -416,6 +416,17 @@ static int rdtgroup_kmode_cpus_show(struct kernfs_open_file *of,
 	return ret;
 }
 
+static void resctrl_configure_kmode_global(const struct cpumask *mask,
+					   struct rdtgroup *rdtgrp,
+					   bool enable)
+{
+	resctrl_arch_configure_kmode_global(mask, rdtgrp->closid,
+					    resctrl_kmode.ctrl == KMODE_ASSIGN,
+					    rdtgrp->mon.rmid,
+					    resctrl_kmode.mon == KMODE_ASSIGN,
+					    enable);
+}
+
 /*
  * Update the PGR_ASSOC MSR on all cpus in @cpu_mask,
  *
@@ -1125,6 +1136,48 @@ static int resctrl_kernel_mode_show(struct kernfs_open_file *of,
 	return 0;
 }
 
+/*
+ * resctrl_kmode_global_online_cpu() - Program kernel mode association for @cpu
+ * @cpu: CPU that has just been brought online
+ *
+ * When RESCTRL_GLOBAL_ENABLE_PER_CPU is active, add @cpu to the active kernel
+ * mode and program the corresponding CLOSID and/or RMID association.
+ */
+static void resctrl_kmode_global_online_cpu(unsigned int cpu)
+{
+	struct rdtgroup *rdtgrp = resctrl_kmode.rdtgrp;
+
+	if (resctrl_kmode.active != RESCTRL_GLOBAL_ENABLE_PER_CPU || !rdtgrp)
+		return;
+
+	resctrl_configure_kmode_global(cpumask_of(cpu), rdtgrp, true);
+
+	cpumask_set_cpu(cpu, &rdtgrp->kmode_cpu_mask);
+}
+
+/*
+ * resctrl_kmode_global_offline_cpu() - Clear kernel mode association for @cpu
+ * @cpu: CPU being taken offline.
+ *
+ * When RESCTRL_GLOBAL_ENABLE_PER_CPU is active and @cpu is in kmode_cpu_mask,
+ * the association is disabled on @cpu and @cpu is removed from that mask.
+ * A CPU outside the mask is left unchanged.
+ */
+static void resctrl_kmode_global_offline_cpu(unsigned int cpu)
+{
+	struct rdtgroup *rdtgrp = resctrl_kmode.rdtgrp;
+
+	if (resctrl_kmode.active != RESCTRL_GLOBAL_ENABLE_PER_CPU || !rdtgrp)
+		return;
+
+	if (!cpumask_test_cpu(cpu, &rdtgrp->kmode_cpu_mask))
+		return;
+
+	cpumask_clear_cpu(cpu, &rdtgrp->kmode_cpu_mask);
+
+	resctrl_configure_kmode_global(cpumask_of(cpu), rdtgrp, false);
+}
+
 void *rdt_kn_parent_priv(struct kernfs_node *kn)
 {
 	/*
@@ -4882,6 +4935,7 @@ void resctrl_online_cpu(unsigned int cpu)
 	mutex_lock(&rdtgroup_mutex);
 	/* The CPU is set in default rdtgroup after online. */
 	cpumask_set_cpu(cpu, &rdtgroup_default.cpu_mask);
+	resctrl_kmode_global_online_cpu(cpu);
 	mutex_unlock(&rdtgroup_mutex);
 }
 
@@ -4925,6 +4979,8 @@ void resctrl_offline_cpu(unsigned int cpu)
 		}
 	}
 
+	resctrl_kmode_global_offline_cpu(cpu);
+
 	if (!l3->mon_capable)
 		goto out_unlock;
 
-- 
2.43.0