[PATCH v7 14/19] fs/resctrl: Deactivate kernel mode when the group is removed

[Babu Moger <babu.moger@amd.com>]

The RESCTRL_GLOBAL_ENABLE_PER_CPU allows CPUs to use different CLOSID
and/or RMID association when transitioning from user mode to kernel mode.
When active, the mode is enabled or disabled on a per-CPU basis as part a
backing resource group.

When the backing group is removed, the configured kernel mode settings are
no longer valid. Deactivate the kernel mode and restore it to
RESCTRL_INHERIT_USER, the default mode in which kernel work inherits the
user task's CLOSID and/or RMID.

Signed-off-by: Babu Moger <babu.moger@amd.com>
---
v7: Removed rdtgroup_kmode_global_detach() and merged everything to
    rdtgroup_kmode_global_deactivate().
    Removed detach from rdtgroup_rmdir_ctrl().
    Updated the changelog and code comments.

v6: Rewrite changelog (not a bugfix).
    Shorten the code comments.

v5: Took care of couple of Sashiko comments on previous patch.
    https://sashiko.dev/#/patchset/cover.1783461016.git.babu.moger@amd.com?part=10
    Re-wrote the changelog.
    Renamed rdtgroup_config_kmode_reset() to rdtgroup_kmode_deactivate().

v4: Re-wrote the changelog.
    Added the call free_all_child_rdtgrp() and rmdir_all_sub()
    Simplified the code comments.

v3: New patch to handle the kernel_mode clean up.
---
 Documentation/filesystems/resctrl.rst |  3 +++
 fs/resctrl/rdtgroup.c                 | 30 +++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/Documentation/filesystems/resctrl.rst b/Documentation/filesystems/resctrl.rst
index 033d405338c0..e57361dd342e 100644
--- a/Documentation/filesystems/resctrl.rst
+++ b/Documentation/filesystems/resctrl.rst
@@ -594,6 +594,9 @@ The top level of the "info" directory contains the following file(s):
 			When "mon=inherit", or monitoring is unsupported, a CTRL_MON group
 			is shown as "<CTRL_MON>/".
 
+			Removing the backing group deactivates the current kernel mode and
+			restores to "inherit_user".
+
 		When the mode is not active, the output lists the supported parameters
 		and their default values, which are used if the mode is activated
 		without new values.
diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c
index af4c929d7ed4..97d99406de00 100644
--- a/fs/resctrl/rdtgroup.c
+++ b/fs/resctrl/rdtgroup.c
@@ -1178,6 +1178,29 @@ static void resctrl_kmode_global_offline_cpu(unsigned int cpu)
 	resctrl_configure_kmode_global(cpumask_of(cpu), rdtgrp, false);
 }
 
+/*
+ * rdtgroup_kmode_global_deactivate() - Deactivate global kernel mode
+ * @rdtgrp: Resctrl group backing the active kernel mode.
+ *
+ * If @rdtgrp backs the active RESCTRL_GLOBAL_ENABLE_PER_CPU mode, disable
+ * the mode, clear kernel mode state, and restore the default
+ * RESCTRL_INHERIT_USER configuration.
+ */
+static void rdtgroup_kmode_global_deactivate(struct rdtgroup *rdtgrp)
+{
+	if (resctrl_kmode.active != RESCTRL_GLOBAL_ENABLE_PER_CPU ||
+	    resctrl_kmode.rdtgrp != rdtgrp)
+		return;
+
+	resctrl_configure_kmode_global(&rdtgrp->kmode_cpu_mask, rdtgrp, false);
+	cpumask_clear(&rdtgrp->kmode_cpu_mask);
+
+	resctrl_kmode.rdtgrp = NULL;
+	resctrl_kmode.active = RESCTRL_INHERIT_USER;
+	resctrl_kmode.ctrl = KMODE_ASSIGN;
+	resctrl_kmode.mon = KMODE_ASSIGN;
+}
+
 void *rdt_kn_parent_priv(struct kernfs_node *kn)
 {
 	/*
@@ -3214,6 +3237,7 @@ static void free_all_child_rdtgrp(struct rdtgroup *rdtgrp)
 
 	head = &rdtgrp->mon.crdtgrp_list;
 	list_for_each_entry_safe(sentry, stmp, head, mon.crdtgrp_list) {
+		rdtgroup_kmode_global_deactivate(sentry);
 		rdtgroup_unassign_cntrs(sentry);
 		free_rmid(sentry->closid, sentry->mon.rmid);
 		list_del(&sentry->mon.crdtgrp_list);
@@ -3251,6 +3275,7 @@ static void rmdir_all_sub(void)
 		cpumask_or(&rdtgroup_default.cpu_mask,
 			   &rdtgroup_default.cpu_mask, &rdtgrp->cpu_mask);
 
+		rdtgroup_kmode_global_deactivate(rdtgrp);
 		rdtgroup_unassign_cntrs(rdtgrp);
 
 		if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP ||
@@ -3346,6 +3371,7 @@ static void resctrl_fs_teardown(void)
 		return;
 
 	rmdir_all_sub();
+	rdtgroup_kmode_global_deactivate(&rdtgroup_default);
 	rdtgroup_unassign_cntrs(&rdtgroup_default);
 	mon_put_kn_priv();
 	rdt_pseudo_lock_release();
@@ -4381,6 +4407,8 @@ static int rdtgroup_rmdir_mon(struct rdtgroup *rdtgrp, cpumask_var_t tmpmask)
 	u32 closid, rmid;
 	int cpu;
 
+	rdtgroup_kmode_global_deactivate(rdtgrp);
+
 	/* Give any tasks back to the parent group */
 	rdt_move_group_tasks(rdtgrp, prdtgrp, tmpmask);
 
@@ -4431,6 +4459,8 @@ static int rdtgroup_rmdir_ctrl(struct rdtgroup *rdtgrp, cpumask_var_t tmpmask)
 	u32 closid, rmid;
 	int cpu;
 
+	rdtgroup_kmode_global_deactivate(rdtgrp);
+
 	/* Give any tasks back to the default group */
 	rdt_move_group_tasks(rdtgrp, &rdtgroup_default, tmpmask);
 
-- 
2.43.0