[PATCH 4/7] virt: steal_governor: serialize sampling against CPU hotplug

[Yury Norov <ynorov@nvidia.com>]

Steal governor is not safe against a concurrent CPU hotplug operation
because it can change active and preferred cpumasks during the governor
sampling.

To address the race, the governor would take the cpus_write_lock(). But
that may be impractical for the loadable module.

Instead, implement the following locking policy:

 - if the code takes cpus_write_lock(), it's enough to write-access
   the preferred cpumask;
 - if the code takes cpus_read_lock(), it must also take
   cpu_preferred_mutex in order to modify the preferred mask.

The core scheduler and hotplug paths already modify the preferred cpumask
under the cpus_write_lock(), so no change in core logic is required.

This provides a common update protocol for multiple policy drivers
without requiring writers to take the broader CPU hotplug write lock.

Fixes: 27d47ebce4d6 ("virt/steal_governor: Implement steal_governor policy loop")
Signed-off-by: Yury Norov <ynorov@nvidia.com>
---
 drivers/virt/steal_governor.c | 17 +++++++++++++----
 include/linux/cpumask.h       | 11 ++++++++++-
 lib/cpumask.c                 |  5 +++++
 3 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/drivers/virt/steal_governor.c b/drivers/virt/steal_governor.c
index 26b6bcca27ed..8cd5c0f3e3ef 100644
--- a/drivers/virt/steal_governor.c
+++ b/drivers/virt/steal_governor.c
@@ -23,6 +23,7 @@
 #include <linux/ktime.h>
 #include <linux/math64.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/sched/isolation.h>
 #include <linux/topology.h>
 #include <linux/types.h>
@@ -139,7 +140,8 @@ static void decrease_preferred_cpus(void)
 	int target_cpu = nr_cpu_ids;
 	int cpu;
 
-	guard(cpus_read_lock)();
+	lockdep_assert_cpus_held();
+	lockdep_assert_held(&cpu_preferred_mutex);
 	cpu = cpumask_first_and(housekeeping_cpumask(HK_TYPE_KERNEL_NOISE),
 				cpu_preferred_mask);
 	if (cpu >= nr_cpu_ids)
@@ -179,7 +181,8 @@ static void increase_preferred_cpus(void)
 {
 	int first_cpu, cpu;
 
-	guard(cpus_read_lock)();
+	lockdep_assert_cpus_held();
+	lockdep_assert_held(&cpu_preferred_mutex);
 	first_cpu = cpumask_first_andnot(cpu_active_mask, cpu_preferred_mask);
 
 	/* All CPUs are preferred. Nothing to increase further */
@@ -193,6 +196,9 @@ static void increase_preferred_cpus(void)
 
 static bool preferred_cpus_valid(void)
 {
+	lockdep_assert_cpus_held();
+	lockdep_assert_held(&cpu_preferred_mutex);
+
 	if (cpumask_empty(cpu_preferred_mask)) {
 		pr_err("empty preferred mask. stopping\n");
 		return false;
@@ -213,6 +219,9 @@ static void steal_governor_loop(struct work_struct *work)
 	u64 curr_steal, delta_steal, delta_ns, steal_ratio;
 	ktime_t now;
 
+	guard(cpus_read_lock)();
+	guard(mutex)(&cpu_preferred_mutex);
+
 	now = ktime_get();
 	delta_ns = ktime_to_ns(ktime_sub(now, sg_ctx.time));
 
@@ -238,8 +247,6 @@ static void steal_governor_loop(struct work_struct *work)
 	 * module restores to active if CPU hotplug breaks those assumptions.
 	 */
 	if (!preferred_cpus_valid()) {
-		guard(cpus_read_lock)();
-
 		reset_preferred_cpumask();
 		return;
 	}
@@ -276,6 +283,8 @@ static void __exit steal_governor_exit(void)
 {
 	disable_delayed_work_sync(&sg_ctx.work);
 	guard(cpus_read_lock)();
+	guard(mutex)(&cpu_preferred_mutex);
+
 	reset_preferred_cpumask();
 	pr_info("disabled\n");
 }
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index b13d099474bc..3716213d0730 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -124,6 +124,14 @@ extern struct cpumask __cpu_dying_mask;
 
 #ifdef CONFIG_PREFERRED_CPU
 extern struct cpumask __cpu_preferred_mask;
+
+/*
+ * Drivers must take cpus_read_lock(), then cpu_preferred_mutex and
+ * hold both across mask validation and updates. If cpus_write_lock()
+ * is held, it guarantees the exclusive access to the mask, thus
+ * holding cpu_preferred_mutex isn't necessary.
+ */
+extern struct mutex cpu_preferred_mutex;
 #else
 #define __cpu_preferred_mask __cpu_active_mask
 #endif
@@ -1199,7 +1207,8 @@ void init_cpu_possible(const struct cpumask *src);
  * reset_preferred_cpumask - reset preferred CPUs to the active mask.
  *
  * Callers must serialize against CPU hotplug and other preferred cpumask
- * writers.
+ * writers. Requires cpus_read_lock() and cpu_preferred_mutex; or
+ * cpus_write_lock().
  */
 static inline void reset_preferred_cpumask(void)
 {
diff --git a/lib/cpumask.c b/lib/cpumask.c
index 5adb9874fbd0..2e16e19a2bdb 100644
--- a/lib/cpumask.c
+++ b/lib/cpumask.c
@@ -166,3 +166,8 @@ unsigned int cpumask_any_distribute(const struct cpumask *srcp)
 	return next;
 }
 EXPORT_SYMBOL(cpumask_any_distribute);
+
+#ifdef CONFIG_PREFERRED_CPU
+DEFINE_MUTEX(cpu_preferred_mutex);
+EXPORT_SYMBOL_GPL(cpu_preferred_mutex);
+#endif
-- 
2.53.0