[RFC PATCH 09/19] sched/topology: Introduce percpu sd_nohz for nohz state tracking

[K Prateek Nayak <kprateek.nayak@amd.com>]

Add a per-CPU "sd_nohz" variable to track domains in the local CPU's
hierarchy that have "sd->shared" set for nohz idle tracking.

Convert cpu_set_sd_state_*() to use "sd_nohz" instead of "sd_llc"
increasing the scope of tracking to all CPUs with non-NULL sd hierarchy
attached as opposed to just those CPUs with a "sd_llc" domain.

"sd_nohz" will be used in a subsequent commit to transition
nohz_balancer_kick() to use the distributed nohz idle tracking.

Reviewed-by: Gautham R. Shenoy <gautham.shenoy@amd.com>
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
 kernel/sched/fair.c     |  4 ++--
 kernel/sched/sched.h    |  2 ++
 kernel/sched/topology.c | 22 ++++++++++++++++++++++
 3 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index dee0ded7f40d..624b3753c818 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -12451,7 +12451,7 @@ static void set_cpu_sd_state_busy(int cpu)
 
 	guard(rcu)();
 
-	sd = rcu_dereference(per_cpu(sd_llc, cpu));
+	sd = rcu_dereference(per_cpu(sd_nohz, cpu));
 	if (!sd || !sd->nohz_idle)
 		return;
 
@@ -12481,7 +12481,7 @@ static void set_cpu_sd_state_idle(int cpu)
 
 	guard(rcu)();
 
-	sd = rcu_dereference(per_cpu(sd_llc, cpu));
+	sd = rcu_dereference(per_cpu(sd_nohz, cpu));
 	if (!sd || sd->nohz_idle)
 		return;
 
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 9ae3e2993641..35ffb3926334 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3097,6 +3097,8 @@ extern void cfs_bandwidth_usage_dec(void);
 
 #define nohz_flags(cpu)		(&cpu_rq(cpu)->nohz_flags)
 
+DECLARE_PER_CPU(struct sched_domain __rcu *, sd_nohz);
+
 extern void nohz_balance_exit_idle(struct rq *rq);
 #else /* !CONFIG_NO_HZ_COMMON: */
 static inline void nohz_balance_exit_idle(struct rq *rq) { }
diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index d71c60d99313..f230892528c7 100644
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -466,6 +466,8 @@ struct s_data {
 
 #ifdef CONFIG_NO_HZ_COMMON
 
+DEFINE_PER_CPU(struct sched_domain __rcu *, sd_nohz);
+
 static int __fallback_sds_alloc(struct s_data *d, unsigned long *visited_nodes)
 {
 	int j;
@@ -532,6 +534,23 @@ static void claim_fallback_sds(struct s_data *d)
 	}
 }
 
+static void update_nohz_domain(int cpu)
+{
+	struct sched_domain *sd = highest_flag_domain(cpu, SD_SHARE_LLC);
+
+	/*
+	 * If sd_llc doesn't exist, use the lowest sd for nohz idle
+	 * tracking. cpu_attach_domain() already ensures sd->shared is
+	 * assigned for the lowest domain if sd_llc doesn't exist after
+	 * degeneration of the hierarchy.
+	 */
+	if (!sd)
+		sd = rcu_dereference(cpu_rq(cpu)->sd);
+
+	WARN_ON_ONCE(sd && !sd->shared);
+	rcu_assign_pointer(per_cpu(sd_nohz, cpu), sd);
+}
+
 #else /* !CONFIG_NO_HZ_COMMON */
 
 static inline int __fallback_sds_alloc(struct s_data *d, unsigned long *visited_nodes)
@@ -542,6 +561,7 @@ static inline int __fallback_sds_alloc(struct s_data *d, unsigned long *visited_
 static inline void __fallback_sds_free(struct s_data *d) { }
 static inline void assign_fallback_sds(struct s_data *d, struct sched_domain *sd, int cpu) { }
 static inline void claim_fallback_sds(struct s_data *d) { }
+static inline void update_nohz_domain(int cpu) { }
 
 #endif /* CONFIG_NO_HZ_COMMON */
 
@@ -804,6 +824,8 @@ static void update_top_cache_domain(int cpu)
 
 	sd = lowest_flag_domain(cpu, SD_ASYM_CPUCAPACITY_FULL);
 	rcu_assign_pointer(per_cpu(sd_asym_cpucapacity, cpu), sd);
+
+	update_nohz_domain(cpu);
 }
 
 /*
-- 
2.34.1