[RFC PATCH 05/19] sched/topology: Attach new hierarchy in rq_attach_root()

[K Prateek Nayak <kprateek.nayak@amd.com>]

Gregory Haskins introduced the concept of root domain in commit
57d885fea0da ("sched: add sched-domain roots") which put
rq_attach_root() before rcu_assign_pointer(rq->sd, sd).

Further development around dynamic sched domain allocation, and
update_top_cache_domain() that came later relied on the new sched
domain hierarchy and was naturaturally done after rq->sd was updated.

Since subsequent changes around nohz idle CPU tracking creates a
dependency for the fair_sched_class->offline() callback to use the old
hierarchy and the fair_sched_class->online() callback to use the
updated hierarchy, update rq->sd and the top cache domains within
rq_attach_root() between the two callbacks.

This modifies the flow as follows:

       old flow                              new flow
       ========                              ========

  rq_attach_root()                   rq_attach_root()
    set_rq_offline();                  set_rq_offline(); /* Operate on old sd_llc */
    set_rq_online();                   rq->sd = sd;
  rq->sd = sd;                         update_top_cache_domain()
  update_top_cache_domain()            set_rq_online(); /* Operate on new sd_llc */

No functional changes intended.

Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
 kernel/sched/core.c     |  2 +-
 kernel/sched/sched.h    |  2 +-
 kernel/sched/topology.c | 38 ++++++++++++++++++++++++--------------
 3 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index feb750aae71b..771fc6239386 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -8798,7 +8798,7 @@ void __init sched_init(void)
 
 		INIT_LIST_HEAD(&rq->cfs_tasks);
 
-		rq_attach_root(rq, &def_root_domain);
+		rq_attach_root(rq, &def_root_domain, NULL);
 #ifdef CONFIG_NO_HZ_COMMON
 		rq->last_blocked_load_update_tick = jiffies;
 		atomic_set(&rq->nohz_flags, 0);
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index b5367c514c14..9ae3e2993641 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1020,7 +1020,7 @@ struct root_domain {
 
 extern void init_defrootdomain(void);
 extern int sched_init_domains(const struct cpumask *cpu_map);
-extern void rq_attach_root(struct rq *rq, struct root_domain *rd);
+extern void rq_attach_root(struct rq *rq, struct root_domain *rd, struct sched_domain *sd);
 extern void sched_get_rd(struct root_domain *rd);
 extern void sched_put_rd(struct root_domain *rd);
 
diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index f3ebb9e8fe00..a059641e12e5 100644
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -469,9 +469,12 @@ static void free_rootdomain(struct rcu_head *rcu)
 	kfree(rd);
 }
 
-void rq_attach_root(struct rq *rq, struct root_domain *rd)
+static void update_top_cache_domain(int cpu);
+
+void rq_attach_root(struct rq *rq, struct root_domain *rd, struct sched_domain *sd)
 {
 	struct root_domain *old_rd = NULL;
+	int cpu = cpu_of(rq);
 	struct rq_flags rf;
 
 	rq_lock_irqsave(rq, &rf);
@@ -494,6 +497,8 @@ void rq_attach_root(struct rq *rq, struct root_domain *rd)
 	}
 
 	atomic_inc(&rd->refcount);
+	rcu_assign_pointer(rq->sd, sd);
+	update_top_cache_domain(cpu);
 	rq->rd = rd;
 
 	cpumask_set_cpu(rq->cpu, rd->span);
@@ -711,16 +716,27 @@ static void update_top_cache_domain(int cpu)
 	rcu_assign_pointer(per_cpu(sd_asym_cpucapacity, cpu), sd);
 }
 
+struct s_data {
+	struct sched_domain_shared * __percpu *sds;
+	struct sched_domain * __percpu *sd;
+	struct root_domain	*rd;
+};
+
 /*
  * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
  * hold the hotplug lock.
  */
 static void
-cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
+cpu_attach_domain(struct s_data *d, int cpu)
 {
+	struct root_domain *rd = d->rd;
+	struct sched_domain *sd = NULL;
 	struct rq *rq = cpu_rq(cpu);
 	struct sched_domain *tmp;
 
+	if (d->sd)
+		sd = *per_cpu_ptr(d->sd, cpu);
+
 	/* Remove the sched domains which do not contribute to scheduling. */
 	for (tmp = sd; tmp; ) {
 		struct sched_domain *parent = tmp->parent;
@@ -776,21 +792,13 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
 
 	sched_domain_debug(sd, cpu);
 
-	rq_attach_root(rq, rd);
 	tmp = rq->sd;
-	rcu_assign_pointer(rq->sd, sd);
+	rq_attach_root(rq, rd, sd);
 	dirty_sched_domain_sysctl(cpu);
 	destroy_sched_domains(tmp);
 
-	update_top_cache_domain(cpu);
 }
 
-struct s_data {
-	struct sched_domain_shared * __percpu *sds;
-	struct sched_domain * __percpu *sd;
-	struct root_domain	*rd;
-};
-
 enum s_alloc {
 	sa_rootdomain,
 	sa_sd,
@@ -2642,9 +2650,8 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
 	rcu_read_lock();
 	for_each_cpu(i, cpu_map) {
 		rq = cpu_rq(i);
-		sd = *per_cpu_ptr(d.sd, i);
 
-		cpu_attach_domain(sd, d.rd, i);
+		cpu_attach_domain(&d, i);
 
 		if (lowest_flag_domain(i, SD_CLUSTER))
 			has_cluster = true;
@@ -2749,6 +2756,9 @@ int __init sched_init_domains(const struct cpumask *cpu_map)
 static void detach_destroy_domains(const struct cpumask *cpu_map)
 {
 	unsigned int cpu = cpumask_any(cpu_map);
+	struct s_data d = {
+		.rd = &def_root_domain,
+	};
 	int i;
 
 	if (rcu_access_pointer(per_cpu(sd_asym_cpucapacity, cpu)))
@@ -2759,7 +2769,7 @@ static void detach_destroy_domains(const struct cpumask *cpu_map)
 
 	rcu_read_lock();
 	for_each_cpu(i, cpu_map)
-		cpu_attach_domain(NULL, &def_root_domain, i);
+		cpu_attach_domain(&d, i);
 	rcu_read_unlock();
 }
 
-- 
2.34.1