[PATCH 2/4] sched_ext: Collect CID construction state in a private builder

[Zhang Qiao <zhangqiao22@huawei.com>]

The running counters and scratch masks are about to cross function
boundaries in both directions, which would grow the parameter lists of
every subsequent extraction step. Collect the build-only state instead
- the under-construction tables, the seven scratch masks borrowed from
scx_cid_init() and the running cid and index counters - in a
file-private scx_cid_builder and thread it through the stages
extracted so far.

No behavior change.

Signed-off-by: Zhang Qiao <zhangqiao22@huawei.com>
---
 kernel/sched/ext/cid.c | 188 ++++++++++++++++++++++-------------------
 1 file changed, 103 insertions(+), 85 deletions(-)

diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c
index b6be8806a7d5..ee55f101e5c5 100644
--- a/kernel/sched/ext/cid.c
+++ b/kernel/sched/ext/cid.c
@@ -165,9 +165,22 @@ void scx_cid_retire_tables(void)
 	call_rcu(&tbls->rcu, scx_cid_tables_free_rcufn);
 }
 
-static void scx_cid_build_notopo(struct scx_cid_tables *tbls, u32 shard_size,
-				 u32 *next_cid, s32 *next_shard_idx,
-				 struct cpumask *online_no_topo)
+struct scx_cid_builder {
+	struct scx_cid_tables *tbls;
+	struct cpumask *to_walk;
+	struct cpumask *node_scratch;
+	struct cpumask *llc_scratch;
+	struct cpumask *cluster_scratch;
+	struct cpumask *core_scratch;
+	struct cpumask *llc_fallback;
+	struct cpumask *online_no_topo;
+	u32 next_cid;
+	s32 next_node_idx, next_llc_idx, next_core_idx;
+	s32 next_cluster_idx, next_shard_idx;
+	u32 shard_size, max_cids;
+};
+
+static void scx_cid_build_notopo(struct scx_cid_builder *b)
 {
 	u32 notopo_in_shard;
 	s32 notopo_shard_cid, notopo_shard_idx;
@@ -175,42 +188,41 @@ static void scx_cid_build_notopo(struct scx_cid_tables *tbls, u32 shard_size,
 
 	/*
 	 * No-topo section: any possible cpu without a cid - normally just the
-	 * not-online ones. Pack into shards of up to min(@shard_size,
+	 * not-online ones. Pack into shards of up to min(@b->shard_size,
 	 * SCX_CID_SHARD_MAX_CPUS) cids so that every cid has a valid shard
-	 * assignment and the hard cap holds even with a large @shard_size.
-	 * Collect any currently-online cpus that land here in @online_no_topo
+	 * assignment and the hard cap holds even with a large @b->shard_size.
+	 * Collect any currently-online cpus that land here in @b->online_no_topo
 	 * so we can warn about them at the end.
 	 */
-	notopo_in_shard = min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS);
+	notopo_in_shard = min_t(u32, b->shard_size, SCX_CID_SHARD_MAX_CPUS);
 	notopo_shard_cid = -1;
 	notopo_shard_idx = -1;
 
 	for_each_cpu(cpu, cpu_possible_mask) {
-		if (tbls->cpu_to_cid[cpu] != -1)
+		if (b->tbls->cpu_to_cid[cpu] != -1)
 			continue;
 		if (cpu_online(cpu))
-			cpumask_set_cpu(cpu, online_no_topo);
+			cpumask_set_cpu(cpu, b->online_no_topo);
 
-		cid = (*next_cid)++;
-		tbls->cid_to_cpu[cid] = cpu;
-		tbls->cpu_to_cid[cpu] = cid;
+		cid = b->next_cid++;
+		b->tbls->cid_to_cpu[cid] = cpu;
+		b->tbls->cpu_to_cid[cpu] = cid;
 
-		if (notopo_in_shard >= min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS)) {
+		if (notopo_in_shard >= min_t(u32, b->shard_size, SCX_CID_SHARD_MAX_CPUS)) {
 			notopo_shard_cid = cid;
-			notopo_shard_idx = (*next_shard_idx)++;
+			notopo_shard_idx = b->next_shard_idx++;
 			notopo_in_shard = 0;
 		}
 		notopo_in_shard++;
 
-		tbls->cid_to_shard[cid] = notopo_shard_idx;
-		tbls->topo[cid] = SCX_CID_TOPO_NEG;
-		tbls->topo[cid].shard_cid = notopo_shard_cid;
-		tbls->topo[cid].shard_idx = notopo_shard_idx;
+		b->tbls->cid_to_shard[cid] = notopo_shard_idx;
+		b->tbls->topo[cid] = SCX_CID_TOPO_NEG;
+		b->tbls->topo[cid].shard_cid = notopo_shard_cid;
+		b->tbls->topo[cid].shard_idx = notopo_shard_idx;
 	}
 }
 
-static void scx_cid_finalize_tables(struct scx_cid_tables *tbls, u32 next_cid,
-				    s32 next_shard_idx)
+static void scx_cid_finalize_tables(struct scx_cid_builder *b)
 {
 	s32 cid;
 
@@ -219,15 +231,15 @@ static void scx_cid_finalize_tables(struct scx_cid_tables *tbls, u32 next_cid,
 	 * cid ranges by construction: base_cid is the first cid landing in a
 	 * shard, nr_cids is the count.
 	 */
-	for (cid = 0; cid < next_cid; cid++) {
-		s32 sidx = tbls->cid_to_shard[cid];
+	for (cid = 0; cid < b->next_cid; cid++) {
+		s32 sidx = b->tbls->cid_to_shard[cid];
 
-		if (tbls->shard_ranges[sidx].nr_cids == 0)
-			tbls->shard_ranges[sidx].base_cid = cid;
-		tbls->shard_ranges[sidx].nr_cids++;
+		if (b->tbls->shard_ranges[sidx].nr_cids == 0)
+			b->tbls->shard_ranges[sidx].base_cid = cid;
+		b->tbls->shard_ranges[sidx].nr_cids++;
 	}
 
-	tbls->nr_shards = next_shard_idx;
+	b->tbls->nr_shards = b->next_shard_idx;
 }
 
 /**
@@ -253,11 +265,7 @@ s32 scx_cid_init(struct scx_sched *sch)
 	cpumask_var_t llc_fallback __free(free_cpumask_var) = CPUMASK_VAR_NULL;
 	cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL;
 	struct scx_cid_tables *tbls;
-	u32 next_cid = 0;
-	s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0;
-	s32 next_cluster_idx = 0;
-	s32 next_shard_idx = 0;
-	u32 shard_size, max_cids;
+	struct scx_cid_builder b = {};
 	s32 cpu, si;
 
 	/* CMASK_MAX_WORDS in cid.bpf.h covers NR_CPUS up to 8192 */
@@ -266,8 +274,8 @@ s32 scx_cid_init(struct scx_sched *sch)
 	lockdep_assert_cpus_held();
 	lockdep_assert_held(&scx_enable_mutex);
 
-	shard_size = sch->ops.cid_shard_size ?: SCX_CID_SHARD_SIZE_DFL;
-	max_cids = min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS);
+	b.shard_size = sch->ops.cid_shard_size ?: SCX_CID_SHARD_SIZE_DFL;
+	b.max_cids = min_t(u32, b.shard_size, SCX_CID_SHARD_MAX_CPUS);
 
 	tbls = scx_cid_alloc_tables();
 	if (!tbls)
@@ -287,16 +295,25 @@ s32 scx_cid_init(struct scx_sched *sch)
 	    !zalloc_cpumask_var(&online_no_topo, GFP_KERNEL))
 		return -ENOMEM;
 
+	b.tbls = tbls;
+	b.to_walk = to_walk;
+	b.node_scratch = node_scratch;
+	b.llc_scratch = llc_scratch;
+	b.cluster_scratch = cluster_scratch;
+	b.core_scratch = core_scratch;
+	b.llc_fallback = llc_fallback;
+	b.online_no_topo = online_no_topo;
+
 	/* -1 sentinels for sparse-possible cpu id holes (0 is a valid cid) */
 	for (cpu = 0; cpu < nr_cpu_ids; cpu++)
 		tbls->cpu_to_cid[cpu] = -1;
 
-	cpumask_copy(to_walk, cpu_online_mask);
+	cpumask_copy(b.to_walk, cpu_online_mask);
 
-	while (!cpumask_empty(to_walk)) {
-		s32 next_cpu = cpumask_first(to_walk);
+	while (!cpumask_empty(b.to_walk)) {
+		s32 next_cpu = cpumask_first(b.to_walk);
 		s32 nid = cpu_to_node(next_cpu);
-		s32 node_cid = next_cid;
+		s32 node_cid = b.next_cid;
 		s32 node_idx;
 
 		/*
@@ -304,38 +321,38 @@ s32 scx_cid_init(struct scx_sched *sch)
 		 * cid. cpumask_of_node(-1) is undefined.
 		 */
 		if (nid < 0) {
-			cpumask_clear_cpu(next_cpu, to_walk);
+			cpumask_clear_cpu(next_cpu, b.to_walk);
 			continue;
 		}
 
-		node_idx = next_node_idx++;
+		node_idx = b.next_node_idx++;
 
-		/* node_scratch = to_walk & this node */
-		cpumask_and(node_scratch, to_walk, cpumask_of_node(nid));
-		if (WARN_ON_ONCE(!cpumask_test_cpu(next_cpu, node_scratch)))
+		/* b.node_scratch = b.to_walk & this node */
+		cpumask_and(b.node_scratch, b.to_walk, cpumask_of_node(nid));
+		if (WARN_ON_ONCE(!cpumask_test_cpu(next_cpu, b.node_scratch)))
 			return -EINVAL;
 
-		while (!cpumask_empty(node_scratch)) {
-			s32 ncpu = cpumask_first(node_scratch);
-			const struct cpumask *llc_mask = cpu_llc_mask(ncpu, llc_fallback);
-			s32 llc_cid = next_cid;
-			s32 llc_idx = next_llc_idx++;
+		while (!cpumask_empty(b.node_scratch)) {
+			s32 ncpu = cpumask_first(b.node_scratch);
+			const struct cpumask *llc_mask = cpu_llc_mask(ncpu, b.llc_fallback);
+			s32 llc_cid = b.next_cid;
+			s32 llc_idx = b.next_llc_idx++;
 			u32 cores_per_shard, nr_large;
 			u32 shard_local = 0, cores_in_shard = 0, cids_in_shard = 0;
 			s32 shard_cid, shard_idx;
 			s32 cluster_cid = -1, cluster_idx = -1;
 
-			/* llc_scratch = node_scratch & this llc */
-			cpumask_and(llc_scratch, node_scratch, llc_mask);
-			if (WARN_ON_ONCE(!cpumask_test_cpu(ncpu, llc_scratch)))
+			/* b.llc_scratch = b.node_scratch & this llc */
+			cpumask_and(b.llc_scratch, b.node_scratch, llc_mask);
+			if (WARN_ON_ONCE(!cpumask_test_cpu(ncpu, b.llc_scratch)))
 				return -EINVAL;
 
-			calc_shard_layout(llc_scratch, shard_size, &cores_per_shard, &nr_large);
-			shard_cid = next_cid;
-			shard_idx = next_shard_idx++;
-			tbls->shard_node[shard_idx] = nid;
+			calc_shard_layout(b.llc_scratch, b.shard_size, &cores_per_shard, &nr_large);
+			shard_cid = b.next_cid;
+			shard_idx = b.next_shard_idx++;
+			b.tbls->shard_node[shard_idx] = nid;
 
-			while (!cpumask_empty(llc_scratch)) {
+			while (!cpumask_empty(b.llc_scratch)) {
 				const struct cpumask *sib;
 				s32 core_cid, core_idx, lcpu, ccpu;
 				u32 max_cores, cids_in_core;
@@ -345,24 +362,26 @@ s32 scx_cid_init(struct scx_sched *sch)
 				 * on, so that a cluster is a contiguous cid
 				 * range like the core, LLC and node levels.
 				 */
-				if (cpumask_empty(cluster_scratch)) {
-					s32 xcpu = cpumask_first(llc_scratch);
+				if (cpumask_empty(b.cluster_scratch)) {
+					s32 xcpu = cpumask_first(b.llc_scratch);
 
-					cpumask_or(cluster_scratch, topology_cluster_cpumask(xcpu),
+					cpumask_or(b.cluster_scratch,
+						   topology_cluster_cpumask(xcpu),
 						   topology_sibling_cpumask(xcpu));
-					cpumask_and(cluster_scratch, cluster_scratch, llc_scratch);
-					cluster_cid = next_cid;
-					cluster_idx = next_cluster_idx++;
+					cpumask_and(b.cluster_scratch, b.cluster_scratch,
+						    b.llc_scratch);
+					cluster_cid = b.next_cid;
+					cluster_idx = b.next_cluster_idx++;
 				}
 
-				lcpu = cpumask_first(cluster_scratch);
+				lcpu = cpumask_first(b.cluster_scratch);
 				sib = topology_sibling_cpumask(lcpu);
-				core_cid = next_cid;
-				core_idx = next_core_idx++;
+				core_cid = b.next_cid;
+				core_idx = b.next_core_idx++;
 
-				/* core_scratch = cluster_scratch & this core */
-				cpumask_and(core_scratch, cluster_scratch, sib);
-				if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch)))
+				/* b.core_scratch = b.cluster_scratch & this core */
+				cpumask_and(b.core_scratch, b.cluster_scratch, sib);
+				if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, b.core_scratch)))
 					return -EINVAL;
 
 				/*
@@ -371,28 +390,28 @@ s32 scx_cid_init(struct scx_sched *sch)
 				 * shard sizes under uneven SMT. Never start an
 				 * empty shard.
 				 */
-				cids_in_core = cpumask_weight(core_scratch);
+				cids_in_core = cpumask_weight(b.core_scratch);
 				max_cores = cores_per_shard + (shard_local < nr_large ? 1 : 0);
 				if (cores_in_shard &&
 				    (cores_in_shard >= max_cores ||
-				     cids_in_shard + cids_in_core > max_cids)) {
+				     cids_in_shard + cids_in_core > b.max_cids)) {
 					shard_local++;
 					cores_in_shard = 0;
 					cids_in_shard = 0;
-					shard_cid = next_cid;
-					shard_idx = next_shard_idx++;
-					tbls->shard_node[shard_idx] = nid;
+					shard_cid = b.next_cid;
+					shard_idx = b.next_shard_idx++;
+					b.tbls->shard_node[shard_idx] = nid;
 				}
 				cores_in_shard++;
 				cids_in_shard += cids_in_core;
 
-				for_each_cpu(ccpu, core_scratch) {
-					s32 cid = next_cid++;
+				for_each_cpu(ccpu, b.core_scratch) {
+					s32 cid = b.next_cid++;
 
-					tbls->cid_to_cpu[cid] = ccpu;
-					tbls->cpu_to_cid[ccpu] = cid;
-					tbls->cid_to_shard[cid] = shard_idx;
-					tbls->topo[cid] = (struct scx_cid_topo){
+					b.tbls->cid_to_cpu[cid] = ccpu;
+					b.tbls->cpu_to_cid[ccpu] = cid;
+					b.tbls->cid_to_shard[cid] = shard_idx;
+					b.tbls->topo[cid] = (struct scx_cid_topo){
 						.core_cid = core_cid,
 						.core_idx = core_idx,
 						.llc_cid = llc_cid,
@@ -405,17 +424,16 @@ s32 scx_cid_init(struct scx_sched *sch)
 						.cluster_idx = cluster_idx,
 					};
 
-					cpumask_clear_cpu(ccpu, cluster_scratch);
-					cpumask_clear_cpu(ccpu, llc_scratch);
-					cpumask_clear_cpu(ccpu, node_scratch);
-					cpumask_clear_cpu(ccpu, to_walk);
+					cpumask_clear_cpu(ccpu, b.cluster_scratch);
+					cpumask_clear_cpu(ccpu, b.llc_scratch);
+					cpumask_clear_cpu(ccpu, b.node_scratch);
+					cpumask_clear_cpu(ccpu, b.to_walk);
 				}
 			}
 		}
 	}
 
-	scx_cid_build_notopo(tbls, shard_size, &next_cid, &next_shard_idx,
-			     online_no_topo);
+	scx_cid_build_notopo(&b);
 
 	if (!cpumask_empty(llc_fallback))
 		pr_warn("scx_cid: cpus without cacheinfo, using node mask as llc: %*pbl\n",
@@ -424,7 +442,7 @@ s32 scx_cid_init(struct scx_sched *sch)
 		pr_warn("scx_cid: online cpus with no usable topology: %*pbl\n",
 			cpumask_pr_args(online_no_topo));
 
-	scx_cid_finalize_tables(tbls, next_cid, next_shard_idx);
+	scx_cid_finalize_tables(&b);
 	return 0;
 }
 
-- 
2.43.0