[PATCH 3/4] sched_ext: Extract construction of a single LLC CID range

[Zhang Qiao <zhangqiao22@huawei.com>]

Extract the per-LLC walk into scx_cid_build_llc(), which consumes one
LLC worth of cpus from b->node_scratch and appends their cores,
clusters and shards. All per-LLC state - the llc cid and index, the
shard layout counters and the cluster carry - is local to the helper;
cross-LLC state stays in the builder.

The two WARN_ON_ONCE checks now return -EINVAL to the caller instead
of returning straight out of scx_cid_init(); the value and its effect
on the error path are unchanged.

Signed-off-by: Zhang Qiao <zhangqiao22@huawei.com>
---
 kernel/sched/ext/cid.c | 206 ++++++++++++++++++++++-------------------
 1 file changed, 109 insertions(+), 97 deletions(-)

diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c
index ee55f101e5c5..54eacdc23d32 100644
--- a/kernel/sched/ext/cid.c
+++ b/kernel/sched/ext/cid.c
@@ -180,6 +180,110 @@ struct scx_cid_builder {
 	u32 shard_size, max_cids;
 };
 
+static s32 scx_cid_build_llc(struct scx_cid_builder *b, s32 nid,
+			     s32 node_cid, s32 node_idx)
+{
+	s32 ncpu = cpumask_first(b->node_scratch);
+	const struct cpumask *llc_mask = cpu_llc_mask(ncpu, b->llc_fallback);
+	s32 llc_cid = b->next_cid;
+	s32 llc_idx = b->next_llc_idx++;
+	u32 cores_per_shard, nr_large;
+	u32 shard_local = 0, cores_in_shard = 0, cids_in_shard = 0;
+	s32 shard_cid, shard_idx;
+	s32 cluster_cid = -1, cluster_idx = -1;
+
+	/* b->llc_scratch = b->node_scratch & this llc */
+	cpumask_and(b->llc_scratch, b->node_scratch, llc_mask);
+	if (WARN_ON_ONCE(!cpumask_test_cpu(ncpu, b->llc_scratch)))
+		return -EINVAL;
+
+	calc_shard_layout(b->llc_scratch, b->shard_size, &cores_per_shard, &nr_large);
+	shard_cid = b->next_cid;
+	shard_idx = b->next_shard_idx++;
+	b->tbls->shard_node[shard_idx] = nid;
+
+	while (!cpumask_empty(b->llc_scratch)) {
+		const struct cpumask *sib;
+		s32 core_cid, core_idx, lcpu, ccpu;
+		u32 max_cores, cids_in_core;
+
+		/*
+		 * Take the cores of one cluster before moving
+		 * on, so that a cluster is a contiguous cid
+		 * range like the core, LLC and node levels.
+		 */
+		if (cpumask_empty(b->cluster_scratch)) {
+			s32 xcpu = cpumask_first(b->llc_scratch);
+
+			cpumask_or(b->cluster_scratch,
+				   topology_cluster_cpumask(xcpu),
+				   topology_sibling_cpumask(xcpu));
+			cpumask_and(b->cluster_scratch, b->cluster_scratch,
+				    b->llc_scratch);
+			cluster_cid = b->next_cid;
+			cluster_idx = b->next_cluster_idx++;
+		}
+
+		lcpu = cpumask_first(b->cluster_scratch);
+		sib = topology_sibling_cpumask(lcpu);
+		core_cid = b->next_cid;
+		core_idx = b->next_core_idx++;
+
+		/* b->core_scratch = b->cluster_scratch & this core */
+		cpumask_and(b->core_scratch, b->cluster_scratch, sib);
+		if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, b->core_scratch)))
+			return -EINVAL;
+
+		/*
+		 * Advance to a new shard when either core or
+		 * cid count reaches max. The latter bounds
+		 * shard sizes under uneven SMT. Never start an
+		 * empty shard.
+		 */
+		cids_in_core = cpumask_weight(b->core_scratch);
+		max_cores = cores_per_shard + (shard_local < nr_large ? 1 : 0);
+		if (cores_in_shard &&
+		    (cores_in_shard >= max_cores ||
+		     cids_in_shard + cids_in_core > b->max_cids)) {
+			shard_local++;
+			cores_in_shard = 0;
+			cids_in_shard = 0;
+			shard_cid = b->next_cid;
+			shard_idx = b->next_shard_idx++;
+			b->tbls->shard_node[shard_idx] = nid;
+		}
+		cores_in_shard++;
+		cids_in_shard += cids_in_core;
+
+		for_each_cpu(ccpu, b->core_scratch) {
+			s32 cid = b->next_cid++;
+
+			b->tbls->cid_to_cpu[cid] = ccpu;
+			b->tbls->cpu_to_cid[ccpu] = cid;
+			b->tbls->cid_to_shard[cid] = shard_idx;
+			b->tbls->topo[cid] = (struct scx_cid_topo){
+				.core_cid = core_cid,
+				.core_idx = core_idx,
+				.llc_cid = llc_cid,
+				.llc_idx = llc_idx,
+				.node_cid = node_cid,
+				.node_idx = node_idx,
+				.shard_cid = shard_cid,
+				.shard_idx = shard_idx,
+				.cluster_cid = cluster_cid,
+				.cluster_idx = cluster_idx,
+			};
+
+			cpumask_clear_cpu(ccpu, b->cluster_scratch);
+			cpumask_clear_cpu(ccpu, b->llc_scratch);
+			cpumask_clear_cpu(ccpu, b->node_scratch);
+			cpumask_clear_cpu(ccpu, b->to_walk);
+		}
+	}
+
+	return 0;
+}
+
 static void scx_cid_build_notopo(struct scx_cid_builder *b)
 {
 	u32 notopo_in_shard;
@@ -333,103 +437,11 @@ s32 scx_cid_init(struct scx_sched *sch)
 			return -EINVAL;
 
 		while (!cpumask_empty(b.node_scratch)) {
-			s32 ncpu = cpumask_first(b.node_scratch);
-			const struct cpumask *llc_mask = cpu_llc_mask(ncpu, b.llc_fallback);
-			s32 llc_cid = b.next_cid;
-			s32 llc_idx = b.next_llc_idx++;
-			u32 cores_per_shard, nr_large;
-			u32 shard_local = 0, cores_in_shard = 0, cids_in_shard = 0;
-			s32 shard_cid, shard_idx;
-			s32 cluster_cid = -1, cluster_idx = -1;
-
-			/* b.llc_scratch = b.node_scratch & this llc */
-			cpumask_and(b.llc_scratch, b.node_scratch, llc_mask);
-			if (WARN_ON_ONCE(!cpumask_test_cpu(ncpu, b.llc_scratch)))
-				return -EINVAL;
-
-			calc_shard_layout(b.llc_scratch, b.shard_size, &cores_per_shard, &nr_large);
-			shard_cid = b.next_cid;
-			shard_idx = b.next_shard_idx++;
-			b.tbls->shard_node[shard_idx] = nid;
-
-			while (!cpumask_empty(b.llc_scratch)) {
-				const struct cpumask *sib;
-				s32 core_cid, core_idx, lcpu, ccpu;
-				u32 max_cores, cids_in_core;
-
-				/*
-				 * Take the cores of one cluster before moving
-				 * on, so that a cluster is a contiguous cid
-				 * range like the core, LLC and node levels.
-				 */
-				if (cpumask_empty(b.cluster_scratch)) {
-					s32 xcpu = cpumask_first(b.llc_scratch);
-
-					cpumask_or(b.cluster_scratch,
-						   topology_cluster_cpumask(xcpu),
-						   topology_sibling_cpumask(xcpu));
-					cpumask_and(b.cluster_scratch, b.cluster_scratch,
-						    b.llc_scratch);
-					cluster_cid = b.next_cid;
-					cluster_idx = b.next_cluster_idx++;
-				}
-
-				lcpu = cpumask_first(b.cluster_scratch);
-				sib = topology_sibling_cpumask(lcpu);
-				core_cid = b.next_cid;
-				core_idx = b.next_core_idx++;
-
-				/* b.core_scratch = b.cluster_scratch & this core */
-				cpumask_and(b.core_scratch, b.cluster_scratch, sib);
-				if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, b.core_scratch)))
-					return -EINVAL;
-
-				/*
-				 * Advance to a new shard when either core or
-				 * cid count reaches max. The latter bounds
-				 * shard sizes under uneven SMT. Never start an
-				 * empty shard.
-				 */
-				cids_in_core = cpumask_weight(b.core_scratch);
-				max_cores = cores_per_shard + (shard_local < nr_large ? 1 : 0);
-				if (cores_in_shard &&
-				    (cores_in_shard >= max_cores ||
-				     cids_in_shard + cids_in_core > b.max_cids)) {
-					shard_local++;
-					cores_in_shard = 0;
-					cids_in_shard = 0;
-					shard_cid = b.next_cid;
-					shard_idx = b.next_shard_idx++;
-					b.tbls->shard_node[shard_idx] = nid;
-				}
-				cores_in_shard++;
-				cids_in_shard += cids_in_core;
-
-				for_each_cpu(ccpu, b.core_scratch) {
-					s32 cid = b.next_cid++;
-
-					b.tbls->cid_to_cpu[cid] = ccpu;
-					b.tbls->cpu_to_cid[ccpu] = cid;
-					b.tbls->cid_to_shard[cid] = shard_idx;
-					b.tbls->topo[cid] = (struct scx_cid_topo){
-						.core_cid = core_cid,
-						.core_idx = core_idx,
-						.llc_cid = llc_cid,
-						.llc_idx = llc_idx,
-						.node_cid = node_cid,
-						.node_idx = node_idx,
-						.shard_cid = shard_cid,
-						.shard_idx = shard_idx,
-						.cluster_cid = cluster_cid,
-						.cluster_idx = cluster_idx,
-					};
-
-					cpumask_clear_cpu(ccpu, b.cluster_scratch);
-					cpumask_clear_cpu(ccpu, b.llc_scratch);
-					cpumask_clear_cpu(ccpu, b.node_scratch);
-					cpumask_clear_cpu(ccpu, b.to_walk);
-				}
-			}
+			s32 ret;
+
+			ret = scx_cid_build_llc(&b, nid, node_cid, node_idx);
+			if (ret)
+				return ret;
 		}
 	}
 
-- 
2.43.0