[PATCH v4 20/26] sched/deadline: Credit a rebuild to the CPU recorded with the task's bandwidth

[Mathieu Desnoyers <mathieu.desnoyers@efficios.com>]

A rebuild of the root domains' accounting counts each deadline task in
the root domain of the first active CPU of its cpuset,
dl_add_task_root_domain() through dl_get_task_effective_cpus(): a guess
that admission did not make, wrong three ways:

 - the cpuset of a task on a housekeeping CPU can hold CPUs isolated by
   isolcpus="domain" beside it, as the root cpuset does, and without
   cpusets the mask is every CPU the task may run on: an isolated CPU
   coming first has the task counted in the default root domain, which
   underflows when the task leaves the class, or dies;
 - a task confined to isolated CPUs that all go offline, which nothing
   refuses, has no active CPU left, and the rebuild BUG()s;
 - a throttled task whose CPUs went offline is moved, with its
   bandwidth, to an active CPU by dl_task_offline_migration(), and the
   rebuild counts it again where its cpuset says.

Take the CPU recorded with the task's bandwidth where it is active and
the task's mask allows it: the rebuild counts the bandwidth where
admission, or the last move, put it. Otherwise take the first active CPU
of the task's own mask, a housekeeping one where it has any: a change of
partitions sets the masks of its tasks before its rebuild, while the
housekeeping mask that dl_get_task_effective_cpus() reads lags it. The
cpuset answers for a task with no active CPU of its own, as before, and
a task with no active CPU at all is left out: the generation of
"sched/deadline: Keep departing bandwidth across a root domain rebuild"
keeps its release out as well, and dl_task_offline_migration() counts it
where it moves it.

Found by the Sashiko AI reviews of the first three postings of this
series.

Fixes: f9a25f776d78 ("cpusets: Rebuild root domain deadline accounting information")
Cc: stable@vger.kernel.org
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
 kernel/sched/deadline.c | 74 +++++++++++++++++++++++++++++++++++------
 1 file changed, 63 insertions(+), 11 deletions(-)

diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index f169cf35e313..ed951246e3d2 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -878,14 +878,20 @@ static struct rq *dl_task_offline_migration(struct rq *rq, struct task_struct *p
 			__dl_sub(dl_b, p->dl.dl_bw,
 				 cpumask_weight(src_rq->rd->span));
 			raw_spin_unlock(&dl_b->lock);
-
-			dl_b = &later_rq->rd->dl_bw;
-			raw_spin_lock(&dl_b->lock);
-			__dl_add(dl_b, p->dl.dl_bw,
-				 cpumask_weight(later_rq->rd->span));
-			raw_spin_unlock(&dl_b->lock);
 		}
+		/*
+		 * Counted from here in the root domain the task moves to,
+		 * also where a rebuild left it out for want of an active
+		 * CPU: a walk that reaches it after this finds the
+		 * generation current and leaves it alone.
+		 */
+		dl_b = &later_rq->rd->dl_bw;
+		raw_spin_lock(&dl_b->lock);
+		__dl_add(dl_b, p->dl.dl_bw,
+			 cpumask_weight(later_rq->rd->span));
+		raw_spin_unlock(&dl_b->lock);
 		p->dl.dl_bw_cpu = later_rq->cpu;
+		p->dl.dl_bw_gen = dl_bw_gen;
 	}
 
 	set_task_cpu(p, later_rq->cpu);
@@ -3656,6 +3662,47 @@ static void dl_get_task_effective_cpus(struct task_struct *p, struct cpumask *cp
 	cpuset_cpus_allowed_locked(p, cpus);
 }
 
+/*
+ * A CPU of the root domain that a rebuild of the accounting counts @p's
+ * bandwidth in. The one recorded with it, where it is active and the
+ * cpuset of @p allows it: that of the root domain that admitted the
+ * bandwidth, or that the last move put it in, which the cpuset alone does
+ * not tell where it holds CPUs of two root domains, housekeeping ones
+ * beside the ones isolcpus="domain" leaves to the default root domain. The
+ * record answers as well for a task whose own CPUs went offline, moved to
+ * an active one by dl_task_offline_migration(). Else the first active CPU
+ * the cpuset allows, a housekeeping one where it has any; and nr_cpu_ids
+ * where it allows none, which the isolated CPUs allow, no deadline task
+ * of theirs keeping the last of them online.
+ */
+static unsigned int dl_task_rebuild_cpu(struct task_struct *p)
+{
+	struct cpumask *msk = this_cpu_cpumask_var_ptr(local_cpu_mask_dl);
+	int recorded = p->dl.dl_bw_cpu;
+	unsigned int cpu;
+
+	/*
+	 * The task's own CPUs first: a change of cpusets sets the mask of
+	 * each task before the rebuild it makes, where the housekeeping mask
+	 * that dl_get_task_effective_cpus() reads lags that change. With
+	 * none of them active, the CPU recorded is where
+	 * dl_task_offline_migration() moved the task; else its cpuset says.
+	 */
+	cpumask_and(msk, &p->cpus_mask, cpu_active_mask);
+	if (cpumask_empty(msk)) {
+		if (recorded >= 0 && cpu_active(recorded))
+			return recorded;
+		dl_get_task_effective_cpus(p, msk);
+		cpumask_and(msk, msk, cpu_active_mask);
+	}
+	if (recorded >= 0 && cpumask_test_cpu(recorded, msk))
+		return recorded;
+	cpu = cpumask_first_and(msk, housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT));
+	if (cpu < nr_cpu_ids)
+		return cpu;
+	return cpumask_first(msk);
+}
+
 /* From dl_rebuild_rd_accounting(), under the locks it asserts. */
 void dl_add_task_root_domain(struct task_struct *p)
 {
@@ -3663,7 +3710,6 @@ void dl_add_task_root_domain(struct task_struct *p)
 	struct rq *rq;
 	struct dl_bw *dl_b;
 	unsigned int cpu;
-	struct cpumask *msk;
 
 	raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
 	/*
@@ -3677,10 +3723,16 @@ void dl_add_task_root_domain(struct task_struct *p)
 		return;
 	}
 
-	msk = this_cpu_cpumask_var_ptr(local_cpu_mask_dl);
-	dl_get_task_effective_cpus(p, msk);
-	cpu = cpumask_first_and(cpu_active_mask, msk);
-	BUG_ON(cpu >= nr_cpu_ids);
+	cpu = dl_task_rebuild_cpu(p);
+	if (cpu >= nr_cpu_ids) {
+		/*
+		 * No active CPU to count this task's bandwidth: leave it out
+		 * of this rebuild, where dl_bw_gen keeps its release out as
+		 * well, until one has a CPU for it.
+		 */
+		raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags);
+		return;
+	}
 
 	raw_spin_lock(&dl_departed_lock);
 	/*
-- 
2.47.3