[PATCH] sched/fair: Avoid transient double accounting for delayed tasks

[K Prateek Nayak <kprateek.nayak@amd.com>]

When a task is delayed, h_nr_runnable is adjusted via set_delayed().

A dequeue of delayed task with DEQUEUE_DELAYED would decrement
cfs_rq->h_nr_runnable again in dequeue_hierarchy() and then correct it
immediately in __dequeue_task() via clear_delayed().

Adding:

   WARN_ON_ONCE(((int)cfs_rq->h_nr_runnable) < 0);

in dequeue_hierarchy() after h_nr_runnable accounting is tripped
reliably during early boot.

Although the cfs_rq->h_nr_runnable accounting is transient, this
incorrect value gets cached in se->runnable_weight and can affect PELT
calculation until the next enqueue / dequeue adjusts the runnable
weight.

Similarly, a task enqueue should never reach enqueue_hierarchy()
for just ENQUEUE_WAKEUP && se->sched_delayed. Wakeup of delayed tasks
must couple ENQUEUE_DELAYED which gets correctly handled in
enqueue_task_fair().

Avoid incorrect accounting in __dequeue_task() and warn for
incorrect wakeup flags if delayed tasks reaches enqueue with just
ENQUEUE_WAKEUP.

Fixes: 85570f10a4c6 ("sched/eevdf: Move to a single runqueue")
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
 kernel/sched/fair.c | 61 ++++++++++++++++++++-------------------------
 1 file changed, 27 insertions(+), 34 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index d4cc0caee626..e61b365a7b11 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -6614,19 +6614,8 @@ static void set_delayed(struct sched_entity *se)
 {
 	struct cfs_rq *cfs_rq;
 
-	/*
-	 * Delayed se of cfs_rq have no tasks queued on them.
-	 * Do not adjust h_nr_runnable since __dequeue_task()
-	 * will account it for blocked tasks.
-	 *
-	 * This check can be removed because when flat pick
-	 * patches get merged as only task can get delayed,
-	 * same for clear_delayed().
-	 */
-	if (!entity_is_task(se)) {
-		se->sched_delayed = 1;
-		return;
-	}
+	/* Only tasks can be delayed. */
+	WARN_ON_ONCE(!entity_is_task(se));
 
 	/*
 	 * Drop a task leaving the runnable set.
@@ -6640,20 +6629,18 @@ static void set_delayed(struct sched_entity *se)
 		cfs_rq->h_nr_runnable--;
 }
 
+static void __clear_delayed(struct sched_entity *se)
+{
+	/* Only tasks can be delayed. */
+	WARN_ON_ONCE(!entity_is_task(se));
+	se->sched_delayed = 0;
+}
+
 static void clear_delayed(struct sched_entity *se)
 {
 	struct cfs_rq *cfs_rq;
 
-	se->sched_delayed = 0;
-
-	/*
-	 * Delayed se of cfs_rq have no tasks queued on them.
-	 * Do not adjust h_nr_runnable since a dequeue has
-	 * already accounted for it or an enqueue of a task
-	 * below it will account for it in enqueue_task_fair().
-	 */
-	if (!entity_is_task(se))
-		return;
+	__clear_delayed(se);
 
 	/*
 	 * Re-add on wake, after sched_delayed is cleared. On a final delayed
@@ -8177,13 +8164,16 @@ requeue_delayed_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
 static unsigned long enqueue_hierarchy(struct task_struct *p, int flags)
 {
 	unsigned long weight = NICE_0_LOAD;
-	int task_new = !(flags & ENQUEUE_WAKEUP);
 	struct sched_entity *se = &p->se;
 	int h_nr_idle = task_has_idle_policy(p);
 	int h_nr_runnable = 1;
 	struct cfs_rq *cfs_rq;
 
-	if (task_new && se->sched_delayed)
+	/*
+	 * Accounted via requeue_delayed_entity() calling
+	 * clear_delayed() when delayed task wakes up.
+	 */
+	if (se->sched_delayed)
 		h_nr_runnable = 0;
 
 	for_each_sched_entity(se, cfs_rq) {
@@ -8258,6 +8248,9 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 		return;
 	}
 
+	/* Wakeup of delayed tasks must be handled above. */
+	WARN_ON_ONCE(!task_new && se->sched_delayed);
+
 	/*
 	 * If in_iowait is set, the code below may not trigger any cpufreq
 	 * utilization updates, so do it here explicitly with the IOWAIT flag
@@ -8275,9 +8268,6 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 	if (curr)
 		place_entity(cfs_rq, se, flags);
 
-	if (se->on_rq && se->sched_delayed)
-		requeue_delayed_entity(cfs_rq, se);
-
 	weight = enqueue_hierarchy(p, flags);
 
 	if (!curr) {
@@ -8317,16 +8307,15 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 static void dequeue_hierarchy(struct task_struct *p, int flags)
 {
 	struct sched_entity *se = &p->se;
-	bool task_sleep = flags & DEQUEUE_SLEEP;
-	bool task_delayed = flags & DEQUEUE_DELAYED;
 	bool task_throttled = flags & DEQUEUE_THROTTLE;
 	int h_nr_idle = task_has_idle_policy(p);
-	int h_nr_runnable = 0;
+	int h_nr_runnable = 1;
 	struct cfs_rq *cfs_rq;
 	bool dequeue = true;
 
-	if (task_sleep || task_delayed || !se->sched_delayed)
-		h_nr_runnable = 1;
+	/* Already accounted via set_delayed(). */
+	if (se->sched_delayed)
+		h_nr_runnable = 0;
 
 	for_each_sched_entity(se, cfs_rq) {
 		update_curr(cfs_rq);
@@ -8417,7 +8406,11 @@ static bool __dequeue_task(struct rq *rq, struct task_struct *p, int flags)
 		rq->next_balance = jiffies;
 
 	if (task_delayed) {
-		clear_delayed(se);
+		/*
+		 * Skip accounting since set_delayed() has already
+		 * adjusted it. Just clear se->sched_delayed.
+		 */
+		__clear_delayed(se);
 
 		WARN_ON_ONCE(!task_sleep);
 		WARN_ON_ONCE(p->on_rq != 1);

base-commit: d7e670601af6c564ccf6d3b2ddf70ee4f41a6d1c
-- 
2.34.1