[PATCH 3/5] sched/core: Rank DL server picks as deadline in the core-wide pick

[Fernand Sieber <sieberf@amazon.com>]

Commit 4b26cfdd3956 ("sched/core: Fix priority checking for DL server
picks") made the core-wide selection rank a task picked through a DL
server as deadline, so that a fair task competing with RT tasks under
core scheduling still gets its server's bandwidth. It keys on
p->dl_server, which pick_task() used to set.

Since commit bd9bbc96e835 ("sched: Rework dl_server"), pick_task()
records the server in rq->dl_server and p->dl_server is only set once
the task is switched in. The core-wide selection still ranks each
sibling's pick with prio_less(), so a task the sibling's server has
just picked ranks as a plain fair or ext task and loses to an RT task
on another sibling. The server's CPU then runs a task with the RT
task's cookie or is forced idle, and the fair task gets none of the
server's bandwidth for as long as the RT task runs.

Rank each sibling's pick by the server it came through, rq_i->dl_server.
prio_less() keeps using p->dl_server for the core tree, whose tasks are
not being picked.

Fixes: bd9bbc96e835 ("sched: Rework dl_server")
Signed-off-by: Fernand Sieber <sieberf@amazon.com>
---
 kernel/sched/core.c | 50 ++++++++++++++++++++++++++++++---------------
 1 file changed, 34 insertions(+), 16 deletions(-)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 61d4e6e990bba..ffe8a41380d0e 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -196,12 +196,13 @@ __read_mostly int scheduler_running;
 DEFINE_STATIC_KEY_FALSE(__sched_core_enabled);
 
 /* kernel prio, less is more */
-static inline int __task_prio(const struct task_struct *p)
+static inline int __task_prio(const struct task_struct *p,
+			      const struct sched_dl_entity *dl_server)
 {
 	if (p->sched_class == &stop_sched_class) /* trumps deadline */
 		return -2;
 
-	if (p->dl_server)
+	if (dl_server)
 		return -1; /* deadline */
 
 	if (rt_or_dl_prio(p->prio))
@@ -223,12 +224,19 @@ static inline int __task_prio(const struct task_struct *p)
  * ge(a,b) := !l(a,b)
  */
 
-/* real prio, less is less */
-static inline bool prio_less(const struct task_struct *a,
-			     const struct task_struct *b, bool in_fi)
+/*
+ * real prio, less is less
+ *
+ * @a_dl_server and @b_dl_server are the DL servers @a and @b run on
+ * behalf of, if any.
+ */
+static inline bool __prio_less(const struct task_struct *a,
+			       const struct sched_dl_entity *a_dl_server,
+			       const struct task_struct *b,
+			       const struct sched_dl_entity *b_dl_server,
+			       bool in_fi)
 {
-
-	int pa = __task_prio(a), pb = __task_prio(b);
+	int pa = __task_prio(a, a_dl_server), pb = __task_prio(b, b_dl_server);
 
 	if (-pa < -pb)
 		return true;
@@ -239,18 +247,13 @@ static inline bool prio_less(const struct task_struct *a,
 	if (pa == -1) { /* dl_prio() doesn't work because of stop_class above */
 		const struct sched_dl_entity *a_dl, *b_dl;
 
-		a_dl = &a->dl;
 		/*
-		 * Since,'a' and 'b' can be CFS tasks served by DL server,
+		 * Since 'a' and 'b' can be tasks served by a DL server,
 		 * __task_prio() can return -1 (for DL) even for those. In that
 		 * case, get to the dl_server's DL entity.
 		 */
-		if (a->dl_server)
-			a_dl = a->dl_server;
-
-		b_dl = &b->dl;
-		if (b->dl_server)
-			b_dl = b->dl_server;
+		a_dl = a_dl_server ?: &a->dl;
+		b_dl = b_dl_server ?: &b->dl;
 
 		return !dl_time_before(a_dl->deadline, b_dl->deadline);
 	}
@@ -266,6 +269,12 @@ static inline bool prio_less(const struct task_struct *a,
 	return false;
 }
 
+static inline bool prio_less(const struct task_struct *a,
+			     const struct task_struct *b, bool in_fi)
+{
+	return __prio_less(a, a->dl_server, b, b->dl_server, in_fi);
+}
+
 static inline bool __sched_core_less(const struct task_struct *a,
 				     const struct task_struct *b)
 {
@@ -6297,6 +6306,7 @@ pick_next_task(struct rq *rq, struct rq_flags *rf)
 	__must_hold(__rq_lockp(rq))
 {
 	struct task_struct *next, *p, *max;
+	struct sched_dl_entity *max_dl_server;
 	bool fi_before = false;
 	bool core_clock_updated = (rq == rq->core);
 	unsigned long cookie;
@@ -6410,6 +6420,7 @@ pick_next_task(struct rq *rq, struct rq_flags *rf)
 	 * Tie-break prio towards the current CPU
 	 */
 	max = NULL;
+	max_dl_server = NULL;
 	for_each_core_cpu_wrap(i, rq) {
 		rq_i = cpu_rq(i);
 
@@ -6432,8 +6443,15 @@ pick_next_task(struct rq *rq, struct rq_flags *rf)
 
 		rq_i->core_pick = p;
 
-		if (!max || prio_less(max, p, fi_before))
+		/*
+		 * p->dl_server is only set once @p runs; rank a pick that
+		 * came through a DL server by that server.
+		 */
+		if (!max || __prio_less(max, max_dl_server, p, rq_i->dl_server,
+					fi_before)) {
 			max = p;
+			max_dl_server = rq_i->dl_server;
+		}
 	}
 
 	cookie = rq->core->core_cookie = max->core_cookie;
-- 
2.47.3




Amazon Development Centre (South Africa) (Proprietary) Limited
29 Gogosoa Street, Observatory, Cape Town, Western Cape, 7925, South Africa
Registration Number: 2004 / 034463 / 07