[PATCH 3/4] sched/fair: Add cookie checks on wake idle path

[Fernand Sieber <sieberf@amazon.com>]

The wake_affine_idle() function determines whether the previous CPU or the
waking CPU are suitable for running a waking task. Currently it does not
consider core scheduling constraints.

Add cookie compatibility checks to prevent considering a CPU idle when
placing the task there would immediately cause force idle due to an
incompatible sibling task. This reduces unnecessary force idle scenarios
in the wake-up path.

Signed-off-by: Fernand Sieber <sieberf@amazon.com>
---
 kernel/sched/fair.c  | 19 +++++++++++++------
 kernel/sched/sched.h | 33 ++++++++++++++++++++++++++-------
 2 files changed, 39 insertions(+), 13 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 67746899809e..78b36225a039 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -7235,7 +7235,7 @@ static int wake_wide(struct task_struct *p)
  *			  for the overloaded case.
  */
 static int
-wake_affine_idle(int this_cpu, int prev_cpu, int sync)
+wake_affine_idle(struct task_struct *p, int this_cpu, int prev_cpu, int sync)
 {
 	/*
 	 * If this_cpu is idle, it implies the wakeup is from interrupt
@@ -7249,17 +7249,24 @@ wake_affine_idle(int this_cpu, int prev_cpu, int sync)
 	 * a cpufreq perspective, it's better to have higher utilisation
 	 * on one CPU.
 	 */
-	if (available_idle_cpu(this_cpu) && cpus_share_cache(this_cpu, prev_cpu))
-		return available_idle_cpu(prev_cpu) ? prev_cpu : this_cpu;
+	if (available_idle_cpu(this_cpu) &&
+		cpus_share_cache(this_cpu, prev_cpu) &&
+		sched_core_cookie_match(cpu_rq(this_cpu), p)) {
+		return available_idle_cpu(prev_cpu) &&
+			sched_core_cookie_match(cpu_rq(prev_cpu), p) ?
+			prev_cpu : this_cpu;
+	}
 
 	if (sync) {
 		struct rq *rq = cpu_rq(this_cpu);
 
-		if ((rq->nr_running - cfs_h_nr_delayed(rq)) == 1)
+		if (((rq->nr_running - cfs_h_nr_delayed(rq)) == 1) &&
+		    sched_core_cookie_match_sync(rq, p))
 			return this_cpu;
 	}
 
-	if (available_idle_cpu(prev_cpu))
+	if (available_idle_cpu(prev_cpu) &&
+		sched_core_cookie_match(cpu_rq(prev_cpu), p))
 		return prev_cpu;
 
 	return nr_cpumask_bits;
@@ -7314,7 +7321,7 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p,
 	int target = nr_cpumask_bits;
 
 	if (sched_feat(WA_IDLE))
-		target = wake_affine_idle(this_cpu, prev_cpu, sync);
+		target = wake_affine_idle(p, this_cpu, prev_cpu, sync);
 
 	if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits)
 		target = wake_affine_weight(sd, p, this_cpu, prev_cpu, sync);
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 4e7080123a4c..97cc8c66519e 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1386,27 +1386,41 @@ extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_f
  * A special case is that the task's cookie always matches with CPU's core
  * cookie if the CPU is in an idle core.
  */
-static inline bool sched_core_cookie_match(struct rq *rq, struct task_struct *p)
+static inline bool __sched_core_cookie_match(struct rq *rq,
+					     struct task_struct *p,
+					     bool sync)
 {
-	bool idle_core = true;
 	int cpu;
 
 	/* Ignore cookie match if core scheduler is not enabled on the CPU. */
 	if (!sched_core_enabled(rq))
 		return true;
 
+	if (rq->core->core_cookie == p->core_cookie)
+		return true;
+
 	for_each_cpu(cpu, cpu_smt_mask(cpu_of(rq))) {
-		if (!available_idle_cpu(cpu)) {
-			idle_core = false;
-			break;
-		}
+		if (sync && cpu_of(rq) == cpu)
+			continue;
+		if (!available_idle_cpu(cpu))
+			return false;
 	}
 
 	/*
 	 * A CPU in an idle core is always the best choice for tasks with
 	 * cookies.
 	 */
-	return idle_core || rq->core->core_cookie == p->core_cookie;
+	return true;
+}
+
+static inline bool sched_core_cookie_match(struct rq *rq, struct task_struct *p)
+{
+	return __sched_core_cookie_match(rq, p, false);
+}
+
+static inline bool sched_core_cookie_match_sync(struct rq *rq, struct task_struct *p)
+{
+	return __sched_core_cookie_match(rq, p, true);
 }
 
 static inline bool sched_group_cookie_match(struct rq *rq,
@@ -1464,6 +1478,11 @@ static inline bool sched_core_cookie_match(struct rq *rq, struct task_struct *p)
 	return true;
 }
 
+static inline bool sched_core_cookie_match_sync(struct rq *rq, struct task_struct *p)
+{
+	return true;
+}
+
 static inline bool sched_group_cookie_match(struct rq *rq,
 					    struct task_struct *p,
 					    struct sched_group *group)
-- 
2.43.0




Amazon Development Centre (South Africa) (Proprietary) Limited
29 Gogosoa Street, Observatory, Cape Town, Western Cape, 7925, South Africa
Registration Number: 2004 / 034463 / 07