[PATCH] sched_ext: Test scx_has_subs() inline before calling sub-sched hooks

[PATCH] sched_ext: Test scx_has_subs() inline before calling sub-sched hooks

[Usama Arif]

scx_reenq_reject(), scx_process_sync_ecaps() and scx_resolve_local_dsq()
return early if there are no sub-schedulers, but clang emits the full
prologue before the static branch. Without sub-schedulers, we still pay
for a call and for saving and restoring up to six registers, on every
wakeup, dispatch and local DSQ insert.

Let's move the scx_has_subs() test into inline wrappers, so that these
calls go away completely. The wrappers keep lockdep_assert_rq_held().

In a pinned perf bench sched pipe test on x86-64 with clang 22, this
reduced kernel instructions by 52-54 per context switch with scx_simple,
scx_lavd and scx_layered.

No functional change intended.

Signed-off-by: Usama Arif <usama.arif@linux.dev>
---
 kernel/sched/ext/sub.c | 25 +++++++++----------------
 kernel/sched/ext/sub.h | 35 +++++++++++++++++++++++++++++++----
 2 files changed, 40 insertions(+), 20 deletions(-)

diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
index 6463cebd88985..10567196be96c 100644
--- a/kernel/sched/ext/sub.c
+++ b/kernel/sched/ext/sub.c
@@ -675,7 +675,7 @@ void scx_rescue_init(struct rq *rq)
 }
 
 /**
- * scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert
+ * __scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert
  * @sch: enqueuing sub-sched
  * @rq: rq whose local DSQ @p targets
  * @p: task being inserted
@@ -695,12 +695,9 @@ void scx_rescue_init(struct rq *rq)
  * to and run by its nearest non-bypassing ancestor. If root is bypassing, it
  * always holds all caps.
  */
-struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
-					     struct task_struct *p, u64 *enq_flags)
+struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
+					       struct task_struct *p, u64 *enq_flags)
 {
-	if (!scx_has_subs())
-		return &rq->scx.local_dsq;
-
 	s32 cid = __scx_cpu_to_cid(cpu_of(rq));
 	struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch;
 	u64 needed = scx_caps_for_enq(*enq_flags);
@@ -783,14 +780,12 @@ bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p)
  * scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT.
  * Rejection can't happen for root.
  */
-void scx_reenq_reject(struct rq *rq)
+void __scx_reenq_reject(struct rq *rq)
 {
 	LIST_HEAD(tasks);
 	struct task_struct *p, *n;
 
-	lockdep_assert_rq_held(rq);
-
-	if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list))
+	if (list_empty(&rq->scx.reject_dsq.list))
 		return;
 
 	/*
@@ -926,7 +921,7 @@ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid)
 	struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
 
 	/*
-	 * Pairs with smp_mb() in scx_process_sync_ecaps(). Either the check
+	 * Pairs with smp_mb() in __scx_process_sync_ecaps(). Either the check
 	 * below sees the node off the list and queues it, or the in-flight sync
 	 * sees the caps[] update made before this call.
 	 */
@@ -951,7 +946,7 @@ static void discard_queued_syncs(struct rq *rq)
 }
 
 /**
- * scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[]
+ * __scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[]
  * @rq: the cid's cpu rq
  * @prev: @rq's previous task from the in-progress dispatch
  *
@@ -963,16 +958,14 @@ static void discard_queued_syncs(struct rq *rq)
  * learns the cid's idle state. Such a gain arms the per-rq
  * %SCX_RQ_SUB_IDLE_RENOTIFY gate so the next idle pick delivers it.
  */
-void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
+void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
 {
 	s32 cpu = cpu_of(rq);
 	s32 cid, shard;
 	struct llist_node *batch, *pos, *tmp;
 	u64 lost_all = 0;
 
-	lockdep_assert_rq_held(rq);
-
-	if (!scx_has_subs() || likely(llist_empty(&rq->scx.ecaps_to_sync)))
+	if (likely(llist_empty(&rq->scx.ecaps_to_sync)))
 		return;
 
 	/*
diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h
index f7bcdfda8dd85..6a7076b9a1ddb 100644
--- a/kernel/sched/ext/sub.h
+++ b/kernel/sched/ext/sub.h
@@ -28,16 +28,16 @@ bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux);
 void scx_free_pshards(struct scx_sched *sch);
 s32 scx_alloc_pshards(struct scx_sched *sch);
 void scx_init_root_caps(struct scx_sched *sch);
-void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev);
+void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev);
 void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch);
 void scx_online_ecaps(struct rq *rq);
 void scx_offline_ecaps(struct rq *rq);
 void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu);
 void scx_discard_stale_ecaps_syncs(void);
-struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
-					     struct task_struct *p, u64 *enq_flags);
+struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
+					       struct task_struct *p, u64 *enq_flags);
 bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p);
-void scx_reenq_reject(struct rq *rq);
+void __scx_reenq_reject(struct rq *rq);
 void scx_rescue_charge(struct rq *rq, s64 delta_exec);
 void scx_rescue_end(struct rq *rq);
 bool scx_rescue_keep(struct rq *rq, struct task_struct *p);
@@ -72,6 +72,33 @@ static inline void scx_dec_has_subs(struct scx_sched *sch)
 		static_branch_dec(&__scx_has_subs);
 }
 
+/* hot-path hooks, gated inline so that a root-only system skips the calls */
+static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
+{
+	lockdep_assert_rq_held(rq);
+
+	if (scx_has_subs())
+		__scx_process_sync_ecaps(rq, prev);
+}
+
+static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch,
+							   struct rq *rq,
+							   struct task_struct *p,
+							   u64 *enq_flags)
+{
+	if (!scx_has_subs())
+		return &rq->scx.local_dsq;
+	return __scx_resolve_local_dsq(sch, rq, p, enq_flags);
+}
+
+static inline void scx_reenq_reject(struct rq *rq)
+{
+	lockdep_assert_rq_held(rq);
+
+	if (scx_has_subs())
+		__scx_reenq_reject(rq);
+}
+
 #else	/* CONFIG_EXT_SUB_SCHED */
 
 static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; }
-- 
2.53.0-Meta

Re: [PATCH] sched_ext: Test scx_has_subs() inline before calling sub-sched hooks

[Tejun Heo]

Applied to sched_ext/for-7.4.

Thanks.

--
tejun