* [PATCH 1/6] sched_ext: Factor out cid_pshard()
2026-10-17 16:45 [PATCHSET v2 sched_ext/for-7.4] sched_ext: Report a sub-scheduler's ecaps on attach, detach and cpu hotplug Tejun Heo
@ 2026-10-17 16:45 ` Tejun Heo
2026-10-17 16:45 ` [PATCH 2/6] sched_ext: Free a sub-scheduler a grace period after discarding its ecaps syncs Tejun Heo
` (4 subsequent siblings)
5 siblings, 0 replies; 11+ messages in thread
From: Tejun Heo @ 2026-10-17 16:45 UTC (permalink / raw)
To: sched-ext
Cc: David Vernet, Andrea Righi, Changwoo Min, Emil Tsalapatis,
David Dai, linux-kernel, Tejun Heo
Factor the cid to pshard lookup out of the ecaps sync paths into
cid_pshard() so that new sync paths can share it. No functional change.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
kernel/sched/ext/sub.c | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)
diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
index 1345e4174775..7edad2ddaa54 100644
--- a/kernel/sched/ext/sub.c
+++ b/kernel/sched/ext/sub.c
@@ -1063,6 +1063,12 @@ static void discard_queued_syncs(struct rq *rq)
init_llist_node(pos);
}
+/* the pshard holding @sch's caps on @cid */
+static struct scx_pshard *cid_pshard(struct scx_sched *sch, s32 cid)
+{
+ return sch->pshard[rcu_dereference_all(scx_cid_to_shard)[cid]];
+}
+
/**
* __scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[]
* @rq: the cid's cpu rq
@@ -1079,7 +1085,7 @@ static void discard_queued_syncs(struct rq *rq)
void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
{
s32 cpu = cpu_of(rq);
- s32 cid, shard;
+ s32 cid;
struct llist_node *batch, *pos, *tmp;
u64 lost_all = 0;
@@ -1101,13 +1107,12 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
/* @cid is valid here: the cpu is active with queued syncs */
cid = __scx_cpu_to_cid(cpu);
- shard = rcu_dereference_all(scx_cid_to_shard)[cid];
batch = llist_del_all(&rq->scx.ecaps_to_sync);
llist_for_each_safe(pos, tmp, batch) {
struct scx_sched_pcpu *pcpu =
container_of(pos, struct scx_sched_pcpu, ecaps_to_sync_node);
- struct scx_pshard *ps = pcpu->sch->pshard[shard];
+ struct scx_pshard *ps = cid_pshard(pcpu->sch, cid);
u64 old, ecaps, lost, gained;
init_llist_node(pos);
@@ -1229,7 +1234,7 @@ void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch)
void scx_online_ecaps(struct rq *rq)
{
struct scx_sched *root, *pos;
- s32 cid, shard;
+ s32 cid;
/*
* Only a live hierarchy can have ecaps to reseed. This also keeps the
@@ -1244,7 +1249,6 @@ void scx_online_ecaps(struct rq *rq)
root = scx_root_protected();
cid = __scx_cpu_to_cid(cpu_of(rq));
- shard = rcu_dereference_all(scx_cid_to_shard)[cid];
scx_for_each_descendant_pre(pos, root) {
struct scx_pshard *ps;
@@ -1253,7 +1257,7 @@ void scx_online_ecaps(struct rq *rq)
if (!pos->level)
continue;
- ps = pos->pshard[shard];
+ ps = cid_pshard(pos, cid);
guard(raw_spinlock)(&ps->lock);
queue_sync_ecaps(pos, cid);
}
--
2.56.0
^ permalink raw reply [flat|nested] 11+ messages in thread* [PATCH 2/6] sched_ext: Free a sub-scheduler a grace period after discarding its ecaps syncs
2026-10-17 16:45 [PATCHSET v2 sched_ext/for-7.4] sched_ext: Report a sub-scheduler's ecaps on attach, detach and cpu hotplug Tejun Heo
2026-10-17 16:45 ` [PATCH 1/6] sched_ext: Factor out cid_pshard() Tejun Heo
@ 2026-10-17 16:45 ` Tejun Heo
2026-10-17 16:45 ` [PATCH 3/6] sched_ext: Report a sub-scheduler's ecaps on attach and detach Tejun Heo
` (3 subsequent siblings)
5 siblings, 0 replies; 11+ messages in thread
From: Tejun Heo @ 2026-10-17 16:45 UTC (permalink / raw)
To: sched-ext
Cc: David Vernet, Andrea Righi, Changwoo Min, Emil Tsalapatis,
David Dai, linux-kernel, Tejun Heo
The free path removes a sub-scheduler's ecaps sync node from each cpu's list
and then cycles the rq lock until the node reads off-list, as a sync batch
can hold the node privately. Cycling the lock is correct only because the
sched is bypassing and a bypassing sched's sync calls no op, so the batch
touches the pcpu within one rq lock hold.
Tying the free path to what the sync does for a bypassing sched is unusual
and brittle, and it gets in the way of changing the sync. Delivering ecaps
to a bypassing sched or its parent would break the wait, as a delivery drops
the rq lock.
Wait for the batches with RCU instead, the usual way to wait for lockless
users before a free. Discard the nodes after the first grace period as
before, and free the sched after a second one. The first grace period waits
for the paths that found the sched before it was unlinked, such as the
sub-cap kfuncs and the cpu hotplug hooks. Until they finish, they can queue
its syncs, and a cpu can take a queued node into a sync batch that the first
grace period doesn't wait for. After it, nothing queues the syncs, and the
discards remove the nodes still queued.
A batch that took a node keeps irqs off from llist_del_all() to its last
pcpu write, even across the rq lock drops of a delivery. The second grace
period waits for it whether or not it called an op. The exit and disable
irq_works, the helper kthread and the bypass load balancing timer are
stopped ahead of the per-cpu checks as before, so they move into
scx_sched_quiesce_rcu_work() with the checks.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
kernel/sched/ext/ext.c | 76 ++++++++++++++++++++++++++++++------------
kernel/sched/ext/sub.c | 51 +++++++++++-----------------
2 files changed, 74 insertions(+), 53 deletions(-)
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index a955403ba8e5..d28902dc48ff 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -5983,10 +5983,6 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
struct scx_dispatch_q *dsq;
int cpu, node;
- irq_work_sync(&sch->propagate_exit_irq_work);
- irq_work_sync(&sch->disable_irq_work);
- kthread_destroy_worker(sch->helper);
- timer_shutdown_sync(&sch->bypass_lb_timer);
free_cpumask_var(sch->bypass_lb_donee_cpumask);
free_cpumask_var(sch->bypass_lb_resched_cpumask);
free_cpumask_var(sch->stall_cpus);
@@ -6004,22 +6000,6 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
for_each_possible_cpu(cpu) {
struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
- /*
- * $sch would have entered bypass mode before the RCU grace
- * period. As that blocks new deferrals, all
- * deferred_reenq_local_node's must be off-list by now.
- */
- WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node));
-
- /* remove the queued ecaps sync so the pcpu can be freed */
- scx_discard_ecaps_to_sync(cpu, pcpu);
-
- /*
- * Bypass blocks new kicks. Flush the kick irq_work so this
- * pcpu's to_kick_node is off the list before it is freed.
- */
- irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work);
- WARN_ON_ONCE(!list_empty(&pcpu->to_kick_node));
free_cpumask_var(pcpu->cpus_to_kick);
free_cpumask_var(pcpu->cpus_to_kick_if_idle);
free_cpumask_var(pcpu->cpus_to_preempt);
@@ -6061,11 +6041,65 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
kfree(sch);
}
+/**
+ * scx_sched_quiesce_rcu_work - Quiesce a sched and queue its free
+ * @work: &scx_sched.rcu_work
+ *
+ * $sch is freed after two RCU grace periods because of its ecaps sync nodes.
+ *
+ * The first grace period runs between scx_kobj_release() and this function. It
+ * waits for the paths that found $sch before scx_unlink_sched(), such as the
+ * sub-cap kfuncs and the cpu hotplug hooks. Until they finish, they can queue
+ * $sch's syncs, and a cpu can take a queued node into a sync batch. The first
+ * grace period doesn't wait for a batch that started after it began.
+ *
+ * Nothing queues $sch's syncs now. Remove the nodes still on the cpus' lists. A
+ * batch that took a node can still write $sch's pcpu, and a delivery in the
+ * batch can drop the rq lock before that write. The batch keeps irqs off from
+ * taking the node to its last pcpu write, so the second grace period waits for
+ * it before scx_sched_free_rcu_work() frees $sch.
+ */
+static void scx_sched_quiesce_rcu_work(struct work_struct *work)
+{
+ struct rcu_work *rcu_work = to_rcu_work(work);
+ struct scx_sched *sch = container_of(rcu_work, struct scx_sched, rcu_work);
+ s32 cpu;
+
+ irq_work_sync(&sch->propagate_exit_irq_work);
+ irq_work_sync(&sch->disable_irq_work);
+ kthread_destroy_worker(sch->helper);
+ timer_shutdown_sync(&sch->bypass_lb_timer);
+
+ for_each_possible_cpu(cpu) {
+ struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
+
+ /*
+ * $sch would have entered bypass mode before the first RCU
+ * grace period. As that blocks new deferrals, all
+ * deferred_reenq_local_node's must be off-list by now.
+ */
+ WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node));
+
+ /* remove the queued ecaps sync so the pcpu can be freed */
+ scx_discard_ecaps_to_sync(cpu, pcpu);
+
+ /*
+ * Bypass blocks new kicks. Flush the kick irq_work so this
+ * pcpu's to_kick_node is off the list before it is freed.
+ */
+ irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work);
+ WARN_ON_ONCE(!list_empty(&pcpu->to_kick_node));
+ }
+
+ INIT_RCU_WORK(&sch->rcu_work, scx_sched_free_rcu_work);
+ queue_rcu_work(system_dfl_wq, &sch->rcu_work);
+}
+
static void scx_kobj_release(struct kobject *kobj)
{
struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);
- INIT_RCU_WORK(&sch->rcu_work, scx_sched_free_rcu_work);
+ INIT_RCU_WORK(&sch->rcu_work, scx_sched_quiesce_rcu_work);
queue_rcu_work(system_dfl_wq, &sch->rcu_work);
}
diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
index 7edad2ddaa54..1923bdc33d6b 100644
--- a/kernel/sched/ext/sub.c
+++ b/kernel/sched/ext/sub.c
@@ -1318,8 +1318,10 @@ static void clear_all_caps(struct scx_sched *sch)
}
/*
- * @pcpu's sched was unhashed before the grace period, so nothing re-queues its
- * sync node. Remove the node from @rq's pending list so the pcpu can be freed.
+ * Remove @pcpu's ecaps sync node from @cpu's pending list. The first grace
+ * period has passed, so nothing queues the node again. The second grace period
+ * waits for a sync batch that took the node earlier, see
+ * scx_sched_quiesce_rcu_work().
*/
void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu)
{
@@ -1327,43 +1329,28 @@ void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu)
struct llist_node *head = NULL, *tail = NULL;
struct llist_node *pos, *tmp;
+ guard(rq_lock_irqsave)(rq);
+
+ if (!llist_on_list(&pcpu->ecaps_to_sync_node))
+ return;
+
/*
* llist can't unlink a single node. Take all queued nodes, drop @pcpu's
* and resplice the rest. Nodes in the taken batch read as on-list
* throughout, so queue_sync_ecaps() stays correct.
*/
- if (llist_on_list(&pcpu->ecaps_to_sync_node)) {
- scoped_guard (rq_lock_irqsave, rq) {
- llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) {
- if (pos == &pcpu->ecaps_to_sync_node) {
- init_llist_node(pos);
- } else {
- pos->next = head;
- head = pos;
- if (!tail)
- tail = pos;
- }
- }
- if (head)
- llist_add_batch(head, tail, &rq->scx.ecaps_to_sync);
- }
- }
-
- /*
- * An in-flight scx_process_sync_ecaps() batch may still hold the node
- * privately across dispatch-induced rq unlocks, reading as on-list.
- *
- * Because a bypassing sched gets no op call, init_llist_node() and all
- * @pcpu accesses share one contiguous lock hold, off-list under the rq
- * lock means @pcpu won't be accessed again.
- */
- while (true) {
- scoped_guard (rq_lock_irqsave, rq) {
- if (!llist_on_list(&pcpu->ecaps_to_sync_node))
- return;
+ llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) {
+ if (pos == &pcpu->ecaps_to_sync_node) {
+ init_llist_node(pos);
+ } else {
+ pos->next = head;
+ head = pos;
+ if (!tail)
+ tail = pos;
}
- cpu_relax();
}
+ if (head)
+ llist_add_batch(head, tail, &rq->scx.ecaps_to_sync);
}
/**
--
2.56.0
^ permalink raw reply [flat|nested] 11+ messages in thread* [PATCH 3/6] sched_ext: Report a sub-scheduler's ecaps on attach and detach
2026-10-17 16:45 [PATCHSET v2 sched_ext/for-7.4] sched_ext: Report a sub-scheduler's ecaps on attach, detach and cpu hotplug Tejun Heo
2026-10-17 16:45 ` [PATCH 1/6] sched_ext: Factor out cid_pshard() Tejun Heo
2026-10-17 16:45 ` [PATCH 2/6] sched_ext: Free a sub-scheduler a grace period after discarding its ecaps syncs Tejun Heo
@ 2026-10-17 16:45 ` Tejun Heo
2026-10-17 16:45 ` [PATCH 4/6] sched_ext: Deliver ecaps only while the root has the cpu online Tejun Heo
` (2 subsequent siblings)
5 siblings, 0 replies; 11+ messages in thread
From: Tejun Heo @ 2026-10-17 16:45 UTC (permalink / raw)
To: sched-ext
Cc: David Vernet, Andrea Righi, Changwoo Min, Emil Tsalapatis,
David Dai, linux-kernel, Tejun Heo
A sub-scheduler learns which cids it holds through ops.sub_ecaps_updated(),
but the caps granted from ops.sub_attach() reach it only after its enable
bypass lifts, when it is already scheduling its tasks.
Report the caps at the end of the enable, before bypass lifts. On disable,
report their loss before any task goes back to the parent, so that the
parent hears of the loss of every cap it was told of. ops.sub_detach()
remains where the parent cleans up after the sub-scheduler. Deliveries no
longer depend on bypass, so ops.sub_ecaps_updated() and
ops.sub_child_ecaps_updated() also run while a scheduler bypasses. The
replay at unbypass goes, and the enable's bypass returns to after
ops.sub_attach().
A sub-scheduler's ecaps now change only with the deliveries that report
them, so that whenever its ops run under the rq lock, its caps are the ones
it was last told of. A sync before the attach report leaves them alone, and
the detach report zeroes them as it delivers the loss. A cpu going offline
still drops its caps without a report, and those are the only changes not
delivered. clear_all_caps() drops that unreported loss on inactive cpus.
A report reschedules only the cpus the sub-scheduler was ever granted a cap
on. Rescheduling every cpu would disrupt a large machine split many ways,
and which cpus still owe a delivery can't be read off the caps.
The report waits on an rcuwait woken from hard irq work. On PREEMPT_RT, a
regular irq_work queued by a cpu going down can be left behind when hotplug
parks the irq_work thread, and a hard irq can't take a wait queue's lock.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
kernel/sched/ext/ext.c | 7 +-
kernel/sched/ext/internal.h | 50 ++++++-
kernel/sched/ext/sub.c | 253 ++++++++++++++++++++++++++----------
kernel/sched/ext/sub.h | 2 -
4 files changed, 228 insertions(+), 84 deletions(-)
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index d28902dc48ff..3deda2d1f7f8 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -6746,6 +6746,9 @@ static void unbypass_renotify_idle(struct rq *rq, struct scx_sched *pos,
*
* - ops.dispatch() is ignored.
*
+ * - ops.sub_ecaps_updated() and ops.sub_child_ecaps_updated() are still called.
+ * The bypassed tasks are on the bypass DSQs, out of their reach.
+ *
* - dispatch_one() does not report %SCX_DSP_PREV on non-zero slice as slice
* can't be trusted. Whenever a tick triggers, the running task is rotated to
* the tail of the queue.
@@ -6825,10 +6828,8 @@ void scx_bypass(struct scx_sched *sch, bool bypass)
pcpu->flags |= SCX_SCHED_PCPU_BYPASSING;
} else {
pcpu->flags &= ~SCX_SCHED_PCPU_BYPASSING;
- if (was_bypassing) {
+ if (was_bypassing)
unbypass_renotify_idle(rq, pos, pcpu);
- scx_unbypass_replay_ecaps(rq, pos);
- }
}
}
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index b408993af794..8532c58ddbfa 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -878,9 +878,13 @@ struct sched_ext_ops {
* @args: argument container, see the struct definition
*
* The sub-scheduler holds no caps by this point and can no longer
- * affect any cid. Its caps went away without a
- * sub_child_ecaps_updated() report, so whatever was delegated to it,
- * e.g. cpuperf targets, is the parent's to restore here.
+ * affect any cid. Every cap reported through
+ * ops.sub_child_ecaps_updated() was also reported lost there before the
+ * sub-scheduler's tasks came back to this sched. Nothing is reported
+ * for a sub-scheduler whose enable failed before its tasks were handed
+ * over. If this sched is being disabled too, it can miss these reports.
+ * On a cpu whose loss wasn't reported, whatever was delegated, e.g.
+ * cpuperf targets, is the parent's to restore here.
*/
void (*sub_detach)(struct scx_sub_detach_args *args);
@@ -914,6 +918,14 @@ struct sched_ext_ops {
* change is in effect on the cpu. Runs in dispatch context with rq lock
* held, and can perform all operations allowed in ops.dispatch()
* including inserting/moving tasks.
+ *
+ * Deliveries start after this sched's tasks are handed to it and before
+ * it makes its first scheduling decision. The caps granted from
+ * ops.sub_attach() and any change since are reported then. A cpu that
+ * isn't online at that point reports them once it comes online.
+ * Deliveries continue while this sched bypasses. Its tasks are then on
+ * the bypass DSQs, where this op can't insert or move them. Deliveries
+ * stop once this sched is being disabled.
*/
void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after);
@@ -940,11 +952,16 @@ struct sched_ext_ops {
* @before: the child's effective caps as of the last delivery
* @after: the child's effective caps now
*
- * Invoked right after the child's sub_ecaps_updated(), once a grant or
- * revoke is in effect on the cpu. From here on the parent can act on a
+ * Invoked right after the child's ops.sub_ecaps_updated(), once a
+ * change is in effect on the cpu. From here on the parent can act on a
* cap the child lost, e.g. reset the cpuperf target or schedule on the
- * cid. A detaching child's caps go away without a report, see
- * sub_detach(). A cpu going offline drops caps without a report.
+ * cid. A disabling child gets no ops.sub_ecaps_updated(), so only this
+ * op is invoked for its loss. A cpu going offline drops caps without a
+ * report.
+ *
+ * Deliveries start as for the child's ops.sub_ecaps_updated(), continue
+ * while either sched bypasses and stop once this sched is being
+ * disabled.
*
* Runs in dispatch context with rq lock held, and can perform all
* operations allowed in ops.dispatch() including inserting/moving
@@ -1414,6 +1431,10 @@ struct scx_sched;
enum scx_sched_pcpu_flags {
SCX_SCHED_PCPU_BYPASSING = 1LLU << 0,
+ /* ecaps deliveries are enabled, see ecaps_enable() */
+ SCX_SCHED_PCPU_ECAPS_ENABLED = 1LLU << 1,
+ /* a report is pending, see ecaps_report() */
+ SCX_SCHED_PCPU_ECAPS_REPORT = 1LLU << 2,
};
/* dispatch buf */
@@ -1474,6 +1495,16 @@ struct scx_sched_pcpu {
struct llist_node ecaps_to_sync_node;
/* owed a forced update_idle() re-notify on this cpu */
bool idle_renotify;
+ /*
+ * The sched was granted a cap on this cpu at some point. ecaps_report()
+ * reschedules only these cpus, as rescheduling every cpu on each
+ * sub-sched enable and disable would disrupt a large machine split many
+ * ways. Exactly which cpus owe a delivery is hard to track, as
+ * deliveries are asynchronous and reading caps[] and ecaps doesn't
+ * tell. A cpu granted a cap at some point is a simple superset. Written
+ * under pshard->lock.
+ */
+ bool had_caps;
/*
* ecaps as of the last sub_ecaps_updated() and
* sub_child_ecaps_updated() delivery. See __scx_process_sync_ecaps().
@@ -1751,6 +1782,11 @@ struct scx_sched {
bool linked; /* on ->children, see scx_link_sched() */
bool sub_attached;
+ /* ecaps_enable() has run, under scx_enable_mutex */
+ bool ecaps_enabled;
+
+ /* marked cpus whose sync is pending, see ecaps_report() */
+ atomic_t nr_ecaps_reports;
#endif /* CONFIG_EXT_SUB_SCHED */
/*
diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
index 1923bdc33d6b..46b13a1767bb 100644
--- a/kernel/sched/ext/sub.c
+++ b/kernel/sched/ext/sub.c
@@ -42,6 +42,16 @@ static s64 scx_rescue_sat_delta_ns;
static unsigned long scx_rescue_decay_halflife;
static unsigned long scx_rescue_overload_after;
+/*
+ * ecaps_report() waits here for its marked syncs. A cpu going down can clear
+ * its mark, and on PREEMPT_RT a regular irq_work queued there can be left
+ * behind when hotplug parks the irq_work thread, so the wakeup is hard irq
+ * work. A hard irq can't take a wait queue's lock, which sleeps on PREEMPT_RT.
+ * scx_enable_mutex serializes the reports, so the single waiter sleeps on an
+ * rcuwait.
+ */
+static struct rcuwait ecaps_report_wait = __RCUWAIT_INITIALIZER(ecaps_report_wait);
+
/**
* scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk
* @pos: current position
@@ -964,7 +974,7 @@ static void caps_updated_deliver(struct list_head *to_deliver)
* so !has_op means the op is absent or the sched isn't
* live yet - e.g. caps grant from ops.sub_attach().
* Either way don't consume - leave for
- * scx_sub_seed_caps() to deliver once live.
+ * scx_sub_seed_caps() to deliver once has_op is set.
*/
scoped_guard (raw_spinlock, &cu->lock) {
if (cu->caps && SCX_HAS_OP(sch, sub_caps_updated) &&
@@ -992,8 +1002,8 @@ static void caps_updated_deliver(struct list_head *to_deliver)
/*
* Deliver caps owed to @sch that couldn't be delivered earlier (e.g. a grant
- * taken during its sub_attach(), before has_op was set). Called once @sch is
- * enabled.
+ * taken during its sub_attach(), before has_op was set). Called once has_op is
+ * set.
*/
static void scx_sub_seed_caps(struct scx_sched *sch)
{
@@ -1069,14 +1079,31 @@ static struct scx_pshard *cid_pshard(struct scx_sched *sch, s32 cid)
return sch->pshard[rcu_dereference_all(scx_cid_to_shard)[cid]];
}
+static void ecaps_report_irq_workfn(struct irq_work *irq_work)
+{
+ rcuwait_wake_up(&ecaps_report_wait);
+}
+
+static struct irq_work ecaps_report_irq_work = IRQ_WORK_INIT_HARD(ecaps_report_irq_workfn);
+
+/*
+ * @pcpu's marked sync has been processed. The waiter in ecaps_report() can't be
+ * woken under the rq lock, so the last completion wakes it through irq_work.
+ */
+static void ecaps_report_done(struct scx_sched_pcpu *pcpu)
+{
+ pcpu->flags &= ~SCX_SCHED_PCPU_ECAPS_REPORT;
+ if (atomic_dec_and_test(&pcpu->sch->nr_ecaps_reports))
+ irq_work_queue(&ecaps_report_irq_work);
+}
+
/**
* __scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[]
* @rq: the cid's cpu rq
* @prev: @rq's previous task from the in-progress dispatch
*
* pshard->caps[] is the target configuration. pcpu->ecaps is the effective
- * transposed copy owned by the cid's cpu and written only here under @rq's
- * lock.
+ * transposed copy owned by the cid's cpu and written under @rq's lock.
*
* A sched that newly gains baseline access here is owed an update_idle() so it
* learns the cid's idle state. Such a gain arms the per-rq
@@ -1113,6 +1140,7 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
struct scx_sched_pcpu *pcpu =
container_of(pos, struct scx_sched_pcpu, ecaps_to_sync_node);
struct scx_pshard *ps = cid_pshard(pcpu->sch, cid);
+ bool report = pcpu->flags & SCX_SCHED_PCPU_ECAPS_REPORT;
u64 old, ecaps, lost, gained;
init_llist_node(pos);
@@ -1120,8 +1148,17 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
/* pairs with smp_mb() in queue_sync_ecaps(), see there */
smp_mb();
+ /*
+ * Before the sched's has_op is set, only the parent's op would
+ * run, and the change would never reach the sched.
+ * ecaps_enable() enables deliveries on each cpu once has_op is
+ * set. Until then, a sync leaves the ecaps alone.
+ */
old = READ_ONCE(pcpu->ecaps);
- ecaps = calc_effective_caps(ps, cid);
+ if (pcpu->flags & SCX_SCHED_PCPU_ECAPS_ENABLED)
+ ecaps = calc_effective_caps(ps, cid);
+ else
+ ecaps = old;
WRITE_ONCE(pcpu->ecaps, ecaps);
lost = old & ~ecaps;
@@ -1139,25 +1176,28 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
* up there. The rq recorded in it keeps the dispatches
* targeting @rq.
*
- * Bypass propagates down the hierarchy, so a sched that isn't
- * bypassing has no bypassing parent. The child's bypass state
- * gates both deliveries. Both report the same before value, so
- * one reported_ecaps covers them.
+ * Both report the same before value, so one reported_ecaps
+ * covers them.
*/
- if (ecaps != pcpu->reported_ecaps && !scx_bypassing(pcpu->sch, cpu)) {
+ if (ecaps != pcpu->reported_ecaps &&
+ (pcpu->flags & SCX_SCHED_PCPU_ECAPS_ENABLED)) {
struct scx_sched *parent = scx_parent(pcpu->sch);
struct scx_dsp_ctx *dspc;
/* stash @prev so nested dispatches can access it */
rq->scx.sub_dispatch_prev = prev;
- if (SCX_HAS_OP(pcpu->sch, sub_ecaps_updated)) {
+
+ /* a disabling sched's ops are not called */
+ if (SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) &&
+ likely(!READ_ONCE(pcpu->sch->aborting))) {
dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx;
dspc->rq = rq;
SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq,
scx_cpu_arg(cpu), pcpu->reported_ecaps, ecaps);
scx_flush_dispatch_buf(pcpu->sch, rq);
}
- if (SCX_HAS_OP(parent, sub_child_ecaps_updated)) {
+ if (SCX_HAS_OP(parent, sub_child_ecaps_updated) &&
+ likely(!READ_ONCE(parent->aborting))) {
dspc = &this_cpu_ptr(parent->pcpu)->dsp_ctx;
dspc->rq = rq;
SCX_CALL_OP(parent, sub_child_ecaps_updated, rq,
@@ -1169,6 +1209,9 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
pcpu->reported_ecaps = ecaps;
}
+ if (report)
+ ecaps_report_done(pcpu);
+
/*
* Gaining baseline access owes an update_idle() so the sched
* learns the cpu's idle state. Arm the per-rq gate so the next
@@ -1191,41 +1234,6 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
scx_schedule_reenq_local(rq, SCX_REENQ_CAP_REVOKE);
}
-/**
- * scx_unbypass_replay_ecaps - Replay a bypass-suppressed ecaps notification
- * @rq: rq of the cpu leaving bypass
- * @sch: scheduler that just left bypass on @rq's cpu
- *
- * scx_process_sync_ecaps() consumes syncs while bypassing without delivering
- * ops.sub_ecaps_updated(), leaving reported_ecaps stale. Nothing re-queues a
- * sync when bypass lifts, so without a replay a cid that never changes again
- * would never be notified. The attach-time initial grants are the acute case
- * as they are consumed during the enable bypass window. Re-queue a sync for
- * any undelivered delta so the next dispatch delivers it.
- */
-void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch)
-{
- s32 cpu = cpu_of(rq);
- struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
- struct scx_pshard *ps;
- s32 cid;
-
- lockdep_assert_rq_held(rq);
-
- /* root holds every cap and never uses ecaps */
- if (!sch->level)
- return;
-
- if (READ_ONCE(pcpu->ecaps) == pcpu->reported_ecaps)
- return;
-
- cid = __scx_cpu_to_cid(cpu);
- ps = sch->pshard[rcu_dereference_all(scx_cid_to_shard)[cid]];
-
- guard(raw_spinlock)(&ps->lock);
- queue_sync_ecaps(sch, cid);
-}
-
/*
* A cpu came back. Re-seed each sub-sched's ecaps on the cpu's cid. The sync
* recomputes effective caps from the pshard and fires ops.sub_ecaps_updated()
@@ -1281,24 +1289,32 @@ void scx_offline_ecaps(struct rq *rq)
root = scx_root_protected();
scx_for_each_descendant_pre(pos, root) {
+ struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu);
+
/* root holds every cap and never uses ecaps */
if (!pos->level)
continue;
- WRITE_ONCE(per_cpu_ptr(pos->pcpu, cpu)->ecaps, 0);
+ WRITE_ONCE(pcpu->ecaps, 0);
+
+ /* deliveries have stopped on this cpu, clear its mark */
+ if (pcpu->flags & SCX_SCHED_PCPU_ECAPS_REPORT)
+ ecaps_report_done(pcpu);
}
}
/*
- * Clear every cap @sch holds. The pshard caps go first as they are the source a
- * pending sync recomputes ecaps from. ecaps are then zeroed directly for the
- * cap checks.
+ * Clear every cap @sch holds in the pshards. The report that follows delivers
+ * the loss and zeroes the ecaps. The caller holds off cpu hotplug across this
+ * and the report.
*/
static void clear_all_caps(struct scx_sched *sch)
{
s32 si, cpu;
u32 cap_bit;
+ lockdep_assert_cpus_held();
+
/* enable may have failed before scx_alloc_pshards() */
if (!sch->pshard)
return;
@@ -1312,8 +1328,13 @@ static void clear_all_caps(struct scx_sched *sch)
}
for_each_possible_cpu(cpu) {
+ struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
+
guard(rq_lock_irqsave)(cpu_rq(cpu));
- WRITE_ONCE(per_cpu_ptr(sch->pcpu, cpu)->ecaps, 0);
+
+ /* drop an unreported loss, as the report skips inactive cpus */
+ if (!cpu_active(cpu))
+ pcpu->reported_ecaps = 0;
}
}
@@ -1331,6 +1352,9 @@ void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu)
guard(rq_lock_irqsave)(rq);
+ /* ecaps_report() waited for every mark it set */
+ WARN_ON_ONCE(pcpu->flags & SCX_SCHED_PCPU_ECAPS_REPORT);
+
if (!llist_on_list(&pcpu->ecaps_to_sync_node))
return;
@@ -1657,6 +1681,76 @@ static inline s32 scx_cgroup_claim_subtree(struct scx_sched *sch) { return 0; }
static inline void scx_cgroup_return_subtree(struct scx_sched *sch) {}
#endif
+/*
+ * Enable @sch's ecaps deliveries on every possible cpu, so that a cpu inactive
+ * during the attach report delivers from the reseed at its activation.
+ */
+static void ecaps_enable(struct scx_sched *sch)
+{
+ s32 cpu;
+
+ for_each_possible_cpu(cpu) {
+ guard(rq_lock_irqsave)(cpu_rq(cpu));
+ per_cpu_ptr(sch->pcpu, cpu)->flags |= SCX_SCHED_PCPU_ECAPS_ENABLED;
+ }
+ sch->ecaps_enabled = true;
+}
+
+/**
+ * ecaps_report - Report @sch's ecaps and wait
+ * @sch: sub-scheduler to report
+ *
+ * Mark every cpu @sch was ever granted a cap on, deliver its pending change
+ * through the dispatch path and wait for it. An inactive cpu is skipped. A cpu
+ * going down meanwhile has its mark cleared by scx_offline_ecaps().
+ *
+ * Called with scx_enable_mutex held. A sched's attach and detach reports must
+ * not overlap, as the mark is one bit per pcpu.
+ */
+static void ecaps_report(struct scx_sched *sch)
+{
+ s32 cpu;
+
+ lockdep_assert_held(&scx_enable_mutex);
+
+ for_each_possible_cpu(cpu) {
+ struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
+ struct rq *rq = cpu_rq(cpu);
+ struct scx_pshard *ps;
+ s32 cid;
+
+ guard(rq_lock_irqsave)(rq);
+
+ /* syncs on an inactive cpu are discarded */
+ if (!cpu_active(cpu))
+ continue;
+
+ cid = __scx_cpu_to_cid(cpu);
+ ps = cid_pshard(sch, cid);
+ guard(raw_spinlock)(&ps->lock);
+
+ if (!pcpu->had_caps)
+ continue;
+
+ pcpu->flags |= SCX_SCHED_PCPU_ECAPS_REPORT;
+ atomic_inc(&sch->nr_ecaps_reports);
+ queue_sync_ecaps(sch, cid);
+
+ /*
+ * The report needs an ext dispatch on this cpu. A kick is
+ * dropped while the root bypasses, e.g. for PM, so reschedule
+ * directly. A fair task can hold the cpu, so also start the ext
+ * server, which picks the ext class within its period.
+ */
+ update_rq_clock(rq);
+ dl_server_start(&rq->ext_server);
+ resched_curr(rq);
+ }
+
+ rcuwait_wait_event(&ecaps_report_wait, !atomic_read(&sch->nr_ecaps_reports),
+ TASK_UNINTERRUPTIBLE);
+}
+
void scx_sub_disable(struct scx_sched *sch)
{
struct scx_sched *parent = scx_parent(sch);
@@ -1678,6 +1772,24 @@ void scx_sub_disable(struct scx_sched *sch)
* disable ops.
*/
mutex_lock(&scx_enable_mutex);
+
+ /*
+ * Clear @sch's caps and report the loss before any task goes back to
+ * the parent. A disabling sched is granted nothing, so the caps stay
+ * cleared. Hold off cpu hotplug across both. The report covers every
+ * active cpu and the clear drops the unreported loss on the rest, so
+ * that no loss reaches the parent after ops.sub_detach().
+ *
+ * An enable that failed before its attach report has nothing to report.
+ * If it failed before scx_link_sched(), the root's disable doesn't wait
+ * for it and may have retired the cid tables that the report reads.
+ */
+ scoped_guard(cpus_read_lock) {
+ clear_all_caps(sch);
+ if (sch->ecaps_enabled)
+ ecaps_report(sch);
+ }
+
percpu_down_write(&scx_fork_rwsem);
scx_cgroup_lock();
@@ -1785,12 +1897,6 @@ void scx_sub_disable(struct scx_sched *sch)
scx_unlink_sched(sch);
- /*
- * The parent restores what it delegated in ops.sub_detach(). @sch must
- * hold no caps by then.
- */
- clear_all_caps(sch);
-
mutex_unlock(&scx_enable_mutex);
/*
@@ -1950,15 +2056,6 @@ void scx_sub_enable_workfn(struct kthread_work *work)
if (ret)
goto err_disable;
- /*
- * Bypass before @sch is linked and grants can reach it. The parent's
- * delivery advances reported_ecaps, so a sync consumed before @sch's
- * ops are registered would reach the parent and never be replayed to
- * @sch. While bypassed, the syncs are consumed without a delivery and
- * replayed to both at unbypass.
- */
- scx_bypass(sch, true);
-
ret = scx_link_sched(sch);
if (ret)
goto err_disable;
@@ -2004,6 +2101,8 @@ void scx_sub_enable_workfn(struct kthread_work *work)
}
sch->sub_attached = true;
+ scx_bypass(sch, true);
+
for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++)
if (((void (**)(void))ops)[i])
set_bit(i, sch->has_op);
@@ -2145,10 +2244,16 @@ void scx_sub_enable_workfn(struct kthread_work *work)
scx_cgroup_unlock();
percpu_up_write(&scx_fork_rwsem);
- scx_bypass(sch, false);
-
- /* @sch is enabled; deliver any caps owed since its sub_attach() */
+ /*
+ * Report the caps granted from ops.sub_attach() before bypass lifts and
+ * @sch makes its first scheduling decision. The report can wait, so it
+ * runs outside the fork rwsem.
+ */
scx_sub_seed_caps(sch);
+ ecaps_enable(sch);
+ ecaps_report(sch);
+
+ scx_bypass(sch, false);
pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name);
kobject_uevent(&sch->kobj, KOBJ_ADD);
@@ -2578,8 +2683,12 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps,
* the cid gains baseline access, so the holder
* learns of an already-idle cid.
*/
- scx_cmask_for_each_cid(cid, changed_cids)
+ scx_cmask_for_each_cid(cid, changed_cids) {
+ s32 cpu = __scx_cid_to_cpu(cid);
+
+ per_cpu_ptr(child->pcpu, cpu)->had_caps = true;
queue_sync_ecaps(child, cid);
+ }
}
}
diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h
index b76d750463f0..63769205c013 100644
--- a/kernel/sched/ext/sub.h
+++ b/kernel/sched/ext/sub.h
@@ -32,7 +32,6 @@ void scx_free_pshards(struct scx_sched *sch);
s32 scx_alloc_pshards(struct scx_sched *sch);
void scx_init_root_caps(struct scx_sched *sch);
void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev);
-void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch);
void scx_online_ecaps(struct rq *rq);
void scx_offline_ecaps(struct rq *rq);
void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu);
@@ -110,7 +109,6 @@ static inline void scx_free_pshards(struct scx_sched *sch) {}
static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; }
static inline void scx_init_root_caps(struct scx_sched *sch) {}
static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) {}
-static inline void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) {}
static inline void scx_online_ecaps(struct rq *rq) {}
static inline void scx_offline_ecaps(struct rq *rq) {}
static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) {}
--
2.56.0
^ permalink raw reply [flat|nested] 11+ messages in thread* [PATCH 4/6] sched_ext: Deliver ecaps only while the root has the cpu online
2026-10-17 16:45 [PATCHSET v2 sched_ext/for-7.4] sched_ext: Report a sub-scheduler's ecaps on attach, detach and cpu hotplug Tejun Heo
` (2 preceding siblings ...)
2026-10-17 16:45 ` [PATCH 3/6] sched_ext: Report a sub-scheduler's ecaps on attach and detach Tejun Heo
@ 2026-10-17 16:45 ` Tejun Heo
2026-10-17 16:45 ` [PATCH 5/6] sched_ext: Report the ecaps loss when a cpu goes offline Tejun Heo
2026-10-17 16:45 ` [PATCH 6/6] sched_ext: Leave suspend and hibernation out of the ecaps reports Tejun Heo
5 siblings, 0 replies; 11+ messages in thread
From: Tejun Heo @ 2026-10-17 16:45 UTC (permalink / raw)
To: sched-ext
Cc: David Vernet, Andrea Righi, Changwoo Min, Emil Tsalapatis,
David Dai, linux-kernel, Tejun Heo
A sub-scheduler's ecaps ops run in dispatch context, and ops.cpu_online()
and ops.cpu_offline() document that the cpu doesn't call ops.dispatch()
outside them. The reseed of a cpu coming back is queued before the root's
ops.cpu_online(), so the ops of a sub-scheduler and its parent can run for a
cpu the root still considers offline.
Add SCX_RQ_OPS_ONLINE, set while the root considers the cpu online, from its
ops.cpu_online() or its enable until its ops.cpu_offline(). A sync outside
that span leaves the ecaps alone, and the reseed, now queued after
ops.cpu_online(), reports the net change. SCX_RQ_ONLINE can't be used, as
the core changes it ahead of the root's view on both edges.
A cpu going offline still drops its caps without a report.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
kernel/sched/ext/ext.c | 18 ++++++++----
kernel/sched/ext/internal.h | 6 ++++
kernel/sched/ext/sub.c | 57 ++++++++++++++++++++++++++++++-------
kernel/sched/ext/sub.h | 2 ++
kernel/sched/sched.h | 7 +++++
5 files changed, 74 insertions(+), 16 deletions(-)
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 3deda2d1f7f8..b7a24ff34483 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -4117,11 +4117,6 @@ static void handle_hotplug(struct rq *rq, bool online)
if (scx_enabled())
scx_idle_update_selcpu_topology(&sch->ops);
- if (online)
- scx_online_ecaps(rq);
- else
- scx_offline_ecaps(rq);
-
/*
* The tables can't be retired while this function is running as the
* retirement is inside cpus_read_lock. However, scx_cpu_arg() is
@@ -4150,6 +4145,14 @@ static void handle_hotplug(struct rq *rq, bool online)
}
}
+ /*
+ * ecaps are delivered for @cpu only after ops.cpu_online() and before
+ * ops.cpu_offline(). The ecaps ops may read online_cmask, so
+ * scx_offline_ecaps() and scx_online_ecaps() run after its update.
+ */
+ if (!online)
+ scx_offline_ecaps(rq);
+
if (online && SCX_HAS_OP(sch, cpu_online))
SCX_CALL_OP(sch, cpu_online, NULL, cpu_or_cid);
else if (!online && SCX_HAS_OP(sch, cpu_offline))
@@ -4159,6 +4162,9 @@ static void handle_hotplug(struct rq *rq, bool online)
SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG,
"cpu %d going %s, exiting scheduler", cpu,
online ? "online" : "offline");
+
+ if (online)
+ scx_online_ecaps(rq);
}
void scx_rq_activate(struct rq *rq)
@@ -8293,6 +8299,8 @@ static void scx_root_enable_workfn(struct kthread_work *work)
goto err_disable;
}
+ scx_init_rq_ops_online();
+
/*
* Make the scheduler instance visible. Must be inside cpus_read_lock().
* See handle_hotplug().
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 8532c58ddbfa..d03180f91ea6 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -926,6 +926,10 @@ struct sched_ext_ops {
* Deliveries continue while this sched bypasses. Its tasks are then on
* the bypass DSQs, where this op can't insert or move them. Deliveries
* stop once this sched is being disabled.
+ *
+ * Deliveries for a cid happen only while the root considers the cid's
+ * cpu online, never before the root's ops.cpu_online() or after its
+ * ops.cpu_offline() for the cpu.
*/
void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after);
@@ -979,6 +983,7 @@ struct sched_ext_ops {
*
* @cpu just came online. @cpu will not call ops.enqueue() or
* ops.dispatch(), nor run tasks associated with other CPUs beforehand.
+ * ops.sub_child_ecaps_updated() runs for @cpu only after this op.
*/
void (*cpu_online)(s32 cpu);
@@ -988,6 +993,7 @@ struct sched_ext_ops {
*
* @cpu is going offline. @cpu will not call ops.enqueue() or
* ops.dispatch(), nor run tasks associated with other CPUs afterwards.
+ * ops.sub_child_ecaps_updated() runs for @cpu only before this op.
*/
void (*cpu_offline)(s32 cpu);
diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
index 46b13a1767bb..5fb4a4a06422 100644
--- a/kernel/sched/ext/sub.c
+++ b/kernel/sched/ext/sub.c
@@ -1152,13 +1152,19 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
* Before the sched's has_op is set, only the parent's op would
* run, and the change would never reach the sched.
* ecaps_enable() enables deliveries on each cpu once has_op is
- * set. Until then, a sync leaves the ecaps alone.
+ * set. Deliveries also need the cpu online to the root, see
+ * handle_hotplug().
+ *
+ * A sync that can't deliver leaves the ecaps alone. It may
+ * still carry a mark that is cleared below. Don't skip the rest
+ * of the loop for it.
*/
old = READ_ONCE(pcpu->ecaps);
- if (pcpu->flags & SCX_SCHED_PCPU_ECAPS_ENABLED)
- ecaps = calc_effective_caps(ps, cid);
- else
+ if (!(pcpu->flags & SCX_SCHED_PCPU_ECAPS_ENABLED) ||
+ !(rq->scx.flags & SCX_RQ_OPS_ONLINE))
ecaps = old;
+ else
+ ecaps = calc_effective_caps(ps, cid);
WRITE_ONCE(pcpu->ecaps, ecaps);
lost = old & ~ecaps;
@@ -1180,7 +1186,8 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
* covers them.
*/
if (ecaps != pcpu->reported_ecaps &&
- (pcpu->flags & SCX_SCHED_PCPU_ECAPS_ENABLED)) {
+ (pcpu->flags & SCX_SCHED_PCPU_ECAPS_ENABLED) &&
+ (rq->scx.flags & SCX_RQ_OPS_ONLINE)) {
struct scx_sched *parent = scx_parent(pcpu->sch);
struct scx_dsp_ctx *dspc;
@@ -1235,15 +1242,42 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
}
/*
- * A cpu came back. Re-seed each sub-sched's ecaps on the cpu's cid. The sync
- * recomputes effective caps from the pshard and fires ops.sub_ecaps_updated()
- * only on a real change since offline.
+ * Called with cpus_read_lock() held before the root is published. The cpus
+ * active now are online to the root without ops.cpu_online(), so they can
+ * deliver ecaps. handle_hotplug() maintains %SCX_RQ_OPS_ONLINE from here on.
+ */
+void scx_init_rq_ops_online(void)
+{
+ s32 cpu;
+
+ lockdep_assert_cpus_held();
+
+ for_each_possible_cpu(cpu) {
+ struct rq *rq = cpu_rq(cpu);
+
+ guard(rq_lock_irqsave)(rq);
+ if (cpu_active(cpu))
+ rq->scx.flags |= SCX_RQ_OPS_ONLINE;
+ else
+ rq->scx.flags &= ~SCX_RQ_OPS_ONLINE;
+ }
+}
+
+/*
+ * A cpu came back and the root's ops.cpu_online() has run. Let the cpu deliver
+ * ecaps again and reseed each sub-sched's ecaps on its cid. The sync recomputes
+ * effective caps from the pshard and fires ops.sub_ecaps_updated() only on a
+ * real change since offline.
*/
void scx_online_ecaps(struct rq *rq)
{
struct scx_sched *root, *pos;
s32 cid;
+ guard(rq_lock_irqsave)(rq);
+
+ rq->scx.flags |= SCX_RQ_OPS_ONLINE;
+
/*
* Only a live hierarchy can have ecaps to reseed. This also keeps the
* table reads below away from an enable that failed before publishing
@@ -1253,8 +1287,6 @@ void scx_online_ecaps(struct rq *rq)
if (!scx_enabled())
return;
- guard(rq_lock_irqsave)(rq);
-
root = scx_root_protected();
cid = __scx_cpu_to_cid(cpu_of(rq));
@@ -1272,7 +1304,8 @@ void scx_online_ecaps(struct rq *rq)
}
/*
- * A cpu is going down. Zero each sub-sched's in-effect ecaps so cap checks
+ * A cpu is going down, and the root's ops.cpu_offline() follows. Stop the cpu's
+ * ecaps deliveries and zero each sub-sched's in-effect ecaps so cap checks
* treat the cpu as capless while offline. Pending and late-queued syncs are
* discarded at consumption by scx_process_sync_ecaps() while the cpu is
* inactive. Leave reported_ecaps. Ownership is unchanged, so the
@@ -1301,6 +1334,8 @@ void scx_offline_ecaps(struct rq *rq)
if (pcpu->flags & SCX_SCHED_PCPU_ECAPS_REPORT)
ecaps_report_done(pcpu);
}
+
+ rq->scx.flags &= ~SCX_RQ_OPS_ONLINE;
}
/*
diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h
index 63769205c013..46c7e2a578d1 100644
--- a/kernel/sched/ext/sub.h
+++ b/kernel/sched/ext/sub.h
@@ -32,6 +32,7 @@ void scx_free_pshards(struct scx_sched *sch);
s32 scx_alloc_pshards(struct scx_sched *sch);
void scx_init_root_caps(struct scx_sched *sch);
void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev);
+void scx_init_rq_ops_online(void);
void scx_online_ecaps(struct rq *rq);
void scx_offline_ecaps(struct rq *rq);
void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu);
@@ -109,6 +110,7 @@ static inline void scx_free_pshards(struct scx_sched *sch) {}
static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; }
static inline void scx_init_root_caps(struct scx_sched *sch) {}
static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) {}
+static inline void scx_init_rq_ops_online(void) {}
static inline void scx_online_ecaps(struct rq *rq) {}
static inline void scx_offline_ecaps(struct rq *rq) {}
static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) {}
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 37d4808d7f19..c6f1f178e368 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -787,6 +787,13 @@ enum scx_rq_flags {
*/
SCX_RQ_ONLINE = 1 << 0,
SCX_RQ_CAN_STOP_TICK = 1 << 1,
+ /*
+ * Set while the root considers the cpu online, from its
+ * ops.cpu_online() until its ops.cpu_offline(). ecaps are delivered
+ * only while it is set. %SCX_RQ_ONLINE follows the core's rq state
+ * instead, which changes ahead of the root's view on both edges.
+ */
+ SCX_RQ_OPS_ONLINE = 1 << 2,
SCX_RQ_CLK_VALID = 1 << 5, /* RQ clock is fresh and valid */
SCX_RQ_BAL_CB_PENDING = 1 << 6, /* must queue a cb after dispatching */
SCX_RQ_SUB_IDLE_RENOTIFY = 1 << 7, /* sub-scheds are owed update_idle() */
--
2.56.0
^ permalink raw reply [flat|nested] 11+ messages in thread* [PATCH 5/6] sched_ext: Report the ecaps loss when a cpu goes offline
2026-10-17 16:45 [PATCHSET v2 sched_ext/for-7.4] sched_ext: Report a sub-scheduler's ecaps on attach, detach and cpu hotplug Tejun Heo
` (3 preceding siblings ...)
2026-10-17 16:45 ` [PATCH 4/6] sched_ext: Deliver ecaps only while the root has the cpu online Tejun Heo
@ 2026-10-17 16:45 ` Tejun Heo
2026-10-17 16:45 ` [PATCH 6/6] sched_ext: Leave suspend and hibernation out of the ecaps reports Tejun Heo
5 siblings, 0 replies; 11+ messages in thread
From: Tejun Heo @ 2026-10-17 16:45 UTC (permalink / raw)
To: sched-ext
Cc: David Vernet, Andrea Righi, Changwoo Min, Emil Tsalapatis,
David Dai, linux-kernel, Tejun Heo
When a cpu goes offline, a sub-scheduler's effective caps on its cid are
cleared, but neither the sub-scheduler nor its parent is notified. The
sub-scheduler keeps treating the cid as usable, and the parent can't act on
the loss, e.g. reset the cpuperf target.
Report the loss before the root's ops.cpu_offline() and the regain after its
ops.cpu_online(). The zeroing at offline was the last change to a sched's
ecaps outside a delivery, so they now change only with one, and syncs on an
inactive cpu are processed instead of discarded. Suspend and hibernation now
report their cpu downs and ups as well. This is temporary, and a later patch
leaves them out.
Two things need care. A cpu going down marks a sync for each sub-scheduler,
and its hotplug thread waits until each marked sync has been processed. The
cpu's own ext picks process them, and the ext server is started in case a
fair task holds the cpu. Only the marked syncs count, so a flood of changes
on the cid can't hold up the hotplug.
The detach report reaches only active cpus. It holds off cpu hotplug, so a
cpu it skips has reported its loss as it went down, and no unreported loss
is left to drop.
Drop the cpuperf reset from scx_qmap's ops.sub_detach(), as its
ops.sub_child_ecaps_updated() now resets the targets on the loss reports.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
kernel/sched/ext/internal.h | 22 ++--
kernel/sched/ext/sub.c | 232 ++++++++++++++++++++-------------
kernel/sched/sched.h | 1 +
tools/sched_ext/scx_qmap.bpf.c | 25 +---
4 files changed, 160 insertions(+), 120 deletions(-)
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index d03180f91ea6..0d20661fa4eb 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -927,9 +927,14 @@ struct sched_ext_ops {
* the bypass DSQs, where this op can't insert or move them. Deliveries
* stop once this sched is being disabled.
*
+ * Until this sched is being disabled, whenever an op of this sched runs
+ * under a cid's rq lock, its effective caps on the cid are the @after
+ * of the last invocation for the cid.
+ *
* Deliveries for a cid happen only while the root considers the cid's
* cpu online, never before the root's ops.cpu_online() or after its
- * ops.cpu_offline() for the cpu.
+ * ops.cpu_offline() for the cpu. A cpu going offline reports the loss
+ * of every cap on it, and a cpu coming back reports the regain.
*/
void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after);
@@ -960,12 +965,14 @@ struct sched_ext_ops {
* change is in effect on the cpu. From here on the parent can act on a
* cap the child lost, e.g. reset the cpuperf target or schedule on the
* cid. A disabling child gets no ops.sub_ecaps_updated(), so only this
- * op is invoked for its loss. A cpu going offline drops caps without a
- * report.
+ * op is invoked for its loss. On a cpu going offline, a parent other
+ * than the root loses its own caps on the cid too, possibly before this
+ * op is invoked. If so, it can't act on the cid when this op runs.
*
* Deliveries start as for the child's ops.sub_ecaps_updated(), continue
* while either sched bypasses and stop once this sched is being
- * disabled.
+ * disabled. A cpu going offline or coming back is reported as for the
+ * child's ops.sub_ecaps_updated().
*
* Runs in dispatch context with rq lock held, and can perform all
* operations allowed in ops.dispatch() including inserting/moving
@@ -1441,6 +1448,8 @@ enum scx_sched_pcpu_flags {
SCX_SCHED_PCPU_ECAPS_ENABLED = 1LLU << 1,
/* a report is pending, see ecaps_report() */
SCX_SCHED_PCPU_ECAPS_REPORT = 1LLU << 2,
+ /* a cpu going down waits for this sync, see scx_offline_ecaps() */
+ SCX_SCHED_PCPU_ECAPS_OFFLINE = 1LLU << 3,
};
/* dispatch buf */
@@ -1511,11 +1520,6 @@ struct scx_sched_pcpu {
* under pshard->lock.
*/
bool had_caps;
- /*
- * ecaps as of the last sub_ecaps_updated() and
- * sub_child_ecaps_updated() delivery. See __scx_process_sync_ecaps().
- */
- u64 reported_ecaps;
/*
* Decaying rescue runtime consumed on this cpu, see
diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
index 5fb4a4a06422..5395d6f2a17b 100644
--- a/kernel/sched/ext/sub.c
+++ b/kernel/sched/ext/sub.c
@@ -52,6 +52,9 @@ static unsigned long scx_rescue_overload_after;
*/
static struct rcuwait ecaps_report_wait = __RCUWAIT_INITIALIZER(ecaps_report_wait);
+/* a cpu going down waits here for its marked syncs, see scx_offline_ecaps() */
+static DECLARE_WAIT_QUEUE_HEAD(ecaps_offline_waitq);
+
/**
* scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk
* @pos: current position
@@ -1040,8 +1043,8 @@ static u64 calc_effective_caps(struct scx_pshard *ps, s32 cid)
* @sch: sched to update
* @cid: cid to update
*
- * Queue an ecaps update for @sch's @cid and kick the cpu so that it syncs in
- * dispatch_one().
+ * Queue an ecaps update for @sch's @cid and kick the cpu, if active, so that it
+ * syncs in dispatch_one().
*/
static void queue_sync_ecaps(struct scx_sched *sch, s32 cid)
{
@@ -1049,28 +1052,26 @@ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid)
struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
/*
- * Pairs with smp_mb() in __scx_process_sync_ecaps(). Either the check
- * below sees the node off the list and queues it, or the in-flight sync
- * sees the caps[] update made before this call.
+ * Pairs with smp_mb() in __scx_process_sync_ecaps(). The caller has
+ * written the state a sync reads, caps[] or cpu_active(). Either the
+ * check below finds the node off the list and queues a sync, or a sync
+ * in flight is still ahead of its reads and sees the new state.
*/
smp_mb();
/* @cid's pshard->lock excludes concurrent queueing attempts */
if (llist_on_list(&pcpu->ecaps_to_sync_node))
return;
- if (llist_add(&pcpu->ecaps_to_sync_node, &cpu_rq(cpu)->scx.ecaps_to_sync))
- scx_kick_cpu(sch->ancestors[0], cpu, 0);
-}
-
-/* discard @rq's queued ecaps syncs */
-static void discard_queued_syncs(struct rq *rq)
-{
- struct llist_node *pos, *tmp;
-
- lockdep_assert_rq_held(rq);
- llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync))
- init_llist_node(pos);
+ /*
+ * A cpu going down processes its syncs while its hotplug thread waits
+ * in scx_offline_ecaps(). That function reschedules the cpu itself, and
+ * a kick for every change on the cid would only keep preempting the
+ * hotplug thread. Don't kick an inactive cpu.
+ */
+ if (llist_add(&pcpu->ecaps_to_sync_node, &cpu_rq(cpu)->scx.ecaps_to_sync) &&
+ cpu_active(cpu))
+ scx_kick_cpu(sch->ancestors[0], cpu, 0);
}
/* the pshard holding @sch's caps on @cid */
@@ -1097,6 +1098,29 @@ static void ecaps_report_done(struct scx_sched_pcpu *pcpu)
irq_work_queue(&ecaps_report_irq_work);
}
+static void ecaps_offline_irq_workfn(struct irq_work *irq_work)
+{
+ wake_up(&ecaps_offline_waitq);
+}
+
+static DEFINE_IRQ_WORK(ecaps_offline_irq_work, ecaps_offline_irq_workfn);
+
+/*
+ * @pcpu's sync marked by scx_offline_ecaps() has been processed or discarded.
+ * The waiter can't be woken under the rq lock, so the last completion wakes it
+ * through irq_work. The marks are all done before the cpu reaches
+ * __cpu_disable(), past which a queued irq_work would never run.
+ */
+static void ecaps_offline_done(struct rq *rq, struct scx_sched_pcpu *pcpu)
+{
+ lockdep_assert_rq_held(rq);
+
+ pcpu->flags &= ~SCX_SCHED_PCPU_ECAPS_OFFLINE;
+ WRITE_ONCE(rq->scx.nr_ecaps_offline, rq->scx.nr_ecaps_offline - 1);
+ if (!rq->scx.nr_ecaps_offline)
+ irq_work_queue(&ecaps_offline_irq_work);
+}
+
/**
* __scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[]
* @rq: the cid's cpu rq
@@ -1105,6 +1129,11 @@ static void ecaps_report_done(struct scx_sched_pcpu *pcpu)
* pshard->caps[] is the target configuration. pcpu->ecaps is the effective
* transposed copy owned by the cid's cpu and written under @rq's lock.
*
+ * A sched's ecaps change only with a delivery, so that whenever an op of the
+ * sched runs under @rq's lock, its ecaps are what its deliveries reported. A
+ * sync that can't deliver leaves them alone, and a later one that can reports
+ * the net change.
+ *
* A sched that newly gains baseline access here is owed an update_idle() so it
* learns the cid's idle state. Such a gain arms the per-rq
* %SCX_RQ_SUB_IDLE_RENOTIFY gate so the next idle pick delivers it.
@@ -1119,20 +1148,7 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
if (likely(llist_empty(&rq->scx.ecaps_to_sync)))
return;
- /*
- * ecaps are zeroed while the cpu is inactive and must stay zero.
- * Discard queued syncs instead of processing them - the
- * scx_online_ecaps() reseed re-syncs every sched on activation.
- * cpu_active() clears before the offline zeroing and sets before the
- * reseed is queued, so this test can neither miss a racing sync nor
- * eat the reseed.
- */
- if (unlikely(!cpu_active(cpu))) {
- discard_queued_syncs(rq);
- return;
- }
-
- /* @cid is valid here: the cpu is active with queued syncs */
+ /* @cid stays valid while a root is enabled */
cid = __scx_cpu_to_cid(cpu);
batch = llist_del_all(&rq->scx.ecaps_to_sync);
@@ -1141,6 +1157,7 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
container_of(pos, struct scx_sched_pcpu, ecaps_to_sync_node);
struct scx_pshard *ps = cid_pshard(pcpu->sch, cid);
bool report = pcpu->flags & SCX_SCHED_PCPU_ECAPS_REPORT;
+ bool offline = pcpu->flags & SCX_SCHED_PCPU_ECAPS_OFFLINE;
u64 old, ecaps, lost, gained;
init_llist_node(pos);
@@ -1155,6 +1172,10 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
* set. Deliveries also need the cpu online to the root, see
* handle_hotplug().
*
+ * An inactive cpu that can deliver is going down and reads as
+ * capless. A delivery below can drop the rq lock and the cpu's
+ * active state can change between syncs, so test it per sync.
+ *
* A sync that can't deliver leaves the ecaps alone. It may
* still carry a mark that is cleared below. Don't skip the rest
* of the loop for it.
@@ -1163,8 +1184,10 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
if (!(pcpu->flags & SCX_SCHED_PCPU_ECAPS_ENABLED) ||
!(rq->scx.flags & SCX_RQ_OPS_ONLINE))
ecaps = old;
- else
+ else if (cpu_active(cpu))
ecaps = calc_effective_caps(ps, cid);
+ else
+ ecaps = 0;
WRITE_ONCE(pcpu->ecaps, ecaps);
lost = old & ~ecaps;
@@ -1181,13 +1204,8 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
* scheduling can differ from @rq's cpu, so the context is set
* up there. The rq recorded in it keeps the dispatches
* targeting @rq.
- *
- * Both report the same before value, so one reported_ecaps
- * covers them.
*/
- if (ecaps != pcpu->reported_ecaps &&
- (pcpu->flags & SCX_SCHED_PCPU_ECAPS_ENABLED) &&
- (rq->scx.flags & SCX_RQ_OPS_ONLINE)) {
+ if (ecaps != old) {
struct scx_sched *parent = scx_parent(pcpu->sch);
struct scx_dsp_ctx *dspc;
@@ -1200,7 +1218,7 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx;
dspc->rq = rq;
SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq,
- scx_cpu_arg(cpu), pcpu->reported_ecaps, ecaps);
+ scx_cpu_arg(cpu), old, ecaps);
scx_flush_dispatch_buf(pcpu->sch, rq);
}
if (SCX_HAS_OP(parent, sub_child_ecaps_updated) &&
@@ -1209,15 +1227,16 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
dspc->rq = rq;
SCX_CALL_OP(parent, sub_child_ecaps_updated, rq,
pcpu->sch->ops.sub_cgroup_id, scx_cpu_arg(cpu),
- pcpu->reported_ecaps, ecaps);
+ old, ecaps);
scx_flush_dispatch_buf(parent, rq);
}
rq->scx.sub_dispatch_prev = NULL;
- pcpu->reported_ecaps = ecaps;
}
if (report)
ecaps_report_done(pcpu);
+ if (offline)
+ ecaps_offline_done(rq, pcpu);
/*
* Gaining baseline access owes an update_idle() so the sched
@@ -1263,11 +1282,26 @@ void scx_init_rq_ops_online(void)
}
}
+/*
+ * Make @rq's cpu dispatch through the ext class to process its ecaps syncs. A
+ * kick is dropped while the root bypasses, e.g. for PM, and an inactive cpu
+ * isn't kicked, so reschedule directly. A fair task can hold the cpu, so also
+ * start the ext server, which picks the ext class within its period.
+ */
+static void resched_sync_ecaps(struct rq *rq)
+{
+ lockdep_assert_rq_held(rq);
+
+ update_rq_clock(rq);
+ dl_server_start(&rq->ext_server);
+ resched_curr(rq);
+}
+
/*
* A cpu came back and the root's ops.cpu_online() has run. Let the cpu deliver
* ecaps again and reseed each sub-sched's ecaps on its cid. The sync recomputes
- * effective caps from the pshard and fires ops.sub_ecaps_updated() only on a
- * real change since offline.
+ * effective caps from the pshard and reports any change since the last
+ * delivery.
*/
void scx_online_ecaps(struct rq *rq)
{
@@ -1301,40 +1335,69 @@ void scx_online_ecaps(struct rq *rq)
guard(raw_spinlock)(&ps->lock);
queue_sync_ecaps(pos, cid);
}
+
+ resched_sync_ecaps(rq);
}
/*
- * A cpu is going down, and the root's ops.cpu_offline() follows. Stop the cpu's
- * ecaps deliveries and zero each sub-sched's in-effect ecaps so cap checks
- * treat the cpu as capless while offline. Pending and late-queued syncs are
- * discarded at consumption by scx_process_sync_ecaps() while the cpu is
- * inactive. Leave reported_ecaps. Ownership is unchanged, so the
- * scx_online_ecaps() reseed reports only a genuine delta. No callback fires
- * here.
+ * A cpu is going down, and the root's ops.cpu_offline() follows. Queue a marked
+ * sync for each sub-sched to report the loss and wait until each has been
+ * processed. Then stop the cpu's ecaps deliveries, so that none comes after
+ * ops.cpu_offline().
+ *
+ * This runs on the cpu's hotplug thread. While it sleeps, the cpu keeps
+ * scheduling, and a pick through the ext class processes the syncs, see
+ * resched_sync_ecaps(). Only the marked syncs count, so a flood of changes on
+ * the cid can't hold up the hotplug. After the wait, every sub-sched's ecaps on
+ * the cpu are zero, and the later syncs have nothing to deliver.
*/
void scx_offline_ecaps(struct rq *rq)
{
s32 cpu = cpu_of(rq);
struct scx_sched *root, *pos;
+ u32 nr = 0;
+ s32 cid;
- guard(rq_lock_irqsave)(rq);
+ /*
+ * ecaps and the tables need a live root, see scx_online_ecaps(). The
+ * ext class isn't picked without one, so the syncs wouldn't be
+ * processed and the wait would never end. scx_enabled() can't change
+ * during hotplug.
+ */
+ if (scx_enabled()) {
+ scoped_guard (rq_lock_irqsave, rq) {
+ root = scx_root_protected();
+ cid = __scx_cpu_to_cid(cpu);
- root = scx_root_protected();
+ scx_for_each_descendant_pre(pos, root) {
+ struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu);
+ struct scx_pshard *ps;
- scx_for_each_descendant_pre(pos, root) {
- struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu);
+ /* root holds every cap and never uses ecaps */
+ if (!pos->level)
+ continue;
- /* root holds every cap and never uses ecaps */
- if (!pos->level)
- continue;
+ /*
+ * A batch that has yet to reach the sync sees
+ * the mark. A sync already being processed has
+ * left the list and is queued again below.
+ */
+ pcpu->flags |= SCX_SCHED_PCPU_ECAPS_OFFLINE;
+ nr++;
+
+ ps = cid_pshard(pos, cid);
+ guard(raw_spinlock)(&ps->lock);
+ queue_sync_ecaps(pos, cid);
+ }
- WRITE_ONCE(pcpu->ecaps, 0);
+ WRITE_ONCE(rq->scx.nr_ecaps_offline, nr);
+ resched_sync_ecaps(rq);
+ }
- /* deliveries have stopped on this cpu, clear its mark */
- if (pcpu->flags & SCX_SCHED_PCPU_ECAPS_REPORT)
- ecaps_report_done(pcpu);
+ wait_event(ecaps_offline_waitq, !READ_ONCE(rq->scx.nr_ecaps_offline));
}
+ guard(rq_lock_irqsave)(rq);
rq->scx.flags &= ~SCX_RQ_OPS_ONLINE;
}
@@ -1345,7 +1408,7 @@ void scx_offline_ecaps(struct rq *rq)
*/
static void clear_all_caps(struct scx_sched *sch)
{
- s32 si, cpu;
+ s32 si;
u32 cap_bit;
lockdep_assert_cpus_held();
@@ -1361,16 +1424,6 @@ static void clear_all_caps(struct scx_sched *sch)
for (cap_bit = 0; cap_bit < __SCX_NR_CAPS; cap_bit++)
scx_cmask_clear(&ps->caps[cap_bit].cmask);
}
-
- for_each_possible_cpu(cpu) {
- struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
-
- guard(rq_lock_irqsave)(cpu_rq(cpu));
-
- /* drop an unreported loss, as the report skips inactive cpus */
- if (!cpu_active(cpu))
- pcpu->reported_ecaps = 0;
- }
}
/*
@@ -1401,6 +1454,9 @@ void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu)
llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) {
if (pos == &pcpu->ecaps_to_sync_node) {
init_llist_node(pos);
+ /* a cpu going down may be waiting for the sync */
+ if (pcpu->flags & SCX_SCHED_PCPU_ECAPS_OFFLINE)
+ ecaps_offline_done(rq, pcpu);
} else {
pos->next = head;
head = pos;
@@ -1428,9 +1484,11 @@ void scx_discard_stale_ecaps_syncs(void)
for_each_possible_cpu(cpu) {
struct rq *rq = cpu_rq(cpu);
+ struct llist_node *pos, *tmp;
guard(rq_lock_irqsave)(rq);
- discard_queued_syncs(rq);
+ llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync))
+ init_llist_node(pos);
}
}
@@ -1736,8 +1794,14 @@ static void ecaps_enable(struct scx_sched *sch)
* @sch: sub-scheduler to report
*
* Mark every cpu @sch was ever granted a cap on, deliver its pending change
- * through the dispatch path and wait for it. An inactive cpu is skipped. A cpu
- * going down meanwhile has its mark cleared by scx_offline_ecaps().
+ * through the dispatch path and wait for it. An inactive cpu is skipped. It may
+ * already be offline, where the mark would never be processed and the wait
+ * would never end. On attach, the reseed at its activation reports its caps. On
+ * detach, its loss was reported as it went down.
+ *
+ * If a marked cpu goes down meanwhile, it clears the mark while still online,
+ * as ecaps_report_irq_work would never run on an offline cpu. A cpu going down
+ * waits for the marked sync in scx_offline_ecaps().
*
* Called with scx_enable_mutex held. A sched's attach and detach reports must
* not overlap, as the mark is one bit per pcpu.
@@ -1756,7 +1820,6 @@ static void ecaps_report(struct scx_sched *sch)
guard(rq_lock_irqsave)(rq);
- /* syncs on an inactive cpu are discarded */
if (!cpu_active(cpu))
continue;
@@ -1770,16 +1833,7 @@ static void ecaps_report(struct scx_sched *sch)
pcpu->flags |= SCX_SCHED_PCPU_ECAPS_REPORT;
atomic_inc(&sch->nr_ecaps_reports);
queue_sync_ecaps(sch, cid);
-
- /*
- * The report needs an ext dispatch on this cpu. A kick is
- * dropped while the root bypasses, e.g. for PM, so reschedule
- * directly. A fair task can hold the cpu, so also start the ext
- * server, which picks the ext class within its period.
- */
- update_rq_clock(rq);
- dl_server_start(&rq->ext_server);
- resched_curr(rq);
+ resched_sync_ecaps(rq);
}
rcuwait_wait_event(&ecaps_report_wait, !atomic_read(&sch->nr_ecaps_reports),
@@ -1811,9 +1865,9 @@ void scx_sub_disable(struct scx_sched *sch)
/*
* Clear @sch's caps and report the loss before any task goes back to
* the parent. A disabling sched is granted nothing, so the caps stay
- * cleared. Hold off cpu hotplug across both. The report covers every
- * active cpu and the clear drops the unreported loss on the rest, so
- * that no loss reaches the parent after ops.sub_detach().
+ * cleared. Hold off cpu hotplug across both so that a cpu the report
+ * skips has finished going down. Its loss was reported as it went down,
+ * see scx_offline_ecaps().
*
* An enable that failed before its attach report has nothing to report.
* If it failed before scx_link_sched(), the root's disable doesn't wait
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index c6f1f178e368..11fef351a112 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -844,6 +844,7 @@ struct scx_rq {
/* sched of the open running session, see scx_cid_sched_update() */
struct scx_sched *sched;
struct llist_head ecaps_to_sync; /* pending ecaps syncs */
+ u32 nr_ecaps_offline; /* marked syncs a down waits for */
struct task_struct *sub_dispatch_prev;
#endif
cpumask_var_t cpus_to_sync;
diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c
index 61143a2d79f8..96de4bc02569 100644
--- a/tools/sched_ext/scx_qmap.bpf.c
+++ b/tools/sched_ext/scx_qmap.bpf.c
@@ -2168,35 +2168,16 @@ s32 BPF_STRUCT_OPS(qmap_sub_attach, struct scx_sub_attach_args *args)
void BPF_STRUCT_OPS(qmap_sub_detach, struct scx_sub_detach_args *args)
{
u64 cgid = args->ops->sub_cgroup_id;
- s32 nr_cids = qa.nr_cids;
- s32 pos = qa.part.rr_pos;
- u64 holder_cgid;
- bool reset_pool;
- s32 i, cid;
-
- if (nr_cids < 0 || nr_cids > SCX_QMAP_MAX_CPUS || pos < 0 || pos >= MAX_PARTS) {
- scx_bpf_error("-ERANGE");
- return;
- }
+ s32 i;
/*
- * The child's caps are gone without a report, see ops.sub_detach().
- * cpuperf targets persist until the next write. Reset the child's excl
- * cids, and the pool unless another child holds it: that child's revoke
- * report or its own detach resets it then.
+ * qmap_sub_child_ecaps_updated() has reset the cpuperf targets on the
+ * loss reports.
*/
- holder_cgid = qa.part.rr_slots[pos];
- reset_pool = !holder_cgid || holder_cgid == cgid;
-
for (i = 0; i < MAX_SUB_SCHEDS; i++) {
if (qa.sub_sched_ctxs[i].cgroup_id != cgid)
continue;
- bpf_arena_for(cid, 0, nr_cids)
- if (cmask_test(cid, &qa.sub_sched_ctxs[i].granted_cids.mask) ||
- (reset_pool && cmask_test(cid, &qa.rr_cids.mask)))
- scx_bpf_cidperf_set(cid, SCX_CPUPERF_ONE);
-
qa.sub_sched_ctxs[i].cgroup_id = 0;
qa.sub_sched_ctxs[i].weight = 100;
cmask_init(&qa.sub_sched_ctxs[i].granted_cids.mask, 0, qa.nr_cids);
--
2.56.0
^ permalink raw reply [flat|nested] 11+ messages in thread* [PATCH 6/6] sched_ext: Leave suspend and hibernation out of the ecaps reports
2026-10-17 16:45 [PATCHSET v2 sched_ext/for-7.4] sched_ext: Report a sub-scheduler's ecaps on attach, detach and cpu hotplug Tejun Heo
` (4 preceding siblings ...)
2026-10-17 16:45 ` [PATCH 5/6] sched_ext: Report the ecaps loss when a cpu goes offline Tejun Heo
@ 2026-10-17 16:45 ` Tejun Heo
5 siblings, 0 replies; 11+ messages in thread
From: Tejun Heo @ 2026-10-17 16:45 UTC (permalink / raw)
To: sched-ext
Cc: David Vernet, Andrea Righi, Changwoo Min, Emil Tsalapatis,
David Dai, linux-kernel, Tejun Heo
Suspend and hibernation take down every non-boot cpu and bring them back. A
loss and a regain per cpu to every sub-scheduler would be noise. Leave them
out: a cpu they take down reports no loss and keeps the caps last delivered,
and the reseed after resume reports only the net change. They are the downs
with cpuhp_tasks_frozen set between the PM notifications. A kexec jump sets
the flag too but sends no PM notification.
Two things need care. A cpu that suspend or hibernation takes down doesn't
wait for its syncs, and a pending ecaps_report() mark on it is processed by
its last pick before the stopper takes it down.
The detach report reaches only active cpus, and the loss on a cpu that
suspend or hibernation took down was never delivered. The detach waits for
those cpus to come back, so every loss is delivered. They come back before
the tasks thaw, so the wait doesn't hold up the freezer. If one fails to
come back, the hierarchy is aborted at the end of the PM cycle.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
kernel/sched/ext/ext.c | 15 ++++
kernel/sched/ext/internal.h | 5 +-
kernel/sched/ext/sub.c | 143 +++++++++++++++++++++++++++++++++---
kernel/sched/ext/sub.h | 6 ++
4 files changed, 156 insertions(+), 13 deletions(-)
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index b7a24ff34483..4e6ac45634b1 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -4105,6 +4105,7 @@ static void handle_hotplug(struct rq *rq, bool online)
s32 cpu_or_cid = cpu;
atomic_long_inc(&scx_hotplug_seq);
+ scx_update_frozen_cpus(cpu, online);
/*
* scx_root updates are protected by cpus_read_lock() and will stay
@@ -9407,6 +9408,20 @@ static int scx_pm_handler(struct notifier_block *nb, unsigned long event, void *
{
struct scx_sched *sch;
+ /* track the PM cycle whether or not a root is loaded */
+ switch (event) {
+ case PM_HIBERNATION_PREPARE:
+ case PM_SUSPEND_PREPARE:
+ case PM_RESTORE_PREPARE:
+ scx_sub_pm_begin();
+ break;
+ case PM_POST_HIBERNATION:
+ case PM_POST_SUSPEND:
+ case PM_POST_RESTORE:
+ scx_sub_pm_end();
+ break;
+ }
+
guard(rcu)();
sch = rcu_dereference(scx_root);
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 0d20661fa4eb..50eb4e54435b 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -934,7 +934,10 @@ struct sched_ext_ops {
* Deliveries for a cid happen only while the root considers the cid's
* cpu online, never before the root's ops.cpu_online() or after its
* ops.cpu_offline() for the cpu. A cpu going offline reports the loss
- * of every cap on it, and a cpu coming back reports the regain.
+ * of every cap on it, and a cpu coming back reports the regain. A cpu
+ * that suspend or hibernation takes down reports no loss and keeps its
+ * caps while down. Only its net change, if any, is reported once it is
+ * back.
*/
void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after);
diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
index 5395d6f2a17b..b248fcfafe4c 100644
--- a/kernel/sched/ext/sub.c
+++ b/kernel/sched/ext/sub.c
@@ -55,6 +55,19 @@ static struct rcuwait ecaps_report_wait = __RCUWAIT_INITIALIZER(ecaps_report_wai
/* a cpu going down waits here for its marked syncs, see scx_offline_ecaps() */
static DECLARE_WAIT_QUEUE_HEAD(ecaps_offline_waitq);
+/*
+ * The cpus that suspend or hibernation took down and that haven't come back.
+ * Their losses are not delivered, see scx_offline_ecaps(). Hotplug updates the
+ * set under cpus_write_lock(), and scx_sub_pm_end() clears it under
+ * cpus_read_lock(). A cpu that fails to come back stays in the set until
+ * regular hotplug brings it back or the PM cycle ends.
+ */
+static struct cpumask scx_frozen_cpus;
+static DECLARE_WAIT_QUEUE_HEAD(scx_frozen_cpus_waitq);
+
+/* between the PREPARE and POST notifications of a PM cycle */
+static bool scx_pm_in_progress;
+
/**
* scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk
* @pos: current position
@@ -1121,6 +1134,17 @@ static void ecaps_offline_done(struct rq *rq, struct scx_sched_pcpu *pcpu)
irq_work_queue(&ecaps_offline_irq_work);
}
+/*
+ * Test whether the hotplug operation in progress is suspend or hibernation
+ * taking a cpu down or bringing it back. A kexec jump also sets
+ * cpuhp_tasks_frozen but sends no PM notification, so outside a PM cycle its
+ * cpu downs deliver their losses as in regular hotplug.
+ */
+static bool scx_pm_frozen(void)
+{
+ return cpuhp_tasks_frozen && READ_ONCE(scx_pm_in_progress);
+}
+
/**
* __scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[]
* @rq: the cid's cpu rq
@@ -1175,6 +1199,8 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
* An inactive cpu that can deliver is going down and reads as
* capless. A delivery below can drop the rq lock and the cpu's
* active state can change between syncs, so test it per sync.
+ * Suspend and hibernation don't deliver the loss of a cpu they
+ * take down, see scx_offline_ecaps().
*
* A sync that can't deliver leaves the ecaps alone. It may
* still carry a mark that is cleared below. Don't skip the rest
@@ -1186,6 +1212,8 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
ecaps = old;
else if (cpu_active(cpu))
ecaps = calc_effective_caps(ps, cid);
+ else if (scx_pm_frozen())
+ ecaps = old;
else
ecaps = 0;
WRITE_ONCE(pcpu->ecaps, ecaps);
@@ -1260,6 +1288,63 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
scx_schedule_reenq_local(rq, SCX_REENQ_CAP_REVOKE);
}
+void scx_sub_pm_begin(void)
+{
+ WRITE_ONCE(scx_pm_in_progress, true);
+}
+
+/*
+ * Called on every cpu down and up whether or not a root is loaded, so that the
+ * set stays exact when a root is enabled or disabled during a PM cycle. A cpu
+ * that failed to come back from suspend or hibernation can come back through
+ * regular hotplug before the PM cycle ends, e.g. through the SMT control. The
+ * reseed at its activation reports the net change, so any up removes the cpu
+ * from the set.
+ */
+void scx_update_frozen_cpus(s32 cpu, bool online)
+{
+ if (online) {
+ if (!cpumask_test_cpu(cpu, &scx_frozen_cpus))
+ return;
+ __cpumask_clear_cpu(cpu, &scx_frozen_cpus);
+ if (cpumask_empty(&scx_frozen_cpus))
+ wake_up_all(&scx_frozen_cpus_waitq);
+ } else if (scx_pm_frozen()) {
+ __cpumask_set_cpu(cpu, &scx_frozen_cpus);
+ }
+}
+
+/*
+ * The PM cycle is over. A cpu still in the set failed to come back, and its
+ * loss can't be delivered anymore. That breaks the guarantee that every ecaps
+ * change reaches the sub-scheds and their parents, so abort every sched. A root
+ * without sub-scheds is owed no reports and keeps running.
+ *
+ * Abort before clearing the set and waking the detaches in scx_sub_disable(),
+ * so that no parent's ops.sub_detach() runs with that loss unreported. No exit
+ * cpu is recorded, as the cpu running this isn't the one that failed.
+ */
+void scx_sub_pm_end(void)
+{
+ /* hotplug updates the set under cpus_write_lock() */
+ guard(cpus_read_lock)();
+
+ if (unlikely(!cpumask_empty(&scx_frozen_cpus)) && scx_has_subs()) {
+ struct scx_sched *root;
+
+ guard(rcu)();
+ root = rcu_dereference(scx_root);
+ if (root)
+ __scx_exit(root, SCX_EXIT_ERROR, 0, -1,
+ "%u cpu(s) failed to come back after suspend or hibernation",
+ cpumask_weight(&scx_frozen_cpus));
+ }
+
+ WRITE_ONCE(scx_pm_in_progress, false);
+ cpumask_clear(&scx_frozen_cpus);
+ wake_up_all(&scx_frozen_cpus_waitq);
+}
+
/*
* Called with cpus_read_lock() held before the root is published. The cpus
* active now are online to the root without ops.cpu_online(), so they can
@@ -1350,6 +1435,11 @@ void scx_online_ecaps(struct rq *rq)
* resched_sync_ecaps(). Only the marked syncs count, so a flood of changes on
* the cid can't hold up the hotplug. After the wait, every sub-sched's ecaps on
* the cpu are zero, and the later syncs have nothing to deliver.
+ *
+ * The loss on a cpu that suspend or hibernation takes down is not delivered, so
+ * that the scheds don't see a PM cycle as cpus leaving and coming back. The cpu
+ * keeps the caps last delivered until it comes back, and the reseed then
+ * reports the net change.
*/
void scx_offline_ecaps(struct rq *rq)
{
@@ -1364,7 +1454,7 @@ void scx_offline_ecaps(struct rq *rq)
* processed and the wait would never end. scx_enabled() can't change
* during hotplug.
*/
- if (scx_enabled()) {
+ if (scx_enabled() && !scx_pm_frozen()) {
scoped_guard (rq_lock_irqsave, rq) {
root = scx_root_protected();
cid = __scx_cpu_to_cid(cpu);
@@ -1797,11 +1887,16 @@ static void ecaps_enable(struct scx_sched *sch)
* through the dispatch path and wait for it. An inactive cpu is skipped. It may
* already be offline, where the mark would never be processed and the wait
* would never end. On attach, the reseed at its activation reports its caps. On
- * detach, its loss was reported as it went down.
+ * detach, its loss was reported as it went down. A cpu that suspend or
+ * hibernation took down is back by then, or the hierarchy is aborting, see
+ * scx_sub_disable().
*
* If a marked cpu goes down meanwhile, it clears the mark while still online,
- * as ecaps_report_irq_work would never run on an offline cpu. A cpu going down
- * waits for the marked sync in scx_offline_ecaps().
+ * as ecaps_report_irq_work would never run on an offline cpu. A regular down
+ * waits for the marked sync in scx_offline_ecaps(). Suspend and hibernation
+ * don't wait, but the last pick on the cpu before the stopper takes it down, or
+ * a core scheduling sibling's pick of its rq, goes through the ext class and
+ * processes the marked sync.
*
* Called with scx_enable_mutex held. A sched's attach and detach reports must
* not overlap, as the mark is one bit per pcpu.
@@ -1869,15 +1964,39 @@ void scx_sub_disable(struct scx_sched *sch)
* skips has finished going down. Its loss was reported as it went down,
* see scx_offline_ecaps().
*
- * An enable that failed before its attach report has nothing to report.
- * If it failed before scx_link_sched(), the root's disable doesn't wait
- * for it and may have retired the cid tables that the report reads.
+ * The loss on a cpu that suspend or hibernation took down is not
+ * delivered. Wait for such cpus to come back so that the report reaches
+ * them. They go down only after the freezer has frozen the tasks and
+ * come back before the tasks thaw, so the wait doesn't hold up the
+ * freezer. If one fails to come back, the wait lasts until the PM cycle
+ * ends, and scx_sub_pm_end() aborts every sched first.
+ *
+ * The check and the report run in one cpus_read_lock() section so that
+ * no cpu goes down for PM between them. The report also needs
+ * scx_enable_mutex, see ecaps_report(). A cpu coming back takes
+ * cpus_write_lock(), so the wait drops cpus_read_lock(). It drops
+ * scx_enable_mutex too so that other enables and disables don't wait
+ * behind the PM cycle, and retakes both in order.
+ *
+ * An enable that failed before its attach report has nothing to report
+ * and doesn't wait. If it failed before scx_link_sched(), the root's
+ * disable doesn't wait for it and may have retired the cid tables that
+ * the report reads.
*/
- scoped_guard(cpus_read_lock) {
- clear_all_caps(sch);
- if (sch->ecaps_enabled)
- ecaps_report(sch);
- }
+ cpus_read_lock();
+ while (sch->ecaps_enabled && !cpumask_empty(&scx_frozen_cpus)) {
+ cpus_read_unlock();
+ mutex_unlock(&scx_enable_mutex);
+ /* lockless hint, the loop retests under cpus_read_lock() */
+ wait_event_idle(scx_frozen_cpus_waitq,
+ data_race(cpumask_empty(&scx_frozen_cpus)));
+ mutex_lock(&scx_enable_mutex);
+ cpus_read_lock();
+ }
+ clear_all_caps(sch);
+ if (sch->ecaps_enabled)
+ ecaps_report(sch);
+ cpus_read_unlock();
percpu_down_write(&scx_fork_rwsem);
scx_cgroup_lock();
diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h
index 46c7e2a578d1..eb1dba6d51e2 100644
--- a/kernel/sched/ext/sub.h
+++ b/kernel/sched/ext/sub.h
@@ -32,6 +32,9 @@ void scx_free_pshards(struct scx_sched *sch);
s32 scx_alloc_pshards(struct scx_sched *sch);
void scx_init_root_caps(struct scx_sched *sch);
void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev);
+void scx_sub_pm_begin(void);
+void scx_update_frozen_cpus(s32 cpu, bool online);
+void scx_sub_pm_end(void);
void scx_init_rq_ops_online(void);
void scx_online_ecaps(struct rq *rq);
void scx_offline_ecaps(struct rq *rq);
@@ -110,6 +113,9 @@ static inline void scx_free_pshards(struct scx_sched *sch) {}
static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; }
static inline void scx_init_root_caps(struct scx_sched *sch) {}
static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) {}
+static inline void scx_sub_pm_begin(void) {}
+static inline void scx_update_frozen_cpus(s32 cpu, bool online) {}
+static inline void scx_sub_pm_end(void) {}
static inline void scx_init_rq_ops_online(void) {}
static inline void scx_online_ecaps(struct rq *rq) {}
static inline void scx_offline_ecaps(struct rq *rq) {}
--
2.56.0
^ permalink raw reply [flat|nested] 11+ messages in thread