* [PATCH 1/2] sched_ext: Fix idle state tracking across idle repicks
2026-09-14 23:42 [PATCHSET sched_ext/for-7.3-fixes] sched_ext: Idle repick notifications and online cid mask Tejun Heo
@ 2026-09-14 23:42 ` Tejun Heo
2026-09-15 6:23 ` Andrea Righi
2026-09-14 23:42 ` [PATCH 2/2] sched_ext: Maintain an online cid mask in the scheduler arena Tejun Heo
1 sibling, 1 reply; 6+ messages in thread
From: Tejun Heo @ 2026-09-14 23:42 UTC (permalink / raw)
To: David Vernet, Andrea Righi, Changwoo Min
Cc: Emil Tsalapatis, sched-ext, linux-kernel, Tejun Heo
An idle CPU can be reserved and kicked without receiving a task. When it
picks idle again, the builtin idle masks recover the reservation, but
ops.update_idle() is not called. The BPF scheduler's own idle tracking can
leave the CPU unavailable until another real idle transition. This is a hole
in the notification interface: BPF schedulers cannot reliably recover unused
idle reservations, which is required for cid-form schedulers to maintain
their own idle tracking.
Fix this by notifying on idle-to-idle picks and making these notifications
automatic for cid-form schedulers. Keep CPU-form schedulers opt-in because
existing callbacks may restart idle accounting or repeat actions intended
only for idle entry. The known cid-form users, scx_qmap in tools/sched_ext
and scx_nitosis in the scx repository, have idempotent update_idle() bodies.
All sub-schedulers use the cid form, so this is a root property and can use
a static key.
scx_root_enable_workfn() must initialize idle tracking from sch->ops, where
the automatic cid flag has been set. The local ops pointer still refers to
the caller-provided table. Using it would leave the static key disabled for
cid-form schedulers that omit the flag.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
kernel/sched/ext/ext.c | 3 +-
kernel/sched/ext/idle.c | 60 +++++++++++--------
kernel/sched/ext/internal.h | 21 ++++++-
tools/sched_ext/include/scx/compat.h | 2 +
.../sched_ext/include/scx/enum_defs.autogen.h | 1 +
.../sched_ext/include/scx/enums_abi.autogen.h | 3 +-
6 files changed, 60 insertions(+), 30 deletions(-)
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 83999203a63a..934605dfd950 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -7251,6 +7251,7 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
*/
if (cmd->is_cid_type) {
sch->ops_cid = *cmd->ops_cid;
+ sch->ops_cid.flags |= SCX_OPS_UPDATE_IDLE_TO_IDLE;
sch->is_cid_type = true;
} else {
sch->ops = *cmd->ops;
@@ -7575,7 +7576,7 @@ static void scx_root_enable_workfn(struct kthread_work *work)
goto err_disable;
}
- scx_idle_enable(ops);
+ scx_idle_enable(&sch->ops);
/*
* A cid-form scheduler finalizes its cid layout in ops.init_cids(),
diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c
index aa9fb6de0ad6..ea2fe5030fc4 100644
--- a/kernel/sched/ext/idle.c
+++ b/kernel/sched/ext/idle.c
@@ -20,6 +20,9 @@ static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled);
/* Enable/disable per-node idle cpumasks */
static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_per_node);
+/* Idle-to-idle notifications are a property of the root hierarchy. */
+static DEFINE_STATIC_KEY_FALSE(scx_update_idle_to_idle);
+
/* Enable/disable LLC aware optimizations */
static DEFINE_STATIC_KEY_FALSE(scx_selcpu_topo_llc);
@@ -734,13 +737,12 @@ static void update_builtin_idle(int cpu, bool idle)
}
/*
- * Notify schedulers of an idle transition on @cpu's cid, delivering to every
- * sched that holds %SCX_CAP_BASE on the cid (the root holds every cap). A real
- * transition (@do_notify) reaches all holders. A forced one (@root_renotify for
- * the root, a sub-sched's idle_renotify marker for a sub) reaches only the owed
- * scheds.
+ * Notify schedulers holding %SCX_CAP_BASE on @rq's cid (the root holds every
+ * cap). Real transitions and enabled idle repicks (@notify_all) reach all
+ * holders. A forced notification (@root_renotify for the root, a sub-sched's
+ * idle_renotify marker for a sub) reaches only the owed scheds.
*/
-static void scx_idle_notify(struct rq *rq, bool idle, bool do_notify, bool root_renotify)
+static void scx_idle_notify(struct rq *rq, bool idle, bool notify_all, bool root_renotify)
{
s32 cpu = cpu_of(rq);
s32 cid = scx_cpu_arg(cpu);
@@ -751,7 +753,7 @@ static void scx_idle_notify(struct rq *rq, bool idle, bool do_notify, bool root_
/* with no sub-sched, only the root can be owed a notification */
if (!scx_has_subs()) {
- if ((do_notify || root_renotify) &&
+ if ((notify_all || root_renotify) &&
SCX_HAS_OP(root, update_idle) && !scx_bypassing(root, cpu))
SCX_CALL_OP(root, update_idle, rq, cid, idle);
return;
@@ -775,8 +777,8 @@ static void scx_idle_notify(struct rq *rq, bool idle, bool do_notify, bool root_
forced = true;
}
#endif
- if ((do_notify || forced) && SCX_HAS_OP(pos, update_idle) &&
- !scx_bypassing(pos, cpu))
+ if ((notify_all || forced) &&
+ SCX_HAS_OP(pos, update_idle) && !scx_bypassing(pos, cpu))
SCX_CALL_OP(pos, update_idle, rq, cid, idle);
pos = scx_next_descendant_pre(pos, root);
}
@@ -786,21 +788,22 @@ static void scx_idle_notify(struct rq *rq, bool idle, bool do_notify, bool root_
* Update the idle state of a CPU to @idle.
*
* If @do_notify is true, ops.update_idle() is invoked to notify the scx
- * scheduler of an actual idle state transition (idle to busy or vice
- * versa). If @do_notify is false, only the idle state in the idle masks is
- * refreshed without invoking ops.update_idle().
+ * scheduler of an actual idle state transition (idle to busy or vice versa). If
+ * @do_notify is false, refresh the idle masks and notify schedulers with
+ * %SCX_OPS_UPDATE_IDLE_TO_IDLE or an outstanding forced notification.
*
* This distinction is necessary, because an idle CPU can be "reserved" and
- * awakened via scx_bpf_pick_idle_cpu() + scx_bpf_kick_cpu(), marking it as
- * busy even if no tasks are dispatched. In this case, the CPU may return
- * to idle without a true state transition. Refreshing the idle masks
- * without invoking ops.update_idle() ensures accurate idle state tracking
- * while avoiding unnecessary updates and maintaining balanced state
- * transitions.
+ * awakened via scx_bpf_pick_idle_cpu() + scx_bpf_kick_cpu(), marking it as busy
+ * even if no tasks are dispatched. In this case, the CPU may return to idle
+ * without a true state transition. Refreshing idle tracking restores the unused
+ * reservation. Schedulers receiving repeated notifications must distinguish
+ * idle refreshes from transitions for accounting.
*/
void __scx_update_idle(struct rq *rq, bool idle, bool do_notify)
{
int cpu = cpu_of(rq);
+ u32 renotify;
+ bool notify_all;
lockdep_assert_rq_held(rq);
@@ -818,20 +821,19 @@ void __scx_update_idle(struct rq *rq, bool idle, bool do_notify)
* An idle pick also fires it to flush a forced notify owed to a sched
* that missed transitions while bypassed or on a cid it just gained.
* unbypass_renotify_idle() and scx_process_sync_ecaps() arm the per-rq
- * gates, and scx_idle_notify() targets the owed scheds.
+ * gates, and scx_idle_notify() targets the owed scheds. Schedulers with
+ * SCX_OPS_UPDATE_IDLE_TO_IDLE receive every idle pick.
*
* This must come after the builtin idle update so that BPF schedulers
* can create interlocking between ops.update_idle() and ops.enqueue() -
* either enqueue() sees the idle bit or update_idle() sees the task
* that enqueue() queued.
*/
- if (do_notify ||
- (idle && (rq->scx.flags &
- (SCX_RQ_SUB_IDLE_RENOTIFY | SCX_RQ_ROOT_IDLE_RENOTIFY)))) {
- bool root_renotify = rq->scx.flags & SCX_RQ_ROOT_IDLE_RENOTIFY;
-
- rq->scx.flags &= ~(SCX_RQ_SUB_IDLE_RENOTIFY | SCX_RQ_ROOT_IDLE_RENOTIFY);
- scx_idle_notify(rq, idle, do_notify, root_renotify);
+ notify_all = do_notify || (idle && static_branch_unlikely(&scx_update_idle_to_idle));
+ renotify = rq->scx.flags & (SCX_RQ_SUB_IDLE_RENOTIFY | SCX_RQ_ROOT_IDLE_RENOTIFY);
+ if (notify_all || (idle && renotify)) {
+ rq->scx.flags &= ~renotify;
+ scx_idle_notify(rq, idle, notify_all, renotify & SCX_RQ_ROOT_IDLE_RENOTIFY);
}
}
@@ -869,6 +871,11 @@ void scx_idle_enable(struct sched_ext_ops *ops)
else
static_branch_disable_cpuslocked(&scx_builtin_idle_per_node);
+ if (ops->flags & SCX_OPS_UPDATE_IDLE_TO_IDLE)
+ static_branch_enable_cpuslocked(&scx_update_idle_to_idle);
+ else
+ static_branch_disable_cpuslocked(&scx_update_idle_to_idle);
+
reset_idle_masks(ops);
}
@@ -876,6 +883,7 @@ void scx_idle_disable(void)
{
static_branch_disable(&scx_builtin_idle_enabled);
static_branch_disable(&scx_builtin_idle_per_node);
+ static_branch_disable(&scx_update_idle_to_idle);
}
/********************************************************************************
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 076a351bb3f2..2586879e9084 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -215,6 +215,14 @@ enum scx_ops_flags {
*/
SCX_OPS_TID_TO_TASK = 1LLU << 8,
+ /*
+ * Call ops.update_idle() with idle=true when an idle CPU picks idle
+ * again. Custom idle tracking can use this to restore an unused idle
+ * reservation after a kick that did not dispatch a task. Automatically
+ * enabled for cid-form schedulers.
+ */
+ SCX_OPS_UPDATE_IDLE_TO_IDLE = 1LLU << 9,
+
SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE |
SCX_OPS_ENQ_LAST |
SCX_OPS_ENQ_EXITING |
@@ -223,7 +231,8 @@ enum scx_ops_flags {
SCX_OPS_SWITCH_PARTIAL |
SCX_OPS_BUILTIN_IDLE_PER_NODE |
SCX_OPS_ALWAYS_ENQ_IMMED |
- SCX_OPS_TID_TO_TASK,
+ SCX_OPS_TID_TO_TASK |
+ SCX_OPS_UPDATE_IDLE_TO_IDLE,
/* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */
__SCX_OPS_INTERNAL_MASK = 0xffLLU << 56,
@@ -572,6 +581,14 @@ struct sched_ext_ops {
*
* Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle
* tracking.
+ *
+ * With %SCX_OPS_UPDATE_IDLE_TO_IDLE, this operation is also called with
+ * @idle true when an idle CPU picks idle again without leaving the
+ * idle state. The flag is automatically enabled for cid-form
+ * schedulers. Such notifications must not restart idle accounting or
+ * repeat actions that require an actual idle entry. Track the previous
+ * idle state separately from availability bits cleared by idle
+ * reservations.
*/
void (*update_idle)(s32 cpu, bool idle);
@@ -1029,7 +1046,7 @@ struct sched_ext_ops {
* Differences from sched_ext_ops:
* - select_cpu -> select_cid (returns cid)
* - dispatch -> dispatch (cpu arg is now cid)
- * - update_idle -> update_idle (cpu arg is now cid)
+ * - update_idle -> update_idle (cid arg, also called on idle repicks)
* - set_cpumask -> set_cmask (cmask instead of cpumask)
* - cpu_online -> cid_online
* - cpu_offline -> cid_offline
diff --git a/tools/sched_ext/include/scx/compat.h b/tools/sched_ext/include/scx/compat.h
index 7c12df45fdba..b96f2cd8b593 100644
--- a/tools/sched_ext/include/scx/compat.h
+++ b/tools/sched_ext/include/scx/compat.h
@@ -214,6 +214,7 @@ static inline bool __COMPAT_struct_has_field(const char *type, const char *field
#define SCX_OPS_ALLOW_QUEUED_WAKEUP SCX_OPS_FLAG(SCX_OPS_ALLOW_QUEUED_WAKEUP)
#define SCX_OPS_BUILTIN_IDLE_PER_NODE SCX_OPS_FLAG(SCX_OPS_BUILTIN_IDLE_PER_NODE)
#define SCX_OPS_ALWAYS_ENQ_IMMED SCX_OPS_FLAG(SCX_OPS_ALWAYS_ENQ_IMMED)
+#define SCX_OPS_UPDATE_IDLE_TO_IDLE SCX_OPS_FLAG(SCX_OPS_UPDATE_IDLE_TO_IDLE)
#define SCX_PICK_IDLE_FLAG(name) __COMPAT_ENUM_OR_ZERO("scx_pick_idle_cpu_flags", #name)
@@ -273,6 +274,7 @@ static inline long scx_hotplug_seq(void)
* - v6.19: ops.cgroup_set_idle()
* - v7.1: ops.sub_attach(), ops.sub_detach(), ops.sub_cgroup_id
* - v7.3: ops.rescue_bandwidth_ppt, ops.rescue_quantum_us
+ * - v7.3: SCX_OPS_UPDATE_IDLE_TO_IDLE
*/
#define __SCX_OPS_OPEN(__ops_name, __scx_name, __ops_struct) ({ \
struct __scx_name *__oskel; \
diff --git a/tools/sched_ext/include/scx/enum_defs.autogen.h b/tools/sched_ext/include/scx/enum_defs.autogen.h
index 63b6b14b19bd..c7a7baae95f4 100644
--- a/tools/sched_ext/include/scx/enum_defs.autogen.h
+++ b/tools/sched_ext/include/scx/enum_defs.autogen.h
@@ -164,6 +164,7 @@
#define HAVE_SCX_OPS_BUILTIN_IDLE_PER_NODE
#define HAVE_SCX_OPS_ALWAYS_ENQ_IMMED
#define HAVE_SCX_OPS_TID_TO_TASK
+#define HAVE_SCX_OPS_UPDATE_IDLE_TO_IDLE
#define HAVE_SCX_OPS_ALL_FLAGS
#define HAVE___SCX_OPS_INTERNAL_MASK
#define HAVE_SCX_OPS_HAS_CPU_PREEMPT
diff --git a/tools/sched_ext/include/scx/enums_abi.autogen.h b/tools/sched_ext/include/scx/enums_abi.autogen.h
index d53899764f5a..609ccacaba7e 100644
--- a/tools/sched_ext/include/scx/enums_abi.autogen.h
+++ b/tools/sched_ext/include/scx/enums_abi.autogen.h
@@ -176,7 +176,8 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[]
{ "scx_ops_flags", "SCX_OPS_BUILTIN_IDLE_PER_NODE", 0x40LLU },
{ "scx_ops_flags", "SCX_OPS_ALWAYS_ENQ_IMMED", 0x80LLU },
{ "scx_ops_flags", "SCX_OPS_TID_TO_TASK", 0x100LLU },
- { "scx_ops_flags", "SCX_OPS_ALL_FLAGS", 0x1ffLLU },
+ { "scx_ops_flags", "SCX_OPS_UPDATE_IDLE_TO_IDLE", 0x200LLU },
+ { "scx_ops_flags", "SCX_OPS_ALL_FLAGS", 0x3ffLLU },
{ "scx_ops_flags", "__SCX_OPS_INTERNAL_MASK", 0xff00000000000000LLU },
{ "scx_ops_flags", "SCX_OPS_HAS_CPU_PREEMPT", 0x100000000000000LLU },
{ "scx_ops_state", "SCX_OPSS_NONE", 0x0LLU },
--
2.55.0
^ permalink raw reply [flat|nested] 6+ messages in thread* [PATCH 2/2] sched_ext: Maintain an online cid mask in the scheduler arena
2026-09-14 23:42 [PATCHSET sched_ext/for-7.3-fixes] sched_ext: Idle repick notifications and online cid mask Tejun Heo
2026-09-14 23:42 ` [PATCH 1/2] sched_ext: Fix idle state tracking across idle repicks Tejun Heo
@ 2026-09-14 23:42 ` Tejun Heo
2026-09-15 5:44 ` Andrea Righi
1 sibling, 1 reply; 6+ messages in thread
From: Tejun Heo @ 2026-09-14 23:42 UTC (permalink / raw)
To: David Vernet, Andrea Righi, Changwoo Min
Cc: Emil Tsalapatis, sched-ext, linux-kernel, Tejun Heo
Schedulers on the default cid mapping treat [0, nr_online_cids) as the
online set and restart on hotplug. Schedulers that install their own mapping
with scx_bpf_cid_override() have no way to learn which cids are online: the
count no longer identifies members and the CPU-form cpumask is unusable from
cid programs. This is an obvious hole in the cid API.
Add scx_bpf_online_cmask(), a kernel-maintained cmask in the scheduler's
arena, allocated alongside the per-CPU scratch masks and populated after the
cid mapping is finalized and before ops.init(), for child schedulers too.
The pointer stays valid through ops.exit() with no reference to take. It is
the arena offset as a void pointer, the same form struct_ops arena arguments
arrive in. The verifier types the void return as a scalar for the program's
arena cast.
The mask follows the SCX hotplug notifications: seeded from cpu_active_mask
and updated before ops.cid_online/offline() runs, so it lags cpu_online_mask
only inside a hotplug transition. Updates walk the scheduler list under the
lock that also serializes unlinking. Reads are live, not atomic snapshots.
Root initialization excludes hotplug.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
kernel/sched/ext/ext.c | 74 ++++++++++++++++++++++--
kernel/sched/ext/internal.h | 3 +-
kernel/sched/ext/sub.c | 10 ++--
tools/sched_ext/include/scx/common.bpf.h | 1 +
4 files changed, 77 insertions(+), 11 deletions(-)
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 934605dfd950..54a1d84469bf 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -3670,8 +3670,20 @@ static void handle_hotplug(struct rq *rq, bool online)
s16 *tbl = rcu_dereference_check(scx_cpu_to_cid_tbl,
lockdep_is_cpus_held());
- if (tbl)
+ if (tbl) {
+ struct scx_sched *pos;
+
cpu_or_cid = tbl[cpu];
+
+ guard(raw_spinlock_irqsave)(&scx_sched_lock);
+ list_for_each_entry(pos, &scx_sched_all, all) {
+ struct scx_cmask *mask = pos->online_cmask;
+
+ if (mask)
+ __assign_bit(cpu_or_cid, (unsigned long *)mask->bits,
+ online);
+ }
+ }
}
if (online && SCX_HAS_OP(sch, cpu_online))
@@ -5280,12 +5292,17 @@ static void free_exit_info(struct scx_exit_info *ei);
static const char *scx_exit_reason(enum scx_exit_kind kind);
static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind);
-s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch)
+s32 scx_alloc_kern_arena_objs(struct scx_sched *sch)
{
size_t size = struct_size_t(struct scx_cmask, bits,
SCX_CMASK_NR_WORDS(num_possible_cpus()));
+ struct scx_cmask *online;
+ struct scx_cmask_ref ref;
int cpu;
+ /* hotplug stays excluded until the online mask is published */
+ lockdep_assert_cpus_held();
+
if (!sch->is_cid_type || !sch->arena_pool)
return 0;
@@ -5301,15 +5318,28 @@ s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch)
return -ENOMEM;
scx_cmask_init(*slot, 0, num_possible_cpus());
}
+
+ /* pack the online mask alongside the scratch masks */
+ online = scx_arena_alloc(sch, size);
+ if (!online)
+ return -ENOMEM;
+
+ scoped_guard(rcu) {
+ scx_cmask_ref_init_kern(sch, online, 0, num_possible_cpus(), &ref);
+ scx_cmask_ref_from_cpumask(&ref, cpu_active_mask);
+ }
+ sch->online_cmask = online;
+
return 0;
}
-static void scx_set_cmask_scratch_free(struct scx_sched *sch)
+static void scx_free_kern_arena_objs(struct scx_sched *sch)
{
size_t size = struct_size_t(struct scx_cmask, bits,
SCX_CMASK_NR_WORDS(num_possible_cpus()));
int cpu;
+ scx_arena_free(sch, sch->online_cmask, size);
if (!sch->set_cmask_scratch)
return;
@@ -5396,7 +5426,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL);
free_exit_info(sch->exit_info);
- scx_set_cmask_scratch_free(sch);
+ scx_free_kern_arena_objs(sch);
scx_arena_pool_destroy(sch);
if (sch->arena_map)
bpf_map_put(sch->arena_map);
@@ -7602,7 +7632,7 @@ static void scx_root_enable_workfn(struct kthread_work *work)
goto err_disable;
}
- ret = scx_set_cmask_scratch_alloc(sch);
+ ret = scx_alloc_kern_arena_objs(sch);
if (ret) {
cpus_read_unlock();
goto err_disable;
@@ -10339,13 +10369,44 @@ __bpf_kfunc u32 scx_bpf_nr_cids(void)
* hotplug, which lets schedulers treat [0, nr_online_cids) as the online
* range. Schedulers that prefer to handle hotplug without a restart should
* install a custom mapping via scx_bpf_cid_override() and track onlining
- * through the ops.cid_online / ops.cid_offline callbacks.
+ * through the ops.cid_online / ops.cid_offline callbacks, starting from the
+ * mask scx_bpf_online_cmask() returns.
*/
__bpf_kfunc u32 scx_bpf_nr_online_cids(void)
{
return num_online_cpus();
}
+/**
+ * scx_bpf_online_cmask - Return the online cid mask in the scheduler arena
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * Return a read-only cmask covering [0, scx_bpf_nr_cids()) which the kernel
+ * keeps in the scheduler's arena, or NULL if the calling program is not
+ * associated with a live cid-form scheduler. The mask follows the SCX hotplug
+ * notifications: a cid's bit is updated before ops.cid_online/offline() runs
+ * for it. The pointer is valid from ops.init() through ops.exit(). Root
+ * ops.init() runs with hotplug excluded. Other contexts can observe concurrent
+ * updates.
+ */
+__bpf_kfunc const void *scx_bpf_online_cmask(const struct bpf_prog_aux *aux)
+{
+ struct scx_sched *sch;
+ struct scx_cmask *online;
+
+ guard(rcu)();
+
+ sch = scx_prog_sched(aux);
+ if (unlikely(!sch))
+ return NULL;
+ online = sch->online_cmask;
+ if (unlikely(!online))
+ return NULL;
+
+ /* BPF rebases by the low 32 bits, like __arena callback args */
+ return (void *)((unsigned long)online - sch->arena_kern_base);
+}
+
/**
* scx_bpf_this_cid - Return the cid of the CPU this program is running on
*
@@ -10709,6 +10770,7 @@ BTF_ID_FLAGS(func, scx_bpf_nr_node_ids)
BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids)
BTF_ID_FLAGS(func, scx_bpf_nr_cids)
BTF_ID_FLAGS(func, scx_bpf_nr_online_cids)
+BTF_ID_FLAGS(func, scx_bpf_online_cmask, KF_IMPLICIT_ARGS | KF_ARENA_RET)
BTF_ID_FLAGS(func, scx_bpf_this_cid)
BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE)
BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE)
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 2586879e9084..067cddf47220 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -1572,6 +1572,7 @@ struct scx_sched {
* and passes it to the callback's __arena argument.
*/
struct scx_cmask * __percpu *set_cmask_scratch;
+ struct scx_cmask *online_cmask;
DECLARE_BITMAP(has_op, SCX_OPI_END);
@@ -2098,7 +2099,7 @@ void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p);
void scx_cgroup_lock(void);
void scx_cgroup_unlock(void);
#endif
-s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch);
+s32 scx_alloc_kern_arena_objs(struct scx_sched *sch);
void scx_disable_bypass_dsp(struct scx_sched *sch);
void scx_bypass(struct scx_sched *sch, bool bypass);
s32 scx_link_sched(struct scx_sched *sch);
diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
index 385302d19914..f7aeb1488566 100644
--- a/kernel/sched/ext/sub.c
+++ b/kernel/sched/ext/sub.c
@@ -1805,6 +1805,12 @@ void scx_sub_enable_workfn(struct kthread_work *work)
goto err_disable;
}
+ scoped_guard(cpus_read_lock) {
+ ret = scx_alloc_kern_arena_objs(sch);
+ if (ret)
+ goto err_disable;
+ }
+
if (sch->ops.init) {
ret = SCX_CALL_OP_RET(sch, init, NULL);
if (ret) {
@@ -1815,10 +1821,6 @@ void scx_sub_enable_workfn(struct kthread_work *work)
sch->exit_info->flags |= SCX_EFLAG_INITIALIZED;
}
- ret = scx_set_cmask_scratch_alloc(sch);
- if (ret)
- goto err_disable;
-
struct scx_sub_attach_args sub_attach_args = {
.ops = &sch->ops,
.cgroup_path = sch->cgrp_path,
diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h
index 76f5e025e107..2ddb01a059fd 100644
--- a/tools/sched_ext/include/scx/common.bpf.h
+++ b/tools/sched_ext/include/scx/common.bpf.h
@@ -113,6 +113,7 @@ s32 scx_bpf_this_cid(void) __ksym __weak;
struct task_struct *scx_bpf_cid_curr(s32 cid) __ksym __weak;
u32 scx_bpf_nr_cids(void) __ksym __weak;
u32 scx_bpf_nr_online_cids(void) __ksym __weak;
+const void __arena *scx_bpf_online_cmask(void) __ksym __weak;
u32 scx_bpf_cidperf_cap(s32 cid) __ksym __weak;
u32 scx_bpf_cidperf_cur(s32 cid) __ksym __weak;
s32 scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak;
--
2.55.0
^ permalink raw reply [flat|nested] 6+ messages in thread