* [PATCH 1/2] sched/core: Preserve WF_RQ_SELECTED across queued wakeups
2026-10-23 14:48 [PATCH 0/2] sched: Fix SCX_ENQ_CPU_SELECTED for queued wakeups Kuba Piecuch
@ 2026-10-23 14:48 ` Kuba Piecuch
2026-10-23 14:48 ` [PATCH 2/2] selftests/sched_ext: Test SCX_ENQ_CPU_SELECTED with " Kuba Piecuch
1 sibling, 0 replies; 3+ messages in thread
From: Kuba Piecuch @ 2026-10-23 14:48 UTC (permalink / raw)
To: Tejun Heo, Peter Zijlstra
Cc: Kuba Piecuch, David Vernet, Andrea Righi, Changwoo Min,
sched-ext, linux-kernel
ENQUEUE_RQ_SELECTED tells a sched_class that ->select_task_rq() was
called for a task wakeup. sched_ext exposes this flag to BPF schedulers
as SCX_ENQ_CPU_SELECTED. However, the flag is lost for queued wakeups,
which a BPF scheduler can opt into with SCX_OPS_ALLOW_QUEUED_WAKEUP.
This also results in undercounting SCX_EV_SELECT_CPU_FALLBACK events.
Preserve WF_RQ_SELECTED for queued wakeups in the same way WF_MIGRATED
is. This way, ttwu_activate() receives WF_RQ_SELECTED for queued wakeups
and passes ENQUEUE_RQ_SELECTED to activate_task().
Fixes: 3539c6411a7c ("sched_ext: Implement SCX_OPS_ALLOW_QUEUED_WAKEUP")
Signed-off-by: Kuba Piecuch <jpiecuch@google.com>
---
include/linux/sched.h | 6 ++++--
kernel/sched/core.c | 10 +++++++++-
2 files changed, 13 insertions(+), 3 deletions(-)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 12a7e094a3e6..9dabffea1c54 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1006,7 +1006,7 @@ struct task_struct {
/* Unserialized, strictly 'current' */
/*
- * This field must not be in the scheduler word above due to wakelist
+ * These fields must not be in the scheduler word above due to wakelist
* queueing no longer being serialized by p->on_cpu. However:
*
* p->XXX = X; ttwu()
@@ -1016,9 +1016,11 @@ struct task_struct {
* p->on_rq = 0; p->sched_remote_wakeup = Y;
*
* guarantees all stores of 'current' are visible before
- * ->sched_remote_wakeup gets used, so it can be in this word.
+ * ->sched_remote_wakeup and ->sched_rq_selected get used, so they can
+ * be in this word.
*/
unsigned sched_remote_wakeup:1;
+ unsigned sched_rq_selected:1;
#ifdef CONFIG_RT_MUTEXES
unsigned sched_rt_mutex:1;
#endif
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 0ab15f03cb94..d8495735166d 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -3975,13 +3975,20 @@ void sched_ttwu_pending(void *arg)
update_rq_clock(rq);
llist_for_each_entry_safe(p, t, llist, wake_entry.llist) {
+ int wake_flags = 0;
+
if (WARN_ON_ONCE(p->on_cpu))
smp_cond_load_acquire(&p->on_cpu, !VAL);
if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq)))
set_task_cpu(p, cpu_of(rq));
- ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf);
+ if (p->sched_remote_wakeup)
+ wake_flags |= WF_MIGRATED;
+ if (p->sched_rq_selected)
+ wake_flags |= WF_RQ_SELECTED;
+
+ ttwu_do_activate(rq, p, wake_flags, &rf);
}
/*
@@ -4025,6 +4032,7 @@ static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags
struct rq *rq = cpu_rq(cpu);
p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
+ p->sched_rq_selected = !!(wake_flags & WF_RQ_SELECTED);
WRITE_ONCE(rq->ttwu_pending, 1);
#ifdef CONFIG_SMP
--
2.56.0.257.gf7497c7d27-goog
^ permalink raw reply [flat|nested] 3+ messages in thread* [PATCH 2/2] selftests/sched_ext: Test SCX_ENQ_CPU_SELECTED with queued wakeups
2026-10-23 14:48 [PATCH 0/2] sched: Fix SCX_ENQ_CPU_SELECTED for queued wakeups Kuba Piecuch
2026-10-23 14:48 ` [PATCH 1/2] sched/core: Preserve WF_RQ_SELECTED across " Kuba Piecuch
@ 2026-10-23 14:48 ` Kuba Piecuch
1 sibling, 0 replies; 3+ messages in thread
From: Kuba Piecuch @ 2026-10-23 14:48 UTC (permalink / raw)
To: Tejun Heo, Peter Zijlstra
Cc: Kuba Piecuch, David Vernet, Andrea Righi, Changwoo Min,
sched-ext, linux-kernel
Add a test that checks that SCX_ENQ_CPU_SELECTED on a wakeup
ops.enqueue() is set if and only if ops.select_cpu() ran for that wakeup.
The scheduler sets SCX_OPS_ALLOW_QUEUED_WAKEUP so that queued wakeups
are covered too.
ops.select_cpu() records in task storage that it ran and on which CPU.
ops.enqueue() checks that record against SCX_ENQ_CPU_SELECTED. Pairs of
tasks ping-pong over pipes to generate the wakeups. An enqueue on a CPU
other than the one that ran ops.select_cpu() is counted as a queued
wakeup. The test is skipped if none are seen.
This test was written with the help of an AI coding assistant.
Signed-off-by: Kuba Piecuch <jpiecuch@google.com>
---
tools/testing/selftests/sched_ext/Makefile | 1 +
.../sched_ext/enq_cpu_selected.bpf.c | 107 +++++++++++++
.../selftests/sched_ext/enq_cpu_selected.c | 149 ++++++++++++++++++
3 files changed, 257 insertions(+)
create mode 100644 tools/testing/selftests/sched_ext/enq_cpu_selected.bpf.c
create mode 100644 tools/testing/selftests/sched_ext/enq_cpu_selected.c
diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile
index c5d3a2eaea7d..2e21e28426e2 100644
--- a/tools/testing/selftests/sched_ext/Makefile
+++ b/tools/testing/selftests/sched_ext/Makefile
@@ -175,6 +175,7 @@ auto-test-targets := \
ddsp_vtimelocal_fail \
dsp_local_on \
enable_cmask \
+ enq_cpu_selected \
enq_select_cpu \
exit \
hotplug \
diff --git a/tools/testing/selftests/sched_ext/enq_cpu_selected.bpf.c b/tools/testing/selftests/sched_ext/enq_cpu_selected.bpf.c
new file mode 100644
index 000000000000..80e40592332a
--- /dev/null
+++ b/tools/testing/selftests/sched_ext/enq_cpu_selected.bpf.c
@@ -0,0 +1,107 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Verify that SCX_ENQ_CPU_SELECTED is set on a wakeup ops.enqueue() if and
+ * only if ops.select_cpu() was called for that wakeup. The scheduler sets
+ * SCX_OPS_ALLOW_QUEUED_WAKEUP so that the wakeups which are delivered through
+ * the ttwu wakelist (and thus enqueued on a CPU other than the one that ran
+ * ops.select_cpu()) are covered too.
+ */
+
+#include <scx/common.bpf.h>
+
+char _license[] SEC("license") = "GPL";
+
+UEI_DEFINE(uei);
+
+struct task_ctx {
+ bool select_cpu_called;
+ s32 select_cpu_on;
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
+ __uint(map_flags, BPF_F_NO_PREALLOC);
+ __type(key, int);
+ __type(value, struct task_ctx);
+} task_ctx_stor SEC(".maps");
+
+/*
+ * nr_selected: wakeup enqueues preceded by ops.select_cpu()
+ * nr_queued: of those, enqueues on a CPU other than the one which ran
+ * ops.select_cpu(), i.e. delivered through the wakelist
+ * nr_missing: ops.select_cpu() was called, SCX_ENQ_CPU_SELECTED is clear
+ * nr_spurious: ops.select_cpu() wasn't called, SCX_ENQ_CPU_SELECTED is set
+ */
+u64 nr_selected, nr_queued, nr_missing, nr_spurious;
+
+s32 BPF_STRUCT_OPS(enq_cpu_selected_select_cpu, struct task_struct *p,
+ s32 prev_cpu, u64 wake_flags)
+{
+ struct task_ctx *tctx;
+
+ /* fork enqueues don't carry SCX_ENQ_WAKEUP, only track ttwu */
+ if (!(wake_flags & SCX_WAKE_TTWU))
+ return prev_cpu;
+
+ tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
+ if (tctx) {
+ tctx->select_cpu_called = true;
+ tctx->select_cpu_on = bpf_get_smp_processor_id();
+ }
+
+ return prev_cpu;
+}
+
+void BPF_STRUCT_OPS(enq_cpu_selected_enqueue, struct task_struct *p,
+ u64 enq_flags)
+{
+ struct task_ctx *tctx;
+
+ tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
+ if (tctx) {
+ bool called = tctx->select_cpu_called;
+ bool flagged = enq_flags & SCX_ENQ_CPU_SELECTED;
+
+ tctx->select_cpu_called = false;
+
+ if (enq_flags & SCX_ENQ_WAKEUP) {
+ if (called) {
+ __sync_fetch_and_add(&nr_selected, 1);
+ if (bpf_get_smp_processor_id() != tctx->select_cpu_on)
+ __sync_fetch_and_add(&nr_queued, 1);
+ }
+ if (called && !flagged)
+ __sync_fetch_and_add(&nr_missing, 1);
+ if (!called && flagged)
+ __sync_fetch_and_add(&nr_spurious, 1);
+ }
+ }
+
+ scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
+}
+
+s32 BPF_STRUCT_OPS(enq_cpu_selected_init_task, struct task_struct *p,
+ struct scx_init_task_args *args)
+{
+ if (!bpf_task_storage_get(&task_ctx_stor, p, 0,
+ BPF_LOCAL_STORAGE_GET_F_CREATE))
+ return -ENOMEM;
+
+ return 0;
+}
+
+void BPF_STRUCT_OPS(enq_cpu_selected_exit, struct scx_exit_info *ei)
+{
+ UEI_RECORD(uei, ei);
+}
+
+SEC(".struct_ops.link")
+struct sched_ext_ops enq_cpu_selected_ops = {
+ .select_cpu = (void *)enq_cpu_selected_select_cpu,
+ .enqueue = (void *)enq_cpu_selected_enqueue,
+ .init_task = (void *)enq_cpu_selected_init_task,
+ .exit = (void *)enq_cpu_selected_exit,
+ .flags = SCX_OPS_ALLOW_QUEUED_WAKEUP,
+ .name = "enq_cpu_selected",
+ .timeout_ms = 1000U,
+};
diff --git a/tools/testing/selftests/sched_ext/enq_cpu_selected.c b/tools/testing/selftests/sched_ext/enq_cpu_selected.c
new file mode 100644
index 000000000000..efcec91ce0d4
--- /dev/null
+++ b/tools/testing/selftests/sched_ext/enq_cpu_selected.c
@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Verify that SCX_ENQ_CPU_SELECTED reliably tells ops.enqueue() whether
+ * ops.select_cpu() was called, including for queued (wakelist) wakeups.
+ */
+#include <bpf/bpf.h>
+#include <scx/common.h>
+#include <signal.h>
+#include <stdio.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include "enq_cpu_selected.bpf.skel.h"
+#include "scx_test.h"
+
+#define NR_PAIRS 4
+#define WARMUP_MS 200
+#define RUN_MS 2000
+
+static enum scx_test_status setup(void **ctx)
+{
+ struct enq_cpu_selected *skel;
+
+ skel = enq_cpu_selected__open();
+ SCX_FAIL_IF(!skel, "Failed to open");
+ SCX_ENUM_INIT(skel);
+ SCX_FAIL_IF(enq_cpu_selected__load(skel), "Failed to load skel");
+
+ *ctx = skel;
+
+ return SCX_TEST_PASS;
+}
+
+/* Bounce a byte back and forth between two tasks to generate wakeups. */
+static void pingpong(int rfd, int wfd, bool initiator)
+{
+ char c = 0;
+
+ if (initiator && write(wfd, &c, 1) != 1)
+ exit(1);
+
+ while (true) {
+ if (read(rfd, &c, 1) != 1)
+ exit(1);
+ if (write(wfd, &c, 1) != 1)
+ exit(1);
+ }
+}
+
+static int spawn_pair(pid_t *pids)
+{
+ int a2b[2], b2a[2];
+
+ if (pipe(a2b) || pipe(b2a))
+ return -errno;
+
+ pids[0] = fork();
+ if (pids[0] < 0)
+ return -errno;
+ if (!pids[0])
+ pingpong(b2a[0], a2b[1], true);
+
+ pids[1] = fork();
+ if (pids[1] < 0)
+ return -errno;
+ if (!pids[1])
+ pingpong(a2b[0], b2a[1], false);
+
+ close(a2b[0]);
+ close(a2b[1]);
+ close(b2a[0]);
+ close(b2a[1]);
+
+ return 0;
+}
+
+static enum scx_test_status run(void *ctx)
+{
+ struct enq_cpu_selected *skel = ctx;
+ pid_t pids[NR_PAIRS * 2] = {};
+ enum scx_test_status ret = SCX_TEST_PASS;
+ struct bpf_link *link;
+ int i;
+
+ link = bpf_map__attach_struct_ops(skel->maps.enq_cpu_selected_ops);
+ SCX_FAIL_IF(!link, "Failed to attach scheduler");
+
+ for (i = 0; i < NR_PAIRS; i++) {
+ if (spawn_pair(&pids[i * 2])) {
+ SCX_ERR("Failed to spawn workers");
+ ret = SCX_TEST_FAIL;
+ goto out;
+ }
+ }
+
+ /* skip the wakeups racing the scheduler enable */
+ usleep(WARMUP_MS * 1000);
+ skel->bss->nr_selected = 0;
+ skel->bss->nr_queued = 0;
+ skel->bss->nr_missing = 0;
+ skel->bss->nr_spurious = 0;
+
+ usleep(RUN_MS * 1000);
+
+ fprintf(stderr, "selected=%llu queued=%llu missing=%llu spurious=%llu\n",
+ (unsigned long long)skel->bss->nr_selected,
+ (unsigned long long)skel->bss->nr_queued,
+ (unsigned long long)skel->bss->nr_missing,
+ (unsigned long long)skel->bss->nr_spurious);
+
+ if (skel->bss->nr_missing || skel->bss->nr_spurious) {
+ SCX_ERR("SCX_ENQ_CPU_SELECTED doesn't match ops.select_cpu() calls");
+ ret = SCX_TEST_FAIL;
+ } else if (!skel->bss->nr_queued) {
+ fprintf(stderr, "No queued wakeups observed, skipping\n");
+ ret = SCX_TEST_SKIP;
+ }
+
+ if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) {
+ SCX_ERR("Scheduler exited unexpectedly (kind=%llu)",
+ (unsigned long long)skel->data->uei.kind);
+ ret = SCX_TEST_FAIL;
+ }
+out:
+ for (i = 0; i < NR_PAIRS * 2; i++) {
+ if (pids[i] > 0) {
+ kill(pids[i], SIGKILL);
+ waitpid(pids[i], NULL, 0);
+ }
+ }
+ bpf_link__destroy(link);
+
+ return ret;
+}
+
+static void cleanup(void *ctx)
+{
+ struct enq_cpu_selected *skel = ctx;
+
+ enq_cpu_selected__destroy(skel);
+}
+
+struct scx_test enq_cpu_selected = {
+ .name = "enq_cpu_selected",
+ .description = "Verify SCX_ENQ_CPU_SELECTED on queued wakeups",
+ .setup = setup,
+ .run = run,
+ .cleanup = cleanup,
+};
+REGISTER_SCX_TEST(&enq_cpu_selected)
--
2.56.0.257.gf7497c7d27-goog
^ permalink raw reply [flat|nested] 3+ messages in thread