* [PATCH v2 1/2] sched/core: Preserve WF_RQ_SELECTED across queued wakeups
2026-10-26 11:44 [PATCH v2 0/2] sched: Fix SCX_ENQ_CPU_SELECTED for queued wakeups Kuba Piecuch
@ 2026-10-26 11:44 ` Kuba Piecuch
2026-10-26 11:44 ` [PATCH v2 2/2] selftests/sched_ext: Test SCX_ENQ_CPU_SELECTED with " Kuba Piecuch
1 sibling, 0 replies; 3+ messages in thread
From: Kuba Piecuch @ 2026-10-26 11:44 UTC (permalink / raw)
To: Tejun Heo, Peter Zijlstra
Cc: Kuba Piecuch, David Vernet, Andrea Righi, Changwoo Min,
sched-ext, linux-kernel
ENQUEUE_RQ_SELECTED tells a sched_class that ->select_task_rq() was
called for a task wakeup. sched_ext exposes this flag to BPF schedulers
as SCX_ENQ_CPU_SELECTED.
When a BPF scheduler sets SCX_OPS_ALLOW_QUEUED_WAKEUP, a wakeup can be
delivered through the ttwu wakelist in two ways:
1) from the p->on_cpu path, before ->select_task_rq() is called. In this
case WF_RQ_SELECTED is not set and SCX_ENQ_CPU_SELECTED is correctly
clear;
2) from ttwu_queue(), after ->select_task_rq() has been called. In this
case WF_RQ_SELECTED is set in wake_flags, but sched_ttwu_pending()
only carries WF_MIGRATED across the wakelist, so the flag is dropped
and ops.enqueue() is wrongly told that ops.select_cpu() was skipped.
This also results in undercounting SCX_EV_SELECT_CPU_FALLBACK events.
Fix case 2 by preserving WF_RQ_SELECTED across the wakelist in the same
way WF_MIGRATED is, so that ttwu_do_activate() passes
ENQUEUE_RQ_SELECTED to activate_task(). Case 1 is unaffected.
Fixes: 3539c6411a7c ("sched_ext: Implement SCX_OPS_ALLOW_QUEUED_WAKEUP")
Acked-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Andrea Righi <arighi@nvidia.com>
Signed-off-by: Kuba Piecuch <jpiecuch@google.com>
---
include/linux/sched.h | 6 ++++--
kernel/sched/core.c | 10 +++++++++-
2 files changed, 13 insertions(+), 3 deletions(-)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 12a7e094a3e6..9dabffea1c54 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1006,7 +1006,7 @@ struct task_struct {
/* Unserialized, strictly 'current' */
/*
- * This field must not be in the scheduler word above due to wakelist
+ * These fields must not be in the scheduler word above due to wakelist
* queueing no longer being serialized by p->on_cpu. However:
*
* p->XXX = X; ttwu()
@@ -1016,9 +1016,11 @@ struct task_struct {
* p->on_rq = 0; p->sched_remote_wakeup = Y;
*
* guarantees all stores of 'current' are visible before
- * ->sched_remote_wakeup gets used, so it can be in this word.
+ * ->sched_remote_wakeup and ->sched_rq_selected get used, so they can
+ * be in this word.
*/
unsigned sched_remote_wakeup:1;
+ unsigned sched_rq_selected:1;
#ifdef CONFIG_RT_MUTEXES
unsigned sched_rt_mutex:1;
#endif
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 0ab15f03cb94..d8495735166d 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -3975,13 +3975,20 @@ void sched_ttwu_pending(void *arg)
update_rq_clock(rq);
llist_for_each_entry_safe(p, t, llist, wake_entry.llist) {
+ int wake_flags = 0;
+
if (WARN_ON_ONCE(p->on_cpu))
smp_cond_load_acquire(&p->on_cpu, !VAL);
if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq)))
set_task_cpu(p, cpu_of(rq));
- ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf);
+ if (p->sched_remote_wakeup)
+ wake_flags |= WF_MIGRATED;
+ if (p->sched_rq_selected)
+ wake_flags |= WF_RQ_SELECTED;
+
+ ttwu_do_activate(rq, p, wake_flags, &rf);
}
/*
@@ -4025,6 +4032,7 @@ static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags
struct rq *rq = cpu_rq(cpu);
p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
+ p->sched_rq_selected = !!(wake_flags & WF_RQ_SELECTED);
WRITE_ONCE(rq->ttwu_pending, 1);
#ifdef CONFIG_SMP
--
2.56.0.257.gf7497c7d27-goog
^ permalink raw reply [flat|nested] 3+ messages in thread* [PATCH v2 2/2] selftests/sched_ext: Test SCX_ENQ_CPU_SELECTED with queued wakeups
2026-10-26 11:44 [PATCH v2 0/2] sched: Fix SCX_ENQ_CPU_SELECTED for queued wakeups Kuba Piecuch
2026-10-26 11:44 ` [PATCH v2 1/2] sched/core: Preserve WF_RQ_SELECTED across " Kuba Piecuch
@ 2026-10-26 11:44 ` Kuba Piecuch
1 sibling, 0 replies; 3+ messages in thread
From: Kuba Piecuch @ 2026-10-26 11:44 UTC (permalink / raw)
To: Tejun Heo, Peter Zijlstra
Cc: Kuba Piecuch, David Vernet, Andrea Righi, Changwoo Min,
sched-ext, linux-kernel
Add a test that checks that SCX_ENQ_CPU_SELECTED on a wakeup
ops.enqueue() is set if and only if ops.select_cpu() ran for that wakeup.
The scheduler sets SCX_OPS_ALLOW_QUEUED_WAKEUP so that queued wakeups
are covered too.
ops.select_cpu() records in task storage that it ran and on which CPU.
ops.enqueue() checks that record against SCX_ENQ_CPU_SELECTED. Pairs of
tasks ping-pong over pipes to generate the wakeups. Each task is
affined to its own two CPUs, so ops.select_cpu() is still called, but
the waker and the wakee never share a CPU and the wakee's CPU has
nothing else to run, which ttwu_queue_cond() requires for queueing
within an LLC. Tasks stay on prev_cpu and are dispatched to the local
DSQ. An enqueue on a CPU other than the one that ran ops.select_cpu() is
counted as a queued wakeup. The test is skipped if there are fewer than
four CPUs or if no queued wakeups are seen.
This test was written with the help of an AI coding assistant.
Signed-off-by: Kuba Piecuch <jpiecuch@google.com>
---
tools/testing/selftests/sched_ext/Makefile | 1 +
.../sched_ext/enq_cpu_selected.bpf.c | 111 ++++++++++
.../selftests/sched_ext/enq_cpu_selected.c | 194 ++++++++++++++++++
3 files changed, 306 insertions(+)
create mode 100644 tools/testing/selftests/sched_ext/enq_cpu_selected.bpf.c
create mode 100644 tools/testing/selftests/sched_ext/enq_cpu_selected.c
diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile
index c5d3a2eaea7d..2e21e28426e2 100644
--- a/tools/testing/selftests/sched_ext/Makefile
+++ b/tools/testing/selftests/sched_ext/Makefile
@@ -175,6 +175,7 @@ auto-test-targets := \
ddsp_vtimelocal_fail \
dsp_local_on \
enable_cmask \
+ enq_cpu_selected \
enq_select_cpu \
exit \
hotplug \
diff --git a/tools/testing/selftests/sched_ext/enq_cpu_selected.bpf.c b/tools/testing/selftests/sched_ext/enq_cpu_selected.bpf.c
new file mode 100644
index 000000000000..5b5df2b0bdcd
--- /dev/null
+++ b/tools/testing/selftests/sched_ext/enq_cpu_selected.bpf.c
@@ -0,0 +1,111 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Verify that SCX_ENQ_CPU_SELECTED is set on a wakeup ops.enqueue() if and
+ * only if ops.select_cpu() was called for that wakeup. The scheduler sets
+ * SCX_OPS_ALLOW_QUEUED_WAKEUP so that the wakeups which are delivered through
+ * the ttwu wakelist (and thus enqueued on a CPU other than the one that ran
+ * ops.select_cpu()) are covered too.
+ *
+ * Tasks stay on prev_cpu and run from its local DSQ, so a wakeup crosses CPUs
+ * whenever the waker and the wakee have disjoint affinities, which the
+ * userspace part of the test arranges.
+ */
+
+#include <scx/common.bpf.h>
+
+char _license[] SEC("license") = "GPL";
+
+UEI_DEFINE(uei);
+
+struct task_ctx {
+ bool select_cpu_called;
+ s32 select_cpu_on;
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
+ __uint(map_flags, BPF_F_NO_PREALLOC);
+ __type(key, int);
+ __type(value, struct task_ctx);
+} task_ctx_stor SEC(".maps");
+
+/*
+ * nr_selected: wakeup enqueues preceded by ops.select_cpu()
+ * nr_queued: of those, enqueues on a CPU other than the one which ran
+ * ops.select_cpu(), i.e. delivered through the wakelist
+ * nr_missing: ops.select_cpu() was called, SCX_ENQ_CPU_SELECTED is clear
+ * nr_spurious: ops.select_cpu() wasn't called, SCX_ENQ_CPU_SELECTED is set
+ */
+u64 nr_selected, nr_queued, nr_missing, nr_spurious;
+
+s32 BPF_STRUCT_OPS(enq_cpu_selected_select_cpu, struct task_struct *p,
+ s32 prev_cpu, u64 wake_flags)
+{
+ struct task_ctx *tctx;
+
+ /* fork enqueues don't carry SCX_ENQ_WAKEUP, only track ttwu */
+ if (!(wake_flags & SCX_WAKE_TTWU))
+ return prev_cpu;
+
+ tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
+ if (tctx) {
+ tctx->select_cpu_called = true;
+ tctx->select_cpu_on = bpf_get_smp_processor_id();
+ }
+
+ return prev_cpu;
+}
+
+void BPF_STRUCT_OPS(enq_cpu_selected_enqueue, struct task_struct *p,
+ u64 enq_flags)
+{
+ struct task_ctx *tctx;
+
+ tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
+ if (tctx) {
+ bool called = tctx->select_cpu_called;
+ bool flagged = enq_flags & SCX_ENQ_CPU_SELECTED;
+
+ tctx->select_cpu_called = false;
+
+ if (enq_flags & SCX_ENQ_WAKEUP) {
+ if (called) {
+ __sync_fetch_and_add(&nr_selected, 1);
+ if (bpf_get_smp_processor_id() != tctx->select_cpu_on)
+ __sync_fetch_and_add(&nr_queued, 1);
+ }
+ if (called && !flagged)
+ __sync_fetch_and_add(&nr_missing, 1);
+ if (!called && flagged)
+ __sync_fetch_and_add(&nr_spurious, 1);
+ }
+ }
+
+ scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags);
+}
+
+s32 BPF_STRUCT_OPS(enq_cpu_selected_init_task, struct task_struct *p,
+ struct scx_init_task_args *args)
+{
+ if (!bpf_task_storage_get(&task_ctx_stor, p, 0,
+ BPF_LOCAL_STORAGE_GET_F_CREATE))
+ return -ENOMEM;
+
+ return 0;
+}
+
+void BPF_STRUCT_OPS(enq_cpu_selected_exit, struct scx_exit_info *ei)
+{
+ UEI_RECORD(uei, ei);
+}
+
+SEC(".struct_ops.link")
+struct sched_ext_ops enq_cpu_selected_ops = {
+ .select_cpu = (void *)enq_cpu_selected_select_cpu,
+ .enqueue = (void *)enq_cpu_selected_enqueue,
+ .init_task = (void *)enq_cpu_selected_init_task,
+ .exit = (void *)enq_cpu_selected_exit,
+ .flags = SCX_OPS_ALLOW_QUEUED_WAKEUP,
+ .name = "enq_cpu_selected",
+ .timeout_ms = 1000U,
+};
diff --git a/tools/testing/selftests/sched_ext/enq_cpu_selected.c b/tools/testing/selftests/sched_ext/enq_cpu_selected.c
new file mode 100644
index 000000000000..674c855530ed
--- /dev/null
+++ b/tools/testing/selftests/sched_ext/enq_cpu_selected.c
@@ -0,0 +1,194 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Verify that SCX_ENQ_CPU_SELECTED reliably tells ops.enqueue() whether
+ * ops.select_cpu() was called, including for queued (wakelist) wakeups.
+ */
+#include <bpf/bpf.h>
+#include <scx/common.h>
+#include <sched.h>
+#include <signal.h>
+#include <stdio.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include "enq_cpu_selected.bpf.skel.h"
+#include "scx_test.h"
+
+#define MAX_PAIRS 4
+/* ops.select_cpu() is skipped for tasks that can only run on one CPU */
+#define CPUS_PER_TASK 2
+#define WARMUP_MS 200
+#define RUN_MS 2000
+
+static int nr_pairs;
+static cpu_set_t masks[MAX_PAIRS * 2];
+
+/*
+ * Give each task its own CPUS_PER_TASK CPUs so that every wakeup crosses CPUs
+ * and the wakee's CPU has nothing else to run. ttwu_queue_cond() requires the
+ * latter for queueing a wakeup between CPUs sharing the LLC.
+ */
+static int setup_masks(void)
+{
+ int i, cpu, nr_cpus = 0;
+ cpu_set_t allowed;
+
+ if (sched_getaffinity(0, sizeof(allowed), &allowed))
+ return -errno;
+
+ for (i = 0; i < MAX_PAIRS * 2; i++)
+ CPU_ZERO(&masks[i]);
+
+ for (cpu = 0; cpu < CPU_SETSIZE; cpu++) {
+ if (nr_cpus == MAX_PAIRS * 2 * CPUS_PER_TASK)
+ break;
+ if (!CPU_ISSET(cpu, &allowed))
+ continue;
+ CPU_SET(cpu, &masks[nr_cpus / CPUS_PER_TASK]);
+ nr_cpus++;
+ }
+
+ return nr_cpus / (2 * CPUS_PER_TASK);
+}
+
+static enum scx_test_status setup(void **ctx)
+{
+ struct enq_cpu_selected *skel;
+
+ nr_pairs = setup_masks();
+ SCX_FAIL_IF(nr_pairs < 0, "Failed to get CPU affinity");
+ if (!nr_pairs) {
+ fprintf(stderr, "Needs at least %d CPUs, skipping\n",
+ 2 * CPUS_PER_TASK);
+ return SCX_TEST_SKIP;
+ }
+
+ skel = enq_cpu_selected__open();
+ SCX_FAIL_IF(!skel, "Failed to open");
+ SCX_ENUM_INIT(skel);
+ SCX_FAIL_IF(enq_cpu_selected__load(skel), "Failed to load skel");
+
+ *ctx = skel;
+
+ return SCX_TEST_PASS;
+}
+
+/* Bounce a byte back and forth between two tasks to generate wakeups. */
+static void pingpong(int rfd, int wfd, bool initiator, const cpu_set_t *mask)
+{
+ char c = 0;
+
+ if (sched_setaffinity(0, sizeof(*mask), mask))
+ exit(1);
+
+ if (initiator && write(wfd, &c, 1) != 1)
+ exit(1);
+
+ while (true) {
+ if (read(rfd, &c, 1) != 1)
+ exit(1);
+ if (write(wfd, &c, 1) != 1)
+ exit(1);
+ }
+}
+
+static int spawn_pair(pid_t *pids, const cpu_set_t *pair_masks)
+{
+ int a2b[2], b2a[2];
+
+ if (pipe(a2b) || pipe(b2a))
+ return -errno;
+
+ pids[0] = fork();
+ if (pids[0] < 0)
+ return -errno;
+ if (!pids[0])
+ pingpong(b2a[0], a2b[1], true, &pair_masks[0]);
+
+ pids[1] = fork();
+ if (pids[1] < 0)
+ return -errno;
+ if (!pids[1])
+ pingpong(a2b[0], b2a[1], false, &pair_masks[1]);
+
+ close(a2b[0]);
+ close(a2b[1]);
+ close(b2a[0]);
+ close(b2a[1]);
+
+ return 0;
+}
+
+static enum scx_test_status run(void *ctx)
+{
+ struct enq_cpu_selected *skel = ctx;
+ pid_t pids[MAX_PAIRS * 2] = {};
+ enum scx_test_status ret = SCX_TEST_PASS;
+ struct bpf_link *link;
+ int i;
+
+ link = bpf_map__attach_struct_ops(skel->maps.enq_cpu_selected_ops);
+ SCX_FAIL_IF(!link, "Failed to attach scheduler");
+
+ for (i = 0; i < nr_pairs; i++) {
+ if (spawn_pair(&pids[i * 2], &masks[i * 2])) {
+ SCX_ERR("Failed to spawn workers");
+ ret = SCX_TEST_FAIL;
+ goto out;
+ }
+ }
+
+ /* skip the wakeups racing the scheduler enable */
+ usleep(WARMUP_MS * 1000);
+ skel->bss->nr_selected = 0;
+ skel->bss->nr_queued = 0;
+ skel->bss->nr_missing = 0;
+ skel->bss->nr_spurious = 0;
+
+ usleep(RUN_MS * 1000);
+
+ fprintf(stderr, "selected=%llu queued=%llu missing=%llu spurious=%llu\n",
+ (unsigned long long)skel->bss->nr_selected,
+ (unsigned long long)skel->bss->nr_queued,
+ (unsigned long long)skel->bss->nr_missing,
+ (unsigned long long)skel->bss->nr_spurious);
+
+ if (skel->bss->nr_missing || skel->bss->nr_spurious) {
+ SCX_ERR("SCX_ENQ_CPU_SELECTED doesn't match ops.select_cpu() calls");
+ ret = SCX_TEST_FAIL;
+ } else if (!skel->bss->nr_queued) {
+ fprintf(stderr, "No queued wakeups observed, skipping\n");
+ ret = SCX_TEST_SKIP;
+ }
+
+ if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) {
+ SCX_ERR("Scheduler exited unexpectedly (kind=%llu)",
+ (unsigned long long)skel->data->uei.kind);
+ ret = SCX_TEST_FAIL;
+ }
+out:
+ for (i = 0; i < nr_pairs * 2; i++) {
+ if (pids[i] > 0) {
+ kill(pids[i], SIGKILL);
+ waitpid(pids[i], NULL, 0);
+ }
+ }
+ bpf_link__destroy(link);
+
+ return ret;
+}
+
+static void cleanup(void *ctx)
+{
+ struct enq_cpu_selected *skel = ctx;
+
+ enq_cpu_selected__destroy(skel);
+}
+
+struct scx_test enq_cpu_selected = {
+ .name = "enq_cpu_selected",
+ .description = "Verify SCX_ENQ_CPU_SELECTED on queued wakeups",
+ .setup = setup,
+ .run = run,
+ .cleanup = cleanup,
+};
+REGISTER_SCX_TEST(&enq_cpu_selected)
--
2.56.0.257.gf7497c7d27-goog
^ permalink raw reply [flat|nested] 3+ messages in thread