* [RFC PATCH 01/16] sched/core: Break activation of blocked task into a separate helper
2026-08-26 6:28 [RFC PATCH 00/16][PoC] sched/core: Alternate approach to sleeping-owner handling in PROXY_EXEC K Prateek Nayak
@ 2026-08-26 6:28 ` K Prateek Nayak
2026-08-26 6:28 ` [RFC PATCH 02/16] sched/core: Use enqueue/dequeue flags instead of task_on_rq_migrating() K Prateek Nayak
` (14 subsequent siblings)
15 siblings, 0 replies; 23+ messages in thread
From: K Prateek Nayak @ 2026-08-26 6:28 UTC (permalink / raw)
To: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
Andrea Righi, linux-kernel
Cc: Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, Waiman Long, K Prateek Nayak
Add activate_blocked_task() helper to activate a blocked task
(Blocked task is !p->on_rq || p->se.sched_delayed).
This is preparatory patch to clean up the task_on_rq_migrating() usage
in on the enqueue path in the sched_class callbacks to make way for
proxy activation on the enqueue path.
No functional changes intended.
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
kernel/sched/core.c | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 1bf60d78c920..7d1ffcc33fed 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2216,7 +2216,7 @@ inline bool dequeue_task(struct rq *rq, struct task_struct *p, int flags)
return p->sched_class->dequeue_task(rq, p, flags);
}
-void activate_task(struct rq *rq, struct task_struct *p, int flags)
+static void __activate_task(struct rq *rq, struct task_struct *p, int flags)
{
if (task_on_rq_migrating(p))
flags |= ENQUEUE_MIGRATED;
@@ -2227,6 +2227,16 @@ void activate_task(struct rq *rq, struct task_struct *p, int flags)
ASSERT_EXCLUSIVE_WRITER(p->on_rq);
}
+void activate_task(struct rq *rq, struct task_struct *p, int en_flags)
+{
+ __activate_task(rq, p, en_flags);
+}
+
+static void activate_blocked_task(struct rq *rq, struct task_struct *p, int en_flags)
+{
+ __activate_task(rq, p, en_flags);
+}
+
void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
{
WARN_ON_ONCE(flags & DEQUEUE_SLEEP);
@@ -3822,7 +3832,7 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
atomic_dec(&task_rq(p)->nr_iowait);
}
- activate_task(rq, p, en_flags);
+ activate_blocked_task(rq, p, en_flags);
wakeup_preempt(rq, p, wake_flags);
ttwu_do_wakeup(p);
@@ -4961,7 +4971,7 @@ void wake_up_new_task(struct task_struct *p)
update_rq_clock(rq);
post_init_entity_util_avg(p);
- activate_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_INITIAL);
+ activate_blocked_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_INITIAL);
trace_sched_wakeup_new(p);
wakeup_preempt(rq, p, wake_flags);
if (p->sched_class->task_woken) {
--
2.34.1
^ permalink raw reply [flat|nested] 23+ messages in thread* [RFC PATCH 02/16] sched/core: Use enqueue/dequeue flags instead of task_on_rq_migrating()
2026-08-26 6:28 [RFC PATCH 00/16][PoC] sched/core: Alternate approach to sleeping-owner handling in PROXY_EXEC K Prateek Nayak
2026-08-26 6:28 ` [RFC PATCH 01/16] sched/core: Break activation of blocked task into a separate helper K Prateek Nayak
@ 2026-08-26 6:28 ` K Prateek Nayak
2026-08-26 6:28 ` [RFC PATCH 03/16] sched/fair: Use enqueue flags for DO_ATTACH in update_load_avg() K Prateek Nayak
` (13 subsequent siblings)
15 siblings, 0 replies; 23+ messages in thread
From: K Prateek Nayak @ 2026-08-26 6:28 UTC (permalink / raw)
To: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
Andrea Righi, linux-kernel
Cc: Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, Waiman Long, K Prateek Nayak
With the blocked task activation path converted to use a separate
helper, add the DEQUEUE_MIGRATING and ENQUEUE_MIGRATING flag in
deactivate_task() and activate_task().
Avoid checking task_on_rq_migrating() in the sched class callbacks and
instead depend on flags to know if the enqueue/dequeue is a part of a
task migration.
No functional changes intended.
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
kernel/sched/core.c | 7 ++-----
kernel/sched/deadline.c | 6 ------
kernel/sched/fair.c | 6 +++---
kernel/sched/stats.h | 2 +-
4 files changed, 6 insertions(+), 15 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 7d1ffcc33fed..4cd69b08b415 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2218,9 +2218,6 @@ inline bool dequeue_task(struct rq *rq, struct task_struct *p, int flags)
static void __activate_task(struct rq *rq, struct task_struct *p, int flags)
{
- if (task_on_rq_migrating(p))
- flags |= ENQUEUE_MIGRATED;
-
enqueue_task(rq, p, flags);
WRITE_ONCE(p->on_rq, TASK_ON_RQ_QUEUED);
@@ -2229,7 +2226,7 @@ static void __activate_task(struct rq *rq, struct task_struct *p, int flags)
void activate_task(struct rq *rq, struct task_struct *p, int en_flags)
{
- __activate_task(rq, p, en_flags);
+ __activate_task(rq, p, en_flags | ENQUEUE_MIGRATING);
}
static void activate_blocked_task(struct rq *rq, struct task_struct *p, int en_flags)
@@ -2249,7 +2246,7 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
* dequeue_task() and cleared *after* enqueue_task().
*/
- dequeue_task(rq, p, flags);
+ dequeue_task(rq, p, flags | DEQUEUE_MIGRATING);
}
static void block_task(struct rq *rq, struct task_struct *p, unsigned long task_state)
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 3e89b3abeb27..9abda560c633 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -2530,9 +2530,6 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags)
check_schedstat_required();
update_stats_wait_start_dl(dl_rq, dl_se);
- if (task_on_rq_migrating(p))
- flags |= ENQUEUE_MIGRATING;
-
enqueue_dl_entity(dl_se, flags);
if (dl_server(dl_se))
@@ -2552,9 +2549,6 @@ static bool dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags)
{
update_curr_dl(rq);
- if (task_on_rq_migrating(p))
- flags |= DEQUEUE_MIGRATING;
-
dequeue_dl_entity(&p->dl, flags);
if (!p->dl.dl_throttled && !dl_server(&p->dl))
dequeue_pushable_dl_task(rq, p);
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 6d2fa0cd7777..244f1c68c041 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -6265,7 +6265,7 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
action = UPDATE_TG;
if (entity_is_task(se)) {
- if (task_on_rq_migrating(task_of(se)))
+ if (flags & DEQUEUE_MIGRATING)
action |= DO_DETACH;
if (sleep && !(flags & DEQUEUE_DELAYED))
@@ -6663,7 +6663,7 @@ static void dequeue_throttled_task(struct task_struct *p, int flags)
* task is migrating off its old cfs_rq, detach
* the task's load from its old cfs_rq.
*/
- if (task_on_rq_migrating(p))
+ if (flags & DEQUEUE_MIGRATING)
detach_task_cfs_rq(p);
}
@@ -7981,7 +7981,7 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags)
break;
}
flags |= DEQUEUE_SLEEP;
- flags &= ~(DEQUEUE_DELAYED | DEQUEUE_SPECIAL);
+ flags &= ~(DEQUEUE_DELAYED | DEQUEUE_SPECIAL | DEQUEUE_MIGRATING);
}
for_each_sched_entity(se) {
diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h
index ebe0a7765f98..d0d9cff168a2 100644
--- a/kernel/sched/stats.h
+++ b/kernel/sched/stats.h
@@ -137,7 +137,7 @@ static inline void psi_enqueue(struct task_struct *p, int flags)
if (p->se.sched_delayed) {
/* CPU migration of "sleeping" task */
- WARN_ON_ONCE(!(flags & ENQUEUE_MIGRATED));
+ WARN_ON_ONCE(!(flags & ENQUEUE_MIGRATING));
if (p->in_memstall)
set |= TSK_MEMSTALL;
if (p->in_iowait)
--
2.34.1
^ permalink raw reply [flat|nested] 23+ messages in thread* [RFC PATCH 03/16] sched/fair: Use enqueue flags for DO_ATTACH in update_load_avg()
2026-08-26 6:28 [RFC PATCH 00/16][PoC] sched/core: Alternate approach to sleeping-owner handling in PROXY_EXEC K Prateek Nayak
2026-08-26 6:28 ` [RFC PATCH 01/16] sched/core: Break activation of blocked task into a separate helper K Prateek Nayak
2026-08-26 6:28 ` [RFC PATCH 02/16] sched/core: Use enqueue/dequeue flags instead of task_on_rq_migrating() K Prateek Nayak
@ 2026-08-26 6:28 ` K Prateek Nayak
2026-08-26 15:36 ` Andrea Righi
2026-08-26 6:28 ` [RFC PATCH 04/16] sched/core: Activate blocked donor when no owner is found K Prateek Nayak
` (12 subsequent siblings)
15 siblings, 1 reply; 23+ messages in thread
From: K Prateek Nayak @ 2026-08-26 6:28 UTC (permalink / raw)
To: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
Andrea Righi, linux-kernel
Cc: Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, Waiman Long, K Prateek Nayak
proxy_set_cpu() skips calling ->migrate_task_rq() callback for blocked
donor migrations and can lead to incorrect PELT accounting.
enqueue_task() -> update_load_avg() depends on migrate_task_rq_fair() to
clear the p->se.avg.last_update_time and use this as a gating condition
to do DO_ATTACH which is skipped for blocked donors.
Use migrate instead of se->avg.last_update_time indicator for adding
DO_ATTACH action. Add a defensive WARN_ON_ONCE() to catch any cases that
are not covered by the MIGRATING | MIGRATED | INITIAL flags.
Fixes: b049b81bdff6 ("sched: Handle blocked-waiter migration (and return migration)")
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
kernel/sched/fair.c | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 244f1c68c041..ed397c259509 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -5654,7 +5654,7 @@ static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s
decayed = update_cfs_rq_load_avg(now, cfs_rq);
decayed |= propagate_entity_load_avg(se);
- if (!se->avg.last_update_time && (flags & DO_ATTACH)) {
+ if (flags & DO_ATTACH) {
/*
* DO_ATTACH means we're here from enqueue_entity().
@@ -6110,6 +6110,7 @@ static void
enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
{
bool curr = cfs_rq->curr == se;
+ int action = UPDATE_TG;
/*
* If we're the current task, we must renormalise before calling
@@ -6120,6 +6121,11 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
update_curr(cfs_rq);
+ if (flags & (ENQUEUE_MIGRATING | ENQUEUE_MIGRATED | ENQUEUE_INITIAL))
+ action |= DO_ATTACH;
+ else
+ WARN_ON_ONCE(!se->avg.last_update_time);
+
/*
* When enqueuing a sched_entity, we must:
* - Update loads to have both entity and cfs_rq synced with now.
@@ -6129,7 +6135,7 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
* its group cfs_rq
* - Add its new weight to cfs_rq->load.weight
*/
- update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH);
+ update_load_avg(cfs_rq, se, action);
se_update_runnable(se);
/*
* XXX update_load_avg() above will have attached us to the pelt sum;
--
2.34.1
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [RFC PATCH 03/16] sched/fair: Use enqueue flags for DO_ATTACH in update_load_avg()
2026-08-26 6:28 ` [RFC PATCH 03/16] sched/fair: Use enqueue flags for DO_ATTACH in update_load_avg() K Prateek Nayak
@ 2026-08-26 15:36 ` Andrea Righi
0 siblings, 0 replies; 23+ messages in thread
From: Andrea Righi @ 2026-08-26 15:36 UTC (permalink / raw)
To: K Prateek Nayak
Cc: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
linux-kernel, Dietmar Eggemann, Steven Rostedt, Ben Segall,
Mel Gorman, Valentin Schneider, Waiman Long
Hi Prateek,
On Wed, Aug 26, 2026 at 06:28:47AM +0000, K Prateek Nayak wrote:
> proxy_set_cpu() skips calling ->migrate_task_rq() callback for blocked
> donor migrations and can lead to incorrect PELT accounting.
>
> enqueue_task() -> update_load_avg() depends on migrate_task_rq_fair() to
> clear the p->se.avg.last_update_time and use this as a gating condition
> to do DO_ATTACH which is skipped for blocked donors.
>
> Use migrate instead of se->avg.last_update_time indicator for adding
> DO_ATTACH action. Add a defensive WARN_ON_ONCE() to catch any cases that
> are not covered by the MIGRATING | MIGRATED | INITIAL flags.
>
> Fixes: b049b81bdff6 ("sched: Handle blocked-waiter migration (and return migration)")
> Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
> ---
> kernel/sched/fair.c | 10 ++++++++--
> 1 file changed, 8 insertions(+), 2 deletions(-)
>
> diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
> index 244f1c68c041..ed397c259509 100644
> --- a/kernel/sched/fair.c
> +++ b/kernel/sched/fair.c
> @@ -5654,7 +5654,7 @@ static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s
> decayed = update_cfs_rq_load_avg(now, cfs_rq);
> decayed |= propagate_entity_load_avg(se);
>
> - if (!se->avg.last_update_time && (flags & DO_ATTACH)) {
> + if (flags & DO_ATTACH) {
>
> /*
> * DO_ATTACH means we're here from enqueue_entity().
nit: the comment below still says:
* !last_update_time means we've passed through
* migrate_task_rq_fair() indicating we migrated.
We should probably update it, perhaps:
/*
* DO_ATTACH means the entity needs to be attached to this
* cfs_rq's load-tracking sums, either on initial enqueue
* or after migration.
*/
or something along these lines.
> @@ -6110,6 +6110,7 @@ static void
> enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
> {
> bool curr = cfs_rq->curr == se;
> + int action = UPDATE_TG;
>
> /*
> * If we're the current task, we must renormalise before calling
> @@ -6120,6 +6121,11 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
>
> update_curr(cfs_rq);
>
> + if (flags & (ENQUEUE_MIGRATING | ENQUEUE_MIGRATED | ENQUEUE_INITIAL))
> + action |= DO_ATTACH;
> + else
> + WARN_ON_ONCE(!se->avg.last_update_time);
> +
> /*
> * When enqueuing a sched_entity, we must:
> * - Update loads to have both entity and cfs_rq synced with now.
> @@ -6129,7 +6135,7 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
> * its group cfs_rq
> * - Add its new weight to cfs_rq->load.weight
> */
> - update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH);
> + update_load_avg(cfs_rq, se, action);
> se_update_runnable(se);
> /*
> * XXX update_load_avg() above will have attached us to the pelt sum;
> --
> 2.34.1
>
Thanks,
-Andrea
^ permalink raw reply [flat|nested] 23+ messages in thread
* [RFC PATCH 04/16] sched/core: Activate blocked donor when no owner is found
2026-08-26 6:28 [RFC PATCH 00/16][PoC] sched/core: Alternate approach to sleeping-owner handling in PROXY_EXEC K Prateek Nayak
` (2 preceding siblings ...)
2026-08-26 6:28 ` [RFC PATCH 03/16] sched/fair: Use enqueue flags for DO_ATTACH in update_load_avg() K Prateek Nayak
@ 2026-08-26 6:28 ` K Prateek Nayak
2026-08-26 15:56 ` Andrea Righi
2026-08-26 6:28 ` [RFC PATCH 05/16] sched/core: Do not queue blocked donor on a delayed owner K Prateek Nayak
` (11 subsequent siblings)
15 siblings, 1 reply; 23+ messages in thread
From: K Prateek Nayak @ 2026-08-26 6:28 UTC (permalink / raw)
To: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
Andrea Righi, linux-kernel
Cc: Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, Waiman Long, K Prateek Nayak
mutex_unlock_slowpath() follows:
if (owner & MUTEX_FLAG_HANDOFF)
break /* ... and do __mutex_handoff() */
if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, __owner_flags(owner))) {
if (owner & MUTEX_FLAG_WAITERS)
break; /* ... and wake up the forst waiter. */
MUTEX_FLAG_HANDOFF is only set by first-waiter after it has been woken
up and in absence of MUTEX_FLAG_HANDOFF, the owner clears itself from
the lock_word and wakes up the first waiter to try a
__mutex_trylock_or_handoff().
MUTEX_FLAG_HANDOFF exists to prevent new optimistic spinners from
trying to hijack the lock from waiter all the time and potentially
starving them but it is not necessary for MUTEX_FLAG_HANDOFF to be
always set in presence of a waiter.
If a blocked donor is deactivated when no owner is observed, it may not
be woken up until it becomes the first waiter and is naturally woken up
which breaks proxy in the interim.
Wake up the blocked donor and allow it to grab the lock when no owner is
observed. If the task manages to grab the lock, the block chain will
follow at the next proxy migration. If the task fails to grab the lock,
same situation is restored and everyone migrated to the CPU of new
owner.
Fixes: f13beb010e4a ("sched: Have try_to_wake_up() handle return-migration for PROXY_WAKING case")
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
XXX: Is there a better way to handle this? If we can confirm a owner in
find_proxy_task(), we don't need to do a spurious wakeup of every task
observing !owner.
proxy_resched_idle() until owner appears in an option but it will spin
until next owner appears.
---
kernel/sched/core.c | 23 ++++++++++++++++++++---
1 file changed, 20 insertions(+), 3 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 4cd69b08b415..9d9db7ccf01f 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6809,6 +6809,19 @@ static inline void proxy_reacquire_rq_lock(struct rq *rq, struct rq_flags *rf)
update_rq_clock(rq);
}
+static void
+proxy_activate(struct rq *rq, struct rq_flags *rf, struct task_struct *p)
+ __must_hold(__rq_lockp(rq))
+{
+ lockdep_assert_rq_held(rq);
+ proxy_resched_idle(rq);
+ proxy_release_rq_lock(rq, rf);
+
+ wake_up_process(p);
+
+ proxy_reacquire_rq_lock(rq, rf);
+}
+
/*
* If the blocked-on relationship crosses CPUs, migrate @p to the
* owner's CPU.
@@ -6934,14 +6947,15 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
/*
* If there is no owner, either clear blocked_on
* and return p (if it is current and safe to
- * just run on this rq), or return-migrate the task.
+ * just run on this rq), or wake the task to try
+ * and grab the lock it is blocked on.
*/
__clear_task_blocked_on(p, NULL);
- if (task_current(rq, p)) {
+ if (task_current(rq, p) || p->wake_cpu == task_cpu(p)) {
p->is_blocked = 0;
return p;
}
- goto deactivate;
+ goto activate;
}
if (!READ_ONCE(owner->on_rq) || owner->se.sched_delayed) {
@@ -7029,6 +7043,9 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
}
return owner;
+activate:
+ proxy_activate(rq, rf, p);
+ return NULL;
deactivate:
proxy_deactivate(rq, p);
return NULL;
--
2.34.1
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [RFC PATCH 04/16] sched/core: Activate blocked donor when no owner is found
2026-08-26 6:28 ` [RFC PATCH 04/16] sched/core: Activate blocked donor when no owner is found K Prateek Nayak
@ 2026-08-26 15:56 ` Andrea Righi
2026-08-26 17:20 ` K Prateek Nayak
0 siblings, 1 reply; 23+ messages in thread
From: Andrea Righi @ 2026-08-26 15:56 UTC (permalink / raw)
To: K Prateek Nayak
Cc: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
linux-kernel, Dietmar Eggemann, Steven Rostedt, Ben Segall,
Mel Gorman, Valentin Schneider, Waiman Long
On Wed, Aug 26, 2026 at 06:28:48AM +0000, K Prateek Nayak wrote:
> mutex_unlock_slowpath() follows:
>
> if (owner & MUTEX_FLAG_HANDOFF)
> break /* ... and do __mutex_handoff() */
>
> if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, __owner_flags(owner))) {
> if (owner & MUTEX_FLAG_WAITERS)
> break; /* ... and wake up the forst waiter. */
nit: s/forst/first/
>
> MUTEX_FLAG_HANDOFF is only set by first-waiter after it has been woken
> up and in absence of MUTEX_FLAG_HANDOFF, the owner clears itself from
> the lock_word and wakes up the first waiter to try a
> __mutex_trylock_or_handoff().
>
> MUTEX_FLAG_HANDOFF exists to prevent new optimistic spinners from
> trying to hijack the lock from waiter all the time and potentially
> starving them but it is not necessary for MUTEX_FLAG_HANDOFF to be
> always set in presence of a waiter.
>
> If a blocked donor is deactivated when no owner is observed, it may not
> be woken up until it becomes the first waiter and is naturally woken up
> which breaks proxy in the interim.
>
> Wake up the blocked donor and allow it to grab the lock when no owner is
> observed. If the task manages to grab the lock, the block chain will
> follow at the next proxy migration. If the task fails to grab the lock,
> same situation is restored and everyone migrated to the CPU of new
> owner.
>
> Fixes: f13beb010e4a ("sched: Have try_to_wake_up() handle return-migration for PROXY_WAKING case")
> Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
> ---
> XXX: Is there a better way to handle this? If we can confirm a owner in
> find_proxy_task(), we don't need to do a spurious wakeup of every task
> observing !owner.
>
> proxy_resched_idle() until owner appears in an option but it will spin
> until next owner appears.
IIUC, the owner can remain NULL until the waiter selected by mutex_unlock() gets
CPU time and acquires the mutex, so proxy_resched_idle() could spin for longer
than just the unlock critical section.
Maybe we could instead force a handoff from mutex_unlock_slowpath() when proxy
execution is enabled and the mutex has waiters? This would keep the owner
identifiable and avoid waking every task that happens to observe !owner.
> ---
> kernel/sched/core.c | 23 ++++++++++++++++++++---
> 1 file changed, 20 insertions(+), 3 deletions(-)
>
> diff --git a/kernel/sched/core.c b/kernel/sched/core.c
> index 4cd69b08b415..9d9db7ccf01f 100644
> --- a/kernel/sched/core.c
> +++ b/kernel/sched/core.c
> @@ -6809,6 +6809,19 @@ static inline void proxy_reacquire_rq_lock(struct rq *rq, struct rq_flags *rf)
> update_rq_clock(rq);
> }
>
> +static void
> +proxy_activate(struct rq *rq, struct rq_flags *rf, struct task_struct *p)
> + __must_hold(__rq_lockp(rq))
> +{
> + lockdep_assert_rq_held(rq);
> + proxy_resched_idle(rq);
> + proxy_release_rq_lock(rq, rf);
> +
> + wake_up_process(p);
> +
> + proxy_reacquire_rq_lock(rq, rf);
> +}
> +
> /*
> * If the blocked-on relationship crosses CPUs, migrate @p to the
> * owner's CPU.
> @@ -6934,14 +6947,15 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
> /*
> * If there is no owner, either clear blocked_on
> * and return p (if it is current and safe to
> - * just run on this rq), or return-migrate the task.
> + * just run on this rq), or wake the task to try
> + * and grab the lock it is blocked on.
> */
> __clear_task_blocked_on(p, NULL);
> - if (task_current(rq, p)) {
> + if (task_current(rq, p) || p->wake_cpu == task_cpu(p)) {
> p->is_blocked = 0;
> return p;
> }
> - goto deactivate;
> + goto activate;
> }
>
> if (!READ_ONCE(owner->on_rq) || owner->se.sched_delayed) {
> @@ -7029,6 +7043,9 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
> }
> return owner;
>
> +activate:
> + proxy_activate(rq, rf, p);
> + return NULL;
> deactivate:
> proxy_deactivate(rq, p);
> return NULL;
> --
> 2.34.1
>
Thanks,
-Andrea
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [RFC PATCH 04/16] sched/core: Activate blocked donor when no owner is found
2026-08-26 15:56 ` Andrea Righi
@ 2026-08-26 17:20 ` K Prateek Nayak
2026-08-28 6:04 ` K Prateek Nayak
0 siblings, 1 reply; 23+ messages in thread
From: K Prateek Nayak @ 2026-08-26 17:20 UTC (permalink / raw)
To: Andrea Righi
Cc: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
linux-kernel, Dietmar Eggemann, Steven Rostedt, Ben Segall,
Mel Gorman, Valentin Schneider, Waiman Long
Hello Andrea,
On 8/26/2026 9:26 PM, Andrea Righi wrote:
> On Wed, Aug 26, 2026 at 06:28:48AM +0000, K Prateek Nayak wrote:
>> mutex_unlock_slowpath() follows:
>>
>> if (owner & MUTEX_FLAG_HANDOFF)
>> break /* ... and do __mutex_handoff() */
>>
>> if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, __owner_flags(owner))) {
>> if (owner & MUTEX_FLAG_WAITERS)
>> break; /* ... and wake up the forst waiter. */
>
> nit: s/forst/first/
Ack
>
>>
>> MUTEX_FLAG_HANDOFF is only set by first-waiter after it has been woken
>> up and in absence of MUTEX_FLAG_HANDOFF, the owner clears itself from
>> the lock_word and wakes up the first waiter to try a
>> __mutex_trylock_or_handoff().
>>
>> MUTEX_FLAG_HANDOFF exists to prevent new optimistic spinners from
>> trying to hijack the lock from waiter all the time and potentially
>> starving them but it is not necessary for MUTEX_FLAG_HANDOFF to be
>> always set in presence of a waiter.
>>
>> If a blocked donor is deactivated when no owner is observed, it may not
>> be woken up until it becomes the first waiter and is naturally woken up
>> which breaks proxy in the interim.
>>
>> Wake up the blocked donor and allow it to grab the lock when no owner is
>> observed. If the task manages to grab the lock, the block chain will
>> follow at the next proxy migration. If the task fails to grab the lock,
>> same situation is restored and everyone migrated to the CPU of new
>> owner.
>>
>> Fixes: f13beb010e4a ("sched: Have try_to_wake_up() handle return-migration for PROXY_WAKING case")
>> Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
>> ---
>> XXX: Is there a better way to handle this? If we can confirm a owner in
>> find_proxy_task(), we don't need to do a spurious wakeup of every task
>> observing !owner.
>>
>> proxy_resched_idle() until owner appears in an option but it will spin
>> until next owner appears.
>
> IIUC, the owner can remain NULL until the waiter selected by mutex_unlock() gets
> CPU time and acquires the mutex, so proxy_resched_idle() could spin for longer
> than just the unlock critical section.
>
> Maybe we could instead force a handoff from mutex_unlock_slowpath() when proxy
> execution is enabled and the mutex has waiters? This would keep the owner
> identifiable and avoid waking every task that happens to observe !owner.
I think that negates some of the benefits of the optimistic spinning +
mutex_try_lock(). I'll see if it makes any difference to the benchmark
results if we always force a handoff for MUTEX_FLAG_WAITERS.
wait_lock should give enough guarantee that the waiter cannot simple
disappear before the handoff after MUTEX_FLAG_PICKUP is set since
waiter has to try at least one mutex_trylock() under wait_lock before
checking for pending signals.
Thank you for taking a look at the series and ack on the comment on
the previous patch.
>
>> ---
>> kernel/sched/core.c | 23 ++++++++++++++++++++---
>> 1 file changed, 20 insertions(+), 3 deletions(-)
>>
>> diff --git a/kernel/sched/core.c b/kernel/sched/core.c
>> index 4cd69b08b415..9d9db7ccf01f 100644
>> --- a/kernel/sched/core.c
>> +++ b/kernel/sched/core.c
>> @@ -6809,6 +6809,19 @@ static inline void proxy_reacquire_rq_lock(struct rq *rq, struct rq_flags *rf)
>> update_rq_clock(rq);
>> }
>>
>> +static void
>> +proxy_activate(struct rq *rq, struct rq_flags *rf, struct task_struct *p)
>> + __must_hold(__rq_lockp(rq))
>> +{
>> + lockdep_assert_rq_held(rq);
>> + proxy_resched_idle(rq);
>> + proxy_release_rq_lock(rq, rf);
>> +
>> + wake_up_process(p);
>> +
>> + proxy_reacquire_rq_lock(rq, rf);
>> +}
>> +
>> /*
>> * If the blocked-on relationship crosses CPUs, migrate @p to the
>> * owner's CPU.
>> @@ -6934,14 +6947,15 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
>> /*
>> * If there is no owner, either clear blocked_on
>> * and return p (if it is current and safe to
>> - * just run on this rq), or return-migrate the task.
>> + * just run on this rq), or wake the task to try
>> + * and grab the lock it is blocked on.
>> */
>> __clear_task_blocked_on(p, NULL);
>> - if (task_current(rq, p)) {
>> + if (task_current(rq, p) || p->wake_cpu == task_cpu(p)) {
>> p->is_blocked = 0;
>> return p;
>> }
>> - goto deactivate;
>> + goto activate;
>> }
>>
>> if (!READ_ONCE(owner->on_rq) || owner->se.sched_delayed) {
>> @@ -7029,6 +7043,9 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
>> }
>> return owner;
>>
>> +activate:
>> + proxy_activate(rq, rf, p);
>> + return NULL;
>> deactivate:
>> proxy_deactivate(rq, p);
>> return NULL;
--
Thanks and Regards,
Prateek
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [RFC PATCH 04/16] sched/core: Activate blocked donor when no owner is found
2026-08-26 17:20 ` K Prateek Nayak
@ 2026-08-28 6:04 ` K Prateek Nayak
0 siblings, 0 replies; 23+ messages in thread
From: K Prateek Nayak @ 2026-08-28 6:04 UTC (permalink / raw)
To: Andrea Righi, Peter Zijlstra, John Stultz
Cc: Suleiman Souhlal, Ingo Molnar, Juri Lelli, Vincent Guittot,
Will Deacon, Boqun Feng, linux-kernel, Dietmar Eggemann,
Steven Rostedt, Ben Segall, Mel Gorman, Valentin Schneider,
Waiman Long
Hello Andrea,
On 8/26/2026 10:50 PM, K Prateek Nayak wrote:
>>> XXX: Is there a better way to handle this? If we can confirm a owner in
>>> find_proxy_task(), we don't need to do a spurious wakeup of every task
>>> observing !owner.
>>>
>>> proxy_resched_idle() until owner appears in an option but it will spin
>>> until next owner appears.
>>
>> IIUC, the owner can remain NULL until the waiter selected by mutex_unlock() gets
>> CPU time and acquires the mutex, so proxy_resched_idle() could spin for longer
>> than just the unlock critical section.
>>
>> Maybe we could instead force a handoff from mutex_unlock_slowpath() when proxy
>> execution is enabled and the mutex has waiters? This would keep the owner
>> identifiable and avoid waking every task that happens to observe !owner.
>
> I think that negates some of the benefits of the optimistic spinning +
> mutex_try_lock(). I'll see if it makes any difference to the benchmark
> results if we always force a handoff for MUTEX_FLAG_WAITERS.
>
> wait_lock should give enough guarantee that the waiter cannot simple
> disappear before the handoff after MUTEX_FLAG_PICKUP is set since
> waiter has to try at least one mutex_trylock() under wait_lock before
> checking for pending signals.
Below are the results from few experiments. All diffs pasted below are
based on John's tree at:
https://github.com/johnstultz-work/linux-dev.git proxy-exec-v31-7.2-rc4
at commit 06ac43db4d8e ("[ANNOTATION] === Needs confirmation of
functionality past this point ===") with CONFIG_SCHED_PROXY_EXEC=y.
All diffs are very experimental: virtme-ng or testing with a disposable
environment is recommended.
============================
Experiment 1: Simple Handoff
============================
If I do a simple handoff like below, sched-messaging goes pretty bad:
diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index 8a85912d7ee6..da14a49e4fa2 100644
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -1009,7 +1009,7 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
MUTEX_WARN_ON(__owner_task(owner) != current);
MUTEX_WARN_ON(owner & MUTEX_FLAG_PICKUP);
- if (sched_proxy_exec() && current->blocked_donor) {
+ if (sched_proxy_exec() && (current->blocked_donor || (owner & MUTEX_FLAG_WAITERS))) {
/* force handoff if we have a blocked_donor */
owner = MUTEX_FLAG_HANDOFF;
break;
---
With just a simple handoff on waiters, we have:
==================================================================
Test : sched-messaging
Units : Normalized time in seconds
Interpretation: Lower is better
Statistic : AMean
==================================================================
Test: vanilla handoff
1-groups: 3.12 (0.00 pct) 3.47 (-11.21 pct) *
2-groups: 3.43 (0.00 pct) 4.33 (-26.23 pct) *
4-groups: 4.05 (0.00 pct) 5.95 (-46.91 pct)
8-groups: 4.29 (0.00 pct) 9.56 (-122.84 pct)
16-groups: 5.89 (0.00 pct) 12.29 (-108.65 pct)
* Data points have > 10% run to run variance on all versions
==================================================
Experiment 2: Allow steal until next task is found
==================================================
If we open the opportunity to allow stealing of mutex until the next waiter
is found, the results are ever so slightly slightly better:
diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index 8a85912d7ee6..5ebb2624b331 100644
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -92,7 +92,16 @@ static inline struct task_struct *__mutex_trylock_common(struct mutex *lock, boo
unsigned long task = owner & ~MUTEX_FLAGS;
if (task) {
- if (flags & MUTEX_FLAG_PICKUP) {
+ if (sched_proxy_exec() && (flags & MUTEX_FLAG_STEAL)) {
+ /*
+ * STEAL cannot be set after HANDOFF has been
+ * initiated. If STEAL is set, clear it and
+ * preserve other flags
+ */
+ MUTEX_WARN_ON(flags & (MUTEX_FLAG_PICKUP));
+ flags &= ~MUTEX_FLAG_STEAL;
+ task = curr;
+ }else if (flags & MUTEX_FLAG_PICKUP) {
if (task != curr)
break;
flags &= ~MUTEX_FLAG_PICKUP;
@@ -104,7 +113,7 @@ static inline struct task_struct *__mutex_trylock_common(struct mutex *lock, boo
break;
}
} else {
- MUTEX_WARN_ON(flags & (MUTEX_FLAG_HANDOFF | MUTEX_FLAG_PICKUP));
+ MUTEX_WARN_ON(flags & (MUTEX_FLAG_HANDOFF | MUTEX_FLAG_PICKUP | MUTEX_FLAG_STEAL));
task = curr;
}
@@ -274,7 +283,29 @@ static void __mutex_handoff(struct mutex *lock, struct task_struct *task)
new |= (unsigned long)task;
if (task)
new |= MUTEX_FLAG_PICKUP;
+ if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, new))
+ break;
+ }
+}
+
+static void __mutex_steal(struct mutex *lock, struct task_struct *task)
+{
+ unsigned long owner = atomic_long_read(&lock->owner);
+
+ for (;;) {
+ unsigned long new;
+ /* Lock was successfully stolen. */
+ if (__owner_task(owner) != current)
+ break;
+
+ MUTEX_WARN_ON(!(__owner_flags(owner) & MUTEX_FLAG_STEAL));
+ MUTEX_WARN_ON(owner & MUTEX_FLAG_PICKUP);
+
+ new = (owner & MUTEX_FLAG_WAITERS);
+ new |= (unsigned long)task;
+ if (task)
+ new |= MUTEX_FLAG_PICKUP;
if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, new))
break;
}
@@ -389,7 +420,17 @@ bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner,
lockdep_assert_preemption_disabled();
- while (__mutex_owner(lock) == owner) {
+ for (;;) {
+ unsigned long __owner = atomic_long_read(&lock->owner);
+
+ /* If the owner changed, break out. */
+ if (__owner_task(__owner) != owner)
+ break;
+
+ /* If lock can be stolen, break out. */
+ if (sched_proxy_exec() && (__owner_flags(__owner) & MUTEX_FLAG_STEAL))
+ break;
+
/*
* Ensure we emit the owner->on_cpu, dereference _after_
* checking lock->owner still matches owner. And we already
@@ -985,6 +1026,7 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
struct mutex_waiter *waiter;
unsigned long owner;
unsigned long flags;
+ bool steal;
mutex_release(&lock->dep_map, ip);
__release(lock);
@@ -1006,19 +1048,31 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
*/
owner = atomic_long_read(&lock->owner);
for (;;) {
+ unsigned long owner_flags;
+
MUTEX_WARN_ON(__owner_task(owner) != current);
MUTEX_WARN_ON(owner & MUTEX_FLAG_PICKUP);
- if (sched_proxy_exec() && current->blocked_donor) {
- /* force handoff if we have a blocked_donor */
- owner = MUTEX_FLAG_HANDOFF;
- break;
- }
-
if (owner & MUTEX_FLAG_HANDOFF)
break;
- if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, __owner_flags(owner))) {
+ owner_flags = __owner_flags(owner);
+ if (sched_proxy_exec()) {
+ if (current->blocked_donor) {
+ /* force handoff if we have a blocked_donor */
+ owner = MUTEX_FLAG_HANDOFF;
+ break;
+ }
+
+ if (owner & MUTEX_FLAG_WAITERS)
+ owner_flags = owner | MUTEX_FLAG_STEAL;
+ }
+
+ if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, owner_flags)) {
+ if (owner_flags & MUTEX_FLAG_STEAL) {
+ steal = true;
+ break;
+ }
if (owner & MUTEX_FLAG_WAITERS)
break;
@@ -1071,6 +1125,9 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
if (owner & MUTEX_FLAG_HANDOFF)
__mutex_handoff(lock, next);
+ if (sched_proxy_exec() && steal)
+ __mutex_steal(lock, next);
+
raw_spin_unlock(¤t->blocked_lock);
raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
if (next) {
diff --git a/kernel/locking/mutex.h b/kernel/locking/mutex.h
index 3e263e98e5fc..eb4180745da0 100644
--- a/kernel/locking/mutex.h
+++ b/kernel/locking/mutex.h
@@ -33,8 +33,9 @@ struct mutex_waiter {
#define MUTEX_FLAG_WAITERS 0x01
#define MUTEX_FLAG_HANDOFF 0x02
#define MUTEX_FLAG_PICKUP 0x04
+#define MUTEX_FLAG_STEAL 0x08
-#define MUTEX_FLAGS 0x07
+#define MUTEX_FLAGS 0x0F
/*
* Internal helper function; C doesn't allow us to hide it :/
---
With that small steal opportunity, we have:
==================================================================
Test : sched-messaging
Units : Normalized time in seconds
Interpretation: Lower is better
Statistic : AMean
==================================================================
Test: vanilla handoff steal + handoff
1-groups: 3.12 (0.00 pct) 3.47 (-11.21 pct) 3.63 (-16.34 pct) *
2-groups: 3.43 (0.00 pct) 4.33 (-26.23 pct) 4.14 (-20.69 pct) *
4-groups: 4.05 (0.00 pct) 5.95 (-46.91 pct) 5.45 (-34.56 pct)
8-groups: 4.29 (0.00 pct) 9.56 (-122.84 pct) 7.80 (-81.81 pct)
16-groups: 5.89 (0.00 pct) 12.29 (-108.65 pct) 11.89 (-101.86 pct)
* Data points have > 10% run to run variance on all versions
So, the opportunity must be extended to allow stealing until the the waiter
wakes up for !HANDOFF cases. Few complications with that are:
o We cannot continue to persist the old owner after mutex_unlock() since that
owner can die, block on other mutex, etc. and that breaks queuing on owner
since new waiters can go and block on a dead task / task blocked on an
incorrect owner.
o We cannot allow steal after handoff to new owner because __mutex_lock() will
resolve to new owner that hasn't woken up yet and waiters start queuing on
it but a concurrent task can come steal the lock and break proxy. Not very
intuitive; adds a lot of complexity.
============================================
Experiment 3: STEAL + Temporary swap to idle
============================================
the unlock will temporarily swap to rq->idle of the lock owner's CPU with
MUTEX_FLAG_STEAL set to allow grabbing the task until the the waiter wakes
up and manages to grab the task itself for !HANDOFF cases. With that,
numbers are very close:
diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index 8a85912d7ee6..187f95544453 100644
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -92,7 +92,16 @@ static inline struct task_struct *__mutex_trylock_common(struct mutex *lock, boo
unsigned long task = owner & ~MUTEX_FLAGS;
if (task) {
- if (flags & MUTEX_FLAG_PICKUP) {
+ if (sched_proxy_exec() && (flags & MUTEX_FLAG_STEAL)) {
+ /*
+ * STEAL cannot be set after HANDOFF has been
+ * initiated. If STEAL is set, clear it and
+ * preserve other flags
+ */
+ MUTEX_WARN_ON(flags & (MUTEX_FLAG_PICKUP));
+ flags &= ~MUTEX_FLAG_STEAL;
+ task = curr;
+ }else if (flags & MUTEX_FLAG_PICKUP) {
if (task != curr)
break;
flags &= ~MUTEX_FLAG_PICKUP;
@@ -104,7 +113,7 @@ static inline struct task_struct *__mutex_trylock_common(struct mutex *lock, boo
break;
}
} else {
- MUTEX_WARN_ON(flags & (MUTEX_FLAG_HANDOFF | MUTEX_FLAG_PICKUP));
+ MUTEX_WARN_ON(flags & (MUTEX_FLAG_HANDOFF | MUTEX_FLAG_PICKUP | MUTEX_FLAG_STEAL));
task = curr;
}
@@ -242,7 +251,41 @@ __mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter)
__must_hold(&lock->wait_lock)
{
if (list_empty(&waiter->list)) {
- __mutex_clear_flag(lock, MUTEX_FLAGS);
+ /*
+ * The last waiter can be interrupted before the full
+ * unlock with STEAL is done.
+ *
+ * LOCK lock->wait_lock
+ *
+ * __mutex_trylock()
+ * // Sees old owner __mutex_unlock_slowpath()
+ * return owner; atomic_long_cmpxchg_release(&owner, idle | STEAL)
+ * // Succeeds
+ * if (signal_pending())
+ * goto err;
+ *
+ * err:
+ * __mutex_remove_waiter()
+ * __mutex_clear_flag(MUTEX_FLAGS)
+ * lock->first_waiter = NULL;
+ *
+ * UNLOCK lock->wait_lock LOCK lock->wait_lock
+ * waiter = lock->first_waiter; // NULL
+ * // No wakeup
+ *
+ * !!! lock->owner stuck as rq->idle without STEAL set !!!
+ *
+ * Persis the STEAL flag to prevent an idle task to
+ * linger as lock owner. __mutex_trylock_fast() will
+ * fail temporarily for first contender but following
+ * __mutex_trylock_common() will do the right thing.
+ *
+ * XXX: This can also be solved by doing a
+ * atomic_try_cmpxchg() or__mutex_clear_flag() in
+ * __mutex_unlock_slowpath() if steal is set but no
+ * waiter is found under lock->wait_lock.
+ */
+ __mutex_clear_flag(lock, MUTEX_FLAGS & ~MUTEX_FLAG_STEAL);
lock->first_waiter = NULL;
} else {
if (lock->first_waiter == waiter)
@@ -274,7 +317,6 @@ static void __mutex_handoff(struct mutex *lock, struct task_struct *task)
new |= (unsigned long)task;
if (task)
new |= MUTEX_FLAG_PICKUP;
-
if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, new))
break;
}
@@ -389,7 +431,17 @@ bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner,
lockdep_assert_preemption_disabled();
- while (__mutex_owner(lock) == owner) {
+ for (;;) {
+ unsigned long __owner = atomic_long_read(&lock->owner);
+
+ /* If the owner changed, break out. */
+ if (__owner_task(__owner) != owner)
+ break;
+
+ /* If lock can be stolen, break out. */
+ if (sched_proxy_exec() && (__owner_flags(__owner) & MUTEX_FLAG_STEAL))
+ break;
+
/*
* Ensure we emit the owner->on_cpu, dereference _after_
* checking lock->owner still matches owner. And we already
@@ -1006,19 +1058,42 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
*/
owner = atomic_long_read(&lock->owner);
for (;;) {
+ unsigned long owner_flags;
+
MUTEX_WARN_ON(__owner_task(owner) != current);
MUTEX_WARN_ON(owner & MUTEX_FLAG_PICKUP);
- if (sched_proxy_exec() && current->blocked_donor) {
- /* force handoff if we have a blocked_donor */
- owner = MUTEX_FLAG_HANDOFF;
- break;
- }
-
if (owner & MUTEX_FLAG_HANDOFF)
break;
- if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, __owner_flags(owner))) {
+ owner_flags = __owner_flags(owner);
+ if (sched_proxy_exec()) {
+ if (current->blocked_donor) {
+ /* force handoff if we have a blocked_donor */
+ owner = MUTEX_FLAG_HANDOFF;
+ break;
+ }
+
+ if (owner & MUTEX_FLAG_WAITERS) {
+ unsigned long idle;
+ /*
+ * Swap the owner to current CPU's idle task
+ * with a STEAL flag.
+ *
+ * The lock is free to be stolen and
+ * __mutex_owner() will resolve to idle task
+ * that is always ->on_rq on this CPU.
+ *
+ * Proxy donors will temporarily migrate here
+ * before a wakeup or an optimistic spinner
+ * can grab the lock.
+ */
+ idle = (unsigned long)idle_task(raw_smp_processor_id());
+ owner_flags = idle | MUTEX_FLAG_STEAL | owner_flags;
+ }
+ }
+
+ if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, owner_flags)) {
if (owner & MUTEX_FLAG_WAITERS)
break;
diff --git a/kernel/locking/mutex.h b/kernel/locking/mutex.h
index 3e263e98e5fc..eb4180745da0 100644
--- a/kernel/locking/mutex.h
+++ b/kernel/locking/mutex.h
@@ -33,8 +33,9 @@ struct mutex_waiter {
#define MUTEX_FLAG_WAITERS 0x01
#define MUTEX_FLAG_HANDOFF 0x02
#define MUTEX_FLAG_PICKUP 0x04
+#define MUTEX_FLAG_STEAL 0x08
-#define MUTEX_FLAGS 0x07
+#define MUTEX_FLAGS 0x0F
/*
* Internal helper function; C doesn't allow us to hide it :/
---
The results with temporary switch to idle + STEAL are:
==================================================================
Test : sched-messaging
Units : Normalized time in seconds
Interpretation: Lower is better
Statistic : AMean
==================================================================
Test: vanilla handoff STEAL + handoff idle + STEAL
1-groups: 3.12 (0.00 pct) 3.47 (-11.21 pct) 3.63 (-16.34 pct) 3.59 (-15.06 pct)
2-groups: 3.43 (0.00 pct) 4.33 (-26.23 pct) 4.14 (-20.69 pct) 3.48 (-1.45 pct)
4-groups: 4.05 (0.00 pct) 5.95 (-46.91 pct) 5.45 (-34.56 pct) 4.00 (1.23 pct)
8-groups: 4.29 (0.00 pct) 9.56 (-122.84 pct) 7.80 (-81.81 pct) 4.31 (-0.46 pct)
16-groups: 5.89 (0.00 pct) 12.29 (-108.65 pct) 11.89 (-101.86 pct) 5.91 (-0.33 pct)
* Data points have > 10% run to run variance on all versions
My machine has held up for some time with Experiment 3 so I'm
fairly confident at the very least mutual exclusion is holding
up - I haven't seen any lockups / hung task either so hopefully
other bits are fine too :-)
--
Thanks and Regards,
Prateek
^ permalink raw reply [flat|nested] 23+ messages in thread
* [RFC PATCH 05/16] sched/core: Do not queue blocked donor on a delayed owner
2026-08-26 6:28 [RFC PATCH 00/16][PoC] sched/core: Alternate approach to sleeping-owner handling in PROXY_EXEC K Prateek Nayak
` (3 preceding siblings ...)
2026-08-26 6:28 ` [RFC PATCH 04/16] sched/core: Activate blocked donor when no owner is found K Prateek Nayak
@ 2026-08-26 6:28 ` K Prateek Nayak
2026-08-26 6:28 ` [RFC PATCH 06/16] sched/core: Queue blocked donor onto sleeping owner for chain-wakeup K Prateek Nayak
` (10 subsequent siblings)
15 siblings, 0 replies; 23+ messages in thread
From: K Prateek Nayak @ 2026-08-26 6:28 UTC (permalink / raw)
To: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
Andrea Righi, linux-kernel
Cc: Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, Waiman Long, K Prateek Nayak
Delayed owner have a stable task_cpu() since they are ON_RQ_QUEUED. Move
the donor to delayed owner's CPU and block the delayed owner completely
if find_proxy_task() resolves to them similar to what pick would have
done.
This avoids needing to handle chain-wakeup for delayed tasks,
simplifying the ttwu_runnable() handling in the subsequent commits.
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
kernel/sched/core.c | 19 +++++++++++++++++--
1 file changed, 17 insertions(+), 2 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 9d9db7ccf01f..b4a2771eb290 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6958,8 +6958,8 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
goto activate;
}
- if (!READ_ONCE(owner->on_rq) || owner->se.sched_delayed) {
- /* XXX Don't handle blocked owners/delayed dequeue yet */
+ if (!READ_ONCE(owner->on_rq)) {
+ /* XXX Don't handle blocked owners yet */
if (curr_in_chain)
return proxy_resched_idle(rq);
__clear_task_blocked_on(p, NULL);
@@ -7025,6 +7025,18 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
*/
return proxy_resched_idle(rq);
}
+
+ /*
+ * Delayed owner is on the same CPU and rq_lock is held. Treat this
+ * like pick resolving to the delayed owner and block the task
+ * completely before retrying the pick.
+ *
+ * This prevents the need to resolve "owner->blocked_head" chain for
+ * wakeup of delayed task and fast-path them.
+ */
+ if (owner->se.sched_delayed)
+ goto block_delayed_owner;
+
/*
* OK, now we're absolutely sure @owner is on this
* rq, therefore holding @rq->lock is sufficient to
@@ -7052,6 +7064,9 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
migrate_task:
proxy_migrate_task(rq, rf, p, owner_cpu);
return NULL;
+block_delayed_owner:
+ dequeue_task(rq, owner, DEQUEUE_SLEEP | DEQUEUE_DELAYED);
+ return NULL;
}
#else /* SCHED_PROXY_EXEC */
static struct task_struct *
--
2.34.1
^ permalink raw reply [flat|nested] 23+ messages in thread* [RFC PATCH 06/16] sched/core: Queue blocked donor onto sleeping owner for chain-wakeup
2026-08-26 6:28 [RFC PATCH 00/16][PoC] sched/core: Alternate approach to sleeping-owner handling in PROXY_EXEC K Prateek Nayak
` (4 preceding siblings ...)
2026-08-26 6:28 ` [RFC PATCH 05/16] sched/core: Do not queue blocked donor on a delayed owner K Prateek Nayak
@ 2026-08-26 6:28 ` K Prateek Nayak
2026-08-26 16:20 ` Andrea Righi
2026-08-26 6:28 ` [RFC PATCH 07/16] sched/core: Avoid delaying blocked donors queued on sleeping owner K Prateek Nayak
` (9 subsequent siblings)
15 siblings, 1 reply; 23+ messages in thread
From: K Prateek Nayak @ 2026-08-26 6:28 UTC (permalink / raw)
To: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
Andrea Righi, linux-kernel
Cc: Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, Waiman Long, K Prateek Nayak
Add a blocked_donor to sleeping owner's "blocked_head" when
find_proxy_task() resolves to a blocked task. Once added to the list,
double check if the owner has woken up by checking "owner->on_rq".
If the owner has woken up, remove the task from "blocked_head" and
continue try find_proxy_task() agiain to re-evaluate the state of owner
and take the correct steps.
Since find_proxy_task() is called with wait_lock held, which the owner
needs during unlock, it is guaranteed that owner cannot disappear under
us in the process.
The added data memebers in task_struct serve the following pusrpose:
- blocked_head: Contains the blocked donors queued on us
- blocked_node: The list head used to queue onto blocked_head of a
sleeping owner
- sleeping_owner: Sleeping owner on which the task is queued.
Co-developed-by: John Stultz <jstultz@google.com>
Signed-off-by: John Stultz <jstultz@google.com>
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
include/linux/sched.h | 5 ++++
init/init_task.c | 5 ++++
kernel/fork.c | 5 ++++
kernel/sched/core.c | 68 +++++++++++++++++++++++++++++++++++++++++--
4 files changed, 81 insertions(+), 2 deletions(-)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 373bcc0598d1..bf0f4b6be7c4 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1251,6 +1251,11 @@ struct task_struct {
struct mutex *blocked_on; /* lock we're blocked on */
raw_spinlock_t blocked_lock;
+#ifdef CONFIG_SCHED_PROXY_EXEC
+ struct list_head blocked_head; /* tasks blocked on this task */
+ struct list_head blocked_node; /* our entry on someone elses blocked_head */
+ struct task_struct *sleeping_owner; /* task our blocked_node is enqueued on */
+#endif
/*
* The task that is boosting this task; a back link for the current
diff --git a/init/init_task.c b/init/init_task.c
index b67ef6040a65..a097c0def4c4 100644
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -211,6 +211,11 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
&init_task.alloc_lock),
#endif
.blocked_donor = NULL,
+#ifdef CONFIG_SCHED_PROXY_EXEC
+ .blocked_head = LIST_HEAD_INIT(init_task.blocked_head),
+ .blocked_node = LIST_HEAD_INIT(init_task.blocked_node),
+ .sleeping_owner = NULL,
+#endif
#ifdef CONFIG_RT_MUTEXES
.pi_waiters = RB_ROOT_CACHED,
.pi_top_task = NULL,
diff --git a/kernel/fork.c b/kernel/fork.c
index f0e2e131a9a5..88f2b6e08c46 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -2247,6 +2247,11 @@ __latent_entropy struct task_struct *copy_process(
p->blocked_on = NULL; /* not blocked yet */
p->blocked_donor = NULL; /* nobody is boosting p yet */
+#ifdef CONFIG_SCHED_PROXY_EXEC
+ INIT_LIST_HEAD(&p->blocked_head);
+ INIT_LIST_HEAD(&p->blocked_node);
+ p->sleeping_owner = NULL;
+#endif
#ifdef CONFIG_BCACHE
p->sequential_io = 0;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index b4a2771eb290..e53967a126a9 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2229,6 +2229,16 @@ void activate_task(struct rq *rq, struct task_struct *p, int en_flags)
__activate_task(rq, p, en_flags | ENQUEUE_MIGRATING);
}
+#ifdef CONFIG_SCHED_PROXY_EXEC
+
+static void __proxy_dequeue_from_owner(struct task_struct *p)
+{
+ list_del_init(&p->blocked_node);
+ WRITE_ONCE(p->sleeping_owner, NULL);
+}
+
+#endif /* CONFIG_SCHED_PROXY_EXEC */
+
static void activate_blocked_task(struct rq *rq, struct task_struct *p, int en_flags)
{
__activate_task(rq, p, en_flags);
@@ -6872,6 +6882,43 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf,
proxy_reacquire_rq_lock(rq, rf);
}
+static void proxy_enqueue_on_owner(struct rq *rq, struct task_struct *owner,
+ struct task_struct *p)
+{
+ lockdep_assert_rq_held(rq);
+ lockdep_assert_held(&owner->blocked_lock);
+
+ WARN_ON(!p->on_rq);
+ WARN_ON(p->sleeping_owner);
+
+ WRITE_ONCE(p->sleeping_owner, owner);
+ list_add(&p->blocked_node, &owner->blocked_head);
+ proxy_resched_idle(rq);
+
+ /*
+ * Order against __activate_blocked_task_slowpath() checking
+ * owner->blocked_list after setting owner->on_rq.
+ */
+ smp_mb();
+
+ if (READ_ONCE(owner->on_rq)) {
+ /*
+ * owner has woken up and may miss activating us.
+ * Remove ourself from "owner->blocked_head" and try
+ * find_proxy_task() again considering the owner's
+ * new state.
+ */
+ __proxy_dequeue_from_owner(p);
+ return;
+ }
+
+ /*
+ * Owner is fully blocked. __activate_blocked_task_slowpath()
+ * will see us on the list during wakeup and DTRT.
+ */
+ block_task(rq, p, READ_ONCE(p->__state));
+}
+
/*
* Find runnable lock owner to proxy for mutex blocked donor
*
@@ -6962,8 +7009,25 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
/* XXX Don't handle blocked owners yet */
if (curr_in_chain)
return proxy_resched_idle(rq);
- __clear_task_blocked_on(p, NULL);
- goto deactivate;
+ /*
+ * If !@owner->on_rq, holding @rq->lock will not pin the task,
+ * so we cannot drop @mutex->wait_lock until we're sure its a blocked
+ * task on this rq.
+ *
+ * We use @owner->blocked_lock to serialize against ttwu_activate().
+ * Either we see its new owner->on_rq or it will see our list_add().
+ */
+ WARN_ON(owner == p);
+
+ raw_spin_unlock(&p->blocked_lock);
+ raw_spin_lock(&owner->blocked_lock);
+
+ proxy_enqueue_on_owner(rq, owner, p);
+
+ raw_spin_unlock(&owner->blocked_lock);
+ raw_spin_lock(&p->blocked_lock);
+
+ return NULL; /* retry task selection */
}
owner_cpu = task_cpu(owner);
--
2.34.1
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [RFC PATCH 06/16] sched/core: Queue blocked donor onto sleeping owner for chain-wakeup
2026-08-26 6:28 ` [RFC PATCH 06/16] sched/core: Queue blocked donor onto sleeping owner for chain-wakeup K Prateek Nayak
@ 2026-08-26 16:20 ` Andrea Righi
2026-08-27 3:51 ` K Prateek Nayak
0 siblings, 1 reply; 23+ messages in thread
From: Andrea Righi @ 2026-08-26 16:20 UTC (permalink / raw)
To: K Prateek Nayak
Cc: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
linux-kernel, Dietmar Eggemann, Steven Rostedt, Ben Segall,
Mel Gorman, Valentin Schneider, Waiman Long
On Wed, Aug 26, 2026 at 06:28:50AM +0000, K Prateek Nayak wrote:
> Add a blocked_donor to sleeping owner's "blocked_head" when
> find_proxy_task() resolves to a blocked task. Once added to the list,
> double check if the owner has woken up by checking "owner->on_rq".
>
> If the owner has woken up, remove the task from "blocked_head" and
> continue try find_proxy_task() agiain to re-evaluate the state of owner
> and take the correct steps.
nit: s/agiain/again/
>
> Since find_proxy_task() is called with wait_lock held, which the owner
> needs during unlock, it is guaranteed that owner cannot disappear under
> us in the process.
>
> The added data memebers in task_struct serve the following pusrpose:
nit:
s/memebers/members/
s/pusrpose/purpose/
>
> - blocked_head: Contains the blocked donors queued on us
> - blocked_node: The list head used to queue onto blocked_head of a
> sleeping owner
> - sleeping_owner: Sleeping owner on which the task is queued.
>
> Co-developed-by: John Stultz <jstultz@google.com>
> Signed-off-by: John Stultz <jstultz@google.com>
> Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
> ---
> include/linux/sched.h | 5 ++++
> init/init_task.c | 5 ++++
> kernel/fork.c | 5 ++++
> kernel/sched/core.c | 68 +++++++++++++++++++++++++++++++++++++++++--
> 4 files changed, 81 insertions(+), 2 deletions(-)
>
> diff --git a/include/linux/sched.h b/include/linux/sched.h
> index 373bcc0598d1..bf0f4b6be7c4 100644
> --- a/include/linux/sched.h
> +++ b/include/linux/sched.h
> @@ -1251,6 +1251,11 @@ struct task_struct {
>
> struct mutex *blocked_on; /* lock we're blocked on */
> raw_spinlock_t blocked_lock;
> +#ifdef CONFIG_SCHED_PROXY_EXEC
> + struct list_head blocked_head; /* tasks blocked on this task */
> + struct list_head blocked_node; /* our entry on someone elses blocked_head */
> + struct task_struct *sleeping_owner; /* task our blocked_node is enqueued on */
> +#endif
>
> /*
> * The task that is boosting this task; a back link for the current
> diff --git a/init/init_task.c b/init/init_task.c
> index b67ef6040a65..a097c0def4c4 100644
> --- a/init/init_task.c
> +++ b/init/init_task.c
> @@ -211,6 +211,11 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
> &init_task.alloc_lock),
> #endif
> .blocked_donor = NULL,
> +#ifdef CONFIG_SCHED_PROXY_EXEC
> + .blocked_head = LIST_HEAD_INIT(init_task.blocked_head),
> + .blocked_node = LIST_HEAD_INIT(init_task.blocked_node),
> + .sleeping_owner = NULL,
> +#endif
> #ifdef CONFIG_RT_MUTEXES
> .pi_waiters = RB_ROOT_CACHED,
> .pi_top_task = NULL,
> diff --git a/kernel/fork.c b/kernel/fork.c
> index f0e2e131a9a5..88f2b6e08c46 100644
> --- a/kernel/fork.c
> +++ b/kernel/fork.c
> @@ -2247,6 +2247,11 @@ __latent_entropy struct task_struct *copy_process(
>
> p->blocked_on = NULL; /* not blocked yet */
> p->blocked_donor = NULL; /* nobody is boosting p yet */
> +#ifdef CONFIG_SCHED_PROXY_EXEC
> + INIT_LIST_HEAD(&p->blocked_head);
> + INIT_LIST_HEAD(&p->blocked_node);
> + p->sleeping_owner = NULL;
> +#endif
>
> #ifdef CONFIG_BCACHE
> p->sequential_io = 0;
> diff --git a/kernel/sched/core.c b/kernel/sched/core.c
> index b4a2771eb290..e53967a126a9 100644
> --- a/kernel/sched/core.c
> +++ b/kernel/sched/core.c
> @@ -2229,6 +2229,16 @@ void activate_task(struct rq *rq, struct task_struct *p, int en_flags)
> __activate_task(rq, p, en_flags | ENQUEUE_MIGRATING);
> }
>
> +#ifdef CONFIG_SCHED_PROXY_EXEC
> +
> +static void __proxy_dequeue_from_owner(struct task_struct *p)
> +{
> + list_del_init(&p->blocked_node);
> + WRITE_ONCE(p->sleeping_owner, NULL);
> +}
> +
> +#endif /* CONFIG_SCHED_PROXY_EXEC */
> +
> static void activate_blocked_task(struct rq *rq, struct task_struct *p, int en_flags)
> {
> __activate_task(rq, p, en_flags);
> @@ -6872,6 +6882,43 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf,
> proxy_reacquire_rq_lock(rq, rf);
> }
>
> +static void proxy_enqueue_on_owner(struct rq *rq, struct task_struct *owner,
> + struct task_struct *p)
> +{
> + lockdep_assert_rq_held(rq);
> + lockdep_assert_held(&owner->blocked_lock);
> +
> + WARN_ON(!p->on_rq);
> + WARN_ON(p->sleeping_owner);
> +
> + WRITE_ONCE(p->sleeping_owner, owner);
> + list_add(&p->blocked_node, &owner->blocked_head);
> + proxy_resched_idle(rq);
> +
> + /*
> + * Order against __activate_blocked_task_slowpath() checking
> + * owner->blocked_list after setting owner->on_rq.
This should be owner->blocked_head not owner->blocked_list, right?
> + */
> + smp_mb();
> +
> + if (READ_ONCE(owner->on_rq)) {
> + /*
> + * owner has woken up and may miss activating us.
> + * Remove ourself from "owner->blocked_head" and try
> + * find_proxy_task() again considering the owner's
> + * new state.
> + */
> + __proxy_dequeue_from_owner(p);
> + return;
> + }
> +
> + /*
> + * Owner is fully blocked. __activate_blocked_task_slowpath()
> + * will see us on the list during wakeup and DTRT.
> + */
> + block_task(rq, p, READ_ONCE(p->__state));
> +}
> +
> /*
> * Find runnable lock owner to proxy for mutex blocked donor
> *
> @@ -6962,8 +7009,25 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
> /* XXX Don't handle blocked owners yet */
> if (curr_in_chain)
> return proxy_resched_idle(rq);
> - __clear_task_blocked_on(p, NULL);
> - goto deactivate;
> + /*
> + * If !@owner->on_rq, holding @rq->lock will not pin the task,
> + * so we cannot drop @mutex->wait_lock until we're sure its a blocked
> + * task on this rq.
> + *
> + * We use @owner->blocked_lock to serialize against ttwu_activate().
> + * Either we see its new owner->on_rq or it will see our list_add().
> + */
> + WARN_ON(owner == p);
> +
> + raw_spin_unlock(&p->blocked_lock);
> + raw_spin_lock(&owner->blocked_lock);
> +
> + proxy_enqueue_on_owner(rq, owner, p);
> +
> + raw_spin_unlock(&owner->blocked_lock);
> + raw_spin_lock(&p->blocked_lock);
> +
> + return NULL; /* retry task selection */
> }
>
> owner_cpu = task_cpu(owner);
> --
> 2.34.1
>
Thanks,
-Andrea
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [RFC PATCH 06/16] sched/core: Queue blocked donor onto sleeping owner for chain-wakeup
2026-08-26 16:20 ` Andrea Righi
@ 2026-08-27 3:51 ` K Prateek Nayak
0 siblings, 0 replies; 23+ messages in thread
From: K Prateek Nayak @ 2026-08-27 3:51 UTC (permalink / raw)
To: Andrea Righi
Cc: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
linux-kernel, Dietmar Eggemann, Steven Rostedt, Ben Segall,
Mel Gorman, Valentin Schneider, Waiman Long
Hello Andrea,
On 8/26/2026 9:50 PM, Andrea Righi wrote:
>> +static void proxy_enqueue_on_owner(struct rq *rq, struct task_struct *owner,
>> + struct task_struct *p)
>> +{
>> + lockdep_assert_rq_held(rq);
>> + lockdep_assert_held(&owner->blocked_lock);
>> +
>> + WARN_ON(!p->on_rq);
>> + WARN_ON(p->sleeping_owner);
>> +
>> + WRITE_ONCE(p->sleeping_owner, owner);
>> + list_add(&p->blocked_node, &owner->blocked_head);
>> + proxy_resched_idle(rq);
>> +
>> + /*
>> + * Order against __activate_blocked_task_slowpath() checking
>> + * owner->blocked_list after setting owner->on_rq.
>
> This should be owner->blocked_head not owner->blocked_list, right?
Correct! My bad. Also the slow-path is proxy_activate_blocked_task()
that gets introduced in Patch 14. I missed renaming it here.
And ack on other comments too above.
>
>> + */
>> + smp_mb();
>> +
>> + if (READ_ONCE(owner->on_rq)) {
>> + /*
>> + * owner has woken up and may miss activating us.
>> + * Remove ourself from "owner->blocked_head" and try
>> + * find_proxy_task() again considering the owner's
>> + * new state.
>> + */
>> + __proxy_dequeue_from_owner(p);
>> + return;
>> + }
>> +
>> + /*
>> + * Owner is fully blocked. __activate_blocked_task_slowpath()
>> + * will see us on the list during wakeup and DTRT.
>> + */
>> + block_task(rq, p, READ_ONCE(p->__state));
>> +}
--
Thanks and Regards,
Prateek
^ permalink raw reply [flat|nested] 23+ messages in thread
* [RFC PATCH 07/16] sched/core: Avoid delaying blocked donors queued on sleeping owner
2026-08-26 6:28 [RFC PATCH 00/16][PoC] sched/core: Alternate approach to sleeping-owner handling in PROXY_EXEC K Prateek Nayak
` (5 preceding siblings ...)
2026-08-26 6:28 ` [RFC PATCH 06/16] sched/core: Queue blocked donor onto sleeping owner for chain-wakeup K Prateek Nayak
@ 2026-08-26 6:28 ` K Prateek Nayak
2026-08-26 6:28 ` [RFC PATCH 08/16] sched/deadline: Prepare for blocking and proxy activation with MIGRATING flag K Prateek Nayak
` (8 subsequent siblings)
15 siblings, 0 replies; 23+ messages in thread
From: K Prateek Nayak @ 2026-08-26 6:28 UTC (permalink / raw)
To: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
Andrea Righi, linux-kernel
Cc: Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, Waiman Long, K Prateek Nayak
Subsequent patches will rely on the fact that tasks on a sleeping
owner's chain are all owner by one CPU (task_cpu() will all resolve to a
single CPU).
With delayed task, this assumption will not be true since the delayed
task is owned by the CPU where it was dequeued until the pick fully
blocks it.
Block the task queued on sleeping owner completely to avoid this
complication. Further optimizations can explore moving the delayed task
to the p->blocked_cpu and wait until it is dequeued there but for this
PoC, the simpler approach has been adopted.
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
kernel/sched/core.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index e53967a126a9..d635f5452a1b 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6916,7 +6916,7 @@ static void proxy_enqueue_on_owner(struct rq *rq, struct task_struct *owner,
* Owner is fully blocked. __activate_blocked_task_slowpath()
* will see us on the list during wakeup and DTRT.
*/
- block_task(rq, p, READ_ONCE(p->__state));
+ block_task(rq, p, READ_ONCE(p->__state) | TASK_WAKING);
}
/*
--
2.34.1
^ permalink raw reply [flat|nested] 23+ messages in thread* [RFC PATCH 08/16] sched/deadline: Prepare for blocking and proxy activation with MIGRATING flag
2026-08-26 6:28 [RFC PATCH 00/16][PoC] sched/core: Alternate approach to sleeping-owner handling in PROXY_EXEC K Prateek Nayak
` (6 preceding siblings ...)
2026-08-26 6:28 ` [RFC PATCH 07/16] sched/core: Avoid delaying blocked donors queued on sleeping owner K Prateek Nayak
@ 2026-08-26 6:28 ` K Prateek Nayak
2026-08-26 6:28 ` [RFC PATCH 09/16] sched/core: Track CPU where the task was blocked on K Prateek Nayak
` (7 subsequent siblings)
15 siblings, 0 replies; 23+ messages in thread
From: K Prateek Nayak @ 2026-08-26 6:28 UTC (permalink / raw)
To: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
Andrea Righi, linux-kernel
Cc: Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, Waiman Long, K Prateek Nayak
Blocking deadline task sets task_not_contending() to retain bandwidth
and start zero-lag timer on the rq and depends on migrate_task_rq_dl()
via set_task_cpu() to remove them if the task migrates at wakeup.
When DEQUEUE_SLEEP is coupled with DEQUEUE_MIGRATING, the task_cpu() can
be switched via proxy_set_task_cpu() after the task is dequeued.
To balance this, enqueue path will add ENQUEUE_MIGRATING on the activate
path for symmetry with migrated blocking.
Treat SLEEP or WAKEUP with MIGRATING as simple migration and prevent
bandwidth modifiction, inactive timers programming on the rq since
task_rq() may resolve differently.
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
XXX: Does this break EDF if we are blocking with 0-lag point still ahead
of us and not set task_non_contending()?
---
kernel/sched/deadline.c | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 9abda560c633..886ba6bfa868 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -471,7 +471,7 @@ static void task_contending(struct sched_dl_entity *dl_se, int flags)
* If this is a non-deadline task that has been boosted,
* do nothing
*/
- if (dl_se->dl_runtime == 0)
+ if (dl_se->dl_runtime == 0 || (flags & ENQUEUE_MIGRATING))
return;
if (flags & ENQUEUE_MIGRATED)
@@ -2477,8 +2477,13 @@ static void dequeue_dl_entity(struct sched_dl_entity *dl_se, int flags)
* way, because from GRUB's point of view the same thing is happening
* (the task moves from "active contending" to "active non contending"
* or "inactive")
+ *
+ * XXX: Proxy execution can block task with DEQUEUE_MIGRATING and
+ * switch the task_cpu() underneath without going through
+ * migrate_task_rq_dl(). Do not leave any bandwidth or timer reference
+ * back when DEQUEUE_SLEEP is coupled with DEQUEUE_MIGRATING
*/
- if (flags & DEQUEUE_SLEEP)
+ if ((flags & (DEQUEUE_SLEEP|DEQUEUE_MIGRATING)) == DEQUEUE_SLEEP)
task_non_contending(dl_se, true);
}
--
2.34.1
^ permalink raw reply [flat|nested] 23+ messages in thread* [RFC PATCH 09/16] sched/core: Track CPU where the task was blocked on
2026-08-26 6:28 [RFC PATCH 00/16][PoC] sched/core: Alternate approach to sleeping-owner handling in PROXY_EXEC K Prateek Nayak
` (7 preceding siblings ...)
2026-08-26 6:28 ` [RFC PATCH 08/16] sched/deadline: Prepare for blocking and proxy activation with MIGRATING flag K Prateek Nayak
@ 2026-08-26 6:28 ` K Prateek Nayak
2026-08-26 6:28 ` [RFC PATCH 10/16] sched/core: Introduce p->is_linked to track if task is queued on sleeping owner K Prateek Nayak
` (6 subsequent siblings)
15 siblings, 0 replies; 23+ messages in thread
From: K Prateek Nayak @ 2026-08-26 6:28 UTC (permalink / raw)
To: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
Andrea Righi, linux-kernel
Cc: Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, Waiman Long, K Prateek Nayak
Track the CPU where task was blocked on when running with
sched_proxy_exec().
This is used to re-direct the activation of blocked donors queued on the
blocked_head via the said CPU in the activation slowpath that will be
added in the subsequent patches.
p->wake_cpu or task_cpu() is not sufficient for this purpose since
p->wake_cpu is not stable when !task_on_rq_queued() and there is a
window between set_task_cpu() and activet_blocked_task() in the wakeup
path that needs to be plugged in.
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
include/linux/sched.h | 1 +
kernel/sched/core.c | 16 ++++++++++++++++
kernel/sched/sched.h | 6 ++++++
3 files changed, 23 insertions(+)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index bf0f4b6be7c4..2360fdb1ce3f 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1255,6 +1255,7 @@ struct task_struct {
struct list_head blocked_head; /* tasks blocked on this task */
struct list_head blocked_node; /* our entry on someone elses blocked_head */
struct task_struct *sleeping_owner; /* task our blocked_node is enqueued on */
+ int blocked_cpu; /* CPU where task was blocked. */
#endif
/*
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index d635f5452a1b..b5cc5612a12c 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -3773,6 +3773,22 @@ static inline void proxy_reset_donor(struct rq *rq)
resched_curr(rq);
}
+void __proxy_block_task(struct task_struct *p)
+{
+ if (!sched_proxy_exec())
+ return;
+
+ /*
+ * This is the CPU where the __active_blocked_donor() slowpath will
+ * resume activation of blocked donors queued on this task.
+ *
+ * This exists to cover the race between set_task_cpu() changing the
+ * task_cpu() before task is transitioned to TASK_ON_RQ_QUEUED in
+ * activate_blocked_task().
+ */
+ WRITE_ONCE(p->blocked_cpu, task_cpu(p));
+}
+
/*
* Checks to see if task p has been proxy-migrated to another rq
* and needs to be returned. If so, we deactivate the task here
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 56d9c09c485e..0d90506c34f1 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1445,11 +1445,15 @@ static inline void rq_set_donor(struct rq *rq, struct task_struct *t)
{
rcu_assign_pointer(rq->donor, t);
}
+
+void __proxy_block_task(struct task_struct *p);
#else
static inline void rq_set_donor(struct rq *rq, struct task_struct *t)
{
/* Do nothing */
}
+
+static inline void __proxy_block_task(struct task_struct *p) {}
#endif
#ifdef CONFIG_SCHED_CORE
@@ -3039,6 +3043,8 @@ static inline void __block_task(struct rq *rq, struct task_struct *p)
delayacct_blkio_start();
}
+ __proxy_block_task(p);
+
ASSERT_EXCLUSIVE_WRITER(p->on_rq);
/*
--
2.34.1
^ permalink raw reply [flat|nested] 23+ messages in thread* [RFC PATCH 10/16] sched/core: Introduce p->is_linked to track if task is queued on sleeping owner
2026-08-26 6:28 [RFC PATCH 00/16][PoC] sched/core: Alternate approach to sleeping-owner handling in PROXY_EXEC K Prateek Nayak
` (8 preceding siblings ...)
2026-08-26 6:28 ` [RFC PATCH 09/16] sched/core: Track CPU where the task was blocked on K Prateek Nayak
@ 2026-08-26 6:28 ` K Prateek Nayak
2026-08-26 6:28 ` [RFC PATCH 11/16] sched/core: Prepare to inspect ->is_linked alongside ->on_rq during wakeup K Prateek Nayak
` (5 subsequent siblings)
15 siblings, 0 replies; 23+ messages in thread
From: K Prateek Nayak @ 2026-08-26 6:28 UTC (permalink / raw)
To: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
Andrea Righi, linux-kernel
Cc: Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, Waiman Long, K Prateek Nayak
Add a new p->is_linked state that indicates if the task, currently
acting as a blocked donor, is queued on a sleeping owner.
This will be used in the subsequent patches to influence ttwu path in
addition to ->on_rq state for linked proxy donors to consolidate
their removal from owner's blocked_head in a single location.
Clearing of p->is_linked will be added in subsequent patches when ttwu
and activate_blocked_task() paths are plugged to work with the new
scheme.
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
include/linux/sched.h | 2 +-
kernel/sched/core.c | 16 ++++++++++++++++
2 files changed, 17 insertions(+), 1 deletion(-)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 2360fdb1ce3f..ec65abf090a4 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -854,8 +854,8 @@ struct task_struct {
u8 on_cpu;
u8 on_rq;
+ u8 is_linked;
u8 is_blocked;
- u8 __pad;
struct __call_single_node wake_entry;
unsigned int wakee_flips;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index b5cc5612a12c..d3cdfcb400d1 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -630,6 +630,18 @@ int task_llc(const struct task_struct *p)
* if the task is blocked. Traditionally this would mirror p->on_rq, however
* due things like DELAY_DEQUEUE and PROXY_EXEC, this can diverge.
*
+ * p->is_linked <- { 0, 1 }:
+ *
+ * is set by proxy_enqueue_on_onwer() and cleared by either
+ * proxy_needs_return() or proxy_dequeue_from_owner() and tracks if task is
+ * queued on a sleeping owner's blocked_head. Although this could have been
+ * a p->on_rq state, it really represents the state of task when it is off
+ * the rq and is tracked separately for convenience.
+ *
+ * is_linked is always modified under __task_rq_lock() or when
+ * task_on_rq_migrating() during chain-wakeup where concurrent
+ * __task_rq_lock() is stalled for the task.
+ *
* task_cpu(p): is changed by set_task_cpu(), the rules are:
*
* - Don't call set_task_cpu() on a blocked task:
@@ -2235,6 +2247,7 @@ static void __proxy_dequeue_from_owner(struct task_struct *p)
{
list_del_init(&p->blocked_node);
WRITE_ONCE(p->sleeping_owner, NULL);
+ WRITE_ONCE(p->is_linked, 0);
}
#endif /* CONFIG_SCHED_PROXY_EXEC */
@@ -4307,6 +4320,7 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
*/
WARN_ON_ONCE(p->se.sched_delayed);
WARN_ON_ONCE(p->is_blocked);
+ WARN_ON_ONCE(p->is_linked);
/* If p is current, we know we can run here, so clear blocked_on */
clear_task_blocked_on(p, NULL);
if (!ttwu_state_match(p, state, &success))
@@ -4611,6 +4625,7 @@ static void __sched_fork(u64 clone_flags, struct task_struct *p)
/* A delayed task cannot be in clone(). */
WARN_ON_ONCE(p->se.sched_delayed);
WARN_ON_ONCE(p->is_blocked);
+ WARN_ON_ONCE(p->is_linked);
#ifdef CONFIG_FAIR_GROUP_SCHED
p->se.cfs_rq = NULL;
@@ -6909,6 +6924,7 @@ static void proxy_enqueue_on_owner(struct rq *rq, struct task_struct *owner,
WRITE_ONCE(p->sleeping_owner, owner);
list_add(&p->blocked_node, &owner->blocked_head);
+ WRITE_ONCE(p->is_linked, 1);
proxy_resched_idle(rq);
/*
--
2.34.1
^ permalink raw reply [flat|nested] 23+ messages in thread* [RFC PATCH 11/16] sched/core: Prepare to inspect ->is_linked alongside ->on_rq during wakeup
2026-08-26 6:28 [RFC PATCH 00/16][PoC] sched/core: Alternate approach to sleeping-owner handling in PROXY_EXEC K Prateek Nayak
` (9 preceding siblings ...)
2026-08-26 6:28 ` [RFC PATCH 10/16] sched/core: Introduce p->is_linked to track if task is queued on sleeping owner K Prateek Nayak
@ 2026-08-26 6:28 ` K Prateek Nayak
2026-08-26 6:28 ` [RFC PATCH 12/16] sched:core: Add MIGRATING flags when blocking and activating linked donors K Prateek Nayak
` (4 subsequent siblings)
15 siblings, 0 replies; 23+ messages in thread
From: K Prateek Nayak @ 2026-08-26 6:28 UTC (permalink / raw)
To: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
Andrea Righi, linux-kernel
Cc: Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, Waiman Long, K Prateek Nayak
To handle wakeup of blocked donors queued on owner's blocked_head via
ttwu_runnable(), prepare to inspect p->is_blocked in addition to
p->on_rq on paths that demand grabbing rq_lock to prevent "->on_rq"
transitions.
Unionize p->on_rq with p->is_linked to derive a 16-bit p->needs_rq_sync
state that can be atomically inspected on all architectures.
p->needs_rq_sync check follows the same ordering requirements as
p->on_rq check.
Subsequent tasks will guarntee ordering of p->is_linked against p->on_rq
transitions ensuring that loading p->needs_rq_sync state is enough to
cover both and the readers don't falsely observe them to be cleared if
read individually during their transition.
get_wchan() has been reflowed using guards as a pert of the change to
prevent awkward indentation.
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
include/linux/sched.h | 18 +++++++++++++--
kernel/sched/core.c | 53 +++++++++++++++++++++++++++++++++++++++----
2 files changed, 64 insertions(+), 7 deletions(-)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index ec65abf090a4..76de348fb88b 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -852,9 +852,23 @@ struct task_struct {
struct alloc_tag *alloc_tag;
#endif
+ union {
+ struct {
+ u8 on_rq;
+ u8 is_linked;
+ };
+ /*
+ * Allows inspecting on_rq and is_linked
+ * atomically with a single read outside
+ * the task_rq_lock().
+ *
+ * See the comment above the load in
+ * try_to_wake_up() for ordering
+ * guarantees.
+ */
+ u16 needs_rq_sync;
+ };
u8 on_cpu;
- u8 on_rq;
- u8 is_linked;
u8 is_blocked;
struct __call_single_node wake_entry;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index d3cdfcb400d1..957f30632b82 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2171,12 +2171,16 @@ unsigned long get_wchan(struct task_struct *p)
return 0;
/* Only get wchan if task is blocked and we can keep it that way. */
- raw_spin_lock_irq(&p->pi_lock);
+ guard(raw_spinlock_irq)(&p->pi_lock);
+
state = READ_ONCE(p->__state);
+ if (state == TASK_RUNNING || state == TASK_WAKING)
+ return 0;
+
smp_rmb(); /* see try_to_wake_up() */
- if (state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq)
+
+ if (!READ_ONCE(p->needs_rq_sync))
ip = __get_wchan(p);
- raw_spin_unlock_irq(&p->pi_lock);
return ip;
}
@@ -4365,10 +4369,49 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
* Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in
* __schedule(). See the comment for smp_mb__after_spinlock().
*
+ * Additionally, this also guards against activation of blocked
+ * donors queued on a sleeping owner when a wakeup races for
+ * the same task with sched_proxy_exec().
+ *
+ * proxy_enqueue_on_owner()
+ * STORE p->is_linked = 1
+ *
+ * block_task() try_to_wake_up()
+ * smp_mb()
+ * STRORE p->on_rq = 0 LOAD p->needs_rq_sync
+ *
+ * The read below will atomically observe either p->on_rq or
+ * p->is_linked being set and put the task on ttwu_runnable()
+ * path.
+ *
+ * On the chain-wakeup path, p->on_rq is first transitioned to
+ * TASK_ON_RQ_MIGRATING before p->is_linked is cleared and the
+ * task is transitioned to TASK_ON_RQ_QUEUED
+ *
+ * activate_blocked_task()
+ * task_rq_lock() ttwu_runnable()
+ * STORE donor->on_rq = MIGRATING task_rq_lock()
+ * # STALL
+ * smp_wmb();
+ *
+ * proxy_dequeue_from_owner()
+ * STORE donor->is_linked = 0
+ *
+ * activate_task()
+ * STORE donor->on_rq = QUEUED
+ *
+ * rq_unlock() # ACQUIRED
+ * LOAD donor->needs_rq_sync
+ *
+ * In this case too try_to_wake_up() will correctly observe
+ * either p->on_rq != 0 or p->is_linked != 0 by atomically
+ * reading p->needs_rq_sync when the task is being activated as
+ * a part of chain wakeup.
+ *
* A similar smp_rmb() lives in __task_needs_rq_lock().
*/
smp_rmb();
- if (READ_ONCE(p->on_rq) && ttwu_runnable(p, wake_flags))
+ if (READ_ONCE(p->needs_rq_sync) && ttwu_runnable(p, wake_flags))
break;
/*
@@ -4488,7 +4531,7 @@ static bool __task_needs_rq_lock(struct task_struct *p)
* See try_to_wake_up() for a longer comment.
*/
smp_rmb();
- if (p->on_rq)
+ if (READ_ONCE(p->needs_rq_sync))
return true;
/*
--
2.34.1
^ permalink raw reply [flat|nested] 23+ messages in thread* [RFC PATCH 12/16] sched:core: Add MIGRATING flags when blocking and activating linked donors
2026-08-26 6:28 [RFC PATCH 00/16][PoC] sched/core: Alternate approach to sleeping-owner handling in PROXY_EXEC K Prateek Nayak
` (10 preceding siblings ...)
2026-08-26 6:28 ` [RFC PATCH 11/16] sched/core: Prepare to inspect ->is_linked alongside ->on_rq during wakeup K Prateek Nayak
@ 2026-08-26 6:28 ` K Prateek Nayak
2026-08-26 6:28 ` [RFC PATCH 13/16] sched/core: Use p->is_linked state to unlink from sleeping owner early K Prateek Nayak
` (3 subsequent siblings)
15 siblings, 0 replies; 23+ messages in thread
From: K Prateek Nayak @ 2026-08-26 6:28 UTC (permalink / raw)
To: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
Andrea Righi, linux-kernel
Cc: Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, Waiman Long, K Prateek Nayak
Linked donor can go through a proxy_set_task_cpu() on the way to
blocking. Add DEQUEUE_MIGRATING with DEQUEUE_SLEEP when blocking proxy
donors and add ENQUEUE_MIGRATING on the activation path if the task woke
up for reason other than proxy chain-wakeup.
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
include/linux/sched.h | 1 +
kernel/sched/core.c | 9 ++++++++-
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 76de348fb88b..a05240b52806 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1003,6 +1003,7 @@ struct task_struct {
unsigned sched_contributes_to_load:1;
unsigned sched_migrated:1;
unsigned sched_task_hot:1;
+ unsigned sched_migrated_on_blocking:1;
/* Force alignment to the next boundary: */
unsigned :0;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 957f30632b82..ff499d4532ad 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2285,6 +2285,10 @@ static void block_task(struct rq *rq, struct task_struct *p, unsigned long task_
!(task_state & TASK_NOLOAD) &&
!(task_state & TASK_FROZEN);
+ p->sched_migrated_on_blocking = !!p->is_linked;
+ if (p->is_linked)
+ flags |= DEQUEUE_MIGRATING;
+
if (unlikely(is_special_task_state(task_state)))
flags |= DEQUEUE_SPECIAL;
@@ -3415,7 +3419,7 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
trace_sched_migrate_task(p, new_cpu);
if (task_cpu(p) != new_cpu) {
- if (p->sched_class->migrate_task_rq)
+ if (!p->sched_migrated_on_blocking && p->sched_class->migrate_task_rq)
p->sched_class->migrate_task_rq(p, new_cpu);
p->se.nr_migrations++;
perf_event_task_migrate(p);
@@ -3863,6 +3867,9 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
if (p->sched_contributes_to_load)
rq->nr_uninterruptible--;
+ if (p->sched_migrated_on_blocking)
+ en_flags |= ENQUEUE_MIGRATING;
+
if (wake_flags & WF_RQ_SELECTED)
en_flags |= ENQUEUE_RQ_SELECTED;
if (wake_flags & WF_MIGRATED)
--
2.34.1
^ permalink raw reply [flat|nested] 23+ messages in thread* [RFC PATCH 13/16] sched/core: Use p->is_linked state to unlink from sleeping owner early
2026-08-26 6:28 [RFC PATCH 00/16][PoC] sched/core: Alternate approach to sleeping-owner handling in PROXY_EXEC K Prateek Nayak
` (11 preceding siblings ...)
2026-08-26 6:28 ` [RFC PATCH 12/16] sched:core: Add MIGRATING flags when blocking and activating linked donors K Prateek Nayak
@ 2026-08-26 6:28 ` K Prateek Nayak
2026-08-26 6:28 ` [RFC PATCH 14/16] sched/core: Introduce chain-wakeup to activate blocked donors K Prateek Nayak
` (2 subsequent siblings)
15 siblings, 0 replies; 23+ messages in thread
From: K Prateek Nayak @ 2026-08-26 6:28 UTC (permalink / raw)
To: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
Andrea Righi, linux-kernel
Cc: Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, Waiman Long, K Prateek Nayak
Use p->is_linked indicator to unlink the task from a sleeping owner's
blocked node early and continue to proxy_needs_return() on the wakeup
path.
proxy_needs_return() is now the single point of proxy donor handling on
the wakeup path. The cases that handle enqueue of owner is as follows:
if (!owner->on_rq) {
guard(raw_spinlock)(&owner->blocked_lock) activate_task(owner)
enqueue_on_owner() owner->on_rq = QUEUED / MIGRATING
smp_mb() smp_mb()
if (owner->on_rq) /* Raced */ if (!list_empty(owner->blocked_head))
dequeue_from_owner() guard(raw_spinlock)(&owner->blocked_lock)
activate_blocked_donor()
Case that handles concurrent activation of blocked_donor is as follows:
try_to_wake_up(p)
guard(raw_spinlock)(&p->pi_lock)
/* State matches */
if (p->on_rq || p->is_linked) activate_blocked_donor(owner)
ttwu_runnable() if (!list_empty(owner->blocked_head)
task_rq_lock(p) rq_lock(cpu_rq(owner->blocked_cpu))
/* Same as task_rq(p) */
Either activate_blocked_donor() wins and activates task or
ttwu_runnable() wins and completes the wakeup after dequeuing from
sleeping owner.
activate_blocked_donor() changes are added in the subsequent patch.
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
kernel/sched/core.c | 176 +++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 174 insertions(+), 2 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index ff499d4532ad..e6abcea97683 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2252,6 +2252,96 @@ static void __proxy_dequeue_from_owner(struct task_struct *p)
list_del_init(&p->blocked_node);
WRITE_ONCE(p->sleeping_owner, NULL);
WRITE_ONCE(p->is_linked, 0);
+ /*
+ * p->blocked_cpu is intentionally left un-altered
+ * since the wakeup path requires it to grab the
+ * correct rq for chain wakeup if we are here from
+ * the activation path.
+ *
+ * "p->blocked_cpu" only has significance for blocked
+ * task on "blocked_head" and a stale value in the
+ * field is harmless.
+ */
+}
+
+static void proxy_dequeue_from_owner(struct task_struct *p)
+{
+ struct task_struct *owner = p->sleeping_owner;
+
+ scoped_guard(raw_spinlock, &owner->blocked_lock) {
+ WARN_ON_ONCE(p->sleeping_owner != owner);
+ __proxy_dequeue_from_owner(p);
+ }
+}
+
+static bool proxy_try_dequeue_from_owner(struct task_struct *p)
+{
+ struct task_struct *owner = READ_ONCE(p->sleeping_owner);
+
+ /*
+ * Task is outside __task_rq_lock(). These fields
+ * can race with a chain-wakeup.
+ */
+ if (!owner || READ_ONCE(p->on_rq))
+ return false;
+
+ /*
+ * The chain-wakeup slowpath grabs both the rq_lock() and the
+ * owner->blocked_lock while removing the tasks on
+ * owner->blocked_head.
+ *
+ * If there is no concurrent chain-wakeup for this task, it is
+ * safe to just hold the owner->blocked_lock which serializes
+ * the removal like:
+ *
+ * proxy_try_dequeue_from_owner(donor) proxy_activate_blocked_donor(owner)
+ * LOCK rq_lock(p->blocked_cpu)
+ * LOCK &owner->blocked_lock LOCK &owner->blocked_lock
+ *
+ * # Defensive checks STORE donor->on_rq = MIGRATING
+ * __proxy_dequeue_from_owner() smp_mb()
+ *
+ * UNLOCK &owner->blocked_lock UNLOCK &owner->blocked_lock
+ *
+ * # If proxy_try_dequeue fails
+ * __task_rq_lock(donor) __proxy_dequeue_from_owner(donor)
+ * if (donor->on_rq == MIGRATING) __activate_task(donor)
+ * cpu_relax() STORE p->on_rq = 1
+ *
+ * # Sees consistent state.
+ *
+ * Either proxy_activate_blocked_donor() manages to dequeue the donor
+ * and stall the __task_rq_lock() until the task is actually stable
+ * or proxy_try_dequeue_from_owner() wins and manages to remove the
+ * task from "owner->blocked_head" first.
+ */
+ scoped_guard(raw_spinlock, &owner->blocked_lock) {
+ /*
+ * Chain-wakeup has raced to queue the task.
+ * Try again with __task_rq_lock() held.
+ */
+ if (READ_ONCE(p->on_rq))
+ return false;
+
+ /*
+ * Something changes in the link. Try again with
+ * __task_rq_lock() held via proxy_needs_return()
+ * where both p->on_rq and p->is_linked is stable.
+ */
+ if (!READ_ONCE(p->is_linked) || owner != READ_ONCE(p->sleeping_owner))
+ return false;
+
+ __proxy_dequeue_from_owner(p);
+ }
+
+ return true;
+}
+
+#else /* !CONFIG_SCHED_PROXY_EXEC */
+
+static bool proxy_try_dequeue_from_owner(struct task_struct *p)
+{
+ return false;
}
#endif /* CONFIG_SCHED_PROXY_EXEC */
@@ -3799,6 +3889,40 @@ void __proxy_block_task(struct task_struct *p)
if (!sched_proxy_exec())
return;
+ if (unlikely(p->is_linked && p->blocked_cpu != task_cpu(p))) {
+ WRITE_ONCE(p->on_rq, TASK_ON_RQ_MIGRATING);
+ ASSERT_EXCLUSIVE_WRITER(p->on_rq);
+
+ /* Move iowait signal to new CPU. */
+ if (p->in_iowait) {
+ atomic_dec(&task_rq(p)->nr_iowait);
+ atomic_inc(&cpu_rq(p->blocked_cpu)->nr_iowait);
+ }
+
+ /*
+ * Moved the linked task to p->blocked_cpu which is now
+ * resposnible for chain wakeup. TASK_ON_RQ_MIGRATING
+ * above stalls task_rq_lock() until p->on_rq is
+ * transitioned to 0 (See ___task_rq_lock()).
+ */
+ proxy_set_task_cpu(p, p->blocked_cpu);
+
+ /*
+ * Preserve p->blocked_cpu for any potential future
+ * additions to p->blocked_head. The chain should be
+ * re-directed to the same "blocked_cpu".
+ *
+ * XXX: Technically, it is possible to keep the
+ * "blocked_cpu" for this sub-chain different
+ * compared to our ancestors but the chain-wakeup
+ * would need to juggle multiple locks then to
+ * ensure it doesn't race with concurrent wakeups.
+ * Matching "blocked_cpu" requires only a single
+ * rq_lock which is simpler.
+ */
+ return;
+ }
+
/*
* This is the CPU where the __active_blocked_donor() slowpath will
* resume activation of blocked donors queued on this task.
@@ -3819,6 +3943,25 @@ void __proxy_block_task(struct task_struct *p)
*/
static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p)
{
+ if (!sched_proxy_exec())
+ return false;
+
+ /* See try_to_wake_up() for ordering guarantees. */
+ if (unlikely(p->is_linked)) {
+ /*
+ * Task is queued on an owner's blocked_node.
+ *
+ * Dequeue from owner's blocked_node before going
+ * further. Do not take the fast-path since this task is
+ * fully blocked and it is necessary to clear
+ * p->blocked_on indicator before waking up.
+ */
+ WARN_ON_ONCE(p->on_rq);
+ proxy_dequeue_from_owner(p);
+ clear_task_blocked_on(p, NULL);
+ return true;
+ }
+
/*
* Typically per __set_task_cpu(), task_cpu(p) == p->wake_cpu.
*
@@ -3922,10 +4065,34 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
*/
static int ttwu_runnable(struct task_struct *p, int wake_flags)
{
+ struct rq *rq;
+
+ if (sched_proxy_exec() && READ_ONCE(p->is_linked)) {
+ /*
+ * If p->is_linked put us on the ttwu_runnable() path,
+ * try to de-link the blocked donor outside the
+ * __task_rq_lock() first.
+ *
+ * If the task state changed, a wakeup raced with
+ * ttwu_runnable() (see proxy_try_dequeue_from_owner()
+ * for interleaving). Try again with rq_lock held where
+ * state is guaranteed to be stable.
+ */
+ if (proxy_try_dequeue_from_owner(p)) {
+ WARN_ON_ONCE(READ_ONCE(p->on_rq));
+ clear_task_blocked_on(p, NULL);
+ return 0;
+ }
+ }
+
ACQUIRE(__task_rq_lock, guard)(p);
- struct rq *rq = guard.rq;
+ rq = guard.rq;
- if (!task_on_rq_queued(p))
+ /*
+ * schedule() managed to block the task and there is
+ * no unlinking needed. Continue with wakeup.
+ */
+ if (!p->needs_rq_sync)
return 0;
update_rq_clock(rq);
@@ -6972,6 +7139,11 @@ static void proxy_enqueue_on_owner(struct rq *rq, struct task_struct *owner,
WARN_ON(!p->on_rq);
WARN_ON(p->sleeping_owner);
+ /*
+ * See __proxy_block_task() on how p->blocked_cpu is used
+ * to re-direct the task to a handler CPU when
+ */
+ WRITE_ONCE(p->blocked_cpu, READ_ONCE(owner->blocked_cpu));
WRITE_ONCE(p->sleeping_owner, owner);
list_add(&p->blocked_node, &owner->blocked_head);
WRITE_ONCE(p->is_linked, 1);
--
2.34.1
^ permalink raw reply [flat|nested] 23+ messages in thread* [RFC PATCH 14/16] sched/core: Introduce chain-wakeup to activate blocked donors
2026-08-26 6:28 [RFC PATCH 00/16][PoC] sched/core: Alternate approach to sleeping-owner handling in PROXY_EXEC K Prateek Nayak
` (12 preceding siblings ...)
2026-08-26 6:28 ` [RFC PATCH 13/16] sched/core: Use p->is_linked state to unlink from sleeping owner early K Prateek Nayak
@ 2026-08-26 6:28 ` K Prateek Nayak
2026-08-26 6:28 ` [RFC PATCH 15/16] locking/mutex: Track locks owned by a task in a per-task counter K Prateek Nayak
2026-08-26 6:29 ` [RFC PATCH 16/16] sched/core: Set activation of non lock-holders to fast-path K Prateek Nayak
15 siblings, 0 replies; 23+ messages in thread
From: K Prateek Nayak @ 2026-08-26 6:28 UTC (permalink / raw)
To: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
Andrea Righi, linux-kernel
Cc: Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, Waiman Long, K Prateek Nayak
Activating blocked task with sched_proxy_exec() needed grabbing
p->blocked_lock for all wakeups to prevent missing additions to
blocked_head during wakeup.
With proxy_enqueue_on_owner() converted to plug the race between enqueue
on owner and owner's wakeup, simplify the wakeup path with the following
three step process:
activate_blocked_task(owner)
/* Part 1: Enqueue race */ enqueue_on_owner(owner, p)
owner->on_rq = TASK_ON_RQ_MIGRATING; list_add(&p->blocked_node, &owner->blocked_head)
smp_mb(); smp_mb();
if (list_empty(&owner->blocked_head)) if (owner->on_rq) /* Raced */
/* list_del_init(&p->blocked_node)
* Fast-path: Future additions will return;
* see owner->on_rq != 0 and dequeue /*
* themself. * Owner has to observe !list_empty() after
*/ * this point so it is safe to block.
__activate_task(p) */
block_task(p);
/* END: Part 1 */
enqueue_on_owner() holds mutex->wait_lock and "owner" needs to grab the
same wait_lock during unlock so "owner" cannot disappear during
enqueue_on_owner().
/* Part 2: Prepare migration chain */
rq_unlock(rq);
LIST_HEAD(migration_list)
LIST_HEAD(wakeup_list)
try_to_wake_up()
/* Sees proxy_task_is_linked(p) */
ttwu_runnable(donor)
rq_lock(rq_of(owner->blocked_cpu)) task_rq_lock(donor) /* Same rq */
do {
list_for_each_entry(donor, &owner->blocked_head) if (task_is_linked(donor)
donor->on_rq = TASK_ON_RQ_MIGRATING /* Part 1 */ list_del_init(&p->blocked_node)
if (list_empty(&owner->blocked_head) activate_blocked_task(donor)
list_add(&doner, &migration_list) /* Same as left */
else
list_add(&donor, &wakeup_list)
list_add(&owner, &wakeup_list)
owner = list_next_entry(&migration_list)
} while (!list_empty(&migration_list))
rq_unlock(rq_of(owner->blocked_cpu))
/* END: Part 2 */
Since owner->blocked_cpu matches with task_cpu() of the entire chain
(including tasks queued on the blocked donors, and task queued on them,
and so on), grabbing "rq_of(owner->blocked_cpu)" is enough to stall any
concurrent wakeups for any of the linked tasks.
If try_to_wake_up() wins, donor will remove itself from the chain and
continue its wakeup as if it is a blocked task waking up.
If activate_blocked_task() wins, it will set all the donors to
TASK_ON_RQ_MIGRATING, repeating Part 1, to prevent other tasks from
queueing on it form then on and iterates its "blocked_head". This is
done in a breadth-first fashion until full chain is set to
TASK_ON_RQ_MIGRATING.
"owner->blocked_head" traversal and manipulations are still guarded by
"owner->blocked_lock" which has not been depicted here.
/* Part 3: Wakeup chain */
rq_lock(rq);
__activate_task(p); /* Activate top owner first */
/* Activate the entire chain next */
list_for_each_entry(donor, &owner->blocked_head)
list_del_init(&donor->blocked_node)
__activate_task(donor)
/* END: Part 3 */
The wakeup steps skipped for donors is performormed during the chain
wakeup.
Advantages of this method:
o Does not involve juggling p->pi_lock, rq_lock, owner->blocked_lock
in addition to checking task state. Task on chain is fully blocked
and only one rq_lock + onwer_blocked lock can hold off
ttwu_runnable() from waking them.
o No need to borrow reference to "owner" before queuing. The mechanism
ensures p->blocked_owner remains valid as long as is task is on chain.
Disadvantages:
o Since chain wakeups funnel through a single CPU, it is possible for
increased lock contention however large chain wakeups and concurrent
wakeups of donors for reasons other than an unlock is rare.
Co-developed-by: John Stultz <jstultz@google.com>
Signed-off-by: John Stultz <jstultz@google.com>
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
kernel/sched/core.c | 183 +++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 181 insertions(+), 2 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index e6abcea97683..3153ad007447 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2247,6 +2247,8 @@ void activate_task(struct rq *rq, struct task_struct *p, int en_flags)
#ifdef CONFIG_SCHED_PROXY_EXEC
+static inline void proxy_set_task_cpu(struct task_struct *p, int cpu);
+
static void __proxy_dequeue_from_owner(struct task_struct *p)
{
list_del_init(&p->blocked_node);
@@ -2337,6 +2339,183 @@ static bool proxy_try_dequeue_from_owner(struct task_struct *p)
return true;
}
+static void
+proxy_activate_blocked_task(struct rq *rq, struct task_struct *p, int en_flags)
+{
+ int iowait_count = 0, load_contrib_count = 0;
+ struct task_struct *donor, *owner, *tmp;
+ int blocked_cpu, this_cpu = cpu_of(rq);
+ LIST_HEAD(migration_head);
+ LIST_HEAD(wakeup_head);
+ struct rq *blocked_rq;
+ bool needs_migration;
+
+ WRITE_ONCE(p->on_rq, TASK_ON_RQ_MIGRATING);
+ ASSERT_EXCLUSIVE_WRITER(p->on_rq);
+
+ /*
+ * Pairs against smp_mb() in proxy_enqueue_on_owner() which
+ * orders ownwer->on_rq state against the blocked_head addition.
+ */
+ smp_mb();
+
+ /*
+ * Fast-path: Tasks blocking on us will see p->on_rq updated and
+ * will bail out natturally in proxy_enqueue_on_owner().
+ *
+ * proxy_enqueue_on_owner() holds the wait_lock to prevent owner
+ * from running and disappearing before the transient task can
+ * observe p->on_rq change and dequeue itself.
+ */
+ if (list_empty(&p->blocked_head)) {
+ __activate_task(rq, p, en_flags);
+ return;
+ }
+
+ /*
+ * Slow-path: Since we need to wakeup tasks (and task queued on
+ * them, and task queued on them, and ... you get the gist) we
+ * need to grab some locks.
+ *
+ * Lucky for us, we have simplified this via p->blocked_cpu
+ * which is the task_cpu() for the entire chain. Wakeup is
+ * broken into two parts.
+ *
+ * Part1: Under the rq_lock + blocked lock, mark all tasks on
+ * the chain as TASK_ON_RQ_MIGRATING in breadth-first manner.
+ *
+ * Part2: Active all the tasks in bulk.
+ */
+
+ /* Part 1: Prepare the blocked donor chain. */
+ blocked_cpu = p->blocked_cpu;
+ blocked_rq = cpu_rq(blocked_cpu);
+
+ /*
+ * Blocked chain is linked to a different rq. Drop the rq_lock
+ * and migrate the chain over before wkaing it up here.
+ */
+ needs_migration = blocked_rq != rq;
+ if (needs_migration) {
+ raw_spin_rq_unlock(rq);
+ raw_spin_rq_lock(blocked_rq);
+ }
+
+ owner = p;
+
+ do {
+ guard(raw_spinlock)(&owner->blocked_lock);
+
+ /* Iterate the tasks blocked on this owner. */
+ list_for_each_entry_safe(donor, tmp, &owner->blocked_head, blocked_node) {
+ /* Everything hinges on this assumption. */
+ WARN_ON_ONCE(task_cpu(donor) != blocked_cpu);
+ WARN_ON_ONCE(donor->se.sched_delayed);
+
+ WRITE_ONCE(donor->on_rq, TASK_ON_RQ_MIGRATING);
+ ASSERT_EXCLUSIVE_WRITER(donor->on_rq);
+
+ if (donor->in_iowait) {
+ delayacct_blkio_end(donor);
+ iowait_count++;
+ }
+
+ if (donor->sched_contributes_to_load)
+ load_contrib_count++;
+
+ /*
+ * Pairs against the smp_mb() in proxy_enqueue_on_owner().
+ * See the comment at the beginning of the function.
+ */
+ smp_mb();
+
+ /*
+ * If the donor has more task queued on it,
+ * add it to the migration list else directly
+ * queue it onto the wakeup list.
+ */
+ if (!list_empty(&donor->blocked_head)) {
+ list_move_tail(&donor->blocked_node, &migration_head);
+ } else {
+ list_move_tail(&donor->blocked_node, &wakeup_head);
+ }
+ }
+
+ owner = list_first_entry_or_null(&migration_head,
+ typeof(*owner),
+ blocked_node);
+
+ /*
+ * Move the owner to the wakeup list before preparing
+ * the sub-chain queued on it.
+ */
+ if (owner)
+ list_move_tail(&owner->blocked_node, &wakeup_head);
+
+ } while (owner);
+
+ /* Part 2: Bulk wakeup */
+
+ atomic_sub(iowait_count, &blocked_rq->nr_iowait);
+
+ if (needs_migration) {
+ raw_spin_rq_unlock(blocked_rq);
+ raw_spin_rq_lock(rq);
+
+ update_rq_clock(rq);
+ en_flags |= ENQUEUE_NOCLOCK;
+ }
+
+ rq->nr_uninterruptible -= load_contrib_count;
+
+ if (!(en_flags & ENQUEUE_NOCLOCK)) {
+ update_rq_clock(rq);
+ en_flags |= ENQUEUE_NOCLOCK;
+ }
+
+ /* Activate the top owner first. */
+ __activate_task(rq, p, en_flags);
+
+ /*
+ * Add ENQUEUE_MIGRATED for all tasks on the chain
+ * since they are all sched_migrated_on_blocking
+ * and have skipped ->migrate_task_rq() callback.
+ */
+ en_flags |= ENQUEUE_MIGRATING;
+
+ list_for_each_entry_safe(donor, tmp, &wakeup_head, blocked_node) {
+ __proxy_dequeue_from_owner(donor);
+
+ /*
+ * Correct the task_cpu() before activating.
+ *
+ * Since chain preparation already adds enough barrier
+ * after storing p->on_rq = MIGRATING, no additional
+ * barriers are required after modifying p->is_linked
+ * via __proxy_dequeue_from_owner() above.
+ */
+ if (needs_migration)
+ proxy_set_task_cpu(donor, this_cpu);
+
+ __activate_task(rq, donor, en_flags);
+ wakeup_preempt(rq, donor, en_flags);
+ /*
+ * Blocked tasks do not add push callbacks and it is
+ * safe to skip calling ->task_woken() here.
+ */
+ }
+}
+
+static void activate_blocked_task(struct rq *rq, struct task_struct *p, int en_flags)
+{
+ if (!sched_proxy_exec()) {
+ __activate_task(rq, p, en_flags);
+ return;
+ }
+
+ proxy_activate_blocked_task(rq, p, en_flags);
+}
+
#else /* !CONFIG_SCHED_PROXY_EXEC */
static bool proxy_try_dequeue_from_owner(struct task_struct *p)
@@ -2344,13 +2523,13 @@ static bool proxy_try_dequeue_from_owner(struct task_struct *p)
return false;
}
-#endif /* CONFIG_SCHED_PROXY_EXEC */
-
static void activate_blocked_task(struct rq *rq, struct task_struct *p, int en_flags)
{
__activate_task(rq, p, en_flags);
}
+#endif /* CONFIG_SCHED_PROXY_EXEC */
+
void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
{
WARN_ON_ONCE(flags & DEQUEUE_SLEEP);
--
2.34.1
^ permalink raw reply [flat|nested] 23+ messages in thread* [RFC PATCH 15/16] locking/mutex: Track locks owned by a task in a per-task counter
2026-08-26 6:28 [RFC PATCH 00/16][PoC] sched/core: Alternate approach to sleeping-owner handling in PROXY_EXEC K Prateek Nayak
` (13 preceding siblings ...)
2026-08-26 6:28 ` [RFC PATCH 14/16] sched/core: Introduce chain-wakeup to activate blocked donors K Prateek Nayak
@ 2026-08-26 6:28 ` K Prateek Nayak
2026-08-26 6:29 ` [RFC PATCH 16/16] sched/core: Set activation of non lock-holders to fast-path K Prateek Nayak
15 siblings, 0 replies; 23+ messages in thread
From: K Prateek Nayak @ 2026-08-26 6:28 UTC (permalink / raw)
To: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
Andrea Righi, linux-kernel
Cc: Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, Waiman Long, K Prateek Nayak
With sleeping owner handling, the task activation path will need to be
extra careful when waking up a task that can resolve to a mutex_owner().
Track the lock acquisitions in a per-task counter to selectively put
mutex owner activations onto a slowpath.
The per-task accounting is done locklessly on the current task. This is
only relevant for a blocking owner and barrie in __block_task() ensures
the observer sees the correct "lock_nesting" before p->on_rq is
transitioned to 0.
The counter will be used in subsequent patches to optimize
activate_blocked_task() and __proxy_block_task().
One exception to the lock tracking rule is the increment before
schedule_preempt_disabled() and decrement after to fix the former
mis-count.
This is necessary for mutex since a handoff can set fully blocked task
as mutex_owner and, at that moment, other tasks can start queuing
themselves onto the blocked owner which requires the wakeup path to go
via slow-path and ensure blocked waiters (if any) are activated.
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
include/linux/sched.h | 6 +++++
kernel/locking/mutex.c | 57 ++++++++++++++++++++++++++++++++----------
2 files changed, 50 insertions(+), 13 deletions(-)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index a05240b52806..cb58992814fb 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1272,6 +1272,12 @@ struct task_struct {
struct task_struct *sleeping_owner; /* task our blocked_node is enqueued on */
int blocked_cpu; /* CPU where task was blocked. */
#endif
+ /*
+ * HACK: These bits should technically live in CONFIG_SCHED_PROXY_EXEC
+ * block and only account when sched_proxy_exec() is true but for the
+ * PoC keep it outside and assume proxy is always enabled ;-)
+ */
+ unsigned int lock_nesting;
/*
* The task that is boosting this task; a back link for the current
diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index 8a85912d7ee6..23a63e117c3e 100644
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -84,7 +84,8 @@ unsigned long mutex_get_owner(struct mutex *lock)
*/
static inline struct task_struct *__mutex_trylock_common(struct mutex *lock, bool handoff)
{
- unsigned long owner, curr = (unsigned long)current;
+ struct task_struct *cur_task = current;
+ unsigned long owner, curr = (unsigned long)cur_task;
owner = atomic_long_read(&lock->owner);
for (;;) { /* must loop, can race against a flag */
@@ -109,8 +110,10 @@ static inline struct task_struct *__mutex_trylock_common(struct mutex *lock, boo
}
if (atomic_long_try_cmpxchg_acquire(&lock->owner, &owner, task | flags)) {
- if (task == curr)
+ if (task == curr) {
+ cur_task->lock_nesting++;
return NULL;
+ }
break;
}
}
@@ -153,13 +156,16 @@ EXPORT_SYMBOL(mutex_init_generic);
static __always_inline bool __mutex_trylock_fast(struct mutex *lock)
__cond_acquires(true, lock)
{
- unsigned long curr = (unsigned long)current;
+ struct task_struct *cur_task = current;
+ unsigned long curr = (unsigned long)cur_task;
unsigned long zero = 0UL;
MUTEX_WARN_ON(lock->magic != lock);
- if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr))
+ if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr)) {
+ cur_task->lock_nesting++;
return true;
+ }
return false;
}
@@ -167,9 +173,15 @@ static __always_inline bool __mutex_trylock_fast(struct mutex *lock)
static __always_inline bool __mutex_unlock_fast(struct mutex *lock)
__cond_releases(true, lock)
{
- unsigned long curr = (unsigned long)current;
+ struct task_struct *cur_task = current;
+ unsigned long curr = (unsigned long)cur_task;
+
+ if (atomic_long_try_cmpxchg_release(&lock->owner, &curr, 0UL)) {
+ cur_task->lock_nesting--;
+ return true;
+ }
- return atomic_long_try_cmpxchg_release(&lock->owner, &curr, 0UL);
+ return false;
}
#else /* !CONFIG_DEBUG_LOCK_ALLOC */
@@ -723,8 +735,23 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);
+ /*
+ * Mutex handoff can set the current as the lock owner while
+ * it is blocked. Increment "nesting_count" on the way to
+ * schedule() to notify the wakeup path that __mutex_owner()
+ * can resolve to this task on the way back.
+ */
+ current->lock_nesting++;
+
schedule_preempt_disabled();
+ /*
+ * Fix the guard against handoff. If the task got the lock,
+ * __mutex_trylock*() will increment the lock_nesting when the
+ * acquisition is finalized before exiting.
+ */
+ current->lock_nesting--;
+
first = lock->first_waiter == &waiter;
raw_spin_lock_irqsave(&lock->wait_lock, flags);
@@ -981,7 +1008,7 @@ EXPORT_SYMBOL_GPL(ww_mutex_lock_interruptible);
static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)
__releases(lock)
{
- struct task_struct *donor, *next = NULL;
+ struct task_struct *cur_task, *donor, *next = NULL;
struct mutex_waiter *waiter;
unsigned long owner;
unsigned long flags;
@@ -997,6 +1024,10 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
* missed.
*/
guard(preempt)();
+
+ cur_task = current;
+ cur_task->lock_nesting--;
+
/*
* Release the lock before (potentially) taking the spinlock such that
* other contenders can get on with things ASAP.
@@ -1006,10 +1037,10 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
*/
owner = atomic_long_read(&lock->owner);
for (;;) {
- MUTEX_WARN_ON(__owner_task(owner) != current);
+ MUTEX_WARN_ON(__owner_task(owner) != cur_task);
MUTEX_WARN_ON(owner & MUTEX_FLAG_PICKUP);
- if (sched_proxy_exec() && current->blocked_donor) {
+ if (sched_proxy_exec() && cur_task->blocked_donor) {
/* force handoff if we have a blocked_donor */
owner = MUTEX_FLAG_HANDOFF;
break;
@@ -1027,7 +1058,7 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
}
raw_spin_lock_irqsave(&lock->wait_lock, flags);
- raw_spin_lock(¤t->blocked_lock);
+ raw_spin_lock(&cur_task->blocked_lock);
debug_mutex_unlock(lock);
if (sched_proxy_exec()) {
@@ -1036,7 +1067,7 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
* current through this lock, hand the lock to that task, as that
* is the highest waiter, as selected by the scheduling function.
*/
- donor = current->blocked_donor;
+ donor = cur_task->blocked_donor;
if (donor) {
struct mutex *next_lock;
@@ -1045,7 +1076,7 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
if (next_lock == lock) {
next = get_task_struct(donor);
__clear_task_blocked_on(next, lock);
- current->blocked_donor = NULL;
+ cur_task->blocked_donor = NULL;
}
raw_spin_unlock(&donor->blocked_lock);
}
@@ -1071,7 +1102,7 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
if (owner & MUTEX_FLAG_HANDOFF)
__mutex_handoff(lock, next);
- raw_spin_unlock(¤t->blocked_lock);
+ raw_spin_unlock(&cur_task->blocked_lock);
raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
if (next) {
wake_up_process(next);
--
2.34.1
^ permalink raw reply [flat|nested] 23+ messages in thread* [RFC PATCH 16/16] sched/core: Set activation of non lock-holders to fast-path
2026-08-26 6:28 [RFC PATCH 00/16][PoC] sched/core: Alternate approach to sleeping-owner handling in PROXY_EXEC K Prateek Nayak
` (14 preceding siblings ...)
2026-08-26 6:28 ` [RFC PATCH 15/16] locking/mutex: Track locks owned by a task in a per-task counter K Prateek Nayak
@ 2026-08-26 6:29 ` K Prateek Nayak
15 siblings, 0 replies; 23+ messages in thread
From: K Prateek Nayak @ 2026-08-26 6:29 UTC (permalink / raw)
To: John Stultz, Suleiman Souhlal, Ingo Molnar, Peter Zijlstra,
Juri Lelli, Vincent Guittot, Will Deacon, Boqun Feng,
Andrea Righi, linux-kernel
Cc: Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, Waiman Long, K Prateek Nayak
Tasks with a 0 count in "p->lock_nesting" cannot resolve to a
__mutex_owner(). Set actiavations and blocking of these task to
fast-path (plain __activate_task()) to avoid the overheads.
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
kernel/sched/core.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 3153ad007447..3ab4cd36f887 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2508,7 +2508,7 @@ proxy_activate_blocked_task(struct rq *rq, struct task_struct *p, int en_flags)
static void activate_blocked_task(struct rq *rq, struct task_struct *p, int en_flags)
{
- if (!sched_proxy_exec()) {
+ if (likely(!sched_proxy_exec() || !p->lock_nesting)) {
__activate_task(rq, p, en_flags);
return;
}
@@ -4065,7 +4065,7 @@ static inline void proxy_reset_donor(struct rq *rq)
void __proxy_block_task(struct task_struct *p)
{
- if (!sched_proxy_exec())
+ if (!sched_proxy_exec() || likely(!p->lock_nesting))
return;
if (unlikely(p->is_linked && p->blocked_cpu != task_cpu(p))) {
--
2.34.1
^ permalink raw reply [flat|nested] 23+ messages in thread