[PATCH] sched/core: Preserve wake flags across queued wakeups

[PATCH] sched/core: Preserve wake flags across queued wakeups

[Shubhang Kaushik (Ampere)]

The queued wakeup path only remembers whether the wakee migrated. When
the target CPU drains the wakelist, sched_ttwu_pending() turns that back
into either WF_MIGRATED or 0 and passes it to ttwu_do_activate().

That drops the rest of the wake flags seen by the direct wakeup path. In
particular, WF_SYNC, WF_TTWU and WF_RQ_SELECTED can still matter for
wakeup preemption, so queued wakeups can end up making a different
decision from a direct wakeup.

Store the relevant wake_flags bits instead of a single boolean and pass
them through when the wakelist is drained. Keep WF_CURRENT_CPU out of
the saved mask as it is a placement hint for CPU selection, not an
activation flag.

Signed-off-by: Shubhang Kaushik (Ampere) <sh@gentwo.org>
---
This aims to preserve activation-relevant wake flags across queued
remote wakeups, keeping the queued and direct wakeup paths consistent when
calling ttwu_do_activate().

Tested on an Ampere Altra 80 CPU system with perf bench sched messaging and
hackbench. Results were within run-to-run noise of the unpatched mainline
baseline.

Baseline: mainline origin/master at d96fcfe1b7f9.
---
 include/linux/sched.h | 6 +++---
 kernel/sched/core.c   | 9 +++++++--
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 373bcc0598d10b4256a11f8c8373ece78fa5e0e5..5d55cdf696c13ed1c95869a1b6fb4dc1485727dc 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1003,12 +1003,12 @@ struct task_struct {
 	 * schedule()			  if (p->on_rq && ..) // false
 	 *   smp_mb__after_spinlock();	  if (smp_load_acquire(&p->on_cpu) && //true
 	 *   deactivate_task()		      ttwu_queue_wakelist())
-	 *     p->on_rq = 0;			p->sched_remote_wakeup = Y;
+	 *     p->on_rq = 0;			p->sched_remote_wakeup_flags = Y;
 	 *
 	 * guarantees all stores of 'current' are visible before
-	 * ->sched_remote_wakeup gets used, so it can be in this word.
+	 * ->sched_remote_wakeup_flags gets used, so it can be in this word.
 	 */
-	unsigned			sched_remote_wakeup:1;
+	unsigned			sched_remote_wakeup_flags:8;
 #ifdef CONFIG_RT_MUTEXES
 	unsigned			sched_rt_mutex:1;
 #endif
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 96226707c2f6135341aa779b8262f113e103d8ad..0d2918b66be277ec09109022bf3ede4b9ec8220c 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -3902,13 +3902,17 @@ void sched_ttwu_pending(void *arg)
 	update_rq_clock(rq);
 
 	llist_for_each_entry_safe(p, t, llist, wake_entry.llist) {
+		int wake_flags;
+
 		if (WARN_ON_ONCE(p->on_cpu))
 			smp_cond_load_acquire(&p->on_cpu, !VAL);
 
 		if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq)))
 			set_task_cpu(p, cpu_of(rq));
 
-		ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf);
+		wake_flags = p->sched_remote_wakeup_flags;
+		p->sched_remote_wakeup_flags = 0;
+		ttwu_do_activate(rq, p, wake_flags, &rf);
 	}
 
 	/*
@@ -3951,7 +3955,8 @@ static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags
 {
 	struct rq *rq = cpu_rq(cpu);
 
-	p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
+	p->sched_remote_wakeup_flags = wake_flags &
+		(WF_TTWU | WF_SYNC | WF_MIGRATED | WF_RQ_SELECTED);
 
 	WRITE_ONCE(rq->ttwu_pending, 1);
 #ifdef CONFIG_SMP

---
base-commit: d96fcfe1b7f94ac742984ae7986b94a116abff1b
change-id: 20260710-b4-sched-ttwu-wake-flags-e87ada0e65e1

Best regards,
-- 
Shubhang Kaushik (Ampere) <sh@gentwo.org>

Re: [PATCH] sched/core: Preserve wake flags across queued wakeups

[K Prateek Nayak]

Hello Shubhang,

On 7/11/2026 10:47 AM, Shubhang Kaushik (Ampere) wrote:
> diff --git a/include/linux/sched.h b/include/linux/sched.h
> index 373bcc0598d10b4256a11f8c8373ece78fa5e0e5..5d55cdf696c13ed1c95869a1b6fb4dc1485727dc 100644
> --- a/include/linux/sched.h
> +++ b/include/linux/sched.h
> @@ -1003,12 +1003,12 @@ struct task_struct {
>          * schedule()                     if (p->on_rq && ..) // false
>          *   smp_mb__after_spinlock();    if (smp_load_acquire(&p->on_cpu) && //true
>          *   deactivate_task()                ttwu_queue_wakelist())
> -        *     p->on_rq = 0;                    p->sched_remote_wakeup = Y;
> +        *     p->on_rq = 0;                    p->sched_remote_wakeup_flags = Y;
>          *
>          * guarantees all stores of 'current' are visible before
> -        * ->sched_remote_wakeup gets used, so it can be in this word.
> +        * ->sched_remote_wakeup_flags gets used, so it can be in this word.
>          */
> -       unsigned                        sched_remote_wakeup:1;
> +       unsigned                        sched_remote_wakeup_flags:8;

If we are keeping a byte for it, we can just make it a u8 just after
the "personality" variable.

The reason to keep it at a byte boundary from the rq_lock serialized
stuff was to prevent clobbering them but that would be moot now since a
byte sized writes is always atomic across all architectures afaik.
(https://lore.kernel.org/lkml/20201116142005.GE3121392@hirez.programming.kicks-ass.net/)


>  #ifdef CONFIG_RT_MUTEXES
>         unsigned                        sched_rt_mutex:1;
>  #endif
> diff --git a/kernel/sched/core.c b/kernel/sched/core.c
> index 96226707c2f6135341aa779b8262f113e103d8ad..0d2918b66be277ec09109022bf3ede4b9ec8220c 100644
> --- a/kernel/sched/core.c
> +++ b/kernel/sched/core.c
> @@ -3902,13 +3902,17 @@ void sched_ttwu_pending(void *arg)
>         update_rq_clock(rq);
> 
>         llist_for_each_entry_safe(p, t, llist, wake_entry.llist) {
> +               int wake_flags;
> +
>                 if (WARN_ON_ONCE(p->on_cpu))
>                         smp_cond_load_acquire(&p->on_cpu, !VAL);
> 
>                 if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq)))
>                         set_task_cpu(p, cpu_of(rq));
> 
> -               ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf);
> +               wake_flags = p->sched_remote_wakeup_flags;
> +               p->sched_remote_wakeup_flags = 0;

No need to reset it. This path is only used for wakeups and next remote
wakeup would set these flags correctly before putting the task on the
list.

> +               ttwu_do_activate(rq, p, wake_flags, &rf);

On aside note: Only preempt_sync() in wakeup_preempt_fair() cares for
this but if the ttwu manages to find an idle CPU, it would preempt the
idle tasks anyways and if there are no idle CPUs and WF_SYNC works, it
will never use the sched_ttwu_pending() path since it is a wakeup on
the local CPU.

So, the question is, are these flags even relevant with the remote
enqueue path?

>         }
> 
>         /*
> @@ -3951,7 +3955,8 @@ static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags
>  {
>         struct rq *rq = cpu_rq(cpu);
> 
> -       p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
> +       p->sched_remote_wakeup_flags = wake_flags &
> +               (WF_TTWU | WF_SYNC | WF_MIGRATED | WF_RQ_SELECTED);

You can define a WF_MASK next to all the WF_* flags definition and
use it here. It is slightly easier on the eyes ;-)

> 
>         WRITE_ONCE(rq->ttwu_pending, 1);
>  #ifdef CONFIG_SMP
> 
-- 
Thanks and Regards,
Prateek

Re: [PATCH] sched/core: Preserve wake flags across queued wakeups

[Shubhang]

[-- Attachment #1: Type: text/plain, Size: 4744 bytes --]

Hi Prateek,

Thanks for the review.

Agreed on the cleanup points. I’ll fold them into v2. The old thread you
linked makes the storage concern clear as well, so v2 will use a 
standalone u8 outside the scheduler bitfields.

Yes, if the target CPU is idle, these flags do not really change the 
outcome. But the case I was looking at is the remote wakelist 
path for CPUs that do not share cache with the waker, where 
ttwu_queue_cond() can still queue the wakeup without requiring the target 
rq to be idle.

In that case the target CPU may drain the wakelist while running a 
non-idle task, so the queued path can still reach wakeup_preempt() with 
the saved wake flags. There is also the p->on_cpu handoff path, which can 
queue before select_task_rq() and still carry WF_SYNC/WF_TTWU.

I’ll send a v2 with these cleanups and tone down the changelog to frame 
this as preserving direct-vs-queued wakeup semantics.

Regards,
Shubhang Kaushik

On Mon, 13 Jul 2026, K Prateek Nayak wrote:

> Hello Shubhang,
>
> On 7/11/2026 10:47 AM, Shubhang Kaushik (Ampere) wrote:
>> diff --git a/include/linux/sched.h b/include/linux/sched.h
>> index 373bcc0598d10b4256a11f8c8373ece78fa5e0e5..5d55cdf696c13ed1c95869a1b6fb4dc1485727dc 100644
>> --- a/include/linux/sched.h
>> +++ b/include/linux/sched.h
>> @@ -1003,12 +1003,12 @@ struct task_struct {
>>          * schedule()                     if (p->on_rq && ..) // false
>>          *   smp_mb__after_spinlock();    if (smp_load_acquire(&p->on_cpu) && //true
>>          *   deactivate_task()                ttwu_queue_wakelist())
>> -        *     p->on_rq = 0;                    p->sched_remote_wakeup = Y;
>> +        *     p->on_rq = 0;                    p->sched_remote_wakeup_flags = Y;
>>          *
>>          * guarantees all stores of 'current' are visible before
>> -        * ->sched_remote_wakeup gets used, so it can be in this word.
>> +        * ->sched_remote_wakeup_flags gets used, so it can be in this word.
>>          */
>> -       unsigned                        sched_remote_wakeup:1;
>> +       unsigned                        sched_remote_wakeup_flags:8;
>
> If we are keeping a byte for it, we can just make it a u8 just after
> the "personality" variable.
>
> The reason to keep it at a byte boundary from the rq_lock serialized
> stuff was to prevent clobbering them but that would be moot now since a
> byte sized writes is always atomic across all architectures afaik.
> (https://lore.kernel.org/lkml/20201116142005.GE3121392@hirez.programming.kicks-ass.net/)
>
>
>>  #ifdef CONFIG_RT_MUTEXES
>>         unsigned                        sched_rt_mutex:1;
>>  #endif
>> diff --git a/kernel/sched/core.c b/kernel/sched/core.c
>> index 96226707c2f6135341aa779b8262f113e103d8ad..0d2918b66be277ec09109022bf3ede4b9ec8220c 100644
>> --- a/kernel/sched/core.c
>> +++ b/kernel/sched/core.c
>> @@ -3902,13 +3902,17 @@ void sched_ttwu_pending(void *arg)
>>         update_rq_clock(rq);
>>
>>         llist_for_each_entry_safe(p, t, llist, wake_entry.llist) {
>> +               int wake_flags;
>> +
>>                 if (WARN_ON_ONCE(p->on_cpu))
>>                         smp_cond_load_acquire(&p->on_cpu, !VAL);
>>
>>                 if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq)))
>>                         set_task_cpu(p, cpu_of(rq));
>>
>> -               ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf);
>> +               wake_flags = p->sched_remote_wakeup_flags;
>> +               p->sched_remote_wakeup_flags = 0;
>
> No need to reset it. This path is only used for wakeups and next remote
> wakeup would set these flags correctly before putting the task on the
> list.
>
>> +               ttwu_do_activate(rq, p, wake_flags, &rf);
>
> On aside note: Only preempt_sync() in wakeup_preempt_fair() cares for
> this but if the ttwu manages to find an idle CPU, it would preempt the
> idle tasks anyways and if there are no idle CPUs and WF_SYNC works, it
> will never use the sched_ttwu_pending() path since it is a wakeup on
> the local CPU.
>
> So, the question is, are these flags even relevant with the remote
> enqueue path?
>
>>         }
>>
>>         /*
>> @@ -3951,7 +3955,8 @@ static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags
>>  {
>>         struct rq *rq = cpu_rq(cpu);
>>
>> -       p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
>> +       p->sched_remote_wakeup_flags = wake_flags &
>> +               (WF_TTWU | WF_SYNC | WF_MIGRATED | WF_RQ_SELECTED);
>
> You can define a WF_MASK next to all the WF_* flags definition and
> use it here. It is slightly easier on the eyes ;-)
>
>>
>>         WRITE_ONCE(rq->ttwu_pending, 1);
>>  #ifdef CONFIG_SMP
>>
> -- 
> Thanks and Regards,
> Prateek
>