[PATCH 00/12] Various fixes and x86 support

[PATCH 00/12] Various fixes and x86 support

[Peter Zijlstra]

Hi,

Result of review of the base bits that were already merged, and x86 support.

[PATCH 01/12] task_work: Fix NMI race condition

[Peter Zijlstra]

  __schedule()
  // disable irqs
      <NMI>
	  task_work_add(current, work, TWA_NMI_CURRENT);
      </NMI>
  // current = next;
  // enable irqs
      <IRQ>
	  task_work_set_notify_irq()
	  test_and_set_tsk_thread_flag(current,
                                       TIF_NOTIFY_RESUME); // wrong task!
      </IRQ>
  // original task skips task work on its next return to user (or exit!)

Fixes: 466e4d801cd4 ("task_work: Add TWA_NMI_CURRENT as an additional notify mode.")
Reported-by: Josh Poimboeuf <jpoimboe@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 kernel/task_work.c |    8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

--- a/kernel/task_work.c
+++ b/kernel/task_work.c
@@ -9,7 +9,12 @@ static struct callback_head work_exited;
 #ifdef CONFIG_IRQ_WORK
 static void task_work_set_notify_irq(struct irq_work *entry)
 {
-	test_and_set_tsk_thread_flag(current, TIF_NOTIFY_RESUME);
+	/*
+	 * no-op IPI
+	 *
+	 * TWA_NMI_CURRENT will already have set the TIF flag, all
+	 * this interrupt does it tickle the return-to-user path.
+	 */
 }
 static DEFINE_PER_CPU(struct irq_work, irq_work_NMI_resume) =
 	IRQ_WORK_INIT_HARD(task_work_set_notify_irq);
@@ -86,6 +91,7 @@ int task_work_add(struct task_struct *ta
 		break;
 #ifdef CONFIG_IRQ_WORK
 	case TWA_NMI_CURRENT:
+		set_tsk_thread_flag(current, TIF_NOTIFY_RESUME);
 		irq_work_queue(this_cpu_ptr(&irq_work_NMI_resume));
 		break;
 #endif

Re: [PATCH 01/12] task_work: Fix NMI race condition

[Steven Rostedt]

On Wed, 24 Sep 2025 09:59:49 +0200
Peter Zijlstra <peterz@infradead.org> wrote:

>   __schedule()
>   // disable irqs
>       <NMI>
> 	  task_work_add(current, work, TWA_NMI_CURRENT);
>       </NMI>
>   // current = next;
>   // enable irqs
>       <IRQ>
> 	  task_work_set_notify_irq()
> 	  test_and_set_tsk_thread_flag(current,
>                                        TIF_NOTIFY_RESUME); // wrong task!
>       </IRQ>
>   // original task skips task work on its next return to user (or exit!)
> 
> Fixes: 466e4d801cd4 ("task_work: Add TWA_NMI_CURRENT as an additional notify mode.")
> Reported-by: Josh Poimboeuf <jpoimboe@kernel.org>
> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>

Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>

-- Steve

[tip: perf/core] task_work: Fix NMI race condition

[tip-bot2 for Peter Zijlstra]

The following commit has been merged into the perf/core branch of tip:

Commit-ID:     ef1ea98c8fffe227e5319215d84a53fa2a4bcebc
Gitweb:        https://git.kernel.org/tip/ef1ea98c8fffe227e5319215d84a53fa2a4bcebc
Author:        Peter Zijlstra <peterz@infradead.org>
AuthorDate:    Mon, 22 Sep 2025 15:47:00 +02:00
Committer:     Peter Zijlstra <peterz@infradead.org>
CommitterDate: Wed, 29 Oct 2025 10:29:54 +01:00

task_work: Fix NMI race condition

  __schedule()
  // disable irqs
      <NMI>
	  task_work_add(current, work, TWA_NMI_CURRENT);
      </NMI>
  // current = next;
  // enable irqs
      <IRQ>
	  task_work_set_notify_irq()
	  test_and_set_tsk_thread_flag(current,
                                       TIF_NOTIFY_RESUME); // wrong task!
      </IRQ>
  // original task skips task work on its next return to user (or exit!)

Fixes: 466e4d801cd4 ("task_work: Add TWA_NMI_CURRENT as an additional notify mode.")
Reported-by: Josh Poimboeuf <jpoimboe@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Link: https://patch.msgid.link/20250924080118.425949403@infradead.org
---
 kernel/task_work.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/kernel/task_work.c b/kernel/task_work.c
index d1efec5..0f7519f 100644
--- a/kernel/task_work.c
+++ b/kernel/task_work.c
@@ -9,7 +9,12 @@ static struct callback_head work_exited; /* all we need is ->next == NULL */
 #ifdef CONFIG_IRQ_WORK
 static void task_work_set_notify_irq(struct irq_work *entry)
 {
-	test_and_set_tsk_thread_flag(current, TIF_NOTIFY_RESUME);
+	/*
+	 * no-op IPI
+	 *
+	 * TWA_NMI_CURRENT will already have set the TIF flag, all
+	 * this interrupt does it tickle the return-to-user path.
+	 */
 }
 static DEFINE_PER_CPU(struct irq_work, irq_work_NMI_resume) =
 	IRQ_WORK_INIT_HARD(task_work_set_notify_irq);
@@ -86,6 +91,7 @@ int task_work_add(struct task_struct *task, struct callback_head *work,
 		break;
 #ifdef CONFIG_IRQ_WORK
 	case TWA_NMI_CURRENT:
+		set_tsk_thread_flag(current, TIF_NOTIFY_RESUME);
 		irq_work_queue(this_cpu_ptr(&irq_work_NMI_resume));
 		break;
 #endif

[PATCH 02/12] unwind: Shorten lines

[Peter Zijlstra]

There are some exceptionally long lines that cause ugly wrapping.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 include/linux/unwind_deferred.h |   18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

--- a/include/linux/unwind_deferred.h
+++ b/include/linux/unwind_deferred.h
@@ -8,7 +8,9 @@
 
 struct unwind_work;
 
-typedef void (*unwind_callback_t)(struct unwind_work *work, struct unwind_stacktrace *trace, u64 cookie);
+typedef void (*unwind_callback_t)(struct unwind_work *work,
+				  struct unwind_stacktrace *trace,
+				  u64 cookie);
 
 struct unwind_work {
 	struct list_head		list;
@@ -68,9 +70,17 @@ static __always_inline void unwind_reset
 static inline void unwind_task_init(struct task_struct *task) {}
 static inline void unwind_task_free(struct task_struct *task) {}
 
-static inline int unwind_user_faultable(struct unwind_stacktrace *trace) { return -ENOSYS; }
-static inline int unwind_deferred_init(struct unwind_work *work, unwind_callback_t func) { return -ENOSYS; }
-static inline int unwind_deferred_request(struct unwind_work *work, u64 *timestamp) { return -ENOSYS; }
+static inline int unwind_user_faultable(struct unwind_stacktrace *trace)
+{ return -ENOSYS; }
+
+static inline int
+unwind_deferred_init(struct unwind_work *work, unwind_callback_t func)
+{ return -ENOSYS; }
+
+static inline int
+unwind_deferred_request(struct unwind_work *work, u64 *timestamp)
+{ return -ENOSYS; }
+
 static inline void unwind_deferred_cancel(struct unwind_work *work) {}
 
 static inline void unwind_deferred_task_exit(struct task_struct *task) {}

Re: [PATCH 02/12] unwind: Shorten lines

[Steven Rostedt]

On Wed, 24 Sep 2025 09:59:50 +0200
Peter Zijlstra <peterz@infradead.org> wrote:

> There are some exceptionally long lines that cause ugly wrapping.
> 
> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>

Acked-by: Steven Rostedt (Google) <rostedt@goodmis.org>

-- Steve

[tip: perf/core] unwind: Shorten lines

[tip-bot2 for Peter Zijlstra]

The following commit has been merged into the perf/core branch of tip:

Commit-ID:     c31b9d2f589463a7cb286467a618b3b598654890
Gitweb:        https://git.kernel.org/tip/c31b9d2f589463a7cb286467a618b3b598654890
Author:        Peter Zijlstra <peterz@infradead.org>
AuthorDate:    Mon, 22 Sep 2025 15:44:10 +02:00
Committer:     Peter Zijlstra <peterz@infradead.org>
CommitterDate: Wed, 29 Oct 2025 10:29:54 +01:00

unwind: Shorten lines

There are some exceptionally long lines that cause ugly wrapping.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Link: https://patch.msgid.link/20250924080118.545274393@infradead.org
---
 include/linux/unwind_deferred.h | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/include/linux/unwind_deferred.h b/include/linux/unwind_deferred.h
index 26122d0..25f4dff 100644
--- a/include/linux/unwind_deferred.h
+++ b/include/linux/unwind_deferred.h
@@ -8,7 +8,9 @@
 
 struct unwind_work;
 
-typedef void (*unwind_callback_t)(struct unwind_work *work, struct unwind_stacktrace *trace, u64 cookie);
+typedef void (*unwind_callback_t)(struct unwind_work *work,
+				  struct unwind_stacktrace *trace,
+				  u64 cookie);
 
 struct unwind_work {
 	struct list_head		list;
@@ -68,9 +70,17 @@ static __always_inline void unwind_reset_info(void)
 static inline void unwind_task_init(struct task_struct *task) {}
 static inline void unwind_task_free(struct task_struct *task) {}
 
-static inline int unwind_user_faultable(struct unwind_stacktrace *trace) { return -ENOSYS; }
-static inline int unwind_deferred_init(struct unwind_work *work, unwind_callback_t func) { return -ENOSYS; }
-static inline int unwind_deferred_request(struct unwind_work *work, u64 *timestamp) { return -ENOSYS; }
+static inline int unwind_user_faultable(struct unwind_stacktrace *trace)
+{ return -ENOSYS; }
+
+static inline int
+unwind_deferred_init(struct unwind_work *work, unwind_callback_t func)
+{ return -ENOSYS; }
+
+static inline int
+unwind_deferred_request(struct unwind_work *work, u64 *timestamp)
+{ return -ENOSYS; }
+
 static inline void unwind_deferred_cancel(struct unwind_work *work) {}
 
 static inline void unwind_deferred_task_exit(struct task_struct *task) {}

[PATCH 03/12] unwind: Add required include files

[Peter Zijlstra]

To be self sufficient, the file needs to include linux/types.h. This
provides things like u32/u64 and struct callback_head.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 include/linux/unwind_deferred_types.h |    2 ++
 1 file changed, 2 insertions(+)

--- a/include/linux/unwind_deferred_types.h
+++ b/include/linux/unwind_deferred_types.h
@@ -2,6 +2,8 @@
 #ifndef _LINUX_UNWIND_USER_DEFERRED_TYPES_H
 #define _LINUX_UNWIND_USER_DEFERRED_TYPES_H
 
+#include <linux/types.h>
+
 struct unwind_cache {
 	unsigned long		unwind_completed;
 	unsigned int		nr_entries;

Re: [PATCH 03/12] unwind: Add required include files

[Steven Rostedt]

On Wed, 24 Sep 2025 09:59:51 +0200
Peter Zijlstra <peterz@infradead.org> wrote:

> To be self sufficient, the file needs to include linux/types.h. This
> provides things like u32/u64 and struct callback_head.
> 
> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>

Acked-by: Steven Rostedt (Google) <rostedt@goodmis.org>

-- Steve

[tip: perf/core] unwind: Add required include files

[tip-bot2 for Peter Zijlstra]

The following commit has been merged into the perf/core branch of tip:

Commit-ID:     b1164c7d118defb01a885b53f56e3336db784df7
Gitweb:        https://git.kernel.org/tip/b1164c7d118defb01a885b53f56e3336db784df7
Author:        Peter Zijlstra <peterz@infradead.org>
AuthorDate:    Mon, 22 Sep 2025 15:44:59 +02:00
Committer:     Peter Zijlstra <peterz@infradead.org>
CommitterDate: Wed, 29 Oct 2025 10:29:55 +01:00

unwind: Add required include files

To be self sufficient, the file needs to include linux/types.h. This
provides things like u32/u64 and struct callback_head.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Link: https://patch.msgid.link/20250924080118.665787071@infradead.org
---
 include/linux/unwind_deferred_types.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/include/linux/unwind_deferred_types.h b/include/linux/unwind_deferred_types.h
index 33b62ac..29452ff 100644
--- a/include/linux/unwind_deferred_types.h
+++ b/include/linux/unwind_deferred_types.h
@@ -2,6 +2,8 @@
 #ifndef _LINUX_UNWIND_USER_DEFERRED_TYPES_H
 #define _LINUX_UNWIND_USER_DEFERRED_TYPES_H
 
+#include <linux/types.h>
+
 struct unwind_cache {
 	unsigned long		unwind_completed;
 	unsigned int		nr_entries;

[PATCH 04/12] unwind: Simplify unwind_reset_info()

[Peter Zijlstra]

Invert the condition of the first if and make it an early exit to
reduce an indent level for the rest fo the function.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 include/linux/unwind_deferred.h |   26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

--- a/include/linux/unwind_deferred.h
+++ b/include/linux/unwind_deferred.h
@@ -45,22 +45,22 @@ void unwind_deferred_task_exit(struct ta
 static __always_inline void unwind_reset_info(void)
 {
 	struct unwind_task_info *info = &current->unwind_info;
-	unsigned long bits;
+	unsigned long bits = info->unwind_mask;
 
 	/* Was there any unwinding? */
-	if (unlikely(info->unwind_mask)) {
-		bits = info->unwind_mask;
-		do {
-			/* Is a task_work going to run again before going back */
-			if (bits & UNWIND_PENDING)
-				return;
-		} while (!try_cmpxchg(&info->unwind_mask, &bits, 0UL));
-		current->unwind_info.id.id = 0;
+	if (likely(!bits))
+		return;
 
-		if (unlikely(info->cache)) {
-			info->cache->nr_entries = 0;
-			info->cache->unwind_completed = 0;
-		}
+	do {
+		/* Is a task_work going to run again before going back */
+		if (bits & UNWIND_PENDING)
+			return;
+	} while (!try_cmpxchg(&info->unwind_mask, &bits, 0UL));
+	current->unwind_info.id.id = 0;
+
+	if (unlikely(info->cache)) {
+		info->cache->nr_entries = 0;
+		info->cache->unwind_completed = 0;
 	}
 }
 

Re: [PATCH 04/12] unwind: Simplify unwind_reset_info()

[Steven Rostedt]

On Wed, 24 Sep 2025 09:59:52 +0200
Peter Zijlstra <peterz@infradead.org> wrote:

> Invert the condition of the first if and make it an early exit to
> reduce an indent level for the rest fo the function.
> 
> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>

Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>

-- Steve

[tip: perf/core] unwind: Simplify unwind_reset_info()

[tip-bot2 for Peter Zijlstra]

The following commit has been merged into the perf/core branch of tip:

Commit-ID:     52a1ec718b3eb6da29a76d05a662365a997139cc
Gitweb:        https://git.kernel.org/tip/52a1ec718b3eb6da29a76d05a662365a997139cc
Author:        Peter Zijlstra <peterz@infradead.org>
AuthorDate:    Mon, 22 Sep 2025 15:46:00 +02:00
Committer:     Peter Zijlstra <peterz@infradead.org>
CommitterDate: Wed, 29 Oct 2025 10:29:55 +01:00

unwind: Simplify unwind_reset_info()

Invert the condition of the first if and make it an early exit to
reduce an indent level for the rest fo the function.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Link: https://patch.msgid.link/20250924080118.777916262@infradead.org
---
 include/linux/unwind_deferred.h | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/include/linux/unwind_deferred.h b/include/linux/unwind_deferred.h
index 25f4dff..196e12c 100644
--- a/include/linux/unwind_deferred.h
+++ b/include/linux/unwind_deferred.h
@@ -46,22 +46,22 @@ void unwind_deferred_task_exit(struct task_struct *task);
 static __always_inline void unwind_reset_info(void)
 {
 	struct unwind_task_info *info = &current->unwind_info;
-	unsigned long bits;
+	unsigned long bits = info->unwind_mask;
 
 	/* Was there any unwinding? */
-	if (unlikely(info->unwind_mask)) {
-		bits = info->unwind_mask;
-		do {
-			/* Is a task_work going to run again before going back */
-			if (bits & UNWIND_PENDING)
-				return;
-		} while (!try_cmpxchg(&info->unwind_mask, &bits, 0UL));
-		current->unwind_info.id.id = 0;
-
-		if (unlikely(info->cache)) {
-			info->cache->nr_entries = 0;
-			info->cache->unwind_completed = 0;
-		}
+	if (likely(!bits))
+		return;
+
+	do {
+		/* Is a task_work going to run again before going back */
+		if (bits & UNWIND_PENDING)
+			return;
+	} while (!try_cmpxchg(&info->unwind_mask, &bits, 0UL));
+	current->unwind_info.id.id = 0;
+
+	if (unlikely(info->cache)) {
+		info->cache->nr_entries = 0;
+		info->cache->unwind_completed = 0;
 	}
 }
 

[PATCH 05/12] unwind: Add comment to unwind_deferred_task_exit()

[Peter Zijlstra]

Explain why unwind_deferred_task_exit() exist and its constraints.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 kernel/exit.c |    7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -934,7 +934,6 @@ void __noreturn do_exit(long code)
 
 	tsk->exit_code = code;
 	taskstats_exit(tsk, group_dead);
-	unwind_deferred_task_exit(tsk);
 	trace_sched_process_exit(tsk, group_dead);
 
 	/*
@@ -945,6 +944,12 @@ void __noreturn do_exit(long code)
 	 * gets woken up by child-exit notifications.
 	 */
 	perf_event_exit_task(tsk);
+	/*
+	 * PF_EXITING (above) ensures unwind_deferred_request() will no
+	 * longer add new unwinds. While exit_mm() (below) will destroy the
+	 * abaility to do unwinds.
+	 */
+	unwind_deferred_task_exit(tsk);
 
 	exit_mm();
 

Re: [PATCH 05/12] unwind: Add comment to unwind_deferred_task_exit()

[Steven Rostedt]

On Wed, 24 Sep 2025 09:59:53 +0200
Peter Zijlstra <peterz@infradead.org> wrote:

> Explain why unwind_deferred_task_exit() exist and its constraints.
> 
> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
> ---
>  kernel/exit.c |    7 ++++++-
>  1 file changed, 6 insertions(+), 1 deletion(-)
> 
> --- a/kernel/exit.c
> +++ b/kernel/exit.c
> @@ -934,7 +934,6 @@ void __noreturn do_exit(long code)
>  
>  	tsk->exit_code = code;
>  	taskstats_exit(tsk, group_dead);
> -	unwind_deferred_task_exit(tsk);
>  	trace_sched_process_exit(tsk, group_dead);
>  
>  	/*
> @@ -945,6 +944,12 @@ void __noreturn do_exit(long code)
>  	 * gets woken up by child-exit notifications.
>  	 */
>  	perf_event_exit_task(tsk);
> +	/*
> +	 * PF_EXITING (above) ensures unwind_deferred_request() will no
> +	 * longer add new unwinds. While exit_mm() (below) will destroy the
> +	 * abaility to do unwinds.


I would state that it also flushes any unwind that is currently pending, as
exit_mm() will prevent it from happening.

-- Steve


> +	 */
> +	unwind_deferred_task_exit(tsk);
>  
>  	exit_mm();
>  
> 

Re: [PATCH 05/12] unwind: Add comment to unwind_deferred_task_exit()

[Peter Zijlstra]

On Wed, Oct 01, 2025 at 11:35:05AM -0400, Steven Rostedt wrote:
> On Wed, 24 Sep 2025 09:59:53 +0200
> Peter Zijlstra <peterz@infradead.org> wrote:
> 
> > Explain why unwind_deferred_task_exit() exist and its constraints.
> > 
> > Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
> > ---
> >  kernel/exit.c |    7 ++++++-
> >  1 file changed, 6 insertions(+), 1 deletion(-)
> > 
> > --- a/kernel/exit.c
> > +++ b/kernel/exit.c
> > @@ -934,7 +934,6 @@ void __noreturn do_exit(long code)
> >  
> >  	tsk->exit_code = code;
> >  	taskstats_exit(tsk, group_dead);
> > -	unwind_deferred_task_exit(tsk);
> >  	trace_sched_process_exit(tsk, group_dead);
> >  
> >  	/*
> > @@ -945,6 +944,12 @@ void __noreturn do_exit(long code)
> >  	 * gets woken up by child-exit notifications.
> >  	 */
> >  	perf_event_exit_task(tsk);
> > +	/*
> > +	 * PF_EXITING (above) ensures unwind_deferred_request() will no
> > +	 * longer add new unwinds. While exit_mm() (below) will destroy the
> > +	 * abaility to do unwinds.
> 
> 
> I would state that it also flushes any unwind that is currently pending, as
> exit_mm() will prevent it from happening.

It now reads:

+       /*
+        * PF_EXITING (above) ensures unwind_deferred_request() will no
+        * longer add new unwinds. While exit_mm() (below) will destroy the
+        * abaility to do unwinds. So flush any pending unwinds here.
+        */
+       unwind_deferred_task_exit(tsk);

Re: [PATCH 05/12] unwind: Add comment to unwind_deferred_task_exit()

[Steven Rostedt]

On Mon, 20 Oct 2025 12:16:02 +0200
Peter Zijlstra <peterz@infradead.org> wrote:

> > I would state that it also flushes any unwind that is currently pending, as
> > exit_mm() will prevent it from happening.  
> 
> It now reads:
> 
> +       /*
> +        * PF_EXITING (above) ensures unwind_deferred_request() will no
> +        * longer add new unwinds. While exit_mm() (below) will destroy the
> +        * abaility to do unwinds. So flush any pending unwinds here.
> +        */
> +       unwind_deferred_task_exit(tsk);

Thanks,

Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>

-- Steve

[tip: perf/core] unwind: Add comment to unwind_deferred_task_exit()

[tip-bot2 for Peter Zijlstra]

The following commit has been merged into the perf/core branch of tip:

Commit-ID:     ae577ea0bc5249c483da09670f784dbc288c80b6
Gitweb:        https://git.kernel.org/tip/ae577ea0bc5249c483da09670f784dbc288c80b6
Author:        Peter Zijlstra <peterz@infradead.org>
AuthorDate:    Mon, 22 Sep 2025 15:46:27 +02:00
Committer:     Peter Zijlstra <peterz@infradead.org>
CommitterDate: Wed, 29 Oct 2025 10:29:55 +01:00

unwind: Add comment to unwind_deferred_task_exit()

Explain why unwind_deferred_task_exit() exist and its constraints.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Link: https://patch.msgid.link/20250924080118.893367437@infradead.org
---
 kernel/exit.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/kernel/exit.c b/kernel/exit.c
index 9f74e8f..5f6e78e 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -939,7 +939,6 @@ void __noreturn do_exit(long code)
 
 	tsk->exit_code = code;
 	taskstats_exit(tsk, group_dead);
-	unwind_deferred_task_exit(tsk);
 	trace_sched_process_exit(tsk, group_dead);
 
 	/*
@@ -950,6 +949,12 @@ void __noreturn do_exit(long code)
 	 * gets woken up by child-exit notifications.
 	 */
 	perf_event_exit_task(tsk);
+	/*
+	 * PF_EXITING (above) ensures unwind_deferred_request() will no
+	 * longer add new unwinds. While exit_mm() (below) will destroy the
+	 * abaility to do unwinds. So flush any pending unwinds here.
+	 */
+	unwind_deferred_task_exit(tsk);
 
 	exit_mm();
 

[PATCH 06/12] unwind: Fix unwind_deferred_request() vs NMI

[Peter Zijlstra]

task_work_add(RWA_RESUME) isn't NMI-safe, use TWA_NMI_CURRENT when
used from NMI context.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 kernel/unwind/deferred.c |   10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

--- a/kernel/unwind/deferred.c
+++ b/kernel/unwind/deferred.c
@@ -231,6 +231,7 @@ void unwind_deferred_task_exit(struct ta
 int unwind_deferred_request(struct unwind_work *work, u64 *cookie)
 {
 	struct unwind_task_info *info = &current->unwind_info;
+	int twa_mode = TWA_RESUME;
 	unsigned long old, bits;
 	unsigned long bit;
 	int ret;
@@ -246,8 +247,11 @@ int unwind_deferred_request(struct unwin
 	 * Trigger a warning to make it obvious that an architecture
 	 * is using this in NMI when it should not be.
 	 */
-	if (WARN_ON_ONCE(!CAN_USE_IN_NMI && in_nmi()))
-		return -EINVAL;
+	if (in_nmi()) {
+		if (WARN_ON_ONCE(!CAN_USE_IN_NMI))
+			return -EINVAL;
+		twa_mode = TWA_NMI_CURRENT;
+	}
 
 	/* Do not allow cancelled works to request again */
 	bit = READ_ONCE(work->bit);
@@ -285,7 +289,7 @@ int unwind_deferred_request(struct unwin
 	}
 
 	/* The work has been claimed, now schedule it. */
-	ret = task_work_add(current, &info->work, TWA_RESUME);
+	ret = task_work_add(current, &info->work, twa_mode);
 
 	if (WARN_ON_ONCE(ret))
 		WRITE_ONCE(info->unwind_mask, 0);

Re: [PATCH 06/12] unwind: Fix unwind_deferred_request() vs NMI

[Steven Rostedt]

On Wed, 24 Sep 2025 09:59:54 +0200
Peter Zijlstra <peterz@infradead.org> wrote:

> task_work_add(RWA_RESUME) isn't NMI-safe, use TWA_NMI_CURRENT when
> used from NMI context.
> 
> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>

Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>

-- Steve

[tip: perf/core] unwind: Fix unwind_deferred_request() vs NMI

[tip-bot2 for Peter Zijlstra]

The following commit has been merged into the perf/core branch of tip:

Commit-ID:     a38a64712e740d6e9df6940a997a47f5fab7efa2
Gitweb:        https://git.kernel.org/tip/a38a64712e740d6e9df6940a997a47f5fab7efa2
Author:        Peter Zijlstra <peterz@infradead.org>
AuthorDate:    Mon, 22 Sep 2025 15:47:56 +02:00
Committer:     Peter Zijlstra <peterz@infradead.org>
CommitterDate: Wed, 29 Oct 2025 10:29:56 +01:00

unwind: Fix unwind_deferred_request() vs NMI

task_work_add(RWA_RESUME) isn't NMI-safe, use TWA_NMI_CURRENT when
used from NMI context.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Link: https://patch.msgid.link/20250924080119.005422353@infradead.org
---
 kernel/unwind/deferred.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/kernel/unwind/deferred.c b/kernel/unwind/deferred.c
index dc6040a..d2cd3a7 100644
--- a/kernel/unwind/deferred.c
+++ b/kernel/unwind/deferred.c
@@ -231,6 +231,7 @@ void unwind_deferred_task_exit(struct task_struct *task)
 int unwind_deferred_request(struct unwind_work *work, u64 *cookie)
 {
 	struct unwind_task_info *info = &current->unwind_info;
+	int twa_mode = TWA_RESUME;
 	unsigned long old, bits;
 	unsigned long bit;
 	int ret;
@@ -246,8 +247,11 @@ int unwind_deferred_request(struct unwind_work *work, u64 *cookie)
 	 * Trigger a warning to make it obvious that an architecture
 	 * is using this in NMI when it should not be.
 	 */
-	if (WARN_ON_ONCE(!CAN_USE_IN_NMI && in_nmi()))
-		return -EINVAL;
+	if (in_nmi()) {
+		if (WARN_ON_ONCE(!CAN_USE_IN_NMI))
+			return -EINVAL;
+		twa_mode = TWA_NMI_CURRENT;
+	}
 
 	/* Do not allow cancelled works to request again */
 	bit = READ_ONCE(work->bit);
@@ -285,7 +289,7 @@ int unwind_deferred_request(struct unwind_work *work, u64 *cookie)
 	}
 
 	/* The work has been claimed, now schedule it. */
-	ret = task_work_add(current, &info->work, TWA_RESUME);
+	ret = task_work_add(current, &info->work, twa_mode);
 
 	if (WARN_ON_ONCE(ret))
 		WRITE_ONCE(info->unwind_mask, 0);

[PATCH 07/12] unwind: Clarify calling context

[Peter Zijlstra]

The get_cookie() function hard relies on IRQs being disabled, but this
isn't immediately obvious when reading the function.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 kernel/unwind/deferred.c |    2 ++
 1 file changed, 2 insertions(+)

--- a/kernel/unwind/deferred.c
+++ b/kernel/unwind/deferred.c
@@ -79,6 +79,8 @@ static u64 get_cookie(struct unwind_task
 {
 	u32 cnt = 1;
 
+	lockdep_assert_irqs_disabled();
+
 	if (info->id.cpu)
 		return info->id.id;
 

Re: [PATCH 07/12] unwind: Clarify calling context

[Steven Rostedt]

On Wed, 24 Sep 2025 09:59:55 +0200
Peter Zijlstra <peterz@infradead.org> wrote:

> The get_cookie() function hard relies on IRQs being disabled, but this
> isn't immediately obvious when reading the function.
> 
> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>

Acked-by: Steven Rostedt (Google) <rostedt@goodmis.org>

-- Steve

> ---
>  kernel/unwind/deferred.c |    2 ++
>  1 file changed, 2 insertions(+)
> 
> --- a/kernel/unwind/deferred.c
> +++ b/kernel/unwind/deferred.c
> @@ -79,6 +79,8 @@ static u64 get_cookie(struct unwind_task
>  {
>  	u32 cnt = 1;
>  
> +	lockdep_assert_irqs_disabled();
> +
>  	if (info->id.cpu)
>  		return info->id.id;
>  
> 

[tip: perf/core] unwind: Clarify calling context

[tip-bot2 for Peter Zijlstra]

The following commit has been merged into the perf/core branch of tip:

Commit-ID:     1e74829f36b5db19afc3d17f0a3750e9573710ae
Gitweb:        https://git.kernel.org/tip/1e74829f36b5db19afc3d17f0a3750e9573710ae
Author:        Peter Zijlstra <peterz@infradead.org>
AuthorDate:    Mon, 22 Sep 2025 15:49:19 +02:00
Committer:     Peter Zijlstra <peterz@infradead.org>
CommitterDate: Wed, 29 Oct 2025 10:29:56 +01:00

unwind: Clarify calling context

The get_cookie() function hard relies on IRQs being disabled, but this
isn't immediately obvious when reading the function.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Link: https://patch.msgid.link/20250924080119.122507632@infradead.org
---
 kernel/unwind/deferred.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/kernel/unwind/deferred.c b/kernel/unwind/deferred.c
index d2cd3a7..6395192 100644
--- a/kernel/unwind/deferred.c
+++ b/kernel/unwind/deferred.c
@@ -79,6 +79,8 @@ static u64 get_cookie(struct unwind_task_info *info)
 {
 	u32 cnt = 1;
 
+	lockdep_assert_irqs_disabled();
+
 	if (info->id.cpu)
 		return info->id.id;
 

[PATCH 08/12] unwind: Simplify unwind_user_faultable()

[Peter Zijlstra]

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 kernel/unwind/deferred.c |    6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

--- a/kernel/unwind/deferred.c
+++ b/kernel/unwind/deferred.c
@@ -128,17 +128,15 @@ int unwind_user_faultable(struct unwind_
 
 	cache = info->cache;
 	trace->entries = cache->entries;
-
-	if (cache->nr_entries) {
+	trace->nr = cache->nr_entries;
+	if (trace->nr) {
 		/*
 		 * The user stack has already been previously unwound in this
 		 * entry context.  Skip the unwind and use the cache.
 		 */
-		trace->nr = cache->nr_entries;
 		return 0;
 	}
 
-	trace->nr = 0;
 	unwind_user(trace, UNWIND_MAX_ENTRIES);
 
 	cache->nr_entries = trace->nr;

Re: [PATCH 08/12] unwind: Simplify unwind_user_faultable()

[Steven Rostedt]

On Wed, 24 Sep 2025 09:59:56 +0200
Peter Zijlstra <peterz@infradead.org> wrote:

> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
> ---
>  kernel/unwind/deferred.c |    6 ++----
>  1 file changed, 2 insertions(+), 4 deletions(-)
> 
> --- a/kernel/unwind/deferred.c
> +++ b/kernel/unwind/deferred.c
> @@ -128,17 +128,15 @@ int unwind_user_faultable(struct unwind_
>  
>  	cache = info->cache;
>  	trace->entries = cache->entries;
> -
> -	if (cache->nr_entries) {
> +	trace->nr = cache->nr_entries;
> +	if (trace->nr) {
>  		/*
>  		 * The user stack has already been previously unwound in this
>  		 * entry context.  Skip the unwind and use the cache.
>  		 */
> -		trace->nr = cache->nr_entries;
>  		return 0;
>  	}

Could we turn the above into:

	/*
	 * If the user stack has already been previously unwound in this
	 * entry context.  Skip the unwind and use the cache.
	 */
	if (trace->nr)
		return 0;

So we could remove the squiggly brackets?

-- Steve



>  
> -	trace->nr = 0;
>  	unwind_user(trace, UNWIND_MAX_ENTRIES);
>  
>  	cache->nr_entries = trace->nr;
> 

Re: [PATCH 08/12] unwind: Simplify unwind_user_faultable()

[Peter Zijlstra]

On Wed, Oct 01, 2025 at 11:40:53AM -0400, Steven Rostedt wrote:
> On Wed, 24 Sep 2025 09:59:56 +0200
> Peter Zijlstra <peterz@infradead.org> wrote:
> 
> > Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
> > ---
> >  kernel/unwind/deferred.c |    6 ++----
> >  1 file changed, 2 insertions(+), 4 deletions(-)
> > 
> > --- a/kernel/unwind/deferred.c
> > +++ b/kernel/unwind/deferred.c
> > @@ -128,17 +128,15 @@ int unwind_user_faultable(struct unwind_
> >  
> >  	cache = info->cache;
> >  	trace->entries = cache->entries;
> > -
> > -	if (cache->nr_entries) {
> > +	trace->nr = cache->nr_entries;
> > +	if (trace->nr) {
> >  		/*
> >  		 * The user stack has already been previously unwound in this
> >  		 * entry context.  Skip the unwind and use the cache.
> >  		 */
> > -		trace->nr = cache->nr_entries;
> >  		return 0;
> >  	}
> 
> Could we turn the above into:
> 
> 	/*
> 	 * If the user stack has already been previously unwound in this
> 	 * entry context.  Skip the unwind and use the cache.
> 	 */
> 	if (trace->nr)
> 		return 0;
> 
> So we could remove the squiggly brackets?

This tarriff nonsense must be really bad if you're re-cycling them so
aggressively :-)

Sure, done.

[tip: perf/core] unwind: Simplify unwind_user_faultable()

[tip-bot2 for Peter Zijlstra]

The following commit has been merged into the perf/core branch of tip:

Commit-ID:     42b9138f81fc22c36128f9524bb21bc9eabfb1b8
Gitweb:        https://git.kernel.org/tip/42b9138f81fc22c36128f9524bb21bc9eabfb1b8
Author:        Peter Zijlstra <peterz@infradead.org>
AuthorDate:    Mon, 22 Sep 2025 15:49:14 +02:00
Committer:     Peter Zijlstra <peterz@infradead.org>
CommitterDate: Wed, 29 Oct 2025 10:29:56 +01:00

unwind: Simplify unwind_user_faultable()

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20250924080119.271671514@infradead.org
---
 kernel/unwind/deferred.c | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/kernel/unwind/deferred.c b/kernel/unwind/deferred.c
index 6395192..09617d8 100644
--- a/kernel/unwind/deferred.c
+++ b/kernel/unwind/deferred.c
@@ -128,17 +128,14 @@ int unwind_user_faultable(struct unwind_stacktrace *trace)
 
 	cache = info->cache;
 	trace->entries = cache->entries;
-
-	if (cache->nr_entries) {
-		/*
-		 * The user stack has already been previously unwound in this
-		 * entry context.  Skip the unwind and use the cache.
-		 */
-		trace->nr = cache->nr_entries;
+	trace->nr = cache->nr_entries;
+	/*
+	 * The user stack has already been previously unwound in this
+	 * entry context.  Skip the unwind and use the cache.
+	 */
+	if (trace->nr)
 		return 0;
-	}
 
-	trace->nr = 0;
 	unwind_user(trace, UNWIND_MAX_ENTRIES);
 
 	cache->nr_entries = trace->nr;

[PATCH 09/12] unwind: Make unwind_task_info::unwind_mask consistent

[Peter Zijlstra]

The unwind_task_info::unwind_mask was manipulated using a mixture of:

  regular store
  WRITE_ONCE()
  try_cmpxchg()
  set_bit()
  atomic_long_*()

Clean up and make it consistently atomic_long_t.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 include/linux/unwind_deferred.h       |    4 ++--
 include/linux/unwind_deferred_types.h |    3 ++-
 kernel/unwind/deferred.c              |   17 +++++++++--------
 3 files changed, 13 insertions(+), 11 deletions(-)

--- a/include/linux/unwind_deferred.h
+++ b/include/linux/unwind_deferred.h
@@ -46,7 +46,7 @@ void unwind_deferred_task_exit(struct ta
 static __always_inline void unwind_reset_info(void)
 {
 	struct unwind_task_info *info = &current->unwind_info;
-	unsigned long bits = info->unwind_mask;
+	unsigned long bits = atomic_long_read(&info->unwind_mask);
 
 	/* Was there any unwinding? */
 	if (likely(!bits))
@@ -56,7 +56,7 @@ static __always_inline void unwind_reset
 		/* Is a task_work going to run again before going back */
 		if (bits & UNWIND_PENDING)
 			return;
-	} while (!try_cmpxchg(&info->unwind_mask, &bits, 0UL));
+	} while (!atomic_long_try_cmpxchg(&info->unwind_mask, &bits, 0UL));
 	current->unwind_info.id.id = 0;
 
 	if (unlikely(info->cache)) {
--- a/include/linux/unwind_deferred_types.h
+++ b/include/linux/unwind_deferred_types.h
@@ -3,6 +3,7 @@
 #define _LINUX_UNWIND_USER_DEFERRED_TYPES_H
 
 #include <linux/types.h>
+#include <linux/atomic.h>
 
 struct unwind_cache {
 	unsigned long		unwind_completed;
@@ -32,7 +33,7 @@ union unwind_task_id {
 };
 
 struct unwind_task_info {
-	unsigned long		unwind_mask;
+	atomic_long_t		unwind_mask;
 	struct unwind_cache	*cache;
 	struct callback_head	work;
 	union unwind_task_id	id;
--- a/kernel/unwind/deferred.c
+++ b/kernel/unwind/deferred.c
@@ -53,7 +53,7 @@ DEFINE_STATIC_SRCU(unwind_srcu);
 
 static inline bool unwind_pending(struct unwind_task_info *info)
 {
-	return test_bit(UNWIND_PENDING_BIT, &info->unwind_mask);
+	return atomic_long_read(&info->unwind_mask) & UNWIND_USED;
 }
 
 /*
@@ -142,7 +142,7 @@ int unwind_user_faultable(struct unwind_
 	cache->nr_entries = trace->nr;
 
 	/* Clear nr_entries on way back to user space */
-	set_bit(UNWIND_USED_BIT, &info->unwind_mask);
+	atomic_long_or(UNWIND_USED, &info->unwind_mask);
 
 	return 0;
 }
@@ -160,7 +160,7 @@ static void process_unwind_deferred(stru
 
 	/* Clear pending bit but make sure to have the current bits */
 	bits = atomic_long_fetch_andnot(UNWIND_PENDING,
-				  (atomic_long_t *)&info->unwind_mask);
+					&info->unwind_mask);
 	/*
 	 * From here on out, the callback must always be called, even if it's
 	 * just an empty trace.
@@ -265,7 +265,7 @@ int unwind_deferred_request(struct unwin
 
 	*cookie = get_cookie(info);
 
-	old = READ_ONCE(info->unwind_mask);
+	old = atomic_long_read(&info->unwind_mask);
 
 	/* Is this already queued or executed */
 	if (old & bit)
@@ -278,7 +278,7 @@ int unwind_deferred_request(struct unwin
 	 * to have a callback.
 	 */
 	bits = UNWIND_PENDING | bit;
-	old = atomic_long_fetch_or(bits, (atomic_long_t *)&info->unwind_mask);
+	old = atomic_long_fetch_or(bits, &info->unwind_mask);
 	if (old & bits) {
 		/*
 		 * If the work's bit was set, whatever set it had better
@@ -292,7 +292,7 @@ int unwind_deferred_request(struct unwin
 	ret = task_work_add(current, &info->work, twa_mode);
 
 	if (WARN_ON_ONCE(ret))
-		WRITE_ONCE(info->unwind_mask, 0);
+		atomic_long_set(&info->unwind_mask, 0);
 
 	return ret;
 }
@@ -324,7 +324,8 @@ void unwind_deferred_cancel(struct unwin
 	guard(rcu)();
 	/* Clear this bit from all threads */
 	for_each_process_thread(g, t) {
-		clear_bit(bit, &t->unwind_info.unwind_mask);
+		atomic_long_andnot(UNWIND_USED,
+				   &t->unwind_info.unwind_mask);
 		if (t->unwind_info.cache)
 			clear_bit(bit, &t->unwind_info.cache->unwind_completed);
 	}
@@ -354,7 +355,7 @@ void unwind_task_init(struct task_struct
 
 	memset(info, 0, sizeof(*info));
 	init_task_work(&info->work, unwind_deferred_task_work);
-	info->unwind_mask = 0;
+	atomic_long_set(&info->unwind_mask, 0);
 }
 
 void unwind_task_free(struct task_struct *task)

Re: [PATCH 09/12] unwind: Make unwind_task_info::unwind_mask consistent

[Steven Rostedt]

On Wed, 24 Sep 2025 09:59:57 +0200
Peter Zijlstra <peterz@infradead.org> wrote:

> @@ -324,7 +324,8 @@ void unwind_deferred_cancel(struct unwin
>  	guard(rcu)();
>  	/* Clear this bit from all threads */
>  	for_each_process_thread(g, t) {
> -		clear_bit(bit, &t->unwind_info.unwind_mask);
> +		atomic_long_andnot(UNWIND_USED,
> +				   &t->unwind_info.unwind_mask);

Shouldn't this be:

		atomic_long_andnot(BIT(bit), &t->unwind_info.unwind_mask);

 ?

As BIT(bit) != UNWIND_USED.

-- Steve

>  		if (t->unwind_info.cache)
>  			clear_bit(bit, &t->unwind_info.cache->unwind_completed);
>  	}

Re: [PATCH 09/12] unwind: Make unwind_task_info::unwind_mask consistent

[Peter Zijlstra]

On Wed, Oct 01, 2025 at 11:47:02AM -0400, Steven Rostedt wrote:
> On Wed, 24 Sep 2025 09:59:57 +0200
> Peter Zijlstra <peterz@infradead.org> wrote:
> 
> > @@ -324,7 +324,8 @@ void unwind_deferred_cancel(struct unwin
> >  	guard(rcu)();
> >  	/* Clear this bit from all threads */
> >  	for_each_process_thread(g, t) {
> > -		clear_bit(bit, &t->unwind_info.unwind_mask);
> > +		atomic_long_andnot(UNWIND_USED,
> > +				   &t->unwind_info.unwind_mask);
> 
> Shouldn't this be:
> 
> 		atomic_long_andnot(BIT(bit), &t->unwind_info.unwind_mask);
> 
>  ?
> 

Yeah, copy-pasta failure there. Thanks!

[tip: perf/core] unwind: Make unwind_task_info::unwind_mask consistent

[tip-bot2 for Peter Zijlstra]

The following commit has been merged into the perf/core branch of tip:

Commit-ID:     639214f65b1db87c6992eadf93079ff0d8768c2d
Gitweb:        https://git.kernel.org/tip/639214f65b1db87c6992eadf93079ff0d8768c2d
Author:        Peter Zijlstra <peterz@infradead.org>
AuthorDate:    Mon, 22 Sep 2025 16:09:17 +02:00
Committer:     Peter Zijlstra <peterz@infradead.org>
CommitterDate: Wed, 29 Oct 2025 10:29:57 +01:00

unwind: Make unwind_task_info::unwind_mask consistent

The unwind_task_info::unwind_mask was manipulated using a mixture of:

  regular store
  WRITE_ONCE()
  try_cmpxchg()
  set_bit()
  atomic_long_*()

Clean up and make it consistently atomic_long_t.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20250924080119.384384486@infradead.org
---
 include/linux/unwind_deferred.h       |  4 ++--
 include/linux/unwind_deferred_types.h |  3 ++-
 kernel/unwind/deferred.c              | 17 +++++++++--------
 3 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/include/linux/unwind_deferred.h b/include/linux/unwind_deferred.h
index 196e12c..f4743c8 100644
--- a/include/linux/unwind_deferred.h
+++ b/include/linux/unwind_deferred.h
@@ -46,7 +46,7 @@ void unwind_deferred_task_exit(struct task_struct *task);
 static __always_inline void unwind_reset_info(void)
 {
 	struct unwind_task_info *info = &current->unwind_info;
-	unsigned long bits = info->unwind_mask;
+	unsigned long bits = atomic_long_read(&info->unwind_mask);
 
 	/* Was there any unwinding? */
 	if (likely(!bits))
@@ -56,7 +56,7 @@ static __always_inline void unwind_reset_info(void)
 		/* Is a task_work going to run again before going back */
 		if (bits & UNWIND_PENDING)
 			return;
-	} while (!try_cmpxchg(&info->unwind_mask, &bits, 0UL));
+	} while (!atomic_long_try_cmpxchg(&info->unwind_mask, &bits, 0UL));
 	current->unwind_info.id.id = 0;
 
 	if (unlikely(info->cache)) {
diff --git a/include/linux/unwind_deferred_types.h b/include/linux/unwind_deferred_types.h
index 29452ff..0a4c8dd 100644
--- a/include/linux/unwind_deferred_types.h
+++ b/include/linux/unwind_deferred_types.h
@@ -3,6 +3,7 @@
 #define _LINUX_UNWIND_USER_DEFERRED_TYPES_H
 
 #include <linux/types.h>
+#include <linux/atomic.h>
 
 struct unwind_cache {
 	unsigned long		unwind_completed;
@@ -32,7 +33,7 @@ union unwind_task_id {
 };
 
 struct unwind_task_info {
-	unsigned long		unwind_mask;
+	atomic_long_t		unwind_mask;
 	struct unwind_cache	*cache;
 	struct callback_head	work;
 	union unwind_task_id	id;
diff --git a/kernel/unwind/deferred.c b/kernel/unwind/deferred.c
index 09617d8..a88fb48 100644
--- a/kernel/unwind/deferred.c
+++ b/kernel/unwind/deferred.c
@@ -53,7 +53,7 @@ DEFINE_STATIC_SRCU(unwind_srcu);
 
 static inline bool unwind_pending(struct unwind_task_info *info)
 {
-	return test_bit(UNWIND_PENDING_BIT, &info->unwind_mask);
+	return atomic_long_read(&info->unwind_mask) & UNWIND_PENDING;
 }
 
 /*
@@ -141,7 +141,7 @@ int unwind_user_faultable(struct unwind_stacktrace *trace)
 	cache->nr_entries = trace->nr;
 
 	/* Clear nr_entries on way back to user space */
-	set_bit(UNWIND_USED_BIT, &info->unwind_mask);
+	atomic_long_or(UNWIND_USED, &info->unwind_mask);
 
 	return 0;
 }
@@ -159,7 +159,7 @@ static void process_unwind_deferred(struct task_struct *task)
 
 	/* Clear pending bit but make sure to have the current bits */
 	bits = atomic_long_fetch_andnot(UNWIND_PENDING,
-				  (atomic_long_t *)&info->unwind_mask);
+					&info->unwind_mask);
 	/*
 	 * From here on out, the callback must always be called, even if it's
 	 * just an empty trace.
@@ -264,7 +264,7 @@ int unwind_deferred_request(struct unwind_work *work, u64 *cookie)
 
 	*cookie = get_cookie(info);
 
-	old = READ_ONCE(info->unwind_mask);
+	old = atomic_long_read(&info->unwind_mask);
 
 	/* Is this already queued or executed */
 	if (old & bit)
@@ -277,7 +277,7 @@ int unwind_deferred_request(struct unwind_work *work, u64 *cookie)
 	 * to have a callback.
 	 */
 	bits = UNWIND_PENDING | bit;
-	old = atomic_long_fetch_or(bits, (atomic_long_t *)&info->unwind_mask);
+	old = atomic_long_fetch_or(bits, &info->unwind_mask);
 	if (old & bits) {
 		/*
 		 * If the work's bit was set, whatever set it had better
@@ -291,7 +291,7 @@ int unwind_deferred_request(struct unwind_work *work, u64 *cookie)
 	ret = task_work_add(current, &info->work, twa_mode);
 
 	if (WARN_ON_ONCE(ret))
-		WRITE_ONCE(info->unwind_mask, 0);
+		atomic_long_set(&info->unwind_mask, 0);
 
 	return ret;
 }
@@ -323,7 +323,8 @@ void unwind_deferred_cancel(struct unwind_work *work)
 	guard(rcu)();
 	/* Clear this bit from all threads */
 	for_each_process_thread(g, t) {
-		clear_bit(bit, &t->unwind_info.unwind_mask);
+		atomic_long_andnot(BIT(bit),
+				   &t->unwind_info.unwind_mask);
 		if (t->unwind_info.cache)
 			clear_bit(bit, &t->unwind_info.cache->unwind_completed);
 	}
@@ -353,7 +354,7 @@ void unwind_task_init(struct task_struct *task)
 
 	memset(info, 0, sizeof(*info));
 	init_task_work(&info->work, unwind_deferred_task_work);
-	info->unwind_mask = 0;
+	atomic_long_set(&info->unwind_mask, 0);
 }
 
 void unwind_task_free(struct task_struct *task)

[PATCH 10/12] unwind: Simplify unwind_user_next_fp() alignment check

[Peter Zijlstra]

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 kernel/unwind/user.c |    4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

--- a/kernel/unwind/user.c
+++ b/kernel/unwind/user.c
@@ -19,7 +19,6 @@ static int unwind_user_next_fp(struct un
 {
 	const struct unwind_user_frame *frame = &fp_frame;
 	unsigned long cfa, fp, ra;
-	unsigned int shift;
 
 	if (frame->use_fp) {
 		if (state->fp < state->sp)
@@ -37,8 +36,7 @@ static int unwind_user_next_fp(struct un
 		return -EINVAL;
 
 	/* Make sure that the address is word aligned */
-	shift = sizeof(long) == 4 ? 2 : 3;
-	if (cfa & ((1 << shift) - 1))
+	if (cfa & (sizeof(long) - 1))
 		return -EINVAL;
 
 	/* Find the Return Address (RA) */

Re: [PATCH 10/12] unwind: Simplify unwind_user_next_fp() alignment check

[Steven Rostedt]

On Wed, 24 Sep 2025 09:59:58 +0200
Peter Zijlstra <peterz@infradead.org> wrote:

I would add a change log. Something like:

  sizeof(long) is 4 or 8. Where 4 = 1 << 2 and 8 = 1 << 3.
  Calculating shift to be 2 or 3 and then passing that variable into
  (1 << shift) is the same as just using sizeof(long).

I blame lack of sleep for writing that code :-p

> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>

Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>

-- Steve

> ---
>  kernel/unwind/user.c |    4 +---
>  1 file changed, 1 insertion(+), 3 deletions(-)
> 
> --- a/kernel/unwind/user.c
> +++ b/kernel/unwind/user.c
> @@ -19,7 +19,6 @@ static int unwind_user_next_fp(struct un
>  {
>  	const struct unwind_user_frame *frame = &fp_frame;
>  	unsigned long cfa, fp, ra;
> -	unsigned int shift;
>  
>  	if (frame->use_fp) {
>  		if (state->fp < state->sp)
> @@ -37,8 +36,7 @@ static int unwind_user_next_fp(struct un
>  		return -EINVAL;
>  
>  	/* Make sure that the address is word aligned */
> -	shift = sizeof(long) == 4 ? 2 : 3;
> -	if (cfa & ((1 << shift) - 1))
> +	if (cfa & (sizeof(long) - 1))
>  		return -EINVAL;
>  
>  	/* Find the Return Address (RA) */
> 

Re: [PATCH 10/12] unwind: Simplify unwind_user_next_fp() alignment check

[Peter Zijlstra]

On Wed, Oct 01, 2025 at 11:55:24AM -0400, Steven Rostedt wrote:
> On Wed, 24 Sep 2025 09:59:58 +0200
> Peter Zijlstra <peterz@infradead.org> wrote:
> 
> I would add a change log. Something like:
> 
>   sizeof(long) is 4 or 8. Where 4 = 1 << 2 and 8 = 1 << 3.
>   Calculating shift to be 2 or 3 and then passing that variable into
>   (1 << shift) is the same as just using sizeof(long).

I've made it: "2^log_2(n) == n". I find it very hard to spend that many
words on something this trivial :-)

Re: [PATCH 10/12] unwind: Simplify unwind_user_next_fp() alignment check

[Steven Rostedt]

On Mon, 20 Oct 2025 12:28:14 +0200
Peter Zijlstra <peterz@infradead.org> wrote:

> On Wed, Oct 01, 2025 at 11:55:24AM -0400, Steven Rostedt wrote:
> > On Wed, 24 Sep 2025 09:59:58 +0200
> > Peter Zijlstra <peterz@infradead.org> wrote:
> > 
> > I would add a change log. Something like:
> > 
> >   sizeof(long) is 4 or 8. Where 4 = 1 << 2 and 8 = 1 << 3.
> >   Calculating shift to be 2 or 3 and then passing that variable into
> >   (1 << shift) is the same as just using sizeof(long).  
> 
> I've made it: "2^log_2(n) == n". I find it very hard to spend that many
> words on something this trivial :-)

What you don't like Common Core Math[1]?

-- Steve

[1] https://3010tangents.wordpress.com/2015/03/03/common-core/

Re: [PATCH 10/12] unwind: Simplify unwind_user_next_fp() alignment check

[Peter Zijlstra]

On Wed, Oct 22, 2025 at 11:20:32AM -0400, Steven Rostedt wrote:
> On Mon, 20 Oct 2025 12:28:14 +0200
> Peter Zijlstra <peterz@infradead.org> wrote:
> 
> > On Wed, Oct 01, 2025 at 11:55:24AM -0400, Steven Rostedt wrote:
> > > On Wed, 24 Sep 2025 09:59:58 +0200
> > > Peter Zijlstra <peterz@infradead.org> wrote:
> > > 
> > > I would add a change log. Something like:
> > > 
> > >   sizeof(long) is 4 or 8. Where 4 = 1 << 2 and 8 = 1 << 3.
> > >   Calculating shift to be 2 or 3 and then passing that variable into
> > >   (1 << shift) is the same as just using sizeof(long).  
> > 
> > I've made it: "2^log_2(n) == n". I find it very hard to spend that many
> > words on something this trivial :-)
> 
> What you don't like Common Core Math[1]?

I'm not sure I'm qualified to comment on that particular topic. Yes,
there are many equivalent ways of doing 'math' and it is fun to explore
them all.

Specifically: 3000-1 and 2999+1 should both be mastered and both involve
the (decimal) carry bit. If your curriculum does not provide, or treats
one as more difficult from the other, something is wrong.

The thing I've noticed with my own kids though, is that teachers often
don't master these basics themselves, and find it very hard to properly
explain these things. This then makes me wonder HTF they ever got to be
a teacher, in face of their obvious incompetence, but that's another
problem.

[ I am now having to explain and derive the abc formula for finding the
  roots of the 2nd grade polynomial to my kid because curriculum mostly
  'forgot' about proofs and provides 'tricks' instead of understanding.

  Also, there is a very nice tie-in with this derivation of the abc
  formula to the starting point of calculus, notably the expression for
  finding the extreme is of course f'(x)=0, but is found here through
  simple algebra. ]

Anyway, I find throwing heaps of natural language at a trivial statement
makes it more opaque, rather than clearer.

[tip: perf/core] unwind: Simplify unwind_user_next_fp() alignment check

[tip-bot2 for Peter Zijlstra]

The following commit has been merged into the perf/core branch of tip:

Commit-ID:     5578534e4b92350995a20068f2e6ea3186c62d7f
Gitweb:        https://git.kernel.org/tip/5578534e4b92350995a20068f2e6ea3186c62d7f
Author:        Peter Zijlstra <peterz@infradead.org>
AuthorDate:    Tue, 23 Sep 2025 13:04:09 +02:00
Committer:     Peter Zijlstra <peterz@infradead.org>
CommitterDate: Wed, 29 Oct 2025 10:29:57 +01:00

unwind: Simplify unwind_user_next_fp() alignment check

  2^log_2(n) == n

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Link: https://patch.msgid.link/20250924080119.497867836@infradead.org
---
 kernel/unwind/user.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/kernel/unwind/user.c b/kernel/unwind/user.c
index 97a8415..9dcde79 100644
--- a/kernel/unwind/user.c
+++ b/kernel/unwind/user.c
@@ -19,7 +19,6 @@ static int unwind_user_next_fp(struct unwind_user_state *state)
 {
 	const struct unwind_user_frame *frame = &fp_frame;
 	unsigned long cfa, fp, ra;
-	unsigned int shift;
 
 	if (frame->use_fp) {
 		if (state->fp < state->sp)
@@ -37,8 +36,7 @@ static int unwind_user_next_fp(struct unwind_user_state *state)
 		return -EINVAL;
 
 	/* Make sure that the address is word aligned */
-	shift = sizeof(long) == 4 ? 2 : 3;
-	if (cfa & ((1 << shift) - 1))
+	if (cfa & (sizeof(long) - 1))
 		return -EINVAL;
 
 	/* Find the Return Address (RA) */

[PATCH 11/12] unwind: Implement compat fp unwind

[Peter Zijlstra]

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 include/linux/unwind_user_types.h |    1 +
 kernel/unwind/user.c              |   24 ++++++++++++++++++++----
 2 files changed, 21 insertions(+), 4 deletions(-)

--- a/include/linux/unwind_user_types.h
+++ b/include/linux/unwind_user_types.h
@@ -36,6 +36,7 @@ struct unwind_user_state {
 	unsigned long				ip;
 	unsigned long				sp;
 	unsigned long				fp;
+	unsigned int				ws;
 	enum unwind_user_type			current_type;
 	unsigned int				available_types;
 	bool					done;
--- a/kernel/unwind/user.c
+++ b/kernel/unwind/user.c
@@ -15,6 +15,21 @@ static const struct unwind_user_frame fp
 #define for_each_user_frame(state) \
 	for (unwind_user_start(state); !(state)->done; unwind_user_next(state))
 
+static inline int
+get_user_word(unsigned long *word, unsigned long base, int off, int size)
+{
+	unsigned long __user *addr = (void __user *)base + (off * size);
+#ifdef CONFIG_COMPAT
+	if (size == sizeof(int)) {
+		unsigned int data;
+		int ret = get_user(data, (unsigned int __user *)addr);
+		*word = data;
+		return ret;
+	}
+#endif
+	return get_user(*word, addr);
+}
+
 static int unwind_user_next_fp(struct unwind_user_state *state)
 {
 	const struct unwind_user_frame *frame = &fp_frame;
@@ -29,21 +44,21 @@ static int unwind_user_next_fp(struct un
 	}
 
 	/* Get the Canonical Frame Address (CFA) */
-	cfa += frame->cfa_off;
+	cfa += state->ws * frame->cfa_off;
 
 	/* stack going in wrong direction? */
 	if (cfa <= state->sp)
 		return -EINVAL;
 
 	/* Make sure that the address is word aligned */
-	if (cfa & (sizeof(long) - 1))
+	if (cfa & (state->ws - 1))
 		return -EINVAL;
 
 	/* Find the Return Address (RA) */
-	if (get_user(ra, (unsigned long *)(cfa + frame->ra_off)))
+	if (get_user_word(&ra, cfa, frame->ra_off, state->ws))
 		return -EINVAL;
 
-	if (frame->fp_off && get_user(fp, (unsigned long __user *)(cfa + frame->fp_off)))
+	if (frame->fp_off && get_user_word(&fp, cfa, frame->fp_off, state->ws))
 		return -EINVAL;
 
 	state->ip = ra;
@@ -100,6 +115,7 @@ static int unwind_user_start(struct unwi
 	state->ip = instruction_pointer(regs);
 	state->sp = user_stack_pointer(regs);
 	state->fp = frame_pointer(regs);
+	state->ws = compat_user_mode(regs) ? sizeof(int) : sizeof(long);
 
 	return 0;
 }

Re: [PATCH 11/12] unwind: Implement compat fp unwind

[Jens Remus]

Hello Peter, Steve, Josh, and Indu,

while rebasing the unwind user sframe series on top of this series and
https://lore.kernel.org/linux-trace-kernel/20251007214008.080852573@kernel.org/
I ran into the following issue:

On 9/24/2025 9:59 AM, Peter Zijlstra wrote:

> --- a/include/linux/unwind_user_types.h
> +++ b/include/linux/unwind_user_types.h
> @@ -36,6 +36,7 @@ struct unwind_user_state {
>  	unsigned long				ip;
>  	unsigned long				sp;
>  	unsigned long				fp;
> +	unsigned int				ws;

Factoring out the word size (ws) from the CFA, FP, and RA offsets is
clever.  Wondering though whether that would be an issue for unwind user
sframe.  Do all architectures guarantee that those offsets are aligned
to the native word size?

>  	enum unwind_user_type			current_type;
>  	unsigned int				available_types;
>  	bool					done;

> --- a/kernel/unwind/user.c
> +++ b/kernel/unwind/user.c

> @@ -29,21 +44,21 @@ static int unwind_user_next_fp(struct un
>  	}
>  
>  	/* Get the Canonical Frame Address (CFA) */
> -	cfa += frame->cfa_off;
> +	cfa += state->ws * frame->cfa_off;

In SFrame the CFA, FP, and RA offsets are unscaled.  Would it be ok, if
unwind user sframe would factor state->ws from those offset values?  What
if they were not aligned?  unwind user sframe would then have to fail.

@Indu: Thought from a SFrame perspective?

>  
>  	/* stack going in wrong direction? */
>  	if (cfa <= state->sp)
>  		return -EINVAL;
>  
>  	/* Make sure that the address is word aligned */
> -	if (cfa & (sizeof(long) - 1))
> +	if (cfa & (state->ws - 1))
>  		return -EINVAL;

Alternatively using a state->ws of 1 in uwind user sframe would defeat
this alignment check.

>  
>  	/* Find the Return Address (RA) */
> -	if (get_user(ra, (unsigned long *)(cfa + frame->ra_off)))
> +	if (get_user_word(&ra, cfa, frame->ra_off, state->ws))
>  		return -EINVAL;
>  
> -	if (frame->fp_off && get_user(fp, (unsigned long __user *)(cfa + frame->fp_off)))
> +	if (frame->fp_off && get_user_word(&fp, cfa, frame->fp_off, state->ws))
>  		return -EINVAL;
>  
>  	state->ip = ra;
> @@ -100,6 +115,7 @@ static int unwind_user_start(struct unwi
>  	state->ip = instruction_pointer(regs);
>  	state->sp = user_stack_pointer(regs);
>  	state->fp = frame_pointer(regs);
> +	state->ws = compat_user_mode(regs) ? sizeof(int) : sizeof(long);
>  
>  	return 0;
>  }
Thanks and regards,
Jens
-- 
Jens Remus
Linux on Z Development (D3303)
+49-7031-16-1128 Office
jremus@de.ibm.com

IBM

IBM Deutschland Research & Development GmbH; Vorsitzender des Aufsichtsrats: Wolfgang Wendt; Geschäftsführung: David Faller; Sitz der Gesellschaft: Böblingen; Registergericht: Amtsgericht Stuttgart, HRB 243294
IBM Data Privacy Statement: https://www.ibm.com/privacy/

Re: [PATCH 11/12] unwind: Implement compat fp unwind

[Jens Remus]

Hello Peter!

On 10/17/2025 5:47 PM, Jens Remus wrote:
> while rebasing the unwind user sframe series on top of this series and
> https://lore.kernel.org/linux-trace-kernel/20251007214008.080852573@kernel.org/
> I ran into the following issue:
> 
> On 9/24/2025 9:59 AM, Peter Zijlstra wrote:
> 
>> --- a/include/linux/unwind_user_types.h
>> +++ b/include/linux/unwind_user_types.h
>> @@ -36,6 +36,7 @@ struct unwind_user_state {
>>  	unsigned long				ip;
>>  	unsigned long				sp;
>>  	unsigned long				fp;
>> +	unsigned int				ws;
> 
> Factoring out the word size (ws) from the CFA, FP, and RA offsets is
> clever.  Wondering though whether that would be an issue for unwind user
> sframe.  Do all architectures guarantee that those offsets are aligned
> to the native word size?
> 
>>  	enum unwind_user_type			current_type;
>>  	unsigned int				available_types;
>>  	bool					done;
> 
>> --- a/kernel/unwind/user.c
>> +++ b/kernel/unwind/user.c
> 
>> @@ -29,21 +44,21 @@ static int unwind_user_next_fp(struct un
>>  	}
>>  
>>  	/* Get the Canonical Frame Address (CFA) */
>> -	cfa += frame->cfa_off;
>> +	cfa += state->ws * frame->cfa_off;
> 
> In SFrame the CFA, FP, and RA offsets are unscaled.  Would it be ok, if
> unwind user sframe would factor state->ws from those offset values?  What
> if they were not aligned?  unwind user sframe would then have to fail.

Sorry that I did not immediately think about the most obvious solution
tho above issues:  to not factor out the word size from the frame CFA,
FP, and RA offsets.  What do you think about making the following
changes to this and giyour subsequent patch?  That would work nicely
with unwind user sframe.


diff --git a/kernel/unwind/user.c b/kernel/unwind/user.c
--- a/kernel/unwind/user.c
+++ b/kernel/unwind/user.c
@@ -8,19 +8,15 @@
 #include <linux/unwind_user.h>
 #include <linux/uaccess.h>
 
-static const struct unwind_user_frame fp_frame = {
-	ARCH_INIT_USER_FP_FRAME
-};
-
 #define for_each_user_frame(state) \
 	for (unwind_user_start(state); !(state)->done; unwind_user_next(state))
 
 static inline int
-get_user_word(unsigned long *word, unsigned long base, int off, int size)
+get_user_word(unsigned long *word, unsigned long base, int off, unsigned int ws)
 {
-	unsigned long __user *addr = (void __user *)base + (off * size);
+	unsigned long __user *addr = (void __user *)base + off;
 #ifdef CONFIG_COMPAT
-	if (size == sizeof(int)) {
+	if (ws == sizeof(int)) {
 		unsigned int data;
 		int ret = get_user(data, (unsigned int __user *)addr);
 		*word = data;
@@ -32,6 +28,9 @@ get_user_word(unsigned long *word, unsigned long base, int off, int size)
 
 static int unwind_user_next_fp(struct unwind_user_state *state)
 {
+	const struct unwind_user_frame fp_frame = {
+		ARCH_INIT_USER_FP_FRAME(state->ws)
+	};
 	const struct unwind_user_frame *frame = &fp_frame;
 	unsigned long cfa, fp, ra;
 
@@ -44,7 +43,7 @@ static int unwind_user_next_fp(struct unwind_user_state *state)
 	}
 
 	/* Get the Canonical Frame Address (CFA) */
-	cfa += state->ws * frame->cfa_off;
+	cfa += frame->cfa_off;
 
 	/* stack going in wrong direction? */
 	if (cfa <= state->sp)


diff --git a/arch/x86/include/asm/unwind_user.h b/arch/x86/include/asm/unwind_user.h
--- a/arch/x86/include/asm/unwind_user.h
+++ b/arch/x86/include/asm/unwind_user.h
@@ -2,10 +2,10 @@
 #ifndef _ASM_X86_UNWIND_USER_H
 #define _ASM_X86_UNWIND_USER_H
 
-#define ARCH_INIT_USER_FP_FRAME				\
-	.cfa_off	=  2,				\
-	.ra_off		= -1,				\
-	.fp_off		= -2,				\
+#define ARCH_INIT_USER_FP_FRAME(ws)			\
+	.cfa_off	=  2*(ws),			\
+	.ra_off		= -1*(ws),			\
+	.fp_off		= -2*(ws),			\
 	.use_fp		= true,
 
 #endif /* _ASM_X86_UNWIND_USER_H */


Thanks and regards,
Jens
-- 
Jens Remus
Linux on Z Development (D3303)
+49-7031-16-1128 Office
jremus@de.ibm.com

IBM

IBM Deutschland Research & Development GmbH; Vorsitzender des Aufsichtsrats: Wolfgang Wendt; Geschäftsführung: David Faller; Sitz der Gesellschaft: Böblingen; Registergericht: Amtsgericht Stuttgart, HRB 243294
IBM Data Privacy Statement: https://www.ibm.com/privacy/

Re: [PATCH 11/12] unwind: Implement compat fp unwind

[Peter Zijlstra]

On Mon, Oct 20, 2025 at 11:16:45AM +0200, Jens Remus wrote:
> Hello Peter!
> 
> On 10/17/2025 5:47 PM, Jens Remus wrote:
> > while rebasing the unwind user sframe series on top of this series and
> > https://lore.kernel.org/linux-trace-kernel/20251007214008.080852573@kernel.org/
> > I ran into the following issue:
> > 
> > On 9/24/2025 9:59 AM, Peter Zijlstra wrote:
> > 
> >> --- a/include/linux/unwind_user_types.h
> >> +++ b/include/linux/unwind_user_types.h
> >> @@ -36,6 +36,7 @@ struct unwind_user_state {
> >>  	unsigned long				ip;
> >>  	unsigned long				sp;
> >>  	unsigned long				fp;
> >> +	unsigned int				ws;
> > 
> > Factoring out the word size (ws) from the CFA, FP, and RA offsets is
> > clever.  Wondering though whether that would be an issue for unwind user
> > sframe.  Do all architectures guarantee that those offsets are aligned
> > to the native word size?
> > 
> >>  	enum unwind_user_type			current_type;
> >>  	unsigned int				available_types;
> >>  	bool					done;
> > 
> >> --- a/kernel/unwind/user.c
> >> +++ b/kernel/unwind/user.c
> > 
> >> @@ -29,21 +44,21 @@ static int unwind_user_next_fp(struct un
> >>  	}
> >>  
> >>  	/* Get the Canonical Frame Address (CFA) */
> >> -	cfa += frame->cfa_off;
> >> +	cfa += state->ws * frame->cfa_off;
> > 
> > In SFrame the CFA, FP, and RA offsets are unscaled.  Would it be ok, if
> > unwind user sframe would factor state->ws from those offset values?  What
> > if they were not aligned?  unwind user sframe would then have to fail.
> 
> Sorry that I did not immediately think about the most obvious solution
> tho above issues:  to not factor out the word size from the frame CFA,
> FP, and RA offsets.  What do you think about making the following
> changes to this and giyour subsequent patch?  That would work nicely
> with unwind user sframe.


Yes, this should do nicely. I've made the changes, I'll do a test build
and then push out to the robots.

Thanks!

Re: [PATCH 11/12] unwind: Implement compat fp unwind

[Peter Zijlstra]

On Mon, Oct 20, 2025 at 12:39:40PM +0200, Peter Zijlstra wrote:

> Yes, this should do nicely. I've made the changes, I'll do a test build
> and then push out to the robots.

Ok, this now lives here:

  git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git unwind/cleanup

Suppose this all comes back clean from the robots, where shall I merge
it? tip/perf/core, tip/x86/core ?

Re: [PATCH 11/12] unwind: Implement compat fp unwind

[Steven Rostedt]

On Mon, 20 Oct 2025 12:48:07 +0200
Peter Zijlstra <peterz@infradead.org> wrote:

> On Mon, Oct 20, 2025 at 12:39:40PM +0200, Peter Zijlstra wrote:
> 
> > Yes, this should do nicely. I've made the changes, I'll do a test build
> > and then push out to the robots.  
> 
> Ok, this now lives here:
> 
>   git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git unwind/cleanup
> 
> Suppose this all comes back clean from the robots, where shall I merge
> it? tip/perf/core, tip/x86/core ?

I've been basing all my perf work off of tip/perf/core, so perhaps use that branch?

Thanks!

-- Steve

Re: [PATCH 11/12] unwind: Implement compat fp unwind

[Peter Zijlstra]

On Wed, Oct 22, 2025 at 11:23:44AM -0400, Steven Rostedt wrote:
> On Mon, 20 Oct 2025 12:48:07 +0200
> Peter Zijlstra <peterz@infradead.org> wrote:
> 
> > On Mon, Oct 20, 2025 at 12:39:40PM +0200, Peter Zijlstra wrote:
> > 
> > > Yes, this should do nicely. I've made the changes, I'll do a test build
> > > and then push out to the robots.  
> > 
> > Ok, this now lives here:
> > 
> >   git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git unwind/cleanup
> > 
> > Suppose this all comes back clean from the robots, where shall I merge
> > it? tip/perf/core, tip/x86/core ?
> 
> I've been basing all my perf work off of tip/perf/core, so perhaps use that branch?

Right, I'll move the pile over to queue/perf/core and once blessed by
the robots shove it into -tip.

Re: [PATCH 11/12] unwind: Implement compat fp unwind

[Jens Remus]

Hello Peter!

On 10/20/2025 12:39 PM, Peter Zijlstra wrote:
> On Mon, Oct 20, 2025 at 11:16:45AM +0200, Jens Remus wrote:
>> On 10/17/2025 5:47 PM, Jens Remus wrote:

>>> In SFrame the CFA, FP, and RA offsets are unscaled.  Would it be ok, if
>>> unwind user sframe would factor state->ws from those offset values?  What
>>> if they were not aligned?  unwind user sframe would then have to fail.
>>
>> Sorry that I did not immediately think about the most obvious solution
>> tho above issues:  to not factor out the word size from the frame CFA,
>> FP, and RA offsets.  What do you think about making the following
>> changes to this and giyour subsequent patch?  That would work nicely
>> with unwind user sframe.
> 
> 
> Yes, this should do nicely. I've made the changes, I'll do a test build
> and then push out to the robots.

Thanks!  Looking at your following updated patch I found that your
change from "pointer to const struct unwind_user_frame" to
"const struct unwind_user_frame" (done for obvious reasons) will require
unwind user sframe to undo this when refactoring unwind_user_next_fp()
into unwind_user_next_common().  Would that be the usual procedure or
could you leave it as "pointer to const struct unwind_user_frame" for
now?

https://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git/commit/?h=unwind/cleanup&id=f3624d64ba4862067b620fbd5bfbc0bfaf5368ae

Thanks and regards,
Jens
-- 
Jens Remus
Linux on Z Development (D3303)
+49-7031-16-1128 Office
jremus@de.ibm.com

IBM

IBM Deutschland Research & Development GmbH; Vorsitzender des Aufsichtsrats: Wolfgang Wendt; Geschäftsführung: David Faller; Sitz der Gesellschaft: Böblingen; Registergericht: Amtsgericht Stuttgart, HRB 243294
IBM Data Privacy Statement: https://www.ibm.com/privacy/

Re: [PATCH 11/12] unwind: Implement compat fp unwind

[Peter Zijlstra]

On Wed, Oct 22, 2025 at 04:55:01PM +0200, Jens Remus wrote:
> Hello Peter!
> 
> On 10/20/2025 12:39 PM, Peter Zijlstra wrote:
> > On Mon, Oct 20, 2025 at 11:16:45AM +0200, Jens Remus wrote:
> >> On 10/17/2025 5:47 PM, Jens Remus wrote:
> 
> >>> In SFrame the CFA, FP, and RA offsets are unscaled.  Would it be ok, if
> >>> unwind user sframe would factor state->ws from those offset values?  What
> >>> if they were not aligned?  unwind user sframe would then have to fail.
> >>
> >> Sorry that I did not immediately think about the most obvious solution
> >> tho above issues:  to not factor out the word size from the frame CFA,
> >> FP, and RA offsets.  What do you think about making the following
> >> changes to this and giyour subsequent patch?  That would work nicely
> >> with unwind user sframe.
> > 
> > 
> > Yes, this should do nicely. I've made the changes, I'll do a test build
> > and then push out to the robots.
> 
> Thanks!  Looking at your following updated patch I found that your
> change from "pointer to const struct unwind_user_frame" to
> "const struct unwind_user_frame" (done for obvious reasons) will require
> unwind user sframe to undo this when refactoring unwind_user_next_fp()
> into unwind_user_next_common().  Would that be the usual procedure or
> could you leave it as "pointer to const struct unwind_user_frame" for
> now?

Ah, I see, that is here:

  https://lkml.kernel.org/r/20251022144326.4082059-9-jremus@linux.ibm.com

Yeah, just change it there. It is a bit weird to have this indirection
at this point.

Re: [PATCH 11/12] unwind: Implement compat fp unwind

[Peter Zijlstra]

On Fri, Oct 17, 2025 at 05:47:26PM +0200, Jens Remus wrote:
> On 9/24/2025 9:59 AM, Peter Zijlstra wrote:
> 
> > --- a/include/linux/unwind_user_types.h
> > +++ b/include/linux/unwind_user_types.h
> > @@ -36,6 +36,7 @@ struct unwind_user_state {
> >  	unsigned long				ip;
> >  	unsigned long				sp;
> >  	unsigned long				fp;
> > +	unsigned int				ws;
> 
> Factoring out the word size (ws) from the CFA, FP, and RA offsets is
> clever.  Wondering though whether that would be an issue for unwind user
> sframe.  Do all architectures guarantee that those offsets are aligned
> to the native word size?

I would hope so, but this is all opt-in, I'm sure the first architecture
with an unaligned stack trying to support this will let us know ;-)

> >  	/* Make sure that the address is word aligned */
> > -	if (cfa & (sizeof(long) - 1))
> > +	if (cfa & (state->ws - 1))
> >  		return -EINVAL;
> 
> Alternatively using a state->ws of 1 in uwind user sframe would defeat
> this alignment check.

Indeed. Or rather, with a words size of 1, everything is aligned :-)

Re: [PATCH 11/12] unwind: Implement compat fp unwind

[Steven Rostedt]

On Wed, 24 Sep 2025 09:59:59 +0200
Peter Zijlstra <peterz@infradead.org> wrote:

> @@ -100,6 +115,7 @@ static int unwind_user_start(struct unwi
>  	state->ip = instruction_pointer(regs);
>  	state->sp = user_stack_pointer(regs);
>  	state->fp = frame_pointer(regs);
> +	state->ws = compat_user_mode(regs) ? sizeof(int) : sizeof(long);

compat_user_mode() is an architecture function (only defined in arm64 and now x86).

s390 doesn't implement it and regs can't be used to tell if it's compat or
not (although Jens tells me the task_struct can).

To do this properly in generic code, we should add a a:

	unwind_compat_mode(struct pt_regs *regs);

call in include/linux/unwind_user.h:

#ifndef unwind_compat_mode
static inline bool unwind_compat_mode(struct pt_regs *regs)
{
	return false;
}
#endif

And then in the x86 and arm64 asm/unwind_user.h:

static inline bool unwind_compat_mode(struct pt_regs *regs)
{
	return compat_user_mode(regs);
}
#define unwind_compat_mode unwind_compat_mode

-- Steve


>  
>  	return 0;
>  }
> 

Re: [PATCH 11/12] unwind: Implement compat fp unwind

[Peter Zijlstra]

On Wed, Oct 22, 2025 at 02:31:40PM -0400, Steven Rostedt wrote:
> On Wed, 24 Sep 2025 09:59:59 +0200
> Peter Zijlstra <peterz@infradead.org> wrote:
> 
> > @@ -100,6 +115,7 @@ static int unwind_user_start(struct unwi
> >  	state->ip = instruction_pointer(regs);
> >  	state->sp = user_stack_pointer(regs);
> >  	state->fp = frame_pointer(regs);
> > +	state->ws = compat_user_mode(regs) ? sizeof(int) : sizeof(long);
> 
> compat_user_mode() is an architecture function (only defined in arm64 and now x86).
> 
> s390 doesn't implement it and regs can't be used to tell if it's compat or
> not (although Jens tells me the task_struct can).

I've made this:

	state->ws = unwind_user_word_size(regs);

And then every arch will need to implement this. The x86 implementations
is:

static inline int unwind_user_word_size(struct pt_regs *regs)
{
#ifdef CONFIG_X86_64
       if (!user_64bit_mode(regs))
               return sizeof(int);
#endif
       return sizeof(long);
}

Re: [PATCH 11/12] unwind: Implement compat fp unwind

[Peter Zijlstra]

On Fri, Oct 24, 2025 at 04:10:56PM +0200, Peter Zijlstra wrote:
> On Wed, Oct 22, 2025 at 02:31:40PM -0400, Steven Rostedt wrote:
> > On Wed, 24 Sep 2025 09:59:59 +0200
> > Peter Zijlstra <peterz@infradead.org> wrote:
> > 
> > > @@ -100,6 +115,7 @@ static int unwind_user_start(struct unwi
> > >  	state->ip = instruction_pointer(regs);
> > >  	state->sp = user_stack_pointer(regs);
> > >  	state->fp = frame_pointer(regs);
> > > +	state->ws = compat_user_mode(regs) ? sizeof(int) : sizeof(long);
> > 
> > compat_user_mode() is an architecture function (only defined in arm64 and now x86).
> > 
> > s390 doesn't implement it and regs can't be used to tell if it's compat or
> > not (although Jens tells me the task_struct can).
> 
> I've made this:
> 
> 	state->ws = unwind_user_word_size(regs);

Ooh, how about I do:

	if (!state->ws) {
		state->done = true;
		return -EINVAL; // nobody cares about this return value
	}

> 
> And then every arch will need to implement this. The x86 implementations
> is:
> 
> static inline int unwind_user_word_size(struct pt_regs *regs)
> {
	if (regs->flags & X86_VM_MASK)
		return 0;
> #ifdef CONFIG_X86_64
>        if (!user_64bit_mode(regs))
>                return sizeof(int);
> #endif
>        return sizeof(long);
> }

Then we flat out refuse to unwind VM86, which is slightly different from
the current code (which would still record regs->ip), but meh.

[tip: perf/core] unwind: Implement compat fp unwind

[tip-bot2 for Peter Zijlstra]

The following commit has been merged into the perf/core branch of tip:

Commit-ID:     c79dd946e370af3537edb854f210cba3a94b4516
Gitweb:        https://git.kernel.org/tip/c79dd946e370af3537edb854f210cba3a94b4516
Author:        Peter Zijlstra <peterz@infradead.org>
AuthorDate:    Tue, 23 Sep 2025 13:27:34 +02:00
Committer:     Peter Zijlstra <peterz@infradead.org>
CommitterDate: Wed, 29 Oct 2025 10:29:57 +01:00

unwind: Implement compat fp unwind

It is important to be able to unwind compat tasks too.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20250924080119.613695709@infradead.org
---
 include/linux/unwind_user_types.h |  1 +-
 kernel/unwind/user.c              | 40 +++++++++++++++++++++---------
 2 files changed, 30 insertions(+), 11 deletions(-)

diff --git a/include/linux/unwind_user_types.h b/include/linux/unwind_user_types.h
index a449f15..938f7e6 100644
--- a/include/linux/unwind_user_types.h
+++ b/include/linux/unwind_user_types.h
@@ -36,6 +36,7 @@ struct unwind_user_state {
 	unsigned long				ip;
 	unsigned long				sp;
 	unsigned long				fp;
+	unsigned int				ws;
 	enum unwind_user_type			current_type;
 	unsigned int				available_types;
 	bool					done;
diff --git a/kernel/unwind/user.c b/kernel/unwind/user.c
index 9dcde79..6428715 100644
--- a/kernel/unwind/user.c
+++ b/kernel/unwind/user.c
@@ -8,19 +8,32 @@
 #include <linux/unwind_user.h>
 #include <linux/uaccess.h>
 
-static const struct unwind_user_frame fp_frame = {
-	ARCH_INIT_USER_FP_FRAME
-};
-
 #define for_each_user_frame(state) \
 	for (unwind_user_start(state); !(state)->done; unwind_user_next(state))
 
+static inline int
+get_user_word(unsigned long *word, unsigned long base, int off, unsigned int ws)
+{
+	unsigned long __user *addr = (void __user *)base + off;
+#ifdef CONFIG_COMPAT
+	if (ws == sizeof(int)) {
+		unsigned int data;
+		int ret = get_user(data, (unsigned int __user *)addr);
+		*word = data;
+		return ret;
+	}
+#endif
+	return get_user(*word, addr);
+}
+
 static int unwind_user_next_fp(struct unwind_user_state *state)
 {
-	const struct unwind_user_frame *frame = &fp_frame;
+	const struct unwind_user_frame frame = {
+		ARCH_INIT_USER_FP_FRAME(state->ws)
+	};
 	unsigned long cfa, fp, ra;
 
-	if (frame->use_fp) {
+	if (frame.use_fp) {
 		if (state->fp < state->sp)
 			return -EINVAL;
 		cfa = state->fp;
@@ -29,26 +42,26 @@ static int unwind_user_next_fp(struct unwind_user_state *state)
 	}
 
 	/* Get the Canonical Frame Address (CFA) */
-	cfa += frame->cfa_off;
+	cfa += frame.cfa_off;
 
 	/* stack going in wrong direction? */
 	if (cfa <= state->sp)
 		return -EINVAL;
 
 	/* Make sure that the address is word aligned */
-	if (cfa & (sizeof(long) - 1))
+	if (cfa & (state->ws - 1))
 		return -EINVAL;
 
 	/* Find the Return Address (RA) */
-	if (get_user(ra, (unsigned long *)(cfa + frame->ra_off)))
+	if (get_user_word(&ra, cfa, frame.ra_off, state->ws))
 		return -EINVAL;
 
-	if (frame->fp_off && get_user(fp, (unsigned long __user *)(cfa + frame->fp_off)))
+	if (frame.fp_off && get_user_word(&fp, cfa, frame.fp_off, state->ws))
 		return -EINVAL;
 
 	state->ip = ra;
 	state->sp = cfa;
-	if (frame->fp_off)
+	if (frame.fp_off)
 		state->fp = fp;
 	return 0;
 }
@@ -100,6 +113,11 @@ static int unwind_user_start(struct unwind_user_state *state)
 	state->ip = instruction_pointer(regs);
 	state->sp = user_stack_pointer(regs);
 	state->fp = frame_pointer(regs);
+	state->ws = unwind_user_word_size(regs);
+	if (!state->ws) {
+		state->done = true;
+		return -EINVAL;
+	}
 
 	return 0;
 }

[PATCH 12/12] unwind_user/x86: Enable frame pointer unwinding on x86

[Peter Zijlstra]

From: Josh Poimboeuf <jpoimboe@kernel.org>

Use ARCH_INIT_USER_FP_FRAME to describe how frame pointers are unwound
on x86, and enable CONFIG_HAVE_UNWIND_USER_FP accordingly so the
unwind_user interfaces can be used.

Signed-off-by: Josh Poimboeuf <jpoimboe@kernel.org>
Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 arch/x86/Kconfig                   |    1 +
 arch/x86/include/asm/ptrace.h      |    9 +++++++++
 arch/x86/include/asm/unwind_user.h |   11 +++++++++++
 3 files changed, 21 insertions(+)
 create mode 100644 arch/x86/include/asm/unwind_user.h

--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -297,6 +297,7 @@ config X86
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_UACCESS_VALIDATION		if HAVE_OBJTOOL
 	select HAVE_UNSTABLE_SCHED_CLOCK
+	select HAVE_UNWIND_USER_FP		if X86_64
 	select HAVE_USER_RETURN_NOTIFIER
 	select HAVE_GENERIC_VDSO
 	select VDSO_GETRANDOM			if X86_64
--- a/arch/x86/include/asm/ptrace.h
+++ b/arch/x86/include/asm/ptrace.h
@@ -255,6 +255,15 @@ static inline bool any_64bit_mode(struct
 #endif
 }
 
+static inline bool compat_user_mode(struct pt_regs *regs)
+{
+#ifdef CONFIG_X86_64
+	return !user_64bit_mode(regs);
+#else
+	return false;
+#endif
+}
+
 #ifdef CONFIG_X86_64
 #define current_user_stack_pointer()	current_pt_regs()->sp
 #define compat_user_stack_pointer()	current_pt_regs()->sp
--- /dev/null
+++ b/arch/x86/include/asm/unwind_user.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_X86_UNWIND_USER_H
+#define _ASM_X86_UNWIND_USER_H
+
+#define ARCH_INIT_USER_FP_FRAME				\
+	.cfa_off	=  2,				\
+	.ra_off		= -1,				\
+	.fp_off		= -2,				\
+	.use_fp		= true,
+
+#endif /* _ASM_X86_UNWIND_USER_H */