* [PATCH RFC v4 01/13] entry: Pass pt_regs to irqentry_exit_cond_resched()
2026-09-18 14:49 [PATCH RFC v4 00/13] rcu-tasks: build Tasks RCU on Tasks Trace readers in trampolines Josef Bacik
@ 2026-09-18 14:49 ` Josef Bacik
2026-09-18 14:49 ` [PATCH RFC v4 02/13] rcu-tasks: Add a Tasks RCU implementation for reader-marked trampolines Josef Bacik
` (11 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Josef Bacik @ 2026-09-18 14:49 UTC (permalink / raw)
To: Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai,
Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai
Cc: Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Josef Bacik
The irq-exit preemption path is about to need the interrupted context's
registers to decide whether the preemption may be reported to Tasks RCU
as a quiescent state. irqentry_exit_to_kernel_mode_preempt() already
has them; hand them down through irqentry_exit_cond_resched(), its
PREEMPT_DYNAMIC static-call and static-key variants, and
raw_irqentry_exit_cond_resched(). The only caller outside the generic
entry code is Xen PV's upcall handler, which has regs as well.
No functional change.
Assisted-by: LLM
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
---
arch/x86/xen/enlighten_pv.c | 2 +-
include/linux/irq-entry-common.h | 13 +++++++------
kernel/entry/common.c | 6 +++---
3 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c
index 2c64b388f616..3d85035f5624 100644
--- a/arch/x86/xen/enlighten_pv.c
+++ b/arch/x86/xen/enlighten_pv.c
@@ -739,7 +739,7 @@ __visible noinstr void xen_pv_evtchn_do_upcall(struct pt_regs *regs)
inhcall = get_and_clear_inhcall();
if (inhcall && !WARN_ON_ONCE(state.exit_rcu)) {
- irqentry_exit_cond_resched();
+ irqentry_exit_cond_resched(regs);
instrumentation_end();
restore_inhcall(inhcall);
} else {
diff --git a/include/linux/irq-entry-common.h b/include/linux/irq-entry-common.h
index 0bb6c03481fa..e19b41ee6b18 100644
--- a/include/linux/irq-entry-common.h
+++ b/include/linux/irq-entry-common.h
@@ -343,24 +343,25 @@ typedef struct irqentry_state {
/**
* irqentry_exit_cond_resched - Conditionally reschedule on return from interrupt
+ * @regs: Pointer to pt_regs of interrupted context
*
* Conditional reschedule with additional sanity checks.
*/
-void raw_irqentry_exit_cond_resched(void);
+void raw_irqentry_exit_cond_resched(struct pt_regs *regs);
#ifdef CONFIG_PREEMPT_DYNAMIC
#if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
#define irqentry_exit_cond_resched_dynamic_enabled raw_irqentry_exit_cond_resched
#define irqentry_exit_cond_resched_dynamic_disabled NULL
DECLARE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched);
-#define irqentry_exit_cond_resched() static_call(irqentry_exit_cond_resched)()
+#define irqentry_exit_cond_resched(regs) static_call(irqentry_exit_cond_resched)(regs)
#elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);
-void dynamic_irqentry_exit_cond_resched(void);
-#define irqentry_exit_cond_resched() dynamic_irqentry_exit_cond_resched()
+void dynamic_irqentry_exit_cond_resched(struct pt_regs *regs);
+#define irqentry_exit_cond_resched(regs) dynamic_irqentry_exit_cond_resched(regs)
#endif
#else /* CONFIG_PREEMPT_DYNAMIC */
-#define irqentry_exit_cond_resched() raw_irqentry_exit_cond_resched()
+#define irqentry_exit_cond_resched(regs) raw_irqentry_exit_cond_resched(regs)
#endif /* CONFIG_PREEMPT_DYNAMIC */
/**
@@ -465,7 +466,7 @@ static inline void irqentry_exit_to_kernel_mode_preempt(struct pt_regs *regs,
return;
if (IS_ENABLED(CONFIG_PREEMPTION))
- irqentry_exit_cond_resched();
+ irqentry_exit_cond_resched(regs);
}
/**
diff --git a/kernel/entry/common.c b/kernel/entry/common.c
index e3d381fd3d25..e4acd50bd81a 100644
--- a/kernel/entry/common.c
+++ b/kernel/entry/common.c
@@ -134,7 +134,7 @@ static inline bool arch_irqentry_exit_need_resched(void);
static inline bool arch_irqentry_exit_need_resched(void) { return true; }
#endif
-void raw_irqentry_exit_cond_resched(void)
+void raw_irqentry_exit_cond_resched(struct pt_regs *regs)
{
if (!preempt_count()) {
/* Sanity check RCU and thread stack */
@@ -150,11 +150,11 @@ void raw_irqentry_exit_cond_resched(void)
DEFINE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched);
#elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);
-void dynamic_irqentry_exit_cond_resched(void)
+void dynamic_irqentry_exit_cond_resched(struct pt_regs *regs)
{
if (!static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched))
return;
- raw_irqentry_exit_cond_resched();
+ raw_irqentry_exit_cond_resched(regs);
}
#endif
#endif
--
2.55.0
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH RFC v4 02/13] rcu-tasks: Add a Tasks RCU implementation for reader-marked trampolines
2026-09-18 14:49 [PATCH RFC v4 00/13] rcu-tasks: build Tasks RCU on Tasks Trace readers in trampolines Josef Bacik
2026-09-18 14:49 ` [PATCH RFC v4 01/13] entry: Pass pt_regs to irqentry_exit_cond_resched() Josef Bacik
@ 2026-09-18 14:49 ` Josef Bacik
2026-09-18 14:49 ` [PATCH RFC v4 03/13] kprobes: Expose the optprobe jump window to Tasks RCU Josef Bacik
` (10 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Josef Bacik @ 2026-09-18 14:49 UTC (permalink / raw)
To: Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai,
Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai
Cc: Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Josef Bacik
Tasks RCU waits for every task to pass through a voluntary context
switch, usermode or idle, because a preempted task might be sitting in a
trampoline that is about to be freed and nothing marks it as such. With
PREEMPT_LAZY that is a poor fit for servers: cond_resched() is a no-op,
so a CPU-bound kthread only ever leaves the CPU by preemption, and one
such kthread holds every synchronize_rcu_tasks() caller -- ftrace and
BPF trampoline teardown under their mutexes, the kprobe jump optimizer
under text_mutex and cpus_read_lock() -- hostage for as long as it runs.
Following the discussion on v2, take the other road: let the
architecture make its trampolines Tasks Trace RCU readers. When an
architecture selects HAVE_RCU_TRAMPOLINE_READERS it promises that every
trampoline whose lifetime Tasks RCU guards enters rcu_read_lock_trace()
(or its assembly equivalent) before calling out and leaves it before
returning, so a task anywhere inside such a call-out, preempted or not,
is an ordinary Tasks Trace reader.
That leaves the few instructions of trampoline text before the reader is
entered and after it is left (plus, in a later patch, the bytes a kprobe
jump optimization is about to overwrite). A task can only linger there
by being interrupted there, and such text never calls anything that
schedules, so instead of tracking tasks we track CPUs: every pass
through __schedule() is a per-CPU quiescent event, except that the one
context switch that can catch a task at an arbitrary instruction -- a
preemption from irq exit -- first records the interrupted IP in the task
and parks it on a per-CPU list for the duration (reusing the fields and
lists the classic flavor keeps for its exit-path bookkeeping), and, if
the IP is inside such "unmarked" text, puts the task on a short holdout
list; the task takes itself off at its next context switch outside such
a preemption or irq-exit check that finds it elsewhere. Usermode (the
existing tick hook, or a nohz_full CPU in an RCU extended quiescent
state) and idle count as well. rcu_tasks_trampoline_text() does the
classification: anything outside core and module text, plus an arch hook
for things like static ftrace stubs and return thunks.
The grace period, run by the existing rcu_tasks kthread so that
call_rcu_tasks(), synchronize_rcu_tasks() and rcu_barrier_tasks() keep
their names and callers, is: wait for every online CPU to context switch
or be seen in an RCU extended quiescent state (nudging stragglers with
resched_cpu() after a jiffy), drain the holdout list as it stood,
synchronize_rcu_tasks_trace() for everything inside the readers, then
one more CPU pass and drain for tasks that have since left the reader
into the trailing instructions. That is bounded by a few jiffies,
preempt-off latency and an SRCU grace period rather than by the longest
stretch any task runs without sleeping, needs no per-task scan, and
makes cond_resched_tasks_rcu_qs() unnecessary on such architectures.
Unlike the classic flavor it also waits for an idle task caught in a
trampoline, since an idle CPU only counts while RCU is not watching it.
rcu_tasks_wait_irq_preempted() walks the parked lists for the one caller
(the kprobe jump optimizer, later in the series) that makes ordinary
text unsafe to be parked in and so has to wait out tasks that were
preempted there before it said so.
The classic implementation is untouched and remains the default; the
new one is built only as CONFIG_TASKS_RCU_TRAMPOLINE_READERS when the
architecture opts in and uses the generic irq entry code, whose
reschedule check gains the rcu_tasks_irq_resched() call. Nothing
selects it yet.
Suggested-by: Paul E. McKenney <paulmck@kernel.org>
Suggested-by: Alexei Starovoitov <ast@kernel.org>
Assisted-by: LLM
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
---
include/linux/rcupdate.h | 24 ++-
include/linux/sched.h | 1 +
kernel/entry/common.c | 8 +-
kernel/fork.c | 1 +
kernel/rcu/Kconfig | 22 +++
kernel/rcu/tasks.h | 459 ++++++++++++++++++++++++++++++++++++++++++++++-
kernel/rcu/update.c | 2 +
7 files changed, 508 insertions(+), 9 deletions(-)
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 44c07a66edff..79e4b14f83b9 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -50,6 +50,23 @@ token_context_lock_instance(RCU, RCU_BH);
/* Exported common interfaces */
void call_rcu(struct rcu_head *head, rcu_callback_t func);
void rcu_barrier_tasks(void);
+
+/*
+ * Trampoline-reader Tasks RCU (CONFIG_TASKS_RCU_TRAMPOLINE_READERS), see
+ * kernel/rcu/tasks.h. rcu_tasks_irq_resched_enter()/_exit() bracket the
+ * irq-exit preemption; rcu_tasks_trampoline_text() and the arch_ override
+ * classify an interrupted IP; rcu_tasks_wait_irq_preempted() lets a caller
+ * wait out tasks already preempted somewhere it is about to make unsafe.
+ */
+void rcu_tasks_irq_resched_enter(unsigned long ip);
+void rcu_tasks_irq_resched_exit(void);
+bool rcu_tasks_trampoline_text(unsigned long ip);
+bool arch_rcu_tasks_trampoline_text(unsigned long ip);
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+void rcu_tasks_wait_irq_preempted(bool (*inside)(unsigned long ip));
+#else
+static inline void rcu_tasks_wait_irq_preempted(bool (*inside)(unsigned long ip)) { }
+#endif
void synchronize_rcu(void);
/*
@@ -180,11 +197,16 @@ static inline void rcu_nocb_flush_deferred_wakeup(void) { }
#ifdef CONFIG_TASKS_RCU_GENERIC
# ifdef CONFIG_TASKS_RCU
-# define rcu_tasks_classic_qs(t, preempt) \
+# ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+void rcu_tasks_note_qs(struct task_struct *t, bool preempt);
+# define rcu_tasks_classic_qs(t, preempt) rcu_tasks_note_qs((t), (preempt))
+# else
+# define rcu_tasks_classic_qs(t, preempt) \
do { \
if (!(preempt) && READ_ONCE((t)->rcu_tasks_holdout)) \
WRITE_ONCE((t)->rcu_tasks_holdout, false); \
} while (0)
+# endif
void call_rcu_tasks(struct rcu_head *head, rcu_callback_t func);
void synchronize_rcu_tasks(void);
void rcu_tasks_torture_stats_print(char *tt, char *tf);
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 8b3d47a325cc..15beb44caa2c 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -957,6 +957,7 @@ struct task_struct {
u8 rcu_tasks_holdout;
u8 rcu_tasks_idx;
int rcu_tasks_idle_cpu;
+ unsigned long rcu_tasks_irq_ip;
struct list_head rcu_tasks_holdout_list;
int rcu_tasks_exit_cpu;
struct list_head rcu_tasks_exit_list;
diff --git a/kernel/entry/common.c b/kernel/entry/common.c
index e4acd50bd81a..94318519998c 100644
--- a/kernel/entry/common.c
+++ b/kernel/entry/common.c
@@ -6,6 +6,7 @@
#include <linux/jump_label.h>
#include <linux/kmsan.h>
#include <linux/livepatch.h>
+#include <linux/rcupdate.h>
#include <linux/resume_user_mode.h>
#include <linux/tick.h>
@@ -141,8 +142,13 @@ void raw_irqentry_exit_cond_resched(struct pt_regs *regs)
rcu_irq_exit_check_preempt();
if (IS_ENABLED(CONFIG_DEBUG_ENTRY))
WARN_ON_ONCE(!on_thread_stack());
- if (need_resched() && arch_irqentry_exit_need_resched())
+ if (need_resched() && arch_irqentry_exit_need_resched()) {
+ if (IS_ENABLED(CONFIG_TASKS_RCU_TRAMPOLINE_READERS))
+ rcu_tasks_irq_resched_enter(instruction_pointer(regs));
preempt_schedule_irq();
+ if (IS_ENABLED(CONFIG_TASKS_RCU_TRAMPOLINE_READERS))
+ rcu_tasks_irq_resched_exit();
+ }
}
}
#ifdef CONFIG_PREEMPT_DYNAMIC
diff --git a/kernel/fork.c b/kernel/fork.c
index 416758c8a3d4..8077336bb136 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1871,6 +1871,7 @@ static inline void rcu_copy_process(struct task_struct *p)
p->rcu_tasks_holdout = false;
INIT_LIST_HEAD(&p->rcu_tasks_holdout_list);
p->rcu_tasks_idle_cpu = -1;
+ p->rcu_tasks_irq_ip = 0;
INIT_LIST_HEAD(&p->rcu_tasks_exit_list);
#endif /* #ifdef CONFIG_TASKS_RCU */
#ifdef CONFIG_TASKS_TRACE_RCU
diff --git a/kernel/rcu/Kconfig b/kernel/rcu/Kconfig
index 332df7a7a634..bbab14bc14c3 100644
--- a/kernel/rcu/Kconfig
+++ b/kernel/rcu/Kconfig
@@ -107,6 +107,28 @@ config TASKS_RCU
default NEED_TASKS_RCU && PREEMPTION
select IRQ_WORK
+config HAVE_RCU_TRAMPOLINE_READERS
+ bool
+ help
+ Select this if the architecture uses the generic irq entry code and
+ every trampoline whose lifetime Tasks RCU guards on it (ftrace
+ trampolines, kprobe out-of-line and optimized-probe slots, BPF
+ trampolines, out-of-line ftrace direct-call trampolines) enters a
+ Tasks Trace RCU read-side critical section before calling out of
+ the trampoline and leaves it before returning, and any core text
+ that runs on behalf of such a trampoline outside that reader is
+ reported by arch_rcu_tasks_trampoline_text(). The assembly readers
+ use the this_cpu_inc() form of SRCU-fast, hence !NEED_SRCU_NMI_SAFE.
+
+config TASKS_RCU_TRAMPOLINE_READERS
+ def_bool TASKS_RCU && HAVE_RCU_TRAMPOLINE_READERS && GENERIC_IRQ_ENTRY && !NEED_SRCU_NMI_SAFE
+ select TASKS_TRACE_RCU
+ help
+ Implement the Tasks RCU grace period as a per-CPU pass over
+ context switches and irq-exit reschedules outside trampoline text
+ plus a Tasks Trace RCU grace period, instead of waiting for every
+ task to voluntarily context switch. See kernel/rcu/tasks.h.
+
config FORCE_TASKS_RUDE_RCU
bool "Force selection of Tasks Rude RCU"
depends on RCU_EXPERT
diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h
index 627295396cd9..f03be742be48 100644
--- a/kernel/rcu/tasks.h
+++ b/kernel/rcu/tasks.h
@@ -152,7 +152,7 @@ static struct rcu_tasks rt_name = \
.kname = #rt_name, \
}
-#ifdef CONFIG_TASKS_RCU
+#if defined(CONFIG_TASKS_RCU) && !defined(CONFIG_TASKS_RCU_TRAMPOLINE_READERS)
/* Report delay of scan exiting tasklist in rcu_tasks_postscan(). */
static void tasks_rcu_exit_stall(struct timer_list *unused);
@@ -802,7 +802,7 @@ static void rcu_tasks_torture_stats_print_generic(struct rcu_tasks *rtp, char *t
#endif // #ifndef CONFIG_TINY_RCU
-#if defined(CONFIG_TASKS_RCU)
+#if defined(CONFIG_TASKS_RCU) && !defined(CONFIG_TASKS_RCU_TRAMPOLINE_READERS)
////////////////////////////////////////////////////////////////////////
//
@@ -897,10 +897,444 @@ static void rcu_tasks_wait_gp(struct rcu_tasks *rtp)
rtp->postgp_func(rtp);
}
-#endif /* #if defined(CONFIG_TASKS_RCU) */
+#endif /* #if defined(CONFIG_TASKS_RCU) && !defined(CONFIG_TASKS_RCU_TRAMPOLINE_READERS) */
#ifdef CONFIG_TASKS_RCU
+static int rcu_tasks_lazy_ms = -1;
+module_param(rcu_tasks_lazy_ms, int, 0444);
+
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+
+////////////////////////////////////////////////////////////////////////
+//
+// Tasks RCU for architectures whose trampolines are Tasks Trace RCU
+// readers (CONFIG_HAVE_RCU_TRAMPOLINE_READERS).
+//
+// On these architectures every piece of text whose lifetime Tasks RCU
+// guards -- ftrace trampolines, kprobe optinsn slots, BPF trampoline
+// images, out-of-line ftrace direct-call trampolines -- enters a Tasks
+// Trace RCU read-side critical section before calling out of itself and
+// leaves it before returning, so a task anywhere inside such a call-out,
+// preempted or not, is an ordinary rcu_read_lock_trace() reader and
+// synchronize_rcu_tasks_trace() waits for it.
+//
+// What that cannot cover is the handful of instructions in the trampoline
+// before the reader is entered and after it is left, and the one user that
+// has no trampoline at all: the bytes after a kprobe that the jump
+// optimizer is about to overwrite. A task can only linger in such
+// "unmarked" text by being interrupted there; unmarked text never calls
+// anything that could schedule. So a context switch on a CPU tells us that
+// whatever that CPU was running is out of unmarked text, with one
+// exception: a preemption from the irq-exit path, which can happen at any
+// instruction boundary. That path has the interrupted pt_regs in hand, so
+// just before it preempts it records the IP in the task and checks it
+// (rcu_tasks_trampoline_text()); if it is inside unmarked text the task
+// goes on a short holdout list first, and takes itself off again at its
+// next context switch outside such a preemption or its next irq-exit
+// check that finds it elsewhere. With that, every pass through
+// __schedule() is a per-CPU quiescent event, as are usermode and idle.
+//
+// A grace period is then:
+//
+// 1. Wait for every online CPU to context switch or be found in an RCU
+// extended quiescent state (deep idle, nohz_full userspace), nudging
+// stragglers with resched_cpu(). Afterwards no task is in the leading
+// unmarked instructions of a dying trampoline unless it is on the
+// holdout list.
+// 2. Wait for the holdout list (as it stood) to drain.
+// 3. synchronize_rcu_tasks_trace(), for everything inside the readers.
+// 4. Repeat 1 and 2 for tasks that have since left the reader and are in
+// the trailing unmarked instructions.
+//
+// which is bounded by a few jiffies plus preempt-off latency plus an SRCU
+// grace period, independent of how long any task runs without sleeping.
+// Unlike the classic implementation this does wait for an idle task caught
+// in a trampoline, since an idle CPU only counts while RCU is not watching
+// it.
+
+static void rcu_tasks_tramp_wait_gp(struct rcu_tasks *rtp);
+void call_rcu_tasks(struct rcu_head *rhp, rcu_callback_t func);
+DEFINE_RCU_TASKS(rcu_tasks, rcu_tasks_tramp_wait_gp, call_rcu_tasks, "RCU Tasks");
+
+/* Per-CPU count of Tasks RCU quiescent events, and the GP kthread's snapshot. */
+static DEFINE_PER_CPU(unsigned long, rcu_tasks_qs_seq);
+static DEFINE_PER_CPU(unsigned long, rcu_tasks_qs_snap);
+
+/*
+ * Tasks currently switched out by an irq-exit preemption are kept, with the
+ * interrupted IP, on the per-CPU rtp_exit_list of the CPU that preempted them
+ * (reusing the list, lock and task_struct fields the classic flavor uses for
+ * its exit-path bookkeeping, which this flavor does not need), so that
+ * rcu_tasks_wait_irq_preempted() can find them without a tasklist scan and
+ * regardless of where they are in exit.
+ */
+
+/* Tasks last seen preempted inside unmarked trampoline text. */
+static LIST_HEAD(rcu_tasks_tramp_holdouts);
+static DEFINE_RAW_SPINLOCK(rcu_tasks_tramp_lock);
+
+/* CPUs / holdouts the current grace period is still waiting for. */
+static struct cpumask rcu_tasks_pending_cpus;
+static LIST_HEAD(rcu_tasks_gp_holdouts);
+
+/**
+ * arch_rcu_tasks_trampoline_text - Does the architecture treat @ip as unmarked trampoline text?
+ * @ip: kernel text address inside core kernel text
+ *
+ * See rcu_tasks_trampoline_text(). Architectures override this to flag
+ * core text that runs on behalf of a trampoline outside its Tasks Trace
+ * reader, e.g. static ftrace entry stubs or return thunks that hold a
+ * trampoline address they are about to jump to.
+ */
+bool __weak arch_rcu_tasks_trampoline_text(unsigned long ip)
+{
+ return false;
+}
+
+/**
+ * rcu_tasks_trampoline_text - Is @ip in text Tasks RCU protects but no reader marks?
+ * @ip: an interrupted instruction pointer
+ *
+ * True when a task interrupted at @ip may be executing, or about to enter
+ * or return into, text whose lifetime depends on synchronize_rcu_tasks()
+ * without being inside the Tasks Trace reader that text takes around its
+ * call-outs:
+ *
+ * - anything outside core kernel and module text (ftrace and BPF
+ * trampolines, kprobe slots and other dynamically allocated text; this
+ * deliberately does not ask is_ftrace_trampoline() and friends, since
+ * text being torn down may already be unregistered there);
+ * - whatever the architecture adds via arch_rcu_tasks_trampoline_text().
+ *
+ * A false positive only makes the task a holdout until its next quiescent
+ * event. Called with interrupts disabled from the irq-exit path.
+ */
+bool rcu_tasks_trampoline_text(unsigned long ip)
+{
+ if (core_kernel_text(ip))
+ return arch_rcu_tasks_trampoline_text(ip);
+ return !is_module_text_address(ip);
+}
+NOKPROBE_SYMBOL(rcu_tasks_trampoline_text);
+
+/* Note a Tasks RCU quiescent event on this CPU. */
+static void rcu_tasks_qs_event(void)
+{
+ unsigned long *seq;
+
+ guard(preempt_notrace)();
+ seq = this_cpu_ptr(&rcu_tasks_qs_seq);
+ /* Order a preceding rcu_tasks_tramp_hold() before the count. */
+ smp_store_release(seq, *seq + 1);
+}
+
+static void rcu_tasks_tramp_hold(struct task_struct *t)
+{
+ unsigned long flags;
+
+ if (t->rcu_tasks_holdout)
+ return;
+ raw_spin_lock_irqsave(&rcu_tasks_tramp_lock, flags);
+ list_add_tail(&t->rcu_tasks_holdout_list, &rcu_tasks_tramp_holdouts);
+ WRITE_ONCE(t->rcu_tasks_holdout, true);
+ raw_spin_unlock_irqrestore(&rcu_tasks_tramp_lock, flags);
+}
+
+static void rcu_tasks_tramp_release(struct task_struct *t)
+{
+ unsigned long flags;
+
+ if (likely(!t->rcu_tasks_holdout))
+ return;
+ raw_spin_lock_irqsave(&rcu_tasks_tramp_lock, flags);
+ list_del_init(&t->rcu_tasks_holdout_list);
+ WRITE_ONCE(t->rcu_tasks_holdout, false);
+ raw_spin_unlock_irqrestore(&rcu_tasks_tramp_lock, flags);
+}
+
+/**
+ * rcu_tasks_irq_resched_enter - Tasks RCU hook for the irq-exit reschedule check
+ * @ip: instruction pointer of the interrupted (task-level) context
+ *
+ * Called with interrupts disabled when an interrupt returning to kernel
+ * mode is about to preempt_schedule_irq(), the one context switch that can
+ * catch a task inside unmarked trampoline text. Record where the task is
+ * parked for as long as it is (rcu_tasks_wait_irq_preempted() looks at
+ * that), and if it is inside such text make it a holdout before
+ * __schedule() reports the quiescent event; if it is not, this is as good
+ * as a voluntary switch for ending an earlier hold.
+ */
+void rcu_tasks_irq_resched_enter(unsigned long ip)
+{
+ struct task_struct *t = current;
+ struct rcu_tasks_percpu *rtpcp = this_cpu_ptr(rcu_tasks.rtpcpu);
+
+ lockdep_assert_irqs_disabled();
+ WRITE_ONCE(t->rcu_tasks_irq_ip, ip);
+ t->rcu_tasks_exit_cpu = smp_processor_id();
+ raw_spin_lock_rcu_node(rtpcp);
+ list_add(&t->rcu_tasks_exit_list, &rtpcp->rtp_exit_list);
+ raw_spin_unlock_rcu_node(rtpcp);
+
+ if (unlikely(rcu_tasks_trampoline_text(ip)))
+ rcu_tasks_tramp_hold(t);
+ else
+ rcu_tasks_tramp_release(t);
+}
+NOKPROBE_SYMBOL(rcu_tasks_irq_resched_enter);
+
+/**
+ * rcu_tasks_irq_resched_exit - preempt_schedule_irq() has returned
+ *
+ * The task is running again (possibly elsewhere) and about to return to the
+ * interrupted context; it is no longer parked anywhere.
+ */
+void rcu_tasks_irq_resched_exit(void)
+{
+ struct task_struct *t = current;
+ struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rcu_tasks.rtpcpu, t->rcu_tasks_exit_cpu);
+
+ lockdep_assert_irqs_disabled();
+ raw_spin_lock_rcu_node(rtpcp);
+ list_del_init(&t->rcu_tasks_exit_list);
+ raw_spin_unlock_rcu_node(rtpcp);
+ WRITE_ONCE(t->rcu_tasks_irq_ip, 0);
+}
+NOKPROBE_SYMBOL(rcu_tasks_irq_resched_exit);
+
+/**
+ * rcu_tasks_note_qs - Tasks RCU hook for a context switch or explicit QS
+ * @t: current
+ * @preempt: this is a preemption rather than a voluntary switch
+ *
+ * Every pass through __schedule() (and cond_resched_tasks_rcu_qs(), and a
+ * tick from userspace or idle) is a quiescent event for this CPU: unmarked
+ * trampoline text never calls anything that schedules, and the irq-exit
+ * path has already made @t a holdout if it is preempting inside such text.
+ * Any of these outside an irq-exit preemption also shows @t itself to be
+ * outside, ending an earlier hold -- including cond_resched() under
+ * PREEMPT_DYNAMIC's none/voluntary modes, where the irq-exit path is off.
+ */
+void rcu_tasks_note_qs(struct task_struct *t, bool preempt)
+{
+ WARN_ON_ONCE(t != current);
+ if (!READ_ONCE(t->rcu_tasks_irq_ip))
+ rcu_tasks_tramp_release(t);
+ rcu_tasks_qs_event();
+}
+EXPORT_SYMBOL_GPL(rcu_tasks_note_qs); /* cond_resched_tasks_rcu_qs() */
+
+/**
+ * rcu_tasks_wait_irq_preempted - wait for tasks irq-preempted inside @inside
+ * @inside: predicate on a task's recorded irq-exit preemption IP
+ *
+ * For a caller about to make some ordinary text unsafe to be parked in
+ * (the kprobe jump optimizer): once the caller has arranged for
+ * rcu_tasks_trampoline_text() to cover that text, new irq-exit preemptions
+ * there become holdouts, but a task preempted there earlier is invisible
+ * to the grace period. Wait until no parked task's recorded preemption IP
+ * is inside; a following synchronize_rcu_tasks() then covers the rest.
+ * The leading synchronize_rcu() orders the caller's arrangement against
+ * preemptions in flight, which run with interrupts disabled.
+ */
+void rcu_tasks_wait_irq_preempted(bool (*inside)(unsigned long ip))
+{
+ struct task_struct *t;
+ unsigned long flags;
+ int cpu, kick;
+ bool found;
+
+ synchronize_rcu();
+ for (;;) {
+ found = false;
+ for_each_possible_cpu(cpu) {
+ struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rcu_tasks.rtpcpu, cpu);
+
+ kick = -1;
+ raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
+ list_for_each_entry(t, &rtpcp->rtp_exit_list, rcu_tasks_exit_list) {
+ if (inside(READ_ONCE(t->rcu_tasks_irq_ip))) {
+ found = true;
+ if (task_curr(t))
+ kick = task_cpu(t);
+ }
+ }
+ raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
+ if (kick >= 0)
+ resched_cpu(kick);
+ }
+ if (!found)
+ return;
+ schedule_timeout_uninterruptible(1);
+ }
+}
+
+/* Has @cpu passed a quiescent event since the snapshot, or need it not? */
+static bool rcu_tasks_cpu_quiescent(int cpu)
+{
+ if (!cpu_online(cpu))
+ return true;
+ /* Pairs with the release in rcu_tasks_qs_event(). */
+ if (smp_load_acquire(per_cpu_ptr(&rcu_tasks_qs_seq, cpu)) !=
+ per_cpu(rcu_tasks_qs_snap, cpu))
+ return true;
+ /*
+ * Idle or nohz_full userspace in an RCU extended quiescent state: no
+ * task-level kernel frames can be live in a trampoline there, and
+ * whatever ran before has switched out. An idle CPU that RCU is
+ * watching (an interrupt from idle, or the traceable part of the idle
+ * loop) is deliberately not let through: the idle task may be in a
+ * trampoline with that interrupt on top, and since it is never
+ * preempted from irq exit nothing else would catch it. It gets the
+ * resched_cpu() like anyone else and counts once the idle loop itself
+ * schedules, which it cannot do from inside a trampoline.
+ */
+ return !(ct_rcu_watching_cpu(cpu) & CT_RCU_WATCHING);
+}
+
+/* Rate-limited stall report; returns true if the caller should add detail. */
+static bool rcu_tasks_tramp_stall(struct rcu_tasks *rtp, unsigned long *lastreport,
+ const char *what)
+{
+ int rtst = READ_ONCE(rcu_task_stall_timeout);
+
+ if (rtst <= 0 || !time_after(jiffies, *lastreport + rtst))
+ return false;
+ *lastreport = jiffies;
+ pr_err("INFO: %s: %s, grace period %lu is %lu jiffies old\n", rtp->kname,
+ what, rcu_seq_current(&rtp->tasks_gp_seq), jiffies - rtp->gp_start);
+ return true;
+}
+
+/*
+ * Steps 1/4: wait until every online CPU has context switched or is in an
+ * RCU extended quiescent state. A CPU that has neither after a jiffy is
+ * asked to switch with resched_cpu(), which takes it through
+ * rcu_tasks_irq_resched_enter() and __schedule() (or, from userspace, a
+ * guest or the idle loop, straight to __schedule()).
+ */
+static void rcu_tasks_tramp_wait_cpus(struct rcu_tasks *rtp, unsigned long *lastreport)
+{
+ struct cpumask *pending = &rcu_tasks_pending_cpus;
+ unsigned long start;
+ int cpu;
+
+ /*
+ * The quiescent events run with preemption (in practice interrupts)
+ * disabled, so after this any event we go on to count began after the
+ * caller's updates -- the unpublished trampoline, and whatever
+ * rcu_tasks_trampoline_text() consults -- were visible to it.
+ */
+ synchronize_rcu();
+
+ start = jiffies;
+ for_each_online_cpu(cpu) {
+ per_cpu(rcu_tasks_qs_snap, cpu) = READ_ONCE(per_cpu(rcu_tasks_qs_seq, cpu));
+ __cpumask_set_cpu(cpu, pending);
+ }
+ /* Snapshots before the checks below; pairs with rcu_tasks_qs_event(). */
+ smp_mb();
+
+ for (;;) {
+ for_each_cpu(cpu, pending)
+ if (rcu_tasks_cpu_quiescent(cpu))
+ __cpumask_clear_cpu(cpu, pending);
+ if (cpumask_empty(pending))
+ break;
+ if (time_after(jiffies, start)) {
+ for_each_cpu(cpu, pending)
+ resched_cpu(cpu);
+ rtp->n_ipis += cpumask_weight(pending);
+ }
+ schedule_timeout_idle(1);
+ if (rcu_tasks_tramp_stall(rtp, lastreport, "CPUs without a quiescent event"))
+ pr_err("\tCPUs: %*pbl\n", cpumask_pr_args(pending));
+ }
+}
+
+/*
+ * Steps 2/4: wait for the tasks that were holdouts when we looked to stop
+ * being holdouts. They are moved to a private list so that tasks becoming
+ * holdouts later (in live trampolines) cannot keep us here; each removes
+ * itself via rcu_tasks_tramp_release() wherever it is queued.
+ */
+static void rcu_tasks_tramp_wait_holdouts(struct rcu_tasks *rtp, unsigned long *lastreport)
+{
+ struct task_struct *t;
+ unsigned long flags;
+ int cpu;
+
+ raw_spin_lock_irqsave(&rcu_tasks_tramp_lock, flags);
+ list_splice_tail_init(&rcu_tasks_tramp_holdouts, &rcu_tasks_gp_holdouts);
+ raw_spin_unlock_irqrestore(&rcu_tasks_tramp_lock, flags);
+
+ for (;;) {
+ struct cpumask *kick = &rcu_tasks_pending_cpus;
+ struct task_struct *show[8];
+ int nshow = 0, i;
+ bool empty, report;
+
+ report = rcu_tasks_tramp_stall(rtp, lastreport,
+ "tasks preempted in trampoline text");
+ cpumask_clear(kick);
+ raw_spin_lock_irqsave(&rcu_tasks_tramp_lock, flags);
+ empty = list_empty(&rcu_tasks_gp_holdouts);
+ list_for_each_entry(t, &rcu_tasks_gp_holdouts, rcu_tasks_holdout_list) {
+ if (task_curr(t))
+ __cpumask_set_cpu(task_cpu(t), kick);
+ if (report && nshow < ARRAY_SIZE(show))
+ show[nshow++] = get_task_struct(t);
+ }
+ raw_spin_unlock_irqrestore(&rcu_tasks_tramp_lock, flags);
+ /* Never printk under the lock the irq-exit path takes. */
+ for (i = 0; i < nshow; i++) {
+ sched_show_task(show[i]);
+ put_task_struct(show[i]);
+ }
+ if (empty)
+ break;
+ for_each_cpu(cpu, kick)
+ resched_cpu(cpu);
+ rtp->n_ipis += cpumask_weight(kick);
+ schedule_timeout_idle(1);
+ }
+}
+
+/* Wait for one trampoline-reader Tasks RCU grace period. */
+static void rcu_tasks_tramp_wait_gp(struct rcu_tasks *rtp)
+{
+ unsigned long lastreport = jiffies;
+
+ set_tasks_gp_state(rtp, RTGS_WAIT_SCAN_HOLDOUTS);
+ rcu_tasks_tramp_wait_cpus(rtp, &lastreport);
+ rcu_tasks_tramp_wait_holdouts(rtp, &lastreport);
+
+ set_tasks_gp_state(rtp, RTGS_WAIT_READERS);
+ synchronize_rcu_tasks_trace();
+
+ set_tasks_gp_state(rtp, RTGS_SCAN_HOLDOUTS);
+ rcu_tasks_tramp_wait_cpus(rtp, &lastreport);
+ rcu_tasks_tramp_wait_holdouts(rtp, &lastreport);
+
+ set_tasks_gp_state(rtp, RTGS_POST_GP);
+}
+
+static int __init rcu_spawn_tasks_kthread(void)
+{
+ rcu_tasks.gp_sleep = HZ / 10;
+ if (rcu_tasks_lazy_ms >= 0)
+ rcu_tasks.lazy_jiffies = msecs_to_jiffies(rcu_tasks_lazy_ms);
+ rcu_tasks.wait_state = TASK_IDLE;
+ rcu_spawn_tasks_kthread_generic(&rcu_tasks);
+ return 0;
+}
+
+void exit_tasks_rcu_start(void) { }
+void exit_tasks_rcu_finish(void) { }
+
+#else /* #ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS */
+
////////////////////////////////////////////////////////////////////////
//
// Simple variant of RCU whose quiescent states are voluntary context
@@ -1173,6 +1607,8 @@ static void tasks_rcu_exit_stall(struct timer_list *unused)
#endif // #ifndef CONFIG_TINY_RCU
}
+#endif /* #else #ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS */
+
/**
* call_rcu_tasks() - Queue an RCU for invocation task-based grace period
* @rhp: structure to be used for queueing the RCU updates.
@@ -1187,6 +1623,12 @@ static void tasks_rcu_exit_stall(struct timer_list *unused)
* primitives analogous to rcu_read_lock() and rcu_read_unlock() because
* this primitive is intended to determine that all tasks have passed
* through a safe state, not so much for data-structure synchronization.
+ * On CONFIG_TASKS_RCU_TRAMPOLINE_READERS kernels a preemption outside
+ * trampoline text also ends one, and a reader whose protected window
+ * spans preemptible code must additionally be a Tasks Trace RCU reader
+ * (rcu_read_lock_trace(), as the trampolines there take around their
+ * call-outs); an arbitrary stretch of preemptible kernel code is not
+ * protected.
*
* See the description of call_rcu() for more detailed information on
* memory ordering guarantees.
@@ -1205,7 +1647,9 @@ EXPORT_SYMBOL_GPL(call_rcu_tasks);
* executing rcu-tasks read-side critical sections have elapsed. These
* read-side critical sections are delimited by calls to schedule(),
* cond_resched_tasks_rcu_qs(), idle execution, userspace execution, calls
- * to synchronize_rcu_tasks(), and (in theory, anyway) cond_resched().
+ * to synchronize_rcu_tasks(), and (in theory, anyway) cond_resched();
+ * on CONFIG_TASKS_RCU_TRAMPOLINE_READERS kernels also by preemption
+ * outside trampoline text, see call_rcu_tasks().
*
* This is a very specialized primitive, intended only for a few uses in
* tracing and other situations requiring manipulation of function
@@ -1233,9 +1677,7 @@ void rcu_barrier_tasks(void)
}
EXPORT_SYMBOL_GPL(rcu_barrier_tasks);
-static int rcu_tasks_lazy_ms = -1;
-module_param(rcu_tasks_lazy_ms, int, 0444);
-
+#ifndef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
static int __init rcu_spawn_tasks_kthread(void)
{
rcu_tasks.gp_sleep = HZ / 10;
@@ -1251,6 +1693,7 @@ static int __init rcu_spawn_tasks_kthread(void)
rcu_spawn_tasks_kthread_generic(&rcu_tasks);
return 0;
}
+#endif /* #ifndef CONFIG_TASKS_RCU_TRAMPOLINE_READERS */
#if !defined(CONFIG_TINY_RCU)
void show_rcu_tasks_classic_gp_kthread(void)
@@ -1279,6 +1722,7 @@ void rcu_tasks_get_gp_data(int *flags, unsigned long *gp_seq)
}
EXPORT_SYMBOL_GPL(rcu_tasks_get_gp_data);
+#ifndef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
/*
* Protect against tasklist scan blind spot while the task is exiting and
* may be removed from the tasklist. Do this by adding the task to yet
@@ -1322,6 +1766,7 @@ void exit_tasks_rcu_finish(void)
list_del_init(&t->rcu_tasks_exit_list);
raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
}
+#endif /* #ifndef CONFIG_TASKS_RCU_TRAMPOLINE_READERS */
#else /* #ifdef CONFIG_TASKS_RCU */
void exit_tasks_rcu_start(void) { }
diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c
index b62735a67884..a122b8d1effb 100644
--- a/kernel/rcu/update.c
+++ b/kernel/rcu/update.c
@@ -40,7 +40,9 @@
#include <linux/tick.h>
#include <linux/rcupdate_wait.h>
#include <linux/sched/isolation.h>
+#include <linux/context_tracking_state.h>
#include <linux/kprobes.h>
+#include <linux/module.h>
#include <linux/slab.h>
#include <linux/irq_work.h>
#include <linux/rcupdate_trace.h>
--
2.55.0
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH RFC v4 03/13] kprobes: Expose the optprobe jump window to Tasks RCU
2026-09-18 14:49 [PATCH RFC v4 00/13] rcu-tasks: build Tasks RCU on Tasks Trace readers in trampolines Josef Bacik
2026-09-18 14:49 ` [PATCH RFC v4 01/13] entry: Pass pt_regs to irqentry_exit_cond_resched() Josef Bacik
2026-09-18 14:49 ` [PATCH RFC v4 02/13] rcu-tasks: Add a Tasks RCU implementation for reader-marked trampolines Josef Bacik
@ 2026-09-18 14:49 ` Josef Bacik
2026-09-18 14:49 ` [PATCH RFC v4 04/13] ftrace: Mark modules hosting direct-call trampolines for " Josef Bacik
` (9 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Josef Bacik @ 2026-09-18 14:49 UTC (permalink / raw)
To: Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai,
Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai
Cc: Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Josef Bacik
kprobe_optimizer() is the one synchronize_rcu_tasks() user that is not
about trampoline text: it waits for tasks that were interrupted on an
instruction boundary inside the bytes it is about to overwrite with the
optimized jump, so that none of them resumes into the middle of the new
instruction. Those bytes are ordinary kernel or module text with no
Tasks Trace reader around them, so on CONFIG_TASKS_RCU_TRAMPOLINE_READERS
kernels the irq-exit quiescent-state check has to be told about them.
Add kprobe_in_optimized_region(), a lockless and conservative form of
get_optimized_kprobe() that reports whether any registered kprobe lies
within MAX_OPTIMIZED_LENGTH before the given address regardless of its
optimization state, and have rcu_tasks_trampoline_text() consult it for
core and module text so that a task interrupted there becomes a holdout
rather than a quiescent event. The hash walk only runs while
kprobe_optimizer() is actually inside its synchronize_rcu_tasks(),
tracked by a flag it sets around the call; otherwise the check is a
single load. That check cannot see a task that was already preempted in
the region before the flag went up (possibly before the kprobe even
existed), and the new grace period does not otherwise wait for a
preempted task to run again, so before synchronize_rcu_tasks() the
optimizer calls rcu_tasks_wait_irq_preempted() to wait until no parked
task's recorded irq-exit preemption IP is inside such a region; its
leading synchronize_rcu() also publishes the flag to every (interrupts-
disabled) check in flight. The kprobe hash is RCU-protected and every
free path waits for a grace period after unhashing, so the lockless walk
from the irq-exit path is safe.
On other configurations the flag is set and cleared but nothing reads
it and rcu_tasks_wait_irq_preempted() is a stub; the classic
implementation already waits for such tasks.
Assisted-by: LLM
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
---
include/linux/kprobes.h | 8 +++++++-
kernel/kprobes.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++++
kernel/rcu/tasks.h | 11 ++++++++---
3 files changed, 65 insertions(+), 4 deletions(-)
diff --git a/include/linux/kprobes.h b/include/linux/kprobes.h
index e6de7ae55bda..74cc48c04417 100644
--- a/include/linux/kprobes.h
+++ b/include/linux/kprobes.h
@@ -530,11 +530,17 @@ static inline bool is_kprobe_insn_slot(unsigned long addr)
}
#endif /* !CONFIG_KPROBES */
-#ifndef CONFIG_OPTPROBES
+#ifdef CONFIG_OPTPROBES
+bool kprobe_in_optimized_region(unsigned long addr);
+#else /* !CONFIG_OPTPROBES */
static inline bool is_kprobe_optinsn_slot(unsigned long addr)
{
return false;
}
+static inline bool kprobe_in_optimized_region(unsigned long addr)
+{
+ return false;
+}
#endif /* !CONFIG_OPTPROBES */
#ifdef CONFIG_KRETPROBES
diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index 6337da5cab9e..e460fba83e4a 100644
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -511,6 +511,48 @@ static struct kprobe *get_optimized_kprobe(kprobe_opcode_t *addr)
return NULL;
}
+/*
+ * True while kprobe_optimizer() is waiting for its Tasks RCU grace period.
+ * Only in that window can an interruption inside an optprobe's jump region
+ * matter to it, so kprobe_in_optimized_region() does no work otherwise.
+ */
+static bool kprobe_optimizer_waiting;
+
+/**
+ * kprobe_in_optimized_region - Could @addr be inside bytes a jump-optimized
+ * kprobe replaces?
+ * @addr: kernel text address, typically an interrupted instruction pointer
+ *
+ * kprobe_optimizer() relies on synchronize_rcu_tasks() to wait for tasks that
+ * were interrupted on an instruction boundary inside the region about to be
+ * overwritten by the optimized jump. Where Tasks RCU is built on
+ * reader-marked trampolines that region has no reader, so the irq-exit
+ * quiescent-state check asks this instead (see rcu_tasks_trampoline_text()).
+ * This is the lockless, conservative form of get_optimized_kprobe(): it does
+ * not care whether the kprobe found is, or ever will be, optimized. May be
+ * called from any context with preemption disabled; the kprobe hash is
+ * RCU-protected and every free path waits for a grace period after unhashing.
+ *
+ * The hash walk only runs while the optimizer is actually waiting. A task
+ * that was preempted in such a region before the flag went up is invisible
+ * to that check, so the optimizer first waits those out by their recorded
+ * preemption IP (rcu_tasks_wait_irq_preempted(), whose leading
+ * synchronize_rcu() also publishes the flag to every check in flight).
+ */
+bool kprobe_in_optimized_region(unsigned long addr)
+{
+ int i;
+
+ if (!READ_ONCE(kprobe_optimizer_waiting))
+ return false;
+
+ for (i = 1; i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++)
+ if (get_kprobe((kprobe_opcode_t *)addr - i))
+ return true;
+ return false;
+}
+NOKPROBE_SYMBOL(kprobe_in_optimized_region);
+
/* Optimization staging list, protected by 'kprobe_mutex' */
static LIST_HEAD(optimizing_list);
static LIST_HEAD(unoptimizing_list);
@@ -644,8 +686,16 @@ static void kprobe_optimizer(void)
* to 2nd-Nth byte of jump instruction. This wait is for avoiding it.
* Note that on non-preemptive kernel, this is transparently converted
* to synchronoze_sched() to wait for all interrupts to have completed.
+ * kprobe_optimizer_waiting lets a reader-marked-trampoline Tasks RCU
+ * recognise tasks interrupted in such a region while we wait, and
+ * rcu_tasks_wait_irq_preempted() (a no-op elsewhere) first waits
+ * out any that were preempted there before we said so; see
+ * kprobe_in_optimized_region().
*/
+ WRITE_ONCE(kprobe_optimizer_waiting, true);
+ rcu_tasks_wait_irq_preempted(kprobe_in_optimized_region);
synchronize_rcu_tasks();
+ WRITE_ONCE(kprobe_optimizer_waiting, false);
/* Step 3: Optimize kprobes after quiesence period */
do_optimize_kprobes();
diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h
index f03be742be48..eb1388dd8a61 100644
--- a/kernel/rcu/tasks.h
+++ b/kernel/rcu/tasks.h
@@ -1005,7 +1005,9 @@ bool __weak arch_rcu_tasks_trampoline_text(unsigned long ip)
* trampolines, kprobe slots and other dynamically allocated text; this
* deliberately does not ask is_ftrace_trampoline() and friends, since
* text being torn down may already be unregistered there);
- * - whatever the architecture adds via arch_rcu_tasks_trampoline_text().
+ * - whatever the architecture adds via arch_rcu_tasks_trampoline_text();
+ * - the bytes after a kprobe that a pending jump optimization is about to
+ * overwrite, the one synchronize_rcu_tasks() user with no trampoline.
*
* A false positive only makes the task a holdout until its next quiescent
* event. Called with interrupts disabled from the irq-exit path.
@@ -1013,8 +1015,11 @@ bool __weak arch_rcu_tasks_trampoline_text(unsigned long ip)
bool rcu_tasks_trampoline_text(unsigned long ip)
{
if (core_kernel_text(ip))
- return arch_rcu_tasks_trampoline_text(ip);
- return !is_module_text_address(ip);
+ return arch_rcu_tasks_trampoline_text(ip) ||
+ kprobe_in_optimized_region(ip);
+ if (is_module_text_address(ip))
+ return kprobe_in_optimized_region(ip);
+ return true;
}
NOKPROBE_SYMBOL(rcu_tasks_trampoline_text);
--
2.55.0
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH RFC v4 04/13] ftrace: Mark modules hosting direct-call trampolines for Tasks RCU
2026-09-18 14:49 [PATCH RFC v4 00/13] rcu-tasks: build Tasks RCU on Tasks Trace readers in trampolines Josef Bacik
` (2 preceding siblings ...)
2026-09-18 14:49 ` [PATCH RFC v4 03/13] kprobes: Expose the optprobe jump window to Tasks RCU Josef Bacik
@ 2026-09-18 14:49 ` Josef Bacik
2026-09-18 14:49 ` [PATCH RFC v4 05/13] x86/ftrace: Take a Tasks Trace reader around ftrace_caller's call-out Josef Bacik
` (8 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Josef Bacik @ 2026-09-18 14:49 UTC (permalink / raw)
To: Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai,
Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai
Cc: Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Josef Bacik
An out-of-line direct trampoline registered with register_ftrace_direct()
is kept alive only by Tasks RCU while a task executes it or is preempted
in something it called; ftrace_shutdown()'s synchronize_rcu_tasks() is
what stops rmmod freeing it under such a task. Where Tasks RCU is built
on reader-marked trampolines, such a trampoline must be a Tasks Trace
reader across its call-out like the ftrace and BPF trampolines are, so
document that in register_ftrace_direct().
That still leaves the few instructions before the reader is entered and
after it is left. For BPF images those are in dynamically allocated
text that rcu_tasks_trampoline_text() already treats as unmarked
trampoline text, but the in-tree samples (and any similar user) place
their trampolines in module .text. Add a sticky
module::ftrace_direct_tramp flag (under CONFIG_TASKS_RCU_TRAMPOLINE_READERS,
its only consumer), set by every register/modify path when the direct
address is module text, and have rcu_tasks_trampoline_text() treat a task
interrupted anywhere in such a module as a potential holdout. Other
modules' text is unaffected.
Assisted-by: LLM
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
---
include/linux/module.h | 7 +++++++
kernel/rcu/tasks.h | 18 +++++++++++++++---
kernel/trace/ftrace.c | 39 +++++++++++++++++++++++++++++++++++++++
3 files changed, 61 insertions(+), 3 deletions(-)
diff --git a/include/linux/module.h b/include/linux/module.h
index 96cc98568eea..82ca4f774725 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -521,6 +521,13 @@ struct module {
unsigned int num_ftrace_callsites;
unsigned long *ftrace_callsites;
#endif
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+ /*
+ * An ftrace direct-call trampoline lives in this module's text; see
+ * rcu_tasks_trampoline_text(). Sticky once set.
+ */
+ bool ftrace_direct_tramp;
+#endif
#ifdef CONFIG_KPROBES
void *kprobes_text_start;
unsigned int kprobes_text_size;
diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h
index eb1388dd8a61..42ea6e0e61cb 100644
--- a/kernel/rcu/tasks.h
+++ b/kernel/rcu/tasks.h
@@ -1006,6 +1006,8 @@ bool __weak arch_rcu_tasks_trampoline_text(unsigned long ip)
* deliberately does not ask is_ftrace_trampoline() and friends, since
* text being torn down may already be unregistered there);
* - whatever the architecture adds via arch_rcu_tasks_trampoline_text();
+ * - the text of a module that hosts an out-of-line ftrace direct-call
+ * trampoline (see ftrace_direct_mark_module());
* - the bytes after a kprobe that a pending jump optimization is about to
* overwrite, the one synchronize_rcu_tasks() user with no trampoline.
*
@@ -1014,12 +1016,22 @@ bool __weak arch_rcu_tasks_trampoline_text(unsigned long ip)
*/
bool rcu_tasks_trampoline_text(unsigned long ip)
{
+ bool ret = true;
+
if (core_kernel_text(ip))
return arch_rcu_tasks_trampoline_text(ip) ||
kprobe_in_optimized_region(ip);
- if (is_module_text_address(ip))
- return kprobe_in_optimized_region(ip);
- return true;
+
+#ifdef CONFIG_MODULES
+ scoped_guard(rcu) {
+ struct module *mod = __module_text_address(ip);
+
+ if (mod)
+ ret = READ_ONCE(mod->ftrace_direct_tramp) ||
+ kprobe_in_optimized_region(ip);
+ }
+#endif
+ return ret;
}
NOKPROBE_SYMBOL(rcu_tasks_trampoline_text);
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 53d5db60bfa5..f69f71591358 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -6076,6 +6076,29 @@ static void reset_direct(struct ftrace_ops *ops, unsigned long addr)
ops->trampoline = 0;
}
+/*
+ * A direct trampoline may live in module text rather than in dynamically
+ * allocated text that rcu_tasks_trampoline_text() recognises on its own (see
+ * samples/ftrace/ftrace-direct*.c). The trampoline itself must be a Tasks
+ * Trace reader across its call-out (see register_ftrace_direct()); marking the
+ * owning module here covers the instructions before it enters that reader and
+ * after it leaves it, where a task interrupted in the module's text must not be
+ * counted as Tasks-RCU quiescent, so that ftrace_shutdown()'s
+ * synchronize_rcu_tasks() still keeps the module text from being freed under
+ * it.
+ */
+static void ftrace_direct_mark_module(unsigned long addr)
+{
+#if defined(CONFIG_MODULES) && defined(CONFIG_TASKS_RCU_TRAMPOLINE_READERS)
+ struct module *mod;
+
+ guard(rcu)();
+ mod = __module_text_address(addr);
+ if (mod)
+ WRITE_ONCE(mod->ftrace_direct_tramp, true);
+#endif
+}
+
/**
* register_ftrace_direct - Call a custom trampoline directly
* for multiple functions registered in @ops
@@ -6090,6 +6113,17 @@ static void reset_direct(struct ftrace_ops *ops, unsigned long addr)
* and save the parameters of the function being traced, and restore them
* (or inject new ones if needed), before returning.
*
+ * Nothing but Tasks RCU keeps the trampoline at @addr alive while a task is
+ * executing it or is preempted in something it called. On architectures that
+ * select HAVE_RCU_TRAMPOLINE_READERS, Tasks RCU only waits for such a task if
+ * it is a Tasks Trace RCU reader, so the trampoline must enter one
+ * (rcu_read_lock_trace() or an assembly equivalent) before calling out and
+ * leave it before returning, just as that option requires of the in-kernel
+ * ftrace and BPF trampolines. The few instructions before and after are
+ * covered by the irq-exit check: automatically for trampolines outside kernel
+ * and module text (e.g. BPF images), and via ftrace_direct_mark_module() for
+ * trampolines in module text.
+ *
* Returns:
* 0 on success
* -EINVAL - The @ops object was already registered with this call or
@@ -6169,6 +6203,7 @@ int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
ops->flags |= MULTI_FLAGS;
ops->trampoline = FTRACE_REGS_ADDR;
ops->direct_call = addr;
+ ftrace_direct_mark_module(addr);
err = register_ftrace_function_nolock(ops);
if (err)
@@ -6237,6 +6272,8 @@ __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
lockdep_assert_held_once(&direct_mutex);
+ ftrace_direct_mark_module(addr);
+
/* Enable the tmp_ops to have the same functions as the direct ops */
ftrace_ops_init(&tmp_ops);
tmp_ops.func_hash = ops->func_hash;
@@ -6419,6 +6456,7 @@ int update_ftrace_direct_add(struct ftrace_ops *ops, struct ftrace_hash *hash)
hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
if (__ftrace_lookup_ip(direct_functions, entry->ip))
goto out_unlock;
+ ftrace_direct_mark_module(entry->direct);
}
}
@@ -6702,6 +6740,7 @@ int update_ftrace_direct_mod(struct ftrace_ops *ops, struct ftrace_hash *hash, b
tmp = __ftrace_lookup_ip(direct_hash, entry->ip);
if (!tmp)
continue;
+ ftrace_direct_mark_module(entry->direct);
tmp->direct = entry->direct;
}
}
--
2.55.0
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH RFC v4 05/13] x86/ftrace: Take a Tasks Trace reader around ftrace_caller's call-out
2026-09-18 14:49 [PATCH RFC v4 00/13] rcu-tasks: build Tasks RCU on Tasks Trace readers in trampolines Josef Bacik
` (3 preceding siblings ...)
2026-09-18 14:49 ` [PATCH RFC v4 04/13] ftrace: Mark modules hosting direct-call trampolines for " Josef Bacik
@ 2026-09-18 14:49 ` Josef Bacik
2026-09-18 14:49 ` [PATCH RFC v4 06/13] x86/kprobes: Take a Tasks Trace reader in the optprobe template Josef Bacik
` (7 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Josef Bacik @ 2026-09-18 14:49 UTC (permalink / raw)
To: Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai,
Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai
Cc: Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Josef Bacik
For HAVE_RCU_TRAMPOLINE_READERS the ftrace trampolines must be Tasks
Trace RCU readers while they call out, since that -- and not the absence
of a voluntary context switch -- is what synchronize_rcu_tasks() will
wait for before ftrace_shutdown() frees a dynamic trampoline or its ops.
Open-code rcu_read_lock_trace() and rcu_read_unlock_trace() in
ftrace_caller and ftrace_regs_caller: bump current->trc_reader_nesting
and, for the outermost reader, do the SRCU-fast per-CPU increment on
rcu_tasks_trace_srcu_struct and stash the counter pointer in
current->trc_reader_scp, exactly as the C inlines do (including the
smp_mb() when CONFIG_TASKS_TRACE_RCU_NO_MB is not set). The lock sits
before the function_trace_op load, because between that load and the
call the ops pointer is protected only by Tasks RCU, and the unlock
after the call returns. The sequences are inside the region that
create_trampoline() copies for per-ops trampolines; their %rip-relative
references are fixed up by text_poke_apply_relocation() like
CALL_DEPTH_ACCOUNT's. %rax and %rcx are dead at both points.
Two pieces of core text still run outside that reader while holding
the address of a Tasks-RCU-protected trampoline they are about to
enter: the static stubs themselves, whose direct-call tails keep a BPF
trampoline address on the stack until the final RET, and, under
CONFIG_MITIGATION_RETHUNK, the return thunk that RET expands to. Add an
ftrace_static_tramp_end marker after ftrace_stub_direct_tramp and linker
symbols around .text..__x86.return_thunk and .text..__x86.rethunk_safe,
and provide arch_rcu_tasks_trampoline_text() covering
[ftrace_caller, ftrace_static_tramp_end) and both thunk ranges so the
irq-exit check treats a task interrupted there as a holdout.
All of this is built only under CONFIG_TASKS_RCU_TRAMPOLINE_READERS,
which x86 does not enable until a later patch.
Assisted-by: LLM
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
---
arch/x86/kernel/asm-offsets.c | 8 +++++
arch/x86/kernel/ftrace.c | 43 +++++++++++++++++++++++++++
arch/x86/kernel/ftrace_64.S | 69 +++++++++++++++++++++++++++++++++++++++++++
arch/x86/kernel/vmlinux.lds.S | 4 +++
4 files changed, 124 insertions(+)
diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index 081816888f7a..876c3986419a 100644
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -9,6 +9,7 @@
#include <linux/crypto.h>
#include <crypto/aria.h>
#include <linux/sched.h>
+#include <linux/srcu.h>
#include <linux/stddef.h>
#include <linux/hardirq.h>
#include <linux/suspend.h>
@@ -46,6 +47,13 @@ static void __used common(void)
#ifdef CONFIG_STACKPROTECTOR
OFFSET(TASK_stack_canary, task_struct, stack_canary);
#endif
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+ OFFSET(TASK_trc_reader_nesting, task_struct, trc_reader_nesting);
+ OFFSET(TASK_trc_reader_scp, task_struct, trc_reader_scp);
+ OFFSET(SRCU_srcu_ctrp, srcu_struct, srcu_ctrp);
+ OFFSET(SRCU_CTR_srcu_locks, srcu_ctr, srcu_locks);
+ OFFSET(SRCU_CTR_srcu_unlocks, srcu_ctr, srcu_unlocks);
+#endif
BLANK();
OFFSET(pbe_address, pbe, address);
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index 17d6edfcb7e0..9babaed483eb 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -275,6 +275,49 @@ static inline void tramp_free(void *tramp)
execmem_free(tramp);
}
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+extern void ftrace_static_tramp_end(void);
+extern char __return_thunk_start[], __return_thunk_end[];
+extern char __rethunk_safe_start[], __rethunk_safe_end[];
+
+/*
+ * The SRCU-fast increments in TRACE_RCU_READ_LOCK/UNLOCK (ftrace_64.S) are the
+ * this_cpu_inc() form.
+ */
+static_assert(!IS_ENABLED(CONFIG_NEED_SRCU_NMI_SAFE));
+
+/*
+ * See rcu_tasks_trampoline_text(). Some core kernel text behaves like a
+ * trampoline for Tasks RCU purposes because a task executing there outside
+ * any Tasks Trace reader may still be about to enter a Tasks-RCU-protected
+ * trampoline whose address it already holds:
+ *
+ * - the static ftrace_caller / ftrace_regs_caller / ftrace_stub_direct_tramp
+ * stubs, which carry a direct-call target on the stack until their final
+ * RET, and
+ * - the return thunks that RET expands to under CONFIG_MITIGATION_RETHUNK,
+ * which run after leaving the stubs above and before landing in that
+ * target.
+ */
+bool arch_rcu_tasks_trampoline_text(unsigned long ip)
+{
+ if (ip >= (unsigned long)ftrace_caller &&
+ ip < (unsigned long)ftrace_static_tramp_end)
+ return true;
+#ifdef CONFIG_MITIGATION_RETPOLINE
+ if (ip >= (unsigned long)__return_thunk_start &&
+ ip < (unsigned long)__return_thunk_end)
+ return true;
+#endif
+#ifdef CONFIG_MITIGATION_SRSO
+ if (ip >= (unsigned long)__rethunk_safe_start &&
+ ip < (unsigned long)__rethunk_safe_end)
+ return true;
+#endif
+ return false;
+}
+#endif /* CONFIG_TASKS_RCU_TRAMPOLINE_READERS */
+
/* Defined as markers to the end of the ftrace default trampolines */
extern void ftrace_regs_caller_end(void);
extern void ftrace_caller_end(void);
diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S
index 62c1c93aa1c6..5d8cb3861978 100644
--- a/arch/x86/kernel/ftrace_64.S
+++ b/arch/x86/kernel/ftrace_64.S
@@ -7,6 +7,7 @@
#include <linux/cfi_types.h>
#include <linux/linkage.h>
#include <asm/asm-offsets.h>
+#include <asm/percpu.h>
#include <asm/ptrace.h>
#include <asm/ftrace.h>
#include <asm/nospec-branch.h>
@@ -145,6 +146,53 @@ SYM_FUNC_END(ftrace_stub_graph)
#ifdef CONFIG_DYNAMIC_FTRACE
+/*
+ * Open-coded rcu_read_lock_trace() / rcu_read_unlock_trace(), see
+ * include/linux/rcupdate_trace.h and CONFIG_HAVE_RCU_TRAMPOLINE_READERS: the
+ * trampoline and the ftrace_ops it is about to load are kept alive by Tasks
+ * RCU only while we are inside this reader, so the lock must precede the
+ * function_trace_op load and the unlock must follow the call. These live
+ * inside the region copied into dynamic trampolines; the %rip-relative
+ * references are fixed up by text_poke_apply_relocation() in
+ * create_trampoline(). Clobbers %rax, %rcx and flags.
+ */
+.macro TRACE_RCU_READ_LOCK
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+ movq PER_CPU_VAR(current_task), %rcx
+ movl TASK_trc_reader_nesting(%rcx), %eax
+ incl TASK_trc_reader_nesting(%rcx)
+ testl %eax, %eax
+ jnz .Ltrl_nested_\@
+ movq rcu_tasks_trace_srcu_struct+SRCU_srcu_ctrp(%rip), %rax
+ incq %gs:SRCU_CTR_srcu_locks(%rax)
+ movq %rax, TASK_trc_reader_scp(%rcx)
+#ifndef CONFIG_TASKS_TRACE_RCU_NO_MB
+ lock addl $0, -4(%rsp) /* smp_mb() */
+#endif
+.Ltrl_nested_\@:
+#endif
+.endm
+
+.macro TRACE_RCU_READ_UNLOCK
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+ movq PER_CPU_VAR(current_task), %rcx
+ movl TASK_trc_reader_nesting(%rcx), %eax
+ subl $1, %eax
+ jnz .Ltru_nested_\@
+ /* Outermost: pick up scp before an interrupt can see nesting == 0. */
+ movq TASK_trc_reader_scp(%rcx), %rax
+ movl $0, TASK_trc_reader_nesting(%rcx)
+#ifndef CONFIG_TASKS_TRACE_RCU_NO_MB
+ lock addl $0, -4(%rsp) /* smp_mb() */
+#endif
+ incq %gs:SRCU_CTR_srcu_unlocks(%rax)
+ jmp .Ltru_done_\@
+.Ltru_nested_\@:
+ movl %eax, TASK_trc_reader_nesting(%rcx)
+.Ltru_done_\@:
+#endif
+.endm
+
SYM_FUNC_START(__fentry__)
ANNOTATE_NOENDBR
CALL_DEPTH_ACCOUNT
@@ -163,6 +211,8 @@ SYM_FUNC_START(ftrace_caller)
leaq MCOUNT_REG_SIZE+8(%rsp), %rcx
movq %rcx, RSP(%rsp)
+ TRACE_RCU_READ_LOCK
+
SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL)
ANNOTATE_NOENDBR
/* Load the ftrace_ops into the 3rd parameter */
@@ -181,6 +231,8 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
ANNOTATE_NOENDBR
call ftrace_stub
+ TRACE_RCU_READ_UNLOCK
+
/* Handlers can change the RIP */
movq RIP(%rsp), %rax
movq %rax, MCOUNT_REG_SIZE(%rsp)
@@ -209,6 +261,8 @@ SYM_FUNC_START(ftrace_regs_caller)
CALL_DEPTH_ACCOUNT
+ TRACE_RCU_READ_LOCK
+
SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL)
ANNOTATE_NOENDBR
/* Load the ftrace_ops into the 3rd parameter */
@@ -246,6 +300,8 @@ SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL)
ANNOTATE_NOENDBR
call ftrace_stub
+ TRACE_RCU_READ_UNLOCK
+
/* Copy flags back to SS, to restore them */
movq EFLAGS(%rsp), %rax
movq %rax, MCOUNT_REG_SIZE(%rsp)
@@ -328,6 +384,19 @@ SYM_FUNC_START(ftrace_stub_direct_tramp)
RET
SYM_FUNC_END(ftrace_stub_direct_tramp)
+/*
+ * [ftrace_caller, ftrace_static_tramp_end) is treated as trampoline text by
+ * rcu_tasks_trampoline_text(): outside TRACE_RCU_READ_LOCK/UNLOCK the stubs
+ * may still hold a direct-call trampoline address (ORIG_RAX / the return
+ * address they RET to) that only Tasks RCU keeps alive. With return thunks
+ * the RET itself runs elsewhere; arch_rcu_tasks_trampoline_text() covers
+ * those too.
+ */
+SYM_CODE_START_NOALIGN(ftrace_static_tramp_end)
+ UNWIND_HINT_UNDEFINED
+ ANNOTATE_NOENDBR
+SYM_CODE_END(ftrace_static_tramp_end)
+
#else /* ! CONFIG_DYNAMIC_FTRACE */
SYM_FUNC_START(__fentry__)
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 2438b89a4620..e546283dc267 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -151,7 +151,9 @@ SECTIONS
* definition.
*/
. = srso_alias_untrain_ret | (1 << 2) | (1 << 8) | (1 << 14) | (1 << 20);
+ __rethunk_safe_start = .;
*(.text..__x86.rethunk_safe)
+ __rethunk_safe_end = .;
#endif
ALIGN_ENTRY_TEXT_END
@@ -162,7 +164,9 @@ SECTIONS
SOFTIRQENTRY_TEXT
#ifdef CONFIG_MITIGATION_RETPOLINE
*(.text..__x86.indirect_thunk)
+ __return_thunk_start = .;
*(.text..__x86.return_thunk)
+ __return_thunk_end = .;
#endif
STATIC_CALL_TEXT
*(.gnu.warning)
--
2.55.0
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH RFC v4 06/13] x86/kprobes: Take a Tasks Trace reader in the optprobe template
2026-09-18 14:49 [PATCH RFC v4 00/13] rcu-tasks: build Tasks RCU on Tasks Trace readers in trampolines Josef Bacik
` (4 preceding siblings ...)
2026-09-18 14:49 ` [PATCH RFC v4 05/13] x86/ftrace: Take a Tasks Trace reader around ftrace_caller's call-out Josef Bacik
@ 2026-09-18 14:49 ` Josef Bacik
2026-09-18 14:49 ` [PATCH RFC v4 07/13] bpf, x86: Take a Tasks Trace reader in the trampoline around its call-outs Josef Bacik
` (6 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Josef Bacik @ 2026-09-18 14:49 UTC (permalink / raw)
To: Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai,
Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai
Cc: Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Josef Bacik
The jump-optimized kprobe template calls optimized_callback() from a
dynamically allocated slot with preemption enabled, and only Tasks RCU
keeps that slot alive under a task preempted in the callback. For
HAVE_RCU_TRAMPOLINE_READERS that means the template must be a Tasks
Trace reader across the call, so open-code rcu_read_lock_trace() and
rcu_read_unlock_trace() around it as ftrace_64.S does. The template
lives in .rodata and is memcpy()d into each slot without relocation
processing, so the references to current_task and
rcu_tasks_trace_srcu_struct are absolute (R_X86_64_32S, relocated for
KASLR like any other) rather than %rip-relative. %rax and %rcx have
already been saved by SAVE_REGS_STRING and are dead after the call.
The slot itself is dynamically allocated text, so the instructions
before the lock and after the unlock are covered by the irq-exit check.
64-bit only; 32-bit x86 does not take part.
Assisted-by: LLM
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
---
arch/x86/kernel/kprobes/opt.c | 44 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 44 insertions(+)
diff --git a/arch/x86/kernel/kprobes/opt.c b/arch/x86/kernel/kprobes/opt.c
index 3f8fea52619f..68a5de6cdabe 100644
--- a/arch/x86/kernel/kprobes/opt.c
+++ b/arch/x86/kernel/kprobes/opt.c
@@ -31,6 +31,7 @@
#include <asm/set_memory.h>
#include <asm/sections.h>
#include <asm/nospec-branch.h>
+#include <asm/asm-offsets.h>
#include "common.h"
@@ -101,6 +102,47 @@ static void synthesize_set_arg1(kprobe_opcode_t *addr, unsigned long val)
*(unsigned long *)addr = val;
}
+/*
+ * Open-coded rcu_read_lock_trace() / rcu_read_unlock_trace() around the call
+ * to optimized_callback(), see CONFIG_HAVE_RCU_TRAMPOLINE_READERS and the
+ * equivalent macros in ftrace_64.S. The template is memcpy()d into the slot
+ * without relocation processing, so memory references must be absolute
+ * rather than %rip-relative. %rax and %rcx are free at both points.
+ */
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+#ifndef CONFIG_TASKS_TRACE_RCU_NO_MB
+#define OPTPROBE_TRACE_RCU_MB " lock addl $0, -4(%rsp)\n"
+#else
+#define OPTPROBE_TRACE_RCU_MB
+#endif
+#define OPTPROBE_TRACE_RCU_READ_LOCK \
+ " movq %gs:current_task, %rcx\n" \
+ " movl " __stringify(TASK_trc_reader_nesting) "(%rcx), %eax\n" \
+ " incl " __stringify(TASK_trc_reader_nesting) "(%rcx)\n" \
+ " testl %eax, %eax\n" \
+ " jnz 1f\n" \
+ " movq rcu_tasks_trace_srcu_struct+" __stringify(SRCU_srcu_ctrp) ", %rax\n" \
+ " incq %gs:" __stringify(SRCU_CTR_srcu_locks) "(%rax)\n" \
+ " movq %rax, " __stringify(TASK_trc_reader_scp) "(%rcx)\n" \
+ OPTPROBE_TRACE_RCU_MB \
+ "1:\n"
+#define OPTPROBE_TRACE_RCU_READ_UNLOCK \
+ " movq %gs:current_task, %rcx\n" \
+ " movl " __stringify(TASK_trc_reader_nesting) "(%rcx), %eax\n" \
+ " subl $1, %eax\n" \
+ " jnz 2f\n" \
+ " movq " __stringify(TASK_trc_reader_scp) "(%rcx), %rax\n" \
+ " movl $0, " __stringify(TASK_trc_reader_nesting) "(%rcx)\n" \
+ OPTPROBE_TRACE_RCU_MB \
+ " incq %gs:" __stringify(SRCU_CTR_srcu_unlocks) "(%rax)\n" \
+ " jmp 3f\n" \
+ "2: movl %eax, " __stringify(TASK_trc_reader_nesting) "(%rcx)\n" \
+ "3:\n"
+#else
+#define OPTPROBE_TRACE_RCU_READ_LOCK
+#define OPTPROBE_TRACE_RCU_READ_UNLOCK
+#endif
+
asm (
".pushsection .rodata\n"
".global optprobe_template_entry\n"
@@ -114,6 +156,7 @@ asm (
"optprobe_template_clac:\n"
ASM_NOP3
SAVE_REGS_STRING
+ OPTPROBE_TRACE_RCU_READ_LOCK
" movq %rsp, %rsi\n"
".global optprobe_template_val\n"
"optprobe_template_val:\n"
@@ -122,6 +165,7 @@ asm (
".global optprobe_template_call\n"
"optprobe_template_call:\n"
ASM_NOP5
+ OPTPROBE_TRACE_RCU_READ_UNLOCK
/* Copy 'regs->flags' into 'regs->ss'. */
" movq 18*8(%rsp), %rdx\n"
" movq %rdx, 20*8(%rsp)\n"
--
2.55.0
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH RFC v4 07/13] bpf, x86: Take a Tasks Trace reader in the trampoline around its call-outs
2026-09-18 14:49 [PATCH RFC v4 00/13] rcu-tasks: build Tasks RCU on Tasks Trace readers in trampolines Josef Bacik
` (5 preceding siblings ...)
2026-09-18 14:49 ` [PATCH RFC v4 06/13] x86/kprobes: Take a Tasks Trace reader in the optprobe template Josef Bacik
@ 2026-09-18 14:49 ` Josef Bacik
2026-09-18 21:26 ` Alexei Starovoitov
2026-09-18 14:49 ` [PATCH RFC v4 08/13] arm64: ftrace: Take a Tasks Trace reader around ftrace_caller's call-out Josef Bacik
` (5 subsequent siblings)
12 siblings, 1 reply; 15+ messages in thread
From: Josef Bacik @ 2026-09-18 14:49 UTC (permalink / raw)
To: Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai,
Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai
Cc: Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Josef Bacik
On HAVE_RCU_TRAMPOLINE_READERS kernels Tasks RCU keeps a BPF trampoline
image allocated only while a task using it is a Tasks Trace RCU reader
or is executing text that rcu_tasks_trampoline_text() recognises. The
image itself is such text, but the C glue and the programs it calls are
not, and only sleepable programs take rcu_read_lock_trace() today.
Have the x86-64 JIT open-code rcu_read_lock_trace() and
rcu_read_unlock_trace() in the trampoline, as ftrace_64.S does for
ftrace_caller: one reader from just after the frame is set up to just
before the original function is called, covering __bpf_tramp_enter()
and the fentry and fmod_ret programs, and a second one from just after
the original function returns to just before the final register
restore, covering the fexit programs and __bpf_tramp_exit(). The
original function itself runs outside both, since it may run for a long
time and the image is pinned by im->pcref across it. Trampolines that
do not call the original function get a single reader around all their
programs. The second reader is entered before ip_after_call, so the
ip_after_call -> ip_epilogue jump that bpf_tramp_image_put() patches in
is inside it, and the fmod_ret early-exit branch lands after that point
still holding the first reader, so exactly one is held on every path.
The sequence uses r10 and r11, which are scratch at each emission point,
and references current_task and rcu_tasks_trace_srcu_struct by absolute
sign-extended address, the form the JIT already relies on for
this_cpu_off. Sleepable programs' own rcu_read_lock_trace() simply
nests. Nothing is emitted on other configurations.
Suggested-by: Alexei Starovoitov <ast@kernel.org>
Assisted-by: LLM
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
---
arch/x86/net/bpf_jit_comp.c | 113 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 113 insertions(+)
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 2853e87797a7..c991f7ceacdf 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -14,6 +14,7 @@
#include <linux/memory.h>
#include <linux/sort.h>
#include <linux/execmem.h>
+#include <linux/rcupdate_trace.h>
#include <asm/extable.h>
#include <asm/ftrace.h>
#include <asm/set_memory.h>
@@ -722,6 +723,97 @@ static void emit_indirect_jump(u8 **pprog, int bpf_reg, u8 *ip)
*pprog = prog;
}
+/*
+ * Open-coded rcu_read_lock_trace() / rcu_read_unlock_trace() for the
+ * trampoline, see CONFIG_HAVE_RCU_TRAMPOLINE_READERS and the equivalent
+ * macros in arch/x86/kernel/ftrace_64.S. The image is not relocated, so
+ * current_task and rcu_tasks_trace_srcu_struct are referenced by absolute
+ * (sign-extended 32-bit) address, the form the JIT already relies on for
+ * this_cpu_off. Uses r10 and r11, which are scratch at every emission
+ * point, and clobbers flags.
+ *
+ * lock: unlock:
+ * mov r11, gs:[current_task] mov r11, gs:[current_task]
+ * mov r10d, [r11+nesting] mov r10d, [r11+nesting]
+ * inc dword ptr [r11+nesting] sub r10d, 1
+ * test r10d, r10d jnz 2f
+ * jnz 1f mov r10, [r11+scp]
+ * mov r10, [&srcu.srcu_ctrp] mov dword ptr [r11+nesting], 0
+ * inc qword ptr gs:[r10+locks] (smp_mb)
+ * mov [r11+scp], r10 inc qword ptr gs:[r10+unlocks]
+ * (smp_mb) jmp 3f
+ * 1: 2: mov [r11+nesting], r10d
+ * 3:
+ */
+static void emit_trace_rcu_reader(u8 **pprog, bool lock)
+{
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+ const u32 nesting = offsetof(struct task_struct, trc_reader_nesting);
+ const u32 scp = offsetof(struct task_struct, trc_reader_scp);
+ const bool mb = !IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB);
+ u8 *prog = *pprog;
+
+ BUILD_BUG_ON(IS_ENABLED(CONFIG_NEED_SRCU_NMI_SAFE));
+ BUILD_BUG_ON(offsetof(struct srcu_ctr, srcu_locks) != 0);
+ BUILD_BUG_ON(offsetof(struct srcu_ctr, srcu_unlocks) != 8);
+
+ /* mov r11, gs:[abs32 current_task] */
+ EMIT2(0x65, 0x4C);
+ EMIT3(0x8B, 0x1C, 0x25);
+ EMIT((u32)(unsigned long)¤t_task, 4);
+ /* mov r10d, dword ptr [r11 + nesting] */
+ EMIT3(0x45, 0x8B, 0x93);
+ EMIT(nesting, 4);
+
+ if (lock) {
+ /* inc dword ptr [r11 + nesting] */
+ EMIT3(0x41, 0xFF, 0x83);
+ EMIT(nesting, 4);
+ /* test r10d, r10d */
+ EMIT3(0x45, 0x85, 0xD2);
+ /* jnz 1f */
+ EMIT2(X86_JNE, 8 + 4 + 7 + (mb ? 6 : 0));
+ /* mov r10, qword ptr [abs32 &rcu_tasks_trace_srcu_struct.srcu_ctrp] */
+ EMIT4(0x4C, 0x8B, 0x14, 0x25);
+ EMIT((u32)(unsigned long)&rcu_tasks_trace_srcu_struct.srcu_ctrp, 4);
+ /* inc qword ptr gs:[r10] */
+ EMIT4(0x65, 0x49, 0xFF, 0x02);
+ /* mov qword ptr [r11 + scp], r10 */
+ EMIT3(0x4D, 0x89, 0x93);
+ EMIT(scp, 4);
+ /* smp_mb(): lock add dword ptr [rsp - 4], 0 */
+ if (mb)
+ EMIT2_off32(0xF0, 0x83, 0x00FC2444);
+ /* 1: */
+ } else {
+ /* sub r10d, 1 */
+ EMIT4(0x41, 0x83, 0xEA, 0x01);
+ /* jnz 2f */
+ EMIT2(X86_JNE, 7 + 11 + (mb ? 6 : 0) + 5 + 2);
+ /* mov r10, qword ptr [r11 + scp] */
+ EMIT3(0x4D, 0x8B, 0x93);
+ EMIT(scp, 4);
+ /* mov dword ptr [r11 + nesting], 0 */
+ EMIT3(0x41, 0xC7, 0x83);
+ EMIT(nesting, 4);
+ EMIT(0, 4);
+ if (mb)
+ EMIT2_off32(0xF0, 0x83, 0x00FC2444);
+ /* inc qword ptr gs:[r10 + 8] */
+ EMIT4(0x65, 0x49, 0xFF, 0x42);
+ EMIT1(0x08);
+ /* jmp 3f */
+ EMIT2(0xEB, 7);
+ /* 2: mov dword ptr [r11 + nesting], r10d */
+ EMIT3(0x45, 0x89, 0x93);
+ EMIT(nesting, 4);
+ /* 3: */
+ }
+
+ *pprog = prog;
+#endif
+}
+
static void emit_return(u8 **pprog, u8 *ip)
{
u8 *prog = *pprog;
@@ -3610,6 +3702,16 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
/* mov QWORD PTR [rbp - rbx_off], rbx */
emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_6, -rbx_off);
+ /*
+ * Tasks RCU keeps this image alive only while we are a Tasks Trace
+ * reader; the instructions before this point (and after the final
+ * unlock) are covered by the irq-exit IP check. One reader spans
+ * __bpf_tramp_enter() and the fentry/fmod_ret progs, a second one
+ * the fexit progs and __bpf_tramp_exit(); the original function runs
+ * outside both, with the image pinned by im->pcref instead.
+ */
+ emit_trace_rcu_reader(&prog, true);
+
func_meta = nr_regs;
/* Store number of argument registers of the traced function */
emit_store_stack_imm64(&prog, BPF_REG_0, -func_meta_off, func_meta);
@@ -3660,6 +3762,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
}
if (flags & BPF_TRAMP_F_CALL_ORIG) {
+ emit_trace_rcu_reader(&prog, false);
restore_regs(m, &prog, regs_off);
save_args(m, &prog, arg_stack_off, true, flags, 0);
@@ -3682,6 +3785,13 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
}
/* remember return value in a stack for bpf prog to access */
emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
+ /*
+ * Second reader. Taken before ip_after_call so that the
+ * ip_after_call -> ip_epilogue jump patched in at teardown is
+ * inside it too; the fmod_ret early exit jumps past this still
+ * holding the first reader, so either way exactly one is held.
+ */
+ emit_trace_rcu_reader(&prog, true);
im->ip_after_call = image + (prog - (u8 *)rw_image);
emit_nops(&prog, X86_PATCH_SIZE);
}
@@ -3737,6 +3847,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack_size);
}
+ /* Remaining instructions are covered by the irq-exit IP check. */
+ emit_trace_rcu_reader(&prog, false);
+
/* restore return value of orig_call or fentry prog back into RAX */
if (save_ret)
emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8);
--
2.55.0
^ permalink raw reply [flat|nested] 15+ messages in thread* Re: [PATCH RFC v4 07/13] bpf, x86: Take a Tasks Trace reader in the trampoline around its call-outs
2026-09-18 14:49 ` [PATCH RFC v4 07/13] bpf, x86: Take a Tasks Trace reader in the trampoline around its call-outs Josef Bacik
@ 2026-09-18 21:26 ` Alexei Starovoitov
0 siblings, 0 replies; 15+ messages in thread
From: Alexei Starovoitov @ 2026-09-18 21:26 UTC (permalink / raw)
To: Josef Bacik, Paul E. McKenney, Frederic Weisbecker,
Neeraj Upadhyay, Joel Fernandes, Boqun Feng, Thomas Gleixner,
Peter Zijlstra, Steven Rostedt, Masami Hiramatsu, Mark Rutland,
Jiri Olsa, Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko,
x86, Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai,
Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai
Cc: Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel
On Fri Sep 18, 2026 at 2:49 PM UTC, Josef Bacik wrote:
> On HAVE_RCU_TRAMPOLINE_READERS kernels Tasks RCU keeps a BPF trampoline
> image allocated only while a task using it is a Tasks Trace RCU reader
> or is executing text that rcu_tasks_trampoline_text() recognises. The
> image itself is such text, but the C glue and the programs it calls are
> not, and only sleepable programs take rcu_read_lock_trace() today.
>
> Have the x86-64 JIT open-code rcu_read_lock_trace() and
> rcu_read_unlock_trace() in the trampoline, as ftrace_64.S does for
> ftrace_caller: one reader from just after the frame is set up to just
> before the original function is called, covering __bpf_tramp_enter()
> and the fentry and fmod_ret programs, and a second one from just after
> the original function returns to just before the final register
> restore, covering the fexit programs and __bpf_tramp_exit(). The
> original function itself runs outside both, since it may run for a long
> time and the image is pinned by im->pcref across it. Trampolines that
> do not call the original function get a single reader around all their
> programs. The second reader is entered before ip_after_call, so the
> ip_after_call -> ip_epilogue jump that bpf_tramp_image_put() patches in
> is inside it, and the fmod_ret early-exit branch lands after that point
> still holding the first reader, so exactly one is held on every path.
>
> The sequence uses r10 and r11, which are scratch at each emission point,
> and references current_task and rcu_tasks_trace_srcu_struct by absolute
> sign-extended address, the form the JIT already relies on for
> this_cpu_off. Sleepable programs' own rcu_read_lock_trace() simply
> nests. Nothing is emitted on other configurations.
>
> Suggested-by: Alexei Starovoitov <ast@kernel.org>
> Assisted-by: LLM
> Signed-off-by: Josef Bacik <josef@toxicpanda.com>
> ---
> arch/x86/net/bpf_jit_comp.c | 113 ++++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 113 insertions(+)
>
> diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
> index 2853e87797a7..c991f7ceacdf 100644
> --- a/arch/x86/net/bpf_jit_comp.c
> +++ b/arch/x86/net/bpf_jit_comp.c
> @@ -14,6 +14,7 @@
> #include <linux/memory.h>
> #include <linux/sort.h>
> #include <linux/execmem.h>
> +#include <linux/rcupdate_trace.h>
> #include <asm/extable.h>
> #include <asm/ftrace.h>
> #include <asm/set_memory.h>
> @@ -722,6 +723,97 @@ static void emit_indirect_jump(u8 **pprog, int bpf_reg, u8 *ip)
> *pprog = prog;
> }
>
> +/*
> + * Open-coded rcu_read_lock_trace() / rcu_read_unlock_trace() for the
> + * trampoline, see CONFIG_HAVE_RCU_TRAMPOLINE_READERS and the equivalent
> + * macros in arch/x86/kernel/ftrace_64.S. The image is not relocated, so
> + * current_task and rcu_tasks_trace_srcu_struct are referenced by absolute
> + * (sign-extended 32-bit) address, the form the JIT already relies on for
> + * this_cpu_off. Uses r10 and r11, which are scratch at every emission
> + * point, and clobbers flags.
> + *
> + * lock: unlock:
> + * mov r11, gs:[current_task] mov r11, gs:[current_task]
> + * mov r10d, [r11+nesting] mov r10d, [r11+nesting]
> + * inc dword ptr [r11+nesting] sub r10d, 1
> + * test r10d, r10d jnz 2f
> + * jnz 1f mov r10, [r11+scp]
> + * mov r10, [&srcu.srcu_ctrp] mov dword ptr [r11+nesting], 0
> + * inc qword ptr gs:[r10+locks] (smp_mb)
> + * mov [r11+scp], r10 inc qword ptr gs:[r10+unlocks]
> + * (smp_mb) jmp 3f
> + * 1: 2: mov [r11+nesting], r10d
> + * 3:
> + */
> +static void emit_trace_rcu_reader(u8 **pprog, bool lock)
> +{
> +#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
> + const u32 nesting = offsetof(struct task_struct, trc_reader_nesting);
> + const u32 scp = offsetof(struct task_struct, trc_reader_scp);
nesting_off ?
scp_off ?
Otherwise
EMIT(nesting, 4);
is a bit confusing.
> + const bool mb = !IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB);
> + u8 *prog = *pprog;
> +
> + BUILD_BUG_ON(IS_ENABLED(CONFIG_NEED_SRCU_NMI_SAFE));
> + BUILD_BUG_ON(offsetof(struct srcu_ctr, srcu_locks) != 0);
> + BUILD_BUG_ON(offsetof(struct srcu_ctr, srcu_unlocks) != 8);
Looks too hardcoded here.
Why not to use the same approach as with offsetof() few lines above?
Overall looks ok,
but I wonder what Paul will say that rcu_read_lock_tasks_trace()
becomes baked in into JITs and will be pretty hard to change.
We also lose rcu_tt lockdep runtime checks.
So lockdep might get confused?
^ permalink raw reply [flat|nested] 15+ messages in thread
* [PATCH RFC v4 08/13] arm64: ftrace: Take a Tasks Trace reader around ftrace_caller's call-out
2026-09-18 14:49 [PATCH RFC v4 00/13] rcu-tasks: build Tasks RCU on Tasks Trace readers in trampolines Josef Bacik
` (6 preceding siblings ...)
2026-09-18 14:49 ` [PATCH RFC v4 07/13] bpf, x86: Take a Tasks Trace reader in the trampoline around its call-outs Josef Bacik
@ 2026-09-18 14:49 ` Josef Bacik
2026-09-18 14:49 ` [PATCH RFC v4 09/13] bpf, arm64: Take a Tasks Trace reader in the trampoline around its call-outs Josef Bacik
` (4 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Josef Bacik @ 2026-09-18 14:49 UTC (permalink / raw)
To: Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai,
Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai
Cc: Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Josef Bacik
For HAVE_RCU_TRAMPOLINE_READERS the ftrace trampoline must be a Tasks
Trace RCU reader while it calls out, since that is what
synchronize_rcu_tasks() will wait for before ftrace_shutdown() frees an
ftrace_ops (or, with CALL_OPS, lets its owner free it) under a task
preempted in the callback.
Open-code rcu_read_lock_trace() and rcu_read_unlock_trace() around the
call to ops->func in ftrace_caller: bump current->trc_reader_nesting via
sp_el0 and, for the outermost reader, do the SRCU-fast per-CPU increment
on rcu_tasks_trace_srcu_struct and stash the counter pointer in
current->trc_reader_scp, as the C inlines do (including the dmb when
CONFIG_TASKS_TRACE_RCU_NO_MB is not set). The per-CPU increment is an
LL/SC add on this CPU's counter; being migrated between reading the
per-CPU offset and the store-exclusive only means another CPU's counter
is incremented atomically instead, which SRCU sums over anyway.
x12-x16 are free at both points.
arm64 has no return thunks and, with CALL_OPS, no dynamic ftrace
trampolines, but ftrace_caller itself carries the ops pointer in x11
from before the reader is entered and a direct-call BPF trampoline
address in x17 until the final br/ret after it is left, so mark the end
of the static trampoline text and provide arch_rcu_tasks_trampoline_text()
covering [ftrace_caller, ftrace_static_tramp_end).
Built only under CONFIG_TASKS_RCU_TRAMPOLINE_READERS, which arm64 does
not enable until a later patch.
Assisted-by: LLM
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
---
arch/arm64/kernel/asm-offsets.c | 8 +++++
arch/arm64/kernel/entry-ftrace.S | 74 ++++++++++++++++++++++++++++++++++++++++
arch/arm64/kernel/ftrace.c | 20 +++++++++++
3 files changed, 102 insertions(+)
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 9c853ed3ceab..f6a8fb1f9b43 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -10,6 +10,7 @@
#include <linux/arm_sdei.h>
#include <linux/sched.h>
+#include <linux/srcu.h>
#include <linux/ftrace.h>
#include <linux/kexec.h>
#include <linux/mm.h>
@@ -39,6 +40,13 @@ int main(void)
DEFINE(TSK_STACK, offsetof(struct task_struct, stack));
#ifdef CONFIG_STACKPROTECTOR
DEFINE(TSK_STACK_CANARY, offsetof(struct task_struct, stack_canary));
+#endif
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+ DEFINE(TSK_TRC_READER_NESTING, offsetof(struct task_struct, trc_reader_nesting));
+ DEFINE(TSK_TRC_READER_SCP, offsetof(struct task_struct, trc_reader_scp));
+ DEFINE(SRCU_SRCU_CTRP, offsetof(struct srcu_struct, srcu_ctrp));
+ DEFINE(SRCU_CTR_SRCU_LOCKS, offsetof(struct srcu_ctr, srcu_locks));
+ DEFINE(SRCU_CTR_SRCU_UNLOCKS, offsetof(struct srcu_ctr, srcu_unlocks));
#endif
BLANK();
DEFINE(THREAD_CPU_CONTEXT, offsetof(struct task_struct, thread.cpu_context));
diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S
index 025140caafe7..fc2805eb9e15 100644
--- a/arch/arm64/kernel/entry-ftrace.S
+++ b/arch/arm64/kernel/entry-ftrace.S
@@ -14,6 +14,72 @@
#include <asm/insn.h>
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
+/*
+ * Open-coded rcu_read_lock_trace() / rcu_read_unlock_trace(), see
+ * include/linux/rcupdate_trace.h and CONFIG_HAVE_RCU_TRAMPOLINE_READERS. The
+ * whole of ftrace_caller is treated as trampoline text by the irq-exit check
+ * (see arch_rcu_tasks_trampoline_text()), so these only need to bracket the
+ * call out to ops->func; everything before the lock and after the unlock,
+ * including the direct-call tails that carry a BPF trampoline address in x17,
+ * is covered by that.
+ *
+ * The SRCU-fast per-CPU increment is done LL/SC on this CPU's counter; being
+ * migrated between reading the per-CPU offset and the store-exclusive only
+ * means another CPU's counter is (atomically) incremented, which SRCU sums
+ * over anyway. Ordering between the nesting count and the scp stash only
+ * matters against interrupts on this CPU, which observe program order.
+ * Clobbers x12-x16 and the flags.
+ */
+ .macro trace_rcu_srcu_inc, addr:req, tmp:req, wtmp2:req
+8888: ldxr \tmp, [\addr]
+ add \tmp, \tmp, #1
+ stxr \wtmp2, \tmp, [\addr]
+ cbnz \wtmp2, 8888b
+ .endm
+
+ .macro trace_rcu_read_lock
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+ mrs x12, sp_el0 // current
+ ldr w13, [x12, #TSK_TRC_READER_NESTING]
+ add w14, w13, #1
+ str w14, [x12, #TSK_TRC_READER_NESTING]
+ cbnz w13, .Ltrl_nested\@ // interrupted a reader: done
+ ldr_l x13, rcu_tasks_trace_srcu_struct + SRCU_SRCU_CTRP
+ str x13, [x12, #TSK_TRC_READER_SCP]
+ get_this_cpu_offset x14
+ add x14, x14, x13
+ add x14, x14, #SRCU_CTR_SRCU_LOCKS
+ trace_rcu_srcu_inc x14, x15, w16
+#ifndef CONFIG_TASKS_TRACE_RCU_NO_MB
+ dmb ish
+#endif
+.Ltrl_nested\@:
+#endif
+ .endm
+
+ .macro trace_rcu_read_unlock
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+ mrs x12, sp_el0 // current
+ ldr w13, [x12, #TSK_TRC_READER_NESTING]
+ subs w13, w13, #1
+ b.ne .Ltru_nested\@
+ /* Outermost: pick up scp before an interrupt can see nesting == 0. */
+ ldr x14, [x12, #TSK_TRC_READER_SCP]
+ str wzr, [x12, #TSK_TRC_READER_NESTING]
+#ifndef CONFIG_TASKS_TRACE_RCU_NO_MB
+ dmb ish
+#endif
+ get_this_cpu_offset x15
+ add x14, x14, x15
+ add x14, x14, #SRCU_CTR_SRCU_UNLOCKS
+ trace_rcu_srcu_inc x14, x15, w16
+ b .Ltru_done\@
+.Ltru_nested\@:
+ str w13, [x12, #TSK_TRC_READER_NESTING]
+.Ltru_done\@:
+#endif
+ .endm
+
/*
* Due to -fpatchable-function-entry=2, the compiler has placed two NOPs before
* the regular function prologue. For an enabled callsite, ftrace_init_nop() and
@@ -94,6 +160,8 @@ SYM_CODE_START(ftrace_caller)
stp x29, x30, [sp, #FREGS_SIZE]
add x29, sp, #FREGS_SIZE
+ trace_rcu_read_lock
+
/* Prepare arguments for the tracer func */
sub x0, x30, #AARCH64_INSN_SIZE // ip (callsite's BL insn)
mov x1, x9 // parent_ip (callsite's LR)
@@ -111,6 +179,8 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
bl ftrace_stub // func(ip, parent_ip, op, regs)
#endif
+ trace_rcu_read_unlock
+
/*
* At the callsite x0-x8 and x19-x30 were live. Any C code will have preserved
* x19-x29 per the AAPCS, and we created frame records upon entry, so we need
@@ -178,6 +248,10 @@ SYM_CODE_START(ftrace_stub_direct_tramp)
SYM_CODE_END(ftrace_stub_direct_tramp)
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
+/* End of [ftrace_caller, ...) for arch_rcu_tasks_trampoline_text(). */
+SYM_CODE_START(ftrace_static_tramp_end)
+SYM_CODE_END(ftrace_static_tramp_end)
+
#else /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */
/*
diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c
index e1a3c0b3a051..5f4193f15cd9 100644
--- a/arch/arm64/kernel/ftrace.c
+++ b/arch/arm64/kernel/ftrace.c
@@ -17,6 +17,26 @@
#include <asm/insn.h>
#include <asm/text-patching.h>
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+extern void ftrace_static_tramp_end(void);
+
+/* The SRCU-fast increments in entry-ftrace.S are the this_cpu_inc() form. */
+static_assert(!IS_ENABLED(CONFIG_NEED_SRCU_NMI_SAFE));
+
+/*
+ * See rcu_tasks_trampoline_text(). ftrace_caller and ftrace_stub_direct_tramp
+ * are core kernel text but must be treated as trampolines: a task interrupted
+ * in them outside the Tasks Trace reader may be carrying an ops pointer (x11)
+ * or a direct-call BPF trampoline address (x17) whose lifetime is guarded only
+ * by Tasks RCU.
+ */
+bool arch_rcu_tasks_trampoline_text(unsigned long ip)
+{
+ return ip >= (unsigned long)ftrace_caller &&
+ ip < (unsigned long)ftrace_static_tramp_end;
+}
+#endif
+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
struct fregs_offset {
const char *name;
--
2.55.0
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH RFC v4 09/13] bpf, arm64: Take a Tasks Trace reader in the trampoline around its call-outs
2026-09-18 14:49 [PATCH RFC v4 00/13] rcu-tasks: build Tasks RCU on Tasks Trace readers in trampolines Josef Bacik
` (7 preceding siblings ...)
2026-09-18 14:49 ` [PATCH RFC v4 08/13] arm64: ftrace: Take a Tasks Trace reader around ftrace_caller's call-out Josef Bacik
@ 2026-09-18 14:49 ` Josef Bacik
2026-09-18 14:49 ` [PATCH RFC v4 10/13] samples: ftrace: Make the direct-call trampolines Tasks Trace readers Josef Bacik
` (3 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Josef Bacik @ 2026-09-18 14:49 UTC (permalink / raw)
To: Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai,
Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai
Cc: Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Josef Bacik
Same scheme as x86: have the arm64 BPF JIT open-code rcu_read_lock_trace()
and rcu_read_unlock_trace() in the trampoline, one reader from after the
callee-saved registers are stored to just before the original function
is called (covering __bpf_tramp_enter() and the fentry/fmod_ret programs)
and a second from just after it returns to just before those registers
are restored (covering the fexit programs and __bpf_tramp_exit()), with
the original function itself outside both and pinned by im->pcref. The
second reader is entered before ip_after_call and the fmod_ret cbnz
lands past it still holding the first, so exactly one is held on every
path; trampolines without an original call get a single reader.
The sequence mirrors entry-ftrace.S: current via sp_el0,
trc_reader_nesting bumped, and for the outermost reader the SRCU-fast
per-CPU counter incremented LL/SC and the counter pointer stashed in
trc_reader_scp (plus the dmb when CONFIG_TASKS_TRACE_RCU_NO_MB is not
set). x10-x15 are scratch at every emission point. Nothing is emitted
on other configurations.
Suggested-by: Alexei Starovoitov <ast@kernel.org>
Assisted-by: LLM
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
---
arch/arm64/net/bpf_jit_comp.c | 90 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 90 insertions(+)
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index c18e005a41db..87f9d53caf2c 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -14,6 +14,7 @@
#include <linux/filter.h>
#include <linux/memory.h>
#include <linux/printk.h>
+#include <linux/rcupdate_trace.h>
#include <linux/slab.h>
#include <asm/asm-extable.h>
@@ -2591,6 +2592,74 @@ static void emit_arena_arg_conv(struct jit_ctx *ctx, u8 dst, u8 src, bool nullab
emit(A64_SUB(0, dst, src, base_lo), ctx);
}
+/*
+ * Open-coded rcu_read_lock_trace() / rcu_read_unlock_trace() for the
+ * trampoline, see CONFIG_HAVE_RCU_TRAMPOLINE_READERS and the equivalent macros
+ * in arch/arm64/kernel/entry-ftrace.S. The SRCU-fast per-CPU increment is an
+ * LL/SC add on this CPU's counter; being migrated between reading the per-CPU
+ * offset and the store-exclusive only means another CPU's counter is
+ * incremented atomically instead, which SRCU sums over anyway. Uses x10-x15,
+ * which are scratch at every emission point, and the flags.
+ */
+static void emit_trace_rcu_reader(struct jit_ctx *ctx, bool lock)
+{
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+ const int nesting = offsetof(struct task_struct, trc_reader_nesting);
+ const int scp = offsetof(struct task_struct, trc_reader_scp);
+ const bool mb = !IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB);
+ const u8 tsk = A64_R(10), n = A64_R(11), tmp = A64_R(12);
+ const u8 ctr = A64_R(13), addr = A64_R(14), val = A64_R(15);
+
+ BUILD_BUG_ON(IS_ENABLED(CONFIG_NEED_SRCU_NMI_SAFE));
+ /* LDR/STR (immediate, unsigned offset) ranges */
+ BUILD_BUG_ON((nesting & 3) || nesting >= SZ_16K || (scp & 7) || scp >= SZ_32K);
+
+ emit(A64_MRS_SP_EL0(tsk), ctx); /* current */
+ emit(A64_LDR32I(n, tsk, nesting), ctx);
+ if (lock) {
+ emit(A64_ADD_I(0, tmp, n, 1), ctx);
+ emit(A64_STR32I(tmp, tsk, nesting), ctx);
+ /* interrupted a reader: done */
+ emit(A64_CBNZ(0, n, 12 + mb), ctx);
+ /* scp = rcu_tasks_trace_srcu_struct.srcu_ctrp; current->trc_reader_scp = scp */
+ emit_addr_mov_i64(ctr, (u64)&rcu_tasks_trace_srcu_struct.srcu_ctrp, ctx);
+ emit(A64_LDR64I(ctr, ctr, 0), ctx);
+ emit(A64_STR64I(ctr, tsk, scp), ctx);
+ } else {
+ emit(A64_SUB_I(0, n, n, 1), ctx);
+ /* still nested: just store the count */
+ emit(A64_CBNZ(0, n, 11 + mb), ctx);
+ /* outermost: pick up scp before an interrupt can see nesting == 0 */
+ emit(A64_LDR64I(ctr, tsk, scp), ctx);
+ emit(A64_STR32I(A64_ZR, tsk, nesting), ctx);
+ if (mb)
+ emit(A64_DMB_ISH, ctx);
+ }
+ /* this_cpu_inc(scp->srcu_locks / srcu_unlocks) */
+ if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN))
+ emit(A64_MRS_TPIDR_EL2(addr), ctx);
+ else
+ emit(A64_MRS_TPIDR_EL1(addr), ctx);
+ emit(A64_ADD(1, addr, addr, ctr), ctx);
+ if (!lock)
+ emit(A64_ADD_I(1, addr, addr, offsetof(struct srcu_ctr, srcu_unlocks)), ctx);
+ emit(A64_LDXR(1, val, addr), ctx);
+ emit(A64_ADD_I(1, val, val, 1), ctx);
+ emit(A64_STXR(1, val, addr, tmp), ctx);
+ emit(A64_CBNZ(0, tmp, -3), ctx);
+ if (lock) {
+ if (mb)
+ emit(A64_DMB_ISH, ctx);
+ /* 1: */
+ } else {
+ emit(A64_B(2), ctx);
+ /* 2: */
+ emit(A64_STR32I(n, tsk, nesting), ctx);
+ /* 3: */
+ }
+#endif
+}
+
static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
const struct btf_func_model *m, const struct arg_aux *a,
bool for_call_origin, bool is_struct_ops, u64 arena_base)
@@ -2854,6 +2923,16 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx);
emit(A64_STR64I(A64_R(20), A64_SP, regs_off + 8), ctx);
+ /*
+ * Tasks RCU keeps this image alive only while we are a Tasks Trace
+ * reader; the instructions before this point (and after the final
+ * unlock) are covered by the irq-exit IP check. One reader spans
+ * __bpf_tramp_enter() and the fentry/fmod_ret progs, a second one the
+ * fexit progs and __bpf_tramp_exit(); the original function runs
+ * outside both, with the image pinned by im->pcref instead.
+ */
+ emit_trace_rcu_reader(ctx, true);
+
if (flags & BPF_TRAMP_F_CALL_ORIG) {
/* for the first pass, assume the worst case */
if (!ctx->image)
@@ -2898,12 +2977,20 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
if (flags & BPF_TRAMP_F_CALL_ORIG) {
/* the original func takes kernel addresses, never converted ones */
save_args(ctx, bargs_off, oargs_off, m, a, true, is_struct_ops, 0);
+ emit_trace_rcu_reader(ctx, false);
/* call original func */
emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx);
emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx);
emit(A64_RET(A64_R(10)), ctx);
/* store return value */
emit(A64_STR64I(A64_R(0), A64_SP, retval_off), ctx);
+ /*
+ * Second reader. Taken before ip_after_call so that the branch
+ * to the epilogue patched in at teardown is inside it too; the
+ * fmod_ret early exit lands past this still holding the first
+ * reader, so either way exactly one is held.
+ */
+ emit_trace_rcu_reader(ctx, true);
/* reserve a nop for bpf_tramp_image_put */
im->ip_after_call = ctx->ro_image + ctx->idx;
emit(A64_NOP, ctx);
@@ -2945,6 +3032,9 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
if (flags & BPF_TRAMP_F_RESTORE_REGS)
restore_args(ctx, bargs_off, a->regs_for_args);
+ /* Remaining instructions are covered by the irq-exit IP check. */
+ emit_trace_rcu_reader(ctx, false);
+
/* restore callee saved register x19 and x20 */
emit(A64_LDR64I(A64_R(19), A64_SP, regs_off), ctx);
emit(A64_LDR64I(A64_R(20), A64_SP, regs_off + 8), ctx);
--
2.55.0
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH RFC v4 10/13] samples: ftrace: Make the direct-call trampolines Tasks Trace readers
2026-09-18 14:49 [PATCH RFC v4 00/13] rcu-tasks: build Tasks RCU on Tasks Trace readers in trampolines Josef Bacik
` (8 preceding siblings ...)
2026-09-18 14:49 ` [PATCH RFC v4 09/13] bpf, arm64: Take a Tasks Trace reader in the trampoline around its call-outs Josef Bacik
@ 2026-09-18 14:49 ` Josef Bacik
2026-09-18 14:49 ` [PATCH RFC v4 11/13] rcutorture: Make Tasks RCU readers Tasks Trace readers where required Josef Bacik
` (2 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Josef Bacik @ 2026-09-18 14:49 UTC (permalink / raw)
To: Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai,
Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai
Cc: Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Josef Bacik
The sample direct trampolines are exactly the kind of out-of-line
register_ftrace_direct() user whose lifetime depends on Tasks RCU
waiting for a task inside them: nothing else stops rmmod while a task is
preempted in my_direct_func(). On HAVE_RCU_TRAMPOLINE_READERS
architectures that wait only covers Tasks Trace RCU readers, so give the
samples a small shared header with rcu_read_lock_trace() and
rcu_read_unlock_trace() open-coded as instruction strings for x86-64 and
arm64 -- the same sequences as ftrace_64.S and entry-ftrace.S, using
caller-saved non-argument scratch registers -- and bracket every
call-out with them. The instructions outside the bracket are module
text, covered by ftrace_direct_mark_module(). Other architectures get
empty definitions.
Assisted-by: LLM
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
---
samples/ftrace/ftrace-direct-modify.c | 9 ++
samples/ftrace/ftrace-direct-multi-modify.c | 9 ++
samples/ftrace/ftrace-direct-multi.c | 5 ++
samples/ftrace/ftrace-direct-too.c | 5 ++
samples/ftrace/ftrace-direct.c | 5 ++
samples/ftrace/ftrace-direct.h | 126 ++++++++++++++++++++++++++++
6 files changed, 159 insertions(+)
diff --git a/samples/ftrace/ftrace-direct-modify.c b/samples/ftrace/ftrace-direct-modify.c
index 164d9dd6fd92..937c8d8c2a1b 100644
--- a/samples/ftrace/ftrace-direct-modify.c
+++ b/samples/ftrace/ftrace-direct-modify.c
@@ -2,6 +2,7 @@
#include <linux/module.h>
#include <linux/kthread.h>
#include <linux/ftrace.h>
+#include "ftrace-direct.h"
#if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32)
#include <asm/asm-offsets.h>
#endif
@@ -73,7 +74,9 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ TRACE_RCU_READ_LOCK
" call my_direct_func1\n"
+ TRACE_RCU_READ_UNLOCK
" leave\n"
ASM_RET
" .size my_tramp1, .-my_tramp1\n"
@@ -85,7 +88,9 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ TRACE_RCU_READ_LOCK
" call my_direct_func2\n"
+ TRACE_RCU_READ_UNLOCK
" leave\n"
ASM_RET
" .size my_tramp2, .-my_tramp2\n"
@@ -141,11 +146,13 @@ asm (
" .globl my_tramp1\n"
" my_tramp1:"
" hint 34\n" // bti c
+ TRACE_RCU_READ_LOCK
" sub sp, sp, #16\n"
" stp x9, x30, [sp]\n"
" bl my_direct_func1\n"
" ldp x30, x9, [sp]\n"
" add sp, sp, #16\n"
+ TRACE_RCU_READ_UNLOCK
" ret x9\n"
" .size my_tramp1, .-my_tramp1\n"
@@ -153,11 +160,13 @@ asm (
" .globl my_tramp2\n"
" my_tramp2:"
" hint 34\n" // bti c
+ TRACE_RCU_READ_LOCK
" sub sp, sp, #16\n"
" stp x9, x30, [sp]\n"
" bl my_direct_func2\n"
" ldp x30, x9, [sp]\n"
" add sp, sp, #16\n"
+ TRACE_RCU_READ_UNLOCK
" ret x9\n"
" .size my_tramp2, .-my_tramp2\n"
" .popsection\n"
diff --git a/samples/ftrace/ftrace-direct-multi-modify.c b/samples/ftrace/ftrace-direct-multi-modify.c
index b03766c6217b..e12e5c8b83f0 100644
--- a/samples/ftrace/ftrace-direct-multi-modify.c
+++ b/samples/ftrace/ftrace-direct-multi-modify.c
@@ -2,6 +2,7 @@
#include <linux/module.h>
#include <linux/kthread.h>
#include <linux/ftrace.h>
+#include "ftrace-direct.h"
#if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32)
#include <asm/asm-offsets.h>
#endif
@@ -77,10 +78,12 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ TRACE_RCU_READ_LOCK
" pushq %rdi\n"
" movq 8(%rbp), %rdi\n"
" call my_direct_func1\n"
" popq %rdi\n"
+ TRACE_RCU_READ_UNLOCK
" leave\n"
ASM_RET
" .size my_tramp1, .-my_tramp1\n"
@@ -92,10 +95,12 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ TRACE_RCU_READ_LOCK
" pushq %rdi\n"
" movq 8(%rbp), %rdi\n"
" call my_direct_func2\n"
" popq %rdi\n"
+ TRACE_RCU_READ_UNLOCK
" leave\n"
ASM_RET
" .size my_tramp2, .-my_tramp2\n"
@@ -154,6 +159,7 @@ asm (
" .globl my_tramp1\n"
" my_tramp1:"
" hint 34\n" // bti c
+ TRACE_RCU_READ_LOCK
" sub sp, sp, #32\n"
" stp x9, x30, [sp]\n"
" str x0, [sp, #16]\n"
@@ -162,6 +168,7 @@ asm (
" ldp x30, x9, [sp]\n"
" ldr x0, [sp, #16]\n"
" add sp, sp, #32\n"
+ TRACE_RCU_READ_UNLOCK
" ret x9\n"
" .size my_tramp1, .-my_tramp1\n"
@@ -169,6 +176,7 @@ asm (
" .globl my_tramp2\n"
" my_tramp2:"
" hint 34\n" // bti c
+ TRACE_RCU_READ_LOCK
" sub sp, sp, #32\n"
" stp x9, x30, [sp]\n"
" str x0, [sp, #16]\n"
@@ -177,6 +185,7 @@ asm (
" ldp x30, x9, [sp]\n"
" ldr x0, [sp, #16]\n"
" add sp, sp, #32\n"
+ TRACE_RCU_READ_UNLOCK
" ret x9\n"
" .size my_tramp2, .-my_tramp2\n"
" .popsection\n"
diff --git a/samples/ftrace/ftrace-direct-multi.c b/samples/ftrace/ftrace-direct-multi.c
index 3fe6ddaf0b69..a970464ed378 100644
--- a/samples/ftrace/ftrace-direct-multi.c
+++ b/samples/ftrace/ftrace-direct-multi.c
@@ -3,6 +3,7 @@
#include <linux/mm.h> /* for handle_mm_fault() */
#include <linux/ftrace.h>
+#include "ftrace-direct.h"
#include <linux/sched/stat.h>
#if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32)
#include <asm/asm-offsets.h>
@@ -56,10 +57,12 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ TRACE_RCU_READ_LOCK
" pushq %rdi\n"
" movq 8(%rbp), %rdi\n"
" call my_direct_func\n"
" popq %rdi\n"
+ TRACE_RCU_READ_UNLOCK
" leave\n"
ASM_RET
" .size my_tramp, .-my_tramp\n"
@@ -101,6 +104,7 @@ asm (
" .globl my_tramp\n"
" my_tramp:"
" hint 34\n" // bti c
+ TRACE_RCU_READ_LOCK
" sub sp, sp, #32\n"
" stp x9, x30, [sp]\n"
" str x0, [sp, #16]\n"
@@ -109,6 +113,7 @@ asm (
" ldp x30, x9, [sp]\n"
" ldr x0, [sp, #16]\n"
" add sp, sp, #32\n"
+ TRACE_RCU_READ_UNLOCK
" ret x9\n"
" .size my_tramp, .-my_tramp\n"
" .popsection\n"
diff --git a/samples/ftrace/ftrace-direct-too.c b/samples/ftrace/ftrace-direct-too.c
index bf2411aa6fd7..abc098c2ab7a 100644
--- a/samples/ftrace/ftrace-direct-too.c
+++ b/samples/ftrace/ftrace-direct-too.c
@@ -3,6 +3,7 @@
#include <linux/mm.h> /* for handle_mm_fault() */
#include <linux/ftrace.h>
+#include "ftrace-direct.h"
#if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32)
#include <asm/asm-offsets.h>
#endif
@@ -61,6 +62,7 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ TRACE_RCU_READ_LOCK
" pushq %rdi\n"
" pushq %rsi\n"
" pushq %rdx\n"
@@ -70,6 +72,7 @@ asm (
" popq %rdx\n"
" popq %rsi\n"
" popq %rdi\n"
+ TRACE_RCU_READ_UNLOCK
" leave\n"
ASM_RET
" .size my_tramp, .-my_tramp\n"
@@ -110,6 +113,7 @@ asm (
" .globl my_tramp\n"
" my_tramp:"
" hint 34\n" // bti c
+ TRACE_RCU_READ_LOCK
" sub sp, sp, #48\n"
" stp x9, x30, [sp]\n"
" stp x0, x1, [sp, #16]\n"
@@ -119,6 +123,7 @@ asm (
" ldp x0, x1, [sp, #16]\n"
" ldp x2, x3, [sp, #32]\n"
" add sp, sp, #48\n"
+ TRACE_RCU_READ_UNLOCK
" ret x9\n"
" .size my_tramp, .-my_tramp\n"
" .popsection\n"
diff --git a/samples/ftrace/ftrace-direct.c b/samples/ftrace/ftrace-direct.c
index 5368c8c39cbb..99b65ad2fccc 100644
--- a/samples/ftrace/ftrace-direct.c
+++ b/samples/ftrace/ftrace-direct.c
@@ -3,6 +3,7 @@
#include <linux/sched.h> /* for wake_up_process() */
#include <linux/ftrace.h>
+#include "ftrace-direct.h"
#if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32)
#include <asm/asm-offsets.h>
#endif
@@ -54,9 +55,11 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ TRACE_RCU_READ_LOCK
" pushq %rdi\n"
" call my_direct_func\n"
" popq %rdi\n"
+ TRACE_RCU_READ_UNLOCK
" leave\n"
ASM_RET
" .size my_tramp, .-my_tramp\n"
@@ -97,6 +100,7 @@ asm (
" .globl my_tramp\n"
" my_tramp:"
" hint 34\n" // bti c
+ TRACE_RCU_READ_LOCK
" sub sp, sp, #32\n"
" stp x9, x30, [sp]\n"
" str x0, [sp, #16]\n"
@@ -104,6 +108,7 @@ asm (
" ldp x30, x9, [sp]\n"
" ldr x0, [sp, #16]\n"
" add sp, sp, #32\n"
+ TRACE_RCU_READ_UNLOCK
" ret x9\n"
" .size my_tramp, .-my_tramp\n"
" .popsection\n"
diff --git a/samples/ftrace/ftrace-direct.h b/samples/ftrace/ftrace-direct.h
new file mode 100644
index 000000000000..726f67048ff5
--- /dev/null
+++ b/samples/ftrace/ftrace-direct.h
@@ -0,0 +1,126 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _SAMPLES_FTRACE_DIRECT_H
+#define _SAMPLES_FTRACE_DIRECT_H
+
+#include <linux/stringify.h>
+
+/*
+ * A direct-call trampoline is entered with no lock, refcount or RCU marker
+ * held; only Tasks RCU keeps it (and, for a module, its text) alive while a
+ * task is inside it or preempted in something it called. On architectures
+ * that select HAVE_RCU_TRAMPOLINE_READERS, Tasks RCU only waits for such a
+ * task while it is a Tasks Trace RCU reader, so the trampoline must enter one
+ * before calling out and leave it afterwards, exactly like the ftrace and BPF
+ * trampolines do. See register_ftrace_direct(). The instructions before the
+ * lock and after the unlock are covered by ftrace_direct_mark_module().
+ *
+ * These are rcu_read_lock_trace() / rcu_read_unlock_trace() open-coded as
+ * instruction strings for use inside the samples' asm() trampolines, after
+ * the versions in arch/x86/kernel/ftrace_64.S and
+ * arch/arm64/kernel/entry-ftrace.S. The scratch registers are caller-saved
+ * and not argument registers, so they are dead on entry to and exit from an
+ * fentry trampoline; the flags are clobbered.
+ *
+ * The generated asm-offsets.h is only pulled in on the architectures that need
+ * it here: it is not generally safe to include from C (e.g. PPC32's TASK_SIZE
+ * and arm64's TRAMP_VALIAS clash with the C definitions).
+ */
+#if defined(CONFIG_TASKS_RCU_TRAMPOLINE_READERS) && defined(CONFIG_X86_64)
+
+#include <asm/asm-offsets.h>
+
+#ifndef CONFIG_TASKS_TRACE_RCU_NO_MB
+#define TRACE_RCU_MB " lock addl $0, -4(%rsp)\n"
+#else
+#define TRACE_RCU_MB
+#endif
+
+#define TRACE_RCU_READ_LOCK \
+ " movq %gs:current_task(%rip), %r11\n" \
+ " movl " __stringify(TASK_trc_reader_nesting) "(%r11), %r10d\n" \
+ " incl " __stringify(TASK_trc_reader_nesting) "(%r11)\n" \
+ " testl %r10d, %r10d\n" \
+ " jnz 771f\n" \
+ " movq rcu_tasks_trace_srcu_struct+" __stringify(SRCU_srcu_ctrp) "(%rip), %r10\n" \
+ " incq %gs:" __stringify(SRCU_CTR_srcu_locks) "(%r10)\n" \
+ " movq %r10, " __stringify(TASK_trc_reader_scp) "(%r11)\n" \
+ TRACE_RCU_MB \
+ "771:\n"
+
+#define TRACE_RCU_READ_UNLOCK \
+ " movq %gs:current_task(%rip), %r11\n" \
+ " movl " __stringify(TASK_trc_reader_nesting) "(%r11), %r10d\n" \
+ " subl $1, %r10d\n" \
+ " jnz 772f\n" \
+ " movq " __stringify(TASK_trc_reader_scp) "(%r11), %r10\n" \
+ " movl $0, " __stringify(TASK_trc_reader_nesting) "(%r11)\n" \
+ TRACE_RCU_MB \
+ " incq %gs:" __stringify(SRCU_CTR_srcu_unlocks) "(%r10)\n" \
+ " jmp 773f\n" \
+ "772: movl %r10d, " __stringify(TASK_trc_reader_nesting) "(%r11)\n" \
+ "773:\n"
+
+#elif defined(CONFIG_TASKS_RCU_TRAMPOLINE_READERS) && defined(CONFIG_ARM64)
+
+#include <asm/alternative-macros.h>
+#include <asm/cpucaps.h>
+/* arm64's asm-offsets.h redefines TRAMP_VALIAS from <asm/fixmap.h>. */
+#pragma push_macro("TRAMP_VALIAS")
+#undef TRAMP_VALIAS
+#include <asm/asm-offsets.h>
+#pragma pop_macro("TRAMP_VALIAS")
+
+#ifndef CONFIG_TASKS_TRACE_RCU_NO_MB
+#define TRACE_RCU_MB " dmb ish\n"
+#else
+#define TRACE_RCU_MB
+#endif
+
+#define TRACE_RCU_SRCU_CTRP "rcu_tasks_trace_srcu_struct+" __stringify(SRCU_SRCU_CTRP)
+
+/* x14 = this CPU's offset; then atomically increment the long at x14 + \areg */
+#define TRACE_RCU_PERCPU_INC(areg) \
+ ALTERNATIVE(" mrs x14, tpidr_el1\n", " mrs x14, tpidr_el2\n", \
+ ARM64_HAS_VIRT_HOST_EXTN) \
+ " add x14, x14, " areg "\n" \
+ "778: ldxr x15, [x14]\n" \
+ " add x15, x15, #1\n" \
+ " stxr w16, x15, [x14]\n" \
+ " cbnz w16, 778b\n"
+
+#define TRACE_RCU_READ_LOCK \
+ " mrs x12, sp_el0\n" \
+ " ldr w13, [x12, #" __stringify(TSK_TRC_READER_NESTING) "]\n" \
+ " add w14, w13, #1\n" \
+ " str w14, [x12, #" __stringify(TSK_TRC_READER_NESTING) "]\n" \
+ " cbnz w13, 771f\n" \
+ " adrp x13, " TRACE_RCU_SRCU_CTRP "\n" \
+ " ldr x13, [x13, #:lo12:" TRACE_RCU_SRCU_CTRP "]\n" \
+ " str x13, [x12, #" __stringify(TSK_TRC_READER_SCP) "]\n" \
+ " add x13, x13, #" __stringify(SRCU_CTR_SRCU_LOCKS) "\n" \
+ TRACE_RCU_PERCPU_INC("x13") \
+ TRACE_RCU_MB \
+ "771:\n"
+
+#define TRACE_RCU_READ_UNLOCK \
+ " mrs x12, sp_el0\n" \
+ " ldr w13, [x12, #" __stringify(TSK_TRC_READER_NESTING) "]\n" \
+ " subs w13, w13, #1\n" \
+ " b.ne 772f\n" \
+ " ldr x13, [x12, #" __stringify(TSK_TRC_READER_SCP) "]\n" \
+ " str wzr, [x12, #" __stringify(TSK_TRC_READER_NESTING) "]\n" \
+ TRACE_RCU_MB \
+ " add x13, x13, #" __stringify(SRCU_CTR_SRCU_UNLOCKS) "\n" \
+ TRACE_RCU_PERCPU_INC("x13") \
+ " b 773f\n" \
+ "772: str w13, [x12, #" __stringify(TSK_TRC_READER_NESTING) "]\n" \
+ "773:\n"
+
+#else
+
+#define TRACE_RCU_READ_LOCK
+#define TRACE_RCU_READ_UNLOCK
+
+#endif
+
+#endif /* _SAMPLES_FTRACE_DIRECT_H */
--
2.55.0
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH RFC v4 11/13] rcutorture: Make Tasks RCU readers Tasks Trace readers where required
2026-09-18 14:49 [PATCH RFC v4 00/13] rcu-tasks: build Tasks RCU on Tasks Trace readers in trampolines Josef Bacik
` (9 preceding siblings ...)
2026-09-18 14:49 ` [PATCH RFC v4 10/13] samples: ftrace: Make the direct-call trampolines Tasks Trace readers Josef Bacik
@ 2026-09-18 14:49 ` Josef Bacik
2026-09-18 14:49 ` [PATCH RFC v4 12/13] rcu-tasks-trace: Assert no reader is held on return to userspace Josef Bacik
2026-09-18 14:49 ` [PATCH RFC v4 13/13] x86, arm64: Build Tasks RCU on Tasks Trace readers in trampolines Josef Bacik
12 siblings, 0 replies; 15+ messages in thread
From: Josef Bacik @ 2026-09-18 14:49 UTC (permalink / raw)
To: Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai,
Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai
Cc: Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Josef Bacik
rcutorture's "tasks" flavor has empty readlock/readunlock hooks because
a classic Tasks RCU reader is simply code that does not block. Under
CONFIG_TASKS_RCU_TRAMPOLINE_READERS a preemption outside trampoline text
is also a quiescent state, and the thing real readers (trampolines) do
to stay protected across their call-outs is take rcu_read_lock_trace(),
so have the torture readers do the same there. Otherwise a preempted
torture reader would rightly be treated as quiescent and the test would
report false-positive too-short grace periods.
Assisted-by: LLM
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
---
kernel/rcu/rcutorture.c | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 794937e13e7c..ab870ef09af0 100644
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -1142,13 +1142,23 @@ static struct rcu_torture_ops trivial_preempt_ops = {
* Definitions for RCU-tasks torture testing.
*/
+/*
+ * A classic Tasks RCU reader is any stretch of kernel code that does not
+ * voluntarily block. With CONFIG_TASKS_RCU_TRAMPOLINE_READERS a preemption
+ * outside trampoline text also ends it, and what a trampoline does to stay
+ * protected across its call-out is take a Tasks Trace reader, so model that.
+ */
static int tasks_torture_read_lock(void)
{
+ if (IS_ENABLED(CONFIG_TASKS_RCU_TRAMPOLINE_READERS))
+ rcu_read_lock_trace();
return 0;
}
static void tasks_torture_read_unlock(int idx)
{
+ if (IS_ENABLED(CONFIG_TASKS_RCU_TRAMPOLINE_READERS))
+ rcu_read_unlock_trace();
}
static void rcu_tasks_torture_deferred_free(struct rcu_torture *p)
--
2.55.0
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH RFC v4 12/13] rcu-tasks-trace: Assert no reader is held on return to userspace
2026-09-18 14:49 [PATCH RFC v4 00/13] rcu-tasks: build Tasks RCU on Tasks Trace readers in trampolines Josef Bacik
` (10 preceding siblings ...)
2026-09-18 14:49 ` [PATCH RFC v4 11/13] rcutorture: Make Tasks RCU readers Tasks Trace readers where required Josef Bacik
@ 2026-09-18 14:49 ` Josef Bacik
2026-09-18 14:49 ` [PATCH RFC v4 13/13] x86, arm64: Build Tasks RCU on Tasks Trace readers in trampolines Josef Bacik
12 siblings, 0 replies; 15+ messages in thread
From: Josef Bacik @ 2026-09-18 14:49 UTC (permalink / raw)
To: Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai,
Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai
Cc: Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Josef Bacik
With trampolines now taking rcu_read_lock_trace() open-coded from
assembly and JIT-emitted code, an unbalanced reader would silently
turn every later Tasks Trace grace period on that task into a stall.
No task can legitimately reach userspace with current->trc_reader_nesting
non-zero, so under CONFIG_PROVE_RCU check it in the generic entry
code's return-to-user validation, next to the existing kmap and lockdep
assertions. Compiles away otherwise.
Assisted-by: LLM
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
---
include/linux/irq-entry-common.h | 2 ++
include/linux/rcupdate_trace.h | 7 +++++++
2 files changed, 9 insertions(+)
diff --git a/include/linux/irq-entry-common.h b/include/linux/irq-entry-common.h
index e19b41ee6b18..d42568a488a7 100644
--- a/include/linux/irq-entry-common.h
+++ b/include/linux/irq-entry-common.h
@@ -5,6 +5,7 @@
#include <linux/context_tracking.h>
#include <linux/hrtimer_rearm.h>
#include <linux/kmsan.h>
+#include <linux/rcupdate_trace.h>
#include <linux/rseq_entry.h>
#include <linux/static_call_types.h>
#include <linux/syscalls.h>
@@ -214,6 +215,7 @@ static __always_inline void __exit_to_user_mode_validate(void)
{
/* Ensure that kernel state is sane for a return to userspace */
kmap_assert_nomap();
+ rcu_tasks_trace_assert_idle();
lockdep_assert_irqs_disabled();
lockdep_sys_exit();
}
diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h
index 273c59a03251..6034c6b19479 100644
--- a/include/linux/rcupdate_trace.h
+++ b/include/linux/rcupdate_trace.h
@@ -211,6 +211,12 @@ unsigned long rcu_tasks_trace_batches_completed(void);
// Placeholders to enable stepwise transition.
void __init rcu_tasks_trace_suppress_unused(void);
+/* A task must never reach userspace inside an rcu_read_lock_trace() reader. */
+static inline void rcu_tasks_trace_assert_idle(void)
+{
+ WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && READ_ONCE(current->trc_reader_nesting));
+}
+
#else
static inline unsigned long rcu_tasks_trace_batches_completed(void) { return 0; }
/*
@@ -220,6 +226,7 @@ static inline unsigned long rcu_tasks_trace_batches_completed(void) { return 0;
static inline void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func) { BUG(); }
static inline void rcu_read_lock_trace(void) { BUG(); }
static inline void rcu_read_unlock_trace(void) { BUG(); }
+static inline void rcu_tasks_trace_assert_idle(void) { }
#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
DEFINE_LOCK_GUARD_0(rcu_tasks_trace,
--
2.55.0
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH RFC v4 13/13] x86, arm64: Build Tasks RCU on Tasks Trace readers in trampolines
2026-09-18 14:49 [PATCH RFC v4 00/13] rcu-tasks: build Tasks RCU on Tasks Trace readers in trampolines Josef Bacik
` (11 preceding siblings ...)
2026-09-18 14:49 ` [PATCH RFC v4 12/13] rcu-tasks-trace: Assert no reader is held on return to userspace Josef Bacik
@ 2026-09-18 14:49 ` Josef Bacik
12 siblings, 0 replies; 15+ messages in thread
From: Josef Bacik @ 2026-09-18 14:49 UTC (permalink / raw)
To: Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai,
Paul E. McKenney, Frederic Weisbecker, Neeraj Upadhyay,
Joel Fernandes, Boqun Feng, Thomas Gleixner, Peter Zijlstra,
Steven Rostedt, Masami Hiramatsu, Mark Rutland, Jiri Olsa,
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, x86,
Catalin Marinas, Will Deacon, Puranjay Mohan, Xu Kuohai
Cc: Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Andy Lutomirski, Josh Triplett, Uladzislau Rezki,
Mathieu Desnoyers, Lai Jiangshan, Zqiang, Juergen Gross,
Luis Chamberlain, Ihor Solodrai, linux-kernel, rcu,
linux-trace-kernel, bpf, linux-arm-kernel, xen-devel,
Josef Bacik
With the preceding patches every trampoline whose lifetime Tasks RCU
guards on x86-64 and arm64 -- ftrace_caller and its copies, the optprobe
template, BPF trampolines, and the sample direct-call
trampolines -- is a Tasks Trace RCU reader around its call-out, and the
text outside that reader is known to rcu_tasks_trampoline_text().
Select HAVE_RCU_TRAMPOLINE_READERS on both (x86-64 with SMP for Tree
SRCU and DYNAMIC_FTRACE, which is where its ftrace_caller changes and
arch_rcu_tasks_trampoline_text() live; arm64 with
DYNAMIC_FTRACE_WITH_ARGS likewise), which switches CONFIG_TASKS_RCU to the
implementation added earlier in the series: a Tasks RCU grace period
becomes a per-CPU pass over context switches and irq-exit reschedules
outside trampoline text plus a Tasks Trace grace period, bounded by a
few jiffies and preempt-off latency instead of by the longest stretch
any task runs without sleeping.
Other architectures keep the classic implementation. Update
Documentation/RCU and the FORCE_TASKS_RCU help text to describe the
variant and the obligation it places on trampolines.
Assisted-by: LLM
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
---
.../RCU/Design/Requirements/Requirements.rst | 24 ++++++++++++++++++++++
Documentation/RCU/checklist.rst | 7 ++++++-
arch/arm64/Kconfig | 1 +
arch/x86/Kconfig | 1 +
kernel/rcu/Kconfig | 6 ++++--
5 files changed, 36 insertions(+), 3 deletions(-)
diff --git a/Documentation/RCU/Design/Requirements/Requirements.rst b/Documentation/RCU/Design/Requirements/Requirements.rst
index 8101fe6229d5..c059c463e083 100644
--- a/Documentation/RCU/Design/Requirements/Requirements.rst
+++ b/Documentation/RCU/Design/Requirements/Requirements.rst
@@ -2756,6 +2756,30 @@ synchronize_rcu(), and rcu_barrier(), respectively. In
three APIs are therefore implemented by separate functions that check
for voluntary context switches.
+Architectures that select ``CONFIG_HAVE_RCU_TRAMPOLINE_READERS`` keep the
+same three APIs but implement the grace period differently
+(``CONFIG_TASKS_RCU_TRAMPOLINE_READERS``). There, every trampoline whose
+lifetime Tasks RCU guards enters a Tasks Trace RCU read-side critical
+section (rcu_read_lock_trace() or its assembly equivalent) before calling
+out and leaves it before returning, so a task anywhere inside such a
+call-out is an ordinary Tasks Trace reader whether or not it is
+preempted. The few trampoline instructions outside that reader can only
+be occupied by a task that was interrupted there, so the grace period
+additionally waits for each CPU to pass through a context switch, and the
+irq-exit preemption path, the only switch that can catch a task inside
+such text (rcu_tasks_trampoline_text()), briefly makes such a task a
+holdout until it is next seen elsewhere. On such kernels an involuntary
+context switch outside trampoline text *is* a Tasks-RCU quiescent state,
+a Tasks RCU grace period no longer depends on how long any task runs
+without sleeping, cond_resched_tasks_rcu_qs() is unnecessary, and the
+obligation moves to the trampolines: anything that relies on
+synchronize_rcu_tasks() to protect code a task may be preempted in must
+take the Tasks Trace reader (see register_ftrace_direct()), or, where
+that is impossible because the code is ordinary text with no trampoline
+of its own, make it known to rcu_tasks_trampoline_text() and wait out
+tasks already parked there with rcu_tasks_wait_irq_preempted(), as the
+kprobe jump optimizer does.
+
Tasks Rude RCU
~~~~~~~~~~~~~~
diff --git a/Documentation/RCU/checklist.rst b/Documentation/RCU/checklist.rst
index 4b30f701225f..87dd506bffef 100644
--- a/Documentation/RCU/checklist.rst
+++ b/Documentation/RCU/checklist.rst
@@ -252,7 +252,12 @@ over a rather long period of time, but improvements are always welcome!
a. If the updater uses synchronize_rcu_tasks() or
call_rcu_tasks(), then the readers must refrain from
executing voluntary context switches, that is, from
- blocking.
+ blocking. On architectures that select
+ CONFIG_HAVE_RCU_TRAMPOLINE_READERS a reader must in
+ addition be a Tasks Trace RCU reader (that is what the
+ trampolines there do around their call-outs) or be text
+ that rcu_tasks_trampoline_text() recognises; an arbitrary
+ stretch of preemptible kernel code is not protected.
b. If the updater uses call_rcu_tasks_trace()
or synchronize_rcu_tasks_trace(), then the
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index b5a51b0ef944..bf0e56006863 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -218,6 +218,7 @@ config ARM64
select HAVE_PERF_REGS
select HAVE_PERF_USER_STACK_DUMP
select HAVE_PREEMPT_DYNAMIC_KEY
+ select HAVE_RCU_TRAMPOLINE_READERS if DYNAMIC_FTRACE_WITH_ARGS
select HAVE_REGS_AND_STACK_ACCESS_API
select HAVE_RELIABLE_STACKTRACE
select HAVE_POSIX_CPU_TIMERS_TASK_WORK
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 15fd9ec5ecac..64c3814eb745 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -288,6 +288,7 @@ config X86
select MMU_GATHER_RCU_TABLE_FREE
select MMU_GATHER_MERGE_VMAS
select HAVE_POSIX_CPU_TIMERS_TASK_WORK
+ select HAVE_RCU_TRAMPOLINE_READERS if X86_64 && SMP && DYNAMIC_FTRACE
select HAVE_REGS_AND_STACK_ACCESS_API
select HAVE_RELIABLE_STACKTRACE if UNWINDER_ORC || STACK_VALIDATION
select HAVE_FUNCTION_ARG_ACCESS_API
diff --git a/kernel/rcu/Kconfig b/kernel/rcu/Kconfig
index bbab14bc14c3..341b68b972ef 100644
--- a/kernel/rcu/Kconfig
+++ b/kernel/rcu/Kconfig
@@ -95,8 +95,10 @@ config FORCE_TASKS_RCU
help
This option force-enables a task-based RCU implementation
that uses only voluntary context switch (not preemption!),
- idle, and user-mode execution as quiescent states. Not for
- manual selection in most cases.
+ idle, and user-mode execution as quiescent states, or, on
+ HAVE_RCU_TRAMPOLINE_READERS architectures, the variant built on
+ Tasks Trace RCU readers in trampolines. Not for manual
+ selection in most cases.
config NEED_TASKS_RCU
bool
--
2.55.0
^ permalink raw reply [flat|nested] 15+ messages in thread