From: Song Liu <song@kernel.org>
To: linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org,
live-patching@vger.kernel.org
Cc: rostedt@goodmis.org, mhiramat@kernel.org, mark.rutland@arm.com,
kernel-team@meta.com, Song Liu <song@kernel.org>
Subject: [PATCH v2 04/16] bpf: Mark the BPF trampoline helpers notrace
Date: Thu, 22 Oct 2026 18:10:37 -0700 [thread overview]
Message-ID: <20261023011049.1164625-5-song@kernel.org> (raw)
In-Reply-To: <20261023011049.1164625-1-song@kernel.org>
The BPF trampoline calls __bpf_prog_enter*() before each program and
__bpf_prog_exit*() after it, and calls __bpf_tramp_enter() and
__bpf_tramp_exit() around the traced function. These functions are
notrace: if anything they call could run a BPF program, attaching a
program there would recurse without bound. The recursion check in
__bpf_prog_enter_recur() does not prevent this, because the trampoline
calls the enter and exit functions even for a program it then skips.
For the same reason, btf_id_deny already rejects attaching to the
out-of-line migrate_disable() and migrate_enable(), and, with
CONFIG_PREEMPT_RCU, to __rcu_read_lock() and __rcu_read_unlock().
Many of the helpers they call are plain inline functions:
- bpf_set_run_ctx(), bpf_reset_run_ctx(),
bpf_prog_get_recursion_context() and
bpf_prog_put_recursion_context();
- __migrate_disable() and __migrate_enable();
- the !CONFIG_PREEMPT_RCU __rcu_read_lock() and __rcu_read_unlock(),
rcu_read_unlock() and rcu_read_unlock_migrate(), and the lockdep
annotations rcu_lock_acquire(), rcu_try_lock_acquire() and
rcu_lock_release();
- rcu_read_lock_trace(), rcu_read_unlock_trace() and
srcu_lock_release();
- percpu_ref_get(), percpu_ref_put(), percpu_ref_get_many(),
percpu_ref_put_many() and __ref_is_percpu();
- local_inc() and local_add() on x86, called through u64_stats_inc()
and u64_stats_add() from bpf_prog_inc_misses_counter() and
__update_prog_stats().
An out-of-line copy of any of them is untraced only because inline
implies notrace. Mark them notrace explicitly. No functional change.
Assisted-by: Claude:claude-opus-5
Signed-off-by: Song Liu <song@kernel.org>
---
arch/x86/include/asm/local.h | 4 ++--
include/linux/bpf.h | 8 ++++----
include/linux/percpu-refcount.h | 12 ++++++------
include/linux/rcupdate.h | 14 +++++++-------
include/linux/rcupdate_trace.h | 4 ++--
include/linux/sched.h | 8 ++++----
include/linux/srcu.h | 2 +-
7 files changed, 26 insertions(+), 26 deletions(-)
diff --git a/arch/x86/include/asm/local.h b/arch/x86/include/asm/local.h
index 4957018fef3e..1749c4481000 100644
--- a/arch/x86/include/asm/local.h
+++ b/arch/x86/include/asm/local.h
@@ -16,7 +16,7 @@ typedef struct {
#define local_read(l) atomic_long_read(&(l)->a)
#define local_set(l, i) atomic_long_set(&(l)->a, (i))
-static inline void local_inc(local_t *l)
+static inline void notrace local_inc(local_t *l)
{
asm volatile(_ASM_INC "%0"
: "+m" (l->a.counter));
@@ -28,7 +28,7 @@ static inline void local_dec(local_t *l)
: "+m" (l->a.counter));
}
-static inline void local_add(long i, local_t *l)
+static inline void notrace local_add(long i, local_t *l)
{
asm volatile(_ASM_ADD "%1,%0"
: "+m" (l->a.counter)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 7719f6528445..22bf8e285117 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -2156,7 +2156,7 @@ struct bpf_struct_ops_common_value {
enum bpf_struct_ops_state state;
};
-static inline bool bpf_prog_get_recursion_context(struct bpf_prog *prog)
+static inline bool notrace bpf_prog_get_recursion_context(struct bpf_prog *prog)
{
#ifdef CONFIG_ARM64
u8 rctx = interrupt_context_level();
@@ -2176,7 +2176,7 @@ static inline bool bpf_prog_get_recursion_context(struct bpf_prog *prog)
#endif
}
-static inline void bpf_prog_put_recursion_context(struct bpf_prog *prog)
+static inline void notrace bpf_prog_put_recursion_context(struct bpf_prog *prog)
{
#ifdef CONFIG_ARM64
u8 rctx = interrupt_context_level();
@@ -2547,7 +2547,7 @@ struct bpf_tramp_run_ctx {
struct bpf_run_ctx *saved_run_ctx;
};
-static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx)
+static inline notrace struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx)
{
struct bpf_run_ctx *old_ctx = NULL;
@@ -2558,7 +2558,7 @@ static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx)
return old_ctx;
}
-static inline void bpf_reset_run_ctx(struct bpf_run_ctx *old_ctx)
+static inline void notrace bpf_reset_run_ctx(struct bpf_run_ctx *old_ctx)
{
#ifdef CONFIG_BPF_SYSCALL
current->bpf_ctx = old_ctx;
diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h
index d73a1c08c3e3..41db7d25b802 100644
--- a/include/linux/percpu-refcount.h
+++ b/include/linux/percpu-refcount.h
@@ -155,8 +155,8 @@ static inline void percpu_ref_kill(struct percpu_ref *ref)
* because doing so forces the compiler to generate two conditional
* branches as it can't assume that @ref->percpu_count is not NULL.
*/
-static inline bool __ref_is_percpu(struct percpu_ref *ref,
- unsigned long __percpu **percpu_countp)
+static inline bool notrace __ref_is_percpu(struct percpu_ref *ref,
+ unsigned long __percpu **percpu_countp)
{
unsigned long percpu_ptr;
@@ -195,7 +195,7 @@ static inline bool __ref_is_percpu(struct percpu_ref *ref,
*
* This function is safe to call as long as @ref is between init and exit.
*/
-static inline void percpu_ref_get_many(struct percpu_ref *ref, unsigned long nr)
+static inline void notrace percpu_ref_get_many(struct percpu_ref *ref, unsigned long nr)
{
unsigned long __percpu *percpu_count;
@@ -217,7 +217,7 @@ static inline void percpu_ref_get_many(struct percpu_ref *ref, unsigned long nr)
*
* This function is safe to call as long as @ref is between init and exit.
*/
-static inline void percpu_ref_get(struct percpu_ref *ref)
+static inline void notrace percpu_ref_get(struct percpu_ref *ref)
{
percpu_ref_get_many(ref, 1);
}
@@ -323,7 +323,7 @@ static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
*
* This function is safe to call as long as @ref is between init and exit.
*/
-static inline void percpu_ref_put_many(struct percpu_ref *ref, unsigned long nr)
+static inline void notrace percpu_ref_put_many(struct percpu_ref *ref, unsigned long nr)
{
unsigned long __percpu *percpu_count;
@@ -346,7 +346,7 @@ static inline void percpu_ref_put_many(struct percpu_ref *ref, unsigned long nr)
*
* This function is safe to call as long as @ref is between init and exit.
*/
-static inline void percpu_ref_put(struct percpu_ref *ref)
+static inline void notrace percpu_ref_put(struct percpu_ref *ref)
{
percpu_ref_put_many(ref, 1);
}
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 5e95acc33989..1d26aee3c71c 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -98,12 +98,12 @@ void __rcu_read_unlock(void);
void rcu_read_unlock_strict(void);
#endif
-static inline void __rcu_read_lock(void)
+static inline void notrace __rcu_read_lock(void)
{
preempt_disable();
}
-static inline void __rcu_read_unlock(void)
+static inline void notrace __rcu_read_unlock(void)
{
if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
rcu_read_unlock_strict();
@@ -295,17 +295,17 @@ extern struct lockdep_map rcu_callback_map;
#ifdef CONFIG_DEBUG_LOCK_ALLOC
-static inline void rcu_lock_acquire(struct lockdep_map *map)
+static inline void notrace rcu_lock_acquire(struct lockdep_map *map)
{
lock_acquire(map, 0, 0, 2, 0, NULL, _THIS_IP_);
}
-static inline void rcu_try_lock_acquire(struct lockdep_map *map)
+static inline void notrace rcu_try_lock_acquire(struct lockdep_map *map)
{
lock_acquire(map, 0, 1, 2, 0, NULL, _THIS_IP_);
}
-static inline void rcu_lock_release(struct lockdep_map *map)
+static inline void notrace rcu_lock_release(struct lockdep_map *map)
{
lock_release(map, _THIS_IP_);
}
@@ -863,7 +863,7 @@ static __always_inline void rcu_read_lock(void)
*
* See rcu_read_lock() for more information.
*/
-static inline void rcu_read_unlock(void)
+static inline void notrace rcu_read_unlock(void)
__releases_shared(RCU)
{
RCU_LOCKDEP_WARN(!rcu_is_watching(),
@@ -982,7 +982,7 @@ static __always_inline void rcu_read_lock_dont_migrate(void)
rcu_read_lock();
}
-static inline void rcu_read_unlock_migrate(void)
+static inline void notrace rcu_read_unlock_migrate(void)
__releases_shared(RCU)
{
rcu_read_unlock();
diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h
index cee89e51e45c..94a44773b87d 100644
--- a/include/linux/rcupdate_trace.h
+++ b/include/linux/rcupdate_trace.h
@@ -93,7 +93,7 @@ static inline void rcu_read_unlock_tasks_trace(struct srcu_ctr __percpu *scp)
*
* For more details, please see the documentation for rcu_read_lock().
*/
-static inline void rcu_read_lock_trace(void)
+static inline void notrace rcu_read_lock_trace(void)
{
struct task_struct *t = current;
@@ -117,7 +117,7 @@ static inline void rcu_read_lock_trace(void)
*
* For more details, please see the documentation for rcu_read_unlock().
*/
-static inline void rcu_read_unlock_trace(void)
+static inline void notrace rcu_read_unlock_trace(void)
{
struct srcu_ctr __percpu *scp;
struct task_struct *t = current;
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 373bcc0598d1..8ec1f4172180 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2429,7 +2429,7 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
#endif
#define this_rq_pinned() (*(unsigned int *)((void *)this_rq_raw() + RQ_nr_pinned))
-static inline void __migrate_enable(void)
+static inline void notrace __migrate_enable(void)
{
struct task_struct *p = current;
@@ -2464,7 +2464,7 @@ static inline void __migrate_enable(void)
this_rq_pinned()--;
}
-static inline void __migrate_disable(void)
+static inline void notrace __migrate_disable(void)
{
struct task_struct *p = current;
@@ -2484,8 +2484,8 @@ static inline void __migrate_disable(void)
p->migration_disabled = 1;
}
#else /* !COMPILE_OFFSETS */
-static inline void __migrate_disable(void) { }
-static inline void __migrate_enable(void) { }
+static inline void notrace __migrate_disable(void) { }
+static inline void notrace __migrate_enable(void) { }
#endif /* !COMPILE_OFFSETS */
/*
diff --git a/include/linux/srcu.h b/include/linux/srcu.h
index a54ce9e808b9..bce8cac45474 100644
--- a/include/linux/srcu.h
+++ b/include/linux/srcu.h
@@ -188,7 +188,7 @@ static inline void srcu_lock_acquire(struct lockdep_map *map)
}
/* Annotates a srcu_read_lock() */
-static inline void srcu_lock_release(struct lockdep_map *map)
+static inline void notrace srcu_lock_release(struct lockdep_map *map)
{
lock_map_release(map);
}
--
2.53.0-Meta
next prev parent reply other threads:[~2026-10-23 1:11 UTC|newest]
Thread overview: 17+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-10-23 1:10 [PATCH v2 00/16] Move notrace from inline to __always_inline Song Liu
2026-10-23 1:10 ` [PATCH v2 01/16] x86/paravirt: Mark native pv_ops helpers notrace Song Liu
2026-10-23 1:10 ` [PATCH v2 02/16] x86: Mark CPU state helpers used by resume and kexec notrace Song Liu
2026-10-23 1:10 ` [PATCH v2 03/16] bpf: Mark bpf_spin_lock helpers notrace Song Liu
2026-10-23 1:10 ` Song Liu [this message]
2026-10-23 1:10 ` [PATCH v2 05/16] bpf: Mark the bpf_mem_alloc fast path " Song Liu
2026-10-23 1:10 ` [PATCH v2 06/16] u64_stats: Mark the update helpers used by BPF notrace Song Liu
2026-10-23 1:10 ` [PATCH v2 07/16] tracepoint: Mark __do_trace_##name() notrace Song Liu
2026-10-23 1:10 ` [PATCH v2 08/16] srcu: Mark the fast notrace read-side helpers notrace Song Liu
2026-10-23 1:10 ` [PATCH v2 09/16] arm64: Mark the preempt count " Song Liu
2026-10-23 1:10 ` [PATCH v2 10/16] powerpc, csky, mips, sh: Mark the accessors behind current notrace Song Liu
2026-10-23 1:10 ` [PATCH v2 11/16] rcu: Mark the Tiny RCU rcu_is_watching() notrace Song Liu
2026-10-23 1:10 ` [PATCH v2 12/16] tracing: Mark the tracing_gen_ctx helpers notrace Song Liu
2026-10-23 1:10 ` [PATCH v2 13/16] ktime: Mark ktime_to_ns() notrace Song Liu
2026-10-23 1:10 ` [PATCH v2 14/16] kallsyms: Mark the kernel text address checks notrace Song Liu
2026-10-23 1:10 ` [PATCH v2 15/16] uaccess: Mark pagefault_disable() and pagefault_enable() notrace Song Liu
2026-10-23 1:10 ` [PATCH v2 16/16] compiler_types: Move notrace from inline to __always_inline Song Liu
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261023011049.1164625-5-song@kernel.org \
--to=song@kernel.org \
--cc=kernel-team@meta.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-trace-kernel@vger.kernel.org \
--cc=live-patching@vger.kernel.org \
--cc=mark.rutland@arm.com \
--cc=mhiramat@kernel.org \
--cc=rostedt@goodmis.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®