* [RFC PATCH v2 01/11] ptwrite uprobes: Add infrastructure for ptwrite uprobes
2026-09-17 23:00 ptwrite uprobes v2 Andi Kleen
@ 2026-09-17 23:00 ` Andi Kleen
2026-09-17 23:00 ` [RFC PATCH v2 02/11] ptwrite uprobes: Add minimal low level support for x86 Andi Kleen
` (9 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Andi Kleen @ 2026-09-17 23:00 UTC (permalink / raw)
To: Masami Hiramatsu
Cc: Oleg Nesterov, Peter Zijlstra, linux-kernel, linux-trace-kernel,
x86, tglx, jolsa, linux-perf-users, adrian.hunter, Andi Kleen
uprobes currently always require entering the kernel to log anything.
While that works well, it is rather slow.
Modern Intel CPUs have the ptwrite instruction, which can log data to
the Processor Trace buffer. Add support in uprobes for patching PTWRITE
instead of kernel entries. When a user collects Processor Trace with perf
the logged data will appear in the PT log, otherwise the instructions will
be nops.
PTWRITE provides much faster logging, but it is missing various
features that the full featured probes have.
The instrumentation is similar to optimized uprobes. Add a trampoline
page. Replace the original instruction (5 byte nop) with a jump
to the trampoline. The nop restriction will be relaxed in a later patch.
The trampoline does ptwrites and then jumps back.
Build the high-level infrastructure without any x86-64-specific
parts (except for one data structure). Add register/unregister, basic data
structures and high level hooks. Add weak stubs to handle the no uprobes
or different architectures case.
Assisted-by: omp:gpt-5.6-luna
Signed-off-by: Andi Kleen <ak@kernel.org>
---
arch/Kconfig | 3 +
include/linux/uprobes.h | 77 +++++++++++++++++++-
kernel/events/uprobes.c | 151 ++++++++++++++++++++++++++++++++++++++++
kernel/fork.c | 1 +
4 files changed, 230 insertions(+), 2 deletions(-)
diff --git a/arch/Kconfig b/arch/Kconfig
index 45c657772362..2a3f2e6d4882 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -208,6 +208,9 @@ config UPROBES
managed by the kernel and kept transparent to the probed
application. )
+config ARCH_HAS_UPROBES_PTWRITE
+ bool
+
config HAVE_64BIT_ALIGNED_ACCESS
def_bool 64BIT && !HAVE_EFFICIENT_UNALIGNED_ACCESS
help
diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h
index d34dbc0fbbfe..6b20d61cf737 100644
--- a/include/linux/uprobes.h
+++ b/include/linux/uprobes.h
@@ -23,9 +23,11 @@ struct uprobe;
struct vm_area_struct;
struct mm_struct;
struct inode;
+struct file;
struct notifier_block;
struct page;
struct srcu_ctr;
+struct uprobe_ptwrite_desc;
/*
* Allowed return values from uprobe consumer's handler callback
@@ -187,6 +189,37 @@ struct xol_area;
struct uprobes_state {
struct xol_area *xol_area;
+#ifdef CONFIG_ARCH_HAS_UPROBES_PTWRITE
+ /* Ptwrite pages and metadata protected by mm mmap write lock. */
+ struct hlist_head head_ptwrite;
+#endif
+};
+
+#define UPROBE_PTWRITE_MAX_ARGS 8
+
+/*
+ * Header word: event_id<<48 | nargs<<40 | UPROBE_PTW_HDR_MAGIC (bits 39..0).
+ */
+#define UPROBE_PTW_HDR_MAGIC 0x5054525731UL /* "PTRW1" */
+
+enum uprobe_ptwrite_src {
+ UPROBE_PTW_SRC_REG, /* value = live GPR (index in .reg) */
+ UPROBE_PTW_SRC_IMM, /* value = constant (.val), stored in stub data slot */
+};
+
+struct uprobe_ptwrite_arg {
+ u8 src; /* enum uprobe_ptwrite_src */
+ u8 reg; /* x86-64 GPR index (0=rax..15=r15) for SRC_REG */
+ u8 size; /* declared type size 1/2/4/8 (decoder hint) */
+ u8 reserved;
+ u64 val; /* SRC_IMM: constant; SRC_REG: unused */
+};
+
+struct uprobe_ptwrite_desc {
+ u16 event_id; /* identifier carried in the header word */
+ u8 nargs;
+ u8 flags;
+ struct uprobe_ptwrite_arg args[UPROBE_PTWRITE_MAX_ARGS];
};
typedef int (*uprobe_write_verify_t)(struct page *page, unsigned long vaddr,
@@ -205,6 +238,37 @@ extern int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma,
uprobe_opcode_t *insn, int nbytes, uprobe_write_verify_t verify, bool is_register, bool do_update_ref_ctr,
void *data);
extern struct uprobe *uprobe_register(struct inode *inode, loff_t offset, loff_t ref_ctr_offset, struct uprobe_consumer *uc);
+extern struct uprobe *uprobe_register_ptwrite(struct inode *inode,
+ struct file *file, loff_t offset,
+ struct uprobe_consumer *uc,
+ const struct uprobe_ptwrite_desc *desc);
+extern bool arch_uprobe_ptwrite_supported(void);
+extern int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
+ const struct uprobe_ptwrite_desc *desc);
+extern int arch_uprobe_install_ptwrite(struct arch_uprobe *auprobe,
+ struct vm_area_struct *vma,
+ unsigned long vaddr);
+extern int arch_uprobe_uninstall_ptwrite(struct arch_uprobe *auprobe,
+ struct vm_area_struct *vma,
+ unsigned long vaddr);
+
+enum uprobe_ptwrite_fetch_kind {
+ UPROBE_PTW_FETCH_REG, /* live GPR */
+ UPROBE_PTW_FETCH_STACKP, /* stack pointer value ($stack) */
+ UPROBE_PTW_FETCH_STACKN, /* [SP + imm] ($stackN, imm pre-scaled) */
+ UPROBE_PTW_FETCH_MEMREG, /* [GPR + imm] (imm = disp32) */
+ UPROBE_PTW_FETCH_IMM, /* constant */
+};
+
+struct uprobe_ptwrite_fetch {
+ enum uprobe_ptwrite_fetch_kind kind;
+ unsigned int reg; /* pt_regs member offset */
+ u64 imm; /* IMM value / MEMREG disp / STACKN off */
+};
+
+extern int arch_uprobe_ptwrite_fetch(struct uprobe_ptwrite_arg *a,
+ const struct uprobe_ptwrite_fetch *f);
+
extern int uprobe_apply(struct uprobe *uprobe, struct uprobe_consumer *uc, bool);
extern void uprobe_unregister_nosync(struct uprobe *uprobe, struct uprobe_consumer *uc);
extern void uprobe_unregister_sync(void);
@@ -236,6 +300,8 @@ extern void uprobe_handle_trampoline(struct pt_regs *regs);
extern void *arch_uretprobe_trampoline(unsigned long *psize);
extern unsigned long uprobe_get_trampoline_vaddr(void);
extern void uprobe_copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len);
+extern void arch_uprobe_clear_state(struct mm_struct *mm);
+extern void arch_uprobe_init_state(struct mm_struct *mm);
extern void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr);
extern void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr);
extern unsigned long arch_uprobe_get_xol_area(void);
@@ -254,6 +320,13 @@ uprobe_register(struct inode *inode, loff_t offset, loff_t ref_ctr_offset, struc
{
return ERR_PTR(-ENOSYS);
}
+static inline struct uprobe *
+uprobe_register_ptwrite(struct inode *inode, struct file *file, loff_t offset,
+ struct uprobe_consumer *uc,
+ const struct uprobe_ptwrite_desc *desc)
+{
+ return ERR_PTR(-ENOSYS);
+}
static inline int
uprobe_apply(struct uprobe* uprobe, struct uprobe_consumer *uc, bool add)
{
@@ -280,8 +353,8 @@ static inline void uprobe_start_dup_mmap(void)
static inline void uprobe_end_dup_mmap(void)
{
}
-static inline void
-uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm)
+static inline void uprobe_dup_mmap(struct mm_struct *oldmm,
+ struct mm_struct *newmm)
{
}
static inline void uprobe_notify_resume(struct pt_regs *regs)
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 290c23e273e6..30c28625bb5f 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -59,6 +59,9 @@ DEFINE_STATIC_SRCU_FAST_UPDOWN(uretprobes_srcu);
/* Have a copy of original instruction */
#define UPROBE_COPY_INSN 0
+/* PTWRITE uprobe */
+#define UPROBE_PTWRITE 1
+
struct uprobe {
struct rb_node rb_node; /* node in the rb tree */
refcount_t ref;
@@ -1162,6 +1165,18 @@ static int install_breakpoint(struct uprobe *uprobe, struct vm_area_struct *vma,
if (ret)
return ret;
+ if (test_bit(UPROBE_PTWRITE, &uprobe->flags)) {
+ first_uprobe = !mm_flags_test(MMF_HAS_UPROBES, mm);
+ if (first_uprobe)
+ mm_flags_set(MMF_HAS_UPROBES, mm);
+
+ ret = arch_uprobe_install_ptwrite(&uprobe->arch, vma, vaddr);
+ if (!ret)
+ mm_flags_clear(MMF_RECALC_UPROBES, mm);
+ else if (first_uprobe)
+ mm_flags_clear(MMF_HAS_UPROBES, mm);
+ return ret;
+ }
/*
* set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(),
* the task can hit this breakpoint right after __replace_page().
@@ -1185,6 +1200,9 @@ static int remove_breakpoint(struct uprobe *uprobe, struct vm_area_struct *vma,
struct mm_struct *mm = vma->vm_mm;
mm_flags_set(MMF_RECALC_UPROBES, mm);
+ if (test_bit(UPROBE_PTWRITE, &uprobe->flags))
+ return arch_uprobe_uninstall_ptwrite(&uprobe->arch, vma, vaddr);
+
return set_orig_insn(&uprobe->arch, vma, vaddr);
}
@@ -1423,6 +1441,12 @@ struct uprobe *uprobe_register(struct inode *inode,
return uprobe;
down_write(&uprobe->register_rwsem);
+ if (test_bit(UPROBE_PTWRITE, &uprobe->flags)) {
+ up_write(&uprobe->register_rwsem);
+ put_uprobe(uprobe);
+ return ERR_PTR(-EBUSY);
+ }
+
consumer_add(uprobe, uc);
ret = register_for_each_vma(uprobe, uc);
up_write(&uprobe->register_rwsem);
@@ -1442,6 +1466,131 @@ struct uprobe *uprobe_register(struct inode *inode,
}
EXPORT_SYMBOL_GPL(uprobe_register);
+/*
+ * Architecture state and PTWRITE hooks: weak defaults so the generic core
+ * builds on any architecture.
+ */
+void __weak arch_uprobe_init_state(struct mm_struct *mm)
+{
+}
+
+void __weak arch_uprobe_clear_state(struct mm_struct *mm)
+{
+}
+
+bool __weak arch_uprobe_ptwrite_supported(void)
+{
+ return false;
+}
+
+int __weak arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
+ const struct uprobe_ptwrite_desc *desc)
+{
+ return -EOPNOTSUPP;
+}
+
+int __weak arch_uprobe_install_ptwrite(struct arch_uprobe *auprobe,
+ struct vm_area_struct *vma,
+ unsigned long vaddr)
+{
+ return -EOPNOTSUPP;
+}
+
+int __weak arch_uprobe_uninstall_ptwrite(struct arch_uprobe *auprobe,
+ struct vm_area_struct *vma,
+ unsigned long vaddr)
+{
+ return 0;
+}
+
+int __weak arch_uprobe_ptwrite_fetch(struct uprobe_ptwrite_arg *arg,
+ const struct uprobe_ptwrite_fetch *fetch)
+{
+ return -EOPNOTSUPP;
+}
+
+/**
+ * uprobe_register_ptwrite - register a PTWRITE uprobe
+ * @inode: the probed file's inode
+ * @file: open file used while populating the instruction page cache
+ * @offset: offset from the start of the file
+ * @uc: consumer controlling probe lifetime
+ * @desc: requested values to emit
+ */
+struct uprobe *uprobe_register_ptwrite(struct inode *inode, struct file *file,
+ loff_t offset, struct uprobe_consumer *uc,
+ const struct uprobe_ptwrite_desc *desc)
+{
+ struct uprobe *uprobe;
+ int ret;
+
+ if (!file || !uc)
+ return ERR_PTR(-EINVAL);
+
+ if (!arch_uprobe_ptwrite_supported())
+ return ERR_PTR(-EOPNOTSUPP);
+
+ if (!desc || desc->nargs == 0 || desc->nargs > UPROBE_PTWRITE_MAX_ARGS)
+ return ERR_PTR(-EINVAL);
+
+ if (!inode->i_mapping->a_ops->read_folio &&
+ !shmem_mapping(inode->i_mapping))
+ return ERR_PTR(-EIO);
+
+ /* Racy, just to catch the obvious mistakes */
+ if (offset < 0)
+ return ERR_PTR(-EINVAL);
+ if (offset > i_size_read(inode))
+ return ERR_PTR(-EINVAL);
+ if (!IS_ALIGNED(offset, UPROBE_SWBP_INSN_SIZE))
+ return ERR_PTR(-EINVAL);
+
+ uprobe = alloc_uprobe(inode, offset, 0);
+ if (IS_ERR(uprobe))
+ return uprobe;
+
+ down_write(&uprobe->register_rwsem);
+
+ /*
+ * Do not repurpose an existing uprobe. UPROBE_COPY_INSN remains set
+ * after its last consumer is detached and closes the deferred-removal
+ * window where the consumer list alone is not a sufficient mode check.
+ */
+ if (test_bit(UPROBE_COPY_INSN, &uprobe->flags) ||
+ !list_empty(&uprobe->consumers)) {
+ ret = -EBUSY;
+ goto out;
+ }
+
+ /* Build the mm-independent stub template once, at registration. */
+ ret = arch_uprobe_ptwrite_prepare(&uprobe->arch, desc);
+ if (ret)
+ goto out;
+
+
+ set_bit(UPROBE_PTWRITE, &uprobe->flags);
+ consumer_add(uprobe, uc);
+ ret = register_for_each_vma(uprobe, uc);
+ up_write(&uprobe->register_rwsem);
+
+ if (ret) {
+ uprobe_unregister_nosync(uprobe, uc);
+ /*
+ * Registration might have partially succeeded. Clean
+ * everything up.
+ */
+ uprobe_unregister_sync();
+ return ERR_PTR(ret);
+ }
+
+ return uprobe;
+out:
+ up_write(&uprobe->register_rwsem);
+ put_uprobe(uprobe);
+ return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(uprobe_register_ptwrite);
+
/**
* uprobe_apply - add or remove the breakpoints according to @uc->filter
* @uprobe: uprobe which "owns" the breakpoint
@@ -1826,6 +1975,8 @@ void uprobe_clear_state(struct mm_struct *mm)
delayed_uprobe_remove(NULL, mm);
mutex_unlock(&delayed_uprobe_lock);
+ arch_uprobe_clear_state(mm);
+
if (!area)
return;
diff --git a/kernel/fork.c b/kernel/fork.c
index a5934a317634..461a7b8b9e1b 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1076,6 +1076,7 @@ static void mm_init_uprobes_state(struct mm_struct *mm)
{
#ifdef CONFIG_UPROBES
mm->uprobes_state.xol_area = NULL;
+ arch_uprobe_init_state(mm);
#endif
}
--
2.54.0
^ permalink raw reply [flat|nested] 12+ messages in thread* [RFC PATCH v2 02/11] ptwrite uprobes: Add minimal low level support for x86
2026-09-17 23:00 ptwrite uprobes v2 Andi Kleen
2026-09-17 23:00 ` [RFC PATCH v2 01/11] ptwrite uprobes: Add infrastructure for ptwrite uprobes Andi Kleen
@ 2026-09-17 23:00 ` Andi Kleen
2026-09-17 23:00 ` [RFC PATCH v2 03/11] ptwrite uprobes: Add a sample module to exercise interface Andi Kleen
` (8 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Andi Kleen @ 2026-09-17 23:00 UTC (permalink / raw)
To: Masami Hiramatsu
Cc: Oleg Nesterov, Peter Zijlstra, linux-kernel, linux-trace-kernel,
x86, tglx, jolsa, linux-perf-users, adrian.hunter, Andi Kleen
Add more data structures and the x86 machinery to generate the PTWRITE
instructions for a ptwrite uprobe. The probe executes PTWRITEs and then
jumps back to the original code. In this variant only patching
5 byte nops is supported.
The instructions are pre-generated to templates and then patched when
setting up the final user page.
The patching code uses 3 phase patching similar to int3_update.
The ptwrite stub emits a header with a magic value and the number of
arguments, and then the actual probed values.
There is no separate config option for ptwrite uprobes, it is just tied
to the main uprobes config.
Some limitations in the current implementation:
- The probed 5 byte area cannot cross a page.
- The allocated stubs in the user program are only freed on exit.
Assisted-by: omp:gpt-5.6-luna
Signed-off-by: Andi Kleen <ak@kernel.org>
---
arch/x86/Kconfig | 1 +
arch/x86/include/asm/mmu_context.h | 17 +-
arch/x86/include/asm/uprobes.h | 26 ++
arch/x86/kernel/uprobes.c | 654 +++++++++++++++++++++++++++++
4 files changed, 697 insertions(+), 1 deletion(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 15fd9ec5ecac..7fda5edb0c06 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -102,6 +102,7 @@ config X86
select ARCH_HAS_HW_PTE_YOUNG
select ARCH_HAS_NONLEAF_PMD_YOUNG if PGTABLE_LEVELS > 2
select ARCH_HAS_UACCESS_FLUSHCACHE if X86_64
+ select ARCH_HAS_UPROBES_PTWRITE if X86_64
select ARCH_HAS_COPY_MC if X86_64
select ARCH_HAS_SET_MEMORY
select ARCH_HAS_SET_DIRECT_MAP
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index ef5b507de34e..5a5b7e2a4f5d 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -215,12 +215,27 @@ static inline void arch_dup_pkeys(struct mm_struct *oldmm,
#endif
}
+/* Put here to avoid asm/uprobes.h's dependencies */
+extern int uprobe_ptwrite_dup_mmap(struct mm_struct *oldmm,
+ struct mm_struct *newmm);
+
static inline int arch_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
{
+ int ret;
+
arch_dup_pkeys(oldmm, mm);
paravirt_enter_mmap(mm);
dup_lam(oldmm, mm);
- return ldt_dup_context(oldmm, mm);
+
+ ret = ldt_dup_context(oldmm, mm);
+ if (ret)
+ return ret;
+#ifdef CONFIG_UPROBES
+ ret = uprobe_ptwrite_dup_mmap(oldmm, mm);
+ if (ret)
+ return ret;
+#endif
+ return 0;
}
static inline void arch_exit_mmap(struct mm_struct *mm)
diff --git a/arch/x86/include/asm/uprobes.h b/arch/x86/include/asm/uprobes.h
index 362210c79998..20d086a945c5 100644
--- a/arch/x86/include/asm/uprobes.h
+++ b/arch/x86/include/asm/uprobes.h
@@ -11,6 +11,7 @@
*/
#include <linux/notifier.h>
+#include <linux/rcupdate.h>
typedef u8 uprobe_opcode_t;
@@ -23,10 +24,34 @@ typedef u8 uprobe_opcode_t;
enum {
ARCH_UPROBE_FLAG_CAN_OPTIMIZE = 0,
ARCH_UPROBE_FLAG_OPTIMIZE_FAIL = 1,
+ ARCH_UPROBE_FLAG_PTWRITE = 2,
};
struct uprobe_xol_ops;
+/*
+ * ptwrite probe state. The stub template (code + data slots) is built
+ * once at registration (mm-independent except the final jmp's rel32, patched
+ * per-mm at install). Block layout:
+ * [ptwriteq hdr(%rip)] [arg emissions] [jmp probe+5] [u64 slots: header, imms]
+ */
+struct uprobe_ptwrite_arch {
+ u8 stub[256];
+ u8 stub_len; /* code + data, whole block */
+ u8 jmp_off; /* offset of the final jmp's rel32 field */
+ u8 ndata; /* number of u64 data slots */
+ u8 orig[MAX_UINSN_BYTES]; /* pristine file bytes, before generic analysis */
+};
+
+/* Per-mm page holding generated ptwrite stub blocks, like tramp_mapping. */
+struct uprobe_ptwrite_page {
+ struct hlist_node node;
+ struct rcu_head rcu;
+ struct page *page; /* stub blocks written via kmap */
+ unsigned long vaddr; /* mapping base */
+ u16 cursor; /* next free block offset */
+};
+
struct arch_uprobe {
union {
u8 insn[MAX_UINSN_BYTES];
@@ -51,6 +76,7 @@ struct arch_uprobe {
} push;
};
+ struct uprobe_ptwrite_arch ptwrite;
unsigned long flags;
};
diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c
index 65a2de82ecd2..7fcdc4bf5197 100644
--- a/arch/x86/kernel/uprobes.c
+++ b/arch/x86/kernel/uprobes.c
@@ -15,11 +15,15 @@
#include <linux/syscalls.h>
#include <linux/kdebug.h>
+#include <linux/highmem.h>
+#include <linux/mm.h>
#include <asm/processor.h>
#include <asm/insn.h>
#include <asm/insn-eval.h>
#include <asm/mmu_context.h>
#include <asm/nops.h>
+#include <asm/cpufeature.h>
+#include <asm/cpuid/api.h>
/* Post-execution fixups. */
@@ -719,6 +723,134 @@ static struct vm_area_struct *get_uprobe_trampoline(struct mm_struct *mm, unsign
&tramp_mapping);
}
+void arch_uprobe_init_state(struct mm_struct *mm)
+{
+ INIT_HLIST_HEAD(&mm->uprobes_state.head_ptwrite);
+}
+
+static void free_uprobe_ptwrite_page_rcu(struct rcu_head *rcu)
+{
+ struct uprobe_ptwrite_page *ptw =
+ container_of(rcu, struct uprobe_ptwrite_page, rcu);
+
+ __free_page(ptw->page);
+ kfree(ptw);
+}
+
+void arch_uprobe_clear_state(struct mm_struct *mm)
+{
+ struct uprobes_state *state = &mm->uprobes_state;
+ struct uprobe_ptwrite_page *ptw;
+ struct hlist_node *n;
+
+ hlist_for_each_entry_safe(ptw, n, &state->head_ptwrite, node) {
+ hlist_del_rcu(&ptw->node);
+ call_rcu(&ptw->rcu, free_uprobe_ptwrite_page_rcu);
+ }
+}
+
+static vm_fault_t ptwrite_fault(const struct vm_special_mapping *sm,
+ struct vm_area_struct *vma, struct vm_fault *vmf)
+{
+ struct uprobes_state *state = &vma->vm_mm->uprobes_state;
+ struct uprobe_ptwrite_page *ptw;
+
+ rcu_read_lock();
+ hlist_for_each_entry_rcu(ptw, &state->head_ptwrite, node) {
+ if (ptw->vaddr == vma->vm_start) {
+ vmf->page = ptw->page;
+ get_page(vmf->page);
+ rcu_read_unlock();
+ return 0;
+ }
+ }
+ rcu_read_unlock();
+ return VM_FAULT_SIGBUS;
+}
+
+static int ptwrite_mremap(const struct vm_special_mapping *sm,
+ struct vm_area_struct *new_vma)
+{
+ return -EPERM;
+}
+
+static const struct vm_special_mapping ptwrite_mapping = {
+ .name = "[uprobes-ptwrite]",
+ .fault = ptwrite_fault,
+ .mremap = ptwrite_mremap,
+};
+
+static bool __in_uprobe_ptwrite(struct mm_struct *mm, unsigned long ip)
+{
+ struct vm_area_struct *vma = vma_lookup(mm, ip);
+
+ return vma && vma_is_special_mapping(vma, &ptwrite_mapping);
+}
+
+static struct vm_area_struct *
+install_uprobe_ptwrite_vma(struct mm_struct *mm, unsigned long addr)
+{
+ return _install_special_mapping(mm, addr, PAGE_SIZE,
+ VM_READ | VM_EXEC | VM_MAYEXEC | VM_MAYREAD |
+ VM_IO | VM_DONTCOPY, &ptwrite_mapping);
+}
+
+int uprobe_ptwrite_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm)
+{
+ struct uprobes_state *old_state = &oldmm->uprobes_state;
+ struct uprobes_state *new_state = &newmm->uprobes_state;
+ struct uprobe_ptwrite_page *ptw, *new;
+ struct vm_area_struct *vma;
+ struct hlist_node *n;
+
+ mmap_assert_write_locked(oldmm);
+ mmap_assert_write_locked(newmm);
+ hlist_for_each_entry(ptw, &old_state->head_ptwrite, node) {
+ void *src, *dst;
+
+ /*
+ * Not using __GFP_ACCOUNT here because it triggered
+ * a deadlock.
+ */
+ new = kzalloc_obj(*new, GFP_KERNEL);
+ if (!new)
+ goto fail;
+ new->page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+ if (!new->page) {
+ kfree(new);
+ goto fail;
+ }
+
+ src = kmap_local_page(ptw->page);
+ dst = kmap_local_page(new->page);
+ memcpy(dst, src, PAGE_SIZE);
+ kunmap_local(dst);
+ kunmap_local(src);
+ new->vaddr = ptw->vaddr;
+ new->cursor = ptw->cursor;
+
+ vma = install_uprobe_ptwrite_vma(newmm, new->vaddr);
+ if (IS_ERR(vma)) {
+ __free_page(new->page);
+ kfree(new);
+ goto fail;
+ }
+
+ /* Publish the copied page fields before readers can find it. */
+ smp_wmb();
+ hlist_add_head_rcu(&new->node, &new_state->head_ptwrite);
+ }
+
+ return 0;
+
+fail:
+ hlist_for_each_entry_safe(new, n, &new_state->head_ptwrite, node) {
+ WARN_ON_ONCE(do_munmap(newmm, new->vaddr, PAGE_SIZE, NULL));
+ }
+ arch_uprobe_clear_state(newmm);
+ return -ENOMEM;
+}
+
static bool __in_uprobe_trampoline(struct mm_struct *mm, unsigned long ip)
{
struct vm_area_struct *vma = vma_lookup(mm, ip);
@@ -869,11 +1001,13 @@ enum {
EXPECT_SWBP,
EXPECT_OPTIMIZED,
EXPECT_SWBP_OPTIMIZED,
+ EXPECT_BYTE,
};
struct write_opcode_ctx {
unsigned long base;
int expect;
+ u8 expect_byte;
};
/*
@@ -901,6 +1035,10 @@ static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t *
if (is_swbp_opt_insns(&old_opcode[0]))
return 1;
break;
+ case EXPECT_BYTE:
+ if (old_opcode[0] == ctx->expect_byte)
+ return 1;
+ break;
}
return -1;
@@ -1064,6 +1202,53 @@ static int int3_update_unoptimize(struct arch_uprobe *auprobe, struct vm_area_st
return 0;
}
+/*
+ * Modify a five-byte instruction by using INT3 breakpoints on SMP.
+ * The caller supplies the byte expected before the update and controls
+ * whether the anonymous page and reference counter are updated on the
+ * final write.
+ */
+static int text_poke_5byte(struct arch_uprobe *auprobe, struct vm_area_struct *vma,
+ unsigned long vaddr, u8 *new5, u8 expect_byte,
+ bool skip_int3, bool is_register, bool final_is_register,
+ bool do_update_ref_ctr, bool *first_phase_done)
+{
+ uprobe_opcode_t int3 = UPROBE_SWBP_INSN;
+ struct write_opcode_ctx ctx = {
+ .base = vaddr,
+ .expect = EXPECT_BYTE,
+ .expect_byte = expect_byte,
+ };
+ int err;
+
+ if (!skip_int3) {
+ err = uprobe_write(auprobe, vma, vaddr, &int3, 1, verify_insn,
+ is_register, false, &ctx);
+ if (err)
+ return err;
+ }
+ if (first_phase_done)
+ *first_phase_done = true;
+
+ smp_text_poke_sync_each_cpu();
+
+ ctx.expect = EXPECT_SWBP;
+ err = uprobe_write(auprobe, vma, vaddr + 1, new5 + 1, 4, verify_insn,
+ is_register, false, &ctx);
+ if (err)
+ return err;
+
+ smp_text_poke_sync_each_cpu();
+
+ err = uprobe_write(auprobe, vma, vaddr, new5, 1, verify_insn,
+ final_is_register, do_update_ref_ctr, &ctx);
+ if (err)
+ return err;
+
+ smp_text_poke_sync_each_cpu();
+ return 0;
+}
+
static int swbp_optimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma,
unsigned long vaddr, unsigned long tramp)
{
@@ -1102,6 +1287,475 @@ static int copy_from_vaddr(struct mm_struct *mm, unsigned long vaddr, void *dst,
return 0;
}
+/*
+ * ptwrite uprobes: trap-free user-mode instrumentation.
+ *
+ * Block layout (mm-independent template, built at registration):
+ * ptwriteq hdr(%rip) ; header: event_id<<48 | nargs<<40 | magic
+ * ptwriteq %reg / imm(%rip) ; one per arg
+ * jmp probe+5 ; rel32 patched per-mm at install
+ * [u64 slots: header, imm values]
+ */
+
+static int ptwrite_emit_reg(u8 *p, u8 reg)
+{
+ /* ptwriteq %reg : F3 REX.W[.B] 0F AE /4, modrm = 11 100 rrr */
+ *p++ = 0xf3;
+ *p++ = (reg & 8) ? 0x49 : 0x48; /* REX.W, +REX.B for r8-r15 */
+ *p++ = 0x0f;
+ *p++ = 0xae;
+ *p++ = 0xe0 | (reg & 7);
+ return 5;
+}
+
+static int ptwrite_emit_riprel(u8 *p, s32 disp)
+{
+ /*
+ * ptwriteq disp32(%rip) : F3 48 0F AE 25 <disp32> (9 bytes)
+ * modrm 0x25 = mod 00, reg 100 (/4, PTWRITE), rm 101 (RIP-relative).
+ */
+ *p++ = 0xf3;
+ *p++ = 0x48;
+ *p++ = 0x0f;
+ *p++ = 0xae;
+ *p++ = 0x25;
+ memcpy(p, &disp, 4);
+ return 9;
+}
+
+bool arch_uprobe_ptwrite_supported(void)
+{
+ u32 eax, ebx, ecx, edx;
+
+ if (!boot_cpu_has(X86_FEATURE_INTEL_PT))
+ return false;
+ if (boot_cpu_data.cpuid_level < 0x14)
+ return false;
+
+ /* CPUID.(EAX=14H, ECX=0):EBX[4] = PTWRITE */
+ cpuid_count(0x14, 0, &eax, &ebx, &ecx, &edx);
+ return !!(ebx & BIT(4));
+}
+
+/*
+ * x86-64 pt_regs member offset -> GPR index (0=rax..15=r15), matching
+ * the uprobe_ptwrite_arg.reg convention used by the stub generator.
+ * The offsets are what the generic trace-probe register parser
+ * (regs_query_register_offset) puts into FETCH_OP_REG.params.
+ */
+static const struct {
+ unsigned int off;
+ u8 idx;
+} ptwrite_reg_map[] = {
+ { .off = offsetof(struct pt_regs, ax), .idx = 0 },
+ { .off = offsetof(struct pt_regs, cx), .idx = 1 },
+ { .off = offsetof(struct pt_regs, dx), .idx = 2 },
+ { .off = offsetof(struct pt_regs, bx), .idx = 3 },
+ { .off = offsetof(struct pt_regs, sp), .idx = 4 },
+ { .off = offsetof(struct pt_regs, bp), .idx = 5 },
+ { .off = offsetof(struct pt_regs, si), .idx = 6 },
+ { .off = offsetof(struct pt_regs, di), .idx = 7 },
+ { .off = offsetof(struct pt_regs, r8), .idx = 8 },
+ { .off = offsetof(struct pt_regs, r9), .idx = 9 },
+ { .off = offsetof(struct pt_regs, r10), .idx = 10 },
+ { .off = offsetof(struct pt_regs, r11), .idx = 11 },
+ { .off = offsetof(struct pt_regs, r12), .idx = 12 },
+ { .off = offsetof(struct pt_regs, r13), .idx = 13 },
+ { .off = offsetof(struct pt_regs, r14), .idx = 14 },
+ { .off = offsetof(struct pt_regs, r15), .idx = 15 },
+};
+
+/* Compile the register, stack-pointer, and immediate fetch forms. */
+int arch_uprobe_ptwrite_fetch(struct uprobe_ptwrite_arg *a,
+ const struct uprobe_ptwrite_fetch *f)
+{
+ int i, idx = -1;
+
+ switch (f->kind) {
+ case UPROBE_PTW_FETCH_REG:
+ for (i = 0; i < ARRAY_SIZE(ptwrite_reg_map); i++) {
+ if (ptwrite_reg_map[i].off == f->reg) {
+ idx = ptwrite_reg_map[i].idx;
+ break;
+ }
+ }
+ if (idx < 0)
+ return -EINVAL;
+ a->src = UPROBE_PTW_SRC_REG;
+ a->reg = idx;
+ break;
+ case UPROBE_PTW_FETCH_STACKP:
+ a->src = UPROBE_PTW_SRC_REG;
+ a->reg = 4; /* rsp */
+ break;
+ case UPROBE_PTW_FETCH_IMM:
+ a->src = UPROBE_PTW_SRC_IMM;
+ a->val = f->imm;
+ break;
+ default:
+ return -EINVAL;
+ }
+ return 0;
+}
+
+int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
+ const struct uprobe_ptwrite_desc *desc)
+{
+ struct uprobe_ptwrite_arch *ptw = &auprobe->ptwrite;
+ u8 *code = ptw->stub, *p = ptw->stub;
+ u16 imm_off[UPROBE_PTWRITE_MAX_ARGS];
+ unsigned int data_off;
+ unsigned int hdr_off = 0;
+ unsigned int imm_idx = 0, n_imm = 0;
+ u64 hdr;
+ int i;
+
+ if (!desc || desc->nargs == 0)
+ return -EINVAL;
+ if (desc->nargs > UPROBE_PTWRITE_MAX_ARGS)
+ return -E2BIG;
+ if (desc->flags)
+ return -EINVAL;
+
+ /* The generic registration path copied these bytes before this hook. */
+ memcpy(ptw->orig, auprobe->insn, sizeof(ptw->orig));
+
+ for (i = 0; i < desc->nargs; i++) {
+ switch (desc->args[i].src) {
+ case UPROBE_PTW_SRC_REG:
+ if (desc->args[i].reg > 15)
+ return -EINVAL;
+ break;
+ case UPROBE_PTW_SRC_IMM:
+ if (n_imm >= ARRAY_SIZE(imm_off))
+ return -E2BIG;
+ n_imm++;
+ break;
+ default:
+ return -EINVAL;
+ }
+ }
+
+ /* header word emission (disp32 patched below) */
+ p += ptwrite_emit_riprel(p, 0);
+
+ for (i = 0; i < desc->nargs; i++) {
+ if (desc->args[i].src == UPROBE_PTW_SRC_REG) {
+ p += ptwrite_emit_reg(p, desc->args[i].reg);
+ } else {
+ imm_off[imm_idx++] = p - code;
+ p += ptwrite_emit_riprel(p, 0);
+ }
+ }
+
+ /* final jmp back to probe+5; rel32 patched per-mm at install */
+ *p++ = 0xe9;
+ if (p - code > U8_MAX)
+ return -E2BIG;
+ ptw->jmp_off = p - code;
+ p += 4;
+
+ data_off = (p - code + 7) & ~7UL;
+ if (data_off + 8 * (1 + n_imm) > sizeof(ptw->stub))
+ return -E2BIG;
+
+ /* data slots: header, then imm values in emission order */
+ hdr = ((u64)desc->event_id << 48) | ((u64)desc->nargs << 40);
+ *(u64 *)(code + data_off) = hdr;
+
+ /* patch the header's disp32: hdr slot - end of header insn */
+ *(s32 *)(code + hdr_off + 5) = (s32)(data_off - (hdr_off + 9));
+
+ imm_idx = 0;
+ for (i = 0; i < desc->nargs; i++) {
+ if (desc->args[i].src != UPROBE_PTW_SRC_IMM)
+ continue;
+ *(s32 *)(code + imm_off[imm_idx] + 5) =
+ (s32)((data_off + 8 * (1 + imm_idx)) - (imm_off[imm_idx] + 9));
+ *(u64 *)(code + data_off + 8 * (1 + imm_idx)) = desc->args[i].val;
+ imm_idx++;
+ }
+
+ ptw->stub_len = data_off + 8 * (1 + n_imm);
+ ptw->ndata = 1 + n_imm;
+ return 0;
+}
+#undef PTW_NEED
+
+/*
+ * Find a free PAGE_SIZE area in @mm within +/-2GB of the probe (so the jmp
+ * rel32 at the probe can reach the stub). Caller holds mmap_write_lock(mm).
+ * Returns an address or a negative errno encoded as unsigned long.
+ */
+static unsigned long find_ptwrite_page_area(struct mm_struct *mm,
+ unsigned long vaddr)
+{
+ VMA_ITERATOR(vmi, mm, 0);
+ struct vm_area_struct *vma;
+ unsigned long low, high, prev, call_end;
+ const unsigned long call_range = (unsigned long)INT_MAX + 1;
+
+ mmap_assert_write_locked(mm);
+ if (check_add_overflow(vaddr, 5UL, &call_end))
+ return -ENOMEM;
+ if (call_end < call_range)
+ low = PAGE_SIZE;
+ else
+ low = call_end - call_range;
+ if (low < PAGE_SIZE)
+ low = PAGE_SIZE;
+ if (low > ULONG_MAX - (PAGE_SIZE - 1))
+ return -ENOMEM;
+ low = PAGE_ALIGN(low);
+
+ if (check_add_overflow(call_end, (unsigned long)INT_MAX, &high))
+ high = ULONG_MAX;
+ high = min(high, TASK_SIZE_MAX);
+ if (low >= high)
+ return -ENOMEM;
+
+ prev = low;
+ for_each_vma(vmi, vma) {
+ if (vma->vm_start >= high)
+ break;
+ if (vma->vm_end <= prev)
+ continue;
+ if (vma->vm_start > prev && vma->vm_start - prev >= PAGE_SIZE)
+ return prev;
+ if (vma->vm_end > prev) {
+ if (vma->vm_end > ULONG_MAX - (PAGE_SIZE - 1))
+ return -ENOMEM;
+ prev = PAGE_ALIGN(vma->vm_end);
+ if (prev >= high)
+ return -ENOMEM;
+ }
+ }
+ if (prev < high && high - prev >= PAGE_SIZE)
+ return prev;
+ return -ENOMEM;
+}
+
+static struct uprobe_ptwrite_page *
+create_uprobe_ptwrite_page(struct mm_struct *mm, unsigned long vaddr)
+{
+ struct uprobe_ptwrite_page *ptw;
+ struct vm_area_struct *vma;
+ unsigned long area;
+
+ area = find_ptwrite_page_area(mm, vaddr);
+ if (IS_ERR_VALUE(area))
+ return NULL;
+
+ mmap_assert_write_locked(mm);
+
+ ptw = kzalloc_obj(*ptw, GFP_KERNEL);
+ if (!ptw)
+ return NULL;
+
+ ptw->page = alloc_page(GFP_HIGHUSER | __GFP_ZERO);
+ if (!ptw->page) {
+ kfree(ptw);
+ return NULL;
+ }
+ ptw->vaddr = area;
+
+ vma = install_uprobe_ptwrite_vma(mm, area);
+ if (IS_ERR(vma)) {
+ __free_page(ptw->page);
+ kfree(ptw);
+ return NULL;
+ }
+ return ptw;
+}
+static struct uprobe_ptwrite_page *
+get_uprobe_ptwrite_page(struct mm_struct *mm, unsigned long vaddr,
+ unsigned int len)
+{
+ struct uprobes_state *state = &mm->uprobes_state;
+ struct uprobe_ptwrite_page *ptw;
+ mmap_assert_write_locked(mm);
+
+ /* a block larger than a page can never be placed */
+ if (len > PAGE_SIZE)
+ return NULL;
+
+ /* Both helpers use the signed 32-bit range of the generated jump. */
+ hlist_for_each_entry(ptw, &state->head_ptwrite, node)
+ if (is_reachable_by_call(ptw->vaddr + ptw->cursor, vaddr) &&
+ ptw->cursor + len <= PAGE_SIZE)
+ return ptw;
+
+ /* no reachable page with room: allocate a fresh one (cursor 0) */
+ ptw = create_uprobe_ptwrite_page(mm, vaddr);
+ if (!ptw)
+ return NULL;
+ /* Order page initialization before publishing the page on the RCU list. */
+ smp_wmb();
+
+ hlist_add_head_rcu(&ptw->node, &state->head_ptwrite);
+ return ptw;
+}
+
+/* Probe site must be a 5-byte NOP that does not cross a page boundary. */
+static int ptwrite_validate_site(const u8 *orig, unsigned long vaddr)
+{
+ struct insn insn;
+ int ret;
+ int off = 0;
+
+ /*
+ * The 5 displaced bytes must be NOPs: either one 5-byte NOP
+ * (nopl 0x0(%rax,%rax,1)) or a run of shorter NOPs summing to
+ * exactly 5 (gcc -fpatchable-function-entry=5 emits 5 x 0x90 on
+ * modern toolchains). Any non-NOP byte, or a NOP crossing the
+ * 5-byte window, is rejected.
+ */
+ while (off < 5) {
+ ret = insn_decode(&insn, orig + off, 5 - off, INSN_MODE_64);
+ if (ret < 0)
+ return -EINVAL;
+ if (insn.length < 1 || insn.length > 5 - off ||
+ !insn_is_nop(&insn))
+ return -EINVAL;
+ off += insn.length;
+ }
+ if (off != 5)
+ return -EINVAL;
+ if (PAGE_SIZE - (vaddr & ~PAGE_MASK) < 5)
+ return -EINVAL;
+ return 0;
+}
+
+static bool ptwrite_rel32(unsigned long from, unsigned long to, s32 *rel)
+{
+ s64 delta = (s64)to - (s64)from;
+
+ if (delta < INT_MIN || delta > INT_MAX)
+ return false;
+ *rel = (s32)delta;
+ return true;
+}
+
+static bool ptwrite_is_installed(struct mm_struct *mm, unsigned long vaddr,
+ const u8 *insn5)
+{
+ struct __packed __arch_relative_insn {
+ u8 op;
+ s32 raddr;
+ } *jmp = (struct __arch_relative_insn *)insn5;
+ s64 target;
+
+ if (jmp->op != 0xe9)
+ return false;
+ target = (s64)vaddr + 5 + (s64)jmp->raddr;
+ if (target < PAGE_SIZE || target >= TASK_SIZE_MAX)
+ return false;
+ return __in_uprobe_ptwrite(mm, (unsigned long)target);
+}
+
+/*
+ * Install a JMP rel32 at the probe site using the 3-phase SMP-safe poke.
+ * On failure, restores the original instruction so the site is never
+ * left half-poked.
+ */
+static int ptwrite_text_poke(struct arch_uprobe *auprobe,
+ struct vm_area_struct *vma, unsigned long vaddr,
+ unsigned long stub_addr)
+{
+ u8 jmp5[5] = { 0xe9, 0, 0, 0, 0 };
+ bool first_phase_done;
+ s32 rel;
+ int err;
+
+ if (!ptwrite_rel32(vaddr + 5, stub_addr, &rel))
+ return -ERANGE;
+ memcpy(jmp5 + 1, &rel, 4);
+
+ err = text_poke_5byte(auprobe, vma, vaddr, jmp5,
+ auprobe->ptwrite.orig[0], false, true, true,
+ false, &first_phase_done);
+ if (err && first_phase_done) {
+ int restore_err;
+
+ /* Restore only after INT3 was successfully installed. */
+ restore_err = text_poke_5byte(auprobe, vma, vaddr,
+ auprobe->ptwrite.orig, UPROBE_SWBP_INSN,
+ true, true, true, false, NULL);
+ if (restore_err)
+ return restore_err;
+ }
+ return err;
+}
+
+int arch_uprobe_install_ptwrite(struct arch_uprobe *auprobe,
+ struct vm_area_struct *vma, unsigned long vaddr)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ struct uprobe_ptwrite_page *ptw;
+ struct uprobe_ptwrite_arch *ptw_a = &auprobe->ptwrite;
+ unsigned long block_off, stub_addr;
+ u8 *kaddr, orig[5];
+ s32 rel;
+ int ret;
+
+ if (!is_64bit_mm(mm))
+ return -EOPNOTSUPP;
+ /* A three-phase poke and this single-page read both require one page. */
+ if (PAGE_SIZE - (vaddr & ~PAGE_MASK) < 5)
+ return -EINVAL;
+ mmap_assert_write_locked(mm);
+
+ ret = copy_from_vaddr(mm, vaddr, orig, sizeof(orig));
+ if (ret)
+ return ret;
+ if (ptwrite_is_installed(mm, vaddr, orig))
+ return 0;
+
+ ret = ptwrite_validate_site(orig, vaddr);
+ if (ret)
+ return ret;
+
+ ptw = get_uprobe_ptwrite_page(mm, vaddr, ptw_a->stub_len);
+ if (!ptw)
+ return -ENOMEM;
+
+ block_off = ptw->cursor;
+ if (block_off > PAGE_SIZE ||
+ ptw_a->stub_len > PAGE_SIZE - block_off)
+ return -ENOMEM;
+ stub_addr = ptw->vaddr + block_off;
+ if (!ptwrite_rel32(stub_addr + ptw_a->jmp_off + 4,
+ vaddr + 5, &rel))
+ return -ERANGE;
+
+ kaddr = kmap_local_page(ptw->page);
+ memcpy(kaddr + block_off, ptw_a->stub, ptw_a->stub_len);
+ /* ptwrite_mapping rejects mremap, so this per-mm rel32 remains valid. */
+ memcpy(kaddr + block_off + ptw_a->jmp_off, &rel, sizeof(rel));
+ kunmap_local(kaddr);
+
+ ret = ptwrite_text_poke(auprobe, vma, vaddr, stub_addr);
+ if (!ret)
+ ptw->cursor = block_off + ptw_a->stub_len;
+ return ret;
+}
+
+int arch_uprobe_uninstall_ptwrite(struct arch_uprobe *auprobe,
+ struct vm_area_struct *vma, unsigned long vaddr)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ u8 cur[5];
+
+ mmap_assert_write_locked(mm);
+ if (copy_from_vaddr(mm, vaddr, cur, sizeof(cur)) ||
+ !ptwrite_is_installed(mm, vaddr, cur))
+ return 0;
+
+ return text_poke_5byte(auprobe, vma, vaddr, auprobe->ptwrite.orig,
+ UPROBE_SWBP_INSN, false, false, false, false, NULL);
+}
+
+
static bool __is_optimized(struct mm_struct *mm, uprobe_opcode_t *insn, unsigned long vaddr)
{
struct __packed __arch_relative_insn {
--
2.54.0
^ permalink raw reply [flat|nested] 12+ messages in thread* [RFC PATCH v2 03/11] ptwrite uprobes: Add a sample module to exercise interface
2026-09-17 23:00 ptwrite uprobes v2 Andi Kleen
2026-09-17 23:00 ` [RFC PATCH v2 01/11] ptwrite uprobes: Add infrastructure for ptwrite uprobes Andi Kleen
2026-09-17 23:00 ` [RFC PATCH v2 02/11] ptwrite uprobes: Add minimal low level support for x86 Andi Kleen
@ 2026-09-17 23:00 ` Andi Kleen
2026-09-17 23:00 ` [RFC PATCH v2 04/11] ptwrite uprobes: Add support to tracing infrastructure Andi Kleen
` (7 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Andi Kleen @ 2026-09-17 23:00 UTC (permalink / raw)
To: Masami Hiramatsu
Cc: Oleg Nesterov, Peter Zijlstra, linux-kernel, linux-trace-kernel,
x86, tglx, jolsa, linux-perf-users, adrian.hunter, Andi Kleen
Add a basic test module and a test program for uprobes ptwrite.
The module allows to configure and register a ptwrite uprobe in a
executable.
Use this for testing. It is not intended as a production interface.
Assisted-by: omp:gpt-5.6-luna
Signed-off-by: Andi Kleen <ak@kernel.org>
---
samples/Kconfig | 9 ++
samples/Makefile | 1 +
samples/uprobe-ptwrite/Makefile | 9 ++
samples/uprobe-ptwrite/test_prog.c | 38 +++++
samples/uprobe-ptwrite/uprobe_ptwrite_test.c | 153 +++++++++++++++++++
5 files changed, 210 insertions(+)
create mode 100644 samples/uprobe-ptwrite/Makefile
create mode 100644 samples/uprobe-ptwrite/test_prog.c
create mode 100644 samples/uprobe-ptwrite/uprobe_ptwrite_test.c
diff --git a/samples/Kconfig b/samples/Kconfig
index a75e8e78330d..79da903edb62 100644
--- a/samples/Kconfig
+++ b/samples/Kconfig
@@ -322,6 +322,15 @@ config SAMPLE_HUNG_TASK
Reading these files with multiple processes triggers hung task
detection by holding locks for a long time (256 seconds).
+config SAMPLE_UPROBE_PTWRITE
+ tristate "Build ptwrite uprobe test module -- loadable module only"
+ depends on UPROBES && X86_64 && m
+ help
+ Builds a prototype ptwrite uprobe driver for testing Intel PTWRITE
+ support. Use the module parameters to select a file, offset, arguments,
+ and event ID. This interface is for testing and is not intended for
+ production use.
+
source "samples/rust/Kconfig"
source "samples/damon/Kconfig"
diff --git a/samples/Makefile b/samples/Makefile
index 07641e177bd8..0c970d46653e 100644
--- a/samples/Makefile
+++ b/samples/Makefile
@@ -44,4 +44,5 @@ obj-$(CONFIG_SAMPLE_DAMON_WSSE) += damon/
obj-$(CONFIG_SAMPLE_DAMON_PRCL) += damon/
obj-$(CONFIG_SAMPLE_DAMON_MTIER) += damon/
obj-$(CONFIG_SAMPLE_HUNG_TASK) += hung_task/
+obj-$(CONFIG_SAMPLE_UPROBE_PTWRITE) += uprobe-ptwrite/
obj-$(CONFIG_SAMPLE_TSM_MR) += tsm-mr/
diff --git a/samples/uprobe-ptwrite/Makefile b/samples/uprobe-ptwrite/Makefile
new file mode 100644
index 000000000000..0b81311dd4b8
--- /dev/null
+++ b/samples/uprobe-ptwrite/Makefile
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# ptwrite uprobe prototype sample.
+#
+# Module build (in-tree or via make M=samples/uprobe-ptwrite).
+# The userspace test target is built by the in-tree selftest harness in
+# tools/testing/selftests/uprobes.
+
+obj-$(CONFIG_SAMPLE_UPROBE_PTWRITE) += uprobe_ptwrite_test.o
diff --git a/samples/uprobe-ptwrite/test_prog.c b/samples/uprobe-ptwrite/test_prog.c
new file mode 100644
index 000000000000..1b57c522e215
--- /dev/null
+++ b/samples/uprobe-ptwrite/test_prog.c
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * test_prog - userspace target for the ptwrite uprobe test.
+ *
+ * target() carries a 5-byte NOP (nopl 0(%rax,%rax,1) = 0f 1f 44 00 00) at
+ * its entry. A probe installed at that file offset emits rdi/rsi (a, b)
+ * via PTWRITE. In the test loop a == i and b == i + 1.
+ */
+#include <stdio.h>
+
+#ifndef noinline
+#define noinline __attribute__((noinline))
+#endif
+
+static noinline unsigned long
+target(unsigned long a, unsigned long b)
+{
+ /*
+ * 5-byte NOP: nopl 0(%rax,%rax,1) = 0f 1f 44 00 00.
+ * Emitted as raw bytes: the assembler would otherwise shrink
+ * "nopl (%rax,%rax,1)" to the 4-byte form (0f 1f 04 00), which
+ * the probe site validation rejects (jmp rel32 needs 5 bytes).
+ */
+ asm volatile(".globl target_site\n\t"
+ "target_site:\n\t"
+ ".byte 0x0f, 0x1f, 0x44, 0x00, 0x00");
+ return a * 31 + b;
+}
+
+int main(void)
+{
+ unsigned long i, acc = 0;
+
+ for (i = 0; i < 2000; i++)
+ acc += target(i, i + 1);
+ printf("acc=%lu\n", acc);
+ return 0;
+}
diff --git a/samples/uprobe-ptwrite/uprobe_ptwrite_test.c b/samples/uprobe-ptwrite/uprobe_ptwrite_test.c
new file mode 100644
index 000000000000..94d01bf18faa
--- /dev/null
+++ b/samples/uprobe-ptwrite/uprobe_ptwrite_test.c
@@ -0,0 +1,153 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * uprobe_ptwrite_test - minimal ptwrite uprobe prototype driver.
+ *
+ * Registers a trap-free ptwrite uprobe at a user-specified file offset and
+ * emits the requested live registers / immediates into an externally
+ * configured Intel PT stream (no kernel entry at probe-hit time):
+ *
+ * perf record -e intel_pt/ptw=1,fup_on_ptw=1//u -o perf.data ./test_prog
+ *
+ * Usage (module params):
+ * path=/path/to/prog file to probe
+ * offset=0xADDR file offset of the probe site
+ * args="r0,r1,i0x42" comma-separated;
+ * r<N> = x86-64 GPR index 0..15,
+ * i<hex> = immediate constant,
+ * event_id=0x1234 identifier carried in the PTW header word
+ */
+#include <linux/module.h>
+#include <linux/uprobes.h>
+#include <linux/fs.h>
+#include <linux/shmem_fs.h>
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+static char *path = "/nonexistent";
+module_param(path, charp, 0444);
+MODULE_PARM_DESC(path, "path of the binary to probe");
+
+static ulong offset;
+module_param(offset, ulong, 0444);
+MODULE_PARM_DESC(offset, "file offset of the 5-byte NOP to probe");
+
+static ushort event_id = 0x1234;
+module_param(event_id, ushort, 0444);
+MODULE_PARM_DESC(event_id, "event id carried in the PTW header word");
+
+static char *args = "r0";
+module_param(args, charp, 0444);
+MODULE_PARM_DESC(args, "comma-separated args: r<N> GPR, i<hex> immediate, m<N>[:disp][:4|8] memory");
+
+static struct file *probe_file;
+static struct uprobe *probe;
+static struct uprobe_consumer consumer;
+static struct uprobe_ptwrite_desc desc;
+
+static int parse_probe_args(void)
+{
+ char *s, *p, *tok;
+ unsigned int n = 0;
+
+ s = kstrdup(args, GFP_KERNEL);
+ if (!s)
+ return -ENOMEM;
+
+ p = s;
+ while ((tok = strsep(&p, ",")) != NULL) {
+ struct uprobe_ptwrite_arg *a;
+
+ if (n >= UPROBE_PTWRITE_MAX_ARGS) {
+ pr_err("too many args\n");
+ goto err;
+ }
+ a = &desc.args[n];
+
+ a->size = 8;
+ if (tok[0] == 'r') {
+ unsigned long reg;
+
+ if (kstrtoul(tok + 1, 10, ®) || reg > 15) {
+ pr_err("bad reg '%s'\n", tok);
+ goto err;
+ }
+ a->src = UPROBE_PTW_SRC_REG;
+ a->reg = reg;
+ } else if (tok[0] == 'i') {
+ unsigned long long v;
+
+ if (kstrtoull(tok + 1, 0, &v)) {
+ pr_err("bad imm '%s'\n", tok);
+ goto err;
+ }
+ a->src = UPROBE_PTW_SRC_IMM;
+ a->val = v;
+
+ } else {
+ pr_err("bad arg '%s'\n", tok);
+ goto err;
+ }
+ n++;
+ }
+ if (!n) {
+ pr_err("need 1..%d args\n", UPROBE_PTWRITE_MAX_ARGS);
+ goto err;
+ }
+ desc.nargs = n;
+ kfree(s);
+ return 0;
+err:
+ kfree(s);
+ return -EINVAL;
+}
+
+static int __init uprobe_ptwrite_test_init(void)
+{
+ struct inode *inode;
+ int ret;
+
+ desc.event_id = event_id;
+ ret = parse_probe_args();
+ if (ret)
+ return ret;
+
+ probe_file = filp_open(path, O_RDONLY, 0);
+ if (IS_ERR(probe_file))
+ return PTR_ERR(probe_file);
+
+ inode = file_inode(probe_file);
+ if (!inode->i_mapping->a_ops->read_folio &&
+ !shmem_mapping(inode->i_mapping)) {
+ pr_err("unsupported mapping\n");
+ ret = -EIO;
+ goto out_file;
+ }
+
+ probe = uprobe_register_ptwrite(inode, probe_file, offset, &consumer, &desc);
+ if (IS_ERR(probe)) {
+ ret = PTR_ERR(probe);
+ pr_err("register failed: %d\n", ret);
+ goto out_file;
+ }
+
+ pr_info("probe %s+0x%lx, %u args, event_id=0x%x\n",
+ path, offset, desc.nargs, desc.event_id);
+ return 0;
+
+out_file:
+ fput(probe_file);
+ return ret;
+}
+
+static void __exit uprobe_ptwrite_test_exit(void)
+{
+ uprobe_unregister_nosync(probe, &consumer);
+ uprobe_unregister_sync();
+ fput(probe_file);
+ pr_info("unregistered\n");
+}
+
+module_init(uprobe_ptwrite_test_init);
+module_exit(uprobe_ptwrite_test_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("ptwrite uprobes testing driver");
--
2.54.0
^ permalink raw reply [flat|nested] 12+ messages in thread* [RFC PATCH v2 04/11] ptwrite uprobes: Add support to tracing infrastructure
2026-09-17 23:00 ptwrite uprobes v2 Andi Kleen
` (2 preceding siblings ...)
2026-09-17 23:00 ` [RFC PATCH v2 03/11] ptwrite uprobes: Add a sample module to exercise interface Andi Kleen
@ 2026-09-17 23:00 ` Andi Kleen
2026-09-17 23:00 ` [RFC PATCH v2 05/11] ptwrite uprobes: Factor file-backed instruction reads Andi Kleen
` (6 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Andi Kleen @ 2026-09-17 23:00 UTC (permalink / raw)
To: Masami Hiramatsu
Cc: Oleg Nesterov, Peter Zijlstra, linux-kernel, linux-trace-kernel,
x86, tglx, jolsa, linux-perf-users, adrian.hunter, Andi Kleen
Hook up the low level x86 ptwrite uprobes code to the generic trace
uprobes events parser, so that the new probes can be set up. The
interface is similar to classic probes, but there is new ptw: syntax
and various restrictions.
Add minimal docs.
Architectures without the ptwrite backend are handled by weak stubs.
Assisted-by: omp:gpt-5.6-luna
Signed-off-by: Andi Kleen <ak@kernel.org>
---
Documentation/trace/uprobetracer.rst | 46 +++++-
arch/x86/Kconfig | 1 +
arch/x86/include/asm/uprobes.h | 12 +-
include/linux/uprobes.h | 4 +
kernel/trace/Kconfig | 6 +
kernel/trace/trace_uprobe.c | 219 +++++++++++++++++++++++++--
6 files changed, 272 insertions(+), 16 deletions(-)
diff --git a/Documentation/trace/uprobetracer.rst b/Documentation/trace/uprobetracer.rst
index 01f6a780fb04..56d951d1fa5b 100644
--- a/Documentation/trace/uprobetracer.rst
+++ b/Documentation/trace/uprobetracer.rst
@@ -19,8 +19,8 @@ However unlike kprobe-event tracer, the uprobe event interface expects the
user to calculate the offset of the probepoint in the object.
You can also use /sys/kernel/tracing/dynamic_events instead of
-uprobe_events. That interface will provide unified access to other
-dynamic events too.
+uprobe_events. That interface provides unified access to other event types
+too.
Synopsis of uprobe_tracer
-------------------------
@@ -29,6 +29,7 @@ Synopsis of uprobe_tracer
p[:[GRP/][EVENT]] PATH:OFFSET [FETCHARGS] : Set a uprobe
r[:[GRP/][EVENT]] PATH:OFFSET [FETCHARGS] : Set a return uprobe (uretprobe)
p[:[GRP/][EVENT]] PATH:OFFSET%return [FETCHARGS] : Set a return uprobe (uretprobe)
+ ptw[:[GRP/][EVENT]] PATH:OFFSET [FETCHARGS] : Set a trap-free ptwrite uprobe
-:[GRP/][EVENT] : Clear uprobe or uretprobe event
GRP : Group name. If omitted, "uprobes" is the default value.
@@ -76,6 +77,47 @@ offset, and container-size (usually 32). The syntax is::
For $comm, the default type is "string"; any other type is invalid.
+ptwrite uprobes (ptw:)
+----------------------
+
+See ptwrite-uprobes.rst for more details.
+
+A ``ptw:`` probe emits values into an Intel Processor Trace data stream.
+It is faster than standard uprobes, but has restrictions. The Intel Processor
+Trace recording must be configured separately.
+
+Requirements and restrictions:
+
+- Intel CPU with PTWRITE support (``/sys/devices/intel_pt/format/ptw`` must exist)
+- Entry probes only: ``r:``/``%return`` and the SDT reference counter
+ ``(REF)`` are rejected.
+- The probe site must be a 5-byte NOP or a punnable instruction (see
+ ptwrite-uprobes.rst). For five ``0x90`` bytes from GCC's
+ ``-fpatchable-function-entry=5``, append ``%multinop`` to the offset.
+- The event does not produce ring-buffer records. It provides a type registry
+ (``events/GRP/EVENT/format``) and the wire ``event_id``
+ (``events/GRP/EVENT/id``) for an external decoder. Filters and perf
+ attach are rejected because filtering would need kernel entry.
+
+Wire format: each probe hit emits a 64-bit header word ``event_id<<48 |
+nargs<<40 | 0x5054525731`` followed by ``nargs`` PTWRITE payloads. Perf
+exposes each payload through its existing ``u64`` field. Decode with
+``perf script`` and ``tools/perf/scripts/python/uprobe-ptwrite-decode.py``:
+
+::
+
+ perf record -e intel_pt/ptw=1,fup_on_ptw=1/u -o perf.data -- ./app
+ perf script --itrace=qwe -s uprobe-ptwrite-decode.py
+
+``fup_on_ptw=1`` increases the overhead, but improves decoding reliability
+because the exact IP of each probe is logged.
+
+Example::
+
+ echo 'ptw:t /bin/app:0x1234 %rdi %rsi \0x42' > /sys/kernel/tracing/uprobe_events
+ echo 1 > /sys/kernel/tracing/events/uprobes/t/enable
+
+
Event Profiling
---------------
You can check the total number of probe hits per event via
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 7fda5edb0c06..4c67ff306950 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -423,6 +423,7 @@ config HAVE_INTEL_TXT
config ARCH_SUPPORTS_UPROBES
def_bool y
+ select UPROBE_EVENTS_PTWRITE if X86_64 && UPROBE_EVENTS
config FIX_EARLYCON_MEM
def_bool y
diff --git a/arch/x86/include/asm/uprobes.h b/arch/x86/include/asm/uprobes.h
index 20d086a945c5..b51907c1d465 100644
--- a/arch/x86/include/asm/uprobes.h
+++ b/arch/x86/include/asm/uprobes.h
@@ -29,6 +29,14 @@ enum {
struct uprobe_xol_ops;
+/*
+ * Stub block array size. Worst case = 250 B (8 MEM args, rsp bases, fault
+ * table); 288 leaves 38 B slack. A file-scope static_assert in
+ * arch/x86/kernel/uprobes.c re-derives the worst case; prepare() also
+ * enforces it with -E2BIG at runtime.
+ */
+#define UPROBE_PTWRITE_STUB_SIZE 288
+
/*
* ptwrite probe state. The stub template (code + data slots) is built
* once at registration (mm-independent except the final jmp's rel32, patched
@@ -36,8 +44,8 @@ struct uprobe_xol_ops;
* [ptwriteq hdr(%rip)] [arg emissions] [jmp probe+5] [u64 slots: header, imms]
*/
struct uprobe_ptwrite_arch {
- u8 stub[256];
- u8 stub_len; /* code + data, whole block */
+ u8 stub[UPROBE_PTWRITE_STUB_SIZE];
+ u16 stub_len; /* code + data + fault table, whole block */
u8 jmp_off; /* offset of the final jmp's rel32 field */
u8 ndata; /* number of u64 data slots */
u8 orig[MAX_UINSN_BYTES]; /* pristine file bytes, before generic analysis */
diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h
index 6b20d61cf737..6cea2944ca8b 100644
--- a/include/linux/uprobes.h
+++ b/include/linux/uprobes.h
@@ -205,8 +205,12 @@ struct uprobes_state {
enum uprobe_ptwrite_src {
UPROBE_PTW_SRC_REG, /* value = live GPR (index in .reg) */
UPROBE_PTW_SRC_IMM, /* value = constant (.val), stored in stub data slot */
+ UPROBE_PTW_SRC_MEM, /* value = [.reg + disp32] (v2); requires ALLOW_MEM */
};
+/* uprobe_ptwrite_desc.flags */
+#define UPROBE_PTWRITE_FL_ALLOW_MEM BIT(0) /* SRC_MEM args enabled */
+
struct uprobe_ptwrite_arg {
u8 src; /* enum uprobe_ptwrite_src */
u8 reg; /* x86-64 GPR index (0=rax..15=r15) for SRC_REG */
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index 0ab5916575a9..edbae5aa8dd3 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -848,6 +848,12 @@ config UPROBE_EVENTS
This option is required if you plan to use perf-probe subcommand
of perf tools on user space applications.
+config UPROBE_EVENTS_PTWRITE
+ bool
+ depends on UPROBE_EVENTS
+ help
+ ptwrite uprobes ("ptw:" tracefs event type).
+
config EPROBE_EVENTS
bool "Enable event-based dynamic events"
depends on TRACING
diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c
index 22cc3c8181b8..ab8ad3f6083d 100644
--- a/kernel/trace/trace_uprobe.c
+++ b/kernel/trace/trace_uprobe.c
@@ -18,6 +18,7 @@
#include <linux/security.h>
#include <linux/string.h>
#include <linux/uaccess.h>
+#include <linux/fs.h>
#include <linux/uprobes.h>
#include "trace.h"
@@ -66,6 +67,9 @@ struct trace_uprobe {
unsigned long offset;
unsigned long ref_ctr_offset;
unsigned long __percpu *nhits;
+ bool is_ptwrite;
+ struct uprobe_ptwrite_desc ptwrite_desc;
+ /* tp.args[] is a flex array and must remain the last member */
struct trace_probe tp;
};
@@ -275,7 +279,7 @@ static bool trace_uprobe_is_busy(struct dyn_event *ev)
{
struct trace_uprobe *tu = to_trace_uprobe(ev);
- return trace_probe_is_enabled(&tu->tp);
+ return trace_probe_is_enabled(&tu->tp) || tu->uprobe;
}
static bool trace_uprobe_match_command_head(struct trace_uprobe *tu,
@@ -510,7 +514,8 @@ static int register_trace_uprobe(struct trace_uprobe *tu)
old_tu = find_probe_event(trace_probe_name(&tu->tp),
trace_probe_group_name(&tu->tp));
if (old_tu) {
- if (is_ret_probe(tu) != is_ret_probe(old_tu)) {
+ if (is_ret_probe(tu) != is_ret_probe(old_tu) ||
+ tu->is_ptwrite != old_tu->is_ptwrite) {
trace_probe_log_set_index(0);
trace_probe_log_err(0, DIFF_PROBE_TYPE);
return -EEXIST;
@@ -535,9 +540,85 @@ static int register_trace_uprobe(struct trace_uprobe *tu)
DEFINE_FREE(free_trace_uprobe, struct trace_uprobe *, free_trace_uprobe(_T))
+/*
+ * Compile one parsed fetch arg into a ptwrite descriptor entry. The
+ * arch-independent part: decode the fetch chain, reject shapes the
+ * scratch-free stub cannot emit, and hand the rest to the arch hook.
+ */
+static int ptwrite_compile_arg(struct trace_uprobe *tu, int i)
+{
+ struct fetch_insn *code = tu->tp.args[i].code;
+ struct uprobe_ptwrite_arg *a = &tu->ptwrite_desc.args[i];
+ struct uprobe_ptwrite_fetch f;
+
+ if (code[1].op == FETCH_OP_ST_MEM || code[1].op == FETCH_OP_ST_UMEM) {
+ if (code[2].op != FETCH_OP_END)
+ return -EINVAL;
+ if (code[0].op != FETCH_OP_REG) {
+ /*
+ * +off($stackN): the STACK op derefs [rsp+8N] to a
+ * POINTER, and ST_MEM derefs that pointer (load-of-
+ * load). The scratch-free stub has no register to
+ * hold the intermediate pointer, so reject.
+ */
+ return -EINVAL;
+ }
+ if (tu->tp.args[i].type->size != 4 &&
+ tu->tp.args[i].type->size != 8)
+ return -EINVAL; /* memory derefs are u32 or u64 */
+ f.kind = UPROBE_PTW_FETCH_MEMREG;
+ f.reg = code[0].param;
+ f.imm = code[1].offset;
+ goto compile;
+ }
+
+ /*
+ * $stackN: [STACK, ST_RAW, END], the deref is folded inside the
+ * STACK op (get_user_stack_nth reads [rsp + 8N]).
+ */
+ if (code[0].op == FETCH_OP_STACK &&
+ code[1].op == FETCH_OP_ST_RAW && code[2].op == FETCH_OP_END) {
+ if (tu->tp.args[i].type->size != 4 &&
+ tu->tp.args[i].type->size != 8)
+ return -EINVAL; /* stack slots are u32 or u64 */
+ f.kind = UPROBE_PTW_FETCH_STACKN;
+ f.imm = 8L * code[0].param;
+ goto compile;
+ }
+
+ if (code[1].op != FETCH_OP_ST_RAW || code[2].op != FETCH_OP_END)
+ return -EINVAL;
+
+ switch (code->op) {
+ case FETCH_OP_REG: /* %reg */
+ f.kind = UPROBE_PTW_FETCH_REG;
+ f.reg = code->param;
+ break;
+ case FETCH_OP_STACKP: /* $stack: SP value, never faults */
+ f.kind = UPROBE_PTW_FETCH_STACKP;
+ break;
+ case FETCH_OP_IMM: /* \IMM */
+ f.kind = UPROBE_PTW_FETCH_IMM;
+ f.imm = code->immediate;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+compile:
+ if (f.kind == UPROBE_PTW_FETCH_MEMREG ||
+ f.kind == UPROBE_PTW_FETCH_STACKN)
+ tu->ptwrite_desc.flags |= UPROBE_PTWRITE_FL_ALLOW_MEM;
+ if (arch_uprobe_ptwrite_fetch(a, &f))
+ return -EINVAL;
+ a->size = tu->tp.args[i].type->size;
+ return 0;
+}
+
/*
* Argument syntax:
* - Add uprobe: p|r[:[GRP/][EVENT]] PATH:OFFSET[%return][(REF)] [FETCHARGS]
+ * - Add ptwrite uprobe: ptw[:[GRP/][EVENT]] PATH:OFFSET [FETCHARGS]
*/
static int __trace_uprobe_create(int argc, const char **argv)
{
@@ -553,10 +634,17 @@ static int __trace_uprobe_create(int argc, const char **argv)
char *buf __free(kfree) = NULL;
enum probe_print_type ptype;
bool is_return = false;
- int i, ret;
+ bool is_ptwrite = false;
+ int i, ret, arg_start = 2;
ref_ctr_offset = 0;
+ if (!strncmp(argv[0], "ptw:", 4) || !strcmp(argv[0], "ptw")) {
+ if (!IS_ENABLED(CONFIG_UPROBE_EVENTS_PTWRITE))
+ return -EOPNOTSUPP; /* no arch backend configured */
+ is_ptwrite = true;
+ }
+
switch (argv[0][0]) {
case 'r':
is_return = true;
@@ -572,13 +660,17 @@ static int __trace_uprobe_create(int argc, const char **argv)
trlog = trace_probe_log_init("trace_uprobe", argc, argv);
- if (argc - 2 > MAX_TRACE_ARGS) {
+ if (argc - 2 > MAX_TRACE_ARGS ||
+ (is_ptwrite && argc - 2 > UPROBE_PTWRITE_MAX_ARGS)) {
trace_probe_log_set_index(2);
trace_probe_log_err(0, TOO_MANY_ARGS);
return -E2BIG;
}
- if (argv[0][1] == ':')
+ if (is_ptwrite)
+ event = argv[0][3] == ':' && argv[0][4] ?
+ &argv[0][4] : NULL;
+ else if (argv[0][1] == ':')
event = &argv[0][2];
if (!strchr(argv[1], '/'))
@@ -608,6 +700,10 @@ static int __trace_uprobe_create(int argc, const char **argv)
/* Parse reference counter offset if specified. */
rctr = strchr(arg, '(');
+ if (rctr && is_ptwrite) {
+ trace_probe_log_err(rctr - filename, BAD_REFCNT);
+ return -EINVAL; /* SDT ref-counting needs kernel updates */
+ }
if (rctr) {
rctr_end = strchr(rctr, ')');
if (!rctr_end) {
@@ -632,7 +728,10 @@ static int __trace_uprobe_create(int argc, const char **argv)
/* Check if there is %return suffix */
tmp = strchr(arg, '%');
- if (tmp) {
+ if (tmp && is_ptwrite) {
+ trace_probe_log_err(tmp - filename, BAD_ADDR_SUFFIX);
+ return -EINVAL;
+ } else if (tmp) {
if (!strcmp(tmp, "%return")) {
*tmp = '\0';
is_return = true;
@@ -677,7 +776,8 @@ static int __trace_uprobe_create(int argc, const char **argv)
buf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
if (!buf)
return -ENOMEM;
- snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_0x%lx", 'p', tail, offset);
+ snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_0x%lx",
+ is_ptwrite ? 't' : 'p', tail, offset);
event = buf;
kfree(tail);
}
@@ -712,6 +812,25 @@ static int __trace_uprobe_create(int argc, const char **argv)
return ret;
}
+ if (is_ptwrite) {
+ if (!argc) {
+ trace_probe_log_set_index(2);
+ trace_probe_log_err(0, NO_ARG_BODY);
+ return -EINVAL; /* core rejects desc->nargs == 0 */
+ }
+ tu->is_ptwrite = true;
+ tu->ptwrite_desc.nargs = argc;
+ tu->ptwrite_desc.flags = 0;
+ for (i = 0; i < argc; i++) {
+ ret = ptwrite_compile_arg(tu, i);
+ if (ret) {
+ trace_probe_log_set_index(i + arg_start);
+ trace_probe_log_err(0, BAD_FETCH_ARG);
+ return ret;
+ }
+ }
+ }
+
ptype = is_ret_probe(tu) ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL;
ret = traceprobe_set_print_fmt(&tu->tp, ptype);
if (ret < 0)
@@ -754,9 +873,24 @@ static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev)
char c = is_ret_probe(tu) ? 'r' : 'p';
int i;
- seq_printf(m, "%c:%s/%s %s:0x%0*lx", c, trace_probe_group_name(&tu->tp),
- trace_probe_name(&tu->tp), tu->filename,
- (int)(sizeof(void *) * 2), tu->offset);
+ if (tu->is_ptwrite) {
+ seq_printf(m, "ptw:%s/%s %s:0x%0*lx",
+ trace_probe_group_name(&tu->tp),
+ trace_probe_name(&tu->tp), tu->filename,
+ (int)(sizeof(void *) * 2), tu->offset);
+ if (tu->ptwrite_desc.flags & UPROBE_PTWRITE_FL_NO_LEAD_PACE) {
+ seq_puts(m, "%nopace");
+ if (tu->ptwrite_desc.flags & UPROBE_PTWRITE_FL_ALLOW_NOP_RUN)
+ seq_putc(m, ' ');
+ }
+ if (tu->ptwrite_desc.flags & UPROBE_PTWRITE_FL_ALLOW_NOP_RUN)
+ seq_puts(m, "%multinop");
+ } else {
+ seq_printf(m, "%c:%s/%s %s:0x%0*lx", c,
+ trace_probe_group_name(&tu->tp),
+ trace_probe_name(&tu->tp), tu->filename,
+ (int)(sizeof(void *) * 2), tu->offset);
+ }
if (tu->ref_ctr_offset)
seq_printf(m, "(0x%lx)", tu->ref_ctr_offset);
@@ -1107,9 +1241,26 @@ static int trace_uprobe_enable(struct trace_uprobe *tu, filter_func_t filter)
{
struct inode *inode = d_real_inode(tu->path.dentry);
struct uprobe *uprobe;
+ struct file *file;
- tu->consumer.filter = filter;
- uprobe = uprobe_register(inode, tu->offset, tu->ref_ctr_offset, &tu->consumer);
+ if (tu->is_ptwrite) {
+ if (filter) {
+ /* PTWRITE probes have no kernel entry to evaluate a filter. */
+ return -EINVAL;
+ }
+ file = dentry_open(&tu->path, O_RDONLY, current_cred());
+ if (IS_ERR(file))
+ return PTR_ERR(file);
+ tu->ptwrite_desc.event_id =
+ trace_probe_event_call(&tu->tp)->event.type;
+ uprobe = uprobe_register_ptwrite(inode, file, tu->offset,
+ &tu->consumer, &tu->ptwrite_desc);
+ fput(file);
+ } else {
+ tu->consumer.filter = filter;
+ uprobe = uprobe_register(inode, tu->offset,
+ tu->ref_ctr_offset, &tu->consumer);
+ }
if (IS_ERR(uprobe))
return PTR_ERR(uprobe);
@@ -1148,6 +1299,28 @@ static int probe_event_enable(struct trace_event_call *call,
tp = trace_probe_primary_from_call(call);
if (WARN_ON_ONCE(!tp))
return -ENODEV;
+ tu = container_of(tp, struct trace_uprobe, tp);
+
+ if (tu->is_ptwrite) {
+ if (filter || !file || file->filter)
+ return -EINVAL;
+ enabled = trace_probe_is_enabled(tp);
+ ret = trace_probe_add_file(tp, file);
+ if (ret < 0)
+ return ret;
+ if (enabled)
+ return 0;
+ list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {
+ ret = trace_uprobe_enable(tu, NULL);
+ if (ret) {
+ __probe_event_disable(tp);
+ trace_probe_remove_file(tp, file);
+ return ret;
+ }
+ }
+ return 0;
+ }
+
enabled = trace_probe_is_enabled(tp);
/* This may also change "enabled" state */
@@ -1201,11 +1374,22 @@ static void probe_event_disable(struct trace_event_call *call,
struct trace_event_file *file)
{
struct trace_probe *tp;
+ struct trace_uprobe *tu;
tp = trace_probe_primary_from_call(call);
if (WARN_ON_ONCE(!tp))
return;
+ tu = container_of(tp, struct trace_uprobe, tp);
+ if (tu->is_ptwrite) {
+ if (trace_probe_remove_file(tp, file) < 0)
+ return;
+ if (trace_probe_is_enabled(tp))
+ return; /* other instances still enabled */
+ __probe_event_disable(tp);
+ return;
+ }
+
if (!trace_probe_is_enabled(tp))
return;
@@ -1493,6 +1677,13 @@ int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type,
}
#endif /* CONFIG_PERF_EVENTS */
+static bool ptwrite_event(struct trace_event_call *call)
+{
+ struct trace_uprobe *tu = trace_uprobe_primary_from_call(call);
+
+ return tu && tu->is_ptwrite;
+}
+
static int
trace_uprobe_register(struct trace_event_call *event, enum trace_reg type,
void *data)
@@ -1516,9 +1707,13 @@ trace_uprobe_register(struct trace_event_call *event, enum trace_reg type,
return 0;
case TRACE_REG_PERF_OPEN:
+ if (ptwrite_event(event))
+ return -EINVAL; /* no perf attach to ptwrite events */
return uprobe_perf_open(event, data);
case TRACE_REG_PERF_CLOSE:
+ if (ptwrite_event(event))
+ return -EINVAL;
return uprobe_perf_close(event, data);
#endif
--
2.54.0
^ permalink raw reply [flat|nested] 12+ messages in thread* [RFC PATCH v2 05/11] ptwrite uprobes: Factor file-backed instruction reads
2026-09-17 23:00 ptwrite uprobes v2 Andi Kleen
` (3 preceding siblings ...)
2026-09-17 23:00 ` [RFC PATCH v2 04/11] ptwrite uprobes: Add support to tracing infrastructure Andi Kleen
@ 2026-09-17 23:00 ` Andi Kleen
2026-09-17 23:00 ` [RFC PATCH v2 06/11] ptwrite uprobes: Add basic memory references Andi Kleen
` (5 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Andi Kleen @ 2026-09-17 23:00 UTC (permalink / raw)
To: Masami Hiramatsu
Cc: Oleg Nesterov, Peter Zijlstra, linux-kernel, linux-trace-kernel,
x86, tglx, jolsa, linux-perf-users, adrian.hunter, Andi Kleen
Refactor copy_insn into a more generic uprobe_copy_from_file.
The existing copy_insn is still there, but uses the generic
version now. The generic version will be used in later patches.
No semantic change intended.
Assisted-by: omp:gpt-5.6-luna
Signed-off-by: Andi Kleen <ak@kernel.org>
---
include/linux/uprobes.h | 2 ++
kernel/events/uprobes.c | 59 +++++++++++++++++++++++++++++------------
2 files changed, 44 insertions(+), 17 deletions(-)
diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h
index 6cea2944ca8b..6fa70f3f648c 100644
--- a/include/linux/uprobes.h
+++ b/include/linux/uprobes.h
@@ -304,6 +304,8 @@ extern void uprobe_handle_trampoline(struct pt_regs *regs);
extern void *arch_uretprobe_trampoline(unsigned long *psize);
extern unsigned long uprobe_get_trampoline_vaddr(void);
extern void uprobe_copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len);
+extern int uprobe_copy_from_file(struct inode *inode, struct file *file,
+ loff_t offset, void *buf, int size);
extern void arch_uprobe_clear_state(struct mm_struct *mm);
extern void arch_uprobe_init_state(struct mm_struct *mm);
extern void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr);
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 30c28625bb5f..894196f3089f 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -1073,30 +1073,55 @@ static int __copy_insn(struct address_space *mapping, struct file *filp,
return 0;
}
-static int copy_insn(struct uprobe *uprobe, struct file *filp)
+/**
+ * uprobe_copy_from_file - read bytes from a file's page cache
+ * @inode: the file's inode
+ * @file: file used by the filesystem's read_folio callback
+ * @offset: byte offset into the file
+ * @buf: destination buffer
+ * @size: number of bytes to read (may cross page boundaries)
+ *
+ * Callers that require a full instruction must check that the requested size
+ * was copied.
+ */
+int uprobe_copy_from_file(struct inode *inode, struct file *file,
+ loff_t offset, void *buf, int size)
{
- struct address_space *mapping = uprobe->inode->i_mapping;
- loff_t offs = uprobe->offset;
- void *insn = &uprobe->arch.insn;
- int size = sizeof(uprobe->arch.insn);
- int len, err = -EIO;
+ struct address_space *mapping = inode->i_mapping;
+ int len, copied = 0, err;
- /* Copy only available bytes, -EIO if nothing was read */
- do {
- if (offs >= i_size_read(uprobe->inode))
+ if (offset < 0 || size < 0)
+ return -EINVAL;
+
+ while (copied < size) {
+ if (offset >= i_size_read(inode))
break;
- len = min_t(int, size, PAGE_SIZE - (offs & ~PAGE_MASK));
- err = __copy_insn(mapping, filp, insn, len, offs);
+ /* Let the page cache zero-fill bytes past i_size in the final page. */
+ len = min_t(int, size - copied,
+ PAGE_SIZE - (offset & ~PAGE_MASK));
+ err = __copy_insn(mapping, file, buf + copied, len, offset);
if (err)
- break;
+ return err;
+
+ copied += len;
+ offset += len;
+ }
+ return copied;
+}
- insn += len;
- offs += len;
- size -= len;
- } while (size);
+static int copy_insn(struct uprobe *uprobe, struct file *filp)
+{
+ int ret;
- return err;
+ ret = uprobe_copy_from_file(uprobe->inode, filp, uprobe->offset,
+ &uprobe->arch.insn,
+ sizeof(uprobe->arch.insn));
+ if (ret < 0)
+ return ret;
+ if (!ret)
+ return -EIO;
+ return 0;
}
static int prepare_uprobe(struct uprobe *uprobe, struct file *file,
--
2.54.0
^ permalink raw reply [flat|nested] 12+ messages in thread* [RFC PATCH v2 06/11] ptwrite uprobes: Add basic memory references
2026-09-17 23:00 ptwrite uprobes v2 Andi Kleen
` (4 preceding siblings ...)
2026-09-17 23:00 ` [RFC PATCH v2 05/11] ptwrite uprobes: Factor file-backed instruction reads Andi Kleen
@ 2026-09-17 23:00 ` Andi Kleen
2026-09-17 23:00 ` [RFC PATCH v2 07/11] ptwrite uprobes: Add multinop support Andi Kleen
` (4 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Andi Kleen @ 2026-09-17 23:00 UTC (permalink / raw)
To: Masami Hiramatsu
Cc: Oleg Nesterov, Peter Zijlstra, linux-kernel, linux-trace-kernel,
x86, tglx, jolsa, linux-perf-users, adrian.hunter, Andi Kleen
Add support for memory references. Currently this is only
simple cases, no indirect memory references or strings,
that would require saving/restoring registers. Only 8 and 4 byte
memory references are supported.
This doesn't have any fault handling yet, so a bad reference
in the probe could result in crashing the application.
It increases the kernel memory overhead of each probe to roughly
1 KB.
Assisted-by: omp:gpt-5.6-luna
Signed-off-by: Andi Kleen <ak@kernel.org>
---
Documentation/trace/uprobetracer.rst | 7 ++
arch/x86/include/asm/uprobes.h | 2 +-
arch/x86/kernel/uprobes.c | 90 +++++++++++++++++---
include/linux/uprobes.h | 4 +-
samples/uprobe-ptwrite/uprobe_ptwrite_test.c | 40 ++++++++-
5 files changed, 128 insertions(+), 15 deletions(-)
diff --git a/Documentation/trace/uprobetracer.rst b/Documentation/trace/uprobetracer.rst
index 56d951d1fa5b..5d891a96c4fe 100644
--- a/Documentation/trace/uprobetracer.rst
+++ b/Documentation/trace/uprobetracer.rst
@@ -94,6 +94,13 @@ Requirements and restrictions:
- The probe site must be a 5-byte NOP or a punnable instruction (see
ptwrite-uprobes.rst). For five ``0x90`` bytes from GCC's
``-fpatchable-function-entry=5``, append ``%multinop`` to the offset.
+- FETCHARGS: register names (``%di``, ``%r8``, ...), ``$stack`` (the
+ stack pointer value), immediates (``\IMM``), and memory sources:
+ ``$stackN`` (the Nth stack slot, ``[%rsp + 8N]``) and ``+off(FETCHARG)``
+ dereferences (for example, ``+8(%di)`` = ``[%rdi + 8]``). ``u64`` sources
+ use ``ptwriteq``; ``u32``/``s32``/``x32`` sources use ``ptwritel`` and read
+ four bytes. Strings, bitfields, and indirect dereferences are not supported.
+- Memory accesses can fault.
- The event does not produce ring-buffer records. It provides a type registry
(``events/GRP/EVENT/format``) and the wire ``event_id``
(``events/GRP/EVENT/id``) for an external decoder. Filters and perf
diff --git a/arch/x86/include/asm/uprobes.h b/arch/x86/include/asm/uprobes.h
index b51907c1d465..e5a668ba5ad6 100644
--- a/arch/x86/include/asm/uprobes.h
+++ b/arch/x86/include/asm/uprobes.h
@@ -45,7 +45,7 @@ struct uprobe_xol_ops;
*/
struct uprobe_ptwrite_arch {
u8 stub[UPROBE_PTWRITE_STUB_SIZE];
- u16 stub_len; /* code + data + fault table, whole block */
+ u16 stub_len; /* code + data, whole block */
u8 jmp_off; /* offset of the final jmp's rel32 field */
u8 ndata; /* number of u64 data slots */
u8 orig[MAX_UINSN_BYTES]; /* pristine file bytes, before generic analysis */
diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c
index 7fcdc4bf5197..0029b66cd64e 100644
--- a/arch/x86/kernel/uprobes.c
+++ b/arch/x86/kernel/uprobes.c
@@ -1365,7 +1365,10 @@ static const struct {
{ .off = offsetof(struct pt_regs, r15), .idx = 15 },
};
-/* Compile the register, stack-pointer, and immediate fetch forms. */
+/*
+ * Compile one tracefs fetch arg (arch-neutral form, see
+ * uprobe_ptwrite_fetch) into a ptwrite descriptor entry.
+ */
int arch_uprobe_ptwrite_fetch(struct uprobe_ptwrite_arg *a,
const struct uprobe_ptwrite_fetch *f)
{
@@ -1373,6 +1376,7 @@ int arch_uprobe_ptwrite_fetch(struct uprobe_ptwrite_arg *a,
switch (f->kind) {
case UPROBE_PTW_FETCH_REG:
+ case UPROBE_PTW_FETCH_MEMREG:
for (i = 0; i < ARRAY_SIZE(ptwrite_reg_map); i++) {
if (ptwrite_reg_map[i].off == f->reg) {
idx = ptwrite_reg_map[i].idx;
@@ -1380,15 +1384,23 @@ int arch_uprobe_ptwrite_fetch(struct uprobe_ptwrite_arg *a,
}
}
if (idx < 0)
- return -EINVAL;
- a->src = UPROBE_PTW_SRC_REG;
+ return -EINVAL; /* not an x86-64 GPR */
+ a->src = f->kind == UPROBE_PTW_FETCH_REG ?
+ UPROBE_PTW_SRC_REG : UPROBE_PTW_SRC_MEM;
a->reg = idx;
+ if (f->kind == UPROBE_PTW_FETCH_MEMREG)
+ a->val = (u64)(s32)f->imm;
break;
- case UPROBE_PTW_FETCH_STACKP:
+ case UPROBE_PTW_FETCH_STACKP: /* $stack: SP value, never faults */
a->src = UPROBE_PTW_SRC_REG;
a->reg = 4; /* rsp */
break;
- case UPROBE_PTW_FETCH_IMM:
+ case UPROBE_PTW_FETCH_STACKN: /* [rsp + imm] */
+ a->src = UPROBE_PTW_SRC_MEM;
+ a->reg = 4; /* rsp */
+ a->val = f->imm;
+ break;
+ case UPROBE_PTW_FETCH_IMM: /* \IMM */
a->src = UPROBE_PTW_SRC_IMM;
a->val = f->imm;
break;
@@ -1398,6 +1410,26 @@ int arch_uprobe_ptwrite_fetch(struct uprobe_ptwrite_arg *a,
return 0;
}
+/*
+ * Worst-case stub block: header ptwriteq (9) + max memory args (10 bytes
+ * each, including a SIB byte) + final jmp (5), rounded up; data adds one
+ * header slot and one slot per immediate. Keep the bound below the stub size.
+ */
+static_assert((((9 + UPROBE_PTWRITE_MAX_ARGS * 10 + 5 + 7) & ~7) +
+ 8 * (1 + UPROBE_PTWRITE_MAX_ARGS)) <= UPROBE_PTWRITE_STUB_SIZE,
+ "worst-case ptwrite stub block exceeds UPROBE_PTWRITE_STUB_SIZE");
+
+static bool ptwrite_has_room(const u8 *base, const u8 *p, size_t len)
+{
+ return p >= base && (size_t)(p - base) <=
+ sizeof(((struct uprobe_ptwrite_arch *)0)->stub) - len;
+}
+
+#define PTW_NEED(_len) do { \
+ if (!ptwrite_has_room(code, p, (_len))) \
+ return -E2BIG; \
+ } while (0)
+
int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
const struct uprobe_ptwrite_desc *desc)
{
@@ -1414,7 +1446,7 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
return -EINVAL;
if (desc->nargs > UPROBE_PTWRITE_MAX_ARGS)
return -E2BIG;
- if (desc->flags)
+ if (desc->flags & ~UPROBE_PTWRITE_FL_ALLOW_MEM)
return -EINVAL;
/* The generic registration path copied these bytes before this hook. */
@@ -1431,24 +1463,62 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
return -E2BIG;
n_imm++;
break;
+ case UPROBE_PTW_SRC_MEM:
+ if (!(desc->flags & UPROBE_PTWRITE_FL_ALLOW_MEM))
+ return -EINVAL;
+ if (desc->args[i].reg > 15)
+ return -EINVAL;
+ if (desc->args[i].size != 4 &&
+ desc->args[i].size != 8)
+ return -EINVAL;
+ break;
default:
return -EINVAL;
}
}
/* header word emission (disp32 patched below) */
+ PTW_NEED(9);
p += ptwrite_emit_riprel(p, 0);
for (i = 0; i < desc->nargs; i++) {
- if (desc->args[i].src == UPROBE_PTW_SRC_REG) {
+ switch (desc->args[i].src) {
+ case UPROBE_PTW_SRC_REG:
+ PTW_NEED(5);
p += ptwrite_emit_reg(p, desc->args[i].reg);
- } else {
+ break;
+ case UPROBE_PTW_SRC_IMM:
+ if (imm_idx >= ARRAY_SIZE(imm_off))
+ return -E2BIG;
+ PTW_NEED(9);
imm_off[imm_idx++] = p - code;
p += ptwrite_emit_riprel(p, 0);
+ break;
+ case UPROBE_PTW_SRC_MEM: {
+ u8 reg = desc->args[i].reg;
+ bool wide = desc->args[i].size == 8;
+ unsigned int arg_len = (wide ? 9 : 8) +
+ ((reg & 7) == 4) + (!wide && (reg & 8));
+
+ PTW_NEED(arg_len);
+ *p++ = 0xf3;
+ if (wide)
+ *p++ = (reg & 8) ? 0x49 : 0x48; /* REX.W */
+ else if (reg & 8)
+ *p++ = 0x41; /* REX.B only (32-bit operand) */
+ *p++ = 0x0f;
+ *p++ = 0xae;
+ *p++ = 0xa0 | (reg & 7); /* mod 10, reg /4, rm reg */
+ if ((reg & 7) == 4) /* SIB escape: base rsp/esp/r12 */
+ *p++ = 0x24;
+ p += 4;
+ break;
+ }
}
}
/* final jmp back to probe+5; rel32 patched per-mm at install */
+ PTW_NEED(5);
*p++ = 0xe9;
if (p - code > U8_MAX)
return -E2BIG;
@@ -1460,7 +1530,8 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
return -E2BIG;
/* data slots: header, then imm values in emission order */
- hdr = ((u64)desc->event_id << 48) | ((u64)desc->nargs << 40);
+ hdr = ((u64)desc->event_id << 48) | ((u64)desc->nargs << 40) |
+ UPROBE_PTW_HDR_MAGIC;
*(u64 *)(code + data_off) = hdr;
/* patch the header's disp32: hdr slot - end of header insn */
@@ -1755,7 +1826,6 @@ int arch_uprobe_uninstall_ptwrite(struct arch_uprobe *auprobe,
UPROBE_SWBP_INSN, false, false, false, false, NULL);
}
-
static bool __is_optimized(struct mm_struct *mm, uprobe_opcode_t *insn, unsigned long vaddr)
{
struct __packed __arch_relative_insn {
diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h
index 6fa70f3f648c..f6ffb0637991 100644
--- a/include/linux/uprobes.h
+++ b/include/linux/uprobes.h
@@ -213,10 +213,10 @@ enum uprobe_ptwrite_src {
struct uprobe_ptwrite_arg {
u8 src; /* enum uprobe_ptwrite_src */
- u8 reg; /* x86-64 GPR index (0=rax..15=r15) for SRC_REG */
+ u8 reg; /* x86-64 GPR index (0=rax..15=r15) for SRC_REG/SRC_MEM */
u8 size; /* declared type size 1/2/4/8 (decoder hint) */
u8 reserved;
- u64 val; /* SRC_IMM: constant; SRC_REG: unused */
+ u64 val; /* SRC_IMM: constant, SRC_MEM: disp32 (low 32 bits) */
};
struct uprobe_ptwrite_desc {
diff --git a/samples/uprobe-ptwrite/uprobe_ptwrite_test.c b/samples/uprobe-ptwrite/uprobe_ptwrite_test.c
index 94d01bf18faa..55e23ff27dd5 100644
--- a/samples/uprobe-ptwrite/uprobe_ptwrite_test.c
+++ b/samples/uprobe-ptwrite/uprobe_ptwrite_test.c
@@ -11,9 +11,11 @@
* Usage (module params):
* path=/path/to/prog file to probe
* offset=0xADDR file offset of the probe site
- * args="r0,r1,i0x42" comma-separated;
+ * args="r0,r1,i0x42,m3,m2:0x8,m4:0x10:4" comma-separated;
* r<N> = x86-64 GPR index 0..15,
* i<hex> = immediate constant,
+ * m<N>[:<disp>][:<size>] = memory arg [reg + disp32],
+ * size 4 (u32 load) or 8 (u64 load, default)
* event_id=0x1234 identifier carried in the PTW header word
*/
#include <linux/module.h>
@@ -81,7 +83,41 @@ static int parse_probe_args(void)
}
a->src = UPROBE_PTW_SRC_IMM;
a->val = v;
-
+ } else if (tok[0] == 'm') {
+ /*
+ * m<R>[:<disp>][:<size>]: memory arg [reg + disp32],
+ * size 4 (u32 load) or 8 (u64 load, default)
+ */
+ char *colon = strchr(tok, ':');
+ char *szs = NULL;
+ unsigned long reg;
+ long long disp = 0;
+ unsigned long size = 8;
+
+ if (colon) {
+ *colon = '\0';
+ szs = strchr(colon + 1, ':');
+ if (szs)
+ *szs++ = '\0';
+ }
+ if (kstrtoul(tok + 1, 10, ®) || reg > 15) {
+ pr_err("bad mem reg '%s'\n", tok);
+ goto err;
+ }
+ if (colon && kstrtoll(colon + 1, 0, &disp)) {
+ pr_err("bad mem disp '%s'\n", colon + 1);
+ goto err;
+ }
+ if (szs && (kstrtoul(szs, 10, &size) ||
+ (size != 4 && size != 8))) {
+ pr_err("bad mem size '%s'\n", szs);
+ goto err;
+ }
+ a->src = UPROBE_PTW_SRC_MEM;
+ a->reg = reg;
+ a->val = (u64)(s32)disp;
+ a->size = size;
+ desc.flags |= UPROBE_PTWRITE_FL_ALLOW_MEM;
} else {
pr_err("bad arg '%s'\n", tok);
goto err;
--
2.54.0
^ permalink raw reply [flat|nested] 12+ messages in thread* [RFC PATCH v2 07/11] ptwrite uprobes: Add multinop support
2026-09-17 23:00 ptwrite uprobes v2 Andi Kleen
` (5 preceding siblings ...)
2026-09-17 23:00 ` [RFC PATCH v2 06/11] ptwrite uprobes: Add basic memory references Andi Kleen
@ 2026-09-17 23:00 ` Andi Kleen
2026-09-17 23:00 ` [RFC PATCH v2 08/11] ptwrite uprobes: Support instruction punning Andi Kleen
` (3 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Andi Kleen @ 2026-09-17 23:00 UTC (permalink / raw)
To: Masami Hiramatsu
Cc: Oleg Nesterov, Peter Zijlstra, linux-kernel, linux-trace-kernel,
x86, tglx, jolsa, linux-perf-users, adrian.hunter, Andi Kleen
GCC's -fpatchable-function-entry=5 may emit five one-byte NOPs. Normally
that's not safe to patch because some might jump into a later nop.
But for the gcc case it's safe because nobody jumps into the nops.
Add a %multinop that allows the user opting into patching these sites.
This way patching for the gcc instrumentation works.
Assisted-by: omp:gpt-5.6-luna
Signed-off-by: Andi Kleen <ak@kernel.org>
---
arch/x86/include/asm/uprobes.h | 24 ++++++--
arch/x86/kernel/uprobes.c | 63 +++++++++++++++++---
include/linux/uprobes.h | 2 +
kernel/trace/trace_uprobe.c | 58 +++++++++++++-----
samples/uprobe-ptwrite/uprobe_ptwrite_test.c | 6 ++
5 files changed, 126 insertions(+), 27 deletions(-)
diff --git a/arch/x86/include/asm/uprobes.h b/arch/x86/include/asm/uprobes.h
index e5a668ba5ad6..c46b3fe09025 100644
--- a/arch/x86/include/asm/uprobes.h
+++ b/arch/x86/include/asm/uprobes.h
@@ -30,12 +30,23 @@ enum {
struct uprobe_xol_ops;
/*
- * Stub block array size. Worst case = 250 B (8 MEM args, rsp bases, fault
- * table); 288 leaves 38 B slack. A file-scope static_assert in
- * arch/x86/kernel/uprobes.c re-derives the worst case; prepare() also
- * enforces it with -E2BIG at runtime.
+ * Stub block size. Conservative worst case is 298 bytes: 9-byte header,
+ * one lead fence, eight 21-byte memory forms with one 3-byte fence each,
+ * a 16-byte original-instruction copy, a 5-byte return jump, alignment,
+ * and 66 bytes of data/fault metadata. 384 leaves room. A static_assert in
+ * arch/x86/kernel/uprobes.c checks the bound; prepare() also checks it with
+ * -E2BIG.
*/
-#define UPROBE_PTWRITE_STUB_SIZE 288
+#define UPROBE_PTWRITE_STUB_SIZE 384
+
+
+/*
+ * Word pacing: insert this many LFENCEs between emitted ptwrite words and
+ * before the first word, unless UPROBE_PTWRITE_FL_NO_LEAD_PACE is requested.
+ */
+#define UPROBE_PTWRITE_SERIALIZE_LFENCES 1 /* LFENCEs per word gap */
+/* The encoded LFENCE instruction occupies three bytes. */
+#define UPROBE_PTWRITE_LFENCE_SIZE 3
/*
* ptwrite probe state. The stub template (code + data slots) is built
@@ -49,6 +60,9 @@ struct uprobe_ptwrite_arch {
u8 jmp_off; /* offset of the final jmp's rel32 field */
u8 ndata; /* number of u64 data slots */
u8 orig[MAX_UINSN_BYTES]; /* pristine file bytes, before generic analysis */
+ u16 ft_off; /* fault table offset within the block (0 if none) */
+ u8 nft; /* number of fault entries */
+ bool allow_nop_run; /* accept a five-byte run of 0x90 */
};
/* Per-mm page holding generated ptwrite stub blocks, like tramp_mapping. */
diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c
index 0029b66cd64e..f915dda8bcb4 100644
--- a/arch/x86/kernel/uprobes.c
+++ b/arch/x86/kernel/uprobes.c
@@ -1323,6 +1323,24 @@ static int ptwrite_emit_riprel(u8 *p, s32 disp)
return 9;
}
+static int ptwrite_emit_lfence(u8 *p)
+{
+ *p++ = 0x0f;
+ *p++ = 0xae;
+ *p++ = 0xe8; /* LFENCE */
+ return UPROBE_PTWRITE_LFENCE_SIZE;
+}
+
+/* The default pacing: one or more fences per word gap. */
+static int ptwrite_emit_lfences(u8 *p)
+{
+ int i;
+
+ for (i = 0; i < UPROBE_PTWRITE_SERIALIZE_LFENCES; i++)
+ p += ptwrite_emit_lfence(p);
+ return UPROBE_PTWRITE_SERIALIZE_LFENCES *
+ UPROBE_PTWRITE_LFENCE_SIZE;
+}
bool arch_uprobe_ptwrite_supported(void)
{
u32 eax, ebx, ecx, edx;
@@ -1411,13 +1429,20 @@ int arch_uprobe_ptwrite_fetch(struct uprobe_ptwrite_arg *a,
}
/*
- * Worst-case stub block: header ptwriteq (9) + max memory args (10 bytes
- * each, including a SIB byte) + final jmp (5), rounded up; data adds one
- * header slot and one slot per immediate. Keep the bound below the stub size.
+ * Worst-case paced stub before instruction punning: a 9-byte header, one
+ * lead fence, one fence after the header, one fence between each argument,
+ * the largest memory form (10 bytes), and the return jump. Data adds one
+ * header slot and one slot per immediate.
*/
-static_assert((((9 + UPROBE_PTWRITE_MAX_ARGS * 10 + 5 + 7) & ~7) +
- 8 * (1 + UPROBE_PTWRITE_MAX_ARGS)) <= UPROBE_PTWRITE_STUB_SIZE,
- "worst-case ptwrite stub block exceeds UPROBE_PTWRITE_STUB_SIZE");
+static_assert((((9 + UPROBE_PTWRITE_SERIALIZE_LFENCES *
+ UPROBE_PTWRITE_LFENCE_SIZE +
+ UPROBE_PTWRITE_MAX_ARGS *
+ (10 + UPROBE_PTWRITE_SERIALIZE_LFENCES *
+ UPROBE_PTWRITE_LFENCE_SIZE) +
+ 5 + 7) & ~7) +
+ 8 * (1 + UPROBE_PTWRITE_MAX_ARGS)) <=
+ UPROBE_PTWRITE_STUB_SIZE,
+ "worst-case ptwrite stub block exceeds UPROBE_PTWRITE_STUB_SIZE");
static bool ptwrite_has_room(const u8 *base, const u8 *p, size_t len)
{
@@ -1439,6 +1464,7 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
unsigned int data_off;
unsigned int hdr_off = 0;
unsigned int imm_idx = 0, n_imm = 0;
+ bool paced = false;
u64 hdr;
int i;
@@ -1446,7 +1472,9 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
return -EINVAL;
if (desc->nargs > UPROBE_PTWRITE_MAX_ARGS)
return -E2BIG;
- if (desc->flags & ~UPROBE_PTWRITE_FL_ALLOW_MEM)
+ if (desc->flags & ~(UPROBE_PTWRITE_FL_ALLOW_MEM |
+ UPROBE_PTWRITE_FL_NO_LEAD_PACE |
+ UPROBE_PTWRITE_FL_ALLOW_NOP_RUN))
return -EINVAL;
/* The generic registration path copied these bytes before this hook. */
@@ -1477,9 +1505,22 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
}
}
+ paced = !(desc->flags & UPROBE_PTWRITE_FL_NO_LEAD_PACE);
+ if (paced) {
+ PTW_NEED(UPROBE_PTWRITE_SERIALIZE_LFENCES *
+ UPROBE_PTWRITE_LFENCE_SIZE);
+ p += ptwrite_emit_lfences(p);
+ }
+
/* header word emission (disp32 patched below) */
PTW_NEED(9);
+ hdr_off = p - code;
p += ptwrite_emit_riprel(p, 0);
+ if (paced) {
+ PTW_NEED(UPROBE_PTWRITE_SERIALIZE_LFENCES *
+ UPROBE_PTWRITE_LFENCE_SIZE);
+ p += ptwrite_emit_lfences(p);
+ }
for (i = 0; i < desc->nargs; i++) {
switch (desc->args[i].src) {
@@ -1515,9 +1556,14 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
break;
}
}
+ if (paced && i + 1 < desc->nargs) {
+ PTW_NEED(UPROBE_PTWRITE_SERIALIZE_LFENCES *
+ UPROBE_PTWRITE_LFENCE_SIZE);
+ p += ptwrite_emit_lfences(p);
+ }
}
- /* final jmp back to probe+5; rel32 patched per-mm at install */
+ /* final jmp back to probe+len; rel32 patched per-mm at install */
PTW_NEED(5);
*p++ = 0xe9;
if (p - code > U8_MAX)
@@ -1549,6 +1595,7 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
ptw->stub_len = data_off + 8 * (1 + n_imm);
ptw->ndata = 1 + n_imm;
+ ptw->allow_nop_run = desc->flags & UPROBE_PTWRITE_FL_ALLOW_NOP_RUN;
return 0;
}
#undef PTW_NEED
diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h
index f6ffb0637991..5f16799c9fb9 100644
--- a/include/linux/uprobes.h
+++ b/include/linux/uprobes.h
@@ -210,6 +210,8 @@ enum uprobe_ptwrite_src {
/* uprobe_ptwrite_desc.flags */
#define UPROBE_PTWRITE_FL_ALLOW_MEM BIT(0) /* SRC_MEM args enabled */
+#define UPROBE_PTWRITE_FL_NO_LEAD_PACE BIT(1) /* don't slow down probes */
+#define UPROBE_PTWRITE_FL_ALLOW_NOP_RUN BIT(2) /* accept five 1-byte NOPs */
struct uprobe_ptwrite_arg {
u8 src; /* enum uprobe_ptwrite_src */
diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c
index ab8ad3f6083d..a226b89e5b9a 100644
--- a/kernel/trace/trace_uprobe.c
+++ b/kernel/trace/trace_uprobe.c
@@ -635,6 +635,8 @@ static int __trace_uprobe_create(int argc, const char **argv)
enum probe_print_type ptype;
bool is_return = false;
bool is_ptwrite = false;
+ bool is_nopace = false;
+ bool is_nop_run = false;
int i, ret, arg_start = 2;
ref_ctr_offset = 0;
@@ -660,13 +662,6 @@ static int __trace_uprobe_create(int argc, const char **argv)
trlog = trace_probe_log_init("trace_uprobe", argc, argv);
- if (argc - 2 > MAX_TRACE_ARGS ||
- (is_ptwrite && argc - 2 > UPROBE_PTWRITE_MAX_ARGS)) {
- trace_probe_log_set_index(2);
- trace_probe_log_err(0, TOO_MANY_ARGS);
- return -E2BIG;
- }
-
if (is_ptwrite)
event = argv[0][3] == ':' && argv[0][4] ?
&argv[0][4] : NULL;
@@ -728,9 +723,19 @@ static int __trace_uprobe_create(int argc, const char **argv)
/* Check if there is %return suffix */
tmp = strchr(arg, '%');
+ if (tmp && is_ptwrite && !strcmp(tmp, "%nopace")) {
+ *tmp = '\0';
+ is_nopace = true;
+ tmp = NULL;
+ }
if (tmp && is_ptwrite) {
- trace_probe_log_err(tmp - filename, BAD_ADDR_SUFFIX);
- return -EINVAL;
+ if (!strcmp(tmp, "%multinop")) {
+ *tmp = '\0';
+ is_nop_run = true;
+ } else {
+ trace_probe_log_err(tmp - filename, BAD_ADDR_SUFFIX);
+ return -EINVAL;
+ }
} else if (tmp) {
if (!strcmp(tmp, "%return")) {
*tmp = '\0';
@@ -747,6 +752,28 @@ static int __trace_uprobe_create(int argc, const char **argv)
trace_probe_log_err(arg - filename, BAD_UPROBE_OFFS);
return ret;
}
+ if (is_ptwrite && arg_start < argc &&
+ !strcmp(argv[arg_start], "%nopace")) {
+ is_nopace = true;
+ arg_start++;
+ }
+ if (is_ptwrite) {
+ while (arg_start < argc && !strcmp(argv[arg_start], "%multinop")) {
+ is_nop_run = true;
+ arg_start++;
+ }
+ if (arg_start < argc && !strcmp(argv[arg_start], "%nopace")) {
+ is_nopace = true;
+ arg_start++;
+ }
+ }
+
+ if (argc - arg_start > MAX_TRACE_ARGS ||
+ (is_ptwrite && argc - arg_start > UPROBE_PTWRITE_MAX_ARGS)) {
+ trace_probe_log_set_index(arg_start);
+ trace_probe_log_err(0, TOO_MANY_ARGS);
+ return -E2BIG;
+ }
/* setup a probe */
trace_probe_log_set_index(0);
@@ -782,8 +809,8 @@ static int __trace_uprobe_create(int argc, const char **argv)
kfree(tail);
}
- argc -= 2;
- argv += 2;
+ argc -= arg_start;
+ argv += arg_start;
tu = alloc_trace_uprobe(group, event, argc, is_return);
if (IS_ERR(tu)) {
@@ -806,7 +833,7 @@ static int __trace_uprobe_create(int argc, const char **argv)
/* parse arguments */
for (i = 0; i < argc; i++) {
- trace_probe_log_set_index(i + 2);
+ trace_probe_log_set_index(i + arg_start);
ret = traceprobe_parse_probe_arg(&tu->tp, i, argv[i], ctx);
if (ret)
return ret;
@@ -814,13 +841,16 @@ static int __trace_uprobe_create(int argc, const char **argv)
if (is_ptwrite) {
if (!argc) {
- trace_probe_log_set_index(2);
+ trace_probe_log_set_index(arg_start);
trace_probe_log_err(0, NO_ARG_BODY);
return -EINVAL; /* core rejects desc->nargs == 0 */
}
tu->is_ptwrite = true;
tu->ptwrite_desc.nargs = argc;
- tu->ptwrite_desc.flags = 0;
+ tu->ptwrite_desc.flags = is_nop_run ?
+ UPROBE_PTWRITE_FL_ALLOW_NOP_RUN : 0;
+ if (is_nopace)
+ tu->ptwrite_desc.flags |= UPROBE_PTWRITE_FL_NO_LEAD_PACE;
for (i = 0; i < argc; i++) {
ret = ptwrite_compile_arg(tu, i);
if (ret) {
diff --git a/samples/uprobe-ptwrite/uprobe_ptwrite_test.c b/samples/uprobe-ptwrite/uprobe_ptwrite_test.c
index 55e23ff27dd5..ca28d511b60c 100644
--- a/samples/uprobe-ptwrite/uprobe_ptwrite_test.c
+++ b/samples/uprobe-ptwrite/uprobe_ptwrite_test.c
@@ -17,6 +17,7 @@
* m<N>[:<disp>][:<size>] = memory arg [reg + disp32],
* size 4 (u32 load) or 8 (u64 load, default)
* event_id=0x1234 identifier carried in the PTW header word
+ * allow_nop_run=1 accept five one-byte NOPs at the site
*/
#include <linux/module.h>
#include <linux/uprobes.h>
@@ -36,6 +37,10 @@ static ushort event_id = 0x1234;
module_param(event_id, ushort, 0444);
MODULE_PARM_DESC(event_id, "event id carried in the PTW header word");
+static bool allow_nop_run;
+module_param(allow_nop_run, bool, 0444);
+MODULE_PARM_DESC(allow_nop_run, "accept five one-byte NOPs at the site");
+
static char *args = "r0";
module_param(args, charp, 0444);
MODULE_PARM_DESC(args, "comma-separated args: r<N> GPR, i<hex> immediate, m<N>[:disp][:4|8] memory");
@@ -142,6 +147,7 @@ static int __init uprobe_ptwrite_test_init(void)
int ret;
desc.event_id = event_id;
+ desc.flags = allow_nop_run ? UPROBE_PTWRITE_FL_ALLOW_NOP_RUN : 0;
ret = parse_probe_args();
if (ret)
return ret;
--
2.54.0
^ permalink raw reply [flat|nested] 12+ messages in thread* [RFC PATCH v2 08/11] ptwrite uprobes: Support instruction punning
2026-09-17 23:00 ptwrite uprobes v2 Andi Kleen
` (6 preceding siblings ...)
2026-09-17 23:00 ` [RFC PATCH v2 07/11] ptwrite uprobes: Add multinop support Andi Kleen
@ 2026-09-17 23:00 ` Andi Kleen
2026-09-17 23:00 ` [RFC PATCH v2 09/11] ptwrite uprobes: Use atomic patching for multinop sites Andi Kleen
` (2 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Andi Kleen @ 2026-09-17 23:00 UTC (permalink / raw)
To: Masami Hiramatsu
Cc: Oleg Nesterov, Peter Zijlstra, linux-kernel, linux-trace-kernel,
x86, tglx, jolsa, linux-perf-users, adrian.hunter, Andi Kleen
The previous ptwrite instrumentation only worked on 5 byte+ nops because
it needs to patch in a 5 byte branch.
However that is somewhat limiting because it means most code cannot
be probed. Use a simplified variant of the instruction
punning technique from Chamith et al. "Instruction Punning: Lightweight
instrumentation for x86-64". Patch only the one-byte branch opcode and
reuse the existing 4 following bytes in the code as the branch target.
If someone branches to the remaining bytes they are still executed
in the original way because they didn't change.
This requires placing a target trampoline page at the right address. If
the area is not available or points to kernel space it doesn't work.
In general it is somewhat unreliable for non PIE executables because
the target is often negative and ends up in kernel space. However
on modern Linux distributions near all binaries are PIE and high
up in the address space with ample gaps around them, which makes
punning much more successful.
Testing every instruction in a PIE-linked Debian 13 /bin/bash:
- bash has around 210k instructions
- of which around 10k are directly patchable 5 byte nops
- running bash for 100 times with ASLR 76.4% of the instructions
are always pun probeable
- with 23% of the instruction never being punnable
- the rest sometimes depending on ASLR luck.
So punning is not perfect, but it works most of the time for these
kind of binaries, and is much better than just nops.
If the probing fails it's possible for the harness to move the probe
around until it finds a better target. Sometimes you're just
lucky on rerun with ASLR. Or alternatively just use a classic
uprobe.
Return unique errnos for all instruction error cases so the harness
can make informed decisions.
Add the low-level machinery for punning. Classify
the instruction, poke the target and map trampolines to the right place.
Various special cases are not supported to simplify the code.
One difference to the nop probing is that the previous instruction
needs to be copied and fixed up (analogous to classic uprobes)
The punning technique could be also used with optimized
uprobes, but this patch only applies it to ptwrite probes.
Assisted-by: omp:gpt-5.6-luna
Signed-off-by: Andi Kleen <ak@kernel.org>
---
arch/x86/include/asm/uprobes.h | 24 +-
arch/x86/kernel/uprobes.c | 460 ++++++++++++++++++++++++++++++---
include/linux/uprobes.h | 4 +-
kernel/events/uprobes.c | 18 +-
4 files changed, 461 insertions(+), 45 deletions(-)
diff --git a/arch/x86/include/asm/uprobes.h b/arch/x86/include/asm/uprobes.h
index c46b3fe09025..53d88e37a771 100644
--- a/arch/x86/include/asm/uprobes.h
+++ b/arch/x86/include/asm/uprobes.h
@@ -39,6 +39,8 @@ struct uprobe_xol_ops;
*/
#define UPROBE_PTWRITE_STUB_SIZE 384
+/* the out-of-line original-instruction copy slot (x86 max insn length) */
+#define UPROBE_PTWRITE_COPY_SIZE MAX_UINSN_BYTES
/*
* Word pacing: insert this many LFENCEs between emitted ptwrite words and
@@ -50,14 +52,19 @@ struct uprobe_xol_ops;
/*
* ptwrite probe state. The stub template (code + data slots) is built
- * once at registration (mm-independent except the final jmp's rel32, patched
- * per-mm at install). Block layout:
- * [ptwriteq hdr(%rip)] [arg emissions] [jmp probe+5] [u64 slots: header, imms]
+ * once at registration. Only the final jmp's rel32 and the copy's
+ * disp/rel fields are patched per-mm at install. Block layout:
+ * [ptwriteq hdr(%rip)] [arg emissions] [orig-insn copy]
+ * [jmp probe+len] [u64 slots: header, imms]
*/
struct uprobe_ptwrite_arch {
u8 stub[UPROBE_PTWRITE_STUB_SIZE];
u16 stub_len; /* code + data, whole block */
u8 jmp_off; /* offset of the final jmp's rel32 field */
+ u8 copy_off; /* offset of the out-of-line instruction copy */
+ u8 len; /* copy length (0 = drop); back-jmp = vaddr+len */
+ u8 disp_off; /* rip-relative disp32 offset in the copy (0 = none) */
+ s32 disp; /* original disp32 (delta-patched per-mm) */
u8 ndata; /* number of u64 data slots */
u8 orig[MAX_UINSN_BYTES]; /* pristine file bytes, before generic analysis */
u16 ft_off; /* fault table offset within the block (0 if none) */
@@ -72,6 +79,17 @@ struct uprobe_ptwrite_page {
struct page *page; /* stub blocks written via kmap */
unsigned long vaddr; /* mapping base */
u16 cursor; /* next free block offset */
+ u16 nblocks;
+ struct {
+ u16 off; /* block offset in the page */
+ u16 len; /* generated block length */
+ u8 orig0; /* original site byte 0 (pun restore) */
+ u8 pun; /* instruction-pun mechanism (single-byte poke) */
+ u8 site_len; /* original instruction length (pun identity) */
+ s32 site_off; /* probe site - page base (idempotent reinstall) */
+ u8 site_insn[MAX_UINSN_BYTES]; /* original bytes (pun identity) */
+ } index[PAGE_SIZE / 32]; /* exact: min block = 32 B (nargs >= 1), */
+ /* so <= 128 blocks fit a page */
};
struct arch_uprobe {
diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c
index f915dda8bcb4..af9b36a219d3 100644
--- a/arch/x86/kernel/uprobes.c
+++ b/arch/x86/kernel/uprobes.c
@@ -1019,8 +1019,17 @@ static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t *
{
struct write_opcode_ctx *ctx = data;
uprobe_opcode_t old_opcode[OPT_INSN_SIZE];
+ int len;
- uprobe_copy_from_page(page, ctx->base, old_opcode, OPT_INSN_SIZE);
+ /*
+ * Byte-state checks need only the first byte. Optimized-state checks
+ * inspect the complete ten-byte instruction.
+ */
+ len = ctx->expect == EXPECT_OPTIMIZED ||
+ ctx->expect == EXPECT_SWBP_OPTIMIZED ? OPT_INSN_SIZE : 1;
+ if (PAGE_SIZE - (ctx->base & ~PAGE_MASK) < len)
+ return -1;
+ uprobe_copy_from_page(page, ctx->base, old_opcode, len);
switch (ctx->expect) {
case EXPECT_SWBP:
@@ -1450,12 +1459,19 @@ static bool ptwrite_has_room(const u8 *base, const u8 *p, size_t len)
sizeof(((struct uprobe_ptwrite_arch *)0)->stub) - len;
}
+static bool pun_site_is_nop(const u8 *orig, bool allow_nop_run);
+static int pun_classify_insn(struct insn *insn, u8 *disp_off, s32 *disp);
+static int pun_decode_site(struct inode *inode, struct file *file,
+ loff_t offset, u8 *copy,
+ u8 *disp_off, s32 *disp, bool allow_nop_run);
#define PTW_NEED(_len) do { \
if (!ptwrite_has_room(code, p, (_len))) \
return -E2BIG; \
} while (0)
int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
+ struct inode *inode, struct file *file,
+ loff_t offset,
const struct uprobe_ptwrite_desc *desc)
{
struct uprobe_ptwrite_arch *ptw = &auprobe->ptwrite;
@@ -1466,7 +1482,7 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
unsigned int imm_idx = 0, n_imm = 0;
bool paced = false;
u64 hdr;
- int i;
+ int i, ret;
if (!desc || desc->nargs == 0)
return -EINVAL;
@@ -1563,6 +1579,13 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
}
}
+ /* the out-of-line original-instruction copy slot (patched per-mm) */
+ PTW_NEED(UPROBE_PTWRITE_COPY_SIZE);
+ if (p - code > U8_MAX)
+ return -E2BIG;
+ ptw->copy_off = p - code;
+ p += UPROBE_PTWRITE_COPY_SIZE;
+
/* final jmp back to probe+len; rel32 patched per-mm at install */
PTW_NEED(5);
*p++ = 0xe9;
@@ -1596,6 +1619,15 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
ptw->stub_len = data_off + 8 * (1 + n_imm);
ptw->ndata = 1 + n_imm;
ptw->allow_nop_run = desc->flags & UPROBE_PTWRITE_FL_ALLOW_NOP_RUN;
+
+ ret = pun_decode_site(inode, file, offset, code + ptw->copy_off,
+ &ptw->disp_off, &ptw->disp,
+ ptw->allow_nop_run);
+ if (ret < 0)
+ return ret;
+ ptw->len = ret;
+ memset(code + ptw->copy_off + ptw->len, 0x90,
+ UPROBE_PTWRITE_COPY_SIZE - ptw->len);
return 0;
}
#undef PTW_NEED
@@ -1654,15 +1686,10 @@ static unsigned long find_ptwrite_page_area(struct mm_struct *mm,
}
static struct uprobe_ptwrite_page *
-create_uprobe_ptwrite_page(struct mm_struct *mm, unsigned long vaddr)
+create_uprobe_ptwrite_page_at(struct mm_struct *mm, unsigned long area)
{
struct uprobe_ptwrite_page *ptw;
struct vm_area_struct *vma;
- unsigned long area;
-
- area = find_ptwrite_page_area(mm, vaddr);
- if (IS_ERR_VALUE(area))
- return NULL;
mmap_assert_write_locked(mm);
@@ -1685,6 +1712,17 @@ create_uprobe_ptwrite_page(struct mm_struct *mm, unsigned long vaddr)
}
return ptw;
}
+
+static struct uprobe_ptwrite_page *
+create_uprobe_ptwrite_page(struct mm_struct *mm, unsigned long vaddr)
+{
+ unsigned long area = find_ptwrite_page_area(mm, vaddr);
+
+ if (IS_ERR_VALUE(area) || security_mmap_addr(area))
+ return NULL;
+ return create_uprobe_ptwrite_page_at(mm, area);
+}
+
static struct uprobe_ptwrite_page *
get_uprobe_ptwrite_page(struct mm_struct *mm, unsigned long vaddr,
unsigned int len)
@@ -1714,34 +1752,131 @@ get_uprobe_ptwrite_page(struct mm_struct *mm, unsigned long vaddr,
return ptw;
}
-/* Probe site must be a 5-byte NOP that does not cross a page boundary. */
-static int ptwrite_validate_site(const u8 *orig, unsigned long vaddr)
+/*
+ * A run of short NOPs is accepted only when requested. This validation does
+ * not make the three-phase poke safe for threads that already passed byte 0.
+ */
+static bool ptwrite_is_nop_run(const u8 *orig)
+{
+ return !memchr_inv(orig, 0x90, 5);
+}
+
+static bool pun_site_is_nop(const u8 *orig, bool allow_nop_run)
{
struct insn insn;
int ret;
- int off = 0;
- /*
- * The 5 displaced bytes must be NOPs: either one 5-byte NOP
- * (nopl 0x0(%rax,%rax,1)) or a run of shorter NOPs summing to
- * exactly 5 (gcc -fpatchable-function-entry=5 emits 5 x 0x90 on
- * modern toolchains). Any non-NOP byte, or a NOP crossing the
- * 5-byte window, is rejected.
- */
- while (off < 5) {
- ret = insn_decode(&insn, orig + off, 5 - off, INSN_MODE_64);
- if (ret < 0)
- return -EINVAL;
- if (insn.length < 1 || insn.length > 5 - off ||
- !insn_is_nop(&insn))
- return -EINVAL;
- off += insn.length;
+ ret = insn_decode(&insn, orig, 5, INSN_MODE_64);
+ if (ret < 0)
+ return false;
+ if (insn.length == 5 && insn_is_nop(&insn))
+ return true;
+ if (!allow_nop_run)
+ return false;
+ return ptwrite_is_nop_run(orig);
+}
+
+/* Identify the explicitly opted-in run of five one-byte NOPs. */
+static bool ptwrite_site_is_multinop(const u8 *orig, bool allow_nop_run)
+{
+ return allow_nop_run && ptwrite_is_nop_run(orig);
+}
+
+/*
+ * Classify the site's single instruction for out-of-line execution.
+ * Returns the length, or a negative errno when it cannot run safely out of
+ * line.
+ */
+static int pun_classify_insn(struct insn *insn, u8 *disp_off, s32 *disp)
+{
+ u8 op = insn->opcode.bytes[0];
+
+ switch (op) {
+ case 0xcc: /* int3 */
+ case 0xcd: /* int imm8 */
+ case 0xce: /* into */
+ case 0xcf: /* iret */
+ case 0xf1: /* int1 */
+ case 0xea: /* jmp far */
+ case 0x9a: /* call far */
+ /* Could be handled with special case code. */
+ case 0xe8: /* call rel32 */
+ case 0xe0: /* loopne rel8: cannot run out of line */
+ case 0xe1: /* loope rel8 */
+ case 0xe2: /* loop rel8 */
+ case 0xe3: /* jecxz/jrcxz */
+ /* These two could be handled if the offsets fit */
+ case 0xe9: /* jmp rel32 */
+ case 0xeb: /* jmp rel8 */
+ case 0x70 ... 0x7f: /* jcc rel8 */
+ return -EOPNOTSUPP;
+ }
+ /* XBEGIN's rel32 abort target is IP-relative, not RIP-relative. */
+ if (op == 0xc7 && insn->modrm.nbytes &&
+ X86_MODRM_MOD(insn->modrm.value) == 3 &&
+ X86_MODRM_REG(insn->modrm.value) == 7 &&
+ X86_MODRM_RM(insn->modrm.value) == 0)
+ return -EOPNOTSUPP;
+ if (op == 0x0f) {
+ switch (insn->opcode.bytes[1]) {
+ case 0x05: /* syscall */
+ case 0x34: /* sysenter */
+ case 0x35: /* sysexit */
+ return -EOPNOTSUPP;
+ }
+ /* jcc rel32: could be handled if offsets fit */
+ if (insn->opcode.bytes[1] >= 0x80 &&
+ insn->opcode.bytes[1] <= 0x8f)
+ return -EOPNOTSUPP;
+ /* Allow endbranch because this is incompatible with CET anyways */
+ }
+ if (op == 0xff) {
+ u8 reg = X86_MODRM_REG(insn->modrm.value);
+
+ /* call/lcall/jmp-far indirect */
+ if (reg == 2 || reg == 3 || reg == 5)
+ return -EOPNOTSUPP;
+ }
+
+ if (insn_rip_relative(insn)) {
+ *disp_off = insn_offset_displacement(insn);
+ insn_get_displacement(insn);
+ *disp = insn->displacement.value;
}
- if (off != 5)
+ return insn->length;
+}
+
+/*
+ * Read the site's instruction bytes from the file and classify them.
+ * The bytes are identical in every mm, so the copy is mm-independent.
+ */
+static int pun_decode_site(struct inode *inode, struct file *file,
+ loff_t offset, u8 *copy,
+ u8 *disp_off, s32 *disp, bool allow_nop_run)
+{
+ u8 buf[MAX_UINSN_BYTES] = {};
+ struct insn insn;
+ int ret;
+
+ ret = uprobe_copy_from_file(inode, file, offset, buf,
+ MAX_UINSN_BYTES);
+ if (ret < 0)
+ return ret;
+ if (!ret)
+ return -EIO;
+
+ if (pun_site_is_nop(buf, allow_nop_run))
+ return 0;
+
+ /* Check single instruction */
+ if (insn_decode(&insn, buf, MAX_UINSN_BYTES, INSN_MODE_64))
return -EINVAL;
- if (PAGE_SIZE - (vaddr & ~PAGE_MASK) < 5)
+ if (insn.length < 1 || insn.length > MAX_UINSN_BYTES)
return -EINVAL;
- return 0;
+
+ /* the original bytes verbatim; classify only validates them */
+ memcpy(copy, buf, insn.length);
+ return pun_classify_insn(&insn, disp_off, disp);
}
static bool ptwrite_rel32(unsigned long from, unsigned long to, s32 *rel)
@@ -1805,16 +1940,184 @@ static int ptwrite_text_poke(struct arch_uprobe *auprobe,
return err;
}
+static int pun_text_poke(struct arch_uprobe *auprobe,
+ struct vm_area_struct *vma,
+ unsigned long vaddr, u8 e9,
+ struct write_opcode_ctx *ctx)
+{
+ int err;
+
+ err = uprobe_write(auprobe, vma, vaddr, &e9, 1, verify_insn,
+ true, false, ctx);
+ if (err)
+ return err;
+ smp_text_poke_sync_each_cpu();
+ return 0;
+}
+
+static int pun_install(struct arch_uprobe *auprobe,
+ struct vm_area_struct *vma, unsigned long vaddr,
+ const u8 *orig)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ struct uprobe_ptwrite_page *ptw;
+ struct uprobe_ptwrite_arch *ptw_a = &auprobe->ptwrite;
+ struct uprobes_state *state = &mm->uprobes_state;
+ struct write_opcode_ctx ctx = {
+ .base = vaddr,
+ .expect = EXPECT_BYTE,
+ .expect_byte = orig[0],
+ };
+ unsigned long t, page_base, block_off, stub_addr;
+ s64 site_delta, target;
+ s32 jump_rel, disp32, orig_rel;
+ u8 site_len;
+ bool found = false;
+ bool nop_fallback = ptwrite_site_is_multinop(orig,
+ ptw_a->allow_nop_run) &&
+ (vaddr & 7);
+ u8 *kaddr;
+ int b, ret;
+
+ mmap_assert_write_locked(mm);
+ if (nop_fallback) {
+ hlist_for_each_entry(ptw, &state->head_ptwrite, node) {
+ site_delta = (s64)vaddr - (s64)ptw->vaddr;
+ if (site_delta < INT_MIN || site_delta > INT_MAX)
+ continue;
+ for (b = 0; b < smp_load_acquire(&ptw->nblocks); b++)
+ if (!ptw->index[b].pun &&
+ ptw->index[b].site_off == (s32)site_delta &&
+ ptw->index[b].site_len == 5 &&
+ !memcmp(ptw->index[b].site_insn, orig, 5))
+ break;
+ if (b >= smp_load_acquire(&ptw->nblocks))
+ continue;
+ if (!__in_uprobe_ptwrite(mm, ptw->vaddr))
+ continue;
+ return ptwrite_text_poke(auprobe, vma, vaddr,
+ ptw->vaddr + ptw->index[b].off);
+ }
+ ptw = get_uprobe_ptwrite_page(mm, vaddr, ptw_a->stub_len);
+ if (!ptw)
+ return -ENOMEM;
+ block_off = ptw->cursor;
+ } else {
+ memcpy(&orig_rel, orig + 1, sizeof(orig_rel));
+ target = (s64)vaddr + 5 + (s64)orig_rel;
+ if (target < PAGE_SIZE || target >= TASK_SIZE_MAX)
+ return -EADDRNOTAVAIL;
+ t = (unsigned long)target;
+ page_base = t & PAGE_MASK;
+ block_off = t & (PAGE_SIZE - 1);
+ if (block_off + ptw_a->stub_len > PAGE_SIZE)
+ return -ENOSPC;
+
+ /* reuse an existing ptwrite page at the target, else map a new one */
+ hlist_for_each_entry(ptw, &state->head_ptwrite, node) {
+ if (ptw->vaddr == page_base) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ if (vma_lookup(mm, page_base))
+ return -EADDRNOTAVAIL; /* target page occupied */
+ ptw = create_uprobe_ptwrite_page_at(mm, page_base);
+ if (!ptw)
+ return -ENOMEM;
+ /* Order page initialization before publishing it to fault readers. */
+ smp_wmb();
+ hlist_add_head_rcu(&ptw->node, &state->head_ptwrite);
+ }
+ }
+
+ site_delta = (s64)vaddr - (s64)ptw->vaddr;
+ if (site_delta < INT_MIN || site_delta > INT_MAX)
+ return -ERANGE;
+
+ for (b = 0; b < ptw->nblocks; b++) {
+ u16 old_off = ptw->index[b].off;
+ u16 old_len = ptw->index[b].len;
+
+ if (block_off + ptw_a->stub_len <= old_off ||
+ old_off + old_len <= block_off)
+ continue;
+ if (old_off == block_off && ptw->index[b].pun &&
+ old_len == ptw_a->stub_len &&
+ ptw->index[b].site_off == (s32)site_delta &&
+ ptw->index[b].site_len == ptw_a->len &&
+ !memcmp(ptw->index[b].site_insn, ptw_a->orig,
+ ptw_a->len))
+ return pun_text_poke(auprobe, vma, vaddr, 0xe9, &ctx);
+ return -EADDRNOTAVAIL;
+ }
+ if (ptw->nblocks >= ARRAY_SIZE(ptw->index))
+ return -ENOMEM;
+
+ stub_addr = ptw->vaddr + block_off;
+ if (!ptwrite_rel32(stub_addr + ptw_a->jmp_off + 4,
+ vaddr + (ptw_a->len ? ptw_a->len : 5), &jump_rel))
+ return -ERANGE;
+ if (ptw_a->len && ptw_a->disp_off) {
+ s64 d = (s64)ptw_a->disp + (s64)vaddr -
+ (s64)(stub_addr + ptw_a->copy_off);
+
+ if (d < INT_MIN || d > INT_MAX)
+ return -ERANGE;
+ disp32 = (s32)d;
+ }
+
+ /*
+ * A NOP fallback needs a synthetic rel32 at the site, so it uses
+ * the full five-byte poke and restore path rather than punning.
+ */
+ site_len = nop_fallback ? 5 : ptw_a->len;
+ ptw->index[ptw->nblocks].off = block_off;
+ ptw->index[ptw->nblocks].len = ptw_a->stub_len;
+ ptw->index[ptw->nblocks].pun = !nop_fallback;
+ ptw->index[ptw->nblocks].orig0 = orig[0];
+ ptw->index[ptw->nblocks].site_len = site_len;
+ ptw->index[ptw->nblocks].site_off = (s32)site_delta;
+ memcpy(ptw->index[ptw->nblocks].site_insn, ptw_a->orig, site_len);
+ smp_store_release(&ptw->nblocks, ptw->nblocks + 1);
+
+ kaddr = kmap_local_page(ptw->page);
+ memcpy(kaddr + block_off, ptw_a->stub, ptw_a->stub_len);
+ memcpy(kaddr + block_off + ptw_a->jmp_off, &jump_rel, sizeof(jump_rel));
+ if (ptw_a->len && ptw_a->disp_off)
+ memcpy(kaddr + block_off + ptw_a->copy_off + ptw_a->disp_off,
+ &disp32, sizeof(disp32));
+ kunmap_local(kaddr);
+
+ if (nop_fallback)
+ ret = ptwrite_text_poke(auprobe, vma, vaddr, stub_addr);
+ else
+ ret = pun_text_poke(auprobe, vma, vaddr, 0xe9, &ctx);
+ if (ret) {
+ /* Publish rollback before readers observe the reduced block count. */
+ smp_store_release(&ptw->nblocks, ptw->nblocks - 1);
+ return ret;
+ }
+
+ if (block_off + ptw_a->stub_len > ptw->cursor)
+ ptw->cursor = block_off + ptw_a->stub_len;
+ return 0;
+}
+
int arch_uprobe_install_ptwrite(struct arch_uprobe *auprobe,
struct vm_area_struct *vma, unsigned long vaddr)
{
struct mm_struct *mm = vma->vm_mm;
+ struct uprobes_state *state = &mm->uprobes_state;
struct uprobe_ptwrite_page *ptw;
struct uprobe_ptwrite_arch *ptw_a = &auprobe->ptwrite;
unsigned long block_off, stub_addr;
u8 *kaddr, orig[5];
+ s64 site_delta;
s32 rel;
int ret;
+ int b;
if (!is_64bit_mm(mm))
return -EOPNOTSUPP;
@@ -1829,10 +2132,28 @@ int arch_uprobe_install_ptwrite(struct arch_uprobe *auprobe,
if (ptwrite_is_installed(mm, vaddr, orig))
return 0;
- ret = ptwrite_validate_site(orig, vaddr);
- if (ret)
- return ret;
+ if (!pun_site_is_nop(orig, ptw_a->allow_nop_run))
+ return pun_install(auprobe, vma, vaddr, orig);
+ hlist_for_each_entry(ptw, &state->head_ptwrite, node) {
+ site_delta = (s64)vaddr - (s64)ptw->vaddr;
+ if (site_delta < INT_MIN || site_delta > INT_MAX)
+ continue;
+ /* Acquire the published count before reading block metadata. */
+ for (b = 0; b < smp_load_acquire(&ptw->nblocks); b++)
+ if (!ptw->index[b].pun &&
+ ptw->index[b].site_off == (s32)site_delta &&
+ ptw->index[b].site_len == sizeof(orig) &&
+ !memcmp(ptw->index[b].site_insn, orig, sizeof(orig)))
+ break;
+ /* Recheck the published count with acquire ordering. */
+ if (b >= smp_load_acquire(&ptw->nblocks))
+ continue;
+ if (!__in_uprobe_ptwrite(mm, ptw->vaddr))
+ continue;
+ return ptwrite_text_poke(auprobe, vma, vaddr,
+ ptw->vaddr + ptw->index[b].off);
+ }
ptw = get_uprobe_ptwrite_page(mm, vaddr, ptw_a->stub_len);
if (!ptw)
return -ENOMEM;
@@ -1845,6 +2166,19 @@ int arch_uprobe_install_ptwrite(struct arch_uprobe *auprobe,
if (!ptwrite_rel32(stub_addr + ptw_a->jmp_off + 4,
vaddr + 5, &rel))
return -ERANGE;
+ site_delta = (s64)vaddr - (s64)ptw->vaddr;
+ if (site_delta < INT_MIN || site_delta > INT_MAX)
+ return -ERANGE;
+
+ ptw->index[ptw->nblocks].off = block_off;
+ ptw->index[ptw->nblocks].len = ptw_a->stub_len;
+ ptw->index[ptw->nblocks].pun = 0;
+ ptw->index[ptw->nblocks].orig0 = orig[0];
+ ptw->index[ptw->nblocks].site_len = sizeof(orig);
+ ptw->index[ptw->nblocks].site_off = (s32)site_delta;
+ memcpy(ptw->index[ptw->nblocks].site_insn, orig, sizeof(orig));
+ /* Publish initialized metadata before exposing the probe jump. */
+ smp_store_release(&ptw->nblocks, ptw->nblocks + 1);
kaddr = kmap_local_page(ptw->page);
memcpy(kaddr + block_off, ptw_a->stub, ptw_a->stub_len);
@@ -1862,15 +2196,69 @@ int arch_uprobe_uninstall_ptwrite(struct arch_uprobe *auprobe,
struct vm_area_struct *vma, unsigned long vaddr)
{
struct mm_struct *mm = vma->vm_mm;
+ struct uprobe_ptwrite_arch *ptw_a = &auprobe->ptwrite;
+ struct uprobes_state *state = &mm->uprobes_state;
u8 cur[5];
+ struct uprobe_ptwrite_page *ptw, *found_page = NULL;
+ s32 rel;
+ s64 target, site_delta;
+ unsigned long page_base, boff;
+ int b, ret;
+ struct write_opcode_ctx ctx = {
+ .base = vaddr,
+ .expect = EXPECT_BYTE,
+ .expect_byte = 0xe9,
+ };
mmap_assert_write_locked(mm);
- if (copy_from_vaddr(mm, vaddr, cur, sizeof(cur)) ||
- !ptwrite_is_installed(mm, vaddr, cur))
+
+ ret = copy_from_vaddr(mm, vaddr, cur, sizeof(cur));
+ if (ret)
+ return ret;
+ if (!ptwrite_is_installed(mm, vaddr, cur))
return 0;
- return text_poke_5byte(auprobe, vma, vaddr, auprobe->ptwrite.orig,
- UPROBE_SWBP_INSN, false, false, false, false, NULL);
+ memcpy(&rel, cur + 1, sizeof(rel));
+ target = (s64)vaddr + 5 + (s64)rel;
+ if (target < PAGE_SIZE || target >= TASK_SIZE_MAX)
+ return text_poke_5byte(auprobe, vma, vaddr, ptw_a->orig,
+ 0xe9, false, false, false, false, NULL);
+ page_base = (unsigned long)target & PAGE_MASK;
+ boff = (unsigned long)target & (PAGE_SIZE - 1);
+ hlist_for_each_entry(ptw, &state->head_ptwrite, node)
+ if (ptw->vaddr == page_base) {
+ found_page = ptw;
+ break;
+ }
+ if (!found_page)
+ return 0;
+ /* Acquire the published count before reading block metadata. */
+ for (b = 0; b < smp_load_acquire(&found_page->nblocks); b++)
+ if (found_page->index[b].off == boff)
+ break;
+ if (b >= smp_load_acquire(&found_page->nblocks))
+ return 0;
+ site_delta = (s64)vaddr - (s64)found_page->vaddr;
+ if (site_delta < INT_MIN || site_delta > INT_MAX)
+ return 0;
+ if (found_page->index[b].site_off != (s32)site_delta ||
+ found_page->index[b].site_len !=
+ (found_page->index[b].pun ? ptw_a->len : sizeof(cur)) ||
+ memcmp(found_page->index[b].site_insn, ptw_a->orig,
+ found_page->index[b].site_len))
+ return 0;
+ if (found_page->index[b].pun) {
+ u8 orig0 = found_page->index[b].orig0;
+
+ ret = uprobe_write(auprobe, vma, vaddr, &orig0, 1,
+ verify_insn, false, false, &ctx);
+ if (ret)
+ return ret;
+ smp_text_poke_sync_each_cpu();
+ return 0;
+ }
+ return text_poke_5byte(auprobe, vma, vaddr, ptw_a->orig,
+ 0xe9, false, false, false, false, NULL);
}
static bool __is_optimized(struct mm_struct *mm, uprobe_opcode_t *insn, unsigned long vaddr)
diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h
index 5f16799c9fb9..85779cff3414 100644
--- a/include/linux/uprobes.h
+++ b/include/linux/uprobes.h
@@ -250,7 +250,9 @@ extern struct uprobe *uprobe_register_ptwrite(struct inode *inode,
const struct uprobe_ptwrite_desc *desc);
extern bool arch_uprobe_ptwrite_supported(void);
extern int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
- const struct uprobe_ptwrite_desc *desc);
+ struct inode *inode, struct file *file,
+ loff_t offset,
+ const struct uprobe_ptwrite_desc *desc);
extern int arch_uprobe_install_ptwrite(struct arch_uprobe *auprobe,
struct vm_area_struct *vma,
unsigned long vaddr);
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 894196f3089f..78bc847d73cc 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -1509,6 +1509,8 @@ bool __weak arch_uprobe_ptwrite_supported(void)
}
int __weak arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
+ struct inode *inode, struct file *file,
+ loff_t offset,
const struct uprobe_ptwrite_desc *desc)
{
return -EOPNOTSUPP;
@@ -1586,13 +1588,19 @@ struct uprobe *uprobe_register_ptwrite(struct inode *inode, struct file *file,
ret = -EBUSY;
goto out;
}
-
- /* Build the mm-independent stub template once, at registration. */
- ret = arch_uprobe_ptwrite_prepare(&uprobe->arch, desc);
+ /*
+ * Prepare the immutable PTWRITE stub before exposing the uprobe. The
+ * copy-instruction flag also keeps the normal XOL preparation path out.
+ */
+ ret = copy_insn(uprobe, file);
if (ret)
goto out;
-
-
+ ret = arch_uprobe_ptwrite_prepare(&uprobe->arch, inode, file, offset,
+ desc);
+ if (ret)
+ goto out;
+ smp_wmb();
+ set_bit(UPROBE_COPY_INSN, &uprobe->flags);
set_bit(UPROBE_PTWRITE, &uprobe->flags);
consumer_add(uprobe, uc);
ret = register_for_each_vma(uprobe, uc);
--
2.54.0
^ permalink raw reply [flat|nested] 12+ messages in thread* [RFC PATCH v2 09/11] ptwrite uprobes: Use atomic patching for multinop sites
2026-09-17 23:00 ptwrite uprobes v2 Andi Kleen
` (7 preceding siblings ...)
2026-09-17 23:00 ` [RFC PATCH v2 08/11] ptwrite uprobes: Support instruction punning Andi Kleen
@ 2026-09-17 23:00 ` Andi Kleen
2026-09-17 23:00 ` [RFC PATCH v2 10/11] ptwrite uprobes: Add a tutorial and overview documentation Andi Kleen
2026-09-17 23:00 ` [RFC PATCH v2 11/11] ptwrite uprobes: Add kernel self tests Andi Kleen
10 siblings, 0 replies; 12+ messages in thread
From: Andi Kleen @ 2026-09-17 23:00 UTC (permalink / raw)
To: Masami Hiramatsu
Cc: Oleg Nesterov, Peter Zijlstra, linux-kernel, linux-trace-kernel,
x86, tglx, jolsa, linux-perf-users, adrian.hunter, Andi Kleen
The earlier multinop patching is not quite safe because the cross
modified CPU could be already executing on a later nop when the
cross patching occurs. The Intel SDM allows cross modification
by larger stores as long as they are aligned. AMD has a similar
guarantee.
Support GCC function-entry patch sites is the main motivation for
multinop, and these sites are always aligned.
So enforce 8 bytes alignment of the multinop and use a safe RMW 8 byte store
to overwrite the 5 byte sequence. This assumes that the code is not
changing in parallel, but if that happens cross modification safety
is probably the smallest of the issues.
Assisted-by: omp:gpt-5.6-luna
Signed-off-by: Andi Kleen <ak@kernel.org>
---
arch/x86/kernel/uprobes.c | 183 +++++++++++++++++++++++---------------
kernel/events/uprobes.c | 15 +++-
2 files changed, 123 insertions(+), 75 deletions(-)
diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c
index af9b36a219d3..114905bb69a8 100644
--- a/arch/x86/kernel/uprobes.c
+++ b/arch/x86/kernel/uprobes.c
@@ -17,6 +17,7 @@
#include <linux/kdebug.h>
#include <linux/highmem.h>
#include <linux/mm.h>
+#include <linux/security.h>
#include <asm/processor.h>
#include <asm/insn.h>
#include <asm/insn-eval.h>
@@ -828,6 +829,8 @@ int uprobe_ptwrite_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm)
kunmap_local(src);
new->vaddr = ptw->vaddr;
new->cursor = ptw->cursor;
+ new->nblocks = ptw->nblocks;
+ memcpy(new->index, ptw->index, sizeof(new->index));
vma = install_uprobe_ptwrite_vma(newmm, new->vaddr);
if (IS_ERR(vma)) {
@@ -1448,7 +1451,7 @@ static_assert((((9 + UPROBE_PTWRITE_SERIALIZE_LFENCES *
UPROBE_PTWRITE_MAX_ARGS *
(10 + UPROBE_PTWRITE_SERIALIZE_LFENCES *
UPROBE_PTWRITE_LFENCE_SIZE) +
- 5 + 7) & ~7) +
+ UPROBE_PTWRITE_COPY_SIZE + 5 + 7) & ~7) +
8 * (1 + UPROBE_PTWRITE_MAX_ARGS)) <=
UPROBE_PTWRITE_STUB_SIZE,
"worst-case ptwrite stub block exceeds UPROBE_PTWRITE_STUB_SIZE");
@@ -1460,6 +1463,7 @@ static bool ptwrite_has_room(const u8 *base, const u8 *p, size_t len)
}
static bool pun_site_is_nop(const u8 *orig, bool allow_nop_run);
+static bool ptwrite_site_is_multinop(const u8 *orig, bool allow_nop_run);
static int pun_classify_insn(struct insn *insn, u8 *disp_off, s32 *disp);
static int pun_decode_site(struct inode *inode, struct file *file,
loff_t offset, u8 *copy,
@@ -1494,7 +1498,18 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
return -EINVAL;
/* The generic registration path copied these bytes before this hook. */
+ ptw->allow_nop_run =
+ desc->flags & UPROBE_PTWRITE_FL_ALLOW_NOP_RUN;
memcpy(ptw->orig, auprobe->insn, sizeof(ptw->orig));
+ /*
+ * File mappings preserve page offsets, so an unaligned file offset
+ * cannot become an aligned runtime address. Reject it before the probe
+ * is exposed; install-time failures for a future mapping are otherwise
+ * not observable through the tracefs enable operation.
+ */
+ if (ptwrite_site_is_multinop(ptw->orig, ptw->allow_nop_run) &&
+ !IS_ALIGNED(offset, sizeof(u64)))
+ return -EINVAL;
for (i = 0; i < desc->nargs; i++) {
switch (desc->args[i].src) {
@@ -1618,7 +1633,7 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe,
ptw->stub_len = data_off + 8 * (1 + n_imm);
ptw->ndata = 1 + n_imm;
- ptw->allow_nop_run = desc->flags & UPROBE_PTWRITE_FL_ALLOW_NOP_RUN;
+
ret = pun_decode_site(inode, file, offset, code + ptw->copy_off,
&ptw->disp_off, &ptw->disp,
@@ -1753,8 +1768,9 @@ get_uprobe_ptwrite_page(struct mm_struct *mm, unsigned long vaddr,
}
/*
- * A run of short NOPs is accepted only when requested. This validation does
- * not make the three-phase poke safe for threads that already passed byte 0.
+ * A run of short NOPs is accepted only when requested. It is patched with
+ * an aligned eight-byte read-modify-write, preserving the following bytes;
+ * code is not expected to change concurrently.
*/
static bool ptwrite_is_nop_run(const u8 *orig)
{
@@ -1940,6 +1956,42 @@ static int ptwrite_text_poke(struct arch_uprobe *auprobe,
return err;
}
+/*
+ * Replace an aligned five-byte NOP run with a JMP in one eight-byte store.
+ * The trailing three bytes are read from the existing text. We assume
+ * nobody else is changing it. This is covered by the Intel/AMD "aligned store"
+ * cross modifying guarantee.
+ */
+static int ptwrite_multinop_text_poke(struct arch_uprobe *auprobe,
+ struct vm_area_struct *vma,
+ unsigned long vaddr,
+ unsigned long stub_addr)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ struct write_opcode_ctx ctx = {
+ .base = vaddr,
+ .expect = EXPECT_BYTE,
+ .expect_byte = 0x90,
+ };
+ u8 patch[sizeof(u64)];
+ s32 rel;
+ int err;
+
+ if (!IS_ALIGNED(vaddr, sizeof(u64)))
+ return -EINVAL;
+ if (!ptwrite_rel32(vaddr + 5, stub_addr, &rel))
+ return -ERANGE;
+ err = copy_from_vaddr(mm, vaddr, patch, sizeof(patch));
+ if (err)
+ return err;
+ patch[0] = 0xe9;
+ memcpy(&patch[1], &rel, sizeof(rel));
+ err = uprobe_write(auprobe, vma, vaddr, patch, sizeof(patch),
+ verify_insn, true, false, &ctx);
+ if (!err)
+ smp_text_poke_sync_each_cpu();
+ return err;
+}
static int pun_text_poke(struct arch_uprobe *auprobe,
struct vm_area_struct *vma,
unsigned long vaddr, u8 e9,
@@ -1971,65 +2023,40 @@ static int pun_install(struct arch_uprobe *auprobe,
unsigned long t, page_base, block_off, stub_addr;
s64 site_delta, target;
s32 jump_rel, disp32, orig_rel;
- u8 site_len;
bool found = false;
- bool nop_fallback = ptwrite_site_is_multinop(orig,
- ptw_a->allow_nop_run) &&
- (vaddr & 7);
u8 *kaddr;
int b, ret;
mmap_assert_write_locked(mm);
- if (nop_fallback) {
- hlist_for_each_entry(ptw, &state->head_ptwrite, node) {
- site_delta = (s64)vaddr - (s64)ptw->vaddr;
- if (site_delta < INT_MIN || site_delta > INT_MAX)
- continue;
- for (b = 0; b < smp_load_acquire(&ptw->nblocks); b++)
- if (!ptw->index[b].pun &&
- ptw->index[b].site_off == (s32)site_delta &&
- ptw->index[b].site_len == 5 &&
- !memcmp(ptw->index[b].site_insn, orig, 5))
- break;
- if (b >= smp_load_acquire(&ptw->nblocks))
- continue;
- if (!__in_uprobe_ptwrite(mm, ptw->vaddr))
- continue;
- return ptwrite_text_poke(auprobe, vma, vaddr,
- ptw->vaddr + ptw->index[b].off);
+ memcpy(&orig_rel, orig + 1, sizeof(orig_rel));
+ target = (s64)vaddr + 5 + (s64)orig_rel;
+ if (target < PAGE_SIZE || target >= TASK_SIZE_MAX)
+ return -EADDRNOTAVAIL;
+ t = (unsigned long)target;
+ page_base = t & PAGE_MASK;
+ ret = security_mmap_addr(page_base);
+ if (ret)
+ return ret;
+ block_off = t & (PAGE_SIZE - 1);
+ if (block_off + ptw_a->stub_len > PAGE_SIZE)
+ return -ENOSPC;
+
+ /* Reuse an existing ptwrite page at the target, else map a new one. */
+ hlist_for_each_entry(ptw, &state->head_ptwrite, node) {
+ if (ptw->vaddr == page_base) {
+ found = true;
+ break;
}
- ptw = get_uprobe_ptwrite_page(mm, vaddr, ptw_a->stub_len);
+ }
+ if (!found) {
+ if (vma_lookup(mm, page_base))
+ return -EADDRNOTAVAIL;
+ ptw = create_uprobe_ptwrite_page_at(mm, page_base);
if (!ptw)
return -ENOMEM;
- block_off = ptw->cursor;
- } else {
- memcpy(&orig_rel, orig + 1, sizeof(orig_rel));
- target = (s64)vaddr + 5 + (s64)orig_rel;
- if (target < PAGE_SIZE || target >= TASK_SIZE_MAX)
- return -EADDRNOTAVAIL;
- t = (unsigned long)target;
- page_base = t & PAGE_MASK;
- block_off = t & (PAGE_SIZE - 1);
- if (block_off + ptw_a->stub_len > PAGE_SIZE)
- return -ENOSPC;
-
- /* reuse an existing ptwrite page at the target, else map a new one */
- hlist_for_each_entry(ptw, &state->head_ptwrite, node) {
- if (ptw->vaddr == page_base) {
- found = true;
- break;
- }
- }
- if (!found) {
- if (vma_lookup(mm, page_base))
- return -EADDRNOTAVAIL; /* target page occupied */
- ptw = create_uprobe_ptwrite_page_at(mm, page_base);
- if (!ptw)
- return -ENOMEM;
- /* Order page initialization before publishing it to fault readers. */
- smp_wmb();
- hlist_add_head_rcu(&ptw->node, &state->head_ptwrite);
- }
+ /* Publish initialized page fields before fault readers find it. */
+ smp_wmb();
+ hlist_add_head_rcu(&ptw->node, &state->head_ptwrite);
}
site_delta = (s64)vaddr - (s64)ptw->vaddr;
@@ -2068,19 +2095,13 @@ static int pun_install(struct arch_uprobe *auprobe,
disp32 = (s32)d;
}
- /*
- * A NOP fallback needs a synthetic rel32 at the site, so it uses
- * the full five-byte poke and restore path rather than punning.
- */
- site_len = nop_fallback ? 5 : ptw_a->len;
ptw->index[ptw->nblocks].off = block_off;
ptw->index[ptw->nblocks].len = ptw_a->stub_len;
- ptw->index[ptw->nblocks].pun = !nop_fallback;
+ ptw->index[ptw->nblocks].pun = 1;
ptw->index[ptw->nblocks].orig0 = orig[0];
- ptw->index[ptw->nblocks].site_len = site_len;
+ ptw->index[ptw->nblocks].site_len = ptw_a->len;
ptw->index[ptw->nblocks].site_off = (s32)site_delta;
- memcpy(ptw->index[ptw->nblocks].site_insn, ptw_a->orig, site_len);
- smp_store_release(&ptw->nblocks, ptw->nblocks + 1);
+ memcpy(ptw->index[ptw->nblocks].site_insn, ptw_a->orig, ptw_a->len);
kaddr = kmap_local_page(ptw->page);
memcpy(kaddr + block_off, ptw_a->stub, ptw_a->stub_len);
@@ -2089,11 +2110,10 @@ static int pun_install(struct arch_uprobe *auprobe,
memcpy(kaddr + block_off + ptw_a->copy_off + ptw_a->disp_off,
&disp32, sizeof(disp32));
kunmap_local(kaddr);
+ /* Publish initialized metadata before exposing the probe jump. */
+ smp_store_release(&ptw->nblocks, ptw->nblocks + 1);
- if (nop_fallback)
- ret = ptwrite_text_poke(auprobe, vma, vaddr, stub_addr);
- else
- ret = pun_text_poke(auprobe, vma, vaddr, 0xe9, &ctx);
+ ret = pun_text_poke(auprobe, vma, vaddr, 0xe9, &ctx);
if (ret) {
/* Publish rollback before readers observe the reduced block count. */
smp_store_release(&ptw->nblocks, ptw->nblocks - 1);
@@ -2129,6 +2149,9 @@ int arch_uprobe_install_ptwrite(struct arch_uprobe *auprobe,
ret = copy_from_vaddr(mm, vaddr, orig, sizeof(orig));
if (ret)
return ret;
+ if (ptwrite_site_is_multinop(orig, ptw_a->allow_nop_run) &&
+ !IS_ALIGNED(vaddr, sizeof(u64)))
+ return -EINVAL;
if (ptwrite_is_installed(mm, vaddr, orig))
return 0;
@@ -2151,8 +2174,13 @@ int arch_uprobe_install_ptwrite(struct arch_uprobe *auprobe,
continue;
if (!__in_uprobe_ptwrite(mm, ptw->vaddr))
continue;
- return ptwrite_text_poke(auprobe, vma, vaddr,
- ptw->vaddr + ptw->index[b].off);
+ if (ptwrite_site_is_multinop(orig, ptw_a->allow_nop_run))
+ ret = ptwrite_multinop_text_poke(auprobe, vma, vaddr,
+ ptw->vaddr + ptw->index[b].off);
+ else
+ ret = ptwrite_text_poke(auprobe, vma, vaddr,
+ ptw->vaddr + ptw->index[b].off);
+ return ret;
}
ptw = get_uprobe_ptwrite_page(mm, vaddr, ptw_a->stub_len);
if (!ptw)
@@ -2186,10 +2214,17 @@ int arch_uprobe_install_ptwrite(struct arch_uprobe *auprobe,
memcpy(kaddr + block_off + ptw_a->jmp_off, &rel, sizeof(rel));
kunmap_local(kaddr);
- ret = ptwrite_text_poke(auprobe, vma, vaddr, stub_addr);
- if (!ret)
- ptw->cursor = block_off + ptw_a->stub_len;
- return ret;
+ if (ptwrite_site_is_multinop(orig, ptw_a->allow_nop_run))
+ ret = ptwrite_multinop_text_poke(auprobe, vma, vaddr, stub_addr);
+ else
+ ret = ptwrite_text_poke(auprobe, vma, vaddr, stub_addr);
+ if (ret) {
+ /* Publish rollback before readers use the reduced block count. */
+ smp_store_release(&ptw->nblocks, ptw->nblocks - 1);
+ return ret;
+ }
+ ptw->cursor = block_off + ptw_a->stub_len;
+ return 0;
}
int arch_uprobe_uninstall_ptwrite(struct arch_uprobe *auprobe,
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 78bc847d73cc..18c5df46a509 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -192,7 +192,20 @@ void uprobe_copy_from_page(struct page *page, unsigned long vaddr, void *dst, in
static void copy_to_page(struct page *page, unsigned long vaddr, const void *src, int len)
{
void *kaddr = kmap_local_page(page);
- memcpy(kaddr + (vaddr & ~PAGE_MASK), src, len);
+ void *dst = kaddr + (vaddr & ~PAGE_MASK);
+
+ /*
+ * Atomic eight-byte stores are required for safe cross-modification of
+ * live user text; other writes use the ordinary byte-copy path.
+ */
+ if (len == sizeof(u64) && IS_ALIGNED(vaddr, sizeof(u64))) {
+ u64 value;
+
+ memcpy(&value, src, sizeof(value));
+ WRITE_ONCE(*(u64 *)dst, value);
+ } else {
+ memcpy(dst, src, len);
+ }
kunmap_local(kaddr);
}
--
2.54.0
^ permalink raw reply [flat|nested] 12+ messages in thread* [RFC PATCH v2 10/11] ptwrite uprobes: Add a tutorial and overview documentation
2026-09-17 23:00 ptwrite uprobes v2 Andi Kleen
` (8 preceding siblings ...)
2026-09-17 23:00 ` [RFC PATCH v2 09/11] ptwrite uprobes: Use atomic patching for multinop sites Andi Kleen
@ 2026-09-17 23:00 ` Andi Kleen
2026-09-17 23:00 ` [RFC PATCH v2 11/11] ptwrite uprobes: Add kernel self tests Andi Kleen
10 siblings, 0 replies; 12+ messages in thread
From: Andi Kleen @ 2026-09-17 23:00 UTC (permalink / raw)
To: Masami Hiramatsu
Cc: Oleg Nesterov, Peter Zijlstra, linux-kernel, linux-trace-kernel,
x86, tglx, jolsa, linux-perf-users, adrian.hunter, Andi Kleen
Document the PTWRITE uprobe interface, syntax, requirements,
limitations, and usage examples.
Assisted-by: omp:gpt-5.6-luna
Signed-off-by: Andi Kleen <ak@kernel.org>
---
Documentation/trace/index.rst | 1 +
Documentation/trace/ptwrite-uprobes.rst | 387 ++++++++++++++++++++++++
2 files changed, 388 insertions(+)
create mode 100644 Documentation/trace/ptwrite-uprobes.rst
diff --git a/Documentation/trace/index.rst b/Documentation/trace/index.rst
index f4058e8e92e3..4ae7b158804b 100644
--- a/Documentation/trace/index.rst
+++ b/Documentation/trace/index.rst
@@ -90,6 +90,7 @@ interactions.
.. toctree::
:maxdepth: 1
+ ptwrite-uprobes
user_events
uprobetracer
diff --git a/Documentation/trace/ptwrite-uprobes.rst b/Documentation/trace/ptwrite-uprobes.rst
new file mode 100644
index 000000000000..5268b8aed9de
--- /dev/null
+++ b/Documentation/trace/ptwrite-uprobes.rst
@@ -0,0 +1,387 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+===============
+ptwrite uprobes
+===============
+
+.. contents:: :local:
+
+Introduction
+============
+
+A classic uprobe enters the kernel for each probe. That causes overhead
+when the probe is executed frequently.
+
+ptwrite uprobes instead rely on hardware tracing that doesn't enter
+the kernel. It uses the ``PTWRITE`` instruction available on modern
+Intel CPUs to log data to the Processor Trace buffer. Processor
+Trace is configured and recorded by Linux perf.
+
+There are some limitations of the scheme (see below)
+but it is a lot faster than classic uprobes.
+
+Performance
+===========
+
+Measured on the development kernel in a KVM guest (2 vCPUs) running
+on an Alder Lake laptop with a probe on a hot function called in a
+tight loop.
+
+ +---------------------------------------+----------------------+--------------------+------------------------+
+ | mode (2-arg probe) | PT off (% of classic)| full (% of classic)| snapshot (% of classic)|
+ +=======================================+======================+====================+========================+
+ | classic uprobe (tracefs) | 100% | 100% | 100% |
+ | classic uprobe (perf probe, tracefs) | 104% | | |
+ | classic uprobe (perf probe, perf ring)| | 156% | |
+ | ptwrite %nopace | 2% | 2% | 8% |
+ | ptwrite default | 7% | 7% | 11% |
+ | perf probe ``--ptwrite`` | 7% | 7% | 12% |
+ +---------------------------------------+----------------------+--------------------+------------------------+
+
+The percentages are normalized to the classic tracefs uprobe in each
+recording mode; lower values represent lower cost per hit.
+
+A ``%nopace`` probe costs roughly **2%** as much as a classic uprobe
+with PT off (about 98% less). The default pacing (on unless ``%nopace``
+is given) costs roughly 7% as much (about 93% less) in the same mode.
+The default pacing slows down the probes to avoid data loss when they are
+too tightly spaced.
+
+``snapshot`` refers to ``perf record`` snapshot mode (``-S``) which doesn't
+save the PT ring buffer constantly.
+
+
+Requirements
+============
+
+- An Intel CPU with Intel PT and PTWRITE. When running as a guest Intel PT
+ needs to be exposed to the guest.
+ PT/PTWRITE are available when ``/sys/devices/intel_pt/format/ptw`` exists.
+- A kernel with ``CONFIG_UPROBE_EVENTS`` enabled.
+
+Quick start (tracefs)
+=====================
+
+Pick a probe site, register a probe at its file offset, enable it, run the
+program under PT, decode.
+
+Example 1: probe an existing instruction (punning)
+--------------------------------------------------
+
+Build a small program and probe the entry of ``main``::
+
+ $ cat > t.c <<'EOF'
+ #include <stdio.h>
+
+ __attribute__((noinline, noipa)) static unsigned long
+ target(unsigned long a, unsigned long b)
+ {
+ return a * 31 + b;
+ }
+
+ int main(void)
+ {
+ unsigned long i, acc = 0;
+ for (i = 0; i < 100; i++)
+ acc += target(i, i + 1);
+ printf("acc=%lu\n", acc);
+ return 0;
+ }
+ EOF
+ $ gcc -O2 -no-pie -fno-inline -o t t.c
+
+``objdump -F`` prints the file offset of every instruction::
+
+ $ objdump -d -F t | sed -n "/<main> (File Offset/,+1p"
+ 0000000000401040 <main> (File Offset: 0x1040):
+ 401040: 55 push %rbp
+
+``1040`` is the file offset of ``main``'s first instruction, exactly
+what the probe line needs. Register the probe there, enable it, run
+the program under PT and decode::
+
+ # echo "ptw:e t:0x1040 %di %si" > /sys/kernel/tracing/uprobe_events
+ # echo 1 > /sys/kernel/tracing/events/uprobes/e/enable
+ # perf record -e intel_pt/ptw=1,fup_on_ptw=1/u -o perf.data ./t
+ # perf script --itrace=qwe -s uprobe-ptwrite-decode.py -i perf.data
+ record 1: event=uprobes/e id=0x6c2 args=[1, 140728356930600]
+ summary: records=1 dropped=0 stray=0 unknown=0 errors=0
+
+Since it isn't a nop the instruction is "punned": byte 0 is changed
+to a jump to a trampoline that logs the data and returns to the
+previous execution.
+
+Punning is a probabilistic method that depends on the existing
+instruction bytes and the placement of the executable in memory.
+It has a high chance of success on PIE/PIC binaries, but tends
+to work poorly on non PIE main executables.
+When punning is not possible the probe is rejected at install
+time. Options in this case:
+
+- Move the probe site to a different instruction which may work.
+- Rebuild with -fPIE if the main executable is not PIE.
+- Enable or disable /proc/sys/kernel/randomize_va_space. If the
+ randomization is enabled it may also just work on a rerun of
+ the program.
+- Fall back to a classic uprobe.
+- Insert a 5-byte NOP which is always supported (see below).
+
+Example 2: an explicit 5-byte NOP (inline assembly)
+---------------------------------------------------
+
+Add a 5-byte NOP at the probe point::
+
+ $ cat > t.c <<'EOF'
+ #include <stdio.h>
+
+ __attribute__((noinline)) static unsigned long
+ target(unsigned long a, unsigned long b)
+ {
+ asm volatile(".byte 0x0f, 0x1f, 0x44, 0x00, 0x00"); /* nopl */
+ return a * 31 + b;
+ }
+
+ int main(void)
+ {
+ unsigned long i, acc = 0;
+ for (i = 0; i < 100; i++)
+ acc += target(i, i + 1);
+ printf("acc=%lu\n", acc);
+ return 0;
+ }
+ EOF
+ $ gcc -O2 -no-pie -o t t.c
+
+ $ objdump -d -F t | sed -n "/<target> (File Offset/,+1p"
+ 0000000000401170 <target> (File Offset: 0x1170):
+ 401170: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1)
+
+Probe it exactly like example 1::
+
+ # echo "ptw:e t:0x1170 %di %si" > /sys/kernel/tracing/uprobe_events
+ # echo 1 > /sys/kernel/tracing/events/uprobes/e/enable
+ # perf record -e intel_pt/ptw=1,fup_on_ptw=1/u -o perf.data ./t
+ # perf script --itrace=qwe -s uprobe-ptwrite-decode.py -i perf.data
+ record 99: event=uprobes/e id=0x6c2 args=[98, 99]
+ record 100: event=uprobes/e id=0x6c2 args=[99, 100]
+ summary: records=100 dropped=0 stray=0 unknown=0 errors=0
+
+Configuring ptwrite uprobes
+===========================
+
+ptwrite uprobes is configured like normal uprobes by writing
+commands to ``/sys/kernel/tracing/uprobe_events``.
+
+ ptw[:[GRP/][EVENT]] PATH:OFFSET [FETCHARGS] : set a ptwrite probe
+ -:[GRP/][EVENT] : clear a probe
+
+ GRP : group name. If omitted, "uprobes" is the default (the
+ event appears under events/uprobes/).
+ EVENT : event name. If omitted, one is generated from PATH+OFFSET.
+ PATH : path to an executable or a library.
+ OFFSET : file offset of the probe site (0x-prefixed hex, see above).
+ FETCHARGS: probe arguments, up to 8 (see "Argument syntax" below).
+
+After creating the ptwrite uprobe it becomes available with its name
+in ``/sys/kernel/tracing/uprobe_events``. There it can be enabled
+by writing 1 to its enable field. However it only logs data
+when a Linux perf PT recording session with ptw=1 is active.
+
+perf probe
+----------
+
+``perf probe --ptwrite -x <file>`` creates ptwrite uprobes instead of
+the classic trap-based ones. The example below uses SDT probes.
+
+(this requires installing systemtap-devel or an equivalent package)
+
+ $ cat > t.c <<'EOF'
+ #include <stdio.h>
+ #include <sys/sdt.h>
+ __attribute__((noinline, noclone)) static unsigned long
+ target(unsigned long a, unsigned long b)
+ {
+ unsigned long local = a * 2;
+ STAP_PROBE1(test, rarg, a);
+ STAP_PROBE1(test, carg, 42);
+ STAP_PROBE2(test, marg, &local, b);
+ return a * 31 + b;
+ }
+ int main(void)
+ {
+ unsigned long i, acc = 0;
+ for (i = 0; i < 20; i++) {
+ acc += target(i, i + 1);
+ asm volatile("pause");
+ }
+ printf("acc=%lu\n", acc);
+ return 0;
+ }
+ EOF
+ $ gcc -O2 -no-pie -o t t.c
+
+The first probe point (``rarg``) is a nop 9 bytes into ``target``::
+
+ $ objdump -d t | sed -n "/<target>:/,+3p"
+ 0000000000401180 <target>:
+ 401180: 48 8d 04 3f lea (%rdi,%rdi,1),%rax
+ 401184: 48 89 44 24 f8 mov %rax,-0x8(%rsp)
+ 401189: 90 nop
+
+Probe it with ``perf probe --ptwrite`` using the function+offset
+form, then enable, capture and delete it like any ptwrite probe::
+
+ $ perf probe --ptwrite -x ./t --add "target+9 %di %si"
+ Added new event:
+ probe_t:target (on target+9 in ./t with %di %si)
+
+ # the tracefs line it wrote:
+ # ptw:probe_t/target ./t:0x1189 arg1=%di arg2=%si
+
+ # echo 1 > /sys/kernel/tracing/events/probe_t/target/enable
+ # perf record -e intel_pt/ptw=1,fup_on_ptw=1/u -o perf.data ./t
+ # perf script --itrace=qwe -s uprobe-ptwrite-decode.py -i perf.data
+ record 1: event=probe_t/target id=0x6a9 args=[0, 1]
+ record 20: event=probe_t/target id=0x6a9 args=[19, 20]
+ summary: records=20 dropped=0 stray=0 unknown=0 errors=0
+ # perf probe -d probe_t:target
+
+A ``nop`` instruction, as used by SDT probes, is not guaranteed to
+be ptwrite patchable. It needs a 5-byte NOP, but it can
+often be punned. If punning fails, the kernel reports
+``failed to install`` and the probe has to be moved to another site.
+
+GCC's ``-fpatchable-function-entry=5`` may emit five one-byte NOPs.
+To use that site, add ``%multinop`` to the tracefs probe offset::
+
+ # echo "ptw:e t:0x1170%multinop %di %si" > /sys/kernel/tracing/uprobe_events
+
+The five-byte run must start at an 8-byte-aligned address because it is
+patched with an atomic eight-byte store. An unaligned ``%multinop`` site is
+rejected; without ``%multinop``, the run is treated as a pun and may not
+always succeed.
+
+perf probe uses the standard argument syntax for the ptwrite subset
+(registers, ``$stack``/``$stackN``, ``+disp(%reg)`` memory reads, and
+``\0x2a``-style constants). Strings, arrays and typed suffixes are not
+supported by the ptwrite stub and are rejected by the kernel.
+``%return`` is refused (ptwrite probes are entry-only), and the mode
+requires ``-x``. The probes are enabled, captured and deleted like
+classic probes (``perf probe -l``, ``perf probe -d``).
+They carry the default (LFENCE) pacing. ``%nopace`` cannot be selected
+through perf probe. Write the tracefs line by hand for that.
+
+Argument syntax
+---------------
+
+ptwrite uprobes only support a limited number of argument types
+compared to classic uprobes.
+
+``ptw:<name> <path>:<offset> <arg> ... [options]`` where each ``<arg>`` is one
+of:
+
+- ``%di``, ``%si``, ``%ax`` ...: a live register.
+- ``\IMM``: a fixed constant (stored in the stub), e.g. ``\0x42``.
+- ``$stack``: the stack pointer value (never faults).
+- ``$stackN``: the Nth stack slot (``[%rsp + 8N]``). ``u64`` uses an
+ 8-byte load on the fault-fixup path; ``u32``/``s32``/``x32`` use a 4-byte
+ load.
+- ``+<disp>(%reg)``: read memory at ``[reg + disp]``. ``u64`` uses an
+ 8-byte load; ``u32``/``s32``/``x32`` use a 4-byte load (``ptwritel``).
+ A bad address writes ``0``.
+
+Options
+-------
+
+``%nopace`` disables artificial slowdown of the probes. This can cause
+data loss when they are tightly spaced or have many arguments, but
+speeds up the probes (see the benchmark section above).
+
+``%multinop`` lets users probe a 5-byte nop sequence that is not one
+instruction. A program could jump to a later nop, which would break when
+the probe rewrites the site.
+
+However there is a common case where gcc's -fpatchable-function-entry=5
+generates 5 nops for each function that are convenient points
+for patching, and nobody jumps into the middle of them.
+
+The five one-byte NOP sequence must be 8-byte aligned.
+
+The encoding format
+===================
+
+Each probe writes a header and the arguments to the PT stream.
+
+The header is a 64-bit word. Each argument is one PTWRITE payload exposed by
+perf as a ``u64`` value.
+
+ header word: bits 63..48 event id (matches the tracefs id in sysfs)
+ bits 47..40 number of argument words
+ bits 39..0 fixed magic 0x5054525731 ("PTRW1")
+ arguments: one PTWRITE payload per FETCHARG
+
+If the program itself also executes own ``PTWRITE``, those values mix with the
+uprobe output in the stream. The decoder uses the header magic to identify
+uprobe records. Other values are printed as ``manual ptwrite:`` lines (with
+their IP when ``fup_on_ptw`` is set) and counted in the summary's ``stray``
+field.
+
+To also print the decoded branch stream alongside the records, add
+``b`` to the itrace options and drop the ``q``
+
+ # ``perf script --itrace=web -s uprobe-ptwrite-decode.py -i perf.data``
+
+Each decoded branch prints as a ``branch:`` line (from => to, with
+symbols where resolvable), interleaved with the probe records and any
+manual ptwrites in delivery order.
+
+Other events in the recording, including classic uprobes, tracepoints,
+and sample events, are printed as ``event:`` lines unless disabled
+by the decoder.
+
+Unsupported instructions for probes
+===================================
+
+The following instructions are always refused for instrumentation:
+
+- Traps: ``int3``, ``int1``, ``int imm8``, ``into``, ``iret``
+ because they save the IP.
+- System instructions: ``syscall``, ``sysenter``, ``sysexit``
+ for similar reasons.
+- Far control flow: ``jmp far``, ``call far``, and the indirect far
+ forms (call-far, jmp-far).
+- Relative branches: ``jmp rel8/rel32``, ``jcc rel8/rel32``,
+ ``loop*``, ``jecxz/jrcxz``, and ``call rel32``. Out-of-line execution
+ would change their relative targets, and a call's return address would
+ point into the stub.
+- Indirect ``call``: the return-address problem also applies to the
+ register and memory forms.
+- If the 5 byte area of the instruction crosses a page boundary it
+ currently cannot be probed (this applies to nop probes too).
+
+Other restrictions
+==================
+
+- Each probe needs 4K of process memory and roughly 1K extra in the kernel.
+- The probe pages are currently only freed on process exit.
+- Return probes (``%return``/``r:``): ptwrite probes are entry-only
+ and ``%return`` is refused.
+- The SDT reference counter (``(REF)``).
+- EBPF, perf actions, filters, event predicates, histograms, triggers,
+ profiling and similar advanced trace features are all not supported
+ since they would require a kernel entry. However some basic filtering
+ is possible at the perf recording level, for example limit the scope
+ to a CPU or to a process. PT also supports address filter ranges
+ that allow filtering by IP.
+- More than one probe at the same site
+- Only 4 and 8 byte memory references are supported.
+- Fetch argument variety: classic probes fetch strings
+ (``:string``/``:ustring``), arrays, bitfields, nested derefs,
+ ``$retval``, ``$comm`` and ``$argN``. Ptwrite probes only take live
+ registers, ``\IMM`` constants, ``$stack``/``$stackN`` and
+ ``+disp(%reg)`` memory reads. Memory reads are 8-byte words for ``u64``
+ and 4-byte words for ``u32``/``s32``/``x32``.
+ (some of this could be relaxed, but it would require a writable stack)
+- Like normal uprobes one byte of the instruction stream is overwritten
+ (or 5 bytes for the nop case). If the program reads its own code
+ it might see different values.
--
2.54.0
^ permalink raw reply [flat|nested] 12+ messages in thread* [RFC PATCH v2 11/11] ptwrite uprobes: Add kernel self tests
2026-09-17 23:00 ptwrite uprobes v2 Andi Kleen
` (9 preceding siblings ...)
2026-09-17 23:00 ` [RFC PATCH v2 10/11] ptwrite uprobes: Add a tutorial and overview documentation Andi Kleen
@ 2026-09-17 23:00 ` Andi Kleen
10 siblings, 0 replies; 12+ messages in thread
From: Andi Kleen @ 2026-09-17 23:00 UTC (permalink / raw)
To: Masami Hiramatsu
Cc: Oleg Nesterov, Peter Zijlstra, linux-kernel, linux-trace-kernel,
x86, tglx, jolsa, linux-perf-users, adrian.hunter, Andi Kleen
Add kernel selftests for PTWRITE uprobes. Exercise instruction punning,
multinop patching, fork/exec, and removal of probes.
Assisted-by: omp:gpt-5.6-luna
Signed-off-by: Andi Kleen <ak@kernel.org>
---
tools/testing/selftests/Makefile | 1 +
.../test.d/kprobe/uprobe_syntax_errors.tc | 40 ++++
tools/testing/selftests/uprobes/Makefile | 14 ++
tools/testing/selftests/uprobes/ptw_probe.c | 156 +++++++++++++
tools/testing/selftests/uprobes/run_ptw.sh | 219 ++++++++++++++++++
5 files changed, 430 insertions(+)
create mode 100644 tools/testing/selftests/uprobes/Makefile
create mode 100644 tools/testing/selftests/uprobes/ptw_probe.c
create mode 100755 tools/testing/selftests/uprobes/run_ptw.sh
diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile
index 2d960626750e..02f36cbf98ae 100644
--- a/tools/testing/selftests/Makefile
+++ b/tools/testing/selftests/Makefile
@@ -135,6 +135,7 @@ TARGETS += tpm2
TARGETS += tty
TARGETS += ublk
TARGETS += uevent
+TARGETS += uprobes
TARGETS += user_events
TARGETS += vDSO
TARGETS += mm
diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/uprobe_syntax_errors.tc b/tools/testing/selftests/ftrace/test.d/kprobe/uprobe_syntax_errors.tc
index e12dc967ec76..53f30c0b7d1b 100644
--- a/tools/testing/selftests/ftrace/test.d/kprobe/uprobe_syntax_errors.tc
+++ b/tools/testing/selftests/ftrace/test.d/kprobe/uprobe_syntax_errors.tc
@@ -33,4 +33,44 @@ if grep -q "\$current.*" README; then
check_error 'p /bin/sh:10 ^$current:u8' # BAD_VAR
fi
+# ptwrite options may be written as an offset suffix or as separate tokens.
+# Use /bin/sh's executable entry so registration reaches the parser options.
+ptw_off=
+if command -v readelf >/dev/null 2>&1; then
+ ptw_entry=$(readelf -hW /bin/sh |
+ awk '/Entry point address:/{print $NF; exit}')
+ ptw_load_off=$(readelf -lW /bin/sh |
+ awk '$1 == "LOAD" && $0 ~ / R E/ {print $2; exit}')
+ ptw_load_vaddr=$(readelf -lW /bin/sh |
+ awk '$1 == "LOAD" && $0 ~ / R E/ {print $3; exit}')
+ if [ -n "$ptw_entry" ] && [ -n "$ptw_load_off" ] &&
+ [ -n "$ptw_load_vaddr" ]; then
+ ptw_off=$(( $(printf "%d" "$ptw_load_off") +
+ $(printf "%d" "$ptw_entry") -
+ $(printf "%d" "$ptw_load_vaddr") ))
+ fi
+fi
+if [ "$(uname -m)" = x86_64 ] &&
+ [ -e /sys/devices/intel_pt/format/ptw ] && [ -n "$ptw_off" ]; then
+check_good_ptw() {
+ echo > uprobe_events # Clear any probe left by an earlier case.
+ echo "$1" > uprobe_events
+ grep -q 'ptw:uprobes/ptw_parser' uprobe_events
+ local ret=$?
+ echo "-:ptw_parser" > uprobe_events
+ return "$ret"
+}
+
+check_good_ptw "ptw:ptw_parser /bin/sh:$ptw_off%multinop %di" || exit 1
+check_good_ptw "ptw:ptw_parser /bin/sh:$ptw_off%nopace %multinop %di" || exit 1
+check_good_ptw "ptw:ptw_parser /bin/sh:$ptw_off %multinop %nopace %di" || exit 1
+
+check_error "ptw:ptw_parser /bin/sh:$ptw_off^%return %di" # BAD_ADDR_SUFFIX
+check_error "ptw:ptw_parser /bin/sh:$ptw_off^%unknown %di" # BAD_ADDR_SUFFIX
+if grep -q '\$comm' README; then
+ check_error "ptw:ptw_parser /bin/sh:$ptw_off %multinop %di ^\$comm" # BAD_FETCH_ARG
+fi
+echo > uprobe_events
+fi
+
exit 0
diff --git a/tools/testing/selftests/uprobes/Makefile b/tools/testing/selftests/uprobes/Makefile
new file mode 100644
index 000000000000..ee97442f8c9d
--- /dev/null
+++ b/tools/testing/selftests/uprobes/Makefile
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0
+# ptwrite uprobe selftests (x86-64).
+ARCH ?= $(shell uname -m 2>/dev/null || echo not)
+CFLAGS += -O2 -Wall -no-pie
+
+TEST_PROGS := run_ptw.sh
+
+ifneq ($(filter x86 x86_64,$(ARCH)),)
+TEST_GEN_FILES := ptw_probe
+else
+TEST_GEN_FILES :=
+endif
+
+include ../lib.mk
diff --git a/tools/testing/selftests/uprobes/ptw_probe.c b/tools/testing/selftests/uprobes/ptw_probe.c
new file mode 100644
index 000000000000..b232fbd4d2aa
--- /dev/null
+++ b/tools/testing/selftests/uprobes/ptw_probe.c
@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ptw_probe - ptwrite uprobe selftest target.
+ */
+#include <stdio.h>
+#include <stdint.h>
+#include <string.h>
+
+static __attribute__((noipa)) uint64_t
+punfn(uint64_t a)
+{
+ uint32_t v;
+
+ asm volatile("mov $0xfff10000, %%eax\n\tmovl %%eax, %0"
+ : "=r"(v) : : "rax");
+ return v ^ (a * 0x9e3779b97f4a7c15ULL);
+}
+
+static __attribute__((noipa)) uint64_t
+jcc8(uint64_t a)
+{
+ asm volatile("jne 1f\n\tmovabs $0x1111111111111111, %%rax\n\t"
+ "1:" : "+a"(a) : : "cc");
+ return a;
+}
+
+static __attribute__((noipa)) uint64_t
+nopfn(uint64_t a)
+{
+ asm volatile("nop\n\t"
+ ".p2align 3, 0x90\n\t"
+ ".globl nopfn_site\n\t"
+ "nopfn_site:\n\t"
+ "nop\n\tnop\n\tnop\n\tnop\n\tnop" ::: "memory");
+ return a * 31 + 7;
+}
+
+/* These symbols are labels defined by the inline assembly above. */
+extern const uint8_t nopfn_site[];
+extern const uint8_t nopfn_unaligned_site[];
+
+static __attribute__((noipa)) uint64_t
+unaligned_nopfn(uint64_t a)
+{
+ asm volatile(".p2align 3, 0x90\n\t"
+ "nop\n\t"
+ ".globl nopfn_unaligned_site\n\t"
+ "nopfn_unaligned_site:\n\t"
+ "nop\n\tnop\n\tnop\n\tnop\n\tnop" ::: "memory");
+ return a * 13 + 5;
+}
+
+static __attribute__((noipa)) uint64_t
+nop5(uint64_t a)
+{
+ asm volatile(".byte 0x0f, 0x1f, 0x44, 0x00, 0x00" ::: "memory");
+ return a * 7 + 3;
+}
+
+static __attribute__((noipa)) uint64_t
+rzfn(uint64_t a)
+{
+ uint64_t v;
+
+ asm volatile("movq %0, -8(%%rsp)" : : "r"(a) : "memory");
+ asm volatile(".globl rz_probe_site\n\t"
+ "rz_probe_site:\n\t"
+ "nop\n\tnop\n\tnop\n\tnop\n\tnop" ::: "memory");
+ asm volatile("movq -8(%%rsp), %0" : "=r"(v) : : "memory");
+ return v ^ 0x55;
+}
+
+static uint8_t load_site_byte(const uint8_t *p)
+{
+ return __atomic_load_n(p, __ATOMIC_RELAXED);
+}
+
+static void dump_site(const char *name, const uint8_t *p)
+{
+ printf("SITE %s %02x%02x%02x%02x%02x\n", name,
+ load_site_byte(p + 0), load_site_byte(p + 1),
+ load_site_byte(p + 2), load_site_byte(p + 3),
+ load_site_byte(p + 4));
+}
+
+static int check_installed(const char *name, const uint8_t *p, uint64_t vaddr)
+{
+ uint32_t rel_u;
+ int32_t rel;
+ unsigned long long target, s, e;
+ FILE *f;
+ char line[256];
+ int found = 0;
+
+ if (load_site_byte(p + 0) != 0xe9)
+ return 1; /* not installed */
+ rel_u = (uint32_t)load_site_byte(p + 1) |
+ ((uint32_t)load_site_byte(p + 2) << 8) |
+ ((uint32_t)load_site_byte(p + 3) << 16) |
+ ((uint32_t)load_site_byte(p + 4) << 24);
+ rel = (int32_t)rel_u;
+ target = vaddr + 5 + (int64_t)rel;
+ f = fopen("/proc/self/maps", "r");
+ if (!f)
+ return -1;
+ while (fgets(line, sizeof(line), f)) {
+ if (!strstr(line, "[uprobes-ptwrite]"))
+ continue;
+ if (sscanf(line, "%llx-%llx", &s, &e) == 2 &&
+ target >= s && target < e) {
+ found = 1;
+ break;
+ }
+ }
+ fclose(f);
+ printf("INSTALL %s %s (target %llx)\n", name,
+ found ? "ok" : "BAD-TARGET", (unsigned long long)target);
+ return found ? 0 : 2;
+}
+
+int main(int argc, char **argv)
+{
+ uint64_t acc = 0x1122334455667788ULL;
+ int i, r, bad = 0;
+
+
+ for (i = 0; i < 100; i++)
+ acc = nopfn(acc + i);
+ for (i = 0; i < 100; i++)
+ acc = unaligned_nopfn(acc + i);
+ for (i = 0; i < 100; i++)
+ acc = nop5(acc + i);
+ for (i = 0; i < 100; i++)
+ acc = rzfn(acc + i);
+ for (i = 0; i < 100; i++)
+ acc = punfn(acc + i);
+ for (i = 0; i < 100; i++)
+ acc = jcc8(acc + i);
+
+ dump_site("punfn", (const uint8_t *)&punfn);
+ dump_site("jcc8", (const uint8_t *)&jcc8);
+ dump_site("nopfn", nopfn_site);
+ dump_site("unaligned_nopfn", nopfn_unaligned_site);
+ dump_site("nop5", (const uint8_t *)&nop5);
+
+ r = check_installed("punfn", (const uint8_t *)&punfn, (uint64_t)&punfn);
+ bad |= r == 2;
+ r = check_installed("nopfn", nopfn_site, (uint64_t)nopfn_site);
+ bad |= r == 2;
+ r = check_installed("nop5", (const uint8_t *)&nop5, (uint64_t)&nop5);
+ bad |= r == 2;
+
+ printf("PTW-PROBE acc=%llx %s\n", (unsigned long long)acc,
+ bad ? "INSTALL-BAD" : "ok");
+ return bad ? 1 : 0;
+}
diff --git a/tools/testing/selftests/uprobes/run_ptw.sh b/tools/testing/selftests/uprobes/run_ptw.sh
new file mode 100755
index 000000000000..a16b7adeabb7
--- /dev/null
+++ b/tools/testing/selftests/uprobes/run_ptw.sh
@@ -0,0 +1,219 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+# run_ptw.sh - ptwrite uprobes selftests.
+# Root + tracefs + an x86-64 CPU with PTWRITE required
+DIR=$(dirname "$(readlink -f "$0")")
+BIN="$DIR/ptw_probe"
+TR=/sys/kernel/tracing
+EV="$TR/uprobe_events"
+PTW=/sys/devices/intel_pt/format/ptw
+
+cleanup() {
+ if [ -e "$TR/events/uprobes/pw/enable" ]; then
+ echo 0 > "$TR/events/uprobes/pw/enable" 2>/dev/null
+ fi
+ echo "-:pw" >> "$EV" 2>/dev/null
+ echo "-:bad" >> "$EV" 2>/dev/null
+}
+
+if [ ! -e "$PTW" ]; then
+ echo "1..0 # SKIP PTWRITE unavailable"
+ exit 0
+fi
+if [ ! -e "$EV" ] || [ "$(id -u)" != 0 ] || [ ! -x "$BIN" ] ||
+ ! command -v objdump >/dev/null 2>&1 ||
+ ! command -v readelf >/dev/null 2>&1; then
+ echo "1..0 # SKIP missing tracefs, root, ptw_probe, objdump, or readelf"
+ exit 0
+fi
+trap cleanup EXIT
+
+# the probe sites: the entry instructions
+elf_off() {
+ local v=$1
+ local base=$(readelf -l "$BIN" 2>/dev/null |
+ awk '/LOAD/{if ($1=="LOAD") {print $3; exit}}')
+ [ -n "$base" ] && printf "0x%x" $((v - base))
+}
+
+PUN_V=$(objdump -d "$BIN" | awk '/^[0-9a-f]+ <punfn>:/{print $1;exit}' | tr -d ':')
+JCC_V=$(objdump -d "$BIN" |
+ awk '/^[0-9a-f]+ <jcc8>:/ {f=1; next} f&&/jne/{print $1; exit}' |
+ tr -d ':')
+NOP_V=$(objdump -d "$BIN" | awk '/^[0-9a-f]+ <nopfn_site>:/{print $1;exit}' | tr -d ':')
+NOP5_V=$(objdump -d "$BIN" | awk '/^[0-9a-f]+ <nop5>:/{print $1;exit}' | tr -d ':')
+UNOP_V=$(objdump -d "$BIN" |
+ awk '/^[0-9a-f]+ <nopfn_unaligned_site>:/{print $1;exit}' | tr -d ':')
+RZ_V=$(objdump -d "$BIN" | awk '/^[0-9a-f]+ <rz_probe_site>:/{print $1;exit}' | tr -d ':')
+if [ -z "$PUN_V" ] || [ -z "$JCC_V" ] || [ -z "$FLT_V" ] ||
+ [ -z "$NOP_V" ] || [ -z "$NOP5_V" ] || [ -z "$UNOP_V" ] ||
+ [ -z "$RZ_V" ]; then
+ echo "1..0 # SKIP unable to resolve ptw_probe symbols"
+ exit 0
+fi
+PUN_OFF=$(elf_off 0x$PUN_V)
+JCC_OFF=$(elf_off 0x$JCC_V)
+NOP_OFF=$(elf_off 0x$NOP_V)
+NOP5_OFF=$(elf_off 0x$NOP5_V)
+UNOP_OFF=$(elf_off 0x$UNOP_V)
+RZ_OFF=$(elf_off 0x$RZ_V)
+
+echo "1..10"
+failures=0
+
+# baseline (unprobed)
+base_out=$("$BIN"); base_rc=$?
+base=$(printf '%s' "$base_out" | sed -n 's/.*acc=\([0-9a-f]*\).*/\1/p')
+base_sites=$(printf '%s' "$base_out" | grep '^SITE ')
+[ -z "$base" ] && base=0
+
+# run one probed invocation: $run_rc = exit code, $probe = the acc
+run_one() {
+ out=$("$BIN")
+ run_rc=$?
+ probe=$(printf '%s' "$out" | sed -n 's/.*acc=\([0-9a-f]*\).*/\1/p')
+}
+
+# the site bytes of a fresh invocation must equal the baseline
+sites_match() {
+ [ "$(printf '%s' "$base_sites")" = \
+ "$("$BIN" | grep '^SITE ')" ]
+}
+
+# emit the TAP line and count failures (tap <num> <ok|not|skip> <desc>)
+tap() {
+ if [ "$2" = ok ]; then
+ echo "ok $1 - $3"
+ elif [ "$2" = skip ]; then
+ echo "ok $1 - $3 # SKIP"
+ else
+ echo "not ok $1 - $3"
+ failures=$((failures + 1))
+ fi
+}
+
+# Install a probe at the site, run the probed binary once, and check the
+# run against the baseline (acc, exit, restored site bytes). A
+# create/enable failure is fatal.
+# Usage: probe_run <num> <offset> <args> <desc>
+probe_run() {
+ local num=$1 off=$2 args=$3 desc=$4
+
+ if ! echo "ptw:pw $BIN:$off $args" >> "$EV" 2>/dev/null; then
+ tap "$num" not "$desc (probe create failed)"
+ exit 1
+ fi
+ if ! echo 1 > "$TR/events/uprobes/pw/enable" 2>/dev/null; then
+ tap "$num" not "$desc (probe enable failed)"
+ exit 1
+ fi
+ run_one
+ echo 0 > "$TR/events/uprobes/pw/enable" 2>/dev/null
+ echo "-:pw" >> "$EV" 2>/dev/null
+ if [ "$probe" = "$base" ] && [ "$run_rc" -eq 0 ] && sites_match; then
+ tap "$num" ok "$desc"
+ else
+ tap "$num" not "$desc (base $base probed $probe rc $run_rc)"
+ fi
+}
+
+# 1: pun out-of-line execution preserves the site instruction's effect
+probe_run 1 $PUN_OFF "%di %si" \
+ "pun out-of-line execution preserves the instruction effect"
+
+# 2: a relative branch site must be refused at enable
+if ! echo "ptw:pw $BIN:$JCC_OFF %di %si" >> "$EV" 2>/dev/null; then
+ tap 2 ok "rel8 jcc site rejected at create"
+else
+ if echo 1 > "$TR/events/uprobes/pw/enable" 2>/dev/null; then
+ echo 0 > "$TR/events/uprobes/pw/enable" 2>/dev/null
+ tap 2 not "rel8 jcc site enabled (expected rejection)"
+ else
+ tap 2 ok "rel8 jcc site rejected (no re-encode)"
+ fi
+ echo "-:pw" >> "$EV" 2>/dev/null
+fi
+
+
+# 3: mini-stress (50 fork/execs survive)
+if ! echo "ptw:pw $BIN:$PUN_OFF %di %si" >> "$EV" 2>/dev/null ||
+ ! echo 1 > "$TR/events/uprobes/pw/enable" 2>/dev/null; then
+ tap 3 not "churn mini-stress setup failed"
+else
+ fails=0
+ for i in $(seq 1 50); do
+ "$BIN" >/dev/null 2>&1 || fails=$((fails + 1))
+ done
+ echo 0 > "$TR/events/uprobes/pw/enable" 2>/dev/null
+ if [ "$fails" -eq 0 ] && sites_match; then
+ tap 3 ok "churn mini-stress (50 execs, 0 failures)"
+ else
+ tap 3 not "churn mini-stress ($fails/50 failed)"
+ fi
+fi
+echo "-:pw" >> "$EV" 2>/dev/null
+
+# 4: an aligned 5x1-byte NOP run takes the atomic patch path
+probe_run 4 "${NOP_OFF}%multinop" "%di %si" \
+ "aligned NOP-composition install (probe fires, target valid, site restored)"
+
+# 5: the single 5-byte NOP keeps the classic 3-phase poke
+probe_run 5 $NOP5_OFF "%di %si" \
+ "single 5-byte-NOP 3-phase install (probe fires, target valid, site restored)"
+
+# 6: enable/disable flip loop (50 re-installs stay correct)
+if ! echo "ptw:pw $BIN:$NOP5_OFF %di %si" >> "$EV" 2>/dev/null ||
+ [ ! -e "$TR/events/uprobes/pw/enable" ]; then
+ tap 6 not "flip loop setup failed"
+else
+ fails=0
+ for i in $(seq 1 50); do
+ echo 1 > "$TR/events/uprobes/pw/enable" 2>/dev/null ||
+ fails=$((fails + 1))
+ "$BIN" >/dev/null 2>&1 || fails=$((fails + 1))
+ echo 0 > "$TR/events/uprobes/pw/enable" 2>/dev/null ||
+ fails=$((fails + 1))
+ done
+ if [ "$fails" -eq 0 ] && sites_match; then
+ tap 6 ok "enable/disable flip loop (50 flips, 0 failures, site restored)"
+ else
+ tap 6 not "flip loop ($fails failures)"
+ fi
+fi
+echo "-:pw" >> "$EV" 2>/dev/null
+
+# 7: a 4-arg paced probe at a site with a stack-local sentinel
+probe_run 7 $RZ_OFF "%di %si %dx %r8" \
+ "4-arg paced probe"
+
+# 8: %nopace attached to the offset remains accepted
+probe_run 8 "${PUN_OFF}%nopace" "%di %si" \
+ "%nopace offset suffix is accepted"
+
+# 9: %nopace as a separate option remains accepted
+probe_run 9 "$PUN_OFF" "%nopace %di %si" \
+ "%nopace separate option is accepted"
+
+# 10: unknown ptwrite options must be rejected by tracefs
+if echo "ptw:bad $BIN:${PUN_OFF}%unknown %di %si" >> "$EV" 2>/dev/null; then
+ echo "-:bad" >> "$EV" 2>/dev/null
+ tap 10 not "unknown ptwrite option accepted"
+else
+ tap 10 ok "unknown ptwrite option rejected"
+fi
+
+# 11: an unaligned 5x1-byte NOP run cannot use the atomic patch path
+if ! echo "ptw:pw $BIN:${UNOP_OFF}%multinop %di %si" >> "$EV" 2>/dev/null; then
+ tap 11 ok "unaligned NOP-composition site rejected at create"
+else
+ if echo 1 > "$TR/events/uprobes/pw/enable" 2>/dev/null; then
+ echo 0 > "$TR/events/uprobes/pw/enable" 2>/dev/null
+ tap 11 not "unaligned NOP-composition site enabled"
+ else
+ tap 11 ok "unaligned NOP-composition site rejected at enable"
+ fi
+ echo "-:pw" >> "$EV" 2>/dev/null
+fi
+
+# the kselftest runner uses only the exit code
+[ "$failures" -eq 0 ] || exit 1
--
2.54.0
^ permalink raw reply [flat|nested] 12+ messages in thread