From: Dylan Hatch <dylanbhatch@google.com>
To: Catalin Marinas <catalin.marinas@arm.com>,
Will Deacon <will@kernel.org>,
Mark Rutland <mark.rutland@arm.com>
Cc: Herbert Xu <herbert@gondor.apana.org.au>,
"David S . Miller" <davem@davemloft.net>,
Nathan Chancellor <nathan@kernel.org>,
Nicolas Schier <nsc@kernel.org>,
Nick Desaulniers <ndesaulniers@google.com>,
Bill Wendling <morbo@google.com>,
Justin Stitt <justinstitt@google.com>,
Weinan Liu <wnliu@google.com>, Jens Remus <jremus@linux.ibm.com>,
linux-arm-kernel@lists.infradead.org,
linux-kernel@vger.kernel.org, linux-kbuild@vger.kernel.org,
linux-crypto@vger.kernel.org, llvm@lists.linux.dev,
Josh Poimboeuf <jpoimboe@kernel.org>,
Steven Rostedt <rostedt@goodmis.org>,
Ard Biesheuvel <ardb@kernel.org>,
Indu Bhagat <ibhagatgnu@gmail.com>,
Prasanna Kumar T S M <ptsm@linux.microsoft.com>,
live-patching@vger.kernel.org,
Dylan Hatch <dylanbhatch@google.com>
Subject: [RFC PATCH 08/12] arm64, unwind: add in-kernel .lr_unwind lookup and validation
Date: Tue, 17 Nov 2026 04:35:00 +0000 [thread overview]
Message-ID: <20261117043504.1021063-9-dylanbhatch@google.com> (raw)
In-Reply-To: <20261117043504.1021063-1-dylanbhatch@google.com>
Add runtime lookup and optional boot-time validation for the vmlinux
.lr_unwind_fde and .lr_unwind_ranges tables.
lr_unwind_find() binary searches the sorted .lr_unwind_fde table with
monotonic bounds checking, validates start IPs against arm64 kernel text
sections (.text, .init.text, .exit.text, and .rodata.text), and scans
the inline or out-of-line instruction ranges to determine the exception
boundary unwind mode (LR_UNWIND_LIVE, LR_UNWIND_FP, or LR_UNWIND_SP) and
any SP-relative frame-record offset.
Suggested-by: Jens Remus <jremus@linux.ibm.com>
Signed-off-by: Dylan Hatch <dylanbhatch@google.com>
---
arch/arm64/Kconfig | 9 +
arch/arm64/include/asm/lr_unwind.h | 31 ++++
arch/arm64/include/asm/sections.h | 3 +
arch/arm64/kernel/Makefile | 5 +
arch/arm64/kernel/lr_unwind.c | 274 +++++++++++++++++++++++++++++
arch/arm64/kernel/setup.c | 3 +
arch/arm64/kernel/vmlinux.lds.S | 2 +
7 files changed, 327 insertions(+)
create mode 100644 arch/arm64/kernel/lr_unwind.c
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index aa09c7ea61be3..1503b8a0b7021 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -2517,6 +2517,15 @@ config ARM64_LR_UNWIND
live in x30, saved in the frame record at [x29], or saved in the
frame record at an SP-relative offset).
+config ARM64_LR_UNWIND_VALIDATION
+ bool "Validate .lr_unwind sections at boot and module load"
+ depends on ARM64_LR_UNWIND
+ help
+ Validate every FDE and range entry in .lr_unwind_fde and
+ .lr_unwind_ranges when initializing vmlinux and module unwind
+ tables. This is a debugging option that adds boot and module load
+ overhead and should normally be disabled in production.
+
config ARM64_CONTPTE
bool "Contiguous PTE mappings for user memory" if EXPERT
depends on TRANSPARENT_HUGEPAGE
diff --git a/arch/arm64/include/asm/lr_unwind.h b/arch/arm64/include/asm/lr_unwind.h
index 81c743994c4d5..bfb6abed05f76 100644
--- a/arch/arm64/include/asm/lr_unwind.h
+++ b/arch/arm64/include/asm/lr_unwind.h
@@ -108,4 +108,35 @@ static __always_inline void lr_fde_unpack_inline(u32 desc, lr_range_t ranges[4])
#define LR_OOL_IDX(d) (((d) >> 6) & LR_OOL_IDX_MAX)
#define LR_OOL_COUNT(d) ((d) & LR_OOL_COUNT_MAX)
+#ifdef __KERNEL__
+
+#include <linux/errno.h>
+#include <linux/init.h>
+
+struct lr_unwind_section {
+ const struct lr_fde *fdes;
+ unsigned int num_fdes;
+ const lr_range_t *ranges;
+ unsigned int num_ranges;
+};
+
+#ifdef CONFIG_ARM64_LR_UNWIND
+
+void __init init_lr_unwind_table(void);
+int lr_unwind_find(unsigned long ip, enum lr_unwind_mode *mode,
+ unsigned int *sp_off);
+
+#else /* !CONFIG_ARM64_LR_UNWIND */
+
+static inline void __init init_lr_unwind_table(void) {}
+static inline int lr_unwind_find(unsigned long ip, enum lr_unwind_mode *mode,
+ unsigned int *sp_off)
+{
+ return -EINVAL;
+}
+
+#endif /* CONFIG_ARM64_LR_UNWIND */
+
+#endif /* __KERNEL__ */
+
#endif /* __ASM_LR_UNWIND_H */
diff --git a/arch/arm64/include/asm/sections.h b/arch/arm64/include/asm/sections.h
index 51b0d594239eb..2f950aa28225f 100644
--- a/arch/arm64/include/asm/sections.h
+++ b/arch/arm64/include/asm/sections.h
@@ -23,6 +23,9 @@ extern char __irqentry_text_start[], __irqentry_text_end[];
extern char __mmuoff_data_start[], __mmuoff_data_end[];
extern char __entry_tramp_text_start[], __entry_tramp_text_end[];
extern char __relocate_new_kernel_start[], __relocate_new_kernel_end[];
+extern char _srodatatext[], _erodatatext[];
+extern char __start_lr_unwind_fde[], __stop_lr_unwind_fde[];
+extern char __start_lr_unwind_ranges[], __stop_lr_unwind_ranges[];
static inline size_t entry_tramp_text_size(void)
{
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index fc5d1a0c38b3c..f07a0cead3ee3 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -18,12 +18,16 @@ CFLAGS_syscall.o += -fno-stack-protector
# can significantly impact performance. Avoid instrumenting the stack trace
# collection code to minimize this impact.
KASAN_SANITIZE_stacktrace.o := n
+KASAN_SANITIZE_lr_unwind.o := n
+KCSAN_SANITIZE_lr_unwind.o := n
+UBSAN_SANITIZE_lr_unwind.o := n
# It's not safe to invoke KCOV when portions of the kernel environment aren't
# available or are out-of-sync with HW state. Since `noinstr` doesn't always
# inhibit KCOV instrumentation, disable it for the entire compilation unit.
KCOV_INSTRUMENT_entry-common.o := n
KCOV_INSTRUMENT_idle.o := n
+KCOV_INSTRUMENT_lr_unwind.o := n
# Softirq stack switching can run outside interrupt context.
KCOV_INSTRUMENT_irq.o := n
@@ -71,6 +75,7 @@ obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
obj-$(CONFIG_VMCORE_INFO) += vmcore_info.o
obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o
obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o
+obj-$(CONFIG_ARM64_LR_UNWIND) += lr_unwind.o
obj-$(CONFIG_ARM64_MPAM) += mpam.o
obj-$(CONFIG_ARM64_MTE) += mte.o
obj-y += vdso-wrap.o
diff --git a/arch/arm64/kernel/lr_unwind.c b/arch/arm64/kernel/lr_unwind.c
new file mode 100644
index 0000000000000..53e2c78a8bdaf
--- /dev/null
+++ b/arch/arm64/kernel/lr_unwind.c
@@ -0,0 +1,274 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Kernel .lr_unwind lookup and validation
+ */
+
+#define pr_fmt(fmt) "lr_unwind: " fmt
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/kprobes.h>
+#include <linux/limits.h>
+#include <linux/printk.h>
+
+#include <asm/lr_unwind.h>
+#include <asm/sections.h>
+
+static bool lr_unwind_init __ro_after_init;
+static struct lr_unwind_section kernel_lr_sec __ro_after_init;
+
+static const char *objname(const struct lr_unwind_section *sec)
+{
+ return "vmlinux";
+}
+
+#define dbg_sec(sec, fmt, ...) \
+ pr_debug("(%s): " fmt, objname(sec), ##__VA_ARGS__)
+
+static __always_inline unsigned long lr_fde_start_addr(const struct lr_fde *fde)
+{
+ return (unsigned long)&fde->start_offset + fde->start_offset;
+}
+
+static __always_inline bool is_kernel_ip_valid(unsigned long ip)
+{
+ if (ip >= (unsigned long)_stext && ip < (unsigned long)_etext)
+ return true;
+
+ if (system_state < SYSTEM_FREEING_INITMEM &&
+ ip >= (unsigned long)_sinittext && ip < (unsigned long)_einittext)
+ return true;
+
+ /* .exit.text is retained in vmlinux on arm64. */
+ if (ip >= (unsigned long)__exittext_begin &&
+ ip < (unsigned long)__exittext_end)
+ return true;
+
+ /*
+ * .rodata.text is never executed from the kernel mapping, but it still
+ * has unwind data.
+ */
+ if (ip >= (unsigned long)_srodatatext &&
+ ip < (unsigned long)_erodatatext)
+ return true;
+
+ return false;
+}
+
+static __always_inline bool is_sec_ip_valid(const struct lr_unwind_section *sec,
+ unsigned long ip)
+{
+ return is_kernel_ip_valid(ip);
+}
+
+static __always_inline int __find_fde(const struct lr_unwind_section *sec,
+ unsigned long ip,
+ const struct lr_fde **fde_out)
+{
+ unsigned long func_addr_low = 0, func_addr_high = ULONG_MAX;
+ unsigned int low = 0, high = sec->num_fdes;
+ const struct lr_fde *found = NULL;
+
+ while (low < high) {
+ unsigned int mid = low + ((high - low) / 2);
+ const struct lr_fde *fde = &sec->fdes[mid];
+ unsigned long func_addr = lr_fde_start_addr(fde);
+
+ if (ip >= func_addr) {
+ if (func_addr < func_addr_low)
+ return -EFAULT;
+
+ func_addr_low = func_addr;
+ found = fde;
+ low = mid + 1;
+ } else {
+ if (func_addr > func_addr_high)
+ return -EFAULT;
+
+ func_addr_high = func_addr;
+ high = mid;
+ }
+ }
+
+ if (!found || !is_sec_ip_valid(sec, lr_fde_start_addr(found)))
+ return -EINVAL;
+
+ *fde_out = found;
+ return 0;
+}
+
+static __always_inline int __lr_unwind_find(const struct lr_unwind_section *sec,
+ unsigned long ip,
+ enum lr_unwind_mode *mode,
+ unsigned int *sp_off)
+{
+ lr_range_t inline_ranges[4];
+ const lr_range_t *ranges;
+ const struct lr_fde *fde;
+ unsigned long insn_off;
+ u32 desc, cnt;
+ int ret;
+
+ if (ip & 0x3)
+ return -EINVAL;
+
+ ret = __find_fde(sec, ip, &fde);
+ if (ret)
+ return ret;
+
+ insn_off = (ip - lr_fde_start_addr(fde)) >> 2;
+ desc = fde->range_desc;
+
+ if (likely(desc & LR_FDE_INLINE)) {
+ lr_fde_unpack_inline(desc, inline_ranges);
+ ranges = inline_ranges;
+ cnt = 4;
+ } else {
+ u32 idx = LR_OOL_IDX(desc);
+
+ cnt = LR_OOL_COUNT(desc);
+ if (!cnt || idx + cnt > sec->num_ranges)
+ return -EFAULT;
+ ranges = &sec->ranges[idx];
+ }
+
+ for (u32 i = 0; i < cnt; i++) {
+ u32 len = LR_RANGE_LEN(ranges[i]);
+
+ if (insn_off < len) {
+ enum lr_unwind_mode m = LR_RANGE_MODE(ranges[i]);
+
+ if (unlikely(m > LR_UNWIND_SP))
+ return -EFAULT;
+ *mode = m;
+ *sp_off = LR_FDE_SP_OFF_BYTES(desc);
+ return 0;
+ }
+ insn_off -= len;
+ }
+
+ /* Make sure IP is not in an unannotated gap past this FDE */
+ return -EINVAL;
+}
+
+noinstr int lr_unwind_find(unsigned long ip, enum lr_unwind_mode *mode,
+ unsigned int *sp_off)
+{
+ if (!mode || !sp_off)
+ return -EINVAL;
+
+ if (is_kernel_ip_valid(ip)) {
+ if (!lr_unwind_init)
+ return -EINVAL;
+
+ return __lr_unwind_find(&kernel_lr_sec, ip, mode, sp_off);
+ }
+
+ return -EINVAL;
+}
+NOKPROBE_SYMBOL(lr_unwind_find);
+
+#ifdef CONFIG_ARM64_LR_UNWIND_VALIDATION
+
+static int lr_unwind_validate_section(const struct lr_unwind_section *sec)
+{
+ unsigned long prev_end = 0;
+
+ for (unsigned int i = 0; i < sec->num_fdes; i++) {
+ const struct lr_fde *fde = &sec->fdes[i];
+ unsigned long ip = lr_fde_start_addr(fde);
+ u32 desc = fde->range_desc;
+ u32 total_insns = 0;
+
+ if ((ip & 0x3) || !is_sec_ip_valid(sec, ip)) {
+ dbg_sec(sec, "FDE %u invalid start IP 0x%lx\n", i, ip);
+ return -EINVAL;
+ }
+
+ if (ip < prev_end) {
+ dbg_sec(sec, "FDE %u unsorted or overlapping\n", i);
+ return -EFAULT;
+ }
+
+ if (desc & LR_FDE_INLINE) {
+ total_insns = LR_INLINE_R0(desc) +
+ LR_INLINE_R1(desc) +
+ LR_INLINE_R2(desc) +
+ LR_INLINE_R3(desc);
+ } else {
+ u32 idx = LR_OOL_IDX(desc);
+ u32 cnt = LR_OOL_COUNT(desc);
+
+ if (!cnt || idx + cnt > sec->num_ranges) {
+ dbg_sec(sec, "FDE %u OOL range out of bounds\n", i);
+ return -EFAULT;
+ }
+ for (u32 j = 0; j < cnt; j++) {
+ lr_range_t r = sec->ranges[idx + j];
+ u32 len = LR_RANGE_LEN(r);
+
+ if (!len || LR_RANGE_MODE(r) > LR_UNWIND_SP) {
+ dbg_sec(sec, "FDE %u invalid OOL entry\n", i);
+ return -EINVAL;
+ }
+ total_insns += len;
+ }
+ }
+
+ if (!total_insns) {
+ dbg_sec(sec, "FDE %u has empty range\n", i);
+ return -EINVAL;
+ }
+
+ prev_end = ip + ((unsigned long)total_insns << 2);
+ if (!is_sec_ip_valid(sec, prev_end - 4)) {
+ dbg_sec(sec, "FDE %u end IP out of bounds\n", i);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+#else /* !CONFIG_ARM64_LR_UNWIND_VALIDATION */
+
+static int lr_unwind_validate_section(const struct lr_unwind_section *sec)
+{
+ return 0;
+}
+
+#endif /* CONFIG_ARM64_LR_UNWIND_VALIDATION */
+
+static void dbg_print_section(const struct lr_unwind_section *sec)
+{
+ dbg_sec(sec, "fdes:0x%lx (%u) ranges:0x%lx (%u)\n",
+ (unsigned long)sec->fdes, sec->num_fdes,
+ (unsigned long)sec->ranges, sec->num_ranges);
+}
+
+void __init init_lr_unwind_table(void)
+{
+ size_t fde_bytes = (unsigned long)__stop_lr_unwind_fde -
+ (unsigned long)__start_lr_unwind_fde;
+ size_t range_bytes = (unsigned long)__stop_lr_unwind_ranges -
+ (unsigned long)__start_lr_unwind_ranges;
+
+ if (!fde_bytes || (fde_bytes % sizeof(struct lr_fde)) ||
+ (range_bytes % sizeof(lr_range_t))) {
+ pr_warn("invalid vmlinux .lr_unwind section sizes\n");
+ return;
+ }
+
+ kernel_lr_sec.fdes = (const struct lr_fde *)__start_lr_unwind_fde;
+ kernel_lr_sec.num_fdes = fde_bytes / sizeof(struct lr_fde);
+ kernel_lr_sec.ranges = (const lr_range_t *)__start_lr_unwind_ranges;
+ kernel_lr_sec.num_ranges = range_bytes / sizeof(lr_range_t);
+
+ if (lr_unwind_validate_section(&kernel_lr_sec)) {
+ pr_warn("invalid vmlinux .lr_unwind section\n");
+ return;
+ }
+
+ lr_unwind_init = true;
+ dbg_print_section(&kernel_lr_sec);
+}
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index a707deba6ae70..a19563e382ebd 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -43,6 +43,7 @@
#include <asm/cpufeature.h>
#include <asm/cpu_ops.h>
#include <asm/kasan.h>
+#include <asm/lr_unwind.h>
#include <asm/numa.h>
#include <asm/scs.h>
#include <asm/sections.h>
@@ -373,6 +374,8 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p)
"This indicates a broken bootloader or old kernel\n",
boot_args[1], boot_args[2], boot_args[3]);
}
+
+ init_lr_unwind_table();
}
static inline bool cpu_can_disable(unsigned int cpu)
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index b8f2963d10030..dc6c4a041f691 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -256,12 +256,14 @@ SECTIONS
/* code sections that are never executed via the kernel mapping */
.rodata.text : {
+ _srodatatext = .;
TRAMP_TEXT
HIBERNATE_TEXT
KEXEC_TEXT
IDMAP_TEXT
. = ALIGN(PAGE_SIZE);
}
+ _erodatatext = .;
idmap_pg_dir = .;
. += PAGE_SIZE;
--
2.56.0.679.gc527c65306-goog
next prev parent reply other threads:[~2026-11-17 4:35 UTC|newest]
Thread overview: 13+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-17 4:34 [RFC PATCH 00/12] arm64, unwind: add .lr_unwind unwinder for reliable stacktrace Dylan Hatch
2026-11-17 4:34 ` [RFC PATCH 01/12] arm64, unwind: build with .eh_frame and add gen_lr_unwind skeleton Dylan Hatch
2026-11-17 4:34 ` [RFC PATCH 02/12] arm64: factor .eh_frame parser out of patch-scs.c into <asm/eh_frame.h> Dylan Hatch
2026-11-17 4:34 ` [RFC PATCH 03/12] arm64: eh_frame: track CFA and x29/x30 register state in CFI parser Dylan Hatch
2026-11-17 4:34 ` [RFC PATCH 04/12] arm64, unwind: parse .eh_frame into .lr_unwind entries in gen_lr_unwind Dylan Hatch
2026-11-17 4:34 ` [RFC PATCH 05/12] scripts/gen_lr_unwind: emit .lr_unwind_fde and .lr_unwind_ranges Dylan Hatch
2026-11-17 4:34 ` [RFC PATCH 06/12] scripts/gen_lr_unwind: add --dump option to inspect .lr_unwind tables Dylan Hatch
2026-11-17 4:34 ` [RFC PATCH 07/12] scripts/sorttable: sort .lr_unwind_fde by start_offset Dylan Hatch
2026-11-17 4:35 ` Dylan Hatch [this message]
2026-11-17 4:35 ` [RFC PATCH 09/12] arm64/module, unwind: add .lr_unwind support for modules Dylan Hatch
2026-11-17 4:35 ` [RFC PATCH 10/12] arm64: entry: add unwind info for call_on_irq_stack() Dylan Hatch
2026-11-17 4:35 ` [RFC PATCH 11/12] arm64, crypto/lib: annotate leaf functions with CFI info Dylan Hatch
2026-11-17 4:35 ` [RFC PATCH 12/12] unwind: arm64: use .lr_unwind to unwind interrupt frames Dylan Hatch
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261117043504.1021063-9-dylanbhatch@google.com \
--to=dylanbhatch@google.com \
--cc=ardb@kernel.org \
--cc=catalin.marinas@arm.com \
--cc=davem@davemloft.net \
--cc=herbert@gondor.apana.org.au \
--cc=ibhagatgnu@gmail.com \
--cc=jpoimboe@kernel.org \
--cc=jremus@linux.ibm.com \
--cc=justinstitt@google.com \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-crypto@vger.kernel.org \
--cc=linux-kbuild@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=live-patching@vger.kernel.org \
--cc=llvm@lists.linux.dev \
--cc=mark.rutland@arm.com \
--cc=morbo@google.com \
--cc=nathan@kernel.org \
--cc=ndesaulniers@google.com \
--cc=nsc@kernel.org \
--cc=ptsm@linux.microsoft.com \
--cc=rostedt@goodmis.org \
--cc=will@kernel.org \
--cc=wnliu@google.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®