[PATCH 0/6] x86: Harden syscall dispatch

[PATCH 0/6] x86: Harden syscall dispatch

[Peter Zijlstra]

Hi!

These few patches harden the syscall dispatch by making -fno-jump-table
unconditional for x86_64 and then having all SYSCALL*() macros mark the actual
syscall entry points with __noendbr.

Notably, we already have -fno-jump-table for KERNEL_IBT || RETPOLINE, which
means effectively everybody already has this. However, making this
unconditional has the nice side effect that the compilers must never generate
array accesses and it is safe to remove array_index_nospec().

Additionally, since dispatch is now forced to be a binary decision tree with
direct (tail) calls, there are no valid indirect calls to any of the syscall
functions and they can be marked __noendbr.

This means anything trying to use the sys_call_table[] will fail (on hardware
sporting IBT).

All this makes the normal system call hijack a lot more difficult.

Patches also at:

  git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git x86/syscall

---
 arch/x86/Kconfig                       |  4 +---
 arch/x86/Makefile                      | 27 +++++++---------------
 arch/x86/entry/calling.h               |  1 -
 arch/x86/entry/common.c                |  2 +-
 arch/x86/entry/entry_64.S              |  2 +-
 arch/x86/entry/entry_64_compat.S       |  6 ++---
 arch/x86/entry/entry_fred.c            |  6 ++---
 arch/x86/entry/syscall_32.c            | 20 ++++++++++-------
 arch/x86/entry/syscall_64.c            | 24 +++++++++++++-------
 arch/x86/include/asm/entry-common.h    |  2 +-
 arch/x86/include/asm/fred.h            |  6 ++---
 arch/x86/include/asm/syscall.h         | 10 ++++-----
 arch/x86/include/asm/syscall_wrapper.h | 23 ++++++++++---------
 arch/x86/lib/clear_page_64.S           |  1 -
 arch/x86/lib/copy_user_64.S            |  1 -
 arch/x86/lib/copy_user_uncached_64.S   |  1 -
 arch/x86/lib/getuser.S                 |  4 ----
 arch/x86/lib/hweight.S                 |  3 ---
 arch/x86/lib/putuser.S                 |  4 ----
 include/linux/annotate.h               |  1 +
 include/linux/instrumentation.h        | 17 ++++++++++++++
 include/linux/objtool_types.h          |  1 +
 tools/include/linux/objtool_types.h    |  1 +
 tools/objtool/check.c                  | 41 ++++++++++++++++++++++++++--------
 24 files changed, 118 insertions(+), 90 deletions(-)

[PATCH 1/6] x86_64/entry: Unconditionally use -fno-jump-table

[Peter Zijlstra]

Currently we force -fno-jump-table for RETPOLINE=y || X86_KERNEL_IBT=y, which
means that it is effectively disabled for most everyone these days.

Simply unconditionally disable jump tables for x86_64 to take away the edge
case.

Doing so, allows removing the array_index_nospec() barrier in the syscall
dispatch since it cannot be an array index anymore.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 arch/x86/Kconfig            |    4 +---
 arch/x86/Makefile           |   27 ++++++++-------------------
 arch/x86/entry/syscall_32.c |    2 ++
 arch/x86/entry/syscall_64.c |    2 --
 4 files changed, 11 insertions(+), 24 deletions(-)

--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -2366,10 +2366,8 @@ config CC_OPT_INDIRECT_BRANCH_CS_PREFIX
 config CC_OPT_CF_PROTECTION_NONE
 	def_string "$(cc-option-str,-fcf-protection=none)"
 
-# with jump tables off: the compilers use NOTRACK for them, which kernel IBT
-# does not allow
 config CC_OPT_CF_PROTECTION_BRANCH
-	def_string "$(cc-option-str,-fcf-protection=branch -fno-jump-tables)"
+	def_string "$(cc-option-str,-fcf-protection=branch)"
 
 config CC_OPT_FALIGN_JUMPS
 	def_string "$(cc-option-str,-falign-jumps=1)"
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -90,17 +90,8 @@ CC_FLAGS_FPU += -mhard-float
 endif
 
 ifeq ($(CONFIG_X86_KERNEL_IBT),y)
-#
-# Kernel IBT has S_CET.NOTRACK_EN=0, as such the compilers must not generate
-# NOTRACK prefixes. Current generation compilers unconditionally employ NOTRACK
-# for jump-tables, as such, disable jump-tables for now.
-#
-# (jump-tables are implicitly disabled by RETPOLINE)
-#
-#   https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104816
-#
 KBUILD_CFLAGS += $(CONFIG_CC_OPT_CF_PROTECTION_BRANCH)
-KBUILD_RUSTFLAGS += -Zcf-protection=branch $(if $(call rustc-min-version,109300),-Cjump-tables=n,-Zno-jump-tables)
+KBUILD_RUSTFLAGS += -Zcf-protection=branch
 else
 KBUILD_CFLAGS += $(CONFIG_CC_OPT_CF_PROTECTION_NONE)
 endif
@@ -178,6 +169,13 @@ endif
         KBUILD_RUSTFLAGS += -Ccode-model=kernel
 
         percpu_seg := gs
+
+	# Due to retpolines and cf-protection=branch's implicit NOTRACK usage
+	# for jump-tables, blanked disable jump-tables for all x86_64 builds to
+	# get a consistent behaviour across configurations. This allows
+	# removing some array_index_nospec() usage.
+	KBUILD_CFLAGS += -fno-jump-tables
+	KBUILD_RUSTFLAGS += $(if $(call rustc-min-version,109300),-Cjump-tables=n,-Zno-jump-tables)
 endif
 
 ifeq ($(CONFIG_STACKPROTECTOR),y)
@@ -214,15 +212,6 @@ KBUILD_CFLAGS += -fno-asynchronous-unwin
 ifdef CONFIG_MITIGATION_RETPOLINE
   KBUILD_CFLAGS += $(RETPOLINE_CFLAGS)
   KBUILD_RUSTFLAGS += $(RETPOLINE_RUSTFLAGS)
-  # Additionally, avoid generating expensive indirect jumps which
-  # are subject to retpolines for small number of switch cases.
-  # LLVM turns off jump table generation by default when under
-  # retpoline builds, however, gcc does not for x86. This has
-  # only been fixed starting from gcc stable version 8.4.0 and
-  # onwards, but not for older ones. See gcc bug #86952.
-  ifndef CONFIG_CC_IS_CLANG
-    KBUILD_CFLAGS += -fno-jump-tables
-  endif
 endif
 
 ifdef CONFIG_MITIGATION_SLS
--- a/arch/x86/entry/syscall_32.c
+++ b/arch/x86/entry/syscall_32.c
@@ -75,7 +75,9 @@ early_param("ia32_emulation", ia32_emula
 static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs, unsigned long nr)
 {
 	if (likely(nr < IA32_NR_syscalls)) {
+#ifdef CONFIG_X86_32
 		nr = array_index_nospec(nr, IA32_NR_syscalls);
+#endif
 		regs->ax = ia32_sys_call(regs, (unsigned int)nr);
 	}
 }
--- a/arch/x86/entry/syscall_64.c
+++ b/arch/x86/entry/syscall_64.c
@@ -57,7 +57,6 @@ static noinline long x32_sys_call(const
 static __always_inline bool do_syscall_x64(struct pt_regs *regs, unsigned long nr)
 {
 	if (likely(nr < NR_syscalls)) {
-		nr = array_index_nospec(nr, NR_syscalls);
 		regs->ax = x64_sys_call(regs, (unsigned int)nr);
 		return true;
 	}
@@ -70,7 +69,6 @@ static __always_inline void do_syscall_x
 	nr -= __X32_SYSCALL_BIT;
 
 	if (IS_ENABLED(CONFIG_X86_X32_ABI) && likely(nr < X32_NR_syscalls)) {
-		nr = array_index_nospec(nr, X32_NR_syscalls);
 		regs->ax = x32_sys_call(regs, (unsigned int)nr);
 	}
 }

[PATCH 2/6] objtool: Add instrumentation_weak()

[Peter Zijlstra]

The strict pairing of instrumentation_begin()/instrumentation_end() gets in the
way of tail-call optimizations, since it requires writing code like:

  int bar(void);

  noinstr int foo(void)
  {
    int ret;
    instrumentation_begin();
    ret = bar();
    instrumentation_end();
    return ret;
  }

Introduce instrumentation_weak(), which is like instrumentation_begin() but
does not nest and does not pair. This way one can write:

  noinstr int foo(void)
  {
    instrumentation_weak();
    return bar();
  }

And get tail-calls without objtool getting upset.

Reported-by: kernel test robot <lkp@intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 include/linux/annotate.h            |    1 +
 include/linux/instrumentation.h     |   17 +++++++++++++++++
 include/linux/objtool_types.h       |    1 +
 tools/include/linux/objtool_types.h |    1 +
 tools/objtool/check.c               |   17 ++++++++++++-----
 5 files changed, 32 insertions(+), 5 deletions(-)

--- a/include/linux/annotate.h
+++ b/include/linux/annotate.h
@@ -71,6 +71,7 @@
  */
 #define ANNOTATE_INSTR_BEGIN(label)	ASM_ANNOTATE_LABEL(label, ANNOTYPE_INSTR_BEGIN)
 #define ANNOTATE_INSTR_END(label)	ASM_ANNOTATE_LABEL(label, ANNOTYPE_INSTR_END)
+#define ANNOTATE_INSTR_WEAK(label)	ASM_ANNOTATE_LABEL(label, ANNOTYPE_INSTR_WEAK)
 /*
  * objtool annotation to ignore the alternatives and only consider the original
  * instruction(s).
--- a/include/linux/instrumentation.h
+++ b/include/linux/instrumentation.h
@@ -52,9 +52,26 @@
 		     : : "i" (c));					\
 })
 #define instrumentation_end() __instrumentation_end(__COUNTER__)
+
+/*
+ * A very much weaker variant of instrumentation_begin(). It does not nest and
+ * it does not pair with instrumentation_end().
+ *
+ * Its purpose is to allow calling instrumented code while preserving
+ * tail-call optimization, something which is otherwise impossible with the
+ * need to call instrumentation_end().
+ */
+#define __instrumentation_weak(c) ({					\
+	asm volatile(__stringify(c) ": nop\n\t"				\
+		     ANNOTATE_INSTR_WEAK(__ASM_BREF(c))		\
+		     : : "i" (c));					\
+})
+#define instrumentation_weak() __instrumentation_weak(__COUNTER__)
+
 #else /* !CONFIG_NOINSTR_VALIDATION */
 # define instrumentation_begin()	do { } while(0)
 # define instrumentation_end()		do { } while(0)
+# define instrumentation_weak()		do { } while(0)
 #endif /* CONFIG_NOINSTR_VALIDATION */
 
 #endif /* __LINUX_INSTRUMENTATION_H */
--- a/include/linux/objtool_types.h
+++ b/include/linux/objtool_types.h
@@ -66,6 +66,7 @@ struct unwind_hint {
 #define ANNOTYPE_INTRA_FUNCTION_CALL	7
 #define ANNOTYPE_REACHABLE		8
 #define ANNOTYPE_NOCFI			9
+#define ANNOTYPE_INSTR_WEAK		10
 
 #define ANNOTYPE_DATA_SPECIAL		1
 
--- a/tools/include/linux/objtool_types.h
+++ b/tools/include/linux/objtool_types.h
@@ -66,6 +66,7 @@ struct unwind_hint {
 #define ANNOTYPE_INTRA_FUNCTION_CALL	7
 #define ANNOTYPE_REACHABLE		8
 #define ANNOTYPE_NOCFI			9
+#define ANNOTYPE_INSTR_WEAK		10
 
 #define ANNOTYPE_DATA_SPECIAL		1
 
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -2435,11 +2435,15 @@ static int __annotate_late(struct objtoo
 		break;
 
 	case ANNOTYPE_INSTR_BEGIN:
-		insn->instr++;
+		insn->instr += 2;
 		break;
 
 	case ANNOTYPE_INSTR_END:
-		insn->instr--;
+		insn->instr -= 2;
+		break;
+
+	case ANNOTYPE_INSTR_WEAK:
+		insn->instr += 1;
 		break;
 
 	case ANNOTYPE_UNRET_BEGIN:
@@ -3535,6 +3539,11 @@ static int validate_sibling_call(struct
 				 struct instruction *insn,
 				 struct insn_state *state)
 {
+	if (state->noinstr && state->instr > 1) {
+		WARN_INSN(insn, "sibling call with instrumentation enabled");
+		return 1;
+	}
+
 	if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
 		WARN_INSN(insn, "sibling call from callable instruction with modified stack frame");
 		return 1;
@@ -3545,7 +3554,7 @@ static int validate_sibling_call(struct
 
 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
 {
-	if (state->noinstr && state->instr > 0) {
+	if (state->noinstr && state->instr > 1) {
 		WARN_INSN(insn, "return with instrumentation enabled");
 		return 1;
 	}
@@ -3646,8 +3655,6 @@ static bool skip_alt_group(struct instru
 
 static int validate_branch(struct objtool_file *file, struct symbol *func,
 			   struct instruction *insn, struct insn_state state);
-static int do_validate_branch(struct objtool_file *file, struct symbol *func,
-			      struct instruction *insn, struct insn_state *state);
 
 static int validate_insn(struct objtool_file *file, struct symbol *func,
 			 struct instruction *insn, struct insn_state *statep,

[PATCH 3/6] x86_64/entry: Make syscall dispatch noinstr

[Peter Zijlstra]

Being able to put breakpoints and the like inside syscall dispatch is
'unfortunate'. Simply push noinstr out one more layer.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 arch/x86/entry/syscall_32.c |    3 ++-
 arch/x86/entry/syscall_64.c |   13 +++++++++++--
 2 files changed, 13 insertions(+), 3 deletions(-)

--- a/arch/x86/entry/syscall_32.c
+++ b/arch/x86/entry/syscall_32.c
@@ -43,8 +43,9 @@ const sys_call_ptr_t sys_call_table[] =
 #define __SYSCALL(nr, sym) case nr: return __ia32_##sym(regs);
 
 /* The unsigned int @nr argument is intentional as it creates denser code in a 64-bit build */
-static noinline long ia32_sys_call(const struct pt_regs *regs, unsigned int nr)
+static noinstr long ia32_sys_call(const struct pt_regs *regs, unsigned int nr)
 {
+	instrumentation_weak();
 	switch (nr) {
 	#include <asm/syscalls_32.h>
 	default: return __ia32_sys_ni_syscall(regs);
--- a/arch/x86/entry/syscall_64.c
+++ b/arch/x86/entry/syscall_64.c
@@ -34,17 +34,26 @@ const sys_call_ptr_t sys_call_table[] =
 #define __SYSCALL(nr, sym) case nr: return __x64_##sym(regs);
 
 /* The unsigned int @nr argument is intentional as it creates denser code */
-static noinline long x64_sys_call(const struct pt_regs *regs, unsigned int nr)
+static noinstr long x64_sys_call(const struct pt_regs *regs, unsigned int nr)
 {
+	/*
+	 * Because -fno-jump-tables, this compiles into a binary branch tree
+	 * rather than a jump-table. As such @nr is not used as an array
+	 * index. Additionally, this is an out-of-line function on purpose,
+	 * such that all the actual syscall function calls are tail-calls,
+	 * returning to our caller for the common bits.
+	 */
+	instrumentation_weak();
 	switch (nr) {
 	#include <asm/syscalls_64.h>
 	default: return __x64_sys_ni_syscall(regs);
 	}
 }
 
-static noinline long x32_sys_call(const struct pt_regs *regs, unsigned int nr)
+static noinstr long x32_sys_call(const struct pt_regs *regs, unsigned int nr)
 {
 #ifdef CONFIG_X86_X32_ABI
+	instrumentation_weak();
 	switch (nr) {
 	#include <asm/syscalls_x32.h>
 	default: return __x64_sys_ni_syscall(regs);

[PATCH 4/6] x86_64/entry: Mark all syscalls __noendbr

[Peter Zijlstra]

Since system call dispatch is now strictly through direct calls, there is no
reason for the syscalls to be called indirectly. Mark them all __noendbr.

Notably, the sys_call_table[] is purely a lookup for tracing and the like,
nobody actually (should) dispatch anything through it, and with this we make
sure nobody *can*.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 arch/x86/entry/syscall_32.c            |    5 +++--
 arch/x86/entry/syscall_64.c            |    7 ++++---
 arch/x86/include/asm/syscall_wrapper.h |   23 ++++++++++++-----------
 tools/objtool/check.c                  |    9 ++++++++-
 4 files changed, 27 insertions(+), 17 deletions(-)

--- a/arch/x86/entry/syscall_32.c
+++ b/arch/x86/entry/syscall_32.c
@@ -12,6 +12,7 @@
 #include <asm/traps.h>
 #include <asm/cpufeature.h>
 #include <asm/syscall.h>
+#include <asm/ibt.h>
 
 #ifdef CONFIG_IA32_EMULATION
 #define __SYSCALL_WITH_COMPAT(nr, native, compat)	__SYSCALL(nr, compat)
@@ -19,8 +20,8 @@
 #define __SYSCALL_WITH_COMPAT(nr, native, compat)	__SYSCALL(nr, native)
 #endif
 
-#define __SYSCALL(nr, sym) extern long __ia32_##sym(const struct pt_regs *);
-#define __SYSCALL_NORETURN(nr, sym) extern long __noreturn __ia32_##sym(const struct pt_regs *);
+#define __SYSCALL(nr, sym) extern __noendbr long __ia32_##sym(const struct pt_regs *);
+#define __SYSCALL_NORETURN(nr, sym) extern __noendbr long __noreturn __ia32_##sym(const struct pt_regs *);
 #include <asm/syscalls_32.h>
 #undef  __SYSCALL
 
--- a/arch/x86/entry/syscall_64.c
+++ b/arch/x86/entry/syscall_64.c
@@ -8,9 +8,10 @@
 #include <linux/entry-common.h>
 #include <linux/nospec.h>
 #include <asm/syscall.h>
+#include <asm/ibt.h>
 
-#define __SYSCALL(nr, sym) extern long __x64_##sym(const struct pt_regs *);
-#define __SYSCALL_NORETURN(nr, sym) extern long __noreturn __x64_##sym(const struct pt_regs *);
+#define __SYSCALL(nr, sym) extern __noendbr long __x64_##sym(const struct pt_regs *);
+#define __SYSCALL_NORETURN(nr, sym) extern __noendbr long __noreturn __x64_##sym(const struct pt_regs *);
 #include <asm/syscalls_64.h>
 #ifdef CONFIG_X86_X32_ABI
 #include <asm/syscalls_x32.h>
@@ -25,7 +26,7 @@
  * kernel/trace/trace_syscalls.c still wants to know the system
  * call address.
  */
-#define __SYSCALL(nr, sym) __x64_##sym,
+#define __SYSCALL(nr, sym) (void *)&__x64_##sym,
 const sys_call_ptr_t sys_call_table[] = {
 #include <asm/syscalls_64.h>
 };
--- a/arch/x86/include/asm/syscall_wrapper.h
+++ b/arch/x86/include/asm/syscall_wrapper.h
@@ -7,9 +7,10 @@
 #define _ASM_X86_SYSCALL_WRAPPER_H
 
 #include <asm/ptrace.h>
+#include <asm/ibt.h>
 
-extern long __x64_sys_ni_syscall(const struct pt_regs *regs);
-extern long __ia32_sys_ni_syscall(const struct pt_regs *regs);
+extern __noendbr long __x64_sys_ni_syscall(const struct pt_regs *regs);
+extern __noendbr long __ia32_sys_ni_syscall(const struct pt_regs *regs);
 
 /*
  * Instead of the generic __SYSCALL_DEFINEx() definition, the x86 version takes
@@ -83,22 +84,22 @@ extern long __ia32_sys_ni_syscall(const
 			__MAP(x, __SC_TYPE, __VA_ARGS__))		\
 
 #define __SYS_STUB0(abi, name)						\
-	long __##abi##_##name(const struct pt_regs *regs);		\
+	long __noendbr __##abi##_##name(const struct pt_regs *regs);	\
 	ALLOW_ERROR_INJECTION(__##abi##_##name, ERRNO);			\
-	long __##abi##_##name(const struct pt_regs *regs)		\
+	long __noendbr __##abi##_##name(const struct pt_regs *regs)	\
 		__alias(__do_##name);
 
 #define __SYS_STUBx(abi, name, ...)					\
-	long __##abi##_##name(const struct pt_regs *regs);		\
+	long __noendbr __##abi##_##name(const struct pt_regs *regs);	\
 	ALLOW_ERROR_INJECTION(__##abi##_##name, ERRNO);			\
-	long __##abi##_##name(const struct pt_regs *regs)		\
+	long __noendbr __##abi##_##name(const struct pt_regs *regs)	\
 	{								\
 		return __se_##name(__VA_ARGS__);			\
 	}
 
 #define __COND_SYSCALL(abi, name)					\
-	__weak long __##abi##_##name(const struct pt_regs *__unused);	\
-	__weak long __##abi##_##name(const struct pt_regs *__unused)	\
+	__weak long __noendbr __##abi##_##name(const struct pt_regs *__unused);	\
+	__weak long __noendbr __##abi##_##name(const struct pt_regs *__unused)	\
 	{								\
 		return sys_ni_syscall();				\
 	}
@@ -257,8 +258,8 @@ extern long __ia32_sys_ni_syscall(const
  * For VSYSCALLS, we need to declare these three syscalls with the new
  * pt_regs-based calling convention for in-kernel use.
  */
-long __x64_sys_getcpu(const struct pt_regs *regs);
-long __x64_sys_gettimeofday(const struct pt_regs *regs);
-long __x64_sys_time(const struct pt_regs *regs);
+long __noendbr __x64_sys_getcpu(const struct pt_regs *regs);
+long __noendbr __x64_sys_gettimeofday(const struct pt_regs *regs);
+long __noendbr __x64_sys_time(const struct pt_regs *regs);
 
 #endif /* _ASM_X86_SYSCALL_WRAPPER_H */
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -4672,8 +4672,15 @@ static int validate_ibt(struct objtool_f
 		    !strcmp(sec->name, "__patchable_function_entries"))
 			continue;
 
-		for_each_reloc(sec->rsec, reloc)
+		for_each_reloc(sec->rsec, reloc) {
+			struct symbol *sym = find_symbol_containing(reloc->sec->base,
+								    reloc_offset(reloc));
+
+			if (sym && !strcmp(sym->name, "sys_call_table"))
+				continue;
+
 			warnings += validate_ibt_data_reloc(file, reloc);
+		}
 	}
 
 	return warnings;

[PATCH 5/6] objtool: Allow EXPORT_SYMBOL() of __noendbr functions

[Peter Zijlstra]

Objtool IBT validation does two things:

 - it checks that all text references have an ENDBR instruction (WARN);
 - it finds all ENDBR instructions that have no references (SEAL).

The first ensures that all function pointers are of functions that have ENDBR,
since doing indirect calls requires this. The flip side is that you can have
text references that are not function pointers in the sense of later calling
them. Eg. ip_within_syscall_gap() will take the address of symbols, but will
never call them. Any such location that is referenced but does not have ENDBR
requires an ANNOTATE_NOENDBR.

The second is used to create a list of ENDBR instructions to SEAL. That is, at
boot time the kernel will replace all these ENDBR instructions with a NOP
instruction, limiting indirect targets.

Now, EXPORT_SYMBOL*(sym) is counted as a reference. This is important because
even if the kernel itself does not reference the symbol, a module still might.
The flip side is that exporting a __noendbr function needs an ANNOTATE_NOENDBR.

Relax this by supressing the objtool-WARN for .export_symbol references. This
allows exporting of __noendbr without further ANNOTATE_NOENDBR. Remove any such
now redudant annotation.

Reported-by: kernel test robot <lkp@intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 arch/x86/entry/calling.h             |    1 -
 arch/x86/entry/entry_64.S            |    2 +-
 arch/x86/entry/entry_64_compat.S     |    6 +++---
 arch/x86/lib/clear_page_64.S         |    1 -
 arch/x86/lib/copy_user_64.S          |    1 -
 arch/x86/lib/copy_user_uncached_64.S |    1 -
 arch/x86/lib/getuser.S               |    4 ----
 arch/x86/lib/hweight.S               |    3 ---
 arch/x86/lib/putuser.S               |    4 ----
 tools/objtool/check.c                |   15 ++++++++++++---
 10 files changed, 16 insertions(+), 22 deletions(-)

--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -432,7 +432,6 @@ For 32-bit we have the following convent
 /* rdi:	arg1 ... normal C conventions. rax is saved/restored. */
 .macro THUNK name, func
 SYM_FUNC_START(\name)
-	ANNOTATE_NOENDBR
 	pushq %rbp
 	movq %rsp, %rbp
 
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -95,7 +95,7 @@ SYM_CODE_START(entry_SYSCALL_64)
 	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
 
 SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL)
-	ANNOTATE_NOENDBR
+	ANNOTATE_NOENDBR // ip_within_syscall_gap
 
 	/* Construct struct pt_regs on stack */
 	pushq	$__USER_DS				/* pt_regs->ss */
--- a/arch/x86/entry/entry_64_compat.S
+++ b/arch/x86/entry/entry_64_compat.S
@@ -196,7 +196,7 @@ SYM_CODE_START(entry_SYSCALL_compat)
 	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
 
 SYM_INNER_LABEL(entry_SYSCALL_compat_safe_stack, SYM_L_GLOBAL)
-	ANNOTATE_NOENDBR
+	ANNOTATE_NOENDBR // ip_within_syscall_gap
 
 	/* Construct struct pt_regs on stack */
 	pushq	$__USER_DS		/* pt_regs->ss */
@@ -260,7 +260,7 @@ SYM_INNER_LABEL(entry_SYSCALL_compat_aft
          */
 	movq	RSP-ORIG_RAX(%rsp), %rsp
 SYM_INNER_LABEL(entry_SYSRETL_compat_unsafe_stack, SYM_L_GLOBAL)
-	ANNOTATE_NOENDBR
+	ANNOTATE_NOENDBR // ip_within_syscall_gap
 
 	/*
 	 * The original userspace %rsp (RSP-ORIG_RAX(%rsp)) is stored
@@ -280,7 +280,7 @@ SYM_INNER_LABEL(entry_SYSRETL_compat_uns
 	CLEAR_CPU_BUFFERS
 	sysretl
 SYM_INNER_LABEL(entry_SYSRETL_compat_end, SYM_L_GLOBAL)
-	ANNOTATE_NOENDBR
+	ANNOTATE_NOENDBR // ip_within_syscall_gap
 	int3
 SYM_CODE_END(entry_SYSCALL_compat)
 
--- a/arch/x86/lib/clear_page_64.S
+++ b/arch/x86/lib/clear_page_64.S
@@ -42,7 +42,6 @@ EXPORT_SYMBOL_GPL(__clear_pages_unrolled
  * rcx: uncleared bytes or 0 if successful.
  */
 SYM_FUNC_START(rep_stos_alternative)
-	ANNOTATE_NOENDBR
 	cmpq $64,%rcx
 	jae .Lunrolled
 
--- a/arch/x86/lib/copy_user_64.S
+++ b/arch/x86/lib/copy_user_64.S
@@ -32,7 +32,6 @@
  * it simpler for us, we can clobber rsi/rdi and rax freely.
  */
 SYM_FUNC_START(rep_movs_alternative)
-	ANNOTATE_NOENDBR
 	cmpq $64,%rcx
 	jae .Llarge
 
--- a/arch/x86/lib/copy_user_uncached_64.S
+++ b/arch/x86/lib/copy_user_uncached_64.S
@@ -28,7 +28,6 @@
  * rax uncopied bytes or 0 if successful.
  */
 SYM_FUNC_START(copy_to_nontemporal)
-	ANNOTATE_NOENDBR
 	/* If destination is not 7-byte aligned, we'll have to align it */
 	testb $7,%dil
 	jne .Lalign
--- a/arch/x86/lib/getuser.S
+++ b/arch/x86/lib/getuser.S
@@ -58,7 +58,6 @@
 
 	.text
 SYM_FUNC_START(__get_user_1)
-	ANNOTATE_NOENDBR
 	check_range size=1
 	ASM_STAC
 	UACCESS movzbl (%_ASM_AX),%edx
@@ -69,7 +68,6 @@ SYM_FUNC_END(__get_user_1)
 EXPORT_SYMBOL(__get_user_1)
 
 SYM_FUNC_START(__get_user_2)
-	ANNOTATE_NOENDBR
 	check_range size=2
 	ASM_STAC
 	UACCESS movzwl (%_ASM_AX),%edx
@@ -80,7 +78,6 @@ SYM_FUNC_END(__get_user_2)
 EXPORT_SYMBOL(__get_user_2)
 
 SYM_FUNC_START(__get_user_4)
-	ANNOTATE_NOENDBR
 	check_range size=4
 	ASM_STAC
 	UACCESS movl (%_ASM_AX),%edx
@@ -91,7 +88,6 @@ SYM_FUNC_END(__get_user_4)
 EXPORT_SYMBOL(__get_user_4)
 
 SYM_FUNC_START(__get_user_8)
-	ANNOTATE_NOENDBR
 #ifndef CONFIG_X86_64
 	xor %ecx,%ecx
 #endif
--- a/arch/x86/lib/hweight.S
+++ b/arch/x86/lib/hweight.S
@@ -10,8 +10,6 @@
  * %rdi: w
  */
 SYM_FUNC_START(__sw_hweight32)
-	ANNOTATE_NOENDBR
-
 #ifdef CONFIG_X86_64
 	movl %edi, %eax				# w
 #endif
@@ -44,7 +42,6 @@ EXPORT_SYMBOL(__sw_hweight32)
  */
 #ifdef CONFIG_X86_64
 SYM_FUNC_START(__sw_hweight64)
-	ANNOTATE_NOENDBR
 	pushq   %rdi
 	pushq   %rdx
 
--- a/arch/x86/lib/putuser.S
+++ b/arch/x86/lib/putuser.S
@@ -46,7 +46,6 @@
 
 .text
 SYM_FUNC_START(__put_user_1)
-	ANNOTATE_NOENDBR
 	check_range size=1
 	ASM_STAC
 1:	movb %al,(%_ASM_CX)
@@ -57,7 +56,6 @@ SYM_FUNC_END(__put_user_1)
 EXPORT_SYMBOL(__put_user_1)
 
 SYM_FUNC_START(__put_user_2)
-	ANNOTATE_NOENDBR
 	check_range size=2
 	ASM_STAC
 2:	movw %ax,(%_ASM_CX)
@@ -68,7 +66,6 @@ SYM_FUNC_END(__put_user_2)
 EXPORT_SYMBOL(__put_user_2)
 
 SYM_FUNC_START(__put_user_4)
-	ANNOTATE_NOENDBR
 	check_range size=4
 	ASM_STAC
 3:	movl %eax,(%_ASM_CX)
@@ -79,7 +76,6 @@ SYM_FUNC_END(__put_user_4)
 EXPORT_SYMBOL(__put_user_4)
 
 SYM_FUNC_START(__put_user_8)
-	ANNOTATE_NOENDBR
 	check_range size=8
 	ASM_STAC
 4:	mov %_ASM_AX,(%_ASM_CX)
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -4597,7 +4597,7 @@ static int validate_ibt_insn(struct objt
 }
 
 static int validate_ibt_data_reloc(struct objtool_file *file,
-				   struct reloc *reloc)
+				   struct reloc *reloc, bool warn)
 {
 	struct instruction *dest;
 
@@ -4611,7 +4611,7 @@ static int validate_ibt_data_reloc(struc
 		return 0;
 	}
 
-	if (dest->noendbr)
+	if (dest->noendbr || !warn)
 		return 0;
 
 	WARN_FUNC(reloc->sec->base, reloc_offset(reloc),
@@ -4636,6 +4636,7 @@ static int validate_ibt(struct objtool_f
 		warnings += validate_ibt_insn(file, insn);
 
 	for_each_sec(file->elf, sec) {
+		bool warn = true;
 
 		/* Already done by validate_ibt_insn() */
 		if (is_text_sec(sec))
@@ -4672,6 +4673,14 @@ static int validate_ibt(struct objtool_f
 		    !strcmp(sec->name, "__patchable_function_entries"))
 			continue;
 
+		/*
+		 * EXPORT_SYMBOL() must be counted towards an ENDBR usage,
+		 * such symbols must not be implicitly sealed. But it is fine
+		 * to EXPORT an explicitly (__noendbr) sealed function.
+		 */
+		if (!strcmp(sec->name, ".export_symbol"))
+			warn = false;
+
 		for_each_reloc(sec->rsec, reloc) {
 			struct symbol *sym = find_symbol_containing(reloc->sec->base,
 								    reloc_offset(reloc));
@@ -4679,7 +4688,7 @@ static int validate_ibt(struct objtool_f
 			if (sym && !strcmp(sym->name, "sys_call_table"))
 				continue;
 
-			warnings += validate_ibt_data_reloc(file, reloc);
+			warnings += validate_ibt_data_reloc(file, reloc, warn);
 		}
 	}
 

[PATCH 6/6] x86_64/entry: Mark C noinstr __noendbr

[Peter Zijlstra]

Mark all the C noinstr entry functions __noendbr where possible. Nobody has any
business of doing indirect calls to this.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 arch/x86/entry/common.c             |    2 +-
 arch/x86/entry/entry_fred.c         |    6 +++---
 arch/x86/entry/syscall_32.c         |   10 +++++-----
 arch/x86/entry/syscall_64.c         |    6 +++---
 arch/x86/include/asm/entry-common.h |    2 +-
 arch/x86/include/asm/fred.h         |    6 +++---
 arch/x86/include/asm/syscall.h      |   10 +++++-----
 7 files changed, 21 insertions(+), 21 deletions(-)

--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -15,7 +15,7 @@
  * purely a signal that an NMI/IRQ is pending, i.e. the event that triggered
  * the VM-Exit is held pending until it's unblocked in the host.
  */
-noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector)
+noinstr __noendbr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector)
 {
 	if (event_type == EVENT_TYPE_EXTINT) {
 #ifdef CONFIG_X86_64
--- a/arch/x86/entry/entry_fred.c
+++ b/arch/x86/entry/entry_fred.c
@@ -234,7 +234,7 @@ static noinstr void fred_swexc(struct pt
 	}
 }
 
-__visible noinstr void fred_entry_from_user(struct pt_regs *regs)
+__visible noinstr __noendbr void fred_entry_from_user(struct pt_regs *regs)
 {
 	unsigned long error_code = regs->orig_ax;
 
@@ -266,7 +266,7 @@ __visible noinstr void fred_entry_from_u
 	return fred_bad_type(regs, error_code);
 }
 
-__visible noinstr void fred_entry_from_kernel(struct pt_regs *regs)
+__visible noinstr __noendbr void fred_entry_from_kernel(struct pt_regs *regs)
 {
 	unsigned long error_code = regs->orig_ax;
 
@@ -295,7 +295,7 @@ __visible noinstr void fred_entry_from_k
 }
 
 #if IS_ENABLED(CONFIG_KVM_INTEL)
-__visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs)
+__visible noinstr __noendbr void __fred_entry_from_kvm(struct pt_regs *regs)
 {
 	switch (regs->fred_ss.type) {
 	case EVENT_TYPE_EXTINT:
--- a/arch/x86/entry/syscall_32.c
+++ b/arch/x86/entry/syscall_32.c
@@ -124,7 +124,7 @@ static __always_inline bool int80_is_ext
  *   eax:				system call number
  *   ebx, ecx, edx, esi, edi, ebp:	arg1 - arg 6
  */
-__visible noinstr void do_int80_emulation(struct pt_regs *regs)
+__visible noinstr __noendbr void do_int80_emulation(struct pt_regs *regs)
 {
 	long nr;
 
@@ -239,7 +239,7 @@ DEFINE_FREDENTRY_RAW(int80_emulation)
 #else /* CONFIG_IA32_EMULATION */
 
 /* Handles int $0x80 on a 32bit kernel */
-__visible noinstr void do_int80_syscall_32(struct pt_regs *regs)
+__visible noinstr __noendbr void do_int80_syscall_32(struct pt_regs *regs)
 {
 	long nr = syscall_32_enter(regs);
 
@@ -259,7 +259,7 @@ __visible noinstr void do_int80_syscall_
 }
 #endif /* !CONFIG_IA32_EMULATION */
 
-static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
+static noinstr __noendbr bool __do_fast_syscall_32(struct pt_regs *regs)
 {
 	long nr = syscall_32_enter(regs);
 	int res;
@@ -291,7 +291,7 @@ static noinstr bool __do_fast_syscall_32
 }
 
 /* Returns true to return using SYSEXIT/SYSRETL, or false to use IRET */
-__visible noinstr bool do_fast_syscall_32(struct pt_regs *regs)
+__visible noinstr __noendbr bool do_fast_syscall_32(struct pt_regs *regs)
 {
 	/*
 	 * Called using the internal vDSO SYSENTER/SYSCALL32 calling
@@ -338,7 +338,7 @@ __visible noinstr bool do_fast_syscall_3
 }
 
 /* Returns true to return using SYSEXIT/SYSRETL, or false to use IRET */
-__visible noinstr bool do_SYSENTER_32(struct pt_regs *regs)
+__visible noinstr __noendbr bool do_SYSENTER_32(struct pt_regs *regs)
 {
 	/* SYSENTER loses RSP, but the vDSO saved it in RBP. */
 	regs->sp = regs->bp;
--- a/arch/x86/entry/syscall_64.c
+++ b/arch/x86/entry/syscall_64.c
@@ -35,7 +35,7 @@ const sys_call_ptr_t sys_call_table[] =
 #define __SYSCALL(nr, sym) case nr: return __x64_##sym(regs);
 
 /* The unsigned int @nr argument is intentional as it creates denser code */
-static noinstr long x64_sys_call(const struct pt_regs *regs, unsigned int nr)
+static noinstr __noendbr long x64_sys_call(const struct pt_regs *regs, unsigned int nr)
 {
 	/*
 	 * Because -fno-jump-tables, this compiles into a binary branch tree
@@ -51,7 +51,7 @@ static noinstr long x64_sys_call(const s
 	}
 }
 
-static noinstr long x32_sys_call(const struct pt_regs *regs, unsigned int nr)
+static noinstr __noendbr long x32_sys_call(const struct pt_regs *regs, unsigned int nr)
 {
 #ifdef CONFIG_X86_X32_ABI
 	instrumentation_weak();
@@ -84,7 +84,7 @@ static __always_inline void do_syscall_x
 }
 
 /* Returns true to return using SYSRET, or false to use IRET */
-__visible noinstr bool do_syscall_64(struct pt_regs *regs, long nr)
+__visible noinstr __noendbr bool do_syscall_64(struct pt_regs *regs, long nr)
 {
 	if (likely(syscall_enter_from_user_mode_randomize_stack(regs, &nr))) {
 		instrumentation_begin();
--- a/arch/x86/include/asm/entry-common.h
+++ b/arch/x86/include/asm/entry-common.h
@@ -96,6 +96,6 @@ static __always_inline void arch_exit_to
 }
 #define arch_exit_to_user_mode arch_exit_to_user_mode
 
-extern void x86_entry_from_kvm(unsigned int entry_type, unsigned int vector);
+extern __noendbr void x86_entry_from_kvm(unsigned int entry_type, unsigned int vector);
 
 #endif
--- a/arch/x86/include/asm/fred.h
+++ b/arch/x86/include/asm/fred.h
@@ -67,9 +67,9 @@ void asm_fred_entrypoint_user(void);
 void asm_fred_entrypoint_kernel(void);
 void asm_fred_entry_from_kvm(struct fred_ss);
 
-__visible void fred_entry_from_user(struct pt_regs *regs);
-__visible void fred_entry_from_kernel(struct pt_regs *regs);
-__visible void __fred_entry_from_kvm(struct pt_regs *regs);
+__visible __noendbr void fred_entry_from_user(struct pt_regs *regs);
+__visible __noendbr void fred_entry_from_kernel(struct pt_regs *regs);
+__visible __noendbr void __fred_entry_from_kvm(struct pt_regs *regs);
 
 /* Can be called from noinstr code, thus __always_inline */
 static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned int vector)
--- a/arch/x86/include/asm/syscall.h
+++ b/arch/x86/include/asm/syscall.h
@@ -164,13 +164,13 @@ static inline int syscall_get_arch(struc
 		? AUDIT_ARCH_I386 : AUDIT_ARCH_X86_64;
 }
 
-bool do_syscall_64(struct pt_regs *regs, long nr);
-void do_int80_emulation(struct pt_regs *regs);
+__noendbr bool do_syscall_64(struct pt_regs *regs, long nr);
+__noendbr void do_int80_emulation(struct pt_regs *regs);
 
 #endif	/* CONFIG_X86_32 */
 
-void do_int80_syscall_32(struct pt_regs *regs);
-bool do_fast_syscall_32(struct pt_regs *regs);
-bool do_SYSENTER_32(struct pt_regs *regs);
+__noendbr void do_int80_syscall_32(struct pt_regs *regs);
+__noendbr bool do_fast_syscall_32(struct pt_regs *regs);
+__noendbr bool do_SYSENTER_32(struct pt_regs *regs);
 
 #endif	/* _ASM_X86_SYSCALL_H */

Re: [PATCH 1/6] x86_64/entry: Unconditionally use -fno-jump-table

[Breno Leitao]

On Mon, Nov 02, 2026 at 12:25:50PM +0100, Peter Zijlstra wrote:
> Currently we force -fno-jump-table for RETPOLINE=y || X86_KERNEL_IBT=y, which
> means that it is effectively disabled for most everyone these days.
> 
> Simply unconditionally disable jump tables for x86_64 to take away the edge
> case.
> 
> Doing so, allows removing the array_index_nospec() barrier in the syscall
> dispatch since it cannot be an array index anymore.
> 
> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>

Thanks for the patchset.

Tested-by: Breno Leitao <leitao@debian.org>