From: Josh Poimboeuf <jpoimboe@kernel.org>
To: x86@kernel.org
Cc: linux-kernel@vger.kernel.org, Borislav Petkov <bp@alien8.de>,
Peter Zijlstra <peterz@infradead.org>,
Babu Moger <babu.moger@amd.com>,
Paolo Bonzini <pbonzini@redhat.com>,
Sean Christopherson <seanjc@google.com>,
David.Kaplan@amd.com, Andrew Cooper <andrew.cooper3@citrix.com>,
Nikolay Borisov <nik.borisov@suse.com>,
gregkh@linuxfoundation.org, Thomas Gleixner <tglx@linutronix.de>
Subject: [PATCH 17/22] x86/srso: Disentangle rethunk-dependent options
Date: Sun, 20 Aug 2023 18:19:14 -0700 [thread overview]
Message-ID: <58449e5b95c192d076cb02bc1ee20b5713cd3c02.1692580085.git.jpoimboe@kernel.org> (raw)
In-Reply-To: <cover.1692580085.git.jpoimboe@kernel.org>
CONFIG_RETHUNK, CONFIG_CPU_UNRET_ENTRY and CONFIG_CPU_SRSO are all
tangled up. De-spaghettify the code a bit.
Some of the rethunk-related code has been shuffled around within the
'.text..__x86.return_thunk' section, but otherwise there are no
functional changes. srso_alias_untrain_ret() and srso_alias_safe_ret()
((which are very address-sensitive) haven't moved.
Signed-off-by: Josh Poimboeuf <jpoimboe@kernel.org>
---
arch/x86/include/asm/nospec-branch.h | 25 +++--
arch/x86/kernel/cpu/bugs.c | 5 +-
arch/x86/kernel/vmlinux.lds.S | 7 +-
arch/x86/lib/retpoline.S | 158 +++++++++++++++------------
4 files changed, 109 insertions(+), 86 deletions(-)
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 6c14fd1f5912..51e3f1a287d2 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -289,19 +289,17 @@
* where we have a stack but before any RET instruction.
*/
.macro UNTRAIN_RET
-#if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \
- defined(CONFIG_CALL_DEPTH_TRACKING) || defined(CONFIG_CPU_SRSO)
+#if defined(CONFIG_RETHUNK) || defined(CONFIG_CPU_IBPB_ENTRY)
VALIDATE_UNRET_END
ALTERNATIVE_3 "", \
CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \
"call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \
- __stringify(RESET_CALL_DEPTH), X86_FEATURE_CALL_DEPTH
+ __stringify(RESET_CALL_DEPTH), X86_FEATURE_CALL_DEPTH
#endif
.endm
.macro UNTRAIN_RET_VM
-#if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \
- defined(CONFIG_CALL_DEPTH_TRACKING) || defined(CONFIG_CPU_SRSO)
+#if defined(CONFIG_RETHUNK) || defined(CONFIG_CPU_IBPB_ENTRY)
VALIDATE_UNRET_END
ALTERNATIVE_3 "", \
CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \
@@ -311,8 +309,7 @@
.endm
.macro UNTRAIN_RET_FROM_CALL
-#if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \
- defined(CONFIG_CALL_DEPTH_TRACKING) || defined(CONFIG_CPU_SRSO)
+#if defined(CONFIG_RETHUNK) || defined(CONFIG_CPU_IBPB_ENTRY)
VALIDATE_UNRET_END
ALTERNATIVE_3 "", \
CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \
@@ -348,6 +345,20 @@ extern void __x86_return_thunk(void);
static inline void __x86_return_thunk(void) {}
#endif
+#ifdef CONFIG_CPU_UNRET_ENTRY
+extern void retbleed_return_thunk(void);
+#else
+static inline void retbleed_return_thunk(void) {}
+#endif
+
+#ifdef CONFIG_CPU_SRSO
+extern void srso_return_thunk(void);
+extern void srso_alias_return_thunk(void);
+#else
+static inline void srso_return_thunk(void) {}
+static inline void srso_alias_return_thunk(void) {}
+#endif
+
extern void retbleed_return_thunk(void);
extern void srso_return_thunk(void);
extern void srso_alias_return_thunk(void);
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index f24c0f7e3e8a..73d10e54fc1f 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -63,7 +63,7 @@ EXPORT_SYMBOL_GPL(x86_pred_cmd);
static DEFINE_MUTEX(spec_ctrl_mutex);
-void (*x86_return_thunk)(void) __ro_after_init = &__x86_return_thunk;
+void (*x86_return_thunk)(void) __ro_after_init = __x86_return_thunk;
/* Update SPEC_CTRL MSR and its cached copy unconditionally */
static void update_spec_ctrl(u64 val)
@@ -1042,8 +1042,7 @@ static void __init retbleed_select_mitigation(void)
setup_force_cpu_cap(X86_FEATURE_RETHUNK);
setup_force_cpu_cap(X86_FEATURE_UNRET);
- if (IS_ENABLED(CONFIG_RETHUNK))
- x86_return_thunk = retbleed_return_thunk;
+ x86_return_thunk = retbleed_return_thunk;
if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 83d41c2601d7..9188834e56c9 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -139,10 +139,7 @@ SECTIONS
STATIC_CALL_TEXT
ALIGN_ENTRY_TEXT_BEGIN
-#ifdef CONFIG_CPU_SRSO
*(.text..__x86.rethunk_untrain)
-#endif
-
ENTRY_TEXT
#ifdef CONFIG_CPU_SRSO
@@ -520,12 +517,12 @@ INIT_PER_CPU(irq_stack_backing_store);
"fixed_percpu_data is not at start of per-cpu area");
#endif
-#ifdef CONFIG_RETHUNK
+#ifdef CONFIG_CPU_UNRET_ENTRY
. = ASSERT((retbleed_return_thunk & 0x3f) == 0, "retbleed_return_thunk not cacheline-aligned");
-. = ASSERT((srso_safe_ret & 0x3f) == 0, "srso_safe_ret not cacheline-aligned");
#endif
#ifdef CONFIG_CPU_SRSO
+. = ASSERT((srso_safe_ret & 0x3f) == 0, "srso_safe_ret not cacheline-aligned");
/*
* GNU ld cannot do XOR until 2.41.
* https://sourceware.org/git/?p=binutils-gdb.git;a=commit;h=f6f78318fca803c4907fb8d7f6ded8295f1947b1
diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
index 5e3b016c6d3e..e5be0ecf3ce0 100644
--- a/arch/x86/lib/retpoline.S
+++ b/arch/x86/lib/retpoline.S
@@ -126,12 +126,13 @@ SYM_CODE_END(__x86_indirect_jump_thunk_array)
#include <asm/GEN-for-each-reg.h>
#undef GEN
#endif
-/*
- * This function name is magical and is used by -mfunction-return=thunk-extern
- * for the compiler to generate JMPs to it.
- */
+
#ifdef CONFIG_RETHUNK
+ .section .text..__x86.return_thunk
+
+#ifdef CONFIG_CPU_SRSO
+
/*
* srso_alias_untrain_ret() and srso_alias_safe_ret() are placed at
* special addresses:
@@ -147,9 +148,7 @@ SYM_CODE_END(__x86_indirect_jump_thunk_array)
*
* As a result, srso_alias_safe_ret() becomes a safe return.
*/
-#ifdef CONFIG_CPU_SRSO
- .section .text..__x86.rethunk_untrain
-
+ .pushsection .text..__x86.rethunk_untrain
SYM_START(srso_alias_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE)
UNWIND_HINT_FUNC
ANNOTATE_NOENDBR
@@ -157,17 +156,9 @@ SYM_START(srso_alias_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE)
lfence
jmp srso_alias_return_thunk
SYM_FUNC_END(srso_alias_untrain_ret)
+ .popsection
- .section .text..__x86.rethunk_safe
-#else
-/* dummy definition for alternatives */
-SYM_START(srso_alias_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE)
- ANNOTATE_UNRET_SAFE
- ret
- int3
-SYM_FUNC_END(srso_alias_untrain_ret)
-#endif
-
+ .pushsection .text..__x86.rethunk_safe
SYM_START(srso_alias_safe_ret, SYM_L_GLOBAL, SYM_A_NONE)
lea 8(%_ASM_SP), %_ASM_SP
UNWIND_HINT_FUNC
@@ -175,8 +166,7 @@ SYM_START(srso_alias_safe_ret, SYM_L_GLOBAL, SYM_A_NONE)
ret
int3
SYM_FUNC_END(srso_alias_safe_ret)
-
- .section .text..__x86.return_thunk
+ .popsection
SYM_CODE_START(srso_alias_return_thunk)
UNWIND_HINT_FUNC
@@ -185,6 +175,56 @@ SYM_CODE_START(srso_alias_return_thunk)
ud2
SYM_CODE_END(srso_alias_return_thunk)
+/*
+ * SRSO untraining sequence for Zen1/2, similar to retbleed_untrain_ret()
+ * above. On kernel entry, srso_untrain_ret() is executed which is a
+ *
+ * movabs $0xccccc30824648d48,%rax
+ *
+ * and when the return thunk executes the inner label srso_safe_ret()
+ * later, it is a stack manipulation and a RET which is mispredicted and
+ * thus a "safe" one to use.
+ */
+ .align 64
+ .skip 64 - (srso_safe_ret - srso_untrain_ret), 0xcc
+SYM_START(srso_untrain_ret, SYM_L_LOCAL, SYM_A_NONE)
+ ANNOTATE_NOENDBR
+ .byte 0x48, 0xb8
+
+/*
+ * This forces the function return instruction to speculate into a trap
+ * (UD2 in srso_return_thunk() below). This RET will then mispredict
+ * and execution will continue at the return site read from the top of
+ * the stack.
+ */
+SYM_INNER_LABEL(srso_safe_ret, SYM_L_GLOBAL)
+ lea 8(%_ASM_SP), %_ASM_SP
+ ret
+ int3
+ int3
+ /* end of movabs */
+ lfence
+ call srso_safe_ret
+ ud2
+SYM_CODE_END(srso_safe_ret)
+SYM_FUNC_END(srso_untrain_ret)
+
+SYM_CODE_START(srso_return_thunk)
+ UNWIND_HINT_FUNC
+ ANNOTATE_NOENDBR
+ call srso_safe_ret
+ ud2
+SYM_CODE_END(srso_return_thunk)
+
+#define JMP_SRSO_UNTRAIN_RET "jmp srso_untrain_ret"
+#define JMP_SRSO_ALIAS_UNTRAIN_RET "jmp srso_alias_untrain_ret"
+#else /* !CONFIG_CPU_SRSO */
+#define JMP_SRSO_UNTRAIN_RET "ud2"
+#define JMP_SRSO_ALIAS_UNTRAIN_RET "ud2"
+#endif /* CONFIG_CPU_SRSO */
+
+#ifdef CONFIG_CPU_UNRET_ENTRY
+
/*
* Some generic notes on the untraining sequences:
*
@@ -264,64 +304,21 @@ SYM_CODE_END(retbleed_return_thunk)
int3
SYM_FUNC_END(retbleed_untrain_ret)
-/*
- * SRSO untraining sequence for Zen1/2, similar to retbleed_untrain_ret()
- * above. On kernel entry, srso_untrain_ret() is executed which is a
- *
- * movabs $0xccccc30824648d48,%rax
- *
- * and when the return thunk executes the inner label srso_safe_ret()
- * later, it is a stack manipulation and a RET which is mispredicted and
- * thus a "safe" one to use.
- */
- .align 64
- .skip 64 - (srso_safe_ret - srso_untrain_ret), 0xcc
-SYM_START(srso_untrain_ret, SYM_L_LOCAL, SYM_A_NONE)
- ANNOTATE_NOENDBR
- .byte 0x48, 0xb8
+#define JMP_RETBLEED_UNTRAIN_RET "jmp retbleed_untrain_ret"
+#else /* !CONFIG_CPU_UNRET_ENTRY */
+#define JMP_RETBLEED_UNTRAIN_RET "ud2"
+#endif /* CONFIG_CPU_UNRET_ENTRY */
-/*
- * This forces the function return instruction to speculate into a trap
- * (UD2 in srso_return_thunk() below). This RET will then mispredict
- * and execution will continue at the return site read from the top of
- * the stack.
- */
-SYM_INNER_LABEL(srso_safe_ret, SYM_L_GLOBAL)
- lea 8(%_ASM_SP), %_ASM_SP
- ret
- int3
- int3
- /* end of movabs */
- lfence
- call srso_safe_ret
- ud2
-SYM_CODE_END(srso_safe_ret)
-SYM_FUNC_END(srso_untrain_ret)
-
-SYM_CODE_START(srso_return_thunk)
- UNWIND_HINT_FUNC
- ANNOTATE_NOENDBR
- call srso_safe_ret
- ud2
-SYM_CODE_END(srso_return_thunk)
+#if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_SRSO)
SYM_FUNC_START(entry_untrain_ret)
- ALTERNATIVE_2 "jmp retbleed_untrain_ret", \
- "jmp srso_untrain_ret", X86_FEATURE_SRSO, \
- "jmp srso_alias_untrain_ret", X86_FEATURE_SRSO_ALIAS
+ ALTERNATIVE_2 JMP_RETBLEED_UNTRAIN_RET, \
+ JMP_SRSO_UNTRAIN_RET, X86_FEATURE_SRSO, \
+ JMP_SRSO_ALIAS_UNTRAIN_RET, X86_FEATURE_SRSO_ALIAS
SYM_FUNC_END(entry_untrain_ret)
__EXPORT_THUNK(entry_untrain_ret)
-SYM_CODE_START(__x86_return_thunk)
- UNWIND_HINT_FUNC
- ANNOTATE_NOENDBR
- ANNOTATE_UNRET_SAFE
- ret
- int3
-SYM_CODE_END(__x86_return_thunk)
-EXPORT_SYMBOL(__x86_return_thunk)
-
-#endif /* CONFIG_RETHUNK */
+#endif /* CONFIG_CPU_UNRET_ENTRY || CONFIG_CPU_SRSO */
#ifdef CONFIG_CALL_DEPTH_TRACKING
@@ -356,3 +353,22 @@ SYM_FUNC_START(__x86_return_skl)
SYM_FUNC_END(__x86_return_skl)
#endif /* CONFIG_CALL_DEPTH_TRACKING */
+
+/*
+ * This function name is magical and is used by -mfunction-return=thunk-extern
+ * for the compiler to generate JMPs to it.
+ *
+ * This code is only used during kernel boot or module init. All
+ * 'JMP __x86_return_thunk' sites are changed to something else by
+ * apply_returns().
+ */
+SYM_CODE_START(__x86_return_thunk)
+ UNWIND_HINT_FUNC
+ ANNOTATE_NOENDBR
+ ANNOTATE_UNRET_SAFE
+ ret
+ int3
+SYM_CODE_END(__x86_return_thunk)
+EXPORT_SYMBOL(__x86_return_thunk)
+
+#endif /* CONFIG_RETHUNK */
--
2.41.0
next prev parent reply other threads:[~2023-08-21 1:19 UTC|newest]
Thread overview: 63+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-08-21 1:18 [PATCH 00/22] SRSO fixes/cleanups Josh Poimboeuf
2023-08-21 1:18 ` [PATCH 01/22] x86/srso: Fix srso_show_state() side effect Josh Poimboeuf
2023-08-21 5:42 ` Nikolay Borisov
2023-08-21 6:04 ` Borislav Petkov
2023-08-21 16:17 ` Josh Poimboeuf
2023-08-22 5:23 ` Borislav Petkov
2023-08-21 1:18 ` [PATCH 02/22] x86/srso: Set CPUID feature bits independently of bug or mitigation status Josh Poimboeuf
2023-08-21 5:42 ` Nikolay Borisov
2023-08-21 9:27 ` Andrew Cooper
2023-08-21 14:06 ` Borislav Petkov
2023-08-23 5:20 ` Borislav Petkov
2023-08-23 12:22 ` Andrew Cooper
2023-08-24 4:24 ` Borislav Petkov
2023-08-24 22:04 ` Josh Poimboeuf
2023-08-25 6:42 ` Borislav Petkov
2023-08-21 13:59 ` Borislav Petkov
2023-08-21 1:19 ` [PATCH 03/22] KVM: x86: Support IBPB_BRTYPE and SBPB Josh Poimboeuf
2023-08-21 9:34 ` Andrew Cooper
2023-08-21 16:23 ` Josh Poimboeuf
2023-08-21 16:35 ` Sean Christopherson
2023-08-21 16:46 ` Nikolay Borisov
2023-08-21 16:50 ` Sean Christopherson
2023-08-21 17:05 ` Josh Poimboeuf
2023-08-24 16:39 ` Sean Christopherson
2023-08-24 17:07 ` Josh Poimboeuf
2023-08-21 16:49 ` Sean Christopherson
2023-08-21 1:19 ` [PATCH 04/22] x86/srso: Fix SBPB enablement for spec_rstack_overflow=off Josh Poimboeuf
2023-08-21 14:16 ` Borislav Petkov
2023-08-21 16:36 ` Josh Poimboeuf
2023-08-22 5:54 ` Borislav Petkov
2023-08-22 6:07 ` Borislav Petkov
2023-08-22 21:59 ` Josh Poimboeuf
2023-08-23 1:27 ` Borislav Petkov
2023-08-21 1:19 ` [PATCH 05/22] x86/srso: Fix SBPB enablement for mitigations=off Josh Poimboeuf
2023-08-23 5:57 ` Borislav Petkov
2023-08-23 20:55 ` Josh Poimboeuf
2023-08-23 23:02 ` Josh Poimboeuf
2023-08-21 1:19 ` [PATCH 06/22] x86/srso: Print actual mitigation if requested mitigation isn't possible Josh Poimboeuf
2023-08-23 6:06 ` Borislav Petkov
2023-08-21 1:19 ` [PATCH 07/22] x86/srso: Remove default case in srso_select_mitigation() Josh Poimboeuf
2023-08-23 6:18 ` Borislav Petkov
2023-08-21 1:19 ` [PATCH 08/22] x86/srso: Downgrade retbleed IBPB warning to informational message Josh Poimboeuf
2023-08-24 4:43 ` Borislav Petkov
2023-08-21 1:19 ` [PATCH 09/22] x86/srso: Simplify exit paths Josh Poimboeuf
2023-08-21 1:19 ` [PATCH 10/22] x86/srso: Print mitigation for retbleed IBPB case Josh Poimboeuf
2023-08-24 4:48 ` Borislav Petkov
2023-08-24 21:40 ` Josh Poimboeuf
2023-08-21 1:19 ` [PATCH 11/22] x86/srso: Slight simplification Josh Poimboeuf
2023-08-24 4:55 ` Borislav Petkov
2023-08-21 1:19 ` [PATCH 12/22] x86/srso: Remove redundant X86_FEATURE_ENTRY_IBPB check Josh Poimboeuf
2023-08-25 7:09 ` Borislav Petkov
2023-08-21 1:19 ` [PATCH 13/22] x86/srso: Fix vulnerability reporting for missing microcode Josh Poimboeuf
2023-08-25 7:25 ` Borislav Petkov
2023-08-21 1:19 ` [PATCH 14/22] x86/srso: Fix unret validation dependencies Josh Poimboeuf
2023-08-21 1:19 ` [PATCH 15/22] x86/alternatives: Remove faulty optimization Josh Poimboeuf
2023-08-21 1:19 ` [PATCH 16/22] x86/srso: Unexport untraining functions Josh Poimboeuf
2023-08-21 5:50 ` Nikolay Borisov
2023-08-21 1:19 ` Josh Poimboeuf [this message]
2023-08-21 1:19 ` [PATCH 18/22] x86/rethunk: Use SYM_CODE_START[_LOCAL]_NOALIGN macros Josh Poimboeuf
2023-08-21 1:19 ` [PATCH 19/22] x86/srso: Improve i-cache locality for alias mitigation Josh Poimboeuf
2023-08-21 1:19 ` [PATCH 20/22] x86/retpoline: Remove .text..__x86.return_thunk section Josh Poimboeuf
2023-08-21 1:19 ` [PATCH 21/22] x86/nospec: Refactor UNTRAIN_RET[_*] Josh Poimboeuf
2023-08-21 1:19 ` [PATCH 22/22] x86/calldepth: Rename __x86_return_skl() to call_depth_return_thunk() Josh Poimboeuf
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=58449e5b95c192d076cb02bc1ee20b5713cd3c02.1692580085.git.jpoimboe@kernel.org \
--to=jpoimboe@kernel.org \
--cc=David.Kaplan@amd.com \
--cc=andrew.cooper3@citrix.com \
--cc=babu.moger@amd.com \
--cc=bp@alien8.de \
--cc=gregkh@linuxfoundation.org \
--cc=linux-kernel@vger.kernel.org \
--cc=nik.borisov@suse.com \
--cc=pbonzini@redhat.com \
--cc=peterz@infradead.org \
--cc=seanjc@google.com \
--cc=tglx@linutronix.de \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®