[PATCH] x86/fpu: Avoid kernel-address leaks in the FXSAVE workaround

[Kyle Zeng <kylebot@openai.com>]

On AMD CPUs without XSaveErPtr, restoring x87 state without a pending
exception can leave FDP/FIP/FOP unchanged.  The existing workaround
replaces the previous context's pointers by executing FILD on a kernel
fpstate.  This exposes the kernel instruction and operand addresses
instead.  On TCG qemu64, an unprivileged task can recover the kernel
text slide with FNSTENV at ELF entry, even though FXSAVE64 reports zero
error pointers.

Use FNINIT to clear the pointers without recording new addresses.  It
also clears pending exceptions without waiting for them, and the
following restore reloads the intended x87 state.

Apply the workaround in the FXRSTOR and FP-capable XRSTOR helpers,
including the safe and user-sigframe variants.  This also covers the
init-state and direct sigreturn paths, which must not retain pointers
from kernel FPU use or another task.  Retain preparation for the
no-FXSR FRSTOR fallback too: it was covered by the old common
workaround, and TCG qemu64 can also leave the legacy pointers unchanged
after that restore.  Only initialize x87 when the restore mask includes
FP, so partial and supervisor-only restores keep the unrequested state
intact.

Fixes: 18bd057b1408 ("[PATCH] i386/x86-64: Fix x87 information leak between processes")
Cc: stable@vger.kernel.org
Assisted-by: Codex:gpt-6-astra
Signed-off-by: Kyle Zeng <kylebot@openai.com>
---
 arch/x86/kernel/fpu/core.c     | 14 --------------
 arch/x86/kernel/fpu/internal.h | 19 +++++++++++++++++++
 arch/x86/kernel/fpu/legacy.h   | 14 ++++++++++++++
 arch/x86/kernel/fpu/xstate.h   |  5 +++++
 4 files changed, 38 insertions(+), 14 deletions(-)

diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c
index d1aeecd57f5e..9113f55b80d8 100644
--- a/arch/x86/kernel/fpu/core.c
+++ b/arch/x86/kernel/fpu/core.c
@@ -161,20 +161,6 @@ void save_fpregs_to_fpstate(struct fpu *fpu)
 
 void restore_fpregs_from_fpstate(struct fpstate *fpstate, u64 mask)
 {
-	/*
-	 * AMD K7/K8 and later CPUs up to Zen don't save/restore
-	 * FDP/FIP/FOP unless an exception is pending. Clear the x87 state
-	 * here by setting it to fixed values.  "m" is a random variable
-	 * that should be in L1.
-	 */
-	if (unlikely(static_cpu_has_bug(X86_BUG_FXSAVE_LEAK))) {
-		asm volatile(
-			"fnclex\n\t"
-			"emms\n\t"
-			"fildl %[addr]"	/* set F?P to defined value */
-			: : [addr] "m" (*fpstate));
-	}
-
 	if (use_xsave()) {
 		/*
 		 * Dynamically enabled features are enabled in XCR0, but
diff --git a/arch/x86/kernel/fpu/internal.h b/arch/x86/kernel/fpu/internal.h
index 975de070c9c9..3b4d178edc31 100644
--- a/arch/x86/kernel/fpu/internal.h
+++ b/arch/x86/kernel/fpu/internal.h
@@ -2,6 +2,9 @@
 #ifndef __X86_KERNEL_FPU_INTERNAL_H
 #define __X86_KERNEL_FPU_INTERNAL_H
 
+#include <asm/cpufeature.h>
+#include <asm/fpu/types.h>
+
 extern struct fpstate init_fpstate;
 
 /* CPU feature check wrappers */
@@ -15,6 +18,22 @@ static __always_inline __pure bool use_fxsr(void)
 	return cpu_feature_enabled(X86_FEATURE_FXSR);
 }
 
+/*
+ * AMD CPUs without XSaveErPtr may leave FDP/FIP/FOP unchanged on restore
+ * when no x87 exception is pending.  Using an x87 load to overwrite them
+ * leaks the kernel instruction and operand addresses through FNSTENV.
+ *
+ * FNINIT clears the pointers without recording any new ones.  Only do this
+ * when the x87 state is about to be replaced; a partial XRSTOR must leave
+ * unrequested components alone.
+ */
+static inline void fpregs_restore_prepare(u64 mask)
+{
+	if (unlikely(static_cpu_has_bug(X86_BUG_FXSAVE_LEAK)) &&
+	    (mask & XFEATURE_MASK_FP))
+		asm volatile("fninit");
+}
+
 #ifdef CONFIG_X86_DEBUG_FPU
 # define WARN_ON_FPU(x) WARN_ON_ONCE(x)
 #else
diff --git a/arch/x86/kernel/fpu/legacy.h b/arch/x86/kernel/fpu/legacy.h
index 098f367bb8a7..fe825c9a00c7 100644
--- a/arch/x86/kernel/fpu/legacy.h
+++ b/arch/x86/kernel/fpu/legacy.h
@@ -4,6 +4,8 @@
 
 #include <asm/fpu/types.h>
 
+#include "internal.h"
+
 extern unsigned int mxcsr_feature_mask;
 
 static inline void ldmxcsr(u32 mxcsr)
@@ -63,6 +65,8 @@ static inline int fxsave_to_user_sigframe(struct fxregs_state __user *fx)
 
 static inline void fxrstor(struct fxregs_state *fx)
 {
+	fpregs_restore_prepare(XFEATURE_MASK_FP);
+
 	if (IS_ENABLED(CONFIG_X86_32))
 		kernel_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx));
 	else
@@ -71,6 +75,8 @@ static inline void fxrstor(struct fxregs_state *fx)
 
 static inline int fxrstor_safe(struct fxregs_state *fx)
 {
+	fpregs_restore_prepare(XFEATURE_MASK_FP);
+
 	if (IS_ENABLED(CONFIG_X86_32))
 		return kernel_insn_err(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx));
 	else
@@ -79,6 +85,8 @@ static inline int fxrstor_safe(struct fxregs_state *fx)
 
 static inline int fxrstor_from_user_sigframe(struct fxregs_state __user *fx)
 {
+	fpregs_restore_prepare(XFEATURE_MASK_FP);
+
 	if (IS_ENABLED(CONFIG_X86_32))
 		return user_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx));
 	else
@@ -87,16 +95,22 @@ static inline int fxrstor_from_user_sigframe(struct fxregs_state __user *fx)
 
 static inline void frstor(struct fregs_state *fx)
 {
+	fpregs_restore_prepare(XFEATURE_MASK_FP);
+
 	kernel_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx));
 }
 
 static inline int frstor_safe(struct fregs_state *fx)
 {
+	fpregs_restore_prepare(XFEATURE_MASK_FP);
+
 	return kernel_insn_err(frstor %[fx], "=m" (*fx), [fx] "m" (*fx));
 }
 
 static inline int frstor_from_user_sigframe(struct fregs_state __user *fx)
 {
+	fpregs_restore_prepare(XFEATURE_MASK_FP);
+
 	return user_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx));
 }
 
diff --git a/arch/x86/kernel/fpu/xstate.h b/arch/x86/kernel/fpu/xstate.h
index 38a2862f09d3..ac378d839534 100644
--- a/arch/x86/kernel/fpu/xstate.h
+++ b/arch/x86/kernel/fpu/xstate.h
@@ -7,6 +7,8 @@
 #include <asm/fpu/xcr.h>
 #include <asm/msr.h>
 
+#include "internal.h"
+
 #ifdef CONFIG_X86_64
 DECLARE_PER_CPU(u64, xfd_state);
 #endif
@@ -240,6 +242,7 @@ static inline void os_xrstor(struct fpstate *fpstate, u64 mask)
 	u32 hmask = mask >> 32;
 
 	xfd_validate_state(fpstate, mask, true);
+	fpregs_restore_prepare(mask);
 	XSTATE_XRESTORE(&fpstate->regs.xsave, lmask, hmask);
 }
 
@@ -334,6 +337,7 @@ static inline int xrstor_from_user_sigframe(struct xregs_state __user *buf, u64
 	int err;
 
 	xfd_validate_state(x86_task_fpu(current)->fpstate, mask, true);
+	fpregs_restore_prepare(mask);
 
 	stac();
 	XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);
@@ -355,6 +359,7 @@ static inline int os_xrstor_safe(struct fpstate *fpstate, u64 mask)
 
 	/* Ensure that XFD is up to date */
 	xfd_update_state(fpstate);
+	fpregs_restore_prepare(mask);
 
 	if (cpu_feature_enabled(X86_FEATURE_XSAVES))
 		XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);
-- 
2.53.0