[PATCH 04/13] x86/coco/sev: fix zeropad #VC on unaccepted memory

[Kevin Loughlin <kevinloughlin@google.com>]

When dynamic memory unacceptance is active on AMD SEV-SNP, an accepted
4KB page may sit immediately adjacent to a 2MB block that has been
unaccepted via PVALIDATE(validate=false). Speculative unaligned loads
across the page boundary via load_unaligned_zeropad() (such as in
dentry path walk) trigger a hardware #VC exception with error code
SVM_EXIT_PAGE_NOT_VALIDATED (0x404) or SVM_EXIT_NPF (0x400) on the
unvalidated neighbor page.

Define SVM_EXIT_PAGE_NOT_VALIDATED (0x404) and invoke fixup_exception()
in vc_raw_handle_exception() when an SVM_EXIT_PAGE_NOT_VALIDATED or
SVM_EXIT_NPF #VC exception occurs in kernel mode at an EX_TYPE_ZEROPAD
instruction before dispatching to vc_handle_exitcode(), matching the
existing EX_TYPE_ZEROPAD handling in the Intel TDX #VE handler
(ve_raise_fault()).

Signed-off-by: Kevin Loughlin <kevinloughlin@google.com>
---
 arch/x86/coco/sev/vc-handle.c   | 29 +++++++++++++++++++++++++++++
 arch/x86/include/uapi/asm/svm.h |  2 ++
 2 files changed, 31 insertions(+)

diff --git a/arch/x86/coco/sev/vc-handle.c b/arch/x86/coco/sev/vc-handle.c
index f25d302efceb..1ed00771d87d 100644
--- a/arch/x86/coco/sev/vc-handle.c
+++ b/arch/x86/coco/sev/vc-handle.c
@@ -18,6 +18,7 @@
 #include <linux/io.h>
 #include <linux/psp-sev.h>
 #include <linux/efi.h>
+#include <linux/extable.h>
 #include <uapi/linux/sev-guest.h>
 
 #include <asm/init.h>
@@ -872,6 +873,29 @@ static __always_inline bool vc_from_invalid_context(struct pt_regs *regs)
 	return is_vc2_stack(sp) && !is_vc2_stack(prev_sp);
 }
 
+static bool vc_try_fixup_zeropad(struct es_em_ctxt *ctxt, struct pt_regs *regs,
+				 unsigned long error_code)
+{
+	unsigned long addr, fault_addr;
+
+	if ((error_code != SVM_EXIT_PAGE_NOT_VALIDATED &&
+	     error_code != SVM_EXIT_NPF) ||
+	    user_mode(regs) ||
+	    ex_get_fixup_type(regs->ip) != EX_TYPE_ZEROPAD)
+		return false;
+
+	addr = (unsigned long)insn_get_addr_ref(&ctxt->insn, regs);
+	if (addr == ~0ul)
+		return false;
+
+	/*
+	 * load_unaligned_zeropad() faults when an unaligned word load crosses
+	 * into an unaccepted page; round up to the crossed page boundary.
+	 */
+	fault_addr = (addr + sizeof(long) - 1) & PAGE_MASK;
+	return fixup_exception(regs, X86_TRAP_VC, error_code, fault_addr);
+}
+
 static bool vc_raw_handle_exception(struct pt_regs *regs, unsigned long error_code)
 {
 	struct ghcb_state state;
@@ -885,6 +909,11 @@ static bool vc_raw_handle_exception(struct pt_regs *regs, unsigned long error_co
 	vc_ghcb_invalidate(ghcb);
 	result = vc_init_em_ctxt(&ctxt, regs, error_code);
 
+	if (result == ES_OK && vc_try_fixup_zeropad(&ctxt, regs, error_code)) {
+		__sev_put_ghcb(&state);
+		return true;
+	}
+
 	if (result == ES_OK)
 		result = vc_handle_exitcode(&ctxt, ghcb, error_code);
 
diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h
index e80676185092..38b8e0c2f5e2 100644
--- a/arch/x86/include/uapi/asm/svm.h
+++ b/arch/x86/include/uapi/asm/svm.h
@@ -101,6 +101,7 @@
 #define SVM_EXIT_AVIC_INCOMPLETE_IPI		0x401
 #define SVM_EXIT_AVIC_UNACCELERATED_ACCESS	0x402
 #define SVM_EXIT_VMGEXIT       0x403
+#define SVM_EXIT_PAGE_NOT_VALIDATED	0x404
 #define SVM_EXIT_PML_FULL	0x407
 
 /* SEV-ES software-defined VMGEXIT events */
@@ -237,6 +238,7 @@
 	{ SVM_EXIT_AVIC_INCOMPLETE_IPI,		"avic_incomplete_ipi" }, \
 	{ SVM_EXIT_AVIC_UNACCELERATED_ACCESS,   "avic_unaccelerated_access" }, \
 	{ SVM_EXIT_VMGEXIT,		"vmgexit" }, \
+	{ SVM_EXIT_PAGE_NOT_VALIDATED,	"page_not_validated" }, \
 	{ SVM_EXIT_PML_FULL,		"pml_full" }, \
 	{ SVM_VMGEXIT_MMIO_READ,	"vmgexit_mmio_read" }, \
 	{ SVM_VMGEXIT_MMIO_WRITE,	"vmgexit_mmio_write" }, \
-- 
2.56.0.307.g5fe7bd680c-goog