[PATCH v2 15/20] KVM: nSVM: Use a gfn_to_pfn_cache for the vmcb12 page

[David Woodhouse <dwmw2@infradead.org>]

From: David Woodhouse <dwmw@amazon.co.uk>

Replace the transient map/unmap of the vmcb12 page on every nested
VMRUN (control/save copy-in) and nested VM-exit (state copy-back) with
a gfn_to_pfn_cache keyed on vmcb12_gpa. With unmanaged guest memory
the transient map is a memremap/memunmap cycle on every L1<->L2
transition; the cache persists the kernel mapping for as long as the
gPA and its translation are unchanged. An L1 which multiplexes
several L2s on one vCPU re-keys the cache on each switch, paying a
refresh; the common single-L2 case pays nothing.

No pinning is needed, unlike the nVMX APIC pages: SVM never hands an
L1-owned physical address to the CPU in vmcb02 (the merged MSRPM and
the IOPM are kernel-owned pages), so the vmcb12 page is only ever
accessed by KVM itself, under the SRCU read lock, with the usual
check/refresh protocol.

Dirty marking moves from unconditional-at-unmap to the actual write
paths: the SVM_EXIT_ERR reflection in the VMRUN consistency-check
failure path, and the copy-back at nested VM-exit.

The SMM enter/leave and VMLOAD/VMSAVE emulation paths keep the
transient map: they are cold, and VMLOAD/VMSAVE takes an arbitrary
gPA from RAX which would thrash a single-slot cache.
svm_leave_smm() now validates the vmcb12 GPA it reads from SMRAM, as
VMRUN does for RAX; the copy-back through the cache would otherwise
fail on an unaligned address at the eventual nested VM-exit. The
refused RSM still ends in TRIPLE_FAULT, but at the corrupt-SMRAM
instruction itself, before any L2 state is committed, as for the
other SMRAM consistency checks here.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
 arch/x86/kvm/svm/nested.c | 27 ++++++++++++++++++++-------
 arch/x86/kvm/svm/svm.c    | 15 +++++++++++++++
 arch/x86/kvm/svm/svm.h    |  3 +++
 3 files changed, 38 insertions(+), 7 deletions(-)

diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index ab187abc11b5..32854c8b2bbc 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -1098,12 +1098,13 @@ static int nested_svm_copy_vmcb12_to_cache(struct kvm_vcpu *vcpu, u64 vmcb12_gpa
 	struct vcpu_svm *svm = to_svm(vcpu);
 	struct vmcb *vmcb12;
 	int r = 0;
+	int idx;
 
-	CLASS(kvm_vcpu_map_local, m)(vcpu, gpa_to_gfn(vmcb12_gpa));
-	if (m.ret)
+	idx = kvm_gpc_lock_page(&svm->nested.vmcb12_cache, vmcb12_gpa);
+	if (idx < 0)
 		return -EFAULT;
 
-	vmcb12 = m.map.hva;
+	vmcb12 = svm->nested.vmcb12_cache.khva;
 	nested_copy_vmcb_control_to_cache(svm, &vmcb12->control);
 	nested_copy_vmcb_save_to_cache(svm, &vmcb12->save);
 
@@ -1114,9 +1115,11 @@ static int nested_svm_copy_vmcb12_to_cache(struct kvm_vcpu *vcpu, u64 vmcb12_gpa
 		vmcb12->control.event_inj = 0;
 		vmcb12->control.event_inj_err = 0;
 		svm_set_gif(svm, false);
+		kvm_gpc_mark_dirty_in_slot(&svm->nested.vmcb12_cache);
 		r = -EINVAL;
 	}
 
+	kvm_gpc_unlock(&svm->nested.vmcb12_cache, idx);
 	return r;
 }
 
@@ -1261,12 +1264,14 @@ static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu)
 	struct vcpu_svm *svm = to_svm(vcpu);
 	struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
 	struct vmcb *vmcb12;
+	int idx;
 
-	CLASS(kvm_vcpu_map_local, m)(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa));
-	if (m.ret)
-		return m.ret;
+	idx = kvm_gpc_lock_page(&svm->nested.vmcb12_cache,
+				  svm->nested.vmcb12_gpa);
+	if (idx < 0)
+		return idx;
 
-	vmcb12 = m.map.hva;
+	vmcb12 = svm->nested.vmcb12_cache.khva;
 
 	vmcb12->save.es     = vmcb02->save.es;
 	vmcb12->save.cs     = vmcb02->save.cs;
@@ -1321,6 +1326,9 @@ static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu)
 				       vmcb12->control.exit_int_info_err,
 				       KVM_ISA_SVM);
 
+	kvm_gpc_mark_dirty_in_slot(&svm->nested.vmcb12_cache);
+	kvm_gpc_unlock(&svm->nested.vmcb12_cache, idx);
+
 	return 0;
 }
 
@@ -1506,6 +1514,9 @@ int svm_allocate_nested(struct vcpu_svm *svm)
 		goto err_free_vmcb02;
 
 	svm->nested.vmcb02.cpu = -1;
+
+	kvm_gpc_init(&svm->nested.vmcb12_cache, svm->vcpu.kvm);
+
 	svm->nested.initialized = true;
 	return 0;
 
@@ -1525,6 +1536,8 @@ void svm_free_nested(struct vcpu_svm *svm)
 	svm_vcpu_free_msrpm(svm->nested.msrpm);
 	svm->nested.msrpm = NULL;
 
+	kvm_gpc_deactivate(&svm->nested.vmcb12_cache);
+
 	__free_page(virt_to_page(svm->nested.vmcb02.ptr));
 	svm->nested.vmcb02.ptr = NULL;
 
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index 136069dfcd44..b78727cca9b1 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -5206,6 +5206,21 @@ static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram)
 	if (!(smram64->efer & EFER_SVME))
 		return 1;
 
+	/*
+	 * The vmcb12 GPA read back from SMRAM feeds the vmcb12 cache at
+	 * the eventual nested VM-exit, whose copy-back would fail on an
+	 * unaligned or illegal address. Refusing the RSM here ends in
+	 * TRIPLE_FAULT just the same (em_rsm() shoots a refused RSM),
+	 * but it fails before any L2 state has been committed, at the
+	 * instruction whose SMRAM contents are corrupt, matching how
+	 * this function already treats other inconsistent SMRAM state —
+	 * rather than at an unrelated nested VM-exit later. VMRUN
+	 * (#GP) and KVM_SET_NESTED_STATE (-EINVAL) police the same
+	 * field on their own entry points.
+	 */
+	if (!page_address_valid(vcpu, smram64->svm_guest_vmcb_gpa))
+		return 1;
+
 	CLASS(kvm_vcpu_map_local, m)(vcpu, gpa_to_gfn(smram64->svm_guest_vmcb_gpa));
 	if (m.ret)
 		return 1;
diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h
index 1b8f3f2057d9..fe964a961305 100644
--- a/arch/x86/kvm/svm/svm.h
+++ b/arch/x86/kvm/svm/svm.h
@@ -219,6 +219,9 @@ struct svm_nested_state {
 	u64 last_vmcb12_gpa;
 	u64 last_bus_lock_rip;
 
+	/* Cached mapping of the vmcb12 page, keyed on vmcb12_gpa */
+	struct gfn_to_pfn_cache vmcb12_cache;
+
 	/*
 	 * The MSR permissions map used for vmcb02, which is the merge result
 	 * of vmcb01 and vmcb12
-- 
2.43.0