[PATCH v2 17/20] KVM: nSVM: Cache L1's IO permissions map pages

[David Woodhouse <dwmw2@infradead.org>]

From: David Woodhouse <dwmw@amazon.co.uk>

The per-intercept IOPM check on every trapped L2 IO instruction reads
L1's IO permissions map with kvm_vcpu_read_guest() — a
memremap/memunmap cycle per access with unmanaged guest memory. Cache
the three IOPM pages in gfn_to_pfn_caches keyed on iopm_base_pa, as
for the MSR permissions map.

Unlike the MSRPM accesses, an IOPM read is not naturally aligned: a
two-byte read for a port range straddling a page boundary (e.g. ports
0x7ff8-0x7fff at bytes 0xfff-0x1000) spans two pages, so read byte by
byte, each from its own page's cache.

Failure semantics are preserved: an unreadable IOPM reflects the
intercept to L1 (NESTED_EXIT_DONE).

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
 arch/x86/kvm/svm/nested.c | 71 +++++++++++++++++++++++++++++----------
 arch/x86/kvm/svm/svm.c    |  2 ++
 arch/x86/kvm/svm/svm.h    |  7 ++++
 3 files changed, 63 insertions(+), 17 deletions(-)

diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index 4e22cd7a38e1..a682b7446c3e 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -1519,10 +1519,36 @@ static void nested_svm_triple_fault(struct kvm_vcpu *vcpu)
 	nested_svm_simple_vmexit(to_svm(vcpu), SVM_EXIT_SHUTDOWN);
 }
 
+/*
+ * Initialize the nested gfn_to_pfn_caches, once per vCPU. Unlike the
+ * allocations in svm_allocate_nested(), the caches persist across
+ * EFER.SVME toggles: svm_free_nested() only deactivates them, and
+ * reuse goes through kvm_gpc_activate() without re-init.
+ */
+void svm_init_nested_caches(struct vcpu_svm *svm)
+{
+	int i;
+
+	kvm_gpc_init(&svm->nested.vmcb12_cache, svm->vcpu.kvm);
+
+	/*
+	 * KVM only ever reads the MSR and IO permissions maps, which L1 may
+	 * keep in memory that its VMM maps read-only: map the caches
+	 * read-only (which also means never marking them dirty), matching
+	 * the read semantics of the kvm_vcpu_read_guest() calls they
+	 * replace.
+	 */
+	for (i = 0; i < ARRAY_SIZE(svm->nested.msrpm12_cache); i++)
+		__kvm_gpc_init(&svm->nested.msrpm12_cache[i], svm->vcpu.kvm,
+			       false, true, NULL, 0);
+	for (i = 0; i < ARRAY_SIZE(svm->nested.iopm12_cache); i++)
+		__kvm_gpc_init(&svm->nested.iopm12_cache[i], svm->vcpu.kvm,
+			       false, true, NULL, 0);
+}
+
 int svm_allocate_nested(struct vcpu_svm *svm)
 {
 	struct page *vmcb02_page;
-	int i;
 
 	if (svm->nested.initialized)
 		return 0;
@@ -1539,18 +1565,6 @@ int svm_allocate_nested(struct vcpu_svm *svm)
 
 	svm->nested.vmcb02.cpu = -1;
 
-	kvm_gpc_init(&svm->nested.vmcb12_cache, svm->vcpu.kvm);
-
-	/*
-	 * KVM only ever reads the MSR permissions map, which L1 may keep in
-	 * memory that its VMM maps read-only: map the cache read-only
-	 * (which also means never marking it dirty), matching the read
-	 * semantics of the kvm_vcpu_read_guest() calls it replaces.
-	 */
-	for (i = 0; i < ARRAY_SIZE(svm->nested.msrpm12_cache); i++)
-		__kvm_gpc_init(&svm->nested.msrpm12_cache[i], svm->vcpu.kvm,
-			       false, true, NULL, 0);
-
 	svm->nested.initialized = true;
 	return 0;
 
@@ -1575,6 +1589,8 @@ void svm_free_nested(struct vcpu_svm *svm)
 	kvm_gpc_deactivate(&svm->nested.vmcb12_cache);
 	for (i = 0; i < ARRAY_SIZE(svm->nested.msrpm12_cache); i++)
 		kvm_gpc_deactivate(&svm->nested.msrpm12_cache[i]);
+	for (i = 0; i < ARRAY_SIZE(svm->nested.iopm12_cache); i++)
+		kvm_gpc_deactivate(&svm->nested.iopm12_cache[i]);
 
 	__free_page(virt_to_page(svm->nested.vmcb02.ptr));
 	svm->nested.vmcb02.ptr = NULL;
@@ -1654,7 +1670,7 @@ static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
 	unsigned port, size, iopm_len;
 	u16 val, mask;
 	u8 start_bit;
-	u64 gpa;
+	int i;
 
 	if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_IOIO_PROT)))
 		return NESTED_EXIT_HOST;
@@ -1662,14 +1678,35 @@ static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
 	port = svm->vmcb->control.exit_info_1 >> 16;
 	size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
 		SVM_IOIO_SIZE_SHIFT;
-	gpa  = svm->nested.ctl.iopm_base_pa + (port / 8);
 	start_bit = port % 8;
 	iopm_len = (start_bit + size > 8) ? 2 : 1;
 	mask = (0xf >> (4 - size)) << start_bit;
 	val = 0;
 
-	if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
-		return NESTED_EXIT_DONE;
+	/*
+	 * Read byte by byte: a two-byte read at the end of a page (e.g.
+	 * ports 0x7ff8-0x7fff spanning bytes 0xfff-0x1000) crosses into
+	 * the next page of the IOPM, i.e. the next cache.
+	 */
+	for (i = 0; i < iopm_len; i++) {
+		unsigned long offset = port / 8 + i;
+		struct gfn_to_pfn_cache *gpc;
+		gpa_t gpa;
+		int idx;
+
+		if (WARN_ON_ONCE(offset >= IOPM_SIZE))
+			return NESTED_EXIT_DONE;
+
+		gpc = &svm->nested.iopm12_cache[offset >> PAGE_SHIFT];
+		gpa = svm->nested.ctl.iopm_base_pa + (offset & PAGE_MASK);
+
+		idx = kvm_gpc_lock_page(gpc, gpa);
+		if (idx < 0)
+			return NESTED_EXIT_DONE;
+
+		val |= *(u8 *)(gpc->khva + offset_in_page(offset)) << (i * 8);
+		kvm_gpc_unlock(gpc, idx);
+	}
 
 	return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
 }
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index b78727cca9b1..f601e0650c36 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -1382,6 +1382,8 @@ static int svm_vcpu_create(struct kvm_vcpu *vcpu)
 	svm->vmcb01.pa = __sme_set(page_to_pfn(vmcb01_page) << PAGE_SHIFT);
 	svm_switch_vmcb(svm, &svm->vmcb01);
 
+	svm_init_nested_caches(svm);
+
 	svm->guest_state_loaded = false;
 
 	return 0;
diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h
index 8a5dff5a9ee3..256eefb27a83 100644
--- a/arch/x86/kvm/svm/svm.h
+++ b/arch/x86/kvm/svm/svm.h
@@ -229,6 +229,12 @@ struct svm_nested_state {
 	 */
 	struct gfn_to_pfn_cache msrpm12_cache[MSRPM_SIZE / PAGE_SIZE];
 
+	/*
+	 * Cached mappings of the three pages of L1's IO permissions map,
+	 * keyed on ctl.iopm_base_pa, for the per-intercept checks.
+	 */
+	struct gfn_to_pfn_cache iopm12_cache[IOPM_SIZE / PAGE_SIZE];
+
 	/*
 	 * The MSR permissions map used for vmcb02, which is the merge result
 	 * of vmcb01 and vmcb12
@@ -904,6 +910,7 @@ int __init nested_svm_init_msrpm_merge_offsets(void);
 
 int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb_gpa, bool from_vmrun);
 void svm_leave_nested(struct kvm_vcpu *vcpu);
+void svm_init_nested_caches(struct vcpu_svm *svm);
 void svm_free_nested(struct vcpu_svm *svm);
 int svm_allocate_nested(struct vcpu_svm *svm);
 int nested_svm_vmrun(struct kvm_vcpu *vcpu);
-- 
2.43.0