[RFC PATCH kernel 17/17] x86/sev: Flush IOMMU TLB for trusted devices

[Alexey Kardashevskiy <aik@amd.com>]

IOMMU performs RMP checks when SNP is enabled, the results are
cached along with the IOMMU translations. When a VM lowers permission
of a mapped page (moves to a lower VMPL level or from read+write to
read-only or private to shared), the cached RMP check results require
invalidation.

At the moment the only way to invalidate IOMMU cache is the RMPUPDATE
instruction which flushes all IOMMU TLBs. It is a host privileged
instruction so a VM needs a way to ensure the host has done it.
Note that the guest's RMPADJUST/PVALIDATE do not flush IOMMU TLBs.

The host implements a new "IOMMU TLB Flush" VMGEXIT code which is
advertised via bit#11 in the GHCB Hypervisor capabilities.

Use RMPUPDATE in the following way:
- allocate a page per VCPU (to allow lockless flushing);
- When invalidation is needed, copy two patterns (A and B) to the page;
- invalidate the page so the host can make it shared;
- use new GHCB call to request RMPUPDATE on the host;
- the host makes the page shared;
- the host clears pattern A;
- the host makes the page private again;
- the host returns to the guest;
- check if pattern A has changed and pattern B has not;
- if the above failed, panic().

The patterns are located far enough to not hit the same cache line to
work with the cipher text hiding feature.

The host can choose to not execute the request, WARN_ON if this
is the case. Further patches will attempt to handle this in other way.

Signed-off-by: Alexey Kardashevskiy <aik@amd.com>
---
 arch/x86/include/asm/sev-common.h |  2 +
 arch/x86/include/uapi/asm/svm.h   |  3 +
 arch/x86/coco/sev/core.c          | 92 ++++++++++++++++++++
 3 files changed, 97 insertions(+)

diff --git a/arch/x86/include/asm/sev-common.h b/arch/x86/include/asm/sev-common.h
index ff763c3c5d63..51abf8d061fa 100644
--- a/arch/x86/include/asm/sev-common.h
+++ b/arch/x86/include/asm/sev-common.h
@@ -138,6 +138,7 @@ enum psc_op {
 #define GHCB_HV_FT_SNP_AP_CREATION	BIT_ULL(1)
 #define GHCB_HV_FT_SNP_MULTI_VMPL	BIT_ULL(5)
 #define GHCB_HV_FT_SNP_SEV_TIO		BIT_ULL(7)
+#define GHCB_HV_FT_SNP_IOMMU_TLB_FLUSH	BIT_ULL(11)
 
 /*
  * SNP Page State Change NAE event
@@ -210,6 +211,7 @@ struct snp_psc_desc {
 #define GHCB_TERM_SECURE_TSC		10	/* Secure TSC initialization failed */
 #define GHCB_TERM_SVSM_CA_REMAP_FAIL	11	/* SVSM is present but CA could not be remapped */
 #define GHCB_TERM_SAVIC_FAIL		12	/* Secure AVIC-specific failure */
+#define GHCB_TERM_IOMMUTLB_FLUSH	13	/* IOMMUTLB flush failed for SEV-TIO device */
 
 #define GHCB_RESP_CODE(v)		((v) & GHCB_MSR_INFO_MASK)
 
diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h
index 93597ad492bf..269050942c8e 100644
--- a/arch/x86/include/uapi/asm/svm.h
+++ b/arch/x86/include/uapi/asm/svm.h
@@ -160,6 +160,8 @@
 #define SVM_VMGEXIT_SEV_TIO_OP_UNBIND	1
 #define SVM_VMGEXIT_SEV_TIO_OP_RUN	2
 #define SVM_VMGEXIT_SEV_TIO_OP_STOP	3
+#define SVM_VMGEXIT_IOMMU_TLB_FLUSH		0x80000022ull
+#define SVM_VMGEXIT_IOMMU_TLB_FLUSH_NO_ACTION	1
 #define SVM_VMGEXIT_HV_FEATURES			0x8000fffdull
 #define SVM_VMGEXIT_TERM_REQUEST		0x8000fffeull
 #define SVM_VMGEXIT_TERM_REASON(reason_set, reason_code)	\
@@ -285,6 +287,7 @@
 	{ SVM_VMGEXIT_AP_CREATION,	"vmgexit_ap_creation" }, \
 	{ SVM_VMGEXIT_SEV_TIO_GR,	"vmgexit_sev_tio_guest_request" }, \
 	{ SVM_VMGEXIT_SEV_TIO_OP,	"vmgexit_sev_tio_op" }, \
+	{ SVM_VMGEXIT_IOMMU_TLB_FLUSH, "vmgexit_sev_tio_iommu_tlb_flush" }, \
 	{ SVM_VMGEXIT_HV_FEATURES,	"vmgexit_hypervisor_feature" }, \
 	{ SVM_EXIT_ERR,         "invalid_guest_state" }
 
diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c
index ed0e4546d5e5..aa5a3abb4796 100644
--- a/arch/x86/coco/sev/core.c
+++ b/arch/x86/coco/sev/core.c
@@ -44,6 +44,7 @@
 #include <asm/cpuid/api.h>
 #include <asm/cmdline.h>
 #include <asm/msr.h>
+#include <asm/archrandom.h>
 
 #include "internal.h"
 
@@ -103,6 +104,36 @@ static unsigned long snp_tsc_freq_khz __ro_after_init;
 
 DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data);
 DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
+DEFINE_PER_CPU(u8 *, iommu_tlb_flush_ghcb_page);
+static atomic_t sev_tio_devices_num;
+
+static int alloc_iommu_tlb_flush_ghcb_pages(void)
+{
+	unsigned int cpu;
+	struct page *pg;
+	void *p;
+
+	/*
+	 * Allocate per CPU pages while encrypted DMA is not happening yet
+	 * and smashing is cheap.
+	 */
+	for_each_possible_cpu(cpu) {
+		if (per_cpu(iommu_tlb_flush_ghcb_page, cpu))
+			continue;
+
+		pg = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL, 0);
+		if (!pg)
+			return -ENOMEM;
+
+		p = page_to_virt(pg);
+		/* Trigger psmash in the host os now to avoid psmash race later */
+		snp_set_memory_shared((unsigned long)p, 1);
+		snp_set_memory_private((unsigned long)p, 1);
+		per_cpu(iommu_tlb_flush_ghcb_page, cpu) = p;
+	}
+
+	return 0;
+}
 
 int sev_tio_op(u32 guest_rid, unsigned int op, u64 *fw_err, u64 *tdi_id)
 {
@@ -111,6 +142,24 @@ int sev_tio_op(u32 guest_rid, unsigned int op, u64 *fw_err, u64 *tdi_id)
 	struct ghcb *ghcb;
 	int ret;
 
+	if (!(sev_hv_features & GHCB_HV_FT_SNP_SEV_TIO))
+		return -EPERM;
+
+	if (op == SVM_VMGEXIT_SEV_TIO_OP_RUN || op == SVM_VMGEXIT_SEV_TIO_OP_STOP) {
+		if (!(sev_hv_features & GHCB_HV_FT_SNP_IOMMU_TLB_FLUSH))
+			return -EPERM;
+
+		if (op == SVM_VMGEXIT_SEV_TIO_OP_RUN) {
+			if (atomic_inc_return(&sev_tio_devices_num) == 1) {
+				ret = alloc_iommu_tlb_flush_ghcb_pages();
+				if (ret)
+					return ret;
+			}
+		} else if (atomic_dec_return(&sev_tio_devices_num) == 0) {
+			/* Do cleanup or leave it like this? */
+		}
+	}
+
 	/* __sev_get_ghcb() needs IRQs disabled because it uses per-CPU GHCB. */
 	guard(irqsave)();
 
@@ -347,6 +396,42 @@ static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc)
 	return ret;
 }
 
+static int ghcb_flush_iommu_tlb(struct ghcb *ghcb)
+{
+	/* AES encrypts with 16 byte blocks */
+	unsigned long s1[BITS_TO_LONGS(128)], s2[BITS_TO_LONGS(128)];
+	void *p = this_cpu_read(iommu_tlb_flush_ghcb_page), *p2;
+	struct es_em_ctxt ctxt;
+	int ret;
+
+	if (!p)
+		return -ENOMEM;
+
+	/* Keep patterns apart far enough to not share the same cache line */
+	p2 = (u8 *) p + 2048;
+
+	vc_ghcb_invalidate(ghcb);
+
+	BUILD_BUG_ON(ARRAY_SIZE(s1) != 2);
+	if (!rdrand_long(s1) || !rdrand_long(s1 + 1) ||
+	    !rdrand_long(s2) || !rdrand_long(s2 + 1))
+		return -EFAULT;
+
+	memcpy(p, s1, sizeof(s1));
+	memcpy(p2, s2, sizeof(s2));
+
+	pvalidate((unsigned long) p, RMP_PG_SIZE_4K, false);
+	ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_IOMMU_TLB_FLUSH, __pa(p), 0);
+	pvalidate((unsigned long) p, RMP_PG_SIZE_4K, true);
+
+	/* Ensure that the host change is visible */
+	smp_mb();
+
+	if (!memcmp(p, s1, sizeof(s1)) || memcmp(p2, s2, sizeof(s2)))
+		return -EFAULT;
+
+	return 0;
+}
 static unsigned long __set_pages_state(struct snp_psc_desc *data, unsigned long vaddr,
 				       unsigned long vaddr_end, int op)
 {
@@ -404,6 +489,13 @@ static unsigned long __set_pages_state(struct snp_psc_desc *data, unsigned long
 	if (!ghcb || vmgexit_psc(ghcb, data))
 		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
 
+	if (atomic_read(&sev_tio_devices_num)) {
+		int ret = ghcb_flush_iommu_tlb(ghcb);
+
+		if (ret)
+			sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_IOMMUTLB_FLUSH);
+	}
+
 	__sev_put_ghcb(&state);
 
 	local_irq_restore(flags);
-- 
2.55.0