[PATCH v5 16/24] iommu/amd: Add IOMMUFD vDevice and DevID mapping

[Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>]

IOMMUFD vDevice allocation supplies the guest device ID (virt_id).
The AMD vIOMMU Device ID mapping table translates that gdev_id to the
host device ID for virtualised IOMMU commands.

Add vdevice_init, which validates a PCI device and virt_id, then
programs the mapping through VFCTRL via amd_viommu_set_device_mapping().
Get the host struct device via iommufd_vdevice_to_device(). Set
vdev->destroy to restore an idle mapping when the vDevice is removed
while the vIOMMU is still alive.

Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
---
 drivers/iommu/amd/amd_iommu_types.h | 12 +++++
 drivers/iommu/amd/amd_viommu.h      | 10 ++++
 drivers/iommu/amd/iommufd.c         | 52 ++++++++++++++++++++
 drivers/iommu/amd/viommu.c          | 73 ++++++++++++++++++++++++++++-
 4 files changed, 146 insertions(+), 1 deletion(-)

diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index fc8e1378c7ed..07eaeeb72e38 100644
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -547,6 +547,14 @@ struct amd_iommu_viommu {
 	u64 *devid_table;
 	u64 *domid_table;
 
+	/*
+	 * Serializes VFCTRL Guest DID map CONTROL0/CONTROL1 doorbells
+	 * (DevID and DomID table updates) for this gid. Those registers
+	 * are shared by every vDevice on this vIOMMU; igroup locks are
+	 * not. Do not nest with trans_devid_lock.
+	 */
+	struct mutex vfctrl_lock;
+
 	/* Offset for mmap() of guest VF MMIO; set after iommufd_viommu_alloc_mmap(). */
 	unsigned long vfmmio_mmap_offset;
 };
@@ -1113,6 +1121,10 @@ struct amd_irte_ops {
 	void (*clear_allocated)(struct irq_remap_table *, int);
 };
 
+struct amd_iommu_vdevice {
+	struct iommufd_vdevice core;
+};
+
 #ifdef CONFIG_IRQ_REMAP
 extern struct amd_irte_ops irte_32_ops;
 extern struct amd_irte_ops irte_128_ops;
diff --git a/drivers/iommu/amd/amd_viommu.h b/drivers/iommu/amd/amd_viommu.h
index 580cad0f9f5a..1022ae20c170 100644
--- a/drivers/iommu/amd/amd_viommu.h
+++ b/drivers/iommu/amd/amd_viommu.h
@@ -8,6 +8,9 @@
 
 struct amd_iommu;
 
+/* 16-bit guest DevID index; not VIOMMU_MAX_GID. */
+#define VIOMMU_MAX_GDEVID	0xFFFF
+
 #if IS_ENABLED(CONFIG_AMD_IOMMU_IOMMUFD)
 
 int amd_viommu_init(struct amd_iommu *iommu);
@@ -20,6 +23,8 @@ int amd_viommu_init_one(struct amd_iommu *iommu, struct amd_iommu_viommu *viommu
 
 void amd_viommu_uninit_one(struct amd_iommu *iommu, struct amd_iommu_viommu *viommu);
 
+void amd_viommu_set_device_mapping(struct amd_iommu_viommu *aviommu,
+				   u16 hdev_id, u16 gdev_id);
 #else
 
 /*
@@ -41,6 +46,11 @@ static inline u64 amd_viommu_get_vfmmio_addr(struct amd_iommu *iommu, u16 gid)
 	return 0;
 }
 
+static inline void amd_viommu_set_device_mapping(struct amd_iommu_viommu *aviommu,
+						 u16 hdev_id, u16 gdev_id)
+{
+}
+
 #endif /* CONFIG_AMD_IOMMU_IOMMUFD */
 
 #endif /* AMD_VIOMMU_H */
diff --git a/drivers/iommu/amd/iommufd.c b/drivers/iommu/amd/iommufd.c
index 97de07d3ad10..5760b183ce62 100644
--- a/drivers/iommu/amd/iommufd.c
+++ b/drivers/iommu/amd/iommufd.c
@@ -70,6 +70,7 @@ int amd_iommufd_viommu_init(struct iommufd_viommu *viommu, struct iommu_domain *
 	if (ret)
 		return ret;
 
+	mutex_init(&aviommu->vfctrl_lock);
 	xa_init_flags(&aviommu->gdomid_array, XA_FLAGS_ALLOC1);
 	aviommu->parent = pdom;
 
@@ -121,6 +122,7 @@ int amd_iommufd_viommu_init(struct iommufd_viommu *viommu, struct iommu_domain *
 err_mmap:
 	amd_iommu_gid_free(iommu, aviommu->gid);
 err_gid:
+	mutex_destroy(&aviommu->vfctrl_lock);
 	return ret;
 }
 
@@ -140,6 +142,54 @@ static void amd_iommufd_viommu_destroy(struct iommufd_viommu *viommu)
 	iommufd_viommu_destroy_mmap(&aviommu->core, aviommu->vfmmio_mmap_offset);
 	amd_viommu_uninit_one(iommu, aviommu);
 	amd_iommu_gid_free(iommu, aviommu->gid);
+	mutex_destroy(&aviommu->vfctrl_lock);
+}
+
+/*
+ * Called from drivers/iommu/iommufd/viommu.c: iommufd_vdevice_abort()
+ *
+ * Restore an idle DevID map entry for this virt_id. Every DevID mapping
+ * table slot must have V=1 for both active and non-active devices. A
+ * non-valid entry raises INVALID_DEVICE_REQUEST when the IOMMU processes
+ * INVALIDATE_IOMMU_ALL. Use host device ID 0: that RID is the IOMMU PCI
+ * function and does not normally host an endpoint.
+ */
+static void _amd_viommu_vdevice_destroy(struct iommufd_vdevice *vdev)
+{
+	struct iommufd_viommu *viommu = vdev->viommu;
+	struct amd_iommu_viommu *aviommu =
+		container_of(viommu, struct amd_iommu_viommu, core);
+
+	amd_viommu_set_device_mapping(aviommu, 0, (u16)vdev->virt_id);
+}
+
+/*
+ * Called from drivers/iommu/iommufd/viommu.c: iommufd_vdevice_alloc_ioctl()
+ */
+static int _amd_viommu_vdevice_init(struct iommufd_vdevice *vdev)
+{
+	struct device *dev = iommufd_vdevice_to_device(vdev);
+	struct pci_dev *pdev;
+	struct iommufd_viommu *viommu = vdev->viommu;
+	struct amd_iommu_viommu *aviommu = container_of(viommu, struct amd_iommu_viommu, core);
+
+	if (!dev_is_pci(dev))
+		return -EINVAL;
+	pdev = to_pci_dev(dev);
+
+	if (vdev->virt_id > VIOMMU_MAX_GDEVID) {
+		pr_err("%s: Invalid virtual ID (virt_id=%#llx)\n", __func__, vdev->virt_id);
+		return -EINVAL;
+	}
+
+	pr_debug("%s: gid=%#x, hdev_id=%#x, gdev_id=%#llx\n", __func__,
+		 aviommu->gid, pci_dev_id(pdev), vdev->virt_id);
+
+	amd_viommu_set_device_mapping(aviommu, pci_dev_id(pdev),
+				      (u16)vdev->virt_id);
+	vdev->destroy = _amd_viommu_vdevice_destroy;
+
+	return 0;
 }
 
 /*
@@ -149,4 +199,6 @@ static void amd_iommufd_viommu_destroy(struct iommufd_viommu *viommu)
 static const struct iommufd_viommu_ops amd_viommu_ops = {
 	.alloc_domain_nested = amd_iommu_alloc_domain_nested,
 	.destroy = amd_iommufd_viommu_destroy,
+	.vdevice_size = VDEVICE_STRUCT_SIZE(struct amd_iommu_vdevice, core),
+	.vdevice_init = _amd_viommu_vdevice_init,
 };
diff --git a/drivers/iommu/amd/viommu.c b/drivers/iommu/amd/viommu.c
index 06a21f14b2a5..556864eaaa0d 100644
--- a/drivers/iommu/amd/viommu.c
+++ b/drivers/iommu/amd/viommu.c
@@ -6,6 +6,7 @@
 #define pr_fmt(fmt)     "AMD-Vi: " fmt
 #define dev_fmt(fmt)    pr_fmt(fmt)
 
+#include <linux/delay.h>
 #include <linux/iommu.h>
 #include <linux/iommufd.h>
 #include <uapi/linux/iommufd.h>
@@ -29,7 +30,6 @@
  * VIOMMU_MAX_GDEVID / VIOMMU_MAX_GDOMID are 16-bit guest DevID
  * and DomID table indices (hardware mapping-table width), not GID.
  */
-#define VIOMMU_MAX_GDEVID		0xFFFF
 #define VIOMMU_DEVID_MAPPING_BASE	0x1000000000ULL
 #define VIOMMU_DEVID_MAPPING_ENTRY_SIZE	(1 << 20)
 
@@ -45,6 +45,8 @@ static_assert(VIOMMU_DEVID_MAPPING_ENTRY_SIZE >=
 static_assert(VIOMMU_DOMID_MAPPING_ENTRY_SIZE >=
 	      (VIOMMU_MAX_GDOMID + 1ULL) * sizeof(u64));
 
+#define VIOMMU_VFCTRL_GUEST_DID_MAP_CONTROL0_OFFSET	0x00
+
 static void __init amd_viommu_vf_vfcntl_unmap(struct amd_iommu *iommu)
 {
 	if (iommu->vfctrl_base) {
@@ -439,6 +441,75 @@ static void __maybe_unused free_private_vm_region(struct amd_iommu *iommu, u64 *
 	*entry = NULL;
 }
 
+#define DEVID_ENTRY_GDEVID_MASK		GENMASK_ULL(61, 46)
+#define DEVID_ENTRY_HDEVID_MASK		GENMASK_ULL(29, 14)
+#define DEVID_ENTRY_WRITE		BIT_ULL(63)
+#define DEVID_ENTRY_VALID		BIT_ULL(0)
+
+/*
+ * CONTROL0/CONTROL1 bit 63 (WRITE) is a doorbell: hardware copies one
+ * entry into the IPA-backed table and clears WRITE when that DMA write
+ * retires. This is not a command-buffer operation;
+ * iommu_completion_wait() does not drain it. A readq() of CONTROL_EN
+ * is also unrelated. Poll WRITE=0 before ringing a new doorbell and
+ * after each write so the table update is retired before the caller
+ * proceeds (DTE commit, unmap/free).
+ */
+static void vfctrl_wait_write_clear(struct amd_iommu *iommu, u16 gid,
+				    unsigned int offset)
+{
+	u8 __iomem *reg = VIOMMU_VFCTRL_MMIO_BASE(iommu, gid) + offset;
+	int i;
+
+	for (i = 0; i < LOOP_TIMEOUT; i++) {
+		if (!(readq(reg) & DEVID_ENTRY_WRITE))
+			return;
+		udelay(1);
+	}
+	pr_err("%s: WRITE timeout gid=%#x offset=%#x val=%#llx\n",
+	       __func__, gid, offset, readq(reg));
+}
+
+/*
+ * Guest DevID table update via VFCTRL CONTROL0.
+ */
+static void set_device_mapping_unlocked(struct amd_iommu *iommu, u16 gid,
+					u16 hdev_id, u16 gdev_id)
+{
+	u64 val;
+	u8 __iomem *vfctrl = VIOMMU_VFCTRL_MMIO_BASE(iommu, gid);
+
+	val = FIELD_PREP(DEVID_ENTRY_GDEVID_MASK, gdev_id) |
+	      FIELD_PREP(DEVID_ENTRY_HDEVID_MASK, hdev_id) |
+	      DEVID_ENTRY_WRITE | DEVID_ENTRY_VALID;
+
+	vfctrl_wait_write_clear(iommu, gid,
+			VIOMMU_VFCTRL_GUEST_DID_MAP_CONTROL0_OFFSET);
+	writeq(val, vfctrl + VIOMMU_VFCTRL_GUEST_DID_MAP_CONTROL0_OFFSET);
+	vfctrl_wait_write_clear(iommu, gid,
+			VIOMMU_VFCTRL_GUEST_DID_MAP_CONTROL0_OFFSET);
+}
+
+/*
+ * Program the DevID via VFCTRL CONTROL0. Waits for WRITE to clear so
+ * the IPA table DMA has retired before return. Serialized by
+ * aviommu->vfctrl_lock: CONTROL0 is shared by all vDevices on this gid.
+ */
+void amd_viommu_set_device_mapping(struct amd_iommu_viommu *aviommu,
+				   u16 hdev_id, u16 gdev_id)
+{
+	struct amd_iommu *iommu =
+		container_of(aviommu->core.iommu_dev, struct amd_iommu, iommu);
+
+	pr_debug("%s: iommu_devid=%#x, gid=%#x, hdev_id=%#x, gdev_id=%#x\n",
+		 __func__, pci_dev_id(iommu->dev), aviommu->gid, hdev_id,
+		 gdev_id);
+
+	mutex_lock(&aviommu->vfctrl_lock);
+	set_device_mapping_unlocked(iommu, aviommu->gid, hdev_id, gdev_id);
+	mutex_unlock(&aviommu->vfctrl_lock);
+}
+
 void amd_viommu_uninit_one(struct amd_iommu *iommu, struct amd_iommu_viommu *aviommu)
 {
 	pr_debug("%s: gid=%u\n", __func__, aviommu->gid);
-- 
2.34.1