[PATCH v1 6/8] iommu: Introduce TDISP T=0 state for confidential IOMMUs

[Nicolin Chen <nicolinc@nvidia.com>]

TDISP devices have two independent DMA streams, T=1 and T=0. In several
platforms these streams terminate into different IOMMU environments with
different translation tables. For such platforms the IOMMU driver in Linux
handles exactly one of the streams. Typically we expect the T=1 stream to
be linked to the vIOMMU and the T=0 stream to simply be left as a no-iommu
identity configuration.

As the HW has these two different DMA translation environments, the iommu
subsystem must participate in switching between them. When the device uses
T=0, its T=1 vIOMMU path should be left in BLOCKING to disable it, and a
simplified "T=0 IDENTITY" mode is setup within the DMA API.

    [device: T=0] ===> [hypervisor] ===> [memory]
          |
          +-- (guest association) --> [confidential IOMMU: blocking DMA]

Setting the vIOMMU to BLOCKING also gives up its control of ATS, which is
now managed by the core code as part of simplified T=0 mode.

Suggested-by: Jason Gunthorpe <jgg@nvidia.com>
Assisted-by: Claude:claude-opus-5
Signed-off-by: Nicolin Chen <nicolinc@nvidia.com>
---
 drivers/iommu/Makefile     |  3 ++
 drivers/iommu/iommu-priv.h | 19 +++++++++++++
 include/linux/iommu.h      |  4 +++
 drivers/iommu/iommu-cc.c   | 57 ++++++++++++++++++++++++++++++++++++++
 drivers/iommu/iommu.c      | 17 ++++++++++--
 5 files changed, 98 insertions(+), 2 deletions(-)
 create mode 100644 drivers/iommu/iommu-cc.c

diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 2f05725eaab18..dc91d4cf0ce11 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -6,6 +6,9 @@ obj-$(CONFIG_RISCV_IOMMU) += riscv/
 obj-$(CONFIG_GENERIC_PT) += generic_pt/fmt/
 obj-$(CONFIG_HYPERV) += hyperv/
 obj-$(CONFIG_IOMMU_API) += iommu.o
+ifdef CONFIG_ARCH_HAS_CC_PLATFORM
+obj-$(CONFIG_IOMMU_API) += iommu-cc.o
+endif
 obj-$(CONFIG_IOMMU_SUPPORT) += iommu-pages.o
 obj-$(CONFIG_IOMMU_API) += iommu-traces.o
 obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o
diff --git a/drivers/iommu/iommu-priv.h b/drivers/iommu/iommu-priv.h
index aaffad5854fc9..06e233ac4662f 100644
--- a/drivers/iommu/iommu-priv.h
+++ b/drivers/iommu/iommu-priv.h
@@ -8,6 +8,25 @@
 #include <linux/iommu-debug-pagealloc.h>
 #include <linux/msi.h>
 
+#ifdef CONFIG_ARCH_HAS_CC_PLATFORM
+void iommu_tdisp_enter_t0(struct device *dev);
+void iommu_tdisp_exit_t0(struct device *dev);
+#else
+static inline void iommu_tdisp_enter_t0(struct device *dev)
+{
+}
+
+static inline void iommu_tdisp_exit_t0(struct device *dev)
+{
+}
+#endif
+
+static inline bool iommu_using_t0_stream(struct device *dev)
+{
+	return dev->iommu->iommu_dev->confidential &&
+	       !dev->iommu->tdisp_t1;
+}
+
 static inline const struct iommu_ops *dev_iommu_ops(struct device *dev)
 {
 	/*
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index bd68532e7f3be..e3ed0dc538e0d 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -871,6 +871,9 @@ struct iommu_fault_param {
  * @max_pasids:  number of PASIDs this device can consume
  * @attach_deferred: the dma domain attachment is deferred
  * @pci_32bit_workaround: Limit DMA allocations to 32-bit IOVAs
+ * @tdisp_t1: device uses its confidential IOMMU's T=1 stream. A zero value
+ *            means T=0, where the T=1 stream remains blocked while the core
+ *            owns required DMA operations such as ATS.
  * @require_direct: device requires IOMMU_RESV_DIRECT regions
  * @shadow_on_flush: IOTLB flushes are used to sync shadow tables
  *
@@ -886,6 +889,7 @@ struct dev_iommu {
 	u32				max_pasids;
 	u32				attach_deferred:1;
 	u32				pci_32bit_workaround:1;
+	u32				tdisp_t1:1;
 	u32				require_direct:1;
 	u32				shadow_on_flush:1;
 };
diff --git a/drivers/iommu/iommu-cc.c b/drivers/iommu/iommu-cc.c
new file mode 100644
index 0000000000000..d107b39696a08
--- /dev/null
+++ b/drivers/iommu/iommu-cc.c
@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * TDISP devices can have separate T=0 and T=1 DMA streams that terminate in
+ * different IOMMU environments. The Linux IOMMU driver manages the T=1 stream
+ * while the T=0 stream uses the physical path.
+ *
+ *   [device: T=0] ===> [hypervisor] ===> [memory]
+ *       |
+ *       +-- (guest association) --> [confidential IOMMU: blocking DMA]
+ */
+#include <linux/pci.h>
+#include <linux/pci-ats.h>
+
+#include "iommu-priv.h"
+
+/**
+ * iommu_tdisp_enter_t0 - Prepare @dev while TDISP is T=0
+ * @dev: device entering TDISP T=0
+ *
+ * The IOMMU core invokes this helper for a device on a confidential IOMMU
+ * while TDISP is T=0. The device remains associated with its IOMMU while the
+ * core owns DMA operations required in this state. Enable ATS here when the
+ * device requires it, since the IOMMU driver does not operate ATS while in
+ * BLOCKED.
+ */
+void iommu_tdisp_enter_t0(struct device *dev)
+{
+	struct pci_dev *pdev;
+	int ret;
+
+	if (!dev_is_pci(dev))
+		return;
+
+	pdev = to_pci_dev(dev);
+	if (!pci_ats_required(pdev))
+		return;
+
+	ret = pci_enable_ats(pdev, PCI_ATS_MIN_STU);
+	if (ret)
+		dev_warn(dev, "cannot enable ATS while TDISP is T=0\n");
+}
+
+/**
+ * iommu_tdisp_exit_t0 - Undo iommu_tdisp_enter_t0()
+ * @dev: device leaving TDISP T=0
+ */
+void iommu_tdisp_exit_t0(struct device *dev)
+{
+	struct pci_dev *pdev;
+
+	if (!dev_is_pci(dev))
+		return;
+
+	pdev = to_pci_dev(dev);
+	if (pci_ats_required(pdev) && pdev->ats_enabled)
+		pci_disable_ats(pdev);
+}
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index feff390727d13..7ac6569af6773 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -535,9 +535,17 @@ static int iommu_init_device(struct device *dev)
 	}
 	dev->iommu->iommu_dev = iommu_dev;
 
+	if (iommu_using_t0_stream(dev)) {
+		/*
+		 * After probe the iommu has to leave the T=1 stream in BLOCKING,
+		 * while the T=0 stream will go through the physical path.
+		 */
+		iommu_tdisp_enter_t0(dev);
+	}
+
 	ret = iommu_device_link(iommu_dev, dev);
 	if (ret)
-		goto err_release;
+		goto err_tdisp_t0;
 
 	group = ops->device_group(dev);
 	if (WARN_ON_ONCE(group == NULL))
@@ -555,7 +563,9 @@ static int iommu_init_device(struct device *dev)
 
 err_unlink:
 	iommu_device_unlink(iommu_dev, dev);
-err_release:
+err_tdisp_t0:
+	if (iommu_using_t0_stream(dev))
+		iommu_tdisp_exit_t0(dev);
 	if (ops->release_device)
 		ops->release_device(dev);
 err_module_put:
@@ -573,6 +583,9 @@ static void iommu_deinit_device(struct device *dev)
 
 	lockdep_assert_held(&group->mutex);
 
+	if (iommu_using_t0_stream(dev))
+		iommu_tdisp_exit_t0(dev);
+
 	iommu_device_unlink(dev->iommu->iommu_dev, dev);
 
 	/*
-- 
2.43.0