[RFC PATCH v2 0/4] iommu/riscv: Add irqbypass support without MSI page table

[RFC PATCH v2 0/4] iommu/riscv: Add irqbypass support without MSI page table

[fangyu.yu]

From: Fangyu Yu <fangyu.yu@linux.alibaba.com>

The MSI page table is an optional feature of the RISC-V IOMMU
specification. Hardware without the MSI_FLAT capability has no such
table, and MSI writes from devices are translated by the second-stage
page table like any other DMA.

Since the second-stage page table governs where those MSI writes land,
forwarding is still possible without an MSI page table: map the guest
IMSIC GPA to the host IMSIC HPA in the second-stage domain and device
MSI writes reach the vCPU's interrupt file directly, bypassing the host
MSI path entirely.

This patch implements irqbypass using second-stage page table, so it
is a functional supplement for IOMMU without MSI page table.

This series is based on Andrew Jones' v3 irqbypass support series [1]

[1] https://lore.kernel.org/linux-riscv/20260928143113.49838-1-andrew.jones@oss.qualcomm.com/

---
Changes in v2:
    - Following Jason's and Shuai's suggestion, the atomic PTE replace operation
      is moved into generic_pt as a new replace_leaf op (new patch 1), and the
      riscv driver now uses it to repoint the guest IMSIC GPA mapping instead of
      walking the second-stage page table in the driver (patch 3).
    - Link to v1:
      https://lore.kernel.org/linux-riscv/20261008140001.94508-1-fangyu.yu@linux.alibaba.com/

Fangyu Yu (4):
  iommupt: Add replace_leaf to atomically swap a leaf OA
  iommu/riscv: Extract IRQ forwarding payload validation
  iommu/riscv: Add guest IMSIC GPA mapping helpers
  iommu/riscv: Support IRQ forwarding without MSI page tables

 drivers/iommu/generic_pt/fmt/riscv.h      |  47 ++++++
 drivers/iommu/generic_pt/iommu_pt.h       |  65 ++++++++
 drivers/iommu/generic_pt/kunit_iommu.h    |   7 +
 drivers/iommu/generic_pt/kunit_iommu_pt.h |  54 +++++++
 drivers/iommu/riscv/iommu-ir.c            | 178 ++++++++++++++++++++--
 drivers/iommu/riscv/iommu.c               |  43 ++++++
 drivers/iommu/riscv/iommu.h               |  12 ++
 include/linux/generic_pt/iommu.h          |   4 +
 8 files changed, 401 insertions(+), 9 deletions(-)

-- 
2.50.1

[RFC PATCH v2 1/4] iommupt: Add replace_leaf to atomically swap a leaf OA

[fangyu.yu]

From: Fangyu Yu <fangyu.yu@linux.alibaba.com>

Some users need to repoint a leaf entry to a different physical page
without ever making the IOVA non-present. RISC-V IRQ forwarding without
an MSI page table maps the guest IMSIC page in the second-stage domain
and must follow the page when a vCPU's VS-file host page moves; an
unmap/map sequence would leave a window where device MSI writes fault.

Add a pt_entry_replace_oa() format op that atomically swaps the OA
stored in a leaf entry that fits in a single PTE word via cmpxchg,
validating that the entry still holds the expected old OA, and a
pt_iommu_riscv_64_replace_leaf() export that walks to the leaf. Formats
that cannot do this atomically do not provide the op, following the
pt_iommu_fmt_hw_info() pattern. Covered by a kunit case checking the
granule and superpage swaps, stale OA rejection, and missing
translation.

Signed-off-by: Fangyu Yu <fangyu.yu@linux.alibaba.com>
---
 drivers/iommu/generic_pt/fmt/riscv.h      | 47 ++++++++++++++++
 drivers/iommu/generic_pt/iommu_pt.h       | 65 +++++++++++++++++++++++
 drivers/iommu/generic_pt/kunit_iommu.h    |  7 +++
 drivers/iommu/generic_pt/kunit_iommu_pt.h | 54 +++++++++++++++++++
 include/linux/generic_pt/iommu.h          |  4 ++
 5 files changed, 177 insertions(+)

diff --git a/drivers/iommu/generic_pt/fmt/riscv.h b/drivers/iommu/generic_pt/fmt/riscv.h
index 2f4838eb44af..38c837805df0 100644
--- a/drivers/iommu/generic_pt/fmt/riscv.h
+++ b/drivers/iommu/generic_pt/fmt/riscv.h
@@ -27,6 +27,7 @@
 
 #include <linux/bitfield.h>
 #include <linux/container_of.h>
+#include <linux/errno.h>
 #include <linux/log2.h>
 #include <linux/sizes.h>
 
@@ -272,6 +273,52 @@ static inline bool riscvpt_dirty_supported(struct pt_common *common)
 }
 #define pt_dirty_supported riscvpt_dirty_supported
 
+/**
+ * pt_entry_replace_oa() - Atomically replace the OA stored in a leaf
+ * @pts: Table entry to change
+ * @old_oa: OA the entry must currently translate to
+ * @new_oa: OA to store in the entry
+ *
+ * Atomically swap the OA stored in a single PTE word leaf entry, keeping
+ * every other bit of the entry unchanged. The entry must currently translate
+ * to @old_oa so a stale view of the table is detectable. Both @old_oa and
+ * @new_oa must be aligned to the size the entry translates. Entries that do
+ * not fit in a single PTE word, such as contiguous entries, are rejected.
+ *
+ * Return: 0 on success, -EAGAIN if the entry changed concurrently, -EADDRINUSE
+ * if the entry does not translate to @old_oa, -EOPNOTSUPP if the entry spans
+ * multiple PTE words, -EINVAL if an argument is malformed.
+ */
+static inline int riscvpt_entry_replace_oa(struct pt_state *pts,
+					   pt_oaddr_t old_oa,
+					   pt_oaddr_t new_oa)
+{
+	pt_riscv_entry_t *tablep =
+		pt_cur_table(pts, pt_riscv_entry_t) + pts->index;
+	pt_riscv_entry_t old = pts->entry, new;
+	unsigned int oasz_lg2 = pt_table_item_lg2sz(pts);
+	u64 ppn = (u64)new_oa >> PT_GRANULE_LG2SZ;
+
+	/* Only a leaf entry that fits in a single PTE word can be replaced. */
+	if (pt_entry_num_contig_lg2(pts))
+		return -EOPNOTSUPP;
+
+	if (oalog2_mod(old_oa, oasz_lg2) || oalog2_mod(new_oa, oasz_lg2) ||
+	    ppn > FIELD_MAX(RISCVPT_PPN))
+		return -EINVAL;
+
+	/* The leaf must still hold the OA the caller expects to replace. */
+	if (pt_entry_oa(pts) != old_oa)
+		return -EADDRINUSE;
+
+	new = (old & ~(pt_riscv_entry_t)RISCVPT_PPN) |
+	      FIELD_PREP(RISCVPT_PPN, ppn);
+	if (!try_cmpxchg64(tablep, &old, new))
+		return -EAGAIN;
+	return 0;
+}
+#define pt_entry_replace_oa riscvpt_entry_replace_oa
+
 /* --- iommu */
 #include <linux/generic_pt/iommu.h>
 #include <linux/iommu.h>
diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h
index 3d1fdf1e81d8..5e296992c4df 100644
--- a/drivers/iommu/generic_pt/iommu_pt.h
+++ b/drivers/iommu/generic_pt/iommu_pt.h
@@ -375,6 +375,71 @@ static int __maybe_unused NS(set_dirty)(struct pt_iommu *iommu_table,
 	return pt_walk_range(&range, __set_dirty, NULL);
 }
 
+#ifdef pt_entry_replace_oa
+struct pt_replace_leaf_args {
+	pt_oaddr_t old_oa;
+	pt_oaddr_t new_oa;
+};
+
+static int __replace_leaf(struct pt_range *range, void *arg,
+			  unsigned int level, struct pt_table_p *table)
+{
+	struct pt_state pts = pt_init(range, level, table);
+	struct pt_replace_leaf_args *args = arg;
+
+	switch (pt_load_single_entry(&pts)) {
+	case PT_ENTRY_EMPTY:
+		return -ENOENT;
+	case PT_ENTRY_TABLE:
+		return pt_descend(&pts, arg, __replace_leaf);
+	case PT_ENTRY_OA:
+		return pt_entry_replace_oa(&pts, args->old_oa, args->new_oa);
+	}
+	return -ENOENT;
+}
+
+/**
+ * replace_leaf() - Atomically replace the OA stored in a leaf entry
+ * @domain: Table to manipulate
+ * @iova: IO virtual address to locate the leaf
+ * @old_oa: OA the leaf must currently translate to
+ * @new_oa: OA to store in the leaf
+ *
+ * Locate the leaf entry translating @iova and atomically swap its stored OA
+ * from @old_oa to @new_oa, leaving the entry attributes unchanged. HW
+ * walking the table concurrently observes either the old or the new OA. The
+ * caller must invalidate any translation caches before assuming HW no longer
+ * observes the old OA.
+ *
+ * Context: There is no internal locking. The caller must serialize this with
+ * any concurrent map/unmap touching @iova, as with map_range().
+ *
+ * Returns: 0 on success. -EAGAIN indicates the entry changed concurrently,
+ * retry. -ENOENT if @iova has no translation. -EADDRINUSE if the leaf does
+ * not translate to @old_oa. -EOPNOTSUPP if the leaf does not fit in a single
+ * PTE word, such as a contiguous entry.
+ */
+int DOMAIN_NS(replace_leaf)(struct iommu_domain *domain, dma_addr_t iova,
+			    phys_addr_t old_oa, phys_addr_t new_oa)
+{
+	struct pt_iommu *iommu_table =
+		container_of(domain, struct pt_iommu, domain);
+	struct pt_replace_leaf_args args = {
+		.old_oa = old_oa,
+		.new_oa = new_oa,
+	};
+	struct pt_range range;
+	int ret;
+
+	ret = make_range(common_from_iommu(iommu_table), &range, iova, 1);
+	if (ret)
+		return ret;
+
+	return pt_walk_range(&range, __replace_leaf, &args);
+}
+EXPORT_SYMBOL_NS_GPL(DOMAIN_NS(replace_leaf), "GENERIC_PT_IOMMU");
+#endif
+
 struct pt_iommu_collect_args {
 	struct iommupt_pending_gather pending;
 	/* Fail if any OAs are within the range */
diff --git a/drivers/iommu/generic_pt/kunit_iommu.h b/drivers/iommu/generic_pt/kunit_iommu.h
index 22c9e4c4dd97..d52b82c9942c 100644
--- a/drivers/iommu/generic_pt/kunit_iommu.h
+++ b/drivers/iommu/generic_pt/kunit_iommu.h
@@ -16,6 +16,13 @@
 int pt_iommu_init(struct pt_iommu_table *fmt_table,
 		  const struct pt_iommu_table_cfg *cfg, gfp_t gfp);
 
+#ifdef pt_entry_replace_oa
+#define pt_iommu_replace_leaf \
+	CONCATENATE(CONCATENATE(pt_iommu_, PTPFX), replace_leaf)
+int pt_iommu_replace_leaf(struct iommu_domain *domain, dma_addr_t iova,
+			  phys_addr_t old_oa, phys_addr_t new_oa);
+#endif
+
 /* The format can provide a list of configurations it would like to test */
 #ifdef kunit_fmt_cfgs
 static const void *kunit_pt_gen_params_cfg(struct kunit *test, const void *prev,
diff --git a/drivers/iommu/generic_pt/kunit_iommu_pt.h b/drivers/iommu/generic_pt/kunit_iommu_pt.h
index ece1c9b8c55d..8707d55418d3 100644
--- a/drivers/iommu/generic_pt/kunit_iommu_pt.h
+++ b/drivers/iommu/generic_pt/kunit_iommu_pt.h
@@ -427,6 +427,57 @@ static void test_mixed(struct kunit *test)
 	check_iova(test, start, oa, len);
 }
 
+#ifdef pt_entry_replace_oa
+static void test_replace_leaf(struct kunit *test)
+{
+	struct kunit_iommu_priv *priv = test->priv;
+	struct pt_range range = pt_top_range(priv->common);
+	pt_vaddr_t va = range.va + priv->smallest_pgsz * 256;
+	phys_addr_t pa = log2_set_mod(priv->test_oa, 0,
+				      priv->smallest_pgsz_lg2);
+	phys_addr_t new_pa = log2_set_mod(priv->test_oa + priv->smallest_pgsz,
+					  0, priv->smallest_pgsz_lg2);
+	int ret;
+
+	do_map(test, va, pa, priv->smallest_pgsz);
+
+	/* The leaf OA can be swapped in place */
+	ret = pt_iommu_replace_leaf(&priv->domain, va, pa, new_pa);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	check_iova(test, va, new_pa, priv->smallest_pgsz);
+
+	/* A stale old OA is detected */
+	ret = pt_iommu_replace_leaf(&priv->domain, va, pa, pa);
+	KUNIT_ASSERT_EQ(test, ret, -EADDRINUSE);
+
+	/* A not-present IOVA reports -ENOENT */
+	ret = pt_iommu_replace_leaf(&priv->domain, va + priv->smallest_pgsz,
+				    pa, new_pa);
+	KUNIT_ASSERT_EQ(test, ret, -ENOENT);
+
+	do_unmap(test, va, priv->smallest_pgsz);
+
+	if (!(priv->safe_pgsize_bitmap & SZ_2M))
+		return;
+
+	/* A leaf at a higher level can be swapped in place too */
+	va = ALIGN(va, SZ_2M);
+	pa = log2_set_mod(priv->test_oa, 0, ilog2(SZ_2M));
+	new_pa = log2_set_mod(priv->test_oa + SZ_2M, 0, ilog2(SZ_2M));
+	do_map(test, va, pa, SZ_2M);
+
+	ret = pt_iommu_replace_leaf(&priv->domain, va, pa, new_pa);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	check_iova(test, va, new_pa, SZ_2M);
+
+	/* A stale old OA is detected */
+	ret = pt_iommu_replace_leaf(&priv->domain, va, pa, pa);
+	KUNIT_ASSERT_EQ(test, ret, -EADDRINUSE);
+
+	do_unmap(test, va, SZ_2M);
+}
+#endif
+
 static struct kunit_case iommu_test_cases[] = {
 	KUNIT_CASE_FMT(test_increase_level),
 	KUNIT_CASE_FMT(test_map_simple),
@@ -435,6 +486,9 @@ static struct kunit_case iommu_test_cases[] = {
 	KUNIT_CASE_FMT(test_random_map),
 	KUNIT_CASE_FMT(test_pgsize_boundary),
 	KUNIT_CASE_FMT(test_mixed),
+#ifdef pt_entry_replace_oa
+	KUNIT_CASE_FMT(test_replace_leaf),
+#endif
 	{},
 };
 
diff --git a/include/linux/generic_pt/iommu.h b/include/linux/generic_pt/iommu.h
index 2eed9b78f56c..e9042b509013 100644
--- a/include/linux/generic_pt/iommu.h
+++ b/include/linux/generic_pt/iommu.h
@@ -254,6 +254,10 @@ struct pt_iommu_cfg {
 	int pt_iommu_##fmt##_read_and_clear_dirty(                             \
 		struct iommu_domain *domain, unsigned long iova, size_t size,  \
 		unsigned long flags, struct iommu_dirty_bitmap *dirty);        \
+	/* Only defined by formats whose fmt header has pt_entry_replace_oa */ \
+	int pt_iommu_##fmt##_replace_leaf(struct iommu_domain *domain,         \
+					  dma_addr_t iova, phys_addr_t old_oa, \
+					  phys_addr_t new_oa);                 \
 	int pt_iommu_##fmt##_init(struct pt_iommu_##fmt *table,                \
 				  const struct pt_iommu_##fmt##_cfg *cfg,      \
 				  gfp_t gfp);                                  \
-- 
2.50.1

[RFC PATCH v2 2/4] iommu/riscv: Extract IRQ forwarding payload validation

[fangyu.yu]

From: Fangyu Yu <fangyu.yu@linux.alibaba.com>

Move the owner and MSI address mask/pattern checks from
riscv_iommu_ir_activate() into a separate helper so a second forwarding
backend can validate the same fields. No functional change.

Signed-off-by: Fangyu Yu <fangyu.yu@linux.alibaba.com>
---
 drivers/iommu/riscv/iommu-ir.c | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/drivers/iommu/riscv/iommu-ir.c b/drivers/iommu/riscv/iommu-ir.c
index b3f0a56475ed..b96baff9986d 100644
--- a/drivers/iommu/riscv/iommu-ir.c
+++ b/drivers/iommu/riscv/iommu-ir.c
@@ -104,15 +104,10 @@ static void riscv_iommu_ir_set_target(struct riscv_iommu_msipte *msipte,
 	}
 }
 
-static int riscv_iommu_ir_activate(struct riscv_iommu_msi_table *msi_table,
-				   struct riscv_iommu_device *iommu,
-				   struct riscv_iommu_ir_vcpu_info *vcpu_info)
+static int riscv_iommu_ir_validate_vcpu_info(const struct riscv_iommu_ir_vcpu_info *vcpu_info)
 {
 	u64 pattern = vcpu_info->msi_addr_pattern;
 	u64 mask = vcpu_info->msi_addr_mask;
-	u64 required_caps = 0;
-	size_t nr_ptes;
-	int ret;
 
 	if (!vcpu_info->owner || (pattern & mask) ||
 	    ((pattern | mask) & ~RISCV_IOMMU_DC_MSI_ADDR_MASK))
@@ -129,6 +124,21 @@ static int riscv_iommu_ir_activate(struct riscv_iommu_msi_table *msi_table,
 			return -EINVAL;
 	}
 
+	return 0;
+}
+
+static int riscv_iommu_ir_activate(struct riscv_iommu_msi_table *msi_table,
+				   struct riscv_iommu_device *iommu,
+				   struct riscv_iommu_ir_vcpu_info *vcpu_info)
+{
+	u64 required_caps = 0;
+	size_t nr_ptes;
+	int ret;
+
+	ret = riscv_iommu_ir_validate_vcpu_info(vcpu_info);
+	if (ret)
+		return ret;
+
 	nr_ptes = riscv_iommu_ir_nr_ptes(vcpu_info);
 
 	ret = riscv_iommu_ir_validate_targets(vcpu_info, nr_ptes, &required_caps);
-- 
2.50.1

[RFC PATCH v2 3/4] iommu/riscv: Add guest IMSIC GPA mapping helpers

[fangyu.yu]

From: Fangyu Yu <fangyu.yu@linux.alibaba.com>

IOMMU implementations without the MSI_FLAT capability translate MSI
writes through the second-stage page table, so IRQ forwarding on them
will need guest IMSIC pages mapped into the second-stage domain.

Add an xarray to the MSI table that tracks the HPA installed for each
mapped guest IMSIC GPA, and two helpers for use with the MSI table lock
held: riscv_iommu_msi_table_map_gpa() installs the initial 4 KiB
mapping, and riscv_iommu_msi_table_replace_gpa() repoints the leaf PTE
when a vCPU's VS-file host page moves, using the generic page table
replace_leaf op instead of walking the table in the driver.

Signed-off-by: Fangyu Yu <fangyu.yu@linux.alibaba.com>
---
 drivers/iommu/riscv/iommu.c | 43 +++++++++++++++++++++++++++++++++++++
 drivers/iommu/riscv/iommu.h | 12 +++++++++++
 2 files changed, 55 insertions(+)

diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c
index e2e77469ea3c..88e131ce96c1 100644
--- a/drivers/iommu/riscv/iommu.c
+++ b/drivers/iommu/riscv/iommu.c
@@ -1217,6 +1217,42 @@ void riscv_iommu_msi_table_inval_all(struct riscv_iommu_msi_table *msi_table)
 	riscv_iommu_iotlb_inval(domain, &gather);
 }
 
+int riscv_iommu_msi_table_map_gpa(struct riscv_iommu_msi_table *msi_table,
+				  dma_addr_t gpa, phys_addr_t hpa)
+{
+	struct riscv_iommu_domain *domain =
+		container_of(msi_table, struct riscv_iommu_domain, msi_table);
+	const int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
+
+	/* Guest IMSIC GPA mapping only exists in second-stage translations. */
+	if (!domain->gscid)
+		return -EOPNOTSUPP;
+
+	return iommu_map(&domain->domain, gpa, hpa, IMSIC_MMIO_PAGE_SZ, prot,
+			 GFP_ATOMIC);
+}
+
+int riscv_iommu_msi_table_replace_gpa(struct riscv_iommu_msi_table *msi_table,
+				      dma_addr_t gpa, phys_addr_t old_hpa,
+				      phys_addr_t new_hpa)
+{
+	struct riscv_iommu_domain *domain =
+		container_of(msi_table, struct riscv_iommu_domain, msi_table);
+	int ret;
+
+	/* Guest IMSIC GPA mapping only exists in second-stage translations. */
+	if (!domain->gscid)
+		return -EOPNOTSUPP;
+
+	ret = pt_iommu_riscv_64_replace_leaf(&domain->domain, gpa, old_hpa,
+					     new_hpa);
+	if (ret == -EAGAIN) {
+		/* The leaf changed concurrently; the caller revalidates. */
+		return -EADDRINUSE;
+	}
+	return ret;
+}
+
 #define RISCV_IOMMU_FSC_BARE 0
 /*
  * This function sends IOTINVAL commands as required by the RISC-V
@@ -1425,6 +1461,8 @@ static void riscv_iommu_iotlb_sync(struct iommu_domain *iommu_domain,
 static void riscv_iommu_free_paging_domain(struct iommu_domain *iommu_domain)
 {
 	struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
+	struct riscv_iommu_noflat_imsic *imsic;
+	unsigned long index;
 
 	WARN_ON(!list_empty(&domain->bonds));
 
@@ -1435,6 +1473,10 @@ static void riscv_iommu_free_paging_domain(struct iommu_domain *iommu_domain)
 	if (domain->gscid > 0)
 		ida_free(&riscv_iommu_gscids, domain->gscid);
 
+	xa_for_each(&domain->msi_table.noflat_imsics, index, imsic)
+		kfree(imsic);
+	xa_destroy(&domain->msi_table.noflat_imsics);
+
 	pt_iommu_deinit(&domain->riscvpt.iommu);
 	iommu_free_pages(domain->msi_table.root);
 	kfree(domain);
@@ -1676,6 +1718,7 @@ riscv_iommu_domain_alloc_paging_flags(struct device *dev, u32 flags,
 	INIT_LIST_HEAD_RCU(&domain->bonds);
 	raw_spin_lock_init(&domain->lock);
 	raw_spin_lock_init(&domain->msi_table.lock);
+	xa_init(&domain->msi_table.noflat_imsics);
 	mutex_init(&domain->mutex);
 	iommu = dev_to_iommu(dev);
 	cfg.common.hw_max_oasz_lg2 = 56;
diff --git a/drivers/iommu/riscv/iommu.h b/drivers/iommu/riscv/iommu.h
index 9852962e245b..8d5f44de2688 100644
--- a/drivers/iommu/riscv/iommu.h
+++ b/drivers/iommu/riscv/iommu.h
@@ -18,6 +18,7 @@
 #include <linux/irqdomain.h>
 #include <linux/rcupdate.h>
 #include <linux/sizes.h>
+#include <linux/xarray.h>
 
 #include "iommu-bits.h"
 
@@ -74,6 +75,11 @@ struct riscv_iommu_device {
 	struct irq_domain *irqdomain;
 };
 
+/* Tracks a guest IMSIC GPA mapped into an S2 domain on IOMMUs without MSI_FLAT. */
+struct riscv_iommu_noflat_imsic {
+	phys_addr_t hpa;
+};
+
 struct riscv_iommu_msi_table {
 	/* Protects attachment, interrupt forwarding state, and MSI PTE updates. */
 	raw_spinlock_t lock;
@@ -84,6 +90,7 @@ struct riscv_iommu_msi_table {
 	u64 msi_addr_pattern;
 	const void *owner;
 	u64 required_caps; /* RISCV_IOMMU_CAPABILITIES_* required by active MSI PTEs */
+	struct xarray noflat_imsics;
 };
 
 /* Private IOMMU data for managed devices, dev_iommu_priv_* */
@@ -109,6 +116,11 @@ bool riscv_iommu_msi_table_check_caps(struct riscv_iommu_msi_table *msi_table, u
 void riscv_iommu_msi_table_inval(struct riscv_iommu_msi_table *msi_table, unsigned long addr);
 void riscv_iommu_msi_table_inval_all(struct riscv_iommu_msi_table *msi_table);
 void riscv_iommu_msi_table_update(struct riscv_iommu_msi_table *msi_table, bool activate);
+int riscv_iommu_msi_table_map_gpa(struct riscv_iommu_msi_table *msi_table,
+				  dma_addr_t gpa, phys_addr_t hpa);
+int riscv_iommu_msi_table_replace_gpa(struct riscv_iommu_msi_table *msi_table,
+				      dma_addr_t gpa, phys_addr_t old_hpa,
+				      phys_addr_t new_hpa);
 
 #ifdef CONFIG_RISCV_IMSIC
 void riscv_iommu_ir_irq_domain_remove(struct riscv_iommu_device *iommu);
-- 
2.50.1

[RFC PATCH v2 4/4] iommu/riscv: Support IRQ forwarding without MSI page tables

[fangyu.yu]

From: Fangyu Yu <fangyu.yu@linux.alibaba.com>

IOMMU implementations without the MSI_FLAT capability have no MSI page
table, and their MSI writes are translated by the second-stage page
table. Forward interrupts there by mapping the guest IMSIC GPA to the
host IMSIC HPA in the second-stage domain, so device MSI writes reach
the vCPU's interrupt file directly.

Dispatch on the IOMMU capabilities in the IRQ forwarding entry: with
MSI_FLAT nothing changes; without it, validate that every target is an
IMSIC page in the guest MSI address window, install the GPA mappings,
and replace the leaf PTE via the generic page table replace_leaf op
when a vCPU migration moves the VS-file host page. Mappings persist
for the lifetime of the domain; disabling forwarding or a failed
operation only drops the per-IRQ state so the IRQ falls back to host
delivery.

Signed-off-by: Fangyu Yu <fangyu.yu@linux.alibaba.com>
---
 drivers/iommu/riscv/iommu-ir.c | 156 ++++++++++++++++++++++++++++++++-
 1 file changed, 153 insertions(+), 3 deletions(-)

diff --git a/drivers/iommu/riscv/iommu-ir.c b/drivers/iommu/riscv/iommu-ir.c
index b96baff9986d..bc474e44805d 100644
--- a/drivers/iommu/riscv/iommu-ir.c
+++ b/drivers/iommu/riscv/iommu-ir.c
@@ -271,14 +271,159 @@ static int riscv_iommu_ir_update_target(struct riscv_iommu_msi_table *msi_table,
 	return 0;
 }
 
-static int riscv_iommu_ir_irq_set_vcpu_affinity_locked(struct irq_data *data,
+static int riscv_iommu_ir_validate_noflat_target(const struct riscv_iommu_ir_vcpu_info *vcpu_info,
+						 const struct riscv_iommu_ir_target *target)
+{
+	u64 addr = target->gpa >> IMSIC_MMIO_PAGE_SHIFT;
+
+	if (!IS_ALIGNED(target->gpa, IMSIC_MMIO_PAGE_SZ) ||
+	    (addr & ~vcpu_info->msi_addr_mask) != vcpu_info->msi_addr_pattern)
+		return -EINVAL;
+
+	/* Without MSI_FLAT there is no MSI page table, so only IMSIC targets work. */
+	if (target->type != RISCV_IOMMU_IR_TARGET_IMSIC)
+		return -EOPNOTSUPP;
+
+	if (!IS_ALIGNED(target->hpa, IMSIC_MMIO_PAGE_SZ))
+		return -EINVAL;
+
+	return 0;
+}
+
+/* Caller must hold msi_table->lock. */
+static int riscv_iommu_ir_map_guest_imsic(struct riscv_iommu_msi_table *msi_table,
+					  const struct riscv_iommu_ir_target *target)
+{
+	unsigned long index = target->gpa >> IMSIC_MMIO_PAGE_SHIFT;
+	struct riscv_iommu_noflat_imsic *imsic;
+	int ret;
+
+	imsic = xa_load(&msi_table->noflat_imsics, index);
+	if (imsic) {
+		if (imsic->hpa == target->hpa)
+			return 0;
+
+		ret = riscv_iommu_msi_table_replace_gpa(msi_table, target->gpa,
+							imsic->hpa, target->hpa);
+		if (ret)
+			return ret;
+	} else {
+		imsic = kzalloc_obj(*imsic, GFP_ATOMIC);
+		if (!imsic)
+			return -ENOMEM;
+
+		ret = xa_err(xa_store(&msi_table->noflat_imsics, index, imsic,
+				      GFP_ATOMIC));
+		if (ret) {
+			kfree(imsic);
+			return ret;
+		}
+
+		ret = riscv_iommu_msi_table_map_gpa(msi_table, target->gpa, target->hpa);
+		if (ret) {
+			xa_erase(&msi_table->noflat_imsics, index);
+			kfree(imsic);
+			return ret;
+		}
+	}
+
+	imsic->hpa = target->hpa;
+	riscv_iommu_msi_table_inval(msi_table, target->gpa);
+	return 0;
+}
+
+static int riscv_iommu_ir_activate_noflat(struct riscv_iommu_msi_table *msi_table,
+					  struct riscv_iommu_ir_vcpu_info *vcpu_info)
+{
+	int ret;
+
+	ret = riscv_iommu_ir_validate_vcpu_info(vcpu_info);
+	if (ret)
+		return ret;
+
+	if (!vcpu_info->targets || !vcpu_info->nr_targets)
+		return -EINVAL;
+
+	for (unsigned int i = 0; i < vcpu_info->nr_targets; i++) {
+		ret = riscv_iommu_ir_validate_noflat_target(vcpu_info, &vcpu_info->targets[i]);
+		if (ret)
+			return ret;
+	}
+
+	for (unsigned int i = 0; i < vcpu_info->nr_targets; i++) {
+		ret = riscv_iommu_ir_map_guest_imsic(msi_table, &vcpu_info->targets[i]);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int riscv_iommu_ir_update_target_noflat(struct riscv_iommu_msi_table *msi_table,
+					       struct riscv_iommu_ir_vcpu_info *vcpu_info)
+{
+	const struct riscv_iommu_ir_target *target = &vcpu_info->target;
+	int ret;
+
+	ret = riscv_iommu_ir_validate_vcpu_info(vcpu_info);
+	if (ret)
+		return ret;
+
+	ret = riscv_iommu_ir_validate_noflat_target(vcpu_info, target);
+	if (ret)
+		return ret;
+
+	if (!xa_load(&msi_table->noflat_imsics, target->gpa >> IMSIC_MMIO_PAGE_SHIFT))
+		return -EINVAL;
+
+	return riscv_iommu_ir_map_guest_imsic(msi_table, target);
+}
+
+static int riscv_iommu_ir_irq_set_vcpu_affinity_noflat(struct irq_data *data,
 						       struct riscv_iommu_info *info,
 						       struct riscv_iommu_ir_vcpu_info *vcpu_info,
 						       struct riscv_iommu_msi_table *msi_table)
+{
+	int ret;
+
+	if (!vcpu_info) {
+		/* Mappings persist; only the per-IRQ forwarding state is dropped. */
+		if (irqd_is_forwarded_to_vcpu(data)) {
+			irqd_clr_forwarded_to_vcpu(data);
+			info->nr_forwarded_irqs--;
+		}
+		return 0;
+	}
+
+	ret = vcpu_info->cmd == RISCV_IOMMU_IR_FORWARD ?
+		riscv_iommu_ir_activate_noflat(msi_table, vcpu_info) :
+		riscv_iommu_ir_update_target_noflat(msi_table, vcpu_info);
+	if (!ret) {
+		if (!irqd_is_forwarded_to_vcpu(data)) {
+			irqd_set_forwarded_to_vcpu(data);
+			info->nr_forwarded_irqs++;
+		}
+	} else if (irqd_is_forwarded_to_vcpu(data)) {
+		irqd_clr_forwarded_to_vcpu(data);
+		info->nr_forwarded_irqs--;
+	}
+
+	return ret;
+}
+
+static int riscv_iommu_ir_irq_set_vcpu_affinity_locked(struct irq_data *data,
+						       struct riscv_iommu_info *info,
+						       struct riscv_iommu_ir_vcpu_info *vcpu_info,
+						       struct riscv_iommu_msi_table *msi_table,
+						       bool noflat)
 {
 	struct riscv_iommu_device *iommu = data->domain->host_data;
 	int ret;
 
+	if (noflat)
+		return riscv_iommu_ir_irq_set_vcpu_affinity_noflat(data, info,
+								    vcpu_info, msi_table);
+
 	if (!vcpu_info) {
 		if (WARN_ON_ONCE(!msi_table->nr_forwarded_irqs || !info->nr_forwarded_irqs))
 			return -EINVAL;
@@ -332,10 +477,12 @@ static int riscv_iommu_ir_irq_set_vcpu_affinity_locked(struct irq_data *data,
 static int riscv_iommu_ir_irq_set_vcpu_affinity(struct irq_data *data, void *arg)
 {
 	struct riscv_iommu_ir_vcpu_info *vcpu_info = arg;
+	struct riscv_iommu_device *iommu = data->domain->host_data;
 	struct riscv_iommu_msi_table *msi_table;
 	struct riscv_iommu_info *info;
 	struct msi_desc *desc;
 	struct device *dev;
+	bool noflat;
 	int ret;
 
 	if (!vcpu_info && !irqd_is_forwarded_to_vcpu(data))
@@ -354,6 +501,8 @@ static int riscv_iommu_ir_irq_set_vcpu_affinity(struct irq_data *data, void *arg
 	if (WARN_ON_ONCE(!info))
 		return -EINVAL;
 
+	noflat = !(iommu->caps & RISCV_IOMMU_CAPABILITIES_MSI_FLAT);
+
 	scoped_guard(rcu) {
 		/*
 		 * RCU keeps the table alive, but the device may switch domains before
@@ -361,7 +510,7 @@ static int riscv_iommu_ir_irq_set_vcpu_affinity(struct irq_data *data, void *arg
 		 */
 		for (;;) {
 			msi_table = riscv_iommu_msi_table_rcu(info);
-			if (!msi_table || !msi_table->root)
+			if (!msi_table || (!noflat && !msi_table->root))
 				return -EOPNOTSUPP;
 
 			raw_spin_lock(&msi_table->lock);
@@ -371,7 +520,8 @@ static int riscv_iommu_ir_irq_set_vcpu_affinity(struct irq_data *data, void *arg
 		}
 	}
 
-	ret = riscv_iommu_ir_irq_set_vcpu_affinity_locked(data, info, vcpu_info, msi_table);
+	ret = riscv_iommu_ir_irq_set_vcpu_affinity_locked(data, info, vcpu_info,
+							  msi_table, noflat);
 	raw_spin_unlock(&msi_table->lock);
 
 	return ret;
-- 
2.50.1