[PATCH v3 13/19] iommu/riscv: Implement irq_compose_msi_msg for IMSIC remapping

[Andrew Jones <andrew.jones@oss.qualcomm.com>]

Wire up irq_compose_msi_msg() on the IOMMU-IR irq chip to look up the
pre-mapped IOVA for the target IMSIC page and rewrite the composed
message to use it, redirecting MSI writes through the IOMMU instead
of landing at the raw IMSIC physical address.

The lookup is O(1), since the "extract" function used to build the
table's index is also used here, which is necessary since the number
of IMSICs may eventually be in the hundreds and irq_compose_msi_msg()
may run in atomic context.

domain->msi_iova is read here without domain->mutex, since compose can
run in atomic context, and RCU only protects info->domain itself. Pair
smp_store_release()/smp_load_acquire() on domain->msi_iova to ensure
it's observably filled before publishing its pointer. That guarantees
a non-NULL domain->msi_iova observed here is safe to use.

A NULL domain->msi_iova falls back to the raw IMSIC physical address,
same as when info->domain itself is NULL, since a live paging domain
should not normally reach compose without a populated table.

Signed-off-by: Andrew Jones <andrew.jones@oss.qualcomm.com>
---
 drivers/iommu/riscv/iommu-ir.c | 49 +++++++++++++++++++++++++++++++---
 1 file changed, 45 insertions(+), 4 deletions(-)

diff --git a/drivers/iommu/riscv/iommu-ir.c b/drivers/iommu/riscv/iommu-ir.c
index 7d9a1eaca92e..98be3e3fddad 100644
--- a/drivers/iommu/riscv/iommu-ir.c
+++ b/drivers/iommu/riscv/iommu-ir.c
@@ -91,17 +91,56 @@ static int riscv_iommu_ir_build_msi_iova(struct riscv_iommu_domain *domain, stru
 			return ret;
 	}
 
-	domain->msi_iova = no_free_ptr(msi_iova);
+	/* Pair with smp_load_acquire() in riscv_iommu_ir_compose_msi_msg() */
+	smp_store_release(&domain->msi_iova, no_free_ptr(msi_iova));
 
 	return 0;
 }
 
+static void riscv_iommu_ir_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
+{
+	struct riscv_iommu_info *info = data->domain->host_data;
+	struct riscv_iommu_domain *domain;
+	dma_addr_t *msi_iova;
+	struct msi_desc *desc;
+	phys_addr_t pa;
+	size_t idx;
+
+	BUG_ON(irq_chip_compose_msi_msg(data->parent_data, msg));
+
+	desc = irq_data_get_msi_desc(data);
+	if (WARN_ON_ONCE(!desc))
+		return;
+
+	guard(rcu)();
+
+	domain = rcu_dereference(info->domain);
+	if (!domain) {
+		msi_desc_set_iommu_msi_iova(desc, 0, 0);
+		return;
+	}
+
+	/* Pair with smp_store_release() in riscv_iommu_ir_build_msi_iova() */
+	msi_iova = smp_load_acquire(&domain->msi_iova);
+	if (!msi_iova) {
+		msi_desc_set_iommu_msi_iova(desc, 0, 0);
+		return;
+	}
+
+	pa = ((u64)msg->address_hi << 32) | msg->address_lo;
+	idx = riscv_iommu_ir_msi_iova_idx(pa);
+
+	msi_desc_set_iommu_msi_iova(desc, msi_iova[idx], IMSIC_MMIO_PAGE_SHIFT);
+	msi_msg_set_addr(desc, msg, pa);
+}
+
 static struct irq_chip riscv_iommu_ir_irq_chip = {
 	.name			= "IOMMU-IR",
 	.irq_ack		= irq_chip_ack_parent,
 	.irq_mask		= irq_chip_mask_parent,
 	.irq_unmask		= irq_chip_unmask_parent,
 	.irq_set_affinity	= irq_chip_set_affinity_parent,
+	.irq_compose_msi_msg	= riscv_iommu_ir_compose_msi_msg,
 };
 
 static int riscv_iommu_ir_irq_domain_alloc_irqs(struct irq_domain *irqdomain,
@@ -267,10 +306,12 @@ int riscv_iommu_ir_attach_paging_domain(struct iommu_domain *iommu_domain, struc
 		if (msi_iova) {
 			guard(mutex)(&domain->mutex);
 
-			if (domain->msi_iova)
+			if (domain->msi_iova) {
 				kfree(msi_iova);
-			else
-				domain->msi_iova = msi_iova;
+			} else {
+				/* Pair with smp_load_acquire() in riscv_iommu_ir_compose_msi_msg() */
+				smp_store_release(&domain->msi_iova, msi_iova);
+			}
 
 			return 0;
 		}
-- 
2.43.0