[PATCH v2 11/11] iommu/arm-smmu-v3: Enable PRI for PCI device in arm_smmu_probe_device()

[Nicolin Chen <nicolinc@nvidia.com>]

Now PRI requests can be correctly handled. Enable the PCI cap when probing
a PCI device. Also flush the priq in arm_smmu_attach_release().

Set the per-device outstanding request budget to the full priq depth, same
as intel-iommu's per-device PRQ_DEPTH choice. A fixed per-device cap won't
prevent multiple PRI-capable devices from potentially exceeding the priq's
capacity; priq overflow is recoverable per the SMMUv3 spec, and it is rare
in practice.

select PCI_PRI in Kconfig like other IOMMUs, gated on PCI so the build can
stay clean for non-PCI ARM SMMUv3 configurations.

Signed-off-by: Nicolin Chen <nicolinc@nvidia.com>
---
 drivers/iommu/arm/Kconfig                   |  1 +
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 21 +++++++++++++++++++--
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/drivers/iommu/arm/Kconfig b/drivers/iommu/arm/Kconfig
index 5fac08b89deea..855934d08f866 100644
--- a/drivers/iommu/arm/Kconfig
+++ b/drivers/iommu/arm/Kconfig
@@ -79,6 +79,7 @@ config ARM_SMMU_V3
 	select IOMMU_API
 	select IOMMU_IO_PGTABLE_LPAE
 	select GENERIC_MSI_IRQ
+	select PCI_PRI if PCI
 	select IOMMUFD_DRIVER if IOMMUFD
 	help
 	  Support for implementations of the ARM System MMU architecture
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 87e4880a145f1..648db9a24b582 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -3149,7 +3149,7 @@ static int arm_smmu_enable_iopf(struct arm_smmu_master *master,
 	 * device-specific fault handlers and don't need IOPF, so this is not a
 	 * failure.
 	 */
-	if (!master->stall_enabled)
+	if (!master->stall_enabled && !master->pri_enabled)
 		return 0;
 
 	/* We're not keeping track of SIDs in fault events */
@@ -3352,6 +3352,12 @@ void arm_smmu_attach_release(struct arm_smmu_attach_state *state)
 			if (smmu->evtq.q.irq)
 				synchronize_irq(smmu->evtq.q.irq);
 		}
+		/* Drain the hardware priq */
+		if (master->pri_enabled) {
+			arm_smmu_drain_queue_for_iopf(smmu, &smmu->priq.q);
+			if (smmu->priq.q.irq)
+				synchronize_irq(smmu->priq.q.irq);
+		}
 		/* Pending events might be in the combined_irq handler */
 		if (smmu->combined_irq)
 			synchronize_irq(smmu->combined_irq);
@@ -4282,8 +4288,17 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev)
 
 	if (dev_is_pci(dev)) {
 		unsigned int stu = __ffs(smmu->pgsize_bitmap);
+		struct pci_dev *pdev = to_pci_dev(dev);
 
-		pci_prepare_ats(to_pci_dev(dev), stu);
+		if (!pci_prepare_ats(pdev, stu) && pci_pri_supported(pdev) &&
+		    (smmu->features & ARM_SMMU_FEAT_PRI) && smmu->evtq.iopf) {
+			unsigned int reqs = 1 << smmu->priq.q.llq.max_n_shift;
+
+			if (!pci_reset_pri(pdev) && !pci_enable_pri(pdev, reqs))
+				master->pri_enabled = true;
+			else
+				dev_warn(master->dev, "failed to enable PRI\n");
+		}
 	}
 
 	return &smmu->iommu;
@@ -4299,6 +4314,8 @@ static void arm_smmu_release_device(struct device *dev)
 
 	WARN_ON(master->iopf_refcount);
 
+	if (master->pri_enabled)
+		pci_disable_pri(to_pci_dev(master->dev));
 	arm_smmu_disable_pasid(master);
 	arm_smmu_remove_master(master);
 	if (arm_smmu_cdtab_allocated(&master->cd_table))
-- 
2.43.0