[PATCH RFC v8 9/9] iommu/arm-smmu-v3: Quiesce PASID before mm release unlinks

["Peng Fan (OSS)" <peng.fan@oss.nxp.com>]

From: Peng Fan <peng.fan@nxp.com>

arm_smmu_blocking_set_dev_pasid() unlinked the PASID master_domain
before clearing the CD and invalidating the ATC. arm_smmu_mm_release()
only takes devices_lock, so it could miss the removed entry, return after
arm_smmu_domain_inv(), and let exit_mmap() free page tables while the
PASID CD was still valid and could be re-fetched.

Clear the CD and invalidate ATS before unlinking the master_domain, so
mm_release either sees the entry and writes a faulting CD itself, or
sees it removed only after the CD was invalidated.

mm_release must not change used_ssids while quiescing, otherwise a write
racing with clear_cd() can resurrect the PASID count. Add a no-account
CD write for mm_release, and re-clear the CD without accounting after
unlinking to drop a faulting CD written in the race window.

For the last-PASID downgrade, do not propagate attach_dev() failures:
the IOMMU core requires set_dev_pasid() not to fail in this path. The
private CD table is kept allocated for reuse.

Signed-off-by: Peng Fan <peng.fan@nxp.com>
---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c |  4 +-
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c     | 95 ++++++++++++++++---------
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h     |  7 +-
 3 files changed, 67 insertions(+), 39 deletions(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
index f65c60652c0d..23dbe6ff479d 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
@@ -178,8 +178,8 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
 			continue;
 		arm_smmu_make_sva_cd(&target, master, NULL,
 				     smmu_domain->cd.asid);
-		arm_smmu_write_cd_entry(master, master_domain->ssid, cdptr,
-					&target);
+		arm_smmu_write_cd_entry_no_account(master, master_domain->ssid,
+						   cdptr, &target);
 	}
 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
 
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 5a6bed0ca7f1..f22a3b6455a5 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -1675,6 +1675,26 @@ void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid,
 	arm_smmu_write_entry(&cd_writer.writer, cdptr->data, target->data);
 }
 
+/*
+ * Quiesce the CD without changing the PASID accounting. mmu_notifier release
+ * writes a valid, faulting CD while the detach path owns the used_ssids
+ * transition, so it must not resurrect the PASID count.
+ */
+void arm_smmu_write_cd_entry_no_account(struct arm_smmu_master *master, int ssid,
+					struct arm_smmu_cd *cdptr,
+					const struct arm_smmu_cd *target)
+{
+	struct arm_smmu_cd_writer cd_writer = {
+		.writer = {
+			.ops = &arm_smmu_cd_writer_ops,
+			.master = master,
+		},
+		.ssid = ssid,
+	};
+
+	arm_smmu_write_entry(&cd_writer.writer, cdptr->data, target->data);
+}
+
 void arm_smmu_make_s1_cd(struct arm_smmu_cd *target,
 			 struct arm_smmu_master *master,
 			 struct arm_smmu_domain *smmu_domain)
@@ -1734,6 +1754,23 @@ void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid)
 	arm_smmu_write_cd_entry(master, ssid, cdptr, &target);
 }
 
+static void arm_smmu_clear_cd_no_account(struct arm_smmu_master *master,
+					 ioasid_t ssid)
+{
+	struct arm_smmu_cd target = {};
+	struct arm_smmu_cd *cdptr;
+
+	if (!arm_smmu_cdtab_allocated(&master->cd_table))
+		return;
+	cdptr = arm_smmu_master_get_cd_ptr(master, ssid);
+	if (!cdptr) {
+		/* Only ats_always_on allows a NULL CD on default substream */
+		WARN_ON(!master->ats_always_on || ssid);
+		return;
+	}
+	arm_smmu_write_cd_entry_no_account(master, ssid, cdptr, &target);
+}
+
 static int arm_smmu_alloc_cd_tables(struct arm_smmu_device *smmu,
 				    struct arm_smmu_ctx_desc_cfg *cd_table,
 				    u8 ssid_bits, bool ats_always_on)
@@ -4176,61 +4213,49 @@ static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain,
 		.old_domain = old_domain,
 		.ssid = pasid,
 	};
-	int ret = 0;
 
 	mutex_lock(&arm_smmu_asid_lock);
 	arm_smmu_attach_prepare_invs(&state, NULL);
 	/*
-	 * Unlink the PASID from the domain before invalidating its CD.
-	 * arm_smmu_mm_release() walks this list with only devices_lock held
-	 * and writes a valid, faulting CD; unlinking first prevents it from
-	 * resurrecting used_ssids after the last PASID is detached.
+	 * Invalidate the CD before unlinking the PASID from the domain.
+	 * arm_smmu_mm_release() walks this list with only devices_lock held:
+	 * if it sees the master_domain it writes a valid, faulting CD; if it
+	 * does not, the CD has already been invalidated here. Either way the
+	 * page tables can be freed after mm_release() returns.
 	 */
-	state.old_master_domain = arm_smmu_remove_master_domain(
-		master, &smmu_domain->domain, pasid);
 	arm_smmu_clear_cd(master, pasid);
 	if (master->ats_enabled)
 		arm_smmu_atc_inv_master(master, pasid);
+	state.old_master_domain = arm_smmu_remove_master_domain(
+		master, &smmu_domain->domain, pasid);
+	/*
+	 * mm_release() may have written a faulting CD in the window above.
+	 * Drop it without touching used_ssids so the CD is invalid again and
+	 * a later PASID attach accounts for the install correctly.
+	 */
+	arm_smmu_clear_cd_no_account(master, pasid);
 	arm_smmu_install_old_domain_invs(&state);
 	mutex_unlock(&arm_smmu_asid_lock);
 	arm_smmu_attach_release(&state);
 
 	/*
 	 * When the last user of the CD table goes away, switch the STE back to
-	 * the RID domain's CD table (S1) or to the native bypass/abort STE
-	 * (identity/blocked). Do not call attach_dev() for an S1 paging domain:
-	 * this is a teardown path where allocation failures cannot be
-	 * propagated. The private CD table is kept allocated for reuse and
-	 * freed when the device goes away.
+	 * the RID domain's shared CD table (S1) or to the native bypass/abort
+	 * STE (identity/blocked), by re-attaching its sid_domain. Do not
+	 * propagate attach_dev() failures from this teardown path. The private
+	 * CD table is kept allocated for reuse and freed when the device goes
+	 * away.
 	 */
 	if (!arm_smmu_ssids_in_use(&master->cd_table)) {
 		struct iommu_domain *sid_domain =
 			iommu_driver_get_domain_for_dev(master->dev);
+		bool ats_always_on = master->ats_always_on &&
+				     sid_domain->type != IOMMU_DOMAIN_BLOCKED;
 
-		if (!master->ats_always_on ||
-		    sid_domain->type == IOMMU_DOMAIN_BLOCKED) {
-			if ((sid_domain->type & __IOMMU_DOMAIN_PAGING) &&
-			    to_smmu_domain(sid_domain)->stage ==
-				    ARM_SMMU_DOMAIN_S1) {
-				struct arm_smmu_ste ste;
-				struct arm_smmu_domain *sid_smmu_domain =
-					to_smmu_domain(sid_domain);
-
-				mutex_lock(&arm_smmu_asid_lock);
-				arm_smmu_make_cdtable_ste(
-					&ste, master, &sid_smmu_domain->cd_table,
-					master->ats_enabled,
-					STRTAB_STE_1_S1DSS_SSID0);
-				arm_smmu_install_ste_for_dev(master, &ste);
-				mutex_unlock(&arm_smmu_asid_lock);
-			} else if (sid_domain->type == IOMMU_DOMAIN_IDENTITY ||
-				   sid_domain->type == IOMMU_DOMAIN_BLOCKED) {
-				ret = sid_domain->ops->attach_dev(sid_domain, dev,
-								  sid_domain);
-			}
-		}
+		if (!ats_always_on)
+			sid_domain->ops->attach_dev(sid_domain, dev, sid_domain);
 	}
-	return ret;
+	return 0;
 }
 
 static int arm_smmu_attach_dev_ste(struct iommu_domain *domain,
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index cd659590dfe5..79623d8283f5 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -1096,8 +1096,8 @@ struct arm_smmu_domain {
 
 	/*
 	 * CD table shared by all non-PASID devices attached to this S1
-	 * domain. A master uses a private table only when it needs
-	 * substreams (PASID) or ATS-always-on/S1DSS.
+	 * domain. A master uses a private table when it needs substreams
+	 * (PASID), ATS-always-on/S1DSS, or the stall model.
 	 */
 	struct arm_smmu_ctx_desc_cfg	cd_table;
 
@@ -1212,6 +1212,9 @@ void arm_smmu_make_s1_cd(struct arm_smmu_cd *target,
 void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid,
 			     struct arm_smmu_cd *cdptr,
 			     const struct arm_smmu_cd *target);
+void arm_smmu_write_cd_entry_no_account(struct arm_smmu_master *master, int ssid,
+					struct arm_smmu_cd *cdptr,
+					const struct arm_smmu_cd *target);
 
 int arm_smmu_set_pasid(struct arm_smmu_master *master,
 		       struct arm_smmu_domain *smmu_domain, ioasid_t pasid,

-- 
2.34.1