[PATCH v5 04/15] iommu/arm-smmu-v3: Drain in-flight fault events on domain detach

[Nicolin Chen <nicolinc@nvidia.com>]

When a device leaves a domain, fault events for the old domain may remain
in the SMMU event queue or the IOPF workqueue. If the IOMMU core frees that
domain before those events are handled, the work may use freed memory.

Start with the hardware queue by using arm_smmu_wait_for_queue_drained() to
count entries consumed by the threaded IRQ handler, and poll the EVTQ when
an IOPF-enabled attachment ends. This prevents a pending IRQ from queuing
old-domain work after the drain. Its until_empty mode can drain the CMDQ as
well during suspend and runtime PM.

queue_poll() cannot be used because it is an atomic busy-wait that expects
hardware to consume entries. The EVTQ and PRIQ are drained by threaded IRQ
handlers, so a busy-wait could starve a handler sharing the same CPU on a
non-preemptible kernel. The new helper sleeps, and might_sleep() catches an
atomic-context caller even when the queue is already empty.

Sample the deadline ahead of the queue state that it judges, and honour it
only after the exit conditions, so a drain that completes while this poll
is preempted still returns success instead of a spurious timeout. This is
the same ordering that poll_timeout_us() guarantees.

Note that a drained event is dequeued, but not necessarily handled, since
queue_remove_raw() moves the MMIO CONS before the threaded IRQ handler gets
to push the event onto the IOPF workqueue. A subsequent change will invoke
synchronize_irq() and iopf_queue_flush_dev() to close that gap, and it will
act on the errno of a timed-out drain too.

Fixes: cfea71aea921 ("iommu/arm-smmu-v3: Put iopf enablement in the domain attach path")
Cc: stable@vger.kernel.org # v6.16
Reviewed-by: Jonathan Cameron <jonathan.cameron@oss.qualcomm.com>
Assisted-by: LLM
Signed-off-by: Nicolin Chen <nicolinc@nvidia.com>
---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h |   2 +
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 101 ++++++++++++++++++++
 2 files changed, 103 insertions(+)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index de7e4284658a1..21b00b9296b31 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -189,6 +189,8 @@ struct arm_vsmmu;
 #define Q_WRP(llq, p)			((p) & (1 << (llq)->max_n_shift))
 /* A position is Q_WRP | Q_IDX, wrapping at twice the queue capacity */
 #define Q_POS(llq, p)			(Q_WRP(llq, p) | Q_IDX(llq, p))
+/* Entries between two positions, i.e. how far @b leads @a */
+#define Q_DIFF(llq, a, b)		Q_POS(llq, (b) - (a))
 #define Q_OVERFLOW_FLAG			(1U << 31)
 #define Q_OVF(p)			((p) & Q_OVERFLOW_FLAG)
 #define Q_ENT(q, p)			((q)->base +			\
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index b908a8af31442..9e657d075f387 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -948,6 +948,97 @@ static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,
 					   cmds->num, true);
 }
 
+/**
+ * arm_smmu_wait_for_queue_drained - Wait for an SMMU queue to be drained
+ * @smmu: the SMMU device
+ * @q: the queue to be drained
+ * @until_empty: target selection
+ *
+ * With @until_empty == true (for CMDQ), exit once the queue is observed empty:
+ *
+ *   cons0                cons                                prod
+ *     |                   |                                   |
+ *  ---+###################+===================================+--->
+ *                         |<--------- undrained==0? --------->|
+ *
+ * With @until_empty == false (for EVTQ/PRIQ), exit once "drained" reaches its
+ * target: "pending" (i.e. prod0 - cons0, frozen at the entry time):
+ *
+ *   cons0                cons                 prod0         (prod)
+ *     |<---- drained ---->|                     |             |
+ *  ---+###################+=====================+=============+--->
+ *     |<--------------- pending --------------->|
+ *
+ * Note that a drained entry is dequeued, but not necessarily handled: the
+ * EVTQ/PRIQ callers must follow up with a synchronize_irq() to wait for the
+ * threaded IRQ handler to finish handling the dequeued entries.
+ *
+ * Context: Process context; may sleep.
+ * Return: 0 on success or a negative errno on timeout.
+ */
+static int arm_smmu_wait_for_queue_drained(struct arm_smmu_device *smmu,
+					   struct arm_smmu_queue *q,
+					   bool until_empty)
+{
+	ktime_t timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
+	u32 cons, prod, pending;
+	u32 drained = 0;
+
+	might_sleep();
+
+	cons = readl_relaxed(q->cons_reg);
+	prod = readl_relaxed(q->prod_reg);
+	/* The exit target: the number of entries in the queue at entry */
+	pending = Q_DIFF(&q->llq, cons, prod);
+
+	while (true) {
+		u32 prev, undrained;
+		bool expired;
+
+		/*
+		 * Sample the deadline ahead of the queue state it judges, but
+		 * break only after the exit conditions below, so a queue that
+		 * drained during a long preemption still exits with a success.
+		 */
+		expired = ktime_compare(ktime_get(), timeout) > 0;
+
+		/* Accumulate the entries consumed since the last poll */
+		prev = cons;
+		cons = readl_relaxed(q->cons_reg);
+		drained += Q_DIFF(&q->llq, prev, cons);
+
+		prod = readl_relaxed(q->prod_reg);
+		undrained = Q_DIFF(&q->llq, cons, prod);
+
+		/* Exit on an empty queue, regardless of until_empty */
+		if (!undrained)
+			return 0;
+
+		/* Snapshot mode: exit once the pending entries are drained */
+		if (!until_empty && drained >= pending)
+			return 0;
+
+		/*
+		 * A timeout means the consumer might be stuck. In theory, if it
+		 * moves 2 * qsize entries or more within a single poll interval
+		 * Q_DIFF() will wrap and undercount drained: that could trigger
+		 * a spurious warning too, if the queue was never once observed
+		 * empty. Yet, that much consumption in such a short interval is
+		 * unrealistic.
+		 */
+		if (expired)
+			break;
+
+		/* The consumer might be a threaded IRQ handler. Yield to it */
+		fsleep(100);
+	}
+
+	dev_warn_ratelimited(smmu->dev,
+			     "queue drain timed out at prod=0x%x cons=0x%x\n",
+			     prod, cons);
+	return -ETIMEDOUT;
+}
+
 static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused,
 				   struct iommu_page_response *resp)
 {
@@ -3318,12 +3409,22 @@ void arm_smmu_attach_release(struct arm_smmu_attach_state *state)
 {
 	struct arm_smmu_master_domain *master_domain = state->old_master_domain;
 	struct arm_smmu_master *master = state->master;
+	struct arm_smmu_device *smmu = master->smmu;
 
+	lockdep_assert_not_held(&arm_smmu_asid_lock);
 	iommu_group_mutex_assert(master->dev);
 
 	if (!master_domain)
 		return;
 
+	/*
+	 * In-flight fault work references the old domain via its attach handle,
+	 * which the IOMMU core might free once this returns. Drain the hardware
+	 * eventq, so that a pending event cannot turn into new fault work.
+	 */
+	if (master_domain->using_iopf && master->stall_enabled)
+		arm_smmu_wait_for_queue_drained(smmu, &smmu->evtq.q, false);
+
 	arm_smmu_disable_iopf(master, master_domain);
 	kfree(master_domain);
 	state->old_master_domain = NULL;
-- 
2.43.0