[PATCH] iommu/dma: Allocate flush queues on first use

[Michal Clapinski <mclapinski@google.com>]

iommu_dma_init_domain() allocates the per-CPU flush queue of a DMA-FQ
domain upfront. That is 10KiB per possible CPU, so on a 224-CPU machine
it costs 2.2MiB and ~570us per domain. Since every device gets probed at
boot, large servers pay this for hundreds of domains in pci_iommu_init(),
even though many of them (PCIe ports, CPU uncore devices, ...) will
never do streaming DMA.

dma-iommu already copes with a flush queue appearing on a live domain,
as the sysfs DMA -> DMA-FQ transition does just that. So only allocate
it on the domain's first unmap, from a workqueue since unmaps can happen
in atomic context, and use strict invalidation until it is in place. If
the allocation fails, downgrade the domain to IOMMU_DOMAIN_DMA as before.

On a 224-CPU Sapphire Rapids server with 431 IOMMU groups, this cuts the
time spent in pci_iommu_init() from 330ms to 83ms.

Signed-off-by: Michal Clapinski <mclapinski@google.com>
---
 drivers/iommu/dma-iommu.c | 78 ++++++++++++++++++++++++++++++++++++---
 1 file changed, 72 insertions(+), 6 deletions(-)

diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 1f624f241ac1..e61cd016b424 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -56,6 +56,11 @@ struct iommu_dma_options {
 	unsigned int	fq_timeout;
 };
 
+/* Bits in iommu_dma_cookie::fq_flags */
+enum {
+	IOMMU_DMA_FQ_REQUESTED,		/* fq_work has been queued */
+};
+
 struct iommu_dma_cookie {
 	struct iova_domain iovad;
 	struct list_head msi_page_list;
@@ -74,6 +79,10 @@ struct iommu_dma_cookie {
 	atomic_t fq_timer_on;
 	/* Domain for flush queue callback; NULL if flush queue not in use */
 	struct iommu_domain *fq_domain;
+	/* Domain to allocate a flush queue for on its first unmap, if any */
+	struct iommu_domain *fq_lazy_domain;
+	unsigned long fq_flags;
+	struct work_struct fq_work;
 	/* Options for dma-iommu use */
 	struct iommu_dma_options options;
 };
@@ -330,7 +339,12 @@ static int iommu_dma_init_fq_percpu(struct iommu_dma_cookie *cookie)
 	return 0;
 }
 
-/* sysfs updates are serialised by the mutex of the group owning @domain */
+/*
+ * sysfs updates are serialised by the mutex of the group owning @domain. They
+ * only ever target a domain of type IOMMU_DOMAIN_DMA. iommu_dma_fq_work(), the
+ * only other caller, acts on a domain of type IOMMU_DOMAIN_DMA_FQ and only
+ * changes it to IOMMU_DOMAIN_DMA once it is done, so they can't race.
+ */
 int iommu_dma_init_fq(struct iommu_domain *domain)
 {
 	struct iommu_dma_cookie *cookie = domain->iova_cookie;
@@ -363,6 +377,48 @@ int iommu_dma_init_fq(struct iommu_domain *domain)
 	return 0;
 }
 
+static void iommu_dma_fq_work(struct work_struct *work)
+{
+	struct iommu_dma_cookie *cookie = container_of(work,
+						       struct iommu_dma_cookie,
+						       fq_work);
+	struct iommu_domain *domain = cookie->fq_lazy_domain;
+
+	/*
+	 * On failure, fall back to strict invalidation as iommu_dma_init_domain()
+	 * would, and say so in the domain type, so that it is reported correctly
+	 * and writing DMA-FQ to the group's sysfs type can retry. That only acts
+	 * on IOMMU_DOMAIN_DMA domains, so it can't run concurrently with us.
+	 * Don't take the group mutex here: the group may be waiting for us in
+	 * iommu_put_dma_cookie() with it held.
+	 */
+	if (iommu_dma_init_fq(domain))
+		WRITE_ONCE(domain->type, IOMMU_DOMAIN_DMA);
+}
+
+/*
+ * A flush queue is a sizeable per-CPU allocation, but it is only of any use
+ * once a driver unmaps DMA through the domain. Many domains never get there,
+ * e.g. those of bridges and other devices that are probed by the IOMMU core
+ * but never do DMA, so set the flush queue up on the first unmap rather than
+ * when the domain is initialised. Unmaps keep using strict invalidation until
+ * the flush queue is in place. This may be called from atomic context, so do
+ * the allocation from a workqueue.
+ *
+ * The work is queued at most once per domain. Check the flag before trying to
+ * set it, so that concurrent unmaps don't all have to take its cache line
+ * exclusively until the flush queue is in place.
+ */
+static void iommu_dma_request_fq(struct iommu_dma_cookie *cookie)
+{
+	if (likely(!cookie->fq_lazy_domain) ||
+	    test_bit(IOMMU_DMA_FQ_REQUESTED, &cookie->fq_flags) ||
+	    test_and_set_bit(IOMMU_DMA_FQ_REQUESTED, &cookie->fq_flags))
+		return;
+
+	queue_work(system_dfl_wq, &cookie->fq_work);
+}
+
 /**
  * iommu_get_dma_cookie - Acquire DMA-API resources for a domain
  * @domain: IOMMU domain to prepare for DMA-API usage
@@ -379,6 +435,7 @@ int iommu_get_dma_cookie(struct iommu_domain *domain)
 		return -ENOMEM;
 
 	INIT_LIST_HEAD(&cookie->msi_page_list);
+	INIT_WORK(&cookie->fq_work, iommu_dma_fq_work);
 	domain->cookie_type = IOMMU_COOKIE_DMA_IOVA;
 	domain->iova_cookie = cookie;
 	return 0;
@@ -428,6 +485,7 @@ void iommu_put_dma_cookie(struct iommu_domain *domain)
 	struct iommu_dma_msi_page *msi, *tmp;
 
 	if (cookie->iovad.granule) {
+		cancel_work_sync(&cookie->fq_work);
 		iommu_dma_free_fq(cookie);
 		put_iova_domain(&cookie->iovad);
 	}
@@ -715,11 +773,17 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, struct device *dev
 
 	iommu_dma_init_options(&cookie->options, dev);
 
-	/* If the FQ fails we can simply fall back to strict mode */
-	if (domain->type == IOMMU_DOMAIN_DMA_FQ &&
-	    (!iommu_domain_supports_fq(dev, domain) ||
-	     iommu_dma_init_fq(domain)))
-		domain->type = IOMMU_DOMAIN_DMA;
+	/*
+	 * The flush queue is allocated on first use, see iommu_dma_request_fq().
+	 * If it can't be used, or its allocation fails, we simply stay in
+	 * strict mode.
+	 */
+	if (domain->type == IOMMU_DOMAIN_DMA_FQ) {
+		if (iommu_domain_supports_fq(dev, domain))
+			cookie->fq_lazy_domain = domain;
+		else
+			domain->type = IOMMU_DOMAIN_DMA;
+	}
 
 	return iova_reserve_iommu_regions(dev, domain);
 }
@@ -835,6 +899,8 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
 	size = iova_align(iovad, size + iova_off);
 	iommu_iotlb_gather_init(&iotlb_gather);
 	iotlb_gather.queued = READ_ONCE(cookie->fq_domain);
+	if (!iotlb_gather.queued)
+		iommu_dma_request_fq(cookie);
 
 	unmapped = iommu_unmap_fast(domain, dma_addr, size, &iotlb_gather);
 	WARN_ON(unmapped != size);
-- 
2.56.0.212.g129ede20d9-goog