[PATCH v3] iommu/iova: Move CPU magazine init to first insert

[Logan Odell <loganodell@google.com>]

A large amount of memory may be allocated for these magazines on
machines with a lot of IOMMU groups and CPU cores. Not all may be used
as some devices may be unused or be bound to drivers that do not use the
DMA-API. Furthermore, some drivers may not use all levels or CPUs.

Move the initialization of the loaded and prev magazines for each CPU on
the first attempt to try to insert a freed IOVA to them.

Signed-off-by: Logan Odell <loganodell@google.com>
Signed-off-by: Michal Clapinski <mclapinski@google.com>
---
v2: only rebase
v3: Combine the initilaization logic with new magazine logic in __iova_rcache_insert
---

Change-Id: I987b6dfb2b1125d8da8618fff540f315b99bad13
---
 drivers/iommu/iova.c | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index 021daf6528de..82d52eb12e4c 100644
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -621,6 +621,9 @@ iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad)
 	unsigned long flags;
 	int i;
 
+	if (!mag)
+		return;
+
 	spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
 
 	for (i = 0 ; i < mag->size; ++i) {
@@ -739,12 +742,6 @@ int iova_domain_init_rcaches(struct iova_domain *iovad)
 			cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
 
 			spin_lock_init(&cpu_rcache->lock);
-			cpu_rcache->loaded = iova_magazine_alloc(GFP_KERNEL);
-			cpu_rcache->prev = iova_magazine_alloc(GFP_KERNEL);
-			if (!cpu_rcache->loaded || !cpu_rcache->prev) {
-				ret = -ENOMEM;
-				goto out_err;
-			}
 		}
 	}
 
@@ -777,19 +774,23 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
 	cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);
 	spin_lock_irqsave(&cpu_rcache->lock, flags);
 
-	if (!iova_magazine_full(cpu_rcache->loaded)) {
+	if (cpu_rcache->loaded && !iova_magazine_full(cpu_rcache->loaded)) {
 		can_insert = true;
-	} else if (!iova_magazine_full(cpu_rcache->prev)) {
+	} else if (cpu_rcache->prev && !iova_magazine_full(cpu_rcache->prev)) {
 		swap(cpu_rcache->prev, cpu_rcache->loaded);
 		can_insert = true;
 	} else {
 		struct iova_magazine *new_mag = iova_magazine_alloc(GFP_ATOMIC);
 
 		if (new_mag) {
-			spin_lock(&rcache->lock);
-			iova_depot_push(rcache, cpu_rcache->loaded);
-			spin_unlock(&rcache->lock);
-			schedule_delayed_work(&rcache->work, IOVA_DEPOT_DELAY);
+			if (cpu_rcache->loaded && !cpu_rcache->prev) {
+				cpu_rcache->prev = cpu_rcache->loaded;
+			} else if (cpu_rcache->loaded) {
+				spin_lock(&rcache->lock);
+				iova_depot_push(rcache, cpu_rcache->loaded);
+				spin_unlock(&rcache->lock);
+				schedule_delayed_work(&rcache->work, IOVA_DEPOT_DELAY);
+			}
 
 			cpu_rcache->loaded = new_mag;
 			can_insert = true;
@@ -831,9 +832,9 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache,
 	cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);
 	spin_lock_irqsave(&cpu_rcache->lock, flags);
 
-	if (!iova_magazine_empty(cpu_rcache->loaded)) {
+	if (cpu_rcache->loaded && !iova_magazine_empty(cpu_rcache->loaded)) {
 		has_pfn = true;
-	} else if (!iova_magazine_empty(cpu_rcache->prev)) {
+	} else if (cpu_rcache->prev && !iova_magazine_empty(cpu_rcache->prev)) {
 		swap(cpu_rcache->prev, cpu_rcache->loaded);
 		has_pfn = true;
 	} else {
-- 
2.55.0.229.g6434b31f56-goog