[RFC 1/2] dma-mapping: Export arm_iommu_{alloc,free}_iova() functions

[RFC 1/2] dma-mapping: Export arm_iommu_{alloc,free}_iova() functions

[Hiroshi DOYU]

Export __{alloc,free}_iova() as arm_iommu_{alloc,free}_iova().

There are some cases that IOVA allocation and mapping have to be done
seperately, especially for perf optimization reasons. This patch
allows client modules to {alloc,free} IOVA space by themselves without
backing up actual pages for that area.

Signed-off-by: Hiroshi DOYU <hdoyu@nvidia.com>
---
 arch/arm/include/asm/dma-iommu.h |    4 ++++
 arch/arm/mm/dma-mapping.c        |   31 +++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 0 deletions(-)

diff --git a/arch/arm/include/asm/dma-iommu.h b/arch/arm/include/asm/dma-iommu.h
index 799b094..2595928 100644
--- a/arch/arm/include/asm/dma-iommu.h
+++ b/arch/arm/include/asm/dma-iommu.h
@@ -30,5 +30,9 @@ void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping);
 int arm_iommu_attach_device(struct device *dev,
 					struct dma_iommu_mapping *mapping);
 
+dma_addr_t arm_iommu_alloc_iova(struct device *dev, size_t size);
+
+void arm_iommu_free_iova(struct device *dev, dma_addr_t addr, size_t size);
+
 #endif /* __KERNEL__ */
 #endif
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index afb5e7a..bca1715 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -1041,6 +1041,21 @@ static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,
 	return mapping->base + (start << (mapping->order + PAGE_SHIFT));
 }
 
+/**
+ * arm_iommu_alloc_iova
+ * @dev: valid struct device pointer
+ * @size: size of buffer to allocate
+ *
+ * Allocate IOVA address range
+ */
+dma_addr_t arm_iommu_alloc_iova(struct device *dev, size_t size)
+{
+	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+
+	return __alloc_iova(mapping, size);
+}
+EXPORT_SYMBOL_GPL(arm_iommu_alloc_iova);
+
 static inline void __free_iova(struct dma_iommu_mapping *mapping,
 			       dma_addr_t addr, size_t size)
 {
@@ -1055,6 +1070,22 @@ static inline void __free_iova(struct dma_iommu_mapping *mapping,
 	spin_unlock_irqrestore(&mapping->lock, flags);
 }
 
+/**
+ * arm_iommu_free_iova
+ * @dev: valid struct device pointer
+ * @iova: iova address being free'ed
+ * @size: size of buffer to allocate
+ *
+ * Free IOVA address range
+ */
+void arm_iommu_free_iova(struct device *dev, dma_addr_t addr, size_t size)
+{
+	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+
+	__free_iova(mapping, addr, size);
+}
+EXPORT_SYMBOL_GPL(arm_iommu_free_iova);
+
 static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
 {
 	struct page **pages;
-- 
1.7.5.4

[RFC 2/2] dma-mapping: Enable IOVA mapping with specific address

[Hiroshi DOYU]

Enable IOVA (un)mapping at a specific IOVA address, independent of
allocating/freeing IOVA area, introducing the following
dma_(un)map_page_*at*() functions:

	dma_map_page_at()
	dma_unmap_page_at()

The above create a mapping between pre-allocated iova and a page, and
remov just a mapping, leaving iova itself allocated. At mapping, it
also checks if IOVA is already reserved or not.

There are the version with the prefix "arm_iommu_", and they are
exactly same as the above.

Signed-off-by: Hiroshi DOYU <hdoyu@nvidia.com>
---
 arch/arm/include/asm/dma-iommu.h   |   29 +++++++-
 arch/arm/include/asm/dma-mapping.h |    1 +
 arch/arm/mm/dma-mapping.c          |  158 +++++++++++++++++++++++++++---------
 3 files changed, 150 insertions(+), 38 deletions(-)

diff --git a/arch/arm/include/asm/dma-iommu.h b/arch/arm/include/asm/dma-iommu.h
index 2595928..99eba3d 100644
--- a/arch/arm/include/asm/dma-iommu.h
+++ b/arch/arm/include/asm/dma-iommu.h
@@ -30,9 +30,36 @@ void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping);
 int arm_iommu_attach_device(struct device *dev,
 					struct dma_iommu_mapping *mapping);
 
-dma_addr_t arm_iommu_alloc_iova(struct device *dev, size_t size);
+dma_addr_t arm_iommu_alloc_iova_at(struct device *dev, dma_addr_t addr,
+				size_t size);
+
+static inline dma_addr_t arm_iommu_alloc_iova(struct device *dev, size_t size)
+{
+	return arm_iommu_alloc_iova_at(dev, DMA_ANON_ADDR, size);
+}
 
 void arm_iommu_free_iova(struct device *dev, dma_addr_t addr, size_t size);
 
+dma_addr_t arm_iommu_map_page_at(struct device *dev, struct page *page,
+			 dma_addr_t addr, unsigned long offset, size_t size,
+			 enum dma_data_direction dir, struct dma_attrs *attrs);
+
+static inline dma_addr_t dma_map_page_at(struct device *d, struct page *p,
+					 dma_addr_t a, size_t o, size_t s,
+					 enum dma_data_direction r)
+{
+	return arm_iommu_map_page_at(d, p, a, o, s, r, 0);
+}
+
+void arm_iommu_unmap_page_at(struct device *dev, dma_addr_t handle,
+			     size_t size, enum dma_data_direction dir,
+			     struct dma_attrs *attrs);
+
+static inline void dma_unmap_page_at(struct device *d, dma_addr_t a, size_t s,
+				     enum dma_data_direction r)
+{
+	return arm_iommu_unmap_page_at(d, a, s, r, 0);
+}
+
 #endif /* __KERNEL__ */
 #endif
diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h
index bbef15d..b73eb73 100644
--- a/arch/arm/include/asm/dma-mapping.h
+++ b/arch/arm/include/asm/dma-mapping.h
@@ -12,6 +12,7 @@
 #include <asm/memory.h>
 
 #define DMA_ERROR_CODE	(~0)
+#define DMA_ANON_ADDR	(~0)
 extern struct dma_map_ops arm_dma_ops;
 
 static inline struct dma_map_ops *get_dma_ops(struct device *dev)
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index bca1715..b98e668 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -1013,48 +1013,65 @@ fs_initcall(dma_debug_do_init);
 
 /* IOMMU */
 
-static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,
-				      size_t size)
+static dma_addr_t __alloc_iova_at(struct dma_iommu_mapping *mapping,
+				     dma_addr_t iova, size_t size)
 {
 	unsigned int order = get_order(size);
 	unsigned int align = 0;
-	unsigned int count, start;
+	unsigned int count, start, orig = 0;
 	unsigned long flags;
+	bool anon = (iova == DMA_ANON_ADDR) ? true : false;
 
 	count = ((PAGE_ALIGN(size) >> PAGE_SHIFT) +
 		 (1 << mapping->order) - 1) >> mapping->order;
 
-	if (order > mapping->order)
+	if (anon && (order > mapping->order))
 		align = (1 << (order - mapping->order)) - 1;
 
 	spin_lock_irqsave(&mapping->lock, flags);
-	start = bitmap_find_next_zero_area(mapping->bitmap, mapping->bits, 0,
-					   count, align);
-	if (start > mapping->bits) {
-		spin_unlock_irqrestore(&mapping->lock, flags);
-		return DMA_ERROR_CODE;
-	}
+	if (!anon)
+		orig = (iova - mapping->base) >> (mapping->order + PAGE_SHIFT);
+
+	start = bitmap_find_next_zero_area(mapping->bitmap, mapping->bits,
+					   orig, count, align);
+	if (start > mapping->bits)
+		goto not_found;
+
+	if (!anon && (orig != start))
+		goto not_found;
 
 	bitmap_set(mapping->bitmap, start, count);
 	spin_unlock_irqrestore(&mapping->lock, flags);
 
 	return mapping->base + (start << (mapping->order + PAGE_SHIFT));
+
+not_found:
+	spin_unlock_irqrestore(&mapping->lock, flags);
+	return DMA_ERROR_CODE;
+}
+
+static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,
+				      size_t size)
+{
+	return __alloc_iova_at(mapping, DMA_ANON_ADDR, size);
 }
 
 /**
- * arm_iommu_alloc_iova
+ * arm_iommu_alloc_iova_at
  * @dev: valid struct device pointer
+ * @iova: iova address being requested. Set DMA_ANON_ADDR for arbitral
  * @size: size of buffer to allocate
  *
  * Allocate IOVA address range
  */
-dma_addr_t arm_iommu_alloc_iova(struct device *dev, size_t size)
+dma_addr_t arm_iommu_alloc_iova_at(struct device *dev, dma_addr_t iova,
+				size_t size)
 {
 	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
 
-	return __alloc_iova(mapping, size);
+	return __alloc_iova_at(mapping, iova, size);
 }
-EXPORT_SYMBOL_GPL(arm_iommu_alloc_iova);
+EXPORT_SYMBOL_GPL(arm_iommu_alloc_iova_at);
 
 static inline void __free_iova(struct dma_iommu_mapping *mapping,
 			       dma_addr_t addr, size_t size)
@@ -1507,6 +1524,41 @@ void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
 			__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
 }
 
+static dma_addr_t __arm_iommu_map_page_at(struct device *dev, struct page *page,
+			  dma_addr_t req, unsigned long offset, size_t size,
+			  enum dma_data_direction dir, struct dma_attrs *attrs)
+{
+	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+	dma_addr_t dma_addr;
+	int ret, len = PAGE_ALIGN(size + offset);
+
+	if (!arch_is_coherent())
+		__dma_page_cpu_to_dev(page, offset, size, dir);
+
+	dma_addr = __alloc_iova_at(mapping, req, len);
+	if (dma_addr == DMA_ERROR_CODE) {
+		if (req == DMA_ANON_ADDR)
+			return DMA_ERROR_CODE;
+		/*
+		 * Verified that iova(req) is reserved in advance if
+		 * @req is specified.
+		 */
+		dma_addr = req;
+	}
+
+	if (req != DMA_ANON_ADDR)
+		BUG_ON(dma_addr != req);
+
+	ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, 0);
+	if (ret < 0)
+		goto fail;
+
+	return dma_addr + offset;
+fail:
+	if (req == DMA_ANON_ADDR)
+		__free_iova(mapping, dma_addr, len);
+	return DMA_ERROR_CODE;
+}
 
 /**
  * arm_iommu_map_page
@@ -1522,25 +1574,47 @@ static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
 	     unsigned long offset, size_t size, enum dma_data_direction dir,
 	     struct dma_attrs *attrs)
 {
-	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
-	dma_addr_t dma_addr;
-	int ret, len = PAGE_ALIGN(size + offset);
+	return __arm_iommu_map_page_at(dev, page, DMA_ANON_ADDR,
+				       offset, size, dir, attrs);
+}
 
-	if (!arch_is_coherent())
-		__dma_page_cpu_to_dev(page, offset, size, dir);
+/**
+ * arm_iommu_map_page_at
+ * @dev: valid struct device pointer
+ * @page: page that buffer resides in
+ * @req: iova address being requested. Set DMA_ANON_ADDR for arbitral
+ * @offset: offset into page for start of buffer
+ * @size: size of buffer to map
+ * @dir: DMA transfer direction
+ *
+ * The version with a specified iova address of arm_iommu_map_page().
+ */
+dma_addr_t arm_iommu_map_page_at(struct device *dev, struct page *page,
+		 dma_addr_t req, unsigned long offset, size_t size,
+		 enum dma_data_direction dir, struct dma_attrs *attrs)
+{
+	return __arm_iommu_map_page_at(dev, page, req, offset, size, dir,
+				       attrs);
+}
+EXPORT_SYMBOL_GPL(arm_iommu_map_page_at);
 
-	dma_addr = __alloc_iova(mapping, len);
-	if (dma_addr == DMA_ERROR_CODE)
-		return dma_addr;
+static inline int __arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
+		size_t size, enum dma_data_direction dir,
+		struct dma_attrs *attrs)
+{
+	dma_addr_t iova = handle & PAGE_MASK;
+	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
+	int offset = handle & ~PAGE_MASK;
+	int len = PAGE_ALIGN(size + offset);
 
-	ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, 0);
-	if (ret < 0)
-		goto fail;
+	if (!iova)
+		return -EINVAL;
 
-	return dma_addr + offset;
-fail:
-	__free_iova(mapping, dma_addr, len);
-	return DMA_ERROR_CODE;
+	if (!arch_is_coherent())
+		__dma_page_dev_to_cpu(page, offset, size, dir);
+
+	iommu_unmap(mapping->domain, iova, len);
+	return 0;
 }
 
 /**
@@ -1558,20 +1632,30 @@ static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
 {
 	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
 	dma_addr_t iova = handle & PAGE_MASK;
-	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
-	int offset = handle & ~PAGE_MASK;
 	int len = PAGE_ALIGN(size + offset);
 
-	if (!iova)
+	if (__arm_iommu_unmap_page(dev, handle, size, dir, attrs))
 		return;
-
-	if (!arch_is_coherent())
-		__dma_page_dev_to_cpu(page, offset, size, dir);
-
-	iommu_unmap(mapping->domain, iova, len);
 	__free_iova(mapping, iova, len);
 }
 
+/**
+ * arm_iommu_unmap_page_at
+ * @dev: valid struct device pointer
+ * @handle: DMA address of buffer
+ * @size: size of buffer (same as passed to dma_map_page)
+ * @dir: DMA transfer direction (same as passed to dma_map_page)
+ *
+ * The version without freeing iova of arm_iommu_unmap_page().
+ */
+void arm_iommu_unmap_page_at(struct device *dev, dma_addr_t handle,
+		size_t size, enum dma_data_direction dir,
+		struct dma_attrs *attrs)
+{
+	__arm_iommu_unmap_page(dev, handle, size, dir, attrs);
+}
+EXPORT_SYMBOL_GPL(arm_iommu_unmap_page_at);
+
 static void arm_iommu_sync_single_for_cpu(struct device *dev,
 		dma_addr_t handle, size_t size, enum dma_data_direction dir)
 {
-- 
1.7.5.4