[PATCH v2 3/7] dma-mapping: Keep DMA memory private for capable devices

[Nicolin Chen <nicolinc@nvidia.com>]

Architecture code can require devices to use unencrypted memory. This makes
the DMA API use shared memory for coherent allocations and bounce streaming
mappings through SWIOTLB.

A device marked with DEV_FLAG_DMA_CC_PRIVATE can reach private memory
directly. Add dma_require_decrypted() to combine force_dma_unencrypted()
with the new device flag, and use it throughout the generic DMA code.

Signed-off-by: Nicolin Chen <nicolinc@nvidia.com>
---
 include/linux/dma-direct.h |  7 ++++++-
 kernel/dma/direct.c        | 16 ++++++++--------
 kernel/dma/mapping.c       |  2 +-
 kernel/dma/swiotlb.c       |  2 +-
 4 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h
index daa31a1adf7b0..fb35ca27941cc 100644
--- a/include/linux/dma-direct.h
+++ b/include/linux/dma-direct.h
@@ -134,6 +134,11 @@ static inline bool force_dma_unencrypted(struct device *dev)
 }
 #endif /* CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED */
 
+static inline bool dma_require_decrypted(struct device *dev)
+{
+	return !dev_dma_cc_private(dev) && force_dma_unencrypted(dev);
+}
+
 static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size,
 		bool is_ram, unsigned long attrs)
 {
@@ -146,7 +151,7 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size,
 	 * requires unencrypted DMA addresses. Treat it as not DMA-capable
 	 * so the caller can fall back to a suitable SWIOTLB pool.
 	 */
-	if (!(attrs & DMA_ATTR_CC_SHARED) && force_dma_unencrypted(dev))
+	if (!(attrs & DMA_ATTR_CC_SHARED) && dma_require_decrypted(dev))
 		return false;
 
 	if (is_ram && !IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) &&
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 25e9a86835fd3..b3b9ff090c590 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -41,7 +41,7 @@ static inline struct page *dma_direct_to_page(struct device *dev,
 
 u64 dma_direct_get_required_mask(struct device *dev)
 {
-	bool require_decrypted = force_dma_unencrypted(dev);
+	bool require_decrypted = dma_require_decrypted(dev);
 	phys_addr_t phys = ((phys_addr_t)max_pfn << PAGE_SHIFT) - 1;
 	u64 max_dma = phys_to_dma_direct(dev, phys, require_decrypted);
 
@@ -72,7 +72,7 @@ static gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 *phys_limit)
 
 bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
 {
-	bool require_decrypted = force_dma_unencrypted(dev);
+	bool require_decrypted = dma_require_decrypted(dev);
 	dma_addr_t dma_addr = phys_to_dma_direct(dev, phys, require_decrypted);
 
 	if (dma_addr == DMA_MAPPING_ERROR)
@@ -226,7 +226,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 	struct page *page;
 	void *cpu_addr;
 
-	if (force_dma_unencrypted(dev))
+	if (dma_require_decrypted(dev))
 		attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
 
 	if (attrs & __DMA_ATTR_ALLOC_CC_SHARED) {
@@ -385,7 +385,7 @@ void dma_direct_free(struct device *dev, size_t size,
 	 * If the allocation used decrypted/shared backing pages, restore
 	 * the encryption state on free.
 	 */
-	if (force_dma_unencrypted(dev))
+	if (dma_require_decrypted(dev))
 		attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
 
 	if (attrs & __DMA_ATTR_ALLOC_CC_SHARED)
@@ -452,7 +452,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
 	struct page *page;
 	void *cpu_addr;
 
-	if (force_dma_unencrypted(dev))
+	if (dma_require_decrypted(dev))
 		attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
 
 	if ((attrs & __DMA_ATTR_ALLOC_CC_SHARED) && dma_direct_use_pool(dev, gfp))
@@ -496,7 +496,7 @@ void dma_direct_free_pages(struct device *dev, size_t size,
 	 * if the device had requested for an unencrypted buffer,
 	 * convert it to encrypted on free
 	 */
-	bool mark_mem_encrypted = force_dma_unencrypted(dev);
+	bool mark_mem_encrypted = dma_require_decrypted(dev);
 
 	/* If page is not from an atomic pool, dma_free_from_pool_page() fails */
 	if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
@@ -665,7 +665,7 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma,
 	const pgoff_t pgoff_end = vma_end_pgoff(vma);
 	int ret = -ENXIO;
 
-	if (force_dma_unencrypted(dev))
+	if (dma_require_decrypted(dev))
 		attrs |= DMA_ATTR_CC_SHARED;
 
 	vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);
@@ -814,7 +814,7 @@ size_t dma_direct_max_mapping_size(struct device *dev)
 	/* If SWIOTLB is active, use its maximum mapping size */
 	if (is_swiotlb_active(dev) &&
 	    (dma_addressing_limited(dev) || is_swiotlb_force_bounce(dev) ||
-	     force_dma_unencrypted(dev)))
+	     dma_require_decrypted(dev)))
 		return swiotlb_max_mapping_size(dev);
 
 	return SIZE_MAX;
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
index ded585b39fb92..0252473edd0b5 100644
--- a/kernel/dma/mapping.c
+++ b/kernel/dma/mapping.c
@@ -673,7 +673,7 @@ void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
 		return NULL;
 	}
 
-	if (force_dma_unencrypted(dev))
+	if (dma_require_decrypted(dev))
 		attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
 
 	if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) {
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 2c57b49ac7c06..ddfc4533934a2 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -1550,7 +1550,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
 	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
 
 		/* swiotlb pool is incorrect for this device */
-		if (unlikely(mem->cc_shared != force_dma_unencrypted(dev)))
+		if (unlikely(mem->cc_shared != dma_require_decrypted(dev)))
 			return (phys_addr_t)DMA_MAPPING_ERROR;
 
 	} else if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) {
-- 
2.43.0