[RFC PATCH kernel 08/17] x86/dma: Add ARCH_HAS_PHYS_TO_DMA

[Alexey Kardashevskiy <aik@amd.com>]

Confidential VMs work with both trusted and legacy (untrusted devices).
The trusted devices allow DMA to/from encrypted guest memory while legacy
devices have to use SWIOTLB or share pages for DMA.

In case of P2P between trusted and legacy devices, the trusted device has
to perform DMA with T=1 (a PCIe IDE TLP bit saying "encrypt") but the target
may be shared.

One way of allowing the above on AMD SEV is using IOMMU sDTE vTOM feature
which is a bus address:
- below that address all accesses target private guest memory;
- above - shared memory.

Effectively this adds some high bit in a DMA handle to mark "shared" (on
AMD).

Copy the generic phys_to_dma / dma_to_phys helpers from
include/linux/dma-direct.h into arch/x86/include/asm/dma-direct.h and
select ARCH_HAS_PHYS_TO_DMA for x86.

The x86 implementation is the same as the generic code for
__phys_to_dma, dma_range_map handling, and SME encryption, with two
additions for confidential devices:
 - cc_shared_dma_offset and cc_private_dma_offset in dev_archdata
 - when TDI is accepted is true, phys_to_dma() adds the shared or
   private offset (selected by DMA_ATTR_CC_SHARED), and dma_to_phys()
   subtracts it on the way back

phys_to_dma() also gains an attrs argument so dma_capable() and swiotlb can
pass mapping attributes through. Update the call sites accordingly.

Signed-off-by: Alexey Kardashevskiy <aik@amd.com>
---

DMA_ATTR_CC_SHARED or __DMA_ATTR_ALLOC_CC_SHARED?
---
 arch/x86/Kconfig                  |  1 +
 arch/x86/include/asm/device.h     |  2 +
 arch/x86/include/asm/dma-direct.h | 77 ++++++++++++++++++++
 include/linux/dma-direct.h        |  2 +-
 drivers/xen/swiotlb-xen.c         |  2 +-
 kernel/dma/swiotlb.c              |  2 +-
 6 files changed, 83 insertions(+), 3 deletions(-)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 15fd9ec5ecac..c4df431e74b2 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -112,6 +112,7 @@ config X86
 	select ARCH_HAS_UBSAN
 	select ARCH_HAS_DEBUG_WX
 	select ARCH_HAS_ZONE_DMA_SET if EXPERT
+	select ARCH_HAS_PHYS_TO_DMA
 	select ARCH_HAVE_NMI_SAFE_CMPXCHG
 	select ARCH_HAVE_EXTRA_ELF_NOTES
 	select ARCH_MEMORY_ORDER_TSO
diff --git a/arch/x86/include/asm/device.h b/arch/x86/include/asm/device.h
index 7c0a52ca2f4d..0dbc2c125522 100644
--- a/arch/x86/include/asm/device.h
+++ b/arch/x86/include/asm/device.h
@@ -3,6 +3,8 @@
 #define _ASM_X86_DEVICE_H
 
 struct dev_archdata {
+	dma_addr_t cc_shared_dma_offset;
+	dma_addr_t cc_private_dma_offset;
 };
 
 struct pdev_archdata {
diff --git a/arch/x86/include/asm/dma-direct.h b/arch/x86/include/asm/dma-direct.h
new file mode 100644
index 000000000000..37074c87bbec
--- /dev/null
+++ b/arch/x86/include/asm/dma-direct.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef ASM_X86_DMA_DIRECT_H
+#define ASM_X86_DMA_DIRECT_H 1
+
+#include <linux/pci-tsm.h>
+
+static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
+{
+	if (dev->dma_range_map)
+		return translate_phys_to_dma(dev, paddr);
+
+	return paddr;
+}
+
+static inline bool __device_cc_accepted(struct device *dev)
+{
+	if (!dev || !dev_is_pci(dev) || !to_pci_dev(dev)->tsm)
+		return false;
+
+	return test_bit(PCI_TSM_F_ACCEPT, &to_pci_dev(dev)->tsm->flags);
+}
+
+static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr, unsigned long attrs)
+{
+	if (__device_cc_accepted(dev)) {
+		if (attrs & DMA_ATTR_CC_SHARED)
+			return __phys_to_dma(dev, paddr) + dev->archdata.cc_shared_dma_offset;
+
+		return __phys_to_dma(dev, paddr) + dev->archdata.cc_private_dma_offset;
+	}
+
+	return dma_addr_encrypted(__phys_to_dma(dev, paddr));
+}
+
+static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
+{
+	phys_addr_t paddr;
+
+	if (__device_cc_accepted(dev)) {
+		if (dev->archdata.cc_shared_dma_offset &&
+		    daddr >= dev->archdata.cc_shared_dma_offset)
+			return daddr - dev->archdata.cc_shared_dma_offset;
+
+		if (dev->archdata.cc_private_dma_offset &&
+		    daddr >= dev->archdata.cc_private_dma_offset)
+			return daddr - dev->archdata.cc_private_dma_offset;
+	}
+
+	daddr = dma_addr_canonical(daddr);
+	if (dev->dma_range_map)
+		paddr = translate_dma_to_phys(dev, daddr);
+	else
+		paddr = daddr;
+
+	return paddr;
+}
+
+static inline dma_addr_t phys_to_dma_unencrypted(struct device *dev, phys_addr_t paddr)
+{
+	if (__device_cc_accepted(dev))
+		return __phys_to_dma(dev, paddr) + dev->archdata.cc_shared_dma_offset;
+
+	return dma_addr_unencrypted(__phys_to_dma(dev, paddr));
+}
+
+static inline dma_addr_t phys_to_dma_encrypted(struct device *dev, phys_addr_t paddr)
+{
+	if (__device_cc_accepted(dev))
+		return __phys_to_dma(dev, paddr) + dev->archdata.cc_private_dma_offset;
+
+	return dma_addr_encrypted(__phys_to_dma(dev, paddr));
+}
+
+#define phys_to_dma_unencrypted phys_to_dma_unencrypted
+#define phys_to_dma_encrypted phys_to_dma_encrypted
+
+#endif /* ASM_X86_DMA_DIRECT_H */
diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h
index daa31a1adf7b..616b66ccf322 100644
--- a/include/linux/dma-direct.h
+++ b/include/linux/dma-direct.h
@@ -150,7 +150,7 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size,
 		return false;
 
 	if (is_ram && !IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) &&
-	    min(addr, end) < phys_to_dma(dev, PFN_PHYS(min_low_pfn)))
+	    min(addr, end) < phys_to_dma(dev, PFN_PHYS(min_low_pfn), attrs))
 		return false;
 
 	return end <= min_not_zero(*dev->dma_mask, dev->bus_dma_limit);
diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
index e2538824ef52..915c174f8b56 100644
--- a/drivers/xen/swiotlb-xen.c
+++ b/drivers/xen/swiotlb-xen.c
@@ -55,7 +55,7 @@ static inline phys_addr_t xen_phys_to_bus(struct device *dev, phys_addr_t paddr)
 
 static inline dma_addr_t xen_phys_to_dma(struct device *dev, phys_addr_t paddr)
 {
-	return phys_to_dma(dev, xen_phys_to_bus(dev, paddr));
+	return phys_to_dma(dev, xen_phys_to_bus(dev, paddr), 0);
 }
 
 static inline phys_addr_t xen_bus_to_phys(struct device *dev,
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index ded7016a46a7..d7c7d15740ae 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -1764,7 +1764,7 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
 	phys_addr_t swiotlb_addr;
 	dma_addr_t dma_addr;
 
-	trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size);
+	trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr, attrs), size);
 
 	swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, 0, dir, &attrs);
 	if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
-- 
2.55.0