[RFC PATCH 5/6] KVM: guest_memfd: Add dma-buf backing

[Fred Griffoul <griffoul@gmail.com>]

From: Fred Griffoul <fgriffo@amazon.co.uk>

A memory owner that exports a dma-buf should be able to use that one
object for both guest stage-2 mappings and device IOMMU mappings. A
second, KVM-specific backing interface would duplicate the description
of the memory layout and the rules for invalidating it.

Add GUEST_MEMFD_FLAG_USE_DMABUF and a dmabuf_fd field to
KVM_CREATE_GUEST_MEMFD. This backing has its own kvm_gmem_ops.
guest_memfd attaches to the buffer as a revocable importer without a
device. At creation, it asks for the run at offset 0 and refuses MMIO
or unknown attributes, so a buffer that KVM can never map fails early.
Offset 0 may be unbacked.

On a fault, guest_memfd takes the reservation once. It asks get_phys()
about the PUD-sized, PMD-sized and page-sized blocks around the faulting
page, in that order, and maps the first block that one aligned RAM run
covers. If the page is not backed or the run is not supported, the
fault returns -EFAULT. A READONLY run is mapped without stage-2 write
permission.

The invalidation callback only runs KVM's invalidation over the
affected range. It does not allocate memory or call the exporter. A
fault that read the layout during the change sees mmu_invalidate_seq
move and retries.

Callbacks can start as soon as guest_memfd attaches. A dma-buf-backed
file is therefore added to the inode's list of guest_memfd files under
filemap_invalidate_lock.

mmap() uses the exporter's dma-buf mmap. fallocate() and populate are
not supported. Import the DMA_BUF symbol namespace so that a
CONFIG_KVM=m build passes modpost.

Signed-off-by: Fred Griffoul <fgriffo@amazon.co.uk>
---
 include/linux/kvm_host.h       |   1 +
 include/uapi/linux/kvm.h       |   6 +-
 tools/include/uapi/linux/kvm.h |   6 +-
 virt/kvm/Kconfig               |   1 +
 virt/kvm/guest_memfd.c         | 239 +++++++++++++++++++++++++++++++--
 5 files changed, 243 insertions(+), 10 deletions(-)

diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index f84f3ab44acf..0158dc46580b 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -834,6 +834,7 @@ static inline u64 kvm_gmem_get_supported_flags(struct kvm *kvm)
 
 	if (!kvm || kvm_arch_supports_gmem_init_shared(kvm))
 		flags |= GUEST_MEMFD_FLAG_INIT_SHARED;
+	flags |= GUEST_MEMFD_FLAG_USE_DMABUF;
 
 	return flags;
 }
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index 419011097fa8..f8c83013773d 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -1654,11 +1654,15 @@ struct kvm_memory_attributes {
 #define KVM_CREATE_GUEST_MEMFD	_IOWR(KVMIO,  0xd4, struct kvm_create_guest_memfd)
 #define GUEST_MEMFD_FLAG_MMAP		(1ULL << 0)
 #define GUEST_MEMFD_FLAG_INIT_SHARED	(1ULL << 1)
+#define GUEST_MEMFD_FLAG_USE_DMABUF	(1ULL << 2)
 
 struct kvm_create_guest_memfd {
 	__u64 size;
 	__u64 flags;
-	__u64 reserved[6];
+	/* With USE_DMABUF, memory comes from this dma-buf's leading bytes. */
+	__s32 dmabuf_fd;
+	__u32 pad;
+	__u64 reserved[5];
 };
 
 #define KVM_PRE_FAULT_MEMORY	_IOWR(KVMIO, 0xd5, struct kvm_pre_fault_memory)
diff --git a/tools/include/uapi/linux/kvm.h b/tools/include/uapi/linux/kvm.h
index d0c0c8605976..ebeb2bc353ed 100644
--- a/tools/include/uapi/linux/kvm.h
+++ b/tools/include/uapi/linux/kvm.h
@@ -1644,11 +1644,15 @@ struct kvm_memory_attributes {
 #define KVM_CREATE_GUEST_MEMFD	_IOWR(KVMIO,  0xd4, struct kvm_create_guest_memfd)
 #define GUEST_MEMFD_FLAG_MMAP		(1ULL << 0)
 #define GUEST_MEMFD_FLAG_INIT_SHARED	(1ULL << 1)
+#define GUEST_MEMFD_FLAG_USE_DMABUF	(1ULL << 2)
 
 struct kvm_create_guest_memfd {
 	__u64 size;
 	__u64 flags;
-	__u64 reserved[6];
+	/* With USE_DMABUF, memory comes from this dma-buf's leading bytes. */
+	__s32 dmabuf_fd;
+	__u32 pad;
+	__u64 reserved[5];
 };
 
 #define KVM_PRE_FAULT_MEMORY	_IOWR(KVMIO, 0xd5, struct kvm_pre_fault_memory)
diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig
index 794976b88c6f..7ac4c8a371eb 100644
--- a/virt/kvm/Kconfig
+++ b/virt/kvm/Kconfig
@@ -105,6 +105,7 @@ config KVM_GENERIC_MEMORY_ATTRIBUTES
 
 config KVM_GUEST_MEMFD
        select XARRAY_MULTI
+       select DMA_SHARED_BUFFER
        bool
 
 config HAVE_KVM_ARCH_GMEM_PREPARE
diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c
index 0f4bf2cc5e8e..f401cc560eb4 100644
--- a/virt/kvm/guest_memfd.c
+++ b/virt/kvm/guest_memfd.c
@@ -1,4 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
+#include <linux/dma-buf.h>
+#include <linux/dma-resv.h>
+#include <linux/module.h>
 #include <linux/anon_inodes.h>
 #include <linux/backing-dev.h>
 #include <linux/falloc.h>
@@ -44,6 +47,8 @@ struct gmem_inode {
 	struct list_head gmem_file_list;
 
 	u64 flags;
+	struct dma_buf *dmabuf;
+	struct dma_buf_attachment *attach;
 };
 
 static __always_inline struct gmem_inode *GMEM_I(struct inode *inode)
@@ -477,21 +482,37 @@ static const struct vm_operations_struct kvm_gmem_vm_ops = {
 #endif
 };
 
-static int kvm_gmem_native_mmap(struct file *file, struct vm_area_struct *vma)
+static int kvm_gmem_validate_mmap(struct file *file,
+				  struct vm_area_struct *vma)
 {
 	if (!kvm_gmem_supports_mmap(file_inode(file)))
 		return -ENODEV;
-
 	if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) !=
-	    (VM_SHARED | VM_MAYSHARE)) {
+	    (VM_SHARED | VM_MAYSHARE))
 		return -EINVAL;
-	}
+	return 0;
+}
 
-	vma->vm_ops = &kvm_gmem_vm_ops;
+static int kvm_gmem_native_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	int ret = kvm_gmem_validate_mmap(file, vma);
 
+	if (ret)
+		return ret;
+	vma->vm_ops = &kvm_gmem_vm_ops;
 	return 0;
 }
 
+static int kvm_gmem_dmabuf_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	struct dma_buf *dmabuf = GMEM_I(file_inode(file))->dmabuf;
+	int ret = kvm_gmem_validate_mmap(file, vma);
+
+	if (ret)
+		return ret;
+	return dma_buf_mmap(dmabuf, vma, vma->vm_pgoff);
+}
+
 /* File-op dispatchers — thin: they all go through the backing's ops table. */
 
 static int kvm_gmem_fops_release(struct inode *inode, struct file *file)
@@ -617,6 +638,172 @@ bool __weak kvm_arch_supports_gmem_init_shared(struct kvm *kvm)
 	return true;
 }
 
+/* ---- a dma-buf behind the file ----------------------------------------- */
+
+/*
+ * Refuse a buffer whose first run guest_memfd can never map, such as MMIO,
+ * at creation rather than on every fault. A gap at offset 0 is legal. This
+ * is advisory: each fault checks the run it uses.
+ */
+static int gmem_dmabuf_check(struct dma_buf_attachment *attach, u64 size)
+{
+	struct phys_vec pv;
+	u32 attr;
+	int ret;
+
+	dma_resv_lock(attach->dmabuf->resv, NULL);
+	ret = dma_buf_get_phys(attach, 0, size, &pv, &attr);
+	dma_resv_unlock(attach->dmabuf->resv);
+	if (ret == -ENOENT)
+		return 0;
+	if (ret)
+		return ret;
+	if (!dma_buf_phys_attr_known(attr) ||
+	    dma_buf_phys_attr_type(attr) != DMA_BUF_PHYS_ATTR_RAM)
+		return -EOPNOTSUPP;
+	return 0;
+}
+
+static void gmem_dmabuf_invalidate_range(struct dma_buf_attachment *attach,
+					 unsigned long offset,
+					 unsigned long len)
+{
+	struct inode *inode = attach->importer_priv;
+	pgoff_t npages = i_size_read(inode) >> PAGE_SHIFT;
+	pgoff_t start = offset >> PAGE_SHIFT;
+	pgoff_t end = min_t(pgoff_t, DIV_ROUND_UP(offset + len, PAGE_SIZE),
+			    npages);
+
+	if (start >= end)
+		return;
+	/* Lock order: dma-buf reservation, file invalidation, then mmu_lock. */
+	filemap_invalidate_lock(inode->i_mapping);
+	kvm_gmem_invalidate_start(inode, start, end);
+	kvm_gmem_invalidate_end(inode, start, end);
+	filemap_invalidate_unlock(inode->i_mapping);
+}
+
+static void gmem_dmabuf_invalidate(struct dma_buf_attachment *attach)
+{
+	struct inode *inode = attach->importer_priv;
+
+	gmem_dmabuf_invalidate_range(attach, 0, i_size_read(inode));
+}
+
+static const struct dma_buf_attach_ops gmem_dmabuf_attach_ops = {
+	.allow_peer2peer	= true,
+	.invalidate_mappings	= gmem_dmabuf_invalidate,
+	.invalidate_mappings_range = gmem_dmabuf_invalidate_range,
+};
+
+static int kvm_gmem_dmabuf_attach(struct inode *inode, int dmabuf_fd)
+{
+	struct gmem_inode *gi = GMEM_I(inode);
+	struct dma_buf_attachment *attach;
+	struct dma_buf *dmabuf;
+	int ret;
+
+	dmabuf = dma_buf_get(dmabuf_fd);
+	if (IS_ERR(dmabuf))
+		return PTR_ERR(dmabuf);
+	if (dmabuf->size < i_size_read(inode)) {
+		ret = -EINVAL;
+		goto put;
+	}
+
+	gi->dmabuf = dmabuf;
+	attach = dma_buf_dynamic_attach(dmabuf, NULL, &gmem_dmabuf_attach_ops,
+					inode);
+	if (IS_ERR(attach)) {
+		ret = PTR_ERR(attach);
+		goto clear_put;
+	}
+	gi->attach = attach;
+	ret = gmem_dmabuf_check(attach, i_size_read(inode));
+	if (ret)
+		goto detach;
+	return 0;
+
+detach:
+	dma_buf_detach(dmabuf, attach);
+	gi->attach = NULL;
+clear_put:
+	gi->dmabuf = NULL;
+put:
+	dma_buf_put(dmabuf);
+	return ret;
+}
+
+static void kvm_gmem_dmabuf_release(struct inode *inode)
+{
+	struct gmem_inode *gi = GMEM_I(inode);
+
+	if (!gi->dmabuf)
+		return;
+	dma_buf_detach(gi->dmabuf, gi->attach);
+	dma_buf_put(gi->dmabuf);
+	gi->attach = NULL;
+	gi->dmabuf = NULL;
+}
+
+static int gmem_dmabuf_get_pfn(struct file *file, struct kvm *kvm,
+			       struct kvm_memory_slot *slot, gfn_t gfn,
+			       kvm_pfn_t *pfn, struct page **page,
+			       int *max_order, bool *writable)
+{
+	static const int orders[] = { PUD_ORDER, PMD_ORDER, 0 };
+	struct gmem_inode *gi = GMEM_I(file_inode(file));
+	struct gmem_file *f = gmem_file_of(file);
+	pgoff_t index = kvm_gmem_get_index(slot, gfn);
+	pgoff_t npages = i_size_read(file_inode(file)) >> PAGE_SHIFT;
+	unsigned int oi;
+	int ret = -EFAULT;
+
+	if (file != READ_ONCE(slot->gmem.file))
+		return -EFAULT;
+	if (xa_load(&f->bindings, index) != slot)
+		return -EIO;
+
+	/* A block maps at an order only if one aligned RAM run covers it. */
+	dma_resv_lock(gi->dmabuf->resv, NULL);
+	for (oi = max_order ? 0 : ARRAY_SIZE(orders) - 1;
+	     oi < ARRAY_SIZE(orders); oi++) {
+		int order = orders[oi];
+		pgoff_t bs = ALIGN_DOWN(index, 1UL << order);
+		u64 blen = (u64)PAGE_SIZE << order;
+		struct phys_vec pv;
+		u32 attr;
+
+		if (bs + (1UL << order) > npages)
+			continue;
+		if (dma_buf_get_phys(gi->attach, (u64)bs << PAGE_SHIFT, blen,
+				     &pv, &attr) ||
+		    pv.len != blen || !IS_ALIGNED(pv.paddr, blen) ||
+		    !dma_buf_phys_attr_known(attr) ||
+		    dma_buf_phys_attr_type(attr) != DMA_BUF_PHYS_ATTR_RAM)
+			continue;
+
+		*pfn = PHYS_PFN(pv.paddr) + (index - bs);
+		*page = NULL;
+		if (max_order)
+			*max_order = order;
+		if (writable && (attr & DMA_BUF_PHYS_ATTR_READONLY))
+			*writable = false;
+		ret = 0;
+		break;
+	}
+	dma_resv_unlock(gi->dmabuf->resv);
+	return ret;
+}
+
+static int kvm_gmem_dmabuf_populate(struct file *file, struct kvm *kvm,
+				    struct kvm_memory_slot *slot, gfn_t gfn,
+				    kvm_pfn_t *pfn, struct page *src_page,
+				    int order)
+{
+	return -EOPNOTSUPP;
+}
+
 static int  kvm_gmem_native_bind(struct file *file, struct kvm *kvm,
 				 struct kvm_memory_slot *slot, loff_t offset);
 static void kvm_gmem_native_unbind(struct file *slot_file, struct kvm *kvm,
@@ -654,7 +841,17 @@ static const struct kvm_gmem_ops kvm_gmem_native_ops = {
 	.fallocate = kvm_gmem_native_fallocate,
 };
 
-static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)
+static const struct kvm_gmem_ops kvm_gmem_dmabuf_ops = {
+	.bind      = kvm_gmem_native_bind,
+	.unbind    = kvm_gmem_native_unbind,
+	.get_pfn   = gmem_dmabuf_get_pfn,
+	.populate  = kvm_gmem_dmabuf_populate,
+	.release   = kvm_gmem_native_release,
+	.mmap      = kvm_gmem_dmabuf_mmap,
+};
+
+static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags,
+			     int dmabuf_fd)
 {
 	static const char *name = "[kvm-gmem]";
 	struct gmem_file *f;
@@ -695,6 +892,12 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)
 
 	GMEM_I(inode)->flags = flags;
 
+	if (flags & GUEST_MEMFD_FLAG_USE_DMABUF) {
+		err = kvm_gmem_dmabuf_attach(inode, dmabuf_fd);
+		if (err)
+			goto err_inode;
+	}
+
 	file = alloc_file_pseudo(inode, kvm_gmem_mnt, name, O_RDWR, &kvm_gmem_fops);
 	if (IS_ERR(file)) {
 		err = PTR_ERR(file);
@@ -703,12 +906,17 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)
 
 	file->f_flags |= O_LARGEFILE;
 	file->private_data = &f->backing;
-	f->backing.ops = &kvm_gmem_native_ops;
+	f->backing.ops = flags & GUEST_MEMFD_FLAG_USE_DMABUF ?
+			 &kvm_gmem_dmabuf_ops : &kvm_gmem_native_ops;
 
 	kvm_get_kvm(kvm);
 	f->kvm = kvm;
 	xa_init(&f->bindings);
+	if (flags & GUEST_MEMFD_FLAG_USE_DMABUF)
+		filemap_invalidate_lock(inode->i_mapping);
 	list_add(&f->entry, &GMEM_I(inode)->gmem_file_list);
+	if (flags & GUEST_MEMFD_FLAG_USE_DMABUF)
+		filemap_invalidate_unlock(inode->i_mapping);
 
 	fd_install(fd, file);
 	return fd;
@@ -734,8 +942,11 @@ int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args)
 
 	if (size <= 0 || !PAGE_ALIGNED(size))
 		return -EINVAL;
+	if (args->pad ||
+	    (!(flags & GUEST_MEMFD_FLAG_USE_DMABUF) && args->dmabuf_fd))
+		return -EINVAL;
 
-	return __kvm_gmem_create(kvm, size, flags);
+	return __kvm_gmem_create(kvm, size, flags, args->dmabuf_fd);
 }
 
 /*
@@ -1211,6 +1422,8 @@ static struct inode *kvm_gmem_alloc_inode(struct super_block *sb)
 	mpol_shared_policy_init(&gi->policy, NULL);
 
 	gi->flags = 0;
+	gi->dmabuf = NULL;
+	gi->attach = NULL;
 	INIT_LIST_HEAD(&gi->gmem_file_list);
 	return &gi->vfs_inode;
 }
@@ -1225,11 +1438,19 @@ static void kvm_gmem_free_inode(struct inode *inode)
 	kmem_cache_free(kvm_gmem_inode_cachep, GMEM_I(inode));
 }
 
+static void kvm_gmem_evict_inode(struct inode *inode)
+{
+	kvm_gmem_dmabuf_release(inode);
+	truncate_inode_pages_final(&inode->i_data);
+	clear_inode(inode);
+}
+
 static const struct super_operations kvm_gmem_super_operations = {
 	.statfs		= simple_statfs,
 	.alloc_inode	= kvm_gmem_alloc_inode,
 	.destroy_inode	= kvm_gmem_destroy_inode,
 	.free_inode	= kvm_gmem_free_inode,
+	.evict_inode	= kvm_gmem_evict_inode,
 };
 
 static int kvm_gmem_init_fs_context(struct fs_context *fc)
@@ -1292,3 +1513,5 @@ void kvm_gmem_exit(void)
 	rcu_barrier();
 	kmem_cache_destroy(kvm_gmem_inode_cachep);
 }
+
+MODULE_IMPORT_NS("DMA_BUF");
-- 
2.47.3