[PATCH RFC v3 68/88] vfio/pci: Use iov_iter for MMIO read and write

[Ackerley Tng via B4 Relay <devnull+ackerleytng.google.com@kernel.org>]

From: Ackerley Tng <ackerleytng@google.com>

Refactor the internal MMIO read and write helpers in vfio_pci_rdwr to
operate on struct iov_iter rather than directly calling copy_to_user
and copy_from_user with user pointers.

Keep the existing vfio_pci_core_do_io_rw interface unchanged by
importing user buffers into an iov_iter via import_ubuf before calling
the internal implementation. The overhead of initializing the iov_iter
on userspace accesses is negligible compared to PCIe bus cycle latency.

This decouples low-level PCI MMIO access from userspace copy semantics,
allowing in-kernel callers to service I/O using kernel buffers (such
as ITER_KVEC) without user copy overhead or page fault recursion.

For delegation of read/write() to guest_memfd, this helps avoid AB-BA
deadlocks between mapping.invalidate_lock and mm->mmap_lock when MMIO read
and write operations are invoked under filemap invalidation locks.

No functional change intended.

Signed-off-by: Ackerley Tng <ackerleytng@google.com>
---
 drivers/vfio/pci/vfio_pci_rdwr.c | 107 ++++++++++++++++++++++++---------------
 1 file changed, 65 insertions(+), 42 deletions(-)

diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c
index e5f1f60513709..cc64722d1ab9a 100644
--- a/drivers/vfio/pci/vfio_pci_rdwr.c
+++ b/drivers/vfio/pci/vfio_pci_rdwr.c
@@ -91,34 +91,34 @@ VFIO_IOREAD(32)
 VFIO_IOREAD(64)
 
 #define VFIO_IORDWR(size)						\
-static int vfio_pci_iordwr##size(struct vfio_pci_core_device *vdev,\
-				bool iswrite, bool test_mem,		\
-				void __iomem *io, char __user *buf,	\
-				loff_t off, size_t *filled)		\
-{									\
-	u##size val;							\
-	int ret;							\
-									\
-	if (iswrite) {							\
-		if (copy_from_user(&val, buf, sizeof(val)))		\
-			return -EFAULT;					\
-									\
-		ret = vfio_pci_core_iowrite##size(vdev, test_mem,	\
-						  val, io + off);	\
-		if (ret)						\
-			return ret;					\
-	} else {							\
-		ret = vfio_pci_core_ioread##size(vdev, test_mem,	\
-						 &val, io + off);	\
-		if (ret)						\
-			return ret;					\
-									\
-		if (copy_to_user(buf, &val, sizeof(val)))		\
-			return -EFAULT;					\
-	}								\
-									\
-	*filled = sizeof(val);						\
-	return 0;							\
+static int vfio_pci_iordwr##size(struct vfio_pci_core_device *vdev,		\
+				bool iswrite, bool test_mem,			\
+				void __iomem *io, struct iov_iter *iter,	\
+				loff_t off, size_t *filled)			\
+{										\
+	u##size val;								\
+	int ret;								\
+										\
+	if (iswrite) {								\
+		if (!copy_from_iter_full(&val, sizeof(val), iter))		\
+			return -EFAULT;						\
+										\
+		ret = vfio_pci_core_iowrite##size(vdev, test_mem,		\
+						  val, io + off);		\
+		if (ret)							\
+			return ret;						\
+	} else {								\
+		ret = vfio_pci_core_ioread##size(vdev, test_mem,		\
+						 &val, io + off);		\
+		if (ret)							\
+			return ret;						\
+										\
+		if (!copy_to_iter_full(&val, sizeof(val), iter))		\
+			return -EFAULT;						\
+	}									\
+										\
+	*filled = sizeof(val);							\
+	return 0;								\
 }									\
 
 VFIO_IORDWR(8)
@@ -132,11 +132,12 @@ VFIO_IORDWR(64)
  * reads with -1.  This is intended for handling MSI-X vector tables and
  * leftover space for ROM BARs.
  */
-ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
-			       void __iomem *io, char __user *buf,
-			       loff_t off, size_t count, size_t x_start,
-			       size_t x_end, bool iswrite,
-			       enum vfio_pci_io_width max_width)
+static ssize_t __vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev,
+					bool test_mem, void __iomem *io,
+					struct iov_iter *iter, loff_t off,
+					size_t count, size_t x_start,
+					size_t x_end, bool iswrite,
+					enum vfio_pci_io_width max_width)
 {
 	ssize_t done = 0;
 	int ret;
@@ -153,25 +154,25 @@ ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
 
 		if (fillable >= 8 && !(off % 8) && max_width >= 8) {
 			ret = vfio_pci_iordwr64(vdev, iswrite, test_mem,
-						io, buf, off, &filled);
+						io, iter, off, &filled);
 			if (ret)
 				return ret;
 
 		} else if (fillable >= 4 && !(off % 4) && max_width >= 4) {
 			ret = vfio_pci_iordwr32(vdev, iswrite, test_mem,
-						io, buf, off, &filled);
+						io, iter, off, &filled);
 			if (ret)
 				return ret;
 
 		} else if (fillable >= 2 && !(off % 2) && max_width >= 2) {
 			ret = vfio_pci_iordwr16(vdev, iswrite, test_mem,
-						io, buf, off, &filled);
+						io, iter, off, &filled);
 			if (ret)
 				return ret;
 
 		} else if (fillable) {
 			ret = vfio_pci_iordwr8(vdev, iswrite, test_mem,
-					       io, buf, off, &filled);
+					       io, iter, off, &filled);
 			if (ret)
 				return ret;
 
@@ -179,23 +180,45 @@ ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
 			/* Fill reads with -1, drop writes */
 			filled = min(count, (size_t)(x_end - off));
 			if (!iswrite) {
-				u8 val = 0xFF;
-				size_t i;
+				static const u8 ff_buf[64] = { [0 ... 63] = 0xFF };
+				size_t cur = filled;
+
+				while (cur) {
+					size_t step = min(cur, sizeof(ff_buf));
 
-				for (i = 0; i < filled; i++)
-					if (copy_to_user(buf + i, &val, 1))
+					if (!copy_to_iter_full(ff_buf, step, iter))
 						return -EFAULT;
+					cur -= step;
+				}
+			} else {
+				iov_iter_advance(iter, filled);
 			}
 		}
 
 		count -= filled;
 		done += filled;
 		off += filled;
-		buf += filled;
 	}
 
 	return done;
 }
+
+ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
+			       void __iomem *io, char __user *buf,
+			       loff_t off, size_t count, size_t x_start,
+			       size_t x_end, bool iswrite,
+			       enum vfio_pci_io_width max_width)
+{
+	struct iov_iter iter;
+	int ret;
+
+	ret = import_ubuf(iswrite ? ITER_SOURCE : ITER_DEST, buf, count, &iter);
+	if (ret)
+		return ret;
+
+	return __vfio_pci_core_do_io_rw(vdev, test_mem, io, &iter, off, count,
+					x_start, x_end, iswrite, max_width);
+}
 EXPORT_SYMBOL_GPL(vfio_pci_core_do_io_rw);
 
 ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf,

-- 
2.56.0.455.g6c5a503b38-goog