[PATCH 01/13] virtio_ring: support per-virtqueue DMA mapping ops

[Kevin Loughlin <kevinloughlin@google.com>]

Allow individual virtqueues to override the device-wide DMA mapping
operations via virtqueue_set_map_ops(). When custom per-virtqueue
mapping operations are configured, vring_map_one_sg(), vring_unmap_one(),
vring_mapping_error(), and the DMA sync helpers dispatch through
vq->map_ops instead of the parent virtio_device DMA API path.

Replace vring_map_single() with vring_map_indirect() and
vring_unmap_indirect() using the device-level mapping path (vdev->map
or dma_map_page_attrs()) so that indirect descriptor tables continue to
be DMA-mapped for host VMM access even when a virtqueue overrides buffer
mapping via vq->map_ops.

Signed-off-by: Kevin Loughlin <kevinloughlin@google.com>
---
 drivers/virtio/virtio_ring.c | 136 ++++++++++++++++++++++++-----------
 include/linux/virtio.h       |   4 ++
 2 files changed, 98 insertions(+), 42 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index db678f5a80e0..9c6a1e9f0a0b 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -261,6 +261,7 @@ struct vring_virtqueue {
 	bool we_own_ring;
 
 	union virtio_map map;
+	const struct virtio_map_ops *map_ops;
 
 #ifdef DEBUG
 	/* They're supposed to lock for us. */
@@ -477,16 +478,32 @@ static struct device *vring_dma_dev(const struct vring_virtqueue *vq)
 	return vq->map.dma_dev;
 }
 
+static inline const struct virtio_map_ops *vring_get_map_ops(const struct vring_virtqueue *vq)
+{
+	if (vq->map_ops)
+		return vq->map_ops;
+	return vq->vq.vdev->map;
+}
+
+void virtqueue_set_map_ops(struct virtqueue *_vq,
+			   const struct virtio_map_ops *map_ops)
+{
+	struct vring_virtqueue *vq = to_vvq(_vq);
+
+	vq->map_ops = map_ops;
+}
+EXPORT_SYMBOL_GPL(virtqueue_set_map_ops);
+
 static int vring_mapping_error(const struct vring_virtqueue *vq,
 			       dma_addr_t addr)
 {
-	struct virtio_device *vdev = vq->vq.vdev;
+	const struct virtio_map_ops *map_ops = vring_get_map_ops(vq);
 
 	if (!vq->use_map_api)
 		return 0;
 
-	if (vdev->map)
-		return vdev->map->mapping_error(vq->map, addr);
+	if (map_ops)
+		return map_ops->mapping_error(vq->map, addr);
 	else
 		return dma_mapping_error(vring_dma_dev(vq), addr);
 }
@@ -530,15 +547,45 @@ static int vring_map_one_sg(const struct vring_virtqueue *vq, struct scatterlist
 	return 0;
 }
 
-static dma_addr_t vring_map_single(const struct vring_virtqueue *vq,
-				   void *cpu_addr, size_t size,
-				   enum dma_data_direction direction)
+static int vring_map_indirect(const struct vring_virtqueue *vq, void *cpu_addr,
+			      size_t size, dma_addr_t *addr)
 {
-	if (!vq->use_map_api)
-		return (dma_addr_t)virt_to_phys(cpu_addr);
+	struct virtio_device *vdev = vq->vq.vdev;
+
+	if (!vq->use_map_api) {
+		kmsan_handle_dma(virt_to_phys(cpu_addr), size, DMA_TO_DEVICE);
+		*addr = (dma_addr_t)virt_to_phys(cpu_addr);
+		return 0;
+	}
+
+	if (vdev->map) {
+		*addr = vdev->map->map_page(vq->map, virt_to_page(cpu_addr),
+					    offset_in_page(cpu_addr), size,
+					    DMA_TO_DEVICE, 0);
+		if (vdev->map->mapping_error(vq->map, *addr))
+			return -ENOMEM;
+		return 0;
+	}
+
+	*addr = dma_map_page_attrs(vring_dma_dev(vq), virt_to_page(cpu_addr),
+				   offset_in_page(cpu_addr), size,
+				   DMA_TO_DEVICE, 0);
+	if (dma_mapping_error(vring_dma_dev(vq), *addr))
+		return -ENOMEM;
 
-	return virtqueue_map_single_attrs(&vq->vq, cpu_addr,
-					  size, direction, 0);
+	return 0;
+}
+
+static void vring_unmap_indirect(const struct vring_virtqueue *vq,
+				 dma_addr_t addr, size_t size)
+{
+	struct virtio_device *vdev = vq->vq.vdev;
+
+	if (vdev->map)
+		vdev->map->unmap_page(vq->map, addr, size, DMA_TO_DEVICE, 0);
+	else
+		dma_unmap_page_attrs(vring_dma_dev(vq), addr, size,
+				     DMA_TO_DEVICE, 0);
 }
 
 static void virtqueue_init(struct vring_virtqueue *vq, u32 num)
@@ -576,6 +623,8 @@ static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq,
 	if (flags & VRING_DESC_F_INDIRECT) {
 		if (!vq->use_map_api)
 			goto out;
+		vring_unmap_indirect(vq, extra->addr, extra->len);
+		goto out;
 	} else if (!vring_need_unmap_buffer(vq, extra))
 		goto out;
 
@@ -761,10 +810,11 @@ static inline int virtqueue_add_split(struct vring_virtqueue *vq,
 
 	if (indirect) {
 		/* Now that the indirect table is filled in, map it. */
-		dma_addr_t addr = vring_map_single(
-			vq, desc, total_sg * sizeof(struct vring_desc),
-			DMA_TO_DEVICE);
-		if (vring_mapping_error(vq, addr))
+		dma_addr_t addr;
+
+		if (vring_map_indirect(vq, desc,
+				       total_sg * sizeof(struct vring_desc),
+				       &addr))
 			goto unmap_release;
 
 		virtqueue_add_desc_split(vq, vq->split.vring.desc,
@@ -1370,6 +1420,7 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
 #endif
 	vq->map = map;
 	vq->use_map_api = vring_use_map_api(vdev);
+	vq->map_ops = NULL;
 
 	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
 		!context;
@@ -1488,6 +1539,8 @@ static void vring_unmap_extra_packed(const struct vring_virtqueue *vq,
 	if (flags & VRING_DESC_F_INDIRECT) {
 		if (!vq->use_map_api)
 			return;
+		vring_unmap_indirect(vq, extra->addr, extra->len);
+		return;
 	} else if (!vring_need_unmap_buffer(vq, extra))
 		return;
 
@@ -1585,10 +1638,9 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
 	}
 
 	/* Now that the indirect table is filled in, map it. */
-	addr = vring_map_single(vq, desc,
-			total_sg * sizeof(struct vring_packed_desc),
-			DMA_TO_DEVICE);
-	if (vring_mapping_error(vq, addr))
+	if (vring_map_indirect(vq, desc,
+			       total_sg * sizeof(struct vring_packed_desc),
+			       &addr))
 		goto unmap_release;
 
 	vq->packed.vring.desc[head].addr = cpu_to_le64(addr);
@@ -2620,6 +2672,7 @@ static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
 #endif
 	vq->map = map;
 	vq->use_map_api = vring_use_map_api(vdev);
+	vq->map_ops = NULL;
 
 	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
 		!context;
@@ -3064,7 +3117,7 @@ struct device *virtqueue_dma_dev(struct virtqueue *_vq)
 {
 	struct vring_virtqueue *vq = to_vvq(_vq);
 
-	if (vq->use_map_api && !_vq->vdev->map)
+	if (vq->use_map_api && !vring_get_map_ops(vq))
 		return vq->map.dma_dev;
 	else
 		return NULL;
@@ -3809,18 +3862,17 @@ EXPORT_SYMBOL_GPL(virtqueue_map_free_coherent);
  */
 dma_addr_t virtqueue_map_page_attrs(const struct virtqueue *_vq,
 				    struct page *page,
-				    unsigned long offset,
-				    size_t size,
+				    unsigned long offset, size_t size,
 				    enum dma_data_direction dir,
 				    unsigned long attrs)
 {
 	const struct vring_virtqueue *vq = to_vvq(_vq);
-	struct virtio_device *vdev = _vq->vdev;
+	const struct virtio_map_ops *map_ops = vring_get_map_ops(vq);
 
-	if (vdev->map)
-		return vdev->map->map_page(vq->map,
-					   page, offset, size,
-					   dir, attrs);
+	if (map_ops)
+		return map_ops->map_page(vq->map,
+					 page, offset, size,
+					 dir, attrs);
 
 	return dma_map_page_attrs(vring_dma_dev(vq),
 				  page, offset, size,
@@ -3842,11 +3894,11 @@ void virtqueue_unmap_page_attrs(const struct virtqueue *_vq,
 				unsigned long attrs)
 {
 	const struct vring_virtqueue *vq = to_vvq(_vq);
-	struct virtio_device *vdev = _vq->vdev;
+	const struct virtio_map_ops *map_ops = vring_get_map_ops(vq);
 
-	if (vdev->map)
-		vdev->map->unmap_page(vq->map,
-				      map_handle, size, dir, attrs);
+	if (map_ops)
+		map_ops->unmap_page(vq->map,
+				    map_handle, size, dir, attrs);
 	else
 		dma_unmap_page_attrs(vring_dma_dev(vq), map_handle,
 				     size, dir, attrs);
@@ -3941,13 +3993,13 @@ EXPORT_SYMBOL_GPL(virtqueue_map_mapping_error);
 bool virtqueue_map_need_sync(const struct virtqueue *_vq, dma_addr_t addr)
 {
 	const struct vring_virtqueue *vq = to_vvq(_vq);
-	struct virtio_device *vdev = _vq->vdev;
+	const struct virtio_map_ops *map_ops = vring_get_map_ops(vq);
 
 	if (!vq->use_map_api)
 		return false;
 
-	if (vdev->map)
-		return vdev->map->need_sync(vq->map, addr);
+	if (map_ops)
+		return map_ops->need_sync(vq->map, addr);
 	else
 		return dma_need_sync(vring_dma_dev(vq), addr);
 }
@@ -3971,14 +4023,14 @@ void virtqueue_map_sync_single_range_for_cpu(const struct virtqueue *_vq,
 					     enum dma_data_direction dir)
 {
 	const struct vring_virtqueue *vq = to_vvq(_vq);
-	struct virtio_device *vdev = _vq->vdev;
+	const struct virtio_map_ops *map_ops = vring_get_map_ops(vq);
 
 	if (!vq->use_map_api)
 		return;
 
-	if (vdev->map)
-		vdev->map->sync_single_for_cpu(vq->map,
-					       addr + offset, size, dir);
+	if (map_ops)
+		map_ops->sync_single_for_cpu(vq->map,
+					     addr + offset, size, dir);
 	else
 		dma_sync_single_range_for_cpu(vring_dma_dev(vq),
 					      addr, offset, size, dir);
@@ -4002,15 +4054,15 @@ void virtqueue_map_sync_single_range_for_device(const struct virtqueue *_vq,
 						enum dma_data_direction dir)
 {
 	const struct vring_virtqueue *vq = to_vvq(_vq);
-	struct virtio_device *vdev = _vq->vdev;
+	const struct virtio_map_ops *map_ops = vring_get_map_ops(vq);
 
 	if (!vq->use_map_api)
 		return;
 
-	if (vdev->map)
-		vdev->map->sync_single_for_device(vq->map,
-						  addr + offset,
-						  size, dir);
+	if (map_ops)
+		map_ops->sync_single_for_device(vq->map,
+						addr + offset,
+						size, dir);
 	else
 		dma_sync_single_range_for_device(vring_dma_dev(vq), addr,
 						 offset, size, dir);
diff --git a/include/linux/virtio.h b/include/linux/virtio.h
index f923e42cfd01..20fe333d6778 100644
--- a/include/linux/virtio.h
+++ b/include/linux/virtio.h
@@ -44,6 +44,7 @@ struct virtqueue {
 };
 
 struct vduse_vq_group;
+struct virtio_map_ops;
 
 union virtio_map {
 	/* Device that performs DMA */
@@ -294,6 +295,9 @@ void virtqueue_map_free_coherent(struct virtio_device *vdev,
 				 size_t size, void *vaddr,
 				 dma_addr_t dma_handle);
 
+void virtqueue_set_map_ops(struct virtqueue *_vq,
+			   const struct virtio_map_ops *map_ops);
+
 dma_addr_t virtqueue_map_page_attrs(const struct virtqueue *_vq,
 				    struct page *page,
 				    unsigned long offset,
-- 
2.56.0.307.g5fe7bd680c-goog