[PATCH net v5 3/4] netdevsim: support devmem RX and TX bindings

[Mina Almasry <almasrymina@google.com>]

Set a 64-bit DMA mask and UINT_MAX max segment size on nsim_bus_dev in
nsim_bus_probe() when direct DMA mapping is supported, enable
NETMEM_TX_DMA on the netdevsim net_device, and configure page_pool
instances with PP_FLAG_ALLOW_UNREADABLE_NETMEM and queue_idx, plus
PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV when a DMA mask and
PP_DMA_INDEX_BITS are available, so devmem bind-rx and bind-tx netlink
operations can be tested against netdevsim in software selftests.

The pp_hold debugfs file allocates pages from the queue 0 page pool, so
refuse it when that pool is absent or backed by a memory provider, and
take the netdev instance lock there so a concurrent bind-rx cannot swap
the pool in between. Since netdevsim forwards TX skbs in software
without a DMA engine, drop unreadable or net_iov skbs in
nsim_start_xmit().

Cc: Tariq Toukan <tariqt@nvidia.com>
Cc: Kaifeng Wang <kaifengw@google.com>
Signed-off-by: Mina Almasry <almasrymina@google.com>
Reviewed-by: Dragos Tatulea <dtatulea@nvidia.com>
Acked-by: Stanislav Fomichev <sdf@fomichev.me>
---
v5:
- Configure the DMA mask and dma_parms in nsim_bus_probe() gated on
  dma_pci_p2pdma_supported(), and require PP_DMA_INDEX_BITS before
  setting PP_FLAG_DMA_MAP so page_pool_scrub() can unmap inflight pages
  on 32-bit builds (Sashiko).
- Check for NULL ns->rq in nsim_pp_hold_write() and drop unreadable or
  net_iov skbs at the start of nsim_start_xmit() (Sashiko).

v3:
- Only request page pool DMA mapping when CONFIG_HAS_DMA is set
  (Sashiko).
- Refuse pp_hold on a page pool backed by a memory provider, under the
  netdev instance lock (Sashiko).
- Drop unreadable skbs in the software PSP path when it would have to
  checksum the payload (Sashiko).
---
 drivers/net/netdevsim/bus.c       |  9 +++++++++
 drivers/net/netdevsim/netdev.c    | 33 +++++++++++++++++++++++++------
 drivers/net/netdevsim/netdevsim.h |  1 +
 3 files changed, 37 insertions(+), 6 deletions(-)

diff --git a/drivers/net/netdevsim/bus.c b/drivers/net/netdevsim/bus.c
index b792ee10b1c1a..5f8f0c2f76939 100644
--- a/drivers/net/netdevsim/bus.c
+++ b/drivers/net/netdevsim/bus.c
@@ -5,6 +5,7 @@
 
 #include <linux/completion.h>
 #include <linux/device.h>
+#include <linux/dma-mapping.h>
 #include <linux/idr.h>
 #include <linux/kernel.h>
 #include <linux/list.h>
@@ -416,6 +417,14 @@ static int nsim_bus_probe(struct device *dev)
 {
 	struct nsim_bus_dev *nsim_bus_dev = to_nsim_bus_dev(dev);
 
+	if (dma_pci_p2pdma_supported(dev) &&
+	    !dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(64))) {
+		dev->dma_parms = &nsim_bus_dev->dma_parms;
+		dma_set_max_seg_size(dev, UINT_MAX);
+	} else {
+		dev->dma_mask = NULL;
+	}
+
 	return nsim_drv_probe(nsim_bus_dev);
 }
 
diff --git a/drivers/net/netdevsim/netdev.c b/drivers/net/netdevsim/netdev.c
index eebc02ccc4a92..0372a9bd63bbb 100644
--- a/drivers/net/netdevsim/netdev.c
+++ b/drivers/net/netdevsim/netdev.c
@@ -133,6 +133,10 @@ static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	int dr;
 
 	rcu_read_lock();
+	if (!skb_frags_readable(skb) ||
+	    (skb_shinfo(skb)->nr_frags &&
+	     skb_frag_is_net_iov(&skb_shinfo(skb)->frags[0])))
+		goto out_drop_any;
 	if (!nsim_ipsec_tx(ns, skb))
 		goto out_drop_any;
 
@@ -422,19 +426,27 @@ static int nsim_poll(struct napi_struct *napi, int budget)
 	return done;
 }
 
-static int nsim_create_page_pool(struct page_pool **p, struct napi_struct *napi)
+static int nsim_create_page_pool(struct net_device *dev, struct page_pool **p,
+				 struct napi_struct *napi, int idx)
 {
+	struct netdevsim *ns = netdev_priv(dev);
 	struct page_pool_params params = {
 		.order = 0,
 		.pool_size = NSIM_RING_SIZE,
+		.flags = PP_FLAG_ALLOW_UNREADABLE_NETMEM,
+		.max_len = PAGE_SIZE,
 		.nid = NUMA_NO_NODE,
-		.dev = &napi->dev->dev,
+		.dev = &ns->nsim_bus_dev->dev,
 		.napi = napi,
 		.dma_dir = DMA_BIDIRECTIONAL,
-		.netdev = napi->dev,
+		.netdev = dev,
+		.queue_idx = idx,
 	};
 	struct page_pool *pool;
 
+	if (ns->nsim_bus_dev->dev.dma_mask && PP_DMA_INDEX_BITS)
+		params.flags |= PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
+
 	pool = page_pool_create(&params);
 	if (IS_ERR(pool))
 		return PTR_ERR(pool);
@@ -458,7 +470,7 @@ static int nsim_init_napi(struct netdevsim *ns)
 	for (i = 0; i < dev->num_rx_queues; i++) {
 		rq = ns->rq[i];
 
-		err = nsim_create_page_pool(&rq->page_pool, &rq->napi);
+		err = nsim_create_page_pool(dev, &rq->page_pool, &rq->napi, i);
 		if (err)
 			goto err_pp_destroy;
 	}
@@ -770,14 +782,16 @@ nsim_queue_mem_alloc(struct net_device *dev,
 	if (ns->rq_reset_mode == 1) {
 		if (!netif_running(ns->netdev))
 			return -ENETDOWN;
-		return nsim_create_page_pool(&qmem->pp, &ns->rq[idx]->napi);
+		return nsim_create_page_pool(dev, &qmem->pp,
+					     &ns->rq[idx]->napi, idx);
 	}
 
 	qmem->rq = nsim_queue_alloc();
 	if (!qmem->rq)
 		return -ENOMEM;
 
-	err = nsim_create_page_pool(&qmem->rq->page_pool, &qmem->rq->napi);
+	err = nsim_create_page_pool(dev, &qmem->rq->page_pool, &qmem->rq->napi,
+				    idx);
 	if (err)
 		goto err_free;
 
@@ -937,12 +951,16 @@ nsim_pp_hold_write(struct file *file, const char __user *data,
 		return ret;
 
 	rtnl_lock();
+	netdev_lock(ns->netdev);
 	ret = count;
 	if (val == !!ns->page)
 		goto exit;
 
 	if (!netif_running(ns->netdev) && val) {
 		ret = -ENETDOWN;
+	} else if (val && (!ns->rq ||
+			   page_pool_is_unreadable(ns->rq[0]->page_pool))) {
+		ret = -EOPNOTSUPP;
 	} else if (val) {
 		ns->page = page_pool_dev_alloc_pages(ns->rq[0]->page_pool);
 		if (!ns->page)
@@ -954,6 +972,7 @@ nsim_pp_hold_write(struct file *file, const char __user *data,
 	}
 
 exit:
+	netdev_unlock(ns->netdev);
 	rtnl_unlock();
 	return ret;
 }
@@ -1059,6 +1078,8 @@ static int nsim_init_netdevsim(struct netdevsim *ns)
 	ns->netdev->netdev_ops = &nsim_netdev_ops;
 	ns->netdev->stat_ops = &nsim_stat_ops;
 	ns->netdev->queue_mgmt_ops = &nsim_queue_mgmt_ops;
+	if (ns->nsim_bus_dev->dev.dma_mask)
+		ns->netdev->netmem_tx = NETMEM_TX_DMA;
 	netdev_lockdep_set_classes(ns->netdev);
 
 	err = nsim_udp_tunnels_info_create(ns->nsim_dev, ns->netdev);
diff --git a/drivers/net/netdevsim/netdevsim.h b/drivers/net/netdevsim/netdevsim.h
index fd5599cd3bb9f..2aa7ea924396e 100644
--- a/drivers/net/netdevsim/netdevsim.h
+++ b/drivers/net/netdevsim/netdevsim.h
@@ -476,6 +476,7 @@ int nsim_setup_tc(struct net_device *dev, enum tc_setup_type type,
 #define NSIM_BUS_DEV_MAX_VFS 4
 struct nsim_bus_dev {
 	struct device dev;
+	struct device_dma_parameters dma_parms;
 	struct list_head list;
 	unsigned int port_count;
 	unsigned int num_queues; /* Number of queues for each port on this bus */
-- 
2.56.0.385.gac671c819a-goog