[PATCH net-next V2 11/13] net: add netdev_unbind_dma_dev() helper

[Tariq Toukan <tariqt@nvidia.com>]

From: Dragos Tatulea <dtatulea@nvidia.com>

Currently the netdev code that binds dmabufs to a DMA device assumes
that the device is the netdev's own PCI device, and the unbind cleanup
is done under that assumption.

With the introduction of data direct in mlx5, this assumption no longer
holds: the DMA device is not the netdev's PCI device. When the data
direct DMA device is unbound, there is no way to clean up the bindings.

Add a helper that unbinds the dmabufs of a closed or detached netdev
from a DMA device, independent of the memory provider in use:

- RX: uninstall the memory provider of every RX queue whose DMA device
  is the departing one or is gone. The uninstall callback unmaps the
  buffers, for devmem and io_uring zcrx alike. The queues get regular
  page pools on the next open.

- devmem TX: detach and unmap the TX bindings of the netdev mapped
  against the device, as netdev unregister does. New sends fail and
  in-flight skbs are dropped at xmit.

The netdev must be closed or detached so that the hardware no longer
accesses the mappings and the qdiscs hold no validated skbs. A detached
netdev also works because the driver has already torn down the queues
and new bindings can't be added.

For now leased queues owned by virtual netdevs are not supported by this
API. mlx5 refuses zero-copy while RX queues are leased in data direct
mode.

The helper only unmaps. The providers and bindings stay allocated until
their users release them.

Signed-off-by: Dragos Tatulea <dtatulea@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
---
 include/net/netdev_queues.h |  1 +
 net/core/devmem.c           | 42 ++++++++++++++++++++++++++++++
 net/core/devmem.h           |  6 +++++
 net/core/netdev_queues.c    | 51 +++++++++++++++++++++++++++++++++++++
 4 files changed, 100 insertions(+)

diff --git a/include/net/netdev_queues.h b/include/net/netdev_queues.h
index c5335e935b7d..f3f174f342b9 100644
--- a/include/net/netdev_queues.h
+++ b/include/net/netdev_queues.h
@@ -235,6 +235,7 @@ void netdev_queue_config(struct net_device *dev, int rxq,
 			 struct netdev_queue_config *qcfg);
 
 bool netif_rxq_has_unreadable_mp(struct net_device *dev, unsigned int rxq_idx);
+int netdev_unbind_dma_dev(struct net_device *dev, struct device *dma_dev);
 
 /**
  * DOC: Lockless queue stopping / waking helpers.
diff --git a/net/core/devmem.c b/net/core/devmem.c
index dd3987bbd9f6..bc8ae12b2318 100644
--- a/net/core/devmem.c
+++ b/net/core/devmem.c
@@ -556,6 +556,48 @@ bool netdev_has_dmabuf_binding(struct net_device *dev, struct device *dma_dev)
 }
 EXPORT_SYMBOL_GPL(netdev_has_dmabuf_binding);
 
+/* Unmap the bindings of a closed @dev that use @dma_dev, the way netdev
+ * unregister does. Senders that raced the detach drop their skbs at xmit.
+ */
+void net_devmem_unbind_dma_dev(struct net_device *dev, struct device *dma_dev)
+{
+	struct net_devmem_dmabuf_binding *binding;
+	unsigned long xa_idx;
+	bool match, live;
+
+	netdev_assert_locked(dev);
+
+	xa_lock(&net_devmem_dmabuf_bindings);
+	xa_for_each(&net_devmem_dmabuf_bindings, xa_idx, binding) {
+		if (READ_ONCE(binding->dev) != dev)
+			continue;
+
+		/* Hide the binding from TX lookups while it is checked. */
+		live = xa_get_mark(&net_devmem_dmabuf_bindings, xa_idx,
+				   NET_DEVMEM_BINDING_LIVE);
+		if (live)
+			__xa_clear_mark(&net_devmem_dmabuf_bindings, xa_idx,
+					NET_DEVMEM_BINDING_LIVE);
+		xa_unlock(&net_devmem_dmabuf_bindings);
+
+		mutex_lock(&binding->lock);
+		match = net_devmem_binding_mapped_to(binding, dma_dev);
+		mutex_unlock(&binding->lock);
+		if (match)
+			net_devmem_dmabuf_binding_unmap(binding);
+
+		xa_lock(&net_devmem_dmabuf_bindings);
+		if (match) {
+			WRITE_ONCE(binding->vdev, NULL);
+			WRITE_ONCE(binding->dev, NULL);
+		} else if (live) {
+			__xa_set_mark(&net_devmem_dmabuf_bindings, xa_idx,
+				      NET_DEVMEM_BINDING_LIVE);
+		}
+	}
+	xa_unlock(&net_devmem_dmabuf_bindings);
+}
+
 void net_devmem_net_iov_inc_ref(struct net_iov *niov)
 {
 	net_devmem_dmabuf_binding_get(net_devmem_iov_binding(niov));
diff --git a/net/core/devmem.h b/net/core/devmem.h
index c4f476947dfa..06a86efc807d 100644
--- a/net/core/devmem.h
+++ b/net/core/devmem.h
@@ -106,6 +106,7 @@ void net_devmem_unlock_dev(struct net_device *dev,
 			   netdevice_tracker *dev_tracker);
 void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding);
 void net_devmem_dev_unregister(struct net_device *dev);
+void net_devmem_unbind_dma_dev(struct net_device *dev, struct device *dma_dev);
 int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx,
 				    struct net_devmem_dmabuf_binding *binding,
 				    struct netlink_ext_ack *extack);
@@ -209,6 +210,11 @@ static inline void net_devmem_dev_unregister(struct net_device *dev)
 {
 }
 
+static inline void net_devmem_unbind_dma_dev(struct net_device *dev,
+					     struct device *dma_dev)
+{
+}
+
 static inline int
 net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx,
 				struct net_devmem_dmabuf_binding *binding,
diff --git a/net/core/netdev_queues.c b/net/core/netdev_queues.c
index 27cb6397bdd4..71cc7bfc087d 100644
--- a/net/core/netdev_queues.c
+++ b/net/core/netdev_queues.c
@@ -1,10 +1,12 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+#include <net/netdev_lock.h>
 #include <net/netdev_queues.h>
 #include <net/netdev_rx_queue.h>
 #include <net/xdp_sock_drv.h>
 
 #include "dev.h"
+#include "devmem.h"
 
 static struct device *
 __netdev_queue_get_dma_dev(struct net_device *dev, unsigned int idx)
@@ -58,6 +60,55 @@ struct device *netdev_queue_get_dma_dev(struct net_device *dev,
 	return dma_dev;
 }
 
+/**
+ * netdev_unbind_dma_dev() - drop the memory providers of a departing DMA device
+ * @dev:	net_device, closed or detached by the caller
+ * @dma_dev:	DMA device the driver is about to lose
+ *
+ * Uninstalls the memory provider of every RX queue of @dev that maps
+ * against @dma_dev. Unmaps the devmem bindings of @dev that use @dma_dev.
+ * The bindings stay allocated until their users release them.
+ *
+ * The provider of a leased RX queue is owned by the virtual netdev and
+ * can't be torn down here. Drivers must not report a DMA device they can
+ * lose for leased queues; such queues are skipped with a warning.
+ *
+ * Return: 0 on success, -EBUSY if @dev is running and present.
+ */
+int netdev_unbind_dma_dev(struct net_device *dev, struct device *dma_dev)
+{
+	unsigned int i;
+
+	netdev_assert_locked(dev);
+
+	if (netif_running(dev) && netif_device_present(dev))
+		return -EBUSY;
+
+	for (i = 0; i < dev->real_num_rx_queues; i++) {
+		struct netdev_rx_queue *rxq = __netif_get_rx_queue(dev, i);
+		struct pp_memory_provider_params p = rxq->mp_params;
+		struct device *queue_dev;
+
+		if (!p.mp_ops)
+			continue;
+
+		queue_dev = __netdev_queue_get_dma_dev(dev, i);
+		if (queue_dev && queue_dev != dma_dev)
+			continue;
+
+		if (WARN_ON_ONCE(rxq->lease))
+			continue;
+
+		memset(&rxq->mp_params, 0, sizeof(rxq->mp_params));
+		__netif_mp_uninstall_rxq(rxq, &p);
+	}
+
+	net_devmem_unbind_dma_dev(dev, dma_dev);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(netdev_unbind_dma_dev);
+
 bool netdev_can_create_queue(const struct net_device *dev,
 			     struct netlink_ext_ack *extack)
 {
-- 
2.44.0