Re: [PATCH net-next v1 05/11] xsk: Install UMEM page pool providers in the core

[Mina Almasry <almasrymina@google.com>]

On Tue, Nov 10, 2026 at 2:21 AM Björn Töpel <bjorn@kernel.org> wrote:
>
> A page_pool driver should not need its own AF_XDP setup. When it sets
> netdev_queue_mgmt_ops.xsk_page_pool, a zero-copy bind installs the
> UMEM provider through the queue API, as devmem and io_uring do. The
> driver only judges the chunk size, headroom and multi-buffer mode
> against its MTU and XDP program, in XDP_VALIDATE_XSK_POOL; MTU and
> program changes of a bound device get the same check. The UMEM
> headroom belongs to the socket, so the queue gets it as rx_headroom,
> next to rx_page_size.
>
> After a queue restart, the core checks that the new page pools took
> the provider. A readable provider must back the header pool, where an
> unsplit packet lands, so pools declare the buffers they supply in
> rx_roles. The old queue's pools carry the same provider during the
> check, so a per-queue generation, mp_gen, tells the new ones apart.
>
> Generic XDP runs outside the queue NAPI, so it cannot feed the socket.
>
> The provider needs no xdp_buff_xsk per chunk. The buffer pool
> allocates them only for copy mode and driver zero copy, which saves
> 128 bytes per chunk.
>
> Signed-off-by: Björn Töpel <bjorn@kernel.org>
> ---
>  include/linux/netdevice.h     |   8 ++
>  include/net/netdev_queues.h   |  12 ++
>  include/net/netdev_rx_queue.h |   2 +
>  include/net/page_pool/types.h |  12 ++
>  include/net/xdp_sock.h        |   4 +
>  include/net/xdp_sock_drv.h    |  45 +++++++
>  include/net/xsk_buff_pool.h   |   3 +
>  net/core/dev.c                |  16 +++
>  net/core/netdev_config.c      |   2 +
>  net/core/netdev_rx_queue.c    |   9 +-
>  net/core/page_pool.c          |   6 +-
>  net/core/page_pool_user.c     |  25 ++--
>  net/xdp/xsk.h                 |   7 +-
>  net/xdp/xsk_buff_pool.c       |  69 ++++++++---
>  net/xdp/xsk_pp.c              | 221 ++++++++++++++++++++++++++++++++++
>  15 files changed, 410 insertions(+), 31 deletions(-)
>
> diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
> index 435be7880d7e..11e51e684003 100644
> --- a/include/linux/netdevice.h
> +++ b/include/linux/netdevice.h
> @@ -77,6 +77,7 @@ struct udp_tunnel_info;
>  struct udp_tunnel_nic_info;
>  struct udp_tunnel_nic;
>  struct bpf_prog;
> +struct xsk_rx_check;
>  struct xdp_buff;
>  struct xdp_frame;
>  struct xdp_metadata_ops;
> @@ -1010,6 +1011,8 @@ enum bpf_netdev_command {
>         BPF_OFFLOAD_MAP_ALLOC,
>         BPF_OFFLOAD_MAP_FREE,
>         XDP_SETUP_XSK_POOL,
> +       /* Check an AF_XDP page pool setup, see netdev_queue_mgmt_ops */
> +       XDP_VALIDATE_XSK_POOL,
>  };
>
>  struct bpf_prog_offload_ops;
> @@ -1048,6 +1051,11 @@ struct netdev_bpf {
>                         struct xsk_buff_pool *pool;
>                         u16 queue_id;
>                 } xsk;
> +               /* XDP_VALIDATE_XSK_POOL */
> +               struct {
> +                       const struct xsk_rx_check *check;
> +                       struct netlink_ext_ack *extack;
> +               } xsk_check;
>         };
>  };
>
> diff --git a/include/net/netdev_queues.h b/include/net/netdev_queues.h
> index c5335e935b7d..ad5f0420c21d 100644
> --- a/include/net/netdev_queues.h
> +++ b/include/net/netdev_queues.h
> @@ -45,12 +45,15 @@ struct netdev_config {
>  /**
>   * struct netdev_queue_config - rendered configuration for an RX queue
>   * @rx_page_size:      Size of one RX page-pool allocation.
> + * @rx_headroom:       Space before the packet in a head buffer, including
> + *                     XDP_PACKET_HEADROOM. Zero selects the driver default.
>   * @rx_ring_size:      Configured size of the regular RX ring.
>   * @rx_mini_ring_size: Configured size of the RX mini ring.
>   * @rx_jumbo_ring_size:        Configured size of the RX jumbo ring.
>   */
>  struct netdev_queue_config {
>         u32     rx_page_size;
> +       u32     rx_headroom;
>         u32     rx_ring_size;
>         u32     rx_mini_ring_size;
>         u32     rx_jumbo_ring_size;
> @@ -156,6 +159,8 @@ void netdev_stat_queue_sum(struct net_device *netdev,
>  enum {
>         /* The queue checks and honours the page size qcfg parameter */
>         QCFG_RX_PAGE_SIZE       = 0x1,
> +       /* The queue checks and honours the headroom qcfg parameter */
> +       QCFG_RX_HEADROOM        = 0x2,
>  };
>
>  /**
> @@ -199,6 +204,12 @@ enum {
>   *
>   * @supported_params:  Bitmask of supported parameters, see QCFG_*.
>   *
> + * @xsk_page_pool:     AF_XDP zero copy installs the UMEM as the page pool
> + *                     memory provider of the RX queue. The driver selects it
> + *                     with PP_FLAG_MEM_PROVIDER and PP_RX_ROLE_HEAD, honours
> + *                     whole buffers and QCFG_RX_HEADROOM, and accepts or
> + *                     rejects the setup in XDP_VALIDATE_XSK_POOL.
> + *
>   * Note that @ndo_queue_mem_alloc and @ndo_queue_mem_free may be called while
>   * the interface is closed. @ndo_queue_start and @ndo_queue_stop will only
>   * be called for an interface which is open.
> @@ -229,6 +240,7 @@ struct netdev_queue_mgmt_ops {
>                                     struct netlink_ext_ack *extack);
>
>         unsigned int supported_params;
> +       bool xsk_page_pool;
>  };
>
>  void netdev_queue_config(struct net_device *dev, int rxq,
> diff --git a/include/net/netdev_rx_queue.h b/include/net/netdev_rx_queue.h
> index 9415a94d333d..989120cbf452 100644
> --- a/include/net/netdev_rx_queue.h
> +++ b/include/net/netdev_rx_queue.h
> @@ -31,6 +31,8 @@ struct netdev_rx_queue {
>         struct napi_struct              *napi;
>         struct netdev_queue_config      qcfg;
>         struct pp_memory_provider_params mp_params;
> +       /* stamps the page pools of each queue allocation */
> +       u32                             mp_gen;
>
>         /* If a queue is leased, then the lease pointer is always
>          * valid. From the physical device it points to the virtual
> diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h
> index 0d48c05f1462..e9f3af91e305 100644
> --- a/include/net/page_pool/types.h
> +++ b/include/net/page_pool/types.h
> @@ -40,6 +40,12 @@
>   */
>  #define PP_FLAG_MEM_PROVIDER   BIT(4)
>
> +/* RX buffers a page pool supplies, see page_pool_params.rx_roles */
> +enum pp_rx_role {
> +       PP_RX_ROLE_HEAD         = BIT(0),       /* packet headers */
> +       PP_RX_ROLE_PAYLOAD      = BIT(1),       /* payload after header split */
> +};
> +
>  #define PP_FLAG_ALL            (PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV | \
>                                  PP_FLAG_SYSTEM_POOL | \
>                                  PP_FLAG_ALLOW_UNREADABLE_NETMEM | \
> @@ -94,6 +100,8 @@ struct pp_alloc_cache {
>   * @queue_idx: queue idx this page_pool is being created for.
>   * @flags:     PP_FLAG_DMA_MAP, PP_FLAG_DMA_SYNC_DEV, PP_FLAG_SYSTEM_POOL,
>   *             PP_FLAG_ALLOW_UNREADABLE_NETMEM, PP_FLAG_MEM_PROVIDER.
> + * @rx_roles:  &enum pp_rx_role mask, for pools that select a memory
> + *             provider. Zero means payload buffers.
>   */
>  struct page_pool_params {
>         struct_group_tagged(page_pool_params_fast, fast,
> @@ -110,6 +118,7 @@ struct page_pool_params {
>                 struct net_device *netdev;
>                 unsigned int queue_idx;
>                 unsigned int    flags;
> +               unsigned int    rx_roles;
>  /* private: used by test code only */
>                 void (*init_callback)(netmem_ref netmem, void *arg);
>                 void *init_arg;
> @@ -182,6 +191,7 @@ struct pp_memory_provider_params {
>         void *mp_priv;
>         const struct memory_provider_ops *mp_ops;
>         u32 rx_page_size;
> +       u32 rx_headroom;
>  };
>
>  struct page_pool {
> @@ -245,6 +255,8 @@ struct page_pool {
>
>         void *mp_priv;
>         const struct memory_provider_ops *mp_ops;
> +       /* queue allocation that selected the provider */
> +       u32 mp_gen;
>
>         struct xarray dma_mapped;
>
> diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h
> index 6e70b320b399..e0e22ef839e1 100644
> --- a/include/net/xdp_sock.h
> +++ b/include/net/xdp_sock.h
> @@ -19,6 +19,10 @@
>  struct net_device;
>  struct xsk_queue;
>  struct xdp_buff;
> +struct memory_provider_ops;
> +
> +/* The memory provider of AF_XDP page pool zero copy */
> +extern const struct memory_provider_ops xsk_pp_ops;
>
>  struct xdp_umem {
>         void *addrs;
> diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h
> index d94aeb506379..5bb1949a0aa7 100644
> --- a/include/net/xdp_sock_drv.h
> +++ b/include/net/xdp_sock_drv.h
> @@ -22,8 +22,29 @@ struct xsk_cb_desc {
>         u8 bytes;
>  };
>
> +/**
> + * struct xsk_rx_check - AF_XDP page pool RX setup a driver must accept
> + * @buf_size: size of each RX buffer
> + * @headroom: space before the packet in a head buffer, including
> + *     XDP_PACKET_HEADROOM. The UMEM headroom is the rest.
> + * @mtu: MTU of the device
> + * @sg: the socket takes packets in more than one buffer
> + * @prog: XDP program to check, or NULL for the current one
> + */
> +struct xsk_rx_check {
> +       u32 buf_size;
> +       u32 headroom;
> +       unsigned int mtu;
> +       bool sg;
> +       struct bpf_prog *prog;
> +};
> +
>  #ifdef CONFIG_XDP_SOCKETS
>
> +int xsk_validate_dev(struct net_device *dev, unsigned int mtu,
> +                    struct bpf_prog *prog, enum bpf_xdp_mode mode,
> +                    struct netlink_ext_ack *extack);
> +struct xsk_buff_pool *xsk_rxq_pool(struct net_device *dev, u16 queue_id);
>  void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries);
>  bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc);
>  u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max);
> @@ -36,6 +57,11 @@ void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool);
>  void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool);
>  bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool);
>
> +static inline bool xsk_pool_uses_sg(struct xsk_buff_pool *pool)
> +{
> +       return pool->umem->flags & XDP_UMEM_SG_FLAG;
> +}
> +
>  static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool)
>  {
>         return XDP_PACKET_HEADROOM + pool->headroom;
> @@ -351,6 +377,20 @@ static inline void xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool *pool,
>
>  #else
>
> +static inline int xsk_validate_dev(struct net_device *dev, unsigned int mtu,
> +                                  struct bpf_prog *prog,
> +                                  enum bpf_xdp_mode mode,
> +                                  struct netlink_ext_ack *extack)
> +{
> +       return 0;
> +}
> +
> +static inline struct xsk_buff_pool *xsk_rxq_pool(struct net_device *dev,
> +                                                u16 queue_id)
> +{
> +       return NULL;
> +}
> +
>  static inline void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries)
>  {
>  }
> @@ -397,6 +437,11 @@ static inline bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool)
>         return false;
>  }
>
> +static inline bool xsk_pool_uses_sg(struct xsk_buff_pool *pool)
> +{
> +       return false;
> +}
> +
>  static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool)
>  {
>         return 0;
> diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h
> index 02e89350faf9..6d7cfdf99a8b 100644
> --- a/include/net/xsk_buff_pool.h
> +++ b/include/net/xsk_buff_pool.h
> @@ -11,6 +11,7 @@
>  #include <net/xdp.h>
>
>  struct xsk_buff_pool;
> +struct xsk_pp;
>  struct xdp_rxq_info;
>  struct xsk_cb_desc;
>  struct xsk_queue;
> @@ -54,6 +55,8 @@ struct xsk_buff_pool {
>         spinlock_t xsk_tx_list_lock;
>         refcount_t users;
>         struct xdp_umem *umem;
> +       /* page pool memory provider, see net/xdp/xsk_pp.c */
> +       struct xsk_pp *pp;
>         struct work_struct work;
>         /* Protects generic receive in shared and non-shared umem mode. */
>         spinlock_t rx_lock;
> diff --git a/net/core/dev.c b/net/core/dev.c
> index 663fe3e32ea0..7e985fb82488 100644
> --- a/net/core/dev.c
> +++ b/net/core/dev.c
> @@ -163,6 +163,7 @@
>  #include <net/page_pool/types.h>
>  #include <net/page_pool/helpers.h>
>  #include <net/page_pool/memory_provider.h>
> +#include <net/xdp_sock_drv.h>
>  #include <net/rps.h>
>  #include <linux/phy_link_topology.h>
>
> @@ -10535,6 +10536,8 @@ u8 dev_xdp_sb_prog_count(struct net_device *dev)
>
>  int netif_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf)
>  {
> +       int err;
> +
>         if (!dev->netdev_ops->ndo_bpf)
>                 return -EOPNOTSUPP;
>
> @@ -10552,6 +10555,13 @@ int netif_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf)
>                 return -EBUSY;
>         }
>
> +       if (bpf->command == XDP_SETUP_PROG && bpf->prog) {
> +               err = xsk_validate_dev(dev, dev->mtu, bpf->prog, XDP_MODE_DRV,
> +                                      bpf->extack);
> +               if (err)
> +                       return err;
> +       }
> +
>         return dev->netdev_ops->ndo_bpf(dev, bpf);
>  }
>  EXPORT_SYMBOL_GPL(netif_xdp_propagate);
> @@ -10629,6 +10639,12 @@ static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode,
>                 return -EBUSY;
>         }
>
> +       if (mode != XDP_MODE_HW && prog) {
> +               err = xsk_validate_dev(dev, dev->mtu, prog, mode, extack);
> +               if (err)
> +                       return err;
> +       }
> +
>         memset(&xdp, 0, sizeof(xdp));
>         xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG;
>         xdp.extack = extack;
> diff --git a/net/core/netdev_config.c b/net/core/netdev_config.c
> index 1975de42a60d..97e33764737b 100644
> --- a/net/core/netdev_config.c
> +++ b/net/core/netdev_config.c
> @@ -88,6 +88,8 @@ static int __netdev_queue_config(struct net_device *dev, int rxq_idx,
>         mpp = &__netif_get_rx_queue(dev, rxq_idx)->mp_params;
>         if (mpp->rx_page_size)
>                 qcfg->rx_page_size = mpp->rx_page_size;
> +       if (mpp->rx_headroom)
> +               qcfg->rx_headroom = mpp->rx_headroom;
>         err = validate_cb(dev, qcfg, extack);
>         if (err)
>                 return err;
> diff --git a/net/core/netdev_rx_queue.c b/net/core/netdev_rx_queue.c
> index 9e14a747f9b5..4e2f906fbc73 100644
> --- a/net/core/netdev_rx_queue.c
> +++ b/net/core/netdev_rx_queue.c
> @@ -6,6 +6,7 @@
>  #include <net/netdev_queues.h>
>  #include <net/netdev_rx_queue.h>
>  #include <net/page_pool/memory_provider.h>
> +#include <net/xdp_sock.h>
>
>  #include "dev.h"
>  #include "page_pool_priv.h"
> @@ -141,6 +142,7 @@ static int netdev_rx_queue_reconfig(struct net_device *dev,
>                 goto err_free_new_mem;
>         }
>
> +       rxq->mp_gen++;
>         err = qops->ndo_queue_mem_alloc(dev, qcfg_new, new_mem, rxq_idx);
>         if (err)
>                 goto err_free_old_mem;
> @@ -235,6 +237,10 @@ static int __netif_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx,
>                 NL_SET_ERR_MSG(extack, "device does not support: rx_page_size");
>                 return -EOPNOTSUPP;
>         }
> +       if (p->rx_headroom && !(qops->supported_params & QCFG_RX_HEADROOM)) {
> +               NL_SET_ERR_MSG(extack, "device does not support: rx_headroom");
> +               return -EOPNOTSUPP;
> +       }
>
>         rxq = __netif_get_rx_queue(dev, rxq_idx);
>         if (rxq->mp_params.mp_ops) {
> @@ -242,7 +248,8 @@ static int __netif_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx,
>                 return -EEXIST;
>         }
>  #ifdef CONFIG_XDP_SOCKETS
> -       if (rxq->pool) {
> +       /* An AF_XDP pool owns the queue, unless it installs its provider. */
> +       if (rxq->pool && p->mp_ops != &xsk_pp_ops) {
>                 NL_SET_ERR_MSG(extack, "designated queue already in use by AF_XDP");
>                 return -EBUSY;
>         }
> diff --git a/net/core/page_pool.c b/net/core/page_pool.c
> index 8237fda598ab..e897e79d31f1 100644
> --- a/net/core/page_pool.c
> +++ b/net/core/page_pool.c
> @@ -215,7 +215,8 @@ bool netmem_can_retain(netmem_ref netmem)
>  EXPORT_SYMBOL(netmem_can_retain);
>
>  static int page_pool_select_mp(struct page_pool *pool,
> -                              const struct pp_memory_provider_params *mpp)
> +                              const struct pp_memory_provider_params *mpp,
> +                              u32 rxq_mp_gen)
>  {
>         const struct memory_provider_ops *ops = mpp->mp_ops;
>         unsigned int flags = pool->slow.flags;
> @@ -234,6 +235,7 @@ static int page_pool_select_mp(struct page_pool *pool,
>
>         pool->mp_priv = mpp->mp_priv;
>         pool->mp_ops = ops;
> +       pool->mp_gen = rxq_mp_gen;
>         return 0;
>  }
>
> @@ -326,7 +328,7 @@ static int page_pool_init(struct page_pool *pool,
>                 netdev_assert_locked(pool->slow.netdev);
>                 rxq = __netif_get_rx_queue(pool->slow.netdev,
>                                            pool->slow.queue_idx);
> -               err = page_pool_select_mp(pool, &rxq->mp_params);
> +               err = page_pool_select_mp(pool, &rxq->mp_params, rxq->mp_gen);
>                 if (err)
>                         goto free_ptr_ring;
>         }
> diff --git a/net/core/page_pool_user.c b/net/core/page_pool_user.c
> index f30f3779624c..b69da0f74756 100644
> --- a/net/core/page_pool_user.c
> +++ b/net/core/page_pool_user.c
> @@ -398,28 +398,37 @@ void page_pool_unlist(struct page_pool *pool)
>         mutex_unlock(&page_pools_lock);
>  }
>
> +/* Check that the queue allocation that just ran put the provider in every
> + * pool role it needs: headers for a readable provider, payload otherwise.
> + * Pools of earlier allocations do not count.
> + */
>  int page_pool_check_memory_provider(struct net_device *dev,
>                                     struct netdev_rx_queue *rxq)
>  {
> -       void *binding = rxq->mp_params.mp_priv;
> +       const struct memory_provider_ops *ops = rxq->mp_params.mp_ops;
> +       void *mp_priv = rxq->mp_params.mp_priv;
> +       unsigned int need, have = 0;
>         struct page_pool *pool;
>         struct hlist_node *n;
>
> -       if (!binding)
> +       if (!mp_priv)
>                 return 0;
>
> +       need = PP_RX_ROLE_PAYLOAD;
> +       if (ops->caps & MP_CAP_READABLE)
> +               need = PP_RX_ROLE_HEAD;
> +
>         mutex_lock(&page_pools_lock);
>         hlist_for_each_entry_safe(pool, n, &dev->page_pools, user.list) {
> -               if (pool->mp_priv != binding)
> +               if (pool->mp_priv != mp_priv || pool->mp_gen != rxq->mp_gen ||
> +                   pool->slow.queue_idx != get_netdev_rx_queue_index(rxq))
>                         continue;
>
> -               if (pool->slow.queue_idx == get_netdev_rx_queue_index(rxq)) {
> -                       mutex_unlock(&page_pools_lock);
> -                       return 0;
> -               }
> +               have |= pool->slow.rx_roles ? : PP_RX_ROLE_PAYLOAD;
>         }
>         mutex_unlock(&page_pools_lock);
> -       return -ENODATA;
> +
> +       return (have & need) == need ? 0 : -ENODATA;
>  }
>
>  static void page_pool_unreg_netdev_wipe(struct net_device *netdev)
> diff --git a/net/xdp/xsk.h b/net/xdp/xsk.h
> index 4dab7b9a1487..9699e9a499a3 100644
> --- a/net/xdp/xsk.h
> +++ b/net/xdp/xsk.h
> @@ -36,10 +36,13 @@ void xsk_map_try_sock_delete(struct xsk_map *map, struct xdp_sock *xs,
>                              struct xdp_sock __rcu **map_entry);
>  struct xsk_pp;
>
> -extern const struct memory_provider_ops xsk_pp_ops;
> -
>  struct xsk_pp *xsk_pp_create(struct xsk_buff_pool *pool);
>  void xsk_pp_put(struct xsk_pp *pp);
> +bool xsk_pp_supported(struct net_device *dev);
> +int xsk_pp_bind(struct xsk_buff_pool *pool, struct net_device *dev,
> +               u16 queue_id);
> +void xsk_pp_unbind(struct xsk_buff_pool *pool);
> +int xp_alloc_heads(struct xsk_buff_pool *pool);
>  void xsk_clear_pool_at_qid(struct net_device *dev, u16 queue_id);
>  int xsk_reg_pool_at_qid(struct net_device *dev, struct xsk_buff_pool *pool,
>                         u16 queue_id);
> diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c
> index 18966bbdb5b7..c620a0c32c17 100644
> --- a/net/xdp/xsk_buff_pool.c
> +++ b/net/xdp/xsk_buff_pool.c
> @@ -38,6 +38,8 @@ void xp_destroy(struct xsk_buff_pool *pool)
>         if (!pool)
>                 return;
>
> +       if (pool->pp)
> +               xsk_pp_put(pool->pp);
>         kvfree(pool->tx_descs);
>         kvfree(pool->heads);
>         kvfree(pool);
> @@ -62,18 +64,13 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs,
>  {
>         bool unaligned = umem->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG;
>         struct xsk_buff_pool *pool;
> -       struct xdp_buff_xsk *xskb;
> -       u32 i, entries;
> +       u32 entries;
>
>         entries = unaligned ? umem->chunks : 0;
>         pool = kvzalloc_flex(*pool, free_heads, entries);
>         if (!pool)
>                 goto out;
>
> -       pool->heads = kvzalloc_objs(*pool->heads, umem->chunks);
> -       if (!pool->heads)
> -               goto out;
> -
>         if (xs->tx)
>                 if (xp_alloc_tx_descs(pool, xs, max_segs))
>                         goto out;
> @@ -104,10 +101,30 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs,
>         pool->fq = xs->fq_tmp;
>         pool->cq = xs->cq_tmp;
>
> -       for (i = 0; i < pool->free_heads_cnt; i++) {
> +       return pool;
> +
> +out:
> +       xp_destroy(pool);
> +       return NULL;
> +}
> +
> +/* Only copy mode and driver zero copy use heads */
> +int xp_alloc_heads(struct xsk_buff_pool *pool)
> +{
> +       struct xdp_buff_xsk *xskb;
> +       u32 i;
> +
> +       if (pool->heads)
> +               return 0;
> +
> +       pool->heads = kvzalloc_objs(*pool->heads, pool->heads_cnt);
> +       if (!pool->heads)
> +               return -ENOMEM;
> +
> +       for (i = 0; i < pool->heads_cnt; i++) {
>                 xskb = &pool->heads[i];
>                 xskb->pool = pool;
> -               xskb->xdp.frame_sz = umem->chunk_size - umem->headroom;
> +               xskb->xdp.frame_sz = pool->chunk_size - pool->headroom;
>                 INIT_LIST_HEAD(&xskb->list_node);
>                 if (pool->unaligned)
>                         pool->free_heads[i] = xskb;
> @@ -115,11 +132,7 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs,
>                         xp_init_xskb_addr(xskb, pool, (u64)i * pool->chunk_size);
>         }
>
> -       return pool;
> -
> -out:
> -       xp_destroy(pool);
> -       return NULL;
> +       return 0;
>  }
>
>  void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq)
> @@ -212,9 +225,24 @@ int xp_assign_dev(struct xsk_buff_pool *pool,
>          */
>         hlist_add_head(&pool->dev_node, &netdev->xsk_pools);
>
> -       if (force_copy)
> +       if (force_copy) {
>                 /* For copy-mode, we are done. */
> +               err = xp_alloc_heads(pool);
> +               goto out;
> +       }
> +
> +       if (xsk_pp_supported(netdev)) {
> +               err = xsk_pp_bind(pool, netdev, queue_id);
> +               if (err)
> +                       goto err_unreg_pool;
> +
> +               pool->umem->zc = true;
>                 return 0;
> +       }
> +
> +       err = xp_alloc_heads(pool);
> +       if (err)
> +               goto out;
>
>         if ((netdev->xdp_features & NETDEV_XDP_ACT_XSK) != NETDEV_XDP_ACT_XSK) {
>                 err = -EOPNOTSUPP;
> @@ -267,8 +295,10 @@ int xp_assign_dev(struct xsk_buff_pool *pool,
>  err_unreg_xsk:
>         xp_disable_drv_zc(pool);
>  err_unreg_pool:
> -       if (!force_zc)
> -               err = 0; /* fallback to copy mode */
> +       /* An old page pool may still own chunks of a failed provider. */
> +       if (!force_zc && !pool->pp)
> +               err = xp_alloc_heads(pool); /* fallback to copy mode */
> +out:
>         if (err) {
>                 pool->umem->flags = umem_flags;
>                 xsk_clear_pool_at_qid(netdev, queue_id);
> @@ -301,7 +331,10 @@ static void __xp_clear_dev(struct xsk_buff_pool *pool)
>         struct net_device *netdev = pool->netdev;
>
>         netdev_assert_locked_ops_compat(netdev);
> -       xp_disable_drv_zc(pool);
> +       if (pool->pp)
> +               xsk_pp_unbind(pool);
> +       else
> +               xp_disable_drv_zc(pool);
>         xsk_clear_pool_at_qid(netdev, pool->queue_id);
>         hlist_del(&pool->dev_node);
>         pool->netdev = NULL;
> @@ -474,7 +507,7 @@ static void xp_check_dma_contiguity(struct xsk_dma_map *dma_map)
>
>  static int xp_init_dma_info(struct xsk_buff_pool *pool, struct xsk_dma_map *dma_map)
>  {
> -       if (!pool->unaligned) {
> +       if (!pool->unaligned && pool->heads) {
>                 u32 i;
>
>                 for (i = 0; i < pool->heads_cnt; i++) {
> diff --git a/net/xdp/xsk_pp.c b/net/xdp/xsk_pp.c
> index fdd55265a6cd..c2214e718cdb 100644
> --- a/net/xdp/xsk_pp.c
> +++ b/net/xdp/xsk_pp.c
> @@ -8,6 +8,9 @@
>   *       +-----RX ring-------+
>   */
>
> +#include <net/netdev_lock.h>
> +#include <net/netdev_queues.h>
> +#include <net/netdev_rx_queue.h>
>  #include <net/page_pool/helpers.h>
>  #include <net/page_pool/memory_provider.h>
>  #include <net/xdp_sock_drv.h>
> @@ -15,6 +18,8 @@
>  #include "xsk.h"
>  #include "xsk_queue.h"
>
> +#define XSK_PP_DMA_ATTR        (DMA_ATTR_WEAK_ORDERING | DMA_ATTR_SKIP_CPU_SYNC)
> +
>  enum xsk_pp_chunk {
>         XSK_PP_USER,    /* owned by userspace */
>         XSK_PP_POOL,    /* owned by a page pool */
> @@ -30,6 +35,8 @@ struct xsk_pp {
>         refcount_t refs;
>         u32 nr_pools;
>         bool dma_need_sync;
> +       /* RX queue of the provider, NULL once it is removed */
> +       struct netdev_rx_queue *rxq;
>         u8 *state;
>         u32 reuse_cnt;
>         u32 reuse[];
> @@ -236,11 +243,225 @@ static int xsk_pp_nl_fill(void *mp_priv, struct sk_buff *rsp,
>         return 0;
>  }
>
> +/* The queue no longer uses the provider, and its DMA stopped */
> +static void xsk_pp_detach(struct xsk_pp *pp)
> +{
> +       pp->rxq = NULL;
> +       spin_lock_bh(&pp->lock);
> +       pp->fq = NULL;
> +       spin_unlock_bh(&pp->lock);
> +       xp_dma_unmap(pp->pool, XSK_PP_DMA_ATTR);
> +}
> +
> +/* The device unregisters, and is closed */
> +static void xsk_pp_uninstall(void *mp_priv, struct netdev_rx_queue *rxq)
> +{
> +       xsk_pp_detach(mp_priv);
> +}
> +
>  const struct memory_provider_ops xsk_pp_ops = {
>         .init           = xsk_pp_init,
>         .destroy        = xsk_pp_destroy,
>         .alloc_netmems  = xsk_pp_alloc_netmems,
>         .release_netmem = xsk_pp_release_netmem,
>         .nl_fill        = xsk_pp_nl_fill,
> +       .uninstall      = xsk_pp_uninstall,
>         .caps           = MP_CAP_READABLE,
>  };
> +
> +static bool xsk_pp_abandon(struct xsk_pp *pp)
> +{
> +       struct xsk_buff_pool *pool = pp->pool;
> +       bool busy;
> +       u32 idx;
> +
> +       spin_lock_bh(&pp->lock);
> +       busy = pp->nr_pools;
> +       while (!busy && pp->reuse_cnt) {
> +               idx = pp->reuse[--pp->reuse_cnt];
> +               pp->state[idx] = XSK_PP_USER;
> +               xp_free(&pool->heads[idx]);
> +       }
> +       spin_unlock_bh(&pp->lock);
> +
> +       return !busy;
> +}
> +
> +static int xsk_pp_validate(struct net_device *dev, struct xsk_buff_pool *pool,
> +                          unsigned int mtu, struct bpf_prog *prog,
> +                          struct netlink_ext_ack *extack)
> +{
> +       struct xsk_rx_check check = {
> +               .buf_size = pool->chunk_size,
> +               .headroom = xsk_pool_get_headroom(pool),
> +               .mtu = mtu,
> +               .sg = xsk_pool_uses_sg(pool),
> +               .prog = prog,
> +       };
> +       struct netdev_bpf bpf = {
> +               .command = XDP_VALIDATE_XSK_POOL,
> +               .xsk_check.check = &check,
> +               .xsk_check.extack = extack,
> +       };
> +
> +       return dev->netdev_ops->ndo_bpf(dev, &bpf);
> +}
> +
> +/**
> + * xsk_validate_dev - check AF_XDP page pool queues against a new setting
> + * @dev: device of the queues
> + * @mtu: MTU to check
> + * @prog: XDP program to check, or NULL for the current one
> + * @mode: XDP mode of @prog
> + * @extack: netlink extended ack
> + *
> + * Context: holds the instance lock of @dev, or RTNL for a device without it.
> + * Return: 0 if the driver accepts all queues of @dev that have an AF_XDP
> + * memory provider, or its error.
> + */
> +int xsk_validate_dev(struct net_device *dev, unsigned int mtu,
> +                    struct bpf_prog *prog, enum bpf_xdp_mode mode,
> +                    struct netlink_ext_ack *extack)
> +{
> +       struct netdev_rx_queue *rxq;
> +       struct xsk_pp *pp;
> +       unsigned int i;
> +       int err;
> +
> +       netdev_assert_locked_ops_compat(dev);
> +
> +       for (i = 0; i < dev->real_num_rx_queues; i++) {
> +               rxq = __netif_get_rx_queue(dev, i);
> +               if (rxq->mp_params.mp_ops != &xsk_pp_ops)
> +                       continue;
> +
> +               /* Generic XDP runs outside the queue NAPI */
> +               if (mode == XDP_MODE_SKB) {
> +                       NL_SET_ERR_MSG(extack,
> +                                      "generic XDP cannot feed an AF_XDP page pool socket");
> +                       return -EBUSY;
> +               }
> +
> +               pp = rxq->mp_params.mp_priv;
> +               err = xsk_pp_validate(dev, pp->pool, mtu, prog, extack);
> +               if (err)
> +                       return err;
> +       }
> +
> +       return 0;
> +}
> +EXPORT_SYMBOL_GPL(xsk_validate_dev);
> +
> +/**
> + * xsk_rxq_pool - get the AF_XDP pool that provides memory to an RX queue
> + * @dev: device of the queue
> + * @queue_id: RX queue
> + *
> + * The pool stays valid while the queue that the caller allocates runs.
> + *
> + * Context: holds the instance lock of @dev.
> + * Return: the buffer pool, or NULL.
> + */
> +struct xsk_buff_pool *xsk_rxq_pool(struct net_device *dev, u16 queue_id)
> +{
> +       struct netdev_rx_queue *rxq;
> +       struct xsk_pp *pp;
> +
> +       netdev_assert_locked(dev);
> +
> +       if (queue_id >= dev->real_num_rx_queues)
> +               return NULL;
> +
> +       rxq = __netif_get_rx_queue(dev, queue_id);
> +       if (rxq->mp_params.mp_ops != &xsk_pp_ops)
> +               return NULL;
> +
> +       pp = rxq->mp_params.mp_priv;
> +       return pp->pool;
> +}
> +EXPORT_SYMBOL_GPL(xsk_rxq_pool);
> +
> +bool xsk_pp_supported(struct net_device *dev)
> +{
> +       const struct netdev_queue_mgmt_ops *qops = dev->queue_mgmt_ops;
> +
> +       return qops && qops->xsk_page_pool;
> +}
> +
> +/* Called with RTNL and the instance lock of @dev. On error, @pool->pp stays
> + * set if an old page pool still owns chunks; copy mode must not take the pool.
> + */
> +int xsk_pp_bind(struct xsk_buff_pool *pool, struct net_device *dev,
> +               u16 queue_id)
> +{
> +       struct pp_memory_provider_params mp = { .mp_ops = &xsk_pp_ops };
> +       struct device *dma_dev;
> +       struct xsk_pp *pp;
> +       int err;
> +
> +       if (queue_id >= dev->real_num_rx_queues || pool->unaligned)
> +               return -EOPNOTSUPP;
> +       if ((dev->xdp_features & NETDEV_XDP_ACT_XSK) != NETDEV_XDP_ACT_XSK ||
> +           (xsk_pool_uses_sg(pool) && dev->xdp_zc_max_segs == 1))
> +               return -EOPNOTSUPP;
> +       if (dev_xdp_prog_id(dev, XDP_MODE_SKB))
> +               return -EOPNOTSUPP;
> +       if (!netif_all_mps_have_cap(dev, MP_CAP_READABLE))
> +               return -EBUSY;
> +
> +       err = xsk_pp_validate(dev, pool, dev->mtu, NULL, NULL);
> +       if (err)
> +               return err;
> +
> +       dma_dev = netdev_queue_get_dma_dev(dev, queue_id, NETDEV_QUEUE_TYPE_RX);
> +       if (!dma_dev)
> +               return -EOPNOTSUPP;
> +
> +       err = xp_dma_map(pool, dma_dev, XSK_PP_DMA_ATTR, pool->umem->pgs,
> +                        pool->umem->npgs);
> +       if (err)
> +               return err;
> +
> +       pp = xsk_pp_create(pool);
> +       if (IS_ERR(pp)) {
> +               xp_dma_unmap(pool, XSK_PP_DMA_ATTR);
> +               return PTR_ERR(pp);
> +       }
> +
> +       mp.mp_priv = pp;
> +       mp.rx_page_size = pool->chunk_size;
> +       mp.rx_headroom = xsk_pool_get_headroom(pool);
> +       err = netif_mp_open_rxq(dev, queue_id, &mp, NULL);
> +       if (!err) {
> +               pp->rxq = __netif_get_rx_queue(dev, queue_id);
> +               pool->pp = pp;
> +               pool->xdp_zc_max_segs = dev->xdp_zc_max_segs;
> +               return 0;
> +       }
> +
> +       xp_dma_unmap(pool, XSK_PP_DMA_ATTR);
> +       /* Copy mode takes the chunks that the provider holds */
> +       if (!xp_alloc_heads(pool) && xsk_pp_abandon(pp))
> +               xsk_pp_put(pp);
> +       else
> +               pool->pp = pp;
> +
> +       return err;
> +}
> +
> +/* Called with RTNL and the instance lock of the device */
> +void xsk_pp_unbind(struct xsk_buff_pool *pool)
> +{
> +       struct xsk_pp *pp = pool->pp;
> +       struct pp_memory_provider_params mp = {
> +               .mp_ops = &xsk_pp_ops,
> +               .mp_priv = pp,
> +       };
> +
> +       /* Unregister removed the provider already */
> +       if (!pp->rxq)
> +               return;
> +
> +       netif_mp_close_rxq(pool->netdev, pool->queue_id, &mp);
> +       xsk_pp_detach(pp);

Need to handle netif_mp_close_rxq() failing here maybe?

-- 
Thanks,
Mina