[PATCH net-next 0/2] net: airoha: Add XDP support

[PATCH net-next 0/2] net: airoha: Add XDP support

[Til Kaiser]

Hello,

This series adds XDP support to the Airoha Ethernet driver.

Patch 1 adds the RX- and TX-path support needed for XDP_PASS, XDP_DROP,
XDP_TX, and XDP_REDIRECT.

Patch 2 exposes XDP and page-pool statistics through ethtool.

Thanks,
Til

Til Kaiser (2):
  net: airoha: Add XDP support
  net: airoha: Add XDP statistics

 drivers/net/ethernet/airoha/Kconfig      |   1 +
 drivers/net/ethernet/airoha/airoha_eth.c | 493 +++++++++++++++++++++--
 drivers/net/ethernet/airoha/airoha_eth.h |  32 +-
 3 files changed, 496 insertions(+), 30 deletions(-)

-- 
2.55.0

[PATCH net-next 1/2] net: airoha: Add XDP support

[Til Kaiser]

Implement eXpress Data Path (XDP) support for the Airoha Ethernet driver.

XDP programs are attached per net_device via the ndo_bpf hook. The BPF
program reference is stored as an RCU-protected pointer in struct
airoha_gdm_dev and replaced atomically on program load/unload via
rcu_replace_pointer().

To support XDP, the following changes are made to the RX path:

- The page pool DMA direction is switched from DMA_FROM_DEVICE to
  DMA_BIDIRECTIONAL, which is required for XDP_TX since the same page
  can be reused for transmit.
- The RX headroom is increased from NET_SKB_PAD to XDP_PACKET_HEADROOM
  so that XDP programs have the required headroom available in the data
  buffer.
- RX buffers use a full page rather than a half-page fragment. This leaves
  sufficient space for XDP headroom, skb_shared_info tailroom, and a
  standard 1500-byte MTU.
- AIROHA_RX_MAX_BUF_SIZE expresses the maximum MTU that fits in an RX
  buffer after headroom, shared-info, Ethernet, VLAN, and FCS overhead.
  MTU changes are rejected when an XDP program is loaded and the requested
  MTU would exceed this limit.
- xdp_rxq_info is registered for each RX queue during initialisation and
  unregistered on cleanup, with the page pool set as the memory model.

The XDP program is run in airoha_run_xdp() for each received buffer.
The following actions are supported:

- XDP_PASS: the buffer is passed to the normal networking stack.
- XDP_TX: the buffer is transmitted back out the same interface using
  the new airoha_xdp_xmit_back() helper.
- XDP_REDIRECT: the buffer is redirected to another interface or map
  via xdp_do_redirect(), with xdp_do_flush() deferred to the end of
  the NAPI poll via a per-queue xdp_flush flag.
- XDP_DROP (and unknown actions): the page is returned to the page pool.

Two new TX path helpers are introduced:

- airoha_xdp_submit_frame() maps an xdp_frame for DMA (or reuses the
  page pool DMA address for XDP_TX) and enqueues it into a TX ring,
  handling multi-buffer frames via skb_shared_info fragments.
- airoha_xdp_xmit_back() converts an xdp_buff to an xdp_frame and
  submits it to the TX queue corresponding to the current CPU.

The ndo_xdp_xmit handler airoha_xdp_xmit() is also added to allow
XDP redirect from other drivers into the Airoha interface. Frames are
submitted using airoha_xdp_submit_frame() with dma_map=true.

struct airoha_queue_entry is extended with an is_xdpf flag and an
xdp_frame pointer (in a union with the existing sk_buff pointer) so
that the TX completion path can correctly free either XDP frames or
SKBs.

The advertised XDP feature flags are set to:
  NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | NETDEV_XDP_ACT_NDO_XMIT

Signed-off-by: Til Kaiser <mail@tk154.de>
---
 drivers/net/ethernet/airoha/airoha_eth.c | 386 +++++++++++++++++++++--
 drivers/net/ethernet/airoha/airoha_eth.h |  19 +-
 2 files changed, 375 insertions(+), 30 deletions(-)

diff --git a/drivers/net/ethernet/airoha/airoha_eth.c b/drivers/net/ethernet/airoha/airoha_eth.c
index 64619e9a704d..71b25f225a8a 100644
--- a/drivers/net/ethernet/airoha/airoha_eth.c
+++ b/drivers/net/ethernet/airoha/airoha_eth.c
@@ -13,6 +13,7 @@
 #include <net/page_pool/helpers.h>
 #include <net/pkt_cls.h>
 #include <uapi/linux/ppp_defs.h>
+#include <linux/bpf_trace.h>
 
 #include "airoha_regs.h"
 #include "airoha_eth.h"
@@ -657,6 +658,255 @@ airoha_qdma_get_gdm_dev(struct airoha_eth *eth, struct airoha_qdma_desc *desc)
 	return port->devs[d] ? port->devs[d] : ERR_PTR(-ENODEV);
 }
 
+static void airoha_unmap_xmit_buf(struct airoha_eth *eth,
+				  struct airoha_queue_entry *e)
+{
+	switch (e->dma_type) {
+	case AIROHA_DMA_MAP_PAGE:
+		dma_unmap_page(eth->dev, e->dma_addr, e->dma_len,
+			       DMA_TO_DEVICE);
+		break;
+	case AIROHA_DMA_MAP_SINGLE:
+		dma_unmap_single(eth->dev, e->dma_addr, e->dma_len,
+				 DMA_TO_DEVICE);
+		break;
+	case AIROHA_DMA_UNMAPPED:
+	default:
+		break;
+	}
+	e->dma_type = AIROHA_DMA_UNMAPPED;
+}
+
+static int airoha_xdp_submit_frame(struct airoha_gdm_dev *dev,
+				   struct xdp_frame *xdpf,
+				   struct airoha_queue *q,
+				   int qid, u32 msg0, u32 msg1,
+				   bool dma_map)
+{
+	struct airoha_queue_entry *e, *next_e;
+	int len = xdpf->len, nr_frags, i;
+	struct airoha_qdma_desc *desc;
+	struct skb_shared_info *sinfo;
+	void *data = xdpf->data;
+	u16 index, next_index;
+	LIST_HEAD(tx_list);
+	dma_addr_t addr;
+	u32 val;
+
+	sinfo = xdp_get_shared_info_from_frame(xdpf);
+	nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;
+
+	if (q->queued >= q->ndesc - 1 - nr_frags)
+		return -EBUSY;
+
+	for (i = 0; i <= nr_frags; i++) {
+		if (dma_map) {
+			if (i == 0)
+				addr = dma_map_single(dev->eth->dev, data, len,
+						      DMA_TO_DEVICE);
+			else
+				addr = dma_map_page(dev->eth->dev, virt_to_page(data),
+						    offset_in_page(data), len,
+						    DMA_TO_DEVICE);
+			if (unlikely(dma_mapping_error(dev->eth->dev, addr)))
+				goto unmap;
+		} else {
+			struct page *page = virt_to_head_page(data);
+
+			addr = page_pool_get_dma_addr(page) +
+			       (data - (void *)page_address(page));
+			dma_sync_single_for_device(dev->eth->dev, addr, len,
+						   DMA_BIDIRECTIONAL);
+		}
+
+		e = list_first_entry(&q->tx_list, struct airoha_queue_entry, list);
+		list_move_tail(&e->list, &tx_list);
+
+		index = e - q->entry;
+		desc = &q->desc[index];
+
+		e->is_xdpf = true;
+		e->xdpf = (i == nr_frags) ? xdpf : NULL;
+		e->dma_addr = addr;
+		e->dma_len = len;
+		if (dma_map)
+			e->dma_type = i == 0 ? AIROHA_DMA_MAP_SINGLE : AIROHA_DMA_MAP_PAGE;
+		else
+			e->dma_type = AIROHA_DMA_UNMAPPED;
+
+		next_e = list_first_entry(&q->tx_list, struct airoha_queue_entry, list);
+		next_index = next_e - q->entry;
+
+		val = FIELD_PREP(QDMA_DESC_LEN_MASK, len);
+		if (i < nr_frags)
+			val |= FIELD_PREP(QDMA_DESC_MORE_MASK, 1);
+		WRITE_ONCE(desc->ctrl, cpu_to_le32(val));
+		WRITE_ONCE(desc->addr, cpu_to_le32(addr));
+		val = FIELD_PREP(QDMA_DESC_NEXT_ID_MASK, next_index);
+		WRITE_ONCE(desc->data, cpu_to_le32(val));
+		WRITE_ONCE(desc->msg0, cpu_to_le32(msg0));
+		WRITE_ONCE(desc->msg1, cpu_to_le32(msg1));
+		WRITE_ONCE(desc->msg2, cpu_to_le32(0xffff));
+
+		q->queued++;
+
+		if (i < nr_frags) {
+			skb_frag_t *frag = &sinfo->frags[i];
+
+			data = skb_frag_address(frag);
+			len = skb_frag_size(frag);
+		}
+	}
+
+	return next_index;
+
+unmap:
+	list_for_each_entry(e, &tx_list, list) {
+		airoha_unmap_xmit_buf(dev->eth, e);
+		e->is_xdpf = false;
+		e->xdpf = NULL;
+		q->queued--;
+	}
+	list_splice(&tx_list, &q->tx_list);
+
+	return -ENOMEM;
+}
+
+static int airoha_xdp_xmit_back(struct net_device *netdev, struct xdp_buff *xdp,
+				struct airoha_qdma *qdma)
+{
+	struct airoha_gdm_dev *dev = netdev_priv(netdev);
+	struct airoha_queue *q;
+	struct xdp_frame *xdpf;
+	u32 msg0, msg1;
+	int qid, index;
+	u8 fport;
+
+	xdpf = xdp_convert_buff_to_frame(xdp);
+	if (unlikely(!xdpf))
+		return -EOVERFLOW;
+
+	qid = airoha_qdma_get_txq(qdma, smp_processor_id());
+	q = &qdma->q_tx[qid];
+
+	msg0 = FIELD_PREP(QDMA_ETH_TXMSG_CHAN_MASK,
+			  qid / AIROHA_NUM_QOS_QUEUES) |
+	       FIELD_PREP(QDMA_ETH_TXMSG_QUEUE_MASK,
+			  qid % AIROHA_NUM_QOS_QUEUES);
+
+	fport = airoha_get_fe_port(dev);
+	msg1 = FIELD_PREP(QDMA_ETH_TXMSG_NBOQ_MASK, dev->nbq) |
+	       FIELD_PREP(QDMA_ETH_TXMSG_FPORT_MASK, fport) |
+	       FIELD_PREP(QDMA_ETH_TXMSG_METER_MASK, 0x7f);
+
+	spin_lock(&q->lock);
+	index = airoha_xdp_submit_frame(dev, xdpf, q, qid, msg0, msg1, false);
+	if (unlikely(index < 0)) {
+		spin_unlock(&q->lock);
+		return index;
+	}
+
+	airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid),
+			TX_RING_CPU_IDX_MASK,
+			FIELD_PREP(TX_RING_CPU_IDX_MASK, index));
+	spin_unlock(&q->lock);
+
+	return 0;
+}
+
+static int airoha_xdp_xmit(struct net_device *netdev, int n,
+			   struct xdp_frame **frames, u32 flags)
+{
+	int qid, index = 0, last_index = -1, i, drops = 0;
+	struct airoha_gdm_dev *dev = netdev_priv(netdev);
+	struct airoha_qdma *qdma;
+	struct airoha_queue *q;
+	u32 msg0, msg1;
+	u8 fport;
+
+	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
+		return -EINVAL;
+
+	rcu_read_lock();
+	qdma = rcu_dereference(dev->qdma);
+	if (!qdma) {
+		rcu_read_unlock();
+		return -ENODEV;
+	}
+
+	qid = airoha_qdma_get_txq(qdma, smp_processor_id());
+	q = &qdma->q_tx[qid];
+
+	msg0 = FIELD_PREP(QDMA_ETH_TXMSG_CHAN_MASK,
+			  qid / AIROHA_NUM_QOS_QUEUES) |
+	       FIELD_PREP(QDMA_ETH_TXMSG_QUEUE_MASK,
+			  qid % AIROHA_NUM_QOS_QUEUES);
+
+	fport = airoha_get_fe_port(dev);
+	msg1 = FIELD_PREP(QDMA_ETH_TXMSG_NBOQ_MASK, dev->nbq) |
+	       FIELD_PREP(QDMA_ETH_TXMSG_FPORT_MASK, fport) |
+	       FIELD_PREP(QDMA_ETH_TXMSG_METER_MASK, 0x7f);
+
+	spin_lock(&q->lock);
+	for (i = 0; i < n; i++) {
+		index = airoha_xdp_submit_frame(dev, frames[i], q, qid, msg0, msg1, true);
+		if (unlikely(index < 0)) {
+			xdp_return_frame_rx_napi(frames[i]);
+			drops++;
+		} else {
+			last_index = index;
+		}
+	}
+	if (n > drops && (flags & XDP_XMIT_FLUSH))
+		airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid),
+				TX_RING_CPU_IDX_MASK,
+				FIELD_PREP(TX_RING_CPU_IDX_MASK, last_index));
+	spin_unlock(&q->lock);
+	rcu_read_unlock();
+
+	return n - drops;
+}
+
+static bool airoha_run_xdp(struct net_device *netdev, struct bpf_prog *prog,
+			   struct xdp_buff *xdp, struct airoha_queue *q,
+			   struct airoha_queue_entry *e, struct page *page)
+{
+	u32 act = bpf_prog_run_xdp(prog, xdp);
+
+	switch (act) {
+	case XDP_PASS:
+		return false;
+	case XDP_TX:
+		if (unlikely(airoha_xdp_xmit_back(netdev, xdp, q->qdma) < 0)) {
+			trace_xdp_exception(netdev, prog, act);
+			page_pool_put_full_page(q->page_pool, page, true);
+		} else {
+			e->buf = NULL;
+		}
+		break;
+	case XDP_REDIRECT:
+		if (unlikely(xdp_do_redirect(netdev, xdp, prog) < 0)) {
+			trace_xdp_exception(netdev, prog, act);
+			page_pool_put_full_page(q->page_pool, page, true);
+		} else {
+			q->xdp_flush = true;
+			e->buf = NULL;
+		}
+		break;
+	default:
+		bpf_warn_invalid_xdp_action(netdev, prog, act);
+		fallthrough;
+	case XDP_ABORTED:
+		trace_xdp_exception(netdev, prog, act);
+		fallthrough;
+	case XDP_DROP:
+		page_pool_put_full_page(q->page_pool, page, true);
+		break;
+	}
+
+	return true;
+}
+
 static int airoha_qdma_rx_process(struct airoha_queue *q, int budget)
 {
 	enum dma_data_direction dir = page_pool_get_dma_dir(q->page_pool);
@@ -698,6 +948,27 @@ static int airoha_qdma_rx_process(struct airoha_queue *q, int budget)
 
 		netdev = netdev_from_priv(dev);
 		if (!q->skb) { /* first buffer */
+			struct bpf_prog *xdp_prog;
+
+			rcu_read_lock();
+			xdp_prog = rcu_dereference(dev->xdp_prog);
+			if (xdp_prog) {
+				struct xdp_buff xdp;
+
+				xdp_init_buff(&xdp, q->buf_size, &q->xdp_rxq);
+				xdp_prepare_buff(&xdp, e->buf - AIROHA_RX_HEADROOM,
+						 AIROHA_RX_HEADROOM, len, false);
+
+				if (airoha_run_xdp(netdev, xdp_prog, &xdp, q, e, page)) {
+					rcu_read_unlock();
+					continue;
+				}
+
+				len = xdp.data_end - xdp.data;
+				e->buf = xdp.data;
+			}
+			rcu_read_unlock();
+
 			q->skb = napi_build_skb(e->buf - AIROHA_RX_HEADROOM,
 						q->buf_size);
 			if (!q->skb)
@@ -779,6 +1050,11 @@ static int airoha_qdma_rx_napi_poll(struct napi_struct *napi, int budget)
 		done += cur;
 	} while (cur && done < budget);
 
+	if (q->xdp_flush) {
+		xdp_do_flush();
+		q->xdp_flush = false;
+	}
+
 	if (done < budget && napi_complete(napi)) {
 		struct airoha_qdma *qdma = q->qdma;
 		int i, qid = q - &qdma->q_rx[0];
@@ -804,17 +1080,17 @@ static int airoha_qdma_init_rx_queue(struct airoha_queue *q,
 		.order = 0,
 		.pool_size = 256,
 		.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
-		.dma_dir = DMA_FROM_DEVICE,
+		.dma_dir = DMA_BIDIRECTIONAL,
 		.max_len = PAGE_SIZE,
 		.nid = NUMA_NO_NODE,
 		.dev = qdma->eth->dev,
 		.napi = &q->napi,
 	};
+	int qid = q - &qdma->q_rx[0], thr, err;
 	struct airoha_eth *eth = qdma->eth;
-	int qid = q - &qdma->q_rx[0], thr;
 	dma_addr_t dma_addr;
 
-	q->buf_size = PAGE_SIZE / 2;
+	q->buf_size = PAGE_SIZE;
 	q->qdma = qdma;
 
 	q->entry = devm_kzalloc(eth->dev, ndesc * sizeof(*q->entry),
@@ -829,15 +1105,22 @@ static int airoha_qdma_init_rx_queue(struct airoha_queue *q,
 
 	q->page_pool = page_pool_create(&pp_params);
 	if (IS_ERR(q->page_pool)) {
-		int err = PTR_ERR(q->page_pool);
-
-		q->page_pool = NULL;
-		return err;
+		err = PTR_ERR(q->page_pool);
+		goto err_page_pool_create;
 	}
 
 	q->ndesc = ndesc;
 	netif_napi_add(eth->napi_dev, &q->napi, airoha_qdma_rx_napi_poll);
 
+	err = xdp_rxq_info_reg(&q->xdp_rxq, eth->napi_dev, qid, q->napi.napi_id);
+	if (err)
+		goto err_xdp_rxq_info_reg;
+
+	err = xdp_rxq_info_reg_mem_model(&q->xdp_rxq, MEM_TYPE_PAGE_POOL,
+					 q->page_pool);
+	if (err)
+		goto err_xdp_rxq_info_reg_mem_model;
+
 	airoha_qdma_wr(qdma, REG_RX_RING_BASE(qid), dma_addr);
 	airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid),
 			RX_RING_SIZE_MASK,
@@ -853,6 +1136,14 @@ static int airoha_qdma_init_rx_queue(struct airoha_queue *q,
 	airoha_qdma_fill_rx_queue(q);
 
 	return 0;
+
+err_xdp_rxq_info_reg_mem_model:
+	xdp_rxq_info_unreg(&q->xdp_rxq);
+err_xdp_rxq_info_reg:
+	page_pool_destroy(q->page_pool);
+err_page_pool_create:
+	q->page_pool = NULL;
+	return err;
 }
 
 static void airoha_qdma_cleanup_rx_queue(struct airoha_queue *q)
@@ -882,6 +1173,9 @@ static void airoha_qdma_cleanup_rx_queue(struct airoha_queue *q)
 		q->queued--;
 	}
 
+	if (xdp_rxq_info_is_reg(&q->xdp_rxq))
+		xdp_rxq_info_unreg(&q->xdp_rxq);
+
 	q->head = q->tail;
 	/* Set RX_DMA_IDX to RX_CPU_IDX to notify the hw the QDMA RX ring is
 	 * empty.
@@ -949,25 +1243,6 @@ static void airoha_qdma_wake_netdev_txqs(struct airoha_queue *q)
 	q->txq_stopped = false;
 }
 
-static void airoha_unmap_xmit_buf(struct airoha_eth *eth,
-				  struct airoha_queue_entry *e)
-{
-	switch (e->dma_type) {
-	case AIROHA_DMA_MAP_PAGE:
-		dma_unmap_page(eth->dev, e->dma_addr, e->dma_len,
-			       DMA_TO_DEVICE);
-		break;
-	case AIROHA_DMA_MAP_SINGLE:
-		dma_unmap_single(eth->dev, e->dma_addr, e->dma_len,
-				 DMA_TO_DEVICE);
-		break;
-	case AIROHA_DMA_UNMAPPED:
-	default:
-		break;
-	}
-	e->dma_type = AIROHA_DMA_UNMAPPED;
-}
-
 static int airoha_qdma_tx_napi_poll(struct napi_struct *napi, int budget)
 {
 	struct airoha_tx_irq_queue *irq_q;
@@ -1037,7 +1312,12 @@ static int airoha_qdma_tx_napi_poll(struct napi_struct *napi, int budget)
 		WRITE_ONCE(desc->msg1, 0);
 		q->queued--;
 
-		if (skb) {
+		if (e->is_xdpf) {
+			if (e->xdpf)
+				xdp_return_frame(e->xdpf);
+			e->is_xdpf = false;
+			e->xdpf = NULL;
+		} else if (e->skb) {
 			struct airoha_gdm_dev *dev = netdev_priv(skb->dev);
 			u16 qidx = skb_get_queue_mapping(skb);
 			struct netdev_queue *txq;
@@ -1218,7 +1498,12 @@ static void airoha_qdma_tx_cleanup(struct airoha_qdma *qdma)
 			WRITE_ONCE(desc->msg1, 0);
 			WRITE_ONCE(desc->msg2, 0);
 
-			if (skb) {
+			if (e->is_xdpf) {
+				if (e->xdpf)
+					xdp_return_frame(e->xdpf);
+				e->is_xdpf = false;
+				e->xdpf = NULL;
+			} else if (skb) {
 				struct netdev_queue *txq;
 
 				txq = skb_get_tx_queue(skb->dev, skb);
@@ -2201,6 +2486,12 @@ static int airoha_dev_change_mtu(struct net_device *netdev, int mtu)
 	struct airoha_gdm_dev *dev = netdev_priv(netdev);
 	struct airoha_gdm_port *port = dev->port;
 
+	if (rcu_access_pointer(dev->xdp_prog) && mtu > AIROHA_RX_MAX_BUF_SIZE) {
+		netdev_err(netdev, "MTU too large for XDP (max %lu)\n",
+			   AIROHA_RX_MAX_BUF_SIZE);
+		return -EINVAL;
+	}
+
 	WRITE_ONCE(netdev->mtu, mtu);
 	if (port->users)
 		airoha_dev_set_xmit_frame_size(netdev);
@@ -2377,6 +2668,7 @@ static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb,
 
 		list_move_tail(&e->list, &tx_list);
 		e->skb = i == nr_frags - 1 ? skb : NULL;
+		e->is_xdpf = false;
 		e->dma_addr = addr;
 		e->dma_len = len;
 
@@ -3321,6 +3613,39 @@ static int airoha_tc_setup_qdisc_htb(struct net_device *netdev,
 	return 0;
 }
 
+static int airoha_xdp_setup(struct net_device *netdev, struct bpf_prog *prog,
+			    struct netlink_ext_ack *extack)
+{
+	struct airoha_gdm_dev *dev = netdev_priv(netdev);
+	struct bpf_prog *old_prog;
+
+	if (netdev->features & NETIF_F_LRO) {
+		NL_SET_ERR_MSG_MOD(extack, "XDP is not supported with LRO");
+		return -EOPNOTSUPP;
+	}
+
+	if (netdev->mtu > AIROHA_RX_MAX_BUF_SIZE) {
+		NL_SET_ERR_MSG_MOD(extack, "MTU too large for XDP");
+		return -EOPNOTSUPP;
+	}
+
+	old_prog = rcu_replace_pointer(dev->xdp_prog, prog, lockdep_rtnl_is_held());
+	if (old_prog)
+		bpf_prog_put(old_prog);
+
+	return 0;
+}
+
+static int airoha_xdp_bpf(struct net_device *netdev, struct netdev_bpf *xdp)
+{
+	switch (xdp->command) {
+	case XDP_SETUP_PROG:
+		return airoha_xdp_setup(netdev, xdp->prog, xdp->extack);
+	default:
+		return -EINVAL;
+	}
+}
+
 static int airoha_dev_tc_setup(struct net_device *dev,
 			       enum tc_setup_type type, void *type_data)
 {
@@ -3347,6 +3672,8 @@ static const struct net_device_ops airoha_netdev_ops = {
 	.ndo_get_stats64        = airoha_dev_get_stats64,
 	.ndo_set_mac_address	= airoha_dev_set_macaddr,
 	.ndo_setup_tc		= airoha_dev_tc_setup,
+	.ndo_bpf		= airoha_xdp_bpf,
+	.ndo_xdp_xmit		= airoha_xdp_xmit,
 };
 
 static const struct ethtool_ops airoha_ethtool_ops = {
@@ -3435,6 +3762,9 @@ static int airoha_alloc_gdm_device(struct airoha_eth *eth,
 			      NETIF_F_HW_TC;
 	netdev->features |= netdev->hw_features;
 	netdev->vlan_features = netdev->hw_features;
+	netdev->xdp_features = NETDEV_XDP_ACT_BASIC |
+				NETDEV_XDP_ACT_REDIRECT |
+				NETDEV_XDP_ACT_NDO_XMIT;
 	SET_NETDEV_DEV(netdev, eth->dev);
 
 	/* reserve hw queues for HTB offloading */
diff --git a/drivers/net/ethernet/airoha/airoha_eth.h b/drivers/net/ethernet/airoha/airoha_eth.h
index 8277c1c87bb3..cfbc0b8f5bf9 100644
--- a/drivers/net/ethernet/airoha/airoha_eth.h
+++ b/drivers/net/ethernet/airoha/airoha_eth.h
@@ -7,6 +7,7 @@
 #ifndef AIROHA_ETH_H
 #define AIROHA_ETH_H
 
+#include <linux/bpf.h>
 #include <linux/debugfs.h>
 #include <linux/etherdevice.h>
 #include <linux/iopoll.h>
@@ -15,6 +16,7 @@
 #include <linux/reset.h>
 #include <linux/soc/airoha/airoha_offload.h>
 #include <net/dsa.h>
+#include <net/xdp.h>
 
 #define AIROHA_MAX_NUM_GDM_PORTS	4
 #define AIROHA_MAX_NUM_GDM_DEVS		2
@@ -34,7 +36,11 @@
 #define AIROHA_FE_MC_MAX_VLAN_TABLE	64
 #define AIROHA_FE_MC_MAX_VLAN_PORT	16
 #define AIROHA_NUM_TX_IRQ		2
-#define AIROHA_RX_HEADROOM		(NET_SKB_PAD + NET_IP_ALIGN)
+#define AIROHA_RX_HEADROOM		(XDP_PACKET_HEADROOM + NET_IP_ALIGN)
+#define AIROHA_RX_PAD			(AIROHA_RX_HEADROOM + \
+					 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
+#define AIROHA_RX_MAX_BUF_SIZE		(PAGE_SIZE - AIROHA_RX_PAD - \
+					 VLAN_ETH_HLEN - ETH_FCS_LEN)
 #define AIROHA_RX_LEN(_n)		((_n) - AIROHA_RX_HEADROOM)
 #define HW_DSCP_NUM			2048
 #define IRQ_QUEUE_LEN(_n)		((_n) ? 1024 : 2048)
@@ -182,12 +188,16 @@ struct airoha_queue_entry {
 		void *buf;
 		struct {
 			struct list_head list;
-			struct sk_buff *skb;
+			union {
+				struct sk_buff *skb;
+				struct xdp_frame *xdpf;
+			};
 			enum airoha_dma_map_type dma_type;
 		};
 	};
 	dma_addr_t dma_addr;
 	u16 dma_len;
+	bool is_xdpf;
 };
 
 struct airoha_queue {
@@ -211,6 +221,9 @@ struct airoha_queue {
 	struct page_pool *page_pool;
 	struct sk_buff *skb;
 
+	struct xdp_rxq_info xdp_rxq;
+	bool xdp_flush;
+
 	struct list_head tx_list;
 };
 
@@ -589,6 +602,8 @@ struct airoha_gdm_dev {
 	unsigned long flags;
 	int nbq;
 
+	struct bpf_prog __rcu *xdp_prog;
+
 	struct airoha_hw_stats stats;
 
 	/* Serialize netdev_tx_completed_queue() calls per TX queue during
-- 
2.55.0

Re: [PATCH net-next 1/2] net: airoha: Add XDP support

[Lorenzo Bianconi]

[-- Attachment #1: Type: text/plain, Size: 22666 bytes --]

> Implement eXpress Data Path (XDP) support for the Airoha Ethernet driver.

Hi Til,

thanks to work on it. Some comments inline.

Regards,
Lorenzo

> 
> XDP programs are attached per net_device via the ndo_bpf hook. The BPF
> program reference is stored as an RCU-protected pointer in struct
> airoha_gdm_dev and replaced atomically on program load/unload via
> rcu_replace_pointer().
> 
> To support XDP, the following changes are made to the RX path:
> 
> - The page pool DMA direction is switched from DMA_FROM_DEVICE to
>   DMA_BIDIRECTIONAL, which is required for XDP_TX since the same page
>   can be reused for transmit.
> - The RX headroom is increased from NET_SKB_PAD to XDP_PACKET_HEADROOM
>   so that XDP programs have the required headroom available in the data
>   buffer.
> - RX buffers use a full page rather than a half-page fragment. This leaves
>   sufficient space for XDP headroom, skb_shared_info tailroom, and a
>   standard 1500-byte MTU.
> - AIROHA_RX_MAX_BUF_SIZE expresses the maximum MTU that fits in an RX
>   buffer after headroom, shared-info, Ethernet, VLAN, and FCS overhead.
>   MTU changes are rejected when an XDP program is loaded and the requested
>   MTU would exceed this limit.
> - xdp_rxq_info is registered for each RX queue during initialisation and
>   unregistered on cleanup, with the page pool set as the memory model.
> 
> The XDP program is run in airoha_run_xdp() for each received buffer.
> The following actions are supported:
> 
> - XDP_PASS: the buffer is passed to the normal networking stack.
> - XDP_TX: the buffer is transmitted back out the same interface using
>   the new airoha_xdp_xmit_back() helper.
> - XDP_REDIRECT: the buffer is redirected to another interface or map
>   via xdp_do_redirect(), with xdp_do_flush() deferred to the end of
>   the NAPI poll via a per-queue xdp_flush flag.
> - XDP_DROP (and unknown actions): the page is returned to the page pool.
> 
> Two new TX path helpers are introduced:
> 
> - airoha_xdp_submit_frame() maps an xdp_frame for DMA (or reuses the
>   page pool DMA address for XDP_TX) and enqueues it into a TX ring,
>   handling multi-buffer frames via skb_shared_info fragments.
> - airoha_xdp_xmit_back() converts an xdp_buff to an xdp_frame and
>   submits it to the TX queue corresponding to the current CPU.
> 
> The ndo_xdp_xmit handler airoha_xdp_xmit() is also added to allow
> XDP redirect from other drivers into the Airoha interface. Frames are
> submitted using airoha_xdp_submit_frame() with dma_map=true.
> 
> struct airoha_queue_entry is extended with an is_xdpf flag and an
> xdp_frame pointer (in a union with the existing sk_buff pointer) so
> that the TX completion path can correctly free either XDP frames or
> SKBs.
> 
> The advertised XDP feature flags are set to:
>   NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | NETDEV_XDP_ACT_NDO_XMIT
> 
> Signed-off-by: Til Kaiser <mail@tk154.de>
> ---
>  drivers/net/ethernet/airoha/airoha_eth.c | 386 +++++++++++++++++++++--
>  drivers/net/ethernet/airoha/airoha_eth.h |  19 +-
>  2 files changed, 375 insertions(+), 30 deletions(-)
> 
> diff --git a/drivers/net/ethernet/airoha/airoha_eth.c b/drivers/net/ethernet/airoha/airoha_eth.c
> index 64619e9a704d..71b25f225a8a 100644
> --- a/drivers/net/ethernet/airoha/airoha_eth.c
> +++ b/drivers/net/ethernet/airoha/airoha_eth.c
> @@ -13,6 +13,7 @@
>  #include <net/page_pool/helpers.h>
>  #include <net/pkt_cls.h>
>  #include <uapi/linux/ppp_defs.h>
> +#include <linux/bpf_trace.h>

please respect alphabetic order

>  
>  #include "airoha_regs.h"
>  #include "airoha_eth.h"
> @@ -657,6 +658,255 @@ airoha_qdma_get_gdm_dev(struct airoha_eth *eth, struct airoha_qdma_desc *desc)
>  	return port->devs[d] ? port->devs[d] : ERR_PTR(-ENODEV);
>  }
>  
> +static void airoha_unmap_xmit_buf(struct airoha_eth *eth,
> +				  struct airoha_queue_entry *e)
> +{
> +	switch (e->dma_type) {
> +	case AIROHA_DMA_MAP_PAGE:
> +		dma_unmap_page(eth->dev, e->dma_addr, e->dma_len,
> +			       DMA_TO_DEVICE);
> +		break;
> +	case AIROHA_DMA_MAP_SINGLE:
> +		dma_unmap_single(eth->dev, e->dma_addr, e->dma_len,
> +				 DMA_TO_DEVICE);
> +		break;
> +	case AIROHA_DMA_UNMAPPED:
> +	default:
> +		break;
> +	}
> +	e->dma_type = AIROHA_DMA_UNMAPPED;
> +}
> +
> +static int airoha_xdp_submit_frame(struct airoha_gdm_dev *dev,
> +				   struct xdp_frame *xdpf,
> +				   struct airoha_queue *q,
> +				   int qid, u32 msg0, u32 msg1,
> +				   bool dma_map)

can you please run checkpatch.pl on the patch?

> +{
> +	struct airoha_queue_entry *e, *next_e;
> +	int len = xdpf->len, nr_frags, i;
> +	struct airoha_qdma_desc *desc;
> +	struct skb_shared_info *sinfo;
> +	void *data = xdpf->data;
> +	u16 index, next_index;
> +	LIST_HEAD(tx_list);
> +	dma_addr_t addr;
> +	u32 val;
> +
> +	sinfo = xdp_get_shared_info_from_frame(xdpf);
> +	nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;
> +
> +	if (q->queued >= q->ndesc - 1 - nr_frags)

nit: I guess it is more readable if you do:

	if (q->queued + 1 + nr_frags >= q->ndesc)
		return -EBUSY;

> +		return -EBUSY;
> +
> +	for (i = 0; i <= nr_frags; i++) {
> +		if (dma_map) {
> +			if (i == 0)
> +				addr = dma_map_single(dev->eth->dev, data, len,
> +						      DMA_TO_DEVICE);
> +			else
> +				addr = dma_map_page(dev->eth->dev, virt_to_page(data),
> +						    offset_in_page(data), len,
> +						    DMA_TO_DEVICE);
> +			if (unlikely(dma_mapping_error(dev->eth->dev, addr)))
> +				goto unmap;
> +		} else {
> +			struct page *page = virt_to_head_page(data);
> +
> +			addr = page_pool_get_dma_addr(page) +
> +			       (data - (void *)page_address(page));
> +			dma_sync_single_for_device(dev->eth->dev, addr, len,
> +						   DMA_BIDIRECTIONAL);
> +		}
> +
> +		e = list_first_entry(&q->tx_list, struct airoha_queue_entry, list);
> +		list_move_tail(&e->list, &tx_list);
> +
> +		index = e - q->entry;
> +		desc = &q->desc[index];
> +
> +		e->is_xdpf = true;
> +		e->xdpf = (i == nr_frags) ? xdpf : NULL;

nit: you do not need brackets here.

> +		e->dma_addr = addr;
> +		e->dma_len = len;
> +		if (dma_map)
> +			e->dma_type = i == 0 ? AIROHA_DMA_MAP_SINGLE : AIROHA_DMA_MAP_PAGE;
> +		else
> +			e->dma_type = AIROHA_DMA_UNMAPPED;

I thin you can move this chunk in the above if/else block

> +
> +		next_e = list_first_entry(&q->tx_list, struct airoha_queue_entry, list);
> +		next_index = next_e - q->entry;
> +
> +		val = FIELD_PREP(QDMA_DESC_LEN_MASK, len);
> +		if (i < nr_frags)
> +			val |= FIELD_PREP(QDMA_DESC_MORE_MASK, 1);
> +		WRITE_ONCE(desc->ctrl, cpu_to_le32(val));
> +		WRITE_ONCE(desc->addr, cpu_to_le32(addr));
> +		val = FIELD_PREP(QDMA_DESC_NEXT_ID_MASK, next_index);
> +		WRITE_ONCE(desc->data, cpu_to_le32(val));
> +		WRITE_ONCE(desc->msg0, cpu_to_le32(msg0));
> +		WRITE_ONCE(desc->msg1, cpu_to_le32(msg1));
> +		WRITE_ONCE(desc->msg2, cpu_to_le32(0xffff));
> +
> +		q->queued++;
> +
> +		if (i < nr_frags) {
> +			skb_frag_t *frag = &sinfo->frags[i];
> +
> +			data = skb_frag_address(frag);
> +			len = skb_frag_size(frag);
> +		}
> +	}
> +
> +	return next_index;
> +
> +unmap:
> +	list_for_each_entry(e, &tx_list, list) {
> +		airoha_unmap_xmit_buf(dev->eth, e);
> +		e->is_xdpf = false;
> +		e->xdpf = NULL;
> +		q->queued--;
> +	}
> +	list_splice(&tx_list, &q->tx_list);
> +
> +	return -ENOMEM;
> +}
> +
> +static int airoha_xdp_xmit_back(struct net_device *netdev, struct xdp_buff *xdp,
> +				struct airoha_qdma *qdma)
> +{
> +	struct airoha_gdm_dev *dev = netdev_priv(netdev);
> +	struct airoha_queue *q;
> +	struct xdp_frame *xdpf;
> +	u32 msg0, msg1;
> +	int qid, index;
> +	u8 fport;
> +
> +	xdpf = xdp_convert_buff_to_frame(xdp);
> +	if (unlikely(!xdpf))
> +		return -EOVERFLOW;
> +
> +	qid = airoha_qdma_get_txq(qdma, smp_processor_id());
> +	q = &qdma->q_tx[qid];
> +
> +	msg0 = FIELD_PREP(QDMA_ETH_TXMSG_CHAN_MASK,
> +			  qid / AIROHA_NUM_QOS_QUEUES) |
> +	       FIELD_PREP(QDMA_ETH_TXMSG_QUEUE_MASK,
> +			  qid % AIROHA_NUM_QOS_QUEUES);
> +
> +	fport = airoha_get_fe_port(dev);
> +	msg1 = FIELD_PREP(QDMA_ETH_TXMSG_NBOQ_MASK, dev->nbq) |
> +	       FIELD_PREP(QDMA_ETH_TXMSG_FPORT_MASK, fport) |
> +	       FIELD_PREP(QDMA_ETH_TXMSG_METER_MASK, 0x7f);

msg0/msg1 are in common with airoha_xdp_xmit(), right? Can you please move them
in airoha_xdp_submit_frame()?

> +
> +	spin_lock(&q->lock);
> +	index = airoha_xdp_submit_frame(dev, xdpf, q, qid, msg0, msg1, false);
> +	if (unlikely(index < 0)) {
> +		spin_unlock(&q->lock);
> +		return index;
> +	}
> +
> +	airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid),
> +			TX_RING_CPU_IDX_MASK,
> +			FIELD_PREP(TX_RING_CPU_IDX_MASK, index));
> +	spin_unlock(&q->lock);
> +
> +	return 0;
> +}
> +
> +static int airoha_xdp_xmit(struct net_device *netdev, int n,
> +			   struct xdp_frame **frames, u32 flags)
> +{
> +	int qid, index = 0, last_index = -1, i, drops = 0;
> +	struct airoha_gdm_dev *dev = netdev_priv(netdev);
> +	struct airoha_qdma *qdma;
> +	struct airoha_queue *q;
> +	u32 msg0, msg1;
> +	u8 fport;
> +
> +	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
> +		return -EINVAL;
> +
> +	rcu_read_lock();
> +	qdma = rcu_dereference(dev->qdma);
> +	if (!qdma) {
> +		rcu_read_unlock();
> +		return -ENODEV;
> +	}
> +
> +	qid = airoha_qdma_get_txq(qdma, smp_processor_id());
> +	q = &qdma->q_tx[qid];
> +
> +	msg0 = FIELD_PREP(QDMA_ETH_TXMSG_CHAN_MASK,
> +			  qid / AIROHA_NUM_QOS_QUEUES) |
> +	       FIELD_PREP(QDMA_ETH_TXMSG_QUEUE_MASK,
> +			  qid % AIROHA_NUM_QOS_QUEUES);
> +
> +	fport = airoha_get_fe_port(dev);
> +	msg1 = FIELD_PREP(QDMA_ETH_TXMSG_NBOQ_MASK, dev->nbq) |
> +	       FIELD_PREP(QDMA_ETH_TXMSG_FPORT_MASK, fport) |
> +	       FIELD_PREP(QDMA_ETH_TXMSG_METER_MASK, 0x7f);
> +
> +	spin_lock(&q->lock);
> +	for (i = 0; i < n; i++) {
> +		index = airoha_xdp_submit_frame(dev, frames[i], q, qid, msg0, msg1, true);
> +		if (unlikely(index < 0)) {
> +			xdp_return_frame_rx_napi(frames[i]);
> +			drops++;
> +		} else {
> +			last_index = index;
> +		}
> +	}
> +	if (n > drops && (flags & XDP_XMIT_FLUSH))

Do you mean to skip it if drops == n? If so, I guess it is more readable to
count number of successful transmissions


> +		airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid),
> +				TX_RING_CPU_IDX_MASK,
> +				FIELD_PREP(TX_RING_CPU_IDX_MASK, last_index));
> +	spin_unlock(&q->lock);
> +	rcu_read_unlock();
> +
> +	return n - drops;

same here

> +}
> +
> +static bool airoha_run_xdp(struct net_device *netdev, struct bpf_prog *prog,
> +			   struct xdp_buff *xdp, struct airoha_queue *q,
> +			   struct airoha_queue_entry *e, struct page *page)
> +{
> +	u32 act = bpf_prog_run_xdp(prog, xdp);
> +
> +	switch (act) {
> +	case XDP_PASS:
> +		return false;
> +	case XDP_TX:
> +		if (unlikely(airoha_xdp_xmit_back(netdev, xdp, q->qdma) < 0)) {
> +			trace_xdp_exception(netdev, prog, act);
> +			page_pool_put_full_page(q->page_pool, page, true);
> +		} else {
> +			e->buf = NULL;
> +		}
> +		break;
> +	case XDP_REDIRECT:
> +		if (unlikely(xdp_do_redirect(netdev, xdp, prog) < 0)) {
> +			trace_xdp_exception(netdev, prog, act);
> +			page_pool_put_full_page(q->page_pool, page, true);
> +		} else {
> +			q->xdp_flush = true;
> +			e->buf = NULL;
> +		}
> +		break;
> +	default:
> +		bpf_warn_invalid_xdp_action(netdev, prog, act);
> +		fallthrough;
> +	case XDP_ABORTED:
> +		trace_xdp_exception(netdev, prog, act);
> +		fallthrough;
> +	case XDP_DROP:
> +		page_pool_put_full_page(q->page_pool, page, true);
> +		break;
> +	}
> +
> +	return true;
> +}
> +
>  static int airoha_qdma_rx_process(struct airoha_queue *q, int budget)
>  {
>  	enum dma_data_direction dir = page_pool_get_dma_dir(q->page_pool);
> @@ -698,6 +948,27 @@ static int airoha_qdma_rx_process(struct airoha_queue *q, int budget)
>  
>  		netdev = netdev_from_priv(dev);
>  		if (!q->skb) { /* first buffer */
> +			struct bpf_prog *xdp_prog;

nit: prog seems better to me

> +
> +			rcu_read_lock();
> +			xdp_prog = rcu_dereference(dev->xdp_prog);
> +			if (xdp_prog) {
> +				struct xdp_buff xdp;
> +
> +				xdp_init_buff(&xdp, q->buf_size, &q->xdp_rxq);
> +				xdp_prepare_buff(&xdp, e->buf - AIROHA_RX_HEADROOM,
> +						 AIROHA_RX_HEADROOM, len, false);
> +
> +				if (airoha_run_xdp(netdev, xdp_prog, &xdp, q, e, page)) {
> +					rcu_read_unlock();
> +					continue;
> +				}
> +
> +				len = xdp.data_end - xdp.data;
> +				e->buf = xdp.data;
> +			}
> +			rcu_read_unlock();
> +
>  			q->skb = napi_build_skb(e->buf - AIROHA_RX_HEADROOM,
>  						q->buf_size);
>  			if (!q->skb)
> @@ -779,6 +1050,11 @@ static int airoha_qdma_rx_napi_poll(struct napi_struct *napi, int budget)
>  		done += cur;
>  	} while (cur && done < budget);
>  
> +	if (q->xdp_flush) {
> +		xdp_do_flush();
> +		q->xdp_flush = false;
> +	}
> +
>  	if (done < budget && napi_complete(napi)) {
>  		struct airoha_qdma *qdma = q->qdma;
>  		int i, qid = q - &qdma->q_rx[0];
> @@ -804,17 +1080,17 @@ static int airoha_qdma_init_rx_queue(struct airoha_queue *q,
>  		.order = 0,
>  		.pool_size = 256,
>  		.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
> -		.dma_dir = DMA_FROM_DEVICE,
> +		.dma_dir = DMA_BIDIRECTIONAL,
>  		.max_len = PAGE_SIZE,
>  		.nid = NUMA_NO_NODE,
>  		.dev = qdma->eth->dev,
>  		.napi = &q->napi,
>  	};
> +	int qid = q - &qdma->q_rx[0], thr, err;
>  	struct airoha_eth *eth = qdma->eth;
> -	int qid = q - &qdma->q_rx[0], thr;
>  	dma_addr_t dma_addr;
>  
> -	q->buf_size = PAGE_SIZE / 2;
> +	q->buf_size = PAGE_SIZE;

I guess this can introduce a performance penalty in the non-xdp case since now
we need a full page for a single buffer while in the current codebase we can
have 2 fragments in a single page. Moreover, it seems to me you are not using
latest codebase here.

>  	q->qdma = qdma;
>  
>  	q->entry = devm_kzalloc(eth->dev, ndesc * sizeof(*q->entry),
> @@ -829,15 +1105,22 @@ static int airoha_qdma_init_rx_queue(struct airoha_queue *q,
>  
>  	q->page_pool = page_pool_create(&pp_params);
>  	if (IS_ERR(q->page_pool)) {
> -		int err = PTR_ERR(q->page_pool);
> -
> -		q->page_pool = NULL;
> -		return err;
> +		err = PTR_ERR(q->page_pool);
> +		goto err_page_pool_create;
>  	}
>  
>  	q->ndesc = ndesc;
>  	netif_napi_add(eth->napi_dev, &q->napi, airoha_qdma_rx_napi_poll);
>  
> +	err = xdp_rxq_info_reg(&q->xdp_rxq, eth->napi_dev, qid, q->napi.napi_id);
> +	if (err)
> +		goto err_xdp_rxq_info_reg;
> +
> +	err = xdp_rxq_info_reg_mem_model(&q->xdp_rxq, MEM_TYPE_PAGE_POOL,
> +					 q->page_pool);
> +	if (err)
> +		goto err_xdp_rxq_info_reg_mem_model;
> +
>  	airoha_qdma_wr(qdma, REG_RX_RING_BASE(qid), dma_addr);
>  	airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid),
>  			RX_RING_SIZE_MASK,
> @@ -853,6 +1136,14 @@ static int airoha_qdma_init_rx_queue(struct airoha_queue *q,
>  	airoha_qdma_fill_rx_queue(q);
>  
>  	return 0;
> +
> +err_xdp_rxq_info_reg_mem_model:
> +	xdp_rxq_info_unreg(&q->xdp_rxq);
> +err_xdp_rxq_info_reg:
> +	page_pool_destroy(q->page_pool);
> +err_page_pool_create:
> +	q->page_pool = NULL;
> +	return err;
>  }
>  
>  static void airoha_qdma_cleanup_rx_queue(struct airoha_queue *q)
> @@ -882,6 +1173,9 @@ static void airoha_qdma_cleanup_rx_queue(struct airoha_queue *q)
>  		q->queued--;
>  	}
>  
> +	if (xdp_rxq_info_is_reg(&q->xdp_rxq))
> +		xdp_rxq_info_unreg(&q->xdp_rxq);
> +
>  	q->head = q->tail;
>  	/* Set RX_DMA_IDX to RX_CPU_IDX to notify the hw the QDMA RX ring is
>  	 * empty.
> @@ -949,25 +1243,6 @@ static void airoha_qdma_wake_netdev_txqs(struct airoha_queue *q)
>  	q->txq_stopped = false;
>  }
>  
> -static void airoha_unmap_xmit_buf(struct airoha_eth *eth,
> -				  struct airoha_queue_entry *e)
> -{
> -	switch (e->dma_type) {
> -	case AIROHA_DMA_MAP_PAGE:
> -		dma_unmap_page(eth->dev, e->dma_addr, e->dma_len,
> -			       DMA_TO_DEVICE);
> -		break;
> -	case AIROHA_DMA_MAP_SINGLE:
> -		dma_unmap_single(eth->dev, e->dma_addr, e->dma_len,
> -				 DMA_TO_DEVICE);
> -		break;
> -	case AIROHA_DMA_UNMAPPED:
> -	default:
> -		break;
> -	}
> -	e->dma_type = AIROHA_DMA_UNMAPPED;
> -}
> -
>  static int airoha_qdma_tx_napi_poll(struct napi_struct *napi, int budget)
>  {
>  	struct airoha_tx_irq_queue *irq_q;
> @@ -1037,7 +1312,12 @@ static int airoha_qdma_tx_napi_poll(struct napi_struct *napi, int budget)
>  		WRITE_ONCE(desc->msg1, 0);
>  		q->queued--;
>  
> -		if (skb) {
> +		if (e->is_xdpf) {
> +			if (e->xdpf)
> +				xdp_return_frame(e->xdpf);
> +			e->is_xdpf = false;
> +			e->xdpf = NULL;
> +		} else if (e->skb) {
>  			struct airoha_gdm_dev *dev = netdev_priv(skb->dev);
>  			u16 qidx = skb_get_queue_mapping(skb);
>  			struct netdev_queue *txq;
> @@ -1218,7 +1498,12 @@ static void airoha_qdma_tx_cleanup(struct airoha_qdma *qdma)
>  			WRITE_ONCE(desc->msg1, 0);
>  			WRITE_ONCE(desc->msg2, 0);
>  
> -			if (skb) {
> +			if (e->is_xdpf) {
> +				if (e->xdpf)
> +					xdp_return_frame(e->xdpf);
> +				e->is_xdpf = false;
> +				e->xdpf = NULL;
> +			} else if (skb) {
>  				struct netdev_queue *txq;
>  
>  				txq = skb_get_tx_queue(skb->dev, skb);
> @@ -2201,6 +2486,12 @@ static int airoha_dev_change_mtu(struct net_device *netdev, int mtu)
>  	struct airoha_gdm_dev *dev = netdev_priv(netdev);
>  	struct airoha_gdm_port *port = dev->port;
>  
> +	if (rcu_access_pointer(dev->xdp_prog) && mtu > AIROHA_RX_MAX_BUF_SIZE) {
> +		netdev_err(netdev, "MTU too large for XDP (max %lu)\n",
> +			   AIROHA_RX_MAX_BUF_SIZE);
> +		return -EINVAL;
> +	}
> +
>  	WRITE_ONCE(netdev->mtu, mtu);
>  	if (port->users)
>  		airoha_dev_set_xmit_frame_size(netdev);
> @@ -2377,6 +2668,7 @@ static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb,
>  
>  		list_move_tail(&e->list, &tx_list);
>  		e->skb = i == nr_frags - 1 ? skb : NULL;
> +		e->is_xdpf = false;
>  		e->dma_addr = addr;
>  		e->dma_len = len;
>  
> @@ -3321,6 +3613,39 @@ static int airoha_tc_setup_qdisc_htb(struct net_device *netdev,
>  	return 0;
>  }
>  
> +static int airoha_xdp_setup(struct net_device *netdev, struct bpf_prog *prog,
> +			    struct netlink_ext_ack *extack)
> +{
> +	struct airoha_gdm_dev *dev = netdev_priv(netdev);
> +	struct bpf_prog *old_prog;
> +
> +	if (netdev->features & NETIF_F_LRO) {
> +		NL_SET_ERR_MSG_MOD(extack, "XDP is not supported with LRO");
> +		return -EOPNOTSUPP;
> +	}
> +
> +	if (netdev->mtu > AIROHA_RX_MAX_BUF_SIZE) {
> +		NL_SET_ERR_MSG_MOD(extack, "MTU too large for XDP");
> +		return -EOPNOTSUPP;
> +	}
> +
> +	old_prog = rcu_replace_pointer(dev->xdp_prog, prog, lockdep_rtnl_is_held());
> +	if (old_prog)
> +		bpf_prog_put(old_prog);
> +
> +	return 0;
> +}
> +
> +static int airoha_xdp_bpf(struct net_device *netdev, struct netdev_bpf *xdp)

nit: airoha_dev_bpf()?

> +{
> +	switch (xdp->command) {
> +	case XDP_SETUP_PROG:
> +		return airoha_xdp_setup(netdev, xdp->prog, xdp->extack);
> +	default:
> +		return -EINVAL;
> +	}
> +}
> +
>  static int airoha_dev_tc_setup(struct net_device *dev,
>  			       enum tc_setup_type type, void *type_data)
>  {
> @@ -3347,6 +3672,8 @@ static const struct net_device_ops airoha_netdev_ops = {
>  	.ndo_get_stats64        = airoha_dev_get_stats64,
>  	.ndo_set_mac_address	= airoha_dev_set_macaddr,
>  	.ndo_setup_tc		= airoha_dev_tc_setup,
> +	.ndo_bpf		= airoha_xdp_bpf,
> +	.ndo_xdp_xmit		= airoha_xdp_xmit,
>  };
>  
>  static const struct ethtool_ops airoha_ethtool_ops = {
> @@ -3435,6 +3762,9 @@ static int airoha_alloc_gdm_device(struct airoha_eth *eth,
>  			      NETIF_F_HW_TC;
>  	netdev->features |= netdev->hw_features;
>  	netdev->vlan_features = netdev->hw_features;
> +	netdev->xdp_features = NETDEV_XDP_ACT_BASIC |
> +				NETDEV_XDP_ACT_REDIRECT |
> +				NETDEV_XDP_ACT_NDO_XMIT;

please align it:

	netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
			       NETDEV_XDP_ACT_NDO_XMIT;

>  	SET_NETDEV_DEV(netdev, eth->dev);
>  
>  	/* reserve hw queues for HTB offloading */
> diff --git a/drivers/net/ethernet/airoha/airoha_eth.h b/drivers/net/ethernet/airoha/airoha_eth.h
> index 8277c1c87bb3..cfbc0b8f5bf9 100644
> --- a/drivers/net/ethernet/airoha/airoha_eth.h
> +++ b/drivers/net/ethernet/airoha/airoha_eth.h
> @@ -7,6 +7,7 @@
>  #ifndef AIROHA_ETH_H
>  #define AIROHA_ETH_H
>  
> +#include <linux/bpf.h>
>  #include <linux/debugfs.h>
>  #include <linux/etherdevice.h>
>  #include <linux/iopoll.h>
> @@ -15,6 +16,7 @@
>  #include <linux/reset.h>
>  #include <linux/soc/airoha/airoha_offload.h>
>  #include <net/dsa.h>
> +#include <net/xdp.h>
>  
>  #define AIROHA_MAX_NUM_GDM_PORTS	4
>  #define AIROHA_MAX_NUM_GDM_DEVS		2
> @@ -34,7 +36,11 @@
>  #define AIROHA_FE_MC_MAX_VLAN_TABLE	64
>  #define AIROHA_FE_MC_MAX_VLAN_PORT	16
>  #define AIROHA_NUM_TX_IRQ		2
> -#define AIROHA_RX_HEADROOM		(NET_SKB_PAD + NET_IP_ALIGN)
> +#define AIROHA_RX_HEADROOM		(XDP_PACKET_HEADROOM + NET_IP_ALIGN)
> +#define AIROHA_RX_PAD			(AIROHA_RX_HEADROOM + \
> +					 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
> +#define AIROHA_RX_MAX_BUF_SIZE		(PAGE_SIZE - AIROHA_RX_PAD - \
> +					 VLAN_ETH_HLEN - ETH_FCS_LEN)

same as above, I think this can introduce a performance penalty in the non-xdp
case. Can you please check? (e.g. on a 10Gbps link).

>  #define AIROHA_RX_LEN(_n)		((_n) - AIROHA_RX_HEADROOM)
>  #define HW_DSCP_NUM			2048
>  #define IRQ_QUEUE_LEN(_n)		((_n) ? 1024 : 2048)
> @@ -182,12 +188,16 @@ struct airoha_queue_entry {
>  		void *buf;
>  		struct {
>  			struct list_head list;
> -			struct sk_buff *skb;
> +			union {
> +				struct sk_buff *skb;
> +				struct xdp_frame *xdpf;
> +			};
>  			enum airoha_dma_map_type dma_type;
>  		};
>  	};
>  	dma_addr_t dma_addr;
>  	u16 dma_len;
> +	bool is_xdpf;
>  };
>  
>  struct airoha_queue {
> @@ -211,6 +221,9 @@ struct airoha_queue {
>  	struct page_pool *page_pool;
>  	struct sk_buff *skb;
>  
> +	struct xdp_rxq_info xdp_rxq;
> +	bool xdp_flush;
> +
>  	struct list_head tx_list;
>  };
>  
> @@ -589,6 +602,8 @@ struct airoha_gdm_dev {
>  	unsigned long flags;
>  	int nbq;
>  
> +	struct bpf_prog __rcu *xdp_prog;

nit: prog

> +
>  	struct airoha_hw_stats stats;
>  
>  	/* Serialize netdev_tx_completed_queue() calls per TX queue during
> -- 
> 2.55.0
> 
> 

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 228 bytes --]

[PATCH net-next 2/2] net: airoha: Add XDP statistics

[Til Kaiser]

Track and expose per-device XDP statistics via ethtool, covering both
the RX and TX sides of the XDP data path.

A new struct airoha_xdp_stats is introduced and embedded in the existing
struct airoha_hw_stats. It contains the following counters, all protected
by the existing u64_stats_sync:

  rx_xdp_pass            - frames passed to the networking stack
  rx_xdp_aborted         - frames with XDP_ABORTED or unknown action
  rx_xdp_drop            - frames dropped by the BPF program or on error
  rx_xdp_tx              - frames sent back out via XDP_TX
  rx_xdp_tx_errors       - XDP_TX failures
  rx_xdp_redirect        - frames successfully redirected
  rx_xdp_redirect_errors - XDP_REDIRECT failures
  tx_xdp_xmit            - frames successfully submitted via ndo_xdp_xmit
  tx_xdp_xmit_errors     - frames dropped in ndo_xdp_xmit

The airoha_run_xdp() function is reworked to update the appropriate
counter after each XDP action. The airoha_xdp_xmit() ndo_xdp_xmit handler
is updated to record the number of successfully transmitted and dropped
frames.

The ethtool interface is extended with three new callbacks:

- get_strings: emits the XDP counter names followed by the page pool
  stat strings obtained from page_pool_ethtool_stats_get_strings().
- get_sset_count: returns the combined count of XDP and page pool stats.
- get_ethtool_stats: reads the XDP counters under the u64_stats_sync
  seqcount and appends page pool stats collected from all active RX
  queues via page_pool_get_stats().

PAGE_POOL_STATS is selected in Kconfig to enable the page pool statistics
infrastructure required by the ethtool callbacks.

Signed-off-by: Til Kaiser <mail@tk154.de>
---
 drivers/net/ethernet/airoha/Kconfig      |   1 +
 drivers/net/ethernet/airoha/airoha_eth.c | 131 ++++++++++++++++++++---
 drivers/net/ethernet/airoha/airoha_eth.h |  14 +++
 3 files changed, 133 insertions(+), 13 deletions(-)

diff --git a/drivers/net/ethernet/airoha/Kconfig b/drivers/net/ethernet/airoha/Kconfig
index 1f6640a15fc9..3fa7683a79e9 100644
--- a/drivers/net/ethernet/airoha/Kconfig
+++ b/drivers/net/ethernet/airoha/Kconfig
@@ -20,6 +20,7 @@ config NET_AIROHA
 	depends on NET_DSA || !NET_DSA
 	select NET_AIROHA_NPU
 	select PAGE_POOL
+	select PAGE_POOL_STATS
 	help
 	  This driver supports the gigabit ethernet MACs in the
 	  Airoha SoC family.
diff --git a/drivers/net/ethernet/airoha/airoha_eth.c b/drivers/net/ethernet/airoha/airoha_eth.c
index 71b25f225a8a..29b837204842 100644
--- a/drivers/net/ethernet/airoha/airoha_eth.c
+++ b/drivers/net/ethernet/airoha/airoha_eth.c
@@ -864,6 +864,11 @@ static int airoha_xdp_xmit(struct net_device *netdev, int n,
 	spin_unlock(&q->lock);
 	rcu_read_unlock();
 
+	u64_stats_update_begin(&dev->stats.syncp);
+	dev->stats.xdp_stats.tx_xdp_xmit += n - drops;
+	dev->stats.xdp_stats.tx_xdp_xmit_errors += drops;
+	u64_stats_update_end(&dev->stats.syncp);
+
 	return n - drops;
 }
 
@@ -871,40 +876,56 @@ static bool airoha_run_xdp(struct net_device *netdev, struct bpf_prog *prog,
 			   struct xdp_buff *xdp, struct airoha_queue *q,
 			   struct airoha_queue_entry *e, struct page *page)
 {
+	struct airoha_gdm_dev *dev = netdev_priv(netdev);
+	struct airoha_hw_stats *hw_stats = &dev->stats;
 	u32 act = bpf_prog_run_xdp(prog, xdp);
+	u64 *count;
 
 	switch (act) {
 	case XDP_PASS:
-		return false;
+		count = &hw_stats->xdp_stats.rx_xdp_pass;
+		goto update_stats;
 	case XDP_TX:
 		if (unlikely(airoha_xdp_xmit_back(netdev, xdp, q->qdma) < 0)) {
+			count = &hw_stats->xdp_stats.rx_xdp_tx_errors;
 			trace_xdp_exception(netdev, prog, act);
-			page_pool_put_full_page(q->page_pool, page, true);
-		} else {
-			e->buf = NULL;
+			break;
 		}
-		break;
+
+		e->buf = NULL;
+		count = &hw_stats->xdp_stats.rx_xdp_tx;
+		goto update_stats;
 	case XDP_REDIRECT:
 		if (unlikely(xdp_do_redirect(netdev, xdp, prog) < 0)) {
+			count = &hw_stats->xdp_stats.rx_xdp_redirect_errors;
 			trace_xdp_exception(netdev, prog, act);
-			page_pool_put_full_page(q->page_pool, page, true);
-		} else {
-			q->xdp_flush = true;
-			e->buf = NULL;
+			break;
 		}
-		break;
+
+		e->buf = NULL;
+		q->xdp_flush = true;
+		count = &hw_stats->xdp_stats.rx_xdp_redirect;
+		goto update_stats;
 	default:
 		bpf_warn_invalid_xdp_action(netdev, prog, act);
 		fallthrough;
 	case XDP_ABORTED:
+		count = &hw_stats->xdp_stats.rx_xdp_aborted;
 		trace_xdp_exception(netdev, prog, act);
-		fallthrough;
+		break;
 	case XDP_DROP:
-		page_pool_put_full_page(q->page_pool, page, true);
+		count = &hw_stats->xdp_stats.rx_xdp_drop;
 		break;
 	}
 
-	return true;
+	page_pool_put_full_page(q->page_pool, page, true);
+
+update_stats:
+	u64_stats_update_begin(&hw_stats->syncp);
+	*count = *count + 1;
+	u64_stats_update_end(&hw_stats->syncp);
+
+	return act != XDP_PASS;
 }
 
 static int airoha_qdma_rx_process(struct airoha_queue *q, int budget)
@@ -3676,12 +3697,96 @@ static const struct net_device_ops airoha_netdev_ops = {
 	.ndo_xdp_xmit		= airoha_xdp_xmit,
 };
 
+#define AIROHA_ETHTOOL_XDP_STAT(x) { #x, \
+				  offsetof(struct airoha_hw_stats, xdp_stats.x) / \
+				  sizeof(u64) }
+
+static const struct {
+	const char name[ETH_GSTRING_LEN];
+	u32 offset;
+} airoha_ethtool_xdp_stats[] = {
+	AIROHA_ETHTOOL_XDP_STAT(rx_xdp_redirect),
+	AIROHA_ETHTOOL_XDP_STAT(rx_xdp_redirect_errors),
+	AIROHA_ETHTOOL_XDP_STAT(rx_xdp_pass),
+	AIROHA_ETHTOOL_XDP_STAT(rx_xdp_aborted),
+	AIROHA_ETHTOOL_XDP_STAT(rx_xdp_drop),
+	AIROHA_ETHTOOL_XDP_STAT(rx_xdp_tx),
+	AIROHA_ETHTOOL_XDP_STAT(rx_xdp_tx_errors),
+	AIROHA_ETHTOOL_XDP_STAT(tx_xdp_xmit),
+	AIROHA_ETHTOOL_XDP_STAT(tx_xdp_xmit_errors),
+};
+
+static void airoha_ethtool_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
+{
+	int i;
+
+	switch (stringset) {
+	case ETH_SS_STATS:
+		for (i = 0; i < ARRAY_SIZE(airoha_ethtool_xdp_stats); i++) {
+			strscpy(data, airoha_ethtool_xdp_stats[i].name, ETH_GSTRING_LEN);
+			data += ETH_GSTRING_LEN;
+		}
+		page_pool_ethtool_stats_get_strings(data);
+		break;
+	}
+}
+
+static int airoha_ethtool_get_sset_count(struct net_device *netdev, int stringset)
+{
+	switch (stringset) {
+	case ETH_SS_STATS:
+		return ARRAY_SIZE(airoha_ethtool_xdp_stats) + page_pool_ethtool_stats_get_count();
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static void airoha_ethtool_get_ethtool_stats(struct net_device *netdev,
+					     struct ethtool_stats *stats,
+					     u64 *data)
+{
+	struct airoha_gdm_dev *dev = netdev_priv(netdev);
+	struct airoha_hw_stats *hw_stats = &dev->stats;
+	u64 *hw_stats_base = (u64 *)hw_stats;
+	struct page_pool_stats pp_stats = {};
+	struct airoha_qdma *qdma;
+	unsigned int start;
+	int i;
+
+	if (netif_running(netdev))
+		airoha_update_hw_stats(dev);
+
+	do {
+		start = u64_stats_fetch_begin(&hw_stats->syncp);
+		for (i = 0; i < ARRAY_SIZE(airoha_ethtool_xdp_stats); i++)
+			data[i] = hw_stats_base[airoha_ethtool_xdp_stats[i].offset];
+	} while (u64_stats_fetch_retry(&hw_stats->syncp, start));
+
+	rcu_read_lock();
+	qdma = rcu_dereference(dev->qdma);
+	if (qdma) {
+		for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
+			struct airoha_queue *q = &qdma->q_rx[i];
+
+			if (q->page_pool)
+				page_pool_get_stats(q->page_pool, &pp_stats);
+		}
+	}
+	rcu_read_unlock();
+
+	page_pool_ethtool_stats_get(data + ARRAY_SIZE(airoha_ethtool_xdp_stats),
+				    &pp_stats);
+}
+
 static const struct ethtool_ops airoha_ethtool_ops = {
 	.get_drvinfo		= airoha_ethtool_get_drvinfo,
 	.get_eth_mac_stats      = airoha_ethtool_get_mac_stats,
 	.get_rmon_stats		= airoha_ethtool_get_rmon_stats,
 	.get_link_ksettings	= phy_ethtool_get_link_ksettings,
 	.get_link		= ethtool_op_get_link,
+	.get_strings		= airoha_ethtool_get_strings,
+	.get_sset_count		= airoha_ethtool_get_sset_count,
+	.get_ethtool_stats	= airoha_ethtool_get_ethtool_stats,
 };
 
 static int airoha_metadata_dst_alloc(struct airoha_gdm_port *port)
diff --git a/drivers/net/ethernet/airoha/airoha_eth.h b/drivers/net/ethernet/airoha/airoha_eth.h
index cfbc0b8f5bf9..d2ac295c49e9 100644
--- a/drivers/net/ethernet/airoha/airoha_eth.h
+++ b/drivers/net/ethernet/airoha/airoha_eth.h
@@ -236,6 +236,18 @@ struct airoha_tx_irq_queue {
 	u32 *q;
 };
 
+struct airoha_xdp_stats {
+	u64 rx_xdp_redirect;
+	u64 rx_xdp_redirect_errors;
+	u64 rx_xdp_pass;
+	u64 rx_xdp_aborted;
+	u64 rx_xdp_drop;
+	u64 rx_xdp_tx;
+	u64 rx_xdp_tx_errors;
+	u64 tx_xdp_xmit;
+	u64 tx_xdp_xmit_errors;
+};
+
 struct airoha_hw_stats {
 	struct u64_stats_sync syncp;
 
@@ -259,6 +271,8 @@ struct airoha_hw_stats {
 	u64 rx_jabber;
 	u64 rx_len[7];
 
+	struct airoha_xdp_stats xdp_stats;
+
 	struct {
 		/* Previous HW register values for 32-bit counter delta
 		 * tracking. Storing the last seen value and accumulating
-- 
2.55.0

Re: [PATCH net-next 2/2] net: airoha: Add XDP statistics

[Lorenzo Bianconi]

[-- Attachment #1: Type: text/plain, Size: 10127 bytes --]

> Track and expose per-device XDP statistics via ethtool, covering both
> the RX and TX sides of the XDP data path.
> 
> A new struct airoha_xdp_stats is introduced and embedded in the existing
> struct airoha_hw_stats. It contains the following counters, all protected
> by the existing u64_stats_sync:
> 
>   rx_xdp_pass            - frames passed to the networking stack
>   rx_xdp_aborted         - frames with XDP_ABORTED or unknown action
>   rx_xdp_drop            - frames dropped by the BPF program or on error
>   rx_xdp_tx              - frames sent back out via XDP_TX
>   rx_xdp_tx_errors       - XDP_TX failures
>   rx_xdp_redirect        - frames successfully redirected
>   rx_xdp_redirect_errors - XDP_REDIRECT failures
>   tx_xdp_xmit            - frames successfully submitted via ndo_xdp_xmit
>   tx_xdp_xmit_errors     - frames dropped in ndo_xdp_xmit
> 
> The airoha_run_xdp() function is reworked to update the appropriate
> counter after each XDP action. The airoha_xdp_xmit() ndo_xdp_xmit handler
> is updated to record the number of successfully transmitted and dropped
> frames.
> 
> The ethtool interface is extended with three new callbacks:
> 
> - get_strings: emits the XDP counter names followed by the page pool
>   stat strings obtained from page_pool_ethtool_stats_get_strings().
> - get_sset_count: returns the combined count of XDP and page pool stats.
> - get_ethtool_stats: reads the XDP counters under the u64_stats_sync
>   seqcount and appends page pool stats collected from all active RX
>   queues via page_pool_get_stats().
> 
> PAGE_POOL_STATS is selected in Kconfig to enable the page pool statistics
> infrastructure required by the ethtool callbacks.
> 
> Signed-off-by: Til Kaiser <mail@tk154.de>
> ---
>  drivers/net/ethernet/airoha/Kconfig      |   1 +
>  drivers/net/ethernet/airoha/airoha_eth.c | 131 ++++++++++++++++++++---
>  drivers/net/ethernet/airoha/airoha_eth.h |  14 +++
>  3 files changed, 133 insertions(+), 13 deletions(-)
> 
> diff --git a/drivers/net/ethernet/airoha/Kconfig b/drivers/net/ethernet/airoha/Kconfig
> index 1f6640a15fc9..3fa7683a79e9 100644
> --- a/drivers/net/ethernet/airoha/Kconfig
> +++ b/drivers/net/ethernet/airoha/Kconfig
> @@ -20,6 +20,7 @@ config NET_AIROHA
>  	depends on NET_DSA || !NET_DSA
>  	select NET_AIROHA_NPU
>  	select PAGE_POOL
> +	select PAGE_POOL_STATS
>  	help
>  	  This driver supports the gigabit ethernet MACs in the
>  	  Airoha SoC family.
> diff --git a/drivers/net/ethernet/airoha/airoha_eth.c b/drivers/net/ethernet/airoha/airoha_eth.c
> index 71b25f225a8a..29b837204842 100644
> --- a/drivers/net/ethernet/airoha/airoha_eth.c
> +++ b/drivers/net/ethernet/airoha/airoha_eth.c
> @@ -864,6 +864,11 @@ static int airoha_xdp_xmit(struct net_device *netdev, int n,
>  	spin_unlock(&q->lock);
>  	rcu_read_unlock();
>  
> +	u64_stats_update_begin(&dev->stats.syncp);
> +	dev->stats.xdp_stats.tx_xdp_xmit += n - drops;
> +	dev->stats.xdp_stats.tx_xdp_xmit_errors += drops;
> +	u64_stats_update_end(&dev->stats.syncp);
> +
>  	return n - drops;
>  }
>  
> @@ -871,40 +876,56 @@ static bool airoha_run_xdp(struct net_device *netdev, struct bpf_prog *prog,
>  			   struct xdp_buff *xdp, struct airoha_queue *q,
>  			   struct airoha_queue_entry *e, struct page *page)
>  {
> +	struct airoha_gdm_dev *dev = netdev_priv(netdev);
> +	struct airoha_hw_stats *hw_stats = &dev->stats;
>  	u32 act = bpf_prog_run_xdp(prog, xdp);
> +	u64 *count;
>  
>  	switch (act) {
>  	case XDP_PASS:
> -		return false;
> +		count = &hw_stats->xdp_stats.rx_xdp_pass;
> +		goto update_stats;
>  	case XDP_TX:
>  		if (unlikely(airoha_xdp_xmit_back(netdev, xdp, q->qdma) < 0)) {
> +			count = &hw_stats->xdp_stats.rx_xdp_tx_errors;
>  			trace_xdp_exception(netdev, prog, act);
> -			page_pool_put_full_page(q->page_pool, page, true);
> -		} else {
> -			e->buf = NULL;
> +			break;
>  		}
> -		break;
> +
> +		e->buf = NULL;
> +		count = &hw_stats->xdp_stats.rx_xdp_tx;
> +		goto update_stats;
>  	case XDP_REDIRECT:
>  		if (unlikely(xdp_do_redirect(netdev, xdp, prog) < 0)) {
> +			count = &hw_stats->xdp_stats.rx_xdp_redirect_errors;
>  			trace_xdp_exception(netdev, prog, act);
> -			page_pool_put_full_page(q->page_pool, page, true);
> -		} else {
> -			q->xdp_flush = true;
> -			e->buf = NULL;
> +			break;
>  		}
> -		break;
> +
> +		e->buf = NULL;
> +		q->xdp_flush = true;
> +		count = &hw_stats->xdp_stats.rx_xdp_redirect;
> +		goto update_stats;
>  	default:
>  		bpf_warn_invalid_xdp_action(netdev, prog, act);
>  		fallthrough;
>  	case XDP_ABORTED:
> +		count = &hw_stats->xdp_stats.rx_xdp_aborted;
>  		trace_xdp_exception(netdev, prog, act);
> -		fallthrough;
> +		break;
>  	case XDP_DROP:
> -		page_pool_put_full_page(q->page_pool, page, true);
> +		count = &hw_stats->xdp_stats.rx_xdp_drop;
>  		break;
>  	}
>  
> -	return true;
> +	page_pool_put_full_page(q->page_pool, page, true);
> +
> +update_stats:
> +	u64_stats_update_begin(&hw_stats->syncp);
> +	*count = *count + 1;
> +	u64_stats_update_end(&hw_stats->syncp);

Would it better if we update the stats in airoha_qdma_rx_process() passing a
airoha_xdp_stats struct to airoha_run_xdp(). I guess doing so we can even drop
xdp_flush field in airoha_qdma_queue struct.

> +
> +	return act != XDP_PASS;
>  }
>  
>  static int airoha_qdma_rx_process(struct airoha_queue *q, int budget)
> @@ -3676,12 +3697,96 @@ static const struct net_device_ops airoha_netdev_ops = {
>  	.ndo_xdp_xmit		= airoha_xdp_xmit,
>  };
>  
> +#define AIROHA_ETHTOOL_XDP_STAT(x) { #x, \
> +				  offsetof(struct airoha_hw_stats, xdp_stats.x) / \
> +				  sizeof(u64) }
> +
> +static const struct {
> +	const char name[ETH_GSTRING_LEN];
> +	u32 offset;
> +} airoha_ethtool_xdp_stats[] = {
> +	AIROHA_ETHTOOL_XDP_STAT(rx_xdp_redirect),
> +	AIROHA_ETHTOOL_XDP_STAT(rx_xdp_redirect_errors),
> +	AIROHA_ETHTOOL_XDP_STAT(rx_xdp_pass),
> +	AIROHA_ETHTOOL_XDP_STAT(rx_xdp_aborted),
> +	AIROHA_ETHTOOL_XDP_STAT(rx_xdp_drop),
> +	AIROHA_ETHTOOL_XDP_STAT(rx_xdp_tx),
> +	AIROHA_ETHTOOL_XDP_STAT(rx_xdp_tx_errors),
> +	AIROHA_ETHTOOL_XDP_STAT(tx_xdp_xmit),
> +	AIROHA_ETHTOOL_XDP_STAT(tx_xdp_xmit_errors),
> +};
> +
> +static void airoha_ethtool_get_strings(struct net_device *netdev, u32 stringset, u8 *data)

nit I like old style :)

	static void airoha_ethtool_get_strings(struct net_device *netdev,
					       u32 stringset, u8 *data)
	{
		....
	}

> +{
> +	int i;
> +
> +	switch (stringset) {
> +	case ETH_SS_STATS:
> +		for (i = 0; i < ARRAY_SIZE(airoha_ethtool_xdp_stats); i++) {
> +			strscpy(data, airoha_ethtool_xdp_stats[i].name, ETH_GSTRING_LEN);
> +			data += ETH_GSTRING_LEN;
> +		}
> +		page_pool_ethtool_stats_get_strings(data);
> +		break;
> +	}
> +}
> +
> +static int airoha_ethtool_get_sset_count(struct net_device *netdev, int stringset)
> +{
> +	switch (stringset) {
> +	case ETH_SS_STATS:
> +		return ARRAY_SIZE(airoha_ethtool_xdp_stats) + page_pool_ethtool_stats_get_count();

same here:
		return ARRAY_SIZE(airoha_ethtool_xdp_stats) +
		       page_pool_ethtool_stats_get_count();

> +	default:
> +		return -EOPNOTSUPP;
> +	}
> +}
> +
> +static void airoha_ethtool_get_ethtool_stats(struct net_device *netdev,
> +					     struct ethtool_stats *stats,
> +					     u64 *data)
> +{
> +	struct airoha_gdm_dev *dev = netdev_priv(netdev);
> +	struct airoha_hw_stats *hw_stats = &dev->stats;
> +	u64 *hw_stats_base = (u64 *)hw_stats;
> +	struct page_pool_stats pp_stats = {};
> +	struct airoha_qdma *qdma;
> +	unsigned int start;
> +	int i;
> +
> +	if (netif_running(netdev))
> +		airoha_update_hw_stats(dev);
> +
> +	do {
> +		start = u64_stats_fetch_begin(&hw_stats->syncp);
> +		for (i = 0; i < ARRAY_SIZE(airoha_ethtool_xdp_stats); i++)
> +			data[i] = hw_stats_base[airoha_ethtool_xdp_stats[i].offset];
> +	} while (u64_stats_fetch_retry(&hw_stats->syncp, start));
> +
> +	rcu_read_lock();
> +	qdma = rcu_dereference(dev->qdma);

IIRC airoha_ethtool_get_ethtool_stats() is running under RTNL so you can just
use airoha_qdma_deref() here.

Regards,
Lorenzo

> +	if (qdma) {
> +		for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
> +			struct airoha_queue *q = &qdma->q_rx[i];
> +
> +			if (q->page_pool)
> +				page_pool_get_stats(q->page_pool, &pp_stats);
> +		}
> +	}
> +	rcu_read_unlock();
> +
> +	page_pool_ethtool_stats_get(data + ARRAY_SIZE(airoha_ethtool_xdp_stats),
> +				    &pp_stats);
> +}
> +
>  static const struct ethtool_ops airoha_ethtool_ops = {
>  	.get_drvinfo		= airoha_ethtool_get_drvinfo,
>  	.get_eth_mac_stats      = airoha_ethtool_get_mac_stats,
>  	.get_rmon_stats		= airoha_ethtool_get_rmon_stats,
>  	.get_link_ksettings	= phy_ethtool_get_link_ksettings,
>  	.get_link		= ethtool_op_get_link,
> +	.get_strings		= airoha_ethtool_get_strings,
> +	.get_sset_count		= airoha_ethtool_get_sset_count,
> +	.get_ethtool_stats	= airoha_ethtool_get_ethtool_stats,
>  };
>  
>  static int airoha_metadata_dst_alloc(struct airoha_gdm_port *port)
> diff --git a/drivers/net/ethernet/airoha/airoha_eth.h b/drivers/net/ethernet/airoha/airoha_eth.h
> index cfbc0b8f5bf9..d2ac295c49e9 100644
> --- a/drivers/net/ethernet/airoha/airoha_eth.h
> +++ b/drivers/net/ethernet/airoha/airoha_eth.h
> @@ -236,6 +236,18 @@ struct airoha_tx_irq_queue {
>  	u32 *q;
>  };
>  
> +struct airoha_xdp_stats {
> +	u64 rx_xdp_redirect;
> +	u64 rx_xdp_redirect_errors;
> +	u64 rx_xdp_pass;
> +	u64 rx_xdp_aborted;
> +	u64 rx_xdp_drop;
> +	u64 rx_xdp_tx;
> +	u64 rx_xdp_tx_errors;
> +	u64 tx_xdp_xmit;
> +	u64 tx_xdp_xmit_errors;
> +};
> +
>  struct airoha_hw_stats {
>  	struct u64_stats_sync syncp;
>  
> @@ -259,6 +271,8 @@ struct airoha_hw_stats {
>  	u64 rx_jabber;
>  	u64 rx_len[7];
>  
> +	struct airoha_xdp_stats xdp_stats;
> +
>  	struct {
>  		/* Previous HW register values for 32-bit counter delta
>  		 * tracking. Storing the last seen value and accumulating
> -- 
> 2.55.0
> 
> 

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 228 bytes --]