* [PATCH net-next V3 1/2] net/mlx5e: Batch NAPI RCU grace periods on channel close
2026-11-19 10:39 [PATCH net-next V3 0/2] net/mlx5e: Batch RCU grace periods on channel open and close Tariq Toukan
@ 2026-11-19 10:39 ` Tariq Toukan
2026-11-19 10:39 ` [PATCH net-next V3 2/2] net/mlx5e: Batch ICOSQ trigger RCU synchronization Tariq Toukan
1 sibling, 0 replies; 3+ messages in thread
From: Tariq Toukan @ 2026-11-19 10:39 UTC (permalink / raw)
To: Andrew Lunn, David S. Miller, Eric Dumazet, Jakub Kicinski,
netdev, Paolo Abeni
Cc: Alexei Starovoitov, bpf, Carolina Jubran, Cosmin Ratiu,
Daniel Borkmann, Dragos Tatulea, Gal Pressman,
Jesper Dangaard Brouer, John Fastabend, Kees Cook,
Leon Romanovsky, open list, linux-rdma, Li RongQing, Mark Bloch,
Richard Cochran, Saeed Mahameed, Stanislav Fomichev,
Tariq Toukan
From: Carolina Jubran <cjubran@nvidia.com>
Channel close currently invokes synchronize_net() repeatedly while
tearing down each channel. RQ, ICOSQ, TX SQ and NAPI teardown each
require a grace period, causing the number of waits to scale with the
number of channels. TX SQ teardown adds a wait for each traffic class,
while XDP and QoS can add further calls.
Batch these waits across all channels. Split channel deactivate and
close into pre_sync and post_sync phases. First clear the ENABLED state
on all queues, then wait for a single grace period before continuing
with the teardown. Once NAPI is disabled on all channels, wait for one
more grace period, as the poll may still arm the CQs after
napi_complete_done().
Similarly, close the queues and unlink all NAPI instances with
__netif_napi_del_locked(), issue one synchronize_net() after all of
them have been unlinked, and only then free the channels.
Keep the existing per object synchronize_net() calls for single queue
paths. Drop the one in mlx5e_close_xdpsq(), as its callers already
wait for a grace period or run before NAPI is enabled.
248 channels, rcu_normal=1, ip link set eth2 down:
Before: 11.837s
After: 0.817s (~14.5x faster)
Signed-off-by: Carolina Jubran <cjubran@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
---
drivers/net/ethernet/mellanox/mlx5/core/en.h | 4 +
.../net/ethernet/mellanox/mlx5/core/en/ptp.c | 27 +++-
.../net/ethernet/mellanox/mlx5/core/en/ptp.h | 4 +-
.../net/ethernet/mellanox/mlx5/core/en/qos.c | 42 +++++-
.../net/ethernet/mellanox/mlx5/core/en/qos.h | 5 +-
.../net/ethernet/mellanox/mlx5/core/en/xdp.h | 2 -
.../mellanox/mlx5/core/en/xsk/setup.c | 22 ++-
.../mellanox/mlx5/core/en/xsk/setup.h | 2 +
.../net/ethernet/mellanox/mlx5/core/en_main.c | 126 ++++++++++++++----
9 files changed, 185 insertions(+), 49 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index a3945195f454..657a7d59c845 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -1158,8 +1158,10 @@ int mlx5e_ptp_rx_manage_fs_ctx(struct mlx5e_priv *priv, void *ctx);
int mlx5e_flush_rq(struct mlx5e_rq *rq, int curr_state);
void mlx5e_activate_rq(struct mlx5e_rq *rq);
+void mlx5e_deactivate_rq_pre_sync(struct mlx5e_rq *rq);
void mlx5e_deactivate_rq(struct mlx5e_rq *rq);
void mlx5e_activate_icosq(struct mlx5e_icosq *icosq);
+void mlx5e_deactivate_icosq_pre_sync(struct mlx5e_icosq *icosq);
void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq);
int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,
@@ -1169,6 +1171,8 @@ int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix,
struct mlx5e_txqsq *sq, int tc, u16 qos_queue_group_id,
struct mlx5e_sq_stats *sq_stats);
void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq);
+void mlx5e_deactivate_txqsq_pre_sync(struct mlx5e_txqsq *sq);
+void mlx5e_deactivate_txqsq_post_sync(struct mlx5e_txqsq *sq);
void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq);
void mlx5e_free_txqsq(struct mlx5e_txqsq *sq);
void mlx5e_tx_disable_queue(struct netdev_queue *txq);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
index 2edb55afff1d..f67f6beaf190 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
@@ -928,10 +928,17 @@ int mlx5e_ptp_open(struct mlx5e_priv *priv, struct mlx5e_params *params,
return err;
}
-void mlx5e_ptp_close(struct mlx5e_ptp *c)
+void mlx5e_ptp_close_pre_sync(struct mlx5e_ptp *c)
{
+ /* NAPI is disabled and synced, or was never enabled. */
mlx5e_ptp_close_queues(c);
- netif_napi_del_locked(&c->napi);
+ __netif_napi_del_locked(&c->napi);
+}
+
+void mlx5e_ptp_close(struct mlx5e_ptp *c)
+{
+ mlx5e_ptp_close_pre_sync(c);
+ synchronize_net(); /* Sync with NAPI and page pool readers. */
kvfree(c);
}
@@ -954,18 +961,28 @@ void mlx5e_ptp_activate_channel(struct mlx5e_ptp *c)
mlx5e_trigger_napi_sched(&c->napi);
}
-void mlx5e_ptp_deactivate_channel(struct mlx5e_ptp *c)
+void mlx5e_ptp_deactivate_channel_pre_sync(struct mlx5e_ptp *c)
{
int tc;
if (test_bit(MLX5E_PTP_STATE_RX, c->state)) {
netif_queue_set_napi(c->netdev, c->rq.ix, NETDEV_QUEUE_TYPE_RX, NULL);
- mlx5e_deactivate_rq(&c->rq);
+ mlx5e_deactivate_rq_pre_sync(&c->rq);
+ }
+
+ if (test_bit(MLX5E_PTP_STATE_TX, c->state)) {
+ for (tc = 0; tc < c->num_tc; tc++)
+ mlx5e_deactivate_txqsq_pre_sync(&c->ptpsq[tc].txqsq);
}
+}
+
+void mlx5e_ptp_deactivate_channel_post_sync(struct mlx5e_ptp *c)
+{
+ int tc;
if (test_bit(MLX5E_PTP_STATE_TX, c->state)) {
for (tc = 0; tc < c->num_tc; tc++)
- mlx5e_deactivate_txqsq(&c->ptpsq[tc].txqsq);
+ mlx5e_deactivate_txqsq_post_sync(&c->ptpsq[tc].txqsq);
}
napi_disable_locked(&c->napi);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h
index 2a457a2ed707..47d4bae5f354 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h
@@ -128,9 +128,11 @@ static inline bool mlx5e_ptpsq_metadata_freelist_empty(struct mlx5e_ptpsq *ptpsq
int mlx5e_ptp_open(struct mlx5e_priv *priv, struct mlx5e_params *params,
u8 lag_port, struct mlx5e_ptp **cp);
+void mlx5e_ptp_close_pre_sync(struct mlx5e_ptp *c);
void mlx5e_ptp_close(struct mlx5e_ptp *c);
void mlx5e_ptp_activate_channel(struct mlx5e_ptp *c);
-void mlx5e_ptp_deactivate_channel(struct mlx5e_ptp *c);
+void mlx5e_ptp_deactivate_channel_pre_sync(struct mlx5e_ptp *c);
+void mlx5e_ptp_deactivate_channel_post_sync(struct mlx5e_ptp *c);
int mlx5e_ptp_get_rqn(struct mlx5e_ptp *c, u32 *rqn);
int mlx5e_ptp_alloc_rx_fs(struct mlx5e_flow_steering *fs,
const struct mlx5e_profile *profile);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c
index 3176549eff3e..604c9e67c16a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c
@@ -233,7 +233,7 @@ void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid)
kfree(sq);
}
-void mlx5e_qos_close_queues(struct mlx5e_channel *c)
+void mlx5e_qos_close_queues(struct mlx5e_channel *c, bool sync_needed)
{
struct mlx5e_txqsq __rcu **qos_sqs;
int i;
@@ -241,7 +241,8 @@ void mlx5e_qos_close_queues(struct mlx5e_channel *c)
qos_sqs = rcu_replace_pointer(c->qos_sqs, NULL, lockdep_is_held(&c->priv->state_lock));
if (!qos_sqs)
return;
- synchronize_rcu(); /* Sync with NAPI. */
+ if (sync_needed)
+ synchronize_rcu(); /* Sync with NAPI. */
for (i = 0; i < c->qos_sqs_size; i++) {
struct mlx5e_txqsq *sq;
@@ -263,7 +264,7 @@ void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs)
int i;
for (i = 0; i < chs->num; i++)
- mlx5e_qos_close_queues(chs->c[i]);
+ mlx5e_qos_close_queues(chs->c[i], true);
}
int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs)
@@ -326,11 +327,10 @@ void mlx5e_qos_activate_queues(struct mlx5e_priv *priv)
mlx5e_htb_enumerate_leaves(priv->htb, mlx5e_activate_qos_sq, priv);
}
-void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c)
+void mlx5e_qos_deactivate_queues_pre_sync(struct mlx5e_channel *c)
{
struct mlx5e_params *params = &c->priv->channels.params;
struct mlx5e_txqsq __rcu **qos_sqs;
- u16 txq_ix;
int i;
qos_sqs = mlx5e_state_dereference(c->priv, c->qos_sqs);
@@ -346,7 +346,30 @@ void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c)
continue;
qos_dbg(c->mdev, "Deactivate QoS SQ qid %u\n", qid);
- mlx5e_deactivate_txqsq(sq);
+ mlx5e_deactivate_txqsq_pre_sync(sq);
+ }
+}
+
+void mlx5e_qos_deactivate_queues_post_sync(struct mlx5e_channel *c)
+{
+ struct mlx5e_params *params = &c->priv->channels.params;
+ struct mlx5e_txqsq __rcu **qos_sqs;
+ int i;
+
+ qos_sqs = mlx5e_state_dereference(c->priv, c->qos_sqs);
+ if (!qos_sqs)
+ return;
+
+ for (i = 0; i < c->qos_sqs_size; i++) {
+ u16 qid = params->num_channels * i + c->ix;
+ struct mlx5e_txqsq *sq;
+ u16 txq_ix;
+
+ sq = mlx5e_state_dereference(c->priv, qos_sqs[i]);
+ if (!sq)
+ continue;
+
+ mlx5e_deactivate_txqsq_post_sync(sq);
txq_ix = mlx5e_qid_from_qos(&c->priv->channels, qid);
@@ -361,7 +384,12 @@ void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs)
int i;
for (i = 0; i < chs->num; i++)
- mlx5e_qos_deactivate_queues(chs->c[i]);
+ mlx5e_qos_deactivate_queues_pre_sync(chs->c[i]);
+
+ synchronize_net();
+
+ for (i = 0; i < chs->num; i++)
+ mlx5e_qos_deactivate_queues_post_sync(chs->c[i]);
}
void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.h b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.h
index 4947afa23b73..57abeb5887b6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.h
@@ -28,9 +28,10 @@ void mlx5e_reset_qdisc(struct net_device *dev, u16 qid);
int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs);
void mlx5e_qos_activate_queues(struct mlx5e_priv *priv);
-void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c);
+void mlx5e_qos_deactivate_queues_pre_sync(struct mlx5e_channel *c);
+void mlx5e_qos_deactivate_queues_post_sync(struct mlx5e_channel *c);
void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs);
-void mlx5e_qos_close_queues(struct mlx5e_channel *c);
+void mlx5e_qos_close_queues(struct mlx5e_channel *c, bool sync_needed);
void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs);
int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
index 5b8f4094d553..10fd91eda9b4 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
@@ -136,8 +136,6 @@ static inline void mlx5e_xdp_tx_disable(struct mlx5e_priv *priv)
clear_bit(MLX5E_STATE_XDP_ACTIVE, &priv->state);
clear_bit(MLX5E_STATE_XDP_TX_ENABLED, &priv->state);
- /* Let other device's napi(s) and XSK wakeups see our new state. */
- synchronize_net();
}
static inline bool mlx5e_xdp_tx_is_enabled(struct mlx5e_priv *priv)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
index 11500fd213a5..1bdccb823eb1 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
@@ -163,11 +163,9 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
return err;
}
-void mlx5e_close_xsk(struct mlx5e_channel *c)
+void mlx5e_close_xsk_queues(struct mlx5e_channel *c)
{
- clear_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
- synchronize_net(); /* Sync with NAPI. */
-
+ /* Caller guarantees that NAPI can no longer access the XSK queues. */
mlx5e_close_rq(&c->xskrq);
mlx5e_close_cq(&c->xskrq.cq);
mlx5e_close_xdpsq(&c->xsksq);
@@ -177,6 +175,13 @@ void mlx5e_close_xsk(struct mlx5e_channel *c)
memset(&c->xsksq, 0, sizeof(c->xsksq));
}
+void mlx5e_close_xsk(struct mlx5e_channel *c)
+{
+ clear_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
+ synchronize_net(); /* Sync with NAPI. */
+ mlx5e_close_xsk_queues(c);
+}
+
void mlx5e_activate_xsk(struct mlx5e_channel *c)
{
/* ICOSQ recovery deactivates RQs. Suspend the recovery to avoid
@@ -189,7 +194,7 @@ void mlx5e_activate_xsk(struct mlx5e_channel *c)
/* TX queue is created active. */
}
-void mlx5e_deactivate_xsk(struct mlx5e_channel *c)
+void mlx5e_deactivate_xsk_pre_sync(struct mlx5e_channel *c)
{
/* ICOSQ recovery may reactivate XSKRQ if clear_bit is called in the
* middle of recovery. Suspend the recovery to avoid it.
@@ -197,7 +202,12 @@ void mlx5e_deactivate_xsk(struct mlx5e_channel *c)
mlx5e_reporter_icosq_suspend_recovery(c);
clear_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state);
mlx5e_reporter_icosq_resume_recovery(c);
- synchronize_net(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */
/* TX queue is disabled on close. */
}
+
+void mlx5e_deactivate_xsk(struct mlx5e_channel *c)
+{
+ mlx5e_deactivate_xsk_pre_sync(c);
+ synchronize_net(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h
index 664ec78192c3..203cff64fcef 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h
@@ -16,8 +16,10 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
struct mlx5e_channel_param *cparam,
struct xsk_buff_pool *pool,
struct mlx5e_channel *c);
+void mlx5e_close_xsk_queues(struct mlx5e_channel *c);
void mlx5e_close_xsk(struct mlx5e_channel *c);
void mlx5e_activate_xsk(struct mlx5e_channel *c);
+void mlx5e_deactivate_xsk_pre_sync(struct mlx5e_channel *c);
void mlx5e_deactivate_xsk(struct mlx5e_channel *c);
#endif /* __MLX5_EN_XSK_SETUP_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index d8563f863844..65e86b4cef85 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -1458,9 +1458,14 @@ void mlx5e_activate_rq(struct mlx5e_rq *rq)
set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
}
-void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
+void mlx5e_deactivate_rq_pre_sync(struct mlx5e_rq *rq)
{
clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
+}
+
+void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
+{
+ mlx5e_deactivate_rq_pre_sync(rq);
synchronize_net(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */
}
@@ -1954,13 +1959,15 @@ void mlx5e_tx_disable_queue(struct netdev_queue *txq)
__netif_tx_unlock_bh(txq);
}
-void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq)
+void mlx5e_deactivate_txqsq_pre_sync(struct mlx5e_txqsq *sq)
{
- struct mlx5_wq_cyc *wq = &sq->wq;
-
netif_queue_set_napi(sq->netdev, sq->txq_ix, NETDEV_QUEUE_TYPE_TX, NULL);
clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
- synchronize_net(); /* Sync with NAPI to prevent netif_tx_wake_queue. */
+}
+
+void mlx5e_deactivate_txqsq_post_sync(struct mlx5e_txqsq *sq)
+{
+ struct mlx5_wq_cyc *wq = &sq->wq;
mlx5e_tx_disable_queue(sq->txq);
@@ -1978,6 +1985,13 @@ void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq)
}
}
+void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq)
+{
+ mlx5e_deactivate_txqsq_pre_sync(sq);
+ synchronize_net(); /* Sync with NAPI to prevent netif_tx_wake_queue. */
+ mlx5e_deactivate_txqsq_post_sync(sq);
+}
+
void mlx5e_close_txqsq(struct mlx5e_txqsq *sq)
{
struct mlx5_core_dev *mdev = sq->mdev;
@@ -2089,9 +2103,14 @@ void mlx5e_activate_icosq(struct mlx5e_icosq *icosq)
set_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state);
}
-void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq)
+void mlx5e_deactivate_icosq_pre_sync(struct mlx5e_icosq *icosq)
{
clear_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state);
+}
+
+void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq)
+{
+ mlx5e_deactivate_icosq_pre_sync(icosq);
synchronize_net(); /* Sync with NAPI. */
}
@@ -2145,8 +2164,8 @@ void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq)
{
struct mlx5e_channel *c = sq->channel;
+ /* Caller guarantees that NAPI can no longer access the XDP SQ. */
clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
- synchronize_net(); /* Sync with NAPI. */
mlx5e_destroy_sq(c->mdev, sq->sqn);
mlx5e_free_xdpsq_descs(sq);
@@ -2936,35 +2955,52 @@ static void mlx5e_activate_channel(struct mlx5e_channel *c)
netif_queue_set_napi(c->netdev, c->ix, NETDEV_QUEUE_TYPE_RX, &c->napi);
}
-static void mlx5e_deactivate_channel(struct mlx5e_channel *c)
+static void mlx5e_deactivate_channel_pre_sync(struct mlx5e_channel *c)
{
int tc;
netif_queue_set_napi(c->netdev, c->ix, NETDEV_QUEUE_TYPE_RX, NULL);
if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
- mlx5e_deactivate_xsk(c);
+ mlx5e_deactivate_xsk_pre_sync(c);
else
- mlx5e_deactivate_rq(&c->rq);
+ mlx5e_deactivate_rq_pre_sync(&c->rq);
if (c->async_icosq)
- mlx5e_deactivate_icosq(c->async_icosq);
- mlx5e_deactivate_icosq(&c->icosq);
+ mlx5e_deactivate_icosq_pre_sync(c->async_icosq);
+ mlx5e_deactivate_icosq_pre_sync(&c->icosq);
+ for (tc = 0; tc < c->num_tc; tc++)
+ mlx5e_deactivate_txqsq_pre_sync(&c->sq[tc]);
+ mlx5e_qos_deactivate_queues_pre_sync(c);
+}
+
+static void mlx5e_deactivate_channel_post_sync(struct mlx5e_channel *c)
+{
+ int tc;
+
for (tc = 0; tc < c->num_tc; tc++)
- mlx5e_deactivate_txqsq(&c->sq[tc]);
- mlx5e_qos_deactivate_queues(c);
+ mlx5e_deactivate_txqsq_post_sync(&c->sq[tc]);
+ mlx5e_qos_deactivate_queues_post_sync(c);
napi_disable_locked(&c->napi);
}
-static void mlx5e_close_channel(struct mlx5e_channel *c)
+static void mlx5e_close_channel_pre_sync(struct mlx5e_channel *c)
{
- if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
- mlx5e_close_xsk(c);
+ /* NAPI is disabled and synced, or was never enabled, so the queues
+ * can be destroyed without another grace period.
+ */
+ if (test_and_clear_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
+ mlx5e_close_xsk_queues(c);
mlx5e_close_queues(c);
- mlx5e_qos_close_queues(c);
- netif_napi_del_locked(&c->napi);
+ mlx5e_qos_close_queues(c, false);
+ __netif_napi_del_locked(&c->napi);
+}
+static void mlx5e_close_channel(struct mlx5e_channel *c)
+{
+ mlx5e_close_channel_pre_sync(c);
+ synchronize_net(); /* Sync with NAPI and page pool readers. */
kvfree(c);
}
@@ -3060,32 +3096,63 @@ static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs)
return err ? -ETIMEDOUT : 0;
}
-static void mlx5e_deactivate_channels(struct mlx5e_channels *chs)
+static void mlx5e_deactivate_channels_pre_sync(struct mlx5e_channels *chs)
{
int i;
if (chs->ptp)
- mlx5e_ptp_deactivate_channel(chs->ptp);
+ mlx5e_ptp_deactivate_channel_pre_sync(chs->ptp);
for (i = 0; i < chs->num; i++)
- mlx5e_deactivate_channel(chs->c[i]);
+ mlx5e_deactivate_channel_pre_sync(chs->c[i]);
}
-void mlx5e_close_channels(struct mlx5e_channels *chs)
+static void mlx5e_deactivate_channels_post_sync(struct mlx5e_channels *chs)
+{
+ int i;
+
+ if (chs->ptp)
+ mlx5e_ptp_deactivate_channel_post_sync(chs->ptp);
+
+ for (i = 0; i < chs->num; i++)
+ mlx5e_deactivate_channel_post_sync(chs->c[i]);
+}
+
+static void mlx5e_close_channels_pre_sync(struct mlx5e_channels *chs)
+{
+ int i;
+
+ if (chs->ptp)
+ mlx5e_ptp_close_pre_sync(chs->ptp);
+
+ for (i = 0; i < chs->num; i++)
+ mlx5e_close_channel_pre_sync(chs->c[i]);
+}
+
+static void mlx5e_close_channels_post_sync(struct mlx5e_channels *chs)
{
int i;
if (chs->ptp) {
- mlx5e_ptp_close(chs->ptp);
+ kvfree(chs->ptp);
chs->ptp = NULL;
}
+
for (i = 0; i < chs->num; i++)
- mlx5e_close_channel(chs->c[i]);
+ kvfree(chs->c[i]);
kfree(chs->c);
chs->num = 0;
}
+void mlx5e_close_channels(struct mlx5e_channels *chs)
+{
+ mlx5e_close_channels_pre_sync(chs);
+ /* One grace period for NAPI and page pool readers. */
+ synchronize_net();
+ mlx5e_close_channels_post_sync(chs);
+}
+
static int mlx5e_modify_tirs_packet_merge(struct mlx5e_priv *priv)
{
struct mlx5e_rx_res *res = priv->rx_res;
@@ -3394,7 +3461,14 @@ void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv)
netif_tx_disable(priv->netdev);
mlx5e_xdp_tx_disable(priv);
- mlx5e_deactivate_channels(&priv->channels);
+ mlx5e_deactivate_channels_pre_sync(&priv->channels);
+ /* One grace period for XDP, RQ, ICOSQ, TX, QoS, XSK and PTP. */
+ synchronize_net();
+ mlx5e_deactivate_channels_post_sync(&priv->channels);
+ /* napi_disable() can return while the poll still arms the CQs.
+ * Wait for it, so close can destroy the queues right away.
+ */
+ synchronize_net();
}
static int mlx5e_switch_priv_params(struct mlx5e_priv *priv,
--
2.44.0
^ permalink raw reply [flat|nested] 3+ messages in thread* [PATCH net-next V3 2/2] net/mlx5e: Batch ICOSQ trigger RCU synchronization
2026-11-19 10:39 [PATCH net-next V3 0/2] net/mlx5e: Batch RCU grace periods on channel open and close Tariq Toukan
2026-11-19 10:39 ` [PATCH net-next V3 1/2] net/mlx5e: Batch NAPI RCU grace periods on channel close Tariq Toukan
@ 2026-11-19 10:39 ` Tariq Toukan
1 sibling, 0 replies; 3+ messages in thread
From: Tariq Toukan @ 2026-11-19 10:39 UTC (permalink / raw)
To: Andrew Lunn, David S. Miller, Eric Dumazet, Jakub Kicinski,
netdev, Paolo Abeni
Cc: Alexei Starovoitov, bpf, Carolina Jubran, Cosmin Ratiu,
Daniel Borkmann, Dragos Tatulea, Gal Pressman,
Jesper Dangaard Brouer, John Fastabend, Kees Cook,
Leon Romanovsky, open list, linux-rdma, Li RongQing, Mark Bloch,
Richard Cochran, Saeed Mahameed, Stanislav Fomichev,
Tariq Toukan
From: Carolina Jubran <cjubran@nvidia.com>
During channel activation, mlx5e_activate_channels() triggers the ICOSQ
once for every channel in order to kick NAPI. mlx5e_trigger_napi_icosq()
sets MLX5E_SQ_STATE_LOCK_NEEDED and waits for an RCU grace period before
serializing the trigger with ICOSQ users in NAPI context.
Mark all channel ICOSQs as requiring locking first, wait for a single
grace period, and then trigger all of them.
Keep mlx5e_trigger_napi_icosq() with its existing synchronization
semantics for standalone callers.
248 channels, rcu_normal=1, ip link set eth2 up:
Before: 4.087s
After: 1.080s (~3.8x faster)
Signed-off-by: Carolina Jubran <cjubran@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
---
.../net/ethernet/mellanox/mlx5/core/en_main.c | 28 +++++++++++++++----
1 file changed, 22 insertions(+), 6 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 65e86b4cef85..e599968d06d7 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -2799,14 +2799,11 @@ static int mlx5e_channel_stats_alloc(struct mlx5e_priv *priv, int ix, int cpu)
return 0;
}
-void mlx5e_trigger_napi_icosq(struct mlx5e_channel *c)
+static void mlx5e_trigger_napi_icosq_post_sync(struct mlx5e_channel *c)
{
struct mlx5e_icosq *sq = &c->icosq;
bool locked;
- set_bit(MLX5E_SQ_STATE_LOCK_NEEDED, &sq->state);
- synchronize_net();
-
locked = mlx5e_icosq_sync_lock(sq);
mlx5e_trigger_irq(sq);
mlx5e_icosq_sync_unlock(sq, locked);
@@ -2814,6 +2811,13 @@ void mlx5e_trigger_napi_icosq(struct mlx5e_channel *c)
clear_bit(MLX5E_SQ_STATE_LOCK_NEEDED, &sq->state);
}
+void mlx5e_trigger_napi_icosq(struct mlx5e_channel *c)
+{
+ set_bit(MLX5E_SQ_STATE_LOCK_NEEDED, &c->icosq.state);
+ synchronize_net();
+ mlx5e_trigger_napi_icosq_post_sync(c);
+}
+
void mlx5e_trigger_napi_async_icosq(struct mlx5e_channel *c)
{
struct mlx5e_icosq *sq = c->async_icosq;
@@ -3057,6 +3061,19 @@ int mlx5e_open_channels(struct mlx5e_priv *priv,
return err;
}
+static void mlx5e_trigger_napi_icosqs(struct mlx5e_channels *chs)
+{
+ int i;
+
+ for (i = 0; i < chs->num; i++)
+ set_bit(MLX5E_SQ_STATE_LOCK_NEEDED, &chs->c[i]->icosq.state);
+
+ synchronize_net();
+
+ for (i = 0; i < chs->num; i++)
+ mlx5e_trigger_napi_icosq_post_sync(chs->c[i]);
+}
+
static void mlx5e_activate_channels(struct mlx5e_priv *priv, struct mlx5e_channels *chs)
{
int i;
@@ -3067,8 +3084,7 @@ static void mlx5e_activate_channels(struct mlx5e_priv *priv, struct mlx5e_channe
if (priv->htb)
mlx5e_qos_activate_queues(priv);
- for (i = 0; i < chs->num; i++)
- mlx5e_trigger_napi_icosq(chs->c[i]);
+ mlx5e_trigger_napi_icosqs(chs);
if (chs->ptp)
mlx5e_ptp_activate_channel(chs->ptp);
--
2.44.0
^ permalink raw reply [flat|nested] 3+ messages in thread