[RFC net-next 6/7] net/mlx5e: kTLS, Keep the TX queue of an offloaded socket stable

[Tariq Toukan <tariqt@nvidia.com>]

kTLS TX WQEs update the TIS crypto and record tracking state at
execution time, and WQEs of different SQs are not ordered. A connection
must not move to another SQ while WQEs that it posted to its SQ may be
pending.

With TLSDEV_F_STICKY_TXQ, the stack keeps the socket's TX queue while
the socket has packets in flight. mlx5e_select_queue() uses that queue
to pick the channel only: with multiple TCs, it adds the offset of the
skb's UP, derived from its DSCP or VLAN priority, and with HTB, it picks
the QoS queue of the skb's priority. A connection whose packets change
priority moves to another SQ at any time.

For a SOCK_STICKY_TXQ socket, cache the final queue in the socket, and
reuse it while TCP does not allow a move (ooo_okay is clear). A priority
change then takes effect once no packet of the connection is in flight.
TCP sets ooo_okay when it transmits the skb, and it may be stale by the
time the skb is selected, e.g. after an upper device's qdisc reordered
the socket's skbs, so a move also requires that no other skb of the
socket is in flight. A cached PTP queue is not reused, these are
selected for PTP event packets only.

The remaining moves are covered by the net fixes "net/mlx5e: Drain old
TX SQs before activating new channels", for channel replacement and the
TX queue count changes that come with it, and "net/mlx5e: kTLS, Hold a
dropped skb until its posted WQEs complete", which keeps sk_wmem_alloc
held while WQEs of a dropped skb may be pending. When an HTB leaf is
deleted or becomes an inner class, its queue is removed or given to
another class only after its queued packets were dropped and its SQ
closed, so its connections have nothing in flight and pick a queue anew.
An skb that netem partially orphaned holds no sk_wmem_alloc charge, but
it carries no data for the device, and does not post WQEs that name the
TIS since "net/mlx5e: kTLS, Post TX context params only for skbs that
use them".

Opt in to TLSDEV_F_STICKY_TXQ.

Found by LLM-assisted code inspection, while answering a question on
kTLS TX TIS reuse for TLS 1.3 rekey on netdev, and reproduced on
ConnectX-6 Dx with forced queue moves.

Reproduced on ConnectX-6 Dx, with 128 channels, mqprio with 8 TCs and
DSCP trust mapping DSCP 46 to priority 5, by flipping the IP_TOS of 32
kTLS TX offloaded connections between 0 and DSCP 46 every 5 ms while
they stream pattern data for 60 seconds to a software kTLS receiver that
checks every byte. Without this patch, 115 of 147 connections failed
decryption. With it, none did, and the connections still reached the TC5
queues.

Fixes: d2ead1f360e8 ("net/mlx5e: Add kTLS TX HW offload support")
Link: https://lore.kernel.org/netdev/20261022181751.1349984-1-rjethwani@purestorage.com/
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
---
 .../net/ethernet/mellanox/mlx5/core/en/selq.c | 92 ++++++++++++++++---
 .../mellanox/mlx5/core/en_accel/ktls.c        |  1 +
 2 files changed, 81 insertions(+), 12 deletions(-)

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/selq.c b/drivers/net/ethernet/mellanox/mlx5/core/en/selq.c
index 8318a26bc263..dfaee2c3624e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/selq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/selq.c
@@ -5,6 +5,7 @@
 #include <linux/slab.h>
 #include <linux/netdevice.h>
 #include <linux/rcupdate.h>
+#include <net/sock.h>
 #include "en.h"
 #include "en/ptp.h"
 #include "en/htb.h"
@@ -190,22 +191,12 @@ static int mlx5e_select_htb_queue(struct mlx5e_priv *priv, struct sk_buff *skb,
 	return mlx5e_htb_get_txq_by_classid(priv->htb, classid);
 }
 
-u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb,
-		       struct net_device *sb_dev)
+static u16 __mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb,
+				struct mlx5e_selq_params *selq)
 {
 	struct mlx5e_priv *priv = netdev_priv(dev);
-	struct mlx5e_selq_params *selq;
 	int txq_ix, up;
 
-	selq = rcu_dereference_bh(priv->selq.active);
-
-	/* This is a workaround needed only for the mlx5e_netdev_change_profile
-	 * flow that zeroes out the whole priv without unregistering the netdev
-	 * and without preventing ndo_select_queue from being called.
-	 */
-	if (unlikely(!selq))
-		return 0;
-
 	if (likely(!selq->is_special_queues)) {
 		/* No special queues, netdev_pick_tx returns one of the regular ones. */
 
@@ -266,3 +257,80 @@ u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb,
 
 	return txq_ix + up * selq->num_channels;
 }
+
+static bool mlx5e_sk_sticky_txq(const struct sock *sk)
+{
+	return IS_ENABLED(CONFIG_MLX5_EN_TLS) && sk && sk_fullsock(sk) &&
+	       sock_flag(sk, SOCK_STICKY_TXQ);
+}
+
+/* TCP sets skb->ooo_okay when it transmits the skb, and it may be stale by
+ * now, e.g. after an upper device's qdisc reordered the socket's skbs.
+ * Allow a move only if no other skb of the socket is in flight: a
+ * decrypted skb holds its own sk_wmem_alloc charge, any other skb may not.
+ */
+static bool mlx5e_sk_txq_may_move(const struct sk_buff *skb)
+{
+	int own = skb_is_decrypted(skb) ? skb->truesize : 0;
+
+	return skb->ooo_okay && sk_wmem_alloc_get(skb->sk) <= own;
+}
+
+/* A SOCK_STICKY_TXQ socket must not move to another TX queue while it has
+ * packets in flight, see TLSDEV_F_STICKY_TXQ. netdev_pick_tx() keeps the
+ * socket's queue, but the UP offset and the HTB queue are derived from each
+ * skb. Reuse the final queue of the socket's previous skb, unless TCP allows
+ * a move.
+ */
+static int mlx5e_select_sticky_queue(struct net_device *dev,
+				     struct sk_buff *skb,
+				     struct mlx5e_selq_params *selq)
+{
+	int txq_ix;
+
+	if (mlx5e_sk_txq_may_move(skb))
+		return -1;
+
+	txq_ix = sk_tx_queue_get(skb->sk);
+	if (txq_ix < 0 || txq_ix >= dev->real_num_tx_queues)
+		return -1;
+
+	/* The PTP queues are selected for PTP event packets only. */
+	if (selq->is_ptp && txq_ix >= selq->num_regular_queues &&
+	    txq_ix < selq->num_regular_queues + selq->num_tcs)
+		return -1;
+
+	return txq_ix;
+}
+
+u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb,
+		       struct net_device *sb_dev)
+{
+	struct mlx5e_priv *priv = netdev_priv(dev);
+	struct mlx5e_selq_params *selq;
+	struct sock *sk = skb->sk;
+	int txq_ix;
+
+	selq = rcu_dereference_bh(priv->selq.active);
+
+	/* This is a workaround needed only for the mlx5e_netdev_change_profile
+	 * flow that zeroes out the whole priv without unregistering the netdev
+	 * and without preventing ndo_select_queue from being called.
+	 */
+	if (unlikely(!selq))
+		return 0;
+
+	if (likely(!mlx5e_sk_sticky_txq(sk)))
+		return __mlx5e_select_queue(dev, skb, selq);
+
+	txq_ix = mlx5e_select_sticky_queue(dev, skb, selq);
+	if (txq_ix >= 0)
+		return txq_ix;
+
+	txq_ix = __mlx5e_select_queue(dev, skb, selq);
+	/* Cache the final queue, as netdev_pick_tx() does with its own pick. */
+	if (rcu_access_pointer(sk->sk_dst_cache))
+		sk_tx_queue_set(sk, txq_ix);
+
+	return txq_ix;
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c
index 337201f12895..248ccae9f8e8 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c
@@ -87,6 +87,7 @@ static int mlx5e_ktls_resync(struct net_device *netdev,
 }
 
 static const struct tlsdev_ops mlx5e_ktls_ops = {
+	.flags = TLSDEV_F_STICKY_TXQ,
 	.tls_dev_add = mlx5e_ktls_add,
 	.tls_dev_del = mlx5e_ktls_del,
 	.tls_dev_resync = mlx5e_ktls_resync,
-- 
2.44.0