[PATCH net V2 2/4] net/mlx5: Lag, limit aggregated TX speed by PCIe bandwidth

[Tariq Toukan <tariqt@nvidia.com>]

From: Or Har-Toov <ohartoov@nvidia.com>

FW initializes max_tx_speed to the minimum of the port speed and the
PCI bandwidth. Modifying it with a value computed by different rules
changes the meaning of the field rather than updating it.

Limit the aggregated LAG TX speed by the PCIe bandwidth. Clamping each
uplink separately and then summing could let the aggregate reach a
multiple of the link's real bandwidth, so clamp the sum once instead,
counting only the ports that contribute to it.

This assumes the bonded PFs share a single PCIe link, which holds when
they are functions of the same device. In Socket Direct, each port spans
multiple mdevs on separate PCIe links, so the PCIe budget for that port
is the sum of its mdevs' PCIe bandwidths. Where Socket Direct ports do
not have equal PCIe budgets, the aggregate is clamped to the narrowest
port, which under-reports rather than over-reports the available
bandwidth.

pcie_bandwidth_mbps() reads the endpoint link only, without traversing
upstream bridges, and uses the nominal per-lane transfer rate times the
link width, without encoding overhead. FW initializes max_tx_speed from
the same endpoint-only link status, so using the same calculation keeps
the clamp consistent with the value FW programs before LAG. Ethernet
speeds and max_tx_speed are likewise nominal rates, so the comparison is
like for like. A more accurate bandwidth estimate would make the
aggregate differ from the per-vport value FW sets up for the same link.

If the PCIe bandwidth of a contributing port cannot be read, the speed
calculation fails instead of returning an unclamped or partial result.
A device that has no Link Status register, an RC integrated endpoint,
has no PCIe budget to apply and is not clamped.

Fixes: 50f1d188c580 ("net/mlx5: Propagate LAG effective max_tx_speed to vports")
Signed-off-by: Or Har-Toov <ohartoov@nvidia.com>
Reviewed-by: Mark Bloch <mbloch@nvidia.com>
Reviewed-by: Shay Drori <shayd@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
---
 .../net/ethernet/mellanox/mlx5/core/lag/lag.c | 79 ++++++++++++++++++-
 drivers/pci/pci.c                             | 16 ++++
 include/linux/pci.h                           |  2 +
 3 files changed, 95 insertions(+), 2 deletions(-)

v1: https://lore.kernel.org/all/20260910102432.3845360-5-tariqt@nvidia.com/

Changes in v2:
- Clamp the aggregate sum once by the narrowest PCIe link instead of
  clamping each uplink before summing, which could let the aggregate
  exceed the real bandwidth
- Document the shared PCIe link assumption in the changelog
- Add pcie_bandwidth_mbps() to drivers/pci/pci.c, which reads
  PCI_EXP_LNKSTA once and decodes both speed and width from that
  value, avoiding a race between separate reads of speed and width
- In Socket Direct, sum the PCIe bandwidths of all mdevs in the same
  port group instead of taking the minimum, since those links are
  independent
- Fail the aggregate and max speed calculations when a PCIe bandwidth
  cannot be read, instead of skipping the clamp; an RC integrated
  endpoint has no Link Status and is not clamped

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
index 57755ceab8a7..05e6d0aa31fc 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
@@ -1432,10 +1432,55 @@ static bool mlx5_lag_should_disable_lag(struct mlx5_lag *ldev, bool do_bond)
 	       ldev->mode != MLX5_LAG_MODE_MPESW;
 }
 
+static int mlx5_lag_dev_pcie_bw(struct mlx5_core_dev *dev, u32 *bw)
+{
+	if (pci_pcie_type(dev->pdev) == PCI_EXP_TYPE_RC_END) {
+		*bw = U32_MAX;
+		return 0;
+	}
+
+	*bw = pcie_bandwidth_mbps(dev->pdev);
+	return *bw ? 0 : -EIO;
+}
+
+static int mlx5_lag_port_pcie_bw(struct mlx5_lag *ldev, struct lag_func *port,
+				 u32 *bw)
+{
+	struct lag_func *member;
+	bool unlimited = false;
+	u32 member_bw;
+	int idx;
+	int err;
+
+	if (!port->group_id)
+		return mlx5_lag_dev_pcie_bw(port->dev, bw);
+
+	*bw = 0;
+	mlx5_lag_for_each(idx, 0, ldev, port->group_id) {
+		member = mlx5_lag_pf(ldev, idx);
+		if (!member)
+			continue;
+
+		err = mlx5_lag_dev_pcie_bw(member->dev, &member_bw);
+		if (err)
+			return err;
+
+		if (member_bw == U32_MAX)
+			unlimited = true;
+		else
+			*bw += member_bw;
+	}
+
+	if (unlimited)
+		*bw = U32_MAX;
+	return 0;
+}
+
 static int mlx5_lag_get_devices_oper_speed(struct mlx5_lag *ldev,
 					   u32 *sum_speed)
 {
 	bool mpesw = ldev->mode == MLX5_LAG_MODE_MPESW;
+	u32 pci_bw = U32_MAX;
 	int pf_idx;
 
 	*sum_speed = 0;
@@ -1443,6 +1488,7 @@ static int mlx5_lag_get_devices_oper_speed(struct mlx5_lag *ldev,
 		u8 opmod = MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT;
 		struct mlx5_core_dev *pf_mdev;
 		struct lag_func *pf;
+		u32 dev_pci_bw;
 		u32 speed;
 		u8 state;
 		int ret;
@@ -1477,9 +1523,20 @@ static int mlx5_lag_get_devices_oper_speed(struct mlx5_lag *ldev,
 			return ret;
 		}
 
+		ret = mlx5_lag_port_pcie_bw(ldev, pf, &dev_pci_bw);
+		if (ret) {
+			mlx5_core_dbg(pf_mdev,
+				      "Failed to get PCIe bandwidth (err=%d)\n",
+				      ret);
+			return ret;
+		}
+
+		pci_bw = min(pci_bw, dev_pci_bw);
 		*sum_speed += speed;
 	}
 
+	*sum_speed = min(*sum_speed, pci_bw);
+
 	return 0;
 }
 
@@ -1510,6 +1567,7 @@ void mlx5_lag_reset_agg_speed(struct mlx5_lag *ldev)
 #ifdef CONFIG_MLX5_ESWITCH
 static int mlx5_lag_get_devices_max_speed(struct mlx5_lag *ldev, u32 *max_speed)
 {
+	u32 pci_bw = U32_MAX;
 	bool take_max;
 	int pf_idx;
 
@@ -1521,6 +1579,7 @@ static int mlx5_lag_get_devices_max_speed(struct mlx5_lag *ldev, u32 *max_speed)
 	mlx5_ldev_for_each(pf_idx, 0, ldev) {
 		struct mlx5_core_dev *pf_mdev;
 		struct lag_func *pf;
+		u32 dev_pci_bw;
 		u32 speed;
 		int ret;
 
@@ -1539,10 +1598,26 @@ static int mlx5_lag_get_devices_max_speed(struct mlx5_lag *ldev, u32 *max_speed)
 			return ret;
 		}
 
-		*max_speed = take_max ?
-			max(*max_speed, speed) : *max_speed + speed;
+		ret = mlx5_lag_port_pcie_bw(ldev, pf, &dev_pci_bw);
+		if (ret) {
+			mlx5_core_dbg(pf_mdev,
+				      "Failed to get PCIe bandwidth (err=%d)\n",
+				      ret);
+			return ret;
+		}
+
+		if (take_max) {
+			speed = min(speed, dev_pci_bw);
+			*max_speed = max(*max_speed, speed);
+		} else {
+			pci_bw = min(pci_bw, dev_pci_bw);
+			*max_speed += speed;
+		}
 	}
 
+	if (!take_max)
+		*max_speed = min(*max_speed, pci_bw);
+
 	return 0;
 }
 
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index 8cc6a89130b5..064eef237d6f 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -6006,6 +6006,22 @@ int pcie_link_speed_mbps(struct pci_dev *pdev)
 }
 EXPORT_SYMBOL(pcie_link_speed_mbps);
 
+u32 pcie_bandwidth_mbps(struct pci_dev *pdev)
+{
+	u16 lnksta;
+	u32 width;
+	int speed;
+
+	if (pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta))
+		return 0;
+	speed = pcie_dev_speed_mbps(to_pcie_link_speed(lnksta));
+	if (speed <= 0)
+		return 0;
+	width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta);
+	return speed * width;
+}
+EXPORT_SYMBOL_GPL(pcie_bandwidth_mbps);
+
 /**
  * pcie_bandwidth_available - determine minimum link settings of a PCIe
  *			      device and its bandwidth limitation
diff --git a/include/linux/pci.h b/include/linux/pci.h
index 5ffcf5ce8c96..202bc8680601 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -1479,6 +1479,8 @@ u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev,
 			     enum pci_bus_speed *speed,
 			     enum pcie_link_width *width);
 int pcie_link_speed_mbps(struct pci_dev *pdev);
+u32 pcie_bandwidth_mbps(struct pci_dev *pdev);
+
 void pcie_print_link_status(struct pci_dev *dev);
 int pcie_reset_flr(struct pci_dev *dev, bool probe);
 int pcie_flr(struct pci_dev *dev);
-- 
2.44.0