[net-next v3 3/3] bnge: Reserve ring resources for upcoming RoCE MPC support

[Siva Reddy Kallam <siva.kallam@broadcom.com>]

From: Sachin Holla <sachin.holla@broadcom.com>

Reserve one additional TX ring and one additional completion ring
per RoCE-enabled function for forthcoming RoCE MPC support.

Reduce the L2 ring limits before adding MPC resources to the firmware
reservation demand. Exclude MPC resources from firmware grants before
adjusting L2 counts, allowing partial grants that leave sufficient
resources for L2 and MPC.

The corresponding RoCE MPC allocation and use will follow in a
separate RDMA patch series. Until then, these reservations remain
unused and reduce the resources available to L2.

Disable RoCE before selecting default resources when the reported ring
limits cannot accommodate MPC and minimum L2 operation, preserving
the L2 ring budget on resource-constrained functions.

Rebuild the default RSS indirection table when the effective L2 RX
queue count changes, even if the firmware RX grant remains unchanged.
Initialize the table after successful default-ring setup so a failed
reservation followed by a successful retry cannot skip initialization.

Signed-off-by: Sachin Holla <sachin.holla@broadcom.com>
Signed-off-by: Siva Reddy Kallam <siva.kallam@broadcom.com>
Signed-off-by: Saswati Das <saswati.das@broadcom.com>
Reviewed-by: Dharmender Garg <dharmender.garg@broadcom.com>
---
 .../net/ethernet/broadcom/bnge/bnge_core.c    |  9 +++
 .../ethernet/broadcom/bnge/bnge_hwrm_lib.c    |  3 +-
 .../net/ethernet/broadcom/bnge/bnge_resc.c    | 56 +++++++++++++++++--
 3 files changed, 61 insertions(+), 7 deletions(-)

diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_core.c b/drivers/net/ethernet/broadcom/bnge/bnge_core.c
index 68b74eb2c3a2..c5d51b8ee9d1 100644
--- a/drivers/net/ethernet/broadcom/bnge/bnge_core.c
+++ b/drivers/net/ethernet/broadcom/bnge/bnge_core.c
@@ -312,6 +312,15 @@ static int bnge_probe_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	max_irqs = bnge_get_max_irq(pdev);
 	bnge_set_max_func_irqs(bd, max_irqs);
 
+	/* Preserve L2 operation if the additional MPC rings cannot fit. */
+	if (bnge_is_roce_en(bd) &&
+	    (bd->hw_resc.max_tx_rings <= 1 ||
+	     bd->hw_resc.max_cp_rings <= 2)) {
+		dev_warn(bd->dev,
+			 "Not enough rings for RoCE MPC, disabling RoCE\n");
+		bd->flags &= ~BNGE_EN_ROCE;
+	}
+
 	bnge_aux_init_dflt_config(bd);
 
 	rc = bnge_net_init_dflt_config(bd);
diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c b/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c
index 651c5e783516..01547311add8 100644
--- a/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c
+++ b/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c
@@ -441,7 +441,8 @@ static int bnge_hwrm_get_rings(struct bnge_dev *bd)
 	tx = hw_resc->resv_tx_rings;
 	if (bnge_is_agg_reqd(bd))
 		rx >>= 1;
-	if (cp < (rx + tx)) {
+	/* RoCE ring counts are adjusted after excluding MPC resources. */
+	if (!bnge_is_roce_en(bd) && cp < (rx + tx)) {
 		rc = bnge_fix_rings_count(&rx, &tx, cp, false);
 		if (rc)
 			goto get_rings_exit;
diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_resc.c b/drivers/net/ethernet/broadcom/bnge/bnge_resc.c
index 65ef34536e36..90185482da33 100644
--- a/drivers/net/ethernet/broadcom/bnge/bnge_resc.c
+++ b/drivers/net/ethernet/broadcom/bnge/bnge_resc.c
@@ -110,9 +110,24 @@ static u16 bnge_nqs_demand(struct bnge_dev *bd)
 	return bd->nq_nr_rings + bnge_aux_get_msix(bd);
 }
 
+static u16 bnge_tx_rings_demand(struct bnge_dev *bd)
+{
+	u16 tx_rings = bd->tx_nr_rings;
+
+	if (bnge_is_roce_en(bd))
+		tx_rings += 1; /* For MPC TX ring */
+
+	return tx_rings;
+}
+
 static u16 bnge_cprs_demand(struct bnge_dev *bd)
 {
-	return bd->tx_nr_rings + bd->rx_nr_rings;
+	u16 cprs = bd->tx_nr_rings + bd->rx_nr_rings;
+
+	if (bnge_is_roce_en(bd))
+		cprs += 1; /* For MPC CQ ring */
+
+	return cprs;
 }
 
 static u16 bnge_get_avail_msix(struct bnge_dev *bd, int num)
@@ -251,7 +266,7 @@ static bool bnge_need_reserve_rings(struct bnge_dev *bd)
 	u16 nqs = bnge_nqs_demand(bd);
 	u16 vnic;
 
-	if (hw_resc->resv_tx_rings != bd->tx_nr_rings)
+	if (hw_resc->resv_tx_rings != bnge_tx_rings_demand(bd))
 		return true;
 
 	vnic = bnge_get_total_vnics(bd, rx);
@@ -271,8 +286,8 @@ static bool bnge_need_reserve_rings(struct bnge_dev *bd)
 int bnge_reserve_rings(struct bnge_dev *bd)
 {
 	u16 aux_dflt_msix = bnge_aux_get_dflt_msix(bd);
+	u16 rx_rings, old_rx_rings, old_rx_nr_rings;
 	struct bnge_hw_rings hwr = {0};
-	u16 rx_rings, old_rx_rings;
 	u16 nq = bd->nq_nr_rings;
 	u16 aux_msix = 0;
 	bool sh = false;
@@ -294,11 +309,11 @@ int bnge_reserve_rings(struct bnge_dev *bd)
 		hwr.nq = bnge_nqs_demand(bd);
 	}
 
-	hwr.tx = bd->tx_nr_rings;
+	hwr.tx = bnge_tx_rings_demand(bd);
 	hwr.rx = bd->rx_nr_rings;
 	if (bd->flags & BNGE_EN_SHARED_CHNL)
 		sh = true;
-	hwr.cmpl = hwr.rx + hwr.tx;
+	hwr.cmpl = bnge_cprs_demand(bd);
 
 	hwr.vnic = bnge_get_total_vnics(bd, hwr.rx);
 
@@ -308,6 +323,7 @@ int bnge_reserve_rings(struct bnge_dev *bd)
 	hwr.rss_ctx = bnge_rss_ctxs_in_use(bd, &hwr);
 	hwr.stat = bnge_func_stat_ctxs_demand(bd);
 	old_rx_rings = bd->hw_resc.resv_rx_rings;
+	old_rx_nr_rings = bd->rx_nr_rings;
 
 	rc = bnge_hwrm_reserve_rings(bd, &hwr);
 	if (rc)
@@ -315,6 +331,14 @@ int bnge_reserve_rings(struct bnge_dev *bd)
 
 	bnge_copy_reserved_rings(bd, &hwr);
 
+	/* Exclude the MPC reservation before adjusting the L2 rings. */
+	if (bnge_is_roce_en(bd)) {
+		if (hwr.tx <= 1 || hwr.cmpl <= 2)
+			return -ENOMEM;
+		hwr.tx--;
+		hwr.cmpl--;
+	}
+
 	rx_rings = hwr.rx;
 	if (bnge_is_agg_reqd(bd)) {
 		if (hwr.rx >= 2)
@@ -330,12 +354,19 @@ int bnge_reserve_rings(struct bnge_dev *bd)
 	hwr.nq = min_t(u16, hwr.nq, hwr.stat);
 
 	/* Adjust the rings */
+	rc = bnge_fix_rings_count(&rx_rings, &hwr.tx, hwr.cmpl, false);
+	if (rc)
+		return rc;
 	rc = bnge_adjust_rings(bd, &rx_rings, &hwr.tx, hwr.nq, sh);
+	if (rc)
+		return rc;
 	if (bnge_is_agg_reqd(bd))
 		hwr.rx = rx_rings << 1;
 	tx_cp = hwr.tx;
 	hwr.nq = sh ? max_t(u16, tx_cp, rx_rings) : tx_cp + rx_rings;
 	bd->tx_nr_rings = hwr.tx;
+	if (!bd->num_tc)
+		bd->tx_nr_rings_per_tc = bd->tx_nr_rings;
 
 	if (rx_rings != bd->rx_nr_rings)
 		dev_warn(bd->dev, "RX rings resv reduced to %d than earlier %d requested\n",
@@ -347,7 +378,8 @@ int bnge_reserve_rings(struct bnge_dev *bd)
 	if (!bnge_rings_ok(&hwr))
 		return -ENOMEM;
 
-	if (old_rx_rings != bd->hw_resc.resv_rx_rings)
+	if (old_rx_rings != bd->hw_resc.resv_rx_rings ||
+	    old_rx_nr_rings != bd->rx_nr_rings)
 		bnge_set_dflt_rss_indir_tbl(bd);
 
 	if (!bnge_aux_registered(bd)) {
@@ -452,6 +484,17 @@ static void _bnge_get_max_rings(struct bnge_dev *bd, u16 *max_rx,
 
 	max_cp = bnge_get_max_func_cp_rings(bd);
 
+	/* Leave one TX ring and one completion ring for RoCE MPC. */
+	if (bnge_is_roce_en(bd)) {
+		if (*max_tx <= 1 || max_cp <= 2) {
+			*max_rx = 0;
+			*max_tx = 0;
+			return;
+		}
+		(*max_tx)--;
+		max_cp--;
+	}
+
 	/* Fix RX and TX rings according to number of CPs available */
 	rc = bnge_fix_rings_count(max_rx, max_tx, max_cp, false);
 	if (rc) {
@@ -595,6 +638,7 @@ int bnge_net_init_dflt_config(struct bnge_dev *bd)
 	if (rc)
 		goto err_free_tbl;
 
+	bnge_set_dflt_rss_indir_tbl(bd);
 	hw_resc = &bd->hw_resc;
 	bd->max_fltr = hw_resc->max_rx_em_flows + hw_resc->max_rx_wm_flows +
 		       BNGE_L2_FLTR_MAX_FLTR;
-- 
2.43.5