From: Joe Damato <joe@dama.to>
To: netdev@vger.kernel.org, Michael Chan <michael.chan@broadcom.com>,
Pavan Chebbi <pavan.chebbi@broadcom.com>,
Andrew Lunn <andrew+netdev@lunn.ch>,
"David S. Miller" <davem@davemloft.net>,
Eric Dumazet <edumazet@kernel.org>,
Jakub Kicinski <kuba@kernel.org>, Paolo Abeni <pabeni@redhat.com>,
Prashant Sreedharan <prashant@broadcom.com>
Cc: edumazet@google.com, horms@kernel.org,
selvin.xavier@broadcom.com, kalesh-anakkur.purayil@broadcom.com,
linux-kernel@vger.kernel.org, Joe Damato <joe@dama.to>
Subject: [PATCH net v6 3/5] bnxt_en: stop DMA before releasing rings the firmware did not free
Date: Fri, 20 Nov 2026 07:28:25 -0800 [thread overview]
Message-ID: <20261120152830.699712-4-joe@dama.to> (raw)
In-Reply-To: <20261120152830.699712-1-joe@dama.to>
When HWRM_RING_FREE is not answered, bnxt_hwrm_ring_free() clears
fw_ring_id and __bnxt_close_nic() goes on to call bnxt_free_mem(), which
unmaps the ring memory and the RX buffers that the FW may still be
using.
This is reachable in production. On a BCM57504 the first sign is the TX
watchdog; the close that follows times out a subset of its RING_FREEs and
the driver releases those rings anyway:
05:30:12 NETDEV WATCHDOG: transmit queue 0 timed out 6073 ms
05:30:12 Resp cmpl intr err msg: 0x51 x20
05:30:12 hwrm_ring_free type 1 failed x12
05:30:12 hwrm_ring_free type 2 failed x8
05:30:12 AMD-Vi: IO_PAGE_FAULT x3
Count the TX, RX and AGG rings whose RING_FREE got no response and
report that to the caller, which stops DMA before the ring memory is
released. This is skipped during a firmware reset or after a fatal
firmware error, where the reset path already handles the device.
The device stays unusable until the driver is rebound.
Fixes: 74608fc98d28 ("bnxt_en: Ring free response from close path should use completion ring")
Signed-off-by: Joe Damato <joe@dama.to>
---
drivers/net/ethernet/broadcom/bnxt/bnxt.c | 61 ++++++++++++++++++-----
1 file changed, 48 insertions(+), 13 deletions(-)
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index c3413354e758..600a8c866f3c 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -7774,21 +7774,31 @@ static void bnxt_clear_one_cp_ring(struct bnxt *bp, struct bnxt_cp_ring_info *cp
memset(cpr->cp_desc_ring[i], 0, size);
}
-static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
+static int bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
{
+ int stuck = 0;
u32 type;
int i;
if (!bp->bnapi)
- return;
+ return 0;
+ /* -EBUSY means the firmware never answered, so it may still own
+ * the ring.
+ */
for (i = 0; i < bp->tx_nr_rings; i++)
- bnxt_hwrm_tx_ring_free(bp, &bp->tx_ring[i], close_path);
+ if (bnxt_hwrm_tx_ring_free(bp, &bp->tx_ring[i],
+ close_path) == -EBUSY)
+ stuck++;
bnxt_cancel_dim(bp);
for (i = 0; i < bp->rx_nr_rings; i++) {
- bnxt_hwrm_rx_ring_free(bp, &bp->rx_ring[i], close_path);
- bnxt_hwrm_rx_agg_ring_free(bp, &bp->rx_ring[i], close_path);
+ struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
+
+ if (bnxt_hwrm_rx_ring_free(bp, rxr, close_path) == -EBUSY)
+ stuck++;
+ if (bnxt_hwrm_rx_agg_ring_free(bp, rxr, close_path) == -EBUSY)
+ stuck++;
}
/* The completion rings are about to be freed. After that the
@@ -7818,6 +7828,26 @@ static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
}
}
+
+ if (!stuck)
+ return 0;
+
+ netdev_err(bp->dev, "Firmware did not free %d ring(s)\n", stuck);
+ return -EIO;
+}
+
+static void bnxt_stop_dma(struct bnxt *bp)
+{
+ /* A firmware reset resets the device, and the fatal error path
+ * already disables it.
+ */
+ if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) ||
+ test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
+ return;
+
+ netdev_err(bp->dev,
+ "Disabling DMA before releasing ring memory, the driver must be rebound to recover\n");
+ pci_clear_master(bp->pdev);
}
static int __bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
@@ -10852,16 +10882,19 @@ static void bnxt_clear_vnic(struct bnxt *bp)
bnxt_hwrm_vnic_ctx_free(bp);
}
-static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
- bool irq_re_init)
+static int bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
+ bool irq_re_init)
{
+ int rc;
+
bnxt_clear_vnic(bp);
- bnxt_hwrm_ring_free(bp, close_path);
+ rc = bnxt_hwrm_ring_free(bp, close_path);
bnxt_hwrm_ring_grp_free(bp);
if (irq_re_init) {
bnxt_hwrm_stat_ctx_free(bp);
bnxt_hwrm_free_tunnel_ports(bp);
}
+ return rc;
}
static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
@@ -11383,15 +11416,15 @@ static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
return 0;
err_out:
- bnxt_hwrm_resource_free(bp, 0, true);
+ if (bnxt_hwrm_resource_free(bp, 0, true))
+ bnxt_stop_dma(bp);
return rc;
}
static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
{
- bnxt_hwrm_resource_free(bp, 1, irq_re_init);
- return 0;
+ return bnxt_hwrm_resource_free(bp, 1, irq_re_init);
}
static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
@@ -13462,7 +13495,8 @@ int bnxt_half_open_nic(struct bnxt *bp)
*/
void bnxt_half_close_nic(struct bnxt *bp)
{
- bnxt_hwrm_resource_free(bp, false, true);
+ if (bnxt_hwrm_resource_free(bp, false, true))
+ bnxt_stop_dma(bp);
bnxt_del_napi(bp);
bnxt_free_skbs(bp);
bnxt_free_mem(bp, true);
@@ -13541,7 +13575,8 @@ static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
if (BNXT_SUPPORTS_MULTI_RSS_CTX(bp))
bnxt_clear_rss_ctxs(bp);
/* Flush rings and disable interrupts */
- bnxt_shutdown_nic(bp, irq_re_init);
+ if (bnxt_shutdown_nic(bp, irq_re_init))
+ bnxt_stop_dma(bp);
/* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
--
2.53.1-Meta
next prev parent reply other threads:[~2026-11-20 15:28 UTC|newest]
Thread overview: 6+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-20 15:28 [PATCH net v6 0/5] bnxt_en: Make RING FREE more robust Joe Damato
2026-11-20 15:28 ` [PATCH net v6 1/5] bnxt_en: return the RING_FREE status to callers Joe Damato
2026-11-20 15:28 ` [PATCH net v6 2/5] bnxt_en: check HWRM response if completion never arrives Joe Damato
2026-11-20 15:28 ` Joe Damato [this message]
2026-11-20 15:28 ` [PATCH net v6 4/5] bnxt_en: refuse to open a device with stopped DMA Joe Damato
2026-11-20 15:28 ` [PATCH net v6 5/5] bnxt_en: stop the ULP around the reset task Joe Damato
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261120152830.699712-4-joe@dama.to \
--to=joe@dama.to \
--cc=andrew+netdev@lunn.ch \
--cc=davem@davemloft.net \
--cc=edumazet@google.com \
--cc=edumazet@kernel.org \
--cc=horms@kernel.org \
--cc=kalesh-anakkur.purayil@broadcom.com \
--cc=kuba@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=michael.chan@broadcom.com \
--cc=netdev@vger.kernel.org \
--cc=pabeni@redhat.com \
--cc=pavan.chebbi@broadcom.com \
--cc=prashant@broadcom.com \
--cc=selvin.xavier@broadcom.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®