From: Naveen Mamindlapalli <naveenm@marvell.com>
To: <davem@davemloft.net>, <edumazet@google.com>, <kuba@kernel.org>,
<pabeni@redhat.com>, <netdev@vger.kernel.org>,
<linux-kernel@vger.kernel.org>, <sgoutham@marvell.com>
Cc: Naveen Mamindlapalli <naveenm@marvell.com>
Subject: [net-next PATCH v2 6/6] octeontx2-af: Set XOFF on other child transmit schedulers during SMQ flush
Date: Mon, 12 Jun 2023 11:34:24 +0530 [thread overview]
Message-ID: <20230612060424.1427-7-naveenm@marvell.com> (raw)
In-Reply-To: <20230612060424.1427-1-naveenm@marvell.com>
When multiple transmit scheduler queues feed a TL1 transmit link, the
SMQ flush initiated on a low priority queue might get stuck when a high
priority queue fully subscribes the transmit link. This inturn effects
interface teardown. To avoid this, temporarily XOFF all TL1's other
immediate child transmit scheduler queues and also clear any rate limit
configuration on all the scheduler queues in SMQ(flush) hierarchy.
Signed-off-by: Naveen Mamindlapalli <naveenm@marvell.com>
Signed-off-by: Sunil Goutham <sgoutham@marvell.com>
---
.../net/ethernet/marvell/octeontx2/af/rvu.h | 16 +++
.../ethernet/marvell/octeontx2/af/rvu_nix.c | 130 +++++++++++++++++-
2 files changed, 144 insertions(+), 2 deletions(-)
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
index c07d826e36d1..b5a7ee63508c 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
@@ -285,6 +285,22 @@ struct nix_mark_format {
u32 *cfg;
};
+/* smq(flush) to tl1 cir/pir info */
+struct nix_smq_tree_ctx {
+ u64 cir_off;
+ u64 cir_val;
+ u64 pir_off;
+ u64 pir_val;
+};
+
+/* smq flush context */
+struct nix_smq_flush_ctx {
+ int smq;
+ u16 tl1_schq;
+ u16 tl2_schq;
+ struct nix_smq_tree_ctx smq_tree_ctx[NIX_TXSCH_LVL_CNT];
+};
+
struct npc_pkind {
struct rsrc_bmap rsrc;
u32 *pfchan_map;
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
index 23149036be77..601ef269aa29 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
@@ -2114,9 +2114,121 @@ int rvu_mbox_handler_nix_txsch_alloc(struct rvu *rvu,
return rc;
}
+static void nix_smq_flush_fill_ctx(struct rvu *rvu, int blkaddr, int smq,
+ struct nix_smq_flush_ctx *smq_flush_ctx)
+{
+ struct nix_smq_tree_ctx *smq_tree_ctx;
+ u64 parent_off, regval;
+ u16 schq;
+ int lvl;
+
+ smq_flush_ctx->smq = smq;
+
+ schq = smq;
+ for (lvl = NIX_TXSCH_LVL_SMQ; lvl <= NIX_TXSCH_LVL_TL1; lvl++) {
+ smq_tree_ctx = &smq_flush_ctx->smq_tree_ctx[lvl];
+ if (lvl == NIX_TXSCH_LVL_TL1) {
+ smq_flush_ctx->tl1_schq = schq;
+ smq_tree_ctx->cir_off = NIX_AF_TL1X_CIR(schq);
+ smq_tree_ctx->pir_off = 0;
+ smq_tree_ctx->pir_val = 0;
+ parent_off = 0;
+ } else if (lvl == NIX_TXSCH_LVL_TL2) {
+ smq_flush_ctx->tl2_schq = schq;
+ smq_tree_ctx->cir_off = NIX_AF_TL2X_CIR(schq);
+ smq_tree_ctx->pir_off = NIX_AF_TL2X_PIR(schq);
+ parent_off = NIX_AF_TL2X_PARENT(schq);
+ } else if (lvl == NIX_TXSCH_LVL_TL3) {
+ smq_tree_ctx->cir_off = NIX_AF_TL3X_CIR(schq);
+ smq_tree_ctx->pir_off = NIX_AF_TL3X_PIR(schq);
+ parent_off = NIX_AF_TL3X_PARENT(schq);
+ } else if (lvl == NIX_TXSCH_LVL_TL4) {
+ smq_tree_ctx->cir_off = NIX_AF_TL4X_CIR(schq);
+ smq_tree_ctx->pir_off = NIX_AF_TL4X_PIR(schq);
+ parent_off = NIX_AF_TL4X_PARENT(schq);
+ } else if (lvl == NIX_TXSCH_LVL_MDQ) {
+ smq_tree_ctx->cir_off = NIX_AF_MDQX_CIR(schq);
+ smq_tree_ctx->pir_off = NIX_AF_MDQX_PIR(schq);
+ parent_off = NIX_AF_MDQX_PARENT(schq);
+ }
+ /* save cir/pir register values */
+ smq_tree_ctx->cir_val = rvu_read64(rvu, blkaddr, smq_tree_ctx->cir_off);
+ if (smq_tree_ctx->pir_off)
+ smq_tree_ctx->pir_val = rvu_read64(rvu, blkaddr, smq_tree_ctx->pir_off);
+
+ /* get parent txsch node */
+ if (parent_off) {
+ regval = rvu_read64(rvu, blkaddr, parent_off);
+ schq = (regval >> 16) & 0x1FF;
+ }
+ }
+}
+
+static void nix_smq_flush_enadis_xoff(struct rvu *rvu, int blkaddr,
+ struct nix_smq_flush_ctx *smq_flush_ctx, bool enable)
+{
+ struct nix_txsch *txsch;
+ struct nix_hw *nix_hw;
+ u64 regoff;
+ int tl2;
+
+ nix_hw = get_nix_hw(rvu->hw, blkaddr);
+ if (!nix_hw)
+ return;
+
+ /* loop through all TL2s with matching PF_FUNC */
+ txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL2];
+ for (tl2 = 0; tl2 < txsch->schq.max; tl2++) {
+ /* skip the smq(flush) TL2 */
+ if (tl2 == smq_flush_ctx->tl2_schq)
+ continue;
+ /* skip unused TL2s */
+ if (TXSCH_MAP_FLAGS(txsch->pfvf_map[tl2]) & NIX_TXSCHQ_FREE)
+ continue;
+ /* skip if PF_FUNC doesn't match */
+ if ((TXSCH_MAP_FUNC(txsch->pfvf_map[tl2]) & ~RVU_PFVF_FUNC_MASK) !=
+ (TXSCH_MAP_FUNC(txsch->pfvf_map[smq_flush_ctx->tl2_schq] &
+ ~RVU_PFVF_FUNC_MASK)))
+ continue;
+ /* enable/disable XOFF */
+ regoff = NIX_AF_TL2X_SW_XOFF(tl2);
+ if (enable)
+ rvu_write64(rvu, blkaddr, regoff, 0x1);
+ else
+ rvu_write64(rvu, blkaddr, regoff, 0x0);
+ }
+}
+
+static void nix_smq_flush_enadis_rate(struct rvu *rvu, int blkaddr,
+ struct nix_smq_flush_ctx *smq_flush_ctx, bool enable)
+{
+ u64 cir_off, pir_off, cir_val, pir_val;
+ struct nix_smq_tree_ctx *smq_tree_ctx;
+ int lvl;
+
+ for (lvl = NIX_TXSCH_LVL_SMQ; lvl <= NIX_TXSCH_LVL_TL1; lvl++) {
+ smq_tree_ctx = &smq_flush_ctx->smq_tree_ctx[lvl];
+ cir_off = smq_tree_ctx->cir_off;
+ cir_val = smq_tree_ctx->cir_val;
+ pir_off = smq_tree_ctx->pir_off;
+ pir_val = smq_tree_ctx->pir_val;
+
+ if (enable) {
+ rvu_write64(rvu, blkaddr, cir_off, cir_val);
+ if (lvl != NIX_TXSCH_LVL_TL1)
+ rvu_write64(rvu, blkaddr, pir_off, pir_val);
+ } else {
+ rvu_write64(rvu, blkaddr, cir_off, 0x0);
+ if (lvl != NIX_TXSCH_LVL_TL1)
+ rvu_write64(rvu, blkaddr, pir_off, 0x0);
+ }
+ }
+}
+
static int nix_smq_flush(struct rvu *rvu, int blkaddr,
int smq, u16 pcifunc, int nixlf)
{
+ struct nix_smq_flush_ctx *smq_flush_ctx;
int pf = rvu_get_pf(pcifunc);
u8 cgx_id = 0, lmac_id = 0;
int err, restore_tx_en = 0;
@@ -2136,6 +2248,14 @@ static int nix_smq_flush(struct rvu *rvu, int blkaddr,
lmac_id, true);
}
+ /* XOFF all TL2s whose parent TL1 matches SMQ tree TL1 */
+ smq_flush_ctx = kzalloc(sizeof(*smq_flush_ctx), GFP_KERNEL);
+ if (!smq_flush_ctx)
+ return -ENOMEM;
+ nix_smq_flush_fill_ctx(rvu, blkaddr, smq, smq_flush_ctx);
+ nix_smq_flush_enadis_xoff(rvu, blkaddr, smq_flush_ctx, true);
+ nix_smq_flush_enadis_rate(rvu, blkaddr, smq_flush_ctx, false);
+
cfg = rvu_read64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq));
/* Do SMQ flush and set enqueue xoff */
cfg |= BIT_ULL(50) | BIT_ULL(49);
@@ -2150,8 +2270,14 @@ static int nix_smq_flush(struct rvu *rvu, int blkaddr,
err = rvu_poll_reg(rvu, blkaddr,
NIX_AF_SMQX_CFG(smq), BIT_ULL(49), true);
if (err)
- dev_err(rvu->dev,
- "NIXLF%d: SMQ%d flush failed\n", nixlf, smq);
+ dev_info(rvu->dev,
+ "NIXLF%d: SMQ%d flush failed, txlink might be busy\n",
+ nixlf, smq);
+
+ /* clear XOFF on TL2s */
+ nix_smq_flush_enadis_rate(rvu, blkaddr, smq_flush_ctx, true);
+ nix_smq_flush_enadis_xoff(rvu, blkaddr, smq_flush_ctx, false);
+ kfree(smq_flush_ctx);
rvu_cgx_enadis_rx_bp(rvu, pf, true);
/* restore cgx tx state */
--
2.39.0.198.ga38d39a4c5
next prev parent reply other threads:[~2023-06-12 6:05 UTC|newest]
Thread overview: 8+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-06-12 6:04 [net-next PATCH v2 0/6] RVU NIX AF driver updates Naveen Mamindlapalli
2023-06-12 6:04 ` [net-next PATCH v2 1/6] octeontx2-af: Add devlink option to adjust mcam high prio zone entries Naveen Mamindlapalli
2023-06-12 6:04 ` [net-next PATCH v2 2/6] octeontx2-af: extend RSS supported offload types Naveen Mamindlapalli
2023-06-12 6:04 ` [net-next PATCH v2 3/6] octeontx2-af: cn10k: Set NIX DWRR MTU for CN10KB silicon Naveen Mamindlapalli
2023-06-12 6:04 ` [net-next PATCH v2 4/6] octeontx2-af: Enable LBK links only when switch mode is on Naveen Mamindlapalli
2023-06-12 6:04 ` [net-next PATCH v2 5/6] octeontx2-af: add option to toggle DROP_RE enable in rx cfg Naveen Mamindlapalli
2023-06-12 6:04 ` Naveen Mamindlapalli [this message]
2023-06-13 9:00 ` [net-next PATCH v2 0/6] RVU NIX AF driver updates patchwork-bot+netdevbpf
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230612060424.1427-7-naveenm@marvell.com \
--to=naveenm@marvell.com \
--cc=davem@davemloft.net \
--cc=edumazet@google.com \
--cc=kuba@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=netdev@vger.kernel.org \
--cc=pabeni@redhat.com \
--cc=sgoutham@marvell.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®