From: Konstantin Taranov <kotaranov@linux.microsoft.com>
To: kotaranov@microsoft.com, snsanghvi@microsoft.com,
longli@microsoft.com, jgg@ziepe.ca, leon@kernel.org
Cc: linux-rdma@vger.kernel.org, linux-kernel@vger.kernel.org
Subject: [PATCH rdma-next v4 1/5] RDMA/mana_ib: Optimize shadow queue bookkeeping
Date: Wed, 23 Sep 2026 06:11:51 -0700 [thread overview]
Message-ID: <20260923131155.4055875-2-kotaranov@linux.microsoft.com> (raw)
In-Reply-To: <20260923131155.4055875-1-kotaranov@linux.microsoft.com>
From: Konstantin Taranov <kotaranov@microsoft.com>
Pack the posted WQE size and send opcode into compact bitfields. Move the
UD receive shadow fields into the common shadow_wqe_header. This
simplifies the code and lets all QP and WQ types share one structure.
Use release and acquire operations so posting and polling can safely run
on different CPUs.
Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
---
drivers/infiniband/hw/mana/cq.c | 35 ++++++++++++-----------
drivers/infiniband/hw/mana/qp.c | 4 +--
drivers/infiniband/hw/mana/shadow_queue.h | 29 +++++++++----------
drivers/infiniband/hw/mana/wr.c | 15 +++++-----
4 files changed, 40 insertions(+), 43 deletions(-)
diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c
index 6f9ac8b4a..14a48409e 100644
--- a/drivers/infiniband/hw/mana/cq.c
+++ b/drivers/infiniband/hw/mana/cq.c
@@ -190,15 +190,15 @@ static inline void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe
{
struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem;
- struct ud_sq_shadow_wqe *shadow_wqe;
+ struct shadow_wqe_header *shadow_wqe;
shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq);
if (!shadow_wqe)
return;
- shadow_wqe->header.error_code = rdma_cqe->ud_send.vendor_error;
+ shadow_wqe->error_code = rdma_cqe->ud_send.vendor_error;
- wq->tail += shadow_wqe->header.posted_wqe_size;
+ wq->tail += shadow_wqe->wqe_size_in_bu;
shadow_queue_advance_next_to_complete(&qp->shadow_sq);
}
@@ -206,7 +206,7 @@ static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe
{
struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem;
- struct ud_rq_shadow_wqe *shadow_wqe;
+ struct shadow_wqe_header *shadow_wqe;
shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_rq);
if (!shadow_wqe)
@@ -214,9 +214,9 @@ static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe
shadow_wqe->byte_len = rdma_cqe->ud_recv.msg_len;
shadow_wqe->src_qpn = rdma_cqe->ud_recv.src_qpn;
- shadow_wqe->header.error_code = IB_WC_SUCCESS;
+ shadow_wqe->error_code = IB_WC_SUCCESS;
- wq->tail += shadow_wqe->header.posted_wqe_size;
+ wq->tail += shadow_wqe->wqe_size_in_bu;
shadow_queue_advance_next_to_complete(&qp->shadow_rq);
}
@@ -238,21 +238,20 @@ static void mana_handle_cqe(struct mana_ib_dev *mdev, struct gdma_comp *cqe)
}
static void fill_verbs_from_shadow_wqe(struct mana_ib_qp *qp, struct ib_wc *wc,
- const struct shadow_wqe_header *shadow_wqe)
+ const struct shadow_wqe_header *shadow_wqe,
+ enum ib_wc_opcode opcode)
{
- const struct ud_rq_shadow_wqe *ud_wqe = (const struct ud_rq_shadow_wqe *)shadow_wqe;
-
wc->wr_id = shadow_wqe->wr_id;
wc->status = shadow_wqe->error_code;
- wc->opcode = shadow_wqe->opcode;
+ wc->opcode = opcode;
wc->vendor_err = shadow_wqe->error_code;
wc->wc_flags = 0;
wc->qp = &qp->ibqp;
wc->pkey_index = 0;
- if (shadow_wqe->opcode == IB_WC_RECV) {
- wc->byte_len = ud_wqe->byte_len;
- wc->src_qp = ud_wqe->src_qpn;
+ if (opcode == IB_WC_RECV) {
+ wc->byte_len = shadow_wqe->byte_len;
+ wc->src_qp = shadow_wqe->src_qpn;
wc->wc_flags |= IB_WC_GRH;
}
}
@@ -270,7 +269,8 @@ static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc
if (wc_index >= nwc)
goto out;
- fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe);
+ fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe,
+ shadow_wqe->send_opcode);
shadow_queue_advance_consumer(&qp->shadow_sq);
wc_index++;
}
@@ -283,7 +283,8 @@ static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc
if (wc_index >= nwc)
goto out;
- fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe);
+ fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe,
+ IB_WC_RECV);
shadow_queue_advance_consumer(&qp->shadow_rq);
wc_index++;
}
@@ -296,13 +297,13 @@ static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc
static void mana_drain_gsi_sq(struct mana_ib_qp *qp)
{
struct mana_ib_cq *cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
- struct ud_sq_shadow_wqe *shadow_wqe;
+ struct shadow_wqe_header *shadow_wqe;
unsigned long flags;
spin_lock_irqsave(&cq->cq_lock, flags);
while ((shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq))
!= NULL) {
- shadow_wqe->header.error_code = IB_WC_GENERAL_ERR;
+ shadow_wqe->error_code = IB_WC_GENERAL_ERR;
shadow_queue_advance_next_to_complete(&qp->shadow_sq);
}
spin_unlock_irqrestore(&cq->cq_lock, flags);
diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
index fac43b3a5..79c9cc944 100644
--- a/drivers/infiniband/hw/mana/qp.c
+++ b/drivers/infiniband/hw/mana/qp.c
@@ -748,13 +748,13 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
doorbell = mdev->gdma_dev->doorbell;
err = create_shadow_queue(&qp->shadow_rq, attr->cap.max_recv_wr,
- sizeof(struct ud_rq_shadow_wqe));
+ sizeof(struct shadow_wqe_header));
if (err) {
ibdev_err(&mdev->ib_dev, "Failed to create shadow rq err %d\n", err);
goto destroy_queues;
}
err = create_shadow_queue(&qp->shadow_sq, attr->cap.max_send_wr,
- sizeof(struct ud_sq_shadow_wqe));
+ sizeof(struct shadow_wqe_header));
if (err) {
ibdev_err(&mdev->ib_dev, "Failed to create shadow sq err %d\n", err);
goto destroy_shadow_queues;
diff --git a/drivers/infiniband/hw/mana/shadow_queue.h b/drivers/infiniband/hw/mana/shadow_queue.h
index a4b3818f9..a127edf7a 100644
--- a/drivers/infiniband/hw/mana/shadow_queue.h
+++ b/drivers/infiniband/hw/mana/shadow_queue.h
@@ -7,22 +7,14 @@
#define _MANA_SHADOW_QUEUE_H_
struct shadow_wqe_header {
- u16 opcode;
- u16 error_code;
- u32 posted_wqe_size;
u64 wr_id;
-};
-
-struct ud_rq_shadow_wqe {
- struct shadow_wqe_header header;
+ u64 wqe_size_in_bu : 8;
+ u64 send_opcode : 4;
+ u64 error_code : 16;
u32 byte_len;
u32 src_qpn;
};
-struct ud_sq_shadow_wqe {
- struct shadow_wqe_header header;
-};
-
struct shadow_queue {
/* Unmasked producer index, Incremented on wqe posting */
u64 prod_idx;
@@ -57,12 +49,14 @@ static inline void destroy_shadow_queue(struct shadow_queue *queue)
static inline bool shadow_queue_full(struct shadow_queue *queue)
{
- return (queue->prod_idx - queue->cons_idx) >= queue->length;
+ /* Do not reuse an entry until the poller has finished reading it. */
+ return (queue->prod_idx - smp_load_acquire(&queue->cons_idx)) >= queue->length;
}
static inline bool shadow_queue_empty(struct shadow_queue *queue)
{
- return queue->prod_idx == queue->cons_idx;
+ /* Pair with posting's release of the initialized shadow WQE. */
+ return smp_load_acquire(&queue->prod_idx) == queue->cons_idx;
}
static inline void *
@@ -91,7 +85,8 @@ shadow_queue_get_next_to_consume(const struct shadow_queue *queue)
static inline void *
shadow_queue_get_next_to_complete(struct shadow_queue *queue)
{
- if (queue->next_to_complete_idx == queue->prod_idx)
+ /* Observe initialized WQE fields published by the posting CPU. */
+ if (queue->next_to_complete_idx == smp_load_acquire(&queue->prod_idx))
return NULL;
return shadow_queue_get_element(queue, queue->next_to_complete_idx);
@@ -99,12 +94,14 @@ shadow_queue_get_next_to_complete(struct shadow_queue *queue)
static inline void shadow_queue_advance_producer(struct shadow_queue *queue)
{
- queue->prod_idx++;
+ /* Publish all WQE fields to CQ polling on another CPU. */
+ smp_store_release(&queue->prod_idx, queue->prod_idx + 1);
}
static inline void shadow_queue_advance_consumer(struct shadow_queue *queue)
{
- queue->cons_idx++;
+ /* Finish WC generation and queue-tail updates before allowing reuse. */
+ smp_store_release(&queue->cons_idx, queue->cons_idx + 1);
}
static inline void shadow_queue_advance_next_to_complete(struct shadow_queue *queue)
diff --git a/drivers/infiniband/hw/mana/wr.c b/drivers/infiniband/hw/mana/wr.c
index 36a1d506f..8cd2980d4 100644
--- a/drivers/infiniband/hw/mana/wr.c
+++ b/drivers/infiniband/hw/mana/wr.c
@@ -14,7 +14,7 @@ static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr *
struct gdma_posted_wqe_info wqe_info = {0};
struct gdma_sge gdma_sgl[MAX_WR_SGL_NUM];
struct gdma_wqe_request wqe_req = {0};
- struct ud_rq_shadow_wqe *shadow_wqe;
+ struct shadow_wqe_header *shadow_wqe;
int err, i;
if (shadow_queue_full(&qp->shadow_rq))
@@ -37,9 +37,8 @@ static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr *
shadow_wqe = shadow_queue_producer_entry(&qp->shadow_rq);
memset(shadow_wqe, 0, sizeof(*shadow_wqe));
- shadow_wqe->header.opcode = IB_WC_RECV;
- shadow_wqe->header.wr_id = wr->wr_id;
- shadow_wqe->header.posted_wqe_size = wqe_info.wqe_size_in_bu;
+ shadow_wqe->wr_id = wr->wr_id;
+ shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu;
shadow_queue_advance_producer(&qp->shadow_rq);
mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue);
@@ -82,7 +81,7 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr
struct gdma_posted_wqe_info wqe_info = {0};
struct gdma_wqe_request wqe_req = {0};
struct rdma_send_oob send_oob = {0};
- struct ud_sq_shadow_wqe *shadow_wqe;
+ struct shadow_wqe_header *shadow_wqe;
int err, i;
if (!ndev) {
@@ -132,9 +131,9 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr
qp->ud_qp.sq_psn++;
shadow_wqe = shadow_queue_producer_entry(&qp->shadow_sq);
memset(shadow_wqe, 0, sizeof(*shadow_wqe));
- shadow_wqe->header.opcode = IB_WC_SEND;
- shadow_wqe->header.wr_id = wr->wr.wr_id;
- shadow_wqe->header.posted_wqe_size = wqe_info.wqe_size_in_bu;
+ shadow_wqe->send_opcode = IB_WC_SEND;
+ shadow_wqe->wr_id = wr->wr.wr_id;
+ shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu;
shadow_queue_advance_producer(&qp->shadow_sq);
mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue);
--
2.43.0
next prev parent reply other threads:[~2026-09-23 13:12 UTC|newest]
Thread overview: 6+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-09-23 13:11 [PATCH rdma-next v4 0/5] RDMA/mana_ib: Streamline kernel UD/GSI posting and completion handling Konstantin Taranov
2026-09-23 13:11 ` Konstantin Taranov [this message]
2026-09-23 13:11 ` [PATCH rdma-next v4 2/5] RDMA/mana_ib: Revise UD send posting and WQE definitions Konstantin Taranov
2026-09-23 13:11 ` [PATCH rdma-next v4 3/5] RDMA/mana_ib: Revise UD receive posting with GDMA_WR_IB_SGL Konstantin Taranov
2026-09-23 13:11 ` [PATCH rdma-next v4 4/5] RDMA/mana_ib: Make kernel CQ arming robust Konstantin Taranov
2026-09-23 13:11 ` [PATCH rdma-next v4 5/5] RDMA/mana_ib: Poll UD completions and flush software error QPs Konstantin Taranov
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260923131155.4055875-2-kotaranov@linux.microsoft.com \
--to=kotaranov@linux.microsoft.com \
--cc=jgg@ziepe.ca \
--cc=kotaranov@microsoft.com \
--cc=leon@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-rdma@vger.kernel.org \
--cc=longli@microsoft.com \
--cc=snsanghvi@microsoft.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®