From: Konstantin Taranov <kotaranov@linux.microsoft.com>
To: kotaranov@microsoft.com, snsanghvi@microsoft.com,
longli@microsoft.com, jgg@ziepe.ca, leon@kernel.org
Cc: linux-rdma@vger.kernel.org, linux-kernel@vger.kernel.org
Subject: [PATCH rdma-next 2/5] RDMA/mana_ib: Revise UD send posting and WQE definitions
Date: Wed, 16 Sep 2026 06:48:32 -0700 [thread overview]
Message-ID: <20260916134835.2380971-3-kotaranov@linux.microsoft.com> (raw)
In-Reply-To: <20260916134835.2380971-1-kotaranov@linux.microsoft.com>
From: Konstantin Taranov <kotaranov@microsoft.com>
Batch a linked list of UD or GSI send requests behind one doorbell,
returning the first failed WR while publishing all successful entries.
Move the send PSN into common QP state and describe the extended RDMA
send and receive OOB layouts needed by subsequent RC support.
Keep UD's inline OOB explicitly at INLINE_OOB_LARGE_SIZE even though
the shared software OOB structure now describes larger formats. UD
therefore continues using the existing GDMA encoder; extended GDMA
WQE encoding and RC feature gating remain in their later patches.
Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
---
drivers/infiniband/hw/mana/mana_ib.h | 40 ++++++++++++++++++-
drivers/infiniband/hw/mana/wr.c | 58 ++++++++++++++++------------
2 files changed, 72 insertions(+), 26 deletions(-)
diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
index 4da110b30..619578f16 100644
--- a/drivers/infiniband/hw/mana/mana_ib.h
+++ b/drivers/infiniband/hw/mana/mana_ib.h
@@ -206,7 +206,6 @@ enum mana_ud_queue_type {
struct mana_ib_ud_qp {
struct mana_ib_queue queues[MANA_UD_QUEUE_TYPE_MAX];
- u32 sq_psn;
};
struct mana_ib_qp {
@@ -222,6 +221,7 @@ struct mana_ib_qp {
/* The port on the IB device, starting with 1 */
u32 port;
+ u32 sq_psn;
struct list_head cq_send_list;
struct list_head cq_recv_list;
@@ -525,7 +525,16 @@ struct mana_rnic_set_qp_state_resp {
enum WQE_OPCODE_TYPES {
WQE_TYPE_UD_SEND = 0,
+ WQE_TYPE_RC_SEND = 2,
+ WQE_TYPE_RC_SEND_IMM = 3,
+ WQE_TYPE_RC_SEND_INV = 4,
+ WQE_TYPE_WRITE = 5,
+ WQE_TYPE_WRITE_IMM = 6,
+ WQE_TYPE_READ = 7,
WQE_TYPE_UD_RECV = 8,
+ WQE_TYPE_RC_RECV = 9,
+ WQE_TYPE_REG_MR = 10,
+ WQE_TYPE_LOCAL_INV = 12,
}; /* HW DATA */
struct rdma_send_oob {
@@ -544,7 +553,36 @@ struct rdma_send_oob {
u32 reserved1;
u32 reserved2;
} ud_send;
+ union {
+ u32 immediate;
+ u32 invalidate_key;
+ } rc_send;
+ struct {
+ u32 address_hi;
+ u32 address_low;
+ u32 rkey;
+ u32 dma_len;
+ } rdma;
+ struct {
+ u32 mkey;
+ } mm;
};
+ union {
+ u32 immediate_ext;
+ struct {
+ u16 rsn;
+ u16 reserved;
+ } read;
+ };
+ u32 fsn : 24;
+ u32 reserved2 : 8;
+}; /* HW DATA */
+
+struct rdma_recv_oob {
+ u32 psn_start : 24;
+ u32 reserved1 : 8;
+ u32 msn : 24;
+ u32 reserved2 : 8;
}; /* HW DATA */
struct mana_rdma_cqe {
diff --git a/drivers/infiniband/hw/mana/wr.c b/drivers/infiniband/hw/mana/wr.c
index 8cd2980d4..f25df2fad 100644
--- a/drivers/infiniband/hw/mana/wr.c
+++ b/drivers/infiniband/hw/mana/wr.c
@@ -71,7 +71,7 @@ int mana_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
return err;
}
-static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr)
+static int mana_ib_post_send_ud_one(struct mana_ib_qp *qp, const struct ib_ud_wr *wr)
{
struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
struct mana_ib_ah *ah = container_of(wr->ah, struct mana_ib_ah, ibah);
@@ -84,11 +84,8 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr
struct shadow_wqe_header *shadow_wqe;
int err, i;
- if (!ndev) {
- ibdev_dbg(&mdev->ib_dev, "Invalid port %u in QP %u\n",
- qp->port, qp->ibqp.qp_num);
+ if (!ndev)
return -EINVAL;
- }
if (wr->wr.opcode != IB_WR_SEND)
return -EINVAL;
@@ -110,7 +107,7 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr
wqe_req.num_sge = wr->wr.num_sge + 1;
wqe_req.sgl = gdma_sgl;
- wqe_req.inline_oob_size = sizeof(struct rdma_send_oob);
+ wqe_req.inline_oob_size = INLINE_OOB_LARGE_SIZE;
wqe_req.inline_oob_data = &send_oob;
wqe_req.flags = GDMA_WR_OOB_IN_SGL;
wqe_req.client_data_unit = ib_mtu_enum_to_int(ib_mtu_int_to_enum(ndev->mtu));
@@ -119,7 +116,7 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr
send_oob.fence = !!(wr->wr.send_flags & IB_SEND_FENCE);
send_oob.signaled = !!(wr->wr.send_flags & IB_SEND_SIGNALED);
send_oob.solicited = !!(wr->wr.send_flags & IB_SEND_SOLICITED);
- send_oob.psn = qp->ud_qp.sq_psn;
+ send_oob.psn = qp->sq_psn;
send_oob.ssn_or_rqpn = wr->remote_qpn;
send_oob.ud_send.remote_qkey =
qp->ibqp.qp_type == IB_QPT_GSI ? IB_QP1_QKEY : wr->remote_qkey;
@@ -128,40 +125,51 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr
if (err)
return err;
- qp->ud_qp.sq_psn++;
+ qp->sq_psn++;
shadow_wqe = shadow_queue_producer_entry(&qp->shadow_sq);
memset(shadow_wqe, 0, sizeof(*shadow_wqe));
- shadow_wqe->send_opcode = IB_WC_SEND;
shadow_wqe->wr_id = wr->wr.wr_id;
shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu;
shadow_queue_advance_producer(&qp->shadow_sq);
- mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue);
return 0;
}
-int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
- const struct ib_send_wr **bad_wr)
+static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_send_wr *wr,
+ const struct ib_send_wr **bad_wr)
{
+ struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
+ struct gdma_queue *queue = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem;
+ bool ring_sq = false;
int err = 0;
- struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
for (; wr; wr = wr->next) {
- switch (ibqp->qp_type) {
- case IB_QPT_UD:
- case IB_QPT_GSI:
- err = mana_ib_post_send_ud(qp, ud_wr(wr));
- if (unlikely(err)) {
- *bad_wr = wr;
- return err;
- }
+ err = mana_ib_post_send_ud_one(qp, ud_wr(wr));
+ if (unlikely(err)) {
+ *bad_wr = wr;
break;
- default:
- ibdev_dbg(ibqp->device, "Posting send wr on qp type %u is not supported\n",
- ibqp->qp_type);
- return -EINVAL;
}
+ ring_sq = true;
}
+ if (ring_sq)
+ mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue);
+
return err;
}
+
+int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
+ const struct ib_send_wr **bad_wr)
+{
+ struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
+
+ switch (ibqp->qp_type) {
+ case IB_QPT_UD:
+ case IB_QPT_GSI:
+ return mana_ib_post_send_ud(qp, wr, bad_wr);
+ default:
+ /* Unsupported QP type */
+ *bad_wr = wr;
+ return -EINVAL;
+ }
+}
--
2.43.0
next prev parent reply other threads:[~2026-09-16 13:49 UTC|newest]
Thread overview: 6+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-09-16 13:48 [PATCH rdma-next 0/5] RDMA/mana_ib: Streamline kernel UD/GSI posting and completion handling Konstantin Taranov
2026-09-16 13:48 ` [PATCH rdma-next 1/5] RDMA/mana_ib: Optimize shadow queue bookkeeping Konstantin Taranov
2026-09-16 13:48 ` Konstantin Taranov [this message]
2026-09-16 13:48 ` [PATCH rdma-next 3/5] RDMA/mana_ib: Revise UD receive posting with GDMA_WR_IB_SGL Konstantin Taranov
2026-09-16 13:48 ` [PATCH rdma-next 4/5] RDMA/mana_ib: Make kernel CQ arming robust Konstantin Taranov
2026-09-16 13:48 ` [PATCH rdma-next 5/5] RDMA/mana_ib: Poll UD completions and flush software error QPs Konstantin Taranov
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260916134835.2380971-3-kotaranov@linux.microsoft.com \
--to=kotaranov@linux.microsoft.com \
--cc=jgg@ziepe.ca \
--cc=kotaranov@microsoft.com \
--cc=leon@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-rdma@vger.kernel.org \
--cc=longli@microsoft.com \
--cc=snsanghvi@microsoft.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®