From: Konstantin Taranov <kotaranov@linux.microsoft.com>
To: kotaranov@microsoft.com, snsanghvi@microsoft.com,
longli@microsoft.com, jgg@ziepe.ca, leon@kernel.org
Cc: linux-rdma@vger.kernel.org, linux-kernel@vger.kernel.org
Subject: [PATCH rdma-next 10/10] RDMA/mana_ib: Drain kernel receive and send queues
Date: Thu, 1 Oct 2026 11:20:15 -0700 [thread overview]
Message-ID: <20261001182015.1757203-11-kotaranov@linux.microsoft.com> (raw)
In-Reply-To: <20261001182015.1757203-1-kotaranov@linux.microsoft.com>
From: Konstantin Taranov <kotaranov@microsoft.com>
Implement the core drain callbacks by moving the QP to ERR, posting a
completion-bearing marker, and waiting until that marker is flushed.
Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
---
drivers/infiniband/hw/mana/device.c | 2 +
drivers/infiniband/hw/mana/mana_ib.h | 2 +
drivers/infiniband/hw/mana/qp.c | 91 ++++++++++++++++++++++++++++
3 files changed, 95 insertions(+)
diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c
index cdaada7f6e8a..07c7fe678ff7 100644
--- a/drivers/infiniband/hw/mana/device.c
+++ b/drivers/infiniband/hw/mana/device.c
@@ -38,6 +38,8 @@ static const struct ib_device_ops mana_ib_dev_ops = {
.destroy_rwq_ind_table = mana_ib_destroy_rwq_ind_table,
.destroy_wq = mana_ib_destroy_wq,
.disassociate_ucontext = mana_ib_disassociate_ucontext,
+ .drain_rq = mana_ib_drain_rq,
+ .drain_sq = mana_ib_drain_sq,
.get_dma_mr = mana_ib_get_dma_mr,
.get_link_layer = mana_ib_get_link_layer,
.get_port_immutable = mana_ib_get_port_immutable,
diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
index 406b85abdef2..6eb06a1036ab 100644
--- a/drivers/infiniband/hw/mana/mana_ib.h
+++ b/drivers/infiniband/hw/mana/mana_ib.h
@@ -1059,6 +1059,8 @@ int mana_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
const struct ib_send_wr **bad_wr);
+void mana_ib_drain_sq(struct ib_qp *ibqp);
+void mana_ib_drain_rq(struct ib_qp *ibqp);
void mana_drain_gsi_sqs(struct mana_ib_dev *mdev);
int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
index ed6dbd66bd36..95374896ea82 100644
--- a/drivers/infiniband/hw/mana/qp.c
+++ b/drivers/infiniband/hw/mana/qp.c
@@ -1331,3 +1331,94 @@ int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
return -ENOENT;
}
+
+struct mana_ib_drain_cqe {
+ struct ib_cqe cqe;
+ struct completion done;
+};
+
+static void mana_ib_drain_qp_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+ struct mana_ib_drain_cqe *cqe = container_of(wc->wr_cqe,
+ struct mana_ib_drain_cqe,
+ cqe);
+ complete(&cqe->done);
+}
+
+void mana_ib_drain_sq(struct ib_qp *ibqp)
+{
+ struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
+ struct mana_ib_drain_cqe sdrain;
+ struct ib_rdma_wr swr = {
+ .wr = {
+ .next = NULL,
+ .wr_cqe = &sdrain.cqe,
+ .opcode = IB_WR_SEND,
+ },
+ };
+ const struct ib_send_wr *bad_swr;
+ struct ib_cq *cq = ibqp->send_cq;
+ int ret;
+
+ ret = ib_modify_qp(ibqp, &attr, IB_QP_STATE);
+ if (ret) {
+ ibdev_err(ibqp->device, "Failed to drain SQ: modify QP returned %d\n", ret);
+ return;
+ }
+
+ sdrain.cqe.done = mana_ib_drain_qp_done;
+ init_completion(&sdrain.done);
+
+ ret = ib_post_send(ibqp, &swr.wr, &bad_swr);
+ if (ret) {
+ ibdev_err(ibqp->device, "Failed to post SQ drain WR: %d\n", ret);
+ return;
+ }
+
+ if (cq->comp_handler)
+ cq->comp_handler(cq, cq->cq_context);
+
+ if (cq->poll_ctx == IB_POLL_DIRECT) {
+ while (!wait_for_completion_timeout(&sdrain.done, HZ / 10))
+ ib_process_cq_direct(cq, -1);
+ } else {
+ wait_for_completion(&sdrain.done);
+ }
+}
+
+void mana_ib_drain_rq(struct ib_qp *ibqp)
+{
+ struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
+ struct mana_ib_drain_cqe rdrain;
+ struct ib_recv_wr rwr = {
+ .next = NULL,
+ .wr_cqe = &rdrain.cqe,
+ };
+ struct ib_cq *cq = ibqp->recv_cq;
+ int ret;
+
+ ret = ib_modify_qp(ibqp, &attr, IB_QP_STATE);
+ if (ret) {
+ ibdev_err(ibqp->device, "Failed to drain RQ: modify QP returned %d\n", ret);
+ return;
+ }
+
+ rdrain.cqe.done = mana_ib_drain_qp_done;
+ init_completion(&rdrain.done);
+
+ ret = ib_post_recv(ibqp, &rwr, NULL);
+ if (ret) {
+ ibdev_err(ibqp->device, "Failed to post RQ drain WR: %d\n", ret);
+ return;
+ }
+
+ if (cq->comp_handler)
+ cq->comp_handler(cq, cq->cq_context);
+
+ if (cq->poll_ctx == IB_POLL_DIRECT) {
+ while (!wait_for_completion_timeout(&rdrain.done, HZ / 10))
+ ib_process_cq_direct(cq, -1);
+ } else {
+ wait_for_completion(&rdrain.done);
+ }
+}
--
2.43.0
prev parent reply other threads:[~2026-10-01 18:21 UTC|newest]
Thread overview: 11+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-10-01 18:20 [PATCH rdma-next 00/10] RDMA/mana_ib: Add kernel RC and fast registration support Konstantin Taranov
2026-10-01 18:20 ` [PATCH rdma-next 01/10] RDMA/mana_ib: Allocate and map fast-registration MRs Konstantin Taranov
2026-10-01 18:20 ` [PATCH rdma-next 02/10] RDMA/mana: Create and destroy kernel RC QPs Konstantin Taranov
2026-10-01 18:20 ` [PATCH rdma-next 03/10] net/mana: Extend GDMA encoding for new RDMA WQEs Konstantin Taranov
2026-10-01 18:20 ` [PATCH rdma-next 04/10] RDMA/mana_ib: Maintain kernel RC QP state Konstantin Taranov
2026-10-01 18:20 ` [PATCH rdma-next 05/10] RDMA/mana_ib: Post receive WRs on kernel RC QPs Konstantin Taranov
2026-10-01 18:20 ` [PATCH rdma-next 06/10] RDMA/mana_ib: Post send and memory-management WRs on " Konstantin Taranov
2026-10-01 18:20 ` [PATCH rdma-next 07/10] RDMA/mana_ib: Poll RC completions using PSN and FSN progress Konstantin Taranov
2026-10-01 18:20 ` [PATCH rdma-next 08/10] RDMA/mana_ib: Flush and notify CQs when kernel QPs enter ERR Konstantin Taranov
2026-10-01 18:20 ` [PATCH rdma-next 09/10] RDMA/mana_ib: Handle error CQEs for RC QPs Konstantin Taranov
2026-10-01 18:20 ` Konstantin Taranov [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261001182015.1757203-11-kotaranov@linux.microsoft.com \
--to=kotaranov@linux.microsoft.com \
--cc=jgg@ziepe.ca \
--cc=kotaranov@microsoft.com \
--cc=leon@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-rdma@vger.kernel.org \
--cc=longli@microsoft.com \
--cc=snsanghvi@microsoft.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®