From: Konstantin Taranov <kotaranov@linux.microsoft.com>
To: kotaranov@microsoft.com, shirazsaleem@microsoft.com,
pabeni@redhat.com, haiyangz@microsoft.com, kys@microsoft.com,
edumazet@google.com, kuba@kernel.org, davem@davemloft.net,
decui@microsoft.com, wei.liu@kernel.org,
sharmaajay@microsoft.com, longli@microsoft.com, jgg@ziepe.ca,
leon@kernel.org
Cc: linux-rdma@vger.kernel.org, linux-kernel@vger.kernel.org,
netdev@vger.kernel.org, linux-hyperv@vger.kernel.org
Subject: [PATCH rdma-next 11/13] RDMA/mana_ib: extend mana QP table
Date: Mon, 20 Jan 2025 09:27:17 -0800 [thread overview]
Message-ID: <1737394039-28772-12-git-send-email-kotaranov@linux.microsoft.com> (raw)
In-Reply-To: <1737394039-28772-1-git-send-email-kotaranov@linux.microsoft.com>
From: Konstantin Taranov <kotaranov@microsoft.com>
Enable mana QP table to store UD/GSI QPs.
For send queues, set the most significant bit to one,
as send and receive WQs can have the same ID in mana.
Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
Reviewed-by: Shiraz Saleem <shirazsaleem@microsoft.com>
---
drivers/infiniband/hw/mana/main.c | 2 +-
drivers/infiniband/hw/mana/mana_ib.h | 8 ++-
drivers/infiniband/hw/mana/qp.c | 78 ++++++++++++++++++++++++++--
3 files changed, 83 insertions(+), 5 deletions(-)
diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c
index b0c55cb..114e391 100644
--- a/drivers/infiniband/hw/mana/main.c
+++ b/drivers/infiniband/hw/mana/main.c
@@ -704,7 +704,7 @@ mana_ib_event_handler(void *ctx, struct gdma_queue *q, struct gdma_event *event)
switch (event->type) {
case GDMA_EQE_RNIC_QP_FATAL:
qpn = event->details[0];
- qp = mana_get_qp_ref(mdev, qpn);
+ qp = mana_get_qp_ref(mdev, qpn, false);
if (!qp)
break;
if (qp->ibqp.event_handler) {
diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
index bd34ad6..5e4ca55 100644
--- a/drivers/infiniband/hw/mana/mana_ib.h
+++ b/drivers/infiniband/hw/mana/mana_ib.h
@@ -23,6 +23,9 @@
/* MANA doesn't have any limit for MR size */
#define MANA_IB_MAX_MR_SIZE U64_MAX
+/* Send queue ID mask */
+#define MANA_SENDQ_MASK BIT(31)
+
/*
* The hardware limit of number of MRs is greater than maximum number of MRs
* that can possibly represent in 24 bits
@@ -438,11 +441,14 @@ static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev)
}
static inline struct mana_ib_qp *mana_get_qp_ref(struct mana_ib_dev *mdev,
- uint32_t qid)
+ u32 qid, bool is_sq)
{
struct mana_ib_qp *qp;
unsigned long flag;
+ if (is_sq)
+ qid |= MANA_SENDQ_MASK;
+
xa_lock_irqsave(&mdev->qp_table_wq, flag);
qp = xa_load(&mdev->qp_table_wq, qid);
if (qp)
diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
index 051ea03..2528046 100644
--- a/drivers/infiniband/hw/mana/qp.c
+++ b/drivers/infiniband/hw/mana/qp.c
@@ -444,18 +444,82 @@ static enum gdma_queue_type mana_ib_queue_type(struct ib_qp_init_attr *attr, u32
return type;
}
+static int mana_table_store_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
+{
+ return xa_insert_irq(&mdev->qp_table_wq, qp->ibqp.qp_num, qp,
+ GFP_KERNEL);
+}
+
+static void mana_table_remove_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
+{
+ xa_erase_irq(&mdev->qp_table_wq, qp->ibqp.qp_num);
+}
+
+static int mana_table_store_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
+{
+ u32 qids = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].id | MANA_SENDQ_MASK;
+ u32 qidr = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id;
+ int err;
+
+ err = xa_insert_irq(&mdev->qp_table_wq, qids, qp, GFP_KERNEL);
+ if (err)
+ return err;
+
+ err = xa_insert_irq(&mdev->qp_table_wq, qidr, qp, GFP_KERNEL);
+ if (err)
+ goto remove_sq;
+
+ return 0;
+
+remove_sq:
+ xa_erase_irq(&mdev->qp_table_wq, qids);
+ return err;
+}
+
+static void mana_table_remove_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
+{
+ u32 qids = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].id | MANA_SENDQ_MASK;
+ u32 qidr = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id;
+
+ xa_erase_irq(&mdev->qp_table_wq, qids);
+ xa_erase_irq(&mdev->qp_table_wq, qidr);
+}
+
static int mana_table_store_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
{
refcount_set(&qp->refcount, 1);
init_completion(&qp->free);
- return xa_insert_irq(&mdev->qp_table_wq, qp->ibqp.qp_num, qp,
- GFP_KERNEL);
+
+ switch (qp->ibqp.qp_type) {
+ case IB_QPT_RC:
+ return mana_table_store_rc_qp(mdev, qp);
+ case IB_QPT_UD:
+ case IB_QPT_GSI:
+ return mana_table_store_ud_qp(mdev, qp);
+ default:
+ ibdev_dbg(&mdev->ib_dev, "Unknown QP type for storing in mana table, %d\n",
+ qp->ibqp.qp_type);
+ }
+
+ return -EINVAL;
}
static void mana_table_remove_qp(struct mana_ib_dev *mdev,
struct mana_ib_qp *qp)
{
- xa_erase_irq(&mdev->qp_table_wq, qp->ibqp.qp_num);
+ switch (qp->ibqp.qp_type) {
+ case IB_QPT_RC:
+ mana_table_remove_rc_qp(mdev, qp);
+ break;
+ case IB_QPT_UD:
+ case IB_QPT_GSI:
+ mana_table_remove_ud_qp(mdev, qp);
+ break;
+ default:
+ ibdev_dbg(&mdev->ib_dev, "Unknown QP type for removing from mana table, %d\n",
+ qp->ibqp.qp_type);
+ return;
+ }
mana_put_qp_ref(qp);
wait_for_completion(&qp->free);
}
@@ -586,8 +650,14 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i)
qp->ud_qp.queues[i].kmem->id = qp->ud_qp.queues[i].id;
+ err = mana_table_store_qp(mdev, qp);
+ if (err)
+ goto destroy_qp;
+
return 0;
+destroy_qp:
+ mana_ib_gd_destroy_ud_qp(mdev, qp);
destroy_shadow_queues:
destroy_shadow_queue(&qp->shadow_rq);
destroy_shadow_queue(&qp->shadow_sq);
@@ -770,6 +840,8 @@ static int mana_ib_destroy_ud_qp(struct mana_ib_qp *qp, struct ib_udata *udata)
container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
int i;
+ mana_table_remove_qp(mdev, qp);
+
destroy_shadow_queue(&qp->shadow_rq);
destroy_shadow_queue(&qp->shadow_sq);
--
2.43.0
next prev parent reply other threads:[~2025-01-20 17:27 UTC|newest]
Thread overview: 31+ messages / expand[flat|nested] mbox.gz Atom feed top
2025-01-20 17:27 [PATCH rdma-next 00/13] RDMA/mana_ib: Enable CM for mana_ib Konstantin Taranov
2025-01-20 17:27 ` [PATCH rdma-next 01/13] RDMA/mana_ib: Allow registration of DMA-mapped memory in PDs Konstantin Taranov
2025-01-23 5:17 ` Long Li
2025-01-20 17:27 ` [PATCH rdma-next 02/13] RDMA/mana_ib: implement get_dma_mr Konstantin Taranov
2025-01-23 5:18 ` Long Li
2025-01-20 17:27 ` [PATCH rdma-next 03/13] RDMA/mana_ib: helpers to allocate kernel queues Konstantin Taranov
2025-01-23 5:25 ` Long Li
2025-01-20 17:27 ` [PATCH rdma-next 04/13] RDMA/mana_ib: create kernel-level CQs Konstantin Taranov
2025-01-23 5:36 ` Long Li
2025-01-28 12:50 ` Konstantin Taranov
2025-01-29 0:48 ` Long Li
2025-01-20 17:27 ` [PATCH rdma-next 05/13] RDMA/mana_ib: Create and destroy UD/GSI QP Konstantin Taranov
2025-01-23 5:40 ` Long Li
2025-01-20 17:27 ` [PATCH rdma-next 06/13] RDMA/mana_ib: UD/GSI QP creation for kernel Konstantin Taranov
2025-01-23 5:45 ` Long Li
2025-01-20 17:27 ` [PATCH rdma-next 07/13] RDMA/mana_ib: create/destroy AH Konstantin Taranov
2025-01-23 5:53 ` Long Li
2025-01-20 17:27 ` [PATCH rdma-next 08/13] net/mana: fix warning in the writer of client oob Konstantin Taranov
2025-01-23 22:48 ` Long Li
2025-01-20 17:27 ` [PATCH rdma-next 09/13] RDMA/mana_ib: UD/GSI work requests Konstantin Taranov
2025-01-23 18:20 ` Long Li
2025-01-23 19:03 ` Jason Gunthorpe
2025-01-20 17:27 ` [PATCH rdma-next 10/13] RDMA/mana_ib: implement req_notify_cq Konstantin Taranov
2025-01-23 5:57 ` Long Li
2025-01-20 17:27 ` Konstantin Taranov [this message]
2025-01-23 6:02 ` [PATCH rdma-next 11/13] RDMA/mana_ib: extend mana QP table Long Li
2025-01-20 17:27 ` [PATCH rdma-next 12/13] RDMA/mana_ib: polling of CQs for GSI/UD Konstantin Taranov
2025-01-23 19:17 ` Long Li
2025-01-20 17:27 ` [PATCH rdma-next 13/13] RDMA/mana_ib: indicate CM support Konstantin Taranov
2025-01-23 19:18 ` Long Li
2025-02-03 11:56 ` [PATCH rdma-next 00/13] RDMA/mana_ib: Enable CM for mana_ib Leon Romanovsky
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1737394039-28772-12-git-send-email-kotaranov@linux.microsoft.com \
--to=kotaranov@linux.microsoft.com \
--cc=davem@davemloft.net \
--cc=decui@microsoft.com \
--cc=edumazet@google.com \
--cc=haiyangz@microsoft.com \
--cc=jgg@ziepe.ca \
--cc=kotaranov@microsoft.com \
--cc=kuba@kernel.org \
--cc=kys@microsoft.com \
--cc=leon@kernel.org \
--cc=linux-hyperv@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-rdma@vger.kernel.org \
--cc=longli@microsoft.com \
--cc=netdev@vger.kernel.org \
--cc=pabeni@redhat.com \
--cc=sharmaajay@microsoft.com \
--cc=shirazsaleem@microsoft.com \
--cc=wei.liu@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
Powered by JetHome