From: Daniel Wagner <dwagner@suse.de>
To: linux-nvme@lists.infradead.org
Cc: linux-kernel@vger.kernel.org, Chaitanya Kulkarni <kch@nvidia.com>,
Sagi Grimberg <sagi@grimberg.me>, Hannes Reinecke <hare@suse.de>,
James Smart <jsmart2021@gmail.com>,
Daniel Wagner <dwagner@suse.de>
Subject: [RFC v3 1/9] nvme-rdma: stream line queue functions arguments
Date: Thu, 4 May 2023 11:12:51 +0200 [thread overview]
Message-ID: <20230504091259.29100-2-dwagner@suse.de> (raw)
In-Reply-To: <20230504091259.29100-1-dwagner@suse.de>
In preparation to move common code from the fabrics driver to fabrics.c,
we stream line the low level functions. This allows any common code just
to pass in nvme subsystem global types, such as 'struct nvme_ctrl'
instead of the driver specialized types 'struct nvme_rdma_ctrl'.
Signed-off-by: Daniel Wagner <dwagner@suse.de>
---
drivers/nvme/host/rdma.c | 62 ++++++++++++++++++++++++++--------------
1 file changed, 40 insertions(+), 22 deletions(-)
diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index 0eb79696fb73..92e5d0ccf3a9 100644
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -132,6 +132,11 @@ static inline struct nvme_rdma_ctrl *to_rdma_ctrl(struct nvme_ctrl *ctrl)
return container_of(ctrl, struct nvme_rdma_ctrl, ctrl);
}
+static inline int nvme_rdma_queue_id(struct nvme_rdma_queue *queue)
+{
+ return queue - queue->ctrl->queues;
+}
+
static LIST_HEAD(device_list);
static DEFINE_MUTEX(device_list_mutex);
@@ -566,13 +571,19 @@ static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue)
return ret;
}
-static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,
- int idx, size_t queue_size)
+static int nvme_rdma_alloc_queue(struct nvme_ctrl *nctrl, int idx)
{
+ struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
struct nvme_rdma_queue *queue;
struct sockaddr *src_addr = NULL;
+ size_t queue_size;
int ret;
+ if (idx == 0)
+ queue_size = NVME_AQ_DEPTH;
+ else
+ queue_size = ctrl->ctrl.sqsize + 1;
+
queue = &ctrl->queues[idx];
mutex_init(&queue->queue_lock);
queue->ctrl = ctrl;
@@ -636,16 +647,22 @@ static void __nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
ib_drain_qp(queue->qp);
}
-static void nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
+static void nvme_rdma_stop_queue(struct nvme_ctrl *nctrl, int qid)
{
+ struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
+ struct nvme_rdma_queue *queue = &ctrl->queues[qid];
+
mutex_lock(&queue->queue_lock);
if (test_and_clear_bit(NVME_RDMA_Q_LIVE, &queue->flags))
__nvme_rdma_stop_queue(queue);
mutex_unlock(&queue->queue_lock);
}
-static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)
+static void nvme_rdma_free_queue(struct nvme_ctrl *nctrl, int qid)
{
+ struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
+ struct nvme_rdma_queue *queue = &ctrl->queues[qid];
+
if (!test_and_clear_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags))
return;
@@ -659,7 +676,7 @@ static void nvme_rdma_free_io_queues(struct nvme_rdma_ctrl *ctrl)
int i;
for (i = 1; i < ctrl->ctrl.queue_count; i++)
- nvme_rdma_free_queue(&ctrl->queues[i]);
+ nvme_rdma_free_queue(&ctrl->ctrl, i);
}
static void nvme_rdma_stop_io_queues(struct nvme_rdma_ctrl *ctrl)
@@ -667,18 +684,19 @@ static void nvme_rdma_stop_io_queues(struct nvme_rdma_ctrl *ctrl)
int i;
for (i = 1; i < ctrl->ctrl.queue_count; i++)
- nvme_rdma_stop_queue(&ctrl->queues[i]);
+ nvme_rdma_stop_queue(&ctrl->ctrl, i);
}
-static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx)
+static int nvme_rdma_start_queue(struct nvme_ctrl *nctrl, int idx)
{
+ struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
struct nvme_rdma_queue *queue = &ctrl->queues[idx];
int ret;
if (idx)
- ret = nvmf_connect_io_queue(&ctrl->ctrl, idx);
+ ret = nvmf_connect_io_queue(nctrl, idx);
else
- ret = nvmf_connect_admin_queue(&ctrl->ctrl);
+ ret = nvmf_connect_admin_queue(nctrl);
if (!ret) {
set_bit(NVME_RDMA_Q_LIVE, &queue->flags);
@@ -697,7 +715,7 @@ static int nvme_rdma_start_io_queues(struct nvme_rdma_ctrl *ctrl,
int i, ret = 0;
for (i = first; i < last; i++) {
- ret = nvme_rdma_start_queue(ctrl, i);
+ ret = nvme_rdma_start_queue(&ctrl->ctrl, i);
if (ret)
goto out_stop_queues;
}
@@ -706,7 +724,7 @@ static int nvme_rdma_start_io_queues(struct nvme_rdma_ctrl *ctrl,
out_stop_queues:
for (i--; i >= first; i--)
- nvme_rdma_stop_queue(&ctrl->queues[i]);
+ nvme_rdma_stop_queue(&ctrl->ctrl, i);
return ret;
}
@@ -768,8 +786,7 @@ static int nvme_rdma_alloc_io_queues(struct nvme_rdma_ctrl *ctrl)
}
for (i = 1; i < ctrl->ctrl.queue_count; i++) {
- ret = nvme_rdma_alloc_queue(ctrl, i,
- ctrl->ctrl.sqsize + 1);
+ ret = nvme_rdma_alloc_queue(&ctrl->ctrl, i);
if (ret)
goto out_free_queues;
}
@@ -778,7 +795,7 @@ static int nvme_rdma_alloc_io_queues(struct nvme_rdma_ctrl *ctrl)
out_free_queues:
for (i--; i >= 1; i--)
- nvme_rdma_free_queue(&ctrl->queues[i]);
+ nvme_rdma_free_queue(&ctrl->ctrl, i);
return ret;
}
@@ -806,7 +823,7 @@ static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl)
sizeof(struct nvme_command), DMA_TO_DEVICE);
ctrl->async_event_sqe.data = NULL;
}
- nvme_rdma_free_queue(&ctrl->queues[0]);
+ nvme_rdma_free_queue(&ctrl->ctrl, 0);
}
static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
@@ -815,7 +832,7 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
bool pi_capable = false;
int error;
- error = nvme_rdma_alloc_queue(ctrl, 0, NVME_AQ_DEPTH);
+ error = nvme_rdma_alloc_queue(&ctrl->ctrl, 0);
if (error)
return error;
@@ -850,7 +867,7 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
}
- error = nvme_rdma_start_queue(ctrl, 0);
+ error = nvme_rdma_start_queue(&ctrl->ctrl, 0);
if (error)
goto out_remove_admin_tag_set;
@@ -877,7 +894,7 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
nvme_quiesce_admin_queue(&ctrl->ctrl);
blk_sync_queue(ctrl->ctrl.admin_q);
out_stop_queue:
- nvme_rdma_stop_queue(&ctrl->queues[0]);
+ nvme_rdma_stop_queue(&ctrl->ctrl, 0);
nvme_cancel_admin_tagset(&ctrl->ctrl);
out_remove_admin_tag_set:
if (new)
@@ -889,7 +906,7 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
ctrl->async_event_sqe.data = NULL;
}
out_free_queue:
- nvme_rdma_free_queue(&ctrl->queues[0]);
+ nvme_rdma_free_queue(&ctrl->ctrl, 0);
return error;
}
@@ -962,7 +979,7 @@ static void nvme_rdma_teardown_admin_queue(struct nvme_rdma_ctrl *ctrl,
{
nvme_quiesce_admin_queue(&ctrl->ctrl);
blk_sync_queue(ctrl->ctrl.admin_q);
- nvme_rdma_stop_queue(&ctrl->queues[0]);
+ nvme_rdma_stop_queue(&ctrl->ctrl, 0);
nvme_cancel_admin_tagset(&ctrl->ctrl);
if (remove) {
nvme_unquiesce_admin_queue(&ctrl->ctrl);
@@ -1113,7 +1130,7 @@ static int nvme_rdma_setup_ctrl(struct nvme_rdma_ctrl *ctrl, bool new)
destroy_admin:
nvme_quiesce_admin_queue(&ctrl->ctrl);
blk_sync_queue(ctrl->ctrl.admin_q);
- nvme_rdma_stop_queue(&ctrl->queues[0]);
+ nvme_rdma_stop_queue(&ctrl->ctrl, 0);
nvme_cancel_admin_tagset(&ctrl->ctrl);
if (new)
nvme_remove_admin_tag_set(&ctrl->ctrl);
@@ -1960,9 +1977,10 @@ static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
static void nvme_rdma_complete_timed_out(struct request *rq)
{
struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
+ struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
struct nvme_rdma_queue *queue = req->queue;
- nvme_rdma_stop_queue(queue);
+ nvme_rdma_stop_queue(ctrl, nvme_rdma_queue_id(queue));
nvmf_complete_timed_out_request(rq);
}
--
2.40.0
next prev parent reply other threads:[~2023-05-04 9:13 UTC|newest]
Thread overview: 13+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-05-04 9:12 [RFC v3 0/9] Unifying fabrics drivers Daniel Wagner
2023-05-04 9:12 ` Daniel Wagner [this message]
2023-05-04 9:12 ` [RFC v3 2/9] nvme-rdma: factor rdma specific queue init code out Daniel Wagner
2023-05-04 9:12 ` [RFC v3 3/9] nvme-tcp: move error and connect work to nvme_ctrl Daniel Wagner
2023-05-04 9:12 ` [RFC v3 4/9] nvme-rdma: use error and connect work from nvme_ctrl Daniel Wagner
2023-05-04 9:12 ` [RFC v3 5/9] nvme-fabrics: add fabric state machine Daniel Wagner
2023-05-04 9:12 ` [RFC v3 6/9] nvme-tcp: replace state machine with generic one Daniel Wagner
2023-05-04 9:12 ` [RFC v3 7/9] nvme-rdma: " Daniel Wagner
2023-05-04 9:12 ` [RFC v3 8/9] nvme: move queue flags to middle layer Daniel Wagner
2023-05-04 9:42 ` Sagi Grimberg
2023-05-04 9:12 ` [RFC v3 9/9] nvme: introduce setup_transport() Daniel Wagner
2023-05-04 10:23 ` Sagi Grimberg
2023-05-04 10:25 ` [RFC v3 0/9] Unifying fabrics drivers Sagi Grimberg
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230504091259.29100-2-dwagner@suse.de \
--to=dwagner@suse.de \
--cc=hare@suse.de \
--cc=jsmart2021@gmail.com \
--cc=kch@nvidia.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-nvme@lists.infradead.org \
--cc=sagi@grimberg.me \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®