[PATCH RFC 3/3] RDMA/rtrs: Post client read requests inline

[Andrei-Alexandru Bleortu <me@andrei-z.com>]

The RTRS client sends a read request as an RDMA Write with Immediate of
a small message: the user message, the RTRS header and one memory
descriptor, about 60 bytes for RNBD. The message is posted by reference
and the QP asks for no inline data, so the HCA has to DMA-read it from
host memory before sending. That read is on the latency path of every
read.

Let rtrs_cq_qp_create() take the amount of inline data to ask for.
The client asks for RTRS_CLT_MAX_INLINE_DATA, enough for a read
request with one descriptor and up to 64 bytes of user message, and
the server asks for none. If the device refuses, the QP is created
without inline data. The client posts a read request with
IB_SEND_INLINE when it fits in the inline size the device granted.
Write requests, whose message travels behind the data in the same
WR, are not changed.

Providers copy inline data from sg_list[].addr as a kernel virtual
address, so the inline path puts req->iu->buf in the on-stack SGE.

Assisted-by: LLM
Signed-off-by: Andrei-Alexandru Bleortu <me@andrei-z.com>
---
 drivers/infiniband/ulp/rtrs/rtrs-clt.c | 17 +++++++++++++++--
 drivers/infiniband/ulp/rtrs/rtrs-pri.h | 10 +++++++++-
 drivers/infiniband/ulp/rtrs/rtrs-srv.c |  2 +-
 drivers/infiniband/ulp/rtrs/rtrs.c     | 22 +++++++++++++++++-----
 4 files changed, 42 insertions(+), 9 deletions(-)

diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
index bf3a0377da3f..4a552e0459b2 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
@@ -489,6 +489,17 @@ static int rtrs_post_send_rdma(struct rtrs_clt_con *con,
 				      req->iu->dma_addr,
 				      req->sg_size, DMA_TO_DEVICE);
 
+	/*
+	 * Post the request inline if it fits in the granted inline size, so
+	 * the HCA does not have to DMA-read it from host memory.  The CPU
+	 * copies inline data from a kernel virtual address, so put
+	 * req->iu->buf in the SGE.
+	 */
+	if (req->sg_size <= con->c.max_inline_data) {
+		sge.addr = (uintptr_t)req->iu->buf;
+		flags |= IB_SEND_INLINE;
+	}
+
 	return rtrs_iu_post_rdma_write_imm(&con->c, req->iu, &sge, 1,
 					    rbuf->rkey, rbuf->addr + off,
 					    imm, flags, wr, NULL);
@@ -1714,11 +1725,13 @@ static int create_con_cq_qp(struct rtrs_clt_con *con)
 	if (con->c.cid >= clt_path->s.irq_con_num)
 		err = rtrs_cq_qp_create(&clt_path->s, &con->c, max_send_sge,
 					cq_vector, cq_num, max_send_wr,
-					max_recv_wr, IB_POLL_DIRECT);
+					max_recv_wr, RTRS_CLT_MAX_INLINE_DATA,
+					IB_POLL_DIRECT);
 	else
 		err = rtrs_cq_qp_create(&clt_path->s, &con->c, max_send_sge,
 					cq_vector, cq_num, max_send_wr,
-					max_recv_wr, IB_POLL_SOFTIRQ);
+					max_recv_wr, RTRS_CLT_MAX_INLINE_DATA,
+					IB_POLL_SOFTIRQ);
 	/*
 	 * In case of error we do not bother to clean previous allocations,
 	 * since destroy_con_cq_qp() must be called.
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-pri.h b/drivers/infiniband/ulp/rtrs/rtrs-pri.h
index 59529d593869..bcba80d3c214 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-pri.h
+++ b/drivers/infiniband/ulp/rtrs/rtrs-pri.h
@@ -97,6 +97,7 @@ struct rtrs_con {
 	int                     nr_cqe;
 	atomic_t		wr_cnt;
 	atomic_t		sq_wr_avail;
+	u32			max_inline_data;
 };
 
 struct rtrs_path {
@@ -286,6 +287,13 @@ struct rtrs_msg_rdma_read {
 	struct rtrs_sg_desc    desc[];
 };
 
+/*
+ * Inline data the client asks for on its QPs: enough for a read request
+ * with one memory descriptor and a user message of up to 64 bytes.
+ */
+#define RTRS_CLT_MAX_INLINE_DATA (sizeof(struct rtrs_msg_rdma_read) + \
+				  sizeof(struct rtrs_sg_desc) + 64)
+
 /**
  * struct rtrs_msg_rdma_write - Message transferred to server with RDMA-Write
  * @type:		always @RTRS_MSG_WRITE
@@ -325,7 +333,7 @@ int rtrs_post_recv_empty(struct rtrs_con *con, struct ib_cqe *cqe);
 int rtrs_cq_qp_create(struct rtrs_path *path, struct rtrs_con *con,
 		      u32 max_send_sge, int cq_vector, int nr_cqe,
 		      u32 max_send_wr, u32 max_recv_wr,
-		      enum ib_poll_context poll_ctx);
+		      u32 max_inline_data, enum ib_poll_context poll_ctx);
 void rtrs_cq_qp_destroy(struct rtrs_con *con);
 
 void rtrs_init_hb(struct rtrs_path *path, struct ib_cqe *cqe,
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c
index 5314599d29fe..30fa4c7f64dc 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c
@@ -1759,7 +1759,7 @@ static int create_con(struct rtrs_srv_path *srv_path,
 
 	/* TODO: SOFTIRQ can be faster, but be careful with softirq context */
 	err = rtrs_cq_qp_create(&srv_path->s, &con->c, 1, cq_vector, cq_num,
-				 max_send_wr, max_recv_wr,
+				 max_send_wr, max_recv_wr, 0,
 				 IB_POLL_WORKQUEUE);
 	if (err) {
 		rtrs_err(s, "rtrs_cq_qp_create(), err: %pe\n", ERR_PTR(err));
diff --git a/drivers/infiniband/ulp/rtrs/rtrs.c b/drivers/infiniband/ulp/rtrs/rtrs.c
index a642626e0ecb..08b8b1abb1d3 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs.c
@@ -253,9 +253,10 @@ static int create_cq(struct rtrs_con *con, int cq_vector, int nr_cqe,
 }
 
 static int create_qp(struct rtrs_con *con, struct ib_pd *pd,
-		     u32 max_send_wr, u32 max_recv_wr, u32 max_sge)
+		     u32 max_send_wr, u32 max_recv_wr, u32 max_sge,
+		     u32 max_inline_data)
 {
-	struct ib_qp_init_attr init_attr = {};
+	struct ib_qp_init_attr init_attr = {}, attr;
 	struct rdma_cm_id *cm_id = con->cm_id;
 	int ret;
 
@@ -271,13 +272,24 @@ static int create_qp(struct rtrs_con *con, struct ib_pd *pd,
 	init_attr.recv_cq = con->cq;
 	init_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
 
-	ret = rdma_create_qp(cm_id, pd, &init_attr);
+	/*
+	 * If the device cannot provide the requested inline data, create the
+	 * QP without it.
+	 */
+	attr = init_attr;
+	attr.cap.max_inline_data = max_inline_data;
+	ret = rdma_create_qp(cm_id, pd, &attr);
+	if (ret && max_inline_data) {
+		attr = init_attr;
+		ret = rdma_create_qp(cm_id, pd, &attr);
+	}
 	if (ret) {
 		rtrs_err(con->path, "Creating QP failed, err: %pe\n",
 			 ERR_PTR(ret));
 		return ret;
 	}
 	con->qp = cm_id->qp;
+	con->max_inline_data = attr.cap.max_inline_data;
 
 	return ret;
 }
@@ -296,7 +308,7 @@ static void destroy_cq(struct rtrs_con *con)
 int rtrs_cq_qp_create(struct rtrs_path *path, struct rtrs_con *con,
 		       u32 max_send_sge, int cq_vector, int nr_cqe,
 		       u32 max_send_wr, u32 max_recv_wr,
-		       enum ib_poll_context poll_ctx)
+		       u32 max_inline_data, enum ib_poll_context poll_ctx)
 {
 	int err;
 
@@ -305,7 +317,7 @@ int rtrs_cq_qp_create(struct rtrs_path *path, struct rtrs_con *con,
 		return err;
 
 	err = create_qp(con, path->dev->ib_pd, max_send_wr, max_recv_wr,
-			max_send_sge);
+			max_send_sge, max_inline_data);
 	if (err) {
 		destroy_cq(con);
 		return err;
-- 
2.56.0