[PATCH 08/11] RDMA/cxi: add native memory registration

[Doug Ledford <doug.ledford@hpe.com>]

Add memory region registration on top of the core driver's pinned
mapping and its memory descriptors.  A registration returns, with the
usual lkey/rkey, the native mapping description (VA, IOVA, length,
page and huge page shift, descriptor ID and LAC) so the user space
provider can address the region without a second query.

Remote keys have 48 significant bits: a requested key wider than that
is rejected with EOPNOTSUPP rather than truncated.

Responses use ib_respond_udata().

checkpatch --strict (100 columns) still reports the following in this
patch. They need coordination with the internal repositories this code
is maintained in and will be fixed in V2:
- CHECK: PARENTHESIS_ALIGNMENT (1)

Co-developed-by: Iraimani P <iraimani.pavadai@hpe.com>
Signed-off-by: Iraimani P <iraimani.pavadai@hpe.com>
Signed-off-by: Doug Ledford <doug.ledford@hpe.com>
---
 drivers/infiniband/hw/cxi/cxiverbs_mr.c | 192 ++++++++++++++++++++++++++++++++
 1 file changed, 192 insertions(+)

diff --git a/drivers/infiniband/hw/cxi/cxiverbs_mr.c b/drivers/infiniband/hw/cxi/cxiverbs_mr.c
new file mode 100644
index 000000000000..97ca5f9a7bb1
--- /dev/null
+++ b/drivers/infiniband/hw/cxi/cxiverbs_mr.c
@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * CXI InfiniBand RDMA driver — Memory Registration (Phase 2).
+ * Copyright 2025 Hewlett Packard Enterprise Development LP
+ *
+ * Maps user memory into the CXI ATU (Address Translation Unit).
+ *
+ * Key design points
+ * -----------------
+ * lkey = low 32 bits of (md->id << 3) | lac
+ *   The LAC (Local Address Context) is the hardware protection domain
+ *   for local DMA.  md->id is the ATU MD index.
+ *
+ * rkey = random 32-bit token
+ *   Used as match_bits in the portal target LE for RDMA WRITE/READ.
+ *   The remote initiator must present this token for protected access.
+ *
+ * ODP (On-Demand Paging):
+ *   IB_ACCESS_ON_DEMAND selects native ATS/PRS mapping instead of pinning.
+ *   The native ATS/PASID/HMM paths own invalidation and notifier lifetimes.
+ *   The verbs provider must not install a second notifier or clear native
+ *   mappings from an MMU callback.
+ */
+
+#include <linux/slab.h>
+#include <linux/random.h>
+#include <linux/mm.h>
+
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_umem.h>
+
+#include "cxiverbs.h"
+
+/* =======================================================================
+ * reg_user_mr
+ * =======================================================================
+ */
+
+struct ib_mr *cxiverbs_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
+				  u64 iova, int access,
+				  struct ib_dmah *dmah,
+				  struct ib_udata *udata)
+{
+	struct cxiverbs_ucontext *uctxt;
+	struct cxiverbs_pd *pd = to_cxiverbs_pd(ibpd);
+	struct cxi_ibv_reg_mr_resp resp = {};
+	struct cxi_ibv_reg_mr cmd = {};
+	struct cxiverbs_mr *mr;
+	bool is_odp   = !!(access & IB_ACCESS_ON_DEMAND);
+	u32 cxi_flags = CXI_MAP_USER_ADDR;
+	int ret;
+
+	if (!ibpd->uobject)
+		return ERR_PTR(-EOPNOTSUPP);
+	uctxt = to_cxiverbs_ucontext(ibpd->uobject->context);
+
+	if (!pd->lni_ref)
+		return ERR_PTR(-EINVAL);
+	if (udata && udata->inlen >= sizeof(cmd)) {
+		ret = ib_copy_from_udata(&cmd, udata, sizeof(cmd));
+		if (ret)
+			return ERR_PTR(ret);
+		if (cmd.flags & ~(CXI_REG_MR_USER_RKEY | CXI_REG_MR_NATIVE_MAP))
+			return ERR_PTR(-EOPNOTSUPP);
+	}
+
+	mr = kzalloc_obj(*mr);
+	if (!mr)
+		return ERR_PTR(-ENOMEM);
+	mr->cadev = uctxt->cadev;
+
+	if (is_odp) {
+		/* ATS mode: PCIe Address Translation Services / Page Request */
+		cxi_flags |= CXI_MAP_ATS;
+	} else {
+		/* Pin-based: pages locked in RAM for the MR lifetime */
+		cxi_flags |= CXI_MAP_PIN;
+	}
+
+	if (access & (IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE))
+		cxi_flags |= CXI_MAP_WRITE;
+	/* Local read is always needed for SGLs used in post_send */
+	cxi_flags |= CXI_MAP_READ;
+	if (cmd.flags & CXI_REG_MR_NATIVE_MAP) {
+		if (is_odp || !(cmd.native_flags & CXI_MAP_PIN) ||
+		    !(cmd.native_flags & (CXI_MAP_READ | CXI_MAP_WRITE)) ||
+		    (cmd.native_flags & ~(CXI_MAP_PIN | CXI_MAP_READ |
+				  CXI_MAP_WRITE | CXI_MAP_ALLOC_MD))) {
+			ret = -EOPNOTSUPP;
+			goto err_free_mr;
+		}
+		cxi_flags = CXI_MAP_USER_ADDR | cmd.native_flags;
+	}
+
+	/* Map into the ATU under the context's JobID-anchored LNI. */
+	mr->md = uctxt->cadev->cxi_map(pd->lni_ref->lni,
+				     (uintptr_t)start, length,
+					 cxi_flags, NULL);
+	if (IS_ERR(mr->md)) {
+		ret = PTR_ERR(mr->md);
+		mr->md = NULL;
+		goto err_free_mr;
+	}
+
+	mr->lac = mr->md->lac;
+
+	/*
+	 * The narrow keys alias the low word of these, so a caller limited to
+	 * 32 bits gets a usable truncation as long as the descriptor index
+	 * stays under 2^29.
+	 */
+	mr->ibmr.lkey64 = ((u64)mr->lac & CXI_LKEY64_LAC_MASK) |
+			  (((u64)mr->md->id & CXI_LKEY64_MD_MASK) <<
+			   CXI_LKEY64_MD_SHIFT);
+	/*
+	 * Bit 31 keeps the *narrow* key non-zero, so an rkey stays distinct
+	 * from the zero match value SEND uses whether a peer presents the full
+	 * key or only its low word.
+	 */
+	mr->ibmr.rkey64 = (get_random_u64() & CXI_RKEY_MAX) | BIT_ULL(31);
+
+	if (udata && udata->inlen >= sizeof(cmd)) {
+		ret = ib_copy_from_udata(&cmd, udata, sizeof(cmd));
+		if (ret)
+			goto err_unmap;
+
+		/* Taken as given: a narrow value makes a narrow region. */
+		if (cmd.flags & CXI_REG_MR_USER_RKEY) {
+			if (cmd.rkey64 > CXI_RKEY_MAX) {
+				ret = -EOPNOTSUPP;
+				goto err_unmap;
+			}
+			if (!cmd.rkey64) {
+				ret = -EINVAL;
+				goto err_unmap;
+			}
+			mr->ibmr.rkey64 = cmd.rkey64;
+		}
+	}
+	mr->ibmr.lkey = (u32)mr->ibmr.lkey64;
+	mr->ibmr.rkey = (u32)mr->ibmr.rkey64;
+	mr->ibmr.length = length;
+	mr->ibmr.iova   = iova ? iova : start;
+
+	resp.lkey64 = mr->ibmr.lkey64;
+	resp.rkey64 = mr->ibmr.rkey64;
+	resp.iova = CXI_VA_TO_IOVA(mr->md, start);
+	resp.info.va = mr->md->va;
+	resp.info.iova = mr->md->iova;
+	resp.info.length = mr->md->len;
+	resp.info.page_shift = mr->md->page_shift;
+	resp.info.huge_shift = mr->md->huge_shift;
+	resp.info.id = mr->md->id;
+	resp.info.lac = mr->md->lac;
+
+	if (udata) {
+		ret = ib_respond_udata(udata, resp);
+		if (ret)
+			goto err_unmap;
+	}
+
+	return &mr->ibmr;
+
+err_unmap:
+	uctxt->cadev->cxi_unmap(mr->md);
+err_free_mr:
+	kfree(mr);
+	return ERR_PTR(ret);
+}
+
+/* =======================================================================
+ * dereg_mr
+ * =======================================================================
+ */
+
+int cxiverbs_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
+{
+	struct cxiverbs_mr *mr = to_cxiverbs_mr(ibmr);
+	int ret;
+
+	if (atomic_read(&mr->native_users))
+		return -EBUSY;
+	if (mr->md) {
+		ret = mr->cadev->cxi_unmap(mr->md);
+		if (ret)
+			return ret;
+		mr->md = NULL;
+	}
+
+	kfree(mr);
+	return 0;
+}

-- 
2.56.0