[PATCH 09/11] RDMA/cxi: add services jobs profiles queues and counting triggers

[Doug Ledford <doug.ledford@hpe.com>]

Add the private uverbs object methods through which a privileged
process provisions and drives native Cassini resources.  They exist
so a launcher or workload manager can set policy on the NIC and the
provider can reach native queues without the kernel interpreting every
command:

 - services: allocate, enable, query and destroy a CXI service, with
   VNI, traffic class and resource limits; retry and range VNI modes
   are selectable
 - jobs and job keys, LNI and PD PID allocation and query
 - communication profiles, event queues (with optional pinned PCT
   registration and PCT passthrough), command queues including a target
   command queue and a restricted trigger mode, and counting events
   with a user supplied write-back page
 - priority wait descriptors: a pollable file descriptor that holds the
   LNI and reports event and status wakeups, with positional reads so
   the generation can be consumed without seeking
 - a privileged mapping of the device CSR aperture and an
   administrative inbound-wait method, both requiring CAP_SYS_ADMIN
   and a C1/C2 physical function

Lifetime rules worth knowing when reviewing: a native object stays
allocated until the last VMA that maps it is released, even if the
uverbs object was destroyed first; counting events are accounted
against the LNI and freed together with it; the write-back page for a
counting event must be user memory and is pinned, never a kernel
page, because the core driver treats a kernel address passed this
way as device memory.

Known open issue, deferred to the V2 SR-IOV update (Sashiko H87,
duplicate M88): on a physical function cxi_eq_free() cannot fail. On a
virtual function it fails when the PF mailbox is unreachable; if that
happens while the file is closed or the device is removed, the RDMA core
drops the EQ object without native cleanup and the VF EQ and its
references leak. Explicit EQ destroys are retried.

Known limitation, deferred to V2: this patch carries SR-IOV
virtual-function code (VF-to-PF mailbox requests, mailbox opcodes or VF
resource handling) from an older, incomplete version of the Cassini
SR-IOV support. V2 of this series will include the updated SR-IOV
implementation, so issues in the VF paths should be ignored for now.

checkpatch --strict (100 columns) still reports the following in this
patch. They need coordination with the internal repositories this code
is maintained in and will be fixed in V2:
- CHECK: PARENTHESIS_ALIGNMENT (82), OPEN_ENDED_LINE (23),
  UNCOMMENTED_DEFINITION (1)

Co-developed-by: Mamila Santosh <santosh-kumar.mamila@hpe.com>
Signed-off-by: Mamila Santosh <santosh-kumar.mamila@hpe.com>
Signed-off-by: Doug Ledford <doug.ledford@hpe.com>
---
 drivers/infiniband/hw/cxi/cxiverbs_ct.c | 1906 +++++++++++++++++++++++++++++++
 1 file changed, 1906 insertions(+)

diff --git a/drivers/infiniband/hw/cxi/cxiverbs_ct.c b/drivers/infiniband/hw/cxi/cxiverbs_ct.c
new file mode 100644
index 000000000000..c8f8974c649b
--- /dev/null
+++ b/drivers/infiniband/hw/cxi/cxiverbs_ct.c
@@ -0,0 +1,1906 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * cxiverbs_ct.c: Counting Triggers custom uverbs for the CXI RDMA driver.
+ * Copyright 2025 Hewlett Packard Enterprise Development LP
+ *
+ * Exposes CXI Counting Triggers (CTs) to userspace via the RDMA ioctl()
+ * interface.  A CT is a hardware counter that the NIC decrements when a
+ * triggered operation completes; when the counter hits zero the deferred
+ * command is dispatched.
+ *
+ * ABI:
+ *   CXI_IB_OBJECT_CT / CXI_IB_METHOD_CT_ALLOC
+ *     - allocates a CT against the caller's ucontext LNI
+ *     - returns: ctn, writeback-page mmap params, doorbell CSR mmap params
+ *   CXI_IB_METHOD_CT_DESTROY
+ *     - frees all CT resources; uses default destroy handler
+ *
+ * UVERBS_MODULE_NAME must be defined before uverbs_named_ioctl.h.
+ */
+#define UVERBS_MODULE_NAME cxiverbs
+#include <rdma/uverbs_named_ioctl.h>
+
+#include <linux/slab.h>
+#include <linux/gfp.h>
+#include <linux/file.h>
+#include <linux/poll.h>
+#include <linux/refcount.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/uverbs_ioctl.h>
+#include <uapi/rdma/cxiverbs-abi.h>
+#include "cxiverbs.h"
+
+static struct cxiverbs_ucontext *cxiverbs_get_ucontext(
+		struct uverbs_attr_bundle *attrs)
+{
+	struct ib_ucontext *context = ib_uverbs_get_ucontext(attrs);
+
+	if (IS_ERR(context))
+		return ERR_CAST(context);
+	return to_cxiverbs_ucontext(context);
+}
+
+struct cxiverbs_csr {
+	struct cxiverbs_mmap_entry *mapping;
+};
+
+static int cxiverbs_csr_cleanup(struct ib_uobject *uobj,
+			       enum rdma_remove_reason why,
+			       struct uverbs_attr_bundle *attrs)
+{
+	struct cxiverbs_csr *csr = uobj->object;
+
+	rdma_user_mmap_entry_remove(&csr->mapping->rdma_entry);
+	kfree(csr);
+	return 0;
+}
+
+static int UVERBS_HANDLER(CXI_IB_METHOD_CSR_MAP)(
+		struct uverbs_attr_bundle *attrs)
+{
+	struct ib_uobject *uobj =
+		uverbs_attr_get_uobject(attrs, CXI_IB_ATTR_CSR_HANDLE);
+	struct cxi_ibv_csr_info info = {};
+	struct cxiverbs_ucontext *ctx;
+	struct cxiverbs_csr *csr;
+	struct cxi_dev *cdev;
+	phys_addr_t physical;
+	size_t length;
+	u32 generation;
+	int ret;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+	ctx = cxiverbs_get_ucontext(attrs);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+	cdev = ctx->cadev->cxi_dev;
+	generation = cdev->prop.cassini_version & 0xf0;
+	if (!cdev->is_physfn ||
+	    (generation != CASSINI_1 && generation != CASSINI_2))
+		return -EOPNOTSUPP;
+	cxi_get_csrs_range(cdev, &physical, &length);
+	if (!length || !IS_ALIGNED(physical, PAGE_SIZE) ||
+	    !IS_ALIGNED(length, PAGE_SIZE))
+		return -EINVAL;
+	csr = kzalloc_obj(*csr, GFP_KERNEL);
+	if (!csr)
+		return -ENOMEM;
+	csr->mapping = cxiverbs_mmap_entry_alloc_admin(&ctx->ibucontext,
+						     physical, length,
+						     &info.offset);
+	if (IS_ERR(csr->mapping)) {
+		ret = PTR_ERR(csr->mapping);
+		kfree(csr);
+		return ret;
+	}
+	info.length = length;
+	uobj->object = csr;
+	uverbs_finalize_uobj_create(attrs, CXI_IB_ATTR_CSR_HANDLE);
+	return uverbs_copy_to(attrs, CXI_IB_ATTR_CSR_INFO, &info, sizeof(info));
+}
+
+DECLARE_UVERBS_NAMED_METHOD(CXI_IB_METHOD_CSR_MAP,
+	UVERBS_ATTR_IDR(CXI_IB_ATTR_CSR_HANDLE, CXI_IB_OBJECT_CSR,
+		       UVERBS_ACCESS_NEW, UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_CSR_INFO,
+			    UVERBS_ATTR_TYPE(struct cxi_ibv_csr_info),
+			    UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_METHOD_DESTROY(CXI_IB_METHOD_CSR_UNMAP,
+	UVERBS_ATTR_IDR(CXI_IB_ATTR_CSR_HANDLE, CXI_IB_OBJECT_CSR,
+		       UVERBS_ACCESS_DESTROY, UA_MANDATORY));
+
+static int
+UVERBS_HANDLER(CXI_IB_METHOD_CSR_INBOUND_WAIT)(struct uverbs_attr_bundle *attrs)
+{
+	struct cxiverbs_ucontext *ctx;
+	struct cxi_dev *cdev;
+	u32 generation;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+	ctx = cxiverbs_get_ucontext(attrs);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+	cdev = ctx->cadev->cxi_dev;
+	generation = cdev->prop.cassini_version & 0xf0;
+	if (!cdev->is_physfn ||
+	    (generation != CASSINI_1 && generation != CASSINI_2))
+		return -EOPNOTSUPP;
+	return cxi_inbound_wait(cdev);
+}
+
+DECLARE_UVERBS_NAMED_METHOD(CXI_IB_METHOD_CSR_INBOUND_WAIT);
+
+DECLARE_UVERBS_NAMED_OBJECT(CXI_IB_OBJECT_CSR,
+	UVERBS_TYPE_ALLOC_IDR(cxiverbs_csr_cleanup),
+	&UVERBS_METHOD(CXI_IB_METHOD_CSR_MAP),
+	&UVERBS_METHOD(CXI_IB_METHOD_CSR_UNMAP),
+	&UVERBS_METHOD(CXI_IB_METHOD_CSR_INBOUND_WAIT));
+
+struct cxiverbs_lni *cxiverbs_lni_get(struct cxiverbs_ucontext *ctx, u32 handle)
+{
+	struct cxiverbs_lni *lni = NULL;
+	struct cxiverbs_lni *candidate;
+	unsigned long index;
+
+	mutex_lock(&ctx->resource_lock);
+	if (handle != CXI_IBV_LNI_ANY) {
+		lni = xa_load(&ctx->lnis, handle);
+	} else {
+		xa_for_each(&ctx->lnis, index, candidate) {
+			if (lni) {
+				lni = NULL;
+				break;
+			}
+			lni = candidate;
+		}
+	}
+	if (lni)
+		atomic_inc(&lni->usecnt);
+	mutex_unlock(&ctx->resource_lock);
+	return lni ? lni : ERR_PTR(-EINVAL);
+}
+
+void cxiverbs_lni_put(struct cxiverbs_lni *lni)
+{
+	atomic_dec(&lni->usecnt);
+}
+
+static int cxiverbs_lni_cleanup(struct ib_uobject *uobj,
+			       enum rdma_remove_reason why,
+			       struct uverbs_attr_bundle *attrs)
+{
+	struct cxiverbs_lni *lni = uobj->object;
+	struct cxiverbs_ucontext *ctx = lni->ctx;
+
+	mutex_lock(&ctx->resource_lock);
+	if (atomic_read(&lni->usecnt)) {
+		mutex_unlock(&ctx->resource_lock);
+		return -EBUSY;
+	}
+	xa_erase(&ctx->lnis, lni->handle);
+	mutex_unlock(&ctx->resource_lock);
+	ctx->cadev->lni_free(lni->lni);
+	kfree(lni);
+	return 0;
+}
+
+static int UVERBS_HANDLER(CXI_IB_METHOD_LNI_ALLOC)(
+		struct uverbs_attr_bundle *attrs)
+{
+	struct ib_uobject *uobj =
+		uverbs_attr_get_uobject(attrs, CXI_IB_ATTR_LNI_ALLOC_HANDLE);
+	struct cxiverbs_ucontext *ctx = cxiverbs_get_ucontext(attrs);
+	struct cxiverbs_lni *lni;
+	u32 svc_id;
+	int ret;
+
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+	ret = uverbs_copy_from(&svc_id, attrs, CXI_IB_ATTR_LNI_ALLOC_SVC_ID);
+	if (ret)
+		return ret;
+	if (!svc_id)
+		return -EINVAL;
+	lni = kzalloc_obj(*lni, GFP_KERNEL);
+	if (!lni)
+		return -ENOMEM;
+	lni->lni = ctx->cadev->lni_alloc(ctx->cadev->cxi_dev, svc_id);
+	if (IS_ERR(lni->lni)) {
+		ret = PTR_ERR(lni->lni);
+		goto err_free;
+	}
+	lni->ctx = ctx;
+	lni->svc_id = svc_id;
+	lni->handle = uobj->id;
+	atomic_set(&lni->usecnt, 0);
+	mutex_lock(&ctx->resource_lock);
+	ret = xa_insert(&ctx->lnis, lni->handle, lni, GFP_KERNEL);
+	mutex_unlock(&ctx->resource_lock);
+	if (ret)
+		goto err_lni;
+	uobj->object = lni;
+	uverbs_finalize_uobj_create(attrs, CXI_IB_ATTR_LNI_ALLOC_HANDLE);
+	return uverbs_copy_to(attrs, CXI_IB_ATTR_LNI_ALLOC_ID,
+			     &lni->lni->id, sizeof(lni->lni->id));
+err_lni:
+	ctx->cadev->lni_free(lni->lni);
+err_free:
+	kfree(lni);
+	return ret;
+}
+
+DECLARE_UVERBS_NAMED_METHOD(CXI_IB_METHOD_LNI_ALLOC,
+	UVERBS_ATTR_IDR(CXI_IB_ATTR_LNI_ALLOC_HANDLE, CXI_IB_OBJECT_LNI,
+			UVERBS_ACCESS_NEW, UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_LNI_ALLOC_SVC_ID,
+			   UVERBS_ATTR_TYPE(u32), UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_LNI_ALLOC_ID,
+			    UVERBS_ATTR_TYPE(u32), UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_METHOD_DESTROY(CXI_IB_METHOD_LNI_DESTROY,
+	UVERBS_ATTR_IDR(CXI_IB_ATTR_LNI_DESTROY_HANDLE, CXI_IB_OBJECT_LNI,
+			UVERBS_ACCESS_DESTROY, UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_OBJECT(CXI_IB_OBJECT_LNI,
+	UVERBS_TYPE_ALLOC_IDR(cxiverbs_lni_cleanup),
+	&UVERBS_METHOD(CXI_IB_METHOD_LNI_ALLOC),
+	&UVERBS_METHOD(CXI_IB_METHOD_LNI_DESTROY));
+
+static int UVERBS_HANDLER(CXI_IB_METHOD_PD_PID_ALLOC)(
+		struct uverbs_attr_bundle *attrs)
+{
+	struct ib_pd *ibpd = uverbs_attr_get_obj(attrs, CXI_IB_ATTR_PD_PID_HANDLE);
+	struct cxiverbs_pd *pd = to_cxiverbs_pd(ibpd);
+	struct cxiverbs_dev *dev = to_cxiverbs_dev(ibpd->device);
+	u32 flags = 0;
+	u32 value;
+	int pid;
+	int ret;
+
+	ret = uverbs_get_flags32(&flags, attrs, CXI_IB_ATTR_PD_PID_FLAGS,
+				CXI_IB_PD_PID_ABSOLUTE);
+	if (ret)
+		return ret;
+	if (flags || !dev->cadev->pid_alloc || !dev->cadev->pid_free)
+		return -EOPNOTSUPP;
+	mutex_lock(&pd->resource_lock);
+	if (pd->pidonfep_valid) {
+		ret = -EBUSY;
+		goto out_unlock;
+	}
+	pid = dev->cadev->pid_alloc(pd->lni_ref->lni);
+	if (pid < 0) {
+		ret = pid;
+		goto out_unlock;
+	}
+	value = pid;
+	ret = uverbs_copy_to(attrs, CXI_IB_ATTR_PD_PID_VALUE,
+			     &value, sizeof(value));
+	if (ret) {
+		dev->cadev->pid_free(pd->lni_ref->lni, pid);
+		goto out_unlock;
+	}
+	pd->pidonfep = pid;
+	pd->pidonfep_valid = true;
+out_unlock:
+	mutex_unlock(&pd->resource_lock);
+	return ret;
+}
+
+static int UVERBS_HANDLER(CXI_IB_METHOD_PD_PID_QUERY)(
+		struct uverbs_attr_bundle *attrs)
+{
+	struct ib_pd *ibpd = uverbs_attr_get_obj(attrs, CXI_IB_ATTR_PD_PID_HANDLE);
+	struct cxiverbs_pd *pd = to_cxiverbs_pd(ibpd);
+	int ret;
+
+	mutex_lock(&pd->resource_lock);
+	ret = pd->pidonfep_valid ?
+		uverbs_copy_to(attrs, CXI_IB_ATTR_PD_PID_VALUE,
+			       &pd->pidonfep, sizeof(pd->pidonfep)) : -ENOENT;
+	mutex_unlock(&pd->resource_lock);
+	return ret;
+}
+
+DECLARE_UVERBS_NAMED_METHOD(CXI_IB_METHOD_PD_PID_ALLOC,
+	UVERBS_ATTR_IDR(CXI_IB_ATTR_PD_PID_HANDLE, UVERBS_OBJECT_PD,
+			UVERBS_ACCESS_WRITE, UA_MANDATORY),
+	UVERBS_ATTR_FLAGS_IN(CXI_IB_ATTR_PD_PID_FLAGS, u32, UA_OPTIONAL),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_PD_PID_VALUE,
+			    UVERBS_ATTR_TYPE(u32), UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_METHOD(CXI_IB_METHOD_PD_PID_QUERY,
+	UVERBS_ATTR_IDR(CXI_IB_ATTR_PD_PID_HANDLE, UVERBS_OBJECT_PD,
+			UVERBS_ACCESS_READ, UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_PD_PID_VALUE,
+			    UVERBS_ATTR_TYPE(u32), UA_MANDATORY));
+
+DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_PD,
+	&UVERBS_METHOD(CXI_IB_METHOD_PD_PID_ALLOC),
+	&UVERBS_METHOD(CXI_IB_METHOD_PD_PID_QUERY));
+
+static int UVERBS_HANDLER(CXI_IB_METHOD_SVC_ALLOC)(
+		struct uverbs_attr_bundle *attrs)
+{
+	struct cxiverbs_ucontext *ctx = cxiverbs_get_ucontext(attrs);
+	struct cxi_ibv_svc_config config;
+	struct cxi_svc_desc desc = {};
+	struct cxi_svc_fail_info fail = {};
+	struct cxi_limits *limits[] = {
+		&desc.limits.ptes, &desc.limits.txqs, &desc.limits.tgqs,
+		&desc.limits.eqs, &desc.limits.cts, &desc.limits.acs,
+		&desc.limits.tles, &desc.limits.les,
+	};
+	u32 svc_id;
+	unsigned int index;
+	unsigned int vni_index;
+	int ret;
+
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+	if (!capable(CAP_NET_ADMIN))
+		return -EPERM;
+	ret = uverbs_copy_from(&config, attrs, CXI_IB_ATTR_SVC_CONFIG);
+	if (ret)
+		return ret;
+	if (config.is_system_svc && !capable(CAP_SYS_ADMIN))
+		return -EPERM;
+	if (config.is_system_svc > 1 || config.is_range > 1 ||
+	    config.vni_count > CXI_IB_SVC_MAX_VNIS ||
+	    config.vni_min || config.vni_max ||
+	    (config.tc_mask >> CXI_TC_MAX) ||
+	    (config.flags & ~(CXI_IB_SVC_LIMITS | CXI_IB_SVC_CREATE_TC)))
+		return -EINVAL;
+	if (config.is_range) {
+		if (config.is_system_svc || config.vni_count ||
+		    !config.vnis[0] || config.vnis[0] > config.vnis[1] ||
+		    config.vnis[2] || config.vnis[3])
+			return -EINVAL;
+	} else if (config.is_system_svc) {
+		if (config.svc_id || config.vni_count || config.tc_mask ||
+		    (config.flags & CXI_IB_SVC_CREATE_TC) ||
+		    !(config.flags & CXI_IB_SVC_LIMITS) ||
+		    config.vnis[0] || config.vnis[1] || config.vnis[2] ||
+		    config.vnis[3])
+			return -EINVAL;
+		for (index = 0; index < ARRAY_SIZE(limits); index++) {
+			u16 expected = index == CXI_IB_SVC_EQS ||
+				       index == CXI_IB_SVC_ACS ? 1 : 0;
+
+			if (config.maximum[index] != expected ||
+			    config.reserved[index] != expected)
+				return -EINVAL;
+		}
+	} else if (!config.vni_count || !config.tc_mask) {
+		return -EINVAL;
+	}
+	for (index = 0; index < CXI_IB_SVC_MAX_VNIS; index++) {
+		if (index >= config.vni_count) {
+			if (!config.is_range && config.vnis[index] &&
+			    !config.is_system_svc)
+				return -EINVAL;
+			continue;
+		}
+		if (!config.vnis[index])
+			return -EINVAL;
+		for (vni_index = 0; vni_index < index; vni_index++)
+			if (config.vnis[index] == config.vnis[vni_index])
+				return -EINVAL;
+	}
+	desc.svc_id = config.svc_id;
+	desc.enable = true;
+	desc.restricted_members = true;
+	desc.members[0].type = CXI_SVC_MEMBER_UID;
+	desc.members[0].svc_member.uid = config.uid;
+	desc.is_system_svc = config.is_system_svc;
+	desc.restricted_tcs = !config.is_system_svc;
+	desc.resource_limits = !!(config.flags & CXI_IB_SVC_LIMITS);
+	for (index = 0; index < ARRAY_SIZE(limits); index++) {
+		if (config.reserved[index] > config.maximum[index])
+			return -EINVAL;
+		limits[index]->max = config.maximum[index];
+		limits[index]->res = config.reserved[index];
+	}
+	for (index = 0; index < CXI_TC_MAX; index++) {
+		desc.tcs[index] = !!(config.tc_mask & BIT(index));
+		if (!desc.tcs[index] || !(config.flags & CXI_IB_SVC_CREATE_TC))
+			continue;
+		ret = ctx->cadev->tc_cfg(ctx->cadev->cxi_dev, index,
+					CXI_TC_TYPE_DEFAULT);
+		if (ret && ret != -EEXIST)
+			return ret;
+	}
+	if (config.is_system_svc) {
+		desc.restricted_vnis = true;
+	} else if (config.is_range) {
+		desc.restricted_vnis = false;
+	} else {
+		desc.restricted_vnis = true;
+		desc.num_vld_vnis = config.vni_count;
+		memcpy(desc.vnis, config.vnis,
+		       config.vni_count * sizeof(config.vnis[0]));
+	}
+	ret = ctx->cadev->svc_alloc(ctx->cadev->cxi_dev, &desc, &fail,
+				   "cxidv-service");
+	if (ret < 0)
+		return ret;
+	svc_id = ret;
+	if (config.is_range) {
+		ret = ctx->cadev->svc_set_vni_range(ctx->cadev->cxi_dev, svc_id,
+						   config.vnis[0], config.vnis[1]);
+		if (ret)
+			goto err_destroy;
+		ret = ctx->cadev->svc_enable(ctx->cadev->cxi_dev, svc_id, true);
+		if (ret)
+			goto err_destroy;
+	}
+	ret = uverbs_copy_to(attrs, CXI_IB_ATTR_SVC_ID, &svc_id, sizeof(svc_id));
+	if (!ret)
+		return 0;
+err_destroy:
+	ctx->cadev->svc_destroy(ctx->cadev->cxi_dev, svc_id);
+	return ret;
+}
+
+static int UVERBS_HANDLER(CXI_IB_METHOD_SVC_DESTROY)(
+		struct uverbs_attr_bundle *attrs)
+{
+	struct cxiverbs_ucontext *ctx = cxiverbs_get_ucontext(attrs);
+	u32 svc_id;
+	int ret;
+
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+	if (!capable(CAP_NET_ADMIN))
+		return -EPERM;
+	ret = uverbs_copy_from(&svc_id, attrs, CXI_IB_ATTR_SVC_ID);
+	return ret ? ret : ctx->cadev->svc_destroy(ctx->cadev->cxi_dev, svc_id);
+}
+
+static int UVERBS_HANDLER(CXI_IB_METHOD_SVC_ENABLE)(
+		struct uverbs_attr_bundle *attrs)
+{
+	struct cxiverbs_ucontext *ctx = cxiverbs_get_ucontext(attrs);
+	u32 svc_id;
+	u32 enable;
+	int ret;
+
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+	if (!capable(CAP_NET_ADMIN))
+		return -EPERM;
+	ret = uverbs_copy_from(&svc_id, attrs, CXI_IB_ATTR_SVC_ID);
+	if (ret)
+		return ret;
+	ret = uverbs_copy_from(&enable, attrs, CXI_IB_ATTR_SVC_FLAGS);
+	if (ret)
+		return ret;
+	if (enable > 1)
+		return -EINVAL;
+	return ctx->cadev->svc_enable(ctx->cadev->cxi_dev, svc_id, enable);
+}
+
+static int UVERBS_HANDLER(CXI_IB_METHOD_SVC_QUERY)(
+		struct uverbs_attr_bundle *attrs)
+{
+	struct cxiverbs_ucontext *ctx = cxiverbs_get_ucontext(attrs);
+	struct cxi_ibv_svc_config config = {};
+	struct cxi_svc_desc desc;
+	const struct cxi_limits *limits[] = {
+		&desc.limits.ptes, &desc.limits.txqs, &desc.limits.tgqs,
+		&desc.limits.eqs, &desc.limits.cts, &desc.limits.acs,
+		&desc.limits.tles, &desc.limits.les,
+	};
+	unsigned int first, last, index;
+	int ret;
+
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+	ret = uverbs_copy_from(&config.svc_id, attrs, CXI_IB_ATTR_SVC_ID);
+	if (ret)
+		return ret;
+	ret = ctx->cadev->svc_get(ctx->cadev->cxi_dev, config.svc_id, &desc);
+	if (ret)
+		return ret;
+	if (desc.enable)
+		config.flags |= CXI_IB_SVC_ENABLED;
+	if (desc.resource_limits)
+		config.flags |= CXI_IB_SVC_LIMITS;
+	config.is_system_svc = desc.is_system_svc;
+	if (uverbs_attr_is_valid(attrs, CXI_IB_ATTR_SVC_FLAGS)) {
+		ret = uverbs_copy_to(attrs, CXI_IB_ATTR_SVC_FLAGS,
+				     &config.flags, sizeof(config.flags));
+		if (ret)
+			return ret;
+	}
+	if (uverbs_attr_is_valid(attrs, CXI_IB_ATTR_SVC_USAGE)) {
+		struct cxi_rsrc_use native = {};
+		struct cxi_ibv_svc_rsrc_use usage = {};
+
+		BUILD_BUG_ON(sizeof(native.in_use) != sizeof(usage.in_use));
+		ret = ctx->cadev->svc_rsrc_get(ctx->cadev->cxi_dev,
+					      config.svc_id, &native);
+		if (ret)
+			return ret;
+		usage.svc_id = native.svc_id;
+		usage.tle_pool_id = native.tle_pool_id;
+		memcpy(usage.in_use, native.in_use, sizeof(usage.in_use));
+		ret = uverbs_copy_to(attrs, CXI_IB_ATTR_SVC_USAGE,
+				     &usage, sizeof(usage));
+		if (ret)
+			return ret;
+	}
+	if (uverbs_attr_is_valid(attrs, CXI_IB_ATTR_SVC_LPR)) {
+		u32 lpr;
+
+		ret = cxi_svc_get_lpr(ctx->cadev->cxi_dev, config.svc_id);
+		if (ret < 0)
+			return ret;
+		lpr = ret;
+		ret = uverbs_copy_to(attrs, CXI_IB_ATTR_SVC_LPR, &lpr, sizeof(lpr));
+		if (ret)
+			return ret;
+	}
+	if (!uverbs_attr_is_valid(attrs, CXI_IB_ATTR_SVC_CONFIG))
+		return uverbs_attr_is_valid(attrs, CXI_IB_ATTR_SVC_FLAGS) ||
+		       uverbs_attr_is_valid(attrs, CXI_IB_ATTR_SVC_USAGE) ||
+		       uverbs_attr_is_valid(attrs, CXI_IB_ATTR_SVC_LPR) ? 0 : -EINVAL;
+	if (desc.restricted_vnis) {
+		if ((!desc.num_vld_vnis && !desc.is_system_svc) ||
+		    desc.num_vld_vnis > CXI_IB_SVC_MAX_VNIS)
+			return -EOPNOTSUPP;
+		config.vni_count = desc.num_vld_vnis;
+		if (desc.num_vld_vnis)
+			memcpy(config.vnis, desc.vnis,
+			       desc.num_vld_vnis * sizeof(config.vnis[0]));
+	} else {
+		ret = ctx->cadev->svc_get_vni_range(ctx->cadev->cxi_dev,
+						   config.svc_id, &first, &last);
+		if (ret)
+			return ret;
+		config.is_range = 1;
+		config.vnis[0] = first;
+		config.vnis[1] = last;
+	}
+	for (index = 0; index < CXI_TC_MAX; index++)
+		if (!desc.is_system_svc &&
+		    (!desc.restricted_tcs || desc.tcs[index]))
+			config.tc_mask |= BIT(index);
+	config.uid = desc.members[0].svc_member.uid;
+	for (index = 0; index < ARRAY_SIZE(limits); index++) {
+		config.maximum[index] = limits[index]->max;
+		config.reserved[index] = limits[index]->res;
+	}
+	return uverbs_copy_to(attrs, CXI_IB_ATTR_SVC_CONFIG,
+			     &config, sizeof(config));
+}
+
+DECLARE_UVERBS_NAMED_METHOD(CXI_IB_METHOD_SVC_ALLOC,
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_SVC_CONFIG,
+			   UVERBS_ATTR_TYPE(struct cxi_ibv_svc_config), UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_SVC_ID,
+			    UVERBS_ATTR_TYPE(u32), UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_METHOD(CXI_IB_METHOD_SVC_DESTROY,
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_SVC_ID,
+			   UVERBS_ATTR_TYPE(u32), UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_METHOD(CXI_IB_METHOD_SVC_ENABLE,
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_SVC_ID,
+			   UVERBS_ATTR_TYPE(u32), UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_SVC_FLAGS,
+			   UVERBS_ATTR_TYPE(u32), UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_METHOD(CXI_IB_METHOD_SVC_QUERY,
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_SVC_ID,
+			   UVERBS_ATTR_TYPE(u32), UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_SVC_CONFIG,
+			    UVERBS_ATTR_TYPE(struct cxi_ibv_svc_config), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_SVC_FLAGS,
+			    UVERBS_ATTR_TYPE(u32), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_SVC_USAGE,
+			    UVERBS_ATTR_TYPE(struct cxi_ibv_svc_rsrc_use), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_SVC_LPR,
+			    UVERBS_ATTR_TYPE(u32), UA_OPTIONAL));
+
+DECLARE_UVERBS_GLOBAL_METHODS(CXI_IB_OBJECT_SVC,
+	&UVERBS_METHOD(CXI_IB_METHOD_SVC_ALLOC),
+	&UVERBS_METHOD(CXI_IB_METHOD_SVC_DESTROY),
+	&UVERBS_METHOD(CXI_IB_METHOD_SVC_QUERY),
+	&UVERBS_METHOD(CXI_IB_METHOD_SVC_ENABLE));
+
+struct cxiverbs_wait {
+	struct ib_uobject uobj;
+	struct cxiverbs_lni *lni;
+	wait_queue_head_t poll_wait;
+	struct mutex read_lock;
+	atomic64_t generation;
+	atomic64_t cleared;
+	bool initialized;
+	bool dead;
+};
+
+static ssize_t cxiverbs_wait_read(struct file *file, char __user *buffer,
+				size_t length, loff_t *offset)
+{
+	struct cxiverbs_wait *wait =
+		container_of(file->private_data, struct cxiverbs_wait, uobj);
+	u64 generation;
+	char value = '1';
+	int ret;
+
+	if (!length)
+		return 0;
+	ret = mutex_lock_interruptible(&wait->read_lock);
+	if (ret)
+		return ret;
+	if (READ_ONCE(wait->dead)) {
+		ret = -EIO;
+		goto unlock;
+	}
+	generation = atomic64_read(&wait->generation);
+	if (copy_to_user(buffer, &value, sizeof(value))) {
+		ret = -EFAULT;
+		goto unlock;
+	}
+	atomic64_set(&wait->cleared, generation);
+	ret = sizeof(value);
+unlock:
+	mutex_unlock(&wait->read_lock);
+	return ret;
+}
+
+static __poll_t cxiverbs_wait_poll(struct file *file, poll_table *table)
+{
+	struct cxiverbs_wait *wait =
+		container_of(file->private_data, struct cxiverbs_wait, uobj);
+
+	poll_wait(file, &wait->poll_wait, table);
+	if (READ_ONCE(wait->dead))
+		return EPOLLHUP | EPOLLERR;
+	if (atomic64_read(&wait->generation) != atomic64_read(&wait->cleared))
+		return EPOLLPRI;
+	return 0;
+}
+
+static const struct file_operations cxiverbs_wait_fops = {
+	.owner = THIS_MODULE,
+	.read = cxiverbs_wait_read,
+	.poll = cxiverbs_wait_poll,
+	.release = uverbs_uobject_fd_release,
+	.llseek = noop_llseek,
+};
+
+static void cxiverbs_wait_cleanup(struct ib_uobject *object,
+				 enum rdma_remove_reason reason)
+{
+	struct cxiverbs_wait *wait =
+		container_of(object, struct cxiverbs_wait, uobj);
+
+	if (!wait->initialized)
+		return;
+	WRITE_ONCE(wait->dead, true);
+	wake_up_interruptible_poll(&wait->poll_wait, EPOLLHUP | EPOLLERR);
+	if (wait->lni) {
+		cxiverbs_lni_put(wait->lni);
+		wait->lni = NULL;
+	}
+}
+
+static int UVERBS_HANDLER(CXI_IB_METHOD_WAIT_ALLOC)(
+		struct uverbs_attr_bundle *attrs)
+{
+	struct ib_uobject *object =
+		uverbs_attr_get_uobject(attrs, CXI_IB_ATTR_WAIT_FD);
+	struct file *file = object->object;
+	struct cxiverbs_wait *wait =
+		container_of(object, struct cxiverbs_wait, uobj);
+	struct cxiverbs_ucontext *ctx;
+	u32 handle;
+	int ret;
+
+	init_waitqueue_head(&wait->poll_wait);
+	mutex_init(&wait->read_lock);
+	atomic64_set(&wait->generation, 1);
+	atomic64_set(&wait->cleared, 0);
+	wait->initialized = true;
+	ctx = cxiverbs_get_ucontext(attrs);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+	ret = uverbs_copy_from(&handle, attrs, CXI_IB_ATTR_WAIT_LNI);
+	if (ret)
+		return ret;
+	wait->lni = cxiverbs_lni_get(ctx, handle);
+	if (IS_ERR(wait->lni)) {
+		ret = PTR_ERR(wait->lni);
+		wait->lni = NULL;
+		return ret;
+	}
+	file->f_mode |= FMODE_PREAD;
+	return 0;
+}
+
+DECLARE_UVERBS_NAMED_METHOD(CXI_IB_METHOD_WAIT_ALLOC,
+	UVERBS_ATTR_FD(CXI_IB_ATTR_WAIT_FD, CXI_IB_OBJECT_WAIT,
+		       UVERBS_ACCESS_NEW, UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_WAIT_LNI,
+			   UVERBS_ATTR_TYPE(u32), UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_OBJECT(CXI_IB_OBJECT_WAIT,
+	UVERBS_TYPE_ALLOC_FD(sizeof(struct cxiverbs_wait),
+			     cxiverbs_wait_cleanup,
+			     &cxiverbs_wait_fops, "[cxiwait]", O_RDONLY),
+	&UVERBS_METHOD(CXI_IB_METHOD_WAIT_ALLOC));
+
+static void cxiverbs_wait_notify(void *data)
+{
+	struct file *file = data;
+	struct cxiverbs_wait *wait =
+		container_of(file->private_data, struct cxiverbs_wait, uobj);
+
+	if (READ_ONCE(wait->dead))
+		return;
+	atomic64_inc(&wait->generation);
+	wake_up_interruptible_poll(&wait->poll_wait, EPOLLPRI);
+}
+
+static int cxiverbs_eq_get_wait(struct uverbs_attr_bundle *attrs, u16 attr,
+			       struct cxiverbs_lni *lni, struct file **file)
+{
+	struct ib_uobject *object;
+	struct cxiverbs_wait *wait;
+
+	if (!uverbs_attr_is_valid(attrs, attr))
+		return 0;
+	object = uverbs_attr_get_uobject(attrs, attr);
+	wait = container_of(object, struct cxiverbs_wait, uobj);
+	if (READ_ONCE(wait->dead) || wait->lni != lni)
+		return -EINVAL;
+	*file = get_file(object->object);
+	return 0;
+}
+
+struct cxiverbs_native_eq {
+	struct cxiverbs_lni *lni;
+	struct cxiverbs_mr *mr;
+	struct file *event_wait;
+	struct file *status_wait;
+	struct cxi_eq *eq;
+	struct cxiverbs_mmap_entry *doorbell;
+	atomic_t users;
+};
+
+/*
+ * Runs once the hardware EQ is gone and the last doorbell mapping is closed.
+ * Only drops references and frees memory, so it cannot fail.
+ */
+static void cxiverbs_native_eq_release(void *owner)
+{
+	struct cxiverbs_native_eq *queue = owner;
+
+	if (queue->status_wait)
+		fput(queue->status_wait);
+	if (queue->event_wait)
+		fput(queue->event_wait);
+	if (queue->mr)
+		atomic_dec(&queue->mr->native_users);
+	cxiverbs_lni_put(queue->lni);
+	kfree(queue);
+}
+
+static int cxiverbs_native_eq_cleanup(struct ib_uobject *object,
+				     enum rdma_remove_reason reason,
+				     struct uverbs_attr_bundle *attrs)
+{
+	struct cxiverbs_native_eq *queue = object->object;
+	int ret;
+
+	if (atomic_read(&queue->users))
+		return -EBUSY;
+	/*
+	 * eq_free() can fail (a VF may be unable to reach the PF) and leaves
+	 * the EQ untouched when it does. Do it here, where rdma-core can
+	 * report the error and retry, rather than from mmap_free(), which
+	 * has no way to fail.
+	 */
+	ret = queue->lni->ctx->cadev->eq_free(queue->eq);
+	if (ret)
+		return ret;
+	/*
+	 * The queue, and the LNI reference it holds, stay alive until the last
+	 * doorbell mapping is closed, so the EQ number cannot be recycled
+	 * while the page is still mapped.
+	 */
+	if (queue->doorbell)
+		rdma_user_mmap_entry_remove(&queue->doorbell->rdma_entry);
+	else
+		cxiverbs_native_eq_release(queue);
+	return 0;
+}
+
+static int UVERBS_HANDLER(CXI_IB_METHOD_EQ_ALLOC)(struct uverbs_attr_bundle *attrs)
+{
+	struct cxiverbs_ucontext *ctx = cxiverbs_get_ucontext(attrs);
+	struct ib_uobject *object = uverbs_attr_get_uobject(attrs, CXI_IB_ATTR_EQ_HANDLE);
+	struct ib_mr *ibmr = NULL;
+	struct cxiverbs_mr *mr = NULL;
+	struct cxiverbs_native_eq *queue;
+	struct cxi_ibv_eq_config config;
+	struct cxi_ibv_eq_info info = {};
+	struct cxi_eq_attr native = {};
+	phys_addr_t physical;
+	size_t size;
+	u64 offset;
+	u32 lni_handle;
+	u32 flags = 0;
+	bool passthrough;
+	int ret;
+
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+	ret = uverbs_get_flags32(&flags, attrs, CXI_IB_ATTR_EQ_FLAGS,
+				CXI_IB_EQ_REGISTER_PCT | CXI_IB_EQ_PASSTHROUGH);
+	if (ret)
+		return ret;
+	passthrough = flags & CXI_IB_EQ_PASSTHROUGH;
+	if (passthrough && !(flags & CXI_IB_EQ_REGISTER_PCT))
+		return -EINVAL;
+	if (passthrough == uverbs_attr_is_valid(attrs, CXI_IB_ATTR_EQ_MR))
+		return -EINVAL;
+	if (!passthrough) {
+		ibmr = uverbs_attr_get_obj(attrs, CXI_IB_ATTR_EQ_MR);
+		mr = to_cxiverbs_mr(ibmr);
+	}
+	if (flags & CXI_IB_EQ_REGISTER_PCT) {
+		struct cxi_dev *cdev = ctx->cadev->cxi_dev;
+		u32 generation = cdev->prop.cassini_version & 0xf0;
+
+		if (!capable(CAP_SYS_ADMIN))
+			return -EPERM;
+		if (!cdev->is_physfn ||
+		    (generation != CASSINI_1 && generation != CASSINI_2))
+			return -EOPNOTSUPP;
+	}
+	ret = uverbs_copy_from(&lni_handle, attrs, CXI_IB_ATTR_EQ_LNI);
+	if (ret)
+		return ret;
+	ret = uverbs_copy_from(&config, attrs, CXI_IB_ATTR_EQ_CONFIG);
+	if (ret)
+		return ret;
+	if (!config.length)
+		return -EINVAL;
+	if (mr && (config.address < mr->md->va ||
+		   config.length > mr->md->len ||
+		   config.address - mr->md->va > mr->md->len - config.length))
+		return -EINVAL;
+	queue = kzalloc_obj(*queue, GFP_KERNEL);
+	if (!queue)
+		return -ENOMEM;
+	queue->lni = cxiverbs_lni_get(ctx, lni_handle);
+	if (IS_ERR(queue->lni)) {
+		ret = PTR_ERR(queue->lni);
+		goto free_queue;
+	}
+	if (mr && to_cxiverbs_pd(ibmr->pd)->lni_ref != queue->lni) {
+		ret = -EINVAL;
+		goto put_lni;
+	}
+	queue->mr = mr;
+	if (mr)
+		atomic_inc(&mr->native_users);
+	native.queue = (void *)(uintptr_t)config.address;
+	native.queue_len = config.length;
+	native.flags = CXI_EQ_USER;
+	if (flags & CXI_IB_EQ_REGISTER_PCT)
+		native.flags |= CXI_EQ_REGISTER_PCT;
+	if (passthrough)
+		native.flags |= CXI_EQ_PASSTHROUGH;
+	native.status_thresh_base = config.threshold_base;
+	native.status_thresh_delta = config.threshold_delta;
+	native.status_thresh_count = config.threshold_count;
+	native.ec_delay = config.ec_delay;
+	native.cpu_affinity = config.cpu;
+	native.reserved_slots = config.reserved_slots;
+	ret = cxiverbs_eq_get_wait(attrs, CXI_IB_ATTR_EQ_EVENT_WAIT,
+				   queue->lni, &queue->event_wait);
+	if (ret)
+		goto put_waits;
+	ret = cxiverbs_eq_get_wait(attrs, CXI_IB_ATTR_EQ_STATUS_WAIT,
+				   queue->lni, &queue->status_wait);
+	if (ret)
+		goto put_waits;
+	queue->eq = ctx->cadev->eq_alloc(queue->lni->lni, mr ? mr->md : NULL,
+		&native, queue->event_wait ? cxiverbs_wait_notify : NULL,
+		queue->event_wait,
+		queue->status_wait ? cxiverbs_wait_notify : NULL,
+		queue->status_wait);
+	if (IS_ERR(queue->eq)) {
+		ret = PTR_ERR(queue->eq);
+		goto put_waits;
+	}
+	ret = ctx->cadev->eq_user_info(queue->eq, &physical, &size);
+	if (ret)
+		goto destroy_eq;
+	if (flags & CXI_IB_EQ_REGISTER_PCT)
+		queue->doorbell = cxiverbs_mmap_entry_alloc_admin(
+			&ctx->ibucontext, physical, size, &offset);
+	else
+		queue->doorbell = cxiverbs_mmap_entry_alloc(
+			&ctx->ibucontext, true, physical, NULL, size, &offset);
+	if (IS_ERR(queue->doorbell)) {
+		ret = PTR_ERR(queue->doorbell);
+		queue->doorbell = NULL;
+		goto destroy_eq;
+	}
+	queue->doorbell->owner = queue;
+	queue->doorbell->release_owner = cxiverbs_native_eq_release;
+	info.doorbell_offset = offset;
+	info.doorbell_size = size;
+	info.eqn = queue->eq->eqn;
+	object->object = queue;
+	uverbs_finalize_uobj_create(attrs, CXI_IB_ATTR_EQ_HANDLE);
+	return uverbs_copy_to(attrs, CXI_IB_ATTR_EQ_INFO, &info, sizeof(info));
+destroy_eq:
+	/*
+	 * The hardware EQ exists, so let rdma-core unwind it through
+	 * cxiverbs_native_eq_cleanup(). If eq_free() fails there, the core
+	 * keeps the object and retries instead of leaking it.
+	 */
+	object->object = queue;
+	uverbs_finalize_uobj_create(attrs, CXI_IB_ATTR_EQ_HANDLE);
+	return ret;
+put_waits:
+	if (queue->status_wait)
+		fput(queue->status_wait);
+	if (queue->event_wait)
+		fput(queue->event_wait);
+	if (mr)
+		atomic_dec(&mr->native_users);
+put_lni:
+	cxiverbs_lni_put(queue->lni);
+free_queue:
+	kfree(queue);
+	return ret;
+}
+
+DECLARE_UVERBS_NAMED_METHOD(CXI_IB_METHOD_EQ_ALLOC,
+	UVERBS_ATTR_IDR(CXI_IB_ATTR_EQ_HANDLE, CXI_IB_OBJECT_EQ,
+			UVERBS_ACCESS_NEW, UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_EQ_LNI, UVERBS_ATTR_TYPE(u32), UA_MANDATORY),
+	UVERBS_ATTR_IDR(CXI_IB_ATTR_EQ_MR, UVERBS_OBJECT_MR,
+			UVERBS_ACCESS_READ, UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_EQ_CONFIG,
+			   UVERBS_ATTR_TYPE(struct cxi_ibv_eq_config), UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_EQ_INFO,
+			    UVERBS_ATTR_TYPE(struct cxi_ibv_eq_info),
+			    UA_MANDATORY),
+	UVERBS_ATTR_FLAGS_IN(CXI_IB_ATTR_EQ_FLAGS, u32, UA_OPTIONAL),
+	UVERBS_ATTR_FD(CXI_IB_ATTR_EQ_EVENT_WAIT, CXI_IB_OBJECT_WAIT,
+		       UVERBS_ACCESS_READ, UA_OPTIONAL),
+	UVERBS_ATTR_FD(CXI_IB_ATTR_EQ_STATUS_WAIT, CXI_IB_OBJECT_WAIT,
+		       UVERBS_ACCESS_READ, UA_OPTIONAL));
+
+DECLARE_UVERBS_NAMED_METHOD_DESTROY(CXI_IB_METHOD_EQ_DESTROY,
+	UVERBS_ATTR_IDR(CXI_IB_ATTR_EQ_HANDLE, CXI_IB_OBJECT_EQ,
+			UVERBS_ACCESS_DESTROY, UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_OBJECT(CXI_IB_OBJECT_EQ,
+	UVERBS_TYPE_ALLOC_IDR(cxiverbs_native_eq_cleanup),
+	&UVERBS_METHOD(CXI_IB_METHOD_EQ_ALLOC),
+	&UVERBS_METHOD(CXI_IB_METHOD_EQ_DESTROY));
+
+struct cxiverbs_cp {
+	struct cxiverbs_lni *lni;
+	struct cxi_cp *cp;
+};
+
+static int cxiverbs_cp_cleanup(struct ib_uobject *object,
+			      enum rdma_remove_reason reason,
+			      struct uverbs_attr_bundle *attrs)
+{
+	struct cxiverbs_cp *profile = object->object;
+
+	profile->lni->ctx->cadev->cp_free(profile->cp);
+	cxiverbs_lni_put(profile->lni);
+	kfree(profile);
+	return 0;
+}
+
+static int UVERBS_HANDLER(CXI_IB_METHOD_CP_ALLOC)(struct uverbs_attr_bundle *attrs)
+{
+	struct ib_uobject *object = uverbs_attr_get_uobject(attrs, CXI_IB_ATTR_CP_HANDLE);
+	struct cxiverbs_ucontext *ctx = cxiverbs_get_ucontext(attrs);
+	struct cxiverbs_cp *profile;
+	struct cxi_ibv_cp_info info;
+	u32 lni_handle, vni, tc, type;
+	int ret;
+
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+	ret = uverbs_copy_from(&lni_handle, attrs, CXI_IB_ATTR_CP_LNI);
+	if (ret)
+		return ret;
+	ret = uverbs_copy_from(&vni, attrs, CXI_IB_ATTR_CP_VNI);
+	if (ret)
+		return ret;
+	ret = uverbs_copy_from(&tc, attrs, CXI_IB_ATTR_CP_TC);
+	if (ret)
+		return ret;
+	ret = uverbs_copy_from(&type, attrs, CXI_IB_ATTR_CP_TYPE);
+	if (ret)
+		return ret;
+	if (!vni || vni > U16_MAX || tc >= CXI_TC_MAX || type >= CXI_TC_TYPE_MAX)
+		return -EINVAL;
+	profile = kzalloc_obj(*profile, GFP_KERNEL);
+	if (!profile)
+		return -ENOMEM;
+	profile->lni = cxiverbs_lni_get(ctx, lni_handle);
+	if (IS_ERR(profile->lni)) {
+		ret = PTR_ERR(profile->lni);
+		goto free_profile;
+	}
+	profile->cp = ctx->cadev->cp_alloc(profile->lni->lni, vni, tc, type);
+	if (IS_ERR(profile->cp)) {
+		ret = PTR_ERR(profile->cp);
+		goto put_lni;
+	}
+	info.vni = profile->cp->vni_pcp;
+	info.tc = profile->cp->tc;
+	info.type = profile->cp->tc_type;
+	info.lcid = profile->cp->lcid;
+	object->object = profile;
+	uverbs_finalize_uobj_create(attrs, CXI_IB_ATTR_CP_HANDLE);
+	return uverbs_copy_to(attrs, CXI_IB_ATTR_CP_INFO, &info, sizeof(info));
+put_lni:
+	cxiverbs_lni_put(profile->lni);
+free_profile:
+	kfree(profile);
+	return ret;
+}
+
+DECLARE_UVERBS_NAMED_METHOD(CXI_IB_METHOD_CP_ALLOC,
+	UVERBS_ATTR_IDR(CXI_IB_ATTR_CP_HANDLE, CXI_IB_OBJECT_CP,
+			UVERBS_ACCESS_NEW, UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_CP_LNI, UVERBS_ATTR_TYPE(u32), UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_CP_VNI, UVERBS_ATTR_TYPE(u32), UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_CP_TC, UVERBS_ATTR_TYPE(u32), UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_CP_TYPE, UVERBS_ATTR_TYPE(u32), UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_CP_INFO,
+			    UVERBS_ATTR_TYPE(struct cxi_ibv_cp_info), UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_METHOD_DESTROY(CXI_IB_METHOD_CP_DESTROY,
+	UVERBS_ATTR_IDR(CXI_IB_ATTR_CP_HANDLE, CXI_IB_OBJECT_CP,
+			UVERBS_ACCESS_DESTROY, UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_OBJECT(CXI_IB_OBJECT_CP,
+	UVERBS_TYPE_ALLOC_IDR(cxiverbs_cp_cleanup),
+	&UVERBS_METHOD(CXI_IB_METHOD_CP_ALLOC),
+	&UVERBS_METHOD(CXI_IB_METHOD_CP_DESTROY));
+
+/* -----------------------------------------------------------------------
+ * Per-CT kernel object.  Heap-allocated; freed in cxiverbs_ct_cleanup().
+ * -----------------------------------------------------------------------
+ */
+struct cxiverbs_ct {
+	refcount_t refs;
+	struct cxi_ct            *ct;       /* CXI counting trigger          */
+	struct cxiverbs_lni *lni_ref;
+	struct page              *wb_page;  /* writeback-buffer page         */
+	struct cxiverbs_mmap_entry *wb_mmap;  /* userspace mmap: writeback buf */
+	struct cxiverbs_mmap_entry *db_mmap;  /* userspace mmap: CT doorbell   */
+};
+
+static void cxiverbs_ct_release(void *owner)
+{
+	struct cxiverbs_ct *ct = owner;
+
+	if (!refcount_dec_and_test(&ct->refs))
+		return;
+	if (ct->ct)
+		cxi_ct_free(ct->ct);
+	if (ct->wb_page)
+		unpin_user_page(ct->wb_page);
+	cxiverbs_lni_put(ct->lni_ref);
+	kfree(ct);
+}
+
+/* -----------------------------------------------------------------------
+ * IDR object cleanup — called on CT_DESTROY and on context teardown.
+ * -----------------------------------------------------------------------
+ */
+static int cxiverbs_ct_cleanup(struct ib_uobject *uobj,
+			      enum rdma_remove_reason why,
+			      struct uverbs_attr_bundle *attrs)
+{
+	struct cxiverbs_ct *ct = uobj->object;
+
+	if (ct->db_mmap)
+		rdma_user_mmap_entry_remove(&ct->db_mmap->rdma_entry);
+	if (ct->wb_mmap)
+		rdma_user_mmap_entry_remove(&ct->wb_mmap->rdma_entry);
+	cxiverbs_ct_release(ct);
+	return 0;
+}
+
+/* -----------------------------------------------------------------------
+ * CXI_IB_METHOD_CT_ALLOC handler
+ * -----------------------------------------------------------------------
+ */
+static int UVERBS_HANDLER(CXI_IB_METHOD_CT_ALLOC)(
+		struct uverbs_attr_bundle *attrs)
+{
+	struct ib_uobject  *uobj   = uverbs_attr_get_uobject(attrs,
+					CXI_IB_ATTR_CT_ALLOC_HANDLE);
+	struct ib_ucontext *ibctx  = ib_uverbs_get_ucontext(attrs);
+	struct cxiverbs_ucontext *uctx;
+	struct cxi_adev *cadev;
+	struct cxiverbs_ct   *ct_obj;
+	struct c_ct_writeback *wb;
+	phys_addr_t db_phys;
+	size_t      db_sz;
+	u64         wb_offset = 0, db_offset = 0;
+	u64         wb_len, db_len;
+	u64 wb_address;
+	u32         ctn;
+	u32 lni_handle = CXI_IBV_LNI_ANY;
+	int         ret;
+
+	if (IS_ERR(ibctx))
+		return PTR_ERR(ibctx);
+
+	uctx = to_cxiverbs_ucontext(ibctx);
+	cadev = uctx->cadev;
+	ret = uverbs_copy_from(&wb_address, attrs,
+			       CXI_IB_ATTR_CT_ALLOC_WB_ADDRESS);
+	if (ret)
+		return ret;
+	if (!wb_address || !IS_ALIGNED(wb_address, PAGE_SIZE) ||
+	    wb_address != (unsigned long)wb_address ||
+	    !access_ok(u64_to_user_ptr(wb_address), PAGE_SIZE))
+		return -EINVAL;
+
+	ct_obj = kzalloc_obj(*ct_obj);
+	if (!ct_obj)
+		return -ENOMEM;
+	if (uverbs_attr_is_valid(attrs, CXI_IB_ATTR_CT_ALLOC_LNI_HANDLE)) {
+		ret = uverbs_copy_from(&lni_handle, attrs,
+				       CXI_IB_ATTR_CT_ALLOC_LNI_HANDLE);
+		if (ret)
+			goto err_free_object;
+	}
+	ct_obj->lni_ref = cxiverbs_lni_get(uctx, lni_handle);
+	if (IS_ERR(ct_obj->lni_ref)) {
+		ret = PTR_ERR(ct_obj->lni_ref);
+		goto err_free_object;
+	}
+	refcount_set(&ct_obj->refs, 1);
+
+	ret = pin_user_pages_fast(wb_address, 1, FOLL_WRITE | FOLL_LONGTERM,
+				  &ct_obj->wb_page);
+	if (ret != 1) {
+		ct_obj->wb_page = NULL;
+		if (ret >= 0)
+			ret = -EFAULT;
+		goto err_release;
+	}
+	wb = (struct c_ct_writeback *)(unsigned long)wb_address;
+
+	ct_obj->ct = cadev->ct_alloc(ct_obj->lni_ref->lni, wb, true);
+	if (IS_ERR(ct_obj->ct)) {
+		ret = PTR_ERR(ct_obj->ct);
+		ct_obj->ct = NULL;
+		goto err_release;
+	}
+
+	/* Get the physical address of the CT doorbell CSR for mmap. */
+	ret = cxi_ct_user_info(ct_obj->ct, &db_phys, &db_sz);
+	if (ret)
+		goto err_release;
+
+	wb_len = PAGE_SIZE;
+	db_len = (u64)db_sz;
+
+	/* Create mmap entry for the writeback buffer page. */
+	refcount_inc(&ct_obj->refs);
+	ct_obj->wb_mmap = cxiverbs_mmap_entry_alloc_owned(ibctx, false, 0,
+				ct_obj->wb_page, wb_len, &wb_offset, ct_obj,
+				cxiverbs_ct_release);
+	if (IS_ERR(ct_obj->wb_mmap)) {
+		ret = PTR_ERR(ct_obj->wb_mmap);
+		ct_obj->wb_mmap = NULL;
+		refcount_dec(&ct_obj->refs);
+		goto err_release;
+	}
+
+	/* Create mmap entry for the CT doorbell CSR (IO mapping). */
+	refcount_inc(&ct_obj->refs);
+	ct_obj->db_mmap = cxiverbs_mmap_entry_alloc_owned(ibctx, true, db_phys,
+				NULL, db_len, &db_offset, ct_obj,
+				cxiverbs_ct_release);
+	if (IS_ERR(ct_obj->db_mmap)) {
+		ret = PTR_ERR(ct_obj->db_mmap);
+		ct_obj->db_mmap = NULL;
+		refcount_dec(&ct_obj->refs);
+		goto err_remove_wb;
+	}
+
+	ctn = ct_obj->ct->ctn;
+
+	/* Commit the uobject — after this, returning an error will cause the
+	 * framework to call cxiverbs_ct_cleanup() automatically.
+	 */
+	uobj->object = ct_obj;
+	uverbs_finalize_uobj_create(attrs, CXI_IB_ATTR_CT_ALLOC_HANDLE);
+
+	ret = uverbs_copy_to(attrs, CXI_IB_ATTR_CT_ALLOC_CTN,
+			     &ctn, sizeof(ctn));
+	if (ret)
+		return ret;
+
+	ret = uverbs_copy_to(attrs, CXI_IB_ATTR_CT_ALLOC_WB_MMAP_OFFSET,
+			     &wb_offset, sizeof(wb_offset));
+	if (ret)
+		return ret;
+
+	ret = uverbs_copy_to(attrs, CXI_IB_ATTR_CT_ALLOC_WB_MMAP_LENGTH,
+			     &wb_len, sizeof(wb_len));
+	if (ret)
+		return ret;
+
+	ret = uverbs_copy_to(attrs, CXI_IB_ATTR_CT_ALLOC_DB_MMAP_OFFSET,
+			     &db_offset, sizeof(db_offset));
+	if (ret)
+		return ret;
+
+	ret = uverbs_copy_to(attrs, CXI_IB_ATTR_CT_ALLOC_DB_MMAP_LENGTH,
+			     &db_len, sizeof(db_len));
+	return ret;
+
+err_remove_wb:
+	rdma_user_mmap_entry_remove(&ct_obj->wb_mmap->rdma_entry);
+err_release:
+	cxiverbs_ct_release(ct_obj);
+	return ret;
+err_free_object:
+	kfree(ct_obj);
+	return ret;
+}
+
+/* -----------------------------------------------------------------------
+ * Method definitions
+ * -----------------------------------------------------------------------
+ */
+DECLARE_UVERBS_NAMED_METHOD(
+	CXI_IB_METHOD_CT_ALLOC,
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_CT_ALLOC_WB_ADDRESS,
+			   UVERBS_ATTR_TYPE(u64), UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_CT_ALLOC_LNI_HANDLE,
+			   UVERBS_ATTR_TYPE(u32), UA_OPTIONAL),
+	UVERBS_ATTR_IDR(CXI_IB_ATTR_CT_ALLOC_HANDLE,
+			CXI_IB_OBJECT_CT,
+			UVERBS_ACCESS_NEW,
+			UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_CT_ALLOC_CTN,
+			    UVERBS_ATTR_TYPE(u32),
+			    UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_CT_ALLOC_WB_MMAP_OFFSET,
+			    UVERBS_ATTR_TYPE(u64),
+			    UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_CT_ALLOC_WB_MMAP_LENGTH,
+			    UVERBS_ATTR_TYPE(u64),
+			    UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_CT_ALLOC_DB_MMAP_OFFSET,
+			    UVERBS_ATTR_TYPE(u64),
+			    UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_CT_ALLOC_DB_MMAP_LENGTH,
+			    UVERBS_ATTR_TYPE(u64),
+			    UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_METHOD_DESTROY(
+	CXI_IB_METHOD_CT_DESTROY,
+	UVERBS_ATTR_IDR(CXI_IB_ATTR_CT_DESTROY_HANDLE,
+			CXI_IB_OBJECT_CT,
+			UVERBS_ACCESS_DESTROY,
+			UA_MANDATORY));
+
+/* -----------------------------------------------------------------------
+ * Object definition
+ * -----------------------------------------------------------------------
+ */
+DECLARE_UVERBS_NAMED_OBJECT(
+	CXI_IB_OBJECT_CT,
+	UVERBS_TYPE_ALLOC_IDR(cxiverbs_ct_cleanup),
+	&UVERBS_METHOD(CXI_IB_METHOD_CT_ALLOC),
+	&UVERBS_METHOD(CXI_IB_METHOD_CT_DESTROY));
+
+struct cxiverbs_cmdq {
+	refcount_t refs;
+	struct cxiverbs_lni *lni;
+	struct cxi_cq *cq;
+	struct cxiverbs_mmap_entry *commands;
+	struct cxiverbs_mmap_entry *doorbell;
+};
+
+static void cxiverbs_cmdq_release(void *owner)
+{
+	struct cxiverbs_cmdq *cmdq = owner;
+
+	if (!refcount_dec_and_test(&cmdq->refs))
+		return;
+	if (cmdq->cq)
+		cmdq->lni->ctx->cadev->cq_free(cmdq->cq);
+	cxiverbs_lni_put(cmdq->lni);
+	kfree(cmdq);
+}
+
+static int cxiverbs_cmdq_cleanup(struct ib_uobject *object,
+				enum rdma_remove_reason reason,
+				struct uverbs_attr_bundle *attrs)
+{
+	struct cxiverbs_cmdq *cmdq = object->object;
+
+	if (cmdq->doorbell)
+		rdma_user_mmap_entry_remove(&cmdq->doorbell->rdma_entry);
+	if (cmdq->commands)
+		rdma_user_mmap_entry_remove(&cmdq->commands->rdma_entry);
+	cxiverbs_cmdq_release(cmdq);
+	return 0;
+}
+
+static int UVERBS_HANDLER(CXI_IB_METHOD_CMDQ_ALLOC)(
+		struct uverbs_attr_bundle *attrs)
+{
+	struct ib_uobject *object =
+		uverbs_attr_get_uobject(attrs, CXI_IB_ATTR_CMDQ_HANDLE);
+	struct cxi_cq_alloc_opts opts = { .flags = CXI_CQ_USER };
+	struct cxi_ibv_cmdq_info info = {};
+	struct cxiverbs_ucontext *ctx;
+	struct cxiverbs_cmdq *cmdq;
+	struct page *pages;
+	phys_addr_t doorbell;
+	size_t commands_size;
+	size_t doorbell_size;
+	u64 offset;
+	u32 handle;
+	u32 flags = 0;
+	int ret;
+
+	ctx = cxiverbs_get_ucontext(attrs);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+	if (!ctx->cadev->cq_alloc || !ctx->cadev->cq_free ||
+	    !ctx->cadev->cq_user_info)
+		return -EOPNOTSUPP;
+	ret = uverbs_copy_from(&handle, attrs, CXI_IB_ATTR_CMDQ_LNI);
+	if (ret)
+		return ret;
+	ret = uverbs_copy_from(&opts.count, attrs, CXI_IB_ATTR_CMDQ_COUNT);
+	if (ret)
+		return ret;
+	if (!opts.count || opts.count > CXI_MAX_CQ_COUNT)
+		return -EINVAL;
+	if (uverbs_attr_is_valid(attrs, CXI_IB_ATTR_CMDQ_FLAGS)) {
+		ret = uverbs_copy_from(&flags, attrs, CXI_IB_ATTR_CMDQ_FLAGS);
+		if (ret)
+			return ret;
+	}
+	if (flags & ~CXI_IB_CMDQ_TRIGGER)
+		return -EINVAL;
+	if (flags & CXI_IB_CMDQ_TRIGGER)
+		opts.flags |= CXI_CQ_IS_TX | CXI_CQ_TX_WITH_TRIG_CMDS;
+	cmdq = kzalloc_obj(*cmdq, GFP_KERNEL);
+	if (!cmdq)
+		return -ENOMEM;
+	cmdq->lni = cxiverbs_lni_get(ctx, handle);
+	if (IS_ERR(cmdq->lni)) {
+		ret = PTR_ERR(cmdq->lni);
+		kfree(cmdq);
+		return ret;
+	}
+	refcount_set(&cmdq->refs, 1);
+	cmdq->cq = ctx->cadev->cq_alloc(cmdq->lni->lni, NULL, &opts,
+				       NUMA_NO_NODE);
+	if (IS_ERR(cmdq->cq)) {
+		ret = PTR_ERR(cmdq->cq);
+		cmdq->cq = NULL;
+		goto release;
+	}
+	ret = ctx->cadev->cq_user_info(cmdq->cq, &commands_size, &pages,
+				      &doorbell, &doorbell_size);
+	if (ret)
+		goto release;
+	info.count = cmdq->cq->size;
+	info.index = cmdq->cq->idx;
+	info.commands_length = commands_size;
+	info.doorbell_length = doorbell_size;
+	refcount_inc(&cmdq->refs);
+	cmdq->commands = cxiverbs_mmap_entry_alloc_owned(&ctx->ibucontext,
+			false, 0, pages, commands_size, &offset, cmdq,
+			cxiverbs_cmdq_release);
+	if (IS_ERR(cmdq->commands)) {
+		ret = PTR_ERR(cmdq->commands);
+		cmdq->commands = NULL;
+		refcount_dec(&cmdq->refs);
+		goto release;
+	}
+	info.commands_offset = offset;
+	refcount_inc(&cmdq->refs);
+	cmdq->doorbell = cxiverbs_mmap_entry_alloc_owned(&ctx->ibucontext,
+			true, doorbell, NULL, doorbell_size, &offset, cmdq,
+			cxiverbs_cmdq_release);
+	if (IS_ERR(cmdq->doorbell)) {
+		ret = PTR_ERR(cmdq->doorbell);
+		cmdq->doorbell = NULL;
+		refcount_dec(&cmdq->refs);
+		rdma_user_mmap_entry_remove(&cmdq->commands->rdma_entry);
+		goto release;
+	}
+	info.doorbell_offset = offset;
+	object->object = cmdq;
+	uverbs_finalize_uobj_create(attrs, CXI_IB_ATTR_CMDQ_HANDLE);
+	return uverbs_copy_to(attrs, CXI_IB_ATTR_CMDQ_INFO,
+			     &info, sizeof(info));
+release:
+	cxiverbs_cmdq_release(cmdq);
+	return ret;
+}
+
+DECLARE_UVERBS_NAMED_METHOD(CXI_IB_METHOD_CMDQ_ALLOC,
+	UVERBS_ATTR_IDR(CXI_IB_ATTR_CMDQ_HANDLE, CXI_IB_OBJECT_CMDQ,
+		       UVERBS_ACCESS_NEW, UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_CMDQ_LNI,
+			   UVERBS_ATTR_TYPE(u32), UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_CMDQ_COUNT,
+			   UVERBS_ATTR_TYPE(u32), UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_CMDQ_INFO,
+			    UVERBS_ATTR_TYPE(struct cxi_ibv_cmdq_info),
+			    UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_CMDQ_FLAGS, UVERBS_ATTR_TYPE(u32)));
+
+DECLARE_UVERBS_NAMED_METHOD_DESTROY(CXI_IB_METHOD_CMDQ_DESTROY,
+	UVERBS_ATTR_IDR(CXI_IB_ATTR_CMDQ_HANDLE, CXI_IB_OBJECT_CMDQ,
+		       UVERBS_ACCESS_DESTROY, UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_OBJECT(CXI_IB_OBJECT_CMDQ,
+	UVERBS_TYPE_ALLOC_IDR(cxiverbs_cmdq_cleanup),
+	&UVERBS_METHOD(CXI_IB_METHOD_CMDQ_ALLOC),
+	&UVERBS_METHOD(CXI_IB_METHOD_CMDQ_DESTROY));
+
+static int cxiverbs_copy_job_tc_limit(struct uverbs_attr_bundle *attrs,
+				    int max_attr, int res_attr,
+				    struct cxi_limits *limit)
+{
+	int ret;
+
+	if (uverbs_attr_is_valid(attrs, max_attr)) {
+		ret = uverbs_copy_from(&limit->max, attrs, max_attr);
+		if (ret)
+			return ret;
+	}
+	if (uverbs_attr_is_valid(attrs, res_attr)) {
+		ret = uverbs_copy_from(&limit->res, attrs, res_attr);
+		if (ret)
+			return ret;
+	}
+
+	return limit->max < limit->res ? -EINVAL : 0;
+}
+
+static u32 cxiverbs_tc_key(u32 tc, u32 tc_type)
+{
+	return (tc_type << 16) | tc;
+}
+
+static int UVERBS_HANDLER(CXI_IB_METHOD_JOB_TC_BIND)(
+		struct uverbs_attr_bundle *attrs)
+{
+	struct ib_ucontext *ibctx = ib_uverbs_get_ucontext(attrs);
+	struct cxiverbs_ucontext *uctx;
+	struct cxiverbs_dev *dev;
+	struct cxiverbs_job_entry *job;
+	struct cxiverbs_job_tc_binding *binding = NULL;
+	struct cxiverbs_tc_entry *tc_entry = NULL;
+	struct cxi_rsrc_limits limits = {};
+	u32 flags = 0;
+	u32 tc_key;
+	u32 job_id;
+	u32 svc_id;
+	u32 bound_svc_id;
+	u32 tc;
+	u32 tc_type = CXI_TC_TYPE_DEFAULT;
+	bool create_tc;
+	bool created_tc = false;
+	int ret;
+
+	if (IS_ERR(ibctx))
+		return PTR_ERR(ibctx);
+	if (!capable(CAP_NET_ADMIN))
+		return -EPERM;
+
+	uctx = to_cxiverbs_ucontext(ibctx);
+	dev = to_cxiverbs_dev(ibctx->device);
+	if (!uctx->cadev->job_tc_bind)
+		return -EOPNOTSUPP;
+
+	ret = uverbs_get_flags32(&flags, attrs,
+				  CXI_IB_ATTR_JOB_TC_BIND_FLAGS,
+				  CXI_IB_JOB_TC_F_CREATE);
+	if (ret)
+		return ret;
+
+	ret = uverbs_copy_from(&job_id, attrs, CXI_IB_ATTR_JOB_TC_BIND_JOB_ID);
+	if (ret)
+		return ret;
+	ret = uverbs_copy_from(&tc, attrs, CXI_IB_ATTR_JOB_TC_BIND_TC);
+	if (ret)
+		return ret;
+
+	svc_id = 0;
+	if (uverbs_attr_is_valid(attrs, CXI_IB_ATTR_JOB_TC_BIND_SVC_ID)) {
+		ret = uverbs_copy_from(&svc_id, attrs,
+				       CXI_IB_ATTR_JOB_TC_BIND_SVC_ID);
+		if (ret)
+			return ret;
+	}
+	if (uverbs_attr_is_valid(attrs, CXI_IB_ATTR_JOB_TC_BIND_TC_TYPE)) {
+		ret = uverbs_copy_from(&tc_type, attrs,
+				       CXI_IB_ATTR_JOB_TC_BIND_TC_TYPE);
+		if (ret)
+			return ret;
+	}
+
+	ret = cxiverbs_copy_job_tc_limit(attrs,
+			CXI_IB_ATTR_JOB_TC_BIND_PTES_MAX,
+			CXI_IB_ATTR_JOB_TC_BIND_PTES_RES,
+			&limits.ptes);
+	if (ret)
+		return ret;
+	ret = cxiverbs_copy_job_tc_limit(attrs,
+			CXI_IB_ATTR_JOB_TC_BIND_TXQS_MAX,
+			CXI_IB_ATTR_JOB_TC_BIND_TXQS_RES,
+			&limits.txqs);
+	if (ret)
+		return ret;
+	ret = cxiverbs_copy_job_tc_limit(attrs,
+			CXI_IB_ATTR_JOB_TC_BIND_TGQS_MAX,
+			CXI_IB_ATTR_JOB_TC_BIND_TGQS_RES,
+			&limits.tgqs);
+	if (ret)
+		return ret;
+	ret = cxiverbs_copy_job_tc_limit(attrs,
+			CXI_IB_ATTR_JOB_TC_BIND_EQS_MAX,
+			CXI_IB_ATTR_JOB_TC_BIND_EQS_RES,
+			&limits.eqs);
+	if (ret)
+		return ret;
+	ret = cxiverbs_copy_job_tc_limit(attrs,
+			CXI_IB_ATTR_JOB_TC_BIND_CTS_MAX,
+			CXI_IB_ATTR_JOB_TC_BIND_CTS_RES,
+			&limits.cts);
+	if (ret)
+		return ret;
+	ret = cxiverbs_copy_job_tc_limit(attrs,
+			CXI_IB_ATTR_JOB_TC_BIND_LES_MAX,
+			CXI_IB_ATTR_JOB_TC_BIND_LES_RES,
+			&limits.les);
+	if (ret)
+		return ret;
+	ret = cxiverbs_copy_job_tc_limit(attrs,
+			CXI_IB_ATTR_JOB_TC_BIND_TLES_MAX,
+			CXI_IB_ATTR_JOB_TC_BIND_TLES_RES,
+			&limits.tles);
+	if (ret)
+		return ret;
+	ret = cxiverbs_copy_job_tc_limit(attrs,
+			CXI_IB_ATTR_JOB_TC_BIND_ACS_MAX,
+			CXI_IB_ATTR_JOB_TC_BIND_ACS_RES,
+			&limits.acs);
+	if (ret)
+		return ret;
+
+	mutex_lock(&dev->jobs_lock);
+	job = uctx->job;
+	if (!job || job->job_id != job_id) {
+		ret = -EINVAL;
+		goto out_unlock;
+	}
+
+	create_tc = flags & CXI_IB_JOB_TC_F_CREATE;
+	tc_key = cxiverbs_tc_key(tc, tc_type);
+	tc_entry = xa_load(&dev->tcs, tc_key);
+	if (create_tc) {
+		if (tc_entry) {
+			ret = -EEXIST;
+			goto out_unlock;
+		}
+	}
+
+	if (tc_entry) {
+		list_for_each_entry(binding, &tc_entry->job_list, entry) {
+			if (binding->job == job) {
+				ret = -EEXIST;
+				goto out_unlock;
+			}
+		}
+		binding = NULL;
+	}
+
+	binding = kzalloc_obj(*binding);
+	if (!binding) {
+		ret = -ENOMEM;
+		goto out_unlock;
+	}
+
+	ret = uctx->cadev->job_tc_bind(uctx->cadev->cxi_dev, job_id, svc_id,
+					 tc, tc_type, create_tc, &limits,
+					 &bound_svc_id, &created_tc);
+	if (ret)
+		goto err_free_binding;
+
+	if (!tc_entry) {
+		tc_entry = kzalloc_obj(*tc_entry);
+		if (!tc_entry) {
+			ret = -ENOMEM;
+			goto err_clear_created_tc;
+		}
+		tc_entry->key = tc_key;
+		tc_entry->tc = tc;
+		tc_entry->tc_type = tc_type;
+		tc_entry->created = created_tc;
+		INIT_LIST_HEAD(&tc_entry->job_list);
+		ret = xa_insert(&dev->tcs, tc_key, tc_entry, GFP_KERNEL);
+		if (ret) {
+			kfree(tc_entry);
+			tc_entry = NULL;
+			goto err_clear_created_tc;
+		}
+	}
+
+	binding->job = job;
+	binding->svc_id = bound_svc_id;
+	list_add_tail(&binding->entry, &tc_entry->job_list);
+	mutex_unlock(&dev->jobs_lock);
+
+	return uverbs_copy_to(attrs, CXI_IB_ATTR_JOB_TC_BIND_RESP_SVC_ID,
+				    &bound_svc_id, sizeof(bound_svc_id));
+err_clear_created_tc:
+	if (created_tc && uctx->cadev->tc_clear)
+		uctx->cadev->tc_clear(uctx->cadev->cxi_dev, tc, tc_type);
+	if (uctx->cadev->svc_destroy)
+		uctx->cadev->svc_destroy(uctx->cadev->cxi_dev, bound_svc_id);
+err_free_binding:
+	kfree(binding);
+out_unlock:
+	mutex_unlock(&dev->jobs_lock);
+	return ret;
+}
+
+DECLARE_UVERBS_NAMED_METHOD(
+	CXI_IB_METHOD_JOB_TC_BIND,
+	UVERBS_ATTR_FLAGS_IN(CXI_IB_ATTR_JOB_TC_BIND_FLAGS, u32, UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_JOB_ID,
+			    UVERBS_ATTR_TYPE(u32), UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_TC,
+			    UVERBS_ATTR_TYPE(u32), UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_TC_TYPE,
+			    UVERBS_ATTR_TYPE(u32), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_SVC_ID,
+			    UVERBS_ATTR_TYPE(u32), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_PTES_MAX,
+			    UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_PTES_RES,
+			    UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_TXQS_MAX,
+			    UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_TXQS_RES,
+			    UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_TGQS_MAX,
+			    UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_TGQS_RES,
+			    UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_EQS_MAX,
+			    UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_EQS_RES,
+			    UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_CTS_MAX,
+			    UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_CTS_RES,
+			    UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_LES_MAX,
+			    UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_LES_RES,
+			    UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_TLES_MAX,
+			    UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_TLES_RES,
+			    UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_ACS_MAX,
+			    UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_BIND_ACS_RES,
+			    UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_JOB_TC_BIND_RESP_SVC_ID,
+			     UVERBS_ATTR_TYPE(u32), UA_MANDATORY));
+
+static int UVERBS_HANDLER(CXI_IB_METHOD_JOB_TC_CLEAR)(
+		struct uverbs_attr_bundle *attrs)
+{
+	struct ib_ucontext *ibctx = ib_uverbs_get_ucontext(attrs);
+	struct cxiverbs_ucontext *uctx;
+	struct cxiverbs_dev *dev;
+	struct cxiverbs_tc_entry *tc_entry;
+	u32 tc;
+	u32 tc_type = CXI_TC_TYPE_DEFAULT;
+	u32 tc_key;
+	int ret;
+
+	if (IS_ERR(ibctx))
+		return PTR_ERR(ibctx);
+	if (!capable(CAP_NET_ADMIN))
+		return -EPERM;
+
+	uctx = to_cxiverbs_ucontext(ibctx);
+	dev = to_cxiverbs_dev(ibctx->device);
+	if (!uctx->cadev->tc_clear)
+		return -EOPNOTSUPP;
+
+	ret = uverbs_copy_from(&tc, attrs, CXI_IB_ATTR_JOB_TC_CLEAR_TC);
+	if (ret)
+		return ret;
+	if (uverbs_attr_is_valid(attrs, CXI_IB_ATTR_JOB_TC_CLEAR_TC_TYPE)) {
+		ret = uverbs_copy_from(&tc_type, attrs,
+				       CXI_IB_ATTR_JOB_TC_CLEAR_TC_TYPE);
+		if (ret)
+			return ret;
+	}
+
+	if (tc >= CXI_TC_MAX || tc_type >= CXI_TC_TYPE_MAX)
+		return -EINVAL;
+
+	tc_key = cxiverbs_tc_key(tc, tc_type);
+	mutex_lock(&dev->jobs_lock);
+	tc_entry = xa_load(&dev->tcs, tc_key);
+	if (!tc_entry) {
+		ret = -ENOENT;
+		goto out_unlock;
+	}
+	if (!list_empty(&tc_entry->job_list)) {
+		ret = -EBUSY;
+		goto out_unlock;
+	}
+
+	ret = tc_entry->created ?
+		uctx->cadev->tc_clear(uctx->cadev->cxi_dev, tc, tc_type) : 0;
+	if (ret)
+		goto out_unlock;
+	xa_erase(&dev->tcs, tc_key);
+	kfree(tc_entry);
+
+out_unlock:
+	mutex_unlock(&dev->jobs_lock);
+	return ret;
+}
+
+DECLARE_UVERBS_NAMED_METHOD(
+	CXI_IB_METHOD_JOB_TC_CLEAR,
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_CLEAR_TC,
+			    UVERBS_ATTR_TYPE(u32), UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_CLEAR_TC_TYPE,
+			    UVERBS_ATTR_TYPE(u32), UA_OPTIONAL));
+
+static int UVERBS_HANDLER(CXI_IB_METHOD_JOB_TC_QUERY)(
+		struct uverbs_attr_bundle *attrs)
+{
+	struct ib_ucontext *ibctx = ib_uverbs_get_ucontext(attrs);
+	struct cxiverbs_ucontext *uctx;
+	struct cxiverbs_dev *dev;
+	struct cxi_tc_info *tc_info;
+	struct cxi_ibv_tc_entry *entries;
+	u32 max_entries = 0;
+	u32 count = 0;
+	u32 copied;
+	int ret;
+	u32 i;
+
+	if (IS_ERR(ibctx))
+		return PTR_ERR(ibctx);
+
+	uctx = to_cxiverbs_ucontext(ibctx);
+	dev = to_cxiverbs_dev(ibctx->device);
+	if (!uctx->cadev->tc_list)
+		return -EOPNOTSUPP;
+
+	if (uverbs_attr_is_valid(attrs, CXI_IB_ATTR_JOB_TC_QUERY_MAX_ENTRIES)) {
+		ret = uverbs_copy_from(&max_entries, attrs,
+				       CXI_IB_ATTR_JOB_TC_QUERY_MAX_ENTRIES);
+		if (ret)
+			return ret;
+	}
+
+	if (!max_entries || max_entries > CXI_TC_MAX * CXI_TC_TYPE_MAX)
+		max_entries = CXI_TC_MAX * CXI_TC_TYPE_MAX;
+
+	tc_info = kcalloc(max_entries, sizeof(*tc_info), GFP_KERNEL);
+	if (!tc_info)
+		return -ENOMEM;
+	entries = kcalloc(max_entries, sizeof(*entries), GFP_KERNEL);
+	if (!entries) {
+		kfree(tc_info);
+		return -ENOMEM;
+	}
+
+	ret = uctx->cadev->tc_list(uctx->cadev->cxi_dev, tc_info, max_entries,
+				    &count);
+	if (ret)
+		goto out_free;
+
+	copied = min(count, max_entries);
+	mutex_lock(&dev->jobs_lock);
+	for (i = 0; i < copied; i++) {
+		struct cxiverbs_tc_entry *tracked;
+
+		entries[i].tc = tc_info[i].tc;
+		entries[i].tc_type = tc_info[i].tc_type;
+		entries[i].res_req_dscp = tc_info[i].res_req_dscp;
+		entries[i].unres_req_dscp = tc_info[i].unres_req_dscp;
+		entries[i].res_rsp_dscp = tc_info[i].res_rsp_dscp;
+		entries[i].unres_rsp_dscp = tc_info[i].unres_rsp_dscp;
+		entries[i].req_pcp = tc_info[i].req_pcp;
+		entries[i].rsp_pcp = tc_info[i].rsp_pcp;
+		entries[i].ocuset = tc_info[i].ocuset;
+		entries[i].cq_tc = tc_info[i].cq_tc;
+		entries[i].req_bc = tc_info[i].req_bc;
+
+		tracked = xa_load(&dev->tcs,
+				  cxiverbs_tc_key(entries[i].tc, entries[i].tc_type));
+		if (tracked && tracked->created)
+			entries[i].flags |= CXI_IBV_TC_ENTRY_F_CREATED_BY_RDMA;
+	}
+	mutex_unlock(&dev->jobs_lock);
+
+	ret = uverbs_copy_to(attrs, CXI_IB_ATTR_JOB_TC_QUERY_RESP_COUNT,
+			     &count, sizeof(count));
+	if (ret)
+		goto out_free;
+	ret = uverbs_copy_to(attrs, CXI_IB_ATTR_JOB_TC_QUERY_RESP_ENTRIES,
+			     entries, copied * sizeof(*entries));
+
+out_free:
+	kfree(entries);
+	kfree(tc_info);
+	return ret;
+}
+
+DECLARE_UVERBS_NAMED_METHOD(
+	CXI_IB_METHOD_JOB_TC_QUERY,
+	UVERBS_ATTR_PTR_IN(CXI_IB_ATTR_JOB_TC_QUERY_MAX_ENTRIES,
+			    UVERBS_ATTR_TYPE(u32), UA_OPTIONAL),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_JOB_TC_QUERY_RESP_COUNT,
+			     UVERBS_ATTR_TYPE(u32), UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(CXI_IB_ATTR_JOB_TC_QUERY_RESP_ENTRIES,
+			     UVERBS_ATTR_MIN_SIZE(0), UA_MANDATORY));
+
+DECLARE_UVERBS_GLOBAL_METHODS(CXI_IB_OBJECT_JOB_TC,
+	&UVERBS_METHOD(CXI_IB_METHOD_JOB_TC_BIND),
+	&UVERBS_METHOD(CXI_IB_METHOD_JOB_TC_CLEAR),
+	&UVERBS_METHOD(CXI_IB_METHOD_JOB_TC_QUERY));
+
+/* -----------------------------------------------------------------------
+ * Driver uapi_definition table — assigned to ib_dev.driver_def in probe.
+ * -----------------------------------------------------------------------
+ */
+const struct uapi_definition cxiverbs_defs[] = {
+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(CXI_IB_OBJECT_WAIT),
+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(CXI_IB_OBJECT_CSR),
+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(CXI_IB_OBJECT_EQ),
+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(CXI_IB_OBJECT_CP),
+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(CXI_IB_OBJECT_SVC),
+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_PD),
+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(CXI_IB_OBJECT_LNI),
+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(CXI_IB_OBJECT_CT),
+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(CXI_IB_OBJECT_CMDQ),
+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(CXI_IB_OBJECT_JOB_TC),
+	{},
+};

-- 
2.56.0