mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: Wei Huang <wei.huang2@amd.com>
To: <dmaengine@vger.kernel.org>
Cc: <vkoul@kernel.org>, <Frank.Li@kernel.org>, <shuah@kernel.org>,
	<nathan.lynch@amd.com>, <wei.huang2@amd.com>,
	<mario.limonciello@amd.com>, <Stephen.Bates@amd.com>,
	<tycho@kernel.org>, <suravee.suthikulpanit@amd.com>,
	<rientjes@google.com>, <John.Kariuki@amd.com>, <jic23@kernel.org>,
	<weixugc@google.com>, <bhelgaas@google.com>,
	<Shyam.Iyer@dell.com>, <linux-kernel@vger.kernel.org>,
	<linux-kselftest@vger.kernel.org>, <linux-pci@vger.kernel.org>
Subject: [PATCH v1 10/44] dmaengine: sdxi: separate context and submission resources
Date: Thu, 19 Nov 2026 14:22:46 -0600	[thread overview]
Message-ID: <20261119202321.1734355-11-wei.huang2@amd.com> (raw)
In-Reply-To: <20261119202321.1734355-1-wei.huang2@amd.com>

Split common context allocation from kernel submission queue
allocation and make context table programming consume an explicit
control configuration. Kernel and admin contexts retain the existing
SQ/CQ model while later userspace contexts can provide PASID-backed
ring addresses.

Co-developed-by: Nathan Lynch <nathan.lynch@amd.com>
Signed-off-by: Nathan Lynch <nathan.lynch@amd.com>
Signed-off-by: Wei Huang <wei.huang2@amd.com>
---
 drivers/dma/sdxi/context.c | 120 ++++++++++++++++++++++++++-----------
 drivers/dma/sdxi/context.h |   3 +
 drivers/dma/sdxi/device.c  |   2 +
 drivers/dma/sdxi/pci.c     |   1 +
 drivers/dma/sdxi/sdxi.h    |   1 +
 5 files changed, 91 insertions(+), 36 deletions(-)

diff --git a/drivers/dma/sdxi/context.c b/drivers/dma/sdxi/context.c
index a8d065b57bac..834223e7b003 100644
--- a/drivers/dma/sdxi/context.c
+++ b/drivers/dma/sdxi/context.c
@@ -17,6 +17,7 @@
 #include <linux/errno.h>
 #include <linux/idr.h>
 #include <linux/iommu.h>
+#include <linux/overflow.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <asm/barrier.h>
@@ -81,23 +82,14 @@ static struct sdxi_cxt *sdxi_alloc_cxt(struct sdxi_dev *sdxi)
 
 	cxt->sdxi = sdxi;
 	ida_init(&cxt->akey_ida);
-
-	cxt->ring_state = kzalloc_obj(*cxt->ring_state, GFP_KERNEL);
-	if (!cxt->ring_state)
-		return NULL;
-
-	cxt->sq = sdxi_sq_alloc(sdxi, DEFAULT_DESC_RING_ENTRIES);
-	if (!cxt->sq)
-		return NULL;
-
-	cxt->cst_pool = sdxi_cst_pool_create(sdxi, DEFAULT_DESC_RING_ENTRIES);
-	if (!cxt->cst_pool)
-		return NULL;
+	cxt->pasid = IOMMU_NO_PASID;
+	cxt->opb_000_enb = sdxi->op_grp_cap;
 
 	cxt->akey_table = dma_alloc_coherent(dev, sizeof(*cxt->akey_table),
 					     &cxt->akey_table_dma, GFP_KERNEL);
 	if (!cxt->akey_table)
 		return NULL;
+	memset(cxt->akey_table, 0, sizeof(*cxt->akey_table));
 
 	cxt->cxt_ctl = dma_pool_zalloc(sdxi->cxt_ctl_pool, GFP_KERNEL,
 				       &cxt->cxt_ctl_dma);
@@ -107,6 +99,25 @@ static struct sdxi_cxt *sdxi_alloc_cxt(struct sdxi_dev *sdxi)
 	return_ptr(cxt);
 }
 
+static int sdxi_alloc_kernel_submission(struct sdxi_cxt *cxt)
+{
+	cxt->ring_state = kzalloc_obj(*cxt->ring_state, GFP_KERNEL);
+	if (!cxt->ring_state)
+		return -ENOMEM;
+
+	cxt->sq = sdxi_sq_alloc(cxt->sdxi, DEFAULT_DESC_RING_ENTRIES);
+	if (!cxt->sq)
+		return -ENOMEM;
+
+	cxt->cst_pool = sdxi_cst_pool_create(cxt->sdxi,
+					     DEFAULT_DESC_RING_ENTRIES);
+	if (!cxt->cst_pool)
+		return -ENOMEM;
+
+	sdxi_ring_state_init(cxt->ring_state, cxt->sq);
+	return 0;
+}
+
 struct sdxi_cxt_ctl_cfg {
 	dma_addr_t ds_ring_ptr;
 	dma_addr_t cxt_sts_ptr;
@@ -253,9 +264,9 @@ sdxi_get_L1_table(struct sdxi_dev *sdxi, u16 l2_idx)
  * implementation. L1 pages remain allocated until device teardown so a
  * cached L2 pointer can never refer to freed memory.
  */
-static int sdxi_publish_cxt(const struct sdxi_cxt *cxt)
+static int sdxi_publish_cxt(const struct sdxi_cxt *cxt,
+			    const struct sdxi_cxt_ctl_cfg *ctl_cfg)
 {
-	struct sdxi_cxt_ctl_cfg ctl_cfg;
 	struct sdxi_cxt_L1_cfg L1_cfg;
 	struct sdxi_cxt_L1_table *L1_table;
 	struct sdxi_cxt_L1_ent *ent;
@@ -268,16 +279,7 @@ static int sdxi_publish_cxt(const struct sdxi_cxt *cxt)
 		      cxt->id, cxt->sdxi->max_cxtid))
 		return -EINVAL;
 
-	ctl_cfg = (typeof(ctl_cfg)) {
-		.se              = 1,
-		.csa             = 1,
-		.ds_ring_ptr     = sdxi_sq_ring_dma(cxt->sq),
-		.ds_ring_sz      = cxt->sq->count,
-		.cxt_sts_ptr     = sdxi_sq_cxt_sts_dma(cxt->sq),
-		.write_index_ptr = sdxi_sq_write_index_dma(cxt->sq),
-	};
-
-	err = configure_cxt_ctl(cxt->cxt_ctl, &ctl_cfg);
+	err = configure_cxt_ctl(cxt->cxt_ctl, ctl_cfg);
 	if (err)
 		return err;
 
@@ -294,13 +296,13 @@ static int sdxi_publish_cxt(const struct sdxi_cxt *cxt)
 
 	L1_cfg = (typeof(L1_cfg)) {
 		.ka          = 1,
-		.pv          = 0,
+		.pv          = cxt->pasid_valid,
 		.cxt_ctl_ptr = cxt->cxt_ctl_dma,
 		.akey_sz     = akey_table_order(cxt->akey_table),
 		.akey_ptr    = cxt->akey_table_dma,
-		.cxt_pasid   = IOMMU_NO_PASID,
+		.cxt_pasid   = cxt->pasid,
 		.max_buffer  = cxt->sdxi->max_buffer,
-		.opb_000_enb = cxt->sdxi->op_grp_cap,
+		.opb_000_enb = cxt->opb_000_enb,
 	};
 
 	err = configure_L1_entry(ent, &L1_cfg);
@@ -321,23 +323,39 @@ static void free_admin_cxt(void *ptr)
 	sdxi_free_cxt(sdxi->admin_cxt);
 }
 
+static int sdxi_cxt_set_doorbell(struct sdxi_cxt *cxt);
+
 int sdxi_admin_cxt_init(struct sdxi_dev *sdxi)
 {
-	int err;
+	struct sdxi_cxt_ctl_cfg ctl_cfg;
 	struct sdxi_sq *sq;
+	int err;
 
 	struct sdxi_cxt *cxt __free(sdxi_cxt) = sdxi_alloc_cxt(sdxi);
 	if (!cxt)
 		return -ENOMEM;
 
+	err = sdxi_alloc_kernel_submission(cxt);
+	if (err)
+		return err;
+
 	sq = cxt->sq;
 	/* SDXI 1.0 4.1.8.4.b: Set CXT_STS.state to CXTV_RUN. */
 	sq->cxt_sts.state = FIELD_PREP(SDXI_CXT_STS_STATE, CXTV_RUN);
 	cxt->id = SDXI_ADMIN_CXT_ID;
-	cxt->sq->doorbell = sdxi->dbs + cxt->id * sdxi->db_stride;
-	sdxi_ring_state_init(cxt->ring_state, sq);
+	err = sdxi_cxt_set_doorbell(cxt);
+	if (err)
+		return err;
 
-	err = sdxi_publish_cxt(cxt);
+	ctl_cfg = (typeof(ctl_cfg)) {
+		.se              = 1,
+		.csa             = 1,
+		.ds_ring_ptr     = sdxi_sq_ring_dma(cxt->sq),
+		.ds_ring_sz      = cxt->sq->count,
+		.cxt_sts_ptr     = sdxi_sq_cxt_sts_dma(cxt->sq),
+		.write_index_ptr = sdxi_sq_write_index_dma(cxt->sq),
+	};
+	err = sdxi_publish_cxt(cxt, &ctl_cfg);
 	if (err)
 		return err;
 
@@ -382,6 +400,23 @@ static bool sdxi_cxt_id_valid(const struct sdxi_cxt_id *cxt_id)
 	return cxt_id->index > 0;
 }
 
+static int sdxi_cxt_set_doorbell(struct sdxi_cxt *cxt)
+{
+	struct sdxi_dev *sdxi = cxt->sdxi;
+	size_t offset;
+
+	if (check_mul_overflow((size_t)cxt->id, (size_t)sdxi->db_stride,
+			       &offset) ||
+	    offset > sdxi->db_size ||
+	    sdxi->db_stride > sdxi->db_size - offset)
+		return -ERANGE;
+
+	if (WARN_ON_ONCE(!cxt->sq))
+		return -EINVAL;
+	cxt->sq->doorbell = sdxi->dbs + offset;
+	return 0;
+}
+
 /*
  * Transfer the reserved ID to the context and store its pointer in client_cxts.
  * sdxi_free_cxt() releases the ID.
@@ -401,6 +436,9 @@ static void sdxi_cxt_id_assign(struct sdxi_cxt *cxt, struct sdxi_cxt_id *cxt_id)
  */
 struct sdxi_cxt *sdxi_cxt_new(struct sdxi_dev *sdxi)
 {
+	struct sdxi_cxt_ctl_cfg ctl_cfg;
+	int err;
+
 	/*
 	 * Ensure an ID is available before allocating memory for the
 	 * context and its control structures.
@@ -413,13 +451,23 @@ struct sdxi_cxt *sdxi_cxt_new(struct sdxi_dev *sdxi)
 	if (!cxt)
 		return NULL;
 
-	sdxi_cxt_id_assign(cxt, &id);
-
-	cxt->sq->doorbell = sdxi->dbs + cxt->id * sdxi->db_stride;
+	err = sdxi_alloc_kernel_submission(cxt);
+	if (err)
+		return NULL;
 
-	sdxi_ring_state_init(cxt->ring_state, cxt->sq);
+	sdxi_cxt_id_assign(cxt, &id);
+	if (sdxi_cxt_set_doorbell(cxt))
+		return NULL;
 
-	if (sdxi_publish_cxt(cxt))
+	ctl_cfg = (typeof(ctl_cfg)) {
+		.se              = 1,
+		.csa             = 1,
+		.ds_ring_ptr     = sdxi_sq_ring_dma(cxt->sq),
+		.ds_ring_sz      = cxt->sq->count,
+		.cxt_sts_ptr     = sdxi_sq_cxt_sts_dma(cxt->sq),
+		.write_index_ptr = sdxi_sq_write_index_dma(cxt->sq),
+	};
+	if (sdxi_publish_cxt(cxt, &ctl_cfg))
 		return NULL;
 
 	return_ptr(cxt);
diff --git a/drivers/dma/sdxi/context.h b/drivers/dma/sdxi/context.h
index 196207cbca94..8e57565c0707 100644
--- a/drivers/dma/sdxi/context.h
+++ b/drivers/dma/sdxi/context.h
@@ -36,6 +36,9 @@ static inline u8 akey_table_order(const struct sdxi_akey_table *tbl)
 struct sdxi_cxt {
 	struct sdxi_dev *sdxi;
 	u16 id;
+	u32 pasid;
+	u32 opb_000_enb;
+	bool pasid_valid;
 
 	struct sdxi_cxt_ctl *cxt_ctl;
 	dma_addr_t cxt_ctl_dma;
diff --git a/drivers/dma/sdxi/device.c b/drivers/dma/sdxi/device.c
index fa9421a5410b..4b54b14fefc1 100644
--- a/drivers/dma/sdxi/device.c
+++ b/drivers/dma/sdxi/device.c
@@ -370,6 +370,8 @@ static int sdxi_fn_activate(struct sdxi_dev *sdxi)
 	cap0 = sdxi_read64(sdxi, SDXI_MMIO_CAP0);
 	sdxi->db_stride = SZ_4K;
 	sdxi->db_stride *= 1U << FIELD_GET(SDXI_MMIO_CAP0_DB_STRIDE, cap0);
+	if (sdxi->db_stride > sdxi->db_size)
+		return -ENODEV;
 
 	cap1 = sdxi_read64(sdxi, SDXI_MMIO_CAP1);
 	sdxi->op_grp_cap = FIELD_GET(SDXI_MMIO_CAP1_OPB_000_CAP, cap1);
diff --git a/drivers/dma/sdxi/pci.c b/drivers/dma/sdxi/pci.c
index f9f75ff5e179..7b477b7bec55 100644
--- a/drivers/dma/sdxi/pci.c
+++ b/drivers/dma/sdxi/pci.c
@@ -59,6 +59,7 @@ static int sdxi_pci_init(struct sdxi_dev *sdxi)
 	if (IS_ERR(sdxi->dbs))
 		return dev_err_probe(dev, PTR_ERR(sdxi->dbs),
 				     "failed to map doorbell region\n");
+	sdxi->db_size = pci_resource_len(pdev, SDXI_PCI_BAR_DOORBELL);
 
 	/* Mailbox registers are reserved without SR-IOV support. */
 	sdxi->has_mailbox = pdev->is_virtfn ||
diff --git a/drivers/dma/sdxi/sdxi.h b/drivers/dma/sdxi/sdxi.h
index 65e0191391db..fa6c3ef1c58f 100644
--- a/drivers/dma/sdxi/sdxi.h
+++ b/drivers/dma/sdxi/sdxi.h
@@ -74,6 +74,7 @@ struct sdxi_dev {
 	struct device *dev;
 	void __iomem *ctrl_regs;	/* virt addr of ctrl registers */
 	void __iomem *dbs;		/* virt addr of doorbells */
+	size_t db_size;			/* mapped doorbell region size */
 
 	struct sdxi_version sdxi_version;
 
-- 
2.56.1


  parent reply	other threads:[~2026-11-19 20:25 UTC|newest]

Thread overview: 45+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-19 20:22 [PATCH v1 00/44] dmaengine: sdxi: add userspace control and management support Wei Huang
2026-11-19 20:22 ` [PATCH v1 01/44] dmaengine: sdxi: cache hardware version for feature gating Wei Huang
2026-11-19 20:22 ` [PATCH v1 02/44] dmaengine: sdxi: define key and error MMIO registers Wei Huang
2026-11-19 20:22 ` [PATCH v1 03/44] dmaengine: sdxi: define SDXI hardware layouts Wei Huang
2026-11-19 20:22 ` [PATCH v1 04/44] dmaengine: sdxi: encode admin descriptors Wei Huang
2026-11-19 20:22 ` [PATCH v1 05/44] dmaengine: sdxi: support timeout-aware completion waits Wei Huang
2026-11-19 20:22 ` [PATCH v1 06/44] dmaengine: sdxi: support the full context table hierarchy Wei Huang
2026-11-19 20:22 ` [PATCH v1 07/44] dmaengine: sdxi: implement synchronous admin operations Wei Huang
2026-11-19 20:22 ` [PATCH v1 08/44] dmaengine: sdxi: add core RKey table management Wei Huang
2026-11-19 20:22 ` [PATCH v1 09/44] dmaengine: sdxi: handle hardware error logs Wei Huang
2026-11-19 20:22 ` Wei Huang [this message]
2026-11-19 20:22 ` [PATCH v1 11/44] dmaengine: sdxi: support userspace context lifetimes Wei Huang
2026-11-19 20:22 ` [PATCH v1 12/44] dmaengine: sdxi: route context interrupts to eventfd Wei Huang
2026-11-19 20:22 ` [PATCH v1 13/44] dmaengine: sdxi: kill user contexts on fatal errors Wei Huang
2026-11-19 20:22 ` [PATCH v1 14/44] dmaengine: sdxi: define the ioctl UAPI surface Wei Huang
2026-11-19 20:22 ` [PATCH v1 15/44] dmaengine: sdxi: add internal ioctl definitions Wei Huang
2026-11-19 20:22 ` [PATCH v1 16/44] dmaengine: sdxi: register core character devices Wei Huang
2026-11-19 20:22 ` [PATCH v1 17/44] dmaengine: sdxi: implement fixed-size ioctl dispatch Wei Huang
2026-11-19 20:22 ` [PATCH v1 18/44] dmaengine: sdxi: implement userspace context file operations Wei Huang
2026-11-19 20:22 ` [PATCH v1 19/44] dmaengine: sdxi: validate and copy key ioctl payloads Wei Huang
2026-11-19 20:22 ` [PATCH v1 20/44] dmaengine: sdxi: implement RKey management ioctls Wei Huang
2026-11-19 20:22 ` [PATCH v1 21/44] dmaengine: sdxi: implement AKey " Wei Huang
2026-11-19 20:22 ` [PATCH v1 22/44] dmaengine: sdxi: expose operation-group discovery Wei Huang
2026-11-19 20:22 ` [PATCH v1 23/44] dmaengine: sdxi: implement typed vendor admin transport Wei Huang
2026-11-19 20:23 ` [PATCH v1 24/44] dmaengine: sdxi: expose character device attributes Wei Huang
2026-11-19 20:23 ` [PATCH v1 25/44] dmaengine: sdxi: add PF and VF device naming Wei Huang
2026-11-19 20:23 ` [PATCH v1 26/44] dmaengine: sdxi: add SDXI trace events Wei Huang
2026-11-19 20:23 ` [PATCH v1 27/44] dmaengine: sdxi: support suspend and resume Wei Huang
2026-11-19 20:23 ` [PATCH v1 28/44] dmaengine: sdxi: amd: define QoS request formats Wei Huang
2026-11-19 20:23 ` [PATCH v1 29/44] dmaengine: sdxi: amd: add QoS validation scaffolding Wei Huang
2026-11-19 20:23 ` [PATCH v1 30/44] dmaengine: sdxi: amd: build QoS admin descriptors Wei Huang
2026-11-19 20:23 ` [PATCH v1 31/44] dmaengine: sdxi: amd: prevalidate QoS operations Wei Huang
2026-11-19 20:23 ` [PATCH v1 32/44] dmaengine: sdxi: amd: execute QoS vendor operations Wei Huang
2026-11-19 20:23 ` [PATCH v1 33/44] dmaengine: sdxi: amd: register QoS vendor handler Wei Huang
2026-11-19 20:23 ` [PATCH v1 34/44] dmaengine: sdxi: amd: test QoS validation with KUnit Wei Huang
2026-11-19 20:23 ` [PATCH v1 35/44] selftests: sdxi: add build scaffold Wei Huang
2026-11-19 20:23 ` [PATCH v1 36/44] selftests: sdxi: add shared descriptor and context helpers Wei Huang
2026-11-19 20:23 ` [PATCH v1 37/44] selftests: sdxi: test context creation Wei Huang
2026-11-19 20:23 ` [PATCH v1 38/44] selftests: sdxi: test NOP descriptor submission Wei Huang
2026-11-19 20:23 ` [PATCH v1 39/44] selftests: sdxi: test copy submission and ring wrap Wei Huang
2026-11-19 20:23 ` [PATCH v1 40/44] selftests: sdxi: exercise dataplane error paths Wei Huang
2026-11-19 20:23 ` [PATCH v1 41/44] selftests: sdxi: test device ioctls Wei Huang
2026-11-19 20:23 ` [PATCH v1 42/44] selftests: sdxi: test vendor admin ioctls Wei Huang
2026-11-19 20:23 ` [PATCH v1 43/44] selftests: sdxi: test RKey ioctls Wei Huang
2026-11-19 20:23 ` [PATCH v1 44/44] MAINTAINERS: cover SDXI UAPI, vendor files, and selftests Wei Huang

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261119202321.1734355-11-wei.huang2@amd.com \
    --to=wei.huang2@amd.com \
    --cc=Frank.Li@kernel.org \
    --cc=John.Kariuki@amd.com \
    --cc=Shyam.Iyer@dell.com \
    --cc=Stephen.Bates@amd.com \
    --cc=bhelgaas@google.com \
    --cc=dmaengine@vger.kernel.org \
    --cc=jic23@kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-kselftest@vger.kernel.org \
    --cc=linux-pci@vger.kernel.org \
    --cc=mario.limonciello@amd.com \
    --cc=nathan.lynch@amd.com \
    --cc=rientjes@google.com \
    --cc=shuah@kernel.org \
    --cc=suravee.suthikulpanit@amd.com \
    --cc=tycho@kernel.org \
    --cc=vkoul@kernel.org \
    --cc=weixugc@google.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®