[PATCH v3] nvme-tcp: allow setting per-queue io_cpu through sysfs

[PATCH v3] nvme-tcp: allow setting per-queue io_cpu through sysfs

[Saravanan D]

nvme_tcp_set_queue_io_cpu() picks each queue's io_cpu at connect time
as the least loaded CPU in the queue's blk-mq map group, and all socket
work runs there for the connection's lifetime. This decision falls
short when the host partitions its CPUs after connect time. On a 384
cpu multi tenant host with 128 queue controllers, blk-mq folds three
CPUs into every map group, some groups straddle two tenants' cpusets,
and 9% of nvme_tcp_io_work executions ran outside the submitting VM's
cpuset, seen by the neighbor as steal time.

Expose each I/O queue's io_cpu as a writable sysfs attribute

  /sys/class/nvme/nvmeX/tcp_queues/<qid>/io_cpu

so a control plane that owns CPU placement can set it directly instead
of relying on the driver's heuristic. A written value persists across
reconnects, marked by NVME_TCP_Q_IO_CPU_USER. Writing -1 clears the
mark and re-runs the connect time selection. Reading returns the CPU,
or -1 when the queue is unbound.

The store, the connect time selection and queue stop serialize their
accounting of nvme_tcp_cpu_queues under the queue lock.

Suggested-by: Sagi Grimberg <sagi@grimberg.me>
Link: https://lore.kernel.org/linux-nvme/220e9da3-f756-4a16-8de1-d4b171f15009@grimberg.me/
Assisted-by: Claude:claude-opus-4-8 [Claude Code]
Signed-off-by: Saravanan D <saravanand@crusoe.ai>
---
Changes since v2 [1]:
- Replaced the io_cpu_adopt connect option and the submitter adoption
  heuristic with a per queue writable sysfs attribute, following Sagi's
  suggestion [2]. The control plane now sets each queue's io_cpu
  directly, the assignment is kept across reconnects and writing -1
  reverts to the connect time selection.
- Retitled from "nvme-tcp: pin io_cpu to submitter cpu".

The per queue directories follow the blk-mq mq/<hctx> sysfs pattern.

Tested on a 2 socket 384 cpu host with 128 queue controllers. Writing
a cpu number changed the queue's io_cpu to it. Writing an invalid
value was rejected. Writing -1 re-ran the connect time selection.
Pinned queues kept their io_cpu across a controller reset while
unpinned queues received a fresh pick.

The multiple queues per hctx RFC [3] found the same need to steer the
socket work cpu, so this attribute may gain a second user.

[1] https://lore.kernel.org/linux-nvme/20260820083634.71689-1-saravanand@crusoe.ai/
[2] https://lore.kernel.org/linux-nvme/1d56144d-6987-40c1-ac02-b15333db121e@grimberg.me/
[3] https://lore.kernel.org/linux-nvme/20260903152623.614951-1-kbusch@meta.com/

 Documentation/ABI/stable/sysfs-nvme |  12 +++
 drivers/nvme/host/tcp.c             | 153 +++++++++++++++++++++++++++-
 2 files changed, 162 insertions(+), 3 deletions(-)

diff --git a/Documentation/ABI/stable/sysfs-nvme b/Documentation/ABI/stable/sysfs-nvme
index a2f5d0710db4..2bbb5a0b7c2e 100644
--- a/Documentation/ABI/stable/sysfs-nvme
+++ b/Documentation/ABI/stable/sysfs-nvme
@@ -451,3 +451,15 @@ Contact:	Hannes Reinecke <hare@suse.de>
 Description:
 		Shows the subsystem type. Possible values: "discovery",
 		"nvm", "reserved".
+
+What:		/sys/class/nvme/nvmeX/tcp_queues/<qid>/io_cpu
+Date:		September 2026
+KernelVersion:	7.4
+Contact:	Saravanan D <saravanand@crusoe.ai>
+Description:
+		(RW) The CPU that runs the socket work for I/O queue <qid>
+		of an NVMe over TCP controller, selected by the driver at
+		connect time. Writing a CPU number overrides the selection
+		and persists across reconnects. Writing -1 reverts to the
+		driver's selection. Reads show the current CPU, or -1 when
+		the queue is unbound.
diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
index 921934028e0b..22ad1fdf4a12 100644
--- a/drivers/nvme/host/tcp.c
+++ b/drivers/nvme/host/tcp.c
@@ -93,6 +93,7 @@ enum nvme_tcp_queue_flags {
 	NVME_TCP_Q_LIVE		= 1,
 	NVME_TCP_Q_POLLING	= 2,
 	NVME_TCP_Q_IO_CPU_SET	= 3,
+	NVME_TCP_Q_IO_CPU_USER	= 4,
 };
 
 enum nvme_tcp_recv_state {
@@ -102,6 +103,17 @@ enum nvme_tcp_recv_state {
 };
 
 struct nvme_tcp_ctrl;
+struct nvme_tcp_queue;
+
+/*
+ * Allocated per registration and freed by its kobject release, so a
+ * reconnect never reuses a kobject whose release is still pending.
+ */
+struct nvme_tcp_queue_kobj {
+	struct kobject		kobj;
+	struct nvme_tcp_queue	*queue;
+};
+
 struct nvme_tcp_queue {
 	struct socket		*sock;
 	struct work_struct	io_work;
@@ -141,6 +153,8 @@ struct nvme_tcp_queue {
 	int                     tls_err;
 	struct page_frag_cache	pf_cache;
 
+	struct nvme_tcp_queue_kobj *qkobj;
+
 	void (*state_change)(struct sock *);
 	void (*data_ready)(struct sock *);
 	void (*write_space)(struct sock *);
@@ -171,8 +185,11 @@ struct nvme_tcp_ctrl {
 	struct delayed_work	connect_work;
 	struct nvme_tcp_request async_req;
 	u32			io_queues[HCTX_MAX_TYPES];
+	struct kobject		*queues_kobj;
 };
 
+static void nvme_tcp_unregister_queue_sysfs(struct nvme_tcp_queue *queue);
+
 static struct workqueue_struct *nvme_tcp_wq;
 static const struct blk_mq_ops nvme_tcp_mq_ops;
 static const struct blk_mq_ops nvme_tcp_admin_mq_ops;
@@ -1497,6 +1514,8 @@ static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
 	if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
 		return;
 
+	nvme_tcp_unregister_queue_sysfs(queue);
+
 	page_frag_cache_drain(&queue->pf_cache);
 
 	/**
@@ -1716,9 +1735,19 @@ static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)
 	unsigned int *mq_map = NULL;
 	int cpu, min_queues = INT_MAX, io_cpu;
 
+	lockdep_assert_held(&queue->queue_lock);
+
 	if (wq_unbound)
 		goto out;
 
+	/* A user assigned io_cpu is kept across reconnects */
+	if (test_bit(NVME_TCP_Q_IO_CPU_USER, &queue->flags) &&
+	    queue->io_cpu != WORK_CPU_UNBOUND) {
+		if (!test_and_set_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags))
+			atomic_inc(&nvme_tcp_cpu_queues[queue->io_cpu]);
+		goto out;
+	}
+
 	if (nvme_tcp_default_queue(queue))
 		mq_map = set->map[HCTX_TYPE_DEFAULT].mq_map;
 	else if (nvme_tcp_read_queue(queue))
@@ -1836,6 +1865,118 @@ static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl,
 	return ret;
 }
 
+static struct nvme_tcp_queue *nvme_tcp_kobj_to_queue(struct kobject *kobj)
+{
+	return container_of(kobj, struct nvme_tcp_queue_kobj, kobj)->queue;
+}
+
+static ssize_t io_cpu_show(struct kobject *kobj, struct kobj_attribute *attr,
+			   char *buf)
+{
+	struct nvme_tcp_queue *queue = nvme_tcp_kobj_to_queue(kobj);
+	int io_cpu = READ_ONCE(queue->io_cpu);
+
+	return sysfs_emit(buf, "%d\n",
+			  io_cpu == WORK_CPU_UNBOUND ? -1 : io_cpu);
+}
+
+static ssize_t io_cpu_store(struct kobject *kobj, struct kobj_attribute *attr,
+			    const char *buf, size_t count)
+{
+	struct nvme_tcp_queue *queue = nvme_tcp_kobj_to_queue(kobj);
+	int cpu, old;
+	int ret;
+
+	ret = kstrtoint(buf, 0, &cpu);
+	if (ret)
+		return ret;
+	if (cpu != -1 &&
+	    ((unsigned int)cpu >= nr_cpu_ids || !cpu_online(cpu)))
+		return -EINVAL;
+
+	mutex_lock(&queue->queue_lock);
+	if (!test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) {
+		mutex_unlock(&queue->queue_lock);
+		return -ENODEV;
+	}
+	if (cpu == -1) {
+		if (test_and_clear_bit(NVME_TCP_Q_IO_CPU_USER, &queue->flags)) {
+			if (test_and_clear_bit(NVME_TCP_Q_IO_CPU_SET,
+					       &queue->flags))
+				atomic_dec(&nvme_tcp_cpu_queues[queue->io_cpu]);
+			WRITE_ONCE(queue->io_cpu, WORK_CPU_UNBOUND);
+			/* a queue that is not live gets its pick at start */
+			if (test_bit(NVME_TCP_Q_LIVE, &queue->flags))
+				nvme_tcp_set_queue_io_cpu(queue);
+		}
+	} else {
+		old = xchg(&queue->io_cpu, cpu);
+		set_bit(NVME_TCP_Q_IO_CPU_USER, &queue->flags);
+		if (test_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags)) {
+			atomic_dec(&nvme_tcp_cpu_queues[old]);
+			atomic_inc(&nvme_tcp_cpu_queues[cpu]);
+		}
+	}
+	mutex_unlock(&queue->queue_lock);
+
+	return count;
+}
+
+static struct kobj_attribute nvme_tcp_io_cpu_attr =
+	__ATTR(io_cpu, 0644, io_cpu_show, io_cpu_store);
+
+static struct attribute *nvme_tcp_queue_attrs[] = {
+	&nvme_tcp_io_cpu_attr.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(nvme_tcp_queue);
+
+static void nvme_tcp_queue_kobj_release(struct kobject *kobj)
+{
+	kfree(container_of(kobj, struct nvme_tcp_queue_kobj, kobj));
+}
+
+static const struct kobj_type nvme_tcp_queue_ktype = {
+	.sysfs_ops	= &kobj_sysfs_ops,
+	.release	= nvme_tcp_queue_kobj_release,
+	.default_groups	= nvme_tcp_queue_groups,
+};
+
+static void nvme_tcp_register_queue_sysfs(struct nvme_tcp_queue *queue)
+{
+	struct nvme_tcp_ctrl *ctrl = queue->ctrl;
+	struct nvme_tcp_queue_kobj *qkobj;
+
+	if (!ctrl->queues_kobj)
+		ctrl->queues_kobj = kobject_create_and_add("tcp_queues",
+							   &ctrl->ctrl.device->kobj);
+	if (!ctrl->queues_kobj)
+		return;
+
+	qkobj = kzalloc_obj(*qkobj);
+	if (!qkobj)
+		return;
+
+	qkobj->queue = queue;
+	if (kobject_init_and_add(&qkobj->kobj, &nvme_tcp_queue_ktype,
+				 ctrl->queues_kobj, "%d",
+				 nvme_tcp_queue_id(queue))) {
+		kobject_put(&qkobj->kobj);
+		return;
+	}
+	queue->qkobj = qkobj;
+}
+
+static void nvme_tcp_unregister_queue_sysfs(struct nvme_tcp_queue *queue)
+{
+	struct nvme_tcp_queue_kobj *qkobj = queue->qkobj;
+
+	if (!qkobj)
+		return;
+	queue->qkobj = NULL;
+	kobject_put(&qkobj->kobj);
+}
+
 static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
 				key_serial_t pskid)
 {
@@ -1906,7 +2047,8 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
 
 	queue->sock->sk->sk_allocation = GFP_ATOMIC;
 	queue->sock->sk->sk_use_task_frag = false;
-	queue->io_cpu = WORK_CPU_UNBOUND;
+	if (!test_bit(NVME_TCP_Q_IO_CPU_USER, &queue->flags))
+		queue->io_cpu = WORK_CPU_UNBOUND;
 	queue->request = NULL;
 	queue->data_remaining = 0;
 	queue->ddgst_remaining = 0;
@@ -1974,6 +2116,9 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
 
 	set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags);
 
+	if (qid)
+		nvme_tcp_register_queue_sysfs(queue);
+
 	return 0;
 
 err_init_connect:
@@ -2022,10 +2167,9 @@ static void nvme_tcp_stop_queue_nowait(struct nvme_ctrl *nctrl, int qid)
 	if (!test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
 		return;
 
+	mutex_lock(&queue->queue_lock);
 	if (test_and_clear_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags))
 		atomic_dec(&nvme_tcp_cpu_queues[queue->io_cpu]);
-
-	mutex_lock(&queue->queue_lock);
 	if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags))
 		__nvme_tcp_stop_queue(queue);
 	/* Stopping the queue will disable TLS */
@@ -2085,7 +2229,9 @@ static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx)
 	nvme_tcp_setup_sock_ops(queue);
 
 	if (idx) {
+		mutex_lock(&queue->queue_lock);
 		nvme_tcp_set_queue_io_cpu(queue);
+		mutex_unlock(&queue->queue_lock);
 		ret = nvmf_connect_io_queue(nctrl, idx);
 	} else
 		ret = nvmf_connect_admin_queue(nctrl);
@@ -2650,6 +2796,7 @@ static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl)
 
 	nvmf_free_options(nctrl->opts);
 free_ctrl:
+	kobject_put(ctrl->queues_kobj);
 	kfree(ctrl->queues);
 	kfree(ctrl);
 }

base-commit: fd9beb8870736e1c6a0b2351d88a161aaeb2b326
-- 
2.55.0

Re: [PATCH v3] nvme-tcp: allow setting per-queue io_cpu through sysfs

[Nilay Shroff]

On 9/10/26 2:44 AM, Saravanan D wrote:
> nvme_tcp_set_queue_io_cpu() picks each queue's io_cpu at connect time
> as the least loaded CPU in the queue's blk-mq map group, and all socket
> work runs there for the connection's lifetime. This decision falls
> short when the host partitions its CPUs after connect time. On a 384
> cpu multi tenant host with 128 queue controllers, blk-mq folds three
> CPUs into every map group, some groups straddle two tenants' cpusets,
> and 9% of nvme_tcp_io_work executions ran outside the submitting VM's
> cpuset, seen by the neighbor as steal time.
> 
> Expose each I/O queue's io_cpu as a writable sysfs attribute
> 
>    /sys/class/nvme/nvmeX/tcp_queues/<qid>/io_cpu
> 

I wonder if we should avoid introducing a transport-specific tcp_queues hierarchy
here. io_cpu describes CPU placement of an NVMe I/O queue rather than something
inherently specific to TCP, and other transports may have similar queue-level
attributes in the future. Could we instead introduce a generic "queues/<qid>/"
hierarchy under each NVMe controller, e.g.

      /sys/class/nvme/nvmeX/queues/<qid>/

and have transports expose queue-specific attributes there? The "queues/<qid>"
object could be created for every controller, while transports that need queue-
specific sysfs attributes can populate the corresponding <qid> objects. This
would give us a transport-independent ABI and avoid introducing separate
tcp_queues, rdma_queues, etc. hierarchies.

> so a control plane that owns CPU placement can set it directly instead
> of relying on the driver's heuristic. A written value persists across
> reconnects, marked by NVME_TCP_Q_IO_CPU_USER. Writing -1 clears the
> mark and re-runs the connect time selection. Reading returns the CPU,
> or -1 when the queue is unbound.
> 
> The store, the connect time selection and queue stop serialize their
> accounting of nvme_tcp_cpu_queues under the queue lock.
> 
Also, based on the current design, it seems possible to have some queues whose
io_cpu is driver-managed while other queues are explicitly managed by the
control plane. This gives useful flexibility for the tenant use case. However,
looking at the io_cpu sysfs attribute alone, it is not possible to determine
whether the current CPU was selected by the driver or explicitly assigned by
the control plane. So could we expose this state through an additional read-only
attribute alongside io_cpu, for example:

     /sys/class/nvme/nvmeX/queues/<qid>/io_cpu
     /sys/class/nvme/nvmeX/queues/<qid>/managed

where "managed" indicates who currently owns the CPU placement. For example, "1"
could indicate that io_cpu is driver-managed and "0" indicates it is managed by
the control plane or user. This would make the current ownership state observable
without making any wild guess.

Thanks,
--Nilay

Re: [PATCH v3] nvme-tcp: allow setting per-queue io_cpu through sysfs

[Saravanan D]

On Thu, 10 Sep 2026 13:55:59 +0530 Nilay Shroff <nilay@linux.ibm.com> wrote:
> I wonder if we should avoid introducing a transport-specific tcp_queues
> hierarchy here. io_cpu describes CPU placement of an NVMe I/O queue
> rather than something inherently specific to TCP, and other transports
> may have similar queue-level attributes in the future. Could we instead
> introduce a generic "queues/<qid>/" hierarchy under each NVMe
> controller, e.g.
>
>       /sys/class/nvme/nvmeX/queues/<qid>/
>
> and have transports expose queue-specific attributes there?

Thanks for your review.

A controller is backed by exactly one transport, so we can adopt the
transport-neutral /sys/class/nvme/nvmeX/queues/<qid>/ path with the
creation staying outside of core in nvme-tcp, which owns the queue
lifecycle. The core only knows ctrl->queue_count and never sees
individual queue lifecycle.

No other transport has queue attributes today, so a controller backed
by rdma will have no queues directory at all. The path appears only
where a transport populates it, and if another transport grows queue
attributes later, the creation can move into the core without any
ABI change.

> So could we expose this state through an additional read-only
> attribute alongside io_cpu, for example:
>
>      /sys/class/nvme/nvmeX/queues/<qid>/io_cpu
>      /sys/class/nvme/nvmeX/queues/<qid>/managed
>
> where "managed" indicates who currently owns the CPU placement. For
> example, "1" could indicate that io_cpu is driver-managed and "0"
> indicates it is managed by the control plane or user.

Agreed, I can add the read-only managed attribute, 1 when the io_cpu is
driver managed and 0 when the user assigned it in the next version.

Both suggestions look good to me. I will wait for Sagi or Keith to
weigh in before spinning v4.

Saravanan D.

Re: [PATCH v3] nvme-tcp: allow setting per-queue io_cpu through sysfs

[Sagi Grimberg]

On 11/09/2026 9:03, Saravanan D wrote:
> On Thu, 10 Sep 2026 13:55:59 +0530 Nilay Shroff <nilay@linux.ibm.com> wrote:
>> I wonder if we should avoid introducing a transport-specific tcp_queues
>> hierarchy here. io_cpu describes CPU placement of an NVMe I/O queue
>> rather than something inherently specific to TCP, and other transports
>> may have similar queue-level attributes in the future. Could we instead
>> introduce a generic "queues/<qid>/" hierarchy under each NVMe
>> controller, e.g.
>>
>>        /sys/class/nvme/nvmeX/queues/<qid>/
>>
>> and have transports expose queue-specific attributes there?
> Thanks for your review.
>
> A controller is backed by exactly one transport, so we can adopt the
> transport-neutral /sys/class/nvme/nvmeX/queues/<qid>/ path with the
> creation staying outside of core in nvme-tcp, which owns the queue
> lifecycle. The core only knows ctrl->queue_count and never sees
> individual queue lifecycle.
>
> No other transport has queue attributes today, so a controller backed
> by rdma will have no queues directory at all. The path appears only
> where a transport populates it, and if another transport grows queue
> attributes later, the creation can move into the core without any
> ABI change.
>
>> So could we expose this state through an additional read-only
>> attribute alongside io_cpu, for example:
>>
>>       /sys/class/nvme/nvmeX/queues/<qid>/io_cpu
>>       /sys/class/nvme/nvmeX/queues/<qid>/managed
>>
>> where "managed" indicates who currently owns the CPU placement. For
>> example, "1" could indicate that io_cpu is driver-managed and "0"
>> indicates it is managed by the control plane or user.
> Agreed, I can add the read-only managed attribute, 1 when the io_cpu is
> driver managed and 0 when the user assigned it in the next version.
>
> Both suggestions look good to me. I will wait for Sagi or Keith to
> weigh in before spinning v4.

I agree with this approach.

I think that making nvme-core more "aware" of individual controller queues
is a good thing.

Re: [PATCH v3] nvme-tcp: allow setting per-queue io_cpu through sysfs

[Sagi Grimberg]

On 10/09/2026 0:14, Saravanan D wrote:
> nvme_tcp_set_queue_io_cpu() picks each queue's io_cpu at connect time
> as the least loaded CPU in the queue's blk-mq map group, and all socket
> work runs there for the connection's lifetime. This decision falls
> short when the host partitions its CPUs after connect time. On a 384
> cpu multi tenant host with 128 queue controllers, blk-mq folds three
> CPUs into every map group, some groups straddle two tenants' cpusets,
> and 9% of nvme_tcp_io_work executions ran outside the submitting VM's
> cpuset, seen by the neighbor as steal time.
>
> Expose each I/O queue's io_cpu as a writable sysfs attribute
>
>    /sys/class/nvme/nvmeX/tcp_queues/<qid>/io_cpu
>
> so a control plane that owns CPU placement can set it directly instead
> of relying on the driver's heuristic. A written value persists across
> reconnects, marked by NVME_TCP_Q_IO_CPU_USER. Writing -1 clears the
> mark and re-runs the connect time selection. Reading returns the CPU,
> or -1 when the queue is unbound.

No need for NVME_TCP_Q_IO_CPU_USER flag and restoring to the original 
connect time selection. -1 would mean WORK_CPU_UNBOUND. In fact lets 
allow this file to access "unbound" string (same as -1). I don't think 
we need to restore the connect time selection, the user who touches 
these settings is obviously interested to be in control.
>
> The store, the connect time selection and queue stop serialize their
> accounting of nvme_tcp_cpu_queues under the queue lock.

I think that the queue accounting is something we'd do anyways, 
regardless of how the
queue is set. It just represents the io_cpu spread across cpu cores.

>
> Suggested-by: Sagi Grimberg <sagi@grimberg.me>
> Link: https://lore.kernel.org/linux-nvme/220e9da3-f756-4a16-8de1-d4b171f15009@grimberg.me/
> Assisted-by: Claude:claude-opus-4-8 [Claude Code]
> Signed-off-by: Saravanan D <saravanand@crusoe.ai>
> ---
> Changes since v2 [1]:
> - Replaced the io_cpu_adopt connect option and the submitter adoption
>    heuristic with a per queue writable sysfs attribute, following Sagi's
>    suggestion [2]. The control plane now sets each queue's io_cpu
>    directly, the assignment is kept across reconnects and writing -1
>    reverts to the connect time selection.
> - Retitled from "nvme-tcp: pin io_cpu to submitter cpu".
>
> The per queue directories follow the blk-mq mq/<hctx> sysfs pattern.
>
> Tested on a 2 socket 384 cpu host with 128 queue controllers. Writing
> a cpu number changed the queue's io_cpu to it. Writing an invalid
> value was rejected. Writing -1 re-ran the connect time selection.
> Pinned queues kept their io_cpu across a controller reset while
> unpinned queues received a fresh pick.
>
> The multiple queues per hctx RFC [3] found the same need to steer the
> socket work cpu, so this attribute may gain a second user.
>
> [1] https://lore.kernel.org/linux-nvme/20260820083634.71689-1-saravanand@crusoe.ai/
> [2] https://lore.kernel.org/linux-nvme/1d56144d-6987-40c1-ac02-b15333db121e@grimberg.me/
> [3] https://lore.kernel.org/linux-nvme/20260903152623.614951-1-kbusch@meta.com/
>
>   Documentation/ABI/stable/sysfs-nvme |  12 +++
>   drivers/nvme/host/tcp.c             | 153 +++++++++++++++++++++++++++-
>   2 files changed, 162 insertions(+), 3 deletions(-)
>
> diff --git a/Documentation/ABI/stable/sysfs-nvme b/Documentation/ABI/stable/sysfs-nvme
> index a2f5d0710db4..2bbb5a0b7c2e 100644
> --- a/Documentation/ABI/stable/sysfs-nvme
> +++ b/Documentation/ABI/stable/sysfs-nvme
> @@ -451,3 +451,15 @@ Contact:	Hannes Reinecke <hare@suse.de>
>   Description:
>   		Shows the subsystem type. Possible values: "discovery",
>   		"nvm", "reserved".
> +
> +What:		/sys/class/nvme/nvmeX/tcp_queues/<qid>/io_cpu
> +Date:		September 2026
> +KernelVersion:	7.4
> +Contact:	Saravanan D <saravanand@crusoe.ai>
> +Description:
> +		(RW) The CPU that runs the socket work for I/O queue <qid>
> +		of an NVMe over TCP controller, selected by the driver at
> +		connect time. Writing a CPU number overrides the selection
> +		and persists across reconnects. Writing -1 reverts to the
> +		driver's selection. Reads show the current CPU, or -1 when
> +		the queue is unbound.
> diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
> index 921934028e0b..22ad1fdf4a12 100644
> --- a/drivers/nvme/host/tcp.c
> +++ b/drivers/nvme/host/tcp.c
> @@ -93,6 +93,7 @@ enum nvme_tcp_queue_flags {
>   	NVME_TCP_Q_LIVE		= 1,
>   	NVME_TCP_Q_POLLING	= 2,
>   	NVME_TCP_Q_IO_CPU_SET	= 3,
> +	NVME_TCP_Q_IO_CPU_USER	= 4,
>   };
>   
>   enum nvme_tcp_recv_state {
> @@ -102,6 +103,17 @@ enum nvme_tcp_recv_state {
>   };
>   
>   struct nvme_tcp_ctrl;
> +struct nvme_tcp_queue;
> +
> +/*
> + * Allocated per registration and freed by its kobject release, so a
> + * reconnect never reuses a kobject whose release is still pending.
> + */
> +struct nvme_tcp_queue_kobj {
> +	struct kobject		kobj;
> +	struct nvme_tcp_queue	*queue;
> +};

I am wandering if it is time to introduce the core nvme queue:
struct nvme_queue {
         struct kobject        kobj;
         unsigned int          qid;
         u64                          flags;
};

And have the transport queue embed it:
struct nvme_tcp_queue {
         struct nvme_queue        nvmeq;
         ....
};

I suspect it will allow for better abstractions.

Thoughts? Keith, Christoph?

Re: [PATCH v3] nvme-tcp: allow setting per-queue io_cpu through sysfs

[Nilay Shroff]

On 9/12/26 2:41 AM, Sagi Grimberg wrote:
> 
> 
> On 10/09/2026 0:14, Saravanan D wrote:
>> nvme_tcp_set_queue_io_cpu() picks each queue's io_cpu at connect time
>> as the least loaded CPU in the queue's blk-mq map group, and all socket
>> work runs there for the connection's lifetime. This decision falls
>> short when the host partitions its CPUs after connect time. On a 384
>> cpu multi tenant host with 128 queue controllers, blk-mq folds three
>> CPUs into every map group, some groups straddle two tenants' cpusets,
>> and 9% of nvme_tcp_io_work executions ran outside the submitting VM's
>> cpuset, seen by the neighbor as steal time.
>>
>> Expose each I/O queue's io_cpu as a writable sysfs attribute
>>
>>    /sys/class/nvme/nvmeX/tcp_queues/<qid>/io_cpu
>>
>> so a control plane that owns CPU placement can set it directly instead
>> of relying on the driver's heuristic. A written value persists across
>> reconnects, marked by NVME_TCP_Q_IO_CPU_USER. Writing -1 clears the
>> mark and re-runs the connect time selection. Reading returns the CPU,
>> or -1 when the queue is unbound.
> 
> No need for NVME_TCP_Q_IO_CPU_USER flag and restoring to the original connect time selection. -1 would mean WORK_CPU_UNBOUND. In fact lets allow this file to access "unbound" string (same as -1). I don't think we need to restore the connect time selection, the user who touches these settings is obviously interested to be in control.
>>
>> The store, the connect time selection and queue stop serialize their
>> accounting of nvme_tcp_cpu_queues under the queue lock.
> 
> I think that the queue accounting is something we'd do anyways, regardless of how the
> queue is set. It just represents the io_cpu spread across cpu cores.
> 
>>
>> Suggested-by: Sagi Grimberg <sagi@grimberg.me>
>> Link: https://lore.kernel.org/linux-nvme/220e9da3-f756-4a16-8de1-d4b171f15009@grimberg.me/
>> Assisted-by: Claude:claude-opus-4-8 [Claude Code]
>> Signed-off-by: Saravanan D <saravanand@crusoe.ai>
>> ---
>> Changes since v2 [1]:
>> - Replaced the io_cpu_adopt connect option and the submitter adoption
>>    heuristic with a per queue writable sysfs attribute, following Sagi's
>>    suggestion [2]. The control plane now sets each queue's io_cpu
>>    directly, the assignment is kept across reconnects and writing -1
>>    reverts to the connect time selection.
>> - Retitled from "nvme-tcp: pin io_cpu to submitter cpu".
>>
>> The per queue directories follow the blk-mq mq/<hctx> sysfs pattern.
>>
>> Tested on a 2 socket 384 cpu host with 128 queue controllers. Writing
>> a cpu number changed the queue's io_cpu to it. Writing an invalid
>> value was rejected. Writing -1 re-ran the connect time selection.
>> Pinned queues kept their io_cpu across a controller reset while
>> unpinned queues received a fresh pick.
>>
>> The multiple queues per hctx RFC [3] found the same need to steer the
>> socket work cpu, so this attribute may gain a second user.
>>
>> [1] https://lore.kernel.org/linux-nvme/20260820083634.71689-1-saravanand@crusoe.ai/
>> [2] https://lore.kernel.org/linux-nvme/1d56144d-6987-40c1-ac02-b15333db121e@grimberg.me/
>> [3] https://lore.kernel.org/linux-nvme/20260903152623.614951-1-kbusch@meta.com/
>>
>>   Documentation/ABI/stable/sysfs-nvme |  12 +++
>>   drivers/nvme/host/tcp.c             | 153 +++++++++++++++++++++++++++-
>>   2 files changed, 162 insertions(+), 3 deletions(-)
>>
>> diff --git a/Documentation/ABI/stable/sysfs-nvme b/Documentation/ABI/stable/sysfs-nvme
>> index a2f5d0710db4..2bbb5a0b7c2e 100644
>> --- a/Documentation/ABI/stable/sysfs-nvme
>> +++ b/Documentation/ABI/stable/sysfs-nvme
>> @@ -451,3 +451,15 @@ Contact:    Hannes Reinecke <hare@suse.de>
>>   Description:
>>           Shows the subsystem type. Possible values: "discovery",
>>           "nvm", "reserved".
>> +
>> +What:        /sys/class/nvme/nvmeX/tcp_queues/<qid>/io_cpu
>> +Date:        September 2026
>> +KernelVersion:    7.4
>> +Contact:    Saravanan D <saravanand@crusoe.ai>
>> +Description:
>> +        (RW) The CPU that runs the socket work for I/O queue <qid>
>> +        of an NVMe over TCP controller, selected by the driver at
>> +        connect time. Writing a CPU number overrides the selection
>> +        and persists across reconnects. Writing -1 reverts to the
>> +        driver's selection. Reads show the current CPU, or -1 when
>> +        the queue is unbound.
>> diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
>> index 921934028e0b..22ad1fdf4a12 100644
>> --- a/drivers/nvme/host/tcp.c
>> +++ b/drivers/nvme/host/tcp.c
>> @@ -93,6 +93,7 @@ enum nvme_tcp_queue_flags {
>>       NVME_TCP_Q_LIVE        = 1,
>>       NVME_TCP_Q_POLLING    = 2,
>>       NVME_TCP_Q_IO_CPU_SET    = 3,
>> +    NVME_TCP_Q_IO_CPU_USER    = 4,
>>   };
>>   enum nvme_tcp_recv_state {
>> @@ -102,6 +103,17 @@ enum nvme_tcp_recv_state {
>>   };
>>   struct nvme_tcp_ctrl;
>> +struct nvme_tcp_queue;
>> +
>> +/*
>> + * Allocated per registration and freed by its kobject release, so a
>> + * reconnect never reuses a kobject whose release is still pending.
>> + */
>> +struct nvme_tcp_queue_kobj {
>> +    struct kobject        kobj;
>> +    struct nvme_tcp_queue    *queue;
>> +};
> 
> I am wandering if it is time to introduce the core nvme queue:
> struct nvme_queue {
>          struct kobject        kobj;
>          unsigned int          qid;
>          u64                          flags;
> };
> 
> And have the transport queue embed it:
> struct nvme_tcp_queue {
>          struct nvme_queue        nvmeq;
>          ....
> };
> 
> I suspect it will allow for better abstractions.
> 
> Thoughts? Keith, Christoph?

Yes, agreed. I think having a common queue structure would provide
a better abstraction here. However, struct nvme_queue is already
used by the PCI driver for its transport-specific queue representation.
Perhaps we could name common structure as struct nvme_queue_info.

Thanks,
--Nilay