[PATCH 1/3] nvme: Add adaptive PCIe link rate switching function

[PATCH 1/3] nvme: Add adaptive PCIe link rate switching function

[Liao Xuan]

Add support for dynamically adjusting the PCIe link rate of NVMe
devices based on I/O activity. The link is downgraded to a
configurable minimum rate during periods of low activity to save
power, and upgraded to the maximum supported rate as soon as the
workload exceeds a computed threshold, so that peak performance is
maintained under load.

The amount of transferred data is accumulated on the I/O submission
path and aggregated by a per-controller timer. The threshold is
derived from the bandwidth of the minimum and maximum rates such that
the extra time required to transfer the data at the minimum rate does
not exceed the estimated link retraining time.

The feature is currently only supported on Hygon platforms.

Signed-off-by: Liao Xuan <liaoxuan@open-hieco.net>
Signed-off-by: default avatarZheng Tan <tanzheng@kylinos.cn>
---
 drivers/nvme/host/Kconfig        |  10 +
 drivers/nvme/host/Makefile       |   1 +
 drivers/nvme/host/core.c         |   4 +
 drivers/nvme/host/nvme.h         |  42 ++++
 drivers/nvme/host/pci.c          |   4 +
 drivers/nvme/host/speed_switch.c | 367 +++++++++++++++++++++++++++++++
 drivers/pci/pcie/bwctrl.c        |   1 +
 7 files changed, 429 insertions(+)
 create mode 100644 drivers/nvme/host/speed_switch.c

diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig
index 31974c7dd20c..098ec7abba66 100644
--- a/drivers/nvme/host/Kconfig
+++ b/drivers/nvme/host/Kconfig
@@ -134,3 +134,13 @@ config NVME_APPLE
 
 	  To compile this driver as a module, choose M here: the
 	  module will be called nvme-apple.
+
+config NVME_SPEED_SWITCH
+	bool "NVMe adaptive PCIe link rate switching"
+	depends on BLK_DEV_NVME && X86
+	help
+	  This provides support for dynamic PCIe link rate switching for NVMe
+	  devices. When enabled, the NVMe driver can dynamically adjust the
+	  PCIe link speed and width based on workload requirement.
+
+	  If unsure, say N.
diff --git a/drivers/nvme/host/Makefile b/drivers/nvme/host/Makefile
index 67563a69f7dc..02d0200fcc88 100644
--- a/drivers/nvme/host/Makefile
+++ b/drivers/nvme/host/Makefile
@@ -19,6 +19,7 @@ nvme-core-$(CONFIG_BLK_DEV_ZONED)	+= zns.o
 nvme-core-$(CONFIG_FAULT_INJECTION_DEBUG_FS)	+= fault_inject.o
 nvme-core-$(CONFIG_NVME_HWMON)		+= hwmon.o
 nvme-core-$(CONFIG_NVME_HOST_AUTH)	+= auth.o
+nvme-core-$(CONFIG_NVME_SPEED_SWITCH)	+= speed_switch.o
 
 nvme-y					+= pci.o
 
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index beea23d04a70..85e0bfc4d7aa 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -4679,6 +4679,7 @@ static void nvme_scan_work(struct work_struct *work)
 		/* Re-read the ANA log page to not miss updates */
 		queue_work(nvme_wq, &ctrl->ana_work);
 #endif
+	nvme_speed_switch_start(ctrl);
 }
 
 /*
@@ -5103,6 +5104,7 @@ EXPORT_SYMBOL_GPL(nvme_remove_io_tag_set);
 
 void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
 {
+	nvme_speed_switch_exit(ctrl);
 	nvme_mpath_stop(ctrl);
 	nvme_auth_stop(ctrl);
 	nvme_stop_failfast_work(ctrl);
@@ -5318,6 +5320,8 @@ int nvme_add_ctrl(struct nvme_ctrl *ctrl)
 	nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
 	nvme_get_ctrl(ctrl);
 
+	nvme_speed_switch_init(ctrl);
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(nvme_add_ctrl);
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 2cff9fcbf740..0f27c73a99ee 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -339,6 +339,33 @@ enum nvme_ctrl_flags {
 	NVME_CTRL_FROZEN		= 6,
 };
 
+#ifdef CONFIG_NVME_SPEED_SWITCH
+struct nvme_speed_switch_stats {
+	u64 read_bytes;
+	u64 write_bytes;
+};
+
+struct nvme_speed_switch {
+	bool initialized;
+	bool enabled;
+	u8 max_speed;
+	u8 cur_speed;
+	u8 target_speed;
+	u8 min_speed;
+	u32 monitor_interval;	/* ms */
+	u32 threshold;		/* KB */
+	struct timer_list timer;
+	struct work_struct work;
+	struct nvme_speed_switch_stats __percpu *stats;
+	atomic_t timer_active;
+};
+
+enum nvme_speed_timer_state {
+	NVME_SPEED_TIMER_INACTIVE,
+	NVME_SPEED_TIMER_ACTIVE,
+};
+#endif
+
 struct nvme_ctrl {
 	bool comp_seen;
 	bool identified;
@@ -481,8 +508,23 @@ struct nvme_ctrl {
 	enum nvme_dctype dctype;
 
 	u16			awupf; /* 0's based value. */
+#ifdef CONFIG_NVME_SPEED_SWITCH
+	struct nvme_speed_switch speed_switch;
+#endif
 };
 
+#ifdef CONFIG_NVME_SPEED_SWITCH
+void nvme_update_io_stats(struct nvme_ctrl *ctrl, struct request *req);
+void nvme_speed_switch_start(struct nvme_ctrl *ctrl);
+void nvme_speed_switch_init(struct nvme_ctrl *ctrl);
+void nvme_speed_switch_exit(struct nvme_ctrl *ctrl);
+#else
+static inline void nvme_update_io_stats(struct nvme_ctrl *ctrl, struct request *req) {}
+static inline void nvme_speed_switch_start(struct nvme_ctrl *ctrl) {}
+static inline void nvme_speed_switch_init(struct nvme_ctrl *ctrl) {}
+static inline void nvme_speed_switch_exit(struct nvme_ctrl *ctrl) {}
+#endif
+
 static inline enum nvme_ctrl_state nvme_ctrl_state(struct nvme_ctrl *ctrl)
 {
 	return READ_ONCE(ctrl->state);
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 5440cf18b55b..dfba03409dc0 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -1461,6 +1461,9 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
 	ret = nvme_prep_rq(req);
 	if (unlikely(ret))
 		return ret;
+
+	nvme_update_io_stats(&dev->ctrl, req);
+
 	spin_lock(&nvmeq->sq_lock);
 	nvme_sq_copy_cmd(nvmeq, &iod->cmd);
 	nvme_write_sq_db(nvmeq, bd->last);
@@ -1479,6 +1482,7 @@ static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct rq_list *rqlist)
 	while ((req = rq_list_pop(rqlist))) {
 		struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 
+		nvme_update_io_stats(&nvmeq->dev->ctrl, req);
 		nvme_sq_copy_cmd(nvmeq, &iod->cmd);
 	}
 	nvme_write_sq_db(nvmeq, true);
diff --git a/drivers/nvme/host/speed_switch.c b/drivers/nvme/host/speed_switch.c
new file mode 100644
index 000000000000..0a828d04885b
--- /dev/null
+++ b/drivers/nvme/host/speed_switch.c
@@ -0,0 +1,367 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * NVMe link speed switch driver
+ *
+ * Adjusts the PCIe link rate of NVMe devices dynamically based on the I/O
+ * workload: the link is downgraded to a configurable minimum rate during
+ * periods of low activity to save power, and upgraded to the maximum
+ * supported rate as soon as the workload requires it.
+ *
+ * Copyright (c) 2026 Chengdu Haiguang IC Design Co., Ltd.
+ * Copyright (C) 2026 Kylin Software Co., Ltd.
+ *
+ */
+
+#include <linux/blk-mq.h>
+#include <linux/math64.h>
+#include <linux/pci.h>
+#include <linux/percpu.h>
+#include <linux/processor.h>
+
+#include "nvme.h"
+
+/*
+ * PCIe link bandwidth in KB/s for each generation (rows, Gen1..Gen5) and
+ * lane width (columns, x1..x16).  The table is used to derive the I/O
+ * threshold exceed/below which the link can be upgraded/downgraded.
+ */
+static const u32 nvme_speed_table[][5] = {
+	{  312,  625, 1250, 2500, 5000 },   /* Gen1 */
+	{  625, 1250, 2500, 5000, 10000 },  /* Gen2 */
+	{ 1000, 2000, 4000, 8000, 16000 },  /* Gen3 */
+	{ 2000, 4000, 8000, 16000, 32000 }, /* Gen4 */
+	{ 4000, 8000, 16000, 32000, 64000 },/* Gen5 */
+};
+
+static int nvme_bandwidth_index(enum pcie_link_width width)
+{
+	switch (width) {
+	case PCIE_LNK_X1:
+		return 0;
+	case PCIE_LNK_X2:
+		return 1;
+	case PCIE_LNK_X4:
+		return 2;
+	case PCIE_LNK_X8:
+		return 3;
+	case PCIE_LNK_X16:
+		return 4;
+	default:
+		return -EINVAL;
+	}
+}
+
+/*
+ * Read the maximum supported link rate of the upstream port from the Link
+ * Capability register.
+ */
+static int nvme_get_max_link_speed(struct pci_dev *pdev)
+{
+	u16 linkcap;
+
+	pcie_capability_read_word(pdev, PCI_EXP_LNKCAP, &linkcap);
+	return linkcap & PCI_EXP_LNKCAP_SLS;
+}
+
+static enum pci_bus_speed nvme_speed_to_bus_speed(u8 gen)
+{
+	return gen + PCIE_SPEED_2_5GT - 1;
+}
+
+/*
+ * Work item that actually changes the link rate.  The queues are frozen
+ * before retraining to quiesce I/O and resumed once the switch completed.
+ */
+static void nvme_speed_switch_work(struct work_struct *work)
+{
+	struct nvme_speed_switch *sw = container_of(work, struct nvme_speed_switch, work);
+	struct nvme_ctrl *ctrl = container_of(sw, struct nvme_ctrl, speed_switch);
+	struct pci_dev *pdev = to_pci_dev(ctrl->dev);
+	struct pci_dev *bridge = pdev->bus->self;
+	u8 cur_speed = READ_ONCE(sw->cur_speed);
+	int ret;
+
+	if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)
+		return;
+
+	/* Freeze I/O to avoid timeouts during link retraining. */
+	nvme_start_freeze(ctrl);
+	nvme_wait_freeze(ctrl);
+
+	ret = pcie_set_target_speed(bridge, nvme_speed_to_bus_speed(sw->target_speed),
+				    true);
+	if (ret) {
+		dev_warn(ctrl->device,
+			 "failed to set target rate Gen%u (%d), trying rollback to Gen%u\n",
+			 sw->target_speed, ret, cur_speed);
+		ret = pcie_set_target_speed(bridge, nvme_speed_to_bus_speed(cur_speed),
+					    true);
+		if (ret) {
+			dev_err(ctrl->device,
+				"rollback to Gen%u failed (%d), disabling speed switch\n",
+				cur_speed, ret);
+			WRITE_ONCE(sw->enabled, false);
+			nvme_unfreeze(ctrl);
+			return;
+		}
+	} else {
+		WRITE_ONCE(sw->cur_speed, sw->target_speed);
+	}
+
+	nvme_unfreeze(ctrl);
+	dev_dbg(ctrl->device, "link rate changed to Gen%u\n",
+		READ_ONCE(sw->cur_speed));
+}
+
+/*
+ * Compute the I/O threshold (in KB) for a monitoring window.
+ *
+ * The threshold is derived from the following constraint: for the link
+ * speed to be upgraded, the transfer time of the data at the minimum
+ * speed must be greater than the sum of the transfer time at maximum
+ * speed and the link retraining time (100us):
+ *
+ *	data / min_bw >= data / max_bw + 100us
+ *
+ * which yields data <= min_bw * max_bw * 0.1ms / (max_bw - min_bw).
+ * The bandwidths are taken from the link capability of the upstream port
+ * and the link width capability of the device.
+ */
+static u32 nvme_calc_speed_threshold(struct nvme_speed_switch *sw,
+				     enum pcie_link_width max_width)
+{
+	struct nvme_ctrl *ctrl = container_of(sw, struct nvme_ctrl, speed_switch);
+	u64 min_bw, max_bw, x, y;
+	int bw_idx, min_gen = sw->min_speed, max_gen = sw->max_speed;
+	u32 threshold;
+
+	if (min_gen < 1 || min_gen > 5 || max_gen < 1 || max_gen > 5) {
+		dev_warn(ctrl->device,
+			 "invalid link rate range for threshold: min=Gen%d, max=Gen%d\n",
+			 min_gen, max_gen);
+		return 0;
+	}
+
+	bw_idx = nvme_bandwidth_index(max_width);
+	if (bw_idx < 0) {
+		dev_warn(ctrl->device,
+			 "unsupported link width (%d) for threshold calculation\n",
+			 max_width);
+		return 0;
+	}
+
+	min_bw = nvme_speed_table[min_gen - 1][bw_idx];
+	max_bw = nvme_speed_table[max_gen - 1][bw_idx];
+	if (min_bw >= max_bw) {
+		dev_warn(ctrl->device,
+			 "invalid bandwidth for threshold: min=%llu KB/s, max=%llu KB/s\n",
+			 min_bw, max_bw);
+		return 0;
+	}
+
+	x = min_bw * max_bw;
+	y = 10ULL * (max_bw - min_bw);
+	threshold = div64_u64(x + (y >> 1), y);
+
+	dev_dbg(ctrl->device,
+		"I/O threshold: %u KB (min=Gen%d, max=Gen%d, width=x%d)\n",
+		threshold, min_gen, max_gen, max_width);
+	return threshold;
+}
+
+/* Called on the I/O submission path to accumulate the transferred bytes. */
+void nvme_update_io_stats(struct nvme_ctrl *ctrl, struct request *req)
+{
+	struct nvme_speed_switch *sw = &ctrl->speed_switch;
+	struct nvme_speed_switch_stats *stat;
+	enum req_op op = req_op(req);
+	unsigned int bytes = blk_rq_bytes(req);
+
+	if (!READ_ONCE(sw->enabled))
+		return;
+
+	if (op != REQ_OP_READ && op != REQ_OP_WRITE)
+		return;
+
+	if (!sw->stats) {
+		dev_dbg(ctrl->device, "I/O statistics not allocated, skip accounting\n");
+		return;
+	}
+
+	stat = get_cpu_ptr(sw->stats);
+	if (op == REQ_OP_READ)
+		stat->read_bytes += bytes;
+	else
+		stat->write_bytes += bytes;
+	put_cpu_ptr(sw->stats);
+
+	if (atomic_cmpxchg(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE,
+			   NVME_SPEED_TIMER_ACTIVE) != NVME_SPEED_TIMER_INACTIVE)
+		return;
+
+	/* First I/O of the activity period: compute the threshold and arm the timer. */
+	WRITE_ONCE(sw->threshold, nvme_calc_speed_threshold(sw,
+					pcie_get_width_cap(to_pci_dev(ctrl->dev))));
+	if (!READ_ONCE(sw->threshold)) {
+		dev_warn(ctrl->device,
+			 "failed to compute speed switch threshold, keeping current rate\n");
+		atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE);
+		return;
+	}
+	mod_timer(&sw->timer, jiffies + msecs_to_jiffies(READ_ONCE(sw->monitor_interval)));
+}
+EXPORT_SYMBOL_GPL(nvme_update_io_stats);
+
+static void nvme_clear_io_stats(struct nvme_speed_switch *sw)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		struct nvme_speed_switch_stats *stat = per_cpu_ptr(sw->stats, cpu);
+
+		stat->read_bytes = 0;
+		stat->write_bytes = 0;
+	}
+}
+
+/*
+ * Aggregate the per-CPU I/O counters of the last monitoring window and
+ * decide the target rate: the maximum rate if the window exceeded the
+ * threshold, the minimum rate otherwise.
+ */
+static int nvme_check_io_and_decide_speed(struct nvme_speed_switch *sw)
+{
+	struct nvme_ctrl *ctrl = container_of(sw, struct nvme_ctrl, speed_switch);
+	u64 read_bytes = 0, write_bytes = 0;
+	unsigned long read_kb, write_kb;
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		struct nvme_speed_switch_stats *stat = per_cpu_ptr(sw->stats, cpu);
+
+		read_bytes += stat->read_bytes;
+		write_bytes += stat->write_bytes;
+		stat->read_bytes = 0;
+		stat->write_bytes = 0;
+	}
+
+	read_kb = read_bytes / 1024;
+	write_kb = write_bytes / 1024;
+	dev_dbg(ctrl->device,
+		"I/O in window: read=%lu KB, write=%lu KB, threshold=%u KB\n",
+		read_kb, write_kb, READ_ONCE(sw->threshold));
+
+	if (read_kb >= sw->threshold || write_kb >= sw->threshold)
+		return sw->max_speed;
+	return sw->min_speed;
+}
+
+static void nvme_speed_switch_timer_fn(struct timer_list *t)
+{
+	struct nvme_speed_switch *sw = container_of(t, struct nvme_speed_switch, timer);
+
+	if (!READ_ONCE(sw->enabled)) {
+		atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE);
+		return;
+	}
+
+	sw->target_speed = nvme_check_io_and_decide_speed(sw);
+	if (sw->target_speed != READ_ONCE(sw->cur_speed))
+		schedule_work(&sw->work);
+
+	mod_timer(t, jiffies + msecs_to_jiffies(READ_ONCE(sw->monitor_interval)));
+}
+
+static void nvme_speed_switch_params_init(struct nvme_speed_switch *sw)
+{
+	sw->monitor_interval = 100;
+	sw->min_speed = PCI_EXP_LNKSTA_CLS_2_5GB;
+}
+
+void nvme_speed_switch_init(struct nvme_ctrl *ctrl)
+{
+	struct nvme_speed_switch *sw = &ctrl->speed_switch;
+	struct pci_dev *pdev;
+	int max_speed;
+
+	if (boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) {
+		dev_dbg(ctrl->device,
+			"link rate switching is only supported on Hygon platforms\n");
+		return;
+	}
+
+	if (!dev_is_pci(ctrl->dev)) {
+		dev_err(ctrl->device, "link rate switching requires a PCI device\n");
+		return;
+	}
+	pdev = to_pci_dev(ctrl->dev);
+	if (!pdev->bus->self) {
+		dev_err(ctrl->device,
+			"link rate switching requires a PCIe upstream port\n");
+		return;
+	}
+
+	nvme_speed_switch_params_init(sw);
+
+	max_speed = nvme_get_max_link_speed(pdev->bus->self);
+	if (max_speed <= sw->min_speed) {
+		dev_err(ctrl->device,
+			"invalid link rate range: min=Gen%u, max=Gen%d\n",
+			sw->min_speed, max_speed);
+		return;
+	}
+	sw->max_speed = max_speed;
+
+	sw->stats = alloc_percpu(struct nvme_speed_switch_stats);
+	if (!sw->stats) {
+		dev_err(ctrl->device, "failed to allocate per-CPU I/O statistics\n");
+		return;
+	}
+	nvme_clear_io_stats(sw);
+
+	WRITE_ONCE(sw->cur_speed, sw->max_speed);
+	WRITE_ONCE(sw->threshold, nvme_calc_speed_threshold(sw, pcie_get_width_cap(pdev)));
+	if (!READ_ONCE(sw->threshold)) {
+		dev_err(ctrl->device, "failed to compute speed switch threshold\n");
+		free_percpu(sw->stats);
+		sw->stats = NULL;
+		return;
+	}
+
+	INIT_WORK(&sw->work, nvme_speed_switch_work);
+	timer_setup(&sw->timer, nvme_speed_switch_timer_fn, 0);
+	atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE);
+	WRITE_ONCE(sw->enabled, true);
+	sw->initialized = true;
+
+	dev_info(ctrl->device,
+		 "NVMe link rate switching initialized (min Gen%u, max Gen%d)\n",
+		 sw->min_speed, sw->max_speed);
+}
+
+void nvme_speed_switch_start(struct nvme_ctrl *ctrl)
+{
+	struct nvme_speed_switch *sw = &ctrl->speed_switch;
+
+	if (!READ_ONCE(sw->enabled))
+		return;
+
+	atomic_set(&sw->timer_active, NVME_SPEED_TIMER_ACTIVE);
+	mod_timer(&sw->timer, jiffies + msecs_to_jiffies(READ_ONCE(sw->monitor_interval)));
+}
+
+void nvme_speed_switch_exit(struct nvme_ctrl *ctrl)
+{
+	struct nvme_speed_switch *sw = &ctrl->speed_switch;
+
+	if (!sw->initialized)
+		return;
+
+	sw->initialized = false;
+	WRITE_ONCE(sw->enabled, false);
+	timer_delete_sync(&sw->timer);
+	cancel_work_sync(&sw->work);
+
+	free_percpu(sw->stats);
+	sw->stats = NULL;
+}
diff --git a/drivers/pci/pcie/bwctrl.c b/drivers/pci/pcie/bwctrl.c
index d48021bfd844..4b28e26f9531 100644
--- a/drivers/pci/pcie/bwctrl.c
+++ b/drivers/pci/pcie/bwctrl.c
@@ -156,6 +156,7 @@ int pcie_set_target_speed(struct pci_dev *port, enum pci_bus_speed speed_req,
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(pcie_set_target_speed);
 
 static void pcie_bwnotif_enable(struct pcie_device *srv)
 {
-- 
2.43.0

Re: [PATCH 1/3] nvme: Add adaptive PCIe link rate switching function

[Bjorn Helgaas]

[+cc Ilpo, bwctrl]

On Tue, Nov 03, 2026 at 02:25:43PM +0800, Liao Xuan wrote:
> Add support for dynamically adjusting the PCIe link rate of NVMe
> devices based on I/O activity. The link is downgraded to a
> configurable minimum rate during periods of low activity to save
> power, and upgraded to the maximum supported rate as soon as the
> workload exceeds a computed threshold, so that peak performance is
> maintained under load.
> 
> The amount of transferred data is accumulated on the I/O submission
> path and aggregated by a per-controller timer. The threshold is
> derived from the bandwidth of the minimum and maximum rates such that
> the extra time required to transfer the data at the minimum rate does
> not exceed the estimated link retraining time.
> 
> The feature is currently only supported on Hygon platforms.

Why?

> Signed-off-by: Liao Xuan <liaoxuan@open-hieco.net>
> Signed-off-by: default avatarZheng Tan <tanzheng@kylinos.cn>
> ---
>  drivers/nvme/host/Kconfig        |  10 +
>  drivers/nvme/host/Makefile       |   1 +
>  drivers/nvme/host/core.c         |   4 +
>  drivers/nvme/host/nvme.h         |  42 ++++
>  drivers/nvme/host/pci.c          |   4 +
>  drivers/nvme/host/speed_switch.c | 367 +++++++++++++++++++++++++++++++
>  drivers/pci/pcie/bwctrl.c        |   1 +
>  7 files changed, 429 insertions(+)
>  create mode 100644 drivers/nvme/host/speed_switch.c
> 
> diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig
> index 31974c7dd20c..098ec7abba66 100644
> --- a/drivers/nvme/host/Kconfig
> +++ b/drivers/nvme/host/Kconfig
> @@ -134,3 +134,13 @@ config NVME_APPLE
>  
>  	  To compile this driver as a module, choose M here: the
>  	  module will be called nvme-apple.
> +
> +config NVME_SPEED_SWITCH
> +	bool "NVMe adaptive PCIe link rate switching"
> +	depends on BLK_DEV_NVME && X86
> +	help
> +	  This provides support for dynamic PCIe link rate switching for NVMe
> +	  devices. When enabled, the NVMe driver can dynamically adjust the
> +	  PCIe link speed and width based on workload requirement.
> +
> +	  If unsure, say N.
> diff --git a/drivers/nvme/host/Makefile b/drivers/nvme/host/Makefile
> index 67563a69f7dc..02d0200fcc88 100644
> --- a/drivers/nvme/host/Makefile
> +++ b/drivers/nvme/host/Makefile
> @@ -19,6 +19,7 @@ nvme-core-$(CONFIG_BLK_DEV_ZONED)	+= zns.o
>  nvme-core-$(CONFIG_FAULT_INJECTION_DEBUG_FS)	+= fault_inject.o
>  nvme-core-$(CONFIG_NVME_HWMON)		+= hwmon.o
>  nvme-core-$(CONFIG_NVME_HOST_AUTH)	+= auth.o
> +nvme-core-$(CONFIG_NVME_SPEED_SWITCH)	+= speed_switch.o
>  
>  nvme-y					+= pci.o
>  
> diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
> index beea23d04a70..85e0bfc4d7aa 100644
> --- a/drivers/nvme/host/core.c
> +++ b/drivers/nvme/host/core.c
> @@ -4679,6 +4679,7 @@ static void nvme_scan_work(struct work_struct *work)
>  		/* Re-read the ANA log page to not miss updates */
>  		queue_work(nvme_wq, &ctrl->ana_work);
>  #endif
> +	nvme_speed_switch_start(ctrl);
>  }
>  
>  /*
> @@ -5103,6 +5104,7 @@ EXPORT_SYMBOL_GPL(nvme_remove_io_tag_set);
>  
>  void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
>  {
> +	nvme_speed_switch_exit(ctrl);
>  	nvme_mpath_stop(ctrl);
>  	nvme_auth_stop(ctrl);
>  	nvme_stop_failfast_work(ctrl);
> @@ -5318,6 +5320,8 @@ int nvme_add_ctrl(struct nvme_ctrl *ctrl)
>  	nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
>  	nvme_get_ctrl(ctrl);
>  
> +	nvme_speed_switch_init(ctrl);
> +
>  	return 0;
>  }
>  EXPORT_SYMBOL_GPL(nvme_add_ctrl);
> diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
> index 2cff9fcbf740..0f27c73a99ee 100644
> --- a/drivers/nvme/host/nvme.h
> +++ b/drivers/nvme/host/nvme.h
> @@ -339,6 +339,33 @@ enum nvme_ctrl_flags {
>  	NVME_CTRL_FROZEN		= 6,
>  };
>  
> +#ifdef CONFIG_NVME_SPEED_SWITCH
> +struct nvme_speed_switch_stats {
> +	u64 read_bytes;
> +	u64 write_bytes;
> +};
> +
> +struct nvme_speed_switch {
> +	bool initialized;
> +	bool enabled;
> +	u8 max_speed;
> +	u8 cur_speed;
> +	u8 target_speed;
> +	u8 min_speed;
> +	u32 monitor_interval;	/* ms */
> +	u32 threshold;		/* KB */
> +	struct timer_list timer;
> +	struct work_struct work;
> +	struct nvme_speed_switch_stats __percpu *stats;
> +	atomic_t timer_active;
> +};
> +
> +enum nvme_speed_timer_state {
> +	NVME_SPEED_TIMER_INACTIVE,
> +	NVME_SPEED_TIMER_ACTIVE,
> +};
> +#endif
> +
>  struct nvme_ctrl {
>  	bool comp_seen;
>  	bool identified;
> @@ -481,8 +508,23 @@ struct nvme_ctrl {
>  	enum nvme_dctype dctype;
>  
>  	u16			awupf; /* 0's based value. */
> +#ifdef CONFIG_NVME_SPEED_SWITCH
> +	struct nvme_speed_switch speed_switch;
> +#endif
>  };
>  
> +#ifdef CONFIG_NVME_SPEED_SWITCH
> +void nvme_update_io_stats(struct nvme_ctrl *ctrl, struct request *req);
> +void nvme_speed_switch_start(struct nvme_ctrl *ctrl);
> +void nvme_speed_switch_init(struct nvme_ctrl *ctrl);
> +void nvme_speed_switch_exit(struct nvme_ctrl *ctrl);
> +#else
> +static inline void nvme_update_io_stats(struct nvme_ctrl *ctrl, struct request *req) {}
> +static inline void nvme_speed_switch_start(struct nvme_ctrl *ctrl) {}
> +static inline void nvme_speed_switch_init(struct nvme_ctrl *ctrl) {}
> +static inline void nvme_speed_switch_exit(struct nvme_ctrl *ctrl) {}
> +#endif
> +
>  static inline enum nvme_ctrl_state nvme_ctrl_state(struct nvme_ctrl *ctrl)
>  {
>  	return READ_ONCE(ctrl->state);
> diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
> index 5440cf18b55b..dfba03409dc0 100644
> --- a/drivers/nvme/host/pci.c
> +++ b/drivers/nvme/host/pci.c
> @@ -1461,6 +1461,9 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
>  	ret = nvme_prep_rq(req);
>  	if (unlikely(ret))
>  		return ret;
> +
> +	nvme_update_io_stats(&dev->ctrl, req);
> +
>  	spin_lock(&nvmeq->sq_lock);
>  	nvme_sq_copy_cmd(nvmeq, &iod->cmd);
>  	nvme_write_sq_db(nvmeq, bd->last);
> @@ -1479,6 +1482,7 @@ static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct rq_list *rqlist)
>  	while ((req = rq_list_pop(rqlist))) {
>  		struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
>  
> +		nvme_update_io_stats(&nvmeq->dev->ctrl, req);
>  		nvme_sq_copy_cmd(nvmeq, &iod->cmd);
>  	}
>  	nvme_write_sq_db(nvmeq, true);
> diff --git a/drivers/nvme/host/speed_switch.c b/drivers/nvme/host/speed_switch.c
> new file mode 100644
> index 000000000000..0a828d04885b
> --- /dev/null
> +++ b/drivers/nvme/host/speed_switch.c
> @@ -0,0 +1,367 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * NVMe link speed switch driver
> + *
> + * Adjusts the PCIe link rate of NVMe devices dynamically based on the I/O
> + * workload: the link is downgraded to a configurable minimum rate during
> + * periods of low activity to save power, and upgraded to the maximum
> + * supported rate as soon as the workload requires it.
> + *
> + * Copyright (c) 2026 Chengdu Haiguang IC Design Co., Ltd.
> + * Copyright (C) 2026 Kylin Software Co., Ltd.
> + *
> + */
> +
> +#include <linux/blk-mq.h>
> +#include <linux/math64.h>
> +#include <linux/pci.h>
> +#include <linux/percpu.h>
> +#include <linux/processor.h>
> +
> +#include "nvme.h"
> +
> +/*
> + * PCIe link bandwidth in KB/s for each generation (rows, Gen1..Gen5) and
> + * lane width (columns, x1..x16).  The table is used to derive the I/O
> + * threshold exceed/below which the link can be upgraded/downgraded.
> + */
> +static const u32 nvme_speed_table[][5] = {
> +	{  312,  625, 1250, 2500, 5000 },   /* Gen1 */
> +	{  625, 1250, 2500, 5000, 10000 },  /* Gen2 */
> +	{ 1000, 2000, 4000, 8000, 16000 },  /* Gen3 */
> +	{ 2000, 4000, 8000, 16000, 32000 }, /* Gen4 */
> +	{ 4000, 8000, 16000, 32000, 64000 },/* Gen5 */
> +};
> +
> +static int nvme_bandwidth_index(enum pcie_link_width width)
> +{
> +	switch (width) {
> +	case PCIE_LNK_X1:
> +		return 0;
> +	case PCIE_LNK_X2:
> +		return 1;
> +	case PCIE_LNK_X4:
> +		return 2;
> +	case PCIE_LNK_X8:
> +		return 3;
> +	case PCIE_LNK_X16:
> +		return 4;
> +	default:
> +		return -EINVAL;
> +	}
> +}
> +
> +/*
> + * Read the maximum supported link rate of the upstream port from the Link
> + * Capability register.
> + */
> +static int nvme_get_max_link_speed(struct pci_dev *pdev)
> +{
> +	u16 linkcap;
> +
> +	pcie_capability_read_word(pdev, PCI_EXP_LNKCAP, &linkcap);
> +	return linkcap & PCI_EXP_LNKCAP_SLS;
> +}
> +
> +static enum pci_bus_speed nvme_speed_to_bus_speed(u8 gen)
> +{
> +	return gen + PCIE_SPEED_2_5GT - 1;
> +}
> +
> +/*
> + * Work item that actually changes the link rate.  The queues are frozen
> + * before retraining to quiesce I/O and resumed once the switch completed.
> + */
> +static void nvme_speed_switch_work(struct work_struct *work)
> +{
> +	struct nvme_speed_switch *sw = container_of(work, struct nvme_speed_switch, work);
> +	struct nvme_ctrl *ctrl = container_of(sw, struct nvme_ctrl, speed_switch);
> +	struct pci_dev *pdev = to_pci_dev(ctrl->dev);
> +	struct pci_dev *bridge = pdev->bus->self;
> +	u8 cur_speed = READ_ONCE(sw->cur_speed);
> +	int ret;
> +
> +	if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)
> +		return;
> +
> +	/* Freeze I/O to avoid timeouts during link retraining. */
> +	nvme_start_freeze(ctrl);
> +	nvme_wait_freeze(ctrl);
> +
> +	ret = pcie_set_target_speed(bridge, nvme_speed_to_bus_speed(sw->target_speed),
> +				    true);
> +	if (ret) {
> +		dev_warn(ctrl->device,
> +			 "failed to set target rate Gen%u (%d), trying rollback to Gen%u\n",
> +			 sw->target_speed, ret, cur_speed);
> +		ret = pcie_set_target_speed(bridge, nvme_speed_to_bus_speed(cur_speed),
> +					    true);
> +		if (ret) {
> +			dev_err(ctrl->device,
> +				"rollback to Gen%u failed (%d), disabling speed switch\n",
> +				cur_speed, ret);
> +			WRITE_ONCE(sw->enabled, false);
> +			nvme_unfreeze(ctrl);
> +			return;
> +		}
> +	} else {
> +		WRITE_ONCE(sw->cur_speed, sw->target_speed);
> +	}
> +
> +	nvme_unfreeze(ctrl);
> +	dev_dbg(ctrl->device, "link rate changed to Gen%u\n",
> +		READ_ONCE(sw->cur_speed));
> +}
> +
> +/*
> + * Compute the I/O threshold (in KB) for a monitoring window.
> + *
> + * The threshold is derived from the following constraint: for the link
> + * speed to be upgraded, the transfer time of the data at the minimum
> + * speed must be greater than the sum of the transfer time at maximum
> + * speed and the link retraining time (100us):
> + *
> + *	data / min_bw >= data / max_bw + 100us
> + *
> + * which yields data <= min_bw * max_bw * 0.1ms / (max_bw - min_bw).
> + * The bandwidths are taken from the link capability of the upstream port
> + * and the link width capability of the device.
> + */
> +static u32 nvme_calc_speed_threshold(struct nvme_speed_switch *sw,
> +				     enum pcie_link_width max_width)
> +{
> +	struct nvme_ctrl *ctrl = container_of(sw, struct nvme_ctrl, speed_switch);
> +	u64 min_bw, max_bw, x, y;
> +	int bw_idx, min_gen = sw->min_speed, max_gen = sw->max_speed;
> +	u32 threshold;
> +
> +	if (min_gen < 1 || min_gen > 5 || max_gen < 1 || max_gen > 5) {
> +		dev_warn(ctrl->device,
> +			 "invalid link rate range for threshold: min=Gen%d, max=Gen%d\n",
> +			 min_gen, max_gen);
> +		return 0;
> +	}
> +
> +	bw_idx = nvme_bandwidth_index(max_width);
> +	if (bw_idx < 0) {
> +		dev_warn(ctrl->device,
> +			 "unsupported link width (%d) for threshold calculation\n",
> +			 max_width);
> +		return 0;
> +	}
> +
> +	min_bw = nvme_speed_table[min_gen - 1][bw_idx];
> +	max_bw = nvme_speed_table[max_gen - 1][bw_idx];
> +	if (min_bw >= max_bw) {
> +		dev_warn(ctrl->device,
> +			 "invalid bandwidth for threshold: min=%llu KB/s, max=%llu KB/s\n",
> +			 min_bw, max_bw);
> +		return 0;
> +	}
> +
> +	x = min_bw * max_bw;
> +	y = 10ULL * (max_bw - min_bw);
> +	threshold = div64_u64(x + (y >> 1), y);
> +
> +	dev_dbg(ctrl->device,
> +		"I/O threshold: %u KB (min=Gen%d, max=Gen%d, width=x%d)\n",
> +		threshold, min_gen, max_gen, max_width);
> +	return threshold;
> +}
> +
> +/* Called on the I/O submission path to accumulate the transferred bytes. */
> +void nvme_update_io_stats(struct nvme_ctrl *ctrl, struct request *req)
> +{
> +	struct nvme_speed_switch *sw = &ctrl->speed_switch;
> +	struct nvme_speed_switch_stats *stat;
> +	enum req_op op = req_op(req);
> +	unsigned int bytes = blk_rq_bytes(req);
> +
> +	if (!READ_ONCE(sw->enabled))
> +		return;
> +
> +	if (op != REQ_OP_READ && op != REQ_OP_WRITE)
> +		return;
> +
> +	if (!sw->stats) {
> +		dev_dbg(ctrl->device, "I/O statistics not allocated, skip accounting\n");
> +		return;
> +	}
> +
> +	stat = get_cpu_ptr(sw->stats);
> +	if (op == REQ_OP_READ)
> +		stat->read_bytes += bytes;
> +	else
> +		stat->write_bytes += bytes;
> +	put_cpu_ptr(sw->stats);
> +
> +	if (atomic_cmpxchg(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE,
> +			   NVME_SPEED_TIMER_ACTIVE) != NVME_SPEED_TIMER_INACTIVE)
> +		return;
> +
> +	/* First I/O of the activity period: compute the threshold and arm the timer. */
> +	WRITE_ONCE(sw->threshold, nvme_calc_speed_threshold(sw,
> +					pcie_get_width_cap(to_pci_dev(ctrl->dev))));
> +	if (!READ_ONCE(sw->threshold)) {
> +		dev_warn(ctrl->device,
> +			 "failed to compute speed switch threshold, keeping current rate\n");
> +		atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE);
> +		return;
> +	}
> +	mod_timer(&sw->timer, jiffies + msecs_to_jiffies(READ_ONCE(sw->monitor_interval)));
> +}
> +EXPORT_SYMBOL_GPL(nvme_update_io_stats);
> +
> +static void nvme_clear_io_stats(struct nvme_speed_switch *sw)
> +{
> +	int cpu;
> +
> +	for_each_possible_cpu(cpu) {
> +		struct nvme_speed_switch_stats *stat = per_cpu_ptr(sw->stats, cpu);
> +
> +		stat->read_bytes = 0;
> +		stat->write_bytes = 0;
> +	}
> +}
> +
> +/*
> + * Aggregate the per-CPU I/O counters of the last monitoring window and
> + * decide the target rate: the maximum rate if the window exceeded the
> + * threshold, the minimum rate otherwise.
> + */
> +static int nvme_check_io_and_decide_speed(struct nvme_speed_switch *sw)
> +{
> +	struct nvme_ctrl *ctrl = container_of(sw, struct nvme_ctrl, speed_switch);
> +	u64 read_bytes = 0, write_bytes = 0;
> +	unsigned long read_kb, write_kb;
> +	int cpu;
> +
> +	for_each_possible_cpu(cpu) {
> +		struct nvme_speed_switch_stats *stat = per_cpu_ptr(sw->stats, cpu);
> +
> +		read_bytes += stat->read_bytes;
> +		write_bytes += stat->write_bytes;
> +		stat->read_bytes = 0;
> +		stat->write_bytes = 0;
> +	}
> +
> +	read_kb = read_bytes / 1024;
> +	write_kb = write_bytes / 1024;
> +	dev_dbg(ctrl->device,
> +		"I/O in window: read=%lu KB, write=%lu KB, threshold=%u KB\n",
> +		read_kb, write_kb, READ_ONCE(sw->threshold));
> +
> +	if (read_kb >= sw->threshold || write_kb >= sw->threshold)
> +		return sw->max_speed;
> +	return sw->min_speed;
> +}
> +
> +static void nvme_speed_switch_timer_fn(struct timer_list *t)
> +{
> +	struct nvme_speed_switch *sw = container_of(t, struct nvme_speed_switch, timer);
> +
> +	if (!READ_ONCE(sw->enabled)) {
> +		atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE);
> +		return;
> +	}
> +
> +	sw->target_speed = nvme_check_io_and_decide_speed(sw);
> +	if (sw->target_speed != READ_ONCE(sw->cur_speed))
> +		schedule_work(&sw->work);
> +
> +	mod_timer(t, jiffies + msecs_to_jiffies(READ_ONCE(sw->monitor_interval)));
> +}
> +
> +static void nvme_speed_switch_params_init(struct nvme_speed_switch *sw)
> +{
> +	sw->monitor_interval = 100;
> +	sw->min_speed = PCI_EXP_LNKSTA_CLS_2_5GB;
> +}
> +
> +void nvme_speed_switch_init(struct nvme_ctrl *ctrl)
> +{
> +	struct nvme_speed_switch *sw = &ctrl->speed_switch;
> +	struct pci_dev *pdev;
> +	int max_speed;
> +
> +	if (boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) {
> +		dev_dbg(ctrl->device,
> +			"link rate switching is only supported on Hygon platforms\n");
> +		return;
> +	}
> +
> +	if (!dev_is_pci(ctrl->dev)) {
> +		dev_err(ctrl->device, "link rate switching requires a PCI device\n");
> +		return;
> +	}
> +	pdev = to_pci_dev(ctrl->dev);
> +	if (!pdev->bus->self) {
> +		dev_err(ctrl->device,
> +			"link rate switching requires a PCIe upstream port\n");
> +		return;
> +	}
> +
> +	nvme_speed_switch_params_init(sw);
> +
> +	max_speed = nvme_get_max_link_speed(pdev->bus->self);
> +	if (max_speed <= sw->min_speed) {
> +		dev_err(ctrl->device,
> +			"invalid link rate range: min=Gen%u, max=Gen%d\n",
> +			sw->min_speed, max_speed);
> +		return;
> +	}
> +	sw->max_speed = max_speed;
> +
> +	sw->stats = alloc_percpu(struct nvme_speed_switch_stats);
> +	if (!sw->stats) {
> +		dev_err(ctrl->device, "failed to allocate per-CPU I/O statistics\n");
> +		return;
> +	}
> +	nvme_clear_io_stats(sw);
> +
> +	WRITE_ONCE(sw->cur_speed, sw->max_speed);
> +	WRITE_ONCE(sw->threshold, nvme_calc_speed_threshold(sw, pcie_get_width_cap(pdev)));
> +	if (!READ_ONCE(sw->threshold)) {
> +		dev_err(ctrl->device, "failed to compute speed switch threshold\n");
> +		free_percpu(sw->stats);
> +		sw->stats = NULL;
> +		return;
> +	}
> +
> +	INIT_WORK(&sw->work, nvme_speed_switch_work);
> +	timer_setup(&sw->timer, nvme_speed_switch_timer_fn, 0);
> +	atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE);
> +	WRITE_ONCE(sw->enabled, true);
> +	sw->initialized = true;
> +
> +	dev_info(ctrl->device,
> +		 "NVMe link rate switching initialized (min Gen%u, max Gen%d)\n",
> +		 sw->min_speed, sw->max_speed);
> +}
> +
> +void nvme_speed_switch_start(struct nvme_ctrl *ctrl)
> +{
> +	struct nvme_speed_switch *sw = &ctrl->speed_switch;
> +
> +	if (!READ_ONCE(sw->enabled))
> +		return;
> +
> +	atomic_set(&sw->timer_active, NVME_SPEED_TIMER_ACTIVE);
> +	mod_timer(&sw->timer, jiffies + msecs_to_jiffies(READ_ONCE(sw->monitor_interval)));
> +}
> +
> +void nvme_speed_switch_exit(struct nvme_ctrl *ctrl)
> +{
> +	struct nvme_speed_switch *sw = &ctrl->speed_switch;
> +
> +	if (!sw->initialized)
> +		return;
> +
> +	sw->initialized = false;
> +	WRITE_ONCE(sw->enabled, false);
> +	timer_delete_sync(&sw->timer);
> +	cancel_work_sync(&sw->work);
> +
> +	free_percpu(sw->stats);
> +	sw->stats = NULL;
> +}
> diff --git a/drivers/pci/pcie/bwctrl.c b/drivers/pci/pcie/bwctrl.c
> index d48021bfd844..4b28e26f9531 100644
> --- a/drivers/pci/pcie/bwctrl.c
> +++ b/drivers/pci/pcie/bwctrl.c
> @@ -156,6 +156,7 @@ int pcie_set_target_speed(struct pci_dev *port, enum pci_bus_speed speed_req,
>  
>  	return ret;
>  }
> +EXPORT_SYMBOL_GPL(pcie_set_target_speed);
>  
>  static void pcie_bwnotif_enable(struct pcie_device *srv)
>  {
> -- 
> 2.43.0
> 

Re: [PATCH 1/3] nvme: Add adaptive PCIe link rate switching function

[Keith Busch]

On Tue, Nov 03, 2026 at 02:25:43PM +0800, Liao Xuan wrote:
> Add support for dynamically adjusting the PCIe link rate of NVMe
> devices based on I/O activity. The link is downgraded to a
> configurable minimum rate during periods of low activity to save
> power, and upgraded to the maximum supported rate as soon as the
> workload exceeds a computed threshold, so that peak performance is
> maintained under load.

Pretty sure PCIe ASPM L1 states would save more power than downgrading
the link rate, as well as have better entry/exit latency than a software
driven monitor and link retrain.

回复：[PATCH 1/3] nvme: Add adaptive PCIe link rate switching function

[廖煊]

Hi,Keith

Thank you for the review and for raising the ASPM L1 point. I agree that ASPM L1 is very effective for power saving.
However, L1 is entered only when the link is completely idle (no pending transactions). During active but low-utilization I/O, the link cannot enter L1 and typically stays trained at its maximum rate. This patch targets exactly that window: it dynamically downgrades the link rate to the minimum sufficient for the current traffic, and upgrades back to maximum when load rises.
So the two mechanisms cover different link conditions and are complementary rather than conflicting: ASPM L1 handles the idle link, while this patch handles the active-but-underutilized link.

Best regards,
Liao Xuan

------------------------------------------------------------------
发件人：Keith Busch <kbusch@kernel.org>
发送时间：2026年11月4日(周三) 03:08
收件人："廖煊"<liaoxuan@open-hieco.net>
抄　送：axboe<axboe@kernel.dk>; hch<hch@lst.de>; sagi<sagi@grimberg.me>; "linux-nvme"<linux-nvme@lists.infradead.org>; "linux-kernel"<linux-kernel@vger.kernel.org>; default avatarZheng Tan<tanzheng@kylinos.cn>
主　题：Re: [PATCH 1/3] nvme: Add adaptive PCIe link rate switching function


On Tue, Nov 03, 2026 at 02:25:43PM +0800, Liao Xuan wrote:
> Add support for dynamically adjusting the PCIe link rate of NVMe
> devices based on I/O activity. The link is downgraded to a
> configurable minimum rate during periods of low activity to save
> power, and upgraded to the maximum supported rate as soon as the
> workload exceeds a computed threshold, so that peak performance is
> maintained under load.

Pretty sure PCIe ASPM L1 states would save more power than downgrading
the link rate, as well as have better entry/exit latency than a software
driven monitor and link retrain.

回复：[PATCH 1/3] nvme: Add adaptive PCIe link rate switching function

[廖煊]

Hi Bjorn,
Thank you for the review and for the question.

The reason for restricting this feature to Hygon platforms is that the
PCIe link retraining time varies across different CPU platforms.

In this patch, the I/O threshold for downgrading/upgrading the link rate
is derived based on this retraining time. Currently, it is hardcoded as
100us in nvme_calc_speed_threshold(), which is the validated value for
Hygon. Since we have only verified this timing parameter on Hygon platforms,
we added the restriction to avoid potential performance regressions on
other platforms where the retraining time might be different.

That said, the design logic itself is completely generic. To enable it
on other platforms, one would only need to determine their specific link
retraining time to calculate the correct threshold accordingly.

Best regards,
Liao Xuan



------------------------------------------------------------------
发件人：Bjorn Helgaas <helgaas@kernel.org>
发送时间：2026年11月4日(周三) 02:23
收件人："廖煊"<liaoxuan@open-hieco.net>
抄　送：kbusch<kbusch@kernel.org>; axboe<axboe@kernel.dk>; hch<hch@lst.de>; sagi<sagi@grimberg.me>; "linux-nvme"<linux-nvme@lists.infradead.org>; "linux-kernel"<linux-kernel@vger.kernel.org>; default avatarZheng Tan<tanzheng@kylinos.cn>; "Ilpo Järvinen"<ilpo.jarvinen@linux.intel.com>
主　题：Re: [PATCH 1/3] nvme: Add adaptive PCIe link rate switching function


[+cc Ilpo, bwctrl]

On Tue, Nov 03, 2026 at 02:25:43PM +0800, Liao Xuan wrote:
> Add support for dynamically adjusting the PCIe link rate of NVMe
> devices based on I/O activity. The link is downgraded to a
> configurable minimum rate during periods of low activity to save
> power, and upgraded to the maximum supported rate as soon as the
> workload exceeds a computed threshold, so that peak performance is
> maintained under load.
> 
> The amount of transferred data is accumulated on the I/O submission
> path and aggregated by a per-controller timer. The threshold is
> derived from the bandwidth of the minimum and maximum rates such that
> the extra time required to transfer the data at the minimum rate does
> not exceed the estimated link retraining time.
> 
> The feature is currently only supported on Hygon platforms.

Why?

> Signed-off-by: Liao Xuan <liaoxuan@open-hieco.net>
> Signed-off-by: default avatarZheng Tan <tanzheng@kylinos.cn>
> ---
>  drivers/nvme/host/Kconfig        |  10 +
>  drivers/nvme/host/Makefile       |   1 +
>  drivers/nvme/host/core.c         |   4 +
>  drivers/nvme/host/nvme.h         |  42 ++++
>  drivers/nvme/host/pci.c          |   4 +
>  drivers/nvme/host/speed_switch.c | 367 +++++++++++++++++++++++++++++++
>  drivers/pci/pcie/bwctrl.c        |   1 +
>  7 files changed, 429 insertions(+)
>  create mode 100644 drivers/nvme/host/speed_switch.c
> 
> diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig
> index 31974c7dd20c..098ec7abba66 100644
> --- a/drivers/nvme/host/Kconfig
> +++ b/drivers/nvme/host/Kconfig
> @@ -134,3 +134,13 @@ config NVME_APPLE
>  
>     To compile this driver as a module, choose M here: the
>     module will be called nvme-apple.
> +
> +config NVME_SPEED_SWITCH
> + bool "NVMe adaptive PCIe link rate switching"
> + depends on BLK_DEV_NVME && X86
> + help
> +   This provides support for dynamic PCIe link rate switching for NVMe
> +   devices. When enabled, the NVMe driver can dynamically adjust the
> +   PCIe link speed and width based on workload requirement.
> +
> +   If unsure, say N.
> diff --git a/drivers/nvme/host/Makefile b/drivers/nvme/host/Makefile
> index 67563a69f7dc..02d0200fcc88 100644
> --- a/drivers/nvme/host/Makefile
> +++ b/drivers/nvme/host/Makefile
> @@ -19,6 +19,7 @@ nvme-core-$(CONFIG_BLK_DEV_ZONED) += zns.o
>  nvme-core-$(CONFIG_FAULT_INJECTION_DEBUG_FS) += fault_inject.o
>  nvme-core-$(CONFIG_NVME_HWMON)  += hwmon.o
>  nvme-core-$(CONFIG_NVME_HOST_AUTH) += auth.o
> +nvme-core-$(CONFIG_NVME_SPEED_SWITCH) += speed_switch.o
>  
>  nvme-y     += pci.o
>  
> diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
> index beea23d04a70..85e0bfc4d7aa 100644
> --- a/drivers/nvme/host/core.c
> +++ b/drivers/nvme/host/core.c
> @@ -4679,6 +4679,7 @@ static void nvme_scan_work(struct work_struct *work)
>    /* Re-read the ANA log page to not miss updates */
>    queue_work(nvme_wq, &ctrl->ana_work);
>  #endif
> + nvme_speed_switch_start(ctrl);
>  }
>  
>  /*
> @@ -5103,6 +5104,7 @@ EXPORT_SYMBOL_GPL(nvme_remove_io_tag_set);
>  
>  void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
>  {
> + nvme_speed_switch_exit(ctrl);
>   nvme_mpath_stop(ctrl);
>   nvme_auth_stop(ctrl);
>   nvme_stop_failfast_work(ctrl);
> @@ -5318,6 +5320,8 @@ int nvme_add_ctrl(struct nvme_ctrl *ctrl)
>   nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
>   nvme_get_ctrl(ctrl);
>  
> + nvme_speed_switch_init(ctrl);
> +
>   return 0;
>  }
>  EXPORT_SYMBOL_GPL(nvme_add_ctrl);
> diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
> index 2cff9fcbf740..0f27c73a99ee 100644
> --- a/drivers/nvme/host/nvme.h
> +++ b/drivers/nvme/host/nvme.h
> @@ -339,6 +339,33 @@ enum nvme_ctrl_flags {
>   NVME_CTRL_FROZEN  = 6,
>  };
>  
> +#ifdef CONFIG_NVME_SPEED_SWITCH
> +struct nvme_speed_switch_stats {
> + u64 read_bytes;
> + u64 write_bytes;
> +};
> +
> +struct nvme_speed_switch {
> + bool initialized;
> + bool enabled;
> + u8 max_speed;
> + u8 cur_speed;
> + u8 target_speed;
> + u8 min_speed;
> + u32 monitor_interval; /* ms */
> + u32 threshold;  /* KB */
> + struct timer_list timer;
> + struct work_struct work;
> + struct nvme_speed_switch_stats __percpu *stats;
> + atomic_t timer_active;
> +};
> +
> +enum nvme_speed_timer_state {
> + NVME_SPEED_TIMER_INACTIVE,
> + NVME_SPEED_TIMER_ACTIVE,
> +};
> +#endif
> +
>  struct nvme_ctrl {
>   bool comp_seen;
>   bool identified;
> @@ -481,8 +508,23 @@ struct nvme_ctrl {
>   enum nvme_dctype dctype;
>  
>   u16   awupf; /* 0's based value. */
> +#ifdef CONFIG_NVME_SPEED_SWITCH
> + struct nvme_speed_switch speed_switch;
> +#endif
>  };
>  
> +#ifdef CONFIG_NVME_SPEED_SWITCH
> +void nvme_update_io_stats(struct nvme_ctrl *ctrl, struct request *req);
> +void nvme_speed_switch_start(struct nvme_ctrl *ctrl);
> +void nvme_speed_switch_init(struct nvme_ctrl *ctrl);
> +void nvme_speed_switch_exit(struct nvme_ctrl *ctrl);
> +#else
> +static inline void nvme_update_io_stats(struct nvme_ctrl *ctrl, struct request *req) {}
> +static inline void nvme_speed_switch_start(struct nvme_ctrl *ctrl) {}
> +static inline void nvme_speed_switch_init(struct nvme_ctrl *ctrl) {}
> +static inline void nvme_speed_switch_exit(struct nvme_ctrl *ctrl) {}
> +#endif
> +
>  static inline enum nvme_ctrl_state nvme_ctrl_state(struct nvme_ctrl *ctrl)
>  {
>   return READ_ONCE(ctrl->state);
> diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
> index 5440cf18b55b..dfba03409dc0 100644
> --- a/drivers/nvme/host/pci.c
> +++ b/drivers/nvme/host/pci.c
> @@ -1461,6 +1461,9 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
>   ret = nvme_prep_rq(req);
>   if (unlikely(ret))
>    return ret;
> +
> + nvme_update_io_stats(&dev->ctrl, req);
> +
>   spin_lock(&nvmeq->sq_lock);
>   nvme_sq_copy_cmd(nvmeq, &iod->cmd);
>   nvme_write_sq_db(nvmeq, bd->last);
> @@ -1479,6 +1482,7 @@ static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct rq_list *rqlist)
>   while ((req = rq_list_pop(rqlist))) {
>    struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
>  
> +  nvme_update_io_stats(&nvmeq->dev->ctrl, req);
>    nvme_sq_copy_cmd(nvmeq, &iod->cmd);
>   }
>   nvme_write_sq_db(nvmeq, true);
> diff --git a/drivers/nvme/host/speed_switch.c b/drivers/nvme/host/speed_switch.c
> new file mode 100644
> index 000000000000..0a828d04885b
> --- /dev/null
> +++ b/drivers/nvme/host/speed_switch.c
> @@ -0,0 +1,367 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * NVMe link speed switch driver
> + *
> + * Adjusts the PCIe link rate of NVMe devices dynamically based on the I/O
> + * workload: the link is downgraded to a configurable minimum rate during
> + * periods of low activity to save power, and upgraded to the maximum
> + * supported rate as soon as the workload requires it.
> + *
> + * Copyright (c) 2026 Chengdu Haiguang IC Design Co., Ltd.
> + * Copyright (C) 2026 Kylin Software Co., Ltd.
> + *
> + */
> +
> +#include <linux/blk-mq.h>
> +#include <linux/math64.h>
> +#include <linux/pci.h>
> +#include <linux/percpu.h>
> +#include <linux/processor.h>
> +
> +#include "nvme.h"
> +
> +/*
> + * PCIe link bandwidth in KB/s for each generation (rows, Gen1..Gen5) and
> + * lane width (columns, x1..x16).  The table is used to derive the I/O
> + * threshold exceed/below which the link can be upgraded/downgraded.
> + */
> +static const u32 nvme_speed_table[][5] = {
> + {  312,  625, 1250, 2500, 5000 },   /* Gen1 */
> + {  625, 1250, 2500, 5000, 10000 },  /* Gen2 */
> + { 1000, 2000, 4000, 8000, 16000 },  /* Gen3 */
> + { 2000, 4000, 8000, 16000, 32000 }, /* Gen4 */
> + { 4000, 8000, 16000, 32000, 64000 },/* Gen5 */
> +};
> +
> +static int nvme_bandwidth_index(enum pcie_link_width width)
> +{
> + switch (width) {
> + case PCIE_LNK_X1:
> +  return 0;
> + case PCIE_LNK_X2:
> +  return 1;
> + case PCIE_LNK_X4:
> +  return 2;
> + case PCIE_LNK_X8:
> +  return 3;
> + case PCIE_LNK_X16:
> +  return 4;
> + default:
> +  return -EINVAL;
> + }
> +}
> +
> +/*
> + * Read the maximum supported link rate of the upstream port from the Link
> + * Capability register.
> + */
> +static int nvme_get_max_link_speed(struct pci_dev *pdev)
> +{
> + u16 linkcap;
> +
> + pcie_capability_read_word(pdev, PCI_EXP_LNKCAP, &linkcap);
> + return linkcap & PCI_EXP_LNKCAP_SLS;
> +}
> +
> +static enum pci_bus_speed nvme_speed_to_bus_speed(u8 gen)
> +{
> + return gen + PCIE_SPEED_2_5GT - 1;
> +}
> +
> +/*
> + * Work item that actually changes the link rate.  The queues are frozen
> + * before retraining to quiesce I/O and resumed once the switch completed.
> + */
> +static void nvme_speed_switch_work(struct work_struct *work)
> +{
> + struct nvme_speed_switch *sw = container_of(work, struct nvme_speed_switch, work);
> + struct nvme_ctrl *ctrl = container_of(sw, struct nvme_ctrl, speed_switch);
> + struct pci_dev *pdev = to_pci_dev(ctrl->dev);
> + struct pci_dev *bridge = pdev->bus->self;
> + u8 cur_speed = READ_ONCE(sw->cur_speed);
> + int ret;
> +
> + if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)
> +  return;
> +
> + /* Freeze I/O to avoid timeouts during link retraining. */
> + nvme_start_freeze(ctrl);
> + nvme_wait_freeze(ctrl);
> +
> + ret = pcie_set_target_speed(bridge, nvme_speed_to_bus_speed(sw->target_speed),
> +        true);
> + if (ret) {
> +  dev_warn(ctrl->device,
> +    "failed to set target rate Gen%u (%d), trying rollback to Gen%u\n",
> +    sw->target_speed, ret, cur_speed);
> +  ret = pcie_set_target_speed(bridge, nvme_speed_to_bus_speed(cur_speed),
> +         true);
> +  if (ret) {
> +   dev_err(ctrl->device,
> +    "rollback to Gen%u failed (%d), disabling speed switch\n",
> +    cur_speed, ret);
> +   WRITE_ONCE(sw->enabled, false);
> +   nvme_unfreeze(ctrl);
> +   return;
> +  }
> + } else {
> +  WRITE_ONCE(sw->cur_speed, sw->target_speed);
> + }
> +
> + nvme_unfreeze(ctrl);
> + dev_dbg(ctrl->device, "link rate changed to Gen%u\n",
> +  READ_ONCE(sw->cur_speed));
> +}
> +
> +/*
> + * Compute the I/O threshold (in KB) for a monitoring window.
> + *
> + * The threshold is derived from the following constraint: for the link
> + * speed to be upgraded, the transfer time of the data at the minimum
> + * speed must be greater than the sum of the transfer time at maximum
> + * speed and the link retraining time (100us):
> + *
> + * data / min_bw >= data / max_bw + 100us
> + *
> + * which yields data <= min_bw * max_bw * 0.1ms / (max_bw - min_bw).
> + * The bandwidths are taken from the link capability of the upstream port
> + * and the link width capability of the device.
> + */
> +static u32 nvme_calc_speed_threshold(struct nvme_speed_switch *sw,
> +         enum pcie_link_width max_width)
> +{
> + struct nvme_ctrl *ctrl = container_of(sw, struct nvme_ctrl, speed_switch);
> + u64 min_bw, max_bw, x, y;
> + int bw_idx, min_gen = sw->min_speed, max_gen = sw->max_speed;
> + u32 threshold;
> +
> + if (min_gen < 1 || min_gen > 5 || max_gen < 1 || max_gen > 5) {
> +  dev_warn(ctrl->device,
> +    "invalid link rate range for threshold: min=Gen%d, max=Gen%d\n",
> +    min_gen, max_gen);
> +  return 0;
> + }
> +
> + bw_idx = nvme_bandwidth_index(max_width);
> + if (bw_idx < 0) {
> +  dev_warn(ctrl->device,
> +    "unsupported link width (%d) for threshold calculation\n",
> +    max_width);
> +  return 0;
> + }
> +
> + min_bw = nvme_speed_table[min_gen - 1][bw_idx];
> + max_bw = nvme_speed_table[max_gen - 1][bw_idx];
> + if (min_bw >= max_bw) {
> +  dev_warn(ctrl->device,
> +    "invalid bandwidth for threshold: min=%llu KB/s, max=%llu KB/s\n",
> +    min_bw, max_bw);
> +  return 0;
> + }
> +
> + x = min_bw * max_bw;
> + y = 10ULL * (max_bw - min_bw);
> + threshold = div64_u64(x + (y >> 1), y);
> +
> + dev_dbg(ctrl->device,
> +  "I/O threshold: %u KB (min=Gen%d, max=Gen%d, width=x%d)\n",
> +  threshold, min_gen, max_gen, max_width);
> + return threshold;
> +}
> +
> +/* Called on the I/O submission path to accumulate the transferred bytes. */
> +void nvme_update_io_stats(struct nvme_ctrl *ctrl, struct request *req)
> +{
> + struct nvme_speed_switch *sw = &ctrl->speed_switch;
> + struct nvme_speed_switch_stats *stat;
> + enum req_op op = req_op(req);
> + unsigned int bytes = blk_rq_bytes(req);
> +
> + if (!READ_ONCE(sw->enabled))
> +  return;
> +
> + if (op != REQ_OP_READ && op != REQ_OP_WRITE)
> +  return;
> +
> + if (!sw->stats) {
> +  dev_dbg(ctrl->device, "I/O statistics not allocated, skip accounting\n");
> +  return;
> + }
> +
> + stat = get_cpu_ptr(sw->stats);
> + if (op == REQ_OP_READ)
> +  stat->read_bytes += bytes;
> + else
> +  stat->write_bytes += bytes;
> + put_cpu_ptr(sw->stats);
> +
> + if (atomic_cmpxchg(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE,
> +      NVME_SPEED_TIMER_ACTIVE) != NVME_SPEED_TIMER_INACTIVE)
> +  return;
> +
> + /* First I/O of the activity period: compute the threshold and arm the timer. */
> + WRITE_ONCE(sw->threshold, nvme_calc_speed_threshold(sw,
> +     pcie_get_width_cap(to_pci_dev(ctrl->dev))));
> + if (!READ_ONCE(sw->threshold)) {
> +  dev_warn(ctrl->device,
> +    "failed to compute speed switch threshold, keeping current rate\n");
> +  atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE);
> +  return;
> + }
> + mod_timer(&sw->timer, jiffies + msecs_to_jiffies(READ_ONCE(sw->monitor_interval)));
> +}
> +EXPORT_SYMBOL_GPL(nvme_update_io_stats);
> +
> +static void nvme_clear_io_stats(struct nvme_speed_switch *sw)
> +{
> + int cpu;
> +
> + for_each_possible_cpu(cpu) {
> +  struct nvme_speed_switch_stats *stat = per_cpu_ptr(sw->stats, cpu);
> +
> +  stat->read_bytes = 0;
> +  stat->write_bytes = 0;
> + }
> +}
> +
> +/*
> + * Aggregate the per-CPU I/O counters of the last monitoring window and
> + * decide the target rate: the maximum rate if the window exceeded the
> + * threshold, the minimum rate otherwise.
> + */
> +static int nvme_check_io_and_decide_speed(struct nvme_speed_switch *sw)
> +{
> + struct nvme_ctrl *ctrl = container_of(sw, struct nvme_ctrl, speed_switch);
> + u64 read_bytes = 0, write_bytes = 0;
> + unsigned long read_kb, write_kb;
> + int cpu;
> +
> + for_each_possible_cpu(cpu) {
> +  struct nvme_speed_switch_stats *stat = per_cpu_ptr(sw->stats, cpu);
> +
> +  read_bytes += stat->read_bytes;
> +  write_bytes += stat->write_bytes;
> +  stat->read_bytes = 0;
> +  stat->write_bytes = 0;
> + }
> +
> + read_kb = read_bytes / 1024;
> + write_kb = write_bytes / 1024;
> + dev_dbg(ctrl->device,
> +  "I/O in window: read=%lu KB, write=%lu KB, threshold=%u KB\n",
> +  read_kb, write_kb, READ_ONCE(sw->threshold));
> +
> + if (read_kb >= sw->threshold || write_kb >= sw->threshold)
> +  return sw->max_speed;
> + return sw->min_speed;
> +}
> +
> +static void nvme_speed_switch_timer_fn(struct timer_list *t)
> +{
> + struct nvme_speed_switch *sw = container_of(t, struct nvme_speed_switch, timer);
> +
> + if (!READ_ONCE(sw->enabled)) {
> +  atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE);
> +  return;
> + }
> +
> + sw->target_speed = nvme_check_io_and_decide_speed(sw);
> + if (sw->target_speed != READ_ONCE(sw->cur_speed))
> +  schedule_work(&sw->work);
> +
> + mod_timer(t, jiffies + msecs_to_jiffies(READ_ONCE(sw->monitor_interval)));
> +}
> +
> +static void nvme_speed_switch_params_init(struct nvme_speed_switch *sw)
> +{
> + sw->monitor_interval = 100;
> + sw->min_speed = PCI_EXP_LNKSTA_CLS_2_5GB;
> +}
> +
> +void nvme_speed_switch_init(struct nvme_ctrl *ctrl)
> +{
> + struct nvme_speed_switch *sw = &ctrl->speed_switch;
> + struct pci_dev *pdev;
> + int max_speed;
> +
> + if (boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) {
> +  dev_dbg(ctrl->device,
> +   "link rate switching is only supported on Hygon platforms\n");
> +  return;
> + }
> +
> + if (!dev_is_pci(ctrl->dev)) {
> +  dev_err(ctrl->device, "link rate switching requires a PCI device\n");
> +  return;
> + }
> + pdev = to_pci_dev(ctrl->dev);
> + if (!pdev->bus->self) {
> +  dev_err(ctrl->device,
> +   "link rate switching requires a PCIe upstream port\n");
> +  return;
> + }
> +
> + nvme_speed_switch_params_init(sw);
> +
> + max_speed = nvme_get_max_link_speed(pdev->bus->self);
> + if (max_speed <= sw->min_speed) {
> +  dev_err(ctrl->device,
> +   "invalid link rate range: min=Gen%u, max=Gen%d\n",
> +   sw->min_speed, max_speed);
> +  return;
> + }
> + sw->max_speed = max_speed;
> +
> + sw->stats = alloc_percpu(struct nvme_speed_switch_stats);
> + if (!sw->stats) {
> +  dev_err(ctrl->device, "failed to allocate per-CPU I/O statistics\n");
> +  return;
> + }
> + nvme_clear_io_stats(sw);
> +
> + WRITE_ONCE(sw->cur_speed, sw->max_speed);
> + WRITE_ONCE(sw->threshold, nvme_calc_speed_threshold(sw, pcie_get_width_cap(pdev)));
> + if (!READ_ONCE(sw->threshold)) {
> +  dev_err(ctrl->device, "failed to compute speed switch threshold\n");
> +  free_percpu(sw->stats);
> +  sw->stats = NULL;
> +  return;
> + }
> +
> + INIT_WORK(&sw->work, nvme_speed_switch_work);
> + timer_setup(&sw->timer, nvme_speed_switch_timer_fn, 0);
> + atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE);
> + WRITE_ONCE(sw->enabled, true);
> + sw->initialized = true;
> +
> + dev_info(ctrl->device,
> +   "NVMe link rate switching initialized (min Gen%u, max Gen%d)\n",
> +   sw->min_speed, sw->max_speed);
> +}
> +
> +void nvme_speed_switch_start(struct nvme_ctrl *ctrl)
> +{
> + struct nvme_speed_switch *sw = &ctrl->speed_switch;
> +
> + if (!READ_ONCE(sw->enabled))
> +  return;
> +
> + atomic_set(&sw->timer_active, NVME_SPEED_TIMER_ACTIVE);
> + mod_timer(&sw->timer, jiffies + msecs_to_jiffies(READ_ONCE(sw->monitor_interval)));
> +}
> +
> +void nvme_speed_switch_exit(struct nvme_ctrl *ctrl)
> +{
> + struct nvme_speed_switch *sw = &ctrl->speed_switch;
> +
> + if (!sw->initialized)
> +  return;
> +
> + sw->initialized = false;
> + WRITE_ONCE(sw->enabled, false);
> + timer_delete_sync(&sw->timer);
> + cancel_work_sync(&sw->work);
> +
> + free_percpu(sw->stats);
> + sw->stats = NULL;
> +}
> diff --git a/drivers/pci/pcie/bwctrl.c b/drivers/pci/pcie/bwctrl.c
> index d48021bfd844..4b28e26f9531 100644
> --- a/drivers/pci/pcie/bwctrl.c
> +++ b/drivers/pci/pcie/bwctrl.c
> @@ -156,6 +156,7 @@ int pcie_set_target_speed(struct pci_dev *port, enum pci_bus_speed speed_req,
>  
>   return ret;
>  }
> +EXPORT_SYMBOL_GPL(pcie_set_target_speed);
>  
>  static void pcie_bwnotif_enable(struct pcie_device *srv)
>  {
> -- 
> 2.43.0
> 

Re: 回复：[PATCH 1/3] nvme: Add adaptive PCIe link rate switching function

[Bjorn Helgaas]

On Thu, Nov 05, 2026 at 02:30:13PM +0800, 廖煊 wrote:
> Hi Bjorn,
> Thank you for the review and for the question.
> 
> The reason for restricting this feature to Hygon platforms is that the
> PCIe link retraining time varies across different CPU platforms.
> 
> In this patch, the I/O threshold for downgrading/upgrading the link rate
> is derived based on this retraining time. Currently, it is hardcoded as
> 100us in nvme_calc_speed_threshold(), which is the validated value for
> Hygon. Since we have only verified this timing parameter on Hygon platforms,
> we added the restriction to avoid potential performance regressions on
> other platforms where the retraining time might be different.

Seems suspect to hard-code platform parameters like this, but if you
must, at least add a comment to the effect that this depends on the
platform and the current value is for platform X.

Re: 回复：[PATCH 1/3] nvme: Add adaptive PCIe link rate switching function

[Keith Busch]

On Thu, Nov 05, 2026 at 01:50:21PM +0800, 廖煊 wrote:
> However, L1 is entered only when the link is completely idle (no
> pending transactions). During active but low-utilization I/O, the link
> cannot enter L1 and typically stays trained at its maximum rate. This
> patch targets exactly that window: it dynamically downgrades the link
> rate to the minimum sufficient for the current traffic, and upgrades
> back to maximum when load rises.

ASPM operates on microsecond idle periods, so a lightly used link spends
most of its time detected as idle. Your threshold only selects the lower
rate below roughly 1.5MB/s on a Gen4 x4 link, which is essentially idle
where L1 would kick in more readily. Do you have shareable numbers that
show a real power benefit beyond ASPM?

> So the two mechanisms cover different link conditions and are
> complementary rather than conflicting: ASPM L1 handles the idle link,
> while this patch handles the active-but-underutilized link.

I don't think this kind of policy belongs in the kernel, much less in
the nvme driver. There are mechanisms for user space to control this,
though: the bwctrl cooling device lets user space set the link speed,
and diskstats provide utilization. A user space daemon can implement
this without kernel changes.

If you don't want to tie this to the PCIe thermal subsystem, maybe the
current_link_speed attribute could add a writeable callback instead?

回复：回复：[PATCH 1/3] nvme: Add adaptive PCIe link rate switching function

[廖煊]

Hi Bjorn,

To improve generality and make the assumption explicit, I will follow
your suggestion: I will encapsulate the threshold calculation flow in
a separate function, and add a comment there stating that this parameter
is platform-dependent and that the current value is for Hygon platforms.
I will send an updated patch with this change.

Thanks,
Liao Xuan
------------------------------------------------------------------
发件人：Bjorn Helgaas <helgaas@kernel.org>
发送时间：2026年11月6日(周五) 03:22
收件人："廖煊"<liaoxuan@open-hieco.net>
抄　送：kbusch<kbusch@kernel.org>; axboe<axboe@kernel.dk>; hch<hch@lst.de>; sagi<sagi@grimberg.me>; "linux-nvme"<linux-nvme@lists.infradead.org>; "linux-kernel"<linux-kernel@vger.kernel.org>; default avatarZheng Tan<tanzheng@kylinos.cn>; "Ilpo Järvinen"<ilpo.jarvinen@linux.intel.com>
主　题：Re: 回复：[PATCH 1/3] nvme: Add adaptive PCIe link rate switching function


On Thu, Nov 05, 2026 at 02:30:13PM +0800, 廖煊 wrote:
> Hi Bjorn,
> Thank you for the review and for the question.
> 
> The reason for restricting this feature to Hygon platforms is that the
> PCIe link retraining time varies across different CPU platforms.
> 
> In this patch, the I/O threshold for downgrading/upgrading the link rate
> is derived based on this retraining time. Currently, it is hardcoded as
> 100us in nvme_calc_speed_threshold(), which is the validated value for
> Hygon. Since we have only verified this timing parameter on Hygon platforms,
> we added the restriction to avoid potential performance regressions on
> other platforms where the retraining time might be different.

Seems suspect to hard-code platform parameters like this, but if you
must, at least add a comment to the effect that this depends on the
platform and the current value is for platform X.