[PATCH 0/3] Add adaptive NVMe link rate switching function

[PATCH 0/3] Add adaptive NVMe link rate switching function

[Liao Xuan]

From: Liao Xuan <liaoxuan@hygon.cn>

NVMe devices are commonly left trained at the maximum PCIe link rate
even when there is little or no I/O activity, which wastes power in
both the device and the surrounding platform.

This series adds adaptive PCIe link rate switching to the NVMe host
driver. The link is downgraded to a configurable minimum rate when
I/O activity is low to save power, and upgraded back to the maximum
supported rate as soon as the workload requires it, so that peak
performance is maintained under load.

The amount of transferred data is accumulated on the submission path
with per-CPU counters and evaluated by a per-controller timer. The
switch threshold is derived from the bandwidth of the minimum and
maximum link rates such that the extra time required to transfer the
data at the minimum rate does not exceed the estimated link retraining
time. The controller queues are frozen while the link is retrained
and resumed afterwards, so no I/O is lost or times out during the
switch.

The main functions of each patch are as follows:
1. Patch 1 adds the core switching functionality. It uses the target
link speed interface of the PCIe bandwidth controller, which is
exported for this purpose.

2. Patch 2 adds hysteresis and idle detection to the switching policy.
The link is upgraded immediately once the threshold is exceeded, but
only downgraded after a number of consecutive below-threshold windows,
which prevents excessive rate toggling under fluctuating workloads.
Monitoring is stopped after a period of inactivity and re-armed by the
next I/O.

3. Patch 3 exposes the switching parameters (enable, monitor_interval,
min_speed, up_threshold, down_threshold) via sysfs so that the
power/performance trade-off can be tuned at runtime without a reboot.

The feature is currently enabled on Hygon platforms.

Liao Xuan (3):
  nvme: Add adaptive PCIe link rate switching function
  nvme: Add hysteresis and idle detection to link rate switching
  nvme: Expose link rate switching tunables via sysfs

 drivers/nvme/host/Kconfig        |  10 +
 drivers/nvme/host/Makefile       |   1 +
 drivers/nvme/host/core.c         |   4 +
 drivers/nvme/host/nvme.h         |  50 ++++
 drivers/nvme/host/pci.c          |   4 +
 drivers/nvme/host/speed_switch.c | 424 +++++++++++++++++++++++++++++++
 drivers/nvme/host/sysfs.c        | 143 +++++++++++
 drivers/pci/pcie/bwctrl.c        |   1 +
 8 files changed, 637 insertions(+)
 create mode 100644 drivers/nvme/host/speed_switch.c

-- 
2.43.0

[PATCH 1/3] nvme: Add adaptive PCIe link rate switching function

[Liao Xuan]

Add support for dynamically adjusting the PCIe link rate of NVMe
devices based on I/O activity. The link is downgraded to a
configurable minimum rate during periods of low activity to save
power, and upgraded to the maximum supported rate as soon as the
workload exceeds a computed threshold, so that peak performance is
maintained under load.

The amount of transferred data is accumulated on the I/O submission
path and aggregated by a per-controller timer. The threshold is
derived from the bandwidth of the minimum and maximum rates such that
the extra time required to transfer the data at the minimum rate does
not exceed the estimated link retraining time.

The feature is currently only supported on Hygon platforms.

Four NVMe devices were tested on the link with this patch. The power
benefit was measured by sampling the voltage and current of the power
monitor every 1 second, each run lasted 3 minutes and the test was
repeated 5 times.

Test run    Power Benefit(w)
1           5.3236
2           5.2285
3           5.4176
4           5.4538
5           5.3444
Average     5.3536

The adaptive link rate switching saves about 5.3w of system power
on average.

The feature was tested with Gen1 as the minimum link rate. The workload
runs 10 seconds of random read followed by 0.5 seconds of idle, repeated
100 times.

                              Avg IOPS (k)  Avg BW (GiB/s)  Avg clat (us)
With speed switching          2671.2        10.19           378.1
Without speed switching       2681.5        10.23           377.0

The differences are within measurement noise (< 0.5%), which shows
that the adaptive link rate switching does not degrade the baseline
performance and keeps the I/O stable.

In summary, the adaptive link rate switching keeps the I/O performance
essentially unchanged (the differences in IOPS, bandwidth and latency
are all within 0.5% measurement noise) while saving about 5w of
system power, providing an effective way to reduce the platform power
consumption under light I/O without sacrificing throughput or latency.

Signed-off-by: Liao Xuan <liaoxuan@open-hieco.net>
Signed-off-by: Zheng Tan <tanzheng@kylinos.cn>
---
 drivers/nvme/host/Kconfig        |  10 +
 drivers/nvme/host/Makefile       |   1 +
 drivers/nvme/host/core.c         |   4 +
 drivers/nvme/host/nvme.h         |  42 ++++
 drivers/nvme/host/pci.c          |   4 +
 drivers/nvme/host/speed_switch.c | 371 +++++++++++++++++++++++++++++++
 drivers/pci/pcie/bwctrl.c        |   1 +
 7 files changed, 433 insertions(+)
 create mode 100644 drivers/nvme/host/speed_switch.c

diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig
index 31974c7dd..098ec7abb 100644
--- a/drivers/nvme/host/Kconfig
+++ b/drivers/nvme/host/Kconfig
@@ -134,3 +134,13 @@ config NVME_APPLE
 
 	  To compile this driver as a module, choose M here: the
 	  module will be called nvme-apple.
+
+config NVME_SPEED_SWITCH
+	bool "NVMe adaptive PCIe link rate switching"
+	depends on BLK_DEV_NVME && X86
+	help
+	  This provides support for dynamic PCIe link rate switching for NVMe
+	  devices. When enabled, the NVMe driver can dynamically adjust the
+	  PCIe link speed and width based on workload requirement.
+
+	  If unsure, say N.
diff --git a/drivers/nvme/host/Makefile b/drivers/nvme/host/Makefile
index 6414ec968..f33e804e9 100644
--- a/drivers/nvme/host/Makefile
+++ b/drivers/nvme/host/Makefile
@@ -18,6 +18,7 @@ nvme-core-$(CONFIG_BLK_DEV_ZONED)	+= zns.o
 nvme-core-$(CONFIG_FAULT_INJECTION_DEBUG_FS)	+= fault_inject.o
 nvme-core-$(CONFIG_NVME_HWMON)		+= hwmon.o
 nvme-core-$(CONFIG_NVME_HOST_AUTH)	+= auth.o
+nvme-core-$(CONFIG_NVME_SPEED_SWITCH)	+= speed_switch.o
 
 nvme-y					+= pci.o
 
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 453c1f0b2..f190dd05e 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -4627,6 +4627,7 @@ static void nvme_scan_work(struct work_struct *work)
 		/* Re-read the ANA log page to not miss updates */
 		queue_work(nvme_wq, &ctrl->ana_work);
 #endif
+	nvme_speed_switch_start(ctrl);
 }
 
 /*
@@ -5051,6 +5052,7 @@ EXPORT_SYMBOL_GPL(nvme_remove_io_tag_set);
 
 void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
 {
+	nvme_speed_switch_exit(ctrl);
 	nvme_mpath_stop(ctrl);
 	nvme_auth_stop(ctrl);
 	nvme_stop_failfast_work(ctrl);
@@ -5266,6 +5268,8 @@ int nvme_add_ctrl(struct nvme_ctrl *ctrl)
 	nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
 	nvme_get_ctrl(ctrl);
 
+	nvme_speed_switch_init(ctrl);
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(nvme_add_ctrl);
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 824651cc8..ff06f85d1 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -331,6 +331,33 @@ enum nvme_ctrl_flags {
 	NVME_CTRL_FROZEN		= 6,
 };
 
+#ifdef CONFIG_NVME_SPEED_SWITCH
+struct nvme_speed_switch_stats {
+	u64 read_bytes;
+	u64 write_bytes;
+};
+
+struct nvme_speed_switch {
+	bool initialized;
+	bool enabled;
+	u8 max_speed;
+	u8 cur_speed;
+	u8 target_speed;
+	u8 min_speed;
+	u32 monitor_interval;	/* ms */
+	u32 threshold;		/* KB */
+	struct timer_list timer;
+	struct work_struct work;
+	struct nvme_speed_switch_stats __percpu *stats;
+	atomic_t timer_active;
+};
+
+enum nvme_speed_timer_state {
+	NVME_SPEED_TIMER_INACTIVE,
+	NVME_SPEED_TIMER_ACTIVE,
+};
+#endif
+
 struct nvme_ctrl {
 	bool comp_seen;
 	bool identified;
@@ -472,8 +499,23 @@ struct nvme_ctrl {
 	enum nvme_dctype dctype;
 
 	u16			awupf; /* 0's based value. */
+#ifdef CONFIG_NVME_SPEED_SWITCH
+	struct nvme_speed_switch speed_switch;
+#endif
 };
 
+#ifdef CONFIG_NVME_SPEED_SWITCH
+void nvme_update_io_stats(struct nvme_ctrl *ctrl, struct request *req);
+void nvme_speed_switch_start(struct nvme_ctrl *ctrl);
+void nvme_speed_switch_init(struct nvme_ctrl *ctrl);
+void nvme_speed_switch_exit(struct nvme_ctrl *ctrl);
+#else
+static inline void nvme_update_io_stats(struct nvme_ctrl *ctrl, struct request *req) {}
+static inline void nvme_speed_switch_start(struct nvme_ctrl *ctrl) {}
+static inline void nvme_speed_switch_init(struct nvme_ctrl *ctrl) {}
+static inline void nvme_speed_switch_exit(struct nvme_ctrl *ctrl) {}
+#endif
+
 static inline enum nvme_ctrl_state nvme_ctrl_state(struct nvme_ctrl *ctrl)
 {
 	return READ_ONCE(ctrl->state);
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 69932d640..862c38242 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -1455,6 +1455,9 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
 	ret = nvme_prep_rq(req);
 	if (unlikely(ret))
 		return ret;
+
+	nvme_update_io_stats(&dev->ctrl, req);
+
 	spin_lock(&nvmeq->sq_lock);
 	nvme_sq_copy_cmd(nvmeq, &iod->cmd);
 	nvme_write_sq_db(nvmeq, bd->last);
@@ -1473,6 +1476,7 @@ static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct rq_list *rqlist)
 	while ((req = rq_list_pop(rqlist))) {
 		struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 
+		nvme_update_io_stats(&nvmeq->dev->ctrl, req);
 		nvme_sq_copy_cmd(nvmeq, &iod->cmd);
 	}
 	nvme_write_sq_db(nvmeq, true);
diff --git a/drivers/nvme/host/speed_switch.c b/drivers/nvme/host/speed_switch.c
new file mode 100644
index 000000000..88c046fcd
--- /dev/null
+++ b/drivers/nvme/host/speed_switch.c
@@ -0,0 +1,371 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * NVMe link speed switch driver
+ *
+ * Adjusts the PCIe link rate of NVMe devices dynamically based on the I/O
+ * workload: the link is downgraded to a configurable minimum rate during
+ * periods of low activity to save power, and upgraded to the maximum
+ * supported rate as soon as the workload requires it.
+ *
+ * Copyright (c) 2026 Hygon Information Technology Co., Ltd.
+ * Copyright (c) 2026 Kylin Software Co., Ltd.
+ *
+ */
+
+#include <linux/blk-mq.h>
+#include <linux/math64.h>
+#include <linux/pci.h>
+#include <linux/percpu.h>
+#include <linux/processor.h>
+
+#include "nvme.h"
+
+/*
+ * PCIe link bandwidth in KB/s for each generation (rows, Gen1..Gen5) and
+ * lane width (columns, x1..x16).  The table is used to derive the I/O
+ * threshold exceed/below which the link can be upgraded/downgraded.
+ */
+static const u32 nvme_speed_table[][5] = {
+	{  312,  625, 1250, 2500, 5000 },   /* Gen1 */
+	{  625, 1250, 2500, 5000, 10000 },  /* Gen2 */
+	{ 1000, 2000, 4000, 8000, 16000 },  /* Gen3 */
+	{ 2000, 4000, 8000, 16000, 32000 }, /* Gen4 */
+	{ 4000, 8000, 16000, 32000, 64000 },/* Gen5 */
+};
+
+static int nvme_bandwidth_index(enum pcie_link_width width)
+{
+	switch (width) {
+	case PCIE_LNK_X1:
+		return 0;
+	case PCIE_LNK_X2:
+		return 1;
+	case PCIE_LNK_X4:
+		return 2;
+	case PCIE_LNK_X8:
+		return 3;
+	case PCIE_LNK_X16:
+		return 4;
+	default:
+		return -EINVAL;
+	}
+}
+
+/*
+ * Read the maximum supported link rate of the upstream port from the Link
+ * Capability register.
+ */
+static int nvme_get_max_link_speed(struct pci_dev *pdev)
+{
+	u16 linkcap;
+
+	pcie_capability_read_word(pdev, PCI_EXP_LNKCAP, &linkcap);
+
+	return linkcap & PCI_EXP_LNKCAP_SLS;
+}
+
+static enum pci_bus_speed nvme_speed_to_bus_speed(u8 gen)
+{
+	return gen + PCIE_SPEED_2_5GT - 1;
+}
+
+/*
+ * Work item that actually changes the link rate.  The queues are frozen
+ * before retraining to quiesce I/O and resumed once the switch completed.
+ */
+static void nvme_speed_switch_work(struct work_struct *work)
+{
+	struct nvme_speed_switch *sw = container_of(work, struct nvme_speed_switch, work);
+	struct nvme_ctrl *ctrl = container_of(sw, struct nvme_ctrl, speed_switch);
+	struct pci_dev *pdev = to_pci_dev(ctrl->dev);
+	struct pci_dev *bridge = pdev->bus->self;
+	u8 cur_speed = READ_ONCE(sw->cur_speed);
+	int ret;
+
+	if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)
+		return;
+
+	/* Freeze I/O to avoid timeouts during link retraining. */
+	nvme_start_freeze(ctrl);
+	nvme_wait_freeze(ctrl);
+
+	ret = pcie_set_target_speed(bridge, nvme_speed_to_bus_speed(sw->target_speed),
+				    true);
+	if (ret) {
+		dev_warn(ctrl->device,
+			 "failed to set target rate Gen%u (%d), trying rollback to Gen%u\n",
+			 sw->target_speed, ret, cur_speed);
+		ret = pcie_set_target_speed(bridge, nvme_speed_to_bus_speed(cur_speed),
+					    true);
+		if (ret) {
+			dev_err(ctrl->device,
+				"rollback to Gen%u failed (%d), disabling speed switch\n",
+				cur_speed, ret);
+			WRITE_ONCE(sw->enabled, false);
+			nvme_unfreeze(ctrl);
+			return;
+		}
+	} else {
+		WRITE_ONCE(sw->cur_speed, sw->target_speed);
+	}
+
+	nvme_unfreeze(ctrl);
+	dev_dbg(ctrl->device, "link rate changed to Gen%u\n",
+		READ_ONCE(sw->cur_speed));
+}
+
+/*
+ * Compute the I/O threshold (in KB) for a monitoring window.
+ *
+ * The threshold is derived from the following constraint: for the link
+ * speed to be upgraded, the transfer time of the data at the minimum
+ * speed must be greater than the sum of the transfer time at maximum
+ * speed and the link retraining time (100us):
+ *
+ *	data / min_bw >= data / max_bw + 100us
+ *
+ * which yields data <= min_bw * max_bw * 0.1ms / (max_bw - min_bw).
+ * The bandwidths are taken from the link capability of the upstream port
+ * and the link width capability of the device.
+ */
+static u32 nvme_calc_speed_threshold(struct nvme_speed_switch *sw,
+				     enum pcie_link_width max_width)
+{
+	struct nvme_ctrl *ctrl = container_of(sw, struct nvme_ctrl, speed_switch);
+	int bw_idx, min_gen = sw->min_speed, max_gen = sw->max_speed;
+	u64 min_bw, max_bw, x, y;
+	u32 threshold;
+
+	if (min_gen < 1 || min_gen > 5 || max_gen < 1 || max_gen > 5) {
+		dev_warn(ctrl->device,
+			 "invalid link rate range for threshold: min=Gen%d, max=Gen%d\n",
+			 min_gen, max_gen);
+		return 0;
+	}
+
+	bw_idx = nvme_bandwidth_index(max_width);
+	if (bw_idx < 0) {
+		dev_warn(ctrl->device,
+			 "unsupported link width (%d) for threshold calculation\n",
+			 max_width);
+		return 0;
+	}
+
+	min_bw = nvme_speed_table[min_gen - 1][bw_idx];
+	max_bw = nvme_speed_table[max_gen - 1][bw_idx];
+	if (min_bw >= max_bw) {
+		dev_warn(ctrl->device,
+			 "invalid bandwidth for threshold: min=%llu KB/s, max=%llu KB/s\n",
+			 min_bw, max_bw);
+		return 0;
+	}
+
+	x = min_bw * max_bw;
+	y = 10ULL * (max_bw - min_bw);
+	threshold = div64_u64(x + (y >> 1), y);
+
+	dev_dbg(ctrl->device,
+		"I/O threshold: %u KB (min=Gen%d, max=Gen%d, width=x%d)\n",
+		threshold, min_gen, max_gen, max_width);
+
+	return threshold;
+}
+
+/* Called on the I/O submission path to accumulate the transferred bytes. */
+void nvme_update_io_stats(struct nvme_ctrl *ctrl, struct request *req)
+{
+	struct nvme_speed_switch *sw = &ctrl->speed_switch;
+	struct nvme_speed_switch_stats *stat;
+	enum req_op op = req_op(req);
+	unsigned int bytes = blk_rq_bytes(req);
+
+	if (!READ_ONCE(sw->enabled))
+		return;
+
+	if (op != REQ_OP_READ && op != REQ_OP_WRITE)
+		return;
+
+	if (!sw->stats) {
+		dev_dbg(ctrl->device, "I/O statistics not allocated, skip accounting\n");
+		return;
+	}
+
+	stat = get_cpu_ptr(sw->stats);
+	if (op == REQ_OP_READ)
+		stat->read_bytes += bytes;
+	else
+		stat->write_bytes += bytes;
+	put_cpu_ptr(sw->stats);
+
+	if (atomic_cmpxchg(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE,
+			   NVME_SPEED_TIMER_ACTIVE) != NVME_SPEED_TIMER_INACTIVE)
+		return;
+
+	/* First I/O of the activity period: compute the threshold and arm the timer. */
+	WRITE_ONCE(sw->threshold, nvme_calc_speed_threshold(sw,
+					pcie_get_width_cap(to_pci_dev(ctrl->dev))));
+	if (!READ_ONCE(sw->threshold)) {
+		dev_warn(ctrl->device,
+			 "failed to compute speed switch threshold, keeping current rate\n");
+		atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE);
+		return;
+	}
+	mod_timer(&sw->timer, jiffies + msecs_to_jiffies(READ_ONCE(sw->monitor_interval)));
+}
+EXPORT_SYMBOL_GPL(nvme_update_io_stats);
+
+static void nvme_clear_io_stats(struct nvme_speed_switch *sw)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		struct nvme_speed_switch_stats *stat = per_cpu_ptr(sw->stats, cpu);
+
+		stat->read_bytes = 0;
+		stat->write_bytes = 0;
+	}
+}
+
+/*
+ * Aggregate the per-CPU I/O counters of the last monitoring window and
+ * decide the target rate: the maximum rate if the window exceeded the
+ * threshold, the minimum rate otherwise.
+ */
+static int nvme_check_io_and_decide_speed(struct nvme_speed_switch *sw)
+{
+	struct nvme_ctrl *ctrl = container_of(sw, struct nvme_ctrl, speed_switch);
+	u64 read_bytes = 0, write_bytes = 0;
+	unsigned long read_kb, write_kb;
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		struct nvme_speed_switch_stats *stat = per_cpu_ptr(sw->stats, cpu);
+
+		read_bytes += stat->read_bytes;
+		write_bytes += stat->write_bytes;
+		stat->read_bytes = 0;
+		stat->write_bytes = 0;
+	}
+
+	read_kb = read_bytes / 1024;
+	write_kb = write_bytes / 1024;
+	dev_dbg(ctrl->device,
+		"I/O in window: read=%lu KB, write=%lu KB, threshold=%u KB\n",
+		read_kb, write_kb, READ_ONCE(sw->threshold));
+
+	if (read_kb >= sw->threshold || write_kb >= sw->threshold)
+		return sw->max_speed;
+
+	return sw->min_speed;
+}
+
+static void nvme_speed_switch_timer_fn(struct timer_list *t)
+{
+	struct nvme_speed_switch *sw = container_of(t, struct nvme_speed_switch, timer);
+
+	if (!READ_ONCE(sw->enabled)) {
+		atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE);
+		return;
+	}
+
+	sw->target_speed = nvme_check_io_and_decide_speed(sw);
+	if (sw->target_speed != READ_ONCE(sw->cur_speed))
+		schedule_work(&sw->work);
+
+	mod_timer(t, jiffies + msecs_to_jiffies(READ_ONCE(sw->monitor_interval)));
+}
+
+static void nvme_speed_switch_params_init(struct nvme_speed_switch *sw)
+{
+	sw->monitor_interval = 100;
+	sw->min_speed = PCI_EXP_LNKSTA_CLS_2_5GB;
+}
+
+void nvme_speed_switch_init(struct nvme_ctrl *ctrl)
+{
+	struct nvme_speed_switch *sw = &ctrl->speed_switch;
+	struct pci_dev *pdev;
+	int max_speed;
+
+	if (boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) {
+		dev_dbg(ctrl->device,
+			"link rate switching is only supported on Hygon platforms\n");
+		return;
+	}
+
+	if (!dev_is_pci(ctrl->dev)) {
+		dev_err(ctrl->device, "link rate switching requires a PCI device\n");
+		return;
+	}
+
+	pdev = to_pci_dev(ctrl->dev);
+	if (!pdev->bus->self) {
+		dev_err(ctrl->device,
+			"link rate switching requires a PCIe upstream port\n");
+		return;
+	}
+
+	nvme_speed_switch_params_init(sw);
+
+	max_speed = nvme_get_max_link_speed(pdev->bus->self);
+	if (max_speed <= sw->min_speed) {
+		dev_err(ctrl->device,
+			"invalid link rate range: min=Gen%u, max=Gen%d\n",
+			sw->min_speed, max_speed);
+		return;
+	}
+	sw->max_speed = max_speed;
+
+	sw->stats = alloc_percpu(struct nvme_speed_switch_stats);
+	if (!sw->stats) {
+		dev_err(ctrl->device, "failed to allocate per-CPU I/O statistics\n");
+		return;
+	}
+	nvme_clear_io_stats(sw);
+
+	WRITE_ONCE(sw->cur_speed, sw->max_speed);
+	WRITE_ONCE(sw->threshold, nvme_calc_speed_threshold(sw, pcie_get_width_cap(pdev)));
+	if (!READ_ONCE(sw->threshold)) {
+		dev_err(ctrl->device, "failed to compute speed switch threshold\n");
+		free_percpu(sw->stats);
+		sw->stats = NULL;
+		return;
+	}
+
+	INIT_WORK(&sw->work, nvme_speed_switch_work);
+	timer_setup(&sw->timer, nvme_speed_switch_timer_fn, 0);
+	atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE);
+	WRITE_ONCE(sw->enabled, true);
+	sw->initialized = true;
+
+	dev_info(ctrl->device,
+		 "NVMe link rate switching initialized (min Gen%u, max Gen%d)\n",
+		 sw->min_speed, sw->max_speed);
+}
+
+void nvme_speed_switch_start(struct nvme_ctrl *ctrl)
+{
+	struct nvme_speed_switch *sw = &ctrl->speed_switch;
+
+	if (!READ_ONCE(sw->enabled))
+		return;
+
+	atomic_set(&sw->timer_active, NVME_SPEED_TIMER_ACTIVE);
+	mod_timer(&sw->timer, jiffies + msecs_to_jiffies(READ_ONCE(sw->monitor_interval)));
+}
+
+void nvme_speed_switch_exit(struct nvme_ctrl *ctrl)
+{
+	struct nvme_speed_switch *sw = &ctrl->speed_switch;
+
+	if (!sw->initialized)
+		return;
+
+	sw->initialized = false;
+	WRITE_ONCE(sw->enabled, false);
+	timer_delete_sync(&sw->timer);
+	cancel_work_sync(&sw->work);
+
+	free_percpu(sw->stats);
+	sw->stats = NULL;
+}
diff --git a/drivers/pci/pcie/bwctrl.c b/drivers/pci/pcie/bwctrl.c
index c4c8d260b..b2949ed59 100644
--- a/drivers/pci/pcie/bwctrl.c
+++ b/drivers/pci/pcie/bwctrl.c
@@ -178,6 +178,7 @@ int pcie_set_target_speed(struct pci_dev *port, enum pci_bus_speed speed_req,
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(pcie_set_target_speed);
 
 static void pcie_bwnotif_enable(struct pcie_device *srv)
 {
-- 
2.43.0

[PATCH 2/3] nvme: Add hysteresis and idle detection to link rate switching

[Liao Xuan]

The rate switching decision currently toggles the link rate in every
monitoring window based on the I/O of that window alone, which can
cause excessive rate switching when the workload fluctuates around the
threshold.

Add counters that require the threshold to be exceeded for a
consecutive number of windows before the rate is upgraded, and to stay
below the threshold for a consecutive number of windows before it is
downgraded. The upgrade happens immediately (threshold 0 by default)
so that latency is not sacrificed, while the downgrade requires 10
consecutive windows.

Monitoring is stopped after 10 seconds of inactivity and is re-armed by
the next I/O.

Signed-off-by: Liao Xuan <liaoxuan@open-hieco.net>
---
 drivers/nvme/host/nvme.h         |  5 +++
 drivers/nvme/host/speed_switch.c | 52 ++++++++++++++++++++++++++++----
 2 files changed, 51 insertions(+), 6 deletions(-)

diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index ff06f85d1..5d5034601 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -346,6 +346,11 @@ struct nvme_speed_switch {
 	u8 min_speed;
 	u32 monitor_interval;	/* ms */
 	u32 threshold;		/* KB */
+	u32 up_cnt;
+	u32 down_cnt;
+	u32 up_threshold;
+	u32 down_threshold;
+	u32 idle_cnt;
 	struct timer_list timer;
 	struct work_struct work;
 	struct nvme_speed_switch_stats __percpu *stats;
diff --git a/drivers/nvme/host/speed_switch.c b/drivers/nvme/host/speed_switch.c
index 88c046fcd..22f180c81 100644
--- a/drivers/nvme/host/speed_switch.c
+++ b/drivers/nvme/host/speed_switch.c
@@ -228,10 +228,15 @@ static void nvme_clear_io_stats(struct nvme_speed_switch *sw)
 
 /*
  * Aggregate the per-CPU I/O counters of the last monitoring window and
- * decide the target rate: the maximum rate if the window exceeded the
- * threshold, the minimum rate otherwise.
+ * decide the target rate.  Hysteresis counters are used to avoid
+ * excessive rate switching when the workload fluctuates around the
+ * threshold: the rate is only upgraded after the threshold was exceeded
+ * for a number of consecutive windows, and only downgraded after it
+ * stayed below the threshold for a number of consecutive windows.  When
+ * no I/O was observed for 10 seconds the monitoring is stopped.
  */
-static int nvme_check_io_and_decide_speed(struct nvme_speed_switch *sw)
+static int nvme_check_io_and_decide_speed(struct nvme_speed_switch *sw,
+					  bool *io_activity)
 {
 	struct nvme_ctrl *ctrl = container_of(sw, struct nvme_ctrl, speed_switch);
 	u64 read_bytes = 0, write_bytes = 0;
@@ -247,31 +252,62 @@ static int nvme_check_io_and_decide_speed(struct nvme_speed_switch *sw)
 		stat->write_bytes = 0;
 	}
 
+	if (read_bytes || write_bytes) {
+		*io_activity = true;
+		sw->idle_cnt = 0;
+	} else {
+		sw->idle_cnt++;
+	}
+
+	/* Stop monitoring after 10 seconds of inactivity (100 windows of 100ms). */
+	if (sw->idle_cnt >= 100)
+		*io_activity = false;
+
 	read_kb = read_bytes / 1024;
 	write_kb = write_bytes / 1024;
 	dev_dbg(ctrl->device,
 		"I/O in window: read=%lu KB, write=%lu KB, threshold=%u KB\n",
 		read_kb, write_kb, READ_ONCE(sw->threshold));
 
-	if (read_kb >= sw->threshold || write_kb >= sw->threshold)
+	if (read_kb >= sw->threshold || write_kb >= sw->threshold) {
+		sw->up_cnt++;
+		sw->down_cnt = 0;
+	} else {
+		sw->down_cnt++;
+		sw->up_cnt = 0;
+	}
+
+	if (sw->up_cnt > READ_ONCE(sw->up_threshold)) {
+		sw->up_cnt = 0;
 		return sw->max_speed;
+	}
+
+	if (sw->down_cnt > READ_ONCE(sw->down_threshold) || !*io_activity) {
+		sw->down_cnt = 0;
+		return sw->min_speed;
+	}
 
-	return sw->min_speed;
+	return READ_ONCE(sw->cur_speed);
 }
 
 static void nvme_speed_switch_timer_fn(struct timer_list *t)
 {
 	struct nvme_speed_switch *sw = container_of(t, struct nvme_speed_switch, timer);
+	bool io_activity = true;
 
 	if (!READ_ONCE(sw->enabled)) {
 		atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE);
 		return;
 	}
 
-	sw->target_speed = nvme_check_io_and_decide_speed(sw);
+	sw->target_speed = nvme_check_io_and_decide_speed(sw, &io_activity);
 	if (sw->target_speed != READ_ONCE(sw->cur_speed))
 		schedule_work(&sw->work);
 
+	if (!io_activity) {
+		atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE);
+		return;
+	}
 	mod_timer(t, jiffies + msecs_to_jiffies(READ_ONCE(sw->monitor_interval)));
 }
 
@@ -279,6 +315,10 @@ static void nvme_speed_switch_params_init(struct nvme_speed_switch *sw)
 {
 	sw->monitor_interval = 100;
 	sw->min_speed = PCI_EXP_LNKSTA_CLS_2_5GB;
+	/* Upgrade immediately once the threshold is exceeded. */
+	sw->up_threshold = 0;
+	/* Downgrade after 10 consecutive below-threshold windows. */
+	sw->down_threshold = 10;
 }
 
 void nvme_speed_switch_init(struct nvme_ctrl *ctrl)
-- 
2.43.0

[PATCH 3/3] nvme: Expose link rate switching tunables via sysfs

[Liao Xuan]

Add a speed attribute group under /sys/class/nvme/nvmeX/ that
exposes the adaptive link rate switching parameters for runtime tuning:

  enable           - toggle the feature on/off (0/1)
  monitor_interval - I/O monitoring window in milliseconds (>= 100)
  min_speed        - minimum PCIe generation to downgrade to (1-5)
  up_threshold     - windows above the threshold needed to upgrade
  down_threshold   - windows below the threshold needed to downgrade

The attributes are read/write and are removed when the controller is
torn down.

Signed-off-by: Liao Xuan <liaoxuan@open-hieco.net>
---
 drivers/nvme/host/nvme.h         |   3 +
 drivers/nvme/host/speed_switch.c |  15 +++-
 drivers/nvme/host/sysfs.c        | 143 +++++++++++++++++++++++++++++++
 3 files changed, 160 insertions(+), 1 deletion(-)

diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 5d5034601..fb958ff13 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -1076,6 +1076,9 @@ extern const struct attribute_group nvme_dev_attrs_group;
 extern const struct attribute_group nvme_dev_diag_attrs_group;
 extern const struct attribute_group *nvme_subsys_attrs_groups[];
 extern const struct attribute_group *nvme_dev_attr_groups[];
+#ifdef CONFIG_NVME_SPEED_SWITCH
+extern const struct attribute_group nvme_speed_attr_group;
+#endif
 extern const struct block_device_operations nvme_bdev_ops;
 
 void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl);
diff --git a/drivers/nvme/host/speed_switch.c b/drivers/nvme/host/speed_switch.c
index 22f180c81..fce426845 100644
--- a/drivers/nvme/host/speed_switch.c
+++ b/drivers/nvme/host/speed_switch.c
@@ -325,7 +325,7 @@ void nvme_speed_switch_init(struct nvme_ctrl *ctrl)
 {
 	struct nvme_speed_switch *sw = &ctrl->speed_switch;
 	struct pci_dev *pdev;
-	int max_speed;
+	int max_speed, ret;
 
 	if (boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) {
 		dev_dbg(ctrl->device,
@@ -376,6 +376,18 @@ void nvme_speed_switch_init(struct nvme_ctrl *ctrl)
 	timer_setup(&sw->timer, nvme_speed_switch_timer_fn, 0);
 	atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE);
 	WRITE_ONCE(sw->enabled, true);
+
+	ret = sysfs_create_group(&ctrl->device->kobj, &nvme_speed_attr_group);
+	if (ret) {
+		dev_err(ctrl->device,
+			"failed to create link rate switching sysfs group (%d)\n",
+			ret);
+		timer_delete_sync(&sw->timer);
+		free_percpu(sw->stats);
+		sw->stats = NULL;
+		WRITE_ONCE(sw->enabled, false);
+		return;
+	}
 	sw->initialized = true;
 
 	dev_info(ctrl->device,
@@ -405,6 +417,7 @@ void nvme_speed_switch_exit(struct nvme_ctrl *ctrl)
 	WRITE_ONCE(sw->enabled, false);
 	timer_delete_sync(&sw->timer);
 	cancel_work_sync(&sw->work);
+	sysfs_remove_group(&ctrl->device->kobj, &nvme_speed_attr_group);
 
 	free_percpu(sw->stats);
 	sw->stats = NULL;
diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c
index 75b2d69b5..cd899b87b 100644
--- a/drivers/nvme/host/sysfs.c
+++ b/drivers/nvme/host/sysfs.c
@@ -1308,3 +1308,146 @@ const struct attribute_group *nvme_subsys_attrs_groups[] = {
 	&nvme_subsys_attrs_group,
 	NULL,
 };
+
+#ifdef CONFIG_NVME_SPEED_SWITCH
+enum nvme_speed_attr_id {
+	NVME_SPEED_ATTR_ENABLE,
+	NVME_SPEED_ATTR_MONITOR_INTERVAL,
+	NVME_SPEED_ATTR_MIN_SPEED,
+	NVME_SPEED_ATTR_UP_THRESHOLD,
+	NVME_SPEED_ATTR_DOWN_THRESHOLD,
+};
+
+struct nvme_speed_attr {
+	struct device_attribute attr;
+	enum nvme_speed_attr_id id;
+};
+
+static ssize_t nvme_speed_attr_show(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
+	struct nvme_speed_switch *sw = &ctrl->speed_switch;
+	struct nvme_speed_attr *sa =
+		container_of(attr, struct nvme_speed_attr, attr);
+	u32 val;
+
+	switch (sa->id) {
+	case NVME_SPEED_ATTR_ENABLE:
+		val = READ_ONCE(sw->enabled);
+		break;
+	case NVME_SPEED_ATTR_MONITOR_INTERVAL:
+		val = READ_ONCE(sw->monitor_interval);
+		break;
+	case NVME_SPEED_ATTR_MIN_SPEED:
+		val = READ_ONCE(sw->min_speed);
+		break;
+	case NVME_SPEED_ATTR_UP_THRESHOLD:
+		val = READ_ONCE(sw->up_threshold);
+		break;
+	case NVME_SPEED_ATTR_DOWN_THRESHOLD:
+		val = READ_ONCE(sw->down_threshold);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return sysfs_emit(buf, "%u\n", val);
+}
+
+static ssize_t nvme_speed_attr_store(struct device *dev,
+				     struct device_attribute *attr,
+				     const char *buf, size_t count)
+{
+	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
+	struct nvme_speed_switch *sw = &ctrl->speed_switch;
+	struct nvme_speed_attr *sa =
+		container_of(attr, struct nvme_speed_attr, attr);
+	u32 val;
+	int ret;
+
+	ret = kstrtou32(buf, 10, &val);
+	if (ret)
+		return ret;
+
+	switch (sa->id) {
+	case NVME_SPEED_ATTR_ENABLE:
+		if (val > 1)
+			return -EINVAL;
+		if (!val && READ_ONCE(sw->enabled)) {
+			WRITE_ONCE(sw->enabled, false);
+			timer_delete_sync(&sw->timer);
+			cancel_work_sync(&sw->work);
+			atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE);
+		} else
+			WRITE_ONCE(sw->enabled, val);
+		break;
+	case NVME_SPEED_ATTR_MONITOR_INTERVAL:
+		if (val < 100)
+			return -EINVAL;
+		WRITE_ONCE(sw->monitor_interval, val);
+		break;
+	case NVME_SPEED_ATTR_MIN_SPEED:
+		if (val < 1 || val > 5)
+			return -EINVAL;
+		if (val > sw->max_speed)
+			val = sw->max_speed;
+		WRITE_ONCE(sw->min_speed, val);
+		break;
+	case NVME_SPEED_ATTR_UP_THRESHOLD:
+		WRITE_ONCE(sw->up_threshold, val);
+		break;
+	case NVME_SPEED_ATTR_DOWN_THRESHOLD:
+		WRITE_ONCE(sw->down_threshold, val);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return count;
+}
+
+static struct nvme_speed_attr nvme_speed_attr_enable = {
+	.attr = __ATTR(enable, 0644,
+		       nvme_speed_attr_show, nvme_speed_attr_store),
+	.id = NVME_SPEED_ATTR_ENABLE,
+};
+
+static struct nvme_speed_attr nvme_speed_attr_monitor_interval = {
+	.attr = __ATTR(monitor_interval, 0644,
+		       nvme_speed_attr_show, nvme_speed_attr_store),
+	.id = NVME_SPEED_ATTR_MONITOR_INTERVAL,
+};
+
+static struct nvme_speed_attr nvme_speed_attr_min_speed = {
+	.attr = __ATTR(min_speed, 0644,
+		       nvme_speed_attr_show, nvme_speed_attr_store),
+	.id = NVME_SPEED_ATTR_MIN_SPEED,
+};
+
+static struct nvme_speed_attr nvme_speed_attr_up_threshold = {
+	.attr = __ATTR(up_threshold, 0644,
+		       nvme_speed_attr_show, nvme_speed_attr_store),
+	.id = NVME_SPEED_ATTR_UP_THRESHOLD,
+};
+
+static struct nvme_speed_attr nvme_speed_attr_down_threshold = {
+	.attr = __ATTR(down_threshold, 0644,
+		       nvme_speed_attr_show, nvme_speed_attr_store),
+	.id = NVME_SPEED_ATTR_DOWN_THRESHOLD,
+};
+
+static struct attribute *nvme_speed_attrs[] = {
+	&nvme_speed_attr_enable.attr.attr,
+	&nvme_speed_attr_monitor_interval.attr.attr,
+	&nvme_speed_attr_min_speed.attr.attr,
+	&nvme_speed_attr_up_threshold.attr.attr,
+	&nvme_speed_attr_down_threshold.attr.attr,
+	NULL,
+};
+
+const struct attribute_group nvme_speed_attr_group = {
+	.name = "speed",
+	.attrs = nvme_speed_attrs,
+};
+#endif /* CONFIG_NVME_SPEED_SWITCH */
-- 
2.43.0