mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: David Matlack <dmatlack@google.com>
To: kvm@vger.kernel.org, linux-kernel@vger.kernel.org,
	 linux-kselftest@vger.kernel.org, stable@vger.kernel.org
Cc: Abhishek Sahu <abhsahu@nvidia.com>,
	Alex Williamson <alex@shazbot.org>,
	 David Matlack <dmatlack@google.com>,
	Raghavendra Rao Ananta <rananta@google.com>,
	Shuah Khan <shuah@kernel.org>,  Vipin Sharma <vipinsh@google.com>
Subject: [PATCH v2 8/8] vfio: selftests: Add test for device power state transitions
Date: Thu, 19 Nov 2026 00:01:50 +0000	[thread overview]
Message-ID: <20261119000151.442543-9-dmatlack@google.com> (raw)
In-Reply-To: <20261119000151.442543-1-dmatlack@google.com>

Add vfio_pci_power_state_test to exercise VFIO's handling of device
power state changes initiated by userspace through the PM capability in
config space.

The test covers:

 - Closing a device in D3hot and then reopening it, which must leave
   the device in D0.
 - Enabling SR-IOV while the PF is in D3hot, which must return the PF to
   D0 and prevent it from leaving D0 while VFs are enabled.
 - Racing a close of the device while it is in D3hot against enabling
   SR-IOV, both of which transition the device back to D0.

The SR-IOV tests are skipped if the device does not support SR-IOV,
already has VFs enabled, or SR-IOV cannot be configured on it (e.g.
vfio-pci with vfio_pci.enable_sriov=N). If an SR-IOV test case fails,
any VFs it left enabled are disabled so they don't affect later test
cases.

Fail the test if /sys/kernel/warn_count changes between fixture setup
and teardown, since bugs such as the close vs. SR-IOV enable race show
up only as kernel warnings. When that happens, print the warnings
logged to /dev/kmsg during the test so that the failure can be debugged
from the test log.

Signed-off-by: David Matlack <dmatlack@google.com>
---
 tools/testing/selftests/vfio/Makefile         |   1 +
 .../vfio/vfio_pci_power_state_test.c          | 256 ++++++++++++++++++
 2 files changed, 257 insertions(+)
 create mode 100644 tools/testing/selftests/vfio/vfio_pci_power_state_test.c

diff --git a/tools/testing/selftests/vfio/Makefile b/tools/testing/selftests/vfio/Makefile
index 2c32c48db509..e2b152528560 100644
--- a/tools/testing/selftests/vfio/Makefile
+++ b/tools/testing/selftests/vfio/Makefile
@@ -12,6 +12,7 @@ TEST_GEN_PROGS += vfio_iommufd_setup_test
 TEST_GEN_PROGS += vfio_pci_device_test
 TEST_GEN_PROGS += vfio_pci_device_init_perf_test
 TEST_GEN_PROGS += vfio_pci_driver_test
+TEST_GEN_PROGS += vfio_pci_power_state_test
 TEST_GEN_PROGS += vfio_pci_sriov_uapi_test
 
 TEST_FILES += scripts/cleanup.sh
diff --git a/tools/testing/selftests/vfio/vfio_pci_power_state_test.c b/tools/testing/selftests/vfio/vfio_pci_power_state_test.c
new file mode 100644
index 000000000000..5b185c174036
--- /dev/null
+++ b/tools/testing/selftests/vfio/vfio_pci_power_state_test.c
@@ -0,0 +1,256 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <pthread.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <linux/pci_regs.h>
+
+#include <libvfio.h>
+
+#include "kselftest_harness.h"
+
+/* PCI_PM_CTRL_STATE_MASK values */
+#define D0	0
+#define D3HOT	3
+
+static const char *device_bdf;
+
+static u16 power_state_get(struct vfio_pci_device *device, u8 pm_cap)
+{
+	return vfio_pci_config_readw(device, pm_cap + PCI_PM_CTRL) &
+	       PCI_PM_CTRL_STATE_MASK;
+}
+
+static void power_state_set(struct vfio_pci_device *device, u8 pm_cap,
+			    u16 state)
+{
+	u16 pmcsr = vfio_pci_config_readw(device, pm_cap + PCI_PM_CTRL);
+
+	pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
+	pmcsr |= state;
+
+	vfio_pci_config_writew(device, pm_cap + PCI_PM_CTRL, pmcsr);
+}
+
+/*
+ * Set by test cases that enable VFs, so that disable_vfs() can disable any VFs
+ * left enabled if the test case fails. Only set in the child process that runs
+ * the test case.
+ */
+static bool test_enabled_vfs;
+
+/*
+ * Disable VFs left enabled by a failed test case, so they don't affect later
+ * test cases. Called from fixture teardown, which runs when an ASSERT fails,
+ * and from an atexit() handler, which runs when a library helper exits the
+ * test process early (e.g. on a VFIO_ASSERT() failure).
+ */
+static void disable_vfs(void)
+{
+	if (test_enabled_vfs && sysfs_sriov_numvfs_get(device_bdf) > 0)
+		__sysfs_sriov_numvfs_set(device_bdf, 0);
+}
+
+/*
+ * Check whether the test can enable VFs on the device, briefly enabling one VF
+ * to find out. Returns the reason to skip the test, or NULL if it can run.
+ */
+static const char *sriov_skip_reason(void)
+{
+	int ret;
+
+	if (sysfs_sriov_totalvfs_get(device_bdf) <= 0)
+		return "Device does not support SR-IOV";
+
+	if (sysfs_sriov_numvfs_get(device_bdf) != 0)
+		return "SR-IOV is already enabled on the device";
+
+	/* From here on, any VFs on the device were enabled by this test. */
+	test_enabled_vfs = true;
+
+	/* -ENOENT means SR-IOV cannot be configured on the device. */
+	ret = __sysfs_sriov_numvfs_set(device_bdf, 1);
+	if (ret == -ENOENT)
+		return "Cannot enable SR-IOV on the device (vfio-pci requires enable_sriov=1)";
+
+	VFIO_ASSERT_EQ(ret, 0);
+	sysfs_sriov_numvfs_set(device_bdf, 0);
+	return NULL;
+}
+
+FIXTURE(vfio_pci_power_state_test) {
+	struct iommu *iommu;
+	struct vfio_pci_device *device;
+	u8 pm_cap;
+	int warn_count;
+	int kmsg_fd;
+};
+
+FIXTURE_SETUP(vfio_pci_power_state_test)
+{
+	/*
+	 * Snapshot the kernel's warning count so that teardown can check that
+	 * the test did not trigger any kernel warnings, which are otherwise
+	 * invisible to the test. Open /dev/kmsg first so that any warning
+	 * counted after the snapshot can be printed if the check fails.
+	 */
+	self->kmsg_fd = kmsg_open();
+	self->warn_count = sysfs_kernel_warn_count_get();
+
+	self->iommu = iommu_init(default_iommu_mode);
+	self->device = vfio_pci_device_init(device_bdf, self->iommu);
+
+	self->pm_cap = vfio_pci_find_capability(self->device, PCI_CAP_ID_PM);
+	if (!self->pm_cap)
+		SKIP(return, "Device does not have a PM capability\n");
+
+	ASSERT_EQ(power_state_get(self->device, self->pm_cap), D0);
+}
+
+FIXTURE_TEARDOWN(vfio_pci_power_state_test)
+{
+	disable_vfs();
+
+	if (self->device)
+		vfio_pci_device_cleanup(self->device);
+
+	iommu_cleanup(self->iommu);
+
+	if (self->warn_count >= 0) {
+		int warn_count = sysfs_kernel_warn_count_get();
+
+		if (warn_count != self->warn_count && self->kmsg_fd >= 0)
+			kmsg_print_warnings(self->kmsg_fd);
+
+		EXPECT_EQ(warn_count, self->warn_count);
+	}
+
+	if (self->kmsg_fd >= 0)
+		close(self->kmsg_fd);
+}
+
+/*
+ * Put the device into D3hot via config space, close it, and then reopen it.
+ * Closing the device while it is in D3hot must not trigger any kernel warnings
+ * (checked during fixture teardown), and the device must be in D0 when it is
+ * reopened.
+ *
+ * Note that this does not verify that VFIO returns the device to D0 during
+ * close, since that is not reliably observable from userspace: the device may
+ * runtime suspend after it is closed, and opening it also transitions it to
+ * D0.
+ */
+TEST_F(vfio_pci_power_state_test, reopen_after_close_in_d3hot)
+{
+	power_state_set(self->device, self->pm_cap, D3HOT);
+	ASSERT_EQ(power_state_get(self->device, self->pm_cap), D3HOT);
+
+	vfio_pci_device_cleanup(self->device);
+
+	self->device = vfio_pci_device_init(device_bdf, self->iommu);
+	ASSERT_EQ(power_state_get(self->device, self->pm_cap), D0);
+}
+
+/*
+ * Put the device into D3hot via config space and then enable SR-IOV. VFIO
+ * must return the PF to D0 before enabling VFs, and must not allow the PF to
+ * leave D0 while VFs are enabled.
+ */
+TEST_F(vfio_pci_power_state_test, sriov_enable_in_d3hot)
+{
+	const char *skip_reason = sriov_skip_reason();
+
+	if (skip_reason)
+		SKIP(return, "%s", skip_reason);
+
+	power_state_set(self->device, self->pm_cap, D3HOT);
+	ASSERT_EQ(power_state_get(self->device, self->pm_cap), D3HOT);
+
+	sysfs_sriov_numvfs_set(device_bdf, 1);
+	EXPECT_EQ(power_state_get(self->device, self->pm_cap), D0);
+
+	power_state_set(self->device, self->pm_cap, D3HOT);
+	EXPECT_EQ(power_state_get(self->device, self->pm_cap), D0);
+
+	sysfs_sriov_numvfs_set(device_bdf, 0);
+
+	power_state_set(self->device, self->pm_cap, D3HOT);
+	EXPECT_EQ(power_state_get(self->device, self->pm_cap), D3HOT);
+
+	power_state_set(self->device, self->pm_cap, D0);
+	EXPECT_EQ(power_state_get(self->device, self->pm_cap), D0);
+}
+
+struct sriov_enable_args {
+	pthread_barrier_t barrier;
+	int delay_us;
+};
+
+static void *sriov_enable_thread(void *arg)
+{
+	struct sriov_enable_args *args = arg;
+
+	pthread_barrier_wait(&args->barrier);
+	usleep(args->delay_us);
+	sysfs_sriov_numvfs_set(device_bdf, 1);
+	return NULL;
+}
+
+/*
+ * Race closing the device while it is in D3hot against enabling SR-IOV. Both
+ * paths transition the device back to D0, which must be serialized by VFIO,
+ * e.g. to avoid restoring the PF's BARs after VFs have been enabled or
+ * double-freeing the saved PM state.
+ *
+ * The D3hot->D0 transition during close includes a delay (10ms by default)
+ * before the device's state is restored. Use a barrier to start closing the
+ * device and enabling SR-IOV at the same time, and then delay enabling SR-IOV
+ * by a few milliseconds so that it lands inside that window. Vary the delay
+ * across iterations to account for differences in timing across systems and
+ * kernel configs.
+ */
+TEST_F(vfio_pci_power_state_test, close_in_d3hot_races_sriov_enable)
+{
+	struct sriov_enable_args args;
+	pthread_t thread;
+	const char *skip_reason;
+	int i;
+
+	skip_reason = sriov_skip_reason();
+	if (skip_reason)
+		SKIP(return, "%s", skip_reason);
+
+	ASSERT_EQ(pthread_barrier_init(&args.barrier, NULL, 2), 0);
+
+	/*
+	 * Keep the number of iterations low since each one enables and
+	 * disables SR-IOV, which can take over a second.
+	 */
+	for (i = 0; i < 10; i++) {
+		args.delay_us = 2000 + i * 500;
+
+		power_state_set(self->device, self->pm_cap, D3HOT);
+		ASSERT_EQ(power_state_get(self->device, self->pm_cap), D3HOT);
+
+		ASSERT_EQ(pthread_create(&thread, NULL, sriov_enable_thread, &args), 0);
+		pthread_barrier_wait(&args.barrier);
+		vfio_pci_device_cleanup(self->device);
+		self->device = NULL;
+		ASSERT_EQ(pthread_join(thread, NULL), 0);
+
+		sysfs_sriov_numvfs_set(device_bdf, 0);
+
+		self->device = vfio_pci_device_init(device_bdf, self->iommu);
+		ASSERT_EQ(power_state_get(self->device, self->pm_cap), D0);
+	}
+
+	ASSERT_EQ(pthread_barrier_destroy(&args.barrier), 0);
+}
+
+int main(int argc, char *argv[])
+{
+	device_bdf = vfio_selftests_get_bdf(&argc, argv);
+	VFIO_ASSERT_EQ(atexit(disable_vfs), 0);
+
+	return test_harness_run(argc, argv);
+}
-- 
2.56.1.708.g77570a6a1d-goog


      parent reply	other threads:[~2026-11-19  0:02 UTC|newest]

Thread overview: 9+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-19  0:01 [PATCH v2 0/8] vfio/pci: Fix power state race with SR-IOV enablement David Matlack
2026-11-19  0:01 ` [PATCH v2 1/8] vfio/pci: Hold memory_lock for all calls to vfio_pci_set_power_state() David Matlack
2026-11-19  0:01 ` [PATCH v2 2/8] vfio/pci: Annotate lockless accesses to vdev->sriov_active David Matlack
2026-11-19  0:01 ` [PATCH v2 3/8] vfio: selftests: Add __sysfs_sriov_numvfs_set() David Matlack
2026-11-19  0:01 ` [PATCH v2 4/8] vfio: selftests: Skip vfio_pci_sriov_uapi_test if SR-IOV can't be enabled David Matlack
2026-11-19  0:01 ` [PATCH v2 5/8] vfio: selftests: Add vfio_pci_find_capability() David Matlack
2026-11-19  0:01 ` [PATCH v2 6/8] vfio: selftests: Add sysfs_kernel_warn_count_get() David Matlack
2026-11-19  0:01 ` [PATCH v2 7/8] vfio: selftests: Add helpers to print kernel warnings from /dev/kmsg David Matlack
2026-11-19  0:01 ` David Matlack [this message]

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261119000151.442543-9-dmatlack@google.com \
    --to=dmatlack@google.com \
    --cc=abhsahu@nvidia.com \
    --cc=alex@shazbot.org \
    --cc=kvm@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-kselftest@vger.kernel.org \
    --cc=rananta@google.com \
    --cc=shuah@kernel.org \
    --cc=stable@vger.kernel.org \
    --cc=vipinsh@google.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®