[RFC PATCH v3 01/20] vfio/pci: Add SRCU-protected access ops

[Shameer Kolothum <skolothumtho@nvidia.com>]

Before recovering a device, VFIO needs to stop new hardware accesses and
wait for operations already in progress to finish.

Give each open device an SRCU instance to track hardware accesses.
Enable SRCU access tracking only for devices with
pci_recovery_supported set.
Select and call the access operation within the same SRCU read-side
section. Keep copies to and from userspace outside that section so a
page fault cannot hold up recovery.

Introduce live and blocked tables. Initialize the ops pointer to blocked
when the device is created, and select live ops when open finishes.
Close uses a blocking transition that installs blocked ops and drains
SRCU readers before teardown. Open publishes live ops after setup.
Serialize these transitions with access_lock. Internal setup and cleanup
use their own helpers.

Provide an accessor for operations that use their existing synchronization
instead of SRCU. The accessor only returns the selected table. It does
not acquire a lock or keep the device's per-open resources alive.

Subsequent patches convert hardware access paths to these ops and add
support for host PCI error recovery using the same blocked table.

Signed-off-by: Shameer Kolothum <skolothumtho@nvidia.com>
---
 drivers/vfio/pci/vfio_pci_priv.h | 15 ++++++
 include/linux/vfio_pci_core.h    | 10 ++++
 drivers/vfio/pci/vfio_pci_core.c | 86 ++++++++++++++++++++++++++++++++
 3 files changed, 111 insertions(+)

diff --git a/drivers/vfio/pci/vfio_pci_priv.h b/drivers/vfio/pci/vfio_pci_priv.h
index 4e7162234a2e..97b69e4bc8be 100644
--- a/drivers/vfio/pci/vfio_pci_priv.h
+++ b/drivers/vfio/pci/vfio_pci_priv.h
@@ -73,6 +73,21 @@ u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev);
 void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev,
 					u16 cmd);
 
+/* Operations receive kernel buffers only. User copies stay outside SRCU. */
+typedef long (*vfio_pci_access_fn)(struct vfio_pci_core_device *vdev,
+				 void *data);
+long vfio_pci_core_access(struct vfio_pci_core_device *vdev,
+			  vfio_pci_access_fn fn, void *data);
+
+struct vfio_pci_access_ops {
+	/* Select and call within SRCU. The callback receives kernel buffers. */
+	long (*execute)(struct vfio_pci_core_device *vdev,
+			vfio_pci_access_fn fn, void *data);
+};
+
+const struct vfio_pci_access_ops *
+vfio_pci_access_ops(struct vfio_pci_core_device *vdev);
+
 #ifdef CONFIG_VFIO_PCI_IGD
 bool vfio_pci_is_intel_display(struct pci_dev *pdev);
 int vfio_pci_igd_init(struct vfio_pci_core_device *vdev);
diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
index 2f3fa2787737..88aa7b752231 100644
--- a/include/linux/vfio_pci_core.h
+++ b/include/linux/vfio_pci_core.h
@@ -14,6 +14,7 @@
 #include <linux/vfio.h>
 #include <linux/irqbypass.h>
 #include <linux/rcupdate.h>
+#include <linux/srcu.h>
 #include <linux/types.h>
 #include <linux/uuid.h>
 #include <linux/notifier.h>
@@ -97,6 +98,8 @@ static inline int vfio_pci_core_get_dmabuf_phys(
 }
 #endif
 
+struct vfio_pci_access_ops;
+
 struct vfio_pci_core_device {
 	struct vfio_device	vdev;
 	struct pci_dev		*pdev;
@@ -131,6 +134,7 @@ struct vfio_pci_core_device {
 	bool			disable_idle_d3:1;
 	bool			nointxmask:1;
 	bool			disable_vga:1;
+	bool			pci_recovery_supported:1;
 	/* Flags modified at runtime - dedicated storage unit */
 	bool			needs_reset;
 	bool			pm_intx_masked;
@@ -150,6 +154,12 @@ struct vfio_pci_core_device {
 	struct vfio_pci_core_device	*sriov_pf_core_dev;
 	struct notifier_block	nb;
 	struct rw_semaphore	memory_lock;
+	struct srcu_struct	access_srcu;
+	/* Updated under access_lock. */
+	const struct vfio_pci_access_ops __rcu *access_ops;
+	/* Set after open completes, cleared before close tears down state. */
+	bool			device_open;
+	struct mutex		access_lock;	/* protects changes to access ops and flags */
 	struct list_head	dmabufs;
 };
 
diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
index fcbb2da860e2..b7d81233b99b 100644
--- a/drivers/vfio/pci/vfio_pci_core.c
+++ b/drivers/vfio/pci/vfio_pci_core.c
@@ -312,6 +312,65 @@ static void vfio_pci_probe_power_state(struct vfio_pci_core_device *vdev)
 	vdev->needs_pm_restore = !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET);
 }
 
+static long vfio_pci_access_live(struct vfio_pci_core_device *vdev,
+				 vfio_pci_access_fn fn, void *data)
+{
+	return fn(vdev, data);
+}
+
+static long vfio_pci_access_blocked(struct vfio_pci_core_device *vdev,
+				    vfio_pci_access_fn fn, void *data)
+{
+	return -EIO;
+}
+
+static const struct vfio_pci_access_ops vfio_pci_live_ops = {
+	.execute = vfio_pci_access_live,
+};
+
+static const struct vfio_pci_access_ops vfio_pci_blocked_ops = {
+	.execute = vfio_pci_access_blocked,
+};
+
+/*
+ * The ops tables are static and remain valid across open and close.
+ * Ordinary access callbacks run through vfio_pci_core_access(), which
+ * holds SRCU across table selection and callback execution.
+ */
+const struct vfio_pci_access_ops *
+vfio_pci_access_ops(struct vfio_pci_core_device *vdev)
+{
+	if (!vdev->pci_recovery_supported)
+		return &vfio_pci_live_ops;
+	return rcu_dereference_raw(vdev->access_ops);
+}
+
+/* Block new operations and drain ordinary accesses before teardown or recovery. */
+static void vfio_pci_core_block_access(struct vfio_pci_core_device *vdev)
+{
+	lockdep_assert_held(&vdev->access_lock);
+	rcu_assign_pointer(vdev->access_ops, &vfio_pci_blocked_ops);
+
+	synchronize_srcu(&vdev->access_srcu);
+}
+
+long vfio_pci_core_access(struct vfio_pci_core_device *vdev,
+			  vfio_pci_access_fn fn, void *data)
+{
+	const struct vfio_pci_access_ops *ops;
+	long ret;
+	int idx;
+
+	if (!vdev->pci_recovery_supported)
+		return vfio_pci_live_ops.execute(vdev, fn, data);
+
+	idx = srcu_read_lock(&vdev->access_srcu);
+	ops = srcu_dereference(vdev->access_ops, &vdev->access_srcu);
+	ret = ops->execute(vdev, fn, data);
+	srcu_read_unlock(&vdev->access_srcu, idx);
+	return ret;
+}
+
 /*
  * pci_set_power_state() wrapper handling devices which perform a soft reset on
  * D3->D0 transition.  Save state prior to D0/1/2->D3, stash it on the vdev,
@@ -649,10 +708,18 @@ int vfio_pci_core_enable(struct vfio_pci_core_device *vdev)
 	if (!vfio_vga_disabled(vdev) && vfio_pci_is_vga(pdev))
 		vdev->has_vga = true;
 
+	if (vdev->pci_recovery_supported) {
+		ret = init_srcu_struct(&vdev->access_srcu);
+		if (ret)
+			goto out_free_config;
+	}
+
 	vfio_pci_core_map_bars(vdev);
 
 	return 0;
 
+out_free_config:
+	vfio_config_free(vdev);
 out_free_zdev:
 	vfio_pci_zdev_close_device(vdev);
 out_free_state:
@@ -799,6 +866,9 @@ void vfio_pci_core_disable(struct vfio_pci_core_device *vdev)
 	/* Put the pm-runtime usage counter acquired during enable */
 	if (!vdev->disable_idle_d3)
 		pm_runtime_put(&pdev->dev);
+
+	if (vdev->pci_recovery_supported)
+		cleanup_srcu_struct(&vdev->access_srcu);
 }
 EXPORT_SYMBOL_GPL(vfio_pci_core_disable);
 
@@ -807,6 +877,13 @@ void vfio_pci_core_close_device(struct vfio_device *core_vdev)
 	struct vfio_pci_core_device *vdev =
 		container_of(core_vdev, struct vfio_pci_core_device, vdev);
 
+	if (vdev->pci_recovery_supported) {
+		scoped_guard(mutex, &vdev->access_lock) {
+			WRITE_ONCE(vdev->device_open, false);
+			vfio_pci_core_block_access(vdev);
+		}
+	}
+
 	if (vdev->sriov_pf_core_dev) {
 		mutex_lock(&vdev->sriov_pf_core_dev->vf_token->lock);
 		WARN_ON(!vdev->sriov_pf_core_dev->vf_token->users);
@@ -839,6 +916,12 @@ void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev)
 		vdev->sriov_pf_core_dev->vf_token->users++;
 		mutex_unlock(&vdev->sriov_pf_core_dev->vf_token->lock);
 	}
+
+	if (vdev->pci_recovery_supported) {
+		guard(mutex)(&vdev->access_lock);
+		WRITE_ONCE(vdev->device_open, true);
+		rcu_assign_pointer(vdev->access_ops, &vfio_pci_live_ops);
+	}
 }
 EXPORT_SYMBOL_GPL(vfio_pci_core_finish_enable);
 
@@ -2183,6 +2266,8 @@ int vfio_pci_core_init_dev(struct vfio_device *core_vdev)
 	if (ret && ret != -EOPNOTSUPP)
 		return ret;
 	INIT_LIST_HEAD(&vdev->dmabufs);
+	mutex_init(&vdev->access_lock);
+	RCU_INIT_POINTER(vdev->access_ops, &vfio_pci_blocked_ops);
 	init_rwsem(&vdev->memory_lock);
 	xa_init(&vdev->ctx);
 
@@ -2197,6 +2282,7 @@ void vfio_pci_core_release_dev(struct vfio_device *core_vdev)
 
 	mutex_destroy(&vdev->igate);
 	mutex_destroy(&vdev->ioeventfds_lock);
+	mutex_destroy(&vdev->access_lock);
 	kfree(vdev->region);
 	kfree(vdev->pm_save);
 }
-- 
2.43.0