[RFC PATCH v3 19/20] vfio/pci: Add VFIO_DEVICE_FEATURE_PCI_ERROR_RECOVERY

[Shameer Kolothum <skolothumtho@nvidia.com>]

Userspace needs a way to enable host recovery and find out whether it is
still running, succeeded, reset the device or failed.

Add a device feature that registers a recovery eventfd and reports
status flags and a sequence number. SET with eventfd -1 disables
recovery. GET is available while hardware access is blocked. The
sequence number lets userspace detect recovery that finished before it
read the notification.

Reject SET if access is blocked, recovery has failed or a recorded host
recovery is still running. This also applies if userspace had not
enabled recovery when the event started, so a late subscription cannot
report part of an older event as a new one. Updating the eventfd resets
status for the current open. The existing error eventfd is unchanged.

Add the recovery eventfd and signal it when recovery starts, completes
or fails. Release it during close. Signal under access_lock so SET cannot
replace the eventfd and reset status before notification.

Userspace must probe each device and restore guest state after a reset.

Signed-off-by: Shameer Kolothum <skolothumtho@nvidia.com>
---
 include/linux/vfio_pci_core.h    |   1 +
 include/uapi/linux/vfio.h        |  51 ++++++++++++
 drivers/vfio/pci/vfio_pci_core.c | 139 +++++++++++++++++++++++++++++++
 3 files changed, 191 insertions(+)

diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
index cd7d87788de0..896c176c28a0 100644
--- a/include/linux/vfio_pci_core.h
+++ b/include/linux/vfio_pci_core.h
@@ -152,6 +152,7 @@ struct vfio_pci_core_device {
 	int			ioeventfds_nr;
 	struct vfio_pci_eventfd __rcu *err_trigger;
 	struct vfio_pci_eventfd __rcu *req_trigger;
+	struct vfio_pci_eventfd __rcu *pci_recovery_trigger;
 	struct eventfd_ctx	*pm_wake_eventfd_ctx;
 	struct list_head	dummy_resources_list;
 	struct mutex		ioeventfds_lock;
diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
index e41437fa17ad..86a3952ee302 100644
--- a/include/uapi/linux/vfio.h
+++ b/include/uapi/linux/vfio.h
@@ -1555,6 +1555,57 @@ struct vfio_device_feature_zpci_err {
 
 #define VFIO_DEVICE_FEATURE_ZPCI_ERROR 13
 
+/*
+ * Report host PCI error recovery state for this device.
+ *
+ * Userspace must probe the feature on each device before enabling it.
+ *
+ * VFIO_DEVICE_FEATURE_SET with a valid eventfd enables recovery for this
+ * open and requests notifications when each event starts and ends. SET
+ * with eventfd -1 disables recovery. flags and sequence must be zero. SET
+ * returns -EBUSY during recovery, after recovery failure, or while device
+ * access is blocked, and -ENODEV after close has started.
+ * VFIO_PCI_ERR_IRQ_INDEX notifications are unchanged.
+ *
+ * VFIO_DEVICE_FEATURE_GET returns the current state and eventfd = -1. GET
+ * is allowed during recovery, but may wait for a running callback.
+ *
+ * The sequence number increments for each event and resets to zero after
+ * a successful SET. It lets userspace detect multiple events from a single
+ * notification. Recovery may complete before userspace reads the
+ * notification, so userspace must check the sequence and status rather
+ * than wait to observe IN_PROGRESS.
+ *
+ * Ordinary hardware accesses are blocked during recovery. Read and write
+ * requests fail with -EIO, and faults on BAR mappings result in SIGBUS.
+ *
+ * CHANNEL_FROZEN records a frozen channel. DEVICE_RESET records a host
+ * reset, after which userspace must reconfigure interrupts with
+ * VFIO_DEVICE_SET_IRQS.
+ * FAILED indicates that recovery failed and access remains blocked until
+ * close and reopen. Status bits remain set until the next event or SET.
+ * ENABLED indicates that recovery is enabled.
+ *
+ * SET returns -EBUSY while a host recovery event recorded by the driver is
+ * still running, even if recovery was disabled when it was reported.
+ * Enabling recovery does not report events that have already completed.
+ *
+ * Open returns -EBUSY while a recorded host recovery is still running,
+ * including events reported while recovery was disabled.
+ */
+struct vfio_device_pci_error_recovery {
+	__u32 flags;
+#define VFIO_PCI_ERROR_RECOVERY_IN_PROGRESS	(1U << 0)
+#define VFIO_PCI_ERROR_RECOVERY_CHANNEL_FROZEN	(1U << 1)
+#define VFIO_PCI_ERROR_RECOVERY_DEVICE_RESET	(1U << 2)
+#define VFIO_PCI_ERROR_RECOVERY_FAILED		(1U << 3)
+#define VFIO_PCI_ERROR_RECOVERY_ENABLED		(1U << 4)
+	__s32 eventfd;
+	__aligned_u64 sequence;
+};
+
+#define VFIO_DEVICE_FEATURE_PCI_ERROR_RECOVERY 14
+
 /* -------- API for Type1 VFIO IOMMU -------- */
 
 /**
diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
index 5e9451e08263..0897d1f0e082 100644
--- a/drivers/vfio/pci/vfio_pci_core.c
+++ b/drivers/vfio/pci/vfio_pci_core.c
@@ -1043,6 +1043,10 @@ void vfio_pci_core_close_device(struct vfio_device *core_vdev)
 	mutex_lock(&vdev->igate);
 	vfio_pci_eventfd_replace_locked(vdev, &vdev->err_trigger, NULL);
 	vfio_pci_eventfd_replace_locked(vdev, &vdev->req_trigger, NULL);
+	if (vdev->pci_recovery_supported)
+		vfio_pci_eventfd_replace_locked(vdev,
+						&vdev->pci_recovery_trigger,
+						NULL);
 	mutex_unlock(&vdev->igate);
 }
 EXPORT_SYMBOL_GPL(vfio_pci_core_close_device);
@@ -1939,6 +1943,114 @@ static int vfio_pci_core_feature_token(struct vfio_pci_core_device *vdev,
 	return 0;
 }
 
+/* Consumes the eventfd reference on both success and failure. */
+static int vfio_pci_recovery_set(struct vfio_pci_core_device *vdev,
+				 struct eventfd_ctx *ctx, bool enable)
+{
+	int ret;
+
+	mutex_lock(&vdev->access_lock);
+	if (!vdev->device_open) {
+		ret = -ENODEV;
+		goto out_put;
+	}
+	if (rcu_access_pointer(vdev->access_ops) != &vfio_pci_live_ops ||
+	    vdev->pci_recovery_host_active) {
+		ret = -EBUSY;
+		goto out_put;
+	}
+	if (!enable &&
+	    (vdev->pci_recovery_flags &
+	     (VFIO_PCI_RECOVERY_IN_PROGRESS | VFIO_PCI_RECOVERY_FAILED))) {
+		ret = -EBUSY;
+		goto out_put;
+	}
+
+	mutex_lock(&vdev->igate);
+	ret = vfio_pci_eventfd_replace_locked(vdev, &vdev->pci_recovery_trigger,
+					      ctx);
+	mutex_unlock(&vdev->igate);
+	if (ret)
+		goto out_put;
+
+	WRITE_ONCE(vdev->pci_recovery_enabled, enable);
+	/*
+	 * Updating the recovery eventfd resets status for the current open.
+	 * It does not change the access ops or whether host recovery is running.
+	 */
+	WRITE_ONCE(vdev->pci_recovery_flags, 0);
+	vdev->pci_recovery_sequence = 0;
+	vdev->pci_recovery_command_valid = false;
+
+	goto out_unlock;
+
+out_put:
+	if (ctx)
+		eventfd_ctx_put(ctx);
+out_unlock:
+	mutex_unlock(&vdev->access_lock);
+	return ret;
+}
+
+static int
+vfio_pci_core_feature_error_recovery(struct vfio_pci_core_device *vdev, u32 flags,
+				     struct vfio_device_pci_error_recovery __user *arg,
+				     size_t argsz)
+{
+	struct vfio_device_pci_error_recovery state = { .eventfd = -1 };
+	struct eventfd_ctx *ctx = NULL;
+	bool enable;
+	int ret;
+
+	if (!vdev->pci_recovery_supported)
+		return -ENOTTY;
+
+	ret = vfio_check_feature(flags, argsz,
+				 VFIO_DEVICE_FEATURE_GET |
+				 VFIO_DEVICE_FEATURE_SET, sizeof(state));
+	if (ret != 1)
+		return ret;
+
+	if (flags & VFIO_DEVICE_FEATURE_GET) {
+		scoped_guard(mutex, &vdev->access_lock) {
+			u32 rflags = vdev->pci_recovery_flags;
+
+			if (vdev->pci_recovery_enabled)
+				state.flags |= VFIO_PCI_ERROR_RECOVERY_ENABLED;
+			if (rflags & VFIO_PCI_RECOVERY_IN_PROGRESS)
+				state.flags |=
+					VFIO_PCI_ERROR_RECOVERY_IN_PROGRESS;
+			if (rflags & VFIO_PCI_RECOVERY_FROZEN)
+				state.flags |=
+					VFIO_PCI_ERROR_RECOVERY_CHANNEL_FROZEN;
+			if (rflags & VFIO_PCI_RECOVERY_RESET)
+				state.flags |=
+					VFIO_PCI_ERROR_RECOVERY_DEVICE_RESET;
+			if (rflags & VFIO_PCI_RECOVERY_FAILED)
+				state.flags |= VFIO_PCI_ERROR_RECOVERY_FAILED;
+			state.sequence = vdev->pci_recovery_sequence;
+		}
+
+		if (copy_to_user(arg, &state, sizeof(state)))
+			return -EFAULT;
+		return 0;
+	}
+
+	if (copy_from_user(&state, arg, sizeof(state)))
+		return -EFAULT;
+	if (state.flags || state.sequence || state.eventfd < -1)
+		return -EINVAL;
+
+	enable = state.eventfd >= 0;
+	if (enable) {
+		ctx = eventfd_ctx_fdget(state.eventfd);
+		if (IS_ERR(ctx))
+			return PTR_ERR(ctx);
+	}
+
+	return vfio_pci_recovery_set(vdev, ctx, enable);
+}
+
 int vfio_pci_core_ioctl_feature(struct vfio_device *device, u32 flags,
 				void __user *arg, size_t argsz)
 {
@@ -1959,6 +2071,9 @@ int vfio_pci_core_ioctl_feature(struct vfio_device *device, u32 flags,
 		return vfio_pci_core_feature_dma_buf(vdev, flags, arg, argsz);
 	case VFIO_DEVICE_FEATURE_ZPCI_ERROR:
 		return vfio_pci_zdev_feature_err(device, flags, arg, argsz);
+
+	case VFIO_DEVICE_FEATURE_PCI_ERROR_RECOVERY:
+		return vfio_pci_core_feature_error_recovery(vdev, flags, arg, argsz);
 	default:
 		return -ENOTTY;
 	}
@@ -2685,6 +2800,20 @@ void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev)
 }
 EXPORT_SYMBOL_GPL(vfio_pci_core_unregister_device);
 
+static void
+vfio_pci_signal_recovery_event(struct vfio_pci_core_device *vdev)
+{
+	struct vfio_pci_eventfd *eventfd;
+
+	lockdep_assert_held(&vdev->access_lock);
+
+	rcu_read_lock();
+	eventfd = rcu_dereference(vdev->pci_recovery_trigger);
+	if (eventfd)
+		eventfd_signal(eventfd->ctx);
+	rcu_read_unlock();
+}
+
 /*
  * Save PCI_COMMAND, clear bus mastering and set the requested bits.
  * Reject an all-ones read from an inaccessible device.
@@ -2717,6 +2846,7 @@ pci_ers_result_t vfio_pci_core_aer_err_detected(struct vfio_pci_core_device *vde
 	struct vfio_pci_eventfd *eventfd;
 	pci_ers_result_t result = PCI_ERS_RESULT_CAN_RECOVER;
 	unsigned long irq_flags;
+	bool notify_recovery = false;
 	bool nested;
 	u32 flags;
 	int ret = 0;
@@ -2745,6 +2875,7 @@ pci_ers_result_t vfio_pci_core_aer_err_detected(struct vfio_pci_core_device *vde
 		goto out_unlock;
 	}
 
+	notify_recovery = true;
 	vfio_pci_core_block_access(vdev);
 	/*
 	 * Keep the original command register for nested events. Recovery may
@@ -2811,6 +2942,8 @@ pci_ers_result_t vfio_pci_core_aer_err_detected(struct vfio_pci_core_device *vde
 out_memory:
 	up_write(&vdev->memory_lock);
 out_unlock:
+	if (notify_recovery)
+		vfio_pci_signal_recovery_event(vdev);
 	mutex_unlock(&vdev->access_lock);
 
 out:
@@ -2900,6 +3033,8 @@ pci_ers_result_t vfio_pci_core_aer_slot_reset(struct vfio_pci_core_device *vdev)
 		   (vdev->pci_recovery_flags | VFIO_PCI_RECOVERY_FAILED) &
 		   ~VFIO_PCI_RECOVERY_IN_PROGRESS);
 	vdev->pci_recovery_command_valid = false;
+	/* Report failure now. resume() skips recovery that has already finished. */
+	vfio_pci_signal_recovery_event(vdev);
 	mutex_unlock(&vdev->access_lock);
 
 	/* Allow recovery of other devices under the bridge to continue. */
@@ -2911,6 +3046,7 @@ void vfio_pci_core_aer_resume(struct vfio_pci_core_device *vdev)
 {
 	struct pci_dev *pdev = vdev->pdev;
 	unsigned long irq_flags;
+	bool notify_recovery = false;
 	bool notify_intx = false;
 	bool power_up;
 	u32 flags;
@@ -2921,6 +3057,7 @@ void vfio_pci_core_aer_resume(struct vfio_pci_core_device *vdev)
 	if (!(vdev->pci_recovery_flags & VFIO_PCI_RECOVERY_IN_PROGRESS))
 		goto out_unlock;
 
+	notify_recovery = true;
 	/*
 	 * Match IRQ configuration's lock order and keep D0 requests out until
 	 * live ops are installed, so no deferred request can be left behind.
@@ -2983,6 +3120,8 @@ void vfio_pci_core_aer_resume(struct vfio_pci_core_device *vdev)
 	mutex_unlock(&vdev->igate);
 
 out_unlock:
+	if (notify_recovery)
+		vfio_pci_signal_recovery_event(vdev);
 	mutex_unlock(&vdev->access_lock);
 }
 EXPORT_SYMBOL_GPL(vfio_pci_core_aer_resume);
-- 
2.43.0