[RFC PATCH v3 17/20] vfio/pci: Complete recovery in resume()

[Shameer Kolothum <skolothumtho@nvidia.com>]

Add resume() to finish recovery and allow device access again. This also
handles recovery that did not require a reset.

Complete deferred D0 requests and restore PCI_COMMAND, DMA-BUF mappings
and INTx state before installing live ops. Take igate before
memory_lock, matching IRQ configuration, and hold both through
restoration and live-ops installation so no deferred D0 request is
missed.

Use the locked PCI power helper because native PCI recovery already
holds pci_bus_sem for reading. Leave failed devices blocked.

Signed-off-by: Shameer Kolothum <skolothumtho@nvidia.com>
---
 include/linux/vfio_pci_core.h    |  3 ++
 drivers/vfio/pci/vfio_pci.c      |  8 +++
 drivers/vfio/pci/vfio_pci_core.c | 87 ++++++++++++++++++++++++++++++++
 3 files changed, 98 insertions(+)

diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
index d36bca65d180..a7a87c04ee60 100644
--- a/include/linux/vfio_pci_core.h
+++ b/include/linux/vfio_pci_core.h
@@ -147,6 +147,8 @@ struct vfio_pci_core_device {
 	bool			sriov_active;
 	struct pci_saved_state	*pci_saved_state;
 	struct pci_saved_state	*pm_save;
+	/* Deferred D0 request, protected by memory_lock. */
+	bool			power_up_pending;
 	int			ioeventfds_nr;
 	struct vfio_pci_eventfd __rcu *err_trigger;
 	struct vfio_pci_eventfd __rcu *req_trigger;
@@ -289,6 +291,7 @@ int vfio_pci_core_runtime_suspend(struct vfio_pci_core_device *vdev);
 int vfio_pci_core_runtime_resume(struct vfio_pci_core_device *vdev);
 pci_ers_result_t vfio_pci_core_aer_err_detected(struct vfio_pci_core_device *vdev);
 pci_ers_result_t vfio_pci_core_aer_slot_reset(struct vfio_pci_core_device *vdev);
+void vfio_pci_core_aer_resume(struct vfio_pci_core_device *vdev);
 
 /*
  * Per-driver PM/AER trampolines. driver_data points to the driver's private
diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c
index b4b3351cbd85..13f4f1faf90d 100644
--- a/drivers/vfio/pci/vfio_pci.c
+++ b/drivers/vfio/pci/vfio_pci.c
@@ -257,6 +257,13 @@ static pci_ers_result_t vfio_pci_aer_slot_reset(struct pci_dev *pdev)
 	return vfio_pci_core_aer_slot_reset(vdev);
 }
 
+static void vfio_pci_aer_resume(struct pci_dev *pdev)
+{
+	struct vfio_pci_core_device *vdev = pci_get_drvdata(pdev);
+
+	vfio_pci_core_aer_resume(vdev);
+}
+
 static const struct pci_error_handlers vfio_pci_legacy_err_handlers = {
 	.error_detected = vfio_pci_aer_err_detected,
 };
@@ -264,6 +271,7 @@ static const struct pci_error_handlers vfio_pci_legacy_err_handlers = {
 static const struct pci_error_handlers vfio_pci_err_handlers = {
 	.error_detected = vfio_pci_aer_err_detected,
 	.slot_reset = vfio_pci_aer_slot_reset,
+	.resume = vfio_pci_aer_resume,
 };
 
 static struct pci_driver vfio_pci_driver = {
diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
index 039fd864119b..09923a48a547 100644
--- a/drivers/vfio/pci/vfio_pci_core.c
+++ b/drivers/vfio/pci/vfio_pci_core.c
@@ -515,6 +515,13 @@ int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, pci_power_t stat
 	return __vfio_pci_set_power_state(vdev, state, pci_set_power_state);
 }
 
+static int
+vfio_pci_set_power_state_locked(struct vfio_pci_core_device *vdev,
+				pci_power_t state)
+{
+	return __vfio_pci_set_power_state(vdev, state, pci_set_power_state_locked);
+}
+
 /* Avoid waiting for pci_bus_sem under locks needed by PCI recovery. */
 int vfio_pci_try_set_power_state(struct vfio_pci_core_device *vdev,
 				 pci_power_t state)
@@ -2750,6 +2757,86 @@ pci_ers_result_t vfio_pci_core_aer_slot_reset(struct vfio_pci_core_device *vdev)
 }
 EXPORT_SYMBOL_GPL(vfio_pci_core_aer_slot_reset);
 
+void vfio_pci_core_aer_resume(struct vfio_pci_core_device *vdev)
+{
+	struct pci_dev *pdev = vdev->pdev;
+	unsigned long irq_flags;
+	bool notify_intx = false;
+	bool power_up;
+	u32 flags;
+	int ret = 0;
+
+	mutex_lock(&vdev->access_lock);
+	vdev->pci_recovery_host_active = false;
+	if (!(vdev->pci_recovery_flags & VFIO_PCI_RECOVERY_IN_PROGRESS))
+		goto out_unlock;
+
+	/*
+	 * Match IRQ configuration's lock order and keep D0 requests out until
+	 * live ops are installed, so no deferred request can be left behind.
+	 */
+	mutex_lock(&vdev->igate);
+	down_write(&vdev->memory_lock);
+	/*
+	 * Complete a deferred D0 request before restoring PCI_COMMAND.
+	 * After a reset, slot_reset() has already put the device in D0.
+	 */
+	power_up = vdev->power_up_pending;
+	vdev->power_up_pending = false;
+	if (power_up && !(vdev->pci_recovery_flags & VFIO_PCI_RECOVERY_RESET)) {
+		ret = vfio_pci_set_power_state_locked(vdev, PCI_D0);
+		if (ret)
+			goto out_memory;
+	}
+
+	/*
+	 * Hold irqlock while updating PCI_COMMAND, as the INTx handler does.
+	 * Keep masked INTx disabled until vfio_pci_intx_recovery_finish().
+	 */
+	spin_lock_irqsave(&vdev->irqlock, irq_flags);
+	if (!(vdev->pci_recovery_flags & VFIO_PCI_RECOVERY_RESET) &&
+	    vdev->pci_recovery_command_valid) {
+		u16 cmd = vdev->pci_recovery_command;
+
+		cmd = vfio_pci_intx_recovery_update_command(vdev, cmd);
+		ret = pci_write_config_word(pdev, PCI_COMMAND, cmd);
+	}
+	spin_unlock_irqrestore(&vdev->irqlock, irq_flags);
+	if (ret)
+		goto out_memory;
+
+	/*
+	 * Restoring saved state may leave ROM decoding enabled. Disable it
+	 * before allowing access to shared decoders.
+	 */
+	if (pci_resource_start(pdev, PCI_ROM_RESOURCE) &&
+	    !(pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_ENABLE))
+		pci_disable_rom(pdev);
+
+	/* Recovery restores mappings while ordinary access still uses stubs. */
+	if (__vfio_pci_memory_enabled(vdev))
+		vfio_pci_dma_buf_restore(vdev);
+
+out_memory:
+	vdev->pci_recovery_command_valid = false;
+	flags = vdev->pci_recovery_flags & ~VFIO_PCI_RECOVERY_IN_PROGRESS;
+	if (ret) {
+		WRITE_ONCE(vdev->pci_recovery_flags,
+			   flags | VFIO_PCI_RECOVERY_FAILED);
+	} else {
+		WRITE_ONCE(vdev->pci_recovery_flags, flags);
+		notify_intx = vfio_pci_intx_recovery_finish(vdev);
+	}
+	up_write(&vdev->memory_lock);
+	if (notify_intx)
+		vfio_pci_intx_recovery_signal(vdev);
+	mutex_unlock(&vdev->igate);
+
+out_unlock:
+	mutex_unlock(&vdev->access_lock);
+}
+EXPORT_SYMBOL_GPL(vfio_pci_core_aer_resume);
+
 static int vfio_pci_sriov_configure_live(struct vfio_pci_core_device *vdev,
 					 int nr_virtfn)
 {
-- 
2.43.0