[RFC PATCH v3 14/20] vfio/pci: Add INTx recovery operations

[Shameer Kolothum <skolothumtho@nvidia.com>]

Physical INTx interrupts and eventfd unmask requests can arrive while
device access is blocked for recovery. Add blocked operations that
remember unmask requests and keep masking pending interrupts on shared
lines. An explicit mask cancels a pending unmask request.

Select and call INTx operations under irqlock, and hold the same lock
when switching access ops. Process pending unmask requests and publish
live ops under irqlock so requests cannot be lost between these steps.
Keep closed and failed devices blocked.

Provide helpers to mask INTx at recovery entry and preserve its mask
when restoring PCI_COMMAND. The completion caller holds igate and
memory_lock through restoration, then sends any resulting INTx
notification after releasing memory_lock and irqlock.

Signed-off-by: Shameer Kolothum <skolothumtho@nvidia.com>
---
 drivers/vfio/pci/vfio_pci_priv.h  |  22 ++++
 include/linux/vfio_pci_core.h     |   2 +-
 drivers/vfio/pci/vfio_pci_core.c  |  23 ++++
 drivers/vfio/pci/vfio_pci_intrs.c | 183 ++++++++++++++++++++++++++----
 4 files changed, 209 insertions(+), 21 deletions(-)

diff --git a/drivers/vfio/pci/vfio_pci_priv.h b/drivers/vfio/pci/vfio_pci_priv.h
index b02fd1fbcd31..c195ed3e4d9e 100644
--- a/drivers/vfio/pci/vfio_pci_priv.h
+++ b/drivers/vfio/pci/vfio_pci_priv.h
@@ -3,6 +3,7 @@
 #define VFIO_PCI_PRIV_H
 
 #include <linux/vfio_pci_core.h>
+#include <linux/irqreturn.h>
 
 /* Special capability IDs predefined access */
 #define PCI_CAP_ID_INVALID		0xFF	/* default raw access */
@@ -25,6 +26,11 @@ struct vfio_pci_ioeventfd {
 
 bool vfio_pci_intx_mask(struct vfio_pci_core_device *vdev);
 void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev);
+void vfio_pci_intx_recovery_start(struct vfio_pci_core_device *vdev);
+bool vfio_pci_intx_recovery_finish(struct vfio_pci_core_device *vdev);
+void vfio_pci_intx_recovery_signal(struct vfio_pci_core_device *vdev);
+u16 vfio_pci_intx_recovery_update_command(struct vfio_pci_core_device *vdev,
+					  u16 command);
 
 int vfio_pci_eventfd_replace_locked(struct vfio_pci_core_device *vdev,
 				    struct vfio_pci_eventfd __rcu **peventfd,
@@ -84,6 +90,8 @@ typedef long (*vfio_pci_access_fn)(struct vfio_pci_core_device *vdev,
 long vfio_pci_core_access(struct vfio_pci_core_device *vdev,
 			  vfio_pci_access_fn fn, void *data);
 
+struct vfio_pci_irq_ctx;
+
 struct vfio_pci_access_ops {
 	/* Select and call within SRCU. The callback receives kernel buffers. */
 	long (*execute)(struct vfio_pci_core_device *vdev,
@@ -92,10 +100,24 @@ struct vfio_pci_access_ops {
 	int (*set_power_state)(struct vfio_pci_core_device *vdev, pci_power_t state);
 	/* Caller holds memory_lock. */
 	void (*restore_dmabufs)(struct vfio_pci_core_device *vdev);
+	/* Select and call both INTx methods while holding irqlock. */
+	irqreturn_t (*intx_handler)(struct vfio_pci_core_device *vdev,
+				    struct vfio_pci_irq_ctx *ctx);
+	int (*intx_unmask)(struct vfio_pci_core_device *vdev,
+			   struct vfio_pci_irq_ctx *ctx);
 };
 
 const struct vfio_pci_access_ops *
 vfio_pci_access_ops(struct vfio_pci_core_device *vdev);
+void vfio_pci_restore_access(struct vfio_pci_core_device *vdev);
+irqreturn_t vfio_pci_intx_handle(struct vfio_pci_core_device *vdev,
+				 struct vfio_pci_irq_ctx *ctx);
+irqreturn_t vfio_pci_intx_handle_blocked(struct vfio_pci_core_device *vdev,
+					 struct vfio_pci_irq_ctx *ctx);
+int vfio_pci_intx_unmask_live(struct vfio_pci_core_device *vdev,
+			      struct vfio_pci_irq_ctx *ctx);
+int vfio_pci_intx_unmask_blocked(struct vfio_pci_core_device *vdev,
+				 struct vfio_pci_irq_ctx *ctx);
 
 #ifdef CONFIG_VFIO_PCI_IGD
 bool vfio_pci_is_intel_display(struct pci_dev *pdev);
diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
index f2b64420cb07..fadbbaa58770 100644
--- a/include/linux/vfio_pci_core.h
+++ b/include/linux/vfio_pci_core.h
@@ -160,7 +160,7 @@ struct vfio_pci_core_device {
 	struct notifier_block	nb;
 	struct rw_semaphore	memory_lock;
 	struct srcu_struct	access_srcu;
-	/* Updated under access_lock. */
+	/* Updated under access_lock and irqlock. */
 	const struct vfio_pci_access_ops __rcu *access_ops;
 	/* Set after open completes, cleared before close tears down state. */
 	bool			device_open;
diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
index cdc6bafe9861..89cebe2522ba 100644
--- a/drivers/vfio/pci/vfio_pci_core.c
+++ b/drivers/vfio/pci/vfio_pci_core.c
@@ -338,12 +338,16 @@ static const struct vfio_pci_access_ops vfio_pci_live_ops = {
 	.execute = vfio_pci_access_live,
 	.set_power_state = vfio_pci_try_set_power_state,
 	.restore_dmabufs = vfio_pci_dma_buf_restore,
+	.intx_handler = vfio_pci_intx_handle,
+	.intx_unmask = vfio_pci_intx_unmask_live,
 };
 
 static const struct vfio_pci_access_ops vfio_pci_blocked_ops = {
 	.execute = vfio_pci_access_blocked,
 	.set_power_state = vfio_pci_power_blocked,
 	.restore_dmabufs = vfio_pci_restore_dmabufs_blocked,
+	.intx_handler = vfio_pci_intx_handle_blocked,
+	.intx_unmask = vfio_pci_intx_unmask_blocked,
 };
 
 /*
@@ -362,12 +366,27 @@ vfio_pci_access_ops(struct vfio_pci_core_device *vdev)
 /* Block new operations and drain ordinary accesses before teardown or recovery. */
 static void vfio_pci_core_block_access(struct vfio_pci_core_device *vdev)
 {
+	unsigned long flags;
+
 	lockdep_assert_held(&vdev->access_lock);
+	spin_lock_irqsave(&vdev->irqlock, flags);
 	rcu_assign_pointer(vdev->access_ops, &vfio_pci_blocked_ops);
+	spin_unlock_irqrestore(&vdev->irqlock, flags);
 
 	synchronize_srcu(&vdev->access_srcu);
 }
 
+/* Caller finishes internal restoration before allowing ordinary access. */
+void vfio_pci_restore_access(struct vfio_pci_core_device *vdev)
+{
+	lockdep_assert_held(&vdev->access_lock);
+	lockdep_assert_held(&vdev->irqlock);
+	if (vdev->device_open &&
+	    !(vdev->pci_recovery_flags &
+	      (VFIO_PCI_RECOVERY_IN_PROGRESS | VFIO_PCI_RECOVERY_FAILED)))
+		rcu_assign_pointer(vdev->access_ops, &vfio_pci_live_ops);
+}
+
 long vfio_pci_core_access(struct vfio_pci_core_device *vdev,
 			  vfio_pci_access_fn fn, void *data)
 {
@@ -965,13 +984,17 @@ void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev)
 	}
 
 	if (vdev->pci_recovery_supported) {
+		unsigned long flags;
+
 		guard(mutex)(&vdev->access_lock);
 		WRITE_ONCE(vdev->pci_recovery_flags, 0);
 		vdev->pci_recovery_sequence = 0;
 		vdev->pci_recovery_command_valid = false;
 		WRITE_ONCE(vdev->pci_recovery_enabled, false);
 		WRITE_ONCE(vdev->device_open, true);
+		spin_lock_irqsave(&vdev->irqlock, flags);
 		rcu_assign_pointer(vdev->access_ops, &vfio_pci_live_ops);
+		spin_unlock_irqrestore(&vdev->irqlock, flags);
 	}
 }
 EXPORT_SYMBOL_GPL(vfio_pci_core_finish_enable);
diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c
index 64f80f64ff57..f78c7217017d 100644
--- a/drivers/vfio/pci/vfio_pci_intrs.c
+++ b/drivers/vfio/pci/vfio_pci_intrs.c
@@ -29,6 +29,8 @@ struct vfio_pci_irq_ctx {
 	struct virqfd			*mask;
 	char				*name;
 	bool				masked;
+	bool				recovery_masked;
+	bool				unmask_pending;
 	struct irq_bypass_producer	producer;
 };
 
@@ -126,6 +128,10 @@ static bool __vfio_pci_intx_mask(struct vfio_pci_core_device *vdev)
 	if (WARN_ON_ONCE(!ctx))
 		goto out_unlock;
 
+	/* An explicit mask cancels any unmask request saved during recovery. */
+	ctx->recovery_masked = false;
+	ctx->unmask_pending = false;
+
 	if (!ctx->masked) {
 		/*
 		 * Can't use check_and_mask here because we always want to
@@ -162,16 +168,13 @@ bool vfio_pci_intx_mask(struct vfio_pci_core_device *vdev)
  * a signal is necessary, which can then be handled via a work queue
  * or directly depending on the caller.
  */
-static int vfio_pci_intx_unmask_handler(void *opaque, void *data)
+int vfio_pci_intx_unmask_live(struct vfio_pci_core_device *vdev,
+			      struct vfio_pci_irq_ctx *ctx)
 {
-	struct vfio_pci_core_device *vdev = opaque;
 	struct pci_dev *pdev = vdev->pdev;
-	struct vfio_pci_irq_ctx *ctx = data;
-	unsigned long flags;
 	int ret = 0;
 
-	spin_lock_irqsave(&vdev->irqlock, flags);
-
+	lockdep_assert_held(&vdev->irqlock);
 	/*
 	 * Unmasking comes from ioctl or config, so again, have the
 	 * physical bit follow the virtual even when not using INTx.
@@ -179,9 +182,12 @@ static int vfio_pci_intx_unmask_handler(void *opaque, void *data)
 	if (unlikely(!is_intx(vdev))) {
 		if (vdev->pci_2_3)
 			pci_intx(pdev, 1);
-		goto out_unlock;
+		goto out;
 	}
 
+	ctx->unmask_pending = false;
+	ctx->recovery_masked = false;
+
 	if (ctx->masked && !vdev->virq_disabled) {
 		/*
 		 * A pending interrupt here would immediately trigger,
@@ -197,9 +203,28 @@ static int vfio_pci_intx_unmask_handler(void *opaque, void *data)
 		ctx->masked = (ret > 0);
 	}
 
-out_unlock:
-	spin_unlock_irqrestore(&vdev->irqlock, flags);
+out:
+	return ret;
+}
 
+int vfio_pci_intx_unmask_blocked(struct vfio_pci_core_device *vdev,
+				 struct vfio_pci_irq_ctx *ctx)
+{
+	lockdep_assert_held(&vdev->irqlock);
+	if (is_intx(vdev))
+		ctx->unmask_pending = true;
+	return 0;
+}
+
+static int vfio_pci_intx_unmask_handler(void *opaque, void *data)
+{
+	struct vfio_pci_core_device *vdev = opaque;
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&vdev->irqlock, flags);
+	ret = vfio_pci_access_ops(vdev)->intx_unmask(vdev, data);
+	spin_unlock_irqrestore(&vdev->irqlock, flags);
 	return ret;
 }
 
@@ -220,33 +245,151 @@ void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev)
 	mutex_unlock(&vdev->igate);
 }
 
-static irqreturn_t vfio_intx_handler(int irq, void *dev_id)
+/*
+ * Recovery always masks INTx and remembers whether to unmask it afterward.
+ * The interrupt handler masks INTx only if this device has a pending
+ * interrupt, since another device may share the line.
+ */
+static bool vfio_pci_intx_mask_for_recovery(struct vfio_pci_core_device *vdev,
+					    struct vfio_pci_irq_ctx *ctx,
+					    bool quiesce)
 {
-	struct vfio_pci_irq_ctx *ctx = dev_id;
-	struct vfio_pci_core_device *vdev = ctx->vdev;
-	unsigned long flags;
-	int ret = IRQ_NONE;
+	lockdep_assert_held(&vdev->irqlock);
 
-	spin_lock_irqsave(&vdev->irqlock, flags);
+	if (ctx->masked)
+		return false;
 
+	if (!vdev->pci_2_3)
+		disable_irq_nosync(vdev->pdev->irq);
+	else if (quiesce)
+		pci_intx(vdev->pdev, 0);
+	else if (!pci_check_and_mask_intx(vdev->pdev))
+		return false;
+
+	ctx->masked = true;
+	if (quiesce)
+		ctx->recovery_masked = true;
+	return true;
+}
+
+irqreturn_t vfio_pci_intx_handle(struct vfio_pci_core_device *vdev,
+				 struct vfio_pci_irq_ctx *ctx)
+{
+	lockdep_assert_held(&vdev->irqlock);
 	if (!vdev->pci_2_3) {
 		disable_irq_nosync(vdev->pdev->irq);
 		ctx->masked = true;
-		ret = IRQ_HANDLED;
-	} else if (!ctx->masked &&  /* may be shared */
-		   pci_check_and_mask_intx(vdev->pdev)) {
+		return IRQ_HANDLED;
+	}
+	if (!ctx->masked && pci_check_and_mask_intx(vdev->pdev)) {
 		ctx->masked = true;
-		ret = IRQ_HANDLED;
+		return IRQ_HANDLED;
 	}
+	return IRQ_NONE;
+}
+
+irqreturn_t vfio_pci_intx_handle_blocked(struct vfio_pci_core_device *vdev,
+					 struct vfio_pci_irq_ctx *ctx)
+{
+	lockdep_assert_held(&vdev->irqlock);
+	/* A shared interrupt line still needs pending INTx to be masked. */
+	if (vfio_pci_intx_mask_for_recovery(vdev, ctx, false) ||
+	    (ctx->masked && !vdev->pci_2_3))
+		return IRQ_HANDLED;
+	return IRQ_NONE;
+}
+
+static irqreturn_t vfio_intx_handler(int irq, void *dev_id)
+{
+	struct vfio_pci_irq_ctx *ctx = dev_id;
+	struct vfio_pci_core_device *vdev = ctx->vdev;
+	unsigned long flags;
+	irqreturn_t ret;
 
+	spin_lock_irqsave(&vdev->irqlock, flags);
+	ret = vfio_pci_access_ops(vdev)->intx_handler(vdev, ctx);
 	spin_unlock_irqrestore(&vdev->irqlock, flags);
 
 	if (ret == IRQ_HANDLED)
 		vfio_send_intx_eventfd(vdev, ctx);
-
 	return ret;
 }
 
+void vfio_pci_intx_recovery_start(struct vfio_pci_core_device *vdev)
+{
+	struct vfio_pci_irq_ctx *ctx;
+	unsigned long flags;
+
+	lockdep_assert_held(&vdev->access_lock);
+
+	spin_lock_irqsave(&vdev->irqlock, flags);
+	if (!is_intx(vdev))
+		goto out_unlock;
+
+	ctx = vfio_irq_ctx_get(vdev, 0);
+	if (WARN_ON_ONCE(!ctx))
+		goto out_unlock;
+
+	vfio_pci_intx_mask_for_recovery(vdev, ctx, true);
+
+out_unlock:
+	spin_unlock_irqrestore(&vdev->irqlock, flags);
+}
+
+/* Preserve the current INTx mask when restoring PCI_COMMAND. */
+u16 vfio_pci_intx_recovery_update_command(struct vfio_pci_core_device *vdev,
+					  u16 command)
+{
+	struct vfio_pci_irq_ctx *ctx;
+
+	lockdep_assert_held(&vdev->irqlock);
+
+	if (!vdev->pci_2_3 || !is_intx(vdev))
+		return command;
+
+	ctx = vfio_irq_ctx_get(vdev, 0);
+	if (ctx && ctx->masked)
+		command |= PCI_COMMAND_INTX_DISABLE;
+
+	return command;
+}
+
+/*
+ * Restore INTx and allow ordinary access under the same lock. An unmask
+ * request must either be replayed here or reach the live op afterward.
+ */
+bool vfio_pci_intx_recovery_finish(struct vfio_pci_core_device *vdev)
+{
+	struct vfio_pci_irq_ctx *ctx = NULL;
+	unsigned long flags;
+	int ret = 0;
+
+	lockdep_assert_held(&vdev->access_lock);
+	lockdep_assert_held(&vdev->igate);
+	lockdep_assert_held_write(&vdev->memory_lock);
+	spin_lock_irqsave(&vdev->irqlock, flags);
+	if (!is_intx(vdev))
+		goto out_restore;
+
+	ctx = vfio_irq_ctx_get(vdev, 0);
+	if (WARN_ON_ONCE(!ctx))
+		goto out_restore;
+
+	if (ctx->recovery_masked || ctx->unmask_pending)
+		ret = vfio_pci_intx_unmask_live(vdev, ctx);
+
+out_restore:
+	vfio_pci_restore_access(vdev);
+	spin_unlock_irqrestore(&vdev->irqlock, flags);
+	return ret > 0;
+}
+
+void vfio_pci_intx_recovery_signal(struct vfio_pci_core_device *vdev)
+{
+	lockdep_assert_held(&vdev->igate);
+	vfio_send_intx_eventfd(vdev, vfio_irq_ctx_get(vdev, 0));
+}
+
 static int vfio_intx_enable(struct vfio_pci_core_device *vdev,
 			    struct eventfd_ctx *trigger)
 {
-- 
2.43.0