[PATCH 5/6] vfio/mlx5: Use .migration_reset_state() callback

["Michał Winiarski" <michal.winiarski@intel.com>]

Move the migration device state reset code from .reset_done() to
dedicated callback.
Remove the deferred reset mechanism, as it's no longer needed.

Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
---
 drivers/vfio/pci/mlx5/cmd.c  | 15 +++++----
 drivers/vfio/pci/mlx5/cmd.h  |  3 --
 drivers/vfio/pci/mlx5/main.c | 59 +++++++++---------------------------
 3 files changed, 22 insertions(+), 55 deletions(-)

diff --git a/drivers/vfio/pci/mlx5/cmd.c b/drivers/vfio/pci/mlx5/cmd.c
index a92b095b90f6a..de6d786ce7ed1 100644
--- a/drivers/vfio/pci/mlx5/cmd.c
+++ b/drivers/vfio/pci/mlx5/cmd.c
@@ -178,13 +178,13 @@ static int mlx5fv_vf_event(struct notifier_block *nb,
 	case MLX5_PF_NOTIFY_ENABLE_VF:
 		mutex_lock(&mvdev->state_mutex);
 		mvdev->mdev_detach = false;
-		mlx5vf_state_mutex_unlock(mvdev);
+		mutex_unlock(&mvdev->state_mutex);
 		break;
 	case MLX5_PF_NOTIFY_DISABLE_VF:
 		mlx5vf_cmd_close_migratable(mvdev);
 		mutex_lock(&mvdev->state_mutex);
 		mvdev->mdev_detach = true;
-		mlx5vf_state_mutex_unlock(mvdev);
+		mutex_unlock(&mvdev->state_mutex);
 		break;
 	default:
 		break;
@@ -203,7 +203,7 @@ void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev)
 	mutex_lock(&mvdev->state_mutex);
 	mlx5vf_disable_fds(mvdev, NULL);
 	_mlx5vf_free_page_tracker_resources(mvdev);
-	mlx5vf_state_mutex_unlock(mvdev);
+	mutex_unlock(&mvdev->state_mutex);
 }
 
 void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev)
@@ -254,7 +254,6 @@ void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev,
 		goto end;
 
 	mutex_init(&mvdev->state_mutex);
-	spin_lock_init(&mvdev->reset_lock);
 	mvdev->nb.notifier_call = mlx5fv_vf_event;
 	ret = mlx5_sriov_blocking_notifier_register(mvdev->mdev, mvdev->vf_id,
 						    &mvdev->nb);
@@ -1487,7 +1486,7 @@ int mlx5vf_stop_page_tracker(struct vfio_device *vdev)
 	_mlx5vf_free_page_tracker_resources(mvdev);
 	mvdev->log_active = false;
 end:
-	mlx5vf_state_mutex_unlock(mvdev);
+	mutex_unlock(&mvdev->state_mutex);
 	return 0;
 }
 
@@ -1589,7 +1588,7 @@ int mlx5vf_start_page_tracker(struct vfio_device *vdev,
 	mlx5_eq_notifier_register(mdev, &tracker->nb);
 	*page_size = host_qp->tracked_page_size;
 	mvdev->log_active = true;
-	mlx5vf_state_mutex_unlock(mvdev);
+	mutex_unlock(&mvdev->state_mutex);
 	return 0;
 
 err_activate:
@@ -1605,7 +1604,7 @@ int mlx5vf_start_page_tracker(struct vfio_device *vdev,
 err_uar:
 	mlx5_put_uars_page(mdev, tracker->uar);
 end:
-	mlx5vf_state_mutex_unlock(mvdev);
+	mutex_unlock(&mvdev->state_mutex);
 	return err;
 }
 
@@ -1787,6 +1786,6 @@ int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova,
 	if (tracker->is_err)
 		err = -EIO;
 end:
-	mlx5vf_state_mutex_unlock(mvdev);
+	mutex_unlock(&mvdev->state_mutex);
 	return err;
 }
diff --git a/drivers/vfio/pci/mlx5/cmd.h b/drivers/vfio/pci/mlx5/cmd.h
index d7821b5ca7729..e36df1052cf5e 100644
--- a/drivers/vfio/pci/mlx5/cmd.h
+++ b/drivers/vfio/pci/mlx5/cmd.h
@@ -170,7 +170,6 @@ struct mlx5vf_pci_core_device {
 	int vf_id;
 	u16 vhca_id;
 	u8 migrate_cap:1;
-	u8 deferred_reset:1;
 	u8 mdev_detach:1;
 	u8 log_active:1;
 	u8 chunk_mode:1;
@@ -178,8 +177,6 @@ struct mlx5vf_pci_core_device {
 	/* protect migration state */
 	struct mutex state_mutex;
 	enum vfio_device_mig_state mig_state;
-	/* protect the reset_done flow */
-	spinlock_t reset_lock;
 	struct mlx5_vf_migration_file *resuming_migf;
 	struct mlx5_vf_migration_file *saving_migf;
 	struct mlx5_vhca_page_tracker tracker;
diff --git a/drivers/vfio/pci/mlx5/main.c b/drivers/vfio/pci/mlx5/main.c
index 7ec47e736a8e5..ddc6fa346f37c 100644
--- a/drivers/vfio/pci/mlx5/main.c
+++ b/drivers/vfio/pci/mlx5/main.c
@@ -325,7 +325,7 @@ static void mlx5vf_mig_file_save_work(struct work_struct *_work)
 err:
 	mlx5vf_mark_err(migf);
 end:
-	mlx5vf_state_mutex_unlock(mvdev);
+	mutex_unlock(&mvdev->state_mutex);
 	fput(migf->filp);
 }
 
@@ -544,7 +544,7 @@ static long mlx5vf_precopy_ioctl(struct file *filp, unsigned int cmd,
 	}
 
 done:
-	mlx5vf_state_mutex_unlock(mvdev);
+	mutex_unlock(&mvdev->state_mutex);
 	if (copy_to_user((void __user *)arg, &info, minsz))
 		return -EFAULT;
 	return 0;
@@ -552,7 +552,7 @@ static long mlx5vf_precopy_ioctl(struct file *filp, unsigned int cmd,
 err_migf_unlock:
 	mutex_unlock(&migf->lock);
 err_state_unlock:
-	mlx5vf_state_mutex_unlock(mvdev);
+	mutex_unlock(&mvdev->state_mutex);
 	return ret;
 }
 
@@ -972,7 +972,7 @@ static ssize_t mlx5vf_resume_write(struct file *filp, const char __user *buf,
 	if (ret)
 		migf->state = MLX5_MIGF_STATE_ERROR;
 	mutex_unlock(&migf->lock);
-	mlx5vf_state_mutex_unlock(migf->mvdev);
+	mutex_unlock(&migf->mvdev->state_mutex);
 	return ret ? ret : done;
 }
 
@@ -1191,25 +1191,6 @@ mlx5vf_pci_step_device_state_locked(struct mlx5vf_pci_core_device *mvdev,
 	return ERR_PTR(-EINVAL);
 }
 
-/*
- * This function is called in all state_mutex unlock cases to
- * handle a 'deferred_reset' if exists.
- */
-void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev)
-{
-again:
-	spin_lock(&mvdev->reset_lock);
-	if (mvdev->deferred_reset) {
-		mvdev->deferred_reset = false;
-		spin_unlock(&mvdev->reset_lock);
-		mvdev->mig_state = VFIO_DEVICE_STATE_RUNNING;
-		mlx5vf_disable_fds(mvdev, NULL);
-		goto again;
-	}
-	mutex_unlock(&mvdev->state_mutex);
-	spin_unlock(&mvdev->reset_lock);
-}
-
 static struct file *
 mlx5vf_pci_set_device_state(struct vfio_device *vdev,
 			    enum vfio_device_mig_state new_state)
@@ -1238,7 +1219,7 @@ mlx5vf_pci_set_device_state(struct vfio_device *vdev,
 			break;
 		}
 	}
-	mlx5vf_state_mutex_unlock(mvdev);
+	mutex_unlock(&mvdev->state_mutex);
 	return res;
 }
 
@@ -1256,7 +1237,7 @@ static int mlx5vf_pci_get_data_size(struct vfio_device *vdev,
 						    &total_size, 0);
 	if (!ret)
 		*stop_copy_length = total_size;
-	mlx5vf_state_mutex_unlock(mvdev);
+	mutex_unlock(&mvdev->state_mutex);
 	return ret;
 }
 
@@ -1268,32 +1249,22 @@ static int mlx5vf_pci_get_device_state(struct vfio_device *vdev,
 
 	mutex_lock(&mvdev->state_mutex);
 	*curr_state = mvdev->mig_state;
-	mlx5vf_state_mutex_unlock(mvdev);
+	mutex_unlock(&mvdev->state_mutex);
 	return 0;
 }
 
-static void mlx5vf_pci_aer_reset_done(struct pci_dev *pdev)
+static void mlx5vf_pci_reset_device_state(struct vfio_device *vdev)
 {
-	struct mlx5vf_pci_core_device *mvdev = mlx5vf_drvdata(pdev);
+	struct mlx5vf_pci_core_device *mvdev = container_of(
+		vdev, struct mlx5vf_pci_core_device, core_device.vdev);
 
 	if (!mvdev->migrate_cap)
 		return;
 
-	/*
-	 * As the higher VFIO layers are holding locks across reset and using
-	 * those same locks with the mm_lock we need to prevent ABBA deadlock
-	 * with the state_mutex and mm_lock.
-	 * In case the state_mutex was taken already we defer the cleanup work
-	 * to the unlock flow of the other running context.
-	 */
-	spin_lock(&mvdev->reset_lock);
-	mvdev->deferred_reset = true;
-	if (!mutex_trylock(&mvdev->state_mutex)) {
-		spin_unlock(&mvdev->reset_lock);
-		return;
-	}
-	spin_unlock(&mvdev->reset_lock);
-	mlx5vf_state_mutex_unlock(mvdev);
+	mutex_lock(&mvdev->state_mutex);
+	mvdev->mig_state = VFIO_DEVICE_STATE_RUNNING;
+	mlx5vf_disable_fds(mvdev, NULL);
+	mutex_unlock(&mvdev->state_mutex);
 }
 
 static int mlx5vf_pci_open_device(struct vfio_device *core_vdev)
@@ -1325,6 +1296,7 @@ static void mlx5vf_pci_close_device(struct vfio_device *core_vdev)
 static const struct vfio_migration_ops mlx5vf_pci_mig_ops = {
 	.migration_set_state = mlx5vf_pci_set_device_state,
 	.migration_get_state = mlx5vf_pci_get_device_state,
+	.migration_reset_state = mlx5vf_pci_reset_device_state,
 	.migration_get_data_size = mlx5vf_pci_get_data_size,
 };
 
@@ -1417,7 +1389,6 @@ static const struct pci_device_id mlx5vf_pci_table[] = {
 MODULE_DEVICE_TABLE(pci, mlx5vf_pci_table);
 
 static const struct pci_error_handlers mlx5vf_err_handlers = {
-	.reset_done = mlx5vf_pci_aer_reset_done,
 	.error_detected = vfio_pci_core_aer_err_detected,
 };
 
-- 
2.51.2