[PATCH mlx5-next 06/10] RDMA/mlx5: Add new registration stage for data direct users

[Tariq Toukan <tariqt@nvidia.com>]

From: Dragos Tatulea <dtatulea@nvidia.com>

For the upcoming move of data direct to mlx5_core driver,
the API needs to be split into init and register stages.
That's because only the core will do the init phase but
both IB and netdev devices will need to register.

While data direct still lives in mlx5_ib, do all steps in
mlx5_ib_data_direct_init().

While at it, make mlx5_data_direct_query_vuid() private
as it is no longer needed.

This patch has no functional changes. It is a preparation
for moving data direct infrastructure to mlx5_core. For
this reason the new functions are added to the new header.

Signed-off-by: Dragos Tatulea <dtatulea@nvidia.com>
Reviewed-by: Cosmin Ratiu <cratiu@nvidia.com>
Reviewed-by: Leon Romanovsky <leonro@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
---
 drivers/infiniband/hw/mlx5/data_direct.c | 114 ++++++++++++++++++-----
 drivers/infiniband/hw/mlx5/data_direct.h |   5 -
 drivers/infiniband/hw/mlx5/main.c        |  41 ++++----
 drivers/infiniband/hw/mlx5/mlx5_ib.h     |   3 -
 include/linux/mlx5/data_direct.h         |  10 +-
 5 files changed, 118 insertions(+), 55 deletions(-)

diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c
index cc3f741b4f81..f871e3ce9f73 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.c
+++ b/drivers/infiniband/hw/mlx5/data_direct.c
@@ -24,7 +24,8 @@ struct mlx5_data_direct_registration {
 	struct blocking_notifier_head users;
 };
 
-int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid)
+static int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev,
+				       char *out_vuid)
 {
 	u8 out[MLX5_ST_SZ_BYTES(query_vuid_out) +
 	       MLX5_ST_SZ_BYTES(array1024_auto)] = {};
@@ -182,25 +183,65 @@ void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev)
 	memset(&dev->ddr, 0, sizeof(dev->ddr));
 }
 
-int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid,
-			    struct notifier_block *nb)
+static struct mlx5_data_direct_registration *
+mlx5_data_direct_get_reg(struct mlx5_ib_dev *ibdev)
+{
+	struct mlx5_data_direct_registration *reg;
+
+	list_for_each_entry(reg, &mlx5_data_direct_reg_list, list)
+		if (reg->ibdev == ibdev)
+			return reg;
+	return NULL;
+}
+
+static void mlx5_data_direct_bind(struct mlx5_ib_dev *ibdev,
+				  struct mlx5_data_direct_dev *dev)
+{
+	mutex_lock(&ibdev->data_direct_lock);
+	ibdev->data_direct_dev = dev;
+	mutex_unlock(&ibdev->data_direct_lock);
+}
+
+static void
+mlx5_data_direct_do_unbind(struct mlx5_data_direct_registration *reg)
+{
+	struct mlx5_ib_dev *ibdev = reg->ibdev;
+
+	mutex_lock(&ibdev->data_direct_lock);
+	blocking_notifier_call_chain(&reg->users, MLX5_DATA_DIRECT_UNBIND,
+				     NULL);
+	ibdev->data_direct_dev = NULL;
+	mutex_unlock(&ibdev->data_direct_lock);
+}
+
+int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev)
 {
 	struct mlx5_data_direct_registration *reg;
 	struct mlx5_data_direct_dev *dev;
+	int err;
+
+	if (!mlx5_data_direct_supported(ibdev->mdev))
+		return 0;
 
 	reg = kzalloc_obj(*reg);
 	if (!reg)
 		return -ENOMEM;
 
 	reg->ibdev = ibdev;
-	strscpy(reg->vuid, vuid);
 	BLOCKING_INIT_NOTIFIER_HEAD(&reg->users);
-	blocking_notifier_chain_register(&reg->users, nb);
+
+	err = mlx5_data_direct_query_vuid(ibdev->mdev, reg->vuid);
+	if (err) {
+		mlx5_ib_warn(ibdev, "Failed to query VUID, disabling data direct, err=%d\n",
+			     err);
+		kfree(reg);
+		return err;
+	}
 
 	mutex_lock(&mlx5_data_direct_mutex);
 	list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) {
-		if (strcmp(dev->vuid, vuid) == 0) {
-			mlx5_ib_data_direct_bind(ibdev, dev);
+		if (strcmp(dev->vuid, reg->vuid) == 0) {
+			mlx5_data_direct_bind(ibdev, dev);
 			break;
 		}
 	}
@@ -213,25 +254,56 @@ int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid,
 	return 0;
 }
 
-void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev,
-			       struct notifier_block *nb)
+void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev)
 {
 	struct mlx5_data_direct_registration *reg;
 
+	if (!mlx5_data_direct_supported(ibdev->mdev))
+		return;
+
 	mutex_lock(&mlx5_data_direct_mutex);
-	list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {
-		if (reg->ibdev == ibdev) {
-			blocking_notifier_chain_unregister(&reg->users, nb);
-			list_del(&reg->list);
-			kfree(reg);
-			goto end;
-		}
+	reg = mlx5_data_direct_get_reg(ibdev);
+	if (reg) {
+		list_del(&reg->list);
+		mlx5_data_direct_do_unbind(reg);
 	}
+	mutex_unlock(&mlx5_data_direct_mutex);
 
-	WARN_ON(true);
-end:
+	kfree(reg);
+}
+
+int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev,
+			      struct notifier_block *nb)
+{
+	struct mlx5_data_direct_registration *reg;
+
+	if (!mlx5_data_direct_supported(ibdev->mdev))
+		return 0;
+
+	mutex_lock(&mlx5_data_direct_mutex);
+	reg = mlx5_data_direct_get_reg(ibdev);
+	if (reg)
+		blocking_notifier_chain_register(&reg->users, nb);
+	mutex_unlock(&mlx5_data_direct_mutex);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mlx5_data_direct_register);
+
+void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev,
+				 struct notifier_block *nb)
+{
+	struct mlx5_data_direct_registration *reg;
+
+	if (!mlx5_data_direct_supported(ibdev->mdev))
+		return;
+
+	mutex_lock(&mlx5_data_direct_mutex);
+	reg = mlx5_data_direct_get_reg(ibdev);
+	if (reg)
+		blocking_notifier_chain_unregister(&reg->users, nb);
 	mutex_unlock(&mlx5_data_direct_mutex);
 }
+EXPORT_SYMBOL_GPL(mlx5_data_direct_unregister);
 
 static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev)
 {
@@ -240,7 +312,7 @@ static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev)
 	mutex_lock(&mlx5_data_direct_mutex);
 	list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {
 		if (strcmp(dev->vuid, reg->vuid) == 0)
-			mlx5_ib_data_direct_bind(reg->ibdev, dev);
+			mlx5_data_direct_bind(reg->ibdev, dev);
 	}
 
 	/* Add the data direct device to the global list, further IB devices may
@@ -259,9 +331,7 @@ static void mlx5_data_direct_dev_unreg(struct mlx5_data_direct_dev *dev)
 	list_del(&dev->list);
 	list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {
 		if (strcmp(dev->vuid, reg->vuid) == 0)
-			blocking_notifier_call_chain(&reg->users,
-						     MLX5_DATA_DIRECT_UNBIND,
-						     NULL);
+			mlx5_data_direct_do_unbind(reg);
 	}
 	mutex_unlock(&mlx5_data_direct_mutex);
 }
diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h
index 90e0565d7333..f207239461bf 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.h
+++ b/drivers/infiniband/hw/mlx5/data_direct.h
@@ -33,11 +33,6 @@ struct mlx5_data_direct_resources {
 int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev);
 void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev);
 
-int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid);
-int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid,
-			    struct notifier_block *nb);
-void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev,
-			       struct notifier_block *nb);
 int mlx5_data_direct_driver_register(void);
 void mlx5_data_direct_driver_unregister(void);
 
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 0bece3e55109..d0141509870a 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -3945,33 +3945,41 @@ static int mlx5_ib_data_direct_event(struct notifier_block *nb,
 	struct mlx5_ib_dev *dev =
 		container_of(nb, struct mlx5_ib_dev, data_direct_nb);
 
-	if (action == MLX5_DATA_DIRECT_UNBIND)
-		mlx5_ib_data_direct_unbind(dev);
+	if (action != MLX5_DATA_DIRECT_UNBIND)
+		return NOTIFY_DONE;
+
+	mlx5_ib_revoke_data_direct_mrs(dev);
 
 	return NOTIFY_OK;
 }
 
 static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev)
 {
-	char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1] = {};
 	int ret;
 
 	if (!mlx5_data_direct_supported(dev->mdev))
 		return 0;
 
-	ret = mlx5_data_direct_query_vuid(dev->mdev, vuid);
+	ret = mlx5_data_direct_init(dev);
 	if (ret)
 		return ret;
 
 	ret = mlx5_data_direct_create_resources(dev);
 	if (ret)
-		return ret;
+		goto err_resources;
 
 	INIT_LIST_HEAD(&dev->data_direct_mr_list);
 	dev->data_direct_nb.notifier_call = mlx5_ib_data_direct_event;
-	ret = mlx5_data_direct_ib_reg(dev, vuid, &dev->data_direct_nb);
+	ret = mlx5_data_direct_register(dev, &dev->data_direct_nb);
 	if (ret)
-		mlx5_data_direct_free_resources(dev);
+		goto err_register;
+
+	return ret;
+
+err_register:
+	mlx5_data_direct_free_resources(dev);
+err_resources:
+	mlx5_data_direct_cleanup(dev);
 
 	return ret;
 }
@@ -3981,8 +3989,9 @@ static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev)
 	if (!mlx5_data_direct_supported(dev->mdev))
 		return;
 
-	mlx5_data_direct_ib_unreg(dev, &dev->data_direct_nb);
+	mlx5_data_direct_unregister(dev, &dev->data_direct_nb);
 	mlx5_data_direct_free_resources(dev);
+	mlx5_data_direct_cleanup(dev);
 }
 
 static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)
@@ -5008,22 +5017,6 @@ static void mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev *dev)
 		cancel_work_sync(&devr->ports[port].pkey_change_work);
 }
 
-void mlx5_ib_data_direct_bind(struct mlx5_ib_dev *ibdev,
-			      struct mlx5_data_direct_dev *dev)
-{
-	mutex_lock(&ibdev->data_direct_lock);
-	ibdev->data_direct_dev = dev;
-	mutex_unlock(&ibdev->data_direct_lock);
-}
-
-void mlx5_ib_data_direct_unbind(struct mlx5_ib_dev *ibdev)
-{
-	mutex_lock(&ibdev->data_direct_lock);
-	mlx5_ib_revoke_data_direct_mrs(ibdev);
-	ibdev->data_direct_dev = NULL;
-	mutex_unlock(&ibdev->data_direct_lock);
-}
-
 void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
 		      const struct mlx5_ib_profile *profile,
 		      int stage)
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index ff026985e293..e56f3eee7ba1 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -1396,9 +1396,6 @@ int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);
 struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm,
 				struct ib_dm_mr_attr *attr,
 				struct uverbs_attr_bundle *attrs);
-void mlx5_ib_data_direct_bind(struct mlx5_ib_dev *ibdev,
-			      struct mlx5_data_direct_dev *dev);
-void mlx5_ib_data_direct_unbind(struct mlx5_ib_dev *ibdev);
 void mlx5_ib_revoke_data_direct_mrs(struct mlx5_ib_dev *dev);
 
 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
diff --git a/include/linux/mlx5/data_direct.h b/include/linux/mlx5/data_direct.h
index 4556e58c6d03..71f47547aebb 100644
--- a/include/linux/mlx5/data_direct.h
+++ b/include/linux/mlx5/data_direct.h
@@ -7,7 +7,15 @@
 #define _MLX5_DATA_DIRECT_H
 
 struct mlx5_core_dev;
+struct mlx5_ib_dev;
+struct notifier_block;
 
-int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid);
+int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev);
+void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev);
+
+int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev,
+			      struct notifier_block *nb);
+void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev,
+				 struct notifier_block *nb);
 
 #endif
-- 
2.44.0