[PATCH mlx5-next 08/10] RDMA/mlx5: Consolidate data direct state into one object

[Tariq Toukan <tariqt@nvidia.com>]

From: Dragos Tatulea <dtatulea@nvidia.com>

There are three per device data direct objects: mlx5_data_direct_dev,
mlx5_data_direct_resources and mlx5_data_direct_registration with
separate allocations. This patch consolidates all of them into the
registration object.

For this, a few things needed to happen:

- mlx5_ib_dev keeps a pointer to mlx5_data_direct.
- mlx5_data_direct_dev is now accessible from mlx5_data_direct.
  mlx5_data_direct_get_dev() is added as a helper to access
  the device pointer.
- Since mlx5_data_direct is part of mlx5_data_direct_registration,
  getting to the registration is a simple container_of. No need
  for searching the list.

Since mlx5_data_direct_resources no longer has an accurate name,
this patch renames it.

This is a preparation for the upcoming move of data_direct to
mlx5_core.

This patch has no functional changes.

Signed-off-by: Dragos Tatulea <dtatulea@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
---
 drivers/infiniband/hw/mlx5/data_direct.c | 73 ++++++++++--------------
 drivers/infiniband/hw/mlx5/data_direct.h |  7 ---
 drivers/infiniband/hw/mlx5/mlx5_ib.h     |  3 +-
 drivers/infiniband/hw/mlx5/mr.c          |  6 +-
 drivers/infiniband/hw/mlx5/odp.c         |  2 +-
 drivers/infiniband/hw/mlx5/std_types.c   |  2 +-
 drivers/infiniband/hw/mlx5/umr.c         | 11 ++--
 include/linux/mlx5/data_direct.h         | 17 ++++++
 8 files changed, 59 insertions(+), 62 deletions(-)

diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c
index 16afacdd3601..f58a144d95e3 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.c
+++ b/drivers/infiniband/hw/mlx5/data_direct.c
@@ -18,10 +18,10 @@ static LIST_HEAD(mlx5_data_direct_reg_list);
 static DEFINE_MUTEX(mlx5_data_direct_mutex);
 
 struct mlx5_data_direct_registration {
-	struct mlx5_ib_dev *ibdev;
 	char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1];
 	struct list_head list;
 	struct blocking_notifier_head users;
+	struct mlx5_data_direct dd;
 };
 
 static int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev,
@@ -139,8 +139,8 @@ int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev)
 	if (err)
 		goto err_mkey;
 
-	dev->ddr.mkey = mkey;
-	dev->ddr.pdn = pdn;
+	dev->data_direct->mkey = mkey;
+	dev->data_direct->pdn = pdn;
 
 	/* create another mkey with RO support */
 	if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) {
@@ -157,8 +157,8 @@ int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev)
 		err = mlx5_core_create_mkey(mdev, &mkey, in, inlen);
 		/* RO is defined as best effort */
 		if (!err) {
-			dev->ddr.mkey_ro = mkey;
-			dev->ddr.mkey_ro_valid = true;
+			dev->data_direct->mkey_ro = mkey;
+			dev->data_direct->mkey_ro_valid = true;
 		}
 	}
 
@@ -174,38 +174,25 @@ int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev)
 
 void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev)
 {
-	if (dev->ddr.mkey_ro_valid)
-		mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey_ro);
+	if (dev->data_direct->mkey_ro_valid)
+		mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey_ro);
 
-	mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey);
-	mlx5_core_dealloc_pd(dev->mdev, dev->ddr.pdn);
+	mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey);
+	mlx5_core_dealloc_pd(dev->mdev, dev->data_direct->pdn);
 
-	memset(&dev->ddr, 0, sizeof(dev->ddr));
+	memset(dev->data_direct, 0, sizeof(*dev->data_direct));
 }
 
-static struct mlx5_data_direct_registration *
-mlx5_data_direct_get_reg(struct mlx5_ib_dev *ibdev)
-{
-	struct mlx5_data_direct_registration *reg;
-
-	list_for_each_entry(reg, &mlx5_data_direct_reg_list, list)
-		if (reg->ibdev == ibdev)
-			return reg;
-	return NULL;
-}
-
-static void mlx5_data_direct_bind(struct mlx5_ib_dev *ibdev,
+static void mlx5_data_direct_bind(struct mlx5_data_direct_registration *reg,
 				  struct mlx5_data_direct_dev *dev)
 {
-	WRITE_ONCE(ibdev->data_direct_dev, dev);
+	WRITE_ONCE(reg->dd.dev, dev);
 }
 
 static void
 mlx5_data_direct_do_unbind(struct mlx5_data_direct_registration *reg)
 {
-	struct mlx5_ib_dev *ibdev = reg->ibdev;
-
-	WRITE_ONCE(ibdev->data_direct_dev, NULL);
+	WRITE_ONCE(reg->dd.dev, NULL);
 	blocking_notifier_call_chain(&reg->users, MLX5_DATA_DIRECT_UNBIND,
 				     NULL);
 }
@@ -223,7 +210,6 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev)
 	if (!reg)
 		return -ENOMEM;
 
-	reg->ibdev = ibdev;
 	BLOCKING_INIT_NOTIFIER_HEAD(&reg->users);
 
 	err = mlx5_data_direct_query_vuid(ibdev->mdev, reg->vuid);
@@ -234,10 +220,12 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev)
 		return err;
 	}
 
+	ibdev->data_direct = &reg->dd;
+
 	mutex_lock(&mlx5_data_direct_mutex);
 	list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) {
 		if (strcmp(dev->vuid, reg->vuid) == 0) {
-			mlx5_data_direct_bind(ibdev, dev);
+			mlx5_data_direct_bind(reg, dev);
 			break;
 		}
 	}
@@ -257,14 +245,14 @@ void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev)
 	if (!mlx5_data_direct_supported(ibdev->mdev))
 		return;
 
+	reg = container_of(ibdev->data_direct,
+			    struct mlx5_data_direct_registration, dd);
 	mutex_lock(&mlx5_data_direct_mutex);
-	reg = mlx5_data_direct_get_reg(ibdev);
-	if (reg) {
-		list_del(&reg->list);
-		mlx5_data_direct_do_unbind(reg);
-	}
+	list_del(&reg->list);
+	mlx5_data_direct_do_unbind(reg);
 	mutex_unlock(&mlx5_data_direct_mutex);
 
+	ibdev->data_direct = NULL;
 	kfree(reg);
 }
 
@@ -276,11 +264,10 @@ int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev,
 	if (!mlx5_data_direct_supported(ibdev->mdev))
 		return 0;
 
-	mutex_lock(&mlx5_data_direct_mutex);
-	reg = mlx5_data_direct_get_reg(ibdev);
-	if (reg)
-		blocking_notifier_chain_register(&reg->users, nb);
-	mutex_unlock(&mlx5_data_direct_mutex);
+	reg = container_of(ibdev->data_direct,
+			    struct mlx5_data_direct_registration, dd);
+	blocking_notifier_chain_register(&reg->users, nb);
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(mlx5_data_direct_register);
@@ -293,11 +280,9 @@ void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev,
 	if (!mlx5_data_direct_supported(ibdev->mdev))
 		return;
 
-	mutex_lock(&mlx5_data_direct_mutex);
-	reg = mlx5_data_direct_get_reg(ibdev);
-	if (reg)
-		blocking_notifier_chain_unregister(&reg->users, nb);
-	mutex_unlock(&mlx5_data_direct_mutex);
+	reg = container_of(ibdev->data_direct,
+			    struct mlx5_data_direct_registration, dd);
+	blocking_notifier_chain_unregister(&reg->users, nb);
 }
 EXPORT_SYMBOL_GPL(mlx5_data_direct_unregister);
 
@@ -308,7 +293,7 @@ static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev)
 	mutex_lock(&mlx5_data_direct_mutex);
 	list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {
 		if (strcmp(dev->vuid, reg->vuid) == 0)
-			mlx5_data_direct_bind(reg->ibdev, dev);
+			mlx5_data_direct_bind(reg, dev);
 	}
 
 	/* Add the data direct device to the global list, further IB devices may
diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h
index f207239461bf..de2730f00852 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.h
+++ b/drivers/infiniband/hw/mlx5/data_direct.h
@@ -23,13 +23,6 @@ struct mlx5_data_direct_dev {
 	struct list_head list;
 };
 
-struct mlx5_data_direct_resources {
-	u32 pdn;
-	u32 mkey;
-	u32 mkey_ro;
-	u8 mkey_ro_valid :1;
-};
-
 int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev);
 void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev);
 
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 8792e10340c5..6952170f472b 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -1091,7 +1091,7 @@ struct mlx5_macsec {
 struct mlx5_ib_dev {
 	struct ib_device		ib_dev;
 	struct mlx5_core_dev		*mdev;
-	struct mlx5_data_direct_dev	*data_direct_dev;
+	struct mlx5_data_direct		*data_direct;
 	/* Protects data_direct_mr_list and serializes mr
 	 * registration/deregistration with data direct device unbind.
 	 */
@@ -1152,7 +1152,6 @@ struct mlx5_ib_dev {
 	u16 pkey_table_len;
 	u8 lag_ports;
 	struct mlx5_special_mkeys mkeys;
-	struct mlx5_data_direct_resources ddr;
 
 #ifdef CONFIG_MLX5_MACSEC
 	struct mlx5_macsec macsec;
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 1fc61a680f8a..5efe78cb34c1 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -587,7 +587,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem,
 				      populate ? pd : dev->umrc.pd);
 	/* In case a data direct flow, overwrite the pdn field by its internal kernel PD */
 	if (umem->is_dmabuf && ksm_mode)
-		MLX5_SET(mkc, mkc, pd, dev->ddr.pdn);
+		MLX5_SET(mkc, mkc, pd, dev->data_direct->pdn);
 
 	MLX5_SET(mkc, mkc, free, !populate);
 	MLX5_SET(mkc, mkc, access_mode_1_0, access_mode);
@@ -997,7 +997,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset,
 		return ERR_PTR(-EOPNOTSUPP);
 
 	mutex_lock(&dev->data_direct_lock);
-	data_direct_dev = READ_ONCE(dev->data_direct_dev);
+	data_direct_dev = mlx5_data_direct_get_dev(dev->data_direct);
 	if (!data_direct_dev) {
 		ret = -EINVAL;
 		goto end;
@@ -1006,7 +1006,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset,
 	/* If no device's 'data direct mkey' with RO flags exists
 	 * mask it out accordingly.
 	 */
-	if (!dev->ddr.mkey_ro_valid)
+	if (!dev->data_direct->mkey_ro_valid)
 		access_flags &= ~IB_ACCESS_RELAXED_ORDERING;
 	crossed_mr = reg_user_mr_dmabuf(pd, &data_direct_dev->pdev->dev,
 					offset, length, virt_addr, fd,
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index 1badec9bf527..7b9f73b76d0d 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -952,7 +952,7 @@ int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd)
 	int ret;
 
 	if (mr->data_direct)
-		pdn = dev->ddr.pdn;
+		pdn = dev->data_direct->pdn;
 	else
 		pdn = to_mpd(pd)->pdn;
 
diff --git a/drivers/infiniband/hw/mlx5/std_types.c b/drivers/infiniband/hw/mlx5/std_types.c
index df949fa634c9..3828100fdc6c 100644
--- a/drivers/infiniband/hw/mlx5/std_types.c
+++ b/drivers/infiniband/hw/mlx5/std_types.c
@@ -203,7 +203,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)(
 		return PTR_ERR(c);
 	dev = to_mdev(c->ibucontext.device);
 	mutex_lock(&dev->data_direct_lock);
-	data_direct_dev = READ_ONCE(dev->data_direct_dev);
+	data_direct_dev = mlx5_data_direct_get_dev(dev->data_direct);
 	if (!data_direct_dev) {
 		ret = -ENODEV;
 		goto end;
diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c
index 48cae5cc1c1b..1ca83d2d8a0b 100644
--- a/drivers/infiniband/hw/mlx5/umr.c
+++ b/drivers/infiniband/hw/mlx5/umr.c
@@ -760,12 +760,14 @@ static int _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags,
 		}
 
 		if (dd) {
+			struct mlx5_data_direct *mdd = dev->data_direct;
+
 			cur_ksm->va = cpu_to_be64(rdma_block_iter_dma_address(&biter));
 			if (mr->access_flags & IB_ACCESS_RELAXED_ORDERING &&
-			    dev->ddr.mkey_ro_valid)
-				cur_ksm->key = cpu_to_be32(dev->ddr.mkey_ro);
+			    mdd->mkey_ro_valid)
+				cur_ksm->key = cpu_to_be32(mdd->mkey_ro);
 			else
-				cur_ksm->key = cpu_to_be32(dev->ddr.mkey);
+				cur_ksm->key = cpu_to_be32(mdd->mkey);
 			if (mr->umem->is_dmabuf &&
 			    (flags & MLX5_IB_UPD_XLT_ZAP)) {
 				cur_ksm->va = 0;
@@ -810,7 +812,8 @@ int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr,
 	    !(flags & MLX5_IB_UPD_XLT_KEEP_PGSZ)))
 		return -EINVAL;
 
-	return _mlx5r_umr_update_mr_pas(mr, flags, mr_to_mdev(mr)->ddr.pdn,
+	return _mlx5r_umr_update_mr_pas(mr, flags,
+					mr_to_mdev(mr)->data_direct->pdn,
 					true, start_block, nblocks);
 }
 
diff --git a/include/linux/mlx5/data_direct.h b/include/linux/mlx5/data_direct.h
index 71f47547aebb..eaa2dc6bc607 100644
--- a/include/linux/mlx5/data_direct.h
+++ b/include/linux/mlx5/data_direct.h
@@ -6,10 +6,27 @@
 #ifndef _MLX5_DATA_DIRECT_H
 #define _MLX5_DATA_DIRECT_H
 
+#include <linux/compiler.h>
+
 struct mlx5_core_dev;
+struct mlx5_data_direct_dev;
 struct mlx5_ib_dev;
 struct notifier_block;
 
+struct mlx5_data_direct {
+	struct mlx5_data_direct_dev *dev;
+	u32 pdn;
+	u32 mkey;
+	u32 mkey_ro;
+	u8 mkey_ro_valid :1;
+};
+
+static inline struct mlx5_data_direct_dev *
+mlx5_data_direct_get_dev(struct mlx5_data_direct *dd)
+{
+	return dd ? READ_ONCE(dd->dev) : NULL;
+}
+
 int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev);
 void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev);
 
-- 
2.44.0