[PATCH mlx5-next 00/10] mlx5: Move data direct infrastructure to mlx5_core

[PATCH mlx5-next 00/10] mlx5: Move data direct infrastructure to mlx5_core

[Tariq Toukan]

Hi,

Data direct support currently lives in mlx5_ib, which owns the data
direct PCI driver, device matching and resource management. To prepare
for data direct support in netdev, move this infrastructure to mlx5_core
and provide a common interface for its consumers.

The series first replaces the direct IB unbind callback with a blocking
notifier chain, allowing multiple consumers to receive unbind events.
It then separates consumer registration from initialization,
consolidates the per-device state and resources, and keeps IB-specific
locking and MR teardown in mlx5_ib.

With these preparations in place, the final patch moves the shared
implementation and PCI driver to mlx5_core. Data direct initialization
and resource creation run during core load, before consumers attach;
cleanup runs during core unload, after consumers detach.

The data direct PCI driver is also explicitly named "mlx5_dd", replacing
the previous "mlx5_ib" name derived from KBUILD_MODNAME.

This series by Dragos prepares the shared infrastructure for netdev
support.  The netdev consumer will follow in a separate series.

Regards,
Tariq

Dragos Tatulea (10):
  RDMA/mlx5: Notify data direct users via a notifier chain on unbind
  RDMA/mlx5: Add mlx5_data_direct_supported() helper
  RDMA/mlx5: Make the data direct VUID query self-contained
  RDMA/mlx5: Give the data direct PCI driver a proper name
  RDMA/mlx5: Move and rename data direct resource functions
  RDMA/mlx5: Add new registration stage for data direct users
  RDMA/mlx5: Extract IB specific lock out of data direct
  RDMA/mlx5: Consolidate data direct state into one object
  RDMA/mlx5: Pull data_direct resource creation in init phase
  mlx5: Move data direct implementation to mlx5_core

 drivers/infiniband/hw/mlx5/Makefile           |   1 -
 drivers/infiniband/hw/mlx5/cmd.c              |  21 -
 drivers/infiniband/hw/mlx5/cmd.h              |   2 -
 drivers/infiniband/hw/mlx5/data_direct.c      | 227 ----------
 drivers/infiniband/hw/mlx5/data_direct.h      |  23 -
 drivers/infiniband/hw/mlx5/main.c             | 144 +------
 drivers/infiniband/hw/mlx5/mlx5_ib.h          |  18 +-
 drivers/infiniband/hw/mlx5/mr.c               |   7 +-
 drivers/infiniband/hw/mlx5/odp.c              |   2 +-
 drivers/infiniband/hw/mlx5/std_types.c        |   3 +-
 drivers/infiniband/hw/mlx5/umr.c              |  11 +-
 .../net/ethernet/mellanox/mlx5/core/Makefile  |   2 +-
 .../ethernet/mellanox/mlx5/core/data_direct.c | 402 ++++++++++++++++++
 .../net/ethernet/mellanox/mlx5/core/main.c    |  15 +
 .../ethernet/mellanox/mlx5/core/mlx5_core.h   |   5 +
 include/linux/mlx5/data_direct.h              |  47 ++
 include/linux/mlx5/driver.h                   |   8 +
 17 files changed, 517 insertions(+), 421 deletions(-)
 delete mode 100644 drivers/infiniband/hw/mlx5/data_direct.c
 delete mode 100644 drivers/infiniband/hw/mlx5/data_direct.h
 create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/data_direct.c
 create mode 100644 include/linux/mlx5/data_direct.h


base-commit: 36b1d3299d0b6aa51485cc9a79b8d94948d5f3d4
-- 
2.44.0

[PATCH mlx5-next 01/10] RDMA/mlx5: Notify data direct users via a notifier chain on unbind

[Tariq Toukan]

From: Dragos Tatulea <dtatulea@nvidia.com>

Currently during unbind, data direct code calls
mlx5_ib_data_direct_unbind() directly. To make the unbind notification
generic and support multiple users, deliver an MLX5_DATA_DIRECT_UNBIND
event through a blocking_notifier chain kept per registration.

Users register a notifier_block via mlx5_data_direct_ib_reg(); mlx5_ib
embeds one in struct mlx5_ib_dev and, from its notifier callback, runs
the existing mlx5_ib_data_direct_unbind() teardown.

This patch doesn't introduce any functional changes.

Signed-off-by: Dragos Tatulea <dtatulea@nvidia.com>
Reviewed-by: Cosmin Ratiu <cratiu@nvidia.com>
Reviewed-by: Leon Romanovsky <leonro@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
---
 drivers/infiniband/hw/mlx5/data_direct.c | 17 ++++++++++++++---
 drivers/infiniband/hw/mlx5/data_direct.h | 12 ++++++++++--
 drivers/infiniband/hw/mlx5/main.c        | 17 +++++++++++++++--
 drivers/infiniband/hw/mlx5/mlx5_ib.h     |  1 +
 4 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c
index d57484245c38..04affdbdcb12 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.c
+++ b/drivers/infiniband/hw/mlx5/data_direct.c
@@ -4,6 +4,9 @@
  */
 
 #include "mlx5_ib.h"
+
+#include <linux/notifier.h>
+
 #include "data_direct.h"
 
 static LIST_HEAD(mlx5_data_direct_dev_list);
@@ -18,6 +21,7 @@ struct mlx5_data_direct_registration {
 	struct mlx5_ib_dev *ibdev;
 	char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1];
 	struct list_head list;
+	struct blocking_notifier_head users;
 };
 
 static const struct pci_device_id mlx5_data_direct_pci_table[] = {
@@ -78,7 +82,8 @@ static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev)
 	return 0;
 }
 
-int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid)
+int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid,
+			    struct notifier_block *nb)
 {
 	struct mlx5_data_direct_registration *reg;
 	struct mlx5_data_direct_dev *dev;
@@ -89,6 +94,8 @@ int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid)
 
 	reg->ibdev = ibdev;
 	strscpy(reg->vuid, vuid);
+	BLOCKING_INIT_NOTIFIER_HEAD(&reg->users);
+	blocking_notifier_chain_register(&reg->users, nb);
 
 	mutex_lock(&mlx5_data_direct_mutex);
 	list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) {
@@ -106,13 +113,15 @@ int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid)
 	return 0;
 }
 
-void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev)
+void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev,
+			       struct notifier_block *nb)
 {
 	struct mlx5_data_direct_registration *reg;
 
 	mutex_lock(&mlx5_data_direct_mutex);
 	list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {
 		if (reg->ibdev == ibdev) {
+			blocking_notifier_chain_unregister(&reg->users, nb);
 			list_del(&reg->list);
 			kfree(reg);
 			goto end;
@@ -150,7 +159,9 @@ static void mlx5_data_direct_dev_unreg(struct mlx5_data_direct_dev *dev)
 	list_del(&dev->list);
 	list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {
 		if (strcmp(dev->vuid, reg->vuid) == 0)
-			mlx5_ib_data_direct_unbind(reg->ibdev);
+			blocking_notifier_call_chain(&reg->users,
+						     MLX5_DATA_DIRECT_UNBIND,
+						     NULL);
 	}
 	mutex_unlock(&mlx5_data_direct_mutex);
 }
diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h
index 2fd2bdbe8f69..a9bb471949e2 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.h
+++ b/drivers/infiniband/hw/mlx5/data_direct.h
@@ -6,8 +6,14 @@
 #ifndef _MLX5_IB_DATA_DIRECT_H
 #define _MLX5_IB_DATA_DIRECT_H
 
+#include <linux/notifier.h>
+
 struct mlx5_ib_dev;
 
+enum mlx5_data_direct_event {
+	MLX5_DATA_DIRECT_UNBIND,
+};
+
 struct mlx5_data_direct_dev {
 	struct device *device;
 	struct pci_dev *pdev;
@@ -15,8 +21,10 @@ struct mlx5_data_direct_dev {
 	struct list_head list;
 };
 
-int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid);
-void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev);
+int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid,
+			    struct notifier_block *nb);
+void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev,
+			       struct notifier_block *nb);
 int mlx5_data_direct_driver_register(void);
 void mlx5_data_direct_driver_unregister(void);
 
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 02809114fc79..547275dfa40e 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -4019,6 +4019,18 @@ static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev,
 	return false;
 }
 
+static int mlx5_ib_data_direct_event(struct notifier_block *nb,
+				     unsigned long action, void *data)
+{
+	struct mlx5_ib_dev *dev =
+		container_of(nb, struct mlx5_ib_dev, data_direct_nb);
+
+	if (action == MLX5_DATA_DIRECT_UNBIND)
+		mlx5_ib_data_direct_unbind(dev);
+
+	return NOTIFY_OK;
+}
+
 static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev)
 {
 	char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1] = {};
@@ -4037,7 +4049,8 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev)
 		return ret;
 
 	INIT_LIST_HEAD(&dev->data_direct_mr_list);
-	ret = mlx5_data_direct_ib_reg(dev, vuid);
+	dev->data_direct_nb.notifier_call = mlx5_ib_data_direct_event;
+	ret = mlx5_data_direct_ib_reg(dev, vuid, &dev->data_direct_nb);
 	if (ret)
 		mlx5_ib_free_data_direct_resources(dev);
 
@@ -4050,7 +4063,7 @@ static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev)
 	    !MLX5_CAP_GEN_2(dev->mdev, query_vuid))
 		return;
 
-	mlx5_data_direct_ib_unreg(dev);
+	mlx5_data_direct_ib_unreg(dev, &dev->data_direct_nb);
 	mlx5_ib_free_data_direct_resources(dev);
 }
 
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 522984d958bb..97d2aa79e95d 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -1131,6 +1131,7 @@ struct mlx5_ib_dev {
 	spinlock_t		reset_flow_resource_lock;
 	struct list_head	qp_list;
 	struct list_head data_direct_mr_list;
+	struct notifier_block data_direct_nb;
 	/* Array with num_ports elements */
 	struct mlx5_ib_port	*port;
 	struct mlx5_sq_bfreg	bfreg;
-- 
2.44.0

[PATCH mlx5-next 02/10] RDMA/mlx5: Add mlx5_data_direct_supported() helper

[Tariq Toukan]

From: Dragos Tatulea <dtatulea@nvidia.com>

The data direct capability FW checks are duplicated inline in
mlx5_ib_data_direct_init() and mlx5_ib_data_direct_cleanup().

Wrap it in a mlx5_data_direct_supported() helper and use it in both
places, in preparation for moving the data direct matching code to
mlx5_core. For the same reason put it in driver.h instead of
data_direct.h.

No functional change.

Signed-off-by: Dragos Tatulea <dtatulea@nvidia.com>
Reviewed-by: Cosmin Ratiu <cratiu@nvidia.com>
Reviewed-by: Leon Romanovsky <leonro@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
---
 drivers/infiniband/hw/mlx5/main.c | 6 ++----
 include/linux/mlx5/driver.h       | 6 ++++++
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 547275dfa40e..0c57cb39ae7e 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -4036,8 +4036,7 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev)
 	char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1] = {};
 	int ret;
 
-	if (!MLX5_CAP_GEN(dev->mdev, data_direct) ||
-	    !MLX5_CAP_GEN_2(dev->mdev, query_vuid))
+	if (!mlx5_data_direct_supported(dev->mdev))
 		return 0;
 
 	ret = mlx5_cmd_query_vuid(dev->mdev, true, vuid);
@@ -4059,8 +4058,7 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev)
 
 static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev)
 {
-	if (!MLX5_CAP_GEN(dev->mdev, data_direct) ||
-	    !MLX5_CAP_GEN_2(dev->mdev, query_vuid))
+	if (!mlx5_data_direct_supported(dev->mdev))
 		return;
 
 	mlx5_data_direct_ib_unreg(dev, &dev->data_direct_nb);
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index b1871c0821d0..ee29f95ead18 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -1395,4 +1395,10 @@ static inline struct net *mlx5_core_net(struct mlx5_core_dev *dev)
 
 #define MLX5_SW_IMAGE_GUID_MAX_BYTES 9
 
+static inline bool mlx5_data_direct_supported(struct mlx5_core_dev *mdev)
+{
+	return MLX5_CAP_GEN(mdev, data_direct) &&
+	       MLX5_CAP_GEN_2(mdev, query_vuid);
+}
+
 #endif /* MLX5_DRIVER_H */
-- 
2.44.0

[PATCH mlx5-next 03/10] RDMA/mlx5: Make the data direct VUID query self-contained

[Tariq Toukan]

From: Dragos Tatulea <dtatulea@nvidia.com>

mlx5_cmd_query_vuid() in cmd.c is generic over the data_direct flag but
its only caller is the data direct init path, which always passes
data_direct=true.

Move it into data_direct.c as mlx5_data_direct_query_vuid(), dropping
the always-true parameter and hardcoding data_direct=1. This groups the
query with the rest of the data direct code so that it travels together
when the data direct matching code is relocated to mlx5_core, and lets
it become static there once the query is folded into the core bind path.

This function will become private in a subsequent patch.

No functional change.

Signed-off-by: Dragos Tatulea <dtatulea@nvidia.com>
Reviewed-by: Cosmin Ratiu <cratiu@nvidia.com>
Reviewed-by: Leon Romanovsky <leonro@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
---
 drivers/infiniband/hw/mlx5/cmd.c         | 21 ---------------------
 drivers/infiniband/hw/mlx5/cmd.h         |  2 --
 drivers/infiniband/hw/mlx5/data_direct.c | 20 ++++++++++++++++++++
 drivers/infiniband/hw/mlx5/data_direct.h |  3 +++
 drivers/infiniband/hw/mlx5/main.c        |  3 ++-
 include/linux/mlx5/data_direct.h         | 13 +++++++++++++
 6 files changed, 38 insertions(+), 24 deletions(-)
 create mode 100644 include/linux/mlx5/data_direct.h

diff --git a/drivers/infiniband/hw/mlx5/cmd.c b/drivers/infiniband/hw/mlx5/cmd.c
index 7c08e3008927..895b62cc528d 100644
--- a/drivers/infiniband/hw/mlx5/cmd.c
+++ b/drivers/infiniband/hw/mlx5/cmd.c
@@ -245,24 +245,3 @@ int mlx5_cmd_uar_dealloc(struct mlx5_core_dev *dev, u32 uarn, u16 uid)
 	MLX5_SET(dealloc_uar_in, in, uid, uid);
 	return mlx5_cmd_exec_in(dev, dealloc_uar, in);
 }
-
-int mlx5_cmd_query_vuid(struct mlx5_core_dev *dev, bool data_direct,
-			char *out_vuid)
-{
-	u8 out[MLX5_ST_SZ_BYTES(query_vuid_out) +
-		MLX5_ST_SZ_BYTES(array1024_auto)] = {};
-	u8 in[MLX5_ST_SZ_BYTES(query_vuid_in)] = {};
-	char *vuid;
-	int err;
-
-	MLX5_SET(query_vuid_in, in, opcode, MLX5_CMD_OPCODE_QUERY_VUID);
-	MLX5_SET(query_vuid_in, in, vhca_id, MLX5_CAP_GEN(dev, vhca_id));
-	MLX5_SET(query_vuid_in, in, data_direct, data_direct);
-	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
-	if (err)
-		return err;
-
-	vuid = MLX5_ADDR_OF(query_vuid_out, out, vuid);
-	memcpy(out_vuid, vuid, MLX5_ST_SZ_BYTES(array1024_auto));
-	return 0;
-}
diff --git a/drivers/infiniband/hw/mlx5/cmd.h b/drivers/infiniband/hw/mlx5/cmd.h
index e6c88b6ebd0d..e5cd31270443 100644
--- a/drivers/infiniband/hw/mlx5/cmd.h
+++ b/drivers/infiniband/hw/mlx5/cmd.h
@@ -58,6 +58,4 @@ int mlx5_cmd_mad_ifc(struct mlx5_ib_dev *dev, const void *inb, void *outb,
 		     u16 opmod, u8 port);
 int mlx5_cmd_uar_alloc(struct mlx5_core_dev *dev, u32 *uarn, u16 uid);
 int mlx5_cmd_uar_dealloc(struct mlx5_core_dev *dev, u32 uarn, u16 uid);
-int mlx5_cmd_query_vuid(struct mlx5_core_dev *dev, bool data_direct,
-			char *out_vuid);
 #endif /* MLX5_IB_CMD_H */
diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c
index 04affdbdcb12..8d006b611c63 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.c
+++ b/drivers/infiniband/hw/mlx5/data_direct.c
@@ -24,6 +24,26 @@ struct mlx5_data_direct_registration {
 	struct blocking_notifier_head users;
 };
 
+int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid)
+{
+	u8 out[MLX5_ST_SZ_BYTES(query_vuid_out) +
+	       MLX5_ST_SZ_BYTES(array1024_auto)] = {};
+	u8 in[MLX5_ST_SZ_BYTES(query_vuid_in)] = {};
+	char *vuid;
+	int err;
+
+	MLX5_SET(query_vuid_in, in, opcode, MLX5_CMD_OPCODE_QUERY_VUID);
+	MLX5_SET(query_vuid_in, in, vhca_id, MLX5_CAP_GEN(dev, vhca_id));
+	MLX5_SET(query_vuid_in, in, data_direct, 1);
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	if (err)
+		return err;
+
+	vuid = MLX5_ADDR_OF(query_vuid_out, out, vuid);
+	memcpy(out_vuid, vuid, MLX5_ST_SZ_BYTES(array1024_auto));
+	return 0;
+}
+
 static const struct pci_device_id mlx5_data_direct_pci_table[] = {
 	{ PCI_VDEVICE(MELLANOX, 0x2100) }, /* ConnectX-8 Data Direct */
 	{ 0, }
diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h
index a9bb471949e2..fa3a145c06c6 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.h
+++ b/drivers/infiniband/hw/mlx5/data_direct.h
@@ -7,7 +7,9 @@
 #define _MLX5_IB_DATA_DIRECT_H
 
 #include <linux/notifier.h>
+#include <linux/mlx5/data_direct.h>
 
+struct mlx5_core_dev;
 struct mlx5_ib_dev;
 
 enum mlx5_data_direct_event {
@@ -21,6 +23,7 @@ struct mlx5_data_direct_dev {
 	struct list_head list;
 };
 
+int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid);
 int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid,
 			    struct notifier_block *nb);
 void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev,
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 0c57cb39ae7e..8d6914de1442 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -25,6 +25,7 @@
 #include <linux/mlx5/vport.h>
 #include <linux/mlx5/fs.h>
 #include <linux/mlx5/eswitch.h>
+#include <linux/mlx5/data_direct.h>
 #include <linux/mlx5/driver.h>
 #include <linux/mlx5/lag.h>
 #include <linux/list.h>
@@ -4039,7 +4040,7 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev)
 	if (!mlx5_data_direct_supported(dev->mdev))
 		return 0;
 
-	ret = mlx5_cmd_query_vuid(dev->mdev, true, vuid);
+	ret = mlx5_data_direct_query_vuid(dev->mdev, vuid);
 	if (ret)
 		return ret;
 
diff --git a/include/linux/mlx5/data_direct.h b/include/linux/mlx5/data_direct.h
new file mode 100644
index 000000000000..4556e58c6d03
--- /dev/null
+++ b/include/linux/mlx5/data_direct.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/*
+ * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved
+ */
+
+#ifndef _MLX5_DATA_DIRECT_H
+#define _MLX5_DATA_DIRECT_H
+
+struct mlx5_core_dev;
+
+int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid);
+
+#endif
-- 
2.44.0

[PATCH mlx5-next 04/10] RDMA/mlx5: Give the data direct PCI driver a proper name

[Tariq Toukan]

From: Dragos Tatulea <dtatulea@nvidia.com>

The data direct PCI driver uses KBUILD_MODNAME as its driver name,
which resolves to "mlx5_ib". This is misleading (the driver binds
the data direct PCIe function, not an IB device) and would silently
change to "mlx5_core" when the driver is relocated there.

Name the driver explicitly "mlx5_dd".

Signed-off-by: Dragos Tatulea <dtatulea@nvidia.com>
Reviewed-by: Cosmin Ratiu <cratiu@nvidia.com>
Reviewed-by: Leon Romanovsky <leonro@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
---
 drivers/infiniband/hw/mlx5/data_direct.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c
index 8d006b611c63..a80778f2b9b2 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.c
+++ b/drivers/infiniband/hw/mlx5/data_direct.c
@@ -240,7 +240,7 @@ static void mlx5_data_direct_remove(struct pci_dev *pdev)
 }
 
 static struct pci_driver mlx5_data_direct_driver = {
-	.name = KBUILD_MODNAME,
+	.name = "mlx5_dd",
 	.id_table = mlx5_data_direct_pci_table,
 	.probe = mlx5_data_direct_probe,
 	.remove = mlx5_data_direct_remove,
-- 
2.44.0

[PATCH mlx5-next 05/10] RDMA/mlx5: Move and rename data direct resource functions

[Tariq Toukan]

From: Dragos Tatulea <dtatulea@nvidia.com>

Move the data direct resource creation/deletion and resource
struct to be self contained in the data direct file. Rename
the functions to be consistent with the rest.

While at it clear the fields on resource deletion.

This is preparation for moving the data direct code to the
mlx5_core driver. The code has no functional changes.
Keep the resource struct and function declarations in the IB-local
data_direct.h header.

Signed-off-by: Dragos Tatulea <dtatulea@nvidia.com>
Reviewed-by: Cosmin Ratiu <cratiu@nvidia.com>
Reviewed-by: Leon Romanovsky <leonro@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
---
 drivers/infiniband/hw/mlx5/data_direct.c | 80 ++++++++++++++++++++++
 drivers/infiniband/hw/mlx5/data_direct.h | 10 +++
 drivers/infiniband/hw/mlx5/main.c        | 87 +-----------------------
 drivers/infiniband/hw/mlx5/mlx5_ib.h     |  8 +--
 4 files changed, 94 insertions(+), 91 deletions(-)

diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c
index a80778f2b9b2..cc3f741b4f81 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.c
+++ b/drivers/infiniband/hw/mlx5/data_direct.c
@@ -102,6 +102,86 @@ static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev)
 	return 0;
 }
 
+int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
+	struct mlx5_core_dev *mdev = dev->mdev;
+	bool ro_supp = false;
+	void *mkc;
+	u32 mkey;
+	u32 pdn;
+	u32 *in;
+	int err;
+
+	err = mlx5_core_alloc_pd(mdev, &pdn);
+	if (err)
+		return err;
+
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in) {
+		err = -ENOMEM;
+		goto err;
+	}
+
+	MLX5_SET(create_mkey_in, in, data_direct, 1);
+	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
+	MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
+	MLX5_SET(mkc, mkc, lw, 1);
+	MLX5_SET(mkc, mkc, lr, 1);
+	MLX5_SET(mkc, mkc, rw, 1);
+	MLX5_SET(mkc, mkc, rr, 1);
+	MLX5_SET(mkc, mkc, a, 1);
+	MLX5_SET(mkc, mkc, pd, pdn);
+	MLX5_SET(mkc, mkc, length64, 1);
+	MLX5_SET(mkc, mkc, qpn, 0xffffff);
+	err = mlx5_core_create_mkey(mdev, &mkey, in, inlen);
+	if (err)
+		goto err_mkey;
+
+	dev->ddr.mkey = mkey;
+	dev->ddr.pdn = pdn;
+
+	/* create another mkey with RO support */
+	if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) {
+		MLX5_SET(mkc, mkc, relaxed_ordering_write, 1);
+		ro_supp = true;
+	}
+
+	if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)) {
+		MLX5_SET(mkc, mkc, relaxed_ordering_read, 1);
+		ro_supp = true;
+	}
+
+	if (ro_supp) {
+		err = mlx5_core_create_mkey(mdev, &mkey, in, inlen);
+		/* RO is defined as best effort */
+		if (!err) {
+			dev->ddr.mkey_ro = mkey;
+			dev->ddr.mkey_ro_valid = true;
+		}
+	}
+
+	kvfree(in);
+	return 0;
+
+err_mkey:
+	kvfree(in);
+err:
+	mlx5_core_dealloc_pd(mdev, pdn);
+	return err;
+}
+
+void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev)
+{
+	if (dev->ddr.mkey_ro_valid)
+		mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey_ro);
+
+	mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey);
+	mlx5_core_dealloc_pd(dev->mdev, dev->ddr.pdn);
+
+	memset(&dev->ddr, 0, sizeof(dev->ddr));
+}
+
 int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid,
 			    struct notifier_block *nb)
 {
diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h
index fa3a145c06c6..90e0565d7333 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.h
+++ b/drivers/infiniband/hw/mlx5/data_direct.h
@@ -23,6 +23,16 @@ struct mlx5_data_direct_dev {
 	struct list_head list;
 };
 
+struct mlx5_data_direct_resources {
+	u32 pdn;
+	u32 mkey;
+	u32 mkey_ro;
+	u8 mkey_ro_valid :1;
+};
+
+int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev);
+void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev);
+
 int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid);
 int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid,
 			    struct notifier_block *nb);
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 8d6914de1442..0bece3e55109 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -3474,87 +3474,6 @@ static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev)
 	mutex_destroy(&devr->srq_lock);
 }
 
-static int
-mlx5_ib_create_data_direct_resources(struct mlx5_ib_dev *dev)
-{
-	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
-	struct mlx5_core_dev *mdev = dev->mdev;
-	bool ro_supp = false;
-	void *mkc;
-	u32 mkey;
-	u32 pdn;
-	u32 *in;
-	int err;
-
-	err = mlx5_core_alloc_pd(mdev, &pdn);
-	if (err)
-		return err;
-
-	in = kvzalloc(inlen, GFP_KERNEL);
-	if (!in) {
-		err = -ENOMEM;
-		goto err;
-	}
-
-	MLX5_SET(create_mkey_in, in, data_direct, 1);
-	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
-	MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
-	MLX5_SET(mkc, mkc, lw, 1);
-	MLX5_SET(mkc, mkc, lr, 1);
-	MLX5_SET(mkc, mkc, rw, 1);
-	MLX5_SET(mkc, mkc, rr, 1);
-	MLX5_SET(mkc, mkc, a, 1);
-	MLX5_SET(mkc, mkc, pd, pdn);
-	MLX5_SET(mkc, mkc, length64, 1);
-	MLX5_SET(mkc, mkc, qpn, 0xffffff);
-	err = mlx5_core_create_mkey(mdev, &mkey, in, inlen);
-	if (err)
-		goto err_mkey;
-
-	dev->ddr.mkey = mkey;
-	dev->ddr.pdn = pdn;
-
-	/* create another mkey with RO support */
-	if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) {
-		MLX5_SET(mkc, mkc, relaxed_ordering_write, 1);
-		ro_supp = true;
-	}
-
-	if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)) {
-		MLX5_SET(mkc, mkc, relaxed_ordering_read, 1);
-		ro_supp = true;
-	}
-
-	if (ro_supp) {
-		err = mlx5_core_create_mkey(mdev, &mkey, in, inlen);
-		/* RO is defined as best effort */
-		if (!err) {
-			dev->ddr.mkey_ro = mkey;
-			dev->ddr.mkey_ro_valid = true;
-		}
-	}
-
-	kvfree(in);
-	return 0;
-
-err_mkey:
-	kvfree(in);
-err:
-	mlx5_core_dealloc_pd(mdev, pdn);
-	return err;
-}
-
-static void
-mlx5_ib_free_data_direct_resources(struct mlx5_ib_dev *dev)
-{
-
-	if (dev->ddr.mkey_ro_valid)
-		mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey_ro);
-
-	mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey);
-	mlx5_core_dealloc_pd(dev->mdev, dev->ddr.pdn);
-}
-
 static u32 get_core_cap_flags(struct ib_device *ibdev,
 			      struct mlx5_hca_vport_context *rep)
 {
@@ -4044,7 +3963,7 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev)
 	if (ret)
 		return ret;
 
-	ret = mlx5_ib_create_data_direct_resources(dev);
+	ret = mlx5_data_direct_create_resources(dev);
 	if (ret)
 		return ret;
 
@@ -4052,7 +3971,7 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev)
 	dev->data_direct_nb.notifier_call = mlx5_ib_data_direct_event;
 	ret = mlx5_data_direct_ib_reg(dev, vuid, &dev->data_direct_nb);
 	if (ret)
-		mlx5_ib_free_data_direct_resources(dev);
+		mlx5_data_direct_free_resources(dev);
 
 	return ret;
 }
@@ -4063,7 +3982,7 @@ static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev)
 		return;
 
 	mlx5_data_direct_ib_unreg(dev, &dev->data_direct_nb);
-	mlx5_ib_free_data_direct_resources(dev);
+	mlx5_data_direct_free_resources(dev);
 }
 
 static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 97d2aa79e95d..ff026985e293 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -24,6 +24,7 @@
 #include <rdma/mlx5_user_ioctl_cmds.h>
 #include <rdma/mlx5_user_ioctl_verbs.h>
 
+#include "data_direct.h"
 #include "srq.h"
 #include "qp.h"
 #include "macsec.h"
@@ -781,13 +782,6 @@ struct mlx5_ib_port_resources {
 	struct work_struct pkey_change_work;
 };
 
-struct mlx5_data_direct_resources {
-	u32 pdn;
-	u32 mkey;
-	u32 mkey_ro;
-	u8 mkey_ro_valid :1;
-};
-
 struct mlx5_ib_resources {
 	struct ib_cq	*c0;
 	struct mutex cq_lock;
-- 
2.44.0

[PATCH mlx5-next 06/10] RDMA/mlx5: Add new registration stage for data direct users

[Tariq Toukan]

From: Dragos Tatulea <dtatulea@nvidia.com>

For the upcoming move of data direct to mlx5_core driver,
the API needs to be split into init and register stages.
That's because only the core will do the init phase but
both IB and netdev devices will need to register.

While data direct still lives in mlx5_ib, do all steps in
mlx5_ib_data_direct_init().

While at it, make mlx5_data_direct_query_vuid() private
as it is no longer needed.

This patch has no functional changes. It is a preparation
for moving data direct infrastructure to mlx5_core. For
this reason the new functions are added to the new header.

Signed-off-by: Dragos Tatulea <dtatulea@nvidia.com>
Reviewed-by: Cosmin Ratiu <cratiu@nvidia.com>
Reviewed-by: Leon Romanovsky <leonro@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
---
 drivers/infiniband/hw/mlx5/data_direct.c | 114 ++++++++++++++++++-----
 drivers/infiniband/hw/mlx5/data_direct.h |   5 -
 drivers/infiniband/hw/mlx5/main.c        |  41 ++++----
 drivers/infiniband/hw/mlx5/mlx5_ib.h     |   3 -
 include/linux/mlx5/data_direct.h         |  10 +-
 5 files changed, 118 insertions(+), 55 deletions(-)

diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c
index cc3f741b4f81..f871e3ce9f73 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.c
+++ b/drivers/infiniband/hw/mlx5/data_direct.c
@@ -24,7 +24,8 @@ struct mlx5_data_direct_registration {
 	struct blocking_notifier_head users;
 };
 
-int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid)
+static int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev,
+				       char *out_vuid)
 {
 	u8 out[MLX5_ST_SZ_BYTES(query_vuid_out) +
 	       MLX5_ST_SZ_BYTES(array1024_auto)] = {};
@@ -182,25 +183,65 @@ void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev)
 	memset(&dev->ddr, 0, sizeof(dev->ddr));
 }
 
-int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid,
-			    struct notifier_block *nb)
+static struct mlx5_data_direct_registration *
+mlx5_data_direct_get_reg(struct mlx5_ib_dev *ibdev)
+{
+	struct mlx5_data_direct_registration *reg;
+
+	list_for_each_entry(reg, &mlx5_data_direct_reg_list, list)
+		if (reg->ibdev == ibdev)
+			return reg;
+	return NULL;
+}
+
+static void mlx5_data_direct_bind(struct mlx5_ib_dev *ibdev,
+				  struct mlx5_data_direct_dev *dev)
+{
+	mutex_lock(&ibdev->data_direct_lock);
+	ibdev->data_direct_dev = dev;
+	mutex_unlock(&ibdev->data_direct_lock);
+}
+
+static void
+mlx5_data_direct_do_unbind(struct mlx5_data_direct_registration *reg)
+{
+	struct mlx5_ib_dev *ibdev = reg->ibdev;
+
+	mutex_lock(&ibdev->data_direct_lock);
+	blocking_notifier_call_chain(&reg->users, MLX5_DATA_DIRECT_UNBIND,
+				     NULL);
+	ibdev->data_direct_dev = NULL;
+	mutex_unlock(&ibdev->data_direct_lock);
+}
+
+int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev)
 {
 	struct mlx5_data_direct_registration *reg;
 	struct mlx5_data_direct_dev *dev;
+	int err;
+
+	if (!mlx5_data_direct_supported(ibdev->mdev))
+		return 0;
 
 	reg = kzalloc_obj(*reg);
 	if (!reg)
 		return -ENOMEM;
 
 	reg->ibdev = ibdev;
-	strscpy(reg->vuid, vuid);
 	BLOCKING_INIT_NOTIFIER_HEAD(&reg->users);
-	blocking_notifier_chain_register(&reg->users, nb);
+
+	err = mlx5_data_direct_query_vuid(ibdev->mdev, reg->vuid);
+	if (err) {
+		mlx5_ib_warn(ibdev, "Failed to query VUID, disabling data direct, err=%d\n",
+			     err);
+		kfree(reg);
+		return err;
+	}
 
 	mutex_lock(&mlx5_data_direct_mutex);
 	list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) {
-		if (strcmp(dev->vuid, vuid) == 0) {
-			mlx5_ib_data_direct_bind(ibdev, dev);
+		if (strcmp(dev->vuid, reg->vuid) == 0) {
+			mlx5_data_direct_bind(ibdev, dev);
 			break;
 		}
 	}
@@ -213,25 +254,56 @@ int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid,
 	return 0;
 }
 
-void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev,
-			       struct notifier_block *nb)
+void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev)
 {
 	struct mlx5_data_direct_registration *reg;
 
+	if (!mlx5_data_direct_supported(ibdev->mdev))
+		return;
+
 	mutex_lock(&mlx5_data_direct_mutex);
-	list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {
-		if (reg->ibdev == ibdev) {
-			blocking_notifier_chain_unregister(&reg->users, nb);
-			list_del(&reg->list);
-			kfree(reg);
-			goto end;
-		}
+	reg = mlx5_data_direct_get_reg(ibdev);
+	if (reg) {
+		list_del(&reg->list);
+		mlx5_data_direct_do_unbind(reg);
 	}
+	mutex_unlock(&mlx5_data_direct_mutex);
 
-	WARN_ON(true);
-end:
+	kfree(reg);
+}
+
+int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev,
+			      struct notifier_block *nb)
+{
+	struct mlx5_data_direct_registration *reg;
+
+	if (!mlx5_data_direct_supported(ibdev->mdev))
+		return 0;
+
+	mutex_lock(&mlx5_data_direct_mutex);
+	reg = mlx5_data_direct_get_reg(ibdev);
+	if (reg)
+		blocking_notifier_chain_register(&reg->users, nb);
+	mutex_unlock(&mlx5_data_direct_mutex);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mlx5_data_direct_register);
+
+void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev,
+				 struct notifier_block *nb)
+{
+	struct mlx5_data_direct_registration *reg;
+
+	if (!mlx5_data_direct_supported(ibdev->mdev))
+		return;
+
+	mutex_lock(&mlx5_data_direct_mutex);
+	reg = mlx5_data_direct_get_reg(ibdev);
+	if (reg)
+		blocking_notifier_chain_unregister(&reg->users, nb);
 	mutex_unlock(&mlx5_data_direct_mutex);
 }
+EXPORT_SYMBOL_GPL(mlx5_data_direct_unregister);
 
 static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev)
 {
@@ -240,7 +312,7 @@ static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev)
 	mutex_lock(&mlx5_data_direct_mutex);
 	list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {
 		if (strcmp(dev->vuid, reg->vuid) == 0)
-			mlx5_ib_data_direct_bind(reg->ibdev, dev);
+			mlx5_data_direct_bind(reg->ibdev, dev);
 	}
 
 	/* Add the data direct device to the global list, further IB devices may
@@ -259,9 +331,7 @@ static void mlx5_data_direct_dev_unreg(struct mlx5_data_direct_dev *dev)
 	list_del(&dev->list);
 	list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {
 		if (strcmp(dev->vuid, reg->vuid) == 0)
-			blocking_notifier_call_chain(&reg->users,
-						     MLX5_DATA_DIRECT_UNBIND,
-						     NULL);
+			mlx5_data_direct_do_unbind(reg);
 	}
 	mutex_unlock(&mlx5_data_direct_mutex);
 }
diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h
index 90e0565d7333..f207239461bf 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.h
+++ b/drivers/infiniband/hw/mlx5/data_direct.h
@@ -33,11 +33,6 @@ struct mlx5_data_direct_resources {
 int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev);
 void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev);
 
-int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid);
-int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid,
-			    struct notifier_block *nb);
-void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev,
-			       struct notifier_block *nb);
 int mlx5_data_direct_driver_register(void);
 void mlx5_data_direct_driver_unregister(void);
 
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 0bece3e55109..d0141509870a 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -3945,33 +3945,41 @@ static int mlx5_ib_data_direct_event(struct notifier_block *nb,
 	struct mlx5_ib_dev *dev =
 		container_of(nb, struct mlx5_ib_dev, data_direct_nb);
 
-	if (action == MLX5_DATA_DIRECT_UNBIND)
-		mlx5_ib_data_direct_unbind(dev);
+	if (action != MLX5_DATA_DIRECT_UNBIND)
+		return NOTIFY_DONE;
+
+	mlx5_ib_revoke_data_direct_mrs(dev);
 
 	return NOTIFY_OK;
 }
 
 static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev)
 {
-	char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1] = {};
 	int ret;
 
 	if (!mlx5_data_direct_supported(dev->mdev))
 		return 0;
 
-	ret = mlx5_data_direct_query_vuid(dev->mdev, vuid);
+	ret = mlx5_data_direct_init(dev);
 	if (ret)
 		return ret;
 
 	ret = mlx5_data_direct_create_resources(dev);
 	if (ret)
-		return ret;
+		goto err_resources;
 
 	INIT_LIST_HEAD(&dev->data_direct_mr_list);
 	dev->data_direct_nb.notifier_call = mlx5_ib_data_direct_event;
-	ret = mlx5_data_direct_ib_reg(dev, vuid, &dev->data_direct_nb);
+	ret = mlx5_data_direct_register(dev, &dev->data_direct_nb);
 	if (ret)
-		mlx5_data_direct_free_resources(dev);
+		goto err_register;
+
+	return ret;
+
+err_register:
+	mlx5_data_direct_free_resources(dev);
+err_resources:
+	mlx5_data_direct_cleanup(dev);
 
 	return ret;
 }
@@ -3981,8 +3989,9 @@ static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev)
 	if (!mlx5_data_direct_supported(dev->mdev))
 		return;
 
-	mlx5_data_direct_ib_unreg(dev, &dev->data_direct_nb);
+	mlx5_data_direct_unregister(dev, &dev->data_direct_nb);
 	mlx5_data_direct_free_resources(dev);
+	mlx5_data_direct_cleanup(dev);
 }
 
 static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)
@@ -5008,22 +5017,6 @@ static void mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev *dev)
 		cancel_work_sync(&devr->ports[port].pkey_change_work);
 }
 
-void mlx5_ib_data_direct_bind(struct mlx5_ib_dev *ibdev,
-			      struct mlx5_data_direct_dev *dev)
-{
-	mutex_lock(&ibdev->data_direct_lock);
-	ibdev->data_direct_dev = dev;
-	mutex_unlock(&ibdev->data_direct_lock);
-}
-
-void mlx5_ib_data_direct_unbind(struct mlx5_ib_dev *ibdev)
-{
-	mutex_lock(&ibdev->data_direct_lock);
-	mlx5_ib_revoke_data_direct_mrs(ibdev);
-	ibdev->data_direct_dev = NULL;
-	mutex_unlock(&ibdev->data_direct_lock);
-}
-
 void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
 		      const struct mlx5_ib_profile *profile,
 		      int stage)
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index ff026985e293..e56f3eee7ba1 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -1396,9 +1396,6 @@ int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);
 struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm,
 				struct ib_dm_mr_attr *attr,
 				struct uverbs_attr_bundle *attrs);
-void mlx5_ib_data_direct_bind(struct mlx5_ib_dev *ibdev,
-			      struct mlx5_data_direct_dev *dev);
-void mlx5_ib_data_direct_unbind(struct mlx5_ib_dev *ibdev);
 void mlx5_ib_revoke_data_direct_mrs(struct mlx5_ib_dev *dev);
 
 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
diff --git a/include/linux/mlx5/data_direct.h b/include/linux/mlx5/data_direct.h
index 4556e58c6d03..71f47547aebb 100644
--- a/include/linux/mlx5/data_direct.h
+++ b/include/linux/mlx5/data_direct.h
@@ -7,7 +7,15 @@
 #define _MLX5_DATA_DIRECT_H
 
 struct mlx5_core_dev;
+struct mlx5_ib_dev;
+struct notifier_block;
 
-int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid);
+int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev);
+void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev);
+
+int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev,
+			      struct notifier_block *nb);
+void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev,
+				 struct notifier_block *nb);
 
 #endif
-- 
2.44.0

[PATCH mlx5-next 07/10] RDMA/mlx5: Extract IB specific lock out of data direct

[Tariq Toukan]

From: Dragos Tatulea <dtatulea@nvidia.com>

The data_direct_lock from struct mlx5_ib_dev is an IB specific lock.
It protects from unbind while mlx5_data_direct_dev is being accessed.

So move data_direct_lock out of the generic bind/unbind functions
and into the IB specific notifier callback.

data_direct_dev can be set during bind/unbind with a simple WRITE_ONCE,
the lock is no longer needed.

Each user of the data direct registration API has to do their own
locking if necessary.

During unbind, the data_direct_dev is set to NULL before calling the
notifiers to avoid multiple users still accessing the data_direct_dev
while unbind is in progress. This is to avoid another data
direct specific lock which can lead to lock ordering issues.

This patch has no functional changes.

Signed-off-by: Dragos Tatulea <dtatulea@nvidia.com>
Reviewed-by: Cosmin Ratiu <cratiu@nvidia.com>
Reviewed-by: Leon Romanovsky <leonro@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
---
 drivers/infiniband/hw/mlx5/data_direct.c | 8 ++------
 drivers/infiniband/hw/mlx5/main.c        | 2 ++
 drivers/infiniband/hw/mlx5/mlx5_ib.h     | 4 +++-
 drivers/infiniband/hw/mlx5/mr.c          | 2 +-
 drivers/infiniband/hw/mlx5/std_types.c   | 2 +-
 5 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c
index f871e3ce9f73..16afacdd3601 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.c
+++ b/drivers/infiniband/hw/mlx5/data_direct.c
@@ -197,9 +197,7 @@ mlx5_data_direct_get_reg(struct mlx5_ib_dev *ibdev)
 static void mlx5_data_direct_bind(struct mlx5_ib_dev *ibdev,
 				  struct mlx5_data_direct_dev *dev)
 {
-	mutex_lock(&ibdev->data_direct_lock);
-	ibdev->data_direct_dev = dev;
-	mutex_unlock(&ibdev->data_direct_lock);
+	WRITE_ONCE(ibdev->data_direct_dev, dev);
 }
 
 static void
@@ -207,11 +205,9 @@ mlx5_data_direct_do_unbind(struct mlx5_data_direct_registration *reg)
 {
 	struct mlx5_ib_dev *ibdev = reg->ibdev;
 
-	mutex_lock(&ibdev->data_direct_lock);
+	WRITE_ONCE(ibdev->data_direct_dev, NULL);
 	blocking_notifier_call_chain(&reg->users, MLX5_DATA_DIRECT_UNBIND,
 				     NULL);
-	ibdev->data_direct_dev = NULL;
-	mutex_unlock(&ibdev->data_direct_lock);
 }
 
 int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev)
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index d0141509870a..cefbcc816651 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -3948,7 +3948,9 @@ static int mlx5_ib_data_direct_event(struct notifier_block *nb,
 	if (action != MLX5_DATA_DIRECT_UNBIND)
 		return NOTIFY_DONE;
 
+	mutex_lock(&dev->data_direct_lock);
 	mlx5_ib_revoke_data_direct_mrs(dev);
+	mutex_unlock(&dev->data_direct_lock);
 
 	return NOTIFY_OK;
 }
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index e56f3eee7ba1..8792e10340c5 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -1092,7 +1092,9 @@ struct mlx5_ib_dev {
 	struct ib_device		ib_dev;
 	struct mlx5_core_dev		*mdev;
 	struct mlx5_data_direct_dev	*data_direct_dev;
-	/* protect accessing data_direct_dev */
+	/* Protects data_direct_mr_list and serializes mr
+	 * registration/deregistration with data direct device unbind.
+	 */
 	struct mutex			data_direct_lock;
 	struct notifier_block		mdev_events;
 	struct notifier_block		sys_error_events;
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index e6b74955d95d..1fc61a680f8a 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -997,7 +997,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset,
 		return ERR_PTR(-EOPNOTSUPP);
 
 	mutex_lock(&dev->data_direct_lock);
-	data_direct_dev = dev->data_direct_dev;
+	data_direct_dev = READ_ONCE(dev->data_direct_dev);
 	if (!data_direct_dev) {
 		ret = -EINVAL;
 		goto end;
diff --git a/drivers/infiniband/hw/mlx5/std_types.c b/drivers/infiniband/hw/mlx5/std_types.c
index 1ee31611b4b3..df949fa634c9 100644
--- a/drivers/infiniband/hw/mlx5/std_types.c
+++ b/drivers/infiniband/hw/mlx5/std_types.c
@@ -203,7 +203,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)(
 		return PTR_ERR(c);
 	dev = to_mdev(c->ibucontext.device);
 	mutex_lock(&dev->data_direct_lock);
-	data_direct_dev = dev->data_direct_dev;
+	data_direct_dev = READ_ONCE(dev->data_direct_dev);
 	if (!data_direct_dev) {
 		ret = -ENODEV;
 		goto end;
-- 
2.44.0

[PATCH mlx5-next 08/10] RDMA/mlx5: Consolidate data direct state into one object

[Tariq Toukan]

From: Dragos Tatulea <dtatulea@nvidia.com>

There are three per device data direct objects: mlx5_data_direct_dev,
mlx5_data_direct_resources and mlx5_data_direct_registration with
separate allocations. This patch consolidates all of them into the
registration object.

For this, a few things needed to happen:

- mlx5_ib_dev keeps a pointer to mlx5_data_direct.
- mlx5_data_direct_dev is now accessible from mlx5_data_direct.
  mlx5_data_direct_get_dev() is added as a helper to access
  the device pointer.
- Since mlx5_data_direct is part of mlx5_data_direct_registration,
  getting to the registration is a simple container_of. No need
  for searching the list.

Since mlx5_data_direct_resources no longer has an accurate name,
this patch renames it.

This is a preparation for the upcoming move of data_direct to
mlx5_core.

This patch has no functional changes.

Signed-off-by: Dragos Tatulea <dtatulea@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
---
 drivers/infiniband/hw/mlx5/data_direct.c | 73 ++++++++++--------------
 drivers/infiniband/hw/mlx5/data_direct.h |  7 ---
 drivers/infiniband/hw/mlx5/mlx5_ib.h     |  3 +-
 drivers/infiniband/hw/mlx5/mr.c          |  6 +-
 drivers/infiniband/hw/mlx5/odp.c         |  2 +-
 drivers/infiniband/hw/mlx5/std_types.c   |  2 +-
 drivers/infiniband/hw/mlx5/umr.c         | 11 ++--
 include/linux/mlx5/data_direct.h         | 17 ++++++
 8 files changed, 59 insertions(+), 62 deletions(-)

diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c
index 16afacdd3601..f58a144d95e3 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.c
+++ b/drivers/infiniband/hw/mlx5/data_direct.c
@@ -18,10 +18,10 @@ static LIST_HEAD(mlx5_data_direct_reg_list);
 static DEFINE_MUTEX(mlx5_data_direct_mutex);
 
 struct mlx5_data_direct_registration {
-	struct mlx5_ib_dev *ibdev;
 	char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1];
 	struct list_head list;
 	struct blocking_notifier_head users;
+	struct mlx5_data_direct dd;
 };
 
 static int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev,
@@ -139,8 +139,8 @@ int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev)
 	if (err)
 		goto err_mkey;
 
-	dev->ddr.mkey = mkey;
-	dev->ddr.pdn = pdn;
+	dev->data_direct->mkey = mkey;
+	dev->data_direct->pdn = pdn;
 
 	/* create another mkey with RO support */
 	if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) {
@@ -157,8 +157,8 @@ int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev)
 		err = mlx5_core_create_mkey(mdev, &mkey, in, inlen);
 		/* RO is defined as best effort */
 		if (!err) {
-			dev->ddr.mkey_ro = mkey;
-			dev->ddr.mkey_ro_valid = true;
+			dev->data_direct->mkey_ro = mkey;
+			dev->data_direct->mkey_ro_valid = true;
 		}
 	}
 
@@ -174,38 +174,25 @@ int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev)
 
 void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev)
 {
-	if (dev->ddr.mkey_ro_valid)
-		mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey_ro);
+	if (dev->data_direct->mkey_ro_valid)
+		mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey_ro);
 
-	mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey);
-	mlx5_core_dealloc_pd(dev->mdev, dev->ddr.pdn);
+	mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey);
+	mlx5_core_dealloc_pd(dev->mdev, dev->data_direct->pdn);
 
-	memset(&dev->ddr, 0, sizeof(dev->ddr));
+	memset(dev->data_direct, 0, sizeof(*dev->data_direct));
 }
 
-static struct mlx5_data_direct_registration *
-mlx5_data_direct_get_reg(struct mlx5_ib_dev *ibdev)
-{
-	struct mlx5_data_direct_registration *reg;
-
-	list_for_each_entry(reg, &mlx5_data_direct_reg_list, list)
-		if (reg->ibdev == ibdev)
-			return reg;
-	return NULL;
-}
-
-static void mlx5_data_direct_bind(struct mlx5_ib_dev *ibdev,
+static void mlx5_data_direct_bind(struct mlx5_data_direct_registration *reg,
 				  struct mlx5_data_direct_dev *dev)
 {
-	WRITE_ONCE(ibdev->data_direct_dev, dev);
+	WRITE_ONCE(reg->dd.dev, dev);
 }
 
 static void
 mlx5_data_direct_do_unbind(struct mlx5_data_direct_registration *reg)
 {
-	struct mlx5_ib_dev *ibdev = reg->ibdev;
-
-	WRITE_ONCE(ibdev->data_direct_dev, NULL);
+	WRITE_ONCE(reg->dd.dev, NULL);
 	blocking_notifier_call_chain(&reg->users, MLX5_DATA_DIRECT_UNBIND,
 				     NULL);
 }
@@ -223,7 +210,6 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev)
 	if (!reg)
 		return -ENOMEM;
 
-	reg->ibdev = ibdev;
 	BLOCKING_INIT_NOTIFIER_HEAD(&reg->users);
 
 	err = mlx5_data_direct_query_vuid(ibdev->mdev, reg->vuid);
@@ -234,10 +220,12 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev)
 		return err;
 	}
 
+	ibdev->data_direct = &reg->dd;
+
 	mutex_lock(&mlx5_data_direct_mutex);
 	list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) {
 		if (strcmp(dev->vuid, reg->vuid) == 0) {
-			mlx5_data_direct_bind(ibdev, dev);
+			mlx5_data_direct_bind(reg, dev);
 			break;
 		}
 	}
@@ -257,14 +245,14 @@ void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev)
 	if (!mlx5_data_direct_supported(ibdev->mdev))
 		return;
 
+	reg = container_of(ibdev->data_direct,
+			    struct mlx5_data_direct_registration, dd);
 	mutex_lock(&mlx5_data_direct_mutex);
-	reg = mlx5_data_direct_get_reg(ibdev);
-	if (reg) {
-		list_del(&reg->list);
-		mlx5_data_direct_do_unbind(reg);
-	}
+	list_del(&reg->list);
+	mlx5_data_direct_do_unbind(reg);
 	mutex_unlock(&mlx5_data_direct_mutex);
 
+	ibdev->data_direct = NULL;
 	kfree(reg);
 }
 
@@ -276,11 +264,10 @@ int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev,
 	if (!mlx5_data_direct_supported(ibdev->mdev))
 		return 0;
 
-	mutex_lock(&mlx5_data_direct_mutex);
-	reg = mlx5_data_direct_get_reg(ibdev);
-	if (reg)
-		blocking_notifier_chain_register(&reg->users, nb);
-	mutex_unlock(&mlx5_data_direct_mutex);
+	reg = container_of(ibdev->data_direct,
+			    struct mlx5_data_direct_registration, dd);
+	blocking_notifier_chain_register(&reg->users, nb);
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(mlx5_data_direct_register);
@@ -293,11 +280,9 @@ void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev,
 	if (!mlx5_data_direct_supported(ibdev->mdev))
 		return;
 
-	mutex_lock(&mlx5_data_direct_mutex);
-	reg = mlx5_data_direct_get_reg(ibdev);
-	if (reg)
-		blocking_notifier_chain_unregister(&reg->users, nb);
-	mutex_unlock(&mlx5_data_direct_mutex);
+	reg = container_of(ibdev->data_direct,
+			    struct mlx5_data_direct_registration, dd);
+	blocking_notifier_chain_unregister(&reg->users, nb);
 }
 EXPORT_SYMBOL_GPL(mlx5_data_direct_unregister);
 
@@ -308,7 +293,7 @@ static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev)
 	mutex_lock(&mlx5_data_direct_mutex);
 	list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {
 		if (strcmp(dev->vuid, reg->vuid) == 0)
-			mlx5_data_direct_bind(reg->ibdev, dev);
+			mlx5_data_direct_bind(reg, dev);
 	}
 
 	/* Add the data direct device to the global list, further IB devices may
diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h
index f207239461bf..de2730f00852 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.h
+++ b/drivers/infiniband/hw/mlx5/data_direct.h
@@ -23,13 +23,6 @@ struct mlx5_data_direct_dev {
 	struct list_head list;
 };
 
-struct mlx5_data_direct_resources {
-	u32 pdn;
-	u32 mkey;
-	u32 mkey_ro;
-	u8 mkey_ro_valid :1;
-};
-
 int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev);
 void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev);
 
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 8792e10340c5..6952170f472b 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -1091,7 +1091,7 @@ struct mlx5_macsec {
 struct mlx5_ib_dev {
 	struct ib_device		ib_dev;
 	struct mlx5_core_dev		*mdev;
-	struct mlx5_data_direct_dev	*data_direct_dev;
+	struct mlx5_data_direct		*data_direct;
 	/* Protects data_direct_mr_list and serializes mr
 	 * registration/deregistration with data direct device unbind.
 	 */
@@ -1152,7 +1152,6 @@ struct mlx5_ib_dev {
 	u16 pkey_table_len;
 	u8 lag_ports;
 	struct mlx5_special_mkeys mkeys;
-	struct mlx5_data_direct_resources ddr;
 
 #ifdef CONFIG_MLX5_MACSEC
 	struct mlx5_macsec macsec;
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 1fc61a680f8a..5efe78cb34c1 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -587,7 +587,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem,
 				      populate ? pd : dev->umrc.pd);
 	/* In case a data direct flow, overwrite the pdn field by its internal kernel PD */
 	if (umem->is_dmabuf && ksm_mode)
-		MLX5_SET(mkc, mkc, pd, dev->ddr.pdn);
+		MLX5_SET(mkc, mkc, pd, dev->data_direct->pdn);
 
 	MLX5_SET(mkc, mkc, free, !populate);
 	MLX5_SET(mkc, mkc, access_mode_1_0, access_mode);
@@ -997,7 +997,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset,
 		return ERR_PTR(-EOPNOTSUPP);
 
 	mutex_lock(&dev->data_direct_lock);
-	data_direct_dev = READ_ONCE(dev->data_direct_dev);
+	data_direct_dev = mlx5_data_direct_get_dev(dev->data_direct);
 	if (!data_direct_dev) {
 		ret = -EINVAL;
 		goto end;
@@ -1006,7 +1006,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset,
 	/* If no device's 'data direct mkey' with RO flags exists
 	 * mask it out accordingly.
 	 */
-	if (!dev->ddr.mkey_ro_valid)
+	if (!dev->data_direct->mkey_ro_valid)
 		access_flags &= ~IB_ACCESS_RELAXED_ORDERING;
 	crossed_mr = reg_user_mr_dmabuf(pd, &data_direct_dev->pdev->dev,
 					offset, length, virt_addr, fd,
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index 1badec9bf527..7b9f73b76d0d 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -952,7 +952,7 @@ int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd)
 	int ret;
 
 	if (mr->data_direct)
-		pdn = dev->ddr.pdn;
+		pdn = dev->data_direct->pdn;
 	else
 		pdn = to_mpd(pd)->pdn;
 
diff --git a/drivers/infiniband/hw/mlx5/std_types.c b/drivers/infiniband/hw/mlx5/std_types.c
index df949fa634c9..3828100fdc6c 100644
--- a/drivers/infiniband/hw/mlx5/std_types.c
+++ b/drivers/infiniband/hw/mlx5/std_types.c
@@ -203,7 +203,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)(
 		return PTR_ERR(c);
 	dev = to_mdev(c->ibucontext.device);
 	mutex_lock(&dev->data_direct_lock);
-	data_direct_dev = READ_ONCE(dev->data_direct_dev);
+	data_direct_dev = mlx5_data_direct_get_dev(dev->data_direct);
 	if (!data_direct_dev) {
 		ret = -ENODEV;
 		goto end;
diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c
index 48cae5cc1c1b..1ca83d2d8a0b 100644
--- a/drivers/infiniband/hw/mlx5/umr.c
+++ b/drivers/infiniband/hw/mlx5/umr.c
@@ -760,12 +760,14 @@ static int _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags,
 		}
 
 		if (dd) {
+			struct mlx5_data_direct *mdd = dev->data_direct;
+
 			cur_ksm->va = cpu_to_be64(rdma_block_iter_dma_address(&biter));
 			if (mr->access_flags & IB_ACCESS_RELAXED_ORDERING &&
-			    dev->ddr.mkey_ro_valid)
-				cur_ksm->key = cpu_to_be32(dev->ddr.mkey_ro);
+			    mdd->mkey_ro_valid)
+				cur_ksm->key = cpu_to_be32(mdd->mkey_ro);
 			else
-				cur_ksm->key = cpu_to_be32(dev->ddr.mkey);
+				cur_ksm->key = cpu_to_be32(mdd->mkey);
 			if (mr->umem->is_dmabuf &&
 			    (flags & MLX5_IB_UPD_XLT_ZAP)) {
 				cur_ksm->va = 0;
@@ -810,7 +812,8 @@ int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr,
 	    !(flags & MLX5_IB_UPD_XLT_KEEP_PGSZ)))
 		return -EINVAL;
 
-	return _mlx5r_umr_update_mr_pas(mr, flags, mr_to_mdev(mr)->ddr.pdn,
+	return _mlx5r_umr_update_mr_pas(mr, flags,
+					mr_to_mdev(mr)->data_direct->pdn,
 					true, start_block, nblocks);
 }
 
diff --git a/include/linux/mlx5/data_direct.h b/include/linux/mlx5/data_direct.h
index 71f47547aebb..eaa2dc6bc607 100644
--- a/include/linux/mlx5/data_direct.h
+++ b/include/linux/mlx5/data_direct.h
@@ -6,10 +6,27 @@
 #ifndef _MLX5_DATA_DIRECT_H
 #define _MLX5_DATA_DIRECT_H
 
+#include <linux/compiler.h>
+
 struct mlx5_core_dev;
+struct mlx5_data_direct_dev;
 struct mlx5_ib_dev;
 struct notifier_block;
 
+struct mlx5_data_direct {
+	struct mlx5_data_direct_dev *dev;
+	u32 pdn;
+	u32 mkey;
+	u32 mkey_ro;
+	u8 mkey_ro_valid :1;
+};
+
+static inline struct mlx5_data_direct_dev *
+mlx5_data_direct_get_dev(struct mlx5_data_direct *dd)
+{
+	return dd ? READ_ONCE(dd->dev) : NULL;
+}
+
 int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev);
 void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev);
 
-- 
2.44.0

[PATCH mlx5-next 09/10] RDMA/mlx5: Pull data_direct resource creation in init phase

[Tariq Toukan]

From: Dragos Tatulea <dtatulea@nvidia.com>

Since data_direct resource creation happens directly after init,
there's no need to separate them.

This patch moves data direct resource creation in the init phase,
thus simplifying the data direct API. The cleanup path gets the same
treatment.

This keeps the existing IB device lifetime and prepares for moving the
combined initialization and cleanup to mlx5_core load and unload.

The patch has no functional changes.

Signed-off-by: Dragos Tatulea <dtatulea@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
---
 drivers/infiniband/hw/mlx5/data_direct.c | 16 ++++++++++++----
 drivers/infiniband/hw/mlx5/data_direct.h |  3 ---
 drivers/infiniband/hw/mlx5/main.c        |  7 -------
 3 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c
index f58a144d95e3..3878c4aebb2c 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.c
+++ b/drivers/infiniband/hw/mlx5/data_direct.c
@@ -103,7 +103,7 @@ static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev)
 	return 0;
 }
 
-int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev)
+static int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev)
 {
 	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
 	struct mlx5_core_dev *mdev = dev->mdev;
@@ -172,15 +172,13 @@ int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev)
 	return err;
 }
 
-void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev)
+static void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev)
 {
 	if (dev->data_direct->mkey_ro_valid)
 		mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey_ro);
 
 	mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey);
 	mlx5_core_dealloc_pd(dev->mdev, dev->data_direct->pdn);
-
-	memset(dev->data_direct, 0, sizeof(*dev->data_direct));
 }
 
 static void mlx5_data_direct_bind(struct mlx5_data_direct_registration *reg,
@@ -222,6 +220,10 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev)
 
 	ibdev->data_direct = &reg->dd;
 
+	err = mlx5_data_direct_create_resources(ibdev);
+	if (err)
+		goto err_resources;
+
 	mutex_lock(&mlx5_data_direct_mutex);
 	list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) {
 		if (strcmp(dev->vuid, reg->vuid) == 0) {
@@ -236,6 +238,11 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev)
 	list_add_tail(&reg->list, &mlx5_data_direct_reg_list);
 	mutex_unlock(&mlx5_data_direct_mutex);
 	return 0;
+
+err_resources:
+	ibdev->data_direct = NULL;
+	kfree(reg);
+	return err;
 }
 
 void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev)
@@ -252,6 +259,7 @@ void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev)
 	mlx5_data_direct_do_unbind(reg);
 	mutex_unlock(&mlx5_data_direct_mutex);
 
+	mlx5_data_direct_free_resources(ibdev);
 	ibdev->data_direct = NULL;
 	kfree(reg);
 }
diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h
index de2730f00852..49cdd407018a 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.h
+++ b/drivers/infiniband/hw/mlx5/data_direct.h
@@ -23,9 +23,6 @@ struct mlx5_data_direct_dev {
 	struct list_head list;
 };
 
-int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev);
-void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev);
-
 int mlx5_data_direct_driver_register(void);
 void mlx5_data_direct_driver_unregister(void);
 
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index cefbcc816651..941928a782bd 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -3966,10 +3966,6 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev)
 	if (ret)
 		return ret;
 
-	ret = mlx5_data_direct_create_resources(dev);
-	if (ret)
-		goto err_resources;
-
 	INIT_LIST_HEAD(&dev->data_direct_mr_list);
 	dev->data_direct_nb.notifier_call = mlx5_ib_data_direct_event;
 	ret = mlx5_data_direct_register(dev, &dev->data_direct_nb);
@@ -3979,8 +3975,6 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev)
 	return ret;
 
 err_register:
-	mlx5_data_direct_free_resources(dev);
-err_resources:
 	mlx5_data_direct_cleanup(dev);
 
 	return ret;
@@ -3992,7 +3986,6 @@ static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev)
 		return;
 
 	mlx5_data_direct_unregister(dev, &dev->data_direct_nb);
-	mlx5_data_direct_free_resources(dev);
 	mlx5_data_direct_cleanup(dev);
 }
 
-- 
2.44.0

[PATCH mlx5-next 10/10] mlx5: Move data direct implementation to mlx5_core

[Tariq Toukan]

From: Dragos Tatulea <dtatulea@nvidia.com>

As data direct will also be used by netdev devices, move the data direct
code to mlx5_core. Now mlx5_core is responsible for data direct
initialization and resource creation.

Now data_direct is initialized in mlx5_load() before attaching consumers
and clean it up in mlx5_unload() after detaching them. The data direct
PCI driver is moved and registered in mlx5_core.

IB specific work stays in mlx5_ib: notifier registration, mutex
initialization and MR list initialization.

This patch has no functional changes.

Signed-off-by: Dragos Tatulea <dtatulea@nvidia.com>
---
 drivers/infiniband/hw/mlx5/Makefile           |  1 -
 drivers/infiniband/hw/mlx5/data_direct.h      | 29 ------
 drivers/infiniband/hw/mlx5/main.c             | 23 +----
 drivers/infiniband/hw/mlx5/mlx5_ib.h          |  3 +-
 drivers/infiniband/hw/mlx5/mr.c               |  7 +-
 drivers/infiniband/hw/mlx5/odp.c              |  2 +-
 drivers/infiniband/hw/mlx5/std_types.c        |  3 +-
 drivers/infiniband/hw/mlx5/umr.c              |  4 +-
 .../net/ethernet/mellanox/mlx5/core/Makefile  |  2 +-
 .../mellanox/mlx5/core}/data_direct.c         | 91 ++++++++++---------
 .../net/ethernet/mellanox/mlx5/core/main.c    | 15 +++
 .../ethernet/mellanox/mlx5/core/mlx5_core.h   |  5 +
 include/linux/mlx5/data_direct.h              | 23 +++--
 include/linux/mlx5/driver.h                   |  2 +
 14 files changed, 97 insertions(+), 113 deletions(-)
 delete mode 100644 drivers/infiniband/hw/mlx5/data_direct.h
 rename drivers/{infiniband/hw/mlx5 => net/ethernet/mellanox/mlx5/core}/data_direct.c (79%)

diff --git a/drivers/infiniband/hw/mlx5/Makefile b/drivers/infiniband/hw/mlx5/Makefile
index dd7bb377f491..35afc7bc57d2 100644
--- a/drivers/infiniband/hw/mlx5/Makefile
+++ b/drivers/infiniband/hw/mlx5/Makefile
@@ -6,7 +6,6 @@ mlx5_ib-y := ah.o \
 	     cong.o \
 	     counters.o \
 	     cq.o \
-	     data_direct.o \
 	     dm.o \
 	     dmah.o \
 	     doorbell.o \
diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h
deleted file mode 100644
index 49cdd407018a..000000000000
--- a/drivers/infiniband/hw/mlx5/data_direct.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
-/*
- * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved
- */
-
-#ifndef _MLX5_IB_DATA_DIRECT_H
-#define _MLX5_IB_DATA_DIRECT_H
-
-#include <linux/notifier.h>
-#include <linux/mlx5/data_direct.h>
-
-struct mlx5_core_dev;
-struct mlx5_ib_dev;
-
-enum mlx5_data_direct_event {
-	MLX5_DATA_DIRECT_UNBIND,
-};
-
-struct mlx5_data_direct_dev {
-	struct device *device;
-	struct pci_dev *pdev;
-	char *vuid;
-	struct list_head list;
-};
-
-int mlx5_data_direct_driver_register(void);
-void mlx5_data_direct_driver_unregister(void);
-
-#endif
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 941928a782bd..dd2a3996b14e 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -52,7 +52,6 @@
 #include <rdma/mlx5_user_ioctl_cmds.h>
 #include <rdma/ib_ucaps.h>
 #include "macsec.h"
-#include "data_direct.h"
 #include "dmah.h"
 
 #define UVERBS_MODULE_NAME mlx5_ib
@@ -3962,20 +3961,9 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev)
 	if (!mlx5_data_direct_supported(dev->mdev))
 		return 0;
 
-	ret = mlx5_data_direct_init(dev);
-	if (ret)
-		return ret;
-
 	INIT_LIST_HEAD(&dev->data_direct_mr_list);
 	dev->data_direct_nb.notifier_call = mlx5_ib_data_direct_event;
-	ret = mlx5_data_direct_register(dev, &dev->data_direct_nb);
-	if (ret)
-		goto err_register;
-
-	return ret;
-
-err_register:
-	mlx5_data_direct_cleanup(dev);
+	ret = mlx5_data_direct_register(dev->mdev, &dev->data_direct_nb);
 
 	return ret;
 }
@@ -3985,8 +3973,7 @@ static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev)
 	if (!mlx5_data_direct_supported(dev->mdev))
 		return;
 
-	mlx5_data_direct_unregister(dev, &dev->data_direct_nb);
-	mlx5_data_direct_cleanup(dev);
+	mlx5_data_direct_unregister(dev->mdev, &dev->data_direct_nb);
 }
 
 static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)
@@ -5438,9 +5425,6 @@ static int __init mlx5_ib_init(void)
 	ret = mlx5r_rep_init();
 	if (ret)
 		goto rep_err;
-	ret = mlx5_data_direct_driver_register();
-	if (ret)
-		goto dd_err;
 	ret = auxiliary_driver_register(&mlx5r_mp_driver);
 	if (ret)
 		goto mp_err;
@@ -5453,8 +5437,6 @@ static int __init mlx5_ib_init(void)
 drv_err:
 	auxiliary_driver_unregister(&mlx5r_mp_driver);
 mp_err:
-	mlx5_data_direct_driver_unregister();
-dd_err:
 	mlx5r_rep_cleanup();
 rep_err:
 	mlx5_ib_qp_event_cleanup();
@@ -5466,7 +5448,6 @@ static int __init mlx5_ib_init(void)
 
 static void __exit mlx5_ib_cleanup(void)
 {
-	mlx5_data_direct_driver_unregister();
 	auxiliary_driver_unregister(&mlx5r_driver);
 	auxiliary_driver_unregister(&mlx5r_mp_driver);
 	mlx5r_rep_cleanup();
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 6952170f472b..1a3b9cb79184 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -24,7 +24,7 @@
 #include <rdma/mlx5_user_ioctl_cmds.h>
 #include <rdma/mlx5_user_ioctl_verbs.h>
 
-#include "data_direct.h"
+#include <linux/mlx5/data_direct.h>
 #include "srq.h"
 #include "qp.h"
 #include "macsec.h"
@@ -1091,7 +1091,6 @@ struct mlx5_macsec {
 struct mlx5_ib_dev {
 	struct ib_device		ib_dev;
 	struct mlx5_core_dev		*mdev;
-	struct mlx5_data_direct		*data_direct;
 	/* Protects data_direct_mr_list and serializes mr
 	 * registration/deregistration with data direct device unbind.
 	 */
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 5efe78cb34c1..7463b36a36b6 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -44,7 +44,6 @@
 #include "dm.h"
 #include "mlx5_ib.h"
 #include "umr.h"
-#include "data_direct.h"
 #include "dmah.h"
 
 static int mkey_max_umr_order(struct mlx5_ib_dev *dev)
@@ -587,7 +586,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem,
 				      populate ? pd : dev->umrc.pd);
 	/* In case a data direct flow, overwrite the pdn field by its internal kernel PD */
 	if (umem->is_dmabuf && ksm_mode)
-		MLX5_SET(mkc, mkc, pd, dev->data_direct->pdn);
+		MLX5_SET(mkc, mkc, pd, dev->mdev->data_direct->pdn);
 
 	MLX5_SET(mkc, mkc, free, !populate);
 	MLX5_SET(mkc, mkc, access_mode_1_0, access_mode);
@@ -997,7 +996,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset,
 		return ERR_PTR(-EOPNOTSUPP);
 
 	mutex_lock(&dev->data_direct_lock);
-	data_direct_dev = mlx5_data_direct_get_dev(dev->data_direct);
+	data_direct_dev = mlx5_data_direct_get_dev(dev->mdev->data_direct);
 	if (!data_direct_dev) {
 		ret = -EINVAL;
 		goto end;
@@ -1006,7 +1005,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset,
 	/* If no device's 'data direct mkey' with RO flags exists
 	 * mask it out accordingly.
 	 */
-	if (!dev->data_direct->mkey_ro_valid)
+	if (!dev->mdev->data_direct->mkey_ro_valid)
 		access_flags &= ~IB_ACCESS_RELAXED_ORDERING;
 	crossed_mr = reg_user_mr_dmabuf(pd, &data_direct_dev->pdev->dev,
 					offset, length, virt_addr, fd,
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index 7b9f73b76d0d..2e9a704acc44 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -952,7 +952,7 @@ int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd)
 	int ret;
 
 	if (mr->data_direct)
-		pdn = dev->data_direct->pdn;
+		pdn = dev->mdev->data_direct->pdn;
 	else
 		pdn = to_mpd(pd)->pdn;
 
diff --git a/drivers/infiniband/hw/mlx5/std_types.c b/drivers/infiniband/hw/mlx5/std_types.c
index 3828100fdc6c..b705b88dc917 100644
--- a/drivers/infiniband/hw/mlx5/std_types.c
+++ b/drivers/infiniband/hw/mlx5/std_types.c
@@ -10,7 +10,6 @@
 #include <linux/mlx5/eswitch.h>
 #include <linux/mlx5/vport.h>
 #include "mlx5_ib.h"
-#include "data_direct.h"
 
 #define UVERBS_MODULE_NAME mlx5_ib
 #include <rdma/uverbs_named_ioctl.h>
@@ -203,7 +202,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)(
 		return PTR_ERR(c);
 	dev = to_mdev(c->ibucontext.device);
 	mutex_lock(&dev->data_direct_lock);
-	data_direct_dev = mlx5_data_direct_get_dev(dev->data_direct);
+	data_direct_dev = mlx5_data_direct_get_dev(dev->mdev->data_direct);
 	if (!data_direct_dev) {
 		ret = -ENODEV;
 		goto end;
diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c
index 1ca83d2d8a0b..a3bb8d03fd35 100644
--- a/drivers/infiniband/hw/mlx5/umr.c
+++ b/drivers/infiniband/hw/mlx5/umr.c
@@ -760,7 +760,7 @@ static int _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags,
 		}
 
 		if (dd) {
-			struct mlx5_data_direct *mdd = dev->data_direct;
+			struct mlx5_data_direct *mdd = dev->mdev->data_direct;
 
 			cur_ksm->va = cpu_to_be64(rdma_block_iter_dma_address(&biter));
 			if (mr->access_flags & IB_ACCESS_RELAXED_ORDERING &&
@@ -813,7 +813,7 @@ int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr,
 		return -EINVAL;
 
 	return _mlx5r_umr_update_mr_pas(mr, flags,
-					mr_to_mdev(mr)->data_direct->pdn,
+					mr_to_mdev(mr)->mdev->data_direct->pdn,
 					true, start_block, nblocks);
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
index 19e50f0d55af..a1871259210d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
@@ -18,7 +18,7 @@ mlx5_core-y :=	main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \
 		lib/devcom.o lib/pci_vsc.o lib/dm.o lib/fs_ttc.o diag/fs_tracepoint.o \
 		diag/fw_tracer.o diag/crdump.o devlink.o sh_devlink.o diag/rsc_dump.o \
 		diag/reporter_vnic.o fw_reset.o qos.o lib/tout.o lib/aso.o wc.o fs_pool.o \
-		lib/nv_param.o
+		lib/nv_param.o data_direct.o
 
 #
 # Netdev basic
diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/net/ethernet/mellanox/mlx5/core/data_direct.c
similarity index 79%
rename from drivers/infiniband/hw/mlx5/data_direct.c
rename to drivers/net/ethernet/mellanox/mlx5/core/data_direct.c
index 3878c4aebb2c..67717110feab 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/data_direct.c
@@ -3,11 +3,14 @@
  * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved
  */
 
-#include "mlx5_ib.h"
+#include <linux/list.h>
+#include <linux/mlx5/driver.h>
+#include <linux/mlx5/data_direct.h>
+#include <linux/mlx5/mlx5_ifc.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
 
-#include <linux/notifier.h>
-
-#include "data_direct.h"
+#include "mlx5_core.h"
 
 static LIST_HEAD(mlx5_data_direct_dev_list);
 static LIST_HEAD(mlx5_data_direct_reg_list);
@@ -103,10 +106,9 @@ static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev)
 	return 0;
 }
 
-static int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev)
+static int mlx5_data_direct_create_resources(struct mlx5_core_dev *mdev)
 {
 	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
-	struct mlx5_core_dev *mdev = dev->mdev;
 	bool ro_supp = false;
 	void *mkc;
 	u32 mkey;
@@ -139,16 +141,16 @@ static int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev)
 	if (err)
 		goto err_mkey;
 
-	dev->data_direct->mkey = mkey;
-	dev->data_direct->pdn = pdn;
+	mdev->data_direct->mkey = mkey;
+	mdev->data_direct->pdn = pdn;
 
 	/* create another mkey with RO support */
-	if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) {
+	if (MLX5_CAP_GEN(mdev, relaxed_ordering_write)) {
 		MLX5_SET(mkc, mkc, relaxed_ordering_write, 1);
 		ro_supp = true;
 	}
 
-	if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)) {
+	if (MLX5_CAP_GEN(mdev, relaxed_ordering_read)) {
 		MLX5_SET(mkc, mkc, relaxed_ordering_read, 1);
 		ro_supp = true;
 	}
@@ -157,8 +159,8 @@ static int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev)
 		err = mlx5_core_create_mkey(mdev, &mkey, in, inlen);
 		/* RO is defined as best effort */
 		if (!err) {
-			dev->data_direct->mkey_ro = mkey;
-			dev->data_direct->mkey_ro_valid = true;
+			mdev->data_direct->mkey_ro = mkey;
+			mdev->data_direct->mkey_ro_valid = true;
 		}
 	}
 
@@ -172,13 +174,15 @@ static int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev)
 	return err;
 }
 
-static void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev)
+static void mlx5_data_direct_free_resources(struct mlx5_core_dev *mdev)
 {
-	if (dev->data_direct->mkey_ro_valid)
-		mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey_ro);
+	struct mlx5_data_direct *data_direct = mdev->data_direct;
+
+	if (data_direct->mkey_ro_valid)
+		mlx5_core_destroy_mkey(mdev, data_direct->mkey_ro);
 
-	mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey);
-	mlx5_core_dealloc_pd(dev->mdev, dev->data_direct->pdn);
+	mlx5_core_destroy_mkey(mdev, data_direct->mkey);
+	mlx5_core_dealloc_pd(mdev, data_direct->pdn);
 }
 
 static void mlx5_data_direct_bind(struct mlx5_data_direct_registration *reg,
@@ -195,13 +199,13 @@ mlx5_data_direct_do_unbind(struct mlx5_data_direct_registration *reg)
 				     NULL);
 }
 
-int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev)
+int mlx5_data_direct_init(struct mlx5_core_dev *mdev)
 {
 	struct mlx5_data_direct_registration *reg;
 	struct mlx5_data_direct_dev *dev;
 	int err;
 
-	if (!mlx5_data_direct_supported(ibdev->mdev))
+	if (!mlx5_data_direct_supported(mdev))
 		return 0;
 
 	reg = kzalloc_obj(*reg);
@@ -210,17 +214,18 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev)
 
 	BLOCKING_INIT_NOTIFIER_HEAD(&reg->users);
 
-	err = mlx5_data_direct_query_vuid(ibdev->mdev, reg->vuid);
+	err = mlx5_data_direct_query_vuid(mdev, reg->vuid);
 	if (err) {
-		mlx5_ib_warn(ibdev, "Failed to query VUID, disabling data direct, err=%d\n",
-			     err);
+		mlx5_core_warn(mdev,
+			       "Failed to query VUID, disabling data direct, err=%d\n",
+			       err);
 		kfree(reg);
-		return err;
+		return -EINVAL;
 	}
 
-	ibdev->data_direct = &reg->dd;
+	mdev->data_direct = &reg->dd;
 
-	err = mlx5_data_direct_create_resources(ibdev);
+	err = mlx5_data_direct_create_resources(mdev);
 	if (err)
 		goto err_resources;
 
@@ -233,63 +238,63 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev)
 	}
 
 	/* Add the registration to its global list, to be used upon bind/unbind
-	 * of its affiliated data direct device
+	 * of its affiliated data direct device.
 	 */
 	list_add_tail(&reg->list, &mlx5_data_direct_reg_list);
 	mutex_unlock(&mlx5_data_direct_mutex);
 	return 0;
 
 err_resources:
-	ibdev->data_direct = NULL;
+	mdev->data_direct = NULL;
 	kfree(reg);
 	return err;
 }
 
-void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev)
+void mlx5_data_direct_cleanup(struct mlx5_core_dev *mdev)
 {
 	struct mlx5_data_direct_registration *reg;
 
-	if (!mlx5_data_direct_supported(ibdev->mdev))
+	if (!mlx5_data_direct_supported(mdev))
 		return;
 
-	reg = container_of(ibdev->data_direct,
-			    struct mlx5_data_direct_registration, dd);
+	reg = container_of(mdev->data_direct,
+			   struct mlx5_data_direct_registration, dd);
 	mutex_lock(&mlx5_data_direct_mutex);
 	list_del(&reg->list);
 	mlx5_data_direct_do_unbind(reg);
 	mutex_unlock(&mlx5_data_direct_mutex);
 
-	mlx5_data_direct_free_resources(ibdev);
-	ibdev->data_direct = NULL;
+	mlx5_data_direct_free_resources(mdev);
+	mdev->data_direct = NULL;
 	kfree(reg);
 }
 
-int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev,
+int mlx5_data_direct_register(struct mlx5_core_dev *mdev,
 			      struct notifier_block *nb)
 {
 	struct mlx5_data_direct_registration *reg;
 
-	if (!mlx5_data_direct_supported(ibdev->mdev))
+	if (!mlx5_data_direct_supported(mdev))
 		return 0;
 
-	reg = container_of(ibdev->data_direct,
-			    struct mlx5_data_direct_registration, dd);
+	reg = container_of(mdev->data_direct,
+			   struct mlx5_data_direct_registration, dd);
 	blocking_notifier_chain_register(&reg->users, nb);
 
 	return 0;
 }
 EXPORT_SYMBOL_GPL(mlx5_data_direct_register);
 
-void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev,
+void mlx5_data_direct_unregister(struct mlx5_core_dev *mdev,
 				 struct notifier_block *nb)
 {
 	struct mlx5_data_direct_registration *reg;
 
-	if (!mlx5_data_direct_supported(ibdev->mdev))
+	if (!mlx5_data_direct_supported(mdev))
 		return;
 
-	reg = container_of(ibdev->data_direct,
-			    struct mlx5_data_direct_registration, dd);
+	reg = container_of(mdev->data_direct,
+			   struct mlx5_data_direct_registration, dd);
 	blocking_notifier_chain_unregister(&reg->users, nb);
 }
 EXPORT_SYMBOL_GPL(mlx5_data_direct_unregister);
@@ -304,8 +309,8 @@ static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev)
 			mlx5_data_direct_bind(reg, dev);
 	}
 
-	/* Add the data direct device to the global list, further IB devices may
-	 * use it later as well
+	/* Add the data direct device to the global list, further mlx5 devices
+	 * may use it later as well.
 	 */
 	list_add_tail(&dev->list, &mlx5_data_direct_dev_list);
 	mutex_unlock(&mlx5_data_direct_mutex);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 643b4aac2033..34f0375b6851 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -40,6 +40,7 @@
 #include <linux/interrupt.h>
 #include <linux/delay.h>
 #include <linux/mlx5/driver.h>
+#include <linux/mlx5/data_direct.h>
 #include <linux/mlx5/cq.h>
 #include <linux/mlx5/qp.h>
 #include <linux/debugfs.h>
@@ -1339,8 +1340,14 @@ static int mlx5_load(struct mlx5_core_dev *dev)
 	if (err)
 		goto err_traps_reg;
 
+	err = mlx5_data_direct_init(dev);
+	if (err)
+		goto err_data_direct;
+
 	return 0;
 
+err_data_direct:
+	mlx5_devlink_traps_unregister(priv_to_devlink(dev));
 err_traps_reg:
 	mlx5_sf_dev_table_destroy(dev);
 	mlx5_sriov_detach(dev);
@@ -1371,6 +1378,7 @@ static int mlx5_load(struct mlx5_core_dev *dev)
 
 static void mlx5_unload(struct mlx5_core_dev *dev)
 {
+	mlx5_data_direct_cleanup(dev);
 	mlx5_devlink_traps_unregister(priv_to_devlink(dev));
 	mlx5_vhca_event_stop(dev);
 	mlx5_sf_dev_table_destroy(dev);
@@ -2360,6 +2368,10 @@ static int __init mlx5_init(void)
 	if (err)
 		goto err_sf;
 
+	err = mlx5_data_direct_driver_register();
+	if (err)
+		goto err_dd;
+
 	err = pci_register_driver(&mlx5_core_driver);
 	if (err)
 		goto err_pci;
@@ -2367,6 +2379,8 @@ static int __init mlx5_init(void)
 	return 0;
 
 err_pci:
+	mlx5_data_direct_driver_unregister();
+err_dd:
 	mlx5_sf_driver_unregister();
 err_sf:
 	mlx5e_cleanup();
@@ -2378,6 +2392,7 @@ static int __init mlx5_init(void)
 static void __exit mlx5_cleanup(void)
 {
 	pci_unregister_driver(&mlx5_core_driver);
+	mlx5_data_direct_driver_unregister();
 	mlx5_sf_driver_unregister();
 	mlx5e_cleanup();
 	mlx5_unregister_debugfs();
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
index 09e669f83dba..fd93530a3938 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
@@ -530,4 +530,9 @@ static inline bool mlx5_pcie_cong_event_supported(struct mlx5_core_dev *dev)
 
 	return true;
 }
+
+int mlx5_data_direct_init(struct mlx5_core_dev *mdev);
+void mlx5_data_direct_cleanup(struct mlx5_core_dev *mdev);
+int mlx5_data_direct_driver_register(void);
+void mlx5_data_direct_driver_unregister(void);
 #endif /* __MLX5_CORE_H__ */
diff --git a/include/linux/mlx5/data_direct.h b/include/linux/mlx5/data_direct.h
index eaa2dc6bc607..4cff5ff5489d 100644
--- a/include/linux/mlx5/data_direct.h
+++ b/include/linux/mlx5/data_direct.h
@@ -7,11 +7,23 @@
 #define _MLX5_DATA_DIRECT_H
 
 #include <linux/compiler.h>
+#include <linux/list.h>
 
+struct device;
 struct mlx5_core_dev;
-struct mlx5_data_direct_dev;
-struct mlx5_ib_dev;
 struct notifier_block;
+struct pci_dev;
+
+enum mlx5_data_direct_event {
+	MLX5_DATA_DIRECT_UNBIND,
+};
+
+struct mlx5_data_direct_dev {
+	struct device *device;
+	struct pci_dev *pdev;
+	char *vuid;
+	struct list_head list;
+};
 
 struct mlx5_data_direct {
 	struct mlx5_data_direct_dev *dev;
@@ -27,12 +39,9 @@ mlx5_data_direct_get_dev(struct mlx5_data_direct *dd)
 	return dd ? READ_ONCE(dd->dev) : NULL;
 }
 
-int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev);
-void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev);
-
-int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev,
+int mlx5_data_direct_register(struct mlx5_core_dev *mdev,
 			      struct notifier_block *nb);
-void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev,
+void mlx5_data_direct_unregister(struct mlx5_core_dev *mdev,
 				 struct notifier_block *nb);
 
 #endif
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index ee29f95ead18..ed348a82576e 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -707,6 +707,7 @@ struct mlx5_rsvd_gids {
 
 struct mlx5_clock;
 struct mlx5_clock_dev_state;
+struct mlx5_data_direct;
 struct mlx5_dm;
 struct mlx5_fw_tracer;
 struct mlx5_vxlan;
@@ -801,6 +802,7 @@ struct mlx5_core_dev {
 	/* sync write combining state */
 	struct mutex wc_state_lock;
 	struct devlink *shd;
+	struct mlx5_data_direct *data_direct;
 };
 
 struct mlx5_db {
-- 
2.44.0