Re: [PATCH 09/15] yunsilicon: add RDMA device support

["Xin Tian" <tianx@yunsilicon.com>]

On 10/13/2026 2:17 PM, Junxian Huang wrote:
>
> On 2026/10/12 19:44, Xin Tian wrote:
>> Add the RDMA driver's uverbs ABI header (include/uapi/rdma/xsc-abi.h),
>> the ib_device scaffolding and device registration: query_device,
>> query_port, query_pkey, ucontext allocation and mmap, port attributes and
>> the netdev notifier that tracks the port state.
>>
>> The driver reports RDMA_DRIVER_XSC, which is added to the rdma_driver_id
>> enum at the end so existing IDs are untouched.
>>
>> Signed-off-by: Xin Tian <tianx@yunsilicon.com>
>> ---
>>   drivers/infiniband/Kconfig                    |   1 +
>>   drivers/infiniband/hw/Makefile                |   1 +
>>   drivers/infiniband/hw/xsc/Kconfig             |  17 +
>>   drivers/infiniband/hw/xsc/Makefile            |   9 +
>>   drivers/infiniband/hw/xsc/main.c              | 797 ++++++++++++++++++
>>   drivers/infiniband/hw/xsc/xsc_ib.h            | 542 ++++++++++++
>>   .../net/ethernet/yunsilicon/xsc/common/cq.h   |  30 +
>>   .../ethernet/yunsilicon/xsc/common/driver.h   |  95 +++
>>   .../net/ethernet/yunsilicon/xsc/common/qp.h   | 104 +++
>>   .../net/ethernet/yunsilicon/xsc/common/srq.h  |  54 ++
>>   .../net/ethernet/yunsilicon/xsc/pci/Makefile  |   2 +-
>>   .../net/ethernet/yunsilicon/xsc/pci/link.c    |  35 +
>>   include/uapi/rdma/ib_user_ioctl_verbs.h       |   1 +
>>   include/uapi/rdma/xsc-abi.h                   | 295 +++++++
>>   14 files changed, 1982 insertions(+), 1 deletion(-)
>>   create mode 100644 drivers/infiniband/hw/xsc/Kconfig
>>   create mode 100644 drivers/infiniband/hw/xsc/Makefile
>>   create mode 100644 drivers/infiniband/hw/xsc/main.c
>>   create mode 100644 drivers/infiniband/hw/xsc/xsc_ib.h
>>   create mode 100644 drivers/net/ethernet/yunsilicon/xsc/common/cq.h
>>   create mode 100644 drivers/net/ethernet/yunsilicon/xsc/common/driver.h
>>   create mode 100644 drivers/net/ethernet/yunsilicon/xsc/common/qp.h
>>   create mode 100644 drivers/net/ethernet/yunsilicon/xsc/common/srq.h
>>   create mode 100644 drivers/net/ethernet/yunsilicon/xsc/pci/link.c
>>   create mode 100644 include/uapi/rdma/xsc-abi.h
>>
>> diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig
>> index 086195758..b3cc506db 100644
>> --- a/drivers/infiniband/Kconfig
>> +++ b/drivers/infiniband/Kconfig
>> @@ -100,6 +100,7 @@ source "drivers/infiniband/hw/ocrdma/Kconfig"
>>   source "drivers/infiniband/hw/qedr/Kconfig"
>>   source "drivers/infiniband/hw/usnic/Kconfig"
>>   source "drivers/infiniband/hw/vmw_pvrdma/Kconfig"
>> +source "drivers/infiniband/hw/xsc/Kconfig"
>>   source "drivers/infiniband/sw/rdmavt/Kconfig"
>>   endif # !UML
>>   source "drivers/infiniband/sw/rxe/Kconfig"
>> diff --git a/drivers/infiniband/hw/Makefile b/drivers/infiniband/hw/Makefile
>> index c42b22ac3..1bcd2d50a 100644
>> --- a/drivers/infiniband/hw/Makefile
>> +++ b/drivers/infiniband/hw/Makefile
>> @@ -16,3 +16,4 @@ obj-$(CONFIG_INFINIBAND_BNXT_RE)	+= bnxt_re/
>>   obj-$(CONFIG_INFINIBAND_BNG_RE)		+= bng_re/
>>   obj-$(CONFIG_INFINIBAND_ERDMA)		+= erdma/
>>   obj-$(CONFIG_INFINIBAND_IONIC)		+= ionic/
>> +obj-$(CONFIG_INFINIBAND_XSC)	+= xsc/
>> diff --git a/drivers/infiniband/hw/xsc/Kconfig b/drivers/infiniband/hw/xsc/Kconfig
>> new file mode 100644
>> index 000000000..5cf46df2d
>> --- /dev/null
>> +++ b/drivers/infiniband/hw/xsc/Kconfig
>> @@ -0,0 +1,17 @@
>> +# SPDX-License-Identifier: GPL-2.0
>> +# Copyright (C) 2021 - 2026, Shanghai Yunsilicon Technology Co., Ltd.
>> +# All rights reserved.
>> +
>> +config INFINIBAND_XSC
>> +	tristate "Yunsilicon XSC RDMA driver"
>> +	depends on INFINIBAND && PCI && INET
>> +	depends on YUNSILICON_XSC_PCI && YUNSILICON_XSC_ETH
>> +	help
>> +	  This driver provides RDMA support for Yunsilicon XSC devices.
>> +	  It uses the auxiliary device published by the XSC PCI core and,
>> +	  for now, the netdev registered by the XSC ethernet driver to
>> +	  derive the port state and GID.  Because of that netdev
>> +	  dependency it also depends on YUNSILICON_XSC_ETH.
>> +
>> +	  To compile this driver as a module, choose M here. The module
>> +	  will be called xsc_ib.
>> diff --git a/drivers/infiniband/hw/xsc/Makefile b/drivers/infiniband/hw/xsc/Makefile
>> new file mode 100644
>> index 000000000..6c4ec1ca0
>> --- /dev/null
>> +++ b/drivers/infiniband/hw/xsc/Makefile
>> @@ -0,0 +1,9 @@
>> +# SPDX-License-Identifier: GPL-2.0
>> +# Copyright (C) 2021 - 2026, Shanghai Yunsilicon Technology Co., Ltd.
>> +# All rights reserved.
>> +
>> +ccflags-y := -I$(srctree)/drivers/net/ethernet/yunsilicon/xsc
>> +
>> +obj-$(CONFIG_INFINIBAND_XSC) += xsc_ib.o
>> +
>> +xsc_ib-y := main.o
>> diff --git a/drivers/infiniband/hw/xsc/main.c b/drivers/infiniband/hw/xsc/main.c
>> new file mode 100644
>> index 000000000..9d53a89be
>> --- /dev/null
>> +++ b/drivers/infiniband/hw/xsc/main.c
>> @@ -0,0 +1,797 @@
>> +// SPDX-License-Identifier: GPL-2.0
>> +/*
>> + * Copyright (C) 2021 - 2026, Shanghai Yunsilicon Technology Co., Ltd.
>> + * All rights reserved.
>> + */
>> +
>> +#include <linux/module.h>
>> +#include <linux/auxiliary_bus.h>
>> +#include <linux/init.h>
>> +#include <linux/errno.h>
>> +#include <linux/pci.h>
>> +#include <linux/namei.h>
>> +#include <linux/kobject.h>
>> +#include <linux/kernfs.h>
>> +#include <linux/dma-mapping.h>
>> +#include <linux/slab.h>
>> +#include <linux/io-mapping.h>
>> +#include <linux/sched.h>
>> +#include <linux/netdevice.h>
>> +#include "common/xsc_core.h"
>> +#include "common/xsc_hsi.h"
>> +#include "common/xsc_cmd.h"
>> +#include "common/driver.h"
>> +
>> +#include <rdma/ib_user_verbs.h>
>> +#include <rdma/ib_smi.h>
>> +#include <rdma/ib_umem.h>
>> +
>> +#include <rdma/xsc-abi.h>
>> +#include "xsc_ib.h"
>> +
>> +#define DRIVER_NAME "xsc_ib"
>> +
>> +#define XSC_IB_GID_TBL_LEN	256
>> +#define XSC_IB_MAX_PKEYS	0x80
>> +
>> +MODULE_DESCRIPTION("Yunsilicon HCA IB driver");
>> +MODULE_LICENSE("GPL");
>> +
>> +static int xsc_ib_query_device(struct ib_device *ibdev,
>> +			       struct ib_device_attr *props,
>> +			       struct ib_udata *udata)
> Are the comments in this function really needed? I think the codes
> are self-explainable enough.
Hi Junxian,

All ten are fixed in v1, which I'll post shortly.
>> +{
>> +	size_t udata_outlen = (udata) ? udata->outlen : 0;
>> +	struct xsc_ib_dev *dev = to_xdev(ibdev);
>> +	struct xsc_ib_query_device_resp resp;
>> +	union xsc_ib_fw_ver fw_ver;
>> +	int err = -ENOMEM;
>> +	size_t resp_len;
>> +	int max_rq_sg;
>> +	u64 max_tso;
>> +	u64 flags;
>> +
>> +	err = ib_is_udata_in_empty(udata);
>> +	if (err)
>> +		return err;
>> +
>> +	memset(&resp, 0, sizeof(resp));
>> +	memset(props, 0, sizeof(*props));
> The core already handles the zeroing, no need to do this in driver.
Fixed.
>
>> +
>> +	resp_len = sizeof(resp.comp_mask) + sizeof(resp.response_length);
>> +	/*check param*/
>> +	if (udata_outlen && udata_outlen < resp_len)
>> +		return -EINVAL;
>> +
>> +	resp.response_length = resp_len;
>> +
>> +	fw_ver.data = 0;
>> +	fw_ver.s.ver_major = dev->xdev->fw_version_major;
>> +	fw_ver.s.ver_minor = dev->xdev->fw_version_minor;
>> +	fw_ver.s.ver_patch = dev->xdev->fw_version_patch;
>> +	fw_ver.s.ver_tweak = dev->xdev->fw_version_tweak;
>> +	props->fw_ver = fw_ver.data;
>> +
>> +	props->device_cap_flags    = IB_DEVICE_CHANGE_PHY_PORT |
>> +		IB_DEVICE_PORT_ACTIVE_EVENT		|
>> +		IB_DEVICE_SYS_IMAGE_GUID		|
>> +		IB_DEVICE_RC_RNR_NAK_GEN;
>> +	props->kernel_cap_flags =  IBK_BLOCK_MULTICAST_LOOPBACK;
>> +	props->kernel_cap_flags |= IBK_LOCAL_DMA_LKEY;
>> +
>> +	flags = dev->xdev->caps.flags;
>> +	if (flags & XSC_DEV_CAP_FLAG_BAD_PKEY_CNTR)
>> +		props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
>> +	if (flags & XSC_DEV_CAP_FLAG_BAD_QKEY_CNTR)
>> +		props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
>> +	if (flags & XSC_DEV_CAP_FLAG_APM)
>> +		props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
>> +	props->page_size_cap	   = dev->xdev->caps.min_page_sz;
>> +	props->max_mr_size	   = xsc_get_max_mr_size(dev->xdev);
>> +	props->max_qp		   = dev->xdev->caps.max_qp;
>> +	props->max_qp_wr	   = xsc_get_max_qp_depth(dev->xdev);
>> +	max_rq_sg = dev->xdev->caps.max_rq_desc_sz / sizeof(struct xsc_wqe_data_seg);
>> +
>> +	props->max_send_sge = dev->xdev->caps.send_ds_num - XSC_CTRL_SEG_NUM -
>> +		XSC_RADDR_SEG_NUM;
>> +	props->max_recv_sge = dev->xdev->caps.recv_ds_num;
>> +	props->max_sge_rd	   = 1;/*max sge per read wqe*/
>> +	props->max_cq		   = dev->xdev->caps.max_cq;
>> +	props->max_cqe		   = dev->xdev->caps.max_cqes;
>> +	props->max_mr		   = (1 << dev->xdev->caps.log_max_mkey) - 1;
>> +	props->max_pd		   = dev->xdev->caps.max_pd;
>> +	props->max_qp_rd_atom	   = dev->xdev->caps.max_ra_res_qp;
>> +	props->max_qp_init_rd_atom = dev->xdev->caps.max_ra_req_qp;
>> +	props->max_res_rd_atom	   = props->max_qp_rd_atom * props->max_qp;
>> +	props->max_srq		   =
>> +		dev->xdev->caps.log_max_srq ? (1 << dev->xdev->caps.log_max_srq) : 0;
>> +	props->max_srq_wr	   = dev->xdev->caps.max_srq_wqes - 1;
>> +	props->max_srq_sge	   = dev->xdev->caps.log_max_srq ? (max_rq_sg - 1) : 0;
>> +	props->local_ca_ack_delay  = dev->xdev->caps.local_ca_ack_delay;
>> +	props->atomic_cap	   = is_support_rdma_atomic(dev->xdev) ?
>> +		IB_ATOMIC_HCA : IB_ATOMIC_NONE;
>> +	props->masked_atomic_cap   = IB_ATOMIC_HCA;
>> +	props->max_mcast_grp	   =
>> +		dev->xdev->caps.log_max_mcg ? (1 << dev->xdev->caps.log_max_mcg) : 0;
>> +	props->max_mcast_qp_attach = dev->xdev->caps.max_qp_mcg;
>> +	props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
>> +					   props->max_mcast_grp;
>> +
>> +	props->sys_image_guid = dev->xdev->board_info->guid;
>> +	props->vendor_id = dev->xdev->pdev->vendor;
>> +	props->vendor_part_id = dev->xdev->pdev->device;
>> +	props->hw_ver = ((dev->xdev->chip_ver_l & 0xffff) << 16) |
>> +		(dev->xdev->hotfix_num & 0xffff);
>> +	props->max_pkeys = XSC_IB_MAX_PKEYS;
>> +	props->max_wq_type_rq = dev->xdev->caps.max_qp;
>> +	props->hca_core_clock = dev->xdev->caps.hca_core_clock * 1000; /* KHz */
>> +	props->rss_caps.max_rwq_indirection_tables =
>> +		dev->xdev->caps.max_rwq_indirection_tables;
>> +	props->rss_caps.max_rwq_indirection_table_size =
>> +		dev->xdev->caps.max_rwq_indirection_table_size;
>> +	props->rss_caps.supported_qpts = 1 << IB_QPT_RAW_PACKET;
>> +
>> +	/*response tso_caps extend param*/
>> +	if (offsetofend(typeof(resp), tso_caps) <= udata_outlen) {
>> +		max_tso = dev->xdev->caps.log_max_tso ? (1 << dev->xdev->caps.log_max_tso) : 0;
>> +		if (max_tso) {
>> +			resp.tso_caps.max_tso = max_tso;
>> +			resp.tso_caps.supported_qpts |= 1 << IB_QPT_RAW_PACKET;
>> +			resp.response_length += sizeof(resp.tso_caps);
>> +		}
>> +	}
>> +
>> +	/*response rss_caps extend param*/
>> +	if (offsetofend(typeof(resp), rss_caps) <= udata_outlen) {
>> +		resp.rss_caps.rx_hash_function = XSC_RX_HASH_FUNC_TOEPLITZ;
>> +		resp.rss_caps.rx_hash_fields_mask =
>> +			XSC_RX_HASH_SRC_IPV4 |
>> +			XSC_RX_HASH_DST_IPV4 |
>> +			XSC_RX_HASH_SRC_IPV6 |
>> +			XSC_RX_HASH_DST_IPV6 |
>> +			XSC_RX_HASH_SRC_PORT_TCP |
>> +			XSC_RX_HASH_DST_PORT_TCP |
>> +			XSC_RX_HASH_SRC_PORT_UDP |
>> +			XSC_RX_HASH_DST_PORT_UDP |
>> +			XSC_RX_HASH_INNER;
>> +		resp.response_length += sizeof(resp.rss_caps);
>> +	}
>> +
>> +	/*response packet pacing caps*/
>> +	if (offsetofend(typeof(resp), packet_pacing_caps) <= udata_outlen) {
>> +		resp.packet_pacing_caps.qp_rate_limit_max =
>> +			dev->xdev->caps.qp_rate_limit_max;
>> +		resp.packet_pacing_caps.qp_rate_limit_min =
>> +			dev->xdev->caps.qp_rate_limit_min;
>> +		resp.packet_pacing_caps.supported_qpts |= 1 << IB_QPT_RAW_PACKET;
>> +
>> +		resp.response_length += sizeof(resp.packet_pacing_caps);
>> +	}
>> +
>> +	/*copy response data to user*/
>> +	if (udata_outlen) {
>> +		err = ib_copy_to_udata(udata, &resp, resp.response_length);
>> +		if (err) {
>> +			ibdev_err(&dev->ib_dev, "copy response info to udata fail,err=%d\n", err);
>> +			return err;
>> +		}
>> +	}
>> +
>> +	return 0;
>> +}
>> +
>> +static void xsc_calc_link_info(struct xsc_core_device *xdev,
>> +			       struct ib_port_attr *props)
>> +{
>> +	switch (xsc_get_link_speed(xdev)) {
>> +	case MODULE_SPEED_10G:
>> +		props->active_speed = XSC_RDMA_LINK_SPEED_10GB;
>> +		props->active_width = 1;
>> +		break;
>> +	case MODULE_SPEED_25G:
>> +		props->active_speed = XSC_RDMA_LINK_SPEED_25GB;
>> +		props->active_width = 1;
>> +		break;
>> +	case MODULE_SPEED_40G_R4:
>> +		props->active_speed = XSC_RDMA_LINK_SPEED_10GB;
>> +		props->active_width = 2;
>> +		break;
>> +	case MODULE_SPEED_50G_R:
>> +		props->active_speed = XSC_RDMA_LINK_SPEED_50GB;
>> +		props->active_width = 1;
>> +		break;
>> +	case MODULE_SPEED_50G_R2:
>> +		props->active_speed = XSC_RDMA_LINK_SPEED_50GB;
>> +		props->active_width = 1;
>> +		break;
>> +	case MODULE_SPEED_100G_R2:
>> +		props->active_speed = XSC_RDMA_LINK_SPEED_25GB;
>> +		props->active_width = 2;
>> +		break;
>> +	case MODULE_SPEED_100G_R4:
>> +		props->active_speed = XSC_RDMA_LINK_SPEED_25GB;
>> +		props->active_width = 2;
>> +		break;
>> +	case MODULE_SPEED_200G_R4:
>> +		props->active_speed = XSC_RDMA_LINK_SPEED_50GB;
>> +		props->active_width = 2;
>> +		break;
>> +	case MODULE_SPEED_200G_R8:
>> +		props->active_speed = XSC_RDMA_LINK_SPEED_25GB;
>> +		props->active_width = 4;
>> +		break;
>> +	case MODULE_SPEED_400G_R8:
>> +		props->active_speed = XSC_RDMA_LINK_SPEED_50GB;
>> +		props->active_width = 4;
>> +		break;
>> +	case MODULE_SPEED_400G_R4:
>> +		props->active_speed = XSC_RDMA_LINK_SPEED_100GB;
>> +		props->active_width = 2;
>> +		break;
>> +	default:
>> +		props->active_speed = XSC_RDMA_LINK_SPEED_25GB;
>> +		props->active_width = 1;
>> +		break;
>> +	}
>> +}
>> +
>> +static enum rdma_link_layer xsc_ib_port_link_layer(struct ib_device *ibdev, u32 port)
>> +{
>> +	return IB_LINK_LAYER_ETHERNET;
>> +}
>> +
>> +int xsc_ib_query_port(struct ib_device *ibdev, u32 port,
>> +		      struct ib_port_attr *props)
>> +{
>> +	struct xsc_ib_dev *dev = to_xdev(ibdev);
>> +	struct net_device *ndev = dev->netdev;
>> +	struct xsc_core_device *xdev = dev->xdev;
>> +
>> +	if (port < 1 || port > xdev->caps.num_ports) {
>> +		ibdev_warn(&dev->ib_dev, "invalid port number %d\n", port);
>> +		return -EINVAL;
>> +	}
>> +
>> +	memset(props, 0, sizeof(*props));
>> +
>> +	if (netif_running(ndev) && netif_carrier_ok(ndev))
>> +		props->state = IB_PORT_ACTIVE;
>> +	else
>> +		props->state = IB_PORT_DOWN;
> You can use ib_get_curr_port_state() instead of open-coding.
Fixed.
>> +
>> +	props->max_mtu = is_support_8k_mtu(xdev) ? IB_MTU_8192 : IB_MTU_4096;
>> +	props->active_mtu = min(props->max_mtu, xsc_net_to_ib_mtu(ndev->mtu));
> Similarly, use iboe_get_mtu().
Fixed.
>> +	props->gid_tbl_len = XSC_IB_GID_TBL_LEN;
>> +	props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_OTHER_LOCAL_CHANGES_SUP;
>> +	props->max_msg_sz = SZ_1G;
>> +	props->bad_pkey_cntr = 0;
>> +	props->qkey_viol_cntr = 0;
>> +	props->pkey_tbl_len = 1;
>> +	props->lid = 0;
>> +	props->sm_lid = 0;
>> +	props->lmc = 0;
>> +	props->max_vl_num = 0;
>> +	props->sm_sl = 0;
>> +	props->subnet_timeout = 0;
>> +	props->init_type_reply = 0;
>> +	if (!is_support_rdma(xdev)) {
>> +		props->active_width = 1;
>> +		props->active_speed = XSC_RDMA_LINK_SPEED_25GB;
>> +	} else {
>> +		xsc_calc_link_info(xdev, props);
>> +	}
>> +
>> +	props->phys_state = netif_carrier_ok(ndev) ? XSC_RDMA_PHY_STATE_LINK_UP :
>> +				XSC_RDMA_PHY_STATE_DISABLED;
>> +	return 0;
>> +}
>> +
>> +static int xsc_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index,
>> +			     u16 *pkey)
>> +{
>> +	*pkey = 0xffff;
>> +	return 0;
>> +}
>> +
>> +int xsc_ib_alloc_ucontext(struct ib_ucontext *uctx,
>> +			  struct ib_udata *udata)
>> +{
>> +	struct ib_device *ibdev = uctx->device;
>> +	struct xsc_ib_dev *dev = to_xdev(ibdev);
>> +	struct xsc_ib_alloc_ucontext_req req;
>> +	struct xsc_ib_alloc_ucontext_resp resp;
>> +	int err;
>> +
>> +	if (!dev->ib_active)
>> +		return -EAGAIN;
>> +
>> +	err = ib_copy_from_udata(&req, udata, sizeof(req));
>> +	if (err)
>> +		return err;
>> +
>> +	memset(&resp, 0, sizeof(resp));
>> +	resp.qp_tab_size      = dev->xdev->caps.max_qp;
>> +	resp.cache_line_size  = L1_CACHE_BYTES;
>> +	resp.max_sq_desc_sz = dev->xdev->caps.max_sq_desc_sz;
>> +	resp.max_rq_desc_sz = dev->xdev->caps.max_rq_desc_sz;
>> +	resp.max_send_wqebb = dev->xdev->caps.max_wqes;
>> +	resp.max_recv_wr = dev->xdev->caps.max_wqes;
>> +	xsc_get_db_addr(dev->xdev, &resp.qpm_tx_db, &resp.qpm_rx_db,
>> +			&resp.cqm_armdb, &resp.cqm_next_cid_reg, NULL);
>> +	resp.send_ds_num = dev->xdev->caps.send_ds_num;
>> +	resp.recv_ds_num = dev->xdev->caps.recv_ds_num;
>> +	resp.cmds_supp_uhw |= XSC_USER_CMDS_SUPP_UHW_QUERY_DEVICE;
>> +	resp.device_id = dev->xdev->pdev->device;
>> +
>> +	if (req.abi_ver >= ALLOC_UCONTEXT_VER_V1) {
>> +		xsc_get_multidb_info(dev->xdev, &resp.multidb_num, &resp.tx_multidb_base);
>> +		resp.rdma_proto_mode = dev->xdev->rdma_proto_mode;
>> +	}
>> +
>> +	if (is_support_cqe64(dev->xdev))
>> +		resp.hw_feature_flag |= XSC_HW_FEATURE_FLAG_SUPPORT_CQE64;
>> +
>> +	resp.num_ports = dev->xdev->caps.num_ports;
>> +
>> +	return ib_copy_to_udata(udata, &resp, min_t(int, udata->outlen, sizeof(resp)));
>> +}
>> +
>> +void xsc_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
>> +{
>> +}
>> +
>> +static int xsc_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
>> +{
>> +	struct xsc_ib_dev *dev = to_xdev(ibcontext->device);
>> +	struct xsc_core_device *xdev = dev->xdev;
>> +	unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
>> +	resource_size_t reg_base;
>> +	resource_size_t reg_size = vma->vm_end - vma->vm_start;
>> +	u64 tx_db = 0;
>> +	u64 rx_db = 0;
>> +	u64 cq_db = 0;
>> +	u64 cq_reg = 0;
>> +
>> +	xsc_get_db_addr(xdev, &tx_db, &rx_db, &cq_db, &cq_reg, NULL);
>> +	dev_dbg(&xdev->pdev->dev, "offset:0x%lx\n", offset);
>> +
>> +	if (offset == (tx_db & PAGE_MASK))
>> +		reg_base = pci_resource_start(xdev->pdev, xdev->bar_num) + (tx_db & PAGE_MASK);
>> +	else if (offset == (rx_db & PAGE_MASK))
>> +		reg_base = pci_resource_start(xdev->pdev, xdev->bar_num) + (rx_db & PAGE_MASK);
>> +	else if (offset == (cq_reg & PAGE_MASK))
>> +		reg_base = pci_resource_start(xdev->pdev, xdev->bar_num) + (cq_reg & PAGE_MASK);
>> +	else if (offset == (cq_db & PAGE_MASK))
>> +		reg_base = pci_resource_start(xdev->pdev, xdev->bar_num) + (cq_db & PAGE_MASK);
>> +	else if (offset == (QPM_TX_MDB_BASE_REG_ADDR & PAGE_MASK))
>> +		reg_base = pci_resource_start(xdev->pdev, xdev->bar_num) +
>> +			(QPM_TX_MDB_BASE_REG_ADDR & PAGE_MASK);
>> +	else if (offset == (xdev->caps.mdb_base & PAGE_MASK))
>> +		reg_base = pci_resource_start(xdev->pdev, xdev->bar_num) +
>> +			(xdev->caps.mdb_base & PAGE_MASK);
>> +	else
>> +		return -EINVAL;
>> +
>> +	dev_dbg(&xdev->pdev->dev, "regbase:0x%llx", reg_base);
>> +
>> +	reg_base = (xsc_core_is_pf(xdev) && !is_pf_bar_compressed(xdev)) ?
>> +		reg_base - 0xA0000000 : reg_base;
>> +
>> +	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
>> +	return remap_pfn_range(vma, vma->vm_start, reg_base >> PAGE_SHIFT,
>> +			       reg_size, vma->vm_page_prot);
>> +}
>> +
>> +static int xsc_port_immutable(struct ib_device *ibdev, u32 port_num,
>> +			      struct ib_port_immutable *immutable)
>> +{
>> +	struct ib_port_attr attr;
>> +	int err;
>> +
>> +	err = ib_query_port(ibdev, port_num, &attr);
>> +	if (err)
>> +		return err;
>> +
>> +	immutable->pkey_tbl_len = attr.pkey_tbl_len;
>> +	immutable->gid_tbl_len = attr.gid_tbl_len;
>> +	immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE |
>> +				    RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
>> +	immutable->max_mad_size = IB_MGMT_MAD_SIZE * 2;
>> +
>> +	return 0;
>> +}
>> +
>> +static struct net_device *xsc_get_netdev(struct ib_device *ibdev, u32 port_num)
> You should use ib_device_set_netdev() instead of implementing this op.
Fixed.
>> +{
>> +	struct xsc_ib_dev *xsc_ib_dev = to_xdev(ibdev);
>> +	struct net_device *dev = xsc_ib_dev->netdev;
>> +
>> +	if (dev)
>> +		dev_hold(dev);
>> +
>> +	return dev;
>> +}
>> +
>> +static void xsc_get_guid(const u8 *dev_addr, u8 *guid)
>> +{
>> +	u8 mac[ETH_ALEN];
>> +
>> +	/* MAC-48 to EUI-64 mapping */
>> +	memcpy(mac, dev_addr, ETH_ALEN);
>> +	guid[0] = mac[0] ^ 2;
>> +	guid[1] = mac[1];
>> +	guid[2] = mac[2];
>> +	guid[3] = 0xff;
>> +	guid[4] = 0xfe;
>> +	guid[5] = mac[3];
>> +	guid[6] = mac[4];
>> +	guid[7] = mac[5];
>> +}
>> +
>> +static void init_node_data(struct xsc_ib_dev *dev)
>> +{
>> +	strscpy(dev->ib_dev.node_desc, "xsc_node_desc",
>> +		sizeof(dev->ib_dev.node_desc));
>> +	xsc_get_guid(dev->netdev->dev_addr, (u8 *)&dev->ib_dev.node_guid);
>> +}
>> +
>> +static void xsc_ib_event(struct xsc_core_device *dev, void *context,
>> +			 enum xsc_dev_event event, unsigned long data)
>> +{
>> +	struct xsc_ib_dev *ibdev = (struct xsc_ib_dev *)context;
>> +	struct ib_event ibev;
>> +	u8 port = 0;
>> +
>> +	switch (event) {
>> +	case XSC_DEV_EVENT_SYS_ERROR:
>> +		ibdev->ib_active = false;
>> +		ibev.event = IB_EVENT_DEVICE_FATAL;
>> +		break;
>> +
>> +	case XSC_DEV_EVENT_PORT_UP:
>> +		ibev.event = IB_EVENT_PORT_ACTIVE;
>> +		port = *(u8 *)data;
>> +		break;
>> +
>> +	case XSC_DEV_EVENT_PORT_DOWN:
>> +		ibev.event = IB_EVENT_PORT_ERR;
>> +		port = *(u8 *)data;
>> +		break;
>> +
>> +	case XSC_DEV_EVENT_PORT_INITIALIZED:
>> +		/* not used by ULPs */
>> +		return;
>> +
>> +	case XSC_DEV_EVENT_LID_CHANGE:
>> +		ibev.event = IB_EVENT_LID_CHANGE;
>> +		port = *(u8 *)data;
>> +		break;
>> +
>> +	case XSC_DEV_EVENT_PKEY_CHANGE:
>> +		ibev.event = IB_EVENT_PKEY_CHANGE;
>> +		port = *(u8 *)data;
>> +		break;
>> +
>> +	case XSC_DEV_EVENT_GUID_CHANGE:
>> +		ibev.event = IB_EVENT_GID_CHANGE;
>> +		port = *(u8 *)data;
>> +		break;
>> +
>> +	case XSC_DEV_EVENT_CLIENT_REREG:
>> +		ibev.event = IB_EVENT_CLIENT_REREGISTER;
>> +		port = *(u8 *)data;
>> +		break;
>> +	}
>> +
>> +	ibev.device	      = &ibdev->ib_dev;
>> +	ibev.element.port_num = port;
>> +
>> +	if (ibdev->ib_active)
>> +		ib_dispatch_event(&ibev);
>> +}
>> +
>> +static void xsc_ib_get_dev_fw_str(struct ib_device *ibdev, char *str)
>> +{
>> +	struct xsc_core_device *dev = to_xdev(ibdev)->xdev;
>> +	u8 ver_major = dev->fw_version_major;
>> +	u8 ver_minor = dev->fw_version_minor;
>> +	u16 ver_patch = dev->fw_version_patch;
>> +	u32 ver_tweak = dev->fw_version_tweak;
>> +
>> +	if (ver_tweak == 0) {
>> +		snprintf(str, IB_FW_VERSION_NAME_MAX, "v%u.%u.%u",
>> +			 ver_major,  ver_minor, ver_patch);
>> +	} else {
>> +		snprintf(str, IB_FW_VERSION_NAME_MAX, "v%u.%u.%u+%u",
>> +			 ver_major, ver_minor, ver_patch, ver_tweak);
>> +	}
>> +}
>> +
>> +static void xsc_ib_disassociate_ucontext(struct ib_ucontext *ibcontext)
>> +{
>> +}
>> +
>> +static void xsc_ib_dev_setting(struct xsc_ib_dev *dev)
>> +{
>> +	dev->ib_dev.ops.owner		= THIS_MODULE;
>> +	dev->ib_dev.ops.uverbs_abi_ver	= XSC_IB_UVERBS_ABI_VERSION;
>> +	dev->ib_dev.ops.driver_id = RDMA_DRIVER_XSC;
>> +	dev->ib_dev.ops.disassociate_ucontext = xsc_ib_disassociate_ucontext;
>> +	dev->ib_dev.ops.query_device	= xsc_ib_query_device;
>> +	dev->ib_dev.ops.query_port	= xsc_ib_query_port;
>> +	dev->ib_dev.ops.query_pkey	= xsc_ib_query_pkey;
>> +
>> +	dev->ib_dev.ops.alloc_ucontext	= xsc_ib_alloc_ucontext;
>> +	dev->ib_dev.ops.dealloc_ucontext = xsc_ib_dealloc_ucontext;
>> +	dev->ib_dev.ops.mmap		= xsc_ib_mmap;
>> +
>> +
>> +	dev->ib_dev.ops.get_link_layer	= xsc_ib_port_link_layer;
>> +	dev->ib_dev.ops.get_netdev		= xsc_get_netdev;
>> +
>> +
>> +	dev->ib_dev.ops.get_port_immutable		= xsc_port_immutable;
>> +
>> +	dev->ib_dev.ops.get_dev_fw_str	= xsc_ib_get_dev_fw_str;
>> +
>> +	dev->ib_dev.ops INIT_RDMA_OBJ_SIZE(ib_ucontext, xsc_ib_ucontext, ibucontext);
>> +}
>> +
>> +static void xsc_get_port_state(struct net_device *ndev, enum xsc_dev_event *ev)
>> +{
>> +	*ev = XSC_DEV_EVENT_PORT_DOWN;
>> +	if (netif_running(ndev) && netif_carrier_ok(ndev))
>> +		*ev = XSC_DEV_EVENT_PORT_UP;
>> +}
>> +
>> +static int xsc_netdev_event(struct notifier_block *this, unsigned long event, void *ptr)
>> +{
>> +	struct xsc_ib_dev *ibdev = container_of(this, struct xsc_ib_dev, nb);
>> +	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
>> +	enum xsc_dev_event ev;
>> +	u8 port = 1;
>> +
>> +	if (ndev != ibdev->netdev)
>> +		goto done;
>> +
>> +	ibdev_dbg(&ibdev->ib_dev, "netdev notfiy event:%ld\n", event);
>> +	switch (event) {
>> +	case NETDEV_CHANGE:
>> +	case NETDEV_UP:
>> +	case NETDEV_DOWN:
>> +		xsc_get_port_state(ibdev->netdev, &ev);
>> +		xsc_ib_event(ibdev->xdev, ibdev, ev, (unsigned long)&port);
>> +		break;
>> +	default:
>> +		break;
>> +	}
>> +done:
>> +	return NOTIFY_DONE;
>> +}
>> +
>> +static int xsc_register_netdev_notifier(struct xsc_ib_dev *ibdev)
>> +{
>> +	ibdev->nb.notifier_call = xsc_netdev_event;
>> +	return register_netdevice_notifier(&ibdev->nb);
> ib_netdevice_event() already handles these events. You can also implement
> .report_port_event() if needed. Anyhow you probably don't need this
> notifier in driver.
>
>> +}
>> +
>> +static int xsc_unregister_netdev_notifier(struct xsc_ib_dev *ibdev)
>> +{
>> +	return unregister_netdevice_notifier(&ibdev->nb);
>> +}
>> +
>> +static int init_one(struct xsc_core_device *xdev,
>> +		    struct xsc_ib_dev **ibdev)
>> +{
>> +	struct xsc_ib_dev *dev;
>> +	int err;
>> +
>> +	dev = (struct xsc_ib_dev *)ib_alloc_device(xsc_ib_dev, ib_dev);
>> +	if (!dev)
>> +		return -ENOMEM;
>> +
>> +	dev->xdev = xdev;
>> +	dev->netdev = xdev->netdev;
>> +	strscpy(dev->ib_dev.name, "xscale_%d", IB_DEVICE_NAME_MAX);
>> +
>> +	memcpy(dev->xdev->priv.ibdev_name, dev->ib_dev.name, IB_DEVICE_NAME_MAX);
>> +
>> +	if (!xdev->caps.msix_enable)
>> +		dev->num_comp_vectors = 1;
>> +	else
>> +		dev->num_comp_vectors = xdev->dev_res->eq_table.num_comp_vectors;
>> +
>> +	dev->ib_dev.node_type		= RDMA_NODE_IB_CA;
>> +	dev->ib_dev.local_dma_lkey	= 0xFF;
>> +	dev->num_ports		= xdev->caps.num_ports;
>> +	dev->ib_dev.phys_port_cnt     = dev->num_ports;
>> +	dev->ib_dev.num_comp_vectors	= dev->num_comp_vectors;
>> +	dev->ib_dev.dev.parent = &xdev->pdev->dev;
>> +	xsc_ib_dev_setting(dev);
>> +	dev->xdev->priv.cm_dscp = DSCP_PCP_UNSET;
>> +	dev->xdev->priv.cm_pcp = DSCP_PCP_UNSET;
>> +	dev->xdev->priv.force_pcp = DSCP_PCP_UNSET;
>> +	dev->xdev->priv.force_dscp = DSCP_PCP_UNSET;
>> +	dev->xdev->rdma_proto_mode = RDMA_PROTO_ROCEV2;
>> +	dev->xdev->cm_udp_dst_port = ROCE_V2_UDP_DPORT;
>> +	dev->xdev->veroce_udp_dst_port = VEROCE_UDP_DPORT;
>> +
>> +	dev->ib_dev.uverbs_cmd_mask	=
>> +		(1ull << IB_USER_VERBS_CMD_GET_CONTEXT)		|
>> +		(1ull << IB_USER_VERBS_CMD_QUERY_DEVICE)	|
>> +		(1ull << IB_USER_VERBS_CMD_QUERY_PORT)		|
>> +		(1ull << IB_USER_VERBS_CMD_ALLOC_PD)		|
>> +		(1ull << IB_USER_VERBS_CMD_DEALLOC_PD)		|
>> +		(1ull << IB_USER_VERBS_CMD_CREATE_AH)		|
>> +		(1ull << IB_USER_VERBS_CMD_DESTROY_AH)		|
>> +		(1ull << IB_USER_VERBS_CMD_REG_MR)		|
>> +		(1ull << IB_USER_VERBS_CMD_REREG_MR)		|
>> +		(1ull << IB_USER_VERBS_CMD_DEREG_MR)		|
>> +		(1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL)	|
>> +		(1ull << IB_USER_VERBS_CMD_CREATE_CQ)		|
>> +		(1ull << IB_USER_VERBS_CMD_RESIZE_CQ)		|
>> +		(1ull << IB_USER_VERBS_CMD_DESTROY_CQ)		|
>> +		(1ull << IB_USER_VERBS_CMD_CREATE_QP)		|
>> +		(1ull << IB_USER_VERBS_CMD_MODIFY_QP)		|
>> +		(1ull << IB_USER_VERBS_CMD_QUERY_QP)		|
>> +		(1ull << IB_USER_VERBS_CMD_DESTROY_QP)		|
>> +		(1ull << IB_USER_VERBS_CMD_ATTACH_MCAST)	|
>> +		(1ull << IB_USER_VERBS_CMD_DETACH_MCAST)	|
>> +		(1ull << IB_USER_VERBS_CMD_CREATE_SRQ)		|
>> +		(1ull << IB_USER_VERBS_CMD_MODIFY_SRQ)		|
>> +		(1ull << IB_USER_VERBS_CMD_QUERY_SRQ)		|
>> +		(1ull << IB_USER_VERBS_CMD_DESTROY_SRQ)		|
>> +		(1ull << IB_USER_VERBS_CMD_OPEN_QP);
>> +
>> +	init_node_data(dev);
>> +
>> +
>> +
>> +
> Too many blank lines.
Fixed.
>> +	err = ib_register_device(&dev->ib_dev, dev->ib_dev.name,
>> +				 dev->xdev->device);
>> +	if (err)
>> +		goto err_free;
>> +
>> +	rdma_roce_rescan_device(&dev->ib_dev);
> This is already called in
> ib_register_device → ib_cache_setup_one → gid_table_setup_one
Fixed.
>> +	dev->ib_active = true;
>> +	*ibdev = dev;
>> +
>> +	xsc_register_netdev_notifier(dev);
>> +
>> +
>> +
> Too many blank lines.
>
>> +	return 0;
>> +
>> +err_free:
>> +	ib_dealloc_device((struct ib_device *)dev);
>> +
>> +	return err;
>> +}
>> +
>> diff --git a/drivers/infiniband/hw/xsc/xsc_ib.h b/drivers/infiniband/hw/xsc/xsc_ib.h
>> new file mode 100644
>> index 000000000..a2ae382e2
>> --- /dev/null
>> +++ b/drivers/infiniband/hw/xsc/xsc_ib.h
>> @@ -0,0 +1,542 @@
>> +static inline void *xsc_ib_iova_to_virt(struct ib_device *ibdev, dma_addr_t iova)
>> +{
>> +	phys_addr_t phyaddr;
>> +	struct iommu_domain *domain;
>> +
>> +	domain = iommu_get_domain_for_dev(ibdev->dma_device);
>> +	if (likely(domain)) {
>> +		phyaddr = iommu_iova_to_phys(domain, iova);
>> +		phyaddr |= iova & (PAGE_SIZE - 1);
>> +	} else {
>> +		phyaddr = dma_to_phys(ibdev->dma_device, iova);
>> +	}
>> +
>> +	return phys_to_virt(phyaddr);
>> +}
>> +
>> +struct ib_mad_list_head {
> Why are these ib_mad structures defined again in the driver's header?
>
> Junxian
They were a workaround for historical hardware issues and are no longer

needed. They have been removed in v1.

Thanks,
Xin Tian

>
>> +	struct list_head list;
>> +	struct ib_cqe cqe;
>> +	struct ib_mad_queue *mad_queue;
>> +};
>> +
>> +#define IB_MAD_SEND_REQ_MAX_SG 2
>> +struct ib_mad_send_wr_private {
>> +	struct ib_mad_list_head mad_list;
>> +	struct list_head agent_list;
>> +	struct ib_mad_agent_private *mad_agent_priv;
>> +	struct ib_mad_send_buf send_buf;
>> +	u64 header_mapping;
>> +	u64 payload_mapping;
>> +	struct ib_ud_wr send_wr;
>> +	struct ib_sge sg_list[IB_MAD_SEND_REQ_MAX_SG];
>> +	__be64 tid;
>> +	unsigned long timeout;
>> +	int max_retries;
>> +	int retries_left;
>> +	int retry;
>> +	int refcount;
>> +	enum ib_wc_status status;
>> +
>> +	/* RMPP control */
>> +	struct list_head rmpp_list;
>> +	struct ib_rmpp_segment *last_ack_seg;
>> +	struct ib_rmpp_segment *cur_seg;
>> +	int last_ack;
>> +	int seg_num;
>> +	int newwin;
>> +	int pad;
>> +};
>> +
>> +struct ib_mad_private_header {
>> +	struct ib_mad_list_head mad_list;
>> +	struct ib_mad_recv_wc recv_wc;
>> +	struct ib_wc wc;
>> +	u64 mapping;
>> +} __packed;
>> +
>> +struct ib_mad_private {
>> +	struct ib_mad_private_header header;
>> +	size_t mad_size;
>> +	struct ib_grh grh;
>> +	DECLARE_FLEX_ARRAY(u8, mad);
>> +} __packed;
>> +
>> +static inline void *xsc_ib_send_mad_sg_virt_addr(struct ib_device *ibdev,
>> +						 const struct ib_send_wr *wr,
>> +						 int sg)
>> +{
>> +	struct ib_mad_send_wr_private *mad_send_wr;
>> +	struct ib_mad_list_head *mad_list;
>> +	int iommu_state = xsc_ib_iommu_dma_map(ibdev);
>> +
>> +	/* direct dma mapping */
>> +	if (!iommu_state)
>> +		return phys_to_virt(dma_to_phys(ibdev->dma_device, wr->sg_list[sg].addr));
>> +
>> +	if (iommu_state == XSC_IB_IOMMU_MAP_NORMAL)
>> +		return xsc_ib_iova_to_virt(ibdev, wr->sg_list[sg].addr);
>> +
>> +	mad_list = container_of(wr->wr_cqe, struct ib_mad_list_head, cqe);
>> +	mad_send_wr = container_of(mad_list, struct ib_mad_send_wr_private,
>> +				   mad_list);
>> +
>> +	/* sg_list[0] */
>> +	if (sg == 0)
>> +		return mad_send_wr->send_buf.mad;
>> +
>> +	/* sg_list[1] */
>> +	if (mad_send_wr->send_buf.seg_count)
>> +		return ib_get_rmpp_segment(&mad_send_wr->send_buf,
>> +					   mad_send_wr->seg_num);
>> +	return mad_send_wr->send_buf.mad + mad_send_wr->send_buf.hdr_len;
>> +}
>> +
>> +static inline void *xsc_ib_recv_mad_sg_virt_addr(struct ib_device *ibdev,
>> +						 struct ib_wc *wc,
>> +						 u64 sg_addr)
>> +{
>> +	struct ib_mad_private_header *mad_priv_hdr;
>> +	struct ib_mad_private *recv;
>> +	struct ib_mad_list_head *mad_list;
>> +	int iommu_state = xsc_ib_iommu_dma_map(ibdev);
>> +
>> +	/* direct dma mapping */
>> +	if (!iommu_state)
>> +		return phys_to_virt(dma_to_phys(ibdev->dma_device, sg_addr));
>> +
>> +	if (iommu_state == XSC_IB_IOMMU_MAP_NORMAL)
>> +		return xsc_ib_iova_to_virt(ibdev, sg_addr);
>> +
>> +	mad_list = container_of(wc->wr_cqe, struct ib_mad_list_head, cqe);
>> +	mad_priv_hdr = container_of(mad_list, struct ib_mad_private_header, mad_list);
>> +	recv = container_of(mad_priv_hdr, struct ib_mad_private, header);
>> +	return &recv->grh;
>> +}
>> +
>> +struct xsc_ib_burst_info {
>> +	__u32       max_burst_sz;
>> +	__u16       typical_pkt_sz;
>> +	__u16       reserved;
>> +};
>> +
>> +struct xsc_ib_modify_qp {
>> +	__u32		reserved;
>> +	struct xsc_ib_burst_info  burst_info;
>> +	__u32		profile;
>> +};
>> +
>> +#endif /* XSC_IB_H */