[PATCH 06/13] scsi: fnic: Add the NVMe/FC transport path

[Karan Tilak Kumar <kartilak@cisco.com>]

Build fnic_nvme.c and fnic_nvme.h into the driver.

Add NVMe local-port and remote-port registration, I/O submission, DMA
mapping, tag management, completion and abort handling, LS response
handling, and tport/lport cleanup.

Update shared fnic structures, FCPIO descriptors, FDLS state, and
device-command definitions needed by the NVMe transport path.

Reviewed-by: Sesidhar Baddela <sebaddel@cisco.com>
Reviewed-by: Arulprabhu Ponnusamy <arulponn@cisco.com>
Reviewed-by: Gian Carlo Boffa <gcboffa@cisco.com>
Reviewed-by: Arun Easi <aeasi@cisco.com>
Reviewed-by: Hannes Reinecke <hare@kernel.org>
Reviewed-by: Lee Duncan <lduncan@suse.com>
Signed-off-by: Karan Tilak Kumar <kartilak@cisco.com>
Co-developed-by: Hannes Reinecke <hare@kernel.org>
---
 drivers/scsi/fnic/Makefile      |    1 +
 drivers/scsi/fnic/fcpio.h       |   12 +
 drivers/scsi/fnic/fnic.h        |   20 +-
 drivers/scsi/fnic/fnic_fcs.c    |   11 +-
 drivers/scsi/fnic/fnic_fdls.h   |   27 +-
 drivers/scsi/fnic/fnic_io.h     |   23 +-
 drivers/scsi/fnic/fnic_main.c   |   84 +-
 drivers/scsi/fnic/fnic_nvme.c   | 1749 +++++++++++++++++++++++++++++++
 drivers/scsi/fnic/fnic_nvme.h   |  134 +++
 drivers/scsi/fnic/fnic_res.h    |   32 +-
 drivers/scsi/fnic/fnic_scsi.c   |    4 +-
 drivers/scsi/fnic/vnic_devcmd.h |    2 +-
 12 files changed, 2086 insertions(+), 13 deletions(-)
 create mode 100644 drivers/scsi/fnic/fnic_nvme.c
 create mode 100644 drivers/scsi/fnic/fnic_nvme.h

diff --git a/drivers/scsi/fnic/Makefile b/drivers/scsi/fnic/Makefile
index c025e875009e..48c5e0cb8ca6 100644
--- a/drivers/scsi/fnic/Makefile
+++ b/drivers/scsi/fnic/Makefile
@@ -6,6 +6,7 @@ fnic-y	:= \
 	fnic_attrs.o \
 	fnic_isr.o \
 	fnic_main.o \
+	fnic_nvme.o \
 	fnic_res.o \
 	fnic_fcs.o \
 	fdls_disc.o \
diff --git a/drivers/scsi/fnic/fcpio.h b/drivers/scsi/fnic/fcpio.h
index 46eb442a47cb..a98a2a3bfe15 100644
--- a/drivers/scsi/fnic/fcpio.h
+++ b/drivers/scsi/fnic/fcpio.h
@@ -21,6 +21,18 @@
 #define FCPIO_HOST_SEQ_ID_RANGE_START       0x80
 #define FCPIO_HOST_SEQ_ID_RANGE_END         0xff
 
+struct fnic_nvme_io_event {
+	struct list_head links;
+	struct work_struct io_work;
+	void *arg1;
+};
+
+struct fnic_io_event_s {
+	struct list_head links;
+	struct work_struct io_work;
+	void *arg1;
+};
+
 /*
  * Command entry type
  */
diff --git a/drivers/scsi/fnic/fnic.h b/drivers/scsi/fnic/fnic.h
index 54ee52c453ba..86293e112b34 100644
--- a/drivers/scsi/fnic/fnic.h
+++ b/drivers/scsi/fnic/fnic.h
@@ -27,6 +27,7 @@
 #include "vnic_stats.h"
 #include "vnic_scsi.h"
 #include "fnic_fdls.h"
+#include "fnic_nvme.h"
 
 #define DRV_NAME		"fnic"
 #define DRV_DESCRIPTION		"Cisco FCoE HBA Driver"
@@ -80,6 +81,8 @@
 #define FNIC_DEV_RST_ABTS_DONE          BIT(19)
 #define FNIC_DEV_RST_TERM_DONE          BIT(20)
 #define FNIC_DEV_RST_ABTS_PENDING       BIT(21)
+#define FNIC_NVME_ADMIN_IO_TIMER_PENDING BIT(22)
+#define FNIC_NVME_ADMIN_IO              BIT(23)
 
 #define FNIC_FW_RESET_TIMEOUT        60000	/* mSec   */
 #define FNIC_FCOE_MAX_CMD_LEN        16
@@ -238,7 +241,7 @@ extern spinlock_t reset_fnic_list_lock;
 extern struct list_head reset_fnic_list;
 extern struct workqueue_struct *reset_fnic_work_queue;
 extern struct work_struct reset_fnic_work;
-
+extern struct workqueue_struct *fnic_cmpl_queue;
 
 #define FNIC_MAIN_LOGGING 0x01
 #define FNIC_FCS_LOGGING 0x02
@@ -352,6 +355,11 @@ struct fnic_frame_list {
 	int rx_ethhdr_stripped;
 };
 
+struct fnic_tag_t {
+	struct list_head free_list;
+	int tag_id;
+};
+
 struct fnic_event {
 	struct list_head list;
 	struct fnic *fnic;
@@ -469,6 +477,13 @@ struct fnic {
 	struct list_head vlan_list;
 	/*** FIP related data members  -- end ***/
 
+	/* NVME data members */
+	struct sbitmap nvfnic_tag_map;
+	struct work_struct nvme_io_cmpl_work;
+	atomic_t nvme_io_event_queued;
+	struct llist_head nvme_io_event_llist;
+	struct completion *nvme_lport_unreg_done;
+
 	/* copy work queue cache line section */
 	____cacheline_aligned struct vnic_wq_copy hw_copy_wq[FNIC_WQ_COPY_MAX];
 	____cacheline_aligned struct fnic_cpy_wq sw_copy_wq[FNIC_WQ_COPY_MAX];
@@ -522,6 +537,7 @@ int fnic_host_reset(struct Scsi_Host *shost);
 void fnic_reset(struct Scsi_Host *shost);
 int fnic_issue_fc_host_lip(struct Scsi_Host *shost);
 void fnic_get_host_port_state(struct Scsi_Host *shost);
+void fnic_fcpio_reset(struct fnic *fnic);
 int fnic_wq_copy_cmpl_handler(struct fnic *fnic, int copy_work_to_do, unsigned int cq_index);
 int fnic_wq_cmpl_handler(struct fnic *fnic, int);
 int fnic_flogi_reg_handler(struct fnic *fnic, u32);
@@ -568,6 +584,8 @@ void fnic_scsi_unload(struct fnic *fnic);
 void fnic_scsi_unload_cleanup(struct fnic *fnic);
 int fnic_get_debug_info(struct stats_debug_info *info,
 			struct fnic *fnic);
+int free_wq_copy_descs(struct fnic *fnic, struct vnic_wq_copy *wq,
+			unsigned int hwq);
 
 struct fnic_scsi_iter_data {
 	struct fnic *fnic;
diff --git a/drivers/scsi/fnic/fnic_fcs.c b/drivers/scsi/fnic/fnic_fcs.c
index a67d1085d720..ac1febaa8474 100644
--- a/drivers/scsi/fnic/fnic_fcs.c
+++ b/drivers/scsi/fnic/fnic_fcs.c
@@ -155,7 +155,6 @@ void fnic_fdls_init(struct fnic *fnic, int usefip)
 				 iport->hwmac[3], iport->hwmac[4], iport->hwmac[5]);
 
 	INIT_LIST_HEAD(&iport->tport_list);
-	INIT_LIST_HEAD(&iport->tport_list_pending_del);
 
 	fnic_fdls_disc_init(iport);
 }
@@ -728,7 +727,7 @@ fdls_send_fcoe_frame(struct fnic *fnic, void *frame, int frame_size,
 	return ret;
 }
 
-void fnic_send_fcoe_frame(struct fnic_iport_s *iport, void *frame,
+int fnic_send_fcoe_frame(struct fnic_iport_s *iport, void *frame,
 						 int frame_size)
 {
 	struct fnic *fnic = iport->fnic;
@@ -736,7 +735,7 @@ void fnic_send_fcoe_frame(struct fnic_iport_s *iport, void *frame,
 
 	/* If module unload is in-progress, don't send */
 	if (fnic->in_remove)
-		return;
+		return -ESHUTDOWN;
 
 	if (iport->fabric.flags & FNIC_FDLS_FPMA_LEARNT) {
 		srcmac = iport->fpma;
@@ -746,7 +745,7 @@ void fnic_send_fcoe_frame(struct fnic_iport_s *iport, void *frame,
 		dstmac = FCOE_ALL_FCF_MAC;
 	}
 
-	fdls_send_fcoe_frame(fnic, frame, frame_size, srcmac, dstmac);
+	return fdls_send_fcoe_frame(fnic, frame, frame_size, srcmac, dstmac);
 }
 
 int
@@ -1020,6 +1019,8 @@ void fnic_delete_fcp_tports(struct fnic *fnic)
 		fnic_del_tport_timer_sync(fnic, tport);
 		if (IS_FNIC_FCP_INITIATOR(fnic))
 			fnic_fdls_remove_tport(&fnic->iport, tport, flags);
+		else if (IS_FNIC_NVME_INITIATOR(fnic))
+			nvfnic_delete_tport(&fnic->iport, tport, flags);
 	}
 	spin_unlock_irqrestore(&fnic->fnic_lock, flags);
 }
@@ -1059,6 +1060,8 @@ void fnic_tport_event_handler(struct work_struct *work)
 			if (tport->state == FDLS_TGT_STATE_OFFLINING) {
 				if (IS_FNIC_FCP_INITIATOR(fnic))
 					fnic_fdls_remove_tport(&fnic->iport, tport, flags);
+				else if (IS_FNIC_NVME_INITIATOR(fnic))
+					nvfnic_delete_tport(&fnic->iport, tport, flags);
 			} else {
 				FNIC_FCS_DBG(KERN_INFO, fnic,
 							 "remove rport event dropped tport fcid: 0x%x",
diff --git a/drivers/scsi/fnic/fnic_fdls.h b/drivers/scsi/fnic/fnic_fdls.h
index ff1e3bd0e39f..0a68d0fb11b1 100644
--- a/drivers/scsi/fnic/fnic_fdls.h
+++ b/drivers/scsi/fnic/fnic_fdls.h
@@ -97,6 +97,7 @@
 #define fdls_get_state(_fdls_fabric)          ((_fdls_fabric)->state)
 
 #define FNIC_FDMI_ACTIVE    0x8
+#define FNIC_LPORT_NVME_REGISTERED 0x4
 #define FNIC_FIRST_LINK_UP    0x2
 
 #define fdls_set_tport_state(_tport, _state)    (_tport->state = _state)
@@ -122,6 +123,13 @@
 #define FNIC_FRAME_TYPE_TGT_PRLI	0x2800
 #define FNIC_FRAME_TYPE_TGT_ADISC	0x2A00
 #define FNIC_FRAME_TYPE_TGT_LOGO	0x2C00
+#define FNIC_FRAME_TYPE_NVME_LS		0x3000
+
+#define NVFNIC_FCPIO_TAG_POOL_SZ	(2048)
+#define NVFNIC_LS_REQ_OXID_POOL_SZ	(64)
+#define NVFNIC_LS_REQ_OXID_BASE		(0x2500)
+#define FNIC_LPORT_NVME_REGISTERED	0x4
+
 
 struct fnic_fip_fcf_s {
 	uint16_t vlan_id;
@@ -223,6 +231,8 @@ struct fnic_tport_s {
 	struct fc_rport *rport;
 	char str_wwpn[20];
 	char str_wwnn[20];
+	struct list_head ls_req_list;
+	struct nvme_fc_remote_port *nv_rport;
 };
 
 /* OXID pool related structures */
@@ -291,7 +301,6 @@ struct fnic_iport_s {
 	struct fnic_fdls_fip_s fip;
 	struct fnic_fdls_fabric_s fabric;
 	struct list_head tport_list;
-	struct list_head tport_list_pending_del;
 	/* list of tports for which we are yet to send PLOGO */
 	struct list_head inprocess_tport_list;
 	struct list_head deleted_tport_list;
@@ -311,6 +320,13 @@ struct fnic_iport_s {
 	struct fnic_iport_stats iport_stats;
 	char str_wwpn[20];
 	char str_wwnn[20];
+
+	 /* nvme */;
+	void *nvfnic_fcpio_tag[NVFNIC_FCPIO_TAG_POOL_SZ];
+	struct nvme_fc_local_port *nv_lport;
+	struct nvme_fc_port_template *nv_tmpl;
+	struct fnic_oxid_pool_s ls_req_oxid_pool;
+	struct list_head fcpio_list;
 };
 
 struct rport_dd_data_s {
@@ -339,6 +355,7 @@ enum fnic_recv_frame_type_e {
 	FNIC_TPORT_ADISC_RSP,
 	FNIC_TPORT_BLS_ABTS_RSP,
 	FNIC_TPORT_LOGO_RSP,
+	FNIC_LS_REQ_ABTS_RSP,
 
 	/* unsolicited requests */
 	FNIC_BLS_ABTS_REQ,
@@ -371,6 +388,12 @@ enum fnic_port_speeds {
 	DCEM_PORTSPEED_128G = 128000,
 };
 
+static inline bool fdls_tport_is_offline(struct fnic_tport_s *tport)
+{
+	return (tport->state == FDLS_TGT_STATE_OFFLINING ||
+		tport->state == FDLS_TGT_STATE_OFFLINE);
+}
+
 /* Function Declarations */
 /* fdls_disc.c */
 void fnic_fdls_disc_init(struct fnic_iport_s *iport);
@@ -401,7 +424,7 @@ void fdls_fdmi_retry_plogi(struct fnic_iport_s *iport);
 
 /* fnic_fcs.c */
 void fnic_fdls_init(struct fnic *fnic, int usefip);
-void fnic_send_fcoe_frame(struct fnic_iport_s *iport, void *frame,
+int fnic_send_fcoe_frame(struct fnic_iport_s *iport, void *frame,
 	int frame_size);
 void fnic_fcoe_send_vlan_req(struct fnic *fnic);
 int fnic_send_fip_frame(struct fnic_iport_s *iport,
diff --git a/drivers/scsi/fnic/fnic_io.h b/drivers/scsi/fnic/fnic_io.h
index 0d974e040ab7..49689073e78b 100644
--- a/drivers/scsi/fnic/fnic_io.h
+++ b/drivers/scsi/fnic/fnic_io.h
@@ -6,13 +6,19 @@
 #ifndef _FNIC_IO_H_
 #define _FNIC_IO_H_
 
+#include <scsi/fc/fc_fs.h>
 #include <scsi/fc/fc_fcp.h>
 #include "fnic_fdls.h"
+#include "fcpio.h"
+#include <linux/nvme-fc-driver.h>
 
 #define FNIC_DFLT_SG_DESC_CNT  32
 #define FNIC_MAX_SG_DESC_CNT        256     /* Maximum descriptors per sgl */
 #define FNIC_SG_DESC_ALIGN          16      /* Descriptor address alignment */
 
+#define NVME_STAT_ERROR 0x2
+#define NVME_STAT_TASK_SET_FULL 0x3
+
 struct host_sg_desc {
 	__le64 addr;
 	__le32 len;
@@ -39,6 +45,7 @@ enum fnic_ioreq_state {
 	FNIC_IOREQ_ABTS_PENDING,
 	FNIC_IOREQ_ABTS_COMPLETE,
 	FNIC_IOREQ_CMD_COMPLETE,
+	FNIC_IOREQ_RESET_TERM,
 };
 
 struct fnic_io_req {
@@ -53,9 +60,23 @@ struct fnic_io_req {
 	u8 io_completed:1; /* set to 1 when fw completes IO */
 	u32 port_id; /* remote port DID */
 	unsigned long start_time; /* in jiffies */
+	u32 tag;
+	enum fnic_ioreq_state cmd_state;
+	u32 cmd_flags;
+	u32 abts_state;
 	struct completion *abts_done; /* completion for abts */
 	struct completion *dr_done; /* completion for device reset */
-	unsigned int tag;
 	struct scsi_cmnd *sc; /* midlayer's cmd pointer */
+
+	struct vnic_wq_copy *wq;
+	struct llist_node nvfnic_io_cmpl;
+	struct nvmefc_fcp_req *fcp_req;
+	void (*done)(struct fnic_io_req *io_req);
+	unsigned long waitq_start_time;	/* in jiffies */
+
+	struct timer_list admin_io_timer;
+	uint32_t status;
+	struct fnic_tag_t *tag_data;
+	struct fnic_io_event_s io_evt;
 };
 #endif /* _FNIC_IO_H_ */
diff --git a/drivers/scsi/fnic/fnic_main.c b/drivers/scsi/fnic/fnic_main.c
index 0d7828be244d..1831aa5c5911 100644
--- a/drivers/scsi/fnic/fnic_main.c
+++ b/drivers/scsi/fnic/fnic_main.c
@@ -42,12 +42,12 @@ static struct kmem_cache *fdls_frame_cache;
 static struct kmem_cache *fdls_frame_elem_cache;
 static struct kmem_cache *fdls_frame_recv_cache;
 static LIST_HEAD(fnic_list);
-static DEFINE_SPINLOCK(fnic_list_lock);
 static DEFINE_IDA(fnic_ida);
 
 struct work_struct reset_fnic_work;
 LIST_HEAD(reset_fnic_list);
 DEFINE_SPINLOCK(reset_fnic_list_lock);
+DEFINE_SPINLOCK(fnic_list_lock);
 
 /* Supported devices by fnic module */
 static const struct pci_device_id fnic_id_table[] = {
@@ -89,6 +89,15 @@ module_param(fnic_fc_trace_max_pages, uint, S_IRUGO|S_IWUSR);
 MODULE_PARM_DESC(fnic_fc_trace_max_pages,
 		 "Total allocated memory pages for fc trace buffer");
 
+unsigned int nvme_dev_loss_tmo = 30;
+module_param_named(nvme_dev_loss_tmo, nvme_dev_loss_tmo, uint, 0644);
+MODULE_PARM_DESC(nvme_dev_loss_tmo, "configurable NVME dev loss timeout");
+
+unsigned int nvme_max_ios_to_process = 16;
+module_param(nvme_max_ios_to_process, uint, 0644);
+MODULE_PARM_DESC(nvme_max_ios_to_process,
+		"Maximum number of NVME IOs to process per work queue");
+
 static unsigned int fnic_max_qdepth = FNIC_DFLT_QUEUE_DEPTH;
 module_param(fnic_max_qdepth, uint, S_IRUGO|S_IWUSR);
 MODULE_PARM_DESC(fnic_max_qdepth, "Queue depth to report for each LUN");
@@ -100,6 +109,7 @@ MODULE_PARM_DESC(pc_rscn_handling_feature_flag,
 
 struct workqueue_struct *reset_fnic_work_queue;
 struct workqueue_struct *fnic_fip_queue;
+struct workqueue_struct *fnic_cmpl_queue;
 
 static int fnic_sdev_init(struct scsi_device *sdev)
 {
@@ -678,6 +688,59 @@ static int fnic_scsi_drv_init(struct fnic *fnic)
 	return 0;
 }
 
+static int fnic_nvme_drv_init(struct fnic *fnic)
+{
+	int ret;
+	int hwq;
+
+	fnic->fnic_max_tag_id = NVFNIC_FCPIO_TAG_POOL_SZ;
+
+	for (hwq = 0; hwq < fnic->wq_copy_count; hwq++) {
+		fnic->sw_copy_wq[hwq].ioreq_table_size = fnic->fnic_max_tag_id;
+		fnic->sw_copy_wq[hwq].io_req_table =
+			kzalloc((fnic->sw_copy_wq[hwq].ioreq_table_size + 1) *
+					sizeof(struct fnic_io_req *), GFP_KERNEL);
+
+		if (!fnic->sw_copy_wq[hwq].io_req_table) {
+			fnic_free_ioreq_tables_mq(fnic);
+			return -ENOMEM;
+		}
+	}
+
+	dev_info(&fnic->pdev->dev, "fnic copy wqs: %d, Q0 ioreq table size: %d\n",
+			fnic->wq_copy_count, fnic->sw_copy_wq[0].ioreq_table_size);
+
+	if (sbitmap_init_node(&fnic->nvfnic_tag_map, NVFNIC_FCPIO_TAG_POOL_SZ,
+			      -1, GFP_KERNEL, NUMA_NO_NODE, false, true)) {
+		dev_err(&fnic->pdev->dev,
+			"Unable to allocate tag pool\n");
+		ret = -ENOMEM;
+		goto out_free_ioreq_tables;
+	}
+
+	fnic->io_req_pool = mempool_create_slab_pool(2, fnic_io_req_cache);
+	if (!fnic->io_req_pool) {
+		ret = -ENOMEM;
+		goto out_free_sbitmap;
+	}
+
+	init_llist_head(&fnic->nvme_io_event_llist);
+	INIT_WORK(&fnic->nvme_io_cmpl_work, nvfnic_nvme_iodone_work);
+
+	ret = nvfnic_add_lport(fnic);
+	if (ret)
+		goto out_free_req_pool;
+
+	return 0;
+out_free_req_pool:
+	mempool_destroy(fnic->io_req_pool);
+out_free_sbitmap:
+	sbitmap_free(&fnic->nvfnic_tag_map);
+out_free_ioreq_tables:
+	fnic_free_ioreq_tables_mq(fnic);
+	return ret;
+}
+
 void fnic_mq_map_queues_cpus(struct Scsi_Host *host)
 {
 	struct fnic *fnic = *((struct fnic **) shost_priv(host));
@@ -1057,6 +1120,7 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	INIT_LIST_HEAD(&fnic->frame_queue);
 	INIT_LIST_HEAD(&fnic->tx_queue);
 	INIT_LIST_HEAD(&fnic->tport_event_list);
+	init_llist_head(&fnic->nvme_io_event_llist);
 
 	INIT_DELAYED_WORK(&iport->oxid_pool.schedule_oxid_free_retry,
 	fdls_schedule_oxid_free_retry_work);
@@ -1091,6 +1155,10 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		err = fnic_scsi_drv_init(fnic);
 		if (err)
 			goto err_out_scsi_drv_init;
+	} else if (IS_FNIC_NVME_INITIATOR(fnic)) {
+		err = fnic_nvme_drv_init(fnic);
+		if (err)
+			goto err_out_nvme_drv_init;
 	}
 
 	err = fnic_stats_debugfs_init(fnic);
@@ -1113,6 +1181,7 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	fnic_free_ioreq_tables_mq(fnic);
 	if (IS_FNIC_FCP_INITIATOR(fnic))
 		scsi_remove_host(fnic->host);
+err_out_nvme_drv_init:
 err_out_scsi_drv_init:
 	fnic_free_intr(fnic);
 err_out_fnic_request_intr:
@@ -1190,6 +1259,8 @@ static void fnic_remove(struct pci_dev *pdev)
 
 	if (IS_FNIC_FCP_INITIATOR(fnic))
 		fnic_scsi_unload(fnic);
+	else if (IS_FNIC_NVME_INITIATOR(fnic))
+		nvfnic_nvme_unload(fnic);
 
 	if (vnic_dev_get_intr_mode(fnic->vdev) == VNIC_DEV_INTR_MODE_MSI)
 		timer_delete_sync(&fnic->notify_timer);
@@ -1356,6 +1427,14 @@ static int __init fnic_init_module(void)
 		goto err_create_fip_workq;
 	}
 
+	fnic_cmpl_queue =
+		alloc_workqueue("fnic_cmpl_wq", WQ_HIGHPRI | WQ_MEM_RECLAIM, 0);
+	if (!fnic_cmpl_queue) {
+		pr_err("fnic completion work queue create failed\n");
+		err = -ENOMEM;
+		goto err_create_cmpl_workq;
+	}
+
 	if (pc_rscn_handling_feature_flag == PC_RSCN_HANDLING_FEATURE_ON) {
 		reset_fnic_work_queue =
 			create_singlethread_workqueue("reset_fnic_work_queue");
@@ -1387,6 +1466,8 @@ static int __init fnic_init_module(void)
 err_pci_register:
 	fc_release_transport(fnic_fc_transport);
 err_fc_transport:
+	destroy_workqueue(fnic_cmpl_queue);
+err_create_cmpl_workq:
 	destroy_workqueue(fnic_fip_queue);
 err_create_fip_workq:
 	if (pc_rscn_handling_feature_flag == PC_RSCN_HANDLING_FEATURE_ON)
@@ -1415,6 +1496,7 @@ static int __init fnic_init_module(void)
 static void __exit fnic_cleanup_module(void)
 {
 	pci_unregister_driver(&fnic_driver);
+	destroy_workqueue(fnic_cmpl_queue);
 	destroy_workqueue(fnic_event_queue);
 
 	if (pc_rscn_handling_feature_flag == PC_RSCN_HANDLING_FEATURE_ON)
diff --git a/drivers/scsi/fnic/fnic_nvme.c b/drivers/scsi/fnic/fnic_nvme.c
new file mode 100644
index 000000000000..26d4cbb06f50
--- /dev/null
+++ b/drivers/scsi/fnic/fnic_nvme.c
@@ -0,0 +1,1749 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright 2008 Cisco Systems, Inc.  All rights reserved.
+ * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
+ */
+#include <linux/mempool.h>
+#include <linux/errno.h>
+#include <linux/init.h>
+#include <linux/workqueue.h>
+#include <linux/pci.h>
+#include <linux/scatterlist.h>
+#include <linux/skbuff.h>
+#include <linux/spinlock.h>
+#include <linux/kthread.h>
+#include <linux/if_ether.h>
+#include <linux/if_vlan.h>
+#include <linux/delay.h>
+#include <linux/gfp.h>
+#include <linux/nvme.h>
+#include <linux/nvme-fc.h>
+#include "fnic.h"
+#include "fnic_trace.h"
+#include "fdls_fc.h"
+
+int nvfnic_get_sg_count(struct fnic_io_req *io_req)
+{
+	return io_req->fcp_req->sg_cnt;
+}
+
+int nvfnic_dma_map_sgl(struct fnic *fnic, struct fnic_io_req *io_req,
+		       int sg_count)
+{
+	io_req->sgl_list_pa = dma_map_single(&fnic->pdev->dev,
+					     io_req->sgl_list,
+					     sizeof(io_req->sgl_list[0]) *
+					     sg_count, DMA_TO_DEVICE);
+	if (dma_mapping_error(&fnic->pdev->dev, io_req->sgl_list_pa)) {
+		dev_err(&fnic->pdev->dev, "DMA mapping failed\n");
+		return -EBUSY;
+	}
+
+	return 0;
+}
+
+void nvfnic_dma_unmap_sgl(struct fnic *fnic, struct fnic_io_req *io_req)
+{
+	if (io_req->sgl_list_pa)
+		dma_unmap_single(&fnic->pdev->dev, io_req->sgl_list_pa,
+				 sizeof(io_req->sgl_list[0]) * io_req->sgl_cnt,
+				 DMA_TO_DEVICE);
+}
+
+static void nvfnic_update_io_bytes(struct fnic *fnic,
+				   struct fnic_io_req *io_req, u8 opcode)
+{
+	if (opcode == nvme_cmd_read)
+		fnic->fcp_input_bytes += io_req->fcp_req->transferred_length;
+	else if (opcode == nvme_cmd_write)
+		fnic->fcp_output_bytes += io_req->fcp_req->transferred_length;
+}
+
+int
+nvfnic_alloc_fcpio_tag(struct fnic_iport_s *iport, struct fnic_io_req *io_req)
+{
+	struct fnic *fnic = iport->fnic;
+	int tag;
+
+	tag = sbitmap_get(&fnic->nvfnic_tag_map);
+	if (tag >= 0) {
+		WRITE_ONCE(io_req->tag, tag);
+		fnic->sw_copy_wq[0].io_req_table[tag] = io_req;
+		return tag;
+	}
+	return FNIC_NVME_NO_FREE_TAG;
+}
+
+void
+nvfnic_free_fcpio_tag(struct fnic_iport_s *iport, struct fnic_io_req *io_req)
+{
+	struct fnic *fnic = iport->fnic;
+	uint16_t tag = io_req->tag;
+
+	if (tag == FNIC_NVME_NO_FREE_TAG) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			    "Attempting to free invalid tag: 0x%x\n", tag);
+		return;
+	}
+	fnic->sw_copy_wq[0].io_req_table[tag] = NULL;
+	WRITE_ONCE(io_req->tag, FNIC_NVME_NO_FREE_TAG);
+	sbitmap_clear_bit(&fnic->nvfnic_tag_map, tag);
+}
+
+void
+nvfnic_reset_fcpio_tag_pool(struct fnic_iport_s *iport)
+{
+	WARN_ON(sbitmap_weight(&iport->fnic->nvfnic_tag_map));
+}
+
+struct fnic_io_req *
+nvfnic_find_io_req_by_tag(struct fnic *fnic, uint16_t tag)
+{
+	if (tag == FNIC_NVME_NO_FREE_TAG ||
+		!sbitmap_test_bit(&fnic->nvfnic_tag_map, tag))
+		return NULL;
+	return fnic->sw_copy_wq[0].io_req_table[tag];
+}
+
+/*
+ * Unmap the data buffer and sense buffer for an io_req,
+ * also unmap and free the device-private scatter/gather list.
+ */
+void nvfnic_release_nvme_ioreq_buf(struct fnic_iport_s *iport,
+					struct fnic_io_req *io_req)
+{
+	struct fnic *fnic = iport->fnic;
+
+	nvfnic_dma_unmap_sgl(fnic, io_req);
+
+	if (io_req->sgl_cnt)
+		mempool_free(io_req->sgl_list_alloc,
+			     fnic->io_sgl_pool[io_req->sgl_type]);
+}
+
+inline int nvfnic_queue_wq_nvme_copy_desc(struct fnic *fnic,
+					       struct vnic_wq_copy *wq,
+					       struct fnic_io_req *io_req,
+					       int sg_count)
+{
+	struct scatterlist *sg;
+	struct fnic_tport_s *tport = io_req->tport;
+	struct host_sg_desc *desc;
+	unsigned int i;
+	unsigned long intr_flags;
+	int flags;
+	u8 exch_flags;
+	struct scatterlist *sgl;
+	int idx;
+	int ret = 0;
+
+	if (sg_count) {
+		/* For each SGE, create a device desc entry */
+		desc = io_req->sgl_list;
+		sgl = io_req->fcp_req->first_sgl;
+		for_each_sg(sgl, sg, sg_count, i) {
+			desc->addr = cpu_to_le64(sg_dma_address(sg));
+			desc->len = cpu_to_le32(sg_dma_len(sg));
+			desc->_resvd = 0;
+			desc++;
+		}
+
+		ret = nvfnic_dma_map_sgl(fnic, io_req, sg_count);
+		if (ret)
+			return ret;
+	}
+
+	idx = (struct vnic_wq_copy *)wq - &fnic->hw_copy_wq[0];
+
+	/* Enqueue the descriptor in the Copy WQ */
+	spin_lock_irqsave(&fnic->wq_copy_lock[idx], intr_flags);
+
+	if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[idx])
+		free_wq_copy_descs(fnic, wq, idx);
+
+	if (unlikely(!vnic_wq_copy_desc_avail(wq))) {
+		spin_unlock_irqrestore(&fnic->wq_copy_lock[idx], intr_flags);
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			    "Enqueue failure: No descriptors\n");
+		return -EBUSY;
+	}
+
+	flags = 0;
+	if (io_req->fcp_req->io_dir == NVMEFC_FCP_READ)
+		flags = FCPIO_ICMND_RDDATA;
+	else if (io_req->fcp_req->io_dir == NVMEFC_FCP_WRITE)
+		flags = FCPIO_ICMND_WRDATA;
+
+	exch_flags = 0;
+
+	fnic_queue_wq_copy_desc_nvme_io(wq, io_req->tag,
+					exch_flags, io_req->sgl_cnt,
+					io_req->sgl_list_pa, flags,
+					io_req->fcp_req->cmdaddr,
+					io_req->fcp_req->cmdlen,
+					io_req->fcp_req->payload_length,
+					io_req->port_id,
+					tport->max_payload_size, tport->r_a_tov,
+					tport->e_d_tov);
+
+
+	spin_unlock_irqrestore(&fnic->wq_copy_lock[idx], intr_flags);
+	return 0;
+}
+
+bool
+nvfnic_transport_ready(struct fnic_iport_s *iport, struct fnic_tport_s *tport)
+{
+	struct fnic *fnic = iport->fnic;
+
+	if (tport == NULL)
+		return false;
+
+	if (fdls_get_state(&iport->fabric) == FDLS_STATE_LINKDOWN ||
+	    iport->state != FNIC_IPORT_STATE_READY) {
+		return false;
+	}
+
+	if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_IO_BLOCKED)))
+		return false;
+
+	if (fdls_tport_is_offline(tport))
+		return false;
+
+	return true;
+}
+
+int nvfnic_queuecommand(struct fnic_io_req *io_req)
+{
+	struct fnic_iport_s *iport = io_req->iport;
+	struct fnic *fnic = iport->fnic;
+	struct fnic_tport_s *tport = io_req->tport;
+	struct vnic_wq_copy *wq = io_req->wq;
+	int ret = 0;
+	int sg_count = 0;
+	unsigned long ptr;
+	unsigned char *lba;
+	u64 cmd_trace;
+	int idx;
+	struct nvme_fc_cmd_iu *cmdiu = io_req->fcp_req->cmdaddr;
+
+	io_req->cmd_state = FNIC_IOREQ_NOT_INITED;
+	io_req->cmd_flags = FNIC_NO_FLAGS;
+	/* Map the data buffer */
+	sg_count = nvfnic_get_sg_count(io_req);
+	if (sg_count < 0) {
+		FNIC_TRACE(nvfnic_queuecommand, fnic->fnic_num,
+			   io_req->tag, io_req, 0, io_req->fcp_req->io_dir,
+			   sg_count, io_req->cmd_state);
+		FNIC_NVME_DBG(KERN_INFO, fnic, "sg count is less-than-zero\n");
+		ret = -1;
+		goto out;
+	}
+
+	/* Determine the type of scatter/gather list we need */
+	io_req->sgl_cnt = sg_count;
+	io_req->sgl_type = FNIC_SGL_CACHE_DFLT;
+	if (sg_count > FNIC_DFLT_SG_DESC_CNT)
+		io_req->sgl_type = FNIC_SGL_CACHE_MAX;
+
+	if (sg_count) {
+		io_req->sgl_list =
+		    mempool_alloc(fnic->io_sgl_pool[io_req->sgl_type],
+				  GFP_ATOMIC);
+		if (!io_req->sgl_list) {
+			FNIC_NVME_DBG(KERN_INFO, fnic,
+				      "Unable to alloc SGLs\n");
+			ret = -ENOMEM;
+			goto out;
+		}
+
+		/* Cache sgl list allocated address before alignment */
+		io_req->sgl_list_alloc = io_req->sgl_list;
+		ptr = (unsigned long)io_req->sgl_list;
+		if (ptr % FNIC_SG_DESC_ALIGN) {
+			io_req->sgl_list = (struct host_sg_desc *)
+			    (((unsigned long)ptr + FNIC_SG_DESC_ALIGN - 1)
+			     & ~(FNIC_SG_DESC_ALIGN - 1));
+		}
+	}
+
+	io_req->port_id = tport->fcid;
+	io_req->start_time = jiffies;
+	io_req->cmd_state = FNIC_IOREQ_CMD_PENDING;
+	io_req->cmd_flags = FNIC_IO_INITIALIZED;
+
+	/* create copy wq desc and enqueue it */
+	idx = wq - &fnic->hw_copy_wq[0];
+	ret = nvfnic_queue_wq_nvme_copy_desc(fnic, io_req->wq, io_req, sg_count);
+	if (ret) {
+		FNIC_NVME_DBG(KERN_ERR, fnic, "Unable to queue frame\n");
+		/*
+		 * In case another thread cancelled the request,
+		 * refetch the pointer under the lock.
+		 */
+		nvfnic_release_nvme_ioreq_buf(iport, io_req);
+		FNIC_TRACE(nvfnic_queuecommand, fnic->fnic_num,
+			   io_req->tag, io_req->fcp_req, 0, 0, 0,
+			   (((u64)io_req->cmd_flags << 32) | io_req->cmd_state));
+		return ret;
+	}
+	io_req->cmd_flags |= FNIC_IO_ISSUED;
+ out:
+	lba = (char *)&cmdiu->sqe.rw.slba;
+	cmd_trace = ((u64) cmdiu->sqe.rw.opcode << 56 | (u64) lba[4] << 40 |
+		     (u64) lba[5] << 32 | (u64) lba[0] << 24 |
+		     (u64) lba[1] << 16 | (u64) lba[2] << 8 | lba[3]);
+
+	FNIC_TRACE(nvfnic_queuecommand, fnic->fnic_num,
+		   io_req->tag, 0, io_req,
+		   sg_count, cmd_trace,
+		   (((u64)io_req->cmd_flags >> 32) |
+		    io_req->cmd_state));
+
+	return ret;
+}
+
+int nvfnic_fcpio_send(struct nvme_fc_local_port *lport,
+		  struct nvme_fc_remote_port *rport, void *hw_queue_handle,
+		  struct nvmefc_fcp_req *fcp_req)
+{
+	struct fnic_iport_s *iport = lport->private;
+	struct fnic_io_req *io_req;
+	int ret;
+	struct fnic *fnic = iport->fnic;
+	unsigned long flags = 0;
+	struct fnic_tport_s *tport;
+
+	spin_lock_irqsave(&fnic->fnic_lock, flags);
+
+	tport = (struct fnic_tport_s *)rport->private;
+	if (!nvfnic_transport_ready(iport, tport)) {
+		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+		FNIC_NVME_DBG(KERN_INFO, fnic,
+			      "iport: 0x%x tport: 0x%x not ready\n",
+			      iport->fcid, tport->fcid);
+		return -ENODEV;
+	}
+	atomic_inc(&fnic->in_flight);
+
+	io_req = (struct fnic_io_req *) fcp_req->private;
+	io_req->iport = iport;
+	io_req->tport = (struct fnic_tport_s *)rport->private;
+	io_req->fcp_req = fcp_req;
+	io_req->done = nvfnic_fcpio_cmpl;
+	io_req->sgl_list_pa = 0;
+	io_req->wq = hw_queue_handle;
+	init_llist_node(&io_req->nvfnic_io_cmpl);
+
+	io_req->tag = nvfnic_alloc_fcpio_tag(iport, io_req);
+	if (io_req->tag == FNIC_NVME_NO_FREE_TAG) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			    "No free tag available. Failing IO\n");
+		atomic_dec(&fnic->in_flight);
+		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+		return -EBUSY;
+	}
+
+	ret = nvfnic_queuecommand(io_req);
+	if (ret) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "Queuecommand failed tag: 0x%x\n",
+			      io_req->tag);
+		nvfnic_free_fcpio_tag(iport, io_req);
+	}
+
+	spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+	atomic_dec(&fnic->in_flight);
+	return ret;
+}
+
+void nvfnic_fcpio_nvme_fast_cmpl_handler(struct fnic *fnic,
+				       struct fcpio_fw_req *desc)
+{
+	u8 type;
+	u8 hdr_status;
+	struct fcpio_tag ftag;
+	u32 id;
+	struct fnic_io_req *io_req;
+	unsigned long start_time;
+	u64 cmd_trace;
+	char *lba;
+	struct nvme_fc_cmd_iu *cmdiu;
+	struct fnic_iport_s *iport;
+	struct fnic_tport_s *tport;
+	unsigned int tag;
+
+	/* Decode the cmpl description to get the io_req id */
+	fcpio_header_dec(&desc->hdr, &type, &hdr_status, &ftag);
+	fcpio_tag_id_dec(&ftag, &id);
+	tag = id & FNIC_TAG_MASK;
+
+	if (tag >= fnic->fnic_max_tag_id) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			    "Tag out of range tag: 0x%x hdr status: %s\n", tag,
+			    fnic_fcpio_status_to_str(hdr_status));
+		return;
+	}
+	spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
+
+	io_req = nvfnic_find_io_req_by_tag(fnic, tag);
+
+	WARN_ON_ONCE(!io_req);
+	if (!io_req) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "IO req null hdr: %s tag: 0x%x desc: 0x%p\n",
+			      fnic_fcpio_status_to_str(hdr_status), id, desc);
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			    "type: 0x%x status: 0x%x rsvd: 0x%x tag: 0x%x\n",
+			    desc->hdr.type, desc->hdr.status, desc->hdr._resvd,
+			    id);
+		spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+		return;
+	}
+
+	cmdiu = io_req->fcp_req->cmdaddr;
+	if (io_req->tag != tag) {
+		FNIC_NVME_DBG(KERN_INFO, fnic,
+			      "Tag mismatch tag:%d io:0x%x id:0x%x csn:%08x\n",
+			      tag, io_req->tag, id, be32_to_cpu(cmdiu->csn));
+		spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+		return;
+	}
+	iport = io_req->iport;
+	tport = io_req->tport;
+
+	start_time = io_req->start_time;
+
+	/* firmware completed the io */
+	io_req->io_completed = 1;
+	if (io_req->cmd_state == FNIC_IOREQ_ABTS_PENDING) {
+		/*
+		 * set the FNIC_IO_DONE so that this doesn't get
+		 * flagged as 'out of order' if it was not aborted
+		 */
+		io_req->cmd_flags |= FNIC_IO_DONE;
+		io_req->cmd_flags |= FNIC_IO_ABTS_PENDING;
+		if (hdr_status == FCPIO_ABORTED)
+			io_req->cmd_flags |= FNIC_IO_ABORTED;
+		spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+
+		FNIC_NVME_DBG(KERN_INFO, fnic,
+			      "icmnd abts hdr:%d %s tag:0x%x io:%p",
+			      hdr_status, fnic_fcpio_status_to_str(hdr_status),
+			      id, io_req);
+		return;
+	}
+
+	if (io_req->cmd_state != FNIC_IOREQ_CMD_PENDING) {
+		FNIC_NVME_DBG(KERN_INFO, fnic,
+			      "IO freed id:%d tag:0x%x st:0x%x csn:%08x\n",
+			      id, io_req->tag, io_req->cmd_state,
+			      be32_to_cpu(cmdiu->csn));
+		spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+		return;
+	}
+
+	/* Mark the IO as complete */
+	io_req->cmd_state = FNIC_IOREQ_CMD_COMPLETE;
+	switch (hdr_status) {
+	case FCPIO_SUCCESS:
+		io_req->fcp_req->status = 0;
+		io_req->fcp_req->transferred_length =
+		    io_req->fcp_req->payload_length;
+		io_req->fcp_req->rcv_rsplen = 12;
+		break;
+	default:
+		FNIC_NVME_DBG(KERN_ERR, fnic, "HDR status is non-zero\n");
+		io_req->fcp_req->status = NVME_SC_INTERNAL;
+		break;
+	}
+
+	if (hdr_status != FCPIO_SUCCESS) {
+		FNIC_NVME_DBG(KERN_INFO, fnic, "hdr status: %s\n",
+			    fnic_fcpio_status_to_str(hdr_status));
+	}
+
+	io_req->cmd_flags |= FNIC_IO_DONE;
+
+	cmdiu = io_req->fcp_req->cmdaddr;
+	lba = (char *)&cmdiu->sqe.rw.slba;
+	cmd_trace = ((u64) hdr_status << 56) |
+	    (u64) cmdiu->sqe.rw.opcode << 32 |
+	    (u64) lba[0] << 24 | (u64) lba[1] << 16 |
+	    (u64) lba[2] << 8 | lba[3];
+
+	FNIC_TRACE(nvfnic_fcpio_nvme_fast_cmpl_handler, fnic->fnic_num,
+			tag, io_req,
+		   jiffies_to_msecs(jiffies - start_time),
+		   desc, cmd_trace,
+		   (((u64) io_req->cmd_flags << 32) |
+		    io_req->cmd_state));
+
+	nvfnic_update_io_bytes(fnic, io_req, cmdiu->sqe.rw.opcode);
+
+	nvfnic_release_nvme_ioreq_buf(iport, io_req);
+	if (io_req->done)
+		io_req->done(io_req);
+
+	spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+}
+
+void nvfnic_fcpio_ersp_cmpl_handler(struct fnic *fnic,
+				  struct fcpio_fw_req *desc, int sw_flag)
+{
+	u8 type;
+	u8 hdr_status;
+	struct fcpio_tag ftag;
+	u32 id;
+	struct fcpio_nvme_cmpl *nvme_cmpl;
+	struct fnic_io_req *io_req;
+	unsigned long start_time;
+	uint32_t rsplen;
+	struct nvme_fc_ersp_iu *ersp;
+	struct nvme_fc_ersp_iu *nrsp;
+	struct nvme_fc_cmd_iu *cmdiu;
+	struct nvme_command *sqe;
+	struct nvme_completion *cqe;
+	u64 cmd_trace;
+	struct fnic_iport_s *iport;
+	struct fnic_tport_s *tport;
+	unsigned int tag;
+	char *lba;
+
+	/* Decode the cmpl description to get the io_req id */
+	fcpio_header_dec(&desc->hdr, &type, &hdr_status, &ftag);
+	fcpio_tag_id_dec(&ftag, &id);
+	nvme_cmpl = &desc->u.nvme_cmpl;
+	ersp = (struct nvme_fc_ersp_iu *) nvme_cmpl->resp_bytes;
+	tag = id & FNIC_TAG_MASK;
+
+	if (tag >= fnic->fnic_max_tag_id) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "Tag out of range tag:0x%x hdr:%s\n", tag,
+			      fnic_fcpio_status_to_str(hdr_status));
+		return;
+	}
+	spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
+
+	io_req = nvfnic_find_io_req_by_tag(fnic, tag);
+	if (!io_req) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			    "IOREQ is null hdr status: %s tag: 0x%x desc: %p\n",
+			    fnic_fcpio_status_to_str(hdr_status), tag, desc);
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			    "type: 0x%x status: 0x%x rsvd: 0x%x\n",
+			    desc->hdr.type, desc->hdr.status, desc->hdr._resvd);
+		spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+		return;
+	}
+	iport = io_req->iport;
+	tport = io_req->tport;
+	cmdiu = io_req->fcp_req->cmdaddr;
+
+	if (io_req->tag != tag) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "Tag mismatch io:0x%x tag:0x%x id:0x%x\n",
+			      io_req->tag, tag, id);
+		spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+		return;
+	}
+
+	nrsp = (struct nvme_fc_ersp_iu *)io_req->fcp_req->rspaddr;
+	cmdiu = (struct nvme_fc_cmd_iu *)io_req->fcp_req->cmdaddr;
+	sqe = &cmdiu->sqe;
+	cqe = &nrsp->cqe;
+	start_time = io_req->start_time;
+
+	/* firmware completed the io */
+	io_req->io_completed = 1;
+
+	if (io_req->cmd_state == FNIC_IOREQ_ABTS_PENDING) {
+		/*
+		 * set the FNIC_IO_DONE so that this doesn't get
+		 * flagged as 'out of order' if it was not aborted
+		 */
+		io_req->cmd_flags |= FNIC_IO_DONE;
+		io_req->cmd_flags |= FNIC_IO_ABTS_PENDING;
+		if (hdr_status == FCPIO_ABORTED)
+			io_req->cmd_flags |= FNIC_IO_ABORTED;
+
+		spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+		FNIC_NVME_DBG(KERN_INFO, fnic,
+			      "ABTS pending hdr status: %s tag: 0x%x",
+			      fnic_fcpio_status_to_str(hdr_status), tag);
+		return;
+	}
+
+	if (io_req->cmd_state != FNIC_IOREQ_CMD_PENDING) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "IO already freed by abort. tag: 0x%x id: 0x%x\n",
+			      io_req->tag, id);
+		spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+		return;
+	}
+
+	/* Mark the IO as complete */
+	io_req->cmd_state = FNIC_IOREQ_CMD_COMPLETE;
+
+	switch (hdr_status) {
+	case FCPIO_SUCCESS:
+		io_req->fcp_req->status = 0;
+		if (!sw_flag) {
+			io_req->fcp_req->transferred_length =
+			    io_req->fcp_req->payload_length;
+			rsplen = 32;
+			nrsp->iu_len =
+			    cpu_to_be16(sizeof(struct nvme_fc_ersp_iu) / 4);
+			nrsp->xfrd_len =
+			    cpu_to_be32(io_req->fcp_req->payload_length);
+
+			nrsp->ersp_result = 0;
+			cqe->command_id = sqe->common.command_id;
+			cqe->status = 0;
+			cqe->result.u64 = 0;
+		} else {
+			io_req->fcp_req->transferred_length =
+			    be32_to_cpu(ersp->xfrd_len);
+			rsplen = be16_to_cpu(ersp->iu_len * 4);
+			memcpy(io_req->fcp_req->rspaddr, ersp, rsplen);
+		}
+		io_req->fcp_req->rcv_rsplen = rsplen;
+		break;
+
+	default:
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+				"Unexpected header status: %d\n", hdr_status);
+		io_req->fcp_req->status = NVME_SC_INTERNAL;
+		break;
+	}
+
+	if (hdr_status != FCPIO_SUCCESS) {
+		FNIC_NVME_DBG(KERN_ERR, fnic, "hdr status: %s tag: 0x%x\n",
+			    fnic_fcpio_status_to_str(hdr_status), tag);
+	}
+
+	io_req->cmd_flags |= FNIC_IO_DONE;
+
+	lba = (char *) &cmdiu->sqe.rw.slba;
+	cmd_trace = ((u64) hdr_status << 56) |
+	    (u64) ersp->ersp_result << 48 |
+	    (u64) cmdiu->sqe.rw.opcode << 32 |
+	    (u64) lba[0] << 24 | (u64) lba[1] << 16 |
+	    (u64) lba[2] << 8 | lba[3];
+
+	FNIC_TRACE(nvfnic_fcpio_ersp_cmpl_handler, fnic->fnic_num,
+		   tag, io_req,
+		   ((u64) nvme_cmpl->resvd[1] << 56 |
+		    (u64) nvme_cmpl->resvd[0] << 48 |
+		    jiffies_to_msecs(jiffies - start_time)),
+		   desc, cmd_trace,
+		   (((u64) io_req->cmd_flags << 32) |
+		    io_req->cmd_state));
+
+	nvfnic_update_io_bytes(fnic, io_req, cmdiu->sqe.rw.opcode);
+
+	nvfnic_release_nvme_ioreq_buf(iport, io_req);
+
+	/* Call NVME completion function to complete the IO */
+	if (io_req->done)
+		io_req->done(io_req);
+
+	spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+}
+
+void nvfnic_fcpio_nvme_itmf_cmpl_handler(struct fnic *fnic,
+				       struct fcpio_fw_req *desc)
+{
+	u8 type;
+	u8 hdr_status;
+	struct fcpio_tag ftag;
+	u32 id;
+	unsigned int tag;
+	struct fnic_io_req *io_req;
+	struct nvme_fc_cmd_iu *cmd_iu;
+	unsigned long start_time;
+	struct fnic_iport_s *iport;
+	struct fnic_tport_s *tport;
+
+	fcpio_header_dec(&desc->hdr, &type, &hdr_status, &ftag);
+	fcpio_tag_id_dec(&ftag, &id);
+	tag = id & FNIC_TAG_MASK;
+
+	if (tag >= fnic->fnic_max_tag_id) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "Tag out of range id:0x%x tag:0x%x hdr:%s\n",
+			      id, tag, fnic_fcpio_status_to_str(hdr_status));
+		return;
+	}
+	spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
+
+	io_req = nvfnic_find_io_req_by_tag(fnic, tag);
+	WARN_ON_ONCE(!io_req);
+	if (!io_req) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "IOREQ null hdr:%s tag:0x%x desc:%p\n",
+			      fnic_fcpio_status_to_str(hdr_status), tag, desc);
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "type: 0x%x status: 0x%x rsvd: 0x%x\n",
+			      desc->hdr.type, desc->hdr.status, desc->hdr._resvd);
+		spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+		return;
+	}
+
+	cmd_iu = io_req->fcp_req->cmdaddr;
+	FNIC_NVME_DBG(KERN_INFO, fnic,
+	      "Received ITMF completion tag: 0x%x hdr_status: %d csn: 0x%08x\n",
+	      tag, hdr_status, be32_to_cpu(cmd_iu->csn));
+
+	iport = io_req->iport;
+	tport = io_req->tport;
+
+	start_time = io_req->start_time;
+
+	/* Completion of abort cmd */
+	switch (hdr_status) {
+	case FCPIO_SUCCESS:
+		FNIC_NVME_DBG(KERN_DEBUG, fnic,
+				"Abort success received tag: 0x%x id: 0x%x\n",
+			      tag, id);
+		break;
+	case FCPIO_TIMEOUT:
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+				"Abort timeout received tag: 0x%x id: 0x%x\n",
+			      tag, id);
+		break;
+	case FCPIO_ITMF_REJECTED:
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+				"Abort reject received tag: 0x%x id: 0x%x\n",
+			      tag, id);
+		break;
+
+	case FCPIO_IO_NOT_FOUND:
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "Abort IO not found tag:0x%x id:0x%x\n",
+			      tag, id);
+		break;
+	default:
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+				"Abort unknown received tag: 0x%x id: 0x%x\n",
+			    tag, id);
+		break;
+	}
+
+	if (io_req->cmd_state != FNIC_IOREQ_ABTS_PENDING) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "Abort late completion tag:0x%x id:0x%x\n",
+			      tag, id);
+		/* This is a late completion. Ignore it */
+		spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+		return;
+	}
+
+	io_req->abts_state = hdr_status;
+
+	/* If the status is IO not found consider it as success.
+	 * NVME sends abort even if rport is down in which case
+	 * we will get FCPIO_TIMEOUT. Consider this as success.
+	 */
+	if ((hdr_status == FCPIO_IO_NOT_FOUND) ||
+	    (hdr_status == FCPIO_TIMEOUT) ||
+	    (hdr_status == FCPIO_ITMF_REJECTED))
+		io_req->abts_state = FCPIO_SUCCESS;
+
+	io_req->cmd_flags |= FNIC_IO_ABT_TERM_DONE;
+
+
+	if (io_req->abts_state == FCPIO_SUCCESS) {
+		io_req->fcp_req->transferred_length = 0;
+		io_req->fcp_req->rcv_rsplen = 0;
+		io_req->fcp_req->status = NVME_SC_ABORT_REQ;
+
+		nvfnic_release_nvme_ioreq_buf(iport, io_req);
+		if (io_req->done)
+			io_req->done(io_req);
+		spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+		return;
+	}
+	spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+}
+
+bool _cleanup_tport_io(struct sbitmap *map, unsigned int tag,
+			      void *data)
+{
+	struct fnic_tport_s *tport = data;
+	struct fnic_iport_s *iport = tport->iport;
+	struct fnic *fnic = iport->fnic;
+	struct fnic_io_req *io_req;
+	enum fnic_ioreq_state old_ioreq_state;
+	unsigned long flags;
+
+	io_req = nvfnic_find_io_req_by_tag(fnic, tag);
+	if (!io_req || io_req->tport != tport)
+		return true;
+
+	spin_lock_irqsave(&fnic->fnic_lock, flags);
+	if ((io_req->cmd_state == FNIC_IOREQ_ABTS_PENDING) ||
+	    (io_req->cmd_state == FNIC_DEV_RST_TERM_ISSUED)) {
+		FNIC_NVME_DBG(KERN_INFO, fnic,
+			      "Abort already pending 0x%x\n", io_req->tag);
+		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+		return true;
+	}
+
+	FNIC_NVME_DBG(KERN_ERR, fnic,
+		      "io_req tag: 0x%x abort after unregister timeout\n",
+		      io_req->tag);
+
+	old_ioreq_state = io_req->cmd_state;
+	io_req->cmd_state = FNIC_IOREQ_ABTS_PENDING;
+	io_req->abts_state = FCPIO_INVALID_CODE;
+	spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+
+	if (!nvfnic_queue_abort_io_req(fnic, io_req->tag,
+				      FCPIO_ITMF_ABT_TASK_TERM,
+				      io_req)) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			    "Failed to enqueue abort for ioreq tag: 0x%x\n",
+			    io_req->tag);
+		spin_lock_irqsave(&fnic->fnic_lock, flags);
+		io_req->cmd_state = old_ioreq_state;
+		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+	}
+	return true;
+}
+
+void
+nvfnic_cleanup_tport_io(struct fnic *fnic, struct fnic_tport_s *tport)
+{
+	unsigned long flags;
+	struct nvfnic_ls_req *nvfnic_ls_req, *next;
+	struct nvmefc_ls_req *lsreq;
+	uint16_t oxid;
+
+	spin_lock_irqsave(&fnic->fnic_lock, flags);
+	list_for_each_entry_safe(nvfnic_ls_req, next,
+				 &(tport->ls_req_list), list) {
+		lsreq = nvfnic_ls_req->ls_req;
+		if (!lsreq || (lsreq->private == NULL)) {
+			FNIC_NVME_DBG(KERN_INFO, fnic,
+				"fnic_cleanup_tport_io lsreq NULL\n");
+			continue;
+		}
+		list_del(&nvfnic_ls_req->list);
+		lsreq->private = NULL;
+		oxid = nvfnic_ls_req->oxid;
+		fdls_free_oxid(&fnic->iport, oxid, &nvfnic_ls_req->oxid);
+		nvfnic_ls_req->state = FNIC_LS_REQ_CMD_COMPLETE;
+		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+		timer_delete_sync(&nvfnic_ls_req->ls_req_timer);
+		lsreq->done(lsreq, -ENXIO);
+		spin_lock_irqsave(&fnic->fnic_lock, flags);
+	}
+
+	/* For link-down, IOs are freed by firmware reset completion */
+	if (fdls_get_state(&fnic->iport.fabric) == FDLS_STATE_LINKDOWN) {
+		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+		return;
+	}
+
+	spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+
+	sbitmap_for_each_set(&fnic->nvfnic_tag_map, _cleanup_tport_io, tport);
+}
+
+void
+nvfnic_terminate_tport_ls_reqs(struct fnic *fnic, struct fnic_tport_s *tport)
+{
+	struct nvmefc_ls_req *lsreq;
+	struct nvfnic_ls_req *nvfnic_ls_req, *next;
+	int count = 0;
+	uint16_t oxid;
+
+	spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
+	list_for_each_entry_safe(nvfnic_ls_req, next,
+		&(tport->ls_req_list), list) {
+
+		lsreq = nvfnic_ls_req->ls_req;
+		if (!lsreq || (lsreq->private == NULL)) {
+			FNIC_NVME_DBG(KERN_ERR, fnic,
+				"lsreq is NULL\n");
+			continue;
+		}
+		oxid = nvfnic_ls_req->oxid;
+		list_del(&nvfnic_ls_req->list);
+		lsreq->private = NULL;
+		fdls_free_oxid(&fnic->iport, oxid, &nvfnic_ls_req->oxid);
+		nvfnic_ls_req->state = FNIC_LS_REQ_CMD_COMPLETE;
+		spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+		timer_delete_sync(&nvfnic_ls_req->ls_req_timer);
+		lsreq->done(lsreq, -ENXIO);
+		count++;
+		spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
+	}
+	spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+
+	FNIC_NVME_DBG(KERN_INFO, fnic,
+		"fnic_terminate_tport_lsreqs tport: 0x%x: freed lsreq: %d\n",
+		tport->fcid, count);
+}
+
+bool _terminate_tport_ios(struct sbitmap *map, unsigned int tag,
+				       void *data)
+{
+	struct fnic_tport_s *tport = data;
+	struct fnic_iport_s *iport = tport->iport;
+	struct fnic *fnic = iport->fnic;
+	struct fnic_io_req *io_req;
+
+	io_req = fnic->sw_copy_wq[0].io_req_table[tag];
+	if (!io_req)
+		return true;
+
+	if (io_req->tport != tport)
+		return true;
+
+	FNIC_NVME_DBG(KERN_INFO, fnic,
+		      "Terminate tag: 0x%x (tport fcid 0x%x)\n",
+		      io_req->tag, io_req->tport->fcid);
+	nvfnic_fcpio_abort(iport->nv_lport,
+			   tport->nv_rport, NULL, io_req->fcp_req);
+	return true;
+}
+
+void nvfnic_terminate_tport_ios(struct fnic *fnic,
+				     struct fnic_tport_s *tport)
+{
+
+	sbitmap_for_each_set(&fnic->nvfnic_tag_map, _terminate_tport_ios, tport);
+
+}
+
+bool _cleanup_all_nvme_io(struct sbitmap *map, unsigned int tag,
+				 void *data)
+{
+	struct fnic_iport_s *iport = data;
+	struct fnic_io_req *io_req;
+
+	io_req = iport->fnic->sw_copy_wq[0].io_req_table[tag];
+	if (!io_req)
+		return true;
+
+	io_req->cmd_state = FNIC_DEV_RST_TERM_ISSUED;
+	io_req->fcp_req->status = NVME_SC_INTERNAL;
+	io_req->fcp_req->transferred_length = 0;
+	io_req->fcp_req->rcv_rsplen = 0;
+	nvfnic_release_nvme_ioreq_buf(iport, io_req);
+	io_req->done(io_req);
+	return true;
+}
+
+void nvfnic_cleanup_all_nvme_ios(struct fnic *fnic)
+{
+	struct fnic_iport_s *iport = &fnic->iport;
+
+	spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
+	sbitmap_for_each_set(&fnic->nvfnic_tag_map, _cleanup_all_nvme_io,
+			     iport);
+	spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+}
+
+void nvfnic_nvme_zero_devloss_tports(struct fnic *fnic)
+{
+	struct fnic_tport_s *tport, *next;
+
+	spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
+	list_for_each_entry_safe(tport, next, &fnic->iport.tport_list, links) {
+		if (tport->flags & FNIC_FDLS_NVME_REGISTERED) {
+			spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+			nvme_fc_set_remoteport_devloss(tport->nv_rport, 0);
+			spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
+		}
+	}
+	spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+}
+
+void nvfnic_nvme_unload(struct fnic *fnic)
+{
+	int ret = 0, hwq;
+	struct fnic_iport_s *iport = &fnic->iport;
+	unsigned long flags;
+	unsigned int time_wait =  FNIC_NVME_LPORT_REMOVE_WAIT;
+	unsigned int time_remain;
+	DECLARE_COMPLETION_ONSTACK(nvme_lport_unreg_done);
+
+	/* Mark iport state as INIT so that no IOs can be issued from this point */
+	spin_lock_irqsave(&fnic->fnic_lock, flags);
+	fnic->in_remove = 1;
+	fnic->iport.state = FNIC_IPORT_STATE_LINK_WAIT;
+	fnic->nvme_lport_unreg_done = &nvme_lport_unreg_done;
+	spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+
+	/*
+	 * If fnic is already processing link-down or fnic is held
+	 * in disabled state following a reboot we dont need to issue
+	 * firmware reset and unregister remote ports as it is already
+	 * done as part of link down handling.
+	 */
+	if (fdls_get_state(&iport->fabric) == FDLS_STATE_LINKDOWN) {
+		while (fnic->reset_in_progress == IN_PROGRESS) {
+			wait_for_completion_timeout(&fnic->reset_completion_wait,
+						    msecs_to_jiffies(5000));
+			FNIC_NVME_DBG(KERN_INFO, fnic,
+				      "rmmod waiting for reset %p\n", fnic);
+		}
+	} else if (fdls_get_state(&iport->fabric) != FDLS_STATE_INIT)
+		fnic_fcpio_reset(fnic);
+
+	/*
+	 * Mark state so that the workqueue thread stops forwarding
+	 * received frames and link events to the local port. ISR and
+	 * other threads that can queue work items will also stop
+	 * creating work items on the fnic workqueue
+	 */
+	nvfnic_nvme_zero_devloss_tports(fnic);
+	fnic_flush_tport_event_list(fnic);
+	fnic_delete_fcp_tports(fnic);
+
+	if (iport->flags & FNIC_LPORT_NVME_REGISTERED) {
+		ret = nvme_fc_unregister_localport(fnic->iport.nv_lport);
+		if (ret) {
+			FNIC_NVME_DBG(KERN_ERR, fnic,
+				"Unregister nvme localport failed: %d\n", ret);
+			return;
+		}
+
+		time_remain = wait_for_completion_timeout(fnic->nvme_lport_unreg_done,
+			msecs_to_jiffies(time_wait));
+		if (!time_remain) {
+			FNIC_NVME_DBG(KERN_ERR, fnic,
+				      "Local port removal timed out\n");
+			WARN_ON(1);
+		}
+		iport->flags &= ~FNIC_LPORT_NVME_REGISTERED;
+		kfree(iport->nv_tmpl);
+	}
+
+	nvfnic_flush_nvme_io_list(fnic);
+	sbitmap_free(&fnic->nvfnic_tag_map);
+	for (hwq = 0; hwq < fnic->wq_copy_count; hwq++)
+		kfree(fnic->sw_copy_wq[hwq].io_req_table);
+}
+
+struct nvfnic_ls_req*
+nvfnic_find_ls_req(struct fnic_tport_s *tport, uint16_t oxid)
+{
+	struct nvfnic_ls_req *nvfnic_ls_req, *next;
+
+	list_for_each_entry_safe(nvfnic_ls_req, next, &(tport->ls_req_list), list) {
+		if (nvfnic_ls_req->oxid == oxid)
+			return nvfnic_ls_req;
+	}
+	return NULL;
+}
+
+void nvfnic_fcpio_cmpl(struct fnic_io_req *io_req)
+{
+	struct fnic *fnic = io_req->iport->fnic;
+
+	nvfnic_free_fcpio_tag(io_req->iport, io_req);
+
+	llist_add(&io_req->nvfnic_io_cmpl, &fnic->nvme_io_event_llist);
+	atomic_inc(&fnic->nvme_io_event_queued);
+
+	io_req->waitq_start_time = jiffies;
+	queue_work(fnic_cmpl_queue, &fnic->nvme_io_cmpl_work);
+}
+
+void nvfnic_process_ls_abts_rsp(struct fnic_iport_s *iport,
+		struct fc_frame_header *fchdr)
+{
+	uint32_t tport_fcid;
+	struct fnic_tport_s *tport;
+	struct nvfnic_ls_req *nvfnic_ls_req;
+	struct nvmefc_ls_req *lsreq;
+	uint8_t *fcid;
+	uint16_t oxid = FNIC_STD_GET_OX_ID(fchdr);
+	struct fnic *fnic = iport->fnic;
+
+	fcid = FNIC_STD_GET_S_ID(fchdr);
+	tport_fcid = ntoh24(fcid);
+
+	tport = fnic_find_tport_by_fcid(iport, tport_fcid);
+	if (tport == NULL) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "tport: 0x%x not found\n", tport_fcid);
+		return;
+	}
+
+	nvfnic_ls_req = nvfnic_find_ls_req(tport, oxid);
+	if (nvfnic_ls_req == NULL) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "tport: 0x%x lsreq oxid: 0x%x not found\n",
+			      tport_fcid, oxid);
+		return;
+	}
+
+	lsreq = nvfnic_ls_req->ls_req;
+	if ((lsreq == NULL) || (lsreq->private == NULL)) {
+		FNIC_NVME_DBG(KERN_INFO, fnic,
+			      "tport: 0x%x lsreq oxid: 0x%x already aborted\n",
+			      tport_fcid, oxid);
+		return;
+	}
+
+	nvfnic_ls_req->state = FNIC_LS_REQ_ABTS_COMPLETE;
+
+	list_del(&nvfnic_ls_req->list);
+	fdls_free_oxid(iport, oxid, &nvfnic_ls_req->oxid);
+	lsreq->private = NULL;
+	spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+	timer_delete_sync(&nvfnic_ls_req->ls_req_timer);
+	lsreq->done(lsreq, NVME_SC_HOST_ABORTED_CMD);
+	spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
+}
+
+/**
+ * nvfnic_ls_rsp_recv - Handle received NVMe FC link service (LS) response
+ * @iport:  Pointer to the local FNIC port structure
+ * @fchdr:  Pointer to the Fibre Channel frame header for the
+ *          received response
+ * @len:    Length of the received frame
+ *
+ * This function processes link service (LS) responses received from
+ * NVMe Discovery Controllers or regular NVMe subsystems during
+ * association.
+ */
+void nvfnic_ls_rsp_recv(struct fnic_iport_s *iport,
+		struct fc_frame_header *fchdr, int len)
+{
+	uint8_t *fcid;
+	uint32_t tport_fcid;
+	struct fnic_tport_s *tport;
+	struct nvfnic_ls_req *nvfnic_ls_req;
+	struct nvmefc_ls_req *lsreq;
+	uint16_t oxid;
+	struct fnic *fnic = iport->fnic;
+
+	fcid = FNIC_STD_GET_S_ID(fchdr);
+	tport_fcid = ntoh24(fcid);
+
+	tport = fnic_find_tport_by_fcid(iport, tport_fcid);
+	if (!tport) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "tport: 0x%x not found\n", tport_fcid);
+		return;
+	}
+
+	oxid = FNIC_STD_GET_OX_ID(fchdr);
+	nvfnic_ls_req = nvfnic_find_ls_req(tport, oxid);
+	if (!nvfnic_ls_req) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "tport: 0x%x no nvfnic_lsreq for oxid: 0x%x\n",
+			      tport_fcid, oxid);
+		return;
+	}
+
+	lsreq = nvfnic_ls_req->ls_req;
+	if (!lsreq || (lsreq->private == NULL)) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "tport:0x%x lsreq:0x%x already done\n",
+			      tport_fcid, oxid);
+		return;
+	}
+	if (!lsreq->rspaddr) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "tport 0x%x lsreq 0x%x rspaddr NULL\n",
+			      tport_fcid, oxid);
+		return;
+	}
+
+	if ((nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_PENDING) ||
+	    (nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_STARTED)) {
+		FNIC_NVME_DBG(KERN_INFO, fnic,
+			      "tport 0x%x lsreq oxid: 0x%x abts pending\n",
+			      tport_fcid, oxid);
+		return;
+	}
+
+	nvfnic_ls_req->state = FNIC_LS_REQ_CMD_COMPLETE;
+
+	list_del_init(&nvfnic_ls_req->list);
+	lsreq->private = NULL;
+	fdls_free_oxid(iport, oxid, &nvfnic_ls_req->oxid);
+	timer_delete_sync(&nvfnic_ls_req->ls_req_timer);
+
+	FNIC_NVME_DBG(KERN_DEBUG, fnic,
+		      "tport:0x%x lsreq:0x%x completed\n",
+		      tport_fcid, oxid);
+
+	/* Copy the Response */
+	memcpy(lsreq->rspaddr,
+		(void *)((uint8_t *) fchdr + sizeof(struct fc_frame_header)),
+		len - sizeof(struct fc_frame_header));
+
+	spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+	lsreq->done(lsreq, 0);
+	spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
+}
+
+void nvfnic_ls_req_timeout(struct timer_list *t)
+{
+	struct nvfnic_ls_req *nvfnic_ls_req = timer_container_of(nvfnic_ls_req,
+			t, ls_req_timer);
+	struct fnic *fnic = nvfnic_ls_req->fnic;
+	struct nvmefc_ls_req *ls_req = nvfnic_ls_req->ls_req;
+	struct fnic_iport_s *iport = &fnic->iport;
+	struct fnic_tport_s *tport = (struct fnic_tport_s *) nvfnic_ls_req->tport;
+	uint16_t oxid = nvfnic_ls_req->oxid;
+
+	FNIC_NVME_DBG(KERN_INFO, fnic,
+		      "tport: 0x%x lsreq: 0x%x state: %d timeout\n",
+		      tport->fcid, nvfnic_ls_req->oxid,
+		      nvfnic_ls_req->state);
+	spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
+
+	if ((ls_req->private == NULL) ||
+	    (nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_STARTED)) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "tport: 0x%x lsreq: 0x%x already aborted\n",
+			      tport->fcid, nvfnic_ls_req->oxid);
+		spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+		return;
+	}
+
+	if (nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_PENDING) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "tport: 0x%x lsreq: 0x%x abort timeout\n",
+			      tport->fcid, nvfnic_ls_req->oxid);
+
+		list_del(&nvfnic_ls_req->list);
+		ls_req = nvfnic_ls_req->ls_req;
+		fdls_free_oxid(iport, oxid, &nvfnic_ls_req->oxid);
+		ls_req->private = NULL;
+		spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+		ls_req->done(ls_req, -ETIMEDOUT);
+		return;
+	} else if ((nvfnic_ls_req->state == FNIC_LS_REQ_CMD_PENDING) &&
+		   (nvfnic_transport_ready(iport, tport))) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "tport: 0x%x lsreq: 0x%x sending abort\n",
+			      tport->fcid, nvfnic_ls_req->oxid);
+	}
+
+	if (ls_req->private == NULL) {
+		spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+		return;
+	}
+
+	list_del(&nvfnic_ls_req->list);
+	ls_req = nvfnic_ls_req->ls_req;
+	fdls_free_oxid(iport, oxid, &nvfnic_ls_req->oxid);
+	ls_req->private = NULL;
+
+	spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+	ls_req->done(ls_req, -ETIMEDOUT);
+}
+
+void nvfnic_local_port_delete(struct nvme_fc_local_port *lport)
+{
+	struct fnic_iport_s *iport = (struct fnic_iport_s *) lport->private;
+	struct fnic *fnic = iport->fnic;
+	unsigned long flags = 0;
+
+	FNIC_NVME_DBG(KERN_INFO, fnic, "lport delete 0x%x\n",
+		      iport->fcid);
+
+	spin_lock_irqsave(&fnic->fnic_lock, flags);
+	if (fnic->nvme_lport_unreg_done)
+		complete(fnic->nvme_lport_unreg_done);
+	spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+}
+
+void nvfnic_remote_port_delete(struct nvme_fc_remote_port *rport)
+{
+	/*
+	 * Read rport->private without the lock only to find fnic.
+	 * Re-read it under fnic_lock to claim this delete callback, since
+	 * another callback may already have cleared it.
+	 */
+	struct fnic_tport_s *tport = (struct fnic_tport_s *)rport->private;
+	struct fnic_iport_s *iport;
+	struct fnic *fnic = NULL;
+	unsigned long flags = 0;
+
+	if (tport == NULL) {
+		pr_err("Attempt to delete already deleted tport\n");
+		return;
+	}
+
+	iport = tport->iport;
+	fnic = iport->fnic;
+	FNIC_NVME_DBG(KERN_INFO, fnic, "0x%x tport 0x%x\n",
+		      iport->fcid, tport->fcid);
+
+	spin_lock_irqsave(&fnic->fnic_lock, flags);
+	tport = (struct fnic_tport_s *)rport->private;
+	if (tport == NULL) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "NVMe tport callback after NULL set %p\n",
+			      rport);
+		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+		return;
+	}
+
+	iport = tport->iport;
+	rport->private = NULL;
+	fdls_delete_tport(iport, tport);
+
+	if (tport->flags & FNIC_FDLS_NVME_TPORT_CLEANUP_PENDING) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "tport %8x waiting on clean pending\n",
+			      tport->fcid);
+	}
+
+	while (tport->flags & FNIC_FDLS_NVME_TPORT_CLEANUP_PENDING) {
+		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+		msleep(2000);
+		spin_lock_irqsave(&fnic->fnic_lock, flags);
+	}
+
+	list_del(&tport->links);
+
+	if (tport->tport_del_done)
+		complete(tport->tport_del_done);
+
+	tport->flags |= FNIC_TPORT_CAN_BE_FREED;
+	spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+}
+
+int nvfnic_create_queue(struct nvme_fc_local_port *lport,
+		    unsigned int idx, u16 size, void **handle)
+{
+	struct fnic_iport_s *iport = (struct fnic_iport_s *)lport->private;
+	struct fnic *fnic  = iport->fnic;
+
+	FNIC_NVME_DBG(KERN_DEBUG, fnic,
+		      "iport:0x%x queue:%d size:%d\n", iport->fcid, idx, size);
+
+	if (idx > fnic->wq_copy_count)
+		return -EINVAL;
+
+	if (idx == 0) {
+		/* Admin queue */
+		*handle = &fnic->hw_copy_wq[0];
+	} else {
+		/* IO queues */
+		*handle = &fnic->hw_copy_wq[idx-1];
+	}
+
+	return 0;
+}
+
+void nvfnic_ls_req_abort(struct nvme_fc_local_port *lport,
+		   struct nvme_fc_remote_port *rport,
+		   struct nvmefc_ls_req *lsreq)
+{
+	struct fnic_iport_s *iport = lport->private;
+	struct fnic *fnic = iport->fnic;
+	struct fnic_tport_s *tport;
+	struct nvfnic_ls_req *nvfnic_ls_req;
+	uint16_t oxid;
+	int timeout;
+
+	spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
+
+	tport = (struct fnic_tport_s *) rport->private;
+	/* find the request */
+	nvfnic_ls_req = lsreq->private;
+
+	if (nvfnic_ls_req == NULL) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "0x%x null lsreq already scheduled for abort\n",
+			      iport->fcid);
+		spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+		return;
+	}
+
+	if (nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_PENDING) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "0x%x lsreq 0x%x already scheduled for abort\n",
+			      iport->fcid, nvfnic_ls_req->oxid);
+		spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+		return;
+	}
+
+	FNIC_NVME_DBG(KERN_INFO, fnic,
+		      "0x%x lsreq 0x%x abts\n",
+		      iport->fcid, nvfnic_ls_req->oxid);
+
+	nvfnic_ls_req->state = FNIC_LS_REQ_CMD_ABTS_STARTED;
+	spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+	timer_delete_sync(&nvfnic_ls_req->ls_req_timer);
+
+	spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
+	nvfnic_ls_req = lsreq->private;
+
+	if ((nvfnic_ls_req == NULL) ||
+	    (nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_PENDING)) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "lsreq timeout raced with midlayer abort\n");
+		spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+		return;
+	}
+
+	/* Basic validations of the state */
+	if (!nvfnic_transport_ready(iport, tport)) {
+		/* If iport or tport offline, it will be handled from that event */
+		oxid = nvfnic_ls_req->oxid;
+		lsreq->private = NULL;
+		list_del(&nvfnic_ls_req->list);
+		fdls_free_oxid(iport, oxid, &nvfnic_ls_req->oxid);
+		spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+		lsreq->done(lsreq, -ENXIO);
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+				"nvfnic_lsreq_abort transport not ready\n");
+		return;
+	}
+
+	/* Mark the state and flags */
+	nvfnic_ls_req->state = FNIC_LS_REQ_CMD_ABTS_PENDING;
+	timeout = FNIC_LS_REQ_TMO_MSECS(lsreq->timeout);
+	mod_timer(&nvfnic_ls_req->ls_req_timer,
+		  round_jiffies(jiffies + msecs_to_jiffies(timeout)));
+	spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
+}
+
+bool nvfnic_queue_abort_io_req(struct fnic *fnic, int tag,
+			 u32 task_req, struct fnic_io_req *io_req)
+{
+	int idx;
+	unsigned long flags;
+
+	idx = io_req->wq - &fnic->hw_copy_wq[0];
+
+	atomic_inc(&fnic->in_flight);
+
+	spin_lock_irqsave(&fnic->wq_copy_lock[idx], flags);
+
+	if (vnic_wq_copy_desc_avail(io_req->wq) <= fnic->wq_copy_desc_low[idx])
+		free_wq_copy_descs(fnic, io_req->wq, idx);
+
+	if (!vnic_wq_copy_desc_avail(io_req->wq)) {
+		spin_unlock_irqrestore(&fnic->wq_copy_lock[idx], flags);
+		atomic_dec(&fnic->in_flight);
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+				"tag 0x%x failure: no descriptors\n", tag);
+		return false;
+	}
+	fnic_queue_wq_copy_desc_itmf(io_req->wq, tag | FNIC_TAG_ABORT,
+				     0, task_req, tag, NULL, io_req->port_id,
+				     fnic->config.ra_tov, fnic->config.ed_tov);
+
+
+	spin_unlock_irqrestore(&fnic->wq_copy_lock[idx], flags);
+	atomic_dec(&fnic->in_flight);
+
+	return true;
+}
+
+void nvfnic_fcpio_abort(struct nvme_fc_local_port *lport,
+			struct nvme_fc_remote_port *rport,
+			void *hw_queue_handle, struct nvmefc_fcp_req *fcp_req)
+{
+	struct fnic_iport_s *iport = lport->private;
+	struct fnic *fnic = iport->fnic;
+	struct nvme_fc_cmd_iu *cmd_iu = fcp_req->cmdaddr;
+	struct fnic_io_req *io_req = (struct fnic_io_req *)fcp_req->private;
+	unsigned int tag = io_req->tag;
+	unsigned long flags = 0;
+	unsigned int task_req;
+	enum fnic_ioreq_state old_ioreq_state;
+
+	spin_lock_irqsave(&fnic->fnic_lock, flags);
+
+	if (io_req->tag == FNIC_NVME_NO_FREE_TAG) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			      "tag: (0x%x) tport_fcid: 0x%x\n",
+			      io_req->tag, io_req->tport->fcid);
+		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+		return;
+	}
+
+	if (io_req != nvfnic_find_io_req_by_tag(fnic, io_req->tag)) {
+		FNIC_NVME_DBG(KERN_INFO, fnic,
+			      "cmd tag freed or not issued:0x%x sn:0x%08x\n",
+			      io_req->tag, be32_to_cpu(cmd_iu->csn));
+		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+		return;
+	}
+
+	if (io_req->cmd_state == FNIC_IOREQ_CMD_COMPLETE) {
+		FNIC_NVME_DBG(KERN_INFO, fnic,
+			      "IO already completed before abort: 0x%x\n", tag);
+		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+		return;
+	}
+
+	if ((io_req->cmd_state == FNIC_IOREQ_ABTS_PENDING) ||
+	    (io_req->cmd_state == FNIC_DEV_RST_TERM_ISSUED)) {
+		FNIC_NVME_DBG(KERN_INFO, fnic, "abort already pending %d\n",
+			      tag);
+		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+		return;
+	}
+
+	if (io_req->cmd_state != FNIC_IOREQ_CMD_PENDING) {
+		FNIC_NVME_DBG(KERN_INFO, fnic,
+			      "io_req completed or aborted for tag:0x%x\n",
+			      tag);
+		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+		return;
+	}
+
+	FNIC_NVME_DBG(KERN_INFO, fnic, "in abort cmd_sn:%08x %llx tag: 0x%x\n",
+		      be32_to_cpu(cmd_iu->csn),
+		      le64_to_cpu(cmd_iu->sqe.rw.slba), io_req->tag);
+
+	if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_IO_BLOCKED))) {
+		FNIC_NVME_DBG(KERN_INFO, fnic,
+			      "abort tag:0x%x returned during fw reset\n",
+			      io_req->tag);
+		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+		return;
+	}
+	atomic_inc(&fnic->in_flight);
+
+	if (fdls_tport_is_offline(io_req->tport) ||
+	    (io_req->cmd_state == FNIC_IOREQ_RESET_TERM)) {
+		task_req = FCPIO_ITMF_ABT_TASK_TERM;
+	} else {
+		task_req = FCPIO_ITMF_ABT_TASK;
+	}
+
+
+
+	old_ioreq_state = io_req->cmd_state;
+	io_req->cmd_state = FNIC_IOREQ_ABTS_PENDING;
+	io_req->abts_state = FCPIO_INVALID_CODE;
+
+	spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+
+	if (!nvfnic_queue_abort_io_req(fnic, io_req->tag, task_req, io_req)) {
+		FNIC_NVME_DBG(KERN_INFO, fnic,
+			      "Abort io req queue failed\n");
+		spin_lock_irqsave(&fnic->fnic_lock, flags);
+		io_req->cmd_state = old_ioreq_state;
+		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+	}
+	atomic_dec(&fnic->in_flight);
+}
+
+struct
+nvme_fc_port_template nvfnic_port = {
+	.localport_delete = nvfnic_local_port_delete,
+	.remoteport_delete = nvfnic_remote_port_delete,
+	.create_queue = nvfnic_create_queue,
+	.delete_queue = NULL,
+	.ls_req = NULL,
+	.ls_abort = nvfnic_ls_req_abort,
+	.fcp_io = nvfnic_fcpio_send,
+	.fcp_abort = nvfnic_fcpio_abort,
+	.max_hw_queues = 1,
+	.max_sgl_segments = 256,
+	.max_dif_sgl_segments = 64,
+	.dma_boundary = 0xFFFFFFFF,
+	.local_priv_sz = sizeof(struct fnic_iport_s *),
+	.remote_priv_sz = sizeof(struct fnic_tport_s *),
+	.lsrqst_priv_sz = sizeof(struct nvfnic_ls_req),
+	.fcprqst_priv_sz = sizeof(struct fnic_io_req),
+};
+
+void nvfnic_flush_nvme_io_list(struct fnic *fnic)
+{
+	queue_work(fnic_cmpl_queue, &fnic->nvme_io_cmpl_work);
+	flush_work(&fnic->nvme_io_cmpl_work);
+}
+
+void nvfnic_nvme_iodone_work(struct work_struct *work)
+{
+	struct fnic *fnic = container_of(work, struct fnic, nvme_io_cmpl_work);
+	struct llist_node *llnode;
+	struct fnic_io_req *io_req, *tmp;
+
+	llnode = llist_del_all(&fnic->nvme_io_event_llist);
+	llist_for_each_entry_safe(io_req, tmp, llnode, nvfnic_io_cmpl) {
+		atomic_dec(&fnic->nvme_io_event_queued);
+		io_req->fcp_req->done(io_req->fcp_req);
+	}
+}
+
+void nvfnic_exch_reset(struct fnic_iport_s *iport, struct fnic_tport_s *tport)
+{
+	FNIC_NVME_DBG(KERN_DEBUG, iport->fnic,
+			"0x%x: Exchange reset scheduled for tport: 0x%x\n",
+		    iport->fcid, tport->fcid);
+
+	nvfnic_terminate_tport_ls_reqs(iport->fnic, tport);
+	nvfnic_terminate_tport_ios(iport->fnic, tport);
+}
+
+void nvfnic_delete_tport(struct fnic_iport_s *iport,
+						struct fnic_tport_s *tport,
+						unsigned long flags)
+{
+	struct fnic *fnic = iport->fnic;
+	int ret;
+	unsigned int time_wait = FNIC_NVME_TPORT_REMOVE_WAIT;
+	unsigned int time_remain;
+	DECLARE_COMPLETION_ONSTACK(tm_done);
+	unsigned int fcid;
+	int count = 0;
+
+	if (!tport)
+		return;
+
+	fcid = tport->fcid;
+	fdls_set_tport_state(tport, FDLS_TGT_STATE_OFFLINE);
+
+	FNIC_NVME_DBG(KERN_DEBUG, fnic,
+			"0x%x: scheduled deletion for tport: 0x%x\n",
+		    iport->fcid, tport->fcid);
+
+	if (!(tport->flags & FNIC_FDLS_NVME_REGISTERED)) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			"0x%x: tport: 0x%x not registered. Freeing\n",
+		    iport->fcid, tport->fcid);
+		kfree(tport);
+		return;
+	}
+
+	tport->tport_del_done = &tm_done;
+
+	tport->flags |= FNIC_FDLS_TPORT_DELETED;
+	spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+	ret = nvme_fc_unregister_remoteport(tport->nv_rport);
+	if (ret) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			    "tport: 0x%x unregister failed %d\n",
+			    tport->fcid, ret);
+		spin_lock_irqsave(&fnic->fnic_lock, flags);
+		return;
+	}
+	time_remain = wait_for_completion_timeout(tport->tport_del_done,
+				msecs_to_jiffies(time_wait));
+
+	FNIC_NVME_DBG(KERN_DEBUG, fnic,
+		      "tport: 0x%x wait for deletion done\n",
+		      tport->fcid);
+
+	spin_lock_irqsave(&fnic->fnic_lock, flags);
+	tport->tport_del_done = NULL;
+
+	if (!time_remain) {
+		FNIC_NVME_DBG(KERN_ERR, fnic,
+			    "tport: 0x%x nvme midlayer completion timed out\n",
+			    tport->fcid);
+
+		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+		nvfnic_cleanup_tport_io(fnic, tport);
+		spin_lock_irqsave(&fnic->fnic_lock, flags);
+		tport->flags &= ~FNIC_FDLS_NVME_TPORT_CLEANUP_PENDING;
+	} else {
+		while (!(tport->flags & FNIC_TPORT_CAN_BE_FREED) && (count < 8)) {
+			count++;
+			spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+			msleep(2000);
+			spin_lock_irqsave(&fnic->fnic_lock, flags);
+		}
+		if (tport->flags & FNIC_TPORT_CAN_BE_FREED)
+			kfree(tport);
+	}
+	FNIC_NVME_DBG(KERN_INFO, fnic,
+		      "tport: 0x%x delete complete\n", fcid);
+}
+
+int nvfnic_add_tport(struct fnic *fnic, struct fnic_tport_s *tport)
+{
+	struct fnic_iport_s *iport = &fnic->iport;
+	struct nvme_fc_port_info pinfo;
+	int ret = 0;
+
+	FNIC_NVME_DBG(KERN_INFO, fnic,
+		      "Adding tport to nvme wwpn: 0x%llx\n",
+		      tport->wwpn);
+
+	memset(&pinfo, 0, sizeof(struct nvme_fc_port_info));
+
+	pinfo.port_name = tport->wwpn;
+	pinfo.node_name = tport->wwnn;
+	pinfo.port_role = FC_PORT_ROLE_NVME_DISCOVERY | FC_PORT_ROLE_NVME_TARGET;
+	pinfo.port_id = tport->fcid;
+	pinfo.dev_loss_tmo = nvme_dev_loss_tmo;
+
+	ret = nvme_fc_register_remoteport(iport->nv_lport, &pinfo,
+					  &tport->nv_rport);
+	if (ret) {
+		FNIC_NVME_DBG(KERN_INFO, fnic,
+			    "Failed to register tport wwpn: 0x%llx ret: %d\n",
+			    tport->wwpn, ret);
+		return ret;
+	}
+	tport->flags |= FNIC_FDLS_NVME_REGISTERED;
+	tport->nv_rport->private = tport;
+
+	snprintf(tport->str_wwpn, sizeof(tport->str_wwpn), "0x%llx", tport->wwpn);
+	snprintf(tport->str_wwnn, sizeof(tport->str_wwnn), "0x%llx", tport->wwnn);
+	return ret;
+}
+
+int nvfnic_add_lport(struct fnic *fnic)
+{
+	struct nvme_fc_port_info pinfo;
+	struct fnic_iport_s *iport = &fnic->iport;
+	int ret = 0;
+
+	FNIC_NVME_DBG(KERN_INFO, fnic,
+		      "Adding lport nvme wwpn: 0x%llx\n",
+		      iport->wwpn);
+
+	pinfo.node_name = iport->wwnn;
+	pinfo.port_name = iport->wwpn;
+	pinfo.port_role = FC_PORT_ROLE_NVME_INITIATOR;
+	pinfo.port_id = iport->fcid;
+
+	nvfnic_reset_fcpio_tag_pool(iport);
+
+	if (!(iport->flags & FNIC_LPORT_NVME_REGISTERED)) {
+		iport->nv_tmpl = kzalloc_obj(struct nvme_fc_port_template, GFP_ATOMIC);
+		if (!iport->nv_tmpl) {
+			FNIC_FCS_DBG(KERN_INFO, fnic,
+				     "iport:0x%x NVMe tmpl alloc failed\n",
+				     iport->fcid);
+			return -ENOMEM;
+		}
+		memcpy(iport->nv_tmpl, &nvfnic_port,
+		       sizeof(struct nvme_fc_port_template));
+		iport->nv_tmpl->max_hw_queues = fnic->wq_copy_count;
+
+		ret = nvme_fc_register_localport(&pinfo, iport->nv_tmpl,
+						 get_device(&fnic->pdev->dev),
+						 &iport->nv_lport);
+		if (ret) {
+			FNIC_NVME_DBG(KERN_ERR, fnic,
+					"Failed to add wwpn: 0x%llx ret: %d\n",
+					iport->wwpn, ret);
+			kfree(iport->nv_tmpl);
+			return ret;
+		}
+		iport->flags |= FNIC_LPORT_NVME_REGISTERED;
+		iport->nv_lport->private = iport;
+	}
+
+	sprintf(iport->str_wwpn, "0x%llx", iport->wwpn);
+	sprintf(iport->str_wwnn, "0x%llx", iport->wwnn);
+
+	FNIC_NVME_DBG(KERN_INFO, fnic,
+		      "Successfully added lport wwpn: 0x%llx\n",
+		      iport->wwpn);
+	return 0;
+}
diff --git a/drivers/scsi/fnic/fnic_nvme.h b/drivers/scsi/fnic/fnic_nvme.h
new file mode 100644
index 000000000000..22d69f40cd65
--- /dev/null
+++ b/drivers/scsi/fnic/fnic_nvme.h
@@ -0,0 +1,134 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright 2008 Cisco Systems, Inc.  All rights reserved.
+ * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
+ */
+#ifndef _FNIC_NVME_H
+#define _FNIC_NVME_H
+
+#include "fdls_fc.h"
+#include "fnic_fdls.h"
+
+#define FNIC_NVME_TPORT_REMOVE_WAIT (5 * 1000)
+#define FNIC_NVME_TPORT_LIST_EMPTY_WAIT (FNIC_NVME_TPORT_REMOVE_WAIT * 2)
+#define FNIC_NVME_LPORT_REMOVE_WAIT (2 * 60 * 1000)
+
+#define FNIC_LS_REQ_FLAGS_NONE      0x0
+#define FNIC_LS_REQ_FLAGS_ABORTED   0x1
+#define FNIC_LS_REQ_FLAGS_DONE      0x2
+#define FNIC_LS_REQ_ABORT_COMPLETED 0x4
+#define FNIC_STATUS_LS_REQ_ABORTED 0x1
+
+#define FNIC_LS_REQ_MIN_TMO_SECS (2)
+#define FNIC_LS_REQ_MAX_TMO_SECS (5)
+#define FNIC_NVME_ADMIN_IO_TIMEOUT 30000	/* mSec */
+#define FNIC_NVME_NO_FREE_TAG     (0xFFFF)
+
+#define FNIC_LS_REQ_TMO_MSECS(tmo)    (((tmo >= FNIC_LS_REQ_MIN_TMO_SECS) && \
+					(tmo <= FNIC_LS_REQ_MAX_TMO_SECS)) ? \
+					(tmo * 1000) : (FNIC_LS_REQ_MIN_TMO_SECS * 1000))
+
+#define IS_ADMIN_IO(_io_req)     \
+	(NVME_CMD_FLAGS(_io_req) & FNIC_NVME_ADMIN_IO_TIMER_PENDING)
+
+extern unsigned int nvme_max_ios_to_process;
+extern unsigned int nvme_dev_loss_tmo;
+extern spinlock_t fnic_list_lock;
+
+enum nvfnic_lsreq_state_e {
+	FNIC_LS_REQ_CMD_INIT = 0,
+	FNIC_LS_REQ_CMD_PENDING,
+	FNIC_LS_REQ_CMD_ABTS_PENDING,
+	FNIC_LS_REQ_CMD_COMPLETE,
+	FNIC_LS_REQ_ABTS_COMPLETE,
+	FNIC_LS_REQ_CMD_ABTS_STARTED,
+};
+
+struct fnic_nvme_tag {
+	struct list_head free_list;
+	int tag_id;
+};
+
+struct nvfnic_ls_req {
+	struct list_head list;
+	struct nvmefc_ls_req *ls_req;
+	uint16_t oxid;
+	struct timer_list ls_req_timer;
+	struct fnic *fnic;
+	struct fnic_tport_s *tport;
+	int state;
+	unsigned int flags;
+};
+
+int nvfnic_nvme_io_done_handler(void *arg);
+struct fnic_io_req *nvfnic_find_io_req_by_tag(struct fnic *fnic, uint16_t tag);
+void nvfnic_reset_fcpio_tag_pool(struct fnic_iport_s *iport);
+int nvfnic_add_lport(struct fnic *fnic);
+void nvfnic_fcpio_abort(struct nvme_fc_local_port *lport,
+			struct nvme_fc_remote_port *rport,
+			void *hw_queue_handle, struct nvmefc_fcp_req *fcp_req);
+void nvfnic_remote_port_delete(struct nvme_fc_remote_port *rport);
+void nvfnic_local_port_delete(struct nvme_fc_local_port *lport);
+int nvfnic_dma_map_sgl(struct fnic *fnic, struct fnic_io_req *io_req,
+		       int sg_count);
+void nvfnic_dma_unmap_sgl(struct fnic *fnic, struct fnic_io_req *io_req);
+int nvfnic_get_sg_count(struct fnic_io_req *io_req);
+void nvfnic_release_nvme_ioreq_buf(struct fnic_iport_s *iport,
+				   struct fnic_io_req *io_req);
+void nvfnic_dump_nvcmd(struct fnic_io_req *io_req, uint8_t flags);
+bool _cleanup_tport_io(struct sbitmap *map, unsigned int tag, void *data);
+void nvfnic_flush_nvme_io_list(struct fnic *fnic);
+void nvfnic_fcpio_cmpl(struct fnic_io_req *io_req);
+bool nvfnic_transport_ready(struct fnic_iport_s *iport,
+			    struct fnic_tport_s *tport);
+int nvfnic_alloc_fcpio_tag(struct fnic_iport_s *iport,
+				struct fnic_io_req *io_req);
+int nvfnic_queuecommand(struct fnic_io_req *io_req);
+void nvfnic_free_fcpio_tag(struct fnic_iport_s *iport,
+			   struct fnic_io_req *io_req);
+void nvfnic_delete_lport(struct fnic_iport_s *iport);
+int nvfnic_add_tport(struct fnic *fnic, struct fnic_tport_s *tport);
+void nvfnic_cleanup_all_nvme_ios(struct fnic *fnic);
+void nvfnic_delete_tport_work(struct work_struct *work);
+void nvfnic_delete_tport(struct fnic_iport_s *iport,
+			 struct fnic_tport_s *tport, unsigned long flags);
+void nvfnic_fcpio_nvme_fast_cmpl_handler(struct fnic *fnic,
+					 struct fcpio_fw_req *desc);
+void nvfnic_ls_rsp_recv(struct fnic_iport_s *iport,
+			struct fc_frame_header *fchdr, int len);
+void nvfnic_process_ls_abts_rsp(struct fnic_iport_s *iport,
+				struct fc_frame_header *fchdr);
+void nvfnic_admin_io_timeout(struct timer_list *t);
+void nvfnic_fcpio_nvme_itmf_cmpl_handler(struct fnic *fnic,
+					 struct fcpio_fw_req *desc);
+void nvfnic_nvme_zero_devloss_tports(struct fnic *fnic);
+bool nvfnic_queue_abort_io_req(struct fnic *fnic, int tag, u32 task_req,
+			       struct fnic_io_req *io_req);
+void nvfnic_ls_req_abort(struct nvme_fc_local_port *lport,
+			 struct nvme_fc_remote_port *rport,
+			 struct nvmefc_ls_req *lsreq);
+int nvfnic_create_queue(struct nvme_fc_local_port *lport, unsigned int idx,
+			u16 size, void **handle);
+void nvfnic_ls_req_timeout(struct timer_list *t);
+uint16_t nvfnic_alloc_ls_req_oxid(struct fnic_iport_s *iport);
+struct nvfnic_ls_req *nvfnic_find_ls_req(struct fnic_tport_s *tport,
+					 uint16_t oxid);
+void nvfnic_terminate_tport_ios(struct fnic *fnic, struct fnic_tport_s *tport);
+bool _terminate_tport_ios(struct sbitmap *map, unsigned int tag, void *data);
+bool _cleanup_all_nvme_io(struct sbitmap *map, unsigned int tag, void *data);
+void nvfnic_cleanup_all_nvme_ios(struct fnic *fnic);
+int nvfnic_fcpio_send(struct nvme_fc_local_port *lport,
+		      struct nvme_fc_remote_port *rport, void *hw_queue_handle,
+		      struct nvmefc_fcp_req *fcp_req);
+void nvfnic_fcpio_ersp_cmpl_handler(struct fnic *fnic,
+				    struct fcpio_fw_req *desc, int sw_flag);
+void nvfnic_terminate_tport_ls_reqs(struct fnic *fnic,
+				    struct fnic_tport_s *tport);
+void nvfnic_terminate_tport_admin_ios(struct fnic *fnic,
+				      struct fnic_tport_s *tport);
+void nvfnic_cleanup_tport_io(struct fnic *fnic, struct fnic_tport_s *tport);
+void nvfnic_nvme_unload(struct fnic *fnic);
+void nvfnic_exch_reset(struct fnic_iport_s *iport, struct fnic_tport_s *tport);
+extern const char *fnic_fcpio_status_to_str(unsigned int status);
+void nvfnic_nvme_iodone_work(struct work_struct *work);
+#endif				/* _FNIC_NVME_H */
diff --git a/drivers/scsi/fnic/fnic_res.h b/drivers/scsi/fnic/fnic_res.h
index 92a2fcfd3ea9..12a2a356dc13 100644
--- a/drivers/scsi/fnic/fnic_res.h
+++ b/drivers/scsi/fnic/fnic_res.h
@@ -126,7 +126,8 @@ static inline void fnic_queue_wq_copy_desc_itmf(struct vnic_wq_copy *wq,
 	desc->u.itmf.tm_req = tm_req;       /* SCSI Task Management request */
 	desc->u.itmf.t_tag = tm_id;         /* tag of fcpio to be aborted */
 	desc->u.itmf._resvd = 0;
-	memcpy(desc->u.itmf.lun, lun, LUN_ADDRESS);  /* LUN address */
+	if (lun)
+		memcpy(desc->u.itmf.lun, lun, LUN_ADDRESS);  /* LUN address */
 	desc->u.itmf._resvd1 = 0;
 	hton24(desc->u.itmf.d_id, d_id);    /* FC vNIC only: Target D_ID */
 	desc->u.itmf.r_a_tov = r_a_tov;     /* FC vNIC only: R_A_TOV in msec */
@@ -222,6 +223,35 @@ static inline void fnic_queue_rq_desc(struct vnic_rq *rq,
 	vnic_rq_post(rq, os_buf, 0, dma_addr, len);
 }
 
+static inline void fnic_queue_wq_copy_desc_nvme_io(struct vnic_wq_copy *wq,
+						   u32 req_id,
+						   u8 spl_flags,
+						   u32 sgl_cnt,
+						   u64 sgl_addr,
+						   u8 flags, u8 *nvme_cmd_iu,
+						   u16 cmd_len,
+						   u32 data_len,
+						   u32 d_id, u32 mss,
+						   u32 ratov, u32 edtov)
+{
+	struct fcpio_host_req *desc = vnic_wq_copy_next_desc(wq);
+
+	desc->hdr.type = FCPIO_NVME_CMD; /* enum fcpio_type */
+	desc->hdr.status = 0; /* header status entry */
+	desc->hdr._resvd = 0; /* reserved */
+	desc->hdr.tag.u.req_id = req_id; /* id for this request */
+	desc->u.nvcmnd.sgl_cnt = sgl_cnt; /* scatter-gather list count */
+	desc->u.nvcmnd.sgl_addr = sgl_addr; /* scatter-gather list addr */
+	desc->u.nvcmnd._resvd1 = 0; /* reserved: should be 0 */
+	desc->u.nvcmnd.flags = flags; /* command flags */
+	memset(desc->u.nvcmnd.nvme_cmnd, 0, NVME_CMD_SZ);
+	memcpy(desc->u.nvcmnd.nvme_cmnd, nvme_cmd_iu, cmd_len); /* SCSI CDB */
+	desc->u.nvcmnd.cmd_len = cmd_len;
+	desc->u.nvcmnd.data_len = data_len; /* length of data expected */
+	hton24(desc->u.nvcmnd.d_id, d_id); /* FC vNIC only: Target D_ID */
+
+	vnic_wq_copy_post(wq);
+}
 
 struct fnic;
 
diff --git a/drivers/scsi/fnic/fnic_scsi.c b/drivers/scsi/fnic/fnic_scsi.c
index 37d1e136b7b9..04ab384033b1 100644
--- a/drivers/scsi/fnic/fnic_scsi.c
+++ b/drivers/scsi/fnic/fnic_scsi.c
@@ -96,7 +96,7 @@ static const char *fnic_ioreq_state_to_str(unsigned int state)
 	return fnic_ioreq_state_str[state];
 }
 
-static const char *fnic_fcpio_status_to_str(unsigned int status)
+const char *fnic_fcpio_status_to_str(unsigned int status)
 {
 	if (status >= ARRAY_SIZE(fcpio_status_str) || !fcpio_status_str[status])
 		return "unknown";
@@ -186,7 +186,7 @@ fnic_count_lun_ioreqs(struct fnic *fnic, struct scsi_device *scsi_device)
 }
 
 /* Free up Copy Wq descriptors. Called with copy_wq lock held */
-static int free_wq_copy_descs(struct fnic *fnic, struct vnic_wq_copy *wq, unsigned int hwq)
+int free_wq_copy_descs(struct fnic *fnic, struct vnic_wq_copy *wq, unsigned int hwq)
 {
 	/* if no Ack received from firmware, then nothing to clean */
 	if (!fnic->fw_ack_recd[hwq])
diff --git a/drivers/scsi/fnic/vnic_devcmd.h b/drivers/scsi/fnic/vnic_devcmd.h
index f876d223b2b4..20878a3e9e26 100644
--- a/drivers/scsi/fnic/vnic_devcmd.h
+++ b/drivers/scsi/fnic/vnic_devcmd.h
@@ -430,7 +430,7 @@ struct vnic_devcmd_notify {
 struct vnic_devcmd_provinfo {
 	u8 oui[3];
 	u8 type;
-	u8 data[];
+	u8 data[0];
 };
 
 /*
-- 
2.47.1