[PATCH v2 1/5] ufs: mcq: Add supporting functions for mcq abort

[PATCH v2 1/5] ufs: mcq: Add supporting functions for mcq abort

[Bao D. Nguyen]

Add supporting functions to handle ufs abort in mcq mode.

Signed-off-by: Bao D. Nguyen <quic_nguyenb@quicinc.com>
---
 drivers/ufs/core/ufs-mcq.c     | 190 +++++++++++++++++++++++++++++++++++++++++
 drivers/ufs/core/ufshcd-priv.h |  10 +++
 drivers/ufs/core/ufshcd.c      |   2 -
 include/ufs/ufshci.h           |  17 ++++
 4 files changed, 217 insertions(+), 2 deletions(-)

diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c
index 31df052..a463674 100644
--- a/drivers/ufs/core/ufs-mcq.c
+++ b/drivers/ufs/core/ufs-mcq.c
@@ -12,6 +12,9 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include "ufshcd-priv.h"
+#include <linux/delay.h>
+#include <scsi/scsi_cmnd.h>
+#include <linux/bitfield.h>
 
 #define MAX_QUEUE_SUP GENMASK(7, 0)
 #define UFS_MCQ_MIN_RW_QUEUES 2
@@ -27,6 +30,9 @@
 #define MCQ_ENTRY_SIZE_IN_DWORD	8
 #define CQE_UCD_BA GENMASK_ULL(63, 7)
 
+/* Max mcq register polling time in milisecond unit */
+#define MCQ_POLL_MS 500
+
 static int rw_queue_count_set(const char *val, const struct kernel_param *kp)
 {
 	return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_RW_QUEUES,
@@ -429,3 +435,187 @@ int ufshcd_mcq_init(struct ufs_hba *hba)
 	host->host_tagset = 1;
 	return 0;
 }
+
+static int ufshcd_mcq_poll_register(void __iomem *reg, u32 mask,
+				u32 val, unsigned long timeout_ms)
+{
+	unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
+	int err = 0;
+
+	/* ignore bits that we don't intend to wait on */
+	val = val & mask;
+
+	while ((readl(reg) & mask) != val) {
+		udelay(20);
+		if (time_after(jiffies, timeout)) {
+			err = -ETIMEDOUT;
+			break;
+		}
+	}
+
+	return err;
+}
+
+static int ufshcd_mcq_sq_stop(struct ufs_hba *hba, struct ufs_hw_queue *hwq)
+{
+	void __iomem *reg;
+	u32 i = hwq->id;
+	int err;
+
+	writel(SQ_STOP, mcq_opr_base(hba, OPR_SQD, i) + REG_SQRTC);
+	reg = mcq_opr_base(hba, OPR_SQD, i) + REG_SQRTS;
+	err = ufshcd_mcq_poll_register(reg, SQ_STS, SQ_STS, MCQ_POLL_MS);
+	if (err)
+		dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n",
+			__func__, i, err);
+	return err;
+}
+
+static int ufshcd_mcq_sq_start(struct ufs_hba *hba, struct ufs_hw_queue *hwq)
+{
+	void __iomem *reg;
+	u32 i = hwq->id;
+	int err;
+
+	writel(SQ_START, mcq_opr_base(hba, OPR_SQD, i) + REG_SQRTC);
+	reg = mcq_opr_base(hba, OPR_SQD, i) + REG_SQRTS;
+	err = ufshcd_mcq_poll_register(reg, SQ_STS, 0, MCQ_POLL_MS);
+	if (err)
+		dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n",
+			__func__, i, err);
+	return err;
+}
+
+/**
+ * ufshcd_mcq_sq_cleanup - Clean up Submission Queue resources
+ * associated with the pending command.
+ * @hba - per adapter instance.
+ * @task_tag - The command's task tag.
+ * @result - Result of the Clean up operation.
+ *
+ * Returns 0 and result on completion. Returns error code if
+ * the operation fails.
+ */
+int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag, int *result)
+{
+	struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
+	struct scsi_cmnd *cmd = lrbp->cmd;
+	struct ufs_hw_queue *hwq;
+	void __iomem *reg, *opr_sqd_base;
+	u32 nexus, i;
+	int err;
+
+	if (task_tag != hba->nutrs - UFSHCD_NUM_RESERVED) {
+		if (!cmd)
+			return FAILED;
+		hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
+	} else {
+		hwq = hba->dev_cmd_queue;
+	}
+
+	i = hwq->id;
+
+	spin_lock(&hwq->sq_lock);
+
+	/* stop the SQ fetching before working on it */
+	err = ufshcd_mcq_sq_stop(hba, hwq);
+	if (err)
+		goto unlock;
+
+	/* SQCTI = EXT_IID, IID, LUN, Task Tag */
+	nexus = lrbp->lun << 8 | task_tag;
+	opr_sqd_base = mcq_opr_base(hba, OPR_SQD, i);
+	writel(nexus, opr_sqd_base + REG_SQCTI);
+
+	/* SQRTCy.ICU = 1 */
+	writel(SQ_ICU, opr_sqd_base + REG_SQRTC);
+
+	/* Poll SQRTSy.CUS = 1. Return result from SQRTSy.RTC */
+	reg = opr_sqd_base + REG_SQRTS;
+	err = ufshcd_mcq_poll_register(reg, SQ_CUS, SQ_CUS, MCQ_POLL_MS);
+	if (err)
+		dev_err(hba->dev, "%s: failed. hwq=%d, lun=0x%x, tag=%d\n",
+			__func__, i, lrbp->lun, task_tag);
+
+	*result = FIELD_GET(SQ_ICU_ERR_CODE_MASK, readl(reg));
+
+	if (ufshcd_mcq_sq_start(hba, hwq))
+		err = FAILED;
+
+unlock:
+	spin_unlock(&hwq->sq_lock);
+	return err;
+}
+
+static u64 ufshcd_mcq_get_cmd_desc_addr(struct ufs_hba *hba,
+					int task_tag)
+{
+	struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
+	__le32 hi = lrbp->utr_descriptor_ptr->command_desc_base_addr_hi;
+	__le32 lo = lrbp->utr_descriptor_ptr->command_desc_base_addr_lo;
+
+	return le64_to_cpu((__le64)hi << 32 | lo);
+}
+
+/**
+ * ufshcd_mcq_nullify_cmd - Nullify utrd. Host controller does not fetch
+ * transfer with Command Type = 0xF. post the Completion Queue with OCS=ABORTED.
+ * @hba - per adapter instance.
+ * @hwq - Hardware Queue of the nullified utrd.
+ */
+static void ufshcd_mcq_nullify_cmd(struct ufs_hba *hba, struct ufs_hw_queue *hwq)
+{
+	struct utp_transfer_req_desc *utrd;
+	u32 dword_0;
+
+	utrd = (struct utp_transfer_req_desc *)(hwq->sqe_base_addr +
+			hwq->id * sizeof(struct utp_transfer_req_desc));
+	dword_0 = le32_to_cpu(utrd->header.dword_0);
+	dword_0 &= ~UPIU_COMMAND_TYPE_MASK;
+	dword_0 |= FIELD_PREP(UPIU_COMMAND_TYPE_MASK, 0xF);
+	utrd->header.dword_0 = cpu_to_le32(dword_0);
+}
+
+/**
+ * ufshcd_mcq_sqe_search - Search for the cmd in the Submission Queue (SQ)
+ * @hba - per adapter instance.
+ * @hwq - Hardware Queue to be searched.
+ * @task_tag - The command's task tag.
+ *
+ * Returns true if the SQE containing the command is present in the SQ
+ * (not fetched by the controller); returns false if the SQE is not in the SQ.
+ */
+static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba,
+				struct ufs_hw_queue *hwq, int task_tag)
+{
+	struct utp_transfer_req_desc *utrd;
+	u32 mask = hwq->max_entries - 1;
+	bool ret = false;
+	u64 addr, match;
+	u32 sq_head_slot;
+
+	spin_lock(&hwq->sq_lock);
+
+	ufshcd_mcq_sq_stop(hba, hwq);
+	sq_head_slot = ufshcd_mcq_get_sq_head_slot(hwq);
+	if (sq_head_slot == hwq->sq_tail_slot)
+		goto out;
+
+	addr = ufshcd_mcq_get_cmd_desc_addr(hba, task_tag);
+	while (sq_head_slot != hwq->sq_tail_slot) {
+		utrd = (struct utp_transfer_req_desc *)(hwq->sqe_base_addr +
+				sq_head_slot * sizeof(struct utp_transfer_req_desc));
+		match = le64_to_cpu((__le64)utrd->command_desc_base_addr_hi << 32 |
+				utrd->command_desc_base_addr_lo) & CQE_UCD_BA;
+		if (addr == match) {
+			ret = true;
+			goto out;
+		}
+		sq_head_slot = (sq_head_slot + 1) & mask;
+	}
+
+out:
+	ufshcd_mcq_sq_start(hba, hwq);
+	spin_unlock(&hwq->sq_lock);
+	return ret;
+}
diff --git a/drivers/ufs/core/ufshcd-priv.h b/drivers/ufs/core/ufshcd-priv.h
index 529f850..1a40cf7 100644
--- a/drivers/ufs/core/ufshcd-priv.h
+++ b/drivers/ufs/core/ufshcd-priv.h
@@ -78,6 +78,8 @@ struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba,
 unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba,
 				       struct ufs_hw_queue *hwq);
 
+int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag, int *result);
+
 #define UFSHCD_MCQ_IO_QUEUE_OFFSET	1
 #define SD_ASCII_STD true
 #define SD_RAW false
@@ -403,4 +405,12 @@ static inline struct cq_entry *ufshcd_mcq_cur_cqe(struct ufs_hw_queue *q)
 
 	return cqe + q->cq_head_slot;
 }
+
+static inline u32 ufshcd_mcq_get_sq_head_slot(struct ufs_hw_queue *q)
+{
+	u32 val = readl(q->mcq_sq_head);
+
+	return val / sizeof(struct utp_transfer_req_desc);
+}
+
 #endif /* _UFSHCD_PRIV_H_ */
diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c
index 35a3bd9..808387c 100644
--- a/drivers/ufs/core/ufshcd.c
+++ b/drivers/ufs/core/ufshcd.c
@@ -56,7 +56,6 @@
 #define NOP_OUT_RETRIES    10
 /* Timeout after 50 msecs if NOP OUT hangs without response */
 #define NOP_OUT_TIMEOUT    50 /* msecs */
-
 /* Query request retries */
 #define QUERY_REQ_RETRIES 3
 /* Query request timeout */
@@ -173,7 +172,6 @@ EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
 enum {
 	UFSHCD_MAX_CHANNEL	= 0,
 	UFSHCD_MAX_ID		= 1,
-	UFSHCD_NUM_RESERVED	= 1,
 	UFSHCD_CMD_PER_LUN	= 32 - UFSHCD_NUM_RESERVED,
 	UFSHCD_CAN_QUEUE	= 32 - UFSHCD_NUM_RESERVED,
 };
diff --git a/include/ufs/ufshci.h b/include/ufs/ufshci.h
index 11424bb..ceb8664 100644
--- a/include/ufs/ufshci.h
+++ b/include/ufs/ufshci.h
@@ -99,6 +99,9 @@ enum {
 enum {
 	REG_SQHP		= 0x0,
 	REG_SQTP		= 0x4,
+	REG_SQRTC		= 0x8,
+	REG_SQCTI		= 0xC,
+	REG_SQRTS		= 0x10,
 };
 
 enum {
@@ -111,12 +114,26 @@ enum {
 	REG_CQIE		= 0x4,
 };
 
+enum {
+	SQ_START		= 0x0,
+	SQ_STOP			= 0x1,
+	SQ_ICU			= 0x2,
+};
+
+enum {
+	SQ_STS			= 0x1,
+	SQ_CUS			= 0x2,
+};
+
+#define SQ_ICU_ERR_CODE_MASK		GENMASK(7, 4)
+#define UPIU_COMMAND_TYPE_MASK		GENMASK(31, 28)
 #define UFS_MASK(mask, offset)		((mask) << (offset))
 
 /* UFS Version 08h */
 #define MINOR_VERSION_NUM_MASK		UFS_MASK(0xFFFF, 0)
 #define MAJOR_VERSION_NUM_MASK		UFS_MASK(0xFFFF, 16)
 
+#define UFSHCD_NUM_RESERVED	1
 /*
  * Controller UFSHCI version
  * - 2.x and newer use the following scheme:
-- 
2.7.4

[PATCH v2 2/5] ufs: mcq: Add support for clean up mcq resources

[Bao D. Nguyen]

Update ufshcd_clear_cmds() to clean up the mcq resources similar
to the function ufshcd_utrl_clear() does for sdb mode.

Update ufshcd_try_to_abort_task() to support mcq mode so that
this function can be invoked in either mcq or sdb mode.

Signed-off-by: Bao D. Nguyen <quic_nguyenb@quicinc.com>
---
 drivers/ufs/core/ufshcd.c | 45 +++++++++++++++++++++++++++++++++++++++------
 1 file changed, 39 insertions(+), 6 deletions(-)

diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c
index 808387c..ffccb91 100644
--- a/drivers/ufs/core/ufshcd.c
+++ b/drivers/ufs/core/ufshcd.c
@@ -3007,10 +3007,28 @@ static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
  * @mask and wait until the controller confirms that these requests have been
  * cleared.
  */
-static int ufshcd_clear_cmds(struct ufs_hba *hba, u32 mask)
+static int ufshcd_clear_cmds(struct ufs_hba *hba, unsigned long mask)
 {
 	unsigned long flags;
+	int err, tag, result = FAILED;
 
+	if (is_mcq_enabled(hba)) {
+		/*
+		 * MCQ mode. Clean up the MCQ resources similar to
+		 * what the ufshcd_utrl_clear() does for SDB mode.
+		 */
+		for_each_set_bit(tag, &mask, hba->nutrs) {
+			err = ufshcd_mcq_sq_cleanup(hba, tag, &result);
+			if (err || result) {
+				dev_err(hba->dev, "%s: failed tag=%d. err=%d, result=%d\n",
+					__func__, tag, err, result);
+				return FAILED;
+			}
+		}
+		return 0;
+	}
+
+	/* Single Doorbell Mode */
 	/* clear outstanding transaction before retry */
 	spin_lock_irqsave(hba->host->host_lock, flags);
 	ufshcd_utrl_clear(hba, mask);
@@ -3110,7 +3128,7 @@ static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
 		err = -ETIMEDOUT;
 		dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
 			__func__, lrbp->task_tag);
-		if (ufshcd_clear_cmds(hba, 1U << lrbp->task_tag) == 0) {
+		if (ufshcd_clear_cmds(hba, 1UL << lrbp->task_tag) == 0) {
 			/* successfully cleared the command, retry if needed */
 			err = -EAGAIN;
 			/*
@@ -7379,6 +7397,20 @@ static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
 			 */
 			dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
 				__func__, tag);
+			if (is_mcq_enabled(hba)) {
+				/* MCQ mode */
+				if (lrbp->cmd) {
+					/* sleep for max. 200us to stabilize */
+					usleep_range(100, 200);
+					continue;
+				}
+				/* command completed already */
+				dev_err(hba->dev, "%s: cmd at tag=%d is cleared.\n",
+					__func__, tag);
+				goto out;
+			}
+
+			/* Single Doorbell Mode */
 			reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
 			if (reg & (1 << tag)) {
 				/* sleep for max. 200us to stabilize */
@@ -7415,7 +7447,7 @@ static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
 		goto out;
 	}
 
-	err = ufshcd_clear_cmds(hba, 1U << tag);
+	err = ufshcd_clear_cmds(hba, 1UL << tag);
 	if (err)
 		dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
 			__func__, tag, err);
@@ -7445,8 +7477,8 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
 
 	ufshcd_hold(hba, false);
 	reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
-	/* If command is already aborted/completed, return FAILED. */
-	if (!(test_bit(tag, &hba->outstanding_reqs))) {
+	if (!is_mcq_enabled(hba) && !(test_bit(tag, &hba->outstanding_reqs))) {
+		/* If command is already aborted/completed, return FAILED. */
 		dev_err(hba->dev,
 			"%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
 			__func__, tag, hba->outstanding_reqs, reg);
@@ -7475,7 +7507,8 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
 	}
 	hba->req_abort_count++;
 
-	if (!(reg & (1 << tag))) {
+	if (!is_mcq_enabled(hba) && !(reg & (1 << tag))) {
+		/* only execute this code in single doorbell mode */
 		dev_err(hba->dev,
 		"%s: cmd was completed, but without a notifying intr, tag = %d",
 		__func__, tag);
-- 
2.7.4

[PATCH v2 3/5] ufs: mcq: Added ufshcd_mcq_abort()

[Bao D. Nguyen]

Add ufshcd_mcq_abort() to support ufs abort in mcq mode.

Signed-off-by: Bao D. Nguyen <quic_nguyenb@quicinc.com>
---
 drivers/ufs/core/ufs-mcq.c     | 62 ++++++++++++++++++++++++++++++++++++++++++
 drivers/ufs/core/ufshcd-priv.h |  5 +++-
 drivers/ufs/core/ufshcd.c      | 11 ++++++--
 3 files changed, 74 insertions(+), 4 deletions(-)

diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c
index a463674..e3031cd 100644
--- a/drivers/ufs/core/ufs-mcq.c
+++ b/drivers/ufs/core/ufs-mcq.c
@@ -619,3 +619,65 @@ static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba,
 	spin_unlock(&hwq->sq_lock);
 	return ret;
 }
+
+/**
+ * ufshcd_mcq_abort - Abort the command in MCQ.
+ * @cmd - The command to be aborted.
+ *
+ * Returns SUCCESS or FAILED error codes
+ */
+int ufshcd_mcq_abort(struct scsi_cmnd *cmd)
+{
+	struct Scsi_Host *host = cmd->device->host;
+	struct ufs_hba *hba = shost_priv(host);
+	int tag = scsi_cmd_to_rq(cmd)->tag;
+	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+	struct ufs_hw_queue *hwq;
+	int err = FAILED;
+
+	if (!lrbp->cmd) {
+		dev_err(hba->dev,
+			"%s: skip abort. cmd at tag %d already completed.\n",
+			__func__, tag);
+		goto out;
+	}
+
+	/* Skip task abort in case previous aborts failed and report failure */
+	if (lrbp->req_abort_skip) {
+		dev_err(hba->dev, "%s: skip abort. tag %d failed earlier\n",
+			__func__, tag);
+		goto out;
+	}
+
+	hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
+
+	if (ufshcd_mcq_sqe_search(hba, hwq, tag)) {
+		/*
+		 * Failure. The command should not be "stuck" in SQ for
+		 * a long time which resulted in command being aborted.
+		 */
+		dev_err(hba->dev, "%s: cmd found in sq. hwq=%d, tag=%d\n",
+				__func__, hwq->id, tag);
+		/* Set the Command Type to 0xF per the spec */
+		ufshcd_mcq_nullify_cmd(hba, hwq);
+		goto out;
+	}
+
+	/*
+	 * The command is not in the Submission Queue, and it is not
+	 * in the Completion Queue either. Query the device to see if
+	 * the command is being processed in the device.
+	 */
+	if (ufshcd_try_to_abort_task(hba, tag)) {
+		dev_err(hba->dev, "%s: device abort failed %d\n", __func__, err);
+		lrbp->req_abort_skip = true;
+		goto out;
+	}
+
+	err = SUCCESS;
+	if (lrbp->cmd)
+		ufshcd_release_scsi_cmd(hba, lrbp);
+
+out:
+	return err;
+}
diff --git a/drivers/ufs/core/ufshcd-priv.h b/drivers/ufs/core/ufshcd-priv.h
index 1a40cf7..ef66151 100644
--- a/drivers/ufs/core/ufshcd-priv.h
+++ b/drivers/ufs/core/ufshcd-priv.h
@@ -79,7 +79,10 @@ unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba,
 				       struct ufs_hw_queue *hwq);
 
 int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag, int *result);
-
+int ufshcd_mcq_abort(struct scsi_cmnd *cmd);
+int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
+void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
+				struct ufshcd_lrb *lrbp);
 #define UFSHCD_MCQ_IO_QUEUE_OFFSET	1
 #define SD_ASCII_STD true
 #define SD_RAW false
diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c
index ffccb91..84efb52 100644
--- a/drivers/ufs/core/ufshcd.c
+++ b/drivers/ufs/core/ufshcd.c
@@ -299,7 +299,6 @@ static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on);
 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on);
 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
 					 struct ufs_vreg *vreg);
-static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
 static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba,
 						 bool enable);
 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba);
@@ -5438,7 +5437,7 @@ static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
 }
 
 /* Release the resources allocated for processing a SCSI command. */
-static void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
+void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
 				    struct ufshcd_lrb *lrbp)
 {
 	struct scsi_cmnd *cmd = lrbp->cmd;
@@ -7374,7 +7373,7 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
  *
  * Returns zero on success, non-zero on failure
  */
-static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
+int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
 {
 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
 	int err = 0;
@@ -7534,6 +7533,12 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
 		goto release;
 	}
 
+	if (is_mcq_enabled(hba)) {
+		/* MCQ mode. Branch off to handle abort for mcq mode */
+		err = ufshcd_mcq_abort(cmd);
+		goto release;
+	}
+
 	/* Skip task abort in case previous aborts failed and report failure */
 	if (lrbp->req_abort_skip) {
 		dev_err(hba->dev, "%s: skipping abort\n", __func__);
-- 
2.7.4

[PATCH v2 4/5] ufs: mcq: Use ufshcd_mcq_poll_cqe_lock() in mcq mode

[Bao D. Nguyen]

In preparation for adding mcq error handler support, update the mcq
code to use the ufshcd_mcq_poll_cqe_lock() in interrupt context
instead of using ufshcd_mcq_poll_cqe_nolock(). This is to keep
synchronization between mcq interrupt and error handler contexts
because both need to access the mcq hardware in separate contexts.

Signed-off-by: Bao D. Nguyen <quic_nguyenb@quicinc.com>
---
 drivers/ufs/core/ufs-mcq.c     | 6 +++---
 drivers/ufs/core/ufshcd-priv.h | 2 --
 drivers/ufs/core/ufshcd.c      | 2 +-
 drivers/ufs/host/ufs-qcom.c    | 2 +-
 include/ufs/ufshcd.h           | 2 +-
 5 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c
index e3031cd..ede4343 100644
--- a/drivers/ufs/core/ufs-mcq.c
+++ b/drivers/ufs/core/ufs-mcq.c
@@ -283,8 +283,8 @@ static void ufshcd_mcq_process_cqe(struct ufs_hba *hba,
 	ufshcd_compl_one_cqe(hba, tag, cqe);
 }
 
-unsigned long ufshcd_mcq_poll_cqe_nolock(struct ufs_hba *hba,
-					 struct ufs_hw_queue *hwq)
+static unsigned long ufshcd_mcq_poll_cqe_nolock(struct ufs_hba *hba,
+						struct ufs_hw_queue *hwq)
 {
 	unsigned long completed_reqs = 0;
 
@@ -300,7 +300,6 @@ unsigned long ufshcd_mcq_poll_cqe_nolock(struct ufs_hba *hba,
 
 	return completed_reqs;
 }
-EXPORT_SYMBOL_GPL(ufshcd_mcq_poll_cqe_nolock);
 
 unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba,
 				       struct ufs_hw_queue *hwq)
@@ -313,6 +312,7 @@ unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba,
 
 	return completed_reqs;
 }
+EXPORT_SYMBOL_GPL(ufshcd_mcq_poll_cqe_lock);
 
 void ufshcd_mcq_make_queues_operational(struct ufs_hba *hba)
 {
diff --git a/drivers/ufs/core/ufshcd-priv.h b/drivers/ufs/core/ufshcd-priv.h
index ef66151..70b7ffc 100644
--- a/drivers/ufs/core/ufshcd-priv.h
+++ b/drivers/ufs/core/ufshcd-priv.h
@@ -71,8 +71,6 @@ void ufshcd_mcq_config_mac(struct ufs_hba *hba, u32 max_active_cmds);
 void ufshcd_mcq_select_mcq_mode(struct ufs_hba *hba);
 u32 ufshcd_mcq_read_cqis(struct ufs_hba *hba, int i);
 void ufshcd_mcq_write_cqis(struct ufs_hba *hba, u32 val, int i);
-unsigned long ufshcd_mcq_poll_cqe_nolock(struct ufs_hba *hba,
-					 struct ufs_hw_queue *hwq);
 struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba,
 					   struct request *req);
 unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba,
diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c
index 84efb52..fef1907 100644
--- a/drivers/ufs/core/ufshcd.c
+++ b/drivers/ufs/core/ufshcd.c
@@ -6792,7 +6792,7 @@ static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba)
 			ufshcd_mcq_write_cqis(hba, events, i);
 
 		if (events & UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS)
-			ufshcd_mcq_poll_cqe_nolock(hba, hwq);
+			ufshcd_mcq_poll_cqe_lock(hba, hwq);
 	}
 
 	return IRQ_HANDLED;
diff --git a/drivers/ufs/host/ufs-qcom.c b/drivers/ufs/host/ufs-qcom.c
index 6568fdf..34d3814 100644
--- a/drivers/ufs/host/ufs-qcom.c
+++ b/drivers/ufs/host/ufs-qcom.c
@@ -1558,7 +1558,7 @@ static irqreturn_t ufs_qcom_mcq_esi_handler(int irq, void *__hba)
 	struct ufs_hw_queue *hwq = &hba->uhq[id];
 
 	ufshcd_mcq_write_cqis(hba, 0x1, id);
-	ufshcd_mcq_poll_cqe_nolock(hba, hwq);
+	ufshcd_mcq_poll_cqe_lock(hba, hwq);
 
 	return IRQ_HANDLED;
 }
diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h
index 25aab8e..3fca558 100644
--- a/include/ufs/ufshcd.h
+++ b/include/ufs/ufshcd.h
@@ -1246,7 +1246,7 @@ void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val);
 void ufshcd_hba_stop(struct ufs_hba *hba);
 void ufshcd_schedule_eh_work(struct ufs_hba *hba);
 void ufshcd_mcq_write_cqis(struct ufs_hba *hba, u32 val, int i);
-unsigned long ufshcd_mcq_poll_cqe_nolock(struct ufs_hba *hba,
+unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba,
 					 struct ufs_hw_queue *hwq);
 void ufshcd_mcq_enable_esi(struct ufs_hba *hba);
 void ufshcd_mcq_config_esi(struct ufs_hba *hba, struct msi_msg *msg);
-- 
2.7.4

[PATCH v2 5/5] ufs: core: Add error handling for MCQ mode

[Bao D. Nguyen]

Add support for error handling for MCQ mode.

Signed-off-by: Bao D. Nguyen <quic_nguyenb@quicinc.com>
---
 drivers/ufs/core/ufshcd.c | 80 ++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 69 insertions(+), 11 deletions(-)

diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c
index fef1907..e947f7f 100644
--- a/drivers/ufs/core/ufshcd.c
+++ b/drivers/ufs/core/ufshcd.c
@@ -3127,6 +3127,12 @@ static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
 		err = -ETIMEDOUT;
 		dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
 			__func__, lrbp->task_tag);
+
+		/* MCQ mode */
+		if (is_mcq_enabled(hba))
+			return ufshcd_clear_cmds(hba, 1UL << lrbp->task_tag);
+
+		/* SDB mode */
 		if (ufshcd_clear_cmds(hba, 1UL << lrbp->task_tag) == 0) {
 			/* successfully cleared the command, retry if needed */
 			err = -EAGAIN;
@@ -5562,6 +5568,10 @@ static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num)
  */
 static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
 {
+	struct ufshcd_lrb *lrbp;
+	u32 hwq_num, utag;
+	int tag;
+
 	/* Resetting interrupt aggregation counters first and reading the
 	 * DOOR_BELL afterward allows us to handle all the completed requests.
 	 * In order to prevent other interrupts starvation the DB is read once
@@ -5580,7 +5590,22 @@ static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
 	 * Ignore the ufshcd_poll() return value and return IRQ_HANDLED since we
 	 * do not want polling to trigger spurious interrupt complaints.
 	 */
-	ufshcd_poll(hba->host, UFSHCD_POLL_FROM_INTERRUPT_CONTEXT);
+	if (!is_mcq_enabled(hba)) {
+		ufshcd_poll(hba->host, UFSHCD_POLL_FROM_INTERRUPT_CONTEXT);
+		goto out;
+	}
+
+	/* MCQ mode */
+	for (tag = 0; tag < hba->nutrs; tag++) {
+		lrbp = &hba->lrb[tag];
+		if (lrbp->cmd) {
+			utag = blk_mq_unique_tag(scsi_cmd_to_rq(lrbp->cmd));
+			hwq_num = blk_mq_unique_tag_to_hwq(utag);
+			ufshcd_poll(hba->host, hwq_num);
+		}
+	}
+
+out:
 
 	return IRQ_HANDLED;
 }
@@ -6359,18 +6384,36 @@ static bool ufshcd_abort_all(struct ufs_hba *hba)
 	bool needs_reset = false;
 	int tag, ret;
 
-	/* Clear pending transfer requests */
-	for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
-		ret = ufshcd_try_to_abort_task(hba, tag);
-		dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
-			hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
-			ret ? "failed" : "succeeded");
-		if (ret) {
-			needs_reset = true;
-			goto out;
+	if (is_mcq_enabled(hba)) {
+		struct ufshcd_lrb *lrbp;
+		int tag;
+
+		for (tag = 0; tag < hba->nutrs; tag++) {
+			lrbp = &hba->lrb[tag];
+			if (lrbp->cmd) {
+				ret = ufshcd_try_to_abort_task(hba, tag);
+				dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
+					hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
+					ret ? "failed" : "succeeded");
+			}
+			if (ret) {
+				needs_reset = true;
+				goto out;
+			}
+		}
+	} else {
+		/* Clear pending transfer requests */
+		for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
+			ret = ufshcd_try_to_abort_task(hba, tag);
+			dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
+				hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
+				ret ? "failed" : "succeeded");
+			if (ret) {
+				needs_reset = true;
+				goto out;
+			}
 		}
 	}
-
 	/* Clear pending task management requests */
 	for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
 		if (ufshcd_clear_tm_cmd(hba, tag)) {
@@ -7302,6 +7345,8 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
 	unsigned long flags, pending_reqs = 0, not_cleared = 0;
 	struct Scsi_Host *host;
 	struct ufs_hba *hba;
+	struct ufs_hw_queue *hwq;
+	struct ufshcd_lrb *lrbp;
 	u32 pos;
 	int err;
 	u8 resp = 0xF, lun;
@@ -7317,6 +7362,19 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
 		goto out;
 	}
 
+	if (is_mcq_enabled(hba)) {
+		for (pos = 0; pos < hba->nutrs; pos++) {
+			lrbp = &hba->lrb[pos];
+			if (lrbp->cmd && lrbp->lun == lun) {
+				ufshcd_clear_cmds(hba, 1UL << pos);
+				hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(lrbp->cmd));
+				ufshcd_mcq_poll_cqe_lock(hba, hwq);
+			}
+		}
+		err = 0;
+		goto out;
+	}
+
 	/* clear the commands that were pending for corresponding LUN */
 	spin_lock_irqsave(&hba->outstanding_lock, flags);
 	for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs)
-- 
2.7.4

Re: [PATCH v2 1/5] ufs: mcq: Add supporting functions for mcq abort

[Bart Van Assche]

On 4/17/23 14:05, Bao D. Nguyen wrote:
> +/* Max mcq register polling time in milisecond unit */

A nit: please change "millisecond unit" into "milliseconds".

> +static int ufshcd_mcq_poll_register(void __iomem *reg, u32 mask,
> +				u32 val, unsigned long timeout_ms)
> +{
> +	unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
> +	int err = 0;
> +
> +	/* ignore bits that we don't intend to wait on */
> +	val = val & mask;
> +
> +	while ((readl(reg) & mask) != val) {

& has a higher precedence than != so one pair of parentheses can be left 
out.

> +		udelay(20);
> +		if (time_after(jiffies, timeout)) {

Please use time_is_before_jiffies() instead of time_after(jiffies, ...).

> +			err = -ETIMEDOUT;
> +			break;
> +		}
> +	}
> +
> +	return err;
> +}

Please remove the variable 'err' and return the return value directly.

> +
> +static int ufshcd_mcq_sq_stop(struct ufs_hba *hba, struct ufs_hw_queue *hwq)
> +{
> +	void __iomem *reg;
> +	u32 i = hwq->id;

Please use another variable name than 'i' for a hardware queue ID ('id'?).

> +	u32 i = hwq->id;

Same comment here.

> +/**
> + * ufshcd_mcq_sq_cleanup - Clean up Submission Queue resources

A nit: please use lower case text for "submission queue" and also in the 
comments below ("Clean up" -> "clean up").

> +	spin_lock(&hwq->sq_lock);
> +
> +	/* stop the SQ fetching before working on it */
> +	err = ufshcd_mcq_sq_stop(hba, hwq);
> +	if (err)
> +		goto unlock;

No spin locks around delay loops please. Is there anything that prevents 
to change sq_lock from a spin lock into a mutex?

> +static u64 ufshcd_mcq_get_cmd_desc_addr(struct ufs_hba *hba,
> +					int task_tag)
> +{
> +	struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
> +	__le32 hi = lrbp->utr_descriptor_ptr->command_desc_base_addr_hi;
> +	__le32 lo = lrbp->utr_descriptor_ptr->command_desc_base_addr_lo;
> +
> +	return le64_to_cpu((__le64)hi << 32 | lo);
> +}

Please add a new patch at the head of this series that modifies struct 
utp_transfer_req_desc such that command_desc_base_addr_lo and 
command_desc_base_addr_hi are combined into a single __le64 variable.

> +/**
> + * ufshcd_mcq_nullify_cmd - Nullify utrd. Host controller does not fetch
> + * transfer with Command Type = 0xF. post the Completion Queue with OCS=ABORTED.
> + * @hba - per adapter instance.
> + * @hwq - Hardware Queue of the nullified utrd.
> + */
> +static void ufshcd_mcq_nullify_cmd(struct ufs_hba *hba, struct ufs_hw_queue *hwq)
> +{
> +	struct utp_transfer_req_desc *utrd;
> +	u32 dword_0;
> +
> +	utrd = (struct utp_transfer_req_desc *)(hwq->sqe_base_addr +
> +			hwq->id * sizeof(struct utp_transfer_req_desc));

Please double check this function. It has "cmd" in the function name but 
none of the arguments passed to this function allows to uniquely 
identify a command. Is an argument perhaps missing from this function?

Additionally, hwq->sqe_base_addr points to an array of SQE entries. I do 
not understand why hwq->id * sizeof(struct utp_transfer_req_desc) is 
added to that base address. Please clarify.

> +		utrd = (struct utp_transfer_req_desc *)(hwq->sqe_base_addr +
> +				sq_head_slot * sizeof(struct utp_transfer_req_desc));

hwq->sqe_base_addr already has type struct utp_transfer_req_desc * so 
the " * sizeof(struct utp_transfer_req_desc)" part looks wrong to me.

> diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c
> index 35a3bd9..808387c 100644
> --- a/drivers/ufs/core/ufshcd.c
> +++ b/drivers/ufs/core/ufshcd.c
> @@ -56,7 +56,6 @@
>   #define NOP_OUT_RETRIES    10
>   /* Timeout after 50 msecs if NOP OUT hangs without response */
>   #define NOP_OUT_TIMEOUT    50 /* msecs */
> -
>   /* Query request retries */
>   #define QUERY_REQ_RETRIES 3
>   /* Query request timeout */

Is the above change really necessary?

> @@ -173,7 +172,6 @@ EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
>   enum {
>   	UFSHCD_MAX_CHANNEL	= 0,
>   	UFSHCD_MAX_ID		= 1,
> -	UFSHCD_NUM_RESERVED	= 1,
>   	UFSHCD_CMD_PER_LUN	= 32 - UFSHCD_NUM_RESERVED,
>   	UFSHCD_CAN_QUEUE	= 32 - UFSHCD_NUM_RESERVED,
>   };

Same question here - is this change really necessary?

Thanks,

Bart.

Re: [PATCH v2 2/5] ufs: mcq: Add support for clean up mcq resources

[Bart Van Assche]

On 4/17/23 14:05, Bao D. Nguyen wrote:
> @@ -3110,7 +3128,7 @@ static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
>   		err = -ETIMEDOUT;
>   		dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
>   			__func__, lrbp->task_tag);
> -		if (ufshcd_clear_cmds(hba, 1U << lrbp->task_tag) == 0) {
> +		if (ufshcd_clear_cmds(hba, 1UL << lrbp->task_tag) == 0) {
>   			/* successfully cleared the command, retry if needed */
>   			err = -EAGAIN;
>   			/*

Is this change necessary?

> @@ -7379,6 +7397,20 @@ static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
>   			 */
>   			dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
>   				__func__, tag);
> +			if (is_mcq_enabled(hba)) {
> +				/* MCQ mode */
> +				if (lrbp->cmd) {
> +					/* sleep for max. 200us to stabilize */

What is being stabilized here? Please make this comment more clear.

> +					usleep_range(100, 200);
> +					continue;
> +				}
> +				/* command completed already */
> +				dev_err(hba->dev, "%s: cmd at tag=%d is cleared.\n",
> +					__func__, tag);
> +				goto out;
> +			}

Please do not use lrbp->cmd to check whether or not a command has 
completed. See also my patch "scsi: ufs: Fix handling of lrbp->cmd".

> @@ -7415,7 +7447,7 @@ static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
>   		goto out;
>   	}
>   
> -	err = ufshcd_clear_cmds(hba, 1U << tag);
> +	err = ufshcd_clear_cmds(hba, 1UL << tag);
>   	if (err)
>   		dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
>   			__func__, tag, err);

Is this change necessary?

> -	if (!(test_bit(tag, &hba->outstanding_reqs))) {
> +	if (!is_mcq_enabled(hba) && !(test_bit(tag, &hba->outstanding_reqs))) {

Please leave out one pair of superfluous parentheses from the above 
expression.

Thanks,

Bart.

Re: [PATCH v2 3/5] ufs: mcq: Added ufshcd_mcq_abort()

[Bart Van Assche]

On 4/17/23 14:05, Bao D. Nguyen wrote:
> +	if (!lrbp->cmd) {
> +		dev_err(hba->dev,
> +			"%s: skip abort. cmd at tag %d already completed.\n",
> +			__func__, tag);
> +		goto out;
> +	}

Please do not use lrbp->cmd to check whether or not a command has completed.

> +	if (ufshcd_mcq_sqe_search(hba, hwq, tag)) {
> +		/*
> +		 * Failure. The command should not be "stuck" in SQ for
> +		 * a long time which resulted in command being aborted.
> +		 */
> +		dev_err(hba->dev, "%s: cmd found in sq. hwq=%d, tag=%d\n",
> +				__func__, hwq->id, tag);
> +		/* Set the Command Type to 0xF per the spec */
> +		ufshcd_mcq_nullify_cmd(hba, hwq);

The above looks wrong to me. How can ufshcd_mcq_nullify_cmd() identify a 
command if the 'tag' argument is not passed to that function?

> +	/*
> +	 * The command is not in the Submission Queue, and it is not
> +	 * in the Completion Queue either. Query the device to see if
> +	 * the command is being processed in the device.
> +	 */

Please only use capitals if these are required.

> +	if (lrbp->cmd)
> +		ufshcd_release_scsi_cmd(hba, lrbp);

Same comment here - do not use lrbp->cmd to check for completion.

Thanks,

Bart.

Re: [PATCH v2 5/5] ufs: core: Add error handling for MCQ mode

[Bart Van Assche]

On 4/17/23 14:05, Bao D. Nguyen wrote:
> +		/* MCQ mode */
> +		if (is_mcq_enabled(hba))
> +			return ufshcd_clear_cmds(hba, 1UL << lrbp->task_tag);

The above code will trigger an overflow if lrbp->task_tag >= 8 * 
sizeof(unsigned long). That's not acceptable.

>   static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
>   {
> +	struct ufshcd_lrb *lrbp;
> +	u32 hwq_num, utag;
> +	int tag;
> +
>   	/* Resetting interrupt aggregation counters first and reading the
>   	 * DOOR_BELL afterward allows us to handle all the completed requests.
>   	 * In order to prevent other interrupts starvation the DB is read once
> @@ -5580,7 +5590,22 @@ static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
>   	 * Ignore the ufshcd_poll() return value and return IRQ_HANDLED since we
>   	 * do not want polling to trigger spurious interrupt complaints.
>   	 */
> -	ufshcd_poll(hba->host, UFSHCD_POLL_FROM_INTERRUPT_CONTEXT);
> +	if (!is_mcq_enabled(hba)) {
> +		ufshcd_poll(hba->host, UFSHCD_POLL_FROM_INTERRUPT_CONTEXT);
> +		goto out;
> +	}
> +
> +	/* MCQ mode */
> +	for (tag = 0; tag < hba->nutrs; tag++) {
> +		lrbp = &hba->lrb[tag];
> +		if (lrbp->cmd) {
> +			utag = blk_mq_unique_tag(scsi_cmd_to_rq(lrbp->cmd));
> +			hwq_num = blk_mq_unique_tag_to_hwq(utag);
> +			ufshcd_poll(hba->host, hwq_num);
> +		}
> +	}

Is my understanding correct that ufshcd_transfer_req_compl() is only 
called from single doorbell code paths and hence that the above change 
is not necessary?


> +	if (is_mcq_enabled(hba)) {
> +		struct ufshcd_lrb *lrbp;
> +		int tag;
> +
> +		for (tag = 0; tag < hba->nutrs; tag++) {
> +			lrbp = &hba->lrb[tag];
> +			if (lrbp->cmd) {
> +				ret = ufshcd_try_to_abort_task(hba, tag);
> +				dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
> +					hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
> +					ret ? "failed" : "succeeded");
> +			}
> +			if (ret) {
> +				needs_reset = true;
> +				goto out;
> +			}
> +		}
> +	} else {
> +		/* Clear pending transfer requests */
> +		for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
> +			ret = ufshcd_try_to_abort_task(hba, tag);
> +			dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
> +				hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
> +				ret ? "failed" : "succeeded");
> +			if (ret) {
> +				needs_reset = true;
> +				goto out;
> +			}
>   		}
>   	}

Please introduce helper functions for the MCQ and SDB code paths such 
that the nesting level of the above code is reduced.

Thanks,

Bart.

Re: [PATCH v2 1/5] ufs: mcq: Add supporting functions for mcq abort

[Bao D. Nguyen]

Hi Bart,
Thank you so much for a detailed code review.

On 4/25/2023 5:04 PM, Bart Van Assche wrote:
> On 4/17/23 14:05, Bao D. Nguyen wrote:
>> +/* Max mcq register polling time in milisecond unit */
> 
> A nit: please change "millisecond unit" into "milliseconds".
Yes I will change.

> 
>> +static int ufshcd_mcq_poll_register(void __iomem *reg, u32 mask,
>> +                u32 val, unsigned long timeout_ms)
>> +{
>> +    unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
>> +    int err = 0;
>> +
>> +    /* ignore bits that we don't intend to wait on */
>> +    val = val & mask;
>> +
>> +    while ((readl(reg) & mask) != val) {
> 
> & has a higher precedence than != so one pair of parentheses can be left 
> out.
I think it is is actually the other way. & has lower precedence than !=.
Please correct me if I am wrong.

> 
>> +        udelay(20);
>> +        if (time_after(jiffies, timeout)) {
> 
> Please use time_is_before_jiffies() instead of time_after(jiffies, ...).
time_is_before_jiffies() seems to be defined as time_after(). Could you 
please explain the benefits to choose one over the other?

> 
>> +            err = -ETIMEDOUT;
>> +            break;
>> +        }
>> +    }
>> +
>> +    return err;
>> +}
> 
> Please remove the variable 'err' and return the return value directly.
Yes I will change.

>> +
>> +static int ufshcd_mcq_sq_stop(struct ufs_hba *hba, struct 
>> ufs_hw_queue *hwq)
>> +{
>> +    void __iomem *reg;
>> +    u32 i = hwq->id;
> 
> Please use another variable name than 'i' for a hardware queue ID ('id'?).
Yes I will change.

> 
>> +    u32 i = hwq->id;
> 
> Same comment here.
Yes I will change.

> 
>> +/**
>> + * ufshcd_mcq_sq_cleanup - Clean up Submission Queue resources
> 
> A nit: please use lower case text for "submission queue" and also in the 
> comments below ("Clean up" -> "clean up").
The UFS Host Controller specification uses upper case for the Submission 
Queue and Completion Queue, so I tried to follow the the spec language. 
I don't have a preference. I will make the change.

> 
>> +    spin_lock(&hwq->sq_lock);
>> +
>> +    /* stop the SQ fetching before working on it */
>> +    err = ufshcd_mcq_sq_stop(hba, hwq);
>> +    if (err)
>> +        goto unlock;
> 
> No spin locks around delay loops please. Is there anything that prevents 
> to change sq_lock from a spin lock into a mutex?
This function can be called from multiple non-interrupt contexts. I 
needed to prevent concurrent accesses to the sq hw, so yes mutex would 
work better. I will change.

> 
>> +static u64 ufshcd_mcq_get_cmd_desc_addr(struct ufs_hba *hba,
>> +                    int task_tag)
>> +{
>> +    struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
>> +    __le32 hi = lrbp->utr_descriptor_ptr->command_desc_base_addr_hi;
>> +    __le32 lo = lrbp->utr_descriptor_ptr->command_desc_base_addr_lo;
>> +
>> +    return le64_to_cpu((__le64)hi << 32 | lo);
>> +}
> 
> Please add a new patch at the head of this series that modifies struct 
> utp_transfer_req_desc such that command_desc_base_addr_lo and 
> command_desc_base_addr_hi are combined into a single __le64 variable.
Yes, I will add this as a separate patch.

> 
>> +/**
>> + * ufshcd_mcq_nullify_cmd - Nullify utrd. Host controller does not fetch
>> + * transfer with Command Type = 0xF. post the Completion Queue with 
>> OCS=ABORTED.
>> + * @hba - per adapter instance.
>> + * @hwq - Hardware Queue of the nullified utrd.
>> + */
>> +static void ufshcd_mcq_nullify_cmd(struct ufs_hba *hba, struct 
>> ufs_hw_queue *hwq)
>> +{
>> +    struct utp_transfer_req_desc *utrd;
>> +    u32 dword_0;
>> +
>> +    utrd = (struct utp_transfer_req_desc *)(hwq->sqe_base_addr +
>> +            hwq->id * sizeof(struct utp_transfer_req_desc));
> 
> Please double check this function. It has "cmd" in the function name but 
> none of the arguments passed to this function allows to uniquely 
> identify a command. Is an argument perhaps missing from this function?
Yes, I will make the correction to this function and rename it to 
ufshcd_mcq_nullify_sqe()

> 
> Additionally, hwq->sqe_base_addr points to an array of SQE entries. I do 
> not understand why hwq->id * sizeof(struct utp_transfer_req_desc) is 
> added to that base address. Please clarify. >
>> +        utrd = (struct utp_transfer_req_desc *)(hwq->sqe_base_addr +
>> +                sq_head_slot * sizeof(struct utp_transfer_req_desc));
> 
> hwq->sqe_base_addr already has type struct utp_transfer_req_desc * so 
> the " * sizeof(struct utp_transfer_req_desc)" part looks wrong to me.
Yes, I will correct this.

> 
>> diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c
>> index 35a3bd9..808387c 100644
>> --- a/drivers/ufs/core/ufshcd.c
>> +++ b/drivers/ufs/core/ufshcd.c
>> @@ -56,7 +56,6 @@
>>   #define NOP_OUT_RETRIES    10
>>   /* Timeout after 50 msecs if NOP OUT hangs without response */
>>   #define NOP_OUT_TIMEOUT    50 /* msecs */
>> -
>>   /* Query request retries */
>>   #define QUERY_REQ_RETRIES 3
>>   /* Query request timeout */
> 
> Is the above change really necessary?
The blank line was removed by mistake. I will put it back.

>> @@ -173,7 +172,6 @@ EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
>>   enum {
>>       UFSHCD_MAX_CHANNEL    = 0,
>>       UFSHCD_MAX_ID        = 1,
>> -    UFSHCD_NUM_RESERVED    = 1,
>>       UFSHCD_CMD_PER_LUN    = 32 - UFSHCD_NUM_RESERVED,
>>       UFSHCD_CAN_QUEUE    = 32 - UFSHCD_NUM_RESERVED,
>>   };
> 
> Same question here - is this change really necessary?
I am moving the definition of UFSHCD_NUM_RESERVED to 
include/ufs/ufshci.h file so that I can access it from /core/ufs-mcq.c


> Thanks,
> 
> Bart.

Re: [PATCH v2 2/5] ufs: mcq: Add support for clean up mcq resources

[Bao D. Nguyen]

On 4/25/2023 5:08 PM, Bart Van Assche wrote:
> On 4/17/23 14:05, Bao D. Nguyen wrote:
>> @@ -3110,7 +3128,7 @@ static int ufshcd_wait_for_dev_cmd(struct 
>> ufs_hba *hba,
>>           err = -ETIMEDOUT;
>>           dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
>>               __func__, lrbp->task_tag);
>> -        if (ufshcd_clear_cmds(hba, 1U << lrbp->task_tag) == 0) {
>> +        if (ufshcd_clear_cmds(hba, 1UL << lrbp->task_tag) == 0) {
>>               /* successfully cleared the command, retry if needed */
>>               err = -EAGAIN;
>>               /*
> 
> Is this change necessary?
My intention was to support tag greater than 31 and less than 64.
The 1U << only works up to 32 bits.

> 
>> @@ -7379,6 +7397,20 @@ static int ufshcd_try_to_abort_task(struct 
>> ufs_hba *hba, int tag)
>>                */
>>               dev_err(hba->dev, "%s: cmd at tag %d not pending in the 
>> device.\n",
>>                   __func__, tag);
>> +            if (is_mcq_enabled(hba)) {
>> +                /* MCQ mode */
>> +                if (lrbp->cmd) {
>> +                    /* sleep for max. 200us to stabilize */
> 
> What is being stabilized here? Please make this comment more clear.
This is to keep the same operation/timing as in SDB mode.

> 
>> +                    usleep_range(100, 200);
>> +                    continue;
>> +                }
>> +                /* command completed already */
>> +                dev_err(hba->dev, "%s: cmd at tag=%d is cleared.\n",
>> +                    __func__, tag);
>> +                goto out;
>> +            }
> 
> Please do not use lrbp->cmd to check whether or not a command has 
> completed. See also my patch "scsi: ufs: Fix handling of lrbp->cmd".
I have been thinking how to replace lrbp->cmd, but could not find a good 
solution. Throughout this patch series, I am using lrbp->cmd as a way to 
find the pending command that is being aborted and clean up the 
resources associated with it. Any suggestions to achieve it? Thanks.

> 
>> @@ -7415,7 +7447,7 @@ static int ufshcd_try_to_abort_task(struct 
>> ufs_hba *hba, int tag)
>>           goto out;
>>       }
>> -    err = ufshcd_clear_cmds(hba, 1U << tag);
>> +    err = ufshcd_clear_cmds(hba, 1UL << tag);
>>       if (err)
>>           dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err 
>> %d\n",
>>               __func__, tag, err);
> 
> Is this change necessary?
Same as above, I intended  to support 64 > tag > 31

> 
>> -    if (!(test_bit(tag, &hba->outstanding_reqs))) {
>> +    if (!is_mcq_enabled(hba) && !(test_bit(tag, 
>> &hba->outstanding_reqs))) {
> 
> Please leave out one pair of superfluous parentheses from the above 
> expression.
Yes, I will change.

> 
> Thanks,
> 
> Bart.

Re: [PATCH v2 3/5] ufs: mcq: Added ufshcd_mcq_abort()

[Bao D. Nguyen]

On 4/25/2023 5:12 PM, Bart Van Assche wrote:
> On 4/17/23 14:05, Bao D. Nguyen wrote:
>> +    if (!lrbp->cmd) {
>> +        dev_err(hba->dev,
>> +            "%s: skip abort. cmd at tag %d already completed.\n",
>> +            __func__, tag);
>> +        goto out;
>> +    }
> 
> Please do not use lrbp->cmd to check whether or not a command has 
> completed.
Yes. Same comment as in patch #2.

> 
>> +    if (ufshcd_mcq_sqe_search(hba, hwq, tag)) {
>> +        /*
>> +         * Failure. The command should not be "stuck" in SQ for
>> +         * a long time which resulted in command being aborted.
>> +         */
>> +        dev_err(hba->dev, "%s: cmd found in sq. hwq=%d, tag=%d\n",
>> +                __func__, hwq->id, tag);
>> +        /* Set the Command Type to 0xF per the spec */
>> +        ufshcd_mcq_nullify_cmd(hba, hwq);
> 
> The above looks wrong to me. How can ufshcd_mcq_nullify_cmd() identify a 
> command if the 'tag' argument is not passed to that function?
Same comment as in patch #1. I will change this function.

>  >> +    /*
>> +     * The command is not in the Submission Queue, and it is not
>> +     * in the Completion Queue either. Query the device to see if
>> +     * the command is being processed in the device.
>> +     */
> 
> Please only use capitals if these are required.
Yes, I will change.

> 
>> +    if (lrbp->cmd)
>> +        ufshcd_release_scsi_cmd(hba, lrbp);
> 
> Same comment here - do not use lrbp->cmd to check for completion.
Yes.

> 
> Thanks,
> 
> Bart.

Re: [PATCH v2 5/5] ufs: core: Add error handling for MCQ mode

[Bao D. Nguyen]

On 4/25/2023 5:21 PM, Bart Van Assche wrote:
> On 4/17/23 14:05, Bao D. Nguyen wrote:
>> +        /* MCQ mode */
>> +        if (is_mcq_enabled(hba))
>> +            return ufshcd_clear_cmds(hba, 1UL << lrbp->task_tag);
> 
> The above code will trigger an overflow if lrbp->task_tag >= 8 * 
> sizeof(unsigned long). That's not acceptable.
This ufshcd_clear_cmds() uses a bit map. There are multiple places in 
the UFS code have this limitation if the queue depth grows to be greater 
than 64. I am thinking:
1. Current ufs controllers in the market probably support queue depth 64 
or less, so it may not be a problem today if host controller cap is set 
to 64 queue depth, but can be a problem in multiple places in the code 
later.
2. In mcq mode, we can pass a tag number into this api 
ufshcd_clear_cmds(); while in SDB mode, pass the tag's bit mask as before.
3. Use sbitmask() to support large queue depth? Thanks for any suggestions.

> 
>>   static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
>>   {
>> +    struct ufshcd_lrb *lrbp;
>> +    u32 hwq_num, utag;
>> +    int tag;
>> +
>>       /* Resetting interrupt aggregation counters first and reading the
>>        * DOOR_BELL afterward allows us to handle all the completed 
>> requests.
>>        * In order to prevent other interrupts starvation the DB is 
>> read once
>> @@ -5580,7 +5590,22 @@ static irqreturn_t 
>> ufshcd_transfer_req_compl(struct ufs_hba *hba)
>>        * Ignore the ufshcd_poll() return value and return IRQ_HANDLED 
>> since we
>>        * do not want polling to trigger spurious interrupt complaints.
>>        */
>> -    ufshcd_poll(hba->host, UFSHCD_POLL_FROM_INTERRUPT_CONTEXT);
>> +    if (!is_mcq_enabled(hba)) {
>> +        ufshcd_poll(hba->host, UFSHCD_POLL_FROM_INTERRUPT_CONTEXT);
>> +        goto out;
>> +    }
>> +
>> +    /* MCQ mode */
>> +    for (tag = 0; tag < hba->nutrs; tag++) {
>> +        lrbp = &hba->lrb[tag];
>> +        if (lrbp->cmd) {
>> +            utag = blk_mq_unique_tag(scsi_cmd_to_rq(lrbp->cmd));
>> +            hwq_num = blk_mq_unique_tag_to_hwq(utag);
>> +            ufshcd_poll(hba->host, hwq_num);
>> +        }
>> +    }
> 
> Is my understanding correct that ufshcd_transfer_req_compl() is only 
> called from single doorbell code paths and hence that the above change 
> is not necessary?
ufshcd_transfer_req_compl() can be invoked from MCQ mode such as the 
ufshcd_err_handler() as below:
ufshcd_err_handler()-->ufshcd_complete_requests()-->ufshcd_transfer_req_compl()

> 
> 
>> +    if (is_mcq_enabled(hba)) {
>> +        struct ufshcd_lrb *lrbp;
>> +        int tag;
>> +
>> +        for (tag = 0; tag < hba->nutrs; tag++) {
>> +            lrbp = &hba->lrb[tag];
>> +            if (lrbp->cmd) {
>> +                ret = ufshcd_try_to_abort_task(hba, tag);
>> +                dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", 
>> tag,
>> +                    hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
>> +                    ret ? "failed" : "succeeded");
>> +            }
>> +            if (ret) {
>> +                needs_reset = true;
>> +                goto out;
>> +            }
>> +        }
>> +    } else {
>> +        /* Clear pending transfer requests */
>> +        for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
>> +            ret = ufshcd_try_to_abort_task(hba, tag);
>> +            dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
>> +                hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
>> +                ret ? "failed" : "succeeded");
>> +            if (ret) {
>> +                needs_reset = true;
>> +                goto out;
>> +            }
>>           }
>>       }
> 
> Please introduce helper functions for the MCQ and SDB code paths such 
> that the nesting level of the above code is reduced.
Sure. I will change.

> 
> Thanks,
> 
> Bart.

Re: [PATCH v2 2/5] ufs: mcq: Add support for clean up mcq resources

[Bart Van Assche]

On 5/3/23 21:01, Bao D. Nguyen wrote:
> On 4/25/2023 5:08 PM, Bart Van Assche wrote:
>> On 4/17/23 14:05, Bao D. Nguyen wrote:
>>> @@ -3110,7 +3128,7 @@ static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
>>>           err = -ETIMEDOUT;
>>>           dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
>>>               __func__, lrbp->task_tag);
>>> -        if (ufshcd_clear_cmds(hba, 1U << lrbp->task_tag) == 0) {
>>> +        if (ufshcd_clear_cmds(hba, 1UL << lrbp->task_tag) == 0) {
>>>               /* successfully cleared the command, retry if needed */
>>>               err = -EAGAIN;
>>>               /*
>>
>> Is this change necessary?
> My intention was to support tag greater than 31 and less than 64.
> The 1U << only works up to 32 bits.

The UFSHCI 4.0 standard and also the Linux kernel UFS driver support more than
64 outstanding commands so the above change is not sufficient.

>> Please do not use lrbp->cmd to check whether or not a command has completed. See also my patch "scsi: ufs: Fix handling of lrbp->cmd".
> I have been thinking how to replace lrbp->cmd, but could not find a good solution. Throughout this patch series, I am using lrbp->cmd as a way to find the pending command that is being aborted and clean up the resources associated with it. Any suggestions to achieve it? 

Using lrbp->cmd is fine but checking whether or not it is NULL is not OK. How
about using blk_mq_request_started() to check whether or not a request is still
pending?

Thanks,

Bart.

Re: [PATCH v2 5/5] ufs: core: Add error handling for MCQ mode

[Bart Van Assche]

On 5/3/23 21:18, Bao D. Nguyen wrote:
> On 4/25/2023 5:21 PM, Bart Van Assche wrote:
>> On 4/17/23 14:05, Bao D. Nguyen wrote:
>>> +        /* MCQ mode */
>>> +        if (is_mcq_enabled(hba))
>>> +            return ufshcd_clear_cmds(hba, 1UL << lrbp->task_tag);
>>
>> The above code will trigger an overflow if lrbp->task_tag >= 8 * sizeof(unsigned long). That's not acceptable.
> This ufshcd_clear_cmds() uses a bit map. There are multiple places in the UFS code have this limitation if the queue depth grows to be greater than 64. I am thinking:
> 1. Current ufs controllers in the market probably support queue depth 64 or less, so it may not be a problem today if host controller cap is set to 64 queue depth, but can be a problem in multiple places in the code later.
> 2. In mcq mode, we can pass a tag number into this api ufshcd_clear_cmds(); while in SDB mode, pass the tag's bit mask as before.
> 3. Use sbitmask() to support large queue depth? Thanks for any suggestions.

The UFS driver is the only block driver I know that tracks which commands
are pending in a bitmap. Please pass the lrbp pointer or the task_tag directly
to ufshcd_clear_cmds() instead of passing a bitmap to that function. Please
also introduce a loop in ufshcd_eh_device_reset_handler() around the
ufshcd_clear_cmds() call instead of passing a bitmap to ufshcd_clear_cmds().

>>>   static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
>>>   {
>>> +    struct ufshcd_lrb *lrbp;
>>> +    u32 hwq_num, utag;
>>> +    int tag;
>>> +
>>>       /* Resetting interrupt aggregation counters first and reading the
>>>        * DOOR_BELL afterward allows us to handle all the completed requests.
>>>        * In order to prevent other interrupts starvation the DB is read once
>>> @@ -5580,7 +5590,22 @@ static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
>>>        * Ignore the ufshcd_poll() return value and return IRQ_HANDLED since we
>>>        * do not want polling to trigger spurious interrupt complaints.
>>>        */
>>> -    ufshcd_poll(hba->host, UFSHCD_POLL_FROM_INTERRUPT_CONTEXT);
>>> +    if (!is_mcq_enabled(hba)) {
>>> +        ufshcd_poll(hba->host, UFSHCD_POLL_FROM_INTERRUPT_CONTEXT);
>>> +        goto out;
>>> +    }
>>> +
>>> +    /* MCQ mode */
>>> +    for (tag = 0; tag < hba->nutrs; tag++) {
>>> +        lrbp = &hba->lrb[tag];
>>> +        if (lrbp->cmd) {
>>> +            utag = blk_mq_unique_tag(scsi_cmd_to_rq(lrbp->cmd));
>>> +            hwq_num = blk_mq_unique_tag_to_hwq(utag);
>>> +            ufshcd_poll(hba->host, hwq_num);
>>> +        }
>>> +    }
>>
>> Is my understanding correct that ufshcd_transfer_req_compl() is only called from single doorbell code paths and hence that the above change is not necessary?
> ufshcd_transfer_req_compl() can be invoked from MCQ mode such as the ufshcd_err_handler() as below:
> ufshcd_err_handler()-->ufshcd_complete_requests()-->ufshcd_transfer_req_compl()

Since there are multiple statements in ufshcd_transfer_req_compl() that assume
SDB mode (resetting SDB interrupt aggregation and calling ufshcd_poll()), please
move the is_mcq_enabled() test from ufshcd_transfer_req_compl() into the
callers of that function.

Thanks,

Bart.