[PATCH RFT 4/5] dmaengine: dw-edma: Dynamitc append new request during dmaengine running

[Frank Li <Frank.Li@nxp.com>]

This use PCS-CCS-CB-TCB Producer-Consumer Synchronization module, which
support append new DMA request during dmaengine runnings.

Append new request during dmaengine runnings.

But look like hardware have bug, which missed doorbell when engine is
running. So add workaround to push doorbelll again when found engine stop.

Get more than 10% performance gain.

The before
  Rnd read,    4KB, QD=32, 4 jobs:  IOPS=33.4k, BW=130MiB/s (137MB/s)

After
  Rnd read,    4KB, QD=32, 4 jobs:  IOPS=38.8k, BW=151MiB/s (159MB/s)

Signed-off-by: Frank Li <Frank.Li@nxp.com>
---
 drivers/dma/dw-edma/dw-edma-core.c    | 104 ++++++++++++++++++++++++++--------
 drivers/dma/dw-edma/dw-edma-core.h    |   2 +
 drivers/dma/dw-edma/dw-edma-v0-core.c |  22 ++++++-
 3 files changed, 102 insertions(+), 26 deletions(-)

diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index 678bbc4e65f0e2fced6efec88a3af6935d833bc6..5aacd04bd2da4a65aabec48f6631f6f8882eecfd 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -65,6 +65,7 @@ static void dw_edma_core_reset_ll(struct dw_edma_chan *chan)
 	chan->ll_head = 0;
 	chan->ll_end = 0;
 	chan->cb = true;
+	chan->cur_idx = -1;
 
 	dw_edma_core_ll_link(chan, chan->ll_max - 1, chan->cb,
 			     chan->ll_region.paddr);
@@ -82,6 +83,12 @@ static u32 dw_edma_core_get_free_num(struct dw_edma_chan *chan)
 		(chan->ll_max - 1);
 }
 
+static u32 dw_edma_core_get_done_num(struct dw_edma_chan *chan, u32 index)
+{
+	return (index - chan->ll_end + chan->ll_max - 1) % (chan->ll_max - 1);
+}
+
+/* Need hold vc.lock */
 static void dw_edma_core_start(struct dw_edma_desc *desc, bool first)
 {
 	struct dw_edma_chan *chan = desc->chan;
@@ -94,6 +101,11 @@ static void dw_edma_core_start(struct dw_edma_desc *desc, bool first)
 		if (!free)
 			break;
 
+		/* need update link CB before last update last item */
+		if (chan->ll_head == chan->ll_max - 2)
+			dw_edma_core_ll_link(chan, chan->ll_max - 1, chan->cb,
+					     chan->ll_region.paddr);
+
 		/* Enable irq for last free entry or last burst */
 		dw_edma_core_ll_data(chan, &desc->burst[i],
 				     chan->ll_head, chan->cb,
@@ -108,32 +120,36 @@ static void dw_edma_core_start(struct dw_edma_desc *desc, bool first)
 	}
 
 	desc->done_burst = desc->start_burst;
-	desc->start_burst += i;
+	desc->start_burst = i;
 	desc->ll_end = chan->ll_head;
-
-	dw_edma_core_ch_doorbell(chan);
 }
 
+/* Need hold vc.lock */
 static int dw_edma_start_transfer(struct dw_edma_chan *chan)
 {
 	struct dw_edma_desc *desc;
 	struct virt_dma_desc *vd;
 	int index = dw_edma_core_ll_cur_idx(chan);
+	int ret = 0;
 
 	if (index < 0)
 		dw_edma_core_reset_ll(chan);
 
-	vd = vchan_next_desc(&chan->vc);
-	if (!vd)
-		return 0;
+	list_for_each_entry(vd, &chan->vc.desc_issued, node) {
+		if (!dw_edma_core_get_free_num(chan))
+			return ret;
 
-	desc = vd2dw_edma_desc(vd);
-	if (!desc)
-		return 0;
+		desc = vd2dw_edma_desc(vd);
 
-	dw_edma_core_start(desc, !desc->start_burst);
+		if (desc->start_burst != desc->nburst) {
+			dw_edma_core_start(desc, !desc->start_burst);
+			ret = 1;
+		} else {
+			break;
+		}
+	}
 
-	return 1;
+	return ret;
 }
 
 static void dw_hdma_set_callback_result(struct virt_dma_desc *vd,
@@ -161,6 +177,31 @@ static void dw_hdma_set_callback_result(struct virt_dma_desc *vd,
 	res->residue = residue;
 }
 
+/* Need hold vc.lock */
+static void dw_edma_ll_clean_pending(struct dw_edma_chan *chan, int idx)
+{
+	struct virt_dma_desc *vd, *_vd;
+
+	list_for_each_entry_safe(vd, _vd, &chan->vc.desc_issued, node) {
+		struct dw_edma_desc *desc = vd2dw_edma_desc(vd);
+
+		if (desc->start_burst == desc->nburst) {
+			if (dw_edma_core_get_done_num(chan, idx) >=
+			    dw_edma_core_get_done_num(chan, desc->ll_end)) {
+
+				dw_hdma_set_callback_result(vd,
+							    DMA_TRANS_NOERROR);
+				list_del(&vd->node);
+				vchan_cookie_complete(vd);
+				chan->ll_end = desc->ll_end;
+			}
+		} else {
+			break;
+		}
+	}
+
+}
+
 static void dw_edma_device_caps(struct dma_chan *dchan,
 				struct dma_slave_caps *caps)
 {
@@ -272,12 +313,13 @@ static void dw_edma_device_issue_pending(struct dma_chan *dchan)
 		return;
 
 	spin_lock_irqsave(&chan->vc.lock, flags);
-	if (vchan_issue_pending(&chan->vc) && chan->request == EDMA_REQ_NONE &&
-	    chan->status == EDMA_ST_IDLE) {
+	if (vchan_issue_pending(&chan->vc)) {
 		chan->status = EDMA_ST_BUSY;
 		dw_edma_start_transfer(chan);
 	}
 	spin_unlock_irqrestore(&chan->vc.lock, flags);
+
+	dw_edma_core_ch_doorbell(chan);
 }
 
 static enum dma_status
@@ -290,7 +332,23 @@ dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
 	unsigned long flags;
 	enum dma_status ret;
 	u32 residue = 0;
+	int idx;
 
+	ret = dma_cookie_status(dchan, cookie, txstate);
+	if (ret == DMA_COMPLETE)
+		return ret;
+
+	spin_lock_irqsave(&chan->vc.lock, flags);
+	idx = dw_edma_core_ll_cur_idx(chan);
+	if (idx != chan->cur_idx) {
+		chan->cur_idx = idx;
+
+		dw_edma_ll_clean_pending(chan, idx);
+		dw_edma_start_transfer(chan);
+	}
+	spin_unlock_irqrestore(&chan->vc.lock, flags);
+
+	/* check gain because dw_edma_ll_clean_pending() may update cookie */
 	ret = dma_cookie_status(dchan, cookie, txstate);
 	if (ret == DMA_COMPLETE)
 		return ret;
@@ -545,26 +603,20 @@ dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan,
 
 static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
 {
-	struct dw_edma_desc *desc;
 	struct virt_dma_desc *vd;
 	unsigned long flags;
+	int idx;
 
 	spin_lock_irqsave(&chan->vc.lock, flags);
+	idx = dw_edma_core_ll_cur_idx(chan);
+	if (idx != chan->cur_idx) {
+		chan->cur_idx = idx;
+		dw_edma_ll_clean_pending(chan, idx);
+	}
 	vd = vchan_next_desc(&chan->vc);
 	if (vd) {
 		switch (chan->request) {
 		case EDMA_REQ_NONE:
-			desc = vd2dw_edma_desc(vd);
-			if (desc->start_burst >= desc->nburst) {
-				dw_hdma_set_callback_result(vd,
-							    DMA_TRANS_NOERROR);
-				list_del(&vd->node);
-				vchan_cookie_complete(vd);
-				chan->ll_end = desc->ll_end;
-			}
-
-			/* Continue transferring if there are remaining chunks or issued requests.
-			 */
 			chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE;
 			break;
 
@@ -585,6 +637,8 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
 		}
 	}
 	spin_unlock_irqrestore(&chan->vc.lock, flags);
+
+	dw_edma_core_ch_doorbell(chan);
 }
 
 static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
index fd4b086a36441cc3209131e4274d6c47de4d616c..94d49f8359b99a9b0f8ca708edf81ca854dff4c2 100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -108,6 +108,8 @@ struct dw_edma_chan {
 	enum dw_edma_status		status;
 	u8				configured;
 
+	int				cur_idx;
+
 	struct dma_slave_config		config;
 };
 
diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c
index edc71a4dbc798386508e15f44e85c23e7e50f2ee..bb9a1682f943dafef28bcf52ab83f3485068f8ed 100644
--- a/drivers/dma/dw-edma/dw-edma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-edma-v0-core.c
@@ -499,7 +499,6 @@ static void dw_edma_v0_core_ch_doorbell(struct dw_edma_chan *chan)
 
 	dw_edma_v0_sync_ll_data(chan);
 
-	/* Doorbell */
 	SET_RW_32(dw, chan->dir, doorbell,
 		  FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id));
 }
@@ -517,6 +516,27 @@ static int dw_edma_v0_core_ll_cur_idx(struct dw_edma_chan *chan)
 	if (!val)
 		return -EINVAL;
 
+	/*
+	 * DMA engine looks like have hardware bugs, Doorbell will be missed
+	 * if DMA engine running, so last update descriptor have not fetched by
+	 * DMA engine, so DMA engine stop.
+	 *
+	 *	Most like issue happen at
+	 *
+	 *	  DMA Engine		|	SW
+	 *        ======================================
+	 *  1     send Read req for LL
+	 *  2					update LL
+	 *  3					doorbell
+	 *  4	  *Missed doorbell*
+	 *  5     Get old LL data
+	 *  6     DMA stop
+	 *
+	 * Workaround: Push doorbell again when found DMA stop.
+	 */
+	if (dw_edma_v0_core_ch_status(chan) != DMA_IN_PROGRESS)
+		dw_edma_v0_core_ch_doorbell(chan);
+
 	return (val - (paddr & 0xFFFFFFFF)) / EDMA_LL_SZ;
 }
 

-- 
2.34.1