[PATCH v7 6/7] dmaengine: dw-edma: Dynamically append requests while running

[Koichiro Den <den@valinux.co.jp>]

From: Frank Li <Frank.Li@nxp.com>

Allow issue_pending() to fill free LL entries while the channel is busy,
then check progress and restart it if it has stopped. Publish requests
in descriptor order.

Multiple descriptors can now have entries in the LL ring. When resetting
the ring after an abort, mark the other descriptors with pending entries
as aborted too, so they cannot be replayed.

Since dw_edma_start_transfer() can now return zero with LL entries still
pending, keep such channels BUSY on resume.

Signed-off-by: Frank Li <Frank.Li@nxp.com>
Co-developed-by: Koichiro Den <den@valinux.co.jp>
Signed-off-by: Koichiro Den <den@valinux.co.jp>
---
Changes from v6:
  - Use the common descriptor completion helper when aborting pending
    descriptors during an LL reset (Frank).
  - Polish the commit message a bit.
  - Link to v6: https://lore.kernel.org/r/20261030150742.2586776-5-den@valinux.co.jp/

 drivers/dma/dw-edma/dw-edma-core.c | 65 ++++++++++++++++++++++--------
 1 file changed, 49 insertions(+), 16 deletions(-)

diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index e2f2a535d5ad..69b5488282de 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -111,8 +111,22 @@ static void dw_edma_complete_vdesc(struct virt_dma_desc *vd,
 
 static void dw_edma_core_reset_ll(struct dw_edma_chan *chan)
 {
+	struct virt_dma_desc *vd, *tmp;
 	u32 i;
 
+	/*
+	 * We cannot tell which pending entries completed before the reset.
+	 * Abort their descriptors to avoid repeating a transfer.
+	 */
+	list_for_each_entry_safe(vd, tmp, &chan->vc.desc_issued, node) {
+		struct dw_edma_desc *desc = vd2dw_edma_desc(vd);
+
+		if (desc->start_burst == desc->done_burst)
+			continue;
+
+		dw_edma_complete_vdesc(vd, DMA_TRANS_ABORTED);
+	}
+
 	chan->ll_head = 0;
 	chan->ll_done = 0;
 	/* Drop stale CB bits before reusing the circular LL ring. */
@@ -278,21 +292,34 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan)
 {
 	struct dw_edma_desc *desc;
 	struct virt_dma_desc *vd;
+	int ret = 0;
 
-	vd = vchan_next_desc(&chan->vc);
-	if (!vd)
-		return 0;
+	if (chan->non_ll) {
+		vd = vchan_next_desc(&chan->vc);
+		if (!vd)
+			return 0;
 
-	desc = vd2dw_edma_desc(vd);
-	if (!desc)
-		return 0;
+		dw_edma_core_start(vd2dw_edma_desc(vd));
+		return 1;
+	}
 
-	if (!chan->non_ll && !chan->ll_valid)
-		dw_edma_core_reset_ll(chan);
+	list_for_each_entry(vd, &chan->vc.desc_issued, node) {
+		if (!dw_edma_core_get_free_num(chan))
+			break;
+
+		desc = vd2dw_edma_desc(vd);
+		/* Fully published descriptors may still be in flight. */
+		if (desc->start_burst == desc->nburst)
+			continue;
 
-	dw_edma_core_start(desc);
+		if (!chan->ll_valid)
+			dw_edma_core_reset_ll(chan);
 
-	return 1;
+		dw_edma_core_start(desc);
+		ret = 1;
+	}
+
+	return ret;
 }
 
 static void dw_edma_terminate_vdesc(struct virt_dma_desc *vd)
@@ -561,7 +588,8 @@ static int dw_edma_device_resume(struct dma_chan *dchan)
 		err = -EPERM;
 	} else {
 		chan->status = EDMA_ST_BUSY;
-		if (!dw_edma_start_transfer(chan))
+		if (!dw_edma_start_transfer(chan) &&
+		    (chan->non_ll || !dw_edma_ll_pending(chan)))
 			chan->status = EDMA_ST_IDLE;
 		dw_edma_core_ch_maybe_doorbell(chan);
 	}
@@ -610,11 +638,16 @@ static void dw_edma_device_issue_pending(struct dma_chan *dchan)
 
 	spin_lock_irqsave(&chan->vc.lock, flags);
 	if (chan->configured && vchan_issue_pending(&chan->vc) &&
-	    chan->request == EDMA_REQ_NONE &&
-	    chan->status == EDMA_ST_IDLE) {
-		chan->status = EDMA_ST_BUSY;
-		dw_edma_start_transfer(chan);
-		dw_edma_core_ch_maybe_doorbell(chan);
+	    chan->request == EDMA_REQ_NONE) {
+		if (chan->status == EDMA_ST_IDLE) {
+			chan->status = EDMA_ST_BUSY;
+			dw_edma_start_transfer(chan);
+			dw_edma_core_ch_maybe_doorbell(chan);
+		} else if (!chan->non_ll &&
+			   chan->status == EDMA_ST_BUSY) {
+			dw_edma_start_transfer(chan);
+			dw_edma_ll_advance(chan, false);
+		}
 	}
 	spin_unlock_irqrestore(&chan->vc.lock, flags);
 }
-- 
2.54.0