[PATCH v2 3/3] spi: dw: Add DMA support for enhanced memory operations

[Changhuang Liang <changhuang.liang@starfivetech.com>]

Implement DMA support for enhanced SPI memory operations. These
operations are half-duplex, so they work with either one or two DMA
channels.

Add enhanced memory DMA callbacks (get_chan, setup, can_dma, transfer)
to struct dw_spi_dma_ops and hook them into the generic DMA operations.
They reuse the existing channel configuration, submission and wait
helpers.

With two channels, both are requested at probe time and kept for the
lifetime of the controller. With a single channel, as on JHB100, the
channel is bound to one direction by its "tx"/"rx" DMA specifier. The
channel currently held (dws->txchan or dws->rxchan) records which
direction it serves. It is reused as long as the direction stays the
same, and only released and requested for the other direction when
the transfer direction changes.

In dw_spi_exec_enh_mem_op(), record the direction in dws->tx_dir and
use the DMA path when a channel is available for that direction,
can_dma_enh_mem() accepts the transfer, and the transfer exceeds the
FIFO length; otherwise fall back to the interrupt path.

Also move the udelay(5) workaround into dw_spi_enh_write_cmd_addr() so
it applies to both paths, and enable the generic DMA setup in
dw_spi_jhb100_init().

Signed-off-by: Changhuang Liang <changhuang.liang@starfivetech.com>
---
 drivers/spi/spi-dw-core.c | 100 +++++++++++++--------
 drivers/spi/spi-dw-dma.c  | 182 ++++++++++++++++++++++++++++++++++++++
 drivers/spi/spi-dw-mmio.c |   2 +
 drivers/spi/spi-dw.h      |   7 ++
 4 files changed, 255 insertions(+), 36 deletions(-)

diff --git a/drivers/spi/spi-dw-core.c b/drivers/spi/spi-dw-core.c
index 6889f4446ecb..892b3a8787eb 100644
--- a/drivers/spi/spi-dw-core.c
+++ b/drivers/spi/spi-dw-core.c
@@ -937,6 +937,15 @@ static void dw_spi_enh_write_cmd_addr(struct dw_spi *dws, const struct spi_mem_o
 
 		dw_spi_set_cs(mem->spi, false);
 	}
+
+	/*
+	 * FIXME: The exact reason for this delay is not fully understood,
+	 * but empirical testing shows it significantly improves the stability
+	 * of read/write operations. Without this delay, occasional transfer
+	 * errors or timeouts may occur under certain conditions.
+	 * Keeping it as a safeguard based on practical validation.
+	 */
+	udelay(5);
 }
 
 static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op *op)
@@ -948,6 +957,8 @@ static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op *
 	unsigned long long ms;
 	int ret;
 
+	dws->dma_mapped = false;
+
 	switch (op->data.buswidth) {
 	case 0:
 	case 1:
@@ -971,10 +982,13 @@ static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op *
 	cfg.dfs = 8;
 	cfg.freq = clamp(op->max_freq, 0U, dws->max_mem_freq);
 	cfg.ndf = op->data.nbytes;
-	if (op->data.dir == SPI_MEM_DATA_IN)
+	if (op->data.dir == SPI_MEM_DATA_IN) {
 		cfg.tmode = DW_SPI_CTRLR0_TMOD_RO;
-	else
+		dws->tx_dir = false;
+	} else {
 		cfg.tmode = DW_SPI_CTRLR0_TMOD_TO;
+		dws->tx_dir = true;
+	}
 
 	if (op->data.buswidth == op->addr.buswidth &&
 	    op->data.buswidth == op->cmd.buswidth)
@@ -1010,47 +1024,61 @@ static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op *
 		}
 	}
 
-	dw_spi_enh_write_cmd_addr(dws, op, mem);
+	if (dws->dma_nr_chans && dws->dma_ops->dma_enh_mem_get_chan &&
+	    op->data.nbytes > dws->fifo_len) {
+		ret = dws->dma_ops->dma_enh_mem_get_chan(ctlr->dev.parent, dws);
+		if (ret) {
+			/* fall back to the PIO/IRQ path */
+			dev_dbg(&ctlr->dev, "DMA enh mem channel unavailable (%d)\n", ret);
+		} else if (dws->dma_ops->can_dma_enh_mem(ctlr)) {
+			ret = dws->dma_ops->dma_enh_mem_setup(dws);
+			if (ret)
+				dev_err(&ctlr->dev, "DMA enh mem setup failed (%d)\n", ret);
+			else
+				dws->dma_mapped = true;
+		}
+	}
 
-	/*
-	 * FIXME: The exact reason for this delay is not fully understood,
-	 * but empirical testing shows it significantly improves the stability
-	 * of read/write operations. Without this delay, occasional transfer
-	 * errors or timeouts may occur under certain conditions.
-	 * Keeping it as a safeguard based on practical validation.
-	 */
-	udelay(5);
+	if (dws->dma_mapped) {
+		dw_spi_enh_write_cmd_addr(dws, op, mem);
 
-	dw_spi_enh_irq_setup(dws);
+		ret = dws->dma_ops->dma_enh_mem_transfer(mem, op);
 
-	/* Use timeout calculation from spi_transfer_wait() */
-	ms = 8LL * MSEC_PER_SEC * (dws->rx_len ? dws->rx_len : dws->tx_len);
-	do_div(ms, dws->current_freq);
+		dw_spi_stop_mem_op(dws, mem->spi);
+	} else {
+		dw_spi_enh_write_cmd_addr(dws, op, mem);
 
-	/*
-	 * Increase it twice and add 200 ms tolerance, use
-	 * predefined maximum in case of overflow.
-	 */
-	ms += ms + 200;
-	if (ms > UINT_MAX)
-		ms = UINT_MAX;
+		dw_spi_enh_irq_setup(dws);
 
-	ms = wait_for_completion_timeout(&ctlr->xfer_completion,
-					 msecs_to_jiffies(ms));
-	if (ms == 0) {
-		dw_spi_mask_intr(dws, 0xff);
-		synchronize_irq(dws->irq);
-		dws->rx = NULL;
-		dws->tx = NULL;
-		dws->rx_len = 0;
-		dws->tx_len = 0;
-		dw_spi_stop_mem_op(dws, mem->spi);
-		return -EIO;
-	}
+		/* Use timeout calculation from spi_transfer_wait() */
+		ms = 8LL * MSEC_PER_SEC * (dws->rx_len ? dws->rx_len : dws->tx_len);
+		do_div(ms, dws->current_freq);
+
+		/*
+		 * Increase it twice and add 200 ms tolerance, use
+		 * predefined maximum in case of overflow.
+		 */
+		ms += ms + 200;
+		if (ms > UINT_MAX)
+			ms = UINT_MAX;
+
+		ms = wait_for_completion_timeout(&ctlr->xfer_completion,
+						 msecs_to_jiffies(ms));
+		if (ms == 0) {
+			dw_spi_mask_intr(dws, 0xff);
+			synchronize_irq(dws->irq);
+			dws->rx = NULL;
+			dws->tx = NULL;
+			dws->rx_len = 0;
+			dws->tx_len = 0;
+			dw_spi_stop_mem_op(dws, mem->spi);
+			return -EIO;
+		}
 
-	ret = dw_spi_wait_mem_op_done(dws);
+		ret = dw_spi_wait_mem_op_done(dws);
 
-	dw_spi_stop_mem_op(dws, mem->spi);
+		dw_spi_stop_mem_op(dws, mem->spi);
+	}
 
 	return ret;
 }
diff --git a/drivers/spi/spi-dw-dma.c b/drivers/spi/spi-dw-dma.c
index 76d7814f0dcf..a86039057988 100644
--- a/drivers/spi/spi-dw-dma.c
+++ b/drivers/spi/spi-dw-dma.c
@@ -728,6 +728,183 @@ static void dw_spi_dma_stop(struct dw_spi *dws)
 	}
 }
 
+/*
+ * The enhanced SPI memory operations are half-duplex, so a single DMA channel
+ * is enough for them. On a platform with only one channel, the channel is
+ * bound to a direction by its "tx"/"rx" DMA specifier, and whichever channel
+ * is currently held tells which direction it serves. Keep it across
+ * operations and swap it only when the transfer direction changes.
+ */
+static int dw_spi_enh_mem_dma_get_chan(struct device *dev, struct dw_spi *dws)
+{
+	struct dma_chan **chan, **other;
+	const char *name;
+	int ret;
+
+	if (dws->dma_nr_chans == 2)
+		return 0;
+
+	if (dws->tx_dir) {
+		chan = &dws->txchan;
+		other = &dws->rxchan;
+		name = "tx";
+	} else {
+		chan = &dws->rxchan;
+		other = &dws->txchan;
+		name = "rx";
+	}
+
+	if (*chan)
+		return 0;
+
+	dws->ctlr->dma_tx = NULL;
+	dws->ctlr->dma_rx = NULL;
+
+	/*
+	 * With a single physical channel, the held channel must be released
+	 * before it can be requested again with the handshake of the other
+	 * direction. On failure both stay NULL, and the next operation
+	 * retries the request and falls back to PIO meanwhile.
+	 */
+
+	if (*other) {
+		dma_release_channel(*other);
+		*other = NULL;
+	}
+
+	*chan = dma_request_chan(dev, name);
+	if (IS_ERR(*chan)) {
+		ret = PTR_ERR(*chan);
+		*chan = NULL;
+		return ret;
+	}
+
+	ret = dw_spi_dma_caps_init(dws);
+	if (ret) {
+		dma_release_channel(*chan);
+		*chan = NULL;
+		return ret;
+	}
+
+	dw_spi_dma_maxburst_init(dws);
+
+	dws->ctlr->dma_tx = dws->txchan;
+	dws->ctlr->dma_rx = dws->rxchan;
+
+	return 0;
+}
+
+static int dw_spi_enh_mem_dma_setup(struct dw_spi *dws)
+{
+	u16 dma_ctrl, level;
+	int ret;
+
+	/* Setup DMA channels */
+	if (dws->tx_dir) {
+		ret = dw_spi_dma_config_tx(dws);
+		if (ret)
+			return ret;
+
+		dma_ctrl = DW_SPI_DMACR_TDMAE;
+	} else {
+		ret = dw_spi_dma_config_rx(dws);
+		if (ret)
+			return ret;
+
+		dma_ctrl = DW_SPI_DMACR_RDMAE;
+	}
+
+	dw_writel(dws, DW_SPI_DMACR, dma_ctrl);
+
+	/* Clear stale error status, it is checked after the transfer */
+	dw_readl(dws, DW_SPI_ICR);
+
+	reinit_completion(&dws->dma_completion);
+
+	level = min_t(unsigned int, dws->fifo_len / 2, dws->tx_len);
+	dw_writel(dws, DW_SPI_TXFTLR, level);
+
+	level = min_t(unsigned int, dws->fifo_len / 2, dws->rx_len);
+	dw_writel(dws, DW_SPI_RXFTLR, level ? level - 1 : 0);
+
+	return 0;
+}
+
+static bool dw_spi_enh_mem_can_dma(struct spi_controller *ctlr)
+{
+	struct dw_spi *dws = spi_controller_get_devdata(ctlr);
+	unsigned int len = dws->tx_dir ? dws->tx_len : dws->rx_len;
+	enum dma_slave_buswidth dma_bus_width;
+
+	if (len <= dws->fifo_len)
+		return false;
+
+	dma_bus_width = dw_spi_dma_convert_width(dws->n_bytes);
+
+	return dws->dma_addr_widths & BIT(dma_bus_width);
+}
+
+static void dw_spi_enh_mem_dma_stop(struct dw_spi *dws)
+{
+	if (dws->tx_dir) {
+		if (test_bit(DW_SPI_TX_BUSY, &dws->dma_chan_busy)) {
+			dmaengine_terminate_sync(dws->txchan);
+			clear_bit(DW_SPI_TX_BUSY, &dws->dma_chan_busy);
+		}
+	} else {
+		if (test_bit(DW_SPI_RX_BUSY, &dws->dma_chan_busy)) {
+			dmaengine_terminate_sync(dws->rxchan);
+			clear_bit(DW_SPI_RX_BUSY, &dws->dma_chan_busy);
+		}
+	}
+}
+
+static int dw_spi_enh_mem_dma_transfer(struct spi_mem *mem, const struct spi_mem_op *op)
+{
+	struct spi_controller *ctlr = mem->spi->controller;
+	struct dw_spi *dws = spi_controller_get_devdata(ctlr);
+	struct sg_table sgt;
+	int ret;
+
+	ret = spi_controller_dma_map_mem_op_data(ctlr, op, &sgt);
+	if (ret)
+		goto out_clear_dmac;
+
+	if (dws->tx_dir) {
+		ret = dw_spi_dma_submit_tx(dws, sgt.sgl, sgt.nents);
+		if (ret)
+			goto out_unmap;
+
+		dma_async_issue_pending(dws->txchan);
+
+		ret = dw_spi_dma_wait(dws, dws->tx_len, dws->current_freq);
+		if (!ret)
+			ret = dw_spi_dma_wait_tx_done(dws, dws->current_freq);
+	} else {
+		ret = dw_spi_dma_submit_rx(dws, sgt.sgl, sgt.nents);
+		if (ret)
+			goto out_unmap;
+
+		dma_async_issue_pending(dws->rxchan);
+
+		ret = dw_spi_dma_wait(dws, dws->rx_len, dws->current_freq);
+		if (!ret)
+			ret = dw_spi_dma_wait_rx_done(dws);
+	}
+
+	if (ret)
+		dw_spi_enh_mem_dma_stop(dws);
+	else
+		ret = dw_spi_check_status(dws, true);
+
+out_unmap:
+	spi_controller_dma_unmap_mem_op_data(ctlr, op, &sgt);
+out_clear_dmac:
+	dw_writel(dws, DW_SPI_DMACR, 0);
+
+	return ret;
+}
+
 static const struct dw_spi_dma_ops dw_spi_dma_mfld_ops = {
 	.dma_init	= dw_spi_dma_init_mfld,
 	.dma_exit	= dw_spi_dma_exit,
@@ -750,6 +927,11 @@ static const struct dw_spi_dma_ops dw_spi_dma_generic_ops = {
 	.can_dma	= dw_spi_can_dma,
 	.dma_transfer	= dw_spi_dma_transfer,
 	.dma_stop	= dw_spi_dma_stop,
+
+	.dma_enh_mem_get_chan	= dw_spi_enh_mem_dma_get_chan,
+	.dma_enh_mem_setup	= dw_spi_enh_mem_dma_setup,
+	.can_dma_enh_mem	= dw_spi_enh_mem_can_dma,
+	.dma_enh_mem_transfer	= dw_spi_enh_mem_dma_transfer,
 };
 
 void dw_spi_dma_setup_generic(struct dw_spi *dws)
diff --git a/drivers/spi/spi-dw-mmio.c b/drivers/spi/spi-dw-mmio.c
index 8cdb0351605b..9c70947c969c 100644
--- a/drivers/spi/spi-dw-mmio.c
+++ b/drivers/spi/spi-dw-mmio.c
@@ -363,6 +363,8 @@ static int dw_spi_jhb100_init(struct platform_device *pdev,
 
 	dw_spi_jhb100_mask_intr(&dwsmmio->dws, 0xff);
 
+	dw_spi_dma_setup_generic(&dwsmmio->dws);
+
 	return 0;
 }
 
diff --git a/drivers/spi/spi-dw.h b/drivers/spi/spi-dw.h
index 01eb24ae7fe0..bfdf371bcacb 100644
--- a/drivers/spi/spi-dw.h
+++ b/drivers/spi/spi-dw.h
@@ -183,6 +183,12 @@ struct dw_spi_dma_ops {
 			struct spi_transfer *xfer);
 	int (*dma_transfer)(struct dw_spi *dws, struct spi_transfer *xfer);
 	void (*dma_stop)(struct dw_spi *dws);
+
+	/* enh mem dma ops */
+	int (*dma_enh_mem_get_chan)(struct device *dev, struct dw_spi *dws);
+	int (*dma_enh_mem_setup)(struct dw_spi *dws);
+	bool (*can_dma_enh_mem)(struct spi_controller *ctlr);
+	int (*dma_enh_mem_transfer)(struct spi_mem *mem, const struct spi_mem_op *op);
 };
 
 struct dw_spi {
@@ -228,6 +234,7 @@ struct dw_spi {
 	struct dma_chan		*rxchan;
 	u32			rxburst;
 	unsigned int		dma_nr_chans;
+	bool			tx_dir;
 	u32			dma_sg_burst;
 	u32			dma_addr_widths;
 	unsigned long		dma_chan_busy;
-- 
2.25.1