[PATCH v1 0/2] Add enhance SPI DMA support for JHB100 SFC

[PATCH v1 0/2] Add enhance SPI DMA support for JHB100 SFC

[Changhuang Liang]

On the JHB100, there is only one channel available for DMA transfer.
In flash access scenarios, it operates in half-duplex mode, so we
can request this DMA channel for the direction that transfers a
larger amount of data. Different transfer directions need to share
a single DMA channel in a time-multiplexed manner. Therefore, it is
necessary to request the DMA channel at runtime based on the current
transfer direction, and release the DMA channel after use.

This series has already been tested on the JHB100 EVB1.

Changhuang Liang (2):
  spi: dw: Request DMA channels at runtime instead of probe
  spi: dw: Add DMA support for enhanced memory operations

 drivers/spi/spi-dw-core.c | 154 ++++++++++++++------
 drivers/spi/spi-dw-dma.c  | 289 +++++++++++++++++++++++++++++++++++++-
 drivers/spi/spi-dw-mmio.c |   2 +
 drivers/spi/spi-dw.h      |  10 ++
 4 files changed, 404 insertions(+), 51 deletions(-)

--
2.25.1

[PATCH v1 1/2] spi: dw: Request DMA channels at runtime instead of probe

[Changhuang Liang]

The DW SPI controller requests its DMA channels in
dw_spi_add_controller(), which ties them up for the entire lifetime of
the controller. Even a controller that is only ever used for standard
SPI transfers and never for enhanced SPI transfers holds the channels
exclusively from probe onwards, preventing them from being shared with
other users.

At probe time, first check whether the controller has DMA channels,
and release them immediately after the check. Remove the DMA channel
release from the remove path. Implement prepare_transfer_hardware()
and unprepare_transfer_hardware() to acquire and release the channels
later at runtime. This ensures that the DMA channels can be
time-multiplexed between enhanced SPI and standard SPI.

Since DMA is no longer set up at probe and torn down at remove.
dw_spi_dma_exit() now also clears txchan/rxchan, the controller's
dma_tx/dma_rx pointers, so that the state is consistent across
repeated prepare/unprepare cycles. Also check whether the channels
are valid in dw_spi_can_dma().

Signed-off-by: Changhuang Liang <changhuang.liang@starfivetech.com>
---
 drivers/spi/spi-dw-core.c | 47 ++++++++++++++++++++++++++++++++-------
 drivers/spi/spi-dw-dma.c  |  8 +++++++
 2 files changed, 47 insertions(+), 8 deletions(-)

diff --git a/drivers/spi/spi-dw-core.c b/drivers/spi/spi-dw-core.c
index 206d3f9dd83d..04a5b48373e1 100644
--- a/drivers/spi/spi-dw-core.c
+++ b/drivers/spi/spi-dw-core.c
@@ -545,6 +545,39 @@ static int dw_spi_transfer_one(struct spi_controller *ctlr,
 	return 1;
 }
 
+static int dw_spi_prepare_hardware(struct spi_controller *ctlr)
+{
+	struct dw_spi *dws = spi_controller_get_devdata(ctlr);
+	int ret;
+
+	if (!ctlr->can_dma)
+		return 0;
+
+	ret = dws->dma_ops->dma_init(ctlr->dev.parent, dws);
+	if (ret) {
+		/*
+		 * DMA is optional: fall back to the PIO/IRQ path instead of
+		 * failing the message. Use dev_dbg() since this may happen
+		 * on every prepare.
+		 */
+		dev_dbg(&ctlr->dev, "DMA init failed (%d), using PIO\n", ret);
+
+		return 0;
+	}
+
+	return 0;
+}
+
+static int dw_spi_unprepare_hardware(struct spi_controller *ctlr)
+{
+	struct dw_spi *dws = spi_controller_get_devdata(ctlr);
+
+	if (dws->dma_ops && dws->dma_ops->dma_exit)
+		dws->dma_ops->dma_exit(dws);
+
+	return 0;
+}
+
 static inline void dw_spi_abort(struct spi_controller *ctlr)
 {
 	struct dw_spi *dws = spi_controller_get_devdata(ctlr);
@@ -1330,6 +1363,8 @@ int dw_spi_add_controller(struct device *dev, struct dw_spi *dws)
 	ctlr->setup = dw_spi_setup;
 	ctlr->cleanup = dw_spi_cleanup;
 	ctlr->transfer_one = dw_spi_transfer_one;
+	ctlr->prepare_transfer_hardware = dw_spi_prepare_hardware;
+	ctlr->unprepare_transfer_hardware = dw_spi_unprepare_hardware;
 	ctlr->handle_err = dw_spi_handle_err;
 	ctlr->auto_runtime_pm = true;
 
@@ -1354,13 +1389,14 @@ int dw_spi_add_controller(struct device *dev, struct dw_spi *dws)
 	device_property_read_u32(dev, "rx-sample-delay-ns",
 				 &dws->def_rx_sample_dly_ns);
 
-	if (dws->dma_ops && dws->dma_ops->dma_init) {
+	if (dws->dma_ops && dws->dma_ops->dma_init && dws->dma_ops->dma_exit) {
 		ret = dws->dma_ops->dma_init(dev, dws);
 		if (ret == -EPROBE_DEFER) {
 			goto err_free_irq;
 		} else if (ret) {
 			dev_warn(dev, "DMA init failed\n");
 		} else {
+			dws->dma_ops->dma_exit(dws);
 			ctlr->can_dma = dws->dma_ops->can_dma;
 			ctlr->flags |= SPI_CONTROLLER_MUST_TX;
 		}
@@ -1369,15 +1405,13 @@ int dw_spi_add_controller(struct device *dev, struct dw_spi *dws)
 	ret = spi_register_controller(ctlr);
 	if (ret) {
 		dev_err_probe(dev, ret, "problem registering spi controller\n");
-		goto err_dma_exit;
+		goto err_disable_ctlr;
 	}
 
 	dw_spi_debugfs_init(dws);
 	return 0;
 
-err_dma_exit:
-	if (dws->dma_ops && dws->dma_ops->dma_exit)
-		dws->dma_ops->dma_exit(dws);
+err_disable_ctlr:
 	dw_spi_enable_chip(dws, 0);
 err_free_irq:
 	free_irq(dws->irq, ctlr);
@@ -1393,9 +1427,6 @@ void dw_spi_remove_controller(struct dw_spi *dws)
 
 	spi_unregister_controller(dws->ctlr);
 
-	if (dws->dma_ops && dws->dma_ops->dma_exit)
-		dws->dma_ops->dma_exit(dws);
-
 	dw_spi_shutdown_chip(dws);
 
 	free_irq(dws->irq, dws->ctlr);
diff --git a/drivers/spi/spi-dw-dma.c b/drivers/spi/spi-dw-dma.c
index f7d848fec9ab..022eb4fae33d 100644
--- a/drivers/spi/spi-dw-dma.c
+++ b/drivers/spi/spi-dw-dma.c
@@ -211,11 +211,15 @@ static void dw_spi_dma_exit(struct dw_spi *dws)
 	if (dws->txchan) {
 		dmaengine_terminate_sync(dws->txchan);
 		dma_release_channel(dws->txchan);
+		dws->txchan = NULL;
+		dws->ctlr->dma_tx = NULL;
 	}
 
 	if (dws->rxchan) {
 		dmaengine_terminate_sync(dws->rxchan);
 		dma_release_channel(dws->rxchan);
+		dws->rxchan = NULL;
+		dws->ctlr->dma_rx = NULL;
 	}
 }
 
@@ -249,6 +253,10 @@ static bool dw_spi_can_dma(struct spi_controller *ctlr,
 	enum dma_slave_buswidth dma_bus_width;
 	u8 n_bytes = roundup_pow_of_two(BITS_TO_BYTES(xfer->bits_per_word));
 
+	/* Channels are acquired at runtime and may be unavailable */
+	if (!dws->txchan || !dws->rxchan)
+		return false;
+
 	if (xfer->len <= dws->fifo_len)
 		return false;
 
-- 
2.25.1

[PATCH v1 2/2] spi: dw: Add DMA support for enhanced memory operations

[Changhuang Liang]

Implement DMA support for enhanced SPI memory operations. On some
platforms, such as JHB100, only one DMA channel is available for the
enhanced SPI controller, so the channel is allocated dynamically based
on the transfer direction.

Add enhanced memory DMA callbacks (init/exit, setup, can_dma, transfer)
to struct dw_spi_dma_ops and hook them into the generic DMA operations.
The channel is requested per operation and released afterwards.

In dw_spi_exec_enh_mem_op(), record the direction in dws->tx_dir and use
the DMA path when a channel is available, can_dma_enh_mem() accepts the
transfer, and the transfer exceeds the FIFO length; otherwise fall back
to the interrupt path.

Rework dw_spi_dma_wait_tx_done() to take an explicit speed_hz, add one
to TXFLR for the word possibly left in the shift register, and compute
the delay in ns or us, dropping the dependency on the xfer pointer.

Also move the udelay(5) workaround into dw_spi_enh_write_cmd_addr() so
it applies to both paths, and enable the generic DMA setup in
dw_spi_jhb100_init().

Signed-off-by: Changhuang Liang <changhuang.liang@starfivetech.com>
---
 drivers/spi/spi-dw-core.c | 107 ++++++++++-----
 drivers/spi/spi-dw-dma.c  | 281 +++++++++++++++++++++++++++++++++++++-
 drivers/spi/spi-dw-mmio.c |   2 +
 drivers/spi/spi-dw.h      |  10 ++
 4 files changed, 357 insertions(+), 43 deletions(-)

diff --git a/drivers/spi/spi-dw-core.c b/drivers/spi/spi-dw-core.c
index 04a5b48373e1..6da3812f72f6 100644
--- a/drivers/spi/spi-dw-core.c
+++ b/drivers/spi/spi-dw-core.c
@@ -970,6 +970,15 @@ static void dw_spi_enh_write_cmd_addr(struct dw_spi *dws, const struct spi_mem_o
 
 		dw_spi_set_cs(mem->spi, false);
 	}
+
+	/*
+	 * FIXME: The exact reason for this delay is not fully understood,
+	 * but empirical testing shows it significantly improves the stability
+	 * of read/write operations. Without this delay, occasional transfer
+	 * errors or timeouts may occur under certain conditions.
+	 * Keeping it as a safeguard based on practical validation.
+	 */
+	udelay(5);
 }
 
 static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op *op)
@@ -981,6 +990,8 @@ static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op *
 	unsigned long long ms;
 	int ret;
 
+	dws->dma_mapped = false;
+
 	switch (op->data.buswidth) {
 	case 0:
 	case 1:
@@ -1004,10 +1015,13 @@ static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op *
 	cfg.dfs = 8;
 	cfg.freq = clamp(op->max_freq, 0U, dws->max_mem_freq);
 	cfg.ndf = op->data.nbytes;
-	if (op->data.dir == SPI_MEM_DATA_IN)
+	if (op->data.dir == SPI_MEM_DATA_IN) {
 		cfg.tmode = DW_SPI_CTRLR0_TMOD_RO;
-	else
+		dws->tx_dir = false;
+	} else {
 		cfg.tmode = DW_SPI_CTRLR0_TMOD_TO;
+		dws->tx_dir = true;
+	}
 
 	if (op->data.buswidth == op->addr.buswidth &&
 	    op->data.buswidth == op->cmd.buswidth)
@@ -1043,47 +1057,68 @@ static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op *
 		}
 	}
 
-	dw_spi_enh_write_cmd_addr(dws, op, mem);
-
-	/*
-	 * FIXME: The exact reason for this delay is not fully understood,
-	 * but empirical testing shows it significantly improves the stability
-	 * of read/write operations. Without this delay, occasional transfer
-	 * errors or timeouts may occur under certain conditions.
-	 * Keeping it as a safeguard based on practical validation.
-	 */
-	udelay(5);
+	if (dws->dma_ops && dws->dma_ops->dma_enh_mem_init &&
+	    dws->dma_ops->can_dma_enh_mem && op->data.nbytes > dws->fifo_len) {
+		ret = dws->dma_ops->dma_enh_mem_init(ctlr->dev.parent, dws);
+		if (ret) {
+			dev_dbg(&ctlr->dev, "DMA enh mem init failed (%d)\n", ret);
+		} else if (!dws->dma_ops->can_dma_enh_mem(ctlr)) {
+			/* Not worth a DMA transfer: give the channel back. */
+			dws->dma_ops->dma_enh_mem_exit(dws);
+		} else {
+			ret = dws->dma_ops->dma_enh_mem_setup(dws);
+			if (ret) {
+				/* fall back to the PIO/IRQ path */
+				dws->dma_ops->dma_enh_mem_exit(dws);
+				dev_err(&ctlr->dev, "DMA enh mem setup failed (%d)\n", ret);
+			} else {
+				dws->dma_mapped = true;
+			}
+		}
+	}
 
-	dw_spi_enh_irq_setup(dws);
+	if (dws->dma_mapped) {
+		dw_spi_enh_write_cmd_addr(dws, op, mem);
 
-	/* Use timeout calculation from spi_transfer_wait() */
-	ms = 8LL * MSEC_PER_SEC * (dws->rx_len ? dws->rx_len : dws->tx_len);
-	do_div(ms, dws->current_freq);
+		ret = dws->dma_ops->dma_enh_mem_transfer(mem, op);
 
-	/*
-	 * Increase it twice and add 200 ms tolerance, use
-	 * predefined maximum in case of overflow.
-	 */
-	ms += ms + 200;
-	if (ms > UINT_MAX)
-		ms = UINT_MAX;
+		dws->dma_ops->dma_enh_mem_exit(dws);
 
-	ms = wait_for_completion_timeout(&ctlr->xfer_completion,
-					 msecs_to_jiffies(ms));
-	if (ms == 0) {
-		dw_spi_mask_intr(dws, 0xff);
-		synchronize_irq(dws->irq);
-		dws->rx = NULL;
-		dws->tx = NULL;
-		dws->rx_len = 0;
-		dws->tx_len = 0;
 		dw_spi_stop_mem_op(dws, mem->spi);
-		return -EIO;
-	}
+	} else {
+		dw_spi_enh_write_cmd_addr(dws, op, mem);
 
-	ret = dw_spi_wait_mem_op_done(dws);
+		dw_spi_enh_irq_setup(dws);
 
-	dw_spi_stop_mem_op(dws, mem->spi);
+		/* Use timeout calculation from spi_transfer_wait() */
+		ms = 8LL * MSEC_PER_SEC * (dws->rx_len ? dws->rx_len : dws->tx_len);
+		do_div(ms, dws->current_freq);
+
+		/*
+		 * Increase it twice and add 200 ms tolerance, use
+		 * predefined maximum in case of overflow.
+		 */
+		ms += ms + 200;
+		if (ms > UINT_MAX)
+			ms = UINT_MAX;
+
+		ms = wait_for_completion_timeout(&ctlr->xfer_completion,
+						 msecs_to_jiffies(ms));
+		if (ms == 0) {
+			dw_spi_mask_intr(dws, 0xff);
+			synchronize_irq(dws->irq);
+			dws->rx = NULL;
+			dws->tx = NULL;
+			dws->rx_len = 0;
+			dws->tx_len = 0;
+			dw_spi_stop_mem_op(dws, mem->spi);
+			return -EIO;
+		}
+
+		ret = dw_spi_wait_mem_op_done(dws);
+
+		dw_spi_stop_mem_op(dws, mem->spi);
+	}
 
 	return ret;
 }
diff --git a/drivers/spi/spi-dw-dma.c b/drivers/spi/spi-dw-dma.c
index 022eb4fae33d..446c8b1c2dd1 100644
--- a/drivers/spi/spi-dw-dma.c
+++ b/drivers/spi/spi-dw-dma.c
@@ -294,19 +294,28 @@ static inline bool dw_spi_dma_tx_busy(struct dw_spi *dws)
 		(DW_SPI_SR_BUSY | DW_SPI_SR_TF_EMPT)) != DW_SPI_SR_TF_EMPT;
 }
 
-static int dw_spi_dma_wait_tx_done(struct dw_spi *dws,
-				   struct spi_transfer *xfer)
+static int dw_spi_dma_wait_tx_done(struct dw_spi *dws, u32 speed_hz)
 {
 	int retry = DW_SPI_WAIT_RETRIES;
 	struct spi_delay delay;
+	unsigned long ns, us;
 	u32 nents;
 
-	nents = dw_readl(dws, DW_SPI_TXFLR);
-	delay.unit = SPI_DELAY_UNIT_SCK;
-	delay.value = nents * dws->n_bytes * BITS_PER_BYTE;
+	/* Account for the word that may still be in the shift register */
+	nents = dw_readl(dws, DW_SPI_TXFLR) + 1;
+	ns = DIV_ROUND_UP(NSEC_PER_SEC, speed_hz) * nents *
+	     dws->n_bytes * BITS_PER_BYTE;
+	if (ns <= NSEC_PER_USEC) {
+		delay.unit = SPI_DELAY_UNIT_NSECS;
+		delay.value = ns;
+	} else {
+		us = DIV_ROUND_UP(ns, NSEC_PER_USEC);
+		delay.unit = SPI_DELAY_UNIT_USECS;
+		delay.value = clamp_val(us, 0, USHRT_MAX);
+	}
 
 	while (dw_spi_dma_tx_busy(dws) && retry--)
-		spi_delay_exec(&delay, xfer);
+		spi_delay_exec(&delay, NULL);
 
 	if (retry < 0) {
 		dev_err(&dws->ctlr->dev, "Tx hanged up\n");
@@ -667,7 +676,7 @@ static int dw_spi_dma_transfer(struct dw_spi *dws, struct spi_transfer *xfer)
 		return ret;
 
 	if (dws->ctlr->cur_msg->status == -EINPROGRESS) {
-		ret = dw_spi_dma_wait_tx_done(dws, xfer);
+		ret = dw_spi_dma_wait_tx_done(dws, xfer->effective_speed_hz);
 		if (ret)
 			return ret;
 	}
@@ -690,6 +699,258 @@ static void dw_spi_dma_stop(struct dw_spi *dws)
 	}
 }
 
+static int dw_spi_enh_mem_dma_caps_init(struct dw_spi *dws)
+{
+	struct dma_slave_caps caps;
+	int ret;
+
+	if (dws->tx_dir) {
+		ret = dma_get_slave_caps(dws->txchan, &caps);
+		if (ret)
+			return ret;
+
+		if (!(caps.directions & BIT(DMA_MEM_TO_DEV)))
+			return -ENXIO;
+
+		dws->dma_sg_burst = caps.max_sg_burst;
+		dws->dma_addr_widths = caps.dst_addr_widths;
+	} else {
+		ret = dma_get_slave_caps(dws->rxchan, &caps);
+		if (ret)
+			return ret;
+
+		if (!(caps.directions & BIT(DMA_DEV_TO_MEM)))
+			return -ENXIO;
+
+		dws->dma_sg_burst = caps.max_sg_burst;
+		dws->dma_addr_widths = caps.src_addr_widths;
+	}
+
+	return 0;
+}
+
+static void dw_spi_enh_mem_dma_maxburst_init(struct dw_spi *dws)
+{
+	struct dma_slave_caps caps;
+	u32 max_burst, def_burst;
+	int ret;
+
+	def_burst = dws->fifo_len / 2;
+
+	if (dws->tx_dir) {
+		ret = dma_get_slave_caps(dws->txchan, &caps);
+		max_burst = (!ret && caps.max_burst) ? caps.max_burst
+						     : DW_SPI_TX_BURST_LEVEL;
+		dws->txburst = min(max_burst, def_burst);
+		dw_writel(dws, DW_SPI_DMATDLR, dws->txburst);
+	} else {
+		ret = dma_get_slave_caps(dws->rxchan, &caps);
+		max_burst = (!ret && caps.max_burst) ? caps.max_burst
+						     : DW_SPI_RX_BURST_LEVEL;
+		dws->rxburst = min(max_burst, def_burst);
+		dw_writel(dws, DW_SPI_DMARDLR, dws->rxburst - 1);
+	}
+}
+
+static int dw_spi_enh_mem_dma_init_generic(struct device *dev, struct dw_spi *dws)
+{
+	int ret;
+
+	if (dws->tx_dir) {
+		if (dws->txchan)
+			return -EBUSY;
+
+		dws->txchan = dma_request_chan(dev, "tx");
+		if (IS_ERR(dws->txchan)) {
+			ret = PTR_ERR(dws->txchan);
+			dws->txchan = NULL;
+			goto err_exit;
+		}
+	} else {
+		if (dws->rxchan)
+			return -EBUSY;
+
+		dws->rxchan = dma_request_chan(dev, "rx");
+		if (IS_ERR(dws->rxchan)) {
+			ret = PTR_ERR(dws->rxchan);
+			dws->rxchan = NULL;
+			goto err_exit;
+		}
+	}
+
+	init_completion(&dws->dma_completion);
+
+	ret = dw_spi_enh_mem_dma_caps_init(dws);
+	if (ret)
+		goto free_txrxchan;
+
+	dw_spi_enh_mem_dma_maxburst_init(dws);
+
+	if (dws->tx_dir)
+		dws->ctlr->dma_tx = dws->txchan;
+	else
+		dws->ctlr->dma_rx = dws->rxchan;
+
+	return 0;
+
+free_txrxchan:
+	if (dws->tx_dir) {
+		dma_release_channel(dws->txchan);
+		dws->txchan = NULL;
+	} else {
+		dma_release_channel(dws->rxchan);
+		dws->rxchan = NULL;
+	}
+err_exit:
+	return ret;
+}
+
+static void dw_spi_enh_mem_dma_exit(struct dw_spi *dws)
+{
+	if (dws->tx_dir) {
+		if (!dws->txchan)
+			return;
+
+		dmaengine_terminate_sync(dws->txchan);
+		dma_release_channel(dws->txchan);
+		dws->txchan = NULL;
+		dws->ctlr->dma_tx = NULL;
+	} else {
+		if (!dws->rxchan)
+			return;
+
+		dmaengine_terminate_sync(dws->rxchan);
+		dma_release_channel(dws->rxchan);
+		dws->rxchan = NULL;
+		dws->ctlr->dma_rx = NULL;
+	}
+}
+
+static int dw_spi_enh_mem_dma_setup(struct dw_spi *dws)
+{
+	u16 dma_ctrl, level;
+	int ret;
+
+	/* Setup DMA channels */
+	if (dws->tx_dir) {
+		ret = dw_spi_dma_config_tx(dws);
+		if (ret)
+			return ret;
+
+		dma_ctrl = DW_SPI_DMACR_TDMAE;
+	} else {
+		ret = dw_spi_dma_config_rx(dws);
+		if (ret)
+			return ret;
+
+		dma_ctrl = DW_SPI_DMACR_RDMAE;
+	}
+
+	dw_writel(dws, DW_SPI_DMACR, dma_ctrl);
+
+	reinit_completion(&dws->dma_completion);
+
+	level = min_t(unsigned int, dws->fifo_len / 2, dws->tx_len);
+	dw_writel(dws, DW_SPI_TXFTLR, level);
+
+	level = min_t(unsigned int, dws->fifo_len / 2, dws->rx_len);
+	dw_writel(dws, DW_SPI_RXFTLR, level ? level - 1 : 0);
+
+	return 0;
+}
+
+static bool dw_spi_enh_mem_can_dma(struct spi_controller *ctlr)
+{
+	struct dw_spi *dws = spi_controller_get_devdata(ctlr);
+	enum dma_slave_buswidth dma_bus_width;
+
+	if (dws->tx_dir) {
+		if (dws->tx_len <= dws->fifo_len)
+			return false;
+	} else {
+		if (dws->rx_len <= dws->fifo_len)
+			return false;
+	}
+
+	dma_bus_width = dw_spi_dma_convert_width(dws->n_bytes);
+
+	return dws->dma_addr_widths & BIT(dma_bus_width);
+}
+
+static void dw_spi_enh_mem_dma_stop(struct dw_spi *dws)
+{
+	if (dws->tx_dir) {
+		if (test_bit(DW_SPI_TX_BUSY, &dws->dma_chan_busy)) {
+			dmaengine_terminate_sync(dws->txchan);
+			clear_bit(DW_SPI_TX_BUSY, &dws->dma_chan_busy);
+		}
+	} else {
+		if (test_bit(DW_SPI_RX_BUSY, &dws->dma_chan_busy)) {
+			dmaengine_terminate_sync(dws->rxchan);
+			clear_bit(DW_SPI_RX_BUSY, &dws->dma_chan_busy);
+		}
+	}
+}
+
+static int dw_spi_enh_mem_dma_transfer(struct spi_mem *mem, const struct spi_mem_op *op)
+{
+	struct spi_controller *ctlr = mem->spi->controller;
+	struct dw_spi *dws = spi_controller_get_devdata(ctlr);
+	struct sg_table sgt;
+	int ret;
+
+	if (dws->tx_dir) {
+		ret = spi_controller_dma_map_mem_op_data(ctlr, op, &sgt);
+		if (ret)
+			goto err_clear_dmac;
+
+		ret = dw_spi_dma_submit_tx(dws, sgt.sgl, sgt.nents);
+		if (ret)
+			goto err_unmap;
+
+		dma_async_issue_pending(dws->txchan);
+
+		ret = dw_spi_dma_wait(dws, dws->tx_len, dws->current_freq);
+		if (ret)
+			goto err_terminate;
+
+		ret = dw_spi_dma_wait_tx_done(dws, dws->current_freq);
+		if (ret)
+			goto err_terminate;
+
+		goto err_unmap;
+	} else {
+		ret = spi_controller_dma_map_mem_op_data(ctlr, op, &sgt);
+		if (ret)
+			goto err_clear_dmac;
+
+		ret = dw_spi_dma_submit_rx(dws, sgt.sgl, sgt.nents);
+		if (ret)
+			goto err_unmap;
+
+		dma_async_issue_pending(dws->rxchan);
+
+		ret = dw_spi_dma_wait(dws, dws->rx_len, dws->current_freq);
+		if (ret)
+			goto err_terminate;
+
+		ret = dw_spi_dma_wait_rx_done(dws);
+		if (ret)
+			goto err_terminate;
+
+		goto err_unmap;
+	}
+
+err_terminate:
+	dw_spi_enh_mem_dma_stop(dws);
+err_unmap:
+	spi_controller_dma_unmap_mem_op_data(ctlr, op, &sgt);
+err_clear_dmac:
+	dw_writel(dws, DW_SPI_DMACR, 0);
+
+	return ret;
+}
+
 static const struct dw_spi_dma_ops dw_spi_dma_mfld_ops = {
 	.dma_init	= dw_spi_dma_init_mfld,
 	.dma_exit	= dw_spi_dma_exit,
@@ -712,6 +973,12 @@ static const struct dw_spi_dma_ops dw_spi_dma_generic_ops = {
 	.can_dma	= dw_spi_can_dma,
 	.dma_transfer	= dw_spi_dma_transfer,
 	.dma_stop	= dw_spi_dma_stop,
+
+	.dma_enh_mem_init	= dw_spi_enh_mem_dma_init_generic,
+	.dma_enh_mem_exit	= dw_spi_enh_mem_dma_exit,
+	.dma_enh_mem_setup	= dw_spi_enh_mem_dma_setup,
+	.can_dma_enh_mem	= dw_spi_enh_mem_can_dma,
+	.dma_enh_mem_transfer	= dw_spi_enh_mem_dma_transfer,
 };
 
 void dw_spi_dma_setup_generic(struct dw_spi *dws)
diff --git a/drivers/spi/spi-dw-mmio.c b/drivers/spi/spi-dw-mmio.c
index 8cdb0351605b..9c70947c969c 100644
--- a/drivers/spi/spi-dw-mmio.c
+++ b/drivers/spi/spi-dw-mmio.c
@@ -363,6 +363,8 @@ static int dw_spi_jhb100_init(struct platform_device *pdev,
 
 	dw_spi_jhb100_mask_intr(&dwsmmio->dws, 0xff);
 
+	dw_spi_dma_setup_generic(&dwsmmio->dws);
+
 	return 0;
 }
 
diff --git a/drivers/spi/spi-dw.h b/drivers/spi/spi-dw.h
index e62f919f1188..da603c36d5d4 100644
--- a/drivers/spi/spi-dw.h
+++ b/drivers/spi/spi-dw.h
@@ -183,6 +183,13 @@ struct dw_spi_dma_ops {
 			struct spi_transfer *xfer);
 	int (*dma_transfer)(struct dw_spi *dws, struct spi_transfer *xfer);
 	void (*dma_stop)(struct dw_spi *dws);
+
+	/* enh mem dma ops */
+	int (*dma_enh_mem_init)(struct device *dev, struct dw_spi *dws);
+	void (*dma_enh_mem_exit)(struct dw_spi *dws);
+	int (*dma_enh_mem_setup)(struct dw_spi *dws);
+	bool (*can_dma_enh_mem)(struct spi_controller *ctlr);
+	int (*dma_enh_mem_transfer)(struct spi_mem *mem, const struct spi_mem_op *op);
 };
 
 struct dw_spi {
@@ -227,6 +234,9 @@ struct dw_spi {
 	u32			txburst;
 	struct dma_chan		*rxchan;
 	u32			rxburst;
+
+	bool			tx_dir;
+
 	u32			dma_sg_burst;
 	u32			dma_addr_widths;
 	unsigned long		dma_chan_busy;
-- 
2.25.1

Re: [PATCH v1 1/2] spi: dw: Request DMA channels at runtime instead of probe

[Mark Brown]

[-- Attachment #1: Type: text/plain, Size: 1485 bytes --]

On Wed, Sep 23, 2026 at 02:57:04AM -0700, Changhuang Liang wrote:
> The DW SPI controller requests its DMA channels in
> dw_spi_add_controller(), which ties them up for the entire lifetime of
> the controller. Even a controller that is only ever used for standard
> SPI transfers and never for enhanced SPI transfers holds the channels
> exclusively from probe onwards, preventing them from being shared with
> other users.

> +static int dw_spi_prepare_hardware(struct spi_controller *ctlr)
> +{
> +	struct dw_spi *dws = spi_controller_get_devdata(ctlr);
> +	int ret;
> +
> +	if (!ctlr->can_dma)
> +		return 0;
> +
> +	ret = dws->dma_ops->dma_init(ctlr->dev.parent, dws);
> +	if (ret) {
> +		/*
> +		 * DMA is optional: fall back to the PIO/IRQ path instead of
> +		 * failing the message. Use dev_dbg() since this may happen
> +		 * on every prepare.
> +		 */
> +		dev_dbg(&ctlr->dev, "DMA init failed (%d), using PIO\n", ret);
> +
> +		return 0;
> +	}
> +
> +	return 0;
> +}

This seems like a lot of overhead for every existing controller with
DMA, we'll have to go through the request/release cycle whenever DMA
gets used which feels like a bunch of overhead for a hot path -
especially in the fast path in spi_sync().  If some platforms need it
they should be able to opt into it rather than forcing it on every
single platform.

> +static int dw_spi_unprepare_hardware(struct spi_controller *ctlr)
> +{

Adding this also causes overhead since we do the unprepare in the
thread.

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 488 bytes --]

Re: [PATCH v1 2/2] spi: dw: Add DMA support for enhanced memory operations

[Mark Brown]

[-- Attachment #1: Type: text/plain, Size: 5843 bytes --]

On Wed, Sep 23, 2026 at 02:57:05AM -0700, Changhuang Liang wrote:
> Implement DMA support for enhanced SPI memory operations. On some
> platforms, such as JHB100, only one DMA channel is available for the
> enhanced SPI controller, so the channel is allocated dynamically based
> on the transfer direction.
> 
> Add enhanced memory DMA callbacks (init/exit, setup, can_dma, transfer)
> to struct dw_spi_dma_ops and hook them into the generic DMA operations.
> The channel is requested per operation and released afterwards.
> 
> In dw_spi_exec_enh_mem_op(), record the direction in dws->tx_dir and use
> the DMA path when a channel is available, can_dma_enh_mem() accepts the
> transfer, and the transfer exceeds the FIFO length; otherwise fall back
> to the interrupt path.
> 
> Rework dw_spi_dma_wait_tx_done() to take an explicit speed_hz, add one
> to TXFLR for the word possibly left in the shift register, and compute
> the delay in ns or us, dropping the dependency on the xfer pointer.
> 
> Also move the udelay(5) workaround into dw_spi_enh_write_cmd_addr() so
> it applies to both paths, and enable the generic DMA setup in
> dw_spi_jhb100_init().
> 
> Signed-off-by: Changhuang Liang <changhuang.liang@starfivetech.com>
> ---
>  drivers/spi/spi-dw-core.c | 107 ++++++++++-----
>  drivers/spi/spi-dw-dma.c  | 281 +++++++++++++++++++++++++++++++++++++-
>  drivers/spi/spi-dw-mmio.c |   2 +
>  drivers/spi/spi-dw.h      |  10 ++
>  4 files changed, 357 insertions(+), 43 deletions(-)
> 
> diff --git a/drivers/spi/spi-dw-core.c b/drivers/spi/spi-dw-core.c
> index 04a5b48373e1..6da3812f72f6 100644
> --- a/drivers/spi/spi-dw-core.c
> +++ b/drivers/spi/spi-dw-core.c
> @@ -970,6 +970,15 @@ static void dw_spi_enh_write_cmd_addr(struct dw_spi *dws, const struct spi_mem_o
>  
>  		dw_spi_set_cs(mem->spi, false);
>  	}
> +
> +	/*
> +	 * FIXME: The exact reason for this delay is not fully understood,
> +	 * but empirical testing shows it significantly improves the stability
> +	 * of read/write operations. Without this delay, occasional transfer
> +	 * errors or timeouts may occur under certain conditions.
> +	 * Keeping it as a safeguard based on practical validation.
> +	 */
> +	udelay(5);
>  }
>  
>  static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op *op)
> @@ -981,6 +990,8 @@ static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op *
>  	unsigned long long ms;
>  	int ret;
>  
> +	dws->dma_mapped = false;
> +
>  	switch (op->data.buswidth) {
>  	case 0:
>  	case 1:
> @@ -1004,10 +1015,13 @@ static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op *
>  	cfg.dfs = 8;
>  	cfg.freq = clamp(op->max_freq, 0U, dws->max_mem_freq);
>  	cfg.ndf = op->data.nbytes;
> -	if (op->data.dir == SPI_MEM_DATA_IN)
> +	if (op->data.dir == SPI_MEM_DATA_IN) {
>  		cfg.tmode = DW_SPI_CTRLR0_TMOD_RO;
> -	else
> +		dws->tx_dir = false;
> +	} else {
>  		cfg.tmode = DW_SPI_CTRLR0_TMOD_TO;
> +		dws->tx_dir = true;
> +	}
>  
>  	if (op->data.buswidth == op->addr.buswidth &&
>  	    op->data.buswidth == op->cmd.buswidth)
> @@ -1043,47 +1057,68 @@ static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op *
>  		}
>  	}
>  
> -	dw_spi_enh_write_cmd_addr(dws, op, mem);
> -
> -	/*
> -	 * FIXME: The exact reason for this delay is not fully understood,
> -	 * but empirical testing shows it significantly improves the stability
> -	 * of read/write operations. Without this delay, occasional transfer
> -	 * errors or timeouts may occur under certain conditions.
> -	 * Keeping it as a safeguard based on practical validation.
> -	 */
> -	udelay(5);
> +	if (dws->dma_ops && dws->dma_ops->dma_enh_mem_init &&
> +	    dws->dma_ops->can_dma_enh_mem && op->data.nbytes > dws->fifo_len) {
> +		ret = dws->dma_ops->dma_enh_mem_init(ctlr->dev.parent, dws);
> +		if (ret) {
> +			dev_dbg(&ctlr->dev, "DMA enh mem init failed (%d)\n", ret);

This doesn't pay attention to the probe time detection of DMA, and
there's no probe time path for requesting the channels so we'll always
drop and request the channel.

> +	if (dws->dma_mapped) {
> +		dw_spi_enh_write_cmd_addr(dws, op, mem);
>  
> -	/* Use timeout calculation from spi_transfer_wait() */
> -	ms = 8LL * MSEC_PER_SEC * (dws->rx_len ? dws->rx_len : dws->tx_len);
> -	do_div(ms, dws->current_freq);
> +		ret = dws->dma_ops->dma_enh_mem_transfer(mem, op);
>  
> -	/*
> -	 * Increase it twice and add 200 ms tolerance, use
> -	 * predefined maximum in case of overflow.
> -	 */
> -	ms += ms + 200;
> -	if (ms > UINT_MAX)
> -		ms = UINT_MAX;
> +		dws->dma_ops->dma_enh_mem_exit(dws);

Do we check for errors from the hardware anywhere?

> +static int dw_spi_dma_wait_tx_done(struct dw_spi *dws, u32 speed_hz)
>  {
>  	int retry = DW_SPI_WAIT_RETRIES;
>  	struct spi_delay delay;
> +	unsigned long ns, us;
>  	u32 nents;
>  
> -	nents = dw_readl(dws, DW_SPI_TXFLR);
> -	delay.unit = SPI_DELAY_UNIT_SCK;
> -	delay.value = nents * dws->n_bytes * BITS_PER_BYTE;
> +	/* Account for the word that may still be in the shift register */
> +	nents = dw_readl(dws, DW_SPI_TXFLR) + 1;

This affects all DMA users and wasn't really explained, it should
probably be a separate patch.

> +static int dw_spi_enh_mem_dma_caps_init(struct dw_spi *dws)
> +{

> +static void dw_spi_enh_mem_dma_maxburst_init(struct dw_spi *dws)
> +{

These are very close to the non-enhanced versions, it would be better to
factor out the common bits.

> +static int dw_spi_enh_mem_dma_init_generic(struct device *dev, struct dw_spi *dws)
> +{
> +	int ret;
> +
> +	if (dws->tx_dir) {
> +		if (dws->txchan)
> +			return -EBUSY;

Surely if the driver already has a channel there's no error?

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 488 bytes --]

Re: [PATCH v1 1/2] spi: dw: Request DMA channels at runtime instead of probe

[Changhuang Liang]

Hi, Mark

Thanks for the review.

> On Wed, Sep 23, 2026 at 02:57:04AM -0700, Changhuang Liang wrote:
> > The DW SPI controller requests its DMA channels in
> > dw_spi_add_controller(), which ties them up for the entire lifetime of
> > the controller. Even a controller that is only ever used for standard
> > SPI transfers and never for enhanced SPI transfers holds the channels
> > exclusively from probe onwards, preventing them from being shared with
> > other users.
> 
> > +static int dw_spi_prepare_hardware(struct spi_controller *ctlr) {
> > +	struct dw_spi *dws = spi_controller_get_devdata(ctlr);
> > +	int ret;
> > +
> > +	if (!ctlr->can_dma)
> > +		return 0;
> > +
> > +	ret = dws->dma_ops->dma_init(ctlr->dev.parent, dws);
> > +	if (ret) {
> > +		/*
> > +		 * DMA is optional: fall back to the PIO/IRQ path instead of
> > +		 * failing the message. Use dev_dbg() since this may happen
> > +		 * on every prepare.
> > +		 */
> > +		dev_dbg(&ctlr->dev, "DMA init failed (%d), using PIO\n", ret);
> > +
> > +		return 0;
> > +	}
> > +
> > +	return 0;
> > +}
> 
> This seems like a lot of overhead for every existing controller with DMA, we'll
> have to go through the request/release cycle whenever DMA gets used which
> feels like a bunch of overhead for a hot path - especially in the fast path in
> spi_sync().  If some platforms need it they should be able to opt into it rather
> than forcing it on every single platform.
> 

I will reorganize this series, trying to avoid affecting the current flow.

> > +static int dw_spi_unprepare_hardware(struct spi_controller *ctlr) {
> 
> Adding this also causes overhead since we do the unprepare in the thread.

Best Regards,
Changhuang

Re: [PATCH v1 2/2] spi: dw: Add DMA support for enhanced memory operations

[Changhuang Liang]

Hi, Mark

Thanks for the review.

> On Wed, Sep 23, 2026 at 02:57:05AM -0700, Changhuang Liang wrote:
> > Implement DMA support for enhanced SPI memory operations. On some
> > platforms, such as JHB100, only one DMA channel is available for the
> > enhanced SPI controller, so the channel is allocated dynamically based
> > on the transfer direction.
> >
> > Add enhanced memory DMA callbacks (init/exit, setup, can_dma,
> > transfer) to struct dw_spi_dma_ops and hook them into the generic DMA
> operations.
> > The channel is requested per operation and released afterwards.
> >
> > In dw_spi_exec_enh_mem_op(), record the direction in dws->tx_dir and
> > use the DMA path when a channel is available, can_dma_enh_mem()
> > accepts the transfer, and the transfer exceeds the FIFO length;
> > otherwise fall back to the interrupt path.
> >
> > Rework dw_spi_dma_wait_tx_done() to take an explicit speed_hz, add one
> > to TXFLR for the word possibly left in the shift register, and compute
> > the delay in ns or us, dropping the dependency on the xfer pointer.
> >
> > Also move the udelay(5) workaround into dw_spi_enh_write_cmd_addr() so
> > it applies to both paths, and enable the generic DMA setup in
> > dw_spi_jhb100_init().
> >
> > Signed-off-by: Changhuang Liang <changhuang.liang@starfivetech.com>
> > ---
> >  drivers/spi/spi-dw-core.c | 107 ++++++++++-----
> > drivers/spi/spi-dw-dma.c  | 281
> +++++++++++++++++++++++++++++++++++++-
> >  drivers/spi/spi-dw-mmio.c |   2 +
> >  drivers/spi/spi-dw.h      |  10 ++
> >  4 files changed, 357 insertions(+), 43 deletions(-)
> >
> > diff --git a/drivers/spi/spi-dw-core.c b/drivers/spi/spi-dw-core.c
> > index 04a5b48373e1..6da3812f72f6 100644
> > --- a/drivers/spi/spi-dw-core.c
> > +++ b/drivers/spi/spi-dw-core.c
> > @@ -970,6 +970,15 @@ static void dw_spi_enh_write_cmd_addr(struct
> > dw_spi *dws, const struct spi_mem_o
> >
> >  		dw_spi_set_cs(mem->spi, false);
> >  	}
> > +
> > +	/*
> > +	 * FIXME: The exact reason for this delay is not fully understood,
> > +	 * but empirical testing shows it significantly improves the stability
> > +	 * of read/write operations. Without this delay, occasional transfer
> > +	 * errors or timeouts may occur under certain conditions.
> > +	 * Keeping it as a safeguard based on practical validation.
> > +	 */
> > +	udelay(5);
> >  }
> >
> >  static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct
> > spi_mem_op *op) @@ -981,6 +990,8 @@ static int
> dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op
> *
> >  	unsigned long long ms;
> >  	int ret;
> >
> > +	dws->dma_mapped = false;
> > +
> >  	switch (op->data.buswidth) {
> >  	case 0:
> >  	case 1:
> > @@ -1004,10 +1015,13 @@ static int dw_spi_exec_enh_mem_op(struct
> spi_mem *mem, const struct spi_mem_op *
> >  	cfg.dfs = 8;
> >  	cfg.freq = clamp(op->max_freq, 0U, dws->max_mem_freq);
> >  	cfg.ndf = op->data.nbytes;
> > -	if (op->data.dir == SPI_MEM_DATA_IN)
> > +	if (op->data.dir == SPI_MEM_DATA_IN) {
> >  		cfg.tmode = DW_SPI_CTRLR0_TMOD_RO;
> > -	else
> > +		dws->tx_dir = false;
> > +	} else {
> >  		cfg.tmode = DW_SPI_CTRLR0_TMOD_TO;
> > +		dws->tx_dir = true;
> > +	}
> >
> >  	if (op->data.buswidth == op->addr.buswidth &&
> >  	    op->data.buswidth == op->cmd.buswidth) @@ -1043,47 +1057,68
> @@
> > static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct
> spi_mem_op *
> >  		}
> >  	}
> >
> > -	dw_spi_enh_write_cmd_addr(dws, op, mem);
> > -
> > -	/*
> > -	 * FIXME: The exact reason for this delay is not fully understood,
> > -	 * but empirical testing shows it significantly improves the stability
> > -	 * of read/write operations. Without this delay, occasional transfer
> > -	 * errors or timeouts may occur under certain conditions.
> > -	 * Keeping it as a safeguard based on practical validation.
> > -	 */
> > -	udelay(5);
> > +	if (dws->dma_ops && dws->dma_ops->dma_enh_mem_init &&
> > +	    dws->dma_ops->can_dma_enh_mem && op->data.nbytes >
> dws->fifo_len) {
> > +		ret = dws->dma_ops->dma_enh_mem_init(ctlr->dev.parent, dws);
> > +		if (ret) {
> > +			dev_dbg(&ctlr->dev, "DMA enh mem init failed (%d)\n", ret);
> 
> This doesn't pay attention to the probe time detection of DMA, and there's no
> probe time path for requesting the channels so we'll always drop and request
> the channel.
> 

Yes, previously, considering platforms with only one DMA channel where the tx 
and rx channels can only request one at a time, the initial thought was to request 
them at runtime. I'll try to see if I can re-optimize it to request them during the 
probe stage and eliminate this repeated request/release overhead.

> > +	if (dws->dma_mapped) {
> > +		dw_spi_enh_write_cmd_addr(dws, op, mem);
> >
> > -	/* Use timeout calculation from spi_transfer_wait() */
> > -	ms = 8LL * MSEC_PER_SEC * (dws->rx_len ? dws->rx_len : dws->tx_len);
> > -	do_div(ms, dws->current_freq);
> > +		ret = dws->dma_ops->dma_enh_mem_transfer(mem, op);
> >
> > -	/*
> > -	 * Increase it twice and add 200 ms tolerance, use
> > -	 * predefined maximum in case of overflow.
> > -	 */
> > -	ms += ms + 200;
> > -	if (ms > UINT_MAX)
> > -		ms = UINT_MAX;
> > +		dws->dma_ops->dma_enh_mem_exit(dws);
> 
> Do we check for errors from the hardware anywhere?

Currently, the enhance SPI has clock stretching enabled, so there will be no 
FIFO errors. Therefore, it seems that only DMA errors need to be checked?

> > +static int dw_spi_dma_wait_tx_done(struct dw_spi *dws, u32 speed_hz)
> >  {
> >  	int retry = DW_SPI_WAIT_RETRIES;
> >  	struct spi_delay delay;
> > +	unsigned long ns, us;
> >  	u32 nents;
> >
> > -	nents = dw_readl(dws, DW_SPI_TXFLR);
> > -	delay.unit = SPI_DELAY_UNIT_SCK;
> > -	delay.value = nents * dws->n_bytes * BITS_PER_BYTE;
> > +	/* Account for the word that may still be in the shift register */
> > +	nents = dw_readl(dws, DW_SPI_TXFLR) + 1;
> 
> This affects all DMA users and wasn't really explained, it should probably be a
> separate patch.

OK

> > +static int dw_spi_enh_mem_dma_caps_init(struct dw_spi *dws) {
> 
> > +static void dw_spi_enh_mem_dma_maxburst_init(struct dw_spi *dws) {
> 
> These are very close to the non-enhanced versions, it would be better to
> factor out the common bits.

OK, I'll give it a try.

> > +static int dw_spi_enh_mem_dma_init_generic(struct device *dev, struct
> > +dw_spi *dws) {
> > +	int ret;
> > +
> > +	if (dws->tx_dir) {
> > +		if (dws->txchan)
> > +			return -EBUSY;
> 
> Surely if the driver already has a channel there's no error?

The initial idea here was to request the DMA channel at runtime. If the driver 
already has a channel, it would be considered that the previous DMA transfer 
has not yet finished.

Best Regards,
Changhuang