From: Jelly Jia <Jelly.Jia@cixtech.com>
To: vkoul@kernel.org, Frank.Li@kernel.org, robh@kernel.org,
krzk+dt@kernel.org, conor+dt@kernel.org, robin.murphy@arm.com
Cc: devicetree@vger.kernel.org, cix-kernel-upstream@cixtech.com,
dmaengine@vger.kernel.org, linux-arm-kernel@lists.infradead.org,
linux-kernel@vger.kernel.org, Jelly.Jia@cixtech.com
Subject: [PATCH v7 08/10] dmaengine: arm-dma350: Add slave and cyclic transfer support
Date: Tue, 17 Nov 2026 17:43:33 +0800 [thread overview]
Message-ID: <20261117094335.1393735-9-Jelly.Jia@cixtech.com> (raw)
In-Reply-To: <20261117094335.1393735-1-Jelly.Jia@cixtech.com>
Add peripheral scatter-gather and cyclic transfers with descriptor-owned
resource mappings, coherent linked commands and hardware-based residue.
Snapshot channel configuration under the lock, validate per-channel
capabilities, and reject slave preparation without a valid request line.
Describe peripheral hardware commands with named little-endian fields
and fill them directly in the coherent pool. Decode only when loading
the first peripheral command into MMIO or reading saved command sizes.
Keep CPU-endian memcpy/memset commands unchanged. Single-entry slave SG
uses embedded command storage without an extra pool allocation.
Derive SG lengths from the command sizes rather than a separate array.
Rely on the final non-relaxed channel-enable writel for publication
ordering rather than a prep-time DMA write barrier.
Use scoped locking and consistent device_config/device_caps names. Use
managed OF DMA controller registration and clear request/configuration
when channel resources are freed, retaining them across termination.
Signed-off-by: Jelly Jia <Jelly.Jia@cixtech.com>
---
drivers/dma/arm-dma350.c | 765 +++++++++++++++++++++++++++++++++++++--
1 file changed, 736 insertions(+), 29 deletions(-)
diff --git a/drivers/dma/arm-dma350.c b/drivers/dma/arm-dma350.c
index 465a8b9b722a..7b46b4fdcf9c 100644
--- a/drivers/dma/arm-dma350.c
+++ b/drivers/dma/arm-dma350.c
@@ -3,14 +3,23 @@
// Arm DMA-350 driver
#include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/byteorder/generic.h>
#include <linux/cleanup.h>
-#include <linux/dmaengine.h>
#include <linux/dma-mapping.h>
+#include <linux/dmaengine.h>
+#include <linux/dmapool.h>
#include <linux/io.h>
#include <linux/iopoll.h>
-#include <linux/of.h>
+#include <linux/limits.h>
#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_dma.h>
+#include <linux/overflow.h>
#include <linux/platform_device.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/string.h>
#include "dmaengine.h"
#include "virt-dma.h"
@@ -106,6 +115,10 @@
#define CH_FILLVAL 0x38
#define CH_SRCTRIGINCFG 0x4c
#define CH_DESTRIGINCFG 0x50
+#define CH_TRIGINCFG_BLKSIZE GENMASK(23, 16)
+#define CH_TRIGINCFG_MODE GENMASK(11, 10)
+#define CH_TRIGINCFG_TYPE GENMASK(9, 8)
+#define CH_TRIGINCFG_SEL GENMASK(7, 0)
#define CH_LINKATTR 0x70
#define CH_LINK_SHAREATTR GENMASK(9, 8)
#define CH_LINK_MEMATTR GENMASK(7, 0)
@@ -163,6 +176,18 @@ enum ch_ctrl_xtype {
CH_CTRL_XTYPE_FILL = 3
};
+enum ch_trigincfg_mode {
+ CH_TRIGINCFG_MODE_COMMAND = 0,
+ CH_TRIGINCFG_MODE_DMA_FC = 2,
+ CH_TRIGINCFG_MODE_PERIPH_FC = 3
+};
+
+enum ch_trigincfg_type {
+ CH_TRIGINCFG_TYPE_SW = 0,
+ CH_TRIGINCFG_TYPE_HW = 2,
+ CH_TRIGINCFG_TYPE_INTERNAL = 3
+};
+
enum ch_cfg_shareattr {
SHAREATTR_NSH = 0,
SHAREATTR_OSH = 2,
@@ -175,11 +200,44 @@ enum ch_cfg_memattr {
MEMATTR_WB = 0xff
};
+/* Register-update words in the order selected by the slave header. */
+struct d350_slave_cmd {
+ __le32 header;
+ __le32 ctrl;
+ __le64 src_addr;
+ __le64 dst_addr;
+ __le32 xsize;
+ __le32 xsizehi;
+ __le32 src_transcfg;
+ __le32 dst_transcfg;
+ __le32 xaddrinc;
+ __le32 trigincfg;
+ __le64 link_addr;
+} __packed;
+
+struct d350_hw_cmd {
+ struct d350_slave_cmd *cmd;
+ dma_addr_t dma;
+};
+
struct d350_desc {
struct virt_dma_desc vd;
- u32 command[16];
+ union {
+ u32 command[16];
+ struct d350_slave_cmd slave;
+ };
+ enum dma_transfer_direction direction;
+ dma_addr_t resource_dma;
+ size_t resource_size;
+ enum dma_data_direction resource_dir;
+ bool resource_mapped;
+ struct d350_hw_cmd *cmds;
+ size_t ncmds;
size_t bytes;
+ size_t period_len;
+ size_t periods;
u8 tsz;
+ bool cyclic;
};
struct d350_chan {
@@ -190,14 +248,17 @@ struct d350_chan {
enum dma_status status;
dma_cookie_t cookie;
u32 residue;
+ u32 req;
u8 tsz;
bool has_trig;
bool has_wrap;
bool coherent;
+ struct dma_slave_config sconfig;
};
struct d350 {
struct dma_device dma;
+ struct dma_pool *cmd_pool;
int nchan;
int nreq;
struct d350_chan channels[] __counted_by(nchan);
@@ -208,14 +269,447 @@ static inline struct d350_chan *to_d350_chan(struct dma_chan *chan)
return container_of(chan, struct d350_chan, vc.chan);
}
+static inline struct d350 *d350_chan_to_dmac(struct d350_chan *dch)
+{
+ return container_of(dch->vc.chan.device, struct d350, dma);
+}
+
static inline struct d350_desc *to_d350_desc(struct virt_dma_desc *vd)
{
return container_of(vd, struct d350_desc, vd);
}
+static void d350_unmap_resource(struct device *dev, struct d350_desc *desc)
+{
+ if (!desc->resource_mapped)
+ return;
+
+ dma_unmap_resource(dev, desc->resource_dma, desc->resource_size,
+ desc->resource_dir, 0);
+ desc->resource_mapped = false;
+}
+
+static void d350_free_cmds(struct d350 *dmac, struct d350_desc *desc)
+{
+ size_t i;
+
+ for (i = 0; i < desc->ncmds; i++) {
+ if (desc->cmds[i].cmd)
+ dma_pool_free(dmac->cmd_pool, desc->cmds[i].cmd,
+ desc->cmds[i].dma);
+ }
+
+ kfree(desc->cmds);
+}
+
static void d350_desc_free(struct virt_dma_desc *vd)
{
- kfree(to_d350_desc(vd));
+ struct d350_desc *desc = to_d350_desc(vd);
+ struct device *dev = vd->tx.chan->device->dev;
+ struct d350 *dmac = d350_chan_to_dmac(to_d350_chan(vd->tx.chan));
+
+ d350_unmap_resource(dev, desc);
+ d350_free_cmds(dmac, desc);
+ kfree(desc);
+}
+
+static int d350_alloc_cmds(struct dma_chan *dchan, struct d350_desc *desc,
+ size_t ncmds)
+{
+ struct d350 *dmac = d350_chan_to_dmac(to_d350_chan(dchan));
+ size_t i;
+
+ desc->cmds = kcalloc(ncmds, sizeof(*desc->cmds), GFP_NOWAIT);
+ if (!desc->cmds)
+ return -ENOMEM;
+
+ desc->ncmds = ncmds;
+ for (i = 0; i < ncmds; i++) {
+ desc->cmds[i].cmd = dma_pool_alloc(dmac->cmd_pool, GFP_NOWAIT,
+ &desc->cmds[i].dma);
+ if (!desc->cmds[i].cmd) {
+ d350_free_cmds(dmac, desc);
+ desc->cmds = NULL;
+ desc->ncmds = 0;
+ return -ENOMEM;
+ }
+ }
+
+ return 0;
+}
+
+static dma_addr_t d350_map_resource(struct dma_chan *dchan,
+ struct d350_desc *desc,
+ phys_addr_t cpu_addr, size_t size,
+ enum dma_data_direction dir)
+{
+ struct device *dev = dchan->device->dev;
+ dma_addr_t dma_addr;
+
+ dma_addr = dma_map_resource(dev, cpu_addr, size, dir, 0);
+ if (dma_mapping_error(dev, dma_addr)) {
+ dev_err(dev, "map slave failed ch%u phys=%pa size=%zu dir=%d\n",
+ dchan->chan_id, &cpu_addr, size, dir);
+ return DMA_MAPPING_ERROR;
+ }
+
+ desc->resource_dma = dma_addr;
+ desc->resource_size = size;
+ desc->resource_dir = dir;
+ desc->resource_mapped = true;
+
+ return dma_addr;
+}
+
+static bool d350_buswidth_supported(u32 widths, enum dma_slave_buswidth width)
+{
+ return width < BITS_PER_TYPE(widths) && (widths & BIT(width));
+}
+
+static bool d350_slave_supported(struct d350_chan *dch)
+{
+ return dch->has_trig && d350_chan_to_dmac(dch)->nreq > 0;
+}
+
+static u32 d350_chan_widths(struct d350_chan *dch)
+{
+ struct dma_device *dma = dch->vc.chan.device;
+ u32 widths = dma->src_addr_widths | dma->dst_addr_widths;
+
+ return widths & GENMASK(min_t(unsigned int, BIT(dch->tsz),
+ BITS_PER_TYPE(widths) - 1), 1);
+}
+
+static void d350_device_caps(struct dma_chan *chan, struct dma_slave_caps *caps)
+{
+ struct d350_chan *dch = to_d350_chan(chan);
+ u32 widths = d350_chan_widths(dch);
+
+ caps->src_addr_widths &= widths;
+ caps->dst_addr_widths &= widths;
+ if (!d350_slave_supported(dch))
+ caps->directions &= ~(BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM));
+}
+
+static int d350_check_slave_config(struct d350_chan *dch,
+ struct dma_slave_config *config)
+{
+ u32 maxburst = FIELD_MAX(CH_CFG_MAXBURSTLEN) + 1;
+ u32 widths = d350_chan_widths(dch);
+
+ if (!d350_slave_supported(dch))
+ return -EOPNOTSUPP;
+
+ /* The unused direction may leave its bus width unspecified. */
+ if ((config->src_addr_width != DMA_SLAVE_BUSWIDTH_UNDEFINED &&
+ !d350_buswidth_supported(widths, config->src_addr_width)) ||
+ (config->dst_addr_width != DMA_SLAVE_BUSWIDTH_UNDEFINED &&
+ !d350_buswidth_supported(widths, config->dst_addr_width)))
+ return -EINVAL;
+
+ if (config->src_maxburst > maxburst ||
+ config->dst_maxburst > maxburst)
+ return -EINVAL;
+
+ return 0;
+}
+
+static int d350_device_config(struct dma_chan *chan,
+ struct dma_slave_config *config)
+{
+ struct d350_chan *dch = to_d350_chan(chan);
+ struct dma_device *dma = chan->device;
+ int ret;
+
+ ret = d350_check_slave_config(dch, config);
+ if (ret) {
+ dev_err(dma->dev, "invalid slave configuration\n");
+ return ret;
+ }
+
+ guard(spinlock_irqsave)(&dch->vc.lock);
+
+ dch->sconfig = *config;
+
+ return 0;
+}
+
+static struct dma_chan *d350_of_xlate(struct of_phandle_args *dma_spec,
+ struct of_dma *ofdma)
+{
+ struct d350 *dmac = ofdma->of_dma_data;
+ struct d350_chan *dch;
+ struct dma_chan *chan;
+ u32 req;
+ int i;
+
+ if (dma_spec->args_count != 1) {
+ dev_err(dmac->dma.dev, "dma phandle must have one argument\n");
+ return NULL;
+ }
+
+ req = dma_spec->args[0];
+ if (req >= dmac->nreq) {
+ dev_err(dmac->dma.dev, "invalid DMA request %u, have %d\n",
+ req, dmac->nreq);
+ return NULL;
+ }
+
+ for (i = 0; i < dmac->nchan; i++) {
+ dch = &dmac->channels[i];
+ if (!dch->has_trig)
+ continue;
+
+ chan = dma_get_slave_channel(&dch->vc.chan);
+ if (chan) {
+ dch->req = req;
+ return chan;
+ }
+ }
+
+ dev_err(dmac->dma.dev, "no free channel with trigger support\n");
+
+ return NULL;
+}
+
+static u32 d350_device_transcfg(u32 maxburst)
+{
+ return FIELD_PREP(CH_CFG_MAXBURSTLEN, maxburst - 1) |
+ FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_OSH) |
+ FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_DEVICE);
+}
+
+static int d350_slave_params(struct d350_chan *dch,
+ enum dma_transfer_direction direction,
+ phys_addr_t *dev_cpu_addr,
+ enum dma_data_direction *dev_dir,
+ enum dma_slave_buswidth *width, u32 *maxburst)
+{
+ struct dma_slave_config config;
+ int ret;
+
+ scoped_guard(spinlock_irqsave, &dch->vc.lock)
+ config = dch->sconfig;
+
+ ret = d350_check_slave_config(dch, &config);
+ if (ret)
+ return ret;
+ if (dch->req >= d350_chan_to_dmac(dch)->nreq)
+ return -EINVAL;
+
+ if (direction == DMA_MEM_TO_DEV) {
+ *dev_cpu_addr = config.dst_addr;
+ *dev_dir = DMA_FROM_DEVICE;
+ *width = config.dst_addr_width;
+ *maxburst = config.dst_maxburst;
+ } else if (direction == DMA_DEV_TO_MEM) {
+ *dev_cpu_addr = config.src_addr;
+ *dev_dir = DMA_TO_DEVICE;
+ *width = config.src_addr_width;
+ *maxburst = config.src_maxburst;
+ } else {
+ return -EINVAL;
+ }
+
+ return *width && *maxburst ? 0 : -EINVAL;
+}
+
+static void d350_fill_slave_cmd(struct d350_chan *dch, struct d350_desc *desc,
+ struct d350_slave_cmd *cmd,
+ dma_addr_t mem, dma_addr_t dev_dma_addr,
+ size_t len, bool link, dma_addr_t link_addr,
+ enum dma_transfer_direction direction,
+ enum dma_slave_buswidth width, u32 maxburst,
+ enum ch_ctrl_donetype donetype)
+{
+ bool mem_to_dev = direction == DMA_MEM_TO_DEV;
+ dma_addr_t src_addr = mem_to_dev ? mem : dev_dma_addr;
+ dma_addr_t dst_addr = mem_to_dev ? dev_dma_addr : mem;
+ u16 xsize;
+ u16 xsizehi;
+ u32 devcfg;
+ u32 memcfg;
+ u32 trigcfg;
+ u32 header;
+ u32 ctrl;
+
+ desc->tsz = __ffs(width);
+ xsize = lower_16_bits(len >> desc->tsz);
+ xsizehi = upper_16_bits(len >> desc->tsz);
+ devcfg = d350_device_transcfg(maxburst);
+ memcfg = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC;
+
+ trigcfg = FIELD_PREP(CH_TRIGINCFG_BLKSIZE, maxburst - 1) |
+ FIELD_PREP(CH_TRIGINCFG_MODE, CH_TRIGINCFG_MODE_PERIPH_FC) |
+ FIELD_PREP(CH_TRIGINCFG_TYPE, CH_TRIGINCFG_TYPE_HW) |
+ FIELD_PREP(CH_TRIGINCFG_SEL, dch->req);
+
+ header = LINK_CTRL | LINK_SRCADDR | LINK_SRCADDRHI | LINK_DESADDR |
+ LINK_DESADDRHI | LINK_XSIZE | LINK_XSIZEHI | LINK_SRCTRANSCFG |
+ LINK_DESTRANSCFG | LINK_XADDRINC | LINK_LINKADDR |
+ LINK_LINKADDRHI |
+ (mem_to_dev ? LINK_DESTRIGINCFG : LINK_SRCTRIGINCFG);
+ ctrl = (mem_to_dev ? CH_CTRL_USEDESTRIGIN : CH_CTRL_USESRCTRIGIN) |
+ FIELD_PREP(CH_CTRL_TRANSIZE, desc->tsz) |
+ FIELD_PREP(CH_CTRL_XTYPE, CH_CTRL_XTYPE_CONTINUE) |
+ FIELD_PREP(CH_CTRL_DONETYPE, donetype);
+
+ cmd->header = cpu_to_le32(header);
+ cmd->ctrl = cpu_to_le32(ctrl);
+ cmd->src_addr = cpu_to_le64(src_addr);
+ cmd->dst_addr = cpu_to_le64(dst_addr);
+ cmd->xsize = cpu_to_le32(FIELD_PREP(CH_XY_SRC, xsize) |
+ FIELD_PREP(CH_XY_DES, xsize));
+ cmd->xsizehi = cpu_to_le32(FIELD_PREP(CH_XY_SRC, xsizehi) |
+ FIELD_PREP(CH_XY_DES, xsizehi));
+ cmd->src_transcfg = cpu_to_le32(mem_to_dev ? memcfg : devcfg);
+ cmd->dst_transcfg = cpu_to_le32(mem_to_dev ? devcfg : memcfg);
+ cmd->xaddrinc = cpu_to_le32(mem_to_dev ?
+ FIELD_PREP(CH_XY_SRC, 1) : FIELD_PREP(CH_XY_DES, 1));
+ cmd->trigincfg = cpu_to_le32(trigcfg);
+ cmd->link_addr = cpu_to_le64(link_addr | (link ? CH_LINKADDR_EN : 0));
+}
+
+static struct dma_async_tx_descriptor *
+d350_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
+ unsigned int sg_len,
+ enum dma_transfer_direction direction,
+ unsigned long flags, void *context)
+{
+ struct d350_chan *dch = to_d350_chan(dchan);
+ enum dma_data_direction dev_dir;
+ enum dma_slave_buswidth width;
+ struct d350_desc *desc;
+ phys_addr_t dev_cpu_addr;
+ dma_addr_t dev_dma_addr;
+ dma_addr_t mem;
+ struct scatterlist *sg;
+ u32 maxburst;
+ size_t len;
+ int i;
+
+ if (unlikely(!is_slave_direction(direction) || !sg_len))
+ return NULL;
+
+ if (d350_slave_params(dch, direction, &dev_cpu_addr, &dev_dir, &width,
+ &maxburst))
+ return NULL;
+
+ desc = kzalloc_obj(*desc, GFP_NOWAIT);
+ if (!desc)
+ return NULL;
+
+ desc->direction = direction;
+ dev_dma_addr = d350_map_resource(dchan, desc, dev_cpu_addr, width,
+ dev_dir);
+ if (dma_mapping_error(dchan->device->dev, dev_dma_addr))
+ goto err_free_desc;
+
+ if (sg_len > 1 && d350_alloc_cmds(dchan, desc, sg_len))
+ goto err_free_desc;
+
+ for_each_sg(sgl, sg, sg_len, i) {
+ enum ch_ctrl_donetype donetype = CH_CTRL_DONETYPE_CMD;
+ dma_addr_t link_addr = 0;
+ bool link = false;
+ struct d350_slave_cmd *cmd = &desc->slave;
+
+ mem = sg_dma_address(sg);
+ len = sg_dma_len(sg);
+ if (!len || (len >> __ffs(width)) > U32_MAX ||
+ !IS_ALIGNED(len | mem | dev_dma_addr, width))
+ goto err_free_cmds;
+
+ if (sg_len > 1) {
+ cmd = desc->cmds[i].cmd;
+ if (i < sg_len - 1) {
+ link = true;
+ link_addr = desc->cmds[i + 1].dma;
+ donetype = CH_CTRL_DONETYPE_NONE;
+ }
+ }
+
+ if (check_add_overflow(desc->bytes, len, &desc->bytes) ||
+ desc->bytes > U32_MAX)
+ goto err_free_cmds;
+
+ d350_fill_slave_cmd(dch, desc, cmd, mem, dev_dma_addr, len,
+ link, link_addr, direction, width, maxburst,
+ donetype);
+ }
+
+ return vchan_tx_prep(&dch->vc, &desc->vd, flags);
+
+err_free_cmds:
+ d350_free_cmds(d350_chan_to_dmac(dch), desc);
+err_free_desc:
+ d350_unmap_resource(dchan->device->dev, desc);
+ kfree(desc);
+
+ return NULL;
+}
+
+static struct dma_async_tx_descriptor *
+d350_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t buf_addr,
+ size_t buf_len, size_t period_len,
+ enum dma_transfer_direction direction, unsigned long flags)
+{
+ struct d350_chan *dch = to_d350_chan(dchan);
+ struct d350_desc *desc;
+ phys_addr_t dev_cpu_addr;
+ dma_addr_t dev_dma_addr;
+ enum dma_data_direction dev_dir;
+ enum dma_slave_buswidth width;
+ size_t period;
+ u32 maxburst;
+ int ret;
+
+ if (!buf_len || !period_len || buf_len % period_len ||
+ buf_len > U32_MAX || !is_slave_direction(direction))
+ return NULL;
+
+ ret = d350_slave_params(dch, direction, &dev_cpu_addr, &dev_dir, &width,
+ &maxburst);
+ if (ret)
+ return NULL;
+
+ desc = kzalloc_obj(*desc, GFP_NOWAIT);
+ if (!desc)
+ return NULL;
+
+ desc->direction = direction;
+ dev_dma_addr = d350_map_resource(dchan, desc, dev_cpu_addr, width,
+ dev_dir);
+ if (dma_mapping_error(dchan->device->dev, dev_dma_addr))
+ goto err_free_desc;
+
+ if (!IS_ALIGNED(buf_addr | dev_dma_addr | period_len, width))
+ goto err_free_desc;
+
+ desc->bytes = buf_len;
+ desc->period_len = period_len;
+ desc->periods = buf_len / period_len;
+ desc->cyclic = true;
+
+ if (d350_alloc_cmds(dchan, desc, desc->periods))
+ goto err_free_desc;
+
+ for (period = 0; period < desc->periods; period++) {
+ dma_addr_t mem = buf_addr + period * period_len;
+ dma_addr_t next = desc->cmds[(period + 1) % desc->periods].dma;
+
+ d350_fill_slave_cmd(dch, desc, desc->cmds[period].cmd, mem,
+ dev_dma_addr, period_len, true, next,
+ direction, width, maxburst, CH_CTRL_DONETYPE_CMD);
+ }
+
+ return vchan_tx_prep(&dch->vc, &desc->vd, flags);
+
+err_free_desc:
+ d350_unmap_resource(dchan->device->dev, desc);
+ kfree(desc);
+
+ return NULL;
}
static struct dma_async_tx_descriptor *d350_prep_memcpy(struct dma_chan *chan,
@@ -344,6 +838,75 @@ static u32 d350_get_residue(struct d350_chan *dch)
return res << dch->desc->tsz;
}
+static u32 d350_get_sg_residue(struct d350_chan *dch)
+{
+ struct d350_desc *desc = dch->desc;
+ size_t cmd = 0;
+ size_t i;
+ u32 linkaddr;
+ u32 linkaddrhi;
+ u32 hi_new;
+ u32 lo_new;
+ u32 residue;
+ u64 next_cmd;
+ int retries = 3;
+
+ if (!desc->cmds)
+ return d350_get_residue(dch);
+
+ /*
+ * CH_LINKADDR points at the next command. Match it against the command
+ * array to find the command currently executing, then add every later
+ * command which has not started yet.
+ */
+ /* The channel may step the link pointer between the two reads */
+ hi_new = readl_relaxed(dch->base + CH_LINKADDRHI);
+ do {
+ linkaddrhi = hi_new;
+ linkaddr = readl_relaxed(dch->base + CH_LINKADDR);
+ hi_new = readl_relaxed(dch->base + CH_LINKADDRHI);
+ } while (linkaddrhi != hi_new && --retries);
+
+ next_cmd = (linkaddr & ~CH_LINKADDR_EN) | ((u64)linkaddrhi << 32);
+
+ if (!next_cmd) {
+ cmd = desc->ncmds - 1;
+ } else {
+ for (i = 1; i < desc->ncmds; i++) {
+ if (next_cmd == desc->cmds[i].dma) {
+ cmd = i - 1;
+ break;
+ }
+ }
+ if (i == desc->ncmds)
+ return dch->residue;
+ }
+
+ residue = d350_get_residue(dch);
+
+ /*
+ * If the channel stepped to the next command while CH_XSIZE was read,
+ * the size belongs to the command which just finished and the sum below
+ * would count the bytes left in it twice. Keep the residue from the last
+ * completion in that case.
+ */
+ hi_new = readl_relaxed(dch->base + CH_LINKADDRHI);
+ lo_new = readl_relaxed(dch->base + CH_LINKADDR);
+ if (hi_new != linkaddrhi || lo_new != linkaddr)
+ return dch->residue;
+
+ for (i = cmd + 1; i < desc->ncmds; i++) {
+ const struct d350_slave_cmd *cmd = desc->cmds[i].cmd;
+ u32 count;
+
+ count = FIELD_GET(CH_XY_DES, le32_to_cpu(cmd->xsize));
+ count |= FIELD_GET(CH_XY_DES, le32_to_cpu(cmd->xsizehi)) << 16;
+ residue += (u64)count << desc->tsz;
+ }
+
+ return residue;
+}
+
static int d350_terminate_all(struct dma_chan *chan)
{
struct d350_chan *dch = to_d350_chan(chan);
@@ -395,6 +958,74 @@ static u32 d350_desc_bytes(struct d350_desc *desc)
return desc->bytes;
}
+static u32 d350_get_cyclic_residue(struct d350_chan *dch)
+{
+ struct d350_desc *desc = dch->desc;
+ size_t period;
+ size_t next;
+ u32 linkaddr;
+ u32 linkaddrhi;
+ u32 xsize;
+ u32 xsizehi;
+ u32 hi_new;
+ u32 lo_new;
+ u64 next_cmd;
+ u64 remaining;
+ int retries = 3;
+
+ /*
+ * Locate the current period from the next command's address rather than
+ * counting interrupts, since multiple period completions may share a
+ * pending DONE interrupt.
+ */
+ do {
+ linkaddrhi = readl_relaxed(dch->base + CH_LINKADDRHI);
+ linkaddr = readl_relaxed(dch->base + CH_LINKADDR);
+ hi_new = readl_relaxed(dch->base + CH_LINKADDRHI);
+ if (linkaddrhi != hi_new)
+ continue;
+
+ xsizehi = readl_relaxed(dch->base + CH_XSIZEHI);
+ xsize = readl_relaxed(dch->base + CH_XSIZE);
+ hi_new = readl_relaxed(dch->base + CH_XSIZEHI);
+ if (xsizehi != hi_new)
+ continue;
+
+ /* Retry if the link changed while reading the remaining size. */
+ lo_new = readl_relaxed(dch->base + CH_LINKADDR);
+ hi_new = readl_relaxed(dch->base + CH_LINKADDRHI);
+ if (linkaddrhi != hi_new || linkaddr != lo_new)
+ continue;
+
+ next_cmd = (linkaddr & ~CH_LINKADDR_EN) |
+ ((u64)linkaddrhi << 32);
+ for (next = 0; next < desc->periods; next++)
+ if (next_cmd == desc->cmds[next].dma)
+ break;
+ if (next == desc->periods)
+ continue;
+
+ remaining = FIELD_GET(CH_XY_DES, xsize) |
+ ((u64)FIELD_GET(CH_XY_DES, xsizehi) << 16);
+ remaining <<= desc->tsz;
+ if (remaining > desc->period_len)
+ continue;
+
+ period = next ? next - 1 : desc->periods - 1;
+ return desc->bytes - (period + 1) * desc->period_len + remaining;
+ } while (--retries);
+
+ return dch->residue;
+}
+
+static u32 d350_get_active_residue(struct d350_chan *dch)
+{
+ if (dch->desc->cyclic)
+ return d350_get_cyclic_residue(dch);
+
+ return d350_get_sg_residue(dch);
+}
+
static struct virt_dma_desc *d350_find_queued_desc(struct d350_chan *dch,
dma_cookie_t cookie)
{
@@ -421,7 +1052,7 @@ static enum dma_status d350_tx_status(struct dma_chan *chan, dma_cookie_t cookie
if (cookie == dch->cookie) {
status = dch->status;
if (status == DMA_IN_PROGRESS || status == DMA_PAUSED)
- dch->residue = d350_get_residue(dch);
+ dch->residue = d350_get_active_residue(dch);
residue = dch->residue;
} else {
vd = d350_find_queued_desc(dch, cookie);
@@ -438,22 +1069,10 @@ static enum dma_status d350_tx_status(struct dma_chan *chan, dma_cookie_t cookie
return status;
}
-static void d350_start_next(struct d350_chan *dch)
+static void d350_load_mem_cmd(struct d350_chan *dch)
{
- u32 hdr, *reg;
-
- /* Preserve the failed cookie's status until the channel is terminated. */
- if (dch->status == DMA_ERROR)
- return;
-
- dch->desc = to_d350_desc(vchan_next_desc(&dch->vc));
- if (!dch->desc)
- return;
-
- list_del(&dch->desc->vd.node);
- dch->status = DMA_IN_PROGRESS;
- dch->cookie = dch->desc->vd.tx.cookie;
- dch->residue = d350_desc_bytes(dch->desc);
+ u32 *reg;
+ u32 hdr;
hdr = dch->desc->command[0];
reg = &dch->desc->command[1];
@@ -492,6 +1111,55 @@ static void d350_start_next(struct d350_chan *dch)
writel_relaxed(*reg++, dch->base + CH_LINKADDR);
if (hdr & LINK_LINKADDRHI)
writel_relaxed(*reg++, dch->base + CH_LINKADDRHI);
+}
+
+static void d350_write_addr(struct d350_chan *dch, u32 reg, u64 addr)
+{
+ writel_relaxed(lower_32_bits(addr), dch->base + reg);
+ writel_relaxed(upper_32_bits(addr), dch->base + reg + sizeof(u32));
+}
+
+static void d350_load_slave_cmd(struct d350_chan *dch)
+{
+ struct d350_desc *desc = dch->desc;
+ const struct d350_slave_cmd *cmd;
+ u32 trig_reg;
+
+ cmd = desc->cmds ? desc->cmds[0].cmd : &desc->slave;
+ trig_reg = desc->direction == DMA_MEM_TO_DEV ?
+ CH_DESTRIGINCFG : CH_SRCTRIGINCFG;
+
+ writel_relaxed(le32_to_cpu(cmd->ctrl), dch->base + CH_CTRL);
+ d350_write_addr(dch, CH_SRCADDR, le64_to_cpu(cmd->src_addr));
+ d350_write_addr(dch, CH_DESADDR, le64_to_cpu(cmd->dst_addr));
+ writel_relaxed(le32_to_cpu(cmd->xsize), dch->base + CH_XSIZE);
+ writel_relaxed(le32_to_cpu(cmd->xsizehi), dch->base + CH_XSIZEHI);
+ writel_relaxed(le32_to_cpu(cmd->src_transcfg), dch->base + CH_SRCTRANSCFG);
+ writel_relaxed(le32_to_cpu(cmd->dst_transcfg), dch->base + CH_DESTRANSCFG);
+ writel_relaxed(le32_to_cpu(cmd->xaddrinc), dch->base + CH_XADDRINC);
+ writel_relaxed(le32_to_cpu(cmd->trigincfg), dch->base + trig_reg);
+ d350_write_addr(dch, CH_LINKADDR, le64_to_cpu(cmd->link_addr));
+}
+
+static void d350_start_next(struct d350_chan *dch)
+{
+ /* Preserve the failed cookie's status until the channel is terminated. */
+ if (dch->status == DMA_ERROR)
+ return;
+
+ dch->desc = to_d350_desc(vchan_next_desc(&dch->vc));
+ if (!dch->desc)
+ return;
+
+ list_del(&dch->desc->vd.node);
+ dch->status = DMA_IN_PROGRESS;
+ dch->cookie = dch->desc->vd.tx.cookie;
+ dch->residue = d350_desc_bytes(dch->desc);
+
+ if (is_slave_direction(dch->desc->direction))
+ d350_load_slave_cmd(dch);
+ else
+ d350_load_mem_cmd(dch);
writel(CH_CMD_ENABLE, dch->base + CH_CMD);
}
@@ -511,7 +1179,7 @@ static irqreturn_t d350_irq(int irq, void *data)
struct d350_chan *dch = data;
struct virt_dma_desc *vd;
struct d350_desc *desc;
- u32 residue;
+ u32 residue = 0;
u32 ch_status;
u32 irq_status;
u32 errinfo = 0;
@@ -544,18 +1212,26 @@ static irqreturn_t d350_irq(int irq, void *data)
else
vd->tx_result.result = DMA_TRANS_ABORTED;
- residue = d350_get_residue(dch);
+ residue = d350_get_active_residue(dch);
vd->tx_result.residue = residue;
dch->status = DMA_ERROR;
dch->residue = residue;
dch->desc = NULL;
- vchan_cookie_complete(vd);
+ if (desc->cyclic)
+ vchan_terminate_vdesc(vd);
+ else
+ vchan_cookie_complete(vd);
} else {
- dch->status = DMA_COMPLETE;
- dch->residue = 0;
- dch->desc = NULL;
- vchan_cookie_complete(vd);
- d350_start_next(dch);
+ if (desc->cyclic) {
+ dch->residue = d350_get_cyclic_residue(dch);
+ vchan_cyclic_callback(vd);
+ } else {
+ dch->status = DMA_COMPLETE;
+ dch->residue = 0;
+ dch->desc = NULL;
+ vchan_cookie_complete(vd);
+ d350_start_next(dch);
+ }
}
return IRQ_HANDLED;
@@ -584,6 +1260,11 @@ static void d350_free_chan_resources(struct dma_chan *chan)
free_irq(dch->irq, dch);
vchan_synchronize(&dch->vc);
vchan_free_chan_resources(&dch->vc);
+
+ guard(spinlock_irqsave)(&dch->vc.lock);
+
+ dch->req = U32_MAX;
+ memset(&dch->sconfig, 0, sizeof(dch->sconfig));
}
static int d350_probe(struct platform_device *pdev)
@@ -621,6 +1302,11 @@ static int d350_probe(struct platform_device *pdev)
return -ENOMEM;
dmac->nchan = nchan;
+ dmac->cmd_pool = dmam_pool_create("dma350-cmd", dev,
+ sizeof(struct d350_slave_cmd),
+ sizeof(u64), 0);
+ if (!dmac->cmd_pool)
+ return -ENOMEM;
reg = readl_relaxed(base + DMAINFO + DMA_BUILDCFG1);
dmac->nreq = FIELD_GET(DMA_CFG_NUM_TRIGGER_IN, reg);
@@ -639,19 +1325,23 @@ static int d350_probe(struct platform_device *pdev)
dmac->dma.device_free_chan_resources = d350_free_chan_resources;
dma_cap_set(DMA_MEMCPY, dmac->dma.cap_mask);
dmac->dma.device_prep_dma_memcpy = d350_prep_memcpy;
+ dmac->dma.device_prep_slave_sg = d350_prep_slave_sg;
+ dmac->dma.device_prep_dma_cyclic = d350_prep_dma_cyclic;
dmac->dma.device_pause = d350_pause;
dmac->dma.device_resume = d350_resume;
dmac->dma.device_terminate_all = d350_terminate_all;
dmac->dma.device_synchronize = d350_synchronize;
dmac->dma.device_tx_status = d350_tx_status;
dmac->dma.device_issue_pending = d350_issue_pending;
+ dmac->dma.device_config = d350_device_config;
+ dmac->dma.device_caps = d350_device_caps;
INIT_LIST_HEAD(&dmac->dma.channels);
reg = readl_relaxed(base + DMANSECCTRL + NSEC_CTRL);
writel_relaxed(reg | INTREN_ANYCHINTR_EN,
base + DMANSECCTRL + NSEC_CTRL);
- /* Would be nice to have per-channel caps for this... */
+ /* Advertise memset only if every usable channel supports wrap. */
memset = true;
for (int i = 0; i < nchan; i++) {
struct d350_chan *dch = &dmac->channels[i];
@@ -684,8 +1374,17 @@ static int d350_probe(struct platform_device *pdev)
writel_relaxed(reg, dch->base + CH_LINKATTR);
dch->coherent = coherent;
+ /* Request line 0 is valid, so it cannot mean unconfigured. */
+ dch->req = U32_MAX;
dch->vc.desc_free = d350_desc_free;
vchan_init(&dch->vc, &dmac->dma);
+
+ if (d350_slave_supported(dch)) {
+ dma_cap_set(DMA_SLAVE, dmac->dma.cap_mask);
+ dma_cap_set(DMA_CYCLIC, dmac->dma.cap_mask);
+ dmac->dma.directions |= BIT(DMA_MEM_TO_DEV) |
+ BIT(DMA_DEV_TO_MEM);
+ }
}
if (memset) {
@@ -699,6 +1398,14 @@ static int d350_probe(struct platform_device *pdev)
if (ret)
return dev_err_probe(dev, ret, "Failed to register DMA device\n");
+ if (dev->of_node) {
+ ret = devm_of_dma_controller_register(dev, dev->of_node,
+ d350_of_xlate, dmac);
+ if (ret)
+ return dev_err_probe(dev, ret,
+ "Failed to register OF DMA controller\n");
+ }
+
return 0;
}
--
2.54.0
next prev parent reply other threads:[~2026-11-17 9:43 UTC|newest]
Thread overview: 11+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-17 9:43 [PATCH v7 00/10] dmaengine: arm-dma350: Add slave support and Sky1 integration Jelly Jia
2026-11-17 9:43 ` [PATCH v7 01/10] dmaengine: arm-dma350: Fix source trigger bit Jelly Jia
2026-11-17 9:43 ` [PATCH v7 02/10] dmaengine: arm-dma350: Check DMA mask setup errors Jelly Jia
2026-11-17 9:43 ` [PATCH v7 03/10] dmaengine: arm-dma350: Initialize channel coherency Jelly Jia
2026-11-17 9:43 ` [PATCH v7 04/10] dmaengine: arm-dma350: Serialize IRQ and termination state Jelly Jia
2026-11-17 9:43 ` [PATCH v7 05/10] dmaengine: arm-dma350: Report submitted descriptor status Jelly Jia
2026-11-17 9:43 ` [PATCH v7 06/10] dmaengine: arm-dma350: Use managed device registration Jelly Jia
2026-11-17 9:43 ` [PATCH v7 07/10] dmaengine: arm-dma350: Track descriptor byte lengths explicitly Jelly Jia
2026-11-17 9:43 ` Jelly Jia [this message]
2026-11-17 9:43 ` [PATCH v7 09/10] dt-bindings: dma: arm,dma-350: Describe the CIX Sky1 integration Jelly Jia
2026-11-17 9:43 ` [PATCH v7 10/10] dmaengine: arm-dma350: Support integration clocks and resets Jelly Jia
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261117094335.1393735-9-Jelly.Jia@cixtech.com \
--to=jelly.jia@cixtech.com \
--cc=Frank.Li@kernel.org \
--cc=cix-kernel-upstream@cixtech.com \
--cc=conor+dt@kernel.org \
--cc=devicetree@vger.kernel.org \
--cc=dmaengine@vger.kernel.org \
--cc=krzk+dt@kernel.org \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-kernel@vger.kernel.org \
--cc=robh@kernel.org \
--cc=robin.murphy@arm.com \
--cc=vkoul@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®