[PATCH v2 2/5] dmaengine: dw-edma: Add DMA_SG support

Synopsys DesignWare eDMA supports a linked-list mode where each list item
carries independent source and destination addresses, letting multiple
independent memory transfers be described in one linked list and submitted
to the hardware as a single DMA transaction. The IP processes list items
strictly in order, so paired scatter-gather entries are never reordered.

Implement the DMA_SG capability by adding a new EDMA_XFER_DUAL_SG transfer
type and a corresponding struct dw_edma_dual_sg carrying the paired source
and destination SG lists. dw_edma_device_transfer() walks both lists in
lockstep, building a single hardware linked-list; a per-entry length
mismatch or premature list end fails the whole request.

The transfer direction is inferred from the channel hardware polarity
(EDMA_DIR_READ/WRITE), not from dma_slave_config.direction: for local eDMA
(DW_EDMA_CHIP_LOCAL) read channels handle DEV_TO_MEM and write channels
handle MEM_TO_DEV; for remote eDMA the mapping is inverted. PCIe bus
addresses are translated via dw_edma_get_pci_address() for the remote side
of each transfer. dmaengine_slave_config() must still be called before
dmaengine_prep_dma_sg() because dw_edma_device_transfer() gates transfers
on chan->configured, even though the direction field itself is unused by
the DMA_SG path.

Signed-off-by: Sumit Kumar <[email protected]>
---
 drivers/dma/dw-edma/dw-edma-core.c | 87 +++++++++++++++++++++++++++++++++++---
 drivers/dma/dw-edma/dw-edma-core.h | 10 ++++-
 2 files changed, 90 insertions(+), 7 deletions(-)

diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index 1f893dc54c7938a9c45bafd975a4f99fdc0acb38..9fbfa5ad65b5421b164d2ab9883233225e0269e4 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -372,6 +372,7 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer,
 	struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan);
 	enum dma_transfer_direction dir = xfer->direction;
 	struct scatterlist *sg = NULL;
+	struct scatterlist *dst_sg = NULL;
 	struct dw_edma_burst *burst;
 	struct dw_edma_desc *desc;
 	u64 src_addr, dst_addr;
@@ -429,6 +430,9 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer,
 			return NULL;
 		if (!xfer->xfer.il->src_inc || !xfer->xfer.il->dst_inc)
 			return NULL;
+	} else if (xfer->type == EDMA_XFER_DUAL_SG) {
+		if (xfer->xfer.dual_sg.len < 1)
+			return NULL;
 	} else {
 		return NULL;
 	}
@@ -441,16 +445,27 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer,
 		dst_addr = config->dst_addr;
 	}
 
-	if (dir == DMA_DEV_TO_MEM)
-		src_addr = dw_edma_get_pci_address(chan, (phys_addr_t)src_addr);
-	else
-		dst_addr = dw_edma_get_pci_address(chan, (phys_addr_t)dst_addr);
+	/*
+	 * DUAL_SG translates each src/dst sg entry individually below
+	 * (see the burst->sar/burst->dar assignment), so it is exempt
+	 * from the single up-front translation used by other types.
+	 */
+	if (xfer->type != EDMA_XFER_DUAL_SG) {
+		if (dir == DMA_DEV_TO_MEM)
+			src_addr = dw_edma_get_pci_address(chan, (phys_addr_t)src_addr);
+		else
+			dst_addr = dw_edma_get_pci_address(chan, (phys_addr_t)dst_addr);
+	}
 
 	if (xfer->type == EDMA_XFER_CYCLIC) {
 		cnt = xfer->xfer.cyclic.cnt;
 	} else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
 		cnt = xfer->xfer.sg.len;
 		sg = xfer->xfer.sg.sgl;
+	} else if (xfer->type == EDMA_XFER_DUAL_SG) {
+		cnt = xfer->xfer.dual_sg.len;
+		sg = xfer->xfer.dual_sg.src_sgl;
+		dst_sg = xfer->xfer.dual_sg.dst_sgl;
 	} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
 		cnt = xfer->xfer.il->numf * xfer->xfer.il->frame_size;
 		fsz = xfer->xfer.il->frame_size;
@@ -463,12 +478,23 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer,
 	for (i = 0; i < cnt; i++) {
 		if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg)
 			break;
+		/*
+		 * DUAL_SG walks the source and destination lists in lockstep;
+		 * a premature end or a per-entry length mismatch would leave
+		 * the descriptor partially built, so fail the whole request.
+		 */
+		if (xfer->type == EDMA_XFER_DUAL_SG &&
+		    (!sg || !dst_sg || sg_dma_len(sg) != sg_dma_len(dst_sg))) {
+			kfree(desc);
+			return NULL;
+		}
 
 		burst = desc->burst + i;
 
 		if (xfer->type == EDMA_XFER_CYCLIC)
 			burst->sz = xfer->xfer.cyclic.len;
-		else if (xfer->type == EDMA_XFER_SCATTER_GATHER)
+		else if (xfer->type == EDMA_XFER_SCATTER_GATHER ||
+			 xfer->type == EDMA_XFER_DUAL_SG)
 			burst->sz = sg_dma_len(sg);
 		else if (xfer->type == EDMA_XFER_INTERLEAVED)
 			burst->sz = xfer->xfer.il->sgl[i % fsz].size;
@@ -492,6 +518,9 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer,
 				 */
 			} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
 				burst->dar = dst_addr;
+			} else if (xfer->type == EDMA_XFER_DUAL_SG) {
+				burst->sar = dw_edma_get_pci_address(chan, sg_dma_address(sg));
+				burst->dar = sg_dma_address(dst_sg);
 			}
 		} else {
 			burst->dar = dst_addr;
@@ -507,13 +536,19 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer,
 				 * and destination addresses are increased
 				 * by the same portion (data length)
 				 */
-			}  else if (xfer->type == EDMA_XFER_INTERLEAVED) {
+			} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
 				burst->sar = src_addr;
+			} else if (xfer->type == EDMA_XFER_DUAL_SG) {
+				burst->sar = sg_dma_address(sg);
+				burst->dar = dw_edma_get_pci_address(chan, sg_dma_address(dst_sg));
 			}
 		}
 
 		if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
 			sg = sg_next(sg);
+		} else if (xfer->type == EDMA_XFER_DUAL_SG) {
+			sg = sg_next(sg);
+			dst_sg = sg_next(dst_sg);
 		} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
 			struct dma_interleaved_template *il = xfer->xfer.il;
 			struct data_chunk *dc = &il->sgl[i % fsz];
@@ -613,6 +648,44 @@ static void dw_hdma_set_callback_result(struct virt_dma_desc *vd,
 	res->residue = residue;
 }
 
+static struct dma_async_tx_descriptor *
+dw_edma_device_prep_dma_sg(struct dma_chan *dchan,
+			   struct scatterlist *dst_sg, unsigned int dst_nents,
+			   struct scatterlist *src_sg, unsigned int src_nents,
+			   unsigned long flags)
+{
+	struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
+	struct dw_edma_transfer xfer;
+	enum dma_transfer_direction dir;
+
+	if (src_nents != dst_nents || !src_nents)
+		return NULL;
+
+	if (!src_sg || !dst_sg)
+		return NULL;
+
+	/* Determine direction from channel configuration */
+	if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)
+		dir = (chan->dir == EDMA_DIR_READ) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV;
+	else
+		dir = (chan->dir == EDMA_DIR_WRITE) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV;
+
+	xfer.dchan = dchan;
+	xfer.direction = dir;
+	xfer.xfer.dual_sg.src_sgl = src_sg;
+	xfer.xfer.dual_sg.dst_sgl = dst_sg;
+	xfer.xfer.dual_sg.len = src_nents;
+	xfer.flags = flags;
+	xfer.type = EDMA_XFER_DUAL_SG;
+
+	/*
+	 * dw_edma_device_transfer() rejects unconfigured channels, so
+	 * dmaengine_slave_config() must have been called on this channel
+	 * beforehand even though the direction field is unused here.
+	 */
+	return dw_edma_device_transfer(&xfer, dw_edma_device_get_config(dchan, NULL));
+}
+
 static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
 {
 	struct dw_edma_desc *desc;
@@ -997,6 +1070,7 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
 	dma_cap_set(DMA_CYCLIC, dma->cap_mask);
 	dma_cap_set(DMA_PRIVATE, dma->cap_mask);
 	dma_cap_set(DMA_INTERLEAVE, dma->cap_mask);
+	dma_cap_set(DMA_SG, dma->cap_mask);
 	dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
 	dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
 	dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
@@ -1017,6 +1091,7 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
 	dma->device_prep_config_sg = dw_edma_device_prep_config_sg;
 	dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic;
 	dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma;
+	dma->device_prep_dma_sg = dw_edma_device_prep_dma_sg;
 
 	dma_set_max_seg_size(dma->dev, U32_MAX);
 
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
index f6a5ad31756723e6fd72f7d9d903a1204e25c0b0..e3219ae96d84f528fb7d77dbe3a0b3d7c1264f05 100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -38,7 +38,8 @@ enum dw_edma_status {
 enum dw_edma_xfer_type {
 	EDMA_XFER_SCATTER_GATHER = 0,
 	EDMA_XFER_CYCLIC,
-	EDMA_XFER_INTERLEAVED
+	EDMA_XFER_INTERLEAVED,
+	EDMA_XFER_DUAL_SG,
 };
 
 struct dw_edma_chan;
@@ -151,6 +152,12 @@ struct dw_edma_sg {
 	unsigned int			len;
 };
 
+struct dw_edma_dual_sg {
+	struct scatterlist		*src_sgl;
+	struct scatterlist		*dst_sgl;
+	unsigned int			len;
+};
+
 struct dw_edma_cyclic {
 	dma_addr_t			paddr;
 	size_t				len;
@@ -163,6 +170,7 @@ struct dw_edma_transfer {
 		struct dw_edma_sg		sg;
 		struct dw_edma_cyclic		cyclic;
 		struct dma_interleaved_template *il;
+		struct dw_edma_dual_sg		dual_sg;
 	} xfer;
 	enum dma_transfer_direction	direction;
 	unsigned long			flags;

-- 
2.34.1