[PATCH v2 4/5] PCI: epf-mhi: Add batched DMA read support

Add support for batched DMA transfers in the PCI EPF MHI driver to
improve performance when reading multiple buffers from the host.

Implement two variants of the read_batch() callback:
pci_epf_mhi_edma_read_batch() is a DMA-optimized implementation that uses
dmaengine_prep_dma_sg() to transfer multiple buffers in a single DMA
transaction, while pci_epf_mhi_iatu_read_batch() serves as a CPU-copy
fallback for platforms without DMA support by sequentially processing each
buffer via IATU mapping. Wire up read_batch() to the eDMA variant only
when the RX DMA channel advertises the DMA_SG capability, falling back to
the IATU variant otherwise.

On a successful batch, notify completion for every buffer via its
caller-supplied buf_info->cb, mirroring the read_sync/read_async
completion semantics so the MHI stack can free each buffer and raise its
transfer completion event. Make read_batch() a mandatory callback in
mhi_ep_register_controller(), alongside read_sync/write_sync/read_async/
write_async.

This enables the MHI endpoint stack to cache ring data efficiently,
particularly for wraparound scenarios where ring data spans two
non-contiguous memory regions.

Signed-off-by: Sumit Kumar <[email protected]>
---
 drivers/bus/mhi/ep/main.c                    |   3 +-
 drivers/pci/endpoint/functions/pci-epf-mhi.c | 159 +++++++++++++++++++++++++++
 include/linux/mhi_ep.h                       |   7 ++
 3 files changed, 168 insertions(+), 1 deletion(-)

diff --git a/drivers/bus/mhi/ep/main.c b/drivers/bus/mhi/ep/main.c
index 21bc2c50170ff7f0181c0d0d4eefd51db30f34c8..c45240fc481dc3260a61b546c223b6be8e105587 100644
--- a/drivers/bus/mhi/ep/main.c
+++ b/drivers/bus/mhi/ep/main.c
@@ -1459,7 +1459,8 @@ int mhi_ep_register_controller(struct mhi_ep_cntrl *mhi_cntrl,
 		return -EINVAL;
 
 	if (!mhi_cntrl->read_sync || !mhi_cntrl->write_sync ||
-	    !mhi_cntrl->read_async || !mhi_cntrl->write_async)
+	    !mhi_cntrl->read_async || !mhi_cntrl->write_async ||
+	    !mhi_cntrl->read_batch)
 		return -EINVAL;
 
 	ret = mhi_ep_chan_init(mhi_cntrl, config);
diff --git a/drivers/pci/endpoint/functions/pci-epf-mhi.c b/drivers/pci/endpoint/functions/pci-epf-mhi.c
index 6bac69fc84b472c5f1abbeede2b441b5db73c784..ad245148de1462cffc950ba9a9e232405514dfb7 100644
--- a/drivers/pci/endpoint/functions/pci-epf-mhi.c
+++ b/drivers/pci/endpoint/functions/pci-epf-mhi.c
@@ -444,6 +444,162 @@ static int pci_epf_mhi_edma_write(struct mhi_ep_cntrl *mhi_cntrl,
 	return ret;
 }
 
+static int pci_epf_mhi_iatu_read_batch(struct mhi_ep_cntrl *mhi_cntrl,
+				       struct mhi_ep_buf_info *buf_info_array,
+				       u32 num_buffers)
+{
+	struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);
+	struct device *dev = &epf_mhi->epf->dev;
+	u32 i;
+	int ret;
+
+	if (num_buffers == 0)
+		return -EINVAL;
+
+	for (i = 0; i < num_buffers; i++) {
+		ret = pci_epf_mhi_iatu_read(mhi_cntrl, &buf_info_array[i]);
+		if (ret < 0) {
+			dev_err(dev, "Failed to read buffer %u of %u in batch: %d\n",
+				i, num_buffers, ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int pci_epf_mhi_edma_read_batch(struct mhi_ep_cntrl *mhi_cntrl,
+				       struct mhi_ep_buf_info *buf_info_array,
+				       u32 num_buffers)
+{
+	struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);
+	struct device *dma_dev = epf_mhi->epf->epc->dev.parent;
+	struct device *dev = &epf_mhi->epf->dev;
+	struct dma_async_tx_descriptor *desc;
+	struct dma_slave_config config = {};
+	DECLARE_COMPLETION_ONSTACK(complete);
+	struct scatterlist *src_sg;
+	struct scatterlist *dst_sg;
+	unsigned long time_left;
+	struct dma_chan *chan;
+	dma_cookie_t cookie;
+	unsigned int i;
+	int mapped;
+	void *buf;
+	int ret;
+
+	if (num_buffers == 0)
+		return -EINVAL;
+
+	/*
+	 * Single allocation carved into two arrays: src_sg[], dst_sg[].
+	 * Reduces allocator round-trips on the ring-cache hot path. Done
+	 * before taking the lock so direct reclaim cannot stall other
+	 * transfers waiting on epf_mhi->lock.
+	 */
+	buf = kcalloc(num_buffers, 2 * sizeof(*src_sg), GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+	src_sg = buf;
+	dst_sg = src_sg + num_buffers;
+
+	mutex_lock(&epf_mhi->lock);
+
+	chan = epf_mhi->dma_chan_rx;
+	if (!chan) {
+		ret = -ENODEV;
+		goto err_unlock;
+	}
+
+	sg_init_table(src_sg, num_buffers);
+	sg_init_table(dst_sg, num_buffers);
+
+	for (i = 0; i < num_buffers; i++) {
+		/*
+		 * src addresses are PCIe host bus addresses already visible to
+		 * the eDMA engine; no dma_map_sg() is needed for the source list.
+		 */
+		sg_dma_address(&src_sg[i]) = buf_info_array[i].host_addr;
+		sg_dma_len(&src_sg[i]) = buf_info_array[i].size;
+
+		sg_set_buf(&dst_sg[i], buf_info_array[i].dev_addr, buf_info_array[i].size);
+	}
+
+	mapped = dma_map_sg(dma_dev, dst_sg, num_buffers, DMA_FROM_DEVICE);
+	if (!mapped) {
+		dev_err(dev, "Failed to map destination buffers for %u-buffer batch read\n",
+			num_buffers);
+		ret = -EIO;
+		goto err_unlock;
+	}
+
+	config.direction = DMA_DEV_TO_MEM;
+	ret = dmaengine_slave_config(chan, &config);
+	if (ret) {
+		dev_err(dev, "Failed to configure DMA channel for %u-buffer batch read: %d\n",
+			num_buffers, ret);
+		goto err_unmap;
+	}
+
+	desc = dmaengine_prep_dma_sg(chan, dst_sg, num_buffers,
+				     src_sg, num_buffers,
+				     DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
+	if (!desc) {
+		dev_err(dev, "Failed to prepare batch DMA\n");
+		ret = -EIO;
+		goto err_unmap;
+	}
+
+	desc->callback = pci_epf_mhi_dma_callback;
+	desc->callback_param = &complete;
+
+	cookie = dmaengine_submit(desc);
+	ret = dma_submit_error(cookie);
+	if (ret) {
+		dev_err(dev, "Failed to submit DMA\n");
+		if (dmaengine_terminate_sync(chan))
+			dev_err(dev, "Failed to terminate DMA channel after submit failure\n");
+		goto err_unmap;
+	}
+
+	dma_async_issue_pending(chan);
+
+	time_left = wait_for_completion_timeout(&complete,
+						msecs_to_jiffies(PCI_EPF_MHI_DMA_TIMEOUT_MS));
+	if (!time_left) {
+		dev_err(dev, "DMA transfer timeout\n");
+		if (dmaengine_terminate_sync(chan))
+			dev_err(dev, "Failed to terminate DMA channel after timeout\n");
+		ret = -ETIMEDOUT;
+		goto err_unmap;
+	}
+
+	ret = 0;
+
+err_unmap:
+	/* dma_unmap_sg() must use the nents passed to dma_map_sg(), not its return value */
+	dma_unmap_sg(dma_dev, dst_sg, num_buffers, DMA_FROM_DEVICE);
+err_unlock:
+	mutex_unlock(&epf_mhi->lock);
+
+	kfree(buf);
+
+	/*
+	 * On a successful batch, notify completion for every buffer via its
+	 * caller-supplied callback so MHI can free the backing buffer and raise
+	 * the transfer completion event per entry, matching the
+	 * read_sync/read_async semantics. Done after dropping the lock as a
+	 * callback may re-enter the driver and epf_mhi->lock is not reentrant.
+	 */
+	if (!ret) {
+		for (i = 0; i < num_buffers; i++)
+			if (buf_info_array[i].cb)
+				buf_info_array[i].cb(&buf_info_array[i]);
+	}
+
+	return ret;
+}
+
 static void pci_epf_mhi_dma_worker(struct work_struct *work)
 {
 	struct pci_epf_mhi *epf_mhi = container_of(work, struct pci_epf_mhi, dma_work);
@@ -789,11 +945,14 @@ static int pci_epf_mhi_link_up(struct pci_epf *epf)
 	mhi_cntrl->unmap_free = pci_epf_mhi_unmap_free;
 	mhi_cntrl->read_sync = mhi_cntrl->read_async = pci_epf_mhi_iatu_read;
 	mhi_cntrl->write_sync = mhi_cntrl->write_async = pci_epf_mhi_iatu_write;
+	mhi_cntrl->read_batch = pci_epf_mhi_iatu_read_batch;
 	if (info->flags & MHI_EPF_USE_DMA) {
 		mhi_cntrl->read_sync = pci_epf_mhi_edma_read;
 		mhi_cntrl->write_sync = pci_epf_mhi_edma_write;
 		mhi_cntrl->read_async = pci_epf_mhi_edma_read_async;
 		mhi_cntrl->write_async = pci_epf_mhi_edma_write_async;
+		if (dma_has_cap(DMA_SG, epf_mhi->dma_chan_rx->device->cap_mask))
+			mhi_cntrl->read_batch = pci_epf_mhi_edma_read_batch;
 	}
 
 	/* Register the MHI EP controller */
diff --git a/include/linux/mhi_ep.h b/include/linux/mhi_ep.h
index 7b40fc8cbe77ab8419d167e89264b69a817b9fb1..51f66cae937a53a93e9ae76cdcfaac7b1e08283f 100644
--- a/include/linux/mhi_ep.h
+++ b/include/linux/mhi_ep.h
@@ -107,6 +107,11 @@ struct mhi_ep_buf_info {
  * @write_sync: CB function for writing to host memory synchronously
  * @read_async: CB function for reading from host memory asynchronously
  * @write_async: CB function for writing to host memory asynchronously
+ * @read_batch: CB function for reading from host memory in batches synchronously.
+ *              Callers must pass at least one buffer (num_buffers > 0); each
+ *              buf_info entry must have host_addr, dev_addr and size set.
+ *              Each buffer's buf_info->cb, if set, is invoked once the batch
+ *              completes successfully, mirroring read_sync/read_async.
  * @mhi_state: MHI Endpoint state
  * @max_chan: Maximum channels supported by the endpoint controller
  * @mru: MRU (Maximum Receive Unit) value of the endpoint controller
@@ -164,6 +169,8 @@ struct mhi_ep_cntrl {
 	int (*write_sync)(struct mhi_ep_cntrl *mhi_cntrl, struct mhi_ep_buf_info *buf_info);
 	int (*read_async)(struct mhi_ep_cntrl *mhi_cntrl, struct mhi_ep_buf_info *buf_info);
 	int (*write_async)(struct mhi_ep_cntrl *mhi_cntrl, struct mhi_ep_buf_info *buf_info);
+	int (*read_batch)(struct mhi_ep_cntrl *mhi_cntrl, struct mhi_ep_buf_info *buf_info_array,
+			  u32 num_buffers);
 
 	enum mhi_state mhi_state;
 

-- 
2.34.1