[PATCH v2 4/5] crypto: iaa - use bounce buffer for multi-sg decompress input

From: Giovanni Cabiddu <[email protected]>

Since commit e2c3b6b21c77 ("mm: zswap: use SG list decompression APIs
from zsmalloc"), zswap passes the raw zsmalloc SG list directly to
crypto drivers, so a compressed object spanning multiple pages reaches
IAA as a multi-entry source. Such requests currently fall back to
software decompression.

As IAA hardware requires a single DMA source buffer, linearize small
multi-entry sources into a pre-allocated bounce page and submit that to
the hardware instead of falling back to software. Keep the software
fallback only for multi-entry destinations. This recovers most of the
performance lost by using the software fallback.

Store the bounce-page state in the acomp request context alongside the
existing compression CRC, free it through a shared source-unmap helper,
and back the pages with a small module-wide mempool so the path remains
available in reclaim-driven callers.

Signed-off-by: Giovanni Cabiddu <[email protected]>
Signed-off-by: Vinicius Costa Gomes <[email protected]>
---
 drivers/crypto/intel/iaa/iaa_crypto_main.c | 128 ++++++++++++++++++++++++-----
 1 file changed, 106 insertions(+), 22 deletions(-)

diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c
index 9505ca23e6f3..51547c5fcf70 100644
--- a/drivers/crypto/intel/iaa/iaa_crypto_main.c
+++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c
@@ -9,6 +9,7 @@
 #include <linux/sysfs.h>
 #include <linux/device.h>
 #include <linux/iommu.h>
+#include <linux/mempool.h>
 #include <uapi/linux/idxd.h>
 #include <linux/highmem.h>
 #include <linux/sched/smt.h>
@@ -157,6 +158,16 @@ static bool async_mode;
 /* Use interrupts */
 static bool use_irq;
 
+struct iaa_req_ctx {
+	u32 compression_crc;
+	struct page *bounce_src;
+	dma_addr_t bounce_src_dma;
+	unsigned int bounce_src_len;
+};
+
+static mempool_t *iaa_bounce_pool;
+#define IAA_BOUNCE_POOL_SIZE	128
+
 /**
  * set_iaa_sync_mode - Set IAA sync mode
  * @name: The name of the sync mode
@@ -984,6 +995,23 @@ static inline int check_completion(struct device *dev,
 	return ret;
 }
 
+static void iaa_unmap_src(struct device *dev, struct acomp_req *req)
+{
+	struct iaa_req_ctx *req_ctx = acomp_request_ctx(req);
+
+	if (req_ctx->bounce_src) {
+		dma_unmap_page(dev, req_ctx->bounce_src_dma,
+			       req_ctx->bounce_src_len, DMA_TO_DEVICE);
+		mempool_free(req_ctx->bounce_src, iaa_bounce_pool);
+		req_ctx->bounce_src = NULL;
+		req_ctx->bounce_src_dma = 0;
+		req_ctx->bounce_src_len = 0;
+		return;
+	}
+
+	dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE);
+}
+
 static int deflate_generic_decompress(struct acomp_req *req)
 {
 	ACOMP_FBREQ_ON_STACK(fbreq, req);
@@ -1027,6 +1055,7 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc,
 	struct iaa_device_compression_mode *active_compression_mode;
 	struct iaa_compression_ctx *compression_ctx;
 	struct crypto_ctx *ctx = __ctx;
+	struct iaa_req_ctx *req_ctx = acomp_request_ctx(ctx->req);
 	struct iaa_device *iaa_device;
 	struct idxd_device *idxd;
 	struct iaa_wq *iaa_wq;
@@ -1085,10 +1114,9 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc,
 	}
 
 	if (ctx->compress && compression_ctx->verify_compress) {
-		u32 *compression_crc = acomp_request_ctx(ctx->req);
 		dma_addr_t src_addr, dst_addr;
 
-		*compression_crc = idxd_desc->iax_completion->crc;
+		req_ctx->compression_crc = idxd_desc->iax_completion->crc;
 
 		ret = iaa_remap_for_verify(dev, iaa_wq, ctx->req, &src_addr, &dst_addr);
 		if (ret) {
@@ -1111,7 +1139,7 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc,
 	}
 err:
 	dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_FROM_DEVICE);
-	dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_TO_DEVICE);
+	iaa_unmap_src(dev, ctx->req);
 out:
 	if (ret != 0)
 		dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret);
@@ -1131,7 +1159,7 @@ static int iaa_compress(struct crypto_tfm *tfm,	struct acomp_req *req,
 {
 	struct iaa_device_compression_mode *active_compression_mode;
 	struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
-	u32 *compression_crc = acomp_request_ctx(req);
+	struct iaa_req_ctx *req_ctx = acomp_request_ctx(req);
 	struct iaa_device *iaa_device;
 	struct idxd_desc *idxd_desc;
 	struct iax_hw_desc *desc;
@@ -1222,7 +1250,7 @@ static int iaa_compress(struct crypto_tfm *tfm,	struct acomp_req *req,
 	update_total_comp_bytes_out(*dlen);
 	update_wq_comp_bytes(wq, *dlen);
 
-	*compression_crc = idxd_desc->iax_completion->crc;
+	req_ctx->compression_crc = idxd_desc->iax_completion->crc;
 
 	if (!ctx->async_mode)
 		idxd_free_desc(wq, idxd_desc);
@@ -1282,7 +1310,7 @@ static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req,
 {
 	struct iaa_device_compression_mode *active_compression_mode;
 	struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
-	u32 *compression_crc = acomp_request_ctx(req);
+	struct iaa_req_ctx *req_ctx = acomp_request_ctx(req);
 	struct iaa_device *iaa_device;
 	struct idxd_desc *idxd_desc;
 	struct iax_hw_desc *desc;
@@ -1342,10 +1370,10 @@ static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req,
 		goto err;
 	}
 
-	if (*compression_crc != idxd_desc->iax_completion->crc) {
+	if (req_ctx->compression_crc != idxd_desc->iax_completion->crc) {
 		ret = -EINVAL;
-		dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch:"
-			" comp=0x%x, decomp=0x%x\n", *compression_crc,
+		dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch: comp=0x%x, decomp=0x%x\n",
+			req_ctx->compression_crc,
 			idxd_desc->iax_completion->crc);
 		print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET,
 			       8, 1, idxd_desc->iax_completion, 64, 0);
@@ -1485,6 +1513,7 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req,
 
 static int iaa_comp_acompress(struct acomp_req *req)
 {
+	struct iaa_req_ctx *req_ctx = acomp_request_ctx(req);
 	struct iaa_compression_ctx *compression_ctx;
 	struct crypto_tfm *tfm = req->base.tfm;
 	dma_addr_t src_addr, dst_addr;
@@ -1493,6 +1522,10 @@ static int iaa_comp_acompress(struct acomp_req *req)
 	struct idxd_wq *wq;
 	struct device *dev;
 
+	req_ctx->bounce_src = NULL;
+	req_ctx->bounce_src_dma = 0;
+	req_ctx->bounce_src_len = 0;
+
 	compression_ctx = crypto_tfm_ctx(tfm);
 
 	if (!iaa_crypto_enabled) {
@@ -1584,12 +1617,19 @@ static int iaa_comp_acompress(struct acomp_req *req)
 
 static int iaa_comp_adecompress(struct acomp_req *req)
 {
+	struct iaa_req_ctx *req_ctx = acomp_request_ctx(req);
 	struct crypto_tfm *tfm = req->base.tfm;
 	dma_addr_t src_addr, dst_addr;
+	bool use_bounce_src = false;
 	int cpu, ret = 0;
 	struct iaa_wq *iaa_wq;
 	struct device *dev;
 	struct idxd_wq *wq;
+	struct page *page;
+
+	req_ctx->bounce_src = NULL;
+	req_ctx->bounce_src_dma = 0;
+	req_ctx->bounce_src_len = 0;
 
 	if (!iaa_crypto_enabled) {
 		pr_debug("iaa_crypto disabled, not decompressing\n");
@@ -1601,10 +1641,16 @@ static int iaa_comp_adecompress(struct acomp_req *req)
 		return -EINVAL;
 	}
 
-	/* Fall back to software if src or dst has multiple sg entries */
-	if (sg_nents(req->src) > 1 || sg_nents(req->dst) > 1)
+	/* Fall back to software if dst has multiple sg entries */
+	if (sg_nents(req->dst) > 1)
 		return deflate_generic_decompress(req);
 
+	if (sg_nents(req->src) > 1) {
+		if (req->slen > PAGE_SIZE)
+			return deflate_generic_decompress(req);
+		use_bounce_src = true;
+	}
+
 	cpu = get_cpu();
 	wq = wq_table_next_wq(cpu);
 	put_cpu();
@@ -1623,20 +1669,47 @@ static int iaa_comp_adecompress(struct acomp_req *req)
 
 	dev = &wq->idxd->pdev->dev;
 
-	if (!dma_map_sg(dev, req->src, 1, DMA_TO_DEVICE)) {
-		dev_dbg(dev, "couldn't map src sg for iaa device %d, wq %d\n",
-			iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id);
-		iaa_wq_put(wq);
-		return deflate_generic_decompress(req);
+	if (unlikely(use_bounce_src)) {
+		page = mempool_alloc(iaa_bounce_pool, GFP_ATOMIC);
+		if (!page) {
+			iaa_wq_put(wq);
+			return deflate_generic_decompress(req);
+		}
+
+		if (sg_copy_to_buffer(req->src, sg_nents(req->src),
+				      page_address(page), req->slen) != req->slen) {
+			mempool_free(page, iaa_bounce_pool);
+			iaa_wq_put(wq);
+			return deflate_generic_decompress(req);
+		}
+
+		src_addr = dma_map_page(dev, page, 0, req->slen, DMA_TO_DEVICE);
+		if (dma_mapping_error(dev, src_addr)) {
+			mempool_free(page, iaa_bounce_pool);
+			iaa_wq_put(wq);
+			return deflate_generic_decompress(req);
+		}
+
+		req_ctx->bounce_src = page;
+		req_ctx->bounce_src_dma = src_addr;
+		req_ctx->bounce_src_len = req->slen;
+	} else {
+		if (!dma_map_sg(dev, req->src, 1, DMA_TO_DEVICE)) {
+			dev_dbg(dev, "couldn't map src sg for iaa device %d, wq %d\n",
+				iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id);
+			iaa_wq_put(wq);
+			return deflate_generic_decompress(req);
+		}
+
+		src_addr = sg_dma_address(req->src);
+		dev_dbg(dev, "map src %llx req->src %p slen %d sg_len %d\n", src_addr,
+			req->src, req->slen, sg_dma_len(req->src));
 	}
-	src_addr = sg_dma_address(req->src);
-	dev_dbg(dev, "map src %llx req->src %p slen %d sg_len %d\n", src_addr,
-		req->src, req->slen, sg_dma_len(req->src));
 
 	if (!dma_map_sg(dev, req->dst, 1, DMA_FROM_DEVICE)) {
 		dev_dbg(dev, "couldn't map dst sg for iaa device %d, wq %d\n",
 			iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id);
-		dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE);
+		iaa_unmap_src(dev, req);
 		iaa_wq_put(wq);
 		return deflate_generic_decompress(req);
 	}
@@ -1653,7 +1726,7 @@ static int iaa_comp_adecompress(struct acomp_req *req)
 		dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret);
 
 	dma_unmap_sg(dev, req->dst, 1, DMA_FROM_DEVICE);
-	dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE);
+	iaa_unmap_src(dev, req);
 	iaa_wq_put(wq);
 
 	return ret;
@@ -1687,7 +1760,7 @@ static struct acomp_alg iaa_acomp_fixed_deflate = {
 		.cra_driver_name	= "deflate-iaa",
 		.cra_flags		= CRYPTO_ALG_ASYNC,
 		.cra_ctxsize		= sizeof(struct iaa_compression_ctx),
-		.cra_reqsize		= sizeof(u32),
+		.cra_reqsize		= sizeof(struct iaa_req_ctx),
 		.cra_module		= THIS_MODULE,
 		.cra_priority		= IAA_ALG_PRIORITY,
 	}
@@ -1886,6 +1959,12 @@ static int __init iaa_crypto_init_module(void)
 		goto err_aecs_init;
 	}
 
+	iaa_bounce_pool = mempool_create_page_pool(IAA_BOUNCE_POOL_SIZE, 0);
+	if (!iaa_bounce_pool) {
+		ret = -ENOMEM;
+		goto err_bounce_pool;
+	}
+
 	ret = idxd_driver_register(&iaa_crypto_driver);
 	if (ret) {
 		pr_debug("IAA wq sub-driver registration failed\n");
@@ -1919,6 +1998,9 @@ static int __init iaa_crypto_init_module(void)
 err_verify_attr_create:
 	idxd_driver_unregister(&iaa_crypto_driver);
 err_driver_reg:
+	mempool_destroy(iaa_bounce_pool);
+	iaa_bounce_pool = NULL;
+err_bounce_pool:
 	iaa_aecs_cleanup_fixed();
 err_aecs_init:
 
@@ -1935,6 +2017,8 @@ static void __exit iaa_crypto_cleanup_module(void)
 	driver_remove_file(&iaa_crypto_driver.drv,
 			   &driver_attr_verify_compress);
 	idxd_driver_unregister(&iaa_crypto_driver);
+	mempool_destroy(iaa_bounce_pool);
+	iaa_bounce_pool = NULL;
 	iaa_aecs_cleanup_fixed();
 
 	pr_debug("cleaned up\n");

-- 
2.55.0