From: Dave Jiang <dave.jiang@intel.com>
To: Vinicius Costa Gomes <vinicius.gomes@intel.com>,
Vinod Koul <vkoul@kernel.org>, Frank Li <Frank.Li@kernel.org>,
Kristen Accardi <kristen.c.accardi@intel.com>,
Herbert Xu <herbert@gondor.apana.org.au>,
"David S. Miller" <davem@davemloft.net>,
Andrew Morton <akpm@linux-foundation.org>,
Yosry Ahmed <yosry@kernel.org>, Nhat Pham <nphamcs@gmail.com>
Cc: dmaengine@vger.kernel.org, linux-kernel@vger.kernel.org,
linux-crypto@vger.kernel.org,
Giovanni Cabiddu <giovanni.cabiddu@intel.com>
Subject: Re: [PATCH 4/4] crypto: iaa - use bounce buffer for multi-sg decompress input
Date: Tue, 14 Jul 2026 15:01:20 -0700 [thread overview]
Message-ID: <d6b9b2e8-691b-4ccc-8000-ad2388994724@intel.com> (raw)
In-Reply-To: <20260713-iaa-crypto-fixes-zswap-v1-4-65cac23c684d@intel.com>
On 7/13/26 9:10 PM, Vinicius Costa Gomes wrote:
> From: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
>
> Since commit e2c3b6b21c77 ("mm: zswap: use SG list decompression APIs
> from zsmalloc"), zswap passes the raw zsmalloc SG list directly to
> crypto drivers, so a compressed object spanning multiple pages reaches
> IAA as a multi-entry source. Such requests currently fall back to
> software decompression.
>
> As IAA hardware requires a single DMA source buffer, linearize small
> multi-entry sources into a pre-allocated bounce page and submit that to
> the hardware instead of falling back to software. Keep the software
> fallback only for multi-entry destinations. This recovers most of the
> performance lost by using the software fallback.
>
> Store the bounce-page state in the acomp request context alongside the
> existing compression CRC, free it through a shared source-unmap helper,
> and back the pages with a small module-wide mempool so the path remains
> available in reclaim-driven callers.
>
> Signed-off-by: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
Missing Vinicius sign-off.
Reviewed-by: Dave Jiang <dave.jiang@intel.com>
> ---
> drivers/crypto/intel/iaa/iaa_crypto_main.c | 110 ++++++++++++++++++++++++-----
> 1 file changed, 92 insertions(+), 18 deletions(-)
>
> diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c
> index 8f68b1478476..54bde11c454c 100644
> --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c
> +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c
> @@ -9,6 +9,7 @@
> #include <linux/sysfs.h>
> #include <linux/device.h>
> #include <linux/iommu.h>
> +#include <linux/mempool.h>
> #include <uapi/linux/idxd.h>
> #include <linux/highmem.h>
> #include <linux/sched/smt.h>
> @@ -157,6 +158,15 @@ static bool async_mode;
> /* Use interrupts */
> static bool use_irq;
>
> +struct iaa_req_ctx {
> + u32 compression_crc;
> + struct page *bounce_src;
> + struct scatterlist bounce_sg;
> +};
> +
> +static mempool_t *iaa_bounce_pool;
> +#define IAA_BOUNCE_POOL_SIZE 128
> +
> /**
> * set_iaa_sync_mode - Set IAA sync mode
> * @name: The name of the sync mode
> @@ -984,6 +994,19 @@ static inline int check_completion(struct device *dev,
> return ret;
> }
>
> +static void iaa_unmap_src(struct device *dev, struct acomp_req *req)
> +{
> + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req);
> + struct scatterlist *src = req_ctx->bounce_src ? &req_ctx->bounce_sg : req->src;
> +
> + dma_unmap_sg(dev, src, 1, DMA_TO_DEVICE);
> +
> + if (req_ctx->bounce_src) {
> + mempool_free(req_ctx->bounce_src, iaa_bounce_pool);
> + req_ctx->bounce_src = NULL;
> + }
> +}
> +
> static int deflate_fallback(struct acomp_req *req, bool compress)
> {
> ACOMP_FBREQ_ON_STACK(fbreq, req);
> @@ -1040,6 +1063,7 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc,
> struct iaa_device_compression_mode *active_compression_mode;
> struct iaa_compression_ctx *compression_ctx;
> struct crypto_ctx *ctx = __ctx;
> + struct iaa_req_ctx *req_ctx = acomp_request_ctx(ctx->req);
> struct iaa_device *iaa_device;
> struct idxd_device *idxd;
> struct iaa_wq *iaa_wq;
> @@ -1098,10 +1122,9 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc,
> }
>
> if (ctx->compress && compression_ctx->verify_compress) {
> - u32 *compression_crc = acomp_request_ctx(ctx->req);
> dma_addr_t src_addr, dst_addr;
>
> - *compression_crc = idxd_desc->iax_completion->crc;
> + req_ctx->compression_crc = idxd_desc->iax_completion->crc;
>
> ret = iaa_remap_for_verify(dev, iaa_wq, ctx->req, &src_addr, &dst_addr);
> if (ret) {
> @@ -1124,7 +1147,7 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc,
> }
> err:
> dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_FROM_DEVICE);
> - dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_TO_DEVICE);
> + iaa_unmap_src(dev, ctx->req);
> out:
> if (ret != 0)
> dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret);
> @@ -1144,7 +1167,7 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req,
> {
> struct iaa_device_compression_mode *active_compression_mode;
> struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
> - u32 *compression_crc = acomp_request_ctx(req);
> + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req);
> struct iaa_device *iaa_device;
> struct idxd_desc *idxd_desc;
> struct iax_hw_desc *desc;
> @@ -1235,7 +1258,7 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req,
> update_total_comp_bytes_out(*dlen);
> update_wq_comp_bytes(wq, *dlen);
>
> - *compression_crc = idxd_desc->iax_completion->crc;
> + req_ctx->compression_crc = idxd_desc->iax_completion->crc;
>
> if (!ctx->async_mode)
> idxd_free_desc(wq, idxd_desc);
> @@ -1295,7 +1318,7 @@ static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req,
> {
> struct iaa_device_compression_mode *active_compression_mode;
> struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
> - u32 *compression_crc = acomp_request_ctx(req);
> + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req);
> struct iaa_device *iaa_device;
> struct idxd_desc *idxd_desc;
> struct iax_hw_desc *desc;
> @@ -1355,10 +1378,10 @@ static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req,
> goto err;
> }
>
> - if (*compression_crc != idxd_desc->iax_completion->crc) {
> + if (req_ctx->compression_crc != idxd_desc->iax_completion->crc) {
> ret = -EINVAL;
> - dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch:"
> - " comp=0x%x, decomp=0x%x\n", *compression_crc,
> + dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch: comp=0x%x, decomp=0x%x\n",
> + req_ctx->compression_crc,
> idxd_desc->iax_completion->crc);
> print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET,
> 8, 1, idxd_desc->iax_completion, 64, 0);
> @@ -1498,6 +1521,7 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req,
>
> static int iaa_comp_acompress(struct acomp_req *req)
> {
> + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req);
> struct iaa_compression_ctx *compression_ctx;
> struct crypto_tfm *tfm = req->base.tfm;
> dma_addr_t src_addr, dst_addr;
> @@ -1506,6 +1530,8 @@ static int iaa_comp_acompress(struct acomp_req *req)
> struct idxd_wq *wq;
> struct device *dev;
>
> + req_ctx->bounce_src = NULL;
> +
> compression_ctx = crypto_tfm_ctx(tfm);
>
> if (!iaa_crypto_enabled) {
> @@ -1597,12 +1623,18 @@ static int iaa_comp_acompress(struct acomp_req *req)
>
> static int iaa_comp_adecompress(struct acomp_req *req)
> {
> + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req);
> struct crypto_tfm *tfm = req->base.tfm;
> + struct scatterlist *src = req->src;
> dma_addr_t src_addr, dst_addr;
> + bool use_bounce_src = false;
> int cpu, ret = 0;
> struct iaa_wq *iaa_wq;
> struct device *dev;
> struct idxd_wq *wq;
> + struct page *page;
> +
> + req_ctx->bounce_src = NULL;
>
> if (!iaa_crypto_enabled) {
> pr_debug("iaa_crypto disabled, not decompressing\n");
> @@ -1614,10 +1646,16 @@ static int iaa_comp_adecompress(struct acomp_req *req)
> return -EINVAL;
> }
>
> - /* Fall back to software if src or dst has multiple sg entries */
> - if (sg_nents(req->src) > 1 || sg_nents(req->dst) > 1)
> + /* Fall back to software if dst has multiple sg entries */
> + if (sg_nents(req->dst) > 1)
> return deflate_generic_decompress(req);
>
> + if (sg_nents(req->src) > 1) {
> + if (req->slen > PAGE_SIZE)
> + return deflate_generic_decompress(req);
> + use_bounce_src = true;
> + }
> +
> cpu = get_cpu();
> wq = wq_table_next_wq(cpu);
> put_cpu();
> @@ -1636,20 +1674,45 @@ static int iaa_comp_adecompress(struct acomp_req *req)
>
> dev = &wq->idxd->pdev->dev;
>
> - if (!dma_map_sg(dev, req->src, 1, DMA_TO_DEVICE)) {
> + if (unlikely(use_bounce_src)) {
> + page = mempool_alloc(iaa_bounce_pool, GFP_ATOMIC);
> + if (!page) {
> + iaa_wq_put(wq);
> + return deflate_generic_decompress(req);
> + }
> +
> + if (sg_copy_to_buffer(req->src, sg_nents(req->src),
> + page_address(page), req->slen) != req->slen) {
> + mempool_free(page, iaa_bounce_pool);
> + iaa_wq_put(wq);
> + return deflate_generic_decompress(req);
> + }
> +
> + sg_init_table(&req_ctx->bounce_sg, 1);
> + sg_set_page(&req_ctx->bounce_sg, page, req->slen, 0);
> + req_ctx->bounce_src = page;
> + src = &req_ctx->bounce_sg;
> + }
> +
> + if (!dma_map_sg(dev, src, 1, DMA_TO_DEVICE)) {
> dev_dbg(dev, "couldn't map src sg for iaa device %d, wq %d\n",
> iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id);
> + if (req_ctx->bounce_src) {
> + mempool_free(req_ctx->bounce_src, iaa_bounce_pool);
> + req_ctx->bounce_src = NULL;
> + }
> iaa_wq_put(wq);
> return deflate_generic_decompress(req);
> }
> - src_addr = sg_dma_address(req->src);
> - dev_dbg(dev, "map src %llx req->src %p slen %d sg_len %d\n", src_addr,
> - req->src, req->slen, sg_dma_len(req->src));
> + src_addr = sg_dma_address(src);
> + dev_dbg(dev, "dma_map_sg, src_addr %llx, src %p,"
> + " req->slen %d, sg_dma_len(sg) %d\n", src_addr,
> + src, req->slen, sg_dma_len(src));
>
> if (!dma_map_sg(dev, req->dst, 1, DMA_FROM_DEVICE)) {
> dev_dbg(dev, "couldn't map dst sg for iaa device %d, wq %d\n",
> iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id);
> - dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE);
> + iaa_unmap_src(dev, req);
> iaa_wq_put(wq);
> return deflate_generic_decompress(req);
> }
> @@ -1666,7 +1729,7 @@ static int iaa_comp_adecompress(struct acomp_req *req)
> dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret);
>
> dma_unmap_sg(dev, req->dst, 1, DMA_FROM_DEVICE);
> - dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE);
> + iaa_unmap_src(dev, req);
> iaa_wq_put(wq);
>
> return ret;
> @@ -1700,7 +1763,7 @@ static struct acomp_alg iaa_acomp_fixed_deflate = {
> .cra_driver_name = "deflate-iaa",
> .cra_flags = CRYPTO_ALG_ASYNC,
> .cra_ctxsize = sizeof(struct iaa_compression_ctx),
> - .cra_reqsize = sizeof(u32),
> + .cra_reqsize = sizeof(struct iaa_req_ctx),
> .cra_module = THIS_MODULE,
> .cra_priority = IAA_ALG_PRIORITY,
> }
> @@ -1899,6 +1962,12 @@ static int __init iaa_crypto_init_module(void)
> goto err_aecs_init;
> }
>
> + iaa_bounce_pool = mempool_create_page_pool(IAA_BOUNCE_POOL_SIZE, 0);
> + if (!iaa_bounce_pool) {
> + ret = -ENOMEM;
> + goto err_bounce_pool;
> + }
> +
> ret = idxd_driver_register(&iaa_crypto_driver);
> if (ret) {
> pr_debug("IAA wq sub-driver registration failed\n");
> @@ -1932,6 +2001,9 @@ static int __init iaa_crypto_init_module(void)
> err_verify_attr_create:
> idxd_driver_unregister(&iaa_crypto_driver);
> err_driver_reg:
> + mempool_destroy(iaa_bounce_pool);
> + iaa_bounce_pool = NULL;
> +err_bounce_pool:
> iaa_aecs_cleanup_fixed();
> err_aecs_init:
>
> @@ -1948,6 +2020,8 @@ static void __exit iaa_crypto_cleanup_module(void)
> driver_remove_file(&iaa_crypto_driver.drv,
> &driver_attr_verify_compress);
> idxd_driver_unregister(&iaa_crypto_driver);
> + mempool_destroy(iaa_bounce_pool);
> + iaa_bounce_pool = NULL;
> iaa_aecs_cleanup_fixed();
>
> pr_debug("cleaned up\n");
>
prev parent reply other threads:[~2026-07-14 22:01 UTC|newest]
Thread overview: 11+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-07-14 4:10 [PATCH 0/4] crypto: iaa - Fixes for multi entry SG lists Vinicius Costa Gomes
2026-07-14 4:10 ` [PATCH 1/4] dmaengine: idxd: assign all engines to group 0 in IAA defaults Vinicius Costa Gomes
2026-07-14 21:53 ` Dave Jiang
2026-07-14 21:55 ` Dave Jiang
2026-07-15 1:38 ` Vinicius Costa Gomes
2026-07-14 4:10 ` [PATCH 2/4] crypto: iaa - fall back to software for multi-entry scatterlists Vinicius Costa Gomes
2026-07-14 21:54 ` Dave Jiang
2026-07-14 4:10 ` [PATCH 3/4] crypto: iaa - avoid counting fallback decompression bytes Vinicius Costa Gomes
2026-07-14 21:56 ` Dave Jiang
2026-07-14 4:10 ` [PATCH 4/4] crypto: iaa - use bounce buffer for multi-sg decompress input Vinicius Costa Gomes
2026-07-14 22:01 ` Dave Jiang [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=d6b9b2e8-691b-4ccc-8000-ad2388994724@intel.com \
--to=dave.jiang@intel.com \
--cc=Frank.Li@kernel.org \
--cc=akpm@linux-foundation.org \
--cc=davem@davemloft.net \
--cc=dmaengine@vger.kernel.org \
--cc=giovanni.cabiddu@intel.com \
--cc=herbert@gondor.apana.org.au \
--cc=kristen.c.accardi@intel.com \
--cc=linux-crypto@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=nphamcs@gmail.com \
--cc=vinicius.gomes@intel.com \
--cc=vkoul@kernel.org \
--cc=yosry@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox