* [PATCH 1/4] crypto: engine - clear busy and recheck running in the retry pump loop
2026-11-09 7:35 [PATCH 0/4] crypto: engine - add acomp support and adapt Hisilicon HPRE and ZIP drivers Chenghai Huang
@ 2026-11-09 7:35 ` Chenghai Huang
2026-11-09 7:35 ` [PATCH 2/4] crypto: engine - add acomp support Chenghai Huang
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Chenghai Huang @ 2026-11-09 7:35 UTC (permalink / raw)
To: herbert, davem
Cc: linux-kernel, linux-crypto, liulongfang, qianweili, wangzhou1,
linwenkai6
The retry pump loop jumps back to the dequeue path directly, skipping
the idle check that clears engine->busy and rechecks engine->running.
When the loop drains the queue it exits with busy still set, and
crypto_transfer_request() queues a pump only when the engine is not
busy: if the loop ends with the inline error completion in req_err_1,
which queues no pump, the next request is enqueued with nothing to run
it and the engine queue stalls. crypto_engine_stop() also spins
waiting for the stale busy flag, up to failing with -EBUSY, and the
loop keeps submitting to hardware after stop was requested because it
never rechecks running.
Jump to the idle check instead: the loop now clears busy and exits
when the queue is empty, and stops submitting once the engine is no
longer running. Engines without retry support are unaffected.
Fixes: 6a89f492f8e5 ("crypto: engine - support for parallel requests based on retry mechanism")
Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com>
---
crypto/crypto_engine.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/crypto/crypto_engine.c b/crypto/crypto_engine.c
index 3d07dd5de4fa..5b669038acb4 100644
--- a/crypto/crypto_engine.c
+++ b/crypto/crypto_engine.c
@@ -82,6 +82,7 @@ static void crypto_pump_requests(struct crypto_engine *engine,
if (!engine->retry_support && engine->cur_req)
goto out;
+check_idle:
/* Check if the engine queue is idle */
if (!crypto_queue_len(&engine->queue) || !engine->running) {
if (!engine->busy)
@@ -98,7 +99,6 @@ static void crypto_pump_requests(struct crypto_engine *engine,
goto out;
}
-start_request:
/* Get the fist request from the engine queue to handle */
backlog = crypto_get_backlog(&engine->queue);
async_req = crypto_dequeue_request(&engine->queue);
@@ -161,7 +161,7 @@ static void crypto_pump_requests(struct crypto_engine *engine,
/* If retry mechanism is supported, send new requests to engine */
if (engine->retry_support) {
spin_lock_irqsave(&engine->queue_lock, flags);
- goto start_request;
+ goto check_idle;
}
return;
--
2.43.0
^ permalink raw reply [flat|nested] 5+ messages in thread* [PATCH 2/4] crypto: engine - add acomp support
2026-11-09 7:35 [PATCH 0/4] crypto: engine - add acomp support and adapt Hisilicon HPRE and ZIP drivers Chenghai Huang
2026-11-09 7:35 ` [PATCH 1/4] crypto: engine - clear busy and recheck running in the retry pump loop Chenghai Huang
@ 2026-11-09 7:35 ` Chenghai Huang
2026-11-09 7:35 ` [PATCH 3/4] crypto: hisilicon/hpre - use crypto engine framework Chenghai Huang
2026-11-09 7:35 ` [PATCH 4/4] crypto: hisilicon/zip - adapt to use crypto engine Chenghai Huang
3 siblings, 0 replies; 5+ messages in thread
From: Chenghai Huang @ 2026-11-09 7:35 UTC (permalink / raw)
To: herbert, davem
Cc: linux-kernel, linux-crypto, liulongfang, qianweili, wangzhou1,
linwenkai6
Add support for asynchronous compression (acomp) algorithms to the
crypto engine framework: transfer/finalize helpers plus an
acomp_engine_alg registration interface, mirroring the existing
akcipher and kpp support.
As engine.h needs to embed struct acomp_alg by value, move it from
include/crypto/internal/acompress.h to include/crypto/acompress.h.
The internal header still provides it through the existing include.
Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com>
---
crypto/crypto_engine.c | 42 +++++++++++++++++++++++++++++
include/crypto/acompress.h | 33 +++++++++++++++++++++++
include/crypto/engine.h | 13 +++++++++
include/crypto/internal/acompress.h | 33 -----------------------
4 files changed, 88 insertions(+), 33 deletions(-)
diff --git a/crypto/crypto_engine.c b/crypto/crypto_engine.c
index 5b669038acb4..2a0740d9dd63 100644
--- a/crypto/crypto_engine.c
+++ b/crypto/crypto_engine.c
@@ -7,6 +7,7 @@
* Author: Baolin Wang <baolin.wang@linaro.org>
*/
+#include <crypto/internal/acompress.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/akcipher.h>
#include <crypto/internal/engine.h>
@@ -220,6 +221,19 @@ static int crypto_transfer_request_to_engine(struct crypto_engine *engine,
return crypto_transfer_request(engine, req, true);
}
+/**
+ * crypto_transfer_acomp_request_to_engine - transfer one acomp_request
+ * to list into the engine queue
+ * @engine: the hardware engine
+ * @req: the request need to be listed into the engine queue
+ */
+int crypto_transfer_acomp_request_to_engine(struct crypto_engine *engine,
+ struct acomp_req *req)
+{
+ return crypto_transfer_request_to_engine(engine, &req->base);
+}
+EXPORT_SYMBOL_GPL(crypto_transfer_acomp_request_to_engine);
+
/**
* crypto_transfer_aead_request_to_engine - transfer one aead_request
* to list into the engine queue
@@ -285,6 +299,20 @@ int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine,
}
EXPORT_SYMBOL_GPL(crypto_transfer_skcipher_request_to_engine);
+/**
+ * crypto_finalize_acomp_request - finalize one acomp_request if
+ * the request is done
+ * @engine: the hardware engine
+ * @req: the request need to be finalized
+ * @err: error number
+ */
+void crypto_finalize_acomp_request(struct crypto_engine *engine,
+ struct acomp_req *req, int err)
+{
+ return crypto_finalize_request(engine, &req->base, err);
+}
+EXPORT_SYMBOL_GPL(crypto_finalize_acomp_request);
+
/**
* crypto_finalize_aead_request - finalize one aead_request if
* the request is done
@@ -504,6 +532,20 @@ void crypto_engine_exit(struct crypto_engine *engine)
}
EXPORT_SYMBOL_GPL(crypto_engine_exit);
+int crypto_engine_register_acomp(struct acomp_engine_alg *alg)
+{
+ if (!alg->op.do_one_request)
+ return -EINVAL;
+ return crypto_register_acomp(&alg->base);
+}
+EXPORT_SYMBOL_GPL(crypto_engine_register_acomp);
+
+void crypto_engine_unregister_acomp(struct acomp_engine_alg *alg)
+{
+ crypto_unregister_acomp(&alg->base);
+}
+EXPORT_SYMBOL_GPL(crypto_engine_unregister_acomp);
+
int crypto_engine_register_aead(struct aead_engine_alg *alg)
{
if (!alg->op.do_one_request)
diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h
index 5d5358dfab73..0809fa35490a 100644
--- a/include/crypto/acompress.h
+++ b/include/crypto/acompress.h
@@ -122,6 +122,39 @@ struct crypto_acomp {
}
struct comp_alg_common COMP_ALG_COMMON;
+/**
+ * struct acomp_alg - asynchronous compression algorithm
+ *
+ * @compress: Function performs a compress operation
+ * @decompress: Function performs a de-compress operation
+ * @init: Initialize the cryptographic transformation object.
+ * This function is used to initialize the cryptographic
+ * transformation object. This function is called only once at
+ * the instantiation time, right after the transformation context
+ * was allocated. In case the cryptographic hardware has some
+ * special requirements which need to be handled by software, this
+ * function shall check for the precise requirement of the
+ * transformation and put any software fallbacks in place.
+ * @exit: Deinitialize the cryptographic transformation object. This is a
+ * counterpart to @init, used to remove various changes set in
+ * @init.
+ *
+ * @base: Common crypto API algorithm data structure
+ * @calg: Common algorithm data structure shared with scomp
+ * @COMP_ALG_COMMON: see struct comp_alg_common
+ */
+struct acomp_alg {
+ int (*compress)(struct acomp_req *req);
+ int (*decompress)(struct acomp_req *req);
+ int (*init)(struct crypto_acomp *tfm);
+ void (*exit)(struct crypto_acomp *tfm);
+
+ union {
+ struct COMP_ALG_COMMON;
+ struct comp_alg_common calg;
+ };
+};
+
/**
* DOC: Asynchronous Compression API
*
diff --git a/include/crypto/engine.h b/include/crypto/engine.h
index 2e60344437da..2b4e3d0e43ed 100644
--- a/include/crypto/engine.h
+++ b/include/crypto/engine.h
@@ -7,6 +7,7 @@
#ifndef _CRYPTO_ENGINE_H
#define _CRYPTO_ENGINE_H
+#include <crypto/acompress.h>
#include <crypto/aead.h>
#include <crypto/akcipher.h>
#include <crypto/hash.h>
@@ -26,6 +27,11 @@ struct crypto_engine_op {
void *areq);
};
+struct acomp_engine_alg {
+ struct acomp_alg base;
+ struct crypto_engine_op op;
+};
+
struct aead_engine_alg {
struct aead_alg base;
struct crypto_engine_op op;
@@ -51,6 +57,8 @@ struct skcipher_engine_alg {
struct crypto_engine_op op;
};
+int crypto_transfer_acomp_request_to_engine(struct crypto_engine *engine,
+ struct acomp_req *req);
int crypto_transfer_aead_request_to_engine(struct crypto_engine *engine,
struct aead_request *req);
int crypto_transfer_akcipher_request_to_engine(struct crypto_engine *engine,
@@ -61,6 +69,8 @@ int crypto_transfer_kpp_request_to_engine(struct crypto_engine *engine,
struct kpp_request *req);
int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine,
struct skcipher_request *req);
+void crypto_finalize_acomp_request(struct crypto_engine *engine,
+ struct acomp_req *req, int err);
void crypto_finalize_aead_request(struct crypto_engine *engine,
struct aead_request *req, int err);
void crypto_finalize_akcipher_request(struct crypto_engine *engine,
@@ -79,6 +89,9 @@ struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev,
bool rt, int qlen);
void crypto_engine_exit(struct crypto_engine *engine);
+int crypto_engine_register_acomp(struct acomp_engine_alg *alg);
+void crypto_engine_unregister_acomp(struct acomp_engine_alg *alg);
+
int crypto_engine_register_aead(struct aead_engine_alg *alg);
void crypto_engine_unregister_aead(struct aead_engine_alg *alg);
int crypto_engine_register_aeads(struct aead_engine_alg *algs, int count);
diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h
index 9cd37df32dc4..75580e007c0c 100644
--- a/include/crypto/internal/acompress.h
+++ b/include/crypto/internal/acompress.h
@@ -23,39 +23,6 @@
struct acomp_req *name = acomp_fbreq_on_stack_init( \
__##name##_req, (req))
-/**
- * struct acomp_alg - asynchronous compression algorithm
- *
- * @compress: Function performs a compress operation
- * @decompress: Function performs a de-compress operation
- * @init: Initialize the cryptographic transformation object.
- * This function is used to initialize the cryptographic
- * transformation object. This function is called only once at
- * the instantiation time, right after the transformation context
- * was allocated. In case the cryptographic hardware has some
- * special requirements which need to be handled by software, this
- * function shall check for the precise requirement of the
- * transformation and put any software fallbacks in place.
- * @exit: Deinitialize the cryptographic transformation object. This is a
- * counterpart to @init, used to remove various changes set in
- * @init.
- *
- * @base: Common crypto API algorithm data structure
- * @calg: Cmonn algorithm data structure shared with scomp
- * @COMP_ALG_COMMON: see struct comp_alg_common
- */
-struct acomp_alg {
- int (*compress)(struct acomp_req *req);
- int (*decompress)(struct acomp_req *req);
- int (*init)(struct crypto_acomp *tfm);
- void (*exit)(struct crypto_acomp *tfm);
-
- union {
- struct COMP_ALG_COMMON;
- struct comp_alg_common calg;
- };
-};
-
struct crypto_acomp_stream {
spinlock_t lock;
void *ctx;
--
2.43.0
^ permalink raw reply [flat|nested] 5+ messages in thread* [PATCH 3/4] crypto: hisilicon/hpre - use crypto engine framework
2026-11-09 7:35 [PATCH 0/4] crypto: engine - add acomp support and adapt Hisilicon HPRE and ZIP drivers Chenghai Huang
2026-11-09 7:35 ` [PATCH 1/4] crypto: engine - clear busy and recheck running in the retry pump loop Chenghai Huang
2026-11-09 7:35 ` [PATCH 2/4] crypto: engine - add acomp support Chenghai Huang
@ 2026-11-09 7:35 ` Chenghai Huang
2026-11-09 7:35 ` [PATCH 4/4] crypto: hisilicon/zip - adapt to use crypto engine Chenghai Huang
3 siblings, 0 replies; 5+ messages in thread
From: Chenghai Huang @ 2026-11-09 7:35 UTC (permalink / raw)
To: herbert, davem
Cc: linux-kernel, linux-crypto, liulongfang, qianweili, wangzhou1,
linwenkai6
From: Weili Qian <qianweili@huawei.com>
The HPRE driver has no software backlog queue for MAY_BACKLOG requests.
When the hardware queue is full, such requests fail instead of being
queued for later retry.
Migrate DH/RSA/ECDH to the crypto engine framework, which provides a
backlog queue and built-in retry on -ENOSPC. Convert akcipher_alg/kpp_alg
to akcipher_engine_alg/kpp_engine_alg and use the corresponding engine
register/unregister APIs.
Only MAY_BACKLOG requests are routed through the engine; the QM
hardware queue is deep enough to accept other requests directly, so
they skip the engine overhead and fail with -ENOSPC when full, per
the API contract. Non-backlog requests use a direct-send path with
bounded retries; the error code for a full queue changes from -EBUSY
to -ENOSPC in this path.
HPRE_SEND_CNT now counts successful sends instead of attempts.
Signed-off-by: Weili Qian <qianweili@huawei.com>
---
drivers/crypto/hisilicon/Kconfig | 1 +
drivers/crypto/hisilicon/hpre/hpre.h | 4 +
drivers/crypto/hisilicon/hpre/hpre_crypto.c | 414 +++++++++++++++-----
3 files changed, 314 insertions(+), 105 deletions(-)
diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig
index e38a71d96201..bd8390337d09 100644
--- a/drivers/crypto/hisilicon/Kconfig
+++ b/drivers/crypto/hisilicon/Kconfig
@@ -58,6 +58,7 @@ config CRYPTO_DEV_HISI_HPRE
select CRYPTO_DH
select CRYPTO_RSA
select CRYPTO_ECDH
+ select CRYPTO_ENGINE
help
Support for HiSilicon HPRE(High Performance RSA Engine)
accelerator, which can accelerate RSA and DH algorithms.
diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h
index f73d01f6da87..c1843d85bcd1 100644
--- a/drivers/crypto/hisilicon/hpre/hpre.h
+++ b/drivers/crypto/hisilicon/hpre/hpre.h
@@ -5,10 +5,12 @@
#include <linux/list.h>
#include <linux/hisi_acc_qm.h>
+#include <crypto/engine.h>
#define HPRE_SQE_SIZE sizeof(struct hpre_sqe)
#define HPRE_PF_DEF_Q_NUM 64
#define HPRE_PF_DEF_Q_BASE 0
+#define HPRE_MAX_CRYPTO_ENGINE 64
/*
* type used in qm sqc DW6.
@@ -70,6 +72,8 @@ struct hpre_debug {
struct hpre {
struct hisi_qm qm;
struct hpre_debug debug;
+ struct crypto_engine *engine[HPRE_MAX_CRYPTO_ENGINE];
+ u32 engine_num;
};
enum hpre_alg_type {
diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c
index 66a276786a7d..c02c33ba04d4 100644
--- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c
+++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c
@@ -4,6 +4,7 @@
#include <crypto/dh.h>
#include <crypto/ecc_curve.h>
#include <crypto/ecdh.h>
+#include <crypto/engine.h>
#include <crypto/rng.h>
#include <crypto/internal/akcipher.h>
#include <crypto/internal/kpp.h>
@@ -118,6 +119,7 @@ struct hpre_ctx {
/* for high performance core */
u8 enable_hpcore;
bool fallback;
+ struct crypto_engine *engine;
};
struct hpre_asym_request {
@@ -336,7 +338,12 @@ static void hpre_dh_cb(struct hpre_ctx *ctx, void *resp)
atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
- kpp_request_complete(areq, ret);
+ local_bh_disable();
+ if (areq->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
+ crypto_finalize_kpp_request(ctx->engine, areq, ret);
+ else
+ kpp_request_complete(areq, ret);
+ local_bh_enable();
atomic64_inc(&dfx[HPRE_RECV_CNT].value);
}
@@ -357,7 +364,12 @@ static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp)
areq = req->areq.rsa;
areq->dst_len = ctx->key_sz;
hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
- akcipher_request_complete(areq, ret);
+ local_bh_disable();
+ if (areq->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
+ crypto_finalize_akcipher_request(ctx->engine, areq, ret);
+ else
+ akcipher_request_complete(areq, ret);
+ local_bh_enable();
atomic64_inc(&dfx[HPRE_RECV_CNT].value);
}
@@ -386,10 +398,20 @@ static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type)
return -ENODEV;
}
+ hpre = container_of(qp->qm, struct hpre, qm);
+ if (!hpre->engine_num) {
+ hisi_qm_free_qps(&qp, 1);
+ return -ENODEV;
+ }
+ ctx->engine = hpre->engine[qp->qp_id % hpre->engine_num];
+ if (!ctx->engine) {
+ hisi_qm_free_qps(&qp, 1);
+ return -ENODEV;
+ }
+
qp->req_cb = hpre_alg_cb;
ctx->qp = qp;
ctx->dev = &qp->qm->pdev->dev;
- hpre = container_of(ctx->qp->qm, struct hpre, qm);
ctx->hpre = hpre;
ctx->key_sz = 0;
ctx->crt_g2_mode = false;
@@ -448,25 +470,29 @@ static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa)
return 0;
}
-static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg)
+static int hpre_alg_send_message_retry(struct hpre_asym_request *hpre_req)
{
- struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
- int ctr = 0;
+ struct hpre_dfx *dfx = hpre_req->ctx->hpre->debug.dfx;
+ struct hisi_qp *qp = hpre_req->ctx->qp;
+ int cnt = 0;
int ret;
do {
- atomic64_inc(&dfx[HPRE_SEND_CNT].value);
- ret = hisi_qp_send(ctx->qp, msg);
+ ret = hisi_qp_send(qp, &hpre_req->req);
if (ret != -EBUSY)
break;
atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value);
- } while (ctr++ < HPRE_TRY_SEND_TIMES);
+ } while (cnt++ < HPRE_TRY_SEND_TIMES);
- if (likely(!ret))
- return ret;
+ if (ret == -EBUSY)
+ return -ENOSPC;
+
+ if (likely(!ret)) {
+ atomic64_inc(&dfx[HPRE_SEND_CNT].value);
+ return -EINPROGRESS;
+ }
- if (ret != -EBUSY)
- atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value);
+ atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value);
return ret;
}
@@ -501,11 +527,16 @@ static int hpre_dh_compute_value(struct kpp_request *req)
else
msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH);
- /* success */
- ret = hpre_send(ctx, msg);
- if (likely(!ret))
- return -EINPROGRESS;
+ if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG) {
+ ret = crypto_transfer_kpp_request_to_engine(ctx->engine, req);
+ if (ret == -EINPROGRESS || ret == -EBUSY)
+ return ret;
+ goto clear_all;
+ }
+ ret = hpre_alg_send_message_retry(hpre_req);
+ if (ret == -EINPROGRESS)
+ return ret;
clear_all:
hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
@@ -779,10 +810,16 @@ static int hpre_rsa_enc(struct akcipher_request *req)
if (unlikely(ret))
goto clear_all;
- /* success */
- ret = hpre_send(ctx, msg);
- if (likely(!ret))
- return -EINPROGRESS;
+ if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG) {
+ ret = crypto_transfer_akcipher_request_to_engine(ctx->engine, req);
+ if (ret == -EINPROGRESS || ret == -EBUSY)
+ return ret;
+ goto clear_all;
+ }
+
+ ret = hpre_alg_send_message_retry(hpre_req);
+ if (ret == -EINPROGRESS)
+ return ret;
clear_all:
hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
@@ -832,11 +869,16 @@ static int hpre_rsa_dec(struct akcipher_request *req)
if (unlikely(ret))
goto clear_all;
- /* success */
- ret = hpre_send(ctx, msg);
- if (likely(!ret))
- return -EINPROGRESS;
+ if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG) {
+ ret = crypto_transfer_akcipher_request_to_engine(ctx->engine, req);
+ if (ret == -EINPROGRESS || ret == -EBUSY)
+ return ret;
+ goto clear_all;
+ }
+ ret = hpre_alg_send_message_retry(hpre_req);
+ if (ret == -EINPROGRESS)
+ return ret;
clear_all:
hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
@@ -1433,8 +1475,12 @@ static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp)
p = sg_virt(areq->dst);
memmove(p, p + ctx->key_sz - curve_sz, curve_sz);
memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz);
-
- kpp_request_complete(areq, ret);
+ local_bh_disable();
+ if (areq->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
+ crypto_finalize_kpp_request(ctx->engine, areq, ret);
+ else
+ kpp_request_complete(areq, ret);
+ local_bh_enable();
atomic64_inc(&dfx[HPRE_RECV_CNT].value);
}
@@ -1561,9 +1607,16 @@ static int hpre_ecdh_compute_value(struct kpp_request *req)
msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_ECC_MUL);
msg->resv1 = ctx->enable_hpcore << HPRE_ENABLE_HPCORE_SHIFT;
- ret = hpre_send(ctx, msg);
- if (likely(!ret))
- return -EINPROGRESS;
+ if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG) {
+ ret = crypto_transfer_kpp_request_to_engine(ctx->engine, req);
+ if (ret == -EINPROGRESS || ret == -EBUSY)
+ return ret;
+ goto clear_all;
+ }
+
+ ret = hpre_alg_send_message_retry(hpre_req);
+ if (ret == -EINPROGRESS)
+ return ret;
clear_all:
hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
@@ -1679,86 +1732,176 @@ static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm)
hpre_ecc_clear_ctx(ctx, true);
}
-static struct akcipher_alg rsa = {
- .encrypt = hpre_rsa_enc,
- .decrypt = hpre_rsa_dec,
- .set_pub_key = hpre_rsa_setpubkey,
- .set_priv_key = hpre_rsa_setprivkey,
- .max_size = hpre_rsa_max_size,
- .init = hpre_rsa_init_tfm,
- .exit = hpre_rsa_exit_tfm,
- .base = {
- .cra_ctxsize = sizeof(struct hpre_ctx),
- .cra_priority = HPRE_CRYPTO_ALG_PRI,
- .cra_name = "rsa",
- .cra_driver_name = "hpre-rsa",
- .cra_module = THIS_MODULE,
- .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
- },
-};
+static int hpre_alg_send_message(struct hpre_asym_request *hpre_req)
+{
+ struct hpre_dfx *dfx = hpre_req->ctx->hpre->debug.dfx;
+ struct hisi_qp *qp = hpre_req->ctx->qp;
+ int ret;
-static struct kpp_alg dh = {
- .set_secret = hpre_dh_set_secret,
- .generate_public_key = hpre_dh_generate_public_key,
- .compute_shared_secret = hpre_dh_compute_shared_secret,
- .max_size = hpre_dh_max_size,
- .init = hpre_dh_init_tfm,
- .exit = hpre_dh_exit_tfm,
- .base = {
- .cra_ctxsize = sizeof(struct hpre_ctx),
- .cra_priority = HPRE_CRYPTO_ALG_PRI,
- .cra_name = "dh",
- .cra_driver_name = "hpre-dh",
- .cra_module = THIS_MODULE,
- .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
- },
-};
+ ret = hisi_qp_send(qp, &hpre_req->req);
+ if (ret == -EBUSY) {
+ atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value);
+ return -ENOSPC;
+ }
-static struct kpp_alg ecdh_curves[] = {
- {
- .set_secret = hpre_ecdh_set_secret,
- .generate_public_key = hpre_ecdh_generate_public_key,
- .compute_shared_secret = hpre_ecdh_compute_shared_secret,
- .max_size = hpre_ecdh_max_size,
- .init = hpre_ecdh_nist_p192_init_tfm,
- .exit = hpre_ecdh_exit_tfm,
+ if (likely(!ret)) {
+ atomic64_inc(&dfx[HPRE_SEND_CNT].value);
+ return 0;
+ }
+
+ atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value);
+
+ return ret;
+}
+
+static int hpre_dh_do_one_request(struct crypto_engine *engine, void *areq)
+{
+ struct kpp_request *kreq = container_of(areq, struct kpp_request, base);
+ void *tmp = kpp_request_ctx(kreq);
+ struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
+ int ret;
+
+ ret = hpre_alg_send_message(hpre_req);
+ if (ret && ret != -ENOSPC)
+ hpre_hw_data_clr_all(hpre_req->ctx, hpre_req, kreq->dst, kreq->src);
+
+ return ret;
+}
+
+static int hpre_ecdh_do_one_request(struct crypto_engine *engine, void *areq)
+{
+ struct kpp_request *kreq = container_of(areq, struct kpp_request, base);
+ void *tmp = kpp_request_ctx(kreq);
+ struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
+ int ret;
+
+ ret = hpre_alg_send_message(hpre_req);
+ if (ret && ret != -ENOSPC)
+ hpre_ecdh_hw_data_clr_all(hpre_req->ctx, hpre_req, kreq->dst, kreq->src);
+
+ return ret;
+}
+
+static int hpre_rsa_do_one_request(struct crypto_engine *engine, void *areq)
+{
+ struct akcipher_request *akreq =
+ container_of(areq, struct akcipher_request, base);
+ void *tmp = akcipher_request_ctx(akreq);
+ struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
+ int ret;
+
+ ret = hpre_alg_send_message(hpre_req);
+ if (ret && ret != -ENOSPC)
+ hpre_hw_data_clr_all(hpre_req->ctx, hpre_req, akreq->dst, akreq->src);
+
+ return ret;
+}
+
+static struct akcipher_engine_alg rsa = {
+ .base = {
+ .encrypt = hpre_rsa_enc,
+ .decrypt = hpre_rsa_dec,
+ .set_pub_key = hpre_rsa_setpubkey,
+ .set_priv_key = hpre_rsa_setprivkey,
+ .max_size = hpre_rsa_max_size,
+ .init = hpre_rsa_init_tfm,
+ .exit = hpre_rsa_exit_tfm,
.base = {
.cra_ctxsize = sizeof(struct hpre_ctx),
.cra_priority = HPRE_CRYPTO_ALG_PRI,
- .cra_name = "ecdh-nist-p192",
- .cra_driver_name = "hpre-ecdh-nist-p192",
+ .cra_name = "rsa",
+ .cra_driver_name = "hpre-rsa",
.cra_module = THIS_MODULE,
.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
},
- }, {
- .set_secret = hpre_ecdh_set_secret,
- .generate_public_key = hpre_ecdh_generate_public_key,
- .compute_shared_secret = hpre_ecdh_compute_shared_secret,
- .max_size = hpre_ecdh_max_size,
- .init = hpre_ecdh_nist_p256_init_tfm,
- .exit = hpre_ecdh_exit_tfm,
+ },
+ .op = {
+ .do_one_request = hpre_rsa_do_one_request,
+ },
+};
+
+static struct kpp_engine_alg dh = {
+ .base = {
+ .set_secret = hpre_dh_set_secret,
+ .generate_public_key = hpre_dh_generate_public_key,
+ .compute_shared_secret = hpre_dh_compute_shared_secret,
+ .max_size = hpre_dh_max_size,
+ .init = hpre_dh_init_tfm,
+ .exit = hpre_dh_exit_tfm,
.base = {
.cra_ctxsize = sizeof(struct hpre_ctx),
.cra_priority = HPRE_CRYPTO_ALG_PRI,
- .cra_name = "ecdh-nist-p256",
- .cra_driver_name = "hpre-ecdh-nist-p256",
+ .cra_name = "dh",
+ .cra_driver_name = "hpre-dh",
.cra_module = THIS_MODULE,
.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
},
+ },
+ .op = {
+ .do_one_request = hpre_dh_do_one_request,
+ },
+};
+
+static struct kpp_engine_alg ecdh_curves[] = {
+ {
+ .base = {
+ .set_secret = hpre_ecdh_set_secret,
+ .generate_public_key = hpre_ecdh_generate_public_key,
+ .compute_shared_secret = hpre_ecdh_compute_shared_secret,
+ .max_size = hpre_ecdh_max_size,
+ .init = hpre_ecdh_nist_p192_init_tfm,
+ .exit = hpre_ecdh_exit_tfm,
+ .base = {
+ .cra_ctxsize = sizeof(struct hpre_ctx),
+ .cra_priority = HPRE_CRYPTO_ALG_PRI,
+ .cra_name = "ecdh-nist-p192",
+ .cra_driver_name = "hpre-ecdh-nist-p192",
+ .cra_module = THIS_MODULE,
+ .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
+ },
+ },
+ .op = {
+ .do_one_request = hpre_ecdh_do_one_request,
+ },
}, {
- .set_secret = hpre_ecdh_set_secret,
- .generate_public_key = hpre_ecdh_generate_public_key,
- .compute_shared_secret = hpre_ecdh_compute_shared_secret,
- .max_size = hpre_ecdh_max_size,
- .init = hpre_ecdh_nist_p384_init_tfm,
- .exit = hpre_ecdh_exit_tfm,
.base = {
- .cra_ctxsize = sizeof(struct hpre_ctx),
- .cra_priority = HPRE_CRYPTO_ALG_PRI,
- .cra_name = "ecdh-nist-p384",
- .cra_driver_name = "hpre-ecdh-nist-p384",
- .cra_module = THIS_MODULE,
- .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
+ .set_secret = hpre_ecdh_set_secret,
+ .generate_public_key = hpre_ecdh_generate_public_key,
+ .compute_shared_secret = hpre_ecdh_compute_shared_secret,
+ .max_size = hpre_ecdh_max_size,
+ .init = hpre_ecdh_nist_p256_init_tfm,
+ .exit = hpre_ecdh_exit_tfm,
+ .base = {
+ .cra_ctxsize = sizeof(struct hpre_ctx),
+ .cra_priority = HPRE_CRYPTO_ALG_PRI,
+ .cra_name = "ecdh-nist-p256",
+ .cra_driver_name = "hpre-ecdh-nist-p256",
+ .cra_module = THIS_MODULE,
+ .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
+ },
+ },
+ .op = {
+ .do_one_request = hpre_ecdh_do_one_request,
+ },
+ }, {
+ .base = {
+ .set_secret = hpre_ecdh_set_secret,
+ .generate_public_key = hpre_ecdh_generate_public_key,
+ .compute_shared_secret = hpre_ecdh_compute_shared_secret,
+ .max_size = hpre_ecdh_max_size,
+ .init = hpre_ecdh_nist_p384_init_tfm,
+ .exit = hpre_ecdh_exit_tfm,
+ .base = {
+ .cra_ctxsize = sizeof(struct hpre_ctx),
+ .cra_priority = HPRE_CRYPTO_ALG_PRI,
+ .cra_name = "ecdh-nist-p384",
+ .cra_driver_name = "hpre-ecdh-nist-p384",
+ .cra_module = THIS_MODULE,
+ .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
+ },
+ },
+ .op = {
+ .do_one_request = hpre_ecdh_do_one_request,
},
}
};
@@ -1770,8 +1913,7 @@ static int hpre_register_rsa(struct hisi_qm *qm)
if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP))
return 0;
- rsa.base.cra_flags = 0;
- ret = crypto_register_akcipher(&rsa);
+ ret = crypto_engine_register_akcipher(&rsa);
if (ret)
dev_err(&qm->pdev->dev, "failed to register rsa (%d)!\n", ret);
@@ -1783,7 +1925,7 @@ static void hpre_unregister_rsa(struct hisi_qm *qm)
if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP))
return;
- crypto_unregister_akcipher(&rsa);
+ crypto_engine_unregister_akcipher(&rsa);
}
static int hpre_register_dh(struct hisi_qm *qm)
@@ -1793,7 +1935,7 @@ static int hpre_register_dh(struct hisi_qm *qm)
if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP))
return 0;
- ret = crypto_register_kpp(&dh);
+ ret = crypto_engine_register_kpp(&dh);
if (ret)
dev_err(&qm->pdev->dev, "failed to register dh (%d)!\n", ret);
@@ -1805,7 +1947,7 @@ static void hpre_unregister_dh(struct hisi_qm *qm)
if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP))
return;
- crypto_unregister_kpp(&dh);
+ crypto_engine_unregister_kpp(&dh);
}
static int hpre_register_ecdh(struct hisi_qm *qm)
@@ -1816,10 +1958,10 @@ static int hpre_register_ecdh(struct hisi_qm *qm)
return 0;
for (i = 0; i < ARRAY_SIZE(ecdh_curves); i++) {
- ret = crypto_register_kpp(&ecdh_curves[i]);
+ ret = crypto_engine_register_kpp(&ecdh_curves[i]);
if (ret) {
dev_err(&qm->pdev->dev, "failed to register %s (%d)!\n",
- ecdh_curves[i].base.cra_name, ret);
+ ecdh_curves[i].base.base.cra_name, ret);
goto unreg_kpp;
}
}
@@ -1828,7 +1970,7 @@ static int hpre_register_ecdh(struct hisi_qm *qm)
unreg_kpp:
for (--i; i >= 0; --i)
- crypto_unregister_kpp(&ecdh_curves[i]);
+ crypto_engine_unregister_kpp(&ecdh_curves[i]);
return ret;
}
@@ -1841,17 +1983,76 @@ static void hpre_unregister_ecdh(struct hisi_qm *qm)
return;
for (i = ARRAY_SIZE(ecdh_curves) - 1; i >= 0; --i)
- crypto_unregister_kpp(&ecdh_curves[i]);
+ crypto_engine_unregister_kpp(&ecdh_curves[i]);
+}
+
+static void hpre_free_crypto_engine(struct hisi_qm *qm)
+{
+ struct hpre *hpre = container_of(qm, struct hpre, qm);
+ u32 i;
+
+ for (i = 0; i < hpre->engine_num; i++) {
+ if (hpre->engine[i])
+ crypto_engine_exit(hpre->engine[i]);
+ }
+
+ hpre->engine_num = 0;
+}
+
+static int hpre_alloc_crypto_engine(struct hisi_qm *qm)
+{
+ u32 engine_num = min_t(u32, qm->qp_num, HPRE_MAX_CRYPTO_ENGINE);
+ struct hpre *hpre = container_of(qm, struct hpre, qm);
+ struct device *dev = &qm->pdev->dev;
+ u16 qlen = qm->qp_array[0].sq_depth;
+ int ret;
+ u32 i;
+
+ for (i = 0; i < engine_num; i++) {
+ hpre->engine[i] =
+ crypto_engine_alloc_init_and_set(dev, true,
+ false, qlen);
+ if (!hpre->engine[i]) {
+ dev_err(dev, "failed to alloc crypto engine.\n");
+ ret = -ENOMEM;
+ goto free_engine;
+ }
+
+ ret = crypto_engine_start(hpre->engine[i]);
+ if (ret) {
+ dev_err(dev, "failed to start crypto engine.\n");
+ crypto_engine_exit(hpre->engine[i]);
+ hpre->engine[i] = NULL;
+ goto free_engine;
+ }
+ }
+
+ hpre->engine_num = engine_num;
+
+ return 0;
+
+free_engine:
+ for (i = 0; i < engine_num; i++) {
+ if (hpre->engine[i])
+ crypto_engine_exit(hpre->engine[i]);
+ }
+
+ return ret;
}
int hpre_algs_register(struct hisi_qm *qm)
{
- int ret = 0;
+ int ret;
+
+ ret = hpre_alloc_crypto_engine(qm);
+ if (ret)
+ return ret;
mutex_lock(&hpre_algs_lock);
if (hpre_available_devs) {
hpre_available_devs++;
- goto unlock;
+ mutex_unlock(&hpre_algs_lock);
+ return 0;
}
ret = hpre_register_rsa(qm);
@@ -1877,6 +2078,8 @@ int hpre_algs_register(struct hisi_qm *qm)
hpre_unregister_rsa(qm);
unlock:
mutex_unlock(&hpre_algs_lock);
+ hpre_free_crypto_engine(qm);
+
return ret;
}
@@ -1892,4 +2095,5 @@ void hpre_algs_unregister(struct hisi_qm *qm)
unlock:
mutex_unlock(&hpre_algs_lock);
+ hpre_free_crypto_engine(qm);
}
--
2.43.0
^ permalink raw reply [flat|nested] 5+ messages in thread* [PATCH 4/4] crypto: hisilicon/zip - adapt to use crypto engine
2026-11-09 7:35 [PATCH 0/4] crypto: engine - add acomp support and adapt Hisilicon HPRE and ZIP drivers Chenghai Huang
` (2 preceding siblings ...)
2026-11-09 7:35 ` [PATCH 3/4] crypto: hisilicon/hpre - use crypto engine framework Chenghai Huang
@ 2026-11-09 7:35 ` Chenghai Huang
3 siblings, 0 replies; 5+ messages in thread
From: Chenghai Huang @ 2026-11-09 7:35 UTC (permalink / raw)
To: herbert, davem
Cc: linux-kernel, linux-crypto, liulongfang, qianweili, wangzhou1,
linwenkai6
The ZIP driver has no backlog queue: when the hardware queue is full,
requests fail immediately.
Migrate the ZIP driver to the crypto engine framework. Each QP context
borrows an engine from a per-device pool by its QP id, so compression
and decompression queue independently. Requests are sent to the
hardware when it has room; on -ENOSPC they are parked in the engine
queue and retried with their DMA mappings kept, so transient queue
pressure never fails a request. The per-request state moves into the
request context so the engine can retry queued requests, and the soft
fallback for QP allocation failures is kept.
Also make CRYPTO_DEV_HISI_ZIP select CRYPTO_ENGINE.
Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com>
---
drivers/crypto/hisilicon/Kconfig | 1 +
drivers/crypto/hisilicon/zip/zip.h | 5 +
drivers/crypto/hisilicon/zip/zip_crypto.c | 358 +++++++++++++++-------
3 files changed, 245 insertions(+), 119 deletions(-)
diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig
index bd8390337d09..e280b264e427 100644
--- a/drivers/crypto/hisilicon/Kconfig
+++ b/drivers/crypto/hisilicon/Kconfig
@@ -45,6 +45,7 @@ config CRYPTO_DEV_HISI_ZIP
select CRYPTO_DEV_HISI_QM
select CRYPTO_DEFLATE
select CRYPTO_LZ4
+ select CRYPTO_ENGINE
help
Support for HiSilicon ZIP Driver
diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h
index b83f228281ab..9745703df03c 100644
--- a/drivers/crypto/hisilicon/zip/zip.h
+++ b/drivers/crypto/hisilicon/zip/zip.h
@@ -8,6 +8,9 @@
#include <linux/list.h>
#include <linux/hisi_acc_qm.h>
+#include <crypto/engine.h>
+
+#define HZIP_MAX_CRYPTO_ENGINE 64
enum hisi_zip_error_type {
/* negative compression */
@@ -27,6 +30,8 @@ struct hisi_zip {
struct hisi_qm qm;
struct hisi_zip_ctrl *ctrl;
struct hisi_zip_dfx dfx;
+ struct crypto_engine *engine[HZIP_MAX_CRYPTO_ENGINE];
+ u32 engine_num;
};
struct hisi_zip_sqe {
diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c
index 70adde049b53..84951bc74a96 100644
--- a/drivers/crypto/hisilicon/zip/zip_crypto.c
+++ b/drivers/crypto/hisilicon/zip/zip_crypto.c
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2019 HiSilicon Limited. */
+#include <crypto/engine.h>
#include <crypto/internal/acompress.h>
#include <linux/bitfield.h>
#include <linux/bitmap.h>
@@ -25,6 +26,7 @@
#define HZIP_ALG_PRIORITY 300
#define HZIP_SGL_SGE_NR 10
+#define HZIP_INVAL_REQ_ID ((u16)0xFFFF)
#define HZIP_ALG_DEFLATE GENMASK(5, 4)
#define HZIP_ALG_LZ4 BIT(8)
@@ -56,10 +58,11 @@ struct hisi_zip_req {
dma_addr_t dma_dst;
struct hisi_zip_qp_ctx *qp_ctx;
u16 req_id;
+ enum hisi_zip_alg_type alg_type;
+ bool buf_ready;
};
struct hisi_zip_req_q {
- struct hisi_zip_req *q;
unsigned long *req_bitmap;
spinlock_t req_lock;
u16 size;
@@ -71,7 +74,7 @@ struct hisi_zip_qp_ctx {
struct hisi_acc_sgl_pool *sgl_pool;
struct hisi_zip *zip_dev;
struct hisi_zip_ctx *ctx;
- u8 req_type;
+ struct crypto_engine *engine;
};
struct hisi_zip_sqe_ops {
@@ -91,6 +94,7 @@ struct hisi_zip_ctx {
struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM];
const struct hisi_zip_sqe_ops *ops;
bool fallback;
+ u8 req_type;
};
static int sgl_sge_nr_set(const char *val, const struct kernel_param *kp)
@@ -135,12 +139,10 @@ static int hisi_zip_fallback_do_work(struct acomp_req *acomp_req, bool is_decomp
return ret;
}
-static struct hisi_zip_req *hisi_zip_create_req(struct hisi_zip_qp_ctx *qp_ctx,
- struct acomp_req *req)
+static int hisi_zip_create_req(struct hisi_zip_req *req)
{
+ struct hisi_zip_qp_ctx *qp_ctx = req->qp_ctx;
struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
- struct hisi_zip_req *q = req_q->q;
- struct hisi_zip_req *req_cache;
int req_id;
spin_lock(&req_q->req_lock);
@@ -149,28 +151,26 @@ static struct hisi_zip_req *hisi_zip_create_req(struct hisi_zip_qp_ctx *qp_ctx,
if (req_id >= req_q->size) {
spin_unlock(&req_q->req_lock);
dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n");
- return ERR_PTR(-EAGAIN);
+ return -EBUSY;
}
set_bit(req_id, req_q->req_bitmap);
spin_unlock(&req_q->req_lock);
- req_cache = q + req_id;
- req_cache->req_id = req_id;
- req_cache->req = req;
- req_cache->qp_ctx = qp_ctx;
+ req->req_id = req_id;
- return req_cache;
+ return 0;
}
-static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx,
- struct hisi_zip_req *req)
+static void hisi_zip_remove_req(struct hisi_zip_req *req)
{
- struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
+ struct hisi_zip_req_q *req_q = &req->qp_ctx->req_q;
spin_lock(&req_q->req_lock);
clear_bit(req->req_id, req_q->req_bitmap);
spin_unlock(&req_q->req_lock);
+
+ req->req_id = HZIP_INVAL_REQ_ID;
}
static void hisi_zip_fill_addr(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req)
@@ -232,7 +232,7 @@ static void hisi_zip_fill_sqe_type(struct hisi_zip_sqe *sqe, u8 sqe_type)
}
static void hisi_zip_fill_sqe(struct hisi_zip_ctx *ctx, struct hisi_zip_sqe *sqe,
- u8 req_type, struct hisi_zip_req *req)
+ struct hisi_zip_req *req)
{
const struct hisi_zip_sqe_ops *ops = ctx->ops;
@@ -241,25 +241,18 @@ static void hisi_zip_fill_sqe(struct hisi_zip_ctx *ctx, struct hisi_zip_sqe *sqe
ops->fill_addr(sqe, req);
ops->fill_buf_size(sqe, req);
ops->fill_buf_type(sqe, HZIP_SGL);
- ops->fill_req_type(sqe, req_type);
+ ops->fill_req_type(sqe, ctx->req_type);
ops->fill_win_size(sqe, HZIP_16K_WINSZ);
ops->fill_tag(sqe, req);
ops->fill_sqe_type(sqe, ops->sqe_type);
}
-static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx,
- struct hisi_zip_req *req)
+static int hisi_zip_map_req_buffers(struct hisi_zip_req *req)
{
+ struct hisi_zip_qp_ctx *qp_ctx = req->qp_ctx;
struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool;
- struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx;
+ struct device *dev = &qp_ctx->qp->qm->pdev->dev;
struct acomp_req *a_req = req->req;
- struct hisi_qp *qp = qp_ctx->qp;
- struct device *dev = &qp->qm->pdev->dev;
- struct hisi_zip_sqe zip_sqe;
- int ret;
-
- if (unlikely(!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen))
- return -EINVAL;
req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool,
req->req_id << 1, &req->dma_src,
@@ -274,30 +267,49 @@ static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx,
(req->req_id << 1) + 1,
&req->dma_dst, DMA_FROM_DEVICE);
if (IS_ERR(req->hw_dst)) {
- ret = PTR_ERR(req->hw_dst);
- dev_err(dev, "failed to map the dst buffer to hw sgl (%d)!\n",
- ret);
- goto err_unmap_input;
+ dev_err(dev, "failed to map the dst buffer to hw sgl (%ld)!\n",
+ PTR_ERR(req->hw_dst));
+ hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src, DMA_TO_DEVICE);
+ return PTR_ERR(req->hw_dst);
}
- hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp_ctx->req_type, req);
+ req->buf_ready = true;
+
+ return 0;
+}
+
+static void hisi_zip_unmap_req_buffers(struct hisi_zip_req *req)
+{
+ struct hisi_zip_qp_ctx *qp_ctx = req->qp_ctx;
+ struct device *dev = &qp_ctx->qp->qm->pdev->dev;
+ struct acomp_req *a_req = req->req;
+
+ hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst, DMA_FROM_DEVICE);
+ hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src, DMA_TO_DEVICE);
+
+ req->buf_ready = false;
+}
+
+static int hisi_zip_do_work(struct hisi_zip_req *req)
+{
+ struct hisi_zip_qp_ctx *qp_ctx = req->qp_ctx;
+ struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx;
+ struct hisi_qp *qp = qp_ctx->qp;
+ struct hisi_zip_sqe zip_sqe;
+ int ret;
+
+ hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, req);
/* send command to start a task */
- atomic64_inc(&dfx->send_cnt);
ret = hisi_qp_send(qp, &zip_sqe);
- if (unlikely(ret < 0)) {
- atomic64_inc(&dfx->send_busy_cnt);
- ret = -EAGAIN;
- dev_dbg_ratelimited(dev, "failed to send request!\n");
- goto err_unmap_output;
+ if (likely(!ret)) {
+ atomic64_inc(&dfx->send_cnt);
+ return -EINPROGRESS;
}
- return -EINPROGRESS;
+ if (ret == -EBUSY)
+ atomic64_inc(&dfx->send_busy_cnt);
-err_unmap_output:
- hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst, DMA_FROM_DEVICE);
-err_unmap_input:
- hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src, DMA_TO_DEVICE);
return ret;
}
@@ -333,68 +345,111 @@ static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data)
err = -EIO;
}
- hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst, DMA_FROM_DEVICE);
- hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src, DMA_TO_DEVICE);
+ hisi_zip_unmap_req_buffers(req);
acomp_req->dlen = ops->get_dstlen(sqe);
+ hisi_zip_remove_req(req);
- if (acomp_req->base.complete)
- acomp_request_complete(acomp_req, err);
-
- hisi_zip_remove_req(qp_ctx, req);
+ local_bh_disable();
+ crypto_finalize_acomp_request(qp_ctx->engine, acomp_req, err);
+ local_bh_enable();
}
-static int hisi_zip_acompress(struct acomp_req *acomp_req)
+static int hisi_zip_do_comp(struct hisi_zip_req *req)
{
- struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
- struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP];
- struct hisi_zip_req *req;
- struct device *dev;
+ struct hisi_qp *qp = req->qp_ctx->qp;
+ struct device *dev = &qp->qm->pdev->dev;
int ret;
- if (ctx->fallback)
- return hisi_zip_fallback_do_work(acomp_req, 0);
+ if (!req->buf_ready) {
+ ret = hisi_zip_create_req(req);
+ if (unlikely(ret))
+ return -ENOSPC;
+
+ ret = hisi_zip_map_req_buffers(req);
+ if (unlikely(ret))
+ goto remove_req;
+ }
- dev = &qp_ctx->qp->qm->pdev->dev;
+ ret = hisi_zip_do_work(req);
+ if (ret == -EINPROGRESS)
+ return -EINPROGRESS;
- req = hisi_zip_create_req(qp_ctx, acomp_req);
- if (IS_ERR(req))
- return PTR_ERR(req);
+ if (ret == -EBUSY)
+ return -ENOSPC;
- ret = hisi_zip_do_work(qp_ctx, req);
- if (unlikely(ret != -EINPROGRESS)) {
- dev_info_ratelimited(dev, "failed to do compress (%d)!\n", ret);
- hisi_zip_remove_req(qp_ctx, req);
- }
+ /* -EAGAIN or other unexpected errors */
+ dev_info_ratelimited(dev, "failed to do %scompress (%d)!\n",
+ qp->alg_type ? "de" : "", ret);
+ hisi_zip_unmap_req_buffers(req);
+remove_req:
+ hisi_zip_remove_req(req);
+ return ret;
+}
+
+static int hisi_zip_do_one_request(struct crypto_engine *engine, void *areq)
+{
+ struct acomp_req *acomp_req = container_of(areq, struct acomp_req, base);
+ struct hisi_zip_req *req = acomp_request_ctx(acomp_req);
+ int ret;
+
+ ret = hisi_zip_do_comp(req);
+ if (ret == -EINPROGRESS)
+ return 0;
return ret;
}
-static int hisi_zip_adecompress(struct acomp_req *acomp_req)
+static void hisi_zip_release_req_resource(struct hisi_zip_req *req)
+{
+ if (!req->buf_ready)
+ return;
+
+ hisi_zip_unmap_req_buffers(req);
+ hisi_zip_remove_req(req);
+}
+
+static int hisi_zip_do_acomp(struct acomp_req *acomp_req, u8 alg_type)
{
struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
- struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP];
- struct hisi_zip_req *req;
- struct device *dev;
+ struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[alg_type];
+ struct hisi_zip_req *req = acomp_request_ctx(acomp_req);
int ret;
- if (ctx->fallback)
- return hisi_zip_fallback_do_work(acomp_req, 1);
+ if (unlikely(!acomp_req->src || !acomp_req->slen ||
+ !acomp_req->dst || !acomp_req->dlen))
+ return -EINVAL;
+
+ req->req = acomp_req;
+ req->alg_type = alg_type;
+ req->qp_ctx = qp_ctx;
+ req->buf_ready = false;
- dev = &qp_ctx->qp->qm->pdev->dev;
+ if (ctx->fallback)
+ return hisi_zip_fallback_do_work(acomp_req,
+ req->alg_type == HZIP_ALG_TYPE_DECOMP);
- req = hisi_zip_create_req(qp_ctx, acomp_req);
- if (IS_ERR(req))
- return PTR_ERR(req);
+ ret = hisi_zip_do_comp(req);
+ if (ret != -ENOSPC)
+ return ret;
- ret = hisi_zip_do_work(qp_ctx, req);
- if (unlikely(ret != -EINPROGRESS)) {
- dev_info_ratelimited(dev, "failed to do decompress (%d)!\n",
- ret);
- hisi_zip_remove_req(qp_ctx, req);
+ ret = crypto_transfer_acomp_request_to_engine(qp_ctx->engine, acomp_req);
+ if (ret != -EINPROGRESS && ret != -EBUSY) {
+ hisi_zip_release_req_resource(req);
+ return ret;
}
- return ret;
+ return -EINPROGRESS;
+}
+
+static int hisi_zip_acompress(struct acomp_req *acomp_req)
+{
+ return hisi_zip_do_acomp(acomp_req, HZIP_ALG_TYPE_COMP);
+}
+
+static int hisi_zip_adecompress(struct acomp_req *acomp_req)
+{
+ return hisi_zip_do_acomp(acomp_req, HZIP_ALG_TYPE_DECOMP);
}
static int hisi_zip_decompress(struct acomp_req *acomp_req)
@@ -423,6 +478,8 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int
struct hisi_zip *hisi_zip;
int ret, i;
+ hisi_zip_ctx->req_type = req_type;
+
/* alg_type = 0 for compress, 1 for decompress in hw sqe */
for (i = 0; i < HZIP_CTX_Q_NUM; i++)
alg_type[i] = i;
@@ -434,13 +491,22 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int
}
hisi_zip = container_of(qps[0]->qm, struct hisi_zip, qm);
+ if (!hisi_zip->engine_num) {
+ hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM);
+ return -ENODEV;
+ }
for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
qp_ctx = &hisi_zip_ctx->qp_ctx[i];
qp_ctx->ctx = hisi_zip_ctx;
qp_ctx->zip_dev = hisi_zip;
- qp_ctx->req_type = req_type;
qp_ctx->qp = qps[i];
+ qp_ctx->engine =
+ hisi_zip->engine[qps[i]->qp_id % hisi_zip->engine_num];
+ if (!qp_ctx->engine) {
+ hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM);
+ return -ENODEV;
+ }
}
hisi_zip_ctx->ops = &hisi_zip_ops;
@@ -475,26 +541,13 @@ static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx)
if (i == 0)
return ret;
- goto err_free_comp_q;
+ goto err_free_comp_bitmap;
}
spin_lock_init(&req_q->req_lock);
-
- req_q->q = kzalloc_objs(struct hisi_zip_req, req_q->size);
- if (!req_q->q) {
- ret = -ENOMEM;
- if (i == 0)
- goto err_free_comp_bitmap;
- else
- goto err_free_decomp_bitmap;
- }
}
return 0;
-err_free_decomp_bitmap:
- bitmap_free(ctx->qp_ctx[HZIP_QPC_DECOMP].req_q.req_bitmap);
-err_free_comp_q:
- kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.q);
err_free_comp_bitmap:
bitmap_free(ctx->qp_ctx[HZIP_QPC_COMP].req_q.req_bitmap);
return ret;
@@ -504,10 +557,8 @@ static void hisi_zip_release_req_q(struct hisi_zip_ctx *ctx)
{
int i;
- for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
- kfree(ctx->qp_ctx[i].req_q.q);
+ for (i = 0; i < HZIP_CTX_Q_NUM; i++)
bitmap_free(ctx->qp_ctx[i].req_q.req_bitmap);
- }
}
static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx)
@@ -607,12 +658,12 @@ static void hisi_zip_acomp_exit(struct crypto_acomp *tfm)
hisi_zip_ctx_exit(ctx);
}
-static struct acomp_alg hisi_zip_acomp_deflate = {
- .init = hisi_zip_acomp_init,
- .exit = hisi_zip_acomp_exit,
- .compress = hisi_zip_acompress,
- .decompress = hisi_zip_adecompress,
- .base = {
+static struct acomp_engine_alg hisi_zip_acomp_deflate = {
+ .base.init = hisi_zip_acomp_init,
+ .base.exit = hisi_zip_acomp_exit,
+ .base.compress = hisi_zip_acompress,
+ .base.decompress = hisi_zip_adecompress,
+ .base.base = {
.cra_name = "deflate",
.cra_driver_name = "hisi-deflate-acomp",
.cra_flags = CRYPTO_ALG_ASYNC |
@@ -620,7 +671,11 @@ static struct acomp_alg hisi_zip_acomp_deflate = {
.cra_module = THIS_MODULE,
.cra_priority = HZIP_ALG_PRIORITY,
.cra_ctxsize = sizeof(struct hisi_zip_ctx),
- }
+ .cra_reqsize = sizeof(struct hisi_zip_req),
+ },
+ .op = {
+ .do_one_request = hisi_zip_do_one_request,
+ },
};
static int hisi_zip_register_deflate(struct hisi_qm *qm)
@@ -630,7 +685,7 @@ static int hisi_zip_register_deflate(struct hisi_qm *qm)
if (!hisi_zip_alg_support(qm, HZIP_ALG_DEFLATE))
return 0;
- ret = crypto_register_acomp(&hisi_zip_acomp_deflate);
+ ret = crypto_engine_register_acomp(&hisi_zip_acomp_deflate);
if (ret)
dev_err(&qm->pdev->dev, "failed to register to deflate (%d)!\n", ret);
@@ -642,15 +697,15 @@ static void hisi_zip_unregister_deflate(struct hisi_qm *qm)
if (!hisi_zip_alg_support(qm, HZIP_ALG_DEFLATE))
return;
- crypto_unregister_acomp(&hisi_zip_acomp_deflate);
+ crypto_engine_unregister_acomp(&hisi_zip_acomp_deflate);
}
-static struct acomp_alg hisi_zip_acomp_lz4 = {
- .init = hisi_zip_acomp_init,
- .exit = hisi_zip_acomp_exit,
- .compress = hisi_zip_acompress,
- .decompress = hisi_zip_decompress,
- .base = {
+static struct acomp_engine_alg hisi_zip_acomp_lz4 = {
+ .base.init = hisi_zip_acomp_init,
+ .base.exit = hisi_zip_acomp_exit,
+ .base.compress = hisi_zip_acompress,
+ .base.decompress = hisi_zip_decompress,
+ .base.base = {
.cra_name = "lz4",
.cra_driver_name = "hisi-lz4-acomp",
.cra_flags = CRYPTO_ALG_ASYNC |
@@ -658,7 +713,11 @@ static struct acomp_alg hisi_zip_acomp_lz4 = {
.cra_module = THIS_MODULE,
.cra_priority = HZIP_ALG_PRIORITY,
.cra_ctxsize = sizeof(struct hisi_zip_ctx),
- }
+ .cra_reqsize = sizeof(struct hisi_zip_req),
+ },
+ .op = {
+ .do_one_request = hisi_zip_do_one_request,
+ },
};
static int hisi_zip_register_lz4(struct hisi_qm *qm)
@@ -668,7 +727,7 @@ static int hisi_zip_register_lz4(struct hisi_qm *qm)
if (!hisi_zip_alg_support(qm, HZIP_ALG_LZ4))
return 0;
- ret = crypto_register_acomp(&hisi_zip_acomp_lz4);
+ ret = crypto_engine_register_acomp(&hisi_zip_acomp_lz4);
if (ret)
dev_err(&qm->pdev->dev, "failed to register to LZ4 (%d)!\n", ret);
@@ -680,17 +739,75 @@ static void hisi_zip_unregister_lz4(struct hisi_qm *qm)
if (!hisi_zip_alg_support(qm, HZIP_ALG_LZ4))
return;
- crypto_unregister_acomp(&hisi_zip_acomp_lz4);
+ crypto_engine_unregister_acomp(&hisi_zip_acomp_lz4);
+}
+
+static void hisi_zip_free_crypto_engine(struct hisi_qm *qm)
+{
+ struct hisi_zip *hisi_zip = container_of(qm, struct hisi_zip, qm);
+ u32 i;
+
+ for (i = 0; i < hisi_zip->engine_num; i++) {
+ if (hisi_zip->engine[i])
+ crypto_engine_exit(hisi_zip->engine[i]);
+ }
+
+ hisi_zip->engine_num = 0;
+}
+
+static int hisi_zip_alloc_crypto_engine(struct hisi_qm *qm)
+{
+ struct hisi_zip *hisi_zip = container_of(qm, struct hisi_zip, qm);
+ u32 engine_num = min_t(u32, qm->qp_num, HZIP_MAX_CRYPTO_ENGINE);
+ struct device *dev = &qm->pdev->dev;
+ u16 qlen = qm->qp_array[0].sq_depth;
+ int ret;
+ u32 i;
+
+ for (i = 0; i < engine_num; i++) {
+ hisi_zip->engine[i] = crypto_engine_alloc_init_and_set(dev, true,
+ false, qlen);
+ if (!hisi_zip->engine[i]) {
+ dev_err(dev, "failed to alloc crypto engine.\n");
+ ret = -ENOMEM;
+ goto free_engine;
+ }
+
+ ret = crypto_engine_start(hisi_zip->engine[i]);
+ if (ret) {
+ dev_err(dev, "failed to start crypto engine.\n");
+ crypto_engine_exit(hisi_zip->engine[i]);
+ hisi_zip->engine[i] = NULL;
+ goto free_engine;
+ }
+ }
+
+ hisi_zip->engine_num = engine_num;
+
+ return 0;
+
+free_engine:
+ for (i = 0; i < engine_num; i++) {
+ if (hisi_zip->engine[i])
+ crypto_engine_exit(hisi_zip->engine[i]);
+ }
+
+ return ret;
}
int hisi_zip_register_to_crypto(struct hisi_qm *qm)
{
- int ret = 0;
+ int ret;
+
+ ret = hisi_zip_alloc_crypto_engine(qm);
+ if (ret)
+ return ret;
mutex_lock(&zip_algs_lock);
if (zip_available_devs) {
zip_available_devs++;
- goto unlock;
+ mutex_unlock(&zip_algs_lock);
+ return 0;
}
ret = hisi_zip_register_deflate(qm);
@@ -710,6 +827,8 @@ int hisi_zip_register_to_crypto(struct hisi_qm *qm)
hisi_zip_unregister_deflate(qm);
unlock:
mutex_unlock(&zip_algs_lock);
+ hisi_zip_free_crypto_engine(qm);
+
return ret;
}
@@ -724,4 +843,5 @@ void hisi_zip_unregister_from_crypto(struct hisi_qm *qm)
unlock:
mutex_unlock(&zip_algs_lock);
+ hisi_zip_free_crypto_engine(qm);
}
--
2.43.0
^ permalink raw reply [flat|nested] 5+ messages in thread