* [PATCH v2 01/20] crypto: aes - Fix undesired override of some optimized AES modes
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
@ 2026-09-27 22:42 ` Eric Biggers
2026-09-27 22:42 ` [PATCH v2 02/20] lib/crypto: aes-xctr: Pass counter by value to aes_xctr_arch() Eric Biggers
` (18 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:42 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers
The new library APIs for AES encryption modes were wired up to the
traditional crypto API via crypto/aes.c. However, for now the kernel is
still in a transitional state where various architectures still have
architecture-optimized implementations of AES modes in arch/*/crypto/,
wired up to the traditional crypto API only. Because of that, the
crypto/aes.c algorithms were given a cra_priority of only 110 to prevent
them from overriding arch/*/crypto/ in the traditional crypto API.
However, because of how the traditional crypto API works, the
cra_priority trick doesn't work in cases where the relevant algorithm
isn't directly implemented by arch/*/crypto/ but rather is provided by a
template instance using other code in arch/*/crypto/.
For example, x86 doesn't have its own "ccm(aes)" but rather relies on
the "ccm" template constructing it from the x86-optimized "ctr(aes)".
The existence of the library-based "ccm(aes)" prevents that, even though
its priority is lower than what the template would produce.
Thus, "ccm(aes)" ends up using the slower single-block AES code.
Therefore, skip wiring up the relevant library-based code to the
traditional crypto API on architectures where this problem can occur, as
determined by what exists in arch/*/crypto/ for each architecture.
This is ugly, but it's also temporary: these conditions will go away as
architecture-optimized implementations of AES modes are migrated into
the library. But until then, we need to prevent performance regressions
by ensuring that the optimized code continues to be used.
Note: "xts(aes)" is left alone. Though the "xts" template can use
"ecb(aes)" as an inner algorithm, in practice this isn't very efficient
and a dedicated "xts(aes)" is already provided in all the important
cases anyway. (This omission is also consistent with the fact that the
library isn't planned to provide a similar ECB-to-XTS "adapter".)
Fixes: 20df21a482aa ("crypto: aes - Add CBC and CBC-CTS support using library")
Fixes: 8ca62072faa1 ("crypto: aes - Add GCM support using library")
Fixes: f70ad727d1d6 ("crypto: aes - Add CCM support using library")
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
crypto/aes.c | 39 ++++++++++++++++++++++++++++++++++++---
1 file changed, 36 insertions(+), 3 deletions(-)
diff --git a/crypto/aes.c b/crypto/aes.c
index 94791f481e98..5046b887ac9a 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -637,7 +637,17 @@ static struct skcipher_alg skcipher_algs[] = {
.decrypt = crypto_aes_cbc_decrypt,
},
#endif
-#if IS_ENABLED(CONFIG_CRYPTO_CTS)
+#if IS_ENABLED(CONFIG_CRYPTO_CTS) && \
+ /*
+ * Skip registering this when it might block a "better" implementation
+ * from being instantiated via the "cts" template wrapping an arch-
+ * optimized "cbc(aes)" that hasn't yet been migrated into the library.
+ */ \
+ !(IS_ENABLED(CONFIG_ARM) || \
+ IS_ENABLED(CONFIG_ARM64) || \
+ IS_ENABLED(CONFIG_PPC) || \
+ IS_ENABLED(CONFIG_S390) || \
+ IS_ENABLED(CONFIG_SPARC))
{
.base.cra_name = "cts(cbc(aes))",
.base.cra_driver_name = "cts-cbc-aes-lib",
@@ -980,7 +990,18 @@ static __maybe_unused int crypto_aes_ccm_decrypt(struct aead_request *req)
}
static struct aead_alg aead_algs[] = {
-#if IS_ENABLED(CONFIG_CRYPTO_GCM)
+#if IS_ENABLED(CONFIG_CRYPTO_GCM) && \
+ /*
+ * Skip registering these when they might block "better" implementations
+ * from being instantiated via the corresponding templates using
+ * arch-optimized code that hasn't yet been migrated into the library.
+ */ \
+ !(IS_ENABLED(CONFIG_ARM) || \
+ IS_ENABLED(CONFIG_ARM64) || \
+ IS_ENABLED(CONFIG_PPC) || \
+ IS_ENABLED(CONFIG_RISCV) || \
+ IS_ENABLED(CONFIG_S390) || \
+ IS_ENABLED(CONFIG_SPARC))
{
.base.cra_name = "gcm(aes)",
.base.cra_driver_name = "gcm-aes-lib",
@@ -1012,7 +1033,19 @@ static struct aead_alg aead_algs[] = {
.chunksize = AES_BLOCK_SIZE,
},
#endif /* CONFIG_CRYPTO_GCM */
-#if IS_ENABLED(CONFIG_CRYPTO_CCM)
+#if IS_ENABLED(CONFIG_CRYPTO_CCM) && \
+ /*
+ * Skip registering this when it might block a "better" implementation
+ * from being instantiated via the "ccm" template wrapping an arch-
+ * optimized "ctr(aes)" that hasn't yet been migrated into the library.
+ */ \
+ !(IS_ENABLED(CONFIG_ARM) || \
+ IS_ENABLED(CONFIG_ARM64) || \
+ IS_ENABLED(CONFIG_PPC) || \
+ IS_ENABLED(CONFIG_RISCV) || \
+ IS_ENABLED(CONFIG_S390) || \
+ IS_ENABLED(CONFIG_SPARC) || \
+ IS_ENABLED(CONFIG_X86))
{
.base.cra_name = "ccm(aes)",
.base.cra_driver_name = "ccm-aes-lib",
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread* [PATCH v2 02/20] lib/crypto: aes-xctr: Pass counter by value to aes_xctr_arch()
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
2026-09-27 22:42 ` [PATCH v2 01/20] crypto: aes - Fix undesired override of some optimized AES modes Eric Biggers
@ 2026-09-27 22:42 ` Eric Biggers
2026-09-27 22:42 ` [PATCH v2 03/20] lib/crypto: x86/aes: Clean up aes-aesni.S in preparation for AES modes Eric Biggers
` (17 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:42 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers, Thomas Huth
Update the calling convention for aes_xctr_arch() to pass the counter by
value, then do the increment in generic code. This aligns better with
the x86_64 and arm64 assembly code for XCTR, which takes the counter by
value and thus has to be paired with an increment in C code anyway.
Reviewed-by: Thomas Huth <thuth@redhat.com>
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/aes.c | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c
index f1549839b3de..22f096c50242 100644
--- a/lib/crypto/aes.c
+++ b/lib/crypto/aes.c
@@ -1094,7 +1094,7 @@ static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len,
}
#endif
#ifndef aes_xctr_arch
-static bool aes_xctr_arch(u8 *dst, const u8 *src, size_t len, u64 *ctr,
+static bool aes_xctr_arch(u8 *dst, const u8 *src, size_t len, u64 ctr,
const u8 iv[AES_BLOCK_SIZE],
const struct aes_enckey *key)
{
@@ -1150,8 +1150,10 @@ void aes_xctr(u8 *dst, const u8 *src, size_t len, u64 *ctr,
__le64 aes_input[2];
u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long));
- if (likely(aes_xctr_arch(dst, src, len, ctr, iv, key.enc_key)))
+ if (likely(aes_xctr_arch(dst, src, len, *ctr, iv, key.enc_key))) {
+ *ctr += DIV_ROUND_UP(len, AES_BLOCK_SIZE);
return;
+ }
aes_input[1] = get_unaligned((const __le64 *)&iv[8]);
/* Handle the full blocks. */
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread* [PATCH v2 03/20] lib/crypto: x86/aes: Clean up aes-aesni.S in preparation for AES modes
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
2026-09-27 22:42 ` [PATCH v2 01/20] crypto: aes - Fix undesired override of some optimized AES modes Eric Biggers
2026-09-27 22:42 ` [PATCH v2 02/20] lib/crypto: aes-xctr: Pass counter by value to aes_xctr_arch() Eric Biggers
@ 2026-09-27 22:42 ` Eric Biggers
2026-09-27 22:42 ` [PATCH v2 04/20] lib/crypto: x86/aes-ecb: Add AES-NI optimization Eric Biggers
` (16 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:42 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers
Various miscellaneous updates to the assembly code in preparation for
adding implementations of AES modes to the same file:
- Define macros for the function argument registers, function prologues,
and function epilogues to centralize some of the handling of 32-bit vs
64-bit.
- Refactor the actual AES encryption and AES decryption logic into
macros _do_aes and _do_aes_ecb so that some of the modes can reuse it.
- Rename mask to expandkey_mask to differentiate it from the bswap_mask
that will be added.
- Update the prototypes of aes_encrypt_aesni() and aes_decrypt_aesni()
to be dst, src, key so that they will match the mode functions.
Note that this means passing a pointer to the key struct instead of a
(nrounds, rndkeys) pair, similar to what arch/x86/crypto/aes*.S do.
Although this makes the assembly code depend on the format of the key
struct, having one fewer argument makes it easier to accommodate
32-bit mode, and the C glue code becomes slightly simpler.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/x86/aes-aesni.S | 249 +++++++++++++++++++++++++------------
lib/crypto/x86/aes.h | 28 +++--
2 files changed, 185 insertions(+), 92 deletions(-)
diff --git a/lib/crypto/x86/aes-aesni.S b/lib/crypto/x86/aes-aesni.S
index b8c3e104a3be..a56fcb36da18 100644
--- a/lib/crypto/x86/aes-aesni.S
+++ b/lib/crypto/x86/aes-aesni.S
@@ -8,25 +8,126 @@
// AVX. It does use up to SSE4.1, which all CPUs with AES-NI have.
#include <linux/linkage.h>
-.section .rodata
#ifdef __x86_64__
#define RODATA(label) label(%rip)
-#else
+
+#define ARG0 %rdi
+#define ARG1 %rsi
+#define ARG2 %rdx
+#define ARG2_32 %edx
+#define ARG3 %rcx
+#define ARG4 %r8
+#define TMP %rax
+#define TMP_32 %eax
+#define TMP_16 %ax
+#define TMP_8 %al
+
+#else // __x86_64__
+
#define RODATA(label) label
-#endif
+// Caller-save GPRs and the first 3 function arguments, assuming -mregparm=3
+#define ARG0 %eax
+#define ARG1 %edx
+#define ARG2 %ecx
+#define ARG2_32 %ecx
+
+// *Callee*-save GPRs.
+#define ARG3 %edi
+#define ARG4 %esi
+#define TMP %ebx
+#define TMP_32 %ebx
+#define TMP_16 %bx
+#define TMP_8 %bl
+#endif // !__x86_64__
+
+// Offsets in struct aes_key
+#define OFFSETOF_NROUNDS 4
+#define OFFSETOF_ROUNDKEYS 16
+#define OFFSETOF_INVROUNDKEYS 256
+
+.section .rodata
+.p2align 4
+.Lexpandkey_mask:
// A mask for pshufb that extracts the last dword, rotates it right by 8
// bits, and copies the result to all four dwords.
-.p2align 4
-.Lmask:
.byte 13, 14, 15, 12, 13, 14, 15, 12, 13, 14, 15, 12, 13, 14, 15, 12
- // The AES round constants, used during key expansion
.Lrcon:
+ // The AES round constants, used during key expansion
.long 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36
.text
+// In 32-bit mode, push certain callee-saved GPRs and optionally load function
+// arguments from the stack into them. Do nothing in 64-bit mode.
+//
+// TMP is always made available as a temporary register.
+//
+// \uses_arg3 is 0 to not touch ARG3, 1 to make it available as a temporary
+// register only, or 2 to actually load it as an argument from the stack.
+// Likewise for \uses_arg4 and ARG4. It's assumed the -mregparm=3 convention is
+// being used and %esp hasn't been modified since function entry.
+.macro _prologue uses_arg3=0, uses_arg4=0
+#ifdef __i386__
+ .set ARG3_OFFSET, 4 // Assuming -mregparm=3
+ .set ARG3_PUSHED, \uses_arg3
+ .set ARG4_PUSHED, \uses_arg4
+.if \uses_arg3
+ push ARG3
+ .set ARG3_OFFSET, ARG3_OFFSET + 4
+ .if \uses_arg3 == 2
+ mov ARG3_OFFSET(%esp), ARG3
+ .endif
+.endif
+.if \uses_arg4
+ push ARG4
+ .set ARG3_OFFSET, ARG3_OFFSET + 4
+ .if \uses_arg4 == 2
+ mov ARG3_OFFSET+4(%esp), ARG4
+ .endif
+.endif
+ push TMP
+ .set ARG3_OFFSET, ARG3_OFFSET + 4
+#endif // __i386__
+.endm
+
+.macro _reload_arg3
+#ifdef __i386__
+ mov ARG3_OFFSET(%esp), ARG3
+#endif
+.endm
+
+// Undo any pushes that _prologue did, then return.
+.macro _epilogue
+#ifdef __i386__
+ pop TMP
+.if ARG4_PUSHED
+ pop ARG4
+.endif
+.if ARG3_PUSHED
+ pop ARG3
+.endif
+#endif
+ RET
+.endm
+
+.macro _aesenc enc, rndkey, data
+.if \enc
+ aesenc \rndkey, \data
+.else
+ aesdec \rndkey, \data
+.endif
+.endm
+
+.macro _aesenclast enc, rndkeylast, data
+.if \enc
+ aesenclast \rndkeylast, \data
+.else
+ aesdeclast \rndkeylast, \data
+.endif
+.endm
+
// Transform four dwords [a0, a1, a2, a3] in \a into
// [a0, a0^a1, a0^a1^a2, a0^a1^a2^a3]. \tmp is a temporary xmm register.
//
@@ -71,32 +172,18 @@
.endm
.macro _aes_expandkey_aesni is_aes128
-#ifdef __x86_64__
// Arguments
- .set RNDKEYS, %rdi
- .set INV_RNDKEYS, %rsi
- .set IN_KEY, %rdx
+ .set RNDKEYS, ARG0
+ .set INV_RNDKEYS, ARG1
+ .set IN_KEY, ARG2
// Other local variables
- .set RCON_PTR, %rcx
- .set COUNTER, %eax
-#else
- // Arguments, assuming -mregparm=3
- .set RNDKEYS, %eax
- .set INV_RNDKEYS, %edx
- .set IN_KEY, %ecx
-
- // Other local variables
- .set RCON_PTR, %ebx
- .set COUNTER, %esi
-#endif
+ .set RCON_PTR, ARG3
+ .set COUNTER, TMP_32
.set RCON, %xmm6
.set MASK, %xmm7
-#ifdef __i386__
- push %ebx
- push %esi
-#endif
+ _prologue uses_arg3=1
.if \is_aes128
// AES-128: the first round key is simply a copy of the raw key.
@@ -112,7 +199,7 @@
.endif
// Generate the remaining round keys.
- movdqa RODATA(.Lmask), MASK
+ movdqa RODATA(.Lexpandkey_mask), MASK
.if \is_aes128
lea RODATA(.Lrcon), RCON_PTR
mov $10, COUNTER
@@ -176,11 +263,7 @@
movdqu %xmm0, 16(INV_RNDKEYS) // => Last inverse round key
.Ldone\@:
-#ifdef __i386__
- pop %esi
- pop %ebx
-#endif
- RET
+ _epilogue
.endm
// void aes128_expandkey_aesni(u32 rndkeys[], u32 *inv_rndkeys,
@@ -195,67 +278,73 @@ SYM_FUNC_START(aes256_expandkey_aesni)
_aes_expandkey_aesni 0
SYM_FUNC_END(aes256_expandkey_aesni)
-.macro _aes_crypt_aesni enc
-#ifdef __x86_64__
- .set RNDKEYS, %rdi
- .set NROUNDS, %esi
- .set OUT, %rdx
- .set IN, %rcx
-#else
- // Assuming -mregparm=3
- .set RNDKEYS, %eax
- .set NROUNDS, %edx
- .set OUT, %ecx
- .set IN, %ebx // Passed on stack
-#endif
-
-#ifdef __i386__
- push %ebx
- mov 8(%esp), %ebx
-#endif
-
- // Zero-th round
- movdqu (IN), %xmm0
- movdqu (RNDKEYS), %xmm1
- pxor %xmm1, %xmm0
-
- // Normal rounds
- add $16, RNDKEYS
+// AES-encrypt (\enc=1) or AES-decrypt (\enc=0) the AESDATA registers specified
+// in \vecs using the aes_enckey or aes_key pointed to by KEY. RNDKEY must be
+// set to a temporary XMM register, NROUNDS to a temporary 32-bit GPR, and
+// RNDKEY_PTR to a temporary full-size GPR.
+.macro _do_aes enc, vecs:vararg
+ movl OFFSETOF_NROUNDS(KEY), NROUNDS
dec NROUNDS
-.Lnext_round\@:
- movdqu (RNDKEYS), %xmm1
.if \enc
- aesenc %xmm1, %xmm0
+ .set rndkey0_offs, OFFSETOF_ROUNDKEYS
.else
- aesdec %xmm1, %xmm0
+ .set rndkey0_offs, OFFSETOF_INVROUNDKEYS
.endif
- add $16, RNDKEYS
+
+ // Do the zero-th AES round.
+ movdqu rndkey0_offs(KEY), RNDKEY
+.irp i, \vecs
+ pxor RNDKEY, AESDATA\i
+.endr
+ // Do the regular AES rounds.
+ lea rndkey0_offs+16(KEY), RNDKEY_PTR
+.Lnext_round\@:
+ movdqu (RNDKEY_PTR), RNDKEY
+ add $16, RNDKEY_PTR
+.irp i, \vecs
+ _aesenc \enc, RNDKEY, AESDATA\i
+.endr
dec NROUNDS
- jne .Lnext_round\@
+ jnz .Lnext_round\@
+ // Do the last AES round.
+ movdqu (RNDKEY_PTR), RNDKEY
+.irp i, \vecs
+ _aesenclast \enc, RNDKEY, AESDATA\i
+.endr
+.endm
- // Last round
- movdqu (RNDKEYS), %xmm1
-.if \enc
- aesenclast %xmm1, %xmm0
-.else
- aesdeclast %xmm1, %xmm0
-.endif
- movdqu %xmm0, (OUT)
+.macro _do_aes_ecb enc, vecs:vararg
+.irp i, \vecs
+ movdqu \i*16(SRC), AESDATA\i
+.endr
+ _do_aes \enc, \vecs
+.irp i, \vecs
+ movdqu AESDATA\i, \i*16(DST)
+.endr
+.endm
-#ifdef __i386__
- pop %ebx
-#endif
- RET
+.macro _aes_crypt_aesni enc
+ .set DST, ARG0
+ .set SRC, ARG1
+ .set KEY, ARG2
+ .set RNDKEY_PTR, ARG3 // Temporary register for _do_aes
+ .set NROUNDS, TMP_32 // Temporary register for _do_aes
+ .set AESDATA0, %xmm0
+ .set RNDKEY, %xmm1 // Temporary register for _do_aes
+
+ _prologue uses_arg3=1
+ _do_aes_ecb \enc, 0
+ _epilogue
.endm
-// void aes_encrypt_aesni(const u32 rndkeys[], int nrounds,
-// u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]);
+// void aes_encrypt_aesni(u8 dst[AES_BLOCK_SIZE], const u8 src[AES_BLOCK_SIZE],
+// const struct aes_enckey *key);
SYM_FUNC_START(aes_encrypt_aesni)
_aes_crypt_aesni 1
SYM_FUNC_END(aes_encrypt_aesni)
-// void aes_decrypt_aesni(const u32 inv_rndkeys[], int nrounds,
-// u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]);
+// void aes_decrypt_aesni(u8 dst[AES_BLOCK_SIZE], const u8 src[AES_BLOCK_SIZE],
+// const struct aes_key *key);
SYM_FUNC_START(aes_decrypt_aesni)
_aes_crypt_aesni 0
SYM_FUNC_END(aes_decrypt_aesni)
diff --git a/lib/crypto/x86/aes.h b/lib/crypto/x86/aes.h
index b047dee94f57..06146fef06be 100644
--- a/lib/crypto/x86/aes.h
+++ b/lib/crypto/x86/aes.h
@@ -7,16 +7,21 @@
#include <asm/fpu/api.h>
-static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aesni);
+
+/* The assembly code assumes the following offsets. */
+static_assert(offsetof(struct aes_enckey, nrounds) == 4);
+static_assert(offsetof(struct aes_enckey, k.rndkeys) == 16);
+static_assert(offsetof(struct aes_key, inv_k.inv_rndkeys) == 256);
void aes128_expandkey_aesni(u32 rndkeys[], u32 *inv_rndkeys,
const u8 in_key[AES_KEYSIZE_128]);
void aes256_expandkey_aesni(u32 rndkeys[], u32 *inv_rndkeys,
const u8 in_key[AES_KEYSIZE_256]);
-void aes_encrypt_aesni(const u32 rndkeys[], int nrounds,
- u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]);
-void aes_decrypt_aesni(const u32 inv_rndkeys[], int nrounds,
- u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]);
+void aes_encrypt_aesni(u8 dst[AES_BLOCK_SIZE], const u8 src[AES_BLOCK_SIZE],
+ const struct aes_enckey *key);
+void aes_decrypt_aesni(u8 dst[AES_BLOCK_SIZE], const u8 src[AES_BLOCK_SIZE],
+ const struct aes_key *key);
/*
* Expand an AES key using AES-NI if supported and usable or generic code
@@ -36,7 +41,7 @@ static void aes_preparekey_arch(union aes_enckey_arch *k,
u32 *rndkeys = k->rndkeys;
u32 *inv_rndkeys = inv_k ? inv_k->inv_rndkeys : NULL;
- if (static_branch_likely(&have_aes) && key_len != AES_KEYSIZE_192 &&
+ if (static_branch_likely(&have_aesni) && key_len != AES_KEYSIZE_192 &&
irq_fpu_usable()) {
kernel_fpu_begin();
if (key_len == AES_KEYSIZE_128)
@@ -53,9 +58,9 @@ static void aes_encrypt_arch(const struct aes_enckey *key,
u8 out[AES_BLOCK_SIZE],
const u8 in[AES_BLOCK_SIZE])
{
- if (static_branch_likely(&have_aes) && irq_fpu_usable()) {
+ if (static_branch_likely(&have_aesni) && irq_fpu_usable()) {
kernel_fpu_begin();
- aes_encrypt_aesni(key->k.rndkeys, key->nrounds, out, in);
+ aes_encrypt_aesni(out, in, key);
kernel_fpu_end();
} else {
aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in);
@@ -66,10 +71,9 @@ static void aes_decrypt_arch(const struct aes_key *key,
u8 out[AES_BLOCK_SIZE],
const u8 in[AES_BLOCK_SIZE])
{
- if (static_branch_likely(&have_aes) && irq_fpu_usable()) {
+ if (static_branch_likely(&have_aesni) && irq_fpu_usable()) {
kernel_fpu_begin();
- aes_decrypt_aesni(key->inv_k.inv_rndkeys, key->nrounds,
- out, in);
+ aes_decrypt_aesni(out, in, key);
kernel_fpu_end();
} else {
aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds,
@@ -81,5 +85,5 @@ static void aes_decrypt_arch(const struct aes_key *key,
static void aes_mod_init_arch(void)
{
if (boot_cpu_has(X86_FEATURE_AES))
- static_branch_enable(&have_aes);
+ static_branch_enable(&have_aesni);
}
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread* [PATCH v2 04/20] lib/crypto: x86/aes-ecb: Add AES-NI optimization
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
` (2 preceding siblings ...)
2026-09-27 22:42 ` [PATCH v2 03/20] lib/crypto: x86/aes: Clean up aes-aesni.S in preparation for AES modes Eric Biggers
@ 2026-09-27 22:42 ` Eric Biggers
2026-09-27 22:42 ` [PATCH v2 05/20] lib/crypto: x86/aes-cbc: " Eric Biggers
` (15 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:42 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers
Optimize the crypto library's AES-ECB support with AES-NI, bringing its
performance on par with the "ecb-aes-aesni" skcipher algorithm it will
supersede.
The new assembly functions are written from scratch to fit well into the
crypto library. However, they are functionally very similar to the
functions in arch/x86/crypto/aesni-intel_asm.S that they will supersede
and are intended to provide parity with those -- including supporting
32-bit mode, having the inner loops do 4 AES blocks per iteration, etc.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
crypto/aes.c | 2 +-
lib/crypto/x86/aes-aesni.S | 54 ++++++++++++++++++++++++++++++++++++++
lib/crypto/x86/aes.h | 35 ++++++++++++++++++++++++
3 files changed, 90 insertions(+), 1 deletion(-)
diff --git a/crypto/aes.c b/crypto/aes.c
index 5046b887ac9a..b1f71d2b2460 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -610,7 +610,7 @@ static struct skcipher_alg skcipher_algs[] = {
{
.base.cra_name = "ecb(aes)",
.base.cra_driver_name = "ecb-aes-lib",
- .base.cra_priority = 110,
+ .base.cra_priority = IS_ENABLED(CONFIG_X86) ? 300 : 110,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct aes_key),
.base.cra_module = THIS_MODULE,
diff --git a/lib/crypto/x86/aes-aesni.S b/lib/crypto/x86/aes-aesni.S
index a56fcb36da18..52f4ee5c13b9 100644
--- a/lib/crypto/x86/aes-aesni.S
+++ b/lib/crypto/x86/aes-aesni.S
@@ -348,3 +348,57 @@ SYM_FUNC_END(aes_encrypt_aesni)
SYM_FUNC_START(aes_decrypt_aesni)
_aes_crypt_aesni 0
SYM_FUNC_END(aes_decrypt_aesni)
+
+.macro _ecb_crypt enc
+ // Arguments
+ .set DST, ARG0
+ .set SRC, ARG1
+ .set NBLOCKS, ARG2
+ .set NBLOCKS32, ARG2_32 // Used for improved code density
+ .set KEY, ARG3
+
+ // Other local variables
+ .set RNDKEY_PTR, ARG4 // Temporary register for _do_aes
+ .set NROUNDS, TMP_32 // Temporary register for _do_aes
+ .set AESDATA0, %xmm0
+ .set AESDATA1, %xmm1
+ .set AESDATA2, %xmm2
+ .set AESDATA3, %xmm3
+ .set RNDKEY, %xmm4 // Temporary register for _do_aes
+ _prologue uses_arg3=2, uses_arg4=1
+
+ sub $4, NBLOCKS
+ jl .Lecb_loop4_done\@
+.p2align 5
+.Lecb_loop4\@:
+ _do_aes_ecb \enc, 0,1,2,3
+ add $64, SRC
+ add $64, DST
+ sub $4, NBLOCKS
+ jge .Lecb_loop4\@
+.Lecb_loop4_done\@:
+ add $4, NBLOCKS32
+ jz .Lecb_done\@
+
+.Lecb_loop1\@:
+ _do_aes_ecb \enc, 0
+ add $16, SRC
+ add $16, DST
+ dec NBLOCKS32
+ jnz .Lecb_loop1\@
+
+.Lecb_done\@:
+ _epilogue
+.endm
+
+// void aes_ecb_encrypt_aesni(u8 *dst, const u8 *src, long nblocks,
+// const struct aes_enckey *key);
+SYM_FUNC_START(aes_ecb_encrypt_aesni)
+ _ecb_crypt 1
+SYM_FUNC_END(aes_ecb_encrypt_aesni)
+
+// void aes_ecb_decrypt_aesni(u8 *dst, const u8 *src, long nblocks,
+// const struct aes_key *key);
+SYM_FUNC_START(aes_ecb_decrypt_aesni)
+ _ecb_crypt 0
+SYM_FUNC_END(aes_ecb_decrypt_aesni)
diff --git a/lib/crypto/x86/aes.h b/lib/crypto/x86/aes.h
index 06146fef06be..408625750ae9 100644
--- a/lib/crypto/x86/aes.h
+++ b/lib/crypto/x86/aes.h
@@ -12,6 +12,8 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aesni);
/* The assembly code assumes the following offsets. */
static_assert(offsetof(struct aes_enckey, nrounds) == 4);
static_assert(offsetof(struct aes_enckey, k.rndkeys) == 16);
+static_assert(offsetof(struct aes_key, nrounds) == 4);
+static_assert(offsetof(struct aes_key, k.rndkeys) == 16);
static_assert(offsetof(struct aes_key, inv_k.inv_rndkeys) == 256);
void aes128_expandkey_aesni(u32 rndkeys[], u32 *inv_rndkeys,
@@ -81,6 +83,39 @@ static void aes_decrypt_arch(const struct aes_key *key,
}
}
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB)
+void aes_ecb_encrypt_aesni(u8 *dst, const u8 *src, long nblocks,
+ const struct aes_enckey *key);
+void aes_ecb_decrypt_aesni(u8 *dst, const u8 *src, long nblocks,
+ const struct aes_key *key);
+
+/* len is always a positive multiple of AES_BLOCK_SIZE here. */
+#define aes_ecb_encrypt_arch aes_ecb_encrypt_arch
+static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len,
+ const struct aes_enckey *key)
+{
+ if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
+ return false;
+ kernel_fpu_begin();
+ aes_ecb_encrypt_aesni(dst, src, len / AES_BLOCK_SIZE, key);
+ kernel_fpu_end();
+ return true;
+}
+
+/* len is always a positive multiple of AES_BLOCK_SIZE here. */
+#define aes_ecb_decrypt_arch aes_ecb_decrypt_arch
+static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len,
+ const struct aes_key *key)
+{
+ if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
+ return false;
+ kernel_fpu_begin();
+ aes_ecb_decrypt_aesni(dst, src, len / AES_BLOCK_SIZE, key);
+ kernel_fpu_end();
+ return true;
+}
+#endif /* CONFIG_CRYPTO_LIB_AES_ECB */
+
#define aes_mod_init_arch aes_mod_init_arch
static void aes_mod_init_arch(void)
{
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread* [PATCH v2 05/20] lib/crypto: x86/aes-cbc: Add AES-NI optimization
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
` (3 preceding siblings ...)
2026-09-27 22:42 ` [PATCH v2 04/20] lib/crypto: x86/aes-ecb: Add AES-NI optimization Eric Biggers
@ 2026-09-27 22:42 ` Eric Biggers
2026-09-27 22:42 ` [PATCH v2 06/20] lib/crypto: x86/aes-ctr: " Eric Biggers
` (14 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:42 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers
Optimize the crypto library's AES-CBC and AES-CBC-CTS support with
AES-NI, bringing their performance on par with the "cbc-aes-aesni" and
"cts-cbc-aes-aesni" skcipher algorithms they will supersede.
The new assembly functions are written from scratch to fit well into the
crypto library. However, they are functionally very similar to the
functions in arch/x86/crypto/aesni-intel_asm.S that they will supersede
and are intended to provide parity with those -- including supporting
32-bit mode, having the inner loops do 4 AES blocks per iteration, etc.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
crypto/aes.c | 4 +-
lib/crypto/x86/aes-aesni.S | 232 +++++++++++++++++++++++++++++++++++++
lib/crypto/x86/aes.h | 89 ++++++++++++++
3 files changed, 323 insertions(+), 2 deletions(-)
diff --git a/crypto/aes.c b/crypto/aes.c
index b1f71d2b2460..bf8acc553188 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -625,7 +625,7 @@ static struct skcipher_alg skcipher_algs[] = {
{
.base.cra_name = "cbc(aes)",
.base.cra_driver_name = "cbc-aes-lib",
- .base.cra_priority = 110,
+ .base.cra_priority = IS_ENABLED(CONFIG_X86) ? 300 : 110,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct aes_key),
.base.cra_module = THIS_MODULE,
@@ -651,7 +651,7 @@ static struct skcipher_alg skcipher_algs[] = {
{
.base.cra_name = "cts(cbc(aes))",
.base.cra_driver_name = "cts-cbc-aes-lib",
- .base.cra_priority = 110,
+ .base.cra_priority = IS_ENABLED(CONFIG_X86) ? 300 : 110,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct aes_key),
.base.cra_module = THIS_MODULE,
diff --git a/lib/crypto/x86/aes-aesni.S b/lib/crypto/x86/aes-aesni.S
index 52f4ee5c13b9..51c8522416d7 100644
--- a/lib/crypto/x86/aes-aesni.S
+++ b/lib/crypto/x86/aes-aesni.S
@@ -57,6 +57,14 @@
// The AES round constants, used during key expansion
.long 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36
+.Lcts_permute_table:
+ .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
+ .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
+ .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
+ .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
+ .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
+ .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
+
.text
// In 32-bit mode, push certain callee-saved GPRs and optionally load function
@@ -402,3 +410,227 @@ SYM_FUNC_END(aes_ecb_encrypt_aesni)
SYM_FUNC_START(aes_ecb_decrypt_aesni)
_ecb_crypt 0
SYM_FUNC_END(aes_ecb_decrypt_aesni)
+
+// void aes_cbc_encrypt_aesni(u8 *dst, const u8 *src, long nblocks,
+// u8 iv[AES_BLOCK_SIZE],
+// const struct aes_enckey *key);
+//
+// This assumes nblocks > 0.
+SYM_FUNC_START(aes_cbc_encrypt_aesni)
+ // Arguments
+ .set DST, ARG0
+ .set SRC, ARG1
+ .set NBLOCKS, ARG2
+ .set IV_PTR, ARG3
+ .set KEY, ARG4
+
+ // Other local variables
+#ifdef __x86_64__
+ .set RNDKEY_PTR, %r9
+#else
+ .set RNDKEY_PTR, IV_PTR // IV_PTR is clobbered and reloaded later
+#endif
+ .set NROUNDS, TMP_32
+ .set AESDATA0, %xmm0
+ .set PTEXT, %xmm1
+ _prologue uses_arg3=2, uses_arg4=2
+
+ movdqu (IV_PTR), AESDATA0
+.p2align 5
+.Lcbc_enc_loop:
+ movdqu (SRC), PTEXT
+ pxor PTEXT, AESDATA0
+ _do_aes 1, 0
+ movdqu AESDATA0, (DST)
+ add $16, SRC
+ add $16, DST
+ dec NBLOCKS
+ jnz .Lcbc_enc_loop
+
+ // Store the next IV. On 32-bit, reload IV_PTR from stack first.
+ _reload_arg3
+ movdqu AESDATA0, (IV_PTR)
+ _epilogue
+SYM_FUNC_END(aes_cbc_encrypt_aesni)
+
+// void aes_cbc_decrypt_aesni(u8 *dst, const u8 *src, long nblocks,
+// u8 iv[AES_BLOCK_SIZE], const struct aes_key *key);
+SYM_FUNC_START(aes_cbc_decrypt_aesni)
+ // Arguments
+ .set DST, ARG0
+ .set SRC, ARG1
+ .set NBLOCKS, ARG2
+ .set NBLOCKS32, ARG2_32 // Used for improved code density
+ .set IV_PTR, ARG3
+ .set KEY, ARG4
+
+ // Other local variables
+#ifdef __x86_64__
+ .set RNDKEY_PTR, %r9
+#else
+ .set RNDKEY_PTR, IV_PTR // IV_PTR is clobbered and reloaded later
+#endif
+ .set NROUNDS, TMP_32
+ .set AESDATA0, %xmm0
+ .set AESDATA1, %xmm1
+ .set AESDATA2, %xmm2
+ .set AESDATA3, %xmm3
+ .set RNDKEY, %xmm4
+ .set IV, %xmm5
+ .set CTEXT0, %xmm6
+ .set CTEXT1, %xmm7
+#ifdef __x86_64__
+ .set CTEXT2, %xmm8
+ .set CTEXT3, %xmm9
+#endif
+ _prologue uses_arg3=2, uses_arg4=2
+
+ movdqu (IV_PTR), IV
+
+ sub $4, NBLOCKS
+ jl .Lcbc_dec_loop4_done
+.p2align 5
+.Lcbc_dec_loop4:
+ movdqu 0(SRC), AESDATA0
+ movdqu 16(SRC), AESDATA1
+ movdqu 32(SRC), AESDATA2
+ movdqu 48(SRC), AESDATA3
+ movdqa AESDATA0, CTEXT0
+ movdqa AESDATA1, CTEXT1
+#ifdef __x86_64__
+ movdqa AESDATA2, CTEXT2
+ movdqa AESDATA3, CTEXT3
+#endif
+ _do_aes 0, 0,1,2,3
+ pxor IV, AESDATA0
+ pxor CTEXT0, AESDATA1
+ pxor CTEXT1, AESDATA2
+#ifdef __x86_64__
+ pxor CTEXT2, AESDATA3
+ movdqa CTEXT3, IV
+#else
+ movdqu 32(SRC), CTEXT0
+ pxor CTEXT0, AESDATA3
+ movdqu 48(SRC), IV
+#endif
+ movdqu AESDATA0, 0(DST)
+ movdqu AESDATA1, 16(DST)
+ movdqu AESDATA2, 32(DST)
+ movdqu AESDATA3, 48(DST)
+ add $64, SRC
+ add $64, DST
+ sub $4, NBLOCKS
+ jge .Lcbc_dec_loop4
+.Lcbc_dec_loop4_done:
+ add $4, NBLOCKS32
+ jz .Lcbc_dec_done
+
+.Lcbc_dec_loop1:
+ movdqu (SRC), AESDATA0
+ movdqa AESDATA0, CTEXT0
+ _do_aes 0, 0
+ pxor IV, AESDATA0
+ movdqa CTEXT0, IV
+ movdqu AESDATA0, (DST)
+ add $16, SRC
+ add $16, DST
+ dec NBLOCKS32
+ jnz .Lcbc_dec_loop1
+
+.Lcbc_dec_done:
+ // Store the next IV. On 32-bit, reload IV_PTR from stack first.
+ _reload_arg3
+ movdqu IV, (IV_PTR)
+ _epilogue
+SYM_FUNC_END(aes_cbc_decrypt_aesni)
+
+.macro _aes_cbc_cts_crypt enc
+ .set DST, ARG0
+ .set SRC, ARG1
+ .set PN_LEN, ARG2
+ .set IV_PTR, ARG3
+ .set KEY, ARG4
+ .set RNDKEY_PTR, IV_PTR // Temporary register for _do_aes
+ .set NROUNDS, TMP_32 // Temporary register for _do_aes
+ .set RSHIFT_MASK, %xmm0 // range(16-PN_LEN,16) + [0x80, 0x80, ...]
+ // Must be %xmm0 for pblendvb to work.
+ .set LSHIFT_MASK, %xmm1 // [0x80, 0x80, ...] + range(PN_LEN)
+ .set AESDATA0, %xmm2
+ .set AESDATA1, %xmm3
+ .set RNDKEY, %xmm4 // Temporary register for _do_aes
+ .set IV, %xmm5
+
+ _prologue uses_arg3=2, uses_arg4=2
+
+ // Load LSHIFT_MASK and RSHIFT_MASK.
+ lea RODATA(.Lcts_permute_table), TMP
+ movdqu (TMP,PN_LEN), LSHIFT_MASK
+ sub PN_LEN, TMP
+ movdqu 32(TMP), RSHIFT_MASK
+
+ // Load the IV, freeing up IV_PTR to reuse for RNDKEY_PTR.
+ movdqu (IV_PTR), IV
+
+ // Load the second-from-last source block (P[n-1] or C[n-1]) and
+ // AES-en/decrypt it. In the encryption case, XOR it with the IV first.
+ movdqu (SRC), AESDATA0
+.if \enc
+ pxor IV, AESDATA0
+.endif
+ _do_aes \enc, 0
+
+ // Load the last source block (P[n] or C[n]), left-aligned.
+ movdqu (SRC,PN_LEN), AESDATA1
+
+.if \enc
+ // Right-align P[n], XOR it with the first AES output, and AES-encrypt
+ // the result. This gives C[n-1] in AESDATA1.
+ pshufb RSHIFT_MASK, AESDATA1
+ pxor AESDATA0, AESDATA1
+ _do_aes 1, 1
+
+ // Left-align the first AES output to get left-aligned C[n].
+ pshufb LSHIFT_MASK, AESDATA0
+.else
+ // Save a copy of the first AES output in %xmm6 for later.
+ movdqa AESDATA0, %xmm6
+
+ // Left-align the first AES output and XOR it with the
+ // already-left-aligned C[n]. This gives the left-aligned P[n].
+ pshufb LSHIFT_MASK, AESDATA0
+ pxor AESDATA1, AESDATA0
+
+ // Right-align C[n], then use RSHIFT_MASK (in %xmm0 as pblendvb's
+ // implicit operand) as a blend mask to select C[n] and the remaining
+ // bytes from the first AES output, with the result in AESDATA1.
+ pshufb RSHIFT_MASK, AESDATA1
+ pblendvb %xmm6, AESDATA1
+
+ // AES-decrypt the result and XOR with the IV to get P[n-1] in AESDATA1.
+ _do_aes 0, 1
+ pxor IV, AESDATA1
+.endif
+ // Store the left-aligned C[n] or P[n], then C[n-1] or P[n-1].
+ // Note that these stores may overlap.
+ movdqu AESDATA0, (DST,PN_LEN)
+ movdqu AESDATA1, (DST)
+ _epilogue
+.endm
+
+// void aes_cbc_cts_encrypt_aesni(u8 *dst, const u8 *src, long pn_len,
+// const u8 iv[AES_BLOCK_SIZE],
+// const struct aes_enckey *key);
+// void aes_cbc_cts_decrypt_aesni(u8 *dst, const u8 *src, long pn_len,
+// const u8 iv[AES_BLOCK_SIZE],
+// const struct aes_key *key);
+//
+// Use the CS3 variant of ciphertext stealing to encrypt or decrypt the last two
+// blocks in a message. The two blocks are located at src and consist of one
+// full block of length 16 bytes and one possibly-partial block of length 1 <=
+// pn_len <= 16 bytes. The output is written to dst, which may equal src.
+SYM_FUNC_START(aes_cbc_cts_encrypt_aesni)
+ _aes_cbc_cts_crypt 1
+SYM_FUNC_END(aes_cbc_cts_encrypt_aesni)
+SYM_FUNC_START(aes_cbc_cts_decrypt_aesni)
+ _aes_cbc_cts_crypt 0
+SYM_FUNC_END(aes_cbc_cts_decrypt_aesni)
diff --git a/lib/crypto/x86/aes.h b/lib/crypto/x86/aes.h
index 408625750ae9..a0346cb3ecb5 100644
--- a/lib/crypto/x86/aes.h
+++ b/lib/crypto/x86/aes.h
@@ -116,6 +116,95 @@ static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len,
}
#endif /* CONFIG_CRYPTO_LIB_AES_ECB */
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC)
+void aes_cbc_encrypt_aesni(u8 *dst, const u8 *src, long nblocks,
+ u8 iv[AES_BLOCK_SIZE], const struct aes_enckey *key);
+void aes_cbc_decrypt_aesni(u8 *dst, const u8 *src, long nblocks,
+ u8 iv[AES_BLOCK_SIZE], const struct aes_key *key);
+void aes_cbc_cts_encrypt_aesni(u8 *dst, const u8 *src, long pn_len,
+ const u8 iv[AES_BLOCK_SIZE],
+ const struct aes_enckey *key);
+void aes_cbc_cts_decrypt_aesni(u8 *dst, const u8 *src, long pn_len,
+ const u8 iv[AES_BLOCK_SIZE],
+ const struct aes_key *key);
+
+/* len is always a positive multiple of AES_BLOCK_SIZE here. */
+#define aes_cbc_encrypt_arch aes_cbc_encrypt_arch
+static bool aes_cbc_encrypt_arch(u8 *dst, const u8 *src, size_t len,
+ u8 iv[AES_BLOCK_SIZE],
+ const struct aes_enckey *key)
+{
+ if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
+ return false;
+ kernel_fpu_begin();
+ aes_cbc_encrypt_aesni(dst, src, len / AES_BLOCK_SIZE, iv, key);
+ kernel_fpu_end();
+ return true;
+}
+
+/* len is always a positive multiple of AES_BLOCK_SIZE here. */
+#define aes_cbc_decrypt_arch aes_cbc_decrypt_arch
+static bool aes_cbc_decrypt_arch(u8 *dst, const u8 *src, size_t len,
+ u8 iv[AES_BLOCK_SIZE],
+ const struct aes_key *key)
+{
+ if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
+ return false;
+ kernel_fpu_begin();
+ aes_cbc_decrypt_aesni(dst, src, len / AES_BLOCK_SIZE, iv, key);
+ kernel_fpu_end();
+ return true;
+}
+
+/* len can be any value greater than AES_BLOCK_SIZE here. */
+#define aes_cbc_cts_encrypt_arch aes_cbc_cts_encrypt_arch
+static bool aes_cbc_cts_encrypt_arch(u8 *dst, const u8 *src, size_t len,
+ u8 iv[AES_BLOCK_SIZE],
+ const struct aes_enckey *key)
+{
+ const size_t cbc_blocks = (len - AES_BLOCK_SIZE - 1) / AES_BLOCK_SIZE;
+ const size_t pn_len = ((len - 1) % AES_BLOCK_SIZE) + 1;
+
+ if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
+ return false;
+
+ kernel_fpu_begin();
+ if (cbc_blocks) {
+ aes_cbc_encrypt_aesni(dst, src, cbc_blocks, iv, key);
+ dst += cbc_blocks * AES_BLOCK_SIZE;
+ src += cbc_blocks * AES_BLOCK_SIZE;
+ }
+ /* Encrypt the last 17 to 32 bytes. */
+ aes_cbc_cts_encrypt_aesni(dst, src, pn_len, iv, key);
+ kernel_fpu_end();
+ return true;
+}
+
+/* len can be any value greater than AES_BLOCK_SIZE here. */
+#define aes_cbc_cts_decrypt_arch aes_cbc_cts_decrypt_arch
+static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len,
+ u8 iv[AES_BLOCK_SIZE],
+ const struct aes_key *key)
+{
+ const size_t cbc_blocks = (len - AES_BLOCK_SIZE - 1) / AES_BLOCK_SIZE;
+ const size_t pn_len = ((len - 1) % AES_BLOCK_SIZE) + 1;
+
+ if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
+ return false;
+
+ kernel_fpu_begin();
+ if (cbc_blocks) {
+ aes_cbc_decrypt_aesni(dst, src, cbc_blocks, iv, key);
+ dst += cbc_blocks * AES_BLOCK_SIZE;
+ src += cbc_blocks * AES_BLOCK_SIZE;
+ }
+ /* Decrypt the last 17 to 32 bytes. */
+ aes_cbc_cts_decrypt_aesni(dst, src, pn_len, iv, key);
+ kernel_fpu_end();
+ return true;
+}
+#endif /* CONFIG_CRYPTO_LIB_AES_CBC */
+
#define aes_mod_init_arch aes_mod_init_arch
static void aes_mod_init_arch(void)
{
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread* [PATCH v2 06/20] lib/crypto: x86/aes-ctr: Add AES-NI optimization
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
` (4 preceding siblings ...)
2026-09-27 22:42 ` [PATCH v2 05/20] lib/crypto: x86/aes-cbc: " Eric Biggers
@ 2026-09-27 22:42 ` Eric Biggers
2026-09-27 22:42 ` [PATCH v2 07/20] lib/crypto: x86/aes-xts: " Eric Biggers
` (13 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:42 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers
Optimize the crypto library's AES-CTR support with AES-NI, bringing its
performance on par with the "ctr-aes-aesni" skcipher algorithm it will
supersede.
The new assembly function is written from scratch to fit well into the
crypto library and to be more consistent with aes-ctr-avx-x86_64.S than
the code in arch/x86/crypto/aesni-intel_asm.S that it will supersede.
That includes using the "ctr64" convention, where the assembly code is
simplified by making the C code handle incrementing the high 64 bits of
the counter. Unlike the ECB, CBC, and XTS code, 32-bit support is *not*
included for this one, as the existing CTR code didn't have it.
Note: the priority of ctr-aes-lib is left unchanged at 110 temporarily.
It will be increased when the AVX-optimized code is migrated too.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/x86/aes-aesni.S | 130 ++++++++++++++++++++++++++++++++++++-
lib/crypto/x86/aes.h | 54 +++++++++++++++
2 files changed, 183 insertions(+), 1 deletion(-)
diff --git a/lib/crypto/x86/aes-aesni.S b/lib/crypto/x86/aes-aesni.S
index 51c8522416d7..297fe21ba830 100644
--- a/lib/crypto/x86/aes-aesni.S
+++ b/lib/crypto/x86/aes-aesni.S
@@ -4,7 +4,8 @@
//
// Copyright 2026 Google LLC
//
-// The code in this file supports 32-bit and 64-bit CPUs, and it doesn't require
+// The code in this file supports 32-bit and 64-bit CPUs (except for
+// aes_ctr64_crypt_aesni() which supports 64-bit only), and it doesn't require
// AVX. It does use up to SSE4.1, which all CPUs with AES-NI have.
#include <linux/linkage.h>
@@ -48,6 +49,12 @@
.section .rodata
.p2align 4
+#ifdef __x86_64__
+.Lbswap_mask:
+ // A mask for pshufb that byte-reflects the value.
+ .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
+#endif
+
.Lexpandkey_mask:
// A mask for pshufb that extracts the last dword, rotates it right by 8
// bits, and copies the result to all four dwords.
@@ -634,3 +641,124 @@ SYM_FUNC_END(aes_cbc_cts_encrypt_aesni)
SYM_FUNC_START(aes_cbc_cts_decrypt_aesni)
_aes_cbc_cts_crypt 0
SYM_FUNC_END(aes_cbc_cts_decrypt_aesni)
+
+#ifdef __x86_64__
+// void aes_ctr64_crypt_aesni(u8 *dst, const u8 *src, s64 len,
+// const u64 le_ctr[2],
+// const struct aes_enckey *key);
+SYM_FUNC_START(aes_ctr64_crypt_aesni)
+ // Arguments
+ .set DST, ARG0
+ .set SRC, ARG1
+ .set LEN, ARG2
+ .set LEN32, ARG2_32 // Used for improved code density
+ .set LE_CTR_PTR, ARG3 // Used as temp reg after LE_CTR is loaded
+ .set KEY, ARG4
+
+ // Other local variables
+ .set AESDATA0, %xmm0
+ .set AESDATA1, %xmm1
+ .set AESDATA2, %xmm2
+ .set AESDATA3, %xmm3
+ .set LE_CTR, %xmm4 // Current 128-bit little endian counter
+ .set LE_CTR_INC, %xmm5 // Initialized to (u64[])[1, 0]
+ .set BSWAP_MASK, %xmm6
+ .set RNDKEY, %xmm7
+ .set RNDKEY_PTR, LE_CTR_PTR // Temporary register for _do_aes
+ .set NROUNDS, TMP_32 // Temporary register for _do_aes
+
+ // Initialize LE_CTR, BSWAP_MASK, and LE_CTR_INC.
+ movdqu (LE_CTR_PTR), LE_CTR
+ movdqa RODATA(.Lbswap_mask), BSWAP_MASK
+ mov $1, TMP_32
+ movd TMP_32, LE_CTR_INC
+
+ // Encrypt and XOR four blocks (64 bytes) at a time.
+ sub $64, LEN
+ jl .Lctr_loop4_done
+.p2align 5
+.Lctr_loop4:
+.irp i, 0,1,2,3
+ movdqa LE_CTR, AESDATA\i
+ pshufb BSWAP_MASK, AESDATA\i // => big endian counter
+ paddq LE_CTR_INC, LE_CTR
+.endr
+ _do_aes 1, 0,1,2,3
+ // AESDATA[0-3] now contain four keystream blocks.
+.irp i, 0,1,2,3
+ movdqu \i*16(SRC), RNDKEY // Use RNDKEY as temp register.
+ pxor RNDKEY, AESDATA\i
+ movdqu AESDATA\i, \i*16(DST)
+.endr
+ add $64, SRC
+ add $64, DST
+ sub $64, LEN
+ jge .Lctr_loop4
+.Lctr_loop4_done:
+ add $64, LEN32
+ jz .Lctr_done
+
+ // 1 <= LEN <= 63 bytes remain. Prepare four more keystream blocks.
+.irp i, 0,1,2,3
+ movdqa LE_CTR, AESDATA\i
+ pshufb BSWAP_MASK, AESDATA\i // => big endian counter
+ .if \i != 3
+ paddq LE_CTR_INC, LE_CTR
+ .endif
+.endr
+ _do_aes 1, 0,1,2,3
+ // AESDATA[0-3] now contain four keystream blocks.
+
+ // XOR one block (16 bytes) at a time.
+ sub $16, LEN32
+ jl .Lctr_xor_partial
+.Lctr_xor1:
+ movdqu (SRC), RNDKEY // Use RNDKEY as temp register.
+ pxor RNDKEY, AESDATA0
+ movdqu AESDATA0, (DST)
+ movdqa AESDATA1, AESDATA0
+ movdqa AESDATA2, AESDATA1
+ movdqa AESDATA3, AESDATA2
+ add $16, SRC
+ add $16, DST
+ sub $16, LEN32
+ jge .Lctr_xor1
+
+ // XOR the remaining LEN mod 16 bytes.
+.Lctr_xor_partial:
+ test $8, LEN32
+ jz 1f
+ movq AESDATA0, TMP
+ xor (SRC), TMP // XOR 8 bytes.
+ mov TMP, (DST)
+ add $8, SRC
+ add $8, DST
+ psrldq $8, AESDATA0
+1:
+ test $4, LEN32
+ jz 2f
+ movd AESDATA0, TMP_32
+ xor (SRC), TMP_32 // XOR 4 bytes.
+ mov TMP_32, (DST)
+ add $4, SRC
+ add $4, DST
+ psrldq $4, AESDATA0
+2:
+ test $2, LEN32
+ jz 3f
+ movd AESDATA0, TMP_32
+ xor (SRC), TMP_16 // XOR 2 bytes.
+ mov TMP_16, (DST)
+ add $2, SRC
+ add $2, DST
+ psrldq $2, AESDATA0
+3:
+ test $1, LEN32
+ jz .Lctr_done
+ movd AESDATA0, TMP_32
+ xor (SRC), TMP_8 // XOR 1 byte.
+ mov TMP_8, (DST)
+.Lctr_done:
+ RET
+SYM_FUNC_END(aes_ctr64_crypt_aesni)
+#endif // __x86_64__
diff --git a/lib/crypto/x86/aes.h b/lib/crypto/x86/aes.h
index a0346cb3ecb5..2a2b26d10e87 100644
--- a/lib/crypto/x86/aes.h
+++ b/lib/crypto/x86/aes.h
@@ -205,6 +205,60 @@ static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len,
}
#endif /* CONFIG_CRYPTO_LIB_AES_CBC */
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR) && IS_ENABLED(CONFIG_X86_64)
+void aes_ctr64_crypt_aesni(u8 *dst, const u8 *src, s64 len, const u64 le_ctr[2],
+ const struct aes_enckey *key);
+
+static void aes_ctr64_x86(u8 *dst, const u8 *src, size_t len,
+ const u64 le_ctr[2], const struct aes_enckey *key)
+{
+ aes_ctr64_crypt_aesni(dst, src, len, le_ctr, key);
+}
+
+#define aes_ctr_arch aes_ctr_arch
+static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len,
+ u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key)
+{
+ u64 le_ctr[2];
+ u64 ctr64;
+ size_t nblocks;
+ size_t part1_len;
+
+ if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
+ return false;
+
+ ctr64 = le_ctr[0] = get_unaligned_be64(&ctr[8]);
+ le_ctr[1] = get_unaligned_be64(&ctr[0]);
+
+ kernel_fpu_begin();
+
+ nblocks = DIV_ROUND_UP(len, AES_BLOCK_SIZE);
+ ctr64 += nblocks;
+
+ if (likely(ctr64 >= nblocks)) {
+ /* The low 64 bits of the counter won't overflow. */
+ aes_ctr64_x86(dst, src, len, le_ctr, key);
+ } else {
+ /*
+ * The low 64 bits of the counter will overflow. The
+ * assembly doesn't handle this case, so split the
+ * operation into two at the point where the overflow
+ * will occur. After the first part, add the carry bit.
+ */
+ part1_len = min(len, (nblocks - ctr64) * AES_BLOCK_SIZE);
+ aes_ctr64_x86(dst, src, part1_len, le_ctr, key);
+ le_ctr[0] = 0;
+ le_ctr[1]++;
+ aes_ctr64_x86(dst + part1_len, src + part1_len, len - part1_len,
+ le_ctr, key);
+ }
+ kernel_fpu_end();
+ put_unaligned_be64(ctr64, &ctr[8]);
+ put_unaligned_be64(le_ctr[1], &ctr[0]);
+ return true;
+}
+#endif /* CONFIG_CRYPTO_LIB_AES_CTR && CONFIG_X86_64 */
+
#define aes_mod_init_arch aes_mod_init_arch
static void aes_mod_init_arch(void)
{
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread* [PATCH v2 07/20] lib/crypto: x86/aes-xts: Add AES-NI optimization
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
` (5 preceding siblings ...)
2026-09-27 22:42 ` [PATCH v2 06/20] lib/crypto: x86/aes-ctr: " Eric Biggers
@ 2026-09-27 22:42 ` Eric Biggers
2026-09-27 22:42 ` [PATCH v2 08/20] crypto: x86/aes - Drop superseded 32-bit build support Eric Biggers
` (12 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:42 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers
Optimize the crypto library's AES-XTS support with AES-NI, bringing its
performance on par with the "xts-aes-aesni" skcipher algorithm it will
supersede.
The new assembly code is written from scratch to fit well into the
crypto library and to be more consistent with aes-xts-avx-x86_64.S than
the code in arch/x86/crypto/aesni-intel_asm.S that it will supersede.
At a high level it is quite similar though, including doing 4 blocks per
iteration and supporting 32-bit mode for parity with the old code.
The new assembly code also fixes the flaw the old code had where the
encrypted tweaks were spilled to the destination buffer, rather than
kept entirely in registers (64-bit mode) or spilled to the stack (32-bit
mode). The encrypted tweaks are secret values that should not be
exposed to any code that doesn't have access to the key itself.
Note: the priority of xts-aes-lib is left unchanged at 110 temporarily.
It will be increased when the AVX-optimized AES-XTS code is migrated
too. Most systems use the AVX-optimized code; this commit just deals
with support for older CPUs that have AES-NI but not AVX (and 32-bit).
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/x86/aes-aesni.S | 159 +++++++++++++++++++++++++++++++++++++
lib/crypto/x86/aes.h | 44 ++++++++++
2 files changed, 203 insertions(+)
diff --git a/lib/crypto/x86/aes-aesni.S b/lib/crypto/x86/aes-aesni.S
index 297fe21ba830..cbc0cc23f63e 100644
--- a/lib/crypto/x86/aes-aesni.S
+++ b/lib/crypto/x86/aes-aesni.S
@@ -49,6 +49,17 @@
.section .rodata
.p2align 4
+.Lxts_gf_poly:
+ // For XTS: a constant used when advancing the tweak by one block by
+ // multiplying by the polynomial 'x' in GF(2^128). The low 64 bits of
+ // this value represent the polynomial x^7 + x^2 + x + 1; it is the
+ // value that must be XOR'd into the low 64 bits of the tweak each time
+ // a 1 is carried out of the high 64 bits.
+ //
+ // The high 64 bits of this value is just the internal carry bit that
+ // exists when there's a carry out of the low 64 bits of the tweak.
+ .quad 0x87, 1
+
#ifdef __x86_64__
.Lbswap_mask:
// A mask for pshufb that byte-reflects the value.
@@ -762,3 +773,151 @@ SYM_FUNC_START(aes_ctr64_crypt_aesni)
RET
SYM_FUNC_END(aes_ctr64_crypt_aesni)
#endif // __x86_64__
+
+// Given a 128-bit XTS tweak in the xmm register \tweak, compute the next tweak
+// (by multiplying by the polynomial 'x') and write it back to \tweak.
+.macro _next_tweak tweak, tmp
+ pshufd $0x13, \tweak, \tmp
+ paddq \tweak, \tweak
+ psrad $31, \tmp
+ pand GF_POLY, \tmp
+ pxor \tmp, \tweak
+.endm
+
+.macro _aes_xts_crypt enc
+ // Arguments
+ .set DST, ARG0
+ .set SRC, ARG1
+ .set NBLOCKS, ARG2
+ .set NBLOCKS32, ARG2_32 // Used for improved code density
+ .set TWEAK_PTR, ARG3
+ .set KEY, ARG4
+
+ // Other local variables
+#ifdef __x86_64__
+ .set RNDKEY_PTR, %r9
+#else
+ .set RNDKEY_PTR, TWEAK_PTR // TWEAK_PTR is clobbered and reloaded later.
+#endif
+ .set NROUNDS, TMP_32
+ .set AESDATA0, %xmm0
+ .set AESDATA1, %xmm1
+ .set AESDATA2, %xmm2
+ .set AESDATA3, %xmm3
+ .set GF_POLY, %xmm4
+ .set RNDKEY, %xmm5
+ .set TWEAK, %xmm6
+ .set SAVED_TWEAK0, %xmm7
+#ifdef __x86_64__
+ .set SAVED_TWEAK1, %xmm8
+ .set SAVED_TWEAK2, %xmm9
+#endif
+
+ _prologue uses_arg3=2, uses_arg4=2
+#ifdef __i386__
+ // Reserve 16-byte aligned space to spill two tweaks.
+ push %ebp
+ mov %esp, %ebp
+ and $~15, %esp
+ sub $32, %esp
+#endif
+
+ movdqu (TWEAK_PTR), TWEAK
+ movdqa RODATA(.Lxts_gf_poly), GF_POLY
+
+ sub $4, NBLOCKS
+ jl .Lxts_loop4_done\@
+.p2align 5
+.Lxts_loop4\@:
+ // Load the next four source blocks into AESDATA[0-3] and XOR them with
+ // their tweaks, advancing the tweak three times in order to do so.
+ // Save the four tweaks for later; on 64-bit they all fit into
+ // registers, while on 32-bit two tweaks are spilled to the stack.
+.irp i, 0,1,2,3
+ movdqu \i*16(SRC), AESDATA\i
+ pxor TWEAK, AESDATA\i
+ .if \i != 3
+#ifdef __x86_64__
+ movdqa TWEAK, SAVED_TWEAK\i
+#else
+ .if \i == 0
+ movdqa TWEAK, SAVED_TWEAK0
+ .else
+ movdqa TWEAK, (\i-1)*16(%esp)
+ .endif
+#endif
+ _next_tweak TWEAK, RNDKEY
+ .endif
+.endr
+
+ // Encrypt or decrypt the blocks.
+ _do_aes \enc, 0,1,2,3
+
+ // XOR the blocks with the saved tweaks.
+ pxor SAVED_TWEAK0, AESDATA0
+#ifdef __x86_64__
+ pxor SAVED_TWEAK1, AESDATA1
+ pxor SAVED_TWEAK2, AESDATA2
+#else
+ pxor (%esp), AESDATA1
+ pxor 16(%esp), AESDATA2
+#endif
+ pxor TWEAK, AESDATA3
+
+ // Store the encrypted or decrypted blocks.
+.irp i, 0,1,2,3
+ movdqu AESDATA\i, \i*16(DST)
+.endr
+
+ _next_tweak TWEAK, RNDKEY
+ add $64, SRC
+ add $64, DST
+ sub $4, NBLOCKS
+ jge .Lxts_loop4\@
+.Lxts_loop4_done\@:
+ add $4, NBLOCKS32
+ jz .Lxts_done\@
+
+.Lxts_loop1\@:
+ movdqu (SRC), AESDATA0
+ pxor TWEAK, AESDATA0
+ _do_aes \enc, 0
+ pxor TWEAK, AESDATA0
+ movdqu AESDATA0, (DST)
+ _next_tweak TWEAK, RNDKEY
+ add $16, SRC
+ add $16, DST
+ dec NBLOCKS32
+ jnz .Lxts_loop1\@
+
+.Lxts_done\@:
+#ifdef __i386__
+ // Zeroize the stack buffer that tweaks were spilled to.
+ pxor AESDATA0, AESDATA0
+ movdqa AESDATA0, (%esp)
+ movdqa AESDATA0, 16(%esp)
+ mov %ebp, %esp
+ pop %ebp
+#endif
+ // Store the next tweak. On 32-bit, reload TWEAK_PTR from stack first.
+ _reload_arg3
+ movdqu TWEAK, (TWEAK_PTR)
+ _epilogue
+.endm
+
+// void aes_xts_encrypt_aesni(u8 *dst, const u8 *src, long nblocks,
+// u8 tweak[AES_BLOCK_SIZE],
+// const struct aes_key *key);
+// void aes_xts_decrypt_aesni(u8 *dst, const u8 *src, long nblocks,
+// u8 tweak[AES_BLOCK_SIZE],
+// const struct aes_key *key);
+//
+// `tweak` must have already been encrypted by the tweak key; `key` is just the
+// main key. To allow incremental computation, `tweak` is updated to contain
+// the next tweak.
+SYM_FUNC_START(aes_xts_encrypt_aesni)
+ _aes_xts_crypt 1
+SYM_FUNC_END(aes_xts_encrypt_aesni)
+SYM_FUNC_START(aes_xts_decrypt_aesni)
+ _aes_xts_crypt 0
+SYM_FUNC_END(aes_xts_decrypt_aesni)
diff --git a/lib/crypto/x86/aes.h b/lib/crypto/x86/aes.h
index 2a2b26d10e87..54b599bd2581 100644
--- a/lib/crypto/x86/aes.h
+++ b/lib/crypto/x86/aes.h
@@ -259,6 +259,50 @@ static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len,
}
#endif /* CONFIG_CRYPTO_LIB_AES_CTR && CONFIG_X86_64 */
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS)
+void aes_xts_encrypt_aesni(u8 *dst, const u8 *src, long nblocks,
+ u8 tweak[AES_BLOCK_SIZE], const struct aes_key *key);
+void aes_xts_decrypt_aesni(u8 *dst, const u8 *src, long nblocks,
+ u8 tweak[AES_BLOCK_SIZE], const struct aes_key *key);
+
+/* len is always a positive multiple of AES_BLOCK_SIZE here. */
+static __always_inline bool
+aes_xts_crypt_x86(u8 *dst, const u8 *src, size_t len, u8 tweak[AES_BLOCK_SIZE],
+ const struct aes_xts_key *key, bool cont, bool enc)
+{
+ const long nblocks = len / AES_BLOCK_SIZE;
+
+ if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
+ return false;
+
+ kernel_fpu_begin();
+ if (!cont)
+ aes_encrypt_aesni(tweak, tweak, &key->tweak_key);
+ if (enc)
+ aes_xts_encrypt_aesni(dst, src, nblocks, tweak, &key->main_key);
+ else
+ aes_xts_decrypt_aesni(dst, src, nblocks, tweak, &key->main_key);
+ kernel_fpu_end();
+ return true;
+}
+
+#define aes_xts_encrypt_arch aes_xts_encrypt_arch
+static bool aes_xts_encrypt_arch(u8 *dst, const u8 *src, size_t len,
+ u8 tweak[AES_BLOCK_SIZE],
+ const struct aes_xts_key *key, bool cont)
+{
+ return aes_xts_crypt_x86(dst, src, len, tweak, key, cont, true);
+}
+
+#define aes_xts_decrypt_arch aes_xts_decrypt_arch
+static bool aes_xts_decrypt_arch(u8 *dst, const u8 *src, size_t len,
+ u8 tweak[AES_BLOCK_SIZE],
+ const struct aes_xts_key *key, bool cont)
+{
+ return aes_xts_crypt_x86(dst, src, len, tweak, key, cont, false);
+}
+#endif /* CONFIG_CRYPTO_LIB_AES_XTS */
+
#define aes_mod_init_arch aes_mod_init_arch
static void aes_mod_init_arch(void)
{
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread* [PATCH v2 08/20] crypto: x86/aes - Drop superseded 32-bit build support
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
` (6 preceding siblings ...)
2026-09-27 22:42 ` [PATCH v2 07/20] lib/crypto: x86/aes-xts: " Eric Biggers
@ 2026-09-27 22:42 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 09/20] crypto: x86/aes-ecb - Remove superseded ECB skcipher Eric Biggers
` (11 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:42 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers
Now that bare AES, AES-ECB, AES-CBC, AES-CBC-CTS, and AES-XTS are
accelerated on 32-bit x86 kernels via libaes, the equivalent code in
aesni-intel is redundant. Only the 64-bit-only code in aesni-intel is
still needed. Thus, drop the 32-bit build support from aesni-intel.
These AES modes remain accelerated on 32-bit kernels via libaes.
aesni-intel_asm.S is left untouched, as later commits remove that file
entirely one mode at a time anyway. This commit just drops 32-bit
support right away to prevent bisection hazards during that.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/x86/crypto/Kconfig | 6 ++----
arch/x86/crypto/Makefile | 13 +++++++------
arch/x86/crypto/aesni-intel_glue.c | 22 +---------------------
3 files changed, 10 insertions(+), 31 deletions(-)
diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig
index f65d7b83702f..3c4c582e7f2d 100644
--- a/arch/x86/crypto/Kconfig
+++ b/arch/x86/crypto/Kconfig
@@ -4,6 +4,7 @@ menu "Accelerated Cryptographic Algorithms for CPU (x86)"
config CRYPTO_AES_NI_INTEL
tristate "Ciphers: AES, modes: ECB, CBC, CTS, CTR, XCTR, XTS, GCM (AES-NI/VAES)"
+ depends on 64BIT
select CRYPTO_AEAD
select CRYPTO_LIB_AES
select CRYPTO_LIB_GF128MUL
@@ -12,13 +13,10 @@ config CRYPTO_AES_NI_INTEL
AEAD cipher: AES with GCM
Length-preserving ciphers: AES with ECB, CBC, CTS, CTR, XCTR, XTS
- Architecture: x86 (32-bit and 64-bit) using:
+ Architecture: x86_64 using:
- AES-NI (AES new instructions)
- VAES (Vector AES)
- Some algorithm implementations are supported only in 64-bit builds,
- and some have additional prerequisites such as AVX2 or AVX512.
-
config CRYPTO_BLOWFISH_X86_64
tristate "Ciphers: Blowfish, modes: ECB, CBC"
depends on 64BIT
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index e04ff8718d6b..1c3feb6d72b7 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -40,12 +40,13 @@ obj-$(CONFIG_CRYPTO_AEGIS128_AESNI_SSE2) += aegis128-aesni.o
aegis128-aesni-y := aegis128-aesni-asm.o aegis128-aesni-glue.o
obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o
-aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o
-aesni-intel-$(CONFIG_64BIT) += aes-ctr-avx-x86_64.o \
- aes-gcm-aesni-x86_64.o \
- aes-gcm-vaes-avx2.o \
- aes-gcm-vaes-avx512.o \
- aes-xts-avx-x86_64.o
+aesni-intel-y := aesni-intel_asm.o \
+ aesni-intel_glue.o \
+ aes-ctr-avx-x86_64.o \
+ aes-gcm-aesni-x86_64.o \
+ aes-gcm-vaes-avx2.o \
+ aes-gcm-vaes-avx512.o \
+ aes-xts-avx-x86_64.o
obj-$(CONFIG_CRYPTO_SM4_AESNI_AVX_X86_64) += sm4-aesni-avx-x86_64.o
sm4-aesni-avx-x86_64-y := sm4-aesni-avx-asm_64.o sm4_aesni_avx_glue.o
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index f522fff9231e..259e319ff92b 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -80,10 +80,8 @@ asmlinkage void aesni_xts_enc(const struct crypto_aes_ctx *ctx, u8 *out,
asmlinkage void aesni_xts_dec(const struct crypto_aes_ctx *ctx, u8 *out,
const u8 *in, unsigned int len, u8 *iv);
-#ifdef CONFIG_X86_64
asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
const u8 *in, unsigned int len, u8 *iv);
-#endif
static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
{
@@ -319,7 +317,6 @@ static int cts_cbc_decrypt(struct skcipher_request *req)
return skcipher_walk_done(&walk, 0);
}
-#ifdef CONFIG_X86_64
/* This is the non-AVX version. */
static int ctr_crypt_aesni(struct skcipher_request *req)
{
@@ -353,7 +350,6 @@ static int ctr_crypt_aesni(struct skcipher_request *req)
}
return err;
}
-#endif
static int xts_setkey_aesni(struct crypto_skcipher *tfm, const u8 *key,
unsigned int keylen)
@@ -469,8 +465,7 @@ xts_crypt(struct skcipher_request *req, xts_encrypt_iv_func encrypt_iv,
* single-scatterlist-element messages as efficiently as possible. The
* code is 64-bit specific, as it assumes no page mapping is needed.
*/
- if (IS_ENABLED(CONFIG_X86_64) &&
- likely(req->src->length >= req->cryptlen &&
+ if (likely(req->src->length >= req->cryptlen &&
req->dst->length >= req->cryptlen)) {
(*crypt_func)(&ctx->crypt_ctx, sg_virt(req->src),
sg_virt(req->dst), req->cryptlen, req->iv);
@@ -557,7 +552,6 @@ static struct skcipher_alg aesni_skciphers[] = {
.setkey = aesni_skcipher_setkey,
.encrypt = cts_cbc_encrypt,
.decrypt = cts_cbc_decrypt,
-#ifdef CONFIG_X86_64
}, {
.base = {
.cra_name = "ctr(aes)",
@@ -574,7 +568,6 @@ static struct skcipher_alg aesni_skciphers[] = {
.setkey = aesni_skcipher_setkey,
.encrypt = ctr_crypt_aesni,
.decrypt = ctr_crypt_aesni,
-#endif
}, {
.base = {
.cra_name = "xts(aes)",
@@ -594,7 +587,6 @@ static struct skcipher_alg aesni_skciphers[] = {
}
};
-#ifdef CONFIG_X86_64
asmlinkage void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key,
u8 iv[AES_BLOCK_SIZE]);
@@ -1604,18 +1596,6 @@ static void unregister_avx_algs(void)
unregister_aeads(aes_gcm_algs_vaes_avx2);
unregister_aeads(aes_gcm_algs_vaes_avx512);
}
-#else /* CONFIG_X86_64 */
-static struct aead_alg aes_gcm_algs_aesni[0];
-
-static int __init register_avx_algs(void)
-{
- return 0;
-}
-
-static void unregister_avx_algs(void)
-{
-}
-#endif /* !CONFIG_X86_64 */
static const struct x86_cpu_id aesni_cpu_id[] = {
X86_MATCH_FEATURE(X86_FEATURE_AES, NULL),
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread* [PATCH v2 09/20] crypto: x86/aes-ecb - Remove superseded ECB skcipher
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
` (7 preceding siblings ...)
2026-09-27 22:42 ` [PATCH v2 08/20] crypto: x86/aes - Drop superseded 32-bit build support Eric Biggers
@ 2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 10/20] crypto: x86/aes-cbc - Remove superseded CBC skciphers Eric Biggers
` (10 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:43 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers
Now that the AES-ECB library functions are optimized with dedicated
AES-NI code and exposed via the skcipher API via crypto/aes.c, the
similar implementation in aesni-intel is redundant. Remove it.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/x86/crypto/Kconfig | 4 +-
arch/x86/crypto/aesni-intel_asm.S | 121 -----------------------------
arch/x86/crypto/aesni-intel_glue.c | 62 ---------------
3 files changed, 2 insertions(+), 185 deletions(-)
diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig
index 3c4c582e7f2d..f7c4243dd187 100644
--- a/arch/x86/crypto/Kconfig
+++ b/arch/x86/crypto/Kconfig
@@ -3,7 +3,7 @@
menu "Accelerated Cryptographic Algorithms for CPU (x86)"
config CRYPTO_AES_NI_INTEL
- tristate "Ciphers: AES, modes: ECB, CBC, CTS, CTR, XCTR, XTS, GCM (AES-NI/VAES)"
+ tristate "Ciphers: AES, modes: CBC, CTS, CTR, XCTR, XTS, GCM (AES-NI/VAES)"
depends on 64BIT
select CRYPTO_AEAD
select CRYPTO_LIB_AES
@@ -11,7 +11,7 @@ config CRYPTO_AES_NI_INTEL
select CRYPTO_SKCIPHER
help
AEAD cipher: AES with GCM
- Length-preserving ciphers: AES with ECB, CBC, CTS, CTR, XCTR, XTS
+ Length-preserving ciphers: AES with CBC, CTS, CTR, XCTR, XTS
Architecture: x86_64 using:
- AES-NI (AES new instructions)
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index 6abe5e38a6d7..db7f259ee8ab 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -601,127 +601,6 @@ SYM_FUNC_START_LOCAL(_aesni_dec4)
RET
SYM_FUNC_END(_aesni_dec4)
-/*
- * void aesni_ecb_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
- * size_t len)
- */
-SYM_FUNC_START(aesni_ecb_enc)
- FRAME_BEGIN
-#ifndef __x86_64__
- pushl LEN
- pushl KEYP
- pushl KLEN
- movl (FRAME_OFFSET+16)(%esp), KEYP # ctx
- movl (FRAME_OFFSET+20)(%esp), OUTP # dst
- movl (FRAME_OFFSET+24)(%esp), INP # src
- movl (FRAME_OFFSET+28)(%esp), LEN # len
-#endif
- test LEN, LEN # check length
- jz .Lecb_enc_ret
- mov 480(KEYP), KLEN
- cmp $16, LEN
- jb .Lecb_enc_ret
- cmp $64, LEN
- jb .Lecb_enc_loop1
-.align 4
-.Lecb_enc_loop4:
- movups (INP), STATE1
- movups 0x10(INP), STATE2
- movups 0x20(INP), STATE3
- movups 0x30(INP), STATE4
- call _aesni_enc4
- movups STATE1, (OUTP)
- movups STATE2, 0x10(OUTP)
- movups STATE3, 0x20(OUTP)
- movups STATE4, 0x30(OUTP)
- sub $64, LEN
- add $64, INP
- add $64, OUTP
- cmp $64, LEN
- jge .Lecb_enc_loop4
- cmp $16, LEN
- jb .Lecb_enc_ret
-.align 4
-.Lecb_enc_loop1:
- movups (INP), STATE1
- call _aesni_enc1
- movups STATE1, (OUTP)
- sub $16, LEN
- add $16, INP
- add $16, OUTP
- cmp $16, LEN
- jge .Lecb_enc_loop1
-.Lecb_enc_ret:
-#ifndef __x86_64__
- popl KLEN
- popl KEYP
- popl LEN
-#endif
- FRAME_END
- RET
-SYM_FUNC_END(aesni_ecb_enc)
-
-/*
- * void aesni_ecb_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
- * size_t len);
- */
-SYM_FUNC_START(aesni_ecb_dec)
- FRAME_BEGIN
-#ifndef __x86_64__
- pushl LEN
- pushl KEYP
- pushl KLEN
- movl (FRAME_OFFSET+16)(%esp), KEYP # ctx
- movl (FRAME_OFFSET+20)(%esp), OUTP # dst
- movl (FRAME_OFFSET+24)(%esp), INP # src
- movl (FRAME_OFFSET+28)(%esp), LEN # len
-#endif
- test LEN, LEN
- jz .Lecb_dec_ret
- mov 480(KEYP), KLEN
- add $240, KEYP
- cmp $16, LEN
- jb .Lecb_dec_ret
- cmp $64, LEN
- jb .Lecb_dec_loop1
-.align 4
-.Lecb_dec_loop4:
- movups (INP), STATE1
- movups 0x10(INP), STATE2
- movups 0x20(INP), STATE3
- movups 0x30(INP), STATE4
- call _aesni_dec4
- movups STATE1, (OUTP)
- movups STATE2, 0x10(OUTP)
- movups STATE3, 0x20(OUTP)
- movups STATE4, 0x30(OUTP)
- sub $64, LEN
- add $64, INP
- add $64, OUTP
- cmp $64, LEN
- jge .Lecb_dec_loop4
- cmp $16, LEN
- jb .Lecb_dec_ret
-.align 4
-.Lecb_dec_loop1:
- movups (INP), STATE1
- call _aesni_dec1
- movups STATE1, (OUTP)
- sub $16, LEN
- add $16, INP
- add $16, OUTP
- cmp $16, LEN
- jge .Lecb_dec_loop1
-.Lecb_dec_ret:
-#ifndef __x86_64__
- popl KLEN
- popl KEYP
- popl LEN
-#endif
- FRAME_END
- RET
-SYM_FUNC_END(aesni_ecb_dec)
-
/*
* void aesni_cbc_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
* size_t len, u8 *iv)
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 259e319ff92b..c37976d06f45 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -61,10 +61,6 @@ static inline void *aes_align_addr(void *addr)
asmlinkage void aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
unsigned int key_len);
asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in);
-asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
- const u8 *in, unsigned int len);
-asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
- const u8 *in, unsigned int len);
asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
const u8 *in, unsigned int len, u8 *iv);
asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
@@ -117,50 +113,6 @@ static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
return aes_set_key_common(aes_ctx(crypto_skcipher_ctx(tfm)), key, len);
}
-static int ecb_encrypt(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
- struct skcipher_walk walk;
- unsigned int nbytes;
- int err;
-
- err = skcipher_walk_virt(&walk, req, false);
-
- while ((nbytes = walk.nbytes)) {
- kernel_fpu_begin();
- aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
- nbytes & AES_BLOCK_MASK);
- kernel_fpu_end();
- nbytes &= AES_BLOCK_SIZE - 1;
- err = skcipher_walk_done(&walk, nbytes);
- }
-
- return err;
-}
-
-static int ecb_decrypt(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
- struct skcipher_walk walk;
- unsigned int nbytes;
- int err;
-
- err = skcipher_walk_virt(&walk, req, false);
-
- while ((nbytes = walk.nbytes)) {
- kernel_fpu_begin();
- aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
- nbytes & AES_BLOCK_MASK);
- kernel_fpu_end();
- nbytes &= AES_BLOCK_SIZE - 1;
- err = skcipher_walk_done(&walk, nbytes);
- }
-
- return err;
-}
-
static int cbc_encrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@@ -508,20 +460,6 @@ static int xts_decrypt_aesni(struct skcipher_request *req)
static struct skcipher_alg aesni_skciphers[] = {
{
- .base = {
- .cra_name = "ecb(aes)",
- .cra_driver_name = "ecb-aes-aesni",
- .cra_priority = 400,
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
- .cra_module = THIS_MODULE,
- },
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .setkey = aesni_skcipher_setkey,
- .encrypt = ecb_encrypt,
- .decrypt = ecb_decrypt,
- }, {
.base = {
.cra_name = "cbc(aes)",
.cra_driver_name = "cbc-aes-aesni",
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread* [PATCH v2 10/20] crypto: x86/aes-cbc - Remove superseded CBC skciphers
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
` (8 preceding siblings ...)
2026-09-27 22:43 ` [PATCH v2 09/20] crypto: x86/aes-ecb - Remove superseded ECB skcipher Eric Biggers
@ 2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 11/20] crypto: x86/aes-ctr - Remove superseded CTR skcipher Eric Biggers
` (9 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:43 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers
Now that the AES-CBC (and AES-CBC-CTS) library functions are optimized
with dedicated AES-NI code and exposed via the skcipher API via
crypto/aes.c, the similar implementation in aesni-intel is redundant.
Remove it.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/x86/crypto/Kconfig | 4 +-
arch/x86/crypto/aesni-intel_asm.S | 255 -----------------------------
arch/x86/crypto/aesni-intel_glue.c | 195 ----------------------
3 files changed, 2 insertions(+), 452 deletions(-)
diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig
index f7c4243dd187..79e610b5e5bc 100644
--- a/arch/x86/crypto/Kconfig
+++ b/arch/x86/crypto/Kconfig
@@ -3,7 +3,7 @@
menu "Accelerated Cryptographic Algorithms for CPU (x86)"
config CRYPTO_AES_NI_INTEL
- tristate "Ciphers: AES, modes: CBC, CTS, CTR, XCTR, XTS, GCM (AES-NI/VAES)"
+ tristate "Ciphers: AES, modes: CTR, XCTR, XTS, GCM (AES-NI/VAES)"
depends on 64BIT
select CRYPTO_AEAD
select CRYPTO_LIB_AES
@@ -11,7 +11,7 @@ config CRYPTO_AES_NI_INTEL
select CRYPTO_SKCIPHER
help
AEAD cipher: AES with GCM
- Length-preserving ciphers: AES with CBC, CTS, CTR, XCTR, XTS
+ Length-preserving ciphers: AES with CTR, XCTR, XTS
Architecture: x86_64 using:
- AES-NI (AES new instructions)
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index db7f259ee8ab..16c406781b24 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -601,261 +601,6 @@ SYM_FUNC_START_LOCAL(_aesni_dec4)
RET
SYM_FUNC_END(_aesni_dec4)
-/*
- * void aesni_cbc_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
- * size_t len, u8 *iv)
- */
-SYM_FUNC_START(aesni_cbc_enc)
- FRAME_BEGIN
-#ifndef __x86_64__
- pushl IVP
- pushl LEN
- pushl KEYP
- pushl KLEN
- movl (FRAME_OFFSET+20)(%esp), KEYP # ctx
- movl (FRAME_OFFSET+24)(%esp), OUTP # dst
- movl (FRAME_OFFSET+28)(%esp), INP # src
- movl (FRAME_OFFSET+32)(%esp), LEN # len
- movl (FRAME_OFFSET+36)(%esp), IVP # iv
-#endif
- cmp $16, LEN
- jb .Lcbc_enc_ret
- mov 480(KEYP), KLEN
- movups (IVP), STATE # load iv as initial state
-.align 4
-.Lcbc_enc_loop:
- movups (INP), IN # load input
- pxor IN, STATE
- call _aesni_enc1
- movups STATE, (OUTP) # store output
- sub $16, LEN
- add $16, INP
- add $16, OUTP
- cmp $16, LEN
- jge .Lcbc_enc_loop
- movups STATE, (IVP)
-.Lcbc_enc_ret:
-#ifndef __x86_64__
- popl KLEN
- popl KEYP
- popl LEN
- popl IVP
-#endif
- FRAME_END
- RET
-SYM_FUNC_END(aesni_cbc_enc)
-
-/*
- * void aesni_cbc_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
- * size_t len, u8 *iv)
- */
-SYM_FUNC_START(aesni_cbc_dec)
- FRAME_BEGIN
-#ifndef __x86_64__
- pushl IVP
- pushl LEN
- pushl KEYP
- pushl KLEN
- movl (FRAME_OFFSET+20)(%esp), KEYP # ctx
- movl (FRAME_OFFSET+24)(%esp), OUTP # dst
- movl (FRAME_OFFSET+28)(%esp), INP # src
- movl (FRAME_OFFSET+32)(%esp), LEN # len
- movl (FRAME_OFFSET+36)(%esp), IVP # iv
-#endif
- cmp $16, LEN
- jb .Lcbc_dec_just_ret
- mov 480(KEYP), KLEN
- add $240, KEYP
- movups (IVP), IV
- cmp $64, LEN
- jb .Lcbc_dec_loop1
-.align 4
-.Lcbc_dec_loop4:
- movups (INP), IN1
- movaps IN1, STATE1
- movups 0x10(INP), IN2
- movaps IN2, STATE2
-#ifdef __x86_64__
- movups 0x20(INP), IN3
- movaps IN3, STATE3
- movups 0x30(INP), IN4
- movaps IN4, STATE4
-#else
- movups 0x20(INP), IN1
- movaps IN1, STATE3
- movups 0x30(INP), IN2
- movaps IN2, STATE4
-#endif
- call _aesni_dec4
- pxor IV, STATE1
-#ifdef __x86_64__
- pxor IN1, STATE2
- pxor IN2, STATE3
- pxor IN3, STATE4
- movaps IN4, IV
-#else
- pxor IN1, STATE4
- movaps IN2, IV
- movups (INP), IN1
- pxor IN1, STATE2
- movups 0x10(INP), IN2
- pxor IN2, STATE3
-#endif
- movups STATE1, (OUTP)
- movups STATE2, 0x10(OUTP)
- movups STATE3, 0x20(OUTP)
- movups STATE4, 0x30(OUTP)
- sub $64, LEN
- add $64, INP
- add $64, OUTP
- cmp $64, LEN
- jge .Lcbc_dec_loop4
- cmp $16, LEN
- jb .Lcbc_dec_ret
-.align 4
-.Lcbc_dec_loop1:
- movups (INP), IN
- movaps IN, STATE
- call _aesni_dec1
- pxor IV, STATE
- movups STATE, (OUTP)
- movaps IN, IV
- sub $16, LEN
- add $16, INP
- add $16, OUTP
- cmp $16, LEN
- jge .Lcbc_dec_loop1
-.Lcbc_dec_ret:
- movups IV, (IVP)
-.Lcbc_dec_just_ret:
-#ifndef __x86_64__
- popl KLEN
- popl KEYP
- popl LEN
- popl IVP
-#endif
- FRAME_END
- RET
-SYM_FUNC_END(aesni_cbc_dec)
-
-/*
- * void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
- * size_t len, u8 *iv)
- */
-SYM_FUNC_START(aesni_cts_cbc_enc)
- FRAME_BEGIN
-#ifndef __x86_64__
- pushl IVP
- pushl LEN
- pushl KEYP
- pushl KLEN
- movl (FRAME_OFFSET+20)(%esp), KEYP # ctx
- movl (FRAME_OFFSET+24)(%esp), OUTP # dst
- movl (FRAME_OFFSET+28)(%esp), INP # src
- movl (FRAME_OFFSET+32)(%esp), LEN # len
- movl (FRAME_OFFSET+36)(%esp), IVP # iv
- lea .Lcts_permute_table, T1
-#else
- lea .Lcts_permute_table(%rip), T1
-#endif
- mov 480(KEYP), KLEN
- movups (IVP), STATE
- sub $16, LEN
- mov T1, IVP
- add $32, IVP
- add LEN, T1
- sub LEN, IVP
- movups (T1), %xmm4
- movups (IVP), %xmm5
-
- movups (INP), IN1
- add LEN, INP
- movups (INP), IN2
-
- pxor IN1, STATE
- call _aesni_enc1
-
- pshufb %xmm5, IN2
- pxor STATE, IN2
- pshufb %xmm4, STATE
- add OUTP, LEN
- movups STATE, (LEN)
-
- movaps IN2, STATE
- call _aesni_enc1
- movups STATE, (OUTP)
-
-#ifndef __x86_64__
- popl KLEN
- popl KEYP
- popl LEN
- popl IVP
-#endif
- FRAME_END
- RET
-SYM_FUNC_END(aesni_cts_cbc_enc)
-
-/*
- * void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
- * size_t len, u8 *iv)
- */
-SYM_FUNC_START(aesni_cts_cbc_dec)
- FRAME_BEGIN
-#ifndef __x86_64__
- pushl IVP
- pushl LEN
- pushl KEYP
- pushl KLEN
- movl (FRAME_OFFSET+20)(%esp), KEYP # ctx
- movl (FRAME_OFFSET+24)(%esp), OUTP # dst
- movl (FRAME_OFFSET+28)(%esp), INP # src
- movl (FRAME_OFFSET+32)(%esp), LEN # len
- movl (FRAME_OFFSET+36)(%esp), IVP # iv
- lea .Lcts_permute_table, T1
-#else
- lea .Lcts_permute_table(%rip), T1
-#endif
- mov 480(KEYP), KLEN
- add $240, KEYP
- movups (IVP), IV
- sub $16, LEN
- mov T1, IVP
- add $32, IVP
- add LEN, T1
- sub LEN, IVP
- movups (T1), %xmm4
-
- movups (INP), STATE
- add LEN, INP
- movups (INP), IN1
-
- call _aesni_dec1
- movaps STATE, IN2
- pshufb %xmm4, STATE
- pxor IN1, STATE
-
- add OUTP, LEN
- movups STATE, (LEN)
-
- movups (IVP), %xmm0
- pshufb %xmm0, IN1
- pblendvb IN2, IN1
- movaps IN1, STATE
- call _aesni_dec1
-
- pxor IV, STATE
- movups STATE, (OUTP)
-
-#ifndef __x86_64__
- popl KLEN
- popl KEYP
- popl LEN
- popl IVP
-#endif
- FRAME_END
- RET
-SYM_FUNC_END(aesni_cts_cbc_dec)
-
.pushsection .rodata
.align 16
.Lcts_permute_table:
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index c37976d06f45..3fd76dd82e1b 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -61,14 +61,6 @@ static inline void *aes_align_addr(void *addr)
asmlinkage void aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
unsigned int key_len);
asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in);
-asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
- const u8 *in, unsigned int len, u8 *iv);
-asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
- const u8 *in, unsigned int len, u8 *iv);
-asmlinkage void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
- const u8 *in, unsigned int len, u8 *iv);
-asmlinkage void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
- const u8 *in, unsigned int len, u8 *iv);
asmlinkage void aesni_xts_enc(const struct crypto_aes_ctx *ctx, u8 *out,
const u8 *in, unsigned int len, u8 *iv);
@@ -113,162 +105,6 @@ static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
return aes_set_key_common(aes_ctx(crypto_skcipher_ctx(tfm)), key, len);
}
-static int cbc_encrypt(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
- struct skcipher_walk walk;
- unsigned int nbytes;
- int err;
-
- err = skcipher_walk_virt(&walk, req, false);
-
- while ((nbytes = walk.nbytes)) {
- kernel_fpu_begin();
- aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
- nbytes & AES_BLOCK_MASK, walk.iv);
- kernel_fpu_end();
- nbytes &= AES_BLOCK_SIZE - 1;
- err = skcipher_walk_done(&walk, nbytes);
- }
-
- return err;
-}
-
-static int cbc_decrypt(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
- struct skcipher_walk walk;
- unsigned int nbytes;
- int err;
-
- err = skcipher_walk_virt(&walk, req, false);
-
- while ((nbytes = walk.nbytes)) {
- kernel_fpu_begin();
- aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
- nbytes & AES_BLOCK_MASK, walk.iv);
- kernel_fpu_end();
- nbytes &= AES_BLOCK_SIZE - 1;
- err = skcipher_walk_done(&walk, nbytes);
- }
-
- return err;
-}
-
-static int cts_cbc_encrypt(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
- int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
- struct scatterlist *src = req->src, *dst = req->dst;
- struct scatterlist sg_src[2], sg_dst[2];
- struct skcipher_request subreq;
- struct skcipher_walk walk;
- int err;
-
- skcipher_request_set_tfm(&subreq, tfm);
- skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
- NULL, NULL);
-
- if (req->cryptlen <= AES_BLOCK_SIZE) {
- if (req->cryptlen < AES_BLOCK_SIZE)
- return -EINVAL;
- cbc_blocks = 1;
- }
-
- if (cbc_blocks > 0) {
- skcipher_request_set_crypt(&subreq, req->src, req->dst,
- cbc_blocks * AES_BLOCK_SIZE,
- req->iv);
-
- err = cbc_encrypt(&subreq);
- if (err)
- return err;
-
- if (req->cryptlen == AES_BLOCK_SIZE)
- return 0;
-
- dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
- if (req->dst != req->src)
- dst = scatterwalk_ffwd(sg_dst, req->dst,
- subreq.cryptlen);
- }
-
- /* handle ciphertext stealing */
- skcipher_request_set_crypt(&subreq, src, dst,
- req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
- req->iv);
-
- err = skcipher_walk_virt(&walk, &subreq, false);
- if (err)
- return err;
-
- kernel_fpu_begin();
- aesni_cts_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
- walk.nbytes, walk.iv);
- kernel_fpu_end();
-
- return skcipher_walk_done(&walk, 0);
-}
-
-static int cts_cbc_decrypt(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
- int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
- struct scatterlist *src = req->src, *dst = req->dst;
- struct scatterlist sg_src[2], sg_dst[2];
- struct skcipher_request subreq;
- struct skcipher_walk walk;
- int err;
-
- skcipher_request_set_tfm(&subreq, tfm);
- skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
- NULL, NULL);
-
- if (req->cryptlen <= AES_BLOCK_SIZE) {
- if (req->cryptlen < AES_BLOCK_SIZE)
- return -EINVAL;
- cbc_blocks = 1;
- }
-
- if (cbc_blocks > 0) {
- skcipher_request_set_crypt(&subreq, req->src, req->dst,
- cbc_blocks * AES_BLOCK_SIZE,
- req->iv);
-
- err = cbc_decrypt(&subreq);
- if (err)
- return err;
-
- if (req->cryptlen == AES_BLOCK_SIZE)
- return 0;
-
- dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
- if (req->dst != req->src)
- dst = scatterwalk_ffwd(sg_dst, req->dst,
- subreq.cryptlen);
- }
-
- /* handle ciphertext stealing */
- skcipher_request_set_crypt(&subreq, src, dst,
- req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
- req->iv);
-
- err = skcipher_walk_virt(&walk, &subreq, false);
- if (err)
- return err;
-
- kernel_fpu_begin();
- aesni_cts_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
- walk.nbytes, walk.iv);
- kernel_fpu_end();
-
- return skcipher_walk_done(&walk, 0);
-}
-
/* This is the non-AVX version. */
static int ctr_crypt_aesni(struct skcipher_request *req)
{
@@ -460,37 +296,6 @@ static int xts_decrypt_aesni(struct skcipher_request *req)
static struct skcipher_alg aesni_skciphers[] = {
{
- .base = {
- .cra_name = "cbc(aes)",
- .cra_driver_name = "cbc-aes-aesni",
- .cra_priority = 400,
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
- .cra_module = THIS_MODULE,
- },
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .setkey = aesni_skcipher_setkey,
- .encrypt = cbc_encrypt,
- .decrypt = cbc_decrypt,
- }, {
- .base = {
- .cra_name = "cts(cbc(aes))",
- .cra_driver_name = "cts-cbc-aes-aesni",
- .cra_priority = 400,
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
- .cra_module = THIS_MODULE,
- },
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .walksize = 2 * AES_BLOCK_SIZE,
- .setkey = aesni_skcipher_setkey,
- .encrypt = cts_cbc_encrypt,
- .decrypt = cts_cbc_decrypt,
- }, {
.base = {
.cra_name = "ctr(aes)",
.cra_driver_name = "ctr-aes-aesni",
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread* [PATCH v2 11/20] crypto: x86/aes-ctr - Remove superseded CTR skcipher
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
` (9 preceding siblings ...)
2026-09-27 22:43 ` [PATCH v2 10/20] crypto: x86/aes-cbc - Remove superseded CBC skciphers Eric Biggers
@ 2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 12/20] crypto: x86/aes-xts - Remove superseded XTS skcipher Eric Biggers
` (8 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:43 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers
Now that the crypto library's AES-CTR support is optimized with AES-NI
(without AVX), the similar code in aesni-intel is redundant. Remove it.
This only affects the non-AVX implementation ("ctr-aes-aesni"), not
aes-ctr-avx-x86_64.S which is handled later.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/x86/crypto/aesni-intel_asm.S | 125 -----------------------------
arch/x86/crypto/aesni-intel_glue.c | 54 -------------
2 files changed, 179 deletions(-)
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index 16c406781b24..c4d54c4a2c23 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -33,10 +33,6 @@
#define KEY %xmm2
#define IV %xmm3
-#define BSWAP_MASK %xmm10
-#define CTR %xmm11
-#define INC %xmm12
-
#define GF128MUL_MASK %xmm7
#ifdef __x86_64__
@@ -51,7 +47,6 @@
#define T1 %r10
#define TKEYP T1
#define T2 %r11
-#define TCTR_LOW T2
#else
#define AREG %eax
#define KEYP %edi
@@ -610,128 +605,8 @@ SYM_FUNC_END(_aesni_dec4)
.byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
.byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
.byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
-#ifdef __x86_64__
-.Lbswap_mask:
- .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
-#endif
.popsection
-#ifdef __x86_64__
-/*
- * _aesni_inc_init: internal ABI
- * setup registers used by _aesni_inc
- * input:
- * IV
- * output:
- * CTR: == IV, in little endian
- * TCTR_LOW: == lower qword of CTR
- * INC: == 1, in little endian
- * BSWAP_MASK == endian swapping mask
- */
-SYM_FUNC_START_LOCAL(_aesni_inc_init)
- movaps .Lbswap_mask(%rip), BSWAP_MASK
- movaps IV, CTR
- pshufb BSWAP_MASK, CTR
- mov $1, TCTR_LOW
- movq TCTR_LOW, INC
- movq CTR, TCTR_LOW
- RET
-SYM_FUNC_END(_aesni_inc_init)
-
-/*
- * _aesni_inc: internal ABI
- * Increase IV by 1, IV is in big endian
- * input:
- * IV
- * CTR: == IV, in little endian
- * TCTR_LOW: == lower qword of CTR
- * INC: == 1, in little endian
- * BSWAP_MASK == endian swapping mask
- * output:
- * IV: Increase by 1
- * changed:
- * CTR: == output IV, in little endian
- * TCTR_LOW: == lower qword of CTR
- */
-SYM_FUNC_START_LOCAL(_aesni_inc)
- paddq INC, CTR
- add $1, TCTR_LOW
- jnc .Linc_low
- pslldq $8, INC
- paddq INC, CTR
- psrldq $8, INC
-.Linc_low:
- movaps CTR, IV
- pshufb BSWAP_MASK, IV
- RET
-SYM_FUNC_END(_aesni_inc)
-
-/*
- * void aesni_ctr_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
- * size_t len, u8 *iv)
- */
-SYM_FUNC_START(aesni_ctr_enc)
- ANNOTATE_NOENDBR
- FRAME_BEGIN
- cmp $16, LEN
- jb .Lctr_enc_just_ret
- mov 480(KEYP), KLEN
- movups (IVP), IV
- call _aesni_inc_init
- cmp $64, LEN
- jb .Lctr_enc_loop1
-.align 4
-.Lctr_enc_loop4:
- movaps IV, STATE1
- call _aesni_inc
- movups (INP), IN1
- movaps IV, STATE2
- call _aesni_inc
- movups 0x10(INP), IN2
- movaps IV, STATE3
- call _aesni_inc
- movups 0x20(INP), IN3
- movaps IV, STATE4
- call _aesni_inc
- movups 0x30(INP), IN4
- call _aesni_enc4
- pxor IN1, STATE1
- movups STATE1, (OUTP)
- pxor IN2, STATE2
- movups STATE2, 0x10(OUTP)
- pxor IN3, STATE3
- movups STATE3, 0x20(OUTP)
- pxor IN4, STATE4
- movups STATE4, 0x30(OUTP)
- sub $64, LEN
- add $64, INP
- add $64, OUTP
- cmp $64, LEN
- jge .Lctr_enc_loop4
- cmp $16, LEN
- jb .Lctr_enc_ret
-.align 4
-.Lctr_enc_loop1:
- movaps IV, STATE
- call _aesni_inc
- movups (INP), IN
- call _aesni_enc1
- pxor IN, STATE
- movups STATE, (OUTP)
- sub $16, LEN
- add $16, INP
- add $16, OUTP
- cmp $16, LEN
- jge .Lctr_enc_loop1
-.Lctr_enc_ret:
- movups IV, (IVP)
-.Lctr_enc_just_ret:
- FRAME_END
- RET
-SYM_FUNC_END(aesni_ctr_enc)
-
-#endif
-
.section .rodata.cst16.gf128mul_x_ble_mask, "aM", @progbits, 16
.align 16
.Lgf128mul_x_ble_mask:
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 3fd76dd82e1b..d2e5a88b409c 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -41,7 +41,6 @@
#define AESNI_ALIGN 16
#define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
-#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1))
#define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
#define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
#define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
@@ -68,9 +67,6 @@ asmlinkage void aesni_xts_enc(const struct crypto_aes_ctx *ctx, u8 *out,
asmlinkage void aesni_xts_dec(const struct crypto_aes_ctx *ctx, u8 *out,
const u8 *in, unsigned int len, u8 *iv);
-asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
- const u8 *in, unsigned int len, u8 *iv);
-
static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
{
return aes_align_addr(raw_ctx);
@@ -105,40 +101,6 @@ static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
return aes_set_key_common(aes_ctx(crypto_skcipher_ctx(tfm)), key, len);
}
-/* This is the non-AVX version. */
-static int ctr_crypt_aesni(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
- u8 keystream[AES_BLOCK_SIZE];
- struct skcipher_walk walk;
- unsigned int nbytes;
- int err;
-
- err = skcipher_walk_virt(&walk, req, false);
-
- while ((nbytes = walk.nbytes) > 0) {
- kernel_fpu_begin();
- if (nbytes & AES_BLOCK_MASK)
- aesni_ctr_enc(ctx, walk.dst.virt.addr,
- walk.src.virt.addr,
- nbytes & AES_BLOCK_MASK, walk.iv);
- nbytes &= ~AES_BLOCK_MASK;
-
- if (walk.nbytes == walk.total && nbytes > 0) {
- aesni_enc(ctx, keystream, walk.iv);
- crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes - nbytes,
- walk.src.virt.addr + walk.nbytes - nbytes,
- keystream, nbytes);
- crypto_inc(walk.iv, AES_BLOCK_SIZE);
- nbytes = 0;
- }
- kernel_fpu_end();
- err = skcipher_walk_done(&walk, nbytes);
- }
- return err;
-}
-
static int xts_setkey_aesni(struct crypto_skcipher *tfm, const u8 *key,
unsigned int keylen)
{
@@ -296,22 +258,6 @@ static int xts_decrypt_aesni(struct skcipher_request *req)
static struct skcipher_alg aesni_skciphers[] = {
{
- .base = {
- .cra_name = "ctr(aes)",
- .cra_driver_name = "ctr-aes-aesni",
- .cra_priority = 400,
- .cra_blocksize = 1,
- .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
- .cra_module = THIS_MODULE,
- },
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .chunksize = AES_BLOCK_SIZE,
- .setkey = aesni_skcipher_setkey,
- .encrypt = ctr_crypt_aesni,
- .decrypt = ctr_crypt_aesni,
- }, {
.base = {
.cra_name = "xts(aes)",
.cra_driver_name = "xts-aes-aesni",
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread* [PATCH v2 12/20] crypto: x86/aes-xts - Remove superseded XTS skcipher
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
` (10 preceding siblings ...)
2026-09-27 22:43 ` [PATCH v2 11/20] crypto: x86/aes-ctr - Remove superseded CTR skcipher Eric Biggers
@ 2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 13/20] lib/crypto: x86/aes-ctr: Migrate AVX-optimized code into library Eric Biggers
` (7 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:43 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers
Now that the crypto library's AES-XTS support is optimized with AES-NI
(without AVX), the similar code in aesni-intel is redundant. Remove it.
This only affects the non-AVX implementation ("xts-aes-aesni"), not
aes-xts-avx-x86_64.S which is handled later.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/x86/crypto/aesni-intel_asm.S | 617 -----------------------------
arch/x86/crypto/aesni-intel_glue.c | 69 +---
2 files changed, 1 insertion(+), 685 deletions(-)
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index c4d54c4a2c23..b12a0f2bf006 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -20,30 +20,11 @@
#include <linux/objtool.h>
#include <asm/frame.h>
-#define STATE1 %xmm0
-#define STATE2 %xmm4
-#define STATE3 %xmm5
-#define STATE4 %xmm6
-#define STATE STATE1
-#define IN1 %xmm1
-#define IN2 %xmm7
-#define IN3 %xmm8
-#define IN4 %xmm9
-#define IN IN1
-#define KEY %xmm2
-#define IV %xmm3
-
-#define GF128MUL_MASK %xmm7
-
#ifdef __x86_64__
#define AREG %rax
#define KEYP %rdi
#define OUTP %rsi
#define UKEYP OUTP
-#define INP %rdx
-#define LEN %rcx
-#define IVP %r8
-#define KLEN %r9d
#define T1 %r10
#define TKEYP T1
#define T2 %r11
@@ -52,10 +33,6 @@
#define KEYP %edi
#define OUTP AREG
#define UKEYP OUTP
-#define INP %edx
-#define LEN %esi
-#define IVP %ebp
-#define KLEN %ebx
#define T1 %ecx
#define TKEYP T1
#endif
@@ -241,597 +218,3 @@ SYM_FUNC_START(aesni_set_key)
FRAME_END
RET
SYM_FUNC_END(aesni_set_key)
-
-/*
- * void aesni_enc(const void *ctx, u8 *dst, const u8 *src)
- */
-SYM_FUNC_START(aesni_enc)
- FRAME_BEGIN
-#ifndef __x86_64__
- pushl KEYP
- pushl KLEN
- movl (FRAME_OFFSET+12)(%esp), KEYP # ctx
- movl (FRAME_OFFSET+16)(%esp), OUTP # dst
- movl (FRAME_OFFSET+20)(%esp), INP # src
-#endif
- movl 480(KEYP), KLEN # key length
- movups (INP), STATE # input
- call _aesni_enc1
- movups STATE, (OUTP) # output
-#ifndef __x86_64__
- popl KLEN
- popl KEYP
-#endif
- FRAME_END
- RET
-SYM_FUNC_END(aesni_enc)
-
-/*
- * _aesni_enc1: internal ABI
- * input:
- * KEYP: key struct pointer
- * KLEN: round count
- * STATE: initial state (input)
- * output:
- * STATE: finial state (output)
- * changed:
- * KEY
- * TKEYP (T1)
- */
-SYM_FUNC_START_LOCAL(_aesni_enc1)
- movaps (KEYP), KEY # key
- mov KEYP, TKEYP
- pxor KEY, STATE # round 0
- add $0x30, TKEYP
- cmp $24, KLEN
- jb .Lenc128
- lea 0x20(TKEYP), TKEYP
- je .Lenc192
- add $0x20, TKEYP
- movaps -0x60(TKEYP), KEY
- aesenc KEY, STATE
- movaps -0x50(TKEYP), KEY
- aesenc KEY, STATE
-.align 4
-.Lenc192:
- movaps -0x40(TKEYP), KEY
- aesenc KEY, STATE
- movaps -0x30(TKEYP), KEY
- aesenc KEY, STATE
-.align 4
-.Lenc128:
- movaps -0x20(TKEYP), KEY
- aesenc KEY, STATE
- movaps -0x10(TKEYP), KEY
- aesenc KEY, STATE
- movaps (TKEYP), KEY
- aesenc KEY, STATE
- movaps 0x10(TKEYP), KEY
- aesenc KEY, STATE
- movaps 0x20(TKEYP), KEY
- aesenc KEY, STATE
- movaps 0x30(TKEYP), KEY
- aesenc KEY, STATE
- movaps 0x40(TKEYP), KEY
- aesenc KEY, STATE
- movaps 0x50(TKEYP), KEY
- aesenc KEY, STATE
- movaps 0x60(TKEYP), KEY
- aesenc KEY, STATE
- movaps 0x70(TKEYP), KEY
- aesenclast KEY, STATE
- RET
-SYM_FUNC_END(_aesni_enc1)
-
-/*
- * _aesni_enc4: internal ABI
- * input:
- * KEYP: key struct pointer
- * KLEN: round count
- * STATE1: initial state (input)
- * STATE2
- * STATE3
- * STATE4
- * output:
- * STATE1: finial state (output)
- * STATE2
- * STATE3
- * STATE4
- * changed:
- * KEY
- * TKEYP (T1)
- */
-SYM_FUNC_START_LOCAL(_aesni_enc4)
- movaps (KEYP), KEY # key
- mov KEYP, TKEYP
- pxor KEY, STATE1 # round 0
- pxor KEY, STATE2
- pxor KEY, STATE3
- pxor KEY, STATE4
- add $0x30, TKEYP
- cmp $24, KLEN
- jb .L4enc128
- lea 0x20(TKEYP), TKEYP
- je .L4enc192
- add $0x20, TKEYP
- movaps -0x60(TKEYP), KEY
- aesenc KEY, STATE1
- aesenc KEY, STATE2
- aesenc KEY, STATE3
- aesenc KEY, STATE4
- movaps -0x50(TKEYP), KEY
- aesenc KEY, STATE1
- aesenc KEY, STATE2
- aesenc KEY, STATE3
- aesenc KEY, STATE4
-#.align 4
-.L4enc192:
- movaps -0x40(TKEYP), KEY
- aesenc KEY, STATE1
- aesenc KEY, STATE2
- aesenc KEY, STATE3
- aesenc KEY, STATE4
- movaps -0x30(TKEYP), KEY
- aesenc KEY, STATE1
- aesenc KEY, STATE2
- aesenc KEY, STATE3
- aesenc KEY, STATE4
-#.align 4
-.L4enc128:
- movaps -0x20(TKEYP), KEY
- aesenc KEY, STATE1
- aesenc KEY, STATE2
- aesenc KEY, STATE3
- aesenc KEY, STATE4
- movaps -0x10(TKEYP), KEY
- aesenc KEY, STATE1
- aesenc KEY, STATE2
- aesenc KEY, STATE3
- aesenc KEY, STATE4
- movaps (TKEYP), KEY
- aesenc KEY, STATE1
- aesenc KEY, STATE2
- aesenc KEY, STATE3
- aesenc KEY, STATE4
- movaps 0x10(TKEYP), KEY
- aesenc KEY, STATE1
- aesenc KEY, STATE2
- aesenc KEY, STATE3
- aesenc KEY, STATE4
- movaps 0x20(TKEYP), KEY
- aesenc KEY, STATE1
- aesenc KEY, STATE2
- aesenc KEY, STATE3
- aesenc KEY, STATE4
- movaps 0x30(TKEYP), KEY
- aesenc KEY, STATE1
- aesenc KEY, STATE2
- aesenc KEY, STATE3
- aesenc KEY, STATE4
- movaps 0x40(TKEYP), KEY
- aesenc KEY, STATE1
- aesenc KEY, STATE2
- aesenc KEY, STATE3
- aesenc KEY, STATE4
- movaps 0x50(TKEYP), KEY
- aesenc KEY, STATE1
- aesenc KEY, STATE2
- aesenc KEY, STATE3
- aesenc KEY, STATE4
- movaps 0x60(TKEYP), KEY
- aesenc KEY, STATE1
- aesenc KEY, STATE2
- aesenc KEY, STATE3
- aesenc KEY, STATE4
- movaps 0x70(TKEYP), KEY
- aesenclast KEY, STATE1 # last round
- aesenclast KEY, STATE2
- aesenclast KEY, STATE3
- aesenclast KEY, STATE4
- RET
-SYM_FUNC_END(_aesni_enc4)
-
-/*
- * _aesni_dec1: internal ABI
- * input:
- * KEYP: key struct pointer
- * KLEN: key length
- * STATE: initial state (input)
- * output:
- * STATE: finial state (output)
- * changed:
- * KEY
- * TKEYP (T1)
- */
-SYM_FUNC_START_LOCAL(_aesni_dec1)
- movaps (KEYP), KEY # key
- mov KEYP, TKEYP
- pxor KEY, STATE # round 0
- add $0x30, TKEYP
- cmp $24, KLEN
- jb .Ldec128
- lea 0x20(TKEYP), TKEYP
- je .Ldec192
- add $0x20, TKEYP
- movaps -0x60(TKEYP), KEY
- aesdec KEY, STATE
- movaps -0x50(TKEYP), KEY
- aesdec KEY, STATE
-.align 4
-.Ldec192:
- movaps -0x40(TKEYP), KEY
- aesdec KEY, STATE
- movaps -0x30(TKEYP), KEY
- aesdec KEY, STATE
-.align 4
-.Ldec128:
- movaps -0x20(TKEYP), KEY
- aesdec KEY, STATE
- movaps -0x10(TKEYP), KEY
- aesdec KEY, STATE
- movaps (TKEYP), KEY
- aesdec KEY, STATE
- movaps 0x10(TKEYP), KEY
- aesdec KEY, STATE
- movaps 0x20(TKEYP), KEY
- aesdec KEY, STATE
- movaps 0x30(TKEYP), KEY
- aesdec KEY, STATE
- movaps 0x40(TKEYP), KEY
- aesdec KEY, STATE
- movaps 0x50(TKEYP), KEY
- aesdec KEY, STATE
- movaps 0x60(TKEYP), KEY
- aesdec KEY, STATE
- movaps 0x70(TKEYP), KEY
- aesdeclast KEY, STATE
- RET
-SYM_FUNC_END(_aesni_dec1)
-
-/*
- * _aesni_dec4: internal ABI
- * input:
- * KEYP: key struct pointer
- * KLEN: key length
- * STATE1: initial state (input)
- * STATE2
- * STATE3
- * STATE4
- * output:
- * STATE1: finial state (output)
- * STATE2
- * STATE3
- * STATE4
- * changed:
- * KEY
- * TKEYP (T1)
- */
-SYM_FUNC_START_LOCAL(_aesni_dec4)
- movaps (KEYP), KEY # key
- mov KEYP, TKEYP
- pxor KEY, STATE1 # round 0
- pxor KEY, STATE2
- pxor KEY, STATE3
- pxor KEY, STATE4
- add $0x30, TKEYP
- cmp $24, KLEN
- jb .L4dec128
- lea 0x20(TKEYP), TKEYP
- je .L4dec192
- add $0x20, TKEYP
- movaps -0x60(TKEYP), KEY
- aesdec KEY, STATE1
- aesdec KEY, STATE2
- aesdec KEY, STATE3
- aesdec KEY, STATE4
- movaps -0x50(TKEYP), KEY
- aesdec KEY, STATE1
- aesdec KEY, STATE2
- aesdec KEY, STATE3
- aesdec KEY, STATE4
-.align 4
-.L4dec192:
- movaps -0x40(TKEYP), KEY
- aesdec KEY, STATE1
- aesdec KEY, STATE2
- aesdec KEY, STATE3
- aesdec KEY, STATE4
- movaps -0x30(TKEYP), KEY
- aesdec KEY, STATE1
- aesdec KEY, STATE2
- aesdec KEY, STATE3
- aesdec KEY, STATE4
-.align 4
-.L4dec128:
- movaps -0x20(TKEYP), KEY
- aesdec KEY, STATE1
- aesdec KEY, STATE2
- aesdec KEY, STATE3
- aesdec KEY, STATE4
- movaps -0x10(TKEYP), KEY
- aesdec KEY, STATE1
- aesdec KEY, STATE2
- aesdec KEY, STATE3
- aesdec KEY, STATE4
- movaps (TKEYP), KEY
- aesdec KEY, STATE1
- aesdec KEY, STATE2
- aesdec KEY, STATE3
- aesdec KEY, STATE4
- movaps 0x10(TKEYP), KEY
- aesdec KEY, STATE1
- aesdec KEY, STATE2
- aesdec KEY, STATE3
- aesdec KEY, STATE4
- movaps 0x20(TKEYP), KEY
- aesdec KEY, STATE1
- aesdec KEY, STATE2
- aesdec KEY, STATE3
- aesdec KEY, STATE4
- movaps 0x30(TKEYP), KEY
- aesdec KEY, STATE1
- aesdec KEY, STATE2
- aesdec KEY, STATE3
- aesdec KEY, STATE4
- movaps 0x40(TKEYP), KEY
- aesdec KEY, STATE1
- aesdec KEY, STATE2
- aesdec KEY, STATE3
- aesdec KEY, STATE4
- movaps 0x50(TKEYP), KEY
- aesdec KEY, STATE1
- aesdec KEY, STATE2
- aesdec KEY, STATE3
- aesdec KEY, STATE4
- movaps 0x60(TKEYP), KEY
- aesdec KEY, STATE1
- aesdec KEY, STATE2
- aesdec KEY, STATE3
- aesdec KEY, STATE4
- movaps 0x70(TKEYP), KEY
- aesdeclast KEY, STATE1 # last round
- aesdeclast KEY, STATE2
- aesdeclast KEY, STATE3
- aesdeclast KEY, STATE4
- RET
-SYM_FUNC_END(_aesni_dec4)
-
-.pushsection .rodata
-.align 16
-.Lcts_permute_table:
- .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
- .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
- .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
- .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
- .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
- .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
-.popsection
-
-.section .rodata.cst16.gf128mul_x_ble_mask, "aM", @progbits, 16
-.align 16
-.Lgf128mul_x_ble_mask:
- .octa 0x00000000000000010000000000000087
-.previous
-
-/*
- * _aesni_gf128mul_x_ble: Multiply in GF(2^128) for XTS IVs
- * input:
- * IV: current IV
- * GF128MUL_MASK == mask with 0x87 and 0x01
- * output:
- * IV: next IV
- * changed:
- * KEY: == temporary value
- */
-.macro _aesni_gf128mul_x_ble
- pshufd $0x13, IV, KEY
- paddq IV, IV
- psrad $31, KEY
- pand GF128MUL_MASK, KEY
- pxor KEY, IV
-.endm
-
-.macro _aesni_xts_crypt enc
- FRAME_BEGIN
-#ifndef __x86_64__
- pushl IVP
- pushl LEN
- pushl KEYP
- pushl KLEN
- movl (FRAME_OFFSET+20)(%esp), KEYP # ctx
- movl (FRAME_OFFSET+24)(%esp), OUTP # dst
- movl (FRAME_OFFSET+28)(%esp), INP # src
- movl (FRAME_OFFSET+32)(%esp), LEN # len
- movl (FRAME_OFFSET+36)(%esp), IVP # iv
- movdqa .Lgf128mul_x_ble_mask, GF128MUL_MASK
-#else
- movdqa .Lgf128mul_x_ble_mask(%rip), GF128MUL_MASK
-#endif
- movups (IVP), IV
-
- mov 480(KEYP), KLEN
-.if !\enc
- add $240, KEYP
-
- test $15, LEN
- jz .Lxts_loop4\@
- sub $16, LEN
-.endif
-
-.Lxts_loop4\@:
- sub $64, LEN
- jl .Lxts_1x\@
-
- movdqa IV, STATE1
- movdqu 0x00(INP), IN
- pxor IN, STATE1
- movdqu IV, 0x00(OUTP)
-
- _aesni_gf128mul_x_ble
- movdqa IV, STATE2
- movdqu 0x10(INP), IN
- pxor IN, STATE2
- movdqu IV, 0x10(OUTP)
-
- _aesni_gf128mul_x_ble
- movdqa IV, STATE3
- movdqu 0x20(INP), IN
- pxor IN, STATE3
- movdqu IV, 0x20(OUTP)
-
- _aesni_gf128mul_x_ble
- movdqa IV, STATE4
- movdqu 0x30(INP), IN
- pxor IN, STATE4
- movdqu IV, 0x30(OUTP)
-
-.if \enc
- call _aesni_enc4
-.else
- call _aesni_dec4
-.endif
-
- movdqu 0x00(OUTP), IN
- pxor IN, STATE1
- movdqu STATE1, 0x00(OUTP)
-
- movdqu 0x10(OUTP), IN
- pxor IN, STATE2
- movdqu STATE2, 0x10(OUTP)
-
- movdqu 0x20(OUTP), IN
- pxor IN, STATE3
- movdqu STATE3, 0x20(OUTP)
-
- movdqu 0x30(OUTP), IN
- pxor IN, STATE4
- movdqu STATE4, 0x30(OUTP)
-
- _aesni_gf128mul_x_ble
-
- add $64, INP
- add $64, OUTP
- test LEN, LEN
- jnz .Lxts_loop4\@
-
-.Lxts_ret_iv\@:
- movups IV, (IVP)
-
-.Lxts_ret\@:
-#ifndef __x86_64__
- popl KLEN
- popl KEYP
- popl LEN
- popl IVP
-#endif
- FRAME_END
- RET
-
-.Lxts_1x\@:
- add $64, LEN
- jz .Lxts_ret_iv\@
-.if \enc
- sub $16, LEN
- jl .Lxts_cts4\@
-.endif
-
-.Lxts_loop1\@:
- movdqu (INP), STATE
-.if \enc
- pxor IV, STATE
- call _aesni_enc1
-.else
- add $16, INP
- sub $16, LEN
- jl .Lxts_cts1\@
- pxor IV, STATE
- call _aesni_dec1
-.endif
- pxor IV, STATE
- _aesni_gf128mul_x_ble
-
- test LEN, LEN
- jz .Lxts_out\@
-
-.if \enc
- add $16, INP
- sub $16, LEN
- jl .Lxts_cts1\@
-.endif
-
- movdqu STATE, (OUTP)
- add $16, OUTP
- jmp .Lxts_loop1\@
-
-.Lxts_out\@:
- movdqu STATE, (OUTP)
- jmp .Lxts_ret_iv\@
-
-.if \enc
-.Lxts_cts4\@:
- movdqa STATE4, STATE
- sub $16, OUTP
-.Lxts_cts1\@:
-.else
-.Lxts_cts1\@:
- movdqa IV, STATE4
- _aesni_gf128mul_x_ble
-
- pxor IV, STATE
- call _aesni_dec1
- pxor IV, STATE
-.endif
-#ifndef __x86_64__
- lea .Lcts_permute_table, T1
-#else
- lea .Lcts_permute_table(%rip), T1
-#endif
- add LEN, INP /* rewind input pointer */
- add $16, LEN /* # bytes in final block */
- movups (INP), IN1
-
- mov T1, IVP
- add $32, IVP
- add LEN, T1
- sub LEN, IVP
- add OUTP, LEN
-
- movups (T1), %xmm4
- movaps STATE, IN2
- pshufb %xmm4, STATE
- movups STATE, (LEN)
-
- movups (IVP), %xmm0
- pshufb %xmm0, IN1
- pblendvb IN2, IN1
- movaps IN1, STATE
-
-.if \enc
- pxor IV, STATE
- call _aesni_enc1
- pxor IV, STATE
-.else
- pxor STATE4, STATE
- call _aesni_dec1
- pxor STATE4, STATE
-.endif
-
- movups STATE, (OUTP)
- jmp .Lxts_ret\@
-.endm
-
-/*
- * void aesni_xts_enc(const struct crypto_aes_ctx *ctx, u8 *dst,
- * const u8 *src, unsigned int len, le128 *iv)
- */
-SYM_FUNC_START(aesni_xts_enc)
- _aesni_xts_crypt 1
-SYM_FUNC_END(aesni_xts_enc)
-
-/*
- * void aesni_xts_dec(const struct crypto_aes_ctx *ctx, u8 *dst,
- * const u8 *src, unsigned int len, le128 *iv)
- */
-SYM_FUNC_START(aesni_xts_dec)
- _aesni_xts_crypt 0
-SYM_FUNC_END(aesni_xts_dec)
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index d2e5a88b409c..29f07470f442 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -59,13 +59,6 @@ static inline void *aes_align_addr(void *addr)
asmlinkage void aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
unsigned int key_len);
-asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in);
-
-asmlinkage void aesni_xts_enc(const struct crypto_aes_ctx *ctx, u8 *out,
- const u8 *in, unsigned int len, u8 *iv);
-
-asmlinkage void aesni_xts_dec(const struct crypto_aes_ctx *ctx, u8 *out,
- const u8 *in, unsigned int len, u8 *iv);
static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
{
@@ -226,56 +219,6 @@ xts_crypt(struct skcipher_request *req, xts_encrypt_iv_func encrypt_iv,
return xts_crypt_slowpath(req, crypt_func);
}
-static void aesni_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key,
- u8 iv[AES_BLOCK_SIZE])
-{
- aesni_enc(tweak_key, iv, iv);
-}
-
-static void aesni_xts_encrypt(const struct crypto_aes_ctx *key,
- const u8 *src, u8 *dst, int len,
- u8 tweak[AES_BLOCK_SIZE])
-{
- aesni_xts_enc(key, dst, src, len, tweak);
-}
-
-static void aesni_xts_decrypt(const struct crypto_aes_ctx *key,
- const u8 *src, u8 *dst, int len,
- u8 tweak[AES_BLOCK_SIZE])
-{
- aesni_xts_dec(key, dst, src, len, tweak);
-}
-
-static int xts_encrypt_aesni(struct skcipher_request *req)
-{
- return xts_crypt(req, aesni_xts_encrypt_iv, aesni_xts_encrypt);
-}
-
-static int xts_decrypt_aesni(struct skcipher_request *req)
-{
- return xts_crypt(req, aesni_xts_encrypt_iv, aesni_xts_decrypt);
-}
-
-static struct skcipher_alg aesni_skciphers[] = {
- {
- .base = {
- .cra_name = "xts(aes)",
- .cra_driver_name = "xts-aes-aesni",
- .cra_priority = 401,
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = XTS_AES_CTX_SIZE,
- .cra_module = THIS_MODULE,
- },
- .min_keysize = 2 * AES_MIN_KEY_SIZE,
- .max_keysize = 2 * AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .walksize = 2 * AES_BLOCK_SIZE,
- .setkey = xts_setkey_aesni,
- .encrypt = xts_encrypt_aesni,
- .decrypt = xts_decrypt_aesni,
- }
-};
-
asmlinkage void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key,
u8 iv[AES_BLOCK_SIZE]);
@@ -1299,15 +1242,10 @@ static int __init aesni_init(void)
if (!x86_match_cpu(aesni_cpu_id))
return -ENODEV;
- err = crypto_register_skciphers(aesni_skciphers,
- ARRAY_SIZE(aesni_skciphers));
- if (err)
- return err;
-
err = crypto_register_aeads(aes_gcm_algs_aesni,
ARRAY_SIZE(aes_gcm_algs_aesni));
if (err)
- goto unregister_skciphers;
+ return err;
err = register_avx_algs();
if (err)
@@ -1319,9 +1257,6 @@ static int __init aesni_init(void)
unregister_avx_algs();
crypto_unregister_aeads(aes_gcm_algs_aesni,
ARRAY_SIZE(aes_gcm_algs_aesni));
-unregister_skciphers:
- crypto_unregister_skciphers(aesni_skciphers,
- ARRAY_SIZE(aesni_skciphers));
return err;
}
@@ -1329,8 +1264,6 @@ static void __exit aesni_exit(void)
{
crypto_unregister_aeads(aes_gcm_algs_aesni,
ARRAY_SIZE(aes_gcm_algs_aesni));
- crypto_unregister_skciphers(aesni_skciphers,
- ARRAY_SIZE(aesni_skciphers));
unregister_avx_algs();
}
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread* [PATCH v2 13/20] lib/crypto: x86/aes-ctr: Migrate AVX-optimized code into library
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
` (11 preceding siblings ...)
2026-09-27 22:43 ` [PATCH v2 12/20] crypto: x86/aes-xts - Remove superseded XTS skcipher Eric Biggers
@ 2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 14/20] lib/crypto: x86/aes-xts: " Eric Biggers
` (6 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:43 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers
Migrate aes-ctr-avx-x86_64.S into lib/crypto/, wiring it up to the CTR
and XCTR library functions instead of the crypto_skcipher API. It still
remains available through crypto_skcipher via crypto/aes.c.
Some slight adjustments to the assembly code were needed:
- Take 'struct aes_enckey' instead of 'struct crypto_aes_ctx'.
- Upgrade the length argument from 32-bit to 64-bit so that it's
compatible with the library's use of size_t (at least assuming no
lengths over S64_MAX, which seems quite safe to assume...)
- Remove the CFI stubs, as the functions are now called directly.
- Adjust the argument order to match the caller. Not strictly required,
but it's easiest to handle this now when changing the function
prototypes anyway and adding the new glue code.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/x86/crypto/Kconfig | 4 +-
arch/x86/crypto/Makefile | 1 -
arch/x86/crypto/aesni-intel_glue.c | 153 ------------------
crypto/aes.c | 7 +-
lib/crypto/Makefile | 4 +
.../crypto/x86}/aes-ctr-avx-x86_64.S | 103 ++++++------
lib/crypto/x86/aes.h | 80 ++++++++-
7 files changed, 140 insertions(+), 212 deletions(-)
rename {arch/x86/crypto => lib/crypto/x86}/aes-ctr-avx-x86_64.S (88%)
diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig
index 79e610b5e5bc..778ab164c571 100644
--- a/arch/x86/crypto/Kconfig
+++ b/arch/x86/crypto/Kconfig
@@ -3,7 +3,7 @@
menu "Accelerated Cryptographic Algorithms for CPU (x86)"
config CRYPTO_AES_NI_INTEL
- tristate "Ciphers: AES, modes: CTR, XCTR, XTS, GCM (AES-NI/VAES)"
+ tristate "Ciphers: AES, modes: XTS, GCM (AES-NI/VAES)"
depends on 64BIT
select CRYPTO_AEAD
select CRYPTO_LIB_AES
@@ -11,7 +11,7 @@ config CRYPTO_AES_NI_INTEL
select CRYPTO_SKCIPHER
help
AEAD cipher: AES with GCM
- Length-preserving ciphers: AES with CTR, XCTR, XTS
+ Length-preserving ciphers: AES with XTS
Architecture: x86_64 using:
- AES-NI (AES new instructions)
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index 1c3feb6d72b7..0c016ba87373 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -42,7 +42,6 @@ aegis128-aesni-y := aegis128-aesni-asm.o aegis128-aesni-glue.o
obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o
aesni-intel-y := aesni-intel_asm.o \
aesni-intel_glue.o \
- aes-ctr-avx-x86_64.o \
aes-gcm-aesni-x86_64.o \
aes-gcm-vaes-avx2.o \
aes-gcm-vaes-avx512.o \
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 29f07470f442..5aae178bbae3 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -42,7 +42,6 @@
#define AESNI_ALIGN 16
#define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
#define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
-#define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
#define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
struct aesni_xts_ctx {
@@ -60,11 +59,6 @@ static inline void *aes_align_addr(void *addr)
asmlinkage void aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
unsigned int key_len);
-static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
-{
- return aes_align_addr(raw_ctx);
-}
-
static inline struct aesni_xts_ctx *aes_xts_ctx(struct crypto_skcipher *tfm)
{
return aes_align_addr(crypto_skcipher_ctx(tfm));
@@ -88,12 +82,6 @@ static int aes_set_key_common(struct crypto_aes_ctx *ctx,
return 0;
}
-static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
- unsigned int len)
-{
- return aes_set_key_common(aes_ctx(crypto_skcipher_ctx(tfm)), key, len);
-}
-
static int xts_setkey_aesni(struct crypto_skcipher *tfm, const u8 *key,
unsigned int keylen)
{
@@ -222,100 +210,6 @@ xts_crypt(struct skcipher_request *req, xts_encrypt_iv_func encrypt_iv,
asmlinkage void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key,
u8 iv[AES_BLOCK_SIZE]);
-/* __always_inline to avoid indirect call */
-static __always_inline int
-ctr_crypt(struct skcipher_request *req,
- void (*ctr64_func)(const struct crypto_aes_ctx *key,
- const u8 *src, u8 *dst, int len,
- const u64 le_ctr[2]))
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- const struct crypto_aes_ctx *key = aes_ctx(crypto_skcipher_ctx(tfm));
- unsigned int nbytes, p1_nbytes, nblocks;
- struct skcipher_walk walk;
- u64 le_ctr[2];
- u64 ctr64;
- int err;
-
- ctr64 = le_ctr[0] = get_unaligned_be64(&req->iv[8]);
- le_ctr[1] = get_unaligned_be64(&req->iv[0]);
-
- err = skcipher_walk_virt(&walk, req, false);
-
- while ((nbytes = walk.nbytes) != 0) {
- if (nbytes < walk.total) {
- /* Not the end yet, so keep the length block-aligned. */
- nbytes = round_down(nbytes, AES_BLOCK_SIZE);
- nblocks = nbytes / AES_BLOCK_SIZE;
- } else {
- /* It's the end, so include any final partial block. */
- nblocks = DIV_ROUND_UP(nbytes, AES_BLOCK_SIZE);
- }
- ctr64 += nblocks;
-
- kernel_fpu_begin();
- if (likely(ctr64 >= nblocks)) {
- /* The low 64 bits of the counter won't overflow. */
- (*ctr64_func)(key, walk.src.virt.addr,
- walk.dst.virt.addr, nbytes, le_ctr);
- } else {
- /*
- * The low 64 bits of the counter will overflow. The
- * assembly doesn't handle this case, so split the
- * operation into two at the point where the overflow
- * will occur. After the first part, add the carry bit.
- */
- p1_nbytes = min(nbytes, (nblocks - ctr64) * AES_BLOCK_SIZE);
- (*ctr64_func)(key, walk.src.virt.addr,
- walk.dst.virt.addr, p1_nbytes, le_ctr);
- le_ctr[0] = 0;
- le_ctr[1]++;
- (*ctr64_func)(key, walk.src.virt.addr + p1_nbytes,
- walk.dst.virt.addr + p1_nbytes,
- nbytes - p1_nbytes, le_ctr);
- }
- kernel_fpu_end();
- le_ctr[0] = ctr64;
-
- err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
- }
-
- put_unaligned_be64(ctr64, &req->iv[8]);
- put_unaligned_be64(le_ctr[1], &req->iv[0]);
-
- return err;
-}
-
-/* __always_inline to avoid indirect call */
-static __always_inline int
-xctr_crypt(struct skcipher_request *req,
- void (*xctr_func)(const struct crypto_aes_ctx *key,
- const u8 *src, u8 *dst, int len,
- const u8 iv[AES_BLOCK_SIZE], u64 ctr))
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- const struct crypto_aes_ctx *key = aes_ctx(crypto_skcipher_ctx(tfm));
- struct skcipher_walk walk;
- unsigned int nbytes;
- u64 ctr = 1;
- int err;
-
- err = skcipher_walk_virt(&walk, req, false);
- while ((nbytes = walk.nbytes) != 0) {
- if (nbytes < walk.total)
- nbytes = round_down(nbytes, AES_BLOCK_SIZE);
-
- kernel_fpu_begin();
- (*xctr_func)(key, walk.src.virt.addr, walk.dst.virt.addr,
- nbytes, req->iv, ctr);
- kernel_fpu_end();
-
- ctr += DIV_ROUND_UP(nbytes, AES_BLOCK_SIZE);
- err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
- }
- return err;
-}
-
#define DEFINE_AVX_SKCIPHER_ALGS(suffix, driver_name_suffix, priority) \
\
asmlinkage void \
@@ -335,25 +229,6 @@ static int xts_decrypt_##suffix(struct skcipher_request *req) \
return xts_crypt(req, aes_xts_encrypt_iv, aes_xts_decrypt_##suffix); \
} \
\
-asmlinkage void \
-aes_ctr64_crypt_##suffix(const struct crypto_aes_ctx *key, \
- const u8 *src, u8 *dst, int len, const u64 le_ctr[2]);\
- \
-static int ctr_crypt_##suffix(struct skcipher_request *req) \
-{ \
- return ctr_crypt(req, aes_ctr64_crypt_##suffix); \
-} \
- \
-asmlinkage void \
-aes_xctr_crypt_##suffix(const struct crypto_aes_ctx *key, \
- const u8 *src, u8 *dst, int len, \
- const u8 iv[AES_BLOCK_SIZE], u64 ctr); \
- \
-static int xctr_crypt_##suffix(struct skcipher_request *req) \
-{ \
- return xctr_crypt(req, aes_xctr_crypt_##suffix); \
-} \
- \
static struct skcipher_alg skcipher_algs_##suffix[] = {{ \
.base.cra_name = "xts(aes)", \
.base.cra_driver_name = "xts-aes-" driver_name_suffix, \
@@ -368,34 +243,6 @@ static struct skcipher_alg skcipher_algs_##suffix[] = {{ \
.setkey = xts_setkey_aesni, \
.encrypt = xts_encrypt_##suffix, \
.decrypt = xts_decrypt_##suffix, \
-}, { \
- .base.cra_name = "ctr(aes)", \
- .base.cra_driver_name = "ctr-aes-" driver_name_suffix, \
- .base.cra_priority = priority, \
- .base.cra_blocksize = 1, \
- .base.cra_ctxsize = CRYPTO_AES_CTX_SIZE, \
- .base.cra_module = THIS_MODULE, \
- .min_keysize = AES_MIN_KEY_SIZE, \
- .max_keysize = AES_MAX_KEY_SIZE, \
- .ivsize = AES_BLOCK_SIZE, \
- .chunksize = AES_BLOCK_SIZE, \
- .setkey = aesni_skcipher_setkey, \
- .encrypt = ctr_crypt_##suffix, \
- .decrypt = ctr_crypt_##suffix, \
-}, { \
- .base.cra_name = "xctr(aes)", \
- .base.cra_driver_name = "xctr-aes-" driver_name_suffix, \
- .base.cra_priority = priority, \
- .base.cra_blocksize = 1, \
- .base.cra_ctxsize = CRYPTO_AES_CTX_SIZE, \
- .base.cra_module = THIS_MODULE, \
- .min_keysize = AES_MIN_KEY_SIZE, \
- .max_keysize = AES_MAX_KEY_SIZE, \
- .ivsize = AES_BLOCK_SIZE, \
- .chunksize = AES_BLOCK_SIZE, \
- .setkey = aesni_skcipher_setkey, \
- .encrypt = xctr_crypt_##suffix, \
- .decrypt = xctr_crypt_##suffix, \
}}
DEFINE_AVX_SKCIPHER_ALGS(aesni_avx, "aesni-avx", 500);
diff --git a/crypto/aes.c b/crypto/aes.c
index bf8acc553188..568a900bfec8 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -667,7 +667,7 @@ static struct skcipher_alg skcipher_algs[] = {
{
.base.cra_name = "ctr(aes)",
.base.cra_driver_name = "ctr-aes-lib",
- .base.cra_priority = 110,
+ .base.cra_priority = IS_ENABLED(CONFIG_X86) ? 300 : 110,
.base.cra_blocksize = 1,
.base.cra_ctxsize = sizeof(struct aes_enckey),
.base.cra_module = THIS_MODULE,
@@ -684,7 +684,7 @@ static struct skcipher_alg skcipher_algs[] = {
{
.base.cra_name = "xctr(aes)",
.base.cra_driver_name = "xctr-aes-lib",
- .base.cra_priority = 110,
+ .base.cra_priority = IS_ENABLED(CONFIG_X86) ? 300 : 110,
.base.cra_blocksize = 1,
.base.cra_ctxsize = sizeof(struct aes_enckey),
.base.cra_module = THIS_MODULE,
@@ -1044,8 +1044,7 @@ static struct aead_alg aead_algs[] = {
IS_ENABLED(CONFIG_PPC) || \
IS_ENABLED(CONFIG_RISCV) || \
IS_ENABLED(CONFIG_S390) || \
- IS_ENABLED(CONFIG_SPARC) || \
- IS_ENABLED(CONFIG_X86))
+ IS_ENABLED(CONFIG_SPARC))
{
.base.cra_name = "ccm(aes)",
.base.cra_driver_name = "ccm-aes-lib",
diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
index ca068df1f71f..5d5484fc78ea 100644
--- a/lib/crypto/Makefile
+++ b/lib/crypto/Makefile
@@ -52,7 +52,11 @@ endif # CONFIG_PPC
libaes-$(CONFIG_RISCV) += riscv/aes-riscv64-zvkned.o
libaes-$(CONFIG_SPARC) += sparc/aes_asm.o
+
libaes-$(CONFIG_X86) += x86/aes-aesni.o
+ifneq ($(CONFIG_CRYPTO_LIB_AES_CTR),)
+libaes-$(CONFIG_X86_64) += x86/aes-ctr-avx-x86_64.o
+endif
endif # CONFIG_CRYPTO_LIB_AES_ARCH
# clean-files must be defined unconditionally
diff --git a/arch/x86/crypto/aes-ctr-avx-x86_64.S b/lib/crypto/x86/aes-ctr-avx-x86_64.S
similarity index 88%
rename from arch/x86/crypto/aes-ctr-avx-x86_64.S
rename to lib/crypto/x86/aes-ctr-avx-x86_64.S
index 2745918f68ee..c232337899b6 100644
--- a/arch/x86/crypto/aes-ctr-avx-x86_64.S
+++ b/lib/crypto/x86/aes-ctr-avx-x86_64.S
@@ -53,7 +53,10 @@
// See the function definitions at the bottom of the file for more information.
#include <linux/linkage.h>
-#include <linux/cfi_types.h>
+
+// Offsets in struct aes_enckey
+#define OFFSETOF_KEYLEN 0
+#define OFFSETOF_RNDKEYS 16
.section .rodata
.p2align 4
@@ -278,24 +281,25 @@
.endr
// Function arguments
- .set KEY, %rdi // Initially points to the start of the
- // crypto_aes_ctx, then is advanced to
- // point to the index 1 round key
- .set KEY32, %edi // Available as temp register after all
- // keystream blocks have been generated
+ .set DST, %rdi // Pointer to next destination data
.set SRC, %rsi // Pointer to next source data
- .set DST, %rdx // Pointer to next destination data
- .set LEN, %ecx // Remaining length in bytes.
- // Note: _load_partial_block relies on
- // this being in %ecx.
- .set LEN64, %rcx // Zero-extend LEN before using!
- .set LEN8, %cl
+ .set LEN, %rdx // Remaining length in bytes
+ .set LEN32, %edx // Used for improved code density
.if \is_xctr
+ .set XCTR_CTR, %rcx // u64 ctr;
.set XCTR_IV_PTR, %r8 // const u8 iv[AES_BLOCK_SIZE];
- .set XCTR_CTR, %r9 // u64 ctr;
+ .set KEY, %r9 // Initially points to the start of the
+ // aes_enckey, then is advanced to
+ // point to the index 1 round key
+ .set KEY32, %r9d // Available as temp register after all
+ // keystream blocks have been generated
.else
- .set LE_CTR_PTR, %r8 // const u64 le_ctr[2];
+ .set LE_CTR_PTR, %rcx // const u64 le_ctr[2];
+ .set KEY, %r8
+ .set KEY32, %r8d
.endif
+ // Note: the use of _load_partial_block and _store_partial_block at the
+ // end of the function assumes %rcx doesn't hold DST, SRC, LEN, or KEY.
// Additional local variables
.set RNDKEYLAST_PTR, %r10
@@ -355,17 +359,17 @@
vpsllq $1, LE_CTR_INC1, LE_CTR_INC2
// Load the AES key length: 16 (AES-128), 24 (AES-192), or 32 (AES-256).
- movl 480(KEY), %eax
+ movl OFFSETOF_KEYLEN(KEY), %eax
// Compute the pointer to the last round key.
- lea 6*16(KEY, %rax, 4), RNDKEYLAST_PTR
+ lea OFFSETOF_RNDKEYS+6*16(KEY, %rax, 4), RNDKEYLAST_PTR
// Load the zero-th and last round keys.
- _vbroadcast128 (KEY), RNDKEY0
+ _vbroadcast128 OFFSETOF_RNDKEYS(KEY), RNDKEY0
_vbroadcast128 (RNDKEYLAST_PTR), RNDKEYLAST
// Make KEY point to the first round key.
- add $16, KEY
+ add $OFFSETOF_RNDKEYS+16, KEY
// This is the main loop, which encrypts 8 vectors of data at a time.
add $-8*VL, LEN
@@ -382,7 +386,7 @@
add $-8*VL, LEN
jge .Lloop_8x\@
.Lloop_8x_done\@:
- sub $-8*VL, LEN
+ sub $-8*VL, LEN32
jz .Ldone\@
// 1 <= LEN < 8*VL. Generate 2, 4, or 8 more vectors of keystream
@@ -390,7 +394,7 @@
_prepare_2_ctr_vecs \is_xctr, 0, 1
_prepare_2_ctr_vecs \is_xctr, 2, 3
- cmp $4*VL, LEN
+ cmp $4*VL, LEN32
jle .Lenc_tail_atmost4vecs\@
// 4*VL < LEN < 8*VL. Generate 8 vectors of keystream blocks. Use the
@@ -405,23 +409,23 @@
vaesenclast RNDKEYLAST, AESDATA7, AESDATA3
sub $-4*VL, SRC
sub $-4*VL, DST
- add $-4*VL, LEN
- cmp $1*VL-1, LEN
+ add $-4*VL, LEN32
+ cmp $1*VL-1, LEN32
jle .Lxor_tail_partial_vec_0\@
_xor_data 0
- cmp $2*VL-1, LEN
+ cmp $2*VL-1, LEN32
jle .Lxor_tail_partial_vec_1\@
_xor_data 1
- cmp $3*VL-1, LEN
+ cmp $3*VL-1, LEN32
jle .Lxor_tail_partial_vec_2\@
_xor_data 2
- cmp $4*VL-1, LEN
+ cmp $4*VL-1, LEN32
jle .Lxor_tail_partial_vec_3\@
_xor_data 3
jmp .Ldone\@
.Lenc_tail_atmost4vecs\@:
- cmp $2*VL, LEN
+ cmp $2*VL, LEN32
jle .Lenc_tail_atmost2vecs\@
// 2*VL < LEN <= 4*VL. Generate 4 vectors of keystream blocks. Use the
@@ -432,7 +436,7 @@
vaesenclast RNDKEYLAST, AESDATA3, AESDATA1
sub $-2*VL, SRC
sub $-2*VL, DST
- add $-2*VL, LEN
+ add $-2*VL, LEN32
jmp .Lxor_tail_upto2vecs\@
.Lenc_tail_atmost2vecs\@:
@@ -443,16 +447,16 @@
vaesenclast RNDKEYLAST, AESDATA1, AESDATA1
.Lxor_tail_upto2vecs\@:
- cmp $1*VL-1, LEN
+ cmp $1*VL-1, LEN32
jle .Lxor_tail_partial_vec_0\@
_xor_data 0
- cmp $2*VL-1, LEN
+ cmp $2*VL-1, LEN32
jle .Lxor_tail_partial_vec_1\@
_xor_data 1
jmp .Ldone\@
.Lxor_tail_partial_vec_1\@:
- add $-1*VL, LEN
+ add $-1*VL, LEN32
jz .Ldone\@
sub $-1*VL, SRC
sub $-1*VL, DST
@@ -460,7 +464,7 @@
jmp .Lxor_tail_partial_vec_0\@
.Lxor_tail_partial_vec_2\@:
- add $-2*VL, LEN
+ add $-2*VL, LEN32
jz .Ldone\@
sub $-2*VL, SRC
sub $-2*VL, DST
@@ -468,7 +472,7 @@
jmp .Lxor_tail_partial_vec_0\@
.Lxor_tail_partial_vec_3\@:
- add $-3*VL, LEN
+ add $-3*VL, LEN32
jz .Ldone\@
sub $-3*VL, SRC
sub $-3*VL, DST
@@ -479,28 +483,29 @@
// loads/stores are available; otherwise it's a bit harder...
.if USE_AVX512
mov $-1, %rax
- bzhi LEN64, %rax, %rax
+ bzhi LEN, %rax, %rax
kmovq %rax, %k1
vmovdqu8 (SRC), AESDATA1{%k1}{z}
vpxord AESDATA1, AESDATA0, AESDATA0
vmovdqu8 AESDATA0, (DST){%k1}
.else
.if VL == 32
- cmp $16, LEN
+ cmp $16, LEN32
jl 1f
vpxor (SRC), AESDATA0_XMM, AESDATA1_XMM
vmovdqu AESDATA1_XMM, (DST)
add $16, SRC
add $16, DST
- sub $16, LEN
+ sub $16, LEN32
jz .Ldone\@
vextracti128 $1, AESDATA0, AESDATA0_XMM
1:
.endif
- mov LEN, %r10d
+ // Note: this assumes %rcx doesn't hold DST, SRC, LEN, or KEY.
+ mov LEN32, %ecx
_load_partial_block SRC, AESDATA1_XMM, KEY, KEY32
vpxor AESDATA1_XMM, AESDATA0_XMM, AESDATA0_XMM
- mov %r10d, %ecx
+ mov LEN32, %ecx
_store_partial_block AESDATA0_XMM, DST, KEY, KEY32
.endif
@@ -515,13 +520,13 @@
// They have the following prototypes:
//
//
-// void aes_ctr64_crypt_##suffix(const struct crypto_aes_ctx *key,
-// const u8 *src, u8 *dst, int len,
-// const u64 le_ctr[2]);
+// void aes_ctr64_crypt_##suffix(u8 *dst, const u8 *src, s64 len,
+// const u64 le_ctr[2],
+// const struct aes_enckey *key);
//
-// void aes_xctr_crypt_##suffix(const struct crypto_aes_ctx *key,
-// const u8 *src, u8 *dst, int len,
-// const u8 iv[AES_BLOCK_SIZE], u64 ctr);
+// void aes_xctr_crypt_##suffix(u8 *dst, const u8 *src, s64 len, u64 ctr,
+// const u8 iv[AES_BLOCK_SIZE],
+// const struct aes_enckey *key);
//
// Both functions generate |len| bytes of keystream, XOR it with the data from
// |src|, and write the result to |dst|. On non-final calls, |len| must be a
@@ -545,27 +550,27 @@
.set VL, 16
.set USE_AVX512, 0
-SYM_TYPED_FUNC_START(aes_ctr64_crypt_aesni_avx)
+SYM_FUNC_START(aes_ctr64_crypt_aesni_avx)
_aes_ctr_crypt 0
SYM_FUNC_END(aes_ctr64_crypt_aesni_avx)
-SYM_TYPED_FUNC_START(aes_xctr_crypt_aesni_avx)
+SYM_FUNC_START(aes_xctr_crypt_aesni_avx)
_aes_ctr_crypt 1
SYM_FUNC_END(aes_xctr_crypt_aesni_avx)
.set VL, 32
.set USE_AVX512, 0
-SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx2)
+SYM_FUNC_START(aes_ctr64_crypt_vaes_avx2)
_aes_ctr_crypt 0
SYM_FUNC_END(aes_ctr64_crypt_vaes_avx2)
-SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx2)
+SYM_FUNC_START(aes_xctr_crypt_vaes_avx2)
_aes_ctr_crypt 1
SYM_FUNC_END(aes_xctr_crypt_vaes_avx2)
.set VL, 64
.set USE_AVX512, 1
-SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx512)
+SYM_FUNC_START(aes_ctr64_crypt_vaes_avx512)
_aes_ctr_crypt 0
SYM_FUNC_END(aes_ctr64_crypt_vaes_avx512)
-SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx512)
+SYM_FUNC_START(aes_xctr_crypt_vaes_avx512)
_aes_ctr_crypt 1
SYM_FUNC_END(aes_xctr_crypt_vaes_avx512)
diff --git a/lib/crypto/x86/aes.h b/lib/crypto/x86/aes.h
index 54b599bd2581..8ed247ddb8f2 100644
--- a/lib/crypto/x86/aes.h
+++ b/lib/crypto/x86/aes.h
@@ -8,8 +8,12 @@
#include <asm/fpu/api.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aesni);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aesni_avx);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vaes_avx2);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vaes_avx512);
/* The assembly code assumes the following offsets. */
+static_assert(offsetof(struct aes_enckey, len) == 0);
static_assert(offsetof(struct aes_enckey, nrounds) == 4);
static_assert(offsetof(struct aes_enckey, k.rndkeys) == 16);
static_assert(offsetof(struct aes_key, nrounds) == 4);
@@ -208,11 +212,36 @@ static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len,
#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR) && IS_ENABLED(CONFIG_X86_64)
void aes_ctr64_crypt_aesni(u8 *dst, const u8 *src, s64 len, const u64 le_ctr[2],
const struct aes_enckey *key);
+void aes_ctr64_crypt_aesni_avx(u8 *dst, const u8 *src, s64 len,
+ const u64 le_ctr[2],
+ const struct aes_enckey *key);
+void aes_ctr64_crypt_vaes_avx2(u8 *dst, const u8 *src, s64 len,
+ const u64 le_ctr[2],
+ const struct aes_enckey *key);
+void aes_ctr64_crypt_vaes_avx512(u8 *dst, const u8 *src, s64 len,
+ const u64 le_ctr[2],
+ const struct aes_enckey *key);
+void aes_xctr_crypt_aesni_avx(u8 *dst, const u8 *src, s64 len, u64 ctr,
+ const u8 iv[AES_BLOCK_SIZE],
+ const struct aes_enckey *key);
+void aes_xctr_crypt_vaes_avx2(u8 *dst, const u8 *src, s64 len, u64 ctr,
+ const u8 iv[AES_BLOCK_SIZE],
+ const struct aes_enckey *key);
+void aes_xctr_crypt_vaes_avx512(u8 *dst, const u8 *src, s64 len, u64 ctr,
+ const u8 iv[AES_BLOCK_SIZE],
+ const struct aes_enckey *key);
static void aes_ctr64_x86(u8 *dst, const u8 *src, size_t len,
const u64 le_ctr[2], const struct aes_enckey *key)
{
- aes_ctr64_crypt_aesni(dst, src, len, le_ctr, key);
+ if (static_branch_likely(&have_vaes_avx512))
+ aes_ctr64_crypt_vaes_avx512(dst, src, len, le_ctr, key);
+ else if (static_branch_likely(&have_vaes_avx2))
+ aes_ctr64_crypt_vaes_avx2(dst, src, len, le_ctr, key);
+ else if (static_branch_likely(&have_aesni_avx))
+ aes_ctr64_crypt_aesni_avx(dst, src, len, le_ctr, key);
+ else
+ aes_ctr64_crypt_aesni(dst, src, len, le_ctr, key);
}
#define aes_ctr_arch aes_ctr_arch
@@ -257,6 +286,25 @@ static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len,
put_unaligned_be64(le_ctr[1], &ctr[0]);
return true;
}
+
+#define aes_xctr_arch aes_xctr_arch
+static bool aes_xctr_arch(u8 *dst, const u8 *src, size_t len, u64 ctr,
+ const u8 iv[AES_BLOCK_SIZE],
+ const struct aes_enckey *key)
+{
+ if (!static_branch_likely(&have_aesni_avx) ||
+ unlikely(!irq_fpu_usable()))
+ return false;
+ kernel_fpu_begin();
+ if (static_branch_likely(&have_vaes_avx512))
+ aes_xctr_crypt_vaes_avx512(dst, src, len, ctr, iv, key);
+ else if (static_branch_likely(&have_vaes_avx2))
+ aes_xctr_crypt_vaes_avx2(dst, src, len, ctr, iv, key);
+ else
+ aes_xctr_crypt_aesni_avx(dst, src, len, ctr, iv, key);
+ kernel_fpu_end();
+ return true;
+}
#endif /* CONFIG_CRYPTO_LIB_AES_CTR && CONFIG_X86_64 */
#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS)
@@ -306,6 +354,32 @@ static bool aes_xts_decrypt_arch(u8 *dst, const u8 *src, size_t len,
#define aes_mod_init_arch aes_mod_init_arch
static void aes_mod_init_arch(void)
{
- if (boot_cpu_has(X86_FEATURE_AES))
- static_branch_enable(&have_aesni);
+ /* Everything below requires AES-NI. */
+ if (!boot_cpu_has(X86_FEATURE_AES))
+ return;
+ static_branch_enable(&have_aesni);
+
+ /* Everything below requires AVX and is also 64-bit only. */
+ if (!boot_cpu_has(X86_FEATURE_AVX) || !IS_ENABLED(CONFIG_X86_64))
+ return;
+ static_branch_enable(&have_aesni_avx);
+
+ /*
+ * Everything below requires VAES, and also sometimes AVX2, VPCLMULQDQ,
+ * and PCLMULQDQ. Use a single static key for all of them, since in
+ * practice every CPU with VAES also has the others.
+ */
+ if (!boot_cpu_has(X86_FEATURE_VAES) ||
+ !boot_cpu_has(X86_FEATURE_AVX2) ||
+ !boot_cpu_has(X86_FEATURE_VPCLMULQDQ) ||
+ !boot_cpu_has(X86_FEATURE_PCLMULQDQ))
+ return;
+ static_branch_enable(&have_vaes_avx2);
+
+ if (!boot_cpu_has(X86_FEATURE_AVX512BW) ||
+ !boot_cpu_has(X86_FEATURE_AVX512VL) ||
+ !boot_cpu_has(X86_FEATURE_BMI2) ||
+ boot_cpu_has(X86_FEATURE_PREFER_YMM))
+ return;
+ static_branch_enable(&have_vaes_avx512);
}
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread* [PATCH v2 14/20] lib/crypto: x86/aes-xts: Migrate AVX-optimized code into library
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
` (12 preceding siblings ...)
2026-09-27 22:43 ` [PATCH v2 13/20] lib/crypto: x86/aes-ctr: Migrate AVX-optimized code into library Eric Biggers
@ 2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 15/20] lib/crypto: riscv/aes: Copy aes-macros.S to library Eric Biggers
` (5 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:43 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers
Migrate aes-xts-avx-x86_64.S into lib/crypto/, wiring it up to the XTS
library functions instead of the crypto_skcipher API. It still remains
available through crypto_skcipher via crypto/aes.c.
Some adjustments to the assembly code were needed:
- Take 'struct aes_key' instead of 'struct crypto_aes_ctx'.
- Remove the ciphertext stealing support from the assembly code, as the
library implements it in a generic way instead. (This does slightly
reduce performance when the length isn't a multiple of 16 bytes;
however, that case seems to never be reached in practice in the
kernel. So it makes sense to not extensively optimize for it yet.)
- Change 'int len' to 'long nblocks' for compatibility with the
library's use of size_t lengths.
- Remove the CFI stubs, as the functions are now called directly.
- Adjust the argument order to match the caller. Not strictly required,
but it's easiest to handle this now when changing the function
prototypes anyway and adding the new glue code.
This makes the remaining code in aesni-intel_asm.S (which just handled
key expansion) unused, so remove that too.
Don't remove the selection of CRYPTO_SKCIPHER from CRYPTO_AES_NI_INTEL
yet, since it's still needed for the "skcipher_walk" functions.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/x86/crypto/Kconfig | 6 +-
arch/x86/crypto/Makefile | 6 +-
arch/x86/crypto/aesni-intel_asm.S | 220 ----------------
arch/x86/crypto/aesni-intel_glue.c | 239 +-----------------
crypto/aes.c | 2 +-
lib/crypto/Makefile | 3 +
.../crypto/x86}/aes-xts-avx-x86_64.S | 179 ++++---------
lib/crypto/x86/aes.h | 67 ++++-
8 files changed, 121 insertions(+), 601 deletions(-)
delete mode 100644 arch/x86/crypto/aesni-intel_asm.S
rename {arch/x86/crypto => lib/crypto/x86}/aes-xts-avx-x86_64.S (81%)
diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig
index 778ab164c571..c1842c2d7055 100644
--- a/arch/x86/crypto/Kconfig
+++ b/arch/x86/crypto/Kconfig
@@ -3,7 +3,7 @@
menu "Accelerated Cryptographic Algorithms for CPU (x86)"
config CRYPTO_AES_NI_INTEL
- tristate "Ciphers: AES, modes: XTS, GCM (AES-NI/VAES)"
+ tristate "Ciphers: AES, modes: GCM (AES-NI/VAES)"
depends on 64BIT
select CRYPTO_AEAD
select CRYPTO_LIB_AES
@@ -11,12 +11,14 @@ config CRYPTO_AES_NI_INTEL
select CRYPTO_SKCIPHER
help
AEAD cipher: AES with GCM
- Length-preserving ciphers: AES with XTS
Architecture: x86_64 using:
- AES-NI (AES new instructions)
- VAES (Vector AES)
+ Note: this option no longer provides the accelerated XTS, CBC, CTR,
+ and ECB code. For those just use CRYPTO_XTS, CRYPTO_CBC, etc.
+
config CRYPTO_BLOWFISH_X86_64
tristate "Ciphers: Blowfish, modes: ECB, CBC"
depends on 64BIT
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index 0c016ba87373..09bdea45bc3e 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -40,12 +40,10 @@ obj-$(CONFIG_CRYPTO_AEGIS128_AESNI_SSE2) += aegis128-aesni.o
aegis128-aesni-y := aegis128-aesni-asm.o aegis128-aesni-glue.o
obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o
-aesni-intel-y := aesni-intel_asm.o \
- aesni-intel_glue.o \
+aesni-intel-y := aesni-intel_glue.o \
aes-gcm-aesni-x86_64.o \
aes-gcm-vaes-avx2.o \
- aes-gcm-vaes-avx512.o \
- aes-xts-avx-x86_64.o
+ aes-gcm-vaes-avx512.o
obj-$(CONFIG_CRYPTO_SM4_AESNI_AVX_X86_64) += sm4-aesni-avx-x86_64.o
sm4-aesni-avx-x86_64-y := sm4-aesni-avx-asm_64.o sm4_aesni_avx_glue.o
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
deleted file mode 100644
index b12a0f2bf006..000000000000
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ /dev/null
@@ -1,220 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Implement AES algorithm in Intel AES-NI instructions.
- *
- * The white paper of AES-NI instructions can be downloaded from:
- * http://softwarecommunity.intel.com/isn/downloads/intelavx/AES-Instructions-Set_WP.pdf
- *
- * Copyright (C) 2008, Intel Corp.
- * Author: Huang Ying <ying.huang@intel.com>
- * Vinodh Gopal <vinodh.gopal@intel.com>
- * Kahraman Akdemir
- *
- * Copyright (c) 2010, Intel Corporation.
- *
- * Ported x86_64 version to x86:
- * Author: Mathias Krause <minipli@googlemail.com>
- */
-
-#include <linux/linkage.h>
-#include <linux/objtool.h>
-#include <asm/frame.h>
-
-#ifdef __x86_64__
-#define AREG %rax
-#define KEYP %rdi
-#define OUTP %rsi
-#define UKEYP OUTP
-#define T1 %r10
-#define TKEYP T1
-#define T2 %r11
-#else
-#define AREG %eax
-#define KEYP %edi
-#define OUTP AREG
-#define UKEYP OUTP
-#define T1 %ecx
-#define TKEYP T1
-#endif
-
-SYM_FUNC_START_LOCAL(_key_expansion_256a)
- pshufd $0b11111111, %xmm1, %xmm1
- shufps $0b00010000, %xmm0, %xmm4
- pxor %xmm4, %xmm0
- shufps $0b10001100, %xmm0, %xmm4
- pxor %xmm4, %xmm0
- pxor %xmm1, %xmm0
- movaps %xmm0, (TKEYP)
- add $0x10, TKEYP
- RET
-SYM_FUNC_END(_key_expansion_256a)
-SYM_FUNC_ALIAS_LOCAL(_key_expansion_128, _key_expansion_256a)
-
-SYM_FUNC_START_LOCAL(_key_expansion_192a)
- pshufd $0b01010101, %xmm1, %xmm1
- shufps $0b00010000, %xmm0, %xmm4
- pxor %xmm4, %xmm0
- shufps $0b10001100, %xmm0, %xmm4
- pxor %xmm4, %xmm0
- pxor %xmm1, %xmm0
-
- movaps %xmm2, %xmm5
- movaps %xmm2, %xmm6
- pslldq $4, %xmm5
- pshufd $0b11111111, %xmm0, %xmm3
- pxor %xmm3, %xmm2
- pxor %xmm5, %xmm2
-
- movaps %xmm0, %xmm1
- shufps $0b01000100, %xmm0, %xmm6
- movaps %xmm6, (TKEYP)
- shufps $0b01001110, %xmm2, %xmm1
- movaps %xmm1, 0x10(TKEYP)
- add $0x20, TKEYP
- RET
-SYM_FUNC_END(_key_expansion_192a)
-
-SYM_FUNC_START_LOCAL(_key_expansion_192b)
- pshufd $0b01010101, %xmm1, %xmm1
- shufps $0b00010000, %xmm0, %xmm4
- pxor %xmm4, %xmm0
- shufps $0b10001100, %xmm0, %xmm4
- pxor %xmm4, %xmm0
- pxor %xmm1, %xmm0
-
- movaps %xmm2, %xmm5
- pslldq $4, %xmm5
- pshufd $0b11111111, %xmm0, %xmm3
- pxor %xmm3, %xmm2
- pxor %xmm5, %xmm2
-
- movaps %xmm0, (TKEYP)
- add $0x10, TKEYP
- RET
-SYM_FUNC_END(_key_expansion_192b)
-
-SYM_FUNC_START_LOCAL(_key_expansion_256b)
- pshufd $0b10101010, %xmm1, %xmm1
- shufps $0b00010000, %xmm2, %xmm4
- pxor %xmm4, %xmm2
- shufps $0b10001100, %xmm2, %xmm4
- pxor %xmm4, %xmm2
- pxor %xmm1, %xmm2
- movaps %xmm2, (TKEYP)
- add $0x10, TKEYP
- RET
-SYM_FUNC_END(_key_expansion_256b)
-
-/*
- * void aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
- * unsigned int key_len)
- */
-SYM_FUNC_START(aesni_set_key)
- FRAME_BEGIN
-#ifndef __x86_64__
- pushl KEYP
- movl (FRAME_OFFSET+8)(%esp), KEYP # ctx
- movl (FRAME_OFFSET+12)(%esp), UKEYP # in_key
- movl (FRAME_OFFSET+16)(%esp), %edx # key_len
-#endif
- movups (UKEYP), %xmm0 # user key (first 16 bytes)
- movaps %xmm0, (KEYP)
- lea 0x10(KEYP), TKEYP # key addr
- movl %edx, 480(KEYP)
- pxor %xmm4, %xmm4 # xmm4 is assumed 0 in _key_expansion_x
- cmp $24, %dl
- jb .Lenc_key128
- je .Lenc_key192
- movups 0x10(UKEYP), %xmm2 # other user key
- movaps %xmm2, (TKEYP)
- add $0x10, TKEYP
- aeskeygenassist $0x1, %xmm2, %xmm1 # round 1
- call _key_expansion_256a
- aeskeygenassist $0x1, %xmm0, %xmm1
- call _key_expansion_256b
- aeskeygenassist $0x2, %xmm2, %xmm1 # round 2
- call _key_expansion_256a
- aeskeygenassist $0x2, %xmm0, %xmm1
- call _key_expansion_256b
- aeskeygenassist $0x4, %xmm2, %xmm1 # round 3
- call _key_expansion_256a
- aeskeygenassist $0x4, %xmm0, %xmm1
- call _key_expansion_256b
- aeskeygenassist $0x8, %xmm2, %xmm1 # round 4
- call _key_expansion_256a
- aeskeygenassist $0x8, %xmm0, %xmm1
- call _key_expansion_256b
- aeskeygenassist $0x10, %xmm2, %xmm1 # round 5
- call _key_expansion_256a
- aeskeygenassist $0x10, %xmm0, %xmm1
- call _key_expansion_256b
- aeskeygenassist $0x20, %xmm2, %xmm1 # round 6
- call _key_expansion_256a
- aeskeygenassist $0x20, %xmm0, %xmm1
- call _key_expansion_256b
- aeskeygenassist $0x40, %xmm2, %xmm1 # round 7
- call _key_expansion_256a
- jmp .Ldec_key
-.Lenc_key192:
- movq 0x10(UKEYP), %xmm2 # other user key
- aeskeygenassist $0x1, %xmm2, %xmm1 # round 1
- call _key_expansion_192a
- aeskeygenassist $0x2, %xmm2, %xmm1 # round 2
- call _key_expansion_192b
- aeskeygenassist $0x4, %xmm2, %xmm1 # round 3
- call _key_expansion_192a
- aeskeygenassist $0x8, %xmm2, %xmm1 # round 4
- call _key_expansion_192b
- aeskeygenassist $0x10, %xmm2, %xmm1 # round 5
- call _key_expansion_192a
- aeskeygenassist $0x20, %xmm2, %xmm1 # round 6
- call _key_expansion_192b
- aeskeygenassist $0x40, %xmm2, %xmm1 # round 7
- call _key_expansion_192a
- aeskeygenassist $0x80, %xmm2, %xmm1 # round 8
- call _key_expansion_192b
- jmp .Ldec_key
-.Lenc_key128:
- aeskeygenassist $0x1, %xmm0, %xmm1 # round 1
- call _key_expansion_128
- aeskeygenassist $0x2, %xmm0, %xmm1 # round 2
- call _key_expansion_128
- aeskeygenassist $0x4, %xmm0, %xmm1 # round 3
- call _key_expansion_128
- aeskeygenassist $0x8, %xmm0, %xmm1 # round 4
- call _key_expansion_128
- aeskeygenassist $0x10, %xmm0, %xmm1 # round 5
- call _key_expansion_128
- aeskeygenassist $0x20, %xmm0, %xmm1 # round 6
- call _key_expansion_128
- aeskeygenassist $0x40, %xmm0, %xmm1 # round 7
- call _key_expansion_128
- aeskeygenassist $0x80, %xmm0, %xmm1 # round 8
- call _key_expansion_128
- aeskeygenassist $0x1b, %xmm0, %xmm1 # round 9
- call _key_expansion_128
- aeskeygenassist $0x36, %xmm0, %xmm1 # round 10
- call _key_expansion_128
-.Ldec_key:
- sub $0x10, TKEYP
- movaps (KEYP), %xmm0
- movaps (TKEYP), %xmm1
- movaps %xmm0, 240(TKEYP)
- movaps %xmm1, 240(KEYP)
- add $0x10, KEYP
- lea 240-16(TKEYP), UKEYP
-.align 4
-.Ldec_key_loop:
- movaps (KEYP), %xmm0
- aesimc %xmm0, %xmm1
- movaps %xmm1, (UKEYP)
- add $0x10, KEYP
- sub $0x10, UKEYP
- cmp TKEYP, KEYP
- jb .Ldec_key_loop
-#ifndef __x86_64__
- popl KEYP
-#endif
- FRAME_END
- RET
-SYM_FUNC_END(aesni_set_key)
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 5aae178bbae3..234284312bc1 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Support for AES-NI and VAES instructions. This file contains glue code.
- * The real AES implementations are in aesni-intel_asm.S and other .S files.
+ * The real AES implementations are in .S files.
*
* Copyright (C) 2008, Intel Corp.
* Author: Huang Ying <ying.huang@intel.com>
@@ -26,7 +26,6 @@
#include <crypto/b128ops.h>
#include <crypto/gcm.h>
#include <crypto/gf128mul.h>
-#include <crypto/xts.h>
#include <asm/cpu_device_id.h>
#include <asm/simd.h>
#include <crypto/scatterwalk.h>
@@ -38,217 +37,6 @@
#include <linux/spinlock.h>
#include <linux/static_call.h>
-
-#define AESNI_ALIGN 16
-#define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
-#define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
-#define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
-
-struct aesni_xts_ctx {
- struct crypto_aes_ctx tweak_ctx AESNI_ALIGN_ATTR;
- struct crypto_aes_ctx crypt_ctx AESNI_ALIGN_ATTR;
-};
-
-static inline void *aes_align_addr(void *addr)
-{
- if (crypto_tfm_ctx_alignment() >= AESNI_ALIGN)
- return addr;
- return PTR_ALIGN(addr, AESNI_ALIGN);
-}
-
-asmlinkage void aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
- unsigned int key_len);
-
-static inline struct aesni_xts_ctx *aes_xts_ctx(struct crypto_skcipher *tfm)
-{
- return aes_align_addr(crypto_skcipher_ctx(tfm));
-}
-
-static int aes_set_key_common(struct crypto_aes_ctx *ctx,
- const u8 *in_key, unsigned int key_len)
-{
- int err;
-
- if (!crypto_simd_usable())
- return aes_expandkey(ctx, in_key, key_len);
-
- err = aes_check_keylen(key_len);
- if (err)
- return err;
-
- kernel_fpu_begin();
- aesni_set_key(ctx, in_key, key_len);
- kernel_fpu_end();
- return 0;
-}
-
-static int xts_setkey_aesni(struct crypto_skcipher *tfm, const u8 *key,
- unsigned int keylen)
-{
- struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
- int err;
-
- err = xts_verify_key(tfm, key, keylen);
- if (err)
- return err;
-
- keylen /= 2;
-
- /* first half of xts-key is for crypt */
- err = aes_set_key_common(&ctx->crypt_ctx, key, keylen);
- if (err)
- return err;
-
- /* second half of xts-key is for tweak */
- return aes_set_key_common(&ctx->tweak_ctx, key + keylen, keylen);
-}
-
-typedef void (*xts_encrypt_iv_func)(const struct crypto_aes_ctx *tweak_key,
- u8 iv[AES_BLOCK_SIZE]);
-typedef void (*xts_crypt_func)(const struct crypto_aes_ctx *key,
- const u8 *src, u8 *dst, int len,
- u8 tweak[AES_BLOCK_SIZE]);
-
-/* This handles cases where the source and/or destination span pages. */
-static noinline int
-xts_crypt_slowpath(struct skcipher_request *req, xts_crypt_func crypt_func)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- const struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
- int tail = req->cryptlen % AES_BLOCK_SIZE;
- struct scatterlist sg_src[2], sg_dst[2];
- struct skcipher_request subreq;
- struct skcipher_walk walk;
- struct scatterlist *src, *dst;
- int err;
-
- /*
- * If the message length isn't divisible by the AES block size, then
- * separate off the last full block and the partial block. This ensures
- * that they are processed in the same call to the assembly function,
- * which is required for ciphertext stealing.
- */
- if (tail) {
- skcipher_request_set_tfm(&subreq, tfm);
- skcipher_request_set_callback(&subreq,
- skcipher_request_flags(req),
- NULL, NULL);
- skcipher_request_set_crypt(&subreq, req->src, req->dst,
- req->cryptlen - tail - AES_BLOCK_SIZE,
- req->iv);
- req = &subreq;
- }
-
- err = skcipher_walk_virt(&walk, req, false);
-
- while (walk.nbytes) {
- kernel_fpu_begin();
- (*crypt_func)(&ctx->crypt_ctx,
- walk.src.virt.addr, walk.dst.virt.addr,
- walk.nbytes & ~(AES_BLOCK_SIZE - 1), req->iv);
- kernel_fpu_end();
- err = skcipher_walk_done(&walk,
- walk.nbytes & (AES_BLOCK_SIZE - 1));
- }
-
- if (err || !tail)
- return err;
-
- /* Do ciphertext stealing with the last full block and partial block. */
-
- dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
- if (req->dst != req->src)
- dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
-
- skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
- req->iv);
-
- err = skcipher_walk_virt(&walk, req, false);
- if (err)
- return err;
-
- kernel_fpu_begin();
- (*crypt_func)(&ctx->crypt_ctx, walk.src.virt.addr, walk.dst.virt.addr,
- walk.nbytes, req->iv);
- kernel_fpu_end();
-
- return skcipher_walk_done(&walk, 0);
-}
-
-/* __always_inline to avoid indirect call in fastpath */
-static __always_inline int
-xts_crypt(struct skcipher_request *req, xts_encrypt_iv_func encrypt_iv,
- xts_crypt_func crypt_func)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- const struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
-
- if (unlikely(req->cryptlen < AES_BLOCK_SIZE))
- return -EINVAL;
-
- kernel_fpu_begin();
- (*encrypt_iv)(&ctx->tweak_ctx, req->iv);
-
- /*
- * In practice, virtually all XTS plaintexts and ciphertexts are either
- * 512 or 4096 bytes and do not use multiple scatterlist elements. To
- * optimize the performance of these cases, the below fast-path handles
- * single-scatterlist-element messages as efficiently as possible. The
- * code is 64-bit specific, as it assumes no page mapping is needed.
- */
- if (likely(req->src->length >= req->cryptlen &&
- req->dst->length >= req->cryptlen)) {
- (*crypt_func)(&ctx->crypt_ctx, sg_virt(req->src),
- sg_virt(req->dst), req->cryptlen, req->iv);
- kernel_fpu_end();
- return 0;
- }
- kernel_fpu_end();
- return xts_crypt_slowpath(req, crypt_func);
-}
-
-asmlinkage void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key,
- u8 iv[AES_BLOCK_SIZE]);
-
-#define DEFINE_AVX_SKCIPHER_ALGS(suffix, driver_name_suffix, priority) \
- \
-asmlinkage void \
-aes_xts_encrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src, \
- u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]); \
-asmlinkage void \
-aes_xts_decrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src, \
- u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]); \
- \
-static int xts_encrypt_##suffix(struct skcipher_request *req) \
-{ \
- return xts_crypt(req, aes_xts_encrypt_iv, aes_xts_encrypt_##suffix); \
-} \
- \
-static int xts_decrypt_##suffix(struct skcipher_request *req) \
-{ \
- return xts_crypt(req, aes_xts_encrypt_iv, aes_xts_decrypt_##suffix); \
-} \
- \
-static struct skcipher_alg skcipher_algs_##suffix[] = {{ \
- .base.cra_name = "xts(aes)", \
- .base.cra_driver_name = "xts-aes-" driver_name_suffix, \
- .base.cra_priority = priority, \
- .base.cra_blocksize = AES_BLOCK_SIZE, \
- .base.cra_ctxsize = XTS_AES_CTX_SIZE, \
- .base.cra_module = THIS_MODULE, \
- .min_keysize = 2 * AES_MIN_KEY_SIZE, \
- .max_keysize = 2 * AES_MAX_KEY_SIZE, \
- .ivsize = AES_BLOCK_SIZE, \
- .walksize = 2 * AES_BLOCK_SIZE, \
- .setkey = xts_setkey_aesni, \
- .encrypt = xts_encrypt_##suffix, \
- .decrypt = xts_decrypt_##suffix, \
-}}
-
-DEFINE_AVX_SKCIPHER_ALGS(aesni_avx, "aesni-avx", 500);
-DEFINE_AVX_SKCIPHER_ALGS(vaes_avx2, "vaes-avx2", 600);
-DEFINE_AVX_SKCIPHER_ALGS(vaes_avx512, "vaes-avx512", 800);
-
/* The common part of the x86_64 AES-GCM key struct */
struct aes_gcm_key {
/* Expanded AES key and the AES key length in bytes */
@@ -1002,10 +790,6 @@ static int __init register_avx_algs(void)
if (!boot_cpu_has(X86_FEATURE_AVX))
return 0;
- err = crypto_register_skciphers(skcipher_algs_aesni_avx,
- ARRAY_SIZE(skcipher_algs_aesni_avx));
- if (err)
- return err;
err = crypto_register_aeads(aes_gcm_algs_aesni_avx,
ARRAY_SIZE(aes_gcm_algs_aesni_avx));
if (err)
@@ -1022,10 +806,6 @@ static int __init register_avx_algs(void)
!boot_cpu_has(X86_FEATURE_PCLMULQDQ) ||
!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
return 0;
- err = crypto_register_skciphers(skcipher_algs_vaes_avx2,
- ARRAY_SIZE(skcipher_algs_vaes_avx2));
- if (err)
- return err;
err = crypto_register_aeads(aes_gcm_algs_vaes_avx2,
ARRAY_SIZE(aes_gcm_algs_vaes_avx2));
if (err)
@@ -1041,16 +821,10 @@ static int __init register_avx_algs(void)
if (boot_cpu_has(X86_FEATURE_PREFER_YMM)) {
int i;
- for (i = 0; i < ARRAY_SIZE(skcipher_algs_vaes_avx512); i++)
- skcipher_algs_vaes_avx512[i].base.cra_priority = 1;
for (i = 0; i < ARRAY_SIZE(aes_gcm_algs_vaes_avx512); i++)
aes_gcm_algs_vaes_avx512[i].base.cra_priority = 1;
}
- err = crypto_register_skciphers(skcipher_algs_vaes_avx512,
- ARRAY_SIZE(skcipher_algs_vaes_avx512));
- if (err)
- return err;
err = crypto_register_aeads(aes_gcm_algs_vaes_avx512,
ARRAY_SIZE(aes_gcm_algs_vaes_avx512));
if (err)
@@ -1059,19 +833,13 @@ static int __init register_avx_algs(void)
return 0;
}
-#define unregister_skciphers(A) \
- if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \
- crypto_unregister_skciphers((A), ARRAY_SIZE(A))
#define unregister_aeads(A) \
if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \
crypto_unregister_aeads((A), ARRAY_SIZE(A))
static void unregister_avx_algs(void)
{
- unregister_skciphers(skcipher_algs_aesni_avx);
unregister_aeads(aes_gcm_algs_aesni_avx);
- unregister_skciphers(skcipher_algs_vaes_avx2);
- unregister_skciphers(skcipher_algs_vaes_avx512);
unregister_aeads(aes_gcm_algs_vaes_avx2);
unregister_aeads(aes_gcm_algs_vaes_avx512);
}
@@ -1117,6 +885,7 @@ static void __exit aesni_exit(void)
module_init(aesni_init);
module_exit(aesni_exit);
-MODULE_DESCRIPTION("AES cipher and modes, optimized with AES-NI or VAES instructions");
+MODULE_DESCRIPTION("AES-GCM, optimized with AES-NI or VAES instructions");
MODULE_LICENSE("GPL");
-MODULE_ALIAS_CRYPTO("aes");
+MODULE_ALIAS_CRYPTO("gcm(aes)");
+MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))");
diff --git a/crypto/aes.c b/crypto/aes.c
index 568a900bfec8..756fce2157be 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -701,7 +701,7 @@ static struct skcipher_alg skcipher_algs[] = {
{
.base.cra_name = "xts(aes)",
.base.cra_driver_name = "xts-aes-lib",
- .base.cra_priority = 110,
+ .base.cra_priority = IS_ENABLED(CONFIG_X86) ? 300 : 110,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct aes_xts_key),
.base.cra_module = THIS_MODULE,
diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
index 5d5484fc78ea..02d89a226377 100644
--- a/lib/crypto/Makefile
+++ b/lib/crypto/Makefile
@@ -57,6 +57,9 @@ libaes-$(CONFIG_X86) += x86/aes-aesni.o
ifneq ($(CONFIG_CRYPTO_LIB_AES_CTR),)
libaes-$(CONFIG_X86_64) += x86/aes-ctr-avx-x86_64.o
endif
+ifneq ($(CONFIG_CRYPTO_LIB_AES_XTS),)
+libaes-$(CONFIG_X86_64) += x86/aes-xts-avx-x86_64.o
+endif
endif # CONFIG_CRYPTO_LIB_AES_ARCH
# clean-files must be defined unconditionally
diff --git a/arch/x86/crypto/aes-xts-avx-x86_64.S b/lib/crypto/x86/aes-xts-avx-x86_64.S
similarity index 81%
rename from arch/x86/crypto/aes-xts-avx-x86_64.S
rename to lib/crypto/x86/aes-xts-avx-x86_64.S
index a30753a3e207..2c13451ea343 100644
--- a/arch/x86/crypto/aes-xts-avx-x86_64.S
+++ b/lib/crypto/x86/aes-xts-avx-x86_64.S
@@ -80,14 +80,17 @@
* any CPUs that support VAES but not VPCLMULQDQ. If that changes, we might
* need to start also providing an implementation using VAES alone.
*
- * The AES-XTS implementations in this file support everything required by the
- * crypto API, including support for arbitrary input lengths and multi-part
- * processing. However, they are most heavily optimized for the common case of
- * power-of-2 length inputs that are processed in a single part (disk sectors).
+ * These assembly functions don't handle ciphertext stealing, i.e, lengths that
+ * aren't a multiple of 16 bytes. That case is not actually reached in the
+ * current use cases of AES-XTS in the kernel and is just handled by the C code.
*/
#include <linux/linkage.h>
-#include <linux/cfi_types.h>
+
+// Offsets in struct aes_key
+#define OFFSETOF_KEYLEN 0
+#define OFFSETOF_RNDKEYS 16
+#define OFFSETOF_INVRNDKEYS 256
.section .rodata
.p2align 4
@@ -111,16 +114,6 @@
.Llshift_amounts:
.byte 0, 0, 1, 1, 2, 2, 3, 3
- // This table contains constants for vpshufb and vpblendvb, used to
- // handle variable byte shifts and blending during ciphertext stealing
- // on CPUs that don't support AVX512-style masking.
-.Lcts_permute_table:
- .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
- .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
- .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
- .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
- .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
- .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
.text
.macro _define_Vi i
@@ -149,14 +142,13 @@
.endif
// Function parameters
- .set KEY, %rdi // Initially points to crypto_aes_ctx, then is
- // advanced to point to 7th-from-last round key
+ .set DST, %rdi // Pointer to next destination data
.set SRC, %rsi // Pointer to next source data
- .set DST, %rdx // Pointer to next destination data
- .set LEN, %ecx // Remaining length in bytes
- .set LEN8, %cl
- .set LEN64, %rcx
- .set TWEAK, %r8 // Pointer to next tweak
+ .set NBLOCKS, %rdx // Number of blocks remaining
+ .set NBLOCKS32, %edx // Used for improved code density
+ .set TWEAK, %rcx // Pointer to next tweak
+ .set KEY, %r8 // Initially points to struct aes_key, then is
+ // advanced to point to 7th-from-last round key
// %rax holds the AES key length in bytes.
.set KEYLEN, %eax
@@ -468,9 +460,9 @@
// Select either the encryption round keys or the decryption round keys.
.if \enc
- .set OFFS, 0
+ .set OFFS, OFFSETOF_RNDKEYS
.else
- .set OFFS, 240
+ .set OFFS, OFFSETOF_INVRNDKEYS
.endif
// Load the round key for "round 0".
@@ -615,19 +607,8 @@
.macro _aes_xts_crypt enc
_define_aliases
-.if !\enc
- // When decrypting a message whose length isn't a multiple of the AES
- // block length, exclude the last full block from the main loop by
- // subtracting 16 from LEN. This is needed because ciphertext stealing
- // decryption uses the last two tweaks in reverse order. We'll handle
- // the last full block and the partial block specially at the end.
- lea -16(LEN), %eax
- test $15, LEN8
- cmovnz %eax, LEN
-.endif
-
// Load the AES key length: 16 (AES-128), 24 (AES-192), or 32 (AES-256).
- movl 480(KEY), KEYLEN
+ movl OFFSETOF_KEYLEN(KEY), KEYLEN
// Setup the pointer to the round keys and cache as many as possible.
_setup_round_keys \enc
@@ -635,7 +616,7 @@
// Compute the first set of tweaks TWEAK[0-3].
_compute_first_set_of_tweaks
- add $-4*VL, LEN // shorter than 'sub 4*VL' when VL=32
+ sub $4*VL/16, NBLOCKS
jl .Lhandle_remainder\@
.Lmain_loop\@:
@@ -715,13 +696,13 @@
sub $-4*VL, SRC // shorter than 'add 4*VL' when VL=32
sub $-4*VL, DST
- add $-4*VL, LEN
+ sub $4*VL/16, NBLOCKS
jge .Lmain_loop\@
// Check for the uncommon case where the data length isn't a multiple of
// 4*VL. Handle it out-of-line in order to optimize for the common
// case. In the common case, just fall through to the ret.
- test $4*VL-1, LEN8
+ test $(4*VL/16)-1, NBLOCKS32
jnz .Lhandle_remainder\@
.Ldone\@:
// Store the next tweak back to *TWEAK to support continuation calls.
@@ -733,9 +714,9 @@
.Lhandle_remainder\@:
- // En/decrypt any remaining full blocks, one vector at a time.
+ // En/decrypt any remaining blocks, one vector at a time.
.if VL > 16
- add $3*VL, LEN // Undo extra sub of 4*VL, then sub VL.
+ add $3*VL/16, NBLOCKS32
jl .Lvec_at_a_time_done\@
.Lvec_at_a_time\@:
_vmovdqu (SRC), V0
@@ -744,16 +725,16 @@
_next_tweakvec TWEAK0, V0, V1, TWEAK0
add $VL, SRC
add $VL, DST
- sub $VL, LEN
+ sub $VL/16, NBLOCKS32
jge .Lvec_at_a_time\@
.Lvec_at_a_time_done\@:
- add $VL-16, LEN // Undo extra sub of VL, then sub 16.
+ add $VL/16, NBLOCKS32
.else
- add $4*VL-16, LEN // Undo extra sub of 4*VL, then sub 16.
+ add $4*VL/16, NBLOCKS32
.endif
- // En/decrypt any remaining full blocks, one at a time.
- jl .Lblock_at_a_time_done\@
+ // En/decrypt any remaining blocks, one at a time.
+ jz .Ldone\@
.Lblock_at_a_time\@:
vmovdqu (SRC), %xmm0
_aes_crypt \enc, _XMM, TWEAK0_XMM, %xmm0, tmp=%xmm1
@@ -761,92 +742,26 @@
_next_tweak TWEAK0_XMM, %xmm0, TWEAK0_XMM
add $16, SRC
add $16, DST
- sub $16, LEN
- jge .Lblock_at_a_time\@
-.Lblock_at_a_time_done\@:
- add $16, LEN // Undo the extra sub of 16.
- // Now 0 <= LEN <= 15. If LEN is zero, we're done.
- jz .Ldone\@
-
- // Otherwise 1 <= LEN <= 15, but the real remaining length is 16 + LEN.
- // Do ciphertext stealing to process the last 16 + LEN bytes.
-
-.if \enc
- // If encrypting, the main loop already encrypted the last full block to
- // create the CTS intermediate ciphertext. Prepare for the rest of CTS
- // by rewinding the pointers and loading the intermediate ciphertext.
- sub $16, SRC
- sub $16, DST
- vmovdqu (DST), %xmm0
-.else
- // If decrypting, the main loop didn't decrypt the last full block
- // because CTS decryption uses the last two tweaks in reverse order.
- // Do it now by advancing the tweak and decrypting the last full block.
- _next_tweak TWEAK0_XMM, %xmm0, TWEAK1_XMM
- vmovdqu (SRC), %xmm0
- _aes_crypt \enc, _XMM, TWEAK1_XMM, %xmm0, tmp=%xmm1
-.endif
-
-.if USE_AVX512
- // Create a mask that has the first LEN bits set.
- mov $-1, %r9d
- bzhi LEN, %r9d, %r9d
- kmovd %r9d, %k1
-
- // Swap the first LEN bytes of the en/decryption of the last full block
- // with the partial block. Note that to support in-place en/decryption,
- // the load from the src partial block must happen before the store to
- // the dst partial block.
- vmovdqa %xmm0, %xmm1
- vmovdqu8 16(SRC), %xmm0{%k1}
- vmovdqu8 %xmm1, 16(DST){%k1}
-.else
- lea .Lcts_permute_table(%rip), %r9
-
- // Load the src partial block, left-aligned. Note that to support
- // in-place en/decryption, this must happen before the store to the dst
- // partial block.
- vmovdqu (SRC, LEN64, 1), %xmm1
-
- // Shift the first LEN bytes of the en/decryption of the last full block
- // to the end of a register, then store it to DST+LEN. This stores the
- // dst partial block. It also writes to the second part of the dst last
- // full block, but that part is overwritten later.
- vpshufb (%r9, LEN64, 1), %xmm0, %xmm2
- vmovdqu %xmm2, (DST, LEN64, 1)
-
- // Make xmm3 contain [16-LEN,16-LEN+1,...,14,15,0x80,0x80,...].
- sub LEN64, %r9
- vmovdqu 32(%r9), %xmm3
-
- // Shift the src partial block to the beginning of its register.
- vpshufb %xmm3, %xmm1, %xmm1
-
- // Do a blend to generate the src partial block followed by the second
- // part of the en/decryption of the last full block.
- vpblendvb %xmm3, %xmm0, %xmm1, %xmm0
-.endif
- // En/decrypt again and store the last full block.
- _aes_crypt \enc, _XMM, TWEAK0_XMM, %xmm0, tmp=%xmm1
- vmovdqu %xmm0, (DST)
+ dec NBLOCKS32
+ jnz .Lblock_at_a_time\@
jmp .Ldone\@
.endm
-// void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key,
+// void aes_xts_encrypt_iv(const struct aes_enckey *tweak_key,
// u8 iv[AES_BLOCK_SIZE]);
//
// Encrypt |iv| using the AES key |tweak_key| to get the first tweak. Assumes
// that the CPU supports AES-NI and AVX, but not necessarily VAES or AVX512.
-SYM_TYPED_FUNC_START(aes_xts_encrypt_iv)
+SYM_FUNC_START(aes_xts_encrypt_iv)
.set TWEAK_KEY, %rdi
.set IV, %rsi
.set KEYLEN, %eax
.set KEYLEN64, %rax
vmovdqu (IV), %xmm0
- vpxor (TWEAK_KEY), %xmm0, %xmm0
- movl 480(TWEAK_KEY), KEYLEN
- lea -16(TWEAK_KEY, KEYLEN64, 4), TWEAK_KEY
+ vpxor OFFSETOF_RNDKEYS(TWEAK_KEY), %xmm0, %xmm0
+ movl OFFSETOF_KEYLEN(TWEAK_KEY), KEYLEN
+ lea OFFSETOF_RNDKEYS-16(TWEAK_KEY, KEYLEN64, 4), TWEAK_KEY
cmp $24, KEYLEN
jl .Lencrypt_iv_aes128
je .Lencrypt_iv_aes192
@@ -867,39 +782,37 @@ SYM_FUNC_END(aes_xts_encrypt_iv)
// Below are the actual AES-XTS encryption and decryption functions,
// instantiated from the above macro. They all have the following prototype:
//
-// void (*xts_crypt_func)(const struct crypto_aes_ctx *key,
-// const u8 *src, u8 *dst, int len,
-// u8 tweak[AES_BLOCK_SIZE]);
+// void aes_xts_{en,de}crypt_{cpufeature}(u8 *dst, const u8 *src, long nblocks,
+// u8 tweak[AES_BLOCK_SIZE],
+// const struct aes_key *key);
//
-// |key| is the data key. |tweak| contains the next tweak; the encryption of
-// the original IV with the tweak key was already done. This function supports
-// incremental computation, but |len| must always be >= 16 (AES_BLOCK_SIZE), and
-// |len| must be a multiple of 16 except on the last call. If |len| is a
-// multiple of 16, then this function updates |tweak| to contain the next tweak.
+// `tweak` must have already been encrypted by the tweak key; `key` is just the
+// main key. `tweak` is updated to contain the next tweak, allowing for
+// incremental computation.
.set VL, 16
.set USE_AVX512, 0
-SYM_TYPED_FUNC_START(aes_xts_encrypt_aesni_avx)
+SYM_FUNC_START(aes_xts_encrypt_aesni_avx)
_aes_xts_crypt 1
SYM_FUNC_END(aes_xts_encrypt_aesni_avx)
-SYM_TYPED_FUNC_START(aes_xts_decrypt_aesni_avx)
+SYM_FUNC_START(aes_xts_decrypt_aesni_avx)
_aes_xts_crypt 0
SYM_FUNC_END(aes_xts_decrypt_aesni_avx)
.set VL, 32
.set USE_AVX512, 0
-SYM_TYPED_FUNC_START(aes_xts_encrypt_vaes_avx2)
+SYM_FUNC_START(aes_xts_encrypt_vaes_avx2)
_aes_xts_crypt 1
SYM_FUNC_END(aes_xts_encrypt_vaes_avx2)
-SYM_TYPED_FUNC_START(aes_xts_decrypt_vaes_avx2)
+SYM_FUNC_START(aes_xts_decrypt_vaes_avx2)
_aes_xts_crypt 0
SYM_FUNC_END(aes_xts_decrypt_vaes_avx2)
.set VL, 64
.set USE_AVX512, 1
-SYM_TYPED_FUNC_START(aes_xts_encrypt_vaes_avx512)
+SYM_FUNC_START(aes_xts_encrypt_vaes_avx512)
_aes_xts_crypt 1
SYM_FUNC_END(aes_xts_encrypt_vaes_avx512)
-SYM_TYPED_FUNC_START(aes_xts_decrypt_vaes_avx512)
+SYM_FUNC_START(aes_xts_decrypt_vaes_avx512)
_aes_xts_crypt 0
SYM_FUNC_END(aes_xts_decrypt_vaes_avx512)
diff --git a/lib/crypto/x86/aes.h b/lib/crypto/x86/aes.h
index 8ed247ddb8f2..b555290ab075 100644
--- a/lib/crypto/x86/aes.h
+++ b/lib/crypto/x86/aes.h
@@ -16,6 +16,7 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vaes_avx512);
static_assert(offsetof(struct aes_enckey, len) == 0);
static_assert(offsetof(struct aes_enckey, nrounds) == 4);
static_assert(offsetof(struct aes_enckey, k.rndkeys) == 16);
+static_assert(offsetof(struct aes_key, len) == 0);
static_assert(offsetof(struct aes_key, nrounds) == 4);
static_assert(offsetof(struct aes_key, k.rndkeys) == 16);
static_assert(offsetof(struct aes_key, inv_k.inv_rndkeys) == 256);
@@ -312,6 +313,26 @@ void aes_xts_encrypt_aesni(u8 *dst, const u8 *src, long nblocks,
u8 tweak[AES_BLOCK_SIZE], const struct aes_key *key);
void aes_xts_decrypt_aesni(u8 *dst, const u8 *src, long nblocks,
u8 tweak[AES_BLOCK_SIZE], const struct aes_key *key);
+void aes_xts_encrypt_iv(const struct aes_enckey *tweak_key,
+ u8 iv[AES_BLOCK_SIZE]);
+void aes_xts_encrypt_aesni_avx(u8 *dst, const u8 *src, long nblocks,
+ u8 tweak[AES_BLOCK_SIZE],
+ const struct aes_key *key);
+void aes_xts_decrypt_aesni_avx(u8 *dst, const u8 *src, long nblocks,
+ u8 tweak[AES_BLOCK_SIZE],
+ const struct aes_key *key);
+void aes_xts_encrypt_vaes_avx2(u8 *dst, const u8 *src, long nblocks,
+ u8 tweak[AES_BLOCK_SIZE],
+ const struct aes_key *key);
+void aes_xts_decrypt_vaes_avx2(u8 *dst, const u8 *src, long nblocks,
+ u8 tweak[AES_BLOCK_SIZE],
+ const struct aes_key *key);
+void aes_xts_encrypt_vaes_avx512(u8 *dst, const u8 *src, long nblocks,
+ u8 tweak[AES_BLOCK_SIZE],
+ const struct aes_key *key);
+void aes_xts_decrypt_vaes_avx512(u8 *dst, const u8 *src, long nblocks,
+ u8 tweak[AES_BLOCK_SIZE],
+ const struct aes_key *key);
/* len is always a positive multiple of AES_BLOCK_SIZE here. */
static __always_inline bool
@@ -324,12 +345,46 @@ aes_xts_crypt_x86(u8 *dst, const u8 *src, size_t len, u8 tweak[AES_BLOCK_SIZE],
return false;
kernel_fpu_begin();
- if (!cont)
- aes_encrypt_aesni(tweak, tweak, &key->tweak_key);
- if (enc)
- aes_xts_encrypt_aesni(dst, src, nblocks, tweak, &key->main_key);
- else
- aes_xts_decrypt_aesni(dst, src, nblocks, tweak, &key->main_key);
+ if (IS_ENABLED(CONFIG_X86_64) &&
+ static_branch_likely(&have_vaes_avx512)) {
+ if (!cont)
+ aes_xts_encrypt_iv(&key->tweak_key, tweak);
+ if (enc)
+ aes_xts_encrypt_vaes_avx512(dst, src, nblocks, tweak,
+ &key->main_key);
+ else
+ aes_xts_decrypt_vaes_avx512(dst, src, nblocks, tweak,
+ &key->main_key);
+ } else if (IS_ENABLED(CONFIG_X86_64) &&
+ static_branch_likely(&have_vaes_avx2)) {
+ if (!cont)
+ aes_xts_encrypt_iv(&key->tweak_key, tweak);
+ if (enc)
+ aes_xts_encrypt_vaes_avx2(dst, src, nblocks, tweak,
+ &key->main_key);
+ else
+ aes_xts_decrypt_vaes_avx2(dst, src, nblocks, tweak,
+ &key->main_key);
+ } else if (IS_ENABLED(CONFIG_X86_64) &&
+ static_branch_likely(&have_aesni_avx)) {
+ if (!cont)
+ aes_xts_encrypt_iv(&key->tweak_key, tweak);
+ if (enc)
+ aes_xts_encrypt_aesni_avx(dst, src, nblocks, tweak,
+ &key->main_key);
+ else
+ aes_xts_decrypt_aesni_avx(dst, src, nblocks, tweak,
+ &key->main_key);
+ } else {
+ if (!cont)
+ aes_encrypt_aesni(tweak, tweak, &key->tweak_key);
+ if (enc)
+ aes_xts_encrypt_aesni(dst, src, nblocks, tweak,
+ &key->main_key);
+ else
+ aes_xts_decrypt_aesni(dst, src, nblocks, tweak,
+ &key->main_key);
+ }
kernel_fpu_end();
return true;
}
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread* [PATCH v2 15/20] lib/crypto: riscv/aes: Copy aes-macros.S to library
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
` (13 preceding siblings ...)
2026-09-27 22:43 ` [PATCH v2 14/20] lib/crypto: x86/aes-xts: " Eric Biggers
@ 2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 16/20] lib/crypto: riscv/aes: Pass key struct to assembly code Eric Biggers
` (4 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:43 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers
Copy arch/riscv/crypto/aes-macros.S to lib/crypto/riscv/aes-macros.S and
make lib/crypto/riscv/aes-riscv64-zvkned.S include the latter copy.
This makes it possible to change these macros without interfering with
the remaining code in arch/riscv/crypto/.
Of course, the copy in arch/riscv/crypto/ will be removed once the rest
of the AES code there is migrated to the library.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/riscv/aes-macros.S | 166 ++++++++++++++++++++++++++
lib/crypto/riscv/aes-riscv64-zvkned.S | 2 +-
2 files changed, 167 insertions(+), 1 deletion(-)
create mode 100644 lib/crypto/riscv/aes-macros.S
diff --git a/lib/crypto/riscv/aes-macros.S b/lib/crypto/riscv/aes-macros.S
new file mode 100644
index 000000000000..1384164621a5
--- /dev/null
+++ b/lib/crypto/riscv/aes-macros.S
@@ -0,0 +1,166 @@
+/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */
+//
+// This file is dual-licensed, meaning that you can use it under your
+// choice of either of the following two licenses:
+//
+// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
+//
+// Licensed under the Apache License 2.0 (the "License"). You can obtain
+// a copy in the file LICENSE in the source distribution or at
+// https://www.openssl.org/source/license.html
+//
+// or
+//
+// Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu>
+// Copyright (c) 2023, Phoebe Chen <phoebe.chen@sifive.com>
+// Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com>
+// Copyright 2024 Google LLC
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// This file contains macros that are shared by the other aes-*.S files. The
+// generated code of these macros depends on the following RISC-V extensions:
+// - RV64I
+// - RISC-V Vector ('V') with VLEN >= 128
+// - RISC-V Vector AES block cipher extension ('Zvkned')
+
+// Loads the AES round keys from \keyp into vector registers and jumps to code
+// specific to the length of the key. Specifically:
+// - If AES-128, loads round keys into v1-v11 and jumps to \label128.
+// - If AES-192, loads round keys into v1-v13 and jumps to \label192.
+// - If AES-256, loads round keys into v1-v15 and continues onwards.
+//
+// Also sets vl=4 and vtype=e32,m1,ta,ma. Clobbers t0 and t1.
+.macro aes_begin keyp, label128, label192, key_len
+.ifb \key_len
+ lwu t0, 480(\keyp) // t0 = key length in bytes
+.endif
+ li t1, 24 // t1 = key length for AES-192
+ vsetivli zero, 4, e32, m1, ta, ma
+ vle32.v v1, (\keyp)
+ addi \keyp, \keyp, 16
+ vle32.v v2, (\keyp)
+ addi \keyp, \keyp, 16
+ vle32.v v3, (\keyp)
+ addi \keyp, \keyp, 16
+ vle32.v v4, (\keyp)
+ addi \keyp, \keyp, 16
+ vle32.v v5, (\keyp)
+ addi \keyp, \keyp, 16
+ vle32.v v6, (\keyp)
+ addi \keyp, \keyp, 16
+ vle32.v v7, (\keyp)
+ addi \keyp, \keyp, 16
+ vle32.v v8, (\keyp)
+ addi \keyp, \keyp, 16
+ vle32.v v9, (\keyp)
+ addi \keyp, \keyp, 16
+ vle32.v v10, (\keyp)
+ addi \keyp, \keyp, 16
+ vle32.v v11, (\keyp)
+.ifb \key_len
+ blt t0, t1, \label128 // If AES-128, goto label128.
+.else
+ blt \key_len, t1, \label128 // If AES-128, goto label128.
+.endif
+ addi \keyp, \keyp, 16
+ vle32.v v12, (\keyp)
+ addi \keyp, \keyp, 16
+ vle32.v v13, (\keyp)
+.ifb \key_len
+ beq t0, t1, \label192 // If AES-192, goto label192.
+.else
+ beq \key_len, t1, \label192 // If AES-192, goto label192.
+.endif
+ // Else, it's AES-256.
+ addi \keyp, \keyp, 16
+ vle32.v v14, (\keyp)
+ addi \keyp, \keyp, 16
+ vle32.v v15, (\keyp)
+.endm
+
+// Encrypts \data using zvkned instructions, using the round keys loaded into
+// v1-v11 (for AES-128), v1-v13 (for AES-192), or v1-v15 (for AES-256). \keylen
+// is the AES key length in bits. vl and vtype must already be set
+// appropriately. Note that if vl > 4, multiple blocks are encrypted.
+.macro aes_encrypt data, keylen
+ vaesz.vs \data, v1
+ vaesem.vs \data, v2
+ vaesem.vs \data, v3
+ vaesem.vs \data, v4
+ vaesem.vs \data, v5
+ vaesem.vs \data, v6
+ vaesem.vs \data, v7
+ vaesem.vs \data, v8
+ vaesem.vs \data, v9
+ vaesem.vs \data, v10
+.if \keylen == 128
+ vaesef.vs \data, v11
+.elseif \keylen == 192
+ vaesem.vs \data, v11
+ vaesem.vs \data, v12
+ vaesef.vs \data, v13
+.else
+ vaesem.vs \data, v11
+ vaesem.vs \data, v12
+ vaesem.vs \data, v13
+ vaesem.vs \data, v14
+ vaesef.vs \data, v15
+.endif
+.endm
+
+// Same as aes_encrypt, but decrypts instead of encrypts.
+.macro aes_decrypt data, keylen
+.if \keylen == 128
+ vaesz.vs \data, v11
+.elseif \keylen == 192
+ vaesz.vs \data, v13
+ vaesdm.vs \data, v12
+ vaesdm.vs \data, v11
+.else
+ vaesz.vs \data, v15
+ vaesdm.vs \data, v14
+ vaesdm.vs \data, v13
+ vaesdm.vs \data, v12
+ vaesdm.vs \data, v11
+.endif
+ vaesdm.vs \data, v10
+ vaesdm.vs \data, v9
+ vaesdm.vs \data, v8
+ vaesdm.vs \data, v7
+ vaesdm.vs \data, v6
+ vaesdm.vs \data, v5
+ vaesdm.vs \data, v4
+ vaesdm.vs \data, v3
+ vaesdm.vs \data, v2
+ vaesdf.vs \data, v1
+.endm
+
+// Expands to aes_encrypt or aes_decrypt according to \enc, which is 1 or 0.
+.macro aes_crypt data, enc, keylen
+.if \enc
+ aes_encrypt \data, \keylen
+.else
+ aes_decrypt \data, \keylen
+.endif
+.endm
diff --git a/lib/crypto/riscv/aes-riscv64-zvkned.S b/lib/crypto/riscv/aes-riscv64-zvkned.S
index 0d988bc3d37b..7a52ea6c669d 100644
--- a/lib/crypto/riscv/aes-riscv64-zvkned.S
+++ b/lib/crypto/riscv/aes-riscv64-zvkned.S
@@ -48,7 +48,7 @@
.text
.option arch, +zvkned
-#include "../../arch/riscv/crypto/aes-macros.S"
+#include "aes-macros.S"
#define RNDKEYS a0
#define KEY_LEN a1
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread* [PATCH v2 16/20] lib/crypto: riscv/aes: Pass key struct to assembly code
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
` (14 preceding siblings ...)
2026-09-27 22:43 ` [PATCH v2 15/20] lib/crypto: riscv/aes: Copy aes-macros.S to library Eric Biggers
@ 2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 17/20] lib/crypto: riscv/aes-ecb: Migrate optimized code into library Eric Biggers
` (3 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:43 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers
Make the assembly code take the AES key struct directly, rather than the
round keys pointer and key length separately. Make the aes_begin macro
assume this convention, and remove support for the legacy
'struct crypto_aes_ctx' from it since that isn't used here.
This aligns with the convention that is being used (and will continue to
be used) for the AES modes, it makes the C glue code slightly simpler,
and it avoids the unnecessary shuffling around of arguments.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/riscv/aes-macros.S | 25 ++++++++++---------------
lib/crypto/riscv/aes-riscv64-zvkned.S | 17 ++++++++---------
lib/crypto/riscv/aes.h | 18 ++++++++++++------
3 files changed, 30 insertions(+), 30 deletions(-)
diff --git a/lib/crypto/riscv/aes-macros.S b/lib/crypto/riscv/aes-macros.S
index 1384164621a5..1ab18e358474 100644
--- a/lib/crypto/riscv/aes-macros.S
+++ b/lib/crypto/riscv/aes-macros.S
@@ -44,17 +44,20 @@
// - RISC-V Vector ('V') with VLEN >= 128
// - RISC-V Vector AES block cipher extension ('Zvkned')
-// Loads the AES round keys from \keyp into vector registers and jumps to code
-// specific to the length of the key. Specifically:
+// Offsets in struct aes_enckey
+#define OFFSETOF_KEYLEN 0
+#define OFFSETOF_RNDKEYS 16
+
+// Loads the AES round keys from the struct aes_enckey \keyp into vector
+// registers and jumps to code specific to the length of the key. Specifically:
// - If AES-128, loads round keys into v1-v11 and jumps to \label128.
// - If AES-192, loads round keys into v1-v13 and jumps to \label192.
// - If AES-256, loads round keys into v1-v15 and continues onwards.
//
-// Also sets vl=4 and vtype=e32,m1,ta,ma. Clobbers t0 and t1.
-.macro aes_begin keyp, label128, label192, key_len
-.ifb \key_len
- lwu t0, 480(\keyp) // t0 = key length in bytes
-.endif
+// Also sets vl=4 and vtype=e32,m1,ta,ma. Clobbers \keyp, t0, and t1.
+.macro aes_begin keyp, label128, label192
+ lwu t0, OFFSETOF_KEYLEN(\keyp) // t0 = key length in bytes
+ addi \keyp, \keyp, OFFSETOF_RNDKEYS
li t1, 24 // t1 = key length for AES-192
vsetivli zero, 4, e32, m1, ta, ma
vle32.v v1, (\keyp)
@@ -78,20 +81,12 @@
vle32.v v10, (\keyp)
addi \keyp, \keyp, 16
vle32.v v11, (\keyp)
-.ifb \key_len
blt t0, t1, \label128 // If AES-128, goto label128.
-.else
- blt \key_len, t1, \label128 // If AES-128, goto label128.
-.endif
addi \keyp, \keyp, 16
vle32.v v12, (\keyp)
addi \keyp, \keyp, 16
vle32.v v13, (\keyp)
-.ifb \key_len
beq t0, t1, \label192 // If AES-192, goto label192.
-.else
- beq \key_len, t1, \label192 // If AES-192, goto label192.
-.endif
// Else, it's AES-256.
addi \keyp, \keyp, 16
vle32.v v14, (\keyp)
diff --git a/lib/crypto/riscv/aes-riscv64-zvkned.S b/lib/crypto/riscv/aes-riscv64-zvkned.S
index 7a52ea6c669d..fb35f694b5ac 100644
--- a/lib/crypto/riscv/aes-riscv64-zvkned.S
+++ b/lib/crypto/riscv/aes-riscv64-zvkned.S
@@ -50,10 +50,9 @@
#include "aes-macros.S"
-#define RNDKEYS a0
-#define KEY_LEN a1
-#define OUTP a2
-#define INP a3
+#define KEYP a0
+#define OUTP a1
+#define INP a2
.macro __aes_crypt_zvkned enc, keybits
vle32.v v16, (INP)
@@ -63,7 +62,7 @@
.endm
.macro aes_crypt_zvkned enc
- aes_begin RNDKEYS, 128f, 192f, KEY_LEN
+ aes_begin KEYP, 128f, 192f
__aes_crypt_zvkned \enc, 256
128:
__aes_crypt_zvkned \enc, 128
@@ -71,14 +70,14 @@
__aes_crypt_zvkned \enc, 192
.endm
-// void aes_encrypt_zvkned(const u32 rndkeys[], int key_len,
-// u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]);
+// void aes_encrypt_zvkned(const struct aes_enckey *key, u8 out[AES_BLOCK_SIZE],
+// const u8 in[AES_BLOCK_SIZE]);
SYM_FUNC_START(aes_encrypt_zvkned)
aes_crypt_zvkned 1
SYM_FUNC_END(aes_encrypt_zvkned)
-// void aes_decrypt_zvkned(const u32 rndkeys[], int key_len,
-// u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]);
+// void aes_decrypt_zvkned(const struct aes_key *key, u8 out[AES_BLOCK_SIZE],
+// const u8 in[AES_BLOCK_SIZE]);
SYM_FUNC_START(aes_decrypt_zvkned)
aes_crypt_zvkned 0
SYM_FUNC_END(aes_decrypt_zvkned)
diff --git a/lib/crypto/riscv/aes.h b/lib/crypto/riscv/aes.h
index 0b26f58faf2b..9de9dbd1e887 100644
--- a/lib/crypto/riscv/aes.h
+++ b/lib/crypto/riscv/aes.h
@@ -10,10 +10,16 @@
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkned);
-void aes_encrypt_zvkned(const u32 rndkeys[], int key_len,
- u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]);
-void aes_decrypt_zvkned(const u32 rndkeys[], int key_len,
- u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]);
+/* The assembly code assumes the following offsets. */
+static_assert(offsetof(struct aes_enckey, len) == 0);
+static_assert(offsetof(struct aes_enckey, k.rndkeys) == 16);
+static_assert(offsetof(struct aes_key, len) == 0);
+static_assert(offsetof(struct aes_key, k.rndkeys) == 16);
+
+void aes_encrypt_zvkned(const struct aes_enckey *key, u8 out[AES_BLOCK_SIZE],
+ const u8 in[AES_BLOCK_SIZE]);
+void aes_decrypt_zvkned(const struct aes_key *key, u8 out[AES_BLOCK_SIZE],
+ const u8 in[AES_BLOCK_SIZE]);
static void aes_preparekey_arch(union aes_enckey_arch *k,
union aes_invkey_arch *inv_k,
@@ -29,7 +35,7 @@ static void aes_encrypt_arch(const struct aes_enckey *key,
{
if (static_branch_likely(&have_zvkned) && likely(may_use_simd())) {
kernel_vector_begin();
- aes_encrypt_zvkned(key->k.rndkeys, key->len, out, in);
+ aes_encrypt_zvkned(key, out, in);
kernel_vector_end();
} else {
aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in);
@@ -46,7 +52,7 @@ static void aes_decrypt_arch(const struct aes_key *key,
*/
if (static_branch_likely(&have_zvkned) && likely(may_use_simd())) {
kernel_vector_begin();
- aes_decrypt_zvkned(key->k.rndkeys, key->len, out, in);
+ aes_decrypt_zvkned(key, out, in);
kernel_vector_end();
} else {
aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds,
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread* [PATCH v2 17/20] lib/crypto: riscv/aes-ecb: Migrate optimized code into library
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
` (15 preceding siblings ...)
2026-09-27 22:43 ` [PATCH v2 16/20] lib/crypto: riscv/aes: Pass key struct to assembly code Eric Biggers
@ 2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 18/20] lib/crypto: riscv/aes-cbc: " Eric Biggers
` (2 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:43 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers
Instead of exposing the riscv-optimized AES-ECB code via a
riscv-specific crypto_skcipher algorithm, just implement the AES-ECB
library functions. This is simpler, it makes the AES-ECB library
functions be riscv-optimized, and it also fixes the longstanding issue
where the riscv-optimized AES-ECB code was disabled by default. AES-ECB
support still remains available through crypto_skcipher via
crypto/aes.c, but individual architectures no longer need to handle it.
To match what the library expects, update the assembly functions to
operate on struct aes_enckey or struct aes_key rather than struct
crypto_aes_ctx, and adjust the argument order.
Bump up the priority of the corresponding library-based algorithm on
riscv now that it no longer has to be lower than arch/riscv/crypto/.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/riscv/crypto/Kconfig | 4 +-
arch/riscv/crypto/aes-riscv64-glue.c | 61 +-------------------------
arch/riscv/crypto/aes-riscv64-zvkned.S | 39 ----------------
crypto/aes.c | 3 +-
lib/crypto/riscv/aes-riscv64-zvkned.S | 51 +++++++++++++++++++++
lib/crypto/riscv/aes.h | 33 ++++++++++++++
6 files changed, 89 insertions(+), 102 deletions(-)
diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig
index 6905232ddb03..84c41824b433 100644
--- a/arch/riscv/crypto/Kconfig
+++ b/arch/riscv/crypto/Kconfig
@@ -3,13 +3,13 @@
menu "Accelerated Cryptographic Algorithms for CPU (riscv)"
config CRYPTO_AES_RISCV64
- tristate "Ciphers: AES, modes: ECB, CBC, CTS, CTR, XTS"
+ tristate "Ciphers: AES, modes: CBC, CTS, CTR, XTS"
depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
select CRYPTO_LIB_AES
select CRYPTO_SKCIPHER
help
- Length-preserving ciphers: AES with ECB, CBC, CTS, CTR, XTS
+ Length-preserving ciphers: AES with CBC, CTS, CTR, XTS
Architecture: riscv64 using:
- Zvkned vector crypto extension
diff --git a/arch/riscv/crypto/aes-riscv64-glue.c b/arch/riscv/crypto/aes-riscv64-glue.c
index bbd920c9e29d..f7c492dcfd57 100644
--- a/arch/riscv/crypto/aes-riscv64-glue.c
+++ b/arch/riscv/crypto/aes-riscv64-glue.c
@@ -22,11 +22,6 @@
#include <linux/minmax.h>
#include <linux/module.h>
-asmlinkage void aes_ecb_encrypt_zvkned(const struct crypto_aes_ctx *key,
- const u8 *in, u8 *out, size_t len);
-asmlinkage void aes_ecb_decrypt_zvkned(const struct crypto_aes_ctx *key,
- const u8 *in, u8 *out, size_t len);
-
asmlinkage void aes_cbc_encrypt_zvkned(const struct crypto_aes_ctx *key,
const u8 *in, u8 *out, size_t len,
u8 iv[AES_BLOCK_SIZE]);
@@ -86,44 +81,6 @@ static int riscv64_aes_setkey_skcipher(struct crypto_skcipher *tfm,
return riscv64_aes_setkey(ctx, key, keylen);
}
-/* AES-ECB */
-
-static inline int riscv64_aes_ecb_crypt(struct skcipher_request *req, bool enc)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- const struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
- struct skcipher_walk walk;
- unsigned int nbytes;
- int err;
-
- err = skcipher_walk_virt(&walk, req, false);
- while ((nbytes = walk.nbytes) != 0) {
- kernel_vector_begin();
- if (enc)
- aes_ecb_encrypt_zvkned(ctx, walk.src.virt.addr,
- walk.dst.virt.addr,
- nbytes & ~(AES_BLOCK_SIZE - 1));
- else
- aes_ecb_decrypt_zvkned(ctx, walk.src.virt.addr,
- walk.dst.virt.addr,
- nbytes & ~(AES_BLOCK_SIZE - 1));
- kernel_vector_end();
- err = skcipher_walk_done(&walk, nbytes & (AES_BLOCK_SIZE - 1));
- }
-
- return err;
-}
-
-static int riscv64_aes_ecb_encrypt(struct skcipher_request *req)
-{
- return riscv64_aes_ecb_crypt(req, true);
-}
-
-static int riscv64_aes_ecb_decrypt(struct skcipher_request *req)
-{
- return riscv64_aes_ecb_crypt(req, false);
-}
-
/* AES-CBC */
static int riscv64_aes_cbc_crypt(struct skcipher_request *req, bool enc)
@@ -411,21 +368,6 @@ static int riscv64_aes_xts_decrypt(struct skcipher_request *req)
static struct skcipher_alg riscv64_zvkned_aes_skcipher_algs[] = {
{
- .setkey = riscv64_aes_setkey_skcipher,
- .encrypt = riscv64_aes_ecb_encrypt,
- .decrypt = riscv64_aes_ecb_decrypt,
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .walksize = 8 * AES_BLOCK_SIZE, /* matches LMUL=8 */
- .base = {
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct crypto_aes_ctx),
- .cra_priority = 300,
- .cra_name = "ecb(aes)",
- .cra_driver_name = "ecb-aes-riscv64-zvkned",
- .cra_module = THIS_MODULE,
- },
- }, {
.setkey = riscv64_aes_setkey_skcipher,
.encrypt = riscv64_aes_cbc_encrypt,
.decrypt = riscv64_aes_cbc_decrypt,
@@ -555,11 +497,10 @@ static void __exit riscv64_aes_mod_exit(void)
module_init(riscv64_aes_mod_init);
module_exit(riscv64_aes_mod_exit);
-MODULE_DESCRIPTION("AES-ECB/CBC/CTS/CTR/XTS (RISC-V accelerated)");
+MODULE_DESCRIPTION("AES-CBC/CTS/CTR/XTS (RISC-V accelerated)");
MODULE_AUTHOR("Jerry Shih <jerry.shih@sifive.com>");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("aes");
-MODULE_ALIAS_CRYPTO("ecb(aes)");
MODULE_ALIAS_CRYPTO("cbc(aes)");
MODULE_ALIAS_CRYPTO("cts(cbc(aes))");
MODULE_ALIAS_CRYPTO("ctr(aes)");
diff --git a/arch/riscv/crypto/aes-riscv64-zvkned.S b/arch/riscv/crypto/aes-riscv64-zvkned.S
index d0fc4581a380..00f8a06596d3 100644
--- a/arch/riscv/crypto/aes-riscv64-zvkned.S
+++ b/arch/riscv/crypto/aes-riscv64-zvkned.S
@@ -56,45 +56,6 @@
#define LEN a3
#define IVP a4
-.macro __aes_ecb_crypt enc, keylen
- srli t0, LEN, 2
- // t0 is the remaining length in 32-bit words. It's a multiple of 4.
-1:
- vsetvli t1, t0, e32, m8, ta, ma
- sub t0, t0, t1 // Subtract number of words processed
- slli t1, t1, 2 // Words to bytes
- vle32.v v16, (INP)
- aes_crypt v16, \enc, \keylen
- vse32.v v16, (OUTP)
- add INP, INP, t1
- add OUTP, OUTP, t1
- bnez t0, 1b
-
- ret
-.endm
-
-.macro aes_ecb_crypt enc
- aes_begin KEYP, 128f, 192f
- __aes_ecb_crypt \enc, 256
-128:
- __aes_ecb_crypt \enc, 128
-192:
- __aes_ecb_crypt \enc, 192
-.endm
-
-// void aes_ecb_encrypt_zvkned(const struct crypto_aes_ctx *key,
-// const u8 *in, u8 *out, size_t len);
-//
-// |len| must be nonzero and a multiple of 16 (AES_BLOCK_SIZE).
-SYM_FUNC_START(aes_ecb_encrypt_zvkned)
- aes_ecb_crypt 1
-SYM_FUNC_END(aes_ecb_encrypt_zvkned)
-
-// Same prototype and calling convention as the encryption function
-SYM_FUNC_START(aes_ecb_decrypt_zvkned)
- aes_ecb_crypt 0
-SYM_FUNC_END(aes_ecb_decrypt_zvkned)
-
.macro aes_cbc_encrypt keylen
vle32.v v16, (IVP) // Load IV
1:
diff --git a/crypto/aes.c b/crypto/aes.c
index 756fce2157be..9b9e5b413fc4 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -610,7 +610,8 @@ static struct skcipher_alg skcipher_algs[] = {
{
.base.cra_name = "ecb(aes)",
.base.cra_driver_name = "ecb-aes-lib",
- .base.cra_priority = IS_ENABLED(CONFIG_X86) ? 300 : 110,
+ .base.cra_priority = (IS_ENABLED(CONFIG_RISCV) ||
+ IS_ENABLED(CONFIG_X86)) ? 300 : 110,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct aes_key),
.base.cra_module = THIS_MODULE,
diff --git a/lib/crypto/riscv/aes-riscv64-zvkned.S b/lib/crypto/riscv/aes-riscv64-zvkned.S
index fb35f694b5ac..3700028e62ea 100644
--- a/lib/crypto/riscv/aes-riscv64-zvkned.S
+++ b/lib/crypto/riscv/aes-riscv64-zvkned.S
@@ -81,3 +81,54 @@ SYM_FUNC_END(aes_encrypt_zvkned)
SYM_FUNC_START(aes_decrypt_zvkned)
aes_crypt_zvkned 0
SYM_FUNC_END(aes_decrypt_zvkned)
+
+#undef KEYP
+#undef OUTP
+#undef INP
+
+#define DST a0
+#define SRC a1
+#define LEN a2
+#define KEYP a3
+
+.macro __aes_ecb_crypt enc, keylen
+ srli t0, LEN, 2
+ // t0 is the remaining length in 32-bit words. It's a multiple of 4.
+1:
+ vsetvli t1, t0, e32, m8, ta, ma
+ sub t0, t0, t1 // Subtract number of words processed
+ slli t1, t1, 2 // Words to bytes
+ vle32.v v16, (SRC)
+ aes_crypt v16, \enc, \keylen
+ vse32.v v16, (DST)
+ add SRC, SRC, t1
+ add DST, DST, t1
+ bnez t0, 1b
+
+ ret
+.endm
+
+.macro aes_ecb_crypt enc
+ aes_begin KEYP, 128f, 192f
+ __aes_ecb_crypt \enc, 256
+128:
+ __aes_ecb_crypt \enc, 128
+192:
+ __aes_ecb_crypt \enc, 192
+.endm
+
+// void aes_ecb_encrypt_zvkned(u8 *dst, const u8 *src, size_t len,
+// const struct aes_enckey *key);
+//
+// |len| must be nonzero and a multiple of 16 (AES_BLOCK_SIZE).
+SYM_FUNC_START(aes_ecb_encrypt_zvkned)
+ aes_ecb_crypt 1
+SYM_FUNC_END(aes_ecb_encrypt_zvkned)
+
+// void aes_ecb_decrypt_zvkned(u8 *dst, const u8 *src, size_t len,
+// const struct aes_key *key);
+//
+// |len| must be nonzero and a multiple of 16 (AES_BLOCK_SIZE).
+SYM_FUNC_START(aes_ecb_decrypt_zvkned)
+ aes_ecb_crypt 0
+SYM_FUNC_END(aes_ecb_decrypt_zvkned)
diff --git a/lib/crypto/riscv/aes.h b/lib/crypto/riscv/aes.h
index 9de9dbd1e887..5b36d0147d93 100644
--- a/lib/crypto/riscv/aes.h
+++ b/lib/crypto/riscv/aes.h
@@ -60,6 +60,39 @@ static void aes_decrypt_arch(const struct aes_key *key,
}
}
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB)
+void aes_ecb_encrypt_zvkned(u8 *dst, const u8 *src, size_t len,
+ const struct aes_enckey *key);
+void aes_ecb_decrypt_zvkned(u8 *dst, const u8 *src, size_t len,
+ const struct aes_key *key);
+
+/* len is always a positive multiple of AES_BLOCK_SIZE here. */
+#define aes_ecb_encrypt_arch aes_ecb_encrypt_arch
+static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len,
+ const struct aes_enckey *key)
+{
+ if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd()))
+ return false;
+ kernel_vector_begin();
+ aes_ecb_encrypt_zvkned(dst, src, len, key);
+ kernel_vector_end();
+ return true;
+}
+
+/* len is always a positive multiple of AES_BLOCK_SIZE here. */
+#define aes_ecb_decrypt_arch aes_ecb_decrypt_arch
+static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len,
+ const struct aes_key *key)
+{
+ if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd()))
+ return false;
+ kernel_vector_begin();
+ aes_ecb_decrypt_zvkned(dst, src, len, key);
+ kernel_vector_end();
+ return true;
+}
+#endif /* CONFIG_CRYPTO_LIB_AES_ECB */
+
#define aes_mod_init_arch aes_mod_init_arch
static void aes_mod_init_arch(void)
{
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread* [PATCH v2 18/20] lib/crypto: riscv/aes-cbc: Migrate optimized code into library
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
` (16 preceding siblings ...)
2026-09-27 22:43 ` [PATCH v2 17/20] lib/crypto: riscv/aes-ecb: Migrate optimized code into library Eric Biggers
@ 2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 19/20] lib/crypto: riscv/aes-ctr: " Eric Biggers
2026-09-27 22:43 ` [PATCH v2 20/20] lib/crypto: riscv/aes-xts: " Eric Biggers
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:43 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers
Instead of exposing the riscv-optimized AES-CBC and AES-CBC-CTS code via
a riscv-specific crypto_skcipher algorithm, just implement the AES-CBC
and AES-CBC-CTS library functions. This is simpler, it makes the
AES-CBC and AES-CBC-CTS library functions be riscv-optimized, and it
also fixes the longstanding issue where the riscv-optimized AES-CBC and
AES-CBC-CTS code was disabled by default. AES-CBC and AES-CBC-CTS
support still remains available through crypto_skcipher via
crypto/aes.c, but individual architectures no longer need to handle it.
To match what the library expects, update the assembly functions to
operate on struct aes_enckey or struct aes_key rather than struct
crypto_aes_ctx, and adjust the argument order.
Remove the single-block special case from the assembly code, since it's
already handled by lib/crypto/aes.c before calling aes_cbc_cts_*_arch().
Bump up the priority of the corresponding library-based algorithms on
riscv now that they no longer have to be lower than arch/riscv/crypto/.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/riscv/crypto/Kconfig | 4 +-
arch/riscv/crypto/Makefile | 2 +-
arch/riscv/crypto/aes-riscv64-glue.c | 167 +--------------
arch/riscv/crypto/aes-riscv64-zvkned.S | 273 -------------------------
crypto/aes.c | 6 +-
lib/crypto/riscv/aes-riscv64-zvkned.S | 235 +++++++++++++++++++++
lib/crypto/riscv/aes.h | 68 ++++++
7 files changed, 312 insertions(+), 443 deletions(-)
delete mode 100644 arch/riscv/crypto/aes-riscv64-zvkned.S
diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig
index 84c41824b433..0a3f87ad384e 100644
--- a/arch/riscv/crypto/Kconfig
+++ b/arch/riscv/crypto/Kconfig
@@ -3,13 +3,13 @@
menu "Accelerated Cryptographic Algorithms for CPU (riscv)"
config CRYPTO_AES_RISCV64
- tristate "Ciphers: AES, modes: CBC, CTS, CTR, XTS"
+ tristate "Ciphers: AES, modes: CTR, XTS"
depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
select CRYPTO_LIB_AES
select CRYPTO_SKCIPHER
help
- Length-preserving ciphers: AES with CBC, CTS, CTR, XTS
+ Length-preserving ciphers: AES with CTR, XTS
Architecture: riscv64 using:
- Zvkned vector crypto extension
diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile
index 8cf31db57fc4..d8b85afa6d0b 100644
--- a/arch/riscv/crypto/Makefile
+++ b/arch/riscv/crypto/Makefile
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-$(CONFIG_CRYPTO_AES_RISCV64) += aes-riscv64.o
-aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \
+aes-riscv64-y := aes-riscv64-glue.o \
aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o
obj-$(CONFIG_CRYPTO_SM4_RISCV64) += sm4-riscv64.o
diff --git a/arch/riscv/crypto/aes-riscv64-glue.c b/arch/riscv/crypto/aes-riscv64-glue.c
index f7c492dcfd57..97f5369d7e71 100644
--- a/arch/riscv/crypto/aes-riscv64-glue.c
+++ b/arch/riscv/crypto/aes-riscv64-glue.c
@@ -22,17 +22,6 @@
#include <linux/minmax.h>
#include <linux/module.h>
-asmlinkage void aes_cbc_encrypt_zvkned(const struct crypto_aes_ctx *key,
- const u8 *in, u8 *out, size_t len,
- u8 iv[AES_BLOCK_SIZE]);
-asmlinkage void aes_cbc_decrypt_zvkned(const struct crypto_aes_ctx *key,
- const u8 *in, u8 *out, size_t len,
- u8 iv[AES_BLOCK_SIZE]);
-
-asmlinkage void aes_cbc_cts_crypt_zvkned(const struct crypto_aes_ctx *key,
- const u8 *in, u8 *out, size_t len,
- const u8 iv[AES_BLOCK_SIZE], bool enc);
-
asmlinkage void aes_ctr32_crypt_zvkned_zvkb(const struct crypto_aes_ctx *key,
const u8 *in, u8 *out, size_t len,
u8 iv[AES_BLOCK_SIZE]);
@@ -81,110 +70,6 @@ static int riscv64_aes_setkey_skcipher(struct crypto_skcipher *tfm,
return riscv64_aes_setkey(ctx, key, keylen);
}
-/* AES-CBC */
-
-static int riscv64_aes_cbc_crypt(struct skcipher_request *req, bool enc)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- const struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
- struct skcipher_walk walk;
- unsigned int nbytes;
- int err;
-
- err = skcipher_walk_virt(&walk, req, false);
- while ((nbytes = walk.nbytes) != 0) {
- kernel_vector_begin();
- if (enc)
- aes_cbc_encrypt_zvkned(ctx, walk.src.virt.addr,
- walk.dst.virt.addr,
- nbytes & ~(AES_BLOCK_SIZE - 1),
- walk.iv);
- else
- aes_cbc_decrypt_zvkned(ctx, walk.src.virt.addr,
- walk.dst.virt.addr,
- nbytes & ~(AES_BLOCK_SIZE - 1),
- walk.iv);
- kernel_vector_end();
- err = skcipher_walk_done(&walk, nbytes & (AES_BLOCK_SIZE - 1));
- }
-
- return err;
-}
-
-static int riscv64_aes_cbc_encrypt(struct skcipher_request *req)
-{
- return riscv64_aes_cbc_crypt(req, true);
-}
-
-static int riscv64_aes_cbc_decrypt(struct skcipher_request *req)
-{
- return riscv64_aes_cbc_crypt(req, false);
-}
-
-/* AES-CBC-CTS */
-
-static int riscv64_aes_cbc_cts_crypt(struct skcipher_request *req, bool enc)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- const struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
- struct scatterlist sg_src[2], sg_dst[2];
- struct skcipher_request subreq;
- struct scatterlist *src, *dst;
- struct skcipher_walk walk;
- unsigned int cbc_len;
- int err;
-
- if (req->cryptlen < AES_BLOCK_SIZE)
- return -EINVAL;
-
- err = skcipher_walk_virt(&walk, req, false);
- if (err)
- return err;
- /*
- * If the full message is available in one step, decrypt it in one call
- * to the CBC-CTS assembly function. This reduces overhead, especially
- * on short messages. Otherwise, fall back to doing CBC up to the last
- * two blocks, then invoke CTS just for the ciphertext stealing.
- */
- if (unlikely(walk.nbytes != req->cryptlen)) {
- cbc_len = round_down(req->cryptlen - AES_BLOCK_SIZE - 1,
- AES_BLOCK_SIZE);
- skcipher_walk_abort(&walk);
- skcipher_request_set_tfm(&subreq, tfm);
- skcipher_request_set_callback(&subreq,
- skcipher_request_flags(req),
- NULL, NULL);
- skcipher_request_set_crypt(&subreq, req->src, req->dst,
- cbc_len, req->iv);
- err = riscv64_aes_cbc_crypt(&subreq, enc);
- if (err)
- return err;
- dst = src = scatterwalk_ffwd(sg_src, req->src, cbc_len);
- if (req->dst != req->src)
- dst = scatterwalk_ffwd(sg_dst, req->dst, cbc_len);
- skcipher_request_set_crypt(&subreq, src, dst,
- req->cryptlen - cbc_len, req->iv);
- err = skcipher_walk_virt(&walk, &subreq, false);
- if (err)
- return err;
- }
- kernel_vector_begin();
- aes_cbc_cts_crypt_zvkned(ctx, walk.src.virt.addr, walk.dst.virt.addr,
- walk.nbytes, req->iv, enc);
- kernel_vector_end();
- return skcipher_walk_done(&walk, 0);
-}
-
-static int riscv64_aes_cbc_cts_encrypt(struct skcipher_request *req)
-{
- return riscv64_aes_cbc_cts_crypt(req, true);
-}
-
-static int riscv64_aes_cbc_cts_decrypt(struct skcipher_request *req)
-{
- return riscv64_aes_cbc_cts_crypt(req, false);
-}
-
/* AES-CTR */
static int riscv64_aes_ctr_crypt(struct skcipher_request *req)
@@ -366,41 +251,6 @@ static int riscv64_aes_xts_decrypt(struct skcipher_request *req)
/* Algorithm definitions */
-static struct skcipher_alg riscv64_zvkned_aes_skcipher_algs[] = {
- {
- .setkey = riscv64_aes_setkey_skcipher,
- .encrypt = riscv64_aes_cbc_encrypt,
- .decrypt = riscv64_aes_cbc_decrypt,
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .base = {
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct crypto_aes_ctx),
- .cra_priority = 300,
- .cra_name = "cbc(aes)",
- .cra_driver_name = "cbc-aes-riscv64-zvkned",
- .cra_module = THIS_MODULE,
- },
- }, {
- .setkey = riscv64_aes_setkey_skcipher,
- .encrypt = riscv64_aes_cbc_cts_encrypt,
- .decrypt = riscv64_aes_cbc_cts_decrypt,
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .walksize = 4 * AES_BLOCK_SIZE, /* matches LMUL=4 */
- .base = {
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct crypto_aes_ctx),
- .cra_priority = 300,
- .cra_name = "cts(cbc(aes))",
- .cra_driver_name = "cts-cbc-aes-riscv64-zvkned",
- .cra_module = THIS_MODULE,
- },
- }
-};
-
static struct skcipher_alg riscv64_zvkned_zvkb_aes_skcipher_alg = {
.setkey = riscv64_aes_setkey_skcipher,
.encrypt = riscv64_aes_ctr_crypt,
@@ -452,17 +302,11 @@ static int __init riscv64_aes_mod_init(void)
if (riscv_isa_extension_available(NULL, ZVKNED) &&
riscv_vector_vlen() >= 128) {
- err = crypto_register_skciphers(
- riscv64_zvkned_aes_skcipher_algs,
- ARRAY_SIZE(riscv64_zvkned_aes_skcipher_algs));
- if (err)
- return err;
-
if (riscv_isa_extension_available(NULL, ZVKB)) {
err = crypto_register_skcipher(
&riscv64_zvkned_zvkb_aes_skcipher_alg);
if (err)
- goto unregister_zvkned_skcipher_algs;
+ return err;
}
if (riscv64_aes_xts_supported()) {
@@ -478,9 +322,6 @@ static int __init riscv64_aes_mod_init(void)
unregister_zvkned_zvkb_skcipher_alg:
if (riscv_isa_extension_available(NULL, ZVKB))
crypto_unregister_skcipher(&riscv64_zvkned_zvkb_aes_skcipher_alg);
-unregister_zvkned_skcipher_algs:
- crypto_unregister_skciphers(riscv64_zvkned_aes_skcipher_algs,
- ARRAY_SIZE(riscv64_zvkned_aes_skcipher_algs));
return err;
}
@@ -490,18 +331,14 @@ static void __exit riscv64_aes_mod_exit(void)
crypto_unregister_skcipher(&riscv64_zvkned_zvbb_zvkg_aes_skcipher_alg);
if (riscv_isa_extension_available(NULL, ZVKB))
crypto_unregister_skcipher(&riscv64_zvkned_zvkb_aes_skcipher_alg);
- crypto_unregister_skciphers(riscv64_zvkned_aes_skcipher_algs,
- ARRAY_SIZE(riscv64_zvkned_aes_skcipher_algs));
}
module_init(riscv64_aes_mod_init);
module_exit(riscv64_aes_mod_exit);
-MODULE_DESCRIPTION("AES-CBC/CTS/CTR/XTS (RISC-V accelerated)");
+MODULE_DESCRIPTION("AES-CTR/XTS (RISC-V accelerated)");
MODULE_AUTHOR("Jerry Shih <jerry.shih@sifive.com>");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("aes");
-MODULE_ALIAS_CRYPTO("cbc(aes)");
-MODULE_ALIAS_CRYPTO("cts(cbc(aes))");
MODULE_ALIAS_CRYPTO("ctr(aes)");
MODULE_ALIAS_CRYPTO("xts(aes)");
diff --git a/arch/riscv/crypto/aes-riscv64-zvkned.S b/arch/riscv/crypto/aes-riscv64-zvkned.S
deleted file mode 100644
index 00f8a06596d3..000000000000
--- a/arch/riscv/crypto/aes-riscv64-zvkned.S
+++ /dev/null
@@ -1,273 +0,0 @@
-/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */
-//
-// This file is dual-licensed, meaning that you can use it under your
-// choice of either of the following two licenses:
-//
-// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
-//
-// Licensed under the Apache License 2.0 (the "License"). You can obtain
-// a copy in the file LICENSE in the source distribution or at
-// https://www.openssl.org/source/license.html
-//
-// or
-//
-// Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu>
-// Copyright (c) 2023, Phoebe Chen <phoebe.chen@sifive.com>
-// Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com>
-// Copyright 2024 Google LLC
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions
-// are met:
-// 1. Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// 2. Redistributions in binary form must reproduce the above copyright
-// notice, this list of conditions and the following disclaimer in the
-// documentation and/or other materials provided with the distribution.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-// The generated code of this file depends on the following RISC-V extensions:
-// - RV64I
-// - RISC-V Vector ('V') with VLEN >= 128
-// - RISC-V Vector AES block cipher extension ('Zvkned')
-
-#include <linux/linkage.h>
-
-.text
-.option arch, +zvkned
-
-#include "aes-macros.S"
-
-#define KEYP a0
-#define INP a1
-#define OUTP a2
-#define LEN a3
-#define IVP a4
-
-.macro aes_cbc_encrypt keylen
- vle32.v v16, (IVP) // Load IV
-1:
- vle32.v v17, (INP) // Load plaintext block
- vxor.vv v16, v16, v17 // XOR with IV or prev ciphertext block
- aes_encrypt v16, \keylen // Encrypt
- vse32.v v16, (OUTP) // Store ciphertext block
- addi INP, INP, 16
- addi OUTP, OUTP, 16
- addi LEN, LEN, -16
- bnez LEN, 1b
-
- vse32.v v16, (IVP) // Store next IV
- ret
-.endm
-
-.macro aes_cbc_decrypt keylen
- srli LEN, LEN, 2 // Convert LEN from bytes to words
- vle32.v v16, (IVP) // Load IV
-1:
- vsetvli t0, LEN, e32, m4, ta, ma
- vle32.v v20, (INP) // Load ciphertext blocks
- vslideup.vi v16, v20, 4 // Setup prev ciphertext blocks
- addi t1, t0, -4
- vslidedown.vx v24, v20, t1 // Save last ciphertext block
- aes_decrypt v20, \keylen // Decrypt the blocks
- vxor.vv v20, v20, v16 // XOR with prev ciphertext blocks
- vse32.v v20, (OUTP) // Store plaintext blocks
- vmv.v.v v16, v24 // Next "IV" is last ciphertext block
- slli t1, t0, 2 // Words to bytes
- add INP, INP, t1
- add OUTP, OUTP, t1
- sub LEN, LEN, t0
- bnez LEN, 1b
-
- vsetivli zero, 4, e32, m1, ta, ma
- vse32.v v16, (IVP) // Store next IV
- ret
-.endm
-
-// void aes_cbc_encrypt_zvkned(const struct crypto_aes_ctx *key,
-// const u8 *in, u8 *out, size_t len, u8 iv[16]);
-//
-// |len| must be nonzero and a multiple of 16 (AES_BLOCK_SIZE).
-SYM_FUNC_START(aes_cbc_encrypt_zvkned)
- aes_begin KEYP, 128f, 192f
- aes_cbc_encrypt 256
-128:
- aes_cbc_encrypt 128
-192:
- aes_cbc_encrypt 192
-SYM_FUNC_END(aes_cbc_encrypt_zvkned)
-
-// Same prototype and calling convention as the encryption function
-SYM_FUNC_START(aes_cbc_decrypt_zvkned)
- aes_begin KEYP, 128f, 192f
- aes_cbc_decrypt 256
-128:
- aes_cbc_decrypt 128
-192:
- aes_cbc_decrypt 192
-SYM_FUNC_END(aes_cbc_decrypt_zvkned)
-
-.macro aes_cbc_cts_encrypt keylen
-
- // CBC-encrypt all blocks except the last. But don't store the
- // second-to-last block to the output buffer yet, since it will be
- // handled specially in the ciphertext stealing step. Exception: if the
- // message is single-block, still encrypt the last (and only) block.
- li t0, 16
- j 2f
-1:
- vse32.v v16, (OUTP) // Store ciphertext block
- addi OUTP, OUTP, 16
-2:
- vle32.v v17, (INP) // Load plaintext block
- vxor.vv v16, v16, v17 // XOR with IV or prev ciphertext block
- aes_encrypt v16, \keylen // Encrypt
- addi INP, INP, 16
- addi LEN, LEN, -16
- bgt LEN, t0, 1b // Repeat if more than one block remains
-
- // Special case: if the message is a single block, just do CBC.
- beqz LEN, .Lcts_encrypt_done\@
-
- // Encrypt the last two blocks using ciphertext stealing as follows:
- // C[n-1] = Encrypt(Encrypt(P[n-1] ^ C[n-2]) ^ P[n])
- // C[n] = Encrypt(P[n-1] ^ C[n-2])[0..LEN]
- //
- // C[i] denotes the i'th ciphertext block, and likewise P[i] the i'th
- // plaintext block. Block n, the last block, may be partial; its length
- // is 1 <= LEN <= 16. If there are only 2 blocks, C[n-2] means the IV.
- //
- // v16 already contains Encrypt(P[n-1] ^ C[n-2]).
- // INP points to P[n]. OUTP points to where C[n-1] should go.
- // To support in-place encryption, load P[n] before storing C[n].
- addi t0, OUTP, 16 // Get pointer to where C[n] should go
- vsetvli zero, LEN, e8, m1, tu, ma
- vle8.v v17, (INP) // Load P[n]
- vse8.v v16, (t0) // Store C[n]
- vxor.vv v16, v16, v17 // v16 = Encrypt(P[n-1] ^ C[n-2]) ^ P[n]
- vsetivli zero, 4, e32, m1, ta, ma
- aes_encrypt v16, \keylen
-.Lcts_encrypt_done\@:
- vse32.v v16, (OUTP) // Store C[n-1] (or C[n] in single-block case)
- ret
-.endm
-
-#define LEN32 t4 // Length of remaining full blocks in 32-bit words
-#define LEN_MOD16 t5 // Length of message in bytes mod 16
-
-.macro aes_cbc_cts_decrypt keylen
- andi LEN32, LEN, ~15
- srli LEN32, LEN32, 2
- andi LEN_MOD16, LEN, 15
-
- // Save C[n-2] in v28 so that it's available later during the ciphertext
- // stealing step. If there are fewer than three blocks, C[n-2] means
- // the IV, otherwise it means the third-to-last ciphertext block.
- vmv.v.v v28, v16 // IV
- add t0, LEN, -33
- bltz t0, .Lcts_decrypt_loop\@
- andi t0, t0, ~15
- add t0, t0, INP
- vle32.v v28, (t0)
-
- // CBC-decrypt all full blocks. For the last full block, or the last 2
- // full blocks if the message is block-aligned, this doesn't write the
- // correct output blocks (unless the message is only a single block),
- // because it XORs the wrong values with the raw AES plaintexts. But we
- // fix this after this loop without redoing the AES decryptions. This
- // approach allows more of the AES decryptions to be parallelized.
-.Lcts_decrypt_loop\@:
- vsetvli t0, LEN32, e32, m4, ta, ma
- addi t1, t0, -4
- vle32.v v20, (INP) // Load next set of ciphertext blocks
- vmv.v.v v24, v16 // Get IV or last ciphertext block of prev set
- vslideup.vi v24, v20, 4 // Setup prev ciphertext blocks
- vslidedown.vx v16, v20, t1 // Save last ciphertext block of this set
- aes_decrypt v20, \keylen // Decrypt this set of blocks
- vxor.vv v24, v24, v20 // XOR prev ciphertext blocks with decrypted blocks
- vse32.v v24, (OUTP) // Store this set of plaintext blocks
- sub LEN32, LEN32, t0
- slli t0, t0, 2 // Words to bytes
- add INP, INP, t0
- add OUTP, OUTP, t0
- bnez LEN32, .Lcts_decrypt_loop\@
-
- vsetivli zero, 4, e32, m4, ta, ma
- vslidedown.vx v20, v20, t1 // Extract raw plaintext of last full block
- addi t0, OUTP, -16 // Get pointer to last full plaintext block
- bnez LEN_MOD16, .Lcts_decrypt_non_block_aligned\@
-
- // Special case: if the message is a single block, just do CBC.
- li t1, 16
- beq LEN, t1, .Lcts_decrypt_done\@
-
- // Block-aligned message. Just fix up the last 2 blocks. We need:
- //
- // P[n-1] = Decrypt(C[n]) ^ C[n-2]
- // P[n] = Decrypt(C[n-1]) ^ C[n]
- //
- // We have C[n] in v16, Decrypt(C[n]) in v20, and C[n-2] in v28.
- // Together with Decrypt(C[n-1]) ^ C[n-2] from the output buffer, this
- // is everything needed to fix the output without re-decrypting blocks.
- addi t1, OUTP, -32 // Get pointer to where P[n-1] should go
- vxor.vv v20, v20, v28 // Decrypt(C[n]) ^ C[n-2] == P[n-1]
- vle32.v v24, (t1) // Decrypt(C[n-1]) ^ C[n-2]
- vse32.v v20, (t1) // Store P[n-1]
- vxor.vv v20, v24, v16 // Decrypt(C[n-1]) ^ C[n-2] ^ C[n] == P[n] ^ C[n-2]
- j .Lcts_decrypt_finish\@
-
-.Lcts_decrypt_non_block_aligned\@:
- // Decrypt the last two blocks using ciphertext stealing as follows:
- //
- // P[n-1] = Decrypt(C[n] || Decrypt(C[n-1])[LEN_MOD16..16]) ^ C[n-2]
- // P[n] = (Decrypt(C[n-1]) ^ C[n])[0..LEN_MOD16]
- //
- // We already have Decrypt(C[n-1]) in v20 and C[n-2] in v28.
- vmv.v.v v16, v20 // v16 = Decrypt(C[n-1])
- vsetvli zero, LEN_MOD16, e8, m1, tu, ma
- vle8.v v20, (INP) // v20 = C[n] || Decrypt(C[n-1])[LEN_MOD16..16]
- vxor.vv v16, v16, v20 // v16 = Decrypt(C[n-1]) ^ C[n]
- vse8.v v16, (OUTP) // Store P[n]
- vsetivli zero, 4, e32, m1, ta, ma
- aes_decrypt v20, \keylen // v20 = Decrypt(C[n] || Decrypt(C[n-1])[LEN_MOD16..16])
-.Lcts_decrypt_finish\@:
- vxor.vv v20, v20, v28 // XOR with C[n-2]
- vse32.v v20, (t0) // Store last full plaintext block
-.Lcts_decrypt_done\@:
- ret
-.endm
-
-.macro aes_cbc_cts_crypt keylen
- vle32.v v16, (IVP) // Load IV
- beqz a5, .Lcts_decrypt\@
- aes_cbc_cts_encrypt \keylen
-.Lcts_decrypt\@:
- aes_cbc_cts_decrypt \keylen
-.endm
-
-// void aes_cbc_cts_crypt_zvkned(const struct crypto_aes_ctx *key,
-// const u8 *in, u8 *out, size_t len,
-// const u8 iv[16], bool enc);
-//
-// Encrypts or decrypts a message with the CS3 variant of AES-CBC-CTS.
-// This is the variant that unconditionally swaps the last two blocks.
-SYM_FUNC_START(aes_cbc_cts_crypt_zvkned)
- aes_begin KEYP, 128f, 192f
- aes_cbc_cts_crypt 256
-128:
- aes_cbc_cts_crypt 128
-192:
- aes_cbc_cts_crypt 192
-SYM_FUNC_END(aes_cbc_cts_crypt_zvkned)
diff --git a/crypto/aes.c b/crypto/aes.c
index 9b9e5b413fc4..1daaa59d0cdf 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -626,7 +626,8 @@ static struct skcipher_alg skcipher_algs[] = {
{
.base.cra_name = "cbc(aes)",
.base.cra_driver_name = "cbc-aes-lib",
- .base.cra_priority = IS_ENABLED(CONFIG_X86) ? 300 : 110,
+ .base.cra_priority = (IS_ENABLED(CONFIG_RISCV) ||
+ IS_ENABLED(CONFIG_X86)) ? 300 : 110,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct aes_key),
.base.cra_module = THIS_MODULE,
@@ -652,7 +653,8 @@ static struct skcipher_alg skcipher_algs[] = {
{
.base.cra_name = "cts(cbc(aes))",
.base.cra_driver_name = "cts-cbc-aes-lib",
- .base.cra_priority = IS_ENABLED(CONFIG_X86) ? 300 : 110,
+ .base.cra_priority = (IS_ENABLED(CONFIG_RISCV) ||
+ IS_ENABLED(CONFIG_X86)) ? 300 : 110,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct aes_key),
.base.cra_module = THIS_MODULE,
diff --git a/lib/crypto/riscv/aes-riscv64-zvkned.S b/lib/crypto/riscv/aes-riscv64-zvkned.S
index 3700028e62ea..61c1f185c33d 100644
--- a/lib/crypto/riscv/aes-riscv64-zvkned.S
+++ b/lib/crypto/riscv/aes-riscv64-zvkned.S
@@ -132,3 +132,238 @@ SYM_FUNC_END(aes_ecb_encrypt_zvkned)
SYM_FUNC_START(aes_ecb_decrypt_zvkned)
aes_ecb_crypt 0
SYM_FUNC_END(aes_ecb_decrypt_zvkned)
+
+#undef DST
+#undef SRC
+#undef LEN
+#undef KEYP
+
+#define DST a0
+#define SRC a1
+#define LEN a2
+#define IVP a3
+#define KEYP a4
+
+.macro aes_cbc_encrypt keylen
+ vle32.v v16, (IVP) // Load IV
+1:
+ vle32.v v17, (SRC) // Load plaintext block
+ vxor.vv v16, v16, v17 // XOR with IV or prev ciphertext block
+ aes_encrypt v16, \keylen // Encrypt
+ vse32.v v16, (DST) // Store ciphertext block
+ addi SRC, SRC, 16
+ addi DST, DST, 16
+ addi LEN, LEN, -16
+ bnez LEN, 1b
+
+ vse32.v v16, (IVP) // Store next IV
+ ret
+.endm
+
+.macro aes_cbc_decrypt keylen
+ srli LEN, LEN, 2 // Convert LEN from bytes to words
+ vle32.v v16, (IVP) // Load IV
+1:
+ vsetvli t0, LEN, e32, m4, ta, ma
+ vle32.v v20, (SRC) // Load ciphertext blocks
+ vslideup.vi v16, v20, 4 // Setup prev ciphertext blocks
+ addi t1, t0, -4
+ vslidedown.vx v24, v20, t1 // Save last ciphertext block
+ aes_decrypt v20, \keylen // Decrypt the blocks
+ vxor.vv v20, v20, v16 // XOR with prev ciphertext blocks
+ vse32.v v20, (DST) // Store plaintext blocks
+ vmv.v.v v16, v24 // Next "IV" is last ciphertext block
+ slli t1, t0, 2 // Words to bytes
+ add SRC, SRC, t1
+ add DST, DST, t1
+ sub LEN, LEN, t0
+ bnez LEN, 1b
+
+ vsetivli zero, 4, e32, m1, ta, ma
+ vse32.v v16, (IVP) // Store next IV
+ ret
+.endm
+
+// void aes_cbc_encrypt_zvkned(u8 *dst, const u8 *src, size_t len,
+// u8 iv[AES_BLOCK_SIZE],
+// const struct aes_enckey *key);
+//
+// |len| must be nonzero and a multiple of 16 (AES_BLOCK_SIZE).
+SYM_FUNC_START(aes_cbc_encrypt_zvkned)
+ aes_begin KEYP, 128f, 192f
+ aes_cbc_encrypt 256
+128:
+ aes_cbc_encrypt 128
+192:
+ aes_cbc_encrypt 192
+SYM_FUNC_END(aes_cbc_encrypt_zvkned)
+
+// void aes_cbc_decrypt_zvkned(u8 *dst, const u8 *src, size_t len,
+// u8 iv[AES_BLOCK_SIZE],
+// const struct aes_key *key);
+//
+// |len| must be nonzero and a multiple of 16 (AES_BLOCK_SIZE).
+SYM_FUNC_START(aes_cbc_decrypt_zvkned)
+ aes_begin KEYP, 128f, 192f
+ aes_cbc_decrypt 256
+128:
+ aes_cbc_decrypt 128
+192:
+ aes_cbc_decrypt 192
+SYM_FUNC_END(aes_cbc_decrypt_zvkned)
+
+#undef DST
+#undef SRC
+#undef LEN
+#undef IVP
+#undef KEYP
+
+#define DST a0
+#define SRC a1
+#define LEN a2
+#define IVP a3
+#define KEYP a4
+#define ENC a5
+
+.macro aes_cbc_cts_encrypt keylen
+
+ // CBC-encrypt all blocks except the last. But don't store the
+ // second-to-last block to the output buffer yet, since it will be
+ // handled specially in the ciphertext stealing step.
+ li t0, 16
+ j 2f
+1:
+ vse32.v v16, (DST) // Store ciphertext block
+ addi DST, DST, 16
+2:
+ vle32.v v17, (SRC) // Load plaintext block
+ vxor.vv v16, v16, v17 // XOR with IV or prev ciphertext block
+ aes_encrypt v16, \keylen // Encrypt
+ addi SRC, SRC, 16
+ addi LEN, LEN, -16
+ bgt LEN, t0, 1b // Repeat if more than one block remains
+
+ // Encrypt the last two blocks using ciphertext stealing as follows:
+ // C[n-1] = Encrypt(Encrypt(P[n-1] ^ C[n-2]) ^ P[n])
+ // C[n] = Encrypt(P[n-1] ^ C[n-2])[0..LEN]
+ //
+ // C[i] denotes the i'th ciphertext block, and likewise P[i] the i'th
+ // plaintext block. Block n, the last block, may be partial; its length
+ // is 1 <= LEN <= 16. If there are only 2 blocks, C[n-2] means the IV.
+ //
+ // v16 already contains Encrypt(P[n-1] ^ C[n-2]).
+ // SRC points to P[n]. DST points to where C[n-1] should go.
+ // To support in-place encryption, load P[n] before storing C[n].
+ addi t0, DST, 16 // Get pointer to where C[n] should go
+ vsetvli zero, LEN, e8, m1, tu, ma
+ vle8.v v17, (SRC) // Load P[n]
+ vse8.v v16, (t0) // Store C[n]
+ vxor.vv v16, v16, v17 // v16 = Encrypt(P[n-1] ^ C[n-2]) ^ P[n]
+ vsetivli zero, 4, e32, m1, ta, ma
+ aes_encrypt v16, \keylen
+ vse32.v v16, (DST) // Store C[n-1]
+ ret
+.endm
+
+#define LEN32 t4 // Length of remaining full blocks in 32-bit words
+#define LEN_MOD16 t5 // Length of message in bytes mod 16
+
+.macro aes_cbc_cts_decrypt keylen
+ andi LEN32, LEN, ~15
+ srli LEN32, LEN32, 2
+ andi LEN_MOD16, LEN, 15
+
+ // Save C[n-2] in v28 so that it's available later during the ciphertext
+ // stealing step. If there are fewer than three blocks, C[n-2] means
+ // the IV, otherwise it means the third-to-last ciphertext block.
+ vmv.v.v v28, v16 // IV
+ add t0, LEN, -33
+ bltz t0, .Lcts_decrypt_loop\@
+ andi t0, t0, ~15
+ add t0, t0, SRC
+ vle32.v v28, (t0)
+
+ // CBC-decrypt all full blocks. For the last full block, or the last 2
+ // full blocks if the message is block-aligned, this doesn't write the
+ // correct output blocks, because it XORs the wrong values with the raw
+ // AES plaintexts. But we fix this after this loop without redoing the
+ // AES decryptions. This approach allows more of the AES decryptions to
+ // be parallelized.
+.Lcts_decrypt_loop\@:
+ vsetvli t0, LEN32, e32, m4, ta, ma
+ addi t1, t0, -4
+ vle32.v v20, (SRC) // Load next set of ciphertext blocks
+ vmv.v.v v24, v16 // Get IV or last ciphertext block of prev set
+ vslideup.vi v24, v20, 4 // Setup prev ciphertext blocks
+ vslidedown.vx v16, v20, t1 // Save last ciphertext block of this set
+ aes_decrypt v20, \keylen // Decrypt this set of blocks
+ vxor.vv v24, v24, v20 // XOR prev ciphertext blocks with decrypted blocks
+ vse32.v v24, (DST) // Store this set of plaintext blocks
+ sub LEN32, LEN32, t0
+ slli t0, t0, 2 // Words to bytes
+ add SRC, SRC, t0
+ add DST, DST, t0
+ bnez LEN32, .Lcts_decrypt_loop\@
+
+ vsetivli zero, 4, e32, m4, ta, ma
+ vslidedown.vx v20, v20, t1 // Extract raw plaintext of last full block
+ addi t0, DST, -16 // Get pointer to last full plaintext block
+ bnez LEN_MOD16, .Lcts_decrypt_non_block_aligned\@
+
+ // Block-aligned message. Just fix up the last 2 blocks. We need:
+ //
+ // P[n-1] = Decrypt(C[n]) ^ C[n-2]
+ // P[n] = Decrypt(C[n-1]) ^ C[n]
+ //
+ // We have C[n] in v16, Decrypt(C[n]) in v20, and C[n-2] in v28.
+ // Together with Decrypt(C[n-1]) ^ C[n-2] from the output buffer, this
+ // is everything needed to fix the output without re-decrypting blocks.
+ addi t1, DST, -32 // Get pointer to where P[n-1] should go
+ vxor.vv v20, v20, v28 // Decrypt(C[n]) ^ C[n-2] == P[n-1]
+ vle32.v v24, (t1) // Decrypt(C[n-1]) ^ C[n-2]
+ vse32.v v20, (t1) // Store P[n-1]
+ vxor.vv v20, v24, v16 // Decrypt(C[n-1]) ^ C[n-2] ^ C[n] == P[n] ^ C[n-2]
+ j .Lcts_decrypt_finish\@
+
+.Lcts_decrypt_non_block_aligned\@:
+ // Decrypt the last two blocks using ciphertext stealing as follows:
+ //
+ // P[n-1] = Decrypt(C[n] || Decrypt(C[n-1])[LEN_MOD16..16]) ^ C[n-2]
+ // P[n] = (Decrypt(C[n-1]) ^ C[n])[0..LEN_MOD16]
+ //
+ // We already have Decrypt(C[n-1]) in v20 and C[n-2] in v28.
+ vmv.v.v v16, v20 // v16 = Decrypt(C[n-1])
+ vsetvli zero, LEN_MOD16, e8, m1, tu, ma
+ vle8.v v20, (SRC) // v20 = C[n] || Decrypt(C[n-1])[LEN_MOD16..16]
+ vxor.vv v16, v16, v20 // v16 = Decrypt(C[n-1]) ^ C[n]
+ vse8.v v16, (DST) // Store P[n]
+ vsetivli zero, 4, e32, m1, ta, ma
+ aes_decrypt v20, \keylen // v20 = Decrypt(C[n] || Decrypt(C[n-1])[LEN_MOD16..16])
+.Lcts_decrypt_finish\@:
+ vxor.vv v20, v20, v28 // XOR with C[n-2]
+ vse32.v v20, (t0) // Store last full plaintext block
+ ret
+.endm
+
+.macro aes_cbc_cts_crypt keylen
+ vle32.v v16, (IVP) // Load IV
+ beqz ENC, .Lcts_decrypt\@
+ aes_cbc_cts_encrypt \keylen
+.Lcts_decrypt\@:
+ aes_cbc_cts_decrypt \keylen
+.endm
+
+// void aes_cbc_cts_crypt_zvkned(u8 *dst, const u8 *src, size_t len,
+// const u8 iv[AES_BLOCK_SIZE],
+// aes_encrypt_arg key, bool enc);
+//
+// Encrypts or decrypts len >= 17 bytes of data using the CS3 variant of
+// AES-CBC-CTS. This is the variant that always swaps the last two blocks.
+SYM_FUNC_START(aes_cbc_cts_crypt_zvkned)
+ aes_begin KEYP, 128f, 192f
+ aes_cbc_cts_crypt 256
+128:
+ aes_cbc_cts_crypt 128
+192:
+ aes_cbc_cts_crypt 192
+SYM_FUNC_END(aes_cbc_cts_crypt_zvkned)
diff --git a/lib/crypto/riscv/aes.h b/lib/crypto/riscv/aes.h
index 5b36d0147d93..150cda82cb89 100644
--- a/lib/crypto/riscv/aes.h
+++ b/lib/crypto/riscv/aes.h
@@ -93,6 +93,74 @@ static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len,
}
#endif /* CONFIG_CRYPTO_LIB_AES_ECB */
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC)
+void aes_cbc_encrypt_zvkned(u8 *dst, const u8 *src, size_t len,
+ u8 iv[AES_BLOCK_SIZE], const struct aes_enckey *key);
+void aes_cbc_decrypt_zvkned(u8 *dst, const u8 *src, size_t len,
+ u8 iv[AES_BLOCK_SIZE], const struct aes_key *key);
+void aes_cbc_cts_crypt_zvkned(u8 *dst, const u8 *src, size_t len,
+ const u8 iv[AES_BLOCK_SIZE],
+ aes_encrypt_arg key, bool enc);
+
+/* len is always a positive multiple of AES_BLOCK_SIZE here. */
+#define aes_cbc_encrypt_arch aes_cbc_encrypt_arch
+static bool aes_cbc_encrypt_arch(u8 *dst, const u8 *src, size_t len,
+ u8 iv[AES_BLOCK_SIZE],
+ const struct aes_enckey *key)
+{
+ if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd()))
+ return false;
+ kernel_vector_begin();
+ aes_cbc_encrypt_zvkned(dst, src, len, iv, key);
+ kernel_vector_end();
+ return true;
+}
+
+/* len is always a positive multiple of AES_BLOCK_SIZE here. */
+#define aes_cbc_decrypt_arch aes_cbc_decrypt_arch
+static bool aes_cbc_decrypt_arch(u8 *dst, const u8 *src, size_t len,
+ u8 iv[AES_BLOCK_SIZE],
+ const struct aes_key *key)
+{
+ if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd()))
+ return false;
+ kernel_vector_begin();
+ aes_cbc_decrypt_zvkned(dst, src, len, iv, key);
+ kernel_vector_end();
+ return true;
+}
+
+/* len can be any value greater than AES_BLOCK_SIZE here. */
+#define aes_cbc_cts_encrypt_arch aes_cbc_cts_encrypt_arch
+static bool aes_cbc_cts_encrypt_arch(u8 *dst, const u8 *src, size_t len,
+ u8 iv[AES_BLOCK_SIZE],
+ const struct aes_enckey *key)
+{
+ if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd()))
+ return false;
+
+ kernel_vector_begin();
+ aes_cbc_cts_crypt_zvkned(dst, src, len, iv, key, true);
+ kernel_vector_end();
+ return true;
+}
+
+/* len can be any value greater than AES_BLOCK_SIZE here. */
+#define aes_cbc_cts_decrypt_arch aes_cbc_cts_decrypt_arch
+static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len,
+ u8 iv[AES_BLOCK_SIZE],
+ const struct aes_key *key)
+{
+ if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd()))
+ return false;
+
+ kernel_vector_begin();
+ aes_cbc_cts_crypt_zvkned(dst, src, len, iv, key, false);
+ kernel_vector_end();
+ return true;
+}
+#endif /* CONFIG_CRYPTO_LIB_AES_CBC */
+
#define aes_mod_init_arch aes_mod_init_arch
static void aes_mod_init_arch(void)
{
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread* [PATCH v2 19/20] lib/crypto: riscv/aes-ctr: Migrate optimized code into library
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
` (17 preceding siblings ...)
2026-09-27 22:43 ` [PATCH v2 18/20] lib/crypto: riscv/aes-cbc: " Eric Biggers
@ 2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 20/20] lib/crypto: riscv/aes-xts: " Eric Biggers
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:43 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers
Instead of exposing the riscv-optimized AES-CTR code via a
riscv-specific crypto_skcipher algorithm, just implement the AES-CTR
library functions. This is simpler, it makes the AES-CTR library
functions be riscv-optimized, and it also fixes the longstanding issue
where the riscv-optimized AES-CTR code was disabled by default. AES-CTR
support still remains available through crypto_skcipher via
crypto/aes.c, but individual architectures no longer need to handle it.
To match what the library expects, update the assembly functions to
operate on struct aes_enckey rather than struct crypto_aes_ctx, and
adjust the argument order.
Bump up the priority of the corresponding library-based algorithm on
riscv now that it no longer has to be lower than arch/riscv/crypto/.
Also re-enable the library-based "ccm(aes)" and "gcm(aes)".
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/riscv/crypto/Kconfig | 5 +-
arch/riscv/crypto/Makefile | 2 +-
arch/riscv/crypto/aes-riscv64-glue.c | 114 +-----------------
crypto/aes.c | 5 +-
lib/crypto/Makefile | 4 +
.../crypto/riscv}/aes-riscv64-zvkned-zvkb.S | 23 ++--
lib/crypto/riscv/aes.h | 72 ++++++++++-
7 files changed, 94 insertions(+), 131 deletions(-)
rename {arch/riscv/crypto => lib/crypto/riscv}/aes-riscv64-zvkned-zvkb.S (93%)
diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig
index 0a3f87ad384e..0733d4894401 100644
--- a/arch/riscv/crypto/Kconfig
+++ b/arch/riscv/crypto/Kconfig
@@ -3,18 +3,17 @@
menu "Accelerated Cryptographic Algorithms for CPU (riscv)"
config CRYPTO_AES_RISCV64
- tristate "Ciphers: AES, modes: CTR, XTS"
+ tristate "Ciphers: AES, modes: XTS"
depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
select CRYPTO_LIB_AES
select CRYPTO_SKCIPHER
help
- Length-preserving ciphers: AES with CTR, XTS
+ Length-preserving ciphers: AES with XTS
Architecture: riscv64 using:
- Zvkned vector crypto extension
- Zvbb vector extension (XTS)
- - Zvkb vector crypto extension (CTR)
- Zvkg vector crypto extension (XTS)
config CRYPTO_SM4_RISCV64
diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile
index d8b85afa6d0b..08904603fc94 100644
--- a/arch/riscv/crypto/Makefile
+++ b/arch/riscv/crypto/Makefile
@@ -2,7 +2,7 @@
obj-$(CONFIG_CRYPTO_AES_RISCV64) += aes-riscv64.o
aes-riscv64-y := aes-riscv64-glue.o \
- aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o
+ aes-riscv64-zvkned-zvbb-zvkg.o
obj-$(CONFIG_CRYPTO_SM4_RISCV64) += sm4-riscv64.o
sm4-riscv64-y := sm4-riscv64-glue.o sm4-riscv64-zvksed-zvkb.o
diff --git a/arch/riscv/crypto/aes-riscv64-glue.c b/arch/riscv/crypto/aes-riscv64-glue.c
index 97f5369d7e71..a7dcceb77c49 100644
--- a/arch/riscv/crypto/aes-riscv64-glue.c
+++ b/arch/riscv/crypto/aes-riscv64-glue.c
@@ -22,10 +22,6 @@
#include <linux/minmax.h>
#include <linux/module.h>
-asmlinkage void aes_ctr32_crypt_zvkned_zvkb(const struct crypto_aes_ctx *key,
- const u8 *in, u8 *out, size_t len,
- u8 iv[AES_BLOCK_SIZE]);
-
asmlinkage void aes_xts_encrypt_zvkned_zvbb_zvkg(
const struct crypto_aes_ctx *key,
const u8 *in, u8 *out, size_t len,
@@ -62,75 +58,6 @@ static int riscv64_aes_setkey(struct crypto_aes_ctx *ctx,
return aes_expandkey(ctx, key, keylen);
}
-static int riscv64_aes_setkey_skcipher(struct crypto_skcipher *tfm,
- const u8 *key, unsigned int keylen)
-{
- struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
-
- return riscv64_aes_setkey(ctx, key, keylen);
-}
-
-/* AES-CTR */
-
-static int riscv64_aes_ctr_crypt(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- const struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
- unsigned int nbytes, p1_nbytes;
- struct skcipher_walk walk;
- u32 ctr32, nblocks;
- int err;
-
- /* Get the low 32-bit word of the 128-bit big endian counter. */
- ctr32 = get_unaligned_be32(req->iv + 12);
-
- err = skcipher_walk_virt(&walk, req, false);
- while ((nbytes = walk.nbytes) != 0) {
- if (nbytes < walk.total) {
- /* Not the end yet, so keep the length block-aligned. */
- nbytes = round_down(nbytes, AES_BLOCK_SIZE);
- nblocks = nbytes / AES_BLOCK_SIZE;
- } else {
- /* It's the end, so include any final partial block. */
- nblocks = DIV_ROUND_UP(nbytes, AES_BLOCK_SIZE);
- }
- ctr32 += nblocks;
-
- kernel_vector_begin();
- if (ctr32 >= nblocks) {
- /* The low 32-bit word of the counter won't overflow. */
- aes_ctr32_crypt_zvkned_zvkb(ctx, walk.src.virt.addr,
- walk.dst.virt.addr, nbytes,
- req->iv);
- } else {
- /*
- * The low 32-bit word of the counter will overflow.
- * The assembly doesn't handle this case, so split the
- * operation into two at the point where the overflow
- * will occur. After the first part, add the carry bit.
- */
- p1_nbytes = min(nbytes, (nblocks - ctr32) * AES_BLOCK_SIZE);
- aes_ctr32_crypt_zvkned_zvkb(ctx, walk.src.virt.addr,
- walk.dst.virt.addr,
- p1_nbytes, req->iv);
- crypto_inc(req->iv, 12);
-
- if (ctr32) {
- aes_ctr32_crypt_zvkned_zvkb(
- ctx,
- walk.src.virt.addr + p1_nbytes,
- walk.dst.virt.addr + p1_nbytes,
- nbytes - p1_nbytes, req->iv);
- }
- }
- kernel_vector_end();
-
- err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
- }
-
- return err;
-}
-
/* AES-XTS */
struct riscv64_aes_xts_ctx {
@@ -251,25 +178,6 @@ static int riscv64_aes_xts_decrypt(struct skcipher_request *req)
/* Algorithm definitions */
-static struct skcipher_alg riscv64_zvkned_zvkb_aes_skcipher_alg = {
- .setkey = riscv64_aes_setkey_skcipher,
- .encrypt = riscv64_aes_ctr_crypt,
- .decrypt = riscv64_aes_ctr_crypt,
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .chunksize = AES_BLOCK_SIZE,
- .walksize = 4 * AES_BLOCK_SIZE, /* matches LMUL=4 */
- .base = {
- .cra_blocksize = 1,
- .cra_ctxsize = sizeof(struct crypto_aes_ctx),
- .cra_priority = 300,
- .cra_name = "ctr(aes)",
- .cra_driver_name = "ctr-aes-riscv64-zvkned-zvkb",
- .cra_module = THIS_MODULE,
- },
-};
-
static struct skcipher_alg riscv64_zvkned_zvbb_zvkg_aes_skcipher_alg = {
.setkey = riscv64_aes_xts_setkey,
.encrypt = riscv64_aes_xts_encrypt,
@@ -302,43 +210,27 @@ static int __init riscv64_aes_mod_init(void)
if (riscv_isa_extension_available(NULL, ZVKNED) &&
riscv_vector_vlen() >= 128) {
- if (riscv_isa_extension_available(NULL, ZVKB)) {
- err = crypto_register_skcipher(
- &riscv64_zvkned_zvkb_aes_skcipher_alg);
- if (err)
- return err;
- }
-
if (riscv64_aes_xts_supported()) {
err = crypto_register_skcipher(
&riscv64_zvkned_zvbb_zvkg_aes_skcipher_alg);
if (err)
- goto unregister_zvkned_zvkb_skcipher_alg;
+ return err;
}
}
return err;
-
-unregister_zvkned_zvkb_skcipher_alg:
- if (riscv_isa_extension_available(NULL, ZVKB))
- crypto_unregister_skcipher(&riscv64_zvkned_zvkb_aes_skcipher_alg);
- return err;
}
static void __exit riscv64_aes_mod_exit(void)
{
- if (riscv64_aes_xts_supported())
- crypto_unregister_skcipher(&riscv64_zvkned_zvbb_zvkg_aes_skcipher_alg);
- if (riscv_isa_extension_available(NULL, ZVKB))
- crypto_unregister_skcipher(&riscv64_zvkned_zvkb_aes_skcipher_alg);
+ crypto_unregister_skcipher(&riscv64_zvkned_zvbb_zvkg_aes_skcipher_alg);
}
module_init(riscv64_aes_mod_init);
module_exit(riscv64_aes_mod_exit);
-MODULE_DESCRIPTION("AES-CTR/XTS (RISC-V accelerated)");
+MODULE_DESCRIPTION("AES-XTS (RISC-V accelerated)");
MODULE_AUTHOR("Jerry Shih <jerry.shih@sifive.com>");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("aes");
-MODULE_ALIAS_CRYPTO("ctr(aes)");
MODULE_ALIAS_CRYPTO("xts(aes)");
diff --git a/crypto/aes.c b/crypto/aes.c
index 1daaa59d0cdf..1b84953c5deb 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -670,7 +670,8 @@ static struct skcipher_alg skcipher_algs[] = {
{
.base.cra_name = "ctr(aes)",
.base.cra_driver_name = "ctr-aes-lib",
- .base.cra_priority = IS_ENABLED(CONFIG_X86) ? 300 : 110,
+ .base.cra_priority = (IS_ENABLED(CONFIG_RISCV) ||
+ IS_ENABLED(CONFIG_X86)) ? 300 : 110,
.base.cra_blocksize = 1,
.base.cra_ctxsize = sizeof(struct aes_enckey),
.base.cra_module = THIS_MODULE,
@@ -1002,7 +1003,6 @@ static struct aead_alg aead_algs[] = {
!(IS_ENABLED(CONFIG_ARM) || \
IS_ENABLED(CONFIG_ARM64) || \
IS_ENABLED(CONFIG_PPC) || \
- IS_ENABLED(CONFIG_RISCV) || \
IS_ENABLED(CONFIG_S390) || \
IS_ENABLED(CONFIG_SPARC))
{
@@ -1045,7 +1045,6 @@ static struct aead_alg aead_algs[] = {
!(IS_ENABLED(CONFIG_ARM) || \
IS_ENABLED(CONFIG_ARM64) || \
IS_ENABLED(CONFIG_PPC) || \
- IS_ENABLED(CONFIG_RISCV) || \
IS_ENABLED(CONFIG_S390) || \
IS_ENABLED(CONFIG_SPARC))
{
diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
index 02d89a226377..ff34aeda37ba 100644
--- a/lib/crypto/Makefile
+++ b/lib/crypto/Makefile
@@ -51,6 +51,10 @@ endif # !CONFIG_SPE
endif # CONFIG_PPC
libaes-$(CONFIG_RISCV) += riscv/aes-riscv64-zvkned.o
+ifneq ($(CONFIG_CRYPTO_LIB_AES_CTR),)
+libaes-$(CONFIG_RISCV) += riscv/aes-riscv64-zvkned-zvkb.o
+endif
+
libaes-$(CONFIG_SPARC) += sparc/aes_asm.o
libaes-$(CONFIG_X86) += x86/aes-aesni.o
diff --git a/arch/riscv/crypto/aes-riscv64-zvkned-zvkb.S b/lib/crypto/riscv/aes-riscv64-zvkned-zvkb.S
similarity index 93%
rename from arch/riscv/crypto/aes-riscv64-zvkned-zvkb.S
rename to lib/crypto/riscv/aes-riscv64-zvkned-zvkb.S
index 9962d4500587..2981d8182de6 100644
--- a/arch/riscv/crypto/aes-riscv64-zvkned-zvkb.S
+++ b/lib/crypto/riscv/aes-riscv64-zvkned-zvkb.S
@@ -49,11 +49,11 @@
#include "aes-macros.S"
-#define KEYP a0
-#define INP a1
-#define OUTP a2
-#define LEN a3
-#define IVP a4
+#define DST a0
+#define SRC a1
+#define LEN a2
+#define IVP a3
+#define KEYP a4
#define LEN32 a5
#define VL_E32 a6
@@ -110,13 +110,13 @@
// XOR the data with the keystream.
vsetvli t0, LEN, e8, m4, ta, ma
- vle8.v v20, (INP)
+ vle8.v v20, (SRC)
vxor.vv v20, v20, v24
- vse8.v v20, (OUTP)
+ vse8.v v20, (DST)
// Advance the pointers and update the remaining length.
- add INP, INP, t0
- add OUTP, OUTP, t0
+ add SRC, SRC, t0
+ add DST, DST, t0
sub LEN, LEN, t0
sub LEN32, LEN32, VL_E32
srli VL_BLOCKS, VL_E32, 2
@@ -133,9 +133,8 @@
ret
.endm
-// void aes_ctr32_crypt_zvkned_zvkb(const struct crypto_aes_ctx *key,
-// const u8 *in, u8 *out, size_t len,
-// u8 iv[16]);
+// void aes_ctr32_crypt_zvkned_zvkb(u8 *dst, const u8 *src, size_t len,
+// u8 iv[16], const struct aes_enckey *key);
SYM_FUNC_START(aes_ctr32_crypt_zvkned_zvkb)
aes_begin KEYP, 128f, 192f
aes_ctr32_crypt 256
diff --git a/lib/crypto/riscv/aes.h b/lib/crypto/riscv/aes.h
index 150cda82cb89..c04e04cf7019 100644
--- a/lib/crypto/riscv/aes.h
+++ b/lib/crypto/riscv/aes.h
@@ -9,6 +9,7 @@
#include <asm/vector.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkned);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkned_zvkb);
/* The assembly code assumes the following offsets. */
static_assert(offsetof(struct aes_enckey, len) == 0);
@@ -161,10 +162,79 @@ static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len,
}
#endif /* CONFIG_CRYPTO_LIB_AES_CBC */
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR)
+void aes_ctr32_crypt_zvkned_zvkb(u8 *dst, const u8 *src, size_t len, u8 iv[16],
+ const struct aes_enckey *key);
+
+static void aes_ctr_riscv(u8 *dst, const u8 *src, size_t len,
+ u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key)
+{
+ u32 ctr32 = get_unaligned_be32(&ctr[12]);
+ u32 nblocks;
+ size_t part1_len;
+
+ /* This assumes len <= U32_MAX * AES_BLOCK_SIZE. */
+ nblocks = DIV_ROUND_UP(len, AES_BLOCK_SIZE);
+ ctr32 += nblocks;
+
+ if (likely(ctr32 >= nblocks)) {
+ /* The low 32 bits of the counter won't overflow. */
+ aes_ctr32_crypt_zvkned_zvkb(dst, src, len, ctr, key);
+ } else {
+ /*
+ * The low 32 bits of the counter will overflow. The
+ * assembly doesn't handle this case, so split the
+ * operation into two at the point where the overflow
+ * will occur. After the first part, add the carry bit.
+ */
+ part1_len =
+ min(len, (size_t)(nblocks - ctr32) * AES_BLOCK_SIZE);
+ aes_ctr32_crypt_zvkned_zvkb(dst, src, part1_len, ctr, key);
+ for (int i = AES_BLOCK_SIZE - 5; i >= 0; i--) {
+ if (++ctr[i] != 0)
+ break;
+ }
+ if (part1_len < len)
+ aes_ctr32_crypt_zvkned_zvkb(dst + part1_len,
+ src + part1_len,
+ len - part1_len, ctr, key);
+ }
+}
+
+#define aes_ctr_arch aes_ctr_arch
+static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len,
+ u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key)
+{
+ if (!static_branch_likely(&have_zvkned_zvkb) ||
+ unlikely(!may_use_simd()))
+ return false;
+ kernel_vector_begin();
+ while (len) {
+ /*
+ * Split super long messages into chunks that each need at most
+ * 1 carry bit out of the low 32-bit word of the counter. Up to
+ * U32_MAX * AES_BLOCK_SIZE bytes could be allowed per chunk,
+ * but use a lower limit to guard against other errors.
+ */
+ size_t n = min(len, round_down(S32_MAX, 32 * AES_BLOCK_SIZE));
+
+ aes_ctr_riscv(dst, src, n, ctr, key);
+ dst += n;
+ src += n;
+ len -= n;
+ }
+ kernel_vector_end();
+ return true;
+}
+#endif /* CONFIG_CRYPTO_LIB_AES_CTR */
+
#define aes_mod_init_arch aes_mod_init_arch
static void aes_mod_init_arch(void)
{
if (riscv_isa_extension_available(NULL, ZVKNED) &&
- riscv_vector_vlen() >= 128)
+ riscv_vector_vlen() >= 128) {
static_branch_enable(&have_zvkned);
+ if (riscv_isa_extension_available(NULL, ZVKB))
+ static_branch_enable(&have_zvkned_zvkb);
+ }
}
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread* [PATCH v2 20/20] lib/crypto: riscv/aes-xts: Migrate optimized code into library
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
` (18 preceding siblings ...)
2026-09-27 22:43 ` [PATCH v2 19/20] lib/crypto: riscv/aes-ctr: " Eric Biggers
@ 2026-09-27 22:43 ` Eric Biggers
19 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2026-09-27 22:43 UTC (permalink / raw)
To: linux-crypto
Cc: linux-kernel, Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu,
x86, linux-riscv, Eric Biggers
Instead of exposing the riscv-optimized AES-XTS code via a
riscv-specific crypto_skcipher algorithm, just implement the AES-XTS
library functions. This is simpler, it makes the AES-XTS library
functions be riscv-optimized, and it also fixes the longstanding issue
where the riscv-optimized AES-XTS code was disabled by default. AES-XTS
support still remains available through crypto_skcipher via
crypto/aes.c, but individual architectures no longer need to handle it.
To match what the library expects, update the assembly functions to
operate on struct aes_key rather than struct crypto_aes_ctx, adjust the
argument order, and remove the redundant ciphertext stealing support
which is already implemented in a generic way in the library.
Bump up the priority of the corresponding library-based algorithm on
riscv now that it no longer has to be lower than arch/riscv/crypto/.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/riscv/crypto/Kconfig | 14 --
arch/riscv/crypto/Makefile | 4 -
arch/riscv/crypto/aes-macros.S | 166 ------------
arch/riscv/crypto/aes-riscv64-glue.c | 236 ------------------
crypto/aes.c | 3 +-
lib/crypto/Makefile | 3 +
.../riscv}/aes-riscv64-zvkned-zvbb-zvkg.S | 99 ++------
lib/crypto/riscv/aes.h | 53 ++++
8 files changed, 74 insertions(+), 504 deletions(-)
delete mode 100644 arch/riscv/crypto/aes-macros.S
delete mode 100644 arch/riscv/crypto/aes-riscv64-glue.c
rename {arch/riscv/crypto => lib/crypto/riscv}/aes-riscv64-zvkned-zvbb-zvkg.S (74%)
diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig
index 0733d4894401..614f93214862 100644
--- a/arch/riscv/crypto/Kconfig
+++ b/arch/riscv/crypto/Kconfig
@@ -2,20 +2,6 @@
menu "Accelerated Cryptographic Algorithms for CPU (riscv)"
-config CRYPTO_AES_RISCV64
- tristate "Ciphers: AES, modes: XTS"
- depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
- RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
- select CRYPTO_LIB_AES
- select CRYPTO_SKCIPHER
- help
- Length-preserving ciphers: AES with XTS
-
- Architecture: riscv64 using:
- - Zvkned vector crypto extension
- - Zvbb vector extension (XTS)
- - Zvkg vector crypto extension (XTS)
-
config CRYPTO_SM4_RISCV64
tristate "Ciphers: SM4 (ShangMi 4)"
depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile
index 08904603fc94..9f6956cf50b1 100644
--- a/arch/riscv/crypto/Makefile
+++ b/arch/riscv/crypto/Makefile
@@ -1,8 +1,4 @@
# SPDX-License-Identifier: GPL-2.0-only
-obj-$(CONFIG_CRYPTO_AES_RISCV64) += aes-riscv64.o
-aes-riscv64-y := aes-riscv64-glue.o \
- aes-riscv64-zvkned-zvbb-zvkg.o
-
obj-$(CONFIG_CRYPTO_SM4_RISCV64) += sm4-riscv64.o
sm4-riscv64-y := sm4-riscv64-glue.o sm4-riscv64-zvksed-zvkb.o
diff --git a/arch/riscv/crypto/aes-macros.S b/arch/riscv/crypto/aes-macros.S
deleted file mode 100644
index 1384164621a5..000000000000
--- a/arch/riscv/crypto/aes-macros.S
+++ /dev/null
@@ -1,166 +0,0 @@
-/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */
-//
-// This file is dual-licensed, meaning that you can use it under your
-// choice of either of the following two licenses:
-//
-// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
-//
-// Licensed under the Apache License 2.0 (the "License"). You can obtain
-// a copy in the file LICENSE in the source distribution or at
-// https://www.openssl.org/source/license.html
-//
-// or
-//
-// Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu>
-// Copyright (c) 2023, Phoebe Chen <phoebe.chen@sifive.com>
-// Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com>
-// Copyright 2024 Google LLC
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions
-// are met:
-// 1. Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// 2. Redistributions in binary form must reproduce the above copyright
-// notice, this list of conditions and the following disclaimer in the
-// documentation and/or other materials provided with the distribution.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-// This file contains macros that are shared by the other aes-*.S files. The
-// generated code of these macros depends on the following RISC-V extensions:
-// - RV64I
-// - RISC-V Vector ('V') with VLEN >= 128
-// - RISC-V Vector AES block cipher extension ('Zvkned')
-
-// Loads the AES round keys from \keyp into vector registers and jumps to code
-// specific to the length of the key. Specifically:
-// - If AES-128, loads round keys into v1-v11 and jumps to \label128.
-// - If AES-192, loads round keys into v1-v13 and jumps to \label192.
-// - If AES-256, loads round keys into v1-v15 and continues onwards.
-//
-// Also sets vl=4 and vtype=e32,m1,ta,ma. Clobbers t0 and t1.
-.macro aes_begin keyp, label128, label192, key_len
-.ifb \key_len
- lwu t0, 480(\keyp) // t0 = key length in bytes
-.endif
- li t1, 24 // t1 = key length for AES-192
- vsetivli zero, 4, e32, m1, ta, ma
- vle32.v v1, (\keyp)
- addi \keyp, \keyp, 16
- vle32.v v2, (\keyp)
- addi \keyp, \keyp, 16
- vle32.v v3, (\keyp)
- addi \keyp, \keyp, 16
- vle32.v v4, (\keyp)
- addi \keyp, \keyp, 16
- vle32.v v5, (\keyp)
- addi \keyp, \keyp, 16
- vle32.v v6, (\keyp)
- addi \keyp, \keyp, 16
- vle32.v v7, (\keyp)
- addi \keyp, \keyp, 16
- vle32.v v8, (\keyp)
- addi \keyp, \keyp, 16
- vle32.v v9, (\keyp)
- addi \keyp, \keyp, 16
- vle32.v v10, (\keyp)
- addi \keyp, \keyp, 16
- vle32.v v11, (\keyp)
-.ifb \key_len
- blt t0, t1, \label128 // If AES-128, goto label128.
-.else
- blt \key_len, t1, \label128 // If AES-128, goto label128.
-.endif
- addi \keyp, \keyp, 16
- vle32.v v12, (\keyp)
- addi \keyp, \keyp, 16
- vle32.v v13, (\keyp)
-.ifb \key_len
- beq t0, t1, \label192 // If AES-192, goto label192.
-.else
- beq \key_len, t1, \label192 // If AES-192, goto label192.
-.endif
- // Else, it's AES-256.
- addi \keyp, \keyp, 16
- vle32.v v14, (\keyp)
- addi \keyp, \keyp, 16
- vle32.v v15, (\keyp)
-.endm
-
-// Encrypts \data using zvkned instructions, using the round keys loaded into
-// v1-v11 (for AES-128), v1-v13 (for AES-192), or v1-v15 (for AES-256). \keylen
-// is the AES key length in bits. vl and vtype must already be set
-// appropriately. Note that if vl > 4, multiple blocks are encrypted.
-.macro aes_encrypt data, keylen
- vaesz.vs \data, v1
- vaesem.vs \data, v2
- vaesem.vs \data, v3
- vaesem.vs \data, v4
- vaesem.vs \data, v5
- vaesem.vs \data, v6
- vaesem.vs \data, v7
- vaesem.vs \data, v8
- vaesem.vs \data, v9
- vaesem.vs \data, v10
-.if \keylen == 128
- vaesef.vs \data, v11
-.elseif \keylen == 192
- vaesem.vs \data, v11
- vaesem.vs \data, v12
- vaesef.vs \data, v13
-.else
- vaesem.vs \data, v11
- vaesem.vs \data, v12
- vaesem.vs \data, v13
- vaesem.vs \data, v14
- vaesef.vs \data, v15
-.endif
-.endm
-
-// Same as aes_encrypt, but decrypts instead of encrypts.
-.macro aes_decrypt data, keylen
-.if \keylen == 128
- vaesz.vs \data, v11
-.elseif \keylen == 192
- vaesz.vs \data, v13
- vaesdm.vs \data, v12
- vaesdm.vs \data, v11
-.else
- vaesz.vs \data, v15
- vaesdm.vs \data, v14
- vaesdm.vs \data, v13
- vaesdm.vs \data, v12
- vaesdm.vs \data, v11
-.endif
- vaesdm.vs \data, v10
- vaesdm.vs \data, v9
- vaesdm.vs \data, v8
- vaesdm.vs \data, v7
- vaesdm.vs \data, v6
- vaesdm.vs \data, v5
- vaesdm.vs \data, v4
- vaesdm.vs \data, v3
- vaesdm.vs \data, v2
- vaesdf.vs \data, v1
-.endm
-
-// Expands to aes_encrypt or aes_decrypt according to \enc, which is 1 or 0.
-.macro aes_crypt data, enc, keylen
-.if \enc
- aes_encrypt \data, \keylen
-.else
- aes_decrypt \data, \keylen
-.endif
-.endm
diff --git a/arch/riscv/crypto/aes-riscv64-glue.c b/arch/riscv/crypto/aes-riscv64-glue.c
deleted file mode 100644
index a7dcceb77c49..000000000000
--- a/arch/riscv/crypto/aes-riscv64-glue.c
+++ /dev/null
@@ -1,236 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * AES modes using the RISC-V vector crypto extensions
- *
- * Copyright (C) 2023 VRULL GmbH
- * Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
- *
- * Copyright (C) 2023 SiFive, Inc.
- * Author: Jerry Shih <jerry.shih@sifive.com>
- *
- * Copyright 2024 Google LLC
- */
-
-#include <asm/simd.h>
-#include <asm/vector.h>
-#include <crypto/aes.h>
-#include <crypto/internal/simd.h>
-#include <crypto/internal/skcipher.h>
-#include <crypto/scatterwalk.h>
-#include <crypto/xts.h>
-#include <linux/linkage.h>
-#include <linux/minmax.h>
-#include <linux/module.h>
-
-asmlinkage void aes_xts_encrypt_zvkned_zvbb_zvkg(
- const struct crypto_aes_ctx *key,
- const u8 *in, u8 *out, size_t len,
- u8 tweak[AES_BLOCK_SIZE]);
-
-asmlinkage void aes_xts_decrypt_zvkned_zvbb_zvkg(
- const struct crypto_aes_ctx *key,
- const u8 *in, u8 *out, size_t len,
- u8 tweak[AES_BLOCK_SIZE]);
-
-static int riscv64_aes_setkey(struct crypto_aes_ctx *ctx,
- const u8 *key, unsigned int keylen)
-{
- /*
- * For now we just use the generic key expansion, for these reasons:
- *
- * - zvkned's key expansion instructions don't support AES-192.
- * So, non-zvkned fallback code would be needed anyway.
- *
- * - Users of AES in Linux usually don't change keys frequently.
- * So, key expansion isn't performance-critical.
- *
- * - For single-block AES exposed as a "cipher" algorithm, it's
- * necessary to use struct crypto_aes_ctx and initialize its 'key_dec'
- * field with the round keys for the Equivalent Inverse Cipher. This
- * is because with "cipher", decryption can be requested from a
- * context where the vector unit isn't usable, necessitating a
- * fallback to aes_decrypt(). But, zvkned can only generate and use
- * the normal round keys. Of course, it's preferable to not have
- * special code just for "cipher", as e.g. XTS also uses a
- * single-block AES encryption. It's simplest to just use
- * struct crypto_aes_ctx and aes_expandkey() everywhere.
- */
- return aes_expandkey(ctx, key, keylen);
-}
-
-/* AES-XTS */
-
-struct riscv64_aes_xts_ctx {
- struct crypto_aes_ctx ctx1;
- struct aes_enckey tweak_key;
-};
-
-static int riscv64_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
- unsigned int keylen)
-{
- struct riscv64_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
-
- return xts_verify_key(tfm, key, keylen) ?:
- riscv64_aes_setkey(&ctx->ctx1, key, keylen / 2) ?:
- aes_prepareenckey(&ctx->tweak_key, key + keylen / 2, keylen / 2);
-}
-
-static int riscv64_aes_xts_crypt(struct skcipher_request *req, bool enc)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- const struct riscv64_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
- int tail = req->cryptlen % AES_BLOCK_SIZE;
- struct scatterlist sg_src[2], sg_dst[2];
- struct skcipher_request subreq;
- struct scatterlist *src, *dst;
- struct skcipher_walk walk;
- int err;
-
- if (req->cryptlen < AES_BLOCK_SIZE)
- return -EINVAL;
-
- /* Encrypt the IV with the tweak key to get the first tweak. */
- aes_encrypt(&ctx->tweak_key, req->iv, req->iv);
-
- err = skcipher_walk_virt(&walk, req, false);
-
- /*
- * If the message length isn't divisible by the AES block size and the
- * full message isn't available in one step of the scatterlist walk,
- * then separate off the last full block and the partial block. This
- * ensures that they are processed in the same call to the assembly
- * function, which is required for ciphertext stealing.
- */
- if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
- skcipher_walk_abort(&walk);
-
- skcipher_request_set_tfm(&subreq, tfm);
- skcipher_request_set_callback(&subreq,
- skcipher_request_flags(req),
- NULL, NULL);
- skcipher_request_set_crypt(&subreq, req->src, req->dst,
- req->cryptlen - tail - AES_BLOCK_SIZE,
- req->iv);
- req = &subreq;
- err = skcipher_walk_virt(&walk, req, false);
- } else {
- tail = 0;
- }
-
- while (walk.nbytes) {
- unsigned int nbytes = walk.nbytes;
-
- if (nbytes < walk.total)
- nbytes = round_down(nbytes, AES_BLOCK_SIZE);
-
- kernel_vector_begin();
- if (enc)
- aes_xts_encrypt_zvkned_zvbb_zvkg(
- &ctx->ctx1, walk.src.virt.addr,
- walk.dst.virt.addr, nbytes, req->iv);
- else
- aes_xts_decrypt_zvkned_zvbb_zvkg(
- &ctx->ctx1, walk.src.virt.addr,
- walk.dst.virt.addr, nbytes, req->iv);
- kernel_vector_end();
- err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
- }
-
- if (err || likely(!tail))
- return err;
-
- /* Do ciphertext stealing with the last full block and partial block. */
-
- dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
- if (req->dst != req->src)
- dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
-
- skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
- req->iv);
-
- err = skcipher_walk_virt(&walk, req, false);
- if (err)
- return err;
-
- kernel_vector_begin();
- if (enc)
- aes_xts_encrypt_zvkned_zvbb_zvkg(
- &ctx->ctx1, walk.src.virt.addr,
- walk.dst.virt.addr, walk.nbytes, req->iv);
- else
- aes_xts_decrypt_zvkned_zvbb_zvkg(
- &ctx->ctx1, walk.src.virt.addr,
- walk.dst.virt.addr, walk.nbytes, req->iv);
- kernel_vector_end();
-
- return skcipher_walk_done(&walk, 0);
-}
-
-static int riscv64_aes_xts_encrypt(struct skcipher_request *req)
-{
- return riscv64_aes_xts_crypt(req, true);
-}
-
-static int riscv64_aes_xts_decrypt(struct skcipher_request *req)
-{
- return riscv64_aes_xts_crypt(req, false);
-}
-
-/* Algorithm definitions */
-
-static struct skcipher_alg riscv64_zvkned_zvbb_zvkg_aes_skcipher_alg = {
- .setkey = riscv64_aes_xts_setkey,
- .encrypt = riscv64_aes_xts_encrypt,
- .decrypt = riscv64_aes_xts_decrypt,
- .min_keysize = 2 * AES_MIN_KEY_SIZE,
- .max_keysize = 2 * AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .chunksize = AES_BLOCK_SIZE,
- .walksize = 4 * AES_BLOCK_SIZE, /* matches LMUL=4 */
- .base = {
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct riscv64_aes_xts_ctx),
- .cra_priority = 300,
- .cra_name = "xts(aes)",
- .cra_driver_name = "xts-aes-riscv64-zvkned-zvbb-zvkg",
- .cra_module = THIS_MODULE,
- },
-};
-
-static inline bool riscv64_aes_xts_supported(void)
-{
- return riscv_isa_extension_available(NULL, ZVBB) &&
- riscv_isa_extension_available(NULL, ZVKG) &&
- riscv_vector_vlen() < 2048 /* Implementation limitation */;
-}
-
-static int __init riscv64_aes_mod_init(void)
-{
- int err = -ENODEV;
-
- if (riscv_isa_extension_available(NULL, ZVKNED) &&
- riscv_vector_vlen() >= 128) {
- if (riscv64_aes_xts_supported()) {
- err = crypto_register_skcipher(
- &riscv64_zvkned_zvbb_zvkg_aes_skcipher_alg);
- if (err)
- return err;
- }
- }
-
- return err;
-}
-
-static void __exit riscv64_aes_mod_exit(void)
-{
- crypto_unregister_skcipher(&riscv64_zvkned_zvbb_zvkg_aes_skcipher_alg);
-}
-
-module_init(riscv64_aes_mod_init);
-module_exit(riscv64_aes_mod_exit);
-
-MODULE_DESCRIPTION("AES-XTS (RISC-V accelerated)");
-MODULE_AUTHOR("Jerry Shih <jerry.shih@sifive.com>");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS_CRYPTO("aes");
-MODULE_ALIAS_CRYPTO("xts(aes)");
diff --git a/crypto/aes.c b/crypto/aes.c
index 1b84953c5deb..2e61146f6fbb 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -705,7 +705,8 @@ static struct skcipher_alg skcipher_algs[] = {
{
.base.cra_name = "xts(aes)",
.base.cra_driver_name = "xts-aes-lib",
- .base.cra_priority = IS_ENABLED(CONFIG_X86) ? 300 : 110,
+ .base.cra_priority = (IS_ENABLED(CONFIG_RISCV) ||
+ IS_ENABLED(CONFIG_X86)) ? 300 : 110,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct aes_xts_key),
.base.cra_module = THIS_MODULE,
diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
index ff34aeda37ba..d683b8520f55 100644
--- a/lib/crypto/Makefile
+++ b/lib/crypto/Makefile
@@ -54,6 +54,9 @@ libaes-$(CONFIG_RISCV) += riscv/aes-riscv64-zvkned.o
ifneq ($(CONFIG_CRYPTO_LIB_AES_CTR),)
libaes-$(CONFIG_RISCV) += riscv/aes-riscv64-zvkned-zvkb.o
endif
+ifneq ($(CONFIG_CRYPTO_LIB_AES_XTS),)
+libaes-$(CONFIG_RISCV) += riscv/aes-riscv64-zvkned-zvbb-zvkg.o
+endif
libaes-$(CONFIG_SPARC) += sparc/aes_asm.o
diff --git a/arch/riscv/crypto/aes-riscv64-zvkned-zvbb-zvkg.S b/lib/crypto/riscv/aes-riscv64-zvkned-zvbb-zvkg.S
similarity index 74%
rename from arch/riscv/crypto/aes-riscv64-zvkned-zvbb-zvkg.S
rename to lib/crypto/riscv/aes-riscv64-zvkned-zvbb-zvkg.S
index 146fc9cfb268..c45aaed80ce6 100644
--- a/arch/riscv/crypto/aes-riscv64-zvkned-zvbb-zvkg.S
+++ b/lib/crypto/riscv/aes-riscv64-zvkned-zvbb-zvkg.S
@@ -50,14 +50,13 @@
#include "aes-macros.S"
-#define KEYP a0
-#define INP a1
-#define OUTP a2
-#define LEN a3
-#define TWEAKP a4
+#define DST a0
+#define SRC a1
+#define LEN a2
+#define TWEAKP a3
+#define KEYP a4
#define LEN32 a5
-#define TAIL_LEN a6
#define VL a7
#define VLMAX t4
@@ -67,9 +66,6 @@
#define TWEAKS_BREV v20 // LMUL=4 (most of the time)
#define MULTS_BREV v24 // LMUL=4 (most of the time)
#define TMP0 v28
-#define TMP1 v29
-#define TMP2 v30
-#define TMP3 v31
// xts_init initializes the following values:
//
@@ -167,24 +163,21 @@
.endm
.macro __aes_xts_crypt enc, keylen
- // With 16 < len <= 31, there's no main loop, just ciphertext stealing.
- beqz LEN32, .Lcts_without_main_loop\@
-
vsetvli VLMAX, zero, e32, m4, ta, ma
1:
vsetvli VL, LEN32, e32, m4, ta, ma
2:
// Encrypt or decrypt VL/4 blocks.
- vle32.v TMP0, (INP)
+ vle32.v TMP0, (SRC)
vxor.vv TMP0, TMP0, TWEAKS
aes_crypt TMP0, \enc, \keylen
vxor.vv TMP0, TMP0, TWEAKS
- vse32.v TMP0, (OUTP)
+ vse32.v TMP0, (DST)
// Update the pointers and the remaining length.
slli t0, VL, 2
- add INP, INP, t0
- add OUTP, OUTP, t0
+ add SRC, SRC, t0
+ add DST, DST, t0
sub LEN32, LEN32, VL
// Check whether more blocks remain.
@@ -217,73 +210,14 @@
vsetivli zero, 4, e32, m1, ta, ma
vgmul.vv TWEAKS_BREV, MULTS_BREV // Advance to next tweak
- bnez TAIL_LEN, .Lcts\@
-
// Update *TWEAKP to contain the next tweak.
vbrev8.v TWEAKS, TWEAKS_BREV
vse32.v TWEAKS, (TWEAKP)
ret
-
-.Lcts_without_main_loop\@:
- load_x
-.Lcts\@:
- // TWEAKS_BREV now contains the next tweak. Compute the one after that.
- vsetivli zero, 4, e32, m1, ta, ma
- vmv.v.v TMP0, TWEAKS_BREV
- vgmul.vv TMP0, MULTS_BREV
- // Undo the bit reversal of the next two tweaks and store them in TMP1
- // and TMP2, such that TMP1 is the first needed and TMP2 the second.
-.if \enc
- vbrev8.v TMP1, TWEAKS_BREV
- vbrev8.v TMP2, TMP0
-.else
- vbrev8.v TMP1, TMP0
- vbrev8.v TMP2, TWEAKS_BREV
-.endif
-
- // Encrypt/decrypt the last full block.
- vle32.v TMP0, (INP)
- vxor.vv TMP0, TMP0, TMP1
- aes_crypt TMP0, \enc, \keylen
- vxor.vv TMP0, TMP0, TMP1
-
- // Swap the first TAIL_LEN bytes of the above result with the tail.
- // Note that to support in-place encryption/decryption, the load from
- // the input tail must happen before the store to the output tail.
- addi t0, INP, 16
- addi t1, OUTP, 16
- vmv.v.v TMP3, TMP0
- vsetvli zero, TAIL_LEN, e8, m1, tu, ma
- vle8.v TMP0, (t0)
- vse8.v TMP3, (t1)
-
- // Encrypt/decrypt again and store the last full block.
- vsetivli zero, 4, e32, m1, ta, ma
- vxor.vv TMP0, TMP0, TMP2
- aes_crypt TMP0, \enc, \keylen
- vxor.vv TMP0, TMP0, TMP2
- vse32.v TMP0, (OUTP)
-
- ret
.endm
.macro aes_xts_crypt enc
-
- // Check whether the length is a multiple of the AES block size.
- andi TAIL_LEN, LEN, 15
- beqz TAIL_LEN, 1f
-
- // The length isn't a multiple of the AES block size, so ciphertext
- // stealing will be required. Ciphertext stealing involves special
- // handling of the partial block and the last full block, so subtract
- // the length of both from the length to be processed in the main loop.
- sub LEN, LEN, TAIL_LEN
- addi LEN, LEN, -16
-1:
srli LEN32, LEN, 2
- // LEN and LEN32 now contain the total length of the blocks that will be
- // processed in the main loop, in bytes and 32-bit words respectively.
-
xts_init
aes_begin KEYP, 128f, 192f
__aes_xts_crypt \enc, 256
@@ -293,15 +227,14 @@
__aes_xts_crypt \enc, 192
.endm
-// void aes_xts_encrypt_zvkned_zvbb_zvkg(const struct crypto_aes_ctx *key,
-// const u8 *in, u8 *out, size_t len,
-// u8 tweak[16]);
+// void aes_xts_encrypt_zvkned_zvbb_zvkg(u8 *dst, const u8 *src, size_t len,
+// u8 tweak[AES_BLOCK_SIZE],
+// const struct aes_key *key);
//
-// |key| is the data key. |tweak| contains the next tweak; the encryption of
-// the original IV with the tweak key was already done. This function supports
-// incremental computation, but |len| must always be >= 16 (AES_BLOCK_SIZE), and
-// |len| must be a multiple of 16 except on the last call. If |len| is a
-// multiple of 16, then this function updates |tweak| to contain the next tweak.
+// Encrypts `len` bytes of data using AES-XTS using the given `key` and `tweak`.
+// `key` is just the main key; `tweak` is assumed to have already been encrypted
+// by the tweak key. `len` is assumed to be a multiple of 16. `tweak` is
+// updated to contain the next tweak, allowing for incremental computation.
SYM_FUNC_START(aes_xts_encrypt_zvkned_zvbb_zvkg)
aes_xts_crypt 1
SYM_FUNC_END(aes_xts_encrypt_zvkned_zvbb_zvkg)
diff --git a/lib/crypto/riscv/aes.h b/lib/crypto/riscv/aes.h
index c04e04cf7019..36e9218b507f 100644
--- a/lib/crypto/riscv/aes.h
+++ b/lib/crypto/riscv/aes.h
@@ -1,5 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
+ * AES using the RISC-V vector crypto extensions
+ *
* Copyright (C) 2023 VRULL GmbH
* Copyright (C) 2023 SiFive, Inc.
* Copyright 2024 Google LLC
@@ -10,6 +12,7 @@
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkned);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkned_zvkb);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_xts);
/* The assembly code assumes the following offsets. */
static_assert(offsetof(struct aes_enckey, len) == 0);
@@ -228,6 +231,52 @@ static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len,
}
#endif /* CONFIG_CRYPTO_LIB_AES_CTR */
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS)
+void aes_xts_encrypt_zvkned_zvbb_zvkg(u8 *dst, const u8 *src, size_t len,
+ u8 tweak[AES_BLOCK_SIZE],
+ const struct aes_key *key);
+void aes_xts_decrypt_zvkned_zvbb_zvkg(u8 *dst, const u8 *src, size_t len,
+ u8 tweak[AES_BLOCK_SIZE],
+ const struct aes_key *key);
+
+/* len is always a positive multiple of AES_BLOCK_SIZE here. */
+static __always_inline bool
+aes_xts_crypt_riscv(u8 *dst, const u8 *src, size_t len,
+ u8 tweak[AES_BLOCK_SIZE],
+ const struct aes_xts_key *key, bool cont, bool enc)
+{
+ if (!static_branch_likely(&have_xts) || unlikely(!may_use_simd()))
+ return false;
+ kernel_vector_begin();
+ if (!cont)
+ aes_encrypt_zvkned(&key->tweak_key, tweak, tweak);
+ if (enc)
+ aes_xts_encrypt_zvkned_zvbb_zvkg(dst, src, len, tweak,
+ &key->main_key);
+ else
+ aes_xts_decrypt_zvkned_zvbb_zvkg(dst, src, len, tweak,
+ &key->main_key);
+ kernel_vector_end();
+ return true;
+}
+
+#define aes_xts_encrypt_arch aes_xts_encrypt_arch
+static bool aes_xts_encrypt_arch(u8 *dst, const u8 *src, size_t len,
+ u8 tweak[AES_BLOCK_SIZE],
+ const struct aes_xts_key *key, bool cont)
+{
+ return aes_xts_crypt_riscv(dst, src, len, tweak, key, cont, true);
+}
+
+#define aes_xts_decrypt_arch aes_xts_decrypt_arch
+static bool aes_xts_decrypt_arch(u8 *dst, const u8 *src, size_t len,
+ u8 tweak[AES_BLOCK_SIZE],
+ const struct aes_xts_key *key, bool cont)
+{
+ return aes_xts_crypt_riscv(dst, src, len, tweak, key, cont, false);
+}
+#endif /* CONFIG_CRYPTO_LIB_AES_XTS */
+
#define aes_mod_init_arch aes_mod_init_arch
static void aes_mod_init_arch(void)
{
@@ -236,5 +285,9 @@ static void aes_mod_init_arch(void)
static_branch_enable(&have_zvkned);
if (riscv_isa_extension_available(NULL, ZVKB))
static_branch_enable(&have_zvkned_zvkb);
+ if (riscv_isa_extension_available(NULL, ZVBB) &&
+ riscv_isa_extension_available(NULL, ZVKG) &&
+ riscv_vector_vlen() < 2048 /* Implementation limitation */)
+ static_branch_enable(&have_xts);
}
}
--
2.55.0
^ permalink raw reply [flat|nested] 21+ messages in thread