[PATCH 0/3] AES library fixups

[PATCH 0/3] AES library fixups

[Eric Biggers]

A few small fixups on top of
https://lore.kernel.org/linux-crypto/20260927224418.109759-1-ebiggers@kernel.org/

Eric Biggers (3):
  lib/crypto: x86/aes: Set RNDKEY in aes_cbc_encrypt_aesni()
  crypto: aes - Boost priority of encryption modes further
  crypto: x86/aesni-intel - Add transitional selections

 arch/x86/crypto/Kconfig    | 12 ++++++++++--
 crypto/aes.c               | 12 ++++++------
 lib/crypto/x86/aes-aesni.S | 25 +++++++++++++------------
 3 files changed, 29 insertions(+), 20 deletions(-)


base-commit: 69897f5e8552af8835a2c8a36aa7d597e1cf6689
-- 
2.55.0

[PATCH 1/3] lib/crypto: x86/aes: Set RNDKEY in aes_cbc_encrypt_aesni()

[Eric Biggers]

Make aes_cbc_encrypt_aesni() set the RNDKEY register alias explicitly,
rather than reusing the alias left by the previous function (which
worked by chance).

Also consistently comment the register aliases needed by _do_aes.

Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
 lib/crypto/x86/aes-aesni.S | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/lib/crypto/x86/aes-aesni.S b/lib/crypto/x86/aes-aesni.S
index cbc0cc23f63e..6795e82c34c8 100644
--- a/lib/crypto/x86/aes-aesni.S
+++ b/lib/crypto/x86/aes-aesni.S
@@ -444,13 +444,14 @@ SYM_FUNC_START(aes_cbc_encrypt_aesni)
 
 	// Other local variables
 #ifdef __x86_64__
-	.set	RNDKEY_PTR,	%r9
+	.set	RNDKEY_PTR,	%r9	// Temporary register for _do_aes
 #else
-	.set	RNDKEY_PTR,	IV_PTR // IV_PTR is clobbered and reloaded later
+	.set	RNDKEY_PTR,	IV_PTR	// IV_PTR is clobbered and reloaded later
 #endif
-	.set	NROUNDS,	TMP_32
+	.set	NROUNDS,	TMP_32	// Temporary register for _do_aes
 	.set	AESDATA0,	%xmm0
 	.set	PTEXT,		%xmm1
+	.set	RNDKEY,		%xmm2	// Temporary register for _do_aes
 	_prologue	uses_arg3=2, uses_arg4=2
 
 	movdqu		(IV_PTR), AESDATA0
@@ -484,16 +485,16 @@ SYM_FUNC_START(aes_cbc_decrypt_aesni)
 
 	// Other local variables
 #ifdef __x86_64__
-	.set	RNDKEY_PTR,	%r9
+	.set	RNDKEY_PTR,	%r9	// Temporary register for _do_aes
 #else
-	.set	RNDKEY_PTR,	IV_PTR // IV_PTR is clobbered and reloaded later
+	.set	RNDKEY_PTR,	IV_PTR	// IV_PTR is clobbered and reloaded later
 #endif
-	.set	NROUNDS,	TMP_32
+	.set	NROUNDS,	TMP_32	// Temporary register for _do_aes
 	.set	AESDATA0,	%xmm0
 	.set	AESDATA1,	%xmm1
 	.set	AESDATA2,	%xmm2
 	.set	AESDATA3,	%xmm3
-	.set	RNDKEY,		%xmm4
+	.set	RNDKEY,		%xmm4	// Temporary register for _do_aes
 	.set	IV,		%xmm5
 	.set	CTEXT0,		%xmm6
 	.set	CTEXT1,		%xmm7
@@ -575,7 +576,7 @@ SYM_FUNC_END(aes_cbc_decrypt_aesni)
 	.set	LSHIFT_MASK,	%xmm1 // [0x80, 0x80, ...] + range(PN_LEN)
 	.set	AESDATA0,	%xmm2
 	.set	AESDATA1,	%xmm3
-	.set	RNDKEY,		%xmm4 // Temporary register for _do_aes
+	.set	RNDKEY,		%xmm4	// Temporary register for _do_aes
 	.set	IV,		%xmm5
 
 	_prologue	uses_arg3=2, uses_arg4=2
@@ -674,7 +675,7 @@ SYM_FUNC_START(aes_ctr64_crypt_aesni)
 	.set	LE_CTR,		%xmm4	// Current 128-bit little endian counter
 	.set	LE_CTR_INC,	%xmm5	// Initialized to (u64[])[1, 0]
 	.set	BSWAP_MASK,	%xmm6
-	.set	RNDKEY,		%xmm7
+	.set	RNDKEY,		%xmm7	// Temporary register for _do_aes
 	.set	RNDKEY_PTR,	LE_CTR_PTR  // Temporary register for _do_aes
 	.set	NROUNDS,	TMP_32	// Temporary register for _do_aes
 
@@ -795,17 +796,17 @@ SYM_FUNC_END(aes_ctr64_crypt_aesni)
 
 	// Other local variables
 #ifdef __x86_64__
-	.set	RNDKEY_PTR,	%r9
+	.set	RNDKEY_PTR,	%r9	// Temporary register for _do_aes
 #else
 	.set	RNDKEY_PTR,	TWEAK_PTR // TWEAK_PTR is clobbered and reloaded later.
 #endif
-	.set	NROUNDS,	TMP_32
+	.set	NROUNDS,	TMP_32	// Temporary register for _do_aes
 	.set	AESDATA0,	%xmm0
 	.set	AESDATA1,	%xmm1
 	.set	AESDATA2,	%xmm2
 	.set	AESDATA3,	%xmm3
 	.set	GF_POLY,	%xmm4
-	.set	RNDKEY,		%xmm5
+	.set	RNDKEY,		%xmm5	// Temporary register for _do_aes
 	.set	TWEAK,		%xmm6
 	.set	SAVED_TWEAK0,	%xmm7
 #ifdef __x86_64__
-- 
2.55.0

[PATCH 2/3] crypto: aes - Boost priority of encryption modes further

[Eric Biggers]

Fix a regression where drivers/crypto/ccp/ (priority 300) and
drivers/crypto/padlock-aes.c (priority 400) started being able to
override the AES-NI and/or VAES optimized code (previously priority
400-800).

Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
 crypto/aes.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/crypto/aes.c b/crypto/aes.c
index fe1f68a38daa..73f3aa856be4 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -611,7 +611,7 @@ static struct skcipher_alg skcipher_algs[] = {
 		.base.cra_name = "ecb(aes)",
 		.base.cra_driver_name = "ecb-aes-lib",
 		.base.cra_priority = (IS_ENABLED(CONFIG_RISCV) ||
-				      IS_ENABLED(CONFIG_X86)) ? 300 : 110,
+				      IS_ENABLED(CONFIG_X86)) ? 500 : 110,
 		.base.cra_blocksize = AES_BLOCK_SIZE,
 		.base.cra_ctxsize = sizeof(struct aes_key),
 		.base.cra_module = THIS_MODULE,
@@ -627,7 +627,7 @@ static struct skcipher_alg skcipher_algs[] = {
 		.base.cra_name = "cbc(aes)",
 		.base.cra_driver_name = "cbc-aes-lib",
 		.base.cra_priority = (IS_ENABLED(CONFIG_RISCV) ||
-				      IS_ENABLED(CONFIG_X86)) ? 300 : 110,
+				      IS_ENABLED(CONFIG_X86)) ? 500 : 110,
 		.base.cra_blocksize = AES_BLOCK_SIZE,
 		.base.cra_ctxsize = sizeof(struct aes_key),
 		.base.cra_module = THIS_MODULE,
@@ -655,7 +655,7 @@ static struct skcipher_alg skcipher_algs[] = {
 		.base.cra_name = "cts(cbc(aes))",
 		.base.cra_driver_name = "cts-cbc-aes-lib",
 		.base.cra_priority = (IS_ENABLED(CONFIG_RISCV) ||
-				      IS_ENABLED(CONFIG_X86)) ? 300 : 110,
+				      IS_ENABLED(CONFIG_X86)) ? 500 : 110,
 		.base.cra_blocksize = AES_BLOCK_SIZE,
 		.base.cra_ctxsize = sizeof(struct aes_key),
 		.base.cra_module = THIS_MODULE,
@@ -672,7 +672,7 @@ static struct skcipher_alg skcipher_algs[] = {
 		.base.cra_name = "ctr(aes)",
 		.base.cra_driver_name = "ctr-aes-lib",
 		.base.cra_priority = (IS_ENABLED(CONFIG_RISCV) ||
-				      IS_ENABLED(CONFIG_X86)) ? 300 : 110,
+				      IS_ENABLED(CONFIG_X86)) ? 500 : 110,
 		.base.cra_blocksize = 1,
 		.base.cra_ctxsize = sizeof(struct aes_enckey),
 		.base.cra_module = THIS_MODULE,
@@ -689,7 +689,7 @@ static struct skcipher_alg skcipher_algs[] = {
 	{
 		.base.cra_name = "xctr(aes)",
 		.base.cra_driver_name = "xctr-aes-lib",
-		.base.cra_priority = IS_ENABLED(CONFIG_X86) ? 300 : 110,
+		.base.cra_priority = IS_ENABLED(CONFIG_X86) ? 500 : 110,
 		.base.cra_blocksize = 1,
 		.base.cra_ctxsize = sizeof(struct aes_enckey),
 		.base.cra_module = THIS_MODULE,
@@ -713,7 +713,7 @@ static struct skcipher_alg skcipher_algs[] = {
 		.base.cra_name = "xts(aes)",
 		.base.cra_driver_name = "xts-aes-lib",
 		.base.cra_priority = (IS_ENABLED(CONFIG_RISCV) ||
-				      IS_ENABLED(CONFIG_X86)) ? 300 : 110,
+				      IS_ENABLED(CONFIG_X86)) ? 500 : 110,
 		.base.cra_blocksize = AES_BLOCK_SIZE,
 		.base.cra_ctxsize = sizeof(struct aes_xts_key),
 		.base.cra_module = THIS_MODULE,
-- 
2.55.0

[PATCH 3/3] crypto: x86/aesni-intel - Add transitional selections

[Eric Biggers]

Make CRYPTO_AES_NI_INTEL select CRYPTO_AES, CRYPTO_CBC, and CRYPTO_XTS
so that systems with CRYPTO_AES_NI_INTEL enabled are guaranteed to still
get the algorithm needed for their systems to boot with LUKS / dm-crypt.

(Later, when CRYPTO_AES_NI_INTEL is removed entirely, I plan to turn it
into a 'transitional' symbol with these selections.)

While compatibility selections haven't been used for previous migrations
of architecture-optimized code into lib/crypto/, I think it's worthwhile
in this particular case.

Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
 arch/x86/crypto/Kconfig | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig
index c1842c2d7055..20f5e2a17e45 100644
--- a/arch/x86/crypto/Kconfig
+++ b/arch/x86/crypto/Kconfig
@@ -9,6 +9,9 @@ config CRYPTO_AES_NI_INTEL
 	select CRYPTO_LIB_AES
 	select CRYPTO_LIB_GF128MUL
 	select CRYPTO_SKCIPHER
+	select CRYPTO_AES
+	select CRYPTO_CBC
+	select CRYPTO_XTS
 	help
 	  AEAD cipher: AES with GCM
 
@@ -16,8 +19,13 @@ config CRYPTO_AES_NI_INTEL
 	  - AES-NI (AES new instructions)
 	  - VAES (Vector AES)
 
-	  Note: this option no longer provides the accelerated XTS, CBC, CTR,
-	  and ECB code.  For those just use CRYPTO_XTS, CRYPTO_CBC, etc.
+	  The module controlled by this option (aesni-intel.ko) no longer
+	  directly provides the accelerated ECB, CBC, CBC-CTS, CTR, XCTR, and
+	  XTS code.  Those are now provided by CRYPTO_AES (aes.ko) when the
+	  corresponding mode option (CRYPTO_XTS, CRYPTO_CBC, etc.) is enabled.
+
+	  To ensure dm-crypt continues to work in kconfigs with only this option
+	  set, this option selects CRYPTO_AES, CRYPTO_CBC, and CRYPTO_XTS.
 
 config CRYPTO_BLOWFISH_X86_64
 	tristate "Ciphers: Blowfish, modes: ECB, CBC"
-- 
2.55.0