mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: Robert Elliott <elliott@hpe.com>
To: herbert@gondor.apana.org.au, davem@davemloft.net,
	Jason@zx2c4.com, ardb@kernel.org, ap420073@gmail.com,
	David.Laight@ACULAB.COM, ebiggers@kernel.org,
	tim.c.chen@linux.intel.com, peter@n8pjl.ca, tglx@linutronix.de,
	mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com
Cc: linux-crypto@vger.kernel.org, x86@kernel.org,
	linux-kernel@vger.kernel.org, Robert Elliott <elliott@hpe.com>
Subject: [PATCH 10/13] crypto: x86/aegis - yield FPU context during long loops
Date: Mon, 19 Dec 2022 16:02:20 -0600	[thread overview]
Message-ID: <20221219220223.3982176-11-elliott@hpe.com> (raw)
In-Reply-To: <20221219220223.3982176-1-elliott@hpe.com>

Make kernel_fpu_begin() and kernel_fpu_end() calls around each
assembly language function that uses FPU context, rather than
around the entire set (init, ad, crypt, final).

During encryption, periodically check if the kernel scheduler
wants to run something else on the CPU. If so, yield the kernel
FPU context and let the scheduler intervene.  Associated data is
not limited.

Allow the skcipher_walk functions to sleep again, since they are
no longer called inside FPU context.

Fixes: 1d373d4e8e15 ("crypto: x86 - Add optimized AEGIS implementations")
Fixes: ba6771c0a0bc ("crypto: x86/aegis - fix handling chunked inputs and MAY_SLEEP")
Signed-off-by: Robert Elliott <elliott@hpe.com>
---
 arch/x86/crypto/aegis128-aesni-glue.c | 49 ++++++++++++++++++++-------
 1 file changed, 36 insertions(+), 13 deletions(-)

diff --git a/arch/x86/crypto/aegis128-aesni-glue.c b/arch/x86/crypto/aegis128-aesni-glue.c
index 4623189000d8..f99f3e597b3c 100644
--- a/arch/x86/crypto/aegis128-aesni-glue.c
+++ b/arch/x86/crypto/aegis128-aesni-glue.c
@@ -12,8 +12,8 @@
 #include <crypto/internal/skcipher.h>
 #include <crypto/scatterwalk.h>
 #include <linux/module.h>
-#include <asm/fpu/api.h>
 #include <asm/cpu_device_id.h>
+#include <asm/simd.h>
 
 #define AEGIS128_BLOCK_ALIGN 16
 #define AEGIS128_BLOCK_SIZE 16
@@ -85,15 +85,19 @@ static void crypto_aegis128_aesni_process_ad(
 			if (pos > 0) {
 				unsigned int fill = AEGIS128_BLOCK_SIZE - pos;
 				memcpy(buf.bytes + pos, src, fill);
-				crypto_aegis128_aesni_ad(state,
+				kernel_fpu_begin();
+				crypto_aegis128_aesni_ad(state->blocks,
 							 AEGIS128_BLOCK_SIZE,
 							 buf.bytes);
+				kernel_fpu_end();
 				pos = 0;
 				left -= fill;
 				src += fill;
 			}
 
-			crypto_aegis128_aesni_ad(state, left, src);
+			kernel_fpu_begin();
+			crypto_aegis128_aesni_ad(state->blocks, left, src);
+			kernel_fpu_end();
 
 			src += left & ~(AEGIS128_BLOCK_SIZE - 1);
 			left &= AEGIS128_BLOCK_SIZE - 1;
@@ -110,7 +114,9 @@ static void crypto_aegis128_aesni_process_ad(
 
 	if (pos > 0) {
 		memset(buf.bytes + pos, 0, AEGIS128_BLOCK_SIZE - pos);
-		crypto_aegis128_aesni_ad(state, AEGIS128_BLOCK_SIZE, buf.bytes);
+		kernel_fpu_begin();
+		crypto_aegis128_aesni_ad(state->blocks, AEGIS128_BLOCK_SIZE, buf.bytes);
+		kernel_fpu_end();
 	}
 }
 
@@ -118,16 +124,31 @@ static void crypto_aegis128_aesni_process_crypt(
 		struct aegis_state *state, struct skcipher_walk *walk,
 		const struct aegis_crypt_ops *ops)
 {
-	while (walk->nbytes >= AEGIS128_BLOCK_SIZE) {
-		ops->crypt_blocks(state,
-				  round_down(walk->nbytes, AEGIS128_BLOCK_SIZE),
-				  walk->src.virt.addr, walk->dst.virt.addr);
-		skcipher_walk_done(walk, walk->nbytes % AEGIS128_BLOCK_SIZE);
+	if (walk->nbytes >= AEGIS128_BLOCK_SIZE) {
+		kernel_fpu_begin();
+		for (;;) {
+			unsigned int chunk = min(walk->nbytes, 4096U);
+
+			chunk = round_down(chunk, AEGIS128_BLOCK_SIZE);
+
+			ops->crypt_blocks(state->blocks, chunk,
+					  walk->src.virt.addr, walk->dst.virt.addr);
+
+			skcipher_walk_done(walk, walk->nbytes - chunk);
+
+			if (walk->nbytes < AEGIS128_BLOCK_SIZE)
+				break;
+
+			kernel_fpu_yield();
+		}
+		kernel_fpu_end();
 	}
 
 	if (walk->nbytes) {
-		ops->crypt_tail(state, walk->nbytes, walk->src.virt.addr,
+		kernel_fpu_begin();
+		ops->crypt_tail(state->blocks, walk->nbytes, walk->src.virt.addr,
 				walk->dst.virt.addr);
+		kernel_fpu_end();
 		skcipher_walk_done(walk, 0);
 	}
 }
@@ -172,15 +193,17 @@ static void crypto_aegis128_aesni_crypt(struct aead_request *req,
 	struct skcipher_walk walk;
 	struct aegis_state state;
 
-	ops->skcipher_walk_init(&walk, req, true);
+	ops->skcipher_walk_init(&walk, req, false);
 
 	kernel_fpu_begin();
+	crypto_aegis128_aesni_init(&state.blocks, ctx->key.bytes, req->iv);
+	kernel_fpu_end();
 
-	crypto_aegis128_aesni_init(&state, ctx->key.bytes, req->iv);
 	crypto_aegis128_aesni_process_ad(&state, req->src, req->assoclen);
 	crypto_aegis128_aesni_process_crypt(&state, &walk, ops);
-	crypto_aegis128_aesni_final(&state, tag_xor, req->assoclen, cryptlen);
 
+	kernel_fpu_begin();
+	crypto_aegis128_aesni_final(&state.blocks, tag_xor, req->assoclen, cryptlen);
 	kernel_fpu_end();
 }
 
-- 
2.38.1


  parent reply	other threads:[~2022-12-19 22:03 UTC|newest]

Thread overview: 24+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2022-12-19 22:02 [PATCH 00/13] crypto: x86 " Robert Elliott
2022-12-19 22:02 ` [PATCH 01/13] x86: protect simd.h header file Robert Elliott
2022-12-19 22:02 ` [PATCH 02/13] x86: add yield FPU context utility function Robert Elliott
2022-12-19 22:02 ` [PATCH 03/13] crypto: x86/sha - yield FPU context during long loops Robert Elliott
2022-12-20  3:57   ` Herbert Xu
2022-12-30  9:08   ` Herbert Xu
2023-01-12  8:05   ` Herbert Xu
2023-01-12 18:46     ` Eric Biggers
2023-01-13  2:36       ` Herbert Xu
2023-01-13  2:37         ` Herbert Xu
2023-01-13 19:35           ` Elliott, Robert (Servers)
2023-01-16  3:33             ` Herbert Xu
2022-12-19 22:02 ` [PATCH 04/13] crypto: x86/crc " Robert Elliott
2022-12-19 22:02 ` [PATCH 05/13] crypto: x86/sm3 " Robert Elliott
2022-12-19 22:02 ` [PATCH 06/13] crypto: x86/ghash - use u8 rather than char Robert Elliott
2022-12-19 22:02 ` [PATCH 07/13] crypto: x86/ghash - restructure FPU context saving Robert Elliott
2022-12-30  9:04   ` Herbert Xu
2022-12-19 22:02 ` [PATCH 08/13] crypto: x86/ghash - yield FPU context during long loops Robert Elliott
2022-12-19 22:02 ` [PATCH 09/13] crypto: x86/poly - yield FPU context only when needed Robert Elliott
2022-12-19 22:02 ` Robert Elliott [this message]
2022-12-19 22:02 ` [PATCH 11/13] crypto: x86/blake " Robert Elliott
2022-12-19 22:02 ` [PATCH 12/13] crypto: x86/chacha " Robert Elliott
2022-12-19 22:02 ` [PATCH 13/13] crypto: x86/aria " Robert Elliott
2022-12-20 20:02 ` [PATCH 00/13] crypto: x86 - yield FPU context during long loops Eric Biggers

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20221219220223.3982176-11-elliott@hpe.com \
    --to=elliott@hpe.com \
    --cc=David.Laight@ACULAB.COM \
    --cc=Jason@zx2c4.com \
    --cc=ap420073@gmail.com \
    --cc=ardb@kernel.org \
    --cc=bp@alien8.de \
    --cc=dave.hansen@linux.intel.com \
    --cc=davem@davemloft.net \
    --cc=ebiggers@kernel.org \
    --cc=herbert@gondor.apana.org.au \
    --cc=linux-crypto@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=mingo@redhat.com \
    --cc=peter@n8pjl.ca \
    --cc=tglx@linutronix.de \
    --cc=tim.c.chen@linux.intel.com \
    --cc=x86@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®