From: Robert Elliott <elliott@hpe.com>
To: herbert@gondor.apana.org.au, davem@davemloft.net,
Jason@zx2c4.com, ardb@kernel.org, ap420073@gmail.com,
David.Laight@ACULAB.COM, ebiggers@kernel.org,
tim.c.chen@linux.intel.com, peter@n8pjl.ca, tglx@linutronix.de,
mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com
Cc: linux-crypto@vger.kernel.org, x86@kernel.org,
linux-kernel@vger.kernel.org, Robert Elliott <elliott@hpe.com>
Subject: [PATCH 11/13] crypto: x86/blake - yield FPU context only when needed
Date: Mon, 19 Dec 2022 16:02:21 -0600 [thread overview]
Message-ID: <20221219220223.3982176-12-elliott@hpe.com> (raw)
In-Reply-To: <20221219220223.3982176-1-elliott@hpe.com>
The x86 assembly language implementations using SIMD process data
between kernel_fpu_begin() and kernel_fpu_end() calls. That
disables scheduler preemption, so prevents the CPU core from being
used by other threads.
The update() and finup() functions might be called to process
large quantities of data, which can result in RCU stalls and
soft lockups.
Rather than break the processing into 4 KiB passes, each of which
unilaterally calls kernel_fpu_begin() and kernel_fpu_end(),
periodically check if the kernel scheduler wants to run something
else on the CPU. If so, yield the kernel FPU context and let the
scheduler intervene.
Adjust the type of the length arguments everywhere to be unsigned
long rather than size_t to avoid typecasts.
Suggested-by: Herbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: Robert Elliott <elliott@hpe.com>
---
arch/x86/crypto/blake2s-glue.c | 41 ++++++++++++++++---------------
include/crypto/internal/blake2s.h | 8 +++---
lib/crypto/blake2s-generic.c | 12 ++++-----
3 files changed, 31 insertions(+), 30 deletions(-)
diff --git a/arch/x86/crypto/blake2s-glue.c b/arch/x86/crypto/blake2s-glue.c
index aaba21230528..bbb0a67ebb1c 100644
--- a/arch/x86/crypto/blake2s-glue.c
+++ b/arch/x86/crypto/blake2s-glue.c
@@ -12,46 +12,47 @@
#include <linux/sizes.h>
#include <asm/cpufeature.h>
-#include <asm/fpu/api.h>
#include <asm/processor.h>
#include <asm/simd.h>
-asmlinkage void blake2s_compress_ssse3(struct blake2s_state *state,
- const u8 *block, const size_t nblocks,
- const u32 inc);
-asmlinkage void blake2s_compress_avx512(struct blake2s_state *state,
- const u8 *block, const size_t nblocks,
- const u32 inc);
+asmlinkage void blake2s_compress_ssse3(struct blake2s_state *state, const u8 *data,
+ unsigned int nblocks, u32 inc);
+asmlinkage void blake2s_compress_avx512(struct blake2s_state *state, const u8 *data,
+ unsigned int nblocks, u32 inc);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_ssse3);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_avx512);
-void blake2s_compress(struct blake2s_state *state, const u8 *block,
- size_t nblocks, const u32 inc)
+void blake2s_compress(struct blake2s_state *state, const u8 *data,
+ unsigned int nblocks, const u32 inc)
{
/* SIMD disables preemption, so relax after processing each page. */
BUILD_BUG_ON(SZ_4K / BLAKE2S_BLOCK_SIZE < 8);
if (!static_branch_likely(&blake2s_use_ssse3) || !may_use_simd()) {
- blake2s_compress_generic(state, block, nblocks, inc);
+ blake2s_compress_generic(state, data, nblocks, inc);
return;
}
- do {
- const size_t blocks = min_t(size_t, nblocks,
- SZ_4K / BLAKE2S_BLOCK_SIZE);
+ kernel_fpu_begin();
+ for (;;) {
+ const unsigned int chunks = min(nblocks, 4096U / BLAKE2S_BLOCK_SIZE);
- kernel_fpu_begin();
if (IS_ENABLED(CONFIG_AS_AVX512) &&
static_branch_likely(&blake2s_use_avx512))
- blake2s_compress_avx512(state, block, blocks, inc);
+ blake2s_compress_avx512(state, data, chunks, inc);
else
- blake2s_compress_ssse3(state, block, blocks, inc);
- kernel_fpu_end();
+ blake2s_compress_ssse3(state, data, chunks, inc);
- nblocks -= blocks;
- block += blocks * BLAKE2S_BLOCK_SIZE;
- } while (nblocks);
+ nblocks -= chunks;
+
+ if (!nblocks)
+ break;
+
+ data += chunks * BLAKE2S_BLOCK_SIZE;
+ kernel_fpu_yield();
+ }
+ kernel_fpu_end();
}
EXPORT_SYMBOL(blake2s_compress);
diff --git a/include/crypto/internal/blake2s.h b/include/crypto/internal/blake2s.h
index 506d56530ca9..d6df791e6148 100644
--- a/include/crypto/internal/blake2s.h
+++ b/include/crypto/internal/blake2s.h
@@ -10,11 +10,11 @@
#include <crypto/blake2s.h>
#include <linux/string.h>
-void blake2s_compress_generic(struct blake2s_state *state, const u8 *block,
- size_t nblocks, const u32 inc);
+void blake2s_compress_generic(struct blake2s_state *state, const u8 *data,
+ unsigned int nblocks, u32 inc);
-void blake2s_compress(struct blake2s_state *state, const u8 *block,
- size_t nblocks, const u32 inc);
+void blake2s_compress(struct blake2s_state *state, const u8 *data,
+ unsigned int nblocks, u32 inc);
bool blake2s_selftest(void);
diff --git a/lib/crypto/blake2s-generic.c b/lib/crypto/blake2s-generic.c
index 75ccb3e633e6..6a1caa702698 100644
--- a/lib/crypto/blake2s-generic.c
+++ b/lib/crypto/blake2s-generic.c
@@ -37,12 +37,12 @@ static inline void blake2s_increment_counter(struct blake2s_state *state,
state->t[1] += (state->t[0] < inc);
}
-void blake2s_compress(struct blake2s_state *state, const u8 *block,
- size_t nblocks, const u32 inc)
+void blake2s_compress(struct blake2s_state *state, const u8 *data,
+ unsigned int nblocks, u32 inc)
__weak __alias(blake2s_compress_generic);
-void blake2s_compress_generic(struct blake2s_state *state, const u8 *block,
- size_t nblocks, const u32 inc)
+void blake2s_compress_generic(struct blake2s_state *state, const u8 *data,
+ unsigned int nblocks, u32 inc)
{
u32 m[16];
u32 v[16];
@@ -53,7 +53,7 @@ void blake2s_compress_generic(struct blake2s_state *state, const u8 *block,
while (nblocks > 0) {
blake2s_increment_counter(state, inc);
- memcpy(m, block, BLAKE2S_BLOCK_SIZE);
+ memcpy(m, data, BLAKE2S_BLOCK_SIZE);
le32_to_cpu_array(m, ARRAY_SIZE(m));
memcpy(v, state->h, 32);
v[ 8] = BLAKE2S_IV0;
@@ -103,7 +103,7 @@ void blake2s_compress_generic(struct blake2s_state *state, const u8 *block,
for (i = 0; i < 8; ++i)
state->h[i] ^= v[i] ^ v[i + 8];
- block += BLAKE2S_BLOCK_SIZE;
+ data += BLAKE2S_BLOCK_SIZE;
--nblocks;
}
}
--
2.38.1
next prev parent reply other threads:[~2022-12-19 22:03 UTC|newest]
Thread overview: 24+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-12-19 22:02 [PATCH 00/13] crypto: x86 - yield FPU context during long loops Robert Elliott
2022-12-19 22:02 ` [PATCH 01/13] x86: protect simd.h header file Robert Elliott
2022-12-19 22:02 ` [PATCH 02/13] x86: add yield FPU context utility function Robert Elliott
2022-12-19 22:02 ` [PATCH 03/13] crypto: x86/sha - yield FPU context during long loops Robert Elliott
2022-12-20 3:57 ` Herbert Xu
2022-12-30 9:08 ` Herbert Xu
2023-01-12 8:05 ` Herbert Xu
2023-01-12 18:46 ` Eric Biggers
2023-01-13 2:36 ` Herbert Xu
2023-01-13 2:37 ` Herbert Xu
2023-01-13 19:35 ` Elliott, Robert (Servers)
2023-01-16 3:33 ` Herbert Xu
2022-12-19 22:02 ` [PATCH 04/13] crypto: x86/crc " Robert Elliott
2022-12-19 22:02 ` [PATCH 05/13] crypto: x86/sm3 " Robert Elliott
2022-12-19 22:02 ` [PATCH 06/13] crypto: x86/ghash - use u8 rather than char Robert Elliott
2022-12-19 22:02 ` [PATCH 07/13] crypto: x86/ghash - restructure FPU context saving Robert Elliott
2022-12-30 9:04 ` Herbert Xu
2022-12-19 22:02 ` [PATCH 08/13] crypto: x86/ghash - yield FPU context during long loops Robert Elliott
2022-12-19 22:02 ` [PATCH 09/13] crypto: x86/poly - yield FPU context only when needed Robert Elliott
2022-12-19 22:02 ` [PATCH 10/13] crypto: x86/aegis - yield FPU context during long loops Robert Elliott
2022-12-19 22:02 ` Robert Elliott [this message]
2022-12-19 22:02 ` [PATCH 12/13] crypto: x86/chacha - yield FPU context only when needed Robert Elliott
2022-12-19 22:02 ` [PATCH 13/13] crypto: x86/aria " Robert Elliott
2022-12-20 20:02 ` [PATCH 00/13] crypto: x86 - yield FPU context during long loops Eric Biggers
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20221219220223.3982176-12-elliott@hpe.com \
--to=elliott@hpe.com \
--cc=David.Laight@ACULAB.COM \
--cc=Jason@zx2c4.com \
--cc=ap420073@gmail.com \
--cc=ardb@kernel.org \
--cc=bp@alien8.de \
--cc=dave.hansen@linux.intel.com \
--cc=davem@davemloft.net \
--cc=ebiggers@kernel.org \
--cc=herbert@gondor.apana.org.au \
--cc=linux-crypto@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=mingo@redhat.com \
--cc=peter@n8pjl.ca \
--cc=tglx@linutronix.de \
--cc=tim.c.chen@linux.intel.com \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®