* [PATCH v2] lib/crc: arm64: Add NEON PMULL+EOR3 crc64_nvme implementation
@ 2026-11-14 17:29 Shreesh Adiga
2026-11-17 0:01 ` Eric Biggers
` (2 more replies)
0 siblings, 3 replies; 7+ messages in thread
From: Shreesh Adiga @ 2026-11-14 17:29 UTC (permalink / raw)
To: Eric Biggers, Ard Biesheuvel, Demian Shulhan; +Cc: linux-crypto, linux-kernel
Add a 4x-folding implementation for CPUs with FEAT_SHA3 which benefits
newer ARMv9 CPUs like Cortex-X925.
Usage of EOR3 from SHA3 extension improves performance by 20% compared
to the regular PMULL without EOR3 on Cortex-X925 measure in userspace
application via libbenchmark for 512 byte input size.
Measured about 3x speedup for 512 byte input compared with the existing
NEON single fold implementation on Cortex-X925 in userspace.
Signed-off-by: Shreesh Adiga <16567adigashreesh@gmail.com>
---
Changes in v2:
Improved the 4x to 1x reduction by using x^255 and x^319 x2 reduction.
Added static keys for the CPU feature checks instead of regular
branches.
lib/crc/arm64/crc64.h | 24 +++++++++++--
lib/crc/crc64-neon.c | 83 +++++++++++++++++++++++++++++++++++++++++++
2 files changed, 104 insertions(+), 3 deletions(-)
diff --git a/lib/crc/arm64/crc64.h b/lib/crc/arm64/crc64.h
index c7a69e1f3d8f..e5be0ef21140 100644
--- a/lib/crc/arm64/crc64.h
+++ b/lib/crc/arm64/crc64.h
@@ -9,20 +9,38 @@
#include <linux/sizes.h>
u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len);
+u64 crc64_nvme_neon_pmull_eor3(u64 crc, const u8 *p, size_t len);
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_eor3);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
#define crc64_be_arch crc64_be_generic
static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len)
{
- if (len >= 128 && cpu_have_named_feature(PMULL) &&
+ if (len >= 128 && static_branch_likely(&have_pmull) &&
likely(may_use_simd())) {
size_t chunk = len & ~15;
- scoped_ksimd()
- crc = crc64_nvme_neon(crc, p, chunk);
+ scoped_ksimd() {
+ if (static_branch_likely(&have_eor3))
+ crc = crc64_nvme_neon_pmull_eor3(crc, p, chunk);
+ else
+ crc = crc64_nvme_neon(crc, p, chunk);
+ }
p += chunk;
len &= 15;
}
return crc64_nvme_generic(crc, p, len);
}
+
+#define crc64_mod_init_arch crc64_mod_init_arch
+static void crc64_mod_init_arch(void)
+{
+ if (cpu_have_named_feature(PMULL)) {
+ static_branch_enable(&have_pmull);
+ if (cpu_have_named_feature(SHA3))
+ static_branch_enable(&have_eor3);
+ }
+}
diff --git a/lib/crc/crc64-neon.c b/lib/crc/crc64-neon.c
index 4753fb94a4be..767e55edc0e0 100644
--- a/lib/crc/crc64-neon.c
+++ b/lib/crc/crc64-neon.c
@@ -10,6 +10,17 @@
u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len);
+#ifdef CONFIG_ARM64
+u64 crc64_nvme_neon_pmull_eor3(u64 crc, const u8 *p, size_t len);
+
+/* x^575 mod G, x^511 mod G */
+static const u64 fold_consts_4x_val[2] = { 0x0c32cdb31e18a84aULL,
+ 0x62242240ace5045aULL };
+/* x^319 mod G, x^255 mod G */
+static const u64 fold_consts_2x_val[2] = { 0xb0bc2e589204f500ULL,
+ 0xe1e0bb9d45d7a44cULL };
+#endif
+
/* x^191 mod G, x^127 mod G */
static const u64 fold_consts_val[2] = { 0xeadc41fd2ba3d420ULL,
0x21e9761e252621acULL };
@@ -45,3 +56,75 @@ u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len)
return vgetq_lane_u64(v0, 1);
}
+
+#ifdef CONFIG_ARM64
+#define SHA3_TARGET __attribute__((target("arch=armv8.2-a+crypto+sha3")))
+
+SHA3_TARGET
+static inline uint64x2_t fold_single_round(uint64x2_t fold,
+ uint64x2_t temp,
+ uint64x2_t consts)
+{
+ uint64x2_t temp1 = pmull64(consts, fold);
+ uint64x2_t temp2 = pmull64_high(consts, fold);
+
+ return veor3q_u64(temp, temp1, temp2);
+}
+
+SHA3_TARGET
+u64 crc64_nvme_neon_pmull_eor3(u64 crc, const u8 *p, size_t len)
+{
+ /* Assumes len >= 64. */
+ uint64x2_t fold_consts = vld1q_u64(fold_consts_val);
+ uint64x2_t fold_consts_4x = vld1q_u64(fold_consts_4x_val);
+ uint64x2_t fold_consts_2x = vld1q_u64(fold_consts_2x_val);
+ uint64x2_t v0 = { crc, 0 };
+ uint64x2_t zero = { };
+ uint64x2_t v1, v2, v3;
+
+ v0 ^= vreinterpretq_u64_u8(vld1q_u8(p));
+ v1 = vreinterpretq_u64_u8(vld1q_u8(p + 16));
+ v2 = vreinterpretq_u64_u8(vld1q_u8(p + 32));
+ v3 = vreinterpretq_u64_u8(vld1q_u8(p + 48));
+
+ p += 64;
+ len -= 64;
+ while (len >= 64) {
+ uint64x2_t t0 = vreinterpretq_u64_u8(vld1q_u8(p + 0));
+ uint64x2_t t1 = vreinterpretq_u64_u8(vld1q_u8(p + 16));
+ uint64x2_t t2 = vreinterpretq_u64_u8(vld1q_u8(p + 32));
+ uint64x2_t t3 = vreinterpretq_u64_u8(vld1q_u8(p + 48));
+
+ v0 = fold_single_round(v0, t0, fold_consts_4x);
+ v1 = fold_single_round(v1, t1, fold_consts_4x);
+ v2 = fold_single_round(v2, t2, fold_consts_4x);
+ v3 = fold_single_round(v3, t3, fold_consts_4x);
+ p += 64;
+ len -= 64;
+ }
+
+ v0 = fold_single_round(v0, v2, fold_consts_2x);
+ v1 = fold_single_round(v1, v3, fold_consts_2x);
+ v0 = fold_single_round(v0, v1, fold_consts);
+
+ while (len >= 16) {
+ uint64x2_t t0 = vreinterpretq_u64_u8(vld1q_u8(p));
+
+ v0 = fold_single_round(v0, t0, fold_consts);
+ p += 16;
+ len -= 16;
+ }
+
+ /* Multiply the 128-bit value by x^64 and reduce it back to 128 bits. */
+ v0 = vextq_u64(v0, zero, 1) ^ pmull64_hi_lo(fold_consts, v0);
+
+ /* Final Barrett reduction */
+ uint64x2_t bconsts = vld1q_u64(bconsts_val);
+ uint64x2_t final = pmull64(bconsts, v0);
+
+ v0 = veor3q_u64(v0, vextq_u64(zero, final, 1),
+ pmull64_hi_lo(bconsts, final));
+
+ return vgetq_lane_u64(v0, 1);
+}
+#endif
--
2.56.1
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH v2] lib/crc: arm64: Add NEON PMULL+EOR3 crc64_nvme implementation
2026-11-14 17:29 [PATCH v2] lib/crc: arm64: Add NEON PMULL+EOR3 crc64_nvme implementation Shreesh Adiga
@ 2026-11-17 0:01 ` Eric Biggers
2026-11-17 6:50 ` Shreesh Adiga
2026-11-17 0:45 ` Demian Shulhan
2026-11-17 9:00 ` David Laight
2 siblings, 1 reply; 7+ messages in thread
From: Eric Biggers @ 2026-11-17 0:01 UTC (permalink / raw)
To: Shreesh Adiga; +Cc: Ard Biesheuvel, Demian Shulhan, linux-crypto, linux-kernel
On Sat, Nov 14, 2026 at 10:59:51PM +0530, Shreesh Adiga wrote:
> Add a 4x-folding implementation for CPUs with FEAT_SHA3 which benefits
> newer ARMv9 CPUs like Cortex-X925.
> Usage of EOR3 from SHA3 extension improves performance by 20% compared
> to the regular PMULL without EOR3 on Cortex-X925 measure in userspace
> application via libbenchmark for 512 byte input size.
> Measured about 3x speedup for 512 byte input compared with the existing
> NEON single fold implementation on Cortex-X925 in userspace.
Was crc_kunit run? It already has a benchmark in it.
The commit message should also clarify that this benefits ARMv8.2 and
later processors with the sha3 extension, not just ARMv9. (E.g., Apple
M1 is an example of an ARMv8.4 processor that benefits.)
> +#ifdef CONFIG_ARM64
> +#define SHA3_TARGET __attribute__((target("arch=armv8.2-a+crypto+sha3")))
Doesn't this break the build with gcc 8.1? The kernel supports gcc 8.1
and later, but support for the sha3 target and intrinsics were added in
gcc 9.1. Either this code will need to check the gcc version, or inline
assembly will need to be used for eor3 like lib/raid/xor/ does.
> + v0 = fold_single_round(v0, v2, fold_consts_2x);
> + v1 = fold_single_round(v1, v3, fold_consts_2x);
> + v0 = fold_single_round(v0, v1, fold_consts);
> +
> + while (len >= 16) {
> + uint64x2_t t0 = vreinterpretq_u64_u8(vld1q_u8(p));
> +
> + v0 = fold_single_round(v0, t0, fold_consts);
> + p += 16;
> + len -= 16;
> + }
> +
> + /* Multiply the 128-bit value by x^64 and reduce it back to 128 bits. */
> + v0 = vextq_u64(v0, zero, 1) ^ pmull64_hi_lo(fold_consts, v0);
> +
> + /* Final Barrett reduction */
> + uint64x2_t bconsts = vld1q_u64(bconsts_val);
> + uint64x2_t final = pmull64(bconsts, v0);
> +
> + v0 = veor3q_u64(v0, vextq_u64(zero, final, 1),
> + pmull64_hi_lo(bconsts, final));
> +
> + return vgetq_lane_u64(v0, 1);
Note that this is missing several optimizations compared to the x86
version (lib/crc/x86/crc-pclmul-template.S), including using
fold_consts_2x for a tail of 'len & 32', efficiently handling lengths
that aren't a multiple of 16 bytes, and reducing to the final CRC in two
carryless multiplication instructions rather than three.
This is also just one of several CRC variants. This would have to be
done all over again for each variant if they are not going to be
generated by a template like x86 and riscv do.
So we can do this for now, but it isn't a very scalable solution.
- Eric
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH v2] lib/crc: arm64: Add NEON PMULL+EOR3 crc64_nvme implementation
2026-11-14 17:29 [PATCH v2] lib/crc: arm64: Add NEON PMULL+EOR3 crc64_nvme implementation Shreesh Adiga
2026-11-17 0:01 ` Eric Biggers
@ 2026-11-17 0:45 ` Demian Shulhan
2026-11-17 7:06 ` Shreesh Adiga
2026-11-17 9:00 ` David Laight
2 siblings, 1 reply; 7+ messages in thread
From: Demian Shulhan @ 2026-11-17 0:45 UTC (permalink / raw)
To: Shreesh Adiga; +Cc: Eric Biggers, Ard Biesheuvel, linux-crypto, linux-kernel
Hi Shreesh,
> +#define SHA3_TARGET __attribute__((target("arch=armv8.2-a+crypto+sha3")))
> +
> +SHA3_TARGET
> +static inline uint64x2_t fold_single_round(uint64x2_t fold,
> + uint64x2_t temp,
> + uint64x2_t consts)
> +{
> + uint64x2_t temp1 = pmull64(consts, fold);
> + uint64x2_t temp2 = pmull64_high(consts, fold);
> +
> + return veor3q_u64(temp, temp1, temp2);
> +}
> +
> +SHA3_TARGET
> +u64 crc64_nvme_neon_pmull_eor3(u64 crc, const u8 *p, size_t len)
> +{
> + /* Assumes len >= 64. */
> + uint64x2_t fold_consts = vld1q_u64(fold_consts_val);
> + uint64x2_t fold_consts_4x = vld1q_u64(fold_consts_4x_val);
> + uint64x2_t fold_consts_2x = vld1q_u64(fold_consts_2x_val);
Scalar vmull_p64() makes compilers materialize the constants via mov/movk + fmov
(42/97 insns on Clang). SHA3_TARGET also makes codegen fragile: with
-march=armv8.5-a+crypto, GCC stops inlining pmull64*(). On Neoverse-V2, EOR3
contends with PMULL on pipe V0, whereas regular EOR runs on all SIMD pipes.
Using inline-asm pmull/pmull2 and a clean 4x unroll in crc64_nvme_neon() without
EOR3 should give better performance across all sizes and cleaner code.
/* clang -O2 */
24: mov x10, #0x45a
28: movk x10, #0xace5, lsl #16
2c: movk x10, #0x2240, lsl #32
30: movk x10, #0x6224, lsl #48
34: dup v4.2d, x10
38: mov x10, #0xa84a
3c: movk x10, #0x1e18, lsl #16
40: movk x10, #0xcdb3, lsl #32
44: movk x10, #0xc32, lsl #48
48: fmov d5, x10
Thanks,
Demian
сб, 14 лист. 2026 р. о 18:30 Shreesh Adiga <16567adigashreesh@gmail.com> пише:
>
> Add a 4x-folding implementation for CPUs with FEAT_SHA3 which benefits
> newer ARMv9 CPUs like Cortex-X925.
> Usage of EOR3 from SHA3 extension improves performance by 20% compared
> to the regular PMULL without EOR3 on Cortex-X925 measure in userspace
> application via libbenchmark for 512 byte input size.
> Measured about 3x speedup for 512 byte input compared with the existing
> NEON single fold implementation on Cortex-X925 in userspace.
>
> Signed-off-by: Shreesh Adiga <16567adigashreesh@gmail.com>
> ---
> Changes in v2:
> Improved the 4x to 1x reduction by using x^255 and x^319 x2 reduction.
> Added static keys for the CPU feature checks instead of regular
> branches.
>
> lib/crc/arm64/crc64.h | 24 +++++++++++--
> lib/crc/crc64-neon.c | 83 +++++++++++++++++++++++++++++++++++++++++++
> 2 files changed, 104 insertions(+), 3 deletions(-)
>
> diff --git a/lib/crc/arm64/crc64.h b/lib/crc/arm64/crc64.h
> index c7a69e1f3d8f..e5be0ef21140 100644
> --- a/lib/crc/arm64/crc64.h
> +++ b/lib/crc/arm64/crc64.h
> @@ -9,20 +9,38 @@
> #include <linux/sizes.h>
>
> u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len);
> +u64 crc64_nvme_neon_pmull_eor3(u64 crc, const u8 *p, size_t len);
> +
> +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_eor3);
> +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
>
> #define crc64_be_arch crc64_be_generic
>
> static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len)
> {
> - if (len >= 128 && cpu_have_named_feature(PMULL) &&
> + if (len >= 128 && static_branch_likely(&have_pmull) &&
> likely(may_use_simd())) {
> size_t chunk = len & ~15;
>
> - scoped_ksimd()
> - crc = crc64_nvme_neon(crc, p, chunk);
> + scoped_ksimd() {
> + if (static_branch_likely(&have_eor3))
> + crc = crc64_nvme_neon_pmull_eor3(crc, p, chunk);
> + else
> + crc = crc64_nvme_neon(crc, p, chunk);
> + }
>
> p += chunk;
> len &= 15;
> }
> return crc64_nvme_generic(crc, p, len);
> }
> +
> +#define crc64_mod_init_arch crc64_mod_init_arch
> +static void crc64_mod_init_arch(void)
> +{
> + if (cpu_have_named_feature(PMULL)) {
> + static_branch_enable(&have_pmull);
> + if (cpu_have_named_feature(SHA3))
> + static_branch_enable(&have_eor3);
> + }
> +}
> diff --git a/lib/crc/crc64-neon.c b/lib/crc/crc64-neon.c
> index 4753fb94a4be..767e55edc0e0 100644
> --- a/lib/crc/crc64-neon.c
> +++ b/lib/crc/crc64-neon.c
> @@ -10,6 +10,17 @@
>
> u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len);
>
> +#ifdef CONFIG_ARM64
> +u64 crc64_nvme_neon_pmull_eor3(u64 crc, const u8 *p, size_t len);
> +
> +/* x^575 mod G, x^511 mod G */
> +static const u64 fold_consts_4x_val[2] = { 0x0c32cdb31e18a84aULL,
> + 0x62242240ace5045aULL };
> +/* x^319 mod G, x^255 mod G */
> +static const u64 fold_consts_2x_val[2] = { 0xb0bc2e589204f500ULL,
> + 0xe1e0bb9d45d7a44cULL };
> +#endif
> +
> /* x^191 mod G, x^127 mod G */
> static const u64 fold_consts_val[2] = { 0xeadc41fd2ba3d420ULL,
> 0x21e9761e252621acULL };
> @@ -45,3 +56,75 @@ u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len)
>
> return vgetq_lane_u64(v0, 1);
> }
> +
> +#ifdef CONFIG_ARM64
> +#define SHA3_TARGET __attribute__((target("arch=armv8.2-a+crypto+sha3")))
> +
> +SHA3_TARGET
> +static inline uint64x2_t fold_single_round(uint64x2_t fold,
> + uint64x2_t temp,
> + uint64x2_t consts)
> +{
> + uint64x2_t temp1 = pmull64(consts, fold);
> + uint64x2_t temp2 = pmull64_high(consts, fold);
> +
> + return veor3q_u64(temp, temp1, temp2);
> +}
> +
> +SHA3_TARGET
> +u64 crc64_nvme_neon_pmull_eor3(u64 crc, const u8 *p, size_t len)
> +{
> + /* Assumes len >= 64. */
> + uint64x2_t fold_consts = vld1q_u64(fold_consts_val);
> + uint64x2_t fold_consts_4x = vld1q_u64(fold_consts_4x_val);
> + uint64x2_t fold_consts_2x = vld1q_u64(fold_consts_2x_val);
> + uint64x2_t v0 = { crc, 0 };
> + uint64x2_t zero = { };
> + uint64x2_t v1, v2, v3;
> +
> + v0 ^= vreinterpretq_u64_u8(vld1q_u8(p));
> + v1 = vreinterpretq_u64_u8(vld1q_u8(p + 16));
> + v2 = vreinterpretq_u64_u8(vld1q_u8(p + 32));
> + v3 = vreinterpretq_u64_u8(vld1q_u8(p + 48));
> +
> + p += 64;
> + len -= 64;
> + while (len >= 64) {
> + uint64x2_t t0 = vreinterpretq_u64_u8(vld1q_u8(p + 0));
> + uint64x2_t t1 = vreinterpretq_u64_u8(vld1q_u8(p + 16));
> + uint64x2_t t2 = vreinterpretq_u64_u8(vld1q_u8(p + 32));
> + uint64x2_t t3 = vreinterpretq_u64_u8(vld1q_u8(p + 48));
> +
> + v0 = fold_single_round(v0, t0, fold_consts_4x);
> + v1 = fold_single_round(v1, t1, fold_consts_4x);
> + v2 = fold_single_round(v2, t2, fold_consts_4x);
> + v3 = fold_single_round(v3, t3, fold_consts_4x);
> + p += 64;
> + len -= 64;
> + }
> +
> + v0 = fold_single_round(v0, v2, fold_consts_2x);
> + v1 = fold_single_round(v1, v3, fold_consts_2x);
> + v0 = fold_single_round(v0, v1, fold_consts);
> +
> + while (len >= 16) {
> + uint64x2_t t0 = vreinterpretq_u64_u8(vld1q_u8(p));
> +
> + v0 = fold_single_round(v0, t0, fold_consts);
> + p += 16;
> + len -= 16;
> + }
> +
> + /* Multiply the 128-bit value by x^64 and reduce it back to 128 bits. */
> + v0 = vextq_u64(v0, zero, 1) ^ pmull64_hi_lo(fold_consts, v0);
> +
> + /* Final Barrett reduction */
> + uint64x2_t bconsts = vld1q_u64(bconsts_val);
> + uint64x2_t final = pmull64(bconsts, v0);
> +
> + v0 = veor3q_u64(v0, vextq_u64(zero, final, 1),
> + pmull64_hi_lo(bconsts, final));
> +
> + return vgetq_lane_u64(v0, 1);
> +}
> +#endif
> --
> 2.56.1
>
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH v2] lib/crc: arm64: Add NEON PMULL+EOR3 crc64_nvme implementation
2026-11-17 0:01 ` Eric Biggers
@ 2026-11-17 6:50 ` Shreesh Adiga
2026-11-17 7:11 ` Eric Biggers
0 siblings, 1 reply; 7+ messages in thread
From: Shreesh Adiga @ 2026-11-17 6:50 UTC (permalink / raw)
To: Eric Biggers; +Cc: Ard Biesheuvel, Demian Shulhan, linux-crypto, linux-kernel
Hi Eric,
On Tue, Nov 17, 2026 at 5:31 AM Eric Biggers <ebiggers@kernel.org> wrote:
> Was crc_kunit run? It already has a benchmark in it.
No, I have access to only one aarch64 device that is Android Mediatek
Dimensity 9400 SoC without root access.
Therefore I can't run crc_kunit on that, instead I have done userspace
comparisons for the same kernel with libbenchmark.
This is based on my earlier work for FFmpeg:
https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21651
I also have Corporate MacOS M5 and did check inside Ubuntu VM but that
is not accurate due to virtualization.
> The commit message should also clarify that this benefits ARMv8.2 and
> later processors with the sha3 extension, not just ARMv9. (E.g., Apple
> M1 is an example of an ARMv8.4 processor that benefits.)
Sure I will take care of it, but for now I am thinking of first
improving the existing PMULL to have 4x unrolling as per Demian's
suggestion.
Therefore I will revisit EOR3 addition later, will keep this in mind.
> > +#ifdef CONFIG_ARM64
> > +#define SHA3_TARGET __attribute__((target("arch=armv8.2-a+crypto+sha3")))
>
> Doesn't this break the build with gcc 8.1? The kernel supports gcc 8.1
> and later, but support for the sha3 target and intrinsics were added in
> gcc 9.1. Either this code will need to check the gcc version, or inline
> assembly will need to be used for eor3 like lib/raid/xor/ does.
Yes it seems to be unsupported even on GCC 8.2 and works on GCC 8.5
onwards, I wasn't aware of older toolchain support.
Will keep this in mind. Unfortunately compiler explorer is missing
ARM64 GCC 8.1, will check 8.2 support which hopefully should be
accurate.
Will adopt the existing code in lib/raid/xor itself which should make
it compatible with GCC 8.1.
> > + v0 = fold_single_round(v0, v2, fold_consts_2x);
> > + v1 = fold_single_round(v1, v3, fold_consts_2x);
> > + v0 = fold_single_round(v0, v1, fold_consts);
> > +
> > + while (len >= 16) {
> > + uint64x2_t t0 = vreinterpretq_u64_u8(vld1q_u8(p));
> > +
> > + v0 = fold_single_round(v0, t0, fold_consts);
> > + p += 16;
> > + len -= 16;
> > + }
> > +
> > + /* Multiply the 128-bit value by x^64 and reduce it back to 128 bits. */
> > + v0 = vextq_u64(v0, zero, 1) ^ pmull64_hi_lo(fold_consts, v0);
> > +
> > + /* Final Barrett reduction */
> > + uint64x2_t bconsts = vld1q_u64(bconsts_val);
> > + uint64x2_t final = pmull64(bconsts, v0);
> > +
> > + v0 = veor3q_u64(v0, vextq_u64(zero, final, 1),
> > + pmull64_hi_lo(bconsts, final));
> > +
> > + return vgetq_lane_u64(v0, 1);
>
> Note that this is missing several optimizations compared to the x86
> version (lib/crc/x86/crc-pclmul-template.S), including using
> fold_consts_2x for a tail of 'len & 32', efficiently handling lengths
> that aren't a multiple of 16 bytes, and reducing to the final CRC in two
> carryless multiplication instructions rather than three.
I was looking at the crc-pclmul-template.S code and I have a few queries.
In this aarch64 version we only dispatch to this SIMD routine only
when len>=128.
Since we have 128 bytes available, do we have to add the tail of `len
& 32` handling?
Handling len not a multiple of 16 bytes (partial block) I will add but
I am assuming that the dispatch will maintain 128 minimum len.
I was trying to understand your "reducing to the final CRC in two
carryless multiplications rather than three" statement.
From going through the code in crc-pclmul-template.S:
.Lreduce_128bits_to_crc tracing the logic below this label, I see
three clmul operations:
_pclmulqdq CONSTS_XMM, LO64_TERMS, %xmm0, HI64_TERMS, %xmm1
_cond_vex movdqa, OFFSETOF_BARRETT_REDUCTION_CONSTS(CONSTS_PTR), CONSTS_XMM
_pclmulqdq CONSTS_XMM, HI64_TERMS, %xmm0, HI64_TERMS, %xmm1
.if LSB_CRC && \n == 64
_cond_vex punpcklqdq, %xmm1, %xmm2, %xmm2
_pclmulqdq CONSTS_XMM, LO64_TERMS, %xmm1, HI64_TERMS, %xmm1
I even checked the Intel Whitepaper and the reduction for 64-bit CRC
section "Final Reduction of 128-bits" also mentions 3 clmul.
One for using x^128 followed by xor, and then 2 more for final barrett
reduction with mu and poly.
Thus I am confused about accomplishing this with just 2 clmul. Could
you provide details on this?
Based on inputs I am thinking of only extending the PMULL version to
have unrolling with inline assembly for better codegen.
I will revisit EOR3 afterwards when the PMULL base version has
incorporated all the optimizations.
Mostly we can have always_inline function which can be instantiated
with/without EOR3 I think for approach.
Will submit new separate patch for just PMULL unrolling shortly in a
couple of days based on Demian's inputs.
Thanks,
Shreesh
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH v2] lib/crc: arm64: Add NEON PMULL+EOR3 crc64_nvme implementation
2026-11-17 0:45 ` Demian Shulhan
@ 2026-11-17 7:06 ` Shreesh Adiga
0 siblings, 0 replies; 7+ messages in thread
From: Shreesh Adiga @ 2026-11-17 7:06 UTC (permalink / raw)
To: Demian Shulhan; +Cc: Eric Biggers, Ard Biesheuvel, linux-crypto, linux-kernel
Hi Demian,
On Tue, Nov 17, 2026 at 6:15 AM Demian Shulhan <demyansh@gmail.com> wrote:
> Scalar vmull_p64() makes compilers materialize the constants via mov/movk + fmov
> (42/97 insns on Clang). SHA3_TARGET also makes codegen fragile: with
> -march=armv8.5-a+crypto, GCC stops inlining pmull64*(). On Neoverse-V2, EOR3
> contends with PMULL on pipe V0, whereas regular EOR runs on all SIMD pipes.
> Using inline-asm pmull/pmull2 and a clean 4x unroll in crc64_nvme_neon() without
> EOR3 should give better performance across all sizes and cleaner code.
>
> /* clang -O2 */
> 24: mov x10, #0x45a
> 28: movk x10, #0xace5, lsl #16
> 2c: movk x10, #0x2240, lsl #32
> 30: movk x10, #0x6224, lsl #48
> 34: dup v4.2d, x10
> 38: mov x10, #0xa84a
> 3c: movk x10, #0x1e18, lsl #16
> 40: movk x10, #0xcdb3, lsl #32
> 44: movk x10, #0xc32, lsl #48
> 48: fmov d5, x10
Can I assume we want to have the 16-byte constants loaded using LD1
and use inline assembly for PMULL/PMULL2 and have the codegen not
perform the mov+movk+fmov instantiation?
Will look into extending the existing PMULL routine with 4x unrolling
and adding the partial block handling as well similar to x86 version.
I'll assume that 128 byte threshold for invoking this SIMD routine
still holds though, even though this is probably core specific.
For EOR3 though I assume it is still useful for many cores like
Cortex-X925, Cortex-A720, Cortex-X4 that I checked has measurable
speedup at 512B/4KB.
Maybe we can have a CPU core check for special cases like Neoverse-N2
which at the crc64_mod_init_arch we can check and avoid enabling the
static_branch?
Kindly let me know if it is not worth having EOR3 for all cores. For
now I will try to submit the existing PMULL routine enhancement as per
your suggestion.
Thanks,
Shreesh
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH v2] lib/crc: arm64: Add NEON PMULL+EOR3 crc64_nvme implementation
2026-11-17 6:50 ` Shreesh Adiga
@ 2026-11-17 7:11 ` Eric Biggers
0 siblings, 0 replies; 7+ messages in thread
From: Eric Biggers @ 2026-11-17 7:11 UTC (permalink / raw)
To: Shreesh Adiga; +Cc: Ard Biesheuvel, Demian Shulhan, linux-crypto, linux-kernel
On Tue, Nov 17, 2026 at 12:20:08PM +0530, Shreesh Adiga wrote:
> Hi Eric,
>
> On Tue, Nov 17, 2026 at 5:31 AM Eric Biggers <ebiggers@kernel.org> wrote:
> > Was crc_kunit run? It already has a benchmark in it.
> No, I have access to only one aarch64 device that is Android Mediatek
> Dimensity 9400 SoC without root access.
> Therefore I can't run crc_kunit on that, instead I have done userspace
> comparisons for the same kernel with libbenchmark.
> This is based on my earlier work for FFmpeg:
> https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21651
> I also have Corporate MacOS M5 and did check inside Ubuntu VM but that
> is not accurate due to virtualization.
Well, this is a kernel patch. Contributors are expected to test their
kernel patches. It can at least be done with QEMU.
> > The commit message should also clarify that this benefits ARMv8.2 and
> > later processors with the sha3 extension, not just ARMv9. (E.g., Apple
> > M1 is an example of an ARMv8.4 processor that benefits.)
> Sure I will take care of it, but for now I am thinking of first
> improving the existing PMULL to have 4x unrolling as per Demian's
> suggestion.
That would be fine too.
> > Note that this is missing several optimizations compared to the x86
> > version (lib/crc/x86/crc-pclmul-template.S), including using
> > fold_consts_2x for a tail of 'len & 32', efficiently handling lengths
> > that aren't a multiple of 16 bytes, and reducing to the final CRC in two
> > carryless multiplication instructions rather than three.
> I was looking at the crc-pclmul-template.S code and I have a few queries.
> In this aarch64 version we only dispatch to this SIMD routine only
> when len>=128.
> Since we have 128 bytes available, do we have to add the tail of `len
> & 32` handling?
It's still useful, since the length mod 64 might be >= 32.
> Handling len not a multiple of 16 bytes (partial block) I will add but
> I am assuming that the dispatch will maintain 128 minimum len.
There is overhead in entering a kernel-mode NEON section, so it makes
sense to continue to have a minimum length.
> I was trying to understand your "reducing to the final CRC in two
> carryless multiplications rather than three" statement.
> From going through the code in crc-pclmul-template.S:
> .Lreduce_128bits_to_crc tracing the logic below this label, I see
> three clmul operations:
> _pclmulqdq CONSTS_XMM, LO64_TERMS, %xmm0, HI64_TERMS, %xmm1
>
> _cond_vex movdqa, OFFSETOF_BARRETT_REDUCTION_CONSTS(CONSTS_PTR), CONSTS_XMM
> _pclmulqdq CONSTS_XMM, HI64_TERMS, %xmm0, HI64_TERMS, %xmm1
>
> .if LSB_CRC && \n == 64
> _cond_vex punpcklqdq, %xmm1, %xmm2, %xmm2
> _pclmulqdq CONSTS_XMM, LO64_TERMS, %xmm1, HI64_TERMS, %xmm1
>
> I even checked the Intel Whitepaper and the reduction for 64-bit CRC
> section "Final Reduction of 128-bits" also mentions 3 clmul.
> One for using x^128 followed by xor, and then 2 more for final barrett
> reduction with mu and poly.
> Thus I am confused about accomplishing this with just 2 clmul. Could
> you provide details on this?
Sorry, I misremembered. It's indeed 3 minimum. There are just a lot of
implementations out there that unnecessarily do 4, including
lib/crc/arm64/crc-t10dif-core.S for example which still exists.
- Eric
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH v2] lib/crc: arm64: Add NEON PMULL+EOR3 crc64_nvme implementation
2026-11-14 17:29 [PATCH v2] lib/crc: arm64: Add NEON PMULL+EOR3 crc64_nvme implementation Shreesh Adiga
2026-11-17 0:01 ` Eric Biggers
2026-11-17 0:45 ` Demian Shulhan
@ 2026-11-17 9:00 ` David Laight
2 siblings, 0 replies; 7+ messages in thread
From: David Laight @ 2026-11-17 9:00 UTC (permalink / raw)
To: Shreesh Adiga
Cc: Eric Biggers, Ard Biesheuvel, Demian Shulhan, linux-crypto, linux-kernel
On Sat, 14 Nov 2026 22:59:51 +0530
Shreesh Adiga <16567adigashreesh@gmail.com> wrote:
> Add a 4x-folding implementation for CPUs with FEAT_SHA3 which benefits
> newer ARMv9 CPUs like Cortex-X925.
> Usage of EOR3 from SHA3 extension improves performance by 20% compared
> to the regular PMULL without EOR3 on Cortex-X925 measure in userspace
> application via libbenchmark for 512 byte input size.
> Measured about 3x speedup for 512 byte input compared with the existing
> NEON single fold implementation on Cortex-X925 in userspace.
>
> Signed-off-by: Shreesh Adiga <16567adigashreesh@gmail.com>
> ---
> Changes in v2:
> Improved the 4x to 1x reduction by using x^255 and x^319 x2 reduction.
> Added static keys for the CPU feature checks instead of regular
> branches.
>
> lib/crc/arm64/crc64.h | 24 +++++++++++--
> lib/crc/crc64-neon.c | 83 +++++++++++++++++++++++++++++++++++++++++++
> 2 files changed, 104 insertions(+), 3 deletions(-)
>
> diff --git a/lib/crc/arm64/crc64.h b/lib/crc/arm64/crc64.h
> index c7a69e1f3d8f..e5be0ef21140 100644
> --- a/lib/crc/arm64/crc64.h
> +++ b/lib/crc/arm64/crc64.h
> @@ -9,20 +9,38 @@
> #include <linux/sizes.h>
>
> u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len);
> +u64 crc64_nvme_neon_pmull_eor3(u64 crc, const u8 *p, size_t len);
> +
> +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_eor3);
> +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
>
> #define crc64_be_arch crc64_be_generic
>
> static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len)
> {
> - if (len >= 128 && cpu_have_named_feature(PMULL) &&
> + if (len >= 128 && static_branch_likely(&have_pmull) &&
Would it be better to swap the order of the checks?
Otherwise I think you get the 'len >= 128' conditional branch
even when the static branch is false.
David
> likely(may_use_simd())) {
> size_t chunk = len & ~15;
>
> - scoped_ksimd()
> - crc = crc64_nvme_neon(crc, p, chunk);
> + scoped_ksimd() {
> + if (static_branch_likely(&have_eor3))
> + crc = crc64_nvme_neon_pmull_eor3(crc, p, chunk);
> + else
> + crc = crc64_nvme_neon(crc, p, chunk);
> + }
>
> p += chunk;
> len &= 15;
> }
> return crc64_nvme_generic(crc, p, len);
> }
> +
> +#define crc64_mod_init_arch crc64_mod_init_arch
> +static void crc64_mod_init_arch(void)
> +{
> + if (cpu_have_named_feature(PMULL)) {
> + static_branch_enable(&have_pmull);
> + if (cpu_have_named_feature(SHA3))
> + static_branch_enable(&have_eor3);
> + }
> +}
> diff --git a/lib/crc/crc64-neon.c b/lib/crc/crc64-neon.c
> index 4753fb94a4be..767e55edc0e0 100644
> --- a/lib/crc/crc64-neon.c
> +++ b/lib/crc/crc64-neon.c
> @@ -10,6 +10,17 @@
>
> u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len);
>
> +#ifdef CONFIG_ARM64
> +u64 crc64_nvme_neon_pmull_eor3(u64 crc, const u8 *p, size_t len);
> +
> +/* x^575 mod G, x^511 mod G */
> +static const u64 fold_consts_4x_val[2] = { 0x0c32cdb31e18a84aULL,
> + 0x62242240ace5045aULL };
> +/* x^319 mod G, x^255 mod G */
> +static const u64 fold_consts_2x_val[2] = { 0xb0bc2e589204f500ULL,
> + 0xe1e0bb9d45d7a44cULL };
> +#endif
> +
> /* x^191 mod G, x^127 mod G */
> static const u64 fold_consts_val[2] = { 0xeadc41fd2ba3d420ULL,
> 0x21e9761e252621acULL };
> @@ -45,3 +56,75 @@ u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len)
>
> return vgetq_lane_u64(v0, 1);
> }
> +
> +#ifdef CONFIG_ARM64
> +#define SHA3_TARGET __attribute__((target("arch=armv8.2-a+crypto+sha3")))
> +
> +SHA3_TARGET
> +static inline uint64x2_t fold_single_round(uint64x2_t fold,
> + uint64x2_t temp,
> + uint64x2_t consts)
> +{
> + uint64x2_t temp1 = pmull64(consts, fold);
> + uint64x2_t temp2 = pmull64_high(consts, fold);
> +
> + return veor3q_u64(temp, temp1, temp2);
> +}
> +
> +SHA3_TARGET
> +u64 crc64_nvme_neon_pmull_eor3(u64 crc, const u8 *p, size_t len)
> +{
> + /* Assumes len >= 64. */
> + uint64x2_t fold_consts = vld1q_u64(fold_consts_val);
> + uint64x2_t fold_consts_4x = vld1q_u64(fold_consts_4x_val);
> + uint64x2_t fold_consts_2x = vld1q_u64(fold_consts_2x_val);
> + uint64x2_t v0 = { crc, 0 };
> + uint64x2_t zero = { };
> + uint64x2_t v1, v2, v3;
> +
> + v0 ^= vreinterpretq_u64_u8(vld1q_u8(p));
> + v1 = vreinterpretq_u64_u8(vld1q_u8(p + 16));
> + v2 = vreinterpretq_u64_u8(vld1q_u8(p + 32));
> + v3 = vreinterpretq_u64_u8(vld1q_u8(p + 48));
> +
> + p += 64;
> + len -= 64;
> + while (len >= 64) {
> + uint64x2_t t0 = vreinterpretq_u64_u8(vld1q_u8(p + 0));
> + uint64x2_t t1 = vreinterpretq_u64_u8(vld1q_u8(p + 16));
> + uint64x2_t t2 = vreinterpretq_u64_u8(vld1q_u8(p + 32));
> + uint64x2_t t3 = vreinterpretq_u64_u8(vld1q_u8(p + 48));
> +
> + v0 = fold_single_round(v0, t0, fold_consts_4x);
> + v1 = fold_single_round(v1, t1, fold_consts_4x);
> + v2 = fold_single_round(v2, t2, fold_consts_4x);
> + v3 = fold_single_round(v3, t3, fold_consts_4x);
> + p += 64;
> + len -= 64;
> + }
> +
> + v0 = fold_single_round(v0, v2, fold_consts_2x);
> + v1 = fold_single_round(v1, v3, fold_consts_2x);
> + v0 = fold_single_round(v0, v1, fold_consts);
> +
> + while (len >= 16) {
> + uint64x2_t t0 = vreinterpretq_u64_u8(vld1q_u8(p));
> +
> + v0 = fold_single_round(v0, t0, fold_consts);
> + p += 16;
> + len -= 16;
> + }
> +
> + /* Multiply the 128-bit value by x^64 and reduce it back to 128 bits. */
> + v0 = vextq_u64(v0, zero, 1) ^ pmull64_hi_lo(fold_consts, v0);
> +
> + /* Final Barrett reduction */
> + uint64x2_t bconsts = vld1q_u64(bconsts_val);
> + uint64x2_t final = pmull64(bconsts, v0);
> +
> + v0 = veor3q_u64(v0, vextq_u64(zero, final, 1),
> + pmull64_hi_lo(bconsts, final));
> +
> + return vgetq_lane_u64(v0, 1);
> +}
> +#endif
^ permalink raw reply [flat|nested] 7+ messages in thread
end of thread, other threads:[~2026-11-17 9:00 UTC | newest]
Thread overview: 7+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2026-11-14 17:29 [PATCH v2] lib/crc: arm64: Add NEON PMULL+EOR3 crc64_nvme implementation Shreesh Adiga
2026-11-17 0:01 ` Eric Biggers
2026-11-17 6:50 ` Shreesh Adiga
2026-11-17 7:11 ` Eric Biggers
2026-11-17 0:45 ` Demian Shulhan
2026-11-17 7:06 ` Shreesh Adiga
2026-11-17 9:00 ` David Laight
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®