mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: Shreesh Adiga <16567adigashreesh@gmail.com>
To: Eric Biggers <ebiggers@kernel.org>, Ard Biesheuvel <ardb@kernel.org>
Cc: linux-crypto@vger.kernel.org, linux-kernel@vger.kernel.org
Subject: [PATCH] lib/crc: arm: Support partial blocks in NEON crc64_nvme
Date: Sun, 22 Nov 2026 13:51:11 +0530	[thread overview]
Message-ID: <20261122082122.664360-1-16567adigashreesh@gmail.com> (raw)

Provide vqtbl1q_u8() for AArch32 using two vtbl2_u8() lookups, the same
emulation used by lib/raid/raid6/arm/recov_neon_inner.c, and derive the
partial block blend mask with an arithmetic shift right by 7 instead of
the AArch64-only vcgezq_s8(). This lets fold_partial_block() be shared
by both architectures and removes the AArch32-specific requirement that
crc64_nvme_neon() be called with a length that is a multiple of 16.

The AArch32 dispatch code therefore no longer rounds chunks down to a
multiple of 16 bytes. Each chunk is now either 4 KiB or the whole
remaining length, so the tail is processed by NEON instead of falling
back to the generic table-based implementation.

Also document why the arm64 PMULL helpers use inline asm instead of the
intrinsics, and turn the 4-way fold loop into a do-while loop, since
callers guarantee len >= 128.

AArch32 was tested for correctness only (KUnit under QEMU), as no
AArch32-capable hardware was available for benchmarking.

Suggested-by: Eric Biggers <ebiggers@kernel.org>
Signed-off-by: Shreesh Adiga <16567adigashreesh@gmail.com>
---
 lib/crc/arm/crc64-neon.h   | 16 ++++++++++++++++
 lib/crc/arm/crc64.h        |  4 ++--
 lib/crc/arm64/crc64-neon.h | 12 ++++++++++++
 lib/crc/crc64-neon.c       | 24 ++++++++++++++----------
 4 files changed, 44 insertions(+), 12 deletions(-)

diff --git a/lib/crc/arm/crc64-neon.h b/lib/crc/arm/crc64-neon.h
index 645f553220ff..0d0071a92071 100644
--- a/lib/crc/arm/crc64-neon.h
+++ b/lib/crc/arm/crc64-neon.h
@@ -32,3 +32,19 @@ static inline uint64x2_t pmull64_hi_lo(uint64x2_t a, uint64x2_t b)
 
 	return result;
 }
+
+/*
+ * AArch32 does not provide this intrinsic natively because it does not
+ * implement the underlying instruction. AArch32 only provides a 64-bit
+ * wide vtbl.8 instruction, so use that instead.
+ */
+static inline uint8x16_t vqtbl1q_u8(uint8x16_t a, uint8x16_t b)
+{
+	union {
+		uint8x16_t	val;
+		uint8x8x2_t	pair;
+	} __a = { a };
+
+	return vcombine_u8(vtbl2_u8(__a.pair, vget_low_u8(b)),
+			   vtbl2_u8(__a.pair, vget_high_u8(b)));
+}
diff --git a/lib/crc/arm/crc64.h b/lib/crc/arm/crc64.h
index de274288af61..ce0ffe590596 100644
--- a/lib/crc/arm/crc64.h
+++ b/lib/crc/arm/crc64.h
@@ -13,10 +13,10 @@ u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len);
 
 static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len)
 {
-	if (len >= 128 && static_branch_likely(&have_pmull) &&
+	if (static_branch_likely(&have_pmull) && len >= 128 &&
 	    likely(may_use_simd())) {
 		do {
-			size_t chunk = min_t(size_t, len & ~15, SZ_4K);
+			size_t chunk = (len < SZ_4K + 128) ? len : SZ_4K;
 
 			scoped_ksimd()
 				crc = crc64_nvme_neon(crc, p, chunk);
diff --git a/lib/crc/arm64/crc64-neon.h b/lib/crc/arm64/crc64-neon.h
index 700a3a8619f4..d872f4d4e2e7 100644
--- a/lib/crc/arm64/crc64-neon.h
+++ b/lib/crc/arm64/crc64-neon.h
@@ -1,5 +1,17 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
+/*
+ * The PMULL helpers below use inline asm instead of the vmull_p64() and
+ * vmull_high_p64() intrinsics. vmull_p64() takes poly64_t scalars, so the low
+ * lanes have to be extracted with vgetq_lane_u64(). For the fold constants,
+ * compilers fold that extraction into a 64-bit immediate: recent GCC and Clang
+ * build it in a GPR with mov/movk and transfer it with fmov (Clang also
+ * rebuilds the high half used by pmull2 with mov/movk and dup), instead of
+ * loading the constant pair with a single ldr. With asm operating on whole
+ * vector registers, the constants are loaded from .rodata and pmull reads
+ * the low lanes in place.
+ */
+
 static inline uint64x2_t pmull64(uint64x2_t a, uint64x2_t b)
 {
 	uint64x2_t r;
diff --git a/lib/crc/crc64-neon.c b/lib/crc/crc64-neon.c
index 03b46d0abcc4..5b39d6ae1e9c 100644
--- a/lib/crc/crc64-neon.c
+++ b/lib/crc/crc64-neon.c
@@ -32,8 +32,15 @@ static inline uint64x2_t fold_single_round(uint64x2_t fold, uint64x2_t temp,
 	return temp ^ temp1 ^ temp2;
 }
 
-#ifdef CONFIG_ARM64
-/* vqtbl1q_u8 is not available on AArch32. */
+/*
+ * Fold in the final 1 <= len <= 15 bytes. With A = fold, the remaining
+ * polynomial is A*x^(8*len) + B. Rewrite it as C1*x^128 + C2, where
+ * C1 = floor(A / x^(128 - 8*len)) and C2 = (A*x^(8*len) mod x^128) + B,
+ * then fold C1 into C2. t1 selects the len highest-degree bytes of A to
+ * form C1; ~t1 selects the remaining bytes to form A*x^(8*len) mod x^128,
+ * since tbl returns 0 for any index >= 16. The sign of t1 then selects
+ * which bytes of C2 come from B.
+ */
 static inline uint64x2_t fold_partial_block(uint64x2_t fold, uint64x2_t consts,
 					    const u8 *p, size_t len)
 {
@@ -52,16 +59,15 @@ static inline uint64x2_t fold_partial_block(uint64x2_t fold, uint64x2_t consts,
 	a  = vreinterpretq_u64_u8(vqtbl1q_u8(vreinterpretq_u8_u64(fold), t1));
 	t2 = vmvnq_u8(t1);
 	t2 = vqtbl1q_u8(vreinterpretq_u8_u64(fold), t2);
-	t1 = vcgezq_s8(vreinterpretq_s8_u8(t1));
-	b  = vreinterpretq_u64_u8(vbslq_u8(t1, last16, t2));
+	t1 = vreinterpretq_u8_s8(vshrq_n_s8(vreinterpretq_s8_u8(t1), 7));
+	b  = vreinterpretq_u64_u8(vbslq_u8(t1, t2, last16));
 
 	return fold_single_round(a, b, consts);
 }
-#endif
 
 u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len)
 {
-	/* Assumes len >= 64. AArch32 also assumes len is multiple of 16. */
+	/* Assumes len >= 128. */
 	uint64x2_t fold_consts = vld1q_u64(fold_consts_val);
 	uint64x2_t fold_consts_4x = vld1q_u64(fold_consts_4x_val);
 	uint64x2_t fold_consts_2x = vld1q_u64(fold_consts_2x_val);
@@ -76,7 +82,7 @@ u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len)
 
 	p += 64;
 	len -= 64;
-	while (len >= 64) {
+	do {
 		uint64x2_t t0 = vreinterpretq_u64_u8(vld1q_u8(p +  0));
 		uint64x2_t t1 = vreinterpretq_u64_u8(vld1q_u8(p + 16));
 		uint64x2_t t2 = vreinterpretq_u64_u8(vld1q_u8(p + 32));
@@ -88,7 +94,7 @@ u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len)
 		v3 = fold_single_round(v3, t3, fold_consts_4x);
 		p += 64;
 		len -= 64;
-	}
+	} while (len >= 64);
 
 	v0 = fold_single_round(v0, v2, fold_consts_2x);
 	v1 = fold_single_round(v1, v3, fold_consts_2x);
@@ -113,10 +119,8 @@ u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len)
 		len -= 16;
 	}
 
-#ifdef CONFIG_ARM64
 	if (len)
 		v0 = fold_partial_block(v0, fold_consts, p, len);
-#endif
 
 	/* Multiply the 128-bit value by x^64 and reduce it back to 128 bits. */
 	v0 = vextq_u64(v0, zero, 1) ^ pmull64_hi_lo(fold_consts, v0);
-- 
2.56.1


                 reply	other threads:[~2026-11-22  8:21 UTC|newest]

Thread overview: [no followups] expand[flat|nested]  mbox.gz  Atom feed

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261122082122.664360-1-16567adigashreesh@gmail.com \
    --to=16567adigashreesh@gmail.com \
    --cc=ardb@kernel.org \
    --cc=ebiggers@kernel.org \
    --cc=linux-crypto@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®