[PATCH 1/2] lib/crc: loongarch: Use hardware unaligned load in CRC32(C) loops

[PATCH 1/2] lib/crc: loongarch: Use hardware unaligned load in CRC32(C) loops

[Huacai Chen]

With CONFIG_ARCH_STRICT_ALIGN=y the kernel is built with -mstrict-align,
so get_unaligned_le{16,32,64}() compiles into a byte-by-byte load/shift/
or sequence (~22 instructions per 8 bytes). On a Loongson-3C6000 machine
this throttles the CRC pipeline to ~5.7 cycles per 8 bytes (2.8 GB/s),
while the crcc.* instruction itself runs at ~1.1 cycles (latency) / ~0.8
cycles (throughput) -- the CRC unit sits idle waiting for loads.

CPU cores with the UAL feature handle unaligned accesses in hardware,
so issue real ld.{h,w,d} instructions when possible and keep the generic
get_unaligned*() path for cores without UAL.

Measured on Loongson-3C6000 (single core):

    before: 2.8 GB/s
    after:  8.4 GB/s   (~3x)

Signed-off-by: Tianyang Zhang <zhangtianyang@loongson.cn>
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>
---
 lib/crc/loongarch/crc32.h | 39 +++++++++++++++++++++++++++++++++------
 1 file changed, 33 insertions(+), 6 deletions(-)

diff --git a/lib/crc/loongarch/crc32.h b/lib/crc/loongarch/crc32.h
index d34fa4c68632..692a4165eca2 100644
--- a/lib/crc/loongarch/crc32.h
+++ b/lib/crc/loongarch/crc32.h
@@ -24,15 +24,40 @@ do {							\
 #define CRC32(crc, value, size)		_CRC32(crc, value, size, crc)
 #define CRC32C(crc, value, size)	_CRC32(crc, value, size, crcc)
 
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ual);
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32);
 
+static inline u16 get_le16(const void *p)
+{
+	if (static_branch_likely(&have_ual))
+		return *((__le16 *)p);
+	else
+		return get_unaligned_le16(p);
+}
+
+static inline u32 get_le32(const void *p)
+{
+	if (static_branch_likely(&have_ual))
+		return *((__le32 *)p);
+	else
+		return get_unaligned_le32(p);
+}
+
+static inline u64 get_le64(const void *p)
+{
+	if (static_branch_likely(&have_ual))
+		return *((__le64 *)p);
+	else
+		return get_unaligned_le64(p);
+}
+
 static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
 {
 	if (!static_branch_likely(&have_crc32))
 		return crc32_le_base(crc, p, len);
 
 	while (len >= sizeof(u64)) {
-		u64 value = get_unaligned_le64(p);
+		u64 value = get_le64(p);
 
 		CRC32(crc, value, d);
 		p += sizeof(u64);
@@ -40,14 +65,14 @@ static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
 	}
 
 	if (len & sizeof(u32)) {
-		u32 value = get_unaligned_le32(p);
+		u32 value = get_le32(p);
 
 		CRC32(crc, value, w);
 		p += sizeof(u32);
 	}
 
 	if (len & sizeof(u16)) {
-		u16 value = get_unaligned_le16(p);
+		u16 value = get_le16(p);
 
 		CRC32(crc, value, h);
 		p += sizeof(u16);
@@ -68,7 +93,7 @@ static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
 		return crc32c_base(crc, p, len);
 
 	while (len >= sizeof(u64)) {
-		u64 value = get_unaligned_le64(p);
+		u64 value = get_le64(p);
 
 		CRC32C(crc, value, d);
 		p += sizeof(u64);
@@ -76,14 +101,14 @@ static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
 	}
 
 	if (len & sizeof(u32)) {
-		u32 value = get_unaligned_le32(p);
+		u32 value = get_le32(p);
 
 		CRC32C(crc, value, w);
 		p += sizeof(u32);
 	}
 
 	if (len & sizeof(u16)) {
-		u16 value = get_unaligned_le16(p);
+		u16 value = get_le16(p);
 
 		CRC32C(crc, value, h);
 		p += sizeof(u16);
@@ -103,6 +128,8 @@ static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
 #define crc32_mod_init_arch crc32_mod_init_arch
 static void crc32_mod_init_arch(void)
 {
+	if (cpu_has_ual)
+		static_branch_enable(&have_ual);
 	if (cpu_has_crc32)
 		static_branch_enable(&have_crc32);
 }
-- 
2.52.0

[PATCH 2/2] lib/crc: loongarch: Add 3-way parallel CRC32(C) for large buffers

[Huacai Chen]

The crc* loop is a single serial dependency chain: every instruction
consumes the previous one's result. CRC is linear over GF(2), so a
buffer can be split into 3 contiguous lanes processed by independent
chains and combined afterwards:

    crc(A|B|C) = shift(crc(A), |B|+|C|) ^ shift(crc(B), |C|) ^ crc(C)

with lanes B and C seeded with 0. The shift (multiplication by x^(8n)
mod P) is linear, so its multiplier is tabulated at module init as 4x256
tables built from crc32_le_base()/crc32c_base(); each combine then costs
4 lookups instead of a log-time GF(2) square loop.

Lanes use a fixed 2 KiB stride (6 KiB superblock); smaller buffers fall
through to the serial path. The 3-way parallel loop CRC32 is measured on
a Loongson-3C6000 machine (single core, cache-hot, with UAL):

    8 KiB:   8.4 -> 12.0 GB/s
    64 KiB:  8.4 -> 12.4 GB/s   (+45% over the serial hw loop,
                                 ~12x over the generic table)

Correctness was verified bit-exact against crc32c() over all lengths
0..6168 with 3 seeds, plus 3000 random offset/length/seed combinations.

Signed-off-by: Tianyang Zhang <zhangtianyang@loongson.cn>
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>
---
 lib/crc/loongarch/crc32.h | 53 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/lib/crc/loongarch/crc32.h b/lib/crc/loongarch/crc32.h
index 692a4165eca2..1750c6eb4a01 100644
--- a/lib/crc/loongarch/crc32.h
+++ b/lib/crc/loongarch/crc32.h
@@ -12,6 +12,9 @@
 #include <asm/cpu-features.h>
 #include <linux/unaligned.h>
 
+#define CRC3WAY_STRIDE	SZ_2K
+#define CRC3WAY_BLOCK	(3 * CRC3WAY_STRIDE)
+
 #define _CRC32(crc, value, size, type)			\
 do {							\
 	__asm__ __volatile__(				\
@@ -27,6 +30,50 @@ do {							\
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ual);
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32);
 
+static const u8 zero[4] = {0};
+static u32 crc32_k1tab[4][256], crc32_k2tab[4][256];
+static u32 crc32c_k1tab[4][256], crc32c_k2tab[4][256];
+
+static void crc3way_build_table(u32 tab[4][256],
+		u32 (*init)(u32, const u8 *p, size_t), size_t len)
+{
+	int i, j;
+
+	for (i = 0; i < 4; i++)
+		for (j = 0; j < 256; j++)
+			tab[i][j] = init((u32)j << (8 * i), zero, len);
+}
+
+static void crc3way_table_init(void)
+{
+	crc3way_build_table(crc32_k1tab, crc32_le_base, CRC3WAY_STRIDE);
+	crc3way_build_table(crc32_k2tab, crc32_le_base, 2 * CRC3WAY_STRIDE);
+	crc3way_build_table(crc32c_k1tab, crc32c_base, CRC3WAY_STRIDE);
+	crc3way_build_table(crc32c_k2tab, crc32c_base, 2 * CRC3WAY_STRIDE);
+}
+
+static inline u32 crc3way_shift(const u32 tab[4][256], u32 crc)
+{
+	return tab[0][crc & 0xff] ^ tab[1][(crc >> 8) & 0xff] ^
+	       tab[2][(crc >> 16) & 0xff] ^ tab[3][crc >> 24];
+}
+
+#define CRC3WAY_LOOP(crc, p, len, type, k1tab, k2tab)			\
+while ((len) >= CRC3WAY_BLOCK) {					\
+	unsigned int i;							\
+	u32 c0 = (crc), c1 = 0, c2 = 0;					\
+	const u8 *p1 = (p) + CRC3WAY_STRIDE;				\
+	const u8 *p2 = (p) + 2 * CRC3WAY_STRIDE;			\
+									\
+	for (i = 0; i < CRC3WAY_STRIDE; i += sizeof(u64)) {		\
+		_CRC32(c0, get_le64((p) + i), d, type);			\
+		_CRC32(c1, get_le64((p1) + i), d, type);		\
+		_CRC32(c2, get_le64((p2) + i), d, type);		\
+	}								\
+	(p) += CRC3WAY_BLOCK; (len) -= CRC3WAY_BLOCK;			\
+	crc = crc3way_shift(k2tab, c0) ^ crc3way_shift(k1tab, c1) ^ c2;	\
+}
+
 static inline u16 get_le16(const void *p)
 {
 	if (static_branch_likely(&have_ual))
@@ -56,6 +103,8 @@ static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
 	if (!static_branch_likely(&have_crc32))
 		return crc32_le_base(crc, p, len);
 
+	CRC3WAY_LOOP(crc, p, len, crc, crc32_k1tab, crc32_k2tab);
+
 	while (len >= sizeof(u64)) {
 		u64 value = get_le64(p);
 
@@ -92,6 +141,8 @@ static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
 	if (!static_branch_likely(&have_crc32))
 		return crc32c_base(crc, p, len);
 
+	CRC3WAY_LOOP(crc, p, len, crcc, crc32c_k1tab, crc32c_k2tab);
+
 	while (len >= sizeof(u64)) {
 		u64 value = get_le64(p);
 
@@ -128,6 +179,8 @@ static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
 #define crc32_mod_init_arch crc32_mod_init_arch
 static void crc32_mod_init_arch(void)
 {
+	crc3way_table_init();
+
 	if (cpu_has_ual)
 		static_branch_enable(&have_ual);
 	if (cpu_has_crc32)
-- 
2.52.0

Re: [PATCH 2/2] lib/crc: loongarch: Add 3-way parallel CRC32(C) for large buffers

[Eric Biggers]

On Sun, Oct 11, 2026 at 10:29:30PM +0800, Huacai Chen wrote:
> The crc* loop is a single serial dependency chain: every instruction
> consumes the previous one's result. CRC is linear over GF(2), so a
> buffer can be split into 3 contiguous lanes processed by independent
> chains and combined afterwards:
> 
>     crc(A|B|C) = shift(crc(A), |B|+|C|) ^ shift(crc(B), |C|) ^ crc(C)
> 
> with lanes B and C seeded with 0. The shift (multiplication by x^(8n)
> mod P) is linear, so its multiplier is tabulated at module init as 4x256
> tables built from crc32_le_base()/crc32c_base(); each combine then costs
> 4 lookups instead of a log-time GF(2) square loop.
> 
> Lanes use a fixed 2 KiB stride (6 KiB superblock); smaller buffers fall
> through to the serial path. The 3-way parallel loop CRC32 is measured on
> a Loongson-3C6000 machine (single core, cache-hot, with UAL):
> 
>     8 KiB:   8.4 -> 12.0 GB/s
>     64 KiB:  8.4 -> 12.4 GB/s   (+45% over the serial hw loop,
>                                  ~12x over the generic table)
> 
> Correctness was verified bit-exact against crc32c() over all lengths
> 0..6168 with 3 seeds, plus 3000 random offset/length/seed combinations.
> 
> Signed-off-by: Tianyang Zhang <zhangtianyang@loongson.cn>
> Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>

So is this new code testable in QEMU?  That's a prerequisite for adding
new optimized CRC or crypto code.

Please also note that you should just run crc_kunit instead of writing
custom in-kernel CRC test code.

- Eric

Re:[PATCH 1/2] lib/crc: loongarch: Use hardware unaligned load in CRC32(C) loops

[Wentao Guan]

> With CONFIG_ARCH_STRICT_ALIGN=y the kernel is built with -mstrict-align,
> so get_unaligned_le{16,32,64}() compiles into a byte-by-byte load/shift/
> or sequence (~22 instructions per 8 bytes). On a Loongson-3C6000 machine
> this throttles the CRC pipeline to ~5.7 cycles per 8 bytes (2.8 GB/s),
> while the crcc.* instruction itself runs at ~1.1 cycles (latency) / ~0.8
> cycles (throughput) -- the CRC unit sits idle waiting for loads.
> 
> CPU cores with the UAL feature handle unaligned accesses in hardware,
> so issue real ld.{h,w,d} instructions when possible and keep the generic
> get_unaligned*() path for cores without UAL.
> 
> Measured on Loongson-3C6000 (single core):
> 
>     before: 2.8 GB/s
>     after:  8.4 GB/s   (~3x)
> 
> Signed-off-by: Tianyang Zhang <zhangtianyang@loongson.cn>
> Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>
> ---
>  lib/crc/loongarch/crc32.h | 39 +++++++++++++++++++++++++++++++++------
>  1 file changed, 33 insertions(+), 6 deletions(-)
> 
> diff --git a/lib/crc/loongarch/crc32.h b/lib/crc/loongarch/crc32.h
> index d34fa4c68632..692a4165eca2 100644
> --- a/lib/crc/loongarch/crc32.h
> +++ b/lib/crc/loongarch/crc32.h
> @@ -24,15 +24,40 @@ do { \
>  #define CRC32(crc, value, size) _CRC32(crc, value, size, crc)
>  #define CRC32C(crc, value, size) _CRC32(crc, value, size, crcc)
>  
> +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ual);
>  static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32);
>  
> +static inline u16 get_le16(const void *p)
> +{
> + if (static_branch_likely(&have_ual))
> + return *((__le16 *)p);
> + else
> + return get_unaligned_le16(p);
> +}
> +
> +static inline u32 get_le32(const void *p)
> +{
> + if (static_branch_likely(&have_ual))
> + return *((__le32 *)p);
> + else
> + return get_unaligned_le32(p);
> +}
> +
> +static inline u64 get_le64(const void *p)
> +{
> + if (static_branch_likely(&have_ual))
> + return *((__le64 *)p);
> + else
> + return get_unaligned_le64(p);
> +}
> +
>  static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
>  {
>   if (!static_branch_likely(&have_crc32))
>   return crc32_le_base(crc, p, len);
>  
>   while (len >= sizeof(u64)) {
> - u64 value = get_unaligned_le64(p);
> + u64 value = get_le64(p);
>  
>  CRC32(crc, value, d);
>  p += sizeof(u64);
> @@ -40,14 +65,14 @@ static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
>   }
>  
>   if (len & sizeof(u32)) {
> - u32 value = get_unaligned_le32(p);
> + u32 value = get_le32(p);
>  
>  CRC32(crc, value, w);
>  p += sizeof(u32);
>   }
>  
>   if (len & sizeof(u16)) {
> - u16 value = get_unaligned_le16(p);
> + u16 value = get_le16(p);
>  
>   CRC32(crc, value, h);
>   p += sizeof(u16);
> @@ -68,7 +93,7 @@ static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
>  return crc32c_base(crc, p, len);
>  
>   while (len >= sizeof(u64)) {
> - u64 value = get_unaligned_le64(p);
> + u64 value = get_le64(p);
>  
>   CRC32C(crc, value, d);
>   p += sizeof(u64);
> @@ -76,14 +101,14 @@ static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
>   }
>  
>   if (len & sizeof(u32)) {
> - u32 value = get_unaligned_le32(p);
> + u32 value = get_le32(p);
>  
>  CRC32C(crc, value, w);
>  p += sizeof(u32);
>   }
>  
>   if (len & sizeof(u16)) {
> - u16 value = get_unaligned_le16(p);
> + u16 value = get_le16(p);
>  
>   CRC32C(crc, value, h);
>   p += sizeof(u16);
> @@ -103,6 +128,8 @@ static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
>  #define crc32_mod_init_arch crc32_mod_init_arch
>  static void crc32_mod_init_arch(void)
>  {
> + if (cpu_has_ual)
> + static_branch_enable(&have_ual);
>   if (cpu_has_crc32)
>   static_branch_enable(&have_crc32);
>  }
> --
> 2.52.0

Reviewed-by: Wentao Guan <guanwentao@uniontech.com>

Nice performance improved patch~.

Re: [PATCH 1/2] lib/crc: loongarch: Use hardware unaligned load in CRC32(C) loops

[Eric Biggers]

On Sun, Oct 11, 2026 at 10:29:29PM +0800, Huacai Chen wrote:
> +static inline u32 get_le32(const void *p)
> +{
> +	if (static_branch_likely(&have_ual))
> +		return *((__le32 *)p);
> +	else
> +		return get_unaligned_le32(p);
> +}

No, this is just broken.  The bare misaligned access is undefined
behavior in C.  Also, every other use of {get,put}_unaligned_* in the
kernel would need this; it is not specific to the CRC code.  Also, it is
inefficient to compile both code paths into the binary.

The only way that realistically works, as we know from the other
architectures, is for the unaligned access support to be known at
compile time, and to have {get,put}_unaligned_* do the right thing
accordingly.

Since loongarch made the incorrect decision to not specify unaligned
access unconditionally, similar to RISC-V it can be worked around only
by overriding that in the kconfig.  For loongarch that appears that
already be implemented; just unset CONFIG_ARCH_STRICT_ALIGN when you
know the kernel will run only on CPUs that have fixed this bug.

- Eric

Re: [PATCH 1/2] lib/crc: loongarch: Use hardware unaligned load in CRC32(C) loops

[Wentao Guan]

> On Sun, Oct 11, 2026 at 10:29:29PM +0800, Huacai Chen wrote:
> > +static inline u32 get_le32(const void *p)
> > +{
> > + if (static_branch_likely(&have_ual))
> > + return *((__le32 *)p);
> > + else
> > + return get_unaligned_le32(p);
> > +}
> 
> No, this is just broken.  The bare misaligned access is undefined
> behavior in C.  Also, every other use of {get,put}_unaligned_* in the
> kernel would need this; it is not specific to the CRC code.  Also, it is
Right, I got your thought. It not belong to CRC level.

> inefficient to compile both code paths into the binary.
> 
> The only way that realistically works, as we know from the other
> architectures, is for the unaligned access support to be known at
> compile time, and to have {get,put}_unaligned_* do the right thing
> accordingly.
So the optimize it can be move to {get,put}_unaligned_* code?

> 
> Since loongarch made the incorrect decision to not specify unaligned
> access unconditionally, similar to RISC-V it can be worked around only
> by overriding that in the kconfig.  For loongarch that appears that
> already be implemented; just unset CONFIG_ARCH_STRICT_ALIGN when you
> know the kernel will run only on CPUs that have fixed this bug.
I think that we have the use cases to provide the general kernel instead of the two kernel with
different config, so the image increase will be tolerated.

BRs
Wentao Guan