From: Denis Vlasenko <vda@ilport.com.ua>
To: Herbert Xu <herbert@gondor.apana.org.au>
Cc: linux-crypto@vger.kernel.org, davem@davemloft.net,
linux-kernel@vger.kernel.org
Subject: [PATCH 2/4] whirlpool gcc bug fix
Date: Sun, 3 Jul 2005 16:05:23 +0300 [thread overview]
Message-ID: <200507031605.23742.vda@ilport.com.ua> (raw)
In-Reply-To: <200507031557.15416.vda@ilport.com.ua>
[-- Attachment #1: Type: text/plain, Size: 68 bytes --]
wp was not usable: stack overflow with certain gcc versions.
--
vda
[-- Attachment #2: 2.wp.patch --]
[-- Type: text/x-diff, Size: 10119 bytes --]
diff -urpN linux-2.6.12.1.n/crypto/wp512.c linux-2.6.12.2.n/crypto/wp512.c
--- linux-2.6.12.1.n/crypto/wp512.c Sun Jul 3 15:52:55 2005
+++ linux-2.6.12.2.n/crypto/wp512.c Sun Jul 3 15:52:59 2005
@@ -793,80 +793,84 @@ static void wp512_process_buffer(struct
state[6] = block[6] ^ (K[6] = wctx->hash[6]);
state[7] = block[7] ^ (K[7] = wctx->hash[7]);
+/* we do not use L[0] = C0[...] ^ C1[...] ^ ... ^ rc[r]
+** due to gcc -O2 (3.4.3) optimizer bug:
+** this will cause excessive spills (~3K stack used)
+** See http://gcc.gnu.org/bugzilla/show_bug.cgi?id=21141 */
for (r = 1; r <= WHIRLPOOL_ROUNDS; r++) {
- L[0] = C0[(int)(K[0] >> 56) ] ^
- C1[(int)(K[7] >> 48) & 0xff] ^
- C2[(int)(K[6] >> 40) & 0xff] ^
- C3[(int)(K[5] >> 32) & 0xff] ^
- C4[(int)(K[4] >> 24) & 0xff] ^
- C5[(int)(K[3] >> 16) & 0xff] ^
- C6[(int)(K[2] >> 8) & 0xff] ^
- C7[(int)(K[1] ) & 0xff] ^
- rc[r];
-
- L[1] = C0[(int)(K[1] >> 56) ] ^
- C1[(int)(K[0] >> 48) & 0xff] ^
- C2[(int)(K[7] >> 40) & 0xff] ^
- C3[(int)(K[6] >> 32) & 0xff] ^
- C4[(int)(K[5] >> 24) & 0xff] ^
- C5[(int)(K[4] >> 16) & 0xff] ^
- C6[(int)(K[3] >> 8) & 0xff] ^
- C7[(int)(K[2] ) & 0xff];
-
- L[2] = C0[(int)(K[2] >> 56) ] ^
- C1[(int)(K[1] >> 48) & 0xff] ^
- C2[(int)(K[0] >> 40) & 0xff] ^
- C3[(int)(K[7] >> 32) & 0xff] ^
- C4[(int)(K[6] >> 24) & 0xff] ^
- C5[(int)(K[5] >> 16) & 0xff] ^
- C6[(int)(K[4] >> 8) & 0xff] ^
- C7[(int)(K[3] ) & 0xff];
-
- L[3] = C0[(int)(K[3] >> 56) ] ^
- C1[(int)(K[2] >> 48) & 0xff] ^
- C2[(int)(K[1] >> 40) & 0xff] ^
- C3[(int)(K[0] >> 32) & 0xff] ^
- C4[(int)(K[7] >> 24) & 0xff] ^
- C5[(int)(K[6] >> 16) & 0xff] ^
- C6[(int)(K[5] >> 8) & 0xff] ^
- C7[(int)(K[4] ) & 0xff];
-
- L[4] = C0[(int)(K[4] >> 56) ] ^
- C1[(int)(K[3] >> 48) & 0xff] ^
- C2[(int)(K[2] >> 40) & 0xff] ^
- C3[(int)(K[1] >> 32) & 0xff] ^
- C4[(int)(K[0] >> 24) & 0xff] ^
- C5[(int)(K[7] >> 16) & 0xff] ^
- C6[(int)(K[6] >> 8) & 0xff] ^
- C7[(int)(K[5] ) & 0xff];
-
- L[5] = C0[(int)(K[5] >> 56) ] ^
- C1[(int)(K[4] >> 48) & 0xff] ^
- C2[(int)(K[3] >> 40) & 0xff] ^
- C3[(int)(K[2] >> 32) & 0xff] ^
- C4[(int)(K[1] >> 24) & 0xff] ^
- C5[(int)(K[0] >> 16) & 0xff] ^
- C6[(int)(K[7] >> 8) & 0xff] ^
- C7[(int)(K[6] ) & 0xff];
-
- L[6] = C0[(int)(K[6] >> 56) ] ^
- C1[(int)(K[5] >> 48) & 0xff] ^
- C2[(int)(K[4] >> 40) & 0xff] ^
- C3[(int)(K[3] >> 32) & 0xff] ^
- C4[(int)(K[2] >> 24) & 0xff] ^
- C5[(int)(K[1] >> 16) & 0xff] ^
- C6[(int)(K[0] >> 8) & 0xff] ^
- C7[(int)(K[7] ) & 0xff];
-
- L[7] = C0[(int)(K[7] >> 56) ] ^
- C1[(int)(K[6] >> 48) & 0xff] ^
- C2[(int)(K[5] >> 40) & 0xff] ^
- C3[(int)(K[4] >> 32) & 0xff] ^
- C4[(int)(K[3] >> 24) & 0xff] ^
- C5[(int)(K[2] >> 16) & 0xff] ^
- C6[(int)(K[1] >> 8) & 0xff] ^
- C7[(int)(K[0] ) & 0xff];
+ L[0] = C0[BYTE7(K[0])];
+ L[0] ^= C1[BYTE6(K[7])];
+ L[0] ^= C2[BYTE5(K[6])];
+ L[0] ^= C3[BYTE4(K[5])];
+ L[0] ^= C4[BYTE3(K[4])];
+ L[0] ^= C5[BYTE2(K[3])];
+ L[0] ^= C6[BYTE1(K[2])];
+ L[0] ^= C7[BYTE0(K[1])];
+ L[0] ^= rc[r];
+
+ L[1] = C0[BYTE7(K[1])];
+ L[1] ^= C1[BYTE6(K[0])];
+ L[1] ^= C2[BYTE5(K[7])];
+ L[1] ^= C3[BYTE4(K[6])];
+ L[1] ^= C4[BYTE3(K[5])];
+ L[1] ^= C5[BYTE2(K[4])];
+ L[1] ^= C6[BYTE1(K[3])];
+ L[1] ^= C7[BYTE0(K[2])];
+
+ L[2] = C0[BYTE7(K[2])];
+ L[2] ^= C1[BYTE6(K[1])];
+ L[2] ^= C2[BYTE5(K[0])];
+ L[2] ^= C3[BYTE4(K[7])];
+ L[2] ^= C4[BYTE3(K[6])];
+ L[2] ^= C5[BYTE2(K[5])];
+ L[2] ^= C6[BYTE1(K[4])];
+ L[2] ^= C7[BYTE0(K[3])];
+
+ L[3] = C0[BYTE7(K[3])];
+ L[3] ^= C1[BYTE6(K[2])];
+ L[3] ^= C2[BYTE5(K[1])];
+ L[3] ^= C3[BYTE4(K[0])];
+ L[3] ^= C4[BYTE3(K[7])];
+ L[3] ^= C5[BYTE2(K[6])];
+ L[3] ^= C6[BYTE1(K[5])];
+ L[3] ^= C7[BYTE0(K[4])];
+
+ L[4] = C0[BYTE7(K[4])];
+ L[4] ^= C1[BYTE6(K[3])];
+ L[4] ^= C2[BYTE5(K[2])];
+ L[4] ^= C3[BYTE4(K[1])];
+ L[4] ^= C4[BYTE3(K[0])];
+ L[4] ^= C5[BYTE2(K[7])];
+ L[4] ^= C6[BYTE1(K[6])];
+ L[4] ^= C7[BYTE0(K[5])];
+
+ L[5] = C0[BYTE7(K[5])];
+ L[5] ^= C1[BYTE6(K[4])];
+ L[5] ^= C2[BYTE5(K[3])];
+ L[5] ^= C3[BYTE4(K[2])];
+ L[5] ^= C4[BYTE3(K[1])];
+ L[5] ^= C5[BYTE2(K[0])];
+ L[5] ^= C6[BYTE1(K[7])];
+ L[5] ^= C7[BYTE0(K[6])];
+
+ L[6] = C0[BYTE7(K[6])];
+ L[6] ^= C1[BYTE6(K[5])];
+ L[6] ^= C2[BYTE5(K[4])];
+ L[6] ^= C3[BYTE4(K[3])];
+ L[6] ^= C4[BYTE3(K[2])];
+ L[6] ^= C5[BYTE2(K[1])];
+ L[6] ^= C6[BYTE1(K[0])];
+ L[6] ^= C7[BYTE0(K[7])];
+
+ L[7] = C0[BYTE7(K[7])];
+ L[7] ^= C1[BYTE6(K[6])];
+ L[7] ^= C2[BYTE5(K[5])];
+ L[7] ^= C3[BYTE4(K[4])];
+ L[7] ^= C4[BYTE3(K[3])];
+ L[7] ^= C5[BYTE2(K[2])];
+ L[7] ^= C6[BYTE1(K[1])];
+ L[7] ^= C7[BYTE0(K[0])];
K[0] = L[0];
K[1] = L[1];
@@ -877,85 +881,85 @@ static void wp512_process_buffer(struct
K[6] = L[6];
K[7] = L[7];
- L[0] = C0[(int)(state[0] >> 56) ] ^
- C1[(int)(state[7] >> 48) & 0xff] ^
- C2[(int)(state[6] >> 40) & 0xff] ^
- C3[(int)(state[5] >> 32) & 0xff] ^
- C4[(int)(state[4] >> 24) & 0xff] ^
- C5[(int)(state[3] >> 16) & 0xff] ^
- C6[(int)(state[2] >> 8) & 0xff] ^
- C7[(int)(state[1] ) & 0xff] ^
- K[0];
-
- L[1] = C0[(int)(state[1] >> 56) ] ^
- C1[(int)(state[0] >> 48) & 0xff] ^
- C2[(int)(state[7] >> 40) & 0xff] ^
- C3[(int)(state[6] >> 32) & 0xff] ^
- C4[(int)(state[5] >> 24) & 0xff] ^
- C5[(int)(state[4] >> 16) & 0xff] ^
- C6[(int)(state[3] >> 8) & 0xff] ^
- C7[(int)(state[2] ) & 0xff] ^
- K[1];
-
- L[2] = C0[(int)(state[2] >> 56) ] ^
- C1[(int)(state[1] >> 48) & 0xff] ^
- C2[(int)(state[0] >> 40) & 0xff] ^
- C3[(int)(state[7] >> 32) & 0xff] ^
- C4[(int)(state[6] >> 24) & 0xff] ^
- C5[(int)(state[5] >> 16) & 0xff] ^
- C6[(int)(state[4] >> 8) & 0xff] ^
- C7[(int)(state[3] ) & 0xff] ^
- K[2];
-
- L[3] = C0[(int)(state[3] >> 56) ] ^
- C1[(int)(state[2] >> 48) & 0xff] ^
- C2[(int)(state[1] >> 40) & 0xff] ^
- C3[(int)(state[0] >> 32) & 0xff] ^
- C4[(int)(state[7] >> 24) & 0xff] ^
- C5[(int)(state[6] >> 16) & 0xff] ^
- C6[(int)(state[5] >> 8) & 0xff] ^
- C7[(int)(state[4] ) & 0xff] ^
- K[3];
-
- L[4] = C0[(int)(state[4] >> 56) ] ^
- C1[(int)(state[3] >> 48) & 0xff] ^
- C2[(int)(state[2] >> 40) & 0xff] ^
- C3[(int)(state[1] >> 32) & 0xff] ^
- C4[(int)(state[0] >> 24) & 0xff] ^
- C5[(int)(state[7] >> 16) & 0xff] ^
- C6[(int)(state[6] >> 8) & 0xff] ^
- C7[(int)(state[5] ) & 0xff] ^
- K[4];
-
- L[5] = C0[(int)(state[5] >> 56) ] ^
- C1[(int)(state[4] >> 48) & 0xff] ^
- C2[(int)(state[3] >> 40) & 0xff] ^
- C3[(int)(state[2] >> 32) & 0xff] ^
- C4[(int)(state[1] >> 24) & 0xff] ^
- C5[(int)(state[0] >> 16) & 0xff] ^
- C6[(int)(state[7] >> 8) & 0xff] ^
- C7[(int)(state[6] ) & 0xff] ^
- K[5];
-
- L[6] = C0[(int)(state[6] >> 56) ] ^
- C1[(int)(state[5] >> 48) & 0xff] ^
- C2[(int)(state[4] >> 40) & 0xff] ^
- C3[(int)(state[3] >> 32) & 0xff] ^
- C4[(int)(state[2] >> 24) & 0xff] ^
- C5[(int)(state[1] >> 16) & 0xff] ^
- C6[(int)(state[0] >> 8) & 0xff] ^
- C7[(int)(state[7] ) & 0xff] ^
- K[6];
-
- L[7] = C0[(int)(state[7] >> 56) ] ^
- C1[(int)(state[6] >> 48) & 0xff] ^
- C2[(int)(state[5] >> 40) & 0xff] ^
- C3[(int)(state[4] >> 32) & 0xff] ^
- C4[(int)(state[3] >> 24) & 0xff] ^
- C5[(int)(state[2] >> 16) & 0xff] ^
- C6[(int)(state[1] >> 8) & 0xff] ^
- C7[(int)(state[0] ) & 0xff] ^
- K[7];
+ L[0] = C0[BYTE7(state[0])];
+ L[0] ^= C1[BYTE6(state[7])];
+ L[0] ^= C2[BYTE5(state[6])];
+ L[0] ^= C3[BYTE4(state[5])];
+ L[0] ^= C4[BYTE3(state[4])];
+ L[0] ^= C5[BYTE2(state[3])];
+ L[0] ^= C6[BYTE1(state[2])];
+ L[0] ^= C7[BYTE0(state[1])];
+ L[0] ^= K[0];
+
+ L[1] = C0[BYTE7(state[1])];
+ L[1] ^= C1[BYTE6(state[0])];
+ L[1] ^= C2[BYTE5(state[7])];
+ L[1] ^= C3[BYTE4(state[6])];
+ L[1] ^= C4[BYTE3(state[5])];
+ L[1] ^= C5[BYTE2(state[4])];
+ L[1] ^= C6[BYTE1(state[3])];
+ L[1] ^= C7[BYTE0(state[2])];
+ L[1] ^= K[1];
+
+ L[2] = C0[BYTE7(state[2])];
+ L[2] ^= C1[BYTE6(state[1])];
+ L[2] ^= C2[BYTE5(state[0])];
+ L[2] ^= C3[BYTE4(state[7])];
+ L[2] ^= C4[BYTE3(state[6])];
+ L[2] ^= C5[BYTE2(state[5])];
+ L[2] ^= C6[BYTE1(state[4])];
+ L[2] ^= C7[BYTE0(state[3])];
+ L[2] ^= K[2];
+
+ L[3] = C0[BYTE7(state[3])];
+ L[3] ^= C1[BYTE6(state[2])];
+ L[3] ^= C2[BYTE5(state[1])];
+ L[3] ^= C3[BYTE4(state[0])];
+ L[3] ^= C4[BYTE3(state[7])];
+ L[3] ^= C5[BYTE2(state[6])];
+ L[3] ^= C6[BYTE1(state[5])];
+ L[3] ^= C7[BYTE0(state[4])];
+ L[3] ^= K[3];
+
+ L[4] = C0[BYTE7(state[4])];
+ L[4] ^= C1[BYTE6(state[3])];
+ L[4] ^= C2[BYTE5(state[2])];
+ L[4] ^= C3[BYTE4(state[1])];
+ L[4] ^= C4[BYTE3(state[0])];
+ L[4] ^= C5[BYTE2(state[7])];
+ L[4] ^= C6[BYTE1(state[6])];
+ L[4] ^= C7[BYTE0(state[5])];
+ L[4] ^= K[4];
+
+ L[5] = C0[BYTE7(state[5])];
+ L[5] ^= C1[BYTE6(state[4])];
+ L[5] ^= C2[BYTE5(state[3])];
+ L[5] ^= C3[BYTE4(state[2])];
+ L[5] ^= C4[BYTE3(state[1])];
+ L[5] ^= C5[BYTE2(state[0])];
+ L[5] ^= C6[BYTE1(state[7])];
+ L[5] ^= C7[BYTE0(state[6])];
+ L[5] ^= K[5];
+
+ L[6] = C0[BYTE7(state[6])];
+ L[6] ^= C1[BYTE6(state[5])];
+ L[6] ^= C2[BYTE5(state[4])];
+ L[6] ^= C3[BYTE4(state[3])];
+ L[6] ^= C4[BYTE3(state[2])];
+ L[6] ^= C5[BYTE2(state[1])];
+ L[6] ^= C6[BYTE1(state[0])];
+ L[6] ^= C7[BYTE0(state[7])];
+ L[6] ^= K[6];
+
+ L[7] = C0[BYTE7(state[7])];
+ L[7] ^= C1[BYTE6(state[6])];
+ L[7] ^= C2[BYTE5(state[5])];
+ L[7] ^= C3[BYTE4(state[4])];
+ L[7] ^= C4[BYTE3(state[3])];
+ L[7] ^= C5[BYTE2(state[2])];
+ L[7] ^= C6[BYTE1(state[1])];
+ L[7] ^= C7[BYTE0(state[0])];
+ L[7] ^= K[7];
state[0] = L[0];
state[1] = L[1];
next prev parent reply other threads:[~2005-07-03 13:13 UTC|newest]
Thread overview: 8+ messages / expand[flat|nested] mbox.gz Atom feed top
2005-06-20 10:54 [PATCH 0/5] random crypto cleanups Denis Vlasenko
2005-07-03 11:37 ` Herbert Xu
2005-07-03 12:57 ` Denis Vlasenko
2005-07-03 13:00 ` [PATCH 0.5/4 :) ] ROR -> ror32 Denis Vlasenko
2005-07-03 13:03 ` [PATCH 1/4] do not open-code be/le conversions Denis Vlasenko
2005-07-03 13:05 ` Denis Vlasenko [this message]
2005-07-03 13:06 ` [PATCH 3/4] twofish on diet Denis Vlasenko
2005-07-03 13:14 ` [PATCH 4/4] 64 bit rotation Denis Vlasenko
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=200507031605.23742.vda@ilport.com.ua \
--to=vda@ilport.com.ua \
--cc=davem@davemloft.net \
--cc=herbert@gondor.apana.org.au \
--cc=linux-crypto@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®