mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: Usama Arif <usama.arif@linux.dev>
To: Peter Zijlstra <peterz@infradead.org>,
	ndesaulniers@google.com,
	David Laight <david.laight.linux@gmail.com>
Cc: tglx@kernel.org, mingo@redhat.com, bp@alien8.de,
	dave.hansen@linux.intel.com, x86@kernel.org, kees@kernel.org,
	hpa@zytor.com, gustavoars@kernel.org, ardb@kernel.org,
	hca@linux.ibm.com, nathan@kernel.org, ndesaulniers@google.com,
	morbo@google.com, justinstitt@google.com,
	linux-hardening@vger.kernel.org, llvm@lists.linux.dev,
	linux-kernel@vger.kernel.org, riel@surriel.com,
	shakeel.butt@linux.dev, kernel-team@meta.com, rkeilty@meta.com,
	atish.patra@linux.dev, dgreid@meta.com
Subject: Re: [PATCH 2/2] x86/kstack_erase: Use REP STOSQ for large stack fills
Date: Fri, 20 Nov 2026 18:43:38 +0000	[thread overview]
Message-ID: <bad389b1-f4a6-43d4-9531-54b75e067c7a@linux.dev> (raw)
In-Reply-To: <20261120122635.GJ2669272@noisy.programming.kicks-ass.net>



On 20/11/2026 04:26, Peter Zijlstra wrote:
> On Fri, Nov 20, 2026 at 11:45:34AM +0100, Peter Zijlstra wrote:
>> On Thu, Nov 19, 2026 at 01:31:08PM -0800, Usama Arif wrote:
>>> +#ifdef CONFIG_X86_64
>>> +#define __stackleak_poison __stackleak_poison
>>> +static __always_inline void __stackleak_poison(unsigned long erase_low,
>>> +					       unsigned long erase_high,
>>> +					       unsigned long poison)
>>> +{
>>> +	unsigned long count;
>>> +
>>> +	/* Short spans do not amortize the startup cost of REP STOSQ. */
>>> +	if (erase_high <= erase_low + 512) {
>>> +		while (erase_low < erase_high) {
>>> +			*(unsigned long *)erase_low = poison;
>>> +			erase_low += sizeof(unsigned long);
>>> +		}
>>> +		return;
>>> +	}
>>> +
>>> +	count = (erase_high - erase_low) / sizeof(unsigned long);
>>> +	asm volatile("rep stosq"
>>> +		     : "+D" (erase_low), "+c" (count)
>>> +		     : "a" (poison)
>>> +		     : "memory");
>>> +}
>>> +#endif
>>
>> Gemini seems to suggest 16-32 words would be sufficient to amortize the
>> REP STOS setup. It was also adamant I try and unroll that earlier loop.
>>
>> How about something like so, does that make sense? Also, I suppose we
>> should make memset64() __always_inline if we use it here. I'm not at all
>> sold on the unroll here, the asm is pretty horrid for what it does.
> 
> This one is much improved:
> 
> --- a/arch/x86/include/asm/processor.h
> +++ b/arch/x86/include/asm/processor.h
> @@ -28,6 +28,7 @@ struct vm86;
>  #include <asm/vmxfeatures.h>
>  #include <asm/vdso/processor.h>
>  #include <asm/shstk.h>
> +#include <asm/string.h>
>  
>  #include <linux/personality.h>
>  #include <linux/cache.h>
> @@ -580,6 +581,35 @@ static __always_inline bool on_thread_st
>  			       current_stack_pointer) < THREAD_SIZE;
>  }
>  
> +#ifdef CONFIG_X86_64
> +#define __stackleak_poison __stackleak_poison
> +static __always_inline void __stackleak_poison(unsigned long erase_low,
> +					       unsigned long erase_high,
> +					       unsigned long poison)
> +{
> +	unsigned long count = (erase_high - erase_low) / sizeof(unsigned long);
> +
> +	/* Amortize the startup cost of REP STOSQ. */
> +	if (count < 32) {
> +		unsigned long *ptr = (unsigned long *)erase_low;
> +
> +		while (count) {
> +			unsigned long mod = (count-1) % 4;
> +			count -= mod+1;
> +			switch (mod) {
> +			case 3: ptr[count+3] = poison; fallthrough;
> +			case 2: ptr[count+2] = poison; fallthrough;
> +			case 1: ptr[count+1] = poison; fallthrough;
> +			case 0: ptr[count+0] = poison; break;
> +			}
> +		}
> +		return;
> +	}
> +

Hello Nick, Peter and David,

Thanks for the reviews! I should have used memset64 from the get go :)

I have tested 2 possible revisions now as well as this one from Peter,
have added both of them at the end of the reply.

- v2.1 where you loop upto 512B and then memset
- v2.2: 4 stores per loop up to 512 B, then 64-byte aligned memset64()

I timed stackleak_erase() in a Zen 5c guest over a futex/epoll/procfs/unix
syscall:

  No patch                             68.3 ns per erase
  v1 (loop up to 512 B, then REP)           46.6 ns
  v2.1 (loop up to 512 B, memset64())       46.6 ns
  this version                              46.4 ns
  v2.2 (4 stores/iteration, aligned REP)    43.7 ns


The switch doesn't seem to help.

Either way I kept 512 bytes: REP STOSQ only wins from ~430 bytes on Zen
5c against the plain loop (~850 against the unrolled one), and from ~320
on Cooper Lake.


My personal preference is v2.1, as its much simpler code and the difference
in performance between v2.1 and v2.2 isn't actually that much. But if there is
a stronger preference for v2.2, I can send that as it is faster.


v2.1:

diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 77488ff5eb8c7c..3d205ae8566114 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -28,6 +28,7 @@ struct vm86;
 #include <asm/vmxfeatures.h>
 #include <asm/vdso/processor.h>
 #include <asm/shstk.h>
+#include <asm/string.h>

 #include <linux/personality.h>
 #include <linux/cache.h>
@@ -580,6 +581,26 @@ static __always_inline bool on_thread_stack(void)
                               current_stack_pointer) < THREAD_SIZE;
 }

+#if defined(CONFIG_X86_64) && !defined(CONFIG_KMSAN)
+#define __stackleak_poison __stackleak_poison
+static __always_inline void __stackleak_poison(unsigned long erase_low,
+                                              unsigned long erase_high,
+                                              unsigned long poison)
+{
+       /* Short spans do not amortize the startup cost of REP STOSQ. */
+       if (erase_high <= erase_low + 512) {
+               while (erase_low < erase_high) {
+                       *(unsigned long *)erase_low = poison;
+                       erase_low += sizeof(unsigned long);
+               }
+               return;
+       }
+
+       memset64((u64 *)erase_low, poison,
+                (erase_high - erase_low) / sizeof(u64));
+}
+#endif
+
 #ifdef CONFIG_PARAVIRT_XXL
 #include <asm/paravirt.h>
 #else
diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string_64.h
index 2dae70c2b01465..73cc91254b1381 100644
--- a/arch/x86/include/asm/string_64.h
+++ b/arch/x86/include/asm/string_64.h
@@ -54,7 +54,7 @@ static inline void *memset32(uint32_t *s, uint32_t v, size_t n)
 }

 #define __HAVE_ARCH_MEMSET64
-static inline void *memset64(uint64_t *s, uint64_t v, size_t n)
+static __always_inline void *memset64(uint64_t *s, uint64_t v, size_t n)
 {
        const auto s0 = s;
        asm volatile (





v2.2:

diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 77488ff5eb8c7c..5e76e1140908d7 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -28,7 +28,9 @@ struct vm86;
 #include <asm/vmxfeatures.h>
 #include <asm/vdso/processor.h>
 #include <asm/shstk.h>
+#include <asm/string.h>

+#include <linux/align.h>
 #include <linux/personality.h>
 #include <linux/cache.h>
 #include <linux/threads.h>
@@ -580,6 +582,48 @@ static __always_inline bool on_thread_stack(void)
                               current_stack_pointer) < THREAD_SIZE;
 }

+#if defined(CONFIG_X86_64) && !defined(CONFIG_KMSAN)
+#define __stackleak_poison __stackleak_poison
+static __always_inline void __stackleak_poison(unsigned long erase_low,
+                                              unsigned long erase_high,
+                                              unsigned long poison)
+{
+       unsigned long *p = (unsigned long *)erase_low;
+       unsigned long *end = (unsigned long *)erase_high;
+       unsigned long i;
+
+       /* Fewer than four words, or nothing to do. */
+       if (erase_high < erase_low + 4 * sizeof(*p)) {
+               while (p < end)
+                       *p++ = poison;
+               return;
+       }
+
+       /* Four stores per iteration; the last four may overlap. */
+       if (erase_high - erase_low <= 512) {
+               end -= 4;
+               while (p < end) {
+                       p[0] = poison;
+                       p[1] = poison;
+                       p[2] = poison;
+                       p[3] = poison;
+                       p += 4;
+               }
+               end[0] = poison;
+               end[1] = poison;
+               end[2] = poison;
+               end[3] = poison;
+               return;
+       }
+
+       /* REP STOSQ is faster from a 64-byte aligned destination. */
+       for (i = 0; i < 8; i++)
+               p[i] = poison;
+       p = (unsigned long *)ALIGN(erase_low + 1, 64);
+       memset64((u64 *)p, poison, end - p);
+}
+#endif
+
 #ifdef CONFIG_PARAVIRT_XXL
 #include <asm/paravirt.h>
 #else
diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string_64.h
index 2dae70c2b01465..73cc91254b1381 100644
--- a/arch/x86/include/asm/string_64.h
+++ b/arch/x86/include/asm/string_64.h
@@ -54,7 +54,7 @@ static inline void *memset32(uint32_t *s, uint32_t v, size_t n)
 }

 #define __HAVE_ARCH_MEMSET64
-static inline void *memset64(uint64_t *s, uint64_t v, size_t n)
+static __always_inline void *memset64(uint64_t *s, uint64_t v, size_t n)
 {
        const auto s0 = s;
        asm volatile (

> +	memset64((void *)erase_low, poison, count);
> +}
> +#endif
> +
>  #ifdef CONFIG_PARAVIRT_XXL
>  #include <asm/paravirt.h>
>  #else


  reply	other threads:[~2026-11-20 18:43 UTC|newest]

Thread overview: 15+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-19 21:31 [PATCH 0/2] x86/kstack_erase: Faster stack poisoning, plus a startup build fix Usama Arif
2026-11-19 21:31 ` [PATCH 1/2] x86/boot/startup: Disable kernel stack depth tracking Usama Arif
2026-11-19 22:30   ` Nick Desaulniers
2026-11-19 21:31 ` [PATCH 2/2] x86/kstack_erase: Use REP STOSQ for large stack fills Usama Arif
2026-11-19 22:28   ` Nick Desaulniers
2026-11-20 11:38     ` David Laight
2026-11-20 12:42       ` David Laight
2026-11-20 18:46         ` Usama Arif
2026-11-20 10:45   ` Peter Zijlstra
2026-11-20 12:26     ` Peter Zijlstra
2026-11-20 18:43       ` Usama Arif [this message]
2026-11-20 14:02     ` David Laight
2026-11-20 15:35       ` Peter Zijlstra
2026-11-20 16:18         ` David Laight
2026-11-20 18:55   ` David Laight

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=bad389b1-f4a6-43d4-9531-54b75e067c7a@linux.dev \
    --to=usama.arif@linux.dev \
    --cc=ardb@kernel.org \
    --cc=atish.patra@linux.dev \
    --cc=bp@alien8.de \
    --cc=dave.hansen@linux.intel.com \
    --cc=david.laight.linux@gmail.com \
    --cc=dgreid@meta.com \
    --cc=gustavoars@kernel.org \
    --cc=hca@linux.ibm.com \
    --cc=hpa@zytor.com \
    --cc=justinstitt@google.com \
    --cc=kees@kernel.org \
    --cc=kernel-team@meta.com \
    --cc=linux-hardening@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=llvm@lists.linux.dev \
    --cc=mingo@redhat.com \
    --cc=morbo@google.com \
    --cc=nathan@kernel.org \
    --cc=ndesaulniers@google.com \
    --cc=peterz@infradead.org \
    --cc=riel@surriel.com \
    --cc=rkeilty@meta.com \
    --cc=shakeel.butt@linux.dev \
    --cc=tglx@kernel.org \
    --cc=x86@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®