[PATCH 0/2] VMSCAPE optimization for BHI variant

[PATCH 0/2] VMSCAPE optimization for BHI variant

[Pawan Gupta]

Hi All,

These patches aim to improve the performance of a recent mitigation for
VMSCAPE[1] vulnerability. This improvement is relevant for BHI variant of
VMSCAPE that affect Alder Lake and newer processors.

The current mitigation approach uses IBPB on kvm-exit-to-userspace for all
affected range of CPUs. This is an overkill for CPUs that are only affected
by the BHI variant. On such CPUs clearing the branch history is sufficient
for VMSCAPE, and also more apt as the underlying issue is due to poisoned
branch history.

Roadmap:

- First patch introduces clear_bhb_long_loop() for processors with larger
  branch history tables.
- Second patch replaces IBPB on exit-to-userspace with branch history
  clearing sequence.

Below is the iPerf data for transfer between guest and host, comparing IBPB
and BHB-clear mitigation. BHB-clear shows performance improvement over IBPB
in most cases.

Platform: Emerald Rapids
Baseline: vmscape=off

(..._pN below mean N parallel connections)

| iPerf user-net | IBPB    | BHB Clear |
|----------------|---------|-----------|
| UDP 1-vCPU_p1  | -12.5%  |   1.3%    |
| TCP 1-vCPU_p1  | -10.4%  |  -1.5%    |
| TCP 1-vCPU_p1  | -7.5%   |  -3.0%    |
| UDP 4-vCPU_p16 | -3.7%   |  -3.7%    |
| TCP 4-vCPU_p4  | -2.9%   |  -1.4%    |
| UDP 4-vCPU_p4  | -0.6%   |   0.0%    |
| TCP 4-vCPU_p4  |  3.5%   |   0.0%    |

| iPerf bridge-net | IBPB    | BHB Clear |
|------------------|---------|-----------|
| UDP 1-vCPU_p1    | -9.4%   |  -0.4%    |
| TCP 1-vCPU_p1    | -3.9%   |  -0.5%    |
| UDP 4-vCPU_p16   | -2.2%   |  -3.8%    |
| TCP 4-vCPU_p4    | -1.0%   |  -1.0%    |
| TCP 4-vCPU_p4    |  0.5%   |   0.5%    |
| UDP 4-vCPU_p4    |  0.0%   |   0.9%    |
| TCP 1-vCPU_p1    |  0.0%   |   0.9%    |

| iPerf vhost-net | IBPB    | BHB Clear |
|-----------------|---------|-----------|
| UDP 1-vCPU_p1   | -4.3%   |   1.0%    |
| TCP 1-vCPU_p1   | -3.8%   |  -0.5%    |
| TCP 1-vCPU_p1   | -2.7%   |  -0.7%    |
| UDP 4-vCPU_p16  | -0.7%   |  -2.2%    |
| TCP 4-vCPU_p4   | -0.4%   |   0.8%    |
| UDP 4-vCPU_p4   |  0.4%   |  -0.7%    |
| TCP 4-vCPU_p4   |  0.0%   |   0.6%    |

[1] https://comsec.ethz.ch/research/microarch/vmscape-exposing-and-exploiting-incomplete-branch-predictor-isolation-in-cloud-environments/

---
Pawan Gupta (2):
      x86/bhi: Add BHB clearing for CPUs with larger branch history
      x86/vmscape: Replace IBPB with branch history clear on exit to userspace

 Documentation/admin-guide/hw-vuln/vmscape.rst   |  8 +++++
 Documentation/admin-guide/kernel-parameters.txt |  4 ++-
 arch/x86/entry/entry_64.S                       | 47 ++++++++++++++++++-------
 arch/x86/include/asm/cpufeatures.h              |  1 +
 arch/x86/include/asm/entry-common.h             | 12 ++++---
 arch/x86/include/asm/nospec-branch.h            |  5 ++-
 arch/x86/kernel/cpu/bugs.c                      | 44 ++++++++++++++++-------
 arch/x86/kvm/x86.c                              |  5 +--
 8 files changed, 92 insertions(+), 34 deletions(-)
---
base-commit: 4ea5af08590825c79ba2f146482ed54443e22c28
change-id: 20250916-vmscape-bhb-d7d469977f2f

Best regards,
-- 
Pawan

[PATCH 1/2] x86/bhi: Add BHB clearing for CPUs with larger branch history

[Pawan Gupta]

Add a version of clear_bhb_loop() that works on CPUs with larger branch
history table such as Alder Lake and newer. This could serve as a cheaper
alternative to IBPB mitigation for VMSCAPE.

clear_bhb_loop() and the new clear_bhb_long_loop() only differ in the loop
counter. Convert the asm implementation of clear_bhb_loop() into a macro
that is used by both the variants, passing counter as an argument.

There is no difference in the output of:

  $ objdump --disassemble=clear_bhb_loop vmlinux

before and after this commit.

Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
---
 arch/x86/entry/entry_64.S            | 47 ++++++++++++++++++++++++++----------
 arch/x86/include/asm/nospec-branch.h |  3 +++
 2 files changed, 37 insertions(+), 13 deletions(-)

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index ed04a968cc7d0095ab0185b2e3b5beffb7680afd..f5f62af080d8ec6fe81e4dbe78ce44d08e62aa59 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -1499,11 +1499,6 @@ SYM_CODE_END(rewind_stack_and_make_dead)
  * from the branch history tracker in the Branch Predictor, therefore removing
  * user influence on subsequent BTB lookups.
  *
- * It should be used on parts prior to Alder Lake. Newer parts should use the
- * BHI_DIS_S hardware control instead. If a pre-Alder Lake part is being
- * virtualized on newer hardware the VMM should protect against BHI attacks by
- * setting BHI_DIS_S for the guests.
- *
  * CALLs/RETs are necessary to prevent Loop Stream Detector(LSD) from engaging
  * and not clearing the branch history. The call tree looks like:
  *
@@ -1529,11 +1524,12 @@ SYM_CODE_END(rewind_stack_and_make_dead)
  * that all RETs are in the second half of a cacheline to mitigate Indirect
  * Target Selection, rather than taking the slowpath via its_return_thunk.
  */
-SYM_FUNC_START(clear_bhb_loop)
+.macro	__CLEAR_BHB_LOOP outer_loop_count:req, inner_loop_count:req
 	ANNOTATE_NOENDBR
 	push	%rbp
 	mov	%rsp, %rbp
-	movl	$5, %ecx
+
+	movl	$\outer_loop_count, %ecx
 	ANNOTATE_INTRA_FUNCTION_CALL
 	call	1f
 	jmp	5f
@@ -1542,29 +1538,54 @@ SYM_FUNC_START(clear_bhb_loop)
 	 * Shift instructions so that the RET is in the upper half of the
 	 * cacheline and don't take the slowpath to its_return_thunk.
 	 */
-	.skip 32 - (.Lret1 - 1f), 0xcc
+	.skip 32 - (.Lret1_\@ - 1f), 0xcc
 	ANNOTATE_INTRA_FUNCTION_CALL
 1:	call	2f
-.Lret1:	RET
+.Lret1_\@:
+	RET
 	.align 64, 0xcc
 	/*
-	 * As above shift instructions for RET at .Lret2 as well.
+	 * As above shift instructions for RET at .Lret2_\@ as well.
 	 *
-	 * This should be ideally be: .skip 32 - (.Lret2 - 2f), 0xcc
+	 * This should be ideally be: .skip 32 - (.Lret2_\@ - 2f), 0xcc
 	 * but some Clang versions (e.g. 18) don't like this.
 	 */
 	.skip 32 - 18, 0xcc
-2:	movl	$5, %eax
+2:	movl	$\inner_loop_count, %eax
 3:	jmp	4f
 	nop
 4:	sub	$1, %eax
 	jnz	3b
 	sub	$1, %ecx
 	jnz	1b
-.Lret2:	RET
+.Lret2_\@:
+	RET
 5:	lfence
+
 	pop	%rbp
 	RET
+.endm
+
+/*
+ * This should be used on parts prior to Alder Lake. Newer parts should use the
+ * BHI_DIS_S hardware control instead. If a pre-Alder Lake part is being
+ * virtualized on newer hardware the VMM should protect against BHI attacks by
+ * setting BHI_DIS_S for the guests.
+ */
+SYM_FUNC_START(clear_bhb_loop)
+	__CLEAR_BHB_LOOP 5, 5
 SYM_FUNC_END(clear_bhb_loop)
 EXPORT_SYMBOL_GPL(clear_bhb_loop)
 STACK_FRAME_NON_STANDARD(clear_bhb_loop)
+
+/*
+ * A longer version of clear_bhb_loop to ensure that the BHB is cleared on CPUs
+ * with larger branch history tables (i.e. Alder Lake and newer). BHI_DIS_S
+ * protects the kernel, but to mitigate the guest influence on the host
+ * userspace either IBPB or this sequence should be used. See VMSCAPE bug.
+ */
+SYM_FUNC_START(clear_bhb_long_loop)
+	__CLEAR_BHB_LOOP 12, 7
+SYM_FUNC_END(clear_bhb_long_loop)
+EXPORT_SYMBOL_GPL(clear_bhb_long_loop)
+STACK_FRAME_NON_STANDARD(clear_bhb_long_loop)
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index e29f82466f4323aeb2daedb876a34cf686016549..ad7e9d1b3a70cce1f24697e35cecd7761bb1984a 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -388,6 +388,9 @@ extern void write_ibpb(void);
 
 #ifdef CONFIG_X86_64
 extern void clear_bhb_loop(void);
+extern void clear_bhb_long_loop(void);
+#else
+static inline void clear_bhb_long_loop(void) {}
 #endif
 
 extern void (*x86_return_thunk)(void);

-- 
2.34.1

Re: [PATCH 1/2] x86/bhi: Add BHB clearing for CPUs with larger branch history

[Jim Mattson]

On Wed, Sep 24, 2025 at 8:09 PM Pawan Gupta
<pawan.kumar.gupta@linux.intel.com> wrote:
>
> Add a version of clear_bhb_loop() that works on CPUs with larger branch
> history table such as Alder Lake and newer. This could serve as a cheaper
> alternative to IBPB mitigation for VMSCAPE.

Yay!

Can we also use this longer loop as a BHI mitigation on (virtual)
processors with larger branch history tables that don't support
BHI_DIS_S? Today, we just use the short BHB clearing loop and call it
good.

Re: [PATCH 1/2] x86/bhi: Add BHB clearing for CPUs with larger branch history

[Pawan Gupta]

On Thu, Sep 25, 2025 at 10:54:48AM -0700, Jim Mattson wrote:
> On Wed, Sep 24, 2025 at 8:09 PM Pawan Gupta
> <pawan.kumar.gupta@linux.intel.com> wrote:
> >
> > Add a version of clear_bhb_loop() that works on CPUs with larger branch
> > history table such as Alder Lake and newer. This could serve as a cheaper
> > alternative to IBPB mitigation for VMSCAPE.
> 
> Yay!
> 
> Can we also use this longer loop as a BHI mitigation on (virtual)
> processors with larger branch history tables that don't support
> BHI_DIS_S? Today, we just use the short BHB clearing loop and call it
> good.

I believe you are referring to guests that don't enumerate BHI_DIS_S, but
are running on a host that supports BHI_DIS_S. In that case the longer loop
would work to mitigate BHI.

You probably know it already, the longer loop is not an optimal mitigation
compared to BHI_DIS_S. And on CPUs that don't support BHI_DIS_S, short loop
is sufficient.

I think you are talking about some special use cases like a guest migrating
from a CPU that didn't support BHI_DIS_S to a CPU that does. Using the long
loop in that case would be an option.

[PATCH 2/2] x86/vmscape: Replace IBPB with branch history clear on exit to userspace

[Pawan Gupta]

IBPB mitigation for VMSCAPE is an overkill for CPUs that are only affected
by the BHI variant of VMSCAPE. On such CPUs, eIBRS already provides
indirect branch isolation between guest and host userspace. But, a guest
could still poison the branch history.

To mitigate that, use the recently added clear_bhb_long_loop() to isolate
the branch history between guest and userspace. Add cmdline option
'vmscape=auto' that automatically selects the appropriate mitigation based
on the CPU.

Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
---
 Documentation/admin-guide/hw-vuln/vmscape.rst   |  8 +++++
 Documentation/admin-guide/kernel-parameters.txt |  4 ++-
 arch/x86/include/asm/cpufeatures.h              |  1 +
 arch/x86/include/asm/entry-common.h             | 12 ++++---
 arch/x86/include/asm/nospec-branch.h            |  2 +-
 arch/x86/kernel/cpu/bugs.c                      | 44 ++++++++++++++++++-------
 arch/x86/kvm/x86.c                              |  5 +--
 7 files changed, 55 insertions(+), 21 deletions(-)

diff --git a/Documentation/admin-guide/hw-vuln/vmscape.rst b/Documentation/admin-guide/hw-vuln/vmscape.rst
index d9b9a2b6c114c05a7325e5f3c9d42129339b870b..13ca98f952f97daeb28194c3873e945b85eda6a1 100644
--- a/Documentation/admin-guide/hw-vuln/vmscape.rst
+++ b/Documentation/admin-guide/hw-vuln/vmscape.rst
@@ -86,6 +86,10 @@ The possible values in this file are:
    run a potentially malicious guest and issues an IBPB before the first
    exit to userspace after VM-exit.
 
+ * 'Mitigation: Clear BHB before exit to userspace':
+
+   As above conditional BHB clearing mitigation is enabled.
+
  * 'Mitigation: IBPB on VMEXIT':
 
    IBPB is issued on every VM-exit. This occurs when other mitigations like
@@ -108,3 +112,7 @@ The mitigation can be controlled via the ``vmscape=`` command line parameter:
 
    Force vulnerability detection and mitigation even on processors that are
    not known to be affected.
+
+ * ``vmscape=auto``:
+
+   Choose the mitigation based on the VMSCAPE variant the CPU is affected by.
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 5a7a83c411e9c526f8df6d28beb4c784aec3cac9..4596bfcb401f1a89d2dc5ed8c44c83628c9c5dfe 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -8048,9 +8048,11 @@
 
 			off		- disable the mitigation
 			ibpb		- use Indirect Branch Prediction Barrier
-					  (IBPB) mitigation (default)
+					  (IBPB) mitigation
 			force		- force vulnerability detection even on
 					  unaffected processors
+			auto		- (default) automatically select IBPB
+			                  or BHB clear mitigation based on CPU
 
 	vsyscall=	[X86-64,EARLY]
 			Controls the behavior of vsyscalls (i.e. calls to
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 751ca35386b0ef02ad8413321028c15086b3a552..b7382735165210b006449f9f36f59d97d839cfe2 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -496,6 +496,7 @@
 #define X86_FEATURE_TSA_L1_NO		(21*32+12) /* AMD CPU not vulnerable to TSA-L1 */
 #define X86_FEATURE_CLEAR_CPU_BUF_VM	(21*32+13) /* Clear CPU buffers using VERW before VMRUN */
 #define X86_FEATURE_IBPB_EXIT_TO_USER	(21*32+14) /* Use IBPB on exit-to-userspace, see VMSCAPE bug */
+#define X86_FEATURE_CLEAR_BHB_EXIT_TO_USER (21*32+15) /* Clear branch history on exit-to-userspace, see VMSCAPE bug */
 
 /*
  * BUG word(s)
diff --git a/arch/x86/include/asm/entry-common.h b/arch/x86/include/asm/entry-common.h
index ce3eb6d5fdf9f2dba59b7bad24afbfafc8c36918..b7b9af1b641385b8283edf2449578ff65e5bd6df 100644
--- a/arch/x86/include/asm/entry-common.h
+++ b/arch/x86/include/asm/entry-common.h
@@ -94,11 +94,13 @@ static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
 	 */
 	choose_random_kstack_offset(rdtsc());
 
-	/* Avoid unnecessary reads of 'x86_ibpb_exit_to_user' */
-	if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER) &&
-	    this_cpu_read(x86_ibpb_exit_to_user)) {
-		indirect_branch_prediction_barrier();
-		this_cpu_write(x86_ibpb_exit_to_user, false);
+	if (unlikely(this_cpu_read(x86_pred_flush_pending))) {
+		if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER))
+			indirect_branch_prediction_barrier();
+		else if (cpu_feature_enabled(X86_FEATURE_CLEAR_BHB_EXIT_TO_USER))
+			clear_bhb_long_loop();
+
+		this_cpu_write(x86_pred_flush_pending, false);
 	}
 }
 #define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index ad7e9d1b3a70cce1f24697e35cecd7761bb1984a..32d52f32a5e7761fa1988a054a5d40debf67f53f 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -533,7 +533,7 @@ void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature)
 		: "memory");
 }
 
-DECLARE_PER_CPU(bool, x86_ibpb_exit_to_user);
+DECLARE_PER_CPU(bool, x86_pred_flush_pending);
 
 static inline void indirect_branch_prediction_barrier(void)
 {
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 36dcfc5105be9acb6d67a0481949ff03874d5f5d..2f1a86d758777f03bb69b0dc60591108c5660f77 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -109,12 +109,11 @@ DEFINE_PER_CPU(u64, x86_spec_ctrl_current);
 EXPORT_PER_CPU_SYMBOL_GPL(x86_spec_ctrl_current);
 
 /*
- * Set when the CPU has run a potentially malicious guest. An IBPB will
- * be needed to before running userspace. That IBPB will flush the branch
- * predictor content.
+ * Set when the CPU has run a potentially malicious guest. Indicates that a
+ * branch predictor flush is needed before running userspace.
  */
-DEFINE_PER_CPU(bool, x86_ibpb_exit_to_user);
-EXPORT_PER_CPU_SYMBOL_GPL(x86_ibpb_exit_to_user);
+DEFINE_PER_CPU(bool, x86_pred_flush_pending);
+EXPORT_PER_CPU_SYMBOL_GPL(x86_pred_flush_pending);
 
 u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB;
 
@@ -3270,13 +3269,15 @@ enum vmscape_mitigations {
 	VMSCAPE_MITIGATION_AUTO,
 	VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER,
 	VMSCAPE_MITIGATION_IBPB_ON_VMEXIT,
+	VMSCAPE_MITIGATION_BHB_CLEAR_EXIT_TO_USER,
 };
 
 static const char * const vmscape_strings[] = {
-	[VMSCAPE_MITIGATION_NONE]		= "Vulnerable",
+	[VMSCAPE_MITIGATION_NONE]			= "Vulnerable",
 	/* [VMSCAPE_MITIGATION_AUTO] */
-	[VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER]	= "Mitigation: IBPB before exit to userspace",
-	[VMSCAPE_MITIGATION_IBPB_ON_VMEXIT]	= "Mitigation: IBPB on VMEXIT",
+	[VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER]		= "Mitigation: IBPB before exit to userspace",
+	[VMSCAPE_MITIGATION_IBPB_ON_VMEXIT]		= "Mitigation: IBPB on VMEXIT",
+	[VMSCAPE_MITIGATION_BHB_CLEAR_EXIT_TO_USER]	= "Mitigation: Clear BHB before exit to userspace",
 };
 
 static enum vmscape_mitigations vmscape_mitigation __ro_after_init =
@@ -3294,6 +3295,8 @@ static int __init vmscape_parse_cmdline(char *str)
 	} else if (!strcmp(str, "force")) {
 		setup_force_cpu_bug(X86_BUG_VMSCAPE);
 		vmscape_mitigation = VMSCAPE_MITIGATION_AUTO;
+	} else if (!strcmp(str, "auto")) {
+		vmscape_mitigation = VMSCAPE_MITIGATION_AUTO;
 	} else {
 		pr_err("Ignoring unknown vmscape=%s option.\n", str);
 	}
@@ -3304,14 +3307,28 @@ early_param("vmscape", vmscape_parse_cmdline);
 
 static void __init vmscape_select_mitigation(void)
 {
-	if (cpu_mitigations_off() ||
-	    !boot_cpu_has_bug(X86_BUG_VMSCAPE) ||
-	    !boot_cpu_has(X86_FEATURE_IBPB)) {
+	if (cpu_mitigations_off() || !boot_cpu_has_bug(X86_BUG_VMSCAPE)) {
 		vmscape_mitigation = VMSCAPE_MITIGATION_NONE;
 		return;
 	}
 
-	if (vmscape_mitigation == VMSCAPE_MITIGATION_AUTO)
+	if (vmscape_mitigation == VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER &&
+	    !boot_cpu_has(X86_FEATURE_IBPB)) {
+		pr_err("IBPB not supported, switching to AUTO select\n");
+		vmscape_mitigation = VMSCAPE_MITIGATION_AUTO;
+	}
+
+	if (vmscape_mitigation != VMSCAPE_MITIGATION_AUTO)
+		return;
+
+	/*
+	 * CPUs with BHI_CTRL(ADL and newer) can avoid the IBPB and use BHB
+	 * clear sequence. These CPUs are only vulnerable to the BHI variant
+	 * of the VMSCAPE attack.
+	 */
+	if (boot_cpu_has(X86_FEATURE_BHI_CTRL))
+		vmscape_mitigation = VMSCAPE_MITIGATION_BHB_CLEAR_EXIT_TO_USER;
+	else
 		vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER;
 }
 
@@ -3331,6 +3348,8 @@ static void __init vmscape_apply_mitigation(void)
 {
 	if (vmscape_mitigation == VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER)
 		setup_force_cpu_cap(X86_FEATURE_IBPB_EXIT_TO_USER);
+	else if (vmscape_mitigation == VMSCAPE_MITIGATION_BHB_CLEAR_EXIT_TO_USER)
+		setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_EXIT_TO_USER);
 }
 
 #undef pr_fmt
@@ -3422,6 +3441,7 @@ void cpu_bugs_smt_update(void)
 		break;
 	case VMSCAPE_MITIGATION_IBPB_ON_VMEXIT:
 	case VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER:
+	case VMSCAPE_MITIGATION_BHB_CLEAR_EXIT_TO_USER:
 		/*
 		 * Hypervisors can be attacked across-threads, warn for SMT when
 		 * STIBP is not already enabled system-wide.
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 706b6fd56d3c5d29e7f9f6816beeebacb5ef68e6..190f193ef90e6615c5b12530f3f0c6632fda9137 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -11016,8 +11016,9 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 	 * set for the CPU that actually ran the guest, and not the CPU that it
 	 * may migrate to.
 	 */
-	if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER))
-		this_cpu_write(x86_ibpb_exit_to_user, true);
+	if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER) ||
+	    cpu_feature_enabled(X86_FEATURE_CLEAR_BHB_EXIT_TO_USER))
+		this_cpu_write(x86_pred_flush_pending, true);
 
 	/*
 	 * Consume any pending interrupts, including the possible source of

-- 
2.34.1

RE: [PATCH 2/2] x86/vmscape: Replace IBPB with branch history clear on exit to userspace

[Kaplan, David]

[AMD Official Use Only - AMD Internal Distribution Only]

> -----Original Message-----
> From: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
> Sent: Wednesday, September 24, 2025 10:10 PM
> To: x86@kernel.org; H. Peter Anvin <hpa@zytor.com>; Josh Poimboeuf
> <jpoimboe@kernel.org>; Kaplan, David <David.Kaplan@amd.com>; Sean
> Christopherson <seanjc@google.com>; Paolo Bonzini <pbonzini@redhat.com>
> Cc: linux-kernel@vger.kernel.org; kvm@vger.kernel.org; Asit Mallick
> <asit.k.mallick@intel.com>; Tao Zhang <tao1.zhang@intel.com>
> Subject: [PATCH 2/2] x86/vmscape: Replace IBPB with branch history clear on exit
> to userspace
>
> Caution: This message originated from an External Source. Use proper caution
> when opening attachments, clicking links, or responding.
>
>
> IBPB mitigation for VMSCAPE is an overkill for CPUs that are only affected
> by the BHI variant of VMSCAPE. On such CPUs, eIBRS already provides
> indirect branch isolation between guest and host userspace. But, a guest
> could still poison the branch history.
>
> To mitigate that, use the recently added clear_bhb_long_loop() to isolate
> the branch history between guest and userspace. Add cmdline option
> 'vmscape=auto' that automatically selects the appropriate mitigation based
> on the CPU.
>
> Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
> ---
>  Documentation/admin-guide/hw-vuln/vmscape.rst   |  8 +++++
>  Documentation/admin-guide/kernel-parameters.txt |  4 ++-
>  arch/x86/include/asm/cpufeatures.h              |  1 +
>  arch/x86/include/asm/entry-common.h             | 12 ++++---
>  arch/x86/include/asm/nospec-branch.h            |  2 +-
>  arch/x86/kernel/cpu/bugs.c                      | 44 ++++++++++++++++++-------
>  arch/x86/kvm/x86.c                              |  5 +--
>  7 files changed, 55 insertions(+), 21 deletions(-)
>
> diff --git a/Documentation/admin-guide/hw-vuln/vmscape.rst
> b/Documentation/admin-guide/hw-vuln/vmscape.rst
> index
> d9b9a2b6c114c05a7325e5f3c9d42129339b870b..13ca98f952f97daeb28194c3873e
> 945b85eda6a1 100644
> --- a/Documentation/admin-guide/hw-vuln/vmscape.rst
> +++ b/Documentation/admin-guide/hw-vuln/vmscape.rst
> @@ -86,6 +86,10 @@ The possible values in this file are:
>     run a potentially malicious guest and issues an IBPB before the first
>     exit to userspace after VM-exit.
>
> + * 'Mitigation: Clear BHB before exit to userspace':
> +
> +   As above conditional BHB clearing mitigation is enabled.
> +
>   * 'Mitigation: IBPB on VMEXIT':
>
>     IBPB is issued on every VM-exit. This occurs when other mitigations like
> @@ -108,3 +112,7 @@ The mitigation can be controlled via the ``vmscape=``
> command line parameter:
>
>     Force vulnerability detection and mitigation even on processors that are
>     not known to be affected.
> +
> + * ``vmscape=auto``:
> +
> +   Choose the mitigation based on the VMSCAPE variant the CPU is affected by.
> diff --git a/Documentation/admin-guide/kernel-parameters.txt
> b/Documentation/admin-guide/kernel-parameters.txt
> index
> 5a7a83c411e9c526f8df6d28beb4c784aec3cac9..4596bfcb401f1a89d2dc5ed8c44c8
> 3628c9c5dfe 100644
> --- a/Documentation/admin-guide/kernel-parameters.txt
> +++ b/Documentation/admin-guide/kernel-parameters.txt
> @@ -8048,9 +8048,11 @@
>
>                         off             - disable the mitigation
>                         ibpb            - use Indirect Branch Prediction Barrier
> -                                         (IBPB) mitigation (default)
> +                                         (IBPB) mitigation
>                         force           - force vulnerability detection even on
>                                           unaffected processors
> +                       auto            - (default) automatically select IBPB
> +                                         or BHB clear mitigation based on CPU

Many of the other bugs (like srso, l1tf, bhi, etc.) do not have explicit 'auto' options as 'auto' is implied by the lack of an explicit option.  Is there really value in creating an explicit 'auto' option here?

>
>  u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB;
>
> @@ -3270,13 +3269,15 @@ enum vmscape_mitigations {
>         VMSCAPE_MITIGATION_AUTO,
>         VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER,
>         VMSCAPE_MITIGATION_IBPB_ON_VMEXIT,
> +       VMSCAPE_MITIGATION_BHB_CLEAR_EXIT_TO_USER,
>  };
>
>  static const char * const vmscape_strings[] = {
> -       [VMSCAPE_MITIGATION_NONE]               = "Vulnerable",
> +       [VMSCAPE_MITIGATION_NONE]                       = "Vulnerable",
>         /* [VMSCAPE_MITIGATION_AUTO] */
> -       [VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER]  = "Mitigation: IBPB
> before exit to userspace",
> -       [VMSCAPE_MITIGATION_IBPB_ON_VMEXIT]     = "Mitigation: IBPB on
> VMEXIT",
> +       [VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER]          = "Mitigation: IBPB
> before exit to userspace",
> +       [VMSCAPE_MITIGATION_IBPB_ON_VMEXIT]             = "Mitigation: IBPB on
> VMEXIT",
> +       [VMSCAPE_MITIGATION_BHB_CLEAR_EXIT_TO_USER]     = "Mitigation:
> Clear BHB before exit to userspace",
>  };
>
>  static enum vmscape_mitigations vmscape_mitigation __ro_after_init =
> @@ -3294,6 +3295,8 @@ static int __init vmscape_parse_cmdline(char *str)
>         } else if (!strcmp(str, "force")) {
>                 setup_force_cpu_bug(X86_BUG_VMSCAPE);
>                 vmscape_mitigation = VMSCAPE_MITIGATION_AUTO;
> +       } else if (!strcmp(str, "auto")) {
> +               vmscape_mitigation = VMSCAPE_MITIGATION_AUTO;
>         } else {
>                 pr_err("Ignoring unknown vmscape=%s option.\n", str);
>         }
> @@ -3304,14 +3307,28 @@ early_param("vmscape", vmscape_parse_cmdline);
>
>  static void __init vmscape_select_mitigation(void)
>  {
> -       if (cpu_mitigations_off() ||
> -           !boot_cpu_has_bug(X86_BUG_VMSCAPE) ||
> -           !boot_cpu_has(X86_FEATURE_IBPB)) {
> +       if (cpu_mitigations_off() || !boot_cpu_has_bug(X86_BUG_VMSCAPE)) {
>                 vmscape_mitigation = VMSCAPE_MITIGATION_NONE;
>                 return;
>         }

It looks like this patch is based on a tree without vmscape attack vector support, I think you may want to rebase on top of that since it reworked some of this function.

>
> -       if (vmscape_mitigation == VMSCAPE_MITIGATION_AUTO)
> +       if (vmscape_mitigation == VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER
> &&
> +           !boot_cpu_has(X86_FEATURE_IBPB)) {
> +               pr_err("IBPB not supported, switching to AUTO select\n");
> +               vmscape_mitigation = VMSCAPE_MITIGATION_AUTO;
> +       }

I think there's a bug here in case you (theoretically) had a vulnerable CPU that did not have IBPB and did not have BHI_CTRL.  In that case, we should select VMSCAPE_MITIGATION_NONE as we have no mitigation available.  But the code below will still re-select IBPB I believe even though there is no IBPB.

> +
> +       if (vmscape_mitigation != VMSCAPE_MITIGATION_AUTO)
> +               return;
> +
> +       /*
> +        * CPUs with BHI_CTRL(ADL and newer) can avoid the IBPB and use BHB
> +        * clear sequence. These CPUs are only vulnerable to the BHI variant
> +        * of the VMSCAPE attack.
> +        */
> +       if (boot_cpu_has(X86_FEATURE_BHI_CTRL))
> +               vmscape_mitigation =
> VMSCAPE_MITIGATION_BHB_CLEAR_EXIT_TO_USER;
> +       else
>                 vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER;
>  }
>
> @@ -3331,6 +3348,8 @@ static void __init vmscape_apply_mitigation(void)
>  {
>         if (vmscape_mitigation == VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER)
>                 setup_force_cpu_cap(X86_FEATURE_IBPB_EXIT_TO_USER);
> +       else if (vmscape_mitigation ==
> VMSCAPE_MITIGATION_BHB_CLEAR_EXIT_TO_USER)
> +
> setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_EXIT_TO_USER);
>  }
>

--David Kaplan

Re: [PATCH 2/2] x86/vmscape: Replace IBPB with branch history clear on exit to userspace

[Pawan Gupta]

On Thu, Sep 25, 2025 at 06:14:54PM +0000, Kaplan, David wrote:
> [AMD Official Use Only - AMD Internal Distribution Only]
> 
> > -----Original Message-----
> > From: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
> > Sent: Wednesday, September 24, 2025 10:10 PM
> > To: x86@kernel.org; H. Peter Anvin <hpa@zytor.com>; Josh Poimboeuf
> > <jpoimboe@kernel.org>; Kaplan, David <David.Kaplan@amd.com>; Sean
> > Christopherson <seanjc@google.com>; Paolo Bonzini <pbonzini@redhat.com>
> > Cc: linux-kernel@vger.kernel.org; kvm@vger.kernel.org; Asit Mallick
> > <asit.k.mallick@intel.com>; Tao Zhang <tao1.zhang@intel.com>
> > Subject: [PATCH 2/2] x86/vmscape: Replace IBPB with branch history clear on exit
> > to userspace
> >
> > Caution: This message originated from an External Source. Use proper caution
> > when opening attachments, clicking links, or responding.
> >
> >
> > IBPB mitigation for VMSCAPE is an overkill for CPUs that are only affected
> > by the BHI variant of VMSCAPE. On such CPUs, eIBRS already provides
> > indirect branch isolation between guest and host userspace. But, a guest
> > could still poison the branch history.
> >
> > To mitigate that, use the recently added clear_bhb_long_loop() to isolate
> > the branch history between guest and userspace. Add cmdline option
> > 'vmscape=auto' that automatically selects the appropriate mitigation based
> > on the CPU.
> >
> > Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
> > ---
> >  Documentation/admin-guide/hw-vuln/vmscape.rst   |  8 +++++
> >  Documentation/admin-guide/kernel-parameters.txt |  4 ++-
> >  arch/x86/include/asm/cpufeatures.h              |  1 +
> >  arch/x86/include/asm/entry-common.h             | 12 ++++---
> >  arch/x86/include/asm/nospec-branch.h            |  2 +-
> >  arch/x86/kernel/cpu/bugs.c                      | 44 ++++++++++++++++++-------
> >  arch/x86/kvm/x86.c                              |  5 +--
> >  7 files changed, 55 insertions(+), 21 deletions(-)
> >
> > diff --git a/Documentation/admin-guide/hw-vuln/vmscape.rst
> > b/Documentation/admin-guide/hw-vuln/vmscape.rst
> > index
> > d9b9a2b6c114c05a7325e5f3c9d42129339b870b..13ca98f952f97daeb28194c3873e
> > 945b85eda6a1 100644
> > --- a/Documentation/admin-guide/hw-vuln/vmscape.rst
> > +++ b/Documentation/admin-guide/hw-vuln/vmscape.rst
> > @@ -86,6 +86,10 @@ The possible values in this file are:
> >     run a potentially malicious guest and issues an IBPB before the first
> >     exit to userspace after VM-exit.
> >
> > + * 'Mitigation: Clear BHB before exit to userspace':
> > +
> > +   As above conditional BHB clearing mitigation is enabled.
> > +
> >   * 'Mitigation: IBPB on VMEXIT':
> >
> >     IBPB is issued on every VM-exit. This occurs when other mitigations like
> > @@ -108,3 +112,7 @@ The mitigation can be controlled via the ``vmscape=``
> > command line parameter:
> >
> >     Force vulnerability detection and mitigation even on processors that are
> >     not known to be affected.
> > +
> > + * ``vmscape=auto``:
> > +
> > +   Choose the mitigation based on the VMSCAPE variant the CPU is affected by.
> > diff --git a/Documentation/admin-guide/kernel-parameters.txt
> > b/Documentation/admin-guide/kernel-parameters.txt
> > index
> > 5a7a83c411e9c526f8df6d28beb4c784aec3cac9..4596bfcb401f1a89d2dc5ed8c44c8
> > 3628c9c5dfe 100644
> > --- a/Documentation/admin-guide/kernel-parameters.txt
> > +++ b/Documentation/admin-guide/kernel-parameters.txt
> > @@ -8048,9 +8048,11 @@
> >
> >                         off             - disable the mitigation
> >                         ibpb            - use Indirect Branch Prediction Barrier
> > -                                         (IBPB) mitigation (default)
> > +                                         (IBPB) mitigation
> >                         force           - force vulnerability detection even on
> >                                           unaffected processors
> > +                       auto            - (default) automatically select IBPB
> > +                                         or BHB clear mitigation based on CPU
> 
> Many of the other bugs (like srso, l1tf, bhi, etc.) do not have explicit
> 'auto' options as 'auto' is implied by the lack of an explicit option.
> Is there really value in creating an explicit 'auto' option here?

Hmm, so to get the BHB clear mitigation do we advise the users to remove
the vmscape= parameter? That feels a bit weird to me. Also, with
CONFIG_MITIGATION_VMSCAPE=n a user can get IBPB mitigation with
vmscape=ibpb, but there is not way to get the BHB clear mitigation.

> >  u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB;
> >
> > @@ -3270,13 +3269,15 @@ enum vmscape_mitigations {
> >         VMSCAPE_MITIGATION_AUTO,
> >         VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER,
> >         VMSCAPE_MITIGATION_IBPB_ON_VMEXIT,
> > +       VMSCAPE_MITIGATION_BHB_CLEAR_EXIT_TO_USER,
> >  };
> >
> >  static const char * const vmscape_strings[] = {
> > -       [VMSCAPE_MITIGATION_NONE]               = "Vulnerable",
> > +       [VMSCAPE_MITIGATION_NONE]                       = "Vulnerable",
> >         /* [VMSCAPE_MITIGATION_AUTO] */
> > -       [VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER]  = "Mitigation: IBPB
> > before exit to userspace",
> > -       [VMSCAPE_MITIGATION_IBPB_ON_VMEXIT]     = "Mitigation: IBPB on
> > VMEXIT",
> > +       [VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER]          = "Mitigation: IBPB
> > before exit to userspace",
> > +       [VMSCAPE_MITIGATION_IBPB_ON_VMEXIT]             = "Mitigation: IBPB on
> > VMEXIT",
> > +       [VMSCAPE_MITIGATION_BHB_CLEAR_EXIT_TO_USER]     = "Mitigation:
> > Clear BHB before exit to userspace",
> >  };
> >
> >  static enum vmscape_mitigations vmscape_mitigation __ro_after_init =
> > @@ -3294,6 +3295,8 @@ static int __init vmscape_parse_cmdline(char *str)
> >         } else if (!strcmp(str, "force")) {
> >                 setup_force_cpu_bug(X86_BUG_VMSCAPE);
> >                 vmscape_mitigation = VMSCAPE_MITIGATION_AUTO;
> > +       } else if (!strcmp(str, "auto")) {
> > +               vmscape_mitigation = VMSCAPE_MITIGATION_AUTO;
> >         } else {
> >                 pr_err("Ignoring unknown vmscape=%s option.\n", str);
> >         }
> > @@ -3304,14 +3307,28 @@ early_param("vmscape", vmscape_parse_cmdline);
> >
> >  static void __init vmscape_select_mitigation(void)
> >  {
> > -       if (cpu_mitigations_off() ||
> > -           !boot_cpu_has_bug(X86_BUG_VMSCAPE) ||
> > -           !boot_cpu_has(X86_FEATURE_IBPB)) {
> > +       if (cpu_mitigations_off() || !boot_cpu_has_bug(X86_BUG_VMSCAPE)) {
> >                 vmscape_mitigation = VMSCAPE_MITIGATION_NONE;
> >                 return;
> >         }
> 
> It looks like this patch is based on a tree without vmscape attack vector
> support, I think you may want to rebase on top of that since it reworked
> some of this function.

Yes, it is based on upstream. I will rebase it once we are close to a final
version. I tend to base my patches on upstream to avoid any issues when tip
branches get rebased.

> > -       if (vmscape_mitigation == VMSCAPE_MITIGATION_AUTO)
> > +       if (vmscape_mitigation == VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER
> > &&
> > +           !boot_cpu_has(X86_FEATURE_IBPB)) {
> > +               pr_err("IBPB not supported, switching to AUTO select\n");
> > +               vmscape_mitigation = VMSCAPE_MITIGATION_AUTO;
> > +       }
> 
> I think there's a bug here in case you (theoretically) had a vulnerable
> CPU that did not have IBPB and did not have BHI_CTRL. In that case, we
> should select VMSCAPE_MITIGATION_NONE as we have no mitigation available.
> But the code below will still re-select IBPB I believe even though there
> is no IBPB.

Yes, you are right. Let me see how to fix that.

Re: [PATCH 2/2] x86/vmscape: Replace IBPB with branch history clear on exit to userspace

[Pawan Gupta]

On Thu, Sep 25, 2025 at 03:02:57PM -0700, Pawan Gupta wrote:
> On Thu, Sep 25, 2025 at 06:14:54PM +0000, Kaplan, David wrote:
> > > -       if (vmscape_mitigation == VMSCAPE_MITIGATION_AUTO)
> > > +       if (vmscape_mitigation == VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER
> > > &&
> > > +           !boot_cpu_has(X86_FEATURE_IBPB)) {
> > > +               pr_err("IBPB not supported, switching to AUTO select\n");
> > > +               vmscape_mitigation = VMSCAPE_MITIGATION_AUTO;
> > > +       }
> > 
> > I think there's a bug here in case you (theoretically) had a vulnerable
> > CPU that did not have IBPB and did not have BHI_CTRL. In that case, we
> > should select VMSCAPE_MITIGATION_NONE as we have no mitigation available.
> > But the code below will still re-select IBPB I believe even though there
> > is no IBPB.
> 
> Yes, you are right. Let me see how to fix that.

Below should fix it.

---
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 2f1a86d75877..60a2e54155e2 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -3328,8 +3328,10 @@ static void __init vmscape_select_mitigation(void)
 	 */
 	if (boot_cpu_has(X86_FEATURE_BHI_CTRL))
 		vmscape_mitigation = VMSCAPE_MITIGATION_BHB_CLEAR_EXIT_TO_USER;
-	else
+	else if (boot_cpu_has(X86_FEATURE_IBPB))
 		vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER;
+	else
+		vmscape_mitigation = VMSCAPE_MITIGATION_NONE;
 }
 
 static void __init vmscape_update_mitigation(void)

RE: [PATCH 2/2] x86/vmscape: Replace IBPB with branch history clear on exit to userspace

[Kaplan, David]

[AMD Official Use Only - AMD Internal Distribution Only]

> -----Original Message-----
> From: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
> Sent: Thursday, September 25, 2025 5:03 PM
> To: Kaplan, David <David.Kaplan@amd.com>
> Cc: x86@kernel.org; H. Peter Anvin <hpa@zytor.com>; Josh Poimboeuf
> <jpoimboe@kernel.org>; Sean Christopherson <seanjc@google.com>; Paolo
> Bonzini <pbonzini@redhat.com>; linux-kernel@vger.kernel.org;
> kvm@vger.kernel.org; Asit Mallick <asit.k.mallick@intel.com>; Tao Zhang
> <tao1.zhang@intel.com>
> Subject: Re: [PATCH 2/2] x86/vmscape: Replace IBPB with branch history clear on
> exit to userspace
>
> Caution: This message originated from an External Source. Use proper caution
> when opening attachments, clicking links, or responding.
>
>
> On Thu, Sep 25, 2025 at 06:14:54PM +0000, Kaplan, David wrote:
> > [AMD Official Use Only - AMD Internal Distribution Only]
> >
> > > -----Original Message-----
> > > From: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
> > > Sent: Wednesday, September 24, 2025 10:10 PM
> > > To: x86@kernel.org; H. Peter Anvin <hpa@zytor.com>; Josh Poimboeuf
> > > <jpoimboe@kernel.org>; Kaplan, David <David.Kaplan@amd.com>; Sean
> > > Christopherson <seanjc@google.com>; Paolo Bonzini <pbonzini@redhat.com>
> > > Cc: linux-kernel@vger.kernel.org; kvm@vger.kernel.org; Asit Mallick
> > > <asit.k.mallick@intel.com>; Tao Zhang <tao1.zhang@intel.com>
> > > Subject: [PATCH 2/2] x86/vmscape: Replace IBPB with branch history clear on
> exit
> > > to userspace
> > >
> > > Caution: This message originated from an External Source. Use proper caution
> > > when opening attachments, clicking links, or responding.
> > >
> > >
> > > IBPB mitigation for VMSCAPE is an overkill for CPUs that are only affected
> > > by the BHI variant of VMSCAPE. On such CPUs, eIBRS already provides
> > > indirect branch isolation between guest and host userspace. But, a guest
> > > could still poison the branch history.
> > >
> > > To mitigate that, use the recently added clear_bhb_long_loop() to isolate
> > > the branch history between guest and userspace. Add cmdline option
> > > 'vmscape=auto' that automatically selects the appropriate mitigation based
> > > on the CPU.
> > >
> > > Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
> > > ---
> > >  Documentation/admin-guide/hw-vuln/vmscape.rst   |  8 +++++
> > >  Documentation/admin-guide/kernel-parameters.txt |  4 ++-
> > >  arch/x86/include/asm/cpufeatures.h              |  1 +
> > >  arch/x86/include/asm/entry-common.h             | 12 ++++---
> > >  arch/x86/include/asm/nospec-branch.h            |  2 +-
> > >  arch/x86/kernel/cpu/bugs.c                      | 44 ++++++++++++++++++-------
> > >  arch/x86/kvm/x86.c                              |  5 +--
> > >  7 files changed, 55 insertions(+), 21 deletions(-)
> > >
> > > diff --git a/Documentation/admin-guide/hw-vuln/vmscape.rst
> > > b/Documentation/admin-guide/hw-vuln/vmscape.rst
> > > index
> > >
> d9b9a2b6c114c05a7325e5f3c9d42129339b870b..13ca98f952f97daeb28194c3873e
> > > 945b85eda6a1 100644
> > > --- a/Documentation/admin-guide/hw-vuln/vmscape.rst
> > > +++ b/Documentation/admin-guide/hw-vuln/vmscape.rst
> > > @@ -86,6 +86,10 @@ The possible values in this file are:
> > >     run a potentially malicious guest and issues an IBPB before the first
> > >     exit to userspace after VM-exit.
> > >
> > > + * 'Mitigation: Clear BHB before exit to userspace':
> > > +
> > > +   As above conditional BHB clearing mitigation is enabled.
> > > +
> > >   * 'Mitigation: IBPB on VMEXIT':
> > >
> > >     IBPB is issued on every VM-exit. This occurs when other mitigations like
> > > @@ -108,3 +112,7 @@ The mitigation can be controlled via the ``vmscape=``
> > > command line parameter:
> > >
> > >     Force vulnerability detection and mitigation even on processors that are
> > >     not known to be affected.
> > > +
> > > + * ``vmscape=auto``:
> > > +
> > > +   Choose the mitigation based on the VMSCAPE variant the CPU is affected
> by.
> > > diff --git a/Documentation/admin-guide/kernel-parameters.txt
> > > b/Documentation/admin-guide/kernel-parameters.txt
> > > index
> > >
> 5a7a83c411e9c526f8df6d28beb4c784aec3cac9..4596bfcb401f1a89d2dc5ed8c44c8
> > > 3628c9c5dfe 100644
> > > --- a/Documentation/admin-guide/kernel-parameters.txt
> > > +++ b/Documentation/admin-guide/kernel-parameters.txt
> > > @@ -8048,9 +8048,11 @@
> > >
> > >                         off             - disable the mitigation
> > >                         ibpb            - use Indirect Branch Prediction Barrier
> > > -                                         (IBPB) mitigation (default)
> > > +                                         (IBPB) mitigation
> > >                         force           - force vulnerability detection even on
> > >                                           unaffected processors
> > > +                       auto            - (default) automatically select IBPB
> > > +                                         or BHB clear mitigation based on CPU
> >
> > Many of the other bugs (like srso, l1tf, bhi, etc.) do not have explicit
> > 'auto' options as 'auto' is implied by the lack of an explicit option.
> > Is there really value in creating an explicit 'auto' option here?
>
> Hmm, so to get the BHB clear mitigation do we advise the users to remove
> the vmscape= parameter? That feels a bit weird to me. Also, with
> CONFIG_MITIGATION_VMSCAPE=n a user can get IBPB mitigation with
> vmscape=ibpb, but there is not way to get the BHB clear mitigation.
>

Maybe a better solution instead is to add a new option 'vmscape=on'.

If we look at the other most recently added bugs like TSA and ITS, neither have an explicit 'auto' cmdline option.  But they do have 'on' cmdline options.

The difference between 'auto' and 'on' is that 'auto' defers to the attack vector controls while 'on' means 'enable this mitigation if the CPU is vulnerable' (as opposed to 'force' which will enable it even if not vulnerable).

An explicit 'vmscape=on' could give users an option to ensure the mitigation is used (regardless of attack vectors) and could choose the best mitigation (BHB clear if available, otherwise IBPB).

I'd still advise users to not specify any option here unless they know what they're doing.  But an 'on' option would arguably be more consistent with the other recent bugs and maybe meets the needs you're after?

--David Kaplan

Re: [PATCH 2/2] x86/vmscape: Replace IBPB with branch history clear on exit to userspace

[Pawan Gupta]

On Fri, Sep 26, 2025 at 01:39:37PM +0000, Kaplan, David wrote:
> > > > --- a/Documentation/admin-guide/kernel-parameters.txt
> > > > +++ b/Documentation/admin-guide/kernel-parameters.txt
> > > > @@ -8048,9 +8048,11 @@
> > > >
> > > >                         off             - disable the mitigation
> > > >                         ibpb            - use Indirect Branch Prediction Barrier
> > > > -                                         (IBPB) mitigation (default)
> > > > +                                         (IBPB) mitigation
> > > >                         force           - force vulnerability detection even on
> > > >                                           unaffected processors
> > > > +                       auto            - (default) automatically select IBPB
> > > > +                                         or BHB clear mitigation based on CPU
> > >
> > > Many of the other bugs (like srso, l1tf, bhi, etc.) do not have explicit
> > > 'auto' options as 'auto' is implied by the lack of an explicit option.
> > > Is there really value in creating an explicit 'auto' option here?
> >
> > Hmm, so to get the BHB clear mitigation do we advise the users to remove
> > the vmscape= parameter? That feels a bit weird to me. Also, with
> > CONFIG_MITIGATION_VMSCAPE=n a user can get IBPB mitigation with
> > vmscape=ibpb, but there is no way to get the BHB clear mitigation.
> >
> 
> Maybe a better solution instead is to add a new option 'vmscape=on'.
> 
> If we look at the other most recently added bugs like TSA and ITS,
> neither have an explicit 'auto' cmdline option.  But they do have 'on'
> cmdline options.
> 
> The difference between 'auto' and 'on' is that 'auto' defers to the
> attack vector controls while 'on' means 'enable this mitigation if the
> CPU is vulnerable' (as opposed to 'force' which will enable it even if
> not vulnerable).
> 
> An explicit 'vmscape=on' could give users an option to ensure the
> mitigation is used (regardless of attack vectors) and could choose the
> best mitigation (BHB clear if available, otherwise IBPB).
> 
> I'd still advise users to not specify any option here unless they know
> what they're doing.  But an 'on' option would arguably be more consistent
> with the other recent bugs and maybe meets the needs you're after?

Sounds good to me. I'll update the patch.

[PATCH 0/2] VMSCAPE optimization for BHI variant

[Jack Wang]

From: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>

Hi Pawan,

Thx for the patches, I tested them on our Intel SierraForest machine with fio
4k randread/randwrite from guest, qemu virtio-blk, noticed nice performance
improvement comparing to the default IBPB before exit to userspace mitigation.
eg with default IBPB mitigation fio gets 204k IOPS, with this new Clear BHB before exit to userspace 
gets 323k IOPS.

Thx!

Re: [PATCH 0/2] VMSCAPE optimization for BHI variant

[Pawan Gupta]

On Mon, Sep 29, 2025 at 07:12:03AM +0200, Jack Wang wrote:
> From: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
> 
> Hi Pawan,
> 
> Thx for the patches, I tested them on our Intel SierraForest machine with
> fio 4k randread/randwrite from guest, qemu virtio-blk, noticed nice
> performance improvement comparing to the default IBPB before exit to
> userspace mitigation. eg with default IBPB mitigation fio gets 204k IOPS,
> with this new Clear BHB before exit to userspace gets 323k IOPS.

Thanks for sharing the results.

I realized the LFENCE in the clear_bhb_long_loop() is not required. The
ring3 transition after the loop should be serializing anyways. Below patch
gets rid of that LFENCE. It should give some performance boost as well.

--- 8< ---
From: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
Subject: [PATCH] x86/vmscape: Remove LFENCE from BHB clearing long loop

Long loop is used to clear the branch history when switching from a guest
to host userspace. The LFENCE barrier is not required as ring transition
itself acts as a barrier.

Move the prologue, LFENCE and epilogue out of __CLEAR_BHB_LOOP macro to
allow skipping the LFENCE in the long loop variant. Rename the long loop
function to clear_bhb_long_loop_no_barrier() to reflect the change.

Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
---
 arch/x86/entry/entry_64.S            | 32 +++++++++++++++++-----------
 arch/x86/include/asm/entry-common.h  |  2 +-
 arch/x86/include/asm/nospec-branch.h |  4 ++--
 3 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index f5f62af080d8..bb456a3c652e 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -1525,10 +1525,6 @@ SYM_CODE_END(rewind_stack_and_make_dead)
  * Target Selection, rather than taking the slowpath via its_return_thunk.
  */
 .macro	__CLEAR_BHB_LOOP outer_loop_count:req, inner_loop_count:req
-	ANNOTATE_NOENDBR
-	push	%rbp
-	mov	%rsp, %rbp
-
 	movl	$\outer_loop_count, %ecx
 	ANNOTATE_INTRA_FUNCTION_CALL
 	call	1f
@@ -1560,10 +1556,7 @@ SYM_CODE_END(rewind_stack_and_make_dead)
 	jnz	1b
 .Lret2_\@:
 	RET
-5:	lfence
-
-	pop	%rbp
-	RET
+5:
 .endm
 
 /*
@@ -1573,7 +1566,15 @@ SYM_CODE_END(rewind_stack_and_make_dead)
  * setting BHI_DIS_S for the guests.
  */
 SYM_FUNC_START(clear_bhb_loop)
+	ANNOTATE_NOENDBR
+	push	%rbp
+	mov	%rsp, %rbp
+
 	__CLEAR_BHB_LOOP 5, 5
+
+	lfence
+	pop	%rbp
+	RET
 SYM_FUNC_END(clear_bhb_loop)
 EXPORT_SYMBOL_GPL(clear_bhb_loop)
 STACK_FRAME_NON_STANDARD(clear_bhb_loop)
@@ -1584,8 +1585,15 @@ STACK_FRAME_NON_STANDARD(clear_bhb_loop)
  * protects the kernel, but to mitigate the guest influence on the host
  * userspace either IBPB or this sequence should be used. See VMSCAPE bug.
  */
-SYM_FUNC_START(clear_bhb_long_loop)
+SYM_FUNC_START(clear_bhb_long_loop_no_barrier)
+	ANNOTATE_NOENDBR
+	push	%rbp
+	mov	%rsp, %rbp
+
 	__CLEAR_BHB_LOOP 12, 7
-SYM_FUNC_END(clear_bhb_long_loop)
-EXPORT_SYMBOL_GPL(clear_bhb_long_loop)
-STACK_FRAME_NON_STANDARD(clear_bhb_long_loop)
+
+	pop	%rbp
+	RET
+SYM_FUNC_END(clear_bhb_long_loop_no_barrier)
+EXPORT_SYMBOL_GPL(clear_bhb_long_loop_no_barrier)
+STACK_FRAME_NON_STANDARD(clear_bhb_long_loop_no_barrier)
diff --git a/arch/x86/include/asm/entry-common.h b/arch/x86/include/asm/entry-common.h
index b7b9af1b6413..c70454bdd0e3 100644
--- a/arch/x86/include/asm/entry-common.h
+++ b/arch/x86/include/asm/entry-common.h
@@ -98,7 +98,7 @@ static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
 		if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER))
 			indirect_branch_prediction_barrier();
 		else if (cpu_feature_enabled(X86_FEATURE_CLEAR_BHB_EXIT_TO_USER))
-			clear_bhb_long_loop();
+			clear_bhb_long_loop_no_barrier();
 
 		this_cpu_write(x86_pred_flush_pending, false);
 	}
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 32d52f32a5e7..151f5de1a430 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -388,9 +388,9 @@ extern void write_ibpb(void);
 
 #ifdef CONFIG_X86_64
 extern void clear_bhb_loop(void);
-extern void clear_bhb_long_loop(void);
+extern void clear_bhb_long_loop_no_barrier(void);
 #else
-static inline void clear_bhb_long_loop(void) {}
+static inline void clear_bhb_long_loop_no_barrier(void) {}
 #endif
 
 extern void (*x86_return_thunk)(void);

Re: [PATCH 0/2] VMSCAPE optimization for BHI variant

[Jinpu Wang]

On Tue, Sep 30, 2025 at 3:22 AM Pawan Gupta
<pawan.kumar.gupta@linux.intel.com> wrote:
>
> On Mon, Sep 29, 2025 at 07:12:03AM +0200, Jack Wang wrote:
> > From: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
> >
> > Hi Pawan,
> >
> > Thx for the patches, I tested them on our Intel SierraForest machine with
> > fio 4k randread/randwrite from guest, qemu virtio-blk, noticed nice
> > performance improvement comparing to the default IBPB before exit to
> > userspace mitigation. eg with default IBPB mitigation fio gets 204k IOPS,
> > with this new Clear BHB before exit to userspace gets 323k IOPS.
>
> Thanks for sharing the results.
>
> I realized the LFENCE in the clear_bhb_long_loop() is not required. The
> ring3 transition after the loop should be serializing anyways. Below patch
> gets rid of that LFENCE. It should give some performance boost as well.
>
> --- 8< ---
> From: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
> Subject: [PATCH] x86/vmscape: Remove LFENCE from BHB clearing long loop
>
> Long loop is used to clear the branch history when switching from a guest
> to host userspace. The LFENCE barrier is not required as ring transition
> itself acts as a barrier.
>
> Move the prologue, LFENCE and epilogue out of __CLEAR_BHB_LOOP macro to
> allow skipping the LFENCE in the long loop variant. Rename the long loop
> function to clear_bhb_long_loop_no_barrier() to reflect the change.
>
> Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
Yes, I can confirm with this change on top, I get almost same
performance as vmscape=off for fio test.

Thx for pushing the performance further.
> ---
>  arch/x86/entry/entry_64.S            | 32 +++++++++++++++++-----------
>  arch/x86/include/asm/entry-common.h  |  2 +-
>  arch/x86/include/asm/nospec-branch.h |  4 ++--
>  3 files changed, 23 insertions(+), 15 deletions(-)
>
> diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
> index f5f62af080d8..bb456a3c652e 100644
> --- a/arch/x86/entry/entry_64.S
> +++ b/arch/x86/entry/entry_64.S
> @@ -1525,10 +1525,6 @@ SYM_CODE_END(rewind_stack_and_make_dead)
>   * Target Selection, rather than taking the slowpath via its_return_thunk.
>   */
>  .macro __CLEAR_BHB_LOOP outer_loop_count:req, inner_loop_count:req
> -       ANNOTATE_NOENDBR
> -       push    %rbp
> -       mov     %rsp, %rbp
> -
>         movl    $\outer_loop_count, %ecx
>         ANNOTATE_INTRA_FUNCTION_CALL
>         call    1f
> @@ -1560,10 +1556,7 @@ SYM_CODE_END(rewind_stack_and_make_dead)
>         jnz     1b
>  .Lret2_\@:
>         RET
> -5:     lfence
> -
> -       pop     %rbp
> -       RET
> +5:
>  .endm
>
>  /*
> @@ -1573,7 +1566,15 @@ SYM_CODE_END(rewind_stack_and_make_dead)
>   * setting BHI_DIS_S for the guests.
>   */
>  SYM_FUNC_START(clear_bhb_loop)
> +       ANNOTATE_NOENDBR
> +       push    %rbp
> +       mov     %rsp, %rbp
> +
>         __CLEAR_BHB_LOOP 5, 5
> +
> +       lfence
> +       pop     %rbp
> +       RET
>  SYM_FUNC_END(clear_bhb_loop)
>  EXPORT_SYMBOL_GPL(clear_bhb_loop)
>  STACK_FRAME_NON_STANDARD(clear_bhb_loop)
> @@ -1584,8 +1585,15 @@ STACK_FRAME_NON_STANDARD(clear_bhb_loop)
>   * protects the kernel, but to mitigate the guest influence on the host
>   * userspace either IBPB or this sequence should be used. See VMSCAPE bug.
>   */
> -SYM_FUNC_START(clear_bhb_long_loop)
> +SYM_FUNC_START(clear_bhb_long_loop_no_barrier)
> +       ANNOTATE_NOENDBR
> +       push    %rbp
> +       mov     %rsp, %rbp
> +
>         __CLEAR_BHB_LOOP 12, 7
> -SYM_FUNC_END(clear_bhb_long_loop)
> -EXPORT_SYMBOL_GPL(clear_bhb_long_loop)
> -STACK_FRAME_NON_STANDARD(clear_bhb_long_loop)
> +
> +       pop     %rbp
> +       RET
> +SYM_FUNC_END(clear_bhb_long_loop_no_barrier)
> +EXPORT_SYMBOL_GPL(clear_bhb_long_loop_no_barrier)
> +STACK_FRAME_NON_STANDARD(clear_bhb_long_loop_no_barrier)
> diff --git a/arch/x86/include/asm/entry-common.h b/arch/x86/include/asm/entry-common.h
> index b7b9af1b6413..c70454bdd0e3 100644
> --- a/arch/x86/include/asm/entry-common.h
> +++ b/arch/x86/include/asm/entry-common.h
> @@ -98,7 +98,7 @@ static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
>                 if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER))
>                         indirect_branch_prediction_barrier();
>                 else if (cpu_feature_enabled(X86_FEATURE_CLEAR_BHB_EXIT_TO_USER))
> -                       clear_bhb_long_loop();
> +                       clear_bhb_long_loop_no_barrier();
>
>                 this_cpu_write(x86_pred_flush_pending, false);
>         }
> diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
> index 32d52f32a5e7..151f5de1a430 100644
> --- a/arch/x86/include/asm/nospec-branch.h
> +++ b/arch/x86/include/asm/nospec-branch.h
> @@ -388,9 +388,9 @@ extern void write_ibpb(void);
>
>  #ifdef CONFIG_X86_64
>  extern void clear_bhb_loop(void);
> -extern void clear_bhb_long_loop(void);
> +extern void clear_bhb_long_loop_no_barrier(void);
>  #else
> -static inline void clear_bhb_long_loop(void) {}
> +static inline void clear_bhb_long_loop_no_barrier(void) {}
>  #endif
>
>  extern void (*x86_return_thunk)(void);