[PATCH] x86/fpu/xstate: Fix XSTATE_WARN_ON() to emit relevant diagnostics

[PATCH] x86/fpu/xstate: Fix XSTATE_WARN_ON() to emit relevant diagnostics

[Andrew Cooper]

"XSAVE consistency problem" has been reported under Xen, but that's the extent
of my divination skills.

Modify XSTATE_WARN_ON() to force the caller to provide relevant diagnostic
information, and modify each caller suitably.

For check_xstate_against_struct(), this removes a double WARN() where one will
do perfectly fine.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
---
CC: Thomas Gleixner <tglx@linutronix.de>
CC: Ingo Molnar <mingo@redhat.com>
CC: Borislav Petkov <bp@alien8.de>
CC: Dave Hansen <dave.hansen@linux.intel.com>
CC: x86@kernel.org
CC: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>

RFC: CC stable?  This has been wonky debugging for 7 years.

Apparently "size 832 != kernel_size 0" so let the debugging continue...
---
 arch/x86/kernel/fpu/xstate.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c
index c8340156bfd2..28b6478ea531 100644
--- a/arch/x86/kernel/fpu/xstate.c
+++ b/arch/x86/kernel/fpu/xstate.c
@@ -440,8 +440,8 @@ static void __init __xstate_dump_leaves(void)
 	}
 }
 
-#define XSTATE_WARN_ON(x) do {							\
-	if (WARN_ONCE(x, "XSAVE consistency problem, dumping leaves")) {	\
+#define XSTATE_WARN_ON(x, fmt, ...) do {					\
+	if (WARN_ONCE(x, "XSAVE consistency problem: " fmt, ##__VA_ARGS__)) {	\
 		__xstate_dump_leaves();						\
 	}									\
 } while (0)
@@ -554,8 +554,7 @@ static bool __init check_xstate_against_struct(int nr)
 	    (nr >= XFEATURE_MAX) ||
 	    (nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR) ||
 	    ((nr >= XFEATURE_RSRVD_COMP_11) && (nr <= XFEATURE_RSRVD_COMP_16))) {
-		WARN_ONCE(1, "no structure for xstate: %d\n", nr);
-		XSTATE_WARN_ON(1);
+		XSTATE_WARN_ON(1, "No structure for xstate: %d\n", nr);
 		return false;
 	}
 	return true;
@@ -598,12 +597,13 @@ static bool __init paranoid_xstate_size_valid(unsigned int kernel_size)
 		 * XSAVES.
 		 */
 		if (!xsaves && xfeature_is_supervisor(i)) {
-			XSTATE_WARN_ON(1);
+			XSTATE_WARN_ON(1, "Got supervisor feature %d, but XSAVES not advertised\n", i);
 			return false;
 		}
 	}
 	size = xstate_calculate_size(fpu_kernel_cfg.max_features, compacted);
-	XSTATE_WARN_ON(size != kernel_size);
+	XSTATE_WARN_ON(size != kernel_size,
+		       "size %u != kernel_size %u\n", size, kernel_size);
 	return size == kernel_size;
 }
 
-- 
2.11.0

Ping: [PATCH] x86/fpu/xstate: Fix XSTATE_WARN_ON() to emit relevant diagnostics

[Andrew Cooper]

On 10/08/2022 23:19, Andrew Cooper wrote:
> "XSAVE consistency problem" has been reported under Xen, but that's the extent
> of my divination skills.
>
> Modify XSTATE_WARN_ON() to force the caller to provide relevant diagnostic
> information, and modify each caller suitably.
>
> For check_xstate_against_struct(), this removes a double WARN() where one will
> do perfectly fine.
>
> Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
> ---
> CC: Thomas Gleixner <tglx@linutronix.de>
> CC: Ingo Molnar <mingo@redhat.com>
> CC: Borislav Petkov <bp@alien8.de>
> CC: Dave Hansen <dave.hansen@linux.intel.com>
> CC: x86@kernel.org
> CC: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
>
> RFC: CC stable?  This has been wonky debugging for 7 years.
>
> Apparently "size 832 != kernel_size 0" so let the debugging continue...
> ---
>  arch/x86/kernel/fpu/xstate.c | 12 ++++++------
>  1 file changed, 6 insertions(+), 6 deletions(-)
>
> diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c
> index c8340156bfd2..28b6478ea531 100644
> --- a/arch/x86/kernel/fpu/xstate.c
> +++ b/arch/x86/kernel/fpu/xstate.c
> @@ -440,8 +440,8 @@ static void __init __xstate_dump_leaves(void)
>  	}
>  }
>  
> -#define XSTATE_WARN_ON(x) do {							\
> -	if (WARN_ONCE(x, "XSAVE consistency problem, dumping leaves")) {	\
> +#define XSTATE_WARN_ON(x, fmt, ...) do {					\
> +	if (WARN_ONCE(x, "XSAVE consistency problem: " fmt, ##__VA_ARGS__)) {	\
>  		__xstate_dump_leaves();						\
>  	}									\
>  } while (0)
> @@ -554,8 +554,7 @@ static bool __init check_xstate_against_struct(int nr)
>  	    (nr >= XFEATURE_MAX) ||
>  	    (nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR) ||
>  	    ((nr >= XFEATURE_RSRVD_COMP_11) && (nr <= XFEATURE_RSRVD_COMP_16))) {
> -		WARN_ONCE(1, "no structure for xstate: %d\n", nr);
> -		XSTATE_WARN_ON(1);
> +		XSTATE_WARN_ON(1, "No structure for xstate: %d\n", nr);
>  		return false;
>  	}
>  	return true;
> @@ -598,12 +597,13 @@ static bool __init paranoid_xstate_size_valid(unsigned int kernel_size)
>  		 * XSAVES.
>  		 */
>  		if (!xsaves && xfeature_is_supervisor(i)) {
> -			XSTATE_WARN_ON(1);
> +			XSTATE_WARN_ON(1, "Got supervisor feature %d, but XSAVES not advertised\n", i);
>  			return false;
>  		}
>  	}
>  	size = xstate_calculate_size(fpu_kernel_cfg.max_features, compacted);
> -	XSTATE_WARN_ON(size != kernel_size);
> +	XSTATE_WARN_ON(size != kernel_size,
> +		       "size %u != kernel_size %u\n", size, kernel_size);
>  	return size == kernel_size;
>  }
>  

Re: [PATCH] x86/fpu/xstate: Fix XSTATE_WARN_ON() to emit relevant diagnostics

[Borislav Petkov]

On Wed, Aug 10, 2022 at 11:19:09PM +0100, Andrew Cooper wrote:
> "XSAVE consistency problem" has been reported under Xen, but that's the extent
> of my divination skills.
> 
> Modify XSTATE_WARN_ON() to force the caller to provide relevant diagnostic
> information, and modify each caller suitably.
> 
> For check_xstate_against_struct(), this removes a double WARN() where one will
> do perfectly fine.
> 
> Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
> ---
> CC: Thomas Gleixner <tglx@linutronix.de>
> CC: Ingo Molnar <mingo@redhat.com>
> CC: Borislav Petkov <bp@alien8.de>
> CC: Dave Hansen <dave.hansen@linux.intel.com>
> CC: x86@kernel.org
> CC: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
> 
> RFC: CC stable?  This has been wonky debugging for 7 years.
> 
> Apparently "size 832 != kernel_size 0" so let the debugging continue...

I've got a similar bug report from people running Linux guest on some
other prop. HV. And I wanted to give them a debugging patch which dumps
*all* the relevant data along the path of paranoid_xstate_size_valid(),
the loop in there and xstate_calculate_size().

Looking how you might need something like that too, how about you extend
your patch to do that and have it being toggled on by a xstate=debug
cmdline?

It feels like this would be a useful thing to have with the gazillion of
XSTATE features and dynamic buffer allocation...

Thx.

-- 
Regards/Gruss,
    Boris.

https://people.kernel.org/tglx/notes-about-netiquette

Re: [PATCH] x86/fpu/xstate: Fix XSTATE_WARN_ON() to emit relevant diagnostics

[Andrew Cooper]

On 28/09/2022 17:27, Borislav Petkov wrote:
> On Wed, Aug 10, 2022 at 11:19:09PM +0100, Andrew Cooper wrote:
>> "XSAVE consistency problem" has been reported under Xen, but that's the extent
>> of my divination skills.
>>
>> Modify XSTATE_WARN_ON() to force the caller to provide relevant diagnostic
>> information, and modify each caller suitably.
>>
>> For check_xstate_against_struct(), this removes a double WARN() where one will
>> do perfectly fine.
>>
>> Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
>> ---
>> CC: Thomas Gleixner <tglx@linutronix.de>
>> CC: Ingo Molnar <mingo@redhat.com>
>> CC: Borislav Petkov <bp@alien8.de>
>> CC: Dave Hansen <dave.hansen@linux.intel.com>
>> CC: x86@kernel.org
>> CC: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
>>
>> RFC: CC stable?  This has been wonky debugging for 7 years.
>>
>> Apparently "size 832 != kernel_size 0" so let the debugging continue...
> I've got a similar bug report from people running Linux guest on some
> other prop. HV. And I wanted to give them a debugging patch which dumps
> *all* the relevant data along the path of paranoid_xstate_size_valid(),
> the loop in there and xstate_calculate_size().
>
> Looking how you might need something like that too, how about you extend
> your patch to do that and have it being toggled on by a xstate=debug
> cmdline?
>
> It feels like this would be a useful thing to have with the gazillion of
> XSTATE features and dynamic buffer allocation...

So we've actually found and fixed the issue, but XSAVE and therefore
automatically gnarly.

https://xenbits.xen.org/gitweb/?p=xen.git;a=commitdiff;h=c3bd0b83ea5b7c0da6542687436042eeea1e7909

There is no real hardware with XSAVEC but not XSAVES; the spec does try
to distinguish the two, and it's useful for virt to offer XSAVEC without
XSAVES.

CPUID.0xd[1].ebx is spec'd as the total size for XSAVES of all current
XCR0|XSS states.  This is known dodgy already for native, as it leaks
the current MSR_XSS setting into userspace.

I had written the logic originally to hide this dynamic field if XSAVES
wasn't enumerated, but Linux now uses it if XSAVEC is enumerated, to
cross-check what it can see elsewhere in the CPUID state.

I'm pretty sure things will break again when the host MSR_XSS becomes
non-zero, but I have no free time to spend on any of this in the first
place.

Ultimately, the issue here is that there is a privileged state leak, and
Linux is now relying on it for a sanity check.

~Andrew

[tip: x86/fpu] x86/fpu/xstate: Fix XSTATE_WARN_ON() to emit relevant diagnostics

[tip-bot2 for Andrew Cooper]

The following commit has been merged into the x86/fpu branch of tip:

Commit-ID:     48280042f2c6e3ac2cfb1d8b752ab4a7e0baea24
Gitweb:        https://git.kernel.org/tip/48280042f2c6e3ac2cfb1d8b752ab4a7e0baea24
Author:        Andrew Cooper <andrew.cooper3@citrix.com>
AuthorDate:    Wed, 10 Aug 2022 23:19:09 +01:00
Committer:     Borislav Petkov <bp@suse.de>
CommitterDate: Wed, 09 Nov 2022 13:28:31 +01:00

x86/fpu/xstate: Fix XSTATE_WARN_ON() to emit relevant diagnostics

"XSAVE consistency problem" has been reported under Xen, but that's the extent
of my divination skills.

Modify XSTATE_WARN_ON() to force the caller to provide relevant diagnostic
information, and modify each caller suitably.

For check_xstate_against_struct(), this removes a double WARN() where one will
do perfectly fine.

CC stable as this has been wonky debugging for 7 years and it is good to
have there too.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
Signed-off-by: Borislav Petkov <bp@suse.de>
Cc: <stable@vger.kernel.org>
Link: https://lore.kernel.org/r/20220810221909.12768-1-andrew.cooper3@citrix.com
---
 arch/x86/kernel/fpu/xstate.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c
index 59e543b..c2dde46 100644
--- a/arch/x86/kernel/fpu/xstate.c
+++ b/arch/x86/kernel/fpu/xstate.c
@@ -440,8 +440,8 @@ static void __init __xstate_dump_leaves(void)
 	}
 }
 
-#define XSTATE_WARN_ON(x) do {							\
-	if (WARN_ONCE(x, "XSAVE consistency problem, dumping leaves")) {	\
+#define XSTATE_WARN_ON(x, fmt, ...) do {					\
+	if (WARN_ONCE(x, "XSAVE consistency problem: " fmt, ##__VA_ARGS__)) {	\
 		__xstate_dump_leaves();						\
 	}									\
 } while (0)
@@ -554,8 +554,7 @@ static bool __init check_xstate_against_struct(int nr)
 	    (nr >= XFEATURE_MAX) ||
 	    (nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR) ||
 	    ((nr >= XFEATURE_RSRVD_COMP_11) && (nr <= XFEATURE_RSRVD_COMP_16))) {
-		WARN_ONCE(1, "no structure for xstate: %d\n", nr);
-		XSTATE_WARN_ON(1);
+		XSTATE_WARN_ON(1, "No structure for xstate: %d\n", nr);
 		return false;
 	}
 	return true;
@@ -598,12 +597,13 @@ static bool __init paranoid_xstate_size_valid(unsigned int kernel_size)
 		 * XSAVES.
 		 */
 		if (!xsaves && xfeature_is_supervisor(i)) {
-			XSTATE_WARN_ON(1);
+			XSTATE_WARN_ON(1, "Got supervisor feature %d, but XSAVES not advertised\n", i);
 			return false;
 		}
 	}
 	size = xstate_calculate_size(fpu_kernel_cfg.max_features, compacted);
-	XSTATE_WARN_ON(size != kernel_size);
+	XSTATE_WARN_ON(size != kernel_size,
+		       "size %u != kernel_size %u\n", size, kernel_size);
 	return size == kernel_size;
 }