[RFC PATCH 2/3] x86/boot: Set __pgtable_l5_enabled correctly before calling into C code

[Ard Biesheuvel <ardb+git@google.com>]

From: Ard Biesheuvel <ardb@kernel.org>

Ensure that __pgtable_l5_enabled() is set to its permanent value before
calling into any C code that may manipulate page tables or reference any
global variable or object that may be dimensioned differently based on
whether 5-level paging is in use.

This avoids inconsistencies that are difficult to detect, and allows
pgtable_l5_enabled() to be emitted with the 'const' function attribute.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/compressed/head_64.S      | 6 ++++++
 arch/x86/boot/compressed/pgtable_64.c   | 6 +++---
 arch/x86/boot/startup/map_kernel.c      | 1 -
 arch/x86/include/asm/pgtable_64_types.h | 2 +-
 arch/x86/kernel/head64.c                | 2 +-
 arch/x86/kernel/head_64.S               | 7 +++++++
 6 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S
index d9dab940ff62..e6b254a12ca9 100644
--- a/arch/x86/boot/compressed/head_64.S
+++ b/arch/x86/boot/compressed/head_64.S
@@ -454,6 +454,12 @@ SYM_FUNC_START_LOCAL_NOALIGN(.Lrelocated)
 	shrq	$3, %rcx
 	rep	stosq
 
+#ifdef CONFIG_X86_5LEVEL
+	movq	%cr4, %rax
+	shrl	$X86_CR4_LA57_BIT, %eax
+	andl	%eax, __pgtable_l5_enabled(%rip)
+#endif
+
 	call	load_stage2_idt
 
 	/* Pass boot_params to initialize_identity_maps() */
diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c
index 5a6c7a190e5b..0aff7a637f54 100644
--- a/arch/x86/boot/compressed/pgtable_64.c
+++ b/arch/x86/boot/compressed/pgtable_64.c
@@ -11,8 +11,9 @@
 #define BIOS_START_MAX		0x9f000U	/* 640K, absolute maximum */
 
 #ifdef CONFIG_X86_5LEVEL
-/* __pgtable_l5_enabled needs to be in .data to avoid being cleared along with .bss */
-unsigned int __section(".data") __pgtable_l5_enabled;
+unsigned int __pgtable_l5_enabled = 1;
+
+/* These need to be in .data to avoid being cleared along with .bss */
 unsigned int __section(".data") pgdir_shift = 39;
 unsigned int __section(".data") ptrs_per_p4d = 1;
 #endif
@@ -129,7 +130,6 @@ asmlinkage void configure_5level_paging(struct boot_params *bp, void *pgtable)
 		l5_required = true;
 
 		/* Initialize variables for 5-level paging */
-		__pgtable_l5_enabled = 1;
 		pgdir_shift = 48;
 		ptrs_per_p4d = 512;
 	}
diff --git a/arch/x86/boot/startup/map_kernel.c b/arch/x86/boot/startup/map_kernel.c
index 099ae2559336..f3d09e61575b 100644
--- a/arch/x86/boot/startup/map_kernel.c
+++ b/arch/x86/boot/startup/map_kernel.c
@@ -26,7 +26,6 @@ static inline bool check_la57_support(void)
 	if (!(native_read_cr4() & X86_CR4_LA57))
 		return false;
 
-	__pgtable_l5_enabled	= 1;
 	pgdir_shift		= 48;
 	ptrs_per_p4d		= 512;
 	page_offset_base	= __PAGE_OFFSET_BASE_L5;
diff --git a/arch/x86/include/asm/pgtable_64_types.h b/arch/x86/include/asm/pgtable_64_types.h
index 2ca568f56660..2c498d16609c 100644
--- a/arch/x86/include/asm/pgtable_64_types.h
+++ b/arch/x86/include/asm/pgtable_64_types.h
@@ -27,7 +27,7 @@ extern unsigned int __pgtable_l5_enabled;
 #include <asm/alternative.h>
 #include <asm/cpufeatures.h>
 
-static inline bool pgtable_l5_enabled(void)
+static inline bool __attribute_const__ pgtable_l5_enabled(void)
 {
 	asm goto(ALTERNATIVE_TERNARY("jmp 6f", %c[feat], "", "jmp %l[t_no]")
 		"	.pushsection .altinstr_aux,\"ax\"	\n"
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 498b9d6bdf2f..d3d1136ad802 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -49,7 +49,7 @@ SYM_PIC_ALIAS(next_early_pgt);
 pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);
 
 #ifdef CONFIG_X86_5LEVEL
-unsigned int __pgtable_l5_enabled __initdata;
+unsigned int __pgtable_l5_enabled __initdata = 1;
 unsigned int pgdir_shift __ro_after_init = 39;
 EXPORT_SYMBOL(pgdir_shift);
 unsigned int ptrs_per_p4d __ro_after_init = 1;
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 069420853304..1fe74bf828da 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -61,6 +61,13 @@ SYM_CODE_START_NOALIGN(startup_64)
 	/* Set up the stack for verify_cpu() */
 	leaq	__top_init_kernel_stack(%rip), %rsp
 
+#ifdef CONFIG_X86_5LEVEL
+	/* __pgtable_l5_enabled needs to be correct before calling C code */
+	movq	%cr4, %rax
+	shrl	$X86_CR4_LA57_BIT, %eax
+	andl	%eax, __pgtable_l5_enabled(%rip)
+#endif
+
 	/*
 	 * Set up GSBASE.
 	 * Note that on SMP the boot CPU uses the init data section until
-- 
2.49.0.987.g0cc8ee98dc-goog