[PATCH] x86/jailhouse: Cache the Jailhouse CPUID base

[PATCH] x86/jailhouse: Cache the Jailhouse CPUID base

[Jonathan Conder]

This is an optimisation for booting nested VMs. With one level of
nesting, this patch cuts down a 2.5s boot by about 1s.

jailhouse_paravirt() looks for the Jailhouse signature with
cpuid_base_hypervisor(), which tries every possible hypervisor CPUID
base from 0x40000000 to 0x4000ff00, i.e. 256 CPUID instructions in the
worst case. PCI enumeration calls it repeatedly.

CPUID always exits to the hypervisor, so this is slow in virtual
machines. A KVM q35 guest with 16 PCIe root ports performs about 78,000
CPUID exits while booting. Caching the base removes the need for most of
these, assuming the result doesn't change and is the same on every CPU.

Disclaimer: the issue was discovered, and patch authored, with LLM
assistance. I built and tested the patch to confirm that it does speed
things up significantly.

Signed-off-by: Jonathan Conder <jonathan.conder@canonical.com>
---
Tested using the script below, 3 runs per kernel. Built using
x86_64_defconfig, kvm_guest.config and CONFIG_JAILHOUSE_GUEST=y.
I ran it inside a LXD VM on my AMD Ryzen AI 9 HX PRO 370 host.

Before:
uptime           2594ms  2452ms  2771ms
acpi_init         347ms   442ms   486ms
pci_subsys_init  1497ms  1269ms  1612ms

After:
uptime           1446ms  1559ms  1498ms
acpi_init         287ms   357ms   259ms
pci_subsys_init   419ms   505ms   515ms

Reproducer:
#!/bin/bash
# Boot a kernel under QEMU/KVM with 16 PCIe root ports and print how
# long ACPI and PCI init took. Intended to benchmark nested VM boot, so
# run it inside another VM.

set -e

kernel=${1:-arch/x86/boot/bzImage}

tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT

cat > "$tmp/init.c" <<'EOF'
/* Print boot timings, then power off. */
#include <stdio.h>
#include <string.h>
#include <sys/klog.h>
#include <sys/reboot.h>
#include <time.h>

static char log[8 << 20];

int main(void)
{
	struct timespec now;
	char name[64];
	long usecs;
	int len;

	clock_gettime(CLOCK_BOOTTIME, &now);
	printf("REPRO uptime          %ld ms\n", now.tv_sec * 1000 + now.tv_nsec / 1000000);

	len = klogctl(3 /* SYSLOG_ACTION_READ_ALL */, log, sizeof(log) - 1);
	log[len > 0 ? len : 0] = '\0';
	for (char *line = strtok(log, "\n"); line; line = strtok(NULL, "\n")) {
		char *p = strstr(line, "initcall ");

		if (p && sscanf(p, "initcall %63[^+]+%*s returned %*d after %ld usecs", name, &usecs) == 2 &&
		    (!strcmp(name, "pci_subsys_init") || !strcmp(name, "acpi_init")))
			printf("REPRO %-15s %ld ms\n", name, usecs / 1000);
	}

	fflush(stdout);
	reboot(RB_POWER_OFF);
	return 0;
}
EOF

gcc -O2 -static -o "$tmp/init" "$tmp/init.c"
(cd "$tmp" && echo init | cpio --quiet -o -H newc -R 0:0 > initramfs.cpio)

ports=""
for i in {1..16}; do
	ports+=" -device pcie-root-port,id=rp$i,chassis=$i,addr=$((1 + (i - 1) / 8)).$(((i - 1) % 8)),multifunction=on"
done

timeout 300 qemu-system-x86_64 -accel kvm -machine q35 -cpu host -smp 2 -m 1G \
	-nodefaults -display none -serial stdio -no-reboot $ports \
	-kernel "$kernel" -initrd "$tmp/initramfs.cpio" \
	-append "console=ttyS0 quiet initcall_debug log_buf_len=4M panic=-1" \
	< /dev/null > "$tmp/console.log" 2>&1

sed -n 's/^REPRO //p' "$tmp/console.log"

 arch/x86/kernel/jailhouse.c | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/arch/x86/kernel/jailhouse.c b/arch/x86/kernel/jailhouse.c
index f58ce9220e0f..7ace1b331e97 100644
--- a/arch/x86/kernel/jailhouse.c
+++ b/arch/x86/kernel/jailhouse.c
@@ -44,7 +44,7 @@ static void jailhouse_setup_irq(unsigned int irq)
 	mp_save_irq(&mp_irq);
 }
 
-static uint32_t jailhouse_cpuid_base(void)
+static uint32_t __jailhouse_cpuid_base(void)
 {
 	if (boot_cpu_data.cpuid_level < 0 ||
 	    !boot_cpu_has(X86_FEATURE_HYPERVISOR))
@@ -53,6 +53,19 @@ static uint32_t jailhouse_cpuid_base(void)
 	return cpuid_base_hypervisor("Jailhouse\0\0\0", 0);
 }
 
+static uint32_t jailhouse_cpuid_base(void)
+{
+	static u32 base = U32_MAX;
+	u32 val = READ_ONCE(base);
+
+	if (val == U32_MAX) {
+		val = __jailhouse_cpuid_base();
+		WRITE_ONCE(base, val);
+	}
+
+	return val;
+}
+
 static uint32_t __init jailhouse_detect(void)
 {
 	return jailhouse_cpuid_base();
-- 
2.43.0