[PATCH v3 0/1] x86/vmware: Share steal-time counters in encrypted guests

[PATCH v3 0/1] x86/vmware: Share steal-time counters in encrypted guests

[Zack Rusin]

Steal time is the time a virtual CPU was ready to run but waited for the
host to schedule it. For example, in a 100 ms interval a vCPU might run
for 70 ms and wait 30 ms for a physical CPU. Reporting those 30 ms lets
Linux:
- account CPU usage accurately, without charging applications for time
  their vCPU wasn't running;
- base scheduling decisions on the CPU time tasks actually received;
- show host contention in the "st" column of top and vmstat and in
  /proc/stat.

VMware writes a running total into a small per-CPU buffer in guest
memory, and Linux reads it. In a confidential guest (AMD SEV-SNP, Intel
TDX) memory is private by default and the host cannot write it, so the
guest has to share the buffer first. Linux currently registers the
buffer without sharing it, and ESXi then powers off the VM. This only
affects VMs with the steal clock enabled, which ESXi enables by default
only for Photon guests.

Following the v2 discussion, this version uses one packed allocation for
all possible CPUs' counters and shares it before registering any address.
Setup moves to an early initcall so it can use the normal memory-conversion
API. This avoids converting a page in every CPU's per-CPU area and
concentrates sharing in one allocation.

Changes since v2:
- Replace the six infrastructure patches with one VMware-only fix.
- Leave the per-CPU allocator, linker, KVM and early conversion code alone.
- Move ordinary-guest setup to the same initcall.
- Rebase onto v7.3.

Tested on ordinary VMware x86-64 SMP/UP guests, ESXi SEV-SNP and TDX
guests (SMP, UP, 66 vCPUs and percpu_alloc=page), and QEMU/KVM x86-64
guests.

v2: https://lore.kernel.org/r/20260929040256.543767-1-zack.rusin@broadcom.com

Zack Rusin (1):
  x86/vmware: Allocate shared storage for steal-time counters

 arch/x86/kernel/cpu/vmware.c | 130 +++++++++++++++++++++++------------
 1 file changed, 86 insertions(+), 44 deletions(-)


base-commit: c03c1a14507dfaf8b856c2608f9ac216021fc222

[PATCH v3 1/1] x86/vmware: Allocate shared storage for steal-time counters

[Zack Rusin]

Steal time accounts for time a virtual CPU waits to run on the host.
VMware updates a counter in guest memory, but encrypted guests currently
register private memory which the host cannot write.

Allocate the counters together instead of embedding them in every CPU's
per-CPU area, and share them before registering any address. Move setup
for all guests to an early initcall, after the page allocator is available
and before secondary CPUs start, so it can use the normal conversion API.

Keep the allocation for CPU hotplug. Enable accounting only after boot-CPU
registration succeeds, and retain pages whose encryption state is unknown
on failure. A secondary CPU's registration failure must not prevent the
other CPUs from unregistering their counters.

Suggested-by: Borislav Petkov <bp@alien8.de>
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Link: https://lore.kernel.org/r/20261001235950.GHar7z9v3zfk8odoTh@fat_crate.local
Link: https://lore.kernel.org/r/20260309235250.2611115-5-alexey.makhalov@broadcom.com
Co-developed-by: Bo Gan <bo.gan@broadcom.com>
Signed-off-by: Bo Gan <bo.gan@broadcom.com>
Co-developed-by: Alexey Makhalov <alexey.makhalov@broadcom.com>
Signed-off-by: Alexey Makhalov <alexey.makhalov@broadcom.com>
Signed-off-by: Zack Rusin <zack.rusin@broadcom.com>
---
v3: Replace the per-CPU infrastructure series with a packed allocation
and ordinary post-allocator conversion in the VMware driver.

 arch/x86/kernel/cpu/vmware.c | 130 +++++++++++++++++++++++------------
 1 file changed, 86 insertions(+), 44 deletions(-)

diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c
index 34b73573b108..7b7469f2e7d6 100644
--- a/arch/x86/kernel/cpu/vmware.c
+++ b/arch/x86/kernel/cpu/vmware.c
@@ -25,9 +25,12 @@
 #include <linux/init.h>
 #include <linux/export.h>
 #include <linux/clocksource.h>
+#include <linux/cc_platform.h>
 #include <linux/cpu.h>
 #include <linux/efi.h>
+#include <linux/mm.h>
 #include <linux/reboot.h>
+#include <linux/set_memory.h>
 #include <linux/static_call.h>
 #include <linux/sched/cputime.h>
 #include <asm/div64.h>
@@ -145,8 +148,8 @@ static unsigned long vmware_get_tsc_khz(void)
 #ifdef CONFIG_PARAVIRT
 static struct cyc2ns_data vmware_cyc2ns __ro_after_init;
 static bool vmw_sched_clock __initdata = true;
-static DEFINE_PER_CPU_DECRYPTED(struct vmware_steal_time, vmw_steal_time) __aligned(64);
-static bool has_steal_clock;
+static struct vmware_steal_time *vmw_steal_time __ro_after_init;
+static bool has_steal_clock __ro_after_init;
 static bool steal_acc __initdata = true; /* steal time accounting */
 
 static __init int setup_vmw_sched_clock(char *s)
@@ -228,7 +231,7 @@ static bool vmware_is_stealclock_available(void)
  */
 static u64 vmware_steal_clock(int cpu)
 {
-	struct vmware_steal_time *steal = &per_cpu(vmw_steal_time, cpu);
+	struct vmware_steal_time *steal = &vmw_steal_time[cpu];
 	u64 clock;
 
 	if (IS_ENABLED(CONFIG_64BIT))
@@ -253,21 +256,19 @@ static u64 vmware_steal_clock(int cpu)
 			     vmware_cyc2ns.cyc2ns_shift);
 }
 
-static void vmware_register_steal_time(void)
+static bool vmware_register_steal_time(void)
 {
 	int cpu = smp_processor_id();
-	struct vmware_steal_time *st = &per_cpu(vmw_steal_time, cpu);
+	struct vmware_steal_time *st = &vmw_steal_time[cpu];
 
-	if (!has_steal_clock)
-		return;
-
-	if (!stealclock_enable(slow_virt_to_phys(st))) {
-		has_steal_clock = false;
-		return;
+	if (!stealclock_enable(virt_to_phys(st))) {
+		pr_warn("failed to register steal time for CPU %d\n", cpu);
+		return false;
 	}
 
 	pr_info("vmware-stealtime: cpu %d, pa %llx\n",
-		cpu, (unsigned long long) slow_virt_to_phys(st));
+		cpu, (unsigned long long)virt_to_phys(st));
+	return true;
 }
 
 static void vmware_disable_steal_time(void)
@@ -278,12 +279,6 @@ static void vmware_disable_steal_time(void)
 	stealclock_disable();
 }
 
-static void vmware_guest_cpu_init(void)
-{
-	if (has_steal_clock)
-		vmware_register_steal_time();
-}
-
 static void vmware_pv_guest_cpu_reboot(void *unused)
 {
 	vmware_disable_steal_time();
@@ -302,16 +297,11 @@ static struct notifier_block vmware_pv_reboot_nb = {
 };
 
 #ifdef CONFIG_SMP
-static void __init vmware_smp_prepare_boot_cpu(void)
-{
-	vmware_guest_cpu_init();
-	native_smp_prepare_boot_cpu();
-}
-
 static int vmware_cpu_online(unsigned int cpu)
 {
 	local_irq_disable();
-	vmware_guest_cpu_init();
+	if (has_steal_clock)
+		vmware_register_steal_time();
 	local_irq_enable();
 	return 0;
 }
@@ -325,6 +315,76 @@ static int vmware_cpu_down_prepare(unsigned int cpu)
 }
 #endif
 
+static int __init vmware_steal_time_init(void)
+{
+	size_t size = PAGE_ALIGN(nr_cpu_ids * sizeof(*vmw_steal_time));
+	bool encrypted = cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT);
+	unsigned long addr, flags;
+	int ret;
+#ifdef CONFIG_SMP
+	int hp_state;
+#endif
+
+	if (!has_steal_clock)
+		return 0;
+
+	vmw_steal_time = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO);
+	if (!vmw_steal_time) {
+		ret = -ENOMEM;
+		goto disable;
+	}
+	addr = (unsigned long)vmw_steal_time;
+
+	ret = register_reboot_notifier(&vmware_pv_reboot_nb);
+	if (ret)
+		goto free;
+#ifdef CONFIG_SMP
+	hp_state = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
+					     "x86/vmware:online",
+					     vmware_cpu_online,
+					     vmware_cpu_down_prepare);
+	if (hp_state < 0) {
+		ret = hp_state;
+		unregister_reboot_notifier(&vmware_pv_reboot_nb);
+		goto free;
+	}
+#endif
+
+	if (encrypted) {
+		ret = set_memory_decrypted(addr, size >> PAGE_SHIFT);
+		if (ret)
+			goto remove;
+		/* Conversion need not preserve the zeroes. */
+		memset(vmw_steal_time, 0, size);
+	}
+
+	local_irq_save(flags);
+	ret = vmware_register_steal_time() ? 0 : -EIO;
+	local_irq_restore(flags);
+	if (!ret) {
+		static_call_update(pv_steal_clock, vmware_steal_clock);
+		return 0;
+	}
+
+	if (!encrypted || !set_memory_encrypted(addr, size >> PAGE_SHIFT))
+		free_pages_exact(vmw_steal_time, size);
+remove:
+	/* Keep pages with an unknown encryption state out of the allocator. */
+#ifdef CONFIG_SMP
+	cpuhp_remove_state_nocalls(hp_state);
+#endif
+	unregister_reboot_notifier(&vmware_pv_reboot_nb);
+	goto disable;
+free:
+	free_pages_exact(vmw_steal_time, size);
+disable:
+	vmw_steal_time = NULL;
+	has_steal_clock = false;
+	pr_warn("steal time disabled: %d\n", ret);
+	return 0;
+}
+early_initcall(vmware_steal_time_init);
+
 static __init int activate_jump_labels(void)
 {
 	if (has_steal_clock) {
@@ -350,25 +410,7 @@ static void __init vmware_paravirt_ops_setup(void)
 	if (vmw_sched_clock)
 		paravirt_set_sched_clock(vmware_sched_clock);
 
-	if (vmware_is_stealclock_available()) {
-		has_steal_clock = true;
-		static_call_update(pv_steal_clock, vmware_steal_clock);
-
-		/* We use reboot notifier only to disable steal clock */
-		register_reboot_notifier(&vmware_pv_reboot_nb);
-
-#ifdef CONFIG_SMP
-		smp_ops.smp_prepare_boot_cpu =
-			vmware_smp_prepare_boot_cpu;
-		if (cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
-					      "x86/vmware:online",
-					      vmware_cpu_online,
-					      vmware_cpu_down_prepare) < 0)
-			pr_err("vmware_guest: Failed to install cpu hotplug callbacks\n");
-#else
-		vmware_guest_cpu_init();
-#endif
-	}
+	has_steal_clock = vmware_is_stealclock_available();
 }
 #else
 #define vmware_paravirt_ops_setup() do {} while (0)