[PATCH v11 11/23] KVM: x86: Activate master clock from kvm_arch_init_vm()

[David Woodhouse <dwmw2@infradead.org>]

From: David Woodhouse <dwmw@amazon.co.uk>

Initialize the TSC generation and master clock state at VM creation
time, rather than relying on kvm_update_masterclock() from
kvm_arch_vcpu_postcreate().

Set cur_tsc_generation=1, cur_tsc_khz, cur_tsc_scaling_ratio,
cur_tsc_offset, cur_tsc_nsec and kvmclock_offset from an atomic
ktime/TSC snapshot. This establishes a coherent epoch so that:

 - The first vCPU synchronizes to the existing generation (getting
   the pre-computed TSC offset) rather than computing its own.
 - pvclock_update_vm_gtod_copy() succeeds immediately (no zero
   tsc_hz from uninitialized cur_tsc_scaling_ratio).
 - KVM_GET_CLOCK returns valid {host_tsc, realtime} fields as soon
   as any vCPU exists.
 - KVM_[GS]ET_CLOCK_GUEST works without a prior KVM_RUN.
 - No O(N) iteration of all vCPUs on each vCPU creation.

This replaces the KVM_REQ_MASTERCLOCK_UPDATE check and
kvm_update_masterclock() call in kvm_arch_vcpu_postcreate(), which
worked around the missing initialization. A masterclock refresh is
still performed (synchronously, so that it is valid even for userspace
which queries the clock before ever running a vCPU) when the first
vCPU is created, since the VM-creation snapshot can go stale while the
VM has no vCPUs: a clocksource change notifies only VMs with vCPUs,
and a change back to TSC leaves nothing to disagree with the stale
snapshot.

The generation's recorded offset in kvm_synchronize_tsc() can now only
be joined by a vCPU with the same scaling ratio it was recorded with:
the initial generation's offset is recorded with the default ratio
before any vCPU exists, and a vCPU with a different ratio opens a new
generation with its own offset, keeping the generation's ratio and
offset describing the same scaling domain.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
 arch/x86/kvm/x86.c | 76 ++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 64 insertions(+), 12 deletions(-)

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 940168528721..6dbdb60962fe 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -1500,11 +1500,25 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *value, bool user_initiated)
 
 		data = kvm->arch.cur_tsc_write + delta;
 
-		if (!kvm_check_tsc_unstable())
-			offset = kvm->arch.cur_tsc_offset;
-		else
+		/*
+		 * The generation's offset can only be joined by a vCPU which
+		 * scales the TSC by the same ratio it was recorded with; the
+		 * initial generation's offset is recorded at VM creation
+		 * with the default ratio, before any vCPU exists. With a
+		 * different ratio, open a new generation (recording this
+		 * vCPU's own offset) rather than leave the generation's
+		 * ratio and offset describing different scaling domains.
+		 */
+		if (vcpu->arch.l1_tsc_scaling_ratio !=
+		    kvm->arch.cur_tsc_scaling_ratio) {
 			offset = kvm_compute_l1_tsc_offset(vcpu, host_tsc, data);
-		matched = true;
+		} else {
+			if (!kvm_check_tsc_unstable())
+				offset = kvm->arch.cur_tsc_offset;
+			else
+				offset = kvm_compute_l1_tsc_offset(vcpu, host_tsc, data);
+			matched = true;
+		}
 	}
 
 	__kvm_synchronize_tsc(vcpu, offset, data, ns, matched, user_initiated);
@@ -9645,8 +9659,15 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 		return;
 	vcpu_load(vcpu);
 	kvm_synchronize_tsc(vcpu, NULL, false);
-	if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
+
+	/*
+	 * Refresh the VM-creation masterclock snapshot (see
+	 * kvm_arch_init_vm), synchronously so that it is valid even for
+	 * userspace which queries the clock before ever running a vCPU.
+	 */
+	if (vcpu->vcpu_idx == 0)
 		kvm_update_masterclock(vcpu->kvm, NULL);
+
 	vcpu_put(vcpu);
 
 	/* poll control enabled by default */
@@ -10065,6 +10086,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 {
 	int ret;
 	unsigned long flags;
+	u64 kvmclock_host_tsc;
+	s64 kvmclock_ns;
 
 	if (!kvm_is_vm_type_supported(type))
 		return -EINVAL;
@@ -10096,9 +10119,13 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 	seqcount_raw_spinlock_init(&kvm->arch.pvclock_sc, &kvm->arch.tsc_write_lock);
 	ratelimit_state_init(&kvm->arch.kvmclock_update_rs, HZ, 10);
 	ratelimit_set_flags(&kvm->arch.kvmclock_update_rs, RATELIMIT_MSG_ON_RELEASE);
-	kvm->arch.kvmclock_offset = -get_kvmclock_base_ns();
-	kvm->arch.all_vcpus_matched_freq = true;
 
+	kvm->arch.default_tsc_khz = max_tsc_khz ? : tsc_khz;
+	kvm->arch.apic_bus_cycle_ns = APIC_BUS_CYCLE_NS_DEFAULT;
+	kvm->arch.guest_can_read_msr_platform_info = true;
+	kvm->arch.enable_pmu = enable_pmu && !kvm->arch.has_protected_pmu;
+
+	kvm->arch.all_vcpus_matched_freq = true;
 	/*
 	 * Start with all (zero) vCPUs considered matched, so that a VM whose
 	 * vCPUs never reach kvm_track_tsc_matching() — notably when the guest
@@ -10106,16 +10133,41 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 	 * returns early — is still eligible for PVCLOCK_TSC_STABLE_BIT.
 	 */
 	kvm->arch.all_vcpus_matched_tsc = true;
+	kvm->arch.cur_tsc_scaling_ratio = kvm_caps.default_tsc_scaling_ratio;
+	kvm->arch.cur_tsc_khz = kvm->arch.default_tsc_khz;
 
 	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
+
+	/*
+	 * Establish the initial TSC generation for synchronization logic.
+	 * The first vCPU will sync to this, starting its guest TSC at the
+	 * number of cycles elapsed since VM creation rather than at zero.
+	 * This ensures coherence with the kvmclock epoch (also set here),
+	 * and the master clock snapshot taken below is immediately valid.
+	 * It is refreshed again when the first vCPU is created, in case
+	 * the host clocksource changed (and changed back) in between:
+	 * pvclock_gtod_update_fn() does not notify a VM with no vCPUs.
+	 */
+#ifdef CONFIG_X86_64
+	if (!kvm_get_time_and_clockread(&kvmclock_ns, &kvmclock_host_tsc))
+#endif
+	{
+		/*
+		 * If not simultaneous, we want the TSC to be zero slightly
+		 * *after* the kvmclock is zero.
+		 */
+		kvmclock_ns = get_kvmclock_base_ns();
+		kvmclock_host_tsc = rdtsc();
+	}
+	kvm->arch.cur_tsc_write = 0;
+	kvm->arch.cur_tsc_offset = -(s64)kvmclock_host_tsc;
+	kvm->arch.cur_tsc_nsec = kvmclock_ns;
+	kvm->arch.cur_tsc_generation = 1;
+	kvm->arch.kvmclock_offset = -kvmclock_ns;
+
 	pvclock_update_vm_gtod_copy(kvm);
 	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
 
-	kvm->arch.default_tsc_khz = max_tsc_khz ? : tsc_khz;
-	kvm->arch.apic_bus_cycle_ns = APIC_BUS_CYCLE_NS_DEFAULT;
-	kvm->arch.guest_can_read_msr_platform_info = true;
-	kvm->arch.enable_pmu = enable_pmu && !kvm->arch.has_protected_pmu;
-
 #if IS_ENABLED(CONFIG_HYPERV)
 	spin_lock_init(&kvm->arch.hv_root_tdp_lock);
 	kvm->arch.hv_root_tdp = INVALID_PAGE;
-- 
2.43.0