[PATCH v11 08/23] KVM: x86: Re-synchronize TSC after KVM_SET_TSC_KHZ

[David Woodhouse <dwmw2@infradead.org>]

From: David Woodhouse <dwmw@amazon.co.uk>

KVM_SET_TSC_KHZ changes the vCPU's TSC scaling ratio but does not
update the VM-wide cur_tsc_scaling_ratio used by get_kvmclock().
This causes get_kvmclock() to use a stale (default 1:1) ratio when
computing the KVM clock, leading to drift between the host-side
kvmclock and what the guest observes.

Fix this by calling kvm_synchronize_tsc() after changing the TSC
frequency. This:
 - Updates cur_tsc_scaling_ratio (consumed by pvclock_update_vm_gtod_copy)
 - Ensures the TSC value is continuous across the frequency change
 - Triggers kvm_track_tsc_matching() for proper masterclock handling
 - Establishes the new TSC generation for subsequent vCPUs to match

Since kvm_synchronize_tsc() previously inferred "userspace wrote the
TSC" from being given a non-NULL value, and this new call is a purely
kernel-internal resynchronization (including from vCPU creation via
the default frequency), make that an explicit 'user_initiated'
parameter instead. Otherwise the kernel's own call would set
kvm->arch.user_set_tsc and incorrectly enable the 1-second slop
heuristic before userspace has ever written a TSC value, breaking
the restore of a guest with less than one second of runtime.

Only preserve the TSC value across the change for a vCPU which has
actually run. For one which has not, synchronize as for a zero write
instead, joining the current generation. At vCPU creation (which comes
through here with the default frequency) the value read happens to be
zero anyway, because the scaling ratio has not yet been initialized;
make that explicit rather than an accident. For an explicit
KVM_SET_TSC_KHZ on a never-run vCPU, the nonzero value could never
match — userspace has not written a TSC, so the matching window is not
armed — and each such call would open a new TSC generation, orphaning
the matched state of the sibling vCPUs and denying
PVCLOCK_TSC_STABLE_BIT to a multi-vCPU guest whose VMM never writes
the TSCs explicitly. If userspace has already written a TSC, preserve
the value even for a never-run vCPU — a zero write would discard it —
and let the user-write matching window do its usual job.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
 arch/x86/kvm/msrs.c |  2 +-
 arch/x86/kvm/x86.c  | 38 +++++++++++++++++++++++++++++++++-----
 arch/x86/kvm/x86.h  |  2 +-
 3 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/arch/x86/kvm/msrs.c b/arch/x86/kvm/msrs.c
index 26eb30f10c43..d07831b5dfb8 100644
--- a/arch/x86/kvm/msrs.c
+++ b/arch/x86/kvm/msrs.c
@@ -1645,7 +1645,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		break;
 	case MSR_IA32_TSC:
 		if (msr_info->host_initiated) {
-			kvm_synchronize_tsc(vcpu, &data);
+			kvm_synchronize_tsc(vcpu, &data, true);
 		} else if (!vcpu->arch.guest_tsc_protected) {
 			u64 adj = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data) -
 				  vcpu->arch.l1_tsc_offset;
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 77bd3e4bfdcd..f551056d7274 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -1064,6 +1064,7 @@ static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
 {
 	u32 thresh_lo, thresh_hi;
 	int use_scaling = 0;
+	u64 tsc;
 
 	/* tsc_khz can be zero if TSC calibration fails */
 	if (user_tsc_khz == 0) {
@@ -1091,7 +1092,34 @@ static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
 			 user_tsc_khz, thresh_lo, thresh_hi);
 		use_scaling = 1;
 	}
-	return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);
+	/*
+	 * Read the guest TSC before changing the ratio, so we can
+	 * re-synchronize to preserve continuity across the change.
+	 */
+	tsc = kvm_read_l1_tsc(vcpu, rdtsc());
+
+	if (set_tsc_khz(vcpu, user_tsc_khz, use_scaling))
+		return -1;
+
+	/*
+	 * Preserving continuity only makes sense for a vCPU which has
+	 * actually run; for one which has not, synchronize as for a zero
+	 * write, joining the current generation. This makes vCPU creation
+	 * (which comes through here with the default frequency) join the
+	 * generation *explicitly*, rather than by the accident that the
+	 * scaling ratio is still zero at this point so the TSC read above
+	 * is zero anyway — and it stops an explicit KVM_SET_TSC_KHZ on a
+	 * never-run vCPU, whose nonzero TSC value could never match, from
+	 * opening a new generation and orphaning the matched state of its
+	 * siblings. If userspace has already written a TSC, preserve the
+	 * value even for a never-run vCPU — a zero write would discard it —
+	 * and let the user-write matching window do its usual job.
+	 */
+	if (vcpu->arch.last_vmentry_cpu != -1 || vcpu->kvm->arch.user_set_tsc)
+		kvm_synchronize_tsc(vcpu, &tsc, false);
+	else
+		kvm_synchronize_tsc(vcpu, NULL, false);
+	return 0;
 }
 
 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
@@ -1453,9 +1481,9 @@ static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,
 	kvm_track_tsc_matching(vcpu, !matched);
 }
 
-void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value)
+void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *value, bool user_initiated)
 {
-	u64 data = user_value ? *user_value : 0;
+	u64 data = value ? *value : 0;
 	struct kvm *kvm = vcpu->kvm;
 	u64 offset, host_tsc, elapsed;
 	s64 ns;
@@ -1531,7 +1559,7 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value)
 		matched = true;
 	}
 
-	__kvm_synchronize_tsc(vcpu, offset, data, ns, matched, !!user_value);
+	__kvm_synchronize_tsc(vcpu, offset, data, ns, matched, user_initiated);
 	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
 }
 
@@ -9665,7 +9693,7 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 	if (mutex_lock_killable(&vcpu->mutex))
 		return;
 	vcpu_load(vcpu);
-	kvm_synchronize_tsc(vcpu, NULL);
+	kvm_synchronize_tsc(vcpu, NULL, false);
 	if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
 		kvm_update_masterclock(vcpu->kvm, NULL);
 	vcpu_put(vcpu);
diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h
index 436e2fb1396c..de4c86ea5a7d 100644
--- a/arch/x86/kvm/x86.h
+++ b/arch/x86/kvm/x86.h
@@ -330,7 +330,7 @@ uint64_t kvm_get_wall_clock_epoch(struct kvm *kvm);
 bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp);
 int kvm_guest_time_update(struct kvm_vcpu *v);
 
-void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value);
+void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *value, bool user_initiated);
 u64 kvm_scale_tsc(u64 tsc, u64 ratio);
 u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc);
 u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier);
-- 
2.43.0