From: David Woodhouse <dwmw2@infradead.org>
To: kvm@vger.kernel.org
Cc: linux-kernel@vger.kernel.org, linux-kselftest@vger.kernel.org,
rcu@vger.kernel.org, linux-rt-devel@lists.linux.dev,
Paolo Bonzini <pbonzini@redhat.com>,
Sean Christopherson <seanjc@google.com>,
Paul Durrant <paul@xen.org>,
Vitaly Kuznetsov <vkuznets@redhat.com>,
Fred Griffoul <fgriffo@amazon.co.uk>,
"Paul E . McKenney" <paulmck@kernel.org>,
Kunwu Chan <kunwu.chan@linux.dev>,
Kunwu Chan <kunwu.chan@gmail.com>,
syzbot+208f7f3e5f59c11aeb90@syzkaller.appspotmail.com,
David Woodhouse <dwmw@amazon.co.uk>
Subject: [PATCH v2 18/20] KVM: selftests: Add nested transition benchmark
Date: Thu, 5 Nov 2026 02:01:00 +0000 [thread overview]
Message-ID: <20261105020104.2507083-19-dwmw2@infradead.org> (raw)
In-Reply-To: <20261105020104.2507083-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
Measure L1<->L2 transition costs from L1's viewpoint, in TSC cycles:
a reflected-exit phase (cpuid in L2, forwarded to L1: the full
emulated vmexit->vmentry round trip, exercising the vmcb12/vmcs12
access paths), and on SVM an MSR-intercept phase (rdmsr in L2 with
INTERCEPT_MSR_PROT: exercises the per-intercept lookup in L1's MSR
permissions map). Reports min/avg/p50/p99/max over 10k iterations.
Works on both VMX and SVM.
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Assisted-by: LLM
---
tools/testing/selftests/kvm/Makefile.kvm | 1 +
.../kvm/x86/nested_transition_bench.c | 210 ++++++++++++++++++
2 files changed, 211 insertions(+)
create mode 100644 tools/testing/selftests/kvm/x86/nested_transition_bench.c
diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
index dcfe815f4a02..65639e54b554 100644
--- a/tools/testing/selftests/kvm/Makefile.kvm
+++ b/tools/testing/selftests/kvm/Makefile.kvm
@@ -162,6 +162,7 @@ TEST_GEN_PROGS_x86 += x86/recalc_apic_map_test
TEST_GEN_PROGS_x86 += x86/aperfmperf_test
TEST_GEN_PROGS_x86 += x86/guest_memfd_conversions_test
TEST_GEN_PROGS_x86 += x86/vmx_apic_update_test
+TEST_GEN_PROGS_x86 += x86/nested_transition_bench
TEST_GEN_PROGS_x86 += access_tracking_perf_test
TEST_GEN_PROGS_x86 += coalesced_io_test
TEST_GEN_PROGS_x86 += dirty_log_perf_test
diff --git a/tools/testing/selftests/kvm/x86/nested_transition_bench.c b/tools/testing/selftests/kvm/x86/nested_transition_bench.c
new file mode 100644
index 000000000000..ea91ffcc4e0e
--- /dev/null
+++ b/tools/testing/selftests/kvm/x86/nested_transition_bench.c
@@ -0,0 +1,210 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * nested_transition_bench
+ *
+ * Measure nested virtualization transition costs:
+ * 1. L2 exit reflected to L1 (cpuid): full emulated vmexit->vmentry
+ * round trip, exercising the vmcb12/vmcs12 access paths.
+ * 2. L2 MSR access with MSR intercepts enabled: exercises the
+ * per-intercept L1 permissions-bitmap lookup on SVM, and the
+ * bitmap merge path when force_msr_bitmap_recalc is set.
+ *
+ * Reports TSC cycles per iteration (min/avg/p99) from L1's viewpoint.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "test_util.h"
+#include "kvm_util.h"
+#include "processor.h"
+#include "vmx.h"
+#include "svm_util.h"
+
+/*
+ * Keep the iteration count modest: every timed iteration includes a
+ * ucall round trip to userspace to report its sample, which on a
+ * debug (KASAN) kernel costs milliseconds. 10k samples still give a
+ * stable p99.
+ */
+#define N_WARMUP 1000
+#define N_ITERS 10000
+
+
+static uint64_t results[N_ITERS];
+
+/* L2: exit to L1 in a tight loop. */
+static void l2_guest_code(void)
+{
+ u32 eax, ebx, ecx, edx;
+
+ for (;;)
+ cpuid(0, &eax, &ebx, &ecx, &edx);
+}
+
+static void l2_guest_code_msr(void)
+{
+ for (;;)
+ rdmsr(MSR_IA32_TSC_DEADLINE);
+}
+
+/* SVM L1: run L2, timing VMRUN -> #VMEXIT round trips. */
+static void l1_svm_code(struct svm_test_data *svm)
+{
+ struct vmcb *vmcb = svm->vmcb;
+ uint64_t t0, t1;
+ int i;
+
+ generic_svm_setup(svm, l2_guest_code);
+ /*
+ * Unlike VMX, where CPUID unconditionally exits, SVM only
+ * intercepts CPUID if asked; without it L0 emulates L2's cpuid
+ * and resumes L2 directly, and run_guest() never returns.
+ */
+ vmcb->control.intercept |= BIT_ULL(INTERCEPT_CPUID) |
+ BIT_ULL(INTERCEPT_MSR_PROT);
+
+ /*
+ * Set the read-intercept bit for MSR_IA32_TSC_DEADLINE (0x6e0,
+ * MSRPM range 0: bit 2*msr, i.e. byte 0x1b8 bit 0) in L1's MSRPM,
+ * so that L2's rdmsr in phase 2 reflects to L1. L0's per-exit
+ * lookup in this bitmap is the path being measured.
+ */
+ ((u8 *)svm->msr)[(MSR_IA32_TSC_DEADLINE & 0x1fff) * 2 / 8] |=
+ BIT((MSR_IA32_TSC_DEADLINE & 0x1fff) * 2 % 8);
+
+ for (i = 0; i < N_WARMUP; i++) {
+ run_guest(vmcb, svm->vmcb_gpa);
+ GUEST_ASSERT(vmcb->control.exit_code == SVM_EXIT_CPUID);
+ vmcb->save.rip += 2; /* cpuid */
+ }
+
+ for (i = 0; i < N_ITERS; i++) {
+ t0 = rdtsc();
+ run_guest(vmcb, svm->vmcb_gpa);
+ t1 = rdtsc();
+ GUEST_ASSERT(vmcb->control.exit_code == SVM_EXIT_CPUID);
+ vmcb->save.rip += 2;
+ GUEST_SYNC1(t1 - t0);
+ }
+
+ /* Phase 2: MSR intercept path */
+ vmcb->save.rip = (u64)l2_guest_code_msr;
+ for (i = 0; i < N_WARMUP; i++) {
+ run_guest(vmcb, svm->vmcb_gpa);
+ GUEST_ASSERT(vmcb->control.exit_code == SVM_EXIT_MSR);
+ vmcb->save.rip += 2; /* rdmsr */
+ }
+
+ for (i = 0; i < N_ITERS; i++) {
+ t0 = rdtsc();
+ run_guest(vmcb, svm->vmcb_gpa);
+ t1 = rdtsc();
+ GUEST_ASSERT(vmcb->control.exit_code == SVM_EXIT_MSR);
+ vmcb->save.rip += 2; /* rdmsr */
+ GUEST_SYNC1(t1 - t0);
+ }
+
+ GUEST_DONE();
+}
+
+/* VMX L1: run L2, timing VMRESUME -> vmexit round trips. */
+static void l1_vmx_code(struct vmx_pages *vmx)
+{
+ uint64_t t0, t1;
+ int i;
+
+ prepare_for_vmx_operation(vmx);
+ load_vmcs(vmx);
+ prepare_vmcs(vmx, l2_guest_code);
+
+ vmlaunch();
+ /* first exit is here; loop resumes */
+ for (i = 0; i < N_WARMUP; i++) {
+ GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_CPUID);
+ vmwrite(GUEST_RIP, vmread(GUEST_RIP) +
+ vmread(VM_EXIT_INSTRUCTION_LEN));
+ vmresume();
+ }
+
+ for (i = 0; i < N_ITERS; i++) {
+ GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_CPUID);
+ vmwrite(GUEST_RIP, vmread(GUEST_RIP) +
+ vmread(VM_EXIT_INSTRUCTION_LEN));
+ t0 = rdtsc();
+ vmresume();
+ t1 = rdtsc();
+ GUEST_SYNC1(t1 - t0);
+ }
+
+ GUEST_DONE();
+}
+
+static int cmp_u64(const void *a, const void *b)
+{
+ uint64_t x = *(const uint64_t *)a, y = *(const uint64_t *)b;
+
+ return x < y ? -1 : x > y;
+}
+
+static void report(const char *phase, int n)
+{
+ uint64_t sum = 0;
+ int i;
+
+ qsort(results, n, sizeof(results[0]), cmp_u64);
+ for (i = 0; i < n; i++)
+ sum += results[i];
+
+ printf("%-24s n=%d min=%" PRIu64 " avg=%" PRIu64 " p50=%" PRIu64
+ " p99=%" PRIu64 " max=%" PRIu64 " cycles\n",
+ phase, n, results[0], sum / n, results[n / 2],
+ results[(int)(n * 0.99)], results[n - 1]);
+}
+
+int main(int argc, char *argv[])
+{
+ struct kvm_vcpu *vcpu;
+ struct kvm_vm *vm;
+ struct ucall uc;
+ gva_t nested_gva = 0;
+ bool is_svm = kvm_cpu_has(X86_FEATURE_SVM);
+ int n = 0, phase = 0;
+ bool done = false;
+
+ TEST_REQUIRE(is_svm || kvm_cpu_has(X86_FEATURE_VMX));
+
+ vm = vm_create_with_one_vcpu(&vcpu, is_svm ? (void *)l1_svm_code
+ : (void *)l1_vmx_code);
+ if (is_svm)
+ vcpu_alloc_svm(vm, &nested_gva);
+ else
+ vcpu_alloc_vmx(vm, &nested_gva);
+ vcpu_args_set(vcpu, 1, nested_gva);
+
+ while (!done) {
+ vcpu_run(vcpu);
+ switch (get_ucall(vcpu, &uc)) {
+ case UCALL_SYNC:
+ results[n++] = uc.args[0];
+ if (n == N_ITERS) {
+ report(phase == 0 ? "reflected-exit" :
+ "msr-intercept", n);
+ n = 0;
+ phase++;
+ }
+ break;
+ case UCALL_ABORT:
+ REPORT_GUEST_ASSERT(uc);
+ case UCALL_DONE:
+ if (n)
+ report("partial", n);
+ done = true;
+ break;
+ default:
+ TEST_FAIL("Unknown ucall %lu", uc.cmd);
+ }
+ }
+
+ kvm_vm_free(vm);
+ return 0;
+}
--
2.43.0
next prev parent reply other threads:[~2026-11-05 2:01 UTC|newest]
Thread overview: 30+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-05 2:00 [PATCH v2 00/20] KVM: Use atomic SRCU for gfn-to-pfn cache, reinstate guest mode for x86 nesting David Woodhouse
2026-11-05 2:00 ` [PATCH v2 01/20] KVM: x86/xen: Fill the hypercall page with a single guest write David Woodhouse
2026-11-05 2:00 ` [PATCH v2 02/20] KVM: x86/xen: Add KVM_XEN_VCPU_ATTR_TYPE_WRITE_HYPERCALL_PAGE David Woodhouse
2026-11-05 2:00 ` [PATCH v2 03/20] KVM: x86/xen: Extract delivery of event to vCPU into a separate helper David Woodhouse
2026-11-05 2:00 ` [PATCH v2 04/20] KVM: x86/xen: Explicitly tag "shared info" page as never being dirty tracked David Woodhouse
2026-11-05 2:00 ` [PATCH v2 05/20] KVM: x86/xen: Don't dirty track "vCPU info" page David Woodhouse
2026-11-05 2:00 ` [PATCH v2 06/20] KVM: x86: Request the guest TLB flush from record_steal_time() David Woodhouse
2026-11-10 20:19 ` Sean Christopherson
2026-11-10 21:14 ` David Woodhouse
2026-11-05 2:00 ` [PATCH v2 07/20] KVM: selftests: Actually deactivate the HVA-based shared_info cache in xen_shinfo_test David Woodhouse
2026-11-05 2:00 ` [PATCH v2 08/20] KVM: pfncache: Use atomic SRCU for readers instead of a rwlock David Woodhouse
2026-11-05 2:00 ` [PATCH v2 09/20] KVM: x86: Use gfn_to_pfn_cache for steal time / preempted status David Woodhouse
2026-11-05 2:00 ` [PATCH v2 10/20] KVM: pfncache: Add guest-mode pinning (GUEST_USES_PFN successor) David Woodhouse
2026-11-05 2:29 ` sashiko-bot
2026-11-05 2:00 ` [PATCH v2 11/20] KVM: nVMX: Implement cache for L1 MSR bitmap David Woodhouse
2026-11-05 2:00 ` [PATCH v2 12/20] KVM: nVMX: Use pinned pfncache for L1 APIC virtualization pages David Woodhouse
2026-11-05 2:33 ` sashiko-bot
2026-11-05 2:00 ` [PATCH v2 13/20] KVM: x86: Post KVM_REQ_GET_NESTED_STATE_PAGES on memslot updates David Woodhouse
2026-11-05 2:00 ` [PATCH v2 14/20] KVM: selftests: Add nested VMX APIC cache invalidation test David Woodhouse
2026-11-05 2:25 ` sashiko-bot
2026-11-05 2:00 ` [PATCH v2 15/20] KVM: nSVM: Use a gfn_to_pfn_cache for the vmcb12 page David Woodhouse
2026-11-05 2:00 ` [PATCH v2 16/20] KVM: nSVM: Cache L1's MSR permissions map pages David Woodhouse
2026-11-05 2:00 ` [PATCH v2 17/20] KVM: nSVM: Cache L1's IO " David Woodhouse
2026-11-05 2:01 ` David Woodhouse [this message]
2026-11-05 2:01 ` [PATCH v2 19/20] KVM: selftests: Add Xen evtchn vs memslot-update race test David Woodhouse
2026-11-05 2:01 ` [PATCH v2 20/20] KVM: selftests: Add Xen evtchn vs mmu_notifier race test for HVA caches David Woodhouse
2026-11-05 2:28 ` sashiko-bot
2026-11-10 19:52 ` [PATCH v2 00/20] KVM: Use atomic SRCU for gfn-to-pfn cache, reinstate guest mode for x86 nesting David Woodhouse
2026-11-10 20:14 ` Sean Christopherson
2026-11-11 18:38 ` David Woodhouse
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261105020104.2507083-19-dwmw2@infradead.org \
--to=dwmw2@infradead.org \
--cc=dwmw@amazon.co.uk \
--cc=fgriffo@amazon.co.uk \
--cc=kunwu.chan@gmail.com \
--cc=kunwu.chan@linux.dev \
--cc=kvm@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-kselftest@vger.kernel.org \
--cc=linux-rt-devel@lists.linux.dev \
--cc=paul@xen.org \
--cc=paulmck@kernel.org \
--cc=pbonzini@redhat.com \
--cc=rcu@vger.kernel.org \
--cc=seanjc@google.com \
--cc=syzbot+208f7f3e5f59c11aeb90@syzkaller.appspotmail.com \
--cc=vkuznets@redhat.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®