From: Paolo Bonzini <pbonzini@redhat.com>
To: linux-kernel@vger.kernel.org, kvm@vger.kernel.org
Cc: Sean Christopherson <seanjc@google.com>,
Hillf Danton <hdanton@sina.com>,
Steven Rostedt <rostedt@goodmis.org>,
Harry Yoo <harry@kernel.org>,
syzbot+f11512d1e5878ba5e60e@syzkaller.appspotmail.com
Subject: [PATCH 5/5] KVM: x86: move asynchronous page fault handling to vcpu_enter_guest()
Date: Thu, 5 Nov 2026 17:45:31 +0100 [thread overview]
Message-ID: <20261105164531.313849-6-pbonzini@redhat.com> (raw)
In-Reply-To: <20261105164531.313849-1-pbonzini@redhat.com>
Fully handle the PV async #PF in vcpu_enter_guest(). This has several
advantages; first of all nested async #PF code (which has more to do
with host page faults than nested MMU page faults) is moved away from
MMU page fault handling, second host_apf_flags is hidden (though only
partly so, because it is replaced by EXIT_FASTPATH_APF) and the node
put on the stack.
Reported-by: syzbot+f11512d1e5878ba5e60e@syzkaller.appspotmail.com
Link: https://patch.msgid.link/auqAsEk9o6buPNiR@google.com
Co-authored-by: Sean Christopherson <seanjc@google.com>
Signed-off-by: Sean Christopherson <seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/include/asm/kvm_host.h | 6 +++---
arch/x86/kvm/mmu.h | 21 ++++++++++++---------
arch/x86/kvm/mmu/mmu.c | 21 ++-------------------
arch/x86/kvm/svm/nested.c | 4 ----
arch/x86/kvm/svm/svm.c | 14 +++++++-------
arch/x86/kvm/vmx/main.c | 20 +++++++++++---------
arch/x86/kvm/vmx/nested.c | 3 +--
arch/x86/kvm/vmx/vmx.c | 6 +-----
arch/x86/kvm/x86.c | 17 +++++++++++++++--
9 files changed, 52 insertions(+), 60 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 93170c70a17c..f72028efe2ca 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -209,6 +209,7 @@ enum exit_fastpath_completion {
EXIT_FASTPATH_REENTER_GUEST,
EXIT_FASTPATH_EXIT_HANDLED,
EXIT_FASTPATH_EXIT_USERSPACE,
+ EXIT_FASTPATH_APF,
};
typedef enum exit_fastpath_completion fastpath_t;
@@ -950,8 +951,6 @@ struct kvm_vcpu_arch {
u64 msr_int_val; /* MSR_KVM_ASYNC_PF_INT */
u16 vec;
u32 id;
- u32 host_apf_flags;
- struct kvm_task_sleep_node host_apf_node;
bool pageready_pending;
} apf;
@@ -1681,7 +1680,8 @@ struct kvm_x86_ops {
struct x86_instruction_info *info,
enum x86_intercept_stage stage,
struct x86_exception *exception);
- fastpath_t (*handle_exit_irqoff)(struct kvm_vcpu *vcpu);
+ fastpath_t (*handle_exit_irqoff)(struct kvm_vcpu *vcpu,
+ struct kvm_task_sleep_node *host_apf_node);
void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu, bool enable);
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index 3cb2616d88ac..c01a1cf70d77 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -163,17 +163,20 @@ bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu);
int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
u64 fault_address, char *insn, int insn_len);
-static inline void kvm_handle_host_apf_irqoff(struct kvm_vcpu *vcpu, u32 token)
+static inline fastpath_t kvm_handle_host_apf_irqoff(struct kvm_task_sleep_node *host_apf_node,
+ u32 token)
{
- vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags();
- if ((vcpu->arch.apf.host_apf_flags & KVM_PV_REASON_PAGE_NOT_PRESENT) &&
- !kvm_async_pf_queue_task(token, &vcpu->arch.apf.host_apf_node)) {
- /*
- * Nothing to do but regular page fault handling must still
- * be suppressed, so leave host_apf_flags nonzero.
- */
- vcpu->arch.apf.host_apf_flags = KVM_PV_REASON_PAGE_READY;
+ u32 host_apf_flags = kvm_read_and_reset_apf_flags();
+
+ if (!host_apf_flags)
+ return EXIT_FASTPATH_NONE;
+
+ if (!WARN_ON_ONCE(!(host_apf_flags & KVM_PV_REASON_PAGE_NOT_PRESENT))) {
+ kvm_async_pf_queue_task(token, host_apf_node);
+ return EXIT_FASTPATH_APF;
}
+
+ return EXIT_FASTPATH_NONE;
}
void __kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 1f2ae73275aa..a8dfbfb83a6f 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -4948,9 +4948,6 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu,
int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
u64 fault_address, char *insn, int insn_len)
{
- int r = 1;
- u32 flags = vcpu->arch.apf.host_apf_flags;
-
#ifndef CONFIG_X86_64
/* A 64-bit CR2 should be impossible on 32-bit KVM. */
if (KVM_BUG_ON(fault_address >> 32, vcpu->kvm))
@@ -4971,23 +4968,9 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
BUILD_BUG_ON(lower_32_bits(PFERR_SYNTHETIC_MASK));
kvm_request_l1tf_flush_l1d();
- if (!flags) {
- trace_kvm_page_fault(vcpu, fault_address, error_code);
+ trace_kvm_page_fault(vcpu, fault_address, error_code);
- r = kvm_mmu_page_fault(vcpu, fault_address, error_code, insn,
- insn_len);
- } else if (flags & KVM_PV_REASON_PAGE_NOT_PRESENT) {
- vcpu->arch.apf.host_apf_flags = 0;
- local_irq_disable();
- kvm_async_pf_task_wait_schedule(&vcpu->arch.apf.host_apf_node);
- local_irq_enable();
- } else if (flags & KVM_PV_REASON_PAGE_READY) {
- vcpu->arch.apf.host_apf_flags = 0;
- } else {
- WARN_ONCE(1, "Unexpected host async PF flags: %x\n", flags);
- }
-
- return r;
+ return kvm_mmu_page_fault(vcpu, fault_address, error_code, insn, insn_len);
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_handle_page_fault);
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index af0ef678433d..36d9e39af34b 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -1818,10 +1818,6 @@ int nested_svm_exit_special(struct vcpu_svm *svm)
if (svm->vmcb01.ptr->control.intercepts[INTERCEPT_EXCEPTION] &
excp_bits)
return NESTED_EXIT_HOST;
- else if (exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR &&
- svm->vcpu.arch.apf.host_apf_flags)
- /* Trap async PF even if not shadowing */
- return NESTED_EXIT_HOST;
break;
}
case SVM_EXIT_VMMCALL:
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index b720c1ff7134..16d6947d981b 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -3815,7 +3815,6 @@ static void svm_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info,
*error_code = control->event_inj_err;
else
*error_code = 0;
-
}
static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
@@ -3834,7 +3833,7 @@ static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE))
return 0;
- if (is_guest_mode(vcpu)) {
+ if (is_guest_mode(vcpu) && exit_fastpath != EXIT_FASTPATH_APF) {
int vmexit;
trace_kvm_nested_vmexit(vcpu, KVM_ISA_SVM);
@@ -4762,10 +4761,6 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
vcpu_to_pmu(vcpu)->global_ctrl);
}
- /* if exit due to PF check for async PF */
- if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
- kvm_handle_host_apf_irqoff(vcpu, svm->vmcb->control.exit_info_2);
-
kvm_clear_available_registers(vcpu, SVM_REGS_LAZY_LOAD_SET);
trace_kvm_exit(vcpu, KVM_ISA_SVM);
@@ -5091,9 +5086,14 @@ static int svm_check_intercept(struct kvm_vcpu *vcpu,
return ret;
}
-static fastpath_t svm_handle_exit_irqoff(struct kvm_vcpu *vcpu)
+static fastpath_t svm_handle_exit_irqoff(struct kvm_vcpu *vcpu,
+ struct kvm_task_sleep_node *host_apf_node)
{
+ struct vcpu_svm *svm = to_svm(vcpu);
+
switch (to_svm(vcpu)->vmcb->control.exit_code) {
+ case SVM_EXIT_EXCP_BASE + PF_VECTOR:
+ return kvm_handle_host_apf_irqoff(host_apf_node, svm->vmcb->control.exit_info_2);
case SVM_EXIT_EXCP_BASE + MC_VECTOR:
svm_handle_mce(vcpu);
break;
diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c
index b742e2370dd6..6efa4e0051b5 100644
--- a/arch/x86/kvm/vmx/main.c
+++ b/arch/x86/kvm/vmx/main.c
@@ -912,17 +912,18 @@ static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu)
rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);
}
-static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info)
+static fastpath_t handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info,
+ struct kvm_task_sleep_node *host_apf_node)
{
- /* if exit due to PF check for async PF */
if (is_page_fault(intr_info))
- kvm_handle_host_apf_irqoff(vcpu, vt_get_exit_qual(vcpu));
- /* if exit due to NM, handle before interrupts are enabled */
- else if (is_nm_fault(intr_info))
+ return kvm_handle_host_apf_irqoff(host_apf_node, vt_get_exit_qual(vcpu));
+
+ if (is_nm_fault(intr_info))
handle_nm_fault_irqoff(vcpu);
- /* Handle machine checks before interrupts are enabled */
else if (is_machine_check(intr_info))
kvm_machine_check();
+
+ return EXIT_FASTPATH_NONE;
}
static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu,
@@ -941,7 +942,8 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu,
vcpu->arch.at_instruction_boundary = true;
}
-static fastpath_t vt_handle_exit_irqoff(struct kvm_vcpu *vcpu)
+static fastpath_t vt_handle_exit_irqoff(struct kvm_vcpu *vcpu,
+ struct kvm_task_sleep_node *host_apf_node)
{
if (to_vt(vcpu)->emulation_required)
return EXIT_FASTPATH_NONE;
@@ -951,8 +953,8 @@ static fastpath_t vt_handle_exit_irqoff(struct kvm_vcpu *vcpu)
handle_external_interrupt_irqoff(vcpu, vt_get_intr_info(vcpu));
break;
case EXIT_REASON_EXCEPTION_NMI:
- handle_exception_irqoff(vcpu, vt_get_intr_info(vcpu));
- break;
+ return handle_exception_irqoff(vcpu, vt_get_intr_info(vcpu),
+ host_apf_node);
case EXIT_REASON_MCE_DURING_VMENTRY:
kvm_machine_check();
break;
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 516ba03bf608..b7371c584f5a 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -6501,8 +6501,7 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu,
if (is_nmi(intr_info))
return true;
else if (is_page_fault(intr_info))
- return vcpu->arch.apf.host_apf_flags ||
- vmx_need_pf_intercept(vcpu);
+ return vmx_need_pf_intercept(vcpu);
else if (is_debug(intr_info) &&
vcpu->guest_debug &
(KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 82eb891f684c..3b6ad75ba7e7 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -5396,9 +5396,6 @@ static int vmx_handle_page_fault(struct kvm_vcpu *vcpu, u32 error_code)
{
unsigned long cr2 = vt_get_exit_qual(vcpu);
- if (vcpu->arch.apf.host_apf_flags)
- goto handle_pf;
-
/* When using EPT, KVM intercepts #PF only to detect illegal GPAs. */
WARN_ON_ONCE(enable_ept && !allow_smaller_maxphyaddr);
@@ -5429,7 +5426,6 @@ static int vmx_handle_page_fault(struct kvm_vcpu *vcpu, u32 error_code)
return 1;
}
-handle_pf:
return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0);
}
@@ -6733,7 +6729,7 @@ int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
return 1;
}
- if (nested_vmx_reflect_vmexit(vcpu))
+ if (exit_fastpath != EXIT_FASTPATH_APF && nested_vmx_reflect_vmexit(vcpu))
return 1;
}
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index af8cdfd81180..37e721bd2372 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -8116,6 +8116,16 @@ static void kvm_update_cpu_dirty_logging(struct kvm_vcpu *vcpu)
kvm_x86_call(update_cpu_dirty_logging)(vcpu, enable);
}
+static void kvm_handle_pv_async_pf(struct kvm_task_sleep_node *n)
+{
+ if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_KVM_GUEST)))
+ return;
+
+ local_irq_disable();
+ kvm_async_pf_task_wait_schedule(n);
+ local_irq_enable();
+}
+
/*
* Called within kvm->srcu read side.
* Returns 1 to let vcpu_run() continue the guest execution loop without
@@ -8128,9 +8138,9 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
bool req_int_win =
dm_request_for_irq_injection(vcpu) &&
kvm_cpu_accept_dm_intr(vcpu);
+ struct kvm_task_sleep_node host_apf_node = {0};
fastpath_t exit_fastpath;
u64 run_flags, debug_ctl;
-
bool req_immediate_exit = false;
if (kvm_request_pending(vcpu)) {
@@ -8505,7 +8515,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
fpu_sync_guest_vmexit_xfd_state();
if (exit_fastpath == EXIT_FASTPATH_NONE)
- exit_fastpath = kvm_x86_call(handle_exit_irqoff)(vcpu);
+ exit_fastpath = kvm_x86_call(handle_exit_irqoff)(vcpu, &host_apf_node);
if (vcpu->arch.guest_fpu.xfd_err)
wrmsrq(MSR_IA32_XFD_ERR, 0);
@@ -8567,6 +8577,9 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
if (vcpu->arch.apic_attention)
kvm_lapic_sync_from_vapic(vcpu);
+ if (exit_fastpath == EXIT_FASTPATH_APF && host_apf_node.token)
+ kvm_handle_pv_async_pf(&host_apf_node);
+
r = kvm_x86_call(handle_exit)(vcpu, exit_fastpath);
return r;
--
2.56.0
next prev parent reply other threads:[~2026-11-05 16:45 UTC|newest]
Thread overview: 7+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-05 16:45 [PATCH v2 0/5] KVM: remove the dummy allocation case from async page fault handling Paolo Bonzini
2026-11-05 16:45 ` [PATCH 1/5] x86: KVM: take b->lock around wait Paolo Bonzini
2026-11-05 16:45 ` [PATCH 2/5] KVM: x86: register asynchronous page fault waits with interrupts disabled Paolo Bonzini
2026-11-05 16:45 ` [PATCH 3/5] x86: KVM: remove the dummy allocation case from async page fault handling Paolo Bonzini
2026-11-05 16:45 ` [PATCH 4/5] KVM: x86: allow handle_exit_irqoff to return EXIT_FASTPATH_* Paolo Bonzini
2026-11-05 16:45 ` Paolo Bonzini [this message]
2026-11-05 19:08 ` [PATCH 5/5] KVM: x86: move asynchronous page fault handling to vcpu_enter_guest() Sean Christopherson
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261105164531.313849-6-pbonzini@redhat.com \
--to=pbonzini@redhat.com \
--cc=harry@kernel.org \
--cc=hdanton@sina.com \
--cc=kvm@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=rostedt@goodmis.org \
--cc=seanjc@google.com \
--cc=syzbot+f11512d1e5878ba5e60e@syzkaller.appspotmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®