[PATCH V3 0/2] LoongArch: KVM: fix "unreliable stack" issue

[PATCH V3 0/2] LoongArch: KVM: fix "unreliable stack" issue

[Xianglai Li]

When starting multi-core loongarch virtualization on loongarch physical
machine, loading livepatch on the physical machine will cause an error
similar to the following:
[  411.686289] livepatch: klp_try_switch_task: CPU 31/KVM:3116 has an
unreliable stack

The specific test steps are as follows:
1.Start a multi-core virtual machine on a physical machine

2.Enter the following command on the physical machine to turn on the debug
switch:
  echo "file kernel/livepatch/transition.c +p"  > /sys/kernel/debug/\
dynamic_debug/control 


3.Load livepatch:
 modprobe  livepatch-sample 

Through the above steps, similar prints can be viewed in dmesg.

The reason for this issue is that the code of the kvm_exc_entry function
was copied in the function kvm_loongarch_env_init. When the cpu needs to
execute kvm_exc_entry, it will switch to the copied address for execution.
The new address of the kvm_exc_entry function cannot be recognized in ORC,
which eventually leads to the arch_stack_walk_reliable function returning
an error and printing an exception message.

To solve the above problems, we directly compile the switch.S file into
the kernel instead of the module. In this way, the function kvm_exc_entry
will no longer need to be copied.

changlog:
V3<-V2:
1.Replace the EXPORT_SYMBOL macro declaration symbol with the
EXPORT_SYMBOL_FOR_KVM macro

2.Add some comments in kvm_enter_guest

3.Place the correct pc address in era

4.Move .p2align after .text

V2<-V1:
1.Rollback the modification of function parameter types such as
kvm_save_fpu. In the asm-prototypes.h header file, only the parameter types
it depends on are included

Cc: Huacai Chen <chenhuacai@kernel.org>
Cc: WANG Xuerui <kernel@xen0n.name>
Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
Cc: Bibo Mao <maobibo@loongson.cn>
Cc: Charlie Jenkins <charlie@rivosinc.com>
Cc: Xianglai Li <lixianglai@loongson.cn>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: stable@vger.kernel.org
Cc: Tiezhu Yang <yangtiezhu@loongson.cn>

Xianglai Li (2):
  LoongArch: KVM: Compile the switch.S file directly into the kernel
  LoongArch: KVM: fix "unreliable stack" issue

 arch/loongarch/Kbuild                       |  2 +-
 arch/loongarch/include/asm/asm-prototypes.h | 21 +++++++++++++
 arch/loongarch/include/asm/kvm_host.h       |  3 --
 arch/loongarch/kvm/Makefile                 |  2 +-
 arch/loongarch/kvm/main.c                   | 35 ++-------------------
 arch/loongarch/kvm/switch.S                 | 32 ++++++++++++++++---
 6 files changed, 53 insertions(+), 42 deletions(-)


base-commit: 8f0b4cce4481fb22653697cced8d0d04027cb1e8
-- 
2.39.1

[PATCH V3 1/2] LoongArch: KVM: Compile the switch.S file directly into the kernel

[Xianglai Li]

If we directly compile the switch.S file into the kernel, the address of
the kvm_exc_entry function will definitely be within the DMW memory area.
Therefore, we will no longer need to perform a copy relocation of
kvm_exc_entry.

Based on the above description, compile switch.S directly into the kernel,
and then remove the copy relocation execution logic for the kvm_exc_entry
function.

Cc: stable@vger.kernel.org
Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
---
Cc: Huacai Chen <chenhuacai@kernel.org>
Cc: WANG Xuerui <kernel@xen0n.name>
Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
Cc: Bibo Mao <maobibo@loongson.cn>
Cc: Charlie Jenkins <charlie@rivosinc.com>
Cc: Xianglai Li <lixianglai@loongson.cn>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Tiezhu Yang <yangtiezhu@loongson.cn>

 arch/loongarch/Kbuild                       |  2 +-
 arch/loongarch/include/asm/asm-prototypes.h | 21 +++++++++++++
 arch/loongarch/include/asm/kvm_host.h       |  3 --
 arch/loongarch/kvm/Makefile                 |  2 +-
 arch/loongarch/kvm/main.c                   | 35 ++-------------------
 arch/loongarch/kvm/switch.S                 | 22 ++++++++++---
 6 files changed, 43 insertions(+), 42 deletions(-)

diff --git a/arch/loongarch/Kbuild b/arch/loongarch/Kbuild
index beb8499dd8ed..1c7a0dbe5e72 100644
--- a/arch/loongarch/Kbuild
+++ b/arch/loongarch/Kbuild
@@ -3,7 +3,7 @@ obj-y += mm/
 obj-y += net/
 obj-y += vdso/
 
-obj-$(CONFIG_KVM) += kvm/
+obj-$(subst m,y,$(CONFIG_KVM)) += kvm/
 
 # for cleaning
 subdir- += boot
diff --git a/arch/loongarch/include/asm/asm-prototypes.h b/arch/loongarch/include/asm/asm-prototypes.h
index 704066b4f736..e8ce153691e5 100644
--- a/arch/loongarch/include/asm/asm-prototypes.h
+++ b/arch/loongarch/include/asm/asm-prototypes.h
@@ -20,3 +20,24 @@ asmlinkage void noinstr __no_stack_protector ret_from_kernel_thread(struct task_
 								    struct pt_regs *regs,
 								    int (*fn)(void *),
 								    void *fn_arg);
+
+struct kvm_run;
+struct kvm_vcpu;
+
+void kvm_exc_entry(void);
+int  kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu);
+
+struct loongarch_fpu;
+
+#ifdef CONFIG_CPU_HAS_LSX
+void kvm_save_lsx(struct loongarch_fpu *fpu);
+void kvm_restore_lsx(struct loongarch_fpu *fpu);
+#endif
+
+#ifdef CONFIG_CPU_HAS_LASX
+void kvm_save_lasx(struct loongarch_fpu *fpu);
+void kvm_restore_lasx(struct loongarch_fpu *fpu);
+#endif
+
+void kvm_save_fpu(struct loongarch_fpu *fpu);
+void kvm_restore_fpu(struct loongarch_fpu *fpu);
diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h
index e4fe5b8e8149..1a1be10e3803 100644
--- a/arch/loongarch/include/asm/kvm_host.h
+++ b/arch/loongarch/include/asm/kvm_host.h
@@ -85,7 +85,6 @@ struct kvm_context {
 struct kvm_world_switch {
 	int (*exc_entry)(void);
 	int (*enter_guest)(struct kvm_run *run, struct kvm_vcpu *vcpu);
-	unsigned long page_order;
 };
 
 #define MAX_PGTABLE_LEVELS	4
@@ -347,8 +346,6 @@ void kvm_exc_entry(void);
 int  kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu);
 
 extern unsigned long vpid_mask;
-extern const unsigned long kvm_exception_size;
-extern const unsigned long kvm_enter_guest_size;
 extern struct kvm_world_switch *kvm_loongarch_ops;
 
 #define SW_GCSR		(1 << 0)
diff --git a/arch/loongarch/kvm/Makefile b/arch/loongarch/kvm/Makefile
index cb41d9265662..fe665054f824 100644
--- a/arch/loongarch/kvm/Makefile
+++ b/arch/loongarch/kvm/Makefile
@@ -11,7 +11,7 @@ kvm-y += exit.o
 kvm-y += interrupt.o
 kvm-y += main.o
 kvm-y += mmu.o
-kvm-y += switch.o
+obj-y += switch.o
 kvm-y += timer.o
 kvm-y += tlb.o
 kvm-y += vcpu.o
diff --git a/arch/loongarch/kvm/main.c b/arch/loongarch/kvm/main.c
index 80ea63d465b8..67d234540ed4 100644
--- a/arch/loongarch/kvm/main.c
+++ b/arch/loongarch/kvm/main.c
@@ -340,8 +340,7 @@ void kvm_arch_disable_virtualization_cpu(void)
 
 static int kvm_loongarch_env_init(void)
 {
-	int cpu, order, ret;
-	void *addr;
+	int cpu, ret;
 	struct kvm_context *context;
 
 	vmcs = alloc_percpu(struct kvm_context);
@@ -357,30 +356,8 @@ static int kvm_loongarch_env_init(void)
 		return -ENOMEM;
 	}
 
-	/*
-	 * PGD register is shared between root kernel and kvm hypervisor.
-	 * So world switch entry should be in DMW area rather than TLB area
-	 * to avoid page fault reenter.
-	 *
-	 * In future if hardware pagetable walking is supported, we won't
-	 * need to copy world switch code to DMW area.
-	 */
-	order = get_order(kvm_exception_size + kvm_enter_guest_size);
-	addr = (void *)__get_free_pages(GFP_KERNEL, order);
-	if (!addr) {
-		free_percpu(vmcs);
-		vmcs = NULL;
-		kfree(kvm_loongarch_ops);
-		kvm_loongarch_ops = NULL;
-		return -ENOMEM;
-	}
-
-	memcpy(addr, kvm_exc_entry, kvm_exception_size);
-	memcpy(addr + kvm_exception_size, kvm_enter_guest, kvm_enter_guest_size);
-	flush_icache_range((unsigned long)addr, (unsigned long)addr + kvm_exception_size + kvm_enter_guest_size);
-	kvm_loongarch_ops->exc_entry = addr;
-	kvm_loongarch_ops->enter_guest = addr + kvm_exception_size;
-	kvm_loongarch_ops->page_order = order;
+	kvm_loongarch_ops->exc_entry = (void *)kvm_exc_entry;
+	kvm_loongarch_ops->enter_guest = (void *)kvm_enter_guest;
 
 	vpid_mask = read_csr_gstat();
 	vpid_mask = (vpid_mask & CSR_GSTAT_GIDBIT) >> CSR_GSTAT_GIDBIT_SHIFT;
@@ -414,16 +391,10 @@ static int kvm_loongarch_env_init(void)
 
 static void kvm_loongarch_env_exit(void)
 {
-	unsigned long addr;
-
 	if (vmcs)
 		free_percpu(vmcs);
 
 	if (kvm_loongarch_ops) {
-		if (kvm_loongarch_ops->exc_entry) {
-			addr = (unsigned long)kvm_loongarch_ops->exc_entry;
-			free_pages(addr, kvm_loongarch_ops->page_order);
-		}
 		kfree(kvm_loongarch_ops);
 	}
 
diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
index f1768b7a6194..93845ce53651 100644
--- a/arch/loongarch/kvm/switch.S
+++ b/arch/loongarch/kvm/switch.S
@@ -5,6 +5,7 @@
 
 #include <linux/linkage.h>
 #include <asm/asm.h>
+#include <asm/page.h>
 #include <asm/asmmacro.h>
 #include <asm/loongarch.h>
 #include <asm/regdef.h>
@@ -100,10 +101,18 @@
 	 *  -        is still in guest mode, such as pgd table/vmid registers etc,
 	 *  -        will fix with hw page walk enabled in future
 	 * load kvm_vcpu from reserved CSR KVM_VCPU_KS, and save a2 to KVM_TEMP_KS
+	 *
+	 * PGD register is shared between root kernel and kvm hypervisor.
+	 * So world switch entry should be in DMW area rather than TLB area
+	 * to avoid page fault reenter.
+	 *
+	 * In future if hardware pagetable walking is supported, we won't
+	 * need to copy world switch code to DMW area.
 	 */
 	.text
 	.cfi_sections	.debug_frame
 SYM_CODE_START(kvm_exc_entry)
+	.p2align PAGE_SHIFT
 	UNWIND_HINT_UNDEFINED
 	csrwr	a2,   KVM_TEMP_KS
 	csrrd	a2,   KVM_VCPU_KS
@@ -190,8 +199,8 @@ ret_to_host:
 	kvm_restore_host_gpr    a2
 	jr      ra
 
-SYM_INNER_LABEL(kvm_exc_entry_end, SYM_L_LOCAL)
 SYM_CODE_END(kvm_exc_entry)
+EXPORT_SYMBOL(kvm_exc_entry)
 
 /*
  * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu)
@@ -215,8 +224,8 @@ SYM_FUNC_START(kvm_enter_guest)
 	/* Save kvm_vcpu to kscratch */
 	csrwr	a1, KVM_VCPU_KS
 	kvm_switch_to_guest
-SYM_INNER_LABEL(kvm_enter_guest_end, SYM_L_LOCAL)
 SYM_FUNC_END(kvm_enter_guest)
+EXPORT_SYMBOL(kvm_enter_guest)
 
 SYM_FUNC_START(kvm_save_fpu)
 	fpu_save_csr	a0 t1
@@ -224,6 +233,7 @@ SYM_FUNC_START(kvm_save_fpu)
 	fpu_save_cc	a0 t1 t2
 	jr              ra
 SYM_FUNC_END(kvm_save_fpu)
+EXPORT_SYMBOL(kvm_save_fpu)
 
 SYM_FUNC_START(kvm_restore_fpu)
 	fpu_restore_double a0 t1
@@ -231,6 +241,7 @@ SYM_FUNC_START(kvm_restore_fpu)
 	fpu_restore_cc	   a0 t1 t2
 	jr                 ra
 SYM_FUNC_END(kvm_restore_fpu)
+EXPORT_SYMBOL(kvm_restore_fpu)
 
 #ifdef CONFIG_CPU_HAS_LSX
 SYM_FUNC_START(kvm_save_lsx)
@@ -239,6 +250,7 @@ SYM_FUNC_START(kvm_save_lsx)
 	lsx_save_data   a0 t1
 	jr              ra
 SYM_FUNC_END(kvm_save_lsx)
+EXPORT_SYMBOL(kvm_save_lsx)
 
 SYM_FUNC_START(kvm_restore_lsx)
 	lsx_restore_data a0 t1
@@ -246,6 +258,7 @@ SYM_FUNC_START(kvm_restore_lsx)
 	fpu_restore_csr  a0 t1 t2
 	jr               ra
 SYM_FUNC_END(kvm_restore_lsx)
+EXPORT_SYMBOL(kvm_restore_lsx)
 #endif
 
 #ifdef CONFIG_CPU_HAS_LASX
@@ -255,6 +268,7 @@ SYM_FUNC_START(kvm_save_lasx)
 	lasx_save_data  a0 t1
 	jr              ra
 SYM_FUNC_END(kvm_save_lasx)
+EXPORT_SYMBOL(kvm_save_lasx)
 
 SYM_FUNC_START(kvm_restore_lasx)
 	lasx_restore_data a0 t1
@@ -262,10 +276,8 @@ SYM_FUNC_START(kvm_restore_lasx)
 	fpu_restore_csr   a0 t1 t2
 	jr                ra
 SYM_FUNC_END(kvm_restore_lasx)
+EXPORT_SYMBOL(kvm_restore_lasx)
 #endif
-	.section ".rodata"
-SYM_DATA(kvm_exception_size, .quad kvm_exc_entry_end - kvm_exc_entry)
-SYM_DATA(kvm_enter_guest_size, .quad kvm_enter_guest_end - kvm_enter_guest)
 
 #ifdef CONFIG_CPU_HAS_LBT
 STACK_FRAME_NON_STANDARD kvm_restore_fpu
-- 
2.39.1

[PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[Xianglai Li]

Insert the appropriate UNWIND macro definition into the kvm_exc_entry in
the assembly function to guide the generation of correct ORC table entries,
thereby solving the timeout problem of loading the livepatch-sample module
on a physical machine running multiple vcpus virtual machines.

While solving the above problems, we have gained an additional benefit,
that is, we can obtain more call stack information

Stack information that can be obtained before the problem is fixed:
[<0>] kvm_vcpu_block+0x88/0x120 [kvm]
[<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
[<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
[<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
[<0>] kvm_handle_exit+0x160/0x270 [kvm]
[<0>] kvm_exc_entry+0x100/0x1e0

Stack information that can be obtained after the problem is fixed:
[<0>] kvm_vcpu_block+0x88/0x120 [kvm]
[<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
[<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
[<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
[<0>] kvm_handle_exit+0x160/0x270 [kvm]
[<0>] kvm_exc_entry+0x104/0x1e4
[<0>] kvm_enter_guest+0x38/0x11c
[<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
[<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
[<0>] sys_ioctl+0x498/0xf00
[<0>] do_syscall+0x98/0x1d0
[<0>] handle_syscall+0xb8/0x158

Cc: stable@vger.kernel.org
Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
---
Cc: Huacai Chen <chenhuacai@kernel.org>
Cc: WANG Xuerui <kernel@xen0n.name>
Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
Cc: Bibo Mao <maobibo@loongson.cn>
Cc: Charlie Jenkins <charlie@rivosinc.com>
Cc: Xianglai Li <lixianglai@loongson.cn>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Tiezhu Yang <yangtiezhu@loongson.cn>

 arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
index 93845ce53651..a3ea9567dbe5 100644
--- a/arch/loongarch/kvm/switch.S
+++ b/arch/loongarch/kvm/switch.S
@@ -10,6 +10,7 @@
 #include <asm/loongarch.h>
 #include <asm/regdef.h>
 #include <asm/unwind_hints.h>
+#include <linux/kvm_types.h>
 
 #define HGPR_OFFSET(x)		(PT_R0 + 8*x)
 #define GGPR_OFFSET(x)		(KVM_ARCH_GGPR + 8*x)
@@ -110,9 +111,9 @@
 	 * need to copy world switch code to DMW area.
 	 */
 	.text
+	.p2align PAGE_SHIFT
 	.cfi_sections	.debug_frame
 SYM_CODE_START(kvm_exc_entry)
-	.p2align PAGE_SHIFT
 	UNWIND_HINT_UNDEFINED
 	csrwr	a2,   KVM_TEMP_KS
 	csrrd	a2,   KVM_VCPU_KS
@@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
 	/* restore per cpu register */
 	ld.d	u0, a2, KVM_ARCH_HPERCPU
 	addi.d	sp, sp, -PT_SIZE
+	UNWIND_HINT_REGS
 
 	/* Prepare handle exception */
 	or	a0, s0, zero
@@ -200,7 +202,7 @@ ret_to_host:
 	jr      ra
 
 SYM_CODE_END(kvm_exc_entry)
-EXPORT_SYMBOL(kvm_exc_entry)
+EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
 
 /*
  * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu)
@@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
 	/* Save host GPRs */
 	kvm_save_host_gpr a2
 
+	/*
+	 * The csr_era member variable of the pt_regs structure is required
+	 * for unwinding orc to perform stack traceback, so we need to put
+	 * pc into csr_era member variable here.
+	 */
+	pcaddi	t0, 0
+	st.d	t0, a2, PT_ERA
+
 	addi.d	a2, a1, KVM_VCPU_ARCH
 	st.d	sp, a2, KVM_ARCH_HSP
 	st.d	tp, a2, KVM_ARCH_HTP
@@ -225,7 +235,7 @@ SYM_FUNC_START(kvm_enter_guest)
 	csrwr	a1, KVM_VCPU_KS
 	kvm_switch_to_guest
 SYM_FUNC_END(kvm_enter_guest)
-EXPORT_SYMBOL(kvm_enter_guest)
+EXPORT_SYMBOL_FOR_KVM(kvm_enter_guest)
 
 SYM_FUNC_START(kvm_save_fpu)
 	fpu_save_csr	a0 t1
@@ -233,7 +243,7 @@ SYM_FUNC_START(kvm_save_fpu)
 	fpu_save_cc	a0 t1 t2
 	jr              ra
 SYM_FUNC_END(kvm_save_fpu)
-EXPORT_SYMBOL(kvm_save_fpu)
+EXPORT_SYMBOL_FOR_KVM(kvm_save_fpu)
 
 SYM_FUNC_START(kvm_restore_fpu)
 	fpu_restore_double a0 t1
@@ -241,7 +251,7 @@ SYM_FUNC_START(kvm_restore_fpu)
 	fpu_restore_cc	   a0 t1 t2
 	jr                 ra
 SYM_FUNC_END(kvm_restore_fpu)
-EXPORT_SYMBOL(kvm_restore_fpu)
+EXPORT_SYMBOL_FOR_KVM(kvm_restore_fpu)
 
 #ifdef CONFIG_CPU_HAS_LSX
 SYM_FUNC_START(kvm_save_lsx)
@@ -250,7 +260,7 @@ SYM_FUNC_START(kvm_save_lsx)
 	lsx_save_data   a0 t1
 	jr              ra
 SYM_FUNC_END(kvm_save_lsx)
-EXPORT_SYMBOL(kvm_save_lsx)
+EXPORT_SYMBOL_FOR_KVM(kvm_save_lsx)
 
 SYM_FUNC_START(kvm_restore_lsx)
 	lsx_restore_data a0 t1
@@ -258,7 +268,7 @@ SYM_FUNC_START(kvm_restore_lsx)
 	fpu_restore_csr  a0 t1 t2
 	jr               ra
 SYM_FUNC_END(kvm_restore_lsx)
-EXPORT_SYMBOL(kvm_restore_lsx)
+EXPORT_SYMBOL_FOR_KVM(kvm_restore_lsx)
 #endif
 
 #ifdef CONFIG_CPU_HAS_LASX
@@ -268,7 +278,7 @@ SYM_FUNC_START(kvm_save_lasx)
 	lasx_save_data  a0 t1
 	jr              ra
 SYM_FUNC_END(kvm_save_lasx)
-EXPORT_SYMBOL(kvm_save_lasx)
+EXPORT_SYMBOL_FOR_KVM(kvm_save_lasx)
 
 SYM_FUNC_START(kvm_restore_lasx)
 	lasx_restore_data a0 t1
@@ -276,7 +286,7 @@ SYM_FUNC_START(kvm_restore_lasx)
 	fpu_restore_csr   a0 t1 t2
 	jr                ra
 SYM_FUNC_END(kvm_restore_lasx)
-EXPORT_SYMBOL(kvm_restore_lasx)
+EXPORT_SYMBOL_FOR_KVM(kvm_restore_lasx)
 #endif
 
 #ifdef CONFIG_CPU_HAS_LBT
-- 
2.39.1

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[Huacai Chen]

Hi, Xianglai,

On Sat, Dec 27, 2025 at 9:52 AM Xianglai Li <lixianglai@loongson.cn> wrote:
>
> Insert the appropriate UNWIND macro definition into the kvm_exc_entry in
> the assembly function to guide the generation of correct ORC table entries,
> thereby solving the timeout problem of loading the livepatch-sample module
> on a physical machine running multiple vcpus virtual machines.
>
> While solving the above problems, we have gained an additional benefit,
> that is, we can obtain more call stack information
>
> Stack information that can be obtained before the problem is fixed:
> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
> [<0>] kvm_exc_entry+0x100/0x1e0
>
> Stack information that can be obtained after the problem is fixed:
> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
> [<0>] kvm_exc_entry+0x104/0x1e4
> [<0>] kvm_enter_guest+0x38/0x11c
> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
> [<0>] sys_ioctl+0x498/0xf00
> [<0>] do_syscall+0x98/0x1d0
> [<0>] handle_syscall+0xb8/0x158
>
> Cc: stable@vger.kernel.org
> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
> ---
> Cc: Huacai Chen <chenhuacai@kernel.org>
> Cc: WANG Xuerui <kernel@xen0n.name>
> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
> Cc: Bibo Mao <maobibo@loongson.cn>
> Cc: Charlie Jenkins <charlie@rivosinc.com>
> Cc: Xianglai Li <lixianglai@loongson.cn>
> Cc: Thomas Gleixner <tglx@linutronix.de>
> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
>
>  arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
>  1 file changed, 19 insertions(+), 9 deletions(-)
>
> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
> index 93845ce53651..a3ea9567dbe5 100644
> --- a/arch/loongarch/kvm/switch.S
> +++ b/arch/loongarch/kvm/switch.S
> @@ -10,6 +10,7 @@
>  #include <asm/loongarch.h>
>  #include <asm/regdef.h>
>  #include <asm/unwind_hints.h>
> +#include <linux/kvm_types.h>
>
>  #define HGPR_OFFSET(x)         (PT_R0 + 8*x)
>  #define GGPR_OFFSET(x)         (KVM_ARCH_GGPR + 8*x)
> @@ -110,9 +111,9 @@
>          * need to copy world switch code to DMW area.
>          */
>         .text
> +       .p2align PAGE_SHIFT
>         .cfi_sections   .debug_frame
>  SYM_CODE_START(kvm_exc_entry)
> -       .p2align PAGE_SHIFT
>         UNWIND_HINT_UNDEFINED
>         csrwr   a2,   KVM_TEMP_KS
>         csrrd   a2,   KVM_VCPU_KS
> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
>         /* restore per cpu register */
>         ld.d    u0, a2, KVM_ARCH_HPERCPU
>         addi.d  sp, sp, -PT_SIZE
> +       UNWIND_HINT_REGS
>
>         /* Prepare handle exception */
>         or      a0, s0, zero
> @@ -200,7 +202,7 @@ ret_to_host:
>         jr      ra
>
>  SYM_CODE_END(kvm_exc_entry)
> -EXPORT_SYMBOL(kvm_exc_entry)
> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
Why not use EXPORT_SYMBOL_FOR_KVM in the first patch directly?

>
>  /*
>   * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu)
> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
>         /* Save host GPRs */
>         kvm_save_host_gpr a2
>
> +       /*
> +        * The csr_era member variable of the pt_regs structure is required
> +        * for unwinding orc to perform stack traceback, so we need to put
> +        * pc into csr_era member variable here.
> +        */
> +       pcaddi  t0, 0
> +       st.d    t0, a2, PT_ERA
I am still confused here, does this overwrite PT_ERA stored by
kvm_save_host_gpr?

Huacai

> +
>         addi.d  a2, a1, KVM_VCPU_ARCH
>         st.d    sp, a2, KVM_ARCH_HSP
>         st.d    tp, a2, KVM_ARCH_HTP
> @@ -225,7 +235,7 @@ SYM_FUNC_START(kvm_enter_guest)
>         csrwr   a1, KVM_VCPU_KS
>         kvm_switch_to_guest
>  SYM_FUNC_END(kvm_enter_guest)
> -EXPORT_SYMBOL(kvm_enter_guest)
> +EXPORT_SYMBOL_FOR_KVM(kvm_enter_guest)
>
>  SYM_FUNC_START(kvm_save_fpu)
>         fpu_save_csr    a0 t1
> @@ -233,7 +243,7 @@ SYM_FUNC_START(kvm_save_fpu)
>         fpu_save_cc     a0 t1 t2
>         jr              ra
>  SYM_FUNC_END(kvm_save_fpu)
> -EXPORT_SYMBOL(kvm_save_fpu)
> +EXPORT_SYMBOL_FOR_KVM(kvm_save_fpu)
>
>  SYM_FUNC_START(kvm_restore_fpu)
>         fpu_restore_double a0 t1
> @@ -241,7 +251,7 @@ SYM_FUNC_START(kvm_restore_fpu)
>         fpu_restore_cc     a0 t1 t2
>         jr                 ra
>  SYM_FUNC_END(kvm_restore_fpu)
> -EXPORT_SYMBOL(kvm_restore_fpu)
> +EXPORT_SYMBOL_FOR_KVM(kvm_restore_fpu)
>
>  #ifdef CONFIG_CPU_HAS_LSX
>  SYM_FUNC_START(kvm_save_lsx)
> @@ -250,7 +260,7 @@ SYM_FUNC_START(kvm_save_lsx)
>         lsx_save_data   a0 t1
>         jr              ra
>  SYM_FUNC_END(kvm_save_lsx)
> -EXPORT_SYMBOL(kvm_save_lsx)
> +EXPORT_SYMBOL_FOR_KVM(kvm_save_lsx)
>
>  SYM_FUNC_START(kvm_restore_lsx)
>         lsx_restore_data a0 t1
> @@ -258,7 +268,7 @@ SYM_FUNC_START(kvm_restore_lsx)
>         fpu_restore_csr  a0 t1 t2
>         jr               ra
>  SYM_FUNC_END(kvm_restore_lsx)
> -EXPORT_SYMBOL(kvm_restore_lsx)
> +EXPORT_SYMBOL_FOR_KVM(kvm_restore_lsx)
>  #endif
>
>  #ifdef CONFIG_CPU_HAS_LASX
> @@ -268,7 +278,7 @@ SYM_FUNC_START(kvm_save_lasx)
>         lasx_save_data  a0 t1
>         jr              ra
>  SYM_FUNC_END(kvm_save_lasx)
> -EXPORT_SYMBOL(kvm_save_lasx)
> +EXPORT_SYMBOL_FOR_KVM(kvm_save_lasx)
>
>  SYM_FUNC_START(kvm_restore_lasx)
>         lasx_restore_data a0 t1
> @@ -276,7 +286,7 @@ SYM_FUNC_START(kvm_restore_lasx)
>         fpu_restore_csr   a0 t1 t2
>         jr                ra
>  SYM_FUNC_END(kvm_restore_lasx)
> -EXPORT_SYMBOL(kvm_restore_lasx)
> +EXPORT_SYMBOL_FOR_KVM(kvm_restore_lasx)
>  #endif
>
>  #ifdef CONFIG_CPU_HAS_LBT
> --
> 2.39.1
>

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[lixianglai]

Hi Huacai:
> Hi, Xianglai,
>
> On Sat, Dec 27, 2025 at 9:52 AM Xianglai Li <lixianglai@loongson.cn> wrote:
>> Insert the appropriate UNWIND macro definition into the kvm_exc_entry in
>> the assembly function to guide the generation of correct ORC table entries,
>> thereby solving the timeout problem of loading the livepatch-sample module
>> on a physical machine running multiple vcpus virtual machines.
>>
>> While solving the above problems, we have gained an additional benefit,
>> that is, we can obtain more call stack information
>>
>> Stack information that can be obtained before the problem is fixed:
>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>> [<0>] kvm_exc_entry+0x100/0x1e0
>>
>> Stack information that can be obtained after the problem is fixed:
>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>> [<0>] kvm_exc_entry+0x104/0x1e4
>> [<0>] kvm_enter_guest+0x38/0x11c
>> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>> [<0>] sys_ioctl+0x498/0xf00
>> [<0>] do_syscall+0x98/0x1d0
>> [<0>] handle_syscall+0xb8/0x158
>>
>> Cc: stable@vger.kernel.org
>> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
>> ---
>> Cc: Huacai Chen <chenhuacai@kernel.org>
>> Cc: WANG Xuerui <kernel@xen0n.name>
>> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
>> Cc: Bibo Mao <maobibo@loongson.cn>
>> Cc: Charlie Jenkins <charlie@rivosinc.com>
>> Cc: Xianglai Li <lixianglai@loongson.cn>
>> Cc: Thomas Gleixner <tglx@linutronix.de>
>> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
>>
>>   arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
>>   1 file changed, 19 insertions(+), 9 deletions(-)
>>
>> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
>> index 93845ce53651..a3ea9567dbe5 100644
>> --- a/arch/loongarch/kvm/switch.S
>> +++ b/arch/loongarch/kvm/switch.S
>> @@ -10,6 +10,7 @@
>>   #include <asm/loongarch.h>
>>   #include <asm/regdef.h>
>>   #include <asm/unwind_hints.h>
>> +#include <linux/kvm_types.h>
>>
>>   #define HGPR_OFFSET(x)         (PT_R0 + 8*x)
>>   #define GGPR_OFFSET(x)         (KVM_ARCH_GGPR + 8*x)
>> @@ -110,9 +111,9 @@
>>           * need to copy world switch code to DMW area.
>>           */
>>          .text
>> +       .p2align PAGE_SHIFT
>>          .cfi_sections   .debug_frame
>>   SYM_CODE_START(kvm_exc_entry)
>> -       .p2align PAGE_SHIFT
>>          UNWIND_HINT_UNDEFINED
>>          csrwr   a2,   KVM_TEMP_KS
>>          csrrd   a2,   KVM_VCPU_KS
>> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
>>          /* restore per cpu register */
>>          ld.d    u0, a2, KVM_ARCH_HPERCPU
>>          addi.d  sp, sp, -PT_SIZE
>> +       UNWIND_HINT_REGS
>>
>>          /* Prepare handle exception */
>>          or      a0, s0, zero
>> @@ -200,7 +202,7 @@ ret_to_host:
>>          jr      ra
>>
>>   SYM_CODE_END(kvm_exc_entry)
>> -EXPORT_SYMBOL(kvm_exc_entry)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
> Why not use EXPORT_SYMBOL_FOR_KVM in the first patch directly?
Ok,I will put it in the first patch.

>>   /*
>>    * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu)
>> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
>>          /* Save host GPRs */
>>          kvm_save_host_gpr a2
>>
>> +       /*
>> +        * The csr_era member variable of the pt_regs structure is required
>> +        * for unwinding orc to perform stack traceback, so we need to put
>> +        * pc into csr_era member variable here.
>> +        */
>> +       pcaddi  t0, 0
>> +       st.d    t0, a2, PT_ERA
> I am still confused here, does this overwrite PT_ERA stored by
> kvm_save_host_gpr?

No, kvm_save_host_gpr does not overwrite PT_ERA. In fact,
  it is precisely because PT_ERA is not assigned a value that we need to 
add these lines of statements here.

Now the a2 register points to the structure pt_regs. Here, we are 
assigning a value to the pt_regs.csr_era member variable.
Then the function unwind_next_frame uses the pt_regs.csr_era member 
variable as a pc to do the stack traceback.

Thanks,
Xianglai.

> Huacai
>
>> +
>>          addi.d  a2, a1, KVM_VCPU_ARCH
>>          st.d    sp, a2, KVM_ARCH_HSP
>>          st.d    tp, a2, KVM_ARCH_HTP
>> @@ -225,7 +235,7 @@ SYM_FUNC_START(kvm_enter_guest)
>>          csrwr   a1, KVM_VCPU_KS
>>          kvm_switch_to_guest
>>   SYM_FUNC_END(kvm_enter_guest)
>> -EXPORT_SYMBOL(kvm_enter_guest)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_enter_guest)
>>
>>   SYM_FUNC_START(kvm_save_fpu)
>>          fpu_save_csr    a0 t1
>> @@ -233,7 +243,7 @@ SYM_FUNC_START(kvm_save_fpu)
>>          fpu_save_cc     a0 t1 t2
>>          jr              ra
>>   SYM_FUNC_END(kvm_save_fpu)
>> -EXPORT_SYMBOL(kvm_save_fpu)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_save_fpu)
>>
>>   SYM_FUNC_START(kvm_restore_fpu)
>>          fpu_restore_double a0 t1
>> @@ -241,7 +251,7 @@ SYM_FUNC_START(kvm_restore_fpu)
>>          fpu_restore_cc     a0 t1 t2
>>          jr                 ra
>>   SYM_FUNC_END(kvm_restore_fpu)
>> -EXPORT_SYMBOL(kvm_restore_fpu)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_restore_fpu)
>>
>>   #ifdef CONFIG_CPU_HAS_LSX
>>   SYM_FUNC_START(kvm_save_lsx)
>> @@ -250,7 +260,7 @@ SYM_FUNC_START(kvm_save_lsx)
>>          lsx_save_data   a0 t1
>>          jr              ra
>>   SYM_FUNC_END(kvm_save_lsx)
>> -EXPORT_SYMBOL(kvm_save_lsx)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_save_lsx)
>>
>>   SYM_FUNC_START(kvm_restore_lsx)
>>          lsx_restore_data a0 t1
>> @@ -258,7 +268,7 @@ SYM_FUNC_START(kvm_restore_lsx)
>>          fpu_restore_csr  a0 t1 t2
>>          jr               ra
>>   SYM_FUNC_END(kvm_restore_lsx)
>> -EXPORT_SYMBOL(kvm_restore_lsx)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_restore_lsx)
>>   #endif
>>
>>   #ifdef CONFIG_CPU_HAS_LASX
>> @@ -268,7 +278,7 @@ SYM_FUNC_START(kvm_save_lasx)
>>          lasx_save_data  a0 t1
>>          jr              ra
>>   SYM_FUNC_END(kvm_save_lasx)
>> -EXPORT_SYMBOL(kvm_save_lasx)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_save_lasx)
>>
>>   SYM_FUNC_START(kvm_restore_lasx)
>>          lasx_restore_data a0 t1
>> @@ -276,7 +286,7 @@ SYM_FUNC_START(kvm_restore_lasx)
>>          fpu_restore_csr   a0 t1 t2
>>          jr                ra
>>   SYM_FUNC_END(kvm_restore_lasx)
>> -EXPORT_SYMBOL(kvm_restore_lasx)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_restore_lasx)
>>   #endif
>>
>>   #ifdef CONFIG_CPU_HAS_LBT
>> --
>> 2.39.1
>>

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[Jinyang He]

On 2025-12-27 09:27, Xianglai Li wrote:

> Insert the appropriate UNWIND macro definition into the kvm_exc_entry in
> the assembly function to guide the generation of correct ORC table entries,
> thereby solving the timeout problem of loading the livepatch-sample module
> on a physical machine running multiple vcpus virtual machines.
>
> While solving the above problems, we have gained an additional benefit,
> that is, we can obtain more call stack information
>
> Stack information that can be obtained before the problem is fixed:
> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
> [<0>] kvm_exc_entry+0x100/0x1e0
>
> Stack information that can be obtained after the problem is fixed:
> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
> [<0>] kvm_exc_entry+0x104/0x1e4
> [<0>] kvm_enter_guest+0x38/0x11c
> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
> [<0>] sys_ioctl+0x498/0xf00
> [<0>] do_syscall+0x98/0x1d0
> [<0>] handle_syscall+0xb8/0x158
>
> Cc: stable@vger.kernel.org
> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
> ---
> Cc: Huacai Chen <chenhuacai@kernel.org>
> Cc: WANG Xuerui <kernel@xen0n.name>
> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
> Cc: Bibo Mao <maobibo@loongson.cn>
> Cc: Charlie Jenkins <charlie@rivosinc.com>
> Cc: Xianglai Li <lixianglai@loongson.cn>
> Cc: Thomas Gleixner <tglx@linutronix.de>
> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
>
>   arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
>   1 file changed, 19 insertions(+), 9 deletions(-)
>
> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
> index 93845ce53651..a3ea9567dbe5 100644
> --- a/arch/loongarch/kvm/switch.S
> +++ b/arch/loongarch/kvm/switch.S
> @@ -10,6 +10,7 @@
>   #include <asm/loongarch.h>
>   #include <asm/regdef.h>
>   #include <asm/unwind_hints.h>
> +#include <linux/kvm_types.h>
>   
>   #define HGPR_OFFSET(x)		(PT_R0 + 8*x)
>   #define GGPR_OFFSET(x)		(KVM_ARCH_GGPR + 8*x)
> @@ -110,9 +111,9 @@
>   	 * need to copy world switch code to DMW area.
>   	 */
>   	.text
> +	.p2align PAGE_SHIFT
>   	.cfi_sections	.debug_frame
>   SYM_CODE_START(kvm_exc_entry)
> -	.p2align PAGE_SHIFT
>   	UNWIND_HINT_UNDEFINED
>   	csrwr	a2,   KVM_TEMP_KS
>   	csrrd	a2,   KVM_VCPU_KS
> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
>   	/* restore per cpu register */
>   	ld.d	u0, a2, KVM_ARCH_HPERCPU
>   	addi.d	sp, sp, -PT_SIZE
> +	UNWIND_HINT_REGS
>   
>   	/* Prepare handle exception */
>   	or	a0, s0, zero
> @@ -200,7 +202,7 @@ ret_to_host:
>   	jr      ra
>   
>   SYM_CODE_END(kvm_exc_entry)
> -EXPORT_SYMBOL(kvm_exc_entry)
> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
>   
>   /*
>    * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu)
> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
>   	/* Save host GPRs */
>   	kvm_save_host_gpr a2
>   
> +	/*
> +	 * The csr_era member variable of the pt_regs structure is required
> +	 * for unwinding orc to perform stack traceback, so we need to put
> +	 * pc into csr_era member variable here.
> +	 */
> +	pcaddi	t0, 0
> +	st.d	t0, a2, PT_ERA
Hi, Xianglai,

It should use `SYM_CODE_START` to mark the `kvm_enter_guest` rather than
`SYM_FUNC_START`, since the `SYM_FUNC_START` is used to mark "C-likely"
asm functionw. I guess the kvm_enter_guest is something like exception
handler becuase the last instruction is "ertn". So usually it should
mark UNWIND_HINT_REGS where can find last frame info by "$sp".
However, all info is store to "$a2", this mark should be
   `UNWIND_HINT sp_reg=ORC_REG_A2(???) type=UNWIND_HINT_TYPE_REGS`.
I don't konw why save this function internal PC here by `pcaddi t0, 0`,
and I think it is no meaning(, for exception handler, they save last PC
by read CSR.ERA). The `kvm_enter_guest` saves registers by
"$a2"("$sp" - PT_REGS) beyond stack ("$sp"), it is dangerous if IE
is enable. So I wonder if there is really a stacktrace through this 
function?

Jinyang


> +
>   	addi.d	a2, a1, KVM_VCPU_ARCH
>   	st.d	sp, a2, KVM_ARCH_HSP
>   	st.d	tp, a2, KVM_ARCH_HTP
> @@ -225,7 +235,7 @@ SYM_FUNC_START(kvm_enter_guest)
>   	csrwr	a1, KVM_VCPU_KS
>   	kvm_switch_to_guest
>   SYM_FUNC_END(kvm_enter_guest)
> -EXPORT_SYMBOL(kvm_enter_guest)
> +EXPORT_SYMBOL_FOR_KVM(kvm_enter_guest)
>   
>   SYM_FUNC_START(kvm_save_fpu)
>   	fpu_save_csr	a0 t1
> @@ -233,7 +243,7 @@ SYM_FUNC_START(kvm_save_fpu)
>   	fpu_save_cc	a0 t1 t2
>   	jr              ra
>   SYM_FUNC_END(kvm_save_fpu)
> -EXPORT_SYMBOL(kvm_save_fpu)
> +EXPORT_SYMBOL_FOR_KVM(kvm_save_fpu)
>   
>   SYM_FUNC_START(kvm_restore_fpu)
>   	fpu_restore_double a0 t1
> @@ -241,7 +251,7 @@ SYM_FUNC_START(kvm_restore_fpu)
>   	fpu_restore_cc	   a0 t1 t2
>   	jr                 ra
>   SYM_FUNC_END(kvm_restore_fpu)
> -EXPORT_SYMBOL(kvm_restore_fpu)
> +EXPORT_SYMBOL_FOR_KVM(kvm_restore_fpu)
>   
>   #ifdef CONFIG_CPU_HAS_LSX
>   SYM_FUNC_START(kvm_save_lsx)
> @@ -250,7 +260,7 @@ SYM_FUNC_START(kvm_save_lsx)
>   	lsx_save_data   a0 t1
>   	jr              ra
>   SYM_FUNC_END(kvm_save_lsx)
> -EXPORT_SYMBOL(kvm_save_lsx)
> +EXPORT_SYMBOL_FOR_KVM(kvm_save_lsx)
>   
>   SYM_FUNC_START(kvm_restore_lsx)
>   	lsx_restore_data a0 t1
> @@ -258,7 +268,7 @@ SYM_FUNC_START(kvm_restore_lsx)
>   	fpu_restore_csr  a0 t1 t2
>   	jr               ra
>   SYM_FUNC_END(kvm_restore_lsx)
> -EXPORT_SYMBOL(kvm_restore_lsx)
> +EXPORT_SYMBOL_FOR_KVM(kvm_restore_lsx)
>   #endif
>   
>   #ifdef CONFIG_CPU_HAS_LASX
> @@ -268,7 +278,7 @@ SYM_FUNC_START(kvm_save_lasx)
>   	lasx_save_data  a0 t1
>   	jr              ra
>   SYM_FUNC_END(kvm_save_lasx)
> -EXPORT_SYMBOL(kvm_save_lasx)
> +EXPORT_SYMBOL_FOR_KVM(kvm_save_lasx)
>   
>   SYM_FUNC_START(kvm_restore_lasx)
>   	lasx_restore_data a0 t1
> @@ -276,7 +286,7 @@ SYM_FUNC_START(kvm_restore_lasx)
>   	fpu_restore_csr   a0 t1 t2
>   	jr                ra
>   SYM_FUNC_END(kvm_restore_lasx)
> -EXPORT_SYMBOL(kvm_restore_lasx)
> +EXPORT_SYMBOL_FOR_KVM(kvm_restore_lasx)
>   #endif
>   
>   #ifdef CONFIG_CPU_HAS_LBT

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[lixianglai]

Hi Jinyang:
> On 2025-12-27 09:27, Xianglai Li wrote:
>
>> Insert the appropriate UNWIND macro definition into the kvm_exc_entry in
>> the assembly function to guide the generation of correct ORC table 
>> entries,
>> thereby solving the timeout problem of loading the livepatch-sample 
>> module
>> on a physical machine running multiple vcpus virtual machines.
>>
>> While solving the above problems, we have gained an additional benefit,
>> that is, we can obtain more call stack information
>>
>> Stack information that can be obtained before the problem is fixed:
>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>> [<0>] kvm_exc_entry+0x100/0x1e0
>>
>> Stack information that can be obtained after the problem is fixed:
>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>> [<0>] kvm_exc_entry+0x104/0x1e4
>> [<0>] kvm_enter_guest+0x38/0x11c
>> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>> [<0>] sys_ioctl+0x498/0xf00
>> [<0>] do_syscall+0x98/0x1d0
>> [<0>] handle_syscall+0xb8/0x158
>>
>> Cc: stable@vger.kernel.org
>> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
>> ---
>> Cc: Huacai Chen <chenhuacai@kernel.org>
>> Cc: WANG Xuerui <kernel@xen0n.name>
>> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
>> Cc: Bibo Mao <maobibo@loongson.cn>
>> Cc: Charlie Jenkins <charlie@rivosinc.com>
>> Cc: Xianglai Li <lixianglai@loongson.cn>
>> Cc: Thomas Gleixner <tglx@linutronix.de>
>> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
>>
>>   arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
>>   1 file changed, 19 insertions(+), 9 deletions(-)
>>
>> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
>> index 93845ce53651..a3ea9567dbe5 100644
>> --- a/arch/loongarch/kvm/switch.S
>> +++ b/arch/loongarch/kvm/switch.S
>> @@ -10,6 +10,7 @@
>>   #include <asm/loongarch.h>
>>   #include <asm/regdef.h>
>>   #include <asm/unwind_hints.h>
>> +#include <linux/kvm_types.h>
>>     #define HGPR_OFFSET(x)        (PT_R0 + 8*x)
>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>> @@ -110,9 +111,9 @@
>>        * need to copy world switch code to DMW area.
>>        */
>>       .text
>> +    .p2align PAGE_SHIFT
>>       .cfi_sections    .debug_frame
>>   SYM_CODE_START(kvm_exc_entry)
>> -    .p2align PAGE_SHIFT
>>       UNWIND_HINT_UNDEFINED
>>       csrwr    a2,   KVM_TEMP_KS
>>       csrrd    a2,   KVM_VCPU_KS
>> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
>>       /* restore per cpu register */
>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
>>       addi.d    sp, sp, -PT_SIZE
>> +    UNWIND_HINT_REGS
>>         /* Prepare handle exception */
>>       or    a0, s0, zero
>> @@ -200,7 +202,7 @@ ret_to_host:
>>       jr      ra
>>     SYM_CODE_END(kvm_exc_entry)
>> -EXPORT_SYMBOL(kvm_exc_entry)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
>>     /*
>>    * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu)
>> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
>>       /* Save host GPRs */
>>       kvm_save_host_gpr a2
>>   +    /*
>> +     * The csr_era member variable of the pt_regs structure is required
>> +     * for unwinding orc to perform stack traceback, so we need to put
>> +     * pc into csr_era member variable here.
>> +     */
>> +    pcaddi    t0, 0
>> +    st.d    t0, a2, PT_ERA
> Hi, Xianglai,
>
> It should use `SYM_CODE_START` to mark the `kvm_enter_guest` rather than
> `SYM_FUNC_START`, since the `SYM_FUNC_START` is used to mark "C-likely"
> asm functionw. 

Ok, I will use SYM_CODE_START to mark kvm_enter_guest in the next version.

> I guess the kvm_enter_guest is something like exception
> handler becuase the last instruction is "ertn". So usually it should
> mark UNWIND_HINT_REGS where can find last frame info by "$sp".
> However, all info is store to "$a2", this mark should be
>   `UNWIND_HINT sp_reg=ORC_REG_A2(???) type=UNWIND_HINT_TYPE_REGS`.
> I don't konw why save this function internal PC here by `pcaddi t0, 0`,
> and I think it is no meaning(, for exception handler, they save last PC
> by read CSR.ERA). The `kvm_enter_guest` saves registers by
> "$a2"("$sp" - PT_REGS) beyond stack ("$sp"), it is dangerous if IE
> is enable. So I wonder if there is really a stacktrace through this 
> function?
>
The stack backtracking issue in switch.S is rather complex because it 
involves the switching between cpu root-mode and guest-mode:
Real stack backtracking should be divided into two parts:
part 1:
     [<0>] kvm_enter_guest+0x38/0x11c
     [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
     [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
     [<0>] sys_ioctl+0x498/0xf00
     [<0>] do_syscall+0x98/0x1d0
     [<0>] handle_syscall+0xb8/0x158

part 2:
     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
     [<0>] kvm_exc_entry+0x104/0x1e4


In "part 1", after executing kvm_enter_guest, the cpu switches from 
root-mode to guest-mode.
In this case, stack backtracking is indeed very rare.

In "part 2", the cpu switches from the guest-mode to the root-mode,
and most of the stack backtracking occurs during this phase.

To obtain the longest call chain, we save pc in kvm_enter_guest to 
pt_regs.csr_era,
and after restoring the sp of the root-mode cpu in kvm_exc_entry,
The ORC entry was re-established using "UNWIND_HINT_REGS",
  and then we obtained the following stack backtrace as we wanted:

     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
     [<0>] kvm_exc_entry+0x104/0x1e4
     [<0>] kvm_enter_guest+0x38/0x11c
     [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
     [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
     [<0>] sys_ioctl+0x498/0xf00
     [<0>] do_syscall+0x98/0x1d0
     [<0>] handle_syscall+0xb8/0x158

Doing so is equivalent to ignoring the details of the cpu root-mode and 
guest-mode switching.
About what you said in the IE enable phase is dangerous,
interrupts are always off during the cpu root-mode and guest-mode 
switching in kvm_enter_guest and kvm_exc_entry.

Thanks!
Xianglai.

> Jinyang
>
>
>> +
>>       addi.d    a2, a1, KVM_VCPU_ARCH
>>       st.d    sp, a2, KVM_ARCH_HSP
>>       st.d    tp, a2, KVM_ARCH_HTP
>> @@ -225,7 +235,7 @@ SYM_FUNC_START(kvm_enter_guest)
>>       csrwr    a1, KVM_VCPU_KS
>>       kvm_switch_to_guest
>>   SYM_FUNC_END(kvm_enter_guest)
>> -EXPORT_SYMBOL(kvm_enter_guest)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_enter_guest)
>>     SYM_FUNC_START(kvm_save_fpu)
>>       fpu_save_csr    a0 t1
>> @@ -233,7 +243,7 @@ SYM_FUNC_START(kvm_save_fpu)
>>       fpu_save_cc    a0 t1 t2
>>       jr              ra
>>   SYM_FUNC_END(kvm_save_fpu)
>> -EXPORT_SYMBOL(kvm_save_fpu)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_save_fpu)
>>     SYM_FUNC_START(kvm_restore_fpu)
>>       fpu_restore_double a0 t1
>> @@ -241,7 +251,7 @@ SYM_FUNC_START(kvm_restore_fpu)
>>       fpu_restore_cc       a0 t1 t2
>>       jr                 ra
>>   SYM_FUNC_END(kvm_restore_fpu)
>> -EXPORT_SYMBOL(kvm_restore_fpu)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_restore_fpu)
>>     #ifdef CONFIG_CPU_HAS_LSX
>>   SYM_FUNC_START(kvm_save_lsx)
>> @@ -250,7 +260,7 @@ SYM_FUNC_START(kvm_save_lsx)
>>       lsx_save_data   a0 t1
>>       jr              ra
>>   SYM_FUNC_END(kvm_save_lsx)
>> -EXPORT_SYMBOL(kvm_save_lsx)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_save_lsx)
>>     SYM_FUNC_START(kvm_restore_lsx)
>>       lsx_restore_data a0 t1
>> @@ -258,7 +268,7 @@ SYM_FUNC_START(kvm_restore_lsx)
>>       fpu_restore_csr  a0 t1 t2
>>       jr               ra
>>   SYM_FUNC_END(kvm_restore_lsx)
>> -EXPORT_SYMBOL(kvm_restore_lsx)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_restore_lsx)
>>   #endif
>>     #ifdef CONFIG_CPU_HAS_LASX
>> @@ -268,7 +278,7 @@ SYM_FUNC_START(kvm_save_lasx)
>>       lasx_save_data  a0 t1
>>       jr              ra
>>   SYM_FUNC_END(kvm_save_lasx)
>> -EXPORT_SYMBOL(kvm_save_lasx)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_save_lasx)
>>     SYM_FUNC_START(kvm_restore_lasx)
>>       lasx_restore_data a0 t1
>> @@ -276,7 +286,7 @@ SYM_FUNC_START(kvm_restore_lasx)
>>       fpu_restore_csr   a0 t1 t2
>>       jr                ra
>>   SYM_FUNC_END(kvm_restore_lasx)
>> -EXPORT_SYMBOL(kvm_restore_lasx)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_restore_lasx)
>>   #endif
>>     #ifdef CONFIG_CPU_HAS_LBT

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[Jinyang He]

On 2025-12-29 11:53, lixianglai wrote:
> Hi Jinyang:
>> On 2025-12-27 09:27, Xianglai Li wrote:
>>
>>> Insert the appropriate UNWIND macro definition into the 
>>> kvm_exc_entry in
>>> the assembly function to guide the generation of correct ORC table 
>>> entries,
>>> thereby solving the timeout problem of loading the livepatch-sample 
>>> module
>>> on a physical machine running multiple vcpus virtual machines.
>>>
>>> While solving the above problems, we have gained an additional benefit,
>>> that is, we can obtain more call stack information
>>>
>>> Stack information that can be obtained before the problem is fixed:
>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>> [<0>] kvm_exc_entry+0x100/0x1e0
>>>
>>> Stack information that can be obtained after the problem is fixed:
>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>> [<0>] kvm_exc_entry+0x104/0x1e4
>>> [<0>] kvm_enter_guest+0x38/0x11c
>>> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>> [<0>] sys_ioctl+0x498/0xf00
>>> [<0>] do_syscall+0x98/0x1d0
>>> [<0>] handle_syscall+0xb8/0x158
>>>
>>> Cc: stable@vger.kernel.org
>>> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
>>> ---
>>> Cc: Huacai Chen <chenhuacai@kernel.org>
>>> Cc: WANG Xuerui <kernel@xen0n.name>
>>> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
>>> Cc: Bibo Mao <maobibo@loongson.cn>
>>> Cc: Charlie Jenkins <charlie@rivosinc.com>
>>> Cc: Xianglai Li <lixianglai@loongson.cn>
>>> Cc: Thomas Gleixner <tglx@linutronix.de>
>>> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
>>>
>>>   arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
>>>   1 file changed, 19 insertions(+), 9 deletions(-)
>>>
>>> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
>>> index 93845ce53651..a3ea9567dbe5 100644
>>> --- a/arch/loongarch/kvm/switch.S
>>> +++ b/arch/loongarch/kvm/switch.S
>>> @@ -10,6 +10,7 @@
>>>   #include <asm/loongarch.h>
>>>   #include <asm/regdef.h>
>>>   #include <asm/unwind_hints.h>
>>> +#include <linux/kvm_types.h>
>>>     #define HGPR_OFFSET(x)        (PT_R0 + 8*x)
>>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>> @@ -110,9 +111,9 @@
>>>        * need to copy world switch code to DMW area.
>>>        */
>>>       .text
>>> +    .p2align PAGE_SHIFT
>>>       .cfi_sections    .debug_frame
>>>   SYM_CODE_START(kvm_exc_entry)
>>> -    .p2align PAGE_SHIFT
>>>       UNWIND_HINT_UNDEFINED
>>>       csrwr    a2,   KVM_TEMP_KS
>>>       csrrd    a2,   KVM_VCPU_KS
>>> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
>>>       /* restore per cpu register */
>>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
>>>       addi.d    sp, sp, -PT_SIZE
>>> +    UNWIND_HINT_REGS
>>>         /* Prepare handle exception */
>>>       or    a0, s0, zero
>>> @@ -200,7 +202,7 @@ ret_to_host:
>>>       jr      ra
>>>     SYM_CODE_END(kvm_exc_entry)
>>> -EXPORT_SYMBOL(kvm_exc_entry)
>>> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
>>>     /*
>>>    * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu)
>>> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
>>>       /* Save host GPRs */
>>>       kvm_save_host_gpr a2
>>>   +    /*
>>> +     * The csr_era member variable of the pt_regs structure is 
>>> required
>>> +     * for unwinding orc to perform stack traceback, so we need to put
>>> +     * pc into csr_era member variable here.
>>> +     */
>>> +    pcaddi    t0, 0
>>> +    st.d    t0, a2, PT_ERA
>> Hi, Xianglai,
>>
>> It should use `SYM_CODE_START` to mark the `kvm_enter_guest` rather than
>> `SYM_FUNC_START`, since the `SYM_FUNC_START` is used to mark "C-likely"
>> asm functionw. 
>
> Ok, I will use SYM_CODE_START to mark kvm_enter_guest in the next 
> version.
>
>> I guess the kvm_enter_guest is something like exception
>> handler becuase the last instruction is "ertn". So usually it should
>> mark UNWIND_HINT_REGS where can find last frame info by "$sp".
>> However, all info is store to "$a2", this mark should be
>>   `UNWIND_HINT sp_reg=ORC_REG_A2(???) type=UNWIND_HINT_TYPE_REGS`.
>> I don't konw why save this function internal PC here by `pcaddi t0, 0`,
>> and I think it is no meaning(, for exception handler, they save last PC
>> by read CSR.ERA). The `kvm_enter_guest` saves registers by
>> "$a2"("$sp" - PT_REGS) beyond stack ("$sp"), it is dangerous if IE
>> is enable. So I wonder if there is really a stacktrace through this 
>> function?
>>
> The stack backtracking issue in switch.S is rather complex because it 
> involves the switching between cpu root-mode and guest-mode:
> Real stack backtracking should be divided into two parts:
> part 1:
>     [<0>] kvm_enter_guest+0x38/0x11c
>     [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>     [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>     [<0>] sys_ioctl+0x498/0xf00
>     [<0>] do_syscall+0x98/0x1d0
>     [<0>] handle_syscall+0xb8/0x158
>
> part 2:
>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>     [<0>] kvm_exc_entry+0x104/0x1e4
>
>
> In "part 1", after executing kvm_enter_guest, the cpu switches from 
> root-mode to guest-mode.
> In this case, stack backtracking is indeed very rare.
>
> In "part 2", the cpu switches from the guest-mode to the root-mode,
> and most of the stack backtracking occurs during this phase.
>
> To obtain the longest call chain, we save pc in kvm_enter_guest to 
> pt_regs.csr_era,
> and after restoring the sp of the root-mode cpu in kvm_exc_entry,
> The ORC entry was re-established using "UNWIND_HINT_REGS",
>  and then we obtained the following stack backtrace as we wanted:
>
>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>     [<0>] kvm_exc_entry+0x104/0x1e4
I found this might be a coincidence—correct behavior due to the incorrect
UNWIND_HINT_REGS mark and unusual stack adjustment.

First, the kvm_enter_guest contains only a single branch instruction, ertn.
It hardware-jump to the CSR.ERA address directly, jump into kvm_exc_entry.

At this point, the stack layout looks like this:
-------------------------------
   frame from call to `kvm_enter_guest`
-------------------------------  <- $sp
   PT_REGS
-------------------------------  <- $a2

Then kvm_exc_entry adjust stack without save any register (e.g. $ra, $sp)
but still marked UNWIND_HINT_REGS.
After the adjustment:
-------------------------------
   frame from call to `kvm_enter_guest`
-------------------------------
   PT_REGS
-------------------------------  <- $a2, new $sp

During unwinding, when the unwinder reaches kvm_exc_entry,
it meets the mark of PT_REGS and correctly recovers
  pc = regs.csr_era, sp = regs.sp, ra = regs.ra

a) Can we avoid "ertn" rather than `jr reg (or jirl ra, reg, 0)`
instead, like call? The kvm_exc_entry cannot back to kvm_enter_guest
if we use "ertn", so should the kvm_enter_guest appear on the stacktrace?

b) Can we adjust $sp before entering kvm_exc_entry? Then we can mark
UNWIND_HINT_REGS at the beginning of kvm_exc_entry, which something
like ret_from_kernel_thread_asm.

> [<0>] kvm_enter_guest+0x38/0x11c
>     [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>     [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>     [<0>] sys_ioctl+0x498/0xf00
>     [<0>] do_syscall+0x98/0x1d0
>     [<0>] handle_syscall+0xb8/0x158
>
> Doing so is equivalent to ignoring the details of the cpu root-mode 
> and guest-mode switching.
> About what you said in the IE enable phase is dangerous,
> interrupts are always off during the cpu root-mode and guest-mode 
> switching in kvm_enter_guest and kvm_exc_entry.
>

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[Bibo Mao]

On 2025/12/29 下午3:10, Jinyang He wrote:
> 
> On 2025-12-29 11:53, lixianglai wrote:
>> Hi Jinyang:
>>> On 2025-12-27 09:27, Xianglai Li wrote:
>>>
>>>> Insert the appropriate UNWIND macro definition into the 
>>>> kvm_exc_entry in
>>>> the assembly function to guide the generation of correct ORC table 
>>>> entries,
>>>> thereby solving the timeout problem of loading the livepatch-sample 
>>>> module
>>>> on a physical machine running multiple vcpus virtual machines.
>>>>
>>>> While solving the above problems, we have gained an additional benefit,
>>>> that is, we can obtain more call stack information
>>>>
>>>> Stack information that can be obtained before the problem is fixed:
>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>> [<0>] kvm_exc_entry+0x100/0x1e0
>>>>
>>>> Stack information that can be obtained after the problem is fixed:
>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>> [<0>] kvm_exc_entry+0x104/0x1e4
>>>> [<0>] kvm_enter_guest+0x38/0x11c
>>>> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>> [<0>] sys_ioctl+0x498/0xf00
>>>> [<0>] do_syscall+0x98/0x1d0
>>>> [<0>] handle_syscall+0xb8/0x158
>>>>
>>>> Cc: stable@vger.kernel.org
>>>> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
>>>> ---
>>>> Cc: Huacai Chen <chenhuacai@kernel.org>
>>>> Cc: WANG Xuerui <kernel@xen0n.name>
>>>> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
>>>> Cc: Bibo Mao <maobibo@loongson.cn>
>>>> Cc: Charlie Jenkins <charlie@rivosinc.com>
>>>> Cc: Xianglai Li <lixianglai@loongson.cn>
>>>> Cc: Thomas Gleixner <tglx@linutronix.de>
>>>> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
>>>>
>>>>   arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
>>>>   1 file changed, 19 insertions(+), 9 deletions(-)
>>>>
>>>> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
>>>> index 93845ce53651..a3ea9567dbe5 100644
>>>> --- a/arch/loongarch/kvm/switch.S
>>>> +++ b/arch/loongarch/kvm/switch.S
>>>> @@ -10,6 +10,7 @@
>>>>   #include <asm/loongarch.h>
>>>>   #include <asm/regdef.h>
>>>>   #include <asm/unwind_hints.h>
>>>> +#include <linux/kvm_types.h>
>>>>     #define HGPR_OFFSET(x)        (PT_R0 + 8*x)
>>>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>>> @@ -110,9 +111,9 @@
>>>>        * need to copy world switch code to DMW area.
>>>>        */
>>>>       .text
>>>> +    .p2align PAGE_SHIFT
>>>>       .cfi_sections    .debug_frame
>>>>   SYM_CODE_START(kvm_exc_entry)
>>>> -    .p2align PAGE_SHIFT
>>>>       UNWIND_HINT_UNDEFINED
>>>>       csrwr    a2,   KVM_TEMP_KS
>>>>       csrrd    a2,   KVM_VCPU_KS
>>>> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
>>>>       /* restore per cpu register */
>>>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
>>>>       addi.d    sp, sp, -PT_SIZE
>>>> +    UNWIND_HINT_REGS
>>>>         /* Prepare handle exception */
>>>>       or    a0, s0, zero
>>>> @@ -200,7 +202,7 @@ ret_to_host:
>>>>       jr      ra
>>>>     SYM_CODE_END(kvm_exc_entry)
>>>> -EXPORT_SYMBOL(kvm_exc_entry)
>>>> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
>>>>     /*
>>>>    * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu)
>>>> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
>>>>       /* Save host GPRs */
>>>>       kvm_save_host_gpr a2
>>>>   +    /*
>>>> +     * The csr_era member variable of the pt_regs structure is 
>>>> required
>>>> +     * for unwinding orc to perform stack traceback, so we need to put
>>>> +     * pc into csr_era member variable here.
>>>> +     */
>>>> +    pcaddi    t0, 0
>>>> +    st.d    t0, a2, PT_ERA
>>> Hi, Xianglai,
>>>
>>> It should use `SYM_CODE_START` to mark the `kvm_enter_guest` rather than
>>> `SYM_FUNC_START`, since the `SYM_FUNC_START` is used to mark "C-likely"
>>> asm functionw. 
>>
>> Ok, I will use SYM_CODE_START to mark kvm_enter_guest in the next 
>> version.
>>
>>> I guess the kvm_enter_guest is something like exception
>>> handler becuase the last instruction is "ertn". So usually it should
>>> mark UNWIND_HINT_REGS where can find last frame info by "$sp".
>>> However, all info is store to "$a2", this mark should be
>>>   `UNWIND_HINT sp_reg=ORC_REG_A2(???) type=UNWIND_HINT_TYPE_REGS`.
>>> I don't konw why save this function internal PC here by `pcaddi t0, 0`,
>>> and I think it is no meaning(, for exception handler, they save last PC
>>> by read CSR.ERA). The `kvm_enter_guest` saves registers by
>>> "$a2"("$sp" - PT_REGS) beyond stack ("$sp"), it is dangerous if IE
>>> is enable. So I wonder if there is really a stacktrace through this 
>>> function?
>>>
>> The stack backtracking issue in switch.S is rather complex because it 
>> involves the switching between cpu root-mode and guest-mode:
>> Real stack backtracking should be divided into two parts:
>> part 1:
>>     [<0>] kvm_enter_guest+0x38/0x11c
>>     [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>     [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>     [<0>] sys_ioctl+0x498/0xf00
>>     [<0>] do_syscall+0x98/0x1d0
>>     [<0>] handle_syscall+0xb8/0x158
>>
>> part 2:
>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>     [<0>] kvm_exc_entry+0x104/0x1e4
>>
>>
>> In "part 1", after executing kvm_enter_guest, the cpu switches from 
>> root-mode to guest-mode.
>> In this case, stack backtracking is indeed very rare.
>>
>> In "part 2", the cpu switches from the guest-mode to the root-mode,
>> and most of the stack backtracking occurs during this phase.
>>
>> To obtain the longest call chain, we save pc in kvm_enter_guest to 
>> pt_regs.csr_era,
>> and after restoring the sp of the root-mode cpu in kvm_exc_entry,
>> The ORC entry was re-established using "UNWIND_HINT_REGS",
>>  and then we obtained the following stack backtrace as we wanted:
>>
>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>     [<0>] kvm_exc_entry+0x104/0x1e4
> I found this might be a coincidence—correct behavior due to the incorrect
> UNWIND_HINT_REGS mark and unusual stack adjustment.
> 
> First, the kvm_enter_guest contains only a single branch instruction, ertn.
> It hardware-jump to the CSR.ERA address directly, jump into kvm_exc_entry.
> 
> At this point, the stack layout looks like this:
> -------------------------------
>    frame from call to `kvm_enter_guest`
> -------------------------------  <- $sp
>    PT_REGS
> -------------------------------  <- $a2
> 
> Then kvm_exc_entry adjust stack without save any register (e.g. $ra, $sp)
> but still marked UNWIND_HINT_REGS.
> After the adjustment:
> -------------------------------
>    frame from call to `kvm_enter_guest`
> -------------------------------
>    PT_REGS
> -------------------------------  <- $a2, new $sp
> 
> During unwinding, when the unwinder reaches kvm_exc_entry,
> it meets the mark of PT_REGS and correctly recovers
>   pc = regs.csr_era, sp = regs.sp, ra = regs.ra
> 
> a) Can we avoid "ertn" rather than `jr reg (or jirl ra, reg, 0)`
> instead, like call? The kvm_exc_entry cannot back to kvm_enter_guest
> if we use "ertn", so should the kvm_enter_guest appear on the stacktrace?
> 
> b) Can we adjust $sp before entering kvm_exc_entry? Then we can mark
> UNWIND_HINT_REGS at the beginning of kvm_exc_entry, which something
> like ret_from_kernel_thread_asm.
Talk is cheap. Could you show your code?

Regards
Bibo Mao
> 
>> [<0>] kvm_enter_guest+0x38/0x11c
>>     [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>     [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>     [<0>] sys_ioctl+0x498/0xf00
>>     [<0>] do_syscall+0x98/0x1d0
>>     [<0>] handle_syscall+0xb8/0x158
>>
>> Doing so is equivalent to ignoring the details of the cpu root-mode 
>> and guest-mode switching.
>> About what you said in the IE enable phase is dangerous,
>> interrupts are always off during the cpu root-mode and guest-mode 
>> switching in kvm_enter_guest and kvm_exc_entry.
>>
> 

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[lixianglai]

Hi Jinyang:
>
> On 2025-12-29 11:53, lixianglai wrote:
>> Hi Jinyang:
>>> On 2025-12-27 09:27, Xianglai Li wrote:
>>>
>>>> Insert the appropriate UNWIND macro definition into the 
>>>> kvm_exc_entry in
>>>> the assembly function to guide the generation of correct ORC table 
>>>> entries,
>>>> thereby solving the timeout problem of loading the livepatch-sample 
>>>> module
>>>> on a physical machine running multiple vcpus virtual machines.
>>>>
>>>> While solving the above problems, we have gained an additional 
>>>> benefit,
>>>> that is, we can obtain more call stack information
>>>>
>>>> Stack information that can be obtained before the problem is fixed:
>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>> [<0>] kvm_exc_entry+0x100/0x1e0
>>>>
>>>> Stack information that can be obtained after the problem is fixed:
>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>> [<0>] kvm_exc_entry+0x104/0x1e4
>>>> [<0>] kvm_enter_guest+0x38/0x11c
>>>> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>> [<0>] sys_ioctl+0x498/0xf00
>>>> [<0>] do_syscall+0x98/0x1d0
>>>> [<0>] handle_syscall+0xb8/0x158
>>>>
>>>> Cc: stable@vger.kernel.org
>>>> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
>>>> ---
>>>> Cc: Huacai Chen <chenhuacai@kernel.org>
>>>> Cc: WANG Xuerui <kernel@xen0n.name>
>>>> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
>>>> Cc: Bibo Mao <maobibo@loongson.cn>
>>>> Cc: Charlie Jenkins <charlie@rivosinc.com>
>>>> Cc: Xianglai Li <lixianglai@loongson.cn>
>>>> Cc: Thomas Gleixner <tglx@linutronix.de>
>>>> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
>>>>
>>>>   arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
>>>>   1 file changed, 19 insertions(+), 9 deletions(-)
>>>>
>>>> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
>>>> index 93845ce53651..a3ea9567dbe5 100644
>>>> --- a/arch/loongarch/kvm/switch.S
>>>> +++ b/arch/loongarch/kvm/switch.S
>>>> @@ -10,6 +10,7 @@
>>>>   #include <asm/loongarch.h>
>>>>   #include <asm/regdef.h>
>>>>   #include <asm/unwind_hints.h>
>>>> +#include <linux/kvm_types.h>
>>>>     #define HGPR_OFFSET(x)        (PT_R0 + 8*x)
>>>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>>> @@ -110,9 +111,9 @@
>>>>        * need to copy world switch code to DMW area.
>>>>        */
>>>>       .text
>>>> +    .p2align PAGE_SHIFT
>>>>       .cfi_sections    .debug_frame
>>>>   SYM_CODE_START(kvm_exc_entry)
>>>> -    .p2align PAGE_SHIFT
>>>>       UNWIND_HINT_UNDEFINED
>>>>       csrwr    a2,   KVM_TEMP_KS
>>>>       csrrd    a2,   KVM_VCPU_KS
>>>> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
>>>>       /* restore per cpu register */
>>>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
>>>>       addi.d    sp, sp, -PT_SIZE
>>>> +    UNWIND_HINT_REGS
>>>>         /* Prepare handle exception */
>>>>       or    a0, s0, zero
>>>> @@ -200,7 +202,7 @@ ret_to_host:
>>>>       jr      ra
>>>>     SYM_CODE_END(kvm_exc_entry)
>>>> -EXPORT_SYMBOL(kvm_exc_entry)
>>>> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
>>>>     /*
>>>>    * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu)
>>>> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
>>>>       /* Save host GPRs */
>>>>       kvm_save_host_gpr a2
>>>>   +    /*
>>>> +     * The csr_era member variable of the pt_regs structure is 
>>>> required
>>>> +     * for unwinding orc to perform stack traceback, so we need to 
>>>> put
>>>> +     * pc into csr_era member variable here.
>>>> +     */
>>>> +    pcaddi    t0, 0
>>>> +    st.d    t0, a2, PT_ERA
>>> Hi, Xianglai,
>>>
>>> It should use `SYM_CODE_START` to mark the `kvm_enter_guest` rather 
>>> than
>>> `SYM_FUNC_START`, since the `SYM_FUNC_START` is used to mark "C-likely"
>>> asm functionw. 
>>
>> Ok, I will use SYM_CODE_START to mark kvm_enter_guest in the next 
>> version.
>>
>>> I guess the kvm_enter_guest is something like exception
>>> handler becuase the last instruction is "ertn". So usually it should
>>> mark UNWIND_HINT_REGS where can find last frame info by "$sp".
>>> However, all info is store to "$a2", this mark should be
>>>   `UNWIND_HINT sp_reg=ORC_REG_A2(???) type=UNWIND_HINT_TYPE_REGS`.
>>> I don't konw why save this function internal PC here by `pcaddi t0, 0`,
>>> and I think it is no meaning(, for exception handler, they save last PC
>>> by read CSR.ERA). The `kvm_enter_guest` saves registers by
>>> "$a2"("$sp" - PT_REGS) beyond stack ("$sp"), it is dangerous if IE
>>> is enable. So I wonder if there is really a stacktrace through this 
>>> function?
>>>
>> The stack backtracking issue in switch.S is rather complex because it 
>> involves the switching between cpu root-mode and guest-mode:
>> Real stack backtracking should be divided into two parts:
>> part 1:
>>     [<0>] kvm_enter_guest+0x38/0x11c
>>     [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>     [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>     [<0>] sys_ioctl+0x498/0xf00
>>     [<0>] do_syscall+0x98/0x1d0
>>     [<0>] handle_syscall+0xb8/0x158
>>
>> part 2:
>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>     [<0>] kvm_exc_entry+0x104/0x1e4
>>
>>
>> In "part 1", after executing kvm_enter_guest, the cpu switches from 
>> root-mode to guest-mode.
>> In this case, stack backtracking is indeed very rare.
>>
>> In "part 2", the cpu switches from the guest-mode to the root-mode,
>> and most of the stack backtracking occurs during this phase.
>>
>> To obtain the longest call chain, we save pc in kvm_enter_guest to 
>> pt_regs.csr_era,
>> and after restoring the sp of the root-mode cpu in kvm_exc_entry,
>> The ORC entry was re-established using "UNWIND_HINT_REGS",
>>  and then we obtained the following stack backtrace as we wanted:
>>
>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>     [<0>] kvm_exc_entry+0x104/0x1e4
> I found this might be a coincidence—correct behavior due to the incorrect
> UNWIND_HINT_REGS mark and unusual stack adjustment.
>
> First, the kvm_enter_guest contains only a single branch instruction, 
> ertn.
> It hardware-jump to the CSR.ERA address directly, jump into 
> kvm_exc_entry.
>
> At this point, the stack layout looks like this:
> -------------------------------
>   frame from call to `kvm_enter_guest`
> -------------------------------  <- $sp
>   PT_REGS
> -------------------------------  <- $a2
>
> Then kvm_exc_entry adjust stack without save any register (e.g. $ra, $sp)
> but still marked UNWIND_HINT_REGS.
> After the adjustment:
> -------------------------------
>   frame from call to `kvm_enter_guest`
> -------------------------------
>   PT_REGS
> -------------------------------  <- $a2, new $sp
>
> During unwinding, when the unwinder reaches kvm_exc_entry,
> it meets the mark of PT_REGS and correctly recovers
>  pc = regs.csr_era, sp = regs.sp, ra = regs.ra
>
Yes, here unwinder does work as you say.

> a) Can we avoid "ertn" rather than `jr reg (or jirl ra, reg, 0)`
> instead, like call?
No,  we need to rely on the 'ertn instruction return PIE to CRMD IE,
at the same time to ensure that its atomic,
there should be no other instruction than' ertn 'more appropriate here.

> The kvm_exc_entry cannot back to kvm_enter_guest
> if we use "ertn", so should the kvm_enter_guest appear on the stacktrace?
>

It is flexible. As I mentioned above, the cpu completes the switch from 
host-mode to guest mode through kvm_enter_guest,
and then the switch from guest mode to host-mode through kvm_exc_entry. 
When we ignore the details of the host-mode
and guest-mode switching in the middle, we can understand that the host 
cpu has completed kvm_enter_guest->kvm_exc_entry.
 From this perspective, I think it can exist in the call stack, and at 
the same time, we have obtained the maximum call stack information.


> b) Can we adjust $sp before entering kvm_exc_entry? Then we can mark
> UNWIND_HINT_REGS at the beginning of kvm_exc_entry, which something
> like ret_from_kernel_thread_asm.
>
The following command can be used to dump the orc entries of the kernel:
./tools/objtool/objtool --dump vmlinux

You can observe that not all orc entries are generated at the beginning 
of the function.
For example:
handle_tlb_protect
ftrace_stub
handle_reserved

So, is it unnecessary for us to modify UNWIND_HINT_REGS in order to 
place it at the beginning of the function.

If you have a better solution, could you provide an example of the 
modification?
I can test the feasibility of the solution.

Thanks!
Xianglai.

>> [<0>] kvm_enter_guest+0x38/0x11c
>>     [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>     [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>     [<0>] sys_ioctl+0x498/0xf00
>>     [<0>] do_syscall+0x98/0x1d0
>>     [<0>] handle_syscall+0xb8/0x158
>>
>> Doing so is equivalent to ignoring the details of the cpu root-mode 
>> and guest-mode switching.
>> About what you said in the IE enable phase is dangerous,
>> interrupts are always off during the cpu root-mode and guest-mode 
>> switching in kvm_enter_guest and kvm_exc_entry.
>>

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[Jinyang He]

On 2025-12-29 18:11, lixianglai wrote:

> Hi Jinyang:
>>
>> On 2025-12-29 11:53, lixianglai wrote:
>>> Hi Jinyang:
>>>> On 2025-12-27 09:27, Xianglai Li wrote:
>>>>
>>>>> Insert the appropriate UNWIND macro definition into the 
>>>>> kvm_exc_entry in
>>>>> the assembly function to guide the generation of correct ORC table 
>>>>> entries,
>>>>> thereby solving the timeout problem of loading the 
>>>>> livepatch-sample module
>>>>> on a physical machine running multiple vcpus virtual machines.
>>>>>
>>>>> While solving the above problems, we have gained an additional 
>>>>> benefit,
>>>>> that is, we can obtain more call stack information
>>>>>
>>>>> Stack information that can be obtained before the problem is fixed:
>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>> [<0>] kvm_exc_entry+0x100/0x1e0
>>>>>
>>>>> Stack information that can be obtained after the problem is fixed:
>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>> [<0>] kvm_exc_entry+0x104/0x1e4
>>>>> [<0>] kvm_enter_guest+0x38/0x11c
>>>>> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>>> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>>> [<0>] sys_ioctl+0x498/0xf00
>>>>> [<0>] do_syscall+0x98/0x1d0
>>>>> [<0>] handle_syscall+0xb8/0x158
>>>>>
>>>>> Cc: stable@vger.kernel.org
>>>>> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
>>>>> ---
>>>>> Cc: Huacai Chen <chenhuacai@kernel.org>
>>>>> Cc: WANG Xuerui <kernel@xen0n.name>
>>>>> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
>>>>> Cc: Bibo Mao <maobibo@loongson.cn>
>>>>> Cc: Charlie Jenkins <charlie@rivosinc.com>
>>>>> Cc: Xianglai Li <lixianglai@loongson.cn>
>>>>> Cc: Thomas Gleixner <tglx@linutronix.de>
>>>>> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
>>>>>
>>>>>   arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
>>>>>   1 file changed, 19 insertions(+), 9 deletions(-)
>>>>>
>>>>> diff --git a/arch/loongarch/kvm/switch.S 
>>>>> b/arch/loongarch/kvm/switch.S
>>>>> index 93845ce53651..a3ea9567dbe5 100644
>>>>> --- a/arch/loongarch/kvm/switch.S
>>>>> +++ b/arch/loongarch/kvm/switch.S
>>>>> @@ -10,6 +10,7 @@
>>>>>   #include <asm/loongarch.h>
>>>>>   #include <asm/regdef.h>
>>>>>   #include <asm/unwind_hints.h>
>>>>> +#include <linux/kvm_types.h>
>>>>>     #define HGPR_OFFSET(x)        (PT_R0 + 8*x)
>>>>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>>>> @@ -110,9 +111,9 @@
>>>>>        * need to copy world switch code to DMW area.
>>>>>        */
>>>>>       .text
>>>>> +    .p2align PAGE_SHIFT
>>>>>       .cfi_sections    .debug_frame
>>>>>   SYM_CODE_START(kvm_exc_entry)
>>>>> -    .p2align PAGE_SHIFT
>>>>>       UNWIND_HINT_UNDEFINED
>>>>>       csrwr    a2,   KVM_TEMP_KS
>>>>>       csrrd    a2,   KVM_VCPU_KS
>>>>> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
>>>>>       /* restore per cpu register */
>>>>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
>>>>>       addi.d    sp, sp, -PT_SIZE
>>>>> +    UNWIND_HINT_REGS
>>>>>         /* Prepare handle exception */
>>>>>       or    a0, s0, zero
>>>>> @@ -200,7 +202,7 @@ ret_to_host:
>>>>>       jr      ra
>>>>>     SYM_CODE_END(kvm_exc_entry)
>>>>> -EXPORT_SYMBOL(kvm_exc_entry)
>>>>> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
>>>>>     /*
>>>>>    * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu)
>>>>> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
>>>>>       /* Save host GPRs */
>>>>>       kvm_save_host_gpr a2
>>>>>   +    /*
>>>>> +     * The csr_era member variable of the pt_regs structure is 
>>>>> required
>>>>> +     * for unwinding orc to perform stack traceback, so we need 
>>>>> to put
>>>>> +     * pc into csr_era member variable here.
>>>>> +     */
>>>>> +    pcaddi    t0, 0
>>>>> +    st.d    t0, a2, PT_ERA
>>>> Hi, Xianglai,
>>>>
>>>> It should use `SYM_CODE_START` to mark the `kvm_enter_guest` rather 
>>>> than
>>>> `SYM_FUNC_START`, since the `SYM_FUNC_START` is used to mark 
>>>> "C-likely"
>>>> asm functionw. 
>>>
>>> Ok, I will use SYM_CODE_START to mark kvm_enter_guest in the next 
>>> version.
>>>
>>>> I guess the kvm_enter_guest is something like exception
>>>> handler becuase the last instruction is "ertn". So usually it should
>>>> mark UNWIND_HINT_REGS where can find last frame info by "$sp".
>>>> However, all info is store to "$a2", this mark should be
>>>>   `UNWIND_HINT sp_reg=ORC_REG_A2(???) type=UNWIND_HINT_TYPE_REGS`.
>>>> I don't konw why save this function internal PC here by `pcaddi t0, 
>>>> 0`,
>>>> and I think it is no meaning(, for exception handler, they save 
>>>> last PC
>>>> by read CSR.ERA). The `kvm_enter_guest` saves registers by
>>>> "$a2"("$sp" - PT_REGS) beyond stack ("$sp"), it is dangerous if IE
>>>> is enable. So I wonder if there is really a stacktrace through this 
>>>> function?
>>>>
>>> The stack backtracking issue in switch.S is rather complex because 
>>> it involves the switching between cpu root-mode and guest-mode:
>>> Real stack backtracking should be divided into two parts:
>>> part 1:
>>>     [<0>] kvm_enter_guest+0x38/0x11c
>>>     [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>     [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>     [<0>] sys_ioctl+0x498/0xf00
>>>     [<0>] do_syscall+0x98/0x1d0
>>>     [<0>] handle_syscall+0xb8/0x158
>>>
>>> part 2:
>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>     [<0>] kvm_exc_entry+0x104/0x1e4
>>>
>>>
>>> In "part 1", after executing kvm_enter_guest, the cpu switches from 
>>> root-mode to guest-mode.
>>> In this case, stack backtracking is indeed very rare.
>>>
>>> In "part 2", the cpu switches from the guest-mode to the root-mode,
>>> and most of the stack backtracking occurs during this phase.
>>>
>>> To obtain the longest call chain, we save pc in kvm_enter_guest to 
>>> pt_regs.csr_era,
>>> and after restoring the sp of the root-mode cpu in kvm_exc_entry,
>>> The ORC entry was re-established using "UNWIND_HINT_REGS",
>>>  and then we obtained the following stack backtrace as we wanted:
>>>
>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>     [<0>] kvm_exc_entry+0x104/0x1e4
>> I found this might be a coincidence—correct behavior due to the 
>> incorrect
>> UNWIND_HINT_REGS mark and unusual stack adjustment.
>>
>> First, the kvm_enter_guest contains only a single branch instruction, 
>> ertn.
>> It hardware-jump to the CSR.ERA address directly, jump into 
>> kvm_exc_entry.
>>
>> At this point, the stack layout looks like this:
>> -------------------------------
>>   frame from call to `kvm_enter_guest`
>> -------------------------------  <- $sp
>>   PT_REGS
>> -------------------------------  <- $a2
>>
>> Then kvm_exc_entry adjust stack without save any register (e.g. $ra, 
>> $sp)
>> but still marked UNWIND_HINT_REGS.
>> After the adjustment:
>> -------------------------------
>>   frame from call to `kvm_enter_guest`
>> -------------------------------
>>   PT_REGS
>> -------------------------------  <- $a2, new $sp
>>
>> During unwinding, when the unwinder reaches kvm_exc_entry,
>> it meets the mark of PT_REGS and correctly recovers
>>  pc = regs.csr_era, sp = regs.sp, ra = regs.ra
>>
> Yes, here unwinder does work as you say.
>
>> a) Can we avoid "ertn" rather than `jr reg (or jirl ra, reg, 0)`
>> instead, like call?
> No,  we need to rely on the 'ertn instruction return PIE to CRMD IE,
> at the same time to ensure that its atomic,
> there should be no other instruction than' ertn 'more appropriate here.
You are right! I got it.
>
>> The kvm_exc_entry cannot back to kvm_enter_guest
>> if we use "ertn", so should the kvm_enter_guest appear on the 
>> stacktrace?
>>
>
> It is flexible. As I mentioned above, the cpu completes the switch 
> from host-mode to guest mode through kvm_enter_guest,
> and then the switch from guest mode to host-mode through 
> kvm_exc_entry. When we ignore the details of the host-mode
> and guest-mode switching in the middle, we can understand that the 
> host cpu has completed kvm_enter_guest->kvm_exc_entry.
> From this perspective, I think it can exist in the call stack, and at 
> the same time, we have obtained the maximum call stack information.
>
>
>> b) Can we adjust $sp before entering kvm_exc_entry? Then we can mark
>> UNWIND_HINT_REGS at the beginning of kvm_exc_entry, which something
>> like ret_from_kernel_thread_asm.
>>
> The following command can be used to dump the orc entries of the kernel:
> ./tools/objtool/objtool --dump vmlinux
>
> You can observe that not all orc entries are generated at the 
> beginning of the function.
> For example:
> handle_tlb_protect
> ftrace_stub
> handle_reserved
>
> So, is it unnecessary for us to modify UNWIND_HINT_REGS in order to 
> place it at the beginning of the function.
>
> If you have a better solution, could you provide an example of the 
> modification?
> I can test the feasibility of the solution.
>
The expression at the beginning of the function is incorrect (feeling 
sorry).
It should be marked where have all stacktrace info.
Thanks for all the explaining, since I'm unfamiliar with kvm, I need 
these to help my understanding.

Can you try with follows, with save regs by $sp, set more precise era to 
pt_regs, and more unwind hint.


diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
index f1768b7a6194..8ed1d7b72c54 100644
--- a/arch/loongarch/kvm/switch.S
+++ b/arch/loongarch/kvm/switch.S
@@ -14,13 +14,13 @@
  #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)

  .macro kvm_save_host_gpr base
-    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
+    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
      st.d    $r\n, \base, HGPR_OFFSET(\n)
      .endr
  .endm

  .macro kvm_restore_host_gpr base
-    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
+    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
      ld.d    $r\n, \base, HGPR_OFFSET(\n)
      .endr
  .endm
@@ -88,6 +88,7 @@
      /* Load KVM_ARCH register */
      ld.d    a2, a2,    (KVM_ARCH_GGPR + 8 * REG_A2)

+111:
      ertn /* Switch to guest: GSTAT.PGM = 1, ERRCTL.ISERR = 0, 
TLBRPRMD.ISTLBR = 0 */
  .endm

@@ -158,9 +159,10 @@ SYM_CODE_START(kvm_exc_entry)
      csrwr    t0, LOONGARCH_CSR_GTLBC
      ld.d    tp, a2, KVM_ARCH_HTP
      ld.d    sp, a2, KVM_ARCH_HSP
+    UNWIND_HINT_REGS
+
      /* restore per cpu register */
      ld.d    u0, a2, KVM_ARCH_HPERCPU
-    addi.d    sp, sp, -PT_SIZE

      /* Prepare handle exception */
      or    a0, s0, zero
@@ -184,10 +186,11 @@ SYM_CODE_START(kvm_exc_entry)
      csrwr    s1, KVM_VCPU_KS
      kvm_switch_to_guest

+    UNWIND_HINT_UNDEFINED
  ret_to_host:
-    ld.d    a2, a2, KVM_ARCH_HSP
-    addi.d  a2, a2, -PT_SIZE
-    kvm_restore_host_gpr    a2
+    ld.d    sp, a2, KVM_ARCH_HSP
+    kvm_restore_host_gpr    sp
+    addi.d    sp, sp, PT_SIZE
      jr      ra

  SYM_INNER_LABEL(kvm_exc_entry_end, SYM_L_LOCAL)
@@ -200,11 +203,15 @@ SYM_CODE_END(kvm_exc_entry)
   *  a0: kvm_run* run
   *  a1: kvm_vcpu* vcpu
   */
-SYM_FUNC_START(kvm_enter_guest)
+SYM_CODE_START(kvm_enter_guest)
+    UNWIND_HINT_UNDEFINED
      /* Allocate space in stack bottom */
-    addi.d    a2, sp, -PT_SIZE
+    addi.d    sp, sp, -PT_SIZE
      /* Save host GPRs */
-    kvm_save_host_gpr a2
+    kvm_save_host_gpr sp
+    la.pcrel a2, 111f
+    st.d     a2, sp, PT_ERA
+    UNWIND_HINT_REGS

      addi.d    a2, a1, KVM_VCPU_ARCH
      st.d    sp, a2, KVM_ARCH_HSP

Jinyang

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[Bibo Mao]

On 2025/12/29 下午6:41, Jinyang He wrote:
> On 2025-12-29 18:11, lixianglai wrote:
> 
>> Hi Jinyang:
>>>
>>> On 2025-12-29 11:53, lixianglai wrote:
>>>> Hi Jinyang:
>>>>> On 2025-12-27 09:27, Xianglai Li wrote:
>>>>>
>>>>>> Insert the appropriate UNWIND macro definition into the 
>>>>>> kvm_exc_entry in
>>>>>> the assembly function to guide the generation of correct ORC table 
>>>>>> entries,
>>>>>> thereby solving the timeout problem of loading the 
>>>>>> livepatch-sample module
>>>>>> on a physical machine running multiple vcpus virtual machines.
>>>>>>
>>>>>> While solving the above problems, we have gained an additional 
>>>>>> benefit,
>>>>>> that is, we can obtain more call stack information
>>>>>>
>>>>>> Stack information that can be obtained before the problem is fixed:
>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>> [<0>] kvm_exc_entry+0x100/0x1e0
>>>>>>
>>>>>> Stack information that can be obtained after the problem is fixed:
>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>> [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>> [<0>] kvm_enter_guest+0x38/0x11c
>>>>>> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>>>> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>>>> [<0>] sys_ioctl+0x498/0xf00
>>>>>> [<0>] do_syscall+0x98/0x1d0
>>>>>> [<0>] handle_syscall+0xb8/0x158
>>>>>>
>>>>>> Cc: stable@vger.kernel.org
>>>>>> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
>>>>>> ---
>>>>>> Cc: Huacai Chen <chenhuacai@kernel.org>
>>>>>> Cc: WANG Xuerui <kernel@xen0n.name>
>>>>>> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
>>>>>> Cc: Bibo Mao <maobibo@loongson.cn>
>>>>>> Cc: Charlie Jenkins <charlie@rivosinc.com>
>>>>>> Cc: Xianglai Li <lixianglai@loongson.cn>
>>>>>> Cc: Thomas Gleixner <tglx@linutronix.de>
>>>>>> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
>>>>>>
>>>>>>   arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
>>>>>>   1 file changed, 19 insertions(+), 9 deletions(-)
>>>>>>
>>>>>> diff --git a/arch/loongarch/kvm/switch.S 
>>>>>> b/arch/loongarch/kvm/switch.S
>>>>>> index 93845ce53651..a3ea9567dbe5 100644
>>>>>> --- a/arch/loongarch/kvm/switch.S
>>>>>> +++ b/arch/loongarch/kvm/switch.S
>>>>>> @@ -10,6 +10,7 @@
>>>>>>   #include <asm/loongarch.h>
>>>>>>   #include <asm/regdef.h>
>>>>>>   #include <asm/unwind_hints.h>
>>>>>> +#include <linux/kvm_types.h>
>>>>>>     #define HGPR_OFFSET(x)        (PT_R0 + 8*x)
>>>>>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>>>>> @@ -110,9 +111,9 @@
>>>>>>        * need to copy world switch code to DMW area.
>>>>>>        */
>>>>>>       .text
>>>>>> +    .p2align PAGE_SHIFT
>>>>>>       .cfi_sections    .debug_frame
>>>>>>   SYM_CODE_START(kvm_exc_entry)
>>>>>> -    .p2align PAGE_SHIFT
>>>>>>       UNWIND_HINT_UNDEFINED
>>>>>>       csrwr    a2,   KVM_TEMP_KS
>>>>>>       csrrd    a2,   KVM_VCPU_KS
>>>>>> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
>>>>>>       /* restore per cpu register */
>>>>>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
>>>>>>       addi.d    sp, sp, -PT_SIZE
>>>>>> +    UNWIND_HINT_REGS
>>>>>>         /* Prepare handle exception */
>>>>>>       or    a0, s0, zero
>>>>>> @@ -200,7 +202,7 @@ ret_to_host:
>>>>>>       jr      ra
>>>>>>     SYM_CODE_END(kvm_exc_entry)
>>>>>> -EXPORT_SYMBOL(kvm_exc_entry)
>>>>>> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
>>>>>>     /*
>>>>>>    * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu)
>>>>>> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
>>>>>>       /* Save host GPRs */
>>>>>>       kvm_save_host_gpr a2
>>>>>>   +    /*
>>>>>> +     * The csr_era member variable of the pt_regs structure is 
>>>>>> required
>>>>>> +     * for unwinding orc to perform stack traceback, so we need 
>>>>>> to put
>>>>>> +     * pc into csr_era member variable here.
>>>>>> +     */
>>>>>> +    pcaddi    t0, 0
>>>>>> +    st.d    t0, a2, PT_ERA
>>>>> Hi, Xianglai,
>>>>>
>>>>> It should use `SYM_CODE_START` to mark the `kvm_enter_guest` rather 
>>>>> than
>>>>> `SYM_FUNC_START`, since the `SYM_FUNC_START` is used to mark 
>>>>> "C-likely"
>>>>> asm functionw. 
>>>>
>>>> Ok, I will use SYM_CODE_START to mark kvm_enter_guest in the next 
>>>> version.
>>>>
>>>>> I guess the kvm_enter_guest is something like exception
>>>>> handler becuase the last instruction is "ertn". So usually it should
>>>>> mark UNWIND_HINT_REGS where can find last frame info by "$sp".
>>>>> However, all info is store to "$a2", this mark should be
>>>>>   `UNWIND_HINT sp_reg=ORC_REG_A2(???) type=UNWIND_HINT_TYPE_REGS`.
>>>>> I don't konw why save this function internal PC here by `pcaddi t0, 
>>>>> 0`,
>>>>> and I think it is no meaning(, for exception handler, they save 
>>>>> last PC
>>>>> by read CSR.ERA). The `kvm_enter_guest` saves registers by
>>>>> "$a2"("$sp" - PT_REGS) beyond stack ("$sp"), it is dangerous if IE
>>>>> is enable. So I wonder if there is really a stacktrace through this 
>>>>> function?
>>>>>
>>>> The stack backtracking issue in switch.S is rather complex because 
>>>> it involves the switching between cpu root-mode and guest-mode:
>>>> Real stack backtracking should be divided into two parts:
>>>> part 1:
>>>>     [<0>] kvm_enter_guest+0x38/0x11c
>>>>     [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>>     [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>>     [<0>] sys_ioctl+0x498/0xf00
>>>>     [<0>] do_syscall+0x98/0x1d0
>>>>     [<0>] handle_syscall+0xb8/0x158
>>>>
>>>> part 2:
>>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>     [<0>] kvm_exc_entry+0x104/0x1e4
>>>>
>>>>
>>>> In "part 1", after executing kvm_enter_guest, the cpu switches from 
>>>> root-mode to guest-mode.
>>>> In this case, stack backtracking is indeed very rare.
>>>>
>>>> In "part 2", the cpu switches from the guest-mode to the root-mode,
>>>> and most of the stack backtracking occurs during this phase.
>>>>
>>>> To obtain the longest call chain, we save pc in kvm_enter_guest to 
>>>> pt_regs.csr_era,
>>>> and after restoring the sp of the root-mode cpu in kvm_exc_entry,
>>>> The ORC entry was re-established using "UNWIND_HINT_REGS",
>>>>  and then we obtained the following stack backtrace as we wanted:
>>>>
>>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>     [<0>] kvm_exc_entry+0x104/0x1e4
>>> I found this might be a coincidence—correct behavior due to the 
>>> incorrect
>>> UNWIND_HINT_REGS mark and unusual stack adjustment.
>>>
>>> First, the kvm_enter_guest contains only a single branch instruction, 
>>> ertn.
>>> It hardware-jump to the CSR.ERA address directly, jump into 
>>> kvm_exc_entry.
>>>
>>> At this point, the stack layout looks like this:
>>> -------------------------------
>>>   frame from call to `kvm_enter_guest`
>>> -------------------------------  <- $sp
>>>   PT_REGS
>>> -------------------------------  <- $a2
>>>
>>> Then kvm_exc_entry adjust stack without save any register (e.g. $ra, 
>>> $sp)
>>> but still marked UNWIND_HINT_REGS.
>>> After the adjustment:
>>> -------------------------------
>>>   frame from call to `kvm_enter_guest`
>>> -------------------------------
>>>   PT_REGS
>>> -------------------------------  <- $a2, new $sp
>>>
>>> During unwinding, when the unwinder reaches kvm_exc_entry,
>>> it meets the mark of PT_REGS and correctly recovers
>>>  pc = regs.csr_era, sp = regs.sp, ra = regs.ra
>>>
>> Yes, here unwinder does work as you say.
>>
>>> a) Can we avoid "ertn" rather than `jr reg (or jirl ra, reg, 0)`
>>> instead, like call?
>> No,  we need to rely on the 'ertn instruction return PIE to CRMD IE,
>> at the same time to ensure that its atomic,
>> there should be no other instruction than' ertn 'more appropriate here.
> You are right! I got it.
>>
>>> The kvm_exc_entry cannot back to kvm_enter_guest
>>> if we use "ertn", so should the kvm_enter_guest appear on the 
>>> stacktrace?
>>>
>>
>> It is flexible. As I mentioned above, the cpu completes the switch 
>> from host-mode to guest mode through kvm_enter_guest,
>> and then the switch from guest mode to host-mode through 
>> kvm_exc_entry. When we ignore the details of the host-mode
>> and guest-mode switching in the middle, we can understand that the 
>> host cpu has completed kvm_enter_guest->kvm_exc_entry.
>> From this perspective, I think it can exist in the call stack, and at 
>> the same time, we have obtained the maximum call stack information.
>>
>>
>>> b) Can we adjust $sp before entering kvm_exc_entry? Then we can mark
>>> UNWIND_HINT_REGS at the beginning of kvm_exc_entry, which something
>>> like ret_from_kernel_thread_asm.
>>>
>> The following command can be used to dump the orc entries of the kernel:
>> ./tools/objtool/objtool --dump vmlinux
>>
>> You can observe that not all orc entries are generated at the 
>> beginning of the function.
>> For example:
>> handle_tlb_protect
>> ftrace_stub
>> handle_reserved
>>
>> So, is it unnecessary for us to modify UNWIND_HINT_REGS in order to 
>> place it at the beginning of the function.
>>
>> If you have a better solution, could you provide an example of the 
>> modification?
>> I can test the feasibility of the solution.
>>
> The expression at the beginning of the function is incorrect (feeling 
> sorry).
> It should be marked where have all stacktrace info.
> Thanks for all the explaining, since I'm unfamiliar with kvm, I need 
> these to help my understanding.
> 
> Can you try with follows, with save regs by $sp, set more precise era to 
> pt_regs, and more unwind hint.
> 
> 
> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
> index f1768b7a6194..8ed1d7b72c54 100644
> --- a/arch/loongarch/kvm/switch.S
> +++ b/arch/loongarch/kvm/switch.S
> @@ -14,13 +14,13 @@
>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
> 
>   .macro kvm_save_host_gpr base
> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
>       st.d    $r\n, \base, HGPR_OFFSET(\n)
>       .endr
>   .endm
> 
>   .macro kvm_restore_host_gpr base
> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
>       ld.d    $r\n, \base, HGPR_OFFSET(\n)
>       .endr
>   .endm
> @@ -88,6 +88,7 @@
>       /* Load KVM_ARCH register */
>       ld.d    a2, a2,    (KVM_ARCH_GGPR + 8 * REG_A2)
> 
> +111:
>       ertn /* Switch to guest: GSTAT.PGM = 1, ERRCTL.ISERR = 0, 
> TLBRPRMD.ISTLBR = 0 */
>   .endm
> 
> @@ -158,9 +159,10 @@ SYM_CODE_START(kvm_exc_entry)
>       csrwr    t0, LOONGARCH_CSR_GTLBC
>       ld.d    tp, a2, KVM_ARCH_HTP
>       ld.d    sp, a2, KVM_ARCH_HSP
> +    UNWIND_HINT_REGS
> +
>       /* restore per cpu register */
>       ld.d    u0, a2, KVM_ARCH_HPERCPU
> -    addi.d    sp, sp, -PT_SIZE
> 
>       /* Prepare handle exception */
>       or    a0, s0, zero
> @@ -184,10 +186,11 @@ SYM_CODE_START(kvm_exc_entry)
>       csrwr    s1, KVM_VCPU_KS
>       kvm_switch_to_guest
> 
> +    UNWIND_HINT_UNDEFINED
>   ret_to_host:
> -    ld.d    a2, a2, KVM_ARCH_HSP
> -    addi.d  a2, a2, -PT_SIZE
> -    kvm_restore_host_gpr    a2
> +    ld.d    sp, a2, KVM_ARCH_HSP
> +    kvm_restore_host_gpr    sp
> +    addi.d    sp, sp, PT_SIZE
>       jr      ra
> 
>   SYM_INNER_LABEL(kvm_exc_entry_end, SYM_L_LOCAL)
> @@ -200,11 +203,15 @@ SYM_CODE_END(kvm_exc_entry)
>    *  a0: kvm_run* run
>    *  a1: kvm_vcpu* vcpu
>    */
> -SYM_FUNC_START(kvm_enter_guest)
> +SYM_CODE_START(kvm_enter_guest)
> +    UNWIND_HINT_UNDEFINED
>       /* Allocate space in stack bottom */
> -    addi.d    a2, sp, -PT_SIZE
> +    addi.d    sp, sp, -PT_SIZE
>       /* Save host GPRs */
> -    kvm_save_host_gpr a2
> +    kvm_save_host_gpr sp
> +    la.pcrel a2, 111f
> +    st.d     a2, sp, PT_ERA
> +    UNWIND_HINT_REGS
> 
>       addi.d    a2, a1, KVM_VCPU_ARCH
>       st.d    sp, a2, KVM_ARCH_HSP
> 
wow, so quick and wonderful.

Huacai,
  the loongarch kernel and kvm maintainer, what is your point about this?

Regards
Bibo Mao
> Jinyang
> 

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[Huacai Chen]

On Tue, Dec 30, 2025 at 9:21 AM Bibo Mao <maobibo@loongson.cn> wrote:
>
>
>
> On 2025/12/29 下午6:41, Jinyang He wrote:
> > On 2025-12-29 18:11, lixianglai wrote:
> >
> >> Hi Jinyang:
> >>>
> >>> On 2025-12-29 11:53, lixianglai wrote:
> >>>> Hi Jinyang:
> >>>>> On 2025-12-27 09:27, Xianglai Li wrote:
> >>>>>
> >>>>>> Insert the appropriate UNWIND macro definition into the
> >>>>>> kvm_exc_entry in
> >>>>>> the assembly function to guide the generation of correct ORC table
> >>>>>> entries,
> >>>>>> thereby solving the timeout problem of loading the
> >>>>>> livepatch-sample module
> >>>>>> on a physical machine running multiple vcpus virtual machines.
> >>>>>>
> >>>>>> While solving the above problems, we have gained an additional
> >>>>>> benefit,
> >>>>>> that is, we can obtain more call stack information
> >>>>>>
> >>>>>> Stack information that can be obtained before the problem is fixed:
> >>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
> >>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
> >>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
> >>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
> >>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
> >>>>>> [<0>] kvm_exc_entry+0x100/0x1e0
> >>>>>>
> >>>>>> Stack information that can be obtained after the problem is fixed:
> >>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
> >>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
> >>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
> >>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
> >>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
> >>>>>> [<0>] kvm_exc_entry+0x104/0x1e4
> >>>>>> [<0>] kvm_enter_guest+0x38/0x11c
> >>>>>> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
> >>>>>> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
> >>>>>> [<0>] sys_ioctl+0x498/0xf00
> >>>>>> [<0>] do_syscall+0x98/0x1d0
> >>>>>> [<0>] handle_syscall+0xb8/0x158
> >>>>>>
> >>>>>> Cc: stable@vger.kernel.org
> >>>>>> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
> >>>>>> ---
> >>>>>> Cc: Huacai Chen <chenhuacai@kernel.org>
> >>>>>> Cc: WANG Xuerui <kernel@xen0n.name>
> >>>>>> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
> >>>>>> Cc: Bibo Mao <maobibo@loongson.cn>
> >>>>>> Cc: Charlie Jenkins <charlie@rivosinc.com>
> >>>>>> Cc: Xianglai Li <lixianglai@loongson.cn>
> >>>>>> Cc: Thomas Gleixner <tglx@linutronix.de>
> >>>>>> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
> >>>>>>
> >>>>>>   arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
> >>>>>>   1 file changed, 19 insertions(+), 9 deletions(-)
> >>>>>>
> >>>>>> diff --git a/arch/loongarch/kvm/switch.S
> >>>>>> b/arch/loongarch/kvm/switch.S
> >>>>>> index 93845ce53651..a3ea9567dbe5 100644
> >>>>>> --- a/arch/loongarch/kvm/switch.S
> >>>>>> +++ b/arch/loongarch/kvm/switch.S
> >>>>>> @@ -10,6 +10,7 @@
> >>>>>>   #include <asm/loongarch.h>
> >>>>>>   #include <asm/regdef.h>
> >>>>>>   #include <asm/unwind_hints.h>
> >>>>>> +#include <linux/kvm_types.h>
> >>>>>>     #define HGPR_OFFSET(x)        (PT_R0 + 8*x)
> >>>>>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
> >>>>>> @@ -110,9 +111,9 @@
> >>>>>>        * need to copy world switch code to DMW area.
> >>>>>>        */
> >>>>>>       .text
> >>>>>> +    .p2align PAGE_SHIFT
> >>>>>>       .cfi_sections    .debug_frame
> >>>>>>   SYM_CODE_START(kvm_exc_entry)
> >>>>>> -    .p2align PAGE_SHIFT
> >>>>>>       UNWIND_HINT_UNDEFINED
> >>>>>>       csrwr    a2,   KVM_TEMP_KS
> >>>>>>       csrrd    a2,   KVM_VCPU_KS
> >>>>>> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
> >>>>>>       /* restore per cpu register */
> >>>>>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
> >>>>>>       addi.d    sp, sp, -PT_SIZE
> >>>>>> +    UNWIND_HINT_REGS
> >>>>>>         /* Prepare handle exception */
> >>>>>>       or    a0, s0, zero
> >>>>>> @@ -200,7 +202,7 @@ ret_to_host:
> >>>>>>       jr      ra
> >>>>>>     SYM_CODE_END(kvm_exc_entry)
> >>>>>> -EXPORT_SYMBOL(kvm_exc_entry)
> >>>>>> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
> >>>>>>     /*
> >>>>>>    * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu)
> >>>>>> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
> >>>>>>       /* Save host GPRs */
> >>>>>>       kvm_save_host_gpr a2
> >>>>>>   +    /*
> >>>>>> +     * The csr_era member variable of the pt_regs structure is
> >>>>>> required
> >>>>>> +     * for unwinding orc to perform stack traceback, so we need
> >>>>>> to put
> >>>>>> +     * pc into csr_era member variable here.
> >>>>>> +     */
> >>>>>> +    pcaddi    t0, 0
> >>>>>> +    st.d    t0, a2, PT_ERA
> >>>>> Hi, Xianglai,
> >>>>>
> >>>>> It should use `SYM_CODE_START` to mark the `kvm_enter_guest` rather
> >>>>> than
> >>>>> `SYM_FUNC_START`, since the `SYM_FUNC_START` is used to mark
> >>>>> "C-likely"
> >>>>> asm functionw.
> >>>>
> >>>> Ok, I will use SYM_CODE_START to mark kvm_enter_guest in the next
> >>>> version.
> >>>>
> >>>>> I guess the kvm_enter_guest is something like exception
> >>>>> handler becuase the last instruction is "ertn". So usually it should
> >>>>> mark UNWIND_HINT_REGS where can find last frame info by "$sp".
> >>>>> However, all info is store to "$a2", this mark should be
> >>>>>   `UNWIND_HINT sp_reg=ORC_REG_A2(???) type=UNWIND_HINT_TYPE_REGS`.
> >>>>> I don't konw why save this function internal PC here by `pcaddi t0,
> >>>>> 0`,
> >>>>> and I think it is no meaning(, for exception handler, they save
> >>>>> last PC
> >>>>> by read CSR.ERA). The `kvm_enter_guest` saves registers by
> >>>>> "$a2"("$sp" - PT_REGS) beyond stack ("$sp"), it is dangerous if IE
> >>>>> is enable. So I wonder if there is really a stacktrace through this
> >>>>> function?
> >>>>>
> >>>> The stack backtracking issue in switch.S is rather complex because
> >>>> it involves the switching between cpu root-mode and guest-mode:
> >>>> Real stack backtracking should be divided into two parts:
> >>>> part 1:
> >>>>     [<0>] kvm_enter_guest+0x38/0x11c
> >>>>     [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
> >>>>     [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
> >>>>     [<0>] sys_ioctl+0x498/0xf00
> >>>>     [<0>] do_syscall+0x98/0x1d0
> >>>>     [<0>] handle_syscall+0xb8/0x158
> >>>>
> >>>> part 2:
> >>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
> >>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
> >>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
> >>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
> >>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
> >>>>     [<0>] kvm_exc_entry+0x104/0x1e4
> >>>>
> >>>>
> >>>> In "part 1", after executing kvm_enter_guest, the cpu switches from
> >>>> root-mode to guest-mode.
> >>>> In this case, stack backtracking is indeed very rare.
> >>>>
> >>>> In "part 2", the cpu switches from the guest-mode to the root-mode,
> >>>> and most of the stack backtracking occurs during this phase.
> >>>>
> >>>> To obtain the longest call chain, we save pc in kvm_enter_guest to
> >>>> pt_regs.csr_era,
> >>>> and after restoring the sp of the root-mode cpu in kvm_exc_entry,
> >>>> The ORC entry was re-established using "UNWIND_HINT_REGS",
> >>>>  and then we obtained the following stack backtrace as we wanted:
> >>>>
> >>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
> >>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
> >>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
> >>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
> >>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
> >>>>     [<0>] kvm_exc_entry+0x104/0x1e4
> >>> I found this might be a coincidence—correct behavior due to the
> >>> incorrect
> >>> UNWIND_HINT_REGS mark and unusual stack adjustment.
> >>>
> >>> First, the kvm_enter_guest contains only a single branch instruction,
> >>> ertn.
> >>> It hardware-jump to the CSR.ERA address directly, jump into
> >>> kvm_exc_entry.
> >>>
> >>> At this point, the stack layout looks like this:
> >>> -------------------------------
> >>>   frame from call to `kvm_enter_guest`
> >>> -------------------------------  <- $sp
> >>>   PT_REGS
> >>> -------------------------------  <- $a2
> >>>
> >>> Then kvm_exc_entry adjust stack without save any register (e.g. $ra,
> >>> $sp)
> >>> but still marked UNWIND_HINT_REGS.
> >>> After the adjustment:
> >>> -------------------------------
> >>>   frame from call to `kvm_enter_guest`
> >>> -------------------------------
> >>>   PT_REGS
> >>> -------------------------------  <- $a2, new $sp
> >>>
> >>> During unwinding, when the unwinder reaches kvm_exc_entry,
> >>> it meets the mark of PT_REGS and correctly recovers
> >>>  pc = regs.csr_era, sp = regs.sp, ra = regs.ra
> >>>
> >> Yes, here unwinder does work as you say.
> >>
> >>> a) Can we avoid "ertn" rather than `jr reg (or jirl ra, reg, 0)`
> >>> instead, like call?
> >> No,  we need to rely on the 'ertn instruction return PIE to CRMD IE,
> >> at the same time to ensure that its atomic,
> >> there should be no other instruction than' ertn 'more appropriate here.
> > You are right! I got it.
> >>
> >>> The kvm_exc_entry cannot back to kvm_enter_guest
> >>> if we use "ertn", so should the kvm_enter_guest appear on the
> >>> stacktrace?
> >>>
> >>
> >> It is flexible. As I mentioned above, the cpu completes the switch
> >> from host-mode to guest mode through kvm_enter_guest,
> >> and then the switch from guest mode to host-mode through
> >> kvm_exc_entry. When we ignore the details of the host-mode
> >> and guest-mode switching in the middle, we can understand that the
> >> host cpu has completed kvm_enter_guest->kvm_exc_entry.
> >> From this perspective, I think it can exist in the call stack, and at
> >> the same time, we have obtained the maximum call stack information.
> >>
> >>
> >>> b) Can we adjust $sp before entering kvm_exc_entry? Then we can mark
> >>> UNWIND_HINT_REGS at the beginning of kvm_exc_entry, which something
> >>> like ret_from_kernel_thread_asm.
> >>>
> >> The following command can be used to dump the orc entries of the kernel:
> >> ./tools/objtool/objtool --dump vmlinux
> >>
> >> You can observe that not all orc entries are generated at the
> >> beginning of the function.
> >> For example:
> >> handle_tlb_protect
> >> ftrace_stub
> >> handle_reserved
> >>
> >> So, is it unnecessary for us to modify UNWIND_HINT_REGS in order to
> >> place it at the beginning of the function.
> >>
> >> If you have a better solution, could you provide an example of the
> >> modification?
> >> I can test the feasibility of the solution.
> >>
> > The expression at the beginning of the function is incorrect (feeling
> > sorry).
> > It should be marked where have all stacktrace info.
> > Thanks for all the explaining, since I'm unfamiliar with kvm, I need
> > these to help my understanding.
> >
> > Can you try with follows, with save regs by $sp, set more precise era to
> > pt_regs, and more unwind hint.
> >
> >
> > diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
> > index f1768b7a6194..8ed1d7b72c54 100644
> > --- a/arch/loongarch/kvm/switch.S
> > +++ b/arch/loongarch/kvm/switch.S
> > @@ -14,13 +14,13 @@
> >   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
> >
> >   .macro kvm_save_host_gpr base
> > -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
> > +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
> >       st.d    $r\n, \base, HGPR_OFFSET(\n)
> >       .endr
> >   .endm
> >
> >   .macro kvm_restore_host_gpr base
> > -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
> > +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
> >       ld.d    $r\n, \base, HGPR_OFFSET(\n)
> >       .endr
> >   .endm
> > @@ -88,6 +88,7 @@
> >       /* Load KVM_ARCH register */
> >       ld.d    a2, a2,    (KVM_ARCH_GGPR + 8 * REG_A2)
> >
> > +111:
> >       ertn /* Switch to guest: GSTAT.PGM = 1, ERRCTL.ISERR = 0,
> > TLBRPRMD.ISTLBR = 0 */
> >   .endm
> >
> > @@ -158,9 +159,10 @@ SYM_CODE_START(kvm_exc_entry)
> >       csrwr    t0, LOONGARCH_CSR_GTLBC
> >       ld.d    tp, a2, KVM_ARCH_HTP
> >       ld.d    sp, a2, KVM_ARCH_HSP
> > +    UNWIND_HINT_REGS
> > +
> >       /* restore per cpu register */
> >       ld.d    u0, a2, KVM_ARCH_HPERCPU
> > -    addi.d    sp, sp, -PT_SIZE
> >
> >       /* Prepare handle exception */
> >       or    a0, s0, zero
> > @@ -184,10 +186,11 @@ SYM_CODE_START(kvm_exc_entry)
> >       csrwr    s1, KVM_VCPU_KS
> >       kvm_switch_to_guest
> >
> > +    UNWIND_HINT_UNDEFINED
> >   ret_to_host:
> > -    ld.d    a2, a2, KVM_ARCH_HSP
> > -    addi.d  a2, a2, -PT_SIZE
> > -    kvm_restore_host_gpr    a2
> > +    ld.d    sp, a2, KVM_ARCH_HSP
> > +    kvm_restore_host_gpr    sp
> > +    addi.d    sp, sp, PT_SIZE
> >       jr      ra
> >
> >   SYM_INNER_LABEL(kvm_exc_entry_end, SYM_L_LOCAL)
> > @@ -200,11 +203,15 @@ SYM_CODE_END(kvm_exc_entry)
> >    *  a0: kvm_run* run
> >    *  a1: kvm_vcpu* vcpu
> >    */
> > -SYM_FUNC_START(kvm_enter_guest)
> > +SYM_CODE_START(kvm_enter_guest)
> > +    UNWIND_HINT_UNDEFINED
> >       /* Allocate space in stack bottom */
> > -    addi.d    a2, sp, -PT_SIZE
> > +    addi.d    sp, sp, -PT_SIZE
> >       /* Save host GPRs */
> > -    kvm_save_host_gpr a2
> > +    kvm_save_host_gpr sp
> > +    la.pcrel a2, 111f
> > +    st.d     a2, sp, PT_ERA
> > +    UNWIND_HINT_REGS
> >
> >       addi.d    a2, a1, KVM_VCPU_ARCH
> >       st.d    sp, a2, KVM_ARCH_HSP
> >
> wow, so quick and wonderful.
>
> Huacai,
>   the loongarch kernel and kvm maintainer, what is your point about this?
Xianglai's patch really has some problems. But Jinyang's solution is a
little big, can we minimize the changes? For example, don't touch
kvm_exc_entry, only modify kvm_enter_guest. In my opinion the current
problem has no relationship with kvm_exc_entry (but I'm not sure).

Huacai

>
> Regards
> Bibo Mao
> > Jinyang
> >
>
>

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[Bibo Mao]

On 2025/12/29 下午6:41, Jinyang He wrote:
> On 2025-12-29 18:11, lixianglai wrote:
> 
>> Hi Jinyang:
>>>
>>> On 2025-12-29 11:53, lixianglai wrote:
>>>> Hi Jinyang:
>>>>> On 2025-12-27 09:27, Xianglai Li wrote:
>>>>>
>>>>>> Insert the appropriate UNWIND macro definition into the 
>>>>>> kvm_exc_entry in
>>>>>> the assembly function to guide the generation of correct ORC table 
>>>>>> entries,
>>>>>> thereby solving the timeout problem of loading the 
>>>>>> livepatch-sample module
>>>>>> on a physical machine running multiple vcpus virtual machines.
>>>>>>
>>>>>> While solving the above problems, we have gained an additional 
>>>>>> benefit,
>>>>>> that is, we can obtain more call stack information
>>>>>>
>>>>>> Stack information that can be obtained before the problem is fixed:
>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>> [<0>] kvm_exc_entry+0x100/0x1e0
>>>>>>
>>>>>> Stack information that can be obtained after the problem is fixed:
>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>> [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>> [<0>] kvm_enter_guest+0x38/0x11c
>>>>>> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>>>> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>>>> [<0>] sys_ioctl+0x498/0xf00
>>>>>> [<0>] do_syscall+0x98/0x1d0
>>>>>> [<0>] handle_syscall+0xb8/0x158
>>>>>>
>>>>>> Cc: stable@vger.kernel.org
>>>>>> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
>>>>>> ---
>>>>>> Cc: Huacai Chen <chenhuacai@kernel.org>
>>>>>> Cc: WANG Xuerui <kernel@xen0n.name>
>>>>>> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
>>>>>> Cc: Bibo Mao <maobibo@loongson.cn>
>>>>>> Cc: Charlie Jenkins <charlie@rivosinc.com>
>>>>>> Cc: Xianglai Li <lixianglai@loongson.cn>
>>>>>> Cc: Thomas Gleixner <tglx@linutronix.de>
>>>>>> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
>>>>>>
>>>>>>   arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
>>>>>>   1 file changed, 19 insertions(+), 9 deletions(-)
>>>>>>
>>>>>> diff --git a/arch/loongarch/kvm/switch.S 
>>>>>> b/arch/loongarch/kvm/switch.S
>>>>>> index 93845ce53651..a3ea9567dbe5 100644
>>>>>> --- a/arch/loongarch/kvm/switch.S
>>>>>> +++ b/arch/loongarch/kvm/switch.S
>>>>>> @@ -10,6 +10,7 @@
>>>>>>   #include <asm/loongarch.h>
>>>>>>   #include <asm/regdef.h>
>>>>>>   #include <asm/unwind_hints.h>
>>>>>> +#include <linux/kvm_types.h>
>>>>>>     #define HGPR_OFFSET(x)        (PT_R0 + 8*x)
>>>>>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>>>>> @@ -110,9 +111,9 @@
>>>>>>        * need to copy world switch code to DMW area.
>>>>>>        */
>>>>>>       .text
>>>>>> +    .p2align PAGE_SHIFT
>>>>>>       .cfi_sections    .debug_frame
>>>>>>   SYM_CODE_START(kvm_exc_entry)
>>>>>> -    .p2align PAGE_SHIFT
>>>>>>       UNWIND_HINT_UNDEFINED
>>>>>>       csrwr    a2,   KVM_TEMP_KS
>>>>>>       csrrd    a2,   KVM_VCPU_KS
>>>>>> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
>>>>>>       /* restore per cpu register */
>>>>>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
>>>>>>       addi.d    sp, sp, -PT_SIZE
>>>>>> +    UNWIND_HINT_REGS
>>>>>>         /* Prepare handle exception */
>>>>>>       or    a0, s0, zero
>>>>>> @@ -200,7 +202,7 @@ ret_to_host:
>>>>>>       jr      ra
>>>>>>     SYM_CODE_END(kvm_exc_entry)
>>>>>> -EXPORT_SYMBOL(kvm_exc_entry)
>>>>>> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
>>>>>>     /*
>>>>>>    * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu)
>>>>>> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
>>>>>>       /* Save host GPRs */
>>>>>>       kvm_save_host_gpr a2
>>>>>>   +    /*
>>>>>> +     * The csr_era member variable of the pt_regs structure is 
>>>>>> required
>>>>>> +     * for unwinding orc to perform stack traceback, so we need 
>>>>>> to put
>>>>>> +     * pc into csr_era member variable here.
>>>>>> +     */
>>>>>> +    pcaddi    t0, 0
>>>>>> +    st.d    t0, a2, PT_ERA
>>>>> Hi, Xianglai,
>>>>>
>>>>> It should use `SYM_CODE_START` to mark the `kvm_enter_guest` rather 
>>>>> than
>>>>> `SYM_FUNC_START`, since the `SYM_FUNC_START` is used to mark 
>>>>> "C-likely"
>>>>> asm functionw. 
>>>>
>>>> Ok, I will use SYM_CODE_START to mark kvm_enter_guest in the next 
>>>> version.
>>>>
>>>>> I guess the kvm_enter_guest is something like exception
>>>>> handler becuase the last instruction is "ertn". So usually it should
>>>>> mark UNWIND_HINT_REGS where can find last frame info by "$sp".
>>>>> However, all info is store to "$a2", this mark should be
>>>>>   `UNWIND_HINT sp_reg=ORC_REG_A2(???) type=UNWIND_HINT_TYPE_REGS`.
>>>>> I don't konw why save this function internal PC here by `pcaddi t0, 
>>>>> 0`,
>>>>> and I think it is no meaning(, for exception handler, they save 
>>>>> last PC
>>>>> by read CSR.ERA). The `kvm_enter_guest` saves registers by
>>>>> "$a2"("$sp" - PT_REGS) beyond stack ("$sp"), it is dangerous if IE
>>>>> is enable. So I wonder if there is really a stacktrace through this 
>>>>> function?
>>>>>
>>>> The stack backtracking issue in switch.S is rather complex because 
>>>> it involves the switching between cpu root-mode and guest-mode:
>>>> Real stack backtracking should be divided into two parts:
>>>> part 1:
>>>>     [<0>] kvm_enter_guest+0x38/0x11c
>>>>     [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>>     [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>>     [<0>] sys_ioctl+0x498/0xf00
>>>>     [<0>] do_syscall+0x98/0x1d0
>>>>     [<0>] handle_syscall+0xb8/0x158
>>>>
>>>> part 2:
>>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>     [<0>] kvm_exc_entry+0x104/0x1e4
>>>>
>>>>
>>>> In "part 1", after executing kvm_enter_guest, the cpu switches from 
>>>> root-mode to guest-mode.
>>>> In this case, stack backtracking is indeed very rare.
>>>>
>>>> In "part 2", the cpu switches from the guest-mode to the root-mode,
>>>> and most of the stack backtracking occurs during this phase.
>>>>
>>>> To obtain the longest call chain, we save pc in kvm_enter_guest to 
>>>> pt_regs.csr_era,
>>>> and after restoring the sp of the root-mode cpu in kvm_exc_entry,
>>>> The ORC entry was re-established using "UNWIND_HINT_REGS",
>>>>  and then we obtained the following stack backtrace as we wanted:
>>>>
>>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>     [<0>] kvm_exc_entry+0x104/0x1e4
>>> I found this might be a coincidence—correct behavior due to the 
>>> incorrect
>>> UNWIND_HINT_REGS mark and unusual stack adjustment.
>>>
>>> First, the kvm_enter_guest contains only a single branch instruction, 
>>> ertn.
>>> It hardware-jump to the CSR.ERA address directly, jump into 
>>> kvm_exc_entry.
>>>
>>> At this point, the stack layout looks like this:
>>> -------------------------------
>>>   frame from call to `kvm_enter_guest`
>>> -------------------------------  <- $sp
>>>   PT_REGS
>>> -------------------------------  <- $a2
>>>
>>> Then kvm_exc_entry adjust stack without save any register (e.g. $ra, 
>>> $sp)
>>> but still marked UNWIND_HINT_REGS.
>>> After the adjustment:
>>> -------------------------------
>>>   frame from call to `kvm_enter_guest`
>>> -------------------------------
>>>   PT_REGS
>>> -------------------------------  <- $a2, new $sp
>>>
>>> During unwinding, when the unwinder reaches kvm_exc_entry,
>>> it meets the mark of PT_REGS and correctly recovers
>>>  pc = regs.csr_era, sp = regs.sp, ra = regs.ra
>>>
>> Yes, here unwinder does work as you say.
>>
>>> a) Can we avoid "ertn" rather than `jr reg (or jirl ra, reg, 0)`
>>> instead, like call?
>> No,  we need to rely on the 'ertn instruction return PIE to CRMD IE,
>> at the same time to ensure that its atomic,
>> there should be no other instruction than' ertn 'more appropriate here.
> You are right! I got it.
>>
>>> The kvm_exc_entry cannot back to kvm_enter_guest
>>> if we use "ertn", so should the kvm_enter_guest appear on the 
>>> stacktrace?
>>>
>>
>> It is flexible. As I mentioned above, the cpu completes the switch 
>> from host-mode to guest mode through kvm_enter_guest,
>> and then the switch from guest mode to host-mode through 
>> kvm_exc_entry. When we ignore the details of the host-mode
>> and guest-mode switching in the middle, we can understand that the 
>> host cpu has completed kvm_enter_guest->kvm_exc_entry.
>> From this perspective, I think it can exist in the call stack, and at 
>> the same time, we have obtained the maximum call stack information.
>>
>>
>>> b) Can we adjust $sp before entering kvm_exc_entry? Then we can mark
>>> UNWIND_HINT_REGS at the beginning of kvm_exc_entry, which something
>>> like ret_from_kernel_thread_asm.
>>>
>> The following command can be used to dump the orc entries of the kernel:
>> ./tools/objtool/objtool --dump vmlinux
>>
>> You can observe that not all orc entries are generated at the 
>> beginning of the function.
>> For example:
>> handle_tlb_protect
>> ftrace_stub
>> handle_reserved
>>
>> So, is it unnecessary for us to modify UNWIND_HINT_REGS in order to 
>> place it at the beginning of the function.
>>
>> If you have a better solution, could you provide an example of the 
>> modification?
>> I can test the feasibility of the solution.
>>
> The expression at the beginning of the function is incorrect (feeling 
> sorry).
> It should be marked where have all stacktrace info.
> Thanks for all the explaining, since I'm unfamiliar with kvm, I need 
> these to help my understanding.
> 
> Can you try with follows, with save regs by $sp, set more precise era to 
> pt_regs, and more unwind hint.
> 
> 
> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
> index f1768b7a6194..8ed1d7b72c54 100644
> --- a/arch/loongarch/kvm/switch.S
> +++ b/arch/loongarch/kvm/switch.S
> @@ -14,13 +14,13 @@
>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
> 
>   .macro kvm_save_host_gpr base
> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
>       st.d    $r\n, \base, HGPR_OFFSET(\n)
>       .endr
>   .endm
> 
>   .macro kvm_restore_host_gpr base
> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
>       ld.d    $r\n, \base, HGPR_OFFSET(\n)
>       .endr
>   .endm
> @@ -88,6 +88,7 @@
>       /* Load KVM_ARCH register */
>       ld.d    a2, a2,    (KVM_ARCH_GGPR + 8 * REG_A2)
> 
> +111:
>       ertn /* Switch to guest: GSTAT.PGM = 1, ERRCTL.ISERR = 0, 
> TLBRPRMD.ISTLBR = 0 */
>   .endm
> 
> @@ -158,9 +159,10 @@ SYM_CODE_START(kvm_exc_entry)
>       csrwr    t0, LOONGARCH_CSR_GTLBC
>       ld.d    tp, a2, KVM_ARCH_HTP
>       ld.d    sp, a2, KVM_ARCH_HSP
> +    UNWIND_HINT_REGS
> +
>       /* restore per cpu register */
>       ld.d    u0, a2, KVM_ARCH_HPERCPU
> -    addi.d    sp, sp, -PT_SIZE
> 
>       /* Prepare handle exception */
>       or    a0, s0, zero
> @@ -184,10 +186,11 @@ SYM_CODE_START(kvm_exc_entry)
>       csrwr    s1, KVM_VCPU_KS
>       kvm_switch_to_guest
> 
> +    UNWIND_HINT_UNDEFINED
>   ret_to_host:
> -    ld.d    a2, a2, KVM_ARCH_HSP
> -    addi.d  a2, a2, -PT_SIZE
> -    kvm_restore_host_gpr    a2
> +    ld.d    sp, a2, KVM_ARCH_HSP
> +    kvm_restore_host_gpr    sp
> +    addi.d    sp, sp, PT_SIZE
>       jr      ra
> 
>   SYM_INNER_LABEL(kvm_exc_entry_end, SYM_L_LOCAL)
> @@ -200,11 +203,15 @@ SYM_CODE_END(kvm_exc_entry)
>    *  a0: kvm_run* run
>    *  a1: kvm_vcpu* vcpu
>    */
> -SYM_FUNC_START(kvm_enter_guest)
> +SYM_CODE_START(kvm_enter_guest)
> +    UNWIND_HINT_UNDEFINED
>       /* Allocate space in stack bottom */
> -    addi.d    a2, sp, -PT_SIZE
> +    addi.d    sp, sp, -PT_SIZE
>       /* Save host GPRs */
> -    kvm_save_host_gpr a2
> +    kvm_save_host_gpr sp
> +    la.pcrel a2, 111f
> +    st.d     a2, sp, PT_ERA
> +    UNWIND_HINT_REGS
> 
why the label 111f is more accurate?  Supposing there is hw breakpoint 
here and backtrace is called, what is the call trace stack then? obvious 
label 111f is not executed instead.

UNWIND_HINT_REGS is used for nested kernel stack, is that right?
With nested interrupt and exception handlers on LoongArch kernel, is 
UNWIND_HINT_REGS used?

SYM_CODE_START(ret_from_fork_asm)
         UNWIND_HINT_REGS
         move            a1, sp
         bl              ret_from_fork
         STACKLEAK_ERASE
         RESTORE_STATIC
         RESTORE_SOME
         RESTORE_SP_AND_RET
SYM_CODE_END(ret_from_fork_asm)
With this piece of code, what is contents of pt_regs? In generic it is 
called from sys_clone, era is user PC address, is that right? If so,
what is detailed usage in the beginning of ret_from_fork_asm?

Regards
Bibo Mao

>       addi.d    a2, a1, KVM_VCPU_ARCH
>       st.d    sp, a2, KVM_ARCH_HSP
> 
> Jinyang
> 

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[Jinyang He]

On 2025-12-30 10:24, Bibo Mao wrote:

>
>
> On 2025/12/29 下午6:41, Jinyang He wrote:
>> On 2025-12-29 18:11, lixianglai wrote:
>>
>>> Hi Jinyang:
>>>>
>>>> On 2025-12-29 11:53, lixianglai wrote:
>>>>> Hi Jinyang:
>>>>>> On 2025-12-27 09:27, Xianglai Li wrote:
>>>>>>
>>>>>>> Insert the appropriate UNWIND macro definition into the 
>>>>>>> kvm_exc_entry in
>>>>>>> the assembly function to guide the generation of correct ORC 
>>>>>>> table entries,
>>>>>>> thereby solving the timeout problem of loading the 
>>>>>>> livepatch-sample module
>>>>>>> on a physical machine running multiple vcpus virtual machines.
>>>>>>>
>>>>>>> While solving the above problems, we have gained an additional 
>>>>>>> benefit,
>>>>>>> that is, we can obtain more call stack information
>>>>>>>
>>>>>>> Stack information that can be obtained before the problem is fixed:
>>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>> [<0>] kvm_exc_entry+0x100/0x1e0
>>>>>>>
>>>>>>> Stack information that can be obtained after the problem is fixed:
>>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>> [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>>> [<0>] kvm_enter_guest+0x38/0x11c
>>>>>>> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>>>>> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>>>>> [<0>] sys_ioctl+0x498/0xf00
>>>>>>> [<0>] do_syscall+0x98/0x1d0
>>>>>>> [<0>] handle_syscall+0xb8/0x158
>>>>>>>
>>>>>>> Cc: stable@vger.kernel.org
>>>>>>> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
>>>>>>> ---
>>>>>>> Cc: Huacai Chen <chenhuacai@kernel.org>
>>>>>>> Cc: WANG Xuerui <kernel@xen0n.name>
>>>>>>> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
>>>>>>> Cc: Bibo Mao <maobibo@loongson.cn>
>>>>>>> Cc: Charlie Jenkins <charlie@rivosinc.com>
>>>>>>> Cc: Xianglai Li <lixianglai@loongson.cn>
>>>>>>> Cc: Thomas Gleixner <tglx@linutronix.de>
>>>>>>> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
>>>>>>>
>>>>>>>   arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
>>>>>>>   1 file changed, 19 insertions(+), 9 deletions(-)
>>>>>>>
>>>>>>> diff --git a/arch/loongarch/kvm/switch.S 
>>>>>>> b/arch/loongarch/kvm/switch.S
>>>>>>> index 93845ce53651..a3ea9567dbe5 100644
>>>>>>> --- a/arch/loongarch/kvm/switch.S
>>>>>>> +++ b/arch/loongarch/kvm/switch.S
>>>>>>> @@ -10,6 +10,7 @@
>>>>>>>   #include <asm/loongarch.h>
>>>>>>>   #include <asm/regdef.h>
>>>>>>>   #include <asm/unwind_hints.h>
>>>>>>> +#include <linux/kvm_types.h>
>>>>>>>     #define HGPR_OFFSET(x)        (PT_R0 + 8*x)
>>>>>>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>>>>>> @@ -110,9 +111,9 @@
>>>>>>>        * need to copy world switch code to DMW area.
>>>>>>>        */
>>>>>>>       .text
>>>>>>> +    .p2align PAGE_SHIFT
>>>>>>>       .cfi_sections    .debug_frame
>>>>>>>   SYM_CODE_START(kvm_exc_entry)
>>>>>>> -    .p2align PAGE_SHIFT
>>>>>>>       UNWIND_HINT_UNDEFINED
>>>>>>>       csrwr    a2,   KVM_TEMP_KS
>>>>>>>       csrrd    a2,   KVM_VCPU_KS
>>>>>>> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
>>>>>>>       /* restore per cpu register */
>>>>>>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
>>>>>>>       addi.d    sp, sp, -PT_SIZE
>>>>>>> +    UNWIND_HINT_REGS
>>>>>>>         /* Prepare handle exception */
>>>>>>>       or    a0, s0, zero
>>>>>>> @@ -200,7 +202,7 @@ ret_to_host:
>>>>>>>       jr      ra
>>>>>>>     SYM_CODE_END(kvm_exc_entry)
>>>>>>> -EXPORT_SYMBOL(kvm_exc_entry)
>>>>>>> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
>>>>>>>     /*
>>>>>>>    * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu 
>>>>>>> *vcpu)
>>>>>>> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
>>>>>>>       /* Save host GPRs */
>>>>>>>       kvm_save_host_gpr a2
>>>>>>>   +    /*
>>>>>>> +     * The csr_era member variable of the pt_regs structure is 
>>>>>>> required
>>>>>>> +     * for unwinding orc to perform stack traceback, so we need 
>>>>>>> to put
>>>>>>> +     * pc into csr_era member variable here.
>>>>>>> +     */
>>>>>>> +    pcaddi    t0, 0
>>>>>>> +    st.d    t0, a2, PT_ERA
>>>>>> Hi, Xianglai,
>>>>>>
>>>>>> It should use `SYM_CODE_START` to mark the `kvm_enter_guest` 
>>>>>> rather than
>>>>>> `SYM_FUNC_START`, since the `SYM_FUNC_START` is used to mark 
>>>>>> "C-likely"
>>>>>> asm functionw. 
>>>>>
>>>>> Ok, I will use SYM_CODE_START to mark kvm_enter_guest in the next 
>>>>> version.
>>>>>
>>>>>> I guess the kvm_enter_guest is something like exception
>>>>>> handler becuase the last instruction is "ertn". So usually it should
>>>>>> mark UNWIND_HINT_REGS where can find last frame info by "$sp".
>>>>>> However, all info is store to "$a2", this mark should be
>>>>>>   `UNWIND_HINT sp_reg=ORC_REG_A2(???) type=UNWIND_HINT_TYPE_REGS`.
>>>>>> I don't konw why save this function internal PC here by `pcaddi 
>>>>>> t0, 0`,
>>>>>> and I think it is no meaning(, for exception handler, they save 
>>>>>> last PC
>>>>>> by read CSR.ERA). The `kvm_enter_guest` saves registers by
>>>>>> "$a2"("$sp" - PT_REGS) beyond stack ("$sp"), it is dangerous if IE
>>>>>> is enable. So I wonder if there is really a stacktrace through 
>>>>>> this function?
>>>>>>
>>>>> The stack backtracking issue in switch.S is rather complex because 
>>>>> it involves the switching between cpu root-mode and guest-mode:
>>>>> Real stack backtracking should be divided into two parts:
>>>>> part 1:
>>>>>     [<0>] kvm_enter_guest+0x38/0x11c
>>>>>     [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>>>     [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>>>     [<0>] sys_ioctl+0x498/0xf00
>>>>>     [<0>] do_syscall+0x98/0x1d0
>>>>>     [<0>] handle_syscall+0xb8/0x158
>>>>>
>>>>> part 2:
>>>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>     [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>
>>>>>
>>>>> In "part 1", after executing kvm_enter_guest, the cpu switches 
>>>>> from root-mode to guest-mode.
>>>>> In this case, stack backtracking is indeed very rare.
>>>>>
>>>>> In "part 2", the cpu switches from the guest-mode to the root-mode,
>>>>> and most of the stack backtracking occurs during this phase.
>>>>>
>>>>> To obtain the longest call chain, we save pc in kvm_enter_guest to 
>>>>> pt_regs.csr_era,
>>>>> and after restoring the sp of the root-mode cpu in kvm_exc_entry,
>>>>> The ORC entry was re-established using "UNWIND_HINT_REGS",
>>>>>  and then we obtained the following stack backtrace as we wanted:
>>>>>
>>>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>     [<0>] kvm_exc_entry+0x104/0x1e4
>>>> I found this might be a coincidence—correct behavior due to the 
>>>> incorrect
>>>> UNWIND_HINT_REGS mark and unusual stack adjustment.
>>>>
>>>> First, the kvm_enter_guest contains only a single branch 
>>>> instruction, ertn.
>>>> It hardware-jump to the CSR.ERA address directly, jump into 
>>>> kvm_exc_entry.
>>>>
>>>> At this point, the stack layout looks like this:
>>>> -------------------------------
>>>>   frame from call to `kvm_enter_guest`
>>>> -------------------------------  <- $sp
>>>>   PT_REGS
>>>> -------------------------------  <- $a2
>>>>
>>>> Then kvm_exc_entry adjust stack without save any register (e.g. 
>>>> $ra, $sp)
>>>> but still marked UNWIND_HINT_REGS.
>>>> After the adjustment:
>>>> -------------------------------
>>>>   frame from call to `kvm_enter_guest`
>>>> -------------------------------
>>>>   PT_REGS
>>>> -------------------------------  <- $a2, new $sp
>>>>
>>>> During unwinding, when the unwinder reaches kvm_exc_entry,
>>>> it meets the mark of PT_REGS and correctly recovers
>>>>  pc = regs.csr_era, sp = regs.sp, ra = regs.ra
>>>>
>>> Yes, here unwinder does work as you say.
>>>
>>>> a) Can we avoid "ertn" rather than `jr reg (or jirl ra, reg, 0)`
>>>> instead, like call?
>>> No,  we need to rely on the 'ertn instruction return PIE to CRMD IE,
>>> at the same time to ensure that its atomic,
>>> there should be no other instruction than' ertn 'more appropriate here.
>> You are right! I got it.
>>>
>>>> The kvm_exc_entry cannot back to kvm_enter_guest
>>>> if we use "ertn", so should the kvm_enter_guest appear on the 
>>>> stacktrace?
>>>>
>>>
>>> It is flexible. As I mentioned above, the cpu completes the switch 
>>> from host-mode to guest mode through kvm_enter_guest,
>>> and then the switch from guest mode to host-mode through 
>>> kvm_exc_entry. When we ignore the details of the host-mode
>>> and guest-mode switching in the middle, we can understand that the 
>>> host cpu has completed kvm_enter_guest->kvm_exc_entry.
>>> From this perspective, I think it can exist in the call stack, and 
>>> at the same time, we have obtained the maximum call stack information.
>>>
>>>
>>>> b) Can we adjust $sp before entering kvm_exc_entry? Then we can mark
>>>> UNWIND_HINT_REGS at the beginning of kvm_exc_entry, which something
>>>> like ret_from_kernel_thread_asm.
>>>>
>>> The following command can be used to dump the orc entries of the 
>>> kernel:
>>> ./tools/objtool/objtool --dump vmlinux
>>>
>>> You can observe that not all orc entries are generated at the 
>>> beginning of the function.
>>> For example:
>>> handle_tlb_protect
>>> ftrace_stub
>>> handle_reserved
>>>
>>> So, is it unnecessary for us to modify UNWIND_HINT_REGS in order to 
>>> place it at the beginning of the function.
>>>
>>> If you have a better solution, could you provide an example of the 
>>> modification?
>>> I can test the feasibility of the solution.
>>>
>> The expression at the beginning of the function is incorrect (feeling 
>> sorry).
>> It should be marked where have all stacktrace info.
>> Thanks for all the explaining, since I'm unfamiliar with kvm, I need 
>> these to help my understanding.
>>
>> Can you try with follows, with save regs by $sp, set more precise era 
>> to pt_regs, and more unwind hint.
>>
>>
>> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
>> index f1768b7a6194..8ed1d7b72c54 100644
>> --- a/arch/loongarch/kvm/switch.S
>> +++ b/arch/loongarch/kvm/switch.S
>> @@ -14,13 +14,13 @@
>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>
>>   .macro kvm_save_host_gpr base
>> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
>> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
>>       st.d    $r\n, \base, HGPR_OFFSET(\n)
>>       .endr
>>   .endm
>>
>>   .macro kvm_restore_host_gpr base
>> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
>> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
>>       ld.d    $r\n, \base, HGPR_OFFSET(\n)
>>       .endr
>>   .endm
>> @@ -88,6 +88,7 @@
>>       /* Load KVM_ARCH register */
>>       ld.d    a2, a2,    (KVM_ARCH_GGPR + 8 * REG_A2)
>>
>> +111:
>>       ertn /* Switch to guest: GSTAT.PGM = 1, ERRCTL.ISERR = 0, 
>> TLBRPRMD.ISTLBR = 0 */
>>   .endm
>>
>> @@ -158,9 +159,10 @@ SYM_CODE_START(kvm_exc_entry)
>>       csrwr    t0, LOONGARCH_CSR_GTLBC
>>       ld.d    tp, a2, KVM_ARCH_HTP
>>       ld.d    sp, a2, KVM_ARCH_HSP
>> +    UNWIND_HINT_REGS
>> +
>>       /* restore per cpu register */
>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
>> -    addi.d    sp, sp, -PT_SIZE
>>
>>       /* Prepare handle exception */
>>       or    a0, s0, zero
>> @@ -184,10 +186,11 @@ SYM_CODE_START(kvm_exc_entry)
>>       csrwr    s1, KVM_VCPU_KS
>>       kvm_switch_to_guest
>>
>> +    UNWIND_HINT_UNDEFINED
>>   ret_to_host:
>> -    ld.d    a2, a2, KVM_ARCH_HSP
>> -    addi.d  a2, a2, -PT_SIZE
>> -    kvm_restore_host_gpr    a2
>> +    ld.d    sp, a2, KVM_ARCH_HSP
>> +    kvm_restore_host_gpr    sp
>> +    addi.d    sp, sp, PT_SIZE
>>       jr      ra
>>
>>   SYM_INNER_LABEL(kvm_exc_entry_end, SYM_L_LOCAL)
>> @@ -200,11 +203,15 @@ SYM_CODE_END(kvm_exc_entry)
>>    *  a0: kvm_run* run
>>    *  a1: kvm_vcpu* vcpu
>>    */
>> -SYM_FUNC_START(kvm_enter_guest)
>> +SYM_CODE_START(kvm_enter_guest)
>> +    UNWIND_HINT_UNDEFINED
>>       /* Allocate space in stack bottom */
>> -    addi.d    a2, sp, -PT_SIZE
>> +    addi.d    sp, sp, -PT_SIZE
>>       /* Save host GPRs */
>> -    kvm_save_host_gpr a2
>> +    kvm_save_host_gpr sp
>> +    la.pcrel a2, 111f
>> +    st.d     a2, sp, PT_ERA
>> +    UNWIND_HINT_REGS
>>
> why the label 111f is more accurate?  Supposing there is hw breakpoint 
> here and backtrace is called, what is the call trace stack then? 
> obvious label 111f is not executed instead.
Xianglai said marking it as regs can get more stack infos, so I use
UNWIND_HINT_REGS marked here, though it not called. Remove
UNWIND_HINT_REGS thenforbid unwind from here.
This function is called and should usually be marked as "call",
but it is complex by switching the stack and use `ertn` calls
another function.


>
> UNWIND_HINT_REGS is used for nested kernel stack, is that right?
> With nested interrupt and exception handlers on LoongArch kernel, is 
> UNWIND_HINT_REGS used?
>
> SYM_CODE_START(ret_from_fork_asm)
>         UNWIND_HINT_REGS
>         move            a1, sp
>         bl              ret_from_fork
>         STACKLEAK_ERASE
>         RESTORE_STATIC
>         RESTORE_SOME
>         RESTORE_SP_AND_RET
> SYM_CODE_END(ret_from_fork_asm)
> With this piece of code, what is contents of pt_regs? In generic it is 
> called from sys_clone, era is user PC address, is that right? If so,
> what is detailed usage in the beginning of ret_from_fork_asm?
The stacktrace shows the control flow where the PC will go back, so
it is right because when PC is in ret_from_fork_asm, it is already
another thread. The era means it will go back user mode.

Jinyang
>
> Regards
> Bibo Mao
>
>>       addi.d    a2, a1, KVM_VCPU_ARCH
>>       st.d    sp, a2, KVM_ARCH_HSP
>>
>> Jinyang
>>

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[Bibo Mao]

On 2025/12/30 上午11:36, Jinyang He wrote:
> On 2025-12-30 10:24, Bibo Mao wrote:
> 
>>
>>
>> On 2025/12/29 下午6:41, Jinyang He wrote:
>>> On 2025-12-29 18:11, lixianglai wrote:
>>>
>>>> Hi Jinyang:
>>>>>
>>>>> On 2025-12-29 11:53, lixianglai wrote:
>>>>>> Hi Jinyang:
>>>>>>> On 2025-12-27 09:27, Xianglai Li wrote:
>>>>>>>
>>>>>>>> Insert the appropriate UNWIND macro definition into the 
>>>>>>>> kvm_exc_entry in
>>>>>>>> the assembly function to guide the generation of correct ORC 
>>>>>>>> table entries,
>>>>>>>> thereby solving the timeout problem of loading the 
>>>>>>>> livepatch-sample module
>>>>>>>> on a physical machine running multiple vcpus virtual machines.
>>>>>>>>
>>>>>>>> While solving the above problems, we have gained an additional 
>>>>>>>> benefit,
>>>>>>>> that is, we can obtain more call stack information
>>>>>>>>
>>>>>>>> Stack information that can be obtained before the problem is fixed:
>>>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>> [<0>] kvm_exc_entry+0x100/0x1e0
>>>>>>>>
>>>>>>>> Stack information that can be obtained after the problem is fixed:
>>>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>> [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>>>> [<0>] kvm_enter_guest+0x38/0x11c
>>>>>>>> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>>>>>> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>>>>>> [<0>] sys_ioctl+0x498/0xf00
>>>>>>>> [<0>] do_syscall+0x98/0x1d0
>>>>>>>> [<0>] handle_syscall+0xb8/0x158
>>>>>>>>
>>>>>>>> Cc: stable@vger.kernel.org
>>>>>>>> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
>>>>>>>> ---
>>>>>>>> Cc: Huacai Chen <chenhuacai@kernel.org>
>>>>>>>> Cc: WANG Xuerui <kernel@xen0n.name>
>>>>>>>> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
>>>>>>>> Cc: Bibo Mao <maobibo@loongson.cn>
>>>>>>>> Cc: Charlie Jenkins <charlie@rivosinc.com>
>>>>>>>> Cc: Xianglai Li <lixianglai@loongson.cn>
>>>>>>>> Cc: Thomas Gleixner <tglx@linutronix.de>
>>>>>>>> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
>>>>>>>>
>>>>>>>>   arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
>>>>>>>>   1 file changed, 19 insertions(+), 9 deletions(-)
>>>>>>>>
>>>>>>>> diff --git a/arch/loongarch/kvm/switch.S 
>>>>>>>> b/arch/loongarch/kvm/switch.S
>>>>>>>> index 93845ce53651..a3ea9567dbe5 100644
>>>>>>>> --- a/arch/loongarch/kvm/switch.S
>>>>>>>> +++ b/arch/loongarch/kvm/switch.S
>>>>>>>> @@ -10,6 +10,7 @@
>>>>>>>>   #include <asm/loongarch.h>
>>>>>>>>   #include <asm/regdef.h>
>>>>>>>>   #include <asm/unwind_hints.h>
>>>>>>>> +#include <linux/kvm_types.h>
>>>>>>>>     #define HGPR_OFFSET(x)        (PT_R0 + 8*x)
>>>>>>>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>>>>>>> @@ -110,9 +111,9 @@
>>>>>>>>        * need to copy world switch code to DMW area.
>>>>>>>>        */
>>>>>>>>       .text
>>>>>>>> +    .p2align PAGE_SHIFT
>>>>>>>>       .cfi_sections    .debug_frame
>>>>>>>>   SYM_CODE_START(kvm_exc_entry)
>>>>>>>> -    .p2align PAGE_SHIFT
>>>>>>>>       UNWIND_HINT_UNDEFINED
>>>>>>>>       csrwr    a2,   KVM_TEMP_KS
>>>>>>>>       csrrd    a2,   KVM_VCPU_KS
>>>>>>>> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
>>>>>>>>       /* restore per cpu register */
>>>>>>>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
>>>>>>>>       addi.d    sp, sp, -PT_SIZE
>>>>>>>> +    UNWIND_HINT_REGS
>>>>>>>>         /* Prepare handle exception */
>>>>>>>>       or    a0, s0, zero
>>>>>>>> @@ -200,7 +202,7 @@ ret_to_host:
>>>>>>>>       jr      ra
>>>>>>>>     SYM_CODE_END(kvm_exc_entry)
>>>>>>>> -EXPORT_SYMBOL(kvm_exc_entry)
>>>>>>>> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
>>>>>>>>     /*
>>>>>>>>    * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu 
>>>>>>>> *vcpu)
>>>>>>>> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
>>>>>>>>       /* Save host GPRs */
>>>>>>>>       kvm_save_host_gpr a2
>>>>>>>>   +    /*
>>>>>>>> +     * The csr_era member variable of the pt_regs structure is 
>>>>>>>> required
>>>>>>>> +     * for unwinding orc to perform stack traceback, so we need 
>>>>>>>> to put
>>>>>>>> +     * pc into csr_era member variable here.
>>>>>>>> +     */
>>>>>>>> +    pcaddi    t0, 0
>>>>>>>> +    st.d    t0, a2, PT_ERA
>>>>>>> Hi, Xianglai,
>>>>>>>
>>>>>>> It should use `SYM_CODE_START` to mark the `kvm_enter_guest` 
>>>>>>> rather than
>>>>>>> `SYM_FUNC_START`, since the `SYM_FUNC_START` is used to mark 
>>>>>>> "C-likely"
>>>>>>> asm functionw. 
>>>>>>
>>>>>> Ok, I will use SYM_CODE_START to mark kvm_enter_guest in the next 
>>>>>> version.
>>>>>>
>>>>>>> I guess the kvm_enter_guest is something like exception
>>>>>>> handler becuase the last instruction is "ertn". So usually it should
>>>>>>> mark UNWIND_HINT_REGS where can find last frame info by "$sp".
>>>>>>> However, all info is store to "$a2", this mark should be
>>>>>>>   `UNWIND_HINT sp_reg=ORC_REG_A2(???) type=UNWIND_HINT_TYPE_REGS`.
>>>>>>> I don't konw why save this function internal PC here by `pcaddi 
>>>>>>> t0, 0`,
>>>>>>> and I think it is no meaning(, for exception handler, they save 
>>>>>>> last PC
>>>>>>> by read CSR.ERA). The `kvm_enter_guest` saves registers by
>>>>>>> "$a2"("$sp" - PT_REGS) beyond stack ("$sp"), it is dangerous if IE
>>>>>>> is enable. So I wonder if there is really a stacktrace through 
>>>>>>> this function?
>>>>>>>
>>>>>> The stack backtracking issue in switch.S is rather complex because 
>>>>>> it involves the switching between cpu root-mode and guest-mode:
>>>>>> Real stack backtracking should be divided into two parts:
>>>>>> part 1:
>>>>>>     [<0>] kvm_enter_guest+0x38/0x11c
>>>>>>     [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>>>>     [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>>>>     [<0>] sys_ioctl+0x498/0xf00
>>>>>>     [<0>] do_syscall+0x98/0x1d0
>>>>>>     [<0>] handle_syscall+0xb8/0x158
>>>>>>
>>>>>> part 2:
>>>>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>     [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>>
>>>>>>
>>>>>> In "part 1", after executing kvm_enter_guest, the cpu switches 
>>>>>> from root-mode to guest-mode.
>>>>>> In this case, stack backtracking is indeed very rare.
>>>>>>
>>>>>> In "part 2", the cpu switches from the guest-mode to the root-mode,
>>>>>> and most of the stack backtracking occurs during this phase.
>>>>>>
>>>>>> To obtain the longest call chain, we save pc in kvm_enter_guest to 
>>>>>> pt_regs.csr_era,
>>>>>> and after restoring the sp of the root-mode cpu in kvm_exc_entry,
>>>>>> The ORC entry was re-established using "UNWIND_HINT_REGS",
>>>>>>  and then we obtained the following stack backtrace as we wanted:
>>>>>>
>>>>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>     [<0>] kvm_exc_entry+0x104/0x1e4
>>>>> I found this might be a coincidence—correct behavior due to the 
>>>>> incorrect
>>>>> UNWIND_HINT_REGS mark and unusual stack adjustment.
>>>>>
>>>>> First, the kvm_enter_guest contains only a single branch 
>>>>> instruction, ertn.
>>>>> It hardware-jump to the CSR.ERA address directly, jump into 
>>>>> kvm_exc_entry.
>>>>>
>>>>> At this point, the stack layout looks like this:
>>>>> -------------------------------
>>>>>   frame from call to `kvm_enter_guest`
>>>>> -------------------------------  <- $sp
>>>>>   PT_REGS
>>>>> -------------------------------  <- $a2
>>>>>
>>>>> Then kvm_exc_entry adjust stack without save any register (e.g. 
>>>>> $ra, $sp)
>>>>> but still marked UNWIND_HINT_REGS.
>>>>> After the adjustment:
>>>>> -------------------------------
>>>>>   frame from call to `kvm_enter_guest`
>>>>> -------------------------------
>>>>>   PT_REGS
>>>>> -------------------------------  <- $a2, new $sp
>>>>>
>>>>> During unwinding, when the unwinder reaches kvm_exc_entry,
>>>>> it meets the mark of PT_REGS and correctly recovers
>>>>>  pc = regs.csr_era, sp = regs.sp, ra = regs.ra
>>>>>
>>>> Yes, here unwinder does work as you say.
>>>>
>>>>> a) Can we avoid "ertn" rather than `jr reg (or jirl ra, reg, 0)`
>>>>> instead, like call?
>>>> No,  we need to rely on the 'ertn instruction return PIE to CRMD IE,
>>>> at the same time to ensure that its atomic,
>>>> there should be no other instruction than' ertn 'more appropriate here.
>>> You are right! I got it.
>>>>
>>>>> The kvm_exc_entry cannot back to kvm_enter_guest
>>>>> if we use "ertn", so should the kvm_enter_guest appear on the 
>>>>> stacktrace?
>>>>>
>>>>
>>>> It is flexible. As I mentioned above, the cpu completes the switch 
>>>> from host-mode to guest mode through kvm_enter_guest,
>>>> and then the switch from guest mode to host-mode through 
>>>> kvm_exc_entry. When we ignore the details of the host-mode
>>>> and guest-mode switching in the middle, we can understand that the 
>>>> host cpu has completed kvm_enter_guest->kvm_exc_entry.
>>>> From this perspective, I think it can exist in the call stack, and 
>>>> at the same time, we have obtained the maximum call stack information.
>>>>
>>>>
>>>>> b) Can we adjust $sp before entering kvm_exc_entry? Then we can mark
>>>>> UNWIND_HINT_REGS at the beginning of kvm_exc_entry, which something
>>>>> like ret_from_kernel_thread_asm.
>>>>>
>>>> The following command can be used to dump the orc entries of the 
>>>> kernel:
>>>> ./tools/objtool/objtool --dump vmlinux
>>>>
>>>> You can observe that not all orc entries are generated at the 
>>>> beginning of the function.
>>>> For example:
>>>> handle_tlb_protect
>>>> ftrace_stub
>>>> handle_reserved
>>>>
>>>> So, is it unnecessary for us to modify UNWIND_HINT_REGS in order to 
>>>> place it at the beginning of the function.
>>>>
>>>> If you have a better solution, could you provide an example of the 
>>>> modification?
>>>> I can test the feasibility of the solution.
>>>>
>>> The expression at the beginning of the function is incorrect (feeling 
>>> sorry).
>>> It should be marked where have all stacktrace info.
>>> Thanks for all the explaining, since I'm unfamiliar with kvm, I need 
>>> these to help my understanding.
>>>
>>> Can you try with follows, with save regs by $sp, set more precise era 
>>> to pt_regs, and more unwind hint.
>>>
>>>
>>> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
>>> index f1768b7a6194..8ed1d7b72c54 100644
>>> --- a/arch/loongarch/kvm/switch.S
>>> +++ b/arch/loongarch/kvm/switch.S
>>> @@ -14,13 +14,13 @@
>>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>>
>>>   .macro kvm_save_host_gpr base
>>> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
>>> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
>>>       st.d    $r\n, \base, HGPR_OFFSET(\n)
>>>       .endr
>>>   .endm
>>>
>>>   .macro kvm_restore_host_gpr base
>>> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
>>> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
>>>       ld.d    $r\n, \base, HGPR_OFFSET(\n)
>>>       .endr
>>>   .endm
>>> @@ -88,6 +88,7 @@
>>>       /* Load KVM_ARCH register */
>>>       ld.d    a2, a2,    (KVM_ARCH_GGPR + 8 * REG_A2)
>>>
>>> +111:
>>>       ertn /* Switch to guest: GSTAT.PGM = 1, ERRCTL.ISERR = 0, 
>>> TLBRPRMD.ISTLBR = 0 */
>>>   .endm
>>>
>>> @@ -158,9 +159,10 @@ SYM_CODE_START(kvm_exc_entry)
>>>       csrwr    t0, LOONGARCH_CSR_GTLBC
>>>       ld.d    tp, a2, KVM_ARCH_HTP
>>>       ld.d    sp, a2, KVM_ARCH_HSP
>>> +    UNWIND_HINT_REGS
>>> +
>>>       /* restore per cpu register */
>>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
>>> -    addi.d    sp, sp, -PT_SIZE
>>>
>>>       /* Prepare handle exception */
>>>       or    a0, s0, zero
>>> @@ -184,10 +186,11 @@ SYM_CODE_START(kvm_exc_entry)
>>>       csrwr    s1, KVM_VCPU_KS
>>>       kvm_switch_to_guest
>>>
>>> +    UNWIND_HINT_UNDEFINED
>>>   ret_to_host:
>>> -    ld.d    a2, a2, KVM_ARCH_HSP
>>> -    addi.d  a2, a2, -PT_SIZE
>>> -    kvm_restore_host_gpr    a2
>>> +    ld.d    sp, a2, KVM_ARCH_HSP
>>> +    kvm_restore_host_gpr    sp
>>> +    addi.d    sp, sp, PT_SIZE
>>>       jr      ra
>>>
>>>   SYM_INNER_LABEL(kvm_exc_entry_end, SYM_L_LOCAL)
>>> @@ -200,11 +203,15 @@ SYM_CODE_END(kvm_exc_entry)
>>>    *  a0: kvm_run* run
>>>    *  a1: kvm_vcpu* vcpu
>>>    */
>>> -SYM_FUNC_START(kvm_enter_guest)
>>> +SYM_CODE_START(kvm_enter_guest)
>>> +    UNWIND_HINT_UNDEFINED
>>>       /* Allocate space in stack bottom */
>>> -    addi.d    a2, sp, -PT_SIZE
>>> +    addi.d    sp, sp, -PT_SIZE
>>>       /* Save host GPRs */
>>> -    kvm_save_host_gpr a2
>>> +    kvm_save_host_gpr sp
>>> +    la.pcrel a2, 111f
>>> +    st.d     a2, sp, PT_ERA
>>> +    UNWIND_HINT_REGS
>>>
>> why the label 111f is more accurate?  Supposing there is hw breakpoint 
>> here and backtrace is called, what is the call trace stack then? 
>> obvious label 111f is not executed instead.
> Xianglai said marking it as regs can get more stack infos, so I use
> UNWIND_HINT_REGS marked here, though it not called. Remove
> UNWIND_HINT_REGS thenforbid unwind from here.
> This function is called and should usually be marked as "call",
> but it is complex by switching the stack and use `ertn` calls
> another function.
> 
> 
>>
>> UNWIND_HINT_REGS is used for nested kernel stack, is that right?
>> With nested interrupt and exception handlers on LoongArch kernel, is 
>> UNWIND_HINT_REGS used?
>>
>> SYM_CODE_START(ret_from_fork_asm)
>>         UNWIND_HINT_REGS
>>         move            a1, sp
>>         bl              ret_from_fork
>>         STACKLEAK_ERASE
>>         RESTORE_STATIC
>>         RESTORE_SOME
>>         RESTORE_SP_AND_RET
>> SYM_CODE_END(ret_from_fork_asm)
>> With this piece of code, what is contents of pt_regs? In generic it is 
>> called from sys_clone, era is user PC address, is that right? If so,
>> what is detailed usage in the beginning of ret_from_fork_asm?
> The stacktrace shows the control flow where the PC will go back, so
> it is right because when PC is in ret_from_fork_asm, it is already
> another thread. The era means it will go back user mode.
The problem is that user mode era shows unwind with error, and 
user_mode(regs) is not accurate. here is piece of code.
                 pc = regs->csr_era;
                 if (!__kernel_text_address(pc))
                         goto err;
will UNWIND_HINT_END_OF_STACK be better than UNWIND_HINT_REGS?

Regards
Bibo Mao
> 
> Jinyang
>>
>> Regards
>> Bibo Mao
>>
>>>       addi.d    a2, a1, KVM_VCPU_ARCH
>>>       st.d    sp, a2, KVM_ARCH_HSP
>>>
>>> Jinyang
>>>
> 

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[Jinyang He]

On 2025-12-30 12:03, Bibo Mao wrote:

>
>
> On 2025/12/30 上午11:36, Jinyang He wrote:
>> On 2025-12-30 10:24, Bibo Mao wrote:
>>
>>>
>>>
>>> On 2025/12/29 下午6:41, Jinyang He wrote:
>>>> On 2025-12-29 18:11, lixianglai wrote:
>>>>
>>>>> Hi Jinyang:
>>>>>>
>>>>>> On 2025-12-29 11:53, lixianglai wrote:
>>>>>>> Hi Jinyang:
>>>>>>>> On 2025-12-27 09:27, Xianglai Li wrote:
>>>>>>>>
>>>>>>>>> Insert the appropriate UNWIND macro definition into the 
>>>>>>>>> kvm_exc_entry in
>>>>>>>>> the assembly function to guide the generation of correct ORC 
>>>>>>>>> table entries,
>>>>>>>>> thereby solving the timeout problem of loading the 
>>>>>>>>> livepatch-sample module
>>>>>>>>> on a physical machine running multiple vcpus virtual machines.
>>>>>>>>>
>>>>>>>>> While solving the above problems, we have gained an additional 
>>>>>>>>> benefit,
>>>>>>>>> that is, we can obtain more call stack information
>>>>>>>>>
>>>>>>>>> Stack information that can be obtained before the problem is 
>>>>>>>>> fixed:
>>>>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>>> [<0>] kvm_exc_entry+0x100/0x1e0
>>>>>>>>>
>>>>>>>>> Stack information that can be obtained after the problem is 
>>>>>>>>> fixed:
>>>>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>>> [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>>>>> [<0>] kvm_enter_guest+0x38/0x11c
>>>>>>>>> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>>>>>>> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>>>>>>> [<0>] sys_ioctl+0x498/0xf00
>>>>>>>>> [<0>] do_syscall+0x98/0x1d0
>>>>>>>>> [<0>] handle_syscall+0xb8/0x158
>>>>>>>>>
>>>>>>>>> Cc: stable@vger.kernel.org
>>>>>>>>> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
>>>>>>>>> ---
>>>>>>>>> Cc: Huacai Chen <chenhuacai@kernel.org>
>>>>>>>>> Cc: WANG Xuerui <kernel@xen0n.name>
>>>>>>>>> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
>>>>>>>>> Cc: Bibo Mao <maobibo@loongson.cn>
>>>>>>>>> Cc: Charlie Jenkins <charlie@rivosinc.com>
>>>>>>>>> Cc: Xianglai Li <lixianglai@loongson.cn>
>>>>>>>>> Cc: Thomas Gleixner <tglx@linutronix.de>
>>>>>>>>> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
>>>>>>>>>
>>>>>>>>>   arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
>>>>>>>>>   1 file changed, 19 insertions(+), 9 deletions(-)
>>>>>>>>>
>>>>>>>>> diff --git a/arch/loongarch/kvm/switch.S 
>>>>>>>>> b/arch/loongarch/kvm/switch.S
>>>>>>>>> index 93845ce53651..a3ea9567dbe5 100644
>>>>>>>>> --- a/arch/loongarch/kvm/switch.S
>>>>>>>>> +++ b/arch/loongarch/kvm/switch.S
>>>>>>>>> @@ -10,6 +10,7 @@
>>>>>>>>>   #include <asm/loongarch.h>
>>>>>>>>>   #include <asm/regdef.h>
>>>>>>>>>   #include <asm/unwind_hints.h>
>>>>>>>>> +#include <linux/kvm_types.h>
>>>>>>>>>     #define HGPR_OFFSET(x)        (PT_R0 + 8*x)
>>>>>>>>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>>>>>>>> @@ -110,9 +111,9 @@
>>>>>>>>>        * need to copy world switch code to DMW area.
>>>>>>>>>        */
>>>>>>>>>       .text
>>>>>>>>> +    .p2align PAGE_SHIFT
>>>>>>>>>       .cfi_sections    .debug_frame
>>>>>>>>>   SYM_CODE_START(kvm_exc_entry)
>>>>>>>>> -    .p2align PAGE_SHIFT
>>>>>>>>>       UNWIND_HINT_UNDEFINED
>>>>>>>>>       csrwr    a2,   KVM_TEMP_KS
>>>>>>>>>       csrrd    a2,   KVM_VCPU_KS
>>>>>>>>> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
>>>>>>>>>       /* restore per cpu register */
>>>>>>>>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
>>>>>>>>>       addi.d    sp, sp, -PT_SIZE
>>>>>>>>> +    UNWIND_HINT_REGS
>>>>>>>>>         /* Prepare handle exception */
>>>>>>>>>       or    a0, s0, zero
>>>>>>>>> @@ -200,7 +202,7 @@ ret_to_host:
>>>>>>>>>       jr      ra
>>>>>>>>>     SYM_CODE_END(kvm_exc_entry)
>>>>>>>>> -EXPORT_SYMBOL(kvm_exc_entry)
>>>>>>>>> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
>>>>>>>>>     /*
>>>>>>>>>    * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu 
>>>>>>>>> *vcpu)
>>>>>>>>> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
>>>>>>>>>       /* Save host GPRs */
>>>>>>>>>       kvm_save_host_gpr a2
>>>>>>>>>   +    /*
>>>>>>>>> +     * The csr_era member variable of the pt_regs structure 
>>>>>>>>> is required
>>>>>>>>> +     * for unwinding orc to perform stack traceback, so we 
>>>>>>>>> need to put
>>>>>>>>> +     * pc into csr_era member variable here.
>>>>>>>>> +     */
>>>>>>>>> +    pcaddi    t0, 0
>>>>>>>>> +    st.d    t0, a2, PT_ERA
>>>>>>>> Hi, Xianglai,
>>>>>>>>
>>>>>>>> It should use `SYM_CODE_START` to mark the `kvm_enter_guest` 
>>>>>>>> rather than
>>>>>>>> `SYM_FUNC_START`, since the `SYM_FUNC_START` is used to mark 
>>>>>>>> "C-likely"
>>>>>>>> asm functionw. 
>>>>>>>
>>>>>>> Ok, I will use SYM_CODE_START to mark kvm_enter_guest in the 
>>>>>>> next version.
>>>>>>>
>>>>>>>> I guess the kvm_enter_guest is something like exception
>>>>>>>> handler becuase the last instruction is "ertn". So usually it 
>>>>>>>> should
>>>>>>>> mark UNWIND_HINT_REGS where can find last frame info by "$sp".
>>>>>>>> However, all info is store to "$a2", this mark should be
>>>>>>>>   `UNWIND_HINT sp_reg=ORC_REG_A2(???) type=UNWIND_HINT_TYPE_REGS`.
>>>>>>>> I don't konw why save this function internal PC here by `pcaddi 
>>>>>>>> t0, 0`,
>>>>>>>> and I think it is no meaning(, for exception handler, they save 
>>>>>>>> last PC
>>>>>>>> by read CSR.ERA). The `kvm_enter_guest` saves registers by
>>>>>>>> "$a2"("$sp" - PT_REGS) beyond stack ("$sp"), it is dangerous if IE
>>>>>>>> is enable. So I wonder if there is really a stacktrace through 
>>>>>>>> this function?
>>>>>>>>
>>>>>>> The stack backtracking issue in switch.S is rather complex 
>>>>>>> because it involves the switching between cpu root-mode and 
>>>>>>> guest-mode:
>>>>>>> Real stack backtracking should be divided into two parts:
>>>>>>> part 1:
>>>>>>>     [<0>] kvm_enter_guest+0x38/0x11c
>>>>>>>     [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>>>>>     [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>>>>>     [<0>] sys_ioctl+0x498/0xf00
>>>>>>>     [<0>] do_syscall+0x98/0x1d0
>>>>>>>     [<0>] handle_syscall+0xb8/0x158
>>>>>>>
>>>>>>> part 2:
>>>>>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>     [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>>>
>>>>>>>
>>>>>>> In "part 1", after executing kvm_enter_guest, the cpu switches 
>>>>>>> from root-mode to guest-mode.
>>>>>>> In this case, stack backtracking is indeed very rare.
>>>>>>>
>>>>>>> In "part 2", the cpu switches from the guest-mode to the root-mode,
>>>>>>> and most of the stack backtracking occurs during this phase.
>>>>>>>
>>>>>>> To obtain the longest call chain, we save pc in kvm_enter_guest 
>>>>>>> to pt_regs.csr_era,
>>>>>>> and after restoring the sp of the root-mode cpu in kvm_exc_entry,
>>>>>>> The ORC entry was re-established using "UNWIND_HINT_REGS",
>>>>>>>  and then we obtained the following stack backtrace as we wanted:
>>>>>>>
>>>>>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>     [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>> I found this might be a coincidence—correct behavior due to the 
>>>>>> incorrect
>>>>>> UNWIND_HINT_REGS mark and unusual stack adjustment.
>>>>>>
>>>>>> First, the kvm_enter_guest contains only a single branch 
>>>>>> instruction, ertn.
>>>>>> It hardware-jump to the CSR.ERA address directly, jump into 
>>>>>> kvm_exc_entry.
>>>>>>
>>>>>> At this point, the stack layout looks like this:
>>>>>> -------------------------------
>>>>>>   frame from call to `kvm_enter_guest`
>>>>>> -------------------------------  <- $sp
>>>>>>   PT_REGS
>>>>>> -------------------------------  <- $a2
>>>>>>
>>>>>> Then kvm_exc_entry adjust stack without save any register (e.g. 
>>>>>> $ra, $sp)
>>>>>> but still marked UNWIND_HINT_REGS.
>>>>>> After the adjustment:
>>>>>> -------------------------------
>>>>>>   frame from call to `kvm_enter_guest`
>>>>>> -------------------------------
>>>>>>   PT_REGS
>>>>>> -------------------------------  <- $a2, new $sp
>>>>>>
>>>>>> During unwinding, when the unwinder reaches kvm_exc_entry,
>>>>>> it meets the mark of PT_REGS and correctly recovers
>>>>>>  pc = regs.csr_era, sp = regs.sp, ra = regs.ra
>>>>>>
>>>>> Yes, here unwinder does work as you say.
>>>>>
>>>>>> a) Can we avoid "ertn" rather than `jr reg (or jirl ra, reg, 0)`
>>>>>> instead, like call?
>>>>> No,  we need to rely on the 'ertn instruction return PIE to CRMD IE,
>>>>> at the same time to ensure that its atomic,
>>>>> there should be no other instruction than' ertn 'more appropriate 
>>>>> here.
>>>> You are right! I got it.
>>>>>
>>>>>> The kvm_exc_entry cannot back to kvm_enter_guest
>>>>>> if we use "ertn", so should the kvm_enter_guest appear on the 
>>>>>> stacktrace?
>>>>>>
>>>>>
>>>>> It is flexible. As I mentioned above, the cpu completes the switch 
>>>>> from host-mode to guest mode through kvm_enter_guest,
>>>>> and then the switch from guest mode to host-mode through 
>>>>> kvm_exc_entry. When we ignore the details of the host-mode
>>>>> and guest-mode switching in the middle, we can understand that the 
>>>>> host cpu has completed kvm_enter_guest->kvm_exc_entry.
>>>>> From this perspective, I think it can exist in the call stack, and 
>>>>> at the same time, we have obtained the maximum call stack 
>>>>> information.
>>>>>
>>>>>
>>>>>> b) Can we adjust $sp before entering kvm_exc_entry? Then we can mark
>>>>>> UNWIND_HINT_REGS at the beginning of kvm_exc_entry, which something
>>>>>> like ret_from_kernel_thread_asm.
>>>>>>
>>>>> The following command can be used to dump the orc entries of the 
>>>>> kernel:
>>>>> ./tools/objtool/objtool --dump vmlinux
>>>>>
>>>>> You can observe that not all orc entries are generated at the 
>>>>> beginning of the function.
>>>>> For example:
>>>>> handle_tlb_protect
>>>>> ftrace_stub
>>>>> handle_reserved
>>>>>
>>>>> So, is it unnecessary for us to modify UNWIND_HINT_REGS in order 
>>>>> to place it at the beginning of the function.
>>>>>
>>>>> If you have a better solution, could you provide an example of the 
>>>>> modification?
>>>>> I can test the feasibility of the solution.
>>>>>
>>>> The expression at the beginning of the function is incorrect 
>>>> (feeling sorry).
>>>> It should be marked where have all stacktrace info.
>>>> Thanks for all the explaining, since I'm unfamiliar with kvm, I 
>>>> need these to help my understanding.
>>>>
>>>> Can you try with follows, with save regs by $sp, set more precise 
>>>> era to pt_regs, and more unwind hint.
>>>>
>>>>
>>>> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
>>>> index f1768b7a6194..8ed1d7b72c54 100644
>>>> --- a/arch/loongarch/kvm/switch.S
>>>> +++ b/arch/loongarch/kvm/switch.S
>>>> @@ -14,13 +14,13 @@
>>>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>>>
>>>>   .macro kvm_save_host_gpr base
>>>> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
>>>> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
>>>>       st.d    $r\n, \base, HGPR_OFFSET(\n)
>>>>       .endr
>>>>   .endm
>>>>
>>>>   .macro kvm_restore_host_gpr base
>>>> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
>>>> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
>>>>       ld.d    $r\n, \base, HGPR_OFFSET(\n)
>>>>       .endr
>>>>   .endm
>>>> @@ -88,6 +88,7 @@
>>>>       /* Load KVM_ARCH register */
>>>>       ld.d    a2, a2,    (KVM_ARCH_GGPR + 8 * REG_A2)
>>>>
>>>> +111:
>>>>       ertn /* Switch to guest: GSTAT.PGM = 1, ERRCTL.ISERR = 0, 
>>>> TLBRPRMD.ISTLBR = 0 */
>>>>   .endm
>>>>
>>>> @@ -158,9 +159,10 @@ SYM_CODE_START(kvm_exc_entry)
>>>>       csrwr    t0, LOONGARCH_CSR_GTLBC
>>>>       ld.d    tp, a2, KVM_ARCH_HTP
>>>>       ld.d    sp, a2, KVM_ARCH_HSP
>>>> +    UNWIND_HINT_REGS
>>>> +
>>>>       /* restore per cpu register */
>>>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
>>>> -    addi.d    sp, sp, -PT_SIZE
>>>>
>>>>       /* Prepare handle exception */
>>>>       or    a0, s0, zero
>>>> @@ -184,10 +186,11 @@ SYM_CODE_START(kvm_exc_entry)
>>>>       csrwr    s1, KVM_VCPU_KS
>>>>       kvm_switch_to_guest
>>>>
>>>> +    UNWIND_HINT_UNDEFINED
>>>>   ret_to_host:
>>>> -    ld.d    a2, a2, KVM_ARCH_HSP
>>>> -    addi.d  a2, a2, -PT_SIZE
>>>> -    kvm_restore_host_gpr    a2
>>>> +    ld.d    sp, a2, KVM_ARCH_HSP
>>>> +    kvm_restore_host_gpr    sp
>>>> +    addi.d    sp, sp, PT_SIZE
>>>>       jr      ra
>>>>
>>>>   SYM_INNER_LABEL(kvm_exc_entry_end, SYM_L_LOCAL)
>>>> @@ -200,11 +203,15 @@ SYM_CODE_END(kvm_exc_entry)
>>>>    *  a0: kvm_run* run
>>>>    *  a1: kvm_vcpu* vcpu
>>>>    */
>>>> -SYM_FUNC_START(kvm_enter_guest)
>>>> +SYM_CODE_START(kvm_enter_guest)
>>>> +    UNWIND_HINT_UNDEFINED
>>>>       /* Allocate space in stack bottom */
>>>> -    addi.d    a2, sp, -PT_SIZE
>>>> +    addi.d    sp, sp, -PT_SIZE
>>>>       /* Save host GPRs */
>>>> -    kvm_save_host_gpr a2
>>>> +    kvm_save_host_gpr sp
>>>> +    la.pcrel a2, 111f
>>>> +    st.d     a2, sp, PT_ERA
>>>> +    UNWIND_HINT_REGS
>>>>
>>> why the label 111f is more accurate?  Supposing there is hw 
>>> breakpoint here and backtrace is called, what is the call trace 
>>> stack then? obvious label 111f is not executed instead.
>> Xianglai said marking it as regs can get more stack infos, so I use
>> UNWIND_HINT_REGS marked here, though it not called. Remove
>> UNWIND_HINT_REGS thenforbid unwind from here.
>> This function is called and should usually be marked as "call",
>> but it is complex by switching the stack and use `ertn` calls
>> another function.
>>
>>
>>>
>>> UNWIND_HINT_REGS is used for nested kernel stack, is that right?
>>> With nested interrupt and exception handlers on LoongArch kernel, is 
>>> UNWIND_HINT_REGS used?
>>>
>>> SYM_CODE_START(ret_from_fork_asm)
>>>         UNWIND_HINT_REGS
>>>         move            a1, sp
>>>         bl              ret_from_fork
>>>         STACKLEAK_ERASE
>>>         RESTORE_STATIC
>>>         RESTORE_SOME
>>>         RESTORE_SP_AND_RET
>>> SYM_CODE_END(ret_from_fork_asm)
>>> With this piece of code, what is contents of pt_regs? In generic it 
>>> is called from sys_clone, era is user PC address, is that right? If so,
>>> what is detailed usage in the beginning of ret_from_fork_asm?
>> The stacktrace shows the control flow where the PC will go back, so
>> it is right because when PC is in ret_from_fork_asm, it is already
>> another thread. The era means it will go back user mode.
> The problem is that user mode era shows unwind with error, and 
> user_mode(regs) is not accurate. here is piece of code.
>                 pc = regs->csr_era;
>                 if (!__kernel_text_address(pc))
>                         goto err;
> will UNWIND_HINT_END_OF_STACK be better than UNWIND_HINT_REGS?

You are right. And the reason why current the unwinder does not cause
error is in case ORC_TYPE_REGS we process it by user_mode(regs).

Jinyang

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[Bibo Mao]

On 2025/12/30 下午1:53, Jinyang He wrote:
> On 2025-12-30 12:03, Bibo Mao wrote:
> 
>>
>>
>> On 2025/12/30 上午11:36, Jinyang He wrote:
>>> On 2025-12-30 10:24, Bibo Mao wrote:
>>>
>>>>
>>>>
>>>> On 2025/12/29 下午6:41, Jinyang He wrote:
>>>>> On 2025-12-29 18:11, lixianglai wrote:
>>>>>
>>>>>> Hi Jinyang:
>>>>>>>
>>>>>>> On 2025-12-29 11:53, lixianglai wrote:
>>>>>>>> Hi Jinyang:
>>>>>>>>> On 2025-12-27 09:27, Xianglai Li wrote:
>>>>>>>>>
>>>>>>>>>> Insert the appropriate UNWIND macro definition into the 
>>>>>>>>>> kvm_exc_entry in
>>>>>>>>>> the assembly function to guide the generation of correct ORC 
>>>>>>>>>> table entries,
>>>>>>>>>> thereby solving the timeout problem of loading the 
>>>>>>>>>> livepatch-sample module
>>>>>>>>>> on a physical machine running multiple vcpus virtual machines.
>>>>>>>>>>
>>>>>>>>>> While solving the above problems, we have gained an additional 
>>>>>>>>>> benefit,
>>>>>>>>>> that is, we can obtain more call stack information
>>>>>>>>>>
>>>>>>>>>> Stack information that can be obtained before the problem is 
>>>>>>>>>> fixed:
>>>>>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>>>> [<0>] kvm_exc_entry+0x100/0x1e0
>>>>>>>>>>
>>>>>>>>>> Stack information that can be obtained after the problem is 
>>>>>>>>>> fixed:
>>>>>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>>>> [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>>>>>> [<0>] kvm_enter_guest+0x38/0x11c
>>>>>>>>>> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>>>>>>>> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>>>>>>>> [<0>] sys_ioctl+0x498/0xf00
>>>>>>>>>> [<0>] do_syscall+0x98/0x1d0
>>>>>>>>>> [<0>] handle_syscall+0xb8/0x158
>>>>>>>>>>
>>>>>>>>>> Cc: stable@vger.kernel.org
>>>>>>>>>> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
>>>>>>>>>> ---
>>>>>>>>>> Cc: Huacai Chen <chenhuacai@kernel.org>
>>>>>>>>>> Cc: WANG Xuerui <kernel@xen0n.name>
>>>>>>>>>> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
>>>>>>>>>> Cc: Bibo Mao <maobibo@loongson.cn>
>>>>>>>>>> Cc: Charlie Jenkins <charlie@rivosinc.com>
>>>>>>>>>> Cc: Xianglai Li <lixianglai@loongson.cn>
>>>>>>>>>> Cc: Thomas Gleixner <tglx@linutronix.de>
>>>>>>>>>> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
>>>>>>>>>>
>>>>>>>>>>   arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
>>>>>>>>>>   1 file changed, 19 insertions(+), 9 deletions(-)
>>>>>>>>>>
>>>>>>>>>> diff --git a/arch/loongarch/kvm/switch.S 
>>>>>>>>>> b/arch/loongarch/kvm/switch.S
>>>>>>>>>> index 93845ce53651..a3ea9567dbe5 100644
>>>>>>>>>> --- a/arch/loongarch/kvm/switch.S
>>>>>>>>>> +++ b/arch/loongarch/kvm/switch.S
>>>>>>>>>> @@ -10,6 +10,7 @@
>>>>>>>>>>   #include <asm/loongarch.h>
>>>>>>>>>>   #include <asm/regdef.h>
>>>>>>>>>>   #include <asm/unwind_hints.h>
>>>>>>>>>> +#include <linux/kvm_types.h>
>>>>>>>>>>     #define HGPR_OFFSET(x)        (PT_R0 + 8*x)
>>>>>>>>>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>>>>>>>>> @@ -110,9 +111,9 @@
>>>>>>>>>>        * need to copy world switch code to DMW area.
>>>>>>>>>>        */
>>>>>>>>>>       .text
>>>>>>>>>> +    .p2align PAGE_SHIFT
>>>>>>>>>>       .cfi_sections    .debug_frame
>>>>>>>>>>   SYM_CODE_START(kvm_exc_entry)
>>>>>>>>>> -    .p2align PAGE_SHIFT
>>>>>>>>>>       UNWIND_HINT_UNDEFINED
>>>>>>>>>>       csrwr    a2,   KVM_TEMP_KS
>>>>>>>>>>       csrrd    a2,   KVM_VCPU_KS
>>>>>>>>>> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
>>>>>>>>>>       /* restore per cpu register */
>>>>>>>>>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
>>>>>>>>>>       addi.d    sp, sp, -PT_SIZE
>>>>>>>>>> +    UNWIND_HINT_REGS
>>>>>>>>>>         /* Prepare handle exception */
>>>>>>>>>>       or    a0, s0, zero
>>>>>>>>>> @@ -200,7 +202,7 @@ ret_to_host:
>>>>>>>>>>       jr      ra
>>>>>>>>>>     SYM_CODE_END(kvm_exc_entry)
>>>>>>>>>> -EXPORT_SYMBOL(kvm_exc_entry)
>>>>>>>>>> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
>>>>>>>>>>     /*
>>>>>>>>>>    * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu 
>>>>>>>>>> *vcpu)
>>>>>>>>>> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
>>>>>>>>>>       /* Save host GPRs */
>>>>>>>>>>       kvm_save_host_gpr a2
>>>>>>>>>>   +    /*
>>>>>>>>>> +     * The csr_era member variable of the pt_regs structure 
>>>>>>>>>> is required
>>>>>>>>>> +     * for unwinding orc to perform stack traceback, so we 
>>>>>>>>>> need to put
>>>>>>>>>> +     * pc into csr_era member variable here.
>>>>>>>>>> +     */
>>>>>>>>>> +    pcaddi    t0, 0
>>>>>>>>>> +    st.d    t0, a2, PT_ERA
>>>>>>>>> Hi, Xianglai,
>>>>>>>>>
>>>>>>>>> It should use `SYM_CODE_START` to mark the `kvm_enter_guest` 
>>>>>>>>> rather than
>>>>>>>>> `SYM_FUNC_START`, since the `SYM_FUNC_START` is used to mark 
>>>>>>>>> "C-likely"
>>>>>>>>> asm functionw. 
>>>>>>>>
>>>>>>>> Ok, I will use SYM_CODE_START to mark kvm_enter_guest in the 
>>>>>>>> next version.
>>>>>>>>
>>>>>>>>> I guess the kvm_enter_guest is something like exception
>>>>>>>>> handler becuase the last instruction is "ertn". So usually it 
>>>>>>>>> should
>>>>>>>>> mark UNWIND_HINT_REGS where can find last frame info by "$sp".
>>>>>>>>> However, all info is store to "$a2", this mark should be
>>>>>>>>>   `UNWIND_HINT sp_reg=ORC_REG_A2(???) type=UNWIND_HINT_TYPE_REGS`.
>>>>>>>>> I don't konw why save this function internal PC here by `pcaddi 
>>>>>>>>> t0, 0`,
>>>>>>>>> and I think it is no meaning(, for exception handler, they save 
>>>>>>>>> last PC
>>>>>>>>> by read CSR.ERA). The `kvm_enter_guest` saves registers by
>>>>>>>>> "$a2"("$sp" - PT_REGS) beyond stack ("$sp"), it is dangerous if IE
>>>>>>>>> is enable. So I wonder if there is really a stacktrace through 
>>>>>>>>> this function?
>>>>>>>>>
>>>>>>>> The stack backtracking issue in switch.S is rather complex 
>>>>>>>> because it involves the switching between cpu root-mode and 
>>>>>>>> guest-mode:
>>>>>>>> Real stack backtracking should be divided into two parts:
>>>>>>>> part 1:
>>>>>>>>     [<0>] kvm_enter_guest+0x38/0x11c
>>>>>>>>     [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>>>>>>     [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>>>>>>     [<0>] sys_ioctl+0x498/0xf00
>>>>>>>>     [<0>] do_syscall+0x98/0x1d0
>>>>>>>>     [<0>] handle_syscall+0xb8/0x158
>>>>>>>>
>>>>>>>> part 2:
>>>>>>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>>     [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>>>>
>>>>>>>>
>>>>>>>> In "part 1", after executing kvm_enter_guest, the cpu switches 
>>>>>>>> from root-mode to guest-mode.
>>>>>>>> In this case, stack backtracking is indeed very rare.
>>>>>>>>
>>>>>>>> In "part 2", the cpu switches from the guest-mode to the root-mode,
>>>>>>>> and most of the stack backtracking occurs during this phase.
>>>>>>>>
>>>>>>>> To obtain the longest call chain, we save pc in kvm_enter_guest 
>>>>>>>> to pt_regs.csr_era,
>>>>>>>> and after restoring the sp of the root-mode cpu in kvm_exc_entry,
>>>>>>>> The ORC entry was re-established using "UNWIND_HINT_REGS",
>>>>>>>>  and then we obtained the following stack backtrace as we wanted:
>>>>>>>>
>>>>>>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>>     [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>>> I found this might be a coincidence—correct behavior due to the 
>>>>>>> incorrect
>>>>>>> UNWIND_HINT_REGS mark and unusual stack adjustment.
>>>>>>>
>>>>>>> First, the kvm_enter_guest contains only a single branch 
>>>>>>> instruction, ertn.
>>>>>>> It hardware-jump to the CSR.ERA address directly, jump into 
>>>>>>> kvm_exc_entry.
>>>>>>>
>>>>>>> At this point, the stack layout looks like this:
>>>>>>> -------------------------------
>>>>>>>   frame from call to `kvm_enter_guest`
>>>>>>> -------------------------------  <- $sp
>>>>>>>   PT_REGS
>>>>>>> -------------------------------  <- $a2
>>>>>>>
>>>>>>> Then kvm_exc_entry adjust stack without save any register (e.g. 
>>>>>>> $ra, $sp)
>>>>>>> but still marked UNWIND_HINT_REGS.
>>>>>>> After the adjustment:
>>>>>>> -------------------------------
>>>>>>>   frame from call to `kvm_enter_guest`
>>>>>>> -------------------------------
>>>>>>>   PT_REGS
>>>>>>> -------------------------------  <- $a2, new $sp
>>>>>>>
>>>>>>> During unwinding, when the unwinder reaches kvm_exc_entry,
>>>>>>> it meets the mark of PT_REGS and correctly recovers
>>>>>>>  pc = regs.csr_era, sp = regs.sp, ra = regs.ra
>>>>>>>
>>>>>> Yes, here unwinder does work as you say.
>>>>>>
>>>>>>> a) Can we avoid "ertn" rather than `jr reg (or jirl ra, reg, 0)`
>>>>>>> instead, like call?
>>>>>> No,  we need to rely on the 'ertn instruction return PIE to CRMD IE,
>>>>>> at the same time to ensure that its atomic,
>>>>>> there should be no other instruction than' ertn 'more appropriate 
>>>>>> here.
>>>>> You are right! I got it.
>>>>>>
>>>>>>> The kvm_exc_entry cannot back to kvm_enter_guest
>>>>>>> if we use "ertn", so should the kvm_enter_guest appear on the 
>>>>>>> stacktrace?
>>>>>>>
>>>>>>
>>>>>> It is flexible. As I mentioned above, the cpu completes the switch 
>>>>>> from host-mode to guest mode through kvm_enter_guest,
>>>>>> and then the switch from guest mode to host-mode through 
>>>>>> kvm_exc_entry. When we ignore the details of the host-mode
>>>>>> and guest-mode switching in the middle, we can understand that the 
>>>>>> host cpu has completed kvm_enter_guest->kvm_exc_entry.
>>>>>> From this perspective, I think it can exist in the call stack, and 
>>>>>> at the same time, we have obtained the maximum call stack 
>>>>>> information.
>>>>>>
>>>>>>
>>>>>>> b) Can we adjust $sp before entering kvm_exc_entry? Then we can mark
>>>>>>> UNWIND_HINT_REGS at the beginning of kvm_exc_entry, which something
>>>>>>> like ret_from_kernel_thread_asm.
>>>>>>>
>>>>>> The following command can be used to dump the orc entries of the 
>>>>>> kernel:
>>>>>> ./tools/objtool/objtool --dump vmlinux
>>>>>>
>>>>>> You can observe that not all orc entries are generated at the 
>>>>>> beginning of the function.
>>>>>> For example:
>>>>>> handle_tlb_protect
>>>>>> ftrace_stub
>>>>>> handle_reserved
>>>>>>
>>>>>> So, is it unnecessary for us to modify UNWIND_HINT_REGS in order 
>>>>>> to place it at the beginning of the function.
>>>>>>
>>>>>> If you have a better solution, could you provide an example of the 
>>>>>> modification?
>>>>>> I can test the feasibility of the solution.
>>>>>>
>>>>> The expression at the beginning of the function is incorrect 
>>>>> (feeling sorry).
>>>>> It should be marked where have all stacktrace info.
>>>>> Thanks for all the explaining, since I'm unfamiliar with kvm, I 
>>>>> need these to help my understanding.
>>>>>
>>>>> Can you try with follows, with save regs by $sp, set more precise 
>>>>> era to pt_regs, and more unwind hint.
>>>>>
>>>>>
>>>>> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
>>>>> index f1768b7a6194..8ed1d7b72c54 100644
>>>>> --- a/arch/loongarch/kvm/switch.S
>>>>> +++ b/arch/loongarch/kvm/switch.S
>>>>> @@ -14,13 +14,13 @@
>>>>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>>>>
>>>>>   .macro kvm_save_host_gpr base
>>>>> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
>>>>> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
>>>>>       st.d    $r\n, \base, HGPR_OFFSET(\n)
>>>>>       .endr
>>>>>   .endm
>>>>>
>>>>>   .macro kvm_restore_host_gpr base
>>>>> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
>>>>> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
>>>>>       ld.d    $r\n, \base, HGPR_OFFSET(\n)
>>>>>       .endr
>>>>>   .endm
>>>>> @@ -88,6 +88,7 @@
>>>>>       /* Load KVM_ARCH register */
>>>>>       ld.d    a2, a2,    (KVM_ARCH_GGPR + 8 * REG_A2)
>>>>>
>>>>> +111:
>>>>>       ertn /* Switch to guest: GSTAT.PGM = 1, ERRCTL.ISERR = 0, 
>>>>> TLBRPRMD.ISTLBR = 0 */
>>>>>   .endm
>>>>>
>>>>> @@ -158,9 +159,10 @@ SYM_CODE_START(kvm_exc_entry)
>>>>>       csrwr    t0, LOONGARCH_CSR_GTLBC
>>>>>       ld.d    tp, a2, KVM_ARCH_HTP
>>>>>       ld.d    sp, a2, KVM_ARCH_HSP
>>>>> +    UNWIND_HINT_REGS
>>>>> +
>>>>>       /* restore per cpu register */
>>>>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
>>>>> -    addi.d    sp, sp, -PT_SIZE
>>>>>
>>>>>       /* Prepare handle exception */
>>>>>       or    a0, s0, zero
>>>>> @@ -184,10 +186,11 @@ SYM_CODE_START(kvm_exc_entry)
>>>>>       csrwr    s1, KVM_VCPU_KS
>>>>>       kvm_switch_to_guest
>>>>>
>>>>> +    UNWIND_HINT_UNDEFINED
>>>>>   ret_to_host:
>>>>> -    ld.d    a2, a2, KVM_ARCH_HSP
>>>>> -    addi.d  a2, a2, -PT_SIZE
>>>>> -    kvm_restore_host_gpr    a2
>>>>> +    ld.d    sp, a2, KVM_ARCH_HSP
>>>>> +    kvm_restore_host_gpr    sp
>>>>> +    addi.d    sp, sp, PT_SIZE
>>>>>       jr      ra
>>>>>
>>>>>   SYM_INNER_LABEL(kvm_exc_entry_end, SYM_L_LOCAL)
>>>>> @@ -200,11 +203,15 @@ SYM_CODE_END(kvm_exc_entry)
>>>>>    *  a0: kvm_run* run
>>>>>    *  a1: kvm_vcpu* vcpu
>>>>>    */
>>>>> -SYM_FUNC_START(kvm_enter_guest)
>>>>> +SYM_CODE_START(kvm_enter_guest)
>>>>> +    UNWIND_HINT_UNDEFINED
>>>>>       /* Allocate space in stack bottom */
>>>>> -    addi.d    a2, sp, -PT_SIZE
>>>>> +    addi.d    sp, sp, -PT_SIZE
>>>>>       /* Save host GPRs */
>>>>> -    kvm_save_host_gpr a2
>>>>> +    kvm_save_host_gpr sp
>>>>> +    la.pcrel a2, 111f
>>>>> +    st.d     a2, sp, PT_ERA
>>>>> +    UNWIND_HINT_REGS
>>>>>
>>>> why the label 111f is more accurate?  Supposing there is hw 
>>>> breakpoint here and backtrace is called, what is the call trace 
>>>> stack then? obvious label 111f is not executed instead.
>>> Xianglai said marking it as regs can get more stack infos, so I use
>>> UNWIND_HINT_REGS marked here, though it not called. Remove
>>> UNWIND_HINT_REGS thenforbid unwind from here.
>>> This function is called and should usually be marked as "call",
>>> but it is complex by switching the stack and use `ertn` calls
>>> another function.
>>>
>>>
>>>>
>>>> UNWIND_HINT_REGS is used for nested kernel stack, is that right?
>>>> With nested interrupt and exception handlers on LoongArch kernel, is 
>>>> UNWIND_HINT_REGS used?
>>>>
>>>> SYM_CODE_START(ret_from_fork_asm)
>>>>         UNWIND_HINT_REGS
>>>>         move            a1, sp
>>>>         bl              ret_from_fork
>>>>         STACKLEAK_ERASE
>>>>         RESTORE_STATIC
>>>>         RESTORE_SOME
>>>>         RESTORE_SP_AND_RET
>>>> SYM_CODE_END(ret_from_fork_asm)
>>>> With this piece of code, what is contents of pt_regs? In generic it 
>>>> is called from sys_clone, era is user PC address, is that right? If so,
>>>> what is detailed usage in the beginning of ret_from_fork_asm?
>>> The stacktrace shows the control flow where the PC will go back, so
>>> it is right because when PC is in ret_from_fork_asm, it is already
>>> another thread. The era means it will go back user mode.
>> The problem is that user mode era shows unwind with error, and 
>> user_mode(regs) is not accurate. here is piece of code.
>>                 pc = regs->csr_era;
>>                 if (!__kernel_text_address(pc))
>>                         goto err;
>> will UNWIND_HINT_END_OF_STACK be better than UNWIND_HINT_REGS?
> 
> You are right. And the reason why current the unwinder does not cause
> error is in case ORC_TYPE_REGS we process it by user_mode(regs).
user_mode(regs) is unreliable with kernel thread at least:), csr_prmd is 
set for kernel thread with value csr_read32(LOONGARCH_CSR_PRMD), which 
is per-board rather than per thread in function copy_thread.

int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
{
         childksp = (unsigned long)task_stack_page(p) + THREAD_SIZE;

         /* set up new TSS. */
         childregs = (struct pt_regs *) childksp - 1;
         /*  Put the stack after the struct pt_regs.  */
         childksp = (unsigned long) childregs;
         p->thread.csr_prmd = csr_read32(LOONGARCH_CSR_PRMD);
         if (unlikely(args->fn)) {
                 /* kernel thread */
                 childregs->csr_prmd = p->thread.csr_prmd;
                 goto out;
         }

Regards
Bibo Mao

> 
> Jinyang

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[Bibo Mao]

On 2025/12/30 下午1:53, Jinyang He wrote:
> On 2025-12-30 12:03, Bibo Mao wrote:
> 
>>
>>
>> On 2025/12/30 上午11:36, Jinyang He wrote:
>>> On 2025-12-30 10:24, Bibo Mao wrote:
>>>
>>>>
>>>>
>>>> On 2025/12/29 下午6:41, Jinyang He wrote:
>>>>> On 2025-12-29 18:11, lixianglai wrote:
>>>>>
>>>>>> Hi Jinyang:
>>>>>>>
>>>>>>> On 2025-12-29 11:53, lixianglai wrote:
>>>>>>>> Hi Jinyang:
>>>>>>>>> On 2025-12-27 09:27, Xianglai Li wrote:
>>>>>>>>>
>>>>>>>>>> Insert the appropriate UNWIND macro definition into the 
>>>>>>>>>> kvm_exc_entry in
>>>>>>>>>> the assembly function to guide the generation of correct ORC 
>>>>>>>>>> table entries,
>>>>>>>>>> thereby solving the timeout problem of loading the 
>>>>>>>>>> livepatch-sample module
>>>>>>>>>> on a physical machine running multiple vcpus virtual machines.
>>>>>>>>>>
>>>>>>>>>> While solving the above problems, we have gained an additional 
>>>>>>>>>> benefit,
>>>>>>>>>> that is, we can obtain more call stack information
>>>>>>>>>>
>>>>>>>>>> Stack information that can be obtained before the problem is 
>>>>>>>>>> fixed:
>>>>>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>>>> [<0>] kvm_exc_entry+0x100/0x1e0
>>>>>>>>>>
>>>>>>>>>> Stack information that can be obtained after the problem is 
>>>>>>>>>> fixed:
>>>>>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>>>> [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>>>>>> [<0>] kvm_enter_guest+0x38/0x11c
>>>>>>>>>> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>>>>>>>> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>>>>>>>> [<0>] sys_ioctl+0x498/0xf00
>>>>>>>>>> [<0>] do_syscall+0x98/0x1d0
>>>>>>>>>> [<0>] handle_syscall+0xb8/0x158
>>>>>>>>>>
>>>>>>>>>> Cc: stable@vger.kernel.org
>>>>>>>>>> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
>>>>>>>>>> ---
>>>>>>>>>> Cc: Huacai Chen <chenhuacai@kernel.org>
>>>>>>>>>> Cc: WANG Xuerui <kernel@xen0n.name>
>>>>>>>>>> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
>>>>>>>>>> Cc: Bibo Mao <maobibo@loongson.cn>
>>>>>>>>>> Cc: Charlie Jenkins <charlie@rivosinc.com>
>>>>>>>>>> Cc: Xianglai Li <lixianglai@loongson.cn>
>>>>>>>>>> Cc: Thomas Gleixner <tglx@linutronix.de>
>>>>>>>>>> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
>>>>>>>>>>
>>>>>>>>>>   arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
>>>>>>>>>>   1 file changed, 19 insertions(+), 9 deletions(-)
>>>>>>>>>>
>>>>>>>>>> diff --git a/arch/loongarch/kvm/switch.S 
>>>>>>>>>> b/arch/loongarch/kvm/switch.S
>>>>>>>>>> index 93845ce53651..a3ea9567dbe5 100644
>>>>>>>>>> --- a/arch/loongarch/kvm/switch.S
>>>>>>>>>> +++ b/arch/loongarch/kvm/switch.S
>>>>>>>>>> @@ -10,6 +10,7 @@
>>>>>>>>>>   #include <asm/loongarch.h>
>>>>>>>>>>   #include <asm/regdef.h>
>>>>>>>>>>   #include <asm/unwind_hints.h>
>>>>>>>>>> +#include <linux/kvm_types.h>
>>>>>>>>>>     #define HGPR_OFFSET(x)        (PT_R0 + 8*x)
>>>>>>>>>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>>>>>>>>> @@ -110,9 +111,9 @@
>>>>>>>>>>        * need to copy world switch code to DMW area.
>>>>>>>>>>        */
>>>>>>>>>>       .text
>>>>>>>>>> +    .p2align PAGE_SHIFT
>>>>>>>>>>       .cfi_sections    .debug_frame
>>>>>>>>>>   SYM_CODE_START(kvm_exc_entry)
>>>>>>>>>> -    .p2align PAGE_SHIFT
>>>>>>>>>>       UNWIND_HINT_UNDEFINED
>>>>>>>>>>       csrwr    a2,   KVM_TEMP_KS
>>>>>>>>>>       csrrd    a2,   KVM_VCPU_KS
>>>>>>>>>> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
>>>>>>>>>>       /* restore per cpu register */
>>>>>>>>>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
>>>>>>>>>>       addi.d    sp, sp, -PT_SIZE
>>>>>>>>>> +    UNWIND_HINT_REGS
>>>>>>>>>>         /* Prepare handle exception */
>>>>>>>>>>       or    a0, s0, zero
>>>>>>>>>> @@ -200,7 +202,7 @@ ret_to_host:
>>>>>>>>>>       jr      ra
>>>>>>>>>>     SYM_CODE_END(kvm_exc_entry)
>>>>>>>>>> -EXPORT_SYMBOL(kvm_exc_entry)
>>>>>>>>>> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
>>>>>>>>>>     /*
>>>>>>>>>>    * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu 
>>>>>>>>>> *vcpu)
>>>>>>>>>> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
>>>>>>>>>>       /* Save host GPRs */
>>>>>>>>>>       kvm_save_host_gpr a2
>>>>>>>>>>   +    /*
>>>>>>>>>> +     * The csr_era member variable of the pt_regs structure 
>>>>>>>>>> is required
>>>>>>>>>> +     * for unwinding orc to perform stack traceback, so we 
>>>>>>>>>> need to put
>>>>>>>>>> +     * pc into csr_era member variable here.
>>>>>>>>>> +     */
>>>>>>>>>> +    pcaddi    t0, 0
>>>>>>>>>> +    st.d    t0, a2, PT_ERA
>>>>>>>>> Hi, Xianglai,
>>>>>>>>>
>>>>>>>>> It should use `SYM_CODE_START` to mark the `kvm_enter_guest` 
>>>>>>>>> rather than
>>>>>>>>> `SYM_FUNC_START`, since the `SYM_FUNC_START` is used to mark 
>>>>>>>>> "C-likely"
>>>>>>>>> asm functionw. 
>>>>>>>>
>>>>>>>> Ok, I will use SYM_CODE_START to mark kvm_enter_guest in the 
>>>>>>>> next version.
>>>>>>>>
>>>>>>>>> I guess the kvm_enter_guest is something like exception
>>>>>>>>> handler becuase the last instruction is "ertn". So usually it 
>>>>>>>>> should
>>>>>>>>> mark UNWIND_HINT_REGS where can find last frame info by "$sp".
>>>>>>>>> However, all info is store to "$a2", this mark should be
>>>>>>>>>   `UNWIND_HINT sp_reg=ORC_REG_A2(???) type=UNWIND_HINT_TYPE_REGS`.
>>>>>>>>> I don't konw why save this function internal PC here by `pcaddi 
>>>>>>>>> t0, 0`,
>>>>>>>>> and I think it is no meaning(, for exception handler, they save 
>>>>>>>>> last PC
>>>>>>>>> by read CSR.ERA). The `kvm_enter_guest` saves registers by
>>>>>>>>> "$a2"("$sp" - PT_REGS) beyond stack ("$sp"), it is dangerous if IE
>>>>>>>>> is enable. So I wonder if there is really a stacktrace through 
>>>>>>>>> this function?
>>>>>>>>>
>>>>>>>> The stack backtracking issue in switch.S is rather complex 
>>>>>>>> because it involves the switching between cpu root-mode and 
>>>>>>>> guest-mode:
>>>>>>>> Real stack backtracking should be divided into two parts:
>>>>>>>> part 1:
>>>>>>>>     [<0>] kvm_enter_guest+0x38/0x11c
>>>>>>>>     [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>>>>>>     [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>>>>>>     [<0>] sys_ioctl+0x498/0xf00
>>>>>>>>     [<0>] do_syscall+0x98/0x1d0
>>>>>>>>     [<0>] handle_syscall+0xb8/0x158
>>>>>>>>
>>>>>>>> part 2:
>>>>>>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>>     [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>>>>
>>>>>>>>
>>>>>>>> In "part 1", after executing kvm_enter_guest, the cpu switches 
>>>>>>>> from root-mode to guest-mode.
>>>>>>>> In this case, stack backtracking is indeed very rare.
>>>>>>>>
>>>>>>>> In "part 2", the cpu switches from the guest-mode to the root-mode,
>>>>>>>> and most of the stack backtracking occurs during this phase.
>>>>>>>>
>>>>>>>> To obtain the longest call chain, we save pc in kvm_enter_guest 
>>>>>>>> to pt_regs.csr_era,
>>>>>>>> and after restoring the sp of the root-mode cpu in kvm_exc_entry,
>>>>>>>> The ORC entry was re-established using "UNWIND_HINT_REGS",
>>>>>>>>  and then we obtained the following stack backtrace as we wanted:
>>>>>>>>
>>>>>>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>>     [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>>> I found this might be a coincidence—correct behavior due to the 
>>>>>>> incorrect
>>>>>>> UNWIND_HINT_REGS mark and unusual stack adjustment.
>>>>>>>
>>>>>>> First, the kvm_enter_guest contains only a single branch 
>>>>>>> instruction, ertn.
>>>>>>> It hardware-jump to the CSR.ERA address directly, jump into 
>>>>>>> kvm_exc_entry.
>>>>>>>
>>>>>>> At this point, the stack layout looks like this:
>>>>>>> -------------------------------
>>>>>>>   frame from call to `kvm_enter_guest`
>>>>>>> -------------------------------  <- $sp
>>>>>>>   PT_REGS
>>>>>>> -------------------------------  <- $a2
>>>>>>>
>>>>>>> Then kvm_exc_entry adjust stack without save any register (e.g. 
>>>>>>> $ra, $sp)
>>>>>>> but still marked UNWIND_HINT_REGS.
>>>>>>> After the adjustment:
>>>>>>> -------------------------------
>>>>>>>   frame from call to `kvm_enter_guest`
>>>>>>> -------------------------------
>>>>>>>   PT_REGS
>>>>>>> -------------------------------  <- $a2, new $sp
>>>>>>>
>>>>>>> During unwinding, when the unwinder reaches kvm_exc_entry,
>>>>>>> it meets the mark of PT_REGS and correctly recovers
>>>>>>>  pc = regs.csr_era, sp = regs.sp, ra = regs.ra
>>>>>>>
>>>>>> Yes, here unwinder does work as you say.
>>>>>>
>>>>>>> a) Can we avoid "ertn" rather than `jr reg (or jirl ra, reg, 0)`
>>>>>>> instead, like call?
>>>>>> No,  we need to rely on the 'ertn instruction return PIE to CRMD IE,
>>>>>> at the same time to ensure that its atomic,
>>>>>> there should be no other instruction than' ertn 'more appropriate 
>>>>>> here.
>>>>> You are right! I got it.
>>>>>>
>>>>>>> The kvm_exc_entry cannot back to kvm_enter_guest
>>>>>>> if we use "ertn", so should the kvm_enter_guest appear on the 
>>>>>>> stacktrace?
>>>>>>>
>>>>>>
>>>>>> It is flexible. As I mentioned above, the cpu completes the switch 
>>>>>> from host-mode to guest mode through kvm_enter_guest,
>>>>>> and then the switch from guest mode to host-mode through 
>>>>>> kvm_exc_entry. When we ignore the details of the host-mode
>>>>>> and guest-mode switching in the middle, we can understand that the 
>>>>>> host cpu has completed kvm_enter_guest->kvm_exc_entry.
>>>>>> From this perspective, I think it can exist in the call stack, and 
>>>>>> at the same time, we have obtained the maximum call stack 
>>>>>> information.
>>>>>>
>>>>>>
>>>>>>> b) Can we adjust $sp before entering kvm_exc_entry? Then we can mark
>>>>>>> UNWIND_HINT_REGS at the beginning of kvm_exc_entry, which something
>>>>>>> like ret_from_kernel_thread_asm.
>>>>>>>
>>>>>> The following command can be used to dump the orc entries of the 
>>>>>> kernel:
>>>>>> ./tools/objtool/objtool --dump vmlinux
>>>>>>
>>>>>> You can observe that not all orc entries are generated at the 
>>>>>> beginning of the function.
>>>>>> For example:
>>>>>> handle_tlb_protect
>>>>>> ftrace_stub
>>>>>> handle_reserved
>>>>>>
>>>>>> So, is it unnecessary for us to modify UNWIND_HINT_REGS in order 
>>>>>> to place it at the beginning of the function.
>>>>>>
>>>>>> If you have a better solution, could you provide an example of the 
>>>>>> modification?
>>>>>> I can test the feasibility of the solution.
>>>>>>
>>>>> The expression at the beginning of the function is incorrect 
>>>>> (feeling sorry).
>>>>> It should be marked where have all stacktrace info.
>>>>> Thanks for all the explaining, since I'm unfamiliar with kvm, I 
>>>>> need these to help my understanding.
>>>>>
>>>>> Can you try with follows, with save regs by $sp, set more precise 
>>>>> era to pt_regs, and more unwind hint.
>>>>>
>>>>>
>>>>> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
>>>>> index f1768b7a6194..8ed1d7b72c54 100644
>>>>> --- a/arch/loongarch/kvm/switch.S
>>>>> +++ b/arch/loongarch/kvm/switch.S
>>>>> @@ -14,13 +14,13 @@
>>>>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>>>>
>>>>>   .macro kvm_save_host_gpr base
>>>>> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
>>>>> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
>>>>>       st.d    $r\n, \base, HGPR_OFFSET(\n)
>>>>>       .endr
>>>>>   .endm
>>>>>
>>>>>   .macro kvm_restore_host_gpr base
>>>>> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
>>>>> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
>>>>>       ld.d    $r\n, \base, HGPR_OFFSET(\n)
>>>>>       .endr
>>>>>   .endm
>>>>> @@ -88,6 +88,7 @@
>>>>>       /* Load KVM_ARCH register */
>>>>>       ld.d    a2, a2,    (KVM_ARCH_GGPR + 8 * REG_A2)
>>>>>
>>>>> +111:
>>>>>       ertn /* Switch to guest: GSTAT.PGM = 1, ERRCTL.ISERR = 0, 
>>>>> TLBRPRMD.ISTLBR = 0 */
>>>>>   .endm
>>>>>
>>>>> @@ -158,9 +159,10 @@ SYM_CODE_START(kvm_exc_entry)
>>>>>       csrwr    t0, LOONGARCH_CSR_GTLBC
>>>>>       ld.d    tp, a2, KVM_ARCH_HTP
>>>>>       ld.d    sp, a2, KVM_ARCH_HSP
>>>>> +    UNWIND_HINT_REGS
>>>>> +
>>>>>       /* restore per cpu register */
>>>>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
>>>>> -    addi.d    sp, sp, -PT_SIZE
>>>>>
>>>>>       /* Prepare handle exception */
>>>>>       or    a0, s0, zero
>>>>> @@ -184,10 +186,11 @@ SYM_CODE_START(kvm_exc_entry)
>>>>>       csrwr    s1, KVM_VCPU_KS
>>>>>       kvm_switch_to_guest
>>>>>
>>>>> +    UNWIND_HINT_UNDEFINED
>>>>>   ret_to_host:
>>>>> -    ld.d    a2, a2, KVM_ARCH_HSP
>>>>> -    addi.d  a2, a2, -PT_SIZE
>>>>> -    kvm_restore_host_gpr    a2
>>>>> +    ld.d    sp, a2, KVM_ARCH_HSP
>>>>> +    kvm_restore_host_gpr    sp
>>>>> +    addi.d    sp, sp, PT_SIZE
>>>>>       jr      ra
>>>>>
>>>>>   SYM_INNER_LABEL(kvm_exc_entry_end, SYM_L_LOCAL)
>>>>> @@ -200,11 +203,15 @@ SYM_CODE_END(kvm_exc_entry)
>>>>>    *  a0: kvm_run* run
>>>>>    *  a1: kvm_vcpu* vcpu
>>>>>    */
>>>>> -SYM_FUNC_START(kvm_enter_guest)
>>>>> +SYM_CODE_START(kvm_enter_guest)
>>>>> +    UNWIND_HINT_UNDEFINED
>>>>>       /* Allocate space in stack bottom */
>>>>> -    addi.d    a2, sp, -PT_SIZE
>>>>> +    addi.d    sp, sp, -PT_SIZE
>>>>>       /* Save host GPRs */
>>>>> -    kvm_save_host_gpr a2
>>>>> +    kvm_save_host_gpr sp
>>>>> +    la.pcrel a2, 111f
>>>>> +    st.d     a2, sp, PT_ERA
>>>>> +    UNWIND_HINT_REGS
>>>>>
>>>> why the label 111f is more accurate?  Supposing there is hw 
>>>> breakpoint here and backtrace is called, what is the call trace 
>>>> stack then? obvious label 111f is not executed instead.
>>> Xianglai said marking it as regs can get more stack infos, so I use
>>> UNWIND_HINT_REGS marked here, though it not called. Remove
>>> UNWIND_HINT_REGS thenforbid unwind from here.
>>> This function is called and should usually be marked as "call",
>>> but it is complex by switching the stack and use `ertn` calls
>>> another function.
>>>
>>>
>>>>
>>>> UNWIND_HINT_REGS is used for nested kernel stack, is that right?
>>>> With nested interrupt and exception handlers on LoongArch kernel, is 
>>>> UNWIND_HINT_REGS used?
>>>>
>>>> SYM_CODE_START(ret_from_fork_asm)
>>>>         UNWIND_HINT_REGS
>>>>         move            a1, sp
>>>>         bl              ret_from_fork
>>>>         STACKLEAK_ERASE
>>>>         RESTORE_STATIC
>>>>         RESTORE_SOME
>>>>         RESTORE_SP_AND_RET
>>>> SYM_CODE_END(ret_from_fork_asm)
>>>> With this piece of code, what is contents of pt_regs? In generic it 
>>>> is called from sys_clone, era is user PC address, is that right? If so,
>>>> what is detailed usage in the beginning of ret_from_fork_asm?
>>> The stacktrace shows the control flow where the PC will go back, so
>>> it is right because when PC is in ret_from_fork_asm, it is already
>>> another thread. The era means it will go back user mode.
>> The problem is that user mode era shows unwind with error, and 
>> user_mode(regs) is not accurate. here is piece of code.
>>                 pc = regs->csr_era;
>>                 if (!__kernel_text_address(pc))
>>                         goto err;
>> will UNWIND_HINT_END_OF_STACK be better than UNWIND_HINT_REGS?
> 
> You are right. And the reason why current the unwinder does not cause
> error is in case ORC_TYPE_REGS we process it by user_mode(regs).
Any process about UNWIND_HINT_REGS usage, is nested exception unwind 
supported now?

Talking without any actions seems not be style of Loongson :)

Regards
Bibo Mao
> 
> Jinyang
> 

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[Huacai Chen]

On Wed, Apr 8, 2026 at 9:26 AM Bibo Mao <maobibo@loongson.cn> wrote:
>
>
>
> On 2025/12/30 下午1:53, Jinyang He wrote:
> > On 2025-12-30 12:03, Bibo Mao wrote:
> >
> >>
> >>
> >> On 2025/12/30 上午11:36, Jinyang He wrote:
> >>> On 2025-12-30 10:24, Bibo Mao wrote:
> >>>
> >>>>
> >>>>
> >>>> On 2025/12/29 下午6:41, Jinyang He wrote:
> >>>>> On 2025-12-29 18:11, lixianglai wrote:
> >>>>>
> >>>>>> Hi Jinyang:
> >>>>>>>
> >>>>>>> On 2025-12-29 11:53, lixianglai wrote:
> >>>>>>>> Hi Jinyang:
> >>>>>>>>> On 2025-12-27 09:27, Xianglai Li wrote:
> >>>>>>>>>
> >>>>>>>>>> Insert the appropriate UNWIND macro definition into the
> >>>>>>>>>> kvm_exc_entry in
> >>>>>>>>>> the assembly function to guide the generation of correct ORC
> >>>>>>>>>> table entries,
> >>>>>>>>>> thereby solving the timeout problem of loading the
> >>>>>>>>>> livepatch-sample module
> >>>>>>>>>> on a physical machine running multiple vcpus virtual machines.
> >>>>>>>>>>
> >>>>>>>>>> While solving the above problems, we have gained an additional
> >>>>>>>>>> benefit,
> >>>>>>>>>> that is, we can obtain more call stack information
> >>>>>>>>>>
> >>>>>>>>>> Stack information that can be obtained before the problem is
> >>>>>>>>>> fixed:
> >>>>>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
> >>>>>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
> >>>>>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
> >>>>>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
> >>>>>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
> >>>>>>>>>> [<0>] kvm_exc_entry+0x100/0x1e0
> >>>>>>>>>>
> >>>>>>>>>> Stack information that can be obtained after the problem is
> >>>>>>>>>> fixed:
> >>>>>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
> >>>>>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
> >>>>>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
> >>>>>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
> >>>>>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
> >>>>>>>>>> [<0>] kvm_exc_entry+0x104/0x1e4
> >>>>>>>>>> [<0>] kvm_enter_guest+0x38/0x11c
> >>>>>>>>>> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
> >>>>>>>>>> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
> >>>>>>>>>> [<0>] sys_ioctl+0x498/0xf00
> >>>>>>>>>> [<0>] do_syscall+0x98/0x1d0
> >>>>>>>>>> [<0>] handle_syscall+0xb8/0x158
> >>>>>>>>>>
> >>>>>>>>>> Cc: stable@vger.kernel.org
> >>>>>>>>>> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
> >>>>>>>>>> ---
> >>>>>>>>>> Cc: Huacai Chen <chenhuacai@kernel.org>
> >>>>>>>>>> Cc: WANG Xuerui <kernel@xen0n.name>
> >>>>>>>>>> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
> >>>>>>>>>> Cc: Bibo Mao <maobibo@loongson.cn>
> >>>>>>>>>> Cc: Charlie Jenkins <charlie@rivosinc.com>
> >>>>>>>>>> Cc: Xianglai Li <lixianglai@loongson.cn>
> >>>>>>>>>> Cc: Thomas Gleixner <tglx@linutronix.de>
> >>>>>>>>>> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
> >>>>>>>>>>
> >>>>>>>>>>   arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
> >>>>>>>>>>   1 file changed, 19 insertions(+), 9 deletions(-)
> >>>>>>>>>>
> >>>>>>>>>> diff --git a/arch/loongarch/kvm/switch.S
> >>>>>>>>>> b/arch/loongarch/kvm/switch.S
> >>>>>>>>>> index 93845ce53651..a3ea9567dbe5 100644
> >>>>>>>>>> --- a/arch/loongarch/kvm/switch.S
> >>>>>>>>>> +++ b/arch/loongarch/kvm/switch.S
> >>>>>>>>>> @@ -10,6 +10,7 @@
> >>>>>>>>>>   #include <asm/loongarch.h>
> >>>>>>>>>>   #include <asm/regdef.h>
> >>>>>>>>>>   #include <asm/unwind_hints.h>
> >>>>>>>>>> +#include <linux/kvm_types.h>
> >>>>>>>>>>     #define HGPR_OFFSET(x)        (PT_R0 + 8*x)
> >>>>>>>>>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
> >>>>>>>>>> @@ -110,9 +111,9 @@
> >>>>>>>>>>        * need to copy world switch code to DMW area.
> >>>>>>>>>>        */
> >>>>>>>>>>       .text
> >>>>>>>>>> +    .p2align PAGE_SHIFT
> >>>>>>>>>>       .cfi_sections    .debug_frame
> >>>>>>>>>>   SYM_CODE_START(kvm_exc_entry)
> >>>>>>>>>> -    .p2align PAGE_SHIFT
> >>>>>>>>>>       UNWIND_HINT_UNDEFINED
> >>>>>>>>>>       csrwr    a2,   KVM_TEMP_KS
> >>>>>>>>>>       csrrd    a2,   KVM_VCPU_KS
> >>>>>>>>>> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
> >>>>>>>>>>       /* restore per cpu register */
> >>>>>>>>>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
> >>>>>>>>>>       addi.d    sp, sp, -PT_SIZE
> >>>>>>>>>> +    UNWIND_HINT_REGS
> >>>>>>>>>>         /* Prepare handle exception */
> >>>>>>>>>>       or    a0, s0, zero
> >>>>>>>>>> @@ -200,7 +202,7 @@ ret_to_host:
> >>>>>>>>>>       jr      ra
> >>>>>>>>>>     SYM_CODE_END(kvm_exc_entry)
> >>>>>>>>>> -EXPORT_SYMBOL(kvm_exc_entry)
> >>>>>>>>>> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
> >>>>>>>>>>     /*
> >>>>>>>>>>    * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu
> >>>>>>>>>> *vcpu)
> >>>>>>>>>> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
> >>>>>>>>>>       /* Save host GPRs */
> >>>>>>>>>>       kvm_save_host_gpr a2
> >>>>>>>>>>   +    /*
> >>>>>>>>>> +     * The csr_era member variable of the pt_regs structure
> >>>>>>>>>> is required
> >>>>>>>>>> +     * for unwinding orc to perform stack traceback, so we
> >>>>>>>>>> need to put
> >>>>>>>>>> +     * pc into csr_era member variable here.
> >>>>>>>>>> +     */
> >>>>>>>>>> +    pcaddi    t0, 0
> >>>>>>>>>> +    st.d    t0, a2, PT_ERA
> >>>>>>>>> Hi, Xianglai,
> >>>>>>>>>
> >>>>>>>>> It should use `SYM_CODE_START` to mark the `kvm_enter_guest`
> >>>>>>>>> rather than
> >>>>>>>>> `SYM_FUNC_START`, since the `SYM_FUNC_START` is used to mark
> >>>>>>>>> "C-likely"
> >>>>>>>>> asm functionw.
> >>>>>>>>
> >>>>>>>> Ok, I will use SYM_CODE_START to mark kvm_enter_guest in the
> >>>>>>>> next version.
> >>>>>>>>
> >>>>>>>>> I guess the kvm_enter_guest is something like exception
> >>>>>>>>> handler becuase the last instruction is "ertn". So usually it
> >>>>>>>>> should
> >>>>>>>>> mark UNWIND_HINT_REGS where can find last frame info by "$sp".
> >>>>>>>>> However, all info is store to "$a2", this mark should be
> >>>>>>>>>   `UNWIND_HINT sp_reg=ORC_REG_A2(???) type=UNWIND_HINT_TYPE_REGS`.
> >>>>>>>>> I don't konw why save this function internal PC here by `pcaddi
> >>>>>>>>> t0, 0`,
> >>>>>>>>> and I think it is no meaning(, for exception handler, they save
> >>>>>>>>> last PC
> >>>>>>>>> by read CSR.ERA). The `kvm_enter_guest` saves registers by
> >>>>>>>>> "$a2"("$sp" - PT_REGS) beyond stack ("$sp"), it is dangerous if IE
> >>>>>>>>> is enable. So I wonder if there is really a stacktrace through
> >>>>>>>>> this function?
> >>>>>>>>>
> >>>>>>>> The stack backtracking issue in switch.S is rather complex
> >>>>>>>> because it involves the switching between cpu root-mode and
> >>>>>>>> guest-mode:
> >>>>>>>> Real stack backtracking should be divided into two parts:
> >>>>>>>> part 1:
> >>>>>>>>     [<0>] kvm_enter_guest+0x38/0x11c
> >>>>>>>>     [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
> >>>>>>>>     [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
> >>>>>>>>     [<0>] sys_ioctl+0x498/0xf00
> >>>>>>>>     [<0>] do_syscall+0x98/0x1d0
> >>>>>>>>     [<0>] handle_syscall+0xb8/0x158
> >>>>>>>>
> >>>>>>>> part 2:
> >>>>>>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
> >>>>>>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
> >>>>>>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
> >>>>>>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
> >>>>>>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
> >>>>>>>>     [<0>] kvm_exc_entry+0x104/0x1e4
> >>>>>>>>
> >>>>>>>>
> >>>>>>>> In "part 1", after executing kvm_enter_guest, the cpu switches
> >>>>>>>> from root-mode to guest-mode.
> >>>>>>>> In this case, stack backtracking is indeed very rare.
> >>>>>>>>
> >>>>>>>> In "part 2", the cpu switches from the guest-mode to the root-mode,
> >>>>>>>> and most of the stack backtracking occurs during this phase.
> >>>>>>>>
> >>>>>>>> To obtain the longest call chain, we save pc in kvm_enter_guest
> >>>>>>>> to pt_regs.csr_era,
> >>>>>>>> and after restoring the sp of the root-mode cpu in kvm_exc_entry,
> >>>>>>>> The ORC entry was re-established using "UNWIND_HINT_REGS",
> >>>>>>>>  and then we obtained the following stack backtrace as we wanted:
> >>>>>>>>
> >>>>>>>>     [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
> >>>>>>>>     [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
> >>>>>>>>     [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
> >>>>>>>>     [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
> >>>>>>>>     [<0>] kvm_handle_exit+0x160/0x270 [kvm]
> >>>>>>>>     [<0>] kvm_exc_entry+0x104/0x1e4
> >>>>>>> I found this might be a coincidence—correct behavior due to the
> >>>>>>> incorrect
> >>>>>>> UNWIND_HINT_REGS mark and unusual stack adjustment.
> >>>>>>>
> >>>>>>> First, the kvm_enter_guest contains only a single branch
> >>>>>>> instruction, ertn.
> >>>>>>> It hardware-jump to the CSR.ERA address directly, jump into
> >>>>>>> kvm_exc_entry.
> >>>>>>>
> >>>>>>> At this point, the stack layout looks like this:
> >>>>>>> -------------------------------
> >>>>>>>   frame from call to `kvm_enter_guest`
> >>>>>>> -------------------------------  <- $sp
> >>>>>>>   PT_REGS
> >>>>>>> -------------------------------  <- $a2
> >>>>>>>
> >>>>>>> Then kvm_exc_entry adjust stack without save any register (e.g.
> >>>>>>> $ra, $sp)
> >>>>>>> but still marked UNWIND_HINT_REGS.
> >>>>>>> After the adjustment:
> >>>>>>> -------------------------------
> >>>>>>>   frame from call to `kvm_enter_guest`
> >>>>>>> -------------------------------
> >>>>>>>   PT_REGS
> >>>>>>> -------------------------------  <- $a2, new $sp
> >>>>>>>
> >>>>>>> During unwinding, when the unwinder reaches kvm_exc_entry,
> >>>>>>> it meets the mark of PT_REGS and correctly recovers
> >>>>>>>  pc = regs.csr_era, sp = regs.sp, ra = regs.ra
> >>>>>>>
> >>>>>> Yes, here unwinder does work as you say.
> >>>>>>
> >>>>>>> a) Can we avoid "ertn" rather than `jr reg (or jirl ra, reg, 0)`
> >>>>>>> instead, like call?
> >>>>>> No,  we need to rely on the 'ertn instruction return PIE to CRMD IE,
> >>>>>> at the same time to ensure that its atomic,
> >>>>>> there should be no other instruction than' ertn 'more appropriate
> >>>>>> here.
> >>>>> You are right! I got it.
> >>>>>>
> >>>>>>> The kvm_exc_entry cannot back to kvm_enter_guest
> >>>>>>> if we use "ertn", so should the kvm_enter_guest appear on the
> >>>>>>> stacktrace?
> >>>>>>>
> >>>>>>
> >>>>>> It is flexible. As I mentioned above, the cpu completes the switch
> >>>>>> from host-mode to guest mode through kvm_enter_guest,
> >>>>>> and then the switch from guest mode to host-mode through
> >>>>>> kvm_exc_entry. When we ignore the details of the host-mode
> >>>>>> and guest-mode switching in the middle, we can understand that the
> >>>>>> host cpu has completed kvm_enter_guest->kvm_exc_entry.
> >>>>>> From this perspective, I think it can exist in the call stack, and
> >>>>>> at the same time, we have obtained the maximum call stack
> >>>>>> information.
> >>>>>>
> >>>>>>
> >>>>>>> b) Can we adjust $sp before entering kvm_exc_entry? Then we can mark
> >>>>>>> UNWIND_HINT_REGS at the beginning of kvm_exc_entry, which something
> >>>>>>> like ret_from_kernel_thread_asm.
> >>>>>>>
> >>>>>> The following command can be used to dump the orc entries of the
> >>>>>> kernel:
> >>>>>> ./tools/objtool/objtool --dump vmlinux
> >>>>>>
> >>>>>> You can observe that not all orc entries are generated at the
> >>>>>> beginning of the function.
> >>>>>> For example:
> >>>>>> handle_tlb_protect
> >>>>>> ftrace_stub
> >>>>>> handle_reserved
> >>>>>>
> >>>>>> So, is it unnecessary for us to modify UNWIND_HINT_REGS in order
> >>>>>> to place it at the beginning of the function.
> >>>>>>
> >>>>>> If you have a better solution, could you provide an example of the
> >>>>>> modification?
> >>>>>> I can test the feasibility of the solution.
> >>>>>>
> >>>>> The expression at the beginning of the function is incorrect
> >>>>> (feeling sorry).
> >>>>> It should be marked where have all stacktrace info.
> >>>>> Thanks for all the explaining, since I'm unfamiliar with kvm, I
> >>>>> need these to help my understanding.
> >>>>>
> >>>>> Can you try with follows, with save regs by $sp, set more precise
> >>>>> era to pt_regs, and more unwind hint.
> >>>>>
> >>>>>
> >>>>> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
> >>>>> index f1768b7a6194..8ed1d7b72c54 100644
> >>>>> --- a/arch/loongarch/kvm/switch.S
> >>>>> +++ b/arch/loongarch/kvm/switch.S
> >>>>> @@ -14,13 +14,13 @@
> >>>>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
> >>>>>
> >>>>>   .macro kvm_save_host_gpr base
> >>>>> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
> >>>>> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
> >>>>>       st.d    $r\n, \base, HGPR_OFFSET(\n)
> >>>>>       .endr
> >>>>>   .endm
> >>>>>
> >>>>>   .macro kvm_restore_host_gpr base
> >>>>> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
> >>>>> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
> >>>>>       ld.d    $r\n, \base, HGPR_OFFSET(\n)
> >>>>>       .endr
> >>>>>   .endm
> >>>>> @@ -88,6 +88,7 @@
> >>>>>       /* Load KVM_ARCH register */
> >>>>>       ld.d    a2, a2,    (KVM_ARCH_GGPR + 8 * REG_A2)
> >>>>>
> >>>>> +111:
> >>>>>       ertn /* Switch to guest: GSTAT.PGM = 1, ERRCTL.ISERR = 0,
> >>>>> TLBRPRMD.ISTLBR = 0 */
> >>>>>   .endm
> >>>>>
> >>>>> @@ -158,9 +159,10 @@ SYM_CODE_START(kvm_exc_entry)
> >>>>>       csrwr    t0, LOONGARCH_CSR_GTLBC
> >>>>>       ld.d    tp, a2, KVM_ARCH_HTP
> >>>>>       ld.d    sp, a2, KVM_ARCH_HSP
> >>>>> +    UNWIND_HINT_REGS
> >>>>> +
> >>>>>       /* restore per cpu register */
> >>>>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
> >>>>> -    addi.d    sp, sp, -PT_SIZE
> >>>>>
> >>>>>       /* Prepare handle exception */
> >>>>>       or    a0, s0, zero
> >>>>> @@ -184,10 +186,11 @@ SYM_CODE_START(kvm_exc_entry)
> >>>>>       csrwr    s1, KVM_VCPU_KS
> >>>>>       kvm_switch_to_guest
> >>>>>
> >>>>> +    UNWIND_HINT_UNDEFINED
> >>>>>   ret_to_host:
> >>>>> -    ld.d    a2, a2, KVM_ARCH_HSP
> >>>>> -    addi.d  a2, a2, -PT_SIZE
> >>>>> -    kvm_restore_host_gpr    a2
> >>>>> +    ld.d    sp, a2, KVM_ARCH_HSP
> >>>>> +    kvm_restore_host_gpr    sp
> >>>>> +    addi.d    sp, sp, PT_SIZE
> >>>>>       jr      ra
> >>>>>
> >>>>>   SYM_INNER_LABEL(kvm_exc_entry_end, SYM_L_LOCAL)
> >>>>> @@ -200,11 +203,15 @@ SYM_CODE_END(kvm_exc_entry)
> >>>>>    *  a0: kvm_run* run
> >>>>>    *  a1: kvm_vcpu* vcpu
> >>>>>    */
> >>>>> -SYM_FUNC_START(kvm_enter_guest)
> >>>>> +SYM_CODE_START(kvm_enter_guest)
> >>>>> +    UNWIND_HINT_UNDEFINED
> >>>>>       /* Allocate space in stack bottom */
> >>>>> -    addi.d    a2, sp, -PT_SIZE
> >>>>> +    addi.d    sp, sp, -PT_SIZE
> >>>>>       /* Save host GPRs */
> >>>>> -    kvm_save_host_gpr a2
> >>>>> +    kvm_save_host_gpr sp
> >>>>> +    la.pcrel a2, 111f
> >>>>> +    st.d     a2, sp, PT_ERA
> >>>>> +    UNWIND_HINT_REGS
> >>>>>
> >>>> why the label 111f is more accurate?  Supposing there is hw
> >>>> breakpoint here and backtrace is called, what is the call trace
> >>>> stack then? obvious label 111f is not executed instead.
> >>> Xianglai said marking it as regs can get more stack infos, so I use
> >>> UNWIND_HINT_REGS marked here, though it not called. Remove
> >>> UNWIND_HINT_REGS thenforbid unwind from here.
> >>> This function is called and should usually be marked as "call",
> >>> but it is complex by switching the stack and use `ertn` calls
> >>> another function.
> >>>
> >>>
> >>>>
> >>>> UNWIND_HINT_REGS is used for nested kernel stack, is that right?
> >>>> With nested interrupt and exception handlers on LoongArch kernel, is
> >>>> UNWIND_HINT_REGS used?
> >>>>
> >>>> SYM_CODE_START(ret_from_fork_asm)
> >>>>         UNWIND_HINT_REGS
> >>>>         move            a1, sp
> >>>>         bl              ret_from_fork
> >>>>         STACKLEAK_ERASE
> >>>>         RESTORE_STATIC
> >>>>         RESTORE_SOME
> >>>>         RESTORE_SP_AND_RET
> >>>> SYM_CODE_END(ret_from_fork_asm)
> >>>> With this piece of code, what is contents of pt_regs? In generic it
> >>>> is called from sys_clone, era is user PC address, is that right? If so,
> >>>> what is detailed usage in the beginning of ret_from_fork_asm?
> >>> The stacktrace shows the control flow where the PC will go back, so
> >>> it is right because when PC is in ret_from_fork_asm, it is already
> >>> another thread. The era means it will go back user mode.
> >> The problem is that user mode era shows unwind with error, and
> >> user_mode(regs) is not accurate. here is piece of code.
> >>                 pc = regs->csr_era;
> >>                 if (!__kernel_text_address(pc))
> >>                         goto err;
> >> will UNWIND_HINT_END_OF_STACK be better than UNWIND_HINT_REGS?
> >
> > You are right. And the reason why current the unwinder does not cause
> > error is in case ORC_TYPE_REGS we process it by user_mode(regs).
> Any process about UNWIND_HINT_REGS usage, is nested exception unwind
> supported now?
>
> Talking without any actions seems not be style of Loongson :)
From my point of view, Tiezhu's simple solution is acceptable...

Huacai

>
> Regards
> Bibo Mao
> >
> > Jinyang
> >
>

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[Bibo Mao]

On 2026/4/11 下午10:24, Huacai Chen wrote:
> On Wed, Apr 8, 2026 at 9:26 AM Bibo Mao <maobibo@loongson.cn> wrote:
>>
>>
>>
>> On 2025/12/30 下午1:53, Jinyang He wrote:
>>> On 2025-12-30 12:03, Bibo Mao wrote:
>>>
>>>>
>>>>
>>>> On 2025/12/30 上午11:36, Jinyang He wrote:
>>>>> On 2025-12-30 10:24, Bibo Mao wrote:
>>>>>
>>>>>>
>>>>>>
>>>>>> On 2025/12/29 下午6:41, Jinyang He wrote:
>>>>>>> On 2025-12-29 18:11, lixianglai wrote:
>>>>>>>
>>>>>>>> Hi Jinyang:
>>>>>>>>>
>>>>>>>>> On 2025-12-29 11:53, lixianglai wrote:
>>>>>>>>>> Hi Jinyang:
>>>>>>>>>>> On 2025-12-27 09:27, Xianglai Li wrote:
>>>>>>>>>>>
>>>>>>>>>>>> Insert the appropriate UNWIND macro definition into the
>>>>>>>>>>>> kvm_exc_entry in
>>>>>>>>>>>> the assembly function to guide the generation of correct ORC
>>>>>>>>>>>> table entries,
>>>>>>>>>>>> thereby solving the timeout problem of loading the
>>>>>>>>>>>> livepatch-sample module
>>>>>>>>>>>> on a physical machine running multiple vcpus virtual machines.
>>>>>>>>>>>>
>>>>>>>>>>>> While solving the above problems, we have gained an additional
>>>>>>>>>>>> benefit,
>>>>>>>>>>>> that is, we can obtain more call stack information
>>>>>>>>>>>>
>>>>>>>>>>>> Stack information that can be obtained before the problem is
>>>>>>>>>>>> fixed:
>>>>>>>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>>>>>> [<0>] kvm_exc_entry+0x100/0x1e0
>>>>>>>>>>>>
>>>>>>>>>>>> Stack information that can be obtained after the problem is
>>>>>>>>>>>> fixed:
>>>>>>>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>>>>>> [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>>>>>>>> [<0>] kvm_enter_guest+0x38/0x11c
>>>>>>>>>>>> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>>>>>>>>>> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>>>>>>>>>> [<0>] sys_ioctl+0x498/0xf00
>>>>>>>>>>>> [<0>] do_syscall+0x98/0x1d0
>>>>>>>>>>>> [<0>] handle_syscall+0xb8/0x158
>>>>>>>>>>>>
>>>>>>>>>>>> Cc: stable@vger.kernel.org
>>>>>>>>>>>> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
>>>>>>>>>>>> ---
>>>>>>>>>>>> Cc: Huacai Chen <chenhuacai@kernel.org>
>>>>>>>>>>>> Cc: WANG Xuerui <kernel@xen0n.name>
>>>>>>>>>>>> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
>>>>>>>>>>>> Cc: Bibo Mao <maobibo@loongson.cn>
>>>>>>>>>>>> Cc: Charlie Jenkins <charlie@rivosinc.com>
>>>>>>>>>>>> Cc: Xianglai Li <lixianglai@loongson.cn>
>>>>>>>>>>>> Cc: Thomas Gleixner <tglx@linutronix.de>
>>>>>>>>>>>> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
>>>>>>>>>>>>
>>>>>>>>>>>>    arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
>>>>>>>>>>>>    1 file changed, 19 insertions(+), 9 deletions(-)
>>>>>>>>>>>>
>>>>>>>>>>>> diff --git a/arch/loongarch/kvm/switch.S
>>>>>>>>>>>> b/arch/loongarch/kvm/switch.S
>>>>>>>>>>>> index 93845ce53651..a3ea9567dbe5 100644
>>>>>>>>>>>> --- a/arch/loongarch/kvm/switch.S
>>>>>>>>>>>> +++ b/arch/loongarch/kvm/switch.S
>>>>>>>>>>>> @@ -10,6 +10,7 @@
>>>>>>>>>>>>    #include <asm/loongarch.h>
>>>>>>>>>>>>    #include <asm/regdef.h>
>>>>>>>>>>>>    #include <asm/unwind_hints.h>
>>>>>>>>>>>> +#include <linux/kvm_types.h>
>>>>>>>>>>>>      #define HGPR_OFFSET(x)        (PT_R0 + 8*x)
>>>>>>>>>>>>    #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>>>>>>>>>>> @@ -110,9 +111,9 @@
>>>>>>>>>>>>         * need to copy world switch code to DMW area.
>>>>>>>>>>>>         */
>>>>>>>>>>>>        .text
>>>>>>>>>>>> +    .p2align PAGE_SHIFT
>>>>>>>>>>>>        .cfi_sections    .debug_frame
>>>>>>>>>>>>    SYM_CODE_START(kvm_exc_entry)
>>>>>>>>>>>> -    .p2align PAGE_SHIFT
>>>>>>>>>>>>        UNWIND_HINT_UNDEFINED
>>>>>>>>>>>>        csrwr    a2,   KVM_TEMP_KS
>>>>>>>>>>>>        csrrd    a2,   KVM_VCPU_KS
>>>>>>>>>>>> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
>>>>>>>>>>>>        /* restore per cpu register */
>>>>>>>>>>>>        ld.d    u0, a2, KVM_ARCH_HPERCPU
>>>>>>>>>>>>        addi.d    sp, sp, -PT_SIZE
>>>>>>>>>>>> +    UNWIND_HINT_REGS
>>>>>>>>>>>>          /* Prepare handle exception */
>>>>>>>>>>>>        or    a0, s0, zero
>>>>>>>>>>>> @@ -200,7 +202,7 @@ ret_to_host:
>>>>>>>>>>>>        jr      ra
>>>>>>>>>>>>      SYM_CODE_END(kvm_exc_entry)
>>>>>>>>>>>> -EXPORT_SYMBOL(kvm_exc_entry)
>>>>>>>>>>>> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
>>>>>>>>>>>>      /*
>>>>>>>>>>>>     * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu
>>>>>>>>>>>> *vcpu)
>>>>>>>>>>>> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
>>>>>>>>>>>>        /* Save host GPRs */
>>>>>>>>>>>>        kvm_save_host_gpr a2
>>>>>>>>>>>>    +    /*
>>>>>>>>>>>> +     * The csr_era member variable of the pt_regs structure
>>>>>>>>>>>> is required
>>>>>>>>>>>> +     * for unwinding orc to perform stack traceback, so we
>>>>>>>>>>>> need to put
>>>>>>>>>>>> +     * pc into csr_era member variable here.
>>>>>>>>>>>> +     */
>>>>>>>>>>>> +    pcaddi    t0, 0
>>>>>>>>>>>> +    st.d    t0, a2, PT_ERA
>>>>>>>>>>> Hi, Xianglai,
>>>>>>>>>>>
>>>>>>>>>>> It should use `SYM_CODE_START` to mark the `kvm_enter_guest`
>>>>>>>>>>> rather than
>>>>>>>>>>> `SYM_FUNC_START`, since the `SYM_FUNC_START` is used to mark
>>>>>>>>>>> "C-likely"
>>>>>>>>>>> asm functionw.
>>>>>>>>>>
>>>>>>>>>> Ok, I will use SYM_CODE_START to mark kvm_enter_guest in the
>>>>>>>>>> next version.
>>>>>>>>>>
>>>>>>>>>>> I guess the kvm_enter_guest is something like exception
>>>>>>>>>>> handler becuase the last instruction is "ertn". So usually it
>>>>>>>>>>> should
>>>>>>>>>>> mark UNWIND_HINT_REGS where can find last frame info by "$sp".
>>>>>>>>>>> However, all info is store to "$a2", this mark should be
>>>>>>>>>>>    `UNWIND_HINT sp_reg=ORC_REG_A2(???) type=UNWIND_HINT_TYPE_REGS`.
>>>>>>>>>>> I don't konw why save this function internal PC here by `pcaddi
>>>>>>>>>>> t0, 0`,
>>>>>>>>>>> and I think it is no meaning(, for exception handler, they save
>>>>>>>>>>> last PC
>>>>>>>>>>> by read CSR.ERA). The `kvm_enter_guest` saves registers by
>>>>>>>>>>> "$a2"("$sp" - PT_REGS) beyond stack ("$sp"), it is dangerous if IE
>>>>>>>>>>> is enable. So I wonder if there is really a stacktrace through
>>>>>>>>>>> this function?
>>>>>>>>>>>
>>>>>>>>>> The stack backtracking issue in switch.S is rather complex
>>>>>>>>>> because it involves the switching between cpu root-mode and
>>>>>>>>>> guest-mode:
>>>>>>>>>> Real stack backtracking should be divided into two parts:
>>>>>>>>>> part 1:
>>>>>>>>>>      [<0>] kvm_enter_guest+0x38/0x11c
>>>>>>>>>>      [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>>>>>>>>      [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>>>>>>>>      [<0>] sys_ioctl+0x498/0xf00
>>>>>>>>>>      [<0>] do_syscall+0x98/0x1d0
>>>>>>>>>>      [<0>] handle_syscall+0xb8/0x158
>>>>>>>>>>
>>>>>>>>>> part 2:
>>>>>>>>>>      [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>>>>      [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>>>>      [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>>>>      [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>>>>      [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>>>>      [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>> In "part 1", after executing kvm_enter_guest, the cpu switches
>>>>>>>>>> from root-mode to guest-mode.
>>>>>>>>>> In this case, stack backtracking is indeed very rare.
>>>>>>>>>>
>>>>>>>>>> In "part 2", the cpu switches from the guest-mode to the root-mode,
>>>>>>>>>> and most of the stack backtracking occurs during this phase.
>>>>>>>>>>
>>>>>>>>>> To obtain the longest call chain, we save pc in kvm_enter_guest
>>>>>>>>>> to pt_regs.csr_era,
>>>>>>>>>> and after restoring the sp of the root-mode cpu in kvm_exc_entry,
>>>>>>>>>> The ORC entry was re-established using "UNWIND_HINT_REGS",
>>>>>>>>>>   and then we obtained the following stack backtrace as we wanted:
>>>>>>>>>>
>>>>>>>>>>      [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>>>>      [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>>>>      [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>>>>      [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>>>>      [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>>>>      [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>>>>> I found this might be a coincidence—correct behavior due to the
>>>>>>>>> incorrect
>>>>>>>>> UNWIND_HINT_REGS mark and unusual stack adjustment.
>>>>>>>>>
>>>>>>>>> First, the kvm_enter_guest contains only a single branch
>>>>>>>>> instruction, ertn.
>>>>>>>>> It hardware-jump to the CSR.ERA address directly, jump into
>>>>>>>>> kvm_exc_entry.
>>>>>>>>>
>>>>>>>>> At this point, the stack layout looks like this:
>>>>>>>>> -------------------------------
>>>>>>>>>    frame from call to `kvm_enter_guest`
>>>>>>>>> -------------------------------  <- $sp
>>>>>>>>>    PT_REGS
>>>>>>>>> -------------------------------  <- $a2
>>>>>>>>>
>>>>>>>>> Then kvm_exc_entry adjust stack without save any register (e.g.
>>>>>>>>> $ra, $sp)
>>>>>>>>> but still marked UNWIND_HINT_REGS.
>>>>>>>>> After the adjustment:
>>>>>>>>> -------------------------------
>>>>>>>>>    frame from call to `kvm_enter_guest`
>>>>>>>>> -------------------------------
>>>>>>>>>    PT_REGS
>>>>>>>>> -------------------------------  <- $a2, new $sp
>>>>>>>>>
>>>>>>>>> During unwinding, when the unwinder reaches kvm_exc_entry,
>>>>>>>>> it meets the mark of PT_REGS and correctly recovers
>>>>>>>>>   pc = regs.csr_era, sp = regs.sp, ra = regs.ra
>>>>>>>>>
>>>>>>>> Yes, here unwinder does work as you say.
>>>>>>>>
>>>>>>>>> a) Can we avoid "ertn" rather than `jr reg (or jirl ra, reg, 0)`
>>>>>>>>> instead, like call?
>>>>>>>> No,  we need to rely on the 'ertn instruction return PIE to CRMD IE,
>>>>>>>> at the same time to ensure that its atomic,
>>>>>>>> there should be no other instruction than' ertn 'more appropriate
>>>>>>>> here.
>>>>>>> You are right! I got it.
>>>>>>>>
>>>>>>>>> The kvm_exc_entry cannot back to kvm_enter_guest
>>>>>>>>> if we use "ertn", so should the kvm_enter_guest appear on the
>>>>>>>>> stacktrace?
>>>>>>>>>
>>>>>>>>
>>>>>>>> It is flexible. As I mentioned above, the cpu completes the switch
>>>>>>>> from host-mode to guest mode through kvm_enter_guest,
>>>>>>>> and then the switch from guest mode to host-mode through
>>>>>>>> kvm_exc_entry. When we ignore the details of the host-mode
>>>>>>>> and guest-mode switching in the middle, we can understand that the
>>>>>>>> host cpu has completed kvm_enter_guest->kvm_exc_entry.
>>>>>>>>  From this perspective, I think it can exist in the call stack, and
>>>>>>>> at the same time, we have obtained the maximum call stack
>>>>>>>> information.
>>>>>>>>
>>>>>>>>
>>>>>>>>> b) Can we adjust $sp before entering kvm_exc_entry? Then we can mark
>>>>>>>>> UNWIND_HINT_REGS at the beginning of kvm_exc_entry, which something
>>>>>>>>> like ret_from_kernel_thread_asm.
>>>>>>>>>
>>>>>>>> The following command can be used to dump the orc entries of the
>>>>>>>> kernel:
>>>>>>>> ./tools/objtool/objtool --dump vmlinux
>>>>>>>>
>>>>>>>> You can observe that not all orc entries are generated at the
>>>>>>>> beginning of the function.
>>>>>>>> For example:
>>>>>>>> handle_tlb_protect
>>>>>>>> ftrace_stub
>>>>>>>> handle_reserved
>>>>>>>>
>>>>>>>> So, is it unnecessary for us to modify UNWIND_HINT_REGS in order
>>>>>>>> to place it at the beginning of the function.
>>>>>>>>
>>>>>>>> If you have a better solution, could you provide an example of the
>>>>>>>> modification?
>>>>>>>> I can test the feasibility of the solution.
>>>>>>>>
>>>>>>> The expression at the beginning of the function is incorrect
>>>>>>> (feeling sorry).
>>>>>>> It should be marked where have all stacktrace info.
>>>>>>> Thanks for all the explaining, since I'm unfamiliar with kvm, I
>>>>>>> need these to help my understanding.
>>>>>>>
>>>>>>> Can you try with follows, with save regs by $sp, set more precise
>>>>>>> era to pt_regs, and more unwind hint.
>>>>>>>
>>>>>>>
>>>>>>> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
>>>>>>> index f1768b7a6194..8ed1d7b72c54 100644
>>>>>>> --- a/arch/loongarch/kvm/switch.S
>>>>>>> +++ b/arch/loongarch/kvm/switch.S
>>>>>>> @@ -14,13 +14,13 @@
>>>>>>>    #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>>>>>>
>>>>>>>    .macro kvm_save_host_gpr base
>>>>>>> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
>>>>>>> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
>>>>>>>        st.d    $r\n, \base, HGPR_OFFSET(\n)
>>>>>>>        .endr
>>>>>>>    .endm
>>>>>>>
>>>>>>>    .macro kvm_restore_host_gpr base
>>>>>>> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
>>>>>>> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
>>>>>>>        ld.d    $r\n, \base, HGPR_OFFSET(\n)
>>>>>>>        .endr
>>>>>>>    .endm
>>>>>>> @@ -88,6 +88,7 @@
>>>>>>>        /* Load KVM_ARCH register */
>>>>>>>        ld.d    a2, a2,    (KVM_ARCH_GGPR + 8 * REG_A2)
>>>>>>>
>>>>>>> +111:
>>>>>>>        ertn /* Switch to guest: GSTAT.PGM = 1, ERRCTL.ISERR = 0,
>>>>>>> TLBRPRMD.ISTLBR = 0 */
>>>>>>>    .endm
>>>>>>>
>>>>>>> @@ -158,9 +159,10 @@ SYM_CODE_START(kvm_exc_entry)
>>>>>>>        csrwr    t0, LOONGARCH_CSR_GTLBC
>>>>>>>        ld.d    tp, a2, KVM_ARCH_HTP
>>>>>>>        ld.d    sp, a2, KVM_ARCH_HSP
>>>>>>> +    UNWIND_HINT_REGS
>>>>>>> +
>>>>>>>        /* restore per cpu register */
>>>>>>>        ld.d    u0, a2, KVM_ARCH_HPERCPU
>>>>>>> -    addi.d    sp, sp, -PT_SIZE
>>>>>>>
>>>>>>>        /* Prepare handle exception */
>>>>>>>        or    a0, s0, zero
>>>>>>> @@ -184,10 +186,11 @@ SYM_CODE_START(kvm_exc_entry)
>>>>>>>        csrwr    s1, KVM_VCPU_KS
>>>>>>>        kvm_switch_to_guest
>>>>>>>
>>>>>>> +    UNWIND_HINT_UNDEFINED
>>>>>>>    ret_to_host:
>>>>>>> -    ld.d    a2, a2, KVM_ARCH_HSP
>>>>>>> -    addi.d  a2, a2, -PT_SIZE
>>>>>>> -    kvm_restore_host_gpr    a2
>>>>>>> +    ld.d    sp, a2, KVM_ARCH_HSP
>>>>>>> +    kvm_restore_host_gpr    sp
>>>>>>> +    addi.d    sp, sp, PT_SIZE
>>>>>>>        jr      ra
>>>>>>>
>>>>>>>    SYM_INNER_LABEL(kvm_exc_entry_end, SYM_L_LOCAL)
>>>>>>> @@ -200,11 +203,15 @@ SYM_CODE_END(kvm_exc_entry)
>>>>>>>     *  a0: kvm_run* run
>>>>>>>     *  a1: kvm_vcpu* vcpu
>>>>>>>     */
>>>>>>> -SYM_FUNC_START(kvm_enter_guest)
>>>>>>> +SYM_CODE_START(kvm_enter_guest)
>>>>>>> +    UNWIND_HINT_UNDEFINED
>>>>>>>        /* Allocate space in stack bottom */
>>>>>>> -    addi.d    a2, sp, -PT_SIZE
>>>>>>> +    addi.d    sp, sp, -PT_SIZE
>>>>>>>        /* Save host GPRs */
>>>>>>> -    kvm_save_host_gpr a2
>>>>>>> +    kvm_save_host_gpr sp
>>>>>>> +    la.pcrel a2, 111f
>>>>>>> +    st.d     a2, sp, PT_ERA
>>>>>>> +    UNWIND_HINT_REGS
>>>>>>>
>>>>>> why the label 111f is more accurate?  Supposing there is hw
>>>>>> breakpoint here and backtrace is called, what is the call trace
>>>>>> stack then? obvious label 111f is not executed instead.
>>>>> Xianglai said marking it as regs can get more stack infos, so I use
>>>>> UNWIND_HINT_REGS marked here, though it not called. Remove
>>>>> UNWIND_HINT_REGS thenforbid unwind from here.
>>>>> This function is called and should usually be marked as "call",
>>>>> but it is complex by switching the stack and use `ertn` calls
>>>>> another function.
>>>>>
>>>>>
>>>>>>
>>>>>> UNWIND_HINT_REGS is used for nested kernel stack, is that right?
>>>>>> With nested interrupt and exception handlers on LoongArch kernel, is
>>>>>> UNWIND_HINT_REGS used?
>>>>>>
>>>>>> SYM_CODE_START(ret_from_fork_asm)
>>>>>>          UNWIND_HINT_REGS
>>>>>>          move            a1, sp
>>>>>>          bl              ret_from_fork
>>>>>>          STACKLEAK_ERASE
>>>>>>          RESTORE_STATIC
>>>>>>          RESTORE_SOME
>>>>>>          RESTORE_SP_AND_RET
>>>>>> SYM_CODE_END(ret_from_fork_asm)
>>>>>> With this piece of code, what is contents of pt_regs? In generic it
>>>>>> is called from sys_clone, era is user PC address, is that right? If so,
>>>>>> what is detailed usage in the beginning of ret_from_fork_asm?
>>>>> The stacktrace shows the control flow where the PC will go back, so
>>>>> it is right because when PC is in ret_from_fork_asm, it is already
>>>>> another thread. The era means it will go back user mode.
>>>> The problem is that user mode era shows unwind with error, and
>>>> user_mode(regs) is not accurate. here is piece of code.
>>>>                  pc = regs->csr_era;
>>>>                  if (!__kernel_text_address(pc))
>>>>                          goto err;
>>>> will UNWIND_HINT_END_OF_STACK be better than UNWIND_HINT_REGS?
>>>
>>> You are right. And the reason why current the unwinder does not cause
>>> error is in case ORC_TYPE_REGS we process it by user_mode(regs).
>> Any process about UNWIND_HINT_REGS usage, is nested exception unwind
>> supported now?
>>
>> Talking without any actions seems not be style of Loongson :)
>  From my point of view, Tiezhu's simple solution is acceptable...
is nested exception/interrupt unwind supported now?  IIRC only nested 
interrupt with stack switch is supported, nested exception/interrupt 
unwind without stack switch is not supported. Is that right?

Regards
Bibo Mao

> 
> Huacai
> 
>>
>> Regards
>> Bibo Mao
>>>
>>> Jinyang
>>>
>>

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[Huacai Chen]

On Mon, Apr 13, 2026 at 9:01 AM Bibo Mao <maobibo@loongson.cn> wrote:
>
>
>
> On 2026/4/11 下午10:24, Huacai Chen wrote:
> > On Wed, Apr 8, 2026 at 9:26 AM Bibo Mao <maobibo@loongson.cn> wrote:
> >>
> >>
> >>
> >> On 2025/12/30 下午1:53, Jinyang He wrote:
> >>> On 2025-12-30 12:03, Bibo Mao wrote:
> >>>
> >>>>
> >>>>
> >>>> On 2025/12/30 上午11:36, Jinyang He wrote:
> >>>>> On 2025-12-30 10:24, Bibo Mao wrote:
> >>>>>
> >>>>>>
> >>>>>>
> >>>>>> On 2025/12/29 下午6:41, Jinyang He wrote:
> >>>>>>> On 2025-12-29 18:11, lixianglai wrote:
> >>>>>>>
> >>>>>>>> Hi Jinyang:
> >>>>>>>>>
> >>>>>>>>> On 2025-12-29 11:53, lixianglai wrote:
> >>>>>>>>>> Hi Jinyang:
> >>>>>>>>>>> On 2025-12-27 09:27, Xianglai Li wrote:
> >>>>>>>>>>>
> >>>>>>>>>>>> Insert the appropriate UNWIND macro definition into the
> >>>>>>>>>>>> kvm_exc_entry in
> >>>>>>>>>>>> the assembly function to guide the generation of correct ORC
> >>>>>>>>>>>> table entries,
> >>>>>>>>>>>> thereby solving the timeout problem of loading the
> >>>>>>>>>>>> livepatch-sample module
> >>>>>>>>>>>> on a physical machine running multiple vcpus virtual machines.
> >>>>>>>>>>>>
> >>>>>>>>>>>> While solving the above problems, we have gained an additional
> >>>>>>>>>>>> benefit,
> >>>>>>>>>>>> that is, we can obtain more call stack information
> >>>>>>>>>>>>
> >>>>>>>>>>>> Stack information that can be obtained before the problem is
> >>>>>>>>>>>> fixed:
> >>>>>>>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
> >>>>>>>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
> >>>>>>>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
> >>>>>>>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
> >>>>>>>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
> >>>>>>>>>>>> [<0>] kvm_exc_entry+0x100/0x1e0
> >>>>>>>>>>>>
> >>>>>>>>>>>> Stack information that can be obtained after the problem is
> >>>>>>>>>>>> fixed:
> >>>>>>>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
> >>>>>>>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
> >>>>>>>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
> >>>>>>>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
> >>>>>>>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
> >>>>>>>>>>>> [<0>] kvm_exc_entry+0x104/0x1e4
> >>>>>>>>>>>> [<0>] kvm_enter_guest+0x38/0x11c
> >>>>>>>>>>>> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
> >>>>>>>>>>>> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
> >>>>>>>>>>>> [<0>] sys_ioctl+0x498/0xf00
> >>>>>>>>>>>> [<0>] do_syscall+0x98/0x1d0
> >>>>>>>>>>>> [<0>] handle_syscall+0xb8/0x158
> >>>>>>>>>>>>
> >>>>>>>>>>>> Cc: stable@vger.kernel.org
> >>>>>>>>>>>> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
> >>>>>>>>>>>> ---
> >>>>>>>>>>>> Cc: Huacai Chen <chenhuacai@kernel.org>
> >>>>>>>>>>>> Cc: WANG Xuerui <kernel@xen0n.name>
> >>>>>>>>>>>> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
> >>>>>>>>>>>> Cc: Bibo Mao <maobibo@loongson.cn>
> >>>>>>>>>>>> Cc: Charlie Jenkins <charlie@rivosinc.com>
> >>>>>>>>>>>> Cc: Xianglai Li <lixianglai@loongson.cn>
> >>>>>>>>>>>> Cc: Thomas Gleixner <tglx@linutronix.de>
> >>>>>>>>>>>> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
> >>>>>>>>>>>>
> >>>>>>>>>>>>    arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
> >>>>>>>>>>>>    1 file changed, 19 insertions(+), 9 deletions(-)
> >>>>>>>>>>>>
> >>>>>>>>>>>> diff --git a/arch/loongarch/kvm/switch.S
> >>>>>>>>>>>> b/arch/loongarch/kvm/switch.S
> >>>>>>>>>>>> index 93845ce53651..a3ea9567dbe5 100644
> >>>>>>>>>>>> --- a/arch/loongarch/kvm/switch.S
> >>>>>>>>>>>> +++ b/arch/loongarch/kvm/switch.S
> >>>>>>>>>>>> @@ -10,6 +10,7 @@
> >>>>>>>>>>>>    #include <asm/loongarch.h>
> >>>>>>>>>>>>    #include <asm/regdef.h>
> >>>>>>>>>>>>    #include <asm/unwind_hints.h>
> >>>>>>>>>>>> +#include <linux/kvm_types.h>
> >>>>>>>>>>>>      #define HGPR_OFFSET(x)        (PT_R0 + 8*x)
> >>>>>>>>>>>>    #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
> >>>>>>>>>>>> @@ -110,9 +111,9 @@
> >>>>>>>>>>>>         * need to copy world switch code to DMW area.
> >>>>>>>>>>>>         */
> >>>>>>>>>>>>        .text
> >>>>>>>>>>>> +    .p2align PAGE_SHIFT
> >>>>>>>>>>>>        .cfi_sections    .debug_frame
> >>>>>>>>>>>>    SYM_CODE_START(kvm_exc_entry)
> >>>>>>>>>>>> -    .p2align PAGE_SHIFT
> >>>>>>>>>>>>        UNWIND_HINT_UNDEFINED
> >>>>>>>>>>>>        csrwr    a2,   KVM_TEMP_KS
> >>>>>>>>>>>>        csrrd    a2,   KVM_VCPU_KS
> >>>>>>>>>>>> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
> >>>>>>>>>>>>        /* restore per cpu register */
> >>>>>>>>>>>>        ld.d    u0, a2, KVM_ARCH_HPERCPU
> >>>>>>>>>>>>        addi.d    sp, sp, -PT_SIZE
> >>>>>>>>>>>> +    UNWIND_HINT_REGS
> >>>>>>>>>>>>          /* Prepare handle exception */
> >>>>>>>>>>>>        or    a0, s0, zero
> >>>>>>>>>>>> @@ -200,7 +202,7 @@ ret_to_host:
> >>>>>>>>>>>>        jr      ra
> >>>>>>>>>>>>      SYM_CODE_END(kvm_exc_entry)
> >>>>>>>>>>>> -EXPORT_SYMBOL(kvm_exc_entry)
> >>>>>>>>>>>> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
> >>>>>>>>>>>>      /*
> >>>>>>>>>>>>     * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu
> >>>>>>>>>>>> *vcpu)
> >>>>>>>>>>>> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
> >>>>>>>>>>>>        /* Save host GPRs */
> >>>>>>>>>>>>        kvm_save_host_gpr a2
> >>>>>>>>>>>>    +    /*
> >>>>>>>>>>>> +     * The csr_era member variable of the pt_regs structure
> >>>>>>>>>>>> is required
> >>>>>>>>>>>> +     * for unwinding orc to perform stack traceback, so we
> >>>>>>>>>>>> need to put
> >>>>>>>>>>>> +     * pc into csr_era member variable here.
> >>>>>>>>>>>> +     */
> >>>>>>>>>>>> +    pcaddi    t0, 0
> >>>>>>>>>>>> +    st.d    t0, a2, PT_ERA
> >>>>>>>>>>> Hi, Xianglai,
> >>>>>>>>>>>
> >>>>>>>>>>> It should use `SYM_CODE_START` to mark the `kvm_enter_guest`
> >>>>>>>>>>> rather than
> >>>>>>>>>>> `SYM_FUNC_START`, since the `SYM_FUNC_START` is used to mark
> >>>>>>>>>>> "C-likely"
> >>>>>>>>>>> asm functionw.
> >>>>>>>>>>
> >>>>>>>>>> Ok, I will use SYM_CODE_START to mark kvm_enter_guest in the
> >>>>>>>>>> next version.
> >>>>>>>>>>
> >>>>>>>>>>> I guess the kvm_enter_guest is something like exception
> >>>>>>>>>>> handler becuase the last instruction is "ertn". So usually it
> >>>>>>>>>>> should
> >>>>>>>>>>> mark UNWIND_HINT_REGS where can find last frame info by "$sp".
> >>>>>>>>>>> However, all info is store to "$a2", this mark should be
> >>>>>>>>>>>    `UNWIND_HINT sp_reg=ORC_REG_A2(???) type=UNWIND_HINT_TYPE_REGS`.
> >>>>>>>>>>> I don't konw why save this function internal PC here by `pcaddi
> >>>>>>>>>>> t0, 0`,
> >>>>>>>>>>> and I think it is no meaning(, for exception handler, they save
> >>>>>>>>>>> last PC
> >>>>>>>>>>> by read CSR.ERA). The `kvm_enter_guest` saves registers by
> >>>>>>>>>>> "$a2"("$sp" - PT_REGS) beyond stack ("$sp"), it is dangerous if IE
> >>>>>>>>>>> is enable. So I wonder if there is really a stacktrace through
> >>>>>>>>>>> this function?
> >>>>>>>>>>>
> >>>>>>>>>> The stack backtracking issue in switch.S is rather complex
> >>>>>>>>>> because it involves the switching between cpu root-mode and
> >>>>>>>>>> guest-mode:
> >>>>>>>>>> Real stack backtracking should be divided into two parts:
> >>>>>>>>>> part 1:
> >>>>>>>>>>      [<0>] kvm_enter_guest+0x38/0x11c
> >>>>>>>>>>      [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
> >>>>>>>>>>      [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
> >>>>>>>>>>      [<0>] sys_ioctl+0x498/0xf00
> >>>>>>>>>>      [<0>] do_syscall+0x98/0x1d0
> >>>>>>>>>>      [<0>] handle_syscall+0xb8/0x158
> >>>>>>>>>>
> >>>>>>>>>> part 2:
> >>>>>>>>>>      [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
> >>>>>>>>>>      [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
> >>>>>>>>>>      [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
> >>>>>>>>>>      [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
> >>>>>>>>>>      [<0>] kvm_handle_exit+0x160/0x270 [kvm]
> >>>>>>>>>>      [<0>] kvm_exc_entry+0x104/0x1e4
> >>>>>>>>>>
> >>>>>>>>>>
> >>>>>>>>>> In "part 1", after executing kvm_enter_guest, the cpu switches
> >>>>>>>>>> from root-mode to guest-mode.
> >>>>>>>>>> In this case, stack backtracking is indeed very rare.
> >>>>>>>>>>
> >>>>>>>>>> In "part 2", the cpu switches from the guest-mode to the root-mode,
> >>>>>>>>>> and most of the stack backtracking occurs during this phase.
> >>>>>>>>>>
> >>>>>>>>>> To obtain the longest call chain, we save pc in kvm_enter_guest
> >>>>>>>>>> to pt_regs.csr_era,
> >>>>>>>>>> and after restoring the sp of the root-mode cpu in kvm_exc_entry,
> >>>>>>>>>> The ORC entry was re-established using "UNWIND_HINT_REGS",
> >>>>>>>>>>   and then we obtained the following stack backtrace as we wanted:
> >>>>>>>>>>
> >>>>>>>>>>      [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
> >>>>>>>>>>      [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
> >>>>>>>>>>      [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
> >>>>>>>>>>      [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
> >>>>>>>>>>      [<0>] kvm_handle_exit+0x160/0x270 [kvm]
> >>>>>>>>>>      [<0>] kvm_exc_entry+0x104/0x1e4
> >>>>>>>>> I found this might be a coincidence—correct behavior due to the
> >>>>>>>>> incorrect
> >>>>>>>>> UNWIND_HINT_REGS mark and unusual stack adjustment.
> >>>>>>>>>
> >>>>>>>>> First, the kvm_enter_guest contains only a single branch
> >>>>>>>>> instruction, ertn.
> >>>>>>>>> It hardware-jump to the CSR.ERA address directly, jump into
> >>>>>>>>> kvm_exc_entry.
> >>>>>>>>>
> >>>>>>>>> At this point, the stack layout looks like this:
> >>>>>>>>> -------------------------------
> >>>>>>>>>    frame from call to `kvm_enter_guest`
> >>>>>>>>> -------------------------------  <- $sp
> >>>>>>>>>    PT_REGS
> >>>>>>>>> -------------------------------  <- $a2
> >>>>>>>>>
> >>>>>>>>> Then kvm_exc_entry adjust stack without save any register (e.g.
> >>>>>>>>> $ra, $sp)
> >>>>>>>>> but still marked UNWIND_HINT_REGS.
> >>>>>>>>> After the adjustment:
> >>>>>>>>> -------------------------------
> >>>>>>>>>    frame from call to `kvm_enter_guest`
> >>>>>>>>> -------------------------------
> >>>>>>>>>    PT_REGS
> >>>>>>>>> -------------------------------  <- $a2, new $sp
> >>>>>>>>>
> >>>>>>>>> During unwinding, when the unwinder reaches kvm_exc_entry,
> >>>>>>>>> it meets the mark of PT_REGS and correctly recovers
> >>>>>>>>>   pc = regs.csr_era, sp = regs.sp, ra = regs.ra
> >>>>>>>>>
> >>>>>>>> Yes, here unwinder does work as you say.
> >>>>>>>>
> >>>>>>>>> a) Can we avoid "ertn" rather than `jr reg (or jirl ra, reg, 0)`
> >>>>>>>>> instead, like call?
> >>>>>>>> No,  we need to rely on the 'ertn instruction return PIE to CRMD IE,
> >>>>>>>> at the same time to ensure that its atomic,
> >>>>>>>> there should be no other instruction than' ertn 'more appropriate
> >>>>>>>> here.
> >>>>>>> You are right! I got it.
> >>>>>>>>
> >>>>>>>>> The kvm_exc_entry cannot back to kvm_enter_guest
> >>>>>>>>> if we use "ertn", so should the kvm_enter_guest appear on the
> >>>>>>>>> stacktrace?
> >>>>>>>>>
> >>>>>>>>
> >>>>>>>> It is flexible. As I mentioned above, the cpu completes the switch
> >>>>>>>> from host-mode to guest mode through kvm_enter_guest,
> >>>>>>>> and then the switch from guest mode to host-mode through
> >>>>>>>> kvm_exc_entry. When we ignore the details of the host-mode
> >>>>>>>> and guest-mode switching in the middle, we can understand that the
> >>>>>>>> host cpu has completed kvm_enter_guest->kvm_exc_entry.
> >>>>>>>>  From this perspective, I think it can exist in the call stack, and
> >>>>>>>> at the same time, we have obtained the maximum call stack
> >>>>>>>> information.
> >>>>>>>>
> >>>>>>>>
> >>>>>>>>> b) Can we adjust $sp before entering kvm_exc_entry? Then we can mark
> >>>>>>>>> UNWIND_HINT_REGS at the beginning of kvm_exc_entry, which something
> >>>>>>>>> like ret_from_kernel_thread_asm.
> >>>>>>>>>
> >>>>>>>> The following command can be used to dump the orc entries of the
> >>>>>>>> kernel:
> >>>>>>>> ./tools/objtool/objtool --dump vmlinux
> >>>>>>>>
> >>>>>>>> You can observe that not all orc entries are generated at the
> >>>>>>>> beginning of the function.
> >>>>>>>> For example:
> >>>>>>>> handle_tlb_protect
> >>>>>>>> ftrace_stub
> >>>>>>>> handle_reserved
> >>>>>>>>
> >>>>>>>> So, is it unnecessary for us to modify UNWIND_HINT_REGS in order
> >>>>>>>> to place it at the beginning of the function.
> >>>>>>>>
> >>>>>>>> If you have a better solution, could you provide an example of the
> >>>>>>>> modification?
> >>>>>>>> I can test the feasibility of the solution.
> >>>>>>>>
> >>>>>>> The expression at the beginning of the function is incorrect
> >>>>>>> (feeling sorry).
> >>>>>>> It should be marked where have all stacktrace info.
> >>>>>>> Thanks for all the explaining, since I'm unfamiliar with kvm, I
> >>>>>>> need these to help my understanding.
> >>>>>>>
> >>>>>>> Can you try with follows, with save regs by $sp, set more precise
> >>>>>>> era to pt_regs, and more unwind hint.
> >>>>>>>
> >>>>>>>
> >>>>>>> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
> >>>>>>> index f1768b7a6194..8ed1d7b72c54 100644
> >>>>>>> --- a/arch/loongarch/kvm/switch.S
> >>>>>>> +++ b/arch/loongarch/kvm/switch.S
> >>>>>>> @@ -14,13 +14,13 @@
> >>>>>>>    #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
> >>>>>>>
> >>>>>>>    .macro kvm_save_host_gpr base
> >>>>>>> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
> >>>>>>> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
> >>>>>>>        st.d    $r\n, \base, HGPR_OFFSET(\n)
> >>>>>>>        .endr
> >>>>>>>    .endm
> >>>>>>>
> >>>>>>>    .macro kvm_restore_host_gpr base
> >>>>>>> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
> >>>>>>> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
> >>>>>>>        ld.d    $r\n, \base, HGPR_OFFSET(\n)
> >>>>>>>        .endr
> >>>>>>>    .endm
> >>>>>>> @@ -88,6 +88,7 @@
> >>>>>>>        /* Load KVM_ARCH register */
> >>>>>>>        ld.d    a2, a2,    (KVM_ARCH_GGPR + 8 * REG_A2)
> >>>>>>>
> >>>>>>> +111:
> >>>>>>>        ertn /* Switch to guest: GSTAT.PGM = 1, ERRCTL.ISERR = 0,
> >>>>>>> TLBRPRMD.ISTLBR = 0 */
> >>>>>>>    .endm
> >>>>>>>
> >>>>>>> @@ -158,9 +159,10 @@ SYM_CODE_START(kvm_exc_entry)
> >>>>>>>        csrwr    t0, LOONGARCH_CSR_GTLBC
> >>>>>>>        ld.d    tp, a2, KVM_ARCH_HTP
> >>>>>>>        ld.d    sp, a2, KVM_ARCH_HSP
> >>>>>>> +    UNWIND_HINT_REGS
> >>>>>>> +
> >>>>>>>        /* restore per cpu register */
> >>>>>>>        ld.d    u0, a2, KVM_ARCH_HPERCPU
> >>>>>>> -    addi.d    sp, sp, -PT_SIZE
> >>>>>>>
> >>>>>>>        /* Prepare handle exception */
> >>>>>>>        or    a0, s0, zero
> >>>>>>> @@ -184,10 +186,11 @@ SYM_CODE_START(kvm_exc_entry)
> >>>>>>>        csrwr    s1, KVM_VCPU_KS
> >>>>>>>        kvm_switch_to_guest
> >>>>>>>
> >>>>>>> +    UNWIND_HINT_UNDEFINED
> >>>>>>>    ret_to_host:
> >>>>>>> -    ld.d    a2, a2, KVM_ARCH_HSP
> >>>>>>> -    addi.d  a2, a2, -PT_SIZE
> >>>>>>> -    kvm_restore_host_gpr    a2
> >>>>>>> +    ld.d    sp, a2, KVM_ARCH_HSP
> >>>>>>> +    kvm_restore_host_gpr    sp
> >>>>>>> +    addi.d    sp, sp, PT_SIZE
> >>>>>>>        jr      ra
> >>>>>>>
> >>>>>>>    SYM_INNER_LABEL(kvm_exc_entry_end, SYM_L_LOCAL)
> >>>>>>> @@ -200,11 +203,15 @@ SYM_CODE_END(kvm_exc_entry)
> >>>>>>>     *  a0: kvm_run* run
> >>>>>>>     *  a1: kvm_vcpu* vcpu
> >>>>>>>     */
> >>>>>>> -SYM_FUNC_START(kvm_enter_guest)
> >>>>>>> +SYM_CODE_START(kvm_enter_guest)
> >>>>>>> +    UNWIND_HINT_UNDEFINED
> >>>>>>>        /* Allocate space in stack bottom */
> >>>>>>> -    addi.d    a2, sp, -PT_SIZE
> >>>>>>> +    addi.d    sp, sp, -PT_SIZE
> >>>>>>>        /* Save host GPRs */
> >>>>>>> -    kvm_save_host_gpr a2
> >>>>>>> +    kvm_save_host_gpr sp
> >>>>>>> +    la.pcrel a2, 111f
> >>>>>>> +    st.d     a2, sp, PT_ERA
> >>>>>>> +    UNWIND_HINT_REGS
> >>>>>>>
> >>>>>> why the label 111f is more accurate?  Supposing there is hw
> >>>>>> breakpoint here and backtrace is called, what is the call trace
> >>>>>> stack then? obvious label 111f is not executed instead.
> >>>>> Xianglai said marking it as regs can get more stack infos, so I use
> >>>>> UNWIND_HINT_REGS marked here, though it not called. Remove
> >>>>> UNWIND_HINT_REGS thenforbid unwind from here.
> >>>>> This function is called and should usually be marked as "call",
> >>>>> but it is complex by switching the stack and use `ertn` calls
> >>>>> another function.
> >>>>>
> >>>>>
> >>>>>>
> >>>>>> UNWIND_HINT_REGS is used for nested kernel stack, is that right?
> >>>>>> With nested interrupt and exception handlers on LoongArch kernel, is
> >>>>>> UNWIND_HINT_REGS used?
> >>>>>>
> >>>>>> SYM_CODE_START(ret_from_fork_asm)
> >>>>>>          UNWIND_HINT_REGS
> >>>>>>          move            a1, sp
> >>>>>>          bl              ret_from_fork
> >>>>>>          STACKLEAK_ERASE
> >>>>>>          RESTORE_STATIC
> >>>>>>          RESTORE_SOME
> >>>>>>          RESTORE_SP_AND_RET
> >>>>>> SYM_CODE_END(ret_from_fork_asm)
> >>>>>> With this piece of code, what is contents of pt_regs? In generic it
> >>>>>> is called from sys_clone, era is user PC address, is that right? If so,
> >>>>>> what is detailed usage in the beginning of ret_from_fork_asm?
> >>>>> The stacktrace shows the control flow where the PC will go back, so
> >>>>> it is right because when PC is in ret_from_fork_asm, it is already
> >>>>> another thread. The era means it will go back user mode.
> >>>> The problem is that user mode era shows unwind with error, and
> >>>> user_mode(regs) is not accurate. here is piece of code.
> >>>>                  pc = regs->csr_era;
> >>>>                  if (!__kernel_text_address(pc))
> >>>>                          goto err;
> >>>> will UNWIND_HINT_END_OF_STACK be better than UNWIND_HINT_REGS?
> >>>
> >>> You are right. And the reason why current the unwinder does not cause
> >>> error is in case ORC_TYPE_REGS we process it by user_mode(regs).
> >> Any process about UNWIND_HINT_REGS usage, is nested exception unwind
> >> supported now?
> >>
> >> Talking without any actions seems not be style of Loongson :)
> >  From my point of view, Tiezhu's simple solution is acceptable...
> is nested exception/interrupt unwind supported now?  IIRC only nested
> interrupt with stack switch is supported, nested exception/interrupt
> unwind without stack switch is not supported. Is that right?
Nested exception is supported, but it doesn't switch stack; only
interrupt need switch stack.


Huacai

>
> Regards
> Bibo Mao
>
> >
> > Huacai
> >
> >>
> >> Regards
> >> Bibo Mao
> >>>
> >>> Jinyang
> >>>
> >>
>

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[Bibo Mao]

On 2026/4/13 上午11:11, Huacai Chen wrote:
> On Mon, Apr 13, 2026 at 9:01 AM Bibo Mao <maobibo@loongson.cn> wrote:
>>
>>
>>
>> On 2026/4/11 下午10:24, Huacai Chen wrote:
>>> On Wed, Apr 8, 2026 at 9:26 AM Bibo Mao <maobibo@loongson.cn> wrote:
>>>>
>>>>
>>>>
>>>> On 2025/12/30 下午1:53, Jinyang He wrote:
>>>>> On 2025-12-30 12:03, Bibo Mao wrote:
>>>>>
>>>>>>
>>>>>>
>>>>>> On 2025/12/30 上午11:36, Jinyang He wrote:
>>>>>>> On 2025-12-30 10:24, Bibo Mao wrote:
>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>>> On 2025/12/29 下午6:41, Jinyang He wrote:
>>>>>>>>> On 2025-12-29 18:11, lixianglai wrote:
>>>>>>>>>
>>>>>>>>>> Hi Jinyang:
>>>>>>>>>>>
>>>>>>>>>>> On 2025-12-29 11:53, lixianglai wrote:
>>>>>>>>>>>> Hi Jinyang:
>>>>>>>>>>>>> On 2025-12-27 09:27, Xianglai Li wrote:
>>>>>>>>>>>>>
>>>>>>>>>>>>>> Insert the appropriate UNWIND macro definition into the
>>>>>>>>>>>>>> kvm_exc_entry in
>>>>>>>>>>>>>> the assembly function to guide the generation of correct ORC
>>>>>>>>>>>>>> table entries,
>>>>>>>>>>>>>> thereby solving the timeout problem of loading the
>>>>>>>>>>>>>> livepatch-sample module
>>>>>>>>>>>>>> on a physical machine running multiple vcpus virtual machines.
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> While solving the above problems, we have gained an additional
>>>>>>>>>>>>>> benefit,
>>>>>>>>>>>>>> that is, we can obtain more call stack information
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> Stack information that can be obtained before the problem is
>>>>>>>>>>>>>> fixed:
>>>>>>>>>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>>>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>>>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>>>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>>>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>>>>>>>> [<0>] kvm_exc_entry+0x100/0x1e0
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> Stack information that can be obtained after the problem is
>>>>>>>>>>>>>> fixed:
>>>>>>>>>>>>>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>>>>>>>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>>>>>>>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>>>>>>>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>>>>>>>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>>>>>>>> [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>>>>>>>>>> [<0>] kvm_enter_guest+0x38/0x11c
>>>>>>>>>>>>>> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>>>>>>>>>>>> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>>>>>>>>>>>> [<0>] sys_ioctl+0x498/0xf00
>>>>>>>>>>>>>> [<0>] do_syscall+0x98/0x1d0
>>>>>>>>>>>>>> [<0>] handle_syscall+0xb8/0x158
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> Cc: stable@vger.kernel.org
>>>>>>>>>>>>>> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
>>>>>>>>>>>>>> ---
>>>>>>>>>>>>>> Cc: Huacai Chen <chenhuacai@kernel.org>
>>>>>>>>>>>>>> Cc: WANG Xuerui <kernel@xen0n.name>
>>>>>>>>>>>>>> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
>>>>>>>>>>>>>> Cc: Bibo Mao <maobibo@loongson.cn>
>>>>>>>>>>>>>> Cc: Charlie Jenkins <charlie@rivosinc.com>
>>>>>>>>>>>>>> Cc: Xianglai Li <lixianglai@loongson.cn>
>>>>>>>>>>>>>> Cc: Thomas Gleixner <tglx@linutronix.de>
>>>>>>>>>>>>>> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
>>>>>>>>>>>>>>
>>>>>>>>>>>>>>     arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
>>>>>>>>>>>>>>     1 file changed, 19 insertions(+), 9 deletions(-)
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> diff --git a/arch/loongarch/kvm/switch.S
>>>>>>>>>>>>>> b/arch/loongarch/kvm/switch.S
>>>>>>>>>>>>>> index 93845ce53651..a3ea9567dbe5 100644
>>>>>>>>>>>>>> --- a/arch/loongarch/kvm/switch.S
>>>>>>>>>>>>>> +++ b/arch/loongarch/kvm/switch.S
>>>>>>>>>>>>>> @@ -10,6 +10,7 @@
>>>>>>>>>>>>>>     #include <asm/loongarch.h>
>>>>>>>>>>>>>>     #include <asm/regdef.h>
>>>>>>>>>>>>>>     #include <asm/unwind_hints.h>
>>>>>>>>>>>>>> +#include <linux/kvm_types.h>
>>>>>>>>>>>>>>       #define HGPR_OFFSET(x)        (PT_R0 + 8*x)
>>>>>>>>>>>>>>     #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>>>>>>>>>>>>> @@ -110,9 +111,9 @@
>>>>>>>>>>>>>>          * need to copy world switch code to DMW area.
>>>>>>>>>>>>>>          */
>>>>>>>>>>>>>>         .text
>>>>>>>>>>>>>> +    .p2align PAGE_SHIFT
>>>>>>>>>>>>>>         .cfi_sections    .debug_frame
>>>>>>>>>>>>>>     SYM_CODE_START(kvm_exc_entry)
>>>>>>>>>>>>>> -    .p2align PAGE_SHIFT
>>>>>>>>>>>>>>         UNWIND_HINT_UNDEFINED
>>>>>>>>>>>>>>         csrwr    a2,   KVM_TEMP_KS
>>>>>>>>>>>>>>         csrrd    a2,   KVM_VCPU_KS
>>>>>>>>>>>>>> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
>>>>>>>>>>>>>>         /* restore per cpu register */
>>>>>>>>>>>>>>         ld.d    u0, a2, KVM_ARCH_HPERCPU
>>>>>>>>>>>>>>         addi.d    sp, sp, -PT_SIZE
>>>>>>>>>>>>>> +    UNWIND_HINT_REGS
>>>>>>>>>>>>>>           /* Prepare handle exception */
>>>>>>>>>>>>>>         or    a0, s0, zero
>>>>>>>>>>>>>> @@ -200,7 +202,7 @@ ret_to_host:
>>>>>>>>>>>>>>         jr      ra
>>>>>>>>>>>>>>       SYM_CODE_END(kvm_exc_entry)
>>>>>>>>>>>>>> -EXPORT_SYMBOL(kvm_exc_entry)
>>>>>>>>>>>>>> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
>>>>>>>>>>>>>>       /*
>>>>>>>>>>>>>>      * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu
>>>>>>>>>>>>>> *vcpu)
>>>>>>>>>>>>>> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
>>>>>>>>>>>>>>         /* Save host GPRs */
>>>>>>>>>>>>>>         kvm_save_host_gpr a2
>>>>>>>>>>>>>>     +    /*
>>>>>>>>>>>>>> +     * The csr_era member variable of the pt_regs structure
>>>>>>>>>>>>>> is required
>>>>>>>>>>>>>> +     * for unwinding orc to perform stack traceback, so we
>>>>>>>>>>>>>> need to put
>>>>>>>>>>>>>> +     * pc into csr_era member variable here.
>>>>>>>>>>>>>> +     */
>>>>>>>>>>>>>> +    pcaddi    t0, 0
>>>>>>>>>>>>>> +    st.d    t0, a2, PT_ERA
>>>>>>>>>>>>> Hi, Xianglai,
>>>>>>>>>>>>>
>>>>>>>>>>>>> It should use `SYM_CODE_START` to mark the `kvm_enter_guest`
>>>>>>>>>>>>> rather than
>>>>>>>>>>>>> `SYM_FUNC_START`, since the `SYM_FUNC_START` is used to mark
>>>>>>>>>>>>> "C-likely"
>>>>>>>>>>>>> asm functionw.
>>>>>>>>>>>>
>>>>>>>>>>>> Ok, I will use SYM_CODE_START to mark kvm_enter_guest in the
>>>>>>>>>>>> next version.
>>>>>>>>>>>>
>>>>>>>>>>>>> I guess the kvm_enter_guest is something like exception
>>>>>>>>>>>>> handler becuase the last instruction is "ertn". So usually it
>>>>>>>>>>>>> should
>>>>>>>>>>>>> mark UNWIND_HINT_REGS where can find last frame info by "$sp".
>>>>>>>>>>>>> However, all info is store to "$a2", this mark should be
>>>>>>>>>>>>>     `UNWIND_HINT sp_reg=ORC_REG_A2(???) type=UNWIND_HINT_TYPE_REGS`.
>>>>>>>>>>>>> I don't konw why save this function internal PC here by `pcaddi
>>>>>>>>>>>>> t0, 0`,
>>>>>>>>>>>>> and I think it is no meaning(, for exception handler, they save
>>>>>>>>>>>>> last PC
>>>>>>>>>>>>> by read CSR.ERA). The `kvm_enter_guest` saves registers by
>>>>>>>>>>>>> "$a2"("$sp" - PT_REGS) beyond stack ("$sp"), it is dangerous if IE
>>>>>>>>>>>>> is enable. So I wonder if there is really a stacktrace through
>>>>>>>>>>>>> this function?
>>>>>>>>>>>>>
>>>>>>>>>>>> The stack backtracking issue in switch.S is rather complex
>>>>>>>>>>>> because it involves the switching between cpu root-mode and
>>>>>>>>>>>> guest-mode:
>>>>>>>>>>>> Real stack backtracking should be divided into two parts:
>>>>>>>>>>>> part 1:
>>>>>>>>>>>>       [<0>] kvm_enter_guest+0x38/0x11c
>>>>>>>>>>>>       [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>>>>>>>>>>>>       [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>>>>>>>>>>>>       [<0>] sys_ioctl+0x498/0xf00
>>>>>>>>>>>>       [<0>] do_syscall+0x98/0x1d0
>>>>>>>>>>>>       [<0>] handle_syscall+0xb8/0x158
>>>>>>>>>>>>
>>>>>>>>>>>> part 2:
>>>>>>>>>>>>       [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>>>>>>       [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>>>>>>       [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>>>>>>       [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>>>>>>       [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>>>>>>       [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>>>>>>>>
>>>>>>>>>>>>
>>>>>>>>>>>> In "part 1", after executing kvm_enter_guest, the cpu switches
>>>>>>>>>>>> from root-mode to guest-mode.
>>>>>>>>>>>> In this case, stack backtracking is indeed very rare.
>>>>>>>>>>>>
>>>>>>>>>>>> In "part 2", the cpu switches from the guest-mode to the root-mode,
>>>>>>>>>>>> and most of the stack backtracking occurs during this phase.
>>>>>>>>>>>>
>>>>>>>>>>>> To obtain the longest call chain, we save pc in kvm_enter_guest
>>>>>>>>>>>> to pt_regs.csr_era,
>>>>>>>>>>>> and after restoring the sp of the root-mode cpu in kvm_exc_entry,
>>>>>>>>>>>> The ORC entry was re-established using "UNWIND_HINT_REGS",
>>>>>>>>>>>>    and then we obtained the following stack backtrace as we wanted:
>>>>>>>>>>>>
>>>>>>>>>>>>       [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>>>>>>>>>>>>       [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>>>>>>>>>>>>       [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>>>>>>>>>>>>       [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>>>>>>>>>>>>       [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>>>>>>>>>>>>       [<0>] kvm_exc_entry+0x104/0x1e4
>>>>>>>>>>> I found this might be a coincidence—correct behavior due to the
>>>>>>>>>>> incorrect
>>>>>>>>>>> UNWIND_HINT_REGS mark and unusual stack adjustment.
>>>>>>>>>>>
>>>>>>>>>>> First, the kvm_enter_guest contains only a single branch
>>>>>>>>>>> instruction, ertn.
>>>>>>>>>>> It hardware-jump to the CSR.ERA address directly, jump into
>>>>>>>>>>> kvm_exc_entry.
>>>>>>>>>>>
>>>>>>>>>>> At this point, the stack layout looks like this:
>>>>>>>>>>> -------------------------------
>>>>>>>>>>>     frame from call to `kvm_enter_guest`
>>>>>>>>>>> -------------------------------  <- $sp
>>>>>>>>>>>     PT_REGS
>>>>>>>>>>> -------------------------------  <- $a2
>>>>>>>>>>>
>>>>>>>>>>> Then kvm_exc_entry adjust stack without save any register (e.g.
>>>>>>>>>>> $ra, $sp)
>>>>>>>>>>> but still marked UNWIND_HINT_REGS.
>>>>>>>>>>> After the adjustment:
>>>>>>>>>>> -------------------------------
>>>>>>>>>>>     frame from call to `kvm_enter_guest`
>>>>>>>>>>> -------------------------------
>>>>>>>>>>>     PT_REGS
>>>>>>>>>>> -------------------------------  <- $a2, new $sp
>>>>>>>>>>>
>>>>>>>>>>> During unwinding, when the unwinder reaches kvm_exc_entry,
>>>>>>>>>>> it meets the mark of PT_REGS and correctly recovers
>>>>>>>>>>>    pc = regs.csr_era, sp = regs.sp, ra = regs.ra
>>>>>>>>>>>
>>>>>>>>>> Yes, here unwinder does work as you say.
>>>>>>>>>>
>>>>>>>>>>> a) Can we avoid "ertn" rather than `jr reg (or jirl ra, reg, 0)`
>>>>>>>>>>> instead, like call?
>>>>>>>>>> No,  we need to rely on the 'ertn instruction return PIE to CRMD IE,
>>>>>>>>>> at the same time to ensure that its atomic,
>>>>>>>>>> there should be no other instruction than' ertn 'more appropriate
>>>>>>>>>> here.
>>>>>>>>> You are right! I got it.
>>>>>>>>>>
>>>>>>>>>>> The kvm_exc_entry cannot back to kvm_enter_guest
>>>>>>>>>>> if we use "ertn", so should the kvm_enter_guest appear on the
>>>>>>>>>>> stacktrace?
>>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>> It is flexible. As I mentioned above, the cpu completes the switch
>>>>>>>>>> from host-mode to guest mode through kvm_enter_guest,
>>>>>>>>>> and then the switch from guest mode to host-mode through
>>>>>>>>>> kvm_exc_entry. When we ignore the details of the host-mode
>>>>>>>>>> and guest-mode switching in the middle, we can understand that the
>>>>>>>>>> host cpu has completed kvm_enter_guest->kvm_exc_entry.
>>>>>>>>>>   From this perspective, I think it can exist in the call stack, and
>>>>>>>>>> at the same time, we have obtained the maximum call stack
>>>>>>>>>> information.
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>>> b) Can we adjust $sp before entering kvm_exc_entry? Then we can mark
>>>>>>>>>>> UNWIND_HINT_REGS at the beginning of kvm_exc_entry, which something
>>>>>>>>>>> like ret_from_kernel_thread_asm.
>>>>>>>>>>>
>>>>>>>>>> The following command can be used to dump the orc entries of the
>>>>>>>>>> kernel:
>>>>>>>>>> ./tools/objtool/objtool --dump vmlinux
>>>>>>>>>>
>>>>>>>>>> You can observe that not all orc entries are generated at the
>>>>>>>>>> beginning of the function.
>>>>>>>>>> For example:
>>>>>>>>>> handle_tlb_protect
>>>>>>>>>> ftrace_stub
>>>>>>>>>> handle_reserved
>>>>>>>>>>
>>>>>>>>>> So, is it unnecessary for us to modify UNWIND_HINT_REGS in order
>>>>>>>>>> to place it at the beginning of the function.
>>>>>>>>>>
>>>>>>>>>> If you have a better solution, could you provide an example of the
>>>>>>>>>> modification?
>>>>>>>>>> I can test the feasibility of the solution.
>>>>>>>>>>
>>>>>>>>> The expression at the beginning of the function is incorrect
>>>>>>>>> (feeling sorry).
>>>>>>>>> It should be marked where have all stacktrace info.
>>>>>>>>> Thanks for all the explaining, since I'm unfamiliar with kvm, I
>>>>>>>>> need these to help my understanding.
>>>>>>>>>
>>>>>>>>> Can you try with follows, with save regs by $sp, set more precise
>>>>>>>>> era to pt_regs, and more unwind hint.
>>>>>>>>>
>>>>>>>>>
>>>>>>>>> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
>>>>>>>>> index f1768b7a6194..8ed1d7b72c54 100644
>>>>>>>>> --- a/arch/loongarch/kvm/switch.S
>>>>>>>>> +++ b/arch/loongarch/kvm/switch.S
>>>>>>>>> @@ -14,13 +14,13 @@
>>>>>>>>>     #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>>>>>>>>>
>>>>>>>>>     .macro kvm_save_host_gpr base
>>>>>>>>> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
>>>>>>>>> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
>>>>>>>>>         st.d    $r\n, \base, HGPR_OFFSET(\n)
>>>>>>>>>         .endr
>>>>>>>>>     .endm
>>>>>>>>>
>>>>>>>>>     .macro kvm_restore_host_gpr base
>>>>>>>>> -    .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
>>>>>>>>> +    .irp n,1,2,22,23,24,25,26,27,28,29,30,31
>>>>>>>>>         ld.d    $r\n, \base, HGPR_OFFSET(\n)
>>>>>>>>>         .endr
>>>>>>>>>     .endm
>>>>>>>>> @@ -88,6 +88,7 @@
>>>>>>>>>         /* Load KVM_ARCH register */
>>>>>>>>>         ld.d    a2, a2,    (KVM_ARCH_GGPR + 8 * REG_A2)
>>>>>>>>>
>>>>>>>>> +111:
>>>>>>>>>         ertn /* Switch to guest: GSTAT.PGM = 1, ERRCTL.ISERR = 0,
>>>>>>>>> TLBRPRMD.ISTLBR = 0 */
>>>>>>>>>     .endm
>>>>>>>>>
>>>>>>>>> @@ -158,9 +159,10 @@ SYM_CODE_START(kvm_exc_entry)
>>>>>>>>>         csrwr    t0, LOONGARCH_CSR_GTLBC
>>>>>>>>>         ld.d    tp, a2, KVM_ARCH_HTP
>>>>>>>>>         ld.d    sp, a2, KVM_ARCH_HSP
>>>>>>>>> +    UNWIND_HINT_REGS
>>>>>>>>> +
>>>>>>>>>         /* restore per cpu register */
>>>>>>>>>         ld.d    u0, a2, KVM_ARCH_HPERCPU
>>>>>>>>> -    addi.d    sp, sp, -PT_SIZE
>>>>>>>>>
>>>>>>>>>         /* Prepare handle exception */
>>>>>>>>>         or    a0, s0, zero
>>>>>>>>> @@ -184,10 +186,11 @@ SYM_CODE_START(kvm_exc_entry)
>>>>>>>>>         csrwr    s1, KVM_VCPU_KS
>>>>>>>>>         kvm_switch_to_guest
>>>>>>>>>
>>>>>>>>> +    UNWIND_HINT_UNDEFINED
>>>>>>>>>     ret_to_host:
>>>>>>>>> -    ld.d    a2, a2, KVM_ARCH_HSP
>>>>>>>>> -    addi.d  a2, a2, -PT_SIZE
>>>>>>>>> -    kvm_restore_host_gpr    a2
>>>>>>>>> +    ld.d    sp, a2, KVM_ARCH_HSP
>>>>>>>>> +    kvm_restore_host_gpr    sp
>>>>>>>>> +    addi.d    sp, sp, PT_SIZE
>>>>>>>>>         jr      ra
>>>>>>>>>
>>>>>>>>>     SYM_INNER_LABEL(kvm_exc_entry_end, SYM_L_LOCAL)
>>>>>>>>> @@ -200,11 +203,15 @@ SYM_CODE_END(kvm_exc_entry)
>>>>>>>>>      *  a0: kvm_run* run
>>>>>>>>>      *  a1: kvm_vcpu* vcpu
>>>>>>>>>      */
>>>>>>>>> -SYM_FUNC_START(kvm_enter_guest)
>>>>>>>>> +SYM_CODE_START(kvm_enter_guest)
>>>>>>>>> +    UNWIND_HINT_UNDEFINED
>>>>>>>>>         /* Allocate space in stack bottom */
>>>>>>>>> -    addi.d    a2, sp, -PT_SIZE
>>>>>>>>> +    addi.d    sp, sp, -PT_SIZE
>>>>>>>>>         /* Save host GPRs */
>>>>>>>>> -    kvm_save_host_gpr a2
>>>>>>>>> +    kvm_save_host_gpr sp
>>>>>>>>> +    la.pcrel a2, 111f
>>>>>>>>> +    st.d     a2, sp, PT_ERA
>>>>>>>>> +    UNWIND_HINT_REGS
>>>>>>>>>
>>>>>>>> why the label 111f is more accurate?  Supposing there is hw
>>>>>>>> breakpoint here and backtrace is called, what is the call trace
>>>>>>>> stack then? obvious label 111f is not executed instead.
>>>>>>> Xianglai said marking it as regs can get more stack infos, so I use
>>>>>>> UNWIND_HINT_REGS marked here, though it not called. Remove
>>>>>>> UNWIND_HINT_REGS thenforbid unwind from here.
>>>>>>> This function is called and should usually be marked as "call",
>>>>>>> but it is complex by switching the stack and use `ertn` calls
>>>>>>> another function.
>>>>>>>
>>>>>>>
>>>>>>>>
>>>>>>>> UNWIND_HINT_REGS is used for nested kernel stack, is that right?
>>>>>>>> With nested interrupt and exception handlers on LoongArch kernel, is
>>>>>>>> UNWIND_HINT_REGS used?
>>>>>>>>
>>>>>>>> SYM_CODE_START(ret_from_fork_asm)
>>>>>>>>           UNWIND_HINT_REGS
>>>>>>>>           move            a1, sp
>>>>>>>>           bl              ret_from_fork
>>>>>>>>           STACKLEAK_ERASE
>>>>>>>>           RESTORE_STATIC
>>>>>>>>           RESTORE_SOME
>>>>>>>>           RESTORE_SP_AND_RET
>>>>>>>> SYM_CODE_END(ret_from_fork_asm)
>>>>>>>> With this piece of code, what is contents of pt_regs? In generic it
>>>>>>>> is called from sys_clone, era is user PC address, is that right? If so,
>>>>>>>> what is detailed usage in the beginning of ret_from_fork_asm?
>>>>>>> The stacktrace shows the control flow where the PC will go back, so
>>>>>>> it is right because when PC is in ret_from_fork_asm, it is already
>>>>>>> another thread. The era means it will go back user mode.
>>>>>> The problem is that user mode era shows unwind with error, and
>>>>>> user_mode(regs) is not accurate. here is piece of code.
>>>>>>                   pc = regs->csr_era;
>>>>>>                   if (!__kernel_text_address(pc))
>>>>>>                           goto err;
>>>>>> will UNWIND_HINT_END_OF_STACK be better than UNWIND_HINT_REGS?
>>>>>
>>>>> You are right. And the reason why current the unwinder does not cause
>>>>> error is in case ORC_TYPE_REGS we process it by user_mode(regs).
>>>> Any process about UNWIND_HINT_REGS usage, is nested exception unwind
>>>> supported now?
>>>>
>>>> Talking without any actions seems not be style of Loongson :)
>>>   From my point of view, Tiezhu's simple solution is acceptable...
>> is nested exception/interrupt unwind supported now?  IIRC only nested
>> interrupt with stack switch is supported, nested exception/interrupt
>> unwind without stack switch is not supported. Is that right?
> Nested exception is supported, but it doesn't switch stack; only
> interrupt need switch stack.
yes, you are right. Nested exception is supported, UNWIND_HINT_REGS() is 
used in SAVE_SOME() macro.

Regards
Bibo Mao
> 
> 
> Huacai
> 
>>
>> Regards
>> Bibo Mao
>>
>>>
>>> Huacai
>>>
>>>>
>>>> Regards
>>>> Bibo Mao
>>>>>
>>>>> Jinyang
>>>>>
>>>>
>>

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[Bibo Mao]

On 2025/12/27 上午9:27, Xianglai Li wrote:
> Insert the appropriate UNWIND macro definition into the kvm_exc_entry in
> the assembly function to guide the generation of correct ORC table entries,
> thereby solving the timeout problem of loading the livepatch-sample module
> on a physical machine running multiple vcpus virtual machines.
> 
> While solving the above problems, we have gained an additional benefit,
> that is, we can obtain more call stack information
> 
> Stack information that can be obtained before the problem is fixed:
> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
> [<0>] kvm_exc_entry+0x100/0x1e0
> 
> Stack information that can be obtained after the problem is fixed:
> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
> [<0>] kvm_exc_entry+0x104/0x1e4
> [<0>] kvm_enter_guest+0x38/0x11c
> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
> [<0>] sys_ioctl+0x498/0xf00
> [<0>] do_syscall+0x98/0x1d0
> [<0>] handle_syscall+0xb8/0x158
> 
> Cc: stable@vger.kernel.org
> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
> ---
> Cc: Huacai Chen <chenhuacai@kernel.org>
> Cc: WANG Xuerui <kernel@xen0n.name>
> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
> Cc: Bibo Mao <maobibo@loongson.cn>
> Cc: Charlie Jenkins <charlie@rivosinc.com>
> Cc: Xianglai Li <lixianglai@loongson.cn>
> Cc: Thomas Gleixner <tglx@linutronix.de>
> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
> 
>   arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
>   1 file changed, 19 insertions(+), 9 deletions(-)
> 
> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
> index 93845ce53651..a3ea9567dbe5 100644
> --- a/arch/loongarch/kvm/switch.S
> +++ b/arch/loongarch/kvm/switch.S
> @@ -10,6 +10,7 @@
>   #include <asm/loongarch.h>
>   #include <asm/regdef.h>
>   #include <asm/unwind_hints.h>
> +#include <linux/kvm_types.h>
>   
>   #define HGPR_OFFSET(x)		(PT_R0 + 8*x)
>   #define GGPR_OFFSET(x)		(KVM_ARCH_GGPR + 8*x)
> @@ -110,9 +111,9 @@
>   	 * need to copy world switch code to DMW area.
>   	 */
>   	.text
> +	.p2align PAGE_SHIFT
>   	.cfi_sections	.debug_frame
>   SYM_CODE_START(kvm_exc_entry)
> -	.p2align PAGE_SHIFT
>   	UNWIND_HINT_UNDEFINED
>   	csrwr	a2,   KVM_TEMP_KS
>   	csrrd	a2,   KVM_VCPU_KS
> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
>   	/* restore per cpu register */
>   	ld.d	u0, a2, KVM_ARCH_HPERCPU
>   	addi.d	sp, sp, -PT_SIZE
> +	UNWIND_HINT_REGS
>   
>   	/* Prepare handle exception */
>   	or	a0, s0, zero
> @@ -200,7 +202,7 @@ ret_to_host:
>   	jr      ra
>   
>   SYM_CODE_END(kvm_exc_entry)
> -EXPORT_SYMBOL(kvm_exc_entry)
> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
>   
>   /*
>    * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu)
> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
>   	/* Save host GPRs */
>   	kvm_save_host_gpr a2
>   
> +	/*
> +	 * The csr_era member variable of the pt_regs structure is required
> +	 * for unwinding orc to perform stack traceback, so we need to put
> +	 * pc into csr_era member variable here.
> +	 */
> +	pcaddi	t0, 0
> +	st.d	t0, a2, PT_ERA
maybe PRMD need be set with fake pt_regs also, something like this:
         ori     t0, zero, CSR_PRMD_PIE
         st.d	t0, a2, PT_PRMD

Regards
Bibo Mao
> +
>   	addi.d	a2, a1, KVM_VCPU_ARCH
>   	st.d	sp, a2, KVM_ARCH_HSP
>   	st.d	tp, a2, KVM_ARCH_HTP
> @@ -225,7 +235,7 @@ SYM_FUNC_START(kvm_enter_guest)
>   	csrwr	a1, KVM_VCPU_KS
>   	kvm_switch_to_guest
>   SYM_FUNC_END(kvm_enter_guest)
> -EXPORT_SYMBOL(kvm_enter_guest)
> +EXPORT_SYMBOL_FOR_KVM(kvm_enter_guest)
>   
>   SYM_FUNC_START(kvm_save_fpu)
>   	fpu_save_csr	a0 t1
> @@ -233,7 +243,7 @@ SYM_FUNC_START(kvm_save_fpu)
>   	fpu_save_cc	a0 t1 t2
>   	jr              ra
>   SYM_FUNC_END(kvm_save_fpu)
> -EXPORT_SYMBOL(kvm_save_fpu)
> +EXPORT_SYMBOL_FOR_KVM(kvm_save_fpu)
>   
>   SYM_FUNC_START(kvm_restore_fpu)
>   	fpu_restore_double a0 t1
> @@ -241,7 +251,7 @@ SYM_FUNC_START(kvm_restore_fpu)
>   	fpu_restore_cc	   a0 t1 t2
>   	jr                 ra
>   SYM_FUNC_END(kvm_restore_fpu)
> -EXPORT_SYMBOL(kvm_restore_fpu)
> +EXPORT_SYMBOL_FOR_KVM(kvm_restore_fpu)
>   
>   #ifdef CONFIG_CPU_HAS_LSX
>   SYM_FUNC_START(kvm_save_lsx)
> @@ -250,7 +260,7 @@ SYM_FUNC_START(kvm_save_lsx)
>   	lsx_save_data   a0 t1
>   	jr              ra
>   SYM_FUNC_END(kvm_save_lsx)
> -EXPORT_SYMBOL(kvm_save_lsx)
> +EXPORT_SYMBOL_FOR_KVM(kvm_save_lsx)
>   
>   SYM_FUNC_START(kvm_restore_lsx)
>   	lsx_restore_data a0 t1
> @@ -258,7 +268,7 @@ SYM_FUNC_START(kvm_restore_lsx)
>   	fpu_restore_csr  a0 t1 t2
>   	jr               ra
>   SYM_FUNC_END(kvm_restore_lsx)
> -EXPORT_SYMBOL(kvm_restore_lsx)
> +EXPORT_SYMBOL_FOR_KVM(kvm_restore_lsx)
>   #endif
>   
>   #ifdef CONFIG_CPU_HAS_LASX
> @@ -268,7 +278,7 @@ SYM_FUNC_START(kvm_save_lasx)
>   	lasx_save_data  a0 t1
>   	jr              ra
>   SYM_FUNC_END(kvm_save_lasx)
> -EXPORT_SYMBOL(kvm_save_lasx)
> +EXPORT_SYMBOL_FOR_KVM(kvm_save_lasx)
>   
>   SYM_FUNC_START(kvm_restore_lasx)
>   	lasx_restore_data a0 t1
> @@ -276,7 +286,7 @@ SYM_FUNC_START(kvm_restore_lasx)
>   	fpu_restore_csr   a0 t1 t2
>   	jr                ra
>   SYM_FUNC_END(kvm_restore_lasx)
> -EXPORT_SYMBOL(kvm_restore_lasx)
> +EXPORT_SYMBOL_FOR_KVM(kvm_restore_lasx)
>   #endif
>   
>   #ifdef CONFIG_CPU_HAS_LBT
> 

Re: [PATCH V3 2/2] LoongArch: KVM: fix "unreliable stack" issue

[lixianglai]

Hi Bibo Mao:
>
>
> On 2025/12/27 上午9:27, Xianglai Li wrote:
>> Insert the appropriate UNWIND macro definition into the kvm_exc_entry in
>> the assembly function to guide the generation of correct ORC table 
>> entries,
>> thereby solving the timeout problem of loading the livepatch-sample 
>> module
>> on a physical machine running multiple vcpus virtual machines.
>>
>> While solving the above problems, we have gained an additional benefit,
>> that is, we can obtain more call stack information
>>
>> Stack information that can be obtained before the problem is fixed:
>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>> [<0>] kvm_exc_entry+0x100/0x1e0
>>
>> Stack information that can be obtained after the problem is fixed:
>> [<0>] kvm_vcpu_block+0x88/0x120 [kvm]
>> [<0>] kvm_vcpu_halt+0x68/0x580 [kvm]
>> [<0>] kvm_emu_idle+0xd4/0xf0 [kvm]
>> [<0>] kvm_handle_gspr+0x7c/0x700 [kvm]
>> [<0>] kvm_handle_exit+0x160/0x270 [kvm]
>> [<0>] kvm_exc_entry+0x104/0x1e4
>> [<0>] kvm_enter_guest+0x38/0x11c
>> [<0>] kvm_arch_vcpu_ioctl_run+0x26c/0x498 [kvm]
>> [<0>] kvm_vcpu_ioctl+0x200/0xcf8 [kvm]
>> [<0>] sys_ioctl+0x498/0xf00
>> [<0>] do_syscall+0x98/0x1d0
>> [<0>] handle_syscall+0xb8/0x158
>>
>> Cc: stable@vger.kernel.org
>> Signed-off-by: Xianglai Li <lixianglai@loongson.cn>
>> ---
>> Cc: Huacai Chen <chenhuacai@kernel.org>
>> Cc: WANG Xuerui <kernel@xen0n.name>
>> Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
>> Cc: Bibo Mao <maobibo@loongson.cn>
>> Cc: Charlie Jenkins <charlie@rivosinc.com>
>> Cc: Xianglai Li <lixianglai@loongson.cn>
>> Cc: Thomas Gleixner <tglx@linutronix.de>
>> Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
>>
>>   arch/loongarch/kvm/switch.S | 28 +++++++++++++++++++---------
>>   1 file changed, 19 insertions(+), 9 deletions(-)
>>
>> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
>> index 93845ce53651..a3ea9567dbe5 100644
>> --- a/arch/loongarch/kvm/switch.S
>> +++ b/arch/loongarch/kvm/switch.S
>> @@ -10,6 +10,7 @@
>>   #include <asm/loongarch.h>
>>   #include <asm/regdef.h>
>>   #include <asm/unwind_hints.h>
>> +#include <linux/kvm_types.h>
>>     #define HGPR_OFFSET(x)        (PT_R0 + 8*x)
>>   #define GGPR_OFFSET(x)        (KVM_ARCH_GGPR + 8*x)
>> @@ -110,9 +111,9 @@
>>        * need to copy world switch code to DMW area.
>>        */
>>       .text
>> +    .p2align PAGE_SHIFT
>>       .cfi_sections    .debug_frame
>>   SYM_CODE_START(kvm_exc_entry)
>> -    .p2align PAGE_SHIFT
>>       UNWIND_HINT_UNDEFINED
>>       csrwr    a2,   KVM_TEMP_KS
>>       csrrd    a2,   KVM_VCPU_KS
>> @@ -170,6 +171,7 @@ SYM_CODE_START(kvm_exc_entry)
>>       /* restore per cpu register */
>>       ld.d    u0, a2, KVM_ARCH_HPERCPU
>>       addi.d    sp, sp, -PT_SIZE
>> +    UNWIND_HINT_REGS
>>         /* Prepare handle exception */
>>       or    a0, s0, zero
>> @@ -200,7 +202,7 @@ ret_to_host:
>>       jr      ra
>>     SYM_CODE_END(kvm_exc_entry)
>> -EXPORT_SYMBOL(kvm_exc_entry)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
>>     /*
>>    * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu)
>> @@ -215,6 +217,14 @@ SYM_FUNC_START(kvm_enter_guest)
>>       /* Save host GPRs */
>>       kvm_save_host_gpr a2
>>   +    /*
>> +     * The csr_era member variable of the pt_regs structure is required
>> +     * for unwinding orc to perform stack traceback, so we need to put
>> +     * pc into csr_era member variable here.
>> +     */
>> +    pcaddi    t0, 0
>> +    st.d    t0, a2, PT_ERA
> maybe PRMD need be set with fake pt_regs also, something like this:
>         ori     t0, zero, CSR_PRMD_PIE
>         st.d    t0, a2, PT_PRMD
>
Ok, I will fix it in the next version

Thanks,
Xianglai.
> Regards
> Bibo Mao
>> +
>>       addi.d    a2, a1, KVM_VCPU_ARCH
>>       st.d    sp, a2, KVM_ARCH_HSP
>>       st.d    tp, a2, KVM_ARCH_HTP
>> @@ -225,7 +235,7 @@ SYM_FUNC_START(kvm_enter_guest)
>>       csrwr    a1, KVM_VCPU_KS
>>       kvm_switch_to_guest
>>   SYM_FUNC_END(kvm_enter_guest)
>> -EXPORT_SYMBOL(kvm_enter_guest)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_enter_guest)
>>     SYM_FUNC_START(kvm_save_fpu)
>>       fpu_save_csr    a0 t1
>> @@ -233,7 +243,7 @@ SYM_FUNC_START(kvm_save_fpu)
>>       fpu_save_cc    a0 t1 t2
>>       jr              ra
>>   SYM_FUNC_END(kvm_save_fpu)
>> -EXPORT_SYMBOL(kvm_save_fpu)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_save_fpu)
>>     SYM_FUNC_START(kvm_restore_fpu)
>>       fpu_restore_double a0 t1
>> @@ -241,7 +251,7 @@ SYM_FUNC_START(kvm_restore_fpu)
>>       fpu_restore_cc       a0 t1 t2
>>       jr                 ra
>>   SYM_FUNC_END(kvm_restore_fpu)
>> -EXPORT_SYMBOL(kvm_restore_fpu)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_restore_fpu)
>>     #ifdef CONFIG_CPU_HAS_LSX
>>   SYM_FUNC_START(kvm_save_lsx)
>> @@ -250,7 +260,7 @@ SYM_FUNC_START(kvm_save_lsx)
>>       lsx_save_data   a0 t1
>>       jr              ra
>>   SYM_FUNC_END(kvm_save_lsx)
>> -EXPORT_SYMBOL(kvm_save_lsx)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_save_lsx)
>>     SYM_FUNC_START(kvm_restore_lsx)
>>       lsx_restore_data a0 t1
>> @@ -258,7 +268,7 @@ SYM_FUNC_START(kvm_restore_lsx)
>>       fpu_restore_csr  a0 t1 t2
>>       jr               ra
>>   SYM_FUNC_END(kvm_restore_lsx)
>> -EXPORT_SYMBOL(kvm_restore_lsx)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_restore_lsx)
>>   #endif
>>     #ifdef CONFIG_CPU_HAS_LASX
>> @@ -268,7 +278,7 @@ SYM_FUNC_START(kvm_save_lasx)
>>       lasx_save_data  a0 t1
>>       jr              ra
>>   SYM_FUNC_END(kvm_save_lasx)
>> -EXPORT_SYMBOL(kvm_save_lasx)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_save_lasx)
>>     SYM_FUNC_START(kvm_restore_lasx)
>>       lasx_restore_data a0 t1
>> @@ -276,7 +286,7 @@ SYM_FUNC_START(kvm_restore_lasx)
>>       fpu_restore_csr   a0 t1 t2
>>       jr                ra
>>   SYM_FUNC_END(kvm_restore_lasx)
>> -EXPORT_SYMBOL(kvm_restore_lasx)
>> +EXPORT_SYMBOL_FOR_KVM(kvm_restore_lasx)
>>   #endif
>>     #ifdef CONFIG_CPU_HAS_LBT
>>