[PATCH 0/2] x86/xen: don't return from xen_pv_play_dead()

[PATCH 0/2] x86/xen: don't return from xen_pv_play_dead()

[Juergen Gross]

All play_dead() functions but xen_pv_play_dead() don't return to the
caller.

Adapt xen_pv_play_dead() to behave like the other play_dead() variants.

Juergen Gross (2):
  x86/xen: don't let xen_pv_play_dead() return
  x86/xen: mark xen_pv_play_dead() as __noreturn

 arch/x86/xen/smp.h      |  2 ++
 arch/x86/xen/smp_pv.c   | 17 ++++-------------
 arch/x86/xen/xen-head.S |  7 +++++++
 tools/objtool/check.c   |  1 +
 4 files changed, 14 insertions(+), 13 deletions(-)

-- 
2.35.3

[PATCH 1/2] x86/xen: don't let xen_pv_play_dead() return

[Juergen Gross]

A function called via the paravirt play_dead() hook should not return
to the caller.

xen_pv_play_dead() has a problem in this regard, as it currently will
return in case an offlined cpu is brought to life again. This can be
changed only by doing basically a longjmp() to cpu_bringup_and_idle(),
as the hypercall for bringing down the cpu will just return when the
cpu is coming up again. Just re-initializing the cpu isn't possible,
as the Xen hypervisor will deny that operation.

So introduce xen_cpu_bringup_again() resetting the stack and calling
cpu_bringup_and_idle(), which can be called after HYPERVISOR_vcpu_op()
in xen_pv_play_dead().

Signed-off-by: Juergen Gross <jgross@suse.com>
---
 arch/x86/xen/smp.h      |  2 ++
 arch/x86/xen/smp_pv.c   | 13 ++-----------
 arch/x86/xen/xen-head.S |  7 +++++++
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/arch/x86/xen/smp.h b/arch/x86/xen/smp.h
index bd02f9d50107..6e90a312067b 100644
--- a/arch/x86/xen/smp.h
+++ b/arch/x86/xen/smp.h
@@ -21,6 +21,8 @@ void xen_smp_send_reschedule(int cpu);
 void xen_smp_send_call_function_ipi(const struct cpumask *mask);
 void xen_smp_send_call_function_single_ipi(int cpu);
 
+void xen_cpu_bringup_again(unsigned long stack);
+
 struct xen_common_irq {
 	int irq;
 	char *name;
diff --git a/arch/x86/xen/smp_pv.c b/arch/x86/xen/smp_pv.c
index 480be82e9b7b..b40b24382fe3 100644
--- a/arch/x86/xen/smp_pv.c
+++ b/arch/x86/xen/smp_pv.c
@@ -385,17 +385,8 @@ static void xen_pv_play_dead(void) /* used only with HOTPLUG_CPU */
 {
 	play_dead_common();
 	HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(smp_processor_id()), NULL);
-	cpu_bringup();
-	/*
-	 * commit 4b0c0f294 (tick: Cleanup NOHZ per cpu data on cpu down)
-	 * clears certain data that the cpu_idle loop (which called us
-	 * and that we return from) expects. The only way to get that
-	 * data back is to call:
-	 */
-	tick_nohz_idle_enter();
-	tick_nohz_idle_stop_tick_protected();
-
-	cpuhp_online_idle(CPUHP_AP_ONLINE_IDLE);
+	xen_cpu_bringup_again((unsigned long)task_pt_regs(current));
+	BUG();
 }
 
 #else /* !CONFIG_HOTPLUG_CPU */
diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S
index ffaa62167f6e..e36ea4268bd2 100644
--- a/arch/x86/xen/xen-head.S
+++ b/arch/x86/xen/xen-head.S
@@ -76,6 +76,13 @@ SYM_CODE_START(asm_cpu_bringup_and_idle)
 
 	call cpu_bringup_and_idle
 SYM_CODE_END(asm_cpu_bringup_and_idle)
+
+SYM_CODE_START(xen_cpu_bringup_again)
+	UNWIND_HINT_FUNC
+	mov	%rdi, %rsp
+	UNWIND_HINT_REGS
+	call	cpu_bringup_and_idle
+SYM_CODE_END(xen_cpu_bringup_again)
 .popsection
 #endif
 #endif
-- 
2.35.3

[PATCH 2/2] x86/xen: mark xen_pv_play_dead() as __noreturn

[Juergen Gross]

Mark xen_pv_play_dead() and related to that xen_cpu_bringup_again()
as "__noreturn".

Signed-off-by: Juergen Gross <jgross@suse.com>
---
 arch/x86/xen/smp.h    | 2 +-
 arch/x86/xen/smp_pv.c | 4 ++--
 tools/objtool/check.c | 1 +
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/arch/x86/xen/smp.h b/arch/x86/xen/smp.h
index 6e90a312067b..22fb982ff971 100644
--- a/arch/x86/xen/smp.h
+++ b/arch/x86/xen/smp.h
@@ -21,7 +21,7 @@ void xen_smp_send_reschedule(int cpu);
 void xen_smp_send_call_function_ipi(const struct cpumask *mask);
 void xen_smp_send_call_function_single_ipi(int cpu);
 
-void xen_cpu_bringup_again(unsigned long stack);
+void __noreturn xen_cpu_bringup_again(unsigned long stack);
 
 struct xen_common_irq {
 	int irq;
diff --git a/arch/x86/xen/smp_pv.c b/arch/x86/xen/smp_pv.c
index b40b24382fe3..5801f93d567c 100644
--- a/arch/x86/xen/smp_pv.c
+++ b/arch/x86/xen/smp_pv.c
@@ -381,7 +381,7 @@ static void xen_pv_cpu_die(unsigned int cpu)
 	}
 }
 
-static void xen_pv_play_dead(void) /* used only with HOTPLUG_CPU */
+static void __noreturn xen_pv_play_dead(void) /* used only with HOTPLUG_CPU */
 {
 	play_dead_common();
 	HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(smp_processor_id()), NULL);
@@ -400,7 +400,7 @@ static void xen_pv_cpu_die(unsigned int cpu)
 	BUG();
 }
 
-static void xen_pv_play_dead(void)
+static void __noreturn xen_pv_play_dead(void)
 {
 	BUG();
 }
diff --git a/tools/objtool/check.c b/tools/objtool/check.c
index 43ec14c29a60..becdedc2de99 100644
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -189,6 +189,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
 		"snp_abort",
 		"stop_this_cpu",
 		"usercopy_abort",
+		"xen_cpu_bringup_again",
 		"xen_start_kernel",
 	};
 
-- 
2.35.3

Re: [PATCH 0/2] x86/xen: don't return from xen_pv_play_dead()

[Juergen Gross]

[-- Attachment #1.1.1: Type: text/plain, Size: 586 bytes --]

On 25.11.22 07:32, Juergen Gross wrote:
> All play_dead() functions but xen_pv_play_dead() don't return to the
> caller.
> 
> Adapt xen_pv_play_dead() to behave like the other play_dead() variants.
> 
> Juergen Gross (2):
>    x86/xen: don't let xen_pv_play_dead() return
>    x86/xen: mark xen_pv_play_dead() as __noreturn
> 
>   arch/x86/xen/smp.h      |  2 ++
>   arch/x86/xen/smp_pv.c   | 17 ++++-------------
>   arch/x86/xen/xen-head.S |  7 +++++++
>   tools/objtool/check.c   |  1 +
>   4 files changed, 14 insertions(+), 13 deletions(-)
> 

Ping?


Juergen

[-- Attachment #1.1.2: OpenPGP public key --]
[-- Type: application/pgp-keys, Size: 3149 bytes --]

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 495 bytes --]

Re: [PATCH 0/2] x86/xen: don't return from xen_pv_play_dead()

[Peter Zijlstra]

On Fri, Nov 25, 2022 at 07:32:46AM +0100, Juergen Gross wrote:
> All play_dead() functions but xen_pv_play_dead() don't return to the
> caller.
> 
> Adapt xen_pv_play_dead() to behave like the other play_dead() variants.
> 
> Juergen Gross (2):
>   x86/xen: don't let xen_pv_play_dead() return
>   x86/xen: mark xen_pv_play_dead() as __noreturn
> 
>  arch/x86/xen/smp.h      |  2 ++
>  arch/x86/xen/smp_pv.c   | 17 ++++-------------
>  arch/x86/xen/xen-head.S |  7 +++++++
>  tools/objtool/check.c   |  1 +
>  4 files changed, 14 insertions(+), 13 deletions(-)

Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>

Re: [PATCH 0/2] x86/xen: don't return from xen_pv_play_dead()

[Boris Ostrovsky]

On 1/16/23 2:49 AM, Juergen Gross wrote:
> On 25.11.22 07:32, Juergen Gross wrote:
>> All play_dead() functions but xen_pv_play_dead() don't return to the
>> caller.
>>
>> Adapt xen_pv_play_dead() to behave like the other play_dead() variants.
>>
>> Juergen Gross (2):
>>    x86/xen: don't let xen_pv_play_dead() return
>>    x86/xen: mark xen_pv_play_dead() as __noreturn
>>
>>   arch/x86/xen/smp.h      |  2 ++
>>   arch/x86/xen/smp_pv.c   | 17 ++++-------------
>>   arch/x86/xen/xen-head.S |  7 +++++++
>>   tools/objtool/check.c   |  1 +
>>   4 files changed, 14 insertions(+), 13 deletions(-)
>>
>
> Ping?
>
>

Reviewed-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>

[tip: objtool/core] x86/xen: mark xen_pv_play_dead() as __noreturn

[tip-bot2 for Juergen Gross]

The following commit has been merged into the objtool/core branch of tip:

Commit-ID:     1aff0d2658e5a5217d3168e06a90fbb99fa63e80
Gitweb:        https://git.kernel.org/tip/1aff0d2658e5a5217d3168e06a90fbb99fa63e80
Author:        Juergen Gross <jgross@suse.com>
AuthorDate:    Fri, 25 Nov 2022 07:32:48 +01:00
Committer:     Juergen Gross <jgross@suse.com>
CommitterDate: Tue, 17 Jan 2023 09:05:32 +01:00

x86/xen: mark xen_pv_play_dead() as __noreturn

Mark xen_pv_play_dead() and related to that xen_cpu_bringup_again()
as "__noreturn".

Signed-off-by: Juergen Gross <jgross@suse.com>
Reviewed-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20221125063248.30256-3-jgross@suse.com
Signed-off-by: Juergen Gross <jgross@suse.com>
---
 arch/x86/xen/smp.h    | 2 +-
 arch/x86/xen/smp_pv.c | 4 ++--
 tools/objtool/check.c | 1 +
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/arch/x86/xen/smp.h b/arch/x86/xen/smp.h
index 6e90a31..22fb982 100644
--- a/arch/x86/xen/smp.h
+++ b/arch/x86/xen/smp.h
@@ -21,7 +21,7 @@ void xen_smp_send_reschedule(int cpu);
 void xen_smp_send_call_function_ipi(const struct cpumask *mask);
 void xen_smp_send_call_function_single_ipi(int cpu);
 
-void xen_cpu_bringup_again(unsigned long stack);
+void __noreturn xen_cpu_bringup_again(unsigned long stack);
 
 struct xen_common_irq {
 	int irq;
diff --git a/arch/x86/xen/smp_pv.c b/arch/x86/xen/smp_pv.c
index a97b050..a9cf8c8 100644
--- a/arch/x86/xen/smp_pv.c
+++ b/arch/x86/xen/smp_pv.c
@@ -381,7 +381,7 @@ static void xen_pv_cpu_die(unsigned int cpu)
 	}
 }
 
-static void xen_pv_play_dead(void) /* used only with HOTPLUG_CPU */
+static void __noreturn xen_pv_play_dead(void) /* used only with HOTPLUG_CPU */
 {
 	play_dead_common();
 	HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(smp_processor_id()), NULL);
@@ -400,7 +400,7 @@ static void xen_pv_cpu_die(unsigned int cpu)
 	BUG();
 }
 
-static void xen_pv_play_dead(void)
+static void __noreturn xen_pv_play_dead(void)
 {
 	BUG();
 }
diff --git a/tools/objtool/check.c b/tools/objtool/check.c
index 4b7c8b3..68e87b4 100644
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -186,6 +186,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
 		"snp_abort",
 		"stop_this_cpu",
 		"usercopy_abort",
+		"xen_cpu_bringup_again",
 		"xen_start_kernel",
 	};
 

[tip: objtool/core] x86/xen: don't let xen_pv_play_dead() return

[tip-bot2 for Juergen Gross]

The following commit has been merged into the objtool/core branch of tip:

Commit-ID:     076cbf5d216377087e135a0ae81844f7835cdc15
Gitweb:        https://git.kernel.org/tip/076cbf5d216377087e135a0ae81844f7835cdc15
Author:        Juergen Gross <jgross@suse.com>
AuthorDate:    Fri, 25 Nov 2022 07:32:47 +01:00
Committer:     Juergen Gross <jgross@suse.com>
CommitterDate: Tue, 17 Jan 2023 09:05:28 +01:00

x86/xen: don't let xen_pv_play_dead() return

A function called via the paravirt play_dead() hook should not return
to the caller.

xen_pv_play_dead() has a problem in this regard, as it currently will
return in case an offlined cpu is brought to life again. This can be
changed only by doing basically a longjmp() to cpu_bringup_and_idle(),
as the hypercall for bringing down the cpu will just return when the
cpu is coming up again. Just re-initializing the cpu isn't possible,
as the Xen hypervisor will deny that operation.

So introduce xen_cpu_bringup_again() resetting the stack and calling
cpu_bringup_and_idle(), which can be called after HYPERVISOR_vcpu_op()
in xen_pv_play_dead().

Signed-off-by: Juergen Gross <jgross@suse.com>
Reviewed-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20221125063248.30256-2-jgross@suse.com
Signed-off-by: Juergen Gross <jgross@suse.com>
---
 arch/x86/xen/smp.h      |  2 ++
 arch/x86/xen/smp_pv.c   | 13 ++-----------
 arch/x86/xen/xen-head.S |  7 +++++++
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/arch/x86/xen/smp.h b/arch/x86/xen/smp.h
index bd02f9d..6e90a31 100644
--- a/arch/x86/xen/smp.h
+++ b/arch/x86/xen/smp.h
@@ -21,6 +21,8 @@ void xen_smp_send_reschedule(int cpu);
 void xen_smp_send_call_function_ipi(const struct cpumask *mask);
 void xen_smp_send_call_function_single_ipi(int cpu);
 
+void xen_cpu_bringup_again(unsigned long stack);
+
 struct xen_common_irq {
 	int irq;
 	char *name;
diff --git a/arch/x86/xen/smp_pv.c b/arch/x86/xen/smp_pv.c
index 6175f2c..a97b050 100644
--- a/arch/x86/xen/smp_pv.c
+++ b/arch/x86/xen/smp_pv.c
@@ -385,17 +385,8 @@ static void xen_pv_play_dead(void) /* used only with HOTPLUG_CPU */
 {
 	play_dead_common();
 	HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(smp_processor_id()), NULL);
-	cpu_bringup();
-	/*
-	 * commit 4b0c0f294 (tick: Cleanup NOHZ per cpu data on cpu down)
-	 * clears certain data that the cpu_idle loop (which called us
-	 * and that we return from) expects. The only way to get that
-	 * data back is to call:
-	 */
-	tick_nohz_idle_enter();
-	tick_nohz_idle_stop_tick_protected();
-
-	cpuhp_online_idle(CPUHP_AP_ONLINE_IDLE);
+	xen_cpu_bringup_again((unsigned long)task_pt_regs(current));
+	BUG();
 }
 
 #else /* !CONFIG_HOTPLUG_CPU */
diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S
index ffaa621..e36ea42 100644
--- a/arch/x86/xen/xen-head.S
+++ b/arch/x86/xen/xen-head.S
@@ -76,6 +76,13 @@ SYM_CODE_START(asm_cpu_bringup_and_idle)
 
 	call cpu_bringup_and_idle
 SYM_CODE_END(asm_cpu_bringup_and_idle)
+
+SYM_CODE_START(xen_cpu_bringup_again)
+	UNWIND_HINT_FUNC
+	mov	%rdi, %rsp
+	UNWIND_HINT_REGS
+	call	cpu_bringup_and_idle
+SYM_CODE_END(xen_cpu_bringup_again)
 .popsection
 #endif
 #endif