[PATCH 1/2] x86/entry/64: Rearrange SYSCALL entry points for Xen's benefit

[Andy Lutomirski <luto@kernel.org>]

Xen currently fudges RSP on SYSCALL to be compatible with the native
entries.  This has the unfortunate side effect that there are extra
poorly-controlled places with user RSP.  Add better entry points for
Xen to use instead.

This will add a couple of cycles of IRQ latency, but it avoids an
annoying corner case in which an IRQ can be delivered with a
hardware frame that overlaps current_pt_regs.

Signed-off-by: Andy Lutomirski <luto@kernel.org>
---
 arch/x86/entry/entry_64.S        | 20 ++++++++++++++------
 arch/x86/entry/entry_64_compat.S | 10 ++++++----
 2 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 041a37a643e1..f47996910331 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -148,17 +148,19 @@ GLOBAL(entry_SYSCALL_64_after_swapgs)
 	/* Construct struct pt_regs on stack */
 	pushq	$__USER_DS			/* pt_regs->ss */
 	pushq	PER_CPU_VAR(rsp_scratch)	/* pt_regs->sp */
+
 	/*
-	 * Re-enable interrupts.
-	 * We use 'rsp_scratch' as a scratch space, hence irq-off block above
-	 * must execute atomically in the face of possible interrupt-driven
-	 * task preemption. We must enable interrupts only after we're done
-	 * with using rsp_scratch:
+	 * Before this point, interrupts MUST be off to prevent rsp_scratch
+	 * from getting corrupted due to preemption.  Nonetheless, we keep
+	 * interrupts off a little longer.  We eventually want to make it
+	 * all the way to C code with interrupts off, which will considerably
+	 * simplify context tracking.
 	 */
-	ENABLE_INTERRUPTS(CLBR_NONE)
+
 	pushq	%r11				/* pt_regs->flags */
 	pushq	$__USER_CS			/* pt_regs->cs */
 	pushq	%rcx				/* pt_regs->ip */
+GLOBAL(entry_SYSCALL_64_after_hwframe)
 	pushq	%rax				/* pt_regs->orig_ax */
 	pushq	%rdi				/* pt_regs->di */
 	pushq	%rsi				/* pt_regs->si */
@@ -171,6 +173,12 @@ GLOBAL(entry_SYSCALL_64_after_swapgs)
 	pushq	%r11				/* pt_regs->r11 */
 	sub	$(6*8), %rsp			/* pt_regs->bp, bx, r12-15 not saved */
 
+	/*
+	 * Re-enable interrupts.  IRQ tracing already thinks they're
+	 * on because user mode is traced as IRQs-on.
+	 */
+	ENABLE_INTERRUPTS(CLBR_NONE)
+
 	testl	$_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz	tracesys
 entry_SYSCALL_64_fastpath:
diff --git a/arch/x86/entry/entry_64_compat.S b/arch/x86/entry/entry_64_compat.S
index d7571532e7ce..d39495d6446e 100644
--- a/arch/x86/entry/entry_64_compat.S
+++ b/arch/x86/entry/entry_64_compat.S
@@ -312,10 +312,6 @@ ENTRY(entry_SYSCALL_compat)
 	SWAPGS_UNSAFE_STACK
 	movl	%esp, %r8d
 	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
-	ENABLE_INTERRUPTS(CLBR_NONE)
-
-	/* Zero-extending 32-bit regs, do not remove */
-	movl	%eax, %eax
 
 	/* Construct struct pt_regs on stack */
 	pushq	$__USER32_DS		/* pt_regs->ss */
@@ -323,6 +319,7 @@ ENTRY(entry_SYSCALL_compat)
 	pushq	%r11			/* pt_regs->flags */
 	pushq	$__USER32_CS		/* pt_regs->cs */
 	pushq	%rcx			/* pt_regs->ip */
+GLOBAL(entry_SYSCALL_compat_after_hwframe_esp_in_r8d)
 	pushq	%rax			/* pt_regs->orig_ax */
 	pushq	%rdi			/* pt_regs->di */
 	pushq	%rsi			/* pt_regs->si */
@@ -332,6 +329,11 @@ ENTRY(entry_SYSCALL_compat)
 	pushq	$-ENOSYS		/* pt_regs->ax */
 	sub	$(10*8), %rsp		/* pt_regs->r8-11, bp, bx, r12-15 not saved */
 
+	ENABLE_INTERRUPTS(CLBR_NONE)
+
+	/* Zero-extending 32-bit regs, do not remove */
+	movl	%eax, %eax
+
 	/*
 	 * No need to do an access_ok check here because r8 has been
 	 * 32-bit zero extended:
-- 
2.4.3