From: Mark Rutland <mark.rutland@arm.com>
To: linux-kernel@vger.kernel.org
Cc: aou@eecs.berkeley.edu, catalin.marinas@arm.com,
deanbo422@gmail.com, green.hu@gmail.com, guoren@kernel.org,
jonas@southpole.se, kernelfans@gmail.com,
linux-arm-kernel@lists.infradead.org, linux@armlinux.org.uk,
mark.rutland@arm.com, maz@kernel.org, nickhu@andestech.com,
palmer@dabbelt.com, paul.walmsley@sifive.com, shorne@gmail.com,
stefan.kristiansson@saunalahti.fi, tglx@linutronix.de,
tsbogend@alpha.franken.de, vgupta@kernel.org,
vladimir.murzin@arm.com, will@kernel.org
Subject: [PATCH v2 12/17] irq: arm64: perform irqentry in entry code
Date: Tue, 26 Oct 2021 10:24:59 +0100 [thread overview]
Message-ID: <20211026092504.27071-13-mark.rutland@arm.com> (raw)
In-Reply-To: <20211026092504.27071-1-mark.rutland@arm.com>
In preparation for removing HANDLE_DOMAIN_IRQ_IRQENTRY, have arch/arm64
perform all the irqentry accounting in its entry code.
As arch/arm64 already performs portions of the irqentry logic in
enter_from_kernel_mode() and exit_to_kernel_mode(), including
rcu_irq_{enter,exit}(), the only additional calls that need to be made
are to irq_{enter,exit}_rcu(). Removing the calls to
rcu_irq_{enter,exit}() from handle_domain_irq() ensures that we inform
RCU once per IRQ entry and will correctly identify quiescent periods.
Since we should not call irq_{enter,exit}_rcu() when entering a
pseudo-NMI, el1_interrupt() is reworked to have separate __el1_irq() and
__el1_pnmi() paths for regular IRQ and psuedo-NMI entry, with
irq_{enter,exit}_irq() only called for the former.
In preparation for removing HANDLE_DOMAIN_IRQ, the irq regs are managed
in do_interrupt_handler() for both regular IRQ and pseudo-NMI. This is
currently redundant, but not harmful.
For clarity the preemption logic is moved into __el1_irq(). We should
never preempt within a pseudo-NMI, and arm64_enter_nmi() already
enforces this by incrementing the preempt_count, but it's clearer if we
never invoke the preemption logic when entering a pseudo-NMI.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Pingfan Liu <kernelfans@gmail.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Deacon <will@kernel.org>
---
arch/arm64/Kconfig | 1 -
arch/arm64/kernel/entry-common.c | 52 ++++++++++++++++++++++++----------------
2 files changed, 31 insertions(+), 22 deletions(-)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 553239a5a5f7..5c7ae4c3954b 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -134,7 +134,6 @@ config ARM64
select GENERIC_GETTIMEOFDAY
select GENERIC_VDSO_TIME_NS
select HANDLE_DOMAIN_IRQ
- select HANDLE_DOMAIN_IRQ_IRQENTRY
select HARDIRQS_SW_RESEND
select HAVE_MOVE_PMD
select HAVE_MOVE_PUD
diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index 32f9796c4ffe..f7408edf8571 100644
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -17,6 +17,7 @@
#include <asm/daifflags.h>
#include <asm/esr.h>
#include <asm/exception.h>
+#include <asm/irq_regs.h>
#include <asm/kprobes.h>
#include <asm/mmu.h>
#include <asm/processor.h>
@@ -219,22 +220,6 @@ static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)
lockdep_hardirqs_on(CALLER_ADDR0);
}
-static void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs)
-{
- if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))
- arm64_enter_nmi(regs);
- else
- enter_from_kernel_mode(regs);
-}
-
-static void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs)
-{
- if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))
- arm64_exit_nmi(regs);
- else
- exit_to_kernel_mode(regs);
-}
-
static void __sched arm64_preempt_schedule_irq(void)
{
lockdep_assert_irqs_disabled();
@@ -263,10 +248,14 @@ static void __sched arm64_preempt_schedule_irq(void)
static void do_interrupt_handler(struct pt_regs *regs,
void (*handler)(struct pt_regs *))
{
+ struct pt_regs *old_regs = set_irq_regs(regs);
+
if (on_thread_stack())
call_on_irq_stack(regs, handler);
else
handler(regs);
+
+ set_irq_regs(old_regs);
}
extern void (*handle_arch_irq)(struct pt_regs *);
@@ -432,13 +421,22 @@ asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
}
}
-static void noinstr el1_interrupt(struct pt_regs *regs,
- void (*handler)(struct pt_regs *))
+static __always_inline void __el1_pnmi(struct pt_regs *regs,
+ void (*handler)(struct pt_regs *))
{
- write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
+ arm64_enter_nmi(regs);
+ do_interrupt_handler(regs, handler);
+ arm64_exit_nmi(regs);
+}
+
+static __always_inline void __el1_irq(struct pt_regs *regs,
+ void (*handler)(struct pt_regs *))
+{
+ enter_from_kernel_mode(regs);
- enter_el1_irq_or_nmi(regs);
+ irq_enter_rcu();
do_interrupt_handler(regs, handler);
+ irq_exit_rcu();
/*
* Note: thread_info::preempt_count includes both thread_info::count
@@ -449,7 +447,17 @@ static void noinstr el1_interrupt(struct pt_regs *regs,
READ_ONCE(current_thread_info()->preempt_count) == 0)
arm64_preempt_schedule_irq();
- exit_el1_irq_or_nmi(regs);
+ exit_to_kernel_mode(regs);
+}
+static void noinstr el1_interrupt(struct pt_regs *regs,
+ void (*handler)(struct pt_regs *))
+{
+ write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
+
+ if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))
+ __el1_pnmi(regs, handler);
+ else
+ __el1_irq(regs, handler);
}
asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)
@@ -667,7 +675,9 @@ static void noinstr el0_interrupt(struct pt_regs *regs,
if (regs->pc & BIT(55))
arm64_apply_bp_hardening();
+ irq_enter_rcu();
do_interrupt_handler(regs, handler);
+ irq_exit_rcu();
exit_to_user_mode(regs);
}
--
2.11.0
next prev parent reply other threads:[~2021-10-26 9:26 UTC|newest]
Thread overview: 32+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-10-26 9:24 [PATCH v2 00/17] irq: remove handle_domain_{irq,nmi}() Mark Rutland
2021-10-26 9:24 ` [PATCH v2 01/17] irq: mips: avoid nested irq_enter() Mark Rutland
2021-10-26 9:24 ` [PATCH v2 02/17] irq: mips: simplify bcm6345_l1_irq_handle() Mark Rutland
2021-10-26 9:24 ` [PATCH v2 03/17] irq: mips: stop (ab)using handle_domain_irq() Mark Rutland
2021-10-26 9:24 ` [PATCH v2 04/17] irq: mips: simplify do_domain_IRQ() Mark Rutland
2021-10-26 9:24 ` [PATCH v2 05/17] irq: simplify handle_domain_{irq,nmi}() Mark Rutland
2021-10-26 9:24 ` [PATCH v2 06/17] irq: unexport handle_irq_desc() Mark Rutland
2021-10-26 9:24 ` [PATCH v2 07/17] irq: add generic_handle_arch_irq() Mark Rutland
2021-10-26 9:24 ` [PATCH v2 08/17] irq: arc: avoid CONFIG_HANDLE_DOMAIN_IRQ Mark Rutland
2021-10-26 9:24 ` [PATCH v2 09/17] irq: nds32: " Mark Rutland
2021-10-26 9:24 ` [PATCH v2 10/17] irq: add a (temporary) CONFIG_HANDLE_DOMAIN_IRQ_IRQENTRY Mark Rutland
2021-10-26 9:24 ` [PATCH v2 11/17] irq: arm: perform irqentry in entry code Mark Rutland
2021-10-26 9:24 ` Mark Rutland [this message]
2021-10-26 9:25 ` [PATCH v2 13/17] irq: csky: " Mark Rutland
2021-10-26 9:25 ` [PATCH v2 14/17] irq: openrisc: " Mark Rutland
2021-10-26 9:25 ` [PATCH v2 15/17] irq: riscv: " Mark Rutland
2021-10-26 9:25 ` [PATCH v2 16/17] irq: remove CONFIG_HANDLE_DOMAIN_IRQ_IRQENTRY Mark Rutland
2021-10-26 9:25 ` [PATCH v2 17/17] irq: remove handle_domain_{irq,nmi}() Mark Rutland
2022-05-06 20:32 ` Lukas Wunner
2022-05-09 8:54 ` Mark Rutland
2022-05-09 9:09 ` Marc Zyngier
2022-05-09 13:12 ` Thomas Gleixner
2022-05-10 23:36 ` Thomas Gleixner
2022-05-10 12:13 ` Lukas Wunner
2022-05-10 14:15 ` Mark Rutland
2022-05-10 22:52 ` Thomas Gleixner
2022-05-11 8:23 ` Mark Rutland
2022-05-11 8:57 ` Lukas Wunner
2022-05-11 9:27 ` Mark Rutland
2022-05-11 0:11 ` Thomas Gleixner
2022-05-11 8:11 ` Mark Rutland
2021-10-26 10:12 ` [PATCH v2 00/17] " Marc Zyngier
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211026092504.27071-13-mark.rutland@arm.com \
--to=mark.rutland@arm.com \
--cc=aou@eecs.berkeley.edu \
--cc=catalin.marinas@arm.com \
--cc=deanbo422@gmail.com \
--cc=green.hu@gmail.com \
--cc=guoren@kernel.org \
--cc=jonas@southpole.se \
--cc=kernelfans@gmail.com \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux@armlinux.org.uk \
--cc=maz@kernel.org \
--cc=nickhu@andestech.com \
--cc=palmer@dabbelt.com \
--cc=paul.walmsley@sifive.com \
--cc=shorne@gmail.com \
--cc=stefan.kristiansson@saunalahti.fi \
--cc=tglx@linutronix.de \
--cc=tsbogend@alpha.franken.de \
--cc=vgupta@kernel.org \
--cc=vladimir.murzin@arm.com \
--cc=will@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®