[PATCH v2 0/2] riscv: Keep helpers used by noinstr code in noinstr text

[PATCH v2 0/2] riscv: Keep helpers used by noinstr code in noinstr text

[Jisheng Zhang]

A plain inline annotation does not guarantee inlining. In particular,
size optimization can cause helpers called from noinstr functions to
be emitted out of line in .text. Such copies do not inherit the
caller's instrumentation restrictions.
Use __always_inline for the IRQ, idle, stack-check and vector helpers
reachable from non-instrumentable entry paths, including their nested
helpers. Also force inline arch_exit_to_user_mode_prepare().
do_trap_misaligned() is called by some '__trap_section' functions such
as do_trap_load_misaligned() and do_trap_store_misaligned() which are
noinstr code. So we also need to mark do_trap_misaligned() with
__trap_section to keep it in non-instrumentable text as well.
Build-tested the affected entry, trap, idle, stacktrace and KVM objects
with GCC and CONFIG_CC_OPTIMIZE_FOR_SIZE=y. Inspected object symbols
and relocations.

Since v1:
  - add Fixes tag to patch1
  - add more missing __always_inline or noinstr to vector.h and
    kernel_mode_vector.c, these are missing in v1.

Jisheng Zhang (2):
  riscv/traps: Mark do_trap_misaligned() with __trap_section
  riscv: Keep helpers used by noinstr code in non-instrumentable text

 arch/riscv/include/asm/cpuidle.h       |  2 +-
 arch/riscv/include/asm/entry-common.h  |  4 ++--
 arch/riscv/include/asm/irqflags.h      | 14 +++++++-------
 arch/riscv/include/asm/processor.h     |  2 +-
 arch/riscv/include/asm/stacktrace.h    |  2 +-
 arch/riscv/include/asm/vector.h        | 22 +++++++++++-----------
 arch/riscv/kernel/kernel_mode_vector.c | 17 +++++++++--------
 arch/riscv/kernel/traps.c              |  3 ++-
 8 files changed, 34 insertions(+), 32 deletions(-)

-- 
2.55.0

[PATCH v2 1/2] riscv/traps: Mark do_trap_misaligned() with __trap_section

[Jisheng Zhang]

do_trap_misaligned() is called by some '__trap_section' functions such
as do_trap_load_misaligned() and do_trap_store_misaligned() which are
noinstr code. So we also need to mark do_trap_misaligned() with
__trap_section to keep it in non-instrumentable text as well.

Fixes: fd94de9f9e7a ("riscv: misaligned: factorize trap handling")
Signed-off-by: Jisheng Zhang <jszhang@kernel.org>
---
 arch/riscv/kernel/traps.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c
index f28e03f98af0..a170ed343e5a 100644
--- a/arch/riscv/kernel/traps.c
+++ b/arch/riscv/kernel/traps.c
@@ -213,7 +213,8 @@ static const struct {
 	},
 };
 
-static void do_trap_misaligned(struct pt_regs *regs, enum misaligned_access_type type)
+static __trap_section void do_trap_misaligned(struct pt_regs *regs,
+					      enum misaligned_access_type type)
 {
 	irqentry_state_t state;
 
-- 
2.55.0

[PATCH v2 2/2] riscv: Keep helpers used by noinstr code in non-instrumentable text

[Jisheng Zhang]

A plain inline annotation does not guarantee inlining. In particular,
size optimization can cause helpers called from noinstr functions to
be emitted out of line in .text. Such copies do not inherit the
caller's instrumentation restrictions.

Use __always_inline for the IRQ, idle, stack-check and vector helpers
reachable from non-instrumentable entry paths, including their nested
helpers. Also force inline arch_exit_to_user_mode_prepare().

Build-tested the affected entry, trap, idle, stacktrace and KVM objects
with GCC and CONFIG_CC_OPTIMIZE_FOR_SIZE=y. Inspected object symbols
and relocations.

Signed-off-by: Jisheng Zhang <jszhang@kernel.org>
Assisted-by: LLM
---
 arch/riscv/include/asm/cpuidle.h       |  2 +-
 arch/riscv/include/asm/entry-common.h  |  4 ++--
 arch/riscv/include/asm/irqflags.h      | 14 +++++++-------
 arch/riscv/include/asm/processor.h     |  2 +-
 arch/riscv/include/asm/stacktrace.h    |  2 +-
 arch/riscv/include/asm/vector.h        | 22 +++++++++++-----------
 arch/riscv/kernel/kernel_mode_vector.c | 17 +++++++++--------
 7 files changed, 32 insertions(+), 31 deletions(-)

diff --git a/arch/riscv/include/asm/cpuidle.h b/arch/riscv/include/asm/cpuidle.h
index 71fdc607d4bc..edb064a8d2e5 100644
--- a/arch/riscv/include/asm/cpuidle.h
+++ b/arch/riscv/include/asm/cpuidle.h
@@ -10,7 +10,7 @@
 #include <asm/barrier.h>
 #include <asm/processor.h>
 
-static inline void cpu_do_idle(void)
+static __always_inline void cpu_do_idle(void)
 {
 	/*
 	 * Add mb() here to ensure that all
diff --git a/arch/riscv/include/asm/entry-common.h b/arch/riscv/include/asm/entry-common.h
index 34ed149af5d1..90c6d61a2251 100644
--- a/arch/riscv/include/asm/entry-common.h
+++ b/arch/riscv/include/asm/entry-common.h
@@ -7,8 +7,8 @@
 #include <asm/thread_info.h>
 #include <asm/vector.h>
 
-static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
-						  unsigned long ti_work)
+static __always_inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
+							   unsigned long ti_work)
 {
 	if (ti_work & _TIF_RISCV_V_DEFER_RESTORE) {
 		clear_thread_flag(TIF_RISCV_V_DEFER_RESTORE);
diff --git a/arch/riscv/include/asm/irqflags.h b/arch/riscv/include/asm/irqflags.h
index 6fd8cbfcfcc7..0478ea545ceb 100644
--- a/arch/riscv/include/asm/irqflags.h
+++ b/arch/riscv/include/asm/irqflags.h
@@ -10,43 +10,43 @@
 #include <asm/csr.h>
 
 /* read interrupt enabled status */
-static inline unsigned long arch_local_save_flags(void)
+static __always_inline unsigned long arch_local_save_flags(void)
 {
 	return csr_read(CSR_STATUS);
 }
 
 /* unconditionally enable interrupts */
-static inline void arch_local_irq_enable(void)
+static __always_inline void arch_local_irq_enable(void)
 {
 	csr_set(CSR_STATUS, SR_IE);
 }
 
 /* unconditionally disable interrupts */
-static inline void arch_local_irq_disable(void)
+static __always_inline void arch_local_irq_disable(void)
 {
 	csr_clear(CSR_STATUS, SR_IE);
 }
 
 /* get status and disable interrupts */
-static inline unsigned long arch_local_irq_save(void)
+static __always_inline unsigned long arch_local_irq_save(void)
 {
 	return csr_read_clear(CSR_STATUS, SR_IE);
 }
 
 /* test flags */
-static inline int arch_irqs_disabled_flags(unsigned long flags)
+static __always_inline int arch_irqs_disabled_flags(unsigned long flags)
 {
 	return !(flags & SR_IE);
 }
 
 /* test hardware interrupt enable bit */
-static inline int arch_irqs_disabled(void)
+static __always_inline int arch_irqs_disabled(void)
 {
 	return arch_irqs_disabled_flags(arch_local_save_flags());
 }
 
 /* set interrupt enabled status */
-static inline void arch_local_irq_restore(unsigned long flags)
+static __always_inline void arch_local_irq_restore(unsigned long flags)
 {
 	csr_set(CSR_STATUS, flags & SR_IE);
 }
diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h
index e9d339bcb7e9..a921c6f2675a 100644
--- a/arch/riscv/include/asm/processor.h
+++ b/arch/riscv/include/asm/processor.h
@@ -186,7 +186,7 @@ extern void start_thread(struct pt_regs *regs,
 extern unsigned long __get_wchan(struct task_struct *p);
 
 
-static inline void wait_for_interrupt(void)
+static __always_inline void wait_for_interrupt(void)
 {
 	ALT_RISCV_WFI();
 }
diff --git a/arch/riscv/include/asm/stacktrace.h b/arch/riscv/include/asm/stacktrace.h
index b1495a7e06ce..c7c991452e11 100644
--- a/arch/riscv/include/asm/stacktrace.h
+++ b/arch/riscv/include/asm/stacktrace.h
@@ -16,7 +16,7 @@ extern void notrace walk_stackframe(struct task_struct *task, struct pt_regs *re
 extern void dump_backtrace(struct pt_regs *regs, struct task_struct *task,
 			   const char *loglvl);
 
-static inline bool on_thread_stack(void)
+static __always_inline bool on_thread_stack(void)
 {
 	return !(((unsigned long)(current->stack) ^ current_stack_pointer) & ~(THREAD_SIZE - 1));
 }
diff --git a/arch/riscv/include/asm/vector.h b/arch/riscv/include/asm/vector.h
index cd09f32e1ea3..1e8f22484733 100644
--- a/arch/riscv/include/asm/vector.h
+++ b/arch/riscv/include/asm/vector.h
@@ -85,12 +85,12 @@ static __always_inline bool has_xtheadvector(void)
 		return false;
 }
 
-static inline void __riscv_v_vstate_clean(struct pt_regs *regs)
+static __always_inline void __riscv_v_vstate_clean(struct pt_regs *regs)
 {
 	regs->status = __riscv_v_vstate_or(regs->status, CLEAN);
 }
 
-static inline void __riscv_v_vstate_dirty(struct pt_regs *regs)
+static __always_inline void __riscv_v_vstate_dirty(struct pt_regs *regs)
 {
 	regs->status = __riscv_v_vstate_or(regs->status, DIRTY);
 }
@@ -105,7 +105,7 @@ static inline void riscv_v_vstate_on(struct pt_regs *regs)
 	regs->status = __riscv_v_vstate_or(regs->status, INITIAL);
 }
 
-static inline bool riscv_v_vstate_query(struct pt_regs *regs)
+static __always_inline bool riscv_v_vstate_query(struct pt_regs *regs)
 {
 	return !__riscv_v_vstate_check(regs->status, OFF);
 }
@@ -198,7 +198,7 @@ static __always_inline void __vstate_csr_restore(struct __riscv_v_ext_state *src
 	}
 }
 
-static inline void __riscv_v_vstate_save(struct __riscv_v_ext_state *save_to,
+static __always_inline void __riscv_v_vstate_save(struct __riscv_v_ext_state *save_to,
 					 void *datap)
 {
 	unsigned long vl;
@@ -235,7 +235,7 @@ static inline void __riscv_v_vstate_save(struct __riscv_v_ext_state *save_to,
 	riscv_v_disable();
 }
 
-static inline void __riscv_v_vstate_restore(struct __riscv_v_ext_state *restore_from,
+static __always_inline void __riscv_v_vstate_restore(struct __riscv_v_ext_state *restore_from,
 					    void *datap)
 {
 	unsigned long vl;
@@ -272,7 +272,7 @@ static inline void __riscv_v_vstate_restore(struct __riscv_v_ext_state *restore_
 	riscv_v_disable();
 }
 
-static inline void __riscv_v_vstate_discard(void)
+static __always_inline void __riscv_v_vstate_discard(void)
 {
 	unsigned long vl, vtype_inval = 1UL << (BITS_PER_LONG - 1);
 
@@ -300,7 +300,7 @@ static inline void __riscv_v_vstate_discard(void)
 	riscv_v_disable();
 }
 
-static inline void riscv_v_vstate_discard(struct pt_regs *regs)
+static __always_inline void riscv_v_vstate_discard(struct pt_regs *regs)
 {
 	if (riscv_v_vstate_query(regs)) {
 		__riscv_v_vstate_discard();
@@ -317,7 +317,7 @@ static inline void riscv_v_vstate_save(struct __riscv_v_ext_state *vstate,
 	}
 }
 
-static inline void riscv_v_vstate_restore(struct __riscv_v_ext_state *vstate,
+static __always_inline void riscv_v_vstate_restore(struct __riscv_v_ext_state *vstate,
 					  struct pt_regs *regs)
 {
 	if (riscv_v_vstate_query(regs)) {
@@ -341,12 +341,12 @@ static inline bool riscv_preempt_v_dirty(struct task_struct *task)
 	return !!(task->thread.riscv_v_flags & RISCV_PREEMPT_V_DIRTY);
 }
 
-static inline bool riscv_preempt_v_restore(struct task_struct *task)
+static __always_inline bool riscv_preempt_v_restore(struct task_struct *task)
 {
 	return !!(task->thread.riscv_v_flags & RISCV_PREEMPT_V_NEED_RESTORE);
 }
 
-static inline void riscv_preempt_v_clear_dirty(struct task_struct *task)
+static __always_inline void riscv_preempt_v_clear_dirty(struct task_struct *task)
 {
 	barrier();
 	task->thread.riscv_v_flags &= ~RISCV_PREEMPT_V_DIRTY;
@@ -358,7 +358,7 @@ static inline void riscv_preempt_v_set_restore(struct task_struct *task)
 	task->thread.riscv_v_flags |= RISCV_PREEMPT_V_NEED_RESTORE;
 }
 
-static inline bool riscv_preempt_v_started(struct task_struct *task)
+static __always_inline bool riscv_preempt_v_started(struct task_struct *task)
 {
 	return !!(task->thread.riscv_v_flags & RISCV_PREEMPT_V);
 }
diff --git a/arch/riscv/kernel/kernel_mode_vector.c b/arch/riscv/kernel/kernel_mode_vector.c
index b9481a3e40e3..a8b308b3a231 100644
--- a/arch/riscv/kernel/kernel_mode_vector.c
+++ b/arch/riscv/kernel/kernel_mode_vector.c
@@ -124,27 +124,27 @@ static __always_inline u32 *riscv_v_flags_ptr(void)
 	return &current->thread.riscv_v_flags;
 }
 
-static inline void riscv_preempt_v_set_dirty(void)
+static __always_inline void riscv_preempt_v_set_dirty(void)
 {
 	*riscv_v_flags_ptr() |= RISCV_PREEMPT_V_DIRTY;
 }
 
-static inline void riscv_preempt_v_reset_flags(void)
+static __always_inline void riscv_preempt_v_reset_flags(void)
 {
 	*riscv_v_flags_ptr() &= ~(RISCV_PREEMPT_V_DIRTY | RISCV_PREEMPT_V_NEED_RESTORE);
 }
 
-static inline void riscv_v_ctx_depth_inc(void)
+static __always_inline void riscv_v_ctx_depth_inc(void)
 {
 	*riscv_v_flags_ptr() += RISCV_V_CTX_UNIT_DEPTH;
 }
 
-static inline void riscv_v_ctx_depth_dec(void)
+static __always_inline void riscv_v_ctx_depth_dec(void)
 {
 	*riscv_v_flags_ptr() -= RISCV_V_CTX_UNIT_DEPTH;
 }
 
-static inline u32 riscv_v_ctx_get_depth(void)
+static __always_inline u32 riscv_v_ctx_get_depth(void)
 {
 	return *riscv_v_flags_ptr() & RISCV_V_CTX_DEPTH_MASK;
 }
@@ -186,8 +186,8 @@ static int riscv_v_start_kernel_context(void)
 	return 0;
 }
 
-/* low-level V context handling code, called with irq disabled */
-asmlinkage void riscv_v_context_nesting_start(struct pt_regs *regs)
+/* Called from assembly before entry bookkeeping, with interrupts disabled. */
+asmlinkage void noinstr riscv_v_context_nesting_start(struct pt_regs *regs)
 {
 	int depth;
 
@@ -201,7 +201,8 @@ asmlinkage void riscv_v_context_nesting_start(struct pt_regs *regs)
 	riscv_v_ctx_depth_inc();
 }
 
-asmlinkage void riscv_v_context_nesting_end(struct pt_regs *regs)
+/* Called from assembly after exit bookkeeping, with interrupts disabled. */
+asmlinkage void noinstr riscv_v_context_nesting_end(struct pt_regs *regs)
 {
 	struct __riscv_v_ext_state *vstate = &current->thread.kernel_vstate;
 	u32 depth;
-- 
2.55.0