From: Yunhui Cui <cuiyunhui@bytedance.com>
To: pjw@kernel.org, palmer@dabbelt.com, aou@eecs.berkeley.edu,
alex@ghiti.fr, dennis@kernel.org, tj@kernel.org, cl@gentwo.org,
oleg@redhat.com, ast@kernel.org, daniel@iogearbox.net,
andrii@kernel.org, eddyz87@gmail.com, memxor@gmail.com,
martin.lau@linux.dev, song@kernel.org, yonghong.song@linux.dev,
jolsa@kernel.org, emil@etsalapatis.com, ihor.solodrai@linux.dev,
bjorn@kernel.org, pulehui@huawei.com, puranjay@kernel.org,
broonie@kernel.org, cuiyunhui@bytedance.com, dave@stgolabs.net,
david.laight.linux@gmail.com, jszhang@kernel.org,
tglx@kernel.org, conor.dooley@microchip.com, thuth@redhat.com,
ajones@ventanamicro.com, ben.dooks@codethink.co.uk,
rkrcmar@ventanamicro.com, samuel.holland@sifive.com,
zong.li@sifive.com, debug@rivosinc.com, seanwascoding@gmail.com,
andybnac@gmail.com, menglong8.dong@gmail.com,
cyrilbur@tenstorrent.com, wangruikang@iscas.ac.cn,
atishp@rivosinc.com, apatel@ventanamicro.com, arnd@arndb.de,
nathan@kernel.org, nick.desaulniers+lkml@gmail.com,
morbo@google.com, justinstitt@google.com,
qingfang.deng@siflower.com.cn, linux-riscv@lists.infradead.org,
linux-kernel@vger.kernel.org, linux-mm@kvack.org,
bpf@vger.kernel.org, linux-arch@vger.kernel.org,
llvm@lists.linux.dev
Subject: [PATCH v10 2/9] riscv: add infrastructure for preemptible this_cpu operations
Date: Thu, 12 Nov 2026 18:41:42 +0800 [thread overview]
Message-ID: <20261112104149.38589-3-cuiyunhui@bytedance.com> (raw)
In-Reply-To: <20261112104149.38589-1-cuiyunhui@bytedance.com>
RISC-V does not provide architecture-specific this_cpu operations, so
read-modify-write operations use generic fallbacks that disable local
interrupts. Disabling preemption around each architecture-specific
operation would avoid the interrupt-off region, but would still add
preemption-count bookkeeping and a reschedule check to the common path.
Use the recoverable-GPR scheme employed by arm64 so the operations can
remain preemptible. Record the registers used for per-CPU address
generation and repair them on exception return if the task migrated.
Exception entry saves and clears the active state so handlers can use
per-CPU operations independently. Keep this protocol in common assembly
macros so NMI-like entry paths which bypass handle_exception, such as SBI
Supervisor Software Events, can preserve an interrupted operation.
The existing exception return path clears LR/SC reservations, so an
interrupted LR/SC sequence retries after its saved address is repaired.
Suggested-by: David Laight <david.laight.linux@gmail.com>
Link: https://lore.kernel.org/r/20261108213254.67805ef0@pumpkin
Suggested-by: Jisheng Zhang <jszhang@kernel.org>
Link: https://lore.kernel.org/r/avG07pmpcLGkq5i-@xhacker
Signed-off-by: Yunhui Cui <cuiyunhui@bytedance.com>
---
arch/riscv/include/asm/asm.h | 19 ++++++++++++
arch/riscv/include/asm/percpu.h | 43 ++++++++++++++++++++++++++++
arch/riscv/include/asm/ptrace.h | 15 ++++++++++
arch/riscv/include/asm/thread_info.h | 1 +
arch/riscv/kernel/Makefile | 1 +
arch/riscv/kernel/asm-offsets.c | 2 ++
arch/riscv/kernel/entry-common.c | 32 +++++++++++++++++++++
arch/riscv/kernel/entry.S | 10 +++++++
8 files changed, 123 insertions(+)
create mode 100644 arch/riscv/kernel/entry-common.c
diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h
index 06ac38f733b0a..57af10a936d9b 100644
--- a/arch/riscv/include/asm/asm.h
+++ b/arch/riscv/include/asm/asm.h
@@ -81,6 +81,25 @@
/* Common assembly source macros */
+/*
+ * Save and clear the active per-CPU GPR state on exception entry. Entry
+ * paths which bypass handle_exception, such as SBI Supervisor Software
+ * Events (SSE), must pair this with pcpu_gprs_exit around their handlers.
+ */
+.macro pcpu_gprs_entry tsk regs tmp
+ lhu \tmp, TASK_TI_PCPU_GPRS(\tsk)
+ REG_S \tmp, PT_PCPU_GPRS(\regs)
+ sh zero, TASK_TI_PCPU_GPRS(\tsk)
+.endm
+
+/*
+ * Restore the saved state before resuming the interrupted context. tmp must
+ * contain PT_PCPU_GPRS from the interrupted frame.
+ */
+.macro pcpu_gprs_exit tsk tmp
+ sh \tmp, TASK_TI_PCPU_GPRS(\tsk)
+.endm
+
/*
* NOP sequence
*/
diff --git a/arch/riscv/include/asm/percpu.h b/arch/riscv/include/asm/percpu.h
index faf9a75ec66d7..59f7d44f7d3fe 100644
--- a/arch/riscv/include/asm/percpu.h
+++ b/arch/riscv/include/asm/percpu.h
@@ -3,12 +3,55 @@
#ifndef __ASM_PERCPU_H
#define __ASM_PERCPU_H
+#include <linux/bits.h>
+#include <linux/stringify.h>
#include <linux/thread_info.h>
+#include <asm/asm.h>
#include <asm/current.h>
+#include <asm/gpr-num.h>
#define __my_cpu_offset (((struct thread_info *)current)->pcpu_offset)
+#define PCPU_GPR_PCP_SHIFT 0
+#define PCPU_GPR_PCP GENMASK(4, 0)
+#define PCPU_GPR_OFF_SHIFT 5
+#define PCPU_GPR_OFF GENMASK(9, 5)
+#define PCPU_GPR_ADDR_SHIFT 10
+#define PCPU_GPR_ADDR GENMASK(14, 10)
+
+#define __PCPU_GPR_NUM(gpr) ".L__gpr_num_" gpr
+
+#define __PCPU_GPRS_VALUE \
+ "(" \
+ "(" __PCPU_GPR_NUM("%[pcp]") " << " __stringify(PCPU_GPR_PCP_SHIFT) ") | " \
+ "(" __PCPU_GPR_NUM("%[off]") " << " __stringify(PCPU_GPR_OFF_SHIFT) ") | " \
+ "(" __PCPU_GPR_NUM("%[addr]") " << " __stringify(PCPU_GPR_ADDR_SHIFT) ")" \
+ ")"
+
+#define __PCPU_GPRS_ASSERT_DISTINCT \
+ ".if ((" __PCPU_GPR_NUM("%[pcp]") " == " __PCPU_GPR_NUM("%[off]") ") || " \
+ "(" __PCPU_GPR_NUM("%[pcp]") " == " __PCPU_GPR_NUM("%[addr]") ") || " \
+ "(" __PCPU_GPR_NUM("%[off]") " == " __PCPU_GPR_NUM("%[addr]") "))\n" \
+ ".error \"PCPU GPRs overlap\"\n" \
+ ".endif\n"
+
+/*
+ * Mark a region where the per-CPU offset and address registers may need to
+ * be rewritten by exception return after migration. The three registers
+ * must remain distinct for the whole region.
+ */
+#define __PCPU_GPRS_BEGIN \
+ __DEFINE_ASM_GPR_NUMS \
+ __PCPU_GPRS_ASSERT_DISTINCT \
+ "li %[off], " __PCPU_GPRS_VALUE "\n" \
+ "sh %[off], %[gprs]\n" \
+ REG_L " %[off], %[pcpu_offset]\n" \
+ "add %[addr], %[pcp], %[off]\n"
+
+#define __PCPU_GPRS_END \
+ "sh zero, %[gprs]\n"
+
#include <asm-generic/percpu.h>
#endif /* __ASM_PERCPU_H */
diff --git a/arch/riscv/include/asm/ptrace.h b/arch/riscv/include/asm/ptrace.h
index addc8188152f7..ceff8f63bc866 100644
--- a/arch/riscv/include/asm/ptrace.h
+++ b/arch/riscv/include/asm/ptrace.h
@@ -53,6 +53,8 @@ struct pt_regs {
unsigned long cause;
/* a0 value before the syscall */
unsigned long orig_a0;
+ /* Active preemptible per-CPU operation, if any. */
+ unsigned long pcpu_gprs;
};
#define PTRACE_SYSEMU 0x1f
@@ -145,6 +147,19 @@ static inline unsigned long regs_get_register(struct pt_regs *regs,
return *(unsigned long *)((unsigned long)regs + offset);
}
+static inline void regs_set_register(struct pt_regs *regs,
+ unsigned int offset, unsigned long val)
+{
+ if (unlikely(offset > MAX_REG_OFFSET))
+ return;
+
+ /* Register x0 is represented by the first, read-only slot. */
+ if (offset)
+ *(unsigned long *)((unsigned long)regs + offset) = val;
+}
+
+void noinstr riscv_pcpu_regs_fixup(struct pt_regs *regs);
+
/**
* regs_get_kernel_argument() - get Nth function argument in kernel
* @regs: pt_regs of that context
diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h
index f10ba62b61016..51c4076763c1b 100644
--- a/arch/riscv/include/asm/thread_info.h
+++ b/arch/riscv/include/asm/thread_info.h
@@ -63,6 +63,7 @@ struct thread_info {
long user_sp; /* User stack pointer */
unsigned long syscall_work; /* SYSCALL_WORK_ flags */
unsigned long pcpu_offset;
+ unsigned short pcpu_gprs;
#ifdef CONFIG_SHADOW_CALL_STACK
void *scs_base;
void *scs_sp;
diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
index f54c336bf547c..bf786d5dd4de3 100644
--- a/arch/riscv/kernel/Makefile
+++ b/arch/riscv/kernel/Makefile
@@ -52,6 +52,7 @@ obj-y += alternative.o
obj-y += cpu.o
obj-y += cpufeature.o
obj-y += entry.o
+obj-y += entry-common.o
obj-y += irq.o
obj-y += process.o
obj-y += ptrace.o
diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c
index f4f7ee976cd94..38f246a691c60 100644
--- a/arch/riscv/kernel/asm-offsets.c
+++ b/arch/riscv/kernel/asm-offsets.c
@@ -39,6 +39,7 @@ void asm_offsets(void)
OFFSET(TASK_TI_CPU, task_struct, thread_info.cpu);
OFFSET(TASK_TI_PCPU_OFFSET, task_struct, thread_info.pcpu_offset);
+ OFFSET(TASK_TI_PCPU_GPRS, task_struct, thread_info.pcpu_gprs);
OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count);
OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp);
OFFSET(TASK_TI_USER_SP, task_struct, thread_info.user_sp);
@@ -130,6 +131,7 @@ void asm_offsets(void)
OFFSET(PT_STATUS, pt_regs, status);
OFFSET(PT_BADADDR, pt_regs, badaddr);
OFFSET(PT_CAUSE, pt_regs, cause);
+ OFFSET(PT_PCPU_GPRS, pt_regs, pcpu_gprs);
OFFSET(SUSPEND_CONTEXT_REGS, suspend_context, regs);
diff --git a/arch/riscv/kernel/entry-common.c b/arch/riscv/kernel/entry-common.c
new file mode 100644
index 0000000000000..023d856b1679b
--- /dev/null
+++ b/arch/riscv/kernel/entry-common.c
@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/bitfield.h>
+#include <linux/compiler.h>
+#include <linux/percpu.h>
+
+#include <asm/current.h>
+#include <asm/ptrace.h>
+
+/*
+ * If the interrupted context had an active per-CPU operation, rewrite its
+ * saved offset and address registers for the CPU on which it will resume.
+ */
+void noinstr riscv_pcpu_regs_fixup(struct pt_regs *regs)
+{
+ unsigned long pcpu_offset = __my_cpu_offset;
+ unsigned long gprs = regs->pcpu_gprs;
+ unsigned long pcp;
+ unsigned int reg_pcp;
+ unsigned int reg_off;
+ unsigned int reg_addr;
+
+ reg_pcp = FIELD_GET(PCPU_GPR_PCP, gprs);
+ reg_off = FIELD_GET(PCPU_GPR_OFF, gprs);
+ reg_addr = FIELD_GET(PCPU_GPR_ADDR, gprs);
+
+ /* pt_regs offset zero represents x0, followed by x1 through x31. */
+ pcp = regs_get_register(regs, reg_pcp * sizeof(unsigned long));
+ regs_set_register(regs, reg_off * sizeof(unsigned long), pcpu_offset);
+ regs_set_register(regs, reg_addr * sizeof(unsigned long),
+ pcp + pcpu_offset);
+}
diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
index 628b85e3915a7..65298ac828959 100644
--- a/arch/riscv/kernel/entry.S
+++ b/arch/riscv/kernel/entry.S
@@ -198,6 +198,8 @@ SYM_CODE_START(handle_exception)
REG_S s4, PT_CAUSE(sp)
REG_S s5, PT_TP(sp)
+ pcpu_gprs_entry tp sp s0
+
/*
* Set the scratch register to 0, so that if a recursive exception
* occurs, the exception vector knows it came from the kernel
@@ -281,6 +283,14 @@ SYM_CODE_START_NOALIGN(ret_from_exception)
move a0, sp
call riscv_v_context_nesting_end
#endif
+ REG_L t0, PT_PCPU_GPRS(sp)
+ beqz t0, .Lpcpu_regs_restored
+ move a0, sp
+ call riscv_pcpu_regs_fixup
+ REG_L t0, PT_PCPU_GPRS(sp)
+.Lpcpu_regs_restored:
+ pcpu_gprs_exit tp t0
+
REG_L a0, PT_STATUS(sp)
restore_userssp s3, a0
/*
--
2.39.5
next prev parent reply other threads:[~2026-11-12 10:43 UTC|newest]
Thread overview: 15+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-12 10:41 [PATCH v10 0/9] riscv: optimize percpu access Yunhui Cui
2026-11-12 10:41 ` [PATCH v10 1/9] riscv: store percpu offset into thread_info Yunhui Cui
2026-11-12 10:41 ` Yunhui Cui [this message]
2026-11-12 12:02 ` [PATCH v10 2/9] riscv: add infrastructure for preemptible this_cpu operations David Laight
2026-11-12 10:41 ` [PATCH v10 3/9] riscv: implement preemptible this_cpu read and write operations Yunhui Cui
2026-11-12 10:41 ` [PATCH v10 4/9] riscv: implement preemptible this_cpu void RMW operations Yunhui Cui
2026-11-12 10:41 ` [PATCH v10 5/9] riscv: implement preemptible this_cpu add_return operations Yunhui Cui
2026-11-12 11:30 ` bot+bpf-ci
2026-11-12 12:09 ` David Laight
2026-11-12 10:41 ` [PATCH v10 6/9] riscv: implement preemptible this_cpu xchg operations Yunhui Cui
2026-11-12 10:41 ` [PATCH v10 7/9] riscv: implement preemptible this_cpu cmpxchg operations Yunhui Cui
2026-11-12 10:41 ` [PATCH v10 8/9] riscv: implement preemptible this_cpu cmpxchg128 Yunhui Cui
2026-11-12 10:41 ` [PATCH v10 9/9] riscv: select HAVE_CMPXCHG_LOCAL Yunhui Cui
2026-11-12 11:30 ` bot+bpf-ci
2026-11-12 12:10 ` [PATCH v10 0/9] riscv: optimize percpu access Jisheng Zhang
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261112104149.38589-3-cuiyunhui@bytedance.com \
--to=cuiyunhui@bytedance.com \
--cc=ajones@ventanamicro.com \
--cc=alex@ghiti.fr \
--cc=andrii@kernel.org \
--cc=andybnac@gmail.com \
--cc=aou@eecs.berkeley.edu \
--cc=apatel@ventanamicro.com \
--cc=arnd@arndb.de \
--cc=ast@kernel.org \
--cc=atishp@rivosinc.com \
--cc=ben.dooks@codethink.co.uk \
--cc=bjorn@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=broonie@kernel.org \
--cc=cl@gentwo.org \
--cc=conor.dooley@microchip.com \
--cc=cyrilbur@tenstorrent.com \
--cc=daniel@iogearbox.net \
--cc=dave@stgolabs.net \
--cc=david.laight.linux@gmail.com \
--cc=debug@rivosinc.com \
--cc=dennis@kernel.org \
--cc=eddyz87@gmail.com \
--cc=emil@etsalapatis.com \
--cc=ihor.solodrai@linux.dev \
--cc=jolsa@kernel.org \
--cc=jszhang@kernel.org \
--cc=justinstitt@google.com \
--cc=linux-arch@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=linux-riscv@lists.infradead.org \
--cc=llvm@lists.linux.dev \
--cc=martin.lau@linux.dev \
--cc=memxor@gmail.com \
--cc=menglong8.dong@gmail.com \
--cc=morbo@google.com \
--cc=nathan@kernel.org \
--cc=nick.desaulniers+lkml@gmail.com \
--cc=oleg@redhat.com \
--cc=palmer@dabbelt.com \
--cc=pjw@kernel.org \
--cc=pulehui@huawei.com \
--cc=puranjay@kernel.org \
--cc=qingfang.deng@siflower.com.cn \
--cc=rkrcmar@ventanamicro.com \
--cc=samuel.holland@sifive.com \
--cc=seanwascoding@gmail.com \
--cc=song@kernel.org \
--cc=tglx@kernel.org \
--cc=thuth@redhat.com \
--cc=tj@kernel.org \
--cc=wangruikang@iscas.ac.cn \
--cc=yonghong.song@linux.dev \
--cc=zong.li@sifive.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®