From: Yunhui Cui <cuiyunhui@bytedance.com>
To: pjw@kernel.org, palmer@dabbelt.com, aou@eecs.berkeley.edu,
alex@ghiti.fr, dennis@kernel.org, tj@kernel.org, cl@gentwo.org,
oleg@redhat.com, ast@kernel.org, daniel@iogearbox.net,
andrii@kernel.org, eddyz87@gmail.com, memxor@gmail.com,
martin.lau@linux.dev, song@kernel.org, yonghong.song@linux.dev,
jolsa@kernel.org, emil@etsalapatis.com, ihor.solodrai@linux.dev,
bjorn@kernel.org, pulehui@huawei.com, puranjay@kernel.org,
broonie@kernel.org, cuiyunhui@bytedance.com, dave@stgolabs.net,
david.laight.linux@gmail.com, jszhang@kernel.org,
tglx@kernel.org, conor.dooley@microchip.com, thuth@redhat.com,
ajones@ventanamicro.com, ben.dooks@codethink.co.uk,
rkrcmar@ventanamicro.com, samuel.holland@sifive.com,
zong.li@sifive.com, debug@rivosinc.com, seanwascoding@gmail.com,
andybnac@gmail.com, menglong8.dong@gmail.com,
cyrilbur@tenstorrent.com, wangruikang@iscas.ac.cn,
atishp@rivosinc.com, apatel@ventanamicro.com, arnd@arndb.de,
nathan@kernel.org, nick.desaulniers+lkml@gmail.com,
morbo@google.com, justinstitt@google.com,
qingfang.deng@siflower.com.cn, linux-riscv@lists.infradead.org,
linux-kernel@vger.kernel.org, linux-mm@kvack.org,
bpf@vger.kernel.org, linux-arch@vger.kernel.org,
llvm@lists.linux.dev
Subject: [PATCH v10 6/9] riscv: implement preemptible this_cpu xchg operations
Date: Thu, 12 Nov 2026 18:41:46 +0800 [thread overview]
Message-ID: <20261112104149.38589-7-cuiyunhui@bytedance.com> (raw)
In-Reply-To: <20261112104149.38589-1-cuiyunhui@bytedance.com>
Apply the recoverable per-CPU address handling to exchange operations.
Use AMOSWAP for native-width and Zabha subword operations. Where Zabha is
unavailable, use a masked LR/SC sequence that preserves neighboring fields
in the same word.
Signed-off-by: Yunhui Cui <cuiyunhui@bytedance.com>
---
arch/riscv/include/asm/percpu.h | 92 +++++++++++++++++++++++++++++++++
1 file changed, 92 insertions(+)
diff --git a/arch/riscv/include/asm/percpu.h b/arch/riscv/include/asm/percpu.h
index 9ff02f3811550..810f4c5eaa508 100644
--- a/arch/riscv/include/asm/percpu.h
+++ b/arch/riscv/include/asm/percpu.h
@@ -326,6 +326,81 @@ __percpu_add_return_amo_case_16(void __percpu *pcp, unsigned long val)
return __percpu_add_return_8_16(pcp, val, 16);
}
+#define __PERCPU_XCHG_AMO_CASE(sfx, sz) \
+static __always_inline unsigned long \
+__percpu_xchg_case_##sz(void __percpu *pcp, unsigned long val) \
+{ \
+ __PCPU_OP_CONTEXT(); \
+ u##sz ret; \
+ \
+ asm volatile (__PCPU_GPRS_BEGIN \
+ "amoswap" #sfx " %[ret], %[val], 0(%[addr])\n" \
+ __PCPU_GPRS_END \
+ : [gprs] "=m" (__ti->pcpu_gprs), \
+ [off] "=&r" (__off), [addr] "=&r" (__addr), \
+ [ret] "=&r" (ret) \
+ : [pcpu_offset] "m" (__ti->pcpu_offset), \
+ [pcp] "r" (pcp), [val] "r" ((u##sz)val) \
+ : "memory"); \
+ \
+ return ret; \
+}
+
+__PERCPU_XCHG_AMO_CASE(.w, 32)
+#ifdef CONFIG_64BIT
+__PERCPU_XCHG_AMO_CASE(.d, 64)
+#endif
+
+#define __PERCPU_8_16_XCHG_CASE(sz, sfx) \
+static __always_inline unsigned long \
+__percpu_xchg_case_##sz(void __percpu *pcp, unsigned long val) \
+{ \
+ __PCPU_OP_CONTEXT(); \
+ const u32 field_mask = PERCPU_8_16_GET_MASK(sz); \
+ const unsigned long shift = PERCPU_8_16_GET_SHIFT(pcp); \
+ const u32 mask = field_mask << shift; \
+ const u32 new_field = (val & field_mask) << shift; \
+ void __percpu *pcp32 = PERCPU_8_16_GET_PCP32(pcp); \
+ u32 old, new, rc; \
+ u##sz ret; \
+ \
+ if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) && \
+ riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA)) { \
+ asm volatile (__PCPU_GPRS_BEGIN \
+ ".option push\n" \
+ ".option arch, +zabha\n" \
+ "amoswap" #sfx " %[ret], %[val], 0(%[addr])\n" \
+ ".option pop\n" \
+ __PCPU_GPRS_END \
+ : [gprs] "=m" (__ti->pcpu_gprs), \
+ [off] "=&r" (__off), [addr] "=&r" (__addr), \
+ [ret] "=&r" (ret) \
+ : [pcpu_offset] "m" (__ti->pcpu_offset), \
+ [pcp] "r" (pcp), [val] "r" ((u##sz)val) \
+ : "memory"); \
+ return ret; \
+ } \
+ \
+ asm volatile (__PCPU_GPRS_BEGIN \
+ "0: lr.w %[old], 0(%[addr])\n" \
+ "and %[new], %[old], %[inv_mask]\n" \
+ "or %[new], %[new], %[new_field]\n" \
+ "sc.w %[rc], %[new], 0(%[addr])\n" \
+ "bnez %[rc], 0b\n" \
+ __PCPU_GPRS_END \
+ : [gprs] "=m" (__ti->pcpu_gprs), \
+ [off] "=&r" (__off), [addr] "=&r" (__addr), \
+ [old] "=&r" (old), [new] "=&r" (new), [rc] "=&r" (rc) \
+ : [pcpu_offset] "m" (__ti->pcpu_offset), \
+ [pcp] "r" (pcp32), [inv_mask] "r" (~mask), \
+ [new_field] "r" (new_field) \
+ : "memory"); \
+ \
+ return (old & mask) >> shift; \
+}
+
+__PERCPU_8_16_XCHG_CASE(8, .b)
+__PERCPU_8_16_XCHG_CASE(16, .h)
#define _pcp_wrap(op, pcp, ...) \
({ \
op(&(pcp), __VA_ARGS__); \
@@ -338,6 +413,14 @@ __percpu_add_return_amo_case_16(void __percpu *pcp, unsigned long val)
(TYPEOF_UNQUAL(*__pcp_ptr))op(__pcp_ptr, ##args); \
})
+#define _pcp_wrap_xchg(op, pcp, val) \
+({ \
+ __auto_type __pcp_ptr = &(pcp); \
+ \
+ (TYPEOF_UNQUAL(*__pcp_ptr))op(__pcp_ptr, \
+ (unsigned long)(val)); \
+})
+
#define this_cpu_read_1(pcp) _pcp_wrap_return(__percpu_read_8, pcp)
#define this_cpu_read_2(pcp) _pcp_wrap_return(__percpu_read_16, pcp)
#define this_cpu_read_4(pcp) _pcp_wrap_return(__percpu_read_32, pcp)
@@ -390,9 +473,18 @@ __percpu_add_return_amo_case_16(void __percpu *pcp, unsigned long val)
_pcp_wrap_return(__percpu_add_return_amo_case_64, pcp, val)
#endif
+#define this_cpu_xchg_1(pcp, val) _pcp_wrap_xchg(__percpu_xchg_case_8, pcp, val)
+#define this_cpu_xchg_2(pcp, val) _pcp_wrap_xchg(__percpu_xchg_case_16, pcp, val)
+#define this_cpu_xchg_4(pcp, val) _pcp_wrap_xchg(__percpu_xchg_case_32, pcp, val)
+#ifdef CONFIG_64BIT
+#define this_cpu_xchg_8(pcp, val) _pcp_wrap_xchg(__percpu_xchg_case_64, pcp, val)
+#endif
+
#undef PERCPU_RW_OPS
#undef __PERCPU_AMO_OP_CASE
#undef __PERCPU_8_16_OP_CASE
+#undef __PERCPU_XCHG_AMO_CASE
+#undef __PERCPU_8_16_XCHG_CASE
#undef PERCPU_8_16_GET_SHIFT
#undef PERCPU_8_16_GET_MASK
#undef PERCPU_8_16_GET_PCP32
--
2.39.5
next prev parent reply other threads:[~2026-11-12 10:44 UTC|newest]
Thread overview: 15+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-12 10:41 [PATCH v10 0/9] riscv: optimize percpu access Yunhui Cui
2026-11-12 10:41 ` [PATCH v10 1/9] riscv: store percpu offset into thread_info Yunhui Cui
2026-11-12 10:41 ` [PATCH v10 2/9] riscv: add infrastructure for preemptible this_cpu operations Yunhui Cui
2026-11-12 12:02 ` David Laight
2026-11-12 10:41 ` [PATCH v10 3/9] riscv: implement preemptible this_cpu read and write operations Yunhui Cui
2026-11-12 10:41 ` [PATCH v10 4/9] riscv: implement preemptible this_cpu void RMW operations Yunhui Cui
2026-11-12 10:41 ` [PATCH v10 5/9] riscv: implement preemptible this_cpu add_return operations Yunhui Cui
2026-11-12 11:30 ` bot+bpf-ci
2026-11-12 12:09 ` David Laight
2026-11-12 10:41 ` Yunhui Cui [this message]
2026-11-12 10:41 ` [PATCH v10 7/9] riscv: implement preemptible this_cpu cmpxchg operations Yunhui Cui
2026-11-12 10:41 ` [PATCH v10 8/9] riscv: implement preemptible this_cpu cmpxchg128 Yunhui Cui
2026-11-12 10:41 ` [PATCH v10 9/9] riscv: select HAVE_CMPXCHG_LOCAL Yunhui Cui
2026-11-12 11:30 ` bot+bpf-ci
2026-11-12 12:10 ` [PATCH v10 0/9] riscv: optimize percpu access Jisheng Zhang
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261112104149.38589-7-cuiyunhui@bytedance.com \
--to=cuiyunhui@bytedance.com \
--cc=ajones@ventanamicro.com \
--cc=alex@ghiti.fr \
--cc=andrii@kernel.org \
--cc=andybnac@gmail.com \
--cc=aou@eecs.berkeley.edu \
--cc=apatel@ventanamicro.com \
--cc=arnd@arndb.de \
--cc=ast@kernel.org \
--cc=atishp@rivosinc.com \
--cc=ben.dooks@codethink.co.uk \
--cc=bjorn@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=broonie@kernel.org \
--cc=cl@gentwo.org \
--cc=conor.dooley@microchip.com \
--cc=cyrilbur@tenstorrent.com \
--cc=daniel@iogearbox.net \
--cc=dave@stgolabs.net \
--cc=david.laight.linux@gmail.com \
--cc=debug@rivosinc.com \
--cc=dennis@kernel.org \
--cc=eddyz87@gmail.com \
--cc=emil@etsalapatis.com \
--cc=ihor.solodrai@linux.dev \
--cc=jolsa@kernel.org \
--cc=jszhang@kernel.org \
--cc=justinstitt@google.com \
--cc=linux-arch@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=linux-riscv@lists.infradead.org \
--cc=llvm@lists.linux.dev \
--cc=martin.lau@linux.dev \
--cc=memxor@gmail.com \
--cc=menglong8.dong@gmail.com \
--cc=morbo@google.com \
--cc=nathan@kernel.org \
--cc=nick.desaulniers+lkml@gmail.com \
--cc=oleg@redhat.com \
--cc=palmer@dabbelt.com \
--cc=pjw@kernel.org \
--cc=pulehui@huawei.com \
--cc=puranjay@kernel.org \
--cc=qingfang.deng@siflower.com.cn \
--cc=rkrcmar@ventanamicro.com \
--cc=samuel.holland@sifive.com \
--cc=seanwascoding@gmail.com \
--cc=song@kernel.org \
--cc=tglx@kernel.org \
--cc=thuth@redhat.com \
--cc=tj@kernel.org \
--cc=wangruikang@iscas.ac.cn \
--cc=yonghong.song@linux.dev \
--cc=zong.li@sifive.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®