* [PATCH v2 1/3] x86/percpu: Fix x86_this_cpu_variable_test_bit() asm template
@ 2024-04-04 9:42 Uros Bizjak
2024-04-04 9:42 ` [PATCH v2 2/3] x86/percpu: Rewrite x86_this_cpu_test_bit() and friends as macros Uros Bizjak
` (2 more replies)
0 siblings, 3 replies; 6+ messages in thread
From: Uros Bizjak @ 2024-04-04 9:42 UTC (permalink / raw)
To: x86, linux-kernel
Cc: Uros Bizjak, Ingo Molnar, Andy Lutomirski, Brian Gerst,
Denys Vlasenko, H . Peter Anvin, Linus Torvalds, Peter Zijlstra,
Thomas Gleixner, Josh Poimboeuf
Fix x86_this_cpu_variable_test_bit(), which is implemented with
wrong asm template, where argument 2 (count argument) is considered
as percpu variable. However, x86_this_cpu_test_bit() is currently
used exclusively with constant bit number argument, so the called
x86_this_cpu_variable_test_bit() function is never instantiated.
The fix introduces named assembler operands to prevent this kind
of errors.
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
---
v2: Split from the original v1 patch.
---
arch/x86/include/asm/percpu.h | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 20696df5d567..cbfbbe836ee2 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -586,10 +586,11 @@ static inline bool x86_this_cpu_variable_test_bit(int nr,
{
bool oldbit;
- asm volatile("btl "__percpu_arg(2)",%1"
+ asm volatile("btl %[nr], " __percpu_arg([var])
CC_SET(c)
: CC_OUT(c) (oldbit)
- : "m" (*__my_cpu_ptr((unsigned long __percpu *)(addr))), "Ir" (nr));
+ : [var] "m" (*__my_cpu_ptr((unsigned long __percpu *)(addr))),
+ [nr] "Ir" (nr));
return oldbit;
}
--
2.44.0
^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH v2 2/3] x86/percpu: Rewrite x86_this_cpu_test_bit() and friends as macros
2024-04-04 9:42 [PATCH v2 1/3] x86/percpu: Fix x86_this_cpu_variable_test_bit() asm template Uros Bizjak
@ 2024-04-04 9:42 ` Uros Bizjak
2024-04-06 11:07 ` [tip: x86/percpu] " tip-bot2 for Uros Bizjak
2024-04-04 9:42 ` [PATCH v2 3/3] x86/percpu: Introduce raw_cpu_read_long() to reduce ifdeffery Uros Bizjak
2024-04-06 11:07 ` [tip: x86/percpu] x86/percpu: Fix x86_this_cpu_variable_test_bit() asm template tip-bot2 for Uros Bizjak
2 siblings, 1 reply; 6+ messages in thread
From: Uros Bizjak @ 2024-04-04 9:42 UTC (permalink / raw)
To: x86, linux-kernel
Cc: Uros Bizjak, Ingo Molnar, Andy Lutomirski, Brian Gerst,
Denys Vlasenko, H . Peter Anvin, Linus Torvalds, Peter Zijlstra,
Thomas Gleixner, Josh Poimboeuf
Rewrite the whole family of x86_this_cpu_test_bit() functions
as macros, so standard __my_cpu_var() and raw_cpu_read() macros
can be used on percpu variables. This approach considerably
simplifies implementation of functions and also introduces
standard checks on accessed percpu variables.
No functional changes intended.
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
---
v2: Split from the original v1 patch.
---
arch/um/include/asm/cpufeature.h | 3 +-
arch/x86/include/asm/cpufeature.h | 3 +-
arch/x86/include/asm/percpu.h | 54 +++++++++++++------------------
3 files changed, 25 insertions(+), 35 deletions(-)
diff --git a/arch/um/include/asm/cpufeature.h b/arch/um/include/asm/cpufeature.h
index 66fe06db872f..1eb8b834fbec 100644
--- a/arch/um/include/asm/cpufeature.h
+++ b/arch/um/include/asm/cpufeature.h
@@ -38,8 +38,7 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
#define this_cpu_has(bit) \
(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 : \
- x86_this_cpu_test_bit(bit, \
- (unsigned long __percpu *)&cpu_info.x86_capability))
+ x86_this_cpu_test_bit(bit, cpu_info.x86_capability))
/*
* This macro is for detection of features which need kernel
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index f95e1c8b7c01..7d527467ab9c 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -127,8 +127,7 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
#define this_cpu_has(bit) \
(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 : \
- x86_this_cpu_test_bit(bit, \
- (unsigned long __percpu *)&cpu_info.x86_capability))
+ x86_this_cpu_test_bit(bit, cpu_info.x86_capability))
/*
* This macro is for detection of features which need kernel
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index cbfbbe836ee2..d6ff0db32209 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -96,7 +96,7 @@
#endif /* CONFIG_SMP */
#define __my_cpu_type(var) typeof(var) __percpu_seg_override
-#define __my_cpu_ptr(ptr) (__my_cpu_type(*ptr)*)(__force uintptr_t)(ptr)
+#define __my_cpu_ptr(ptr) (__my_cpu_type(*(ptr))*)(__force uintptr_t)(ptr)
#define __my_cpu_var(var) (*__my_cpu_ptr(&(var)))
#define __percpu_arg(x) __percpu_prefix "%" #x
#define __force_percpu_arg(x) __force_percpu_prefix "%" #x
@@ -568,37 +568,29 @@ do { \
#define this_cpu_read_stable_8(pcp) ({ BUILD_BUG(); (typeof(pcp))0; })
#endif
-static __always_inline bool x86_this_cpu_constant_test_bit(unsigned int nr,
- const unsigned long __percpu *addr)
-{
- unsigned long __percpu *a =
- (unsigned long __percpu *)addr + nr / BITS_PER_LONG;
+#define x86_this_cpu_constant_test_bit(_nr, _var) \
+({ \
+ unsigned long __percpu *addr__ = \
+ (unsigned long __percpu *)&(_var) + ((_nr) / BITS_PER_LONG); \
+ !!((1UL << ((_nr) % BITS_PER_LONG)) & raw_cpu_read(*addr__)); \
+})
-#ifdef CONFIG_X86_64
- return ((1UL << (nr % BITS_PER_LONG)) & raw_cpu_read_8(*a)) != 0;
-#else
- return ((1UL << (nr % BITS_PER_LONG)) & raw_cpu_read_4(*a)) != 0;
-#endif
-}
-
-static inline bool x86_this_cpu_variable_test_bit(int nr,
- const unsigned long __percpu *addr)
-{
- bool oldbit;
-
- asm volatile("btl %[nr], " __percpu_arg([var])
- CC_SET(c)
- : CC_OUT(c) (oldbit)
- : [var] "m" (*__my_cpu_ptr((unsigned long __percpu *)(addr))),
- [nr] "Ir" (nr));
-
- return oldbit;
-}
-
-#define x86_this_cpu_test_bit(nr, addr) \
- (__builtin_constant_p((nr)) \
- ? x86_this_cpu_constant_test_bit((nr), (addr)) \
- : x86_this_cpu_variable_test_bit((nr), (addr)))
+#define x86_this_cpu_variable_test_bit(_nr, _var) \
+({ \
+ bool oldbit; \
+ \
+ asm volatile("btl %[nr], " __percpu_arg([var]) \
+ CC_SET(c) \
+ : CC_OUT(c) (oldbit) \
+ : [var] "m" (__my_cpu_var(_var)), \
+ [nr] "rI" (_nr)); \
+ oldbit; \
+})
+
+#define x86_this_cpu_test_bit(_nr, _var) \
+ (__builtin_constant_p(_nr) \
+ ? x86_this_cpu_constant_test_bit(_nr, _var) \
+ : x86_this_cpu_variable_test_bit(_nr, _var))
#include <asm-generic/percpu.h>
--
2.44.0
^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH v2 3/3] x86/percpu: Introduce raw_cpu_read_long() to reduce ifdeffery
2024-04-04 9:42 [PATCH v2 1/3] x86/percpu: Fix x86_this_cpu_variable_test_bit() asm template Uros Bizjak
2024-04-04 9:42 ` [PATCH v2 2/3] x86/percpu: Rewrite x86_this_cpu_test_bit() and friends as macros Uros Bizjak
@ 2024-04-04 9:42 ` Uros Bizjak
2024-04-06 11:07 ` [tip: x86/percpu] " tip-bot2 for Uros Bizjak
2024-04-06 11:07 ` [tip: x86/percpu] x86/percpu: Fix x86_this_cpu_variable_test_bit() asm template tip-bot2 for Uros Bizjak
2 siblings, 1 reply; 6+ messages in thread
From: Uros Bizjak @ 2024-04-04 9:42 UTC (permalink / raw)
To: x86, linux-kernel
Cc: Uros Bizjak, Ingo Molnar, Andy Lutomirski, Brian Gerst,
Denys Vlasenko, H . Peter Anvin, Linus Torvalds, Peter Zijlstra,
Thomas Gleixner, Josh Poimboeuf
Introduce raw_cpu_read_long() macro to slightly reduce ifdeffery
in arch/x86/include/asm/percpu.h.
No functional changes intended.
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
---
v2: Merge the patch with the rest of percpu series.
---
arch/x86/include/asm/percpu.h | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index d6ff0db32209..3bedee1801e2 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -59,12 +59,6 @@
#define __force_percpu_prefix "%%"__stringify(__percpu_seg)":"
#define __my_cpu_offset this_cpu_read(this_cpu_off)
-#ifdef CONFIG_X86_64
-#define __raw_my_cpu_offset raw_cpu_read_8(this_cpu_off);
-#else
-#define __raw_my_cpu_offset raw_cpu_read_4(this_cpu_off);
-#endif
-
/*
* Compared to the generic __my_cpu_offset version, the following
* saves one instruction and avoids clobbering a temp register.
@@ -76,7 +70,7 @@
#ifndef BUILD_VDSO32_64
#define arch_raw_cpu_ptr(_ptr) \
({ \
- unsigned long tcp_ptr__ = __raw_my_cpu_offset; \
+ unsigned long tcp_ptr__ = raw_cpu_read_long(this_cpu_off); \
tcp_ptr__ += (__force unsigned long)(_ptr); \
(typeof(*(_ptr)) __kernel __force *)tcp_ptr__; \
})
@@ -563,9 +557,13 @@ do { \
#define this_cpu_xchg_8(pcp, nval) this_percpu_xchg_op(pcp, nval)
#define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval)
#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, volatile, pcp, ovalp, nval)
+
+#define raw_cpu_read_long(pcp) raw_cpu_read_8(pcp)
#else
/* There is no generic 64 bit read stable operation for 32 bit targets. */
-#define this_cpu_read_stable_8(pcp) ({ BUILD_BUG(); (typeof(pcp))0; })
+#define this_cpu_read_stable_8(pcp) ({ BUILD_BUG(); (typeof(pcp))0; })
+
+#define raw_cpu_read_long(pcp) raw_cpu_read_4(pcp)
#endif
#define x86_this_cpu_constant_test_bit(_nr, _var) \
--
2.44.0
^ permalink raw reply [flat|nested] 6+ messages in thread
* [tip: x86/percpu] x86/percpu: Introduce raw_cpu_read_long() to reduce ifdeffery
2024-04-04 9:42 ` [PATCH v2 3/3] x86/percpu: Introduce raw_cpu_read_long() to reduce ifdeffery Uros Bizjak
@ 2024-04-06 11:07 ` tip-bot2 for Uros Bizjak
0 siblings, 0 replies; 6+ messages in thread
From: tip-bot2 for Uros Bizjak @ 2024-04-06 11:07 UTC (permalink / raw)
To: linux-tip-commits
Cc: Uros Bizjak, Ingo Molnar, Linus Torvalds, x86, linux-kernel
The following commit has been merged into the x86/percpu branch of tip:
Commit-ID: 93cfa544cf9e4771def159002304a2e366cd97af
Gitweb: https://git.kernel.org/tip/93cfa544cf9e4771def159002304a2e366cd97af
Author: Uros Bizjak <ubizjak@gmail.com>
AuthorDate: Thu, 04 Apr 2024 11:42:03 +02:00
Committer: Ingo Molnar <mingo@kernel.org>
CommitterDate: Sat, 06 Apr 2024 12:42:17 +02:00
x86/percpu: Introduce raw_cpu_read_long() to reduce ifdeffery
Introduce raw_cpu_read_long() macro to slightly reduce ifdeffery
in <asm/percpu.h>.
No functional changes intended.
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: https://lore.kernel.org/r/20240404094218.448963-3-ubizjak@gmail.com
---
arch/x86/include/asm/percpu.h | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index d6ff0db..3bedee1 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -59,12 +59,6 @@
#define __force_percpu_prefix "%%"__stringify(__percpu_seg)":"
#define __my_cpu_offset this_cpu_read(this_cpu_off)
-#ifdef CONFIG_X86_64
-#define __raw_my_cpu_offset raw_cpu_read_8(this_cpu_off);
-#else
-#define __raw_my_cpu_offset raw_cpu_read_4(this_cpu_off);
-#endif
-
/*
* Compared to the generic __my_cpu_offset version, the following
* saves one instruction and avoids clobbering a temp register.
@@ -76,7 +70,7 @@
#ifndef BUILD_VDSO32_64
#define arch_raw_cpu_ptr(_ptr) \
({ \
- unsigned long tcp_ptr__ = __raw_my_cpu_offset; \
+ unsigned long tcp_ptr__ = raw_cpu_read_long(this_cpu_off); \
tcp_ptr__ += (__force unsigned long)(_ptr); \
(typeof(*(_ptr)) __kernel __force *)tcp_ptr__; \
})
@@ -563,9 +557,13 @@ do { \
#define this_cpu_xchg_8(pcp, nval) this_percpu_xchg_op(pcp, nval)
#define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval)
#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, volatile, pcp, ovalp, nval)
+
+#define raw_cpu_read_long(pcp) raw_cpu_read_8(pcp)
#else
/* There is no generic 64 bit read stable operation for 32 bit targets. */
-#define this_cpu_read_stable_8(pcp) ({ BUILD_BUG(); (typeof(pcp))0; })
+#define this_cpu_read_stable_8(pcp) ({ BUILD_BUG(); (typeof(pcp))0; })
+
+#define raw_cpu_read_long(pcp) raw_cpu_read_4(pcp)
#endif
#define x86_this_cpu_constant_test_bit(_nr, _var) \
^ permalink raw reply [flat|nested] 6+ messages in thread
* [tip: x86/percpu] x86/percpu: Fix x86_this_cpu_variable_test_bit() asm template
2024-04-04 9:42 [PATCH v2 1/3] x86/percpu: Fix x86_this_cpu_variable_test_bit() asm template Uros Bizjak
2024-04-04 9:42 ` [PATCH v2 2/3] x86/percpu: Rewrite x86_this_cpu_test_bit() and friends as macros Uros Bizjak
2024-04-04 9:42 ` [PATCH v2 3/3] x86/percpu: Introduce raw_cpu_read_long() to reduce ifdeffery Uros Bizjak
@ 2024-04-06 11:07 ` tip-bot2 for Uros Bizjak
2 siblings, 0 replies; 6+ messages in thread
From: tip-bot2 for Uros Bizjak @ 2024-04-06 11:07 UTC (permalink / raw)
To: linux-tip-commits
Cc: Uros Bizjak, Ingo Molnar, Linus Torvalds, H. Peter Anvin, x86,
linux-kernel
The following commit has been merged into the x86/percpu branch of tip:
Commit-ID: 4c3677c077582f8665806def3f6dd35587793c69
Gitweb: https://git.kernel.org/tip/4c3677c077582f8665806def3f6dd35587793c69
Author: Uros Bizjak <ubizjak@gmail.com>
AuthorDate: Thu, 04 Apr 2024 11:42:01 +02:00
Committer: Ingo Molnar <mingo@kernel.org>
CommitterDate: Sat, 06 Apr 2024 12:42:17 +02:00
x86/percpu: Fix x86_this_cpu_variable_test_bit() asm template
Fix x86_this_cpu_variable_test_bit(), which is implemented with an
incorrect asm template, where argument 2 (count argument) is considered
a percpu variable. However, x86_this_cpu_test_bit() is currently
used exclusively with constant bit number argument, so the called
x86_this_cpu_variable_test_bit() function is never instantiated.
The fix introduces named assembler operands to prevent this kind
of error.
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Link: https://lore.kernel.org/r/20240404094218.448963-1-ubizjak@gmail.com
---
arch/x86/include/asm/percpu.h | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 20696df..cbfbbe8 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -586,10 +586,11 @@ static inline bool x86_this_cpu_variable_test_bit(int nr,
{
bool oldbit;
- asm volatile("btl "__percpu_arg(2)",%1"
+ asm volatile("btl %[nr], " __percpu_arg([var])
CC_SET(c)
: CC_OUT(c) (oldbit)
- : "m" (*__my_cpu_ptr((unsigned long __percpu *)(addr))), "Ir" (nr));
+ : [var] "m" (*__my_cpu_ptr((unsigned long __percpu *)(addr))),
+ [nr] "Ir" (nr));
return oldbit;
}
^ permalink raw reply [flat|nested] 6+ messages in thread
* [tip: x86/percpu] x86/percpu: Rewrite x86_this_cpu_test_bit() and friends as macros
2024-04-04 9:42 ` [PATCH v2 2/3] x86/percpu: Rewrite x86_this_cpu_test_bit() and friends as macros Uros Bizjak
@ 2024-04-06 11:07 ` tip-bot2 for Uros Bizjak
0 siblings, 0 replies; 6+ messages in thread
From: tip-bot2 for Uros Bizjak @ 2024-04-06 11:07 UTC (permalink / raw)
To: linux-tip-commits
Cc: Uros Bizjak, Ingo Molnar, Linus Torvalds, x86, linux-kernel
The following commit has been merged into the x86/percpu branch of tip:
Commit-ID: a3f8a3a2cf0b7b3ccb51ee60d51c0b5435c7135a
Gitweb: https://git.kernel.org/tip/a3f8a3a2cf0b7b3ccb51ee60d51c0b5435c7135a
Author: Uros Bizjak <ubizjak@gmail.com>
AuthorDate: Thu, 04 Apr 2024 11:42:02 +02:00
Committer: Ingo Molnar <mingo@kernel.org>
CommitterDate: Sat, 06 Apr 2024 12:42:17 +02:00
x86/percpu: Rewrite x86_this_cpu_test_bit() and friends as macros
Rewrite the whole family of x86_this_cpu_test_bit() functions
as macros, so standard __my_cpu_var() and raw_cpu_read() macros
can be used on percpu variables. This approach considerably
simplifies implementation of functions and also introduces
standard checks on accessed percpu variables.
No functional changes intended.
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: https://lore.kernel.org/r/20240404094218.448963-2-ubizjak@gmail.com
---
arch/um/include/asm/cpufeature.h | 3 +--
arch/x86/include/asm/cpufeature.h | 3 +--
arch/x86/include/asm/percpu.h | 54 ++++++++++++------------------
3 files changed, 25 insertions(+), 35 deletions(-)
diff --git a/arch/um/include/asm/cpufeature.h b/arch/um/include/asm/cpufeature.h
index 66fe06d..1eb8b83 100644
--- a/arch/um/include/asm/cpufeature.h
+++ b/arch/um/include/asm/cpufeature.h
@@ -38,8 +38,7 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
#define this_cpu_has(bit) \
(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 : \
- x86_this_cpu_test_bit(bit, \
- (unsigned long __percpu *)&cpu_info.x86_capability))
+ x86_this_cpu_test_bit(bit, cpu_info.x86_capability))
/*
* This macro is for detection of features which need kernel
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 42157dd..cd4c02d 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -127,8 +127,7 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
#define this_cpu_has(bit) \
(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 : \
- x86_this_cpu_test_bit(bit, \
- (unsigned long __percpu *)&cpu_info.x86_capability))
+ x86_this_cpu_test_bit(bit, cpu_info.x86_capability))
/*
* This macro is for detection of features which need kernel
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index cbfbbe8..d6ff0db 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -96,7 +96,7 @@
#endif /* CONFIG_SMP */
#define __my_cpu_type(var) typeof(var) __percpu_seg_override
-#define __my_cpu_ptr(ptr) (__my_cpu_type(*ptr)*)(__force uintptr_t)(ptr)
+#define __my_cpu_ptr(ptr) (__my_cpu_type(*(ptr))*)(__force uintptr_t)(ptr)
#define __my_cpu_var(var) (*__my_cpu_ptr(&(var)))
#define __percpu_arg(x) __percpu_prefix "%" #x
#define __force_percpu_arg(x) __force_percpu_prefix "%" #x
@@ -568,37 +568,29 @@ do { \
#define this_cpu_read_stable_8(pcp) ({ BUILD_BUG(); (typeof(pcp))0; })
#endif
-static __always_inline bool x86_this_cpu_constant_test_bit(unsigned int nr,
- const unsigned long __percpu *addr)
-{
- unsigned long __percpu *a =
- (unsigned long __percpu *)addr + nr / BITS_PER_LONG;
+#define x86_this_cpu_constant_test_bit(_nr, _var) \
+({ \
+ unsigned long __percpu *addr__ = \
+ (unsigned long __percpu *)&(_var) + ((_nr) / BITS_PER_LONG); \
+ !!((1UL << ((_nr) % BITS_PER_LONG)) & raw_cpu_read(*addr__)); \
+})
-#ifdef CONFIG_X86_64
- return ((1UL << (nr % BITS_PER_LONG)) & raw_cpu_read_8(*a)) != 0;
-#else
- return ((1UL << (nr % BITS_PER_LONG)) & raw_cpu_read_4(*a)) != 0;
-#endif
-}
-
-static inline bool x86_this_cpu_variable_test_bit(int nr,
- const unsigned long __percpu *addr)
-{
- bool oldbit;
-
- asm volatile("btl %[nr], " __percpu_arg([var])
- CC_SET(c)
- : CC_OUT(c) (oldbit)
- : [var] "m" (*__my_cpu_ptr((unsigned long __percpu *)(addr))),
- [nr] "Ir" (nr));
-
- return oldbit;
-}
-
-#define x86_this_cpu_test_bit(nr, addr) \
- (__builtin_constant_p((nr)) \
- ? x86_this_cpu_constant_test_bit((nr), (addr)) \
- : x86_this_cpu_variable_test_bit((nr), (addr)))
+#define x86_this_cpu_variable_test_bit(_nr, _var) \
+({ \
+ bool oldbit; \
+ \
+ asm volatile("btl %[nr], " __percpu_arg([var]) \
+ CC_SET(c) \
+ : CC_OUT(c) (oldbit) \
+ : [var] "m" (__my_cpu_var(_var)), \
+ [nr] "rI" (_nr)); \
+ oldbit; \
+})
+
+#define x86_this_cpu_test_bit(_nr, _var) \
+ (__builtin_constant_p(_nr) \
+ ? x86_this_cpu_constant_test_bit(_nr, _var) \
+ : x86_this_cpu_variable_test_bit(_nr, _var))
#include <asm-generic/percpu.h>
^ permalink raw reply [flat|nested] 6+ messages in thread
end of thread, other threads:[~2024-04-06 11:08 UTC | newest]
Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2024-04-04 9:42 [PATCH v2 1/3] x86/percpu: Fix x86_this_cpu_variable_test_bit() asm template Uros Bizjak
2024-04-04 9:42 ` [PATCH v2 2/3] x86/percpu: Rewrite x86_this_cpu_test_bit() and friends as macros Uros Bizjak
2024-04-06 11:07 ` [tip: x86/percpu] " tip-bot2 for Uros Bizjak
2024-04-04 9:42 ` [PATCH v2 3/3] x86/percpu: Introduce raw_cpu_read_long() to reduce ifdeffery Uros Bizjak
2024-04-06 11:07 ` [tip: x86/percpu] " tip-bot2 for Uros Bizjak
2024-04-06 11:07 ` [tip: x86/percpu] x86/percpu: Fix x86_this_cpu_variable_test_bit() asm template tip-bot2 for Uros Bizjak
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®