From: Paul Burton <paul.burton@mips.com>
To: "linux-mips@vger.kernel.org" <linux-mips@vger.kernel.org>
Cc: Huacai Chen <chenhc@lemote.com>,
Jiaxun Yang <jiaxun.yang@flygoat.com>,
"linux-kernel@vger.kernel.org" <linux-kernel@vger.kernel.org>,
Paul Burton <pburton@wavecomp.com>
Subject: [PATCH v2 16/36] MIPS: bitops: Handle !kernel_uses_llsc first
Date: Tue, 1 Oct 2019 21:53:25 +0000 [thread overview]
Message-ID: <20191001215249.4157062-17-paul.burton@mips.com> (raw)
In-Reply-To: <20191001215249.4157062-1-paul.burton@mips.com>
Reorder conditions in our various bitops functions that check
kernel_uses_llsc such that they handle the !kernel_uses_llsc case first.
This allows us to avoid the need to duplicate the kernel_uses_llsc check
in all the other cases. For functions that don't involve barriers common
to the various implementations, we switch to returning from within each
if block making each case easier to read in isolation.
Signed-off-by: Paul Burton <paul.burton@mips.com>
---
Changes in v2: None
arch/mips/include/asm/bitops.h | 213 ++++++++++++++++-----------------
1 file changed, 105 insertions(+), 108 deletions(-)
diff --git a/arch/mips/include/asm/bitops.h b/arch/mips/include/asm/bitops.h
index 985d6a02f9ea..e300960717e0 100644
--- a/arch/mips/include/asm/bitops.h
+++ b/arch/mips/include/asm/bitops.h
@@ -52,11 +52,16 @@ int __mips_test_and_change_bit(unsigned long nr,
*/
static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
{
- unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
+ unsigned long *m = ((unsigned long *)addr) + (nr >> SZLONG_LOG);
int bit = nr & SZLONG_MASK;
unsigned long temp;
- if (kernel_uses_llsc && R10000_LLSC_WAR) {
+ if (!kernel_uses_llsc) {
+ __mips_set_bit(nr, addr);
+ return;
+ }
+
+ if (R10000_LLSC_WAR) {
__asm__ __volatile__(
" .set push \n"
" .set arch=r4000 \n"
@@ -68,8 +73,11 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
: "=&r" (temp), "=" GCC_OFF_SMALL_ASM() (*m)
: "ir" (1UL << bit), GCC_OFF_SMALL_ASM() (*m)
: __LLSC_CLOBBER);
+ return;
+ }
+
#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
- } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
+ if (__builtin_constant_p(bit)) {
loongson_llsc_mb();
do {
__asm__ __volatile__(
@@ -80,23 +88,23 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
: "ir" (bit), "r" (~0)
: __LLSC_CLOBBER);
} while (unlikely(!temp));
+ return;
+ }
#endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */
- } else if (kernel_uses_llsc) {
- loongson_llsc_mb();
- do {
- __asm__ __volatile__(
- " .set push \n"
- " .set "MIPS_ISA_ARCH_LEVEL" \n"
- " " __LL "%0, %1 # set_bit \n"
- " or %0, %2 \n"
- " " __SC "%0, %1 \n"
- " .set pop \n"
- : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
- : "ir" (1UL << bit)
- : __LLSC_CLOBBER);
- } while (unlikely(!temp));
- } else
- __mips_set_bit(nr, addr);
+
+ loongson_llsc_mb();
+ do {
+ __asm__ __volatile__(
+ " .set push \n"
+ " .set "MIPS_ISA_ARCH_LEVEL" \n"
+ " " __LL "%0, %1 # set_bit \n"
+ " or %0, %2 \n"
+ " " __SC "%0, %1 \n"
+ " .set pop \n"
+ : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
+ : "ir" (1UL << bit)
+ : __LLSC_CLOBBER);
+ } while (unlikely(!temp));
}
/*
@@ -111,11 +119,16 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
*/
static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
{
- unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
+ unsigned long *m = ((unsigned long *)addr) + (nr >> SZLONG_LOG);
int bit = nr & SZLONG_MASK;
unsigned long temp;
- if (kernel_uses_llsc && R10000_LLSC_WAR) {
+ if (!kernel_uses_llsc) {
+ __mips_clear_bit(nr, addr);
+ return;
+ }
+
+ if (R10000_LLSC_WAR) {
__asm__ __volatile__(
" .set push \n"
" .set arch=r4000 \n"
@@ -127,8 +140,11 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
: "ir" (~(1UL << bit))
: __LLSC_CLOBBER);
+ return;
+ }
+
#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
- } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
+ if (__builtin_constant_p(bit)) {
loongson_llsc_mb();
do {
__asm__ __volatile__(
@@ -139,23 +155,23 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
: "ir" (bit)
: __LLSC_CLOBBER);
} while (unlikely(!temp));
+ return;
+ }
#endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */
- } else if (kernel_uses_llsc) {
- loongson_llsc_mb();
- do {
- __asm__ __volatile__(
- " .set push \n"
- " .set "MIPS_ISA_ARCH_LEVEL" \n"
- " " __LL "%0, %1 # clear_bit \n"
- " and %0, %2 \n"
- " " __SC "%0, %1 \n"
- " .set pop \n"
- : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
- : "ir" (~(1UL << bit))
- : __LLSC_CLOBBER);
- } while (unlikely(!temp));
- } else
- __mips_clear_bit(nr, addr);
+
+ loongson_llsc_mb();
+ do {
+ __asm__ __volatile__(
+ " .set push \n"
+ " .set "MIPS_ISA_ARCH_LEVEL" \n"
+ " " __LL "%0, %1 # clear_bit \n"
+ " and %0, %2 \n"
+ " " __SC "%0, %1 \n"
+ " .set pop \n"
+ : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
+ : "ir" (~(1UL << bit))
+ : __LLSC_CLOBBER);
+ } while (unlikely(!temp));
}
/*
@@ -183,12 +199,16 @@ static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *ad
*/
static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
{
+ unsigned long *m = ((unsigned long *)addr) + (nr >> SZLONG_LOG);
int bit = nr & SZLONG_MASK;
+ unsigned long temp;
- if (kernel_uses_llsc && R10000_LLSC_WAR) {
- unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
- unsigned long temp;
+ if (!kernel_uses_llsc) {
+ __mips_change_bit(nr, addr);
+ return;
+ }
+ if (R10000_LLSC_WAR) {
__asm__ __volatile__(
" .set push \n"
" .set arch=r4000 \n"
@@ -200,25 +220,22 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
: "ir" (1UL << bit)
: __LLSC_CLOBBER);
- } else if (kernel_uses_llsc) {
- unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
- unsigned long temp;
+ return;
+ }
- loongson_llsc_mb();
- do {
- __asm__ __volatile__(
- " .set push \n"
- " .set "MIPS_ISA_ARCH_LEVEL" \n"
- " " __LL "%0, %1 # change_bit \n"
- " xor %0, %2 \n"
- " " __SC "%0, %1 \n"
- " .set pop \n"
- : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
- : "ir" (1UL << bit)
- : __LLSC_CLOBBER);
- } while (unlikely(!temp));
- } else
- __mips_change_bit(nr, addr);
+ loongson_llsc_mb();
+ do {
+ __asm__ __volatile__(
+ " .set push \n"
+ " .set "MIPS_ISA_ARCH_LEVEL" \n"
+ " " __LL "%0, %1 # change_bit \n"
+ " xor %0, %2 \n"
+ " " __SC "%0, %1 \n"
+ " .set pop \n"
+ : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
+ : "ir" (1UL << bit)
+ : __LLSC_CLOBBER);
+ } while (unlikely(!temp));
}
/*
@@ -232,15 +249,15 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
static inline int test_and_set_bit(unsigned long nr,
volatile unsigned long *addr)
{
+ unsigned long *m = ((unsigned long *)addr) + (nr >> SZLONG_LOG);
int bit = nr & SZLONG_MASK;
- unsigned long res;
+ unsigned long res, temp;
smp_mb__before_llsc();
- if (kernel_uses_llsc && R10000_LLSC_WAR) {
- unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
- unsigned long temp;
-
+ if (!kernel_uses_llsc) {
+ res = __mips_test_and_set_bit(nr, addr);
+ } else if (R10000_LLSC_WAR) {
__asm__ __volatile__(
" .set push \n"
" .set arch=r4000 \n"
@@ -253,10 +270,7 @@ static inline int test_and_set_bit(unsigned long nr,
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
: "r" (1UL << bit)
: __LLSC_CLOBBER);
- } else if (kernel_uses_llsc) {
- unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
- unsigned long temp;
-
+ } else {
loongson_llsc_mb();
do {
__asm__ __volatile__(
@@ -272,8 +286,7 @@ static inline int test_and_set_bit(unsigned long nr,
} while (unlikely(!res));
res = temp & (1UL << bit);
- } else
- res = __mips_test_and_set_bit(nr, addr);
+ }
smp_llsc_mb();
@@ -291,13 +304,13 @@ static inline int test_and_set_bit(unsigned long nr,
static inline int test_and_set_bit_lock(unsigned long nr,
volatile unsigned long *addr)
{
+ unsigned long *m = ((unsigned long *)addr) + (nr >> SZLONG_LOG);
int bit = nr & SZLONG_MASK;
- unsigned long res;
-
- if (kernel_uses_llsc && R10000_LLSC_WAR) {
- unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
- unsigned long temp;
+ unsigned long res, temp;
+ if (!kernel_uses_llsc) {
+ res = __mips_test_and_set_bit_lock(nr, addr);
+ } else if (R10000_LLSC_WAR) {
__asm__ __volatile__(
" .set push \n"
" .set arch=r4000 \n"
@@ -310,11 +323,7 @@ static inline int test_and_set_bit_lock(unsigned long nr,
: "=&r" (temp), "+m" (*m), "=&r" (res)
: "r" (1UL << bit)
: __LLSC_CLOBBER);
- } else if (kernel_uses_llsc) {
- unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
- unsigned long temp;
-
- loongson_llsc_mb();
+ } else {
do {
__asm__ __volatile__(
" .set push \n"
@@ -329,8 +338,7 @@ static inline int test_and_set_bit_lock(unsigned long nr,
} while (unlikely(!res));
res = temp & (1UL << bit);
- } else
- res = __mips_test_and_set_bit_lock(nr, addr);
+ }
smp_llsc_mb();
@@ -347,15 +355,15 @@ static inline int test_and_set_bit_lock(unsigned long nr,
static inline int test_and_clear_bit(unsigned long nr,
volatile unsigned long *addr)
{
+ unsigned long *m = ((unsigned long *)addr) + (nr >> SZLONG_LOG);
int bit = nr & SZLONG_MASK;
- unsigned long res;
+ unsigned long res, temp;
smp_mb__before_llsc();
- if (kernel_uses_llsc && R10000_LLSC_WAR) {
- unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
- unsigned long temp;
-
+ if (!kernel_uses_llsc) {
+ res = __mips_test_and_clear_bit(nr, addr);
+ } else if (R10000_LLSC_WAR) {
__asm__ __volatile__(
" .set push \n"
" .set arch=r4000 \n"
@@ -370,10 +378,7 @@ static inline int test_and_clear_bit(unsigned long nr,
: "r" (1UL << bit)
: __LLSC_CLOBBER);
#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
- } else if (kernel_uses_llsc && __builtin_constant_p(nr)) {
- unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
- unsigned long temp;
-
+ } else if (__builtin_constant_p(nr)) {
loongson_llsc_mb();
do {
__asm__ __volatile__(
@@ -386,10 +391,7 @@ static inline int test_and_clear_bit(unsigned long nr,
: __LLSC_CLOBBER);
} while (unlikely(!temp));
#endif
- } else if (kernel_uses_llsc) {
- unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
- unsigned long temp;
-
+ } else {
loongson_llsc_mb();
do {
__asm__ __volatile__(
@@ -406,8 +408,7 @@ static inline int test_and_clear_bit(unsigned long nr,
} while (unlikely(!res));
res = temp & (1UL << bit);
- } else
- res = __mips_test_and_clear_bit(nr, addr);
+ }
smp_llsc_mb();
@@ -425,15 +426,15 @@ static inline int test_and_clear_bit(unsigned long nr,
static inline int test_and_change_bit(unsigned long nr,
volatile unsigned long *addr)
{
+ unsigned long *m = ((unsigned long *)addr) + (nr >> SZLONG_LOG);
int bit = nr & SZLONG_MASK;
- unsigned long res;
+ unsigned long res, temp;
smp_mb__before_llsc();
- if (kernel_uses_llsc && R10000_LLSC_WAR) {
- unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
- unsigned long temp;
-
+ if (!kernel_uses_llsc) {
+ res = __mips_test_and_change_bit(nr, addr);
+ } else if (R10000_LLSC_WAR) {
__asm__ __volatile__(
" .set push \n"
" .set arch=r4000 \n"
@@ -446,10 +447,7 @@ static inline int test_and_change_bit(unsigned long nr,
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
: "r" (1UL << bit)
: __LLSC_CLOBBER);
- } else if (kernel_uses_llsc) {
- unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
- unsigned long temp;
-
+ } else {
loongson_llsc_mb();
do {
__asm__ __volatile__(
@@ -465,8 +463,7 @@ static inline int test_and_change_bit(unsigned long nr,
} while (unlikely(!res));
res = temp & (1UL << bit);
- } else
- res = __mips_test_and_change_bit(nr, addr);
+ }
smp_llsc_mb();
--
2.23.0
next prev parent reply other threads:[~2019-10-01 21:54 UTC|newest]
Thread overview: 38+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-10-01 21:53 [PATCH v2 00/36] MIPS: barriers & atomics cleanups Paul Burton
2019-10-01 21:53 ` [PATCH v2 01/36] MIPS: Unify sc beqz definition Paul Burton
2019-10-01 21:53 ` [PATCH v2 02/36] MIPS: Use compact branch for LL/SC loops on MIPSr6+ Paul Burton
2019-10-01 21:53 ` [PATCH v2 03/36] MIPS: barrier: Add __SYNC() infrastructure Paul Burton
2019-10-01 21:53 ` [PATCH v2 04/36] MIPS: barrier: Clean up rmb() & wmb() definitions Paul Burton
2019-10-01 21:53 ` [PATCH v2 05/36] MIPS: barrier: Clean up __smp_mb() definition Paul Burton
2019-10-01 21:53 ` [PATCH v2 06/36] MIPS: barrier: Remove fast_mb() Octeon #ifdef'ery Paul Burton
2019-10-01 21:53 ` [PATCH v2 07/36] MIPS: barrier: Clean up __sync() definition Paul Burton
2019-10-01 21:53 ` [PATCH v2 08/36] MIPS: barrier: Clean up sync_ginv() Paul Burton
2019-10-01 21:53 ` [PATCH v2 09/36] MIPS: atomic: Fix whitespace in ATOMIC_OP macros Paul Burton
2019-10-01 21:53 ` [PATCH v2 10/36] MIPS: atomic: Handle !kernel_uses_llsc first Paul Burton
2019-10-01 21:53 ` [PATCH v2 11/36] MIPS: atomic: Use one macro to generate 32b & 64b functions Paul Burton
2019-10-01 21:53 ` [PATCH v2 12/36] MIPS: atomic: Emit Loongson3 sync workarounds within asm Paul Burton
2019-10-01 21:53 ` [PATCH v2 13/36] MIPS: atomic: Use _atomic barriers in atomic_sub_if_positive() Paul Burton
2019-10-01 21:53 ` [PATCH v2 14/36] MIPS: atomic: Unify 32b & 64b sub_if_positive Paul Burton
2019-10-01 21:53 ` [PATCH v2 15/36] MIPS: atomic: Deduplicate 32b & 64b read, set, xchg, cmpxchg Paul Burton
2019-10-01 21:53 ` Paul Burton [this message]
2019-10-01 21:53 ` [PATCH v2 17/36] MIPS: bitops: Only use ins for bit 16 or higher Paul Burton
2019-10-01 21:53 ` [PATCH v2 18/36] MIPS: bitops: Use MIPS_ISA_REV, not #ifdefs Paul Burton
2019-10-01 21:53 ` [PATCH v2 19/36] MIPS: bitops: ins start position is always an immediate Paul Burton
2019-10-01 21:53 ` [PATCH v2 20/36] MIPS: bitops: Implement test_and_set_bit() in terms of _lock variant Paul Burton
2019-10-01 21:53 ` [PATCH v2 22/36] MIPS: bitops: Use the BIT() macro Paul Burton
2019-10-01 21:53 ` [PATCH v2 21/36] MIPS: bitops: Allow immediates in test_and_{set,clear,change}_bit Paul Burton
2019-10-01 21:53 ` [PATCH v2 23/36] MIPS: bitops: Avoid redundant zero-comparison for non-LLSC Paul Burton
2019-10-01 21:53 ` [PATCH v2 24/36] MIPS: bitops: Abstract LL/SC loops Paul Burton
2019-10-01 21:53 ` [PATCH v2 25/36] MIPS: bitops: Use BIT_WORD() & BITS_PER_LONG Paul Burton
2019-10-01 21:53 ` [PATCH v2 26/36] MIPS: bitops: Emit Loongson3 sync workarounds within asm Paul Burton
2019-10-01 21:53 ` [PATCH v2 27/36] MIPS: bitops: Use smp_mb__before_atomic in test_* ops Paul Burton
2019-10-01 21:53 ` [PATCH v2 28/36] MIPS: cmpxchg: Emit Loongson3 sync workarounds within asm Paul Burton
2019-10-01 21:53 ` [PATCH v2 30/36] MIPS: futex: " Paul Burton
2019-10-01 21:53 ` [PATCH v2 29/36] MIPS: cmpxchg: Omit redundant barriers for Loongson3 Paul Burton
2019-10-01 21:53 ` [PATCH v2 31/36] MIPS: syscall: Emit Loongson3 sync workarounds within asm Paul Burton
2019-10-01 21:53 ` [PATCH v2 32/36] MIPS: barrier: Remove loongson_llsc_mb() Paul Burton
2019-10-01 21:53 ` [PATCH v2 33/36] MIPS: barrier: Make __smp_mb__before_atomic() a no-op for Loongson3 Paul Burton
2019-10-01 21:53 ` [PATCH v2 34/36] MIPS: genex: Add Loongson3 LL/SC workaround to ejtag_debug_handler Paul Burton
2019-10-01 21:53 ` [PATCH v2 35/36] MIPS: genex: Don't reload address unnecessarily Paul Burton
2019-10-01 21:53 ` [PATCH v2 36/36] MIPS: Check Loongson3 LL/SC errata workaround correctness Paul Burton
2019-10-07 17:51 ` [PATCH v2 00/36] MIPS: barriers & atomics cleanups Paul Burton
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20191001215249.4157062-17-paul.burton@mips.com \
--to=paul.burton@mips.com \
--cc=chenhc@lemote.com \
--cc=jiaxun.yang@flygoat.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mips@vger.kernel.org \
--cc=pburton@wavecomp.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®