From: Michel Lespinasse <walken@google.com>
To: Linus Torvalds <torvalds@linux-foundation.org>,
David Howells <dhowells@redhat.com>, Ingo Molnar <mingo@elte.hu>,
Thomas Gleixner <tglx@linutronix.de>
Cc: LKML <linux-kernel@vger.kernel.org>,
Andrew Morton <akpm@linux-foundation.org>,
Mike Waychison <mikew@google.com>,
Suleiman Souhlal <suleiman@google.com>,
Ying Han <yinghan@google.com>,
Michel Lespinasse <walken@google.com>
Subject: [PATCH 05/12] x86 rwsem: take advantage of new RWSEM_WAITING_BIAS semantics
Date: Tue, 11 May 2010 20:20:55 -0700 [thread overview]
Message-ID: <1273634462-2672-6-git-send-email-walken@google.com> (raw)
In-Reply-To: <1273634462-2672-1-git-send-email-walken@google.com>
Now that an arbitrary number of waiting threads are represented by
a single RWSEM_WAITING_BIAS, the up_read and up_write code paths can be
slightly simplified by checking for waiting threads and no active threads
all at once.
Signed-off-by: Michel Lespinasse <walken@google.com>
---
arch/x86/include/asm/rwsem.h | 23 ++++++++++++++---------
arch/x86/lib/rwsem_64.S | 4 +---
arch/x86/lib/semaphore_32.S | 4 +---
3 files changed, 16 insertions(+), 15 deletions(-)
diff --git a/arch/x86/include/asm/rwsem.h b/arch/x86/include/asm/rwsem.h
index 606ede1..a15b84d 100644
--- a/arch/x86/include/asm/rwsem.h
+++ b/arch/x86/include/asm/rwsem.h
@@ -118,7 +118,7 @@ static inline void __down_read(struct rw_semaphore *sem)
{
asm volatile("# beginning down_read\n\t"
LOCK_PREFIX _ASM_INC "(%1)\n\t"
- /* adds 0x00000001, returns the old value */
+ /* adds 0x00000001 */
" jns 1f\n"
" call call_rwsem_down_read_failed\n"
"1:\n\t"
@@ -160,7 +160,7 @@ static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
tmp = RWSEM_ACTIVE_WRITE_BIAS;
asm volatile("# beginning down_write\n\t"
LOCK_PREFIX " xadd %1,(%2)\n\t"
- /* subtract 0x0000ffff, returns the old value */
+ /* adds 0xffff0001, returns the old value */
" test %1,%1\n\t"
/* was the count 0 before? */
" jz 1f\n"
@@ -199,12 +199,15 @@ static inline void __up_read(struct rw_semaphore *sem)
asm volatile("# beginning __up_read\n\t"
LOCK_PREFIX " xadd %1,(%2)\n\t"
/* subtracts 1, returns the old value */
- " jns 1f\n\t"
+ " cmp %4,%1\n\t"
+ /* are there waiting threads and no active threads ? */
+ " jne 1f\n"
" call call_rwsem_wake\n"
"1:\n"
"# ending __up_read\n"
: "+m" (sem->count), "=d" (tmp)
- : "a" (sem), "1" (tmp)
+ : "a" (sem), "1" (tmp),
+ "er" (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_READ_BIAS)
: "memory", "cc");
}
@@ -213,17 +216,19 @@ static inline void __up_read(struct rw_semaphore *sem)
*/
static inline void __up_write(struct rw_semaphore *sem)
{
- rwsem_count_t tmp;
+ rwsem_count_t tmp = -RWSEM_ACTIVE_WRITE_BIAS;
asm volatile("# beginning __up_write\n\t"
LOCK_PREFIX " xadd %1,(%2)\n\t"
- /* tries to transition
- 0xffff0001 -> 0x00000000 */
- " jz 1f\n"
+ /* substracts 0xffff0001, returns the old value */
+ " cmp %4,%1\n\t"
+ /* are there waiting threads and no active threads ? */
+ " jne 1f\n"
" call call_rwsem_wake\n"
"1:\n\t"
"# ending __up_write\n"
: "+m" (sem->count), "=d" (tmp)
- : "a" (sem), "1" (-RWSEM_ACTIVE_WRITE_BIAS)
+ : "a" (sem), "1" (tmp),
+ "er" (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_WRITE_BIAS)
: "memory", "cc");
}
diff --git a/arch/x86/lib/rwsem_64.S b/arch/x86/lib/rwsem_64.S
index 41fcf00..770a387 100644
--- a/arch/x86/lib/rwsem_64.S
+++ b/arch/x86/lib/rwsem_64.S
@@ -60,13 +60,11 @@ ENTRY(call_rwsem_down_write_failed)
ENDPROC(call_rwsem_down_write_failed)
ENTRY(call_rwsem_wake)
- decl %edx /* do nothing if still outstanding active readers */
- jnz 1f
save_common_regs
movq %rax,%rdi
call rwsem_wake
restore_common_regs
-1: ret
+ ret
ENDPROC(call_rwsem_wake)
/* Fix up special calling conventions */
diff --git a/arch/x86/lib/semaphore_32.S b/arch/x86/lib/semaphore_32.S
index 648fe47..63dbf75 100644
--- a/arch/x86/lib/semaphore_32.S
+++ b/arch/x86/lib/semaphore_32.S
@@ -103,15 +103,13 @@ ENTRY(call_rwsem_down_write_failed)
ENTRY(call_rwsem_wake)
CFI_STARTPROC
- decw %dx /* do nothing if still outstanding active readers */
- jnz 1f
push %ecx
CFI_ADJUST_CFA_OFFSET 4
CFI_REL_OFFSET ecx,0
call rwsem_wake
pop %ecx
CFI_ADJUST_CFA_OFFSET -4
-1: ret
+ ret
CFI_ENDPROC
ENDPROC(call_rwsem_wake)
--
1.7.0.1
next prev parent reply other threads:[~2010-05-12 3:23 UTC|newest]
Thread overview: 33+ messages / expand[flat|nested] mbox.gz Atom feed top
2010-05-12 3:20 [PATCH 00/12] rwsem changes + down_read_unfair() proposal Michel Lespinasse
2010-05-12 3:20 ` [PATCH 01/12] rwsem: test for no active locks in __rwsem_do_wake undo code Michel Lespinasse
2010-05-12 3:20 ` [PATCH 02/12] rwsem: use single atomic update for sem count when waking up readers Michel Lespinasse
2010-05-12 3:20 ` [PATCH 03/12] rwsem: let RWSEM_WAITING_BIAS represent any number of waiting threads Michel Lespinasse
2010-05-12 3:20 ` [PATCH 04/12] rwsem: consistently use adjustment variable Michel Lespinasse
2010-05-12 3:20 ` Michel Lespinasse [this message]
2010-05-12 3:20 ` [PATCH 06/12] rwsem: wake queued readers when other readers are active Michel Lespinasse
2010-05-12 3:20 ` [PATCH 07/12] rwsem: wake queued readers when writer blocks on active read lock Michel Lespinasse
2010-05-12 3:20 ` [PATCH 08/12] rwsem: smaller wrappers around rwsem_down_failed_common Michel Lespinasse
2010-05-12 3:20 ` [PATCH 09/12] generic rwsem: implement down_read_unfair Michel Lespinasse
2010-05-12 3:21 ` [PATCH 10/12] rwsem: down_read_unfair infrastructure support Michel Lespinasse
2010-05-12 3:21 ` [PATCH 11/12] x86 rwsem: down_read_unfair implementation Michel Lespinasse
2010-05-12 3:21 ` [PATCH 12/12] Use down_read_unfair() for /sys/<pid>/exe and /sys/<pid>/maps files Michel Lespinasse
2010-05-12 22:53 ` KOSAKI Motohiro
2010-05-12 23:35 ` Michel Lespinasse
2010-05-13 0:32 ` KOSAKI Motohiro
2010-05-12 10:39 ` [PATCH 01/12] rwsem: test for no active locks in __rwsem_do_wake undo code David Howells
2010-05-12 11:01 ` [PATCH 02/12] rwsem: use single atomic update for sem count when waking up readers David Howells
2010-05-13 0:54 ` Michel Lespinasse
2010-05-12 11:36 ` David Howells
2010-05-12 11:45 ` [PATCH 04/12] rwsem: consistently use adjustment variable David Howells
2010-05-13 1:12 ` Michel Lespinasse
2010-05-12 12:10 ` [PATCH 05/12] x86 rwsem: take advantage of new RWSEM_WAITING_BIAS semantics David Howells
2010-05-12 12:22 ` [PATCH 06/12] rwsem: wake queued readers when other readers are active David Howells
2010-05-13 2:39 ` Michel Lespinasse
2010-05-13 5:41 ` Michel Lespinasse
2010-05-12 12:33 ` [PATCH 07/12] rwsem: wake queued readers when writer blocks on active read lock David Howells
2010-05-12 12:36 ` [PATCH 08/12] rwsem: smaller wrappers around rwsem_down_failed_common David Howells
2010-05-12 12:42 ` David Howells
2010-05-13 2:54 ` Michel Lespinasse
2010-05-12 12:46 ` [PATCH 09/12] generic rwsem: implement down_read_unfair David Howells
2010-05-12 13:08 ` [PATCH 11/12] x86 rwsem: down_read_unfair implementation David Howells
2010-05-12 13:10 ` [PATCH 12/12] Use down_read_unfair() for /sys/<pid>/exe and /sys/<pid>/maps files David Howells
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1273634462-2672-6-git-send-email-walken@google.com \
--to=walken@google.com \
--cc=akpm@linux-foundation.org \
--cc=dhowells@redhat.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mikew@google.com \
--cc=mingo@elte.hu \
--cc=suleiman@google.com \
--cc=tglx@linutronix.de \
--cc=torvalds@linux-foundation.org \
--cc=yinghan@google.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®