[PATCH 19/29] x86, tsx: Add support for rwsem elision

[Andi Kleen <andi@firstfloor.org>]

From: Andi Kleen <ak@linux.intel.com>

Add the standard elide wrapper macros to rwsems to enable
lock elision for rwsems. Main target is mmap_sem.

Signed-off-by: Andi Kleen <ak@linux.intel.com>
---
 arch/x86/include/asm/rwsem.h |   23 ++++++++++++++++++++++-
 arch/x86/kernel/rtm-locks.c  |    3 +++
 include/linux/rwsem.h        |    3 +++
 3 files changed, 28 insertions(+), 1 deletions(-)

diff --git a/arch/x86/include/asm/rwsem.h b/arch/x86/include/asm/rwsem.h
index 2dbe4a7..f9297af 100644
--- a/arch/x86/include/asm/rwsem.h
+++ b/arch/x86/include/asm/rwsem.h
@@ -38,6 +38,7 @@
 
 #ifdef __KERNEL__
 #include <asm/asm.h>
+#include <linux/elide.h>
 
 /*
  * The bias values and the counter type limits the number of
@@ -57,11 +58,15 @@
 #define RWSEM_ACTIVE_READ_BIAS		RWSEM_ACTIVE_BIAS
 #define RWSEM_ACTIVE_WRITE_BIAS		(RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
 
+extern bool rwsem_elision;
+
 /*
  * lock for reading
  */
 static inline void __down_read(struct rw_semaphore *sem)
 {
+	if (elide_lock(rwsem_elision, sem->count == 0))
+		return;
 	asm volatile("# beginning down_read\n\t"
 		     LOCK_PREFIX _ASM_INC "(%1)\n\t"
 		     /* adds 0x00000001 */
@@ -80,6 +85,9 @@ static inline void __down_read(struct rw_semaphore *sem)
 static inline int __down_read_trylock(struct rw_semaphore *sem)
 {
 	long result, tmp;
+
+	if (elide_lock(rwsem_elision, sem->count == 0))
+		return 1;
 	asm volatile("# beginning __down_read_trylock\n\t"
 		     "  mov          %0,%1\n\t"
 		     "1:\n\t"
@@ -102,6 +110,9 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)
 static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
 {
 	long tmp;
+
+	if (elide_lock(rwsem_elision, sem->count == 0))
+		return;
 	asm volatile("# beginning down_write\n\t"
 		     LOCK_PREFIX "  xadd      %1,(%2)\n\t"
 		     /* adds 0xffff0001, returns the old value */
@@ -126,7 +137,11 @@ static inline void __down_write(struct rw_semaphore *sem)
  */
 static inline int __down_write_trylock(struct rw_semaphore *sem)
 {
-	long ret = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE,
+	long ret;
+
+	if (elide_lock(rwsem_elision, sem->count == 0))
+		return 1;
+	ret = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE,
 			   RWSEM_ACTIVE_WRITE_BIAS);
 	if (ret == RWSEM_UNLOCKED_VALUE)
 		return 1;
@@ -139,6 +154,9 @@ static inline int __down_write_trylock(struct rw_semaphore *sem)
 static inline void __up_read(struct rw_semaphore *sem)
 {
 	long tmp;
+
+	if (elide_unlock(sem->count == 0))
+		return;
 	asm volatile("# beginning __up_read\n\t"
 		     LOCK_PREFIX "  xadd      %1,(%2)\n\t"
 		     /* subtracts 1, returns the old value */
@@ -157,6 +175,9 @@ static inline void __up_read(struct rw_semaphore *sem)
 static inline void __up_write(struct rw_semaphore *sem)
 {
 	long tmp;
+
+	if (elide_unlock(sem->count == 0))
+		return;
 	asm volatile("# beginning __up_write\n\t"
 		     LOCK_PREFIX "  xadd      %1,(%2)\n\t"
 		     /* subtracts 0xffff0001, returns the old value */
diff --git a/arch/x86/kernel/rtm-locks.c b/arch/x86/kernel/rtm-locks.c
index f3ae8e6..1811028 100644
--- a/arch/x86/kernel/rtm-locks.c
+++ b/arch/x86/kernel/rtm-locks.c
@@ -351,3 +351,6 @@ void init_rtm_spinlocks(void)
 
 __read_mostly bool mutex_elision = true;
 module_param(mutex_elision, bool, 0644);
+
+__read_mostly bool rwsem_elision = true;
+module_param(rwsem_elision, bool, 0644);
diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h
index 8da67d6..436e430 100644
--- a/include/linux/rwsem.h
+++ b/include/linux/rwsem.h
@@ -16,6 +16,8 @@
 
 #include <linux/atomic.h>
 
+#include <linux/elide.h>
+
 struct rw_semaphore;
 
 #ifdef CONFIG_RWSEM_GENERIC_SPINLOCK
@@ -42,6 +44,7 @@ extern struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem);
 /* In all implementations count != 0 means locked */
 static inline int rwsem_is_locked(struct rw_semaphore *sem)
 {
+	elide_abort();
 	return sem->count != 0;
 }
 
-- 
1.7.7.6