From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1757623AbcHWMvT (ORCPT ); Tue, 23 Aug 2016 08:51:19 -0400 Received: from bombadil.infradead.org ([198.137.202.9]:60527 "EHLO bombadil.infradead.org" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1753992AbcHWMvN (ORCPT ); Tue, 23 Aug 2016 08:51:13 -0400 Message-Id: <20160823124856.898171453@infradead.org> User-Agent: quilt/0.63-1 Date: Tue, 23 Aug 2016 14:46:20 +0200 From: Peter Zijlstra To: Linus Torvalds , Waiman Long , Jason Low , Ding Tianhong , Thomas Gleixner , Will Deacon , Ingo Molnar , Imre Deak , Linux Kernel Mailing List , Davidlohr Bueso , Tim Chen , Terry Rudd , "Paul E. McKenney" , Jason Low , Peter Zijlstra Subject: [RFC][PATCH 3/3] locking/mutex: Add lock handoff to avoid starvation References: <20160823124617.015645861@infradead.org> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline; filename=peterz-locking-mutex-steal.patch Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Now that we have an atomic owner field, we can do explicit lock handoff. Use this to avoid starvation. Signed-off-by: Peter Zijlstra (Intel) --- kernel/locking/mutex.c | 44 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -53,6 +53,7 @@ __mutex_init(struct mutex *lock, const c EXPORT_SYMBOL(__mutex_init); #define MUTEX_FLAG_WAITERS 0x01 +#define MUTEX_FLAG_HANDOFF 0x02 #define MUTEX_FLAG_ALL 0x03 @@ -84,6 +85,29 @@ static inline void __mutex_clear_flag(st atomic_long_andnot(flag, &lock->owner); } +static inline bool __mutex_waiter_is_first(struct mutex *lock, struct mutex_waiter *waiter) +{ + return list_first_entry(&lock->wait_list, struct mutex_waiter, list) == waiter; +} + +static void __mutex_handoff(struct mutex *lock, struct task_struct *task) +{ + unsigned long owner = atomic_long_read(&lock->owner); + + for (;;) { + unsigned long old, new; + + new = (owner & MUTEX_FLAG_WAITERS); + new |= (unsigned long)task; + + old = atomic_long_cmpxchg(&lock->owner, owner, new); + if (old == owner) + break; + + owner = old; + } +} + #ifndef CONFIG_DEBUG_LOCK_ALLOC /* * We split the mutex lock/unlock logic into separate fastpath and @@ -414,7 +438,7 @@ static bool mutex_optimistic_spin(struct } #endif -static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock); +static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long owner); /** * mutex_unlock - release the mutex @@ -439,6 +463,9 @@ void __sched mutex_unlock(struct mutex * for (;;) { unsigned long old; + if (owner & MUTEX_FLAG_HANDOFF) + break; + old = atomic_long_cmpxchg_release(&lock->owner, owner, owner & 0x03); if (old == owner) break; @@ -447,7 +474,7 @@ void __sched mutex_unlock(struct mutex * } if (owner & 0x03); - __mutex_unlock_slowpath(lock); + __mutex_unlock_slowpath(lock, owner); } EXPORT_SYMBOL(mutex_unlock); @@ -545,7 +572,7 @@ __mutex_lock_common(struct mutex *lock, list_add_tail(&waiter.list, &lock->wait_list); waiter.task = task; - if (list_first_entry(&lock->wait_list, struct mutex_waiter, list) == &waiter) + if (__mutex_waiter_is_first(lock, &waiter)) __mutex_set_flag(lock, MUTEX_FLAG_WAITERS); lock_contended(&lock->dep_map, ip); @@ -573,8 +600,14 @@ __mutex_lock_common(struct mutex *lock, schedule_preempt_disabled(); spin_lock_mutex(&lock->wait_lock, flags); + if (__mutex_owner(lock) == current) + break; + if (__mutex_trylock(lock)) break; + + if (__mutex_waiter_is_first(lock, &waiter)) + __mutex_set_flag(lock, MUTEX_FLAG_HANDOFF); } __set_task_state(task, TASK_RUNNING); @@ -707,7 +740,7 @@ EXPORT_SYMBOL_GPL(__ww_mutex_lock_interr /* * Release the lock, slowpath: */ -static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock) +static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long owner) { unsigned long flags; WAKE_Q(wake_q); @@ -722,6 +755,9 @@ static noinline void __sched __mutex_unl list_entry(lock->wait_list.next, struct mutex_waiter, list); + if (owner & MUTEX_FLAG_HANDOFF) + __mutex_handoff(lock, waiter->task); + debug_mutex_wake_waiter(lock, waiter); wake_q_add(&wake_q, waiter->task); }