* [PATCH v2 1/6] sched: Constrain locks in sched_submit_work()
2023-08-25 18:10 [PATCH v2 0/6] locking/rtmutex: Avoid PI state recursion through sched_submit_work() Sebastian Andrzej Siewior
@ 2023-08-25 18:10 ` Sebastian Andrzej Siewior
2023-08-25 18:10 ` [PATCH v2 2/6] locking/rtmutex: Avoid unconditional slowpath for DEBUG_RT_MUTEXES Sebastian Andrzej Siewior
` (5 subsequent siblings)
6 siblings, 0 replies; 10+ messages in thread
From: Sebastian Andrzej Siewior @ 2023-08-25 18:10 UTC (permalink / raw)
To: Peter Zijlstra, linux-kernel
Cc: bigeasy, tglx, boqun.feng, bristot, bsegall, dietmar.eggemann,
jstultz, juri.lelli, longman, mgorman, mingo, rostedt, swood,
vincent.guittot, vschneid, will
From: Peter Zijlstra <peterz@infradead.org>
Even though sched_submit_work() is ran from preemptible context,
it is discouraged to have it use blocking locks due to the recursion
potential.
Enforce this.
Signed-off-by: "Peter Zijlstra (Intel)" <peterz@infradead.org>
Link: https://lore.kernel.org/r/20230815111430.154558666@infradead.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
kernel/sched/core.c | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index c52c2eba7c739..bca53ff9d8182 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6735,11 +6735,18 @@ void __noreturn do_task_dead(void)
static inline void sched_submit_work(struct task_struct *tsk)
{
+ static DEFINE_WAIT_OVERRIDE_MAP(sched_map, LD_WAIT_CONFIG);
unsigned int task_flags;
if (task_is_running(tsk))
return;
+ /*
+ * Establish LD_WAIT_CONFIG context to ensure none of the code called
+ * will use a blocking primitive -- which would lead to recursion.
+ */
+ lock_map_acquire_try(&sched_map);
+
task_flags = tsk->flags;
/*
* If a worker goes to sleep, notify and ask workqueue whether it
@@ -6764,6 +6771,8 @@ static inline void sched_submit_work(struct task_struct *tsk)
* make sure to submit it to avoid deadlocks.
*/
blk_flush_plug(tsk->plug, true);
+
+ lock_map_release(&sched_map);
}
static void sched_update_worker(struct task_struct *tsk)
--
2.40.1
^ permalink raw reply [flat|nested] 10+ messages in thread* [PATCH v2 2/6] locking/rtmutex: Avoid unconditional slowpath for DEBUG_RT_MUTEXES
2023-08-25 18:10 [PATCH v2 0/6] locking/rtmutex: Avoid PI state recursion through sched_submit_work() Sebastian Andrzej Siewior
2023-08-25 18:10 ` [PATCH v2 1/6] sched: Constrain locks in sched_submit_work() Sebastian Andrzej Siewior
@ 2023-08-25 18:10 ` Sebastian Andrzej Siewior
2023-08-25 18:10 ` [PATCH v2 3/6] sched: Extract __schedule_loop() Sebastian Andrzej Siewior
` (4 subsequent siblings)
6 siblings, 0 replies; 10+ messages in thread
From: Sebastian Andrzej Siewior @ 2023-08-25 18:10 UTC (permalink / raw)
To: Peter Zijlstra, linux-kernel
Cc: bigeasy, tglx, boqun.feng, bristot, bsegall, dietmar.eggemann,
jstultz, juri.lelli, longman, mgorman, mingo, rostedt, swood,
vincent.guittot, vschneid, will
With DEBUG_RT_MUTEXES enabled the fast-path rt_mutex_cmpxchg_acquire()
always fails and all lock operations take the slow path.
Provide a new helper inline rt_mutex_try_acquire() which maps to
rt_mutex_cmpxchg_acquire() in the non-debug case. For the debug case
it invokes rt_mutex_slowtrylock() which can acquire a non-contended
rtmutex under full debug coverage.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: "Peter Zijlstra (Intel)" <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20230427111937.2745231-4-bigeasy@linutronix.de
Link: https://lore.kernel.org/r/20230815111430.220899937@infradead.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
kernel/locking/rtmutex.c | 21 ++++++++++++++++++++-
kernel/locking/ww_rt_mutex.c | 2 +-
2 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
index 21db0df0eb000..bcec0533a0cc0 100644
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -218,6 +218,11 @@ static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock,
return try_cmpxchg_acquire(&lock->owner, &old, new);
}
+static __always_inline bool rt_mutex_try_acquire(struct rt_mutex_base *lock)
+{
+ return rt_mutex_cmpxchg_acquire(lock, NULL, current);
+}
+
static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,
struct task_struct *old,
struct task_struct *new)
@@ -297,6 +302,20 @@ static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock,
}
+static int __sched rt_mutex_slowtrylock(struct rt_mutex_base *lock);
+
+static __always_inline bool rt_mutex_try_acquire(struct rt_mutex_base *lock)
+{
+ /*
+ * With debug enabled rt_mutex_cmpxchg trylock() will always fail.
+ *
+ * Avoid unconditionally taking the slow path by using
+ * rt_mutex_slow_trylock() which is covered by the debug code and can
+ * acquire a non-contended rtmutex.
+ */
+ return rt_mutex_slowtrylock(lock);
+}
+
static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,
struct task_struct *old,
struct task_struct *new)
@@ -1755,7 +1774,7 @@ static int __sched rt_mutex_slowlock(struct rt_mutex_base *lock,
static __always_inline int __rt_mutex_lock(struct rt_mutex_base *lock,
unsigned int state)
{
- if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
+ if (likely(rt_mutex_try_acquire(lock)))
return 0;
return rt_mutex_slowlock(lock, NULL, state);
diff --git a/kernel/locking/ww_rt_mutex.c b/kernel/locking/ww_rt_mutex.c
index d1473c624105c..c7196de838edc 100644
--- a/kernel/locking/ww_rt_mutex.c
+++ b/kernel/locking/ww_rt_mutex.c
@@ -62,7 +62,7 @@ __ww_rt_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ww_ctx,
}
mutex_acquire_nest(&rtm->dep_map, 0, 0, nest_lock, ip);
- if (likely(rt_mutex_cmpxchg_acquire(&rtm->rtmutex, NULL, current))) {
+ if (likely(rt_mutex_try_acquire(&rtm->rtmutex))) {
if (ww_ctx)
ww_mutex_set_context_fastpath(lock, ww_ctx);
return 0;
--
2.40.1
^ permalink raw reply [flat|nested] 10+ messages in thread* [PATCH v2 3/6] sched: Extract __schedule_loop()
2023-08-25 18:10 [PATCH v2 0/6] locking/rtmutex: Avoid PI state recursion through sched_submit_work() Sebastian Andrzej Siewior
2023-08-25 18:10 ` [PATCH v2 1/6] sched: Constrain locks in sched_submit_work() Sebastian Andrzej Siewior
2023-08-25 18:10 ` [PATCH v2 2/6] locking/rtmutex: Avoid unconditional slowpath for DEBUG_RT_MUTEXES Sebastian Andrzej Siewior
@ 2023-08-25 18:10 ` Sebastian Andrzej Siewior
2023-08-25 18:10 ` [PATCH v2 4/6] sched: Provide rt_mutex specific scheduler helpers Sebastian Andrzej Siewior
` (3 subsequent siblings)
6 siblings, 0 replies; 10+ messages in thread
From: Sebastian Andrzej Siewior @ 2023-08-25 18:10 UTC (permalink / raw)
To: Peter Zijlstra, linux-kernel
Cc: bigeasy, tglx, boqun.feng, bristot, bsegall, dietmar.eggemann,
jstultz, juri.lelli, longman, mgorman, mingo, rostedt, swood,
vincent.guittot, vschneid, will
From: Thomas Gleixner <tglx@linutronix.de>
There are currently two implementations of this basic __schedule()
loop, and there is soon to be a third.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: "Peter Zijlstra (Intel)" <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20230427111937.2745231-2-bigeasy@linutronix.de
Link: https://lore.kernel.org/r/20230815111430.288063671@infradead.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
kernel/sched/core.c | 21 +++++++++++----------
1 file changed, 11 insertions(+), 10 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index bca53ff9d8182..901766a88afc3 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6785,16 +6785,21 @@ static void sched_update_worker(struct task_struct *tsk)
}
}
+static __always_inline void __schedule_loop(unsigned int sched_mode)
+{
+ do {
+ preempt_disable();
+ __schedule(sched_mode);
+ sched_preempt_enable_no_resched();
+ } while (need_resched());
+}
+
asmlinkage __visible void __sched schedule(void)
{
struct task_struct *tsk = current;
sched_submit_work(tsk);
- do {
- preempt_disable();
- __schedule(SM_NONE);
- sched_preempt_enable_no_resched();
- } while (need_resched());
+ __schedule_loop(SM_NONE);
sched_update_worker(tsk);
}
EXPORT_SYMBOL(schedule);
@@ -6858,11 +6863,7 @@ void __sched schedule_preempt_disabled(void)
#ifdef CONFIG_PREEMPT_RT
void __sched notrace schedule_rtlock(void)
{
- do {
- preempt_disable();
- __schedule(SM_RTLOCK_WAIT);
- sched_preempt_enable_no_resched();
- } while (need_resched());
+ __schedule_loop(SM_RTLOCK_WAIT);
}
NOKPROBE_SYMBOL(schedule_rtlock);
#endif
--
2.40.1
^ permalink raw reply [flat|nested] 10+ messages in thread* [PATCH v2 4/6] sched: Provide rt_mutex specific scheduler helpers
2023-08-25 18:10 [PATCH v2 0/6] locking/rtmutex: Avoid PI state recursion through sched_submit_work() Sebastian Andrzej Siewior
` (2 preceding siblings ...)
2023-08-25 18:10 ` [PATCH v2 3/6] sched: Extract __schedule_loop() Sebastian Andrzej Siewior
@ 2023-08-25 18:10 ` Sebastian Andrzej Siewior
2023-08-25 18:10 ` [PATCH v2 5/6] locking/rtmutex: Use " Sebastian Andrzej Siewior
` (2 subsequent siblings)
6 siblings, 0 replies; 10+ messages in thread
From: Sebastian Andrzej Siewior @ 2023-08-25 18:10 UTC (permalink / raw)
To: Peter Zijlstra, linux-kernel
Cc: bigeasy, tglx, boqun.feng, bristot, bsegall, dietmar.eggemann,
jstultz, juri.lelli, longman, mgorman, mingo, rostedt, swood,
vincent.guittot, vschneid, will
From: Peter Zijlstra <peterz@infradead.org>
With PREEMPT_RT there is a rt_mutex recursion problem where
sched_submit_work() can use an rtlock (aka spinlock_t). More
specifically what happens is:
mutex_lock() /* really rt_mutex */
...
__rt_mutex_slowlock_locked()
task_blocks_on_rt_mutex()
// enqueue current task as waiter
// do PI chain walk
rt_mutex_slowlock_block()
schedule()
sched_submit_work()
...
spin_lock() /* really rtlock */
...
__rt_mutex_slowlock_locked()
task_blocks_on_rt_mutex()
// enqueue current task as waiter *AGAIN*
// *CONFUSION*
Fix this by making rt_mutex do the sched_submit_work() early, before
it enqueues itself as a waiter -- before it even knows *if* it will
wait.
[[ basically Thomas' patch but with different naming and a few asserts
added ]]
Originally-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: "Peter Zijlstra (Intel)" <peterz@infradead.org>
Link: https://lore.kernel.org/r/20230815111430.355375399@infradead.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
include/linux/sched.h | 3 +++
include/linux/sched/rt.h | 4 ++++
kernel/sched/core.c | 36 ++++++++++++++++++++++++++++++++----
3 files changed, 39 insertions(+), 4 deletions(-)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 609bde814cb06..0ea7a023c6c73 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -906,6 +906,9 @@ struct task_struct {
* ->sched_remote_wakeup gets used, so it can be in this word.
*/
unsigned sched_remote_wakeup:1;
+#ifdef CONFIG_RT_MUTEXES
+ unsigned sched_rt_mutex:1;
+#endif
/* Bit to tell LSMs we're in execve(): */
unsigned in_execve:1;
diff --git a/include/linux/sched/rt.h b/include/linux/sched/rt.h
index 994c25640e156..b2b9e6eb96830 100644
--- a/include/linux/sched/rt.h
+++ b/include/linux/sched/rt.h
@@ -30,6 +30,10 @@ static inline bool task_is_realtime(struct task_struct *tsk)
}
#ifdef CONFIG_RT_MUTEXES
+extern void rt_mutex_pre_schedule(void);
+extern void rt_mutex_schedule(void);
+extern void rt_mutex_post_schedule(void);
+
/*
* Must hold either p->pi_lock or task_rq(p)->lock.
*/
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 901766a88afc3..bba1ed28608ab 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6738,9 +6738,6 @@ static inline void sched_submit_work(struct task_struct *tsk)
static DEFINE_WAIT_OVERRIDE_MAP(sched_map, LD_WAIT_CONFIG);
unsigned int task_flags;
- if (task_is_running(tsk))
- return;
-
/*
* Establish LD_WAIT_CONFIG context to ensure none of the code called
* will use a blocking primitive -- which would lead to recursion.
@@ -6798,7 +6795,12 @@ asmlinkage __visible void __sched schedule(void)
{
struct task_struct *tsk = current;
- sched_submit_work(tsk);
+#ifdef CONFIG_RT_MUTEXES
+ lockdep_assert(!tsk->sched_rt_mutex);
+#endif
+
+ if (!task_is_running(tsk))
+ sched_submit_work(tsk);
__schedule_loop(SM_NONE);
sched_update_worker(tsk);
}
@@ -7059,6 +7061,32 @@ static void __setscheduler_prio(struct task_struct *p, int prio)
#ifdef CONFIG_RT_MUTEXES
+/*
+ * Would be more useful with typeof()/auto_type but they don't mix with
+ * bit-fields. Since it's a local thing, use int. Keep the generic sounding
+ * name such that if someone were to implement this function we get to compare
+ * notes.
+ */
+#define fetch_and_set(x, v) ({ int _x = (x); (x) = (v); _x; })
+
+void rt_mutex_pre_schedule(void)
+{
+ lockdep_assert(!fetch_and_set(current->sched_rt_mutex, 1));
+ sched_submit_work(current);
+}
+
+void rt_mutex_schedule(void)
+{
+ lockdep_assert(current->sched_rt_mutex);
+ __schedule_loop(SM_NONE);
+}
+
+void rt_mutex_post_schedule(void)
+{
+ sched_update_worker(current);
+ lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0));
+}
+
static inline int __rt_effective_prio(struct task_struct *pi_task, int prio)
{
if (pi_task)
--
2.40.1
^ permalink raw reply [flat|nested] 10+ messages in thread* [PATCH v2 5/6] locking/rtmutex: Use rt_mutex specific scheduler helpers
2023-08-25 18:10 [PATCH v2 0/6] locking/rtmutex: Avoid PI state recursion through sched_submit_work() Sebastian Andrzej Siewior
` (3 preceding siblings ...)
2023-08-25 18:10 ` [PATCH v2 4/6] sched: Provide rt_mutex specific scheduler helpers Sebastian Andrzej Siewior
@ 2023-08-25 18:10 ` Sebastian Andrzej Siewior
2023-09-03 18:54 ` Sultan Alsawaf
2023-08-25 18:10 ` [PATCH v2 6/6] locking/rtmutex: Add a lockdep assert to catch potential nested blocking Sebastian Andrzej Siewior
2023-08-31 9:53 ` [PATCH v2 7/6] locking/rtmutex: Acquire the hb lock via trylock after wait-proxylock Sebastian Andrzej Siewior
6 siblings, 1 reply; 10+ messages in thread
From: Sebastian Andrzej Siewior @ 2023-08-25 18:10 UTC (permalink / raw)
To: Peter Zijlstra, linux-kernel
Cc: bigeasy, tglx, boqun.feng, bristot, bsegall, dietmar.eggemann,
jstultz, juri.lelli, longman, mgorman, mingo, rostedt, swood,
vincent.guittot, vschneid, will
Have rt_mutex use the rt_mutex specific scheduler helpers to avoid
recursion vs rtlock on the PI state.
[[ peterz: adapted to new names ]]
Reported-by: Crystal Wood <swood@redhat.com>
Signed-off-by: "Peter Zijlstra (Intel)" <peterz@infradead.org>
Link: https://lore.kernel.org/r/20230815111430.421408298@infradead.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
kernel/futex/pi.c | 11 +++++++++++
kernel/locking/rtmutex.c | 14 ++++++++++++--
kernel/locking/rwbase_rt.c | 6 ++++++
kernel/locking/rwsem.c | 8 +++++++-
kernel/locking/spinlock_rt.c | 4 ++++
5 files changed, 40 insertions(+), 3 deletions(-)
diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c
index ce2889f123755..f8e65b27d9d6b 100644
--- a/kernel/futex/pi.c
+++ b/kernel/futex/pi.c
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <linux/slab.h>
+#include <linux/sched/rt.h>
#include <linux/sched/task.h>
#include "futex.h"
@@ -1002,6 +1003,12 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl
goto no_block;
}
+ /*
+ * Must be done before we enqueue the waiter, here is unfortunately
+ * under the hb lock, but that *should* work because it does nothing.
+ */
+ rt_mutex_pre_schedule();
+
rt_mutex_init_waiter(&rt_waiter);
/*
@@ -1052,6 +1059,10 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl
if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter))
ret = 0;
+ /*
+ * Waiter is unqueued.
+ */
+ rt_mutex_post_schedule();
no_block:
/*
* Fixup the pi_state owner and possibly acquire the lock if we
diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
index bcec0533a0cc0..a3fe05dfd0d8f 100644
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -1632,7 +1632,7 @@ static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock,
raw_spin_unlock_irq(&lock->wait_lock);
if (!owner || !rtmutex_spin_on_owner(lock, waiter, owner))
- schedule();
+ rt_mutex_schedule();
raw_spin_lock_irq(&lock->wait_lock);
set_current_state(state);
@@ -1661,7 +1661,7 @@ static void __sched rt_mutex_handle_deadlock(int res, int detect_deadlock,
WARN(1, "rtmutex deadlock detected\n");
while (1) {
set_current_state(TASK_INTERRUPTIBLE);
- schedule();
+ rt_mutex_schedule();
}
}
@@ -1756,6 +1756,15 @@ static int __sched rt_mutex_slowlock(struct rt_mutex_base *lock,
unsigned long flags;
int ret;
+ /*
+ * Do all pre-schedule work here, before we queue a waiter and invoke
+ * PI -- any such work that trips on rtlock (PREEMPT_RT spinlock) would
+ * otherwise recurse back into task_blocks_on_rt_mutex() through
+ * rtlock_slowlock() and will then enqueue a second waiter for this
+ * same task and things get really confusing real fast.
+ */
+ rt_mutex_pre_schedule();
+
/*
* Technically we could use raw_spin_[un]lock_irq() here, but this can
* be called in early boot if the cmpxchg() fast path is disabled
@@ -1767,6 +1776,7 @@ static int __sched rt_mutex_slowlock(struct rt_mutex_base *lock,
raw_spin_lock_irqsave(&lock->wait_lock, flags);
ret = __rt_mutex_slowlock_locked(lock, ww_ctx, state);
raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
+ rt_mutex_post_schedule();
return ret;
}
diff --git a/kernel/locking/rwbase_rt.c b/kernel/locking/rwbase_rt.c
index 25ec0239477c2..7d57bfb909001 100644
--- a/kernel/locking/rwbase_rt.c
+++ b/kernel/locking/rwbase_rt.c
@@ -71,6 +71,7 @@ static int __sched __rwbase_read_lock(struct rwbase_rt *rwb,
struct rt_mutex_base *rtm = &rwb->rtmutex;
int ret;
+ rwbase_pre_schedule();
raw_spin_lock_irq(&rtm->wait_lock);
/*
@@ -125,6 +126,7 @@ static int __sched __rwbase_read_lock(struct rwbase_rt *rwb,
rwbase_rtmutex_unlock(rtm);
trace_contention_end(rwb, ret);
+ rwbase_post_schedule();
return ret;
}
@@ -237,6 +239,8 @@ static int __sched rwbase_write_lock(struct rwbase_rt *rwb,
/* Force readers into slow path */
atomic_sub(READER_BIAS, &rwb->readers);
+ rt_mutex_pre_schedule();
+
raw_spin_lock_irqsave(&rtm->wait_lock, flags);
if (__rwbase_write_trylock(rwb))
goto out_unlock;
@@ -248,6 +252,7 @@ static int __sched rwbase_write_lock(struct rwbase_rt *rwb,
if (rwbase_signal_pending_state(state, current)) {
rwbase_restore_current_state();
__rwbase_write_unlock(rwb, 0, flags);
+ rt_mutex_post_schedule();
trace_contention_end(rwb, -EINTR);
return -EINTR;
}
@@ -266,6 +271,7 @@ static int __sched rwbase_write_lock(struct rwbase_rt *rwb,
out_unlock:
raw_spin_unlock_irqrestore(&rtm->wait_lock, flags);
+ rt_mutex_post_schedule();
return 0;
}
diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c
index 9eabd585ce7af..2340b6d90ec6f 100644
--- a/kernel/locking/rwsem.c
+++ b/kernel/locking/rwsem.c
@@ -1427,8 +1427,14 @@ static inline void __downgrade_write(struct rw_semaphore *sem)
#define rwbase_signal_pending_state(state, current) \
signal_pending_state(state, current)
+#define rwbase_pre_schedule() \
+ rt_mutex_pre_schedule()
+
#define rwbase_schedule() \
- schedule()
+ rt_mutex_schedule()
+
+#define rwbase_post_schedule() \
+ rt_mutex_post_schedule()
#include "rwbase_rt.c"
diff --git a/kernel/locking/spinlock_rt.c b/kernel/locking/spinlock_rt.c
index 48a19ed8486d8..842037b2ba548 100644
--- a/kernel/locking/spinlock_rt.c
+++ b/kernel/locking/spinlock_rt.c
@@ -184,9 +184,13 @@ static __always_inline int rwbase_rtmutex_trylock(struct rt_mutex_base *rtm)
#define rwbase_signal_pending_state(state, current) (0)
+#define rwbase_pre_schedule()
+
#define rwbase_schedule() \
schedule_rtlock()
+#define rwbase_post_schedule()
+
#include "rwbase_rt.c"
/*
* The common functions which get wrapped into the rwlock API.
--
2.40.1
^ permalink raw reply [flat|nested] 10+ messages in thread* Re: [PATCH v2 5/6] locking/rtmutex: Use rt_mutex specific scheduler helpers
2023-08-25 18:10 ` [PATCH v2 5/6] locking/rtmutex: Use " Sebastian Andrzej Siewior
@ 2023-09-03 18:54 ` Sultan Alsawaf
2023-09-04 9:03 ` Sebastian Andrzej Siewior
0 siblings, 1 reply; 10+ messages in thread
From: Sultan Alsawaf @ 2023-09-03 18:54 UTC (permalink / raw)
To: Sebastian Andrzej Siewior
Cc: Peter Zijlstra, linux-kernel, tglx, boqun.feng, bristot, bsegall,
dietmar.eggemann, jstultz, juri.lelli, longman, mgorman, mingo,
rostedt, swood, vincent.guittot, vschneid, will
On Fri, Aug 25, 2023 at 08:10:32PM +0200, Sebastian Andrzej Siewior wrote:
> --- a/kernel/locking/rwbase_rt.c
> +++ b/kernel/locking/rwbase_rt.c
> @@ -71,6 +71,7 @@ static int __sched __rwbase_read_lock(struct rwbase_rt *rwb,
> struct rt_mutex_base *rtm = &rwb->rtmutex;
> int ret;
>
> + rwbase_pre_schedule();
> raw_spin_lock_irq(&rtm->wait_lock);
>
> /*
> @@ -125,6 +126,7 @@ static int __sched __rwbase_read_lock(struct rwbase_rt *rwb,
> rwbase_rtmutex_unlock(rtm);
>
> trace_contention_end(rwb, ret);
> + rwbase_post_schedule();
> return ret;
> }
>
> @@ -237,6 +239,8 @@ static int __sched rwbase_write_lock(struct rwbase_rt *rwb,
> /* Force readers into slow path */
> atomic_sub(READER_BIAS, &rwb->readers);
>
> + rt_mutex_pre_schedule();
> +
> raw_spin_lock_irqsave(&rtm->wait_lock, flags);
> if (__rwbase_write_trylock(rwb))
> goto out_unlock;
> @@ -248,6 +252,7 @@ static int __sched rwbase_write_lock(struct rwbase_rt *rwb,
> if (rwbase_signal_pending_state(state, current)) {
> rwbase_restore_current_state();
> __rwbase_write_unlock(rwb, 0, flags);
> + rt_mutex_post_schedule();
> trace_contention_end(rwb, -EINTR);
> return -EINTR;
> }
> @@ -266,6 +271,7 @@ static int __sched rwbase_write_lock(struct rwbase_rt *rwb,
>
> out_unlock:
> raw_spin_unlock_irqrestore(&rtm->wait_lock, flags);
> + rt_mutex_post_schedule();
> return 0;
> }
Shouldn't rwbase_write_lock() use rwbase_{pre|post}_schedule()?
With this change as-is, I observe deadlocks due to lock recursion from
write_lock() specifically, because write_lock() ends up flushing block requests.
Sultan
^ permalink raw reply [flat|nested] 10+ messages in thread* Re: [PATCH v2 5/6] locking/rtmutex: Use rt_mutex specific scheduler helpers
2023-09-03 18:54 ` Sultan Alsawaf
@ 2023-09-04 9:03 ` Sebastian Andrzej Siewior
0 siblings, 0 replies; 10+ messages in thread
From: Sebastian Andrzej Siewior @ 2023-09-04 9:03 UTC (permalink / raw)
To: Sultan Alsawaf
Cc: Peter Zijlstra, linux-kernel, tglx, boqun.feng, bristot, bsegall,
dietmar.eggemann, jstultz, juri.lelli, longman, mgorman, mingo,
rostedt, swood, vincent.guittot, vschneid, will
On 2023-09-03 11:54:41 [-0700], Sultan Alsawaf wrote:
> On Fri, Aug 25, 2023 at 08:10:32PM +0200, Sebastian Andrzej Siewior wrote:
> > --- a/kernel/locking/rwbase_rt.c
> > +++ b/kernel/locking/rwbase_rt.c
> > @@ -71,6 +71,7 @@ static int __sched __rwbase_read_lock(struct rwbase_rt *rwb,
> > struct rt_mutex_base *rtm = &rwb->rtmutex;
> > int ret;
> >
> > + rwbase_pre_schedule();
> > raw_spin_lock_irq(&rtm->wait_lock);
> >
> > /*
> > @@ -125,6 +126,7 @@ static int __sched __rwbase_read_lock(struct rwbase_rt *rwb,
> > rwbase_rtmutex_unlock(rtm);
> >
> > trace_contention_end(rwb, ret);
> > + rwbase_post_schedule();
> > return ret;
> > }
> >
> > @@ -237,6 +239,8 @@ static int __sched rwbase_write_lock(struct rwbase_rt *rwb,
> > /* Force readers into slow path */
> > atomic_sub(READER_BIAS, &rwb->readers);
> >
> > + rt_mutex_pre_schedule();
> > +
> > raw_spin_lock_irqsave(&rtm->wait_lock, flags);
> > if (__rwbase_write_trylock(rwb))
> > goto out_unlock;
> > @@ -248,6 +252,7 @@ static int __sched rwbase_write_lock(struct rwbase_rt *rwb,
> > if (rwbase_signal_pending_state(state, current)) {
> > rwbase_restore_current_state();
> > __rwbase_write_unlock(rwb, 0, flags);
> > + rt_mutex_post_schedule();
> > trace_contention_end(rwb, -EINTR);
> > return -EINTR;
> > }
> > @@ -266,6 +271,7 @@ static int __sched rwbase_write_lock(struct rwbase_rt *rwb,
> >
> > out_unlock:
> > raw_spin_unlock_irqrestore(&rtm->wait_lock, flags);
> > + rt_mutex_post_schedule();
> > return 0;
> > }
>
> Shouldn't rwbase_write_lock() use rwbase_{pre|post}_schedule()?
>
> With this change as-is, I observe deadlocks due to lock recursion from
> write_lock() specifically, because write_lock() ends up flushing block requests.
You are right, it should have been rwbase_{pre|post}_schedule() because
write_lock() is a spinning lock and should not flush the block requests.
Thanks for spotting this.
> Sultan
Sebastian
^ permalink raw reply [flat|nested] 10+ messages in thread
* [PATCH v2 6/6] locking/rtmutex: Add a lockdep assert to catch potential nested blocking
2023-08-25 18:10 [PATCH v2 0/6] locking/rtmutex: Avoid PI state recursion through sched_submit_work() Sebastian Andrzej Siewior
` (4 preceding siblings ...)
2023-08-25 18:10 ` [PATCH v2 5/6] locking/rtmutex: Use " Sebastian Andrzej Siewior
@ 2023-08-25 18:10 ` Sebastian Andrzej Siewior
2023-08-31 9:53 ` [PATCH v2 7/6] locking/rtmutex: Acquire the hb lock via trylock after wait-proxylock Sebastian Andrzej Siewior
6 siblings, 0 replies; 10+ messages in thread
From: Sebastian Andrzej Siewior @ 2023-08-25 18:10 UTC (permalink / raw)
To: Peter Zijlstra, linux-kernel
Cc: bigeasy, tglx, boqun.feng, bristot, bsegall, dietmar.eggemann,
jstultz, juri.lelli, longman, mgorman, mingo, rostedt, swood,
vincent.guittot, vschneid, will
From: Thomas Gleixner <tglx@linutronix.de>
There used to be a BUG_ON(current->pi_blocked_on) in the lock acquisition
functions, but that vanished in one of the rtmutex overhauls.
Bring it back in form of a lockdep assert to catch code paths which take
rtmutex based locks with current::pi_blocked_on != NULL.
Reported-by: Crystal Wood <swood@redhat.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: "Peter Zijlstra (Intel)" <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20230427111937.2745231-5-bigeasy@linutronix.de
Link: https://lore.kernel.org/r/20230815111430.488430699@infradead.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
kernel/locking/rtmutex.c | 2 ++
kernel/locking/rwbase_rt.c | 2 ++
kernel/locking/spinlock_rt.c | 2 ++
3 files changed, 6 insertions(+)
diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
index a3fe05dfd0d8f..4a10e8c16fd2b 100644
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -1784,6 +1784,8 @@ static int __sched rt_mutex_slowlock(struct rt_mutex_base *lock,
static __always_inline int __rt_mutex_lock(struct rt_mutex_base *lock,
unsigned int state)
{
+ lockdep_assert(!current->pi_blocked_on);
+
if (likely(rt_mutex_try_acquire(lock)))
return 0;
diff --git a/kernel/locking/rwbase_rt.c b/kernel/locking/rwbase_rt.c
index 7d57bfb909001..b5e881250fec5 100644
--- a/kernel/locking/rwbase_rt.c
+++ b/kernel/locking/rwbase_rt.c
@@ -133,6 +133,8 @@ static int __sched __rwbase_read_lock(struct rwbase_rt *rwb,
static __always_inline int rwbase_read_lock(struct rwbase_rt *rwb,
unsigned int state)
{
+ lockdep_assert(!current->pi_blocked_on);
+
if (rwbase_read_trylock(rwb))
return 0;
diff --git a/kernel/locking/spinlock_rt.c b/kernel/locking/spinlock_rt.c
index 842037b2ba548..38e292454fccb 100644
--- a/kernel/locking/spinlock_rt.c
+++ b/kernel/locking/spinlock_rt.c
@@ -37,6 +37,8 @@
static __always_inline void rtlock_lock(struct rt_mutex_base *rtm)
{
+ lockdep_assert(!current->pi_blocked_on);
+
if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
rtlock_slowlock(rtm);
}
--
2.40.1
^ permalink raw reply [flat|nested] 10+ messages in thread* [PATCH v2 7/6] locking/rtmutex: Acquire the hb lock via trylock after wait-proxylock.
2023-08-25 18:10 [PATCH v2 0/6] locking/rtmutex: Avoid PI state recursion through sched_submit_work() Sebastian Andrzej Siewior
` (5 preceding siblings ...)
2023-08-25 18:10 ` [PATCH v2 6/6] locking/rtmutex: Add a lockdep assert to catch potential nested blocking Sebastian Andrzej Siewior
@ 2023-08-31 9:53 ` Sebastian Andrzej Siewior
6 siblings, 0 replies; 10+ messages in thread
From: Sebastian Andrzej Siewior @ 2023-08-31 9:53 UTC (permalink / raw)
To: Peter Zijlstra, linux-kernel
Cc: tglx, boqun.feng, bristot, bsegall, dietmar.eggemann, jstultz,
juri.lelli, longman, mgorman, mingo, rostedt, swood,
vincent.guittot, vschneid, will
After rt_mutex_wait_proxy_lock() task_struct::pi_blocked_on is cleared
if current owns the lock. If the operation has been interrupted by a
signal or timeout then pi_blocked_on can be set. This means spin_lock()
*can* overwrite pi_blocked_on on PREEMPT_RT. This has been noticed by
the recently added lockdep-asserts…
The rt_mutex_cleanup_proxy_lock() operation will clear pi_blocked_on
(and update pending waiters as expected) but it must happen under the hb
lock to ensure the same state in rtmutex and userland.
Given all the possibilities it is probably the simplest option to
try-lock the hb lock. In case the lock is occupied a quick nap is
needed. A busy loop can lock up the system if performed by a task with
high priorioty preventing the owner from running.
The rt_mutex_post_schedule() needs to be put before try-lock-loop
because otherwie the schedule() in schedule_hrtimeout() will trip over
the !sched_rt_mutex assert.
Introduce futex_trylock_hblock() to try-lock the hb lock and sleep until
the try-lock operation succeeds. Use it after rt_mutex_wait_proxy_lock()
to acquire the lock.
Suggested-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
On 2023-08-25 20:10:27 [+0200], To Peter Zijlstra wrote:
> but then why over complicate things. Speaking of over complicating, this
> triggered:
> | ------------[ cut here ]------------
> | WARNING: CPU: 6 PID: 2155 at kernel/locking/spinlock_rt.c:40 rt_spin_lock+0x5a/0xf0
…
> Here, rt_mutex_wait_proxy_lock() returned with -EINTR, didn't acquire
> the lock. Later rt_mutex_cleanup_proxy_lock() would clean
> ->pi_blocked_on but that happens after
> | /* Current is not longer pi_blocked_on */
> | spin_lock(q.lock_ptr);
So this seems to do the job. Gross but …
kernel/futex/futex.h | 23 +++++++++++++++++++++++
kernel/futex/pi.c | 10 ++++------
kernel/futex/requeue.c | 4 ++--
kernel/locking/rtmutex.c | 7 +++++--
4 files changed, 34 insertions(+), 10 deletions(-)
diff --git a/kernel/futex/futex.h b/kernel/futex/futex.h
index b5379c0e6d6d1..b0b2a5b35ae57 100644
--- a/kernel/futex/futex.h
+++ b/kernel/futex/futex.h
@@ -254,6 +254,29 @@ double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
spin_unlock(&hb2->lock);
}
+static inline void futex_trylock_hblock(spinlock_t *lock)
+{
+ do {
+ ktime_t chill_time;;
+
+ /*
+ * Current is not longer pi_blocked_on if it owns the lock. It
+ * can still have pi_blocked_on set if the lock acquiring was
+ * interrupted by signal or timeout. The trylock operation does
+ * not clobber pi_blocked_on so it is the only option.
+ * Should the try-lock operation fail then it needs leave the CPU
+ * to avoid a busy loop in case it is the task with the highest
+ * priority.
+ */
+ if (spin_trylock(lock))
+ return;
+
+ chill_time = ktime_set(0, NSEC_PER_MSEC);
+ set_current_state(TASK_UNINTERRUPTIBLE);
+ schedule_hrtimeout(&chill_time, HRTIMER_MODE_REL_HARD);
+ } while (1);
+}
+
/* syscalls */
extern int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, u32
diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c
index f8e65b27d9d6b..1440fdcdbfd8c 100644
--- a/kernel/futex/pi.c
+++ b/kernel/futex/pi.c
@@ -1046,7 +1046,10 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl
ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);
cleanup:
- spin_lock(q.lock_ptr);
+ rt_mutex_post_schedule();
+
+ futex_trylock_hblock(q.lock_ptr);
+
/*
* If we failed to acquire the lock (deadlock/signal/timeout), we must
* first acquire the hb->lock before removing the lock from the
@@ -1058,11 +1061,6 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl
*/
if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter))
ret = 0;
-
- /*
- * Waiter is unqueued.
- */
- rt_mutex_post_schedule();
no_block:
/*
* Fixup the pi_state owner and possibly acquire the lock if we
diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c
index cba8b1a6a4cc2..26888cfa74449 100644
--- a/kernel/futex/requeue.c
+++ b/kernel/futex/requeue.c
@@ -850,8 +850,8 @@ int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags,
pi_mutex = &q.pi_state->pi_mutex;
ret = rt_mutex_wait_proxy_lock(pi_mutex, to, &rt_waiter);
- /* Current is not longer pi_blocked_on */
- spin_lock(q.lock_ptr);
+ futex_trylock_hblock(q.lock_ptr);
+
if (ret && !rt_mutex_cleanup_proxy_lock(pi_mutex, &rt_waiter))
ret = 0;
diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
index 4a10e8c16fd2b..e56585ef489c8 100644
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -1166,10 +1166,13 @@ try_to_take_rt_mutex(struct rt_mutex_base *lock, struct task_struct *task,
* Clear @task->pi_blocked_on. Requires protection by
* @task->pi_lock. Redundant operation for the @waiter == NULL
* case, but conditionals are more expensive than a redundant
- * store.
+ * store. But then there is FUTEX and if rt_mutex_wait_proxy_lock()
+ * did not acquire the lock it try-locks another lock before it clears
+ * @task->pi_blocked_on so we mustn't clear it here premature.
*/
raw_spin_lock(&task->pi_lock);
- task->pi_blocked_on = NULL;
+ if (waiter)
+ task->pi_blocked_on = NULL;
/*
* Finish the lock acquisition. @task is the new owner. If
* other waiters exist we have to insert the highest priority
--
2.40.1
^ permalink raw reply [flat|nested] 10+ messages in thread