* [PATCH 1/7] locking: add opt-in tracking of sleeping lock holders
2026-09-23 5:01 [PATCH 0/7] locking: opt-in tracking of sleeping lock holders Shakeel Butt
@ 2026-09-23 5:01 ` Shakeel Butt
2026-09-23 5:01 ` [PATCH 2/7] locking/rwsem: track holders of opted-in rw_semaphores Shakeel Butt
` (5 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Shakeel Butt @ 2026-09-23 5:01 UTC (permalink / raw)
To: Peter Zijlstra, Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long
Cc: Paul McKenney, Greg Kroah-Hartman, Tejun Heo, Christian Brauner,
Sebastian Andrzej Siewior, Johannes Weiner, Jonathan Corbet,
Meta kernel team, cgroups, driver-core, linux-kernel
A task that stalls while holding a widely shared lock also stalls the
tasks waiting for that lock. One source of such stalls is a memory
allocation by the lock holder, which can get stuck in memory reclaim
because the system is low on memory or because a memcg limit is being
enforced. More concretely, if an allocation made while holding
cgroup_mutex or kernfs_rwsem gets stuck in memory reclaim (global or
memcg), other cgroupfs operations can end up waiting too, even for
unrelated cgroups.
To avoid this, the allocation path (memcg and global) needs a cheap way
to tell whether the task holds such a lock. Locks do not record all
their holders, so ask the task instead: let each task count the sleeping
locks it holds that have opted in to tracking. Read the count with
task_holds_tracked_lock(), current_holds_tracked_lock() or
task_nr_tracked_locks(). BPF can read task->nr_tracked_locks through
BTF. Only the task itself changes its count, so it needs no atomics.
No lock is tracked by default. Until the first lock opts in, the hooks
are a static branch that is off (a 2-byte NOP on x86_64). The first
opt-in turns it on for good.
Signed-off-by: Shakeel Butt <shakeel.butt@linux.dev>
---
MAINTAINERS | 1 +
include/linux/lockholder.h | 119 ++++++++++++++++++++++++++++++++++++
include/linux/sched.h | 8 +++
kernel/Kconfig.locks | 16 +++++
kernel/exit.c | 4 ++
kernel/fork.c | 3 +
kernel/locking/Makefile | 1 +
kernel/locking/lockholder.c | 19 ++++++
8 files changed, 171 insertions(+)
create mode 100644 include/linux/lockholder.h
create mode 100644 kernel/locking/lockholder.c
diff --git a/MAINTAINERS b/MAINTAINERS
index dccddc99ac9a..097e3d9e4973 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -15428,6 +15428,7 @@ F: Documentation/locking/
F: arch/*/include/asm/spinlock*.h
F: include/linux/local_lock*.h
F: include/linux/lockdep*.h
+F: include/linux/lockholder.h
F: include/linux/mutex*.h
F: include/linux/rwlock*.h
F: include/linux/rwsem*.h
diff --git a/include/linux/lockholder.h b/include/linux/lockholder.h
new file mode 100644
index 000000000000..480d8cdd0fbb
--- /dev/null
+++ b/include/linux/lockholder.h
@@ -0,0 +1,119 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Opt-in tracking of sleeping lock holders.
+ *
+ * Each task counts the opted-in sleeping locks it holds. This gives a
+ * cheap way to check whether a task holds a lock that other tasks may be
+ * waiting for, e.g. to avoid throttling it in memory.high enforcement.
+ *
+ * No lock is tracked by default. Until the first lock opts in, the hooks
+ * are a static branch that is off.
+ */
+#ifndef __LINUX_LOCKHOLDER_H
+#define __LINUX_LOCKHOLDER_H
+
+#include <linux/bug.h>
+#include <linux/jump_label.h>
+#include <linux/sched.h>
+
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+
+DECLARE_STATIC_KEY_FALSE(lock_holder_tracking_key);
+
+/*
+ * Turn the hooks on. There is no way to turn them off, as locks never
+ * opt out. May sleep; call after jump_label_init().
+ */
+void lock_holder_tracking_enable(void);
+
+static __always_inline void __lock_holder_acquired(void)
+{
+ current->nr_tracked_locks++;
+}
+
+static __always_inline void __lock_holder_released(void)
+{
+ /*
+ * Only the task changes its own count, so zero here means a missed
+ * acquire hook, not a race. Don't wrap around: the task would look
+ * like a lock holder forever.
+ */
+ if (WARN_ON_ONCE(!current->nr_tracked_locks))
+ return;
+ current->nr_tracked_locks--;
+}
+
+/*
+ * Hooks for lock implementations. @tracked says whether this lock opted
+ * in. It is only evaluated once the static branch is on, so nothing runs
+ * until a lock opts in.
+ */
+#define lock_holder_acquired_if(tracked) \
+do { \
+ if (static_branch_unlikely(&lock_holder_tracking_key) && \
+ (tracked)) \
+ __lock_holder_acquired(); \
+} while (0)
+
+#define lock_holder_released_if(tracked) \
+do { \
+ if (static_branch_unlikely(&lock_holder_tracking_key) && \
+ (tracked)) \
+ __lock_holder_released(); \
+} while (0)
+
+/* For lock types that keep the opt-in in a bool "tracked" member. */
+#define lock_holder_acquired(lock) lock_holder_acquired_if((lock)->tracked)
+#define lock_holder_released(lock) lock_holder_released_if((lock)->tracked)
+
+static inline void lock_holder_task_init(struct task_struct *p)
+{
+ p->nr_tracked_locks = 0;
+}
+
+static inline unsigned int task_nr_tracked_locks(struct task_struct *p)
+{
+ /*
+ * Only @p writes this, so a cross-task read can see a stale value.
+ * Callers asking about another task want a hint, not a guarantee.
+ */
+ return data_race(p->nr_tracked_locks);
+}
+
+#else /* !CONFIG_TRACK_LOCK_HOLDERS */
+
+#define lock_holder_acquired_if(tracked) do { (void)(tracked); } while (0)
+#define lock_holder_released_if(tracked) do { (void)(tracked); } while (0)
+#define lock_holder_acquired(lock) do { (void)(lock); } while (0)
+#define lock_holder_released(lock) do { (void)(lock); } while (0)
+
+static inline void lock_holder_task_init(struct task_struct *p) { }
+
+static inline unsigned int task_nr_tracked_locks(struct task_struct *p)
+{
+ return 0;
+}
+
+#endif /* CONFIG_TRACK_LOCK_HOLDERS */
+
+/**
+ * task_holds_tracked_lock - check if a task holds an opted-in lock
+ * @p: task to check
+ *
+ * For a task other than current, the answer can change at any time.
+ * Always false without CONFIG_TRACK_LOCK_HOLDERS.
+ */
+static inline bool task_holds_tracked_lock(struct task_struct *p)
+{
+ return task_nr_tracked_locks(p) != 0;
+}
+
+/**
+ * current_holds_tracked_lock - check if current holds an opted-in lock
+ */
+static inline bool current_holds_tracked_lock(void)
+{
+ return task_holds_tracked_lock(current);
+}
+
+#endif /* __LINUX_LOCKHOLDER_H */
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 5ce350a616e4..3f7f459e33fd 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1306,6 +1306,14 @@ struct task_struct {
struct held_lock held_locks[MAX_LOCK_DEPTH];
#endif
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+ /*
+ * Number of opted-in sleeping locks held. Only the task itself
+ * changes it, so it needs no atomics. See linux/lockholder.h.
+ */
+ unsigned int nr_tracked_locks;
+#endif
+
#if defined(CONFIG_UBSAN) && !defined(CONFIG_UBSAN_TRAP)
unsigned int in_ubsan;
#endif
diff --git a/kernel/Kconfig.locks b/kernel/Kconfig.locks
index 1c6423aafcd4..9489e6ce72c0 100644
--- a/kernel/Kconfig.locks
+++ b/kernel/Kconfig.locks
@@ -279,3 +279,19 @@ config ARCH_HAS_MMIOWB
config MMIOWB
def_bool y if ARCH_HAS_MMIOWB
depends on SMP
+
+config TRACK_LOCK_HOLDERS
+ bool "Track holders of selected sleeping locks"
+ help
+ Keep a per-task count of held sleeping locks that have opted in
+ to tracking. This lets code check whether a task holds a lock
+ that others may be waiting for, e.g. to avoid stalling it in
+ memcg limit enforcement or under global memory pressure.
+
+ No lock is tracked until one opts in. Until then, the lock hooks
+ are a static branch that is off. The opt-in is a spare bit in a
+ word the lock already has, or existing padding, so on 64-bit no
+ lock grows. Where there is no padding to use, as on 32-bit, some
+ locks grow by a word.
+
+ If unsure, say N.
diff --git a/kernel/exit.c b/kernel/exit.c
index 0430d7c16600..57b05fb19ea3 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -35,6 +35,7 @@
#include <linux/mount.h>
#include <linux/proc_fs.h>
#include <linux/kthread.h>
+#include <linux/lockholder.h>
#include <linux/mempolicy.h>
#include <linux/taskstats_kern.h>
#include <linux/delayacct.h>
@@ -1025,6 +1026,9 @@ void __noreturn do_exit(long code)
* Make sure we are holding no locks:
*/
debug_check_no_locks_held();
+ WARN_ONCE(task_holds_tracked_lock(tsk),
+ "%s/%d exiting while holding %u tracked lock(s)\n",
+ tsk->comm, task_pid_nr(tsk), task_nr_tracked_locks(tsk));
if (tsk->io_context)
exit_io_context(tsk);
diff --git a/kernel/fork.c b/kernel/fork.c
index 39be51048d77..6c461c7e2187 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -94,6 +94,7 @@
#include <linux/sysctl.h>
#include <linux/kcov.h>
#include <linux/livepatch.h>
+#include <linux/lockholder.h>
#include <linux/thread_info.h>
#include <linux/kstack_erase.h>
#include <linux/kasan.h>
@@ -2265,6 +2266,8 @@ __latent_entropy struct task_struct *copy_process(
lockdep_init_task(p);
+ lock_holder_task_init(p);
+
p->blocked_on = NULL; /* not blocked yet */
p->blocked_donor = NULL; /* nobody is boosting p yet */
diff --git a/kernel/locking/Makefile b/kernel/locking/Makefile
index cee1901d4cff..a0945beb304c 100644
--- a/kernel/locking/Makefile
+++ b/kernel/locking/Makefile
@@ -9,6 +9,7 @@ CONTEXT_ANALYSIS_ww_rt_mutex.o := y
CONTEXT_ANALYSIS_rwsem.o := y
obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o
+obj-$(CONFIG_TRACK_LOCK_HOLDERS) += lockholder.o
# Avoid recursion lockdep -> sanitizer -> ... -> lockdep & improve performance.
KASAN_SANITIZE_lockdep.o := n
diff --git a/kernel/locking/lockholder.c b/kernel/locking/lockholder.c
new file mode 100644
index 000000000000..99428c6141c6
--- /dev/null
+++ b/kernel/locking/lockholder.c
@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Opt-in tracking of sleeping lock holders. See linux/lockholder.h.
+ */
+#include <linux/export.h>
+#include <linux/lockholder.h>
+
+/* Exported for inline lock functions used by modules. */
+DEFINE_STATIC_KEY_FALSE(lock_holder_tracking_key);
+EXPORT_SYMBOL_GPL(lock_holder_tracking_key);
+
+void lock_holder_tracking_enable(void)
+{
+ /* static_branch_enable() takes cpus_read_lock() even if already on. */
+ if (static_key_enabled(&lock_holder_tracking_key))
+ return;
+
+ static_branch_enable(&lock_holder_tracking_key);
+}
--
2.53.0-Meta
^ permalink raw reply [flat|nested] 8+ messages in thread* [PATCH 2/7] locking/rwsem: track holders of opted-in rw_semaphores
2026-09-23 5:01 [PATCH 0/7] locking: opt-in tracking of sleeping lock holders Shakeel Butt
2026-09-23 5:01 ` [PATCH 1/7] locking: add " Shakeel Butt
@ 2026-09-23 5:01 ` Shakeel Butt
2026-09-23 5:01 ` [PATCH 3/7] locking/mutex: track holders of opted-in mutexes Shakeel Butt
` (4 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Shakeel Butt @ 2026-09-23 5:01 UTC (permalink / raw)
To: Peter Zijlstra, Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long
Cc: Paul McKenney, Greg Kroah-Hartman, Tejun Heo, Christian Brauner,
Sebastian Andrzej Siewior, Johannes Weiner, Jonathan Corbet,
Meta kernel team, cgroups, driver-core, linux-kernel
Add rwsem_track_holder() to opt an rw_semaphore in to holder tracking.
The count goes up once down_*() has the lock and down before up_*()
releases it. Waiting is not counted, as the task does not hold the lock
yet. The killable, interruptible and trylock variants count only on
success. downgrade_write() keeps the lock held, so it needs no hook.
down_read_non_owner() and up_read_non_owner() can run in different
tasks, which a per-task count cannot follow. Both warn on a tracked
rwsem and neither counts. The warn is on both sides because the two are
not always paired with each other: bpf task_iter takes mm->mmap_lock
with mmap_read_lock_killable() and drops it with up_read_non_owner(),
so warning only on the acquire side would miss it. This matters only
with CONFIG_DEBUG_LOCK_ALLOC, as without it both are defined to plain
down_read() and up_read(), which are counted and balanced.
A lock must opt in before anyone takes it. A task already holding it was
not counted, so its up_*() would make its count wrong.
rwsem_track_holder() warns and does nothing if the rwsem is held, and
__init_rwsem() clears the flag with the rest of count.
The opt-in is bit 3 of ->count, which is reserved. It is sticky: unlike
the other flag bits it survives unlock, so a free tracked rwsem reads
RWSEM_FLAG_TRACKED rather than zero, and rwsem_is_locked() and
rwsem_assert_held_nolockdep() mask it off. Nothing else has to change,
because every other path either adds to count or writes back a value it
read, so the bit comes along.
PREEMPT_RT builds rw_semaphore on rt_mutex_base, which clears ->owner on
unlock, so a bit there would not survive. On 64-bit it has four bytes of
padding after ->wait_lock, so the flag goes there.
Signed-off-by: Shakeel Butt <shakeel.butt@linux.dev>
---
include/linux/rtmutex.h | 11 ++++
include/linux/rwsem.h | 65 +++++++++++++++++++++-
kernel/locking/rtmutex_common.h | 3 ++
kernel/locking/rwsem.c | 96 ++++++++++++++++++++++++++++++---
4 files changed, 167 insertions(+), 8 deletions(-)
diff --git a/include/linux/rtmutex.h b/include/linux/rtmutex.h
index 9e1f012f89db..3001a25705ac 100644
--- a/include/linux/rtmutex.h
+++ b/include/linux/rtmutex.h
@@ -22,6 +22,17 @@ extern int max_lock_depth;
struct rt_mutex_base {
raw_spinlock_t wait_lock;
+#if defined(CONFIG_TRACK_LOCK_HOLDERS) && defined(CONFIG_PREEMPT_RT)
+ /*
+ * Opt-in to holder tracking, set before first use. On PREEMPT_RT
+ * the sleeping locks that can be tracked are built on this one.
+ * On 64-bit the flag fits in the padding after wait_lock, so none
+ * of them grows. Where that padding does not exist, as on 32-bit,
+ * everything built on rt_mutex_base grows by a word, spinlock_t
+ * included.
+ */
+ bool tracked;
+#endif
struct rb_root_cached waiters __guarded_by(&wait_lock);
struct task_struct *owner __guarded_by(&wait_lock);
};
diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h
index 6a1a7bae5f81..fa4f999330ca 100644
--- a/include/linux/rwsem.h
+++ b/include/linux/rwsem.h
@@ -68,17 +68,30 @@ context_lock_struct(rw_semaphore) {
#define RWSEM_UNLOCKED_VALUE 0UL
#define RWSEM_WRITER_LOCKED (1UL << 0)
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+/*
+ * Sticky: set by rwsem_track_holder(), cleared only by init. count bits
+ * 3-7 are reserved, and the lock and unlock paths only add to count or
+ * write back a value they read, so the bit survives. See
+ * kernel/locking/rwsem.c.
+ */
+#define RWSEM_FLAG_TRACKED (1UL << 3)
+#else
+#define RWSEM_FLAG_TRACKED 0UL
+#endif
#define __RWSEM_COUNT_INIT(name) .count = ATOMIC_LONG_INIT(RWSEM_UNLOCKED_VALUE)
static inline int rwsem_is_locked(struct rw_semaphore *sem)
{
- return atomic_long_read(&sem->count) != RWSEM_UNLOCKED_VALUE;
+ return (atomic_long_read(&sem->count) & ~RWSEM_FLAG_TRACKED) !=
+ RWSEM_UNLOCKED_VALUE;
}
static inline void rwsem_assert_held_nolockdep(const struct rw_semaphore *sem)
__assumes_ctx_lock(sem)
{
- WARN_ON(atomic_long_read(&sem->count) == RWSEM_UNLOCKED_VALUE);
+ WARN_ON((atomic_long_read(&sem->count) & ~RWSEM_FLAG_TRACKED) ==
+ RWSEM_UNLOCKED_VALUE);
}
static inline void rwsem_assert_held_write_nolockdep(const struct rw_semaphore *sem)
@@ -87,6 +100,21 @@ static inline void rwsem_assert_held_write_nolockdep(const struct rw_semaphore *
WARN_ON(!(atomic_long_read(&sem->count) & RWSEM_WRITER_LOCKED));
}
+static inline bool rwsem_is_tracked(const struct rw_semaphore *sem)
+{
+ if (!IS_ENABLED(CONFIG_TRACK_LOCK_HOLDERS))
+ return false;
+
+ return atomic_long_read(&sem->count) & RWSEM_FLAG_TRACKED;
+}
+
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+static inline void __rwsem_set_tracked(struct rw_semaphore *sem)
+{
+ atomic_long_or(RWSEM_FLAG_TRACKED, &sem->count);
+}
+#endif
+
/* Common initializer macros and functions */
#ifdef CONFIG_DEBUG_RWSEMS
@@ -157,6 +185,23 @@ context_lock_struct(rw_semaphore) {
#endif
};
+/* The opt-in lives in the padding inside the rtmutex. */
+static inline bool rwsem_is_tracked(const struct rw_semaphore *sem)
+{
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+ return sem->rwbase.rtmutex.tracked;
+#else
+ return false;
+#endif
+}
+
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+static inline void __rwsem_set_tracked(struct rw_semaphore *sem)
+{
+ sem->rwbase.rtmutex.tracked = true;
+}
+#endif
+
#define __RWSEM_INITIALIZER(name) \
{ \
.rwbase = __RWBASE_INITIALIZER(name), \
@@ -223,6 +268,22 @@ static inline void rwsem_assert_held_write(const struct rw_semaphore *sem)
rwsem_assert_held_write_nolockdep(sem);
}
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+/**
+ * rwsem_track_holder - track the holders of an rw_semaphore
+ * @sem: initialized rw_semaphore that no one holds yet
+ *
+ * Count @sem in task_nr_tracked_locks() of the tasks holding it. Use it
+ * for locks whose holders can stall unrelated work. Call it before anyone
+ * can take @sem, e.g. right after init_rwsem(). Do not use
+ * down_read_non_owner() or up_read_non_owner() on a tracked rwsem: they
+ * can run in different tasks, which a per-task count cannot follow.
+ */
+void rwsem_track_holder(struct rw_semaphore *sem);
+#else
+static inline void rwsem_track_holder(struct rw_semaphore *sem) { }
+#endif
+
/*
* lock for reading
*/
diff --git a/kernel/locking/rtmutex_common.h b/kernel/locking/rtmutex_common.h
index c38b7bdea7b3..2228ac11910a 100644
--- a/kernel/locking/rtmutex_common.h
+++ b/kernel/locking/rtmutex_common.h
@@ -180,6 +180,9 @@ enum rtmutex_chainwalk {
static inline void __rt_mutex_base_init(struct rt_mutex_base *lock)
{
+#if defined(CONFIG_TRACK_LOCK_HOLDERS) && defined(CONFIG_PREEMPT_RT)
+ lock->tracked = false;
+#endif
scoped_guard (raw_spinlock_init, &lock->wait_lock) {
lock->waiters = RB_ROOT_CACHED;
lock->owner = NULL;
diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c
index b9c180ac1eee..59e479a53f08 100644
--- a/kernel/locking/rwsem.c
+++ b/kernel/locking/rwsem.c
@@ -25,6 +25,7 @@
#include <linux/sched/signal.h>
#include <linux/sched/clock.h>
#include <linux/export.h>
+#include <linux/lockholder.h>
#include <linux/rwsem.h>
#include <linux/atomic.h>
#include <linux/hung_task.h>
@@ -85,7 +86,8 @@
* Bit 0 - writer locked bit
* Bit 1 - waiters present bit
* Bit 2 - lock handoff bit
- * Bits 3-7 - reserved
+ * Bit 3 - holder tracking opt-in (CONFIG_TRACK_LOCK_HOLDERS, sticky)
+ * Bits 4-7 - reserved
* Bits 8-62 - 55-bit reader count
* Bit 63 - read fail bit
*
@@ -94,7 +96,8 @@
* Bit 0 - writer locked bit
* Bit 1 - waiters present bit
* Bit 2 - lock handoff bit
- * Bits 3-7 - reserved
+ * Bit 3 - holder tracking opt-in (CONFIG_TRACK_LOCK_HOLDERS, sticky)
+ * Bits 4-7 - reserved
* Bits 8-30 - 23-bit reader count
* Bit 31 - read fail bit
*
@@ -261,11 +264,39 @@ static inline bool rwsem_read_trylock(struct rw_semaphore *sem, long *cntp)
return false;
}
+/*
+ * The opt-in flag as it sits in count, for the one path that folds it
+ * into a cmpxchg. The flag is only set once the static branch is on, so
+ * an off branch means no rwsem carries it and the read can be skipped.
+ */
+static inline long rwsem_tracked_flag(const struct rw_semaphore *sem)
+{
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+ if (!static_branch_unlikely(&lock_holder_tracking_key))
+ return 0;
+
+ return atomic_long_read(&sem->count) & RWSEM_FLAG_TRACKED;
+#else
+ return 0;
+#endif
+}
+
static inline bool rwsem_write_trylock(struct rw_semaphore *sem)
{
- long tmp = RWSEM_UNLOCKED_VALUE;
+ /*
+ * A free tracked rwsem reads RWSEM_FLAG_TRACKED, not 0, and the
+ * flag has to survive the acquire, so both the expected and the
+ * new value carry it. Reading it first costs one load but keeps
+ * this to a single cmpxchg; the flag is sticky, so the read cannot
+ * race. This is the only place the write side needs it: everywhere
+ * else count is only added to, or written back from a value that
+ * was read, so the flag survives.
+ */
+ long flag = rwsem_tracked_flag(sem);
+ long tmp = flag;
- if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, RWSEM_WRITER_LOCKED)) {
+ if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp,
+ flag | RWSEM_WRITER_LOCKED)) {
rwsem_set_owner(sem);
return true;
}
@@ -319,6 +350,7 @@ void __init_rwsem(struct rw_semaphore *sem, const char *name,
#ifdef CONFIG_DEBUG_RWSEMS
sem->magic = sem;
#endif
+ /* Clearing count also clears RWSEM_FLAG_TRACKED. */
atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE);
atomic_long_set(&sem->owner, 0L);
scoped_guard (raw_spinlock_init, &sem->wait_lock) {
@@ -1488,6 +1520,7 @@ static inline void __downgrade_write(struct rw_semaphore *sem)
void __init_rwsem(struct rw_semaphore *sem, const char *name,
struct lock_class_key *key)
{
+ /* init_rwbase_rt() also clears the holder tracking opt-in. */
init_rwbase_rt(&(sem)->rwbase);
#ifdef CONFIG_DEBUG_LOCK_ALLOC
@@ -1564,6 +1597,35 @@ static inline bool is_rwsem_reader_owned(struct rw_semaphore *sem)
#endif /* CONFIG_PREEMPT_RT */
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+void rwsem_track_holder(struct rw_semaphore *sem)
+{
+ /*
+ * A task holding @sem now was not counted, so its up_*() would make
+ * its count wrong.
+ */
+ if (WARN_ONCE(rwsem_is_locked(sem),
+ "%s: rwsem is held; opt in before it is published\n",
+ __func__))
+ return;
+
+ /*
+ * Turn the hooks on before the flag, so that a lock carrying the
+ * flag always has them on.
+ */
+ lock_holder_tracking_enable();
+ __rwsem_set_tracked(sem);
+}
+EXPORT_SYMBOL_GPL(rwsem_track_holder);
+#endif
+
+/*
+ * Holder tracking: the count goes up once down_*() has the lock and down
+ * before up_*() releases it. Waiting is not counted, as the task does not
+ * hold the lock yet. downgrade_write() keeps the lock held, so it needs no
+ * hook.
+ */
+
/*
* lock for reading
*/
@@ -1574,6 +1636,7 @@ void __sched down_read(struct rw_semaphore *sem)
rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
LOCK_CONTENDED(sem, __down_read_trylock, __down_read);
+ lock_holder_acquired_if(rwsem_is_tracked(sem));
}
EXPORT_SYMBOL(down_read);
@@ -1588,6 +1651,7 @@ int __sched down_read_interruptible(struct rw_semaphore *sem)
return -EINTR;
}
+ lock_holder_acquired_if(rwsem_is_tracked(sem));
return 0;
}
EXPORT_SYMBOL(down_read_interruptible);
@@ -1603,6 +1667,7 @@ int __sched down_read_killable(struct rw_semaphore *sem)
return -EINTR;
}
+ lock_holder_acquired_if(rwsem_is_tracked(sem));
return 0;
}
EXPORT_SYMBOL(down_read_killable);
@@ -1615,8 +1680,10 @@ int down_read_trylock(struct rw_semaphore *sem)
{
int ret = __down_read_trylock(sem);
- if (ret == 1)
+ if (ret == 1) {
rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_);
+ lock_holder_acquired_if(rwsem_is_tracked(sem));
+ }
return ret;
}
EXPORT_SYMBOL(down_read_trylock);
@@ -1630,6 +1697,7 @@ void __sched down_write(struct rw_semaphore *sem)
might_sleep();
rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
+ lock_holder_acquired_if(rwsem_is_tracked(sem));
}
EXPORT_SYMBOL(down_write);
@@ -1648,6 +1716,7 @@ int __sched down_write_killable(struct rw_semaphore *sem)
return -EINTR;
}
+ lock_holder_acquired_if(rwsem_is_tracked(sem));
return 0;
}
EXPORT_SYMBOL(down_write_killable);
@@ -1660,8 +1729,10 @@ int down_write_trylock(struct rw_semaphore *sem)
{
int ret = __down_write_trylock(sem);
- if (ret == 1)
+ if (ret == 1) {
rwsem_acquire(&sem->dep_map, 0, 1, _RET_IP_);
+ lock_holder_acquired_if(rwsem_is_tracked(sem));
+ }
return ret;
}
@@ -1673,6 +1744,7 @@ EXPORT_SYMBOL(down_write_trylock);
void up_read(struct rw_semaphore *sem)
__no_context_analysis
{
+ lock_holder_released_if(rwsem_is_tracked(sem));
rwsem_release(&sem->dep_map, _RET_IP_);
__up_read(sem);
}
@@ -1684,6 +1756,7 @@ EXPORT_SYMBOL(up_read);
void up_write(struct rw_semaphore *sem)
__no_context_analysis
{
+ lock_holder_released_if(rwsem_is_tracked(sem));
rwsem_release(&sem->dep_map, _RET_IP_);
__up_write(sem);
}
@@ -1708,6 +1781,7 @@ void down_read_nested(struct rw_semaphore *sem, int subclass)
might_sleep();
rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_);
LOCK_CONTENDED(sem, __down_read_trylock, __down_read);
+ lock_holder_acquired_if(rwsem_is_tracked(sem));
}
EXPORT_SYMBOL(down_read_nested);
@@ -1722,6 +1796,7 @@ int down_read_killable_nested(struct rw_semaphore *sem, int subclass)
return -EINTR;
}
+ lock_holder_acquired_if(rwsem_is_tracked(sem));
return 0;
}
EXPORT_SYMBOL(down_read_killable_nested);
@@ -1732,13 +1807,19 @@ void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest)
might_sleep();
rwsem_acquire_nest(&sem->dep_map, 0, 0, nest, _RET_IP_);
LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
+ lock_holder_acquired_if(rwsem_is_tracked(sem));
}
EXPORT_SYMBOL(_down_write_nest_lock);
+/*
+ * The non_owner calls can run in different tasks, which a per-task count
+ * cannot follow, so tracked rwsems must not use them.
+ */
void down_read_non_owner(struct rw_semaphore *sem)
__no_context_analysis
{
might_sleep();
+ WARN_ON_ONCE(rwsem_is_tracked(sem));
__down_read(sem);
/*
* The owner value for a reader-owned lock is mostly for debugging
@@ -1756,6 +1837,7 @@ void down_write_nested(struct rw_semaphore *sem, int subclass)
might_sleep();
rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
+ lock_holder_acquired_if(rwsem_is_tracked(sem));
}
EXPORT_SYMBOL(down_write_nested);
@@ -1771,6 +1853,7 @@ int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass)
return -EINTR;
}
+ lock_holder_acquired_if(rwsem_is_tracked(sem));
return 0;
}
EXPORT_SYMBOL(down_write_killable_nested);
@@ -1779,6 +1862,7 @@ void up_read_non_owner(struct rw_semaphore *sem)
__no_context_analysis
{
DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem);
+ WARN_ON_ONCE(rwsem_is_tracked(sem));
__up_read(sem);
}
EXPORT_SYMBOL(up_read_non_owner);
--
2.53.0-Meta
^ permalink raw reply [flat|nested] 8+ messages in thread* [PATCH 3/7] locking/mutex: track holders of opted-in mutexes
2026-09-23 5:01 [PATCH 0/7] locking: opt-in tracking of sleeping lock holders Shakeel Butt
2026-09-23 5:01 ` [PATCH 1/7] locking: add " Shakeel Butt
2026-09-23 5:01 ` [PATCH 2/7] locking/rwsem: track holders of opted-in rw_semaphores Shakeel Butt
@ 2026-09-23 5:01 ` Shakeel Butt
2026-09-23 5:01 ` [PATCH 4/7] locking/percpu-rwsem: track holders of opted-in percpu_rw_semaphores Shakeel Butt
` (3 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Shakeel Butt @ 2026-09-23 5:01 UTC (permalink / raw)
To: Peter Zijlstra, Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long
Cc: Paul McKenney, Greg Kroah-Hartman, Tejun Heo, Christian Brauner,
Sebastian Andrzej Siewior, Johannes Weiner, Jonathan Corbet,
Meta kernel team, cgroups, driver-core, linux-kernel
Add mutex_track_holder() to opt a mutex in to holder tracking.
On !PREEMPT_RT, current takes a mutex only in __mutex_trylock_common()
and __mutex_trylock_fast(), and releases it only in
__mutex_unlock_fast() and __mutex_unlock_slowpath(). Hooking these four
places covers every mutex_lock*(), mutex_trylock(), ww_mutex_lock*() and
handoff. __mutex_unlock_fast() drops the count after the release, which
needs no care: the count is in current, not in the mutex.
On PREEMPT_RT, the hooks are in __mutex_lock_common(), mutex_trylock(),
_mutex_trylock_nest_lock() and mutex_unlock() in rtmutex_api.c. There,
ww_mutex is an rt_mutex and is not covered. mutex_track_holder() itself
is shared by both builds.
As with rwsem, mutex_track_holder() warns and does nothing if the mutex
is held, and mutex_init() clears the flag with the rest of ->owner.
The opt-in is bit 3 of ->owner, which is below the alignment of a
task_struct pointer. It is sticky: unlike the other flag bits it
survives unlock, so a free tracked mutex reads MUTEX_FLAG_TRACKED rather
than zero.
Bit 3 needs a task_struct aligned to 16 rather than the 8 that bits 0-2
needed. fork_init() aligns to at least L1_CACHE_BYTES, whose smallest
value in the tree is 16, so a static_assert() records the requirement.
Signed-off-by: Shakeel Butt <shakeel.butt@linux.dev>
---
include/linux/mutex.h | 14 +++++++
kernel/locking/mutex.c | 79 ++++++++++++++++++++++++++++++++----
kernel/locking/mutex.h | 48 +++++++++++++++++++++-
kernel/locking/rtmutex_api.c | 36 ++++++++++++++--
4 files changed, 165 insertions(+), 12 deletions(-)
diff --git a/include/linux/mutex.h b/include/linux/mutex.h
index 734048c02f4f..74f54ee83685 100644
--- a/include/linux/mutex.h
+++ b/include/linux/mutex.h
@@ -75,6 +75,20 @@ do { \
*/
#define mutex_init_with_key(mutex, key) __mutex_init((mutex), #mutex, (key))
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+/**
+ * mutex_track_holder - track the holder of a mutex
+ * @lock: initialized mutex that no one holds yet
+ *
+ * Count @lock in task_nr_tracked_locks() of the task holding it. Use it
+ * for locks whose holders can stall unrelated work. Call it before anyone
+ * can take @lock, e.g. right after mutex_init().
+ */
+void mutex_track_holder(struct mutex *lock);
+#else
+static inline void mutex_track_holder(struct mutex *lock) { }
+#endif
+
#ifndef CONFIG_PREEMPT_RT
#define __MUTEX_INITIALIZER(lockname) \
{ .owner = ATOMIC_LONG_INIT(0) \
diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index 942a939cee95..d4b5a3674e39 100644
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -28,6 +28,7 @@
#include <linux/spinlock.h>
#include <linux/interrupt.h>
#include <linux/debug_locks.h>
+#include <linux/lockholder.h>
#include <linux/osq_lock.h>
#include <linux/hung_task.h>
@@ -45,6 +46,7 @@
static void __mutex_init_generic(struct mutex *lock)
{
+ /* Clearing owner also clears MUTEX_FLAG_TRACKED. */
atomic_long_set(&lock->owner, 0);
scoped_guard (raw_spinlock_init, &lock->wait_lock) {
lock->first_waiter = NULL;
@@ -81,6 +83,9 @@ unsigned long mutex_get_owner(struct mutex *lock)
/*
* Returns: __mutex_owner(lock) on failure or NULL on success.
+ *
+ * Only this and __mutex_trylock_fast() take the lock for current, so the
+ * holder count goes up in these two.
*/
static inline struct task_struct *__mutex_trylock_common(struct mutex *lock, bool handoff)
{
@@ -109,8 +114,10 @@ static inline struct task_struct *__mutex_trylock_common(struct mutex *lock, boo
}
if (atomic_long_try_cmpxchg_acquire(&lock->owner, &owner, task | flags)) {
- if (task == curr)
+ if (task == curr) {
+ lock_holder_acquired_if(flags & MUTEX_FLAG_TRACKED);
return NULL;
+ }
break;
}
}
@@ -154,12 +161,28 @@ static __always_inline bool __mutex_trylock_fast(struct mutex *lock)
__cond_acquires(true, lock)
{
unsigned long curr = (unsigned long)current;
- unsigned long zero = 0UL;
+ /*
+ * A free tracked mutex reads MUTEX_FLAG_TRACKED, not 0, and the
+ * flag has to survive the acquire, so both the expected and the
+ * new value carry it. Reading it first keeps this to a single
+ * cmpxchg, and the read cannot race because the flag is sticky.
+ */
+ unsigned long flag = mutex_tracked_flag(lock);
+ unsigned long expected = flag;
MUTEX_WARN_ON(lock->magic != lock);
- if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr))
+ if (atomic_long_try_cmpxchg_acquire(&lock->owner, &expected, curr | flag)) {
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+ /*
+ * The flag is only set once the static branch is on, so a
+ * set flag means tracking is enabled.
+ */
+ if (flag)
+ __lock_holder_acquired();
+#endif
return true;
+ }
return false;
}
@@ -168,8 +191,23 @@ static __always_inline bool __mutex_unlock_fast(struct mutex *lock)
__cond_releases(true, lock)
{
unsigned long curr = (unsigned long)current;
+ /*
+ * A tracked mutex unlocks to MUTEX_FLAG_TRACKED, not to 0. Read the
+ * flag while the mutex is still held, so this needs only a single
+ * cmpxchg and never looks at @lock after giving it away.
+ */
+ unsigned long flag = mutex_tracked_flag(lock);
+ unsigned long owner = curr | flag;
+
+ if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, flag)) {
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+ if (flag)
+ __lock_holder_released();
+#endif
+ return true;
+ }
- return atomic_long_try_cmpxchg_release(&lock->owner, &curr, 0UL);
+ return false;
}
#else /* !CONFIG_DEBUG_LOCK_ALLOC */
@@ -242,7 +280,7 @@ __mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter)
__must_hold(&lock->wait_lock)
{
if (list_empty(&waiter->list)) {
- __mutex_clear_flag(lock, MUTEX_FLAGS);
+ __mutex_clear_flag(lock, MUTEX_STATE_FLAGS);
lock->first_waiter = NULL;
} else {
if (lock->first_waiter == waiter)
@@ -257,7 +295,7 @@ __mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter)
/*
* Give up ownership to a specific task, when @task = NULL, this is equivalent
* to a regular unlock. Sets PICKUP on a handoff, clears HANDOFF, preserves
- * WAITERS. Provides RELEASE semantics like a regular unlock, the
+ * WAITERS and TRACKED. Provides RELEASE semantics like a regular unlock, the
* __mutex_trylock() provides a matching ACQUIRE semantics for the handoff.
*/
static void __mutex_handoff(struct mutex *lock, struct task_struct *task)
@@ -270,7 +308,7 @@ static void __mutex_handoff(struct mutex *lock, struct task_struct *task)
MUTEX_WARN_ON(__owner_task(owner) != current);
MUTEX_WARN_ON(owner & MUTEX_FLAG_PICKUP);
- new = (owner & MUTEX_FLAG_WAITERS);
+ new = (owner & (MUTEX_FLAG_WAITERS | MUTEX_FLAG_TRACKED));
new |= (unsigned long)task;
if (task)
new |= MUTEX_FLAG_PICKUP;
@@ -986,6 +1024,7 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
unsigned long owner;
unsigned long flags;
+ lock_holder_released_if(mutex_is_tracked(lock));
mutex_release(&lock->dep_map, ip);
__release(lock);
@@ -1276,6 +1315,32 @@ __weak int arch_contended_release_trace_reg(void) { return 0; }
__weak void arch_contended_release_trace_unreg(void) { }
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+void mutex_track_holder(struct mutex *lock)
+{
+ /*
+ * A task holding @lock now was not counted, so its mutex_unlock()
+ * would make its count wrong.
+ */
+ if (WARN_ONCE(mutex_is_locked(lock),
+ "%s: mutex is held; opt in before it is published\n",
+ __func__))
+ return;
+
+ /*
+ * Turn the hooks on before the flag, so that a lock carrying the
+ * flag always has them on.
+ */
+ lock_holder_tracking_enable();
+#ifdef CONFIG_PREEMPT_RT
+ lock->rtmutex.tracked = true;
+#else
+ atomic_long_or(MUTEX_FLAG_TRACKED, &lock->owner);
+#endif
+}
+EXPORT_SYMBOL_GPL(mutex_track_holder);
+#endif
+
/**
* atomic_dec_and_mutex_lock - return holding mutex if we dec to 0
* @cnt: the atomic which we are to dec
diff --git a/kernel/locking/mutex.h b/kernel/locking/mutex.h
index 3e263e98e5fc..833f084d5bb0 100644
--- a/kernel/locking/mutex.h
+++ b/kernel/locking/mutex.h
@@ -7,6 +7,7 @@
* Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
*/
#ifndef CONFIG_PREEMPT_RT
+#include <linux/lockholder.h>
#include <linux/mutex.h>
/*
* This is the control structure for tasks blocked on mutex, which resides
@@ -29,12 +30,32 @@ struct mutex_waiter {
* Bit0 indicates a non-empty waiter list; unlock must issue a wakeup.
* Bit1 indicates unlock needs to hand the lock to the top-waiter
* Bit2 indicates handoff has been done and we're waiting for pickup.
+ * Bit3 indicates the lock opted in to holder tracking. Unlike the others
+ * it is sticky: it survives unlock, so an unlocked tracked mutex reads
+ * MUTEX_FLAG_TRACKED rather than 0.
*/
#define MUTEX_FLAG_WAITERS 0x01
#define MUTEX_FLAG_HANDOFF 0x02
#define MUTEX_FLAG_PICKUP 0x04
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+#define MUTEX_FLAG_TRACKED 0x08
+#else
+#define MUTEX_FLAG_TRACKED 0x00
+#endif
+
+/* The state flags, which unlock clears. */
+#define MUTEX_STATE_FLAGS (MUTEX_FLAG_WAITERS | MUTEX_FLAG_HANDOFF | \
+ MUTEX_FLAG_PICKUP)
+#define MUTEX_FLAGS (MUTEX_STATE_FLAGS | MUTEX_FLAG_TRACKED)
-#define MUTEX_FLAGS 0x07
+/*
+ * The flags live below the task_struct pointer in ->owner, so every
+ * task_struct has to be aligned past them. Bits 0-2 needed 8 bytes;
+ * MUTEX_FLAG_TRACKED needs 16. fork_init() aligns task_struct to at
+ * least L1_CACHE_BYTES and init_task is __aligned(L1_CACHE_BYTES), so
+ * that is the value to check; its smallest value in the tree is 16.
+ */
+static_assert(L1_CACHE_BYTES > MUTEX_FLAGS);
/*
* Internal helper function; C doesn't allow us to hide it :/
@@ -48,6 +69,31 @@ static inline struct task_struct *__mutex_owner(struct mutex *lock)
return (struct task_struct *)(atomic_long_read(&lock->owner) & ~MUTEX_FLAGS);
}
+static inline bool mutex_is_tracked(struct mutex *lock)
+{
+ if (!IS_ENABLED(CONFIG_TRACK_LOCK_HOLDERS))
+ return false;
+
+ return atomic_long_read(&lock->owner) & MUTEX_FLAG_TRACKED;
+}
+
+/*
+ * The opt-in flag as it sits in owner, for the fast paths that fold it
+ * into a cmpxchg. The flag is only set once the static branch is on, so
+ * an off branch means no mutex carries it and the read can be skipped.
+ */
+static inline unsigned long mutex_tracked_flag(struct mutex *lock)
+{
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+ if (!static_branch_unlikely(&lock_holder_tracking_key))
+ return 0;
+
+ return atomic_long_read(&lock->owner) & MUTEX_FLAG_TRACKED;
+#else
+ return 0;
+#endif
+}
+
static inline struct mutex *get_task_blocked_on(struct task_struct *p)
{
guard(raw_spinlock_irqsave)(&p->blocked_lock);
diff --git a/kernel/locking/rtmutex_api.c b/kernel/locking/rtmutex_api.c
index eb18b094473c..41ca3c1d92f6 100644
--- a/kernel/locking/rtmutex_api.c
+++ b/kernel/locking/rtmutex_api.c
@@ -2,6 +2,7 @@
/*
* rtmutex API
*/
+#include <linux/lockholder.h>
#include <linux/spinlock.h>
#include <linux/export.h>
@@ -544,10 +545,26 @@ void rt_mutex_debug_task_free(struct task_struct *task)
/* Mutexes */
static void __mutex_rt_init_generic(struct mutex *mutex)
{
+ /* rt_mutex_base_init() also clears the holder tracking opt-in. */
rt_mutex_base_init(&mutex->rtmutex);
debug_check_no_locks_freed((void *)mutex, sizeof(*mutex));
}
+/*
+ * mutex_track_holder() is in mutex.c, shared with !PREEMPT_RT. Here the
+ * opt-in is a flag in the padding inside the rtmutex, not a bit of
+ * ->owner: rt_mutex clears ->owner on unlock, so a bit there would not
+ * survive.
+ */
+static inline bool mutex_is_tracked(struct mutex *lock)
+{
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+ return lock->rtmutex.tracked;
+#else
+ return false;
+#endif
+}
+
static __always_inline int __mutex_lock_common(struct mutex *lock,
unsigned int state,
unsigned int subclass,
@@ -560,10 +577,12 @@ static __always_inline int __mutex_lock_common(struct mutex *lock,
might_sleep();
mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
ret = __rt_mutex_lock(&lock->rtmutex, state);
- if (ret)
+ if (ret) {
mutex_release(&lock->dep_map, ip);
- else
+ } else {
lock_acquired(&lock->dep_map, ip);
+ lock_holder_acquired_if(mutex_is_tracked(lock));
+ }
return ret;
}
@@ -623,8 +642,10 @@ int __sched _mutex_trylock_nest_lock(struct mutex *lock,
return 0;
ret = __rt_mutex_trylock(&lock->rtmutex);
- if (ret)
+ if (ret) {
mutex_acquire_nest(&lock->dep_map, 0, 1, nest_lock, _RET_IP_);
+ lock_holder_acquired_if(mutex_is_tracked(lock));
+ }
return ret;
}
@@ -666,10 +687,16 @@ EXPORT_SYMBOL(mutex_lock_io);
int __sched mutex_trylock(struct mutex *lock)
{
+ int ret;
+
if (IS_ENABLED(CONFIG_DEBUG_RT_MUTEXES) && WARN_ON_ONCE(!in_task()))
return 0;
- return __rt_mutex_trylock(&lock->rtmutex);
+ ret = __rt_mutex_trylock(&lock->rtmutex);
+ if (ret)
+ lock_holder_acquired_if(mutex_is_tracked(lock));
+
+ return ret;
}
EXPORT_SYMBOL(mutex_trylock);
#endif /* !CONFIG_DEBUG_LOCK_ALLOC */
@@ -677,6 +704,7 @@ EXPORT_SYMBOL(mutex_trylock);
void __sched mutex_unlock(struct mutex *lock)
__releases(lock) __no_context_analysis
{
+ lock_holder_released_if(mutex_is_tracked(lock));
mutex_release(&lock->dep_map, _RET_IP_);
__rt_mutex_unlock(&lock->rtmutex);
}
--
2.53.0-Meta
^ permalink raw reply [flat|nested] 8+ messages in thread* [PATCH 4/7] locking/percpu-rwsem: track holders of opted-in percpu_rw_semaphores
2026-09-23 5:01 [PATCH 0/7] locking: opt-in tracking of sleeping lock holders Shakeel Butt
` (2 preceding siblings ...)
2026-09-23 5:01 ` [PATCH 3/7] locking/mutex: track holders of opted-in mutexes Shakeel Butt
@ 2026-09-23 5:01 ` Shakeel Butt
2026-09-23 5:01 ` [PATCH 5/7] locking/selftests: add KUnit tests for lock holder tracking Shakeel Butt
` (2 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Shakeel Butt @ 2026-09-23 5:01 UTC (permalink / raw)
To: Peter Zijlstra, Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long
Cc: Paul McKenney, Greg Kroah-Hartman, Tejun Heo, Christian Brauner,
Sebastian Andrzej Siewior, Johannes Weiner, Jonathan Corbet,
Meta kernel team, cgroups, driver-core, linux-kernel
Add percpu_rwsem_track_holder() to opt a percpu_rw_semaphore in to
holder tracking.
The read fast and slow paths meet before the hook in
percpu_down_read_internal(), so one hook covers both.
percpu_down_read_trylock() counts only on success.
percpu_rwsem_release() and percpu_rwsem_acquire() pass a held lock to
another task. Filesystem freeze uses them to return to user space
holding the lock, and async writes (kiocb_start_write()) use them
because the write may complete in another context. Move the count in
these calls too, so it stays right for each task.
The read side counts inside the preempt-disabled region, so a task is
never preemptible holding @sem without being counted for it.
Signed-off-by: Shakeel Butt <shakeel.butt@linux.dev>
---
include/linux/percpu-rwsem.h | 42 +++++++++++++++++++++++++++++++++++
kernel/locking/percpu-rwsem.c | 28 +++++++++++++++++++++++
2 files changed, 70 insertions(+)
diff --git a/include/linux/percpu-rwsem.h b/include/linux/percpu-rwsem.h
index 39d5bf8e6562..1a131ed8e8f1 100644
--- a/include/linux/percpu-rwsem.h
+++ b/include/linux/percpu-rwsem.h
@@ -8,6 +8,7 @@
#include <linux/wait.h>
#include <linux/rcu_sync.h>
#include <linux/lockdep.h>
+#include <linux/lockholder.h>
#include <linux/cleanup.h>
struct percpu_rw_semaphore {
@@ -16,6 +17,13 @@ struct percpu_rw_semaphore {
struct rcuwait writer;
wait_queue_head_t waiters;
atomic_t block;
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+ /*
+ * Set by percpu_rwsem_track_holder() before first use. Fits in the
+ * padding after block, so the struct does not grow on 64-bit.
+ */
+ bool tracked;
+#endif
#ifdef CONFIG_DEBUG_LOCK_ALLOC
struct lockdep_map dep_map;
#endif
@@ -65,6 +73,11 @@ static inline void percpu_down_read_internal(struct percpu_rw_semaphore *sem,
this_cpu_inc(*sem->read_count);
else
__percpu_down_read(sem, false, freezable); /* Unconditional memory barrier */
+ /*
+ * Count the lock before preemption comes back, so the task is
+ * never preemptible holding @sem but not counted for it.
+ */
+ lock_holder_acquired(sem);
/*
* The preempt_enable() prevents the compiler from
* bleeding the critical section out.
@@ -95,6 +108,12 @@ static inline bool percpu_down_read_trylock(struct percpu_rw_semaphore *sem)
this_cpu_inc(*sem->read_count);
else
ret = __percpu_down_read(sem, true, false); /* Unconditional memory barrier */
+ /*
+ * Count the lock before preemption comes back, as in
+ * percpu_down_read_internal().
+ */
+ if (ret)
+ lock_holder_acquired(sem);
preempt_enable();
/*
* The barrier() from preempt_enable() prevents the compiler from
@@ -111,6 +130,7 @@ extern void __percpu_up_read(struct percpu_rw_semaphore *sem);
static inline void percpu_up_read(struct percpu_rw_semaphore *sem)
{
+ lock_holder_released(sem);
rwsem_release(&sem->dep_map, _RET_IP_);
preempt_disable();
@@ -129,6 +149,20 @@ extern bool percpu_is_read_locked(struct percpu_rw_semaphore *);
extern void percpu_down_write(struct percpu_rw_semaphore *);
extern void percpu_up_write(struct percpu_rw_semaphore *);
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+/**
+ * percpu_rwsem_track_holder - track the holders of a percpu_rw_semaphore
+ * @sem: initialized percpu_rw_semaphore that no one holds yet
+ *
+ * Count @sem in task_nr_tracked_locks() of the tasks holding it. Use it
+ * for locks whose holders can stall unrelated work. Call it before anyone
+ * can take @sem, e.g. right after percpu_init_rwsem().
+ */
+void percpu_rwsem_track_holder(struct percpu_rw_semaphore *sem);
+#else
+static inline void percpu_rwsem_track_holder(struct percpu_rw_semaphore *sem) { }
+#endif
+
DEFINE_GUARD(percpu_read, struct percpu_rw_semaphore *,
percpu_down_read(_T), percpu_up_read(_T))
DEFINE_GUARD_COND(percpu_read, _try, percpu_down_read_trylock(_T))
@@ -156,9 +190,16 @@ extern void percpu_free_rwsem(struct percpu_rw_semaphore *);
#define percpu_rwsem_is_held(sem) lockdep_is_held(sem)
#define percpu_rwsem_assert_held(sem) lockdep_assert_held(sem)
+/*
+ * These pass a held lock to another task. For example, filesystem freeze
+ * returns to user space holding the lock, and thaw, maybe in another
+ * task, releases it. The holder count moves with the lock, as lockdep's
+ * view of it does.
+ */
static inline void percpu_rwsem_release(struct percpu_rw_semaphore *sem,
unsigned long ip)
{
+ lock_holder_released(sem);
lock_release(&sem->dep_map, ip);
}
@@ -166,6 +207,7 @@ static inline void percpu_rwsem_acquire(struct percpu_rw_semaphore *sem,
bool read, unsigned long ip)
{
lock_acquire(&sem->dep_map, 0, 1, read, 1, NULL, ip);
+ lock_holder_acquired(sem);
}
#endif
diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c
index 6c78961fe753..e100c33731ca 100644
--- a/kernel/locking/percpu-rwsem.c
+++ b/kernel/locking/percpu-rwsem.c
@@ -22,6 +22,9 @@ int __percpu_init_rwsem(struct percpu_rw_semaphore *sem,
rcuwait_init(&sem->writer);
init_waitqueue_head(&sem->waiters);
atomic_set(&sem->block, 0);
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+ sem->tracked = false;
+#endif
#ifdef CONFIG_DEBUG_LOCK_ALLOC
debug_check_no_locks_freed((void *)sem, sizeof(*sem));
lockdep_init_map(&sem->dep_map, name, key, 0);
@@ -201,6 +204,28 @@ bool percpu_is_read_locked(struct percpu_rw_semaphore *sem)
}
EXPORT_SYMBOL_GPL(percpu_is_read_locked);
+#ifdef CONFIG_TRACK_LOCK_HOLDERS
+void percpu_rwsem_track_holder(struct percpu_rw_semaphore *sem)
+{
+ /*
+ * A task holding @sem now was not counted, so its percpu_up_*()
+ * would make its count wrong.
+ */
+ if (WARN_ONCE(percpu_is_read_locked(sem) || percpu_is_write_locked(sem),
+ "%s: rwsem is held; opt in before it is published\n",
+ __func__))
+ return;
+
+ /*
+ * Turn the hooks on before the flag, so that a lock carrying the
+ * flag always has them on.
+ */
+ lock_holder_tracking_enable();
+ sem->tracked = true;
+}
+EXPORT_SYMBOL_GPL(percpu_rwsem_track_holder);
+#endif
+
/*
* Return true if the modular sum of the sem->read_count per-CPU variable is
* zero. If this sum is zero, then it is stable due to the fact that if any
@@ -256,11 +281,14 @@ void __sched percpu_down_write(struct percpu_rw_semaphore *sem)
rcuwait_wait_event(&sem->writer, readers_active_check(sem), TASK_UNINTERRUPTIBLE);
if (contended)
trace_contention_end(sem, 0);
+
+ lock_holder_acquired(sem);
}
EXPORT_SYMBOL_GPL(percpu_down_write);
void percpu_up_write(struct percpu_rw_semaphore *sem)
{
+ lock_holder_released(sem);
rwsem_release(&sem->dep_map, _RET_IP_);
if (trace_contended_release_enabled() && wq_has_sleeper(&sem->waiters))
--
2.53.0-Meta
^ permalink raw reply [flat|nested] 8+ messages in thread* [PATCH 5/7] locking/selftests: add KUnit tests for lock holder tracking
2026-09-23 5:01 [PATCH 0/7] locking: opt-in tracking of sleeping lock holders Shakeel Butt
` (3 preceding siblings ...)
2026-09-23 5:01 ` [PATCH 4/7] locking/percpu-rwsem: track holders of opted-in percpu_rw_semaphores Shakeel Butt
@ 2026-09-23 5:01 ` Shakeel Butt
2026-09-23 5:01 ` [PATCH 6/7] Documentation/locking: document sleeping " Shakeel Butt
2026-09-23 5:01 ` [PATCH 7/7] kernfs, cgroup: track holders of the cgroupfs locks Shakeel Butt
6 siblings, 0 replies; 8+ messages in thread
From: Shakeel Butt @ 2026-09-23 5:01 UTC (permalink / raw)
To: Peter Zijlstra, Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long
Cc: Paul McKenney, Greg Kroah-Hartman, Tejun Heo, Christian Brauner,
Sebastian Andrzej Siewior, Johannes Weiner, Jonathan Corbet,
Meta kernel team, cgroups, driver-core, linux-kernel
Add 16 KUnit tests that check task_nr_tracked_locks(current) always
equals the number of opted-in locks the task holds. They cover:
- locks that did not opt in, which must not count
- the mutex, rw_semaphore and percpu_rw_semaphore lock and trylock
calls, including the killable, interruptible, freezable and io ones
- downgrade_write(), the cleanup.h guards, and nested locks released
out of order
- the percpu_rwsem_release()/percpu_rwsem_acquire() hand-off
- re-init clearing the opt-in
Five tests use a helper thread that holds a lock. They check that a
failed trylock does not count, that a lock taken right as the helper
releases it (usually through the slow path) counts exactly once, and
that each task has its own count.
Three cases cover what the opt-in changes besides the count: that an
opted-in but unheld lock still reads unlocked, which the sticky bit
would otherwise break for rwsem_is_locked() and its callers; that
*_track_holder() refuses a lock that is already held; and that the
non_owner rwsem calls warn and do not count. The last needs
CONFIG_DEBUG_LOCK_ALLOC, as without it they are plain down_read() and
up_read(), so it skips otherwise.
Tests compare against the count at the start of the test, not zero.
Passes on UML, x86_64, x86_64 with PROVE_LOCKING + DEBUG_MUTEXES +
DEBUG_RWSEMS (where the mutex fast path is compiled out), and x86_64
PREEMPT_RT:
tools/testing/kunit/kunit.py run --kunitconfig=kernel/locking/ lockholder
Signed-off-by: Shakeel Butt <shakeel.butt@linux.dev>
---
kernel/locking/.kunitconfig | 4 +
kernel/locking/Makefile | 1 +
kernel/locking/lockholder_kunit.c | 741 ++++++++++++++++++++++++++++++
lib/Kconfig.debug | 13 +
4 files changed, 759 insertions(+)
create mode 100644 kernel/locking/.kunitconfig
create mode 100644 kernel/locking/lockholder_kunit.c
diff --git a/kernel/locking/.kunitconfig b/kernel/locking/.kunitconfig
new file mode 100644
index 000000000000..ed243843b8b0
--- /dev/null
+++ b/kernel/locking/.kunitconfig
@@ -0,0 +1,4 @@
+CONFIG_KUNIT=y
+CONFIG_SMP=y
+CONFIG_TRACK_LOCK_HOLDERS=y
+CONFIG_TRACK_LOCK_HOLDERS_KUNIT_TEST=y
diff --git a/kernel/locking/Makefile b/kernel/locking/Makefile
index a0945beb304c..d23b754df40f 100644
--- a/kernel/locking/Makefile
+++ b/kernel/locking/Makefile
@@ -10,6 +10,7 @@ CONTEXT_ANALYSIS_rwsem.o := y
obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o
obj-$(CONFIG_TRACK_LOCK_HOLDERS) += lockholder.o
+obj-$(CONFIG_TRACK_LOCK_HOLDERS_KUNIT_TEST) += lockholder_kunit.o
# Avoid recursion lockdep -> sanitizer -> ... -> lockdep & improve performance.
KASAN_SANITIZE_lockdep.o := n
diff --git a/kernel/locking/lockholder_kunit.c b/kernel/locking/lockholder_kunit.c
new file mode 100644
index 000000000000..6b3c3a66f7df
--- /dev/null
+++ b/kernel/locking/lockholder_kunit.c
@@ -0,0 +1,741 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KUnit tests for sleeping lock holder tracking: check that
+ * task_nr_tracked_locks(current) always equals the number of opted-in
+ * locks the task holds.
+ */
+#include <kunit/test.h>
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/kthread.h>
+#include <linux/lockholder.h>
+#include <linux/mutex.h>
+#include <linux/percpu-rwsem.h>
+#include <linux/rwsem.h>
+#include <linux/sched.h>
+
+struct lockholder_ctx {
+ struct mutex tmutex;
+ struct mutex tmutex2;
+ struct mutex pmutex;
+ struct rw_semaphore tsem;
+ struct rw_semaphore tsem2;
+ struct rw_semaphore psem;
+ struct percpu_rw_semaphore tpcpu;
+ struct percpu_rw_semaphore ppcpu;
+ bool pcpu_ready;
+
+ /* Count at test start. Tests compare against it, not against zero. */
+ unsigned int base;
+
+ /* Helper thread that holds a lock until told to release it. */
+ struct completion helper_holds;
+ struct completion helper_may_release;
+ struct completion helper_done;
+ unsigned int helper_depth_held;
+ unsigned int helper_depth_after;
+};
+
+/* Check the count relative to the start of the test. */
+#define EXPECT_DEPTH(test, ctx, n) \
+ KUNIT_EXPECT_EQ((test), (ctx)->base + (unsigned int)(n), \
+ task_nr_tracked_locks(current))
+
+#define ASSERT_DEPTH(test, ctx, n) \
+ KUNIT_ASSERT_EQ((test), (ctx)->base + (unsigned int)(n), \
+ task_nr_tracked_locks(current))
+
+static int lockholder_init(struct kunit *test)
+{
+ struct lockholder_ctx *ctx;
+
+ ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+ mutex_init(&ctx->tmutex);
+ mutex_init(&ctx->tmutex2);
+ mutex_init(&ctx->pmutex);
+ init_rwsem(&ctx->tsem);
+ init_rwsem(&ctx->tsem2);
+ init_rwsem(&ctx->psem);
+
+ mutex_track_holder(&ctx->tmutex);
+ mutex_track_holder(&ctx->tmutex2);
+ rwsem_track_holder(&ctx->tsem);
+ rwsem_track_holder(&ctx->tsem2);
+
+ KUNIT_ASSERT_EQ(test, 0, percpu_init_rwsem(&ctx->tpcpu));
+ if (percpu_init_rwsem(&ctx->ppcpu)) {
+ percpu_free_rwsem(&ctx->tpcpu);
+ KUNIT_ASSERT_TRUE(test, false);
+ }
+ ctx->pcpu_ready = true;
+ percpu_rwsem_track_holder(&ctx->tpcpu);
+
+ init_completion(&ctx->helper_holds);
+ init_completion(&ctx->helper_may_release);
+ init_completion(&ctx->helper_done);
+
+ ctx->base = task_nr_tracked_locks(current);
+ test->priv = ctx;
+ return 0;
+}
+
+static void lockholder_exit(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+
+ if (ctx && ctx->pcpu_ready) {
+ percpu_free_rwsem(&ctx->tpcpu);
+ percpu_free_rwsem(&ctx->ppcpu);
+ ctx->pcpu_ready = false;
+ }
+}
+
+/* Locks that did not opt in are never counted. */
+static void untracked_locks_are_not_counted(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+
+ mutex_lock(&ctx->pmutex);
+ EXPECT_DEPTH(test, ctx, 0);
+ mutex_unlock(&ctx->pmutex);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ KUNIT_EXPECT_TRUE(test, mutex_trylock(&ctx->pmutex));
+ EXPECT_DEPTH(test, ctx, 0);
+ mutex_unlock(&ctx->pmutex);
+
+ down_read(&ctx->psem);
+ EXPECT_DEPTH(test, ctx, 0);
+ up_read(&ctx->psem);
+
+ down_write(&ctx->psem);
+ EXPECT_DEPTH(test, ctx, 0);
+ up_write(&ctx->psem);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ percpu_down_read(&ctx->ppcpu);
+ EXPECT_DEPTH(test, ctx, 0);
+ percpu_up_read(&ctx->ppcpu);
+
+ percpu_down_write(&ctx->ppcpu);
+ EXPECT_DEPTH(test, ctx, 0);
+ percpu_up_write(&ctx->ppcpu);
+ EXPECT_DEPTH(test, ctx, 0);
+}
+
+static void tracked_mutex_is_counted(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+
+ mutex_lock(&ctx->tmutex);
+ EXPECT_DEPTH(test, ctx, 1);
+ mutex_unlock(&ctx->tmutex);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ KUNIT_ASSERT_EQ(test, 0, mutex_lock_interruptible(&ctx->tmutex));
+ EXPECT_DEPTH(test, ctx, 1);
+ mutex_unlock(&ctx->tmutex);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ KUNIT_ASSERT_EQ(test, 0, mutex_lock_killable(&ctx->tmutex));
+ EXPECT_DEPTH(test, ctx, 1);
+ mutex_unlock(&ctx->tmutex);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ mutex_lock_io(&ctx->tmutex);
+ EXPECT_DEPTH(test, ctx, 1);
+ mutex_unlock(&ctx->tmutex);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ KUNIT_ASSERT_TRUE(test, mutex_trylock(&ctx->tmutex));
+ EXPECT_DEPTH(test, ctx, 1);
+ mutex_unlock(&ctx->tmutex);
+ EXPECT_DEPTH(test, ctx, 0);
+}
+
+static void tracked_rwsem_read_is_counted(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+
+ down_read(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 1);
+ up_read(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ KUNIT_ASSERT_EQ(test, 0, down_read_killable(&ctx->tsem));
+ EXPECT_DEPTH(test, ctx, 1);
+ up_read(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ KUNIT_ASSERT_EQ(test, 0, down_read_interruptible(&ctx->tsem));
+ EXPECT_DEPTH(test, ctx, 1);
+ up_read(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ KUNIT_ASSERT_EQ(test, 1, down_read_trylock(&ctx->tsem));
+ EXPECT_DEPTH(test, ctx, 1);
+ up_read(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 0);
+}
+
+static void tracked_rwsem_write_is_counted(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+
+ down_write(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 1);
+ up_write(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ KUNIT_ASSERT_EQ(test, 0, down_write_killable(&ctx->tsem));
+ EXPECT_DEPTH(test, ctx, 1);
+ up_write(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ KUNIT_ASSERT_EQ(test, 1, down_write_trylock(&ctx->tsem));
+ EXPECT_DEPTH(test, ctx, 1);
+ up_write(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 0);
+}
+
+static void tracked_percpu_rwsem_is_counted(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+
+ percpu_down_read(&ctx->tpcpu);
+ EXPECT_DEPTH(test, ctx, 1);
+ percpu_up_read(&ctx->tpcpu);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ percpu_down_read_freezable(&ctx->tpcpu, false);
+ EXPECT_DEPTH(test, ctx, 1);
+ percpu_up_read(&ctx->tpcpu);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ KUNIT_ASSERT_TRUE(test, percpu_down_read_trylock(&ctx->tpcpu));
+ EXPECT_DEPTH(test, ctx, 1);
+ percpu_up_read(&ctx->tpcpu);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ percpu_down_write(&ctx->tpcpu);
+ EXPECT_DEPTH(test, ctx, 1);
+ percpu_up_write(&ctx->tpcpu);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ scoped_guard(percpu_read, &ctx->tpcpu)
+ EXPECT_DEPTH(test, ctx, 1);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ scoped_guard(percpu_write, &ctx->tpcpu)
+ EXPECT_DEPTH(test, ctx, 1);
+ EXPECT_DEPTH(test, ctx, 0);
+}
+
+/*
+ * percpu_rwsem_release()/percpu_rwsem_acquire() move the count with the
+ * lock.
+ */
+static void percpu_rwsem_handover_moves_the_count(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+
+ percpu_down_write(&ctx->tpcpu);
+ ASSERT_DEPTH(test, ctx, 1);
+
+ /* As if returning to user space with the lock held. */
+ percpu_rwsem_release(&ctx->tpcpu, _THIS_IP_);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ /* Take it back before releasing it. */
+ percpu_rwsem_acquire(&ctx->tpcpu, false, _THIS_IP_);
+ EXPECT_DEPTH(test, ctx, 1);
+
+ percpu_up_write(&ctx->tpcpu);
+ EXPECT_DEPTH(test, ctx, 0);
+}
+
+/* The percpu read slow path, used right after a writer, is counted too. */
+static void percpu_rwsem_slow_read_path_is_counted(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+
+ /* A writer forces readers onto the slow path for a while. */
+ percpu_down_write(&ctx->tpcpu);
+ ASSERT_DEPTH(test, ctx, 1);
+ percpu_up_write(&ctx->tpcpu);
+ ASSERT_DEPTH(test, ctx, 0);
+
+ /*
+ * rcu_sync stays non-idle for a grace period after a writer, so
+ * this most likely uses the slow path.
+ */
+ percpu_down_read(&ctx->tpcpu);
+ EXPECT_DEPTH(test, ctx, 1);
+ percpu_up_read(&ctx->tpcpu);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ KUNIT_EXPECT_TRUE(test, percpu_down_read_trylock(&ctx->tpcpu));
+ EXPECT_DEPTH(test, ctx, 1);
+ percpu_up_read(&ctx->tpcpu);
+ EXPECT_DEPTH(test, ctx, 0);
+}
+
+/* downgrade_write() keeps the lock held, so the count does not change. */
+static void downgrade_write_keeps_the_count(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+
+ down_write(&ctx->tsem);
+ ASSERT_DEPTH(test, ctx, 1);
+
+ downgrade_write(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 1);
+
+ up_read(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 0);
+}
+
+/* Tracked locks add up; untracked ones in between do not count. */
+static void nested_locks_stack(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+
+ mutex_lock(&ctx->tmutex);
+ EXPECT_DEPTH(test, ctx, 1);
+
+ mutex_lock(&ctx->pmutex);
+ EXPECT_DEPTH(test, ctx, 1);
+
+ down_read(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 2);
+
+ down_write(&ctx->psem);
+ EXPECT_DEPTH(test, ctx, 2);
+
+ mutex_lock(&ctx->tmutex2);
+ EXPECT_DEPTH(test, ctx, 3);
+
+ down_read(&ctx->tsem2);
+ EXPECT_DEPTH(test, ctx, 4);
+
+ percpu_down_read(&ctx->tpcpu);
+ EXPECT_DEPTH(test, ctx, 5);
+
+ percpu_down_read(&ctx->ppcpu);
+ EXPECT_DEPTH(test, ctx, 5);
+
+ /* Release in a different order than taken. */
+ up_read(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 4);
+ percpu_up_read(&ctx->ppcpu);
+ EXPECT_DEPTH(test, ctx, 4);
+ up_write(&ctx->psem);
+ EXPECT_DEPTH(test, ctx, 4);
+ percpu_up_read(&ctx->tpcpu);
+ EXPECT_DEPTH(test, ctx, 3);
+ up_read(&ctx->tsem2);
+ EXPECT_DEPTH(test, ctx, 2);
+ mutex_unlock(&ctx->pmutex);
+ EXPECT_DEPTH(test, ctx, 2);
+ mutex_unlock(&ctx->tmutex2);
+ EXPECT_DEPTH(test, ctx, 1);
+ mutex_unlock(&ctx->tmutex);
+ EXPECT_DEPTH(test, ctx, 0);
+}
+
+/* cleanup.h guards are counted too. */
+static void guards_are_counted(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+
+ scoped_guard(mutex, &ctx->tmutex)
+ EXPECT_DEPTH(test, ctx, 1);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ scoped_guard(rwsem_read, &ctx->tsem)
+ EXPECT_DEPTH(test, ctx, 1);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ scoped_guard(rwsem_write, &ctx->tsem)
+ EXPECT_DEPTH(test, ctx, 1);
+ EXPECT_DEPTH(test, ctx, 0);
+}
+
+/*
+ * The helper thread takes a lock and records its own count. It is used to
+ * create contention and to check that counts are per task.
+ */
+enum helper_lock {
+ HELPER_TMUTEX,
+ HELPER_TSEM_WRITE,
+ HELPER_TPCPU_WRITE,
+};
+
+struct helper_arg {
+ struct lockholder_ctx *ctx;
+ enum helper_lock which;
+};
+
+static int lockholder_helper(void *data)
+{
+ struct helper_arg *arg = data;
+ struct lockholder_ctx *ctx = arg->ctx;
+
+ switch (arg->which) {
+ case HELPER_TMUTEX:
+ mutex_lock(&ctx->tmutex);
+ break;
+ case HELPER_TSEM_WRITE:
+ down_write(&ctx->tsem);
+ break;
+ case HELPER_TPCPU_WRITE:
+ percpu_down_write(&ctx->tpcpu);
+ break;
+ }
+
+ ctx->helper_depth_held = task_nr_tracked_locks(current);
+ complete(&ctx->helper_holds);
+
+ wait_for_completion(&ctx->helper_may_release);
+
+ switch (arg->which) {
+ case HELPER_TMUTEX:
+ mutex_unlock(&ctx->tmutex);
+ break;
+ case HELPER_TSEM_WRITE:
+ up_write(&ctx->tsem);
+ break;
+ case HELPER_TPCPU_WRITE:
+ percpu_up_write(&ctx->tpcpu);
+ break;
+ }
+
+ ctx->helper_depth_after = task_nr_tracked_locks(current);
+ complete(&ctx->helper_done);
+
+ /* Wait for kthread_stop(). */
+ while (!kthread_should_stop())
+ schedule_timeout_interruptible(HZ / 10);
+ return 0;
+}
+
+static struct task_struct *start_helper(struct kunit *test,
+ enum helper_lock which)
+{
+ struct lockholder_ctx *ctx = test->priv;
+ struct helper_arg *arg;
+ struct task_struct *t;
+
+ arg = kunit_kzalloc(test, sizeof(*arg), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, arg);
+ arg->ctx = ctx;
+ arg->which = which;
+
+ t = kthread_run(lockholder_helper, arg, "lockholder_kunit");
+ KUNIT_ASSERT_FALSE(test, IS_ERR(t));
+
+ wait_for_completion(&ctx->helper_holds);
+ return t;
+}
+
+static void stop_helper(struct kunit *test, struct task_struct *t)
+{
+ struct lockholder_ctx *ctx = test->priv;
+
+ complete(&ctx->helper_may_release);
+ wait_for_completion(&ctx->helper_done);
+ kthread_stop(t);
+}
+
+/* Each task has its own count, and a new task starts at zero. */
+static void the_count_is_per_task(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+ struct task_struct *t;
+
+ down_read(&ctx->tsem2);
+ ASSERT_DEPTH(test, ctx, 1);
+
+ t = start_helper(test, HELPER_TMUTEX);
+
+ /* The helper's lock is not in our count. */
+ EXPECT_DEPTH(test, ctx, 1);
+ /* The helper started from zero, not from our count. */
+ KUNIT_EXPECT_EQ(test, 1u, ctx->helper_depth_held);
+
+ stop_helper(test, t);
+ KUNIT_EXPECT_EQ(test, 0u, ctx->helper_depth_after);
+
+ EXPECT_DEPTH(test, ctx, 1);
+ up_read(&ctx->tsem2);
+ EXPECT_DEPTH(test, ctx, 0);
+}
+
+/* A failed trylock is not counted. */
+static void failed_trylock_is_not_counted(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+ struct task_struct *t;
+
+ t = start_helper(test, HELPER_TSEM_WRITE);
+
+ KUNIT_EXPECT_EQ(test, 0, down_read_trylock(&ctx->tsem));
+ EXPECT_DEPTH(test, ctx, 0);
+
+ KUNIT_EXPECT_EQ(test, 0, down_write_trylock(&ctx->tsem));
+ EXPECT_DEPTH(test, ctx, 0);
+
+ stop_helper(test, t);
+
+ /* Once the lock is free, the trylock succeeds and counts. */
+ KUNIT_ASSERT_EQ(test, 1, down_read_trylock(&ctx->tsem));
+ EXPECT_DEPTH(test, ctx, 1);
+ up_read(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 0);
+}
+
+/*
+ * A failed mutex_trylock() is not counted, and a contended mutex_lock()
+ * is counted exactly once: not zero (slow path missed) and not twice
+ * (fast and slow path both counted).
+ */
+static void contended_mutex_is_counted_once(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+ struct task_struct *t;
+
+ t = start_helper(test, HELPER_TMUTEX);
+
+ KUNIT_EXPECT_FALSE(test, mutex_trylock(&ctx->tmutex));
+ EXPECT_DEPTH(test, ctx, 0);
+
+ /*
+ * Let the helper release the lock. mutex_lock() below most likely
+ * finds it still held and takes the slow path.
+ */
+ complete(&ctx->helper_may_release);
+
+ mutex_lock(&ctx->tmutex);
+ EXPECT_DEPTH(test, ctx, 1);
+ mutex_unlock(&ctx->tmutex);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ wait_for_completion(&ctx->helper_done);
+ kthread_stop(t);
+}
+
+/* The same for a contended rwsem. */
+static void contended_rwsem_is_counted_once(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+ struct task_struct *t;
+
+ t = start_helper(test, HELPER_TSEM_WRITE);
+
+ complete(&ctx->helper_may_release);
+
+ down_write(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 1);
+ up_write(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ down_read(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 1);
+ up_read(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ wait_for_completion(&ctx->helper_done);
+ kthread_stop(t);
+}
+
+/* The same for a percpu_rw_semaphore, starting with a failed read trylock. */
+static void contended_percpu_rwsem_is_counted_once(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+ struct task_struct *t;
+
+ t = start_helper(test, HELPER_TPCPU_WRITE);
+ KUNIT_EXPECT_EQ(test, 1u, ctx->helper_depth_held);
+
+ KUNIT_EXPECT_FALSE(test, percpu_down_read_trylock(&ctx->tpcpu));
+ EXPECT_DEPTH(test, ctx, 0);
+
+ complete(&ctx->helper_may_release);
+
+ percpu_down_read(&ctx->tpcpu);
+ EXPECT_DEPTH(test, ctx, 1);
+ percpu_up_read(&ctx->tpcpu);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ percpu_down_write(&ctx->tpcpu);
+ EXPECT_DEPTH(test, ctx, 1);
+ percpu_up_write(&ctx->tpcpu);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ wait_for_completion(&ctx->helper_done);
+ KUNIT_EXPECT_EQ(test, 0u, ctx->helper_depth_after);
+ kthread_stop(t);
+}
+
+/* Initializing a lock again clears its opt-in. */
+/*
+ * The opt-in is a sticky bit in the same word that says whether the lock is
+ * held, so an opted-in lock that nobody holds must still read as unlocked.
+ */
+static void opted_in_lock_still_reads_unlocked(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+
+ KUNIT_EXPECT_FALSE(test, rwsem_is_locked(&ctx->tsem));
+ KUNIT_EXPECT_FALSE(test, mutex_is_locked(&ctx->tmutex));
+ KUNIT_EXPECT_FALSE(test, percpu_is_read_locked(&ctx->tpcpu));
+ KUNIT_EXPECT_FALSE(test, percpu_is_write_locked(&ctx->tpcpu));
+
+ down_read(&ctx->tsem);
+ KUNIT_EXPECT_TRUE(test, rwsem_is_locked(&ctx->tsem));
+ up_read(&ctx->tsem);
+ KUNIT_EXPECT_FALSE(test, rwsem_is_locked(&ctx->tsem));
+
+ down_write(&ctx->tsem);
+ KUNIT_EXPECT_TRUE(test, rwsem_is_locked(&ctx->tsem));
+ up_write(&ctx->tsem);
+ KUNIT_EXPECT_FALSE(test, rwsem_is_locked(&ctx->tsem));
+
+ mutex_lock(&ctx->tmutex);
+ KUNIT_EXPECT_TRUE(test, mutex_is_locked(&ctx->tmutex));
+ mutex_unlock(&ctx->tmutex);
+ KUNIT_EXPECT_FALSE(test, mutex_is_locked(&ctx->tmutex));
+
+ percpu_down_write(&ctx->tpcpu);
+ KUNIT_EXPECT_TRUE(test, percpu_is_write_locked(&ctx->tpcpu));
+ percpu_up_write(&ctx->tpcpu);
+ KUNIT_EXPECT_FALSE(test, percpu_is_write_locked(&ctx->tpcpu));
+}
+
+/* Opting in a lock that is already held is refused, and warns. */
+static void track_holder_on_a_held_lock_is_refused(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+
+ mutex_lock(&ctx->pmutex);
+ down_read(&ctx->psem);
+ percpu_down_read(&ctx->ppcpu);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ kunit_warning_suppress(test) {
+ mutex_track_holder(&ctx->pmutex);
+ rwsem_track_holder(&ctx->psem);
+ percpu_rwsem_track_holder(&ctx->ppcpu);
+ KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 3);
+ }
+
+ /* Refused, so the locks are still untracked and nothing is counted. */
+ EXPECT_DEPTH(test, ctx, 0);
+ percpu_up_read(&ctx->ppcpu);
+ up_read(&ctx->psem);
+ mutex_unlock(&ctx->pmutex);
+ EXPECT_DEPTH(test, ctx, 0);
+
+ mutex_lock(&ctx->pmutex);
+ EXPECT_DEPTH(test, ctx, 0);
+ mutex_unlock(&ctx->pmutex);
+}
+
+/*
+ * The non_owner calls can run in different tasks, which a per-task count
+ * cannot follow. Both sides warn on a tracked rwsem and neither counts.
+ *
+ * Without CONFIG_DEBUG_LOCK_ALLOC they are plain down_read()/up_read(),
+ * which are counted and balanced, so there is nothing to test.
+ */
+static void non_owner_on_a_tracked_rwsem_warns(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+
+ if (!IS_ENABLED(CONFIG_DEBUG_LOCK_ALLOC))
+ kunit_skip(test, "non_owner API needs CONFIG_DEBUG_LOCK_ALLOC");
+
+ kunit_warning_suppress(test) {
+ down_read_non_owner(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 0);
+ up_read_non_owner(&ctx->tsem);
+ KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 2);
+ }
+ EXPECT_DEPTH(test, ctx, 0);
+
+ /* An untracked rwsem uses them without warning, and is not counted. */
+ down_read_non_owner(&ctx->psem);
+ EXPECT_DEPTH(test, ctx, 0);
+ up_read_non_owner(&ctx->psem);
+ EXPECT_DEPTH(test, ctx, 0);
+}
+
+static void reinit_clears_the_opt_in(struct kunit *test)
+{
+ struct lockholder_ctx *ctx = test->priv;
+
+ down_read(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 1);
+ up_read(&ctx->tsem);
+
+ init_rwsem(&ctx->tsem);
+ down_read(&ctx->tsem);
+ EXPECT_DEPTH(test, ctx, 0);
+ up_read(&ctx->tsem);
+
+ percpu_down_read(&ctx->tpcpu);
+ EXPECT_DEPTH(test, ctx, 1);
+ percpu_up_read(&ctx->tpcpu);
+
+ percpu_free_rwsem(&ctx->tpcpu);
+ KUNIT_ASSERT_EQ(test, 0, percpu_init_rwsem(&ctx->tpcpu));
+ percpu_down_read(&ctx->tpcpu);
+ EXPECT_DEPTH(test, ctx, 0);
+ percpu_up_read(&ctx->tpcpu);
+
+ mutex_lock(&ctx->tmutex);
+ EXPECT_DEPTH(test, ctx, 1);
+ mutex_unlock(&ctx->tmutex);
+
+ mutex_init(&ctx->tmutex);
+ mutex_lock(&ctx->tmutex);
+ EXPECT_DEPTH(test, ctx, 0);
+ mutex_unlock(&ctx->tmutex);
+ EXPECT_DEPTH(test, ctx, 0);
+}
+
+static struct kunit_case lockholder_test_cases[] = {
+ KUNIT_CASE(untracked_locks_are_not_counted),
+ KUNIT_CASE(tracked_mutex_is_counted),
+ KUNIT_CASE(tracked_rwsem_read_is_counted),
+ KUNIT_CASE(tracked_rwsem_write_is_counted),
+ KUNIT_CASE(tracked_percpu_rwsem_is_counted),
+ KUNIT_CASE(percpu_rwsem_handover_moves_the_count),
+ KUNIT_CASE(percpu_rwsem_slow_read_path_is_counted),
+ KUNIT_CASE(downgrade_write_keeps_the_count),
+ KUNIT_CASE(nested_locks_stack),
+ KUNIT_CASE(guards_are_counted),
+ KUNIT_CASE(the_count_is_per_task),
+ KUNIT_CASE(failed_trylock_is_not_counted),
+ KUNIT_CASE(contended_mutex_is_counted_once),
+ KUNIT_CASE(contended_rwsem_is_counted_once),
+ KUNIT_CASE(contended_percpu_rwsem_is_counted_once),
+ KUNIT_CASE(reinit_clears_the_opt_in),
+ KUNIT_CASE(opted_in_lock_still_reads_unlocked),
+ KUNIT_CASE(track_holder_on_a_held_lock_is_refused),
+ KUNIT_CASE(non_owner_on_a_tracked_rwsem_warns),
+ {}
+};
+
+static struct kunit_suite lockholder_test_suite = {
+ .name = "lockholder",
+ .init = lockholder_init,
+ .exit = lockholder_exit,
+ .test_cases = lockholder_test_cases,
+};
+kunit_test_suite(lockholder_test_suite);
+
+MODULE_DESCRIPTION("KUnit tests for sleeping lock holder tracking");
+MODULE_LICENSE("GPL");
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 163192b2ed7f..67fc86b4fc6f 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1672,6 +1672,19 @@ config WW_MUTEX_SELFTEST
Say M if you want these self tests to build as a module.
Say N if you are unsure.
+config TRACK_LOCK_HOLDERS_KUNIT_TEST
+ tristate "KUnit tests for sleeping lock holder tracking" if !KUNIT_ALL_TESTS
+ depends on KUNIT && TRACK_LOCK_HOLDERS
+ default KUNIT_ALL_TESTS
+ help
+ Check that task_nr_tracked_locks() always equals the number of
+ opted-in locks the task holds.
+
+ For more information on KUnit and unit tests in general, refer to
+ the KUnit documentation in Documentation/dev-tools/kunit/.
+
+ If unsure, say N.
+
config SCF_TORTURE_TEST
tristate "torture tests for smp_call_function*()"
depends on DEBUG_KERNEL
--
2.53.0-Meta
^ permalink raw reply [flat|nested] 8+ messages in thread* [PATCH 6/7] Documentation/locking: document sleeping lock holder tracking
2026-09-23 5:01 [PATCH 0/7] locking: opt-in tracking of sleeping lock holders Shakeel Butt
` (4 preceding siblings ...)
2026-09-23 5:01 ` [PATCH 5/7] locking/selftests: add KUnit tests for lock holder tracking Shakeel Butt
@ 2026-09-23 5:01 ` Shakeel Butt
2026-09-23 5:01 ` [PATCH 7/7] kernfs, cgroup: track holders of the cgroupfs locks Shakeel Butt
6 siblings, 0 replies; 8+ messages in thread
From: Shakeel Butt @ 2026-09-23 5:01 UTC (permalink / raw)
To: Peter Zijlstra, Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long
Cc: Paul McKenney, Greg Kroah-Hartman, Tejun Heo, Christian Brauner,
Sebastian Andrzej Siewior, Johannes Weiner, Jonathan Corbet,
Meta kernel team, cgroups, driver-core, linux-kernel
Describe why the count exists, how a lock opts in, what it costs, where
each lock type updates the count, and what is not covered.
Signed-off-by: Shakeel Butt <shakeel.butt@linux.dev>
---
Documentation/locking/index.rst | 1 +
.../locking/lock-holder-tracking.rst | 168 ++++++++++++++++++
2 files changed, 169 insertions(+)
create mode 100644 Documentation/locking/lock-holder-tracking.rst
diff --git a/Documentation/locking/index.rst b/Documentation/locking/index.rst
index 9278d95b7dcb..6593c924f9a7 100644
--- a/Documentation/locking/index.rst
+++ b/Documentation/locking/index.rst
@@ -9,6 +9,7 @@ Locking
locktypes
lockdep-design
+ lock-holder-tracking
lockstat
locktorture
mutex-design
diff --git a/Documentation/locking/lock-holder-tracking.rst b/Documentation/locking/lock-holder-tracking.rst
new file mode 100644
index 000000000000..a656f3365d97
--- /dev/null
+++ b/Documentation/locking/lock-holder-tracking.rst
@@ -0,0 +1,168 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=============================
+Sleeping lock holder tracking
+=============================
+
+:Date: September 2026
+:Author: Shakeel Butt <shakeel.butt@linux.dev>
+
+Why
+===
+
+A task that stalls while holding a widely shared lock also stalls the
+tasks waiting for that lock. Some kernel code wants to avoid stalling
+such a task.
+
+One source of such stalls is a memory allocation by the lock holder,
+which can get stuck in memory reclaim because the system is low on
+memory or because a memcg limit is being enforced. If an allocation made
+while holding ``cgroup_mutex`` or ``kernfs_rwsem`` gets stuck in reclaim,
+other cgroupfs operations can end up waiting too, even for unrelated
+cgroups.
+
+Knowing that the task holds such a lock lets the allocation path treat it
+differently. Under a memcg limit it can skip the throttling that
+enforcement would otherwise do inline, and leave it for the return to
+user space. Under global pressure it can let the allocation reach the
+memory reserves, so it finishes instead of waiting for reclaim. Those
+policies are not part of this feature; they only need the answer.
+
+What
+====
+
+Locks do not record all their holders, so the answer comes from the task
+instead. Each task counts the opted-in locks it holds::
+
+ static inline bool task_holds_tracked_lock(struct task_struct *p);
+ static inline bool current_holds_tracked_lock(void);
+ static inline unsigned int task_nr_tracked_locks(struct task_struct *p);
+
+Only the task itself changes its count, so the count needs no atomics or
+locking. Without ``CONFIG_TRACK_LOCK_HOLDERS``, these always return
+``false`` or ``0``.
+
+BPF programs can read ``task->nr_tracked_locks`` through BTF.
+
+Opting a lock in
+================
+
+No lock is tracked by default. A lock opts in right after it is
+initialized::
+
+ init_rwsem(&root->kernfs_rwsem);
+ rwsem_track_holder(&root->kernfs_rwsem);
+
+ mutex_init(&foo->lock);
+ mutex_track_holder(&foo->lock);
+
+ percpu_init_rwsem(&foo->sem);
+ percpu_rwsem_track_holder(&foo->sem);
+
+Only a few locks should opt in: those whose holders can stall unrelated
+work. A lock must opt in before anyone can take it. A task already
+holding it was not counted, so its release would make its count wrong.
+All three helpers warn and do nothing if the lock is held. Initializing
+a lock again clears its opt-in.
+
+A tracked rw_semaphore must not use ``down_read_non_owner()`` or
+``up_read_non_owner()``, which can run in different tasks.
+``down_read_non_owner()`` warns on a tracked rwsem and does not count it.
+
+A percpu_rw_semaphore can be passed to another task with
+``percpu_rwsem_release()`` and ``percpu_rwsem_acquire()``, as filesystem
+freeze does. The count moves with the lock.
+
+Cost
+====
+
+With ``CONFIG_TRACK_LOCK_HOLDERS=n``, the generated code does not change.
+
+With it enabled, no lock type grows on 64-bit, so neither do the
+structures that embed them. That relies on padding which is not there on
+every configuration: see the end of this section.
+
+A mutex and an rw_semaphore keep the opt-in in a spare bit of a word they
+already have: bit 3 of ``rw_semaphore::count``, which is reserved, and
+bit 3 of ``mutex::owner``, which is below the alignment of a task_struct
+pointer. The bit is sticky, so unlike the other flag bits it survives
+unlock, and an unlocked lock that has opted in does not read as zero.
+Paths that add to these words, or write back a value they read, keep the
+bit as they are. The three that write an absolute value -
+``rwsem_write_trylock()``, ``__mutex_trylock_fast()`` and
+``__mutex_unlock_fast()`` - read the bit first and put it in both the
+expected and the new value, so each still does a single cmpxchg. Reading
+it needs no lock, because it is sticky.
+
+``percpu_rw_semaphore`` keeps a plain flag, which fits in the padding
+after ``->block``. On ``PREEMPT_RT``, a mutex and an rw_semaphore are
+built on ``rt_mutex_base``, which has no spare bit but does have padding
+after ``->wait_lock``, so the flag goes there.
+
+Those two rely on padding that only exists on 64-bit. On 32-bit
+``percpu_rw_semaphore`` grows by a word, and on 32-bit ``PREEMPT_RT`` so
+does ``rt_mutex_base``, which means everything built on it including
+``spinlock_t``. The sticky bit in ``mutex`` and ``rw_semaphore`` has no
+such problem, as it uses no storage at all.
+
+Until the first lock opts in, each hook is a static branch that is off,
+a 2-byte NOP on x86_64, and the flag reads sit behind it, so every lock
+runs the instructions it ran before. The first opt-in turns the branch
+on for good. After that, an acquire and a release each also read the
+word they are about to update and test the flag, which costs one load
+from a cacheline the lock operation touches anyway.
+``__mutex_trylock_common()`` needs no load of its own, as it already
+has the value.
+
+Where the count changes
+=======================
+
+The count goes up once the lock is taken and down before it is released.
+Waiting is not counted, as the task does not hold the lock yet.
+
+mutex (``!PREEMPT_RT``)
+ Only ``__mutex_trylock_common()`` and ``__mutex_trylock_fast()`` take
+ the lock for current, and only ``__mutex_unlock_fast()`` and
+ ``__mutex_unlock_slowpath()`` release it. These four places cover
+ every ``mutex_lock*()``, ``mutex_trylock()``, ``ww_mutex_lock*()``
+ and handoff. ``__mutex_lock_common()`` has no hook of its own: it
+ reaches ``__mutex_trylock_common()`` through ``__mutex_trylock()``
+ and ``__mutex_trylock_or_handoff()``.
+
+mutex (``PREEMPT_RT``)
+ ``__mutex_lock_common()``, ``mutex_trylock()``,
+ ``_mutex_trylock_nest_lock()`` and ``mutex_unlock()`` in
+ ``rtmutex_api.c``.
+
+rw_semaphore
+ The API functions in ``rwsem.c``, which ``PREEMPT_RT`` and
+ ``!PREEMPT_RT`` share. ``downgrade_write()`` keeps the lock held, so
+ it needs no hook.
+
+percpu_rw_semaphore
+ ``percpu_down_read_internal()``, ``percpu_down_read_trylock()`` and
+ ``percpu_up_read()`` in the header, ``percpu_down_write()`` and
+ ``percpu_up_write()`` in ``percpu-rwsem.c``, and
+ ``percpu_rwsem_acquire()``/``percpu_rwsem_release()``. The read fast
+ and slow paths meet before the hook, so one hook covers both.
+
+Not covered: ``rt_mutex`` used directly, ``ww_mutex`` on ``PREEMPT_RT``
+(it is an ``rt_mutex`` there), ``struct semaphore``, and spinning locks.
+Only ``mutex`` and ``rw_semaphore`` have a ``*_track_holder()`` helper.
+To add a lock type, find it a place to keep the opt-in that does not make
+it bigger, then call ``lock_holder_acquired_if()`` and
+``lock_holder_released_if()`` where it is taken and released. If the lock has a ``bool tracked`` member, as
+``percpu_rw_semaphore`` does, ``lock_holder_acquired()`` and
+``lock_holder_released()`` are the same thing with the member filled in.
+
+Debugging
+=========
+
+``do_exit()`` warns once if a task exits with a non-zero count, and the
+release hook warns once instead of letting the count go below zero.
+Either means a bug, usually a missed hook.
+
+``CONFIG_TRACK_LOCK_HOLDERS_KUNIT_TEST`` tests the count for all three
+lock types::
+
+ tools/testing/kunit/kunit.py run --kunitconfig=kernel/locking/ lockholder
--
2.53.0-Meta
^ permalink raw reply [flat|nested] 8+ messages in thread* [PATCH 7/7] kernfs, cgroup: track holders of the cgroupfs locks
2026-09-23 5:01 [PATCH 0/7] locking: opt-in tracking of sleeping lock holders Shakeel Butt
` (5 preceding siblings ...)
2026-09-23 5:01 ` [PATCH 6/7] Documentation/locking: document sleeping " Shakeel Butt
@ 2026-09-23 5:01 ` Shakeel Butt
6 siblings, 0 replies; 8+ messages in thread
From: Shakeel Butt @ 2026-09-23 5:01 UTC (permalink / raw)
To: Peter Zijlstra, Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long
Cc: Paul McKenney, Greg Kroah-Hartman, Tejun Heo, Christian Brauner,
Sebastian Andrzej Siewior, Johannes Weiner, Jonathan Corbet,
Meta kernel team, cgroups, driver-core, linux-kernel
Opt the cgroupfs kernfs rwsems, cgroup_mutex and cgroup_threadgroup_rwsem
in to lock holder tracking.
Add KERNFS_ROOT_TRACK_LOCK_HOLDERS, which opts in a root's kernfs_rwsem,
kernfs_iattr_rwsem and kernfs_supers_rwsem, and pass it from
cgroup_setup_root(). Other kernfs users have their own roots, so only
cgroupfs is affected. All three are held across work that can allocate,
so a holder stuck in reclaim can stall many other cgroupfs operations;
kernfs_iattr_rwsem is write-held across kernfs_iattrs(), which allocates
with GFP_KERNEL.
cgroup_mutex is taken by every cgroup create, destroy and migrate, and is
held across the same kind of work, so a stalled holder stalls the whole
cgroupfs control plane.
cgroup_threadgroup_rwsem is taken for read on every fork and exit, and
usually for write on migration, so a stalled holder can stall process
creation on the whole machine.
The last two are statically defined, so opt them in from cgroup_init().
That runs after jump_label_init() and before anything can take either
lock.
signal->cgroup_threadgroup_rwsem, used with favordynmods, is left alone.
Its readers also hold cgroup_threadgroup_rwsem, so they are counted
already, and its writer only blocks the process being migrated.
These opt-ins are for a policy that reads the count. On their own they
only turn on the static branch, so they should go in with that policy.
Signed-off-by: Shakeel Butt <shakeel.butt@linux.dev>
---
fs/kernfs/dir.c | 5 +++++
include/linux/kernfs.h | 7 +++++++
kernel/cgroup/cgroup.c | 27 ++++++++++++++++++++++++++-
3 files changed, 38 insertions(+), 1 deletion(-)
diff --git a/fs/kernfs/dir.c b/fs/kernfs/dir.c
index cc6288d5b4cc..1116bb38c1ff 100644
--- a/fs/kernfs/dir.c
+++ b/fs/kernfs/dir.c
@@ -1035,6 +1035,11 @@ struct kernfs_root *kernfs_create_root(struct kernfs_syscall_ops *scops,
init_rwsem(&root->kernfs_rwsem);
init_rwsem(&root->kernfs_iattr_rwsem);
init_rwsem(&root->kernfs_supers_rwsem);
+ if (flags & KERNFS_ROOT_TRACK_LOCK_HOLDERS) {
+ rwsem_track_holder(&root->kernfs_rwsem);
+ rwsem_track_holder(&root->kernfs_iattr_rwsem);
+ rwsem_track_holder(&root->kernfs_supers_rwsem);
+ }
INIT_LIST_HEAD(&root->supers);
rwlock_init(&root->kernfs_rename_lock);
diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h
index 73786b567ccd..ea2efdc5277e 100644
--- a/include/linux/kernfs.h
+++ b/include/linux/kernfs.h
@@ -156,6 +156,13 @@ enum kernfs_root_flag {
* Renames must not change the parent node.
*/
KERNFS_ROOT_INVARIANT_PARENT = 0x0010,
+
+ /*
+ * Track the holders of this root's rwsems (see
+ * CONFIG_TRACK_LOCK_HOLDERS). Use it for roots whose locks are
+ * shared so widely that a stalled holder stalls unrelated work.
+ */
+ KERNFS_ROOT_TRACK_LOCK_HOLDERS = 0x0020,
};
/* type-specific structures for kernfs_node union members */
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 804318ae160e..87d66aae2558 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -2191,11 +2191,18 @@ int cgroup_setup_root(struct cgroup_root *root, u32 ss_mask)
kf_sops = root == &cgrp_dfl_root ?
&cgroup_kf_syscall_ops : &cgroup1_kf_syscall_ops;
+ /*
+ * Track the holders of the cgroupfs kernfs locks. They are held
+ * across work that can charge memory, so a holder stuck in reclaim
+ * can stall many other cgroupfs operations. Other kernfs users have
+ * their own roots and are not affected.
+ */
root->kf_root = kernfs_create_root(kf_sops,
KERNFS_ROOT_CREATE_DEACTIVATED |
KERNFS_ROOT_SUPPORT_EXPORTOP |
KERNFS_ROOT_SUPPORT_USER_XATTR |
- KERNFS_ROOT_INVARIANT_PARENT,
+ KERNFS_ROOT_INVARIANT_PARENT |
+ KERNFS_ROOT_TRACK_LOCK_HOLDERS,
root_cgrp);
if (IS_ERR(root->kf_root)) {
ret = PTR_ERR(root->kf_root);
@@ -6552,6 +6559,24 @@ int __init cgroup_init(void)
get_user_ns(init_cgroup_ns.user_ns);
cgroup_rt_init();
+ /*
+ * cgroup_mutex is taken by every cgroup create, destroy and migrate
+ * and is held across work that can allocate, so a holder stuck in
+ * reclaim stalls the whole cgroupfs control plane.
+ *
+ * cgroup_threadgroup_rwsem is taken for read by fork and exit, and
+ * usually for write by migration, so a stalled holder can stall
+ * process creation on the whole machine.
+ *
+ * Neither can be held yet, as rest_init() has not run.
+ *
+ * signal->cgroup_threadgroup_rwsem is not tracked. Its readers hold
+ * cgroup_threadgroup_rwsem too, so they are counted already, and its
+ * writer only blocks the process being migrated.
+ */
+ mutex_track_holder(&cgroup_mutex);
+ percpu_rwsem_track_holder(&cgroup_threadgroup_rwsem);
+
cgroup_lock();
/*
--
2.53.0-Meta
^ permalink raw reply [flat|nested] 8+ messages in thread