From: Peng Huang <penghuang@google.com>
To: Peter Zijlstra <peterz@infradead.org>,
Ingo Molnar <mingo@redhat.com>, Will Deacon <will@kernel.org>,
Boqun Feng <boqun@kernel.org>, Waiman Long <longman@redhat.com>,
Juri Lelli <juri.lelli@redhat.com>,
Vincent Guittot <vincent.guittot@linaro.org>
Cc: Dietmar Eggemann <dietmar.eggemann@arm.com>,
Steven Rostedt <rostedt@goodmis.org>,
Ben Segall <bsegall@google.com>, Mel Gorman <mgorman@suse.de>,
Valentin Schneider <vschneid@redhat.com>,
K Prateek Nayak <kprateek.nayak@amd.com>,
linux-kernel@vger.kernel.org, Peng Huang <penghuang@google.com>
Subject: [RFC PATCH 1/3] locking/rwsem, sched: track held rwsems and waiter priorities
Date: Thu, 19 Nov 2026 14:49:52 -0800 [thread overview]
Message-ID: <20261119224954.196990-2-penghuang@google.com> (raw)
In-Reply-To: <20261119224954.196990-1-penghuang@google.com>
Track per-task held rw_semaphore pointers in a small fixed-size slot
array (CONFIG_RWSEM_SCHED_BOOST_SLOTS) and track the highest-priority
contending waiter on each rw_semaphore (max_waiter_prio, capped at
MAX_RT_PRIO).
Blocking acquisitions skip irqs/atomic guards to avoid pushf on the
fastpath, while trylock paths keep IRQ/atomic context filtering for
non-owner unlockers such as BPF helpers. Held slots are cleared under
preempt_disable() before releasing sem->count so another CPU cannot
acquire and free the semaphore while a tick/preempt check still sees
it in held_rwsems[].
Also wire up core priority calculation, runtime toggle
(rwsem_sched_boost=), and post-unlock priority restoration.
Signed-off-by: Peng Huang <penghuang@google.com>
---
.../admin-guide/kernel-parameters.txt | 5 +
include/linux/rwsem.h | 28 +++
include/linux/sched.h | 16 ++
init/init_task.c | 4 +
kernel/Kconfig.locks | 15 ++
kernel/locking/rwsem.c | 211 ++++++++++++++++--
kernel/sched/core.c | 102 ++++++++-
kernel/sched/sched.h | 43 ++++
kernel/sched/syscalls.c | 2 +-
9 files changed, 402 insertions(+), 24 deletions(-)
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 176648941aec..2c5207dd9c53 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -6778,6 +6778,11 @@ Kernel parameters
rw [KNL] Mount root device read-write on boot
+ rwsem_sched_boost=
+ [KNL] Enable or disable rwsem scheduler priority
+ boosting when CONFIG_RWSEM_SCHED_BOOST=y.
+ Defaults to 1 (enabled). Format: <bool>
+
S [KNL] Run init in single mode
s390_iommu= [HW,S390]
diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h
index 6a1a7bae5f81..b9188de31881 100644
--- a/include/linux/rwsem.h
+++ b/include/linux/rwsem.h
@@ -16,6 +16,8 @@
#include <linux/atomic.h>
#include <linux/err.h>
#include <linux/cleanup.h>
+#include <linux/jump_label.h>
+#include <linux/sched/prio.h>
#ifdef CONFIG_DEBUG_LOCK_ALLOC
# define __RWSEM_DEP_MAP_INIT(lockname) \
@@ -58,6 +60,9 @@ context_lock_struct(rw_semaphore) {
#endif
raw_spinlock_t wait_lock;
struct rwsem_waiter *first_waiter __guarded_by(&wait_lock);
+#ifdef CONFIG_RWSEM_SCHED_BOOST
+ atomic_t max_waiter_prio;
+#endif
#ifdef CONFIG_DEBUG_RWSEMS
void *magic;
#endif
@@ -101,12 +106,19 @@ static inline void rwsem_assert_held_write_nolockdep(const struct rw_semaphore *
#define __RWSEM_OPT_INIT(lockname)
#endif
+#ifdef CONFIG_RWSEM_SCHED_BOOST
+# define __RWSEM_SCHED_BOOST_INIT .max_waiter_prio = ATOMIC_INIT(MAX_PRIO),
+#else
+# define __RWSEM_SCHED_BOOST_INIT
+#endif
+
#define __RWSEM_INITIALIZER(name) \
{ __RWSEM_COUNT_INIT(name), \
.owner = ATOMIC_LONG_INIT(0), \
__RWSEM_OPT_INIT(name) \
.wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock),\
.first_waiter = NULL, \
+ __RWSEM_SCHED_BOOST_INIT \
__RWSEM_DEBUG_INIT(name) \
__RWSEM_DEP_MAP_INIT(name) }
@@ -327,8 +339,24 @@ extern void up_read_non_owner(struct rw_semaphore *sem) __releases_shared(sem);
# define down_write_nest_lock(sem, nest_lock) down_write(sem)
# define down_write_nested(sem, subclass) down_write(sem)
# define down_write_killable_nested(sem, subclass) down_write_killable(sem)
+#ifdef CONFIG_RWSEM_SCHED_BOOST
+void down_read_non_owner(struct rw_semaphore *sem) __acquires_shared(sem);
+void up_read_non_owner(struct rw_semaphore *sem) __releases_shared(sem);
+#else
# define down_read_non_owner(sem) down_read(sem)
# define up_read_non_owner(sem) up_read(sem)
#endif
+#endif
+
+#ifdef CONFIG_RWSEM_SCHED_BOOST
+extern struct static_key_true rwsem_sched_boost_key;
+static inline void rwsem_sched_boost_set(bool enable)
+{
+ if (enable)
+ static_branch_enable(&rwsem_sched_boost_key);
+ else
+ static_branch_disable(&rwsem_sched_boost_key);
+}
+#endif
#endif /* _LINUX_RWSEM_H */
diff --git a/include/linux/sched.h b/include/linux/sched.h
index e4f6323d7855..9ce2c11aa500 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -78,6 +78,7 @@ struct rcu_node;
struct reclaim_state;
struct root_domain;
struct rq;
+struct rw_semaphore;
struct sched_attr;
struct sched_dl_entity;
struct seq_file;
@@ -1266,6 +1267,13 @@ struct task_struct {
struct rt_mutex_waiter *pi_blocked_on;
#endif
+#ifdef CONFIG_RWSEM_SCHED_BOOST
+ struct rw_semaphore *held_rwsems[CONFIG_RWSEM_SCHED_BOOST_SLOTS];
+ u8 nr_held_rwsems;
+ int rwsem_boost_prio;
+ struct list_head rwsem_rq_node;
+#endif
+
struct mutex *blocked_on; /* lock we're blocked on */
raw_spinlock_t blocked_lock;
@@ -2250,6 +2258,14 @@ static inline void clear_task_blocked_on(struct task_struct *p, struct rt_mutex
}
#endif /* !CONFIG_PREEMPT_RT */
+#ifdef CONFIG_RWSEM_SCHED_BOOST
+void sched_rwsem_recalc_boost(struct task_struct *p);
+#else
+static inline void sched_rwsem_recalc_boost(struct task_struct *p)
+{
+}
+#endif
+
static __always_inline bool need_resched(void)
{
return unlikely(tif_need_resched());
diff --git a/init/init_task.c b/init/init_task.c
index ce7c2b07d855..a52a08f11ffd 100644
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -134,6 +134,10 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
.time_slice = RR_TIMESLICE,
},
.tasks = LIST_HEAD_INIT(init_task.tasks),
+#ifdef CONFIG_RWSEM_SCHED_BOOST
+ .rwsem_boost_prio = MAX_PRIO,
+ .rwsem_rq_node = LIST_HEAD_INIT(init_task.rwsem_rq_node),
+#endif
#ifdef CONFIG_SMP
.pushable_tasks = PLIST_NODE_INIT(init_task.pushable_tasks, MAX_PRIO),
#endif
diff --git a/kernel/Kconfig.locks b/kernel/Kconfig.locks
index 1c6423aafcd4..7487cdab4e59 100644
--- a/kernel/Kconfig.locks
+++ b/kernel/Kconfig.locks
@@ -279,3 +279,18 @@ config ARCH_HAS_MMIOWB
config MMIOWB
def_bool y if ARCH_HAS_MMIOWB
depends on SMP
+
+config RWSEM_SCHED_BOOST_SLOTS
+ int "Maximum held rw_semaphores tracked per task for priority boost"
+ range 0 16
+ default 4
+ depends on !PREEMPT_RT
+ help
+ Number of rw_semaphore pointers recorded per task_struct so the
+ scheduler can detect and boost holders blocking higher-priority
+ waiters across wakeup, preemption, tick, and idle-balance hooks.
+ Set to 0 to disable.
+
+config RWSEM_SCHED_BOOST
+ def_bool y
+ depends on RWSEM_SCHED_BOOST_SLOTS > 0
diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c
index b9c180ac1eee..5289bb4bf5eb 100644
--- a/kernel/locking/rwsem.c
+++ b/kernel/locking/rwsem.c
@@ -30,6 +30,80 @@
#include <linux/hung_task.h>
#include <trace/events/lock.h>
+#ifdef CONFIG_RWSEM_SCHED_BOOST
+/*
+ * Record @sem in a free per-task slot for scheduler boost inspection.
+ * Trylock callers can run in IRQ/atomic contexts (e.g. BPF helpers whose
+ * deferred irq_work uses up_read_non_owner() when irqs_disabled() was true),
+ * while process-context non-owner callers use down_read_non_owner(). Guard
+ * trylocks with @check_atomic so skipped contexts never strand @sem in slots.
+ */
+static inline void rwsem_record_hold(struct rw_semaphore *sem,
+ bool check_atomic)
+{
+ struct task_struct *curr = current;
+ u8 nr;
+ int i;
+
+ if (!static_branch_likely(&rwsem_sched_boost_key))
+ return;
+ if (unlikely(!in_task() ||
+ (check_atomic && (irqs_disabled() || in_atomic()))))
+ return;
+
+ nr = READ_ONCE(curr->nr_held_rwsems);
+ for (i = 0; i < CONFIG_RWSEM_SCHED_BOOST_SLOTS; i++) {
+ if (!curr->held_rwsems[i]) {
+ WRITE_ONCE(curr->held_rwsems[i], sem);
+ WRITE_ONCE(curr->nr_held_rwsems, nr + 1);
+ return;
+ }
+ }
+}
+
+/*
+ * Clear @sem from held_rwsems[] under preempt_disable() BEFORE
+ * releasing sem->count so another CPU cannot acquire/free @sem after
+ * unlock while scheduler inspection still sees it in held_rwsems[].
+ */
+static inline void rwsem_record_release(struct rw_semaphore *sem)
+{
+ struct task_struct *curr = current;
+ u8 nr = READ_ONCE(curr->nr_held_rwsems);
+ int i;
+
+ if (unlikely(!in_task() || !nr))
+ return;
+
+ for (i = 0; i < CONFIG_RWSEM_SCHED_BOOST_SLOTS; i++) {
+ if (curr->held_rwsems[i] == sem) {
+ WRITE_ONCE(curr->held_rwsems[i], NULL);
+ WRITE_ONCE(curr->nr_held_rwsems, nr - 1);
+ return;
+ }
+ }
+}
+
+static inline void rwsem_sched_unboost_if_needed(void)
+{
+ if (unlikely(in_task() && current->rwsem_boost_prio < MAX_PRIO))
+ sched_rwsem_recalc_boost(current);
+}
+#else
+static inline void rwsem_record_hold(struct rw_semaphore *sem,
+ bool check_atomic)
+{
+}
+
+static inline void rwsem_record_release(struct rw_semaphore *sem)
+{
+}
+
+static inline void rwsem_sched_unboost_if_needed(void)
+{
+}
+#endif
+
#ifndef CONFIG_PREEMPT_RT
#include "lock_events.h"
@@ -324,6 +398,9 @@ void __init_rwsem(struct rw_semaphore *sem, const char *name,
scoped_guard (raw_spinlock_init, &sem->wait_lock) {
sem->first_waiter = NULL;
}
+#ifdef CONFIG_RWSEM_SCHED_BOOST
+ atomic_set(&sem->max_waiter_prio, MAX_PRIO);
+#endif
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
osq_lock_init(&sem->osq);
#endif
@@ -349,6 +426,50 @@ enum rwsem_wake_type {
RWSEM_WAKE_READ_OWNED /* Waker thread holds the read lock */
};
+#ifdef CONFIG_RWSEM_SCHED_BOOST
+static inline int rwsem_clamp_waiter_prio(struct task_struct *tsk)
+{
+ /* Cap boost at CFS highest priority (nice -20) to protect RT tasks. */
+ return max_t(int, READ_ONCE(tsk->prio), MAX_RT_PRIO);
+}
+
+static inline void rwsem_update_max_waiter_prio(struct rw_semaphore *sem,
+ struct task_struct *waiter_task)
+{
+ int prio = rwsem_clamp_waiter_prio(waiter_task);
+
+ lockdep_assert_held(&sem->wait_lock);
+ if (prio < atomic_read(&sem->max_waiter_prio))
+ atomic_set(&sem->max_waiter_prio, prio);
+}
+
+static void rwsem_recalc_max_waiter_prio(struct rw_semaphore *sem)
+{
+ struct rwsem_waiter *first = sem->first_waiter;
+ struct rwsem_waiter *w;
+ int prio = MAX_PRIO;
+
+ lockdep_assert_held(&sem->wait_lock);
+ if (first) {
+ prio = min(prio, rwsem_clamp_waiter_prio(first->task));
+ list_for_each_entry(w, &first->list, list)
+ prio = min(prio, rwsem_clamp_waiter_prio(w->task));
+ }
+
+ if (atomic_read(&sem->max_waiter_prio) != prio)
+ atomic_set(&sem->max_waiter_prio, prio);
+}
+#else
+static inline void rwsem_update_max_waiter_prio(struct rw_semaphore *sem,
+ struct task_struct *waiter_task)
+{
+}
+
+static inline void rwsem_recalc_max_waiter_prio(struct rw_semaphore *sem)
+{
+}
+#endif
+
/*
* The typical HZ value is either 250 or 1000. So set the minimum waiting
* time to at least 4ms or 1 jiffy (if it is higher than 4ms) in the wait
@@ -394,9 +515,12 @@ static inline bool
rwsem_del_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)
{
lockdep_assert_held(&sem->wait_lock);
- if (__rwsem_del_waiter(sem, waiter))
+ if (__rwsem_del_waiter(sem, waiter)) {
+ rwsem_recalc_max_waiter_prio(sem);
return true;
+ }
atomic_long_andnot(RWSEM_FLAG_HANDOFF | RWSEM_FLAG_WAITERS, &sem->count);
+ rwsem_recalc_max_waiter_prio(sem);
return false;
}
@@ -542,6 +666,8 @@ static void rwsem_mark_wake(struct rw_semaphore *sem,
break;
} while ((waiter = next) != NULL);
+ rwsem_recalc_max_waiter_prio(sem);
+
adjustment = woken * RWSEM_READER_BIAS - adjustment;
lockevent_cond_inc(rwsem_wake_reader, woken);
@@ -683,6 +809,7 @@ static inline bool rwsem_try_write_lock(struct rw_semaphore *sem,
* success.
*/
__rwsem_del_waiter(sem, waiter);
+ rwsem_recalc_max_waiter_prio(sem);
rwsem_set_owner(sem);
return true;
@@ -1081,6 +1208,7 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
} else {
list_add_tail(&waiter.list, &first->list);
}
+ rwsem_update_max_waiter_prio(sem, current);
/* we're now waiting on the lock, but no longer actively locking */
count = atomic_long_add_return(adjustment, &sem->count);
@@ -1177,6 +1305,7 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)
sem->first_waiter = &waiter;
atomic_long_or(RWSEM_FLAG_WAITERS, &sem->count);
}
+ rwsem_update_max_waiter_prio(sem, current);
/* wait until we successfully acquire the lock */
set_current_state(state);
@@ -1376,7 +1505,7 @@ static inline int __down_write_trylock(struct rw_semaphore *sem)
/*
* unlock after reading
*/
-static inline void __up_read(struct rw_semaphore *sem)
+static inline void __up_read_common(struct rw_semaphore *sem, bool track_owner)
{
long tmp;
@@ -1384,6 +1513,8 @@ static inline void __up_read(struct rw_semaphore *sem)
DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem);
preempt_disable();
+ if (track_owner)
+ rwsem_record_release(sem);
rwsem_clear_reader_owned(sem);
tmp = atomic_long_add_return_release(-RWSEM_READER_BIAS, &sem->count);
DEBUG_RWSEMS_WARN_ON(tmp < 0, sem);
@@ -1395,6 +1526,20 @@ static inline void __up_read(struct rw_semaphore *sem)
rwsem_wake(sem);
}
preempt_enable();
+
+ /* Drop boost after waking waiters and re-enabling preemption. */
+ if (track_owner)
+ rwsem_sched_unboost_if_needed();
+}
+
+static inline void __up_read(struct rw_semaphore *sem)
+{
+ __up_read_common(sem, true);
+}
+
+static inline void __up_read_non_owner(struct rw_semaphore *sem)
+{
+ __up_read_common(sem, false);
}
/*
@@ -1413,6 +1558,7 @@ static inline void __up_write(struct rw_semaphore *sem)
!rwsem_test_oflags(sem, RWSEM_NONSPINNABLE), sem);
preempt_disable();
+ rwsem_record_release(sem);
rwsem_clear_owner(sem);
tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count);
if (unlikely(tmp & RWSEM_FLAG_WAITERS)) {
@@ -1420,6 +1566,7 @@ static inline void __up_write(struct rw_semaphore *sem)
rwsem_wake(sem);
}
preempt_enable();
+ rwsem_sched_unboost_if_needed();
}
/*
@@ -1517,11 +1664,16 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)
return rwbase_read_trylock(&sem->rwbase);
}
-static inline void __up_read(struct rw_semaphore *sem)
+static inline void __up_read_non_owner(struct rw_semaphore *sem)
{
rwbase_read_unlock(&sem->rwbase, TASK_NORMAL);
}
+static inline void __up_read(struct rw_semaphore *sem)
+{
+ __up_read_non_owner(sem);
+}
+
static inline void __sched __down_write(struct rw_semaphore *sem)
{
rwbase_write_lock(&sem->rwbase, TASK_UNINTERRUPTIBLE);
@@ -1574,6 +1726,7 @@ void __sched down_read(struct rw_semaphore *sem)
rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
LOCK_CONTENDED(sem, __down_read_trylock, __down_read);
+ rwsem_record_hold(sem, false);
}
EXPORT_SYMBOL(down_read);
@@ -1588,6 +1741,7 @@ int __sched down_read_interruptible(struct rw_semaphore *sem)
return -EINTR;
}
+ rwsem_record_hold(sem, false);
return 0;
}
EXPORT_SYMBOL(down_read_interruptible);
@@ -1603,6 +1757,7 @@ int __sched down_read_killable(struct rw_semaphore *sem)
return -EINTR;
}
+ rwsem_record_hold(sem, false);
return 0;
}
EXPORT_SYMBOL(down_read_killable);
@@ -1615,8 +1770,10 @@ int down_read_trylock(struct rw_semaphore *sem)
{
int ret = __down_read_trylock(sem);
- if (ret == 1)
+ if (ret == 1) {
rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_);
+ rwsem_record_hold(sem, true);
+ }
return ret;
}
EXPORT_SYMBOL(down_read_trylock);
@@ -1630,6 +1787,7 @@ void __sched down_write(struct rw_semaphore *sem)
might_sleep();
rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
+ rwsem_record_hold(sem, false);
}
EXPORT_SYMBOL(down_write);
@@ -1648,6 +1806,7 @@ int __sched down_write_killable(struct rw_semaphore *sem)
return -EINTR;
}
+ rwsem_record_hold(sem, false);
return 0;
}
EXPORT_SYMBOL(down_write_killable);
@@ -1660,8 +1819,10 @@ int down_write_trylock(struct rw_semaphore *sem)
{
int ret = __down_write_trylock(sem);
- if (ret == 1)
+ if (ret == 1) {
rwsem_acquire(&sem->dep_map, 0, 1, _RET_IP_);
+ rwsem_record_hold(sem, true);
+ }
return ret;
}
@@ -1708,6 +1869,7 @@ void down_read_nested(struct rw_semaphore *sem, int subclass)
might_sleep();
rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_);
LOCK_CONTENDED(sem, __down_read_trylock, __down_read);
+ rwsem_record_hold(sem, false);
}
EXPORT_SYMBOL(down_read_nested);
@@ -1722,6 +1884,7 @@ int down_read_killable_nested(struct rw_semaphore *sem, int subclass)
return -EINTR;
}
+ rwsem_record_hold(sem, false);
return 0;
}
EXPORT_SYMBOL(down_read_killable_nested);
@@ -1732,30 +1895,17 @@ void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest)
might_sleep();
rwsem_acquire_nest(&sem->dep_map, 0, 0, nest, _RET_IP_);
LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
+ rwsem_record_hold(sem, false);
}
EXPORT_SYMBOL(_down_write_nest_lock);
-void down_read_non_owner(struct rw_semaphore *sem)
- __no_context_analysis
-{
- might_sleep();
- __down_read(sem);
- /*
- * The owner value for a reader-owned lock is mostly for debugging
- * purpose only and is not critical to the correct functioning of
- * rwsem. So it is perfectly fine to set it in a preempt-enabled
- * context here.
- */
- __rwsem_set_reader_owned(sem, NULL);
-}
-EXPORT_SYMBOL(down_read_non_owner);
-
void down_write_nested(struct rw_semaphore *sem, int subclass)
__no_context_analysis
{
might_sleep();
rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
+ rwsem_record_hold(sem, false);
}
EXPORT_SYMBOL(down_write_nested);
@@ -1771,15 +1921,34 @@ int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass)
return -EINTR;
}
+ rwsem_record_hold(sem, false);
return 0;
}
EXPORT_SYMBOL(down_write_killable_nested);
+#endif
+
+#if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RWSEM_SCHED_BOOST)
+void down_read_non_owner(struct rw_semaphore *sem)
+ __no_context_analysis
+{
+ might_sleep();
+ __down_read(sem);
+ /*
+ * The owner value for a reader-owned lock is mostly for debugging
+ * purpose only and is not critical to the correct functioning of
+ * rwsem. So it is perfectly fine to set it in a preempt-enabled
+ * context here.
+ */
+ __rwsem_set_reader_owned(sem, NULL);
+}
+EXPORT_SYMBOL(down_read_non_owner);
+
void up_read_non_owner(struct rw_semaphore *sem)
__no_context_analysis
{
DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem);
- __up_read(sem);
+ __up_read_non_owner(sem);
}
EXPORT_SYMBOL(up_read_non_owner);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 5ed660f099a5..5737fe55d52c 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -39,6 +39,7 @@
#include <linux/sched/nohz.h>
#include <linux/sched/rseq_api.h>
#include <linux/sched/rt.h>
+#include <linux/rwsem.h>
#include <linux/context_tracking.h>
#include <linux/cpuset.h>
@@ -1534,12 +1535,108 @@ int tg_nop(struct task_group *tg, void *data)
}
#endif
+#ifdef CONFIG_RWSEM_SCHED_BOOST
+DEFINE_STATIC_KEY_TRUE(rwsem_sched_boost_key);
+
+static int __init rwsem_sched_boost_cmdline(char *str)
+{
+ bool val;
+
+ if (str && !kstrtobool(str, &val) && !val)
+ static_branch_disable(&rwsem_sched_boost_key);
+ return 1;
+}
+__setup("rwsem_sched_boost=", rwsem_sched_boost_cmdline);
+
+static inline bool sched_rwsem_is_boosted(const struct task_struct *p)
+{
+ return p->rwsem_boost_prio < MAX_PRIO;
+}
+
+static inline void sched_rwsem_fork_init(struct task_struct *p)
+{
+ memset(p->held_rwsems, 0, sizeof(p->held_rwsems));
+ p->nr_held_rwsems = 0;
+ p->rwsem_boost_prio = MAX_PRIO;
+ INIT_LIST_HEAD(&p->rwsem_rq_node);
+}
+
+static int sched_rwsem_compute_boost_prio(struct task_struct *p)
+{
+ int best = MAX_PRIO;
+ int i;
+
+ if (!READ_ONCE(p->nr_held_rwsems))
+ return MAX_PRIO;
+
+ for (i = 0; i < CONFIG_RWSEM_SCHED_BOOST_SLOTS; i++) {
+ struct rw_semaphore *sem = READ_ONCE(p->held_rwsems[i]);
+ int wprio;
+
+ if (!sem)
+ continue;
+ wprio = atomic_read(&sem->max_waiter_prio);
+ if (wprio < best)
+ best = wprio;
+ }
+
+ if (best >= MAX_PRIO)
+ return MAX_PRIO;
+ if (best < MAX_RT_PRIO)
+ best = MAX_RT_PRIO;
+ if (best >= p->static_prio && !task_has_idle_policy(p))
+ return MAX_PRIO;
+
+ return best;
+}
+
+/*
+ * Recompute @p's effective priority and EEVDF weight from its held rwsems.
+ * Callers must hold @p's rq->lock (or pi_lock while @p is WAKING) and ensure
+ * @p cannot concurrently execute up_read() + kfree(sem) on another CPU.
+ */
+bool sched_rwsem_check_boost(struct task_struct *p, bool update_load)
+{
+ int new_boost;
+
+ if (p->sched_class != &fair_sched_class)
+ return false;
+
+ new_boost = sched_rwsem_compute_boost_prio(p);
+ if (new_boost == p->rwsem_boost_prio)
+ return false;
+
+ p->rwsem_boost_prio = new_boost;
+ if (!rt_or_dl_prio(p->prio))
+ p->prio = sched_rwsem_effective_prio(p);
+ set_load_weight(p, update_load);
+ return true;
+}
+
+void sched_rwsem_recalc_boost(struct task_struct *p)
+{
+ guard(task_rq_lock)(p);
+
+ if (p->sched_class == &fair_sched_class) {
+ update_rq_clock(task_rq(p));
+ sched_rwsem_check_boost(p, !p->se.sched_delayed);
+ }
+}
+#else
+static inline bool sched_rwsem_is_boosted(const struct task_struct *p)
+{
+ return false;
+}
+
+static inline void sched_rwsem_fork_init(struct task_struct *p) { }
+#endif
+
void set_load_weight(struct task_struct *p, bool update_load)
{
- int prio = p->static_prio - MAX_RT_PRIO;
+ int prio = sched_rwsem_cfs_prio(p) - MAX_RT_PRIO;
struct load_weight lw;
- if (task_has_idle_policy(p)) {
+ if (task_has_idle_policy(p) && !sched_rwsem_is_boosted(p)) {
lw.weight = scale_load(WEIGHT_IDLEPRIO);
lw.inv_weight = WMULT_IDLEPRIO;
} else {
@@ -4888,6 +4985,7 @@ int sched_fork(u64 clone_flags, struct task_struct *p)
* Make sure we do not leak PI boosting priority to the child.
*/
p->prio = current->normal_prio;
+ sched_rwsem_fork_init(p);
uclamp_fork(p);
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 220927453a88..98bc9029f9ae 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -4344,4 +4344,47 @@ void sched_push_current_non_preferred_cpu(struct rq *rq);
static inline void sched_push_current_non_preferred_cpu(struct rq *rq) { }
#endif
+#ifdef CONFIG_RWSEM_SCHED_BOOST
+static inline int sched_rwsem_prio(struct task_struct *p, int base_prio)
+{
+ /*
+ * SCHED_IDLE tasks carry static_prio == NICE_TO_PRIO(0) despite
+ * WEIGHT_IDLEPRIO; use rwsem_boost_prio directly so low-priority
+ * waiters do not boost a SCHED_IDLE holder up to nice 0.
+ */
+ if (task_has_idle_policy(p))
+ return p->rwsem_boost_prio < MAX_PRIO ? p->rwsem_boost_prio
+ : base_prio;
+ return min(base_prio, p->rwsem_boost_prio);
+}
+
+static inline int sched_rwsem_cfs_prio(struct task_struct *p)
+{
+ return sched_rwsem_prio(p, p->static_prio);
+}
+
+static inline int sched_rwsem_effective_prio(struct task_struct *p)
+{
+ return sched_rwsem_prio(p, p->normal_prio);
+}
+
+bool sched_rwsem_check_boost(struct task_struct *p, bool update_load);
+#else
+static inline int sched_rwsem_cfs_prio(const struct task_struct *p)
+{
+ return p->static_prio;
+}
+
+static inline int sched_rwsem_effective_prio(const struct task_struct *p)
+{
+ return p->normal_prio;
+}
+
+static inline bool sched_rwsem_check_boost(struct task_struct *p,
+ bool update_load)
+{
+ return false;
+}
+#endif
+
#endif /* _KERNEL_SCHED_SCHED_H */
diff --git a/kernel/sched/syscalls.c b/kernel/sched/syscalls.c
index 0ec401af80d1..d9e3c61fcfba 100644
--- a/kernel/sched/syscalls.c
+++ b/kernel/sched/syscalls.c
@@ -58,7 +58,7 @@ static int effective_prio(struct task_struct *p)
* to the normal priority:
*/
if (!rt_or_dl_prio(p->prio))
- return p->normal_prio;
+ return sched_rwsem_effective_prio(p);
return p->prio;
}
--
2.56.1.750.gf70dc682cf-goog
next prev parent reply other threads:[~2026-11-19 22:50 UTC|newest]
Thread overview: 4+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-19 22:49 [RFC PATCH 0/3] rwsem: priority boost for holders blocking high-priority waiters Peng Huang
2026-11-19 22:49 ` Peng Huang [this message]
2026-11-19 22:49 ` [RFC PATCH 2/3] sched: boost rwsem holders with contended " Peng Huang
2026-11-19 22:49 ` [RFC PATCH 3/3] locking/rwsem: add KUnit tests for scheduler priority boost Peng Huang
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261119224954.196990-2-penghuang@google.com \
--to=penghuang@google.com \
--cc=boqun@kernel.org \
--cc=bsegall@google.com \
--cc=dietmar.eggemann@arm.com \
--cc=juri.lelli@redhat.com \
--cc=kprateek.nayak@amd.com \
--cc=linux-kernel@vger.kernel.org \
--cc=longman@redhat.com \
--cc=mgorman@suse.de \
--cc=mingo@redhat.com \
--cc=peterz@infradead.org \
--cc=rostedt@goodmis.org \
--cc=vincent.guittot@linaro.org \
--cc=vschneid@redhat.com \
--cc=will@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®