mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: Peng Huang <penghuang@google.com>
To: Peter Zijlstra <peterz@infradead.org>,
	Ingo Molnar <mingo@redhat.com>,  Will Deacon <will@kernel.org>,
	Boqun Feng <boqun@kernel.org>, Waiman Long <longman@redhat.com>,
	 Juri Lelli <juri.lelli@redhat.com>,
	Vincent Guittot <vincent.guittot@linaro.org>
Cc: Dietmar Eggemann <dietmar.eggemann@arm.com>,
	Steven Rostedt <rostedt@goodmis.org>,
	 Ben Segall <bsegall@google.com>, Mel Gorman <mgorman@suse.de>,
	 Valentin Schneider <vschneid@redhat.com>,
	K Prateek Nayak <kprateek.nayak@amd.com>,
	 linux-kernel@vger.kernel.org, Peng Huang <penghuang@google.com>
Subject: [RFC PATCH 2/3] sched: boost rwsem holders with contended high-priority waiters
Date: Thu, 19 Nov 2026 14:49:53 -0800	[thread overview]
Message-ID: <20261119224954.196990-3-penghuang@google.com> (raw)
In-Reply-To: <20261119224954.196990-1-penghuang@google.com>

Keep queued fair tasks holding rwsems on rq->held_rwsem_tasks and check
waiter demand across scheduler wakeup, preemption, tick, and idle
load-balance pull paths.

Uncontended lock/unlock paths never touch rq->held_rwsem_tasks; a task
is linked only if preempted, enqueued, or crossing a tick while holding
an rwsem. Thus rq->held_rwsem_tasks stays empty for the vast majority of
ticks and idle pulls, exiting immediately with zero iteration or
max_waiter_prio lookups.

When a fair task holds a contended rwsem (max_waiter_prio < task prio),
temporarily elevate its effective priority and EEVDF weight without
altering user-visible static_prio/nice, and restore it on unlock or
lazily unlink uncontended unlocks on tick.

Signed-off-by: Peng Huang <penghuang@google.com>
---
 kernel/sched/core.c  | 109 ++++++++++++++++++++++++++++++++++++++++++-
 kernel/sched/fair.c  |  51 ++++++++++++++++++++
 kernel/sched/sched.h |  27 +++++++++++
 3 files changed, 185 insertions(+), 2 deletions(-)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 5737fe55d52c..2ead771c703a 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1561,7 +1561,12 @@ static inline void sched_rwsem_fork_init(struct task_struct *p)
 	INIT_LIST_HEAD(&p->rwsem_rq_node);
 }
 
-static int sched_rwsem_compute_boost_prio(struct task_struct *p)
+static inline void sched_rwsem_rq_init(struct rq *rq)
+{
+	INIT_LIST_HEAD(&rq->held_rwsem_tasks);
+}
+
+int sched_rwsem_compute_boost_prio(struct task_struct *p)
 {
 	int best = MAX_PRIO;
 	int i;
@@ -1613,10 +1618,88 @@ bool sched_rwsem_check_boost(struct task_struct *p, bool update_load)
 	return true;
 }
 
+static inline void sched_rwsem_track_on_rq(struct rq *rq, struct task_struct *p)
+{
+	if (list_empty(&p->rwsem_rq_node))
+		list_add(&p->rwsem_rq_node, &rq->held_rwsem_tasks);
+}
+
+/*
+ * Safe to inspect p->held_rwsems[] here because ttwu() holds p->pi_lock
+ * after smp_cond_load_acquire(&p->on_cpu, !VAL), so @p is off-CPU and
+ * cannot concurrently run up_read()/kfree(sem).
+ */
+static inline void sched_rwsem_wakeup_boost(struct task_struct *p)
+{
+	if (READ_ONCE(p->nr_held_rwsems))
+		sched_rwsem_check_boost(p, false);
+}
+
+static inline void sched_rwsem_enqueue_locked(struct rq *rq,
+					      struct task_struct *p)
+{
+	if (!READ_ONCE(p->nr_held_rwsems) ||
+	    p->sched_class != &fair_sched_class)
+		return;
+
+	sched_rwsem_check_boost(p, false);
+	sched_rwsem_track_on_rq(rq, p);
+}
+
+static inline void sched_rwsem_on_preempt_locked(struct rq *rq,
+						 struct task_struct *prev)
+{
+	if (!READ_ONCE(prev->nr_held_rwsems) || task_rq(prev) != rq ||
+	    !task_on_rq_queued(prev) || prev->se.sched_delayed ||
+	    prev->sched_class != &fair_sched_class)
+		return;
+
+	sched_rwsem_track_on_rq(rq, prev);
+	sched_rwsem_check_boost(prev, true);
+}
+
+static inline void sched_rwsem_tick_locked(struct rq *rq)
+{
+	struct task_struct *curr = rq->curr;
+	struct task_struct *p, *tmp;
+	bool need_resched = false;
+
+	if (READ_ONCE(curr->nr_held_rwsems) &&
+	    curr->sched_class == &fair_sched_class &&
+	    task_rq(curr) == rq && task_on_rq_queued(curr) &&
+	    !curr->se.sched_delayed) {
+		sched_rwsem_track_on_rq(rq, curr);
+		sched_rwsem_check_boost(curr, true);
+	}
+
+	/*
+	 * Lazily unlink tasks whose unlock fastpath skipped rq->lock because
+	 * they were never boosted (!nr_held_rwsems short-circuits sem access),
+	 * or if proxy/delayed-dequeue movement left a stale rq link.
+	 */
+	list_for_each_entry_safe(p, tmp, &rq->held_rwsem_tasks, rwsem_rq_node) {
+		if (!READ_ONCE(p->nr_held_rwsems) ||
+		    task_rq(p) != rq || !task_on_rq_queued(p)) {
+			sched_rwsem_dequeue_locked(p);
+			continue;
+		}
+		if (p == curr || task_on_cpu(rq, p) || p->se.sched_delayed)
+			continue;
+		sched_rwsem_check_boost(p, true);
+		if (p->rwsem_boost_prio < curr->prio)
+			need_resched = true;
+	}
+
+	if (need_resched)
+		resched_curr(rq);
+}
+
 void sched_rwsem_recalc_boost(struct task_struct *p)
 {
 	guard(task_rq_lock)(p);
 
+	if (!READ_ONCE(p->nr_held_rwsems))
+		sched_rwsem_dequeue_locked(p);
 	if (p->sched_class == &fair_sched_class) {
 		update_rq_clock(task_rq(p));
 		sched_rwsem_check_boost(p, !p->se.sched_delayed);
@@ -1629,6 +1712,16 @@ static inline bool sched_rwsem_is_boosted(const struct task_struct *p)
 }
 
 static inline void sched_rwsem_fork_init(struct task_struct *p) { }
+static inline void sched_rwsem_rq_init(struct rq *rq) { }
+static inline void sched_rwsem_wakeup_boost(struct task_struct *p) { }
+
+static inline void sched_rwsem_enqueue_locked(struct rq *rq,
+					      struct task_struct *p) { }
+
+static inline void sched_rwsem_on_preempt_locked(struct rq *rq,
+						 struct task_struct *prev) { }
+
+static inline void sched_rwsem_tick_locked(struct rq *rq) { }
 #endif
 
 void set_load_weight(struct task_struct *p, bool update_load)
@@ -2284,6 +2377,8 @@ void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
 	if (!(flags & ENQUEUE_NOCLOCK))
 		update_rq_clock(rq);
 
+	sched_rwsem_enqueue_locked(rq, p);
+
 	/*
 	 * Can be before ->enqueue_task() because uclamp considers the
 	 * ENQUEUE_DELAYED task before its ->sched_delayed gets cleared
@@ -2323,7 +2418,11 @@ inline bool dequeue_task(struct rq *rq, struct task_struct *p, int flags)
 	 * and mark the task ->sched_delayed.
 	 */
 	uclamp_rq_dec(rq, p);
-	return p->sched_class->dequeue_task(rq, p, flags);
+	if (p->sched_class->dequeue_task(rq, p, flags)) {
+		sched_rwsem_dequeue_locked(p);
+		return true;
+	}
+	return false;
 }
 
 void activate_task(struct rq *rq, struct task_struct *p, int flags)
@@ -4566,6 +4665,8 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
 		 */
 		smp_cond_load_acquire(&p->on_cpu, !VAL);
 
+		sched_rwsem_wakeup_boost(p);
+
 		cpu = select_task_rq(p, p->wake_cpu, &wake_flags);
 		if (task_cpu(p) != cpu) {
 			wake_flags |= WF_MIGRATED;
@@ -5957,6 +6058,7 @@ void sched_tick(void)
 		resched_curr(rq);
 
 	donor->sched_class->task_tick(rq, donor, 0);
+	sched_rwsem_tick_locked(rq);
 	if (sched_feat(LATENCY_WARN))
 		resched_latency = cpu_resched_latency(rq);
 	calc_global_load_tick(rq);
@@ -7391,6 +7493,8 @@ static void __sched notrace __schedule(int sched_mode)
 		switch_count = &prev->nvcsw;
 	}
 
+	sched_rwsem_on_preempt_locked(rq, prev);
+
 pick_again:
 	assert_balance_callbacks_empty(rq);
 	next = pick_next_task(rq, &rf);
@@ -9125,6 +9229,7 @@ void __init sched_init(void)
 		rq->max_idle_balance_cost = sysctl_sched_migration_cost;
 
 		INIT_LIST_HEAD(&rq->cfs_tasks);
+		sched_rwsem_rq_init(rq);
 
 		rq_attach_root(rq, &def_root_domain);
 #ifdef CONFIG_NO_HZ_COMMON
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 0c8af1cac2d5..59ce6321a265 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -11312,6 +11312,53 @@ static struct task_struct *detach_one_task(struct lb_env *env)
 	return NULL;
 }
 
+#ifdef CONFIG_RWSEM_SCHED_BOOST
+/*
+ * On idle pull, prioritize migrating a fair task holding a contended rwsem
+ * so a blocked high-priority waiter is not starved on a busy little core.
+ */
+static int detach_boosted_rwsem_task(struct lb_env *env)
+{
+	struct task_struct *p, *tmp;
+
+	if (env->idle != CPU_IDLE && env->idle != CPU_NEWLY_IDLE)
+		return 0;
+
+	list_for_each_entry_safe(p, tmp, &env->src_rq->held_rwsem_tasks,
+				 rwsem_rq_node) {
+		if (env->src_rq->nr_running <= 1)
+			break;
+		if (task_rq(p) != env->src_rq || !task_on_rq_queued(p)) {
+			sched_rwsem_dequeue_locked(p);
+			continue;
+		}
+		if (task_on_cpu(env->src_rq, p) || p->se.sched_delayed)
+			continue;
+		if (sched_rwsem_compute_boost_prio(p) >= MAX_PRIO)
+			continue;
+		if (!can_migrate_task(p, env))
+			continue;
+
+		/*
+		 * enqueue_task() on dst_rq updates the boosted weight
+		 * via sched_rwsem_enqueue_locked(), avoiding a redundant
+		 * reweight_eevdf() on src_rq before detaching.
+		 */
+		detach_task(p, env);
+		list_add(&p->se.group_node, &env->tasks);
+		env->imbalance = 0;
+		return 1;
+	}
+
+	return 0;
+}
+#else
+static inline int detach_boosted_rwsem_task(struct lb_env *env)
+{
+	return 0;
+}
+#endif
+
 /*
  * detach_tasks() -- tries to detach up to imbalance load/util/tasks from
  * busiest_rq, as part of a balancing operation within domain "sd".
@@ -11339,6 +11386,10 @@ static int detach_tasks(struct lb_env *env)
 	if (env->imbalance <= 0)
 		return 0;
 
+	detached = detach_boosted_rwsem_task(env);
+	if (detached)
+		return detached;
+
 	while (!list_empty(tasks)) {
 		/*
 		 * We don't want to steal all, otherwise we may be treated likewise,
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 98bc9029f9ae..58e53980e71b 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1317,6 +1317,13 @@ struct rq {
 	int			online;
 
 	struct list_head cfs_tasks;
+#ifdef CONFIG_RWSEM_SCHED_BOOST
+	/*
+	 * Queued fair tasks holding an rwsem; protected by rq->lock and
+	 * unlinked on dequeue_task() (incl. migration and TASK_DEAD).
+	 */
+	struct list_head	held_rwsem_tasks;
+#endif
 
 	struct sched_avg	avg_rt;
 	struct sched_avg	avg_dl;
@@ -4368,7 +4375,18 @@ static inline int sched_rwsem_effective_prio(struct task_struct *p)
 	return sched_rwsem_prio(p, p->normal_prio);
 }
 
+int sched_rwsem_compute_boost_prio(struct task_struct *p);
 bool sched_rwsem_check_boost(struct task_struct *p, bool update_load);
+
+/*
+ * Remove @p when it leaves an rq (sleep, migration, class change,
+ * TASK_DEAD via DEQUEUE_SPECIAL, or stale cross-rq cleanup).
+ */
+static inline void sched_rwsem_dequeue_locked(struct task_struct *p)
+{
+	if (!list_empty(&p->rwsem_rq_node))
+		list_del_init(&p->rwsem_rq_node);
+}
 #else
 static inline int sched_rwsem_cfs_prio(const struct task_struct *p)
 {
@@ -4380,11 +4398,20 @@ static inline int sched_rwsem_effective_prio(const struct task_struct *p)
 	return p->normal_prio;
 }
 
+static inline int sched_rwsem_compute_boost_prio(struct task_struct *p)
+{
+	return MAX_PRIO;
+}
+
 static inline bool sched_rwsem_check_boost(struct task_struct *p,
 					   bool update_load)
 {
 	return false;
 }
+
+static inline void sched_rwsem_dequeue_locked(struct task_struct *p)
+{
+}
 #endif
 
 #endif /* _KERNEL_SCHED_SCHED_H */
-- 
2.56.1.750.gf70dc682cf-goog


  parent reply	other threads:[~2026-11-19 22:50 UTC|newest]

Thread overview: 4+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-19 22:49 [RFC PATCH 0/3] rwsem: priority boost for holders blocking " Peng Huang
2026-11-19 22:49 ` [RFC PATCH 1/3] locking/rwsem, sched: track held rwsems and waiter priorities Peng Huang
2026-11-19 22:49 ` Peng Huang [this message]
2026-11-19 22:49 ` [RFC PATCH 3/3] locking/rwsem: add KUnit tests for scheduler priority boost Peng Huang

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261119224954.196990-3-penghuang@google.com \
    --to=penghuang@google.com \
    --cc=boqun@kernel.org \
    --cc=bsegall@google.com \
    --cc=dietmar.eggemann@arm.com \
    --cc=juri.lelli@redhat.com \
    --cc=kprateek.nayak@amd.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=longman@redhat.com \
    --cc=mgorman@suse.de \
    --cc=mingo@redhat.com \
    --cc=peterz@infradead.org \
    --cc=rostedt@goodmis.org \
    --cc=vincent.guittot@linaro.org \
    --cc=vschneid@redhat.com \
    --cc=will@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®