From: K Prateek Nayak <kprateek.nayak@amd.com>
To: Peter Zijlstra <peterz@infradead.org>,
Ingo Molnar <mingo@redhat.com>,
Juri Lelli <juri.lelli@redhat.com>,
Vincent Guittot <vincent.guittot@linaro.org>,
Valentin Schneider <vschneid@redhat.com>,
"Ben Segall" <bsegall@google.com>,
Thomas Gleixner <tglx@linutronix.de>,
"Andy Lutomirski" <luto@kernel.org>,
<linux-kernel@vger.kernel.org>
Cc: Dietmar Eggemann <dietmar.eggemann@arm.com>,
Steven Rostedt <rostedt@goodmis.org>,
Mel Gorman <mgorman@suse.de>,
"Sebastian Andrzej Siewior" <bigeasy@linutronix.de>,
Clark Williams <clrkwllms@kernel.org>,
<linux-rt-devel@lists.linux.dev>, Tejun Heo <tj@kernel.org>,
"Frederic Weisbecker" <frederic@kernel.org>,
Barret Rhoden <brho@google.com>, "Petr Mladek" <pmladek@suse.com>,
Josh Don <joshdon@google.com>, Qais Yousef <qyousef@layalina.io>,
"Paul E. McKenney" <paulmck@kernel.org>,
David Vernet <dvernet@meta.com>,
K Prateek Nayak <kprateek.nayak@amd.com>,
"Gautham R. Shenoy" <gautham.shenoy@amd.com>,
Swapnil Sapkal <swapnil.sapkal@amd.com>
Subject: [RFC PATCH 09/22] sched/fair: Introduce cfs_rq throttled states in preparation for partial throttling
Date: Thu, 20 Feb 2025 09:32:44 +0000 [thread overview]
Message-ID: <20250220093257.9380-10-kprateek.nayak@amd.com> (raw)
In-Reply-To: <20250220093257.9380-1-kprateek.nayak@amd.com>
On introduction of throttle deferral, cfs_rq on encountering a throttle
event with kernel mode preempted entities will be marked partially
throttled allowing only these kernel mode entities to run until they
sleep or exit to userspace.
Introduce "throttled_state" enum to define the three throttle states -
CFS_UNTHROTTLED, CFS_THROTTLED_PARTIAL, and CFS_THROTTLED. In addition
to cfs_rq_throttled() which will now track both partial and complete
throttle, introduce a new helper cfs_rq_h_throttled() to detect a
completely throttled hierarchy.
Convert appropriate cfs_rq_throttled() checks to cfs_rq_h_throttled() to
guard logic that only concerns with a complete throttle. Also take the
opportunity to convert any open coded references to cfs_tr->throttled to
use one of the two helpers.
Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
kernel/sched/fair.c | 67 ++++++++++++++++++++++++++++++++++-----------
1 file changed, 51 insertions(+), 16 deletions(-)
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 4fd70012b479..c84cd2d92343 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -5340,7 +5340,11 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
}
static void check_enqueue_throttle(struct cfs_rq *cfs_rq);
+
+/* cfs_rq is throttled either completely or partially */
static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq);
+/* cfs_rq is throttled completely and the hierarchy is frozen */
+static inline int cfs_rq_h_throttled(struct cfs_rq *cfs_rq);
static void
requeue_delayed_entity(struct sched_entity *se);
@@ -5404,7 +5408,7 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
#ifdef CONFIG_CFS_BANDWIDTH
struct rq *rq = rq_of(cfs_rq);
- if (cfs_rq_throttled(cfs_rq) && !cfs_rq->throttled_clock)
+ if (cfs_rq_h_throttled(cfs_rq) && !cfs_rq->throttled_clock)
cfs_rq->throttled_clock = rq_clock(rq);
if (!cfs_rq->throttled_clock_self)
cfs_rq->throttled_clock_self = rq_clock(rq);
@@ -5448,7 +5452,7 @@ static void set_delayed(struct sched_entity *se)
struct cfs_rq *cfs_rq = cfs_rq_of(se);
cfs_rq->h_nr_runnable--;
- if (cfs_rq_throttled(cfs_rq))
+ if (cfs_rq_h_throttled(cfs_rq))
break;
}
}
@@ -5470,7 +5474,7 @@ static void clear_delayed(struct sched_entity *se)
struct cfs_rq *cfs_rq = cfs_rq_of(se);
cfs_rq->h_nr_runnable++;
- if (cfs_rq_throttled(cfs_rq))
+ if (cfs_rq_h_throttled(cfs_rq))
break;
}
}
@@ -5817,7 +5821,7 @@ static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
if (likely(cfs_rq->runtime_remaining > 0))
return;
- if (cfs_rq->throttled)
+ if (cfs_rq_throttled(cfs_rq))
return;
/*
* if we're unable to extend our runtime we resched so that the active
@@ -5836,11 +5840,37 @@ void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
__account_cfs_rq_runtime(cfs_rq, delta_exec);
}
+/* cfs_rq->throttled states */
+enum throttle_state {
+ /*
+ * cfs_rq is unthrottled; All the queued entities
+ * can be picked to run.
+ */
+ CFS_UNTHROTTLED = 0,
+ /*
+ * cfs_rq is only marked throttled. There are kernel
+ * mode preempted entities that are still runnable.
+ * PELT is not frozen yet.
+ */
+ CFS_THROTTLED_PARTIAL,
+ /*
+ * cfs_rq is fully throttled with PELT frozen. There
+ * are no entities that are considered runnable under
+ * throttle.
+ */
+ CFS_THROTTLED
+};
+
static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
{
return cfs_bandwidth_used() && cfs_rq->throttled;
}
+static inline int cfs_rq_h_throttled(struct cfs_rq *cfs_rq)
+{
+ return cfs_bandwidth_used() && (cfs_rq->throttled == CFS_THROTTLED);
+}
+
/* check whether cfs_rq, or any parent, is throttled */
static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
{
@@ -6011,7 +6041,7 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq)
* Note: distribution will already see us throttled via the
* throttled-list. rq->lock protects completion.
*/
- cfs_rq->throttled = 1;
+ cfs_rq->throttled = CFS_THROTTLED;
SCHED_WARN_ON(cfs_rq->throttled_clock);
if (cfs_rq->nr_queued)
cfs_rq->throttled_clock = rq_clock(rq);
@@ -6028,7 +6058,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
se = cfs_rq->tg->se[cpu_of(rq)];
- cfs_rq->throttled = 0;
+ cfs_rq->throttled = CFS_UNTHROTTLED;
update_rq_clock(rq);
@@ -6080,7 +6110,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
qcfs_rq->h_nr_idle += idle_delta;
/* end evaluation on encountering a throttled cfs_rq */
- if (cfs_rq_throttled(qcfs_rq))
+ if (cfs_rq_h_throttled(qcfs_rq))
goto unthrottle_throttle;
}
@@ -6098,7 +6128,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
qcfs_rq->h_nr_idle += idle_delta;
/* end evaluation on encountering a throttled cfs_rq */
- if (cfs_rq_throttled(qcfs_rq))
+ if (cfs_rq_h_throttled(qcfs_rq))
goto unthrottle_throttle;
}
@@ -6503,7 +6533,7 @@ static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq)
* it's possible for a throttled entity to be forced into a running
* state (e.g. set_curr_task), in this case we're finished.
*/
- if (cfs_rq_throttled(cfs_rq))
+ if (cfs_rq_h_throttled(cfs_rq))
return true;
return throttle_cfs_rq(cfs_rq);
@@ -7029,6 +7059,11 @@ static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
return 0;
}
+static inline int cfs_rq_h_throttled(struct cfs_rq *cfs_rq)
+{
+ return 0;
+}
+
static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
{
return 0;
@@ -7310,7 +7345,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
h_nr_idle = 1;
/* end evaluation on encountering a throttled cfs_rq */
- if (cfs_rq_throttled(cfs_rq))
+ if (cfs_rq_h_throttled(cfs_rq))
goto enqueue_throttle;
flags = ENQUEUE_WAKEUP;
@@ -7337,7 +7372,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
h_nr_idle = 1;
/* end evaluation on encountering a throttled cfs_rq */
- if (cfs_rq_throttled(cfs_rq))
+ if (cfs_rq_h_throttled(cfs_rq))
goto enqueue_throttle;
}
@@ -7431,7 +7466,7 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags)
h_nr_idle = h_nr_queued;
/* end evaluation on encountering a throttled cfs_rq */
- if (cfs_rq_throttled(cfs_rq))
+ if (cfs_rq_h_throttled(cfs_rq))
return 0;
/* Don't dequeue parent if it has other entities besides us */
@@ -7472,8 +7507,8 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags)
if (cfs_rq_is_idle(cfs_rq))
h_nr_idle = h_nr_queued;
- /* end evaluation on encountering a throttled cfs_rq */
- if (cfs_rq_throttled(cfs_rq))
+ /* end evaluation on encountering a throttled cfs_rq hierarchy */
+ if (cfs_rq_h_throttled(cfs_rq))
return 0;
}
@@ -13519,7 +13554,7 @@ static void propagate_entity_cfs_rq(struct sched_entity *se)
{
struct cfs_rq *cfs_rq = cfs_rq_of(se);
- if (cfs_rq_throttled(cfs_rq))
+ if (cfs_rq_h_throttled(cfs_rq))
return;
if (!throttled_hierarchy(cfs_rq))
@@ -13533,7 +13568,7 @@ static void propagate_entity_cfs_rq(struct sched_entity *se)
update_load_avg(cfs_rq, se, UPDATE_TG);
- if (cfs_rq_throttled(cfs_rq))
+ if (cfs_rq_h_throttled(cfs_rq))
break;
if (!throttled_hierarchy(cfs_rq))
--
2.43.0
next prev parent reply other threads:[~2025-02-20 9:36 UTC|newest]
Thread overview: 36+ messages / expand[flat|nested] mbox.gz Atom feed top
2025-02-20 9:32 [RFC PATCH 00/22] sched/fair: Defer CFS throttling to exit to user mode K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 01/22] kernel/entry/common: Move syscall_enter_from_user_mode_work() out of header K Prateek Nayak
2025-02-20 10:43 ` Peter Zijlstra
2025-02-20 10:56 ` K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 02/22] sched/fair: Convert "se->runnable_weight" to unsigned int and pack the struct K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 03/22] [PoC] kernel/entry/common: Mark syscall as a kernel critical section K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 04/22] [PoC] kernel/sched: Inititalize "kernel_cs_count" for new tasks K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 05/22] sched/fair: Track EEVDF stats for entities preempted in kernel mode K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 06/22] sched/fair: Propagate the min_vruntime of kernel mode preempted entity K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 07/22] sched/fair: Propagate preempted entity information up cgroup hierarchy K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 08/22] sched/fair: Allow pick_eevdf() to pick in-kernel entities on throttled hierarchy K Prateek Nayak
2025-02-20 9:32 ` K Prateek Nayak [this message]
2025-02-20 9:32 ` [RFC PATCH 10/22] sched/fair: Prepare throttle_cfs_rq() to allow partial throttling K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 11/22] sched/fair: Prepare unthrottle_cfs_rq() to demote throttle status K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 12/22] sched/fair: Prepare bandwidth distribution to unthrottle partial throttles right away K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 13/22] sched/fair: Correct the throttle status supplied to pick_eevdf() K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 14/22] sched/fair: Mark a task if it was picked on a partially throttled hierarchy K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 15/22] sched/fair: Call resched_curr() from sched_notify_syscall_exit() K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 16/22] sched/fair: Prepare enqueue to partially unthrottle cfs_rq K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 17/22] sched/fair: Clear pick on throttled indicator when task leave fair class K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 18/22] sched/fair: Prepare pick_next_task_fair() to unthrottle a throttled hierarchy K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 19/22] sched/fair: Ignore in-kernel indicators for running task outside of schedule() K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 20/22] sched/fair: Implement determine_throttle_state() for partial throttle K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 21/22] [MAYBE BUGFIX] sched/fair: Group all the se->min_* members together for propagation K Prateek Nayak
2025-02-20 9:32 ` [RFC PATCH 22/22] [DEBUG] sched/fair: Debug pick_eevdf() returning NULL! K Prateek Nayak
2025-02-20 10:55 ` [RFC PATCH 00/22] sched/fair: Defer CFS throttling to exit to user mode Peter Zijlstra
2025-02-20 11:18 ` K Prateek Nayak
2025-02-20 11:32 ` Peter Zijlstra
2025-02-20 12:04 ` K Prateek Nayak
2025-02-21 2:04 ` Josh Don
2025-02-21 3:37 ` K Prateek Nayak
2025-02-21 19:42 ` Josh Don
2025-02-20 15:40 ` Valentin Schneider
2025-02-20 16:58 ` K Prateek Nayak
2025-02-21 1:47 ` Josh Don
2025-02-25 21:13 ` Valentin Schneider
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20250220093257.9380-10-kprateek.nayak@amd.com \
--to=kprateek.nayak@amd.com \
--cc=bigeasy@linutronix.de \
--cc=brho@google.com \
--cc=bsegall@google.com \
--cc=clrkwllms@kernel.org \
--cc=dietmar.eggemann@arm.com \
--cc=dvernet@meta.com \
--cc=frederic@kernel.org \
--cc=gautham.shenoy@amd.com \
--cc=joshdon@google.com \
--cc=juri.lelli@redhat.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-rt-devel@lists.linux.dev \
--cc=luto@kernel.org \
--cc=mgorman@suse.de \
--cc=mingo@redhat.com \
--cc=paulmck@kernel.org \
--cc=peterz@infradead.org \
--cc=pmladek@suse.com \
--cc=qyousef@layalina.io \
--cc=rostedt@goodmis.org \
--cc=swapnil.sapkal@amd.com \
--cc=tglx@linutronix.de \
--cc=tj@kernel.org \
--cc=vincent.guittot@linaro.org \
--cc=vschneid@redhat.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®