* [PATCH v5 1/3] sched/fair: Co-locate cfs_rq and sched_entity
2025-11-07 4:38 [PATCH v5 0/3] sched/fair: Optimize cfs_rq and sched_entity allocation for better data locality Zecheng Li
@ 2025-11-07 4:38 ` Zecheng Li
2025-11-07 4:38 ` [PATCH v5 2/3] sched/fair: Remove task_group->se pointer array Zecheng Li
2025-11-07 4:38 ` [PATCH v5 3/3] sched/fair: Allocate both cfs_rq and sched_entity with per-cpu Zecheng Li
2 siblings, 0 replies; 4+ messages in thread
From: Zecheng Li @ 2025-11-07 4:38 UTC (permalink / raw)
To: Ingo Molnar, Peter Zijlstra, Juri Lelli, Vincent Guittot
Cc: Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, Rik van Riel, Chris Mason,
Madadi Vineeth Reddy, Xu Liu, Blake Jones, Josh Don,
linux-kernel, Zecheng Li
Improve data locality and reduce pointer chasing by allocating struct
cfs_rq and struct sched_entity together for non-root task groups. This
is achieved by introducing a new combined struct cfs_rq_with_se that
holds both objects in a single allocation.
This patch:
- Defines the new struct cfs_rq_with_se.
- Modifies alloc_fair_sched_group() and free_fair_sched_group() to
allocate and free the new struct as a single unit.
- Modifies the per-CPU pointers in task_group->se and task_group->cfs_rq
to point to the members in the new combined structure.
Signed-off-by: Zecheng Li <zecheng@google.com>
---
kernel/sched/fair.c | 23 ++++++++++-------------
kernel/sched/sched.h | 8 ++++++++
2 files changed, 18 insertions(+), 13 deletions(-)
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 273e2871b59e..1676119e302b 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -13359,10 +13359,11 @@ void free_fair_sched_group(struct task_group *tg)
int i;
for_each_possible_cpu(i) {
- if (tg->cfs_rq)
- kfree(tg->cfs_rq[i]);
- if (tg->se)
- kfree(tg->se[i]);
+ if (tg->cfs_rq && tg->cfs_rq[i]) {
+ struct cfs_rq_with_se *combined =
+ container_of(tg->cfs_rq[i], struct cfs_rq_with_se, cfs_rq);
+ kfree(combined);
+ }
}
kfree(tg->cfs_rq);
@@ -13371,6 +13372,7 @@ void free_fair_sched_group(struct task_group *tg)
int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
{
+ struct cfs_rq_with_se *combined;
struct sched_entity *se;
struct cfs_rq *cfs_rq;
int i;
@@ -13387,16 +13389,13 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent));
for_each_possible_cpu(i) {
- cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
+ combined = kzalloc_node(sizeof(struct cfs_rq_with_se),
GFP_KERNEL, cpu_to_node(i));
- if (!cfs_rq)
+ if (!combined)
goto err;
- se = kzalloc_node(sizeof(struct sched_entity_stats),
- GFP_KERNEL, cpu_to_node(i));
- if (!se)
- goto err_free_rq;
-
+ cfs_rq = &combined->cfs_rq;
+ se = &combined->se;
init_cfs_rq(cfs_rq);
init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);
init_entity_runnable_average(se);
@@ -13404,8 +13403,6 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
return 1;
-err_free_rq:
- kfree(cfs_rq);
err:
return 0;
}
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index d04e007608a3..8db53f4d4d06 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -769,6 +769,14 @@ struct cfs_rq {
#endif /* CONFIG_FAIR_GROUP_SCHED */
};
+#ifdef CONFIG_FAIR_GROUP_SCHED
+struct cfs_rq_with_se {
+ struct cfs_rq cfs_rq;
+ /* cfs_rq's sched_entity on parent runqueue */
+ struct sched_entity se ____cacheline_aligned;
+};
+#endif
+
#ifdef CONFIG_SCHED_CLASS_EXT
/* scx_rq->flags, protected by the rq lock */
enum scx_rq_flags {
--
2.51.2.1041.gc1ab5b90ca-goog
^ permalink raw reply [flat|nested] 4+ messages in thread* [PATCH v5 2/3] sched/fair: Remove task_group->se pointer array
2025-11-07 4:38 [PATCH v5 0/3] sched/fair: Optimize cfs_rq and sched_entity allocation for better data locality Zecheng Li
2025-11-07 4:38 ` [PATCH v5 1/3] sched/fair: Co-locate cfs_rq and sched_entity Zecheng Li
@ 2025-11-07 4:38 ` Zecheng Li
2025-11-07 4:38 ` [PATCH v5 3/3] sched/fair: Allocate both cfs_rq and sched_entity with per-cpu Zecheng Li
2 siblings, 0 replies; 4+ messages in thread
From: Zecheng Li @ 2025-11-07 4:38 UTC (permalink / raw)
To: Ingo Molnar, Peter Zijlstra, Juri Lelli, Vincent Guittot
Cc: Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, Rik van Riel, Chris Mason,
Madadi Vineeth Reddy, Xu Liu, Blake Jones, Josh Don,
linux-kernel, Zecheng Li
Now that struct sched_entity is co-located with struct cfs_rq for
non-root task groups, the task_group->se pointer array is redundant. The
associated sched_entity can be loaded directly from the cfs_rq.
This patch performs the access conversion with the helpers:
- is_root_task_group(tg): checks if a task group is the root task group.
It compares the task group's address with the global root_task_group
variable.
- tg_se(tg, cpu): retrieves the cfs_rq and returns the address of the
co-located se. This function checks if tg is the root task group to
ensure behaving the same of previous tg->se[cpu]. Replaces all accesses
that use the tg->se[cpu] pointer array with calls to the new tg_se(tg,
cpu) accessor.
- cfs_rq_se(cfs_rq): simplifies access paths like cfs_rq->tg->se[...] to
use the co-located sched_entity. This function also checks if tg is the
root task group to ensure same behavior.
Since tg_se is not in very hot code paths, and the branch is a register
comparison with an immediate value (`&root_task_group`), the performance
impact is expected to be negligible.
Signed-off-by: Zecheng Li <zecheng@google.com>
---
kernel/sched/core.c | 7 ++-----
kernel/sched/debug.c | 2 +-
kernel/sched/fair.c | 27 ++++++++++-----------------
kernel/sched/sched.h | 29 ++++++++++++++++++++++++-----
4 files changed, 37 insertions(+), 28 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 67b5f2faab36..12ebe1b4c8ae 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -8558,7 +8558,7 @@ void __init sched_init(void)
wait_bit_init();
#ifdef CONFIG_FAIR_GROUP_SCHED
- ptr += 2 * nr_cpu_ids * sizeof(void **);
+ ptr += nr_cpu_ids * sizeof(void **);
#endif
#ifdef CONFIG_RT_GROUP_SCHED
ptr += 2 * nr_cpu_ids * sizeof(void **);
@@ -8567,9 +8567,6 @@ void __init sched_init(void)
ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
#ifdef CONFIG_FAIR_GROUP_SCHED
- root_task_group.se = (struct sched_entity **)ptr;
- ptr += nr_cpu_ids * sizeof(void **);
-
root_task_group.cfs_rq = (struct cfs_rq **)ptr;
ptr += nr_cpu_ids * sizeof(void **);
@@ -9635,7 +9632,7 @@ static int cpu_cfs_stat_show(struct seq_file *sf, void *v)
int i;
for_each_possible_cpu(i) {
- stats = __schedstats_from_se(tg->se[i]);
+ stats = __schedstats_from_se(tg_se(tg, i));
ws += schedstat_val(stats->wait_sum);
}
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 02e16b70a790..16542596d4b0 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -644,7 +644,7 @@ void dirty_sched_domain_sysctl(int cpu)
#ifdef CONFIG_FAIR_GROUP_SCHED
static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group *tg)
{
- struct sched_entity *se = tg->se[cpu];
+ struct sched_entity *se = tg_se(tg, cpu);
#define P(F) SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)F)
#define P_SCHEDSTAT(F) SEQ_printf(m, " .%-30s: %lld\n", \
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 1676119e302b..f9fb07d73a03 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -6021,7 +6021,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
{
struct rq *rq = rq_of(cfs_rq);
struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
- struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];
+ struct sched_entity *se;
/*
* It's possible we are called with !runtime_remaining due to things
@@ -6036,7 +6036,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
if (cfs_rq->runtime_enabled && cfs_rq->runtime_remaining <= 0)
return;
- se = cfs_rq->tg->se[cpu_of(rq)];
+ se = cfs_rq_se(cfs_rq);
cfs_rq->throttled = 0;
@@ -9788,7 +9788,6 @@ static bool __update_blocked_fair(struct rq *rq, bool *done)
{
struct cfs_rq *cfs_rq, *pos;
bool decayed = false;
- int cpu = cpu_of(rq);
/*
* Iterates the task_group tree in a bottom up fashion, see
@@ -9808,7 +9807,7 @@ static bool __update_blocked_fair(struct rq *rq, bool *done)
}
/* Propagate pending load changes to the parent, if any: */
- se = cfs_rq->tg->se[cpu];
+ se = cfs_rq_se(cfs_rq);
if (se && !skip_blocked_update(se))
update_load_avg(cfs_rq_of(se), se, UPDATE_TG);
@@ -9834,8 +9833,7 @@ static bool __update_blocked_fair(struct rq *rq, bool *done)
*/
static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq)
{
- struct rq *rq = rq_of(cfs_rq);
- struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];
+ struct sched_entity *se = cfs_rq_se(cfs_rq);
unsigned long now = jiffies;
unsigned long load;
@@ -13367,7 +13365,6 @@ void free_fair_sched_group(struct task_group *tg)
}
kfree(tg->cfs_rq);
- kfree(tg->se);
}
int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
@@ -13380,9 +13377,6 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL);
if (!tg->cfs_rq)
goto err;
- tg->se = kcalloc(nr_cpu_ids, sizeof(se), GFP_KERNEL);
- if (!tg->se)
- goto err;
tg->shares = NICE_0_LOAD;
@@ -13397,7 +13391,7 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
cfs_rq = &combined->cfs_rq;
se = &combined->se;
init_cfs_rq(cfs_rq);
- init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);
+ init_tg_cfs_entry(tg, cfs_rq, se, i, tg_se(parent, i));
init_entity_runnable_average(se);
}
@@ -13416,7 +13410,7 @@ void online_fair_sched_group(struct task_group *tg)
for_each_possible_cpu(i) {
rq = cpu_rq(i);
- se = tg->se[i];
+ se = tg_se(tg, i);
rq_lock_irq(rq, &rf);
update_rq_clock(rq);
attach_entity_cfs_rq(se);
@@ -13433,7 +13427,7 @@ void unregister_fair_sched_group(struct task_group *tg)
for_each_possible_cpu(cpu) {
struct cfs_rq *cfs_rq = tg->cfs_rq[cpu];
- struct sched_entity *se = tg->se[cpu];
+ struct sched_entity *se = tg_se(tg, cpu);
struct rq *rq = cpu_rq(cpu);
if (se) {
@@ -13470,7 +13464,6 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
init_cfs_rq_runtime(cfs_rq);
tg->cfs_rq[cpu] = cfs_rq;
- tg->se[cpu] = se;
/* se could be NULL for root_task_group */
if (!se)
@@ -13501,7 +13494,7 @@ static int __sched_group_set_shares(struct task_group *tg, unsigned long shares)
/*
* We can't change the weight of the root cgroup.
*/
- if (!tg->se[0])
+ if (is_root_task_group(tg))
return -EINVAL;
shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES));
@@ -13512,7 +13505,7 @@ static int __sched_group_set_shares(struct task_group *tg, unsigned long shares)
tg->shares = shares;
for_each_possible_cpu(i) {
struct rq *rq = cpu_rq(i);
- struct sched_entity *se = tg->se[i];
+ struct sched_entity *se = tg_se(tg, i);
struct rq_flags rf;
/* Propagate contribution to hierarchy */
@@ -13563,7 +13556,7 @@ int sched_group_set_idle(struct task_group *tg, long idle)
for_each_possible_cpu(i) {
struct rq *rq = cpu_rq(i);
- struct sched_entity *se = tg->se[i];
+ struct sched_entity *se = tg_se(tg, i);
struct cfs_rq *grp_cfs_rq = tg->cfs_rq[i];
bool was_idle = cfs_rq_is_idle(grp_cfs_rq);
long idle_task_delta;
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 8db53f4d4d06..1133910a13c2 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -476,8 +476,6 @@ struct task_group {
#endif
#ifdef CONFIG_FAIR_GROUP_SCHED
- /* schedulable entities of this group on each CPU */
- struct sched_entity **se;
/* runqueue "owned" by this group on each CPU */
struct cfs_rq **cfs_rq;
unsigned long shares;
@@ -923,7 +921,8 @@ struct dl_rq {
};
#ifdef CONFIG_FAIR_GROUP_SCHED
-
+/* Check whether a task group is root tg */
+#define is_root_task_group(tg) ((tg) == &root_task_group)
/* An entity is a task if it doesn't "own" a runqueue */
#define entity_is_task(se) (!se->my_q)
@@ -1609,6 +1608,26 @@ static inline struct task_struct *task_of(struct sched_entity *se)
return container_of(se, struct task_struct, se);
}
+static inline struct sched_entity *tg_se(struct task_group *tg, int cpu)
+{
+ if (is_root_task_group(tg))
+ return NULL;
+
+ struct cfs_rq_with_se *combined =
+ container_of(tg->cfs_rq[cpu], struct cfs_rq_with_se, cfs_rq);
+ return &combined->se;
+}
+
+static inline struct sched_entity *cfs_rq_se(struct cfs_rq *cfs_rq)
+{
+ if (is_root_task_group(cfs_rq->tg))
+ return NULL;
+
+ struct cfs_rq_with_se *combined =
+ container_of(cfs_rq, struct cfs_rq_with_se, cfs_rq);
+ return &combined->se;
+}
+
static inline struct cfs_rq *task_cfs_rq(struct task_struct *p)
{
return p->se.cfs_rq;
@@ -2182,8 +2201,8 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
#ifdef CONFIG_FAIR_GROUP_SCHED
set_task_rq_fair(&p->se, p->se.cfs_rq, tg->cfs_rq[cpu]);
p->se.cfs_rq = tg->cfs_rq[cpu];
- p->se.parent = tg->se[cpu];
- p->se.depth = tg->se[cpu] ? tg->se[cpu]->depth + 1 : 0;
+ p->se.parent = tg_se(tg, cpu);
+ p->se.depth = p->se.parent ? p->se.parent->depth + 1 : 0;
#endif
#ifdef CONFIG_RT_GROUP_SCHED
--
2.51.2.1041.gc1ab5b90ca-goog
^ permalink raw reply [flat|nested] 4+ messages in thread* [PATCH v5 3/3] sched/fair: Allocate both cfs_rq and sched_entity with per-cpu
2025-11-07 4:38 [PATCH v5 0/3] sched/fair: Optimize cfs_rq and sched_entity allocation for better data locality Zecheng Li
2025-11-07 4:38 ` [PATCH v5 1/3] sched/fair: Co-locate cfs_rq and sched_entity Zecheng Li
2025-11-07 4:38 ` [PATCH v5 2/3] sched/fair: Remove task_group->se pointer array Zecheng Li
@ 2025-11-07 4:38 ` Zecheng Li
2 siblings, 0 replies; 4+ messages in thread
From: Zecheng Li @ 2025-11-07 4:38 UTC (permalink / raw)
To: Ingo Molnar, Peter Zijlstra, Juri Lelli, Vincent Guittot
Cc: Dietmar Eggemann, Steven Rostedt, Ben Segall, Mel Gorman,
Valentin Schneider, Rik van Riel, Chris Mason,
Madadi Vineeth Reddy, Xu Liu, Blake Jones, Josh Don,
linux-kernel, Zecheng Li
To remove the cfs_rq pointer array in task_group, allocate the combined
cfs_rq and sched_entity using the per-cpu allocator.
This patch implements the following:
- Changes task_group->cfs_rq from struct cfs_rq ** to struct cfs_rq
__percpu *.
- Updates memory allocation in alloc_fair_sched_group() and
free_fair_sched_group() to use alloc_percpu() and free_percpu()
respectively.
- Uses the inline accessor tg_cfs_rq(tg, cpu) with per_cpu_ptr() to
retrieve the pointer to cfs_rq for the given task group and CPU.
- Replaces direct accesses tg->cfs_rq[cpu] with calls to the new
tg_cfs_rq(tg, cpu) helper.
- Handles the root_task_group: since struct rq is already a per-cpu
variable (runqueues), its embedded cfs_rq (rq->cfs) is also per-cpu.
Therefore, we assign root_task_group.cfs_rq = &runqueues.cfs.
- Cleanup the code in initializing the root task group.
This change places each CPU's cfs_rq and sched_entity in its local
per-cpu memory area to remove the per-task_group pointer arrays.
Signed-off-by: Zecheng Li <zecheng@google.com>
---
kernel/sched/core.c | 35 ++++++++++-----------------
kernel/sched/fair.c | 57 +++++++++++++++++---------------------------
kernel/sched/sched.h | 13 ++++++----
3 files changed, 44 insertions(+), 61 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 12ebe1b4c8ae..376d27f4bbdb 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -8542,7 +8542,7 @@ static struct kmem_cache *task_group_cache __ro_after_init;
void __init sched_init(void)
{
- unsigned long ptr = 0;
+ unsigned long __maybe_unused ptr = 0;
int i;
/* Make sure the linker didn't screw up */
@@ -8558,33 +8558,24 @@ void __init sched_init(void)
wait_bit_init();
#ifdef CONFIG_FAIR_GROUP_SCHED
- ptr += nr_cpu_ids * sizeof(void **);
-#endif
-#ifdef CONFIG_RT_GROUP_SCHED
- ptr += 2 * nr_cpu_ids * sizeof(void **);
-#endif
- if (ptr) {
- ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
+ root_task_group.cfs_rq = &runqueues.cfs;
-#ifdef CONFIG_FAIR_GROUP_SCHED
- root_task_group.cfs_rq = (struct cfs_rq **)ptr;
- ptr += nr_cpu_ids * sizeof(void **);
-
- root_task_group.shares = ROOT_TASK_GROUP_LOAD;
- init_cfs_bandwidth(&root_task_group.cfs_bandwidth, NULL);
+ root_task_group.shares = ROOT_TASK_GROUP_LOAD;
+ init_cfs_bandwidth(&root_task_group.cfs_bandwidth, NULL);
#endif /* CONFIG_FAIR_GROUP_SCHED */
#ifdef CONFIG_EXT_GROUP_SCHED
- scx_tg_init(&root_task_group);
+ scx_tg_init(&root_task_group);
#endif /* CONFIG_EXT_GROUP_SCHED */
#ifdef CONFIG_RT_GROUP_SCHED
- root_task_group.rt_se = (struct sched_rt_entity **)ptr;
- ptr += nr_cpu_ids * sizeof(void **);
+ ptr += 2 * nr_cpu_ids * sizeof(void **);
+ ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
+ root_task_group.rt_se = (struct sched_rt_entity **)ptr;
+ ptr += nr_cpu_ids * sizeof(void **);
- root_task_group.rt_rq = (struct rt_rq **)ptr;
- ptr += nr_cpu_ids * sizeof(void **);
+ root_task_group.rt_rq = (struct rt_rq **)ptr;
+ ptr += nr_cpu_ids * sizeof(void **);
#endif /* CONFIG_RT_GROUP_SCHED */
- }
init_defrootdomain();
@@ -9483,7 +9474,7 @@ static int tg_set_cfs_bandwidth(struct task_group *tg,
}
for_each_online_cpu(i) {
- struct cfs_rq *cfs_rq = tg->cfs_rq[i];
+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, i);
struct rq *rq = cfs_rq->rq;
guard(rq_lock_irq)(rq);
@@ -9651,7 +9642,7 @@ static u64 throttled_time_self(struct task_group *tg)
u64 total = 0;
for_each_possible_cpu(i) {
- total += READ_ONCE(tg->cfs_rq[i]->throttled_clock_self_time);
+ total += READ_ONCE(tg_cfs_rq(tg, i)->throttled_clock_self_time);
}
return total;
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index f9fb07d73a03..a5403f5900d9 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -327,7 +327,7 @@ static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
* to a tree or when we reach the top of the tree
*/
if (cfs_rq->tg->parent &&
- cfs_rq->tg->parent->cfs_rq[cpu]->on_list) {
+ tg_cfs_rq(cfs_rq->tg->parent, cpu)->on_list) {
/*
* If parent is already on the list, we add the child
* just before. Thanks to circular linked property of
@@ -335,7 +335,7 @@ static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
* of the list that starts by parent.
*/
list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list,
- &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list));
+ &(tg_cfs_rq(cfs_rq->tg->parent, cpu)->leaf_cfs_rq_list));
/*
* The branch is now connected to its tree so we can
* reset tmp_alone_branch to the beginning of the
@@ -4141,7 +4141,7 @@ static void __maybe_unused clear_tg_offline_cfs_rqs(struct rq *rq)
rcu_read_lock();
list_for_each_entry_rcu(tg, &task_groups, list) {
- struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
clear_tg_load_avg(cfs_rq);
}
@@ -5739,7 +5739,7 @@ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
static inline int lb_throttled_hierarchy(struct task_struct *p, int dst_cpu)
{
- return throttled_hierarchy(task_group(p)->cfs_rq[dst_cpu]);
+ return throttled_hierarchy(tg_cfs_rq(task_group(p), dst_cpu));
}
static inline bool task_is_throttled(struct task_struct *p)
@@ -5885,7 +5885,7 @@ static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags);
static int tg_unthrottle_up(struct task_group *tg, void *data)
{
struct rq *rq = data;
- struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
struct task_struct *p, *tmp;
if (--cfs_rq->throttle_count)
@@ -5956,7 +5956,7 @@ static void record_throttle_clock(struct cfs_rq *cfs_rq)
static int tg_throttle_down(struct task_group *tg, void *data)
{
struct rq *rq = data;
- struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
if (cfs_rq->throttle_count++)
return 0;
@@ -6432,8 +6432,8 @@ static void sync_throttle(struct task_group *tg, int cpu)
if (!tg->parent)
return;
- cfs_rq = tg->cfs_rq[cpu];
- pcfs_rq = tg->parent->cfs_rq[cpu];
+ cfs_rq = tg_cfs_rq(tg, cpu);
+ pcfs_rq = tg_cfs_rq(tg->parent, cpu);
cfs_rq->throttle_count = pcfs_rq->throttle_count;
cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu));
@@ -6625,7 +6625,7 @@ static void __maybe_unused update_runtime_enabled(struct rq *rq)
rcu_read_lock();
list_for_each_entry_rcu(tg, &task_groups, list) {
struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
- struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
raw_spin_lock(&cfs_b->lock);
cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;
@@ -6654,7 +6654,7 @@ static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
rcu_read_lock();
list_for_each_entry_rcu(tg, &task_groups, list) {
- struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
if (!cfs_rq->runtime_enabled)
continue;
@@ -9357,7 +9357,7 @@ static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_
struct cfs_rq *dst_cfs_rq;
#ifdef CONFIG_FAIR_GROUP_SCHED
- dst_cfs_rq = task_group(p)->cfs_rq[dest_cpu];
+ dst_cfs_rq = tg_cfs_rq(task_group(p), dest_cpu);
#else
dst_cfs_rq = &cpu_rq(dest_cpu)->cfs;
#endif
@@ -13094,7 +13094,7 @@ static int task_is_throttled_fair(struct task_struct *p, int cpu)
struct cfs_rq *cfs_rq;
#ifdef CONFIG_FAIR_GROUP_SCHED
- cfs_rq = task_group(p)->cfs_rq[cpu];
+ cfs_rq = tg_cfs_rq(task_group(p), cpu);
#else
cfs_rq = &cpu_rq(cpu)->cfs;
#endif
@@ -13354,42 +13354,31 @@ static void task_change_group_fair(struct task_struct *p)
void free_fair_sched_group(struct task_group *tg)
{
- int i;
-
- for_each_possible_cpu(i) {
- if (tg->cfs_rq && tg->cfs_rq[i]) {
- struct cfs_rq_with_se *combined =
- container_of(tg->cfs_rq[i], struct cfs_rq_with_se, cfs_rq);
- kfree(combined);
- }
- }
-
- kfree(tg->cfs_rq);
+ free_percpu(tg->cfs_rq);
}
int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
{
- struct cfs_rq_with_se *combined;
+ struct cfs_rq_with_se __percpu *combined;
struct sched_entity *se;
struct cfs_rq *cfs_rq;
int i;
- tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL);
- if (!tg->cfs_rq)
+ combined = alloc_percpu_gfp(struct cfs_rq_with_se, GFP_KERNEL);
+ if (!combined)
goto err;
+ tg->cfs_rq = &combined->cfs_rq;
tg->shares = NICE_0_LOAD;
init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent));
for_each_possible_cpu(i) {
- combined = kzalloc_node(sizeof(struct cfs_rq_with_se),
- GFP_KERNEL, cpu_to_node(i));
- if (!combined)
+ cfs_rq = tg_cfs_rq(tg, i);
+ if (!cfs_rq)
goto err;
- cfs_rq = &combined->cfs_rq;
- se = &combined->se;
+ se = tg_se(tg, i);
init_cfs_rq(cfs_rq);
init_tg_cfs_entry(tg, cfs_rq, se, i, tg_se(parent, i));
init_entity_runnable_average(se);
@@ -13426,7 +13415,7 @@ void unregister_fair_sched_group(struct task_group *tg)
destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
for_each_possible_cpu(cpu) {
- struct cfs_rq *cfs_rq = tg->cfs_rq[cpu];
+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu);
struct sched_entity *se = tg_se(tg, cpu);
struct rq *rq = cpu_rq(cpu);
@@ -13463,8 +13452,6 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
cfs_rq->rq = rq;
init_cfs_rq_runtime(cfs_rq);
- tg->cfs_rq[cpu] = cfs_rq;
-
/* se could be NULL for root_task_group */
if (!se)
return;
@@ -13557,7 +13544,7 @@ int sched_group_set_idle(struct task_group *tg, long idle)
for_each_possible_cpu(i) {
struct rq *rq = cpu_rq(i);
struct sched_entity *se = tg_se(tg, i);
- struct cfs_rq *grp_cfs_rq = tg->cfs_rq[i];
+ struct cfs_rq *grp_cfs_rq = tg_cfs_rq(tg, i);
bool was_idle = cfs_rq_is_idle(grp_cfs_rq);
long idle_task_delta;
struct rq_flags rf;
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 1133910a13c2..132e6098c058 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -477,7 +477,7 @@ struct task_group {
#ifdef CONFIG_FAIR_GROUP_SCHED
/* runqueue "owned" by this group on each CPU */
- struct cfs_rq **cfs_rq;
+ struct cfs_rq __percpu *cfs_rq;
unsigned long shares;
/*
* load_avg can be heavily contended at clock tick time, so put
@@ -1607,6 +1607,11 @@ static inline struct task_struct *task_of(struct sched_entity *se)
WARN_ON_ONCE(!entity_is_task(se));
return container_of(se, struct task_struct, se);
}
+/* Access a specific CPU's cfs_rq from a task group */
+static inline struct cfs_rq *tg_cfs_rq(struct task_group *tg, int cpu)
+{
+ return per_cpu_ptr(tg->cfs_rq, cpu);
+}
static inline struct sched_entity *tg_se(struct task_group *tg, int cpu)
{
@@ -1614,7 +1619,7 @@ static inline struct sched_entity *tg_se(struct task_group *tg, int cpu)
return NULL;
struct cfs_rq_with_se *combined =
- container_of(tg->cfs_rq[cpu], struct cfs_rq_with_se, cfs_rq);
+ container_of(tg_cfs_rq(tg, cpu), struct cfs_rq_with_se, cfs_rq);
return &combined->se;
}
@@ -2199,8 +2204,8 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
#endif
#ifdef CONFIG_FAIR_GROUP_SCHED
- set_task_rq_fair(&p->se, p->se.cfs_rq, tg->cfs_rq[cpu]);
- p->se.cfs_rq = tg->cfs_rq[cpu];
+ set_task_rq_fair(&p->se, p->se.cfs_rq, tg_cfs_rq(tg, cpu));
+ p->se.cfs_rq = tg_cfs_rq(tg, cpu);
p->se.parent = tg_se(tg, cpu);
p->se.depth = p->se.parent ? p->se.parent->depth + 1 : 0;
#endif
--
2.51.2.1041.gc1ab5b90ca-goog
^ permalink raw reply [flat|nested] 4+ messages in thread