From: Luo Gengkun <luogengkun@huaweicloud.com>
To: peterz@infradead.org, mingo@redhat.com, juri.lelli@redhat.com,
vincent.guittot@linaro.org, tim.c.chen@linux.intel.com,
yu.c.chen@intel.com, kprateek.nayak@amd.com
Cc: dietmar.eggemann@arm.com, rostedt@goodmis.org,
bsegall@google.com, mgorman@suse.de, vschneid@redhat.com,
linux-kernel@vger.kernel.org
Subject: [PATCH v11 2/2] -- DO NOT APPLY!!! -- sched/cache/debug: Add trace event and sched feature to track scan cost
Date: Wed, 11 Nov 2026 11:04:09 +0000 [thread overview]
Message-ID: <20261111110409.1498035-3-luogengkun@huaweicloud.com> (raw)
In-Reply-To: <20261111110409.1498035-1-luogengkun@huaweicloud.com>
This commit re-adds get_scan_cpumasks() and introduces two sched features
as well as trace events to track the scanning cost for testing purposes.
Signed-off-by: Luo Gengkun <luogengkun@huaweicloud.com>
---
include/trace/events/sched.h | 21 +++++++++++
kernel/sched/fair.c | 69 +++++++++++++++++++++++++++++++++---
kernel/sched/features.h | 1 +
3 files changed, 87 insertions(+), 4 deletions(-)
diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h
index 535860581f15..aced624f198d 100644
--- a/include/trace/events/sched.h
+++ b/include/trace/events/sched.h
@@ -10,6 +10,27 @@
#include <linux/tracepoint.h>
#include <linux/binfmts.h>
+TRACE_EVENT(sched_cache_scan,
+
+ TP_PROTO(struct task_struct *t, int scan),
+
+ TP_ARGS(t, scan),
+
+ TP_STRUCT__entry(
+ __string( comm, t->comm )
+ __field( pid_t, pid )
+ __field( int, scan )
+ ),
+
+ TP_fast_assign(
+ __assign_str(comm);
+ __entry->pid = t->pid;
+ __entry->scan = scan;
+ ),
+
+ TP_printk("comm=%s pid=%d scan=%d", __get_str(comm), __entry->pid,
+ __entry->scan)
+);
/*
* Tracepoint for calling kthread_stop, performed to end a kthread:
*/
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index c97f08eaa9e5..7a363da2cf63 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1807,7 +1807,7 @@ static unsigned long fraction_mm_sched(int cpu,
__update_mm_sched(rq, pcpu_sched);
/* Evict the CPU if this sched group has not run on it for a while */
- if ((rq->cpu_epoch - pcpu_sched->epoch_last_visit) >
+ if (sched_feat(SC_VISIT) && (rq->cpu_epoch - pcpu_sched->epoch_last_visit) >
llc_epoch_affinity_timeout) {
cpumask_clear_cpu(cpu, grp->visited_cpus);
return 0;
@@ -1889,7 +1889,8 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
rq->cpu_runtime += delta_exec;
epoch = rq->cpu_epoch;
pcpu_sched->epoch_last_visit = epoch;
- if (!cpumask_test_cpu(cpu_of(rq), grp->visited_cpus))
+ if (sched_feat(SC_VISIT) &&
+ !cpumask_test_cpu(cpu_of(rq), grp->visited_cpus))
cpumask_set_cpu(cpu_of(rq), grp->visited_cpus);
}
@@ -1941,6 +1942,52 @@ static void task_tick_cache(struct rq *rq, struct task_struct *p)
}
}
+static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p,
+ struct sched_cache_group *grp)
+{
+#ifdef CONFIG_NUMA_BALANCING
+ int cpu, curr_cpu, nid, pref_nid;
+
+ if (!static_branch_likely(&sched_numa_balancing))
+ goto out;
+
+ cpu = READ_ONCE(grp->cpu);
+ if (cpu != -1)
+ nid = cpu_to_node(cpu);
+ curr_cpu = task_cpu(p);
+
+ /*
+ * Scanning in the preferred NUMA node is ideal. However, the NUMA
+ * preferred node is per-task rather than per-process. It is possible
+ * for different threads of the process to have distinct preferred
+ * nodes; consequently, the process-wide preferred LLC may bounce
+ * between different nodes. As a workaround, maintain the scan
+ * CPU mask to also cover the process's current preferred LLC and the
+ * current running node to mitigate the bouncing risk.
+ * TBD: numa_group should be considered during task aggregation.
+ */
+ pref_nid = p->numa_preferred_nid;
+ /* honor the task's preferred node */
+ if (pref_nid == NUMA_NO_NODE)
+ goto out;
+
+ cpumask_or(cpus, cpus, cpumask_of_node(pref_nid));
+
+ /* honor the task's preferred LLC CPU */
+ if (cpu != -1 && !cpumask_test_cpu(cpu, cpus) && nid != NUMA_NO_NODE)
+ cpumask_or(cpus, cpus, cpumask_of_node(nid));
+
+ /* make sure the task's current running node is included */
+ if (!cpumask_test_cpu(curr_cpu, cpus))
+ cpumask_or(cpus, cpus, cpumask_of_node(cpu_to_node(curr_cpu)));
+
+ return;
+
+out:
+#endif
+ cpumask_copy(cpus, cpu_online_mask);
+}
+
static inline void update_avg_scale(u64 *avg, u64 sample)
{
int factor = per_cpu(sd_llc_size, raw_smp_processor_id());
@@ -1963,11 +2010,13 @@ static void task_cache_work(struct callback_head *work)
{
struct sched_cache_group *grp __free(sched_cache_group_put) = NULL;
cpumask_var_t cpus __free(free_cpumask_var) = CPUMASK_VAR_NULL;
+ cpumask_var_t scan_cpus __free(free_cpumask_var) = CPUMASK_VAR_NULL;
int cpu, m_a_cpu = -1, nr_running = 0, curr_cpu;
unsigned long next_scan, now = jiffies;
struct task_struct *p = current, *cur;
unsigned long curr_m_a_occ = 0;
unsigned long m_a_occ = 0;
+ int scanned = 0;
WARN_ON_ONCE(work != &p->cache_work);
@@ -2007,6 +2056,9 @@ static void task_cache_work(struct callback_head *work)
if (!zalloc_cpumask_var(&cpus, GFP_KERNEL))
return;
+ if (!zalloc_cpumask_var(&scan_cpus, GFP_KERNEL))
+ return;
+
scoped_guard (cpus_read_lock) {
guard(rcu)();
@@ -2016,7 +2068,10 @@ static void task_cache_work(struct callback_head *work)
* this way has only just been used by this sched group,
* so its share small, and the next scan will include it.
*/
- cpumask_and(cpus, cpu_online_mask, grp->visited_cpus);
+ if (sched_feat(SC_VISIT))
+ cpumask_and(cpus, cpu_online_mask, grp->visited_cpus);
+ else
+ get_scan_cpumasks(cpus, p, grp);
for_each_cpu(cpu, cpus) {
/* XXX sched_cluster_active */
@@ -2027,7 +2082,11 @@ static void task_cache_work(struct callback_head *work)
if (!sd)
continue;
- for_each_cpu_and(i, sched_domain_span(sd), cpus) {
+ cpumask_copy(scan_cpus, sched_domain_span(sd));
+ if (sched_feat(SC_VISIT))
+ cpumask_and(scan_cpus, scan_cpus, cpus);
+
+ for_each_cpu(i, scan_cpus) {
occ = fraction_mm_sched(i, grp);
a_occ += occ;
if (occ > m_occ) {
@@ -2043,6 +2102,7 @@ static void task_cache_work(struct callback_head *work)
if (cur && !(cur->flags & (PF_EXITING | PF_KTHREAD)) &&
rcu_access_pointer(cur->sched_cache_grp) == grp)
nr_running++;
+ scanned++;
}
/*
@@ -2087,6 +2147,7 @@ static void task_cache_work(struct callback_head *work)
}
update_avg_scale(&grp->nr_running_avg, nr_running);
+ trace_sched_cache_scan(p, scanned);
}
void init_sched_mm(struct task_struct *p)
diff --git a/kernel/sched/features.h b/kernel/sched/features.h
index 8f0dee8fc475..990714ebabfd 100644
--- a/kernel/sched/features.h
+++ b/kernel/sched/features.h
@@ -142,3 +142,4 @@ SCHED_FEAT(LATENCY_WARN, false)
*/
SCHED_FEAT(NI_RANDOM, true)
SCHED_FEAT(NI_RATE, true)
+SCHED_FEAT(SC_VISIT, false)
--
2.34.1
prev parent reply other threads:[~2026-11-11 10:31 UTC|newest]
Thread overview: 4+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-11 11:04 [PATCH v11 0/2] Cache aware scheduling: Reduce the overhead of task_cache_work Luo Gengkun
2026-11-11 11:04 ` [PATCH v11 1/2] sched/cache: Reduce the overhead of task_cache_work by only scan the visisted cpus Luo Gengkun
2026-11-16 7:24 ` Chen Yu
2026-11-11 11:04 ` Luo Gengkun [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261111110409.1498035-3-luogengkun@huaweicloud.com \
--to=luogengkun@huaweicloud.com \
--cc=bsegall@google.com \
--cc=dietmar.eggemann@arm.com \
--cc=juri.lelli@redhat.com \
--cc=kprateek.nayak@amd.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mgorman@suse.de \
--cc=mingo@redhat.com \
--cc=peterz@infradead.org \
--cc=rostedt@goodmis.org \
--cc=tim.c.chen@linux.intel.com \
--cc=vincent.guittot@linaro.org \
--cc=vschneid@redhat.com \
--cc=yu.c.chen@intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®