Re: [Patch v4 02/22] sched/cache: Limit the scan number of CPUs when calculating task occupancy

[Luo Gengkun <luogengkun2@huawei.com>]

On 2026/4/9 21:41, Peter Zijlstra wrote:
> On Thu, Apr 09, 2026 at 09:17:10PM +0800, Luo Gengkun wrote:
>>
>>
>> On 2026/4/2 5:52, Tim Chen wrote:
>>> From: Chen Yu <yu.c.chen@intel.com>
>>>
>>> When NUMA balancing is enabled, the kernel currently iterates over all
>>> online CPUs to aggregate process-wide occupancy data. On large systems,
>>> this global scan introduces significant overhead.
>>>
>>> To reduce scan latency, limit the search to a subset of relevant CPUs:
>>> 1. The task's preferred NUMA node.
>>> 2. The node where the task is currently running.
>>> 3. The node that contains the task's current preferred LLC..
>>>
>>> While focusing solely on the preferred NUMA node is ideal, a
>>> process-wide scan must remain flexible because the "preferred node"
>>> is a per-task attribute. Different threads within the same process may
>>> have different preferred nodes, causing the process-wide preference to
>>> migrate. Maintaining a mask that covers both the preferred and active
>>> running nodes ensures accuracy while significantly reducing the number of
>>> CPUs inspected.
>>
>> To address the issue of scanning overhead, there is a more targeted
>> approach: only scanning the CPUs actually accessed by the process, and
>> evicting these CPUs when they remain unaccessed for a specific period of
>> time.
>>
>> This significantly reduces unnecessary scanning in most scenario. I have
>> attached the patch below for review. Please feel free to integrate or modify
>> these changes.
>>
>> Thansk!
>> Luo Gengkun
> 
> Please fix your MUA, whatever you tried to send is horribly white space
> mangled.
> 
Apologies for the formatting mess. The following should display properly.

 From de33b29f31a882ac8c0038cac4ec659c7fdfe60d Mon Sep 17 00:00:00 2001
From: Luo Gengkun <luogengkun2@huawei.com>
Date: Thu, 9 Apr 2026 07:22:16 +0000
Subject: [PATCH] sched/cache: Reduce the overhead of task_cache_work by only
  scan the visisted cpus.

The overhead of task_cache_work is high, espeically in multi-NUMA system.
Currently, task_cache_work try to find the pref_llc by scan all cpus in the
system. However, most of these scans are meaningless, such as those for
cpus that have never been visited or were accessed a long time ago.

To address this problem, this patch introduces visited_cpus to track the
visited cpus and uses llc_epoch_visited_timeout to evict cpus that have
timed out.

Signed-off-by: Luo Gengkun <luogengkun2@huawei.com>
---
  include/linux/sched.h |  1 +
  kernel/sched/debug.c  |  2 ++
  kernel/sched/fair.c   | 19 ++++++++++++++++---
  kernel/sched/sched.h  |  1 +
  4 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/include/linux/sched.h b/include/linux/sched.h
index dfa4bfd099c6..f2327a13fda8 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2390,6 +2390,7 @@ struct sched_cache_time {
  
  struct sched_cache_stat {
  	struct sched_cache_time __percpu *pcpu_sched;
+	struct cpumask visited_cpus;
  	raw_spinlock_t lock;
  	unsigned long epoch;
  	u64 nr_running_avg;
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 4469e1c152c8..da66acabff4f 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -675,6 +675,8 @@ static __init int sched_init_debug(void)
  			   &llc_epoch_period);
  	debugfs_create_u32("epoch_affinity_timeout", 0644, llc,
  			   &llc_epoch_affinity_timeout);
+	debugfs_create_u32("epoch_visited_timeout", 0644, llc,
+			   &llc_epoch_visited_timeout);
  	debugfs_create_u32("overaggr_pct", 0644, llc,
  			   &llc_overaggr_pct);
  	debugfs_create_u32("imb_pct", 0644, llc,
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index e4e22696a0b1..4ccabc412aa1 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1285,6 +1285,7 @@ static void set_next_buddy(struct sched_entity *se);
  __read_mostly unsigned int llc_aggr_tolerance	= 1;
  __read_mostly unsigned int llc_epoch_period	= EPOCH_PERIOD;
  __read_mostly unsigned int llc_epoch_affinity_timeout = EPOCH_LLC_AFFINITY_TIMEOUT;
+__read_mostly unsigned int llc_epoch_visited_timeout = EPOCH_LLC_AFFINITY_TIMEOUT;
  __read_mostly unsigned int llc_imb_pct		= 20;
  __read_mostly unsigned int llc_overaggr_pct	= 50;
  
@@ -1466,6 +1467,7 @@ void mm_init_sched(struct mm_struct *mm,
  	raw_spin_lock_init(&mm->sc_stat.lock);
  	mm->sc_stat.epoch = epoch;
  	mm->sc_stat.cpu = -1;
+	cpumask_clear(&mm->sc_stat.visited_cpus);
  
  	/*
  	 * The update to mm->sc_stat should not be reordered
@@ -1582,6 +1584,7 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
  		pcpu_sched->runtime += delta_exec;
  		rq->cpu_runtime += delta_exec;
  		epoch = rq->cpu_epoch;
+		cpumask_set_cpu(cpu_of(rq), &mm->sc_stat.visited_cpus);
  	}
  
  	/*
@@ -1724,7 +1727,10 @@ static void task_cache_work(struct callback_head *work)
  		return;
  
  	scoped_guard (cpus_read_lock) {
-		get_scan_cpumasks(cpus, p);
+		if (unlikely(llc_epoch_visited_timeout == 0))
+			get_scan_cpumasks(cpus, p);
+		else
+			cpumask_and(cpus, cpu_online_mask, &mm->sc_stat.visited_cpus);
  
  		for_each_cpu(cpu, cpus) {
  			/* XXX sched_cluster_active */
@@ -1736,8 +1742,15 @@ static void task_cache_work(struct callback_head *work)
  				continue;
  
  			for_each_cpu(i, sched_domain_span(sd)) {
-				occ = fraction_mm_sched(cpu_rq(i),
-							per_cpu_ptr(mm->sc_stat.pcpu_sched, i));
+				struct rq *rq = cpu_rq(i);
+				struct sched_cache_time *pcpu_sched = per_cpu_ptr(mm->sc_stat.pcpu_sched, i);
+				/* Skip the rq that has not been hit for a long time */
+				if (llc_epoch_visited_timeout && (rq->cpu_epoch - pcpu_sched->epoch) >
+				    llc_epoch_visited_timeout) {
+					cpumask_clear_cpu(cpu_of(rq), &mm->sc_stat.visited_cpus);
+					continue;
+				}
+				occ = fraction_mm_sched(rq, pcpu_sched);
  				a_occ += occ;
  				if (occ > m_occ) {
  					m_occ = occ;
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index b757812725f7..93908bb17cc5 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -4041,6 +4041,7 @@ extern int sysctl_sched_cache_user;
  extern unsigned int llc_aggr_tolerance;
  extern unsigned int llc_epoch_period;
  extern unsigned int llc_epoch_affinity_timeout;
+extern unsigned int llc_epoch_visited_timeout;
  extern unsigned int llc_imb_pct;
  extern unsigned int llc_overaggr_pct;
  
-- 
2.34.1