Re: [PATCH v2 23/23] -- DO NOT APPLY!!! -- sched/cache/debug: Display the per LLC occupancy for each process via proc fs

["Chen, Yu C" <yu.c.chen@intel.com>]

On 12/17/2025 5:59 PM, Aaron Lu wrote:
> On Wed, Dec 03, 2025 at 03:07:42PM -0800, Tim Chen wrote:
>> From: Chen Yu <yu.c.chen@intel.com>
>>
>> Debug patch only.
>>
>> Show the per-LLC occupancy in /proc/{PID}/schedstat, with each column
>> corresponding to one LLC. This can be used to verify if the cache-aware
>> load balancer works as expected by aggregating threads onto dedicated LLCs.
>>
>> Suppose there are 2 LLCs and the sampling duration is 10 seconds:
>>
>> Enable the cache aware load balance:
>> 0 12281  <--- LLC0 residency delta is 0, LLC1 is 12 seconds
>> 0 18881
>> 0 16217
>>
>> disable the cache aware load balance:
>> 6497 15802
>> 9299 5435
>> 17811 8278
>>
>> Signed-off-by: Chen Yu <yu.c.chen@intel.com>
>> Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
>> ---
>>   fs/proc/base.c           | 22 ++++++++++++++++++++++
>>   include/linux/mm_types.h | 19 +++++++++++++++++--
>>   include/linux/sched.h    |  3 +++
>>   kernel/sched/fair.c      | 40 ++++++++++++++++++++++++++++++++++++++--
>>   4 files changed, 80 insertions(+), 4 deletions(-)
>>
>> diff --git a/fs/proc/base.c b/fs/proc/base.c
>> index 6299878e3d97..f4be96f4bd01 100644
>> --- a/fs/proc/base.c
>> +++ b/fs/proc/base.c
>> @@ -518,6 +518,28 @@ static int proc_pid_schedstat(struct seq_file *m, struct pid_namespace *ns,
>>   		   (unsigned long long)task->se.sum_exec_runtime,
>>   		   (unsigned long long)task->sched_info.run_delay,
>>   		   task->sched_info.pcount);
>> +#ifdef CONFIG_SCHED_CACHE
>> +	if (sched_cache_enabled()) {
>> +		struct mm_struct *mm = task->mm;
>> +		u64 *llc_runtime;
>> +
>> +		if (!mm)
>> +			return 0;
>> +
>> +		llc_runtime = kcalloc(max_llcs, sizeof(u64), GFP_KERNEL);
>> +		if (!llc_runtime)
>> +			return 0;
>> +
>> +		if (get_mm_per_llc_runtime(task, llc_runtime))
>> +			goto out;
>> +
>> +		for (int i = 0; i < max_llcs; i++)
>> +			seq_printf(m, "%llu ", llc_runtime[i]);
> 
> I feel it is better to also mark the current preferred LLC of this
> process so that I can know how well it works.
> 

Sure.

>> +		seq_puts(m, "\n");
>> +out:
>> +		kfree(llc_runtime);
>> +	}
>> +#endif
>>   
>>   	return 0;
>>   }
> 
> BTW, is there a way to tell if a process is being taken care of by
> 'cache aware scheduling' or it's blocked due to its huge rss or having
> too many threads?
> 
> I used below debug code to get these info through schedstat, but maybe I
> missed something and there is a simpler method?
> 
> diff --git a/fs/proc/base.c b/fs/proc/base.c
> index f4be96f4bd015..c709a1a1bd867 100644
> --- a/fs/proc/base.c
> +++ b/fs/proc/base.c
> @@ -505,6 +505,7 @@ static int proc_pid_stack(struct seq_file *m, struct pid_namespace *ns,
>   #endif
>   
>   #ifdef CONFIG_SCHED_INFO
> +DECLARE_PER_CPU(int, sd_llc_id);
>   /*
>    * Provides /proc/PID/schedstat
>    */
> @@ -522,6 +523,7 @@ static int proc_pid_schedstat(struct seq_file *m, struct pid_namespace *ns,
>   	if (sched_cache_enabled()) {
>   		struct mm_struct *mm = task->mm;
>   		u64 *llc_runtime;
> +		int mm_sched_llc;
>   
>   		if (!mm)
>   			return 0;
> @@ -533,8 +535,17 @@ static int proc_pid_schedstat(struct seq_file *m, struct pid_namespace *ns,
>   		if (get_mm_per_llc_runtime(task, llc_runtime))
>   			goto out;
>   
> +		if (mm->mm_sched_cpu == -1)
> +			mm_sched_llc = -1;
> +		else
> +			mm_sched_llc = per_cpu(sd_llc_id, mm->mm_sched_cpu);

We can use llc_id(mm->mm_sched_cpu).

> +
> +		seq_printf(m, "%llu 0x%x\n", mm->nr_running_avg, mm->mm_sched_flags);
>   		for (int i = 0; i < max_llcs; i++)
> -			seq_printf(m, "%llu ", llc_runtime[i]);
> +			seq_printf(m, "%s%s%llu ",
> +				   i == task->preferred_llc ? "*" : "",
> +				   i == mm_sched_llc ? "?" : "",
> +				   llc_runtime[i]);
>   		seq_puts(m, "\n");
>   out:
>   		kfree(llc_runtime);
> diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
> index 255c22be7312f..06bb106d1b724 100644
> --- a/include/linux/mm_types.h
> +++ b/include/linux/mm_types.h
> @@ -1048,6 +1048,7 @@ struct mm_struct {
>   		raw_spinlock_t mm_sched_lock;
>   		unsigned long mm_sched_epoch;
>   		int mm_sched_cpu;
> +		int mm_sched_flags;
>   		u64 nr_running_avg ____cacheline_aligned_in_smp;
>   #endif
>   
> diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
> index 205208f061bb3..ab1cdba65d389 100644
> --- a/kernel/sched/fair.c
> +++ b/kernel/sched/fair.c
> @@ -1237,12 +1237,20 @@ static inline int get_sched_cache_scale(int mul)
>   	return (1 + (llc_aggr_tolerance - 1) * mul);
>   }
>   
> +#define MM_SCHED_EXCEED_LLC_CAPACITY	1
> +#define MM_SCHED_NO_CACHE_INFO		2
> +#define MM_SCHED_EXCEED_LLC_NR		4
> +#define MM_SCHED_NR_THREADS		8
> +
>   static bool exceed_llc_capacity(struct mm_struct *mm, int cpu)
>   {
>   	unsigned int llc, scale;
>   	struct cacheinfo *ci;
>   	unsigned long rss;
>   
> +	mm->mm_sched_flags &= ~MM_SCHED_NO_CACHE_INFO;
> +	mm->mm_sched_flags &= ~MM_SCHED_EXCEED_LLC_CAPACITY;
> +

Maybe we can do some read-comparison before writing the flags, previously
we found that writing the per-process mm struct is very expensive, so
maybe avoid writing to it as much as possible.

I'll fold your changes and do the test. Thanks!

Thanks,
Chenyu