[PATCH v2] sched/debug, sys_info: Introduce SYS_INFO_CPU_RUNQUEUES

[PATCH v2] sched/debug, sys_info: Introduce SYS_INFO_CPU_RUNQUEUES

[Aaron Tomlin]

When investigating kernel panics, inspectability of per-CPU runqueues
and runnable task states is valuable for diagnosing CPU starvation
priority inversion, etc.

While debugfs (/sys/kernel/debug/sched/debug) exposes runqueue metrics
to userspace, these details are not captured during an automated kernel
panic or crash dump. Capturing per-CPU runqueue state directly into
log_buf fills this diagnostic gap for post-mortem crash analysis.

Introduce SYS_INFO_CPU_RUNQUEUES and its corresponding string token
"cpu_runqueues" to panic_sys_info. Add sched_show_runqueues(), modelled
on print_rq(), to emit per-CPU scheduler diagnostics to the kernel log.

Unlike /sys/kernel/debug/sched/debug which dumps all threads assigned to
a CPU, sched_show_runqueues() only emits threads that are actively
running or queued on the runqueue (via task_on_rq_queued() and
task_current()). This keeps the panic log concise, reflects the true
runqueue depth, and prevents overflowing the printk ring buffer on
systems with high thread counts.

Additionally, to guarantee deadlock and memory safety in panic context:
    - Acquire the runqueue lock using raw_spin_rq_trylock() with
      READ_ONCE() and rcu_dereference() fallback, marking contended
      queues with " (contended)"

    - Wrap the per-CPU inspection in rcu_read_lock() to protect the
      sampled current task (comm and PID) against premature release
      during pr_info() across other callers

    - Omit cgroup group-path printing in print_rq() to avoid acquiring
      cgroup_mutex and traversing kernfs dentries

Suggested-by: Rishil Sandip Shah <rishil1999@outlook.com>
Signed-off-by: Aaron Tomlin <atomlin@atomlin.com>
---
Changes since v1:

 - Resolved Sparse __rcu address space warnings by accessing rq->curr
   via rcu_dereference() in sched_show_runqueues()

 - Link to v1: https://lore.kernel.org/lkml/20260911022844.521413-1-atomlin@atomlin.com/
---
 Documentation/admin-guide/sysctl/kernel.rst |  1 +
 include/linux/sched/debug.h                 |  1 +
 include/linux/sys_info.h                    |  1 +
 kernel/sched/debug.c                        | 76 +++++++++++++++++----
 lib/sys_info.c                              |  4 ++
 5 files changed, 70 insertions(+), 13 deletions(-)

diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst
index b6328cd0f43e..0962b031e035 100644
--- a/Documentation/admin-guide/sysctl/kernel.rst
+++ b/Documentation/admin-guide/sysctl/kernel.rst
@@ -939,6 +939,7 @@ locks           print locks info if CONFIG_LOCKDEP is on
 ftrace          print ftrace buffer
 all_bt          print all CPUs backtrace (if available in the arch)
 blocked_tasks   print only tasks in uninterruptible (blocked) state
+cpu_runqueues   print per-CPU runqueue depth and runnable tasks
 =============   ===================================================
 
 
diff --git a/include/linux/sched/debug.h b/include/linux/sched/debug.h
index 35ed4577a6cc..d4276d8ead29 100644
--- a/include/linux/sched/debug.h
+++ b/include/linux/sched/debug.h
@@ -39,6 +39,7 @@ struct seq_file;
 extern void proc_sched_show_task(struct task_struct *p,
 				 struct pid_namespace *ns, struct seq_file *m);
 extern void proc_sched_set_task(struct task_struct *p);
+extern void sched_show_runqueues(void);
 
 /* Attach to any functions which should be ignored in wchan output. */
 #define __sched		__section(".sched.text")
diff --git a/include/linux/sys_info.h b/include/linux/sys_info.h
index a5bc3ea3d44b..c571aad1e178 100644
--- a/include/linux/sys_info.h
+++ b/include/linux/sys_info.h
@@ -16,6 +16,7 @@
 #define SYS_INFO_PANIC_CONSOLE_REPLAY	0x00000020
 #define SYS_INFO_ALL_BT			0x00000040
 #define SYS_INFO_BLOCKED_TASKS		0x00000080
+#define SYS_INFO_CPU_RUNQUEUES		0x00000100
 
 void sys_info(unsigned long si_mask);
 unsigned long sys_info_parse_param(char *str);
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 72236db67983..79f6b00974bb 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -968,7 +968,8 @@ static void task_group_path(struct task_group *tg, char *path, int plen)
 #endif
 
 static void
-print_task(struct seq_file *m, struct rq *rq, struct task_struct *p)
+print_task(struct seq_file *m, struct rq *rq, struct task_struct *p,
+	   bool show_cgroup_path)
 {
 	if (task_current(rq, p))
 		SEQ_printf(m, ">R");
@@ -996,13 +997,15 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p)
 	SEQ_printf(m, "   %d      %d", task_node(p), task_numa_group_id(p));
 #endif
 #ifdef CONFIG_CGROUP_SCHED
-	SEQ_printf_task_group_path(m, task_group(p), "        %s")
+	if (show_cgroup_path)
+		SEQ_printf_task_group_path(m, task_group(p), "        %s")
 #endif
 
 	SEQ_printf(m, "\n");
 }
 
-static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu)
+static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu,
+		     bool show_cgroup_path, bool queued_only)
 {
 	struct task_struct *g, *p;
 
@@ -1010,31 +1013,36 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu)
 	SEQ_printf(m, "runnable tasks:\n");
 	SEQ_printf(m, " S            task   PID     weight       vruntime   eligible    "
 		   "deadline             slice          sum-exec      switches  "
-		   "prio         wait-time        sum-sleep       sum-block"
+		   "prio         wait-time        sum-sleep       sum-block");
 #ifdef CONFIG_NUMA_BALANCING
-		   "  node   group-id"
+	SEQ_printf(m, "  node   group-id");
 #endif
 #ifdef CONFIG_CGROUP_SCHED
-		   "  group-path"
+	if (show_cgroup_path)
+		SEQ_printf(m, "  group-path");
 #endif
-		   "\n");
+	SEQ_printf(m, "\n");
 	SEQ_printf(m, "-------------------------------------------------------"
 		   "------------------------------------------------------"
-		   "------------------------------------------------------"
+		   "------------------------------------------------------");
 #ifdef CONFIG_NUMA_BALANCING
-		   "--------------"
+	SEQ_printf(m, "--------------");
 #endif
 #ifdef CONFIG_CGROUP_SCHED
-		   "--------------"
+	if (show_cgroup_path)
+		SEQ_printf(m, "--------------");
 #endif
-		   "\n");
+	SEQ_printf(m, "\n");
 
 	rcu_read_lock();
 	for_each_process_thread(g, p) {
 		if (task_cpu(p) != rq_cpu)
 			continue;
 
-		print_task(m, rq, p);
+		if (queued_only && !task_current(rq, p) && !task_on_rq_queued(p))
+			continue;
+
+		print_task(m, rq, p, show_cgroup_path);
 	}
 	rcu_read_unlock();
 }
@@ -1234,7 +1242,7 @@ do {									\
 	print_rt_stats(m, cpu);
 	print_dl_stats(m, cpu);
 
-	print_rq(m, rq, cpu);
+	print_rq(m, rq, cpu, true, false);
 	SEQ_printf(m, "\n");
 }
 
@@ -1322,6 +1330,48 @@ void sysrq_sched_debug_show(void)
 	}
 }
 
+void sched_show_runqueues(void)
+{
+	int cpu;
+
+	pr_info("CPU Runqueues:\n");
+	for_each_online_cpu(cpu) {
+		struct rq *rq = cpu_rq(cpu);
+		struct task_struct *curr;
+		unsigned int nr_running;
+		u64 nr_switches;
+		unsigned long flags;
+		bool locked;
+
+		touch_nmi_watchdog();
+		touch_all_softlockup_watchdogs();
+
+		rcu_read_lock();
+		local_irq_save(flags);
+		locked = raw_spin_rq_trylock(rq);
+		if (locked) {
+			nr_running = rq->nr_running;
+			nr_switches = rq->nr_switches;
+			curr = rcu_dereference(rq->curr);
+			raw_spin_rq_unlock(rq);
+		} else {
+			nr_running = READ_ONCE(rq->nr_running);
+			nr_switches = READ_ONCE(rq->nr_switches);
+			curr = rcu_dereference(rq->curr);
+		}
+		local_irq_restore(flags);
+
+		pr_info("cpu#%d: nr_running:%u switches:%llu curr:%s[%d]%s\n",
+			cpu, nr_running, nr_switches,
+			curr ? curr->comm : "<none>",
+			curr ? task_pid_nr(curr) : -1,
+			locked ? "" : " (contended)");
+
+		print_rq(NULL, rq, cpu, false, true);
+		rcu_read_unlock();
+	}
+}
+
 /*
  * This iterator needs some explanation.
  * It returns 1 for the header position.
diff --git a/lib/sys_info.c b/lib/sys_info.c
index f32a06ec9ed4..fc5bfcc121de 100644
--- a/lib/sys_info.c
+++ b/lib/sys_info.c
@@ -22,6 +22,7 @@ static const char * const si_names[] = {
 	[ilog2(SYS_INFO_PANIC_CONSOLE_REPLAY)]	= "",
 	[ilog2(SYS_INFO_ALL_BT)]		= "all_bt",
 	[ilog2(SYS_INFO_BLOCKED_TASKS)]		= "blocked_tasks",
+	[ilog2(SYS_INFO_CPU_RUNQUEUES)]		= "cpu_runqueues",
 };
 
 /*
@@ -158,6 +159,9 @@ static void __sys_info(unsigned long si_mask)
 
 	if (si_mask & SYS_INFO_BLOCKED_TASKS)
 		show_state_filter(TASK_UNINTERRUPTIBLE);
+
+	if (si_mask & SYS_INFO_CPU_RUNQUEUES)
+		sched_show_runqueues();
 }
 
 void sys_info(unsigned long si_mask)
-- 
2.55.0

Re: [PATCH v2] sched/debug, sys_info: Introduce SYS_INFO_CPU_RUNQUEUES

[Petr Mladek]

On Fri 2026-09-11 21:32:40, Aaron Tomlin wrote:
> When investigating kernel panics, inspectability of per-CPU runqueues
> and runnable task states is valuable for diagnosing CPU starvation
> priority inversion, etc.
> 
> While debugfs (/sys/kernel/debug/sched/debug) exposes runqueue metrics
> to userspace, these details are not captured during an automated kernel
> panic or crash dump. Capturing per-CPU runqueue state directly into
> log_buf fills this diagnostic gap for post-mortem crash analysis.
> 
> Introduce SYS_INFO_CPU_RUNQUEUES and its corresponding string token
> "cpu_runqueues" to panic_sys_info. Add sched_show_runqueues(), modelled
> on print_rq(), to emit per-CPU scheduler diagnostics to the kernel log.
> 
> Unlike /sys/kernel/debug/sched/debug which dumps all threads assigned to
> a CPU, sched_show_runqueues() only emits threads that are actively
> running or queued on the runqueue (via task_on_rq_queued() and
> task_current()). This keeps the panic log concise, reflects the true
> runqueue depth, and prevents overflowing the printk ring buffer on
> systems with high thread counts.
> 
> Additionally, to guarantee deadlock and memory safety in panic context:
>     - Acquire the runqueue lock using raw_spin_rq_trylock() with
>       READ_ONCE() and rcu_dereference() fallback, marking contended
>       queues with " (contended)"
> 
>     - Wrap the per-CPU inspection in rcu_read_lock() to protect the
>       sampled current task (comm and PID) against premature release
>       during pr_info() across other callers
> 
>     - Omit cgroup group-path printing in print_rq() to avoid acquiring
>       cgroup_mutex and traversing kernfs dentries
> 
> --- a/Documentation/admin-guide/sysctl/kernel.rst
> +++ b/Documentation/admin-guide/sysctl/kernel.rst
> @@ -939,6 +939,7 @@ locks           print locks info if CONFIG_LOCKDEP is on
>  ftrace          print ftrace buffer
>  all_bt          print all CPUs backtrace (if available in the arch)
>  blocked_tasks   print only tasks in uninterruptible (blocked) state
> +cpu_runqueues   print per-CPU runqueue depth and runnable tasks

I would keep is short and call it "rq".

>  =============   ===================================================
>  
> --- a/include/linux/sys_info.h
> +++ b/include/linux/sys_info.h
> @@ -16,6 +16,7 @@
>  #define SYS_INFO_PANIC_CONSOLE_REPLAY	0x00000020
>  #define SYS_INFO_ALL_BT			0x00000040
>  #define SYS_INFO_BLOCKED_TASKS		0x00000080
> +#define SYS_INFO_CPU_RUNQUEUES		0x00000100

Similar here: SYS_INFO_RQ

>  void sys_info(unsigned long si_mask);
>  unsigned long sys_info_parse_param(char *str);
> diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
> index 72236db67983..79f6b00974bb 100644
> --- a/kernel/sched/debug.c
> +++ b/kernel/sched/debug.c
> @@ -1322,6 +1330,48 @@ void sysrq_sched_debug_show(void)
>  	}
>  }
>  
> +void sched_show_runqueues(void)
> +{
> +	int cpu;
> +
> +	pr_info("CPU Runqueues:\n");
> +	for_each_online_cpu(cpu) {
> +		struct rq *rq = cpu_rq(cpu);
> +		struct task_struct *curr;
> +		unsigned int nr_running;
> +		u64 nr_switches;
> +		unsigned long flags;
> +		bool locked;
> +
> +		touch_nmi_watchdog();
> +		touch_all_softlockup_watchdogs();
> +
> +		rcu_read_lock();
> +		local_irq_save(flags);
> +		locked = raw_spin_rq_trylock(rq);

Is the trylock needed for all sys_info() callers or just in panic()?
If it is just panic() then I would use it only when oops_in_progress
is set and use raw_spin_rq_lock() otherwise.

> +		if (locked) {
> +			nr_running = rq->nr_running;
> +			nr_switches = rq->nr_switches;
> +			curr = rcu_dereference(rq->curr);
> +			raw_spin_rq_unlock(rq);
> +		} else {
> +			nr_running = READ_ONCE(rq->nr_running);
> +			nr_switches = READ_ONCE(rq->nr_switches);
> +			curr = rcu_dereference(rq->curr);
> +		}
> +		local_irq_restore(flags);
> +
> +		pr_info("cpu#%d: nr_running:%u switches:%llu curr:%s[%d]%s\n",
> +			cpu, nr_running, nr_switches,
> +			curr ? curr->comm : "<none>",
> +			curr ? task_pid_nr(curr) : -1,
> +			locked ? "" : " (contended)");
> +
> +		print_rq(NULL, rq, cpu, false, true);
> +		rcu_read_unlock();
> +	}
> +}

IMHO, it might be a useful feature.

The main question is whether it is acceptable to scheduler
maintainers. It adds some churn. Also they would need to keep in mind
that it can be called in panic().

Best Regards,
Petr

Re: [PATCH v2] sched/debug, sys_info: Introduce SYS_INFO_CPU_RUNQUEUES

[Peter Zijlstra]

On Fri, Sep 11, 2026 at 09:32:40PM -0400, Aaron Tomlin wrote:
> When investigating kernel panics, inspectability of per-CPU runqueues
> and runnable task states is valuable for diagnosing CPU starvation
> priority inversion, etc.
> 
> While debugfs (/sys/kernel/debug/sched/debug) exposes runqueue metrics
> to userspace, these details are not captured during an automated kernel
> panic or crash dump. Capturing per-CPU runqueue state directly into
> log_buf fills this diagnostic gap for post-mortem crash analysis.

Uh, crash-dump preserves everything.


>  	rcu_read_lock();
>  	for_each_process_thread(g, p) {
>  		if (task_cpu(p) != rq_cpu)
>  			continue;
>  
> -		print_task(m, rq, p);
> +		if (queued_only && !task_current(rq, p) && !task_on_rq_queued(p))
> +			continue;
> +
> +		print_task(m, rq, p, show_cgroup_path);
>  	}
>  	rcu_read_unlock();
>  }
> @@ -1234,7 +1242,7 @@ do {									\
>  	print_rt_stats(m, cpu);
>  	print_dl_stats(m, cpu);
>  
> -	print_rq(m, rq, cpu);
> +	print_rq(m, rq, cpu, true, false);
>  	SEQ_printf(m, "\n");
>  }
>  
> @@ -1322,6 +1330,48 @@ void sysrq_sched_debug_show(void)
>  	}
>  }
>  
> +void sched_show_runqueues(void)
> +{
> +	int cpu;
> +
> +	pr_info("CPU Runqueues:\n");
> +	for_each_online_cpu(cpu) {
> +		struct rq *rq = cpu_rq(cpu);
> +		struct task_struct *curr;
> +		unsigned int nr_running;
> +		u64 nr_switches;
> +		unsigned long flags;
> +		bool locked;
> +
> +		touch_nmi_watchdog();
> +		touch_all_softlockup_watchdogs();
> +
> +		rcu_read_lock();
> +		local_irq_save(flags);
> +		locked = raw_spin_rq_trylock(rq);
> +		if (locked) {
> +			nr_running = rq->nr_running;
> +			nr_switches = rq->nr_switches;
> +			curr = rcu_dereference(rq->curr);
> +			raw_spin_rq_unlock(rq);
> +		} else {
> +			nr_running = READ_ONCE(rq->nr_running);
> +			nr_switches = READ_ONCE(rq->nr_switches);
> +			curr = rcu_dereference(rq->curr);
> +		}
> +		local_irq_restore(flags);

This seems to want to avoid deadlocking on rq->lock, but then
print_rq()->print_cfs_stats() will unconditionally take rq->lock again.

So meh.

> +
> +		pr_info("cpu#%d: nr_running:%u switches:%llu curr:%s[%d]%s\n",
> +			cpu, nr_running, nr_switches,
> +			curr ? curr->comm : "<none>",
> +			curr ? task_pid_nr(curr) : -1,
> +			locked ? "" : " (contended)");
> +
> +		print_rq(NULL, rq, cpu, false, true);
> +		rcu_read_unlock();
> +	}
> +}
> +
>  /*
>   * This iterator needs some explanation.
>   * It returns 1 for the header position.
> diff --git a/lib/sys_info.c b/lib/sys_info.c
> index f32a06ec9ed4..fc5bfcc121de 100644
> --- a/lib/sys_info.c
> +++ b/lib/sys_info.c
> @@ -22,6 +22,7 @@ static const char * const si_names[] = {
>  	[ilog2(SYS_INFO_PANIC_CONSOLE_REPLAY)]	= "",
>  	[ilog2(SYS_INFO_ALL_BT)]		= "all_bt",
>  	[ilog2(SYS_INFO_BLOCKED_TASKS)]		= "blocked_tasks",
> +	[ilog2(SYS_INFO_CPU_RUNQUEUES)]		= "cpu_runqueues",
>  };
>  
>  /*
> @@ -158,6 +159,9 @@ static void __sys_info(unsigned long si_mask)
>  
>  	if (si_mask & SYS_INFO_BLOCKED_TASKS)
>  		show_state_filter(TASK_UNINTERRUPTIBLE);
> +
> +	if (si_mask & SYS_INFO_CPU_RUNQUEUES)
> +		sched_show_runqueues();
>  }

I really don't know if this is worth the trouble. I have *never* needed
this.

Re: [PATCH v2] sched/debug, sys_info: Introduce SYS_INFO_CPU_RUNQUEUES

[Aaron Tomlin]

On Tue, Sep 22, 2026 at 04:31:42PM +0200, Petr Mladek wrote:
> On Fri 2026-09-11 21:32:40, Aaron Tomlin wrote:
> > When investigating kernel panics, inspectability of per-CPU runqueues
> > and runnable task states is valuable for diagnosing CPU starvation
> > priority inversion, etc.
> > 
> > While debugfs (/sys/kernel/debug/sched/debug) exposes runqueue metrics
> > to userspace, these details are not captured during an automated kernel
> > panic or crash dump. Capturing per-CPU runqueue state directly into
> > log_buf fills this diagnostic gap for post-mortem crash analysis.
> > 
> > Introduce SYS_INFO_CPU_RUNQUEUES and its corresponding string token
> > "cpu_runqueues" to panic_sys_info. Add sched_show_runqueues(), modelled
> > on print_rq(), to emit per-CPU scheduler diagnostics to the kernel log.
> > 
> > Unlike /sys/kernel/debug/sched/debug which dumps all threads assigned to
> > a CPU, sched_show_runqueues() only emits threads that are actively
> > running or queued on the runqueue (via task_on_rq_queued() and
> > task_current()). This keeps the panic log concise, reflects the true
> > runqueue depth, and prevents overflowing the printk ring buffer on
> > systems with high thread counts.
> > 
> > Additionally, to guarantee deadlock and memory safety in panic context:
> >     - Acquire the runqueue lock using raw_spin_rq_trylock() with
> >       READ_ONCE() and rcu_dereference() fallback, marking contended
> >       queues with " (contended)"
> > 
> >     - Wrap the per-CPU inspection in rcu_read_lock() to protect the
> >       sampled current task (comm and PID) against premature release
> >       during pr_info() across other callers
> > 
> >     - Omit cgroup group-path printing in print_rq() to avoid acquiring
> >       cgroup_mutex and traversing kernfs dentries
> > 
> > --- a/Documentation/admin-guide/sysctl/kernel.rst
> > +++ b/Documentation/admin-guide/sysctl/kernel.rst
> > @@ -939,6 +939,7 @@ locks           print locks info if CONFIG_LOCKDEP is on
> >  ftrace          print ftrace buffer
> >  all_bt          print all CPUs backtrace (if available in the arch)
> >  blocked_tasks   print only tasks in uninterruptible (blocked) state
> > +cpu_runqueues   print per-CPU runqueue depth and runnable tasks
> 
> I would keep is short and call it "rq".

Hi Petr,

Thank you for your review.

Acknowledged, fair enough.

> >  =============   ===================================================
> >  
> > --- a/include/linux/sys_info.h
> > +++ b/include/linux/sys_info.h
> > @@ -16,6 +16,7 @@
> >  #define SYS_INFO_PANIC_CONSOLE_REPLAY	0x00000020
> >  #define SYS_INFO_ALL_BT			0x00000040
> >  #define SYS_INFO_BLOCKED_TASKS		0x00000080
> > +#define SYS_INFO_CPU_RUNQUEUES		0x00000100
> 
> Similar here: SYS_INFO_RQ

Acknowledged.

> >  void sys_info(unsigned long si_mask);
> >  unsigned long sys_info_parse_param(char *str);
> > diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
> > index 72236db67983..79f6b00974bb 100644
> > --- a/kernel/sched/debug.c
> > +++ b/kernel/sched/debug.c
> > @@ -1322,6 +1330,48 @@ void sysrq_sched_debug_show(void)
> >  	}
> >  }
> >  
> > +void sched_show_runqueues(void)
> > +{
> > +	int cpu;
> > +
> > +	pr_info("CPU Runqueues:\n");

I will drop this. In print_rq(), there is often no overarching banner line
emitted beforehand.

> > +	for_each_online_cpu(cpu) {
> > +		struct rq *rq = cpu_rq(cpu);
> > +		struct task_struct *curr;
> > +		unsigned int nr_running;
> > +		u64 nr_switches;
> > +		unsigned long flags;
> > +		bool locked;
> > +
> > +		touch_nmi_watchdog();
> > +		touch_all_softlockup_watchdogs();
> > +
> > +		rcu_read_lock();
> > +		local_irq_save(flags);
> > +		locked = raw_spin_rq_trylock(rq);
> 
> Is the trylock needed for all sys_info() callers or just in panic()?
> If it is just panic() then I would use it only when oops_in_progress
> is set and use raw_spin_rq_lock() otherwise.

Yes, the trylock in sched_show_runqueues() is needed for all callers to
__sys_info(), not just panic().

Because __sys_info() is also called from NMI context (hardlockup), hardirq
context (softlockup), and khungtaskd, oops_in_progress is 0 in those paths.
Using unconditional raw_spin_rq_lock() there risks fatal self-deadlocks if
rq->lock is already held on the local CPU. Furthermore, locking arbitrary
runqueues in a loop risks AB-BA deadlocks with concurrent scheduler
load-balancing.

Using raw_spin_rq_trylock() ensures sched_show_runqueues() remains strictly
non-blocking across all __sys_info() contexts, and tagging contended queues
with " (contended)" adds useful diagnostic value.

As such, I believe the current implementation is sufficient.

> 
> > +		if (locked) {
> > +			nr_running = rq->nr_running;
> > +			nr_switches = rq->nr_switches;
> > +			curr = rcu_dereference(rq->curr);
> > +			raw_spin_rq_unlock(rq);
> > +		} else {
> > +			nr_running = READ_ONCE(rq->nr_running);
> > +			nr_switches = READ_ONCE(rq->nr_switches);
> > +			curr = rcu_dereference(rq->curr);
> > +		}
> > +		local_irq_restore(flags);
> > +
> > +		pr_info("cpu#%d: nr_running:%u switches:%llu curr:%s[%d]%s\n",
> > +			cpu, nr_running, nr_switches,
> > +			curr ? curr->comm : "<none>",
> > +			curr ? task_pid_nr(curr) : -1,
> > +			locked ? "" : " (contended)");
> > +
> > +		print_rq(NULL, rq, cpu, false, true);
> > +		rcu_read_unlock();
> > +	}
> > +}
> 
> IMHO, it might be a useful feature.
> 
> The main question is whether it is acceptable to scheduler
> maintainers. It adds some churn. Also they would need to keep in mind
> that it can be called in panic().

Thank you for your feedback.

Indeed. Regarding the scheduler maintainers (Cc'd Ingo Molnar, Peter
Zijlstra, Juri Lelli, and Vincent Guittot):

The churn was kept strictly confined to kernel/sched/debug.c:
    1.  It reuses existing print_rq()/print_task() helpers with two flags;
        queued_only, to keep output concise and show_cgroup_path, to avoid
        cgroup_mutex.

    2.  sched_show_runqueues() is completely read-only and non-blocking
        (raw_spin_rq_trylock() with READ_ONCE() and rcu_dereference()
        fallback), so it does not impose locking constraints or latency
        risks on core scheduler code, even in panic() or NMI/hardirq
        contexts.


Kind regards,
-- 
Aaron Tomlin

Re: [PATCH v2] sched/debug, sys_info: Introduce SYS_INFO_CPU_RUNQUEUES

[Aaron Tomlin]

On Tue, Sep 22, 2026 at 04:40:27PM +0200, Peter Zijlstra wrote:
> On Fri, Sep 11, 2026 at 09:32:40PM -0400, Aaron Tomlin wrote:
> > When investigating kernel panics, inspectability of per-CPU runqueues
> > and runnable task states is valuable for diagnosing CPU starvation
> > priority inversion, etc.
> > 
> > While debugfs (/sys/kernel/debug/sched/debug) exposes runqueue metrics
> > to userspace, these details are not captured during an automated kernel
> > panic or crash dump. Capturing per-CPU runqueue state directly into
> > log_buf fills this diagnostic gap for post-mortem crash analysis.
> 
> Uh, crash-dump preserves everything.

Hi Peter,

Yes, fair point. The intention here is to preserve this state directly
within the kernel log buffer, particularly for scenarios where kdump is not
configured, the vmcore was truncated or fails to complete.
In such circumstances, dmesg, whether captured via pstore or serial, is
frequently the sole surviving diagnostic artefact.

> 
> 
> >  	rcu_read_lock();
> >  	for_each_process_thread(g, p) {
> >  		if (task_cpu(p) != rq_cpu)
> >  			continue;
> >  
> > -		print_task(m, rq, p);
> > +		if (queued_only && !task_current(rq, p) && !task_on_rq_queued(p))
> > +			continue;
> > +
> > +		print_task(m, rq, p, show_cgroup_path);
> >  	}
> >  	rcu_read_unlock();
> >  }
> > @@ -1234,7 +1242,7 @@ do {									\
> >  	print_rt_stats(m, cpu);
> >  	print_dl_stats(m, cpu);
> >  
> > -	print_rq(m, rq, cpu);
> > +	print_rq(m, rq, cpu, true, false);
> >  	SEQ_printf(m, "\n");
> >  }
> >  
> > @@ -1322,6 +1330,48 @@ void sysrq_sched_debug_show(void)
> >  	}
> >  }
> >  
> > +void sched_show_runqueues(void)
> > +{
> > +	int cpu;
> > +
> > +	pr_info("CPU Runqueues:\n");
> > +	for_each_online_cpu(cpu) {
> > +		struct rq *rq = cpu_rq(cpu);
> > +		struct task_struct *curr;
> > +		unsigned int nr_running;
> > +		u64 nr_switches;
> > +		unsigned long flags;
> > +		bool locked;
> > +
> > +		touch_nmi_watchdog();
> > +		touch_all_softlockup_watchdogs();
> > +
> > +		rcu_read_lock();
> > +		local_irq_save(flags);
> > +		locked = raw_spin_rq_trylock(rq);
> > +		if (locked) {
> > +			nr_running = rq->nr_running;
> > +			nr_switches = rq->nr_switches;
> > +			curr = rcu_dereference(rq->curr);
> > +			raw_spin_rq_unlock(rq);
> > +		} else {
> > +			nr_running = READ_ONCE(rq->nr_running);
> > +			nr_switches = READ_ONCE(rq->nr_switches);
> > +			curr = rcu_dereference(rq->curr);
> > +		}
> > +		local_irq_restore(flags);
> 
> This seems to want to avoid deadlocking on rq->lock, but then
> print_rq()->print_cfs_stats() will unconditionally take rq->lock again.
> 
> So meh.

No, print_rq() does not call print_cfs_stats(). It is print_cpu() (i.e.
used by debugfs and SysRq) that calls print_cfs_stats() and unconditionally
takes rq->lock via print_cfs_rq().

With sched_show_runqueues(), it specifically avoids print_cpu() and only
calls print_rq(NULL, rq, cpu, false, true). Inside print_rq(), it merely
iterates threads under rcu_read_lock() and calls print_task(), neither of
which acquires rq->lock. Hence, rq->lock is never taken again, and the
non-blocking guarantee of the trylock remains intact throughout the entire
dump.

> 
> > +
> > +		pr_info("cpu#%d: nr_running:%u switches:%llu curr:%s[%d]%s\n",
> > +			cpu, nr_running, nr_switches,
> > +			curr ? curr->comm : "<none>",
> > +			curr ? task_pid_nr(curr) : -1,
> > +			locked ? "" : " (contended)");
> > +
> > +		print_rq(NULL, rq, cpu, false, true);
> > +		rcu_read_unlock();
> > +	}
> > +}
> > +
> >  /*
> >   * This iterator needs some explanation.
> >   * It returns 1 for the header position.
> > diff --git a/lib/sys_info.c b/lib/sys_info.c
> > index f32a06ec9ed4..fc5bfcc121de 100644
> > --- a/lib/sys_info.c
> > +++ b/lib/sys_info.c
> > @@ -22,6 +22,7 @@ static const char * const si_names[] = {
> >  	[ilog2(SYS_INFO_PANIC_CONSOLE_REPLAY)]	= "",
> >  	[ilog2(SYS_INFO_ALL_BT)]		= "all_bt",
> >  	[ilog2(SYS_INFO_BLOCKED_TASKS)]		= "blocked_tasks",
> > +	[ilog2(SYS_INFO_CPU_RUNQUEUES)]		= "cpu_runqueues",
> >  };
> >  
> >  /*
> > @@ -158,6 +159,9 @@ static void __sys_info(unsigned long si_mask)
> >  
> >  	if (si_mask & SYS_INFO_BLOCKED_TASKS)
> >  		show_state_filter(TASK_UNINTERRUPTIBLE);
> > +
> > +	if (si_mask & SYS_INFO_CPU_RUNQUEUES)
> > +		sched_show_runqueues();
> >  }
> 
> I really don't know if this is worth the trouble. I have *never* needed
> this.

I understand. However, the requirement primarily arises in production
support. In environments where kdump is not configured or the resulting
vmcore is truncated, dmesg is frequently the sole surviving diagnostic
artefact.

During an NMI-induced panic, capturing per-CPU runqueue depth in the log is
invaluable for identifying CPU starvation, etc. As with the remainder of
sys_info, this functionality is strictly opt-in and disabled by default,
imposing no overhead on systems that do not require it while providing
vital visibility where full memory dumps are unavailable.

Kind regards,
-- 
Aaron Tomlin