[PATCH v2] sched/debug, sys_info: Introduce SYS_INFO_CPU_RUNQUEUES

[PATCH v2] sched/debug, sys_info: Introduce SYS_INFO_CPU_RUNQUEUES

[Aaron Tomlin]

When investigating kernel panics, inspectability of per-CPU runqueues
and runnable task states is valuable for diagnosing CPU starvation
priority inversion, etc.

While debugfs (/sys/kernel/debug/sched/debug) exposes runqueue metrics
to userspace, these details are not captured during an automated kernel
panic or crash dump. Capturing per-CPU runqueue state directly into
log_buf fills this diagnostic gap for post-mortem crash analysis.

Introduce SYS_INFO_CPU_RUNQUEUES and its corresponding string token
"cpu_runqueues" to panic_sys_info. Add sched_show_runqueues(), modelled
on print_rq(), to emit per-CPU scheduler diagnostics to the kernel log.

Unlike /sys/kernel/debug/sched/debug which dumps all threads assigned to
a CPU, sched_show_runqueues() only emits threads that are actively
running or queued on the runqueue (via task_on_rq_queued() and
task_current()). This keeps the panic log concise, reflects the true
runqueue depth, and prevents overflowing the printk ring buffer on
systems with high thread counts.

Additionally, to guarantee deadlock and memory safety in panic context:
    - Acquire the runqueue lock using raw_spin_rq_trylock() with
      READ_ONCE() and rcu_dereference() fallback, marking contended
      queues with " (contended)"

    - Wrap the per-CPU inspection in rcu_read_lock() to protect the
      sampled current task (comm and PID) against premature release
      during pr_info() across other callers

    - Omit cgroup group-path printing in print_rq() to avoid acquiring
      cgroup_mutex and traversing kernfs dentries

Suggested-by: Rishil Sandip Shah <rishil1999@outlook.com>
Signed-off-by: Aaron Tomlin <atomlin@atomlin.com>
---
Changes since v1:

 - Resolved Sparse __rcu address space warnings by accessing rq->curr
   via rcu_dereference() in sched_show_runqueues()

 - Link to v1: https://lore.kernel.org/lkml/20260911022844.521413-1-atomlin@atomlin.com/
---
 Documentation/admin-guide/sysctl/kernel.rst |  1 +
 include/linux/sched/debug.h                 |  1 +
 include/linux/sys_info.h                    |  1 +
 kernel/sched/debug.c                        | 76 +++++++++++++++++----
 lib/sys_info.c                              |  4 ++
 5 files changed, 70 insertions(+), 13 deletions(-)

diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst
index b6328cd0f43e..0962b031e035 100644
--- a/Documentation/admin-guide/sysctl/kernel.rst
+++ b/Documentation/admin-guide/sysctl/kernel.rst
@@ -939,6 +939,7 @@ locks           print locks info if CONFIG_LOCKDEP is on
 ftrace          print ftrace buffer
 all_bt          print all CPUs backtrace (if available in the arch)
 blocked_tasks   print only tasks in uninterruptible (blocked) state
+cpu_runqueues   print per-CPU runqueue depth and runnable tasks
 =============   ===================================================
 
 
diff --git a/include/linux/sched/debug.h b/include/linux/sched/debug.h
index 35ed4577a6cc..d4276d8ead29 100644
--- a/include/linux/sched/debug.h
+++ b/include/linux/sched/debug.h
@@ -39,6 +39,7 @@ struct seq_file;
 extern void proc_sched_show_task(struct task_struct *p,
 				 struct pid_namespace *ns, struct seq_file *m);
 extern void proc_sched_set_task(struct task_struct *p);
+extern void sched_show_runqueues(void);
 
 /* Attach to any functions which should be ignored in wchan output. */
 #define __sched		__section(".sched.text")
diff --git a/include/linux/sys_info.h b/include/linux/sys_info.h
index a5bc3ea3d44b..c571aad1e178 100644
--- a/include/linux/sys_info.h
+++ b/include/linux/sys_info.h
@@ -16,6 +16,7 @@
 #define SYS_INFO_PANIC_CONSOLE_REPLAY	0x00000020
 #define SYS_INFO_ALL_BT			0x00000040
 #define SYS_INFO_BLOCKED_TASKS		0x00000080
+#define SYS_INFO_CPU_RUNQUEUES		0x00000100
 
 void sys_info(unsigned long si_mask);
 unsigned long sys_info_parse_param(char *str);
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 72236db67983..79f6b00974bb 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -968,7 +968,8 @@ static void task_group_path(struct task_group *tg, char *path, int plen)
 #endif
 
 static void
-print_task(struct seq_file *m, struct rq *rq, struct task_struct *p)
+print_task(struct seq_file *m, struct rq *rq, struct task_struct *p,
+	   bool show_cgroup_path)
 {
 	if (task_current(rq, p))
 		SEQ_printf(m, ">R");
@@ -996,13 +997,15 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p)
 	SEQ_printf(m, "   %d      %d", task_node(p), task_numa_group_id(p));
 #endif
 #ifdef CONFIG_CGROUP_SCHED
-	SEQ_printf_task_group_path(m, task_group(p), "        %s")
+	if (show_cgroup_path)
+		SEQ_printf_task_group_path(m, task_group(p), "        %s")
 #endif
 
 	SEQ_printf(m, "\n");
 }
 
-static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu)
+static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu,
+		     bool show_cgroup_path, bool queued_only)
 {
 	struct task_struct *g, *p;
 
@@ -1010,31 +1013,36 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu)
 	SEQ_printf(m, "runnable tasks:\n");
 	SEQ_printf(m, " S            task   PID     weight       vruntime   eligible    "
 		   "deadline             slice          sum-exec      switches  "
-		   "prio         wait-time        sum-sleep       sum-block"
+		   "prio         wait-time        sum-sleep       sum-block");
 #ifdef CONFIG_NUMA_BALANCING
-		   "  node   group-id"
+	SEQ_printf(m, "  node   group-id");
 #endif
 #ifdef CONFIG_CGROUP_SCHED
-		   "  group-path"
+	if (show_cgroup_path)
+		SEQ_printf(m, "  group-path");
 #endif
-		   "\n");
+	SEQ_printf(m, "\n");
 	SEQ_printf(m, "-------------------------------------------------------"
 		   "------------------------------------------------------"
-		   "------------------------------------------------------"
+		   "------------------------------------------------------");
 #ifdef CONFIG_NUMA_BALANCING
-		   "--------------"
+	SEQ_printf(m, "--------------");
 #endif
 #ifdef CONFIG_CGROUP_SCHED
-		   "--------------"
+	if (show_cgroup_path)
+		SEQ_printf(m, "--------------");
 #endif
-		   "\n");
+	SEQ_printf(m, "\n");
 
 	rcu_read_lock();
 	for_each_process_thread(g, p) {
 		if (task_cpu(p) != rq_cpu)
 			continue;
 
-		print_task(m, rq, p);
+		if (queued_only && !task_current(rq, p) && !task_on_rq_queued(p))
+			continue;
+
+		print_task(m, rq, p, show_cgroup_path);
 	}
 	rcu_read_unlock();
 }
@@ -1234,7 +1242,7 @@ do {									\
 	print_rt_stats(m, cpu);
 	print_dl_stats(m, cpu);
 
-	print_rq(m, rq, cpu);
+	print_rq(m, rq, cpu, true, false);
 	SEQ_printf(m, "\n");
 }
 
@@ -1322,6 +1330,48 @@ void sysrq_sched_debug_show(void)
 	}
 }
 
+void sched_show_runqueues(void)
+{
+	int cpu;
+
+	pr_info("CPU Runqueues:\n");
+	for_each_online_cpu(cpu) {
+		struct rq *rq = cpu_rq(cpu);
+		struct task_struct *curr;
+		unsigned int nr_running;
+		u64 nr_switches;
+		unsigned long flags;
+		bool locked;
+
+		touch_nmi_watchdog();
+		touch_all_softlockup_watchdogs();
+
+		rcu_read_lock();
+		local_irq_save(flags);
+		locked = raw_spin_rq_trylock(rq);
+		if (locked) {
+			nr_running = rq->nr_running;
+			nr_switches = rq->nr_switches;
+			curr = rcu_dereference(rq->curr);
+			raw_spin_rq_unlock(rq);
+		} else {
+			nr_running = READ_ONCE(rq->nr_running);
+			nr_switches = READ_ONCE(rq->nr_switches);
+			curr = rcu_dereference(rq->curr);
+		}
+		local_irq_restore(flags);
+
+		pr_info("cpu#%d: nr_running:%u switches:%llu curr:%s[%d]%s\n",
+			cpu, nr_running, nr_switches,
+			curr ? curr->comm : "<none>",
+			curr ? task_pid_nr(curr) : -1,
+			locked ? "" : " (contended)");
+
+		print_rq(NULL, rq, cpu, false, true);
+		rcu_read_unlock();
+	}
+}
+
 /*
  * This iterator needs some explanation.
  * It returns 1 for the header position.
diff --git a/lib/sys_info.c b/lib/sys_info.c
index f32a06ec9ed4..fc5bfcc121de 100644
--- a/lib/sys_info.c
+++ b/lib/sys_info.c
@@ -22,6 +22,7 @@ static const char * const si_names[] = {
 	[ilog2(SYS_INFO_PANIC_CONSOLE_REPLAY)]	= "",
 	[ilog2(SYS_INFO_ALL_BT)]		= "all_bt",
 	[ilog2(SYS_INFO_BLOCKED_TASKS)]		= "blocked_tasks",
+	[ilog2(SYS_INFO_CPU_RUNQUEUES)]		= "cpu_runqueues",
 };
 
 /*
@@ -158,6 +159,9 @@ static void __sys_info(unsigned long si_mask)
 
 	if (si_mask & SYS_INFO_BLOCKED_TASKS)
 		show_state_filter(TASK_UNINTERRUPTIBLE);
+
+	if (si_mask & SYS_INFO_CPU_RUNQUEUES)
+		sched_show_runqueues();
 }
 
 void sys_info(unsigned long si_mask)
-- 
2.55.0