[PATCH 0/3] workqueue: expose per-workqueue stats in debugfs

[PATCH 0/3] workqueue: expose per-workqueue stats in debugfs

[Wilson Felipe Pereira]

struct pool_workqueue already tracks runtime counters in pwq->stats[],
pwq->total_in_flight, and pwq->nr_active, but reading them currently
requires tools/workqueue/wq_monitor.py with drgn and kernel DWARF
debuginfo.

This series:
- Preserves pwq->stats[] in wq->retired_stats[] when unbound pwqs are
  released so cumulative per-workqueue counters do not go backwards.
- Exposes per-workqueue stats via /sys/kernel/debug/workqueue/stats so
  they can be read directly on systems without drgn or debuginfo.
- Updates wq_monitor.py to include wq->retired_stats[] and report a
  'queued' column alongside 'infl'.

Wilson Felipe Pereira (3):
  workqueue: preserve pwq->stats[] when unbound pwqs are released
  workqueue: expose per-workqueue stats in debugfs
  tools/workqueue: report queued items and retired stats in
    wq_monitor.py

 Documentation/core-api/workqueue.rst |  47 +++++++-----
 kernel/workqueue.c                   | 105 +++++++++++++++++++++++++++
 tools/workqueue/wq_monitor.py        |  24 +++++-
 3 files changed, 155 insertions(+), 21 deletions(-)

-- 
2.56.0.218.g1ec904b173-goog

[PATCH 1/3] workqueue: preserve pwq->stats[] when unbound pwqs are released

[Wilson Felipe Pereira]

When an unbound pool_workqueue is replaced (e.g. during CPU hotplug or
apply_workqueue_attrs()), pwq_release_workfn() unlinks and frees the old
pwq along with its pwq->stats[]. This causes cumulative per-workqueue
counters to go backwards.

Add retired_stats[] to struct workqueue_struct and accumulate a released
pwq's stats into wq->retired_stats[] under wq->mutex before freeing it.

Signed-off-by: Wilson Felipe Pereira <wfelipe@google.com>
---
 kernel/workqueue.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index f2e8071a6f4e..88c8cc2b38d8 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -397,6 +397,8 @@ struct workqueue_struct {
 	struct workqueue_attrs	*attrs;	/* PW: workqueue attributes */
 	struct pool_workqueue __rcu *dfl_pwq;   /* PW: only for unbound wqs */
 
+	u64			retired_stats[PWQ_NR_STATS]; /* WQ: stats from released pwqs */
+
 #ifdef CONFIG_SYSFS
 	struct wq_device	*wq_dev;	/* I: for sysfs interface */
 #endif
@@ -5475,8 +5477,12 @@ static void pwq_release_workfn(struct kthread_work *work)
 	 * @wq, and @wq is invalid to access.
 	 */
 	if (!list_empty(&pwq->pwqs_node)) {
+		int i;
+
 		mutex_lock(&wq->mutex);
 		list_del_rcu(&pwq->pwqs_node);
+		for (i = 0; i < PWQ_NR_STATS; i++)
+			wq->retired_stats[i] += pwq->stats[i];
 		is_last = list_empty(&wq->pwqs);
 
 		/*
-- 
2.56.0.218.g1ec904b173-goog

[PATCH 2/3] workqueue: expose per-workqueue stats in debugfs

[Wilson Felipe Pereira]

struct pool_workqueue tracks runtime counters in pwq->stats[],
pwq->total_in_flight, and pwq->nr_active, but reading them currently
requires tools/workqueue/wq_monitor.py with drgn and kernel DWARF
debuginfo. This makes it impractical to inspect workqueue stats on
systems without drgn or debug symbols.

Add /sys/kernel/debug/workqueue/stats to expose per-workqueue stats
directly from the kernel. Iterate over the workqueues list under RCU and
sum wq->retired_stats[] with each pwq's stats[], total_in_flight, and
nr_active to report queued, active, and running work items alongside the
cumulative pwq->stats[] counters.

Signed-off-by: Wilson Felipe Pereira <wfelipe@google.com>
---
 kernel/workqueue.c | 99 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 99 insertions(+)

diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 88c8cc2b38d8..ed0f0be6494a 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -57,6 +57,8 @@
 #include <linux/delay.h>
 #include <linux/irq_work.h>
 #include <linux/math64.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
 
 #include "workqueue_internal.h"
 
@@ -7990,6 +7992,103 @@ static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
 static void workqueue_sysfs_unregister(struct workqueue_struct *wq)	{ }
 #endif	/* CONFIG_SYSFS */
 
+#ifdef CONFIG_DEBUG_FS
+
+static const char *wq_type_str(struct workqueue_struct *wq)
+{
+	if (wq->flags & WQ_BH)
+		return "bh";
+	if (wq->flags & WQ_UNBOUND) {
+		if (wq->flags & __WQ_ORDERED)
+			return "ordered";
+		return "unbound";
+	}
+	return "percpu";
+}
+
+static void *wq_stats_start(struct seq_file *m, loff_t *pos)
+	__acquires(rcu)
+{
+	rcu_read_lock();
+	return seq_list_start_head_rcu(&workqueues, *pos);
+}
+
+static void *wq_stats_next(struct seq_file *m, void *v, loff_t *pos)
+{
+	return seq_list_next_rcu(v, &workqueues, pos);
+}
+
+static void wq_stats_stop(struct seq_file *m, void *v)
+	__releases(rcu)
+{
+	rcu_read_unlock();
+}
+
+static int wq_stats_show(struct seq_file *m, void *v)
+{
+	struct workqueue_struct *wq;
+	struct pool_workqueue *pwq;
+	u64 stats[PWQ_NR_STATS];
+	long in_flight = 0, active = 0;
+	long running, queued;
+	int i;
+
+	if (v == &workqueues) {
+		seq_printf(m, "%-24s %-8s %6s %6s %7s %10s %10s %12s %8s %8s %8s %8s %8s\n",
+			   "workqueue", "type", "queued", "active", "running",
+			   "started", "completed", "cpu_time_us",
+			   "cpu_itsv", "cm_wkup", "repat", "mayday", "rescued");
+		return 0;
+	}
+
+	wq = list_entry(v, struct workqueue_struct, list);
+
+	/* Advisory stats read without wq->mutex or pool->lock. */
+	for (i = 0; i < PWQ_NR_STATS; i++)
+		stats[i] = data_race(wq->retired_stats[i]);
+
+	for_each_pwq(pwq, wq) {
+		for (i = 0; i < PWQ_NR_STATS; i++)
+			stats[i] += data_race(pwq->stats[i]);
+		in_flight += data_race(pwq->total_in_flight);
+		active += data_race(pwq->nr_active);
+	}
+
+	running = max_t(long, stats[PWQ_STAT_STARTED] -
+			      stats[PWQ_STAT_COMPLETED], 0);
+	queued = max_t(long, in_flight - running, 0);
+	active = max_t(long, active, 0);
+
+	seq_printf(m, "%-24s %-8s %6ld %6ld %7ld %10llu %10llu %12llu %8llu %8llu %8llu %8llu %8llu\n",
+		   wq->name, wq_type_str(wq), queued, active, running,
+		   stats[PWQ_STAT_STARTED], stats[PWQ_STAT_COMPLETED],
+		   stats[PWQ_STAT_CPU_TIME], stats[PWQ_STAT_CPU_INTENSIVE],
+		   stats[PWQ_STAT_CM_WAKEUP], stats[PWQ_STAT_REPATRIATED],
+		   stats[PWQ_STAT_MAYDAY], stats[PWQ_STAT_RESCUED]);
+
+	return 0;
+}
+
+static const struct seq_operations wq_stats_sops = {
+	.start	= wq_stats_start,
+	.next	= wq_stats_next,
+	.stop	= wq_stats_stop,
+	.show	= wq_stats_show,
+};
+DEFINE_SEQ_ATTRIBUTE(wq_stats);
+
+static int __init wq_debugfs_init(void)
+{
+	struct dentry *dir;
+
+	dir = debugfs_create_dir("workqueue", NULL);
+	debugfs_create_file("stats", 0444, dir, NULL, &wq_stats_fops);
+	return 0;
+}
+late_initcall(wq_debugfs_init);
+
+#endif	/* CONFIG_DEBUG_FS */
+
 /*
  * Workqueue watchdog.
  *
-- 
2.56.0.218.g1ec904b173-goog

[PATCH 3/3] tools/workqueue: report queued items and retired stats in wq_monitor.py

[Wilson Felipe Pereira]

Update wq_monitor.py to include wq->retired_stats[] when available so
cumulative counters do not drop when unbound pwqs are released. Also sum
pwq->total_in_flight (falling back to pwq->nr_in_flight[] on older
kernels) and add a 'queued' column next to 'infl' to show work items
waiting to execute.

Update Documentation/core-api/workqueue.rst to cover
/sys/kernel/debug/workqueue/stats and the new 'queued' column in
wq_monitor.py.

Signed-off-by: Wilson Felipe Pereira <wfelipe@google.com>
---
 Documentation/core-api/workqueue.rst | 47 +++++++++++++++++-----------
 tools/workqueue/wq_monitor.py        | 24 ++++++++++++--
 2 files changed, 50 insertions(+), 21 deletions(-)

diff --git a/Documentation/core-api/workqueue.rst b/Documentation/core-api/workqueue.rst
index bb770f556568..a4e08231580c 100644
--- a/Documentation/core-api/workqueue.rst
+++ b/Documentation/core-api/workqueue.rst
@@ -701,28 +701,39 @@ See the command's help message for more info.
 Monitoring
 ==========
 
-Use tools/workqueue/wq_monitor.py to monitor workqueue operations: ::
+When ``CONFIG_DEBUG_FS`` is enabled, per-workqueue stats can be read
+directly from ``/sys/kernel/debug/workqueue/stats``: ::
+
+  $ cat /sys/kernel/debug/workqueue/stats
+  workqueue                type     queued active running    started  completed  cpu_time_us cpu_itsv  cm_wkup    repat   mayday  rescued
+  events                   percpu        0      0       0      18545      18545      6100000        0        5        0        0        0
+  events_highpri           percpu        0      0       0          8          8            0        0        0        0        0        0
+  events_long              percpu        0      0       0          3          3            0        0        0        0        0        0
+  events_unbound           unbound       0      0       0      38306      38306       100000        0        0        7        0        0
+
+Alternatively, use tools/workqueue/wq_monitor.py to continuously monitor
+workqueue operations via ``drgn``: ::
 
   $ tools/workqueue/wq_monitor.py events
-                              total  infl  CPUtime  CPUhog CMW/RPR  mayday rescued
-  events                      18545     0      6.1       0       5       -       -
-  events_highpri                  8     0      0.0       0       0       -       -
-  events_long                     3     0      0.0       0       0       -       -
-  events_unbound              38306     0      0.1       -       7       -       -
-  events_freezable                0     0      0.0       0       0       -       -
-  events_power_efficient      29598     0      0.2       0       0       -       -
-  events_freezable_pwr_ef        10     0      0.0       0       0       -       -
+                               total queued  infl  CPUtime CPUitsv CMW/RPR  mayday rescued
+  events                      18545      0     0      6.1       0       5       -       -
+  events_highpri                  8      0     0      0.0       0       0       -       -
+  events_long                     3      0     0      0.0       0       0       -       -
+  events_unbound              38306      0     0      0.1       -       7       -       -
+  events_freezable                0      0     0      0.0       0       0       -       -
+  events_power_efficient      29598      0     0      0.2       0       0       -       -
+  events_freezable_pwr_ef        10      0     0      0.0       0       0       -       -
   sock_diag_events                0     0      0.0       0       0       -       -
 
-                              total  infl  CPUtime  CPUhog CMW/RPR  mayday rescued
-  events                      18548     0      6.1       0       5       -       -
-  events_highpri                  8     0      0.0       0       0       -       -
-  events_long                     3     0      0.0       0       0       -       -
-  events_unbound              38322     0      0.1       -       7       -       -
-  events_freezable                0     0      0.0       0       0       -       -
-  events_power_efficient      29603     0      0.2       0       0       -       -
-  events_freezable_pwr_ef        10     0      0.0       0       0       -       -
-  sock_diag_events                0     0      0.0       0       0       -       -
+                               total queued  infl  CPUtime CPUitsv CMW/RPR  mayday rescued
+  events                      18548      0     0      6.1       0       5       -       -
+  events_highpri                  8      0     0      0.0       0       0       -       -
+  events_long                     3      0     0      0.0       0       0       -       -
+  events_unbound              38322      0     0      0.1       -       7       -       -
+  events_freezable                0      0     0      0.0       0       0       -       -
+  events_power_efficient      29603      0     0      0.2       0       0       -       -
+  events_freezable_pwr_ef        10      0     0      0.0       0       0       -       -
+  sock_diag_events                0      0     0      0.0       0       0       -       -
 
   ...
 
diff --git a/tools/workqueue/wq_monitor.py b/tools/workqueue/wq_monitor.py
index 7f47fa398e3c..2f4176df2a36 100644
--- a/tools/workqueue/wq_monitor.py
+++ b/tools/workqueue/wq_monitor.py
@@ -9,7 +9,9 @@ https://github.com/osandov/drgn.
 
   total    Total number of work items executed by the workqueue.
 
-  infl     The number of currently in-flight work items.
+  queued   The number of work items currently queued and waiting to execute.
+
+  infl     The number of currently executing (in-flight) work items.
 
   CPUtime  Total CPU time consumed by the workqueue in seconds. This is
            sampled from scheduler ticks and only provides ballpark
@@ -55,6 +57,7 @@ workqueues              = prog['workqueues']
 
 WQ_UNBOUND              = prog['WQ_UNBOUND']
 WQ_MEM_RECLAIM          = prog['WQ_MEM_RECLAIM']
+WORK_NR_COLORS          = prog['WORK_NR_COLORS']
 
 PWQ_STAT_STARTED        = prog['PWQ_STAT_STARTED']      # work items started execution
 PWQ_STAT_COMPLETED      = prog['PWQ_STAT_COMPLETED']	# work items completed execution
@@ -72,15 +75,29 @@ class WqStats:
         self.unbound = wq.flags & WQ_UNBOUND != 0
         self.mem_reclaim = wq.flags & WQ_MEM_RECLAIM != 0
         self.stats = [0] * PWQ_NR_STATS
+        self.in_flight = 0
+        try:
+            for i in range(PWQ_NR_STATS):
+                self.stats[i] += int(wq.retired_stats[i])
+        except AttributeError:
+            pass
         for pwq in list_for_each_entry('struct pool_workqueue', wq.pwqs.address_of_(), 'pwqs_node'):
             for i in range(PWQ_NR_STATS):
                 self.stats[i] += int(pwq.stats[i])
+            try:
+                self.in_flight += int(pwq.total_in_flight)
+            except AttributeError:
+                for c in range(WORK_NR_COLORS):
+                    self.in_flight += int(pwq.nr_in_flight[c])
+        self.running = max(self.stats[PWQ_STAT_STARTED] - self.stats[PWQ_STAT_COMPLETED], 0)
+        self.queued = max(self.in_flight - self.running, 0)
 
     def dict(self, now):
         return { 'timestamp'            : now,
                  'name'                 : self.name,
                  'unbound'              : self.unbound,
                  'mem_reclaim'          : self.mem_reclaim,
+                 'queued'               : self.queued,
                  'started'              : self.stats[PWQ_STAT_STARTED],
                  'completed'            : self.stats[PWQ_STAT_COMPLETED],
                  'cpu_time'             : self.stats[PWQ_STAT_CPU_TIME],
@@ -91,7 +108,7 @@ class WqStats:
                  'rescued'              : self.stats[PWQ_STAT_RESCUED], }
 
     def table_header_str():
-        return f'{"":>24} {"total":>8} {"infl":>5} {"CPUtime":>8} '\
+        return f'{"":>24} {"total":>8} {"queued":>6} {"infl":>5} {"CPUtime":>8} '\
             f'{"CPUitsv":>7} {"CMW/RPR":>7} {"mayday":>7} {"rescued":>7}'
 
     def table_row_str(self):
@@ -112,7 +129,8 @@ class WqStats:
 
         out = f'{self.name[-24:]:24} ' \
               f'{self.stats[PWQ_STAT_STARTED]:8} ' \
-              f'{max(self.stats[PWQ_STAT_STARTED] - self.stats[PWQ_STAT_COMPLETED], 0):5} ' \
+              f'{self.queued:6} ' \
+              f'{self.running:5} ' \
               f'{self.stats[PWQ_STAT_CPU_TIME] / 1000000:8.1f} ' \
               f'{cpu_intensive:>7} ' \
               f'{cmw_rpr:>7} ' \
-- 
2.56.0.218.g1ec904b173-goog

Re: [PATCH 2/3] workqueue: expose per-workqueue stats in debugfs

[Tejun Heo]

On Thu, Oct 22, 2026 at 03:53:35PM +0000, Wilson Felipe Pereira wrote:
> struct pool_workqueue tracks runtime counters in pwq->stats[],
> pwq->total_in_flight, and pwq->nr_active, but reading them currently
> requires tools/workqueue/wq_monitor.py with drgn and kernel DWARF
> debuginfo. This makes it impractical to inspect workqueue stats on
> systems without drgn or debug symbols.
> 
> Add /sys/kernel/debug/workqueue/stats to expose per-workqueue stats
> directly from the kernel. Iterate over the workqueues list under RCU and
> sum wq->retired_stats[] with each pwq's stats[], total_in_flight, and
> nr_active to report queued, active, and running work items alongside the
> cumulative pwq->stats[] counters.

This isn't really a justification. What do you use these numbers for? If
this is for live monitoring, why debugfs? If for debugging, what's wrong
with drgn?

Thanks.

-- 
tejun

Re: [PATCH 2/3] workqueue: expose per-workqueue stats in debugfs

[Wilson Felipe Pereira]

Tejun Heo <tj@kernel.org> writes:

> On Thu, Oct 22, 2026 at 03:53:35PM +0000, Wilson Felipe Pereira wrote:
>> struct pool_workqueue tracks runtime counters in pwq->stats[],
>> pwq->total_in_flight, and pwq->nr_active, but reading them currently
>> requires tools/workqueue/wq_monitor.py with drgn and kernel DWARF
>> debuginfo. This makes it impractical to inspect workqueue stats on
>> systems without drgn or debug symbols.
>>
>> Add /sys/kernel/debug/workqueue/stats to expose per-workqueue stats
>> directly from the kernel. Iterate over the workqueues list under RCU and
>> sum wq->retired_stats[] with each pwq's stats[], total_in_flight, and
>> nr_active to report queued, active, and running work items alongside the
>> cumulative pwq->stats[] counters.
>
> This isn't really a justification. What do you use these numbers for? If
> this is for live monitoring, why debugfs? If for debugging, what's wrong
> with drgn?

This will mainly be used for continuous monitoring and exporting the
stats from the machine (CPU time, queued backlog, etc.). Also to quickly
check if any item is queued or stuck.

Why not drgn? For continuous monitoring, running drgn on every machine,
with vmlinux debuginfo, and memory footprint from Python/drgn, while
having CONFIG_PROC_KCORE available, it adds up. Having a simple file
interface, it can be read from an existing monitoring collector.

Why debugfs? If sysfs is picked, not all queues are in
/sys/bus/workqueue/devices/. Using debugfs, a single file can include
all queues. If you have a suggestion, I'm open to change this series.

Thanks!

Re: [PATCH 2/3] workqueue: expose per-workqueue stats in debugfs

[Tejun Heo]

Hello, Wilson.

On Thu, Oct 22, 2026 at 10:46:19PM +0000, Wilson Felipe Pereira wrote:
> Why not drgn? For continuous monitoring, running drgn on every machine,
> with vmlinux debuginfo, and memory footprint from Python/drgn, while
> having CONFIG_PROC_KCORE available, it adds up. Having a simple file
> interface, it can be read from an existing monitoring collector.
>
> Why debugfs? If sysfs is picked, not all queues are in
> /sys/bus/workqueue/devices/. Using debugfs, a single file can include
> all queues. If you have a suggestion, I'm open to change this series.

I'm usually really skeptical when people suggest debugfs as a solution.
If you need continuous monitoring and drgn is too heavy, this is trivial
to do with BPF. The following bpftrace script prints the per-workqueue
stats and needs only BTF, no DWARF debuginfo, /proc/kcore or Python:

  BEGIN
  {
      printf("%-24s %10s %10s %12s %6s %6s %6s\n", "workqueue", "started",
             "completed", "cpu_time_us", "active", "cmw", "mayday");

      $whead = (uint64)kaddr("workqueues");
      $wpos = (uint64)((struct list_head *)$whead)->next;
      for ($i : 0..4096) {
          if ($wpos == $whead) {
              break;
          }
          $wq = (struct workqueue_struct *)($wpos - offsetof(struct workqueue_struct, list));
          $started = (uint64)0; $completed = (uint64)0; $cpu = (uint64)0;
          $cmw = (uint64)0; $mayday = (uint64)0; $active = 0;

          $phead = (uint64)$wq + offsetof(struct workqueue_struct, pwqs);
          $ppos = (uint64)((struct list_head *)$phead)->next;
          for ($j : 0..8192) {
              if ($ppos == $phead) {
                  break;
              }
              $pwq = (struct pool_workqueue *)($ppos - offsetof(struct pool_workqueue, pwqs_node));
              $started += $pwq->stats[PWQ_STAT_STARTED];
              $completed += $pwq->stats[PWQ_STAT_COMPLETED];
              $cpu += $pwq->stats[PWQ_STAT_CPU_TIME];
              $cmw += $pwq->stats[PWQ_STAT_CM_WAKEUP];
              $mayday += $pwq->stats[PWQ_STAT_MAYDAY];
              $active += $pwq->nr_active;
              $ppos = (uint64)((struct list_head *)$ppos)->next;
          }

          printf("%-24s %10lu %10lu %12lu %6d %6lu %6lu\n", $wq->name, $started,
                 $completed, $cpu, $active, $cmw, $mayday);
          $wpos = (uint64)((struct list_head *)$wpos)->next;
      }
      exit();
  }

It took an AI agent about a minute to write and test.

Thanks.

--
tejun