[PATCH 0/3] workqueue: expose per-workqueue stats in debugfs

[PATCH 0/3] workqueue: expose per-workqueue stats in debugfs

[Wilson Felipe Pereira]

struct pool_workqueue already tracks runtime counters in pwq->stats[],
pwq->total_in_flight, and pwq->nr_active, but reading them currently
requires tools/workqueue/wq_monitor.py with drgn and kernel DWARF
debuginfo.

This series:
- Preserves pwq->stats[] in wq->retired_stats[] when unbound pwqs are
  released so cumulative per-workqueue counters do not go backwards.
- Exposes per-workqueue stats via /sys/kernel/debug/workqueue/stats so
  they can be read directly on systems without drgn or debuginfo.
- Updates wq_monitor.py to include wq->retired_stats[] and report a
  'queued' column alongside 'infl'.

Wilson Felipe Pereira (3):
  workqueue: preserve pwq->stats[] when unbound pwqs are released
  workqueue: expose per-workqueue stats in debugfs
  tools/workqueue: report queued items and retired stats in
    wq_monitor.py

 Documentation/core-api/workqueue.rst |  47 +++++++-----
 kernel/workqueue.c                   | 105 +++++++++++++++++++++++++++
 tools/workqueue/wq_monitor.py        |  24 +++++-
 3 files changed, 155 insertions(+), 21 deletions(-)

-- 
2.56.0.218.g1ec904b173-goog

[PATCH 1/3] workqueue: preserve pwq->stats[] when unbound pwqs are released

[Wilson Felipe Pereira]

When an unbound pool_workqueue is replaced (e.g. during CPU hotplug or
apply_workqueue_attrs()), pwq_release_workfn() unlinks and frees the old
pwq along with its pwq->stats[]. This causes cumulative per-workqueue
counters to go backwards.

Add retired_stats[] to struct workqueue_struct and accumulate a released
pwq's stats into wq->retired_stats[] under wq->mutex before freeing it.

Signed-off-by: Wilson Felipe Pereira <wfelipe@google.com>
---
 kernel/workqueue.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index f2e8071a6f4e..88c8cc2b38d8 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -397,6 +397,8 @@ struct workqueue_struct {
 	struct workqueue_attrs	*attrs;	/* PW: workqueue attributes */
 	struct pool_workqueue __rcu *dfl_pwq;   /* PW: only for unbound wqs */
 
+	u64			retired_stats[PWQ_NR_STATS]; /* WQ: stats from released pwqs */
+
 #ifdef CONFIG_SYSFS
 	struct wq_device	*wq_dev;	/* I: for sysfs interface */
 #endif
@@ -5475,8 +5477,12 @@ static void pwq_release_workfn(struct kthread_work *work)
 	 * @wq, and @wq is invalid to access.
 	 */
 	if (!list_empty(&pwq->pwqs_node)) {
+		int i;
+
 		mutex_lock(&wq->mutex);
 		list_del_rcu(&pwq->pwqs_node);
+		for (i = 0; i < PWQ_NR_STATS; i++)
+			wq->retired_stats[i] += pwq->stats[i];
 		is_last = list_empty(&wq->pwqs);
 
 		/*
-- 
2.56.0.218.g1ec904b173-goog

[PATCH 2/3] workqueue: expose per-workqueue stats in debugfs

[Wilson Felipe Pereira]

struct pool_workqueue tracks runtime counters in pwq->stats[],
pwq->total_in_flight, and pwq->nr_active, but reading them currently
requires tools/workqueue/wq_monitor.py with drgn and kernel DWARF
debuginfo. This makes it impractical to inspect workqueue stats on
systems without drgn or debug symbols.

Add /sys/kernel/debug/workqueue/stats to expose per-workqueue stats
directly from the kernel. Iterate over the workqueues list under RCU and
sum wq->retired_stats[] with each pwq's stats[], total_in_flight, and
nr_active to report queued, active, and running work items alongside the
cumulative pwq->stats[] counters.

Signed-off-by: Wilson Felipe Pereira <wfelipe@google.com>
---
 kernel/workqueue.c | 99 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 99 insertions(+)

diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 88c8cc2b38d8..ed0f0be6494a 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -57,6 +57,8 @@
 #include <linux/delay.h>
 #include <linux/irq_work.h>
 #include <linux/math64.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
 
 #include "workqueue_internal.h"
 
@@ -7990,6 +7992,103 @@ static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
 static void workqueue_sysfs_unregister(struct workqueue_struct *wq)	{ }
 #endif	/* CONFIG_SYSFS */
 
+#ifdef CONFIG_DEBUG_FS
+
+static const char *wq_type_str(struct workqueue_struct *wq)
+{
+	if (wq->flags & WQ_BH)
+		return "bh";
+	if (wq->flags & WQ_UNBOUND) {
+		if (wq->flags & __WQ_ORDERED)
+			return "ordered";
+		return "unbound";
+	}
+	return "percpu";
+}
+
+static void *wq_stats_start(struct seq_file *m, loff_t *pos)
+	__acquires(rcu)
+{
+	rcu_read_lock();
+	return seq_list_start_head_rcu(&workqueues, *pos);
+}
+
+static void *wq_stats_next(struct seq_file *m, void *v, loff_t *pos)
+{
+	return seq_list_next_rcu(v, &workqueues, pos);
+}
+
+static void wq_stats_stop(struct seq_file *m, void *v)
+	__releases(rcu)
+{
+	rcu_read_unlock();
+}
+
+static int wq_stats_show(struct seq_file *m, void *v)
+{
+	struct workqueue_struct *wq;
+	struct pool_workqueue *pwq;
+	u64 stats[PWQ_NR_STATS];
+	long in_flight = 0, active = 0;
+	long running, queued;
+	int i;
+
+	if (v == &workqueues) {
+		seq_printf(m, "%-24s %-8s %6s %6s %7s %10s %10s %12s %8s %8s %8s %8s %8s\n",
+			   "workqueue", "type", "queued", "active", "running",
+			   "started", "completed", "cpu_time_us",
+			   "cpu_itsv", "cm_wkup", "repat", "mayday", "rescued");
+		return 0;
+	}
+
+	wq = list_entry(v, struct workqueue_struct, list);
+
+	/* Advisory stats read without wq->mutex or pool->lock. */
+	for (i = 0; i < PWQ_NR_STATS; i++)
+		stats[i] = data_race(wq->retired_stats[i]);
+
+	for_each_pwq(pwq, wq) {
+		for (i = 0; i < PWQ_NR_STATS; i++)
+			stats[i] += data_race(pwq->stats[i]);
+		in_flight += data_race(pwq->total_in_flight);
+		active += data_race(pwq->nr_active);
+	}
+
+	running = max_t(long, stats[PWQ_STAT_STARTED] -
+			      stats[PWQ_STAT_COMPLETED], 0);
+	queued = max_t(long, in_flight - running, 0);
+	active = max_t(long, active, 0);
+
+	seq_printf(m, "%-24s %-8s %6ld %6ld %7ld %10llu %10llu %12llu %8llu %8llu %8llu %8llu %8llu\n",
+		   wq->name, wq_type_str(wq), queued, active, running,
+		   stats[PWQ_STAT_STARTED], stats[PWQ_STAT_COMPLETED],
+		   stats[PWQ_STAT_CPU_TIME], stats[PWQ_STAT_CPU_INTENSIVE],
+		   stats[PWQ_STAT_CM_WAKEUP], stats[PWQ_STAT_REPATRIATED],
+		   stats[PWQ_STAT_MAYDAY], stats[PWQ_STAT_RESCUED]);
+
+	return 0;
+}
+
+static const struct seq_operations wq_stats_sops = {
+	.start	= wq_stats_start,
+	.next	= wq_stats_next,
+	.stop	= wq_stats_stop,
+	.show	= wq_stats_show,
+};
+DEFINE_SEQ_ATTRIBUTE(wq_stats);
+
+static int __init wq_debugfs_init(void)
+{
+	struct dentry *dir;
+
+	dir = debugfs_create_dir("workqueue", NULL);
+	debugfs_create_file("stats", 0444, dir, NULL, &wq_stats_fops);
+	return 0;
+}
+late_initcall(wq_debugfs_init);
+
+#endif	/* CONFIG_DEBUG_FS */
+
 /*
  * Workqueue watchdog.
  *
-- 
2.56.0.218.g1ec904b173-goog

[PATCH 3/3] tools/workqueue: report queued items and retired stats in wq_monitor.py

[Wilson Felipe Pereira]

Update wq_monitor.py to include wq->retired_stats[] when available so
cumulative counters do not drop when unbound pwqs are released. Also sum
pwq->total_in_flight (falling back to pwq->nr_in_flight[] on older
kernels) and add a 'queued' column next to 'infl' to show work items
waiting to execute.

Update Documentation/core-api/workqueue.rst to cover
/sys/kernel/debug/workqueue/stats and the new 'queued' column in
wq_monitor.py.

Signed-off-by: Wilson Felipe Pereira <wfelipe@google.com>
---
 Documentation/core-api/workqueue.rst | 47 +++++++++++++++++-----------
 tools/workqueue/wq_monitor.py        | 24 ++++++++++++--
 2 files changed, 50 insertions(+), 21 deletions(-)

diff --git a/Documentation/core-api/workqueue.rst b/Documentation/core-api/workqueue.rst
index bb770f556568..a4e08231580c 100644
--- a/Documentation/core-api/workqueue.rst
+++ b/Documentation/core-api/workqueue.rst
@@ -701,28 +701,39 @@ See the command's help message for more info.
 Monitoring
 ==========
 
-Use tools/workqueue/wq_monitor.py to monitor workqueue operations: ::
+When ``CONFIG_DEBUG_FS`` is enabled, per-workqueue stats can be read
+directly from ``/sys/kernel/debug/workqueue/stats``: ::
+
+  $ cat /sys/kernel/debug/workqueue/stats
+  workqueue                type     queued active running    started  completed  cpu_time_us cpu_itsv  cm_wkup    repat   mayday  rescued
+  events                   percpu        0      0       0      18545      18545      6100000        0        5        0        0        0
+  events_highpri           percpu        0      0       0          8          8            0        0        0        0        0        0
+  events_long              percpu        0      0       0          3          3            0        0        0        0        0        0
+  events_unbound           unbound       0      0       0      38306      38306       100000        0        0        7        0        0
+
+Alternatively, use tools/workqueue/wq_monitor.py to continuously monitor
+workqueue operations via ``drgn``: ::
 
   $ tools/workqueue/wq_monitor.py events
-                              total  infl  CPUtime  CPUhog CMW/RPR  mayday rescued
-  events                      18545     0      6.1       0       5       -       -
-  events_highpri                  8     0      0.0       0       0       -       -
-  events_long                     3     0      0.0       0       0       -       -
-  events_unbound              38306     0      0.1       -       7       -       -
-  events_freezable                0     0      0.0       0       0       -       -
-  events_power_efficient      29598     0      0.2       0       0       -       -
-  events_freezable_pwr_ef        10     0      0.0       0       0       -       -
+                               total queued  infl  CPUtime CPUitsv CMW/RPR  mayday rescued
+  events                      18545      0     0      6.1       0       5       -       -
+  events_highpri                  8      0     0      0.0       0       0       -       -
+  events_long                     3      0     0      0.0       0       0       -       -
+  events_unbound              38306      0     0      0.1       -       7       -       -
+  events_freezable                0      0     0      0.0       0       0       -       -
+  events_power_efficient      29598      0     0      0.2       0       0       -       -
+  events_freezable_pwr_ef        10      0     0      0.0       0       0       -       -
   sock_diag_events                0     0      0.0       0       0       -       -
 
-                              total  infl  CPUtime  CPUhog CMW/RPR  mayday rescued
-  events                      18548     0      6.1       0       5       -       -
-  events_highpri                  8     0      0.0       0       0       -       -
-  events_long                     3     0      0.0       0       0       -       -
-  events_unbound              38322     0      0.1       -       7       -       -
-  events_freezable                0     0      0.0       0       0       -       -
-  events_power_efficient      29603     0      0.2       0       0       -       -
-  events_freezable_pwr_ef        10     0      0.0       0       0       -       -
-  sock_diag_events                0     0      0.0       0       0       -       -
+                               total queued  infl  CPUtime CPUitsv CMW/RPR  mayday rescued
+  events                      18548      0     0      6.1       0       5       -       -
+  events_highpri                  8      0     0      0.0       0       0       -       -
+  events_long                     3      0     0      0.0       0       0       -       -
+  events_unbound              38322      0     0      0.1       -       7       -       -
+  events_freezable                0      0     0      0.0       0       0       -       -
+  events_power_efficient      29603      0     0      0.2       0       0       -       -
+  events_freezable_pwr_ef        10      0     0      0.0       0       0       -       -
+  sock_diag_events                0      0     0      0.0       0       0       -       -
 
   ...
 
diff --git a/tools/workqueue/wq_monitor.py b/tools/workqueue/wq_monitor.py
index 7f47fa398e3c..2f4176df2a36 100644
--- a/tools/workqueue/wq_monitor.py
+++ b/tools/workqueue/wq_monitor.py
@@ -9,7 +9,9 @@ https://github.com/osandov/drgn.
 
   total    Total number of work items executed by the workqueue.
 
-  infl     The number of currently in-flight work items.
+  queued   The number of work items currently queued and waiting to execute.
+
+  infl     The number of currently executing (in-flight) work items.
 
   CPUtime  Total CPU time consumed by the workqueue in seconds. This is
            sampled from scheduler ticks and only provides ballpark
@@ -55,6 +57,7 @@ workqueues              = prog['workqueues']
 
 WQ_UNBOUND              = prog['WQ_UNBOUND']
 WQ_MEM_RECLAIM          = prog['WQ_MEM_RECLAIM']
+WORK_NR_COLORS          = prog['WORK_NR_COLORS']
 
 PWQ_STAT_STARTED        = prog['PWQ_STAT_STARTED']      # work items started execution
 PWQ_STAT_COMPLETED      = prog['PWQ_STAT_COMPLETED']	# work items completed execution
@@ -72,15 +75,29 @@ class WqStats:
         self.unbound = wq.flags & WQ_UNBOUND != 0
         self.mem_reclaim = wq.flags & WQ_MEM_RECLAIM != 0
         self.stats = [0] * PWQ_NR_STATS
+        self.in_flight = 0
+        try:
+            for i in range(PWQ_NR_STATS):
+                self.stats[i] += int(wq.retired_stats[i])
+        except AttributeError:
+            pass
         for pwq in list_for_each_entry('struct pool_workqueue', wq.pwqs.address_of_(), 'pwqs_node'):
             for i in range(PWQ_NR_STATS):
                 self.stats[i] += int(pwq.stats[i])
+            try:
+                self.in_flight += int(pwq.total_in_flight)
+            except AttributeError:
+                for c in range(WORK_NR_COLORS):
+                    self.in_flight += int(pwq.nr_in_flight[c])
+        self.running = max(self.stats[PWQ_STAT_STARTED] - self.stats[PWQ_STAT_COMPLETED], 0)
+        self.queued = max(self.in_flight - self.running, 0)
 
     def dict(self, now):
         return { 'timestamp'            : now,
                  'name'                 : self.name,
                  'unbound'              : self.unbound,
                  'mem_reclaim'          : self.mem_reclaim,
+                 'queued'               : self.queued,
                  'started'              : self.stats[PWQ_STAT_STARTED],
                  'completed'            : self.stats[PWQ_STAT_COMPLETED],
                  'cpu_time'             : self.stats[PWQ_STAT_CPU_TIME],
@@ -91,7 +108,7 @@ class WqStats:
                  'rescued'              : self.stats[PWQ_STAT_RESCUED], }
 
     def table_header_str():
-        return f'{"":>24} {"total":>8} {"infl":>5} {"CPUtime":>8} '\
+        return f'{"":>24} {"total":>8} {"queued":>6} {"infl":>5} {"CPUtime":>8} '\
             f'{"CPUitsv":>7} {"CMW/RPR":>7} {"mayday":>7} {"rescued":>7}'
 
     def table_row_str(self):
@@ -112,7 +129,8 @@ class WqStats:
 
         out = f'{self.name[-24:]:24} ' \
               f'{self.stats[PWQ_STAT_STARTED]:8} ' \
-              f'{max(self.stats[PWQ_STAT_STARTED] - self.stats[PWQ_STAT_COMPLETED], 0):5} ' \
+              f'{self.queued:6} ' \
+              f'{self.running:5} ' \
               f'{self.stats[PWQ_STAT_CPU_TIME] / 1000000:8.1f} ' \
               f'{cpu_intensive:>7} ' \
               f'{cmw_rpr:>7} ' \
-- 
2.56.0.218.g1ec904b173-goog

Re: [PATCH 2/3] workqueue: expose per-workqueue stats in debugfs

[Tejun Heo]

On Thu, Oct 22, 2026 at 03:53:35PM +0000, Wilson Felipe Pereira wrote:
> struct pool_workqueue tracks runtime counters in pwq->stats[],
> pwq->total_in_flight, and pwq->nr_active, but reading them currently
> requires tools/workqueue/wq_monitor.py with drgn and kernel DWARF
> debuginfo. This makes it impractical to inspect workqueue stats on
> systems without drgn or debug symbols.
> 
> Add /sys/kernel/debug/workqueue/stats to expose per-workqueue stats
> directly from the kernel. Iterate over the workqueues list under RCU and
> sum wq->retired_stats[] with each pwq's stats[], total_in_flight, and
> nr_active to report queued, active, and running work items alongside the
> cumulative pwq->stats[] counters.

This isn't really a justification. What do you use these numbers for? If
this is for live monitoring, why debugfs? If for debugging, what's wrong
with drgn?

Thanks.

-- 
tejun