[PATCH 4/4] workqueue: Detect stalled in-flight work items with empty worklist

[Breno Leitao <leitao@debian.org>]

The workqueue watchdog skips pools with an empty worklist, assuming no
work is pending. However, a single work item that was dequeued and is
now executing on a worker will leave the worklist empty while the worker
is stuck. This means a pool with one hogged worker and no pending work
is invisible to the watchdog.

An example is something like:

  static void stall_work_fn(struct work_struct *work)
  {
        for (;;) {
                mdelay(1000);
                cond_resched();
        }
  }

Fix this by scanning the pool's busy_hash for workers whose
current_start timestamp exceeds the watchdog threshold, independent of
worklist state. The new report_stalled_workers() function iterates all
in-flight workers in a pool and reports each one that has exceeded the
threshold, running as a separate detection path alongside the existing
pool-level last_progress_ts check.

This is an example of the report:

  BUG: workqueue lockup - worker 365:stall_work_fn [wq_stall] stuck in pool cpus=9 node=0 flags=0x0 nice=0 for 33s!
   Showing busy workqueues and worker pools:
   ...

The feature is gated behind a new CONFIG_WQ_WATCHDOG_WORKERS option
(disabled by default) under CONFIG_WQ_WATCHDOG.

Signed-off-by: Breno Leitao <leitao@debian.org>
---
 kernel/workqueue.c | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++--
 lib/Kconfig.debug  | 12 ++++++++++++
 2 files changed, 62 insertions(+), 2 deletions(-)

diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index e527e763162e6..719e14aa4ac56 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -7659,6 +7659,49 @@ static void wq_watchdog_reset_touched(void)
 		per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
 }
 
+#ifdef CONFIG_WQ_WATCHDOG_WORKERS
+/*
+ * Scan all in-flight workers in @pool for stalls. A worker is considered
+ * stalled if its current work item has been executing for longer than @thresh
+ * based on its current_start timestamp. This catches workers that are stuck
+ * regardless of the pool's worklist state or last_progress_ts.
+ */
+static bool report_stalled_workers(struct worker_pool *pool,
+				   unsigned long now,
+				   unsigned long thresh)
+{
+	struct worker *worker;
+	bool stall = false;
+	int bkt;
+
+	/*
+	 * Iterate busy_hash without pool->lock. This is intentionally
+	 * lockless to avoid contention in the watchdog timer path.
+	 * Workers that have been stalled for thresh (typically 30s) are
+	 * unlikely to be transitioning in/out of busy_hash concurrently.
+	 */
+	hash_for_each(pool->busy_hash, bkt, worker, hentry) {
+		if (time_after(now, worker->current_start + thresh)) {
+			pr_emerg("BUG: workqueue lockup - worker ");
+			pr_cont_worker_id(worker);
+			pr_cont(":%ps stuck in pool",
+				worker->current_func);
+			pr_cont_pool_info(pool);
+			pr_cont(" for %us!\n",
+				jiffies_to_msecs(now - worker->current_start) / 1000);
+			stall = true;
+		}
+	}
+	return stall;
+}
+#else
+static bool report_stalled_workers(struct worker_pool *pool,
+				   unsigned long now, unsigned long thresh)
+{
+	return false;
+}
+#endif /* CONFIG_WQ_WATCHDOG_WORKERS */
+
 static void wq_watchdog_timer_fn(struct timer_list *unused)
 {
 	unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ;
@@ -7677,8 +7720,6 @@ static void wq_watchdog_timer_fn(struct timer_list *unused)
 		unsigned long pool_ts, touched, ts;
 
 		pool->cpu_stall = false;
-		if (list_empty(&pool->worklist))
-			continue;
 
 		/*
 		 * If a virtual machine is stopped by the host it can look to
@@ -7686,6 +7727,13 @@ static void wq_watchdog_timer_fn(struct timer_list *unused)
 		 */
 		kvm_check_and_clear_guest_paused();
 
+		/* Check for individual stalled workers in this pool. */
+		if (report_stalled_workers(pool, now, thresh))
+			lockup_detected = true;
+
+		if (list_empty(&pool->worklist))
+			continue;
+
 		/* get the latest of pool and touched timestamps */
 		if (pool->cpu >= 0)
 			touched = READ_ONCE(per_cpu(wq_watchdog_touched_cpu, pool->cpu));
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index ce25a8faf6e9e..dc4bb546b2033 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1320,6 +1320,18 @@ config BOOTPARAM_WQ_STALL_PANIC
 	  This setting can be overridden at runtime via the
 	  workqueue.panic_on_stall kernel parameter.
 
+config WQ_WATCHDOG_WORKERS
+	bool "Detect individual stalled workqueue workers"
+	depends on WQ_WATCHDOG
+	default n
+	help
+	  Say Y here to enable per-worker stall detection. When enabled,
+	  the workqueue watchdog scans all in-flight workers in each pool
+	  and reports any whose current work item has been executing for
+	  longer than the watchdog threshold. This catches stalled workers
+	  even when the pool's worklist is empty or the pool has recently
+	  made forward progress on other work items.
+
 config WQ_CPU_INTENSIVE_REPORT
 	bool "Report per-cpu work items which hog CPU for too long"
 	depends on DEBUG_KERNEL

-- 
2.47.3