[PATCH 0/3] workqueue: Add telemetry tracepoints for CPU hogs, distress, and BH budget yields

[PATCH 0/3] workqueue: Add telemetry tracepoints for CPU hogs, distress, and BH budget yields

[Aaron Tomlin]

Hi Tejun, Lai,

While the workqueue subsystem maintains rich internal telemetry via
pwq->stats[], wq_cpu_intensive_thresh_us, and distress mechanisms, several
critical state transitions and execution anomalies currently lack real-time
event notifications.

Across production fleets and low-latency networking workloads, polling
pwq->stats[] or running drgn scripts is impractical for detecting
intermittent stalls. Tail-latency spikes and packet drops often stem from
softirq overruns or latency-critical work items queueing behind CPU-bound
tasks. These event-driven tracepoints allow zero-overhead eBPF tools and
latency profilers to capture stack traces and kernel context at the exact
moment a starvation event or softirq budget exhaustion occurs.

This patch series introduces lightweight tracepoints for these key
operational boundaries:

Patch 1 adds workqueue_cpu_intensive tracepoint. When a concurrency-managed
worker runs for longer than wq_cpu_intensive_thresh_us without sleeping,
wq_worker_tick() marks it as WORKER_CPU_INTENSIVE and kicks the pool to
prevent queue starvation. The tracepoint will capture the offending work
function, workqueue name, CPU, and elapsed duration.

Patch 2 adds workqueue_mayday and workqueue_rescued tracepoints. One when
worker allocation stalls trigger mayday distress, and another when pending
work items are handed off to the rescuer thread to ensure forward progress.

Patch 3 adds workqueue_bh_budget_yield tracepoint. Bottom-Half (BH)
workqueues enforce execution limits in softirq context (BH_WORKER_JIFFIES
and BH_WORKER_RESTARTS). When a BH worker hits these limits while pending
work remains, it yields and re-raises the softirq. The tracepoint can be
used to identify softirq budget saturation and track whether yielding
occurred due to time slice expiration or restart counts.

Aaron Tomlin (3):
  workqueue: Add workqueue_cpu_intensive tracepoint
  workqueue: Add workqueue_mayday and workqueue_rescued tracepoints
  workqueue: Add workqueue_bh_budget_yield tracepoint

 include/trace/events/workqueue.h | 146 +++++++++++++++++++++++++++++++
 kernel/workqueue.c               |  24 ++++-
 2 files changed, 168 insertions(+), 2 deletions(-)

-- 
2.55.0

[PATCH 1/3] workqueue: Add workqueue_cpu_intensive tracepoint

[Aaron Tomlin]

When a concurrency-managed per-CPU work item runs continuously without
sleeping for longer than wq_cpu_intensive_thresh_us, wq_worker_tick() marks
the worker as WORKER_CPU_INTENSIVE and kicks it out of concurrency
management so that pending work items on the pool are not starved.

While CONFIG_WQ_CPU_INTENSIVE_REPORT logs rate-limited warnings and
pwq->stats[PWQ_STAT_CPU_INTENSIVE] maintains a cumulative counter, there is
currently no tracepoint emitted at the moment of this transition.

Therefore, add the workqueue_cpu_intensive tracepoint, recording the
work_struct pointer and callback function pointer, workqueue name,
executing CPU, and the runtime duration consumed in microseconds.

This enables eBPF profilers, bpftrace, and Ftrace to immediately detect and
attribute CPU-hogging work items in real time.

Signed-off-by: Aaron Tomlin <atomlin@atomlin.com>
---
 include/trace/events/workqueue.h | 39 ++++++++++++++++++++++++++++++++
 kernel/workqueue.c               |  9 ++++++--
 2 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/include/trace/events/workqueue.h b/include/trace/events/workqueue.h
index b0de2bc9ed52..ef0f3a4b73a8 100644
--- a/include/trace/events/workqueue.h
+++ b/include/trace/events/workqueue.h
@@ -126,6 +126,45 @@ TRACE_EVENT(workqueue_execute_end,
 	TP_printk("work struct %p: function %ps", __entry->work, __entry->function)
 );
 
+/**
+ * workqueue_cpu_intensive - called when a work item exceeds cpu_intensive threshold
+ * @pwq:	pointer to struct pool_workqueue
+ * @work:	pointer to struct work_struct
+ * @function:	pointer to worker function
+ * @duration_us: CPU time consumed in microseconds
+ *
+ * This event occurs when a concurrency-managed work item runs for longer
+ * than wq_cpu_intensive_thresh_us without sleeping and is excluded from
+ * concurrency management to prevent stalling other work items.
+ */
+TRACE_EVENT(workqueue_cpu_intensive,
+
+	TP_PROTO(struct pool_workqueue *pwq, struct work_struct *work,
+		 work_func_t function, u64 duration_us),
+
+	TP_ARGS(pwq, work, function, duration_us),
+
+	TP_STRUCT__entry(
+		__field( void *,	work		)
+		__field( void *,	function	)
+		__string( workqueue,	pwq->wq->name	)
+		__field( int,		cpu		)
+		__field( u64,		duration_us	)
+	),
+
+	TP_fast_assign(
+		__entry->work		= work;
+		__entry->function	= function;
+		__assign_str(workqueue);
+		__entry->cpu		= pwq->pool->cpu;
+		__entry->duration_us	= duration_us;
+	),
+
+	TP_printk("work struct=%p function=%ps workqueue=%s cpu=%d duration_us=%llu",
+		  __entry->work, __entry->function, __get_str(workqueue),
+		  __entry->cpu, __entry->duration_us)
+);
+
 #endif /*  _TRACE_WORKQUEUE_H */
 
 /* This part must be outside protection */
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 3c034cbc5bb3..6ee644155b82 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -1532,6 +1532,7 @@ void wq_worker_tick(struct task_struct *task)
 	struct worker *worker = kthread_data(task);
 	struct pool_workqueue *pwq = worker->current_pwq;
 	struct worker_pool *pool = worker->pool;
+	u64 dur;
 
 	if (!pwq)
 		return;
@@ -1557,9 +1558,10 @@ void wq_worker_tick(struct task_struct *task)
 	 * double decrements. The task is releasing the CPU anyway. Let's skip.
 	 * We probably want to make this prettier in the future.
 	 */
+	dur = (READ_ONCE(worker->task->se.sum_exec_runtime) - worker->current_at) /
+	      NSEC_PER_USEC;
 	if ((worker->flags & WORKER_NOT_RUNNING) || READ_ONCE(worker->sleeping) ||
-	    READ_ONCE(worker->task->se.sum_exec_runtime) - worker->current_at <
-	    wq_cpu_intensive_thresh_us * NSEC_PER_USEC)
+	    dur < wq_cpu_intensive_thresh_us)
 		return;
 
 	raw_spin_lock(&pool->lock);
@@ -1572,6 +1574,9 @@ void wq_worker_tick(struct task_struct *task)
 		pwq->stats[PWQ_STAT_CM_WAKEUP]++;
 
 	raw_spin_unlock(&pool->lock);
+
+	trace_workqueue_cpu_intensive(pwq, worker->current_work,
+				      worker->current_func, dur);
 }
 
 /**
-- 
2.55.0

[PATCH 2/3] workqueue: Add workqueue_mayday and workqueue_rescued tracepoints

[Aaron Tomlin]

When a worker pool fails to create a new worker thread within
MAYDAY_INTERVAL (e.g., typically under severe memory pressure where
memory reclaim paths depend on pending work items), send_mayday()
signals distress to the workqueue's rescuer thread. The rescuer then
takes over processing the pending work items via assign_rescuer_work().

While pwq->stats[PWQ_STAT_MAYDAY] and pwq->stats[PWQ_STAT_RESCUED] track
these occurrences cumulatively, there is currently no event-driven
mechanism to observe exactly when mayday distress occurs or which work
items require rescue.

Add two new tracepoints namely workqueue_mayday and workqueue_rescued to
make distress and rescuer execution easily observable (e.g., via Ftrace
or eBPF).

Signed-off-by: Aaron Tomlin <atomlin@atomlin.com>
---
 include/trace/events/workqueue.h | 68 ++++++++++++++++++++++++++++++++
 kernel/workqueue.c               |  2 +
 2 files changed, 70 insertions(+)

diff --git a/include/trace/events/workqueue.h b/include/trace/events/workqueue.h
index ef0f3a4b73a8..013cfa472f6d 100644
--- a/include/trace/events/workqueue.h
+++ b/include/trace/events/workqueue.h
@@ -165,6 +165,74 @@ TRACE_EVENT(workqueue_cpu_intensive,
 		  __entry->cpu, __entry->duration_us)
 );
 
+/**
+ * workqueue_mayday - called when a pool_workqueue sends mayday to rescuer
+ * @pwq:	pointer to struct pool_workqueue
+ *
+ * This event occurs when a worker pool fails to create a new worker
+ * within MAYDAY_INTERVAL and requests the workqueue's rescuer thread to
+ * process pending works.
+ */
+TRACE_EVENT(workqueue_mayday,
+
+	TP_PROTO(struct pool_workqueue *pwq),
+
+	TP_ARGS(pwq),
+
+	TP_STRUCT__entry(
+		__string( workqueue,	pwq->wq->name	)
+		__field( int,		pool_id		)
+		__field( int,		cpu		)
+		__field( int,		nr_active	)
+	),
+
+	TP_fast_assign(
+		__assign_str(workqueue);
+		__entry->pool_id	= pwq->pool->id;
+		__entry->cpu		= pwq->pool->cpu;
+		__entry->nr_active	= pwq->nr_active;
+	),
+
+	TP_printk("workqueue=%s pool_id=%d cpu=%d nr_active=%d",
+		  __get_str(workqueue), __entry->pool_id, __entry->cpu,
+		  __entry->nr_active)
+);
+
+/**
+ * workqueue_rescued - called when a work item is assigned to a rescuer
+ * @pwq:	pointer to struct pool_workqueue
+ * @work:	pointer to struct work_struct
+ * @function:	pointer to worker function
+ *
+ * This event occurs when a work item is claimed by a rescuer thread
+ * to guarantee forward progress.
+ */
+TRACE_EVENT(workqueue_rescued,
+
+	TP_PROTO(struct pool_workqueue *pwq, struct work_struct *work,
+		 work_func_t function),
+
+	TP_ARGS(pwq, work, function),
+
+	TP_STRUCT__entry(
+		__field( void *,	work		)
+		__field( void *,	function	)
+		__string( workqueue,	pwq->wq->name	)
+		__field( int,		cpu		)
+	),
+
+	TP_fast_assign(
+		__entry->work		= work;
+		__entry->function	= function;
+		__assign_str(workqueue);
+		__entry->cpu		= pwq->pool->cpu;
+	),
+
+	TP_printk("work struct=%p function=%ps workqueue=%s cpu=%d",
+		  __entry->work, __entry->function, __get_str(workqueue),
+		  __entry->cpu)
+);
+
 #endif /*  _TRACE_WORKQUEUE_H */
 
 /* This part must be outside protection */
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 6ee644155b82..6f6fe2068389 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -3122,6 +3122,7 @@ static void send_mayday(struct pool_workqueue *pwq)
 		list_add_tail(&pwq->mayday_node, &wq->maydays);
 		wake_up_process(wq->rescuer->task);
 		pwq->stats[PWQ_STAT_MAYDAY]++;
+		trace_workqueue_mayday(pwq);
 	}
 }
 
@@ -3619,6 +3620,7 @@ static bool assign_rescuer_work(struct pool_workqueue *pwq, struct worker *rescu
 	list_for_each_entry_safe_from(work, n, &pool->worklist, entry) {
 		if (get_work_pwq(work) == pwq && assign_work(work, rescuer, &n)) {
 			pwq->stats[PWQ_STAT_RESCUED]++;
+			trace_workqueue_rescued(pwq, work, work->func);
 			/* put the cursor for next search */
 			list_move_tail(&cursor->entry, &n->entry);
 			return true;
-- 
2.55.0

[PATCH 3/3] workqueue: Add workqueue_bh_budget_yield tracepoint

[Aaron Tomlin]

Bottom-Half (BH) workqueues execute work items in softirq context.
To prevent softirqs from starving user and kernel threads, bh_worker()
enforces execution limits (i.e., BH_WORKER_JIFFIES and BH_WORKER_RESTARTS).

When keep_working() is still true but either the time slice or restart
count is exhausted, bh_worker() yields execution and re-raises the
softirq via kick_bh_pool().

Currently, there is no observability into when a BH worker hits these
limits and is forced to yield.

Add the workqueue_bh_budget_yield tracepoint, emitted when bh_worker()
exits the processing loop with pending work items remaining. It records,
the worker pool ID, executing CPU, number of loop restarts consumed, a
boolean flag indicating whether the yield was due to a time slice
timeout, and a boolean flag indicating whether this is a high-priority
BH pool.

Signed-off-by: Aaron Tomlin <atomlin@atomlin.com>
---
 include/trace/events/workqueue.h | 39 ++++++++++++++++++++++++++++++++
 kernel/workqueue.c               | 13 +++++++++++
 2 files changed, 52 insertions(+)

diff --git a/include/trace/events/workqueue.h b/include/trace/events/workqueue.h
index 013cfa472f6d..ad1d0ff1a96d 100644
--- a/include/trace/events/workqueue.h
+++ b/include/trace/events/workqueue.h
@@ -9,6 +9,7 @@
 #include <linux/workqueue.h>
 
 struct pool_workqueue;
+struct worker_pool;
 
 /**
  * workqueue_queue_work - called when a work gets queued
@@ -233,6 +234,44 @@ TRACE_EVENT(workqueue_rescued,
 		  __entry->cpu)
 );
 
+/**
+ * workqueue_bh_budget_yield - called when a BH worker yields due to budget exhaustion
+ * @pool:	pointer to struct worker_pool
+ * @restarts:	number of restarts executed
+ * @timeout:	whether execution hit the time limit (BH_WORKER_JIFFIES)
+ * @highpri:	whether this is a high-priority BH pool
+ *
+ * This event occurs when a bottom-half (BH) worker pool running in softirq
+ * context exhausts its execution time slice or restart limit and must yield
+ * execution.
+ */
+TRACE_EVENT(workqueue_bh_budget_yield,
+
+	TP_PROTO(struct worker_pool *pool, int restarts, bool timeout, bool highpri),
+
+	TP_ARGS(pool, restarts, timeout, highpri),
+
+	TP_STRUCT__entry(
+		__field( int,	pool_id		)
+		__field( int,	cpu		)
+		__field( int,	restarts	)
+		__field( bool,	timeout		)
+		__field( bool,	highpri		)
+	),
+
+	TP_fast_assign(
+		__entry->pool_id	= pool->id;
+		__entry->cpu		= pool->cpu;
+		__entry->restarts	= restarts;
+		__entry->timeout	= timeout;
+		__entry->highpri	= highpri;
+	),
+
+	TP_printk("pool_id=%d cpu=%d restarts=%d timeout=%d highpri=%d",
+		  __entry->pool_id, __entry->cpu, __entry->restarts,
+		  __entry->timeout, __entry->highpri)
+);
+
 #endif /*  _TRACE_WORKQUEUE_H */
 
 /* This part must be outside protection */
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 6f6fe2068389..1c25df68f7ae 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -3756,6 +3756,9 @@ static void bh_worker(struct worker *worker)
 	struct worker_pool *pool = worker->pool;
 	int nr_restarts = BH_WORKER_RESTARTS;
 	unsigned long end = jiffies + BH_WORKER_JIFFIES;
+	bool budget_exhausted = false;
+	bool timeout = false;
+	int executed_restarts = 0;
 
 	worker_lock_callback(pool);
 	raw_spin_lock_irq(&pool->lock);
@@ -3781,12 +3784,22 @@ static void bh_worker(struct worker *worker)
 	} while (keep_working(pool) &&
 		 --nr_restarts && time_before(jiffies, end));
 
+	if (keep_working(pool)) {
+		budget_exhausted = true;
+		timeout = !time_before(jiffies, end);
+		executed_restarts = BH_WORKER_RESTARTS - nr_restarts;
+	}
+
 	worker_set_flags(worker, WORKER_PREP);
 done:
 	worker_enter_idle(worker);
 	kick_pool(pool);
 	raw_spin_unlock_irq(&pool->lock);
 	worker_unlock_callback(pool);
+
+	if (budget_exhausted)
+		trace_workqueue_bh_budget_yield(pool, executed_restarts, timeout,
+						pool->attrs->nice == HIGHPRI_NICE_LEVEL);
 }
 
 /*
-- 
2.55.0