[PATCH v5] hung_task: Deduplicate identical hang reports using explicit blocker tracking

[Aaron Tomlin <atomlin@atomlin.com>]

Currently, during severe lock contention, multiple tasks can hang while
waiting on the exact same resource. The khungtaskd kthread
indiscriminately reports every single instance with a stack trace.
This can roll the kernel ring buffer and prematurely exhaust the
kernel.hung_task_warnings budget. Consequently, the kernel is left
entirely blind to subsequent, unrelated deadlocks.

To preserve the warning budget and ring buffer without sacrificing
observability, this patch introduces a two-tier deduplication mechanism:

    1. Introduces a hung_task_reported in task_struct. If a task remains
       hung across multiple check intervals, khungtaskd suppresses
       redundant stack traces for that specific task until it makes
       progress (verified via context switch counters). Furthermore, it
       is packed into an existing compiler alignment hole, consuming
       zero additional memory

    2. For tasks blocked on a lock, we leverage the
       CONFIG_DETECT_HUNG_TASK_BLOCKER infrastructure. By tracking the
       exact memory address of the blocker lock in a persistent array of
       size 32, we deduplicate warning reports for tasks waiting on the
       same resource across check intervals. If the blocker is already
       tracked, the warning is suppressed and the warning budget is
       preserved

    3. For duplicate tasks, we still print the single-line "INFO: task
       ..." message and trigger tracepoint trace_sched_process_hang().
       It merely skips calling sched_show_task() and
       debug_show_blocker(), printing a concise suppression notice
       instead

    4. Once the hang resolves (i.e. no hung tasks are detected in a
       check round), the budget is restored to the value captured at the
       start of the hang, and the blocker array is cleared, accompanied
       by a recovery message in the ring buffer

Signed-off-by: Aaron Tomlin <atomlin@atomlin.com>
---
Changes since v4:

 - Replaced the stack-local hashmap implementation with a persistent
   array (Petr Mladek)

 - Persistently track blocker addresses across scan intervals. Suppress
   warning reports and keep the sysctl_hung_task_warnings budget intact
   if the blocker is already tracked in the array (Petr Mladek)

 - Reset the warnings budget and clear the blocker array when the hang
   resolves (Petr Mladek)

 - Output a recovery message to the kernel ring buffer upon hang
   resolution

 - Linked to v4: https://lore.kernel.org/lkml/20260627205733.90983-1-atomlin@atomlin.com/

Changes since v3:

 - Deduct from the global budget if printing a full stack trace

 - Pivoted from heuristic Wait Channel hashing to deterministic
   blocker address hashing via CONFIG_DETECT_HUNG_TASK_BLOCKER

 - Replaced the hung_task_reported bit-field with a standalone u8 byte.
   Move hung_task_reported into an existing structural alignment hole
   within task_struct following blocked_lock, resulting in zero overall
   memory footprint increase and optimal cacheline grouping

 - Linked to v3: https://lore.kernel.org/lkml/20260621213756.43225-1-atomlin@atomlin.com/

Changes since v2:

 - Replaced the per-round cache flush with a task_struct bit-field for
   persistent cross-scan tracking, mitigating delayed budget exhaustion

 - Abandoned exact-stack hashing in favour of Wait Channel hashing

 - Transitioned from jhash() to hash_long() to optimise single-pointer
   hashing, and relocated the hash map to the local stack

 - Linked to v2: https://lore.kernel.org/lkml/20260620013559.1537893-1-atomlin@atomlin.com/

Changes since v1:

 - Preserve "INFO:" headers for all hung tasks; suppress only the stack
   dumps for duplicates (Masami Hiramatsu)

 - Print a clear notification when a trace is explicitly suppressed

 - Add #ifdef CONFIG_STACKTRACE guards to prevent Kconfig build errors

 - Optimise overhead by unwinding the stack only if a warning is
   actually going to be printed

 - Linked to v1: https://lore.kernel.org/lkml/20260617184841.1447955-1-atomlin@atomlin.com/
---
 include/linux/hung_task.h |   1 +
 include/linux/sched.h     |   3 +
 kernel/fork.c             |   1 +
 kernel/hung_task.c        | 119 ++++++++++++++++++++++++++++++++++++--
 4 files changed, 118 insertions(+), 6 deletions(-)

diff --git a/include/linux/hung_task.h b/include/linux/hung_task.h
index c4403eeb7144..b19220613402 100644
--- a/include/linux/hung_task.h
+++ b/include/linux/hung_task.h
@@ -37,6 +37,7 @@
 
 #define BLOCKER_TYPE_MASK		0x03UL
 
+
 #ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
 static inline void hung_task_set_blocker(void *lock, unsigned long type)
 {
diff --git a/include/linux/sched.h b/include/linux/sched.h
index b3204a15d512..deb092ff8a9a 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1252,6 +1252,9 @@ struct task_struct {
 	struct mutex			*blocked_on;	/* lock we're blocked on */
 	raw_spinlock_t			blocked_lock;
 
+#ifdef CONFIG_DETECT_HUNG_TASK
+	u8				hung_task_reported;
+#endif
 	/*
 	 * The task that is boosting this task; a back link for the current
 	 * donor stack. Set in schedule() -> find_proxy_task() and only stable
diff --git a/kernel/fork.c b/kernel/fork.c
index 892a95214c54..8a5c08cc42d5 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1569,6 +1569,7 @@ static int copy_mm(u64 clone_flags, struct task_struct *tsk)
 #ifdef CONFIG_DETECT_HUNG_TASK
 	tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;
 	tsk->last_switch_time = 0;
+	tsk->hung_task_reported = 0;
 #endif
 
 	tsk->mm = NULL;
diff --git a/kernel/hung_task.c b/kernel/hung_task.c
index 6fcc94ce4ca9..17c285e83bb7 100644
--- a/kernel/hung_task.c
+++ b/kernel/hung_task.c
@@ -25,6 +25,8 @@
 #include <linux/hung_task.h>
 #include <linux/rwsem.h>
 #include <linux/sys_info.h>
+#include <linux/hash.h>
+#include <linux/array_size.h>
 
 #include <trace/events/sched.h>
 
@@ -59,6 +61,16 @@ static unsigned long __read_mostly sysctl_hung_task_check_interval_secs;
 
 static int __read_mostly sysctl_hung_task_warnings = 10;
 
+#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
+#define HUNG_TASK_BLOCKERS_MAX 32
+static unsigned long hung_task_blockers[HUNG_TASK_BLOCKERS_MAX];
+static int hung_task_blockers_count;
+static bool hung_task_has_active;
+static int warnings_decremented;
+static_assert(HUNG_TASK_BLOCKERS_MAX <= BITS_PER_LONG,
+	      "HUNG_TASK_BLOCKERS_MAX exceeds BITS_PER_LONG");
+#endif
+
 static int __read_mostly did_panic;
 static bool hung_task_call_panic;
 
@@ -125,6 +137,7 @@ static bool task_is_hung(struct task_struct *t, unsigned long timeout)
 	if (switch_count != t->last_switch_count) {
 		t->last_switch_count = switch_count;
 		t->last_switch_time = jiffies;
+		t->hung_task_reported = 0;
 		return false;
 	}
 	if (time_is_after_jiffies(t->last_switch_time + timeout * HZ))
@@ -228,12 +241,14 @@ static inline void debug_show_blocker(struct task_struct *task, unsigned long ti
  * @t: Pointer to the detected hung task.
  * @timeout: Timeout threshold for detecting hung tasks
  * @this_round_count: Count of hung tasks detected in the current iteration
+ * @skip_show_task: Indicating if stack trace should be skipped
  *
  * Print structured information about the specified hung task, if warnings
  * are enabled or if the panic batch threshold is exceeded.
  */
 static void hung_task_info(struct task_struct *t, unsigned long timeout,
-			   unsigned long this_round_count)
+			   unsigned long this_round_count,
+			   unsigned int skip_show_task)
 {
 	trace_sched_process_hang(t);
 
@@ -248,8 +263,16 @@ static void hung_task_info(struct task_struct *t, unsigned long timeout,
 	 * accordingly
 	 */
 	if (sysctl_hung_task_warnings || hung_task_call_panic) {
-		if (sysctl_hung_task_warnings > 0)
+		/*
+		 * Do not exhaust the global warning budget for duplicates;
+		 * only decrement if a full stack trace is being printed.
+		 */
+		if (!skip_show_task && sysctl_hung_task_warnings > 0) {
 			sysctl_hung_task_warnings--;
+#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
+			warnings_decremented++;
+#endif
+		}
 		pr_err("INFO: task %s:%d blocked%s for more than %ld seconds.\n",
 		       t->comm, t->pid, t->in_iowait ? " in I/O wait" : "",
 		       (jiffies - t->last_switch_time) / HZ);
@@ -261,8 +284,12 @@ static void hung_task_info(struct task_struct *t, unsigned long timeout,
 			pr_err("      Blocked by coredump.\n");
 		pr_err("\"echo 0 > /proc/sys/kernel/hung_task_timeout_secs\""
 			" disables this message.\n");
-		sched_show_task(t);
-		debug_show_blocker(t, timeout);
+		if (!skip_show_task) {
+			sched_show_task(t);
+			debug_show_blocker(t, timeout);
+		} else {
+			pr_err("      Stack trace suppressed. Already reported or duplicate\n");
+		}
 
 		if (!sysctl_hung_task_warnings)
 			pr_info("Future hung task reports are suppressed, see sysctl kernel.hung_task_warnings\n");
@@ -306,6 +333,13 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
 	unsigned long this_round_count;
 	int need_warning = sysctl_hung_task_warnings;
 	unsigned long si_mask = hung_task_si_mask;
+	unsigned int skip_show_task;
+	bool scan_completed = false;
+#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
+	unsigned long blocker;
+	static int warnings_reset_value;
+	unsigned long seen_blockers_mask = 0;
+#endif
 
 	/*
 	 * If the system crashed already then all bets are off,
@@ -326,6 +360,7 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
 		}
 
 		if (task_is_hung(t, timeout)) {
+			skip_show_task = t->hung_task_reported;
 			/*
 			 * Increment the global counter so that userspace could
 			 * start migrating tasks ASAP. But count the current
@@ -334,14 +369,86 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
 			 */
 			atomic_long_inc(&sysctl_hung_task_detect_count);
 			this_round_count++;
-			hung_task_info(t, timeout, this_round_count);
+
+#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
+			if (!hung_task_has_active) {
+				warnings_reset_value = sysctl_hung_task_warnings;
+				warnings_decremented = 0;
+				hung_task_has_active = true;
+			}
+			blocker = READ_ONCE(t->blocker);
+			if (blocker) {
+				bool found = false;
+				int i, blocker_idx = -1;
+
+				blocker &= ~BLOCKER_TYPE_MASK;
+				for (i = 0; i < hung_task_blockers_count; i++) {
+					if (hung_task_blockers[i] == blocker) {
+						found = true;
+						blocker_idx = i;
+						break;
+					}
+				}
+				if (found || t->hung_task_reported) {
+					skip_show_task = 1;
+					if (found)
+						seen_blockers_mask |= (1UL << blocker_idx);
+				} else {
+					skip_show_task = 0;
+					if (hung_task_blockers_count < ARRAY_SIZE(hung_task_blockers)) {
+						blocker_idx = hung_task_blockers_count;
+						hung_task_blockers[hung_task_blockers_count++] = blocker;
+						seen_blockers_mask |= (1UL << blocker_idx);
+					} else {
+						pr_warn_once("INFO: tracked blocker array full. Untracked blockers may trigger duplicate warnings\n");
+					}
+				}
+			}
+#endif
+
+			hung_task_info(t, timeout, this_round_count,
+				       skip_show_task);
+			t->hung_task_reported = 1;
 		}
 	}
+	scan_completed = true;
  unlock:
 	rcu_read_unlock();
 
-	if (!this_round_count)
+	if (!this_round_count) {
+#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
+		if (scan_completed && hung_task_has_active) {
+			if (sysctl_hung_task_warnings == warnings_reset_value - warnings_decremented) {
+				pr_info("INFO: hung tasks resolved. Cleared %d tracked blocker(s). Warning budget restored to %d\n",
+					hung_task_blockers_count, warnings_reset_value);
+				sysctl_hung_task_warnings = warnings_reset_value;
+			} else {
+				pr_info("INFO: hung tasks resolved. Cleared %d tracked blocker(s). Warning budget manual change retained at %d\n",
+					hung_task_blockers_count, sysctl_hung_task_warnings);
+			}
+			for (int i = 0; i < ARRAY_SIZE(hung_task_blockers); i++)
+				hung_task_blockers[i] = 0;
+			hung_task_blockers_count = 0;
+			hung_task_has_active = false;
+		}
+#endif
 		return;
+	}
+
+#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
+	if (scan_completed) {
+		int i, write_idx = 0;
+		for (i = 0; i < hung_task_blockers_count; i++) {
+			if (seen_blockers_mask & (1UL << i)) {
+				hung_task_blockers[write_idx++] = hung_task_blockers[i];
+			}
+		}
+		for (i = write_idx; i < hung_task_blockers_count; i++) {
+			hung_task_blockers[i] = 0;
+		}
+		hung_task_blockers_count = write_idx;
+	}
+#endif
 
 	if (need_warning || hung_task_call_panic) {
 		si_mask |= SYS_INFO_LOCKS;
-- 
2.54.0