[PATCH] cgroup: avoid flushing global workqueue in cgroup1_pidlist_destroy_all

[PATCH] cgroup: avoid flushing global workqueue in cgroup1_pidlist_destroy_all

[Junnan Zhang]

From: Junnan Zhang <zhangjn11@chinatelecom.cn>

cgroup1_pidlist_destroy_all() flushes the global
cgroup_pidlist_destroy_wq while destroying a cgroup. Because all cgroup
v1 pidlist destruction work items are queued on the same shared workqueue,
a single slow or stuck work item (e.g. waiting for pidlist_mutex held by a
user-space reader) blocks every concurrent cgroup destruction path.

This can lead to kworker tasks stuck in flush_workqueue() for over
hung_task_timeout seconds, as observed on busy systems running Docker or
Kubernetes workloads.

  INFO: task kworker/0:1:1438499 blocked for more than 120 seconds.
  "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
  kworker/0:1 D 0 1438499 2 0x80000080
  Workqueue: cgroup_destroy css_free_rwork_fn
  ? __schedule+0x296/0x900
  schedule+0x28/0x80
  schedule_timeout+0x1ee/0x3a0
  ? kvm_sched_clock_read+0xd/0x20
  wait_for_completion+0x12c/0x190
  ? wake_up_q+0x70/0x70
  flush_workqueue+0x132/0x430
  ? cgroup1_pidlist_destroy_all+0x7c/0xa0
  cgroup1_pidlist_destroy_all+0x7c/0xa0
  css_free_rwork_fn+0xb5/0x390
  process_one_work+0x195/0x3e0
  worker_thread+0x30/0x390
  ? process_one_work+0x3e0/0x3e0
  kthread+0x113/0x130
  ? kthread_create_worker_on_cpu+0x70/0x70
  ret_from_fork+0x1f/0x40

Fix it by moving the cgroup's pidlists to a local orphan list under
pidlist_mutex, clearing their ->owner pointer, and then cancelling each
pidlist's delayed work outside the lock. The destroy work function now
checks ->owner and skips freeing orphaned pidlists, so
cgroup1_pidlist_destroy_all() can free them safely without flushing the
whole shared workqueue.

Signed-off-by: Junnan Zhang <zhangjn11@chinatelecom.cn>
Signed-off-by: Shouxin Sun <sunshx@chinatelecom.cn>
---
 kernel/cgroup/cgroup-v1.c | 53 ++++++++++++++++++++++++++++-----------
 1 file changed, 38 insertions(+), 15 deletions(-)

diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c
index a4337c9b5287..874fe4dc6ffd 100644
--- a/kernel/cgroup/cgroup-v1.c
+++ b/kernel/cgroup/cgroup-v1.c
@@ -206,13 +206,31 @@ struct cgroup_pidlist {
 void cgroup1_pidlist_destroy_all(struct cgroup *cgrp)
 {
 	struct cgroup_pidlist *l, *tmp_l;
+	LIST_HEAD(orphan);
+
+	/*
+	 * Move pidlists to a local orphan list and mark them as owner-less.
+	 * The destroy work function will see ->owner == NULL and skip freeing.
+	 * We then cancel and free them outside pidlist_mutex to avoid
+	 * flush_workqueue() blocking on the shared workqueue.
+	 */
 
 	mutex_lock(&cgrp->pidlist_mutex);
-	list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links)
-		mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, 0);
+	list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links) {
+		list_del(&l->links);
+		l->owner = NULL;
+		list_add(&l->links, &orphan);
+	}
 	mutex_unlock(&cgrp->pidlist_mutex);
 
-	flush_workqueue(cgroup_pidlist_destroy_wq);
+	list_for_each_entry_safe(l, tmp_l, &orphan, links) {
+		list_del(&l->links);
+		cancel_delayed_work_sync(&l->destroy_dwork);
+		kvfree(l->list);
+		put_pid_ns(l->key.ns);
+		kfree(l);
+	}
+
 	BUG_ON(!list_empty(&cgrp->pidlists));
 }
 
@@ -222,21 +240,26 @@ static void cgroup_pidlist_destroy_work_fn(struct work_struct *work)
 	struct cgroup_pidlist *l = container_of(dwork, struct cgroup_pidlist,
 						destroy_dwork);
 	struct cgroup_pidlist *tofree = NULL;
+	struct cgroup *owner;
 
-	mutex_lock(&l->owner->pidlist_mutex);
+	owner = l->owner;
+	if (owner) {
+		mutex_lock(&owner->pidlist_mutex);
 
-	/*
-	 * Destroy iff we didn't get queued again.  The state won't change
-	 * as destroy_dwork can only be queued while locked.
-	 */
-	if (!delayed_work_pending(dwork)) {
-		list_del(&l->links);
-		kvfree(l->list);
-		put_pid_ns(l->key.ns);
-		tofree = l;
-	}
+		/*
+		 * Destroy iff we didn't get queued again and we're still
+		 * owned by the cgroup.  If ->owner was cleared by
+		 * cgroup1_pidlist_destroy_all(), it will free us.
+		 */
+		if (l->owner == owner && !delayed_work_pending(dwork)) {
+			list_del(&l->links);
+			kvfree(l->list);
+			put_pid_ns(l->key.ns);
+			tofree = l;
+		}
 
-	mutex_unlock(&l->owner->pidlist_mutex);
+		mutex_unlock(&l->owner->pidlist_mutex);
+	}
 	kfree(tofree);
 }
 
-- 
2.43.0

[PATCH v2] cgroup: avoid flushing global workqueue in cgroup1_pidlist_destroy_all

[Junnan Zhang]

From: Junnan Zhang <zhangjn11@chinatelecom.cn>

cgroup1_pidlist_destroy_all() flushes the global
cgroup_pidlist_destroy_wq while destroying a cgroup. Because all cgroup
v1 pidlist destruction work items are queued on the same shared workqueue,
a single slow or stuck work item (e.g. waiting for pidlist_mutex held by a
user-space reader) blocks every concurrent cgroup destruction path.

This can lead to kworker tasks stuck in flush_workqueue() for over
hung_task_timeout seconds, as observed on busy systems running Docker or
Kubernetes workloads.

  INFO: task kworker/0:1:1438499 blocked for more than 120 seconds.
  "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
  kworker/0:1 D 0 1438499 2 0x80000080
  Workqueue: cgroup_destroy css_free_rwork_fn
  ? __schedule+0x296/0x900
  schedule+0x28/0x80
  schedule_timeout+0x1ee/0x3a0
  ? kvm_sched_clock_read+0xd/0x20
  wait_for_completion+0x12c/0x190
  ? wake_up_q+0x70/0x70
  flush_workqueue+0x132/0x430
  ? cgroup1_pidlist_destroy_all+0x7c/0xa0
  cgroup1_pidlist_destroy_all+0x7c/0xa0
  css_free_rwork_fn+0xb5/0x390
  process_one_work+0x195/0x3e0
  worker_thread+0x30/0x390
  ? process_one_work+0x3e0/0x3e0
  kthread+0x113/0x130
  ? kthread_create_worker_on_cpu+0x70/0x70
  ret_from_fork+0x1f/0x40

Fix it by moving the cgroup's pidlists to a local orphan list under
pidlist_mutex, clearing their ->owner pointer, and then cancelling each
pidlist's delayed work outside the lock. The destroy work function now
checks ->owner and skips freeing orphaned pidlists, so
cgroup1_pidlist_destroy_all() can free them safely without flushing the
whole shared workqueue.

Signed-off-by: Junnan Zhang <zhangjn11@chinatelecom.cn>
Signed-off-by: Shouxin Sun <sunshx@chinatelecom.cn>

---
v1 -> v2:
 - Fix a NULL pointer dereference in cgroup_pidlist_destroy_work_fn():
   use the cached owner pointer for mutex_unlock(), as ->owner may have
   been cleared concurrently by cgroup1_pidlist_destroy_all(). Spotted
   by Sashiko AI review.
---
 kernel/cgroup/cgroup-v1.c | 53 ++++++++++++++++++++++++++++-----------
 1 file changed, 38 insertions(+), 15 deletions(-)

diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c
index a4337c9b5287..88b10cd59c7b 100644
--- a/kernel/cgroup/cgroup-v1.c
+++ b/kernel/cgroup/cgroup-v1.c
@@ -206,13 +206,31 @@ struct cgroup_pidlist {
 void cgroup1_pidlist_destroy_all(struct cgroup *cgrp)
 {
 	struct cgroup_pidlist *l, *tmp_l;
+	LIST_HEAD(orphan);
+
+	/*
+	 * Move pidlists to a local orphan list and mark them as owner-less.
+	 * The destroy work function will see ->owner == NULL and skip freeing.
+	 * We then cancel and free them outside pidlist_mutex to avoid
+	 * flush_workqueue() blocking on the shared workqueue.
+	 */
 
 	mutex_lock(&cgrp->pidlist_mutex);
-	list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links)
-		mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, 0);
+	list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links) {
+		list_del(&l->links);
+		l->owner = NULL;
+		list_add(&l->links, &orphan);
+	}
 	mutex_unlock(&cgrp->pidlist_mutex);
 
-	flush_workqueue(cgroup_pidlist_destroy_wq);
+	list_for_each_entry_safe(l, tmp_l, &orphan, links) {
+		list_del(&l->links);
+		cancel_delayed_work_sync(&l->destroy_dwork);
+		kvfree(l->list);
+		put_pid_ns(l->key.ns);
+		kfree(l);
+	}
+
 	BUG_ON(!list_empty(&cgrp->pidlists));
 }
 
@@ -222,21 +240,26 @@ static void cgroup_pidlist_destroy_work_fn(struct work_struct *work)
 	struct cgroup_pidlist *l = container_of(dwork, struct cgroup_pidlist,
 						destroy_dwork);
 	struct cgroup_pidlist *tofree = NULL;
+	struct cgroup *owner;
 
-	mutex_lock(&l->owner->pidlist_mutex);
+	owner = l->owner;
+	if (owner) {
+		mutex_lock(&owner->pidlist_mutex);
 
-	/*
-	 * Destroy iff we didn't get queued again.  The state won't change
-	 * as destroy_dwork can only be queued while locked.
-	 */
-	if (!delayed_work_pending(dwork)) {
-		list_del(&l->links);
-		kvfree(l->list);
-		put_pid_ns(l->key.ns);
-		tofree = l;
-	}
+		/*
+		 * Destroy iff we didn't get queued again and we're still
+		 * owned by the cgroup.  If ->owner was cleared by
+		 * cgroup1_pidlist_destroy_all(), it will free us.
+		 */
+		if (l->owner == owner && !delayed_work_pending(dwork)) {
+			list_del(&l->links);
+			kvfree(l->list);
+			put_pid_ns(l->key.ns);
+			tofree = l;
+		}
 
-	mutex_unlock(&l->owner->pidlist_mutex);
+		mutex_unlock(&owner->pidlist_mutex);
+	}
 	kfree(tofree);
 }
 
-- 
2.43.0

[PATCH v3] cgroup: avoid flushing global workqueue in cgroup1_pidlist_destroy_all

[Junnan Zhang]

From: Junnan Zhang <zhangjn11@chinatelecom.cn>

cgroup1_pidlist_destroy_all() flushes the global
cgroup_pidlist_destroy_wq while destroying a cgroup. Because all cgroup
v1 pidlist destruction work items are queued on the same shared workqueue,
a single slow or stuck work item (e.g. waiting for pidlist_mutex held by a
user-space reader) blocks every concurrent cgroup destruction path.

This can lead to kworker tasks stuck in flush_workqueue() for over
hung_task_timeout seconds, as observed on busy systems running Docker or
Kubernetes workloads.

  INFO: task kworker/0:1:1438499 blocked for more than 120 seconds.
  "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
  kworker/0:1 D 0 1438499 2 0x80000080
  Workqueue: cgroup_destroy css_free_rwork_fn
  ? __schedule+0x296/0x900
  schedule+0x28/0x80
  schedule_timeout+0x1ee/0x3a0
  ? kvm_sched_clock_read+0xd/0x20
  wait_for_completion+0x12c/0x190
  ? wake_up_q+0x70/0x70
  flush_workqueue+0x132/0x430
  ? cgroup1_pidlist_destroy_all+0x7c/0xa0
  cgroup1_pidlist_destroy_all+0x7c/0xa0
  css_free_rwork_fn+0xb5/0x390
  process_one_work+0x195/0x3e0
  worker_thread+0x30/0x390
  ? process_one_work+0x3e0/0x3e0
  kthread+0x113/0x130
  ? kthread_create_worker_on_cpu+0x70/0x70
  ret_from_fork+0x1f/0x40

Fix it by moving the cgroup's pidlists to a local orphan list under
pidlist_mutex, clearing their ->owner pointer, and then cancelling each
pidlist's delayed work outside the lock. The destroy work function now
checks ->owner and skips freeing orphaned pidlists, so
cgroup1_pidlist_destroy_all() can free them safely without flushing the
whole shared workqueue.

Signed-off-by: Junnan Zhang <zhangjn11@chinatelecom.cn>
Signed-off-by: Shouxin Sun <sunshx@chinatelecom.cn>

---
v1 -> v2:
 - Fix a NULL pointer dereference in cgroup_pidlist_destroy_work_fn():
   use the cached owner pointer for mutex_unlock(), as ->owner may have
   been cleared concurrently by cgroup1_pidlist_destroy_all(). Spotted
   by Sashiko AI review.

v2 -> v3:
 - Use READ_ONCE()/WRITE_ONCE() for the ->owner field shared between
   cgroup1_pidlist_destroy_all() and cgroup_pidlist_destroy_work_fn() to
   prevent data races. Spotted by Sashiko AI review.
---
 kernel/cgroup/cgroup-v1.c | 53 ++++++++++++++++++++++++++++-----------
 1 file changed, 38 insertions(+), 15 deletions(-)

diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c
index a4337c9b5287..9abe8b791bb1 100644
--- a/kernel/cgroup/cgroup-v1.c
+++ b/kernel/cgroup/cgroup-v1.c
@@ -206,13 +206,31 @@ struct cgroup_pidlist {
 void cgroup1_pidlist_destroy_all(struct cgroup *cgrp)
 {
 	struct cgroup_pidlist *l, *tmp_l;
+	LIST_HEAD(orphan);
+
+	/*
+	 * Move pidlists to a local orphan list and mark them as owner-less.
+	 * The destroy work function will see ->owner == NULL and skip freeing.
+	 * We then cancel and free them outside pidlist_mutex to avoid
+	 * flush_workqueue() blocking on the shared workqueue.
+	 */
 
 	mutex_lock(&cgrp->pidlist_mutex);
-	list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links)
-		mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, 0);
+	list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links) {
+		list_del(&l->links);
+		WRITE_ONCE(l->owner, NULL);
+		list_add(&l->links, &orphan);
+	}
 	mutex_unlock(&cgrp->pidlist_mutex);
 
-	flush_workqueue(cgroup_pidlist_destroy_wq);
+	list_for_each_entry_safe(l, tmp_l, &orphan, links) {
+		list_del(&l->links);
+		cancel_delayed_work_sync(&l->destroy_dwork);
+		kvfree(l->list);
+		put_pid_ns(l->key.ns);
+		kfree(l);
+	}
+
 	BUG_ON(!list_empty(&cgrp->pidlists));
 }
 
@@ -222,21 +240,26 @@ static void cgroup_pidlist_destroy_work_fn(struct work_struct *work)
 	struct cgroup_pidlist *l = container_of(dwork, struct cgroup_pidlist,
 						destroy_dwork);
 	struct cgroup_pidlist *tofree = NULL;
+	struct cgroup *owner;
 
-	mutex_lock(&l->owner->pidlist_mutex);
+	owner = READ_ONCE(l->owner);
+	if (owner) {
+		mutex_lock(&owner->pidlist_mutex);
 
-	/*
-	 * Destroy iff we didn't get queued again.  The state won't change
-	 * as destroy_dwork can only be queued while locked.
-	 */
-	if (!delayed_work_pending(dwork)) {
-		list_del(&l->links);
-		kvfree(l->list);
-		put_pid_ns(l->key.ns);
-		tofree = l;
-	}
+		/*
+		 * Destroy iff we didn't get queued again and we're still
+		 * owned by the cgroup.  If ->owner was cleared by
+		 * cgroup1_pidlist_destroy_all(), it will free us.
+		 */
+		if (READ_ONCE(l->owner) == owner && !delayed_work_pending(dwork)) {
+			list_del(&l->links);
+			kvfree(l->list);
+			put_pid_ns(l->key.ns);
+			tofree = l;
+		}
 
-	mutex_unlock(&l->owner->pidlist_mutex);
+		mutex_unlock(&owner->pidlist_mutex);
+	}
 	kfree(tofree);
 }
 
-- 
2.43.0

Re: [PATCH v3] cgroup: avoid flushing global workqueue in cgroup1_pidlist_destroy_all

[Junnan Zhang]

Gentle ping.

It's been a couple of weeks since v3, and there hasn't been any
maintainer feedback yet. Could you take a look when you have a
chance? Happy to address any comments or rebase if needed.

Thanks,
Junnan

Re: [PATCH v3] cgroup: avoid flushing global workqueue in cgroup1_pidlist_destroy_all

[Michal Koutný]

[-- Attachment #1: Type: text/plain, Size: 3099 bytes --]

Hi Junnan.

(Sorry for late response, I sketched some notes and then didn't get down
to sent them. Now they're below.)

On Fri, Aug 14, 2026 at 06:20:52PM +0800, Junnan Zhang <zhangjn_dev@163.com> wrote:
> From: Junnan Zhang <zhangjn11@chinatelecom.cn>
> 
> cgroup1_pidlist_destroy_all() flushes the global
> cgroup_pidlist_destroy_wq while destroying a cgroup. Because all cgroup
> v1 pidlist destruction work items are queued on the same shared workqueue,
> a single slow or stuck work item (e.g. waiting for pidlist_mutex held by a
> user-space reader) blocks every concurrent cgroup destruction path.
> 
> This can lead to kworker tasks stuck in flush_workqueue() for over
> hung_task_timeout seconds, as observed on busy systems running Docker or
> Kubernetes workloads.

Since the cgroup_pidlist_destroy_wq is already a dedicated workqueue (no
other conteders), the pursuit of pidlist_mutex holder is a feasible
theory. However, that would also mean:
a) a single reader taking more than hung_task_timeout_secs (that'd be
   a softlockup earlier),
b) starvation of cgroup1_pidlist_destroy_all() by many (queued)
   cgroup_pidlist_start() callers which goes against the second-long
   caching of pidlists,
c) there is large number of nr_cgroups * nr_pidnses which makes the
   caching ineffective

> 
>   INFO: task kworker/0:1:1438499 blocked for more than 120 seconds.
>   "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
>   kworker/0:1 D 0 1438499 2 0x80000080
>   Workqueue: cgroup_destroy css_free_rwork_fn
>   ? __schedule+0x296/0x900
>   schedule+0x28/0x80
>   schedule_timeout+0x1ee/0x3a0
>   ? kvm_sched_clock_read+0xd/0x20
>   wait_for_completion+0x12c/0x190
>   ? wake_up_q+0x70/0x70
>   flush_workqueue+0x132/0x430
>   ? cgroup1_pidlist_destroy_all+0x7c/0xa0
>   cgroup1_pidlist_destroy_all+0x7c/0xa0
>   css_free_rwork_fn+0xb5/0x390
>   process_one_work+0x195/0x3e0
>   worker_thread+0x30/0x390
>   ? process_one_work+0x3e0/0x3e0
>   kthread+0x113/0x130
>   ? kthread_create_worker_on_cpu+0x70/0x70
>   ret_from_fork+0x1f/0x40
> 
> Fix it by moving the cgroup's pidlists to a local orphan list under
> pidlist_mutex, clearing their ->owner pointer, and then cancelling each
> pidlist's delayed work outside the lock. The destroy work function now
> checks ->owner and skips freeing orphaned pidlists, so
> cgroup1_pidlist_destroy_all() can free them safely without flushing the
> whole shared workqueue.

What's the point of the workqueue after this change? (Mainly the
expiration + having process context for the handler.)

The flushing isn't necessary if there's a way how to ensure pidlists
head won't be used after cgrp removal, which the fix should achieve.


So I'd say, the narrow-focused cancellation may work, no need to wait
for other cgroups. OTOH, I'm surprised this v1-issue popped up only now
and whether such a long contention can happen over pidlist_mutex as your
commit message implies. What nr_cgroups, nr_pidnses could cause this in
your theory?

Thanks,
Michal


[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 265 bytes --]

Re: [PATCH v3] cgroup: avoid flushing global workqueue in cgroup1_pidlist_destroy_all

[Ridong Chen]

On 8/31/2026 4:16 PM, Michal Koutný wrote:
> Hi Junnan.
> 
> (Sorry for late response, I sketched some notes and then didn't get down
> to sent them. Now they're below.)
> 
> On Fri, Aug 14, 2026 at 06:20:52PM +0800, Junnan Zhang <zhangjn_dev@163.com> wrote:
>> From: Junnan Zhang <zhangjn11@chinatelecom.cn>
>>
>> cgroup1_pidlist_destroy_all() flushes the global
>> cgroup_pidlist_destroy_wq while destroying a cgroup. Because all cgroup
>> v1 pidlist destruction work items are queued on the same shared workqueue,
>> a single slow or stuck work item (e.g. waiting for pidlist_mutex held by a
>> user-space reader) blocks every concurrent cgroup destruction path.
>>
>> This can lead to kworker tasks stuck in flush_workqueue() for over
>> hung_task_timeout seconds, as observed on busy systems running Docker or
>> Kubernetes workloads.
> 
> Since the cgroup_pidlist_destroy_wq is already a dedicated workqueue (no
> other conteders), the pursuit of pidlist_mutex holder is a feasible
> theory. However, that would also mean:
> a) a single reader taking more than hung_task_timeout_secs (that'd be
>     a softlockup earlier),
> b) starvation of cgroup1_pidlist_destroy_all() by many (queued)
>     cgroup_pidlist_start() callers which goes against the second-long
>     caching of pidlists,
> c) there is large number of nr_cgroups * nr_pidnses which makes the
>     caching ineffective
> 
>>
>>    INFO: task kworker/0:1:1438499 blocked for more than 120 seconds.
>>    "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
>>    kworker/0:1 D 0 1438499 2 0x80000080
>>    Workqueue: cgroup_destroy css_free_rwork_fn
>>    ? __schedule+0x296/0x900
>>    schedule+0x28/0x80
>>    schedule_timeout+0x1ee/0x3a0
>>    ? kvm_sched_clock_read+0xd/0x20
>>    wait_for_completion+0x12c/0x190
>>    ? wake_up_q+0x70/0x70
>>    flush_workqueue+0x132/0x430
>>    ? cgroup1_pidlist_destroy_all+0x7c/0xa0
>>    cgroup1_pidlist_destroy_all+0x7c/0xa0
>>    css_free_rwork_fn+0xb5/0x390
>>    process_one_work+0x195/0x3e0
>>    worker_thread+0x30/0x390
>>    ? process_one_work+0x3e0/0x3e0
>>    kthread+0x113/0x130
>>    ? kthread_create_worker_on_cpu+0x70/0x70
>>    ret_from_fork+0x1f/0x40
>>

I'm not sure we've found the real root cause yet, and the current analysis 
doesn't convince me.

Shouldn't we first figure out why flush_workqueue() waited 120s? Was it because 
there were too many pids, or because someone held pidlist_mutex for too long?

>> Fix it by moving the cgroup's pidlists to a local orphan list under
>> pidlist_mutex, clearing their ->owner pointer, and then cancelling each
>> pidlist's delayed work outside the lock. The destroy work function now
>> checks ->owner and skips freeing orphaned pidlists, so
>> cgroup1_pidlist_destroy_all() can free them safely without flushing the
>> whole shared workqueue.
> 
> What's the point of the workqueue after this change? (Mainly the
> expiration + having process context for the handler.)
> 
> The flushing isn't necessary if there's a way how to ensure pidlists
> head won't be used after cgrp removal, which the fix should achieve.
> 
> 
> So I'd say, the narrow-focused cancellation may work, no need to wait
> for other cgroups. OTOH, I'm surprised this v1-issue popped up only now
> and whether such a long contention can happen over pidlist_mutex as your
> commit message implies. What nr_cgroups, nr_pidnses could cause this in
> your theory?
> 
> Thanks,
> Michal

-- 
Best regards
Ridong

Re: [PATCH v3] cgroup: avoid flushing global workqueue in cgroup1_pidlist_destroy_all

[Junnan Zhang]

Hi Ridong,

> Shouldn't we first figure out why flush_workqueue() waited 120s? Was
> it because there were too many pids, or because someone held
> pidlist_mutex for too long?

That's the same question Michal raised; please see my reply to him in
this thread for the full analysis. The short version: it's neither a
single long mutex holder nor oversized pidlists per se -- the wait is
backlog x per-work latency. flush_workqueue() waits for every work
already queued on the shared wq, which drains serially
(WQ_PERCPU, max_active=1). Container churn keeps queueing destroy
works, and each work must take the owner's pidlist_mutex behind
readers whose pidlist_array_load() runs entirely under that mutex.
A few thousand queued works each delayed by tens of ms is enough to
exceed 120s.

Unfortunately the guest memory dump captured at the incident couldn't
be analyzed with crash, so exact queue depths aren't available; I've
offered to build a synthetic reproducer with measured latency data if
that's needed to move this forward.

Thanks,
Junnan

Re: [PATCH v3] cgroup: avoid flushing global workqueue in cgroup1_pidlist_destroy_all

[Junnan Zhang]

Hi Ridong,

> Shouldn't we first figure out why flush_workqueue() waited 120s? Was
> it because there were too many pids, or because someone held
> pidlist_mutex for too long?

That's the same question Michal raised; please see my reply to him in
this thread for the full analysis. The short version: it's neither a
single long mutex holder nor oversized pidlists per se -- the wait is
backlog x per-work latency. flush_workqueue() waits for every work
already queued on the shared wq, which drains serially
(WQ_PERCPU, max_active=1). Container churn keeps queueing destroy
works, and each work must take the owner's pidlist_mutex behind
readers whose pidlist_array_load() runs entirely under that mutex.
A few thousand queued works each delayed by tens of ms is enough to
exceed 120s.

Unfortunately the guest memory dump captured at the incident couldn't
be analyzed with crash, so exact queue depths aren't available; I've
offered to build a synthetic reproducer with measured latency data if
that's needed to move this forward.

Thanks,
Junnan

Re: [PATCH v3] cgroup: avoid flushing global workqueue in cgroup1_pidlist_destroy_all

[Junnan Zhang]

Hi Michal,

Thanks for the review.

> a) a single reader taking more than hung_task_timeout_secs (that'd be
>    a softlockup earlier),
> b) starvation of cgroup1_pidlist_destroy_all() by many (queued)
>    cgroup_pidlist_start() callers which goes against the second-long
>    caching of pidlists,
> c) there is large number of nr_cgroups * nr_pidnses which makes the
>    caching ineffective

An honest caveat first: this was reported by a customer on a production
Kubernetes node. A guest memory dump was taken at the time, but it
couldn't be analyzed with crash, so I can't give you the exact
nr_cgroups/nr_pidnses.

That said, I believe c) alone is sufficient, and neither a) nor b) is
needed to explain the 120s stall, because the flush latency is
backlog x per-work latency rather than the latency of any single work:

- backlog: flush_workqueue() waits for all works already queued on the
  shared wq, which drains serially (WQ_PERCPU, max_active=1). Container
  churn constantly runs cgroup destruction, and each destroyed cgroup
  queues one work per cached (type, ns) pidlist, so the queue ahead of
  the flusher grows with churn rate x nr_cgroups.

- per-work latency: each destroy work must acquire the owner's
  pidlist_mutex, contending with readers (kubelet/cadvisor/runtime
  scraping cgroup.procs). pidlist_array_load() runs entirely under
  that mutex (css_set walk, kvmalloc, sort), so on a node with frequent
  scraping each work can sit tens of ms behind a reader.

A few thousand queued works each delayed by tens of ms already exceed
hung_task_timeout_secs -- no single reader needs to hold the mutex for
that long (so a) doesn't apply), and each individual work does get the
mutex eventually, so it's not sustained starvation either (so b)
doesn't apply).

More on b): the second-long caching only helps readers that re-read
within that 1s window (e.g. a single `cat` doing several seq_file
iterations). Periodic scrapers like kubelet/cadvisor, with typical
intervals of 10-30s, never hit the cache at all: every scrape round
rebuilds the pidlist under the mutex and queues an expiry work 1s
later. So heavy reader traffic and the existence of the cache are not
in conflict -- the cache is simply ineffective for this access
pattern. It also means the wq steadily carries ~nr_cgroups expiry
works per scrape round, on top of the works queued by churn, which is
what the flusher ends up waiting behind.

On nr_pidnses specifically: containers typically share the pod/host
pid namespace, so in the common case the multiplier is really
nr_cgroups x scraping frequency rather than nr_pidnses. I can't confirm
the customer's pidns usage for the same reason as above.

> OTOH, I'm surprised this v1-issue popped up only now

cgroup v1 is legacy but still the default on widely deployed enterprise
distros, and per-node cgroup density plus metrics scraping frequency
have grown a lot in recent years, so the backlog needed to trip this
only became common recently. Triggering it also requires all three
conditions at once -- many cgroups, frequent full-sweep scraping, and
sustained create/destroy churn -- which is presumably why it isn't
seen more often: missing any one of them, the queue never builds up.
That's my best explanation -- admittedly not provable without an
analyzable dump.

> What's the point of the workqueue after this change? (Mainly the
> expiration + having process context for the handler.)

It still serves the normal path: the deferred expiry that makes the
pidlist cache work, and process context for freeing. The patch only
removes the flush from the cgroup-removal path where, as you noted,
the orphan list guarantees the pidlists head won't be used after cgrp
removal.

If the lack of exact field numbers is a blocker, I can put together a
synthetic reproducer (N cgroups with concurrent cgroup.procs readers
plus destroy churn) and report measured flush_workqueue() latency with
and without the patch. Would that address your concern?

Thanks,
Junnan

Re: [PATCH v3] cgroup: avoid flushing global workqueue in cgroup1_pidlist_destroy_all

[Junnan Zhang]

Hi Michal,

Thanks for the review.

> a) a single reader taking more than hung_task_timeout_secs (that'd be
>    a softlockup earlier),
> b) starvation of cgroup1_pidlist_destroy_all() by many (queued)
>    cgroup_pidlist_start() callers which goes against the second-long
>    caching of pidlists,
> c) there is large number of nr_cgroups * nr_pidnses which makes the
>    caching ineffective

An honest caveat first: this was reported by a customer on a production
Kubernetes node. A guest memory dump was taken at the time, but it
couldn't be analyzed with crash, so I can't give you the exact
nr_cgroups/nr_pidnses.

That said, I believe c) alone is sufficient, and neither a) nor b) is
needed to explain the 120s stall, because the flush latency is
backlog x per-work latency rather than the latency of any single work:

- backlog: flush_workqueue() waits for all works already queued on the
  shared wq, which drains serially (WQ_PERCPU, max_active=1). Container
  churn constantly runs cgroup destruction, and each destroyed cgroup
  queues one work per cached (type, ns) pidlist, so the queue ahead of
  the flusher grows with churn rate x nr_cgroups.

- per-work latency: each destroy work must acquire the owner's
  pidlist_mutex, contending with readers (kubelet/cadvisor/runtime
  scraping cgroup.procs). pidlist_array_load() runs entirely under
  that mutex (css_set walk, kvmalloc, sort), so on a node with frequent
  scraping each work can sit tens of ms behind a reader.

A few thousand queued works each delayed by tens of ms already exceed
hung_task_timeout_secs -- no single reader needs to hold the mutex for
that long (so a) doesn't apply), and each individual work does get the
mutex eventually, so it's not sustained starvation either (so b)
doesn't apply).

More on b): the second-long caching only helps readers that re-read
within that 1s window (e.g. a single `cat` doing several seq_file
iterations). Periodic scrapers like kubelet/cadvisor, with typical
intervals of 10-30s, never hit the cache at all: every scrape round
rebuilds the pidlist under the mutex and queues an expiry work 1s
later. So heavy reader traffic and the existence of the cache are not
in conflict -- the cache is simply ineffective for this access
pattern. It also means the wq steadily carries ~nr_cgroups expiry
works per scrape round, on top of the works queued by churn, which is
what the flusher ends up waiting behind.

On nr_pidnses specifically: containers typically share the pod/host
pid namespace, so in the common case the multiplier is really
nr_cgroups x scraping frequency rather than nr_pidnses. I can't confirm
the customer's pidns usage for the same reason as above.

> OTOH, I'm surprised this v1-issue popped up only now

cgroup v1 is legacy but still the default on widely deployed enterprise
distros, and per-node cgroup density plus metrics scraping frequency
have grown a lot in recent years, so the backlog needed to trip this
only became common recently. Triggering it also requires all three
conditions at once -- many cgroups, frequent full-sweep scraping, and
sustained create/destroy churn -- which is presumably why it isn't
seen more often: missing any one of them, the queue never builds up.
That's my best explanation -- admittedly not provable without an
analyzable dump.

> What's the point of the workqueue after this change? (Mainly the
> expiration + having process context for the handler.)

It still serves the normal path: the deferred expiry that makes the
pidlist cache work, and process context for freeing. The patch only
removes the flush from the cgroup-removal path where, as you noted,
the orphan list guarantees the pidlists head won't be used after cgrp
removal.

If the lack of exact field numbers is a blocker, I can put together a
synthetic reproducer (N cgroups with concurrent cgroup.procs readers
plus destroy churn) and report measured flush_workqueue() latency with
and without the patch. Would that address your concern?

Thanks,
Junnan

Re: [PATCH v3] cgroup: avoid flushing global workqueue in cgroup1_pidlist_destroy_all

[Michal Koutný]

[-- Attachment #1: Type: text/plain, Size: 4648 bytes --]

Hi Junnan.

On Tue, Sep 01, 2026 at 11:39:23AM +0800, Junnan Zhang <zhangjn_dev@163.com> wrote:
> Thanks for the review.
> 
> > a) a single reader taking more than hung_task_timeout_secs (that'd be
> >    a softlockup earlier),
> > b) starvation of cgroup1_pidlist_destroy_all() by many (queued)
> >    cgroup_pidlist_start() callers which goes against the second-long
> >    caching of pidlists,
> > c) there is large number of nr_cgroups * nr_pidnses which makes the
> >    caching ineffective
> 
> An honest caveat first: this was reported by a customer on a production
> Kubernetes node. A guest memory dump was taken at the time, but it
> couldn't be analyzed with crash, so I can't give you the exact
> nr_cgroups/nr_pidnses.
> 
> That said, I believe c) alone is sufficient, and neither a) nor b) is
> needed to explain the 120s stall, because the flush latency is
> backlog x per-work latency rather than the latency of any single work:
> 
> - backlog: flush_workqueue() waits for all works already queued on the
>   shared wq, which drains serially (WQ_PERCPU, max_active=1). Container
>   churn constantly runs cgroup destruction, and each destroyed cgroup
>   queues one work per cached (type, ns) pidlist, so the queue ahead of
>   the flusher grows with churn rate x nr_cgroups.

flush_workqueue() should only block for items present at its invocation
(not later added). I assume large amount of rmdir's in short succession
may build up the queue as a one-off event.

> 
> - per-work latency: each destroy work must acquire the owner's
>   pidlist_mutex, contending with readers (kubelet/cadvisor/runtime
>   scraping cgroup.procs). pidlist_array_load() runs entirely under
>   that mutex (css_set walk, kvmalloc, sort), so on a node with frequent
>   scraping each work can sit tens of ms behind a reader.

OK, lets (over)estimate 100ms per pidlist_mutex passthrough, that gives
lower bound of some 1200 cgroups removed together. (That's a lot but
borderline still practically possible.)

> A few thousand queued works each delayed by tens of ms already exceed
> hung_task_timeout_secs -- no single reader needs to hold the mutex for
> that long (so a) doesn't apply), and each individual work does get the
> mutex eventually, so it's not sustained starvation either (so b)
> doesn't apply).
> 
> More on b): the second-long caching only helps readers that re-read
> within that 1s window (e.g. a single `cat` doing several seq_file
> iterations). Periodic scrapers like kubelet/cadvisor, with typical
> intervals of 10-30s, never hit the cache at all: every scrape round
> rebuilds the pidlist under the mutex and queues an expiry work 1s
> later.

So one reader produces ~0.1 items/s, with the latency number above
around 10 items/s should be dispatched. Or ~100 (parallel) scrapers
should be manageable.

> So heavy reader traffic and the existence of the cache are not
> in conflict -- the cache is simply ineffective for this access
> pattern. It also means the wq steadily carries ~nr_cgroups expiry
> works per scrape round, on top of the works queued by churn, which is
> what the flusher ends up waiting behind.

I understand that an abprubt removal of thousands of cgroups may stress
the pidlist_destroy workqueue.

> On nr_pidnses specifically: containers typically share the pod/host
> pid namespace, so in the common case the multiplier is really
> nr_cgroups x scraping frequency rather than nr_pidnses. I can't confirm
> the customer's pidns usage for the same reason as above.
> 
> > OTOH, I'm surprised this v1-issue popped up only now
> 
> cgroup v1 is legacy but still the default on widely deployed enterprise
> distros, and per-node cgroup density plus metrics scraping frequency
> have grown a lot in recent years, so the backlog needed to trip this
> only became common recently.

I remain somewhat reserved whether only the mere density would cause
thise (as there are other bottlenecks that may start blocking with these
numbers).


> It still serves the normal path: the deferred expiry that makes the
> pidlist cache work, and process context for freeing. The patch only
> removes the flush from the cgroup-removal path where, as you noted,
> the orphan list guarantees the pidlists head won't be used after cgrp
> removal.

Your patch looks correct to me, although, I'd rather not nurture the
venerable code at all. And I've never liked pidlists, so I'd like to
take the opportunity to retire them a bit:

https://github.com/Werkov/linux/commit/8c3d45be95841ede16ceac29d48be59e59ec5a82

Opinions?

Michal

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 265 bytes --]

Re: [PATCH v3] cgroup: avoid flushing global workqueue in cgroup1_pidlist_destroy_all

[Tejun Heo]

On Tue, Sep 08, 2026 at 04:13:11PM +0200, Michal Koutný wrote:
> Your patch looks correct to me, although, I'd rather not nurture the
> venerable code at all. And I've never liked pidlists, so I'd like to
> take the opportunity to retire them a bit:
> 
> https://github.com/Werkov/linux/commit/8c3d45be95841ede16ceac29d48be59e59ec5a82

I'd rather not touch them at all and eventually retire the whole v1
interface.

Thanks.

-- 
tejun

Re: [PATCH v3] cgroup: avoid flushing global workqueue in cgroup1_pidlist_destroy_all

[Junnan Zhang]

Hi Tejun, Michal,

> I'd rather not touch them at all and eventually retire the whole v1
> interface.

Understood, and I have no objection to Michal's approach as the
long-term direction -- routing v1 reads through css_task_iter removes
the pidlist machinery altogether, including the destroy workqueue
behind this hang.

May I ask for some guidance on the near term, though? cgroup v1 is
still the default on widely deployed enterprise distros and will
likely remain in production use for years to come, and this hung
task was observed on a production Kubernetes node. This patch (v3)
is a pure maintenance fix: it changes no user-visible behavior, adds
no interface, and is stable-backportable. It would also remain
relevant for the cgroup_v1_sorted=true path if Michal's change
lands.

Would you consider taking it as a minimal fix? Or, if the consensus
is to leave v1 completely untouched, what would be the recommended
course for production systems that keep hitting this?

Thanks,
Junnan

Re: [PATCH v3] cgroup: avoid flushing global workqueue in cgroup1_pidlist_destroy_all

[Tejun Heo]

Hello,

On Wed, Sep 09, 2026 at 04:46:19PM +0800, Junnan Zhang wrote:
> Would you consider taking it as a minimal fix? Or, if the consensus
> is to leave v1 completely untouched, what would be the recommended
> course for production systems that keep hitting this?

The problem is that the root cause isn't sufficiently established. It's just
difficult to believe cleaning up these lists would take longer than 120s in
itself. Maybe there were other contributing factors - e.g. another
saturating per-cpu work item that was preventing the execution of the
pidlist work item, high memory pressure stalling worker creation, or just
severe CPU contention from bw control or whatnot.

Without the root cause convincingly established, this can't be a "minimal
fix". This can be a proactive behavior improvement, but that's not something
we want to do for cgroup1 code base at this point. Not because we hate
people on cgroup1 but because they're the legacy users on legacy code base.
Nobody is actively working on it and no leading edge testing and adoption
covers it. Any change carries risk of breakage and a change like this can
lead to really subtle problems that can take a long time to diagnose
especially with slow-moving long-tail userbase.

So, yes, I'll take minimal fixes that address real problems (subject to risk
vs. benefit balance of course), but you haven't established that yet.

Thanks.

-- 
tejun

Re: [PATCH v3] cgroup: avoid flushing global workqueue in cgroup1_pidlist_destroy_all

[Junnan Zhang]

Hi Tejun,

Thanks for the clarification -- agreed that the 120s stall needs to
be convincingly attributed before this can count as a minimal fix.

I'll put together a reproducer that drives only cgroup churn plus
cgroup.procs readers on an otherwise idle machine, with measured
cgroup1_pidlist_destroy_all() latencies (and a lowered
hung_task_timeout_secs to catch actual splats), and report back with
the numbers. It'll take me a few days to set up.

Thanks,
Junnan

Re: [PATCH v3] cgroup: avoid flushing global workqueue in cgroup1_pidlist_destroy_all

[Junnan Zhang]

Hi Tejun,

As promised, here are the numbers from a synthetic reproducer.

Common setup (deliberately excluding all the alternative contributors
you mentioned):

- test VM, with/without the patch, no other workload, no memory
  pressure, CPUs far from saturated
- a private v1 named hierarchy (no controllers), so nothing else on
  the system touches cgroup_pidlist_destroy_wq
- some long-lived cgroups holding tasks ("containers"), concurrent
  readers sweeping their cgroup.procs ("monitoring agents"), and churn
  threads doing mkdir/read tasks/rmdir ("container churn")
- latency of cgroup1_pidlist_destroy_all() measured with bpftrace
  kprobe/kretprobe, aggregated per 10s window

I ran two configurations at different scales (same 1000 churners):

  Run A: 8 CPUs, 1G RAM, 200 cgroups x 20 tasks, 100 readers, 300s
           ./cg_flush_repro -s 200 -t 20 -r 100 -c 1000 -d 300

  Run B: 64 CPUs, 16G RAM, 50 cgroups x 1000 tasks, 50 readers, 300s
           ./cg_flush_repro -s 50 -t 1000 -r 50 -c 1000 -d 300

Summary (per 10s window; count = number of calls, avg/max per call):

  Run A unpatched:  count ~400,     avg 71-103 us,   max ~9.8 ms
  Run A patched:    count 157-641,  avg 3-6 us,      max 32-78 us

  Run B unpatched:  count ~40k,     avg 1044-1306 us, max 218-253 ms
                    (tail: ~1000 samples above 2 ms,
                     dozens above 128 ms per window)
  Run B patched:    count ~43-45k,  avg 1 us,        max 131-188 us

Two observations:

1. The coupling scales as predicted: flush latency = backlog x
   per-work latency. Both factors grew from run A to run B -- the
   bigger machine keeps a deeper backlog in flight, and 1000 instead
   of 20 tasks per cgroup make each pidlist rebuild hold pidlist_mutex
   much longer. The unpatched latency grew accordingly: average from
   ~100us to >1ms, max from ~10ms to ~250ms, with a long tail of
   individual flush calls waiting behind the backlog (see the Run B
   histograms below: single flush calls waiting >100ms are not
   outliers but a steady population).

2. With the patch, latency stays flat and microscopic (max < 200us)
   at the same churn rate in both runs -- destroy latency no longer
   depends on the queue backlog at all.

To be upfront: I still have not reproduced the full 120s hang in the
lab; the backlog and per-work latency I can build here remain well
below the production node's. But the unpatched latency grows with
exactly the two factors that were larger in production, while the
patched latency is insensitive to them -- on an otherwise idle
machine, with no competing workqueue items, no worker-creation stalls
and no CPU contention.

The raw bpftrace output (two 10s windows per configuration), the
reproducer (cg_flush_repro.c) and the bpftrace script
(cg_flush_lat.bt) are included below.

Thanks,
Junnan

---
Raw bpftrace output:

Run A, unpatched:

  @count: 406
  @avg_us: 71
  @max_us: 9765
  @lat_us:
  [0]                   16 |@@@@@@                                              |
  [1]                    2 |                                                    |
  [2, 4)                80 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@                     |
  [4, 8)               131 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@|
  [8, 16)              121 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@    |
  [16, 32)              35 |@@@@@@@@@@@@@                                       |
  [32, 64)               7 |@@                                                  |
  [64, 128)              1 |                                                    |
  [128, 256)             5 |@                                                   |
  [256, 512)             3 |@                                                   |
  [512, 1K)              0 |                                                    |
  [1K, 2K)               2 |                                                    |
  [2K, 4K)               0 |                                                    |
  [4K, 8K)               2 |                                                    |
  [8K, 16K)              1 |                                                    |

  @count: 393
  @avg_us: 103
  @max_us: 9582
  @lat_us:
  [0]                   13 |@@@@                                                |
  [1]                    1 |                                                    |
  [2, 4)                61 |@@@@@@@@@@@@@@@@@@@@@                               |
  [4, 8)               150 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@|
  [8, 16)              106 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@                |
  [16, 32)              33 |@@@@@@@@@@@                                         |
  [32, 64)              10 |@@@                                                 |
  [64, 128)              4 |@                                                   |
  [128, 256)             4 |@                                                   |
  [256, 512)             3 |@                                                   |
  [512, 1K)              0 |                                                    |
  [1K, 2K)               2 |                                                    |
  [2K, 4K)               3 |@                                                   |
  [4K, 8K)               2 |                                                    |
  [8K, 16K)              1 |                                                    |

Run A, patched:

  @count: 157
  @avg_us: 6
  @max_us: 32
  @lat_us:
  [0]                    2 |                                                    |
  [1]                    0 |                                                    |
  [2, 4)                 9 |@@@@                                                |
  [4, 8)               117 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@|
  [8, 16)               23 |@@@@@@@@@@                                          |
  [16, 32)               5 |@@                                                  |
  [32, 64)               1 |                                                    |

  @count: 641
  @avg_us: 3
  @max_us: 78
  @lat_us:
  [0]                   57 |@@@@@@@@@@                                          |
  [1]                   30 |@@@@@                                               |
  [2, 4)               281 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@|
  [4, 8)               254 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@     |
  [8, 16)               13 |@@                                                  |
  [16, 32)               5 |                                                    |
  [32, 64)               0 |                                                    |
  [64, 128)              1 |                                                    |

Run B, unpatched:

  @count: 40124
  @avg_us: 1044
  @max_us: 218693
  @lat_us:
  [4, 8)              6822 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@              |
  [8, 16)             3895 |@@@@@@@@@@@@@@@@@@@@@                               |
  [16, 32)            1467 |@@@@@@@@                                            |
  [32, 64)            2848 |@@@@@@@@@@@@@@@                                     |
  [64, 128)           5475 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@                      |
  [128, 256)          9321 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@|
  [256, 512)          7324 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@            |
  [512, 1K)            948 |@@@@@                                               |
  [1K, 2K)             179 |                                                    |
  [2K, 4K)             515 |@@                                                  |
  [4K, 8K)             393 |@@                                                  |
  [8K, 16K)            179 |                                                    |
  [16K, 32K)           528 |@@                                                  |
  [32K, 64K)           123 |                                                    |
  [64K, 128K)           78 |                                                    |
  [128K, 256K)          35 |                                                    |

  @count: 39245
  @avg_us: 1306
  @max_us: 253031
  @lat_us:
  [4, 8)              5188 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@                        |
  [8, 16)             4072 |@@@@@@@@@@@@@@@@@@@@@@                              |
  [16, 32)            1366 |@@@@@@@                                             |
  [32, 64)            2340 |@@@@@@@@@@@@                                        |
  [64, 128)           4782 |@@@@@@@@@@@@@@@@@@@@@@@@@                           |
  [128, 256)          9566 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@|
  [256, 512)          8335 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@       |
  [512, 1K)           1688 |@@@@@@@@@                                           |
  [1K, 2K)             134 |                                                    |
  [2K, 4K)             466 |@@                                                  |
  [4K, 8K)             387 |@@                                                  |
  [8K, 16K)            156 |                                                    |
  [16K, 32K)           457 |@@                                                  |
  [32K, 64K)           112 |                                                    |
  [64K, 128K)          138 |                                                    |
  [128K, 256K)          56 |                                                    |

Run B, patched:

  @count: 45368
  @avg_us: 1
  @max_us: 131
  @lat_us:
  [0]                  569 |                                                    |
  [1]                38421 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@|
  [2, 4)              5330 |@@@@@@@                                             |
  [4, 8)               957 |@                                                   |
  [8, 16)               71 |                                                    |
  [16, 32)              12 |                                                    |
  [32, 64)               1 |                                                    |
  [64, 128)              6 |                                                    |
  [128, 256)             1 |                                                    |

  @count: 42817
  @avg_us: 1
  @max_us: 188
  @lat_us:
  [0]                 1156 |@                                                   |
  [1]                34402 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@|
  [2, 4)              6165 |@@@@@@@@@                                           |
  [4, 8)               919 |@                                                   |
  [8, 16)              147 |                                                    |
  [16, 32)              23 |                                                    |
  [32, 64)               2 |                                                    |
  [64, 128)              1 |                                                    |
  [128, 256)             2 |                                                    |

---
==> cg_flush_repro.c <==

/*
 * cg_flush_repro.c - synthetic reproducer for the cgroup v1 pidlist
 * destroy workqueue backlog (the scenario behind the hung task splat in
 * cgroup1_pidlist_destroy_all() -> flush_workqueue()).
 *
 * It reproduces the three conditions of a busy container node:
 *
 *   1. many long-lived cgroups, each holding many tasks ("containers");
 *   2. concurrent readers sweeping cgroup.procs ("monitoring agents")
 *      with a sweep period >> CGROUP_PIDLIST_DESTROY_DELAY (1s), so the
 *      pidlist cache never hits: every read rebuilds the pidlist under
 *      pidlist_mutex and queues an expiry destroy work 1s later;
 *   3. constant cgroup create/read/destroy churn from several threads,
 *      where each destruction ends up in cgroup1_pidlist_destroy_all()
 *      and, on an unpatched kernel, flush_workqueue() on the shared
 *      cgroup_pidlist_destroy_wq.
 *
 * Compare `cgroup1_pidlist_destroy_all()` latency with and without the
 * fix using cg_flush_lat.bt (bpftrace).
 *
 * Build:  gcc -O2 -pthread -o cg_flush_repro cg_flush_repro.c
 * Usage:  sudo ./cg_flush_repro [-m mnt] [-s seed_cgroups] [-t tasks_per]
 *                               [-r readers] [-c churners] [-d seconds]
 *
 * Needs root. Works on cgroup v2-only systems as well: it mounts its
 * own v1 named hierarchy (no controllers needed, cgroup.procs/tasks
 * still go through the v1 pidlist code).
 *
 * WARNING: this deliberately stresses the kernel. Run it in a test VM,
 * never on a production host.
 */
#define _GNU_SOURCE
#include <errno.h>
#include <fcntl.h>
#include <getopt.h>
#include <pthread.h>
#include <signal.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>

#define NAME_PREFIX	"cgflush"

static const char *mnt = "/tmp/cgflush_v1";
static int nr_seed = 500;	/* long-lived cgroups swept by readers */
static int tasks_per = 50;	/* tasks in each seed cgroup */
static int nr_readers = 16;	/* concurrent cgroup.procs readers */
static int nr_churners = 4;	/* concurrent cgroup create/destroy threads */
static int duration = 300;	/* seconds */

static volatile sig_atomic_t stop;

static atomic_ulong reads_done;
static atomic_ulong churn_done;
static atomic_ulong errors;

static pid_t *children;		/* nr_seed * tasks_per tasks */
static size_t nr_children;

static void on_signal(int sig)
{
	stop = 1;
}

static int write_pid(const char *path, pid_t pid)
{
	char buf[32];
	int len = snprintf(buf, sizeof(buf), "%d", pid);
	int fd = open(path, O_WRONLY);
	int ret = -1;

	if (fd < 0)
		return -1;
	if (write(fd, buf, len) == len)
		ret = 0;
	close(fd);
	return ret;
}

/* Read the whole file so the seq_file start/stop cycle completes. */
static int read_fully(const char *path)
{
	char buf[65536];
	ssize_t n;
	int fd = open(path, O_RDONLY);

	if (fd < 0)
		return -1;
	while ((n = read(fd, buf, sizeof(buf))) > 0)
		;
	close(fd);
	return n < 0 ? -1 : 0;
}

static void *reader_fn(void *arg)
{
	long id = (long)arg;
	unsigned int seed = getpid() ^ (id * 2654435761u);
	char path[256];

	while (!stop) {
		int idx = rand_r(&seed) % nr_seed;

		snprintf(path, sizeof(path), "%s/seed_%d/cgroup.procs", mnt, idx);
		if (read_fully(path) == 0)
			atomic_fetch_add(&reads_done, 1);
		else
			atomic_fetch_add(&errors, 1);
	}
	return NULL;
}

/*
 * Churn: repeatedly create a cgroup, read its tasks file (building a
 * pidlist, hence queueing a destroy work) and remove it.  The cgroup
 * destruction ends up in cgroup1_pidlist_destroy_all().  Each churner
 * thread uses its own dir name prefix so they never collide.
 */
static void *churn_fn(void *arg)
{
	long id = (long)arg;
	char dir[256], path[320];
	unsigned long i = 0;

	while (!stop) {
		snprintf(dir, sizeof(dir), "%s/churn_%ld_%lu", mnt, id, i);
		snprintf(path, sizeof(path), "%s/tasks", dir);

		if (mkdir(dir, 0755) < 0) {
			atomic_fetch_add(&errors, 1);
			usleep(1000);
			continue;
		}
		read_fully(path);
		if (rmdir(dir) == 0)
			atomic_fetch_add(&churn_done, 1);
		i++;
	}
	return NULL;
}

static void spawn_task(size_t cgroup_idx)
{
	pid_t pid = fork();
	char path[256];

	if (pid < 0) {
		perror("fork");
		return;
	}
	if (pid == 0) {
		/* child: just exist as a cgroup member */
		for (;;)
			pause();
	}
	children[nr_children++] = pid;
	snprintf(path, sizeof(path), "%s/seed_%zu/cgroup.procs", mnt, cgroup_idx);
	write_pid(path, pid);
}

static void cleanup(void)
{
	size_t i;
	char path[256];

	for (i = 0; i < nr_children; i++)
		kill(children[i], SIGKILL);
	while (waitpid(-1, NULL, 0) > 0)
		;
	for (i = 0; i < (size_t)nr_seed; i++) {
		snprintf(path, sizeof(path), "%s/seed_%zu", mnt, i);
		rmdir(path);
	}
	umount(mnt);
	rmdir(mnt);
}

int main(int argc, char **argv)
{
	pthread_t *readers, *churners;
	struct sigaction sa = { .sa_handler = on_signal };
	unsigned long reads, churns;
	int opt, i;

	while ((opt = getopt(argc, argv, "m:s:t:r:c:d:")) != -1) {
		switch (opt) {
		case 'm': mnt = optarg; break;
		case 's': nr_seed = atoi(optarg); break;
		case 't': tasks_per = atoi(optarg); break;
		case 'r': nr_readers = atoi(optarg); break;
		case 'c': nr_churners = atoi(optarg); break;
		case 'd': duration = atoi(optarg); break;
		default:
			fprintf(stderr,
				"usage: %s [-m mnt] [-s seed_cgroups] "
				"[-t tasks_per] [-r readers] [-c churners] "
				"[-d seconds]\n",
				argv[0]);
			return 1;
		}
	}

	sigaction(SIGINT, &sa, NULL);
	sigaction(SIGTERM, &sa, NULL);

	/* mount a v1 named hierarchy (no controllers needed) */
	if (mkdir(mnt, 0755) < 0 && errno != EEXIST) {
		perror("mkdir mountpoint");
		return 1;
	}
	if (mount("cgroup", mnt, "cgroup", 0, "none,name=" NAME_PREFIX) < 0 &&
	    errno != EBUSY) {
		perror("mount cgroup v1");
		return 1;
	}

	/* create seed cgroups */
	for (i = 0; i < nr_seed; i++) {
		char dir[256];

		snprintf(dir, sizeof(dir), "%s/seed_%d", mnt, i);
		if (mkdir(dir, 0755) < 0) {
			perror("mkdir seed cgroup");
			goto out_cleanup;
		}
	}

	/* fork tasks (before creating any threads!) and populate seeds */
	children = calloc((size_t)nr_seed * tasks_per, sizeof(pid_t));
	if (!children) {
		perror("calloc");
		goto out_cleanup;
	}
	printf("spawning %d tasks across %d cgroups...\n",
	       nr_seed * tasks_per, nr_seed);
	for (i = 0; i < nr_seed * tasks_per; i++)
		spawn_task(i / tasks_per);
	printf("running: %d readers + %d churners for %ds (Ctrl-C to stop)\n",
	       nr_readers, nr_churners, duration);

	/* start readers and churners */
	readers = calloc(nr_readers, sizeof(pthread_t));
	churners = calloc(nr_churners, sizeof(pthread_t));
	for (i = 0; i < nr_readers; i++)
		pthread_create(&readers[i], NULL, reader_fn, (void *)(long)i);
	for (i = 0; i < nr_churners; i++)
		pthread_create(&churners[i], NULL, churn_fn, (void *)(long)i);

	for (i = 0; i < duration && !stop; i++)
		sleep(1);
	stop = 1;

	for (i = 0; i < nr_churners; i++)
		pthread_join(churners[i], NULL);
	for (i = 0; i < nr_readers; i++)
		pthread_join(readers[i], NULL);

	reads = atomic_load(&reads_done);
	churns = atomic_load(&churn_done);
	printf("done: %lu procs reads (%lu/s), %lu churn cycles (%lu/s), "
	       "%lu errors\n",
	       reads, reads / (duration ?: 1),
	       churns, churns / (duration ?: 1),
	       atomic_load(&errors));

out_cleanup:
	cleanup();
	return 0;
}

==> cg_flush_lat.bt <==

kprobe:cgroup1_pidlist_destroy_all
{
	@start[tid] = nsecs;
}

kretprobe:cgroup1_pidlist_destroy_all
/@start[tid]/
{
	$us = (nsecs - @start[tid]) / 1000;
	@lat_us = hist($us);
	@max_us = max($us);
	@avg_us = avg($us);
	@count = count();
	delete(@start[tid]);
}

interval:s:10
{
	print(@count);
	print(@avg_us);
	print(@max_us);
	print(@lat_us);

	clear(@count);
	clear(@avg_us);
	clear(@max_us);
	clear(@lat_us);

}

END
{
	clear(@start);
}

Re: [PATCH v3] cgroup: avoid flushing global workqueue in cgroup1_pidlist_destroy_all

[Tejun Heo]

On Tue, Sep 15, 2026 at 08:40:10PM +0800, Junnan Zhang wrote:
...
> To be upfront: I still have not reproduced the full 120s hang in the
> lab; the backlog and per-work latency I can build here remain well
> below the production node's. But the unpatched latency grows with
> exactly the two factors that were larger in production, while the
> patched latency is insensitive to them -- on an otherwise idle
> machine, with no competing workqueue items, no worker-creation stalls
> and no CPU contention.

That's a larger than two orders of magnitude gap. There likely were other, a
lot more important, factors.

Thanks.

-- 
tejun