[PATCH v1] mm: unqueue deferred split folios outside the lruvec lock

[PATCH v1] mm: unqueue deferred split folios outside the lruvec lock

[Joanne Koong]

move_folios_to_lru(), folio_batch_move_lru() and folios_put_refs() call
folio_unqueue_deferred_split() on folios where the refcount has dropped
to zero. In these paths, folio_unqueue_deferred_split() is called with
the lruvec lock often held (move_folios_to_lru() always holds it and
folio_batch_move_lru()/folios_put_refs() usually hold it for every folio
in the batch except the first). However, it's not needed to hold the
lruvec lock.

The lruvec lock protects folio->lru and the reclaim LRU lists whereas
the deferred split queue lives on folio->_deferred_list in
deferred_split_lru. The deferred_split_lru is a list_lru with its own
per-memcg per-node lock.

There are already other existing callers that call
folio_unqueue_deferred_split() without holding the lruvec lock (eg
__folio_put(), shrink_folio_list(), __folio_migrate_mapping(), ...).

Add folio_batch_unqueue_deferred_split() and use it in
move_folios_to_lru(), folio_batch_move_lru(), and folios_put_refs() to
unqueue each batch of zero-refcount folios from the deferred split list
after the lruvec lock is released and before
mem_cgroup_uncharge_folios(). This helps shorten the lruvec lock's
critical sections, which run with IRQs disabled and are highly
contended.

Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
---
 mm/folio.c    | 24 +++++++++++++++++++-----
 mm/internal.h |  1 +
 mm/vmscan.c   |  3 ++-
 3 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/mm/folio.c b/mm/folio.c
index 35e242b48870..85144078e635 100644
--- a/mm/folio.c
+++ b/mm/folio.c
@@ -109,6 +109,19 @@ EXPORT_SYMBOL(__folio_put);
 
 typedef void (*move_fn_t)(struct lruvec *lruvec, struct folio *folio);
 
+/*
+ * Unqueue a batch of folios from the deferred split queue. This must be
+ * called after the folios' refcounts have dropped to zero and before they
+ * are uncharged (see comment above __folio_unqueue_deferred_split() for why).
+ */
+void folio_batch_unqueue_deferred_split(struct folio_batch *fbatch)
+{
+	unsigned int i;
+
+	for (i = 0; i < folio_batch_count(fbatch); i++)
+		folio_unqueue_deferred_split(fbatch->folios[i]);
+}
+
 static void lru_add(struct lruvec *lruvec, struct folio *folio)
 {
 	int was_unevictable = folio_test_clear_unevictable(folio);
@@ -177,14 +190,10 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
 		 *
 		 * We're bypassing normal cleanup. Clear flags that are not
 		 * applicable to dead folios.
-		 *
-		 * Since the folio may be part of a huge page, unqueue from
-		 * deferred split list to avoid a dangling list entry.
 		 */
 		if (is_lru_add && folio_ref_freeze(folio, 1)) {
 			__folio_clear_active(folio);
 			__folio_clear_unevictable(folio);
-			folio_unqueue_deferred_split(folio);
 			fbatch->folios[i] = NULL;
 			folio_batch_add(&free_fbatch, folio);
 			continue;
@@ -201,6 +210,11 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
 
 	/* Cleanup filtered dead folios. */
 	if (is_lru_add) {
+		/*
+		 * Since the folios may be part of a huge page, unqueue them
+		 * from the deferred split list to avoid a dangling list entry.
+		 */
+		folio_batch_unqueue_deferred_split(&free_fbatch);
 		mem_cgroup_uncharge_folios(&free_fbatch);
 		free_unref_folios(&free_fbatch);
 	}
@@ -1020,7 +1034,6 @@ void folios_put_refs(struct folio_batch *folios, unsigned int *refs)
 			free_huge_folio(folio);
 			continue;
 		}
-		folio_unqueue_deferred_split(folio);
 		__page_cache_release(folio, &lruvec, &flags);
 
 		if (j != i)
@@ -1035,6 +1048,7 @@ void folios_put_refs(struct folio_batch *folios, unsigned int *refs)
 	}
 
 	folios->nr = j;
+	folio_batch_unqueue_deferred_split(folios);
 	mem_cgroup_uncharge_folios(folios);
 	free_unref_folios(folios);
 }
diff --git a/mm/internal.h b/mm/internal.h
index 0ca863f2639a..ac72577a6e3e 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -760,6 +760,7 @@ static inline bool folio_unqueue_deferred_split(struct folio *folio)
 
 	return __folio_unqueue_deferred_split(folio);
 }
+void folio_batch_unqueue_deferred_split(struct folio_batch *fbatch);
 
 static inline struct folio *page_rmappable_folio(struct page *page)
 {
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 29b31446fc11..94859a830fb0 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2007,9 +2007,9 @@ static unsigned int move_folios_to_lru(struct list_head *list, bool do_rotate)
 		if (unlikely(folio_put_testzero(folio))) {
 			__folio_clear_lru_flags(folio);
 
-			folio_unqueue_deferred_split(folio);
 			if (folio_batch_add(&free_folios, folio) == 0) {
 				lruvec_unlock_irq(lruvec);
+				folio_batch_unqueue_deferred_split(&free_folios);
 				mem_cgroup_uncharge_folios(&free_folios);
 				free_unref_folios(&free_folios);
 				lruvec = NULL;
@@ -2041,6 +2041,7 @@ static unsigned int move_folios_to_lru(struct list_head *list, bool do_rotate)
 		lruvec_unlock_irq(lruvec);
 
 	if (free_folios.nr) {
+		folio_batch_unqueue_deferred_split(&free_folios);
 		mem_cgroup_uncharge_folios(&free_folios);
 		free_unref_folios(&free_folios);
 	}
-- 
2.52.0

Re: [PATCH v1] mm: unqueue deferred split folios outside the lruvec lock

[Lance Yang]

On Tue, Oct 13, 2026 at 03:13:07PM -0700, Joanne Koong wrote:
>move_folios_to_lru(), folio_batch_move_lru() and folios_put_refs() call
>folio_unqueue_deferred_split() on folios where the refcount has dropped
>to zero. In these paths, folio_unqueue_deferred_split() is called with
>the lruvec lock often held (move_folios_to_lru() always holds it and
>folio_batch_move_lru()/folios_put_refs() usually hold it for every folio
>in the batch except the first). However, it's not needed to hold the
>lruvec lock.
>
>The lruvec lock protects folio->lru and the reclaim LRU lists whereas
>the deferred split queue lives on folio->_deferred_list in
>deferred_split_lru. The deferred_split_lru is a list_lru with its own
>per-memcg per-node lock.
>
>There are already other existing callers that call
>folio_unqueue_deferred_split() without holding the lruvec lock (eg
>__folio_put(), shrink_folio_list(), __folio_migrate_mapping(), ...).
>
>Add folio_batch_unqueue_deferred_split() and use it in
>move_folios_to_lru(), folio_batch_move_lru(), and folios_put_refs() to
>unqueue each batch of zero-refcount folios from the deferred split list
>after the lruvec lock is released and before
>mem_cgroup_uncharge_folios(). This helps shorten the lruvec lock's
>critical sections, which run with IRQs disabled and are highly
>contended.

Nice work! Thanks!

BTW, have you seen measurable overhead from unqueueing deferred split
folios under the lruvec lock?

Any before/after numbers for this change, including the cost of the extra
pass over the batch?

Cheers, Lance

>Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
>---
> mm/folio.c    | 24 +++++++++++++++++++-----
> mm/internal.h |  1 +
> mm/vmscan.c   |  3 ++-
> 3 files changed, 22 insertions(+), 6 deletions(-)
>
>diff --git a/mm/folio.c b/mm/folio.c
>index 35e242b48870..85144078e635 100644
>--- a/mm/folio.c
>+++ b/mm/folio.c
>@@ -109,6 +109,19 @@ EXPORT_SYMBOL(__folio_put);
> 
> typedef void (*move_fn_t)(struct lruvec *lruvec, struct folio *folio);
> 
>+/*
>+ * Unqueue a batch of folios from the deferred split queue. This must be
>+ * called after the folios' refcounts have dropped to zero and before they
>+ * are uncharged (see comment above __folio_unqueue_deferred_split() for why).
>+ */
>+void folio_batch_unqueue_deferred_split(struct folio_batch *fbatch)
>+{
>+	unsigned int i;
>+
>+	for (i = 0; i < folio_batch_count(fbatch); i++)
>+		folio_unqueue_deferred_split(fbatch->folios[i]);
>+}
>+
> static void lru_add(struct lruvec *lruvec, struct folio *folio)
> {
> 	int was_unevictable = folio_test_clear_unevictable(folio);
>@@ -177,14 +190,10 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
> 		 *
> 		 * We're bypassing normal cleanup. Clear flags that are not
> 		 * applicable to dead folios.
>-		 *
>-		 * Since the folio may be part of a huge page, unqueue from
>-		 * deferred split list to avoid a dangling list entry.
> 		 */
> 		if (is_lru_add && folio_ref_freeze(folio, 1)) {
> 			__folio_clear_active(folio);
> 			__folio_clear_unevictable(folio);
>-			folio_unqueue_deferred_split(folio);
> 			fbatch->folios[i] = NULL;
> 			folio_batch_add(&free_fbatch, folio);
> 			continue;
>@@ -201,6 +210,11 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
> 
> 	/* Cleanup filtered dead folios. */
> 	if (is_lru_add) {
>+		/*
>+		 * Since the folios may be part of a huge page, unqueue them
>+		 * from the deferred split list to avoid a dangling list entry.
>+		 */
>+		folio_batch_unqueue_deferred_split(&free_fbatch);
> 		mem_cgroup_uncharge_folios(&free_fbatch);
> 		free_unref_folios(&free_fbatch);
> 	}
>@@ -1020,7 +1034,6 @@ void folios_put_refs(struct folio_batch *folios, unsigned int *refs)
> 			free_huge_folio(folio);
> 			continue;
> 		}
>-		folio_unqueue_deferred_split(folio);
> 		__page_cache_release(folio, &lruvec, &flags);
> 
> 		if (j != i)
>@@ -1035,6 +1048,7 @@ void folios_put_refs(struct folio_batch *folios, unsigned int *refs)
> 	}
> 
> 	folios->nr = j;
>+	folio_batch_unqueue_deferred_split(folios);
> 	mem_cgroup_uncharge_folios(folios);
> 	free_unref_folios(folios);
> }
>diff --git a/mm/internal.h b/mm/internal.h
>index 0ca863f2639a..ac72577a6e3e 100644
>--- a/mm/internal.h
>+++ b/mm/internal.h
>@@ -760,6 +760,7 @@ static inline bool folio_unqueue_deferred_split(struct folio *folio)
> 
> 	return __folio_unqueue_deferred_split(folio);
> }
>+void folio_batch_unqueue_deferred_split(struct folio_batch *fbatch);
> 
> static inline struct folio *page_rmappable_folio(struct page *page)
> {
>diff --git a/mm/vmscan.c b/mm/vmscan.c
>index 29b31446fc11..94859a830fb0 100644
>--- a/mm/vmscan.c
>+++ b/mm/vmscan.c
>@@ -2007,9 +2007,9 @@ static unsigned int move_folios_to_lru(struct list_head *list, bool do_rotate)
> 		if (unlikely(folio_put_testzero(folio))) {
> 			__folio_clear_lru_flags(folio);
> 
>-			folio_unqueue_deferred_split(folio);
> 			if (folio_batch_add(&free_folios, folio) == 0) {
> 				lruvec_unlock_irq(lruvec);
>+				folio_batch_unqueue_deferred_split(&free_folios);
> 				mem_cgroup_uncharge_folios(&free_folios);
> 				free_unref_folios(&free_folios);
> 				lruvec = NULL;
>@@ -2041,6 +2041,7 @@ static unsigned int move_folios_to_lru(struct list_head *list, bool do_rotate)
> 		lruvec_unlock_irq(lruvec);
> 
> 	if (free_folios.nr) {
>+		folio_batch_unqueue_deferred_split(&free_folios);
> 		mem_cgroup_uncharge_folios(&free_folios);
> 		free_unref_folios(&free_folios);
> 	}
>-- 
>2.52.0
>
>

Re: [PATCH v1] mm: unqueue deferred split folios outside the lruvec lock

[Barry Song]

On Wed, Oct 14, 2026 at 11:07 AM Lance Yang <lance.yang@linux.dev> wrote:
>
>
> On Tue, Oct 13, 2026 at 03:13:07PM -0700, Joanne Koong wrote:
> >move_folios_to_lru(), folio_batch_move_lru() and folios_put_refs() call
> >folio_unqueue_deferred_split() on folios where the refcount has dropped
> >to zero. In these paths, folio_unqueue_deferred_split() is called with
> >the lruvec lock often held (move_folios_to_lru() always holds it and
> >folio_batch_move_lru()/folios_put_refs() usually hold it for every folio
> >in the batch except the first). However, it's not needed to hold the
> >lruvec lock.
> >
> >The lruvec lock protects folio->lru and the reclaim LRU lists whereas
> >the deferred split queue lives on folio->_deferred_list in
> >deferred_split_lru. The deferred_split_lru is a list_lru with its own
> >per-memcg per-node lock.
> >
> >There are already other existing callers that call
> >folio_unqueue_deferred_split() without holding the lruvec lock (eg
> >__folio_put(), shrink_folio_list(), __folio_migrate_mapping(), ...).
> >
> >Add folio_batch_unqueue_deferred_split() and use it in
> >move_folios_to_lru(), folio_batch_move_lru(), and folios_put_refs() to
> >unqueue each batch of zero-refcount folios from the deferred split list
> >after the lruvec lock is released and before
> >mem_cgroup_uncharge_folios(). This helps shorten the lruvec lock's
> >critical sections, which run with IRQs disabled and are highly
> >contended.
>
> Nice work! Thanks!
>
> BTW, have you seen measurable overhead from unqueueing deferred split
> folios under the lruvec lock?
>
> Any before/after numbers for this change, including the cost of the extra
> pass over the batch?

I recall Joanne mentioning some data on this in the link below. Is that right?

https://lore.kernel.org/linux-mm/CAJnrk1a2PpUGTsCgS7jct2KET4K2zAL+_h+te8G1MaU9bXs99w@mail.gmail.com/

Best Regards
Barry

Re: [PATCH v1] mm: unqueue deferred split folios outside the lruvec lock

[Lance Yang]

On 2026/10/14 11:13, Barry Song wrote:
> On Wed, Oct 14, 2026 at 11:07 AM Lance Yang <lance.yang@linux.dev> wrote:
>>
>>
>> On Tue, Oct 13, 2026 at 03:13:07PM -0700, Joanne Koong wrote:
>>> move_folios_to_lru(), folio_batch_move_lru() and folios_put_refs() call
>>> folio_unqueue_deferred_split() on folios where the refcount has dropped
>>> to zero. In these paths, folio_unqueue_deferred_split() is called with
>>> the lruvec lock often held (move_folios_to_lru() always holds it and
>>> folio_batch_move_lru()/folios_put_refs() usually hold it for every folio
>>> in the batch except the first). However, it's not needed to hold the
>>> lruvec lock.
>>>
>>> The lruvec lock protects folio->lru and the reclaim LRU lists whereas
>>> the deferred split queue lives on folio->_deferred_list in
>>> deferred_split_lru. The deferred_split_lru is a list_lru with its own
>>> per-memcg per-node lock.
>>>
>>> There are already other existing callers that call
>>> folio_unqueue_deferred_split() without holding the lruvec lock (eg
>>> __folio_put(), shrink_folio_list(), __folio_migrate_mapping(), ...).
>>>
>>> Add folio_batch_unqueue_deferred_split() and use it in
>>> move_folios_to_lru(), folio_batch_move_lru(), and folios_put_refs() to
>>> unqueue each batch of zero-refcount folios from the deferred split list
>>> after the lruvec lock is released and before
>>> mem_cgroup_uncharge_folios(). This helps shorten the lruvec lock's
>>> critical sections, which run with IRQs disabled and are highly
>>> contended.
>>
>> Nice work! Thanks!
>>
>> BTW, have you seen measurable overhead from unqueueing deferred split
>> folios under the lruvec lock?
>>
>> Any before/after numbers for this change, including the cost of the extra
>> pass over the batch?
> 
> I recall Joanne mentioning some data on this in the link below. Is that right?
> 
> https://lore.kernel.org/linux-mm/CAJnrk1a2PpUGTsCgS7jct2KET4K2zAL+_h+te8G1MaU9bXs99w@mail.gmail.com/

Thanks, Barry! Yes, those numbers answer my question about the
observed overhead :)

Joanne, have you rerun that benchmark with this patch? I'd be
interested in the before/after runtime and lruvec lock wait.

Cheers, Lance

Re: [PATCH v1] mm: unqueue deferred split folios outside the lruvec lock

[David Hildenbrand (Arm)]

On 10/14/26 00:13, Joanne Koong wrote:
> move_folios_to_lru(), folio_batch_move_lru() and folios_put_refs() call
> folio_unqueue_deferred_split() on folios where the refcount has dropped
> to zero. In these paths, folio_unqueue_deferred_split() is called with
> the lruvec lock often held (move_folios_to_lru() always holds it and
> folio_batch_move_lru()/folios_put_refs() usually hold it for every folio
> in the batch except the first). However, it's not needed to hold the
> lruvec lock.
> 
> The lruvec lock protects folio->lru and the reclaim LRU lists whereas
> the deferred split queue lives on folio->_deferred_list in
> deferred_split_lru. The deferred_split_lru is a list_lru with its own
> per-memcg per-node lock.
> 
> There are already other existing callers that call
> folio_unqueue_deferred_split() without holding the lruvec lock (eg
> __folio_put(), shrink_folio_list(), __folio_migrate_mapping(), ...).
> 
> Add folio_batch_unqueue_deferred_split() and use it in
> move_folios_to_lru(), folio_batch_move_lru(), and folios_put_refs() to
> unqueue each batch of zero-refcount folios from the deferred split list
> after the lruvec lock is released and before
> mem_cgroup_uncharge_folios(). This helps shorten the lruvec lock's
> critical sections, which run with IRQs disabled and are highly
> contended.
> 
> Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
> ---
>  mm/folio.c    | 24 +++++++++++++++++++-----
>  mm/internal.h |  1 +
>  mm/vmscan.c   |  3 ++-
>  3 files changed, 22 insertions(+), 6 deletions(-)
> 
> diff --git a/mm/folio.c b/mm/folio.c
> index 35e242b48870..85144078e635 100644
> --- a/mm/folio.c
> +++ b/mm/folio.c
> @@ -109,6 +109,19 @@ EXPORT_SYMBOL(__folio_put);
>  
>  typedef void (*move_fn_t)(struct lruvec *lruvec, struct folio *folio);
>  
> +/*
> + * Unqueue a batch of folios from the deferred split queue. This must be
> + * called after the folios' refcounts have dropped to zero and before they
> + * are uncharged (see comment above __folio_unqueue_deferred_split() for why).
> + */
> +void folio_batch_unqueue_deferred_split(struct folio_batch *fbatch)
> +{
> +	unsigned int i;
> +
> +	for (i = 0; i < folio_batch_count(fbatch); i++)
> +		folio_unqueue_deferred_split(fbatch->folios[i]);
> +}
> +
>  static void lru_add(struct lruvec *lruvec, struct folio *folio)
>  {
>  	int was_unevictable = folio_test_clear_unevictable(folio);
> @@ -177,14 +190,10 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
>  		 *
>  		 * We're bypassing normal cleanup. Clear flags that are not
>  		 * applicable to dead folios.
> -		 *
> -		 * Since the folio may be part of a huge page, unqueue from
> -		 * deferred split list to avoid a dangling list entry.
>  		 */
>  		if (is_lru_add && folio_ref_freeze(folio, 1)) {
>  			__folio_clear_active(folio);
>  			__folio_clear_unevictable(folio);
> -			folio_unqueue_deferred_split(folio);
>  			fbatch->folios[i] = NULL;
>  			folio_batch_add(&free_fbatch, folio);
>  			continue;
> @@ -201,6 +210,11 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
>  
>  	/* Cleanup filtered dead folios. */
>  	if (is_lru_add) {
> +		/*
> +		 * Since the folios may be part of a huge page, unqueue them
> +		 * from the deferred split list to avoid a dangling list entry.
> +		 */

In general, this lgtm, thanks. I am confused about this existing comment you are
moving.

folios being *part of* a huge page? Huh? A folio might represent a huge page,
but what does the "part of" thing mean here.

Is it simply "Dequeue any large folios" ?

-- 
Cheers,

David