From: Kairui Song via B4 Relay <devnull+kasong.tencent.com@kernel.org>
To: linux-mm@kvack.org
Cc: Andrew Morton <akpm@linux-foundation.org>,
Johannes Weiner <hannes@cmpxchg.org>,
Muchun Song <muchun.song@linux.dev>,
Qi Zheng <qi.zheng@linux.dev>,
Ying Huang <ying.huang@linux.alibaba.com>,
Chris Li <chrisl@kernel.org>, Baoquan He <baoquan.he@linux.dev>,
Nico Pache <nico.pache@linux.dev>,
Usama Arif <usama.arif@linux.dev>,
Michal Hocko <mhocko@kernel.org>,
Roman Gushchin <roman.gushchin@linux.dev>,
Shakeel Butt <shakeel.butt@linux.dev>,
David Hildenbrand <david@kernel.org>,
Lorenzo Stoakes <ljs@kernel.org>,
"Liam R. Howlett" <liam@infradead.org>,
Barry Song <baohua@kernel.org>,
Axel Rasmussen <axelrasmussen@google.com>,
Yuanchu Xie <yuanchu@google.com>, Wei Xu <weixugc@google.com>,
Vlastimil Babka <vbabka@kernel.org>,
Suren Baghdasaryan <surenb@google.com>,
Nhat Pham <nphamcs@gmail.com>,
Youngjun Park <youngjun.park@lge.com>, Zi Yan <ziy@nvidia.com>,
Gregory Price <gourry@gourry.net>,
"Matthew Wilcox (Oracle)" <willy@infradead.org>,
Baolin Wang <baolin.wang@linux.alibaba.com>,
Ryan Roberts <ryan.roberts@arm.com>, Dev Jain <dev.jain@arm.com>,
Lance Yang <lance.yang@linux.dev>,
Hugh Dickins <hughd@google.com>, SJ Park <sj@kernel.org>,
David Rientjes <rientjes@google.com>,
Yu Zhao <yuzhao@google.com>, Vernon Yang <vernon2gm@gmail.com>,
Zicheng Wang <wangzicheng@honor.com>,
Chen Ridong <chenridong@xiaomi.com>,
Tal Zussman <tz2294@columbia.edu>,
Kairui Song <ryncsn@gmail.com>,
Xiang Liu <liuxiang.277@bytedance.com>,
Ehab Ababneh <ehab.ababneh@intel.com>,
Zhao Zhenzhuo <zhaozhengzhuo@uniontech.com>,
Lian Wang <lianux.mm@gmail.com>,
Joanne Koong <joannelkoong@gmail.com>,
Kunwu Chan <kunwu.chan@gmail.com>, Hui Zhu <hui.zhu@linux.dev>,
linux-kernel@vger.kernel.org, cgroups@vger.kernel.org,
damon@lists.linux.dev, linux-doc@vger.kernel.org,
Kairui Song <kasong@tencent.com>
Subject: [PATCH v5 03/17] mm/mglru: make generation page counters atomic
Date: Thu, 19 Nov 2026 05:03:35 +0800 [thread overview]
Message-ID: <20261119-mglru-fg-v5-3-9eee2fe97a2d@tencent.com> (raw)
In-Reply-To: <20261119-mglru-fg-v5-0-9eee2fe97a2d@tencent.com>
From: Kairui Song <kasong@tencent.com>
No feature change, convert them to atomic so we can update them without
holding the LRU lock. A temporary underflow is fine: readers clamp a
negative value to zero, and the counters only need to be eventually
consistent.
Tested-by: Zhao Zhenzhuo <zhaozhengzhuo@uniontech.com>
Signed-off-by: Kairui Song <kasong@tencent.com>
---
include/linux/mm_inline.h | 6 ++----
include/linux/mmzone.h | 2 +-
mm/vmscan.c | 26 +++++++++++++-------------
mm/vmscan.h | 2 +-
4 files changed, 17 insertions(+), 19 deletions(-)
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index 2edb9beef6d3..46beded51c7d 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -265,11 +265,9 @@ static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *foli
VM_WARN_ON_ONCE(old_gen == -1 && new_gen == -1);
if (old_gen >= 0)
- WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone],
- lrugen->nr_pages[old_gen][type][zone] - delta);
+ atomic_long_sub(delta, &lrugen->nr_pages[old_gen][type][zone]);
if (new_gen >= 0)
- WRITE_ONCE(lrugen->nr_pages[new_gen][type][zone],
- lrugen->nr_pages[new_gen][type][zone] + delta);
+ atomic_long_add(delta, &lrugen->nr_pages[new_gen][type][zone]);
/* addition */
if (old_gen < 0) {
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index c161d563ce8f..4c4aad95b0b6 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -554,7 +554,7 @@ struct lru_gen_folio {
/* the multi-gen LRU lists, lazily sorted on eviction */
struct list_head folios[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
/* the multi-gen LRU sizes, eventually consistent */
- long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
+ atomic_long_t nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
/* the exponential moving average of refaulted */
unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS];
/* the exponential moving average of evicted+protected */
diff --git a/mm/vmscan.c b/mm/vmscan.c
index fe1fe1439c7c..8d14831cdf83 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -3350,8 +3350,7 @@ static void reset_batch_size(struct lru_gen_mm_walk *walk)
continue;
walk->nr_pages[gen][type][zone] = 0;
- WRITE_ONCE(lrugen->nr_pages[gen][type][zone],
- lrugen->nr_pages[gen][type][zone] + delta);
+ atomic_long_add(delta, &lrugen->nr_pages[gen][type][zone]);
if (lru_gen_is_active(lruvec, gen))
lru += LRU_ACTIVE;
@@ -3971,11 +3970,8 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
break;
}
flush_lru_batch(head, &batch_end, target_list);
-
- WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone],
- lrugen->nr_pages[old_gen][type][zone] - delta);
- WRITE_ONCE(lrugen->nr_pages[target_gen][type][zone],
- lrugen->nr_pages[target_gen][type][zone] + delta);
+ atomic_long_sub(delta, &lrugen->nr_pages[old_gen][type][zone]);
+ atomic_long_add(delta, &lrugen->nr_pages[target_gen][type][zone]);
if (!remaining)
return false;
}
@@ -4093,8 +4089,8 @@ static bool inc_max_seq(struct lruvec *lruvec, unsigned long seq, int swappiness
for (type = 0; type < ANON_AND_FILE; type++) {
for (zone = 0; zone < MAX_NR_ZONES; zone++) {
enum lru_list lru = type * LRU_INACTIVE_FILE;
- long delta = lrugen->nr_pages[prev][type][zone] -
- lrugen->nr_pages[next][type][zone];
+ long delta = atomic_long_read(&lrugen->nr_pages[prev][type][zone]) -
+ atomic_long_read(&lrugen->nr_pages[next][type][zone]);
if (!delta)
continue;
@@ -4625,8 +4621,13 @@ static void __lru_gen_reparent_memcg(struct lruvec *child_lruvec, struct lruvec
child_lrugen = &child_lruvec->lrugen;
parent_lrugen = &parent_lruvec->lrugen;
+ /*
+ * The child is offline and the lockless updaters, which hold
+ * rcu_read_lock() over the update, avoid racing with reparenting,
+ * so the read/set/add operations below are safe.
+ */
for (gen = 0; gen < MAX_NR_GENS; gen++) {
- long nr_pages = child_lrugen->nr_pages[gen][type][zone];
+ long nr_pages = atomic_long_read(&child_lrugen->nr_pages[gen][type][zone]);
int child_lru_active = lru_gen_is_active(child_lruvec, gen) ? LRU_ACTIVE : 0;
int parent_lru_active = lru_gen_is_active(parent_lruvec, gen) ? LRU_ACTIVE : 0;
@@ -4634,9 +4635,8 @@ static void __lru_gen_reparent_memcg(struct lruvec *child_lruvec, struct lruvec
list_splice_tail_init(&child_lrugen->folios[gen][type][zone],
&parent_lrugen->folios[gen][type][zone]);
- WRITE_ONCE(child_lrugen->nr_pages[gen][type][zone], 0);
- WRITE_ONCE(parent_lrugen->nr_pages[gen][type][zone],
- parent_lrugen->nr_pages[gen][type][zone] + nr_pages);
+ atomic_long_set(&child_lrugen->nr_pages[gen][type][zone], 0);
+ atomic_long_add(nr_pages, &parent_lrugen->nr_pages[gen][type][zone]);
if (lru_gen_is_active(child_lruvec, gen) != lru_gen_is_active(parent_lruvec, gen)) {
__update_lru_size(child_lruvec, lru + child_lru_active, zone, -nr_pages);
diff --git a/mm/vmscan.h b/mm/vmscan.h
index c2cea36289f3..6551f0a6691a 100644
--- a/mm/vmscan.h
+++ b/mm/vmscan.h
@@ -126,7 +126,7 @@ static inline unsigned long lruvec_gen_size(struct lru_gen_folio *lrugen,
unsigned long size = 0;
for (int zone = 0; zone < max_zone; zone++)
- size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
+ size += max(atomic_long_read(&lrugen->nr_pages[gen][type][zone]), 0L);
return size;
}
--
2.56.1
next prev parent reply other threads:[~2026-11-18 21:03 UTC|newest]
Thread overview: 18+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-18 21:03 [PATCH v5 00/17] mm/mglru: frequency guided promotion (MGLRU-FG) and flag cleanup Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 01/17] mm/mglru: rename the generation offset in lru_gen_folio_seq() Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 02/17] mm/memcontrol: allow update of LRU statistic without holding LRU lock Kairui Song via B4 Relay
2026-11-18 21:03 ` Kairui Song via B4 Relay [this message]
2026-11-18 21:03 ` [PATCH v5 04/17] mm/memcg: add folio-based lruvec live helper Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 05/17] mm/mglru: skip the flags cmpxchg when nothing changes Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 06/17] mm/mglru: frequency guided workingset promotion (MGLRU-FG) Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 07/17] mm/mglru: make folio lru referenced times count a generic API Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 08/17] mm/mglru: move add/del LRU size accounting out of lru_gen_update_size() Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 09/17] mm/mglru, gup: mark folios referenced via a fast helper Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 10/17] mm/mglru, smaps: convert to LRU refs based operations for MGLRU Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 11/17] mm/damon: " Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 12/17] mm/khugepaged: " Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 13/17] mm/madvise: adapt for " Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 14/17] mm/huge_memory: mark folio as accessed more accurately on split Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 15/17] mm/mglru: folio LRU refs based active/inactive number accounting Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 16/17] mm/mglru: make folio_inc_lru_refs lruvec lockless Kairui Song via B4 Relay
2026-11-18 21:03 ` [PATCH v5 17/17] mm/mglru: make folio_test_workingset() work based on folio LRU refs Kairui Song via B4 Relay
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261119-mglru-fg-v5-3-9eee2fe97a2d@tencent.com \
--to=devnull+kasong.tencent.com@kernel.org \
--cc=akpm@linux-foundation.org \
--cc=axelrasmussen@google.com \
--cc=baohua@kernel.org \
--cc=baolin.wang@linux.alibaba.com \
--cc=baoquan.he@linux.dev \
--cc=cgroups@vger.kernel.org \
--cc=chenridong@xiaomi.com \
--cc=chrisl@kernel.org \
--cc=damon@lists.linux.dev \
--cc=david@kernel.org \
--cc=dev.jain@arm.com \
--cc=ehab.ababneh@intel.com \
--cc=gourry@gourry.net \
--cc=hannes@cmpxchg.org \
--cc=hughd@google.com \
--cc=hui.zhu@linux.dev \
--cc=joannelkoong@gmail.com \
--cc=kasong@tencent.com \
--cc=kunwu.chan@gmail.com \
--cc=lance.yang@linux.dev \
--cc=liam@infradead.org \
--cc=lianux.mm@gmail.com \
--cc=linux-doc@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=liuxiang.277@bytedance.com \
--cc=ljs@kernel.org \
--cc=mhocko@kernel.org \
--cc=muchun.song@linux.dev \
--cc=nico.pache@linux.dev \
--cc=nphamcs@gmail.com \
--cc=qi.zheng@linux.dev \
--cc=rientjes@google.com \
--cc=roman.gushchin@linux.dev \
--cc=ryan.roberts@arm.com \
--cc=ryncsn@gmail.com \
--cc=shakeel.butt@linux.dev \
--cc=sj@kernel.org \
--cc=surenb@google.com \
--cc=tz2294@columbia.edu \
--cc=usama.arif@linux.dev \
--cc=vbabka@kernel.org \
--cc=vernon2gm@gmail.com \
--cc=wangzicheng@honor.com \
--cc=weixugc@google.com \
--cc=willy@infradead.org \
--cc=ying.huang@linux.alibaba.com \
--cc=youngjun.park@lge.com \
--cc=yuanchu@google.com \
--cc=yuzhao@google.com \
--cc=zhaozhengzhuo@uniontech.com \
--cc=ziy@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®