From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 4231E43B4B5 for ; Wed, 12 Aug 2026 12:18:24 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786537105; cv=none; b=YbNkMtiE9Sii2tQWPRa+XhqOanomVR0/PXBSxCZw7AQSO4NHKgZhwTri7RJ+oGb4/EY6mhIau6hSCnaa2C1T/lgcg38F+FtGO6bPTELjjPCOb44Ds9uSyLfqbx4/jMBh57gdU1reFF5APPkMRmBXw2Zo3KgL4Iq7W2dLDAJkH04= ARC-Message-Signature:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786537105; c=relaxed/simple; bh=gnreT11CPAXENKEyEEEVDO1A02uJWW7lg7OOSQTUHfM=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=hXZ4ckUiB++062kRSljAKlxYeE6sPImGTicIlPiSZiVBYn45IfTiG7w7ZxjRAjbQuqo4F0DB85mSZ7VuoXlZ8MwLRMReec2v1zYytvscO1p1DjbaexForLuP9ABBJmFM5ZtyOoK7YEfV0ARGMxmTnkZyexqMjxSZvJw2UPMOyAE= ARC-Authentication-Results:i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=klp1mR5j; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="klp1mR5j" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 826091F00A3A; Wed, 12 Aug 2026 12:18:20 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1786537104; bh=YTbE3zCASBCCzvf13wSpm/yIS5j72vwri6qsbtqBEHs=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=klp1mR5jnzOgqpzhCOj0WFwg1FgPJMsPSYNOwa6lgfEEIQFXYfuMOitVf9M1XYkq6 IGUAspcT+VDB83shIuawyyQD76502CYv0vKgxgM4BxAefQK2MfCXbbO1iLLsINdsjh KlslEpEoitr+BYqJBjI3e/ETQkdLUZvQ4xyje68bjFyGmUyXvU1JnGORIXnpvvpr7m Jt4in8dWHpPjxlQL2+RyHHtT8kKLQsWuqaIaJ7j25YV47di0urpRAy+FuWXJYitzYx yFJE97Sh5w8wuy/bCnJh4CWWpjWg1Sp7HRetTJknpR4bxWXx1JwSfwSrSFrjdNKOQf bNTadbY0LBN+Q== From: "Barry Song (Xiaomi)" To: akpm@linux-foundation.org, linux-mm@kvack.org Cc: axelrasmussen@google.com, chenridong@xiaomi.com, david@kernel.org, hannes@cmpxchg.org, kasong@tencent.com, lianux.mm@gmail.com, linux-kernel@vger.kernel.org, ljs@kernel.org, lyugaofei@xiaomi.com, mhocko@kernel.org, qi.zheng@linux.dev, shakeel.butt@linux.dev, stevensd@chromium.org, wangzicheng@honor.com, weixugc@google.com, yuanchu@google.com, zhangbo56@xiaomi.com, baolin.wang@linux.alibaba.com, baoquan.he@linux.dev, "Barry Song (Xiaomi)" Subject: [RFC PATCH v4 05/16] mm/mglru: batch update lrugen->nr_pages in inc_min_seq() Date: Wed, 12 Aug 2026 20:16:47 +0800 Message-Id: <20260812121658.69965-6-baohua@kernel.org> X-Mailer: git-send-email 2.39.3 (Apple Git-146) In-Reply-To: <20260812121658.69965-1-baohua@kernel.org> References: <20260812121658.69965-1-baohua@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Currently, folio_inc_gen() updates lrugen->nr_pages for every folio as it advances generations. Instead, accumulate the size changes and update lrugen->nr_pages in a batch after scanning the entire oldest generation, or when the scan stops because remaining reaches zero. Since we only move folios from the oldest generation to the second oldest generation, the active/inactive state cannot change. We can therefore skip __lru_update_size(). Signed-off-by: Barry Song (Xiaomi) --- mm/vmscan.c | 46 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 2fd1e06eb192..c6e3b92c8cae 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3295,20 +3295,21 @@ static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma } /* protect pages accessed multiple times through file descriptors */ -static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio) +static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased) { - int type = folio_is_file_lru(folio); - struct lru_gen_folio *lrugen = &lruvec->lrugen; - int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f); + int new_gen; VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); do { new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; /* folio_update_gen() has promoted this page? */ - if (new_gen >= 0 && new_gen != old_gen) + if (new_gen >= 0 && new_gen != old_gen) { + if (increased) + *increased = false; return new_gen; + } new_gen = (old_gen + 1) % MAX_NR_GENS; @@ -3316,8 +3317,21 @@ static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio) new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags)); - lru_gen_update_size(lruvec, folio, old_gen, new_gen); + if (increased) + *increased = true; + return new_gen; +} +static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio) +{ + int type = folio_is_file_lru(folio); + struct lru_gen_folio *lrugen = &lruvec->lrugen; + int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); + bool gen_increased; + + new_gen = __folio_inc_gen(folio, old_gen, &gen_increased); + if (gen_increased) + lru_gen_update_size(lruvec, folio, old_gen, new_gen); return new_gen; } @@ -3903,6 +3917,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) struct lru_gen_folio *lrugen = &lruvec->lrugen; int hist = lru_hist_from_seq(lrugen->min_seq[type]); int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); + int target_gen = (old_gen + 1) % MAX_NR_GENS; /* For file type, skip the check if swappiness is anon only */ if (type && (swappiness == SWAPPINESS_ANON_ONLY)) @@ -3915,32 +3930,41 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) /* prevent cold/hot inversion if the type is evictable */ for (zone = 0; zone < MAX_NR_ZONES; zone++) { struct list_head *head = &lrugen->folios[old_gen][type][zone]; + unsigned long delta = 0; while (!list_empty(head)) { struct folio *folio = lru_to_folio(head); + long nr_pages = folio_nr_pages(folio); int refs = folio_lru_refs(folio); bool workingset = folio_test_workingset(folio); + bool gen_increased; VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); - new_gen = folio_inc_gen(lruvec, folio); + new_gen = __folio_inc_gen(folio, old_gen, &gen_increased); list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]); - + if (gen_increased) + delta += nr_pages; /* don't count the workingset being lazily promoted */ if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) { int tier = lru_tier_from_refs(refs, workingset); - int delta = folio_nr_pages(folio); WRITE_ONCE(lrugen->protected[hist][type][tier], - lrugen->protected[hist][type][tier] + delta); + lrugen->protected[hist][type][tier] + nr_pages); } if (!--remaining) - return false; + break; } + WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone], + lrugen->nr_pages[old_gen][type][zone] - delta); + WRITE_ONCE(lrugen->nr_pages[target_gen][type][zone], + lrugen->nr_pages[target_gen][type][zone] + delta); + if (!remaining) + return false; } done: reset_ctrl_pos(lruvec, type, true); -- 2.34.1