From: Usama Arif <usama.arif@linux.dev>
To: Andrew Morton <akpm@linux-foundation.org>,
david@kernel.org, chrisl@kernel.org, kasong@tencent.com,
ljs@kernel.org, ziy@nvidia.com, linux-mm@kvack.org
Cc: ying.huang@linux.alibaba.com, Baoquan He <baoquan.he@linux.dev>,
willy@infradead.org, youngjun.park@lge.com, hannes@cmpxchg.org,
riel@surriel.com, shakeel.butt@linux.dev, alex@ghiti.fr,
kas@kernel.org, baohua@kernel.org, dev.jain@arm.com,
baolin.wang@linux.alibaba.com, Nico Pache <nico.pache@linux.dev>,
Liam R. Howlett <liam@infradead.org>,
ryan.roberts@arm.com, Vlastimil Babka <vbabka@kernel.org>,
lance.yang@linux.dev, linux-kernel@vger.kernel.org,
nphamcs@gmail.com, shikemeng@huaweicloud.com, yosry@kernel.org,
qi.zheng@linux.dev, luizcap@redhat.com, kernel-team@meta.com,
Usama Arif <usama.arif@linux.dev>
Subject: [PATCH v8 10/30] mm: make PMD migration-entry splitting explicit
Date: Fri, 2 Oct 2026 02:52:24 -0700 [thread overview]
Message-ID: <20261002095503.3585565-11-usama.arif@linux.dev> (raw)
In-Reply-To: <20261002095503.3585565-1-usama.arif@linux.dev>
__split_huge_pmd() and friends take a "freeze" boolean that every caller
has to pass and almost every caller passes as false. The name says nothing
about what it selects, and the one thing it does select - PTE migration
entries instead of PTE mappings - is only ever wanted by the rmap migration
path.
Rename it to to_migration_entries, keep it private to mm/huge_memory.c,
and add split_pmd_to_migration_entries() for try_to_migrate_one(), the only
caller that wants it.
migrate_vma_split_unmapped_folio() also passed freeze=true, but only ever
runs on a PMD that is already a migration entry, which the generic helper
expands into PTE migration entries either way. Its folio_get() only existed
to balance the put_page() that freeze=true performs, so both go.
split_pmd_to_migration_entries() is only ever handed a present or
device-private PMD, so assert that in __split_huge_pmd_locked() instead of
silently skipping anything else.
Other than that assertion, no functional change intended.
Suggested-by: David Hildenbrand (Arm) <david@kernel.org>
Signed-off-by: Usama Arif <usama.arif@linux.dev>
Reviewed-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
---
include/linux/huge_mm.h | 21 ++++++------
mm/huge_memory.c | 74 ++++++++++++++++++++++++++---------------
mm/memory.c | 4 +--
mm/migrate_device.c | 7 +---
mm/mprotect.c | 2 +-
mm/rmap.c | 8 ++---
6 files changed, 65 insertions(+), 51 deletions(-)
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index 8ca0fa3be2acb..8205e83f27771 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -430,7 +430,7 @@ int folio_memcg_alloc_deferred(struct folio *folio);
void deferred_split_folio(struct folio *folio, bool partially_mapped);
void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
- unsigned long address, bool freeze);
+ unsigned long address);
/**
* pmd_is_huge() - Is this PMD either a huge PMD entry or a software leaf entry?
@@ -462,12 +462,10 @@ static inline bool pmd_is_huge(pmd_t pmd)
do { \
pmd_t *____pmd = (__pmd); \
if (pmd_is_huge(*____pmd)) \
- __split_huge_pmd(__vma, __pmd, __address, \
- false); \
+ __split_huge_pmd(__vma, __pmd, __address); \
} while (0)
-void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
- bool freeze);
+void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address);
void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
unsigned long address);
@@ -590,7 +588,9 @@ static inline bool thp_migration_supported(void)
}
void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address,
- pmd_t *pmd, bool freeze);
+ pmd_t *pmd);
+void split_pmd_to_migration_entries(struct vm_area_struct *vma,
+ unsigned long address, pmd_t *pmd);
bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr,
pmd_t *pmdp, struct folio *folio);
void map_anon_folio_pmd_nopf(struct folio *folio, pmd_t *pmd,
@@ -690,12 +690,13 @@ static inline void deferred_split_folio(struct folio *folio, bool partially_mapp
do { } while (0)
static inline void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
- unsigned long address, bool freeze) {}
+ unsigned long address) {}
static inline void split_huge_pmd_address(struct vm_area_struct *vma,
- unsigned long address, bool freeze) {}
+ unsigned long address) {}
static inline void split_huge_pmd_locked(struct vm_area_struct *vma,
- unsigned long address, pmd_t *pmd,
- bool freeze) {}
+ unsigned long address, pmd_t *pmd) {}
+static inline void split_pmd_to_migration_entries(struct vm_area_struct *vma,
+ unsigned long address, pmd_t *pmd) {}
static inline bool unmap_huge_pmd_locked(struct vm_area_struct *vma,
unsigned long addr, pmd_t *pmdp,
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index ee8d46827ffdc..1df4f619620b4 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -2033,7 +2033,7 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
pte_free(dst_mm, pgtable);
spin_unlock(src_ptl);
spin_unlock(dst_ptl);
- __split_huge_pmd(src_vma, src_pmd, addr, false);
+ __split_huge_pmd(src_vma, src_pmd, addr);
return -EAGAIN;
}
add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
@@ -2257,7 +2257,7 @@ vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf)
folio_unlock(folio);
spin_unlock(vmf->ptl);
fallback:
- __split_huge_pmd(vma, vmf->pmd, vmf->address, false);
+ __split_huge_pmd(vma, vmf->pmd, vmf->address);
return VM_FAULT_FALLBACK;
}
@@ -3190,7 +3190,7 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
}
static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
- unsigned long haddr, bool freeze)
+ unsigned long haddr, bool to_migration_entries)
{
struct mm_struct *mm = vma->vm_mm;
struct folio *folio;
@@ -3208,6 +3208,8 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma);
VM_WARN_ON_ONCE(!pmd_is_valid_softleaf(*pmd) && !pmd_trans_huge(*pmd));
+ VM_WARN_ON_ONCE(to_migration_entries && !pmd_present(*pmd) &&
+ !pmd_is_device_private_entry(*pmd));
count_vm_event(THP_SPLIT_PMD);
@@ -3291,10 +3293,10 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
* folios w.r.t anon exclusive handling. See the comments for
* folio handling and anon_exclusive below.
*/
- if (freeze && anon_exclusive &&
+ if (to_migration_entries && anon_exclusive &&
folio_try_share_anon_rmap_pmd(folio, page))
- freeze = false;
- if (!freeze) {
+ to_migration_entries = false;
+ if (!to_migration_entries) {
rmap_t rmap_flags = RMAP_NONE;
folio_ref_add(folio, HPAGE_PMD_NR - 1);
@@ -3344,13 +3346,14 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
/*
- * Without "freeze", we'll simply split the PMD, propagating the
- * PageAnonExclusive() flag for each PTE by setting it for
- * each subpage -- no need to (temporarily) clear.
+ * When not splitting to migration entries, we'll simply split
+ * the PMD and propagate the PageAnonExclusive() flag for each
+ * PTE by setting it for each page -- no need to (temporarily)
+ * clear.
*
- * With "freeze" we want to replace mapped pages by
- * migration entries right away. This is only possible if we
- * managed to clear PageAnonExclusive() -- see
+ * When splitting to migration entries, we want to replace
+ * mapped pages by migration entries right away. This is only
+ * possible if we managed to clear PageAnonExclusive() -- see
* set_pmd_migration_entry().
*
* In case we cannot clear PageAnonExclusive(), split the PMD
@@ -3359,10 +3362,10 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
* See folio_try_share_anon_rmap_pmd(): invalidate PMD first.
*/
anon_exclusive = PageAnonExclusive(page);
- if (freeze && anon_exclusive &&
+ if (to_migration_entries && anon_exclusive &&
folio_try_share_anon_rmap_pmd(folio, page))
- freeze = false;
- if (!freeze) {
+ to_migration_entries = false;
+ if (!to_migration_entries) {
rmap_t rmap_flags = RMAP_NONE;
folio_ref_add(folio, HPAGE_PMD_NR - 1);
@@ -3387,7 +3390,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
* Note that NUMA hinting access restrictions are not transferred to
* avoid any possibility of altering permissions across VMAs.
*/
- if (freeze || pmd_is_migration_entry(old_pmd)) {
+ if (to_migration_entries || pmd_is_migration_entry(old_pmd)) {
pte_t entry;
swp_entry_t swp_entry;
@@ -3420,8 +3423,8 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) {
/*
* anon_exclusive was already propagated to the relevant
- * pages corresponding to the pte entries when freeze
- * is false.
+ * pages corresponding to the pte entries when
+ * to_migration_entries is false.
*/
if (write)
swp_entry = make_writable_device_private_entry(
@@ -3469,7 +3472,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
if (!pmd_is_migration_entry(*pmd))
folio_remove_rmap_pmd(folio, page, vma);
- if (freeze)
+ if (to_migration_entries)
put_page(page);
smp_wmb(); /* make pte visible before pmd */
@@ -3477,15 +3480,33 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
}
void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address,
- pmd_t *pmd, bool freeze)
+ pmd_t *pmd)
{
VM_WARN_ON_ONCE(!IS_ALIGNED(address, HPAGE_PMD_SIZE));
if (pmd_trans_huge(*pmd) || pmd_is_valid_softleaf(*pmd))
- __split_huge_pmd_locked(vma, pmd, address, freeze);
+ __split_huge_pmd_locked(vma, pmd, address, false);
+}
+
+/**
+ * split_pmd_to_migration_entries() - Split a present or device private PMD into
+ * PTE migration entries.
+ * @vma: The VMA containing the PMD.
+ * @address: The PMD-aligned address the PMD maps.
+ * @pmd: A pointer to the leaf PMD entry.
+ *
+ * For the rmap migration walker, which only ever hands back those two entry
+ * types. Like split_huge_pmd_locked(), the caller must hold the PMD lock and
+ * must already be inside an mmu_notifier invalidate range.
+ */
+void split_pmd_to_migration_entries(struct vm_area_struct *vma,
+ unsigned long address, pmd_t *pmd)
+{
+ VM_WARN_ON_ONCE(!IS_ALIGNED(address, HPAGE_PMD_SIZE));
+ __split_huge_pmd_locked(vma, pmd, address, true);
}
void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
- unsigned long address, bool freeze)
+ unsigned long address)
{
spinlock_t *ptl;
struct mmu_notifier_range range;
@@ -3495,20 +3516,19 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
(address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE);
mmu_notifier_invalidate_range_start(&range);
ptl = pmd_lock(vma->vm_mm, pmd);
- split_huge_pmd_locked(vma, range.start, pmd, freeze);
+ split_huge_pmd_locked(vma, range.start, pmd);
spin_unlock(ptl);
mmu_notifier_invalidate_range_end(&range);
}
-void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
- bool freeze)
+void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address)
{
pmd_t *pmd = mm_find_pmd(vma->vm_mm, address);
if (!pmd)
return;
- __split_huge_pmd(vma, pmd, address, freeze);
+ __split_huge_pmd(vma, pmd, address);
}
static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address)
@@ -3520,7 +3540,7 @@ static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned
if (!IS_ALIGNED(address, HPAGE_PMD_SIZE) &&
range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PMD_SIZE),
ALIGN(address, HPAGE_PMD_SIZE)))
- split_huge_pmd_address(vma, address, false);
+ split_huge_pmd_address(vma, address);
}
void vma_adjust_trans_huge(struct vm_area_struct *vma,
diff --git a/mm/memory.c b/mm/memory.c
index 926276d419202..477d7e359b447 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2096,7 +2096,7 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
next = pmd_addr_end(addr, end);
if (pmd_is_huge(*pmd)) {
if (next - addr != HPAGE_PMD_SIZE)
- __split_huge_pmd(vma, pmd, addr, false);
+ __split_huge_pmd(vma, pmd, addr);
else if (zap_huge_pmd(tlb, vma, pmd, addr)) {
addr = next;
continue;
@@ -6382,7 +6382,7 @@ static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf)
split:
/* COW or write-notify handled on pte level: split pmd. */
- __split_huge_pmd(vma, vmf->pmd, vmf->address, false);
+ __split_huge_pmd(vma, vmf->pmd, vmf->address);
return VM_FAULT_FALLBACK;
}
diff --git a/mm/migrate_device.c b/mm/migrate_device.c
index 0c437004329d9..4a0b61d50d222 100644
--- a/mm/migrate_device.c
+++ b/mm/migrate_device.c
@@ -918,12 +918,7 @@ static int migrate_vma_split_unmapped_folio(struct migrate_vma *migrate,
unsigned long flags;
int ret = 0;
- /*
- * take a reference, since split_huge_pmd_address() with freeze = true
- * drops a reference at the end.
- */
- folio_get(folio);
- split_huge_pmd_address(migrate->vma, addr, true);
+ split_huge_pmd_address(migrate->vma, addr);
ret = folio_split_unmapped(folio, 0);
if (ret)
return ret;
diff --git a/mm/mprotect.c b/mm/mprotect.c
index 2888ee638d872..ee33bbb421008 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -530,7 +530,7 @@ static inline long change_pmd_range(struct mmu_gather *tlb,
if (pmd_is_huge(_pmd)) {
if ((next - addr != HPAGE_PMD_SIZE) ||
pgtable_split_needed(vma, cp_flags)) {
- __split_huge_pmd(vma, pmd, addr, false);
+ __split_huge_pmd(vma, pmd, addr);
/*
* For file-backed, the pmd could have been
* cleared; make sure pmd populated if
diff --git a/mm/rmap.c b/mm/rmap.c
index 5332c52909be1..b762eb85915ea 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -2289,8 +2289,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
* We temporarily have to drop the PTL and
* restart so we can process the PTE-mapped THP.
*/
- split_huge_pmd_locked(vma, pvmw.address,
- pvmw.pmd, false);
+ split_huge_pmd_locked(vma, pvmw.address, pvmw.pmd);
flags &= ~TTU_SPLIT_HUGE_PMD;
page_vma_mapped_walk_restart(&pvmw);
continue;
@@ -2515,13 +2514,12 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
if (flags & TTU_SPLIT_HUGE_PMD) {
/*
- * split_huge_pmd_locked() might leave the
+ * split_pmd_to_migration_entries() might leave the
* folio mapped through PTEs. Retry the walk
* so we can detect this scenario and properly
* abort the walk.
*/
- split_huge_pmd_locked(vma, pvmw.address,
- pvmw.pmd, true);
+ split_pmd_to_migration_entries(vma, pvmw.address, pvmw.pmd);
flags &= ~TTU_SPLIT_HUGE_PMD;
page_vma_mapped_walk_restart(&pvmw);
continue;
--
2.53.0-Meta
next prev parent reply other threads:[~2026-10-02 9:56 UTC|newest]
Thread overview: 34+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-10-02 9:52 [PATCH v8 00/30] mm: PMD-level swap entries for anonymous THPs Usama Arif
2026-10-02 9:52 ` [PATCH v8 01/30] mm: rename pmd_to_softleaf_folio() to pmd_softleaf_to_folio() Usama Arif
2026-10-02 9:52 ` [PATCH v8 02/30] arm64: mm: add PMD swap-exclusive helpers Usama Arif
2026-10-02 9:52 ` [PATCH v8 03/30] loongarch: " Usama Arif
2026-10-02 14:17 ` Huacai Chen
2026-10-02 9:52 ` [PATCH v8 04/30] powerpc: " Usama Arif
2026-10-02 9:52 ` [PATCH v8 05/30] riscv: " Usama Arif
2026-10-02 9:52 ` [PATCH v8 06/30] s390: " Usama Arif
2026-10-02 9:52 ` [PATCH v8 07/30] x86: " Usama Arif
2026-10-02 9:52 ` [PATCH v8 08/30] mm: recognize PMD swap entries in the softleaf layer Usama Arif
2026-10-02 9:52 ` [PATCH v8 09/30] mm/debug_vm_pgtable: test PMD swap-exclusive helpers Usama Arif
2026-10-02 9:52 ` Usama Arif [this message]
2026-10-02 9:52 ` [PATCH v8 11/30] mm: split PMD swap entries into PTE swap entries Usama Arif
2026-10-02 9:52 ` [PATCH v8 12/30] mm/swap: allow duplicating a range of " Usama Arif
2026-10-02 9:52 ` [PATCH v8 13/30] mm: handle PMD swap entries in fork path Usama Arif
2026-10-02 9:52 ` [PATCH v8 14/30] mm: zswap: reject high-order swap cache allocations backed by zswap Usama Arif
2026-10-02 9:52 ` [PATCH v8 15/30] mm: swap in PMD swap entries as whole THPs during swapoff Usama Arif
2026-10-02 9:52 ` [PATCH v8 16/30] fs/proc: account PMD swap entries in smaps Usama Arif
2026-10-02 9:52 ` [PATCH v8 17/30] mm: handle soft-dirty and uffd-wp on PMD swap entries Usama Arif
2026-10-02 9:52 ` [PATCH v8 18/30] mm/hmm: fault PMD swap entries on demand Usama Arif
2026-10-02 9:52 ` [PATCH v8 19/30] mm: free PMD swap entries in zap_huge_pmd() Usama Arif
2026-10-02 9:52 ` [PATCH v8 20/30] mm/madvise: free PMD swap entries with MADV_FREE Usama Arif
2026-10-02 9:52 ` [PATCH v8 21/30] mm/madvise: skip PMD swap entries for MADV_COLD and MADV_PAGEOUT Usama Arif
2026-10-02 9:52 ` [PATCH v8 22/30] mm/madvise: keep PMD swap entries whole for MADV_GUARD_INSTALL/REMOVE Usama Arif
2026-10-02 9:52 ` [PATCH v8 23/30] mm/mincore: report PMD swap-cache residency Usama Arif
2026-10-02 9:52 ` [PATCH v8 24/30] mm/khugepaged: treat PMD swap entries as mapped THPs Usama Arif
2026-10-02 9:52 ` [PATCH v8 25/30] mm: handle PMD swap entries in MADV_WILLNEED Usama Arif
2026-10-02 9:52 ` [PATCH v8 26/30] mm: handle PMD swap entries in UFFDIO_MOVE Usama Arif
2026-10-02 9:52 ` [PATCH v8 27/30] mm: don't PTE-batch a swap-in over a hardware-poisoned subpage Usama Arif
2026-10-02 9:52 ` [PATCH v8 28/30] mm: handle PMD swap entry faults on swap-in Usama Arif
2026-10-02 9:52 ` [PATCH v8 29/30] mm: install PMD swap entries on swap-out Usama Arif
2026-10-02 9:52 ` [PATCH v8 30/30] selftests/mm: add PMD swap entry tests Usama Arif
2026-10-02 14:28 ` [PATCH v8 00/30] mm: PMD-level swap entries for anonymous THPs David Hildenbrand (Arm)
2026-10-02 15:13 ` Zi Yan
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261002095503.3585565-11-usama.arif@linux.dev \
--to=usama.arif@linux.dev \
--cc=akpm@linux-foundation.org \
--cc=alex@ghiti.fr \
--cc=baohua@kernel.org \
--cc=baolin.wang@linux.alibaba.com \
--cc=baoquan.he@linux.dev \
--cc=chrisl@kernel.org \
--cc=david@kernel.org \
--cc=dev.jain@arm.com \
--cc=hannes@cmpxchg.org \
--cc=kas@kernel.org \
--cc=kasong@tencent.com \
--cc=kernel-team@meta.com \
--cc=lance.yang@linux.dev \
--cc=liam@infradead.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=ljs@kernel.org \
--cc=luizcap@redhat.com \
--cc=nico.pache@linux.dev \
--cc=nphamcs@gmail.com \
--cc=qi.zheng@linux.dev \
--cc=riel@surriel.com \
--cc=ryan.roberts@arm.com \
--cc=shakeel.butt@linux.dev \
--cc=shikemeng@huaweicloud.com \
--cc=vbabka@kernel.org \
--cc=willy@infradead.org \
--cc=ying.huang@linux.alibaba.com \
--cc=yosry@kernel.org \
--cc=youngjun.park@lge.com \
--cc=ziy@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®