* [PATCH v2 1/7] kmsan: encapsulate per-page metadata access
2026-11-17 12:06 [PATCH v2 0/7] KMSAN: Move per-page metadata out of struct page Muchun Song
@ 2026-11-17 12:06 ` Muchun Song
2026-11-17 12:06 ` [PATCH v2 2/7] mm/page_ext: fix client data alignment Muchun Song
` (6 subsequent siblings)
7 siblings, 0 replies; 19+ messages in thread
From: Muchun Song @ 2026-11-17 12:06 UTC (permalink / raw)
To: Alexander Potapenko, Andrew Morton
Cc: Marco Elver, Dmitry Vyukov, Chris Down, David Hildenbrand,
Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Johannes Weiner,
Brendan Jackman, Zi Yan, Muchun Song, Joerg Roedel, Lu Baolu,
Pranjal Shrivastava, Ketan, Mostafa Saleh, Sang-Heon Jeon,
Jonathan Corbet, Shuah Khan, Randy Dunlap, Sashiko, kasan-dev,
workflows, linux-doc, linux-kernel, linux-mm, Muchun Song
KMSAN directly accesses page->kmsan_shadow and page->kmsan_origin
throughout the shadow implementation. This couples these users to
storing the metadata pointers in struct page, requiring every access
site to change when the storage moves elsewhere.
Introduce helpers that get and set both metadata page pointers as a pair
and convert the existing users to them.
kmsan_get_shadow_origin_ptr() is the main metadata lookup path for
compiler-instrumented memory accesses and requires both pointers. For
page-backed addresses, resolve the backing page once and obtain the
metadata pair through one lookup instead of calling the single-metadata
path twice. This also avoids duplicate lookups after moving the metadata
storage out of struct page.
No functional change is intended.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
v2:
- Use one page metadata lookup for shadow and origin pointers (suggested
by Chris Down)
---
mm/kmsan/shadow.c | 166 +++++++++++++++++++++++++++++++---------------
1 file changed, 111 insertions(+), 55 deletions(-)
diff --git a/mm/kmsan/shadow.c b/mm/kmsan/shadow.c
index 0c88d89bf0d6..300b48818af1 100644
--- a/mm/kmsan/shadow.c
+++ b/mm/kmsan/shadow.c
@@ -9,6 +9,7 @@
#include <asm/kmsan.h>
#include <asm/tlbflush.h>
+#include <linux/bits.h>
#include <linux/cacheflush.h>
#include <linux/memblock.h>
#include <linux/mm_types.h>
@@ -20,29 +21,46 @@
#include "../vmalloc.h"
#include "kmsan.h"
-#define shadow_page_for(page) ((page)->kmsan_shadow)
+#define KMSAN_META_MASK_SHADOW BIT(0)
+#define KMSAN_META_MASK_ORIGIN BIT(1)
-#define origin_page_for(page) ((page)->kmsan_origin)
+struct page_metadata {
+ struct page *shadow;
+ struct page *origin;
+};
-static void *shadow_ptr_for(struct page *page)
+static inline struct page_metadata get_page_metadata(struct page *page)
{
- return page_address(shadow_page_for(page));
+ return (struct page_metadata) {
+ .shadow = page->kmsan_shadow,
+ .origin = page->kmsan_origin,
+ };
}
-static void *origin_ptr_for(struct page *page)
+static inline void set_page_metadata(struct page *page, struct page_metadata meta)
{
- return page_address(origin_page_for(page));
+ page->kmsan_shadow = meta.shadow;
+ page->kmsan_origin = meta.origin;
}
-static bool page_has_metadata(struct page *page)
+static inline void *shadow_ptr_for(struct page_metadata meta)
{
- return shadow_page_for(page) && origin_page_for(page);
+ return page_address(meta.shadow);
+}
+
+static inline void *origin_ptr_for(struct page_metadata meta)
+{
+ return page_address(meta.origin);
+}
+
+static inline bool page_has_metadata(struct page_metadata meta)
+{
+ return meta.shadow && meta.origin;
}
static void set_no_shadow_origin_page(struct page *page)
{
- shadow_page_for(page) = NULL;
- origin_page_for(page) = NULL;
+ set_page_metadata(page, (struct page_metadata) {});
}
/*
@@ -79,11 +97,56 @@ static struct page *virt_to_page_or_null(void *vaddr)
return NULL;
}
+static __always_inline
+struct shadow_origin_ptr __kmsan_get_metadata(void *address, unsigned int mask)
+{
+ struct shadow_origin_ptr ret = {};
+ struct page_metadata meta;
+ struct page *page;
+ u64 addr = (u64)address;
+ u64 origin_addr = ALIGN_DOWN(addr, KMSAN_ORIGIN_SIZE);
+
+ if (kmsan_internal_is_vmalloc_addr(address) ||
+ kmsan_internal_is_module_addr(address)) {
+ if (mask & KMSAN_META_MASK_SHADOW)
+ ret.shadow = (void *)vmalloc_meta(address, KMSAN_META_SHADOW);
+ if (mask & KMSAN_META_MASK_ORIGIN)
+ ret.origin = (void *)vmalloc_meta((void *)origin_addr,
+ KMSAN_META_ORIGIN);
+ return ret;
+ }
+
+ if (mask & KMSAN_META_MASK_SHADOW) {
+ ret.shadow = arch_kmsan_get_meta_or_null(address, KMSAN_META_SHADOW);
+ if (!ret.shadow)
+ goto lookup_page;
+ }
+ if (mask & KMSAN_META_MASK_ORIGIN)
+ ret.origin = arch_kmsan_get_meta_or_null((void *)origin_addr,
+ KMSAN_META_ORIGIN);
+ if (ret.shadow || ret.origin)
+ return ret;
+
+lookup_page:
+ page = virt_to_page_or_null(address);
+ if (!page)
+ return ret;
+ meta = get_page_metadata(page);
+ if (!page_has_metadata(meta))
+ return ret;
+
+ if (mask & KMSAN_META_MASK_SHADOW)
+ ret.shadow = shadow_ptr_for(meta) + offset_in_page(addr);
+ if (mask & KMSAN_META_MASK_ORIGIN)
+ ret.origin = origin_ptr_for(meta) + offset_in_page(origin_addr);
+
+ return ret;
+}
+
struct shadow_origin_ptr kmsan_get_shadow_origin_ptr(void *address, u64 size,
bool store)
{
struct shadow_origin_ptr ret;
- void *shadow;
/*
* Even if we redirect this memory access to the dummy page, it will
@@ -95,12 +158,10 @@ struct shadow_origin_ptr kmsan_get_shadow_origin_ptr(void *address, u64 size,
goto return_dummy;
KMSAN_WARN_ON(!kmsan_metadata_is_contiguous(address, size));
- shadow = kmsan_get_metadata(address, KMSAN_META_SHADOW);
- if (!shadow)
+ ret = __kmsan_get_metadata(address, KMSAN_META_MASK_SHADOW |
+ KMSAN_META_MASK_ORIGIN);
+ if (!ret.shadow)
goto return_dummy;
-
- ret.shadow = shadow;
- ret.origin = kmsan_get_metadata(address, KMSAN_META_ORIGIN);
return ret;
return_dummy:
@@ -124,46 +185,36 @@ struct shadow_origin_ptr kmsan_get_shadow_origin_ptr(void *address, u64 size,
*/
void *kmsan_get_metadata(void *address, bool is_origin)
{
- u64 addr = (u64)address, off;
- struct page *page;
- void *ret;
-
- if (is_origin)
- addr = ALIGN_DOWN(addr, KMSAN_ORIGIN_SIZE);
- address = (void *)addr;
- if (kmsan_internal_is_vmalloc_addr(address) ||
- kmsan_internal_is_module_addr(address))
- return (void *)vmalloc_meta(address, is_origin);
-
- ret = arch_kmsan_get_meta_or_null(address, is_origin);
- if (ret)
- return ret;
-
- page = virt_to_page_or_null(address);
- if (!page)
- return NULL;
- if (!page_has_metadata(page))
- return NULL;
- off = offset_in_page(addr);
+ struct shadow_origin_ptr ret;
+ unsigned int mask;
- return (is_origin ? origin_ptr_for(page) : shadow_ptr_for(page)) + off;
+ mask = is_origin ? KMSAN_META_MASK_ORIGIN : KMSAN_META_MASK_SHADOW;
+ ret = __kmsan_get_metadata(address, mask);
+ return is_origin ? ret.origin : ret.shadow;
}
void kmsan_copy_page_meta(struct page *dst, struct page *src)
{
+ struct page_metadata dst_meta, src_meta;
+
if (!kmsan_enabled || kmsan_in_runtime())
return;
- if (!dst || !page_has_metadata(dst))
+ if (!dst)
return;
- if (!src || !page_has_metadata(src)) {
+ dst_meta = get_page_metadata(dst);
+ if (!page_has_metadata(dst_meta))
+ return;
+ if (src)
+ src_meta = get_page_metadata(src);
+ if (!src || !page_has_metadata(src_meta)) {
kmsan_internal_unpoison_memory(page_address(dst), PAGE_SIZE,
/*checked*/ false);
return;
}
kmsan_enter_runtime();
- __memcpy(shadow_ptr_for(dst), shadow_ptr_for(src), PAGE_SIZE);
- __memcpy(origin_ptr_for(dst), origin_ptr_for(src), PAGE_SIZE);
+ __memcpy(shadow_ptr_for(dst_meta), shadow_ptr_for(src_meta), PAGE_SIZE);
+ __memcpy(origin_ptr_for(dst_meta), origin_ptr_for(src_meta), PAGE_SIZE);
kmsan_leave_runtime();
}
EXPORT_SYMBOL(kmsan_copy_page_meta);
@@ -171,19 +222,18 @@ EXPORT_SYMBOL(kmsan_copy_page_meta);
void kmsan_alloc_page(struct page *page, unsigned int order, gfp_t flags)
{
bool initialized = (flags & __GFP_ZERO) || !kmsan_enabled;
- struct page *shadow, *origin;
+ struct page_metadata meta;
depot_stack_handle_t handle;
int pages = 1 << order;
if (!page)
return;
- shadow = shadow_page_for(page);
- origin = origin_page_for(page);
+ meta = get_page_metadata(page);
if (initialized) {
- __memset(page_address(shadow), 0, PAGE_SIZE * pages);
- __memset(page_address(origin), 0, PAGE_SIZE * pages);
+ __memset(shadow_ptr_for(meta), 0, PAGE_SIZE * pages);
+ __memset(origin_ptr_for(meta), 0, PAGE_SIZE * pages);
return;
}
@@ -191,7 +241,7 @@ void kmsan_alloc_page(struct page *page, unsigned int order, gfp_t flags)
if (kmsan_in_runtime())
return;
- __memset(page_address(shadow), -1, PAGE_SIZE * pages);
+ __memset(shadow_ptr_for(meta), -1, PAGE_SIZE * pages);
kmsan_enter_runtime();
handle = kmsan_save_stack_with_flags(flags, /*extra_bits*/ 0);
kmsan_leave_runtime();
@@ -200,7 +250,7 @@ void kmsan_alloc_page(struct page *page, unsigned int order, gfp_t flags)
* to just fill the origin pages with @handle.
*/
for (int i = 0; i < PAGE_SIZE * pages / sizeof(handle); i++)
- ((depot_stack_handle_t *)page_address(origin))[i] = handle;
+ ((depot_stack_handle_t *)origin_ptr_for(meta))[i] = handle;
}
void kmsan_free_page(struct page *page, unsigned int order)
@@ -238,8 +288,10 @@ int kmsan_vmap_pages_range_noflush(unsigned long start, unsigned long end,
goto ret;
}
for (int i = 0; i < nr; i++) {
- s_pages[i] = shadow_page_for(pages[i]);
- o_pages[i] = origin_page_for(pages[i]);
+ struct page_metadata meta = get_page_metadata(pages[i]);
+
+ s_pages[i] = meta.shadow;
+ o_pages[i] = meta.origin;
}
prot = PAGE_KERNEL;
@@ -289,10 +341,12 @@ void __init kmsan_init_alloc_meta_for_range(void *start, void *end)
page = virt_to_page_or_null((char *)start + addr);
shadow_p = virt_to_page((char *)shadow + addr);
set_no_shadow_origin_page(shadow_p);
- shadow_page_for(page) = shadow_p;
origin_p = virt_to_page((char *)origin + addr);
set_no_shadow_origin_page(origin_p);
- origin_page_for(page) = origin_p;
+ set_page_metadata(page, (struct page_metadata) {
+ .shadow = shadow_p,
+ .origin = origin_p,
+ });
}
}
@@ -302,7 +356,9 @@ void kmsan_setup_meta(struct page *page, struct page *shadow,
for (int i = 0; i < (1 << order); i++) {
set_no_shadow_origin_page(&shadow[i]);
set_no_shadow_origin_page(&origin[i]);
- shadow_page_for(&page[i]) = &shadow[i];
- origin_page_for(&page[i]) = &origin[i];
+ set_page_metadata(&page[i], (struct page_metadata) {
+ .shadow = &shadow[i],
+ .origin = &origin[i],
+ });
}
}
--
2.54.0
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH v2 2/7] mm/page_ext: fix client data alignment
2026-11-17 12:06 [PATCH v2 0/7] KMSAN: Move per-page metadata out of struct page Muchun Song
2026-11-17 12:06 ` [PATCH v2 1/7] kmsan: encapsulate per-page metadata access Muchun Song
@ 2026-11-17 12:06 ` Muchun Song
2026-11-17 12:06 ` [PATCH v2 3/7] mm/page_ext: always use vmalloc for section page extensions Muchun Song
` (5 subsequent siblings)
7 siblings, 0 replies; 19+ messages in thread
From: Muchun Song @ 2026-11-17 12:06 UTC (permalink / raw)
To: Alexander Potapenko, Andrew Morton
Cc: Marco Elver, Dmitry Vyukov, Chris Down, David Hildenbrand,
Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Johannes Weiner,
Brendan Jackman, Zi Yan, Muchun Song, Joerg Roedel, Lu Baolu,
Pranjal Shrivastava, Ketan, Mostafa Saleh, Sang-Heon Jeon,
Jonathan Corbet, Shuah Khan, Randy Dunlap, Sashiko, kasan-dev,
workflows, linux-doc, linux-kernel, linux-mm, Muchun Song
page_ext lays out client payloads consecutively without aligning their
offsets or the final entry size. This happened to work while the active
payload sizes kept the stride naturally aligned, but IOMMU debug page
allocation added a four-byte payload after data requiring eight-byte
alignment.
When page owner and IOMMU debug page allocation are both enabled on
x86-64, struct page_ext, struct page_owner, and the IOMMU payload use
8, 64, and 4 bytes respectively. This produces a 76-byte entry stride,
leaving struct page_ext and page_owner data in every other entry only
four-byte aligned instead of eight-byte aligned.
Derive a suitable alignment for each payload from the lowest set bit of
its size, capped at unsigned long alignment. Align each client offset
and round the final entry size to the largest active alignment so every
entry preserves the alignment required by all active clients.
Fixes: 466ae6978a5b ("iommu: Add page_ext for IOMMU_DEBUG_PAGEALLOC")
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
v2:
- New patch to fix the existing page_ext client alignment bug
---
mm/page_ext.c | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git a/mm/page_ext.c b/mm/page_ext.c
index b679a8c1f7d7..dbc1bfa258f1 100644
--- a/mm/page_ext.c
+++ b/mm/page_ext.c
@@ -116,8 +116,22 @@ static int __init setup_early_page_ext(char *str)
}
early_param("early_page_ext", setup_early_page_ext);
+static inline size_t page_ext_data_alignment(size_t size)
+{
+ if (!size)
+ return 1;
+
+ /*
+ * A type's size is a multiple of its natural alignment. The lowest
+ * set bit of the size is therefore at least that alignment. Cap it
+ * at unsigned long alignment to avoid excessive per-page padding.
+ */
+ return min_t(size_t, 1UL << __ffs(size), __alignof__(unsigned long));
+}
+
static bool __init invoke_need_callbacks(void)
{
+ size_t max_align = 1;
int i;
int entries = ARRAY_SIZE(page_ext_ops);
bool need = false;
@@ -126,6 +140,7 @@ static bool __init invoke_need_callbacks(void)
if (page_ext_ops[i]->need()) {
if (page_ext_ops[i]->need_shared_flags) {
page_ext_size = sizeof(struct page_ext);
+ max_align = __alignof__(struct page_ext);
break;
}
}
@@ -133,11 +148,17 @@ static bool __init invoke_need_callbacks(void)
for (i = 0; i < entries; i++) {
if (page_ext_ops[i]->need()) {
+ size_t align;
+
+ align = page_ext_data_alignment(page_ext_ops[i]->size);
+ page_ext_size = ALIGN(page_ext_size, align);
page_ext_ops[i]->offset = page_ext_size;
page_ext_size += page_ext_ops[i]->size;
+ max_align = max(max_align, align);
need = true;
}
}
+ page_ext_size = ALIGN(page_ext_size, max_align);
return need;
}
--
2.54.0
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH v2 3/7] mm/page_ext: always use vmalloc for section page extensions
2026-11-17 12:06 [PATCH v2 0/7] KMSAN: Move per-page metadata out of struct page Muchun Song
2026-11-17 12:06 ` [PATCH v2 1/7] kmsan: encapsulate per-page metadata access Muchun Song
2026-11-17 12:06 ` [PATCH v2 2/7] mm/page_ext: fix client data alignment Muchun Song
@ 2026-11-17 12:06 ` Muchun Song
2026-11-18 13:43 ` Mike Rapoport
2026-11-17 12:06 ` [PATCH v2 4/7] mm/page_ext: use common allocation and initialization entry points Muchun Song
` (4 subsequent siblings)
7 siblings, 1 reply; 19+ messages in thread
From: Muchun Song @ 2026-11-17 12:06 UTC (permalink / raw)
To: Alexander Potapenko, Andrew Morton
Cc: Marco Elver, Dmitry Vyukov, Chris Down, David Hildenbrand,
Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Johannes Weiner,
Brendan Jackman, Zi Yan, Muchun Song, Joerg Roedel, Lu Baolu,
Pranjal Shrivastava, Ketan, Mostafa Saleh, Sang-Heon Jeon,
Jonathan Corbet, Shuah Khan, Randy Dunlap, Sashiko, kasan-dev,
workflows, linux-doc, linux-kernel, linux-mm, Muchun Song
SPARSEMEM currently tries to allocate each section's page extensions
from the buddy allocator before falling back to vmalloc. Boot-time
section allocation happens after vmalloc is available, and memory
hotplug allocations do not require physically contiguous backing.
Use vzalloc_node() directly and remove the buddy-specific allocation,
free, and kmemleak bookkeeping. This leaves one runtime backing type
and lets a later change add memblock as the distinct boot-time path
to simplify the code.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
v2:
- Add a preparatory cleanup that always uses vmalloc for SPARSEMEM
section page extensions (suggested by Mike Rapoport)
---
mm/page_ext.c | 21 +++------------------
1 file changed, 3 insertions(+), 18 deletions(-)
diff --git a/mm/page_ext.c b/mm/page_ext.c
index dbc1bfa258f1..3b581242fd52 100644
--- a/mm/page_ext.c
+++ b/mm/page_ext.c
@@ -285,15 +285,9 @@ static struct page_ext *lookup_page_ext(const struct page *page)
static void *__meminit alloc_page_ext(size_t size, int nid)
{
- gfp_t flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN;
- void *addr = NULL;
-
- addr = alloc_pages_exact_nid(nid, size, flags);
- if (addr)
- kmemleak_alloc(addr, size, 1, flags);
- else
- addr = vzalloc_node(size, nid);
+ void *addr;
+ addr = vzalloc_node(size, nid);
if (addr)
memmap_pages_add(DIV_ROUND_UP(size, PAGE_SIZE));
@@ -339,19 +333,10 @@ static int __meminit init_section_page_ext(unsigned long pfn, int nid)
static void free_page_ext(void *addr)
{
size_t table_size;
- struct page *page;
table_size = page_ext_size * PAGES_PER_SECTION;
memmap_pages_add(-1L * (DIV_ROUND_UP(table_size, PAGE_SIZE)));
-
- if (is_vmalloc_addr(addr)) {
- vfree(addr);
- } else {
- page = virt_to_page(addr);
- BUG_ON(PageReserved(page));
- kmemleak_free(addr);
- free_pages_exact(addr, table_size);
- }
+ vfree(addr);
}
static void __free_page_ext(unsigned long pfn)
--
2.54.0
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [PATCH v2 3/7] mm/page_ext: always use vmalloc for section page extensions
2026-11-17 12:06 ` [PATCH v2 3/7] mm/page_ext: always use vmalloc for section page extensions Muchun Song
@ 2026-11-18 13:43 ` Mike Rapoport
0 siblings, 0 replies; 19+ messages in thread
From: Mike Rapoport @ 2026-11-18 13:43 UTC (permalink / raw)
To: Muchun Song
Cc: Alexander Potapenko, Andrew Morton, Marco Elver, Dmitry Vyukov,
Chris Down, David Hildenbrand, Liam R . Howlett, Vlastimil Babka,
Suren Baghdasaryan, Michal Hocko, Johannes Weiner,
Brendan Jackman, Zi Yan, Muchun Song, Joerg Roedel, Lu Baolu,
Pranjal Shrivastava, Ketan, Mostafa Saleh, Sang-Heon Jeon,
Jonathan Corbet, Shuah Khan, Randy Dunlap, Sashiko, kasan-dev,
workflows, linux-doc, linux-kernel, linux-mm
On Tue, Nov 17, 2026 at 08:06:23PM +0800, Muchun Song wrote:
> SPARSEMEM currently tries to allocate each section's page extensions
> from the buddy allocator before falling back to vmalloc. Boot-time
> section allocation happens after vmalloc is available, and memory
> hotplug allocations do not require physically contiguous backing.
>
> Use vzalloc_node() directly and remove the buddy-specific allocation,
> free, and kmemleak bookkeeping. This leaves one runtime backing type
> and lets a later change add memblock as the distinct boot-time path
> to simplify the code.
>
> Signed-off-by: Muchun Song <songmuchun@bytedance.com>
Reviewed-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
> ---
> v2:
> - Add a preparatory cleanup that always uses vmalloc for SPARSEMEM
> section page extensions (suggested by Mike Rapoport)
> ---
> mm/page_ext.c | 21 +++------------------
> 1 file changed, 3 insertions(+), 18 deletions(-)
--
Sincerely yours,
Mike.
^ permalink raw reply [flat|nested] 19+ messages in thread
* [PATCH v2 4/7] mm/page_ext: use common allocation and initialization entry points
2026-11-17 12:06 [PATCH v2 0/7] KMSAN: Move per-page metadata out of struct page Muchun Song
` (2 preceding siblings ...)
2026-11-17 12:06 ` [PATCH v2 3/7] mm/page_ext: always use vmalloc for section page extensions Muchun Song
@ 2026-11-17 12:06 ` Muchun Song
2026-11-18 14:18 ` Mike Rapoport
2026-11-17 12:06 ` [PATCH v2 5/7] mm/page_ext: allocate page extensions early for SPARSEMEM Muchun Song
` (3 subsequent siblings)
7 siblings, 1 reply; 19+ messages in thread
From: Muchun Song @ 2026-11-17 12:06 UTC (permalink / raw)
To: Alexander Potapenko, Andrew Morton
Cc: Marco Elver, Dmitry Vyukov, Chris Down, David Hildenbrand,
Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Johannes Weiner,
Brendan Jackman, Zi Yan, Muchun Song, Joerg Roedel, Lu Baolu,
Pranjal Shrivastava, Ketan, Mostafa Saleh, Sang-Heon Jeon,
Jonathan Corbet, Shuah Khan, Randy Dunlap, Sashiko, kasan-dev,
workflows, linux-doc, linux-kernel, linux-mm, Muchun Song
FLATMEM uses separate page_ext_init_flatmem() and
page_ext_init_flatmem_late() entry points, while SPARSEMEM performs its
work through page_ext_init(). This exposes the memory model in the
generic boot sequence and obscures the allocation and client
initialization phases.
Rename the early entry point to page_ext_alloc() and use
page_ext_init() for client callbacks in both memory models. SPARSEMEM
keeps its existing late allocation behavior for now; its
page_ext_alloc() remains an empty configuration stub.
This also moves FLATMEM client callbacks from immediately after
kmem_cache_init() to after vmalloc_init(). The callbacks still run
after the buddy and slab allocators are available, as required by
page_owner.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
v2:
- Split the common allocation and initialization entry points into a
preparatory patch
- Document the later FLATMEM client callback timing (noted by Chris
Down)
---
include/linux/page_ext.h | 20 +++++---------------
mm/mm_init.c | 7 +------
mm/page_ext.c | 5 ++---
3 files changed, 8 insertions(+), 24 deletions(-)
diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h
index 79c53ec45dfa..911a84bbb3db 100644
--- a/include/linux/page_ext.h
+++ b/include/linux/page_ext.h
@@ -62,13 +62,10 @@ static inline bool early_page_ext_enabled(void)
}
#ifdef CONFIG_SPARSEMEM
-static inline void page_ext_init_flatmem(void)
-{
-}
-extern void page_ext_init(void);
-static inline void page_ext_init_flatmem_late(void)
+static inline void page_ext_alloc(void)
{
}
+void page_ext_init(void);
static inline bool page_ext_iter_next_fast_possible(unsigned long next_pfn)
{
@@ -79,11 +76,8 @@ static inline bool page_ext_iter_next_fast_possible(unsigned long next_pfn)
return next_pfn % PAGES_PER_SECTION;
}
#else
-extern void page_ext_init_flatmem(void);
-extern void page_ext_init_flatmem_late(void);
-static inline void page_ext_init(void)
-{
-}
+void page_ext_alloc(void);
+void page_ext_init(void);
static inline bool page_ext_iter_next_fast_possible(unsigned long next_pfn)
{
@@ -205,11 +199,7 @@ static inline void page_ext_init(void)
{
}
-static inline void page_ext_init_flatmem_late(void)
-{
-}
-
-static inline void page_ext_init_flatmem(void)
+static inline void page_ext_alloc(void)
{
}
diff --git a/mm/mm_init.c b/mm/mm_init.c
index 2367af05570b..7344e7cb8196 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -2711,7 +2711,7 @@ void __init mm_core_init(void)
* page_ext requires contiguous pages,
* bigger than MAX_PAGE_ORDER unless SPARSEMEM.
*/
- page_ext_init_flatmem();
+ page_ext_alloc();
mem_debugging_and_hardening_init();
kfence_alloc_pool_and_metadata();
report_meminit();
@@ -2727,11 +2727,6 @@ void __init mm_core_init(void)
memblock_free_all();
mem_init();
kmem_cache_init();
- /*
- * page_owner must be initialized after buddy is ready, and also after
- * slab is ready so that stack_depot_init() works properly
- */
- page_ext_init_flatmem_late();
kmemleak_init();
ptlock_cache_init();
pgtable_cache_init();
diff --git a/mm/page_ext.c b/mm/page_ext.c
index 3b581242fd52..b41d4a957bc6 100644
--- a/mm/page_ext.c
+++ b/mm/page_ext.c
@@ -180,7 +180,7 @@ static inline struct page_ext *get_entry(void *base, unsigned long index)
}
#ifndef CONFIG_SPARSEMEM
-void __init page_ext_init_flatmem_late(void)
+void __init page_ext_init(void)
{
invoke_init_callbacks();
}
@@ -238,9 +238,8 @@ static int __init alloc_node_page_ext(int nid)
return 0;
}
-void __init page_ext_init_flatmem(void)
+void __init page_ext_alloc(void)
{
-
int nid, fail;
if (!invoke_need_callbacks())
--
2.54.0
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [PATCH v2 4/7] mm/page_ext: use common allocation and initialization entry points
2026-11-17 12:06 ` [PATCH v2 4/7] mm/page_ext: use common allocation and initialization entry points Muchun Song
@ 2026-11-18 14:18 ` Mike Rapoport
0 siblings, 0 replies; 19+ messages in thread
From: Mike Rapoport @ 2026-11-18 14:18 UTC (permalink / raw)
To: Muchun Song
Cc: Alexander Potapenko, Andrew Morton, Marco Elver, Dmitry Vyukov,
Chris Down, David Hildenbrand, Liam R . Howlett, Vlastimil Babka,
Suren Baghdasaryan, Michal Hocko, Johannes Weiner,
Brendan Jackman, Zi Yan, Muchun Song, Joerg Roedel, Lu Baolu,
Pranjal Shrivastava, Ketan, Mostafa Saleh, Sang-Heon Jeon,
Jonathan Corbet, Shuah Khan, Randy Dunlap, Sashiko, kasan-dev,
workflows, linux-doc, linux-kernel, linux-mm
On Tue, Nov 17, 2026 at 08:06:24PM +0800, Muchun Song wrote:
> FLATMEM uses separate page_ext_init_flatmem() and
> page_ext_init_flatmem_late() entry points, while SPARSEMEM performs its
> work through page_ext_init(). This exposes the memory model in the
> generic boot sequence and obscures the allocation and client
> initialization phases.
>
> Rename the early entry point to page_ext_alloc() and use
> page_ext_init() for client callbacks in both memory models. SPARSEMEM
> keeps its existing late allocation behavior for now; its
> page_ext_alloc() remains an empty configuration stub.
>
> This also moves FLATMEM client callbacks from immediately after
> kmem_cache_init() to after vmalloc_init(). The callbacks still run
> after the buddy and slab allocators are available, as required by
> page_owner.
>
> Signed-off-by: Muchun Song <songmuchun@bytedance.com>
Reviewed-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
> ---
> v2:
> - Split the common allocation and initialization entry points into a
> preparatory patch
> - Document the later FLATMEM client callback timing (noted by Chris
> Down)
> ---
> include/linux/page_ext.h | 20 +++++---------------
> mm/mm_init.c | 7 +------
> mm/page_ext.c | 5 ++---
> 3 files changed, 8 insertions(+), 24 deletions(-)
--
Sincerely yours,
Mike.
^ permalink raw reply [flat|nested] 19+ messages in thread
* [PATCH v2 5/7] mm/page_ext: allocate page extensions early for SPARSEMEM
2026-11-17 12:06 [PATCH v2 0/7] KMSAN: Move per-page metadata out of struct page Muchun Song
` (3 preceding siblings ...)
2026-11-17 12:06 ` [PATCH v2 4/7] mm/page_ext: use common allocation and initialization entry points Muchun Song
@ 2026-11-17 12:06 ` Muchun Song
2026-11-18 0:06 ` Chris Down
2026-11-18 14:22 ` Mike Rapoport
2026-11-17 12:06 ` [PATCH v2 6/7] mm/page_ext: inline unchecked page extension lookup Muchun Song
` (2 subsequent siblings)
7 siblings, 2 replies; 19+ messages in thread
From: Muchun Song @ 2026-11-17 12:06 UTC (permalink / raw)
To: Alexander Potapenko, Andrew Morton
Cc: Marco Elver, Dmitry Vyukov, Chris Down, David Hildenbrand,
Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Johannes Weiner,
Brendan Jackman, Zi Yan, Muchun Song, Joerg Roedel, Lu Baolu,
Pranjal Shrivastava, Ketan, Mostafa Saleh, Sang-Heon Jeon,
Jonathan Corbet, Shuah Khan, Randy Dunlap, Sashiko, kasan-dev,
workflows, linux-doc, linux-kernel, linux-mm, Muchun Song
KMSAN needs page extensions when kmsan_init_shadow() installs per-page
metadata, before memblock_free_all() releases memory to the buddy
allocator. FLATMEM makes page extensions available by this point,
whereas SPARSEMEM initializes them later in boot.
Allocate boot-time SPARSEMEM page extensions from memblock to match
FLATMEM's initialization timing. This gives early clients the same
page_ext availability guarantee across both memory models, while runtime
memory hotplug continues to use vmalloc.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
v2:
- Allocate all boot-time SPARSEMEM page extensions from memblock
(suggested by Mike Rapoport)
- Keep memblock allocations above DMA and suppress per-section free
messages (suggested by Chris Down)
---
include/linux/page_ext.h | 9 +--
mm/mm_init.c | 4 --
mm/page_ext.c | 119 +++++++++++++++++++--------------------
3 files changed, 60 insertions(+), 72 deletions(-)
diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h
index 911a84bbb3db..d00320106a58 100644
--- a/include/linux/page_ext.h
+++ b/include/linux/page_ext.h
@@ -61,12 +61,10 @@ static inline bool early_page_ext_enabled(void)
return early_page_ext;
}
-#ifdef CONFIG_SPARSEMEM
-static inline void page_ext_alloc(void)
-{
-}
+void page_ext_alloc(void);
void page_ext_init(void);
+#ifdef CONFIG_SPARSEMEM
static inline bool page_ext_iter_next_fast_possible(unsigned long next_pfn)
{
/*
@@ -76,9 +74,6 @@ static inline bool page_ext_iter_next_fast_possible(unsigned long next_pfn)
return next_pfn % PAGES_PER_SECTION;
}
#else
-void page_ext_alloc(void);
-void page_ext_init(void);
-
static inline bool page_ext_iter_next_fast_possible(unsigned long next_pfn)
{
return true;
diff --git a/mm/mm_init.c b/mm/mm_init.c
index 7344e7cb8196..baa2681857eb 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -2707,10 +2707,6 @@ void __init mm_core_init(void)
build_all_zonelists(NULL);
page_alloc_init_cpuhp();
alloc_tag_sec_init();
- /*
- * page_ext requires contiguous pages,
- * bigger than MAX_PAGE_ORDER unless SPARSEMEM.
- */
page_ext_alloc();
mem_debugging_and_hardening_init();
kfence_alloc_pool_and_metadata();
diff --git a/mm/page_ext.c b/mm/page_ext.c
index b41d4a957bc6..2910727e99e2 100644
--- a/mm/page_ext.c
+++ b/mm/page_ext.c
@@ -12,6 +12,7 @@
#include <linux/rcupdate.h>
#include <linux/pgalloc_tag.h>
#include <linux/iommu-debug-pagealloc.h>
+#include <linux/slab.h>
/*
* struct page extension
@@ -180,11 +181,6 @@ static inline struct page_ext *get_entry(void *base, unsigned long index)
}
#ifndef CONFIG_SPARSEMEM
-void __init page_ext_init(void)
-{
- invoke_init_callbacks();
-}
-
static struct page_ext *lookup_page_ext(const struct page *page)
{
unsigned long pfn = page_to_pfn(page);
@@ -238,24 +234,16 @@ static int __init alloc_node_page_ext(int nid)
return 0;
}
-void __init page_ext_alloc(void)
+static int __init __page_ext_alloc(void)
{
- int nid, fail;
-
- if (!invoke_need_callbacks())
- return;
+ int nid;
- for_each_online_node(nid) {
- fail = alloc_node_page_ext(nid);
- if (fail)
- goto fail;
+ for_each_online_node(nid) {
+ if (alloc_node_page_ext(nid))
+ return -ENOMEM;
}
- pr_info("allocated %ld bytes of page_ext\n", total_usage);
- return;
-fail:
- pr_crit("allocation of page_ext failed.\n");
- panic("Out of memory");
+ return 0;
}
#else /* CONFIG_SPARSEMEM */
@@ -282,13 +270,21 @@ static struct page_ext *lookup_page_ext(const struct page *page)
return get_entry(page_ext, pfn);
}
-static void *__meminit alloc_page_ext(size_t size, int nid)
+static __ref void *alloc_page_ext(size_t size, int nid)
{
void *addr;
+ if (!slab_is_available()) {
+ addr = memblock_alloc_try_nid(size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS),
+ MEMBLOCK_ALLOC_ACCESSIBLE, nid);
+ if (addr)
+ memmap_boot_pages_add(size / PAGE_SIZE);
+ return addr;
+ }
+
addr = vzalloc_node(size, nid);
if (addr)
- memmap_pages_add(DIV_ROUND_UP(size, PAGE_SIZE));
+ memmap_pages_add(size / PAGE_SIZE);
return addr;
}
@@ -304,7 +300,7 @@ static int __meminit init_section_page_ext(unsigned long pfn, int nid)
if (section->page_ext)
return 0;
- table_size = page_ext_size * PAGES_PER_SECTION;
+ table_size = PAGE_ALIGN(page_ext_size * PAGES_PER_SECTION);
base = alloc_page_ext(table_size, nid);
/*
@@ -331,11 +327,19 @@ static int __meminit init_section_page_ext(unsigned long pfn, int nid)
static void free_page_ext(void *addr)
{
- size_t table_size;
+ size_t table_size = PAGE_ALIGN(page_ext_size * PAGES_PER_SECTION);
+ long nr_pages = table_size / PAGE_SIZE;
+
+ if (is_vmalloc_addr(addr)) {
+ memmap_pages_add(-nr_pages);
+ vfree(addr);
+ return;
+ }
- table_size = page_ext_size * PAGES_PER_SECTION;
- memmap_pages_add(-1L * (DIV_ROUND_UP(table_size, PAGE_SIZE)));
- vfree(addr);
+ VM_WARN_ON_ONCE(!PageReserved(virt_to_page(addr)));
+ memmap_boot_pages_add(-nr_pages);
+ kmemleak_free_part_phys(__pa(addr), table_size);
+ free_reserved_area(addr, addr + table_size, -1, NULL);
}
static void __free_page_ext(unsigned long pfn)
@@ -449,52 +453,45 @@ static int __meminit page_ext_callback(struct notifier_block *self,
return notifier_from_errno(ret);
}
-void __init page_ext_init(void)
+static int __init __page_ext_alloc(void)
{
- unsigned long pfn;
- int nid;
-
- if (!invoke_need_callbacks())
- return;
-
- for_each_node_state(nid, N_MEMORY) {
- unsigned long start_pfn, end_pfn;
-
- start_pfn = node_start_pfn(nid);
- end_pfn = node_end_pfn(nid);
- /*
- * start_pfn and end_pfn may not be aligned to SECTION and the
- * page->flags of out of node pages are not initialized. So we
- * scan [start_pfn, the biggest section's pfn < end_pfn) here.
- */
- for (pfn = start_pfn; pfn < end_pfn;
- pfn = ALIGN(pfn + 1, PAGES_PER_SECTION)) {
+ unsigned long start, end, pfn;
+ int i, nid;
+ for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
+ for (pfn = start; pfn < end; pfn = SECTION_ALIGN_UP(pfn + 1)) {
if (!pfn_valid(pfn))
continue;
- /*
- * Nodes's pfns can be overlapping.
- * We know some arch can have a nodes layout such as
- * -------------pfn-------------->
- * N0 | N1 | N2 | N0 | N1 | N2|....
- */
- if (pfn_to_nid(pfn) != nid)
- continue;
if (init_section_page_ext(pfn, nid))
- goto oom;
- cond_resched();
+ return -ENOMEM;
}
}
- hotplug_memory_notifier(page_ext_callback, DEFAULT_CALLBACK_PRI);
- pr_info("allocated %ld bytes of page_ext\n", total_usage);
- invoke_init_callbacks();
- return;
-oom:
- panic("Out of memory");
+ return 0;
+}
+
+#endif
+
+void __init page_ext_alloc(void)
+{
+ if (!invoke_need_callbacks())
+ return;
+ if (__page_ext_alloc())
+ panic("Out of memory");
+
+ pr_info("allocated %ld bytes of page_ext\n", total_usage);
}
+void __init page_ext_init(void)
+{
+ if (!page_ext_size)
+ return;
+
+#ifdef CONFIG_SPARSEMEM
+ hotplug_memory_notifier(page_ext_callback, DEFAULT_CALLBACK_PRI);
#endif
+ invoke_init_callbacks();
+}
/**
* page_ext_lookup() - Lookup a page extension for a PFN.
--
2.54.0
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [PATCH v2 5/7] mm/page_ext: allocate page extensions early for SPARSEMEM
2026-11-17 12:06 ` [PATCH v2 5/7] mm/page_ext: allocate page extensions early for SPARSEMEM Muchun Song
@ 2026-11-18 0:06 ` Chris Down
2026-11-18 3:11 ` Muchun Song
2026-11-18 14:22 ` Mike Rapoport
1 sibling, 1 reply; 19+ messages in thread
From: Chris Down @ 2026-11-18 0:06 UTC (permalink / raw)
To: Muchun Song
Cc: Alexander Potapenko, Andrew Morton, Marco Elver, Dmitry Vyukov,
David Hildenbrand, Liam R . Howlett, Vlastimil Babka,
Mike Rapoport, Suren Baghdasaryan, Michal Hocko, Johannes Weiner,
Brendan Jackman, Zi Yan, Muchun Song, Joerg Roedel, Lu Baolu,
Pranjal Shrivastava, Ketan, Mostafa Saleh, Sang-Heon Jeon,
Jonathan Corbet, Shuah Khan, Randy Dunlap, Sashiko, kasan-dev,
workflows, linux-doc, linux-kernel, linux-mm
Hi Muchun,
Muchun Song writes:
>+ if (!slab_is_available()) {
>+ addr = memblock_alloc_try_nid(size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS),
>+ MEMBLOCK_ALLOC_ACCESSIBLE, nid);
Hmm, unless I'm mistaken, memblock allocates top-down, and with
movablecore= or kernelcore= the top of each node is ZONE_MOVABLE, so this
puts the page_ext tables there. It looks like that then offlining those
memory blocks would fail with EBUSY because of the reserved pages.
I can see why Mike suggested always using memblock here, but only KMSAN
needs the tables before the buddy allocator is up. So I think this needs
to either go back to using memblock only for clients that need it, as v1
did, or keep these allocations below ZONE_MOVABLE.
Thanks!
Chris
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [PATCH v2 5/7] mm/page_ext: allocate page extensions early for SPARSEMEM
2026-11-18 0:06 ` Chris Down
@ 2026-11-18 3:11 ` Muchun Song
2026-11-18 7:00 ` Chris Down
0 siblings, 1 reply; 19+ messages in thread
From: Muchun Song @ 2026-11-18 3:11 UTC (permalink / raw)
To: Chris Down
Cc: Muchun Song, Alexander Potapenko, Andrew Morton, Marco Elver,
Dmitry Vyukov, David Hildenbrand, Liam R . Howlett,
Vlastimil Babka, Mike Rapoport, Suren Baghdasaryan, Michal Hocko,
Johannes Weiner, Brendan Jackman, Zi Yan, Joerg Roedel, Lu Baolu,
Pranjal Shrivastava, Ketan, Mostafa Saleh, Sang-Heon Jeon,
Jonathan Corbet, Shuah Khan, Randy Dunlap, Sashiko, kasan-dev,
workflows, linux-doc, linux-kernel, linux-mm
> On Nov 18, 2026, at 08:06, Chris Down <chris@chrisdown.name> wrote:
>
> Hi Muchun,
Hi Chris,
Thanks for taking a look.
>
> Muchun Song writes:
>> + if (!slab_is_available()) {
>> + addr = memblock_alloc_try_nid(size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS),
>> + MEMBLOCK_ALLOC_ACCESSIBLE, nid);
>
> Hmm, unless I'm mistaken, memblock allocates top-down, and with
> movablecore= or kernelcore= the top of each node is ZONE_MOVABLE, so this
> puts the page_ext tables there. It looks like that then offlining those
> memory blocks would fail with EBUSY because of the reserved pages.
Yes, this is a valid concern. However, early vmemmap backing is also
allocated from memblock before page_ext and can have the same problem, so
handling page_ext alone cannot guarantee successful offlining.
Mike's suggestion considerably simplifies the code, and early page_ext is
only needed by optional debugging and instrumentation users. I would prefer
to keep this approach.
What do you think?
Thanks,
Muchun
>
> I can see why Mike suggested always using memblock here, but only KMSAN
> needs the tables before the buddy allocator is up. So I think this needs
> to either go back to using memblock only for clients that need it, as v1
> did, or keep these allocations below ZONE_MOVABLE.
>
> Thanks!
>
> Chris
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [PATCH v2 5/7] mm/page_ext: allocate page extensions early for SPARSEMEM
2026-11-18 3:11 ` Muchun Song
@ 2026-11-18 7:00 ` Chris Down
2026-11-18 14:17 ` Mike Rapoport
0 siblings, 1 reply; 19+ messages in thread
From: Chris Down @ 2026-11-18 7:00 UTC (permalink / raw)
To: Muchun Song
Cc: Muchun Song, Alexander Potapenko, Andrew Morton, Marco Elver,
Dmitry Vyukov, David Hildenbrand, Liam R . Howlett,
Vlastimil Babka, Mike Rapoport, Suren Baghdasaryan, Michal Hocko,
Johannes Weiner, Brendan Jackman, Zi Yan, Joerg Roedel, Lu Baolu,
Pranjal Shrivastava, Ketan, Mostafa Saleh, Sang-Heon Jeon,
Jonathan Corbet, Shuah Khan, Randy Dunlap, Sashiko, kasan-dev,
workflows, linux-doc, linux-kernel, linux-mm
Hi Muchun,
Muchun Song writes:
>Mike's suggestion considerably simplifies the code, and early page_ext is
>only needed by optional debugging and instrumentation users. I would prefer
>to keep this approach.
You're probably right. But can I cheekily suggest something in the middle
perhaps? :-)
Given the zones are already set up before page_ext_alloc() runs, perhaps
you can pass the start of the node's ZONE_MOVABLE as max_addr to
memblock_alloc_try_nid() when the node has one. The general idea is:
struct zone *movable = &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];
phys_addr_t max_addr = MEMBLOCK_ALLOC_ACCESSIBLE;
if (!zone_is_empty(movable))
max_addr = PFN_PHYS(movable->zone_start_pfn);
addr = memblock_alloc_try_nid(size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS),
max_addr, nid);
if (!addr && max_addr != MEMBLOCK_ALLOC_ACCESSIBLE)
addr = memblock_alloc_try_nid(size, PAGE_SIZE,
__pa(MAX_DMA_ADDRESS),
MEMBLOCK_ALLOC_ACCESSIBLE, nid);
I briefly tried that and it seems to work quite nicely.
But I don't object to it even without that, so feel free to add either way:
Reviewed-by: Chris Down <chris@chrisdown.name>
Thanks!
Chris
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [PATCH v2 5/7] mm/page_ext: allocate page extensions early for SPARSEMEM
2026-11-18 7:00 ` Chris Down
@ 2026-11-18 14:17 ` Mike Rapoport
2026-11-18 15:37 ` Muchun Song
0 siblings, 1 reply; 19+ messages in thread
From: Mike Rapoport @ 2026-11-18 14:17 UTC (permalink / raw)
To: Chris Down
Cc: Muchun Song, Muchun Song, Alexander Potapenko, Andrew Morton,
Marco Elver, Dmitry Vyukov, David Hildenbrand, Liam R . Howlett,
Vlastimil Babka, Suren Baghdasaryan, Michal Hocko,
Johannes Weiner, Brendan Jackman, Zi Yan, Joerg Roedel, Lu Baolu,
Pranjal Shrivastava, Ketan, Mostafa Saleh, Sang-Heon Jeon,
Jonathan Corbet, Shuah Khan, Randy Dunlap, Sashiko, kasan-dev,
workflows, linux-doc, linux-kernel, linux-mm
On Tue, Nov 17, 2026 at 11:00:06PM -0800, Chris Down wrote:
> Hi Muchun,
>
> Muchun Song writes:
> > Mike's suggestion considerably simplifies the code, and early page_ext is
> > only needed by optional debugging and instrumentation users. I would prefer
> > to keep this approach.
>
> You're probably right.
I think so :)
The memap_alloc() that essentially allocates the memory map does
memblock_alloc_try_nid_raw(size, align, min_addr,
MEMBLOCK_ALLOC_NOLEAKTRACE, nid);
so we anyway may end up with reserved memory in ZONE_MOVABLE.
Until recently we had memory map allocation before the zones were setup, so
there wasn't much we could do about it, but ...
> But can I cheekily suggest something in the middle perhaps? :-)
>
> Given the zones are already set up before page_ext_alloc() runs, perhaps
> you can pass the start of the node's ZONE_MOVABLE as max_addr to
> memblock_alloc_try_nid() when the node has one. The general idea is:
>
> struct zone *movable = &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];
> phys_addr_t max_addr = MEMBLOCK_ALLOC_ACCESSIBLE;
>
> if (!zone_is_empty(movable))
> max_addr = PFN_PHYS(movable->zone_start_pfn);
> addr = memblock_alloc_try_nid(size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS),
> max_addr, nid);
> if (!addr && max_addr != MEMBLOCK_ALLOC_ACCESSIBLE)
> addr = memblock_alloc_try_nid(size, PAGE_SIZE,
> __pa(MAX_DMA_ADDRESS),
> MEMBLOCK_ALLOC_ACCESSIBLE, nid);
... with zone extents known we could take it even further :)
page_ext is treated as the memory map in many ways, so we can allocate the
early page_ext sections with memmap_alloc().
And since the latter always runs after zones are set up it could include
this change.
But I think this should come as a separate series that also could move
memmap_boot_pages_add to memmap_alloc().
> I briefly tried that and it seems to work quite nicely.
>
> But I don't object to it even without that, so feel free to add either way:
>
> Reviewed-by: Chris Down <chris@chrisdown.name>
>
> Thanks!
>
> Chris
--
Sincerely yours,
Mike.
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [PATCH v2 5/7] mm/page_ext: allocate page extensions early for SPARSEMEM
2026-11-18 14:17 ` Mike Rapoport
@ 2026-11-18 15:37 ` Muchun Song
0 siblings, 0 replies; 19+ messages in thread
From: Muchun Song @ 2026-11-18 15:37 UTC (permalink / raw)
To: Mike Rapoport
Cc: Muchun Song, Alexander Potapenko, Andrew Morton, Marco Elver,
Dmitry Vyukov, David Hildenbrand, Liam R . Howlett,
Vlastimil Babka, Suren Baghdasaryan, Michal Hocko,
Johannes Weiner, Brendan Jackman, Zi Yan, Joerg Roedel, Lu Baolu,
Pranjal Shrivastava, Ketan, Mostafa Saleh, Sang-Heon Jeon,
Jonathan Corbet, Shuah Khan, Randy Dunlap, Sashiko, kasan-dev,
workflows, linux-doc, linux-kernel, linux-mm, Chris Down
On 2026/11/18 22:17, Mike Rapoport wrote:
> On Tue, Nov 17, 2026 at 11:00:06PM -0800, Chris Down wrote:
>> Hi Muchun,
>>
>> Muchun Song writes:
>>> Mike's suggestion considerably simplifies the code, and early page_ext is
>>> only needed by optional debugging and instrumentation users. I would prefer
>>> to keep this approach.
>> You're probably right.
> I think so :)
> The memap_alloc() that essentially allocates the memory map does
>
> memblock_alloc_try_nid_raw(size, align, min_addr,
> MEMBLOCK_ALLOC_NOLEAKTRACE, nid);
>
> so we anyway may end up with reserved memory in ZONE_MOVABLE.
>
> Until recently we had memory map allocation before the zones were setup, so
> there wasn't much we could do about it, but ...
>
>> But can I cheekily suggest something in the middle perhaps? :-)
>>
>> Given the zones are already set up before page_ext_alloc() runs, perhaps
>> you can pass the start of the node's ZONE_MOVABLE as max_addr to
>> memblock_alloc_try_nid() when the node has one. The general idea is:
>>
>> struct zone *movable = &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];
>> phys_addr_t max_addr = MEMBLOCK_ALLOC_ACCESSIBLE;
>>
>> if (!zone_is_empty(movable))
>> max_addr = PFN_PHYS(movable->zone_start_pfn);
>> addr = memblock_alloc_try_nid(size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS),
>> max_addr, nid);
>> if (!addr && max_addr != MEMBLOCK_ALLOC_ACCESSIBLE)
>> addr = memblock_alloc_try_nid(size, PAGE_SIZE,
>> __pa(MAX_DMA_ADDRESS),
>> MEMBLOCK_ALLOC_ACCESSIBLE, nid);
> ... with zone extents known we could take it even further :)
>
> page_ext is treated as the memory map in many ways, so we can allocate the
> early page_ext sections with memmap_alloc().
> And since the latter always runs after zones are set up it could include
> this change.
>
> But I think this should come as a separate series that also could move
> memmap_boot_pages_add to memmap_alloc().
Agreed. Using memmap_alloc() for early page_ext allocations and
teaching it to avoid ZONE_MOVABLE sounds cleaner. I'll keep this
series as is and handle that in a separate series.
Thanks!
>
>> I briefly tried that and it seems to work quite nicely.
>>
>> But I don't object to it even without that, so feel free to add either way:
>>
>> Reviewed-by: Chris Down <chris@chrisdown.name>
>>
>> Thanks!
>>
>> Chris
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [PATCH v2 5/7] mm/page_ext: allocate page extensions early for SPARSEMEM
2026-11-17 12:06 ` [PATCH v2 5/7] mm/page_ext: allocate page extensions early for SPARSEMEM Muchun Song
2026-11-18 0:06 ` Chris Down
@ 2026-11-18 14:22 ` Mike Rapoport
2026-11-18 15:13 ` Muchun Song
1 sibling, 1 reply; 19+ messages in thread
From: Mike Rapoport @ 2026-11-18 14:22 UTC (permalink / raw)
To: Muchun Song
Cc: Alexander Potapenko, Andrew Morton, Marco Elver, Dmitry Vyukov,
Chris Down, David Hildenbrand, Liam R . Howlett, Vlastimil Babka,
Suren Baghdasaryan, Michal Hocko, Johannes Weiner,
Brendan Jackman, Zi Yan, Muchun Song, Joerg Roedel, Lu Baolu,
Pranjal Shrivastava, Ketan, Mostafa Saleh, Sang-Heon Jeon,
Jonathan Corbet, Shuah Khan, Randy Dunlap, Sashiko, kasan-dev,
workflows, linux-doc, linux-kernel, linux-mm
On Tue, Nov 17, 2026 at 08:06:25PM +0800, Muchun Song wrote:
> KMSAN needs page extensions when kmsan_init_shadow() installs per-page
> metadata, before memblock_free_all() releases memory to the buddy
> allocator. FLATMEM makes page extensions available by this point,
> whereas SPARSEMEM initializes them later in boot.
>
> Allocate boot-time SPARSEMEM page extensions from memblock to match
> FLATMEM's initialization timing. This gives early clients the same
> page_ext availability guarantee across both memory models, while runtime
> memory hotplug continues to use vmalloc.
>
> Signed-off-by: Muchun Song <songmuchun@bytedance.com>
Reviewed-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
with a nit below
> ---
> v2:
> - Allocate all boot-time SPARSEMEM page extensions from memblock
> (suggested by Mike Rapoport)
> - Keep memblock allocations above DMA and suppress per-section free
> messages (suggested by Chris Down)
> ---
> include/linux/page_ext.h | 9 +--
> mm/mm_init.c | 4 --
> mm/page_ext.c | 119 +++++++++++++++++++--------------------
> 3 files changed, 60 insertions(+), 72 deletions(-)
>
> +static __ref void *alloc_page_ext(size_t size, int nid)
> {
> void *addr;
>
> + if (!slab_is_available()) {
> + addr = memblock_alloc_try_nid(size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS),
> + MEMBLOCK_ALLOC_ACCESSIBLE, nid);
> + if (addr)
> + memmap_boot_pages_add(size / PAGE_SIZE);
I think it should be DIV_ROUND_UP().
> + return addr;
> + }
> +
> addr = vzalloc_node(size, nid);
> if (addr)
> - memmap_pages_add(DIV_ROUND_UP(size, PAGE_SIZE));
> + memmap_pages_add(size / PAGE_SIZE);
And here too.
>
> return addr;
> }
> @@ -304,7 +300,7 @@ static int __meminit init_section_page_ext(unsigned long pfn, int nid)
> if (section->page_ext)
> return 0;
>
> - table_size = page_ext_size * PAGES_PER_SECTION;
> + table_size = PAGE_ALIGN(page_ext_size * PAGES_PER_SECTION);
> base = alloc_page_ext(table_size, nid);
>
> /*
--
Sincerely yours,
Mike.
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [PATCH v2 5/7] mm/page_ext: allocate page extensions early for SPARSEMEM
2026-11-18 14:22 ` Mike Rapoport
@ 2026-11-18 15:13 ` Muchun Song
0 siblings, 0 replies; 19+ messages in thread
From: Muchun Song @ 2026-11-18 15:13 UTC (permalink / raw)
To: Mike Rapoport
Cc: Alexander Potapenko, Andrew Morton, Marco Elver, Dmitry Vyukov,
Chris Down, David Hildenbrand, Liam R . Howlett, Vlastimil Babka,
Suren Baghdasaryan, Michal Hocko, Johannes Weiner,
Brendan Jackman, Zi Yan, Joerg Roedel, Lu Baolu,
Pranjal Shrivastava, Ketan, Mostafa Saleh, Muchun Song,
Sang-Heon Jeon, Jonathan Corbet, Shuah Khan, Randy Dunlap,
Sashiko, kasan-dev, workflows, linux-doc, linux-kernel, linux-mm
On 2026/11/18 22:22, Mike Rapoport wrote:
> On Tue, Nov 17, 2026 at 08:06:25PM +0800, Muchun Song wrote:
>> KMSAN needs page extensions when kmsan_init_shadow() installs per-page
>> metadata, before memblock_free_all() releases memory to the buddy
>> allocator. FLATMEM makes page extensions available by this point,
>> whereas SPARSEMEM initializes them later in boot.
>>
>> Allocate boot-time SPARSEMEM page extensions from memblock to match
>> FLATMEM's initialization timing. This gives early clients the same
>> page_ext availability guarantee across both memory models, while runtime
>> memory hotplug continues to use vmalloc.
>>
>> Signed-off-by: Muchun Song <songmuchun@bytedance.com>
> Reviewed-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Thanks for reviewing.
>
> with a nit below
>
>> ---
>> v2:
>> - Allocate all boot-time SPARSEMEM page extensions from memblock
>> (suggested by Mike Rapoport)
>> - Keep memblock allocations above DMA and suppress per-section free
>> messages (suggested by Chris Down)
>> ---
>> include/linux/page_ext.h | 9 +--
>> mm/mm_init.c | 4 --
>> mm/page_ext.c | 119 +++++++++++++++++++--------------------
>> 3 files changed, 60 insertions(+), 72 deletions(-)
>>
>> +static __ref void *alloc_page_ext(size_t size, int nid)
>> {
>> void *addr;
>>
>> + if (!slab_is_available()) {
>> + addr = memblock_alloc_try_nid(size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS),
>> + MEMBLOCK_ALLOC_ACCESSIBLE, nid);
>> + if (addr)
>> + memmap_boot_pages_add(size / PAGE_SIZE);
> I think it should be DIV_ROUND_UP().
size is currently page-aligned by init_section_page_ext(), so both forms are
equivalent here. Is the suggestion to avoid relying on this caller-side
invariant and make it more robust? If so, I'll use DIV_ROUND_UP() for both
accounting calls.
Thanks,
Muchun
>
>> + return addr;
>> + }
>> +
>> addr = vzalloc_node(size, nid);
>> if (addr)
>> - memmap_pages_add(DIV_ROUND_UP(size, PAGE_SIZE));
>> + memmap_pages_add(size / PAGE_SIZE);
> And here too.
>
>>
>> return addr;
>> }
>> @@ -304,7 +300,7 @@ static int __meminit init_section_page_ext(unsigned long pfn, int nid)
>> if (section->page_ext)
>> return 0;
>>
>> - table_size = page_ext_size * PAGES_PER_SECTION;
>> + table_size = PAGE_ALIGN(page_ext_size * PAGES_PER_SECTION);
>> base = alloc_page_ext(table_size, nid);
>>
>> /*
^ permalink raw reply [flat|nested] 19+ messages in thread
* [PATCH v2 6/7] mm/page_ext: inline unchecked page extension lookup
2026-11-17 12:06 [PATCH v2 0/7] KMSAN: Move per-page metadata out of struct page Muchun Song
` (4 preceding siblings ...)
2026-11-17 12:06 ` [PATCH v2 5/7] mm/page_ext: allocate page extensions early for SPARSEMEM Muchun Song
@ 2026-11-17 12:06 ` Muchun Song
2026-11-17 12:06 ` [PATCH v2 7/7] kmsan: fix metadata loss during HugeTLB struct page reinitialization Muchun Song
2026-11-18 0:06 ` [PATCH v2 0/7] KMSAN: Move per-page metadata out of struct page Chris Down
7 siblings, 0 replies; 19+ messages in thread
From: Muchun Song @ 2026-11-17 12:06 UTC (permalink / raw)
To: Alexander Potapenko, Andrew Morton
Cc: Marco Elver, Dmitry Vyukov, Chris Down, David Hildenbrand,
Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Johannes Weiner,
Brendan Jackman, Zi Yan, Muchun Song, Joerg Roedel, Lu Baolu,
Pranjal Shrivastava, Ketan, Mostafa Saleh, Sang-Heon Jeon,
Jonathan Corbet, Shuah Khan, Randy Dunlap, Sashiko, kasan-dev,
workflows, linux-doc, linux-kernel, linux-mm, Muchun Song
The KMSAN runtime needs an unchecked page extension lookup that can be
inlined into its uninstrumented compilation unit while page_ext.c remains
instrumented.
Provide an inline lookup for this use case while keeping the existing RCU
checks for other callers.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
v2:
- Add a preparatory patch that inlines the unchecked lookup (suggested
by Chris Down)
- Use a PFN interface to avoid architecture header-order dependencies
---
include/linux/page_ext.h | 46 ++++++++++++++++++++++++++++++
mm/page_ext.c | 60 +++-------------------------------------
2 files changed, 50 insertions(+), 56 deletions(-)
diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h
index d00320106a58..166f8be11564 100644
--- a/include/linux/page_ext.h
+++ b/include/linux/page_ext.h
@@ -64,7 +64,14 @@ static inline bool early_page_ext_enabled(void)
void page_ext_alloc(void);
void page_ext_init(void);
+static __always_inline struct page_ext *page_ext_entry(void *base, unsigned long index)
+{
+ return base + page_ext_size * index;
+}
+
#ifdef CONFIG_SPARSEMEM
+#define PAGE_EXT_INVALID 0x1
+
static inline bool page_ext_iter_next_fast_possible(unsigned long next_pfn)
{
/*
@@ -73,11 +80,50 @@ static inline bool page_ext_iter_next_fast_possible(unsigned long next_pfn)
*/
return next_pfn % PAGES_PER_SECTION;
}
+
+static __always_inline bool page_ext_invalid(struct page_ext *page_ext)
+{
+ return !page_ext || (((unsigned long)page_ext & PAGE_EXT_INVALID) == PAGE_EXT_INVALID);
+}
+
+static __always_inline struct page_ext *__page_ext_lookup(unsigned long pfn)
+{
+ struct mem_section *section = __pfn_to_section(pfn);
+ struct page_ext *page_ext = READ_ONCE(section->page_ext);
+
+ /*
+ * The sanity checks the page allocator does upon freeing a page can
+ * reach here before the page_ext arrays are allocated when feeding a
+ * range of pages to the allocator for the first time during bootup or
+ * memory hotplug.
+ */
+ if (page_ext_invalid(page_ext))
+ return NULL;
+ return page_ext_entry(page_ext, pfn);
+}
#else
static inline bool page_ext_iter_next_fast_possible(unsigned long next_pfn)
{
return true;
}
+
+static __always_inline struct page_ext *__page_ext_lookup(unsigned long pfn)
+{
+ int nid = pfn_to_nid(pfn);
+ struct page_ext *base = NODE_DATA(nid)->node_page_ext;
+ unsigned long index;
+
+ /*
+ * The sanity checks the page allocator does upon freeing a page can
+ * reach here before the page_ext arrays are allocated when feeding a
+ * range of pages to the allocator for the first time during bootup or
+ * memory hotplug.
+ */
+ if (unlikely(!base))
+ return NULL;
+ index = pfn - round_down(node_start_pfn(nid), MAX_ORDER_NR_PAGES);
+ return page_ext_entry(base, index);
+}
#endif
extern struct page_ext *page_ext_get(const struct page *page);
diff --git a/mm/page_ext.c b/mm/page_ext.c
index 2910727e99e2..0f31e2da2611 100644
--- a/mm/page_ext.c
+++ b/mm/page_ext.c
@@ -63,10 +63,6 @@
* can utilize this callback to initialize the state of it correctly.
*/
-#ifdef CONFIG_SPARSEMEM
-#define PAGE_EXT_INVALID (0x1)
-#endif
-
#if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT)
static bool need_page_idle(void)
{
@@ -175,33 +171,7 @@ static void __init invoke_init_callbacks(void)
}
}
-static inline struct page_ext *get_entry(void *base, unsigned long index)
-{
- return base + page_ext_size * index;
-}
-
#ifndef CONFIG_SPARSEMEM
-static struct page_ext *lookup_page_ext(const struct page *page)
-{
- unsigned long pfn = page_to_pfn(page);
- unsigned long index;
- struct page_ext *base;
-
- WARN_ON_ONCE(!rcu_read_lock_held());
- base = NODE_DATA(page_to_nid(page))->node_page_ext;
- /*
- * The sanity checks the page allocator does upon freeing a
- * page can reach here before the page_ext arrays are
- * allocated when feeding a range of pages to the allocator
- * for the first time during bootup or memory hotplug.
- */
- if (unlikely(!base))
- return NULL;
- index = pfn - round_down(node_start_pfn(page_to_nid(page)),
- MAX_ORDER_NR_PAGES);
- return get_entry(base, index);
-}
-
static int __init alloc_node_page_ext(int nid)
{
struct page_ext *base;
@@ -247,29 +217,6 @@ static int __init __page_ext_alloc(void)
}
#else /* CONFIG_SPARSEMEM */
-static bool page_ext_invalid(struct page_ext *page_ext)
-{
- return !page_ext || (((unsigned long)page_ext & PAGE_EXT_INVALID) == PAGE_EXT_INVALID);
-}
-
-static struct page_ext *lookup_page_ext(const struct page *page)
-{
- unsigned long pfn = page_to_pfn(page);
- struct mem_section *section = __pfn_to_section(pfn);
- struct page_ext *page_ext = READ_ONCE(section->page_ext);
-
- WARN_ON_ONCE(!rcu_read_lock_held());
- /*
- * The sanity checks the page allocator does upon freeing a
- * page can reach here before the page_ext arrays are
- * allocated when feeding a range of pages to the allocator
- * for the first time during bootup or memory hotplug.
- */
- if (page_ext_invalid(page_ext))
- return NULL;
- return get_entry(page_ext, pfn);
-}
-
static __ref void *alloc_page_ext(size_t size, int nid)
{
void *addr;
@@ -360,7 +307,7 @@ static void __free_page_ext(unsigned long pfn)
base = (void *)base - PAGE_EXT_INVALID;
WRITE_ONCE(ms->page_ext, NULL);
- base = get_entry(base, pfn);
+ base = page_ext_entry(base, pfn);
free_page_ext(base);
}
@@ -503,7 +450,8 @@ void __init page_ext_init(void)
*/
struct page_ext *page_ext_lookup(unsigned long pfn)
{
- return lookup_page_ext(pfn_to_page(pfn));
+ WARN_ON_ONCE(!rcu_read_lock_held());
+ return __page_ext_lookup(pfn);
}
/**
@@ -522,7 +470,7 @@ struct page_ext *page_ext_get(const struct page *page)
struct page_ext *page_ext;
rcu_read_lock();
- page_ext = lookup_page_ext(page);
+ page_ext = page_ext_lookup(page_to_pfn(page));
if (!page_ext) {
rcu_read_unlock();
return NULL;
--
2.54.0
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH v2 7/7] kmsan: fix metadata loss during HugeTLB struct page reinitialization
2026-11-17 12:06 [PATCH v2 0/7] KMSAN: Move per-page metadata out of struct page Muchun Song
` (5 preceding siblings ...)
2026-11-17 12:06 ` [PATCH v2 6/7] mm/page_ext: inline unchecked page extension lookup Muchun Song
@ 2026-11-17 12:06 ` Muchun Song
2026-11-18 0:06 ` [PATCH v2 0/7] KMSAN: Move per-page metadata out of struct page Chris Down
7 siblings, 0 replies; 19+ messages in thread
From: Muchun Song @ 2026-11-17 12:06 UTC (permalink / raw)
To: Alexander Potapenko, Andrew Morton
Cc: Marco Elver, Dmitry Vyukov, Chris Down, David Hildenbrand,
Liam R . Howlett, Vlastimil Babka, Mike Rapoport,
Suren Baghdasaryan, Michal Hocko, Johannes Weiner,
Brendan Jackman, Zi Yan, Muchun Song, Joerg Roedel, Lu Baolu,
Pranjal Shrivastava, Ketan, Mostafa Saleh, Sang-Heon Jeon,
Jonathan Corbet, Shuah Khan, Randy Dunlap, Sashiko, kasan-dev,
workflows, linux-doc, linux-kernel, linux-mm, Muchun Song
KMSAN stores each page's shadow and origin pointers in struct page.
Boot-time gigantic HugeTLB pages are reinitialized after
kmsan_init_shadow() installs these pointers, dropping the mappings and
leaving the allocated metadata pages unreachable.
Move the pointers to page_ext, whose boot-time storage is available
before kmsan_init_shadow() and survives struct page reinitialization.
Use the inline unchecked lookup under notrace preemption so metadata
lookup cannot recurse through instrumented hooks.
This shrinks struct page from 80 to 64 bytes on x86-64 KMSAN builds,
making its size a power of two and allowing HugeTLB Vmemmap Optimization
with KMSAN.
Fixes: fde1c4ecf916 ("mm: hugetlb: skip initialization of gigantic tail struct pages if freed by HVO")
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
Reported-by: Sashiko <sashiko-bot@kernel.org>
Closes: https://sashiko.dev/#/bug/linux-ac161ba4-30cc-4bd3-b93c-ab7af8a6f64f
---
v2:
- Update __mm_zero_struct_page() for the smaller struct page.
- Keep the unchecked lookup inline in KMSAN and leave page_ext.c
instrumented (suggested by Chris Down)
- Document page_ext metadata storage and KMSAN HVO support (suggested
by Chris Down)
- Use the common boot-time page_ext allocation instead of need_early
---
Documentation/dev-tools/kmsan.rst | 8 +++---
include/linux/kmsan.h | 2 ++
include/linux/mm.h | 12 +++------
include/linux/mm_types.h | 12 ---------
lib/Kconfig.kmsan | 1 +
mm/kmsan/shadow.c | 45 ++++++++++++++++++++++++++-----
mm/page_ext.c | 4 +++
7 files changed, 52 insertions(+), 32 deletions(-)
diff --git a/Documentation/dev-tools/kmsan.rst b/Documentation/dev-tools/kmsan.rst
index 0dc668b183f6..5330d21d141d 100644
--- a/Documentation/dev-tools/kmsan.rst
+++ b/Documentation/dev-tools/kmsan.rst
@@ -376,13 +376,11 @@ Metadata allocation
There are several places in the kernel for which the metadata is stored.
-1. Each ``struct page`` instance contains two pointers to its shadow and
-origin pages::
+1. Each page frame has a corresponding ``struct page_metadata`` entry containing
+two pointers to its shadow and origin pages::
- struct page {
- ...
+ struct page_metadata {
struct page *shadow, *origin;
- ...
};
At boot-time, the kernel allocates shadow and origin pages for every available
diff --git a/include/linux/kmsan.h b/include/linux/kmsan.h
index 7da9fd506b39..08fb79566235 100644
--- a/include/linux/kmsan.h
+++ b/include/linux/kmsan.h
@@ -15,12 +15,14 @@
#include <linux/types.h>
struct page;
+struct page_ext_operations;
struct kmem_cache;
struct task_struct;
struct scatterlist;
struct urb;
#ifdef CONFIG_KMSAN
+extern struct page_ext_operations kmsan_ops;
/**
* kmsan_task_create() - Initialize KMSAN state for the task.
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 77b5d450fe45..9349209cd2c0 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -133,7 +133,7 @@ extern int mmap_rnd_compat_bits __read_mostly;
* define their own version of this macro in <asm/pgtable.h>
*/
#if BITS_PER_LONG == 64
-/* This function must be updated when the size of struct page grows above 96
+/* This function must be updated when the size of struct page grows above 80
* or reduces below 56. The idea that compiler optimizes out switch()
* statement, and only leaves move/store instructions. Also the compiler can
* combine write statements if they are both assignments and can be reordered,
@@ -144,18 +144,12 @@ static inline void __mm_zero_struct_page(struct page *page)
{
unsigned long *_pp = (void *)page;
- /* Check that struct page is either 56, 64, 72, 80, 88 or 96 bytes */
+ /* Check that struct page is either 56, 64, 72 or 80 bytes */
BUILD_BUG_ON(sizeof(struct page) & 7);
BUILD_BUG_ON(sizeof(struct page) < 56);
- BUILD_BUG_ON(sizeof(struct page) > 96);
+ BUILD_BUG_ON(sizeof(struct page) > 80);
switch (sizeof(struct page)) {
- case 96:
- _pp[11] = 0;
- fallthrough;
- case 88:
- _pp[10] = 0;
- fallthrough;
case 80:
_pp[9] = 0;
fallthrough;
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 1ed35d258d2e..dc73f9f5a51b 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -208,18 +208,6 @@ struct page {
#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
int _last_cpupid;
#endif
-
-#ifdef CONFIG_KMSAN
- /*
- * KMSAN metadata for this page:
- * - shadow page: every bit indicates whether the corresponding
- * bit of the original page is initialized (0) or not (1);
- * - origin page: every 4 bytes contain an id of the stack trace
- * where the uninitialized value was created.
- */
- struct page *kmsan_shadow;
- struct page *kmsan_origin;
-#endif
} _struct_page_alignment;
/*
diff --git a/lib/Kconfig.kmsan b/lib/Kconfig.kmsan
index cae1ddcc18e1..015e85c70793 100644
--- a/lib/Kconfig.kmsan
+++ b/lib/Kconfig.kmsan
@@ -10,6 +10,7 @@ config KMSAN
depends on HAVE_ARCH_KMSAN && HAVE_KMSAN_COMPILER
depends on DEBUG_KERNEL && !KASAN && !KCSAN
depends on !PREEMPT_RT
+ select PAGE_EXTENSION
select STACKDEPOT
select STACKDEPOT_ALWAYS_INIT
help
diff --git a/mm/kmsan/shadow.c b/mm/kmsan/shadow.c
index 300b48818af1..fa2a403d87c2 100644
--- a/mm/kmsan/shadow.c
+++ b/mm/kmsan/shadow.c
@@ -13,6 +13,7 @@
#include <linux/cacheflush.h>
#include <linux/memblock.h>
#include <linux/mm_types.h>
+#include <linux/page_ext.h>
#include <linux/slab.h>
#include <linux/smp.h>
#include <linux/stddef.h>
@@ -24,23 +25,55 @@
#define KMSAN_META_MASK_SHADOW BIT(0)
#define KMSAN_META_MASK_ORIGIN BIT(1)
+/*
+ * KMSAN metadata for this page:
+ * - shadow page: every bit indicates whether the corresponding
+ * bit of the original page is initialized (0) or not (1);
+ * - origin page: every 4 bytes contain an id of the stack trace
+ * where the uninitialized value was created.
+ */
struct page_metadata {
struct page *shadow;
struct page *origin;
};
+static __init bool need_page_metadata(void)
+{
+ return true;
+}
+
+struct page_ext_operations kmsan_ops = {
+ .size = sizeof(struct page_metadata),
+ .need = need_page_metadata,
+};
+
static inline struct page_metadata get_page_metadata(struct page *page)
{
- return (struct page_metadata) {
- .shadow = page->kmsan_shadow,
- .origin = page->kmsan_origin,
- };
+ struct page_metadata meta = {};
+ struct page_ext *page_ext;
+
+ /*
+ * Avoid instrumented preemption debug and tracing hooks while resolving
+ * KMSAN metadata, as they would recurse back into metadata lookup.
+ */
+ preempt_disable_notrace();
+ page_ext = __page_ext_lookup(page_to_pfn(page));
+ if (page_ext)
+ meta = *(struct page_metadata *)page_ext_data(page_ext, &kmsan_ops);
+ preempt_enable_no_resched_notrace();
+
+ return meta;
}
static inline void set_page_metadata(struct page *page, struct page_metadata meta)
{
- page->kmsan_shadow = meta.shadow;
- page->kmsan_origin = meta.origin;
+ struct page_ext *page_ext;
+
+ preempt_disable_notrace();
+ page_ext = __page_ext_lookup(page_to_pfn(page));
+ if (page_ext)
+ *(struct page_metadata *)page_ext_data(page_ext, &kmsan_ops) = meta;
+ preempt_enable_no_resched_notrace();
}
static inline void *shadow_ptr_for(struct page_metadata meta)
diff --git a/mm/page_ext.c b/mm/page_ext.c
index 0f31e2da2611..274c3e778be6 100644
--- a/mm/page_ext.c
+++ b/mm/page_ext.c
@@ -12,6 +12,7 @@
#include <linux/rcupdate.h>
#include <linux/pgalloc_tag.h>
#include <linux/iommu-debug-pagealloc.h>
+#include <linux/kmsan.h>
#include <linux/slab.h>
/*
@@ -87,6 +88,9 @@ static struct page_ext_operations *page_ext_ops[] __initdata = {
#ifdef CONFIG_PAGE_TABLE_CHECK
&page_table_check_ops,
#endif
+#ifdef CONFIG_KMSAN
+ &kmsan_ops,
+#endif
#ifdef CONFIG_IOMMU_DEBUG_PAGEALLOC
&page_iommu_debug_ops,
#endif
--
2.54.0
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [PATCH v2 0/7] KMSAN: Move per-page metadata out of struct page
2026-11-17 12:06 [PATCH v2 0/7] KMSAN: Move per-page metadata out of struct page Muchun Song
` (6 preceding siblings ...)
2026-11-17 12:06 ` [PATCH v2 7/7] kmsan: fix metadata loss during HugeTLB struct page reinitialization Muchun Song
@ 2026-11-18 0:06 ` Chris Down
2026-11-18 3:12 ` Muchun Song
7 siblings, 1 reply; 19+ messages in thread
From: Chris Down @ 2026-11-18 0:06 UTC (permalink / raw)
To: Muchun Song
Cc: Alexander Potapenko, Andrew Morton, Marco Elver, Dmitry Vyukov,
David Hildenbrand, Liam R . Howlett, Vlastimil Babka,
Mike Rapoport, Suren Baghdasaryan, Michal Hocko, Johannes Weiner,
Brendan Jackman, Zi Yan, Muchun Song, Joerg Roedel, Lu Baolu,
Pranjal Shrivastava, Ketan, Mostafa Saleh, Sang-Heon Jeon,
Jonathan Corbet, Shuah Khan, Randy Dunlap, Sashiko, kasan-dev,
workflows, linux-doc, linux-kernel, linux-mm
Hi Muchun,
Thanks! The changes look great, thanks for working on this.
I think patch 5 still has some issues and have replied there, but for patches
1-4, 6, and 7, feel free to add:
Reviewed-by: Chris Down <chris@chrisdown.name>
Tested-by: Chris Down <chris@chrisdown.name>
Thanks!
Chris
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [PATCH v2 0/7] KMSAN: Move per-page metadata out of struct page
2026-11-18 0:06 ` [PATCH v2 0/7] KMSAN: Move per-page metadata out of struct page Chris Down
@ 2026-11-18 3:12 ` Muchun Song
0 siblings, 0 replies; 19+ messages in thread
From: Muchun Song @ 2026-11-18 3:12 UTC (permalink / raw)
To: Chris Down
Cc: Muchun Song, Alexander Potapenko, Andrew Morton, Marco Elver,
Dmitry Vyukov, David Hildenbrand, Liam R . Howlett,
Vlastimil Babka, Mike Rapoport, Suren Baghdasaryan, Michal Hocko,
Johannes Weiner, Brendan Jackman, Zi Yan, Joerg Roedel, Lu Baolu,
Pranjal Shrivastava, Ketan, Mostafa Saleh, Sang-Heon Jeon,
Jonathan Corbet, Shuah Khan, Randy Dunlap, Sashiko, kasan-dev,
workflows, linux-doc, linux-kernel, linux-mm
> On Nov 18, 2026, at 08:06, Chris Down <chris@chrisdown.name> wrote:
>
> Hi Muchun,
>
> Thanks! The changes look great, thanks for working on this.
>
> I think patch 5 still has some issues and have replied there, but for patches
> 1-4, 6, and 7, feel free to add:
>
> Reviewed-by: Chris Down <chris@chrisdown.name>
> Tested-by: Chris Down <chris@chrisdown.name>
Really thanks for your review and test.
Thanks,
Muchun
>
> Thanks!
>
> Chris
^ permalink raw reply [flat|nested] 19+ messages in thread