From: Muchun Song <songmuchun@bytedance.com>
To: Alexander Potapenko <glider@google.com>,
Andrew Morton <akpm@linux-foundation.org>
Cc: Marco Elver <elver@google.com>,
Dmitry Vyukov <dvyukov@google.com>,
Chris Down <chris@chrisdown.name>,
David Hildenbrand <david@kernel.org>,
"Liam R . Howlett" <liam@infradead.org>,
Vlastimil Babka <vbabka@kernel.org>,
Mike Rapoport <rppt@kernel.org>,
Suren Baghdasaryan <surenb@google.com>,
Michal Hocko <mhocko@suse.com>,
Johannes Weiner <hannes@cmpxchg.org>,
Brendan Jackman <brendan.jackman@linux.dev>,
Zi Yan <ziy@nvidia.com>, Muchun Song <muchun.song@linux.dev>,
Joerg Roedel <joerg.roedel@amd.com>,
Lu Baolu <baolu.lu@linux.intel.com>,
Pranjal Shrivastava <praan@google.com>,
Ketan <ketan.kishore@oss.qualcomm.com>,
Mostafa Saleh <smostafa@google.com>,
Sang-Heon Jeon <ekffu200098@gmail.com>,
Jonathan Corbet <corbet@lwn.net>,
Shuah Khan <skhan@linuxfoundation.org>,
Randy Dunlap <rdunlap@infradead.org>,
Sashiko <sashiko-bot@kernel.org>,
kasan-dev@googlegroups.com, workflows@vger.kernel.org,
linux-doc@vger.kernel.org, linux-kernel@vger.kernel.org,
linux-mm@kvack.org, Muchun Song <songmuchun@bytedance.com>
Subject: [PATCH v2 6/7] mm/page_ext: inline unchecked page extension lookup
Date: Tue, 17 Nov 2026 20:06:26 +0800 [thread overview]
Message-ID: <20261117120627.50601-7-songmuchun@bytedance.com> (raw)
In-Reply-To: <20261117120627.50601-1-songmuchun@bytedance.com>
The KMSAN runtime needs an unchecked page extension lookup that can be
inlined into its uninstrumented compilation unit while page_ext.c remains
instrumented.
Provide an inline lookup for this use case while keeping the existing RCU
checks for other callers.
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
---
v2:
- Add a preparatory patch that inlines the unchecked lookup (suggested
by Chris Down)
- Use a PFN interface to avoid architecture header-order dependencies
---
include/linux/page_ext.h | 46 ++++++++++++++++++++++++++++++
mm/page_ext.c | 60 +++-------------------------------------
2 files changed, 50 insertions(+), 56 deletions(-)
diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h
index d00320106a58..166f8be11564 100644
--- a/include/linux/page_ext.h
+++ b/include/linux/page_ext.h
@@ -64,7 +64,14 @@ static inline bool early_page_ext_enabled(void)
void page_ext_alloc(void);
void page_ext_init(void);
+static __always_inline struct page_ext *page_ext_entry(void *base, unsigned long index)
+{
+ return base + page_ext_size * index;
+}
+
#ifdef CONFIG_SPARSEMEM
+#define PAGE_EXT_INVALID 0x1
+
static inline bool page_ext_iter_next_fast_possible(unsigned long next_pfn)
{
/*
@@ -73,11 +80,50 @@ static inline bool page_ext_iter_next_fast_possible(unsigned long next_pfn)
*/
return next_pfn % PAGES_PER_SECTION;
}
+
+static __always_inline bool page_ext_invalid(struct page_ext *page_ext)
+{
+ return !page_ext || (((unsigned long)page_ext & PAGE_EXT_INVALID) == PAGE_EXT_INVALID);
+}
+
+static __always_inline struct page_ext *__page_ext_lookup(unsigned long pfn)
+{
+ struct mem_section *section = __pfn_to_section(pfn);
+ struct page_ext *page_ext = READ_ONCE(section->page_ext);
+
+ /*
+ * The sanity checks the page allocator does upon freeing a page can
+ * reach here before the page_ext arrays are allocated when feeding a
+ * range of pages to the allocator for the first time during bootup or
+ * memory hotplug.
+ */
+ if (page_ext_invalid(page_ext))
+ return NULL;
+ return page_ext_entry(page_ext, pfn);
+}
#else
static inline bool page_ext_iter_next_fast_possible(unsigned long next_pfn)
{
return true;
}
+
+static __always_inline struct page_ext *__page_ext_lookup(unsigned long pfn)
+{
+ int nid = pfn_to_nid(pfn);
+ struct page_ext *base = NODE_DATA(nid)->node_page_ext;
+ unsigned long index;
+
+ /*
+ * The sanity checks the page allocator does upon freeing a page can
+ * reach here before the page_ext arrays are allocated when feeding a
+ * range of pages to the allocator for the first time during bootup or
+ * memory hotplug.
+ */
+ if (unlikely(!base))
+ return NULL;
+ index = pfn - round_down(node_start_pfn(nid), MAX_ORDER_NR_PAGES);
+ return page_ext_entry(base, index);
+}
#endif
extern struct page_ext *page_ext_get(const struct page *page);
diff --git a/mm/page_ext.c b/mm/page_ext.c
index 2910727e99e2..0f31e2da2611 100644
--- a/mm/page_ext.c
+++ b/mm/page_ext.c
@@ -63,10 +63,6 @@
* can utilize this callback to initialize the state of it correctly.
*/
-#ifdef CONFIG_SPARSEMEM
-#define PAGE_EXT_INVALID (0x1)
-#endif
-
#if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT)
static bool need_page_idle(void)
{
@@ -175,33 +171,7 @@ static void __init invoke_init_callbacks(void)
}
}
-static inline struct page_ext *get_entry(void *base, unsigned long index)
-{
- return base + page_ext_size * index;
-}
-
#ifndef CONFIG_SPARSEMEM
-static struct page_ext *lookup_page_ext(const struct page *page)
-{
- unsigned long pfn = page_to_pfn(page);
- unsigned long index;
- struct page_ext *base;
-
- WARN_ON_ONCE(!rcu_read_lock_held());
- base = NODE_DATA(page_to_nid(page))->node_page_ext;
- /*
- * The sanity checks the page allocator does upon freeing a
- * page can reach here before the page_ext arrays are
- * allocated when feeding a range of pages to the allocator
- * for the first time during bootup or memory hotplug.
- */
- if (unlikely(!base))
- return NULL;
- index = pfn - round_down(node_start_pfn(page_to_nid(page)),
- MAX_ORDER_NR_PAGES);
- return get_entry(base, index);
-}
-
static int __init alloc_node_page_ext(int nid)
{
struct page_ext *base;
@@ -247,29 +217,6 @@ static int __init __page_ext_alloc(void)
}
#else /* CONFIG_SPARSEMEM */
-static bool page_ext_invalid(struct page_ext *page_ext)
-{
- return !page_ext || (((unsigned long)page_ext & PAGE_EXT_INVALID) == PAGE_EXT_INVALID);
-}
-
-static struct page_ext *lookup_page_ext(const struct page *page)
-{
- unsigned long pfn = page_to_pfn(page);
- struct mem_section *section = __pfn_to_section(pfn);
- struct page_ext *page_ext = READ_ONCE(section->page_ext);
-
- WARN_ON_ONCE(!rcu_read_lock_held());
- /*
- * The sanity checks the page allocator does upon freeing a
- * page can reach here before the page_ext arrays are
- * allocated when feeding a range of pages to the allocator
- * for the first time during bootup or memory hotplug.
- */
- if (page_ext_invalid(page_ext))
- return NULL;
- return get_entry(page_ext, pfn);
-}
-
static __ref void *alloc_page_ext(size_t size, int nid)
{
void *addr;
@@ -360,7 +307,7 @@ static void __free_page_ext(unsigned long pfn)
base = (void *)base - PAGE_EXT_INVALID;
WRITE_ONCE(ms->page_ext, NULL);
- base = get_entry(base, pfn);
+ base = page_ext_entry(base, pfn);
free_page_ext(base);
}
@@ -503,7 +450,8 @@ void __init page_ext_init(void)
*/
struct page_ext *page_ext_lookup(unsigned long pfn)
{
- return lookup_page_ext(pfn_to_page(pfn));
+ WARN_ON_ONCE(!rcu_read_lock_held());
+ return __page_ext_lookup(pfn);
}
/**
@@ -522,7 +470,7 @@ struct page_ext *page_ext_get(const struct page *page)
struct page_ext *page_ext;
rcu_read_lock();
- page_ext = lookup_page_ext(page);
+ page_ext = page_ext_lookup(page_to_pfn(page));
if (!page_ext) {
rcu_read_unlock();
return NULL;
--
2.54.0
next prev parent reply other threads:[~2026-11-17 12:07 UTC|newest]
Thread overview: 10+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-17 12:06 [PATCH v2 0/7] KMSAN: Move per-page metadata out of struct page Muchun Song
2026-11-17 12:06 ` [PATCH v2 1/7] kmsan: encapsulate per-page metadata access Muchun Song
2026-11-17 12:06 ` [PATCH v2 2/7] mm/page_ext: fix client data alignment Muchun Song
2026-11-17 12:06 ` [PATCH v2 3/7] mm/page_ext: always use vmalloc for section page extensions Muchun Song
2026-11-17 12:06 ` [PATCH v2 4/7] mm/page_ext: use common allocation and initialization entry points Muchun Song
2026-11-17 12:06 ` [PATCH v2 5/7] mm/page_ext: allocate page extensions early for SPARSEMEM Muchun Song
2026-11-18 0:06 ` Chris Down
2026-11-17 12:06 ` Muchun Song [this message]
2026-11-17 12:06 ` [PATCH v2 7/7] kmsan: fix metadata loss during HugeTLB struct page reinitialization Muchun Song
2026-11-18 0:06 ` [PATCH v2 0/7] KMSAN: Move per-page metadata out of struct page Chris Down
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261117120627.50601-7-songmuchun@bytedance.com \
--to=songmuchun@bytedance.com \
--cc=akpm@linux-foundation.org \
--cc=baolu.lu@linux.intel.com \
--cc=brendan.jackman@linux.dev \
--cc=chris@chrisdown.name \
--cc=corbet@lwn.net \
--cc=david@kernel.org \
--cc=dvyukov@google.com \
--cc=ekffu200098@gmail.com \
--cc=elver@google.com \
--cc=glider@google.com \
--cc=hannes@cmpxchg.org \
--cc=joerg.roedel@amd.com \
--cc=kasan-dev@googlegroups.com \
--cc=ketan.kishore@oss.qualcomm.com \
--cc=liam@infradead.org \
--cc=linux-doc@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=mhocko@suse.com \
--cc=muchun.song@linux.dev \
--cc=praan@google.com \
--cc=rdunlap@infradead.org \
--cc=rppt@kernel.org \
--cc=sashiko-bot@kernel.org \
--cc=skhan@linuxfoundation.org \
--cc=smostafa@google.com \
--cc=surenb@google.com \
--cc=vbabka@kernel.org \
--cc=workflows@vger.kernel.org \
--cc=ziy@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®