[PATCH 0/2] mm: optimize HugeTLB bootmem hugepage init

[PATCH 0/2] mm: optimize HugeTLB bootmem hugepage init

[Li Zhe]

This series speeds up the fallback initialization path for HugeTLB tail
struct pages during boot-time hugepage allocation, when HugeTLB vmemmap
optimization (HVO) is unavailable or fails.

Patch 1 factors the existing ZONE_DEVICE template-copy operation into
memmap_init_page_from_template().

Patch 2 reuses that helper in hugetlb_folio_init_tail_vmemmap():
initialize the first tail page normally, copy it into a local template,
then initialize the remaining tail pages by refreshing only PFN-dependent
fields before copying the template.

In the earlier discussion, Mike asked whether the same kind of
template-based initialization also helps the ordinary struct page
initialization path:

https://lore.kernel.org/all/akfB6Qe-jKDEMUxz@kernel.org/

I tested that direction as well, but did not see a measurable benefit
for ordinary struct page initialization. The measurable improvement I
could find is in the HugeTLB bootmem hugepage tail path, where gigantic
pages can require initializing a large number of tail struct pages when
HVO does not cover the allocation.

Tested on an arm64 QEMU/KVM guest with 4K base pages and 64 boot-time 1G
hugepages. HVO is unavailable on arm64 in v7.4-rc1, so the fallback tail
initialization path is exercised.

Results averaged over five boots, measured with a temporary
jiffies-based printk:

  base:             318.4 ms
  with this patch:  202.4 ms

This is a 36.43% reduction in the measured fallback tail initialization
time. The temporary measurement instrumentation is not part of this
series.

Li Zhe (2):
  mm/mm_init: factor out memmap_init_page_from_template()
  mm/hugetlb: optimize bootmem hugepage tail vmemmap init

 mm/hugetlb.c | 23 ++++++++++++++++++-----
 mm/mm_init.c | 29 +++++++++++++++--------------
 mm/mm_init.h |  2 ++
 3 files changed, 35 insertions(+), 19 deletions(-)

-- 
2.20.1

[PATCH 1/2] mm/mm_init: factor out memmap_init_page_from_template()

[Li Zhe]

ZONE_DEVICE memmap initialization already uses a reusable struct page
template when initializing compound tail pages.  The template mostly
contains invariant state, but a few fields still depend on the target
PFN and must be refreshed before copying the template into each
destination struct page.

Factor that operation into memmap_init_page_from_template() so other
boot-time memmap initialization paths can reuse the same helper.

No functional change intended.

Signed-off-by: Li Zhe <lizhe.67@bytedance.com>
---
 mm/mm_init.c | 29 +++++++++++++++--------------
 mm/mm_init.h |  2 ++
 2 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/mm/mm_init.c b/mm/mm_init.c
index 2a0e782a151d..495e20fbdec0 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -625,6 +625,17 @@ void __meminit __init_single_page(struct page *page, unsigned long pfn,
 	vmemmap_optimization_verify_zone(page);
 }
 
+void __meminit memmap_init_page_from_template(struct page *page,
+		unsigned long pfn, unsigned long zone, struct page *template)
+{
+	set_page_section_from_pfn(template, pfn);
+#ifdef WANT_PAGE_VIRTUAL
+	if (!is_highmem_idx(zone))
+		set_page_address(template, __va(pfn << PAGE_SHIFT));
+#endif
+	memcpy_nontemporal(page, template, sizeof(*page));
+}
+
 #ifdef CONFIG_NUMA
 /*
  * During memory init memblocks map pfns to nids. The search is expensive and
@@ -1073,17 +1084,6 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
 	}
 }
 
-static void zone_device_page_init_from_template(struct page *page,
-		unsigned long pfn, struct page *template)
-{
-	set_page_section_from_pfn(template, pfn);
-#ifdef WANT_PAGE_VIRTUAL
-	if (!is_highmem_idx(ZONE_DEVICE))
-		set_page_address(template, __va(pfn << PAGE_SHIFT));
-#endif
-	memcpy_nontemporal(page, template, sizeof(*page));
-}
-
 /*
  * With compound page geometry and when struct pages are stored in ram most
  * tail pages are reused. Consequently, the amount of unique struct pages to
@@ -1136,8 +1136,8 @@ static void __ref memmap_init_compound(struct page *head,
 
 	/* Initialize the remaining tail pages from template. */
 	for (pfn = head_pfn + 2; pfn < end_pfn; pfn++)
-		zone_device_page_init_from_template(pfn_to_page(pfn), pfn,
-						    &template);
+		memmap_init_page_from_template(pfn_to_page(pfn), pfn,
+					       ZONE_DEVICE, &template);
 	prep_compound_head(head, order);
 }
 
@@ -1192,7 +1192,8 @@ void __ref memmap_init_zone_device(struct zone *zone,
 	for (; pfn < end_pfn; pfn += pfns_per_compound) {
 		page = pfn_to_page(pfn);
 
-		zone_device_page_init_from_template(page, pfn, &template);
+		memmap_init_page_from_template(page, pfn, ZONE_DEVICE,
+					       &template);
 
 		if (IS_ALIGNED(pfn, PAGES_PER_SECTION))
 			cond_resched();
diff --git a/mm/mm_init.h b/mm/mm_init.h
index 0034e37dbabd..22c465122b47 100644
--- a/mm/mm_init.h
+++ b/mm/mm_init.h
@@ -111,5 +111,7 @@ void memblock_free_all(void);
 
 void __meminit __init_single_page(struct page *page, unsigned long pfn,
 				unsigned long zone, int nid);
+void __meminit memmap_init_page_from_template(struct page *page,
+		unsigned long pfn, unsigned long zone, struct page *template);
 
 #endif /* __MM_MM_INIT_H */
-- 
2.20.1

[PATCH 2/2] mm/hugetlb: optimize bootmem hugepage tail vmemmap init

[Li Zhe]

Boot-time HugeTLB allocation has to initialize tail struct pages itself
when HugeTLB vmemmap optimization (HVO) is unavailable or fails.  For
gigantic pages, this can cover a large number of tail pages.  Most state
installed by __init_single_page() and prep_compound_tail() is identical
for every tail page in the same huge page; only PFN-dependent fields
need refreshing.

Initialize the first tail page normally, copy it into a local template,
then initialize the remaining tails from that template with
memmap_init_page_from_template().  This avoids repeatedly doing the full
__init_single_page() and prep_compound_tail() sequence for state that is
identical across tail pages in the same huge page.

On an arm64 QEMU/KVM guest using 4K pages and 64 boot-time 1G hugepages
(HVO unavailable on arm64 in v7.4-rc1), the measured fallback tail
initialization time improved by 36.43%, measured with a temporary
jiffies-based printk modeled after the existing timing in
memmap_init_zone_device():

  base:             318.4 ms
  with this patch:  202.4 ms

Signed-off-by: Li Zhe <lizhe.67@bytedance.com>
---
 mm/hugetlb.c | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index da5d04edf7c1..987f16a5ea27 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3219,19 +3219,32 @@ static void __init hugetlb_folio_init_tail_vmemmap(struct folio *folio,
 	enum zone_type zone = folio_zonenum(folio);
 	int nid = folio_nid(folio);
 	struct page *page = folio_page(folio, start_page_number);
+	struct page template;
 	unsigned long head_pfn = folio_pfn(folio);
 	unsigned long pfn, end_pfn = head_pfn + end_page_number;
 	unsigned int order = huge_page_order(h);
 
+	if (start_page_number >= end_page_number)
+		return;
+
 	/*
 	 * As we marked all tail pages with memblock_reserved_mark_noinit(),
 	 * we must initialize them ourselves here.
 	 */
-	for (pfn = head_pfn + start_page_number; pfn < end_pfn; page++, pfn++) {
-		__init_single_page(page, pfn, zone, nid);
-		prep_compound_tail(page, &folio->page, order);
-		set_page_count(page, 0);
-	}
+	pfn = head_pfn + start_page_number;
+
+	/*
+	 * Seed a template from the first real tail page and refresh only
+	 * PFN-dependent fields before copying it.
+	 */
+	__init_single_page(page, pfn, zone, nid);
+	prep_compound_tail(page, &folio->page, order);
+	set_page_count(page, 0);
+	memcpy(&template, page, sizeof(template));
+
+	for (pfn++; pfn < end_pfn; pfn++)
+		memmap_init_page_from_template(pfn_to_page(pfn), pfn, zone,
+					       &template);
 }
 
 static void __init hugetlb_folio_init_vmemmap(struct folio *folio,
-- 
2.20.1

Re: [PATCH 2/2] mm/hugetlb: optimize bootmem hugepage tail vmemmap init

[Balbir Singh]

On Thu, Nov 05, 2026 at 08:06:15PM +0800, Li Zhe wrote:
> Boot-time HugeTLB allocation has to initialize tail struct pages itself
> when HugeTLB vmemmap optimization (HVO) is unavailable or fails.  For
> gigantic pages, this can cover a large number of tail pages.  Most state
> installed by __init_single_page() and prep_compound_tail() is identical
> for every tail page in the same huge page; only PFN-dependent fields
> need refreshing.
> 
> Initialize the first tail page normally, copy it into a local template,
> then initialize the remaining tails from that template with
> memmap_init_page_from_template().  This avoids repeatedly doing the full
> __init_single_page() and prep_compound_tail() sequence for state that is
> identical across tail pages in the same huge page.
> 
> On an arm64 QEMU/KVM guest using 4K pages and 64 boot-time 1G hugepages
> (HVO unavailable on arm64 in v7.4-rc1), the measured fallback tail
> initialization time improved by 36.43%, measured with a temporary
> jiffies-based printk modeled after the existing timing in
> memmap_init_zone_device():
> 
>   base:             318.4 ms
>   with this patch:  202.4 ms
> 
> Signed-off-by: Li Zhe <lizhe.67@bytedance.com>
> ---
>  mm/hugetlb.c | 23 ++++++++++++++++++-----
>  1 file changed, 18 insertions(+), 5 deletions(-)
> 
> diff --git a/mm/hugetlb.c b/mm/hugetlb.c
> index da5d04edf7c1..987f16a5ea27 100644
> --- a/mm/hugetlb.c
> +++ b/mm/hugetlb.c
> @@ -3219,19 +3219,32 @@ static void __init hugetlb_folio_init_tail_vmemmap(struct folio *folio,
>  	enum zone_type zone = folio_zonenum(folio);
>  	int nid = folio_nid(folio);
>  	struct page *page = folio_page(folio, start_page_number);
> +	struct page template;
>  	unsigned long head_pfn = folio_pfn(folio);
>  	unsigned long pfn, end_pfn = head_pfn + end_page_number;
>  	unsigned int order = huge_page_order(h);
>  
> +	if (start_page_number >= end_page_number)
> +		return;
> +

Can this actually happen?

>  	/*
>  	 * As we marked all tail pages with memblock_reserved_mark_noinit(),
>  	 * we must initialize them ourselves here.
>  	 */
> -	for (pfn = head_pfn + start_page_number; pfn < end_pfn; page++, pfn++) {
> -		__init_single_page(page, pfn, zone, nid);
> -		prep_compound_tail(page, &folio->page, order);
> -		set_page_count(page, 0);
> -	}
> +	pfn = head_pfn + start_page_number;
> +
> +	/*
> +	 * Seed a template from the first real tail page and refresh only
> +	 * PFN-dependent fields before copying it.
> +	 */
> +	__init_single_page(page, pfn, zone, nid);
> +	prep_compound_tail(page, &folio->page, order);
> +	set_page_count(page, 0);
> +	memcpy(&template, page, sizeof(template));

Do we need a separate template and page?

> +
> +	for (pfn++; pfn < end_pfn; pfn++)
> +		memmap_init_page_from_template(pfn_to_page(pfn), pfn, zone,
> +					       &template);
>  }
>  
>  static void __init hugetlb_folio_init_vmemmap(struct folio *folio,

Acked-by: Balbir Singh <balbirs@nvidia.com>

Re: [PATCH 1/2] mm/mm_init: factor out memmap_init_page_from_template()

[Balbir Singh]

On Thu, Nov 05, 2026 at 08:06:14PM +0800, Li Zhe wrote:
> ZONE_DEVICE memmap initialization already uses a reusable struct page
> template when initializing compound tail pages.  The template mostly
> contains invariant state, but a few fields still depend on the target
> PFN and must be refreshed before copying the template into each
> destination struct page.
> 
> Factor that operation into memmap_init_page_from_template() so other
> boot-time memmap initialization paths can reuse the same helper.
> 
> No functional change intended.
> 
> Signed-off-by: Li Zhe <lizhe.67@bytedance.com>
> ---
>  mm/mm_init.c | 29 +++++++++++++++--------------
>  mm/mm_init.h |  2 ++
>  2 files changed, 17 insertions(+), 14 deletions(-)
> 
> diff --git a/mm/mm_init.c b/mm/mm_init.c
> index 2a0e782a151d..495e20fbdec0 100644
> --- a/mm/mm_init.c
> +++ b/mm/mm_init.c
> @@ -625,6 +625,17 @@ void __meminit __init_single_page(struct page *page, unsigned long pfn,
>  	vmemmap_optimization_verify_zone(page);
>  }
>  
> +void __meminit memmap_init_page_from_template(struct page *page,
> +		unsigned long pfn, unsigned long zone, struct page *template)
> +{
> +	set_page_section_from_pfn(template, pfn);
> +#ifdef WANT_PAGE_VIRTUAL
> +	if (!is_highmem_idx(zone))
> +		set_page_address(template, __va(pfn << PAGE_SHIFT));
> +#endif
> +	memcpy_nontemporal(page, template, sizeof(*page));
> +}
> +

Can we make the template a const? Why can't we do

void __meminit memmap_init_page_from_template(struct page *page,
		unsigned long pfn, unsigned long zone,
		const struct page *template)
{
	memcpy_nontemporal(page, template, sizeof(*page));

	set_page_section_from_pfn(page, pfn);
#ifdef WANT_PAGE_VIRTUAL
	if (!is_highmem_idx(zone))
		set_page_address(page, __va(pfn << PAGE_SHIFT));
#endif

That way we don't update template once it is set, if template is r/o,
then we don't need both first page and template in the caller to
be different.


>  #ifdef CONFIG_NUMA
>  /*
>   * During memory init memblocks map pfns to nids. The search is expensive and
> @@ -1073,17 +1084,6 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
>  	}
>  }
>  
> -static void zone_device_page_init_from_template(struct page *page,
> -		unsigned long pfn, struct page *template)
> -{
> -	set_page_section_from_pfn(template, pfn);
> -#ifdef WANT_PAGE_VIRTUAL
> -	if (!is_highmem_idx(ZONE_DEVICE))
> -		set_page_address(template, __va(pfn << PAGE_SHIFT));
> -#endif
> -	memcpy_nontemporal(page, template, sizeof(*page));
> -}
> -
>  /*
>   * With compound page geometry and when struct pages are stored in ram most
>   * tail pages are reused. Consequently, the amount of unique struct pages to
> @@ -1136,8 +1136,8 @@ static void __ref memmap_init_compound(struct page *head,
>  
>  	/* Initialize the remaining tail pages from template. */
>  	for (pfn = head_pfn + 2; pfn < end_pfn; pfn++)
> -		zone_device_page_init_from_template(pfn_to_page(pfn), pfn,
> -						    &template);
> +		memmap_init_page_from_template(pfn_to_page(pfn), pfn,
> +					       ZONE_DEVICE, &template);
>  	prep_compound_head(head, order);
>  }
>  
> @@ -1192,7 +1192,8 @@ void __ref memmap_init_zone_device(struct zone *zone,
>  	for (; pfn < end_pfn; pfn += pfns_per_compound) {
>  		page = pfn_to_page(pfn);
>  
> -		zone_device_page_init_from_template(page, pfn, &template);
> +		memmap_init_page_from_template(page, pfn, ZONE_DEVICE,
> +					       &template);
>  
>  		if (IS_ALIGNED(pfn, PAGES_PER_SECTION))
>  			cond_resched();
> diff --git a/mm/mm_init.h b/mm/mm_init.h
> index 0034e37dbabd..22c465122b47 100644
> --- a/mm/mm_init.h
> +++ b/mm/mm_init.h
> @@ -111,5 +111,7 @@ void memblock_free_all(void);
>  
>  void __meminit __init_single_page(struct page *page, unsigned long pfn,
>  				unsigned long zone, int nid);
> +void __meminit memmap_init_page_from_template(struct page *page,
> +		unsigned long pfn, unsigned long zone, struct page *template);
>  
>  #endif /* __MM_MM_INIT_H */
> -- 


Balbir