[PATCH v3 2/2] mm/hugetlb: optimize bootmem hugepage tail vmemmap init

["Li Zhe" <lizhe.67@bytedance.com>]

Boot-time HugeTLB allocation has to initialize tail struct pages itself
when HugeTLB vmemmap optimization (HVO) is unavailable or fails.  For
gigantic pages, this can cover a large number of tail pages.  Most state
installed by __init_single_page() and prep_compound_tail() is identical
for every tail page in the same huge page; only PFN-dependent fields need
refreshing.

Initialize the first tail page normally, then initialize the remaining
tails with hugetlb_init_tail_page().  This copies the already
initialized first tail page with memcpy() and refreshes only the
PFN-dependent fields in the destination page, while avoiding the full
__init_single_page() and prep_compound_tail() sequence for state that is
identical across tail pages in the same huge page.

The fallback path was measured on a v7.4-rc2 base with a temporary
jiffies-based printk modeled after the existing timing in
memmap_init_zone_device(), using 4K base pages and 64 boot-time 1G
hugepages.  HVO is unavailable on arm64; on x86-64,
hugetlb_free_vmemmap=off was used to exercise the fallback path.

  arch    base      with this patch  reduction
  arm64   360.8 ms  256.0 ms         29.05%
  x86-64  171.0 ms  165.0 ms          3.51%

Signed-off-by: Li Zhe <lizhe.67@bytedance.com>
---
 mm/hugetlb.c | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 630c96cd1e88..13efadf64023 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3214,6 +3214,14 @@ static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid)
 	return true;
 }
 
+static void __init hugetlb_init_tail_page(struct page *page, unsigned long pfn,
+					  enum zone_type zone,
+					  const struct page *source)
+{
+	memcpy(page, source, sizeof(*page));
+	memmap_init_refresh_pfn_fields(page, pfn, zone);
+}
+
 /* Initialize [start_page:end_page_number] tail struct pages of a hugepage */
 static void __init hugetlb_folio_init_tail_vmemmap(struct folio *folio,
 					struct hstate *h,
@@ -3231,11 +3239,16 @@ static void __init hugetlb_folio_init_tail_vmemmap(struct folio *folio,
 	 * As we marked all tail pages with memblock_reserved_mark_noinit(),
 	 * we must initialize them ourselves here.
 	 */
-	for (pfn = head_pfn + start_page_number; pfn < end_pfn; page++, pfn++) {
-		__init_single_page(page, pfn, zone, nid);
-		prep_compound_tail(page, &folio->page, order);
-		set_page_count(page, 0);
-	}
+	if (start_page_number >= end_page_number)
+		return;
+
+	pfn = head_pfn + start_page_number;
+	__init_single_page(page, pfn, zone, nid);
+	prep_compound_tail(page, &folio->page, order);
+	set_page_count(page, 0);
+
+	for (pfn++; pfn < end_pfn; pfn++)
+		hugetlb_init_tail_page(pfn_to_page(pfn), pfn, zone, page);
 }
 
 static void __init hugetlb_folio_init_vmemmap(struct folio *folio,
-- 
2.20.1