[PATCH] x86/mm: Split a 1G direct mapping that hot-remove only partly covers

[Cong Wang <xiyou.wangcong@gmail.com>]

From: Cong Wang <cwang@multikernel.io>

Hot-remove works on memory blocks, 128M and up, while the kernel maps
boot memory with 1G pages where it can. Removing a block that sits in
such a 1G page covers only part of it, and remove_pud_table() then
walks the memory that 1G page maps as if it were a PMD table. The
removal succeeds, but it corrupts the first page of the 1G region, may
free that page while still in use, and leaves the removed memory
mapped.

Map the 1G leaf through 2M leaves first, under pgd_lock as
set_memory_*() may change it for the rest of the 1G meanwhile.

Reproduced in a 3G QEMU guest booted with movablecore=1280M. We used a
test module fills the first page of a 1G-mapped region with a known
pattern, then calls offline_and_remove_memory() on one 128M block in
that region. Before this patch, part of the pattern is zeroed and the
1G page stays. After this patch, the pattern is intact, the block is
unmapped, the rest stays mapped through 2M pages, and adding the block
back works.

Fixes: 69ccfe74e16b ("x86/vmemmap: drop handling of 1GB vmemmap ranges")
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@kernel.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Cc: David Hildenbrand <david@kernel.org>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Cong Wang <cwang@multikernel.io>
---
 arch/x86/mm/init_64.c | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 70e682180291..00770b86b270 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -1164,6 +1164,37 @@ remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end,
 		update_page_count(PG_LEVEL_2M, -pages);
 }
 
+/* A hot-remove range may cover only part of a 1G page */
+static bool __meminit split_direct_pud(pud_t *pud)
+{
+	pmd_t *pmd = (pmd_t *)__get_free_page(GFP_KERNEL);
+	unsigned long pfn;
+	pgprot_t prot;
+	int i;
+
+	if (!pmd)
+		return false;
+
+	/* set_memory_*() may split or change it for the rest of the 1G */
+	spin_lock(&pgd_lock);
+	if (!pud_leaf(*pud)) {
+		spin_unlock(&pgd_lock);
+		free_page((unsigned long)pmd);
+		return true;
+	}
+	pfn = pud_pfn(*pud);
+	prot = pud_pgprot(*pud);
+	for (i = 0; i < PTRS_PER_PMD; i++)
+		set_pmd(pmd + i, pfn_pmd(pfn + i * (PMD_SIZE >> PAGE_SHIFT), prot));
+	/* Same frames and attributes, so no flush until remove_pagetable() */
+	pud_populate(&init_mm, pud, pmd);
+	spin_unlock(&pgd_lock);
+
+	update_page_count(PG_LEVEL_1G, -1);
+	update_page_count(PG_LEVEL_2M, PTRS_PER_PMD);
+	return true;
+}
+
 static void __meminit
 remove_pud_table(pud_t *pud_start, unsigned long addr, unsigned long end,
 		 struct vmem_altmap *altmap, bool direct)
@@ -1189,6 +1220,9 @@ remove_pud_table(pud_t *pud_start, unsigned long addr, unsigned long end,
 			continue;
 		}
 
+		if (pud_leaf(*pud) && (!direct || !split_direct_pud(pud)))
+			continue;
+
 		pmd_base = pmd_offset(pud, 0);
 		remove_pmd_table(pmd_base, addr, next, direct, altmap);
 		free_pmd_table(pmd_base, pud);
-- 
2.43.0