[PATCH 40/42] x86, 64bit: remove highmap for not needed ranges

[Yinghai Lu <yinghai@kernel.org>]

add cleanup_highmap_late to remove highmap for initmem, around rodata, and
[_brk_end, all_end).

Kernel Layout:

[    0.000000]   .text: [0x01000000-0x0200df88]
[    0.000000] .rodata: [0x02200000-0x02a1dfff]
[    0.000000]   .data: [0x02c00000-0x02e510ff]
[    0.000000]   .init: [0x02e53000-0x03213fff]
[    0.000000]    .bss: [0x03222000-0x0437cfff]
[    0.000000]    .brk: [0x0437d000-0x043a2fff]

Actually used brk:
[    0.270365] memblock_reserve: [0x0000000437d000-0x00000004383fff] flags 0x0 BRK

Before patch:
---[ High Kernel Mapping ]---
0xffffffff80000000-0xffffffff81000000          16M                           pmd
0xffffffff81000000-0xffffffff82000000          16M     ro         PSE GLB x  pmd
0xffffffff82000000-0xffffffff82011000          68K     ro             GLB x  pte
0xffffffff82011000-0xffffffff82200000        1980K     RW             GLB x  pte
0xffffffff82200000-0xffffffff82a00000           8M     ro         PSE GLB NX pmd
0xffffffff82a00000-0xffffffff82a1e000         120K     ro             GLB NX pte
0xffffffff82a1e000-0xffffffff82c00000        1928K     RW             GLB NX pte
0xffffffff82c00000-0xffffffff82e00000           2M     RW         PSE GLB NX pmd
0xffffffff82e00000-0xffffffff83000000           2M     RW             GLB NX pte
0xffffffff83000000-0xffffffff83200000           2M     RW         PSE GLB NX pmd
0xffffffff83200000-0xffffffff83400000           2M     RW             GLB NX pte
0xffffffff83400000-0xffffffff84400000          16M     RW         PSE GLB NX pmd
0xffffffff84400000-0xffffffffa0000000         444M                           pmd

After patch:
---[ High Kernel Mapping ]---
0xffffffff80000000-0xffffffff81000000          16M                           pmd
0xffffffff81000000-0xffffffff82000000          16M     ro         PSE GLB x  pmd
0xffffffff82000000-0xffffffff82012000          72K     ro             GLB x  pte
0xffffffff82012000-0xffffffff82200000        1976K                           pte
0xffffffff82200000-0xffffffff82a00000           8M     ro         PSE GLB NX pmd
0xffffffff82a00000-0xffffffff82a1e000         120K     ro             GLB NX pte
0xffffffff82a1e000-0xffffffff82c00000        1928K                           pte
0xffffffff82c00000-0xffffffff82e00000           2M     RW         PSE GLB NX pmd
0xffffffff82e00000-0xffffffff82e53000         332K     RW             GLB NX pte
0xffffffff82e53000-0xffffffff83000000        1716K                           pte
0xffffffff83000000-0xffffffff83200000           2M                           pmd
0xffffffff83200000-0xffffffff83214000          80K                           pte
0xffffffff83214000-0xffffffff83400000        1968K     RW             GLB NX pte
0xffffffff83400000-0xffffffff84200000          14M     RW         PSE GLB NX pmd
0xffffffff84200000-0xffffffff84384000        1552K     RW             GLB NX pte
0xffffffff84384000-0xffffffff84400000         496K                           pte
0xffffffff84400000-0xffffffffa0000000         444M                           pmd

So remove some range around rodata.

-v4: adapt it to all_end change.

Signed-off-by: Yinghai Lu <yinghai@kernel.org>
---
 arch/x86/mm/init_64.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 2507b98..38aa59c 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -1010,6 +1010,61 @@ void __init mem_init(void)
 }
 
 #ifdef CONFIG_DEBUG_RODATA
+static void remove_highmap_2m(unsigned long addr)
+{
+	pgd_t *pgd = pgd_offset_k(addr);
+	pud_t *pud = (pud_t *)pgd_page_vaddr(*pgd) + pud_index(addr);
+	pmd_t *pmd = (pmd_t *)pud_page_vaddr(*pud) + pmd_index(addr);
+
+	set_pmd(pmd, __pmd(0));
+}
+
+static void remove_highmap_2m_partial(unsigned long addr, unsigned long end)
+{
+	int i;
+	pgd_t *pgd = pgd_offset_k(addr);
+	pud_t *pud = (pud_t *)pgd_page_vaddr(*pgd) + pud_index(addr);
+	pmd_t *pmd = (pmd_t *)pud_page_vaddr(*pud) + pmd_index(addr);
+	pte_t *pte = (pte_t *)pmd_page_vaddr(*pmd) + pte_index(addr);
+
+	for (i = pte_index(addr); i < pte_index(end - 1) + 1; i++, pte++)
+		set_pte(pte, __pte(0));
+}
+
+static void cleanup_highmap_late(unsigned long start, unsigned long end)
+{
+	unsigned long addr;
+	unsigned long start_2m_aligned = roundup(start, PMD_SIZE);
+	unsigned long end_2m_aligned = rounddown(end, PMD_SIZE);
+
+	start = PFN_ALIGN(start);
+	end &= PAGE_MASK;
+
+	if (start >= end)
+		return;
+
+	if (start < start_2m_aligned) {
+		unsigned long tmp = min(start_2m_aligned, end);
+
+		set_memory_4k(start, (tmp - start) >> PAGE_SHIFT);
+		remove_highmap_2m_partial(start, tmp);
+	}
+
+	for (addr = start_2m_aligned; addr < end_2m_aligned; addr += PMD_SIZE)
+		remove_highmap_2m(addr);
+
+	if (start <= end_2m_aligned && end_2m_aligned < end) {
+		set_memory_4k(end_2m_aligned,
+				(end - end_2m_aligned) >> PAGE_SHIFT);
+		remove_highmap_2m_partial(end_2m_aligned, end);
+	}
+
+	subtract_range(pfn_highmapped, NR_RANGE,
+			__pa_symbol(start) >> PAGE_SHIFT,
+			__pa_symbol(end) >> PAGE_SHIFT);
+	nr_pfn_highmapped = clean_sort_range(pfn_highmapped, NR_RANGE);
+}
+
 const int rodata_test_data = 0xC3;
 EXPORT_SYMBOL_GPL(rodata_test_data);
 
@@ -1058,6 +1113,7 @@ void mark_rodata_ro(void)
 	unsigned long end = (unsigned long) &__end_rodata_hpage_align;
 	unsigned long text_end = PFN_ALIGN(&__stop___ex_table);
 	unsigned long rodata_end = PFN_ALIGN(&__end_rodata);
+	unsigned long data_start = PFN_ALIGN(&_sdata);
 	unsigned long all_end;
 
 	printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n",
@@ -1081,6 +1137,12 @@ void mark_rodata_ro(void)
 	all_end = roundup(_brk_end, PMD_SIZE);
 	set_memory_nx(rodata_start, (all_end - rodata_start) >> PAGE_SHIFT);
 
+	cleanup_highmap_late(text_end, rodata_start);
+	cleanup_highmap_late(rodata_end, data_start);
+	cleanup_highmap_late(PFN_ALIGN(_brk_end), all_end);
+	cleanup_highmap_late((unsigned long)(&__init_begin),
+				(unsigned long)(&__init_end));
+
 	rodata_test();
 
 #ifdef CONFIG_CPA_DEBUG
-- 
1.8.4.5