diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index dd85e093ffdb..cf583209df50 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1220,7 +1220,7 @@ void mark_rodata_ro(void) static void __init declare_vma(struct vm_struct *vma, void *va_start, void *va_end, - unsigned long vm_flags) + unsigned int nr_guard_pages) { phys_addr_t pa_start = __pa_symbol(va_start); unsigned long size = va_end - va_start; @@ -1228,14 +1228,14 @@ static void __init declare_vma(struct vm_struct *vma, BUG_ON(!PAGE_ALIGNED(pa_start)); BUG_ON(!PAGE_ALIGNED(size)); - if (!(vm_flags & VM_NO_GUARD)) - size += PAGE_SIZE; + size += nr_guard_pages * PAGE_SIZE; - vma->addr = va_start; - vma->phys_addr = pa_start; - vma->size = size; - vma->flags = VM_MAP | vm_flags; - vma->caller = __builtin_return_address(0); + vma->addr = va_start; + vma->phys_addr = pa_start; + vma->size = size; + vma->flags = VM_MAP; + vma->nr_guard_pages = nr_guard_pages; + vma->caller = __builtin_return_address(0); vm_area_add_early(vma); } @@ -1376,11 +1376,11 @@ static void __init declare_kernel_vmas(void) { static struct vm_struct vmlinux_seg[KERNEL_SEGMENT_COUNT]; - declare_vma(&vmlinux_seg[0], _text, _etext, VM_NO_GUARD); - declare_vma(&vmlinux_seg[1], __start_rodata, __inittext_begin, VM_NO_GUARD); - declare_vma(&vmlinux_seg[2], __inittext_begin, __inittext_end, VM_NO_GUARD); - declare_vma(&vmlinux_seg[3], __initdata_begin, __initdata_end, VM_NO_GUARD); - declare_vma(&vmlinux_seg[4], _data, _end, 0); + declare_vma(&vmlinux_seg[0], _text, _etext, 0); + declare_vma(&vmlinux_seg[1], __start_rodata, __inittext_begin, 0); + declare_vma(&vmlinux_seg[2], __inittext_begin, __inittext_end, 0); + declare_vma(&vmlinux_seg[3], __initdata_begin, __initdata_end, 0); + declare_vma(&vmlinux_seg[4], _data, _end, VM_DEFAULT_GUARD_PAGES); } void __pi_map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa, diff --git a/arch/loongarch/include/asm/kfence.h b/arch/loongarch/include/asm/kfence.h index da9e93024626..fa9562750312 100644 --- a/arch/loongarch/include/asm/kfence.h +++ b/arch/loongarch/include/asm/kfence.h @@ -23,6 +23,7 @@ static inline bool arch_kfence_init_pool(void) area = __get_vm_area_caller(KFENCE_POOL_SIZE, VM_IOREMAP, KFENCE_AREA_START, KFENCE_AREA_END, + VM_DEFAULT_GUARD_PAGES, __builtin_return_address(0)); if (!area) return false; diff --git a/arch/powerpc/kernel/pci_64.c b/arch/powerpc/kernel/pci_64.c index e27342ef128b..a29fc0350fb3 100644 --- a/arch/powerpc/kernel/pci_64.c +++ b/arch/powerpc/kernel/pci_64.c @@ -133,6 +133,7 @@ void __iomem *ioremap_phb(phys_addr_t paddr, unsigned long size) * reserved 64K legacy region. */ area = __get_vm_area_caller(size, VM_IOREMAP, PHB_IO_BASE, PHB_IO_END, + VM_DEFAULT_GUARD_PAGES, __builtin_return_address(0)); if (!area) return NULL; diff --git a/arch/sh/kernel/cpu/sh4/sq.c b/arch/sh/kernel/cpu/sh4/sq.c index 908a8e09113b..383f42f22dd5 100644 --- a/arch/sh/kernel/cpu/sh4/sq.c +++ b/arch/sh/kernel/cpu/sh4/sq.c @@ -104,7 +104,8 @@ static int __sq_remap(struct sq_mapping *map, pgprot_t prot) struct vm_struct *vma; vma = __get_vm_area_caller(map->size, VM_IOREMAP, map->sq_addr, - SQ_ADDRMAX, __builtin_return_address(0)); + SQ_ADDRMAX, VM_DEFAULT_GUARD_PAGES, + __builtin_return_address(0)); if (!vma) return -ENOMEM; diff --git a/arch/sh/mm/pmb.c b/arch/sh/mm/pmb.c index 482eec50f404..f593fa166d0b 100644 --- a/arch/sh/mm/pmb.c +++ b/arch/sh/mm/pmb.c @@ -444,7 +444,7 @@ void __iomem *pmb_remap_caller(phys_addr_t phys, unsigned long size, * 0xb000...0xc000 range. */ area = __get_vm_area_caller(aligned, VM_IOREMAP, 0xb0000000, - P3SEG, caller); + P3SEG, VM_DEFAULT_GUARD_PAGES, caller); if (!area) return NULL; diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index 323adc93f2dc..1ab9e5cdba5b 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -524,8 +524,8 @@ void __init hyperv_init(void) hv_hypercall_pg = __vmalloc_node_range(PAGE_SIZE, 1, MODULES_VADDR, MODULES_END, GFP_KERNEL, PAGE_KERNEL_ROX, - VM_FLUSH_RESET_PERMS, NUMA_NO_NODE, - __builtin_return_address(0)); + VM_FLUSH_RESET_PERMS, VM_DEFAULT_GUARD_PAGES, + NUMA_NO_NODE, __builtin_return_address(0)); if (hv_hypercall_pg == NULL) goto clean_guest_os_id; diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index 3b02c0c6b371..f5a51fb4518a 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -25,7 +25,6 @@ struct iov_iter; /* in uio.h */ #define VM_USERMAP 0x00000008 /* suitable for remap_vmalloc_range */ #define VM_DMA_COHERENT 0x00000010 /* dma_alloc_coherent */ #define VM_UNINITIALIZED 0x00000020 /* vm_struct is not fully initialized */ -#define VM_NO_GUARD 0x00000040 /* ***DANGEROUS*** don't add guard page */ #define VM_KASAN 0x00000080 /* has allocated kasan shadow memory */ #define VM_FLUSH_RESET_PERMS 0x00000100 /* reset direct map and flush TLB on unmap, can't be freed in atomic context */ #define VM_MAP_PUT_PAGES 0x00000200 /* put pages and free array in vfree */ @@ -41,6 +40,13 @@ struct iov_iter; /* in uio.h */ /* bits [20..32] reserved for arch specific ioremap internals */ +/* + * Default number of unmapped guard pages appended to a vm area to catch + * out-of-bounds accesses. Callers that need a different count (typically + * zero) pass it explicitly to __get_vm_area_node() and friends. + */ +#define VM_DEFAULT_GUARD_PAGES 1 + /* * Maximum alignment for ioremap() regions. * Can be overridden by arch-specific value. @@ -63,6 +69,7 @@ struct vm_struct { unsigned int page_order; #endif unsigned int nr_pages; + unsigned int nr_guard_pages; phys_addr_t phys_addr; const void *caller; unsigned long requested_size; @@ -86,6 +93,7 @@ struct vmap_area { struct vm_struct *vm; /* in "busy" tree */ }; unsigned long flags; /* mark type of vm_map_ram area */ + unsigned int nr_guard_pages; /* unmapped pages trailing this area */ }; /* archs that select HAVE_ARCH_HUGE_VMAP should override one or more of these */ @@ -173,7 +181,8 @@ extern void *__vmalloc_noprof(unsigned long size, gfp_t gfp_mask) __alloc_size(1 extern void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align, unsigned long start, unsigned long end, gfp_t gfp_mask, - pgprot_t prot, unsigned long vm_flags, int node, + pgprot_t prot, unsigned long vm_flags, + unsigned long nr_guard_pages, int node, const void *caller) __alloc_size(1); #define __vmalloc_node_range(...) alloc_hooks(__vmalloc_node_range_noprof(__VA_ARGS__)) @@ -235,12 +244,7 @@ int vmap_pages_range(unsigned long addr, unsigned long end, pgprot_t prot, static inline size_t get_vm_area_size(const struct vm_struct *area) { - if (!(area->flags & VM_NO_GUARD)) - /* return actual size without guard page */ - return area->size - PAGE_SIZE; - else - return area->size; - + return area->size - area->nr_guard_pages * PAGE_SIZE; } extern struct vm_struct *get_vm_area(unsigned long size, unsigned long flags); @@ -249,6 +253,7 @@ extern struct vm_struct *get_vm_area_caller(unsigned long size, extern struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags, unsigned long start, unsigned long end, + unsigned long nr_guard_pages, const void *caller); void free_vm_area(struct vm_struct *area); extern struct vm_struct *remove_vm_area(const void *addr); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index a3c0214ca934..b844d24fba58 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -403,7 +403,8 @@ static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable) return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL, - flags, numa_node, __builtin_return_address(0)); + flags, VM_DEFAULT_GUARD_PAGES, numa_node, + __builtin_return_address(0)); } void *bpf_map_area_alloc(u64 size, int numa_node) diff --git a/kernel/scs.c b/kernel/scs.c index 772488afd5b9..2647c79488e6 100644 --- a/kernel/scs.c +++ b/kernel/scs.c @@ -44,7 +44,8 @@ static void *__scs_alloc(int node) } s = __vmalloc_node_range(SCS_SIZE, 1, VMALLOC_START, VMALLOC_END, - GFP_SCS, PAGE_KERNEL, 0, node, + GFP_SCS, PAGE_KERNEL, 0, + VM_DEFAULT_GUARD_PAGES, node, __builtin_return_address(0)); out: diff --git a/mm/execmem.c b/mm/execmem.c index 084a207e4278..62d6800f3e92 100644 --- a/mm/execmem.c +++ b/mm/execmem.c @@ -39,13 +39,14 @@ static void *execmem_vmalloc(struct execmem_range *range, size_t size, vm_flags |= VM_DEFER_KMEMLEAK; p = __vmalloc_node_range(size, align, start, end, gfp_flags, - pgprot, vm_flags, NUMA_NO_NODE, - __builtin_return_address(0)); + pgprot, vm_flags, VM_DEFAULT_GUARD_PAGES, + NUMA_NO_NODE, __builtin_return_address(0)); if (!p && range->fallback_start) { start = range->fallback_start; end = range->fallback_end; p = __vmalloc_node_range(size, align, start, end, gfp_flags, - pgprot, vm_flags, NUMA_NO_NODE, + pgprot, vm_flags, + VM_DEFAULT_GUARD_PAGES, NUMA_NO_NODE, __builtin_return_address(0)); } @@ -68,12 +69,14 @@ struct vm_struct *execmem_vmap(size_t size) struct vm_struct *area; area = __get_vm_area_node(size, range->alignment, PAGE_SHIFT, VM_ALLOC, - range->start, range->end, NUMA_NO_NODE, + range->start, range->end, + VM_DEFAULT_GUARD_PAGES, NUMA_NO_NODE, GFP_KERNEL, __builtin_return_address(0)); if (!area && range->fallback_start) area = __get_vm_area_node(size, range->alignment, PAGE_SHIFT, VM_ALLOC, range->fallback_start, range->fallback_end, - NUMA_NO_NODE, GFP_KERNEL, __builtin_return_address(0)); + VM_DEFAULT_GUARD_PAGES, NUMA_NO_NODE, + GFP_KERNEL, __builtin_return_address(0)); return area; } diff --git a/mm/internal.h b/mm/internal.h index 5a2ddcf68e0b..d60e45d3b6d9 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1588,8 +1588,9 @@ int migrate_device_coherent_folio(struct folio *folio); struct vm_struct *__get_vm_area_node(unsigned long size, unsigned long align, unsigned long shift, unsigned long vm_flags, unsigned long start, - unsigned long end, int node, gfp_t gfp_mask, - const void *caller); + unsigned long end, + unsigned long nr_guard_pages, int node, + gfp_t gfp_mask, const void *caller); /* * mm/gup.c diff --git a/mm/ioremap.c b/mm/ioremap.c index c36dd9f62fd5..61734b8ec128 100644 --- a/mm/ioremap.c +++ b/mm/ioremap.c @@ -34,7 +34,8 @@ void __iomem *generic_ioremap_prot(phys_addr_t phys_addr, size_t size, size = PAGE_ALIGN(size + offset); area = __get_vm_area_caller(size, VM_IOREMAP, IOREMAP_START, - IOREMAP_END, __builtin_return_address(0)); + IOREMAP_END, VM_DEFAULT_GUARD_PAGES, + __builtin_return_address(0)); if (!area) return NULL; vaddr = (unsigned long)area->addr; diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c index d286e0a04543..5532a735534d 100644 --- a/mm/kasan/shadow.c +++ b/mm/kasan/shadow.c @@ -241,7 +241,7 @@ static int __meminit kasan_mem_notifier(struct notifier_block *nb, ret = __vmalloc_node_range(shadow_size, PAGE_SIZE, shadow_start, shadow_end, GFP_KERNEL, - PAGE_KERNEL, VM_NO_GUARD, + PAGE_KERNEL, 0, 0, pfn_to_nid(mem_data->start_pfn), __builtin_return_address(0)); if (!ret) @@ -676,7 +676,7 @@ int kasan_alloc_module_shadow(void *addr, size_t size, gfp_t gfp_mask) ret = __vmalloc_node_range(shadow_size, 1, shadow_start, shadow_start + shadow_size, GFP_KERNEL, - PAGE_KERNEL, VM_NO_GUARD, NUMA_NO_NODE, + PAGE_KERNEL, 0, 0, NUMA_NO_NODE, __builtin_return_address(0)); if (ret) { diff --git a/mm/vmalloc.c b/mm/vmalloc.c index c31a8615a832..256b1de55080 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -729,7 +729,7 @@ static int check_sparse_vm_area(struct vm_struct *area, unsigned long start, might_sleep(); if (WARN_ON_ONCE(area->flags & VM_FLUSH_RESET_PERMS)) return -EINVAL; - if (WARN_ON_ONCE(area->flags & VM_NO_GUARD)) + if (WARN_ON_ONCE(!area->nr_guard_pages)) return -EINVAL; if (WARN_ON_ONCE(!(area->flags & VM_SPARSE))) return -EINVAL; @@ -2103,11 +2103,13 @@ static struct vmap_area *alloc_vmap_area(unsigned long size, va->va_end = addr + size; va->vm = NULL; va->flags = (va_flags | vn_id); + va->nr_guard_pages = 0; if (vm) { vm->addr = (void *)va->va_start; vm->size = va_size(va); va->vm = vm; + va->nr_guard_pages = vm->nr_guard_pages; } vn = addr_to_node(va->va_start); @@ -3196,7 +3198,8 @@ void clear_vm_uninitialized_flag(struct vm_struct *vm) struct vm_struct *__get_vm_area_node(unsigned long size, unsigned long align, unsigned long shift, unsigned long flags, - unsigned long start, unsigned long end, int node, + unsigned long start, unsigned long end, + unsigned long nr_guard_pages, int node, gfp_t gfp_mask, const void *caller) { struct vmap_area *va; @@ -3216,12 +3219,12 @@ struct vm_struct *__get_vm_area_node(unsigned long size, if (unlikely(!area)) return NULL; - if (!(flags & VM_NO_GUARD)) - size += PAGE_SIZE; + size += nr_guard_pages * PAGE_SIZE; area->flags = flags; area->caller = caller; area->requested_size = requested_size; + area->nr_guard_pages = nr_guard_pages; va = alloc_vmap_area(size, align, start, end, node, gfp_mask, 0, area); if (IS_ERR(va)) { @@ -3246,10 +3249,12 @@ struct vm_struct *__get_vm_area_node(unsigned long size, struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags, unsigned long start, unsigned long end, + unsigned long nr_guard_pages, const void *caller) { return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, start, end, - NUMA_NO_NODE, GFP_KERNEL, caller); + nr_guard_pages, NUMA_NO_NODE, GFP_KERNEL, + caller); } /** @@ -3267,8 +3272,8 @@ struct vm_struct *get_vm_area(unsigned long size, unsigned long flags) { return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, VMALLOC_START, VMALLOC_END, - NUMA_NO_NODE, GFP_KERNEL, - __builtin_return_address(0)); + VM_DEFAULT_GUARD_PAGES, NUMA_NO_NODE, + GFP_KERNEL, __builtin_return_address(0)); } struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, @@ -3276,7 +3281,8 @@ struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, { return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, VMALLOC_START, VMALLOC_END, - NUMA_NO_NODE, GFP_KERNEL, caller); + VM_DEFAULT_GUARD_PAGES, NUMA_NO_NODE, + GFP_KERNEL, caller); } /** @@ -3532,13 +3538,6 @@ void *vmap(struct page **pages, unsigned int count, if (WARN_ON_ONCE(flags & VM_FLUSH_RESET_PERMS)) return NULL; - /* - * Your top guard is someone else's bottom guard. Not having a top - * guard compromises someone else's mappings too. - */ - if (WARN_ON_ONCE(flags & VM_NO_GUARD)) - flags &= ~VM_NO_GUARD; - if (count > totalram_pages()) return NULL; @@ -3959,7 +3958,8 @@ static gfp_t vmalloc_fix_flags(gfp_t flags) * @end: vm area range end * @gfp_mask: flags for the page level allocator * @prot: protection mask for the allocated pages - * @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD) + * @vm_flags: additional vm area flags + * @nr_guard_pages: number of unmapped guard pages to append (0 if none) * @node: node to use for allocation or NUMA_NO_NODE * @caller: caller's return address * @@ -3985,7 +3985,8 @@ static gfp_t vmalloc_fix_flags(gfp_t flags) */ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align, unsigned long start, unsigned long end, gfp_t gfp_mask, - pgprot_t prot, unsigned long vm_flags, int node, + pgprot_t prot, unsigned long vm_flags, + unsigned long nr_guard_pages, int node, const void *caller) { struct vm_struct *area; @@ -4022,8 +4023,8 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align, again: area = __get_vm_area_node(size, align, shift, VM_ALLOC | - VM_UNINITIALIZED | vm_flags, start, end, node, - gfp_mask, caller); + VM_UNINITIALIZED | vm_flags, start, end, + nr_guard_pages, node, gfp_mask, caller); if (!area) { bool nofail = gfp_mask & __GFP_NOFAIL; warn_alloc(gfp_mask, NULL, @@ -4122,7 +4123,8 @@ void *__vmalloc_node_noprof(unsigned long size, unsigned long align, gfp_t gfp_mask, int node, const void *caller) { return __vmalloc_node_range_noprof(size, align, VMALLOC_START, VMALLOC_END, - gfp_mask, PAGE_KERNEL, 0, node, caller); + gfp_mask, PAGE_KERNEL, 0, + VM_DEFAULT_GUARD_PAGES, node, caller); } /* * This is only for performance analysis of vmalloc and stress purpose. @@ -4180,7 +4182,8 @@ void *vmalloc_huge_node_noprof(unsigned long size, gfp_t gfp_mask, int node) gfp_mask = vmalloc_fix_flags(gfp_mask); return __vmalloc_node_range_noprof(size, 1, VMALLOC_START, VMALLOC_END, gfp_mask, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP, - node, __builtin_return_address(0)); + VM_DEFAULT_GUARD_PAGES, node, + __builtin_return_address(0)); } EXPORT_SYMBOL_GPL(vmalloc_huge_node_noprof); @@ -4217,8 +4220,8 @@ void *vmalloc_user_noprof(unsigned long size) { return __vmalloc_node_range_noprof(size, SHMLBA, VMALLOC_START, VMALLOC_END, GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL, - VM_USERMAP, NUMA_NO_NODE, - __builtin_return_address(0)); + VM_USERMAP, VM_DEFAULT_GUARD_PAGES, + NUMA_NO_NODE, __builtin_return_address(0)); } EXPORT_SYMBOL(vmalloc_user_noprof); @@ -4410,8 +4413,8 @@ void *vmalloc_32_user_noprof(unsigned long size) { return __vmalloc_node_range_noprof(size, SHMLBA, VMALLOC_START, VMALLOC_END, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL, - VM_USERMAP, NUMA_NO_NODE, - __builtin_return_address(0)); + VM_USERMAP, VM_DEFAULT_GUARD_PAGES, + NUMA_NO_NODE, __builtin_return_address(0)); } EXPORT_SYMBOL(vmalloc_32_user_noprof); @@ -5463,6 +5466,7 @@ void __init vmalloc_init(void) va->va_start = (unsigned long)tmp->addr; va->va_end = va->va_start + tmp->size; va->vm = tmp; + va->nr_guard_pages = tmp->nr_guard_pages; vn = addr_to_node(va->va_start); insert_vmap_area(va, &vn->busy.root, &vn->busy.head);