From: Andi Kleen <ak@suse.de>
To: Jeremy Fitzhardinge <jeremy@goop.org>, Andi Kleen <ak@suse.de>,
patches@x86-64.org, linux-kernel@vger.kernel.org
Subject: [PATCH] [20/26] x86: Allow percpu variables to be page-aligned
Date: Mon, 30 Apr 2007 01:46:30 +0200 (CEST) [thread overview]
Message-ID: <20070429234630.0DF42151C5@wotan.suse.de> (raw)
In-Reply-To: <20070430146.588463000@suse.de>
From: Jeremy Fitzhardinge <jeremy@goop.org>
Let's allow page-alignment in general for per-cpu data (wanted by Xen, and
Ingo suggested KVM as well).
Because larger alignments can use more room, we increase the max per-cpu
memory to 64k rather than 32k: it's getting a little tight.
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
Signed-off-by: Jeremy Fitzhardinge <jeremy@xensource.com>
Signed-off-by: Andi Kleen <ak@suse.de>
Acked-by: Ingo Molnar <mingo@elte.hu>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
---
arch/alpha/kernel/vmlinux.lds.S | 2 +-
arch/arm/kernel/vmlinux.lds.S | 2 +-
arch/cris/arch-v32/vmlinux.lds.S | 1 +
arch/frv/kernel/vmlinux.lds.S | 1 +
arch/i386/kernel/vmlinux.lds.S | 2 +-
arch/m32r/kernel/vmlinux.lds.S | 2 +-
arch/mips/kernel/vmlinux.lds.S | 2 +-
arch/parisc/kernel/vmlinux.lds.S | 2 +-
arch/powerpc/kernel/setup_64.c | 4 ++--
arch/powerpc/kernel/vmlinux.lds.S | 6 +-----
arch/ppc/kernel/vmlinux.lds.S | 2 +-
arch/s390/kernel/vmlinux.lds.S | 2 +-
arch/sh/kernel/vmlinux.lds.S | 2 +-
arch/sh64/kernel/vmlinux.lds.S | 2 +-
arch/sparc/kernel/vmlinux.lds.S | 2 +-
arch/sparc64/kernel/smp.c | 6 +++---
arch/x86_64/kernel/setup64.c | 4 ++--
arch/x86_64/kernel/vmlinux.lds.S | 2 +-
arch/xtensa/kernel/vmlinux.lds.S | 2 +-
init/main.c | 8 ++------
kernel/module.c | 8 ++++----
21 files changed, 29 insertions(+), 35 deletions(-)
===================================================================
Index: linux/arch/alpha/kernel/vmlinux.lds.S
===================================================================
--- linux.orig/arch/alpha/kernel/vmlinux.lds.S
+++ linux/arch/alpha/kernel/vmlinux.lds.S
@@ -69,7 +69,7 @@ SECTIONS
. = ALIGN(8);
SECURITY_INIT
- . = ALIGN(64);
+ . = ALIGN(8192);
__per_cpu_start = .;
.data.percpu : { *(.data.percpu) }
__per_cpu_end = .;
Index: linux/arch/arm/kernel/vmlinux.lds.S
===================================================================
--- linux.orig/arch/arm/kernel/vmlinux.lds.S
+++ linux/arch/arm/kernel/vmlinux.lds.S
@@ -59,7 +59,7 @@ SECTIONS
usr/built-in.o(.init.ramfs)
__initramfs_end = .;
#endif
- . = ALIGN(64);
+ . = ALIGN(4096);
__per_cpu_start = .;
*(.data.percpu)
__per_cpu_end = .;
Index: linux/arch/cris/arch-v32/vmlinux.lds.S
===================================================================
--- linux.orig/arch/cris/arch-v32/vmlinux.lds.S
+++ linux/arch/cris/arch-v32/vmlinux.lds.S
@@ -91,6 +91,7 @@ SECTIONS
}
SECURITY_INIT
+ . = ALIGN (8192);
__per_cpu_start = .;
.data.percpu : { *(.data.percpu) }
__per_cpu_end = .;
Index: linux/arch/frv/kernel/vmlinux.lds.S
===================================================================
--- linux.orig/arch/frv/kernel/vmlinux.lds.S
+++ linux/arch/frv/kernel/vmlinux.lds.S
@@ -57,6 +57,7 @@ SECTIONS
__alt_instructions_end = .;
.altinstr_replacement : { *(.altinstr_replacement) }
+ . = ALIGN(4096);
__per_cpu_start = .;
.data.percpu : { *(.data.percpu) }
__per_cpu_end = .;
Index: linux/arch/i386/kernel/vmlinux.lds.S
===================================================================
--- linux.orig/arch/i386/kernel/vmlinux.lds.S
+++ linux/arch/i386/kernel/vmlinux.lds.S
@@ -194,7 +194,7 @@ SECTIONS
__initramfs_end = .;
}
#endif
- . = ALIGN(L1_CACHE_BYTES);
+ . = ALIGN(4096);
.data.percpu : AT(ADDR(.data.percpu) - LOAD_OFFSET) {
__per_cpu_start = .;
*(.data.percpu)
Index: linux/arch/m32r/kernel/vmlinux.lds.S
===================================================================
--- linux.orig/arch/m32r/kernel/vmlinux.lds.S
+++ linux/arch/m32r/kernel/vmlinux.lds.S
@@ -110,7 +110,7 @@ SECTIONS
__initramfs_end = .;
#endif
- . = ALIGN(32);
+ . = ALIGN(4096);
__per_cpu_start = .;
.data.percpu : { *(.data.percpu) }
__per_cpu_end = .;
Index: linux/arch/mips/kernel/vmlinux.lds.S
===================================================================
--- linux.orig/arch/mips/kernel/vmlinux.lds.S
+++ linux/arch/mips/kernel/vmlinux.lds.S
@@ -119,7 +119,7 @@ SECTIONS
.init.ramfs : { *(.init.ramfs) }
__initramfs_end = .;
#endif
- . = ALIGN(32);
+ . = ALIGN(_PAGE_SIZE);
__per_cpu_start = .;
.data.percpu : { *(.data.percpu) }
__per_cpu_end = .;
Index: linux/arch/parisc/kernel/vmlinux.lds.S
===================================================================
--- linux.orig/arch/parisc/kernel/vmlinux.lds.S
+++ linux/arch/parisc/kernel/vmlinux.lds.S
@@ -181,7 +181,7 @@ SECTIONS
.init.ramfs : { *(.init.ramfs) }
__initramfs_end = .;
#endif
- . = ALIGN(32);
+ . = ALIGN(ASM_PAGE_SIZE);
__per_cpu_start = .;
.data.percpu : { *(.data.percpu) }
__per_cpu_end = .;
Index: linux/arch/powerpc/kernel/setup_64.c
===================================================================
--- linux.orig/arch/powerpc/kernel/setup_64.c
+++ linux/arch/powerpc/kernel/setup_64.c
@@ -583,14 +583,14 @@ void __init setup_per_cpu_areas(void)
char *ptr;
/* Copy section for each CPU (we discard the original) */
- size = ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES);
+ size = ALIGN(__per_cpu_end - __per_cpu_start, PAGE_SIZE);
#ifdef CONFIG_MODULES
if (size < PERCPU_ENOUGH_ROOM)
size = PERCPU_ENOUGH_ROOM;
#endif
for_each_possible_cpu(i) {
- ptr = alloc_bootmem_node(NODE_DATA(cpu_to_node(i)), size);
+ ptr = alloc_bootmem_pages_node(NODE_DATA(cpu_to_node(i)), size);
if (!ptr)
panic("Cannot allocate cpu data for CPU %d\n", i);
Index: linux/arch/powerpc/kernel/vmlinux.lds.S
===================================================================
--- linux.orig/arch/powerpc/kernel/vmlinux.lds.S
+++ linux/arch/powerpc/kernel/vmlinux.lds.S
@@ -139,11 +139,7 @@ SECTIONS
__initramfs_end = .;
}
#endif
-#ifdef CONFIG_PPC32
- . = ALIGN(32);
-#else
- . = ALIGN(128);
-#endif
+ . = ALIGN(PAGE_SIZE);
.data.percpu : {
__per_cpu_start = .;
*(.data.percpu)
Index: linux/arch/ppc/kernel/vmlinux.lds.S
===================================================================
--- linux.orig/arch/ppc/kernel/vmlinux.lds.S
+++ linux/arch/ppc/kernel/vmlinux.lds.S
@@ -130,7 +130,7 @@ SECTIONS
__ftr_fixup : { *(__ftr_fixup) }
__stop___ftr_fixup = .;
- . = ALIGN(32);
+ . = ALIGN(4096);
__per_cpu_start = .;
.data.percpu : { *(.data.percpu) }
__per_cpu_end = .;
Index: linux/arch/s390/kernel/vmlinux.lds.S
===================================================================
--- linux.orig/arch/s390/kernel/vmlinux.lds.S
+++ linux/arch/s390/kernel/vmlinux.lds.S
@@ -107,7 +107,7 @@ SECTIONS
. = ALIGN(2);
__initramfs_end = .;
#endif
- . = ALIGN(256);
+ . = ALIGN(4096);
__per_cpu_start = .;
.data.percpu : { *(.data.percpu) }
__per_cpu_end = .;
Index: linux/arch/sh/kernel/vmlinux.lds.S
===================================================================
--- linux.orig/arch/sh/kernel/vmlinux.lds.S
+++ linux/arch/sh/kernel/vmlinux.lds.S
@@ -54,7 +54,7 @@ SECTIONS
. = ALIGN(PAGE_SIZE);
.data.page_aligned : { *(.data.page_aligned) }
- . = ALIGN(L1_CACHE_BYTES);
+ . = ALIGN(PAGE_SIZE);
__per_cpu_start = .;
.data.percpu : { *(.data.percpu) }
__per_cpu_end = .;
Index: linux/arch/sh64/kernel/vmlinux.lds.S
===================================================================
--- linux.orig/arch/sh64/kernel/vmlinux.lds.S
+++ linux/arch/sh64/kernel/vmlinux.lds.S
@@ -85,7 +85,7 @@ SECTIONS
. = ALIGN(PAGE_SIZE);
.data.page_aligned : C_PHYS(.data.page_aligned) { *(.data.page_aligned) }
- . = ALIGN(L1_CACHE_BYTES);
+ . = ALIGN(PAGE_SIZE);
__per_cpu_start = .;
.data.percpu : C_PHYS(.data.percpu) { *(.data.percpu) }
__per_cpu_end = . ;
Index: linux/arch/sparc/kernel/vmlinux.lds.S
===================================================================
--- linux.orig/arch/sparc/kernel/vmlinux.lds.S
+++ linux/arch/sparc/kernel/vmlinux.lds.S
@@ -65,7 +65,7 @@ SECTIONS
__initramfs_end = .;
#endif
- . = ALIGN(32);
+ . = ALIGN(4096);
__per_cpu_start = .;
.data.percpu : { *(.data.percpu) }
__per_cpu_end = .;
Index: linux/arch/sparc64/kernel/smp.c
===================================================================
--- linux.orig/arch/sparc64/kernel/smp.c
+++ linux/arch/sparc64/kernel/smp.c
@@ -1343,11 +1343,11 @@ void __init setup_per_cpu_areas(void)
/* Copy section for each CPU (we discard the original) */
goal = PERCPU_ENOUGH_ROOM;
- __per_cpu_shift = 0;
- for (size = 1UL; size < goal; size <<= 1UL)
+ __per_cpu_shift = PAGE_SHIFT;
+ for (size = PAGE_SIZE; size < goal; size <<= 1UL)
__per_cpu_shift++;
- ptr = alloc_bootmem(size * NR_CPUS);
+ ptr = alloc_bootmem_pages(size * NR_CPUS);
__per_cpu_base = ptr - __per_cpu_start;
Index: linux/arch/x86_64/kernel/setup64.c
===================================================================
--- linux.orig/arch/x86_64/kernel/setup64.c
+++ linux/arch/x86_64/kernel/setup64.c
@@ -103,9 +103,9 @@ void __init setup_per_cpu_areas(void)
if (!NODE_DATA(cpu_to_node(i))) {
printk("cpu with no node %d, num_online_nodes %d\n",
i, num_online_nodes());
- ptr = alloc_bootmem(size);
+ ptr = alloc_bootmem_pages(size);
} else {
- ptr = alloc_bootmem_node(NODE_DATA(cpu_to_node(i)), size);
+ ptr = alloc_bootmem_pages_node(NODE_DATA(cpu_to_node(i)), size);
}
if (!ptr)
panic("Cannot allocate cpu data for CPU %d\n", i);
Index: linux/arch/x86_64/kernel/vmlinux.lds.S
===================================================================
--- linux.orig/arch/x86_64/kernel/vmlinux.lds.S
+++ linux/arch/x86_64/kernel/vmlinux.lds.S
@@ -195,7 +195,7 @@ SECTIONS
__initramfs_end = .;
#endif
- . = ALIGN(CONFIG_X86_L1_CACHE_BYTES);
+ . = ALIGN(4096);
__per_cpu_start = .;
.data.percpu : AT(ADDR(.data.percpu) - LOAD_OFFSET) { *(.data.percpu) }
__per_cpu_end = .;
Index: linux/arch/xtensa/kernel/vmlinux.lds.S
===================================================================
--- linux.orig/arch/xtensa/kernel/vmlinux.lds.S
+++ linux/arch/xtensa/kernel/vmlinux.lds.S
@@ -198,7 +198,7 @@ SECTIONS
__ftr_fixup : { *(__ftr_fixup) }
__stop___ftr_fixup = .;
- . = ALIGN(32);
+ . = ALIGN(4096);
__per_cpu_start = .;
.data.percpu : { *(.data.percpu) }
__per_cpu_end = .;
Index: linux/init/main.c
===================================================================
--- linux.orig/init/main.c
+++ linux/init/main.c
@@ -369,12 +369,8 @@ static void __init setup_per_cpu_areas(v
unsigned long nr_possible_cpus = num_possible_cpus();
/* Copy section for each CPU (we discard the original) */
- size = ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES);
-#ifdef CONFIG_MODULES
- if (size < PERCPU_ENOUGH_ROOM)
- size = PERCPU_ENOUGH_ROOM;
-#endif
- ptr = alloc_bootmem(size * nr_possible_cpus);
+ size = ALIGN(PERCPU_ENOUGH_ROOM, PAGE_SIZE);
+ ptr = alloc_bootmem_pages(size * nr_possible_cpus);
for_each_possible_cpu(i) {
__per_cpu_offset[i] = ptr - __per_cpu_start;
Index: linux/kernel/module.c
===================================================================
--- linux.orig/kernel/module.c
+++ linux/kernel/module.c
@@ -346,10 +346,10 @@ static void *percpu_modalloc(unsigned lo
unsigned int i;
void *ptr;
- if (align > SMP_CACHE_BYTES) {
- printk(KERN_WARNING "%s: per-cpu alignment %li > %i\n",
- name, align, SMP_CACHE_BYTES);
- align = SMP_CACHE_BYTES;
+ if (align > PAGE_SIZE) {
+ printk(KERN_WARNING "%s: per-cpu alignment %li > %li\n",
+ name, align, PAGE_SIZE);
+ align = PAGE_SIZE;
}
ptr = __per_cpu_start;
next prev parent reply other threads:[~2007-04-29 23:51 UTC|newest]
Thread overview: 27+ messages / expand[flat|nested] mbox.gz Atom feed top
2007-04-29 23:46 [PATCH] [0/26] x86 candidate patches for review IV: more misc patches Andi Kleen
2007-04-29 23:46 ` [PATCH] [1/26] i386: Add smp_ops interface Andi Kleen
2007-04-29 23:46 ` [PATCH] [2/26] i386: Add machine_ops interface to abstract halting and rebooting Andi Kleen
2007-04-29 23:46 ` [PATCH] [4/26] i386: Remove unneeded externs in nmi.c Andi Kleen
2007-04-29 23:46 ` [PATCH] [6/26] x86: fix amd64-agp aperture validation Andi Kleen
2007-04-29 23:46 ` [PATCH] [7/26] x86_64: Use X86_EFLAGS_IF in x86-64/irqflags.h Andi Kleen
2007-04-29 23:46 ` [PATCH] [8/26] x86_64: fix arithmetic in comment Andi Kleen
2007-04-29 23:46 ` [PATCH] [9/26] x86_64: Fix vmalloc_32 to really allocate <4GB on 64bit platforms Andi Kleen
2007-04-30 5:01 ` Borislav Petkov
2007-04-30 5:49 ` Borislav Petkov
2007-04-30 8:59 ` Andi Kleen
2007-04-29 23:46 ` [PATCH] [10/26] i386: Clean up asm-i386/bugs.h Andi Kleen
2007-04-29 23:46 ` [PATCH] [11/26] i386: clean up identify_cpu Andi Kleen
2007-04-29 23:46 ` [PATCH] [14/26] x86_64: Clean up asm-x86_64/bugs.h Andi Kleen
2007-04-29 23:46 ` [PATCH] [15/26] x86: Don't use MWAIT on AMD Family 10 Andi Kleen
2007-04-29 23:46 ` [PATCH] [16/26] i386: Enable machine check for " Andi Kleen
2007-04-29 23:46 ` [PATCH] [17/26] i386: Use menuconfig objects - APM Andi Kleen
2007-04-29 23:46 ` [PATCH] [18/26] i386: Update smp_call_function* comments Andi Kleen
2007-04-29 23:46 ` [PATCH] [19/26] i386: Enable bank 0 on non K7 Athlon Andi Kleen
2007-04-30 0:09 ` Dave Jones
2007-04-29 23:46 ` Andi Kleen [this message]
2007-04-29 23:46 ` [PATCH] [21/26] x86: Clean up x86 control register and MSR macros (corrected) Andi Kleen
2007-04-29 23:46 ` [PATCH] [22/26] i386: Remove smp_alt_instructions Andi Kleen
2007-04-29 23:46 ` [PATCH] [23/26] i386: Allow boot-time disable of SMP altinstructions Andi Kleen
2007-04-29 23:46 ` [PATCH] [24/26] x86_64: Fix x86_64 compilation with DEBUG_SIG on Andi Kleen
2007-04-29 23:46 ` [PATCH] [25/26] x86_64: x86-64 system crashes when no memory populating Node 0 Andi Kleen
2007-04-29 23:46 ` [PATCH] [26/26] i386: i386 separate hardware-defined TSS from Linux additions Andi Kleen
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20070429234630.0DF42151C5@wotan.suse.de \
--to=ak@suse.de \
--cc=jeremy@goop.org \
--cc=linux-kernel@vger.kernel.org \
--cc=patches@x86-64.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
Powered by JetHome