[PATCH] 1/6 Share common physnode_map code between NUMA-Q and Summit

[PATCH] 1/6 Share common physnode_map code between NUMA-Q and Summit

[Martin J. Bligh]

>From Andy Whitcroft

Share a common physnode_map structure between NUMA-Q and Summit.

diff -urpN -X /home/fletch/.diff.exclude 000-virgin/arch/i386/kernel/i386_ksyms.c 010-common_physmap/arch/i386/kernel/i386_ksyms.c
--- 000-virgin/arch/i386/kernel/i386_ksyms.c	Wed Mar  5 07:36:57 2003
+++ 010-common_physmap/arch/i386/kernel/i386_ksyms.c	Wed Mar  5 08:44:17 2003
@@ -68,6 +68,7 @@ EXPORT_SYMBOL(EISA_bus);
 EXPORT_SYMBOL(MCA_bus);
 #ifdef CONFIG_DISCONTIGMEM
 EXPORT_SYMBOL(node_data);
+EXPORT_SYMBOL(physnode_map);
 #endif
 #ifdef CONFIG_X86_NUMAQ
 EXPORT_SYMBOL(xquad_portio);
diff -urpN -X /home/fletch/.diff.exclude 000-virgin/arch/i386/kernel/numaq.c 010-common_physmap/arch/i386/kernel/numaq.c
--- 000-virgin/arch/i386/kernel/numaq.c	Wed Mar  5 07:36:57 2003
+++ 010-common_physmap/arch/i386/kernel/numaq.c	Wed Mar  5 08:44:17 2003
@@ -31,8 +31,7 @@
 #include <asm/numaq.h>
 
 /* These are needed before the pgdat's are created */
-unsigned long node_start_pfn[MAX_NUMNODES];
-unsigned long node_end_pfn[MAX_NUMNODES];
+extern long node_start_pfn[], node_end_pfn[];
 
 #define	MB_TO_PAGES(addr) ((addr) << (20 - PAGE_SHIFT))
 
@@ -65,25 +64,7 @@ static void __init smp_dump_qct(void)
 	}
 }
 
-/*
- * -----------------------------------------
- *
- * functions related to physnode_map
- *
- * -----------------------------------------
- */
-/*
- * physnode_map keeps track of the physical memory layout of the
- * numaq nodes on a 256Mb break (each element of the array will
- * represent 256Mb of memory and will be marked by the node id.  so,
- * if the first gig is on node 0, and the second gig is on node 1
- * physnode_map will contain:
- * physnode_map[0-3] = 0;
- * physnode_map[4-7] = 1;
- * physnode_map[8- ] = -1;
- */
-int physnode_map[MAX_ELEMENTS] = { [0 ... (MAX_ELEMENTS - 1)] = -1};
-EXPORT_SYMBOL(physnode_map);
+extern int physnode_map[];
 
 /*
  * for each node mark the regions
diff -urpN -X /home/fletch/.diff.exclude 000-virgin/arch/i386/kernel/srat.c 010-common_physmap/arch/i386/kernel/srat.c
--- 000-virgin/arch/i386/kernel/srat.c	Wed Mar  5 07:36:57 2003
+++ 010-common_physmap/arch/i386/kernel/srat.c	Wed Mar  5 08:44:17 2003
@@ -57,8 +57,7 @@ static int num_memory_chunks;		/* total 
 static int zholes_size_init;
 static unsigned long zholes_size[MAX_NUMNODES * MAX_NR_ZONES];
 
-unsigned long node_start_pfn[MAX_NUMNODES];
-unsigned long node_end_pfn[MAX_NUMNODES];
+extern unsigned long node_start_pfn[], node_end_pfn[];
 
 extern void * boot_ioremap(unsigned long, unsigned long);
 
@@ -182,30 +181,19 @@ static __init void chunk_to_zones(unsign
 	}
 }
 
-/*
- * physnode_map keeps track of the physical memory layout of the
- * numaq nodes on a 256Mb break (each element of the array will
- * represent 256Mb of memory and will be marked by the node id.  so,
- * if the first gig is on node 0, and the second gig is on node 1
- * physnode_map will contain:
- * physnode_map[0-3] = 0;
- * physnode_map[4-7] = 1;
- * physnode_map[8- ] = -1;
- */
-int pfnnode_map[MAX_ELEMENTS] = { [0 ... (MAX_ELEMENTS - 1)] = -1};
-EXPORT_SYMBOL(pfnnode_map);
-
-static void __init initialize_pfnnode_map(void)
+static void __init initialize_physnode_map(void)
 {
-	unsigned long topofchunk, cur = 0;
 	int i;
-	
-	for (i = 0; i < num_memory_chunks; i++) {
-		cur = node_memory_chunk[i].start_pfn;
-		topofchunk = node_memory_chunk[i].end_pfn;
-		while (cur < topofchunk) {
-			pfnnode_map[PFN_TO_ELEMENT(cur)] = node_memory_chunk[i].nid;
-			cur ++;
+	unsigned long pfn;
+	struct node_memory_chunk_s *nmcp;
+
+	/* Run the list of memory chunks and fill in the phymap. */
+	nmcp = node_memory_chunk;
+	for (i = num_memory_chunks; --i >= 0; nmcp++) {
+		for (pfn = nmcp->start_pfn; pfn <= nmcp->end_pfn;
+						pfn += PAGES_PER_ELEMENT)
+		{
+			physnode_map[pfn / PAGES_PER_ELEMENT] = (int)nmcp->nid;
 		}
 	}
 }
@@ -272,7 +260,7 @@ static int __init acpi20_parse_srat(stru
 	for (i = 0; i < num_memory_chunks; i++)
 		node_memory_chunk[i].nid = pxm_to_nid_map[node_memory_chunk[i].pxm];
 
-	initialize_pfnnode_map();
+	initialize_physnode_map();
 	
 	printk("pxm bitmap: ");
 	for (i = 0; i < sizeof(pxm_bitmap); i++) {
diff -urpN -X /home/fletch/.diff.exclude 000-virgin/arch/i386/mm/discontig.c 010-common_physmap/arch/i386/mm/discontig.c
--- 000-virgin/arch/i386/mm/discontig.c	Wed Mar  5 07:36:57 2003
+++ 010-common_physmap/arch/i386/mm/discontig.c	Wed Mar  5 08:44:17 2003
@@ -36,11 +36,36 @@
 struct pglist_data *node_data[MAX_NUMNODES];
 bootmem_data_t node0_bdata;
 
+/*
+ * numa interface - we expect the numa architecture specfic code to have
+ *                  populated the following initialisation.
+ *
+ * 1) numnodes         - the total number of nodes configured in the system
+ * 2) physnode_map     - the mapping between a pfn and owning node
+ * 3) node_start_pfn   - the starting page frame number for a node
+ * 3) node_end_pfn     - the ending page fram number for a node
+ */
+
+/*
+ * physnode_map keeps track of the physical memory layout of a generic
+ * numa node on a 256Mb break (each element of the array will
+ * represent 256Mb of memory and will be marked by the node id.  so,
+ * if the first gig is on node 0, and the second gig is on node 1
+ * physnode_map will contain:
+ *
+ *     physnode_map[0-3] = 0;
+ *     physnode_map[4-7] = 1;
+ *     physnode_map[8- ] = -1;
+ */
+int physnode_map[MAX_ELEMENTS] = { [0 ... (MAX_ELEMENTS - 1)] = -1};
+
+unsigned long node_start_pfn[MAX_NUMNODES];
+unsigned long node_end_pfn[MAX_NUMNODES];
+
 extern unsigned long find_max_low_pfn(void);
 extern void find_max_pfn(void);
 extern void one_highpage_init(struct page *, int, int);
 
-extern unsigned long node_start_pfn[], node_end_pfn[];
 extern struct e820map e820;
 extern char _end;
 extern unsigned long highend_pfn, highstart_pfn;
diff -urpN -X /home/fletch/.diff.exclude 000-virgin/include/asm-i386/mmzone.h 010-common_physmap/include/asm-i386/mmzone.h
--- 000-virgin/include/asm-i386/mmzone.h	Wed Mar  5 08:23:16 2003
+++ 010-common_physmap/include/asm-i386/mmzone.h	Wed Mar  5 08:44:27 2003
@@ -10,14 +10,6 @@
 
 #ifdef CONFIG_DISCONTIGMEM
 
-#ifdef CONFIG_X86_NUMAQ
-#include <asm/numaq.h>
-#elif CONFIG_X86_SUMMIT
-#include <asm/srat.h>
-#else
-#define pfn_to_nid(pfn)		(0)
-#endif /* CONFIG_X86_NUMAQ */
-
 extern struct pglist_data *node_data[];
 
 /*
@@ -101,5 +93,38 @@ extern struct pglist_data *node_data[];
  * ( pfn_to_pgdat(pfn) && ((pfn) < node_end_pfn(pfn_to_nid(pfn))) ) 
  */ 
 #define pfn_valid(pfn)          ((pfn) < num_physpages)
+
+/*
+ * generic node memory support, the following assumptions apply:
+ *
+ * 1) memory comes in 256Mb contigious chunks which are either present or not
+ * 2) we will not have more than 64Gb in total
+ *
+ * for now assume that 64Gb is max amount of RAM for whole system
+ *    64Gb / 4096bytes/page = 16777216 pages
+ */
+#define MAX_NR_PAGES 16777216
+#define MAX_ELEMENTS 256
+#define PAGES_PER_ELEMENT (MAX_NR_PAGES/MAX_ELEMENTS)
+
+extern int physnode_map[];
+
+static inline int pfn_to_nid(unsigned long pfn)
+{
+	return(physnode_map[(pfn) / PAGES_PER_ELEMENT]);
+}
+static inline struct pglist_data *pfn_to_pgdat(unsigned long pfn)
+{
+	return(NODE_DATA(pfn_to_nid(pfn)));
+}
+
+#ifdef CONFIG_X86_NUMAQ
+#include <asm/numaq.h>
+#elif CONFIG_X86_SUMMIT
+#include <asm/srat.h>
+#else
+#define pfn_to_nid(pfn)		(0)
+#endif /* CONFIG_X86_NUMAQ */
+
 #endif /* CONFIG_DISCONTIGMEM */
 #endif /* _ASM_MMZONE_H_ */
diff -urpN -X /home/fletch/.diff.exclude 000-virgin/include/asm-i386/numaq.h 010-common_physmap/include/asm-i386/numaq.h
--- 000-virgin/include/asm-i386/numaq.h	Wed Mar  5 07:37:06 2003
+++ 010-common_physmap/include/asm-i386/numaq.h	Wed Mar  5 08:44:17 2003
@@ -28,18 +28,8 @@
 
 #ifdef CONFIG_X86_NUMAQ
 
-/*
- * for now assume that 64Gb is max amount of RAM for whole system
- *    64Gb / 4096bytes/page = 16777216 pages
- */
-#define MAX_NR_PAGES 16777216
-#define MAX_ELEMENTS 256
-#define PAGES_PER_ELEMENT (16777216/256)
-
 extern int physnode_map[];
-#define pfn_to_nid(pfn)	({ physnode_map[(pfn) / PAGES_PER_ELEMENT]; })
-#define pfn_to_pgdat(pfn) NODE_DATA(pfn_to_nid(pfn))
-#define PHYSADDR_TO_NID(pa) pfn_to_nid(pa >> PAGE_SHIFT)
+
 #define MAX_NUMNODES		8
 extern void get_memcfg_numaq(void);
 #define get_memcfg_numa() get_memcfg_numaq()
diff -urpN -X /home/fletch/.diff.exclude 000-virgin/include/asm-i386/srat.h 010-common_physmap/include/asm-i386/srat.h
--- 000-virgin/include/asm-i386/srat.h	Wed Mar  5 07:37:06 2003
+++ 010-common_physmap/include/asm-i386/srat.h	Wed Mar  5 08:44:17 2003
@@ -27,17 +27,7 @@
 #ifndef _ASM_SRAT_H_
 #define _ASM_SRAT_H_
 
-/*
- * each element in pfnnode_map represents 256 MB (2^28) of pages.
- * so, to represent 64GB we need 256 elements.
- */
-#define MAX_ELEMENTS 256
-#define PFN_TO_ELEMENT(pfn) ((pfn)>>(28 - PAGE_SHIFT))
-
-extern int pfnnode_map[];
-#define pfn_to_nid(pfn) ({ pfnnode_map[PFN_TO_ELEMENT(pfn)]; })
-#define pfn_to_pgdat(pfn) NODE_DATA(pfn_to_nid(pfn))
-#define PHYSADDR_TO_NID(pa) pfn_to_nid(pa >> PAGE_SHIFT)
+extern int physnode_map[];
 #define MAX_NUMNODES		8
 extern void get_memcfg_from_srat(void);
 extern unsigned long *get_zholes_size(int);

[PATCH] 2/6 Make CONFIG_NUMA work on non-numa machines.

[Martin J. Bligh]

>From Andy Whitcroft

A few very simple changes in order to make CONFIG_NUMA work everywhere, so 
the distros can build one common binary kernel for distributions.

diff -urpN -X /home/fletch/.diff.exclude 012-pfn_valid/arch/i386/Kconfig 013-numa_x86_pc/arch/i386/Kconfig
--- 012-pfn_valid/arch/i386/Kconfig	Wed Mar  5 07:36:57 2003
+++ 013-numa_x86_pc/arch/i386/Kconfig	Wed Mar  5 07:41:54 2003
@@ -488,7 +488,7 @@ config NR_CPUS
 # Common NUMA Features
 config NUMA
 	bool "Numa Memory Allocation Support"
-	depends on (HIGHMEM64G && (X86_NUMAQ || (X86_SUMMIT && ACPI && !ACPI_HT_ONLY)))
+	depends on (HIGHMEM64G && (X86_NUMAQ || (X86_SUMMIT && ACPI && !ACPI_HT_ONLY))) || X86_PC
 
 config DISCONTIGMEM
 	bool
diff -urpN -X /home/fletch/.diff.exclude 012-pfn_valid/arch/i386/kernel/smpboot.c 013-numa_x86_pc/arch/i386/kernel/smpboot.c
--- 012-pfn_valid/arch/i386/kernel/smpboot.c	Wed Mar  5 07:36:57 2003
+++ 013-numa_x86_pc/arch/i386/kernel/smpboot.c	Wed Mar  5 07:41:54 2003
@@ -966,6 +966,7 @@ static void __init smp_boot_cpus(unsigne
 		if (APIC_init_uniprocessor())
 			printk(KERN_NOTICE "Local APIC not detected."
 					   " Using dummy APIC emulation.\n");
+		map_cpu_to_logical_apicid();
 		return;
 	}
 
diff -urpN -X /home/fletch/.diff.exclude 012-pfn_valid/arch/i386/mm/discontig.c 013-numa_x86_pc/arch/i386/mm/discontig.c
--- 012-pfn_valid/arch/i386/mm/discontig.c	Wed Mar  5 07:41:52 2003
+++ 013-numa_x86_pc/arch/i386/mm/discontig.c	Wed Mar  5 07:41:54 2003
@@ -82,6 +82,36 @@ void *node_remap_start_vaddr[MAX_NUMNODE
 void set_pmd_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags);
 
 /*
+ * FLAT - support for basic PC memory model with discontig enabled, essentially
+ *        a single node with all available processors in it with a flat
+ *        memory map.
+ */
+void __init get_memcfg_numa_flat(void)
+{
+	int pfn;
+
+	printk("NUMA - single node, flat memory mode\n");
+
+	/* Run the memory configuration and find the top of memory. */
+	find_max_pfn();
+	node_start_pfn[0]  = 0;
+	node_end_pfn[0]	  = max_pfn;
+
+	/* Fill in the physnode_map with our simplistic memory model,
+	* all memory is in node 0.
+	*/
+	for (pfn = node_start_pfn[0]; pfn <= node_end_pfn[0];
+	       pfn += PAGES_PER_ELEMENT)
+	{
+		physnode_map[pfn / PAGES_PER_ELEMENT] = 0;
+	}
+
+         /* Indicate there is one node available. */
+	node_set_online(0);
+	numnodes = 1;
+}
+
+/*
  * Find the highest page frame number we have available for the node
  */
 static void __init find_max_pfn_node(int nid)
diff -urpN -X /home/fletch/.diff.exclude 012-pfn_valid/include/asm-i386/mmzone.h 013-numa_x86_pc/include/asm-i386/mmzone.h
--- 012-pfn_valid/include/asm-i386/mmzone.h	Wed Mar  5 07:41:53 2003
+++ 013-numa_x86_pc/include/asm-i386/mmzone.h	Wed Mar  5 07:41:54 2003
@@ -123,6 +123,9 @@ static inline struct pglist_data *pfn_to
 #include <asm/numaq.h>
 #elif CONFIG_X86_SUMMIT
 #include <asm/srat.h>
+#elif CONFIG_X86_PC
+#define get_memcfg_numa get_memcfg_numa_flat
+#define get_zholes_size(n) (0)
 #else
 #define pfn_to_nid(pfn)		(0)
 #endif /* CONFIG_X86_NUMAQ */

[PATCH] 3/6 Convert physnode_map to u8

[Martin J. Bligh]

>From Andy Whitcroft

Convert physnode_map from an int to a u8 to save cachelines.

diff -urpN -X /home/fletch/.diff.exclude 013-numa_x86_pc/arch/i386/kernel/numaq.c 014-physnode_map_u8/arch/i386/kernel/numaq.c
--- 013-numa_x86_pc/arch/i386/kernel/numaq.c	Wed Mar  5 07:41:52 2003
+++ 014-physnode_map_u8/arch/i386/kernel/numaq.c	Wed Mar  5 07:41:55 2003
@@ -64,8 +64,6 @@ static void __init smp_dump_qct(void)
 	}
 }
 
-extern int physnode_map[];
-
 /*
  * for each node mark the regions
  *        TOPOFMEM = hi_shrd_mem_start + hi_shrd_mem_size
diff -urpN -X /home/fletch/.diff.exclude 013-numa_x86_pc/arch/i386/mm/discontig.c 014-physnode_map_u8/arch/i386/mm/discontig.c
--- 013-numa_x86_pc/arch/i386/mm/discontig.c	Wed Mar  5 07:41:54 2003
+++ 014-physnode_map_u8/arch/i386/mm/discontig.c	Wed Mar  5 07:41:55 2003
@@ -57,7 +57,7 @@ bootmem_data_t node0_bdata;
  *     physnode_map[4-7] = 1;
  *     physnode_map[8- ] = -1;
  */
-int physnode_map[MAX_ELEMENTS] = { [0 ... (MAX_ELEMENTS - 1)] = -1};
+u8 physnode_map[MAX_ELEMENTS] = { [0 ... (MAX_ELEMENTS - 1)] = -1};
 
 unsigned long node_start_pfn[MAX_NUMNODES];
 unsigned long node_end_pfn[MAX_NUMNODES];
diff -urpN -X /home/fletch/.diff.exclude 013-numa_x86_pc/include/asm-i386/mmzone.h 014-physnode_map_u8/include/asm-i386/mmzone.h
--- 013-numa_x86_pc/include/asm-i386/mmzone.h	Wed Mar  5 07:41:54 2003
+++ 014-physnode_map_u8/include/asm-i386/mmzone.h	Wed Mar  5 07:41:55 2003
@@ -107,7 +107,7 @@ extern struct pglist_data *node_data[];
 #define MAX_ELEMENTS 256
 #define PAGES_PER_ELEMENT (MAX_NR_PAGES/MAX_ELEMENTS)
 
-extern int physnode_map[];
+extern u8 physnode_map[];
 
 static inline int pfn_to_nid(unsigned long pfn)
 {
diff -urpN -X /home/fletch/.diff.exclude 013-numa_x86_pc/include/asm-i386/numaq.h 014-physnode_map_u8/include/asm-i386/numaq.h
--- 013-numa_x86_pc/include/asm-i386/numaq.h	Wed Mar  5 07:41:52 2003
+++ 014-physnode_map_u8/include/asm-i386/numaq.h	Wed Mar  5 07:41:55 2003
@@ -28,8 +28,6 @@
 
 #ifdef CONFIG_X86_NUMAQ
 
-extern int physnode_map[];
-
 #define MAX_NUMNODES		8
 extern void get_memcfg_numaq(void);
 #define get_memcfg_numa() get_memcfg_numaq()
diff -urpN -X /home/fletch/.diff.exclude 013-numa_x86_pc/include/asm-i386/srat.h 014-physnode_map_u8/include/asm-i386/srat.h
--- 013-numa_x86_pc/include/asm-i386/srat.h	Wed Mar  5 07:41:52 2003
+++ 014-physnode_map_u8/include/asm-i386/srat.h	Wed Mar  5 07:41:55 2003
@@ -27,7 +27,6 @@
 #ifndef _ASM_SRAT_H_
 #define _ASM_SRAT_H_
 
-extern int physnode_map[];
 #define MAX_NUMNODES		8
 extern void get_memcfg_from_srat(void);
 extern unsigned long *get_zholes_size(int);

[PATCH] 4/6 Fix the type of get_zholes_size for NUMA-Q

[Martin J. Bligh]

>From Andy Whitcroft

Fix the type of get_zholes_size for NUMA-Q

diff -urpN -X /home/fletch/.diff.exclude 014-physnode_map_u8/include/asm-i386/numaq.h 015-numaq_zholes_warning/include/asm-i386/numaq.h
--- 014-physnode_map_u8/include/asm-i386/numaq.h	Wed Mar  5 07:41:55 2003
+++ 015-numaq_zholes_warning/include/asm-i386/numaq.h	Wed Mar  5 07:43:31 2003
@@ -157,7 +157,7 @@ struct sys_cfg_data {
         struct	eachquadmem eq[MAX_NUMNODES];	/* indexed by quad id */
 };
 
-static inline unsigned long get_zholes_size(int nid)
+static inline unsigned long *get_zholes_size(int nid)
 {
 	return 0;
 }

[PATCH] 5/6 Provide basic documentation for profiling

[Martin J. Bligh]

People keep asking for this info, and Andrew asked me to put it under the
Documentation directory ... provides really simple instructions for taking
a profile so that users can report performance changes in a useful way.

diff -urpN -X /home/fletch/.diff.exclude 015-numaq_zholes_warning/Documentation/basic_profiling.txt 020-prof_docs/Documentation/basic_profiling.txt
--- 015-numaq_zholes_warning/Documentation/basic_profiling.txt	Wed Dec 31 16:00:00 1969
+++ 020-prof_docs/Documentation/basic_profiling.txt	Wed Mar  5 07:48:44 2003
@@ -0,0 +1,48 @@
+These instructions are deliberately very basic. If you want something clever,
+go read the real docs ;-) Please don't add more stuff, but feel free to 
+correct my mistakes ;-)    (mbligh@aracnet.com)
+Thanks to John Levon, Dave Hansen, et al. for help writing this.
+
+<test> is the thing you're trying to measure.
+Make sure you have the correct System.map / vmlinux referenced!
+IMHO it's easier to use "make install" for linux and hack /sbin/installkernel
+to copy config files, system.map, vmlinux to /boot.
+
+Readprofile
+-----------
+You need a fixed readprofile command for 2.5 ... either get hold of
+a current version from:
+http://www.kernel.org/pub/linux/utils/util-linux/
+or get readprofile binary fixed for 2.5 / akpm's 2.5 patch from 
+ftp://ftp.kernel.org/pub/linux/kernel/people/mbligh/tools/readprofile/
+
+Add "profile=2" to the kernel command line.
+
+clear		readprofile -r
+		<test>
+dump output	readprofile -m /boot/System.map > captured_profile
+
+Oprofile
+--------
+get source (I use 0.5) from http://oprofile.sourceforge.net/
+add "idle=poll" to the kernel command line 
+Configure with CONFIG_PROFILING=y and CONFIG_OPROFILE=y & reboot on new kernel
+./configure --with-kernel-support
+make install
+
+One time setup (pick appropriate one for your CPU):
+P3		opcontrol --setup --vmlinux=/boot/vmlinux \
+		--ctr0-event=CPU_CLK_UNHALTED --ctr0-count=100000
+Athlon/x86-64	opcontrol --setup --vmlinux=/boot/vmlinux \
+		--ctr0-event=RETIRED_INSNS --ctr0-count=100000
+P4		opcontrol --setup --vmlinux=/boot/vmlinux \
+		--ctr0-event=GLOBAL_POWER_EVENTS \
+		--ctr0-unit-mask=1 --ctr0-count=100000
+
+start daemon	opcontrol --start-daemon
+clear		opcontrol --reset
+start		opcontrol --start
+		<test>
+stop		opcontrol --stop
+dump output	oprofpp -dl -i /boot/vmlinux  >  output_file
+

[PATCH] 6/6 cacheline align files_lock

[Martin J. Bligh]

I'm getting a lot of cacheline bounce from .text.lock.file_table due to 
false sharing of the cahceline. The following patch just aligns the lock
in it's own cacheline.

only changes in profile under 50 ticks are:

 -4832   -22.2% .text.lock.file_table
 -6357   -12.8% default_idle
-10374    -6.2% total

Difference in results below (note system times as well as elapsed).

Kernbench: (make -j N vmlinux, where N = 2 x num_cpus)
                              Elapsed      System        User         CPU
                 no-align       44.09       94.38      557.26     1477.00
                    align       44.38       94.18      558.00     1468.25

Kernbench: (make -j N vmlinux, where N = 16 x num_cpus)
                              Elapsed      System        User         CPU
                 no-align       45.53      118.06      560.48     1489.50
                    align       44.84      111.77      560.63     1502.50

Kernbench: (make -j vmlinux, maximal tasks)
                              Elapsed      System        User         CPU
                 no-align       45.17      117.80      560.62     1500.50
                    align       44.94      113.36      560.59     1500.00

diff -urpN -X /home/fletch/.diff.exclude 020-prof_docs/fs/file_table.c 030-align_files_lock/fs/file_table.c
--- 020-prof_docs/fs/file_table.c	Tue Feb 25 23:03:49 2003
+++ 030-align_files_lock/fs/file_table.c	Wed Mar  5 07:49:20 2003
@@ -27,7 +27,7 @@ static LIST_HEAD(anon_list);
 /* And here the free ones sit */
 static LIST_HEAD(free_list);
 /* public *and* exported. Not pretty! */
-spinlock_t files_lock = SPIN_LOCK_UNLOCKED;
+spinlock_t files_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
 
 /* Find an unused file structure and return a pointer to it.
  * Returns NULL, if there are no more free file structures or

Re: [PATCH] 6/6 cacheline align files_lock

[Linus Torvalds]

On Sat, 8 Mar 2003, Christoph Hellwig wrote:
> 
> Agreed, that's what I actually though when looking into that stuff in more
> detail a while ago - I just couldn't remember everything now that Martin
> brought it up again.  Killing the freelist seems like a good idea anyway
> (or rather keep a small list for the reserved filp that is used only
> _after_ kmem_cache_alloc() failed)

Well, the sad part, though, is that the file lists are almost never 
actually _used_, so even if this is made per-superblock that won't get 
over the fact that

 (a) much of the time you'll still get the lock overhead (not very many
     superblocks would be impacted: it would usually spread out the
     current load over two or three super-blocks: pipes, networking and
     "real" filesystem) and

 (b) it would _still_ be for something that almost never happens -
     relatively speaking (ie it's currently used for hanging up terminals
     and for unmounts, and nothing else, I think)

I think we might actually be able to make it really go away, if the file
list was made per-dentry or something like that (and then use the existing
dentry->lock instead of dcache_lock). We already keep track of dentries
for "umount", and if we made the tty layer use another list, we'd get rid 
of this whole lock entirely.

At this point, though, the 2.6.x thing is clearly to just avoid the false 
sharing. But somebody can work on this if they feel like a 2.7.x 
challenge.

		Linus

Re: [PATCH] 6/6 cacheline align files_lock

[Christoph Hellwig]

On Sat, Mar 08, 2003 at 08:10:29AM -0800, Linus Torvalds wrote:
> So to actually fix file_lock, you need to do something else. I _think_
> that "something else" may be to make it be a per-super-block lock, since I
> think that's the only thing the f_list thing is actually used for. Then
> you should probably pass in the superblock pointer to "get_empty_filp()", 
> and _then_ you can get rid of the free list and the current global lock.

Agreed, that's what I actually though when looking into that stuff in more
detail a while ago - I just couldn't remember everything now that Martin
brought it up again.  Killing the freelist seems like a good idea anyway
(or rather keep a small list for the reserved filp that is used only
_after_ kmem_cache_alloc() failed)

Re: [PATCH] 6/6 cacheline align files_lock

[William Lee Irwin III]

On Sat, Mar 08, 2003 at 07:41:42AM -0800, Martin J. Bligh wrote:
> Ironically, the latest scheduler changes *seem* to get rid of about
> 57% of the cost of it .... I have *no* idea why. Diffprofile between
> 64 and 64-bk is just wierd:
>       7222    45.1% page_remove_rmap
>        521    18.4% __copy_from_user_ll
>        422     8.9% __copy_to_user_ll
>        334   105.4% __wake_up
>        248    48.5% pte_alloc_one
>        243   205.9% pgd_ctor
>        211     4.6% vm_enough_memory
>        198     6.3% zap_pte_range
>        179    29.3% kmem_cache_free
>        118    12.2% clear_page_tables
>         99    13.2% __pte_chain_free
>         84     6.1% do_no_page
>         76     6.6% release_pages
>         74     9.0% strnlen_user
>         67     5.6% free_hot_cold_page
>         65    14.4% copy_page_range

The high count for pgd_ctor() is bizarre. It fiddles with less data and
should be called less often than pmd_ctor() or pte_alloc_one(), both of
which take the cache hit for constructing full pages, and pgd_ctor()
can't really hope to do better. This otherwise appears to be in order
of cacheline bounciness. DCU_MISS_OUTSTANDING and IFU_MEM_STALL?

page_remove_rmap() as usual takes top position b/c of its long list
walks; I think you've got something brewing for that already.


-- wli

Re: [PATCH] 6/6 cacheline align files_lock

[Linus Torvalds]

On Sat, 8 Mar 2003, Christoph Hellwig wrote:
> 
> The free list should go away - we have slab for that.  The tty stuff should
> get a per-tty lock.

I doubt either of these would actually fix the lock contention, though.

The tty stuff is not likely to be a real issue for any real load (just how
often do you kill off sessions etc?) And the free list isn't the reason
for the file lock - ues, the file lock protects it, but every time we
touch the free list we touch _real_ lists too (ie either we move a file
from the free list to another list, _or_ we move a unused entry from a
real list to the free list), so we'd need the lock anyway.

So to actually fix file_lock, you need to do something else. I _think_
that "something else" may be to make it be a per-super-block lock, since I
think that's the only thing the f_list thing is actually used for. Then
you should probably pass in the superblock pointer to "get_empty_filp()", 
and _then_ you can get rid of the free list and the current global lock.

Oh, and you need to make the "tty" stuff be a superblock too. Of course, 
it might actually be a perfectly fine thing to make that tty stuff use a 
totally separate pointer chain anyway, the current thing makes me worry 
that "umount()" actually might do the wrong thing if the only file open on 
a filesystem are tty files.

		Linus

Re: [PATCH] 6/6 cacheline align files_lock

[Christoph Hellwig]

On Sat, Mar 08, 2003 at 07:41:42AM -0800, Martin J. Bligh wrote:
> It seems to lock several different things though - free lists, tty lists,
> etc, etc. I could break it up to one per list, but that means taking two
> locks to shift between lists, which I'd prefer not to do if possible.

The free list should go away - we have slab for that.  The tty stuff should
get a per-tty lock.

Re: [PATCH] 6/6 cacheline align files_lock

[Martin J. Bligh]

>> I'm getting a lot of cacheline bounce from .text.lock.file_table due to 
>> false sharing of the cahceline. The following patch just aligns the lock
>> in it's own cacheline.
> 
> What about fixing it's use instead?  

Seriously, I'd love to do that ... files_lock is the highest item of 
the whole kernel compile profile overall, and accounts for about 10%
of all systime.

By I inlining the spinlocks, I can see the heavy users seem to be:

      6278   166.0% get_empty_filp
      5482   239.3% __fput
      4770   254.7% file_move
    -14464  -100.0% .text.lock.file_table

It seems to lock several different things though - free lists, tty lists,
etc, etc. I could break it up to one per list, but that means taking two
locks to shift between lists, which I'd prefer not to do if possible.

Andrew was talking about something more dramatic for it, but I'm not
sure exactly what ... any suggestions on how to drive a stake through
it's heart gladly accepted.

Ironically, the latest scheduler changes *seem* to get rid of about
57% of the cost of it .... I have *no* idea why. Diffprofile between
64 and 64-bk is just wierd:

      7222    45.1% page_remove_rmap
       521    18.4% __copy_from_user_ll
       422     8.9% __copy_to_user_ll
       334   105.4% __wake_up
       248    48.5% pte_alloc_one
       243   205.9% pgd_ctor
       211     4.6% vm_enough_memory
       198     6.3% zap_pte_range
       179    29.3% kmem_cache_free
       118    12.2% clear_page_tables
        99    13.2% __pte_chain_free
        84     6.1% do_no_page
        76     6.6% release_pages
        74     9.0% strnlen_user
        67     5.6% free_hot_cold_page
        65    14.4% copy_page_range
...
       -54   -26.1% generic_file_open
       -54    -7.8% dentry_open
       -55   -16.3% do_lookup
       -61    -1.7% find_get_page
       -74   -58.7% __wake_up_sync
       -74    -8.2% current_kernel_time
       -82   -16.1% file_ra_state_init
      -127    -8.0% schedule
      -132   -19.1% fd_install
      -148    -2.2% page_add_rmap
      -246    -3.0% d_lookup
      -263    -9.3% atomic_dec_and_lock
      -407   -25.7% file_move
      -455   -20.5% __fput
      -853   -21.1% get_empty_filp
      -870    -1.9% default_idle
     -1724    -1.0% total
     -8473   -59.8% .text.lock.file_table

page_remove_rmap I'm not too worried about, as I know objrmap will kill
that already (tried to combine them last night, but got an oops, which
I'm just about to try to recreate with just virgin -bk). But *why*
things are shifting about so much from (presumably) just the scheduler
changes is a mystery to me at least. 

Any ideas?

M.

[PATCH] 6/6 cacheline align files_lock

[Martin J. Bligh]

I'm getting a lot of cacheline bounce from .text.lock.file_table due to 
false sharing of the cahceline. The following patch just aligns the lock
in it's own cacheline.

only changes in profile under 50 ticks are:

 -4832   -22.2% .text.lock.file_table
 -6357   -12.8% default_idle
-10374    -6.2% total

Difference in results below (note system times as well as elapsed).

Kernbench: (make -j N vmlinux, where N = 2 x num_cpus)
                              Elapsed      System        User         CPU
                 no-align       44.09       94.38      557.26     1477.00
                    align       44.38       94.18      558.00     1468.25

Kernbench: (make -j N vmlinux, where N = 16 x num_cpus)
                              Elapsed      System        User         CPU
                 no-align       45.53      118.06      560.48     1489.50
                    align       44.84      111.77      560.63     1502.50

Kernbench: (make -j vmlinux, maximal tasks)
                              Elapsed      System        User         CPU
                 no-align       45.17      117.80      560.62     1500.50
                    align       44.94      113.36      560.59     1500.00

diff -urpN -X /home/fletch/.diff.exclude 020-prof_docs/fs/file_table.c 030-align_files_lock/fs/file_table.c
--- 020-prof_docs/fs/file_table.c	Tue Feb 25 23:03:49 2003
+++ 030-align_files_lock/fs/file_table.c	Wed Mar  5 07:49:20 2003
@@ -27,7 +27,7 @@ static LIST_HEAD(anon_list);
 /* And here the free ones sit */
 static LIST_HEAD(free_list);
 /* public *and* exported. Not pretty! */
-spinlock_t files_lock = SPIN_LOCK_UNLOCKED;
+spinlock_t files_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
 
 /* Find an unused file structure and return a pointer to it.
  * Returns NULL, if there are no more free file structures or