[PATCH] Automatically enable bigsmp when we have more than 8 CPUs

[PATCH] Automatically enable bigsmp when we have more than 8 CPUs

[Venkatesh Pallipadi]

Today, bigsmp mode is only detected with special oem dmi string or a 
boot option. This patch makes the selection of bigsmp automatic, whenever 
there are more 8 CPUs and xAPIC (APIC version 0x14) is supported. 
This patch should only affect systems with more than 8 logical CPUs 
that doesn't belong to any other special sub-archs.

The same issue was also discussed around a year back:
http://www.ussg.iu.edu/hypermail/linux/kernel/0408.0/1679.html
where I had posted a slightly different patch from the one below and that
patch did not go anywhere. The patch below is cleaner than the one in 
above link.

Please apply.

Thanks,
Venki


i386 generic subarchitecture requires explicit dmi strings or command line
to enable bigsmp mode. The patch below removes that restriction, and
uses bigsmp as soon as it finds more than 8 logical CPUs and xAPIC support.

Signed-off-by: Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>

diff -purN linux-2.6.13-rc3/arch/i386/kernel/acpi/boot.c linux-2.6.13-rc3-auto/arch/i386/kernel/acpi/boot.c
--- linux-2.6.13-rc3/arch/i386/kernel/acpi/boot.c	2005-07-29 00:58:06.174571656 -0700
+++ linux-2.6.13-rc3-auto/arch/i386/kernel/acpi/boot.c	2005-07-28 06:44:43.000000000 -0700
@@ -41,7 +41,6 @@
 #ifdef	CONFIG_X86_64
 
 static inline void  acpi_madt_oem_check(char *oem_id, char *oem_table_id) { }
-extern void __init clustered_apic_check(void);
 static inline int ioapic_setup_disabled(void) { return 0; }
 #include <asm/proto.h>
 
@@ -843,7 +842,6 @@ acpi_process_madt(void)
 				acpi_ioapic = 1;
 
 				smp_found_config = 1;
-				clustered_apic_check();
 			}
 		}
 		if (error == -EINVAL) {
diff -purN linux-2.6.13-rc3/arch/i386/kernel/mpparse.c linux-2.6.13-rc3-auto/arch/i386/kernel/mpparse.c
--- linux-2.6.13-rc3/arch/i386/kernel/mpparse.c	2005-07-29 00:58:06.175571504 -0700
+++ linux-2.6.13-rc3-auto/arch/i386/kernel/mpparse.c	2005-07-29 01:38:13.557593808 -0700
@@ -65,6 +65,8 @@ int nr_ioapics;
 int pic_mode;
 unsigned long mp_lapic_addr;
 
+unsigned int def_to_bigsmp = 0;
+
 /* Processor that is doing the boot up */
 unsigned int boot_cpu_physical_apicid = -1U;
 /* Internal processor count */
@@ -213,6 +215,11 @@ static void __init MP_processor_info (st
 		ver = 0x10;
 	}
 	apic_version[m->mpc_apicid] = ver;
+	if ((num_processors > 8) && APIC_XAPIC(ver))
+		def_to_bigsmp = 1;
+	else
+		def_to_bigsmp = 0;
+
 	bios_cpu_apicid[num_processors - 1] = m->mpc_apicid;
 }
 
@@ -479,7 +486,6 @@ static int __init smp_read_mpc(struct mp
 		}
 		++mpc_record;
 	}
-	clustered_apic_check();
 	if (!num_processors)
 		printk(KERN_ERR "SMP mptable: no processors registered!\n");
 	return num_processors;
diff -purN linux-2.6.13-rc3/arch/i386/kernel/setup.c linux-2.6.13-rc3-auto/arch/i386/kernel/setup.c
--- linux-2.6.13-rc3/arch/i386/kernel/setup.c	2005-07-29 00:58:06.177571200 -0700
+++ linux-2.6.13-rc3-auto/arch/i386/kernel/setup.c	2005-07-29 01:51:55.627620168 -0700
@@ -60,6 +60,11 @@
 #include "setup_arch_pre.h"
 #include <bios_ebda.h>
 
+#ifdef  CONFIG_X86_LOCAL_APIC
+#include <mach_apic.h>
+#include <mach_mpparse.h>
+#endif  /* CONFIG_X86_LOCAL_APIC */
+
 /* Forward Declaration. */
 void __init find_max_pfn(void);
 
@@ -1581,9 +1586,6 @@ void __init setup_arch(char **cmdline_p)
 
 	dmi_scan_machine();
 
-#ifdef CONFIG_X86_GENERICARCH
-	generic_apic_probe(*cmdline_p);
-#endif	
 	if (efi_enabled)
 		efi_map_memmap();
 
@@ -1600,6 +1602,14 @@ void __init setup_arch(char **cmdline_p)
 		get_smp_config();
 #endif
 
+#ifdef CONFIG_X86_GENERICARCH
+	generic_apic_probe(*cmdline_p);
+#endif        
+
+#ifdef  CONFIG_X86_LOCAL_APIC
+	clustered_apic_check();
+#endif
+
 	register_memory();
 
 #ifdef CONFIG_VT
diff -purN linux-2.6.13-rc3/arch/i386/mach-generic/bigsmp.c linux-2.6.13-rc3-auto/arch/i386/mach-generic/bigsmp.c
--- linux-2.6.13-rc3/arch/i386/mach-generic/bigsmp.c	2005-07-29 00:58:06.177571200 -0700
+++ linux-2.6.13-rc3-auto/arch/i386/mach-generic/bigsmp.c	2005-07-28 06:44:43.000000000 -0700
@@ -47,7 +47,10 @@ static struct dmi_system_id __initdata b
 
 static __init int probe_bigsmp(void)
 { 
-	dmi_check_system(bigsmp_dmi_table);
+	if (def_to_bigsmp)
+        	dmi_bigsmp = 1;
+	else
+		dmi_check_system(bigsmp_dmi_table);
 	return dmi_bigsmp; 
 } 
 
diff -purN linux-2.6.13-rc3/arch/i386/mach-generic/probe.c linux-2.6.13-rc3-auto/arch/i386/mach-generic/probe.c
--- linux-2.6.13-rc3/arch/i386/mach-generic/probe.c	2005-07-29 00:58:06.178571048 -0700
+++ linux-2.6.13-rc3-auto/arch/i386/mach-generic/probe.c	2005-07-28 06:44:43.000000000 -0700
@@ -24,9 +24,9 @@ struct genapic *genapic = &apic_default;
 
 struct genapic *apic_probe[] __initdata = { 
 	&apic_summit,
-	&apic_bigsmp, 
 	&apic_es7000,
-	&apic_default,	/* must be last */
+	&apic_bigsmp,  /* must be last but one */
+	&apic_default, /* must be last */
 	NULL,
 };
 
diff -purN linux-2.6.13-rc3/include/asm-i386/apicdef.h linux-2.6.13-rc3-auto/include/asm-i386/apicdef.h
--- linux-2.6.13-rc3/include/asm-i386/apicdef.h	2005-07-29 00:58:06.186569832 -0700
+++ linux-2.6.13-rc3-auto/include/asm-i386/apicdef.h	2005-07-29 00:58:40.466358512 -0700
@@ -16,6 +16,7 @@
 #define			GET_APIC_VERSION(x)	((x)&0xFF)
 #define			GET_APIC_MAXLVT(x)	(((x)>>16)&0xFF)
 #define			APIC_INTEGRATED(x)	((x)&0xF0)
+#define			APIC_XAPIC(x)		((x) == 0x14)
 #define		APIC_TASKPRI	0x80
 #define			APIC_TPRI_MASK		0xFF
 #define		APIC_ARBPRI	0x90
diff -purN linux-2.6.13-rc3/include/asm-i386/mpspec.h linux-2.6.13-rc3-auto/include/asm-i386/mpspec.h
--- linux-2.6.13-rc3/include/asm-i386/mpspec.h	2005-07-29 00:58:06.186569832 -0700
+++ linux-2.6.13-rc3-auto/include/asm-i386/mpspec.h	2005-07-29 01:32:46.132370008 -0700
@@ -11,6 +11,7 @@ extern int mp_bus_id_to_local [MAX_MP_BU
 extern int quad_local_to_mp_bus_id [NR_CPUS/4][4];
 extern int mp_bus_id_to_pci_bus [MAX_MP_BUSSES];
 
+extern unsigned int def_to_bigsmp;
 extern unsigned int boot_cpu_physical_apicid;
 extern int smp_found_config;
 extern void find_smp_config (void);

Re: [PATCH] Automatically enable bigsmp when we have more than 8 CPUs

[Andi Kleen]

On Thu, Jul 28, 2005 at 11:51:42AM -0700, Venkatesh Pallipadi wrote:
>  				smp_found_config = 1;
> -				clustered_apic_check();

Removing that here will break x86-64.

> --- linux-2.6.13-rc3/arch/i386/kernel/mpparse.c	2005-07-29 00:58:06.175571504 -0700
> +++ linux-2.6.13-rc3-auto/arch/i386/kernel/mpparse.c	2005-07-29 01:38:13.557593808 -0700
> @@ -65,6 +65,8 @@ int nr_ioapics;
>  int pic_mode;
>  unsigned long mp_lapic_addr;
>  
> +unsigned int def_to_bigsmp = 0;
> +
>  /* Processor that is doing the boot up */
>  unsigned int boot_cpu_physical_apicid = -1U;
>  /* Internal processor count */
> @@ -213,6 +215,11 @@ static void __init MP_processor_info (st
>  		ver = 0x10;
>  	}
>  	apic_version[m->mpc_apicid] = ver;
> +	if ((num_processors > 8) && APIC_XAPIC(ver))
> +		def_to_bigsmp = 1;
> +	else
> +		def_to_bigsmp = 0;
> +

I think this needs at least an CPU vendor == INTEL check, because
bigsmp doesn't work on big AMD systems.

[I have a new physflat i386 subarch pending for that] 

Also I would add a printk somewhere that warns when bigsmp
is not available.

>  	dmi_scan_machine();
>  
> -#ifdef CONFIG_X86_GENERICARCH
> -	generic_apic_probe(*cmdline_p);
> -#endif	
>  	if (efi_enabled)
>  		efi_map_memmap();
>  
> @@ -1600,6 +1602,14 @@ void __init setup_arch(char **cmdline_p)
>  		get_smp_config();
>  #endif
>  
> +#ifdef CONFIG_X86_GENERICARCH
> +	generic_apic_probe(*cmdline_p);
> +#endif        

This move looks risky because ACPI can already switch subarchs. Are you
sure the command line option still works? Testing on summit and es7000
would be good.

-Andi

Re: [PATCH] updated - Automatically enable bigsmp when we have more than 8 CPUs

[Venkatesh Pallipadi]

Below is the updated patch.

Changes from previous version:
* Make sure there are no side-effects on other i386 subarchs.
* Make sure nothing in this patch affects x86_64
* Add additional check for Intel CPUs before switching to bigsmp mode
* Add a warning message if more than 8 CPUs are found with X86_PC subarch

Thanks,
Venki

i386 generic subarchitecture requires explicit dmi strings or command line
to enable bigsmp mode. The patch below removes that restriction, and
uses bigsmp as soon as it finds more than 8 logical CPUs, Intel processors 
and xAPIC support.

Signed-off-by: Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>

diff -purN linux-2.6.13-rc3/arch/i386/kernel/acpi/boot.c linux-2.6.13-rc3-auto/arch/i386/kernel/acpi/boot.c
--- linux-2.6.13-rc3/arch/i386/kernel/acpi/boot.c	2005-07-29 00:58:06.000000000 -0700
+++ linux-2.6.13-rc3-auto/arch/i386/kernel/acpi/boot.c	2005-08-03 10:13:02.000000000 -0700
@@ -833,6 +833,9 @@ acpi_process_madt(void)
 		if (!error) {
 			acpi_lapic = 1;
 
+#ifdef CONFIG_X86_GENERICARCH
+			generic_bigsmp_probe();
+#endif   
 			/*
 			 * Parse MADT IO-APIC entries
 			 */
diff -purN linux-2.6.13-rc3/arch/i386/kernel/mpparse.c linux-2.6.13-rc3-auto/arch/i386/kernel/mpparse.c
--- linux-2.6.13-rc3/arch/i386/kernel/mpparse.c	2005-07-29 00:58:06.000000000 -0700
+++ linux-2.6.13-rc3-auto/arch/i386/kernel/mpparse.c	2005-08-03 14:12:34.156775936 -0700
@@ -65,6 +65,8 @@ int nr_ioapics;
 int pic_mode;
 unsigned long mp_lapic_addr;
 
+unsigned int def_to_bigsmp = 0;
+
 /* Processor that is doing the boot up */
 unsigned int boot_cpu_physical_apicid = -1U;
 /* Internal processor count */
@@ -213,6 +215,13 @@ static void __init MP_processor_info (st
 		ver = 0x10;
 	}
 	apic_version[m->mpc_apicid] = ver;
+	if ((num_processors > 8) && 
+	    APIC_XAPIC(ver) && 
+	    (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL))
+		def_to_bigsmp = 1;
+	else
+		def_to_bigsmp = 0;
+
 	bios_cpu_apicid[num_processors - 1] = m->mpc_apicid;
 }
 
diff -purN linux-2.6.13-rc3/arch/i386/kernel/setup.c linux-2.6.13-rc3-auto/arch/i386/kernel/setup.c
--- linux-2.6.13-rc3/arch/i386/kernel/setup.c	2005-07-29 00:58:06.000000000 -0700
+++ linux-2.6.13-rc3-auto/arch/i386/kernel/setup.c	2005-08-03 14:33:34.450182216 -0700
@@ -1593,8 +1594,13 @@ void __init setup_arch(char **cmdline_p)
 	 */
 	acpi_boot_table_init();
 	acpi_boot_init();
-#endif
 
+#ifdef CONFIG_X86_PC
+	if (def_to_bigsmp) {
+		printk(KERN_WARNING "More than 8 CPUs detected and CONFIG_X86_PC cannot handle it. Use CONFIG_X86_GENERICARCH or CONFIG_X86_BIGSMP.\n");
+	}
+#endif
+#endif
 #ifdef CONFIG_X86_LOCAL_APIC
 	if (smp_found_config)
 		get_smp_config();
diff -purN linux-2.6.13-rc3/arch/i386/mach-generic/bigsmp.c linux-2.6.13-rc3-auto/arch/i386/mach-generic/bigsmp.c
--- linux-2.6.13-rc3/arch/i386/mach-generic/bigsmp.c	2005-07-29 00:58:06.000000000 -0700
+++ linux-2.6.13-rc3-auto/arch/i386/mach-generic/bigsmp.c	2005-08-03 09:54:58.000000000 -0700
@@ -47,7 +47,10 @@ static struct dmi_system_id __initdata b
 
 static __init int probe_bigsmp(void)
 { 
-	dmi_check_system(bigsmp_dmi_table);
+	if (def_to_bigsmp)
+        	dmi_bigsmp = 1;
+	else
+		dmi_check_system(bigsmp_dmi_table);
 	return dmi_bigsmp; 
 } 
 
diff -purN linux-2.6.13-rc3/arch/i386/mach-generic/probe.c linux-2.6.13-rc3-auto/arch/i386/mach-generic/probe.c
--- linux-2.6.13-rc3/arch/i386/mach-generic/probe.c	2005-07-29 00:58:06.000000000 -0700
+++ linux-2.6.13-rc3-auto/arch/i386/mach-generic/probe.c	2005-08-03 14:00:38.533567200 -0700
@@ -30,6 +30,25 @@ struct genapic *apic_probe[] __initdata 
 	NULL,
 };
 
+static int cmdline_apic;
+
+void __init generic_bigsmp_probe(void) 
+{
+	/*
+	 * This routine is used to switch to bigsmp mode when
+	 * - There is no apic= option specified by the user
+	 * - generic_apic_probe() has choosen apic_default as the sub_arch
+	 * - we find more than 8 CPUs in acpi LAPIC listing with xAPIC support
+	 */
+	   
+	if (!cmdline_apic && genapic == &apic_default)
+		if (apic_bigsmp.probe()) {
+			genapic = &apic_bigsmp;
+			printk(KERN_INFO "Overriding APIC driver with %s\n", 
+			       genapic->name);
+		}
+}
+
 void __init generic_apic_probe(char *command_line) 
 { 
 	char *s;
@@ -52,6 +71,7 @@ void __init generic_apic_probe(char *com
 		if (!changed)
 			printk(KERN_ERR "Unknown genapic `%s' specified.\n", s);
 		*p = old;
+		cmdline_apic = changed;
 	} 
 	for (i = 0; !changed && apic_probe[i]; i++) { 
 		if (apic_probe[i]->probe()) {
diff -purN linux-2.6.13-rc3/include/asm-i386/apicdef.h linux-2.6.13-rc3-auto/include/asm-i386/apicdef.h
--- linux-2.6.13-rc3/include/asm-i386/apicdef.h	2005-07-29 00:58:06.000000000 -0700
+++ linux-2.6.13-rc3-auto/include/asm-i386/apicdef.h	2005-08-03 09:54:58.000000000 -0700
@@ -16,6 +16,7 @@
 #define			GET_APIC_VERSION(x)	((x)&0xFF)
 #define			GET_APIC_MAXLVT(x)	(((x)>>16)&0xFF)
 #define			APIC_INTEGRATED(x)	((x)&0xF0)
+#define			APIC_XAPIC(x)		((x) >= 0x14)
 #define		APIC_TASKPRI	0x80
 #define			APIC_TPRI_MASK		0xFF
 #define		APIC_ARBPRI	0x90
diff -purN linux-2.6.13-rc3/include/asm-i386/mach-generic/mach_apic.h linux-2.6.13-rc3-auto/include/asm-i386/mach-generic/mach_apic.h
--- linux-2.6.13-rc3/include/asm-i386/mach-generic/mach_apic.h	2005-07-27 00:02:25.000000000 -0700
+++ linux-2.6.13-rc3-auto/include/asm-i386/mach-generic/mach_apic.h	2005-08-03 10:13:12.000000000 -0700
@@ -28,4 +28,6 @@
 #define enable_apic_mode (genapic->enable_apic_mode)
 #define phys_pkg_id (genapic->phys_pkg_id)
 
+extern void generic_bigsmp_probe(void);
+
 #endif /* __ASM_MACH_APIC_H */
diff -purN linux-2.6.13-rc3/include/asm-i386/mpspec.h linux-2.6.13-rc3-auto/include/asm-i386/mpspec.h
--- linux-2.6.13-rc3/include/asm-i386/mpspec.h	2005-07-29 00:58:06.000000000 -0700
+++ linux-2.6.13-rc3-auto/include/asm-i386/mpspec.h	2005-08-03 09:54:58.000000000 -0700
@@ -11,6 +11,7 @@ extern int mp_bus_id_to_local [MAX_MP_BU
 extern int quad_local_to_mp_bus_id [NR_CPUS/4][4];
 extern int mp_bus_id_to_pci_bus [MAX_MP_BUSSES];
 
+extern unsigned int def_to_bigsmp;
 extern unsigned int boot_cpu_physical_apicid;
 extern int smp_found_config;
 extern void find_smp_config (void);

Re: [PATCH] updated - Automatically enable bigsmp when we have more than 8 CPUs

[Andi Kleen]

On Wed, Aug 03, 2005 at 04:01:14PM -0700, Venkatesh Pallipadi wrote:
> 
> Below is the updated patch.

Looks good. Only minor nit.

> +++ linux-2.6.13-rc3-auto/arch/i386/kernel/setup.c	2005-08-03 14:33:34.450182216 -0700
> @@ -1593,8 +1594,13 @@ void __init setup_arch(char **cmdline_p)
>  	 */
>  	acpi_boot_table_init();
>  	acpi_boot_init();
> -#endif
>  
> +#ifdef CONFIG_X86_PC
> +	if (def_to_bigsmp) {
> +		printk(KERN_WARNING "More than 8 CPUs detected and CONFIG_X86_PC cannot handle it. Use CONFIG_X86_GENERICARCH or CONFIG_X86_BIGSMP.\n");

Isn't that printk line longer than 80 characters?  Will it even fit 
on the screen? 

-Andi

RE: [PATCH] Automatically enable bigsmp when we have more than 8 CPUs

[Pallipadi, Venkatesh]

 

>-----Original Message-----
>From: Andi Kleen [mailto:ak@suse.de] 
>Sent: Wednesday, August 03, 2005 3:20 AM
>To: Pallipadi, Venkatesh
>Cc: Andrew Morton; Andi Kleen; Shah, Rajesh; linux-kernel
>Subject: Re: [PATCH] Automatically enable bigsmp when we have 
>more than 8 CPUs
>
>On Thu, Jul 28, 2005 at 11:51:42AM -0700, Venkatesh Pallipadi wrote:
>>  				smp_found_config = 1;
>> -				clustered_apic_check();
>
>Removing that here will break x86-64.
>

Yes. Ashok sent another patch to fix this part. Lkml subjectline
[patch 1/8] x86_64: Reintroduce clustered_apic_check() for x86_64

>>  
>> +#ifdef CONFIG_X86_GENERICARCH
>> +	generic_apic_probe(*cmdline_p);
>> +#endif        
>
>This move looks risky because ACPI can already switch subarchs. Are you
>sure the command line option still works? Testing on summit and es7000
>would be good.
>

Yes. We also saw some potential issues with ACPI ioapic detection on
other sub-archs after this change, late yesteraday. I am right now 
reworking this patch.

Andrew: Until I send a reworked patch can you remove these two patces
from 
the mm. Thanks.
1) [PATCH] Automatically enable bigsmp when we have more than 8 CPUs
2) [patch 1/8] x86_64: Reintroduce clustered_apic_check() for x86_64

Thanks,
Venki