Re: [PATCH] x86: don't ever patch back to UP if we unplug cpus.

[Suresh Siddha <suresh.b.siddha@intel.com>]

On Fri, 2012-07-27 at 17:08 +0930, Rusty Russell wrote:
> Paul McKenney points out:
> 
>  mean offline overhead is 6251/48=130.2 milliseconds.
> 
>  If I remove the alternatives_smp_switch() from the offline
>  path [...] the mean offline overhead is 550/42=13.1 milliseconds
> 
> Basically, we're never going to get those 120ms back, and the code is
> pretty messy.

I am ok with this proposal. If I follow correctly, you are still
allowing the patching to UP happen on boot if there is only one online
cpu currently but a possiblity of multiple cpu's that can come online
later. Perhaps make this little more explicit in the changelog.

Also,

> +	/* Patch to UP if other cpus not imminent. */
> +	if (noreplace_smp || num_present_cpus() == 1 || setup_max_cpus <= 1) {

shouldn't this be

	if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1))

also, to be consistent with other checks, may be just use
"num_possible_cpus() == 1" check instead of "setup_max_cpus <= 1".

thanks,
suresh

> +		uniproc_patched = true;
>  		alternatives_smp_module_add(NULL, "core kernel",
>  					    __smp_locks, __smp_locks_end,
>  					    _text, _etext);
> -
> -		/* Only switch to UP mode if we don't immediately boot others */
> -		if (num_present_cpus() == 1 || setup_max_cpus <= 1)
> -			alternatives_smp_switch(0);
>  	}
> -#endif
>   	apply_paravirt(__parainstructions, __parainstructions_end);
>  
> -	if (smp_alt_once)
> +	if (!uniproc_patched || num_possible_cpus() == 1)
>  		free_init_pages("SMP alternatives",
>  				(unsigned long)__smp_locks,
>  				(unsigned long)__smp_locks_end);
> diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
> --- a/arch/x86/kernel/smpboot.c
> +++ b/arch/x86/kernel/smpboot.c
> @@ -665,7 +665,8 @@ static int __cpuinit do_boot_cpu(int api
>  	unsigned long boot_error = 0;
>  	int timeout;
>  
> -	alternatives_smp_switch(1);
> +	/* Just in case we booted with a single CPU. */
> +	alternatives_enable_smp();
>  
>  	idle->thread.sp = (unsigned long) (((struct pt_regs *)
>  			  (THREAD_SIZE +  task_stack_page(idle))) - 1);
> @@ -1053,20 +1054,6 @@ out:
>  	preempt_enable();
>  }
>  
> -void arch_disable_nonboot_cpus_begin(void)
> -{
> -	/*
> -	 * Avoid the smp alternatives switch during the disable_nonboot_cpus().
> -	 * In the suspend path, we will be back in the SMP mode shortly anyways.
> -	 */
> -	skip_smp_alternatives = true;
> -}
> -
> -void arch_disable_nonboot_cpus_end(void)
> -{
> -	skip_smp_alternatives = false;
> -}
> -
>  void arch_enable_nonboot_cpus_begin(void)
>  {
>  	set_mtrr_aps_delayed_init();
> @@ -1256,9 +1243,6 @@ void native_cpu_die(unsigned int cpu)
>  		if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
>  			if (system_state == SYSTEM_RUNNING)
>  				pr_info("CPU %u is now offline\n", cpu);
> -
> -			if (1 == num_online_cpus())
> -				alternatives_smp_switch(0);
>  			return;
>  		}
>  		msleep(100);
> diff --git a/kernel/cpu.c b/kernel/cpu.c
> --- a/kernel/cpu.c
> +++ b/kernel/cpu.c
> @@ -439,14 +439,6 @@ EXPORT_SYMBOL_GPL(cpu_up);
>  #ifdef CONFIG_PM_SLEEP_SMP
>  static cpumask_var_t frozen_cpus;
>  
> -void __weak arch_disable_nonboot_cpus_begin(void)
> -{
> -}
> -
> -void __weak arch_disable_nonboot_cpus_end(void)
> -{
> -}
> -
>  int disable_nonboot_cpus(void)
>  {
>  	int cpu, first_cpu, error = 0;
> @@ -458,7 +450,6 @@ int disable_nonboot_cpus(void)
>  	 * with the userspace trying to use the CPU hotplug at the same time
>  	 */
>  	cpumask_clear(frozen_cpus);
> -	arch_disable_nonboot_cpus_begin();
>  
>  	printk("Disabling non-boot CPUs ...\n");
>  	for_each_online_cpu(cpu) {
> @@ -474,8 +465,6 @@ int disable_nonboot_cpus(void)
>  		}
>  	}
>  
> -	arch_disable_nonboot_cpus_end();
> -
>  	if (!error) {
>  		BUG_ON(num_online_cpus() > 1);
>  		/* Make sure the CPUs won't be enabled by someone else */