[PATCH 4/4] x86: Optimize try_ipiless_wakeup avoid idle task lookup

[Venkatesh Pallipadi <venki@google.com>]

Optimize try_ipiless_wakeup with caching of idletask's ti_flags pointer
in a percpu area. Shows a measurable difference in cost of async
smp_call_function_single() to a target CPU that is mwait-idle.

This shows ~50-100 cycles (of total 1200(local) or 1900(remote)) savings on
the IPI send side (as measured with async smp_call_function_single).

Signed-off-by: Venkatesh Pallipadi <venki@google.com>
---
 arch/x86/include/asm/ipiless_wake.h |    7 ++++---
 arch/x86/kernel/smpboot.c           |    4 ++++
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/arch/x86/include/asm/ipiless_wake.h b/arch/x86/include/asm/ipiless_wake.h
index a490dd3..232ce36 100644
--- a/arch/x86/include/asm/ipiless_wake.h
+++ b/arch/x86/include/asm/ipiless_wake.h
@@ -7,6 +7,7 @@
 
 #ifdef CONFIG_SMP
 
+DECLARE_PER_CPU(__u32 *, idletask_ti_flags);
 /*
  * TIF_IN_IPILESS_IDLE CPU being in a idle state with ipiless wakeup
  * capability, without any pending IPIs.
@@ -61,13 +62,13 @@ static inline void do_ipiless_pending_work(void)
 
 static inline int try_ipiless_wakeup(int cpu)
 {
-	struct thread_info *idle_ti = task_thread_info(idle_task(cpu));
+	__u32 *ti_flags = per_cpu(idletask_ti_flags, cpu);
 
-	if (!(idle_ti->flags & _TIF_IN_IPILESS_IDLE))
+	if (!(*ti_flags & _TIF_IN_IPILESS_IDLE))
 		return 0;
 
 	return test_and_clear_bit(TIF_IN_IPILESS_IDLE,
-					(unsigned long *)&idle_ti->flags);
+					(unsigned long *)ti_flags);
 }
 
 #else
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 66d250c..33339e2 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -133,6 +133,8 @@ DEFINE_PER_CPU(cpumask_var_t, cpu_llc_shared_map);
 DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
 EXPORT_PER_CPU_SYMBOL(cpu_info);
 
+DEFINE_PER_CPU(__u32 *, idletask_ti_flags);
+
 atomic_t init_deasserted;
 
 /*
@@ -715,6 +717,7 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu)
 	set_idle_for_cpu(cpu, c_idle.idle);
 do_rest:
 	per_cpu(current_task, cpu) = c_idle.idle;
+	per_cpu(idletask_ti_flags, cpu) = &task_thread_info(c_idle.idle)->flags;
 #ifdef CONFIG_X86_32
 	/* Stack for startup_32 can be just as for start_secondary onwards */
 	irq_ctx_init(cpu);
@@ -1143,6 +1146,7 @@ void __init native_smp_prepare_boot_cpu(void)
 	/* already set me in cpu_online_mask in boot_cpu_init() */
 	cpumask_set_cpu(me, cpu_callout_mask);
 	per_cpu(cpu_state, me) = CPU_ONLINE;
+	per_cpu(idletask_ti_flags, me) = &task_thread_info(current)->flags;
 }
 
 void __init native_smp_cpus_done(unsigned int max_cpus)
-- 
1.7.7.3