Re: [PATCH] sched: Allow per-cpu kernel threads to run on online && !active

[Thomas Gleixner <tglx@linutronix.de>]

On Tue, 1 Mar 2016, Peter Zijlstra wrote:
> Tested-by: Thomas Gleixner <tglx@linutronix.de>

Works nicely especially when we move the set_cpu_active() calls as the last
state in the state machine. See patch below.

Thanks,

	tglx

8<--------------------------

Subject: cpu/hotplug: Handle cpu active as last state
From: Thomas Gleixner <tglx@linutronix.de>
Date: Thu, 03 Mar 2016 12:33:37 +0100

We want to make sure that everything is initialized before we allow scheduling
of arbitrary work on a upcoming CPU and we don't want to have no more random
work on it when we shut it down.

Now that the scheduler handles this nicely via the cpu_active_mask we can put
set_cpu_active() as last action when a cpu is brought up and as first action
when it goes down.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Peter Zijlstra <peterz@infradead.org>
---
 include/linux/cpuhotplug.h |    1 +
 kernel/cpu.c               |   19 +++++++++++++++++--
 kernel/sched/core.c        |   18 ------------------
 3 files changed, 18 insertions(+), 20 deletions(-)

--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -16,6 +16,7 @@ enum cpuhp_state {
 	CPUHP_AP_NOTIFY_ONLINE,
 	CPUHP_AP_ONLINE_DYN,
 	CPUHP_AP_ONLINE_DYN_END		= CPUHP_AP_ONLINE_DYN + 30,
+	CPUHP_AP_ACTIVE,
 	CPUHP_ONLINE,
 };
 
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -904,8 +904,6 @@ void cpuhp_online_idle(enum cpuhp_state
 
 	st->state = CPUHP_AP_ONLINE_IDLE;
 
-	/* The cpu is marked online, set it active now */
-	set_cpu_active(cpu, true);
 	/* Unpark the stopper thread and the hotplug thread of this cpu */
 	stop_machine_unpark(cpu);
 	kthread_unpark(st->thread);
@@ -917,6 +915,18 @@ void cpuhp_online_idle(enum cpuhp_state
 		complete(&st->done);
 }
 
+static int cpu_activate(unsigned int cpu)
+{
+	set_cpu_active(cpu, true);
+	return 0;
+}
+
+static int cpu_deactivate(unsigned int cpu)
+{
+	set_cpu_active(cpu, false);
+	return 0;
+}
+
 /* Requires cpu_add_remove_lock to be held */
 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
 {
@@ -1213,6 +1223,11 @@ static struct cpuhp_step cpuhp_ap_states
 		.startup		= notify_online,
 		.teardown		= notify_down_prepare,
 	},
+	[CPUHP_AP_ACTIVE] = {
+		.name			= "active",
+		.startup		= cpu_activate,
+		.teardown		= cpu_deactivate,
+	},
 #endif
 	[CPUHP_ONLINE] = {
 		.name			= "online",
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5726,23 +5726,6 @@ static int sched_cpu_active(struct notif
 	case CPU_STARTING:
 		set_cpu_rq_start_time();
 		return NOTIFY_OK;
-
-	case CPU_DOWN_FAILED:
-		set_cpu_active(cpu, true);
-		return NOTIFY_OK;
-
-	default:
-		return NOTIFY_DONE;
-	}
-}
-
-static int sched_cpu_inactive(struct notifier_block *nfb,
-					unsigned long action, void *hcpu)
-{
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_DOWN_PREPARE:
-		set_cpu_active((long)hcpu, false);
-		return NOTIFY_OK;
 	default:
 		return NOTIFY_DONE;
 	}
@@ -5761,7 +5744,6 @@ static int __init migration_init(void)
 
 	/* Register cpu active notifiers */
 	cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE);
-	cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE);
 
 	return 0;
 }