[PATCH v2 2/4] sched/idle: Don't handle polling idle as dynticks idle

[Marco Crivellari <marco.crivellari@suse.com>]

TS_FLAG_INIDLE tells nohz whether the tick is managed by the idle loop or
on IRQ exit. With idle=poll it is set while polling, but cpu_idle_poll()
doesn't run the idle loop again on interrupts: a nohz_full kick or the
expiry of a stopped tick isn't re-evaluated until something sets
TIF_NEED_RESCHED. The tick is therefore restarted before polling, and on
nohz_full CPUs it then keeps running after the wake-up and interrupts
userspace.

Enter dynticks idle only on the cpuidle path, from cpuidle_idle_call(),
and leave it before cpu_idle_poll(). While polling, the tick is then
managed on IRQ exit as for a running task: on nohz_full CPUs it stays
stopped if possible, and nothing is left to do on wake-up, so the task
doesn't pay for a tick stop after waking up. On other CPUs, leaving
dynticks idle restarts the tick as before.

tick_nohz_idle_enter() may now be called on each idle loop iteration and
always runs with interrupts disabled; tick_nohz_idle_exit() may be called
outside dynticks idle and with interrupts disabled.

Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Juri Lelli <juri.lelli@redhat.com>
Cc: Vincent Guittot <vincent.guittot@linaro.org>
Cc: Dietmar Eggemann <dietmar.eggemann@arm.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Ben Segall <bsegall@google.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Valentin Schneider <vschneid@redhat.com>
Cc: K Prateek Nayak <kprateek.nayak@amd.com>
Cc: Paul E. McKenney <paulmck@kernel.org>
Cc: Anna-Maria Behnsen <anna-maria@linutronix.de>
Cc: Thomas Gleixner <tglx@kernel.org>
Link: https://lore.kernel.org/all/aQiWfnnSzxsnwa2o@tpad/
Suggested-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Marco Crivellari <marco.crivellari@suse.com>
---
 kernel/sched/idle.c      | 13 +++++++++++--
 kernel/time/tick-sched.c | 27 ++++++++++++++++-----------
 2 files changed, 27 insertions(+), 13 deletions(-)

diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index e0c10a5dbda2..e0e3c06da528 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -193,6 +193,8 @@ static void cpuidle_idle_call(bool stop_tick)
 		return;
 	}
 
+	tick_nohz_idle_enter();
+
 	if (cpuidle_not_available(drv, dev)) {
 		idle_call_stop_or_retain_tick(stop_tick);
 
@@ -303,7 +305,6 @@ static void do_idle(void)
 	 */
 
 	__current_set_polling();
-	tick_nohz_idle_enter();
 
 	while (!need_resched()) {
 
@@ -349,7 +350,15 @@ static void do_idle(void)
 		 * idle as we know that the IPI is going to arrive right away.
 		 */
 		if (cpu_idle_force_poll || tick_check_broadcast_expired()) {
-			tick_nohz_idle_restart_tick();
+			/*
+			 * Polling idle is not dynticks idle: leave it before
+			 * polling so that the tick is managed on IRQ exit as for
+			 * a running task. On nohz_full CPUs it stays stopped and
+			 * nothing is left to do on wake-up that would delay the
+			 * task. On other CPUs, leaving dynticks idle restarts it
+			 * as before.
+			 */
+			tick_nohz_idle_exit();
 			cpu_idle_poll();
 		} else {
 			cpuidle_idle_call(got_tick);
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 645dba919386..0b798dc39511 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -1227,23 +1227,22 @@ static void tick_nohz_clock_wakeup(struct tick_sched *ts)
 /**
  * tick_nohz_idle_enter - prepare for entering idle on the current CPU
  *
- * Called when we start the idle loop.
+ * Called with IRQs disabled on each idle loop iteration that enters
+ * cpuidle. Only the first call enters dynticks idle.
  */
 void tick_nohz_idle_enter(void)
 {
-	struct tick_sched *ts;
+	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
 
-	lockdep_assert_irqs_enabled();
+	lockdep_assert_irqs_disabled();
 
-	local_irq_disable();
+	if (tick_sched_flag_test(ts, TS_FLAG_INIDLE))
+		return;
 
-	ts = this_cpu_ptr(&tick_cpu_sched);
 	WARN_ON_ONCE(ts->timer_expires_base);
 	tick_sched_flag_set(ts, TS_FLAG_INIDLE);
 	ts->idle_entrytime = ktime_get();
 	tick_nohz_clock_sleep(ts);
-
-	local_irq_enable();
 }
 
 /**
@@ -1415,15 +1414,21 @@ static void tick_nohz_idle_update_tick(struct tick_sched *ts, ktime_t now)
  *        as long as possible.
  *   2.2) If the tick has dependencies, restart the tick.
  *
+ * Also called before polling idle, which is not dynticks idle: it may
+ * therefore be called outside dynticks idle, with or without IRQs
+ * disabled.
  */
 void tick_nohz_idle_exit(void)
 {
 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
+	unsigned long flags;
 	ktime_t now;
 
-	local_irq_disable();
+	local_irq_save(flags);
+
+	if (!tick_sched_flag_test(ts, TS_FLAG_INIDLE))
+		goto out;
 
-	WARN_ON_ONCE(!tick_sched_flag_test(ts, TS_FLAG_INIDLE));
 	WARN_ON_ONCE(ts->timer_expires_base);
 
 	tick_sched_flag_clear(ts, TS_FLAG_INIDLE);
@@ -1434,8 +1439,8 @@ void tick_nohz_idle_exit(void)
 		tick_nohz_dyntick_stop(ts, now);
 		tick_nohz_idle_update_tick(ts, now);
 	}
-
-	local_irq_enable();
+out:
+	local_irq_restore(flags);
 }
 
 /*
-- 
2.56.0