[PATCH 1/3] tick/nohz: Re-evaluate the tick on polling idle and on user entry

[Marco Crivellari <marco.crivellari@suse.com>]

On nohz_full CPUs the tick is re-evaluated on IRQ exit and on idle exit.
There are three places where this is not enough:

- A kick handled while the CPU is in polling idle: in idle,
  tick_nohz_irq_exit() only does accounting and relies on the idle loop
  iterating again. HLT and MWAIT return on any IRQ, but cpu_idle_poll()
  only returns on TIF_NEED_RESCHED. The tick always runs in polling idle
  for now, but once the next patch stops it there, a timer enqueued from
  an interrupt on the polling CPU would not be programmed until something
  else wakes the CPU up.

  Set TIF_NEED_RESCHED on the idle task when the kick is handled in idle,
  so that cpu_idle_poll() returns and tick_nohz_idle_exit() re-evaluates
  the tick.

- The return to userspace: if the tick is left running in the kernel,
  for example because a tick dependency was cleared (tick_nohz_dep_clear*()
  don't kick the CPU) or a timer that kept it running was cancelled, it
  fires at least once in userspace before tick_nohz_irq_exit() stops it.

  Stop it from context tracking right before resuming userspace. When the
  tick is already stopped, the common case, this costs a few flag tests
  on each return to userspace.

- The expiry of a stopped tick in polling idle: a stopped tick is
  programmed for the next timer only, and reprogramming it for the
  following one is left to the idle loop, which doesn't run again while
  cpu_idle_poll() spins. Timers that were already queued would then wait
  for an unrelated wakeup.

  Set TIF_NEED_RESCHED in tick_nohz_handler() when the stopped tick
  fires in idle on a nohz_full CPU.

Cc: Anna-Maria Behnsen <anna-maria@linutronix.de>
Cc: Thomas Gleixner <tglx@kernel.org>
Signed-off-by: Marco Crivellari <marco.crivellari@suse.com>
---
 include/linux/tick.h      |  8 ++++++++
 kernel/context_tracking.c |  9 +++++++++
 kernel/time/tick-sched.c  | 41 ++++++++++++++++++++++++++++++++++++++-
 3 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/include/linux/tick.h b/include/linux/tick.h
index b121c5d53e30..8bfbb053f4fd 100644
--- a/include/linux/tick.h
+++ b/include/linux/tick.h
@@ -269,6 +269,7 @@ static inline void tick_dep_clear_signal(struct signal_struct *signal,
 extern void tick_nohz_full_kick_cpu(int cpu);
 extern void __tick_nohz_task_switch(void);
 extern void __init tick_nohz_full_setup(cpumask_var_t cpumask);
+extern void __tick_nohz_user_enter(void);
 #else
 static inline bool tick_nohz_full_enabled(void) { return false; }
 static inline bool tick_nohz_full_cpu(int cpu) { return false; }
@@ -294,6 +295,7 @@ static inline void tick_dep_clear_signal(struct signal_struct *signal,
 static inline void tick_nohz_full_kick_cpu(int cpu) { }
 static inline void __tick_nohz_task_switch(void) { }
 static inline void tick_nohz_full_setup(cpumask_var_t cpumask) { }
+static inline void __tick_nohz_user_enter(void) { }
 #endif
 
 static inline void tick_nohz_task_switch(void)
@@ -302,6 +304,12 @@ static inline void tick_nohz_task_switch(void)
 		__tick_nohz_task_switch();
 }
 
+static inline void tick_nohz_user_enter(void)
+{
+	if (tick_nohz_full_enabled())
+		__tick_nohz_user_enter();
+}
+
 static inline void tick_nohz_user_enter_prepare(void)
 {
 	if (tick_nohz_full_cpu(smp_processor_id()))
diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c
index a743e7ffa6c0..d4274698d6f3 100644
--- a/kernel/context_tracking.c
+++ b/kernel/context_tracking.c
@@ -24,6 +24,7 @@
 #include <linux/export.h>
 #include <linux/kprobes.h>
 #include <trace/events/rcu.h>
+#include <linux/tick.h>
 
 
 DEFINE_PER_CPU(struct context_tracking, context_tracking) = {
@@ -466,10 +467,18 @@ void noinstr __ct_user_enter(enum ctx_state state)
 			 */
 			if (state == CT_STATE_USER) {
 				instrumentation_begin();
+				/*
+				 * On nohz_full CPUs the tick may have been left running in the
+				 * kernel, e.g. kept by a tick dependency that is now gone.
+				 * Stop it now rather than let it interrupt userspace once
+				 * more before the next IRQ exit notices.
+				 */
+				tick_nohz_user_enter();
 				trace_user_enter(0);
 				vtime_user_enter(current);
 				instrumentation_end();
 			}
+
 			/*
 			 * Other than generic entry implementation, we may be past the last
 			 * rescheduling opportunity in the entry code. Trigger a self IPI
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index c6d86e4da8a9..a9f622fdeb12 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -323,8 +323,17 @@ static enum hrtimer_restart tick_nohz_handler(struct hrtimer *timer)
 	 * - to the idle task if in dynticks-idle
 	 * - to IRQ exit if in full-dynticks.
 	 */
-	if (unlikely(tick_sched_flag_test(ts, TS_FLAG_STOPPED)))
+	if (unlikely(tick_sched_flag_test(ts, TS_FLAG_STOPPED))) {
+		/*
+		 * In polling idle on nohz_full the idle loop doesn't iterate on
+		 * interrupts, so it would never reprogram the tick for the next
+		 * timer. Make cpu_idle_poll() return.
+		 */
+		if (tick_sched_flag_test(ts, TS_FLAG_INIDLE) &&
+		    tick_nohz_full_cpu(smp_processor_id()))
+			set_tsk_need_resched(current);
 		return HRTIMER_NORESTART;
+	}
 
 	hrtimer_forward(timer, now, TICK_NSEC);
 
@@ -414,6 +423,12 @@ static void nohz_full_kick_func(struct irq_work *work)
 		if (WARN_ON_ONCE(tick_sched_flag_test(ts, TS_FLAG_INIDLE)))
 			return;
 		tick_nohz_full_update_tick(ts);
+	} else if (tick_sched_flag_test(ts, TS_FLAG_INIDLE)) {
+		/*
+		 * In idle, tick_nohz_irq_exit() doesn't re-evaluate the tick, and
+		 * cpu_idle_poll() only returns on need_resched().
+		 */
+		set_tsk_need_resched(current);
 	}
 }
 
@@ -1071,6 +1086,30 @@ static void tick_nohz_full_update_tick(struct tick_sched *ts)
 	__tick_nohz_full_update_tick(ts, ktime_get());
 }
 
+#ifdef CONFIG_NO_HZ_FULL
+void __tick_nohz_user_enter(void)
+{
+	struct tick_sched *ts;
+	int cpu;
+
+	lockdep_assert_irqs_disabled();
+
+	cpu = smp_processor_id();
+	if (!tick_nohz_full_cpu(cpu))
+		return;
+
+	ts = this_cpu_ptr(&tick_cpu_sched);
+	if (!tick_sched_flag_test(ts, TS_FLAG_NOHZ))
+		return;
+
+	if (tick_sched_flag_test(ts, TS_FLAG_STOPPED))
+		return;
+
+	if (can_stop_full_tick(cpu, ts))
+		tick_nohz_full_stop_tick(ts, cpu);
+}
+#endif
+
 /*
  * A pending softirq outside an IRQ (or softirq disabled section) context
  * should be waiting for ksoftirqd to handle it. Therefore we shouldn't
-- 
2.56.0