[PATCH v2 3/6] hrtimer,sched: Add fuzzy hrtimer mode for HRTICK

[Peter Zijlstra <peterz@infradead.org>]

Upon schedule() HRTICK will cancel the current timer, pick the next
task and reprogram the timer. When schedule() consistently triggers
due to blocking conditions instead of the timer, this leads to endless
reprogramming without ever firing.

Mitigate this with a new hrtimer mode: fuzzy (not really happy with
that name); this mode does two things:

 - skip reprogramming the hardware on timer remove;
 - skip reprogramming the hardware when the new timer
   is after cpu_base->expires_next

Both things are already possible;

 - removing a remote timer will leave the hardware programmed and
   cause a spurious interrupt.
 - this remote CPU adding a timer can skip the reprogramming
   when the timer's expiration is after the (spurious) expiration.

This new timer mode simply causes more of this 'fuzzy' behaviour; it
causes a few spurious interrupts, but similarly avoids endlessly
reprogramming the timer.

This makes the HRTICK match the NO_HRTICK hackbench runs -- the case
where a task never runs until its slice is complete but always goes
sleep early.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 include/linux/hrtimer.h       |    1 +
 include/linux/hrtimer_types.h |    1 +
 kernel/sched/core.c           |    3 ++-
 kernel/time/hrtimer.c         |   16 +++++++++++++++-
 4 files changed, 19 insertions(+), 2 deletions(-)

--- a/include/linux/hrtimer.h
+++ b/include/linux/hrtimer.h
@@ -38,6 +38,7 @@ enum hrtimer_mode {
 	HRTIMER_MODE_PINNED	= 0x02,
 	HRTIMER_MODE_SOFT	= 0x04,
 	HRTIMER_MODE_HARD	= 0x08,
+	HRTIMER_MODE_FUZZY	= 0x10,
 
 	HRTIMER_MODE_ABS_PINNED = HRTIMER_MODE_ABS | HRTIMER_MODE_PINNED,
 	HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL | HRTIMER_MODE_PINNED,
--- a/include/linux/hrtimer_types.h
+++ b/include/linux/hrtimer_types.h
@@ -45,6 +45,7 @@ struct hrtimer {
 	u8				is_rel;
 	u8				is_soft;
 	u8				is_hard;
+	u8				is_fuzzy;
 };
 
 #endif /* _LINUX_HRTIMER_TYPES_H */
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -928,7 +928,8 @@ void hrtick_start(struct rq *rq, u64 del
 static void hrtick_rq_init(struct rq *rq)
 {
 	INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq);
-	hrtimer_setup(&rq->hrtick_timer, hrtick, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
+	hrtimer_setup(&rq->hrtick_timer, hrtick, CLOCK_MONOTONIC,
+		      HRTIMER_MODE_REL_HARD | HRTIMER_MODE_FUZZY);
 }
 #else /* !CONFIG_SCHED_HRTICK: */
 static inline void hrtick_clear(struct rq *rq)
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -1122,7 +1122,7 @@ static void __remove_hrtimer(struct hrti
 	 * an superfluous call to hrtimer_force_reprogram() on the
 	 * remote cpu later on if the same timer gets enqueued again.
 	 */
-	if (reprogram && timer == cpu_base->next_timer)
+	if (!timer->is_fuzzy && reprogram && timer == cpu_base->next_timer)
 		hrtimer_force_reprogram(cpu_base, 1);
 }
 
@@ -1269,6 +1269,19 @@ static int __hrtimer_start_range_ns(stru
 	if (new_base->cpu_base->in_hrtirq)
 		return 0;
 
+	if (timer->is_fuzzy) {
+		/*
+		 * XXX fuzzy implies pinned!  not sure how to deal with
+		 * retrigger_next_event() for the !local case.
+		 */
+		WARN_ON_ONCE(!(mode & HRTIMER_MODE_PINNED));
+		/*
+		 * Notably, by going into hrtimer_reprogram(), it will
+		 * not reprogram if cpu_base->expires_next is earlier.
+		 */
+		return first;
+	}
+
 	if (!force_local) {
 		/*
 		 * If the current CPU base is online, then the timer is
@@ -1645,6 +1658,7 @@ static void __hrtimer_setup(struct hrtim
 	base += hrtimer_clockid_to_base(clock_id);
 	timer->is_soft = softtimer;
 	timer->is_hard = !!(mode & HRTIMER_MODE_HARD);
+	timer->is_fuzzy = !!(mode & HRTIMER_MODE_FUZZY);
 	timer->base = &cpu_base->clock_base[base];
 	timerqueue_init(&timer->node);