[PATCH 5/5] RT - Track which CPUs should get IPI'd on rt-overload

[Gregory Haskins <ghaskins@novell.com>]

The code currently blindly fires IPIs out whenever an overload occurs.
However, there are strict events that govern when a rt-overload exists
(e.g. RT task added to a RQ, or an RT task preempted).  Therefore, we
attempt to efficiently track which CPUs are eligible for rebalancing, and we
only IPI those affected units.

Signed-off-by: Gregory Haskins <ghaskins@novell.com>
CC: Peter W. Morreale <pmorreale@novell.com>
---

 kernel/sched.c |   15 +++++++++++++--
 1 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/kernel/sched.c b/kernel/sched.c
index a28ca9d..6ca5f4f 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -359,6 +359,8 @@ struct rq {
 	unsigned long rto_pulled;
 #endif
 	struct lock_class_key rq_lock_key;
+
+	cpumask_t rto_resched; /* Which of our peers needs rescheduling */
 };
 
 static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
@@ -645,6 +647,9 @@ static inline void inc_rt_tasks(struct task_struct *p, struct rq *rq)
 			smp_wmb();
 			atomic_inc(&rt_overload);
 		}
+
+		cpus_or(rq->rto_resched, rq->rto_resched, p->cpus_allowed);
+		cpu_clear(rq->cpu, rq->rto_resched);
 # endif
 	}
 #endif
@@ -2213,9 +2218,15 @@ static inline void finish_task_switch(struct rq *rq, struct task_struct *prev)
 	 * If we pushed an RT task off the runqueue,
 	 * then kick other CPUs, they might run it:
 	 */
-	if (unlikely(rt_task(current) && rq->rt_nr_running > 1)) {
+	if (unlikely(rt_task(current) && prev->se.on_rq && rt_task(prev))) {
+		cpus_or(rq->rto_resched, rq->rto_resched, prev->cpus_allowed);
+		cpu_clear(rq->cpu, rq->rto_resched);
+	}
+
+	if (unlikely(rq->rt_nr_running > 1 && !cpus_empty(rq->rto_resched))) {
 		schedstat_inc(rq, rto_schedule);
-		smp_send_reschedule_allbutself();
+		smp_send_reschedule_allbutself_cpumask(rq->rto_resched);
+		cpus_clear(rq->rto_resched);
 	}
 #endif
 	prev_state = prev->state;