[RFC PATCH 6/6] sched/rt: Requeue p back if the preemption of check_preempt_equal_prio_common() failed

[Xunlei Pang <xlpang@126.com>]

From: Xunlei Pang <pang.xunlei@linaro.org>

In check_preempt_equal_prio_common(), it requeues "next" ahead
in the "run queue" and want to push current away. But when doing
the actual pushing, if the system state changes, the pushing may
fail as a result.

In this case, p finally becomes the new current and gets running,
while previous current was queued back waiting in the same "run
queue". This broke FIFO.

This patch adds a flag named RT_PREEMPT_PUSHAWAY for task_struct::
rt_preempt, sets it when doing check_preempt_equal_prio_common(),
and clears it if current is away(when it is dequeued). Thus we can
test this flag in p's post_schedule_rt() to judge if the pushing
has happened. If the pushing failed, requeue previous current back
to the head of its "run queue" and start a rescheduling.

Signed-off-by: Xunlei Pang <pang.xunlei@linaro.org>
---
 kernel/sched/rt.c | 75 +++++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 67 insertions(+), 8 deletions(-)

diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index e7d66eb..94789f1 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -258,6 +258,8 @@ int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
 #ifdef CONFIG_SMP
 
 #define RT_PREEMPT_QUEUEAHEAD    1UL
+#define RT_PREEMPT_PUSHAWAY      2UL
+#define RT_PREEMPT_MASK          3UL
 
 /*
  * p(current) was preempted, and to be put ahead of
@@ -268,6 +270,22 @@ static inline bool rt_preempted(struct task_struct *p)
 	return !!(p->rt_preempt & RT_PREEMPT_QUEUEAHEAD);
 }
 
+static inline struct task_struct *rt_preempting_target(struct task_struct *p)
+{
+	return (struct task_struct *) (p->rt_preempt & ~RT_PREEMPT_MASK);
+}
+
+/*
+ * p(new current) is preempting and pushing previous current away.
+ */
+static inline bool rt_preempting(struct task_struct *p)
+{
+	if ((p->rt_preempt & RT_PREEMPT_PUSHAWAY) && rt_preempting_target(p))
+		return true;
+
+	return false;
+}
+
 static inline void clear_rt_preempt(struct task_struct *p)
 {
 	p->rt_preempt = 0;
@@ -375,13 +393,17 @@ static inline int has_pushable_tasks(struct rq *rq)
 	return !plist_head_empty(&rq->rt.pushable_tasks);
 }
 
-static inline void set_post_schedule(struct rq *rq)
+static inline void set_post_schedule(struct rq *rq, struct task_struct *p)
 {
-	/*
-	 * We detect this state here so that we can avoid taking the RQ
-	 * lock again later if there is no need to push
-	 */
-	rq->post_schedule = has_pushable_tasks(rq);
+	if (rt_preempting(p))
+		/* Forced post schedule */
+		rq->post_schedule = 1;
+	else
+		/*
+		 * We detect this state here so that we can avoid taking
+		 * the RQ lock again later if there is no need to push
+		 */
+		rq->post_schedule = has_pushable_tasks(rq);
 }
 
 static void
@@ -430,6 +452,11 @@ static inline bool rt_preempted(struct task_struct *p)
 	return false;
 }
 
+static inline bool rt_preempting(struct task_struct *p)
+{
+	return false;
+}
+
 static inline void clear_rt_preempt(struct task_struct *p)
 {
 }
@@ -472,7 +499,7 @@ static inline int pull_rt_task(struct rq *this_rq)
 	return 0;
 }
 
-static inline void set_post_schedule(struct rq *rq)
+static inline void set_post_schedule(struct rq *rq, struct task_struct *p)
 {
 }
 #endif /* CONFIG_SMP */
@@ -1330,6 +1357,7 @@ static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)
 	dequeue_rt_entity(rt_se);
 
 	dequeue_pushable_task(rq, p);
+	clear_rt_preempt(p);
 }
 
 /*
@@ -1468,6 +1496,11 @@ static void check_preempt_equal_prio_common(struct rq *rq)
 	 * to try and push current away.
 	 */
 	requeue_task_rt(rq, next, 1);
+
+	get_task_struct(curr);
+	curr->rt_preempt |= RT_PREEMPT_PUSHAWAY;
+	next->rt_preempt = (unsigned long) curr;
+	next->rt_preempt |= RT_PREEMPT_PUSHAWAY;
 	resched_curr_preempted_rt(rq);
 }
 
@@ -1590,7 +1623,7 @@ pick_next_task_rt(struct rq *rq, struct task_struct *prev)
 	/* The running task is never eligible for pushing */
 	dequeue_pushable_task(rq, p);
 
-	set_post_schedule(rq);
+	set_post_schedule(rq, p);
 
 	return p;
 }
@@ -2151,6 +2184,32 @@ skip:
 static void post_schedule_rt(struct rq *rq)
 {
 	push_rt_tasks(rq);
+
+	if (rt_preempting(current)) {
+		struct task_struct *target;
+
+		current->rt_preempt = 0;
+		target = rt_preempting_target(current);
+		if (!(target->rt_preempt & RT_PREEMPT_PUSHAWAY))
+			goto out;
+
+		/*
+		 * target still has RT_PREEMPT_PUSHAWAY set which
+		 * means it wasn't pushed away successfully if it
+		 * is still on this rq. thus restore former status
+		 * of current and target if so.
+		 */
+		if (!task_on_rq_queued(target) ||
+		    task_cpu(target) != rq->cpu)
+			goto out;
+
+		/* target is previous current, requeue it back ahead. */
+		requeue_task_rt(rq, target, 1);
+		/* Let's preempt current, loop back to __schedule(). */
+		resched_curr_preempted_rt(rq);
+out:
+		put_task_struct(target);
+	}
 }
 
 /*
-- 
1.9.1