Re: [BUG] cpu hotplug vs scheduler

["Dmitry Adamushko" <dmitry.adamushko@gmail.com>]

[-- Attachment #1: Type: text/plain, Size: 2720 bytes --]

Hi,

> [ ... ]
>
>  [4298303.713901] Call Trace:
>  [4298303.713901]  [<ffffffff804373fe>] schedule+0x414/0x6ab
>  [4298303.713901]  [<ffffffff8023060a>] ? hrtick_set+0x9d/0xe8
>  [4298303.713901]  [<ffffffff8043772f>] ? thread_return+0x9a/0xbf
>  [4298303.713901]  [<ffffffff80231652>] migration_thread+0x185/0x22d
>  [4298303.713901]  [<ffffffff802314cd>] ? migration_thread+0x0/0x22d
>  [4298303.713901]  [<ffffffff8024afe6>] kthread+0x49/0x77
>  [4298303.713901]  [<ffffffff8020d228>] child_rip+0xa/0x12
>  [4298303.713901]  [<ffffffff8024af9d>] ? kthread+0x0/0x77
>  [4298303.713901]  [<ffffffff8020d21e>] ? child_rip+0x0/0x12
>  [4298303.713901]
>  [4298303.713901]
>  [4298303.713901] Code: c0 74 28 48 8b 7b 58 4c 8d 60 f0 48 85 ff 74 10 4c
> 89 e6 e8 df cc ff ff 85 c0 75 04 4c 8b 63 58 4c 89 e6 48 89 df e8 4a e5 ff
> ff <49> 8b 9c 24 58 01 00 00 48 85 db 75 bf 49 83 ec 38 4c 89 ef 4c
>  [4298303.713901] RIP  [<ffffffff8022e722>] pick_next_task_fair+0x55/0x7c
>
>  This seems to be the assignment to cfs_rq after pick_next_entity().

[ cc'ed a few folks. ]


So the cfs-tree likely gets out-of-sync. I pressume, it won't be
reproducible with CONFIG_SCHED_GROUP options being disabled.

Anyway, would you try one of these debug-patches (not sure about the
workability of the second one though :-/)

Let's check what are the values for 'cfs_rq->weight.load/nr_running'.

thanks in advance,

(non-whitespace-damaged versions are enclosed)

---
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -1291,6 +1291,12 @@ static struct task_struct
*pick_next_task_fair(struct rq *rq)

        do {
                se = pick_next_entity(cfs_rq);
+
+               if (unlikely(!se))
+                       printk(KERN_ERR "BUG: se == NULL but
nr_running (%ld), load (%ld),"
+                                       " rq-nr_running (%ld), rq-load (%ld)\n",
+                               cfs_rq->nr_running,
cfs_rq->load.weight, rq->nr_running, rq->load.weight);
+
                cfs_rq = group_cfs_rq(se);
        } while (cfs_rq);


---

--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -1280,6 +1280,8 @@ static void check_preempt_wakeup(struct rq *rq,
struct task_str uct *p)
                resched_task(curr);
 }

+static void sysrq_sched_debug_show(void);
+
 static struct task_struct *pick_next_task_fair(struct rq *rq)
 {
        struct task_struct *p;
@@ -1291,6 +1293,10 @@ static struct task_struct
*pick_next_task_fair(struct rq *rq)

        do {
                se = pick_next_entity(cfs_rq);
+
+               if (unlikely(!se))
+                       sysrq_sched_debug_show();
+
                cfs_rq = group_cfs_rq(se);
        } while (cfs_rq);


---

-- 
Best regards,
Dmitry Adamushko

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: cfs_rq-debug-2.patch --]
[-- Type: text/x-patch; name=cfs_rq-debug-2.patch, Size: 630 bytes --]

diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index e24ecd3..1dcc470 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -1280,6 +1280,8 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p)
 		resched_task(curr);
 }
 
+static void sysrq_sched_debug_show(void);
+
 static struct task_struct *pick_next_task_fair(struct rq *rq)
 {
 	struct task_struct *p;
@@ -1291,6 +1293,10 @@ static struct task_struct *pick_next_task_fair(struct rq *rq)
 
 	do {
 		se = pick_next_entity(cfs_rq);
+
+		if (unlikely(!se))
+			sysrq_sched_debug_show();
+
 		cfs_rq = group_cfs_rq(se);
 	} while (cfs_rq);
 

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #3: cfs_rq-debug-3.patch --]
[-- Type: text/x-patch; name=cfs_rq-debug-3.patch, Size: 542 bytes --]

diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index e24ecd3..e21e020 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -1291,6 +1291,12 @@ static struct task_struct *pick_next_task_fair(struct rq *rq)
 
 	do {
 		se = pick_next_entity(cfs_rq);
+
+		if (unlikely(!se))
+			printk(KERN_ERR "BUG: se == NULL but nr_running (%ld), load (%ld),"
+					" rq-nr_running (%ld), rq-load (%ld)\n",
+				cfs_rq->nr_running, cfs_rq->load.weight, rq->nr_running, rq->load.weight);
+
 		cfs_rq = group_cfs_rq(se);
 	} while (cfs_rq);