testpi-2/1138[CPU#1]: BUG in FREE_WAITER at kernel/rt.c:1368
 [<c011a180>] __WARN_ON+0x60/0x80 (8)
 [<c037c9be>] __down_mutex+0x4fe/0x7fd (48)
 [<c01330d9>] pi_setprio+0xa1/0x642 (104)
 [<c0122196>] lock_timer_base+0x19/0x33 (8)
 [<c037edbd>] _spin_lock_irqsave+0x1d/0x46 (12)
 [<c0122196>] lock_timer_base+0x19/0x33 (8)
 [<c0122196>] lock_timer_base+0x19/0x33 (16)
 [<c01221e8>] __mod_timer+0x38/0xdf (16)
 [<c013aa7b>] sub_preempt_count+0x1a/0x1e (12)
 [<c037e71d>] __down_interruptible+0x923/0xb2b (20)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (16)
 [<c037f3f4>] _raw_spin_unlock+0x12/0x2c (8)
 [<c01344d6>] process_timeout+0x0/0x9 (36)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (64)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (8)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (12)
 [<c0137baa>] down_futex+0x70/0x99 (12)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (12)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (20)
 [<c013a91e>] add_preempt_count_ti+0x1e/0xc6 (100)
 [<c037f3f4>] _raw_spin_unlock+0x12/0x2c (44)
 [<c013c98e>] do_futex+0x96/0xfc (24)
 [<c013caec>] sys_futex+0xf8/0x104 (40)
 [<c0102ce7>] sysenter_past_esp+0x54/0x75 (60)
---------------------------
| preempt count: 00000001 ]
| 1-level deep critical section nesting:
----------------------------------------
.. [<c013a9e0>] .... add_preempt_count+0x1a/0x1e
.....[<00000000>] ..   ( <= stext+0x3feffd68/0x8)

------------------------------
| showing all locks held by: |  (testpi-2/1138 [f6d6c7b0,  69]):
------------------------------

#001:             [c3078fe0] {&base->t_base.lock}
... acquired at:               lock_timer_base+0x19/0x33

testpi-2/1140[CPU#3]: BUG in FREE_WAITER at kernel/rt.c:1368
 [<c011a180>] __WARN_ON+0x60/0x80 (8)
 [<c037c9be>] __down_mutex+0x4fe/0x7fd (48)
 [<c01330d9>] pi_setprio+0xa1/0x642 (104)
 [<c0122196>] lock_timer_base+0x19/0x33 (8)
 [<c037edbd>] _spin_lock_irqsave+0x1d/0x46 (12)
 [<c0122196>] lock_timer_base+0x19/0x33 (8)
 [<c0122196>] lock_timer_base+0x19/0x33 (16)
 [<c01221e8>] __mod_timer+0x38/0xdf (16)
 [<c013aa7b>] sub_preempt_count+0x1a/0x1e (12)
 [<c037e71d>] __down_interruptible+0x923/0xb2b (20)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (16)
 [<c037f3f4>] _raw_spin_unlock+0x12/0x2c (8)
 [<c01344d6>] process_timeout+0x0/0x9 (36)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (64)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (8)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (12)
 [<c0137baa>] down_futex+0x70/0x99 (12)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (12)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (20)
 [<c037f3f4>] _raw_spin_unlock+0x12/0x2c (76)
 [<c013c98e>] do_futex+0x96/0xfc (92)
 [<c013caec>] sys_futex+0xf8/0x104 (40)
 [<c0102ce7>] sysenter_past_esp+0x54/0x75 (60)
---------------------------
| preempt count: 00000001 ]
| 1-level deep critical section nesting:
----------------------------------------
.. [<c013a9e0>] .... add_preempt_count+0x1a/0x1e
.....[<00000000>] ..   ( <= stext+0x3feffd68/0x8)

------------------------------
| showing all locks held by: |  (testpi-2/1140 [f7d980a0,   0]):
------------------------------

#001:             [c3098fe0] {&base->t_base.lock}
... acquired at:               lock_timer_base+0x19/0x33

testpi-2/1139[CPU#0]: BUG in FREE_WAITER at kernel/rt.c:1368
NMI watchdog detected lockup on CPU#1 (5000/5000)
 [<c011a180>] __WARN_ON+0x60/0x80 (8)
 [<c037c9be>] __down_mutex+0x4fe/0x7fd (48)
 [<c013a91e>] add_preempt_count_ti+0x1e/0xc6 (104)
 [<c0122196>] lock_timer_base+0x19/0x33 (8)
 [<c037edbd>] _spin_lock_irqsave+0x1d/0x46 (12)
 [<c0122196>] lock_timer_base+0x19/0x33 (8)
 [<c0122196>] lock_timer_base+0x19/0x33 (16)
 [<c01221e8>] __mod_timer+0x38/0xdf (16)
 [<c013aa7b>] sub_preempt_count+0x1a/0x1e (12)
 [<c037e71d>] __down_interruptible+0x923/0xb2b (20)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (16)
 [<c037f3f4>] _raw_spin_unlock+0x12/0x2c (8)
 [<c01344d6>] process_timeout+0x0/0x9 (36)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (64)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (8)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (12)
 [<c0137baa>] down_futex+0x70/0x99 (12)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (12)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (20)
 [<c037f3f4>] _raw_spin_unlock+0x12/0x2c (76)
 [<c037ec7d>] __lock_text_start+0x1d/0x41 (24)
 [<c013c98e>] do_futex+0x96/0xfc (68)
 [<c013caec>] sys_futex+0xf8/0x104 (40)
 [<c0102ce7>] sysenter_past_esp+0x54/0x75 (60)
---------------------------
| preempt count: 00000001 ]
| 1-level deep critical section nesting:
----------------------------------------
.. [<c013a9e0>] .... add_preempt_count+0x1a/0x1e
.....[<00000000>] ..   ( <= stext+0x3feffd68/0x8)

------------------------------
| showing all locks held by: |  (testpi-2/1139 [f6c347b0,  59]):
------------------------------

#00           [c3068fe01:  ] {&base->t_base.lock}
..[. acquired at:               lock_timer_base+0x19/0x33

testpi-2/1139[CPU#2]: BUG in FREE_WAITER at kernel/rt.c:1368
 [<c011a180>] __WARN_ON+0x60/0x80 (8)
 [<c037c9be>] __down_mutex+0x4fe/0x7fd (48)
 [<c037f3f4>] _raw_spin_unlock+0x12/0x2c (100)
 [<c0122238>] __mod_timer+0x88/0xdf (12)
 [<c037ec7d>] __lock_text_start+0x1d/0x41 (12)
 [<c0122238>] __mod_timer+0x88/0xdf (8)
 [<c0122238>] __mod_timer+0x88/0xdf (16)
 [<c013aa7b>] sub_preempt_count+0x1a/0x1e (12)
 [<c037e71d>] __down_interruptible+0x923/0xb2b (20)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (16)
 [<c037f3f4>] _raw_spin_unlock+0x12/0x2c (8)
 [<c01344d6>] process_timeout+0x0/0x9 (36)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (64)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (8)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (12)
 [<c0137baa>] down_futex+0x70/0x99 (12)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (12)
 [<c013c1ed>] futex_wait_robust+0x151/0x25b (20)
 [<c037f3f4>] _raw_spin_unlock+0x12/0x2c (76)
 [<c037ec7d>] __lock_text_start+0x1d/0x41 (24)
 [<c013c98e>] do_futex+0x96/0xfc (68)
 [<c013caec>] sys_futex+0xf8/0x104 (40)
 [<c0102ce7>] sysenter_past_esp+0x54/0x75 (60)
---------------------------
| preempt count: 00000001 ]
| 1-level deep critical section nesting:
----------------------------------------
.. [<c013a9e0>] .... add_preempt_count+0x1a/0x1e
.....[<00000000>] ..   ( <= stext+0x3feffd68/0x8)

------------------------------
| showing all locks held by: |  (testpi-2/1139 [f6c347b0,  59]):
------------------------------

#001:             [c3088fe0] {&base->t_base.lock}
... acquired at:               __mod_timer+0x88/0xdf


Pid: 14, comm:      softirq-timer/1
EIP: 0060:[<c037f3af>] CPU: 1
EIP is at _raw_spin_lock+0x19/0x25
 EFLAGS: 00000086    Not tainted  (2.6.14-ray-rt1)
EAX: f7f26830 EBX: c03e4e0c ECX: 00000001 EDX: f7f26830
ESI: c3078fe0 EDI: 00000002 EBP: f7f26830 DS: 007b ES: 007b
CR0: 8005003b CR2: b5fb8dcc CR3: 35d67000 CR4: 000006d0
 [<c037c862>] __down_mutex+0x3a2/0x7fd (12)
 [<c037f3f4>] _raw_spin_unlock+0x12/0x2c (12)
 [<c0113828>] load_balance_newidle+0xf1/0x131 (8)
 [<c013aa7b>] sub_preempt_count+0x1a/0x1e (48)
 [<c01319a6>] ktimer_run_queues+0x27/0x131 (24)
 [<c0114551>] __wake_up+0x44/0x69 (8)
 [<c0122b8d>] run_timer_softirq+0x10b/0x3d6 (12)
 [<c037ecff>] _spin_lock_irq+0x1d/0x41 (12)
 [<c0122b8d>] run_timer_softirq+0x10b/0x3d6 (8)
 [<c0122b8d>] run_timer_softirq+0x10b/0x3d6 (16)
 [<c013a91e>] add_preempt_count_ti+0x1e/0xc6 (16)
 [<c011f0e4>] ksoftirqd+0xe7/0x183 (40)
 [<c011effd>] ksoftirqd+0x0/0x183 (32)
 [<c012d811>] kthread+0xa3/0xcd (4)
 [<c012d76e>] kthread+0x0/0xcd (28)
 [<c0100f75>] kernel_thread_helper+0x5/0xb (16)
---------------------------
| preempt count: 00000002 ]
| 2-level deep critical section nesting:
----------------------------------------
.. [<c013a9e0>] .... add_preempt_count+0x1a/0x1e
.....[<00000000>] ..   ( <= stext+0x3feffd68/0x8)
.. [<c013a9e0>] .... add_preempt_count+0x1a/0x1e
.....[<00000000>] ..   ( <= stext+0x3feffd68/0x8)

------------------------------
| showing all locks held by: |  (softirq-timer/1/14 [f7f26830,  98]):
------------------------------