* Re: [LKP] [lkp-robot] 9cf57731b6 [ 8.051016] WARNING: CPU: 1 PID: 58 at lib/list_debug.c:28 __list_add_valid [not found] ` <85eb9dd4-2a20-9c2a-d805-e38cf749f187@intel.com> @ 2018-07-13 10:42 ` Peter Zijlstra 2018-07-15 23:25 ` [tip:sched/core] watchdog/softlockup: Fix cpu_stop_queue_work() double-queue bug tip-bot for Peter Zijlstra 0 siblings, 1 reply; 2+ messages in thread From: Peter Zijlstra @ 2018-07-13 10:42 UTC (permalink / raw) To: Rong Chen; +Cc: kernel test robot, LKP, Ingo Molnar, linux-kernel On Fri, Jul 13, 2018 at 12:32:41PM +0800, Rong Chen wrote: > Thanks, it's fixed with the patch. Ingo, could you merge? --- Subject: watchdog/softlockup: Fix cpu_stop_queue_work double-queue From: Peter Zijlstra <peterz@infradead.org> Date: Wed, 11 Jul 2018 14:34:36 +0200 When scheduling is delayed for longer than the softlockup interrupt period it is possible to double-queue the cpu_stop_work, causing list corruption. Cure this by adding a completion to track the cpu_stop_work's progress. Reported-by: kernel test robot <lkp@intel.com> Tested-by: Rong Chen <rong.a.chen@intel.com> Fixes: 9cf57731b63e ("watchdog/softlockup: Replace "watchdog/%u" threads with cpu_stop_work") Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> --- kernel/watchdog.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/kernel/watchdog.c b/kernel/watchdog.c index b81f777838d5..5470dce212c0 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -330,6 +330,9 @@ static void watchdog_interrupt_count(void) __this_cpu_inc(hrtimer_interrupts); } +static DEFINE_PER_CPU(struct completion, softlockup_completion); +static DEFINE_PER_CPU(struct cpu_stop_work, softlockup_stop_work); + /* * The watchdog thread function - touches the timestamp. * @@ -343,12 +346,11 @@ static int softlockup_fn(void *data) __this_cpu_write(soft_lockup_hrtimer_cnt, __this_cpu_read(hrtimer_interrupts)); __touch_watchdog(); + complete(this_cpu_ptr(&softlockup_completion)); return 0; } -static DEFINE_PER_CPU(struct cpu_stop_work, softlockup_stop_work); - /* watchdog kicker functions */ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) { @@ -364,9 +366,12 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) watchdog_interrupt_count(); /* kick the softlockup detector */ - stop_one_cpu_nowait(smp_processor_id(), - softlockup_fn, NULL, - this_cpu_ptr(&softlockup_stop_work)); + if (completion_done(this_cpu_ptr(&softlockup_completion))) { + reinit_completion(this_cpu_ptr(&softlockup_completion)); + stop_one_cpu_nowait(smp_processor_id(), + softlockup_fn, NULL, + this_cpu_ptr(&softlockup_stop_work)); + } /* .. and repeat */ hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period)); @@ -467,9 +472,13 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) static void watchdog_enable(unsigned int cpu) { struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer); + struct completion *done = this_cpu_ptr(&softlockup_completion); WARN_ON_ONCE(cpu != smp_processor_id()); + init_completion(done); + complete(done); + /* * Start the timer first to prevent the NMI watchdog triggering * before the timer has a chance to fire. @@ -499,6 +508,7 @@ static void watchdog_disable(unsigned int cpu) */ watchdog_nmi_disable(cpu); hrtimer_cancel(hrtimer); + wait_for_completion(this_cpu_ptr(&softlockup_completion)); } static int softlockup_stop_fn(void *data) ^ permalink raw reply [flat|nested] 2+ messages in thread
* [tip:sched/core] watchdog/softlockup: Fix cpu_stop_queue_work() double-queue bug 2018-07-13 10:42 ` [LKP] [lkp-robot] 9cf57731b6 [ 8.051016] WARNING: CPU: 1 PID: 58 at lib/list_debug.c:28 __list_add_valid Peter Zijlstra @ 2018-07-15 23:25 ` tip-bot for Peter Zijlstra 0 siblings, 0 replies; 2+ messages in thread From: tip-bot for Peter Zijlstra @ 2018-07-15 23:25 UTC (permalink / raw) To: linux-tip-commits Cc: linux-kernel, rong.a.chen, mingo, torvalds, hpa, tglx, peterz, lkp Commit-ID: be45bf5395e0886a93fc816bbe41a008ec2e42e2 Gitweb: https://git.kernel.org/tip/be45bf5395e0886a93fc816bbe41a008ec2e42e2 Author: Peter Zijlstra <peterz@infradead.org> AuthorDate: Fri, 13 Jul 2018 12:42:08 +0200 Committer: Ingo Molnar <mingo@kernel.org> CommitDate: Sun, 15 Jul 2018 23:51:19 +0200 watchdog/softlockup: Fix cpu_stop_queue_work() double-queue bug When scheduling is delayed for longer than the softlockup interrupt period it is possible to double-queue the cpu_stop_work, causing list corruption. Cure this by adding a completion to track the cpu_stop_work's progress. Reported-by: kernel test robot <lkp@intel.com> Tested-by: Rong Chen <rong.a.chen@intel.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Fixes: 9cf57731b63e ("watchdog/softlockup: Replace "watchdog/%u" threads with cpu_stop_work") Link: http://lkml.kernel.org/r/20180713104208.GW2494@hirez.programming.kicks-ass.net Signed-off-by: Ingo Molnar <mingo@kernel.org> --- kernel/watchdog.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/kernel/watchdog.c b/kernel/watchdog.c index b81f777838d5..5470dce212c0 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -330,6 +330,9 @@ static void watchdog_interrupt_count(void) __this_cpu_inc(hrtimer_interrupts); } +static DEFINE_PER_CPU(struct completion, softlockup_completion); +static DEFINE_PER_CPU(struct cpu_stop_work, softlockup_stop_work); + /* * The watchdog thread function - touches the timestamp. * @@ -343,12 +346,11 @@ static int softlockup_fn(void *data) __this_cpu_write(soft_lockup_hrtimer_cnt, __this_cpu_read(hrtimer_interrupts)); __touch_watchdog(); + complete(this_cpu_ptr(&softlockup_completion)); return 0; } -static DEFINE_PER_CPU(struct cpu_stop_work, softlockup_stop_work); - /* watchdog kicker functions */ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) { @@ -364,9 +366,12 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) watchdog_interrupt_count(); /* kick the softlockup detector */ - stop_one_cpu_nowait(smp_processor_id(), - softlockup_fn, NULL, - this_cpu_ptr(&softlockup_stop_work)); + if (completion_done(this_cpu_ptr(&softlockup_completion))) { + reinit_completion(this_cpu_ptr(&softlockup_completion)); + stop_one_cpu_nowait(smp_processor_id(), + softlockup_fn, NULL, + this_cpu_ptr(&softlockup_stop_work)); + } /* .. and repeat */ hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period)); @@ -467,9 +472,13 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) static void watchdog_enable(unsigned int cpu) { struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer); + struct completion *done = this_cpu_ptr(&softlockup_completion); WARN_ON_ONCE(cpu != smp_processor_id()); + init_completion(done); + complete(done); + /* * Start the timer first to prevent the NMI watchdog triggering * before the timer has a chance to fire. @@ -499,6 +508,7 @@ static void watchdog_disable(unsigned int cpu) */ watchdog_nmi_disable(cpu); hrtimer_cancel(hrtimer); + wait_for_completion(this_cpu_ptr(&softlockup_completion)); } static int softlockup_stop_fn(void *data) ^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2018-07-15 23:26 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
[not found] <20180711060753.GC17385@shao2-debian>
[not found] ` <20180711123436.GD2476@hirez.programming.kicks-ass.net>
[not found] ` <85eb9dd4-2a20-9c2a-d805-e38cf749f187@intel.com>
2018-07-13 10:42 ` [LKP] [lkp-robot] 9cf57731b6 [ 8.051016] WARNING: CPU: 1 PID: 58 at lib/list_debug.c:28 __list_add_valid Peter Zijlstra
2018-07-15 23:25 ` [tip:sched/core] watchdog/softlockup: Fix cpu_stop_queue_work() double-queue bug tip-bot for Peter Zijlstra
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox
Powered by JetHome