From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S933843AbcI3PTT (ORCPT ); Fri, 30 Sep 2016 11:19:19 -0400 Received: from mail-pf0-f194.google.com ([209.85.192.194]:34537 "EHLO mail-pf0-f194.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S933789AbcI3PTR (ORCPT ); Fri, 30 Sep 2016 11:19:17 -0400 From: Sergey Senozhatsky To: Petr Mladek , Jan Kara Cc: Andrew Morton , Tejun Heo , Calvin Owens , Thomas Gleixner , Mel Gorman , Steven Rostedt , linux-kernel@vger.kernel.org, Sergey Senozhatsky , Sergey Senozhatsky Subject: [RFC][PATCHv2 3/7] printk: introduce per-cpu alt_print seq buffer Date: Sat, 1 Oct 2016 00:17:54 +0900 Message-Id: <20160930151758.8965-4-sergey.senozhatsky@gmail.com> X-Mailer: git-send-email 2.10.0.372.g6fe1b14 In-Reply-To: <20160930151758.8965-1-sergey.senozhatsky@gmail.com> References: <20160930151758.8965-1-sergey.senozhatsky@gmail.com> Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org This patch extends the idea of NMI per-cpu buffers to regions that may cause recursive printk() calls and possible deadlocks. Namely, printk() can't handle printk calls from schedule code or printk() calls from lock debugging code (spin_dump() for instance); because those may be called with `sem->lock' already taken or any other `critical' locks (p->pi_lock, etc.). An example of deadlock can be vprintk_emit() console_unlock() up() << raw_spin_lock_irqsave(&sem->lock, flags); wake_up_process() try_to_wake_up() ttwu_queue() ttwu_activate() activate_task() enqueue_task() enqueue_task_fair() cfs_rq_of() task_of() WARN_ON_ONCE(!entity_is_task(se)) vprintk_emit() console_trylock() down_trylock() raw_spin_lock_irqsave(&sem->lock, flags) ^^^^ deadlock and some other cases. Just like in NMI implementation, the solution uses a per-cpu `printk_func' pointer to 'redirect' printk() calls to a 'safe' callback, that store messages in a per-cpu buffer and flushes them back to logbuf buffer later. Usage example: printk() local_irq_save() alt_printk_enter() // // any printk() call from here will endup in vprintk_alt(), // that stores messages in a special per-CPU buffer. // alt_printk_exit() local_irq_restore() The 'redirection' mechanism, though, has been reworked, as suggested by Petr Mladek. Instead of using a per-cpu @print_func callback we now keep a per-cpu printk-context variable and call either default or nmi vprintk function depending on its value. printk_nmi_entrer/exit and alt_printk_enter/exit, thus, just set/celar corresponding bits in printk-context functions. The patch only adds alt_printk support, we don't use it yet. Signed-off-by: Sergey Senozhatsky --- kernel/printk/alt_printk.c | 117 +++++++++++++++++++++++++++++++++++---------- kernel/printk/internal.h | 43 ++++++++--------- kernel/printk/printk.c | 3 -- 3 files changed, 111 insertions(+), 52 deletions(-) diff --git a/kernel/printk/alt_printk.c b/kernel/printk/alt_printk.c index 7178661..4bc1e7d 100644 --- a/kernel/printk/alt_printk.c +++ b/kernel/printk/alt_printk.c @@ -1,5 +1,5 @@ /* - * alt_printk.c - Safe printk in NMI context + * alt_printk.c - Safe printk for printk-deadlock-prone contexts * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -32,13 +32,13 @@ * is later flushed into the main ring buffer via IRQ work. * * The alternative implementation is chosen transparently - * via @printk_func per-CPU variable. + * by examinig current printk() context mask stored in @alt_printk_ctx + * per-CPU variable. * * The implementation allows to flush the strings also from another CPU. * There are situations when we want to make sure that all buffers * were handled or when IRQs are blocked. */ -DEFINE_PER_CPU(printk_func_t, printk_func) = vprintk_default; static int alt_printk_irq_ready; atomic_t nmi_message_lost; @@ -52,25 +52,21 @@ struct alt_printk_seq_buf { }; static DEFINE_PER_CPU(struct alt_printk_seq_buf, nmi_print_seq); -/* - * Safe printk() for NMI context. It uses a per-CPU buffer to - * store the message. NMIs are not nested, so there is always only - * one writer running. But the buffer might get flushed from another - * CPU, so we need to be careful. - */ -static int vprintk_nmi(const char *fmt, va_list args) +static DEFINE_PER_CPU(struct alt_printk_seq_buf, alt_print_seq); +static DEFINE_PER_CPU(int, alt_printk_ctx); +static DEFINE_PER_CPU(unsigned long, alt_printk_irq_flags); + +static int alt_printk_log_store(struct alt_printk_seq_buf *s, + const char *fmt, va_list args) { - struct alt_printk_seq_buf *s = this_cpu_ptr(&nmi_print_seq); - int add = 0; + int add; size_t len; again: len = atomic_read(&s->len); - if (len >= sizeof(s->buffer)) { - atomic_inc(&nmi_message_lost); + if (len >= sizeof(s->buffer)) return 0; - } /* * Make sure that all old data have been read before the buffer was @@ -110,7 +106,6 @@ static void alt_printk_flush_line(const char *text, int len) printk_deferred("%.*s", len, text); else printk("%.*s", len, text); - } /* @@ -240,6 +235,83 @@ void alt_printk_flush_on_panic(void) alt_printk_flush(); } +/* + * Safe printk() for NMI context. It uses a per-CPU buffer to + * store the message. NMIs are not nested, so there is always only + * one writer running. But the buffer might get flushed from another + * CPU, so we need to be careful. + */ +static int vprintk_nmi(const char *fmt, va_list args) +{ + struct alt_printk_seq_buf *s = this_cpu_ptr(&nmi_print_seq); + int add; + + add = alt_printk_log_store(s, fmt, args); + if (!add) + atomic_inc(&nmi_message_lost); + + return add; +} + +void printk_nmi_enter(void) +{ + this_cpu_or(alt_printk_ctx, ALT_PRINTK_NMI_CONTEXT_MASK); +} + +void printk_nmi_exit(void) +{ + this_cpu_and(alt_printk_ctx, ~ALT_PRINTK_NMI_CONTEXT_MASK); +} + +/* + * Lockless printk(), to avoid deadlocks should the printk() recurse + * into itself. It uses a per-CPU buffer to store the message, just like + * NMI. + */ +static int vprintk_alt(const char *fmt, va_list args) +{ + struct alt_printk_seq_buf *s = this_cpu_ptr(&alt_print_seq); + + return alt_printk_log_store(s, fmt, args); +} + +/* + * Returns with local IRQs disabled. + * Can be preempted by NMI. + */ +void alt_printk_enter(void) +{ + unsigned long flags; + int entry_count; + + local_irq_save(flags); + if (!(this_cpu_read(alt_printk_ctx) & ALT_PRINTK_CONTEXT_MASK)) + this_cpu_write(alt_printk_irq_flags, flags); + this_cpu_inc(alt_printk_ctx); +} + +/* + * Restores local IRQs state saved in alt_printk_enter(). + * Can be preempted by NMI. + */ +void alt_printk_exit(void) +{ + this_cpu_dec(alt_printk_ctx); + if (!(this_cpu_read(alt_printk_ctx) & ALT_PRINTK_CONTEXT_MASK)) + local_irq_restore(this_cpu_read(alt_printk_irq_flags)); +} + +__printf(1, 0) int vprintk_func(const char *fmt, va_list args) +{ + if (this_cpu_read(alt_printk_ctx) & ALT_PRINTK_NMI_CONTEXT_MASK) + return vprintk_nmi(fmt, args); + + if (this_cpu_read(alt_printk_ctx) & ALT_PRINTK_CONTEXT_MASK) + return vprintk_alt(fmt, args); + + return vprintk_default(fmt, args); +} + void __init alt_printk_init(void) { int cpu; @@ -248,6 +320,9 @@ void __init alt_printk_init(void) struct alt_printk_seq_buf *s = &per_cpu(nmi_print_seq, cpu); init_irq_work(&s->work, __alt_printk_flush); + + s = &per_cpu(alt_print_seq, cpu); + init_irq_work(&s->work, __alt_printk_flush); } /* Make sure that IRQ works are initialized before enabling. */ @@ -257,13 +332,3 @@ void __init alt_printk_init(void) /* Flush pending messages that did not have scheduled IRQ works. */ alt_printk_flush(); } - -void printk_nmi_enter(void) -{ - this_cpu_write(printk_func, vprintk_nmi); -} - -void printk_nmi_exit(void) -{ - this_cpu_write(printk_func, vprintk_default); -} diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 7fd2838..6338f5b 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -16,26 +16,8 @@ */ #include -typedef __printf(1, 0) int (*printk_func_t)(const char *fmt, va_list args); - -int __printf(1, 0) vprintk_default(const char *fmt, va_list args); - #ifdef CONFIG_PRINTK_NMI -extern raw_spinlock_t logbuf_lock; - -/* - * printk() could not take logbuf_lock in NMI context. Instead, - * it temporary stores the strings into a per-CPU buffer. - * The alternative implementation is chosen transparently - * via per-CPU variable. - */ -DECLARE_PER_CPU(printk_func_t, printk_func); -static inline __printf(1, 0) int vprintk_func(const char *fmt, va_list args) -{ - return this_cpu_read(printk_func)(fmt, args); -} - extern atomic_t nmi_message_lost; static inline int get_nmi_message_lost(void) { @@ -44,14 +26,29 @@ static inline int get_nmi_message_lost(void) #else /* CONFIG_PRINTK_NMI */ -static inline __printf(1, 0) int vprintk_func(const char *fmt, va_list args) -{ - return vprintk_default(fmt, args); -} - static inline int get_nmi_message_lost(void) { return 0; } #endif /* CONFIG_PRINTK_NMI */ + +#ifdef CONFIG_PRINTK + +#define ALT_PRINTK_CONTEXT_MASK 0x07ffffff +#define ALT_PRINTK_NMI_CONTEXT_MASK 0x08000000 + +extern raw_spinlock_t logbuf_lock; + +__printf(1, 0) int vprintk_default(const char *fmt, va_list args); +__printf(1, 0) int vprintk_func(const char *fmt, va_list args); +void alt_printk_enter(void); +void alt_printk_exit(void); + +#else + +__printf(1, 0) int vprintk_func(const char *fmt, va_list args) { return 0; } +void alt_printk_enter(void) { } +void alt_printk_exit(void) { } + +#endif /* CONFIG_PRINTK */ diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index c7ffbef..ac3d10e 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -2029,9 +2029,6 @@ static size_t msg_print_text(const struct printk_log *msg, enum log_flags prev, static size_t cont_print_text(char *text, size_t size) { return 0; } static bool suppress_message_printing(int level) { return false; } -/* Still needs to be defined for users */ -DEFINE_PER_CPU(printk_func_t, printk_func); - #endif /* CONFIG_PRINTK */ #ifdef CONFIG_EARLY_PRINTK -- 2.10.0.372.g6fe1b14