From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1751456AbeAWIKJ (ORCPT ); Tue, 23 Jan 2018 03:10:09 -0500 Received: from szxga05-in.huawei.com ([45.249.212.191]:4717 "EHLO huawei.com" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1751178AbeAWIIw (ORCPT ); Tue, 23 Jan 2018 03:08:52 -0500 From: To: CC: , , , , Subject: [PATCH RFC 08/16] prcu: Implement PRCU callback processing Date: Tue, 23 Jan 2018 15:59:33 +0800 Message-ID: <1516694381-20333-9-git-send-email-lianglihao@huawei.com> X-Mailer: git-send-email 1.7.12.4 In-Reply-To: <1516694381-20333-1-git-send-email-lianglihao@huawei.com> References: <1516694381-20333-1-git-send-email-lianglihao@huawei.com> MIME-Version: 1.0 Content-Type: text/plain X-Originating-IP: [10.175.102.37] X-CFilter-Loop: Reflected Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org From: Lihao Liang Currently, PRCU core processing only consists of callback processing in prcu_process_callbacks(), which is triggered by the scheduling-clock interrupt. Reviewed-by: Heng Zhang Signed-off-by: Lihao Liang --- include/linux/interrupt.h | 3 ++ include/linux/prcu.h | 8 +++++ kernel/rcu/prcu.c | 86 +++++++++++++++++++++++++++++++++++++++++++++++ kernel/rcu/tree.c | 1 + kernel/time/timer.c | 2 ++ 5 files changed, 100 insertions(+) diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index 0991f973..f05ef62a 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -456,6 +456,9 @@ enum SCHED_SOFTIRQ, HRTIMER_SOFTIRQ, /* Unused, but kept as tools rely on the numbering. Sigh! */ +#ifdef CONFIG_PRCU + PRCU_SOFTIRQ, +#endif RCU_SOFTIRQ, /* Preferable RCU should always be the last softirq */ NR_SOFTIRQS diff --git a/include/linux/prcu.h b/include/linux/prcu.h index e5e09c9b..4e7d5d65 100644 --- a/include/linux/prcu.h +++ b/include/linux/prcu.h @@ -31,11 +31,13 @@ struct prcu_local_struct { unsigned int locked; unsigned int online; unsigned long long version; + unsigned long long cb_version; struct prcu_cblist cblist; }; struct prcu_struct { atomic64_t global_version; + atomic64_t cb_version; atomic_t active_ctr; struct mutex mtx; wait_queue_head_t wait_q; @@ -48,6 +50,9 @@ void synchronize_prcu(void); void call_prcu(struct rcu_head *head, rcu_callback_t func); void prcu_init(void); void prcu_note_context_switch(void); +int prcu_pending(void); +void invoke_prcu_core(void); +void prcu_check_callbacks(void); #else /* #ifdef CONFIG_PRCU */ @@ -57,6 +62,9 @@ void prcu_note_context_switch(void); #define call_prcu() do {} while (0) #define prcu_init() do {} while (0) #define prcu_note_context_switch() do {} while (0) +#define prcu_pending() 0 +#define invoke_prcu_core() do {} while (0) +#define prcu_check_callbacks() do {} while (0) #endif /* #ifdef CONFIG_PRCU */ #endif /* __LINUX_PRCU_H */ diff --git a/kernel/rcu/prcu.c b/kernel/rcu/prcu.c index f198285c..373039c5 100644 --- a/kernel/rcu/prcu.c +++ b/kernel/rcu/prcu.c @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -11,6 +12,7 @@ DEFINE_PER_CPU_SHARED_ALIGNED(struct prcu_local_struct, prcu_local); struct prcu_struct global_prcu = { .global_version = ATOMIC64_INIT(0), + .cb_version = ATOMIC64_INIT(0), .active_ctr = ATOMIC_INIT(0), .mtx = __MUTEX_INITIALIZER(global_prcu.mtx), .wait_q = __WAIT_QUEUE_HEAD_INITIALIZER(global_prcu.wait_q) @@ -27,6 +29,35 @@ static void prcu_cblist_init(struct prcu_cblist *rclp) rclp->len = 0; } +/* + * Dequeue the oldest rcu_head structure from the specified callback list; + * store the callback grace period version number into the version pointer. + */ +static struct rcu_head *prcu_cblist_dequeue(struct prcu_cblist *rclp) +{ + struct rcu_head *rhp; + struct prcu_version_head *vhp; + + rhp = rclp->head; + if (!rhp) { + WARN_ON(vhp); + WARN_ON(rclp->len); + return NULL; + } + + vhp = rclp->version_head; + rclp->version_head = vhp->next; + rclp->head = rhp->next; + rclp->len--; + + if (!rclp->head) { + rclp->tail = &rclp->head; + rclp->version_tail = &rclp->version_head; + } + + return rhp; +} + static inline void prcu_report(struct prcu_local_struct *local) { unsigned long long global_version; @@ -117,6 +148,7 @@ void synchronize_prcu(void) if (atomic_read(&prcu->active_ctr)) wait_event(prcu->wait_q, !atomic_read(&prcu->active_ctr)); + atomic64_set(&prcu->cb_version, version); mutex_unlock(&prcu->mtx); } EXPORT_SYMBOL(synchronize_prcu); @@ -166,6 +198,58 @@ void call_prcu(struct rcu_head *head, rcu_callback_t func) } EXPORT_SYMBOL(call_prcu); +int prcu_pending(void) +{ + struct prcu_local_struct *local = get_cpu_ptr(&prcu_local); + unsigned long long cb_version = local->cb_version; + struct prcu_cblist *rclp = &local->cblist; + + put_cpu_ptr(&prcu_local); + return cb_version < atomic64_read(&prcu->cb_version) && rclp->head; +} + +void invoke_prcu_core(void) +{ + if (cpu_online(smp_processor_id())) + raise_softirq(PRCU_SOFTIRQ); +} + +void prcu_check_callbacks(void) +{ + if (prcu_pending()) + invoke_prcu_core(); +} + +static __latent_entropy void prcu_process_callbacks(struct softirq_action *unused) +{ + unsigned long flags; + unsigned long long cb_version; + struct prcu_local_struct *local; + struct prcu_cblist *rclp; + struct rcu_head *rhp; + struct prcu_version_head *vhp; + + if (cpu_is_offline(smp_processor_id())) + return; + + cb_version = atomic64_read(&prcu->cb_version); + + /* Disable interrupts to prevent races with call_prcu() */ + local_irq_save(flags); + local = this_cpu_ptr(&prcu_local); + rclp = &local->cblist; + rhp = rclp->head; + vhp = rclp->version_head; + for (; rhp && vhp && vhp->version < cb_version; + rhp = rclp->head, vhp = rclp->version_head) { + rhp = prcu_cblist_dequeue(rclp); + debug_rcu_head_unqueue(rhp); + rhp->func(rhp); + } + local->cb_version = cb_version; + local_irq_restore(flags); +} + void prcu_init_local_struct(int cpu) { struct prcu_local_struct *local; @@ -174,6 +258,7 @@ void prcu_init_local_struct(int cpu) local->locked = 0; local->online = 0; local->version = 0; + local->cb_version = 0; prcu_cblist_init(&local->cblist); } @@ -181,6 +266,7 @@ void __init prcu_init(void) { int cpu; + open_softirq(PRCU_SOFTIRQ, prcu_process_callbacks); for_each_possible_cpu(cpu) prcu_init_local_struct(cpu); } diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index e354e475..46910114 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2852,6 +2852,7 @@ void rcu_check_callbacks(int user) { trace_rcu_utilization(TPS("Start scheduler-tick")); increment_cpu_stall_ticks(); + if (user || rcu_is_cpu_rrupt_from_idle()) { /* diff --git a/kernel/time/timer.c b/kernel/time/timer.c index d3f33020..ed863e63 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include @@ -1568,6 +1569,7 @@ void update_process_times(int user_tick) /* Note: this timer irq context must be accounted for as well. */ account_process_tick(p, user_tick); run_local_timers(); + prcu_check_callbacks(); rcu_check_callbacks(user_tick); #ifdef CONFIG_IRQ_WORK if (in_irq()) -- 2.14.1.729.g59c0ea183