mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
To: linux-kernel@vger.kernel.org
Cc: mingo@kernel.org, jiangshanlai@gmail.com, dipankar@in.ibm.com,
	akpm@linux-foundation.org, mathieu.desnoyers@efficios.com,
	josh@joshtriplett.org, tglx@linutronix.de, peterz@infradead.org,
	rostedt@goodmis.org, dhowells@redhat.com, edumazet@google.com,
	fweisbec@gmail.com, oleg@redhat.com, joel@joelfernandes.org,
	"Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Subject: [PATCH tip/core/rcu 01/40] rcu: Introduce grace-period sequence numbers
Date: Mon, 25 Jun 2018 17:08:20 -0700	[thread overview]
Message-ID: <20180626000859.23572-1-paulmck@linux.vnet.ibm.com> (raw)
In-Reply-To: <20180626000841.GA15696@linux.vnet.ibm.com>

This commit adds grace-period sequence numbers (->gp_seq) to the
rcu_state, rcu_node, and rcu_data structures, and updates them.
It also checks for consistency between rsp->gpnum and rsp->gp_seq.
These ->gp_seq counters will eventually replace the existing ->gpnum
and ->completed counters, allowing a single memory access to determine
whether or not a grace period is in progress and if so, which one.
This in turn will enable changes that will reduce ->lock contention on
the leaf rcu_node structures.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcu/tree.c | 16 +++++++++++++++-
 kernel/rcu/tree.h |  3 +++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 2db3475c2924..c50bb88dcab0 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -97,6 +97,7 @@ struct rcu_state sname##_state = { \
 	.gp_state = RCU_GP_IDLE, \
 	.gpnum = 0UL - 300UL, \
 	.completed = 0UL - 300UL, \
+	.gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT, \
 	.barrier_mutex = __MUTEX_INITIALIZER(sname##_state.barrier_mutex), \
 	.name = RCU_STATE_NAME(sname), \
 	.abbr = sabbr, \
@@ -1849,6 +1850,8 @@ static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp,
 		WRITE_ONCE(rdp->gpwrap, false);
 		rcu_gpnum_ovf(rnp, rdp);
 	}
+	if (rdp->gp_seq != rnp->gp_seq)
+		rdp->gp_seq = rnp->gp_seq;
 	return ret;
 }
 
@@ -1910,7 +1913,10 @@ static bool rcu_gp_init(struct rcu_state *rsp)
 	/* Advance to a new grace period and initialize state. */
 	record_gp_stall_check_time(rsp);
 	/* Record GP times before starting GP, hence smp_store_release(). */
+	WARN_ON_ONCE(rsp->gpnum << RCU_SEQ_CTR_SHIFT != rsp->gp_seq);
 	smp_store_release(&rsp->gpnum, rsp->gpnum + 1);
+	smp_mb(); /* Pairs with barriers in stall-warning code. */
+	rcu_seq_start(&rsp->gp_seq);
 	trace_rcu_grace_period(rsp->name, rsp->gpnum, TPS("start"));
 	raw_spin_unlock_irq_rcu_node(rnp);
 
@@ -1984,6 +1990,7 @@ static bool rcu_gp_init(struct rcu_state *rsp)
 		WRITE_ONCE(rnp->gpnum, rsp->gpnum);
 		if (WARN_ON_ONCE(rnp->completed != rsp->completed))
 			WRITE_ONCE(rnp->completed, rsp->completed);
+		WRITE_ONCE(rnp->gp_seq, rsp->gp_seq);
 		if (rnp == rdp->mynode)
 			(void)__note_gp_changes(rsp, rnp, rdp);
 		rcu_preempt_boost_start_gp(rnp);
@@ -2050,6 +2057,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
 {
 	unsigned long gp_duration;
 	bool needgp = false;
+	unsigned long new_gp_seq;
 	struct rcu_data *rdp;
 	struct rcu_node *rnp = rcu_get_root(rsp);
 	struct swait_queue_head *sq;
@@ -2079,12 +2087,15 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
 	 * all of the rcu_node structures before the beginning of the next
 	 * grace period is recorded in any of the rcu_node structures.
 	 */
+	new_gp_seq = rsp->gp_seq;
+	rcu_seq_end(&new_gp_seq);
 	rcu_for_each_node_breadth_first(rsp, rnp) {
 		raw_spin_lock_irq_rcu_node(rnp);
 		if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)))
 			dump_blkd_tasks(rnp, 10);
 		WARN_ON_ONCE(rnp->qsmask);
 		WRITE_ONCE(rnp->completed, rsp->gpnum);
+		WRITE_ONCE(rnp->gp_seq, new_gp_seq);
 		rdp = this_cpu_ptr(rsp->rda);
 		if (rnp == rdp->mynode)
 			needgp = __note_gp_changes(rsp, rnp, rdp) || needgp;
@@ -2098,10 +2109,11 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
 		rcu_gp_slow(rsp, gp_cleanup_delay);
 	}
 	rnp = rcu_get_root(rsp);
-	raw_spin_lock_irq_rcu_node(rnp); /* Order GP before ->completed update. */
+	raw_spin_lock_irq_rcu_node(rnp); /* GP before rsp->gp_seq update. */
 
 	/* Declare grace period done. */
 	WRITE_ONCE(rsp->completed, rsp->gpnum);
+	rcu_seq_end(&rsp->gp_seq);
 	trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end"));
 	rsp->gp_state = RCU_GP_IDLE;
 	/* Check for GP requests since above loop. */
@@ -3609,6 +3621,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
 	rdp->beenonline = true;	 /* We have now been online. */
 	rdp->gpnum = rnp->completed; /* Make CPU later note any new GP. */
 	rdp->completed = rnp->completed;
+	rdp->gp_seq = rnp->gp_seq;
 	rdp->cpu_no_qs.b.norm = true;
 	rdp->rcu_qs_ctr_snap = per_cpu(rcu_dynticks.rcu_qs_ctr, cpu);
 	rdp->core_needs_qs = false;
@@ -3988,6 +4001,7 @@ static void __init rcu_init_one(struct rcu_state *rsp)
 						   &rcu_fqs_class[i], fqs[i]);
 			rnp->gpnum = rsp->gpnum;
 			rnp->completed = rsp->completed;
+			rnp->gp_seq = rsp->gp_seq;
 			rnp->completedqs = rsp->completed;
 			rnp->qsmask = 0;
 			rnp->qsmaskinit = 0;
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 3c1942174c56..50a28d1cf5a1 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -87,6 +87,7 @@ struct rcu_node {
 	unsigned long completed; /* Last GP completed for this node. */
 				/*  This will either be equal to or one */
 				/*  behind the root rcu_node's gpnum. */
+	unsigned long gp_seq;	/* Track rsp->rcu_gp_seq. */
 	unsigned long completedqs; /* All QSes done for this node. */
 	unsigned long qsmask;	/* CPUs or groups that need to switch in */
 				/*  order for current grace period to proceed.*/
@@ -211,6 +212,7 @@ struct rcu_data {
 					/*  in order to detect GP end. */
 	unsigned long	gpnum;		/* Highest gp number that this CPU */
 					/*  is aware of having started. */
+	unsigned long	gp_seq;		/* Track rsp->rcu_gp_seq counter. */
 	unsigned long	rcu_qs_ctr_snap;/* Snapshot of rcu_qs_ctr to check */
 					/*  for rcu_all_qs() invocations. */
 	union rcu_noqs	cpu_no_qs;	/* No QSes yet for this CPU. */
@@ -343,6 +345,7 @@ struct rcu_state {
 						/* Subject to priority boost. */
 	unsigned long gpnum;			/* Current gp number. */
 	unsigned long completed;		/* # of last completed gp. */
+	unsigned long gp_seq;			/* Grace-period sequence #. */
 	struct task_struct *gp_kthread;		/* Task for grace periods. */
 	struct swait_queue_head gp_wq;		/* Where GP task waits. */
 	short gp_flags;				/* Commands for GP task. */
-- 
2.17.1


  reply	other threads:[~2018-06-26  0:07 UTC|newest]

Thread overview: 41+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2018-06-26  0:08 [PATCH tip/core/rcu 0/40] RCU ->gp_seq conversion for v4.19 Paul E. McKenney
2018-06-26  0:08 ` Paul E. McKenney [this message]
2018-06-26  0:08 ` [PATCH tip/core/rcu 02/40] rcu: Move rcu_gp_slow() to ->gp_seq Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 03/40] rcu: Make rcutorture's batches-completed API use ->gp_seq Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 04/40] rcu: Convert rcu_check_gp_kthread_starvation() to GP sequence number Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 05/40] rcu: Make quiescent-state reporting use ->gp_seq Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 06/40] rcu: Convert conditional grace-period primitives to ->gp_seq Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 07/40] rcu: Move RCU's grace-period-change code " Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 08/40] rcu: Convert rcu_gpnum_ovf() " Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 09/40] rcu: Move rcu_implicit_dynticks_qs() " Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 10/40] rcu: Move rcu_try_advance_all_cbs() " Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 11/40] rcu: Move rcu_nocb_gp_get() " Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 12/40] rcu: Move rcu_gp_in_progress() " Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 13/40] rcu: Convert ->rcu_iw_gpnum " Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 14/40] rcu: Convert ->completedqs " Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 15/40] rcu: Convert grace-period requests " Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 16/40] rcu: Make RCU CPU stall warnings use ->gp_seq Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 17/40] rcutorture: Convert rcutorture_get_gp_data() to ->gp_seq Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 18/40] rcu: Move from ->need_future_gp[] to ->gp_seq_needed Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 19/40] rcu: Make rcu_nocb_wait_gp() check if GP already requested Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 20/40] rcu: Convert rcu_grace_period tracepoint to gp_seq Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 21/40] rcu: Convert rcu_future_grace_period " Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 22/40] rcu: Convert rcu_grace_period_init " Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 23/40] rcu: Convert rcu_preempt_task tracepoint to ->gp_seq Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 24/40] rcu: Convert rcu_unlock_preempted_task " Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 25/40] rcu: Convert rcu_quiescent_state_report " Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 26/40] rcu: Convert rcu_fqs " Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 27/40] rcu: Remove ->gpnum and ->completed Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 28/40] rcu: Make simple callback acceleration refer to rdp->gp_seq_needed Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 29/40] doc: Update data-structure documentation for ->gp_seq Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 30/40] doc: Update memory-ordering documentation for ->gp-seq Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 31/40] doc: Update RCU CPU stall-warning documentation Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 32/40] rcu: Don't funnel-lock above leaf node if GP in progress Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 33/40] rcu: Produce last "CleanupMore" trace only if late-breaking request Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 34/40] rcu: Fix cpustart tracepoint gp_seq number Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 35/40] rcu: Make rcu_start_this_gp() check for grace period already started Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 36/40] rcutorture: Correctly handle grace-period sequence wrap Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 37/40] rcu: Regularize resetting of rcu_data wrap indicator Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 38/40] rcu: Rename the grace-period-request variables and parameters Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 39/40] rcu: Use better variable names in funnel locking loop Paul E. McKenney
2018-06-26  0:08 ` [PATCH tip/core/rcu 40/40] rcu: Identify grace period is in progress as we advance up the tree Paul E. McKenney

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20180626000859.23572-1-paulmck@linux.vnet.ibm.com \
    --to=paulmck@linux.vnet.ibm.com \
    --cc=akpm@linux-foundation.org \
    --cc=dhowells@redhat.com \
    --cc=dipankar@in.ibm.com \
    --cc=edumazet@google.com \
    --cc=fweisbec@gmail.com \
    --cc=jiangshanlai@gmail.com \
    --cc=joel@joelfernandes.org \
    --cc=josh@joshtriplett.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=mathieu.desnoyers@efficios.com \
    --cc=mingo@kernel.org \
    --cc=oleg@redhat.com \
    --cc=peterz@infradead.org \
    --cc=rostedt@goodmis.org \
    --cc=tglx@linutronix.de \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®