[PATCH v2 5/7] rcutorture: Run NMI-safe SRCU readers from NMI

[Mathieu Desnoyers <mathieu.desnoyers@efficios.com>]

rcutorture exercises ->call() from NMI, but never runs a reader there.
So nothing tests the NMI-safe reader flavors where it matters most: in
an NMI that interrupts another reader during its counter update.

Run a reader from the same per-CPU perf-counter NMI, when every reader
flavor in use is NMI-safe. These flavors are srcu_read_lock_nmisafe(),
srcu_read_lock_fast() and srcu_read_lock_atomic(). A new ops hook,
->nmi_reader_capable(), reports whether the reader flavors in use are
all NMI-safe. A new nmi_readers module parameter, on by default, enables
these NMI readers. The statistics count these readers as nmi-reads.

The reader loads rcu_torture_current and checks its element as the
other readers do. The element must not be freed before the reader ends.
It must not go through two grace periods either.

The other readers count their pipe and batch values with this_cpu_inc().
On some architectures, an NMI may interrupt it. So the NMI reader counts
its values in arrays of atomic counters of its own. The statistics print
them as "NMI Reader Pipe:" and "NMI Reader Batch:". A too-short grace
period seen in NMI gets the same "!!!" and the same warning as for the
other readers.

The NMI may interrupt lockdep, which then does not record the reader.
So the reader loads the pointer with rcu_dereference_raw().

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
 kernel/rcu/rcutorture.c | 96 ++++++++++++++++++++++++++++++++++++++---
 1 file changed, 90 insertions(+), 6 deletions(-)

diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 93ca7f06371e..9889293c968b 100644
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -117,6 +117,7 @@ torture_param(int, n_barrier_cbs, 0, "# of callbacks/kthreads for barrier testin
 torture_param(int, n_up_down, 32, "# of concurrent up/down hrtimer-based RCU readers");
 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads");
 torture_param(bool, nmi_calls, true, "Exercise ->call() from NMI on nmi_capable flavors");
+torture_param(bool, nmi_readers, true, "Run readers from NMI with NMI-safe reader flavors");
 torture_param(int, nreaders, -1, "Number of RCU reader threads");
 torture_param(bool, nwriters, 1, "Number of RCU writer threads (0 or 1)");
 torture_param(int, object_debug, 0, "Enable debug-object double call_rcu() testing");
@@ -220,6 +221,11 @@ static atomic_long_t n_rcu_torture_timers;
 static atomic_long_t n_rcu_torture_irqs;
 static atomic_long_t n_rcu_torture_nmi_call;
 static atomic_long_t n_rcu_torture_nmi_cb;
+static atomic_long_t n_rcu_torture_nmi_read;
+// Pipe and batch counts of the readers in NMI, as rcu_torture_count and
+// rcu_torture_batch for the others.  An NMI may interrupt this_cpu_inc().
+static atomic_long_t rcu_torture_nmi_pipe[RCU_TORTURE_PIPE_LEN + 1];
+static atomic_long_t rcu_torture_nmi_batch[RCU_TORTURE_PIPE_LEN + 1];
 static long n_barrier_attempts;
 static long n_barrier_successes; /* did rcu_barrier test succeed? */
 static unsigned long n_read_exits;
@@ -438,6 +444,7 @@ struct rcu_torture_ops {
 	long cbflood_max;
 	int irq_capable;
 	int nmi_capable;
+	bool (*nmi_reader_capable)(void); // Are ->readlock() and ->readunlock() NMI-safe?
 	int can_boost;
 	int extendables;
 	int slow_gps;
@@ -790,6 +797,14 @@ static int srcu_torture_read_lock(void)
 	return ret;
 }
 
+// Only these reader flavors may be used from NMI.
+static bool srcu_torture_nmi_reader_capable(void)
+{
+	return reader_flavor &&
+	       !(reader_flavor & ~(SRCU_READ_FLAVOR_NMI | SRCU_READ_FLAVOR_FAST |
+				   SRCU_READ_FLAVOR_ATOMIC));
+}
+
 static void
 srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp)
 {
@@ -981,6 +996,7 @@ static struct rcu_torture_ops srcu_ops = {
 	.have_up_down	= IS_ENABLED(CONFIG_TINY_SRCU)
 				? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST_UPDOWN,
 	.nmi_capable	= 1,
+	.nmi_reader_capable = srcu_torture_nmi_reader_capable,
 	.name		= "srcu"
 };
 
@@ -1049,6 +1065,7 @@ static struct rcu_torture_ops srcud_ops = {
 	.have_up_down	= IS_ENABLED(CONFIG_TINY_SRCU)
 				? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST_UPDOWN,
 	.nmi_capable	= 1,
+	.nmi_reader_capable = srcu_torture_nmi_reader_capable,
 	.name		= "srcud"
 };
 
@@ -2711,6 +2728,10 @@ static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_timer_rand);
  * per-CPU hardware perf counter overflows into an NMI, and its handler submits
  * a preallocated callback via ->call().  One callback per CPU is in flight at a
  * time (guarded by an atomic) to avoid allocating in NMI.
+ *
+ * The same handler also runs a reader, for flavors whose ->nmi_reader_capable()
+ * says that their readers are NMI-safe.  The reader may interrupt another
+ * reader on its CPU, in the middle of its counter update.
  */
 #ifdef CONFIG_PERF_EVENTS
 static struct perf_event_attr rcu_torture_nmi_attr = {
@@ -2735,6 +2756,8 @@ struct rcu_torture_nmi_cb {
 
 static struct perf_event **rcu_torture_nmi_events;
 static int rcu_torture_nmi_hp_state;
+static bool rcu_torture_nmi_do_calls;
+static bool rcu_torture_nmi_do_reads;
 static DEFINE_PER_CPU(struct rcu_torture_nmi_cb, rcu_torture_nmi_cb);
 
 static void rcu_torture_nmi_invoked(struct rcu_head *rhp)
@@ -2745,6 +2768,34 @@ static void rcu_torture_nmi_invoked(struct rcu_head *rhp)
 	atomic_set(&rtncp->inuse, 0);
 }
 
+/*
+ * A reader in NMI.  The NMI may interrupt lockdep, which then does not record
+ * the reader, so the pointer is loaded without a lockdep check.  The element
+ * must not go through two grace periods before the reader ends.
+ * rcu_torture_stats_print() checks this in the pipe counts.
+ */
+static void rcu_torture_nmi_read(void)
+{
+	unsigned long started, completed;
+	struct rcu_torture *p;
+	int idx, pipe_count;
+
+	idx = cur_ops->readlock();
+	started = cur_ops->get_gp_seq();
+	p = rcu_dereference_raw(rcu_torture_current);
+	if (p) {
+		if (!READ_ONCE(p->rtort_mbtest))
+			atomic_inc(&n_rcu_torture_mberror);
+		pipe_count = min(READ_ONCE(p->rtort_pipe_count), RCU_TORTURE_PIPE_LEN);
+		completed = rcutorture_seq_diff(cur_ops->get_gp_seq(), started);
+		completed = min_t(unsigned long, completed, RCU_TORTURE_PIPE_LEN);
+		atomic_long_inc(&rcu_torture_nmi_pipe[pipe_count]);
+		atomic_long_inc(&rcu_torture_nmi_batch[completed]);
+	}
+	cur_ops->readunlock(idx);
+	atomic_long_inc(&n_rcu_torture_nmi_read);
+}
+
 static void rcu_torture_nmi_overflow(struct perf_event *event,
 				     struct perf_sample_data *data,
 				     struct pt_regs *regs)
@@ -2753,10 +2804,12 @@ static void rcu_torture_nmi_overflow(struct perf_event *event,
 
 	if (!in_nmi())
 		return;
-	if (cur_ops->call && !atomic_xchg(&rtncp->inuse, 1)) {
+	if (rcu_torture_nmi_do_calls && !atomic_xchg(&rtncp->inuse, 1)) {
 		atomic_long_inc(&n_rcu_torture_nmi_call);
 		cur_ops->call(&rtncp->rh, rcu_torture_nmi_invoked);
 	}
+	if (rcu_torture_nmi_do_reads)
+		rcu_torture_nmi_read();
 }
 
 static int rcu_torture_nmi_online(unsigned int cpu)
@@ -2789,7 +2842,10 @@ static void rcu_torture_nmi_init(void)
 {
 	int ret;
 
-	if (!nmi_calls || !cur_ops->nmi_capable || !cur_ops->call)
+	rcu_torture_nmi_do_calls = nmi_calls && cur_ops->nmi_capable && cur_ops->call;
+	rcu_torture_nmi_do_reads = nmi_readers && cur_ops->nmi_reader_capable &&
+				   cur_ops->nmi_reader_capable();
+	if (!rcu_torture_nmi_do_calls && !rcu_torture_nmi_do_reads)
 		return;
 	rcu_torture_nmi_events = kzalloc_objs(*rcu_torture_nmi_events,
 					      nr_cpu_ids);
@@ -2815,9 +2871,12 @@ static void rcu_torture_nmi_cleanup(void)
 	}
 	kfree(rcu_torture_nmi_events);
 	rcu_torture_nmi_events = NULL;
-	if (!atomic_long_read(&n_rcu_torture_nmi_call))
+	if (rcu_torture_nmi_do_calls && !atomic_long_read(&n_rcu_torture_nmi_call))
 		pr_alert("%s: nmi_calls set but no ->call() ever issued from NMI, so NMI ->call() went untested (no PMU, or NMIs unavailable here).\n",
 			 __func__);
+	if (rcu_torture_nmi_do_reads && !atomic_long_read(&n_rcu_torture_nmi_read))
+		pr_alert("%s: nmi_readers set but no reader ever ran in NMI, so NMI readers went untested (no PMU, or NMIs unavailable here).\n",
+			 __func__);
 }
 #else /* #ifdef CONFIG_PERF_EVENTS */
 static void rcu_torture_nmi_init(void) { }
@@ -3152,6 +3211,7 @@ rcu_torture_stats_print(void)
 	int cpu;
 	int i;
 	long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
+	int j;
 	long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
 	long n_gpwraps = 0;
 	unsigned long ndowns = 0;
@@ -3183,6 +3243,10 @@ rcu_torture_stats_print(void)
 		if (pipesummary[i] != 0)
 			break;
 	} // The value of variable "i" is used later, so don't clobber it!
+	for (j = RCU_TORTURE_PIPE_LEN; j >= 0; j--) {
+		if (atomic_long_read(&rcu_torture_nmi_pipe[j]))
+			break;
+	} // Same for "j", for the readers in NMI.
 
 	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
 	rtcp = rcu_access_pointer(rcu_torture_current);
@@ -3212,9 +3276,10 @@ rcu_torture_stats_print(void)
 		data_race(n_barrier_attempts),
 		data_race(n_rcu_torture_barrier_error));
 	pr_cont("read-exits: %ld ", data_race(n_read_exits)); // Statistic.
-	pr_cont("nmi-calls: %ld nmi-cbs: %ld ",
+	pr_cont("nmi-calls: %ld nmi-cbs: %ld nmi-reads: %ld ",
 		atomic_long_read(&n_rcu_torture_nmi_call),
-		atomic_long_read(&n_rcu_torture_nmi_cb));
+		atomic_long_read(&n_rcu_torture_nmi_cb),
+		atomic_long_read(&n_rcu_torture_nmi_read));
 	pr_cont("nocb-toggles: %ld:%ld ",
 		atomic_long_read(&n_nocb_offload), atomic_long_read(&n_nocb_deoffload));
 	pr_cont("gpwraps: %ld\n", n_gpwraps);
@@ -3223,7 +3288,7 @@ rcu_torture_stats_print(void)
 	if (atomic_read(&n_rcu_torture_mberror) ||
 	    atomic_read(&n_rcu_torture_mbchk_fail) ||
 	    n_rcu_torture_barrier_error || n_rcu_torture_boost_ktrerror ||
-	    n_rcu_torture_boost_failure || i > 1) {
+	    n_rcu_torture_boost_failure || i > 1 || j > 1) {
 		pr_cont("%s", "!!! ");
 		atomic_inc(&n_rcu_torture_error);
 		WARN_ON_ONCE(atomic_read(&n_rcu_torture_mberror));
@@ -3232,6 +3297,7 @@ rcu_torture_stats_print(void)
 		WARN_ON_ONCE(n_rcu_torture_boost_ktrerror); // no boost kthread
 		WARN_ON_ONCE(n_rcu_torture_boost_failure); // boost failed (TIMER_SOFTIRQ RT prio?)
 		WARN_ON_ONCE(i > 1); // Too-short grace period
+		WARN_ON_ONCE(j > 1); // Too-short grace period, reader in NMI
 	}
 	pr_cont("Reader Pipe: ");
 	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
@@ -3244,6 +3310,20 @@ rcu_torture_stats_print(void)
 		pr_cont(" %ld", batchsummary[i]);
 	pr_cont("\n");
 
+	if (atomic_long_read(&n_rcu_torture_nmi_read)) {
+		pr_alert("%s%s ", torture_type, TORTURE_FLAG);
+		pr_cont("NMI Reader Pipe: ");
+		for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
+			pr_cont(" %ld", atomic_long_read(&rcu_torture_nmi_pipe[i]));
+		pr_cont("\n");
+
+		pr_alert("%s%s ", torture_type, TORTURE_FLAG);
+		pr_cont("NMI Reader Batch: ");
+		for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
+			pr_cont(" %ld", atomic_long_read(&rcu_torture_nmi_batch[i]));
+		pr_cont("\n");
+	}
+
 	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
 	pr_cont("Free-Block Circulation: ");
 	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
@@ -5049,6 +5129,10 @@ rcu_torture_init(void)
 			per_cpu(rcu_torture_batch, cpu)[i] = 0;
 		}
 	}
+	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
+		atomic_long_set(&rcu_torture_nmi_pipe[i], 0);
+		atomic_long_set(&rcu_torture_nmi_batch[i], 0);
+	}
 	err_segs_recorded = 0;
 	rt_read_nsegs = 0;
 
-- 
2.47.3