[PATCH for-v7.5 08/14] rcuscale: Consolidate per-writer arrays into struct writer_state

[Joel Fernandes <jfern@kernel.org>]

The writer kthreads keep their state in five parallel arrays indexed by
writer number: writer_tasks, writer_durations, writer_n_durations,
writer_done and writer_freelists.  Each must be allocated, NULL-checked,
freed and cleared separately, and the cleanup path needs several
guards to cope with any one of them being missing.

Collapse them into a single array of struct writer_state, holding the
task pointer, the duration buffer, the measurement count, the done flag
and the existing struct writer_freelist.

This reduces initialization to one allocation and one out-of-memory
check, and cleanup to a single kfree().  As a side effect, this also
fixes a leak: cleanup only ran its frees under "if (writer_tasks)", so
if the writer_tasks allocation failed while a later allocation (for
example writer_durations or writer_n_durations) still succeeded, those
already-allocated arrays were never freed.  With a single allocation,
that partial-allocation state can no longer occur.

Assisted-by: Codex
Fixes: 8704baab9bc8 ("rcutorture: Add RCU grace-period performance tests")
Signed-off-by: Joel Fernandes <jfern@kernel.org>
Reviewed-by: Bradley Morgan <brads@mainlining.org>
Tested-by: Bradley Morgan <brads@mainlining.org> # Power10
Acked-by: Davidlohr Bueso <dave@stgolabs.net>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
---
 kernel/rcu/rcuscale.c | 125 ++++++++++++++++++------------------------
 1 file changed, 54 insertions(+), 71 deletions(-)

diff --git a/kernel/rcu/rcuscale.c b/kernel/rcu/rcuscale.c
index 38c75c37a07c..41abf05a14d9 100644
--- a/kernel/rcu/rcuscale.c
+++ b/kernel/rcu/rcuscale.c
@@ -115,17 +115,21 @@ struct writer_freelist {
 	struct writer_mblock *ws_mblocks;
 };
 
+struct writer_state {
+	struct writer_freelist wst_fl;
+	struct task_struct *wst_task;
+	u64 *wst_durations;
+	int wst_n_durations;
+	bool wst_done;
+};
+
 static int nrealreaders;
 static int nrealwriters;
 static int nrealexp;
-static struct task_struct **writer_tasks;
 static struct task_struct **reader_tasks;
 static struct task_struct **exp_tasks;
 
-static u64 **writer_durations;
-static bool *writer_done;
-static struct writer_freelist *writer_freelists;
-static int *writer_n_durations;
+static struct writer_state *writer_states;
 static atomic_t n_rcu_scale_reader_started;
 static atomic_t n_rcu_scale_writer_started;
 static atomic_t n_rcu_scale_writer_finished;
@@ -504,9 +508,9 @@ static struct writer_mblock *rcu_scale_alloc(long me)
 	struct writer_freelist *wflp;
 	struct writer_mblock *wmbp;
 
-	if (WARN_ON_ONCE(!writer_freelists))
+	if (WARN_ON_ONCE(!writer_states))
 		return NULL;
-	wflp = &writer_freelists[me];
+	wflp = &writer_states[me].wst_fl;
 	if (llist_empty(&wflp->ws_lhp)) {
 		// ->ws_lhp is private to its rcu_scale_writer task.
 		wmbp = container_of(llist_del_all(&wflp->ws_lhg), struct writer_mblock, wmb_node);
@@ -560,8 +564,9 @@ rcu_scale_writer(void *arg)
 	u64 t;
 	DEFINE_TORTURE_RANDOM(tr);
 	u64 *wdp;
-	u64 *wdpp = writer_durations[me];
-	struct writer_freelist *wflp = &writer_freelists[me];
+	struct writer_state *wsp = &writer_states[me];
+	u64 *wdpp = wsp->wst_durations;
+	struct writer_freelist *wflp = &wsp->wst_fl;
 	struct writer_mblock *wmbp = NULL;
 
 	VERBOSE_SCALEOUT_STRING("rcu_scale_writer task started");
@@ -631,7 +636,7 @@ rcu_scale_writer(void *arg)
 			started = true;
 		if (!done && i >= MIN_MEAS && time_after(jiffies, jdone)) {
 			done = true;
-			WRITE_ONCE(writer_done[me], true);
+			WRITE_ONCE(wsp->wst_done, true);
 			sched_set_normal(current, 0);
 			pr_alert("%s%s rcu_scale_writer %ld has %d measurements\n",
 				 scale_type, SCALE_FLAG, me, MIN_MEAS);
@@ -649,7 +654,7 @@ rcu_scale_writer(void *arg)
 						cur_ops->get_gp_seq();
 				}
 				if (shutdown_secs) {
-					writer_tasks[me] = NULL;
+					wsp->wst_task = NULL;
 					smp_mb(); /* Assign before wake. */
 					rcu_scale_cleanup();
 					kernel_power_off();
@@ -665,10 +670,10 @@ rcu_scale_writer(void *arg)
 
 			if (!atomic_xchg(&dumped, 1)) {
 				for (i = 0; i < nrealwriters; i++) {
-					if (writer_done[i])
+					if (writer_states[i].wst_done)
 						continue;
 					pr_info("%s: Task %ld flags writer %d:\n", __func__, me, i);
-					sched_show_task(writer_tasks[i]);
+					sched_show_task(writer_states[i].wst_task);
 				}
 				if (cur_ops->stats)
 					cur_ops->stats();
@@ -676,7 +681,7 @@ rcu_scale_writer(void *arg)
 		}
 		if (!selfreport && time_after(jiffies, jdone + HZ * (70 + me))) {
 			pr_info("%s: Writer %ld self-report: started %d done %d/%d->%d i %d jdone %lu.\n",
-				__func__, me, started, done, writer_done[me], atomic_read(&n_rcu_scale_writer_finished), i, jiffies - jdone);
+				__func__, me, started, done, wsp->wst_done, atomic_read(&n_rcu_scale_writer_finished), i, jiffies - jdone);
 			selfreport = true;
 		}
 		if (gp_succeeded && started && !alldone && i < MAX_MEAS - 1)
@@ -687,7 +692,7 @@ rcu_scale_writer(void *arg)
 		rcu_scale_free(wmbp);
 		cur_ops->gp_barrier();
 	}
-	writer_n_durations[me] = i_max + 1;
+	wsp->wst_n_durations = i_max + 1;
 	torture_kthread_stopping("rcu_scale_writer");
 	return 0;
 }
@@ -1031,13 +1036,11 @@ rcu_scale_cleanup(void)
 		reader_tasks = NULL;
 	}
 
-	if (writer_tasks) {
+	if (writer_states) {
 		for (i = 0; i < nrealwriters; i++) {
 			torture_stop_kthread(rcu_scale_writer,
-					     writer_tasks[i]);
-			if (!writer_n_durations)
-				continue;
-			j = writer_n_durations[i];
+					     writer_states[i].wst_task);
+			j = writer_states[i].wst_n_durations;
 			pr_alert("%s%s writer %d gps: %d\n",
 				 scale_type, SCALE_FLAG, i, j);
 			ngps += j;
@@ -1051,14 +1054,12 @@ rcu_scale_cleanup(void)
 			 rcuscale_seq_diff(b_rcu_gp_test_finished,
 					   b_rcu_gp_test_started));
 		for (i = 0; i < nrealwriters; i++) {
-			if (!writer_durations)
-				break;
-			if (!writer_n_durations)
-				continue;
-			wdpp = writer_durations[i];
+			struct writer_state *wsp = &writer_states[i];
+
+			wdpp = wsp->wst_durations;
 			if (!wdpp)
 				continue;
-			for (j = 0; j < writer_n_durations[i]; j++) {
+			for (j = 0; j < wsp->wst_n_durations; j++) {
 				wdp = &wdpp[j];
 				pr_alert("%s%s %4d writer-duration: %5d %llu\n",
 					scale_type, SCALE_FLAG,
@@ -1066,34 +1067,23 @@ rcu_scale_cleanup(void)
 				if (j % 100 == 0)
 					schedule_timeout_uninterruptible(1);
 			}
-			kfree(writer_durations[i]);
-			if (writer_freelists) {
+			kfree(wdpp);
+			if (wsp->wst_fl.ws_mblocks) {
 				int ctr = 0;
 				struct llist_node *llnp;
-				struct writer_freelist *wflp = &writer_freelists[i];
 
-				if (wflp->ws_mblocks) {
-					llist_for_each(llnp, wflp->ws_lhg.first)
-						ctr++;
-					llist_for_each(llnp, wflp->ws_lhp.first)
-						ctr++;
-					WARN_ONCE(ctr != gp_async_max,
-						  "%s: ctr = %d gp_async_max = %d\n",
-						  __func__, ctr, gp_async_max);
-					kfree(wflp->ws_mblocks);
-				}
+				llist_for_each(llnp, wsp->wst_fl.ws_lhg.first)
+					ctr++;
+				llist_for_each(llnp, wsp->wst_fl.ws_lhp.first)
+					ctr++;
+				WARN_ONCE(ctr != gp_async_max,
+					  "%s: ctr = %d gp_async_max = %d\n",
+					  __func__, ctr, gp_async_max);
+				kfree(wsp->wst_fl.ws_mblocks);
 			}
 		}
-		kfree(writer_tasks);
-		writer_tasks = NULL;
-		kfree(writer_durations);
-		writer_durations = NULL;
-		kfree(writer_n_durations);
-		writer_n_durations = NULL;
-		kfree(writer_done);
-		writer_done = NULL;
-		kfree(writer_freelists);
-		writer_freelists = NULL;
+		kfree(writer_states);
+		writer_states = NULL;
 	}
 
 	/* Do torture-type-specific cleanup operations.  */
@@ -1187,36 +1177,29 @@ rcu_scale_init(void)
 				goto unwind;
 		}
 	}
-	writer_tasks = kzalloc_objs(writer_tasks[0], nrealwriters);
-	writer_durations = kzalloc_objs(*writer_durations, nrealwriters);
-	writer_n_durations = kzalloc_objs(*writer_n_durations, nrealwriters);
-	writer_done = kzalloc_objs(writer_done[0], nrealwriters);
-	if (gp_async) {
-		if (gp_async_max <= 0) {
-			pr_warn("%s: gp_async_max = %d must be greater than zero.\n",
-				__func__, gp_async_max);
-			WARN_ON_ONCE(IS_BUILTIN(CONFIG_RCU_TORTURE_TEST));
-			firsterr = -EINVAL;
-			goto unwind;
-		}
-		writer_freelists = kzalloc_objs(writer_freelists[0],
-						nrealwriters);
+	writer_states = kzalloc_objs(writer_states[0], nrealwriters);
+	if (gp_async && gp_async_max <= 0) {
+		pr_warn("%s: gp_async_max = %d must be greater than zero.\n",
+			__func__, gp_async_max);
+		WARN_ON_ONCE(IS_BUILTIN(CONFIG_RCU_TORTURE_TEST));
+		firsterr = -EINVAL;
+		goto unwind;
 	}
-	if (!writer_tasks || !writer_durations || !writer_n_durations || !writer_done ||
-	    (gp_async && !writer_freelists)) {
+	if (!writer_states) {
 		SCALEOUT_ERRSTRING("out of memory");
 		firsterr = -ENOMEM;
 		goto unwind;
 	}
 	for (i = 0; i < nrealwriters; i++) {
-		writer_durations[i] =
-			kzalloc_objs(*writer_durations[i], MAX_MEAS);
-		if (!writer_durations[i]) {
+		struct writer_state *wsp = &writer_states[i];
+
+		wsp->wst_durations = kzalloc_objs(*wsp->wst_durations, MAX_MEAS);
+		if (!wsp->wst_durations) {
 			firsterr = -ENOMEM;
 			goto unwind;
 		}
-		if (writer_freelists) {
-			struct writer_freelist *wflp = &writer_freelists[i];
+		if (gp_async) {
+			struct writer_freelist *wflp = &wsp->wst_fl;
 
 			init_llist_head(&wflp->ws_lhg);
 			init_llist_head(&wflp->ws_lhp);
@@ -1234,7 +1217,7 @@ rcu_scale_init(void)
 			}
 		}
 		firsterr = torture_create_kthread(rcu_scale_writer, (void *)i,
-						  writer_tasks[i]);
+						  wsp->wst_task);
 		if (torture_init_error(firsterr))
 			goto unwind;
 	}
-- 
2.54.0