[PATCH] srcutiny: Add atomic SRCU operation checks and state

[Kunwu Chan <kunwu.chan@gmail.com>]

From: Kunwu Chan <kunwu.chan@gmail.com>

Add atomic SRCU operation checks and the associated state to Tiny
SRCU.

An atomic SRCU domain does not use the normal SRCU callback and
grace-period machinery.  In particular, a callback queued with
call_srcu() would never be processed.  Use WARN_ON_ONCE() to reject
call_srcu() and srcu_barrier() on atomic SRCU domains.

For synchronize_srcu(), redirect atomic SRCU domains to
synchronize_srcu_atomic().

Add srcu_reader_flavor to the Tiny SRCU state for these checks.
Tiny SRCU does not currently set the flavor for atomic SRCU domains,
but keeping the flavor in the common state allows the operation
checks to enforce the restriction once atomic flavor tracking is
enabled.

Also initialize srcu_atomic_gp_flag, which was previously left
uninitialized.

Signed-off-by: Kunwu Chan <kunwu.chan@gmail.com>
---
 include/linux/srcutiny.h |  1 +
 kernel/rcu/srcutiny.c    | 12 ++++++++++++
 2 files changed, 13 insertions(+)

diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h
index 47a368f945e3..2b293336525a 100644
--- a/include/linux/srcutiny.h
+++ b/include/linux/srcutiny.h
@@ -20,6 +20,7 @@ struct srcu_struct {
 	u8 srcu_gp_running;		/* GP workqueue running? */
 	u8 srcu_gp_waiting;		/* GP waiting for readers? */
 	u8 srcu_atomic_gp_flag;		/* Serialize atomic GP work.*/
+	u8 srcu_reader_flavor;		/* Values: SRCU_READ_FLAVOR_.*  */
 	unsigned long srcu_idx;		/* Current reader array element in bit 0x2. */
 	unsigned long srcu_idx_max;	/* Furthest future srcu_idx request. */
 	struct swait_queue_head srcu_wq;
diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c
index 26ea4bfbeaf2..22f7716cbb0e 100644
--- a/kernel/rcu/srcutiny.c
+++ b/kernel/rcu/srcutiny.c
@@ -41,6 +41,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp)
 	ssp->srcu_cb_tail = &ssp->srcu_cb_head;
 	ssp->srcu_gp_running = false;
 	ssp->srcu_gp_waiting = false;
+	ssp->srcu_atomic_gp_flag = 0;
 	ssp->srcu_idx = 0;
 	ssp->srcu_idx_max = 0;
 	INIT_WORK(&ssp->srcu_work, srcu_drive_gp);
@@ -289,6 +290,9 @@ EXPORT_SYMBOL_GPL(srcu_defer_drain);
 void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,
 	       rcu_callback_t func)
 {
+	if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC))
+		return;
+
 	if (should_rcu_defer()) {
 		/* A re-entrant call_srcu() during the drain would livelock it. */
 		if (READ_ONCE(srcu_defer_draining) && !in_nmi()) {
@@ -319,6 +323,11 @@ void synchronize_srcu(struct srcu_struct *ssp)
 {
 	struct rcu_synchronize rs;
 
+	if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC)) {
+		synchronize_srcu_atomic(ssp);
+		return;
+	}
+
 	srcu_lock_sync(&ssp->dep_map);
 
 	RCU_LOCKDEP_WARN(lockdep_is_held(ssp) ||
@@ -415,6 +424,9 @@ EXPORT_SYMBOL_GPL(synchronize_srcu_atomic);
 /* Register any deferred callbacks, then wait for all in-flight ones. */
 void srcu_barrier(struct srcu_struct *ssp)
 {
+	if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC))
+		return;
+
 	__srcu_defer_drain(ssp);
 	synchronize_srcu(ssp);
 }
-- 
2.43.0