[PATCH 0/8] refscale updates for v6.19

[PATCH 0/8] refscale updates for v6.19

[Paul E. McKenney]

Hello!

This series contains additions of microbenchmarks to the refscale suite,
and depends on the SRCU series.

1.	Exercise DEFINE_STATIC_SRCU_FAST() and init_srcu_struct_fast().

2.	Add local_irq_disable() and local_irq_save() readers.

3.	Add local_bh_disable() readers.

4.	Add preempt_disable() readers.

5.	Add this_cpu_inc() readers.

6.	Add non-atomic per-CPU increment readers.

7.	Do not diable interrupts for tests involving local_bh_enable().

8.	Add SRCU-fast-updown readers.

						Thanx, Paul

------------------------------------------------------------------------

 b/kernel/rcu/refscale.c |    9 +
 kernel/rcu/refscale.c   |  378 ++++++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 372 insertions(+), 15 deletions(-)

[PATCH 1/8] refscale: Exercise DEFINE_STATIC_SRCU_FAST() and init_srcu_struct_fast()

[Paul E. McKenney]

This commit updates the initialization for the "srcu-fast" scale
type to use DEFINE_STATIC_SRCU_FAST() when reader_flavor is equal to
SRCU_READ_FLAVOR_FAST.

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: <bpf@vger.kernel.org>
---
 kernel/rcu/refscale.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c
index 19841704d8f5..ece77f6d055b 100644
--- a/kernel/rcu/refscale.c
+++ b/kernel/rcu/refscale.c
@@ -184,6 +184,7 @@ static const struct ref_scale_ops rcu_ops = {
 
 // Definitions for SRCU ref scale testing.
 DEFINE_STATIC_SRCU(srcu_refctl_scale);
+DEFINE_STATIC_SRCU_FAST(srcu_fast_refctl_scale);
 static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale;
 
 static void srcu_ref_scale_read_section(const int nloops)
@@ -216,6 +217,12 @@ static const struct ref_scale_ops srcu_ops = {
 	.name		= "srcu"
 };
 
+static bool srcu_fast_sync_scale_init(void)
+{
+	srcu_ctlp = &srcu_fast_refctl_scale;
+	return true;
+}
+
 static void srcu_fast_ref_scale_read_section(const int nloops)
 {
 	int i;
@@ -240,7 +247,7 @@ static void srcu_fast_ref_scale_delay_section(const int nloops, const int udl, c
 }
 
 static const struct ref_scale_ops srcu_fast_ops = {
-	.init		= rcu_sync_scale_init,
+	.init		= srcu_fast_sync_scale_init,
 	.readsection	= srcu_fast_ref_scale_read_section,
 	.delaysection	= srcu_fast_ref_scale_delay_section,
 	.name		= "srcu-fast"
-- 
2.40.1

[PATCH 2/8] refscale: Add local_irq_disable() and local_irq_save() readers

[Paul E. McKenney]

This commit adds refscale readers based on local_irq_disable() and
local_irq_enable() ("refscale.scale_type=irq") and on local_irq_save()
and local_irq_restore ("refscale.scale_type=irqsave").  On my x86 laptop,
these are about 2.8ns and 7.5ns per enable/disable pair, respectively.

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
---
 kernel/rcu/refscale.c | 66 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 65 insertions(+), 1 deletion(-)

diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c
index ece77f6d055b..266a3fa94b54 100644
--- a/kernel/rcu/refscale.c
+++ b/kernel/rcu/refscale.c
@@ -636,6 +636,70 @@ static const struct ref_scale_ops jiffies_ops = {
 	.name		= "jiffies"
 };
 
+static void ref_irq_section(const int nloops)
+{
+	int i;
+
+	preempt_disable();
+	for (i = nloops; i >= 0; i--) {
+		local_irq_disable();
+		local_irq_enable();
+	}
+	preempt_enable();
+}
+
+static void ref_irq_delay_section(const int nloops, const int udl, const int ndl)
+{
+	int i;
+
+	preempt_disable();
+	for (i = nloops; i >= 0; i--) {
+		local_irq_disable();
+		un_delay(udl, ndl);
+		local_irq_enable();
+	}
+	preempt_enable();
+}
+
+static const struct ref_scale_ops irq_ops = {
+	.readsection	= ref_irq_section,
+	.delaysection	= ref_irq_delay_section,
+	.name		= "irq"
+};
+
+static void ref_irqsave_section(const int nloops)
+{
+	unsigned long flags;
+	int i;
+
+	preempt_disable();
+	for (i = nloops; i >= 0; i--) {
+		local_irq_save(flags);
+		local_irq_restore(flags);
+	}
+	preempt_enable();
+}
+
+static void ref_irqsave_delay_section(const int nloops, const int udl, const int ndl)
+{
+	unsigned long flags;
+	int i;
+
+	preempt_disable();
+	for (i = nloops; i >= 0; i--) {
+		local_irq_save(flags);
+		un_delay(udl, ndl);
+		local_irq_restore(flags);
+	}
+	preempt_enable();
+}
+
+static const struct ref_scale_ops irqsave_ops = {
+	.readsection	= ref_irqsave_section,
+	.delaysection	= ref_irqsave_delay_section,
+	.name		= "irqsave"
+};
+
 ////////////////////////////////////////////////////////////////////////
 //
 // Methods leveraging SLAB_TYPESAFE_BY_RCU.
@@ -1172,7 +1236,7 @@ ref_scale_init(void)
 	static const struct ref_scale_ops *scale_ops[] = {
 		&rcu_ops, &srcu_ops, &srcu_fast_ops, RCU_TRACE_OPS RCU_TASKS_OPS
 		&refcnt_ops, &rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops,
-		&acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops,
+		&acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops, &irq_ops, &irqsave_ops,
 		&typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops,
 	};
 
-- 
2.40.1

[PATCH 3/8] refscale: Add local_bh_disable() readers

[Paul E. McKenney]

This commit adds refscale readers based on local_bh_disable() and
local_bh_enable() ("refscale.scale_type=bh").  On my x86 laptop, these
are about 4.9ns.

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
---
 kernel/rcu/refscale.c | 34 +++++++++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c
index 266a3fa94b54..1faed90231ab 100644
--- a/kernel/rcu/refscale.c
+++ b/kernel/rcu/refscale.c
@@ -636,6 +636,37 @@ static const struct ref_scale_ops jiffies_ops = {
 	.name		= "jiffies"
 };
 
+static void ref_bh_section(const int nloops)
+{
+	int i;
+
+	preempt_disable();
+	for (i = nloops; i >= 0; i--) {
+		local_bh_disable();
+		local_bh_enable();
+	}
+	preempt_enable();
+}
+
+static void ref_bh_delay_section(const int nloops, const int udl, const int ndl)
+{
+	int i;
+
+	preempt_disable();
+	for (i = nloops; i >= 0; i--) {
+		local_bh_disable();
+		un_delay(udl, ndl);
+		local_bh_enable();
+	}
+	preempt_enable();
+}
+
+static const struct ref_scale_ops bh_ops = {
+	.readsection	= ref_bh_section,
+	.delaysection	= ref_bh_delay_section,
+	.name		= "bh"
+};
+
 static void ref_irq_section(const int nloops)
 {
 	int i;
@@ -1236,7 +1267,8 @@ ref_scale_init(void)
 	static const struct ref_scale_ops *scale_ops[] = {
 		&rcu_ops, &srcu_ops, &srcu_fast_ops, RCU_TRACE_OPS RCU_TASKS_OPS
 		&refcnt_ops, &rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops,
-		&acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops, &irq_ops, &irqsave_ops,
+		&acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops,
+		&bh_ops, &irq_ops, &irqsave_ops,
 		&typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops,
 	};
 
-- 
2.40.1

[PATCH 4/8] refscale: Add preempt_disable() readers

[Paul E. McKenney]

This commit adds refscale readers based on preempt_disable() and
preempt_enable() ("refscale.scale_type=preempt").  On my x86 laptop, these
are about 2.8ns.

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
---
 kernel/rcu/refscale.c | 33 ++++++++++++++++++++++++++++++++-
 1 file changed, 32 insertions(+), 1 deletion(-)

diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c
index 1faed90231ab..ac9a0b6332ac 100644
--- a/kernel/rcu/refscale.c
+++ b/kernel/rcu/refscale.c
@@ -636,6 +636,37 @@ static const struct ref_scale_ops jiffies_ops = {
 	.name		= "jiffies"
 };
 
+static void ref_preempt_section(const int nloops)
+{
+	int i;
+
+	migrate_disable();
+	for (i = nloops; i >= 0; i--) {
+		preempt_disable();
+		preempt_enable();
+	}
+	migrate_enable();
+}
+
+static void ref_preempt_delay_section(const int nloops, const int udl, const int ndl)
+{
+	int i;
+
+	migrate_disable();
+	for (i = nloops; i >= 0; i--) {
+		preempt_disable();
+		un_delay(udl, ndl);
+		preempt_enable();
+	}
+	migrate_enable();
+}
+
+static const struct ref_scale_ops preempt_ops = {
+	.readsection	= ref_preempt_section,
+	.delaysection	= ref_preempt_delay_section,
+	.name		= "preempt"
+};
+
 static void ref_bh_section(const int nloops)
 {
 	int i;
@@ -1268,7 +1299,7 @@ ref_scale_init(void)
 		&rcu_ops, &srcu_ops, &srcu_fast_ops, RCU_TRACE_OPS RCU_TASKS_OPS
 		&refcnt_ops, &rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops,
 		&acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops,
-		&bh_ops, &irq_ops, &irqsave_ops,
+		&preempt_ops, &bh_ops, &irq_ops, &irqsave_ops,
 		&typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops,
 	};
 
-- 
2.40.1

[PATCH 5/8] refscale: Add this_cpu_inc() readers

[Paul E. McKenney]

This commit adds refscale readers based on this_cpu_inc() and
this_cpu_inc() ("refscale.scale_type=percpuinc").  On my x86 laptop,
these are about 4.5ns per pair.

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
---
 kernel/rcu/refscale.c | 36 ++++++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c
index ac9a0b6332ac..479d86cad652 100644
--- a/kernel/rcu/refscale.c
+++ b/kernel/rcu/refscale.c
@@ -330,6 +330,9 @@ static const struct ref_scale_ops rcu_trace_ops = {
 // Definitions for reference count
 static atomic_t refcnt;
 
+// Definitions acquire-release.
+static DEFINE_PER_CPU(unsigned long, test_acqrel);
+
 static void ref_refcnt_section(const int nloops)
 {
 	int i;
@@ -358,6 +361,34 @@ static const struct ref_scale_ops refcnt_ops = {
 	.name		= "refcnt"
 };
 
+static void ref_percpuinc_section(const int nloops)
+{
+	int i;
+
+	for (i = nloops; i >= 0; i--) {
+		this_cpu_inc(test_acqrel);
+		this_cpu_dec(test_acqrel);
+	}
+}
+
+static void ref_percpuinc_delay_section(const int nloops, const int udl, const int ndl)
+{
+	int i;
+
+	for (i = nloops; i >= 0; i--) {
+		this_cpu_inc(test_acqrel);
+		un_delay(udl, ndl);
+		this_cpu_dec(test_acqrel);
+	}
+}
+
+static const struct ref_scale_ops percpuinc_ops = {
+	.init		= rcu_sync_scale_init,
+	.readsection	= ref_percpuinc_section,
+	.delaysection	= ref_percpuinc_delay_section,
+	.name		= "percpuinc"
+};
+
 // Definitions for rwlock
 static rwlock_t test_rwlock;
 
@@ -501,9 +532,6 @@ static const struct ref_scale_ops lock_irq_ops = {
 	.name		= "lock-irq"
 };
 
-// Definitions acquire-release.
-static DEFINE_PER_CPU(unsigned long, test_acqrel);
-
 static void ref_acqrel_section(const int nloops)
 {
 	unsigned long x;
@@ -1298,7 +1326,7 @@ ref_scale_init(void)
 	static const struct ref_scale_ops *scale_ops[] = {
 		&rcu_ops, &srcu_ops, &srcu_fast_ops, RCU_TRACE_OPS RCU_TASKS_OPS
 		&refcnt_ops, &rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops,
-		&acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops,
+		&percpuinc_ops, &acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops,
 		&preempt_ops, &bh_ops, &irq_ops, &irqsave_ops,
 		&typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops,
 	};
-- 
2.40.1

[PATCH 6/8] refscale: Add non-atomic per-CPU increment readers

[Paul E. McKenney]

This commit adds refscale readers based on READ_ONCE() and WRITE_ONCE()
that are unprotected (can lose counts, "refscale.scale_type=incpercpu"),
preempt-disabled ("refscale.scale_type=incpercpupreempt"),
bh-disabled ("refscale.scale_type=incpercpubh"), and irq-disabled
("refscale.scale_type=incpercpuirqsave").  On my x86 laptop, these are
about 4.3ns, 3.8ns, and 7.3ns per pair, respectively.

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
---
 kernel/rcu/refscale.c | 155 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 153 insertions(+), 2 deletions(-)

diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c
index 479d86cad652..2b247355de40 100644
--- a/kernel/rcu/refscale.c
+++ b/kernel/rcu/refscale.c
@@ -389,6 +389,155 @@ static const struct ref_scale_ops percpuinc_ops = {
 	.name		= "percpuinc"
 };
 
+// Note that this can lose counts in preemptible kernels.
+static void ref_incpercpu_section(const int nloops)
+{
+	int i;
+
+	for (i = nloops; i >= 0; i--) {
+		unsigned long *tap = this_cpu_ptr(&test_acqrel);
+
+		WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);
+		WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);
+	}
+}
+
+static void ref_incpercpu_delay_section(const int nloops, const int udl, const int ndl)
+{
+	int i;
+
+	for (i = nloops; i >= 0; i--) {
+		unsigned long *tap = this_cpu_ptr(&test_acqrel);
+
+		WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);
+		un_delay(udl, ndl);
+		WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);
+	}
+}
+
+static const struct ref_scale_ops incpercpu_ops = {
+	.init		= rcu_sync_scale_init,
+	.readsection	= ref_incpercpu_section,
+	.delaysection	= ref_incpercpu_delay_section,
+	.name		= "incpercpu"
+};
+
+static void ref_incpercpupreempt_section(const int nloops)
+{
+	int i;
+
+	for (i = nloops; i >= 0; i--) {
+		unsigned long *tap;
+
+		preempt_disable();
+		tap = this_cpu_ptr(&test_acqrel);
+		WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);
+		WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);
+		preempt_enable();
+	}
+}
+
+static void ref_incpercpupreempt_delay_section(const int nloops, const int udl, const int ndl)
+{
+	int i;
+
+	for (i = nloops; i >= 0; i--) {
+		unsigned long *tap;
+
+		preempt_disable();
+		tap = this_cpu_ptr(&test_acqrel);
+		WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);
+		un_delay(udl, ndl);
+		WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);
+		preempt_enable();
+	}
+}
+
+static const struct ref_scale_ops incpercpupreempt_ops = {
+	.init		= rcu_sync_scale_init,
+	.readsection	= ref_incpercpupreempt_section,
+	.delaysection	= ref_incpercpupreempt_delay_section,
+	.name		= "incpercpupreempt"
+};
+
+static void ref_incpercpubh_section(const int nloops)
+{
+	int i;
+
+	for (i = nloops; i >= 0; i--) {
+		unsigned long *tap;
+
+		local_bh_disable();
+		tap = this_cpu_ptr(&test_acqrel);
+		WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);
+		WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);
+		local_bh_enable();
+	}
+}
+
+static void ref_incpercpubh_delay_section(const int nloops, const int udl, const int ndl)
+{
+	int i;
+
+	for (i = nloops; i >= 0; i--) {
+		unsigned long *tap;
+
+		local_bh_disable();
+		tap = this_cpu_ptr(&test_acqrel);
+		WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);
+		un_delay(udl, ndl);
+		WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);
+		local_bh_enable();
+	}
+}
+
+static const struct ref_scale_ops incpercpubh_ops = {
+	.init		= rcu_sync_scale_init,
+	.readsection	= ref_incpercpubh_section,
+	.delaysection	= ref_incpercpubh_delay_section,
+	.name		= "incpercpubh"
+};
+
+static void ref_incpercpuirqsave_section(const int nloops)
+{
+	int i;
+	unsigned long flags;
+
+	for (i = nloops; i >= 0; i--) {
+		unsigned long *tap;
+
+		local_irq_save(flags);
+		tap = this_cpu_ptr(&test_acqrel);
+		WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);
+		WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);
+		local_irq_restore(flags);
+	}
+}
+
+static void ref_incpercpuirqsave_delay_section(const int nloops, const int udl, const int ndl)
+{
+	int i;
+	unsigned long flags;
+
+	for (i = nloops; i >= 0; i--) {
+		unsigned long *tap;
+
+		local_irq_save(flags);
+		tap = this_cpu_ptr(&test_acqrel);
+		WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);
+		un_delay(udl, ndl);
+		WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);
+		local_irq_restore(flags);
+	}
+}
+
+static const struct ref_scale_ops incpercpuirqsave_ops = {
+	.init		= rcu_sync_scale_init,
+	.readsection	= ref_incpercpuirqsave_section,
+	.delaysection	= ref_incpercpuirqsave_delay_section,
+	.name		= "incpercpuirqsave"
+};
+
 // Definitions for rwlock
 static rwlock_t test_rwlock;
 
@@ -1325,8 +1474,10 @@ ref_scale_init(void)
 	int firsterr = 0;
 	static const struct ref_scale_ops *scale_ops[] = {
 		&rcu_ops, &srcu_ops, &srcu_fast_ops, RCU_TRACE_OPS RCU_TASKS_OPS
-		&refcnt_ops, &rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops,
-		&percpuinc_ops, &acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops,
+		&refcnt_ops, &percpuinc_ops, &incpercpu_ops, &incpercpupreempt_ops,
+		&incpercpubh_ops, &incpercpuirqsave_ops,
+		&rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops,
+		&sched_clock_ops, &clock_ops, &jiffies_ops,
 		&preempt_ops, &bh_ops, &irq_ops, &irqsave_ops,
 		&typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops,
 	};
-- 
2.40.1

[PATCH 7/8] refscale: Do not diable interrupts for tests involving local_bh_enable()

[Paul E. McKenney]

Some kernel configurations prohibit invoking local_bh_enable() while
interrupts are disabled.  However, refscale disables interrupts to reduce
OS noise during the tests, which results in splats.  This commit therefore
adds an ->enable_irqs flag to the ref_scale_ops structure, and refrains
from disabling interrupts when that flag is set.  This flag is set for
the "bh" and "incpercpubh" scale_type module-parameter values.

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
---
 kernel/rcu/refscale.c | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c
index 2b247355de40..7429ec9f0092 100644
--- a/kernel/rcu/refscale.c
+++ b/kernel/rcu/refscale.c
@@ -136,6 +136,7 @@ struct ref_scale_ops {
 	void (*cleanup)(void);
 	void (*readsection)(const int nloops);
 	void (*delaysection)(const int nloops, const int udl, const int ndl);
+	bool enable_irqs;
 	const char *name;
 };
 
@@ -495,6 +496,7 @@ static const struct ref_scale_ops incpercpubh_ops = {
 	.init		= rcu_sync_scale_init,
 	.readsection	= ref_incpercpubh_section,
 	.delaysection	= ref_incpercpubh_delay_section,
+	.enable_irqs	= true,
 	.name		= "incpercpubh"
 };
 
@@ -872,6 +874,7 @@ static void ref_bh_delay_section(const int nloops, const int udl, const int ndl)
 static const struct ref_scale_ops bh_ops = {
 	.readsection	= ref_bh_section,
 	.delaysection	= ref_bh_delay_section,
+	.enable_irqs	= true,
 	.name		= "bh"
 };
 
@@ -1234,15 +1237,18 @@ ref_scale_reader(void *arg)
 	if (!atomic_dec_return(&n_warmedup))
 		while (atomic_read_acquire(&n_warmedup))
 			rcu_scale_one_reader();
-	// Also keep interrupts disabled.  This also has the effect
-	// of preventing entries into slow path for rcu_read_unlock().
-	local_irq_save(flags);
+	// Also keep interrupts disabled when it is safe to do so, which
+	// it is not for local_bh_enable().  This also has the effect of
+	// preventing entries into slow path for rcu_read_unlock().
+	if (!cur_ops->enable_irqs)
+		local_irq_save(flags);
 	start = ktime_get_mono_fast_ns();
 
 	rcu_scale_one_reader();
 
 	duration = ktime_get_mono_fast_ns() - start;
-	local_irq_restore(flags);
+	if (!cur_ops->enable_irqs)
+		local_irq_restore(flags);
 
 	rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration;
 	// To reduce runtime-skew noise, do maintain-load invocations until
-- 
2.40.1

[PATCH 8/8] refscale: Add SRCU-fast-updown readers

[Paul E. McKenney]

This commit adds refscale readers based on srcu_read_lock_fast_updown()
and srcu_read_lock_fast_updown() ("refscale.scale_type=srcu-fast-updown").
On my x86 laptop, these are about 2.2ns per pair.

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: <bpf@vger.kernel.org>
---
 kernel/rcu/refscale.c | 40 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 39 insertions(+), 1 deletion(-)

diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c
index 7429ec9f0092..07a313782dfd 100644
--- a/kernel/rcu/refscale.c
+++ b/kernel/rcu/refscale.c
@@ -186,6 +186,7 @@ static const struct ref_scale_ops rcu_ops = {
 // Definitions for SRCU ref scale testing.
 DEFINE_STATIC_SRCU(srcu_refctl_scale);
 DEFINE_STATIC_SRCU_FAST(srcu_fast_refctl_scale);
+DEFINE_STATIC_SRCU_FAST_UPDOWN(srcu_fast_updown_refctl_scale);
 static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale;
 
 static void srcu_ref_scale_read_section(const int nloops)
@@ -254,6 +255,42 @@ static const struct ref_scale_ops srcu_fast_ops = {
 	.name		= "srcu-fast"
 };
 
+static bool srcu_fast_updown_sync_scale_init(void)
+{
+	srcu_ctlp = &srcu_fast_updown_refctl_scale;
+	return true;
+}
+
+static void srcu_fast_updown_ref_scale_read_section(const int nloops)
+{
+	int i;
+	struct srcu_ctr __percpu *scp;
+
+	for (i = nloops; i >= 0; i--) {
+		scp = srcu_read_lock_fast_updown(srcu_ctlp);
+		srcu_read_unlock_fast_updown(srcu_ctlp, scp);
+	}
+}
+
+static void srcu_fast_updown_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
+{
+	int i;
+	struct srcu_ctr __percpu *scp;
+
+	for (i = nloops; i >= 0; i--) {
+		scp = srcu_read_lock_fast_updown(srcu_ctlp);
+		un_delay(udl, ndl);
+		srcu_read_unlock_fast_updown(srcu_ctlp, scp);
+	}
+}
+
+static const struct ref_scale_ops srcu_fast_updown_ops = {
+	.init		= srcu_fast_updown_sync_scale_init,
+	.readsection	= srcu_fast_updown_ref_scale_read_section,
+	.delaysection	= srcu_fast_updown_ref_scale_delay_section,
+	.name		= "srcu-fast-updown"
+};
+
 #ifdef CONFIG_TASKS_RCU
 
 // Definitions for RCU Tasks ref scale testing: Empty read markers.
@@ -1479,7 +1516,8 @@ ref_scale_init(void)
 	long i;
 	int firsterr = 0;
 	static const struct ref_scale_ops *scale_ops[] = {
-		&rcu_ops, &srcu_ops, &srcu_fast_ops, RCU_TRACE_OPS RCU_TASKS_OPS
+		&rcu_ops, &srcu_ops, &srcu_fast_ops, &srcu_fast_updown_ops,
+		RCU_TRACE_OPS RCU_TASKS_OPS
 		&refcnt_ops, &percpuinc_ops, &incpercpu_ops, &incpercpupreempt_ops,
 		&incpercpubh_ops, &incpercpuirqsave_ops,
 		&rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops,
-- 
2.40.1

Re: [PATCH 0/8] refscale updates for v6.19

[Frederic Weisbecker]

Hi Paul,

Le Sun, Nov 02, 2025 at 02:49:34PM -0800, Paul E. McKenney a écrit :
> Hello!
> 
> This series contains additions of microbenchmarks to the refscale suite,
> and depends on the SRCU series.
> 
> 1.	Exercise DEFINE_STATIC_SRCU_FAST() and init_srcu_struct_fast().
> 
> 2.	Add local_irq_disable() and local_irq_save() readers.
> 
> 3.	Add local_bh_disable() readers.
> 
> 4.	Add preempt_disable() readers.
> 
> 5.	Add this_cpu_inc() readers.
> 
> 6.	Add non-atomic per-CPU increment readers.
> 
> 7.	Do not diable interrupts for tests involving local_bh_enable().
> 
> 8.	Add SRCU-fast-updown readers.

But this series depends on the "RCU Tasks Trace in terms of SRCU-fast"
patchset which isn't for 6.19, right?

-- 
Frederic Weisbecker
SUSE Labs

Re: [PATCH 0/8] refscale updates for v6.19

[Frederic Weisbecker]

Le Wed, Nov 05, 2025 at 11:56:09PM +0100, Frederic Weisbecker a écrit :
> Hi Paul,
> 
> Le Sun, Nov 02, 2025 at 02:49:34PM -0800, Paul E. McKenney a écrit :
> > Hello!
> > 
> > This series contains additions of microbenchmarks to the refscale suite,
> > and depends on the SRCU series.
> > 
> > 1.	Exercise DEFINE_STATIC_SRCU_FAST() and init_srcu_struct_fast().
> > 
> > 2.	Add local_irq_disable() and local_irq_save() readers.
> > 
> > 3.	Add local_bh_disable() readers.
> > 
> > 4.	Add preempt_disable() readers.
> > 
> > 5.	Add this_cpu_inc() readers.
> > 
> > 6.	Add non-atomic per-CPU increment readers.
> > 
> > 7.	Do not diable interrupts for tests involving local_bh_enable().
> > 
> > 8.	Add SRCU-fast-updown readers.
> 
> But this series depends on the "RCU Tasks Trace in terms of SRCU-fast"
> patchset which isn't for 6.19, right?

Ah it depends on the latest srcu posting, nevermind, got confused in my
inbox :-)

Thanks.

> 
> -- 
> Frederic Weisbecker
> SUSE Labs
> 

-- 
Frederic Weisbecker
SUSE Labs

Re: [PATCH 0/8] refscale updates for v6.19

[Paul E. McKenney]

On Thu, Nov 06, 2025 at 12:01:42AM +0100, Frederic Weisbecker wrote:
> Le Wed, Nov 05, 2025 at 11:56:09PM +0100, Frederic Weisbecker a écrit :
> > Hi Paul,
> > 
> > Le Sun, Nov 02, 2025 at 02:49:34PM -0800, Paul E. McKenney a écrit :
> > > Hello!
> > > 
> > > This series contains additions of microbenchmarks to the refscale suite,
> > > and depends on the SRCU series.
> > > 
> > > 1.	Exercise DEFINE_STATIC_SRCU_FAST() and init_srcu_struct_fast().
> > > 
> > > 2.	Add local_irq_disable() and local_irq_save() readers.
> > > 
> > > 3.	Add local_bh_disable() readers.
> > > 
> > > 4.	Add preempt_disable() readers.
> > > 
> > > 5.	Add this_cpu_inc() readers.
> > > 
> > > 6.	Add non-atomic per-CPU increment readers.
> > > 
> > > 7.	Do not diable interrupts for tests involving local_bh_enable().
> > > 
> > > 8.	Add SRCU-fast-updown readers.
> > 
> > But this series depends on the "RCU Tasks Trace in terms of SRCU-fast"
> > patchset which isn't for 6.19, right?
> 
> Ah it depends on the latest srcu posting, nevermind, got confused in my
> inbox :-)

I confess that my redistribution of that series was a bit abrupt,
apologies!

							Thanx, Paul

> Thanks.
> 
> > 
> > -- 
> > Frederic Weisbecker
> > SUSE Labs
> > 
> 
> -- 
> Frederic Weisbecker
> SUSE Labs

Re: [PATCH 3/8] refscale: Add local_bh_disable() readers

[Sebastian Andrzej Siewior]

On 2025-11-02 14:49:43 [-0800], Paul E. McKenney wrote:
> --- a/kernel/rcu/refscale.c
> +++ b/kernel/rcu/refscale.c
> @@ -636,6 +636,37 @@ static const struct ref_scale_ops jiffies_ops = {
>  	.name		= "jiffies"
>  };
>  
> +static void ref_bh_section(const int nloops)
> +{
> +	int i;
> +
> +	preempt_disable();
> +	for (i = nloops; i >= 0; i--) {
> +		local_bh_disable();

This (preempt off followed by bh off) may cause warnings. That would be
bh is disabled on the CPU, it gets preempted by _this_ which disables
first preemption and then bh. 
I hid the code under CONFIG_PREEMPT_RT_NEEDS_BH_LOCK so it shouldn't be
a problem in the long term I think. So just if you see a warning here
under RT you know why :)

> +		local_bh_enable();
> +	}
> +	preempt_enable();
> +}

Sebastian

Re: [PATCH 3/8] refscale: Add local_bh_disable() readers

[Paul E. McKenney]

On Tue, Nov 11, 2025 at 04:38:03PM +0100, Sebastian Andrzej Siewior wrote:
> On 2025-11-02 14:49:43 [-0800], Paul E. McKenney wrote:
> > --- a/kernel/rcu/refscale.c
> > +++ b/kernel/rcu/refscale.c
> > @@ -636,6 +636,37 @@ static const struct ref_scale_ops jiffies_ops = {
> >  	.name		= "jiffies"
> >  };
> >  
> > +static void ref_bh_section(const int nloops)
> > +{
> > +	int i;
> > +
> > +	preempt_disable();
> > +	for (i = nloops; i >= 0; i--) {
> > +		local_bh_disable();
> 
> This (preempt off followed by bh off) may cause warnings. That would be
> bh is disabled on the CPU, it gets preempted by _this_ which disables
> first preemption and then bh. 
> I hid the code under CONFIG_PREEMPT_RT_NEEDS_BH_LOCK so it shouldn't be
> a problem in the long term I think. So just if you see a warning here
> under RT you know why :)

Huh.  Would migrate_disable() be appropriate?  Or I suppose I could just
let it migrate on RT.  So how about the fix shown below?

							Thanx, Paul

> > +		local_bh_enable();
> > +	}
> > +	preempt_enable();
> > +}
> 
> Sebastian

------------------------------------------------------------------------

diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c
index 07a313782dfd..5a692a3b93fa 100644
--- a/kernel/rcu/refscale.c
+++ b/kernel/rcu/refscale.c
@@ -887,25 +887,29 @@ static void ref_bh_section(const int nloops)
 {
 	int i;
 
-	preempt_disable();
+	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
+		preempt_disable();
 	for (i = nloops; i >= 0; i--) {
 		local_bh_disable();
 		local_bh_enable();
 	}
-	preempt_enable();
+	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
+		preempt_enable();
 }
 
 static void ref_bh_delay_section(const int nloops, const int udl, const int ndl)
 {
 	int i;
 
-	preempt_disable();
+	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
+		preempt_disable();
 	for (i = nloops; i >= 0; i--) {
 		local_bh_disable();
 		un_delay(udl, ndl);
 		local_bh_enable();
 	}
-	preempt_enable();
+	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
+		preempt_enable();
 }
 
 static const struct ref_scale_ops bh_ops = {

Re: [PATCH 3/8] refscale: Add local_bh_disable() readers

[Sebastian Andrzej Siewior]

On 2025-11-11 11:21:04 [-0800], Paul E. McKenney wrote:
> On Tue, Nov 11, 2025 at 04:38:03PM +0100, Sebastian Andrzej Siewior wrote:
> > On 2025-11-02 14:49:43 [-0800], Paul E. McKenney wrote:
> > > --- a/kernel/rcu/refscale.c
> > > +++ b/kernel/rcu/refscale.c
> > > @@ -636,6 +636,37 @@ static const struct ref_scale_ops jiffies_ops = {
> > >  	.name		= "jiffies"
> > >  };
> > >  
> > > +static void ref_bh_section(const int nloops)
> > > +{
> > > +	int i;
> > > +
> > > +	preempt_disable();
> > > +	for (i = nloops; i >= 0; i--) {
> > > +		local_bh_disable();
> > 
> > This (preempt off followed by bh off) may cause warnings. That would be
> > bh is disabled on the CPU, it gets preempted by _this_ which disables
> > first preemption and then bh. 
> > I hid the code under CONFIG_PREEMPT_RT_NEEDS_BH_LOCK so it shouldn't be
> > a problem in the long term I think. So just if you see a warning here
> > under RT you know why :)
> 
> Huh.  Would migrate_disable() be appropriate?  Or I suppose I could just
> let it migrate on RT.  So how about the fix shown below?

Depends on what you want to achieve. Even with that bh-disable you can
be preempted but you can't migrate to another CPU.
That preempt-disable() will keep the RCU read section open during
bh-disable/ enable but migrate_disable() won't. But this not something I
need to explain to you ;) 
If that (to be within a RCU read section) is you intention you could
explicitly add a rcu_read_lock() there.
The change you suggested won't have the problem I mentioned.

> 							Thanx, Paul

Sebastian

Re: [PATCH 3/8] refscale: Add local_bh_disable() readers

[Paul E. McKenney]

On Wed, Nov 12, 2025 at 10:14:52AM +0100, Sebastian Andrzej Siewior wrote:
> On 2025-11-11 11:21:04 [-0800], Paul E. McKenney wrote:
> > On Tue, Nov 11, 2025 at 04:38:03PM +0100, Sebastian Andrzej Siewior wrote:
> > > On 2025-11-02 14:49:43 [-0800], Paul E. McKenney wrote:
> > > > --- a/kernel/rcu/refscale.c
> > > > +++ b/kernel/rcu/refscale.c
> > > > @@ -636,6 +636,37 @@ static const struct ref_scale_ops jiffies_ops = {
> > > >  	.name		= "jiffies"
> > > >  };
> > > >  
> > > > +static void ref_bh_section(const int nloops)
> > > > +{
> > > > +	int i;
> > > > +
> > > > +	preempt_disable();
> > > > +	for (i = nloops; i >= 0; i--) {
> > > > +		local_bh_disable();
> > > 
> > > This (preempt off followed by bh off) may cause warnings. That would be
> > > bh is disabled on the CPU, it gets preempted by _this_ which disables
> > > first preemption and then bh. 
> > > I hid the code under CONFIG_PREEMPT_RT_NEEDS_BH_LOCK so it shouldn't be
> > > a problem in the long term I think. So just if you see a warning here
> > > under RT you know why :)
> > 
> > Huh.  Would migrate_disable() be appropriate?  Or I suppose I could just
> > let it migrate on RT.  So how about the fix shown below?
> 
> Depends on what you want to achieve. Even with that bh-disable you can
> be preempted but you can't migrate to another CPU.

Mostly just trying to measure the overhead of a local_bh_disable()
and local_bh_enable() pair.  Yes, I understand that this is a bit
iffy these days, but it at least gets us some indication of problems
like this one:
https://lore.kernel.org/all/e7d539ed-ced0-4b96-8ecd-048a5b803b85@paulmck-laptop/

> That preempt-disable() will keep the RCU read section open during
> bh-disable/ enable but migrate_disable() won't. But this not something I
> need to explain to you ;) 

Not necessary most of the time, anyway.  ;-)

> If that (to be within a RCU read section) is you intention you could
> explicitly add a rcu_read_lock() there.
> The change you suggested won't have the problem I mentioned.

Very good, thank you!  I am keeping it as an "EXP" (as in "experimental")
commit in my -rcu tree for the time being, given that it sounds like
you are looking to fix this within -rt.

							Thanx, Paul