* [PATCH] lib/percpu_counter: allow concurrent sums with an rwlock
@ 2026-11-16 1:56 Jiebin Sun
2026-11-16 3:16 ` Andrew Morton
2026-11-16 7:30 ` Sebastian Andrzej Siewior
0 siblings, 2 replies; 6+ messages in thread
From: Jiebin Sun @ 2026-11-16 1:56 UTC (permalink / raw)
To: akpm
Cc: dennis, tj, cl, linux-mm, linux-kernel, bigeasy, clrkwllms,
rostedt, linux-rt-devel, tim.c.chen, tianyou.li, wangyang.guo,
jiebin.sun
__percpu_counter_sum() holds a raw spinlock while reading the global
count and per-CPU deltas, serializing concurrent sums of the same
counter. Since a sum changes neither the global count nor the per-CPU
deltas, use rwlock_t on !PREEMPT_RT to allow concurrent sums with a
read lock. Global updates take the write lock. The per-CPU fast path
remains lockless, so sums keep their existing snapshot semantics.
stress-ng dentry reaches this path through ext4_statfs():
statfs -> ext4_statfs -> __percpu_counter_sum
-> native_queued_spin_lock_slowpath
Keep the raw spinlock on PREEMPT_RT, where rwlock_t can sleep.
Global counter updates can run in atomic context and must not sleep.
stress-ng dentry test average throughput change:
+------------------------+------------+--------------------+
| Case | Unit | Average change (%) |
+------------------------+------------+--------------------+
| Dentry, 64 workers | bogo ops/s | +47.34 |
+------------------------+------------+--------------------+
Dentry ran on ext4 with 64 workers pinned to physical CPUs:
stress-ng --dentry 64 --dentries 4096 --timeout 20s
Suggested-by: Tim Chen <tim.c.chen@linux.intel.com>
Reviewed-by: Tianyou Li <tianyou.li@intel.com>
Reviewed-by: Wangyang Guo <wangyang.guo@intel.com>
Signed-off-by: Jiebin Sun <jiebin.sun@intel.com>
---
include/linux/percpu_counter.h | 4 ++
lib/percpu_counter.c | 71 ++++++++++++++++++++++++++--------
2 files changed, 59 insertions(+), 16 deletions(-)
diff --git a/include/linux/percpu_counter.h b/include/linux/percpu_counter.h
index 3a44dd1e33d2..7b28b0e22207 100644
--- a/include/linux/percpu_counter.h
+++ b/include/linux/percpu_counter.h
@@ -20,7 +20,11 @@
#ifdef CONFIG_SMP
struct percpu_counter {
+#ifdef CONFIG_PREEMPT_RT
raw_spinlock_t lock;
+#else
+ rwlock_t lock; /* shared for sums, exclusive for updates */
+#endif
s64 count;
#ifdef CONFIG_HOTPLUG_CPU
struct list_head list; /* All percpu_counters are on a list */
diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c
index 87b261b87b0b..ce0284bacf07 100644
--- a/lib/percpu_counter.c
+++ b/lib/percpu_counter.c
@@ -15,6 +15,46 @@ static LIST_HEAD(percpu_counters);
static DEFINE_SPINLOCK(percpu_counters_lock);
#endif
+/*
+ * Non-RT sums share the lock; global updates take it exclusively.
+ * Keep raw spinlocks on RT: rwlocks may sleep and their irqsave variants
+ * do not disable interrupts. Updates run in atomic contexts and need
+ * IRQ exclusion for CPU-local read-modify-write sequences.
+ */
+#ifdef CONFIG_PREEMPT_RT
+#define fbc_lock(fbc) raw_spin_lock(&(fbc)->lock)
+#define fbc_unlock(fbc) raw_spin_unlock(&(fbc)->lock)
+#define fbc_lock_irqsave(fbc, f) raw_spin_lock_irqsave(&(fbc)->lock, f)
+#define fbc_unlock_irqrestore(fbc, f) raw_spin_unlock_irqrestore(&(fbc)->lock, f)
+#define fbc_read_lock_irqsave fbc_lock_irqsave
+#define fbc_read_unlock_irqrestore fbc_unlock_irqrestore
+#else
+#define fbc_lock(fbc) write_lock(&(fbc)->lock)
+#define fbc_unlock(fbc) write_unlock(&(fbc)->lock)
+#define fbc_lock_irqsave(fbc, f) write_lock_irqsave(&(fbc)->lock, f)
+#define fbc_unlock_irqrestore(fbc, f) write_unlock_irqrestore(&(fbc)->lock, f)
+#define fbc_read_lock_irqsave(fbc, f) read_lock_irqsave(&(fbc)->lock, f)
+#define fbc_read_unlock_irqrestore(fbc, f) read_unlock_irqrestore(&(fbc)->lock, f)
+#endif
+
+static void fbc_lock_init(struct percpu_counter *fbc,
+ struct lock_class_key *key)
+{
+#ifdef CONFIG_PREEMPT_RT
+ raw_spin_lock_init(&fbc->lock);
+ lockdep_set_class(&fbc->lock, key);
+#else
+ rwlock_init(&fbc->lock);
+ /*
+ * rwlock_t normally uses LD_WAIT_CONFIG because it sleeps on RT.
+ * This counter uses a raw spinlock on RT, so it can be taken under
+ * raw spinlocks or with IRQs off on every configuration.
+ */
+ lockdep_init_map_wait(&fbc->lock.dep_map, "&fbc->lock", key, 0,
+ LD_WAIT_SPIN);
+#endif
+}
+
#ifdef CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER
static const struct debug_obj_descr percpu_counter_debug_descr;
@@ -62,13 +102,13 @@ void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
int cpu;
unsigned long flags;
- raw_spin_lock_irqsave(&fbc->lock, flags);
+ fbc_lock_irqsave(fbc, flags);
for_each_possible_cpu(cpu) {
s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
*pcount = 0;
}
fbc->count = amount;
- raw_spin_unlock_irqrestore(&fbc->lock, flags);
+ fbc_unlock_irqrestore(fbc, flags);
}
EXPORT_SYMBOL(percpu_counter_set);
@@ -98,7 +138,7 @@ void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)
count = this_cpu_read(*fbc->counters);
do {
if (unlikely(abs(count + amount) >= batch)) {
- raw_spin_lock_irqsave(&fbc->lock, flags);
+ fbc_lock_irqsave(fbc, flags);
/*
* Note: by now we might have migrated to another CPU
* or the value might have changed.
@@ -106,7 +146,7 @@ void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)
count = __this_cpu_read(*fbc->counters);
fbc->count += count + amount;
__this_cpu_sub(*fbc->counters, count);
- raw_spin_unlock_irqrestore(&fbc->lock, flags);
+ fbc_unlock_irqrestore(fbc, flags);
return;
}
} while (!this_cpu_try_cmpxchg(*fbc->counters, &count, count + amount));
@@ -125,10 +165,10 @@ void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)
local_irq_save(flags);
count = __this_cpu_read(*fbc->counters) + amount;
if (abs(count) >= batch) {
- raw_spin_lock(&fbc->lock);
+ fbc_lock(fbc);
fbc->count += count;
__this_cpu_sub(*fbc->counters, count - amount);
- raw_spin_unlock(&fbc->lock);
+ fbc_unlock(fbc);
} else {
this_cpu_add(*fbc->counters, amount);
}
@@ -148,11 +188,11 @@ void percpu_counter_sync(struct percpu_counter *fbc)
unsigned long flags;
s64 count;
- raw_spin_lock_irqsave(&fbc->lock, flags);
+ fbc_lock_irqsave(fbc, flags);
count = __this_cpu_read(*fbc->counters);
fbc->count += count;
__this_cpu_sub(*fbc->counters, count);
- raw_spin_unlock_irqrestore(&fbc->lock, flags);
+ fbc_unlock_irqrestore(fbc, flags);
}
EXPORT_SYMBOL(percpu_counter_sync);
@@ -174,13 +214,13 @@ s64 __percpu_counter_sum(struct percpu_counter *fbc)
int cpu;
unsigned long flags;
- raw_spin_lock_irqsave(&fbc->lock, flags);
+ fbc_read_lock_irqsave(fbc, flags);
ret = fbc->count;
for_each_cpu_or(cpu, cpu_online_mask, cpu_dying_mask) {
s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
ret += *pcount;
}
- raw_spin_unlock_irqrestore(&fbc->lock, flags);
+ fbc_read_unlock_irqrestore(fbc, flags);
return ret;
}
EXPORT_SYMBOL(__percpu_counter_sum);
@@ -203,8 +243,7 @@ int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount,
}
for (i = 0; i < nr_counters; i++) {
- raw_spin_lock_init(&fbc[i].lock);
- lockdep_set_class(&fbc[i].lock, key);
+ fbc_lock_init(&fbc[i], key);
#ifdef CONFIG_HOTPLUG_CPU
INIT_LIST_HEAD(&fbc[i].list);
#endif
@@ -274,11 +313,11 @@ static int percpu_counter_cpu_dead(unsigned int cpu)
list_for_each_entry(fbc, &percpu_counters, list) {
s32 *pcount;
- raw_spin_lock(&fbc->lock);
+ fbc_lock(fbc);
pcount = per_cpu_ptr(fbc->counters, cpu);
fbc->count += *pcount;
*pcount = 0;
- raw_spin_unlock(&fbc->lock);
+ fbc_unlock(fbc);
}
spin_unlock_irq(&percpu_counters_lock);
#endif
@@ -354,7 +393,7 @@ bool __percpu_counter_limited_add(struct percpu_counter *fbc,
return true;
}
- raw_spin_lock(&fbc->lock);
+ fbc_lock(fbc);
count = fbc->count + amount;
/* Skip percpu_counter_sum() when safe */
@@ -392,7 +431,7 @@ bool __percpu_counter_limited_add(struct percpu_counter *fbc,
fbc->count += count + amount;
__this_cpu_sub(*fbc->counters, count);
out:
- raw_spin_unlock(&fbc->lock);
+ fbc_unlock(fbc);
local_irq_restore(flags);
return good;
}
--
2.52.0
^ permalink raw reply [flat|nested] 6+ messages in thread* Re: [PATCH] lib/percpu_counter: allow concurrent sums with an rwlock
2026-11-16 1:56 [PATCH] lib/percpu_counter: allow concurrent sums with an rwlock Jiebin Sun
@ 2026-11-16 3:16 ` Andrew Morton
2026-11-16 3:44 ` Andrew Morton
2026-11-18 10:16 ` Jiebin Sun
2026-11-16 7:30 ` Sebastian Andrzej Siewior
1 sibling, 2 replies; 6+ messages in thread
From: Andrew Morton @ 2026-11-16 3:16 UTC (permalink / raw)
To: Jiebin Sun
Cc: dennis, tj, cl, linux-mm, linux-kernel, bigeasy, clrkwllms,
rostedt, linux-rt-devel, tim.c.chen, tianyou.li, wangyang.guo
On Mon, 16 Nov 2026 09:56:55 +0800 Jiebin Sun <jiebin.sun@intel.com> wrote:
> __percpu_counter_sum() holds a raw spinlock while reading the global
> count and per-CPU deltas, serializing concurrent sums of the same
> counter. Since a sum changes neither the global count nor the per-CPU
> deltas, use rwlock_t on !PREEMPT_RT to allow concurrent sums with a
> read lock. Global updates take the write lock. The per-CPU fast path
> remains lockless, so sums keep their existing snapshot semantics.
>
> stress-ng dentry reaches this path through ext4_statfs():
>
> statfs -> ext4_statfs -> __percpu_counter_sum
> -> native_queued_spin_lock_slowpath
>
> Keep the raw spinlock on PREEMPT_RT, where rwlock_t can sleep.
> Global counter updates can run in atomic context and must not sleep.
>
> stress-ng dentry test average throughput change:
>
> +------------------------+------------+--------------------+
> | Case | Unit | Average change (%) |
> +------------------------+------------+--------------------+
> | Dentry, 64 workers | bogo ops/s | +47.34 |
> +------------------------+------------+--------------------+
>
> Dentry ran on ext4 with 64 workers pinned to physical CPUs:
>
> stress-ng --dentry 64 --dentries 4096 --timeout 20s
lol, this is a fun patch - it reminds me of the old days. All those
mission-critical workloads which sit in a loop running statfs will love
it!
I had $sidekick do a scan and it told me "the only plausible routine
win is dirty throttling: with low dirty limits, concurrent writers sum
the same counters. ext4 and XFS sum near ENOSPC, where updates also
contend. BFQ's sums are debug-only and serialized. Most other callers
report statistics". You might want to add something like this to
changelog.
Also mention that percpu_counter_sum_positive callers will benefit.
> --- a/include/linux/percpu_counter.h
> +++ b/include/linux/percpu_counter.h
> @@ -20,7 +20,11 @@
> #ifdef CONFIG_SMP
>
> struct percpu_counter {
> +#ifdef CONFIG_PREEMPT_RT
Might be worth a wee comment telling readers why RT is special?
> raw_spinlock_t lock;
> +#else
> + rwlock_t lock; /* shared for sums, exclusive for updates */
> +#endif
> s64 count;
> #ifdef CONFIG_HOTPLUG_CPU
> struct list_head list; /* All percpu_counters are on a list */
>
> --- a/lib/percpu_counter.c
> +++ b/lib/percpu_counter.c
> @@ -15,6 +15,46 @@ static LIST_HEAD(percpu_counters);
> static DEFINE_SPINLOCK(percpu_counters_lock);
> #endif
>
> +/*
> + * Non-RT sums share the lock; global updates take it exclusively.
> + * Keep raw spinlocks on RT: rwlocks may sleep and their irqsave variants
> + * do not disable interrupts. Updates run in atomic contexts and need
> + * IRQ exclusion for CPU-local read-modify-write sequences.
> + */
> +#ifdef CONFIG_PREEMPT_RT
> +#define fbc_lock(fbc) raw_spin_lock(&(fbc)->lock)
> +#define fbc_unlock(fbc) raw_spin_unlock(&(fbc)->lock)
> +#define fbc_lock_irqsave(fbc, f) raw_spin_lock_irqsave(&(fbc)->lock, f)
> +#define fbc_unlock_irqrestore(fbc, f) raw_spin_unlock_irqrestore(&(fbc)->lock, f)
> +#define fbc_read_lock_irqsave fbc_lock_irqsave
> +#define fbc_read_unlock_irqrestore fbc_unlock_irqrestore
> +#else
> +#define fbc_lock(fbc) write_lock(&(fbc)->lock)
> +#define fbc_unlock(fbc) write_unlock(&(fbc)->lock)
> +#define fbc_lock_irqsave(fbc, f) write_lock_irqsave(&(fbc)->lock, f)
> +#define fbc_unlock_irqrestore(fbc, f) write_unlock_irqrestore(&(fbc)->lock, f)
> +#define fbc_read_lock_irqsave(fbc, f) read_lock_irqsave(&(fbc)->lock, f)
> +#define fbc_read_unlock_irqrestore(fbc, f) read_unlock_irqrestore(&(fbc)->lock, f)
> +#endif
Please, no!
https://www.google.com/search?q=why+does+Andrew+Morton+prefer+static+inlines+over+macros
lots of good stuff there ;)
^ permalink raw reply [flat|nested] 6+ messages in thread* Re: [PATCH] lib/percpu_counter: allow concurrent sums with an rwlock
2026-11-16 3:16 ` Andrew Morton
@ 2026-11-16 3:44 ` Andrew Morton
2026-11-16 7:41 ` Sebastian Andrzej Siewior
2026-11-18 10:16 ` Jiebin Sun
1 sibling, 1 reply; 6+ messages in thread
From: Andrew Morton @ 2026-11-16 3:44 UTC (permalink / raw)
To: Jiebin Sun, dennis, tj, cl, linux-mm, linux-kernel, bigeasy,
clrkwllms, rostedt, linux-rt-devel, tim.c.chen, tianyou.li,
wangyang.guo
On Sun, 15 Nov 2026 19:16:36 -0800 Andrew Morton <akpm@linux-foundation.org> wrote:
> > stress-ng dentry test average throughput change:
> >
> > +------------------------+------------+--------------------+
> > | Case | Unit | Average change (%) |
> > +------------------------+------------+--------------------+
> > | Dentry, 64 workers | bogo ops/s | +47.34 |
> > +------------------------+------------+--------------------+
> >
> > Dentry ran on ext4 with 64 workers pinned to physical CPUs:
> >
> > stress-ng --dentry 64 --dentries 4096 --timeout 20s
>
> lol, this is a fun patch
It's *probably* OK, for sane workloads.
But if some workload is busily running percpu_counter_sum on 1000 CPUs,
how is a writer ever to get in there to spill ->counters into ->count?
Modern rwlock_t does implement queuing, so the writer will get in
there eventually. But nothing is simple any more - there are
complications with rwlocks on PREEMPT_RT and with interrupt-time reads
(do we even do that?).
So I think indefinite writer starvation is a risk here?
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [PATCH] lib/percpu_counter: allow concurrent sums with an rwlock
2026-11-16 3:44 ` Andrew Morton
@ 2026-11-16 7:41 ` Sebastian Andrzej Siewior
0 siblings, 0 replies; 6+ messages in thread
From: Sebastian Andrzej Siewior @ 2026-11-16 7:41 UTC (permalink / raw)
To: Andrew Morton
Cc: Jiebin Sun, dennis, tj, cl, linux-mm, linux-kernel, clrkwllms,
rostedt, linux-rt-devel, tim.c.chen, tianyou.li, wangyang.guo
On 2026-11-15 19:44:04 [-0800], Andrew Morton wrote:
> Modern rwlock_t does implement queuing, so the writer will get in
> there eventually.
On the queued-rwlock implementation, if a writer blocks on the lock then
the readers will wait until the writer is done unless the reader is from
interrupt.
There is also seqcount_t where the reader would be fine as long as there
has been no writer during the read and plenty of readers don't block
each other. It might take a while if the writer performs multiple
updates and the reader can't query all CPUs in time.
So it is not easy.
> But nothing is simple any more - there are
> complications with rwlocks on PREEMPT_RT and with interrupt-time reads
> (do we even do that?).
There are a few users from preempt/irq disabled section on PREEMPT_RT
which mandate something like raw_spinlock_t. The rwlock_t will break.
Sebastian
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [PATCH] lib/percpu_counter: allow concurrent sums with an rwlock
2026-11-16 3:16 ` Andrew Morton
2026-11-16 3:44 ` Andrew Morton
@ 2026-11-18 10:16 ` Jiebin Sun
1 sibling, 0 replies; 6+ messages in thread
From: Jiebin Sun @ 2026-11-18 10:16 UTC (permalink / raw)
To: akpm, bigeasy
Cc: dennis, tj, cl, linux-mm, linux-kernel, clrkwllms, rostedt,
linux-rt-devel, tim.c.chen, tianyou.li, wangyang.guo, jiebin.sun
Hi Andrew and Sebastian,
Thanks for the comments and the RT review.
I'll replace the macros with static inline helpers, add the RT comment,
and clarify the practical callers, including percpu_counter_sum_positive(),
in v2.
Interrupt readers can bypass a WAITING writer and delay it if they keep
overlapping. I'm considering the impact this could have on counter
updates. If this needs addressing, we might take the write lock in
__percpu_counter_sum() when called from interrupt context, while keeping
the read lock for ordinary sums.
I also tested buffered writes with fio.
Fio (64 jobs, 3 iterations, 60s each):
+------+------------+-----------------+---------+
| Test | Throughput | CPU utilization | Cycles |
+------+------------+-----------------+---------+
| Fio | +6.91% | -40.41% | -40.80% |
+------+------------+-----------------+---------+
Test configuration: ext4, 32 GiB cgroup memory limit.
Commands:
sysctl -w vm.dirty_bytes=67108864 \
vm.dirty_background_bytes=33554432
fio --name=buffered-write --directory="$TEST_DIR" \
--ioengine=sync --rw=write --bs=64k --size=256m --numjobs=64 \
--direct=0 --fallocate=none --time_based=1 --runtime=60 \
--cpus_allowed=0-31,86-117 --cpus_allowed_policy=split \
--group_reporting --output-format=json
Thanks,
Jiebin
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [PATCH] lib/percpu_counter: allow concurrent sums with an rwlock
2026-11-16 1:56 [PATCH] lib/percpu_counter: allow concurrent sums with an rwlock Jiebin Sun
2026-11-16 3:16 ` Andrew Morton
@ 2026-11-16 7:30 ` Sebastian Andrzej Siewior
1 sibling, 0 replies; 6+ messages in thread
From: Sebastian Andrzej Siewior @ 2026-11-16 7:30 UTC (permalink / raw)
To: Jiebin Sun
Cc: akpm, dennis, tj, cl, linux-mm, linux-kernel, clrkwllms, rostedt,
linux-rt-devel, tim.c.chen, tianyou.li, wangyang.guo
On 2026-11-16 09:56:55 [+0800], Jiebin Sun wrote:
> __percpu_counter_sum() holds a raw spinlock while reading the global
> count and per-CPU deltas, serializing concurrent sums of the same
> counter. Since a sum changes neither the global count nor the per-CPU
> deltas, use rwlock_t on !PREEMPT_RT to allow concurrent sums with a
> read lock. Global updates take the write lock. The per-CPU fast path
> remains lockless, so sums keep their existing snapshot semantics.
>
> stress-ng dentry reaches this path through ext4_statfs():
>
> statfs -> ext4_statfs -> __percpu_counter_sum
> -> native_queued_spin_lock_slowpath
>
> Keep the raw spinlock on PREEMPT_RT, where rwlock_t can sleep.
> Global counter updates can run in atomic context and must not sleep.
>
> stress-ng dentry test average throughput change:
>
> +------------------------+------------+--------------------+
> | Case | Unit | Average change (%) |
> +------------------------+------------+--------------------+
> | Dentry, 64 workers | bogo ops/s | +47.34 |
> +------------------------+------------+--------------------+
>
> Dentry ran on ext4 with 64 workers pinned to physical CPUs:
>
> stress-ng --dentry 64 --dentries 4096 --timeout 20s
>
> Suggested-by: Tim Chen <tim.c.chen@linux.intel.com>
> Reviewed-by: Tianyou Li <tianyou.li@intel.com>
> Reviewed-by: Wangyang Guo <wangyang.guo@intel.com>
> Signed-off-by: Jiebin Sun <jiebin.sun@intel.com>
It looks okay from RT point of view. Is this limited to this benchmark
or is there a real workload behind this?
Sebastian
^ permalink raw reply [flat|nested] 6+ messages in thread
end of thread, other threads:[~2026-11-18 10:12 UTC | newest]
Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2026-11-16 1:56 [PATCH] lib/percpu_counter: allow concurrent sums with an rwlock Jiebin Sun
2026-11-16 3:16 ` Andrew Morton
2026-11-16 3:44 ` Andrew Morton
2026-11-16 7:41 ` Sebastian Andrzej Siewior
2026-11-18 10:16 ` Jiebin Sun
2026-11-16 7:30 ` Sebastian Andrzej Siewior
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®