* [PATCH] lib/percpu_counter: allow concurrent sums with an rwlock
@ 2026-11-16 1:56 Jiebin Sun
2026-11-16 3:16 ` Andrew Morton
0 siblings, 1 reply; 3+ messages in thread
From: Jiebin Sun @ 2026-11-16 1:56 UTC (permalink / raw)
To: akpm
Cc: dennis, tj, cl, linux-mm, linux-kernel, bigeasy, clrkwllms,
rostedt, linux-rt-devel, tim.c.chen, tianyou.li, wangyang.guo,
jiebin.sun
__percpu_counter_sum() holds a raw spinlock while reading the global
count and per-CPU deltas, serializing concurrent sums of the same
counter. Since a sum changes neither the global count nor the per-CPU
deltas, use rwlock_t on !PREEMPT_RT to allow concurrent sums with a
read lock. Global updates take the write lock. The per-CPU fast path
remains lockless, so sums keep their existing snapshot semantics.
stress-ng dentry reaches this path through ext4_statfs():
statfs -> ext4_statfs -> __percpu_counter_sum
-> native_queued_spin_lock_slowpath
Keep the raw spinlock on PREEMPT_RT, where rwlock_t can sleep.
Global counter updates can run in atomic context and must not sleep.
stress-ng dentry test average throughput change:
+------------------------+------------+--------------------+
| Case | Unit | Average change (%) |
+------------------------+------------+--------------------+
| Dentry, 64 workers | bogo ops/s | +47.34 |
+------------------------+------------+--------------------+
Dentry ran on ext4 with 64 workers pinned to physical CPUs:
stress-ng --dentry 64 --dentries 4096 --timeout 20s
Suggested-by: Tim Chen <tim.c.chen@linux.intel.com>
Reviewed-by: Tianyou Li <tianyou.li@intel.com>
Reviewed-by: Wangyang Guo <wangyang.guo@intel.com>
Signed-off-by: Jiebin Sun <jiebin.sun@intel.com>
---
include/linux/percpu_counter.h | 4 ++
lib/percpu_counter.c | 71 ++++++++++++++++++++++++++--------
2 files changed, 59 insertions(+), 16 deletions(-)
diff --git a/include/linux/percpu_counter.h b/include/linux/percpu_counter.h
index 3a44dd1e33d2..7b28b0e22207 100644
--- a/include/linux/percpu_counter.h
+++ b/include/linux/percpu_counter.h
@@ -20,7 +20,11 @@
#ifdef CONFIG_SMP
struct percpu_counter {
+#ifdef CONFIG_PREEMPT_RT
raw_spinlock_t lock;
+#else
+ rwlock_t lock; /* shared for sums, exclusive for updates */
+#endif
s64 count;
#ifdef CONFIG_HOTPLUG_CPU
struct list_head list; /* All percpu_counters are on a list */
diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c
index 87b261b87b0b..ce0284bacf07 100644
--- a/lib/percpu_counter.c
+++ b/lib/percpu_counter.c
@@ -15,6 +15,46 @@ static LIST_HEAD(percpu_counters);
static DEFINE_SPINLOCK(percpu_counters_lock);
#endif
+/*
+ * Non-RT sums share the lock; global updates take it exclusively.
+ * Keep raw spinlocks on RT: rwlocks may sleep and their irqsave variants
+ * do not disable interrupts. Updates run in atomic contexts and need
+ * IRQ exclusion for CPU-local read-modify-write sequences.
+ */
+#ifdef CONFIG_PREEMPT_RT
+#define fbc_lock(fbc) raw_spin_lock(&(fbc)->lock)
+#define fbc_unlock(fbc) raw_spin_unlock(&(fbc)->lock)
+#define fbc_lock_irqsave(fbc, f) raw_spin_lock_irqsave(&(fbc)->lock, f)
+#define fbc_unlock_irqrestore(fbc, f) raw_spin_unlock_irqrestore(&(fbc)->lock, f)
+#define fbc_read_lock_irqsave fbc_lock_irqsave
+#define fbc_read_unlock_irqrestore fbc_unlock_irqrestore
+#else
+#define fbc_lock(fbc) write_lock(&(fbc)->lock)
+#define fbc_unlock(fbc) write_unlock(&(fbc)->lock)
+#define fbc_lock_irqsave(fbc, f) write_lock_irqsave(&(fbc)->lock, f)
+#define fbc_unlock_irqrestore(fbc, f) write_unlock_irqrestore(&(fbc)->lock, f)
+#define fbc_read_lock_irqsave(fbc, f) read_lock_irqsave(&(fbc)->lock, f)
+#define fbc_read_unlock_irqrestore(fbc, f) read_unlock_irqrestore(&(fbc)->lock, f)
+#endif
+
+static void fbc_lock_init(struct percpu_counter *fbc,
+ struct lock_class_key *key)
+{
+#ifdef CONFIG_PREEMPT_RT
+ raw_spin_lock_init(&fbc->lock);
+ lockdep_set_class(&fbc->lock, key);
+#else
+ rwlock_init(&fbc->lock);
+ /*
+ * rwlock_t normally uses LD_WAIT_CONFIG because it sleeps on RT.
+ * This counter uses a raw spinlock on RT, so it can be taken under
+ * raw spinlocks or with IRQs off on every configuration.
+ */
+ lockdep_init_map_wait(&fbc->lock.dep_map, "&fbc->lock", key, 0,
+ LD_WAIT_SPIN);
+#endif
+}
+
#ifdef CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER
static const struct debug_obj_descr percpu_counter_debug_descr;
@@ -62,13 +102,13 @@ void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
int cpu;
unsigned long flags;
- raw_spin_lock_irqsave(&fbc->lock, flags);
+ fbc_lock_irqsave(fbc, flags);
for_each_possible_cpu(cpu) {
s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
*pcount = 0;
}
fbc->count = amount;
- raw_spin_unlock_irqrestore(&fbc->lock, flags);
+ fbc_unlock_irqrestore(fbc, flags);
}
EXPORT_SYMBOL(percpu_counter_set);
@@ -98,7 +138,7 @@ void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)
count = this_cpu_read(*fbc->counters);
do {
if (unlikely(abs(count + amount) >= batch)) {
- raw_spin_lock_irqsave(&fbc->lock, flags);
+ fbc_lock_irqsave(fbc, flags);
/*
* Note: by now we might have migrated to another CPU
* or the value might have changed.
@@ -106,7 +146,7 @@ void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)
count = __this_cpu_read(*fbc->counters);
fbc->count += count + amount;
__this_cpu_sub(*fbc->counters, count);
- raw_spin_unlock_irqrestore(&fbc->lock, flags);
+ fbc_unlock_irqrestore(fbc, flags);
return;
}
} while (!this_cpu_try_cmpxchg(*fbc->counters, &count, count + amount));
@@ -125,10 +165,10 @@ void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)
local_irq_save(flags);
count = __this_cpu_read(*fbc->counters) + amount;
if (abs(count) >= batch) {
- raw_spin_lock(&fbc->lock);
+ fbc_lock(fbc);
fbc->count += count;
__this_cpu_sub(*fbc->counters, count - amount);
- raw_spin_unlock(&fbc->lock);
+ fbc_unlock(fbc);
} else {
this_cpu_add(*fbc->counters, amount);
}
@@ -148,11 +188,11 @@ void percpu_counter_sync(struct percpu_counter *fbc)
unsigned long flags;
s64 count;
- raw_spin_lock_irqsave(&fbc->lock, flags);
+ fbc_lock_irqsave(fbc, flags);
count = __this_cpu_read(*fbc->counters);
fbc->count += count;
__this_cpu_sub(*fbc->counters, count);
- raw_spin_unlock_irqrestore(&fbc->lock, flags);
+ fbc_unlock_irqrestore(fbc, flags);
}
EXPORT_SYMBOL(percpu_counter_sync);
@@ -174,13 +214,13 @@ s64 __percpu_counter_sum(struct percpu_counter *fbc)
int cpu;
unsigned long flags;
- raw_spin_lock_irqsave(&fbc->lock, flags);
+ fbc_read_lock_irqsave(fbc, flags);
ret = fbc->count;
for_each_cpu_or(cpu, cpu_online_mask, cpu_dying_mask) {
s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
ret += *pcount;
}
- raw_spin_unlock_irqrestore(&fbc->lock, flags);
+ fbc_read_unlock_irqrestore(fbc, flags);
return ret;
}
EXPORT_SYMBOL(__percpu_counter_sum);
@@ -203,8 +243,7 @@ int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount,
}
for (i = 0; i < nr_counters; i++) {
- raw_spin_lock_init(&fbc[i].lock);
- lockdep_set_class(&fbc[i].lock, key);
+ fbc_lock_init(&fbc[i], key);
#ifdef CONFIG_HOTPLUG_CPU
INIT_LIST_HEAD(&fbc[i].list);
#endif
@@ -274,11 +313,11 @@ static int percpu_counter_cpu_dead(unsigned int cpu)
list_for_each_entry(fbc, &percpu_counters, list) {
s32 *pcount;
- raw_spin_lock(&fbc->lock);
+ fbc_lock(fbc);
pcount = per_cpu_ptr(fbc->counters, cpu);
fbc->count += *pcount;
*pcount = 0;
- raw_spin_unlock(&fbc->lock);
+ fbc_unlock(fbc);
}
spin_unlock_irq(&percpu_counters_lock);
#endif
@@ -354,7 +393,7 @@ bool __percpu_counter_limited_add(struct percpu_counter *fbc,
return true;
}
- raw_spin_lock(&fbc->lock);
+ fbc_lock(fbc);
count = fbc->count + amount;
/* Skip percpu_counter_sum() when safe */
@@ -392,7 +431,7 @@ bool __percpu_counter_limited_add(struct percpu_counter *fbc,
fbc->count += count + amount;
__this_cpu_sub(*fbc->counters, count);
out:
- raw_spin_unlock(&fbc->lock);
+ fbc_unlock(fbc);
local_irq_restore(flags);
return good;
}
--
2.52.0
^ permalink raw reply [flat|nested] 3+ messages in thread* Re: [PATCH] lib/percpu_counter: allow concurrent sums with an rwlock
2026-11-16 1:56 [PATCH] lib/percpu_counter: allow concurrent sums with an rwlock Jiebin Sun
@ 2026-11-16 3:16 ` Andrew Morton
2026-11-16 3:44 ` Andrew Morton
0 siblings, 1 reply; 3+ messages in thread
From: Andrew Morton @ 2026-11-16 3:16 UTC (permalink / raw)
To: Jiebin Sun
Cc: dennis, tj, cl, linux-mm, linux-kernel, bigeasy, clrkwllms,
rostedt, linux-rt-devel, tim.c.chen, tianyou.li, wangyang.guo
On Mon, 16 Nov 2026 09:56:55 +0800 Jiebin Sun <jiebin.sun@intel.com> wrote:
> __percpu_counter_sum() holds a raw spinlock while reading the global
> count and per-CPU deltas, serializing concurrent sums of the same
> counter. Since a sum changes neither the global count nor the per-CPU
> deltas, use rwlock_t on !PREEMPT_RT to allow concurrent sums with a
> read lock. Global updates take the write lock. The per-CPU fast path
> remains lockless, so sums keep their existing snapshot semantics.
>
> stress-ng dentry reaches this path through ext4_statfs():
>
> statfs -> ext4_statfs -> __percpu_counter_sum
> -> native_queued_spin_lock_slowpath
>
> Keep the raw spinlock on PREEMPT_RT, where rwlock_t can sleep.
> Global counter updates can run in atomic context and must not sleep.
>
> stress-ng dentry test average throughput change:
>
> +------------------------+------------+--------------------+
> | Case | Unit | Average change (%) |
> +------------------------+------------+--------------------+
> | Dentry, 64 workers | bogo ops/s | +47.34 |
> +------------------------+------------+--------------------+
>
> Dentry ran on ext4 with 64 workers pinned to physical CPUs:
>
> stress-ng --dentry 64 --dentries 4096 --timeout 20s
lol, this is a fun patch - it reminds me of the old days. All those
mission-critical workloads which sit in a loop running statfs will love
it!
I had $sidekick do a scan and it told me "the only plausible routine
win is dirty throttling: with low dirty limits, concurrent writers sum
the same counters. ext4 and XFS sum near ENOSPC, where updates also
contend. BFQ's sums are debug-only and serialized. Most other callers
report statistics". You might want to add something like this to
changelog.
Also mention that percpu_counter_sum_positive callers will benefit.
> --- a/include/linux/percpu_counter.h
> +++ b/include/linux/percpu_counter.h
> @@ -20,7 +20,11 @@
> #ifdef CONFIG_SMP
>
> struct percpu_counter {
> +#ifdef CONFIG_PREEMPT_RT
Might be worth a wee comment telling readers why RT is special?
> raw_spinlock_t lock;
> +#else
> + rwlock_t lock; /* shared for sums, exclusive for updates */
> +#endif
> s64 count;
> #ifdef CONFIG_HOTPLUG_CPU
> struct list_head list; /* All percpu_counters are on a list */
>
> --- a/lib/percpu_counter.c
> +++ b/lib/percpu_counter.c
> @@ -15,6 +15,46 @@ static LIST_HEAD(percpu_counters);
> static DEFINE_SPINLOCK(percpu_counters_lock);
> #endif
>
> +/*
> + * Non-RT sums share the lock; global updates take it exclusively.
> + * Keep raw spinlocks on RT: rwlocks may sleep and their irqsave variants
> + * do not disable interrupts. Updates run in atomic contexts and need
> + * IRQ exclusion for CPU-local read-modify-write sequences.
> + */
> +#ifdef CONFIG_PREEMPT_RT
> +#define fbc_lock(fbc) raw_spin_lock(&(fbc)->lock)
> +#define fbc_unlock(fbc) raw_spin_unlock(&(fbc)->lock)
> +#define fbc_lock_irqsave(fbc, f) raw_spin_lock_irqsave(&(fbc)->lock, f)
> +#define fbc_unlock_irqrestore(fbc, f) raw_spin_unlock_irqrestore(&(fbc)->lock, f)
> +#define fbc_read_lock_irqsave fbc_lock_irqsave
> +#define fbc_read_unlock_irqrestore fbc_unlock_irqrestore
> +#else
> +#define fbc_lock(fbc) write_lock(&(fbc)->lock)
> +#define fbc_unlock(fbc) write_unlock(&(fbc)->lock)
> +#define fbc_lock_irqsave(fbc, f) write_lock_irqsave(&(fbc)->lock, f)
> +#define fbc_unlock_irqrestore(fbc, f) write_unlock_irqrestore(&(fbc)->lock, f)
> +#define fbc_read_lock_irqsave(fbc, f) read_lock_irqsave(&(fbc)->lock, f)
> +#define fbc_read_unlock_irqrestore(fbc, f) read_unlock_irqrestore(&(fbc)->lock, f)
> +#endif
Please, no!
https://www.google.com/search?q=why+does+Andrew+Morton+prefer+static+inlines+over+macros
lots of good stuff there ;)
^ permalink raw reply [flat|nested] 3+ messages in thread* Re: [PATCH] lib/percpu_counter: allow concurrent sums with an rwlock
2026-11-16 3:16 ` Andrew Morton
@ 2026-11-16 3:44 ` Andrew Morton
0 siblings, 0 replies; 3+ messages in thread
From: Andrew Morton @ 2026-11-16 3:44 UTC (permalink / raw)
To: Jiebin Sun, dennis, tj, cl, linux-mm, linux-kernel, bigeasy,
clrkwllms, rostedt, linux-rt-devel, tim.c.chen, tianyou.li,
wangyang.guo
On Sun, 15 Nov 2026 19:16:36 -0800 Andrew Morton <akpm@linux-foundation.org> wrote:
> > stress-ng dentry test average throughput change:
> >
> > +------------------------+------------+--------------------+
> > | Case | Unit | Average change (%) |
> > +------------------------+------------+--------------------+
> > | Dentry, 64 workers | bogo ops/s | +47.34 |
> > +------------------------+------------+--------------------+
> >
> > Dentry ran on ext4 with 64 workers pinned to physical CPUs:
> >
> > stress-ng --dentry 64 --dentries 4096 --timeout 20s
>
> lol, this is a fun patch
It's *probably* OK, for sane workloads.
But if some workload is busily running percpu_counter_sum on 1000 CPUs,
how is a writer ever to get in there to spill ->counters into ->count?
Modern rwlock_t does implement queuing, so the writer will get in
there eventually. But nothing is simple any more - there are
complications with rwlocks on PREEMPT_RT and with interrupt-time reads
(do we even do that?).
So I think indefinite writer starvation is a risk here?
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2026-11-16 3:44 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2026-11-16 1:56 [PATCH] lib/percpu_counter: allow concurrent sums with an rwlock Jiebin Sun
2026-11-16 3:16 ` Andrew Morton
2026-11-16 3:44 ` Andrew Morton
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®