* [PATCH 1/3] bitmap: cpumask: introduce and_andnot search helper and iterator
2025-11-19 3:13 [PATCH 0/3] improve group_cpus initialization routines Yury Norov (NVIDIA)
@ 2025-11-19 3:13 ` Yury Norov (NVIDIA)
2025-11-19 3:13 ` [PATCH 2/3] group_cpus: don't call cpumask_weight() prematurely Yury Norov (NVIDIA)
` (2 subsequent siblings)
3 siblings, 0 replies; 6+ messages in thread
From: Yury Norov (NVIDIA) @ 2025-11-19 3:13 UTC (permalink / raw)
To: Andrew Morton, Thomas Gleixner
Cc: Yury Norov (NVIDIA), Rasmus Villemoes, linux-kernel
Like other similar iterators, *_and_andnot helps to get rid of temporary
on-stack bitmaps and associate housekeeping code.
Signed-off-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
---
include/linux/cpumask.h | 22 ++++++++++++++++++++++
include/linux/find.h | 38 ++++++++++++++++++++++++++++++++++++++
lib/find_bit.c | 9 +++++++++
3 files changed, 69 insertions(+)
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index ff8f41ab7ce6..6de16a0e6e7b 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -406,6 +406,28 @@ unsigned int cpumask_random(const struct cpumask *src)
#define for_each_cpu_and(cpu, mask1, mask2) \
for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
+/**
+ * for_each_cpu_and_andnot_from - iterate over every cpu in all masks
+ * @cpu: the (optionally unsigned) integer iterator
+ * @mask1: the first cpumask pointer
+ * @mask2: the second cpumask pointer
+ * @mask3: the third cpumask pointer
+ *
+ * This saves a temporary CPU mask in many places. It is equivalent to:
+ * struct cpumask tmp;
+ * cpumask_and(&tmp, &mask1, &mask2);
+ * cpumask_andnot(&tmp, &tmp, &mask3);
+ * for_each_cpu_from(cpu, &tmp)
+ * ...
+ *
+ * After the loop, cpu is >= nr_cpu_ids.
+ */
+#define for_each_cpu_and_andnot_from(cpu, mask1, mask2, mask3) \
+ for_each_and_andnot_bit_from(cpu, cpumask_bits(mask1), \
+ cpumask_bits(mask2), \
+ cpumask_bits(mask3), \
+ small_cpumask_bits)
+
/**
* for_each_cpu_andnot - iterate over every cpu present in one mask, excluding
* those present in another.
diff --git a/include/linux/find.h b/include/linux/find.h
index 9d720ad92bc1..daf72078c25e 100644
--- a/include/linux/find.h
+++ b/include/linux/find.h
@@ -14,6 +14,9 @@ unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long
unsigned long nbits, unsigned long start);
unsigned long _find_next_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long nbits, unsigned long start);
+unsigned long _find_next_and_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
+ const unsigned long *addr3, unsigned long size,
+ unsigned long n);
unsigned long _find_next_or_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long nbits, unsigned long start);
unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits,
@@ -135,6 +138,36 @@ unsigned long find_next_andnot_bit(const unsigned long *addr1,
}
#endif
+/**
+ * find_next_and_andnot_bit - find the next set bit in *addr1 and *addr2
+ * excluding all the bits in *addr3
+ * @addr1: The first address to base the search on
+ * @addr2: The second address to base the search on
+ * @addr3: The second address to base the search on
+ * @size: The bitmap size in bits
+ * @offset: The bitnumber to start searching at
+ *
+ * Returns the bit number for the next set bit
+ * If no bits found, returns >= @size.
+ */
+static __always_inline
+unsigned long find_next_and_andnot_bit(const unsigned long *addr1,
+ const unsigned long *addr2, const unsigned long *addr3,
+ unsigned long size, unsigned long offset)
+{
+ if (small_const_nbits(size)) {
+ unsigned long val;
+
+ if (unlikely(offset >= size))
+ return size;
+
+ val = *addr1 & *addr2 & ~*addr3;
+ return val ? __ffs(val) : size;
+ }
+
+ return _find_next_and_andnot_bit(addr1, addr2, addr3, size, offset);
+}
+
#ifndef find_next_or_bit
/**
* find_next_or_bit - find the next set bit in either memory regions
@@ -595,6 +628,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned
(bit) = find_next_andnot_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\
(bit)++)
+#define for_each_and_andnot_bit_from(bit, addr1, addr2, addr3, size) \
+ for (; (bit) = find_next_and_andnot_bit((addr1), (addr2), (addr3), \
+ (size), (bit)), (bit) < (size); \
+ (bit)++)
+
#define for_each_or_bit(bit, addr1, addr2, size) \
for ((bit) = 0; \
(bit) = find_next_or_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\
diff --git a/lib/find_bit.c b/lib/find_bit.c
index d4b5a29e3e72..aec79207c566 100644
--- a/lib/find_bit.c
+++ b/lib/find_bit.c
@@ -206,6 +206,15 @@ unsigned long _find_next_andnot_bit(const unsigned long *addr1, const unsigned l
EXPORT_SYMBOL(_find_next_andnot_bit);
#endif
+unsigned long _find_next_and_andnot_bit(const unsigned long *addr1,
+ const unsigned long *addr2,
+ const unsigned long *addr3,
+ unsigned long nbits, unsigned long start)
+{
+ return FIND_NEXT_BIT(addr1[idx] & addr2[idx] & ~addr3[idx], /* nop */, nbits, start);
+}
+EXPORT_SYMBOL(_find_next_and_andnot_bit);
+
#ifndef find_next_or_bit
unsigned long _find_next_or_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long nbits, unsigned long start)
--
2.43.0
^ permalink raw reply [flat|nested] 6+ messages in thread* [PATCH 2/3] group_cpus: don't call cpumask_weight() prematurely
2025-11-19 3:13 [PATCH 0/3] improve group_cpus initialization routines Yury Norov (NVIDIA)
2025-11-19 3:13 ` [PATCH 1/3] bitmap: cpumask: introduce and_andnot search helper and iterator Yury Norov (NVIDIA)
@ 2025-11-19 3:13 ` Yury Norov (NVIDIA)
2025-11-19 3:13 ` [PATCH 3/3] group_cpus: simplify inner loop in grp_spread_init_one() Yury Norov (NVIDIA)
2025-11-28 2:25 ` [PATCH 0/3] improve group_cpus initialization routines Yury Norov
3 siblings, 0 replies; 6+ messages in thread
From: Yury Norov (NVIDIA) @ 2025-11-19 3:13 UTC (permalink / raw)
To: Andrew Morton, Thomas Gleixner
Cc: Yury Norov (NVIDIA), Rasmus Villemoes, linux-kernel
alloc_nodes_groups() and __group_cpus_evenly() call cpumask_weight()
unconditionally in the for_each() loops. cpumask_weight() is O(N), so
the complexity of the functions become O(MAX_NUMNODES * nr_cpu_ids).
This call may be avoided if the nmsk is empty.
Signed-off-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
---
lib/group_cpus.c | 17 ++++++-----------
1 file changed, 6 insertions(+), 11 deletions(-)
diff --git a/lib/group_cpus.c b/lib/group_cpus.c
index 6d08ac05f371..6aae1560b796 100644
--- a/lib/group_cpus.c
+++ b/lib/group_cpus.c
@@ -142,15 +142,11 @@ static void alloc_nodes_groups(unsigned int numgrps,
}
for_each_node_mask(n, nodemsk) {
- unsigned ncpus;
-
- cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]);
- ncpus = cpumask_weight(nmsk);
-
- if (!ncpus)
+ if (!cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]))
continue;
- remaining_ncpus += ncpus;
- node_groups[n].ncpus = ncpus;
+
+ node_groups[n].ncpus = cpumask_weight(nmsk);
+ remaining_ncpus += node_groups[n].ncpus;
}
numgrps = min_t(unsigned, remaining_ncpus, numgrps);
@@ -294,11 +290,10 @@ static int __group_cpus_evenly(unsigned int startgrp, unsigned int numgrps,
continue;
/* Get the cpus on this node which are in the mask */
- cpumask_and(nmsk, cpu_mask, node_to_cpumask[nv->id]);
- ncpus = cpumask_weight(nmsk);
- if (!ncpus)
+ if (!cpumask_and(nmsk, cpu_mask, node_to_cpumask[nv->id]))
continue;
+ ncpus = cpumask_weight(nmsk);
WARN_ON_ONCE(nv->ngroups > ncpus);
/* Account for rounding errors */
--
2.43.0
^ permalink raw reply [flat|nested] 6+ messages in thread* [PATCH 3/3] group_cpus: simplify inner loop in grp_spread_init_one()
2025-11-19 3:13 [PATCH 0/3] improve group_cpus initialization routines Yury Norov (NVIDIA)
2025-11-19 3:13 ` [PATCH 1/3] bitmap: cpumask: introduce and_andnot search helper and iterator Yury Norov (NVIDIA)
2025-11-19 3:13 ` [PATCH 2/3] group_cpus: don't call cpumask_weight() prematurely Yury Norov (NVIDIA)
@ 2025-11-19 3:13 ` Yury Norov (NVIDIA)
2025-11-28 2:25 ` [PATCH 0/3] improve group_cpus initialization routines Yury Norov
3 siblings, 0 replies; 6+ messages in thread
From: Yury Norov (NVIDIA) @ 2025-11-19 3:13 UTC (permalink / raw)
To: Andrew Morton, Thomas Gleixner
Cc: Yury Norov (NVIDIA), Rasmus Villemoes, linux-kernel
Three optimizations for grp_spread_init_one().
1. Drop most of housekeeping code in grp_spread_init_one() with
for_each_cpu_and_andnot_from().
2. Fix Shlemiel the Painter's algorithm by adding 'sibl = cpu' line. This
improves the grp_spread_init_one() complexity from quadratic to linear.
3. Don't clear the nmsk because it's rewritten in the caller code anyways,
and switch to non-atomic bit setter for irqmsk as the mask is local
and implies no concurrency.
Signed-off-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
---
lib/group_cpus.c | 25 ++++++-------------------
1 file changed, 6 insertions(+), 19 deletions(-)
diff --git a/lib/group_cpus.c b/lib/group_cpus.c
index 6aae1560b796..35aba99d8cd0 100644
--- a/lib/group_cpus.c
+++ b/lib/group_cpus.c
@@ -17,27 +17,14 @@ static void grp_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk,
const struct cpumask *siblmsk;
int cpu, sibl;
- for ( ; cpus_per_grp > 0; ) {
- cpu = cpumask_first(nmsk);
-
- /* Should not happen, but I'm too lazy to think about it */
- if (cpu >= nr_cpu_ids)
- return;
-
- cpumask_clear_cpu(cpu, nmsk);
- cpumask_set_cpu(cpu, irqmsk);
- cpus_per_grp--;
-
+ for_each_cpu(cpu, nmsk) {
/* If the cpu has siblings, use them first */
siblmsk = topology_sibling_cpumask(cpu);
- for (sibl = -1; cpus_per_grp > 0; ) {
- sibl = cpumask_next(sibl, siblmsk);
- if (sibl >= nr_cpu_ids)
- break;
- if (!cpumask_test_and_clear_cpu(sibl, nmsk))
- continue;
- cpumask_set_cpu(sibl, irqmsk);
- cpus_per_grp--;
+ sibl = cpu;
+ for_each_cpu_and_andnot_from(sibl, nmsk, siblmsk, irqmsk) {
+ __cpumask_set_cpu(sibl, irqmsk);
+ if (--cpus_per_grp)
+ return;
}
}
}
--
2.43.0
^ permalink raw reply [flat|nested] 6+ messages in thread