* [PATCH 0/3] improve group_cpus initialization routines
@ 2025-09-10 21:08 Yury Norov
2025-09-10 21:08 ` [PATCH 1/3] bitmap: cpumask: introduce and_andnot search helper and iterator Yury Norov
` (2 more replies)
0 siblings, 3 replies; 6+ messages in thread
From: Yury Norov @ 2025-09-10 21:08 UTC (permalink / raw)
To: Thomas Gleixner, Yury Norov, Rasmus Villemoes, linux-kernel
From: Yury Norov (NVIDIA) <yury.norov@gmail.com>
Add missing core API and improve cpumasks usage in the initialization
code.
Yury Norov (NVIDIA) (3):
bitmap: cpumask: introduce and_andnot search helper and iterator
group_cpus: don't call cpumask_weight() prematurely
group_cpus: simplify inner loop in grp_spread_init_one()
include/linux/cpumask.h | 22 +++++++++++++++++++++
include/linux/find.h | 38 +++++++++++++++++++++++++++++++++++++
lib/find_bit.c | 9 +++++++++
lib/group_cpus.c | 42 ++++++++++++-----------------------------
4 files changed, 81 insertions(+), 30 deletions(-)
--
2.43.0
^ permalink raw reply [flat|nested] 6+ messages in thread* [PATCH 1/3] bitmap: cpumask: introduce and_andnot search helper and iterator 2025-09-10 21:08 [PATCH 0/3] improve group_cpus initialization routines Yury Norov @ 2025-09-10 21:08 ` Yury Norov 2025-09-10 21:08 ` [PATCH 2/3] group_cpus: don't call cpumask_weight() prematurely Yury Norov 2025-09-10 21:08 ` [PATCH 3/3] group_cpus: optimize grp_spread_init_one() Yury Norov 2 siblings, 0 replies; 6+ messages in thread From: Yury Norov @ 2025-09-10 21:08 UTC (permalink / raw) To: Thomas Gleixner, Yury Norov, Rasmus Villemoes, linux-kernel From: "Yury Norov (NVIDIA)" <yury.norov@gmail.com> Like other similar iterators, it helps to get rid of temporary on-stack bitmaps and associate housekeeping code. Signed-off-by: Yury Norov (NVIDIA) <yury.norov@gmail.com> --- include/linux/cpumask.h | 22 ++++++++++++++++++++++ include/linux/find.h | 38 ++++++++++++++++++++++++++++++++++++++ lib/find_bit.c | 9 +++++++++ 3 files changed, 69 insertions(+) diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index ff8f41ab7ce6..8f8cb29d56d9 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -406,6 +406,28 @@ unsigned int cpumask_random(const struct cpumask *src) #define for_each_cpu_and(cpu, mask1, mask2) \ for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits) +/** + * for_each_cpu_and_andnot_from - iterate over every cpu in all masks + * @cpu: the (optionally unsigned) integer iterator + * @mask1: the first cpumask pointer + * @mask2: the second cpumask pointer + * @mask3: the third cpumask pointer + * + * This saves a temporary CPU mask in many places. It is equivalent to: + * struct cpumask tmp; + * cpumask_and(&tmp, &mask1, &mask2); + * cpumask_andnot(&tmp, &tmp, &mask3); + * for_each_cpu_from(cpu, &tmp) + * ... + * + * After the loop, cpu is >= nr_cpu_ids. + */ +#define for_each_cpu_and_andnot_from(cpu, mask1, mask2, mask3) \ + for_each_and_andnot_bit_from(cpu, cpumask_bits(mask1), \ + cpumask_bits(mask2), \ + cpumask_bits(mask3), \ + small_cpumask_bits) + /** * for_each_cpu_andnot - iterate over every cpu present in one mask, excluding * those present in another. diff --git a/include/linux/find.h b/include/linux/find.h index 9d720ad92bc1..d43c811aef5c 100644 --- a/include/linux/find.h +++ b/include/linux/find.h @@ -14,6 +14,9 @@ unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long unsigned long nbits, unsigned long start); unsigned long _find_next_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long nbits, unsigned long start); +unsigned long _find_next_and_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, + const unsigned long *addr3, unsigned long size, + unsigned long n); unsigned long _find_next_or_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long nbits, unsigned long start); unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, @@ -135,6 +138,36 @@ unsigned long find_next_andnot_bit(const unsigned long *addr1, } #endif +/** + * find_next_and_andnot_bit - find the next set bit in *addr1 and *addr2 + * excluding all the bits in *addr3 + * @addr1: The first address to base the search on + * @addr2: The second address to base the search on + * @addr3: The third address to base the search on + * @size: The bitmap size in bits + * @offset: The bitnumber to start searching at + * + * Returns the bit number for the next such bit + * If no bits found, returns >= @size. + */ +static __always_inline +unsigned long find_next_and_andnot_bit(const unsigned long *addr1, + const unsigned long *addr2, const unsigned long *addr3, + unsigned long size, unsigned long offset) +{ + if (small_const_nbits(size)) { + unsigned long val; + + if (unlikely(offset >= size)) + return size; + + val = *addr1 & *addr2 & ~*addr3 & GENMASK(size - 1, offset); + return val ? __ffs(val) : size; + } + + return _find_next_and_andnot_bit(addr1, addr2, addr3, size, offset); +} + #ifndef find_next_or_bit /** * find_next_or_bit - find the next set bit in either memory regions @@ -595,6 +628,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned (bit) = find_next_andnot_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\ (bit)++) +#define for_each_and_andnot_bit_from(bit, addr1, addr2, addr3, size) \ + for (; (bit) = find_next_and_andnot_bit((addr1), (addr2), (addr3), \ + (size), (bit)), (bit) < (size); \ + (bit)++) + #define for_each_or_bit(bit, addr1, addr2, size) \ for ((bit) = 0; \ (bit) = find_next_or_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\ diff --git a/lib/find_bit.c b/lib/find_bit.c index d4b5a29e3e72..aec79207c566 100644 --- a/lib/find_bit.c +++ b/lib/find_bit.c @@ -206,6 +206,15 @@ unsigned long _find_next_andnot_bit(const unsigned long *addr1, const unsigned l EXPORT_SYMBOL(_find_next_andnot_bit); #endif +unsigned long _find_next_and_andnot_bit(const unsigned long *addr1, + const unsigned long *addr2, + const unsigned long *addr3, + unsigned long nbits, unsigned long start) +{ + return FIND_NEXT_BIT(addr1[idx] & addr2[idx] & ~addr3[idx], /* nop */, nbits, start); +} +EXPORT_SYMBOL(_find_next_and_andnot_bit); + #ifndef find_next_or_bit unsigned long _find_next_or_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long nbits, unsigned long start) -- 2.43.0 ^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH 2/3] group_cpus: don't call cpumask_weight() prematurely 2025-09-10 21:08 [PATCH 0/3] improve group_cpus initialization routines Yury Norov 2025-09-10 21:08 ` [PATCH 1/3] bitmap: cpumask: introduce and_andnot search helper and iterator Yury Norov @ 2025-09-10 21:08 ` Yury Norov 2025-09-10 21:08 ` [PATCH 3/3] group_cpus: optimize grp_spread_init_one() Yury Norov 2 siblings, 0 replies; 6+ messages in thread From: Yury Norov @ 2025-09-10 21:08 UTC (permalink / raw) To: Thomas Gleixner, Yury Norov, Rasmus Villemoes, linux-kernel From: "Yury Norov (NVIDIA)" <yury.norov@gmail.com> alloc_nodes_groups() and __group_cpus_evenly() call cpumask_weight() unconditionally in the for_each() loops. cpumask_weight() is O(N), so the complexity of the function becomes O(MAX_NUMNODES * nr_cpu_ids). This call may be avoided if the nmsk is empty. Signed-off-by: Yury Norov (NVIDIA) <yury.norov@gmail.com> --- lib/group_cpus.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/group_cpus.c b/lib/group_cpus.c index 6d08ac05f371..6aae1560b796 100644 --- a/lib/group_cpus.c +++ b/lib/group_cpus.c @@ -142,15 +142,11 @@ static void alloc_nodes_groups(unsigned int numgrps, } for_each_node_mask(n, nodemsk) { - unsigned ncpus; - - cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]); - ncpus = cpumask_weight(nmsk); - - if (!ncpus) + if (!cpumask_and(nmsk, cpu_mask, node_to_cpumask[n])) continue; - remaining_ncpus += ncpus; - node_groups[n].ncpus = ncpus; + + node_groups[n].ncpus = cpumask_weight(nmsk); + remaining_ncpus += node_groups[n].ncpus; } numgrps = min_t(unsigned, remaining_ncpus, numgrps); @@ -294,11 +290,10 @@ static int __group_cpus_evenly(unsigned int startgrp, unsigned int numgrps, continue; /* Get the cpus on this node which are in the mask */ - cpumask_and(nmsk, cpu_mask, node_to_cpumask[nv->id]); - ncpus = cpumask_weight(nmsk); - if (!ncpus) + if (!cpumask_and(nmsk, cpu_mask, node_to_cpumask[nv->id])) continue; + ncpus = cpumask_weight(nmsk); WARN_ON_ONCE(nv->ngroups > ncpus); /* Account for rounding errors */ -- 2.43.0 ^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH 3/3] group_cpus: optimize grp_spread_init_one() 2025-09-10 21:08 [PATCH 0/3] improve group_cpus initialization routines Yury Norov 2025-09-10 21:08 ` [PATCH 1/3] bitmap: cpumask: introduce and_andnot search helper and iterator Yury Norov 2025-09-10 21:08 ` [PATCH 2/3] group_cpus: don't call cpumask_weight() prematurely Yury Norov @ 2025-09-10 21:08 ` Yury Norov 2025-09-19 13:56 ` kernel test robot 2 siblings, 1 reply; 6+ messages in thread From: Yury Norov @ 2025-09-10 21:08 UTC (permalink / raw) To: Thomas Gleixner, Yury Norov, Rasmus Villemoes, linux-kernel From: "Yury Norov (NVIDIA)" <yury.norov@gmail.com> Optimizations for grp_spread_init_one(): 1. Drop most of housekeeping code in grp_spread_init_one() by using for_each_cpu_and_andnot_from(). 2. Fix Shlemiel the Painter's algorithm by adding 'sibl = cpu' line. This improves the outer loop complexity from quadratic to linear. 3. Don't clear the nmsk because it's ignored in the caller code anyways, and switch to non-atomic set_cpu() for irqmsk as the mask is local and implies no concurrency. Signed-off-by: Yury Norov (NVIDIA) <yury.norov@gmail.com> --- lib/group_cpus.c | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/lib/group_cpus.c b/lib/group_cpus.c index 6aae1560b796..35aba99d8cd0 100644 --- a/lib/group_cpus.c +++ b/lib/group_cpus.c @@ -17,27 +17,14 @@ static void grp_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk, const struct cpumask *siblmsk; int cpu, sibl; - for ( ; cpus_per_grp > 0; ) { - cpu = cpumask_first(nmsk); - - /* Should not happen, but I'm too lazy to think about it */ - if (cpu >= nr_cpu_ids) - return; - - cpumask_clear_cpu(cpu, nmsk); - cpumask_set_cpu(cpu, irqmsk); - cpus_per_grp--; - + for_each_cpu(cpu, nmsk) { /* If the cpu has siblings, use them first */ siblmsk = topology_sibling_cpumask(cpu); - for (sibl = -1; cpus_per_grp > 0; ) { - sibl = cpumask_next(sibl, siblmsk); - if (sibl >= nr_cpu_ids) - break; - if (!cpumask_test_and_clear_cpu(sibl, nmsk)) - continue; - cpumask_set_cpu(sibl, irqmsk); - cpus_per_grp--; + sibl = cpu; + for_each_cpu_and_andnot_from(sibl, nmsk, siblmsk, irqmsk) { + __cpumask_set_cpu(sibl, irqmsk); + if (--cpus_per_grp) + return; } } } -- 2.43.0 ^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [PATCH 3/3] group_cpus: optimize grp_spread_init_one() 2025-09-10 21:08 ` [PATCH 3/3] group_cpus: optimize grp_spread_init_one() Yury Norov @ 2025-09-19 13:56 ` kernel test robot 0 siblings, 0 replies; 6+ messages in thread From: kernel test robot @ 2025-09-19 13:56 UTC (permalink / raw) To: Yury Norov Cc: oe-lkp, lkp, linux-kernel, Thomas Gleixner, Yury Norov, Rasmus Villemoes, oliver.sang [-- Attachment #1: Type: text/plain, Size: 5869 bytes --] Hello, kernel test robot noticed "BUG:kernel_NULL_pointer_dereference,address" on: commit: 5df2998b7baa3fd4cb66272d1ea5625573b4a63f ("[PATCH 3/3] group_cpus: optimize grp_spread_init_one()") url: https://github.com/intel-lab-lkp/linux/commits/Yury-Norov/bitmap-cpumask-introduce-and_andnot-search-helper-and-iterator/20250911-051023 base: https://git.kernel.org/cgit/linux/kernel/git/tip/tip.git 8ad25ebfa70e86860559b306bbc923c7db4fcac6 patch link: https://lore.kernel.org/all/20250910210850.404834-4-yury.norov@gmail.com/ patch subject: [PATCH 3/3] group_cpus: optimize grp_spread_init_one() in testcase: stress-ng version: stress-ng-x86_64-665b4465f-1_20250912 with following parameters: nr_threads: 100% testtime: 60s test: cpu-online cpufreq_governor: performance config: x86_64-rhel-9.4 compiler: gcc-14 test machine: 104 threads 2 sockets (Skylake) with 192G memory (please refer to attached dmesg/kmsg for entire log/backtrace) If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags | Reported-by: kernel test robot <oliver.sang@intel.com> | Closes: https://lore.kernel.org/oe-lkp/202509192101.5d4e0282-lkp@intel.com The kernel config and materials to reproduce are available at: https://download.01.org/0day-ci/archive/20250919/202509192101.5d4e0282-lkp@intel.com [1] we observed the issue happens randomly in our tests. ========================================================================================= tbox_group/testcase/rootfs/kconfig/compiler/nr_threads/testtime/test/cpufreq_governor: lkp-skl-fpga01/stress-ng/debian-13-x86_64-20250902.cgz/x86_64-rhel-9.4/gcc-14/100%/60s/cpu-online/performance commit: 6ade57f62d272 ("group_cpus: don't call cpumask_weight() prematurely") 5df2998b7baa3 ("group_cpus: optimize grp_spread_init_one()") 6ade57f62d272d3e 5df2998b7baa3fd4cb66272d1ea ---------------- --------------------------- fail:runs %reproduction fail:runs | | | :30 43% 13:30 dmesg.BUG:kernel_NULL_pointer_dereference,address :30 43% 13:30 dmesg.Kernel_panic-not_syncing:Fatal_exception :30 43% 13:30 dmesg.Oops :30 43% 13:30 dmesg.RIP:blk_mq_all_tag_iter and sorry our bot failed to upload correct dmesg in above link [1] attached one FYI. [ 43.544050][ T130] BUG: kernel NULL pointer dereference, address: 0000000000000004 [ 43.552056][ T130] #PF: supervisor read access in kernel mode [ 43.558218][ T130] #PF: error_code(0x0000) - not-present page [ 43.564340][ T130] PGD 0 P4D 0 [ 43.567866][ T130] Oops: Oops: 0000 [#1] SMP PTI [ 43.572855][ T130] CPU: 19 UID: 0 PID: 130 Comm: cpuhp/19 Not tainted 6.17.0-rc1-00014-g5df2998b7baa #1 VOLUNTARY [ 43.583575][ T130] Hardware name: Intel Corporation S2600BT/S2600BT, BIOS SE5C620.86B.1D.01.0147.121320181755 12/13/2018 [ 43.594817][ T130] RIP: 0010:blk_mq_all_tag_iter+0x1a/0x230 [ 43.600779][ T130] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 66 0f 1f 00 0f 1f 44 00 00 41 57 41 56 41 55 41 54 49 89 fc 55 53 48 83 ec 50 <8b> 4f 04 48 89 34 24 48 89 54 24 08 65 48 8b 05 8a b4 48 02 48 89 [ 43.620822][ T130] RSP: 0018:ffffc9000ce5fda0 EFLAGS: 00010286 [ 43.627048][ T130] RAX: 0000604fe9627758 RBX: ffffc9000ce5fe28 RCX: 0000000000000068 [ 43.635176][ T130] RDX: ffffc9000ce5fe28 RSI: ffffffff8199e310 RDI: 0000000000000000 [ 43.643294][ T130] RBP: ffff88810dd98600 R08: ffffffff833d72a0 R09: ffff8881003c8710 [ 43.651410][ T130] R10: ffffc9000ce5fdb0 R11: ffffc9000ce5fdb8 R12: 0000000000000000 [ 43.659516][ T130] R13: 0000000000000013 R14: ffff88810dd98760 R15: ffff8897e08dbde8 [ 43.667621][ T130] FS: 0000000000000000(0000) GS:ffff88985caa3000(0000) knlGS:0000000000000000 [ 43.676681][ T130] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 43.683393][ T130] CR2: 0000000000000004 CR3: 000000303de24004 CR4: 00000000007726f0 [ 43.691491][ T130] PKRU: 55555554 [ 43.695175][ T130] Call Trace: [ 43.698593][ T130] <TASK> [ 43.701657][ T130] ? __call_rcu_common+0xb0/0x2f0 [ 43.707850][ T130] ? xas_load+0xd/0xf0 [ 43.712054][ T130] ? xa_load+0x58/0xb0 [ 43.716259][ T130] blk_mq_hctx_notify_offline+0xd1/0x170 [ 43.722025][ T130] ? __pfx_blk_mq_hctx_notify_offline+0x10/0x10 [ 43.728390][ T130] cpuhp_invoke_callback+0x1c7/0x370 [ 43.733805][ T130] ? __pfx_smpboot_thread_fn+0x10/0x10 [ 43.739386][ T130] cpuhp_thread_fun+0x98/0x170 [ 43.744273][ T130] smpboot_thread_fn+0xc8/0x1b0 [ 43.749254][ T130] kthread+0xe5/0x1f0 [ 43.753363][ T130] ? __pfx_kthread+0x10/0x10 [ 43.758082][ T130] ret_from_fork+0x132/0x170 [ 43.762799][ T130] ? __pfx_kthread+0x10/0x10 [ 43.767513][ T130] ret_from_fork_asm+0x1a/0x30 [ 43.772398][ T130] </TASK> [ 43.775532][ T130] Modules linked in: intel_rapl_msr intel_rapl_common intel_uncore_frequency intel_uncore_frequency_common btrfs blake2b_generic xor raid6_pq skx_edac skx_edac_common nfit libnvdimm sd_mod sg x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel irdma kvm irqbypass ice snd_pcm ghash_clmulni_intel ahci snd_timer rapl gnss snd ast ib_uverbs nvme intel_cstate libahci ipmi_ssif drm_client_lib soundcore binfmt_misc acpi_power_meter mei_me drm_shmem_helper i2c_i801 ib_core ioatdma pcspkr nvme_core libata ipmi_si intel_uncore acpi_ipmi drm_kms_helper mei i2c_smbus intel_pch_thermal lpc_ich dca wmi ipmi_devintf ipmi_msghandler acpi_pad joydev drm fuse nfnetlink [ 43.836729][ T130] CR2: 0000000000000004 [ 43.841005][ T130] ---[ end trace 0000000000000000 ]--- [ 43.855607][ T130] pstore: backend (erst) writing error (-28) -- 0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki [-- Attachment #2: dmesg.xz --] [-- Type: application/x-xz, Size: 30376 bytes --] ^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH 0/3] improve group_cpus initialization routines @ 2025-11-19 3:13 Yury Norov (NVIDIA) 2025-11-19 3:13 ` [PATCH 1/3] bitmap: cpumask: introduce and_andnot search helper and iterator Yury Norov (NVIDIA) 0 siblings, 1 reply; 6+ messages in thread From: Yury Norov (NVIDIA) @ 2025-11-19 3:13 UTC (permalink / raw) To: Andrew Morton, Thomas Gleixner Cc: Yury Norov (NVIDIA), Rasmus Villemoes, linux-kernel Add missing core API and improve cpumasks usage in the initialization code. Yury Norov (NVIDIA) (3): bitmap: cpumask: introduce and_andnot search helper and iterator group_cpus: don't call cpumask_weight() prematurely group_cpus: simplify inner loop in grp_spread_init_one() include/linux/cpumask.h | 22 +++++++++++++++++++++ include/linux/find.h | 38 +++++++++++++++++++++++++++++++++++++ lib/find_bit.c | 9 +++++++++ lib/group_cpus.c | 42 ++++++++++++----------------------------- 4 files changed, 81 insertions(+), 30 deletions(-) -- 2.43.0 ^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH 1/3] bitmap: cpumask: introduce and_andnot search helper and iterator 2025-11-19 3:13 [PATCH 0/3] improve group_cpus initialization routines Yury Norov (NVIDIA) @ 2025-11-19 3:13 ` Yury Norov (NVIDIA) 0 siblings, 0 replies; 6+ messages in thread From: Yury Norov (NVIDIA) @ 2025-11-19 3:13 UTC (permalink / raw) To: Andrew Morton, Thomas Gleixner Cc: Yury Norov (NVIDIA), Rasmus Villemoes, linux-kernel Like other similar iterators, *_and_andnot helps to get rid of temporary on-stack bitmaps and associate housekeeping code. Signed-off-by: Yury Norov (NVIDIA) <yury.norov@gmail.com> --- include/linux/cpumask.h | 22 ++++++++++++++++++++++ include/linux/find.h | 38 ++++++++++++++++++++++++++++++++++++++ lib/find_bit.c | 9 +++++++++ 3 files changed, 69 insertions(+) diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index ff8f41ab7ce6..6de16a0e6e7b 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -406,6 +406,28 @@ unsigned int cpumask_random(const struct cpumask *src) #define for_each_cpu_and(cpu, mask1, mask2) \ for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits) +/** + * for_each_cpu_and_andnot_from - iterate over every cpu in all masks + * @cpu: the (optionally unsigned) integer iterator + * @mask1: the first cpumask pointer + * @mask2: the second cpumask pointer + * @mask3: the third cpumask pointer + * + * This saves a temporary CPU mask in many places. It is equivalent to: + * struct cpumask tmp; + * cpumask_and(&tmp, &mask1, &mask2); + * cpumask_andnot(&tmp, &tmp, &mask3); + * for_each_cpu_from(cpu, &tmp) + * ... + * + * After the loop, cpu is >= nr_cpu_ids. + */ +#define for_each_cpu_and_andnot_from(cpu, mask1, mask2, mask3) \ + for_each_and_andnot_bit_from(cpu, cpumask_bits(mask1), \ + cpumask_bits(mask2), \ + cpumask_bits(mask3), \ + small_cpumask_bits) + /** * for_each_cpu_andnot - iterate over every cpu present in one mask, excluding * those present in another. diff --git a/include/linux/find.h b/include/linux/find.h index 9d720ad92bc1..daf72078c25e 100644 --- a/include/linux/find.h +++ b/include/linux/find.h @@ -14,6 +14,9 @@ unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long unsigned long nbits, unsigned long start); unsigned long _find_next_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long nbits, unsigned long start); +unsigned long _find_next_and_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, + const unsigned long *addr3, unsigned long size, + unsigned long n); unsigned long _find_next_or_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long nbits, unsigned long start); unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, @@ -135,6 +138,36 @@ unsigned long find_next_andnot_bit(const unsigned long *addr1, } #endif +/** + * find_next_and_andnot_bit - find the next set bit in *addr1 and *addr2 + * excluding all the bits in *addr3 + * @addr1: The first address to base the search on + * @addr2: The second address to base the search on + * @addr3: The second address to base the search on + * @size: The bitmap size in bits + * @offset: The bitnumber to start searching at + * + * Returns the bit number for the next set bit + * If no bits found, returns >= @size. + */ +static __always_inline +unsigned long find_next_and_andnot_bit(const unsigned long *addr1, + const unsigned long *addr2, const unsigned long *addr3, + unsigned long size, unsigned long offset) +{ + if (small_const_nbits(size)) { + unsigned long val; + + if (unlikely(offset >= size)) + return size; + + val = *addr1 & *addr2 & ~*addr3; + return val ? __ffs(val) : size; + } + + return _find_next_and_andnot_bit(addr1, addr2, addr3, size, offset); +} + #ifndef find_next_or_bit /** * find_next_or_bit - find the next set bit in either memory regions @@ -595,6 +628,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned (bit) = find_next_andnot_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\ (bit)++) +#define for_each_and_andnot_bit_from(bit, addr1, addr2, addr3, size) \ + for (; (bit) = find_next_and_andnot_bit((addr1), (addr2), (addr3), \ + (size), (bit)), (bit) < (size); \ + (bit)++) + #define for_each_or_bit(bit, addr1, addr2, size) \ for ((bit) = 0; \ (bit) = find_next_or_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\ diff --git a/lib/find_bit.c b/lib/find_bit.c index d4b5a29e3e72..aec79207c566 100644 --- a/lib/find_bit.c +++ b/lib/find_bit.c @@ -206,6 +206,15 @@ unsigned long _find_next_andnot_bit(const unsigned long *addr1, const unsigned l EXPORT_SYMBOL(_find_next_andnot_bit); #endif +unsigned long _find_next_and_andnot_bit(const unsigned long *addr1, + const unsigned long *addr2, + const unsigned long *addr3, + unsigned long nbits, unsigned long start) +{ + return FIND_NEXT_BIT(addr1[idx] & addr2[idx] & ~addr3[idx], /* nop */, nbits, start); +} +EXPORT_SYMBOL(_find_next_and_andnot_bit); + #ifndef find_next_or_bit unsigned long _find_next_or_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long nbits, unsigned long start) -- 2.43.0 ^ permalink raw reply [flat|nested] 6+ messages in thread
end of thread, other threads:[~2025-11-19 3:13 UTC | newest] Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed) -- links below jump to the message on this page -- 2025-09-10 21:08 [PATCH 0/3] improve group_cpus initialization routines Yury Norov 2025-09-10 21:08 ` [PATCH 1/3] bitmap: cpumask: introduce and_andnot search helper and iterator Yury Norov 2025-09-10 21:08 ` [PATCH 2/3] group_cpus: don't call cpumask_weight() prematurely Yury Norov 2025-09-10 21:08 ` [PATCH 3/3] group_cpus: optimize grp_spread_init_one() Yury Norov 2025-09-19 13:56 ` kernel test robot 2025-11-19 3:13 [PATCH 0/3] improve group_cpus initialization routines Yury Norov (NVIDIA) 2025-11-19 3:13 ` [PATCH 1/3] bitmap: cpumask: introduce and_andnot search helper and iterator Yury Norov (NVIDIA)
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®