* [PATCH] md/raid5: grow stripe hash table with the size of stripe cache
@ 2026-11-16 13:39 Miroslav Cimerman
0 siblings, 0 replies; only message in thread
From: Miroslav Cimerman @ 2026-11-16 13:39 UTC (permalink / raw)
To: Song Liu, Yu Kuai
Cc: Li Nan, Xiao Ni, linux-raid, linux-kernel, Anton Gavriliuk
The stripe hash table has a fixed size of one page, 512 buckets with
4 KiB pages. With stripe_cache_size at its current maximum (32768), a
bucket contains 64 stripes on average. Every stripe lookup walks one
chain with interrupts off and the hash lock held, and with random I/O
most lookups miss and walk the whole chain. This costs CPU time on
every request and makes the hash locks more contended.
Grow the table when stripe_cache_size is written, to about
STRIPES_PER_HASH_BUCKET (two) stripes per bucket, rounded up to a
power of two. The new table is allocated before any hash lock is taken,
and the stripes are moved to it with all hash locks held. The table
size stays a power of two and a multiple of NR_STRIPE_HASH_LOCKS, so
every stripe keeps its hash lock. If the allocation fails, the old
table stays. Moving about 24k stripes (only stripes that have been
used are in the table) took about 2 ms with interrupts off on the
Xeon machine below. This happens only when stripe_cache_size is set.
The table starts at one page as before. It needs at most 8 bytes per
stripe: 128 KiB at stripe_cache_size=32768, where the stripe pages
alone take 768 MiB on RAID-6 4+2. The table is not shrunk when
stripe_cache_size is lowered.
2x Xeon E5-2699 v3 (one NUMA node used), RAM disks, chunk 512K,
stripe_cache_size=32768, group_thread_cnt=8, io_uring, without -> with
this patch:
RAID-6 4+2 4k random write, 16 jobs QD64 320k -> 355k IOPS (+11%)
RAID-6 6+2 4k random write, 16 jobs QD64 345k -> 381k IOPS (+10%)
RAID-5 5+1 4k random write, 16 jobs QD64 437k -> 514k IOPS (+18%)
RAID-6 4+2 1M seq write, 4 jobs QD16 4410 -> 4971 MiB/s (+13%)
RAID-6 4+2 4k random read, degraded 543k -> 705k IOPS (+30%)
HPE DL345 Gen11, AMD EPYC 9455, 6x PCIe 5.0 NVMe (KIOXIA CM7-V 3.2 TB),
the same md settings and chunk size:
RAID-6 4+2 4k random write, 16 jobs QD64 501k -> 536k IOPS (+7%)
RAID-6 4+2 2M seq write, 1 job QD64 10342 -> 13523 MiB/s (+31%)
The CPU time per I/O drops by 21-34% on the Xeon, and by 12% (random)
and 49% (sequential) on the EPYC. In the 4k random write tests,
find_get_stripe() drops from 9.2% to 0.6% of all cycles on the Xeon,
and from 3.4% to 0.2% on the EPYC. With md's default stripe_cache_size
nothing changes.
Signed-off-by: Miroslav Cimerman <mc@raidlab.org>
---
drivers/md/raid5.c | 60 +++++++++++++++++++++++++++++++++++++++++++---
drivers/md/raid5.h | 14 ++++++++---
2 files changed, 68 insertions(+), 6 deletions(-)
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index efb4b0d37cb0..5e28ab750398 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -77,7 +77,7 @@ static void raid5_quiesce(struct mddev *mddev, int quiesce);
static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)
{
- int hash = (sect >> RAID5_STRIPE_SHIFT(conf)) & HASH_MASK;
+ int hash = (sect >> RAID5_STRIPE_SHIFT(conf)) & conf->hash_mask;
return &conf->stripe_hashtbl[hash];
}
@@ -7061,6 +7061,57 @@ raid5_show_stripe_cache_size(struct mddev *mddev, char *page)
return ret;
}
+static unsigned int nr_hash_buckets(int nr_stripes)
+{
+ unsigned int buckets = max(nr_stripes / STRIPES_PER_HASH_BUCKET, 1);
+
+ /*
+ * The table size must be a multiple of NR_STRIPE_HASH_LOCKS for a
+ * stripe to keep its lock when the table grows.
+ */
+ BUILD_BUG_ON(NR_HASH_MIN % NR_STRIPE_HASH_LOCKS);
+ return max_t(unsigned int, roundup_pow_of_two(buckets), NR_HASH_MIN);
+}
+
+/*
+ * Grow the stripe hash table so that there are fewer stripes hashed to
+ * a single bucket when the stripe cache is enlarged.
+ *
+ * This is only an optimization. If the allocation of the new table fails,
+ * the old table stays.
+ */
+static void raid5_grow_stripe_hashtbl(struct r5conf *conf, int nr_stripes)
+{
+ unsigned int new_size = nr_hash_buckets(nr_stripes);
+ unsigned int i, old_size;
+ struct hlist_head *new, *old;
+ struct stripe_head *sh;
+ struct hlist_node *tmp;
+
+ lockdep_assert_held(&conf->cache_size_mutex);
+ old = conf->stripe_hashtbl;
+ old_size = conf->hash_mask + 1;
+ if (new_size <= old_size)
+ return;
+ new = kvzalloc_objs(*new, new_size, GFP_KERNEL | __GFP_NOWARN);
+ if (!new)
+ return;
+
+ lock_all_device_hash_locks_irq(conf);
+ conf->stripe_hashtbl = new;
+ conf->hash_mask = new_size - 1;
+ for (i = 0; i < old_size; i++) {
+ hlist_for_each_entry_safe(sh, tmp, &old[i], hash) {
+ hlist_del(&sh->hash);
+ hlist_add_head(&sh->hash,
+ stripe_hash(conf, sh->sector));
+ }
+ }
+ unlock_all_device_hash_locks_irq(conf);
+
+ kvfree(old);
+}
+
int
raid5_set_cache_size(struct mddev *mddev, int size)
{
@@ -7086,6 +7137,7 @@ raid5_set_cache_size(struct mddev *mddev, int size)
result = -ENOMEM;
break;
}
+ raid5_grow_stripe_hashtbl(conf, conf->max_nr_stripes);
mutex_unlock(&conf->cache_size_mutex);
return result;
@@ -7589,7 +7641,7 @@ static void free_conf(struct r5conf *conf)
put_page(conf->disks[i].extra_page);
kfree(conf->disks);
bioset_exit(&conf->bio_split);
- kfree(conf->stripe_hashtbl);
+ kvfree(conf->stripe_hashtbl);
kfree(conf->pending_data);
mempool_destroy(conf->ctx_pool);
@@ -7785,9 +7837,11 @@ static struct r5conf *setup_conf(struct mddev *mddev)
goto abort;
ret = -ENOMEM;
- conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL);
+ conf->stripe_hashtbl = kvzalloc_objs(*conf->stripe_hashtbl,
+ NR_HASH_MIN);
if (!conf->stripe_hashtbl)
goto abort;
+ conf->hash_mask = NR_HASH_MIN - 1;
/* We init hash_locks[0] separately to that it can be used
* as the reference lock in the spin_lock_nest_lock() call
diff --git a/drivers/md/raid5.h b/drivers/md/raid5.h
index 1a61b474fc22..6d1d4aa125f2 100644
--- a/drivers/md/raid5.h
+++ b/drivers/md/raid5.h
@@ -489,8 +489,12 @@ struct disk_info {
#define IO_THRESHOLD 1
#define BYPASS_THRESHOLD 1
-#define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head))
-#define HASH_MASK (NR_HASH - 1)
+/*
+ * The stripe hash table starts at one page and is enlarged with
+ * stripe_cache_size, to about STRIPES_PER_HASH_BUCKET stripes per bucket.
+ */
+#define NR_HASH_MIN (PAGE_SIZE / sizeof(struct hlist_head))
+#define STRIPES_PER_HASH_BUCKET 2
#define MAX_STRIPE_BATCH 8
/* NOTE NR_STRIPE_HASH_LOCKS must remain below 64.
@@ -570,7 +574,11 @@ struct raid5_percpu {
struct r5conf {
struct hlist_head *stripe_hashtbl;
- /* only protect corresponding hash list and inactive_list */
+ unsigned int hash_mask; /* stripe_hashtbl size - 1 */
+ /*
+ * Protect corresponding hash list and inactive_list. All of them are
+ * held to replace stripe_hashtbl and hash_mask during the table growth.
+ */
spinlock_t hash_locks[NR_STRIPE_HASH_LOCKS];
struct mddev *mddev;
int chunk_sectors;
base-commit: 47e1b18f38d74015cb9fd30b5ff2a1eb88e77d31
--
2.53.0
^ permalink raw reply [flat|nested] only message in thread
only message in thread, other threads:[~2026-11-16 13:40 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2026-11-16 13:39 [PATCH] md/raid5: grow stripe hash table with the size of stripe cache Miroslav Cimerman
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®