* [PATCH 1/2] mm/shrinker: keep the per-memcg shrinker units in an xarray
2026-10-19 17:39 [PATCH 0/2] mm/shrinker: per-memcg shrinker units in an xarray, allocated on demand Konstantin Khorenko
@ 2026-10-19 17:39 ` Konstantin Khorenko
2026-10-19 17:39 ` [PATCH 2/2] mm/shrinker: allocate the per-memcg shrinker units on demand Konstantin Khorenko
2026-10-19 18:32 ` [PATCH 0/2] mm/shrinker: per-memcg shrinker units in an xarray, allocated " Harry Yoo
2 siblings, 0 replies; 5+ messages in thread
From: Konstantin Khorenko @ 2026-10-19 17:39 UTC (permalink / raw)
To: Andrew Morton, Dave Chinner, Qi Zheng, Roman Gushchin,
Muchun Song, Johannes Weiner, Michal Hocko, Shakeel Butt
Cc: linux-mm, linux-kernel, cgroups, Konstantin Khorenko
Every memory cgroup keeps, per NUMA node, an array of pointers to its
shrinker_info_units, sized for shrinker_nr_max ids. shrinker_nr_max is
the highest memcg-aware shrinker id ever handed out; every superblock
takes one, and it never comes down.
Keep the units in an xarray per memcg and node instead. Every memcg
then holds just the units it has, not an array sized for the host,
and growing shrinker_nr_max touches nothing that already exists.
The units are still allocated for every memcg up front.
Assisted-by: LLM
Signed-off-by: Konstantin Khorenko <khorenko@virtuozzo.com>
---
include/linux/memcontrol.h | 3 +-
include/linux/shrinker.h | 6 -
mm/memcontrol.c | 1 +
mm/shrinker.c | 312 +++++++++++++++------------------------------
4 files changed, 105 insertions(+), 217 deletions(-)
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index b5a89594c263c..52f3646e244ae 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -23,6 +23,7 @@
#include <linux/writeback.h>
#include <linux/page-flags.h>
#include <linux/shrinker.h>
+#include <linux/xarray.h>
struct mem_cgroup;
struct obj_cgroup;
@@ -93,7 +94,7 @@ struct mem_cgroup_per_node {
struct lruvec_stats_percpu __percpu *lruvec_stats_percpu;
struct lruvec_stats *lruvec_stats;
- struct shrinker_info __rcu *shrinker_info;
+ struct xarray shrinker_units;
#ifdef CONFIG_MEMCG_V1
/*
diff --git a/include/linux/shrinker.h b/include/linux/shrinker.h
index 1a00be90d93a9..2eae68a7ff6c5 100644
--- a/include/linux/shrinker.h
+++ b/include/linux/shrinker.h
@@ -18,12 +18,6 @@ struct shrinker_info_unit {
DECLARE_BITMAP(map, SHRINKER_UNIT_BITS);
};
-struct shrinker_info {
- struct rcu_head rcu;
- int map_nr_max;
- struct shrinker_info_unit *unit[];
-};
-
/*
* This struct is used to pass information from page reclaim to the shrinkers.
* We consolidate the values for easier extension later.
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 7a817181168ed..a13f3984f0e09 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -4110,6 +4110,7 @@ static bool alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
INIT_LIST_HEAD(&pn->objcg_list);
lruvec_init(&pn->lruvec);
+ xa_init(&pn->shrinker_units);
pn->memcg = memcg;
memcg->nodeinfo[node] = pn;
diff --git a/mm/shrinker.c b/mm/shrinker.c
index 4fb424e721bca..74fc6f64459f0 100644
--- a/mm/shrinker.c
+++ b/mm/shrinker.c
@@ -19,48 +19,66 @@ void __init shrinker_unit_cache_init(void)
shrinker_unit_cachep = KMEM_CACHE(shrinker_info_unit, SLAB_PANIC);
}
-static inline int shrinker_unit_size(int nr_items)
+static inline int shrinker_id_to_index(int shrinker_id)
{
- return (DIV_ROUND_UP(nr_items, SHRINKER_UNIT_BITS) * sizeof(struct shrinker_info_unit *));
+ return shrinker_id / SHRINKER_UNIT_BITS;
}
-static inline void shrinker_unit_free(struct shrinker_info *info, int start)
+static inline int shrinker_id_to_offset(int shrinker_id)
{
- struct shrinker_info_unit **unit;
- int nr, i;
+ return shrinker_id % SHRINKER_UNIT_BITS;
+}
- if (!info)
- return;
+static inline int calc_shrinker_id(int index, int offset)
+{
+ return index * SHRINKER_UNIT_BITS + offset;
+}
- unit = info->unit;
- nr = DIV_ROUND_UP(info->map_nr_max, SHRINKER_UNIT_BITS);
+static struct shrinker_info_unit *shrinker_unit_get(struct mem_cgroup *memcg,
+ int nid, int shrinker_id)
+{
+ return xa_load(&memcg->nodeinfo[nid]->shrinker_units,
+ shrinker_id_to_index(shrinker_id));
+}
- for (i = start; i < nr; i++) {
- if (!unit[i])
- break;
+/*
+ * A memcg is visible to expand_shrinker_info() before it goes online, so
+ * a unit may already be there from either side.
+ */
+static int shrinker_unit_install(struct mem_cgroup *memcg, int nid, int index)
+{
+ struct xarray *units = &memcg->nodeinfo[nid]->shrinker_units;
+ struct shrinker_info_unit *unit;
+ int ret;
- kmem_cache_free(shrinker_unit_cachep, unit[i]);
- unit[i] = NULL;
+ if (xa_load(units, index))
+ return 0;
+
+ unit = kmem_cache_alloc_node(shrinker_unit_cachep,
+ GFP_KERNEL | __GFP_ZERO, nid);
+ if (!unit)
+ return -ENOMEM;
+
+ ret = xa_insert(units, index, unit, GFP_KERNEL);
+ if (ret) {
+ kmem_cache_free(shrinker_unit_cachep, unit);
+ if (ret == -EBUSY)
+ ret = 0;
}
+
+ return ret;
}
-static inline int shrinker_unit_alloc(struct shrinker_info *new,
- struct shrinker_info *old, int nid)
+static int shrinker_units_install(struct mem_cgroup *memcg, int start, int end)
{
- struct shrinker_info_unit *unit;
- int nr = DIV_ROUND_UP(new->map_nr_max, SHRINKER_UNIT_BITS);
- int start = old ? DIV_ROUND_UP(old->map_nr_max, SHRINKER_UNIT_BITS) : 0;
- int i;
-
- for (i = start; i < nr; i++) {
- unit = kmem_cache_alloc_node(shrinker_unit_cachep,
- GFP_KERNEL | __GFP_ZERO, nid);
- if (!unit) {
- shrinker_unit_free(new, start);
- return -ENOMEM;
- }
+ int nid, index, ret;
- new->unit[i] = unit;
+ for_each_node(nid) {
+ for (index = start; index < end; index++) {
+ ret = shrinker_unit_install(memcg, nid, index);
+ if (ret)
+ return ret;
+ }
}
return 0;
@@ -68,18 +86,18 @@ static inline int shrinker_unit_alloc(struct shrinker_info *new,
static void __free_shrinker_info(struct mem_cgroup *memcg)
{
- struct mem_cgroup_per_node *pn;
- struct shrinker_info *info;
+ struct shrinker_info_unit *unit;
+ unsigned long index;
int nid;
lockdep_assert_held(&shrinker_mutex);
for_each_node(nid) {
- pn = memcg->nodeinfo[nid];
- info = rcu_dereference_protected(pn->shrinker_info, true);
- shrinker_unit_free(info, 0);
- kvfree(info);
- rcu_assign_pointer(pn->shrinker_info, NULL);
+ struct xarray *units = &memcg->nodeinfo[nid]->shrinker_units;
+
+ xa_for_each(units, index, unit)
+ kmem_cache_free(shrinker_unit_cachep, unit);
+ xa_destroy(units);
}
}
@@ -92,82 +110,22 @@ void free_shrinker_info(struct mem_cgroup *memcg)
int alloc_shrinker_info(struct mem_cgroup *memcg)
{
- int nid, ret = 0;
- int array_size = 0;
+ int ret;
mutex_lock(&shrinker_mutex);
- array_size = shrinker_unit_size(shrinker_nr_max);
- for_each_node(nid) {
- struct shrinker_info *info = kvzalloc_node(sizeof(*info) + array_size,
- GFP_KERNEL, nid);
- if (!info)
- goto err;
- info->map_nr_max = shrinker_nr_max;
- if (shrinker_unit_alloc(info, NULL, nid)) {
- kvfree(info);
- goto err;
- }
- rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_info, info);
- }
+ ret = shrinker_units_install(memcg, 0,
+ shrinker_id_to_index(shrinker_nr_max));
+ if (ret)
+ __free_shrinker_info(memcg);
mutex_unlock(&shrinker_mutex);
return ret;
-
-err:
- __free_shrinker_info(memcg);
- mutex_unlock(&shrinker_mutex);
- return -ENOMEM;
-}
-
-static struct shrinker_info *shrinker_info_protected(struct mem_cgroup *memcg,
- int nid)
-{
- return rcu_dereference_protected(memcg->nodeinfo[nid]->shrinker_info,
- lockdep_is_held(&shrinker_mutex));
-}
-
-static int expand_one_shrinker_info(struct mem_cgroup *memcg, int new_size,
- int old_size, int new_nr_max)
-{
- struct shrinker_info *new, *old;
- struct mem_cgroup_per_node *pn;
- int nid;
-
- for_each_node(nid) {
- pn = memcg->nodeinfo[nid];
- old = shrinker_info_protected(memcg, nid);
- /* Not yet online memcg */
- if (!old)
- return 0;
-
- /* Already expanded this shrinker_info */
- if (new_nr_max <= old->map_nr_max)
- continue;
-
- new = kvzalloc_node(sizeof(*new) + new_size, GFP_KERNEL, nid);
- if (!new)
- return -ENOMEM;
-
- new->map_nr_max = new_nr_max;
-
- memcpy(new->unit, old->unit, old_size);
- if (shrinker_unit_alloc(new, old, nid)) {
- kvfree(new);
- return -ENOMEM;
- }
-
- rcu_assign_pointer(pn->shrinker_info, new);
- kvfree_rcu(old, rcu);
- }
-
- return 0;
}
static int expand_shrinker_info(int new_id)
{
int ret = 0;
int new_nr_max = round_up(new_id + 1, SHRINKER_UNIT_BITS);
- int new_size, old_size = 0;
struct mem_cgroup *memcg;
if (!root_mem_cgroup)
@@ -175,13 +133,11 @@ static int expand_shrinker_info(int new_id)
lockdep_assert_held(&shrinker_mutex);
- new_size = shrinker_unit_size(new_nr_max);
- old_size = shrinker_unit_size(shrinker_nr_max);
-
memcg = mem_cgroup_iter(NULL, NULL, NULL);
do {
- ret = expand_one_shrinker_info(memcg, new_size, old_size,
- new_nr_max);
+ ret = shrinker_units_install(memcg,
+ shrinker_id_to_index(shrinker_nr_max),
+ shrinker_id_to_index(new_nr_max));
if (ret) {
mem_cgroup_iter_break(NULL, memcg);
goto out;
@@ -194,38 +150,20 @@ static int expand_shrinker_info(int new_id)
return ret;
}
-static inline int shrinker_id_to_index(int shrinker_id)
-{
- return shrinker_id / SHRINKER_UNIT_BITS;
-}
-
-static inline int shrinker_id_to_offset(int shrinker_id)
+void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
{
- return shrinker_id % SHRINKER_UNIT_BITS;
-}
+ struct shrinker_info_unit *unit;
-static inline int calc_shrinker_id(int index, int offset)
-{
- return index * SHRINKER_UNIT_BITS + offset;
-}
+ if (shrinker_id < 0 || !memcg || mem_cgroup_is_root(memcg))
+ return;
-void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
-{
- if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
- struct shrinker_info *info;
+ unit = shrinker_unit_get(memcg, nid, shrinker_id);
+ if (WARN_ON_ONCE(!unit))
+ return;
- rcu_read_lock();
- info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
- if (!WARN_ON_ONCE(shrinker_id >= info->map_nr_max)) {
- struct shrinker_info_unit *unit;
-
- unit = info->unit[shrinker_id_to_index(shrinker_id)];
- /* Pairs with smp mb in shrink_slab() */
- smp_mb__before_atomic();
- set_bit(shrinker_id_to_offset(shrinker_id), unit->map);
- }
- rcu_read_unlock();
- }
+ /* Pairs with smp mb in shrink_slab() */
+ smp_mb__before_atomic();
+ set_bit(shrinker_id_to_offset(shrinker_id), unit->map);
}
static DEFINE_IDR(shrinker_idr);
@@ -270,52 +208,44 @@ static void shrinker_memcg_remove(struct shrinker *shrinker)
static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker,
struct mem_cgroup *memcg)
{
- struct shrinker_info *info;
struct shrinker_info_unit *unit;
- long nr_deferred;
- rcu_read_lock();
- info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
- unit = info->unit[shrinker_id_to_index(shrinker->id)];
- nr_deferred = atomic_long_xchg(&unit->nr_deferred[shrinker_id_to_offset(shrinker->id)], 0);
- rcu_read_unlock();
+ unit = shrinker_unit_get(memcg, nid, shrinker->id);
+ if (WARN_ON_ONCE(!unit))
+ return 0;
- return nr_deferred;
+ return atomic_long_xchg(&unit->nr_deferred[shrinker_id_to_offset(shrinker->id)], 0);
}
static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,
struct mem_cgroup *memcg)
{
- struct shrinker_info *info;
struct shrinker_info_unit *unit;
- long nr_deferred;
- rcu_read_lock();
- info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
- unit = info->unit[shrinker_id_to_index(shrinker->id)];
- nr_deferred =
- atomic_long_add_return(nr, &unit->nr_deferred[shrinker_id_to_offset(shrinker->id)]);
- rcu_read_unlock();
+ unit = shrinker_unit_get(memcg, nid, shrinker->id);
+ if (WARN_ON_ONCE(!unit))
+ return 0;
- return nr_deferred;
+ return atomic_long_add_return(nr, &unit->nr_deferred[shrinker_id_to_offset(shrinker->id)]);
}
void reparent_shrinker_deferred(struct mem_cgroup *memcg)
{
- int nid, index, offset;
- long nr;
struct mem_cgroup *parent = parent_mem_cgroup(memcg);
- struct shrinker_info *child_info, *parent_info;
struct shrinker_info_unit *child_unit, *parent_unit;
+ unsigned long index;
+ int nid, offset;
+ long nr;
/* Prevent from concurrent shrinker_info expand */
mutex_lock(&shrinker_mutex);
for_each_node(nid) {
- child_info = shrinker_info_protected(memcg, nid);
- parent_info = shrinker_info_protected(parent, nid);
- for (index = 0; index < shrinker_id_to_index(child_info->map_nr_max); index++) {
- child_unit = child_info->unit[index];
- parent_unit = parent_info->unit[index];
+ struct xarray *parent_units = &parent->nodeinfo[nid]->shrinker_units;
+
+ xa_for_each(&memcg->nodeinfo[nid]->shrinker_units, index, child_unit) {
+ parent_unit = xa_load(parent_units, index);
+ if (WARN_ON_ONCE(!parent_unit))
+ continue;
for (offset = 0; offset < SHRINKER_UNIT_BITS; offset++) {
nr = atomic_long_read(&child_unit->nr_deferred[offset]);
atomic_long_add(nr, &parent_unit->nr_deferred[offset]);
@@ -485,9 +415,9 @@ static unsigned long do_shrink_slab(struct shrink_control *shrinkctl,
static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
struct mem_cgroup *memcg, int priority)
{
- struct shrinker_info *info;
- unsigned long ret, freed = 0;
- int offset, index = 0;
+ struct shrinker_info_unit *unit;
+ unsigned long index, ret, freed = 0;
+ int offset;
if (!mem_cgroup_online(memcg))
return 0;
@@ -495,51 +425,17 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
/*
* lockless algorithm of memcg shrink.
*
- * The shrinker_info may be freed asynchronously via RCU in the
- * expand_one_shrinker_info(), so the rcu_read_lock() needs to be used
- * to ensure the existence of the shrinker_info.
- *
- * The shrinker_info_unit is never freed unless its corresponding memcg
- * is destroyed. Here we already hold the refcount of memcg, so the
- * memcg will not be destroyed, and of course shrinker_info_unit will
- * not be freed.
+ * A unit is only freed together with its memcg, and the caller holds
+ * a reference to the memcg, so the units found by xa_for_each() stay
+ * valid without the RCU lock and do_shrink_slab() may sleep.
*
- * So in the memcg shrink:
- * step 1: use rcu_read_lock() to guarantee existence of the
- * shrinker_info.
- * step 2: after getting shrinker_info_unit we can safely release the
- * RCU lock.
- * step 3: traverse the bitmap and calculate shrinker_id
- * step 4: use rcu_read_lock() to guarantee existence of the shrinker.
- * step 5: use shrinker_id to find the shrinker, then use
- * shrinker_try_get() to guarantee existence of the shrinker,
- * then we can release the RCU lock to do do_shrink_slab() that
- * may sleep.
- * step 6: do shrinker_put() paired with step 5 to put the refcount,
- * if the refcount reaches 0, then wake up the waiter in
- * shrinker_free() by calling complete().
- * Note: here is different from the global shrink, we don't
- * need to acquire the RCU lock to guarantee existence of
- * the shrinker, because we don't need to use this
- * shrinker to traverse the next shrinker in the bitmap.
- * step 7: we have already exited the read-side of rcu critical section
- * before calling do_shrink_slab(), the shrinker_info may be
- * released in expand_one_shrinker_info(), so go back to step 1
- * to reacquire the shrinker_info.
+ * The shrinker is found by id under the RCU lock, and
+ * shrinker_try_get() pins it before the lock is dropped;
+ * shrinker_put() wakes up the waiter in shrinker_free() by calling
+ * complete() when the last reference goes. The next shrinker comes
+ * from the bitmap, not from this shrinker, so nothing else is needed.
*/
-again:
- rcu_read_lock();
- info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
- if (unlikely(!info))
- goto unlock;
-
- if (index < shrinker_id_to_index(info->map_nr_max)) {
- struct shrinker_info_unit *unit;
-
- unit = info->unit[index];
-
- rcu_read_unlock();
-
+ xa_for_each(&memcg->nodeinfo[nid]->shrinker_units, index, unit) {
for_each_set_bit(offset, unit->map, SHRINKER_UNIT_BITS) {
struct shrink_control sc = {
.gfp_mask = gfp_mask,
@@ -594,12 +490,8 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
freed += ret;
shrinker_put(shrinker);
}
-
- index++;
- goto again;
}
-unlock:
- rcu_read_unlock();
+
return freed;
}
#else /* !CONFIG_MEMCG */
--
2.43.0
^ permalink raw reply [flat|nested] 5+ messages in thread* [PATCH 2/2] mm/shrinker: allocate the per-memcg shrinker units on demand
2026-10-19 17:39 [PATCH 0/2] mm/shrinker: per-memcg shrinker units in an xarray, allocated on demand Konstantin Khorenko
2026-10-19 17:39 ` [PATCH 1/2] mm/shrinker: keep the per-memcg shrinker units in an xarray Konstantin Khorenko
@ 2026-10-19 17:39 ` Konstantin Khorenko
2026-10-19 18:32 ` [PATCH 0/2] mm/shrinker: per-memcg shrinker units in an xarray, allocated " Harry Yoo
2 siblings, 0 replies; 5+ messages in thread
From: Konstantin Khorenko @ 2026-10-19 17:39 UTC (permalink / raw)
To: Andrew Morton, Dave Chinner, Qi Zheng, Roman Gushchin,
Muchun Song, Johannes Weiner, Michal Hocko, Shakeel Butt
Cc: linux-mm, linux-kernel, cgroups, Konstantin Khorenko
Instead of a shrinker_info_unit for every SHRINKER_UNIT_BITS ids up to
shrinker_nr_max on every NUMA node, give a memory cgroup only the units
of the shrinkers it holds a list_lru list of.
The rule the code relies on:
A non-root memcg that has a list on a list_lru has the unit of that
list's shrinker, on every node.
The unit is installed before the list is published and lives until the
memcg is freed, so set_shrinker_bit() always finds it. A reader that
finds none knows the memcg never held objects of that shrinker.
Two cases outside the rule:
- the root memcg, where global reclaim keeps the deferred work of
memcg-aware shrinkers; it gets every unit up front and is the only
memcg a shrinker registration still visits;
- under cgroup.memory=nokmem a list_lru is never per-memcg, so it gets
no shrinker id either and no bit can be set.
The install runs with the gfp of the list allocation and can sit under
the i_pages lock, which is taken from interrupts, so the xarray lock is
irq-safe and a node it could not get under the lock is allocated
outside of it, as list_lru does for its own xarray.
The units of a memcg now scale with the shrinkers it and its
descendants hold objects of, not with shrinker_nr_max.
Assisted-by: LLM
Signed-off-by: Konstantin Khorenko <khorenko@virtuozzo.com>
---
include/linux/memcontrol.h | 2 +
mm/list_lru.c | 15 +++++--
mm/memcontrol.c | 7 +---
mm/shrinker.c | 98 +++++++++++++++++++++++++++++-----------------
4 files changed, 77 insertions(+), 45 deletions(-)
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 52f3646e244ae..91641cf082a4a 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -1678,6 +1678,8 @@ static inline u64 mem_cgroup_get_socket_pressure(struct mem_cgroup *memcg)
void shrinker_unit_cache_init(void);
int alloc_shrinker_info(struct mem_cgroup *memcg);
+int memcg_shrinker_unit_alloc(struct mem_cgroup *memcg, int shrinker_id,
+ gfp_t gfp);
void free_shrinker_info(struct mem_cgroup *memcg);
void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id);
void reparent_shrinker_deferred(struct mem_cgroup *memcg);
diff --git a/mm/list_lru.c b/mm/list_lru.c
index 36662d02ff963..73480d8edf641 100644
--- a/mm/list_lru.c
+++ b/mm/list_lru.c
@@ -597,6 +597,12 @@ static int __memcg_list_lru_alloc(struct mem_cgroup *memcg,
parent = parent_mem_cgroup(pos);
}
+ if (!css_is_dying(&pos->css) &&
+ memcg_shrinker_unit_alloc(pos, lru_shrinker_id(lru), gfp)) {
+ kfree(mlru);
+ return -ENOMEM;
+ }
+
if (!mlru) {
mlru = memcg_init_list_lru_one(lru, gfp);
if (!mlru)
@@ -666,13 +672,14 @@ int __list_lru_init(struct list_lru *lru, bool memcg_aware, struct shrinker *shr
int i;
#ifdef CONFIG_MEMCG
- if (shrinker)
+ if (mem_cgroup_kmem_disabled())
+ memcg_aware = false;
+
+ /* A list that is not per-memcg sets no memcg's shrinker bit. */
+ if (shrinker && memcg_aware)
lru->shrinker_id = shrinker->id;
else
lru->shrinker_id = -1;
-
- if (mem_cgroup_kmem_disabled())
- memcg_aware = false;
#endif
lru->node = kzalloc_objs(*lru->node, nr_node_ids);
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index a13f3984f0e09..07075bb220719 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -4110,7 +4110,7 @@ static bool alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
INIT_LIST_HEAD(&pn->objcg_list);
lruvec_init(&pn->lruvec);
- xa_init(&pn->shrinker_units);
+ xa_init_flags(&pn->shrinker_units, XA_FLAGS_LOCK_IRQ);
pn->memcg = memcg;
memcg->nodeinfo[node] = pn;
@@ -4277,11 +4277,6 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
memcg_online_kmem(memcg);
- /*
- * A memcg must be visible for expand_shrinker_info()
- * by the time the maps are allocated. So, we allocate maps
- * here, when mem_cgroup_iter() can't skip it.
- */
if (alloc_shrinker_info(memcg))
goto offline_kmem;
diff --git a/mm/shrinker.c b/mm/shrinker.c
index 74fc6f64459f0..cd9441f149363 100644
--- a/mm/shrinker.c
+++ b/mm/shrinker.c
@@ -42,31 +42,40 @@ static struct shrinker_info_unit *shrinker_unit_get(struct mem_cgroup *memcg,
}
/*
- * A memcg is visible to expand_shrinker_info() before it goes online, so
- * a unit may already be there from either side.
+ * Installs run from the list_lru allocation path with the caller's gfp,
+ * possibly under the i_pages lock, which is taken from interrupts: the
+ * xarray lock is irq-safe and a node it could not get under the lock is
+ * allocated outside of it.
*/
-static int shrinker_unit_install(struct mem_cgroup *memcg, int nid, int index)
+static int shrinker_unit_install(struct mem_cgroup *memcg, int nid, int index,
+ gfp_t gfp)
{
- struct xarray *units = &memcg->nodeinfo[nid]->shrinker_units;
+ XA_STATE(xas, &memcg->nodeinfo[nid]->shrinker_units, index);
struct shrinker_info_unit *unit;
- int ret;
+ unsigned long flags;
- if (xa_load(units, index))
+ if (xa_load(xas.xa, index))
return 0;
- unit = kmem_cache_alloc_node(shrinker_unit_cachep,
- GFP_KERNEL | __GFP_ZERO, nid);
+ unit = kmem_cache_alloc_node(shrinker_unit_cachep, gfp | __GFP_ZERO,
+ nid);
if (!unit)
return -ENOMEM;
- ret = xa_insert(units, index, unit, GFP_KERNEL);
- if (ret) {
+ do {
+ xas_lock_irqsave(&xas, flags);
+ if (!xas_load(&xas)) {
+ xas_store(&xas, unit);
+ if (!xas_error(&xas))
+ unit = NULL;
+ }
+ xas_unlock_irqrestore(&xas, flags);
+ } while (xas_nomem(&xas, gfp));
+
+ if (unit)
kmem_cache_free(shrinker_unit_cachep, unit);
- if (ret == -EBUSY)
- ret = 0;
- }
- return ret;
+ return xas_error(&xas);
}
static int shrinker_units_install(struct mem_cgroup *memcg, int start, int end)
@@ -75,7 +84,8 @@ static int shrinker_units_install(struct mem_cgroup *memcg, int start, int end)
for_each_node(nid) {
for (index = start; index < end; index++) {
- ret = shrinker_unit_install(memcg, nid, index);
+ ret = shrinker_unit_install(memcg, nid, index,
+ GFP_KERNEL);
if (ret)
return ret;
}
@@ -108,10 +118,17 @@ void free_shrinker_info(struct mem_cgroup *memcg)
mutex_unlock(&shrinker_mutex);
}
+/*
+ * Global reclaim keeps the deferred work of memcg-aware shrinkers in the
+ * root's units, and root has no list_lru allocation to install them from.
+ */
int alloc_shrinker_info(struct mem_cgroup *memcg)
{
int ret;
+ if (!mem_cgroup_is_root(memcg))
+ return 0;
+
mutex_lock(&shrinker_mutex);
ret = shrinker_units_install(memcg, 0,
shrinker_id_to_index(shrinker_nr_max));
@@ -124,26 +141,15 @@ int alloc_shrinker_info(struct mem_cgroup *memcg)
static int expand_shrinker_info(int new_id)
{
- int ret = 0;
int new_nr_max = round_up(new_id + 1, SHRINKER_UNIT_BITS);
- struct mem_cgroup *memcg;
-
- if (!root_mem_cgroup)
- goto out;
+ int ret = 0;
lockdep_assert_held(&shrinker_mutex);
- memcg = mem_cgroup_iter(NULL, NULL, NULL);
- do {
- ret = shrinker_units_install(memcg,
+ if (root_mem_cgroup)
+ ret = shrinker_units_install(root_mem_cgroup,
shrinker_id_to_index(shrinker_nr_max),
shrinker_id_to_index(new_nr_max));
- if (ret) {
- mem_cgroup_iter_break(NULL, memcg);
- goto out;
- }
- } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
-out:
if (!ret)
shrinker_nr_max = new_nr_max;
@@ -211,7 +217,8 @@ static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker,
struct shrinker_info_unit *unit;
unit = shrinker_unit_get(memcg, nid, shrinker->id);
- if (WARN_ON_ONCE(!unit))
+ /* No unit: this memcg never held objects of this shrinker. */
+ if (!unit)
return 0;
return atomic_long_xchg(&unit->nr_deferred[shrinker_id_to_offset(shrinker->id)], 0);
@@ -223,12 +230,35 @@ static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,
struct shrinker_info_unit *unit;
unit = shrinker_unit_get(memcg, nid, shrinker->id);
- if (WARN_ON_ONCE(!unit))
+ if (!unit)
return 0;
return atomic_long_add_return(nr, &unit->nr_deferred[shrinker_id_to_offset(shrinker->id)]);
}
+/*
+ * Called before @memcg gets a list on a list_lru of shrinker @shrinker_id:
+ * a map bit is only ever set on a memcg that has such a list.
+ */
+int memcg_shrinker_unit_alloc(struct mem_cgroup *memcg, int shrinker_id,
+ gfp_t gfp)
+{
+ int nid, ret;
+
+ if (shrinker_id < 0 || mem_cgroup_is_root(memcg))
+ return 0;
+
+ for_each_node(nid) {
+ ret = shrinker_unit_install(memcg, nid,
+ shrinker_id_to_index(shrinker_id),
+ gfp);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
void reparent_shrinker_deferred(struct mem_cgroup *memcg)
{
struct mem_cgroup *parent = parent_mem_cgroup(memcg);
@@ -237,14 +267,13 @@ void reparent_shrinker_deferred(struct mem_cgroup *memcg)
int nid, offset;
long nr;
- /* Prevent from concurrent shrinker_info expand */
- mutex_lock(&shrinker_mutex);
for_each_node(nid) {
struct xarray *parent_units = &parent->nodeinfo[nid]->shrinker_units;
xa_for_each(&memcg->nodeinfo[nid]->shrinker_units, index, child_unit) {
parent_unit = xa_load(parent_units, index);
- if (WARN_ON_ONCE(!parent_unit))
+ /* Installed but never listed: nothing deferred in it. */
+ if (!parent_unit)
continue;
for (offset = 0; offset < SHRINKER_UNIT_BITS; offset++) {
nr = atomic_long_read(&child_unit->nr_deferred[offset]);
@@ -252,7 +281,6 @@ void reparent_shrinker_deferred(struct mem_cgroup *memcg)
}
}
}
- mutex_unlock(&shrinker_mutex);
}
#else
static int shrinker_memcg_alloc(struct shrinker *shrinker)
--
2.43.0
^ permalink raw reply [flat|nested] 5+ messages in thread* Re: [PATCH 0/2] mm/shrinker: per-memcg shrinker units in an xarray, allocated on demand
2026-10-19 17:39 [PATCH 0/2] mm/shrinker: per-memcg shrinker units in an xarray, allocated on demand Konstantin Khorenko
2026-10-19 17:39 ` [PATCH 1/2] mm/shrinker: keep the per-memcg shrinker units in an xarray Konstantin Khorenko
2026-10-19 17:39 ` [PATCH 2/2] mm/shrinker: allocate the per-memcg shrinker units on demand Konstantin Khorenko
@ 2026-10-19 18:32 ` Harry Yoo
2026-10-20 16:45 ` Konstantin Khorenko
2 siblings, 1 reply; 5+ messages in thread
From: Harry Yoo @ 2026-10-19 18:32 UTC (permalink / raw)
To: Konstantin Khorenko
Cc: Andrew Morton, Dave Chinner, Qi Zheng, Roman Gushchin,
Muchun Song, Johannes Weiner, Michal Hocko, Shakeel Butt,
linux-mm, linux-kernel, cgroups
On Mon, Oct 19, 2026 at 07:39:22PM +0200, Konstantin Khorenko wrote:
> Every memory cgroup allocates, per NUMA node, one shrinker_info_unit
> per 64 memcg-aware shrinker ids up to shrinker_nr_max, whether or not
> it ever charges an object to those shrinkers, plus a pointer array
> sized for shrinker_nr_max. Every superblock registers such a shrinker,
> and shrinker_nr_max never comes down, so a host that mounts many
> filesystems and runs many cgroups pays for every superblock in every
> cgroup, live or dying, and keeps paying for new cgroups after the
> mounts are gone.
>
> This replaces patch 2/2 of v2, "mm/shrinker: allocate the per-memcg
> shrinker units on demand", with the xarray variant discussed in that
> thread. It applies on top of "mm/shrinker: allocate shrinker_info_unit
> from a dedicated kmem_cache", patch 1/2 of v2 sent on its own as v3:
> https://patch.msgid.link/20261019-vstor-144715-shrinker_info_unit-kmem_cache-v3-1-0fb061e09bf4@virtuozzo.com
>
> Patch 1 keeps the units in an xarray per memcg and node instead of the
> pointer array. A new unit is inserted where it belongs: nothing is
> reallocated, copied or published under RCU when shrinker_nr_max grows,
> and the readers find a unit with xa_load() or walk the present ones
> with xa_for_each(). The units are still allocated up front, so this
> patch changes no behaviour.
>
> Patch 2 allocates a unit only when the cgroup gets a list on that
> shrinker's list_lru, which is where map bits are set: a cgroup pays for
> the shrinkers it holds objects of, not for every superblock on the
> host. The install is an xarray insert with the gfp of the list
> allocation; there is no array to race with, so no lock of its own.
> The root memcg keeps its units for the deferred work of global reclaim
> and is the only memcg a shrinker registration still visits.
>
> Measured on a 4-CPU, single-node VM, 20000 tmpfs mounts (shrinker_nr_max
> 20096) and 2000 memory cgroups, on three kernels:
>
> base v7.3-rc7 with the kmem_cache patch: every cgroup
> has every unit, in a pointer array
> xarray base + this series: units on demand, in an xarray
> old array on demand base + patch 2/2 of v2, rebased: units on demand,
> in the pointer array
>
> old array
> base xarray on demand
> slab per empty cgroup 185 KB 18 KB 22 KB
> shrink_slab_memcg(), empty cgroup 1.6 us 0.2 us 1.6 us
> same, cgroup with objects on 100 sb 1.9 us 0.9 us 1.7 us
> mkdir/rmdir the cgroups, 4000 more
> mounts with them alive same within noise
I think 'old array on demand' is probably good enough, and replacing it
with xarray is not justifiable unless you have a realistic scenario that
would benefit from it?
--
Cheers,
Harry / Hyeonggon
^ permalink raw reply [flat|nested] 5+ messages in thread