Re: [RFC PATCH 2/2] mm/slub: introduce slab parking to reduce list_lock contention

["Vlastimil Babka (SUSE)" <vbabka@kernel.org>]

On 9/11/26 15:06, Hao Li wrote:
> --- >8 ---
> mm/slub: refill prefilled sheaves from the barn
> 
> For evaluation.
> 
> ---
>  mm/slub.c | 118 +++++++++++++++++++++++++++++++++++++++++++++++++++---
>  1 file changed, 112 insertions(+), 6 deletions(-)
> 
> diff --git a/mm/slub.c b/mm/slub.c
> index 27a78d63f537..7d337dbe26a5 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -427,6 +427,8 @@ struct node_barn {
>  	spinlock_t lock;
>  	struct list_head sheaves_full;
>  	struct list_head sheaves_empty;
> +	/* leftovers of prefill or refill, protected by barn->lock */

All the fields are protected by the lock so I don't think it's worth
pointing out specifically.
"prefill or refill" sounds like it may include also internal sheaf refill,
which is not the case (but maybe something worth exploring later)

> +	struct slab_sheaf *sheaf_partial;
>  	unsigned int nr_full;
>  	unsigned int nr_empty;
>  };
> @@ -3183,6 +3185,69 @@ static struct slab_sheaf *barn_get_empty_sheaf(struct node_barn *barn,
>  	return empty;
>  }
>  
> +/*
> + * Exchange @sheaf, which holds fewer objects than requested, for a full sheaf
> + * from the barn. The objects of @sheaf stay in the barn: it becomes the barn's
> + * partial sheaf, or its objects are merged into the existing one, which becomes
> + * a full sheaf when it reaches capacity. That way the leftovers neither occupy
> + * a full-sheaf count nor have to be flushed.
> + *
> + * Returns NULL if the barn has no full sheaf.
> + */
> +static struct slab_sheaf *barn_replace_partial_sheaf(struct kmem_cache *s,
> +						     struct node_barn *barn,
> +						     struct slab_sheaf *sheaf)
> +{
> +	struct slab_sheaf *full, *partial;
> +	unsigned int to_move;
> +	unsigned long flags;
> +
> +	if (!data_race(barn->nr_full))
> +		return NULL;
> +
> +	spin_lock_irqsave(&barn->lock, flags);
> +
> +	if (unlikely(!barn->nr_full)) {
> +		spin_unlock_irqrestore(&barn->lock, flags);
> +		return NULL;

Hmm what if there's no full, but there's sheaf_partial with enough objects
to make a full sheaf together with @sheaf?

> +	}
> +
> +	full = list_first_entry(&barn->sheaves_full, struct slab_sheaf,
> +				barn_list);
> +	list_del(&full->barn_list);
> +	barn->nr_full--;
> +
> +	partial = barn->sheaf_partial;
> +	if (partial) {
> +		to_move = min(sheaf->size, s->sheaf_capacity - partial->size);
> +		sheaf->size -= to_move;
> +		memcpy(&partial->objects[partial->size],
> +		       &sheaf->objects[sheaf->size], to_move * sizeof(void *));
> +		partial->size += to_move;
> +
> +		if (partial->size == s->sheaf_capacity) {
> +			list_add(&partial->barn_list, &barn->sheaves_full);
> +			barn->nr_full++;
> +			barn->sheaf_partial = NULL;
> +		}
> +	}
> +
> +	if (sheaf->size) {
> +		/* no partial sheaf, or it just became full */
> +		barn->sheaf_partial = sheaf;
> +	} else {
> +		list_add(&sheaf->barn_list, &barn->sheaves_empty);
> +		barn->nr_empty++;
> +	}
> +
> +	spin_unlock_irqrestore(&barn->lock, flags);
> +
> +	full->capacity = s->sheaf_capacity;
> +	full->pfmemalloc = false;

I think this belongs more to the caller which deals with the prefilled
sheaves specifically, and this helper is more low level.

Otherwise LGTM!

> +
> +	return full;
> +}
> +
>  /*
>   * The following two functions are used mainly in cases where we have to undo an
>   * intended action due to a race or cpu migration. Thus they do not check the
> @@ -3317,6 +3382,7 @@ static void barn_init(struct node_barn *barn)
>  	spin_lock_init(&barn->lock);
>  	INIT_LIST_HEAD(&barn->sheaves_full);
>  	INIT_LIST_HEAD(&barn->sheaves_empty);
> +	barn->sheaf_partial = NULL;
>  	barn->nr_full = 0;
>  	barn->nr_empty = 0;
>  }
> @@ -3334,6 +3400,10 @@ static void barn_shrink(struct kmem_cache *s, struct node_barn *barn)
>  	barn->nr_full = 0;
>  	list_splice_init(&barn->sheaves_empty, &empty_list);
>  	barn->nr_empty = 0;
> +	if (barn->sheaf_partial) {
> +		list_add(&barn->sheaf_partial->barn_list, &full_list);
> +		barn->sheaf_partial = NULL;
> +	}
>  
>  	spin_unlock_irqrestore(&barn->lock, flags);
>  
> @@ -5233,12 +5303,49 @@ void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int nod
>  }
>  EXPORT_SYMBOL(kmem_cache_alloc_node_noprof);
>  
> +/*
> + * Refill by swapping *@sheafp for a full sheaf from the barn. The old sheaf and
> + * its objects stay in the barn. Returns true when the new sheaf is full.
> + */
> +static bool refill_sheaf_from_barn(struct kmem_cache *s,
> +				   struct slab_sheaf **sheafp)
> +{
> +	struct node_barn *barn = get_barn(s);
> +	struct slab_sheaf *sheaf = *sheafp;
> +	struct slab_sheaf *full;
> +
> +	/* objects from pfmemalloc slabs must not enter the barn */
> +	if (!barn || sheaf->pfmemalloc)
> +		return false;
> +
> +	full = barn_replace_partial_sheaf(s, barn, sheaf);
> +	if (!full)
> +		return false;
> +
> +	stat(s, BARN_GET);
> +	*sheafp = full;
> +
> +	/*
> +	 * The sheaf may hold fewer than capacity objects: rcu_free_sheaf()
> +	 * puts it in the barn as full even when slab_free_hook() has taken
> +	 * some out (KFENCE, KASAN). __prefill_sheaf_pfmemalloc() then
> +	 * continues to refill it.
> +	 */
> +	return full->size == s->sheaf_capacity;
> +}
> +
>  static int __prefill_sheaf_pfmemalloc(struct kmem_cache *s,
> -				      struct slab_sheaf *sheaf, gfp_t gfp)
> +				      struct slab_sheaf **sheafp, gfp_t gfp)
>  {
> +	struct slab_sheaf *sheaf;
>  	gfp_t gfp_nomemalloc;
>  	int ret;
>  
> +	if (refill_sheaf_from_barn(s, sheafp))
> +		return 0;
> +
> +	sheaf = *sheafp;
> +
>  	gfp_nomemalloc = gfp | __GFP_NOMEMALLOC;
>  	if (gfp_pfmemalloc_allowed(gfp))
>  		gfp_nomemalloc |= __GFP_NOWARN;
> @@ -5331,7 +5438,7 @@ kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size)
>  		sheaf->pfmemalloc = false;
>  
>  		if (sheaf->size < size &&
> -		    __prefill_sheaf_pfmemalloc(s, sheaf, gfp)) {
> +		    __prefill_sheaf_pfmemalloc(s, &sheaf, gfp)) {
>  			sheaf_flush_unused(s, sheaf);
>  			free_empty_sheaf(s, sheaf);
>  			sheaf = NULL;
> @@ -5401,11 +5508,10 @@ void kmem_cache_return_sheaf(struct kmem_cache *s, gfp_t gfp,
>   * the given size.
>   *
>   * Return: 0 on success. The sheaf will contain at least @size objects.
> - * The sheaf might have been replaced with a new one if more than
> - * sheaf->capacity objects are requested.
> + * The sheaf might have been replaced with a new one.
>   *
>   * Return: -ENOMEM on failure. Some objects might have been added to the sheaf
> - * but the sheaf will not be replaced.
> + * and the sheaf might have been replaced.
>   *
>   * In practice we always refill to full sheaf's capacity.
>   */
> @@ -5427,7 +5533,7 @@ int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp,
>  
>  	if (likely(sheaf->capacity >= size)) {
>  		if (likely(sheaf->capacity == s->sheaf_capacity))
> -			return __prefill_sheaf_pfmemalloc(s, sheaf, gfp);
> +			return __prefill_sheaf_pfmemalloc(s, sheafp, gfp);
>  
>  		if (!__kmem_cache_alloc_bulk(s, gfp, sheaf->capacity - sheaf->size,
>  					     &sheaf->objects[sheaf->size]))
>