[PATCH V3 01/27] kfifo: add kfifo_alloc_node() helper for NUMA awareness

[PATCH V3 01/27] kfifo: add kfifo_alloc_node() helper for NUMA awareness

[Ming Lei]

Add __kfifo_alloc_node() by refactoring and reusing __kfifo_alloc(),
and define kfifo_alloc_node() macro to support NUMA-aware memory
allocation.

The new __kfifo_alloc_node() function accepts a NUMA node parameter
and uses kmalloc_array_node() instead of kmalloc_array() for
node-specific allocation. The existing __kfifo_alloc() now calls
__kfifo_alloc_node() with NUMA_NO_NODE to maintain backward
compatibility.

This enables users to allocate kfifo buffers on specific NUMA nodes,
which is important for performance in NUMA systems where the kfifo
will be primarily accessed by threads running on specific nodes.

Cc: Stefani Seibold <stefani@seibold.net>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ming Lei <ming.lei@redhat.com>
---
 include/linux/kfifo.h | 27 +++++++++++++++++++++++++++
 lib/kfifo.c           | 13 ++++++++++---
 2 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/include/linux/kfifo.h b/include/linux/kfifo.h
index fd743d4c4b4b..61d1fe014a6c 100644
--- a/include/linux/kfifo.h
+++ b/include/linux/kfifo.h
@@ -369,6 +369,30 @@ __kfifo_int_must_check_helper( \
 }) \
 )
 
+/**
+ * kfifo_alloc_node - dynamically allocates a new fifo buffer on a NUMA node
+ * @fifo: pointer to the fifo
+ * @size: the number of elements in the fifo, this must be a power of 2
+ * @gfp_mask: get_free_pages mask, passed to kmalloc()
+ * @node: NUMA node to allocate memory on
+ *
+ * This macro dynamically allocates a new fifo buffer with NUMA node awareness.
+ *
+ * The number of elements will be rounded-up to a power of 2.
+ * The fifo will be release with kfifo_free().
+ * Return 0 if no error, otherwise an error code.
+ */
+#define kfifo_alloc_node(fifo, size, gfp_mask, node) \
+__kfifo_int_must_check_helper( \
+({ \
+	typeof((fifo) + 1) __tmp = (fifo); \
+	struct __kfifo *__kfifo = &__tmp->kfifo; \
+	__is_kfifo_ptr(__tmp) ? \
+	__kfifo_alloc_node(__kfifo, size, sizeof(*__tmp->type), gfp_mask, node) : \
+	-EINVAL; \
+}) \
+)
+
 /**
  * kfifo_free - frees the fifo
  * @fifo: the fifo to be freed
@@ -902,6 +926,9 @@ __kfifo_uint_must_check_helper( \
 extern int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
 	size_t esize, gfp_t gfp_mask);
 
+extern int __kfifo_alloc_node(struct __kfifo *fifo, unsigned int size,
+	size_t esize, gfp_t gfp_mask, int node);
+
 extern void __kfifo_free(struct __kfifo *fifo);
 
 extern int __kfifo_init(struct __kfifo *fifo, void *buffer,
diff --git a/lib/kfifo.c b/lib/kfifo.c
index a8b2eed90599..195cf0feecc2 100644
--- a/lib/kfifo.c
+++ b/lib/kfifo.c
@@ -22,8 +22,8 @@ static inline unsigned int kfifo_unused(struct __kfifo *fifo)
 	return (fifo->mask + 1) - (fifo->in - fifo->out);
 }
 
-int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
-		size_t esize, gfp_t gfp_mask)
+int __kfifo_alloc_node(struct __kfifo *fifo, unsigned int size,
+		size_t esize, gfp_t gfp_mask, int node)
 {
 	/*
 	 * round up to the next power of 2, since our 'let the indices
@@ -41,7 +41,7 @@ int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
 		return -EINVAL;
 	}
 
-	fifo->data = kmalloc_array(esize, size, gfp_mask);
+	fifo->data = kmalloc_array_node(esize, size, gfp_mask, node);
 
 	if (!fifo->data) {
 		fifo->mask = 0;
@@ -51,6 +51,13 @@ int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
 
 	return 0;
 }
+EXPORT_SYMBOL(__kfifo_alloc_node);
+
+int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
+		size_t esize, gfp_t gfp_mask)
+{
+	return __kfifo_alloc_node(fifo, size, esize, gfp_mask, NUMA_NO_NODE);
+}
 EXPORT_SYMBOL(__kfifo_alloc);
 
 void __kfifo_free(struct __kfifo *fifo)
-- 
2.47.0

Re: [PATCH V3 01/27] kfifo: add kfifo_alloc_node() helper for NUMA awareness

[Andrew Morton]

On Wed, 12 Nov 2025 17:37:39 +0800 Ming Lei <ming.lei@redhat.com> wrote:

> Add __kfifo_alloc_node() by refactoring and reusing __kfifo_alloc(),
> and define kfifo_alloc_node() macro to support NUMA-aware memory
> allocation.
> 
> The new __kfifo_alloc_node() function accepts a NUMA node parameter
> and uses kmalloc_array_node() instead of kmalloc_array() for
> node-specific allocation. The existing __kfifo_alloc() now calls
> __kfifo_alloc_node() with NUMA_NO_NODE to maintain backward
> compatibility.
> 
> This enables users to allocate kfifo buffers on specific NUMA nodes,
> which is important for performance in NUMA systems where the kfifo
> will be primarily accessed by threads running on specific nodes.

I was about to ask "please don't add infrastructure without users", but
I see a "01/27" there.  I wander over to lkml but I can't find 02-27
there either.  Maybe something went wrong.

I prefer to be cc'ed on the entire series, please.

> --- a/include/linux/kfifo.h
> +++ b/include/linux/kfifo.h
> @@ -369,6 +369,30 @@ __kfifo_int_must_check_helper( \
>  }) \
>  )
>  
> +/**
> + * kfifo_alloc_node - dynamically allocates a new fifo buffer on a NUMA node
> + * @fifo: pointer to the fifo
> + * @size: the number of elements in the fifo, this must be a power of 2
> + * @gfp_mask: get_free_pages mask, passed to kmalloc()
> + * @node: NUMA node to allocate memory on
> + *
> + * This macro dynamically allocates a new fifo buffer with NUMA node awareness.
> + *
> + * The number of elements will be rounded-up to a power of 2.
> + * The fifo will be release with kfifo_free().
> + * Return 0 if no error, otherwise an error code.
> + */
> +#define kfifo_alloc_node(fifo, size, gfp_mask, node) \
> +__kfifo_int_must_check_helper( \
> +({ \
> +	typeof((fifo) + 1) __tmp = (fifo); \
> +	struct __kfifo *__kfifo = &__tmp->kfifo; \
> +	__is_kfifo_ptr(__tmp) ? \
> +	__kfifo_alloc_node(__kfifo, size, sizeof(*__tmp->type), gfp_mask, node) : \
> +	-EINVAL; \
> +}) \
> +)

Well this is an eyesore.  Do we really need it?  It seems to be here so
we can check for a programming bug?  Well, don't add programming bugs!

I'm actually not enjoying the existence of __is_kfifo_ptr() at all. 
What  is it all doing?  It's a FIFO for heck's sake, why is this so hard.

> @@ -902,6 +926,9 @@ __kfifo_uint_must_check_helper( \
>  extern int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
>  	size_t esize, gfp_t gfp_mask);
>  
> +extern int __kfifo_alloc_node(struct __kfifo *fifo, unsigned int size,
> +	size_t esize, gfp_t gfp_mask, int node);
> +

Nit: please align things like this:

extern int __kfifo_alloc_node(struct __kfifo *fifo, unsigned int size,
			      size_t esize, gfp_t gfp_mask, int node);

(several places)

Re: [PATCH V3 01/27] kfifo: add kfifo_alloc_node() helper for NUMA awareness

[Ming Lei]

On Wed, Nov 12, 2025 at 11:29:14AM -0800, Andrew Morton wrote:
> On Wed, 12 Nov 2025 17:37:39 +0800 Ming Lei <ming.lei@redhat.com> wrote:
> 
> > Add __kfifo_alloc_node() by refactoring and reusing __kfifo_alloc(),
> > and define kfifo_alloc_node() macro to support NUMA-aware memory
> > allocation.
> > 
> > The new __kfifo_alloc_node() function accepts a NUMA node parameter
> > and uses kmalloc_array_node() instead of kmalloc_array() for
> > node-specific allocation. The existing __kfifo_alloc() now calls
> > __kfifo_alloc_node() with NUMA_NO_NODE to maintain backward
> > compatibility.
> > 
> > This enables users to allocate kfifo buffers on specific NUMA nodes,
> > which is important for performance in NUMA systems where the kfifo
> > will be primarily accessed by threads running on specific nodes.
> 
> I was about to ask "please don't add infrastructure without users", but
> I see a "01/27" there.  I wander over to lkml but I can't find 02-27
> there either.  Maybe something went wrong.

It can be found in lore:

https://lore.kernel.org/all/20251112093808.2134129-1-ming.lei@redhat.com/

> 
> I prefer to be cc'ed on the entire series, please.

OK.

> 
> > --- a/include/linux/kfifo.h
> > +++ b/include/linux/kfifo.h
> > @@ -369,6 +369,30 @@ __kfifo_int_must_check_helper( \
> >  }) \
> >  )
> >  
> > +/**
> > + * kfifo_alloc_node - dynamically allocates a new fifo buffer on a NUMA node
> > + * @fifo: pointer to the fifo
> > + * @size: the number of elements in the fifo, this must be a power of 2
> > + * @gfp_mask: get_free_pages mask, passed to kmalloc()
> > + * @node: NUMA node to allocate memory on
> > + *
> > + * This macro dynamically allocates a new fifo buffer with NUMA node awareness.
> > + *
> > + * The number of elements will be rounded-up to a power of 2.
> > + * The fifo will be release with kfifo_free().
> > + * Return 0 if no error, otherwise an error code.
> > + */
> > +#define kfifo_alloc_node(fifo, size, gfp_mask, node) \
> > +__kfifo_int_must_check_helper( \
> > +({ \
> > +	typeof((fifo) + 1) __tmp = (fifo); \
> > +	struct __kfifo *__kfifo = &__tmp->kfifo; \
> > +	__is_kfifo_ptr(__tmp) ? \
> > +	__kfifo_alloc_node(__kfifo, size, sizeof(*__tmp->type), gfp_mask, node) : \
> > +	-EINVAL; \
> > +}) \
> > +)
> 
> Well this is an eyesore.  Do we really need it?  It seems to be here so
> we can check for a programming bug?  Well, don't add programming bugs!
> 
> I'm actually not enjoying the existence of __is_kfifo_ptr() at all. 
> What  is it all doing?  It's a FIFO for heck's sake, why is this so hard.

It is basically a clone of existing kfifo_alloc().

Do we need to clean kfifo_alloc() first? Otherwise I'd keep the same
pattern with existing definitions.

> 
> > @@ -902,6 +926,9 @@ __kfifo_uint_must_check_helper( \
> >  extern int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
> >  	size_t esize, gfp_t gfp_mask);
> >  
> > +extern int __kfifo_alloc_node(struct __kfifo *fifo, unsigned int size,
> > +	size_t esize, gfp_t gfp_mask, int node);
> > +
> 
> Nit: please align things like this:
> 
> extern int __kfifo_alloc_node(struct __kfifo *fifo, unsigned int size,
> 			      size_t esize, gfp_t gfp_mask, int node);

OK.

Thanks,
Ming

Re: [PATCH V3 01/27] kfifo: add kfifo_alloc_node() helper for NUMA awareness

[Andrew Morton]

On Thu, 13 Nov 2025 09:21:41 +0800 Ming Lei <ming.lei@redhat.com> wrote:

> > > +#define kfifo_alloc_node(fifo, size, gfp_mask, node) \
> > > +__kfifo_int_must_check_helper( \
> > > +({ \
> > > +	typeof((fifo) + 1) __tmp = (fifo); \
> > > +	struct __kfifo *__kfifo = &__tmp->kfifo; \
> > > +	__is_kfifo_ptr(__tmp) ? \
> > > +	__kfifo_alloc_node(__kfifo, size, sizeof(*__tmp->type), gfp_mask, node) : \
> > > +	-EINVAL; \
> > > +}) \
> > > +)
> > 
> > Well this is an eyesore.  Do we really need it?  It seems to be here so
> > we can check for a programming bug?  Well, don't add programming bugs!
> > 
> > I'm actually not enjoying the existence of __is_kfifo_ptr() at all. 
> > What  is it all doing?  It's a FIFO for heck's sake, why is this so hard.
> 
> It is basically a clone of existing kfifo_alloc().
> 
> Do we need to clean kfifo_alloc() first? Otherwise I'd keep the same
> pattern with existing definitions.

Is OK, I was just whining.  Possibly de-over-engineering kifo is a
separate project.

Re: [PATCH V3 01/27] kfifo: add kfifo_alloc_node() helper for NUMA awareness

[Caleb Sander Mateos]

On Wed, Nov 12, 2025 at 1:38 AM Ming Lei <ming.lei@redhat.com> wrote:
>
> Add __kfifo_alloc_node() by refactoring and reusing __kfifo_alloc(),
> and define kfifo_alloc_node() macro to support NUMA-aware memory
> allocation.
>
> The new __kfifo_alloc_node() function accepts a NUMA node parameter
> and uses kmalloc_array_node() instead of kmalloc_array() for
> node-specific allocation. The existing __kfifo_alloc() now calls
> __kfifo_alloc_node() with NUMA_NO_NODE to maintain backward
> compatibility.
>
> This enables users to allocate kfifo buffers on specific NUMA nodes,
> which is important for performance in NUMA systems where the kfifo
> will be primarily accessed by threads running on specific nodes.
>
> Cc: Stefani Seibold <stefani@seibold.net>
> Cc: Andrew Morton <akpm@linux-foundation.org>
> Cc: linux-kernel@vger.kernel.org
> Signed-off-by: Ming Lei <ming.lei@redhat.com>
> ---
>  include/linux/kfifo.h | 27 +++++++++++++++++++++++++++
>  lib/kfifo.c           | 13 ++++++++++---
>  2 files changed, 37 insertions(+), 3 deletions(-)
>
> diff --git a/include/linux/kfifo.h b/include/linux/kfifo.h
> index fd743d4c4b4b..61d1fe014a6c 100644
> --- a/include/linux/kfifo.h
> +++ b/include/linux/kfifo.h
> @@ -369,6 +369,30 @@ __kfifo_int_must_check_helper( \
>  }) \
>  )
>
> +/**
> + * kfifo_alloc_node - dynamically allocates a new fifo buffer on a NUMA node
> + * @fifo: pointer to the fifo
> + * @size: the number of elements in the fifo, this must be a power of 2
> + * @gfp_mask: get_free_pages mask, passed to kmalloc()
> + * @node: NUMA node to allocate memory on
> + *
> + * This macro dynamically allocates a new fifo buffer with NUMA node awareness.
> + *
> + * The number of elements will be rounded-up to a power of 2.
> + * The fifo will be release with kfifo_free().
> + * Return 0 if no error, otherwise an error code.
> + */
> +#define kfifo_alloc_node(fifo, size, gfp_mask, node) \
> +__kfifo_int_must_check_helper( \
> +({ \
> +       typeof((fifo) + 1) __tmp = (fifo); \
> +       struct __kfifo *__kfifo = &__tmp->kfifo; \
> +       __is_kfifo_ptr(__tmp) ? \
> +       __kfifo_alloc_node(__kfifo, size, sizeof(*__tmp->type), gfp_mask, node) : \
> +       -EINVAL; \
> +}) \
> +)
> +
>  /**
>   * kfifo_free - frees the fifo
>   * @fifo: the fifo to be freed
> @@ -902,6 +926,9 @@ __kfifo_uint_must_check_helper( \
>  extern int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
>         size_t esize, gfp_t gfp_mask);
>
> +extern int __kfifo_alloc_node(struct __kfifo *fifo, unsigned int size,
> +       size_t esize, gfp_t gfp_mask, int node);
> +
>  extern void __kfifo_free(struct __kfifo *fifo);
>
>  extern int __kfifo_init(struct __kfifo *fifo, void *buffer,
> diff --git a/lib/kfifo.c b/lib/kfifo.c
> index a8b2eed90599..195cf0feecc2 100644
> --- a/lib/kfifo.c
> +++ b/lib/kfifo.c
> @@ -22,8 +22,8 @@ static inline unsigned int kfifo_unused(struct __kfifo *fifo)
>         return (fifo->mask + 1) - (fifo->in - fifo->out);
>  }
>
> -int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
> -               size_t esize, gfp_t gfp_mask)
> +int __kfifo_alloc_node(struct __kfifo *fifo, unsigned int size,
> +               size_t esize, gfp_t gfp_mask, int node)
>  {
>         /*
>          * round up to the next power of 2, since our 'let the indices
> @@ -41,7 +41,7 @@ int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
>                 return -EINVAL;
>         }
>
> -       fifo->data = kmalloc_array(esize, size, gfp_mask);
> +       fifo->data = kmalloc_array_node(esize, size, gfp_mask, node);
>
>         if (!fifo->data) {
>                 fifo->mask = 0;
> @@ -51,6 +51,13 @@ int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
>
>         return 0;
>  }
> +EXPORT_SYMBOL(__kfifo_alloc_node);
> +
> +int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
> +               size_t esize, gfp_t gfp_mask)
> +{
> +       return __kfifo_alloc_node(fifo, size, esize, gfp_mask, NUMA_NO_NODE);
> +}
>  EXPORT_SYMBOL(__kfifo_alloc);

Is it worth keeping __kfifo_alloc() as an extern function? Seems like
it would make the executable smaller to turn __kfifo_alloc() into a
static inline function that just defers to __kfifo_alloc_node().

Best,
Caleb

>
>  void __kfifo_free(struct __kfifo *fifo)
> --
> 2.47.0
>

Re: [PATCH V3 01/27] kfifo: add kfifo_alloc_node() helper for NUMA awareness

[Ming Lei]

On Fri, Nov 14, 2025 at 08:14:29PM -0800, Caleb Sander Mateos wrote:
> On Wed, Nov 12, 2025 at 1:38 AM Ming Lei <ming.lei@redhat.com> wrote:
> >
> > Add __kfifo_alloc_node() by refactoring and reusing __kfifo_alloc(),
> > and define kfifo_alloc_node() macro to support NUMA-aware memory
> > allocation.
> >
> > The new __kfifo_alloc_node() function accepts a NUMA node parameter
> > and uses kmalloc_array_node() instead of kmalloc_array() for
> > node-specific allocation. The existing __kfifo_alloc() now calls
> > __kfifo_alloc_node() with NUMA_NO_NODE to maintain backward
> > compatibility.
> >
> > This enables users to allocate kfifo buffers on specific NUMA nodes,
> > which is important for performance in NUMA systems where the kfifo
> > will be primarily accessed by threads running on specific nodes.
> >
> > Cc: Stefani Seibold <stefani@seibold.net>
> > Cc: Andrew Morton <akpm@linux-foundation.org>
> > Cc: linux-kernel@vger.kernel.org
> > Signed-off-by: Ming Lei <ming.lei@redhat.com>
> > ---
> >  include/linux/kfifo.h | 27 +++++++++++++++++++++++++++
> >  lib/kfifo.c           | 13 ++++++++++---
> >  2 files changed, 37 insertions(+), 3 deletions(-)
> >
> > diff --git a/include/linux/kfifo.h b/include/linux/kfifo.h
> > index fd743d4c4b4b..61d1fe014a6c 100644
> > --- a/include/linux/kfifo.h
> > +++ b/include/linux/kfifo.h
> > @@ -369,6 +369,30 @@ __kfifo_int_must_check_helper( \
> >  }) \
> >  )
> >
> > +/**
> > + * kfifo_alloc_node - dynamically allocates a new fifo buffer on a NUMA node
> > + * @fifo: pointer to the fifo
> > + * @size: the number of elements in the fifo, this must be a power of 2
> > + * @gfp_mask: get_free_pages mask, passed to kmalloc()
> > + * @node: NUMA node to allocate memory on
> > + *
> > + * This macro dynamically allocates a new fifo buffer with NUMA node awareness.
> > + *
> > + * The number of elements will be rounded-up to a power of 2.
> > + * The fifo will be release with kfifo_free().
> > + * Return 0 if no error, otherwise an error code.
> > + */
> > +#define kfifo_alloc_node(fifo, size, gfp_mask, node) \
> > +__kfifo_int_must_check_helper( \
> > +({ \
> > +       typeof((fifo) + 1) __tmp = (fifo); \
> > +       struct __kfifo *__kfifo = &__tmp->kfifo; \
> > +       __is_kfifo_ptr(__tmp) ? \
> > +       __kfifo_alloc_node(__kfifo, size, sizeof(*__tmp->type), gfp_mask, node) : \
> > +       -EINVAL; \
> > +}) \
> > +)
> > +
> >  /**
> >   * kfifo_free - frees the fifo
> >   * @fifo: the fifo to be freed
> > @@ -902,6 +926,9 @@ __kfifo_uint_must_check_helper( \
> >  extern int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
> >         size_t esize, gfp_t gfp_mask);
> >
> > +extern int __kfifo_alloc_node(struct __kfifo *fifo, unsigned int size,
> > +       size_t esize, gfp_t gfp_mask, int node);
> > +
> >  extern void __kfifo_free(struct __kfifo *fifo);
> >
> >  extern int __kfifo_init(struct __kfifo *fifo, void *buffer,
> > diff --git a/lib/kfifo.c b/lib/kfifo.c
> > index a8b2eed90599..195cf0feecc2 100644
> > --- a/lib/kfifo.c
> > +++ b/lib/kfifo.c
> > @@ -22,8 +22,8 @@ static inline unsigned int kfifo_unused(struct __kfifo *fifo)
> >         return (fifo->mask + 1) - (fifo->in - fifo->out);
> >  }
> >
> > -int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
> > -               size_t esize, gfp_t gfp_mask)
> > +int __kfifo_alloc_node(struct __kfifo *fifo, unsigned int size,
> > +               size_t esize, gfp_t gfp_mask, int node)
> >  {
> >         /*
> >          * round up to the next power of 2, since our 'let the indices
> > @@ -41,7 +41,7 @@ int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
> >                 return -EINVAL;
> >         }
> >
> > -       fifo->data = kmalloc_array(esize, size, gfp_mask);
> > +       fifo->data = kmalloc_array_node(esize, size, gfp_mask, node);
> >
> >         if (!fifo->data) {
> >                 fifo->mask = 0;
> > @@ -51,6 +51,13 @@ int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
> >
> >         return 0;
> >  }
> > +EXPORT_SYMBOL(__kfifo_alloc_node);
> > +
> > +int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
> > +               size_t esize, gfp_t gfp_mask)
> > +{
> > +       return __kfifo_alloc_node(fifo, size, esize, gfp_mask, NUMA_NO_NODE);
> > +}
> >  EXPORT_SYMBOL(__kfifo_alloc);
> 
> Is it worth keeping __kfifo_alloc() as an extern function? Seems like
> it would make the executable smaller to turn __kfifo_alloc() into a
> static inline function that just defers to __kfifo_alloc_node().

OK, will convert to inline __kfifo_alloc() in next version.


Thanks,
Ming