Re: [PATCH v2 01/11] percpu: Introduce percpu hot section

[Uros Bizjak <ubizjak@gmail.com>]

On Wed, Feb 26, 2025 at 7:05 PM Brian Gerst <brgerst@gmail.com> wrote:
>
> Add a subsection to the percpu data for frequently accessed variables
> that should remain cached on each processor.  These varables should not
> be accessed from other processors to avoid cacheline bouncing.
>
> This will replace the pcpu_hot struct on x86, and open up similar
> functionality to other architectures and the kernel core.
>
> Signed-off-by: Brian Gerst <brgerst@gmail.com>
> ---
>  include/asm-generic/vmlinux.lds.h | 10 ++++++++++
>  include/linux/percpu-defs.h       | 12 ++++++++++++
>  2 files changed, 22 insertions(+)
>
> diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
> index 92fc06f7da74..92dd6065fd0a 100644
> --- a/include/asm-generic/vmlinux.lds.h
> +++ b/include/asm-generic/vmlinux.lds.h
> @@ -385,6 +385,11 @@ defined(CONFIG_AUTOFDO_CLANG) || defined(CONFIG_PROPELLER_CLANG)
>         . = ALIGN(PAGE_SIZE);                                           \
>         __nosave_end = .;
>
> +#define CACHE_HOT_DATA(align)                                          \
> +       . = ALIGN(align);                                               \
> +       *(SORT_BY_ALIGNMENT(.data..hot.*))                              \
> +       . = ALIGN(align);
> +
>  #define PAGE_ALIGNED_DATA(page_align)                                  \
>         . = ALIGN(page_align);                                          \
>         *(.data..page_aligned)                                          \
> @@ -1065,6 +1070,10 @@ defined(CONFIG_AUTOFDO_CLANG) || defined(CONFIG_PROPELLER_CLANG)
>         . = ALIGN(PAGE_SIZE);                                           \
>         *(.data..percpu..page_aligned)                                  \
>         . = ALIGN(cacheline);                                           \
> +       __per_cpu_hot_start = .;                                        \
> +       *(SORT_BY_ALIGNMENT(.data..percpu..hot.*))                      \
> +       . = ALIGN(cacheline);                                           \
> +       __per_cpu_hot_end = .;                                          \
>         *(.data..percpu..read_mostly)                                   \
>         . = ALIGN(cacheline);                                           \
>         *(.data..percpu)                                                \
> @@ -1112,6 +1121,7 @@ defined(CONFIG_AUTOFDO_CLANG) || defined(CONFIG_PROPELLER_CLANG)
>                 INIT_TASK_DATA(inittask)                                \
>                 NOSAVE_DATA                                             \
>                 PAGE_ALIGNED_DATA(pagealigned)                          \
> +               CACLE_HOT_DATA(cacheline)                               \

There is a typo in the above macro name.

Uros.

>                 CACHELINE_ALIGNED_DATA(cacheline)                       \
>                 READ_MOSTLY_DATA(cacheline)                             \
>                 DATA_DATA                                               \
> diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h
> index 40d34e032d5b..eb3393f96e5a 100644
> --- a/include/linux/percpu-defs.h
> +++ b/include/linux/percpu-defs.h
> @@ -112,6 +112,18 @@
>  #define DEFINE_PER_CPU(type, name)                                     \
>         DEFINE_PER_CPU_SECTION(type, name, "")
>
> +/*
> + * Declaration/definition used for per-CPU variables that are frequently
> + * accessed and should be in a single cacheline.
> + *
> + * For use only by architecture and core code.
> + */
> +#define DECLARE_PER_CPU_CACHE_HOT(type, name)                          \
> +       DECLARE_PER_CPU_SECTION(type, name, "..hot.." #name)
> +
> +#define DEFINE_PER_CPU_CACHE_HOT(type, name)                           \
> +       DEFINE_PER_CPU_SECTION(type, name, "..hot.." #name)
> +
>  /*
>   * Declaration/definition used for per-CPU variables that must be cacheline
>   * aligned under SMP conditions so that, whilst a particular instance of the
> --
> 2.48.1
>