* [RFC PATCH v0 1/3] x86/ibs: Move IBS caps definitions into its own header
2026-09-24 6:22 [RFC PATCH v0 0/3] pghot: x86: IBS Memory Profiler for hot page promotion Bharata B Rao
@ 2026-09-24 6:22 ` Bharata B Rao
2026-09-24 6:22 ` [RFC PATCH v0 2/3] x86/mm/ibs: In-kernel driver for AMD IBS Memory Profiler Bharata B Rao
` (7 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Bharata B Rao @ 2026-09-24 6:22 UTC (permalink / raw)
To: linux-kernel, linux-mm
Cc: tglx, mingo, bp, dave.hansen, x86, hpa, xin, luto, peterz,
Jonathan.Cameron, gourry, rientjes, sj, weixugc, willy,
ying.huang, ziy, dave, nifan.cxl, xuezhengchu, yiannis, akpm,
david, byungchul, kinseyho, joshua.hahnjy, yuanchu, balbirs,
shivankg, donettom, linyongting, Bharata B Rao
Subsequent patch adds IBS Memory Profiler driver that is
independent of the perf subsystem but needs the CPUID
0x8000001B capability bits. Hence move those bit definitions
out of asm/perf_event.h into a dedicated header so the new
driver can consume them without pulling in perf.
Signed-off-by: Bharata B Rao <bharata@amd.com>
---
arch/x86/include/asm/ibs-caps.h | 85 +++++++++++++++++++++++++++++++
arch/x86/include/asm/perf_event.h | 81 +----------------------------
2 files changed, 86 insertions(+), 80 deletions(-)
create mode 100644 arch/x86/include/asm/ibs-caps.h
diff --git a/arch/x86/include/asm/ibs-caps.h b/arch/x86/include/asm/ibs-caps.h
new file mode 100644
index 000000000000..ddf6c512c8f9
--- /dev/null
+++ b/arch/x86/include/asm/ibs-caps.h
@@ -0,0 +1,85 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_X86_IBS_CAPS_H
+#define _ASM_X86_IBS_CAPS_H
+
+/*
+ * IBS cpuid feature detection
+ */
+
+#define IBS_CPUID_FEATURES 0x8000001b
+
+/*
+ * Same bit mask as for IBS cpuid feature flags (Fn8000_001B_EAX), but
+ * bit 0 is used to indicate the existence of IBS.
+ */
+#define IBS_CAPS_AVAIL (1U<<0)
+#define IBS_CAPS_FETCHSAM (1U<<1)
+#define IBS_CAPS_OPSAM (1U<<2)
+#define IBS_CAPS_RDWROPCNT (1U<<3)
+#define IBS_CAPS_OPCNT (1U<<4)
+#define IBS_CAPS_BRNTRGT (1U<<5)
+#define IBS_CAPS_OPCNTEXT (1U<<6)
+#define IBS_CAPS_RIPINVALIDCHK (1U<<7)
+#define IBS_CAPS_OPBRNFUSE (1U<<8)
+#define IBS_CAPS_FETCHCTLEXTD (1U<<9)
+#define IBS_CAPS_OPDATA4 (1U<<10)
+#define IBS_CAPS_ZEN4 (1U<<11)
+#define IBS_CAPS_OPLDLAT (1U<<12)
+#define IBS_CAPS_DIS (1U<<13)
+#define IBS_CAPS_FETCHLAT (1U<<14)
+#define IBS_CAPS_BIT63_FILTER (1U<<15)
+#define IBS_CAPS_STRMST_RMTSOCKET (1U<<16)
+#define IBS_CAPS_OPDTLBPGSIZE (1U<<19)
+
+#define IBS_CAPS_DEFAULT (IBS_CAPS_AVAIL \
+ | IBS_CAPS_FETCHSAM \
+ | IBS_CAPS_OPSAM)
+
+/*
+ * IBS APIC setup
+ */
+#define IBSCTL 0x1cc
+#define IBSCTL_LVT_OFFSET_VALID (1ULL<<8)
+#define IBSCTL_LVT_OFFSET_MASK 0x0F
+
+/* IBS fetch bits/masks */
+#define IBS_FETCH_L3MISSONLY (1ULL << 59)
+#define IBS_FETCH_RAND_EN (1ULL << 57)
+#define IBS_FETCH_VAL (1ULL << 49)
+#define IBS_FETCH_ENABLE (1ULL << 48)
+#define IBS_FETCH_CNT 0xFFFF0000ULL
+#define IBS_FETCH_MAX_CNT 0x0000FFFFULL
+
+#define IBS_FETCH_2_DIS (1ULL << 0)
+#define IBS_FETCH_2_FETCHLAT_FILTER (0xFULL << 1)
+#define IBS_FETCH_2_FETCHLAT_FILTER_SHIFT (1)
+#define IBS_FETCH_2_EXCL_RIP_63_EQ_1 (1ULL << 5)
+#define IBS_FETCH_2_EXCL_RIP_63_EQ_0 (1ULL << 6)
+
+/*
+ * IBS op bits/masks
+ * The lower 7 bits of the current count are random bits
+ * preloaded by hardware and ignored in software
+ */
+#define IBS_OP_LDLAT_EN (1ULL << 63)
+#define IBS_OP_LDLAT_THRSH (0xFULL << 59)
+#define IBS_OP_LDLAT_THRSH_SHIFT (59)
+#define IBS_OP_CUR_CNT (0xFFF80ULL << 32)
+#define IBS_OP_CUR_CNT_RAND (0x0007FULL << 32)
+#define IBS_OP_CUR_CNT_EXT_MASK (0x7FULL << 52)
+#define IBS_OP_CNT_CTL (1ULL << 19)
+#define IBS_OP_VAL (1ULL << 18)
+#define IBS_OP_ENABLE (1ULL << 17)
+#define IBS_OP_L3MISSONLY (1ULL << 16)
+#define IBS_OP_MAX_CNT 0x0000FFFFULL
+#define IBS_OP_MAX_CNT_EXT 0x007FFFFFULL /* not a register bit mask */
+#define IBS_OP_MAX_CNT_EXT_MASK (0x7FULL << 20) /* separate upper 7 bits */
+#define IBS_RIP_INVALID (1ULL << 38)
+
+#define IBS_OP_2_DIS (1ULL << 0)
+#define IBS_OP_2_EXCL_RIP_63_EQ_0 (1ULL << 1)
+#define IBS_OP_2_EXCL_RIP_63_EQ_1 (1ULL << 2)
+#define IBS_OP_2_STRM_ST_FILTER (1ULL << 3)
+#define IBS_OP_2_STRM_ST_FILTER_SHIFT (3)
+
+#endif /* _ASM_X86_IBS_CAPS_H */
diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h
index 1eb13673e889..de18e63c5c3e 100644
--- a/arch/x86/include/asm/perf_event.h
+++ b/arch/x86/include/asm/perf_event.h
@@ -3,6 +3,7 @@
#define _ASM_X86_PERF_EVENT_H
#include <linux/static_call.h>
+#include <asm/ibs-caps.h>
/*
* Performance event hw details:
@@ -622,86 +623,6 @@ struct arch_pebs_cntr_header {
*/
#define EXT_PERFMON_DEBUG_FEATURES 0x80000022
-/*
- * IBS cpuid feature detection
- */
-
-#define IBS_CPUID_FEATURES 0x8000001b
-
-/*
- * Same bit mask as for IBS cpuid feature flags (Fn8000_001B_EAX), but
- * bit 0 is used to indicate the existence of IBS.
- */
-#define IBS_CAPS_AVAIL (1U<<0)
-#define IBS_CAPS_FETCHSAM (1U<<1)
-#define IBS_CAPS_OPSAM (1U<<2)
-#define IBS_CAPS_RDWROPCNT (1U<<3)
-#define IBS_CAPS_OPCNT (1U<<4)
-#define IBS_CAPS_BRNTRGT (1U<<5)
-#define IBS_CAPS_OPCNTEXT (1U<<6)
-#define IBS_CAPS_RIPINVALIDCHK (1U<<7)
-#define IBS_CAPS_OPBRNFUSE (1U<<8)
-#define IBS_CAPS_FETCHCTLEXTD (1U<<9)
-#define IBS_CAPS_OPDATA4 (1U<<10)
-#define IBS_CAPS_ZEN4 (1U<<11)
-#define IBS_CAPS_OPLDLAT (1U<<12)
-#define IBS_CAPS_DIS (1U<<13)
-#define IBS_CAPS_FETCHLAT (1U<<14)
-#define IBS_CAPS_BIT63_FILTER (1U<<15)
-#define IBS_CAPS_STRMST_RMTSOCKET (1U<<16)
-#define IBS_CAPS_OPDTLBPGSIZE (1U<<19)
-
-#define IBS_CAPS_DEFAULT (IBS_CAPS_AVAIL \
- | IBS_CAPS_FETCHSAM \
- | IBS_CAPS_OPSAM)
-
-/*
- * IBS APIC setup
- */
-#define IBSCTL 0x1cc
-#define IBSCTL_LVT_OFFSET_VALID (1ULL<<8)
-#define IBSCTL_LVT_OFFSET_MASK 0x0F
-
-/* IBS fetch bits/masks */
-#define IBS_FETCH_L3MISSONLY (1ULL << 59)
-#define IBS_FETCH_RAND_EN (1ULL << 57)
-#define IBS_FETCH_VAL (1ULL << 49)
-#define IBS_FETCH_ENABLE (1ULL << 48)
-#define IBS_FETCH_CNT 0xFFFF0000ULL
-#define IBS_FETCH_MAX_CNT 0x0000FFFFULL
-
-#define IBS_FETCH_2_DIS (1ULL << 0)
-#define IBS_FETCH_2_FETCHLAT_FILTER (0xFULL << 1)
-#define IBS_FETCH_2_FETCHLAT_FILTER_SHIFT (1)
-#define IBS_FETCH_2_EXCL_RIP_63_EQ_1 (1ULL << 5)
-#define IBS_FETCH_2_EXCL_RIP_63_EQ_0 (1ULL << 6)
-
-/*
- * IBS op bits/masks
- * The lower 7 bits of the current count are random bits
- * preloaded by hardware and ignored in software
- */
-#define IBS_OP_LDLAT_EN (1ULL << 63)
-#define IBS_OP_LDLAT_THRSH (0xFULL << 59)
-#define IBS_OP_LDLAT_THRSH_SHIFT (59)
-#define IBS_OP_CUR_CNT (0xFFF80ULL << 32)
-#define IBS_OP_CUR_CNT_RAND (0x0007FULL << 32)
-#define IBS_OP_CUR_CNT_EXT_MASK (0x7FULL << 52)
-#define IBS_OP_CNT_CTL (1ULL << 19)
-#define IBS_OP_VAL (1ULL << 18)
-#define IBS_OP_ENABLE (1ULL << 17)
-#define IBS_OP_L3MISSONLY (1ULL << 16)
-#define IBS_OP_MAX_CNT 0x0000FFFFULL
-#define IBS_OP_MAX_CNT_EXT 0x007FFFFFULL /* not a register bit mask */
-#define IBS_OP_MAX_CNT_EXT_MASK (0x7FULL << 20) /* separate upper 7 bits */
-#define IBS_RIP_INVALID (1ULL << 38)
-
-#define IBS_OP_2_DIS (1ULL << 0)
-#define IBS_OP_2_EXCL_RIP_63_EQ_0 (1ULL << 1)
-#define IBS_OP_2_EXCL_RIP_63_EQ_1 (1ULL << 2)
-#define IBS_OP_2_STRM_ST_FILTER (1ULL << 3)
-#define IBS_OP_2_STRM_ST_FILTER_SHIFT (3)
-
#ifdef CONFIG_X86_LOCAL_APIC
extern u32 get_ibs_caps(void);
extern int forward_event_to_ibs(struct perf_event *event);
--
2.34.1
^ permalink raw reply [flat|nested] 10+ messages in thread* [RFC PATCH v0 2/3] x86/mm/ibs: In-kernel driver for AMD IBS Memory Profiler
2026-09-24 6:22 [RFC PATCH v0 0/3] pghot: x86: IBS Memory Profiler for hot page promotion Bharata B Rao
2026-09-24 6:22 ` [RFC PATCH v0 1/3] x86/ibs: Move IBS caps definitions into its own header Bharata B Rao
@ 2026-09-24 6:22 ` Bharata B Rao
2026-09-24 6:22 ` [RFC PATCH v0 3/3] x86/mm/ibs: Add runtime controls for IBS memprofiler Bharata B Rao
` (6 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Bharata B Rao @ 2026-09-24 6:22 UTC (permalink / raw)
To: linux-kernel, linux-mm
Cc: tglx, mingo, bp, dave.hansen, x86, hpa, xin, luto, peterz,
Jonathan.Cameron, gourry, rientjes, sj, weixugc, willy,
ying.huang, ziy, dave, nifan.cxl, xuezhengchu, yiannis, akpm,
david, byungchul, kinseyho, joshua.hahnjy, yuanchu, balbirs,
shivankg, donettom, linyongting, Bharata B Rao
Use IBS (Instruction Based Sampling) Memory Profiler feature
present in AMD Zen6 processors for memory access tracking. The
access information obtained from IBS Memory Profiler is fed to
pghot sub-system for further action using
pghot_record_access(PGHOT_HWHINTS, ...) API.
IBS Memory Profiler as page hotness source is enabled by the
new config option HWMEM_PROFILER.
New system vector 0xea is used for memprofiler interrupt.
This commit adds the driver infrastructure (interrupt setup,
per-CPU ring, workqueue, cpuhp state, pghot call site etc) but
the profiler isn't armed yet and that is done in the next commit.
Register cpuhp on CPUHP_AP_ONLINE_DYN (after workqueue online) so
offline teardown disarms the producer and flush_work() runs while
the per-CPU worker is still bound, before workqueue unbind.
More details about IBS Memory Profiler can be obtained from
the AMD document titled "AMD64 Zen6 Instruction Based Sampling (IBS)
Extensions and Features".
Signed-off-by: Bharata B Rao <bharata@amd.com>
---
Documentation/admin-guide/mm/pghot.rst | 42 +++
arch/x86/Kconfig | 16 +
arch/x86/entry/entry_fred.c | 1 +
arch/x86/include/asm/hardirq.h | 3 +
arch/x86/include/asm/ibs-caps.h | 8 +
arch/x86/include/asm/ibs-mprof.h | 47 +++
arch/x86/include/asm/idtentry.h | 6 +
arch/x86/include/asm/irq_vectors.h | 4 +-
arch/x86/include/asm/msr-index.h | 8 +
arch/x86/kernel/idt.c | 3 +
arch/x86/kernel/irq.c | 3 +
arch/x86/mm/Makefile | 1 +
arch/x86/mm/ibs-mprof.c | 482 +++++++++++++++++++++++++
include/linux/vm_event_item.h | 8 +
mm/Kconfig | 9 +
mm/vmstat.c | 8 +
16 files changed, 648 insertions(+), 1 deletion(-)
create mode 100644 arch/x86/include/asm/ibs-mprof.h
create mode 100644 arch/x86/mm/ibs-mprof.c
diff --git a/Documentation/admin-guide/mm/pghot.rst b/Documentation/admin-guide/mm/pghot.rst
index c7242dc04ea2..df8316175207 100644
--- a/Documentation/admin-guide/mm/pghot.rst
+++ b/Documentation/admin-guide/mm/pghot.rst
@@ -117,3 +117,45 @@ This source is enabled by default through the hint faults bit (0x1) of
to promote hot pages. It can be disabled at runtime by clearing that bit:
# echo 0x0 > /proc/sys/vm/pghot_enabled_sources
+
+Hardware Hints Source
+=====================
+pghot can consume memory access samples reported by hardware profilers.
+This "hardware hints" (hwhints) source feeds hardware-observed accesses
+into pghot for hot page detection and promotion.
+
+Generic in-kernel support for such profilers is controlled by the
+HWMEM_PROFILER config option. It is not enabled directly by the user;
+an in-kernel driver that forwards hardware-observed accesses to pghot
+selects it.
+
+The AMD IBS Memory Profiler (config AMD_IBS_MEMPROF, available on Zen6
+and later AMD CPUs) is one such driver. It uses the AMD Instruction
+Based Sampling (IBS) Memory Profiler facility to sample user memory
+accesses and record them with pghot.
+
+This source can be activated at runtime through the hardware
+hints bit (0x2) of **pghot_enabled_sources**.
+
+# echo 0x2 > /proc/sys/vm/pghot_enabled_sources
+
+HWHINTS Vmstat Counters
+=======================
+Following vmstat counters provide some stats about hardware hints source.
+
+Path: /proc/vmstat
+
+1. **hwhint_total_events**
+ - Number of total hwhint events recorded by hwhints source.
+
+2. **hwhint_useful_events**
+ - Number of actionable events from hwhints source.
+
+3. **hwhint_dropped_events**
+ - Number of events that were dropped due to buffer overrun.
+
+4. **hwhint_dram_accesses**
+ - Number of DRAM accesses reported by hwhints source.
+
+5. **hwhint_extmem_accesses**
+ - Number of external memory (like CXL) accesses reported by hwhints source.
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 15fd9ec5ecac..a25fbe201840 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -1505,6 +1505,22 @@ config AMD_MEM_ENCRYPT
This requires an AMD processor that supports Secure Memory
Encryption (SME).
+config AMD_IBS_MEMPROF
+ bool "AMD IBS Memory Profiler"
+ depends on X86_64 && CPU_SUP_AMD
+ depends on PGHOT
+ select HWMEM_PROFILER
+ help
+ Use the AMD Instruction Based Sampling (IBS) Memory Profiler
+ facility (present on Zen6 and later AMD CPUs) to feed
+ hardware-observed memory accesses into the pghot subsystem
+ for hot-page detection and promotion.
+
+ When disabled, no IBS Memory Profiler MSRs are programmed and
+ the corresponding interrupt handler is not installed.
+
+ If unsure, say N.
+
# Common NUMA Features
config NUMA
bool "NUMA Memory Allocation and Scheduler Support"
diff --git a/arch/x86/entry/entry_fred.c b/arch/x86/entry/entry_fred.c
index fb3594ddf731..f52099e3fa85 100644
--- a/arch/x86/entry/entry_fred.c
+++ b/arch/x86/entry/entry_fred.c
@@ -120,6 +120,7 @@ static idtentry_t sysvec_table[NR_SYSTEM_VECTORS] __ro_after_init = {
SYSVEC(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi),
SYSVEC(POSTED_MSI_NOTIFICATION_VECTOR, posted_msi_notification),
+ SYSVEC(IBS_MEMPROF_VECTOR, ibs_memprof),
};
static bool fred_setup_done __initdata;
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index dea60d66d976..2da4930ea399 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -49,6 +49,9 @@ enum irq_stat_counts {
#endif
#ifdef CONFIG_X86_POSTED_MSI
IRQ_COUNT_POSTED_MSI_NOTIFICATION,
+#endif
+#ifdef CONFIG_AMD_IBS_MEMPROF
+ IRQ_COUNT_IBS_MEMPROF,
#endif
IRQ_COUNT_PIC_APIC_ERROR,
#ifdef CONFIG_X86_IO_APIC
diff --git a/arch/x86/include/asm/ibs-caps.h b/arch/x86/include/asm/ibs-caps.h
index ddf6c512c8f9..1f6c4058a0e3 100644
--- a/arch/x86/include/asm/ibs-caps.h
+++ b/arch/x86/include/asm/ibs-caps.h
@@ -29,6 +29,7 @@
#define IBS_CAPS_FETCHLAT (1U<<14)
#define IBS_CAPS_BIT63_FILTER (1U<<15)
#define IBS_CAPS_STRMST_RMTSOCKET (1U<<16)
+#define IBS_CAPS_MEM_PROFILER (1U<<18)
#define IBS_CAPS_OPDTLBPGSIZE (1U<<19)
#define IBS_CAPS_DEFAULT (IBS_CAPS_AVAIL \
@@ -42,6 +43,13 @@
#define IBSCTL_LVT_OFFSET_VALID (1ULL<<8)
#define IBSCTL_LVT_OFFSET_MASK 0x0F
+/*
+ * IBS Memprofiler setup
+ */
+#define IBSCTL_MPROF_LVT_OFFSET_VALID (1ULL << 24)
+#define IBSCTL_MPROF_LVT_OFFSET_SHIFT 16
+#define IBSCTL_MPROF_LVT_OFFSET_MASK (0xFULL << IBSCTL_MPROF_LVT_OFFSET_SHIFT)
+
/* IBS fetch bits/masks */
#define IBS_FETCH_L3MISSONLY (1ULL << 59)
#define IBS_FETCH_RAND_EN (1ULL << 57)
diff --git a/arch/x86/include/asm/ibs-mprof.h b/arch/x86/include/asm/ibs-mprof.h
new file mode 100644
index 000000000000..3de1880c4126
--- /dev/null
+++ b/arch/x86/include/asm/ibs-mprof.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_X86_IBS_MPROF_H
+#define _ASM_X86_IBS_MPROF_H
+
+/*
+ * All bits are documented here for clarity even if the current
+ * driver doesn't use all of them.
+ */
+
+/* MSR_AMD64_IBS_MPROF_DATA2 bits */
+#define IBS_MPROF_DATA2_DATASRC_MASK 0x7
+#define IBS_MPROF_DATA2_DATASRC_MASK_HIGH 0xC0
+#define IBS_MPROF_DATA2_DATASRC_MASK_HIGH_SHIFT 0x3
+#define IBS_MPROF_DATA2_DATASRC_LCL_CCX 0x1
+#define IBS_MPROF_DATA2_DATASRC_PEER_CCX_NEAR 0x2
+#define IBS_MPROF_DATA2_DATASRC_DRAM 0x3
+#define IBS_MPROF_DATA2_DATASRC_CCX_FAR 0x5
+#define IBS_MPROF_DATA2_DATASRC_DRAM_FAR 0x7
+#define IBS_MPROF_DATA2_DATASRC_EXT_MEM 0x8
+#define IBS_MPROF_DATA2_RMT_NODE BIT_ULL(4)
+#define IBS_MPROF_DATA2_RMT_SOCKET BIT_ULL(9)
+
+/* MSR_AMD64_IBS_MPROF_DATA3 bits */
+#define IBS_MPROF_DATA3_LDOP BIT_ULL(0)
+#define IBS_MPROF_DATA3_STOP BIT_ULL(1)
+#define IBS_MPROF_DATA3_DCMISS BIT_ULL(7)
+#define IBS_MPROF_DATA3_LADDR_VALID BIT_ULL(17)
+#define IBS_MPROF_DATA3_PADDR_VALID BIT_ULL(18)
+#define IBS_MPROF_DATA3_L2MISS BIT_ULL(20)
+#define IBS_MPROF_DATA3_SW_PREFETCH BIT_ULL(21)
+
+/* MSR_AMD64_IBS_MPROF_CTL bits */
+#define IBS_MPROF_CTL_CNT_CTL BIT_ULL(19)
+#define IBS_MPROF_CTL_VAL BIT_ULL(18)
+#define IBS_MPROF_CTL_ENABLE BIT_ULL(17)
+#define IBS_MPROF_CTL_L3MISSONLY BIT_ULL(16)
+#define IBS_MPROF_CTL_MAXCNT_MASK 0x0000FFFFULL
+#define IBS_MPROF_CTL_MAXCNT_EXT_MASK (0x7FULL << 20) /* separate upper 7 bits */
+
+/* MSR_AMD64_IBS_MPROF_CTL2 bits */
+#define IBS_MPROF_CTL2_DISABLE BIT_ULL(0)
+#define IBS_MPROF_CTL2_EXCLUDE_USER BIT_ULL(1)
+#define IBS_MPROF_CTL2_EXCLUDE_KERNEL BIT_ULL(2)
+
+#define IBS_MPROF_SAMPLE_PERIOD 10000
+
+#endif /* _ASM_X86_IBS_MPROF_H */
diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h
index f400cfac69a6..c98833f64a4f 100644
--- a/arch/x86/include/asm/idtentry.h
+++ b/arch/x86/include/asm/idtentry.h
@@ -735,6 +735,12 @@ DECLARE_IDTENTRY_SYSVEC(POSTED_MSI_NOTIFICATION_VECTOR, sysvec_posted_msi_notifi
# define fred_sysvec_posted_msi_notification NULL
# endif
+# ifdef CONFIG_AMD_IBS_MEMPROF
+DECLARE_IDTENTRY_SYSVEC(IBS_MEMPROF_VECTOR, sysvec_ibs_memprof);
+#else
+# define fred_sysvec_ibs_memprof NULL
+#endif
+
#if IS_ENABLED(CONFIG_HYPERV)
DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_hyperv_callback);
DECLARE_IDTENTRY_SYSVEC(HYPERV_REENLIGHTENMENT_VECTOR, sysvec_hyperv_reenlightenment);
diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h
index 85253fc8e384..575f880cb147 100644
--- a/arch/x86/include/asm/irq_vectors.h
+++ b/arch/x86/include/asm/irq_vectors.h
@@ -105,10 +105,12 @@
*/
#define POSTED_MSI_NOTIFICATION_VECTOR 0xeb
+#define IBS_MEMPROF_VECTOR 0xea
+
#define NR_VECTORS 256
#ifdef CONFIG_X86_LOCAL_APIC
-#define FIRST_SYSTEM_VECTOR POSTED_MSI_NOTIFICATION_VECTOR
+#define FIRST_SYSTEM_VECTOR IBS_MEMPROF_VECTOR
#else
#define FIRST_SYSTEM_VECTOR NR_VECTORS
#endif
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index 3a8e51a0c9e8..6e13f53143b0 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -1319,4 +1319,12 @@
* a #GP
*/
+/* AMD IBS Memory Profiler MSRs */
+#define MSR_AMD64_IBS_MPROF_CTL 0xc0010380
+#define MSR_AMD64_IBS_MPROF_CTL2 0xc0010381
+#define MSR_AMD64_IBS_MPROF_DATA2 0xc0010382
+#define MSR_AMD64_IBS_MPROF_DATA3 0xc0010383
+#define MSR_AMD64_IBS_MPROF_LINADDR 0xc0010384
+#define MSR_AMD64_IBS_MPROF_PHYADDR 0xc0010385
+
#endif /* _ASM_X86_MSR_INDEX_H */
diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c
index 90a22e24a9eb..50ec5fc82619 100644
--- a/arch/x86/kernel/idt.c
+++ b/arch/x86/kernel/idt.c
@@ -169,6 +169,9 @@ static const __initconst struct idt_data apic_idts[] = {
# ifdef CONFIG_X86_POSTED_MSI
INTG(POSTED_MSI_NOTIFICATION_VECTOR, asm_sysvec_posted_msi_notification),
# endif
+#ifdef CONFIG_AMD_IBS_MEMPROF
+ INTG(IBS_MEMPROF_VECTOR, asm_sysvec_ibs_memprof),
+#endif
#endif
};
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index 30122f0b3af9..c59a474d9202 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -121,6 +121,9 @@ static const struct irq_stat_info irq_stat_info[IRQ_COUNT_MAX] = {
#endif
#ifdef CONFIG_X86_POSTED_MSI
ISS(POSTED_MSI_NOTIFICATION, "PMN", " Posted MSI notification event\n"),
+#endif
+#ifdef CONFIG_AMD_IBS_MEMPROF
+ ISS(IBS_MEMPROF, "IMP", " IBS Memory Profiler interrupts\n"),
#endif
IDS(PIC_APIC_ERROR, "ERR", " PIC/APIC error interrupts\n"),
#ifdef CONFIG_X86_IO_APIC
diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile
index 3a5364853eab..050a7379d9f7 100644
--- a/arch/x86/mm/Makefile
+++ b/arch/x86/mm/Makefile
@@ -59,3 +59,4 @@ obj-$(CONFIG_X86_MEM_ENCRYPT) += mem_encrypt.o
obj-$(CONFIG_AMD_MEM_ENCRYPT) += mem_encrypt_amd.o
obj-$(CONFIG_AMD_MEM_ENCRYPT) += mem_encrypt_boot.o
+obj-$(CONFIG_AMD_IBS_MEMPROF) += ibs-mprof.o
diff --git a/arch/x86/mm/ibs-mprof.c b/arch/x86/mm/ibs-mprof.c
new file mode 100644
index 000000000000..a455fdc60d8e
--- /dev/null
+++ b/arch/x86/mm/ibs-mprof.c
@@ -0,0 +1,482 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#define pr_fmt(fmt) "amd_ibs_memprof: " fmt
+
+#include <linux/init.h>
+#include <linux/pghot.h>
+#include <linux/percpu.h>
+#include <linux/workqueue.h>
+#include <linux/mm.h>
+#include <linux/vm_event_item.h>
+#include <linux/vmstat.h>
+#include <linux/cpuhotplug.h>
+#include <linux/syscore_ops.h>
+
+#include <asm/ibs-mprof.h>
+#include <asm/ibs-caps.h>
+#include <asm/irq_vectors.h>
+#include <asm/idtentry.h>
+#include <asm/apic.h>
+#include <asm/cpuid/api.h>
+
+#define IBS_NR_SAMPLES 150 /* Percpu sample buffer size */
+
+static DEFINE_PER_CPU(bool, mprof_work_pending);
+
+/*
+ * Basic access info captured for each memory access.
+ */
+struct mprof_sample {
+ unsigned long pfn;
+ unsigned long time; /* jiffies when accessed */
+ int nid; /* Accessing node ID, if known */
+};
+
+/*
+ * Percpu buffer of access samples. Samples are accumulated here
+ * before pushing them to pghot sub-system for further action.
+ */
+struct mprof_sample_pcpu {
+ struct mprof_sample samples[IBS_NR_SAMPLES];
+ int head, tail;
+};
+
+static struct mprof_sample_pcpu __percpu *mprof_s;
+
+/*
+ * Per-CPU work for pushing the percpu access samples to pghot sub-system.
+ *
+ * @cpu records which CPU's sample ring this work item is responsible for
+ * draining.
+ */
+struct mprof_worker {
+ struct work_struct work;
+ unsigned int cpu;
+};
+static DEFINE_PER_CPU(struct mprof_worker, mprof_work);
+static int mprof_cpuhp_state = CPUHP_INVALID;
+
+/*
+ * Record the IBS-reported access sample in percpu buffer.
+ * Called from IBS interrupt handler.
+ */
+static bool mprof_push_sample(unsigned long pfn, int nid, unsigned long time)
+{
+ struct mprof_sample_pcpu *pcpu = raw_cpu_ptr(mprof_s);
+ int head = READ_ONCE(pcpu->head);
+ int tail = READ_ONCE(pcpu->tail);
+ int next = head + 1;
+
+ if (next >= IBS_NR_SAMPLES)
+ next = 0;
+
+ if (next == tail) {
+ count_vm_event(HWHINT_DROPPED_EVENTS);
+ return false;
+ }
+
+ pcpu->samples[head].pfn = pfn;
+ pcpu->samples[head].time = time;
+ pcpu->samples[head].nid = nid;
+
+ /*
+ * Publish the sample slot stores before advancing head; pairs
+ * with the smp_load_acquire() of head in mprof_pop_sample().
+ */
+ smp_store_release(&pcpu->head, next);
+ return true;
+}
+
+static bool mprof_pop_sample(struct mprof_sample_pcpu *pcpu, struct mprof_sample *s)
+{
+ int tail = READ_ONCE(pcpu->tail);
+ /*
+ * Pairs with the smp_store_release() of head in mprof_push_sample();
+ * ensures the sample slot stores are visible before we read the slot.
+ */
+ int head = smp_load_acquire(&pcpu->head);
+ int next = tail + 1;
+
+ if (head == tail)
+ return false;
+
+ if (next >= IBS_NR_SAMPLES)
+ next = 0;
+
+ *s = pcpu->samples[tail];
+
+ /*
+ * Publish the freed slot via tail with release semantics so the plain
+ * sample read above cannot be reordered past it. The ring's one-empty-
+ * slot invariant already keeps the producer from ever writing the slot
+ * being read, so this is not strictly required on x86; it makes the
+ * ordering explicit and symmetric with the head release/acquire, at no
+ * runtime cost (compiler barrier only).
+ */
+ smp_store_release(&pcpu->tail, next);
+ return true;
+}
+
+/*
+ * Drain any samples remaining in @cpu's ring into pghot. Called from the
+ * cpuhp offline callback after flush_work(), with the producer silenced.
+ */
+static void mprof_drain_ring(unsigned int cpu)
+{
+ struct mprof_sample_pcpu *pcpu = per_cpu_ptr(mprof_s, cpu);
+ struct mprof_sample s;
+
+ while (mprof_pop_sample(pcpu, &s))
+ pghot_record_access(s.pfn, s.nid, PGHOT_HWHINTS, s.time);
+}
+
+/*
+ * Remove access samples from percpu buffer and send them
+ * to pghot sub-system for further action.
+ */
+static void mprof_work_handler(struct work_struct *work)
+{
+ struct mprof_worker *mw = container_of(work, struct mprof_worker, work);
+ unsigned int cpu = mw->cpu;
+ struct mprof_sample_pcpu *pcpu = per_cpu_ptr(mprof_s, cpu);
+ struct mprof_sample s;
+
+ for (;;) {
+ while (mprof_pop_sample(pcpu, &s))
+ pghot_record_access(s.pfn, s.nid, PGHOT_HWHINTS,
+ s.time);
+
+ /*
+ * Publish the cleared pending flag before re-checking the
+ * ring so a racing IRQ cannot leave a just-pushed sample
+ * unprocessed.
+ */
+ WRITE_ONCE(per_cpu(mprof_work_pending, cpu), false);
+
+ if (READ_ONCE(pcpu->head) == READ_ONCE(pcpu->tail))
+ return;
+
+ /*
+ * A sample was pushed after we exited the drain loop; the
+ * producer may have skipped re-queuing because pending was
+ * still set. Keep draining until the ring is observably empty
+ * with pending cleared.
+ */
+ }
+}
+
+/*
+ * L3MissOnly + Exclude kernel RIP
+ */
+static void mprof_enable_profiling(void)
+{
+ u64 mprof_config = IBS_MPROF_CTL_CNT_CTL | IBS_MPROF_CTL_L3MISSONLY;
+ unsigned int period = IBS_MPROF_SAMPLE_PERIOD;
+ u64 ctl, ctl2;
+
+ /*
+ * Assemble bits 26:20 and 19:4 of periodic op counter in ctl.
+ * The lower 4 bits are always 0000b.
+ */
+ ctl = (period >> 4) & IBS_MPROF_CTL_MAXCNT_MASK;
+ ctl |= (period & IBS_MPROF_CTL_MAXCNT_EXT_MASK);
+ ctl |= mprof_config;
+ wrmsrq(MSR_AMD64_IBS_MPROF_CTL, ctl);
+
+ /*
+ * Exclude samples that have bit 63 of their RIP set.
+ */
+ ctl2 = IBS_MPROF_CTL2_EXCLUDE_KERNEL;
+ wrmsrq(MSR_AMD64_IBS_MPROF_CTL2, ctl2);
+}
+
+static void mprof_disable_profiling(u64 mem_ctl)
+{
+ mem_ctl &= ~IBS_MPROF_CTL_ENABLE;
+ mem_ctl &= ~IBS_MPROF_CTL_VAL;
+ wrmsrq(MSR_AMD64_IBS_MPROF_CTL, mem_ctl);
+
+ wrmsrq(MSR_AMD64_IBS_MPROF_CTL2, IBS_MPROF_CTL2_DISABLE);
+}
+
+/*
+ * IBS interrupt handler: Process the memory access info reported by IBS.
+ *
+ * Reads the MSRs to collect all the information about the reported
+ * memory access, validates the access, stores the valid sample and
+ * schedules the work on this CPU to further process the sample.
+ */
+static void mprof_overflow_handler(void)
+{
+ u64 mem_ctl, mem_data3, mem_data2, paddr, data_src;
+ struct work_struct *w = &this_cpu_ptr(&mprof_work)->work;
+ unsigned long pfn;
+ struct page *page;
+
+ rdmsrq(MSR_AMD64_IBS_MPROF_CTL, mem_ctl);
+ if (!(mem_ctl & IBS_MPROF_CTL_VAL))
+ return;
+
+ mprof_disable_profiling(mem_ctl);
+ count_vm_event(HWHINT_TOTAL_EVENTS);
+
+ rdmsrq(MSR_AMD64_IBS_MPROF_DATA3, mem_data3);
+
+ /* Data source is valid only if the sample isn't an L1 or L2 hit */
+ if (!(mem_data3 & (IBS_MPROF_DATA3_DCMISS | IBS_MPROF_DATA3_L2MISS))) {
+ count_vm_event(HWHINT_CACHE_ACCESSES);
+ } else {
+ rdmsrq(MSR_AMD64_IBS_MPROF_DATA2, mem_data2);
+ data_src = mem_data2 & IBS_MPROF_DATA2_DATASRC_MASK;
+ data_src |= ((mem_data2 & IBS_MPROF_DATA2_DATASRC_MASK_HIGH) >>
+ IBS_MPROF_DATA2_DATASRC_MASK_HIGH_SHIFT);
+
+ switch (data_src) {
+ case IBS_MPROF_DATA2_DATASRC_DRAM:
+ case IBS_MPROF_DATA2_DATASRC_DRAM_FAR:
+ /* local and remote DRAM are both DRAM accesses */
+ count_vm_event(HWHINT_DRAM_ACCESSES);
+ break;
+ case IBS_MPROF_DATA2_DATASRC_EXT_MEM:
+ count_vm_event(HWHINT_EXTMEM_ACCESSES);
+ break;
+ }
+ }
+
+ /* Is linear addr valid? */
+ if (!(mem_data3 & IBS_MPROF_DATA3_LADDR_VALID))
+ goto handled;
+
+ /* Is phys addr valid? */
+ if (!(mem_data3 & IBS_MPROF_DATA3_PADDR_VALID))
+ goto handled;
+ rdmsrq(MSR_AMD64_IBS_MPROF_PHYADDR, paddr);
+
+ pfn = PHYS_PFN(paddr);
+ page = pfn_to_online_page(pfn);
+ if (!page)
+ goto handled;
+
+ /*
+ * Use the accessing CPU's node as the migration target. On
+ * topologies where all CPUs reside on toptier nodes (the common
+ * case), this is the desired behaviour.
+ */
+ if (!mprof_push_sample(pfn, numa_node_id(), jiffies))
+ goto handled;
+
+ if (!this_cpu_read(mprof_work_pending)) {
+ this_cpu_write(mprof_work_pending, true);
+ schedule_work_on(smp_processor_id(), w);
+ }
+ count_vm_event(HWHINT_USEFUL_EVENTS);
+
+handled:
+ mprof_enable_profiling();
+}
+
+DEFINE_IDTENTRY_SYSVEC(sysvec_ibs_memprof)
+{
+ inc_irq_stat(IBS_MEMPROF);
+ mprof_overflow_handler();
+ apic_eoi();
+}
+
+static int get_mprof_lvt_offset(void)
+{
+ u64 val;
+
+ rdmsrq(MSR_AMD64_IBSCTL, val);
+ if (!(val & IBSCTL_MPROF_LVT_OFFSET_VALID))
+ return -EINVAL;
+
+ return (val & IBSCTL_MPROF_LVT_OFFSET_MASK) >>
+ IBSCTL_MPROF_LVT_OFFSET_SHIFT;
+}
+
+/*
+ * Confirm the IBS memory-profiler extended-LVT offset is valid and routable,
+ * mirroring the perf IBS driver's ibs_eilvt_valid(). The trial uses mask=1 so
+ * it programs nothing that can fire; the per-CPU startup callback unmasks it
+ * later. setup_APIC_eilvt() touches the local APIC, so this must run with
+ * preemption disabled.
+ */
+static bool mprof_eilvt_valid(void)
+{
+ int offset;
+ bool valid = false;
+
+ preempt_disable();
+
+ offset = get_mprof_lvt_offset();
+ if (offset < 0)
+ goto out;
+
+ if (!setup_APIC_eilvt(offset, IBS_MEMPROF_VECTOR, APIC_DELIVERY_MODE_FIXED, 1))
+ valid = true;
+out:
+ preempt_enable();
+ return valid;
+}
+
+/*
+ * Arm the memory profiler on the current CPU: route the extended-LVT entry
+ * to IBS_MEMPROF_VECTOR and program IBS from the published config snapshot.
+ * Runs on the target CPU (cpuhp ONLINE callback and syscore resume).
+ */
+static int mprof_arm_cpu(void)
+{
+ int offset = get_mprof_lvt_offset();
+
+ if (offset < 0) {
+ pr_warn("offset not valid on cpu #%d\n", smp_processor_id());
+ return -ENODEV;
+ }
+
+ if (setup_APIC_eilvt(offset, IBS_MEMPROF_VECTOR, APIC_DELIVERY_MODE_FIXED, 0)) {
+ pr_warn("APIC setup failed on cpu #%d\n", smp_processor_id());
+ return -EBUSY;
+ }
+
+ mprof_enable_profiling();
+ return 0;
+}
+
+/*
+ * Disarm the memory profiler on the current CPU: mask the extended-LVT entry
+ * (silences the producer) and clear the IBS enable/valid bits.
+ */
+static void mprof_disarm_cpu(void)
+{
+ int offset = get_mprof_lvt_offset();
+ u64 mem_ctl;
+
+ if (offset >= 0)
+ setup_APIC_eilvt(offset, IBS_MEMPROF_VECTOR, APIC_DELIVERY_MODE_FIXED, 1);
+
+ rdmsrq(MSR_AMD64_IBS_MPROF_CTL, mem_ctl);
+ mprof_disable_profiling(mem_ctl);
+}
+
+static int x86_amd_ibs_mprof_online(unsigned int cpu)
+{
+ /*
+ * If profiler can't be armed, that must not stop
+ * CPU online. Hence ignoring the return value of
+ * mprof_arm_cpu().
+ */
+ mprof_arm_cpu();
+ return 0;
+}
+
+static int x86_amd_ibs_mprof_offline(unsigned int cpu)
+{
+ struct mprof_worker *mw = per_cpu_ptr(&mprof_work, cpu);
+
+ WARN_ON(cpu != smp_processor_id());
+
+ /*
+ * The teardown callback runs on this CPU before
+ * CPUHP_AP_WORKQUEUE_ONLINE offline, while this CPU's worker is still
+ * per-CPU bound. Silence the producer, flush the pending worker, then
+ * drain any remaining ring samples synchronously.
+ */
+ local_irq_disable();
+ mprof_disarm_cpu();
+ local_irq_enable();
+ flush_work(&mw->work);
+ mprof_drain_ring(cpu);
+ WRITE_ONCE(per_cpu(mprof_work_pending, cpu), false);
+
+ return 0;
+}
+
+#ifdef CONFIG_PM
+/*
+ * The local APIC does not preserve the extended-LVT registers across S3/S4
+ * (lapic_suspend()/lapic_resume() save only the architectural LVTs), and the
+ * boot CPU never re-runs the cpuhp ONLINE callback on resume. Re-arm it via
+ * syscore so the profiler survives suspend/resume; secondary CPUs are handled
+ * by their offline/online cpuhp callbacks.
+ */
+static int mprof_suspend(void *data)
+{
+ mprof_disarm_cpu();
+ return 0;
+}
+
+static void mprof_resume(void *data)
+{
+ mprof_arm_cpu();
+}
+
+static const struct syscore_ops mprof_syscore_ops = {
+ .resume = mprof_resume,
+ .suspend = mprof_suspend,
+};
+
+static struct syscore mprof_syscore = {
+ .ops = &mprof_syscore_ops,
+};
+
+static void mprof_pm_init(void)
+{
+ register_syscore(&mprof_syscore);
+}
+#else
+static void mprof_pm_init(void) {}
+#endif
+
+static int __init mprof_access_profiling_init(void)
+{
+ u32 mprof_caps;
+ int cpu, ret;
+
+ if (!boot_cpu_has(X86_FEATURE_IBS))
+ return -ENODEV;
+
+ mprof_caps = cpuid_eax(IBS_CPUID_FEATURES);
+ if (!(mprof_caps & IBS_CAPS_MEM_PROFILER)) {
+ pr_info("capability is unavailable for access profiling\n");
+ return -ENODEV;
+ }
+
+ /*
+ * Don't initialize the profiler if EILVT offset can't be armed.
+ */
+ if (!mprof_eilvt_valid()) {
+ pr_info("extended-LVT offset unavailable; access profiling disabled\n");
+ return -EINVAL;
+ }
+
+ mprof_s = alloc_percpu_gfp(struct mprof_sample_pcpu, GFP_KERNEL | __GFP_ZERO);
+ if (!mprof_s)
+ return -ENOMEM;
+
+ for_each_possible_cpu(cpu) {
+ struct mprof_worker *mw = per_cpu_ptr(&mprof_work, cpu);
+
+ INIT_WORK(&mw->work, mprof_work_handler);
+ mw->cpu = cpu;
+ }
+
+ ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
+ "x86/amd/ibs_mprof:online",
+ x86_amd_ibs_mprof_online,
+ x86_amd_ibs_mprof_offline);
+ if (ret < 0)
+ goto out_free;
+ mprof_cpuhp_state = ret;
+
+ mprof_pm_init();
+
+ pr_info("IBS Memory Profiler is available for memory access profiling\n");
+ return 0;
+
+out_free:
+ free_percpu(mprof_s);
+ pr_info("IBS Memory Profiler failed to initialize\n");
+ return ret;
+}
+
+device_initcall(mprof_access_profiling_init);
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index 90827e8b597f..ebd001fcf741 100644
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -183,6 +183,14 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
PGHOT_RECORDED_ACCESSES,
PGHOT_REPORTED_HINTFAULTS,
PGHOT_REPORTED_HWHINTS,
+#ifdef CONFIG_HWMEM_PROFILER
+ HWHINT_TOTAL_EVENTS,
+ HWHINT_DRAM_ACCESSES,
+ HWHINT_EXTMEM_ACCESSES,
+ HWHINT_CACHE_ACCESSES,
+ HWHINT_USEFUL_EVENTS,
+ HWHINT_DROPPED_EVENTS,
+#endif /* CONFIG_HWMEM_PROFILER */
#endif /* CONFIG_PGHOT */
NR_VM_EVENT_ITEMS
};
diff --git a/mm/Kconfig b/mm/Kconfig
index 9237d54a6d8b..6ec71eb6f498 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -1536,6 +1536,15 @@ config PGHOT_PRECISE
4 bytes per page against the default one byte per page. Preferable
to enable this on systems with multiple nodes in toptier.
+config HWMEM_PROFILER
+ bool
+ depends on PGHOT
+ help
+ Umbrella symbol enabled by any in-kernel driver that forwards
+ hardware-observed memory accesses to the pghot subsystem (for
+ example AMD_IBS_MEMPROF on x86_64). Drivers select this; users
+ do not enable it directly.
+
source "mm/damon/Kconfig"
endmenu
diff --git a/mm/vmstat.c b/mm/vmstat.c
index cbcbbbbbf297..570615d5f441 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1511,6 +1511,14 @@ const char * const vmstat_text[] = {
[I(PGHOT_RECORDED_ACCESSES)] = "pghot_recorded_accesses",
[I(PGHOT_REPORTED_HINTFAULTS)] = "pghot_reported_hintfaults",
[I(PGHOT_REPORTED_HWHINTS)] = "pghot_reported_hwhints",
+#ifdef CONFIG_HWMEM_PROFILER
+ [I(HWHINT_TOTAL_EVENTS)] = "hwhint_total_events",
+ [I(HWHINT_DRAM_ACCESSES)] = "hwhint_dram_accesses",
+ [I(HWHINT_EXTMEM_ACCESSES)] = "hwhint_extmem_accesses",
+ [I(HWHINT_CACHE_ACCESSES)] = "hwhint_cache_accesses",
+ [I(HWHINT_USEFUL_EVENTS)] = "hwhint_useful_events",
+ [I(HWHINT_DROPPED_EVENTS)] = "hwhint_dropped_events",
+#endif /* CONFIG_HWMEM_PROFILER */
#endif /* CONFIG_PGHOT */
#undef I
#endif /* CONFIG_VM_EVENT_COUNTERS */
--
2.34.1
^ permalink raw reply [flat|nested] 10+ messages in thread* [RFC PATCH v0 3/3] x86/mm/ibs: Add runtime controls for IBS memprofiler
2026-09-24 6:22 [RFC PATCH v0 0/3] pghot: x86: IBS Memory Profiler for hot page promotion Bharata B Rao
2026-09-24 6:22 ` [RFC PATCH v0 1/3] x86/ibs: Move IBS caps definitions into its own header Bharata B Rao
2026-09-24 6:22 ` [RFC PATCH v0 2/3] x86/mm/ibs: In-kernel driver for AMD IBS Memory Profiler Bharata B Rao
@ 2026-09-24 6:22 ` Bharata B Rao
2026-09-24 6:26 ` [RFC PATCH v0 0/3] pghot: x86: IBS Memory Profiler for hot page promotion Bharata B Rao
` (5 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Bharata B Rao @ 2026-09-24 6:22 UTC (permalink / raw)
To: linux-kernel, linux-mm
Cc: tglx, mingo, bp, dave.hansen, x86, hpa, xin, luto, peterz,
Jonathan.Cameron, gourry, rientjes, sj, weixugc, willy,
ying.huang, ziy, dave, nifan.cxl, xuezhengchu, yiannis, akpm,
david, byungchul, kinseyho, joshua.hahnjy, yuanchu, balbirs,
shivankg, donettom, linyongting, Bharata B Rao
Expose runtime tunables for IBS Memory profiler and arm the profiler.
<debugfs>/ibs-mprof/l3miss-only - IbsMemL3MissOnly filter
<debugfs>/ibs-mprof/period - op sample period (IbsMemMaxCnt)
<debugfs>/ibs-mprof/lat-filter - IbsMemLatFltEn load-latency filter
<debugfs>/ibs-mprof/lat-thresh - IbsMemLatThrsh (0x0..0xf)
In addition, a kernel cmdline parameter (ibs-memprof) is introduced
to enable profiling at boot time.
/sys/devices/system/cpu/ibs-mprof/enabled can be used to turn the
profiling on or off.
Enabling the profiler will result in IBS Memory Profiler samples to
be reported to pghot subsystem. pghot will act on these samples only
if hwhints source is enabled explicilty using pghot/enabled_sources
debugfs tunable.
Signed-off-by: Bharata B Rao <bharata@amd.com>
---
.../admin-guide/kernel-parameters.txt | 5 +
Documentation/admin-guide/mm/pghot.rst | 47 ++
arch/x86/include/asm/ibs-mprof.h | 14 +
arch/x86/mm/ibs-mprof.c | 415 +++++++++++++++++-
4 files changed, 466 insertions(+), 15 deletions(-)
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 68647ff4bdd2..3ef091d86c33 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -2235,6 +2235,11 @@ Kernel parameters
syscalls, essentially overriding IA32_EMULATION_DEFAULT_DISABLED at
boot time. When false, unconditionally disables IA32 emulation.
+ ibs-memprof= [X86-64]
+ Format: <bool>
+ When true, IBS Memory Profiler will be enabled at boottime
+ for memory access profiling. It can be changed at runtime too
+ using /sys/devices/system/cpu/ibs-mprof/enabled.
idle= [X86,EARLY]
Format: idle=poll, idle=halt, idle=nomwait
diff --git a/Documentation/admin-guide/mm/pghot.rst b/Documentation/admin-guide/mm/pghot.rst
index df8316175207..97142c2558cd 100644
--- a/Documentation/admin-guide/mm/pghot.rst
+++ b/Documentation/admin-guide/mm/pghot.rst
@@ -159,3 +159,50 @@ Path: /proc/vmstat
5. **hwhint_extmem_accesses**
- Number of external memory (like CXL) accesses reported by hwhints source.
+
+AMD IBS Memory Profiler Tunables
+================================
+sysfs tunables
+--------------
+1. enabled
+
+Path: /sys/devices/system/cpu/ibs-mprof/enabled
+
+- Enable (1) or disable (0) memory access profiling. Enabling here
+ would only arm the memory profiler and results in generation of IBS
+ samples. pghot will act on the reported samples only if hwhints source
+ is enabled in **pghot_enabled_sources**.
+- Default: 0 (Disabled)
+
+Debugfs tunables
+----------------
+Path: /sys/kernel/debug/ibs-mprof/
+
+These tune the AMD IBS Memory Profiler hardware hints source and are
+present only when AMD_IBS_MEMPROF is enabled. A write takes effect on
+all CPUs immediately.
+
+1. **l3miss-only**
+ - When 1, only accesses that miss the L3 cache are sampled. It is
+ recommended to run the memory profiler with L3 miss filtering
+ enabled.
+ - Default: 1
+
+2. **period**
+ - Sample period as the number of ops between samples (IbsMemMaxCnt).
+ - Range: 5008 to 134217712. The written value is rounded down to a
+ multiple of 16 (the low 4 bits of the hardware IbsMemMaxCnt field are
+ always zero), so a read-back may be up to 15 less than what was written.
+ - Default: 10000
+
+3. **lat-filter**
+ - When 1, enable load latency filtering: only loads whose latency
+ exceeds the **lat-thresh** threshold are reported.
+ - Default: 0
+
+4. **lat-thresh**
+ - Load latency threshold, effective only when **lat-filter** is 1. A
+ sample is reported when the load latency exceeds
+ (lat-thresh + 1) * 128 core cycles.
+ - Range: 0x0 to 0xf
+ - Default: 0
diff --git a/arch/x86/include/asm/ibs-mprof.h b/arch/x86/include/asm/ibs-mprof.h
index 3de1880c4126..5262a77645ef 100644
--- a/arch/x86/include/asm/ibs-mprof.h
+++ b/arch/x86/include/asm/ibs-mprof.h
@@ -30,6 +30,9 @@
#define IBS_MPROF_DATA3_SW_PREFETCH BIT_ULL(21)
/* MSR_AMD64_IBS_MPROF_CTL bits */
+#define IBS_MPROF_CTL_LATFLTEN BIT_ULL(63) /* IbsMemLatFltEn */
+#define IBS_MPROF_CTL_LATTHRSH_SHIFT 59
+#define IBS_MPROF_CTL_LATTHRSH_MASK (0xFULL << IBS_MPROF_CTL_LATTHRSH_SHIFT)
#define IBS_MPROF_CTL_CNT_CTL BIT_ULL(19)
#define IBS_MPROF_CTL_VAL BIT_ULL(18)
#define IBS_MPROF_CTL_ENABLE BIT_ULL(17)
@@ -37,6 +40,17 @@
#define IBS_MPROF_CTL_MAXCNT_MASK 0x0000FFFFULL
#define IBS_MPROF_CTL_MAXCNT_EXT_MASK (0x7FULL << 20) /* separate upper 7 bits */
+/*
+ * IbsMemMaxCnt is a 27-bit op count; the low 4 bits are always zero.
+ * The hardware minimum is 16, but such a short interval is impractical
+ * (interrupt storm), so restrict the sample period to a sensible floor.
+ */
+#define IBS_MPROF_MAXCNT_MIN 5008
+#define IBS_MPROF_MAXCNT_MAX ((1U << 27) - 1)
+
+/* IbsMemLatThrsh is a 4-bit field. */
+#define IBS_MPROF_LATTHRSH_MAX 0xF
+
/* MSR_AMD64_IBS_MPROF_CTL2 bits */
#define IBS_MPROF_CTL2_DISABLE BIT_ULL(0)
#define IBS_MPROF_CTL2_EXCLUDE_USER BIT_ULL(1)
diff --git a/arch/x86/mm/ibs-mprof.c b/arch/x86/mm/ibs-mprof.c
index a455fdc60d8e..6f63f18cf474 100644
--- a/arch/x86/mm/ibs-mprof.c
+++ b/arch/x86/mm/ibs-mprof.c
@@ -10,7 +10,14 @@
#include <linux/vm_event_item.h>
#include <linux/vmstat.h>
#include <linux/cpuhotplug.h>
+#include <linux/cpu.h>
#include <linux/syscore_ops.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/mutex.h>
+#include <linux/smp.h>
+#include <linux/uaccess.h>
+#include <linux/string_choices.h>
#include <asm/ibs-mprof.h>
#include <asm/ibs-caps.h>
@@ -56,6 +63,41 @@ struct mprof_worker {
static DEFINE_PER_CPU(struct mprof_worker, mprof_work);
static int mprof_cpuhp_state = CPUHP_INVALID;
+/*
+ * Runtime-configurable profiler parameters, exposed via debugfs. The
+ * interrupt handler consumes the published snapshot on every re-arm, so
+ * configuration changes need no locking on the fast path. Writers are
+ * serialized by mprof_cfg_lock and publish an immutable snapshot; the hot
+ * path only does an smp_load_acquire() of the pointer.
+ */
+struct mprof_config {
+ bool enabled; /* profiling enabled (IbsMemDis inverted) */
+ bool l3miss_only; /* IbsMemL3MissOnly */
+ bool lat_filter; /* IbsMemLatFltEn */
+ u8 lat_thresh; /* IbsMemLatThrsh, 0x0 .. 0xf */
+ u32 period; /* op sample period (IbsMemMaxCnt) */
+ /* Precomputed register values for the interrupt fast path. */
+ u64 ctl;
+ u64 ctl2;
+};
+
+static struct mprof_config mprof_cfg_slots[2];
+static struct mprof_config *mprof_cfg;
+static DEFINE_MUTEX(mprof_cfg_lock);
+
+/*
+ * ibs-memprof: kernel cmdline parameter to arm IBS Memory
+ * Profiler right from boot time.
+ */
+static bool ibs_mprof_enabled __read_mostly;
+
+static int __init setup_ibs_mprof(char *str)
+{
+ return (kstrtobool(str, &ibs_mprof_enabled) == 0);
+}
+
+__setup("ibs-memprof=", setup_ibs_mprof);
+
/*
* Record the IBS-reported access sample in percpu buffer.
* Called from IBS interrupt handler.
@@ -166,28 +208,60 @@ static void mprof_work_handler(struct work_struct *work)
}
/*
- * L3MissOnly + Exclude kernel RIP
+ * Translate the software config into IbsMemCtl / IbsMemCtl2 register values.
+ * Kernel RIP is always excluded: the profiler only samples user memory
+ * accesses.
*/
-static void mprof_enable_profiling(void)
+static void mprof_compose(struct mprof_config *cfg)
{
- u64 mprof_config = IBS_MPROF_CTL_CNT_CTL | IBS_MPROF_CTL_L3MISSONLY;
- unsigned int period = IBS_MPROF_SAMPLE_PERIOD;
- u64 ctl, ctl2;
+ u64 ctl;
+
+ if (!cfg->enabled) {
+ /*
+ * Stop the sample generation only; leave the extended LVT
+ * routed so re-enable does not reprogram the APIC. LVT
+ * mask/unmask is done by mprof_arm_cpu / mprof_disarm_cpu.
+ */
+ ctl = 0;
+ cfg->ctl = ctl;
+ cfg->ctl2 = IBS_MPROF_CTL2_DISABLE;
+ return;
+ }
/*
- * Assemble bits 26:20 and 19:4 of periodic op counter in ctl.
+ * Assemble bits 26:20 and 19:4 of the periodic op counter in ctl.
* The lower 4 bits are always 0000b.
*/
- ctl = (period >> 4) & IBS_MPROF_CTL_MAXCNT_MASK;
- ctl |= (period & IBS_MPROF_CTL_MAXCNT_EXT_MASK);
- ctl |= mprof_config;
- wrmsrq(MSR_AMD64_IBS_MPROF_CTL, ctl);
+ ctl = (cfg->period >> 4) & IBS_MPROF_CTL_MAXCNT_MASK;
+ ctl |= cfg->period & IBS_MPROF_CTL_MAXCNT_EXT_MASK;
+ ctl |= IBS_MPROF_CTL_CNT_CTL | IBS_MPROF_CTL_ENABLE;
- /*
- * Exclude samples that have bit 63 of their RIP set.
- */
- ctl2 = IBS_MPROF_CTL2_EXCLUDE_KERNEL;
- wrmsrq(MSR_AMD64_IBS_MPROF_CTL2, ctl2);
+ if (cfg->l3miss_only)
+ ctl |= IBS_MPROF_CTL_L3MISSONLY;
+
+ if (cfg->lat_filter) {
+ ctl |= IBS_MPROF_CTL_LATFLTEN;
+ ctl |= ((u64)cfg->lat_thresh << IBS_MPROF_CTL_LATTHRSH_SHIFT) &
+ IBS_MPROF_CTL_LATTHRSH_MASK;
+ }
+
+ cfg->ctl = ctl;
+ /* Exclude samples that have bit 63 of their RIP set (kernel). */
+ cfg->ctl2 = IBS_MPROF_CTL2_EXCLUDE_KERNEL;
+}
+
+/*
+ * Program the profiler from the currently published config snapshot. Called
+ * at CPU startup, from the interrupt handler on every re-arm, and via
+ * on_each_cpu() when the config changes.
+ */
+static void mprof_enable_profiling(void)
+{
+ /* Acquire the snapshot; pairs with smp_store_release() in the writers. */
+ struct mprof_config *cfg = smp_load_acquire(&mprof_cfg);
+
+ wrmsrq(MSR_AMD64_IBS_MPROF_CTL, cfg->ctl);
+ wrmsrq(MSR_AMD64_IBS_MPROF_CTL2, cfg->ctl2);
}
static void mprof_disable_profiling(u64 mem_ctl)
@@ -199,6 +273,56 @@ static void mprof_disable_profiling(u64 mem_ctl)
wrmsrq(MSR_AMD64_IBS_MPROF_CTL2, IBS_MPROF_CTL2_DISABLE);
}
+static void mprof_reprogram_this_cpu(void *info)
+{
+ mprof_enable_profiling();
+}
+
+/*
+ * Publish a new config snapshot and program every online CPU.
+ * Callers must hold mprof_cfg_lock and cpus_read_lock. The latter
+ * keeps the online set stable so on_each_cpu() cannot race a CPU's
+ * ONLINE arm path, which might still be reading the previous slot
+ * as this publish overwrites it.
+ */
+static void mprof_publish(const struct mprof_config *newcfg)
+{
+ struct mprof_config *slot;
+
+ lockdep_assert_cpus_held();
+ lockdep_assert_held(&mprof_cfg_lock);
+
+ /* Fill the slot that is not currently published, then flip to it. */
+ slot = (mprof_cfg == &mprof_cfg_slots[0]) ?
+ &mprof_cfg_slots[1] : &mprof_cfg_slots[0];
+ *slot = *newcfg;
+ mprof_compose(slot);
+ /* Publish the fully composed slot; pairs with smp_load_acquire() in readers. */
+ smp_store_release(&mprof_cfg, slot);
+
+ /*
+ * on_each_cpu() with wait serializes against any in-flight interrupt
+ * handler on each CPU, so the previously published slot has no readers
+ * once this returns and can be safely reused by the next writer.
+ */
+ on_each_cpu(mprof_reprogram_this_cpu, NULL, 1);
+}
+
+static void mprof_config_init(void)
+{
+ struct mprof_config *cfg = &mprof_cfg_slots[0];
+
+ if (ibs_mprof_enabled)
+ cfg->enabled = true;
+ cfg->l3miss_only = true;
+ cfg->lat_filter = false;
+ cfg->lat_thresh = 0;
+ cfg->period = IBS_MPROF_SAMPLE_PERIOD;
+ mprof_compose(cfg);
+ /* Publish the initial snapshot; pairs with smp_load_acquire() in readers. */
+ smp_store_release(&mprof_cfg, cfg);
+}
+
/*
* IBS interrupt handler: Process the memory access info reported by IBS.
*
@@ -222,6 +346,13 @@ static void mprof_overflow_handler(void)
rdmsrq(MSR_AMD64_IBS_MPROF_DATA3, mem_data3);
+ /*
+ * If L3 miss filtering is turned off, non load/store
+ * samples may get reported. Ignore them.
+ */
+ if (!(mem_data3 & (IBS_MPROF_DATA3_LDOP | IBS_MPROF_DATA3_STOP)))
+ goto handled;
+
/* Data source is valid only if the sample isn't an L1 or L2 hit */
if (!(mem_data3 & (IBS_MPROF_DATA3_DCMISS | IBS_MPROF_DATA3_L2MISS))) {
count_vm_event(HWHINT_CACHE_ACCESSES);
@@ -423,10 +554,245 @@ static void mprof_pm_init(void)
{
register_syscore(&mprof_syscore);
}
+
+static void mprof_pm_destroy(void)
+{
+ unregister_syscore(&mprof_syscore);
+}
#else
static void mprof_pm_init(void) {}
+static void mprof_pm_destroy(void) {}
#endif
+/*
+ * debugfs interface. Each parameter is a separate file under
+ * <debugfs>/ibs-mprof/. A write validates the value, updates the current
+ * config and actively propagates it to all CPUs via mprof_publish().
+ */
+enum mprof_field {
+ MPROF_L3MISS_ONLY,
+ MPROF_LAT_FILTER,
+ MPROF_LAT_THRESH,
+ MPROF_PERIOD,
+};
+
+static int mprof_parse_uint(const char __user *ubuf, size_t cnt, unsigned int *val)
+{
+ char buf[16];
+
+ if (cnt >= sizeof(buf))
+ return -EINVAL;
+
+ if (copy_from_user(buf, ubuf, cnt))
+ return -EFAULT;
+ buf[cnt] = '\0';
+ if (kstrtouint(buf, 0, val))
+ return -EINVAL;
+ return 0;
+}
+
+static void mprof_store(enum mprof_field field, unsigned int val)
+{
+ struct mprof_config new;
+
+ guard(cpus_read_lock)();
+ guard(mutex)(&mprof_cfg_lock);
+ new = *mprof_cfg;
+ switch (field) {
+ case MPROF_L3MISS_ONLY:
+ new.l3miss_only = val;
+ break;
+ case MPROF_LAT_FILTER:
+ new.lat_filter = val;
+ break;
+ case MPROF_LAT_THRESH:
+ new.lat_thresh = val;
+ break;
+ case MPROF_PERIOD:
+ new.period = val;
+ break;
+ }
+ mprof_publish(&new);
+}
+
+#define MPROF_SHOW(name, field) \
+static int mprof_##name##_show(struct seq_file *m, void *v) \
+{ \
+ unsigned int val; \
+ \
+ mutex_lock(&mprof_cfg_lock); \
+ val = mprof_cfg->field; \
+ mutex_unlock(&mprof_cfg_lock); \
+ seq_printf(m, "%u\n", val); \
+ return 0; \
+} \
+static int mprof_##name##_open(struct inode *inode, struct file *filp) \
+{ \
+ return single_open(filp, mprof_##name##_show, NULL); \
+}
+
+MPROF_SHOW(l3miss_only, l3miss_only)
+MPROF_SHOW(lat_filter, lat_filter)
+MPROF_SHOW(lat_thresh, lat_thresh)
+MPROF_SHOW(period, period)
+
+static ssize_t mprof_l3miss_only_write(struct file *filp, const char __user *ubuf,
+ size_t cnt, loff_t *ppos)
+{
+ unsigned int val;
+ int ret;
+
+ ret = mprof_parse_uint(ubuf, cnt, &val);
+ if (ret)
+ return ret;
+ if (val > 1)
+ return -EINVAL;
+
+ mprof_store(MPROF_L3MISS_ONLY, val);
+ *ppos += cnt;
+ return cnt;
+}
+
+static ssize_t mprof_lat_filter_write(struct file *filp, const char __user *ubuf,
+ size_t cnt, loff_t *ppos)
+{
+ unsigned int val;
+ int ret;
+
+ ret = mprof_parse_uint(ubuf, cnt, &val);
+ if (ret)
+ return ret;
+ if (val > 1)
+ return -EINVAL;
+
+ mprof_store(MPROF_LAT_FILTER, val);
+ *ppos += cnt;
+ return cnt;
+}
+
+static ssize_t mprof_lat_thresh_write(struct file *filp, const char __user *ubuf,
+ size_t cnt, loff_t *ppos)
+{
+ unsigned int val;
+ int ret;
+
+ ret = mprof_parse_uint(ubuf, cnt, &val);
+ if (ret)
+ return ret;
+ if (val > IBS_MPROF_LATTHRSH_MAX)
+ return -EINVAL;
+
+ mprof_store(MPROF_LAT_THRESH, val);
+ *ppos += cnt;
+ return cnt;
+}
+
+static ssize_t mprof_period_write(struct file *filp, const char __user *ubuf,
+ size_t cnt, loff_t *ppos)
+{
+ unsigned int val;
+ int ret;
+
+ ret = mprof_parse_uint(ubuf, cnt, &val);
+ if (ret)
+ return ret;
+
+ /* The lower 4 bits of IBSMemMaxCnt are always zero */
+ val &= ~0xf;
+ if (val < IBS_MPROF_MAXCNT_MIN || val > IBS_MPROF_MAXCNT_MAX)
+ return -EINVAL;
+
+ mprof_store(MPROF_PERIOD, val);
+ *ppos += cnt;
+ return cnt;
+}
+
+#define MPROF_FOPS(name) \
+static const struct file_operations mprof_##name##_fops = { \
+ .open = mprof_##name##_open, \
+ .read = seq_read, \
+ .write = mprof_##name##_write, \
+ .llseek = seq_lseek, \
+ .release = single_release, \
+}
+
+MPROF_FOPS(l3miss_only);
+MPROF_FOPS(lat_filter);
+MPROF_FOPS(lat_thresh);
+MPROF_FOPS(period);
+
+static void mprof_debugfs_init(void)
+{
+ struct dentry *dir = debugfs_create_dir("ibs-mprof", NULL);
+
+ debugfs_create_file("l3miss-only", 0644, dir, NULL, &mprof_l3miss_only_fops);
+ debugfs_create_file("lat-filter", 0644, dir, NULL, &mprof_lat_filter_fops);
+ debugfs_create_file("lat-thresh", 0644, dir, NULL, &mprof_lat_thresh_fops);
+ debugfs_create_file("period", 0644, dir, NULL, &mprof_period_fops);
+}
+
+static ssize_t enabled_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ bool enabled;
+
+ mutex_lock(&mprof_cfg_lock);
+ enabled = mprof_cfg->enabled;
+ mutex_unlock(&mprof_cfg_lock);
+
+ return sysfs_emit(buf, "%s\n", str_enabled_disabled(enabled));
+}
+
+static ssize_t enabled_store(struct device *dev, struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct mprof_config new;
+ bool enabled;
+ int ret;
+
+ ret = kstrtobool(buf, &enabled);
+ if (ret)
+ return ret;
+
+ guard(cpus_read_lock)();
+ guard(mutex)(&mprof_cfg_lock);
+ new = *mprof_cfg;
+ new.enabled = enabled;
+ mprof_publish(&new);
+
+ return count;
+}
+
+static DEVICE_ATTR_RW(enabled);
+
+static struct attribute *ibs_mprof_attributes[] = {
+ &dev_attr_enabled.attr,
+ NULL
+};
+
+static const struct attribute_group ibs_mprof_attr_group = {
+ .name = "ibs-mprof",
+ .attrs = ibs_mprof_attributes,
+};
+
+static int mprof_tunables_init(void)
+{
+ struct device *dev_root;
+ int ret = -ENODEV;
+
+ dev_root = bus_get_dev_root(&cpu_subsys);
+ if (!dev_root)
+ return ret;
+
+ ret = sysfs_create_group(&dev_root->kobj, &ibs_mprof_attr_group);
+ put_device(dev_root);
+ if (ret)
+ return ret;
+
+ mprof_debugfs_init();
+ return 0;
+}
+
static int __init mprof_access_profiling_init(void)
{
u32 mprof_caps;
@@ -460,6 +826,9 @@ static int __init mprof_access_profiling_init(void)
mw->cpu = cpu;
}
+ /* Publish the default config before the startup callback consumes it. */
+ mprof_config_init();
+
ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
"x86/amd/ibs_mprof:online",
x86_amd_ibs_mprof_online,
@@ -470,10 +839,26 @@ static int __init mprof_access_profiling_init(void)
mprof_pm_init();
+ ret = mprof_tunables_init();
+ if (ret)
+ goto out_hp;
+
pr_info("IBS Memory Profiler is available for memory access profiling\n");
return 0;
+out_hp:
+ mprof_pm_destroy();
+ cpuhp_remove_state(mprof_cpuhp_state);
out_free:
+ /*
+ * With ibs-memprof=1 the profiler was armed above, so an interrupt may
+ * have queued a worker that dereferences mprof_s. Producers are now
+ * silenced (cpuhp offline disarmed every CPU, or cpuhp_setup_state
+ * rolled back its arming), so nothing re-schedules; drain any queued or
+ * running worker before freeing the per-CPU buffer it uses.
+ */
+ for_each_possible_cpu(cpu)
+ cancel_work_sync(&per_cpu_ptr(&mprof_work, cpu)->work);
free_percpu(mprof_s);
pr_info("IBS Memory Profiler failed to initialize\n");
return ret;
--
2.34.1
^ permalink raw reply [flat|nested] 10+ messages in thread* Re: [RFC PATCH v0 0/3] pghot: x86: IBS Memory Profiler for hot page promotion
2026-09-24 6:22 [RFC PATCH v0 0/3] pghot: x86: IBS Memory Profiler for hot page promotion Bharata B Rao
` (2 preceding siblings ...)
2026-09-24 6:22 ` [RFC PATCH v0 3/3] x86/mm/ibs: Add runtime controls for IBS memprofiler Bharata B Rao
@ 2026-09-24 6:26 ` Bharata B Rao
2026-09-24 6:27 ` Bharata B Rao
` (4 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Bharata B Rao @ 2026-09-24 6:26 UTC (permalink / raw)
To: linux-kernel, linux-mm
Cc: tglx, mingo, bp, dave.hansen, x86, hpa, xin, luto, peterz,
Jonathan.Cameron, gourry, rientjes, sj, weixugc, willy,
ying.huang, ziy, dave, nifan.cxl, xuezhengchu, yiannis, akpm,
david, byungchul, kinseyho, joshua.hahnjy, yuanchu, balbirs,
shivankg, donettom, linyongting
On 24-Sep-26 11:52 AM, Bharata B Rao wrote:
>
> Detailed per-benchmark tables (throughput/latency + vmstat and pghot
> promotion counters) are posted as replies to this thread.
Micro-benchmark
Multi-threaded application with 64 threads that access memory(8G) at
4K granularity repetitively and randomly. The number of accesses per
thread and the randomness pattern for each thread are fixed beforehand.
The accesses are divided into stores and loads in the ratio of 50:50.
Benchmark threads run on Node 0, while memory is initially provisioned on
CXL node 2 before the accesses start.
Repetitive accesses results in lowertier pages becoming hot and kmigrated
detecting and migrating them. The benchmark score is the time taken to
finish the accesses in microseconds. The sooner it finishes the better it is.
All the numbers shown below are average of 3 runs.
Table 1: Completion time (lower is better), avg of 3 runs
64 threads, random 4K access over 8G, mem on CXL node 2, cpu node 0
Config Time (us) Time (s) Speedup
---------------------------------------
C1 98,485,716 98.5 1.00x
C2 38,636,402 38.6 2.55x
C3 44,270,350 44.3 2.22x
Speedup = C1 (base-NUMAB0) time / config time (>1.00x is faster)
Legend:
C1 = base kernel, NUMAB0 (hot page promotion disabled)
C2 = base kernel, NUMAB2 (hot page promotion enabled)
C3 = pghot kernel, IBS profiler, pghot_freq_threshold=1 (NUMAB0)
Table 2: Page migration / hotness vmstat counters (avg of 3 runs)
'-' = counter not present for that kernel/config
vmstat counter C1 C2 C3
------------------------------------------------------------
pgpromote_success 0 2,097,152 1,981,045
pgpromote_candidate_nrl 0 2,412,042 1,981,045
numa_pages_migrated 0 2,097,152 1,981,045
numa_pte_updates 0 2,097,152 0
numa_hint_faults 0 2,412,042 0
pghot_recorded_accesses - - 1,985,269
pghot_reported_hintfaults - - 0
pghot_reported_hwhints - - 5,991,177
hwhint_total_events - - 5,991,177
hwhint_dram_accesses - - 3,716,462
hwhint_extmem_accesses - - 1,886,078
hwhint_cache_accesses - - 0
hwhint_useful_events - - 5,991,177
hwhint_dropped_events - - 0
Legend:
C1 = base kernel, NUMAB0 (hot page promotion disabled)
C2 = base kernel, NUMAB2 (hot page promotion enabled)
C3 = pghot kernel, IBS profiler, pghot_freq_threshold=1 (NUMAB0)
^ permalink raw reply [flat|nested] 10+ messages in thread* Re: [RFC PATCH v0 0/3] pghot: x86: IBS Memory Profiler for hot page promotion
2026-09-24 6:22 [RFC PATCH v0 0/3] pghot: x86: IBS Memory Profiler for hot page promotion Bharata B Rao
` (3 preceding siblings ...)
2026-09-24 6:26 ` [RFC PATCH v0 0/3] pghot: x86: IBS Memory Profiler for hot page promotion Bharata B Rao
@ 2026-09-24 6:27 ` Bharata B Rao
2026-09-24 6:31 ` Bharata B Rao
` (3 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Bharata B Rao @ 2026-09-24 6:27 UTC (permalink / raw)
To: linux-kernel, linux-mm
Cc: tglx, mingo, bp, dave.hansen, x86, hpa, xin, luto, peterz,
Jonathan.Cameron, gourry, rientjes, sj, weixugc, willy,
ying.huang, ziy, dave, nifan.cxl, xuezhengchu, yiannis, akpm,
david, byungchul, kinseyho, joshua.hahnjy, yuanchu, balbirs,
shivankg, donettom, linyongting
On 24-Sep-26 11:52 AM, Bharata B Rao wrote:
>
> Detailed per-benchmark tables (throughput/latency + vmstat and pghot
> promotion counters) are posted as replies to this thread.
Graph500 tiering comparison: base vs pghot (hwhints)
====================================================
Kernel(s) : base = 7.3.0-rc2-base+
pghot = 7.3.0-rc2-pghot+
Benchmark : Graph500 reference BFS, SCALE=28, edgefactor=16, 128 ranks
Topology : top-tier NUMA node=1 (CPUs+DRAM), lower-tier node=2 (mem-only/CXL)
Note : SKIP_VALIDATION=1 (timing-only); all TEPS carry Graph500 (!) flag.
Figure of merit is harmonic_mean_TEPS (hmean).
Config legend
-------------
C1 : base kernel, no tiering (numa_balancing=0)
C2 : base kernel, NUMAB2 tiering (numa_balancing=2)
C3 : pghot kernel, hwhints source (pghot_enabled_sources=2,
pghot_freq_threshold=1, pghot_target_nid=0 [default],
IBS mem-profiler on, numa_balancing=0, promotion=off)
Column legend
-------------
hmean : harmonic_mean_TEPS (primary Graph500 metric)
hstddev : harmonic_stddev_TEPS
median : median_TEPS
bfs_t : mean BFS time (seconds)
spdup : speedup of hmean vs C1 baseline
Table 1: Performance
--------------------
+--------+------------+----------+------------+---------+--------+
| Config | hmean | hstddev | median | bfs_t | spdup |
| | (TEPS) | (TEPS) | (TEPS) | (sec) | |
+--------+------------+----------+------------+---------+--------+
| C1 | 5.543e+08 | 5.51e+05 | 5.548e+08 | 7.748 | 1.00x |
| C2 | 1.298e+09 | 6.80e+07 | 1.394e+09 | 3.308 | 2.34x |
| C3 | 1.755e+09 | 2.32e+07 | 1.805e+09 | 2.447 | 3.17x |
+--------+------------+----------+------------+---------+--------+
Table 2: Relevant kernel counters (/proc/vmstat deltas over the run)
--------------------------------------------------------------------
Values are accumulated deltas (before -> after) for the whole run.
+----------------------------+-----------+-----------+-----------+
| Counter | C1 | C2 | C3 |
+----------------------------+-----------+-----------+-----------+
| numa_pte_updates | 0 | 25867526 | 0 |
| numa_hint_faults | 0 | 13318442 | 0 |
| numa_pages_migrated | 0 | 13318248 | 3702779 |
| pgpromote_success | 0 | 13317996 | 3702779 |
| pghot_recorded_accesses | 0 | 0 | 3709015 |
| pghot_reported_hintfaults | 0 | 0 | 0 |
| pghot_reported_hwhints | 0 | 0 | 21385228 |
| hwhint_total_events | 0 | 0 | 21385281 |
| hwhint_dram_accesses | 0 | 0 | 16747814 |
| hwhint_extmem_accesses | 0 | 0 | 3697760 |
| hwhint_cache_accesses | 0 | 0 | 0 |
| hwhint_useful_events | 0 | 0 | 21385228 |
| hwhint_dropped_events | 0 | 0 | 0 |
| pgmigrate_success | 26841377 | 40161043 | 30548434 |
+----------------------------+-----------+-----------+-----------+
Key findings
------------
1. Tiering is the dominant win: C2 (base NUMAB2) reaches 2.34x and C3
(pghot hwhints) 3.17x over the untiered baseline (C1), where the
working set is stranded on the lower-tier/CXL node 2.
2. pghot hwhints now clearly leads base NUMAB2 on the official metric:
C3 1.755e9 (3.17x) vs C2 1.298e9 (2.34x) -> ~+35% hmean_TEPS.
3. hwhints/IBS is far more efficient and stable. C3 reaches its higher
hmean while:
- promoting only ~3.70M pages, ~1/4 of C2 (13.32M);
- issuing zero NUMA hint faults / PTE scans (numa_pte_updates=0);
- being ~3x more consistent (hstddev 2.32e7 vs 6.80e7).
IBS reported 21.39M hwhint events (16.75M DRAM + 3.70M ext-mem),
with hwhint_dropped_events=0.
4. median vs harmonic-mean: C2 median (1.394e9) sits well above its
hmean (1.298e9), reflecting high per-BFS variance in the fault-driven
path. C3 median (1.805e9) and hmean (1.755e9) sit close together ->
low variance.
Caveats
-------
* Single run per configuration; the median-vs-hmean spread for C2
indicates non-trivial run-to-run variance. Repeat runs are advisable
before drawing firm quantitative conclusions.
* SKIP_VALIDATION=1 was used (timing-only), so TEPS values are flagged
invalid (!) by Graph500 and are intended for relative comparison only.
^ permalink raw reply [flat|nested] 10+ messages in thread* Re: [RFC PATCH v0 0/3] pghot: x86: IBS Memory Profiler for hot page promotion
2026-09-24 6:22 [RFC PATCH v0 0/3] pghot: x86: IBS Memory Profiler for hot page promotion Bharata B Rao
` (4 preceding siblings ...)
2026-09-24 6:27 ` Bharata B Rao
@ 2026-09-24 6:31 ` Bharata B Rao
2026-09-24 6:33 ` Bharata B Rao
` (2 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Bharata B Rao @ 2026-09-24 6:31 UTC (permalink / raw)
To: linux-kernel, linux-mm
Cc: tglx, mingo, bp, dave.hansen, x86, hpa, xin, luto, peterz,
Jonathan.Cameron, gourry, rientjes, sj, weixugc, willy,
ying.huang, ziy, dave, nifan.cxl, xuezhengchu, yiannis, akpm,
david, byungchul, kinseyho, joshua.hahnjy, yuanchu, balbirs,
shivankg, donettom, linyongting
On 24-Sep-26 11:52 AM, Bharata B Rao wrote:
>
> Detailed per-benchmark tables (throughput/latency + vmstat and pghot
> promotion counters) are posted as replies to this thread.
Benchmark
----------------------------------------------------------------------
NAS Parallel Benchmark BT (Block Tri-diagonal solver), MPI version,
Class D, 16 ranks. Compute threads are bound to node 1 via:
/usr/bin/mpirun -np 16 /usr/bin/numactl --cpunodebind=1 \
.../NPB3.4.4/NPB3.4-MPI/bin/bt.F.x
Test system : 3 NUMA nodes. Nodes 0 and 1 are top-tier DRAM
with CPUs (~252 GiB each); node 2 is a CPU-less CXL lower tier
(~252 GiB). Node distances: 0<->1 = 12, 1<->2 = 50. Before the
measurement phase all provisioned memory (~161.4 GiB across ranks)
is migrated to the CXL node 2, then ranks resume and measurement
begins, so the run starts fully cold on slow memory. This isolates
the effect of hot-page promotion (node 2 -> node 1) on a
bandwidth/latency-sensitive workload.
Kernels
----------------------------------------------------------------------
base = 7.3.0-rc2-base+ (mainline base)
pghot = 7.3.0-rc2-pghot+ (pghot + AMD IBS Memory Profiler)
Legend
----------------------------------------------------------------------
A = base-numab0 Base kernel, NO tiering (numa_balancing=0);
reference case
B = base-numab2 Base kernel, kernel NUMA-balancing tiering
(numa_balancing=2, hint-fault driven promotion,
demotion enabled)
C = pghot-hwhints pghot kernel, promotion driven by the AMD IBS
Memory Profiler (hardware access sampling);
numa_balancing=0 (no NUMA hint faults / scanning)
IBS Memory Profiler configuration (run C)
----------------------------------------------------------------------
vm.pghot_enabled_sources = 2 (HWHINTS source)
vm.pghot_target_nid = 1 (promote to compute node 1,
same target NUMAB2 uses)
vm.pghot_freq_threshold = 1 (promote on 1st access, like
NUMAB2)
vm.pghot_promote_rate_limit_MBps = 65536
vm.pghot_promote_window_ms = 3000
/sys/devices/system/cpu/ibs-mprof/enabled = enabled
ibs-mprof: period=10000, l3miss-only=1, lat-thresh=0, lat-filter=0
Table 1a - Runtime (lower is better)
----------------------------------------------------------------------
Metric A B C
----------------------------------------------------------------------
NPB Time (s) 7564.04 3232.04 3515.97
Speedup vs A (time) 1.00 2.34 2.15
Table 1b - Throughput (higher is better)
----------------------------------------------------------------------
Metric A B C
----------------------------------------------------------------------
NPB Mop/s total 51739.90 121088.51 111310.01
Mop/s ratio vs A 1.00 2.34 2.15
Mop/s ratio vs B 0.43 1.00 0.92
Peak prov. mem (GiB) 161.38 161.38 161.38
Table 2 - vmstat counter deltas (after - before measurement window)
----------------------------------------------------------------------
Counter A B C
----------------------------------------------------------------------
pgpromote_success 0 42181004 39701482
pgpromote_candidate 0 0 0
pgpromote_candidate_nrl 0 42181008 39702291
pgdemote_kswapd 0 0 0
pgdemote_direct 0 0 0
numa_pte_updates 0 42252137 0
numa_hint_faults 0 42182619 0
numa_pages_migrated 0 42182610 39701482
pgmigrate_success 0 42182610 39701482
pghot_recorded_accesses 0 0 39793839
pghot_reported_hintfaults 0 0 0
pghot_reported_hwhints 0 0 553943841
hwhint_total_events 0 0 553950006
hwhint_dram_accesses 0 0 514144414
hwhint_extmem_accesses 0 0 39794778
hwhint_cache_accesses 0 0 0
hwhint_useful_events 0 0 553943842
hwhint_dropped_events 0 0 6147
Key observations
----------------------------------------------------------------------
1. Tiering is essential for BT. With the whole footprint pinned to
the CXL lower tier and no promotion (A), BT is 2.34x slower than
kernel NUMA-balancing tiering (7564 vs 3232 s) and delivers less
than half the throughput (51.7k vs 121.1k Mop/s). BT is
bandwidth/latency-bound and much of its footprint is genuinely
hot, so promotion back to DRAM helps a lot.
2. IBS Memory Profiler drives effective promotion WITHOUT NUMA
balancing. Run C reaches 3516 s / 111.3k Mop/s = 2.15x over the
no-tiering baseline, i.e. ~92% of NUMAB2's throughput, while
numa_pte_updates and numa_hint_faults are both 0 -- no PTE
scanning and no hint faults are involved. Promotion is instead
fed entirely by hardware access samples.
3. Comparable migration volume, different mechanism. C promotes
~39.70M pages (pgpromote_success == pgmigrate_success ==
numa_pages_migrated) vs ~42.18M for NUMAB2 (B) -- about 6% fewer,
consistent with the ~9% runtime gap. Both promote to node 1.
4. IBS sampling is high-volume but precise. With period=10000 and
l3miss-only=1, the profiler recorded ~554M useful events
(hwhint_useful_events / pghot_reported_hwhints = 553,943,84x).
Of ~554M total events, ~514M were DRAM accesses and only
~39.79M were external/CXL accesses (hwhint_extmem_accesses).
That extmem count matches pages promoted (39.70M) and
pghot_recorded_accesses (39.79M) very closely: nearly every
distinct external-memory access observed led to a promotion,
i.e. precise hot-page targeting.
5. Negligible event loss. hwhint_dropped_events = 6,147 out of
~554M total (~0.001%), so the SPSC delivery path kept up with
the hardware sample rate.
6. Migration is promotion-only. pgdemote_* and pgpromote_candidate
are 0 across all runs; memory is pre-staged on node 2 and DRAM
has ample headroom (~252 GiB/node vs ~161 GiB), so pages flow
node 2 -> node 1 with no demotion pressure.
7. Caveats: Verification reported UNSUCCESSFUL in every run
(consistent across all three, so a benchmark class/build
artifact rather than a tiering effect). These are single runs
per config; the ~9% (C vs B) gap should be confirmed with
repeats to establish variance.
Bottom line: the AMD IBS Memory Profiler source drives pghot
promotion purely from hardware access samples -- no NUMA hint
faults, no PTE scanning -- and recovers ~92% of kernel
NUMA-balancing tiering performance on NAS BT, at 2.15x over the
untiered baseline.
^ permalink raw reply [flat|nested] 10+ messages in thread* Re: [RFC PATCH v0 0/3] pghot: x86: IBS Memory Profiler for hot page promotion
2026-09-24 6:22 [RFC PATCH v0 0/3] pghot: x86: IBS Memory Profiler for hot page promotion Bharata B Rao
` (5 preceding siblings ...)
2026-09-24 6:31 ` Bharata B Rao
@ 2026-09-24 6:33 ` Bharata B Rao
2026-09-24 6:35 ` Bharata B Rao
2026-09-24 6:37 ` Bharata B Rao
8 siblings, 0 replies; 10+ messages in thread
From: Bharata B Rao @ 2026-09-24 6:33 UTC (permalink / raw)
To: linux-kernel, linux-mm
Cc: tglx, mingo, bp, dave.hansen, x86, hpa, xin, luto, peterz,
Jonathan.Cameron, gourry, rientjes, sj, weixugc, willy,
ying.huang, ziy, dave, nifan.cxl, xuezhengchu, yiannis, akpm,
david, byungchul, kinseyho, joshua.hahnjy, yuanchu, balbirs,
shivankg, donettom, linyongting
On 24-Sep-26 11:52 AM, Bharata B Rao wrote:
>
> Detailed per-benchmark tables (throughput/latency + vmstat and pghot
> promotion counters) are posted as replies to this thread.
======================================================================
Tiered-memory promotion: baseline vs NUMA-balancing vs DAMON vs pghot
hwhints (IBS) - multithreaded runs
======================================================================
Runs taken on one machine, same workload and sizes. This is a self-contained
comparison of four ways to promote a hot working set from a slow (CXL) tier
to a fast (DRAM) tier:
- baseline : no promotion.
- NUMAB2 : the kernel's NUMA-balancing tier promotion
(numa_balancing=2), driven by NUMA hint faults.
- DAMON fine+quota: in-kernel access monitor with a migrate_hot
scheme (paddr), driven by the kdamond thread.
- pghot hwhints : per-PFN hotness from the IBS Memory Profiler
(hardware sampling), promoted by the kmigrated
thread; shown at two IBS sample periods.
Machine / topology
------------------
- 256 CPUs, 792 GB RAM.
- NUMA: node 0, node 1 = DRAM (fast tier); node 2 = CXL (slow tier,
256 GB). Promotion target = node 0; cold data starts on node 2.
- Kernels: base = 7.3.0-rc2-base+ (baseline, NUMAB2, DAMON);
pghot = 7.3.0-rc2-pghot+ (pghot hwhints).
Workload
--------
- 64-thread pointer chase, 64 GB total buffer, 16 GB hot set, 240 s
timed phase. Buffer relocated to node 2 before the timed phase;
worker threads bound to node 0. "hot set" = 16 GB;
over = (node0 - 16 GB) / 16 GB.
Configurations
--------------
- baseline : numa_balancing=0, no promotion.
- NUMAB2 : numa_balancing=2 (tier promotion via hint faults).
- DAMON : sysfs defaults (sample 5ms, aggr 100ms, update 60s) +
migrate_hot (paddr, nr_accesses>=1) + max_nr_regions=
100000 (min default 10) + quota 128 MiB/1000 ms with
nr_accesses weighting (hottest first).
- hwhints : IBS Memory Profiler as the sole hotness source
(numa_balancing=0), pghot_freq_threshold=1, IBS
l3miss-only=1; IBS sample period 10000 and 5008.
Metric notes
------------
- lat : steady access latency, ns/access, mean of last 25% of samples
(lower is better).
- node0: end-of-run placement on node 0 (numastat -p).
- promoted: bytes migrated to the fast tier (DAMON scheme sz_applied;
NUMAB2/pghot pgpromote_success).
- CPU : cumulative cputime over the 240 s run of the single promoter/
monitor thread - kd = kdamond (DAMON), km = kmigrated (pghot).
NUMAB2 has no single promoter thread; its cost is distributed
across the application threads (scanning + hint faults).
- over: The "over" column is measured against the 16 GB HOT SET, not
against B0: over = (node0 - 16 GB) / 16 GB. B0 promotes
nothing, so its node0 is 0.0 and its over is n/a.
Table. Multithreaded results (64 GB total / 16 GB hot)
------------------------------------------------------
id config kern lat node0 over promoted promoter
(ns) (GB) (GB) CPU
--- ------------------ ----- --- ----- ----- -------- ----------
B0 baseline base 320 0.0 - - -
N2 NUMAB2 base 143 16.0 0% ~14.6* distributed
D3 DAMON fine+quota base 119 23.9 +49% 24.7 kd 13s(5%)
H10 hwhints IBS p10000 pghot 215 4.9 -69% 4.9 km 20s(8%)
H5 hwhints IBS p5008 pghot 92 11.6 -27% 12.0 km 38s(16%)
* N2 pgpromote_success over the timed window is ~14.6 GB; some
promotion happens during the load phase before the vmstat baseline,
so the reliable figure is the placement: node 0 = 16.0 GB (exact).
Supporting counters
-------------------
- N2 : numa_hint_faults 3.73M, numa_pte_updates 12.9M.
- D3 : sz_applied 24.7 GB, qt_exceeds 229, regions settled ~21-71.
- H10: hwhint_total_events 1.40M, pgpromote_success 1.29M.
- H5 : hwhint_total_events 5.91M, pgpromote_success 3.15M.
Findings
--------
1. Accuracy (placement):
- NUMAB2 places exactly the 16 GB hot set (0% over).
- DAMON over-promotes (+49%, 23.9 GB): a coarse-then-quota scheme
bounds volume but still moves ~1.5x the hot set.
- hwhints under-promotes within 240 s (-69% at p10000, -27% at
p5008) because hardware sampling observes only a fraction of
accesses; it had not fully converged at 240 s.
2. IBS sample period is the coverage knob. 10000 -> 5008 roughly
tripled events (1.40M -> 5.91M), nearly tripled placement (4.9 ->
11.6 GB) and cut latency 215 -> 92 ns. period=10000 under-samples
this workload.
3. Latency does not track placement linearly:
- hwhints p5008 reaches the lowest latency (92 ns) with only 11.6 GB
on node 0 - IBS targets the hottest pages, so its smaller
placement is well-chosen.
- DAMON needs 23.9 GB to reach 119 ns.
- NUMAB2 places the exact 16 GB but its steady latency (143 ns) is
higher than its own first interval (119 ns): numa_balancing keeps
scanning and faulting the workload's pages after promotion, an
ongoing per-access tax (12.9M pte updates, 3.73M hint faults).
4. Promoter CPU:
- DAMON kdamond and pghot kmigrated are both out-of-band and cheap
here (kd 13 s; km 20-38 s over 240 s, i.e. ~8-16% of one core).
- NUMAB2's cost is not a single thread; it is paid by the workload
threads via scanning and hint-fault handling.
Caveats
-------
- Two kernels: base (B0/N2/D3) and pghot (H10/H5). No separate
no-promotion baseline was taken on the pghot kernel in this set, so
cross-kernel absolute latency should be read with that in mind;
placement / over-promotion is the primary accuracy metric.
- The CPU column is the concentrated promoter/monitor thread only. It
excludes distributed costs: NUMAB2's scanning + hint-fault handling,
and IBS interrupt/sample handling for hwhints.
- Latency is not isolated to a single cause (bandwidth, working set,
selection quality all plausible); placement is the primary accuracy
metric and latency is supporting.
- hwhints coverage is still ~27% under the 16 GB hot set at 240 s
(p5008); it was still climbing at the end of the run.
^ permalink raw reply [flat|nested] 10+ messages in thread* Re: [RFC PATCH v0 0/3] pghot: x86: IBS Memory Profiler for hot page promotion
2026-09-24 6:22 [RFC PATCH v0 0/3] pghot: x86: IBS Memory Profiler for hot page promotion Bharata B Rao
` (6 preceding siblings ...)
2026-09-24 6:33 ` Bharata B Rao
@ 2026-09-24 6:35 ` Bharata B Rao
2026-09-24 6:37 ` Bharata B Rao
8 siblings, 0 replies; 10+ messages in thread
From: Bharata B Rao @ 2026-09-24 6:35 UTC (permalink / raw)
To: linux-kernel, linux-mm
Cc: tglx, mingo, bp, dave.hansen, x86, hpa, xin, luto, peterz,
Jonathan.Cameron, gourry, rientjes, sj, weixugc, willy,
ying.huang, ziy, dave, nifan.cxl, xuezhengchu, yiannis, akpm,
david, byungchul, kinseyho, joshua.hahnjy, yuanchu, balbirs,
shivankg, donettom, linyongting
On 24-Sep-26 11:52 AM, Bharata B Rao wrote:
>
> Detailed per-benchmark tables (throughput/latency + vmstat and pghot
> promotion counters) are posted as replies to this thread.
llama.cpp tiering benchmark: base vs pghot-hwhints (kernel 7.3.0-rc2)
====================================================================
3-run average per configuration.
Workload: llama-bench (llama.cpp), Mixtral-8x22B-Instruct Q4_K_M (140.6B
params, 79.7 GB), -t 64 -p 512 -n 128 -r 5 --mmap 0. pp512 = prefill
(compute-bound); tg128 = decode (memory-bandwidth-bound).
Setup: 3-node box, N0/N1 DRAM (256 GB each), N2 CXL (distances 10/12/50).
Bench pinned to N1 CPUs with MPOL_PREFERRED_MANY({1}); a 200 GB hot hog on
N1 forces kswapd to naturally demote ~1/3 of the model to CXL; each mode is
then measured while the hog keeps N1 under pressure. pghot target_nid=1,
freq_threshold=1, promote_window=3000ms, rate_limit=65536 MBps, kmigrated
100ms/512. hwhints arms AMD IBS (l3miss-only=1, period=10000).
Legend (columns)
----------------
r1 = base / notier 7.3.0-rc2-base nb=0 src=-
r2 = base / tier (NUMAB2) 7.3.0-rc2-base nb=2 src=-
r3 = pghot / hwhints(IBS) 7.3.0-rc2-pghot nb=0 src=0x2
(nb = kernel.numa_balancing; src = pghot_enabled_sources; 3 runs each)
Table 1 - Throughput, 3-run mean +/- stdev (llama-bench tokens/s)
----------------------------------------------------------------
metric r1 r2 r3
-------------------------------------------------
pp512 mean 69.43 61.34 57.90
stdev 0.85 1.78 2.21
tg128 mean 4.094 4.883 4.975
stdev 0.045 0.053 0.023
tg128 vs r1 1.00x 1.19x 1.22x
pp512 vs r1 1.00x 0.88x 0.83x
Table 1b - per-run values (3 runs), shows consistency
-----------------------------------------------------
r1 r2 r3
-------------------------------------------------
tg128 run1 4.099 4.856 4.949
tg128 run2 4.037 4.836 5.005
tg128 run3 4.146 4.957 4.970
pp512 run1 70.37 63.37 61.01
pp512 run2 68.31 59.03 56.61
pp512 run3 69.60 61.64 56.07
Table 2 - Key vmstat counters, 3-run mean of run-phase deltas (millions)
-----------------------------------------------------------------------
metric r1 r2 r3
------------------------------------------------------
pgpromote_success 0.00 3.07 1.74
pgpromote_candidate 0.00 12.80 12.86
pgdemote_kswapd 2.58 4.78 4.40
pgmigrate_success 2.58 7.85 6.13
numa_pte_updates 0.00 26.06 0.00
numa_hint_faults 0.00 25.12 0.00
pghot_recorded_accesses 0.00 0.00 12.86
pghot_reported_hwhints 0.00 0.00 30.79
hwhint_total_events 0.00 0.00 30.79
hwhint_dram_accesses 0.00 0.00 17.74
hwhint_extmem_accesses 0.00 0.00 12.86
start N2 % (at SIGCONT) 28.3 28.9 27.6
Key observations (3-run averages)
---------------------------------
1. base/tier (r2) tg128 4.883 is +19.3% over base/notier (r1)
4.094; pghot/hwhints (r3) 4.975 is +21.5% over r1 and
+1.9% over r2.
2. Consistency is tight: r3 tg128 4.949-5.005 (stdev 0.023);
r2 4.836-4.957 (stdev 0.053).
3. hwhints armed IBS: ~35M events, ~13M on CXL; pgpromote_candidate
12.9M matches pghot_recorded_accesses 12.9M (1.00x).
4. Prefill (pp512) pays a tiering tax: r2 0.88x, r3 0.83x of r1.
Caveats: base and pghot are different kernels (7.3.0-rc2-base vs -pghot);
the natural-overflow setup settled at ~30% CXL at SIGCONT in all runs;
3 runs per configuration (per-run values in Table 1b).
^ permalink raw reply [flat|nested] 10+ messages in thread* Re: [RFC PATCH v0 0/3] pghot: x86: IBS Memory Profiler for hot page promotion
2026-09-24 6:22 [RFC PATCH v0 0/3] pghot: x86: IBS Memory Profiler for hot page promotion Bharata B Rao
` (7 preceding siblings ...)
2026-09-24 6:35 ` Bharata B Rao
@ 2026-09-24 6:37 ` Bharata B Rao
8 siblings, 0 replies; 10+ messages in thread
From: Bharata B Rao @ 2026-09-24 6:37 UTC (permalink / raw)
To: linux-kernel, linux-mm
Cc: tglx, mingo, bp, dave.hansen, x86, hpa, xin, luto, peterz,
Jonathan.Cameron, gourry, rientjes, sj, weixugc, willy,
ying.huang, ziy, dave, nifan.cxl, xuezhengchu, yiannis, akpm,
david, byungchul, kinseyho, joshua.hahnjy, yuanchu, balbirs,
shivankg, donettom, linyongting
On 24-Sep-26 11:52 AM, Bharata B Rao wrote:
>
> Detailed per-benchmark tables (throughput/latency + vmstat and pghot
> promotion counters) are posted as replies to this thread.
==========================================================================
Redis + memtier: hot-page promotion on a CXL-tiered system
==========================================================================
Benchmark: an in-memory Redis server is loaded with a ~64 GB dataset
(62.2M keys x 1 KB) whose pages are then explicitly migrated to the CXL
node (lower tier); the Redis server and the memtier client run on a top-
tier DRAM node (node 1). The measurement phase drives GET traffic with
memtier (16 threads x 100 conns, PASSES=12) over 50% of the keyspace so
that repeatedly-accessed lower-tier pages become promotion candidates.
Promotions target node 1 (local to the accessing threads): NUMAB uses
the accessing node, pghot uses pghot_target_nid=1.
System : AMD Zen6. Nodes 0,1 = DRAM (top tier);
node 2 = CXL (lower tier). Redis/client pinned to node 1.
Kernels: base=7.3.0-rc2-base+ pghot=7.3.0-rc2-pghot+
Cases:
C1 base/NUMAB0 base kernel, promotion OFF (baseline)
C2 base/NUMAB2 base kernel, NUMAB tiering promotion (hint
faults); promotes to local accessing node
C3 pghot/hwhints-10k pghot, NUMAB=0, source=IBS mprof,
period=10000, freq_thr=1, target_nid=1
C4 pghot/hwhints-5k as C3 but IBS period=5008 (kernel min,
~2x sampling; 5008)
==========================================================================
Table 1: Benchmark metrics (memtier)
==========================================================================
Case Ops/sec vs C1 Avg p50 p99 p99.9
latency (ms) ->
----------------------------------------------------------------------
C1 base/NUMAB0 285,288 ref 179.46 177.15 344.06 358.40
C2 base/NUMAB2 295,475 +3.57% 173.18 168.96 325.63 364.54
C3 pghot/hwhints-10k 285,923 +0.22% 178.91 177.15 344.06 358.40
C4 pghot/hwhints-5k 285,290 +0.00% 179.24 177.15 346.11 358.40
----------------------------------------------------------------------
==========================================================================
Table 2: Page-migration / hotness metrics (vmstat delta)
==========================================================================
Legend: C1=base/NUMAB0 C2=base/NUMAB2 C3=hwhints p=10000
C4=hwhints p=5008 (both hwhints: target_nid=1)
('-' = counter not present on base kernel)
metric C1 C2 C3 C4
-----------------------------------------------------------------
pgpromote_success 0 10,433,506 575,010 1,107,371
numa_pte_updates 0 20,333,730 0 0
numa_hint_faults 0 10,433,506 0 0
numa_pages_migrated 0 10,433,506 575,006 1,107,369
pgmigrate_success 0 10,433,506 575,006 1,107,369
pghot_recorded_accesses - - 575,677 1,110,760
pghot_reported_hwhints - - 964,304 1,967,044
hwhint_total_events - - 964,324 1,967,078
hwhint_dram_accesses - - 388,027 854,679
hwhint_extmem_accesses - - 575,674 1,110,758
hwhint_useful_events - - 964,304 1,967,044
pgdemote_kswapd 0 0 0 0
-----------------------------------------------------------------
==========================================================================
Key observations
==========================================================================
1. NUMAB tiering promotion (C2) helps only marginally: 295,475 vs 285,288
ops/sec (+3.57%), avg latency 179.5 -> 173.2 ms, promoting the full hot set
(10.43M pages / ~39.8 GiB) to the local node.
2. pghot with the IBS hwhints source is flat vs baseline at both periods
(C3 +0.22%, C4 +0.00%). numa_pte_updates / numa_hint_faults are
0 (no NUMA balancing); promotion is purely hardware-sample driven.
3. Sampling density scales promotion linearly (period 10000 -> 5008
doubles reported hwhints 964,304 -> 1,967,044 and promotions 575,010 ->
1,107,371 pages, ~2.19 -> ~4.22 GiB); ext-mem/CXL samples map ~1:1 to
promotions (freq_threshold=1), dram/already-toptier samples are not
promotable.
4. But even ~4.22 GiB is only ~10% of the ~40 GiB hot set, so throughput
does not move. Unlike C2 (which promotes the whole hot set), sampling-
based hwhints at these periods covers too little of the working set within
the run. Reaching C2's gain needs far denser sampling and/or a longer run
so hwhints promotes a large fraction of the hot set.
^ permalink raw reply [flat|nested] 10+ messages in thread