From: Dapeng Mi <dapeng1.mi@linux.intel.com>
To: Peter Zijlstra <peterz@infradead.org>,
Ingo Molnar <mingo@redhat.com>,
Arnaldo Carvalho de Melo <acme@kernel.org>,
Namhyung Kim <namhyung@kernel.org>,
Ian Rogers <irogers@google.com>,
Adrian Hunter <adrian.hunter@intel.com>,
Alexander Shishkin <alexander.shishkin@linux.intel.com>,
Andi Kleen <ak@linux.intel.com>,
Eranian Stephane <eranian@google.com>,
Sandipan Das <sandipan.das@amd.com>,
Andrii Nakryiko <andrii@kernel.org>, Song Liu <song@kernel.org>,
Ravi Bangoria <ravi.bangoria@amd.com>,
Ananth Narayan <ananth.narayan@amd.com>
Cc: linux-kernel@vger.kernel.org, linux-perf-users@vger.kernel.org,
bpf@vger.kernel.org, Dapeng Mi <dapeng1.mi@intel.com>,
Zide Chen <zide.chen@intel.com>,
Falcon Thomas <thomas.falcon@intel.com>,
Xudong Hao <xudong.hao@intel.com>,
Dapeng Mi <dapeng1.mi@linux.intel.com>
Subject: [Patch v6 11/20] perf/x86/intel: Allow LBR read helpers to operate on explicit stacks
Date: Fri, 6 Nov 2026 12:32:48 +0800 [thread overview]
Message-ID: <20261106043257.842864-12-dapeng1.mi@linux.intel.com> (raw)
In-Reply-To: <20261106043257.842864-1-dapeng1.mi@linux.intel.com>
The LBR read helpers currently assume the default stack,
cpuc->lbr_stack, and therefore cannot be reused for other LBR
stacks that may be in use. Extend the *_lbr_read() helpers to take
an explicit LBR stack argument, so they can operate on any LBR stack
instead of only the default one.
Update the related callers, including intel_pmu_store_lbr() and
intel_pmu_lbr_filter(), to pass the appropriate stack explicitly.
No function change intended.
Fixes: c22ac2a3d4bd ("perf: Enable branch record for software events")
Signed-off-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
---
arch/x86/events/amd/brs.c | 2 +-
arch/x86/events/amd/lbr.c | 2 +-
arch/x86/events/intel/core.c | 4 +-
arch/x86/events/intel/lbr.c | 89 ++++++++++++++++++++----------------
arch/x86/events/perf_event.h | 16 ++++---
5 files changed, 64 insertions(+), 49 deletions(-)
diff --git a/arch/x86/events/amd/brs.c b/arch/x86/events/amd/brs.c
index 1906d1257646..475e3699d3ca 100644
--- a/arch/x86/events/amd/brs.c
+++ b/arch/x86/events/amd/brs.c
@@ -316,7 +316,7 @@ void amd_brs_drain(void)
num = tos - start + 1;
- cpu_init_lbr_stack(cpuc, num);
+ cpu_init_lbr_stack(cpu_lbr_stack(cpuc), num);
/*
* BRS is only one pass (saturation) from MSROFF to depth-1
diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c
index 3e4e5e984649..95898a1d9901 100644
--- a/arch/x86/events/amd/lbr.c
+++ b/arch/x86/events/amd/lbr.c
@@ -179,7 +179,7 @@ void amd_pmu_lbr_read(void)
return;
}
- cpu_init_lbr_stack(cpuc, x86_pmu.lbr_nr);
+ cpu_init_lbr_stack(cpu_lbr_stack(cpuc), x86_pmu.lbr_nr);
for (i = 0; i < x86_pmu.lbr_nr; i++) {
entry.from.full = amd_pmu_lbr_get_from(i);
diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
index 42d4d525a729..b7045c05b736 100644
--- a/arch/x86/events/intel/core.c
+++ b/arch/x86/events/intel/core.c
@@ -2826,7 +2826,7 @@ __intel_pmu_snapshot_branch_stack(struct perf_branch_entry *entries,
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
- intel_pmu_lbr_read();
+ intel_pmu_lbr_read(cpu_lbr_stack(cpuc), cpuc->lbr_counters);
cnt = min_t(unsigned int, cnt, cpu_lbr_stack(cpuc)->nr);
memcpy(entries, cpu_lbr_stack(cpuc)->entries,
@@ -4062,7 +4062,7 @@ static int intel_pmu_handle_irq(struct pt_regs *regs)
loops = 0;
again:
- intel_pmu_lbr_read();
+ intel_pmu_lbr_read(cpu_lbr_stack(cpuc), cpuc->lbr_counters);
/*
* Do not clear the LBR frozen status here; keep LBR frozen
diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c
index 62f2f4b9cfde..979b094ae228 100644
--- a/arch/x86/events/intel/lbr.c
+++ b/arch/x86/events/intel/lbr.c
@@ -105,7 +105,9 @@
#define ARCH_LBR_CTL_MASK 0x7f000e
-static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc);
+static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc,
+ struct perf_branch_stack *lbr_stack,
+ u64 *lbr_counters);
static __always_inline bool is_lbr_call_stack_bit_set(u64 config)
{
@@ -741,13 +743,14 @@ void intel_pmu_lbr_disable_all(void)
}
}
-void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
+void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc,
+ struct perf_branch_stack *lbr_stack)
{
unsigned long mask = x86_pmu.lbr_nr - 1;
u64 tos = intel_pmu_lbr_tos();
int i;
- cpu_init_lbr_stack(cpuc, x86_pmu.lbr_nr);
+ cpu_init_lbr_stack(lbr_stack, x86_pmu.lbr_nr);
for (i = 0; i < x86_pmu.lbr_nr; i++) {
unsigned long lbr_idx = (tos - i) & mask;
union {
@@ -760,13 +763,13 @@ void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
rdmsrq(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
- cpu_lbr_stack(cpuc)->entries[i] = (struct perf_branch_entry){
+ lbr_stack->entries[i] = (struct perf_branch_entry){
.from = msr_lastbranch.from,
.to = msr_lastbranch.to,
};
}
- cpu_lbr_stack(cpuc)->nr = i;
- cpu_lbr_stack(cpuc)->hw_idx = tos;
+ lbr_stack->nr = i;
+ lbr_stack->hw_idx = tos;
}
/*
@@ -774,7 +777,8 @@ void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
* is the same as the linear address, allowing us to merge the LIP and EIP
* LBR formats.
*/
-void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
+void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc,
+ struct perf_branch_stack *lbr_stack)
{
bool need_info = false, call_stack = false;
unsigned long mask = x86_pmu.lbr_nr - 1;
@@ -789,7 +793,7 @@ void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
call_stack = true;
}
- cpu_init_lbr_stack(cpuc, num);
+ cpu_init_lbr_stack(lbr_stack, num);
for (i = 0; i < num; i++) {
unsigned long lbr_idx = (tos - i) & mask;
u64 from, to, mis = 0, pred = 0, in_tx = 0, abort = 0;
@@ -850,7 +854,7 @@ void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
if (abort && x86_pmu.lbr_double_abort && out > 0)
out--;
- cpu_lbr_stack(cpuc)->entries[out] = (struct perf_branch_entry){
+ lbr_stack->entries[out] = (struct perf_branch_entry){
.from = from,
.to = to,
.mispred = mis,
@@ -861,8 +865,8 @@ void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
};
out++;
}
- cpu_lbr_stack(cpuc)->nr = out;
- cpu_lbr_stack(cpuc)->hw_idx = tos;
+ lbr_stack->nr = out;
+ lbr_stack->hw_idx = tos;
}
static DEFINE_STATIC_KEY_FALSE(x86_lbr_mispred);
@@ -903,17 +907,19 @@ static __always_inline u16 get_lbr_cycles(u64 info)
static_assert((64 - PERF_BRANCH_ENTRY_INFO_BITS_MAX) > LBR_INFO_BR_CNTR_NUM * LBR_INFO_BR_CNTR_BITS);
-static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc,
- struct lbr_entry *entries)
+static void intel_pmu_store_lbr(struct lbr_entry *entries,
+ struct perf_branch_stack *lbr_stack)
{
+ struct perf_branch_entry *e;
struct lbr_entry *lbr;
u64 from, to, info;
bool mispred;
int i;
- cpu_init_lbr_stack(cpuc, x86_pmu.lbr_nr);
+ cpu_init_lbr_stack(lbr_stack, x86_pmu.lbr_nr);
for (i = 0; i < x86_pmu.lbr_nr; i++) {
lbr = entries ? &entries[i] : NULL;
+ e = &lbr_stack->entries[i];
from = rdlbr_from(i, lbr);
/*
@@ -927,7 +933,7 @@ static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc,
mispred = get_lbr_mispred(info);
- cpu_lbr_stack(cpuc)->entries[i] = (struct perf_branch_entry){
+ *e = (struct perf_branch_entry){
.from = from,
.to = to,
.mispred = mispred,
@@ -948,7 +954,7 @@ static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc,
};
}
- cpu_lbr_stack(cpuc)->nr = i;
+ lbr_stack->nr = i;
}
/*
@@ -999,37 +1005,41 @@ void intel_pmu_lbr_save_brstack(struct perf_sample_data *data,
perf_sample_save_brstack(data, event, cpu_lbr_stack(cpuc), NULL);
}
-static void intel_pmu_arch_lbr_read(struct cpu_hw_events *cpuc)
+static void intel_pmu_arch_lbr_read(struct cpu_hw_events *cpuc,
+ struct perf_branch_stack *lbr_stack)
{
- intel_pmu_store_lbr(cpuc, NULL);
+ intel_pmu_store_lbr(NULL, lbr_stack);
}
-static void intel_pmu_arch_lbr_read_xsave(struct cpu_hw_events *cpuc)
+static void
+intel_pmu_arch_lbr_read_xsave(struct cpu_hw_events *cpuc,
+ struct perf_branch_stack *lbr_stack)
{
struct x86_perf_task_context_arch_lbr_xsave *xsave = cpuc->lbr_xsave;
if (!xsave) {
- intel_pmu_store_lbr(cpuc, NULL);
+ intel_pmu_store_lbr(NULL, lbr_stack);
return;
}
xsave->header.xfeatures = 0;
xsaves(&xsave->xsave, XFEATURE_MASK_LBR);
if (xsave->header.xfeatures & XFEATURE_MASK_LBR) {
- intel_pmu_store_lbr(cpuc, xsave->lbr.entries);
+ intel_pmu_store_lbr(xsave->lbr.entries, lbr_stack);
} else {
/* Clear the LBR stack to avoid leaking stale entries. */
- cpu_lbr_stack(cpuc)->nr = 0;
+ lbr_stack->nr = 0;
}
}
-void intel_pmu_lbr_read(void)
+void intel_pmu_lbr_read(struct perf_branch_stack *lbr_stack,
+ u64 *lbr_counters)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
if (!cpuc->lbr_users || vlbr_exclude_host()) {
- /* Reset lbr_stack.nr to prevent stale data leakage. */
- cpu_lbr_stack(cpuc)->nr = 0;
+ /* Reset lbr_stack->nr to prevent stale data leakage. */
+ lbr_stack->nr = 0;
return;
}
@@ -1042,9 +1052,9 @@ void intel_pmu_lbr_read(void)
if (cpuc->lbr_users == cpuc->lbr_pebs_users)
return;
- x86_pmu.lbr_read(cpuc);
+ x86_pmu.lbr_read(cpuc, lbr_stack);
- intel_pmu_lbr_filter(cpuc);
+ intel_pmu_lbr_filter(cpuc, lbr_stack, lbr_counters);
}
/*
@@ -1227,7 +1237,9 @@ static const int arch_lbr_br_type_map[ARCH_LBR_BR_TYPE_MAP_MAX] = {
* in PERF_SAMPLE_BRANCH_STACK sample may vary.
*/
static void
-intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)
+intel_pmu_lbr_filter(struct cpu_hw_events *cpuc,
+ struct perf_branch_stack *lbr_stack,
+ u64 *lbr_counters)
{
int br_sel = cpuc->br_sel;
int i, j, type, from_plm, to_plm;
@@ -1240,8 +1252,8 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)
((br_sel & X86_BR_TYPE_SAVE) != X86_BR_TYPE_SAVE))
return;
- for (i = 0; i < cpu_lbr_stack(cpuc)->nr; i++) {
- struct perf_branch_entry *be = &cpu_lbr_stack(cpuc)->entries[i];
+ for (i = 0; i < lbr_stack->nr; i++) {
+ struct perf_branch_entry *be = &lbr_stack->entries[i];
from = be->from;
to = be->to;
@@ -1285,22 +1297,21 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)
return;
/* remove all entries with from=0 */
- lbr_nr = cpu_lbr_stack(cpuc)->nr;
+ lbr_nr = lbr_stack->nr;
for (i = 0; i < lbr_nr; ) {
- if (!cpu_lbr_stack(cpuc)->entries[i].from) {
+ if (!lbr_stack->entries[i].from) {
j = i;
while (++j < lbr_nr) {
- cpu_lbr_stack(cpuc)->entries[j-1] =
- cpu_lbr_stack(cpuc)->entries[j];
- cpuc->lbr_counters[j-1] = cpuc->lbr_counters[j];
+ lbr_stack->entries[j-1] = lbr_stack->entries[j];
+ lbr_counters[j-1] = lbr_counters[j];
}
lbr_nr--;
- if (!cpu_lbr_stack(cpuc)->entries[i].from)
+ if (!lbr_stack->entries[i].from)
continue;
}
i++;
}
- cpu_lbr_stack(cpuc)->nr = lbr_nr;
+ lbr_stack->nr = lbr_nr;
}
void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr)
@@ -1314,8 +1325,8 @@ void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr)
else
cpu_lbr_stack(cpuc)->hw_idx = intel_pmu_lbr_tos();
- intel_pmu_store_lbr(cpuc, lbr);
- intel_pmu_lbr_filter(cpuc);
+ intel_pmu_store_lbr(lbr, cpu_lbr_stack(cpuc));
+ intel_pmu_lbr_filter(cpuc, cpu_lbr_stack(cpuc), cpuc->lbr_counters);
}
/*
diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h
index e3d625a6d7eb..fa0af528cd1e 100644
--- a/arch/x86/events/perf_event.h
+++ b/arch/x86/events/perf_event.h
@@ -594,13 +594,13 @@ static __always_inline struct perf_branch_stack *cpu_lbr_stack(struct cpu_hw_eve
return (struct perf_branch_stack *)cpuc->__lbr_stack;
}
-static __always_inline void cpu_init_lbr_stack(struct cpu_hw_events *cpuc, int nr)
+static __always_inline void cpu_init_lbr_stack(struct perf_branch_stack *lbr_stack, int nr)
{
/*
* Initialize the branch-stack count before populating entries[]
* to avoid UBSAN complaints from the __counted_by(nr) flexible array.
*/
- cpu_lbr_stack(cpuc)->nr = nr;
+ lbr_stack->nr = nr;
}
#define __EVENT_CONSTRAINT_RANGE(c, e, n, m, w, o, f) { \
@@ -1207,7 +1207,8 @@ struct x86_pmu {
unsigned int lbr_counters:4;
void (*lbr_reset)(void);
- void (*lbr_read)(struct cpu_hw_events *cpuc);
+ void (*lbr_read)(struct cpu_hw_events *cpuc,
+ struct perf_branch_stack *lbr_stack);
void (*lbr_save)(void *ctx);
void (*lbr_restore)(void *ctx);
@@ -2007,11 +2008,14 @@ void intel_pmu_lbr_enable_all(bool pmi);
void intel_pmu_lbr_disable_all(void);
-void intel_pmu_lbr_read(void);
+void intel_pmu_lbr_read(struct perf_branch_stack *lbr_stack,
+ u64 *lbr_counters);
-void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc);
+void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc,
+ struct perf_branch_stack *lbr_stack);
-void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc);
+void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc,
+ struct perf_branch_stack *lbr_stack);
void intel_pmu_lbr_save(void *ctx);
--
2.34.1
next prev parent reply other threads:[~2026-11-06 4:41 UTC|newest]
Thread overview: 22+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-06 4:32 [Patch v6 00/20] perf/x86: Fix multiple LBR sampling issues Dapeng Mi
2026-11-06 4:32 ` [Patch v6 01/20] perf/x86: Disable LBR if x86_pmu.lbr_nr exceeds MAX_LBR_ENTRIES Dapeng Mi
2026-11-06 4:32 ` [Patch v6 02/20] perf/x86/intel: Validate LBR XSAVE state before updating LBR stack Dapeng Mi
2026-11-06 4:32 ` [Patch v6 03/20] perf/x86/amd/lbr: Limit branch snapshots to valid entries Dapeng Mi
2026-11-06 4:32 ` [Patch v6 04/20] perf/x86/intel/lbr: " Dapeng Mi
2026-11-06 4:32 ` [Patch v6 05/20] perf/x86: Add lbr_enabled to track LBR state Dapeng Mi
2026-11-06 4:32 ` [Patch v6 06/20] perf/x86/amd: Avoid re-enabling PMU and LBR for BPF when disabled Dapeng Mi
2026-11-06 4:32 ` [Patch v6 07/20] perf/x86/intel: Add helper to enable PMU without LBR Dapeng Mi
2026-11-06 4:32 ` [Patch v6 08/20] perf/x86/intel: Avoid re-enabling PMU and LBR for BPF when disabled Dapeng Mi
2026-11-06 4:32 ` [Patch v6 09/20] perf/x86/intel: Keep LBR frozen throughout PMI handling Dapeng Mi
2026-11-06 4:32 ` [Patch v6 10/20] perf: Add counted-bounds tracking for perf_branch_stack Dapeng Mi
2026-11-06 4:32 ` Dapeng Mi [this message]
2026-11-06 4:32 ` [Patch v6 12/20] perf/x86/intel: Use a dedicated LBR stack for BPF snapshotting Dapeng Mi
2026-11-06 4:32 ` [Patch v6 13/20] perf/x86/intel: Always snapshot live hardware LBR state for BPF Dapeng Mi
2026-11-06 4:32 ` [Patch v6 14/20] perf/x86/amd: Refresh LBR stack in PMI only when LBR users are active Dapeng Mi
2026-11-06 4:32 ` [Patch v6 15/20] perf/x86/intel: Refresh LBR stack for non-PEBS LBR events when needed Dapeng Mi
2026-11-06 4:32 ` [Patch v6 16/20] perf/x86/intel: Remove dead lbr_pebs_users Dapeng Mi
2026-11-06 4:32 ` [Patch v6 17/20] perf/x86/intel: Clear raw LBR counters in the LBR stack Dapeng Mi
2026-11-06 4:32 ` [Patch v6 18/20] perf/x86/intel: Refill LBR stack for LBR counter events in PMI handler Dapeng Mi
2026-11-06 4:32 ` [Patch v6 19/20] perf/x86/intel: Pass LBR stack size into store_lbr() Dapeng Mi
2026-11-06 4:32 ` [Patch v6 20/20] perf/core: Don't overwrite shared LBR stack depth Dapeng Mi
2026-11-06 7:23 ` [Patch v6 00/20] perf/x86: Fix multiple LBR sampling issues Mi, Dapeng
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261106043257.842864-12-dapeng1.mi@linux.intel.com \
--to=dapeng1.mi@linux.intel.com \
--cc=acme@kernel.org \
--cc=adrian.hunter@intel.com \
--cc=ak@linux.intel.com \
--cc=alexander.shishkin@linux.intel.com \
--cc=ananth.narayan@amd.com \
--cc=andrii@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=dapeng1.mi@intel.com \
--cc=eranian@google.com \
--cc=irogers@google.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-perf-users@vger.kernel.org \
--cc=mingo@redhat.com \
--cc=namhyung@kernel.org \
--cc=peterz@infradead.org \
--cc=ravi.bangoria@amd.com \
--cc=sandipan.das@amd.com \
--cc=song@kernel.org \
--cc=thomas.falcon@intel.com \
--cc=xudong.hao@intel.com \
--cc=zide.chen@intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®