[Patch v6 15/20] perf/x86/intel: Refresh LBR stack for non-PEBS LBR events when needed

[Dapeng Mi <dapeng1.mi@linux.intel.com>]

PEBS and non-PEBS events share the same LBR stack in cpuc->__lbr_stack.
In the PMI handler, the non-PEBS path refreshes the LBR state before the
PEBS buffer is drained.

When PEBS and non-PEBS LBR events overflow in the same PMI, draining the
PEBS buffer can overwrite the shared LBR stack with the PEBS snapshot.
This leaves stale LBR data from the PEBS record in place, so userspace
can see the wrong branch stack instead of the live LBR state at PMI
entry.

Track whether the LBR state has already been refreshed with a
lbr_refreshed flag. If PEBS events overwrite the shared LBR stack before
the non-PEBS LBR event is processed, clear the flag so the live LBR state
is refreshed again. Remove the unconditional LBR refresh at PMI entry and
initialize the flag as false so the stack is refreshed only when an active
non-PEBS LBR event requires it. Since the LBR state is frozen while the
PMI is being handled, it only needs to be read once for multiple non-PEBS
LBR events in the same interrupt.

Fixes: c22497f5838c ("perf/x86/intel: Support adaptive PEBS v4")
Signed-off-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
---
 arch/x86/events/intel/core.c | 40 +++++++++++++++++++++++++++---------
 1 file changed, 30 insertions(+), 10 deletions(-)

diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
index fbde874af618..b267c68f9467 100644
--- a/arch/x86/events/intel/core.c
+++ b/arch/x86/events/intel/core.c
@@ -3864,8 +3864,9 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
 {
 	struct perf_sample_data data;
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
-	int bit;
+	bool lbr_refreshed = false;
 	int handled = 0;
+	int bit;
 
 	inc_perf_irq_stat();
 
@@ -3906,6 +3907,15 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
 	if (__test_and_clear_bit(GLOBAL_STATUS_BUFFER_OVF_BIT, (unsigned long *)&status)) {
 		u64 pebs_enabled = cpuc->pebs_enabled;
 
+		/*
+		 * Prior to adaptive PEBS, PEBS does not support LBR snapshotting.
+		 * Refresh the LBR stack explicitly when LBR users are present.
+		 */
+		if (cpuc->lbr_users > 0 && !x86_pmu.intel_cap.pebs_baseline) {
+			intel_pmu_lbr_read(cpu_lbr_stack(cpuc), cpuc->lbr_counters);
+			lbr_refreshed = true;
+		}
+
 		handled += x86_pmu_handle_guest_pebs(regs, &data);
 		handled += static_call(x86_pmu_drain_pebs)(regs, &data);
 		/* Ensure no "suspicious NMI" warning for empty PEBS buffer. */
@@ -4003,8 +4013,19 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
 		 *
 		 * The PEBS buffer has to be drained before handling the A-PMI
 		 */
-		if (is_pebs_counter_event_group(event))
+		if (is_pebs_counter_event_group(event)) {
+			/*
+			 * PEBS counter sampling is only supported on adaptive PEBS
+			 * and arch-PEBS, both of which snapshot the LBR stack.
+			 * No explicit LBR refresh is needed here.
+			 */
 			static_call(x86_pmu_drain_pebs)(regs, &data);
+			/*
+			 * Draining the PEBS buffer can overwrite the shared
+			 * LBR stack, so mark lbr_refreshed as false.
+			 */
+			lbr_refreshed = false;
+		}
 
 		last_period = event->hw.last_period;
 
@@ -4013,8 +4034,14 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
 
 		perf_sample_data_init(&data, 0, last_period);
 
-		if (has_branch_stack(event))
+		if (has_branch_stack(event)) {
+			if (!lbr_refreshed) {
+				intel_pmu_lbr_read(cpu_lbr_stack(cpuc),
+						   cpuc->lbr_counters);
+				lbr_refreshed = true;
+			}
 			intel_pmu_lbr_save_brstack(&data, cpuc, event);
+		}
 
 		x86_pmu_clear_perf_regs(regs);
 		x86_pmu_update_perf_regs(event, &data, regs, false);
@@ -4070,13 +4097,6 @@ static int intel_pmu_handle_irq(struct pt_regs *regs)
 
 	loops = 0;
 again:
-	/*
-	 * No need to read the LBR stack when all LBR users are PEBS events,
-	 * since the PEBS handler updates it.
-	 */
-	if (cpuc->lbr_users != cpuc->lbr_pebs_users)
-		intel_pmu_lbr_read(cpu_lbr_stack(cpuc), cpuc->lbr_counters);
-
 	/*
 	 * Do not clear the LBR frozen status here; keep LBR frozen
 	 * throughout PMI handling.
-- 
2.34.1