From: Ian Rogers <irogers@google.com>
To: Peter Zijlstra <peterz@infradead.org>,
Ingo Molnar <mingo@redhat.com>,
Arnaldo Carvalho de Melo <acme@kernel.org>,
Namhyung Kim <namhyung@kernel.org>,
Mark Rutland <mark.rutland@arm.com>,
Alexander Shishkin <alexander.shishkin@linux.intel.com>,
Jiri Olsa <jolsa@kernel.org>, Ian Rogers <irogers@google.com>,
Adrian Hunter <adrian.hunter@intel.com>,
James Clark <james.clark@linaro.org>
Cc: Alexei Starovoitov <ast@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
Andrii Nakryiko <andrii@kernel.org>,
Martin KaFai Lau <martin.lau@linux.dev>,
Eduard Zingerman <eddyz87@gmail.com>, Song Liu <song@kernel.org>,
Yonghong Song <yonghong.song@linux.dev>,
Ihor Solodrai <ihor.solodrai@linux.dev>,
Leo Yan <leo.yan@linux.dev>, Dmitry Vyukov <dvyukov@google.com>,
"Steinar H . Gunderson" <sesse@google.com>,
Stephen Brennan <stephen.s.brennan@oracle.com>,
Thomas Falcon <thomas.falcon@intel.com>,
linux-perf-users@vger.kernel.org, bpf@vger.kernel.org,
linux-kernel@vger.kernel.org
Subject: [PATCH v3 06/19] perf/core: Support deferred unwinding for BUILD_ID_OFFSET sample types
Date: Sun, 15 Nov 2026 23:44:01 -0800 [thread overview]
Message-ID: <20261116074414.3174737-7-irogers@google.com> (raw)
In-Reply-To: <20261116074414.3174737-1-irogers@google.com>
When attr.defer_callchain is set on events using
PERF_SAMPLE_BUILD_ID_OFFSET or PERF_SAMPLE_CALLCHAIN_BUILD_ID_OFFSET,
request a deferred user-space unwind via unwind_deferred_request() and
emit a PERF_CONTEXT_USER_DEFERRED marker and cookie in the sample.
In the faultable task_work callback (perf_unwind_deferred_callback()),
resolve the unwound user IPs to build IDs and file offsets using
stack_map_get_build_id_offset(..., may_fault=true), encode them into the
compressed build ID + offset format, and emit
PERF_RECORD_CALLCHAIN_DEFERRED with PERF_RECORD_MISC_MMAP_BUILD_ID set.
Suggested-by: Namhyung Kim <namhyung@kernel.org>
Signed-off-by: Ian Rogers <irogers@google.com>
---
kernel/events/core.c | 175 ++++++++++++++++++++++++++++++++++++++++---
1 file changed, 164 insertions(+), 11 deletions(-)
diff --git a/kernel/events/core.c b/kernel/events/core.c
index d38ffe52c301..ef40c4442e82 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -460,6 +460,8 @@ static atomic_t nr_bpf_events __read_mostly;
static atomic_t nr_cgroup_events __read_mostly;
static atomic_t nr_text_poke_events __read_mostly;
static atomic_t nr_build_id_events __read_mostly;
+static atomic_t nr_defer_output_build_id_events __read_mostly;
+static atomic_t nr_defer_output_callchain_build_id_events __read_mostly;
static LIST_HEAD(pmus);
static DEFINE_MUTEX(pmus_lock);
@@ -5639,6 +5641,25 @@ static void detach_perf_ctx_data(struct perf_event *event)
}
}
+static bool has_defer_output_callchain_build_id_offset(const struct perf_event *event)
+{
+ return event->attr.defer_output &&
+ (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN_BUILD_ID_OFFSET) &&
+ !(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) &&
+ !event->attr.exclude_callchain_user;
+}
+
+static bool has_defer_output_build_id_offset(const struct perf_event *event)
+{
+ if (!event->attr.defer_output)
+ return false;
+ if (has_defer_output_callchain_build_id_offset(event))
+ return true;
+ return (event->attr.sample_type & PERF_SAMPLE_BUILD_ID_OFFSET) &&
+ (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) ||
+ event->attr.exclude_callchain_user);
+}
+
static void unaccount_event(struct perf_event *event)
{
bool dec = false;
@@ -5652,6 +5673,10 @@ static void unaccount_event(struct perf_event *event)
atomic_dec(&nr_mmap_events);
if (event->attr.build_id)
atomic_dec(&nr_build_id_events);
+ if (has_defer_output_build_id_offset(event))
+ atomic_dec(&nr_defer_output_build_id_events);
+ if (has_defer_output_callchain_build_id_offset(event))
+ atomic_dec(&nr_defer_output_callchain_build_id_events);
if (event->attr.comm)
atomic_dec(&nr_comm_events);
if (event->attr.namespaces)
@@ -8399,6 +8424,8 @@ static void perf_output_read(struct perf_output_handle *handle,
perf_output_read_one(handle, event, enabled, running);
}
+static struct unwind_work perf_unwind_work;
+
static void perf_bpf_to_sample_bid_offset(struct bpf_stack_build_id *bpf_bid)
{
struct perf_sample_build_id_offset *bid = (void *)bpf_bid;
@@ -8434,7 +8461,28 @@ static void perf_output_sample_build_id(struct perf_output_handle *handle,
PERF_RECORD_MISC_USER;
if (sample_bid) {
- if (is_user && has_user_ctx) {
+ u64 defer_cookie = 0;
+ bool defer_user = IS_ENABLED(CONFIG_UNWIND_USER) &&
+ is_user && has_user_ctx &&
+ event->attr.defer_callchain &&
+ (!(sample_type & PERF_SAMPLE_CALLCHAIN) ||
+ event->attr.exclude_callchain_user) &&
+ data->ip == instruction_pointer(task_pt_regs(current));
+
+ if (defer_user &&
+ unwind_deferred_request(&perf_unwind_work, &defer_cookie) < 0)
+ defer_cookie = 0;
+
+ if (defer_cookie) {
+ u64 deferred_bid[4] = {
+ 0,
+ PERF_CONTEXT_USER_DEFERRED,
+ defer_cookie,
+ data->ip,
+ };
+
+ perf_output_put(handle, deferred_bid);
+ } else if (is_user && has_user_ctx) {
if (callchain_bid && data->callchain_build_id &&
data->callchain_build_id->has_ip_bid) {
perf_output_put(handle,
@@ -8811,8 +8859,6 @@ static u64 perf_get_page_size(unsigned long addr)
static struct perf_callchain_entry __empty_callchain = { .nr = 0, };
-static struct unwind_work perf_unwind_work;
-
struct perf_callchain_entry *
perf_callchain(struct perf_event *event, struct pt_regs *regs)
{
@@ -10635,34 +10681,117 @@ void perf_event_bpf_event(struct bpf_prog *prog,
struct perf_callchain_deferred_event {
struct unwind_stacktrace *trace;
+ struct perf_callchain_build_id_entry *bid_entry;
struct {
struct perf_event_header header;
u64 cookie;
} event;
};
+static struct perf_callchain_build_id_entry *
+perf_resolve_build_id_offsets(struct unwind_stacktrace *trace)
+{
+ struct perf_callchain_build_id_entry *entry;
+ struct bpf_stack_build_id *bpf_bids;
+ u16 nr = trace->nr;
+ u32 max_size;
+
+ if (!nr || !current->mm ||
+ !atomic_read(&nr_defer_output_build_id_events))
+ return NULL;
+
+ if (!atomic_read(&nr_defer_output_callchain_build_id_events))
+ nr = 1;
+
+ entry = kzalloc(sizeof(*entry) + nr * sizeof(u64), GFP_KERNEL);
+ if (!entry)
+ return NULL;
+
+ bpf_bids = kcalloc(nr, sizeof(*bpf_bids), GFP_KERNEL);
+ if (!bpf_bids) {
+ kfree(entry);
+ return NULL;
+ }
+
+ for (u16 i = 0; i < nr; i++) {
+ entry->offsets[i] = trace->entries[i];
+ bpf_bids[i].ip = trace->entries[i];
+ }
+
+ stack_map_get_build_id_offset(bpf_bids, nr, true, true);
+
+ max_size = round_down(U16_MAX - sizeof(struct perf_event_header) -
+ 2 * sizeof(u64), sizeof(u64));
+ perf_callchain_encode_build_ids(entry, entry->offsets, nr, 0,
+ bpf_bids, nr, max_size);
+ kfree(bpf_bids);
+ return entry;
+}
+
static void perf_callchain_deferred_output(struct perf_event *event, void *data)
{
struct perf_callchain_deferred_event *deferred_event = data;
+ struct perf_callchain_build_id_entry *bid_entry = deferred_event->bid_entry;
struct perf_output_handle handle;
struct perf_sample_data sample;
u64 nr = deferred_event->trace->nr;
- u16 size, id_size, max_payload;
+ u32 max_offsets = 0, nr_build_ids = 0, nr_offsets = 0;
+ u32 raw_bids_len = 0, aligned_bids_len = 0;
+ bool use_bid;
+ u16 size, misc, id_size, max_payload;
int ret;
if (!event->attr.defer_output)
return;
+ use_bid = has_defer_output_build_id_offset(event);
+ if (use_bid && bid_entry) {
+ if (has_defer_output_callchain_build_id_offset(event))
+ max_offsets = bid_entry->nr_offsets;
+ else
+ max_offsets = min_t(u32, bid_entry->nr_offsets, 1);
+ }
+
id_size = event->attr.sample_id_all ? event->id_header_size : 0;
max_payload = round_down(U16_MAX - sizeof(deferred_event->event) -
sizeof(u64) - id_size, sizeof(u64));
- nr = min_t(u64, nr, max_payload / sizeof(u64));
- size = sizeof(deferred_event->event) + sizeof(u64) + nr * sizeof(u64);
+
+ if (use_bid) {
+ for (u32 i = 0; i < max_offsets; i++) {
+ u8 bid_idx = bid_entry->offsets[i] >> 56;
+ u32 next_nr_bids = nr_build_ids;
+ u32 next_raw_len = raw_bids_len;
+ u32 next_aligned_len;
+
+ if (bid_idx > 0 && bid_idx <= bid_entry->nr_build_ids &&
+ bid_idx > next_nr_bids) {
+ next_nr_bids = bid_idx;
+ next_raw_len = next_nr_bids *
+ (1 + BUILD_ID_SIZE_MAX);
+ }
+ next_aligned_len = ALIGN(next_raw_len, sizeof(u64));
+ if (next_aligned_len + (i + 1) * sizeof(u64) > max_payload)
+ break;
+ nr_build_ids = next_nr_bids;
+ raw_bids_len = next_raw_len;
+ aligned_bids_len = next_aligned_len;
+ nr_offsets = i + 1;
+ }
+ size = sizeof(deferred_event->event) + sizeof(u64) +
+ aligned_bids_len + nr_offsets * sizeof(u64);
+ } else {
+ nr = min_t(u64, nr, max_payload / sizeof(u64));
+ size = sizeof(deferred_event->event) + sizeof(u64) +
+ nr * sizeof(u64);
+ }
+
+ misc = PERF_RECORD_MISC_USER |
+ (use_bid ? PERF_RECORD_MISC_MMAP_BUILD_ID : 0);
/* XXX do we really need sample_id_all for this ??? */
perf_event_header__init(&deferred_event->event.header, &sample,
PERF_RECORD_CALLCHAIN_DEFERRED,
- PERF_RECORD_MISC_USER,
+ misc,
size,
event);
@@ -10672,11 +10801,27 @@ static void perf_callchain_deferred_output(struct perf_event *event, void *data)
return;
perf_output_put(&handle, deferred_event->event);
- perf_output_put(&handle, nr);
- for (u64 i = 0; i < nr; i++) {
- u64 entry = deferred_event->trace->entries[i];
+ if (use_bid) {
+ u64 pad_zero = 0;
+ u32 pad = aligned_bids_len - raw_bids_len;
+
+ perf_output_put(&handle, nr_build_ids);
+ perf_output_put(&handle, nr_offsets);
+ if (raw_bids_len)
+ __output_copy(&handle, bid_entry->build_ids, raw_bids_len);
+ if (pad)
+ __output_copy(&handle, &pad_zero, pad);
+ if (nr_offsets) {
+ __output_copy(&handle, bid_entry->offsets,
+ nr_offsets * sizeof(u64));
+ }
+ } else {
+ perf_output_put(&handle, nr);
+ for (u64 i = 0; i < nr; i++) {
+ u64 entry = deferred_event->trace->entries[i];
- perf_output_put(&handle, entry);
+ perf_output_put(&handle, entry);
+ }
}
perf_event__output_id_sample(event, &handle, &sample);
@@ -10686,14 +10831,18 @@ static void perf_callchain_deferred_output(struct perf_event *event, void *data)
static void perf_unwind_deferred_callback(struct unwind_work *work,
struct unwind_stacktrace *trace, u64 cookie)
{
+ struct perf_callchain_build_id_entry *bid_entry =
+ perf_resolve_build_id_offsets(trace);
struct perf_callchain_deferred_event deferred_event = {
.trace = trace,
+ .bid_entry = bid_entry,
.event = {
.cookie = cookie,
},
};
perf_iterate_sb(perf_callchain_deferred_output, &deferred_event, NULL);
+ kfree(bid_entry);
}
struct perf_text_poke_event {
@@ -13552,6 +13701,10 @@ static void account_event(struct perf_event *event)
atomic_inc(&nr_mmap_events);
if (event->attr.build_id)
atomic_inc(&nr_build_id_events);
+ if (has_defer_output_build_id_offset(event))
+ atomic_inc(&nr_defer_output_build_id_events);
+ if (has_defer_output_callchain_build_id_offset(event))
+ atomic_inc(&nr_defer_output_callchain_build_id_events);
if (event->attr.comm)
atomic_inc(&nr_comm_events);
if (event->attr.namespaces)
--
2.56.1.703.gf848ed807c-goog
next prev parent reply other threads:[~2026-11-16 7:45 UTC|newest]
Thread overview: 20+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-16 7:43 [PATCH v3 00/19] perf/core, perf/tools: Add PERF_SAMPLE_BUILD_ID_OFFSET support Ian Rogers
2026-11-16 7:43 ` [PATCH v3 01/19] perf/core: Protect per-CPU callchain buffer across bpf_overflow_handler() Ian Rogers
2026-11-16 7:43 ` [PATCH v3 02/19] perf/uapi: Factor perf_build_id out into its own top-level struct Ian Rogers
2026-11-16 7:43 ` [PATCH v3 03/19] perf/uapi: Add BUILD_ID_OFFSET sample types to UAPI Ian Rogers
2026-11-16 7:43 ` [PATCH v3 04/19] bpf: Export stack_map_get_build_id_offset() for perf events Ian Rogers
2026-11-16 7:44 ` [PATCH v3 05/19] perf/core: Implement BUILD_ID_OFFSET and CALLCHAIN_BUILD_ID_OFFSET sample types Ian Rogers
2026-11-16 7:44 ` Ian Rogers [this message]
2026-11-16 7:44 ` [PATCH v3 07/19] perf inject: Set ordered_events and ordering_requires_timestamps for --aslr Ian Rogers
2026-11-16 7:44 ` [PATCH v3 08/19] perf session: Avoid double byte-swapping re-parsed deferred samples Ian Rogers
2026-11-16 7:44 ` [PATCH v3 09/19] perf inject: Fix PERF_SAMPLE_READ layout and bounds checks in aslr_tool__process_sample() Ian Rogers
2026-11-16 7:44 ` [PATCH v3 10/19] tools headers UAPI: Sync linux/perf_event.h for struct perf_build_id Ian Rogers
2026-11-16 7:44 ` [PATCH v3 11/19] tools headers UAPI: Sync linux/perf_event.h for BUILD_ID_OFFSET Ian Rogers
2026-11-16 7:44 ` [PATCH v3 12/19] perf: Refactor thread map and symbol APIs to take perf_sample Ian Rogers
2026-11-16 7:44 ` [PATCH v3 13/19] perf maps: Add dsos__find_by_build_id() and maps__find_by_build_id() Ian Rogers
2026-11-16 7:44 ` [PATCH v3 14/19] perf evsel: Parse, synthesize, and dump BUILD_ID_OFFSET sample types Ian Rogers
2026-11-16 7:44 ` [PATCH v3 15/19] perf symbol: Resolve symbols and callchains using build ID and offset Ian Rogers
2026-11-16 7:44 ` [PATCH v3 16/19] perf inject: Add --sample-buildids to convert samples to build ID + offset Ian Rogers
2026-11-16 7:44 ` [PATCH v3 17/19] perf record: Add --buildid-offset option Ian Rogers
2026-11-16 7:44 ` [PATCH v3 18/19] perf test: Add BUILD_ID_OFFSET C unit tests Ian Rogers
2026-11-16 7:44 ` [PATCH v3 19/19] perf test: Add inject --sample-buildids and record --buildid-offset shell tests Ian Rogers
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261116074414.3174737-7-irogers@google.com \
--to=irogers@google.com \
--cc=acme@kernel.org \
--cc=adrian.hunter@intel.com \
--cc=alexander.shishkin@linux.intel.com \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=dvyukov@google.com \
--cc=eddyz87@gmail.com \
--cc=ihor.solodrai@linux.dev \
--cc=james.clark@linaro.org \
--cc=jolsa@kernel.org \
--cc=leo.yan@linux.dev \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-perf-users@vger.kernel.org \
--cc=mark.rutland@arm.com \
--cc=martin.lau@linux.dev \
--cc=mingo@redhat.com \
--cc=namhyung@kernel.org \
--cc=peterz@infradead.org \
--cc=sesse@google.com \
--cc=song@kernel.org \
--cc=stephen.s.brennan@oracle.com \
--cc=thomas.falcon@intel.com \
--cc=yonghong.song@linux.dev \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®