From: Menglong Dong <menglong8.dong@gmail.com>
To: alexei.starovoitov@gmail.com, rostedt@goodmis.org, jolsa@kernel.org
Cc: bpf@vger.kernel.org, Menglong Dong <dongml2@chinatelecom.cn>,
linux-kernel@vger.kernel.org
Subject: [PATCH bpf-next 14/25] bpf: tracing: add multi-link support
Date: Wed, 28 May 2025 11:47:01 +0800 [thread overview]
Message-ID: <20250528034712.138701-15-dongml2@chinatelecom.cn> (raw)
In-Reply-To: <20250528034712.138701-1-dongml2@chinatelecom.cn>
In this commit, we add the support to allow attaching a tracing BPF
program to multi hooks, which is similar to BPF_TRACE_KPROBE_MULTI.
The use case is obvious. For now, we have to create a BPF program for each
kernel function, for which we want to trace, even through all the program
have the same (or similar logic). This can consume extra memory, and make
the program loading slow if we have plenty of kernel function to trace.
The KPROBE_MULTI maybe a alternative, but it can't do what TRACING do. For
example, the kretprobe can't obtain the function args, but the FEXIT can.
For now, we support to create multi-link for fentry/fexit/modify_return
with the following new attach types that we introduce:
BPF_TRACE_FENTRY_MULTI
BPF_TRACE_FEXIT_MULTI
BPF_MODIFY_RETURN_MULTI
We introduce the struct bpf_tracing_multi_link for this purpose, which
can hold all the kernel modules, target bpf program (for attaching to bpf
program) or target btf (for attaching to kernel function) that we
referenced.
During loading, the first target is used for verification by the verifer.
And during attaching, we check the consistency of all the targets with
the first target.
Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
---
include/linux/bpf.h | 11 +
include/linux/bpf_types.h | 1 +
include/uapi/linux/bpf.h | 10 +
kernel/bpf/btf.c | 5 +
kernel/bpf/syscall.c | 365 +++++++++++++++++++++++++++++++++
kernel/bpf/trampoline.c | 7 +-
kernel/bpf/verifier.c | 25 ++-
net/bpf/test_run.c | 3 +
net/core/bpf_sk_storage.c | 2 +
tools/include/uapi/linux/bpf.h | 10 +
10 files changed, 435 insertions(+), 4 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 080bb966d026..7191ad25d519 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1802,6 +1802,17 @@ struct bpf_raw_tp_link {
u64 cookie;
};
+struct bpf_tracing_multi_link {
+ struct bpf_gtramp_link link;
+ enum bpf_attach_type attach_type;
+ struct bpf_prog **tgt_progs;
+ struct btf **tgt_btfs;
+ struct module **mods;
+ u32 prog_cnt;
+ u32 btf_cnt;
+ u32 mods_cnt;
+};
+
struct bpf_link_primer {
struct bpf_link *link;
struct file *file;
diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h
index fa78f49d4a9a..139d5436ce4c 100644
--- a/include/linux/bpf_types.h
+++ b/include/linux/bpf_types.h
@@ -154,3 +154,4 @@ BPF_LINK_TYPE(BPF_LINK_TYPE_PERF_EVENT, perf)
BPF_LINK_TYPE(BPF_LINK_TYPE_KPROBE_MULTI, kprobe_multi)
BPF_LINK_TYPE(BPF_LINK_TYPE_STRUCT_OPS, struct_ops)
BPF_LINK_TYPE(BPF_LINK_TYPE_UPROBE_MULTI, uprobe_multi)
+BPF_LINK_TYPE(BPF_LINK_TYPE_TRACING_MULTI, tracing_multi)
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 16e95398c91c..45dfaf40230e 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -1120,6 +1120,9 @@ enum bpf_attach_type {
BPF_NETKIT_PEER,
BPF_TRACE_KPROBE_SESSION,
BPF_TRACE_UPROBE_SESSION,
+ BPF_TRACE_FENTRY_MULTI,
+ BPF_TRACE_FEXIT_MULTI,
+ BPF_MODIFY_RETURN_MULTI,
__MAX_BPF_ATTACH_TYPE
};
@@ -1144,6 +1147,7 @@ enum bpf_link_type {
BPF_LINK_TYPE_UPROBE_MULTI = 12,
BPF_LINK_TYPE_NETKIT = 13,
BPF_LINK_TYPE_SOCKMAP = 14,
+ BPF_LINK_TYPE_TRACING_MULTI = 15,
__MAX_BPF_LINK_TYPE,
};
@@ -1765,6 +1769,12 @@ union bpf_attr {
*/
__u64 cookie;
} tracing;
+ struct {
+ __u32 cnt;
+ __aligned_u64 tgt_fds;
+ __aligned_u64 btf_ids;
+ __aligned_u64 cookies;
+ } tracing_multi;
struct {
__u32 pf;
__u32 hooknum;
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 64538625ee91..03045f7d428f 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -6100,6 +6100,9 @@ static int btf_validate_prog_ctx_type(struct bpf_verifier_log *log, const struct
case BPF_TRACE_FENTRY:
case BPF_TRACE_FEXIT:
case BPF_MODIFY_RETURN:
+ case BPF_TRACE_FENTRY_MULTI:
+ case BPF_TRACE_FEXIT_MULTI:
+ case BPF_MODIFY_RETURN_MULTI:
/* allow u64* as ctx */
if (btf_is_int(t) && t->size == 8)
return 0;
@@ -6705,6 +6708,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
fallthrough;
case BPF_LSM_CGROUP:
case BPF_TRACE_FEXIT:
+ case BPF_TRACE_FEXIT_MULTI:
/* When LSM programs are attached to void LSM hooks
* they use FEXIT trampolines and when attached to
* int LSM hooks, they use MODIFY_RETURN trampolines.
@@ -6723,6 +6727,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
t = btf_type_by_id(btf, t->type);
break;
case BPF_MODIFY_RETURN:
+ case BPF_MODIFY_RETURN_MULTI:
/* For now the BPF_MODIFY_RETURN can only be attached to
* functions that return an int.
*/
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 60865a27d7d3..0cd989381128 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -37,6 +37,7 @@
#include <linux/trace_events.h>
#include <linux/tracepoint.h>
#include <linux/overflow.h>
+#include <linux/kfunc_md.h>
#include <net/netfilter/nf_bpf_link.h>
#include <net/netkit.h>
@@ -3466,6 +3467,34 @@ static const struct bpf_link_ops bpf_tracing_link_lops = {
.fill_link_info = bpf_tracing_link_fill_link_info,
};
+static int bpf_tracing_check_multi(struct bpf_prog *prog,
+ struct bpf_prog *tgt_prog,
+ struct btf *btf2,
+ const struct btf_type *t2)
+{
+ const struct btf_type *t1;
+ struct btf *btf1;
+
+ /* this case is already valided in bpf_check_attach_target() */
+ if (prog->type == BPF_PROG_TYPE_EXT)
+ return 0;
+
+ btf1 = prog->aux->dst_prog ? prog->aux->dst_prog->aux->btf :
+ prog->aux->attach_btf;
+ if (!btf1)
+ return -EOPNOTSUPP;
+
+ btf2 = btf2 ?: tgt_prog->aux->btf;
+ t1 = prog->aux->attach_func_proto;
+
+ /* the target is the same as the origin one, this is a re-attach */
+ if (t1 == t2)
+ return 0;
+
+ return btf_check_func_part_match(btf1, t1, btf2, t2,
+ prog->aux->accessed_args);
+}
+
static int bpf_tracing_prog_attach(struct bpf_prog *prog,
int tgt_prog_fd,
u32 btf_id,
@@ -3665,6 +3694,335 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
return err;
}
+static void __bpf_tracing_multi_link_release(struct bpf_tracing_multi_link *link)
+{
+ int i;
+
+ if (link->mods_cnt) {
+ for (i = 0; i < link->mods_cnt; i++)
+ module_put(link->mods[i]);
+ kfree(link->mods);
+ }
+
+ if (link->prog_cnt) {
+ for (i = 0; i < link->prog_cnt; i++)
+ bpf_prog_put(link->tgt_progs[i]);
+ kfree(link->tgt_progs);
+ }
+
+ if (link->btf_cnt) {
+ for (i = 0; i < link->btf_cnt; i++)
+ btf_put(link->tgt_btfs[i]);
+ kfree(link->tgt_btfs);
+ }
+
+ kfree(link->link.entries);
+}
+
+static void bpf_tracing_multi_link_release(struct bpf_link *link)
+{
+ struct bpf_tracing_multi_link *multi_link =
+ container_of(link, struct bpf_tracing_multi_link, link.link);
+
+ bpf_gtrampoline_unlink_prog(&multi_link->link);
+ __bpf_tracing_multi_link_release(multi_link);
+}
+
+static void bpf_tracing_multi_link_dealloc(struct bpf_link *link)
+{
+ struct bpf_tracing_multi_link *tr_link =
+ container_of(link, struct bpf_tracing_multi_link, link.link);
+
+ kfree(tr_link);
+}
+
+static void bpf_tracing_multi_link_show_fdinfo(const struct bpf_link *link,
+ struct seq_file *seq)
+{
+ struct bpf_tracing_multi_link *tr_link =
+ container_of(link, struct bpf_tracing_multi_link, link.link);
+ int i;
+
+ for (i = 0; i < tr_link->link.entry_cnt; i++) {
+ seq_printf(seq,
+ "attach_type:\t%d\n"
+ "target_addr:\t%p\n",
+ tr_link->attach_type,
+ tr_link->link.entries[i].addr);
+ }
+}
+
+static const struct bpf_link_ops bpf_tracing_multi_link_lops = {
+ .release = bpf_tracing_multi_link_release,
+ .dealloc = bpf_tracing_multi_link_dealloc,
+ .show_fdinfo = bpf_tracing_multi_link_show_fdinfo,
+};
+
+#define MAX_TRACING_MULTI_CNT 102400
+
+static int bpf_tracing_get_target(u32 fd, struct bpf_prog **tgt_prog,
+ struct btf **tgt_btf)
+{
+ struct bpf_prog *prog = NULL;
+ struct btf *btf = NULL;
+ int err = 0;
+
+ if (fd) {
+ prog = bpf_prog_get(fd);
+ if (!IS_ERR(prog))
+ goto found;
+
+ prog = NULL;
+ /* "fd" is the fd of the kernel module BTF */
+ btf = btf_get_by_fd(fd);
+ if (IS_ERR(btf)) {
+ err = PTR_ERR(btf);
+ goto err;
+ }
+ if (!btf_is_kernel(btf)) {
+ btf_put(btf);
+ err = -EOPNOTSUPP;
+ goto err;
+ }
+ } else {
+ btf = bpf_get_btf_vmlinux();
+ if (IS_ERR(btf)) {
+ err = PTR_ERR(btf);
+ goto err;
+ }
+ if (!btf) {
+ err = -EINVAL;
+ goto err;
+ }
+ btf_get(btf);
+ }
+found:
+ *tgt_prog = prog;
+ *tgt_btf = btf;
+ return 0;
+err:
+ *tgt_prog = NULL;
+ *tgt_btf = NULL;
+ return err;
+}
+
+static int bpf_tracing_multi_link_check(const union bpf_attr *attr, u32 **btf_ids,
+ u32 **tgt_fds, u64 **cookies,
+ u32 cnt)
+{
+ void __user *ubtf_ids;
+ void __user *utgt_fds;
+ void __user *ucookies;
+ void *tmp;
+ int i;
+
+ if (!cnt)
+ return -EINVAL;
+
+ if (cnt > MAX_TRACING_MULTI_CNT)
+ return -E2BIG;
+
+ ucookies = u64_to_user_ptr(attr->link_create.tracing_multi.cookies);
+ if (ucookies) {
+ tmp = kvmalloc_array(cnt, sizeof(**cookies), GFP_KERNEL);
+ if (!tmp)
+ return -ENOMEM;
+
+ *cookies = tmp;
+ if (copy_from_user(tmp, ucookies, cnt * sizeof(**cookies)))
+ return -EFAULT;
+ }
+
+ utgt_fds = u64_to_user_ptr(attr->link_create.tracing_multi.tgt_fds);
+ if (utgt_fds) {
+ tmp = kvmalloc_array(cnt, sizeof(**tgt_fds), GFP_KERNEL);
+ if (!tmp)
+ return -ENOMEM;
+
+ *tgt_fds = tmp;
+ if (copy_from_user(tmp, utgt_fds, cnt * sizeof(**tgt_fds)))
+ return -EFAULT;
+ }
+
+ ubtf_ids = u64_to_user_ptr(attr->link_create.tracing_multi.btf_ids);
+ if (!ubtf_ids)
+ return -EINVAL;
+
+ tmp = kvmalloc_array(cnt, sizeof(**btf_ids), GFP_KERNEL);
+ if (!tmp)
+ return -ENOMEM;
+
+ *btf_ids = tmp;
+ if (copy_from_user(tmp, ubtf_ids, cnt * sizeof(**btf_ids)))
+ return -EFAULT;
+
+ for (i = 0; i < cnt; i++) {
+ if (!(*btf_ids)[i])
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static void bpf_tracing_multi_link_ptr_fill(struct bpf_tracing_multi_link *link,
+ struct ptr_array *progs,
+ struct ptr_array *mods,
+ struct ptr_array *btfs)
+{
+ link->mods = (struct module **) mods->ptrs;
+ link->mods_cnt = mods->cnt;
+ link->tgt_btfs = (struct btf **) btfs->ptrs;
+ link->btf_cnt = btfs->cnt;
+ link->tgt_progs = (struct bpf_prog **) progs->ptrs;
+ link->prog_cnt = progs->cnt;
+}
+
+static int bpf_tracing_prog_attach_multi(const union bpf_attr *attr,
+ struct bpf_prog *prog)
+{
+ struct bpf_tracing_multi_link *link = NULL;
+ u32 cnt, *btf_ids = NULL, *tgt_fds = NULL;
+ struct bpf_link_primer link_primer;
+ struct ptr_array prog_array = { };
+ struct ptr_array btf_array = { };
+ struct ptr_array mod_array = { };
+ u64 *cookies = NULL;
+ int err = 0, i;
+
+ if ((prog->expected_attach_type != BPF_TRACE_FENTRY_MULTI &&
+ prog->expected_attach_type != BPF_TRACE_FEXIT_MULTI &&
+ prog->expected_attach_type != BPF_MODIFY_RETURN_MULTI) ||
+ prog->type != BPF_PROG_TYPE_TRACING)
+ return -EINVAL;
+
+ cnt = attr->link_create.tracing_multi.cnt;
+ err = bpf_tracing_multi_link_check(attr, &btf_ids, &tgt_fds, &cookies,
+ cnt);
+ if (err)
+ goto err_out;
+
+ link = kzalloc(sizeof(*link), GFP_USER);
+ if (!link) {
+ err = -ENOMEM;
+ goto err_out;
+ }
+ link->link.entries = kzalloc(sizeof(*link->link.entries) * cnt,
+ GFP_USER);
+ if (!link->link.entries) {
+ err = -ENOMEM;
+ goto err_out;
+ }
+
+ bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING_MULTI,
+ &bpf_tracing_multi_link_lops, prog);
+ link->attach_type = prog->expected_attach_type;
+
+ mutex_lock(&prog->aux->dst_mutex);
+
+ for (i = 0; i < cnt; i++) {
+ struct bpf_attach_target_info tgt_info = {};
+ struct bpf_gtramp_link_entry *entry;
+ struct bpf_prog *tgt_prog = NULL;
+ u32 tgt_fd, btf_id = btf_ids[i];
+ struct btf *tgt_btf = NULL;
+ struct module *mod = NULL;
+ int nr_regs;
+
+ entry = &link->link.entries[i];
+ tgt_fd = tgt_fds ? tgt_fds[i] : 0;
+ err = bpf_tracing_get_target(tgt_fd, &tgt_prog, &tgt_btf);
+ if (err)
+ goto err_out_unlock;
+
+ if (tgt_prog) {
+ err = bpf_try_add_ptr(&prog_array, tgt_prog);
+ if (err) {
+ bpf_prog_put(tgt_prog);
+ if (err != -EEXIST)
+ goto err_out_unlock;
+ }
+ }
+
+ if (tgt_btf) {
+ err = bpf_try_add_ptr(&btf_array, tgt_btf);
+ if (err) {
+ btf_put(tgt_btf);
+ if (err != -EEXIST)
+ goto err_out_unlock;
+ }
+ }
+
+ prog->aux->attach_tracing_prog = tgt_prog &&
+ tgt_prog->type == BPF_PROG_TYPE_TRACING &&
+ prog->type == BPF_PROG_TYPE_TRACING;
+
+ err = bpf_check_attach_target(NULL, prog, tgt_prog, tgt_btf,
+ btf_id, &tgt_info);
+ if (err)
+ goto err_out_unlock;
+
+ nr_regs = arch_bpf_get_regs_nr(&tgt_info.fmodel);
+ if (nr_regs < 0) {
+ err = nr_regs;
+ goto err_out_unlock;
+ }
+
+ mod = tgt_info.tgt_mod;
+ if (mod) {
+ err = bpf_try_add_ptr(&mod_array, mod);
+ if (err) {
+ module_put(mod);
+ if (err != -EEXIST)
+ goto err_out_unlock;
+ }
+ }
+
+ err = bpf_tracing_check_multi(prog, tgt_prog, tgt_btf,
+ tgt_info.tgt_type);
+ if (err)
+ goto err_out_unlock;
+
+ entry->cookie = cookies ? cookies[i] : 0;
+ entry->addr = (void *)tgt_info.tgt_addr;
+ entry->tgt_prog = tgt_prog;
+ entry->attach_btf = tgt_btf;
+ entry->btf_id = btf_id;
+ entry->nr_args = nr_regs;
+
+ link->link.entry_cnt++;
+ }
+
+ err = bpf_gtrampoline_link_prog(&link->link);
+ if (err)
+ goto err_out_unlock;
+
+ err = bpf_link_prime(&link->link.link, &link_primer);
+ if (err) {
+ bpf_gtrampoline_unlink_prog(&link->link);
+ goto err_out_unlock;
+ }
+
+ bpf_tracing_multi_link_ptr_fill(link, &prog_array, &mod_array,
+ &btf_array);
+ mutex_unlock(&prog->aux->dst_mutex);
+
+ kfree(btf_ids);
+ kfree(tgt_fds);
+ kfree(cookies);
+ return bpf_link_settle(&link_primer);
+err_out_unlock:
+ bpf_tracing_multi_link_ptr_fill(link, &prog_array, &mod_array,
+ &btf_array);
+ __bpf_tracing_multi_link_release(link);
+ mutex_unlock(&prog->aux->dst_mutex);
+err_out:
+ kfree(btf_ids);
+ kfree(tgt_fds);
+ kfree(cookies);
+ kfree(link);
+ return err;
+}
+
static void bpf_raw_tp_link_release(struct bpf_link *link)
{
struct bpf_raw_tp_link *raw_tp =
@@ -4133,6 +4491,9 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type)
case BPF_TRACE_FENTRY:
case BPF_TRACE_FEXIT:
case BPF_MODIFY_RETURN:
+ case BPF_TRACE_FENTRY_MULTI:
+ case BPF_TRACE_FEXIT_MULTI:
+ case BPF_MODIFY_RETURN_MULTI:
return BPF_PROG_TYPE_TRACING;
case BPF_LSM_MAC:
return BPF_PROG_TYPE_LSM;
@@ -5439,6 +5800,10 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr)
ret = bpf_iter_link_attach(attr, uattr, prog);
else if (prog->expected_attach_type == BPF_LSM_CGROUP)
ret = cgroup_bpf_link_attach(attr, prog);
+ else if (prog->expected_attach_type == BPF_TRACE_FENTRY_MULTI ||
+ prog->expected_attach_type == BPF_TRACE_FEXIT_MULTI ||
+ prog->expected_attach_type == BPF_MODIFY_RETURN_MULTI)
+ ret = bpf_tracing_prog_attach_multi(attr, prog);
else
ret = bpf_tracing_prog_attach(prog,
attr->link_create.target_fd,
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 3d7fd59107ed..b92d1d4f1033 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -116,7 +116,9 @@ bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
return (ptype == BPF_PROG_TYPE_TRACING &&
(eatype == BPF_TRACE_FENTRY || eatype == BPF_TRACE_FEXIT ||
- eatype == BPF_MODIFY_RETURN)) ||
+ eatype == BPF_MODIFY_RETURN ||
+ eatype == BPF_TRACE_FENTRY_MULTI || eatype == BPF_TRACE_FEXIT_MULTI ||
+ eatype == BPF_MODIFY_RETURN_MULTI)) ||
(ptype == BPF_PROG_TYPE_LSM && eatype == BPF_LSM_MAC);
}
@@ -515,10 +517,13 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)
{
switch (prog->expected_attach_type) {
case BPF_TRACE_FENTRY:
+ case BPF_TRACE_FENTRY_MULTI:
return BPF_TRAMP_FENTRY;
case BPF_MODIFY_RETURN:
+ case BPF_MODIFY_RETURN_MULTI:
return BPF_TRAMP_MODIFY_RETURN;
case BPF_TRACE_FEXIT:
+ case BPF_TRACE_FEXIT_MULTI:
return BPF_TRAMP_FEXIT;
case BPF_LSM_MAC:
if (!prog->aux->attach_func_proto->type)
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 9c4e29bc98c0..bbfe8ae39f3c 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -16945,10 +16945,13 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char
switch (env->prog->expected_attach_type) {
case BPF_TRACE_FENTRY:
case BPF_TRACE_FEXIT:
+ case BPF_TRACE_FENTRY_MULTI:
+ case BPF_TRACE_FEXIT_MULTI:
range = retval_range(0, 0);
break;
case BPF_TRACE_RAW_TP:
case BPF_MODIFY_RETURN:
+ case BPF_MODIFY_RETURN_MULTI:
return 0;
case BPF_TRACE_ITER:
break;
@@ -22264,7 +22267,9 @@ static int do_misc_fixups(struct bpf_verifier_env *env)
if (prog_type == BPF_PROG_TYPE_TRACING &&
insn->imm == BPF_FUNC_get_func_ret) {
if (eatype == BPF_TRACE_FEXIT ||
- eatype == BPF_MODIFY_RETURN) {
+ eatype == BPF_MODIFY_RETURN ||
+ eatype == BPF_TRACE_FEXIT_MULTI ||
+ eatype == BPF_MODIFY_RETURN_MULTI) {
/* Load nr_args from ctx - 8 */
insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
insn_buf[1] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3);
@@ -23246,7 +23251,9 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
if (tgt_prog->type == BPF_PROG_TYPE_TRACING &&
prog_extension &&
(tgt_prog->expected_attach_type == BPF_TRACE_FENTRY ||
- tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) {
+ tgt_prog->expected_attach_type == BPF_TRACE_FEXIT ||
+ tgt_prog->expected_attach_type == BPF_TRACE_FENTRY_MULTI ||
+ tgt_prog->expected_attach_type == BPF_TRACE_FEXIT_MULTI)) {
/* Program extensions can extend all program types
* except fentry/fexit. The reason is the following.
* The fentry/fexit programs are used for performance
@@ -23345,6 +23352,9 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
case BPF_LSM_CGROUP:
case BPF_TRACE_FENTRY:
case BPF_TRACE_FEXIT:
+ case BPF_MODIFY_RETURN_MULTI:
+ case BPF_TRACE_FENTRY_MULTI:
+ case BPF_TRACE_FEXIT_MULTI:
if (!btf_type_is_func(t)) {
bpf_log(log, "attach_btf_id %u is not a function\n",
btf_id);
@@ -23430,7 +23440,8 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
bpf_log(log, "%s is not sleepable\n", tname);
return ret;
}
- } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) {
+ } else if (prog->expected_attach_type == BPF_MODIFY_RETURN ||
+ prog->expected_attach_type == BPF_MODIFY_RETURN_MULTI) {
if (tgt_prog) {
module_put(mod);
bpf_log(log, "can't modify return codes of BPF programs\n");
@@ -23483,6 +23494,9 @@ static bool can_be_sleepable(struct bpf_prog *prog)
case BPF_TRACE_FEXIT:
case BPF_MODIFY_RETURN:
case BPF_TRACE_ITER:
+ case BPF_TRACE_FENTRY_MULTI:
+ case BPF_TRACE_FEXIT_MULTI:
+ case BPF_MODIFY_RETURN_MULTI:
return true;
default:
return false;
@@ -23557,6 +23571,11 @@ static int check_attach_btf_id(struct bpf_verifier_env *env)
return bpf_iter_prog_supported(prog);
}
+ if (prog->expected_attach_type == BPF_TRACE_FENTRY_MULTI ||
+ prog->expected_attach_type == BPF_TRACE_FEXIT_MULTI ||
+ prog->expected_attach_type == BPF_MODIFY_RETURN_MULTI)
+ return 0;
+
key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id);
tr = bpf_trampoline_get(key, &tgt_info);
if (!tr)
diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c
index aaf13a7d58ed..bcb609425fda 100644
--- a/net/bpf/test_run.c
+++ b/net/bpf/test_run.c
@@ -696,6 +696,8 @@ int bpf_prog_test_run_tracing(struct bpf_prog *prog,
switch (prog->expected_attach_type) {
case BPF_TRACE_FENTRY:
case BPF_TRACE_FEXIT:
+ case BPF_TRACE_FENTRY_MULTI:
+ case BPF_TRACE_FEXIT_MULTI:
if (bpf_fentry_test1(1) != 2 ||
bpf_fentry_test2(2, 3) != 5 ||
bpf_fentry_test3(4, 5, 6) != 15 ||
@@ -709,6 +711,7 @@ int bpf_prog_test_run_tracing(struct bpf_prog *prog,
goto out;
break;
case BPF_MODIFY_RETURN:
+ case BPF_MODIFY_RETURN_MULTI:
ret = bpf_modify_return_test(1, &b);
if (b != 2)
side_effect++;
diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c
index 2e538399757f..c5b1fd714b58 100644
--- a/net/core/bpf_sk_storage.c
+++ b/net/core/bpf_sk_storage.c
@@ -369,6 +369,8 @@ static bool bpf_sk_storage_tracing_allowed(const struct bpf_prog *prog)
return true;
case BPF_TRACE_FENTRY:
case BPF_TRACE_FEXIT:
+ case BPF_TRACE_FENTRY_MULTI:
+ case BPF_TRACE_FEXIT_MULTI:
return !!strncmp(prog->aux->attach_func_name, "bpf_sk_storage",
strlen("bpf_sk_storage"));
default:
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 16e95398c91c..45dfaf40230e 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -1120,6 +1120,9 @@ enum bpf_attach_type {
BPF_NETKIT_PEER,
BPF_TRACE_KPROBE_SESSION,
BPF_TRACE_UPROBE_SESSION,
+ BPF_TRACE_FENTRY_MULTI,
+ BPF_TRACE_FEXIT_MULTI,
+ BPF_MODIFY_RETURN_MULTI,
__MAX_BPF_ATTACH_TYPE
};
@@ -1144,6 +1147,7 @@ enum bpf_link_type {
BPF_LINK_TYPE_UPROBE_MULTI = 12,
BPF_LINK_TYPE_NETKIT = 13,
BPF_LINK_TYPE_SOCKMAP = 14,
+ BPF_LINK_TYPE_TRACING_MULTI = 15,
__MAX_BPF_LINK_TYPE,
};
@@ -1765,6 +1769,12 @@ union bpf_attr {
*/
__u64 cookie;
} tracing;
+ struct {
+ __u32 cnt;
+ __aligned_u64 tgt_fds;
+ __aligned_u64 btf_ids;
+ __aligned_u64 cookies;
+ } tracing_multi;
struct {
__u32 pf;
__u32 hooknum;
--
2.39.5
next prev parent reply other threads:[~2025-05-28 3:50 UTC|newest]
Thread overview: 38+ messages / expand[flat|nested] mbox.gz Atom feed top
2025-05-28 3:46 [PATCH bpf-next 00/25] bpf: tracing " Menglong Dong
2025-05-28 3:46 ` [PATCH bpf-next 01/25] add per-function metadata storage support Menglong Dong
2025-05-28 3:46 ` [PATCH bpf-next 02/25] x86: implement per-function metadata storage for x86 Menglong Dong
2025-05-28 3:46 ` [PATCH bpf-next 03/25] arm64: implement per-function metadata storage for arm64 Menglong Dong
2025-05-28 12:16 ` kernel test robot
2025-05-28 3:46 ` [PATCH bpf-next 04/25] bpf: make kfunc_md support global trampoline link Menglong Dong
2025-05-28 3:46 ` [PATCH bpf-next 05/25] x86,bpf: add bpf_global_caller for global trampoline Menglong Dong
2025-05-28 3:46 ` [PATCH bpf-next 06/25] ftrace: factor out ftrace_direct_update from register_ftrace_direct Menglong Dong
2025-05-28 3:46 ` [PATCH bpf-next 07/25] ftrace: add reset_ftrace_direct_ips Menglong Dong
2025-05-28 3:46 ` [PATCH bpf-next 08/25] bpf: introduce bpf_gtramp_link Menglong Dong
2025-05-28 3:46 ` [PATCH bpf-next 09/25] bpf: tracing: add support to record and check the accessed args Menglong Dong
2025-05-28 3:46 ` [PATCH bpf-next 10/25] bpf: refactor the modules_array to ptr_array Menglong Dong
2025-05-28 3:46 ` [PATCH bpf-next 11/25] bpf: verifier: add btf to the function args of bpf_check_attach_target Menglong Dong
2025-05-28 3:46 ` [PATCH bpf-next 12/25] bpf: verifier: move btf_id_deny to bpf_check_attach_target Menglong Dong
2025-05-28 3:47 ` [PATCH bpf-next 13/25] x86,bpf: factor out __arch_get_bpf_regs_nr Menglong Dong
2025-05-28 3:47 ` Menglong Dong [this message]
2025-05-28 11:34 ` [PATCH bpf-next 14/25] bpf: tracing: add multi-link support kernel test robot
2025-05-28 3:47 ` [PATCH bpf-next 15/25] ftrace: factor out __unregister_ftrace_direct Menglong Dong
2025-05-28 12:37 ` kernel test robot
2025-05-28 3:47 ` [PATCH bpf-next 16/25] ftrace: supporting replace direct ftrace_ops Menglong Dong
2025-05-28 3:47 ` [PATCH bpf-next 17/25] bpf: make trampoline compatible with global trampoline Menglong Dong
2025-05-28 3:47 ` [PATCH bpf-next 18/25] libbpf: don't free btf if tracing_multi progs existing Menglong Dong
2025-05-28 3:47 ` [PATCH bpf-next 19/25] libbpf: support tracing_multi Menglong Dong
2025-05-28 3:47 ` [PATCH bpf-next 20/25] libbpf: add btf type hash lookup support Menglong Dong
2025-05-28 3:47 ` [PATCH bpf-next 21/25] libbpf: add skip_invalid and attach_tracing for tracing_multi Menglong Dong
2025-05-28 3:47 ` [PATCH bpf-next 22/25] selftests/bpf: use the glob_match() from libbpf in test_progs.c Menglong Dong
2025-05-28 3:47 ` [PATCH bpf-next 23/25] selftests/bpf: add get_ksyms and get_addrs to test_progs.c Menglong Dong
2025-05-28 3:47 ` [PATCH bpf-next 24/25] selftests/bpf: add testcases for multi-link of tracing Menglong Dong
2025-05-28 3:47 ` [PATCH bpf-next 25/25] selftests/bpf: add performance bench test for trace prog Menglong Dong
2025-05-28 13:51 ` [PATCH bpf-next 00/25] bpf: tracing multi-link support Steven Rostedt
2025-05-29 1:44 ` Menglong Dong
2025-06-11 3:32 ` Alexei Starovoitov
2025-06-11 12:58 ` Menglong Dong
2025-06-11 16:11 ` Alexei Starovoitov
2025-06-12 0:07 ` Menglong Dong
2025-06-12 0:58 ` Alexei Starovoitov
2025-06-12 3:18 ` Menglong Dong
2025-06-15 8:35 ` Menglong Dong
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20250528034712.138701-15-dongml2@chinatelecom.cn \
--to=menglong8.dong@gmail.com \
--cc=alexei.starovoitov@gmail.com \
--cc=bpf@vger.kernel.org \
--cc=dongml2@chinatelecom.cn \
--cc=jolsa@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=rostedt@goodmis.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®