From: Benjamin Tissoires <bentiss@kernel.org>
To: Alexei Starovoitov <ast@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
Andrii Nakryiko <andrii@kernel.org>,
Martin KaFai Lau <martin.lau@linux.dev>,
Eduard Zingerman <eddyz87@gmail.com>, Song Liu <song@kernel.org>,
Yonghong Song <yonghong.song@linux.dev>,
John Fastabend <john.fastabend@gmail.com>,
KP Singh <kpsingh@kernel.org>,
Stanislav Fomichev <sdf@google.com>, Hao Luo <haoluo@google.com>,
Jiri Olsa <jolsa@kernel.org>, Mykola Lysenko <mykolal@fb.com>,
Shuah Khan <shuah@kernel.org>
Cc: bpf@vger.kernel.org, linux-kernel@vger.kernel.org,
linux-kselftest@vger.kernel.org,
Benjamin Tissoires <bentiss@kernel.org>
Subject: [PATCH bpf-next 14/18] bpf/verifier: add is_sleepable argument to push_callback_call
Date: Tue, 16 Apr 2024 16:08:27 +0200 [thread overview]
Message-ID: <20240416-bpf_wq-v1-14-c9e66092f842@kernel.org> (raw)
In-Reply-To: <20240416-bpf_wq-v1-0-c9e66092f842@kernel.org>
To support sleepable async callbacks, we need to tell push_callback_call()
whether the cb is sleepable or not.
Doing so while checking for the kfunc arguments (when we call
push_callback_call()) is simpler than adding a check for a function
while inside push_callback_call().
When a callback is tagged as sleepable, the verifier now knows that it is
the case and can allow a sleepable callback to happen.
Signed-off-by: Benjamin Tissoires <bentiss@kernel.org>
---
include/linux/bpf_verifier.h | 1 +
kernel/bpf/verifier.c | 26 ++++++++++++++++----------
2 files changed, 17 insertions(+), 10 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 36d19cd32eb5..9db35530c878 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -426,6 +426,7 @@ struct bpf_verifier_state {
* while they are still in use.
*/
bool used_as_loop_entry;
+ bool in_sleepable;
/* first and last insn idx of this verifier state */
u32 first_insn_idx;
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 5e8c1e65fe8c..6a45d88244c6 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1429,6 +1429,7 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state,
}
dst_state->speculative = src->speculative;
dst_state->active_rcu_lock = src->active_rcu_lock;
+ dst_state->in_sleepable = src->in_sleepable;
dst_state->curframe = src->curframe;
dst_state->active_lock.ptr = src->active_lock.ptr;
dst_state->active_lock.id = src->active_lock.id;
@@ -2404,7 +2405,7 @@ static void init_func_state(struct bpf_verifier_env *env,
/* Similar to push_stack(), but for async callbacks */
static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,
int insn_idx, int prev_insn_idx,
- int subprog)
+ int subprog, bool is_sleepable)
{
struct bpf_verifier_stack_elem *elem;
struct bpf_func_state *frame;
@@ -2431,6 +2432,7 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,
* Initialize it similar to do_check_common().
*/
elem->st.branches = 1;
+ elem->st.in_sleepable = is_sleepable;
frame = kzalloc(sizeof(*frame), GFP_KERNEL);
if (!frame)
goto err;
@@ -5278,7 +5280,8 @@ static int map_kptr_match_type(struct bpf_verifier_env *env,
static bool in_sleepable(struct bpf_verifier_env *env)
{
- return env->prog->sleepable;
+ return env->prog->sleepable ||
+ (env->cur_state && env->cur_state->in_sleepable);
}
/* The non-sleepable programs and sleepable programs with explicit bpf_rcu_read_lock()
@@ -9515,7 +9518,7 @@ static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog,
}
static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
- int insn_idx, int subprog,
+ int insn_idx, int subprog, bool is_sleepable,
set_callee_state_fn set_callee_state_cb)
{
struct bpf_verifier_state *state = env->cur_state, *callback_state;
@@ -9550,7 +9553,7 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins
/* there is no real recursion here. timer callbacks are async */
env->subprog_info[subprog].is_async_cb = true;
async_cb = push_async_cb(env, env->subprog_info[subprog].start,
- insn_idx, subprog);
+ insn_idx, subprog, is_sleepable);
if (!async_cb)
return -EFAULT;
callee = async_cb->frame[0];
@@ -10389,15 +10392,15 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
}
break;
case BPF_FUNC_for_each_map_elem:
- err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+ err = push_callback_call(env, insn, insn_idx, meta.subprogno, false,
set_map_elem_callback_state);
break;
case BPF_FUNC_timer_set_callback:
- err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+ err = push_callback_call(env, insn, insn_idx, meta.subprogno, false,
set_timer_callback_state);
break;
case BPF_FUNC_find_vma:
- err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+ err = push_callback_call(env, insn, insn_idx, meta.subprogno, false,
set_find_vma_callback_state);
break;
case BPF_FUNC_snprintf:
@@ -10412,7 +10415,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
if (err)
return err;
if (cur_func(env)->callback_depth < regs[BPF_REG_1].umax_value) {
- err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+ err = push_callback_call(env, insn, insn_idx, meta.subprogno, false,
set_loop_callback_state);
} else {
cur_func(env)->callback_depth = 0;
@@ -10515,7 +10518,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
break;
}
case BPF_FUNC_user_ringbuf_drain:
- err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+ err = push_callback_call(env, insn, insn_idx, meta.subprogno, false,
set_user_ringbuf_callback_state);
break;
}
@@ -12232,7 +12235,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
return err;
if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
- err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+ err = push_callback_call(env, insn, insn_idx, meta.subprogno, false,
set_rbtree_add_callback_state);
if (err) {
verbose(env, "kfunc %s#%d failed callback verification\n",
@@ -17004,6 +17007,9 @@ static bool states_equal(struct bpf_verifier_env *env,
if (old->active_rcu_lock != cur->active_rcu_lock)
return false;
+ if (old->in_sleepable != cur->in_sleepable)
+ return false;
+
/* for states to be equal callsites have to be the same
* and all frame states need to be equivalent
*/
--
2.44.0
next prev parent reply other threads:[~2024-04-16 14:10 UTC|newest]
Thread overview: 32+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-04-16 14:08 [PATCH bpf-next 00/18] Introduce bpf_wq Benjamin Tissoires
2024-04-16 14:08 ` [PATCH bpf-next 01/18] bpf: trampoline: export __bpf_prog_enter/exit_recur Benjamin Tissoires
2024-04-16 14:08 ` [PATCH bpf-next 02/18] bpf: make timer data struct more generic Benjamin Tissoires
2024-04-16 14:08 ` [PATCH bpf-next 03/18] bpf: replace bpf_timer_init with a generic helper Benjamin Tissoires
2024-04-16 14:08 ` [PATCH bpf-next 04/18] bpf: replace bpf_timer_set_callback " Benjamin Tissoires
2024-04-16 14:08 ` [PATCH bpf-next 05/18] bpf: replace bpf_timer_cancel_and_free " Benjamin Tissoires
2024-04-16 14:08 ` [PATCH bpf-next 06/18] bpf: add support for bpf_wq user type Benjamin Tissoires
2024-04-19 6:02 ` Alexei Starovoitov
2024-04-16 14:08 ` [PATCH bpf-next 07/18] tools: sync include/uapi/linux/bpf.h Benjamin Tissoires
2024-04-16 14:08 ` [PATCH bpf-next 08/18] bpf: add support for KF_ARG_PTR_TO_WORKQUEUE Benjamin Tissoires
2024-04-19 6:00 ` Alexei Starovoitov
2024-04-16 14:08 ` [PATCH bpf-next 09/18] bpf: allow struct bpf_wq to be embedded in arraymaps and hashmaps Benjamin Tissoires
2024-04-19 6:05 ` Alexei Starovoitov
2024-04-16 14:08 ` [PATCH bpf-next 10/18] selftests/bpf: add bpf_wq tests Benjamin Tissoires
2024-04-16 14:08 ` [PATCH bpf-next 11/18] bpf: wq: add bpf_wq_init Benjamin Tissoires
2024-04-19 5:25 ` Alexei Starovoitov
2024-04-19 15:12 ` Benjamin Tissoires
2024-04-19 15:34 ` Alexei Starovoitov
2024-04-19 15:55 ` Benjamin Tissoires
2024-04-16 14:08 ` [PATCH bpf-next 12/18] tools: sync include/uapi/linux/bpf.h Benjamin Tissoires
2024-04-16 14:08 ` [PATCH bpf-next 13/18] selftests/bpf: wq: add bpf_wq_init() checks Benjamin Tissoires
2024-04-16 14:08 ` Benjamin Tissoires [this message]
2024-04-16 14:08 ` [PATCH bpf-next 15/18] bpf: wq: add bpf_wq_set_callback_impl Benjamin Tissoires
2024-04-16 14:08 ` [PATCH bpf-next 16/18] selftests/bpf: add checks for bpf_wq_set_callback() Benjamin Tissoires
2024-04-18 3:25 ` Song Liu
2024-04-18 8:55 ` Benjamin Tissoires
2024-04-16 14:08 ` [PATCH bpf-next 17/18] bpf: add bpf_wq_start Benjamin Tissoires
2024-04-19 6:18 ` Alexei Starovoitov
2024-04-19 15:14 ` Benjamin Tissoires
2024-04-19 15:49 ` Alexei Starovoitov
2024-04-19 16:01 ` Benjamin Tissoires
2024-04-16 14:08 ` [PATCH bpf-next 18/18] selftests/bpf: wq: add bpf_wq_start() checks Benjamin Tissoires
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20240416-bpf_wq-v1-14-c9e66092f842@kernel.org \
--to=bentiss@kernel.org \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=eddyz87@gmail.com \
--cc=haoluo@google.com \
--cc=john.fastabend@gmail.com \
--cc=jolsa@kernel.org \
--cc=kpsingh@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-kselftest@vger.kernel.org \
--cc=martin.lau@linux.dev \
--cc=mykolal@fb.com \
--cc=sdf@google.com \
--cc=shuah@kernel.org \
--cc=song@kernel.org \
--cc=yonghong.song@linux.dev \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®