* [PATCH bpf-next 1/4] bpf: Add a list of deferred instruction patches
2026-10-01 9:35 [PATCH bpf-next 0/4] bpf: Apply the rewrites of bpf_do_misc_fixups() at once Qiliang Yuan
@ 2026-10-01 9:35 ` Qiliang Yuan
2026-10-01 10:27 ` bot+bpf-ci
2026-10-01 9:35 ` [PATCH bpf-next 2/4] bpf: Queue the rewrites of bpf_do_misc_fixups() and apply them at once Qiliang Yuan
` (2 subsequent siblings)
3 siblings, 1 reply; 7+ messages in thread
From: Qiliang Yuan @ 2026-10-01 9:35 UTC (permalink / raw)
To: Alexei Starovoitov, Daniel Borkmann, John Fastabend,
Andrii Nakryiko, Eduard Zingerman, Kumar Kartikeya Dwivedi,
Martin KaFai Lau, Song Liu, Yonghong Song, Jiri Olsa,
Emil Tsalapatis, Ihor Solodrai, Shuah Khan
Cc: bpf, linux-kernel, linux-kselftest, Qiliang Yuan
bpf_patch_insn_data() applies a patch right away. It moves the tail of
the instruction and aux data arrays, and it walks the whole program to
fix up branches and every table that names instructions. A pass that
patches M instructions of an N instruction program does O(N * M) work.
pyperf180 from the selftests has 930 bpf_map_lookup_elem() calls on
hash maps. bpf_do_misc_fixups() inlines each of them into 3 instructions
of a 24k instruction program, which takes about a fifth of the time to
load it.
Add a list of patches that a pass queues with bpf_patch_list_add() and
applies with bpf_patch_list_commit() in O(N + inserted instructions).
Until the commit nothing moves, the pass keeps addressing instructions
and aux data by their index in the unpatched program, so patches are
queued in increasing order.
The commit maps each old index to a new one and names a patched
instruction by the first instruction of its patch, as
bpf_patch_insn_data() does. Branches, aux data, subprog starts, line
info, instruction arrays, function pointers and poke descriptors all
follow the map. The new image is built aside first, so a branch that
gets out of range leaves the program untouched.
A patch of a single instruction moves nothing and is applied right
away.
Suggested-by: Eduard Zingerman <eddyz87@gmail.com>
Signed-off-by: Qiliang Yuan <odys.yuan@gmail.com>
---
include/linux/bpf.h | 1 +
include/linux/bpf_verifier.h | 26 ++++
kernel/bpf/bpf_insn_array.c | 18 +++
kernel/bpf/fixups.c | 295 +++++++++++++++++++++++++++++++++++++++++++
kernel/bpf/verifier.c | 1 +
5 files changed, 341 insertions(+)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 4bae3796c42f6..17932ff8b0ae9 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -4299,6 +4299,7 @@ int bpf_insn_array_ready(struct bpf_map *map);
void bpf_insn_array_release(struct bpf_map *map);
void bpf_insn_array_adjust(struct bpf_map *map, u32 first, u32 len);
void bpf_insn_array_adjust_after_remove(struct bpf_map *map, u32 off, u32 len);
+void bpf_insn_array_remap(struct bpf_map *map, const u32 *new_off, u32 cnt, u32 grow);
#ifdef CONFIG_BPF_SYSCALL
void bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image);
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 811342e3c0419..5a968539589f8 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -929,6 +929,27 @@ struct bpf_fd_array {
};
};
+/* one deferred patch: insn 'off' is replaced by 'len' insns */
+struct bpf_insn_patch {
+ u32 off;
+ u32 len;
+ u32 start; /* first insn of the patch in bpf_patch_list.insns */
+ struct bpf_insn orig; /* the replaced insn when the patch was queued */
+};
+
+/*
+ * Patches queued by a rewrite pass, applied together by
+ * bpf_patch_list_commit() in O(prog->len + inserted insns).
+ */
+struct bpf_patch_list {
+ struct bpf_insn_patch *patches;
+ struct bpf_insn *insns;
+ u32 cnt;
+ u32 cap;
+ u32 insn_cnt;
+ u32 insn_cap;
+};
+
/* single container for all structs
* one verifier_env per bpf_check() call
*/
@@ -973,6 +994,7 @@ struct bpf_verifier_env {
bool signature;
u32 insn_aux_data_len;
struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */
+ struct bpf_patch_list patch_list;
const struct bpf_line_info *prev_linfo;
struct bpf_verifier_log log;
struct bpf_diag *diag;
@@ -1834,5 +1856,9 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env);
int bpf_fixup_call_args(struct bpf_verifier_env *env);
int bpf_do_misc_fixups(struct bpf_verifier_env *env);
int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn);
+struct bpf_insn *bpf_patch_list_add(struct bpf_verifier_env *env, u32 off,
+ const struct bpf_insn *patch, u32 len);
+int bpf_patch_list_commit(struct bpf_verifier_env *env);
+void bpf_patch_list_free(struct bpf_verifier_env *env);
#endif /* _LINUX_BPF_VERIFIER_H */
diff --git a/kernel/bpf/bpf_insn_array.c b/kernel/bpf/bpf_insn_array.c
index 92bbb71576ae3..74f07ced85cb0 100644
--- a/kernel/bpf/bpf_insn_array.c
+++ b/kernel/bpf/bpf_insn_array.c
@@ -253,6 +253,24 @@ void bpf_insn_array_adjust(struct bpf_map *map, u32 first, u32 len)
}
}
+/*
+ * Move each offset to new_off[offset], new_off[] covers [0, cnt]. Offsets
+ * past the end of the old program only move by 'grow'.
+ */
+void bpf_insn_array_remap(struct bpf_map *map, const u32 *new_off, u32 cnt, u32 grow)
+{
+ struct bpf_insn_array *insn_array = cast_insn_array(map);
+ u32 off;
+ int i;
+
+ for (i = 0; i < map->max_entries; i++) {
+ off = insn_array->values[i].xlated_off;
+ if (off == INSN_DELETED)
+ continue;
+ insn_array->values[i].xlated_off = off <= cnt ? new_off[off] : off + grow;
+ }
+}
+
void bpf_insn_array_adjust_after_remove(struct bpf_map *map, u32 off, u32 len)
{
struct bpf_insn_array *insn_array = cast_insn_array(map);
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 37cf130ebb57b..39566f3825108 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -404,6 +404,301 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
return __bpf_patch_insn_data(env, off, patch, len, BPF_PATCH_KEEP_TARGET);
}
+/*
+ * Queue the replacement of insn 'off' by 'patch[0..len)'. Nothing moves until
+ * bpf_patch_list_commit(), so a pass keeps addressing insns and their aux data
+ * by their index in env->prog. Patches must be queued in increasing order of
+ * 'off'. Jumps in the patch that leave it are relative to the patch placed at
+ * 'off', the same way as for bpf_patch_insn_data().
+ *
+ * Return the copy of the last insn of the patch, which stays valid until the
+ * next call.
+ */
+struct bpf_insn *bpf_patch_list_add(struct bpf_verifier_env *env, u32 off,
+ const struct bpf_insn *patch, u32 len)
+{
+ struct bpf_patch_list *pl = &env->patch_list;
+ struct bpf_insn_patch *p;
+ struct bpf_insn *insns;
+ u32 cap;
+
+ if (bpf_rewrite_must_abort())
+ return ERR_PTR(-EINTR);
+
+ if (verifier_bug_if(!len || off >= env->prog->len ||
+ (pl->cnt && off <= pl->patches[pl->cnt - 1].off),
+ env, "insn %u patched out of order", off))
+ return ERR_PTR(-EFAULT);
+
+ /* Replacing one insn by one moves nothing, do it right away. */
+ if (len == 1) {
+ env->prog->insnsi[off] = *patch;
+ adjust_insn_aux_data(env, env->prog, off, 1, NULL,
+ BPF_PATCH_KEEP_TARGET);
+ return &env->prog->insnsi[off];
+ }
+
+ if (pl->cnt == pl->cap) {
+ cap = max(pl->cap * 2, 16U);
+ p = kvrealloc(pl->patches, array_size(cap, sizeof(*p)),
+ GFP_KERNEL_ACCOUNT);
+ if (!p)
+ return ERR_PTR(-ENOMEM);
+ pl->patches = p;
+ pl->cap = cap;
+ }
+
+ if (pl->insn_cnt + len > pl->insn_cap) {
+ cap = max3(pl->insn_cap * 2, pl->insn_cnt + len, 64U);
+ insns = kvrealloc(pl->insns, array_size(cap, sizeof(*insns)),
+ GFP_KERNEL_ACCOUNT);
+ if (!insns)
+ return ERR_PTR(-ENOMEM);
+ pl->insns = insns;
+ pl->insn_cap = cap;
+ }
+
+ p = &pl->patches[pl->cnt++];
+ p->off = off;
+ p->len = len;
+ p->start = pl->insn_cnt;
+ p->orig = env->prog->insnsi[off];
+ memcpy(pl->insns + p->start, patch, len * sizeof(*patch));
+ pl->insn_cnt += len;
+
+ return &pl->insns[p->start + len - 1];
+}
+
+void bpf_patch_list_free(struct bpf_verifier_env *env)
+{
+ struct bpf_patch_list *pl = &env->patch_list;
+
+ kvfree(pl->patches);
+ kvfree(pl->insns);
+ memset(pl, 0, sizeof(*pl));
+}
+
+/* Extract the pc-relative operand of a jump, a subprog call or a ld_imm64 of a subprog. */
+static bool insn_get_rel(const struct bpf_insn *insn, s64 *rel, bool *is_imm)
+{
+ u8 code = insn->code;
+
+ if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn)) {
+ *rel = insn->imm;
+ *is_imm = true;
+ return true;
+ }
+ if ((BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) ||
+ BPF_OP(code) == BPF_CALL || BPF_OP(code) == BPF_EXIT)
+ return false;
+
+ *is_imm = code == (BPF_JMP32 | BPF_JA);
+ *rel = *is_imm ? insn->imm : insn->off;
+ return true;
+}
+
+/*
+ * insn is number 'idx' of the 'len' insns that replace insn 'off' of the old
+ * image (an insn that isn't patched is its own patch of length 1). It lands
+ * at 'pos' of the new image. 'new_off' maps the old image to the new one.
+ */
+static int patch_list_adj_insn(struct bpf_insn *insn, const u32 *new_off,
+ u32 cnt, u32 off, u32 idx, u32 len, u32 pos)
+{
+ bool is_imm;
+ s64 rel, tgt;
+
+ if (!insn_get_rel(insn, &rel, &is_imm))
+ return 0;
+
+ /* relative to the first insn of the patch */
+ tgt = (s64)idx + rel + 1;
+ if (tgt >= 0 && tgt < len)
+ return 0;
+
+ /* the patch pushed what follows 'off' down by len - 1 */
+ tgt = tgt < 0 ? off + tgt : off + tgt - len + 1;
+ if (tgt < 0 || tgt > cnt)
+ return -EFAULT;
+
+ rel = (s64)new_off[tgt] - pos - 1;
+ if (is_imm) {
+ if (rel < S32_MIN || rel > S32_MAX)
+ return -ERANGE;
+ insn->imm = rel;
+ } else {
+ if (rel < S16_MIN || rel > S16_MAX)
+ return -ERANGE;
+ insn->off = rel;
+ }
+ return 0;
+}
+
+/*
+ * Fill the aux data of the 'p->len' insns at 'slot' from the aux data 'old'
+ * of the insn they replace, the same way as adjust_insn_aux_data() does.
+ */
+static void patch_list_adj_aux(struct bpf_prog *prog, struct bpf_insn_patch *p,
+ struct bpf_insn *patch,
+ const struct bpf_insn_aux_data *old,
+ struct bpf_insn_aux_data *slot)
+{
+ struct bpf_insn_aux_data *last = slot + p->len - 1;
+ struct bpf_insn *orig = &p->orig;
+ u32 i;
+
+ *last = *old;
+ last->zext_dst = bpf_insn_def32(prog, &patch[p->len - 1]) >= 0;
+ memset(slot, 0, sizeof(*slot) * (p->len - 1));
+
+ for (i = 0; i < p->len - 1; i++) {
+ slot[i].seen = last->seen;
+ slot[i].zext_dst = bpf_insn_def32(prog, &patch[i]) >= 0;
+ if (!memcmp(&patch[i], orig, sizeof(*orig))) {
+ slot[i].non_stack_access = last->non_stack_access;
+ last->non_stack_access = false;
+ } else if (bpf_is_mem_insn(&patch[i])) {
+ slot[i].non_stack_access = true;
+ }
+ }
+
+ if (bpf_is_mem_insn(&patch[p->len - 1]) &&
+ memcmp(&patch[p->len - 1], orig, sizeof(*orig)))
+ last->non_stack_access = true;
+
+ /* indirect jumps to the replaced insn land on the first new one */
+ if (last->indirect_target) {
+ slot[0].indirect_target = 1;
+ last->indirect_target = 0;
+ }
+}
+
+static u32 patch_list_remap(const u32 *new_off, u32 cnt, u32 grow, u32 off)
+{
+ return off <= cnt ? new_off[off] : off + grow;
+}
+
+/*
+ * Apply all queued patches in one go. Everything that names an insn of the
+ * old image follows it to the new one, a patched insn is named by the first
+ * insn of its patch, like with bpf_patch_insn_data().
+ */
+int bpf_patch_list_commit(struct bpf_verifier_env *env)
+{
+ struct bpf_patch_list *pl = &env->patch_list;
+ struct bpf_prog *prog = env->prog;
+ u32 cnt = prog->len, new_cnt, grow = 0, end, i, k, n;
+ struct bpf_insn_aux_data *data;
+ struct bpf_insn *insns = NULL;
+ struct bpf_insn_patch *p;
+ u32 *new_off = NULL;
+ int err = 0;
+
+ if (!pl->cnt)
+ goto out;
+
+ err = -ENOMEM;
+ new_off = kvmalloc_array(cnt + 1, sizeof(*new_off), GFP_KERNEL_ACCOUNT);
+ if (!new_off)
+ goto out;
+
+ for (i = 0, k = 0; i <= cnt; i++) {
+ new_off[i] = i + grow;
+ if (k < pl->cnt && pl->patches[k].off == i)
+ grow += pl->patches[k++].len - 1;
+ }
+ new_cnt = cnt + grow;
+
+ /*
+ * Build the new image aside first, so that a jump that goes out of
+ * range leaves the program untouched.
+ */
+ insns = kvmalloc_array(new_cnt, sizeof(*insns), GFP_KERNEL_ACCOUNT);
+ if (!insns)
+ goto out;
+
+ for (i = 0, k = 0; i < cnt; i++) {
+ const struct bpf_insn *src = &prog->insnsi[i];
+ u32 len = 1;
+
+ if (k < pl->cnt && pl->patches[k].off == i) {
+ src = &pl->insns[pl->patches[k].start];
+ len = pl->patches[k++].len;
+ }
+
+ for (n = 0; n < len; n++) {
+ insns[new_off[i] + n] = src[n];
+ err = patch_list_adj_insn(&insns[new_off[i] + n], new_off,
+ cnt, i, n, len, new_off[i] + n);
+ if (err == -ERANGE) {
+ verbose(env, "insn %d cannot be patched due to 16-bit range\n",
+ env->insn_aux_data[i].orig_idx);
+ goto out;
+ }
+ if (verifier_bug_if(err, env, "insn %u jumps out of the program", i))
+ goto out;
+ }
+ }
+
+ err = -ENOMEM;
+ data = vrealloc(env->insn_aux_data,
+ array_size(new_cnt, sizeof(*data)),
+ GFP_KERNEL_ACCOUNT | __GFP_ZERO);
+ if (!data)
+ goto out;
+ env->insn_aux_data = data;
+
+ prog = bpf_prog_realloc(prog, bpf_prog_size(new_cnt), GFP_USER);
+ if (!prog)
+ goto out;
+ env->prog = prog;
+
+ memcpy(prog->insnsi, insns, sizeof(*insns) * new_cnt);
+ prog->len = new_cnt;
+
+ /* Move aux data from the end, an insn never moves to a lower index. */
+ end = cnt;
+ for (k = pl->cnt; k-- > 0;) {
+ p = &pl->patches[k];
+ memmove(data + new_off[p->off + 1], data + p->off + 1,
+ sizeof(*data) * (end - p->off - 1));
+ patch_list_adj_aux(prog, p, &pl->insns[p->start], &data[p->off],
+ &data[new_off[p->off]]);
+ end = p->off;
+ }
+ env->insn_aux_data_len = new_cnt;
+
+ /* NOTE: fake 'exit' subprog should be updated as well. */
+ for (i = 0; i <= env->subprog_cnt; i++)
+ env->subprog_info[i].start =
+ patch_list_remap(new_off, cnt, grow, env->subprog_info[i].start);
+
+ for (i = 0; i < prog->aux->nr_linfo; i++)
+ prog->aux->linfo[i].insn_off =
+ patch_list_remap(new_off, cnt, grow, prog->aux->linfo[i].insn_off);
+
+ for (i = 0; i < env->insn_array_map_cnt; i++)
+ bpf_insn_array_remap(env->insn_array_maps[i], new_off, cnt, grow);
+
+ for (i = 0; i < env->func_ptr_cnt; i++) {
+ if (env->func_ptrs[i].xlated_off == BPF_FUNC_PTR_DELETED)
+ continue;
+ env->func_ptrs[i].xlated_off =
+ patch_list_remap(new_off, cnt, grow, env->func_ptrs[i].xlated_off);
+ }
+
+ for (i = 0; i < prog->aux->size_poke_tab; i++)
+ prog->aux->poke_tab[i].insn_idx =
+ patch_list_remap(new_off, cnt, grow, prog->aux->poke_tab[i].insn_idx);
+
+ err = 0;
+out:
+ kvfree(insns);
+ kvfree(new_off);
+ bpf_patch_list_free(env);
+ return err;
+}
+
/*
* insn was moved down by delta insns inside its own patch. Operands relative
* to the pc that point in front of the old position did not move with it.
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index b840b3eb9b229..7c1bed0c66c64 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -22753,6 +22753,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
err_free_env:
bpf_free_subprog_jts(env);
vfree(env->insn_aux_data);
+ bpf_patch_list_free(env);
kvfree(env->fd_array);
bpf_stack_liveness_free(env);
kvfree(env->cfg.insn_postorder);
--
2.43.0
^ permalink raw reply [flat|nested] 7+ messages in thread* Re: [PATCH bpf-next 1/4] bpf: Add a list of deferred instruction patches
2026-10-01 9:35 ` [PATCH bpf-next 1/4] bpf: Add a list of deferred instruction patches Qiliang Yuan
@ 2026-10-01 10:27 ` bot+bpf-ci
0 siblings, 0 replies; 7+ messages in thread
From: bot+bpf-ci @ 2026-10-01 10:27 UTC (permalink / raw)
To: odys.yuan, ast, daniel, john.fastabend, andrii, eddyz87, memxor,
martin.lau, song, yonghong.song, jolsa, emil, ihor.solodrai,
shuah
Cc: bpf, linux-kernel, linux-kselftest, odys.yuan, ast, andrii,
daniel, martin.lau, eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 2785 bytes --]
> bpf: Add a list of deferred instruction patches
TL;DR: Nit only: patch_list_adj_aux() duplicates the aux-data propagation
logic of adjust_insn_aux_data() in fixups.c; the review asks whether a shared
helper could keep those rules in one place. Not a bug.
> diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
> index 37cf130ebb57..39566f382510 100644
> --- a/kernel/bpf/fixups.c
> +++ b/kernel/bpf/fixups.c
> @@ -404,6 +404,301 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
[ ... ]
> +/*
> + * Fill the aux data of the 'p->len' insns at 'slot' from the aux data 'old'
> + * of the insn they replace, the same way as adjust_insn_aux_data() does.
> + */
> +static void patch_list_adj_aux(struct bpf_prog *prog, struct bpf_insn_patch *p,
> + struct bpf_insn *patch,
> + const struct bpf_insn_aux_data *old,
> + struct bpf_insn_aux_data *slot)
> +{
> + struct bpf_insn_aux_data *last = slot + p->len - 1;
> + struct bpf_insn *orig = &p->orig;
> + u32 i;
> +
> + *last = *old;
> + last->zext_dst = bpf_insn_def32(prog, &patch[p->len - 1]) >= 0;
> + memset(slot, 0, sizeof(*slot) * (p->len - 1));
> +
> + for (i = 0; i < p->len - 1; i++) {
> + slot[i].seen = last->seen;
> + slot[i].zext_dst = bpf_insn_def32(prog, &patch[i]) >= 0;
> + if (!memcmp(&patch[i], orig, sizeof(*orig))) {
> + slot[i].non_stack_access = last->non_stack_access;
> + last->non_stack_access = false;
> + } else if (bpf_is_mem_insn(&patch[i])) {
> + slot[i].non_stack_access = true;
> + }
> + }
> +
> + if (bpf_is_mem_insn(&patch[p->len - 1]) &&
> + memcmp(&patch[p->len - 1], orig, sizeof(*orig)))
> + last->non_stack_access = true;
> +
> + /* indirect jumps to the replaced insn land on the first new one */
> + if (last->indirect_target) {
> + slot[0].indirect_target = 1;
> + last->indirect_target = 0;
> + }
> +}
this isn't a bug, but patch_list_adj_aux() repeats the per-slot aux fill
that adjust_insn_aux_data() already does for the BPF_PATCH_KEEP_TARGET
case, as its own comment says. The seen propagation, the zext_dst
computation via bpf_insn_def32(), the non_stack_access transfer keyed on
memcmp() against the original insn, the last-slot bpf_is_mem_insn()
marking and the move of indirect_target to the first slot are all
duplicated.
Could adjust_insn_aux_data() and patch_list_adj_aux() share a small helper
that fills 'len' aux slots from the insns that replaced 'orig' (taking the
slot base, the patch insns and the mode), so the aux propagation rules live
in one place?
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/36845462631
^ permalink raw reply [flat|nested] 7+ messages in thread
* [PATCH bpf-next 2/4] bpf: Queue the rewrites of bpf_do_misc_fixups() and apply them at once
2026-10-01 9:35 [PATCH bpf-next 0/4] bpf: Apply the rewrites of bpf_do_misc_fixups() at once Qiliang Yuan
2026-10-01 9:35 ` [PATCH bpf-next 1/4] bpf: Add a list of deferred instruction patches Qiliang Yuan
@ 2026-10-01 9:35 ` Qiliang Yuan
2026-10-01 10:27 ` bot+bpf-ci
2026-10-01 9:35 ` [PATCH bpf-next 3/4] bpf: Drop the constant delta from bpf_do_misc_fixups() Qiliang Yuan
2026-10-01 9:35 ` [PATCH bpf-next 4/4] selftests/bpf: Test jumps around patches of bpf_do_misc_fixups() Qiliang Yuan
3 siblings, 1 reply; 7+ messages in thread
From: Qiliang Yuan @ 2026-10-01 9:35 UTC (permalink / raw)
To: Alexei Starovoitov, Daniel Borkmann, John Fastabend,
Andrii Nakryiko, Eduard Zingerman, Kumar Kartikeya Dwivedi,
Martin KaFai Lau, Song Liu, Yonghong Song, Jiri Olsa,
Emil Tsalapatis, Ihor Solodrai, Shuah Khan
Cc: bpf, linux-kernel, linux-kselftest, Qiliang Yuan
Each rewrite in the main loop of bpf_do_misc_fixups() patches the
program right away, which costs O(prog->len) per rewrite. Queue them
with bpf_patch_list_add() instead and commit them once the loop is done,
before the stack of subprogs is initialized for may_goto.
The loop now walks the unpatched program, delta stays 0. A rewrite that
fixes up the helper call after queueing it goes on to patch_call_imm
with insn pointing to the queued copy of the call, so step insn from
the program by index instead of incrementing it.
Signed-off-by: Qiliang Yuan <odys.yuan@gmail.com>
---
kernel/bpf/fixups.c | 228 +++++++++++++++++-----------------------------------
1 file changed, 72 insertions(+), 156 deletions(-)
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 39566f3825108..4b96f4ee9b3d8 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -2099,13 +2099,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = patch - insn_buf;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2190,13 +2186,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = patch - insn_buf;
}
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2220,13 +2212,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
*patch++ = BPF_MOV64_IMM(insn->dst_reg, 0);
cnt = patch - insn_buf;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2240,13 +2228,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
return -EFAULT;
}
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2293,13 +2277,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
*patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
cnt = patch - insn_buf;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2336,13 +2316,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = may_goto_expand(insn_buf, insn->off, stack_off_cnt,
tail, ARRAY_SIZE(tail));
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
} else if (bpf_is_may_goto_insn(insn)) {
int stack_off = -stack_depth - 8;
@@ -2355,13 +2331,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = may_goto_expand(insn_buf, insn->off, stack_off,
tail, ARRAY_SIZE(tail));
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2392,13 +2364,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = 2;
i++;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2413,13 +2381,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
if (cnt == 0)
goto next_insn;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2500,13 +2464,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
map)->index_mask);
insn_buf[2] = *insn;
cnt = 3;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2534,13 +2494,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[2] = *insn;
cnt = 3;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto patch_call_imm;
}
@@ -2553,13 +2509,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[1] = *insn;
cnt = 2;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto patch_call_imm;
}
@@ -2596,14 +2548,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
if (bpf_map_is_percpu_map(map_ptr->map_type))
prog->jit_required = true;
- new_prog = bpf_patch_insn_data(env, i + delta,
- insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2679,14 +2626,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
BPF_REG_0, 0);
cnt = 3;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf,
- cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2709,13 +2651,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[0] = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0);
cnt = 1;
#endif
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2728,13 +2666,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);
cnt = 3;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
#endif
@@ -2762,13 +2696,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[cnt++] = BPF_JMP_A(1);
insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL);
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2794,13 +2724,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = 1;
}
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2820,13 +2746,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = 2;
}
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2836,12 +2758,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
/* Load IP address from ctx - 16 */
insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16);
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
- if (!new_prog)
- return -ENOMEM;
-
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, 1);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2891,13 +2810,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[10] = BPF_MOV64_IMM(BPF_REG_0, -ENOENT);
cnt = 11;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2909,13 +2824,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0);
cnt = 2;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
patch_call_imm:
@@ -2946,9 +2857,14 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
stack_depth_extra = 0;
}
i++;
- insn++;
+ insn = &prog->insnsi[i + delta];
}
+ ret = bpf_patch_list_commit(env);
+ if (ret)
+ return ret;
+ prog = env->prog;
+
env->prog->aux->stack_depth = subprogs[0].stack_depth;
for (i = 0; i < env->subprog_cnt; i++) {
int delta = bpf_jit_supports_timed_may_goto() ? 2 : 1;
--
2.43.0
^ permalink raw reply [flat|nested] 7+ messages in thread* Re: [PATCH bpf-next 2/4] bpf: Queue the rewrites of bpf_do_misc_fixups() and apply them at once
2026-10-01 9:35 ` [PATCH bpf-next 2/4] bpf: Queue the rewrites of bpf_do_misc_fixups() and apply them at once Qiliang Yuan
@ 2026-10-01 10:27 ` bot+bpf-ci
0 siblings, 0 replies; 7+ messages in thread
From: bot+bpf-ci @ 2026-10-01 10:27 UTC (permalink / raw)
To: odys.yuan, ast, daniel, john.fastabend, andrii, eddyz87, memxor,
martin.lau, song, yonghong.song, jolsa, emil, ihor.solodrai,
shuah
Cc: bpf, linux-kernel, linux-kselftest, odys.yuan, ast, andrii,
daniel, martin.lau, eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 3993 bytes --]
> bpf: Queue the rewrites of bpf_do_misc_fixups() and apply them at once
TL;DR: Possible regression in bpf_do_misc_fixups(): may_goto_expand() now sees
unpatched offsets, so a backward may_goto at exactly -32768 may get a 16-bit
JEQ instead of gotol, and a previously loading program may fail with -ERANGE.
> diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
> index 39566f382510..4b96f4ee9b3d 100644
> --- a/kernel/bpf/fixups.c
> +++ b/kernel/bpf/fixups.c
[ ... ]
> @@ -2336,13 +2316,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
> cnt = may_goto_expand(insn_buf, insn->off, stack_off_cnt,
> tail, ARRAY_SIZE(tail));
>
> - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
> - if (!new_prog)
> - return -ENOMEM;
> -
> - delta += cnt - 1;
> - env->prog = prog = new_prog;
> - insn = new_prog->insnsi + i + delta;
> + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
> + if (IS_ERR(insn))
> + return PTR_ERR(insn);
> goto next_insn;
> } else if (bpf_is_may_goto_insn(insn)) {
> int stack_off = -stack_depth - 8;
> @@ -2355,13 +2331,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
> cnt = may_goto_expand(insn_buf, insn->off, stack_off,
> tail, ARRAY_SIZE(tail));
>
> - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
> - if (!new_prog)
> - return -ENOMEM;
> -
> - delta += cnt - 1;
> - env->prog = prog = new_prog;
> - insn = new_prog->insnsi + i + delta;
> + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
> + if (IS_ERR(insn))
> + return PTR_ERR(insn);
> goto next_insn;
> }
>
Can this change make a program that loaded before get rejected at commit
time?
may_goto_expand() picks between a 16-bit JEQ and JNE + gotol based on the
offset it is given:
off = off >= 0 ? off + tail_cnt : off - 1;
...
if (off == (s16)off) {
insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, off);
} else {
insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 1);
insn_buf[cnt++] = BPF_JMP32_A(off >= 0 ? off : off - 1);
}
Before this change every earlier rewrite was applied right away through
bpf_patch_insn_data(), and bpf_adj_branches() updated the jumps after each
one. So when the loop reached a backward may_goto, insn->off already
included all the growth between the jump target and the may_goto. That is
the final backward distance, and the JEQ-or-gotol choice was made on it.
Now the loop walks the unpatched program, so insn->off is the original,
unpatched offset. The real distance is only computed later, in
patch_list_adj_insn() called from bpf_patch_list_commit():
rel = (s64)new_off[tgt] - pos - 1;
...
if (rel < S16_MIN || rel > S16_MAX)
return -ERANGE;
Take a backward may_goto whose grown offset is exactly -32768 (S16_MIN).
The old code saw off = -32768, computed off - 1 = -32769, which does not
fit in s16, and emitted JNE + gotol, so the program loaded. The new code
sees the smaller original offset, so off - 1 fits in s16 and it emits JEQ.
At commit the JEQ's displacement is -32769, patch_list_adj_insn() returns
-ERANGE, and the program is rejected with "insn %d cannot be patched due to
16-bit range".
The non-timed path under bpf_is_may_goto_insn(), which also calls
may_goto_expand(), has the same problem. Forward may_goto behaves as
before, since the old code also chose JEQ from the unpatched forward
offset. The trigger is narrow, a single exact displacement.
The selftest added later in the series (f8ef7933910d) does not appear to
cover this gotol boundary, and no later commit in the series changes it.
Should the JEQ-or-gotol decision be deferred until the final offsets are
known?
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/36845462631
^ permalink raw reply [flat|nested] 7+ messages in thread
* [PATCH bpf-next 3/4] bpf: Drop the constant delta from bpf_do_misc_fixups()
2026-10-01 9:35 [PATCH bpf-next 0/4] bpf: Apply the rewrites of bpf_do_misc_fixups() at once Qiliang Yuan
2026-10-01 9:35 ` [PATCH bpf-next 1/4] bpf: Add a list of deferred instruction patches Qiliang Yuan
2026-10-01 9:35 ` [PATCH bpf-next 2/4] bpf: Queue the rewrites of bpf_do_misc_fixups() and apply them at once Qiliang Yuan
@ 2026-10-01 9:35 ` Qiliang Yuan
2026-10-01 9:35 ` [PATCH bpf-next 4/4] selftests/bpf: Test jumps around patches of bpf_do_misc_fixups() Qiliang Yuan
3 siblings, 0 replies; 7+ messages in thread
From: Qiliang Yuan @ 2026-10-01 9:35 UTC (permalink / raw)
To: Alexei Starovoitov, Daniel Borkmann, John Fastabend,
Andrii Nakryiko, Eduard Zingerman, Kumar Kartikeya Dwivedi,
Martin KaFai Lau, Song Liu, Yonghong Song, Jiri Olsa,
Emil Tsalapatis, Ihor Solodrai, Shuah Khan
Cc: bpf, linux-kernel, linux-kselftest, Qiliang Yuan
Nothing changes the program while bpf_do_misc_fixups() walks it anymore,
so delta is always 0 in the loop. Index instructions and aux data by i
alone.
Signed-off-by: Qiliang Yuan <odys.yuan@gmail.com>
---
kernel/bpf/fixups.c | 68 ++++++++++++++++++++++++++---------------------------
1 file changed, 34 insertions(+), 34 deletions(-)
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 4b96f4ee9b3d8..0e8c61ede9a5b 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -2044,7 +2044,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
struct bpf_insn *insn_buf = env->insn_buf;
struct bpf_prog *new_prog;
struct bpf_map *map_ptr;
- int i, ret, cnt, delta = 0, cur_subprog = 0;
+ int i, ret, cnt, cur_subprog = 0;
struct bpf_subprog_info *subprogs = env->subprog_info;
u16 stack_depth = subprogs[cur_subprog].stack_depth;
u16 stack_depth_extra = 0;
@@ -2076,7 +2076,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
goto next_insn;
}
- if (env->insn_aux_data[i + delta].needs_zext)
+ if (env->insn_aux_data[i].needs_zext)
/* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */
insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code);
@@ -2099,7 +2099,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = patch - insn_buf;
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
@@ -2186,7 +2186,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = patch - insn_buf;
}
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
@@ -2212,7 +2212,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
*patch++ = BPF_MOV64_IMM(insn->dst_reg, 0);
cnt = patch - insn_buf;
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
@@ -2228,7 +2228,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
return -EFAULT;
}
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
@@ -2243,7 +2243,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
bool issrc, isneg, isimm;
u32 off_reg;
- aux = &env->insn_aux_data[i + delta];
+ aux = &env->insn_aux_data[i];
if (!aux->alu_state ||
aux->alu_state == BPF_ALU_NON_POINTER)
goto next_insn;
@@ -2277,7 +2277,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
*patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
cnt = patch - insn_buf;
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
@@ -2316,7 +2316,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = may_goto_expand(insn_buf, insn->off, stack_off_cnt,
tail, ARRAY_SIZE(tail));
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
@@ -2331,7 +2331,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = may_goto_expand(insn_buf, insn->off, stack_off,
tail, ARRAY_SIZE(tail));
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
@@ -2343,7 +2343,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE)) {
struct bpf_map *map;
- aux = &env->insn_aux_data[i + delta];
+ aux = &env->insn_aux_data[i];
map = env->used_maps[aux->map_index];
if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY)
goto next_insn;
@@ -2364,7 +2364,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = 2;
i++;
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
@@ -2375,13 +2375,13 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
if (insn->src_reg == BPF_PSEUDO_CALL)
goto next_insn;
if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
- ret = bpf_fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt);
+ ret = bpf_fixup_kfunc_call(env, insn, insn_buf, i, &cnt);
if (ret)
return ret;
if (cnt == 0)
goto next_insn;
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
@@ -2418,7 +2418,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn->imm = 0;
insn->code = BPF_JMP | BPF_TAIL_CALL;
- aux = &env->insn_aux_data[i + delta];
+ aux = &env->insn_aux_data[i];
if (env->bpf_capable && !prog->blinding_requested &&
prog->jit_requested &&
!bpf_map_key_poisoned(aux) &&
@@ -2428,7 +2428,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
.reason = BPF_POKE_REASON_TAIL_CALL,
.tail_call.map = aux->map_ptr_state.map_ptr,
.tail_call.key = bpf_map_key_immediate(aux),
- .insn_idx = i + delta,
+ .insn_idx = i,
};
ret = bpf_jit_add_poke_descriptor(prog, &desc);
@@ -2464,13 +2464,13 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
map)->index_mask);
insn_buf[2] = *insn;
cnt = 3;
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
}
- aux = &env->insn_aux_data[i + delta];
+ aux = &env->insn_aux_data[i];
if (aux->arg_prog) {
/* The verifier will process callback_fn as many times as necessary
* with different maps and the register states prepared by
@@ -2494,14 +2494,14 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[2] = *insn;
cnt = 3;
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto patch_call_imm;
}
/* bpf_per_cpu_ptr() and bpf_this_cpu_ptr() */
- if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) {
+ if (env->insn_aux_data[i].call_with_percpu_alloc_ptr) {
/* patch with 'r1 = *(u64 *)(r1 + 0)' since for percpu data,
* bpf_mem_alloc() returns a ptr to the percpu data ptr.
*/
@@ -2509,7 +2509,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[1] = *insn;
cnt = 2;
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto patch_call_imm;
@@ -2529,7 +2529,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn->imm == BPF_FUNC_redirect_map ||
insn->imm == BPF_FUNC_for_each_map_elem ||
insn->imm == BPF_FUNC_map_lookup_percpu_elem)) {
- aux = &env->insn_aux_data[i + delta];
+ aux = &env->insn_aux_data[i];
if (bpf_map_ptr_poisoned(aux))
goto patch_call_imm;
@@ -2548,7 +2548,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
if (bpf_map_is_percpu_map(map_ptr->map_type))
prog->jit_required = true;
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
@@ -2626,7 +2626,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
BPF_REG_0, 0);
cnt = 3;
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
@@ -2651,7 +2651,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[0] = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0);
cnt = 1;
#endif
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
@@ -2666,7 +2666,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);
cnt = 3;
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
@@ -2696,7 +2696,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[cnt++] = BPF_JMP_A(1);
insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL);
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
@@ -2724,7 +2724,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = 1;
}
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
@@ -2746,7 +2746,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = 2;
}
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
@@ -2758,7 +2758,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
/* Load IP address from ctx - 16 */
insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16);
- insn = bpf_patch_list_add(env, i + delta, insn_buf, 1);
+ insn = bpf_patch_list_add(env, i, insn_buf, 1);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
@@ -2810,7 +2810,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[10] = BPF_MOV64_IMM(BPF_REG_0, -ENOENT);
cnt = 11;
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
@@ -2824,7 +2824,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0);
cnt = 2;
- insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
goto next_insn;
@@ -2842,7 +2842,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
}
insn->imm = BPF_CALL_IMM(fn->func);
next_insn:
- if (subprogs[cur_subprog + 1].start == i + delta + 1) {
+ if (subprogs[cur_subprog + 1].start == i + 1) {
subprogs[cur_subprog].stack_depth += stack_depth_extra;
subprogs[cur_subprog].stack_extra = stack_depth_extra;
@@ -2857,7 +2857,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
stack_depth_extra = 0;
}
i++;
- insn = &prog->insnsi[i + delta];
+ insn = &prog->insnsi[i];
}
ret = bpf_patch_list_commit(env);
--
2.43.0
^ permalink raw reply [flat|nested] 7+ messages in thread* [PATCH bpf-next 4/4] selftests/bpf: Test jumps around patches of bpf_do_misc_fixups()
2026-10-01 9:35 [PATCH bpf-next 0/4] bpf: Apply the rewrites of bpf_do_misc_fixups() at once Qiliang Yuan
` (2 preceding siblings ...)
2026-10-01 9:35 ` [PATCH bpf-next 3/4] bpf: Drop the constant delta from bpf_do_misc_fixups() Qiliang Yuan
@ 2026-10-01 9:35 ` Qiliang Yuan
3 siblings, 0 replies; 7+ messages in thread
From: Qiliang Yuan @ 2026-10-01 9:35 UTC (permalink / raw)
To: Alexei Starovoitov, Daniel Borkmann, John Fastabend,
Andrii Nakryiko, Eduard Zingerman, Kumar Kartikeya Dwivedi,
Martin KaFai Lau, Song Liu, Yonghong Song, Jiri Olsa,
Emil Tsalapatis, Ihor Solodrai, Shuah Khan
Cc: bpf, linux-kernel, linux-kselftest, Qiliang Yuan
bpf_do_misc_fixups() now queues its patches and applies them at once.
Check the xlated code for forward and backward jumps to a patched insn,
jumps over patches, and a may_goto whose jump leaves its own patch and
crosses another one.
Signed-off-by: Qiliang Yuan <odys.yuan@gmail.com>
---
tools/testing/selftests/bpf/prog_tests/verifier.c | 2 +
.../selftests/bpf/progs/verifier_patch_list.c | 120 +++++++++++++++++++++
2 files changed, 122 insertions(+)
diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c
index 8a6d341b754ac..af107b0c1595e 100644
--- a/tools/testing/selftests/bpf/prog_tests/verifier.c
+++ b/tools/testing/selftests/bpf/prog_tests/verifier.c
@@ -90,6 +90,7 @@
#include "verifier_netfilter_retcode.skel.h"
#include "verifier_bpf_fastcall.skel.h"
#include "verifier_or_jmp32_k.skel.h"
+#include "verifier_patch_list.skel.h"
#include "verifier_percpu_addr.skel.h"
#include "verifier_precision.skel.h"
#include "verifier_prevent_map_lookup.skel.h"
@@ -273,6 +274,7 @@ void test_verifier_netfilter_ctx(void) { RUN(verifier_netfilter_ctx); }
void test_verifier_netfilter_retcode(void) { RUN(verifier_netfilter_retcode); }
void test_verifier_bpf_fastcall(void) { RUN(verifier_bpf_fastcall); }
void test_verifier_or_jmp32_k(void) { RUN(verifier_or_jmp32_k); }
+void test_verifier_patch_list(void) { RUN(verifier_patch_list); }
void test_verifier_percpu_addr(void) { RUN(verifier_percpu_addr); }
void test_verifier_precision(void) { RUN(verifier_precision); }
void test_verifier_prevent_map_lookup(void) { RUN(verifier_prevent_map_lookup); }
diff --git a/tools/testing/selftests/bpf/progs/verifier_patch_list.c b/tools/testing/selftests/bpf/progs/verifier_patch_list.c
new file mode 100644
index 0000000000000..268cebd179984
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/verifier_patch_list.c
@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include "../../../include/linux/filter.h"
+#include "bpf_misc.h"
+
+/*
+ * bpf_do_misc_fixups() guards each division by a register against division
+ * by zero with a 4 insn patch. Jumps to a patched insn must land on the
+ * first insn of its patch, other jumps must follow the insns they target.
+ */
+
+SEC("raw_tp")
+__description("patch list: forward jumps to and over patched insns")
+__arch_x86_64
+__arch_arm64
+__success
+__xlated("0: call")
+__xlated("1: r1 = r0")
+__xlated("2: r0 = 7")
+__xlated("3: if r1 > 0x5 goto pc+1")
+__xlated("4: r0 = 9")
+__xlated("5: if r1 != 0x0 goto pc+2")
+__xlated("6: w0 ^= w0")
+__xlated("7: goto pc+1")
+__xlated("8: r0 /= r1")
+__xlated("9: if r0 > 0x3 goto pc+4")
+__xlated("10: if r1 != 0x0 goto pc+2")
+__xlated("11: w0 ^= w0")
+__xlated("12: goto pc+1")
+__xlated("13: r0 /= r1")
+__xlated("14: exit")
+__naked void patch_list_forward(void)
+{
+ asm volatile (
+ "call %[bpf_get_prandom_u32];"
+ "r1 = r0;"
+ "r0 = 7;"
+ "if r1 > 5 goto l0_%=;"
+ "r0 = 9;"
+"l0_%=:"
+ "r0 /= r1;"
+ "if r0 > 3 goto l1_%=;"
+ "r0 /= r1;"
+"l1_%=:"
+ "exit;"
+ :
+ : __imm(bpf_get_prandom_u32)
+ : __clobber_all);
+}
+
+SEC("raw_tp")
+__description("patch list: backward jump to a patched insn")
+__arch_x86_64
+__arch_arm64
+__success
+__xlated("0: call")
+__xlated("1: r1 = r0")
+__xlated("2: r2 = 0")
+__xlated("3: if r1 != 0x0 goto pc+2")
+__xlated("4: w0 ^= w0")
+__xlated("5: goto pc+1")
+__xlated("6: r0 /= r1")
+__xlated("7: r2 += 1")
+__xlated("8: if r2 < 0x3 goto pc-6")
+__xlated("9: exit")
+__naked void patch_list_backward(void)
+{
+ asm volatile (
+ "call %[bpf_get_prandom_u32];"
+ "r1 = r0;"
+ "r2 = 0;"
+"l0_%=:"
+ "r0 /= r1;"
+ "r2 += 1;"
+ "if r2 < 3 goto l0_%=;"
+ "exit;"
+ :
+ : __imm(bpf_get_prandom_u32)
+ : __clobber_all);
+}
+
+/* The jump of may_goto leaves its own patch and crosses another one. */
+SEC("raw_tp")
+__description("patch list: may_goto between patched insns")
+__arch_x86_64
+__success
+__xlated("0: *(u64 *)(r10 -16) = 65535")
+__xlated("1: *(u64 *)(r10 -8) = 0")
+__xlated("2: call")
+__xlated("3: r1 = r0")
+__xlated("4: if r1 != 0x0 goto pc+2")
+__xlated("5: w0 ^= w0")
+__xlated("6: goto pc+1")
+__xlated("7: r0 /= r1")
+__xlated("8: r12 = *(u64 *)(r10 -16)")
+__xlated("9: if r12 == 0x0 goto pc+9")
+__xlated("...")
+__xlated("15: if r1 != 0x0 goto pc+2")
+__xlated("16: w0 ^= w0")
+__xlated("17: goto pc+1")
+__xlated("18: r0 /= r1")
+__xlated("19: exit")
+__naked void patch_list_may_goto(void)
+{
+ asm volatile (
+ "call %[bpf_get_prandom_u32];"
+ "r1 = r0;"
+ "r0 /= r1;"
+ ".8byte %[may_goto];"
+ "r0 /= r1;"
+ "exit;"
+ :
+ : __imm(bpf_get_prandom_u32),
+ __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 1 /* offset */, 0))
+ : __clobber_all);
+}
+
+char _license[] SEC("license") = "GPL";
--
2.43.0
^ permalink raw reply [flat|nested] 7+ messages in thread