Re: [PATCH bpf-next v4 3/6] bpf: Inline bpf_iter_num_next() kfunc

On Wed, Jul 29, 2026 at 1:37 PM Puranjay Mohan <[email protected]> wrote:
>
> bpf_iter_num_next() is called on every iteration of a bpf_for() loop and
> is the hot path of the numeric open-coded iterator. It only advances the
> on-stack iterator state and returns a pointer to it, so open-coding it in
> the verifier removes a function call from each loop iteration.
>
> Inline it in bpf_fixup_kfunc_call() by replacing the call with an
> equivalent instruction sequence. R1 holds the pointer to the on-stack
> bpf_iter_num; the returned pointer to s->cur is R1 itself since s->cur is
> the first member.
>
> s->cur and s->end are int, so the kfunc's s->cur + 1 >= s->end test is a
> signed 32-bit comparison of (s->cur + 1) against s->end, with s->cur + 1
> computed as a 32-bit int. Sign-extending both sides of a comparison of
> two int values does not change its result, so the inlined code uses a
> 32-bit compare and needs no sign extension.
>
> Signed-off-by: Puranjay Mohan <[email protected]>
> ---
>  kernel/bpf/verifier.c | 30 ++++++++++++++++++++++++++++++
>  1 file changed, 30 insertions(+)
>
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index 7400515ae1296..3a3d096f3966e 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
> @@ -19810,6 +19810,34 @@ static int inline_bpf_iter_num_new(struct bpf_insn *insn_buf)
>         return i;
>  }
>
> +/*
> + * Inline bpf_iter_num_next(). R1 holds the pointer to the iterator. Keep in sync with the
> + * kfunc in kernel/bpf/bpf_iter.c.
> + */
> +static int inline_bpf_iter_num_next(struct bpf_insn *insn_buf)
> +{
> +       int i = 0;
> +
> +       /*
> +        * s->cur and s->end are int, so the kfunc's s->cur + 1 >= s->end check is a signed 32-bit
> +        * comparison of (s->cur + 1) against s->end and needs no sign extension.
> +        */

let's keep all the explanations in C code implementation, and here
just have brief C code we are implementing using BPF instructions


> +       insn_buf[i++] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, 0);
> +       insn_buf[i++] = BPF_ALU32_IMM(BPF_ADD, BPF_REG_0, 1);
> +       insn_buf[i++] = BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 4);
> +       /* if ((s32)(s->cur + 1) >= (s32)s->end) goto done; */
> +       insn_buf[i++] = BPF_JMP32_REG(BPF_JSGE, BPF_REG_0, BPF_REG_2, 3);
> +       /* s->cur++; return &s->cur; */
> +       insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0);
> +       insn_buf[i++] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
> +       insn_buf[i++] = BPF_JMP_A(2);
> +       /* done: s->cur = s->end = 0; return NULL; */
> +       insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0);
> +       insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, 0);
> +
> +       return i;
> +}
> +
>  int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
>                      struct bpf_insn *insn_buf, int insn_idx, int *cnt)
>  {
> @@ -19941,6 +19969,8 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
>                 *cnt = 6;
>         } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_new]) {
>                 *cnt = inline_bpf_iter_num_new(insn_buf);
> +       } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_next]) {
> +               *cnt = inline_bpf_iter_num_next(insn_buf);
>         }
>
>         if (env->insn_aux_data[insn_idx].arg_prog) {
> --
> 2.53.0-Meta
>