[PATCH bpf-next v5 02/11] bpf: Add helpers to describe the R0:R2 return register pair

LLVM 23 added support for returning a value in two registers for an
__int128, or a struct/union whose size is greater than 8 but not more than
16 bytes: such a value comes back in the R0:R2 register pair, with R2
holding the upper half. See LLVM patches [1] and [2].

Later patches teach the JIT, precision backtracking, live register analysis
and the verifier itself about that convention. All of them need to answer
the same question: does this call return its value in a register pair? Add
the shared helpers up front so they can be used in subsequent patches:

 - bpf_ret_reg_pair() for a BPF subprogram, answered from a per-subprogram
   flag that bpf_compute_subprog_ret_regs() derives once from the BTF
   prototype;
 - bpf_kfunc_ret_reg_pair() for a kfunc call site, answered from the
   btf_func_model that bpf_add_kfunc_call() already built, which is the
   same ret_size the JIT keys the second return register off, so the
   verifier and the generated code cannot disagree.

It also sets jit_required, from the same place the convention is decided
rather than from each site that later comes to model R2, so that no such
site can be missed.

  [1] https://github.com/llvm/llvm-project/pull/190894
  [2] https://github.com/llvm/llvm-project/pull/206876

Signed-off-by: Yonghong Song <[email protected]>
---
 include/linux/bpf_verifier.h |  9 +++++
 kernel/bpf/verifier.c        | 73 ++++++++++++++++++++++++++++++------
 2 files changed, 71 insertions(+), 11 deletions(-)

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 27b43fda9b17..bffd32dca068 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -814,6 +814,8 @@ struct bpf_subprog_info {
 	bool is_async_cb: 1;
 	bool is_exception_cb: 1;
 	bool args_cached: 1;
+	/* true if the return value is passed in the R0:R2 register pair */
+	bool ret_reg_pair: 1;
 	/* true if bpf_fastcall stack region is used by functions that can't be inlined */
 	bool keep_fastcall_stack: 1;
 	bool changes_pkt_data: 1;
@@ -1048,6 +1050,13 @@ static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env
 	return &env->subprog_info[subprog];
 }
 
+static inline bool bpf_ret_reg_pair(struct bpf_verifier_env *env, int subprog)
+{
+	return subprog_info(env, subprog)->ret_reg_pair;
+}
+
+bool bpf_kfunc_ret_reg_pair(struct bpf_verifier_env *env, struct bpf_insn *insn);
+
 struct bpf_call_summary {
 	u8 num_params;
 	bool is_void;
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 03570c693d35..57d14480ded2 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -382,27 +382,60 @@ bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog)
 	return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL;
 }
 
-static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog)
+static const struct btf_type *subprog_ret_type(struct bpf_verifier_env *env, int subprog)
 {
-	const struct btf_type *type, *func, *func_proto;
+	const struct btf_type *func, *func_proto;
 	const struct btf *btf = env->prog->aux->btf;
 	u32 btf_id;
 
+	if (!btf || !env->prog->aux->func_info)
+		return NULL;
+
 	btf_id = env->prog->aux->func_info[subprog].type_id;
 
+	/* Both already validated by prepare_btf_func() at prog load. */
 	func = btf_type_by_id(btf, btf_id);
-	if (verifier_bug_if(!func, env, "btf_id %u not found", btf_id))
-		return false;
-
 	func_proto = btf_type_by_id(btf, func->type);
-	if (!func_proto)
-		return false;
 
-	type = btf_type_skip_modifiers(btf, func_proto->type, NULL);
-	if (!type)
-		return false;
+	return btf_type_skip_modifiers(btf, func_proto->type, NULL);
+}
+
+static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog)
+{
+	const struct btf_type *type = subprog_ret_type(env, subprog);
+
+	return type && btf_type_is_void(type);
+}
+
+static u32 ret_regs_cnt(u32 size)
+{
+	return size > 8 && size <= 16 ? 2 : 1;
+}
+
+static void bpf_compute_subprog_ret_regs(struct bpf_verifier_env *env)
+{
+	const struct btf *btf = env->prog->aux->btf;
+	const struct btf_type *type;
+	int subprog;
+	u32 size;
 
-	return btf_type_is_void(type);
+	for (subprog = 0; subprog < env->subprog_cnt; subprog++) {
+		type = subprog_ret_type(env, subprog);
+		if (!type || !(btf_type_is_struct(type) || btf_type_is_scalar(type)))
+			continue;
+		if (IS_ERR(btf_resolve_size(btf, type, &size)))
+			continue;
+		if (ret_regs_cnt(size) > 1) {
+			subprog_info(env, subprog)->ret_reg_pair = true;
+			/*
+			 * The R0:R2 return convention is only implemented in
+			 * the JIT: the interpreter propagates BPF_R0 alone out
+			 * of a subprogram, so a caller reading R2 would see a
+			 * stale value.
+			 */
+			env->prog->jit_required = 1;
+		}
+	}
 }
 
 static const char *subprog_name(const struct bpf_verifier_env *env, int subprog)
@@ -2456,6 +2489,21 @@ find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset)
 		       sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off);
 }
 
+/*
+ * True if the kfunc called by @insn returns its value in the R0:R2 pair.
+ * Reads the same btf_func_model.ret_size that bpf_add_kfunc_call() validated
+ * and that the JIT keys the second return register off, so the verifier and
+ * the generated code cannot disagree about the convention.
+ */
+bool bpf_kfunc_ret_reg_pair(struct bpf_verifier_env *env, struct bpf_insn *insn)
+{
+	const struct bpf_kfunc_desc *desc;
+
+	desc = find_kfunc_desc(env->prog, insn->imm, insn->off);
+
+	return desc && ret_regs_cnt(desc->func_model.ret_size) > 1;
+}
+
 int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
 		       u16 btf_fd_idx, u8 **func_addr)
 {
@@ -20333,6 +20381,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	if (ret < 0)
 		goto skip_full_check;
 
+	/* must precede the first bpf_ret_reg_pair() user below */
+	bpf_compute_subprog_ret_regs(env);
+
 	ret = bpf_compute_live_registers(env);
 	if (ret < 0)
 		goto skip_full_check;
-- 
2.53.0-Meta