Skip to content

L
linux
Commit ae010757 authored by Eduard Zingerman's avatar Eduard Zingerman Committed by Alexei Starovoitov
Browse files
Options

bpf: rename nocsr -> bpf_fastcall in verifier 

Attribute used by LLVM implementation of the feature had been changed
from no_caller_saved_registers to bpf_fastcall (see [1]).
This commit replaces references to nocsr by references to bpf_fastcall
to keep LLVM and Kernel parts in sync.

[1] https://github.com/llvm/llvm-project/pull/105417Acked-by: default avatarYonghong Song <yonghong.song@linux.dev>
Signed-off-by: default avatarEduard Zingerman <eddyz87@gmail.com>
Link: https://lore.kernel.org/r/20240822084112.3257995-2-eddyz87@gmail.comSigned-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 6d641ca5
Show whitespace changes
Inline Side-by-side
Showing with 84 additions and 85 deletions
include/linux/bpf.h
View file @ ae010757
...@@ -808,12 +808,12 @@ struct bpf_func_proto { ...@@ -808,12 +808,12 @@ struct bpf_func_proto {
bool gpl_only; bool gpl_only;
bool pkt_access; bool pkt_access;
bool might_sleep; bool might_sleep;
/* set to true if helper follows contract for gcc/llvm /* set to true if helper follows contract for llvm
* attribute no_caller_saved_registers: * attribute bpf_fastcall:
* - void functions do not scratch r0 * - void functions do not scratch r0
* - functions taking N arguments scratch only registers r1-rN * - functions taking N arguments scratch only registers r1-rN
*/ */
bool allow_nocsr; bool allow_fastcall;
enum bpf_return_type ret_type; enum bpf_return_type ret_type;
union { union {
struct { struct {
......
include/linux/bpf_verifier.h
View file @ ae010757
...@@ -577,13 +577,13 @@ struct bpf_insn_aux_data { ...@@ -577,13 +577,13 @@ struct bpf_insn_aux_data {
bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */ bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */
u8 alu_state; /* used in combination with alu_limit */ u8 alu_state; /* used in combination with alu_limit */
/* true if STX or LDX instruction is a part of a spill/fill /* true if STX or LDX instruction is a part of a spill/fill
* pattern for a no_caller_saved_registers call. * pattern for a bpf_fastcall call.
*/ */
u8 nocsr_pattern:1; u8 fastcall_pattern:1;
/* for CALL instructions, a number of spill/fill pairs in the /* for CALL instructions, a number of spill/fill pairs in the
* no_caller_saved_registers pattern. * bpf_fastcall pattern.
*/ */
u8 nocsr_spills_num:3; u8 fastcall_spills_num:3;
/* below fields are initialized once */ /* below fields are initialized once */
unsigned int orig_idx; /* original instruction index */ unsigned int orig_idx; /* original instruction index */
...@@ -653,10 +653,10 @@ struct bpf_subprog_info { ...@@ -653,10 +653,10 @@ struct bpf_subprog_info {
u32 linfo_idx; /* The idx to the main_prog->aux->linfo */ u32 linfo_idx; /* The idx to the main_prog->aux->linfo */
u16 stack_depth; /* max. stack depth used by this function */ u16 stack_depth; /* max. stack depth used by this function */
u16 stack_extra; u16 stack_extra;
/* offsets in range [stack_depth .. nocsr_stack_off) /* offsets in range [stack_depth .. fastcall_stack_off)
* are used for no_caller_saved_registers spills and fills. * are used for bpf_fastcall spills and fills.
*/ */
s16 nocsr_stack_off; s16 fastcall_stack_off;
bool has_tail_call: 1; bool has_tail_call: 1;
bool tail_call_reachable: 1; bool tail_call_reachable: 1;
bool has_ld_abs: 1; bool has_ld_abs: 1;
...@@ -664,8 +664,8 @@ struct bpf_subprog_info { ...@@ -664,8 +664,8 @@ struct bpf_subprog_info {
bool is_async_cb: 1; bool is_async_cb: 1;
bool is_exception_cb: 1; bool is_exception_cb: 1;
bool args_cached: 1; bool args_cached: 1;
/* true if nocsr stack region is used by functions that can't be inlined */ /* true if bpf_fastcall stack region is used by functions that can't be inlined */
bool keep_nocsr_stack: 1; bool keep_fastcall_stack: 1;
u8 arg_cnt; u8 arg_cnt;
struct bpf_subprog_arg_info args[MAX_BPF_FUNC_REG_ARGS]; struct bpf_subprog_arg_info args[MAX_BPF_FUNC_REG_ARGS];
......
kernel/bpf/helpers.c
View file @ ae010757
...@@ -158,7 +158,7 @@ const struct bpf_func_proto bpf_get_smp_processor_id_proto = { ...@@ -158,7 +158,7 @@ const struct bpf_func_proto bpf_get_smp_processor_id_proto = {
.func = bpf_get_smp_processor_id, .func = bpf_get_smp_processor_id,
.gpl_only = false, .gpl_only = false,
.ret_type = RET_INTEGER, .ret_type = RET_INTEGER,
.allow_nocsr = true, .allow_fastcall = true,
}; };
BPF_CALL_0(bpf_get_numa_node_id) BPF_CALL_0(bpf_get_numa_node_id)
......
kernel/bpf/verifier.c
View file @ ae010757
...@@ -4579,28 +4579,28 @@ static int get_reg_width(struct bpf_reg_state *reg) ...@@ -4579,28 +4579,28 @@ static int get_reg_width(struct bpf_reg_state *reg)
return fls64(reg->umax_value); return fls64(reg->umax_value);
} }
/* See comment for mark_nocsr_pattern_for_call() */ /* See comment for mark_fastcall_pattern_for_call() */
static void check_nocsr_stack_contract(struct bpf_verifier_env *env, struct bpf_func_state *state, static void check_fastcall_stack_contract(struct bpf_verifier_env *env,
int insn_idx, int off) struct bpf_func_state *state, int insn_idx, int off)
{ {
struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno]; struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno];
struct bpf_insn_aux_data *aux = env->insn_aux_data; struct bpf_insn_aux_data *aux = env->insn_aux_data;
int i; int i;
if (subprog->nocsr_stack_off <= off || aux[insn_idx].nocsr_pattern) if (subprog->fastcall_stack_off <= off || aux[insn_idx].fastcall_pattern)
return; return;
/* access to the region [max_stack_depth .. nocsr_stack_off) /* access to the region [max_stack_depth .. fastcall_stack_off)
* from something that is not a part of the nocsr pattern, * from something that is not a part of the fastcall pattern,
* disable nocsr rewrites for current subprogram by setting * disable fastcall rewrites for current subprogram by setting
* nocsr_stack_off to a value smaller than any possible offset. * fastcall_stack_off to a value smaller than any possible offset.
*/ */
subprog->nocsr_stack_off = S16_MIN; subprog->fastcall_stack_off = S16_MIN;
/* reset nocsr aux flags within subprogram, /* reset fastcall aux flags within subprogram,
* happens at most once per subprogram * happens at most once per subprogram
*/ */
for (i = subprog->start; i < (subprog + 1)->start; ++i) { for (i = subprog->start; i < (subprog + 1)->start; ++i) {
aux[i].nocsr_spills_num = 0; aux[i].fastcall_spills_num = 0;
aux[i].nocsr_pattern = 0; aux[i].fastcall_pattern = 0;
} }
} }
...@@ -4652,7 +4652,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, ...@@ -4652,7 +4652,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
if (err) if (err)
return err; return err;
check_nocsr_stack_contract(env, state, insn_idx, off); check_fastcall_stack_contract(env, state, insn_idx, off);
mark_stack_slot_scratched(env, spi); mark_stack_slot_scratched(env, spi);
if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) { if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) {
bool reg_value_fits; bool reg_value_fits;
...@@ -4787,7 +4787,7 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, ...@@ -4787,7 +4787,7 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env,
return err; return err;
} }
check_nocsr_stack_contract(env, state, insn_idx, min_off); check_fastcall_stack_contract(env, state, insn_idx, min_off);
/* Variable offset writes destroy any spilled pointers in range. */ /* Variable offset writes destroy any spilled pointers in range. */
for (i = min_off; i < max_off; i++) { for (i = min_off; i < max_off; i++) {
u8 new_type, *stype; u8 new_type, *stype;
...@@ -4926,7 +4926,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, ...@@ -4926,7 +4926,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
reg = &reg_state->stack[spi].spilled_ptr; reg = &reg_state->stack[spi].spilled_ptr;
mark_stack_slot_scratched(env, spi); mark_stack_slot_scratched(env, spi);
check_nocsr_stack_contract(env, state, env->insn_idx, off); check_fastcall_stack_contract(env, state, env->insn_idx, off);
if (is_spilled_reg(&reg_state->stack[spi])) { if (is_spilled_reg(&reg_state->stack[spi])) {
u8 spill_size = 1; u8 spill_size = 1;
...@@ -5087,7 +5087,7 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env, ...@@ -5087,7 +5087,7 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env,
min_off = reg->smin_value + off; min_off = reg->smin_value + off;
max_off = reg->smax_value + off; max_off = reg->smax_value + off;
mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);
check_nocsr_stack_contract(env, ptr_state, env->insn_idx, min_off); check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off);
return 0; return 0;
} }
...@@ -6804,13 +6804,13 @@ static int check_stack_slot_within_bounds(struct bpf_verifier_env *env, ...@@ -6804,13 +6804,13 @@ static int check_stack_slot_within_bounds(struct bpf_verifier_env *env,
struct bpf_insn_aux_data *aux = &env->insn_aux_data[env->insn_idx]; struct bpf_insn_aux_data *aux = &env->insn_aux_data[env->insn_idx];
int min_valid_off, max_bpf_stack; int min_valid_off, max_bpf_stack;
/* If accessing instruction is a spill/fill from nocsr pattern, /* If accessing instruction is a spill/fill from bpf_fastcall pattern,
* add room for all caller saved registers below MAX_BPF_STACK. * add room for all caller saved registers below MAX_BPF_STACK.
* In case if nocsr rewrite won't happen maximal stack depth * In case if bpf_fastcall rewrite won't happen maximal stack depth
* would be checked by check_max_stack_depth_subprog(). * would be checked by check_max_stack_depth_subprog().
*/ */
max_bpf_stack = MAX_BPF_STACK; max_bpf_stack = MAX_BPF_STACK;
if (aux->nocsr_pattern) if (aux->fastcall_pattern)
max_bpf_stack += CALLER_SAVED_REGS * BPF_REG_SIZE; max_bpf_stack += CALLER_SAVED_REGS * BPF_REG_SIZE;
if (t == BPF_WRITE || env->allow_uninit_stack) if (t == BPF_WRITE || env->allow_uninit_stack)
...@@ -16118,12 +16118,12 @@ static int visit_func_call_insn(int t, struct bpf_insn *insns, ...@@ -16118,12 +16118,12 @@ static int visit_func_call_insn(int t, struct bpf_insn *insns,
/* Return a bitmask specifying which caller saved registers are /* Return a bitmask specifying which caller saved registers are
* clobbered by a call to a helper *as if* this helper follows * clobbered by a call to a helper *as if* this helper follows
* no_caller_saved_registers contract: * bpf_fastcall contract:
* - includes R0 if function is non-void; * - includes R0 if function is non-void;
* - includes R1-R5 if corresponding parameter has is described * - includes R1-R5 if corresponding parameter has is described
* in the function prototype. * in the function prototype.
*/ */
static u32 helper_nocsr_clobber_mask(const struct bpf_func_proto *fn) static u32 helper_fastcall_clobber_mask(const struct bpf_func_proto *fn)
{ {
u8 mask; u8 mask;
int i; int i;
...@@ -16138,8 +16138,8 @@ static u32 helper_nocsr_clobber_mask(const struct bpf_func_proto *fn) ...@@ -16138,8 +16138,8 @@ static u32 helper_nocsr_clobber_mask(const struct bpf_func_proto *fn)
} }
/* True if do_misc_fixups() replaces calls to helper number 'imm', /* True if do_misc_fixups() replaces calls to helper number 'imm',
* replacement patch is presumed to follow no_caller_saved_registers contract * replacement patch is presumed to follow bpf_fastcall contract
* (see mark_nocsr_pattern_for_call() below). * (see mark_fastcall_pattern_for_call() below).
*/ */
static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm) static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
{ {
...@@ -16153,7 +16153,7 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm) ...@@ -16153,7 +16153,7 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
} }
} }
/* GCC and LLVM define a no_caller_saved_registers function attribute. /* LLVM define a bpf_fastcall function attribute.
* This attribute means that function scratches only some of * This attribute means that function scratches only some of
* the caller saved registers defined by ABI. * the caller saved registers defined by ABI.
* For BPF the set of such registers could be defined as follows: * For BPF the set of such registers could be defined as follows:
...@@ -16163,13 +16163,12 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm) ...@@ -16163,13 +16163,12 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
* *
* The contract between kernel and clang allows to simultaneously use * The contract between kernel and clang allows to simultaneously use
* such functions and maintain backwards compatibility with old * such functions and maintain backwards compatibility with old
* kernels that don't understand no_caller_saved_registers calls * kernels that don't understand bpf_fastcall calls:
* (nocsr for short):
* *
* - for nocsr calls clang allocates registers as-if relevant r0-r5 * - for bpf_fastcall calls clang allocates registers as-if relevant r0-r5
* registers are not scratched by the call; * registers are not scratched by the call;
* *
* - as a post-processing step, clang visits each nocsr call and adds * - as a post-processing step, clang visits each bpf_fastcall call and adds
* spill/fill for every live r0-r5; * spill/fill for every live r0-r5;
* *
* - stack offsets used for the spill/fill are allocated as lowest * - stack offsets used for the spill/fill are allocated as lowest
...@@ -16177,11 +16176,11 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm) ...@@ -16177,11 +16176,11 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
* purposes; * purposes;
* *
* - when kernel loads a program, it looks for such patterns * - when kernel loads a program, it looks for such patterns
* (nocsr function surrounded by spills/fills) and checks if * (bpf_fastcall function surrounded by spills/fills) and checks if
* spill/fill stack offsets are used exclusively in nocsr patterns; * spill/fill stack offsets are used exclusively in fastcall patterns;
* *
* - if so, and if verifier or current JIT inlines the call to the * - if so, and if verifier or current JIT inlines the call to the
* nocsr function (e.g. a helper call), kernel removes unnecessary * bpf_fastcall function (e.g. a helper call), kernel removes unnecessary
* spill/fill pairs; * spill/fill pairs;
* *
* - when old kernel loads a program, presence of spill/fill pairs * - when old kernel loads a program, presence of spill/fill pairs
...@@ -16200,22 +16199,22 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm) ...@@ -16200,22 +16199,22 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
* r0 += r2; * r0 += r2;
* exit; * exit;
* *
* The purpose of mark_nocsr_pattern_for_call is to: * The purpose of mark_fastcall_pattern_for_call is to:
* - look for such patterns; * - look for such patterns;
* - mark spill and fill instructions in env->insn_aux_data[*].nocsr_pattern; * - mark spill and fill instructions in env->insn_aux_data[*].fastcall_pattern;
* - mark set env->insn_aux_data[*].nocsr_spills_num for call instruction; * - mark set env->insn_aux_data[*].fastcall_spills_num for call instruction;
* - update env->subprog_info[*]->nocsr_stack_off to find an offset * - update env->subprog_info[*]->fastcall_stack_off to find an offset
* at which nocsr spill/fill stack slots start; * at which bpf_fastcall spill/fill stack slots start;
* - update env->subprog_info[*]->keep_nocsr_stack. * - update env->subprog_info[*]->keep_fastcall_stack.
* *
* The .nocsr_pattern and .nocsr_stack_off are used by * The .fastcall_pattern and .fastcall_stack_off are used by
* check_nocsr_stack_contract() to check if every stack access to * check_fastcall_stack_contract() to check if every stack access to
* nocsr spill/fill stack slot originates from spill/fill * fastcall spill/fill stack slot originates from spill/fill
* instructions, members of nocsr patterns. * instructions, members of fastcall patterns.
* *
* If such condition holds true for a subprogram, nocsr patterns could * If such condition holds true for a subprogram, fastcall patterns could
* be rewritten by remove_nocsr_spills_fills(). * be rewritten by remove_fastcall_spills_fills().
* Otherwise nocsr patterns are not changed in the subprogram * Otherwise bpf_fastcall patterns are not changed in the subprogram
* (code, presumably, generated by an older clang version). * (code, presumably, generated by an older clang version).
* *
* For example, it is *not* safe to remove spill/fill below: * For example, it is *not* safe to remove spill/fill below:
...@@ -16228,7 +16227,7 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm) ...@@ -16228,7 +16227,7 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
* r0 += r1; exit; * r0 += r1; exit;
* exit; * exit;
*/ */
static void mark_nocsr_pattern_for_call(struct bpf_verifier_env *env, static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env,
struct bpf_subprog_info *subprog, struct bpf_subprog_info *subprog,
int insn_idx, s16 lowest_off) int insn_idx, s16 lowest_off)
{ {
...@@ -16245,8 +16244,8 @@ static void mark_nocsr_pattern_for_call(struct bpf_verifier_env *env, ...@@ -16245,8 +16244,8 @@ static void mark_nocsr_pattern_for_call(struct bpf_verifier_env *env,
if (get_helper_proto(env, call->imm, &fn) < 0) if (get_helper_proto(env, call->imm, &fn) < 0)
/* error would be reported later */ /* error would be reported later */
return; return;
clobbered_regs_mask = helper_nocsr_clobber_mask(fn); clobbered_regs_mask = helper_fastcall_clobber_mask(fn);
can_be_inlined = fn->allow_nocsr && can_be_inlined = fn->allow_fastcall &&
(verifier_inlines_helper_call(env, call->imm) || (verifier_inlines_helper_call(env, call->imm) ||
bpf_jit_inlines_helper_call(call->imm)); bpf_jit_inlines_helper_call(call->imm));
} }
...@@ -16289,36 +16288,36 @@ static void mark_nocsr_pattern_for_call(struct bpf_verifier_env *env, ...@@ -16289,36 +16288,36 @@ static void mark_nocsr_pattern_for_call(struct bpf_verifier_env *env,
if (stx->off != off || ldx->off != off) if (stx->off != off || ldx->off != off)
break; break;
expected_regs_mask &= ~BIT(stx->src_reg); expected_regs_mask &= ~BIT(stx->src_reg);
env->insn_aux_data[insn_idx - i].nocsr_pattern = 1; env->insn_aux_data[insn_idx - i].fastcall_pattern = 1;
env->insn_aux_data[insn_idx + i].nocsr_pattern = 1; env->insn_aux_data[insn_idx + i].fastcall_pattern = 1;
} }
if (i == 1) if (i == 1)
return; return;
/* Conditionally set 'nocsr_spills_num' to allow forward /* Conditionally set 'fastcall_spills_num' to allow forward
* compatibility when more helper functions are marked as * compatibility when more helper functions are marked as
* nocsr at compile time than current kernel supports, e.g: * bpf_fastcall at compile time than current kernel supports, e.g:
* *
* 1: *(u64 *)(r10 - 8) = r1 * 1: *(u64 *)(r10 - 8) = r1
* 2: call A ;; assume A is nocsr for current kernel * 2: call A ;; assume A is bpf_fastcall for current kernel
* 3: r1 = *(u64 *)(r10 - 8) * 3: r1 = *(u64 *)(r10 - 8)
* 4: *(u64 *)(r10 - 8) = r1 * 4: *(u64 *)(r10 - 8) = r1
* 5: call B ;; assume B is not nocsr for current kernel * 5: call B ;; assume B is not bpf_fastcall for current kernel
* 6: r1 = *(u64 *)(r10 - 8) * 6: r1 = *(u64 *)(r10 - 8)
* *
* There is no need to block nocsr rewrite for such program. * There is no need to block bpf_fastcall rewrite for such program.
* Set 'nocsr_pattern' for both calls to keep check_nocsr_stack_contract() happy, * Set 'fastcall_pattern' for both calls to keep check_fastcall_stack_contract() happy,
* don't set 'nocsr_spills_num' for call B so that remove_nocsr_spills_fills() * don't set 'fastcall_spills_num' for call B so that remove_fastcall_spills_fills()
* does not remove spill/fill pair {4,6}. * does not remove spill/fill pair {4,6}.
*/ */
if (can_be_inlined) if (can_be_inlined)
env->insn_aux_data[insn_idx].nocsr_spills_num = i - 1; env->insn_aux_data[insn_idx].fastcall_spills_num = i - 1;
else else
subprog->keep_nocsr_stack = 1; subprog->keep_fastcall_stack = 1;
subprog->nocsr_stack_off = min(subprog->nocsr_stack_off, off); subprog->fastcall_stack_off = min(subprog->fastcall_stack_off, off);
} }
static int mark_nocsr_patterns(struct bpf_verifier_env *env) static int mark_fastcall_patterns(struct bpf_verifier_env *env)
{ {
struct bpf_subprog_info *subprog = env->subprog_info; struct bpf_subprog_info *subprog = env->subprog_info;
struct bpf_insn *insn; struct bpf_insn *insn;
...@@ -16335,12 +16334,12 @@ static int mark_nocsr_patterns(struct bpf_verifier_env *env) ...@@ -16335,12 +16334,12 @@ static int mark_nocsr_patterns(struct bpf_verifier_env *env)
continue; continue;
lowest_off = min(lowest_off, insn->off); lowest_off = min(lowest_off, insn->off);
} }
/* use this offset to find nocsr patterns */ /* use this offset to find fastcall patterns */
for (i = subprog->start; i < (subprog + 1)->start; ++i) { for (i = subprog->start; i < (subprog + 1)->start; ++i) {
insn = env->prog->insnsi + i; insn = env->prog->insnsi + i;
if (insn->code != (BPF_JMP | BPF_CALL)) if (insn->code != (BPF_JMP | BPF_CALL))
continue; continue;
mark_nocsr_pattern_for_call(env, subprog, i, lowest_off); mark_fastcall_pattern_for_call(env, subprog, i, lowest_off);
} }
} }
return 0; return 0;
...@@ -21244,10 +21243,10 @@ static int optimize_bpf_loop(struct bpf_verifier_env *env) ...@@ -21244,10 +21243,10 @@ static int optimize_bpf_loop(struct bpf_verifier_env *env)
return 0; return 0;
} }
/* Remove unnecessary spill/fill pairs, members of nocsr pattern, /* Remove unnecessary spill/fill pairs, members of fastcall pattern,
* adjust subprograms stack depth when possible. * adjust subprograms stack depth when possible.
*/ */
static int remove_nocsr_spills_fills(struct bpf_verifier_env *env) static int remove_fastcall_spills_fills(struct bpf_verifier_env *env)
{ {
struct bpf_subprog_info *subprog = env->subprog_info; struct bpf_subprog_info *subprog = env->subprog_info;
struct bpf_insn_aux_data *aux = env->insn_aux_data; struct bpf_insn_aux_data *aux = env->insn_aux_data;
...@@ -21258,8 +21257,8 @@ static int remove_nocsr_spills_fills(struct bpf_verifier_env *env) ...@@ -21258,8 +21257,8 @@ static int remove_nocsr_spills_fills(struct bpf_verifier_env *env)
int i, j; int i, j;
for (i = 0; i < insn_cnt; i++, insn++) { for (i = 0; i < insn_cnt; i++, insn++) {
if (aux[i].nocsr_spills_num > 0) { if (aux[i].fastcall_spills_num > 0) {
spills_num = aux[i].nocsr_spills_num; spills_num = aux[i].fastcall_spills_num;
/* NOPs would be removed by opt_remove_nops() */ /* NOPs would be removed by opt_remove_nops() */
for (j = 1; j <= spills_num; ++j) { for (j = 1; j <= spills_num; ++j) {
*(insn - j) = NOP; *(insn - j) = NOP;
...@@ -21268,8 +21267,8 @@ static int remove_nocsr_spills_fills(struct bpf_verifier_env *env) ...@@ -21268,8 +21267,8 @@ static int remove_nocsr_spills_fills(struct bpf_verifier_env *env)
modified = true; modified = true;
} }
if ((subprog + 1)->start == i + 1) { if ((subprog + 1)->start == i + 1) {
if (modified && !subprog->keep_nocsr_stack) if (modified && !subprog->keep_fastcall_stack)
subprog->stack_depth = -subprog->nocsr_stack_off; subprog->stack_depth = -subprog->fastcall_stack_off;
subprog++; subprog++;
modified = false; modified = false;
} }
...@@ -22192,7 +22191,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 ...@@ -22192,7 +22191,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
if (ret < 0) if (ret < 0)
goto skip_full_check; goto skip_full_check;
ret = mark_nocsr_patterns(env); ret = mark_fastcall_patterns(env);
if (ret < 0) if (ret < 0)
goto skip_full_check; goto skip_full_check;
...@@ -22209,7 +22208,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 ...@@ -22209,7 +22208,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
* allocate additional slots. * allocate additional slots.
*/ */
if (ret == 0) if (ret == 0)
ret = remove_nocsr_spills_fills(env); ret = remove_fastcall_spills_fills(env);
if (ret == 0) if (ret == 0)
ret = check_max_stack_depth(env); ret = check_max_stack_depth(env);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7