Commit 1ef22b68 authored by Andrii Nakryiko's avatar Andrii Nakryiko Committed by Alexei Starovoitov

bpf: maintain bitmasks across all active frames in __mark_chain_precision

Teach __mark_chain_precision logic to maintain register/stack masks
across all active frames when going from child state to parent state.
Currently this should be mostly no-op, as precision backtracking usually
bails out when encountering subprog entry/exit.

It's not very apparent from the diff due to increased indentation, but
the logic remains the same, except everything is done on specific `fr`
frame index. Calls to bt_clear_reg() and bt_clear_slot() are replaced
with frame-specific bt_clear_frame_reg() and bt_clear_frame_slot(),
where frame index is passed explicitly, instead of using current frame
number.

We also adjust logging to emit affected frame number. And we also add
better logging of human-readable register and stack slot masks, similar
to previous patch.
Signed-off-by: default avatarAndrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/r/20230505043317.3629845-6-andrii@kernel.orgSigned-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent d9439c21
...@@ -3736,7 +3736,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int frame, int r ...@@ -3736,7 +3736,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int frame, int r
struct bpf_func_state *func; struct bpf_func_state *func;
struct bpf_reg_state *reg; struct bpf_reg_state *reg;
bool skip_first = true; bool skip_first = true;
int i, err; int i, fr, err;
if (!env->bpf_capable) if (!env->bpf_capable)
return 0; return 0;
...@@ -3845,21 +3845,22 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int frame, int r ...@@ -3845,21 +3845,22 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int frame, int r
if (!st) if (!st)
break; break;
func = st->frame[frame]; for (fr = bt->frame; fr >= 0; fr--) {
bitmap_from_u64(mask, bt_reg_mask(bt)); func = st->frame[fr];
bitmap_from_u64(mask, bt_frame_reg_mask(bt, fr));
for_each_set_bit(i, mask, 32) { for_each_set_bit(i, mask, 32) {
reg = &func->regs[i]; reg = &func->regs[i];
if (reg->type != SCALAR_VALUE) { if (reg->type != SCALAR_VALUE) {
bt_clear_reg(bt, i); bt_clear_frame_reg(bt, fr, i);
continue; continue;
} }
if (reg->precise) if (reg->precise)
bt_clear_reg(bt, i); bt_clear_frame_reg(bt, fr, i);
else else
reg->precise = true; reg->precise = true;
} }
bitmap_from_u64(mask, bt_stack_mask(bt)); bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr));
for_each_set_bit(i, mask, 64) { for_each_set_bit(i, mask, 64) {
if (i >= func->allocated_stack / BPF_REG_SIZE) { if (i >= func->allocated_stack / BPF_REG_SIZE) {
/* the sequence of instructions: /* the sequence of instructions:
...@@ -3881,21 +3882,26 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int frame, int r ...@@ -3881,21 +3882,26 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int frame, int r
} }
if (!is_spilled_scalar_reg(&func->stack[i])) { if (!is_spilled_scalar_reg(&func->stack[i])) {
bt_clear_slot(bt, i); bt_clear_frame_slot(bt, fr, i);
continue; continue;
} }
reg = &func->stack[i].spilled_ptr; reg = &func->stack[i].spilled_ptr;
if (reg->precise) if (reg->precise)
bt_clear_slot(bt, i); bt_clear_frame_slot(bt, fr, i);
else else
reg->precise = true; reg->precise = true;
} }
if (env->log.level & BPF_LOG_LEVEL2) { if (env->log.level & BPF_LOG_LEVEL2) {
verbose(env, "parent %s regs=%x stack=%llx marks:", fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
!bt_empty(bt) ? "didn't have" : "already had", bt_frame_reg_mask(bt, fr));
bt_reg_mask(bt), bt_stack_mask(bt)); verbose(env, "mark_precise: frame%d: parent state regs=%s ",
fr, env->tmp_str_buf);
fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
bt_frame_stack_mask(bt, fr));
verbose(env, "stack=%s: ", env->tmp_str_buf);
print_verifier_state(env, func, true); print_verifier_state(env, func, true);
} }
}
if (bt_empty(bt)) if (bt_empty(bt))
break; break;
......
...@@ -44,7 +44,7 @@ ...@@ -44,7 +44,7 @@
mark_precise: frame0: regs=r2 stack= before 23\ mark_precise: frame0: regs=r2 stack= before 23\
mark_precise: frame0: regs=r2 stack= before 22\ mark_precise: frame0: regs=r2 stack= before 22\
mark_precise: frame0: regs=r2 stack= before 20\ mark_precise: frame0: regs=r2 stack= before 20\
parent didn't have regs=4 stack=0 marks:\ mark_precise: frame0: parent state regs=r2 stack=:\
mark_precise: frame0: last_idx 19 first_idx 10\ mark_precise: frame0: last_idx 19 first_idx 10\
mark_precise: frame0: regs=r2 stack= before 19\ mark_precise: frame0: regs=r2 stack= before 19\
mark_precise: frame0: regs=r9 stack= before 18\ mark_precise: frame0: regs=r9 stack= before 18\
...@@ -55,7 +55,7 @@ ...@@ -55,7 +55,7 @@
mark_precise: frame0: regs=r9 stack= before 12\ mark_precise: frame0: regs=r9 stack= before 12\
mark_precise: frame0: regs=r9 stack= before 11\ mark_precise: frame0: regs=r9 stack= before 11\
mark_precise: frame0: regs=r9 stack= before 10\ mark_precise: frame0: regs=r9 stack= before 10\
parent already had regs=0 stack=0 marks:", mark_precise: frame0: parent state regs= stack=:",
}, },
{ {
"precise: test 2", "precise: test 2",
...@@ -104,15 +104,15 @@ ...@@ -104,15 +104,15 @@
mark_precise: frame0: regs=r2 stack= before 24\ mark_precise: frame0: regs=r2 stack= before 24\
mark_precise: frame0: regs=r2 stack= before 23\ mark_precise: frame0: regs=r2 stack= before 23\
mark_precise: frame0: regs=r2 stack= before 22\ mark_precise: frame0: regs=r2 stack= before 22\
parent didn't have regs=4 stack=0 marks:\ mark_precise: frame0: parent state regs=r2 stack=:\
mark_precise: frame0: last_idx 20 first_idx 20\ mark_precise: frame0: last_idx 20 first_idx 20\
mark_precise: frame0: regs=r2 stack= before 20\ mark_precise: frame0: regs=r2 stack= before 20\
parent didn't have regs=4 stack=0 marks:\ mark_precise: frame0: parent state regs=r2 stack=:\
mark_precise: frame0: last_idx 19 first_idx 17\ mark_precise: frame0: last_idx 19 first_idx 17\
mark_precise: frame0: regs=r2 stack= before 19\ mark_precise: frame0: regs=r2 stack= before 19\
mark_precise: frame0: regs=r9 stack= before 18\ mark_precise: frame0: regs=r9 stack= before 18\
mark_precise: frame0: regs=r8,r9 stack= before 17\ mark_precise: frame0: regs=r8,r9 stack= before 17\
parent already had regs=0 stack=0 marks:", mark_precise: frame0: parent state regs= stack=:",
}, },
{ {
"precise: cross frame pruning", "precise: cross frame pruning",
...@@ -153,14 +153,14 @@ ...@@ -153,14 +153,14 @@
.prog_type = BPF_PROG_TYPE_XDP, .prog_type = BPF_PROG_TYPE_XDP,
.flags = BPF_F_TEST_STATE_FREQ, .flags = BPF_F_TEST_STATE_FREQ,
.errstr = "mark_precise: frame0: last_idx 5 first_idx 5\ .errstr = "mark_precise: frame0: last_idx 5 first_idx 5\
parent didn't have regs=10 stack=0 marks:\ mark_precise: frame0: parent state regs=r4 stack=:\
mark_precise: frame0: last_idx 4 first_idx 2\ mark_precise: frame0: last_idx 4 first_idx 2\
mark_precise: frame0: regs=r4 stack= before 4\ mark_precise: frame0: regs=r4 stack= before 4\
mark_precise: frame0: regs=r4 stack= before 3\ mark_precise: frame0: regs=r4 stack= before 3\
mark_precise: frame0: regs= stack=-8 before 2\ mark_precise: frame0: regs= stack=-8 before 2\
mark_precise: frame0: falling back to forcing all scalars precise\ mark_precise: frame0: falling back to forcing all scalars precise\
mark_precise: frame0: last_idx 5 first_idx 5\ mark_precise: frame0: last_idx 5 first_idx 5\
parent didn't have regs=1 stack=0 marks:", mark_precise: frame0: parent state regs=r0 stack=:",
.result = VERBOSE_ACCEPT, .result = VERBOSE_ACCEPT,
.retval = -1, .retval = -1,
}, },
...@@ -179,7 +179,7 @@ ...@@ -179,7 +179,7 @@
.prog_type = BPF_PROG_TYPE_XDP, .prog_type = BPF_PROG_TYPE_XDP,
.flags = BPF_F_TEST_STATE_FREQ, .flags = BPF_F_TEST_STATE_FREQ,
.errstr = "mark_precise: frame0: last_idx 6 first_idx 6\ .errstr = "mark_precise: frame0: last_idx 6 first_idx 6\
parent didn't have regs=10 stack=0 marks:\ mark_precise: frame0: parent state regs=r4 stack=:\
mark_precise: frame0: last_idx 5 first_idx 3\ mark_precise: frame0: last_idx 5 first_idx 3\
mark_precise: frame0: regs=r4 stack= before 5\ mark_precise: frame0: regs=r4 stack= before 5\
mark_precise: frame0: regs=r4 stack= before 4\ mark_precise: frame0: regs=r4 stack= before 4\
...@@ -188,7 +188,7 @@ ...@@ -188,7 +188,7 @@
force_precise: frame0: forcing r0 to be precise\ force_precise: frame0: forcing r0 to be precise\
force_precise: frame0: forcing r0 to be precise\ force_precise: frame0: forcing r0 to be precise\
mark_precise: frame0: last_idx 6 first_idx 6\ mark_precise: frame0: last_idx 6 first_idx 6\
parent didn't have regs=1 stack=0 marks:\ mark_precise: frame0: parent state regs=r0 stack=:\
mark_precise: frame0: last_idx 5 first_idx 3\ mark_precise: frame0: last_idx 5 first_idx 3\
mark_precise: frame0: regs=r0 stack= before 5", mark_precise: frame0: regs=r0 stack= before 5",
.result = VERBOSE_ACCEPT, .result = VERBOSE_ACCEPT,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment