Commit 163541e6 authored by Jiong Wang's avatar Jiong Wang Committed by Alexei Starovoitov

arm: bpf: eliminate zero extension code-gen

Cc: Shubham Bansal <illusionist.neo@gmail.com>
Signed-off-by: default avatarJiong Wang <jiong.wang@netronome.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 9d120b41
...@@ -736,6 +736,7 @@ static inline void emit_a32_alu_r64(const bool is64, const s8 dst[], ...@@ -736,6 +736,7 @@ static inline void emit_a32_alu_r64(const bool is64, const s8 dst[],
/* ALU operation */ /* ALU operation */
emit_alu_r(rd[1], rs, true, false, op, ctx); emit_alu_r(rd[1], rs, true, false, op, ctx);
if (!ctx->prog->aux->verifier_zext)
emit_a32_mov_i(rd[0], 0, ctx); emit_a32_mov_i(rd[0], 0, ctx);
} }
...@@ -758,6 +759,7 @@ static inline void emit_a32_mov_r64(const bool is64, const s8 dst[], ...@@ -758,6 +759,7 @@ static inline void emit_a32_mov_r64(const bool is64, const s8 dst[],
struct jit_ctx *ctx) { struct jit_ctx *ctx) {
if (!is64) { if (!is64) {
emit_a32_mov_r(dst_lo, src_lo, ctx); emit_a32_mov_r(dst_lo, src_lo, ctx);
if (!ctx->prog->aux->verifier_zext)
/* Zero out high 4 bytes */ /* Zero out high 4 bytes */
emit_a32_mov_i(dst_hi, 0, ctx); emit_a32_mov_i(dst_hi, 0, ctx);
} else if (__LINUX_ARM_ARCH__ < 6 && } else if (__LINUX_ARM_ARCH__ < 6 &&
...@@ -1060,16 +1062,19 @@ static inline void emit_ldx_r(const s8 dst[], const s8 src, ...@@ -1060,16 +1062,19 @@ static inline void emit_ldx_r(const s8 dst[], const s8 src,
case BPF_B: case BPF_B:
/* Load a Byte */ /* Load a Byte */
emit(ARM_LDRB_I(rd[1], rm, off), ctx); emit(ARM_LDRB_I(rd[1], rm, off), ctx);
if (!ctx->prog->aux->verifier_zext)
emit_a32_mov_i(rd[0], 0, ctx); emit_a32_mov_i(rd[0], 0, ctx);
break; break;
case BPF_H: case BPF_H:
/* Load a HalfWord */ /* Load a HalfWord */
emit(ARM_LDRH_I(rd[1], rm, off), ctx); emit(ARM_LDRH_I(rd[1], rm, off), ctx);
if (!ctx->prog->aux->verifier_zext)
emit_a32_mov_i(rd[0], 0, ctx); emit_a32_mov_i(rd[0], 0, ctx);
break; break;
case BPF_W: case BPF_W:
/* Load a Word */ /* Load a Word */
emit(ARM_LDR_I(rd[1], rm, off), ctx); emit(ARM_LDR_I(rd[1], rm, off), ctx);
if (!ctx->prog->aux->verifier_zext)
emit_a32_mov_i(rd[0], 0, ctx); emit_a32_mov_i(rd[0], 0, ctx);
break; break;
case BPF_DW: case BPF_DW:
...@@ -1359,6 +1364,11 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) ...@@ -1359,6 +1364,11 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
case BPF_ALU64 | BPF_MOV | BPF_X: case BPF_ALU64 | BPF_MOV | BPF_X:
switch (BPF_SRC(code)) { switch (BPF_SRC(code)) {
case BPF_X: case BPF_X:
if (imm == 1) {
/* Special mov32 for zext */
emit_a32_mov_i(dst_hi, 0, ctx);
break;
}
emit_a32_mov_r64(is64, dst, src, ctx); emit_a32_mov_r64(is64, dst, src, ctx);
break; break;
case BPF_K: case BPF_K:
...@@ -1438,6 +1448,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) ...@@ -1438,6 +1448,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
} }
emit_udivmod(rd_lo, rd_lo, rt, ctx, BPF_OP(code)); emit_udivmod(rd_lo, rd_lo, rt, ctx, BPF_OP(code));
arm_bpf_put_reg32(dst_lo, rd_lo, ctx); arm_bpf_put_reg32(dst_lo, rd_lo, ctx);
if (!ctx->prog->aux->verifier_zext)
emit_a32_mov_i(dst_hi, 0, ctx); emit_a32_mov_i(dst_hi, 0, ctx);
break; break;
case BPF_ALU64 | BPF_DIV | BPF_K: case BPF_ALU64 | BPF_DIV | BPF_K:
...@@ -1453,6 +1464,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) ...@@ -1453,6 +1464,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
return -EINVAL; return -EINVAL;
if (imm) if (imm)
emit_a32_alu_i(dst_lo, imm, ctx, BPF_OP(code)); emit_a32_alu_i(dst_lo, imm, ctx, BPF_OP(code));
if (!ctx->prog->aux->verifier_zext)
emit_a32_mov_i(dst_hi, 0, ctx); emit_a32_mov_i(dst_hi, 0, ctx);
break; break;
/* dst = dst << imm */ /* dst = dst << imm */
...@@ -1488,6 +1500,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) ...@@ -1488,6 +1500,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
/* dst = ~dst */ /* dst = ~dst */
case BPF_ALU | BPF_NEG: case BPF_ALU | BPF_NEG:
emit_a32_alu_i(dst_lo, 0, ctx, BPF_OP(code)); emit_a32_alu_i(dst_lo, 0, ctx, BPF_OP(code));
if (!ctx->prog->aux->verifier_zext)
emit_a32_mov_i(dst_hi, 0, ctx); emit_a32_mov_i(dst_hi, 0, ctx);
break; break;
/* dst = ~dst (64 bit) */ /* dst = ~dst (64 bit) */
...@@ -1544,10 +1557,12 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) ...@@ -1544,10 +1557,12 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
#else /* ARMv6+ */ #else /* ARMv6+ */
emit(ARM_UXTH(rd[1], rd[1]), ctx); emit(ARM_UXTH(rd[1], rd[1]), ctx);
#endif #endif
if (!ctx->prog->aux->verifier_zext)
emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx); emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx);
break; break;
case 32: case 32:
/* zero-extend 32 bits into 64 bits */ /* zero-extend 32 bits into 64 bits */
if (!ctx->prog->aux->verifier_zext)
emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx); emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx);
break; break;
case 64: case 64:
...@@ -1838,6 +1853,11 @@ void bpf_jit_compile(struct bpf_prog *prog) ...@@ -1838,6 +1853,11 @@ void bpf_jit_compile(struct bpf_prog *prog)
/* Nothing to do here. We support Internal BPF. */ /* Nothing to do here. We support Internal BPF. */
} }
bool bpf_jit_needs_zext(void)
{
return true;
}
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
{ {
struct bpf_prog *tmp, *orig_prog = prog; struct bpf_prog *tmp, *orig_prog = prog;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment