Commit aee636c4 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller

bpf: do not use reciprocal divide

At first Jakub Zawadzki noticed that some divisions by reciprocal_divide
were not correct. (off by one in some cases)
http://www.wireshark.org/~darkjames/reciprocal-buggy.c

He could also show this with BPF:
http://www.wireshark.org/~darkjames/set-and-dump-filter-k-bug.c

The reciprocal divide in linux kernel is not generic enough,
lets remove its use in BPF, as it is not worth the pain with
current cpus.
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Reported-by: default avatarJakub Zawadzki <darkjames-ws@darkjames.pl>
Cc: Mircea Gherzan <mgherzan@gmail.com>
Cc: Daniel Borkmann <dxchgb@gmail.com>
Cc: Hannes Frederic Sowa <hannes@stressinduktion.org>
Cc: Matt Evans <matt@ozlabs.org>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: David S. Miller <davem@davemloft.net>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent ba42fad0
...@@ -641,10 +641,10 @@ static int build_body(struct jit_ctx *ctx) ...@@ -641,10 +641,10 @@ static int build_body(struct jit_ctx *ctx)
emit(ARM_MUL(r_A, r_A, r_X), ctx); emit(ARM_MUL(r_A, r_A, r_X), ctx);
break; break;
case BPF_S_ALU_DIV_K: case BPF_S_ALU_DIV_K:
/* current k == reciprocal_value(userspace k) */ if (k == 1)
break;
emit_mov_i(r_scratch, k, ctx); emit_mov_i(r_scratch, k, ctx);
/* A = top 32 bits of the product */ emit_udiv(r_A, r_A, r_scratch, ctx);
emit(ARM_UMULL(r_scratch, r_A, r_A, r_scratch), ctx);
break; break;
case BPF_S_ALU_DIV_X: case BPF_S_ALU_DIV_X:
update_on_xread(ctx); update_on_xread(ctx);
......
...@@ -223,10 +223,11 @@ static int bpf_jit_build_body(struct sk_filter *fp, u32 *image, ...@@ -223,10 +223,11 @@ static int bpf_jit_build_body(struct sk_filter *fp, u32 *image,
} }
PPC_DIVWU(r_A, r_A, r_X); PPC_DIVWU(r_A, r_A, r_X);
break; break;
case BPF_S_ALU_DIV_K: /* A = reciprocal_divide(A, K); */ case BPF_S_ALU_DIV_K: /* A /= K */
if (K == 1)
break;
PPC_LI32(r_scratch1, K); PPC_LI32(r_scratch1, K);
/* Top 32 bits of 64bit result -> A */ PPC_DIVWU(r_A, r_A, r_scratch1);
PPC_MULHWU(r_A, r_A, r_scratch1);
break; break;
case BPF_S_ALU_AND_X: case BPF_S_ALU_AND_X:
ctx->seen |= SEEN_XREG; ctx->seen |= SEEN_XREG;
......
...@@ -371,11 +371,13 @@ static int bpf_jit_insn(struct bpf_jit *jit, struct sock_filter *filter, ...@@ -371,11 +371,13 @@ static int bpf_jit_insn(struct bpf_jit *jit, struct sock_filter *filter,
/* dr %r4,%r12 */ /* dr %r4,%r12 */
EMIT2(0x1d4c); EMIT2(0x1d4c);
break; break;
case BPF_S_ALU_DIV_K: /* A = reciprocal_divide(A, K) */ case BPF_S_ALU_DIV_K: /* A /= K */
/* m %r4,<d(K)>(%r13) */ if (K == 1)
EMIT4_DISP(0x5c40d000, EMIT_CONST(K)); break;
/* lr %r5,%r4 */ /* lhi %r4,0 */
EMIT2(0x1854); EMIT4(0xa7480000);
/* d %r4,<d(K)>(%r13) */
EMIT4_DISP(0x5d40d000, EMIT_CONST(K));
break; break;
case BPF_S_ALU_MOD_X: /* A %= X */ case BPF_S_ALU_MOD_X: /* A %= X */
jit->seen |= SEEN_XREG | SEEN_RET0; jit->seen |= SEEN_XREG | SEEN_RET0;
...@@ -391,6 +393,11 @@ static int bpf_jit_insn(struct bpf_jit *jit, struct sock_filter *filter, ...@@ -391,6 +393,11 @@ static int bpf_jit_insn(struct bpf_jit *jit, struct sock_filter *filter,
EMIT2(0x1854); EMIT2(0x1854);
break; break;
case BPF_S_ALU_MOD_K: /* A %= K */ case BPF_S_ALU_MOD_K: /* A %= K */
if (K == 1) {
/* lhi %r5,0 */
EMIT4(0xa7580000);
break;
}
/* lhi %r4,0 */ /* lhi %r4,0 */
EMIT4(0xa7480000); EMIT4(0xa7480000);
/* d %r4,<d(K)>(%r13) */ /* d %r4,<d(K)>(%r13) */
......
...@@ -497,9 +497,20 @@ void bpf_jit_compile(struct sk_filter *fp) ...@@ -497,9 +497,20 @@ void bpf_jit_compile(struct sk_filter *fp)
case BPF_S_ALU_MUL_K: /* A *= K */ case BPF_S_ALU_MUL_K: /* A *= K */
emit_alu_K(MUL, K); emit_alu_K(MUL, K);
break; break;
case BPF_S_ALU_DIV_K: /* A /= K */ case BPF_S_ALU_DIV_K: /* A /= K with K != 0*/
emit_alu_K(MUL, K); if (K == 1)
emit_read_y(r_A); break;
emit_write_y(G0);
#ifdef CONFIG_SPARC32
/* The Sparc v8 architecture requires
* three instructions between a %y
* register write and the first use.
*/
emit_nop();
emit_nop();
emit_nop();
#endif
emit_alu_K(DIV, K);
break; break;
case BPF_S_ALU_DIV_X: /* A /= X; */ case BPF_S_ALU_DIV_X: /* A /= X; */
emit_cmpi(r_X, 0); emit_cmpi(r_X, 0);
......
...@@ -359,15 +359,21 @@ void bpf_jit_compile(struct sk_filter *fp) ...@@ -359,15 +359,21 @@ void bpf_jit_compile(struct sk_filter *fp)
EMIT2(0x89, 0xd0); /* mov %edx,%eax */ EMIT2(0x89, 0xd0); /* mov %edx,%eax */
break; break;
case BPF_S_ALU_MOD_K: /* A %= K; */ case BPF_S_ALU_MOD_K: /* A %= K; */
if (K == 1) {
CLEAR_A();
break;
}
EMIT2(0x31, 0xd2); /* xor %edx,%edx */ EMIT2(0x31, 0xd2); /* xor %edx,%edx */
EMIT1(0xb9);EMIT(K, 4); /* mov imm32,%ecx */ EMIT1(0xb9);EMIT(K, 4); /* mov imm32,%ecx */
EMIT2(0xf7, 0xf1); /* div %ecx */ EMIT2(0xf7, 0xf1); /* div %ecx */
EMIT2(0x89, 0xd0); /* mov %edx,%eax */ EMIT2(0x89, 0xd0); /* mov %edx,%eax */
break; break;
case BPF_S_ALU_DIV_K: /* A = reciprocal_divide(A, K); */ case BPF_S_ALU_DIV_K: /* A /= K */
EMIT3(0x48, 0x69, 0xc0); /* imul imm32,%rax,%rax */ if (K == 1)
EMIT(K, 4); break;
EMIT4(0x48, 0xc1, 0xe8, 0x20); /* shr $0x20,%rax */ EMIT2(0x31, 0xd2); /* xor %edx,%edx */
EMIT1(0xb9);EMIT(K, 4); /* mov imm32,%ecx */
EMIT2(0xf7, 0xf1); /* div %ecx */
break; break;
case BPF_S_ALU_AND_X: case BPF_S_ALU_AND_X:
seen |= SEEN_XREG; seen |= SEEN_XREG;
......
...@@ -36,7 +36,6 @@ ...@@ -36,7 +36,6 @@
#include <asm/uaccess.h> #include <asm/uaccess.h>
#include <asm/unaligned.h> #include <asm/unaligned.h>
#include <linux/filter.h> #include <linux/filter.h>
#include <linux/reciprocal_div.h>
#include <linux/ratelimit.h> #include <linux/ratelimit.h>
#include <linux/seccomp.h> #include <linux/seccomp.h>
#include <linux/if_vlan.h> #include <linux/if_vlan.h>
...@@ -166,7 +165,7 @@ unsigned int sk_run_filter(const struct sk_buff *skb, ...@@ -166,7 +165,7 @@ unsigned int sk_run_filter(const struct sk_buff *skb,
A /= X; A /= X;
continue; continue;
case BPF_S_ALU_DIV_K: case BPF_S_ALU_DIV_K:
A = reciprocal_divide(A, K); A /= K;
continue; continue;
case BPF_S_ALU_MOD_X: case BPF_S_ALU_MOD_X:
if (X == 0) if (X == 0)
...@@ -553,11 +552,6 @@ int sk_chk_filter(struct sock_filter *filter, unsigned int flen) ...@@ -553,11 +552,6 @@ int sk_chk_filter(struct sock_filter *filter, unsigned int flen)
/* Some instructions need special checks */ /* Some instructions need special checks */
switch (code) { switch (code) {
case BPF_S_ALU_DIV_K: case BPF_S_ALU_DIV_K:
/* check for division by zero */
if (ftest->k == 0)
return -EINVAL;
ftest->k = reciprocal_value(ftest->k);
break;
case BPF_S_ALU_MOD_K: case BPF_S_ALU_MOD_K:
/* check for division by zero */ /* check for division by zero */
if (ftest->k == 0) if (ftest->k == 0)
...@@ -853,26 +847,6 @@ void sk_decode_filter(struct sock_filter *filt, struct sock_filter *to) ...@@ -853,26 +847,6 @@ void sk_decode_filter(struct sock_filter *filt, struct sock_filter *to)
to->code = decodes[code]; to->code = decodes[code];
to->jt = filt->jt; to->jt = filt->jt;
to->jf = filt->jf; to->jf = filt->jf;
if (code == BPF_S_ALU_DIV_K) {
/*
* When loaded this rule user gave us X, which was
* translated into R = r(X). Now we calculate the
* RR = r(R) and report it back. If next time this
* value is loaded and RRR = r(RR) is calculated
* then the R == RRR will be true.
*
* One exception. X == 1 translates into R == 0 and
* we can't calculate RR out of it with r().
*/
if (filt->k == 0)
to->k = 1;
else
to->k = reciprocal_value(filt->k);
BUG_ON(reciprocal_value(to->k) != filt->k);
} else
to->k = filt->k; to->k = filt->k;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment