x86, bpf, jit: prevent speculative execution when JIT is enabled

CVE-2017-5753
CVE-2017-5715

When constant blinding is enabled (bpf_jit_harden = 1), this adds
a generic memory barrier (lfence for intel, mfence for AMD) before
emitting x86 jitted code for the BPF_ALU(64)_OR_X and BPF_ALU_LHS_X
(for BPF_REG_AX register) eBPF instructions. This is needed in order
to prevent speculative execution on out of bounds BPF_MAP array
indexes when JIT is enabled. This way an arbitary kernel memory is
not exposed through side-channel attacks.

For more details, please see this Google Project Zero report: tbd
Signed-off-by: Elena Reshetova <elena.reshetova@intel.com>
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Andy Whitcroft <apw@canonical.com>
(cherry picked from commit 33f5e63378ad75331315216b459362b0a5350662)
Signed-off-by: Andy Whitcroft <apw@canonical.com>

arch/x86/net/bpf_jit_comp.c

@@ -106,6 +106,27 @@ static void bpf_flush_icache(void *start, void *end)
 	set_fs(old_fs);
 }
 
+static void emit_memory_barrier(u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+
+	if (bpf_jit_blinding_enabled()) {
+		if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC))
+			/* x86 LFENCE opcode 0F AE E8 */
+			EMIT3(0x0f, 0xae, 0xe8);
+		else if (boot_cpu_has(X86_FEATURE_MFENCE_RDTSC))
+			/* AMD MFENCE opcode 0F AE F0 */
+			EMIT3(0x0f, 0xae, 0xf0);
+		else
+			/* we should never end up here,
+			 * but if we do, better not to emit anything*/
+			return;
+	}
+	*pprog = prog;
+	return;
+}
+
 #define CHOOSE_LOAD_FUNC(K, func) \
 	((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
 
@@ -379,7 +400,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 			case BPF_ADD: b2 = 0x01; break;
 			case BPF_SUB: b2 = 0x29; break;
 			case BPF_AND: b2 = 0x21; break;
-			case BPF_OR: b2 = 0x09; break;
+			case BPF_OR: b2 = 0x09; emit_memory_barrier(&prog); break;
 			case BPF_XOR: b2 = 0x31; break;
 			}
 			if (BPF_CLASS(insn->code) == BPF_ALU64)
@@ -607,6 +628,16 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 		case BPF_ALU64 | BPF_RSH | BPF_X:
 		case BPF_ALU64 | BPF_ARSH | BPF_X:
 
+			/* If blinding is enabled, each
+			 * BPF_LD | BPF_IMM | BPF_DW instruction
+			 * is converted to 4 eBPF instructions with
+			 * BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32)
+			 * always present(number 3). Detect such cases
+			 * and insert memory barriers. */
+			if ((BPF_CLASS(insn->code) == BPF_ALU64)
+				&& (BPF_OP(insn->code) == BPF_LSH)
+				&& (src_reg == BPF_REG_AX))
+				emit_memory_barrier(&prog);
 			/* check for bad case when dst_reg == rcx */
 			if (dst_reg == BPF_REG_4) {
 				/* mov r11, dst_reg */