Commit d1c99108 authored by David Woodhouse's avatar David Woodhouse Committed by Ingo Molnar

Revert "x86/retpoline: Simplify vmexit_fill_RSB()"

This reverts commit 1dde7415. By putting
the RSB filling out of line and calling it, we waste one RSB slot for
returning from the function itself, which means one fewer actual function
call we can make if we're doing the Skylake abomination of call-depth
counting.

It also changed the number of RSB stuffings we do on vmexit from 32,
which was correct, to 16. Let's just stop with the bikeshedding; it
didn't actually *fix* anything anyway.
Signed-off-by: default avatarDavid Woodhouse <dwmw@amazon.co.uk>
Acked-by: default avatarThomas Gleixner <tglx@linutronix.de>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: arjan.van.de.ven@intel.com
Cc: bp@alien8.de
Cc: dave.hansen@intel.com
Cc: jmattson@google.com
Cc: karahmed@amazon.de
Cc: kvm@vger.kernel.org
Cc: pbonzini@redhat.com
Cc: rkrcmar@redhat.com
Link: http://lkml.kernel.org/r/1519037457-7643-4-git-send-email-dwmw@amazon.co.ukSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 8554004a
...@@ -252,8 +252,7 @@ ENTRY(__switch_to_asm) ...@@ -252,8 +252,7 @@ ENTRY(__switch_to_asm)
* exist, overwrite the RSB with entries which capture * exist, overwrite the RSB with entries which capture
* speculative execution to prevent attack. * speculative execution to prevent attack.
*/ */
/* Clobbers %ebx */ FILL_RETURN_BUFFER %ebx, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
FILL_RETURN_BUFFER RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
#endif #endif
/* restore callee-saved registers */ /* restore callee-saved registers */
......
...@@ -364,8 +364,7 @@ ENTRY(__switch_to_asm) ...@@ -364,8 +364,7 @@ ENTRY(__switch_to_asm)
* exist, overwrite the RSB with entries which capture * exist, overwrite the RSB with entries which capture
* speculative execution to prevent attack. * speculative execution to prevent attack.
*/ */
/* Clobbers %rbx */ FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
FILL_RETURN_BUFFER RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
#endif #endif
/* restore callee-saved registers */ /* restore callee-saved registers */
......
...@@ -38,7 +38,4 @@ INDIRECT_THUNK(dx) ...@@ -38,7 +38,4 @@ INDIRECT_THUNK(dx)
INDIRECT_THUNK(si) INDIRECT_THUNK(si)
INDIRECT_THUNK(di) INDIRECT_THUNK(di)
INDIRECT_THUNK(bp) INDIRECT_THUNK(bp)
asmlinkage void __fill_rsb(void);
asmlinkage void __clear_rsb(void);
#endif /* CONFIG_RETPOLINE */ #endif /* CONFIG_RETPOLINE */
...@@ -8,6 +8,50 @@ ...@@ -8,6 +8,50 @@
#include <asm/cpufeatures.h> #include <asm/cpufeatures.h>
#include <asm/msr-index.h> #include <asm/msr-index.h>
/*
* Fill the CPU return stack buffer.
*
* Each entry in the RSB, if used for a speculative 'ret', contains an
* infinite 'pause; lfence; jmp' loop to capture speculative execution.
*
* This is required in various cases for retpoline and IBRS-based
* mitigations for the Spectre variant 2 vulnerability. Sometimes to
* eliminate potentially bogus entries from the RSB, and sometimes
* purely to ensure that it doesn't get empty, which on some CPUs would
* allow predictions from other (unwanted!) sources to be used.
*
* We define a CPP macro such that it can be used from both .S files and
* inline assembly. It's possible to do a .macro and then include that
* from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
*/
#define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */
#define RSB_FILL_LOOPS 16 /* To avoid underflow */
/*
* Google experimented with loop-unrolling and this turned out to be
* the optimal version — two calls, each with their own speculation
* trap should their return address end up getting used, in a loop.
*/
#define __FILL_RETURN_BUFFER(reg, nr, sp) \
mov $(nr/2), reg; \
771: \
call 772f; \
773: /* speculation trap */ \
pause; \
lfence; \
jmp 773b; \
772: \
call 774f; \
775: /* speculation trap */ \
pause; \
lfence; \
jmp 775b; \
774: \
dec reg; \
jnz 771b; \
add $(BITS_PER_LONG/8) * nr, sp;
#ifdef __ASSEMBLY__ #ifdef __ASSEMBLY__
/* /*
...@@ -78,10 +122,17 @@ ...@@ -78,10 +122,17 @@
#endif #endif
.endm .endm
/* This clobbers the BX register */ /*
.macro FILL_RETURN_BUFFER nr:req ftr:req * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
* monstrosity above, manually.
*/
.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
#ifdef CONFIG_RETPOLINE #ifdef CONFIG_RETPOLINE
ALTERNATIVE "", "call __clear_rsb", \ftr ANNOTATE_NOSPEC_ALTERNATIVE
ALTERNATIVE "jmp .Lskip_rsb_\@", \
__stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \
\ftr
.Lskip_rsb_\@:
#endif #endif
.endm .endm
...@@ -156,10 +207,15 @@ extern char __indirect_thunk_end[]; ...@@ -156,10 +207,15 @@ extern char __indirect_thunk_end[];
static inline void vmexit_fill_RSB(void) static inline void vmexit_fill_RSB(void)
{ {
#ifdef CONFIG_RETPOLINE #ifdef CONFIG_RETPOLINE
alternative_input("", unsigned long loops;
"call __fill_rsb",
X86_FEATURE_RETPOLINE, asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
ASM_NO_INPUT_CLOBBER(_ASM_BX, "memory")); ALTERNATIVE("jmp 910f",
__stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
X86_FEATURE_RETPOLINE)
"910:"
: "=r" (loops), ASM_CALL_CONSTRAINT
: : "memory" );
#endif #endif
} }
......
...@@ -28,7 +28,6 @@ lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o insn-eval.o ...@@ -28,7 +28,6 @@ lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o insn-eval.o
lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
lib-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o lib-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
lib-$(CONFIG_RETPOLINE) += retpoline.o lib-$(CONFIG_RETPOLINE) += retpoline.o
OBJECT_FILES_NON_STANDARD_retpoline.o :=y
obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
......
...@@ -7,7 +7,6 @@ ...@@ -7,7 +7,6 @@
#include <asm/alternative-asm.h> #include <asm/alternative-asm.h>
#include <asm/export.h> #include <asm/export.h>
#include <asm/nospec-branch.h> #include <asm/nospec-branch.h>
#include <asm/bitsperlong.h>
.macro THUNK reg .macro THUNK reg
.section .text.__x86.indirect_thunk .section .text.__x86.indirect_thunk
...@@ -47,58 +46,3 @@ GENERATE_THUNK(r13) ...@@ -47,58 +46,3 @@ GENERATE_THUNK(r13)
GENERATE_THUNK(r14) GENERATE_THUNK(r14)
GENERATE_THUNK(r15) GENERATE_THUNK(r15)
#endif #endif
/*
* Fill the CPU return stack buffer.
*
* Each entry in the RSB, if used for a speculative 'ret', contains an
* infinite 'pause; lfence; jmp' loop to capture speculative execution.
*
* This is required in various cases for retpoline and IBRS-based
* mitigations for the Spectre variant 2 vulnerability. Sometimes to
* eliminate potentially bogus entries from the RSB, and sometimes
* purely to ensure that it doesn't get empty, which on some CPUs would
* allow predictions from other (unwanted!) sources to be used.
*
* Google experimented with loop-unrolling and this turned out to be
* the optimal version - two calls, each with their own speculation
* trap should their return address end up getting used, in a loop.
*/
.macro STUFF_RSB nr:req sp:req
mov $(\nr / 2), %_ASM_BX
.align 16
771:
call 772f
773: /* speculation trap */
pause
lfence
jmp 773b
.align 16
772:
call 774f
775: /* speculation trap */
pause
lfence
jmp 775b
.align 16
774:
dec %_ASM_BX
jnz 771b
add $((BITS_PER_LONG/8) * \nr), \sp
.endm
#define RSB_FILL_LOOPS 16 /* To avoid underflow */
ENTRY(__fill_rsb)
STUFF_RSB RSB_FILL_LOOPS, %_ASM_SP
ret
END(__fill_rsb)
EXPORT_SYMBOL_GPL(__fill_rsb)
#define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */
ENTRY(__clear_rsb)
STUFF_RSB RSB_CLEAR_LOOPS, %_ASM_SP
ret
END(__clear_rsb)
EXPORT_SYMBOL_GPL(__clear_rsb)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment