Commit 029f56db authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'x86_asm_for_v5.10' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 asm updates from Borislav Petkov:
 "Two asm wrapper fixes:

   - Use XORL instead of XORQ to avoid a REX prefix and save some bytes
     in the .fixup section, by Uros Bizjak.

   - Replace __force_order dummy variable with a memory clobber to fix
     LLVM requiring a definition for former and to prevent memory
     accesses from still being cached/reordered, by Arvind Sankar"

* tag 'x86_asm_for_v5.10' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/asm: Replace __force_order with a memory clobber
  x86/uaccess: Use XORL %0,%0 in __get_user_asm()
parents 7cd4ecd9 aa5cacdc
...@@ -5,15 +5,6 @@ ...@@ -5,15 +5,6 @@
#include "pgtable.h" #include "pgtable.h"
#include "../string.h" #include "../string.h"
/*
* __force_order is used by special_insns.h asm code to force instruction
* serialization.
*
* It is not referenced from the code, but GCC < 5 with -fPIE would fail
* due to an undefined symbol. Define it to make these ancient GCCs work.
*/
unsigned long __force_order;
#define BIOS_START_MIN 0x20000U /* 128K, less than this is insane */ #define BIOS_START_MIN 0x20000U /* 128K, less than this is insane */
#define BIOS_START_MAX 0x9f000U /* 640K, absolute maximum */ #define BIOS_START_MAX 0x9f000U /* 640K, absolute maximum */
......
...@@ -11,45 +11,47 @@ ...@@ -11,45 +11,47 @@
#include <linux/jump_label.h> #include <linux/jump_label.h>
/* /*
* Volatile isn't enough to prevent the compiler from reordering the * The compiler should not reorder volatile asm statements with respect to each
* read/write functions for the control registers and messing everything up. * other: they should execute in program order. However GCC 4.9.x and 5.x have
* A memory clobber would solve the problem, but would prevent reordering of * a bug (which was fixed in 8.1, 7.3 and 6.5) where they might reorder
* all loads stores around it, which can hurt performance. Solution is to * volatile asm. The write functions are not affected since they have memory
* use a variable and mimic reads and writes to it to enforce serialization * clobbers preventing reordering. To prevent reads from being reordered with
* respect to writes, use a dummy memory operand.
*/ */
extern unsigned long __force_order;
#define __FORCE_ORDER "m"(*(unsigned int *)0x1000UL)
void native_write_cr0(unsigned long val); void native_write_cr0(unsigned long val);
static inline unsigned long native_read_cr0(void) static inline unsigned long native_read_cr0(void)
{ {
unsigned long val; unsigned long val;
asm volatile("mov %%cr0,%0\n\t" : "=r" (val), "=m" (__force_order)); asm volatile("mov %%cr0,%0\n\t" : "=r" (val) : __FORCE_ORDER);
return val; return val;
} }
static __always_inline unsigned long native_read_cr2(void) static __always_inline unsigned long native_read_cr2(void)
{ {
unsigned long val; unsigned long val;
asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order)); asm volatile("mov %%cr2,%0\n\t" : "=r" (val) : __FORCE_ORDER);
return val; return val;
} }
static __always_inline void native_write_cr2(unsigned long val) static __always_inline void native_write_cr2(unsigned long val)
{ {
asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order)); asm volatile("mov %0,%%cr2": : "r" (val) : "memory");
} }
static inline unsigned long __native_read_cr3(void) static inline unsigned long __native_read_cr3(void)
{ {
unsigned long val; unsigned long val;
asm volatile("mov %%cr3,%0\n\t" : "=r" (val), "=m" (__force_order)); asm volatile("mov %%cr3,%0\n\t" : "=r" (val) : __FORCE_ORDER);
return val; return val;
} }
static inline void native_write_cr3(unsigned long val) static inline void native_write_cr3(unsigned long val)
{ {
asm volatile("mov %0,%%cr3": : "r" (val), "m" (__force_order)); asm volatile("mov %0,%%cr3": : "r" (val) : "memory");
} }
static inline unsigned long native_read_cr4(void) static inline unsigned long native_read_cr4(void)
...@@ -64,10 +66,10 @@ static inline unsigned long native_read_cr4(void) ...@@ -64,10 +66,10 @@ static inline unsigned long native_read_cr4(void)
asm volatile("1: mov %%cr4, %0\n" asm volatile("1: mov %%cr4, %0\n"
"2:\n" "2:\n"
_ASM_EXTABLE(1b, 2b) _ASM_EXTABLE(1b, 2b)
: "=r" (val), "=m" (__force_order) : "0" (0)); : "=r" (val) : "0" (0), __FORCE_ORDER);
#else #else
/* CR4 always exists on x86_64. */ /* CR4 always exists on x86_64. */
asm volatile("mov %%cr4,%0\n\t" : "=r" (val), "=m" (__force_order)); asm volatile("mov %%cr4,%0\n\t" : "=r" (val) : __FORCE_ORDER);
#endif #endif
return val; return val;
} }
......
...@@ -418,7 +418,7 @@ do { \ ...@@ -418,7 +418,7 @@ do { \
"2:\n" \ "2:\n" \
".section .fixup,\"ax\"\n" \ ".section .fixup,\"ax\"\n" \
"3: mov %[efault],%[errout]\n" \ "3: mov %[efault],%[errout]\n" \
" xor"itype" %[output],%[output]\n" \ " xorl %k[output],%k[output]\n" \
" jmp 2b\n" \ " jmp 2b\n" \
".previous\n" \ ".previous\n" \
_ASM_EXTABLE_UA(1b, 3b) \ _ASM_EXTABLE_UA(1b, 3b) \
......
...@@ -360,7 +360,7 @@ void native_write_cr0(unsigned long val) ...@@ -360,7 +360,7 @@ void native_write_cr0(unsigned long val)
unsigned long bits_missing = 0; unsigned long bits_missing = 0;
set_register: set_register:
asm volatile("mov %0,%%cr0": "+r" (val), "+m" (__force_order)); asm volatile("mov %0,%%cr0": "+r" (val) : : "memory");
if (static_branch_likely(&cr_pinning)) { if (static_branch_likely(&cr_pinning)) {
if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) { if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) {
...@@ -379,7 +379,7 @@ void native_write_cr4(unsigned long val) ...@@ -379,7 +379,7 @@ void native_write_cr4(unsigned long val)
unsigned long bits_changed = 0; unsigned long bits_changed = 0;
set_register: set_register:
asm volatile("mov %0,%%cr4": "+r" (val), "+m" (cr4_pinned_bits)); asm volatile("mov %0,%%cr4": "+r" (val) : : "memory");
if (static_branch_likely(&cr_pinning)) { if (static_branch_likely(&cr_pinning)) {
if (unlikely((val & cr4_pinned_mask) != cr4_pinned_bits)) { if (unlikely((val & cr4_pinned_mask) != cr4_pinned_bits)) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment