Commit 11f1a4b9 authored by Linus Torvalds's avatar Linus Torvalds

x86: reorganize SMAP handling in user space accesses

This reorganizes how we do the stac/clac instructions in the user access
code.  Instead of adding the instructions directly to the same inline
asm that does the actual user level access and exception handling, add
them at a higher level.

This is mainly preparation for the next step, where we will expose an
interface to allow users to mark several accesses together as being user
space accesses, but it does already clean up some code:

 - the inlined trivial cases of copy_in_user() now do stac/clac just
   once over the accesses: they used to do one pair around the user
   space read, and another pair around the write-back.

 - the {get,put}_user_ex() macros that are used with the catch/try
   handling don't do any stac/clac at all, because that happens in the
   try/catch surrounding them.

Other than those two cleanups that happened naturally from the
re-organization, this should not make any difference. Yet.
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent a5e90b1b
...@@ -134,6 +134,9 @@ extern int __get_user_4(void); ...@@ -134,6 +134,9 @@ extern int __get_user_4(void);
extern int __get_user_8(void); extern int __get_user_8(void);
extern int __get_user_bad(void); extern int __get_user_bad(void);
#define __uaccess_begin() stac()
#define __uaccess_end() clac()
/* /*
* This is a type: either unsigned long, if the argument fits into * This is a type: either unsigned long, if the argument fits into
* that type, or otherwise unsigned long long. * that type, or otherwise unsigned long long.
...@@ -193,10 +196,10 @@ __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL)) ...@@ -193,10 +196,10 @@ __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
#define __put_user_asm_u64(x, addr, err, errret) \ #define __put_user_asm_u64(x, addr, err, errret) \
asm volatile(ASM_STAC "\n" \ asm volatile("\n" \
"1: movl %%eax,0(%2)\n" \ "1: movl %%eax,0(%2)\n" \
"2: movl %%edx,4(%2)\n" \ "2: movl %%edx,4(%2)\n" \
"3: " ASM_CLAC "\n" \ "3:" \
".section .fixup,\"ax\"\n" \ ".section .fixup,\"ax\"\n" \
"4: movl %3,%0\n" \ "4: movl %3,%0\n" \
" jmp 3b\n" \ " jmp 3b\n" \
...@@ -207,10 +210,10 @@ __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL)) ...@@ -207,10 +210,10 @@ __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
: "A" (x), "r" (addr), "i" (errret), "0" (err)) : "A" (x), "r" (addr), "i" (errret), "0" (err))
#define __put_user_asm_ex_u64(x, addr) \ #define __put_user_asm_ex_u64(x, addr) \
asm volatile(ASM_STAC "\n" \ asm volatile("\n" \
"1: movl %%eax,0(%1)\n" \ "1: movl %%eax,0(%1)\n" \
"2: movl %%edx,4(%1)\n" \ "2: movl %%edx,4(%1)\n" \
"3: " ASM_CLAC "\n" \ "3:" \
_ASM_EXTABLE_EX(1b, 2b) \ _ASM_EXTABLE_EX(1b, 2b) \
_ASM_EXTABLE_EX(2b, 3b) \ _ASM_EXTABLE_EX(2b, 3b) \
: : "A" (x), "r" (addr)) : : "A" (x), "r" (addr))
...@@ -304,6 +307,10 @@ do { \ ...@@ -304,6 +307,10 @@ do { \
} \ } \
} while (0) } while (0)
/*
* This doesn't do __uaccess_begin/end - the exception handling
* around it must do that.
*/
#define __put_user_size_ex(x, ptr, size) \ #define __put_user_size_ex(x, ptr, size) \
do { \ do { \
__chk_user_ptr(ptr); \ __chk_user_ptr(ptr); \
...@@ -358,9 +365,9 @@ do { \ ...@@ -358,9 +365,9 @@ do { \
} while (0) } while (0)
#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \ #define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \
asm volatile(ASM_STAC "\n" \ asm volatile("\n" \
"1: mov"itype" %2,%"rtype"1\n" \ "1: mov"itype" %2,%"rtype"1\n" \
"2: " ASM_CLAC "\n" \ "2:\n" \
".section .fixup,\"ax\"\n" \ ".section .fixup,\"ax\"\n" \
"3: mov %3,%0\n" \ "3: mov %3,%0\n" \
" xor"itype" %"rtype"1,%"rtype"1\n" \ " xor"itype" %"rtype"1,%"rtype"1\n" \
...@@ -370,6 +377,10 @@ do { \ ...@@ -370,6 +377,10 @@ do { \
: "=r" (err), ltype(x) \ : "=r" (err), ltype(x) \
: "m" (__m(addr)), "i" (errret), "0" (err)) : "m" (__m(addr)), "i" (errret), "0" (err))
/*
* This doesn't do __uaccess_begin/end - the exception handling
* around it must do that.
*/
#define __get_user_size_ex(x, ptr, size) \ #define __get_user_size_ex(x, ptr, size) \
do { \ do { \
__chk_user_ptr(ptr); \ __chk_user_ptr(ptr); \
...@@ -400,7 +411,9 @@ do { \ ...@@ -400,7 +411,9 @@ do { \
#define __put_user_nocheck(x, ptr, size) \ #define __put_user_nocheck(x, ptr, size) \
({ \ ({ \
int __pu_err; \ int __pu_err; \
__uaccess_begin(); \
__put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \ __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \
__uaccess_end(); \
__builtin_expect(__pu_err, 0); \ __builtin_expect(__pu_err, 0); \
}) })
...@@ -408,7 +421,9 @@ do { \ ...@@ -408,7 +421,9 @@ do { \
({ \ ({ \
int __gu_err; \ int __gu_err; \
unsigned long __gu_val; \ unsigned long __gu_val; \
__uaccess_begin(); \
__get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \ __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \
__uaccess_end(); \
(x) = (__force __typeof__(*(ptr)))__gu_val; \ (x) = (__force __typeof__(*(ptr)))__gu_val; \
__builtin_expect(__gu_err, 0); \ __builtin_expect(__gu_err, 0); \
}) })
...@@ -423,9 +438,9 @@ struct __large_struct { unsigned long buf[100]; }; ...@@ -423,9 +438,9 @@ struct __large_struct { unsigned long buf[100]; };
* aliasing issues. * aliasing issues.
*/ */
#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \ #define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \
asm volatile(ASM_STAC "\n" \ asm volatile("\n" \
"1: mov"itype" %"rtype"1,%2\n" \ "1: mov"itype" %"rtype"1,%2\n" \
"2: " ASM_CLAC "\n" \ "2:\n" \
".section .fixup,\"ax\"\n" \ ".section .fixup,\"ax\"\n" \
"3: mov %3,%0\n" \ "3: mov %3,%0\n" \
" jmp 2b\n" \ " jmp 2b\n" \
...@@ -445,11 +460,11 @@ struct __large_struct { unsigned long buf[100]; }; ...@@ -445,11 +460,11 @@ struct __large_struct { unsigned long buf[100]; };
*/ */
#define uaccess_try do { \ #define uaccess_try do { \
current_thread_info()->uaccess_err = 0; \ current_thread_info()->uaccess_err = 0; \
stac(); \ __uaccess_begin(); \
barrier(); barrier();
#define uaccess_catch(err) \ #define uaccess_catch(err) \
clac(); \ __uaccess_end(); \
(err) |= (current_thread_info()->uaccess_err ? -EFAULT : 0); \ (err) |= (current_thread_info()->uaccess_err ? -EFAULT : 0); \
} while (0) } while (0)
...@@ -547,12 +562,13 @@ extern void __cmpxchg_wrong_size(void) ...@@ -547,12 +562,13 @@ extern void __cmpxchg_wrong_size(void)
__typeof__(ptr) __uval = (uval); \ __typeof__(ptr) __uval = (uval); \
__typeof__(*(ptr)) __old = (old); \ __typeof__(*(ptr)) __old = (old); \
__typeof__(*(ptr)) __new = (new); \ __typeof__(*(ptr)) __new = (new); \
__uaccess_begin(); \
switch (size) { \ switch (size) { \
case 1: \ case 1: \
{ \ { \
asm volatile("\t" ASM_STAC "\n" \ asm volatile("\n" \
"1:\t" LOCK_PREFIX "cmpxchgb %4, %2\n" \ "1:\t" LOCK_PREFIX "cmpxchgb %4, %2\n" \
"2:\t" ASM_CLAC "\n" \ "2:\n" \
"\t.section .fixup, \"ax\"\n" \ "\t.section .fixup, \"ax\"\n" \
"3:\tmov %3, %0\n" \ "3:\tmov %3, %0\n" \
"\tjmp 2b\n" \ "\tjmp 2b\n" \
...@@ -566,9 +582,9 @@ extern void __cmpxchg_wrong_size(void) ...@@ -566,9 +582,9 @@ extern void __cmpxchg_wrong_size(void)
} \ } \
case 2: \ case 2: \
{ \ { \
asm volatile("\t" ASM_STAC "\n" \ asm volatile("\n" \
"1:\t" LOCK_PREFIX "cmpxchgw %4, %2\n" \ "1:\t" LOCK_PREFIX "cmpxchgw %4, %2\n" \
"2:\t" ASM_CLAC "\n" \ "2:\n" \
"\t.section .fixup, \"ax\"\n" \ "\t.section .fixup, \"ax\"\n" \
"3:\tmov %3, %0\n" \ "3:\tmov %3, %0\n" \
"\tjmp 2b\n" \ "\tjmp 2b\n" \
...@@ -582,9 +598,9 @@ extern void __cmpxchg_wrong_size(void) ...@@ -582,9 +598,9 @@ extern void __cmpxchg_wrong_size(void)
} \ } \
case 4: \ case 4: \
{ \ { \
asm volatile("\t" ASM_STAC "\n" \ asm volatile("\n" \
"1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" \ "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" \
"2:\t" ASM_CLAC "\n" \ "2:\n" \
"\t.section .fixup, \"ax\"\n" \ "\t.section .fixup, \"ax\"\n" \
"3:\tmov %3, %0\n" \ "3:\tmov %3, %0\n" \
"\tjmp 2b\n" \ "\tjmp 2b\n" \
...@@ -601,9 +617,9 @@ extern void __cmpxchg_wrong_size(void) ...@@ -601,9 +617,9 @@ extern void __cmpxchg_wrong_size(void)
if (!IS_ENABLED(CONFIG_X86_64)) \ if (!IS_ENABLED(CONFIG_X86_64)) \
__cmpxchg_wrong_size(); \ __cmpxchg_wrong_size(); \
\ \
asm volatile("\t" ASM_STAC "\n" \ asm volatile("\n" \
"1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n" \ "1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n" \
"2:\t" ASM_CLAC "\n" \ "2:\n" \
"\t.section .fixup, \"ax\"\n" \ "\t.section .fixup, \"ax\"\n" \
"3:\tmov %3, %0\n" \ "3:\tmov %3, %0\n" \
"\tjmp 2b\n" \ "\tjmp 2b\n" \
...@@ -618,6 +634,7 @@ extern void __cmpxchg_wrong_size(void) ...@@ -618,6 +634,7 @@ extern void __cmpxchg_wrong_size(void)
default: \ default: \
__cmpxchg_wrong_size(); \ __cmpxchg_wrong_size(); \
} \ } \
__uaccess_end(); \
*__uval = __old; \ *__uval = __old; \
__ret; \ __ret; \
}) })
......
...@@ -56,35 +56,49 @@ int __copy_from_user_nocheck(void *dst, const void __user *src, unsigned size) ...@@ -56,35 +56,49 @@ int __copy_from_user_nocheck(void *dst, const void __user *src, unsigned size)
if (!__builtin_constant_p(size)) if (!__builtin_constant_p(size))
return copy_user_generic(dst, (__force void *)src, size); return copy_user_generic(dst, (__force void *)src, size);
switch (size) { switch (size) {
case 1:__get_user_asm(*(u8 *)dst, (u8 __user *)src, case 1:
__uaccess_begin();
__get_user_asm(*(u8 *)dst, (u8 __user *)src,
ret, "b", "b", "=q", 1); ret, "b", "b", "=q", 1);
__uaccess_end();
return ret; return ret;
case 2:__get_user_asm(*(u16 *)dst, (u16 __user *)src, case 2:
__uaccess_begin();
__get_user_asm(*(u16 *)dst, (u16 __user *)src,
ret, "w", "w", "=r", 2); ret, "w", "w", "=r", 2);
__uaccess_end();
return ret; return ret;
case 4:__get_user_asm(*(u32 *)dst, (u32 __user *)src, case 4:
__uaccess_begin();
__get_user_asm(*(u32 *)dst, (u32 __user *)src,
ret, "l", "k", "=r", 4); ret, "l", "k", "=r", 4);
__uaccess_end();
return ret; return ret;
case 8:__get_user_asm(*(u64 *)dst, (u64 __user *)src, case 8:
__uaccess_begin();
__get_user_asm(*(u64 *)dst, (u64 __user *)src,
ret, "q", "", "=r", 8); ret, "q", "", "=r", 8);
__uaccess_end();
return ret; return ret;
case 10: case 10:
__uaccess_begin();
__get_user_asm(*(u64 *)dst, (u64 __user *)src, __get_user_asm(*(u64 *)dst, (u64 __user *)src,
ret, "q", "", "=r", 10); ret, "q", "", "=r", 10);
if (unlikely(ret)) if (likely(!ret))
return ret; __get_user_asm(*(u16 *)(8 + (char *)dst),
__get_user_asm(*(u16 *)(8 + (char *)dst), (u16 __user *)(8 + (char __user *)src),
(u16 __user *)(8 + (char __user *)src), ret, "w", "w", "=r", 2);
ret, "w", "w", "=r", 2); __uaccess_end();
return ret; return ret;
case 16: case 16:
__uaccess_begin();
__get_user_asm(*(u64 *)dst, (u64 __user *)src, __get_user_asm(*(u64 *)dst, (u64 __user *)src,
ret, "q", "", "=r", 16); ret, "q", "", "=r", 16);
if (unlikely(ret)) if (likely(!ret))
return ret; __get_user_asm(*(u64 *)(8 + (char *)dst),
__get_user_asm(*(u64 *)(8 + (char *)dst), (u64 __user *)(8 + (char __user *)src),
(u64 __user *)(8 + (char __user *)src), ret, "q", "", "=r", 8);
ret, "q", "", "=r", 8); __uaccess_end();
return ret; return ret;
default: default:
return copy_user_generic(dst, (__force void *)src, size); return copy_user_generic(dst, (__force void *)src, size);
...@@ -106,35 +120,51 @@ int __copy_to_user_nocheck(void __user *dst, const void *src, unsigned size) ...@@ -106,35 +120,51 @@ int __copy_to_user_nocheck(void __user *dst, const void *src, unsigned size)
if (!__builtin_constant_p(size)) if (!__builtin_constant_p(size))
return copy_user_generic((__force void *)dst, src, size); return copy_user_generic((__force void *)dst, src, size);
switch (size) { switch (size) {
case 1:__put_user_asm(*(u8 *)src, (u8 __user *)dst, case 1:
__uaccess_begin();
__put_user_asm(*(u8 *)src, (u8 __user *)dst,
ret, "b", "b", "iq", 1); ret, "b", "b", "iq", 1);
__uaccess_end();
return ret; return ret;
case 2:__put_user_asm(*(u16 *)src, (u16 __user *)dst, case 2:
__uaccess_begin();
__put_user_asm(*(u16 *)src, (u16 __user *)dst,
ret, "w", "w", "ir", 2); ret, "w", "w", "ir", 2);
__uaccess_end();
return ret; return ret;
case 4:__put_user_asm(*(u32 *)src, (u32 __user *)dst, case 4:
__uaccess_begin();
__put_user_asm(*(u32 *)src, (u32 __user *)dst,
ret, "l", "k", "ir", 4); ret, "l", "k", "ir", 4);
__uaccess_end();
return ret; return ret;
case 8:__put_user_asm(*(u64 *)src, (u64 __user *)dst, case 8:
__uaccess_begin();
__put_user_asm(*(u64 *)src, (u64 __user *)dst,
ret, "q", "", "er", 8); ret, "q", "", "er", 8);
__uaccess_end();
return ret; return ret;
case 10: case 10:
__uaccess_begin();
__put_user_asm(*(u64 *)src, (u64 __user *)dst, __put_user_asm(*(u64 *)src, (u64 __user *)dst,
ret, "q", "", "er", 10); ret, "q", "", "er", 10);
if (unlikely(ret)) if (likely(!ret)) {
return ret; asm("":::"memory");
asm("":::"memory"); __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst,
__put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst, ret, "w", "w", "ir", 2);
ret, "w", "w", "ir", 2); }
__uaccess_end();
return ret; return ret;
case 16: case 16:
__uaccess_begin();
__put_user_asm(*(u64 *)src, (u64 __user *)dst, __put_user_asm(*(u64 *)src, (u64 __user *)dst,
ret, "q", "", "er", 16); ret, "q", "", "er", 16);
if (unlikely(ret)) if (likely(!ret)) {
return ret; asm("":::"memory");
asm("":::"memory"); __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst,
__put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst, ret, "q", "", "er", 8);
ret, "q", "", "er", 8); }
__uaccess_end();
return ret; return ret;
default: default:
return copy_user_generic((__force void *)dst, src, size); return copy_user_generic((__force void *)dst, src, size);
...@@ -160,39 +190,47 @@ int __copy_in_user(void __user *dst, const void __user *src, unsigned size) ...@@ -160,39 +190,47 @@ int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
switch (size) { switch (size) {
case 1: { case 1: {
u8 tmp; u8 tmp;
__uaccess_begin();
__get_user_asm(tmp, (u8 __user *)src, __get_user_asm(tmp, (u8 __user *)src,
ret, "b", "b", "=q", 1); ret, "b", "b", "=q", 1);
if (likely(!ret)) if (likely(!ret))
__put_user_asm(tmp, (u8 __user *)dst, __put_user_asm(tmp, (u8 __user *)dst,
ret, "b", "b", "iq", 1); ret, "b", "b", "iq", 1);
__uaccess_end();
return ret; return ret;
} }
case 2: { case 2: {
u16 tmp; u16 tmp;
__uaccess_begin();
__get_user_asm(tmp, (u16 __user *)src, __get_user_asm(tmp, (u16 __user *)src,
ret, "w", "w", "=r", 2); ret, "w", "w", "=r", 2);
if (likely(!ret)) if (likely(!ret))
__put_user_asm(tmp, (u16 __user *)dst, __put_user_asm(tmp, (u16 __user *)dst,
ret, "w", "w", "ir", 2); ret, "w", "w", "ir", 2);
__uaccess_end();
return ret; return ret;
} }
case 4: { case 4: {
u32 tmp; u32 tmp;
__uaccess_begin();
__get_user_asm(tmp, (u32 __user *)src, __get_user_asm(tmp, (u32 __user *)src,
ret, "l", "k", "=r", 4); ret, "l", "k", "=r", 4);
if (likely(!ret)) if (likely(!ret))
__put_user_asm(tmp, (u32 __user *)dst, __put_user_asm(tmp, (u32 __user *)dst,
ret, "l", "k", "ir", 4); ret, "l", "k", "ir", 4);
__uaccess_end();
return ret; return ret;
} }
case 8: { case 8: {
u64 tmp; u64 tmp;
__uaccess_begin();
__get_user_asm(tmp, (u64 __user *)src, __get_user_asm(tmp, (u64 __user *)src,
ret, "q", "", "=r", 8); ret, "q", "", "=r", 8);
if (likely(!ret)) if (likely(!ret))
__put_user_asm(tmp, (u64 __user *)dst, __put_user_asm(tmp, (u64 __user *)dst,
ret, "q", "", "er", 8); ret, "q", "", "er", 8);
__uaccess_end();
return ret; return ret;
} }
default: default:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment