Commit d55564cf authored by Linus Torvalds's avatar Linus Torvalds

x86: Make __put_user() generate an out-of-line call

Instead of inlining the stac/mov/clac sequence (which also requires
individual exception table entries and several asm instruction
alternatives entries), just generate "call __put_user_nocheck_X" for the
__put_user() cases, the same way we changed __get_user earlier.

Unlike the get_user() case, we didn't have the same nice infrastructure
to just generate the call with a single case, so this actually has to
change some of the infrastructure in order to do this.  But that only
cleans up the code further.

So now, instead of using a case statement for the sizes, we just do the
same thing we've done on the get_user() side for a long time: use the
size as an immediate constant to the asm, and generate the asm that way
directly.

In order to handle the special case of 64-bit data on a 32-bit kernel, I
needed to change the calling convention slightly: the data is passed in
%eax[:%edx], the pointer in %ecx, and the return value is also returned
in %ecx.  It used to be returned in %eax, but because of how %eax can
now be a double register input, we don't want mix that with a
single-register output.

The actual low-level asm is easier to handle: we'll just share the code
between the checking and non-checking case, with the non-checking case
jumping into the middle of the function.  That may sound a bit too
special, but this code is all very very special anyway, so...

Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent ea6f043f
...@@ -201,11 +201,6 @@ extern int __get_user_bad(void); ...@@ -201,11 +201,6 @@ extern int __get_user_bad(void);
*/ */
#define __get_user(x,ptr) do_get_user_call(get_user_nocheck,x,ptr) #define __get_user(x,ptr) do_get_user_call(get_user_nocheck,x,ptr)
#define __put_user_x(size, x, ptr, __ret_pu) \
asm volatile("call __put_user_" #size : "=a" (__ret_pu) \
: "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
#define __put_user_goto_u64(x, addr, label) \ #define __put_user_goto_u64(x, addr, label) \
...@@ -217,25 +212,41 @@ extern int __get_user_bad(void); ...@@ -217,25 +212,41 @@ extern int __get_user_bad(void);
: : "A" (x), "r" (addr) \ : : "A" (x), "r" (addr) \
: : label) : : label)
#define __put_user_x8(x, ptr, __ret_pu) \
asm volatile("call __put_user_8" : "=a" (__ret_pu) \
: "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
#else #else
#define __put_user_goto_u64(x, ptr, label) \ #define __put_user_goto_u64(x, ptr, label) \
__put_user_goto(x, ptr, "q", "er", label) __put_user_goto(x, ptr, "q", "er", label)
#define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)
#endif #endif
extern void __put_user_bad(void); extern void __put_user_bad(void);
/* /*
* Strange magic calling convention: pointer in %ecx, * Strange magic calling convention: pointer in %ecx,
* value in %eax(:%edx), return value in %eax. clobbers %rbx * value in %eax(:%edx), return value in %ecx. clobbers %rbx
*/ */
extern void __put_user_1(void); extern void __put_user_1(void);
extern void __put_user_2(void); extern void __put_user_2(void);
extern void __put_user_4(void); extern void __put_user_4(void);
extern void __put_user_8(void); extern void __put_user_8(void);
extern void __put_user_nocheck_1(void);
extern void __put_user_nocheck_2(void);
extern void __put_user_nocheck_4(void);
extern void __put_user_nocheck_8(void);
#define do_put_user_call(fn,x,ptr) \
({ \
int __ret_pu; \
register __typeof__(*(ptr)) __val_pu asm("%"_ASM_AX); \
__chk_user_ptr(ptr); \
__val_pu = (x); \
asm volatile("call __" #fn "_%P[size]" \
: "=c" (__ret_pu), \
ASM_CALL_CONSTRAINT \
: "0" (ptr), \
"r" (__val_pu), \
[size] "i" (sizeof(*(ptr))) \
:"ebx"); \
__builtin_expect(__ret_pu, 0); \
})
/** /**
* put_user - Write a simple value into user space. * put_user - Write a simple value into user space.
...@@ -254,32 +265,29 @@ extern void __put_user_8(void); ...@@ -254,32 +265,29 @@ extern void __put_user_8(void);
* *
* Return: zero on success, or -EFAULT on error. * Return: zero on success, or -EFAULT on error.
*/ */
#define put_user(x, ptr) \ #define put_user(x, ptr) ({ might_fault(); do_put_user_call(put_user,x,ptr); })
({ \
int __ret_pu; \ /**
__typeof__(*(ptr)) __pu_val; \ * __put_user - Write a simple value into user space, with less checking.
__chk_user_ptr(ptr); \ * @x: Value to copy to user space.
might_fault(); \ * @ptr: Destination address, in user space.
__pu_val = x; \ *
switch (sizeof(*(ptr))) { \ * Context: User context only. This function may sleep if pagefaults are
case 1: \ * enabled.
__put_user_x(1, __pu_val, ptr, __ret_pu); \ *
break; \ * This macro copies a single simple value from kernel space to user
case 2: \ * space. It supports simple types like char and int, but not larger
__put_user_x(2, __pu_val, ptr, __ret_pu); \ * data types like structures or arrays.
break; \ *
case 4: \ * @ptr must have pointer-to-simple-variable type, and @x must be assignable
__put_user_x(4, __pu_val, ptr, __ret_pu); \ * to the result of dereferencing @ptr.
break; \ *
case 8: \ * Caller must check the pointer with access_ok() before calling this
__put_user_x8(__pu_val, ptr, __ret_pu); \ * function.
break; \ *
default: \ * Return: zero on success, or -EFAULT on error.
__put_user_x(X, __pu_val, ptr, __ret_pu); \ */
break; \ #define __put_user(x, ptr) do_put_user_call(put_user_nocheck,x,ptr)
} \
__builtin_expect(__ret_pu, 0); \
})
#define __put_user_size(x, ptr, size, label) \ #define __put_user_size(x, ptr, size, label) \
do { \ do { \
...@@ -370,21 +378,6 @@ do { \ ...@@ -370,21 +378,6 @@ do { \
: [umem] "m" (__m(addr)), \ : [umem] "m" (__m(addr)), \
[efault] "i" (-EFAULT), "0" (err)) [efault] "i" (-EFAULT), "0" (err))
#define __put_user_nocheck(x, ptr, size) \
({ \
__label__ __pu_label; \
int __pu_err = -EFAULT; \
__typeof__(*(ptr)) __pu_val = (x); \
__typeof__(ptr) __pu_ptr = (ptr); \
__typeof__(size) __pu_size = (size); \
__uaccess_begin(); \
__put_user_size(__pu_val, __pu_ptr, __pu_size, __pu_label); \
__pu_err = 0; \
__pu_label: \
__uaccess_end(); \
__builtin_expect(__pu_err, 0); \
})
/* FIXME: this hack is definitely wrong -AK */ /* FIXME: this hack is definitely wrong -AK */
struct __large_struct { unsigned long buf[100]; }; struct __large_struct { unsigned long buf[100]; };
#define __m(x) (*(struct __large_struct __user *)(x)) #define __m(x) (*(struct __large_struct __user *)(x))
...@@ -401,30 +394,6 @@ struct __large_struct { unsigned long buf[100]; }; ...@@ -401,30 +394,6 @@ struct __large_struct { unsigned long buf[100]; };
: : ltype(x), "m" (__m(addr)) \ : : ltype(x), "m" (__m(addr)) \
: : label) : : label)
/**
* __put_user - Write a simple value into user space, with less checking.
* @x: Value to copy to user space.
* @ptr: Destination address, in user space.
*
* Context: User context only. This function may sleep if pagefaults are
* enabled.
*
* This macro copies a single simple value from kernel space to user
* space. It supports simple types like char and int, but not larger
* data types like structures or arrays.
*
* @ptr must have pointer-to-simple-variable type, and @x must be assignable
* to the result of dereferencing @ptr.
*
* Caller must check the pointer with access_ok() before calling this
* function.
*
* Return: zero on success, or -EFAULT on error.
*/
#define __put_user(x, ptr) \
__put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
extern unsigned long extern unsigned long
copy_from_user_nmi(void *to, const void __user *from, unsigned long n); copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
extern __must_check long extern __must_check long
......
...@@ -25,7 +25,9 @@ ...@@ -25,7 +25,9 @@
* Inputs: %eax[:%edx] contains the data * Inputs: %eax[:%edx] contains the data
* %ecx contains the address * %ecx contains the address
* *
* Outputs: %eax is error code (0 or -EFAULT) * Outputs: %ecx is error code (0 or -EFAULT)
*
* Clobbers: %ebx needed for task pointer
* *
* These functions should not modify any other registers, * These functions should not modify any other registers,
* as they get called from within inline assembly. * as they get called from within inline assembly.
...@@ -38,13 +40,15 @@ SYM_FUNC_START(__put_user_1) ...@@ -38,13 +40,15 @@ SYM_FUNC_START(__put_user_1)
ENTER ENTER
cmp TASK_addr_limit(%_ASM_BX),%_ASM_CX cmp TASK_addr_limit(%_ASM_BX),%_ASM_CX
jae .Lbad_put_user jae .Lbad_put_user
SYM_INNER_LABEL(__put_user_nocheck_1, SYM_L_GLOBAL)
ASM_STAC ASM_STAC
1: movb %al,(%_ASM_CX) 1: movb %al,(%_ASM_CX)
xor %eax,%eax xor %ecx,%ecx
ASM_CLAC ASM_CLAC
ret ret
SYM_FUNC_END(__put_user_1) SYM_FUNC_END(__put_user_1)
EXPORT_SYMBOL(__put_user_1) EXPORT_SYMBOL(__put_user_1)
EXPORT_SYMBOL(__put_user_nocheck_1)
SYM_FUNC_START(__put_user_2) SYM_FUNC_START(__put_user_2)
ENTER ENTER
...@@ -52,13 +56,15 @@ SYM_FUNC_START(__put_user_2) ...@@ -52,13 +56,15 @@ SYM_FUNC_START(__put_user_2)
sub $1,%_ASM_BX sub $1,%_ASM_BX
cmp %_ASM_BX,%_ASM_CX cmp %_ASM_BX,%_ASM_CX
jae .Lbad_put_user jae .Lbad_put_user
SYM_INNER_LABEL(__put_user_nocheck_2, SYM_L_GLOBAL)
ASM_STAC ASM_STAC
2: movw %ax,(%_ASM_CX) 2: movw %ax,(%_ASM_CX)
xor %eax,%eax xor %ecx,%ecx
ASM_CLAC ASM_CLAC
ret ret
SYM_FUNC_END(__put_user_2) SYM_FUNC_END(__put_user_2)
EXPORT_SYMBOL(__put_user_2) EXPORT_SYMBOL(__put_user_2)
EXPORT_SYMBOL(__put_user_nocheck_2)
SYM_FUNC_START(__put_user_4) SYM_FUNC_START(__put_user_4)
ENTER ENTER
...@@ -66,13 +72,15 @@ SYM_FUNC_START(__put_user_4) ...@@ -66,13 +72,15 @@ SYM_FUNC_START(__put_user_4)
sub $3,%_ASM_BX sub $3,%_ASM_BX
cmp %_ASM_BX,%_ASM_CX cmp %_ASM_BX,%_ASM_CX
jae .Lbad_put_user jae .Lbad_put_user
SYM_INNER_LABEL(__put_user_nocheck_4, SYM_L_GLOBAL)
ASM_STAC ASM_STAC
3: movl %eax,(%_ASM_CX) 3: movl %eax,(%_ASM_CX)
xor %eax,%eax xor %ecx,%ecx
ASM_CLAC ASM_CLAC
ret ret
SYM_FUNC_END(__put_user_4) SYM_FUNC_END(__put_user_4)
EXPORT_SYMBOL(__put_user_4) EXPORT_SYMBOL(__put_user_4)
EXPORT_SYMBOL(__put_user_nocheck_4)
SYM_FUNC_START(__put_user_8) SYM_FUNC_START(__put_user_8)
ENTER ENTER
...@@ -80,21 +88,23 @@ SYM_FUNC_START(__put_user_8) ...@@ -80,21 +88,23 @@ SYM_FUNC_START(__put_user_8)
sub $7,%_ASM_BX sub $7,%_ASM_BX
cmp %_ASM_BX,%_ASM_CX cmp %_ASM_BX,%_ASM_CX
jae .Lbad_put_user jae .Lbad_put_user
SYM_INNER_LABEL(__put_user_nocheck_8, SYM_L_GLOBAL)
ASM_STAC ASM_STAC
4: mov %_ASM_AX,(%_ASM_CX) 4: mov %_ASM_AX,(%_ASM_CX)
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
5: movl %edx,4(%_ASM_CX) 5: movl %edx,4(%_ASM_CX)
#endif #endif
xor %eax,%eax xor %ecx,%ecx
ASM_CLAC ASM_CLAC
RET RET
SYM_FUNC_END(__put_user_8) SYM_FUNC_END(__put_user_8)
EXPORT_SYMBOL(__put_user_8) EXPORT_SYMBOL(__put_user_8)
EXPORT_SYMBOL(__put_user_nocheck_8)
SYM_CODE_START_LOCAL(.Lbad_put_user_clac) SYM_CODE_START_LOCAL(.Lbad_put_user_clac)
ASM_CLAC ASM_CLAC
.Lbad_put_user: .Lbad_put_user:
movl $-EFAULT,%eax movl $-EFAULT,%ecx
RET RET
SYM_CODE_END(.Lbad_put_user_clac) SYM_CODE_END(.Lbad_put_user_clac)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment