Commit 965dd3ad authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Michael Ellerman

powerpc/64/syscall: Remove non-volatile GPR save optimisation

powerpc has an optimisation where interrupts avoid saving the
non-volatile (or callee saved) registers to the interrupt stack frame
if they are not required.

Two problems with this are that an interrupt does not always know
whether it will need non-volatiles; and if it does need them, they can
only be saved from the entry-scoped asm code (because we don't control
what the C compiler does with these registers).

system calls are the most difficult: some system calls always require
all registers (e.g., fork, to copy regs into the child). Sometimes
registers are only required under certain conditions (e.g., tracing,
signal delivery). These cases require ugly logic in the call
chains (e.g., ppc_fork), and require a lot of logic to be implemented
in asm.

So remove the optimisation for system calls, and always save NVGPRs on
entry. Modern high performance CPUs are not so sensitive, because the
stores are dense in cache and can be hidden by other expensive work in
the syscall path -- the null syscall selftests benchmark on POWER9 is
not slowed (124.40ns before and 123.64ns after, i.e., within the
noise).

Other interrupts retain the NVGPR optimisation for now.
Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20200225173541.1549955-24-npiggin@gmail.com
parent 71c3b05a
...@@ -98,13 +98,14 @@ END_BTB_FLUSH_SECTION ...@@ -98,13 +98,14 @@ END_BTB_FLUSH_SECTION
std r11,_XER(r1) std r11,_XER(r1)
std r11,_CTR(r1) std r11,_CTR(r1)
std r9,GPR13(r1) std r9,GPR13(r1)
SAVE_NVGPRS(r1)
mflr r10 mflr r10
/* /*
* This clears CR0.SO (bit 28), which is the error indication on * This clears CR0.SO (bit 28), which is the error indication on
* return from this system call. * return from this system call.
*/ */
rldimi r2,r11,28,(63-28) rldimi r2,r11,28,(63-28)
li r11,0xc01 li r11,0xc00
std r10,_LINK(r1) std r10,_LINK(r1)
std r11,_TRAP(r1) std r11,_TRAP(r1)
std r3,ORIG_GPR3(r1) std r3,ORIG_GPR3(r1)
...@@ -323,7 +324,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) ...@@ -323,7 +324,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
/* Traced system call support */ /* Traced system call support */
.Lsyscall_dotrace: .Lsyscall_dotrace:
bl save_nvgprs
addi r3,r1,STACK_FRAME_OVERHEAD addi r3,r1,STACK_FRAME_OVERHEAD
bl do_syscall_trace_enter bl do_syscall_trace_enter
...@@ -408,7 +408,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) ...@@ -408,7 +408,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
mtmsrd r10,1 mtmsrd r10,1
#endif /* CONFIG_PPC_BOOK3E */ #endif /* CONFIG_PPC_BOOK3E */
bl save_nvgprs
addi r3,r1,STACK_FRAME_OVERHEAD addi r3,r1,STACK_FRAME_OVERHEAD
bl do_syscall_trace_leave bl do_syscall_trace_leave
b ret_from_except b ret_from_except
...@@ -442,62 +441,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) ...@@ -442,62 +441,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
_ASM_NOKPROBE_SYMBOL(system_call_common); _ASM_NOKPROBE_SYMBOL(system_call_common);
_ASM_NOKPROBE_SYMBOL(system_call_exit); _ASM_NOKPROBE_SYMBOL(system_call_exit);
/* Save non-volatile GPRs, if not already saved. */
_GLOBAL(save_nvgprs)
ld r11,_TRAP(r1)
andi. r0,r11,1
beqlr-
SAVE_NVGPRS(r1)
clrrdi r0,r11,1
std r0,_TRAP(r1)
blr
_ASM_NOKPROBE_SYMBOL(save_nvgprs);
/*
* The sigsuspend and rt_sigsuspend system calls can call do_signal
* and thus put the process into the stopped state where we might
* want to examine its user state with ptrace. Therefore we need
* to save all the nonvolatile registers (r14 - r31) before calling
* the C code. Similarly, fork, vfork and clone need the full
* register state on the stack so that it can be copied to the child.
*/
_GLOBAL(ppc_fork)
bl save_nvgprs
bl sys_fork
b .Lsyscall_exit
_GLOBAL(ppc_vfork)
bl save_nvgprs
bl sys_vfork
b .Lsyscall_exit
_GLOBAL(ppc_clone)
bl save_nvgprs
bl sys_clone
b .Lsyscall_exit
_GLOBAL(ppc_clone3)
bl save_nvgprs
bl sys_clone3
b .Lsyscall_exit
_GLOBAL(ppc32_swapcontext)
bl save_nvgprs
bl compat_sys_swapcontext
b .Lsyscall_exit
_GLOBAL(ppc64_swapcontext)
bl save_nvgprs
bl sys_swapcontext
b .Lsyscall_exit
_GLOBAL(ppc_switch_endian)
bl save_nvgprs
bl sys_switch_endian
b .Lsyscall_exit
_GLOBAL(ret_from_fork) _GLOBAL(ret_from_fork)
bl schedule_tail bl schedule_tail
REST_NVGPRS(r1) REST_NVGPRS(r1)
...@@ -516,6 +459,17 @@ _GLOBAL(ret_from_kernel_thread) ...@@ -516,6 +459,17 @@ _GLOBAL(ret_from_kernel_thread)
li r3,0 li r3,0
b .Lsyscall_exit b .Lsyscall_exit
/* Save non-volatile GPRs, if not already saved. */
_GLOBAL(save_nvgprs)
ld r11,_TRAP(r1)
andi. r0,r11,1
beqlr-
SAVE_NVGPRS(r1)
clrrdi r0,r11,1
std r0,_TRAP(r1)
blr
_ASM_NOKPROBE_SYMBOL(save_nvgprs);
#ifdef CONFIG_PPC_BOOK3S_64 #ifdef CONFIG_PPC_BOOK3S_64
#define FLUSH_COUNT_CACHE \ #define FLUSH_COUNT_CACHE \
......
...@@ -9,7 +9,9 @@ ...@@ -9,7 +9,9 @@
# #
0 nospu restart_syscall sys_restart_syscall 0 nospu restart_syscall sys_restart_syscall
1 nospu exit sys_exit 1 nospu exit sys_exit
2 nospu fork ppc_fork 2 32 fork ppc_fork sys_fork
2 64 fork sys_fork
2 spu fork sys_ni_syscall
3 common read sys_read 3 common read sys_read
4 common write sys_write 4 common write sys_write
5 common open sys_open compat_sys_open 5 common open sys_open compat_sys_open
...@@ -158,7 +160,9 @@ ...@@ -158,7 +160,9 @@
119 32 sigreturn sys_sigreturn compat_sys_sigreturn 119 32 sigreturn sys_sigreturn compat_sys_sigreturn
119 64 sigreturn sys_ni_syscall 119 64 sigreturn sys_ni_syscall
119 spu sigreturn sys_ni_syscall 119 spu sigreturn sys_ni_syscall
120 nospu clone ppc_clone 120 32 clone ppc_clone sys_clone
120 64 clone sys_clone
120 spu clone sys_ni_syscall
121 common setdomainname sys_setdomainname 121 common setdomainname sys_setdomainname
122 common uname sys_newuname 122 common uname sys_newuname
123 common modify_ldt sys_ni_syscall 123 common modify_ldt sys_ni_syscall
...@@ -240,7 +244,9 @@ ...@@ -240,7 +244,9 @@
186 spu sendfile sys_sendfile64 186 spu sendfile sys_sendfile64
187 common getpmsg sys_ni_syscall 187 common getpmsg sys_ni_syscall
188 common putpmsg sys_ni_syscall 188 common putpmsg sys_ni_syscall
189 nospu vfork ppc_vfork 189 32 vfork ppc_vfork sys_vfork
189 64 vfork sys_vfork
189 spu vfork sys_ni_syscall
190 common ugetrlimit sys_getrlimit compat_sys_getrlimit 190 common ugetrlimit sys_getrlimit compat_sys_getrlimit
191 common readahead sys_readahead compat_sys_readahead 191 common readahead sys_readahead compat_sys_readahead
192 32 mmap2 sys_mmap2 compat_sys_mmap2 192 32 mmap2 sys_mmap2 compat_sys_mmap2
...@@ -316,8 +322,8 @@ ...@@ -316,8 +322,8 @@
248 32 clock_nanosleep sys_clock_nanosleep_time32 248 32 clock_nanosleep sys_clock_nanosleep_time32
248 64 clock_nanosleep sys_clock_nanosleep 248 64 clock_nanosleep sys_clock_nanosleep
248 spu clock_nanosleep sys_clock_nanosleep 248 spu clock_nanosleep sys_clock_nanosleep
249 32 swapcontext ppc_swapcontext ppc32_swapcontext 249 32 swapcontext ppc_swapcontext compat_sys_swapcontext
249 64 swapcontext ppc64_swapcontext 249 64 swapcontext sys_swapcontext
249 spu swapcontext sys_ni_syscall 249 spu swapcontext sys_ni_syscall
250 common tgkill sys_tgkill 250 common tgkill sys_tgkill
251 32 utimes sys_utimes_time32 251 32 utimes sys_utimes_time32
...@@ -456,7 +462,7 @@ ...@@ -456,7 +462,7 @@
361 common bpf sys_bpf 361 common bpf sys_bpf
362 nospu execveat sys_execveat compat_sys_execveat 362 nospu execveat sys_execveat compat_sys_execveat
363 32 switch_endian sys_ni_syscall 363 32 switch_endian sys_ni_syscall
363 64 switch_endian ppc_switch_endian 363 64 switch_endian sys_switch_endian
363 spu switch_endian sys_ni_syscall 363 spu switch_endian sys_ni_syscall
364 common userfaultfd sys_userfaultfd 364 common userfaultfd sys_userfaultfd
365 common membarrier sys_membarrier 365 common membarrier sys_membarrier
...@@ -516,6 +522,8 @@ ...@@ -516,6 +522,8 @@
432 common fsmount sys_fsmount 432 common fsmount sys_fsmount
433 common fspick sys_fspick 433 common fspick sys_fspick
434 common pidfd_open sys_pidfd_open 434 common pidfd_open sys_pidfd_open
435 nospu clone3 ppc_clone3 435 32 clone3 ppc_clone3 sys_clone3
435 64 clone3 sys_clone3
435 spu clone3 sys_ni_syscall
437 common openat2 sys_openat2 437 common openat2 sys_openat2
438 common pidfd_getfd sys_pidfd_getfd 438 common pidfd_getfd sys_pidfd_getfd
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment