Commit bc0ee476 authored by Dave Martin's avatar Dave Martin Committed by Will Deacon

arm64/sve: Core task context handling

This patch adds the core support for switching and managing the SVE
architectural state of user tasks.

Calls to the existing FPSIMD low-level save/restore functions are
factored out as new functions task_fpsimd_{save,load}(), since SVE
now dynamically may or may not need to be handled at these points
depending on the kernel configuration, hardware features discovered
at boot, and the runtime state of the task.  To make these
decisions as fast as possible, const cpucaps are used where
feasible, via the system_supports_sve() helper.

The SVE registers are only tracked for threads that have explicitly
used SVE, indicated by the new thread flag TIF_SVE.  Otherwise, the
FPSIMD view of the architectural state is stored in
thread.fpsimd_state as usual.

When in use, the SVE registers are not stored directly in
thread_struct due to their potentially large and variable size.
Because the task_struct slab allocator must be configured very
early during kernel boot, it is also tricky to configure it
correctly to match the maximum vector length provided by the
hardware, since this depends on examining secondary CPUs as well as
the primary.  Instead, a pointer sve_state in thread_struct points
to a dynamically allocated buffer containing the SVE register data,
and code is added to allocate and free this buffer at appropriate
times.

TIF_SVE is set when taking an SVE access trap from userspace, if
suitable hardware support has been detected.  This enables SVE for
the thread: a subsequent return to userspace will disable the trap
accordingly.  If such a trap is taken without sufficient system-
wide hardware support, SIGILL is sent to the thread instead as if
an undefined instruction had been executed: this may happen if
userspace tries to use SVE in a system where not all CPUs support
it for example.

The kernel will clear TIF_SVE and disable SVE for the thread
whenever an explicit syscall is made by userspace.  For backwards
compatibility reasons and conformance with the spirit of the base
AArch64 procedure call standard, the subset of the SVE register
state that aliases the FPSIMD registers is still preserved across a
syscall even if this happens.  The remainder of the SVE register
state logically becomes zero at syscall entry, though the actual
zeroing work is currently deferred until the thread next tries to
use SVE, causing another trap to the kernel.  This implementation
is suboptimal: in the future, the fastpath case may be optimised
to zero the registers in-place and leave SVE enabled for the task,
where beneficial.

TIF_SVE is also cleared in the following slowpath cases, which are
taken as reasonable hints that the task may no longer use SVE:
 * exec
 * fork and clone

Code is added to sync data between thread.fpsimd_state and
thread.sve_state whenever enabling/disabling SVE, in a manner
consistent with the SVE architectural programmer's model.
Signed-off-by: default avatarDave Martin <Dave.Martin@arm.com>
Reviewed-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Alex Bennée <alex.bennee@linaro.org>
[will: added #include to fix allnoconfig build]
[will: use enable_daif in do_sve_acc]
Signed-off-by: default avatarWill Deacon <will.deacon@arm.com>
parent 22043a3c
...@@ -20,6 +20,8 @@ ...@@ -20,6 +20,8 @@
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
#include <linux/stddef.h>
/* /*
* FP/SIMD storage area has: * FP/SIMD storage area has:
* - FPSR and FPCR * - FPSR and FPCR
...@@ -72,6 +74,20 @@ extern void sve_load_state(void const *state, u32 const *pfpsr, ...@@ -72,6 +74,20 @@ extern void sve_load_state(void const *state, u32 const *pfpsr,
unsigned long vq_minus_1); unsigned long vq_minus_1);
extern unsigned int sve_get_vl(void); extern unsigned int sve_get_vl(void);
#ifdef CONFIG_ARM64_SVE
extern size_t sve_state_size(struct task_struct const *task);
extern void sve_alloc(struct task_struct *task);
extern void fpsimd_release_task(struct task_struct *task);
#else /* ! CONFIG_ARM64_SVE */
static inline void sve_alloc(struct task_struct *task) { }
static inline void fpsimd_release_task(struct task_struct *task) { }
#endif /* ! CONFIG_ARM64_SVE */
/* For use by EFI runtime services calls only */ /* For use by EFI runtime services calls only */
extern void __efi_fpsimd_begin(void); extern void __efi_fpsimd_begin(void);
extern void __efi_fpsimd_end(void); extern void __efi_fpsimd_end(void);
......
...@@ -105,6 +105,8 @@ struct thread_struct { ...@@ -105,6 +105,8 @@ struct thread_struct {
unsigned long tp2_value; unsigned long tp2_value;
#endif #endif
struct fpsimd_state fpsimd_state; struct fpsimd_state fpsimd_state;
void *sve_state; /* SVE registers, if any */
unsigned int sve_vl; /* SVE vector length */
unsigned long fault_address; /* fault info */ unsigned long fault_address; /* fault info */
unsigned long fault_code; /* ESR_EL1 value */ unsigned long fault_code; /* ESR_EL1 value */
struct debug_info debug; /* debugging */ struct debug_info debug; /* debugging */
......
...@@ -63,6 +63,8 @@ struct thread_info { ...@@ -63,6 +63,8 @@ struct thread_info {
void arch_setup_new_exec(void); void arch_setup_new_exec(void);
#define arch_setup_new_exec arch_setup_new_exec #define arch_setup_new_exec arch_setup_new_exec
void arch_release_task_struct(struct task_struct *tsk);
#endif #endif
/* /*
...@@ -92,6 +94,7 @@ void arch_setup_new_exec(void); ...@@ -92,6 +94,7 @@ void arch_setup_new_exec(void);
#define TIF_RESTORE_SIGMASK 20 #define TIF_RESTORE_SIGMASK 20
#define TIF_SINGLESTEP 21 #define TIF_SINGLESTEP 21
#define TIF_32BIT 22 /* 32bit process */ #define TIF_32BIT 22 /* 32bit process */
#define TIF_SVE 23 /* Scalable Vector Extension in use */
#define _TIF_SIGPENDING (1 << TIF_SIGPENDING) #define _TIF_SIGPENDING (1 << TIF_SIGPENDING)
#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
...@@ -105,6 +108,7 @@ void arch_setup_new_exec(void); ...@@ -105,6 +108,7 @@ void arch_setup_new_exec(void);
#define _TIF_UPROBE (1 << TIF_UPROBE) #define _TIF_UPROBE (1 << TIF_UPROBE)
#define _TIF_FSCHECK (1 << TIF_FSCHECK) #define _TIF_FSCHECK (1 << TIF_FSCHECK)
#define _TIF_32BIT (1 << TIF_32BIT) #define _TIF_32BIT (1 << TIF_32BIT)
#define _TIF_SVE (1 << TIF_SVE)
#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \ #define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
_TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \ _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \
......
...@@ -34,6 +34,8 @@ struct undef_hook { ...@@ -34,6 +34,8 @@ struct undef_hook {
void register_undef_hook(struct undef_hook *hook); void register_undef_hook(struct undef_hook *hook);
void unregister_undef_hook(struct undef_hook *hook); void unregister_undef_hook(struct undef_hook *hook);
void force_signal_inject(int signal, int code, struct pt_regs *regs,
unsigned long address);
void arm64_notify_segfault(struct pt_regs *regs, unsigned long addr); void arm64_notify_segfault(struct pt_regs *regs, unsigned long addr);
......
...@@ -599,6 +599,8 @@ el0_sync: ...@@ -599,6 +599,8 @@ el0_sync:
b.eq el0_ia b.eq el0_ia
cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access
b.eq el0_fpsimd_acc b.eq el0_fpsimd_acc
cmp x24, #ESR_ELx_EC_SVE // SVE access
b.eq el0_sve_acc
cmp x24, #ESR_ELx_EC_FP_EXC64 // FP/ASIMD exception cmp x24, #ESR_ELx_EC_FP_EXC64 // FP/ASIMD exception
b.eq el0_fpsimd_exc b.eq el0_fpsimd_exc
cmp x24, #ESR_ELx_EC_SYS64 // configurable trap cmp x24, #ESR_ELx_EC_SYS64 // configurable trap
...@@ -650,6 +652,7 @@ el0_svc_compat: ...@@ -650,6 +652,7 @@ el0_svc_compat:
/* /*
* AArch32 syscall handling * AArch32 syscall handling
*/ */
ldr x16, [tsk, #TSK_TI_FLAGS] // load thread flags
adrp stbl, compat_sys_call_table // load compat syscall table pointer adrp stbl, compat_sys_call_table // load compat syscall table pointer
mov wscno, w7 // syscall number in w7 (r7) mov wscno, w7 // syscall number in w7 (r7)
mov wsc_nr, #__NR_compat_syscalls mov wsc_nr, #__NR_compat_syscalls
...@@ -699,9 +702,19 @@ el0_fpsimd_acc: ...@@ -699,9 +702,19 @@ el0_fpsimd_acc:
mov x1, sp mov x1, sp
bl do_fpsimd_acc bl do_fpsimd_acc
b ret_to_user b ret_to_user
el0_sve_acc:
/*
* Scalable Vector Extension access
*/
enable_daif
ct_user_exit
mov x0, x25
mov x1, sp
bl do_sve_acc
b ret_to_user
el0_fpsimd_exc: el0_fpsimd_exc:
/* /*
* Floating Point or Advanced SIMD exception * Floating Point, Advanced SIMD or SVE exception
*/ */
enable_daif enable_daif
ct_user_exit ct_user_exit
...@@ -849,16 +862,36 @@ ENDPROC(ret_to_user) ...@@ -849,16 +862,36 @@ ENDPROC(ret_to_user)
*/ */
.align 6 .align 6
el0_svc: el0_svc:
ldr x16, [tsk, #TSK_TI_FLAGS] // load thread flags
adrp stbl, sys_call_table // load syscall table pointer adrp stbl, sys_call_table // load syscall table pointer
mov wscno, w8 // syscall number in w8 mov wscno, w8 // syscall number in w8
mov wsc_nr, #__NR_syscalls mov wsc_nr, #__NR_syscalls
#ifndef CONFIG_ARM64_SVE
b el0_svc_naked
#else
tbz x16, #TIF_SVE, el0_svc_naked // Skip unless TIF_SVE set:
bic x16, x16, #_TIF_SVE // discard SVE state
str x16, [tsk, #TSK_TI_FLAGS]
/*
* task_fpsimd_load() won't be called to update CPACR_EL1 in
* ret_to_user unless TIF_FOREIGN_FPSTATE is still set, which only
* happens if a context switch or kernel_neon_begin() or context
* modification (sigreturn, ptrace) intervenes.
* So, ensure that CPACR_EL1 is already correct for the fast-path case:
*/
mrs x9, cpacr_el1
bic x9, x9, #CPACR_EL1_ZEN_EL0EN // disable SVE for el0
msr cpacr_el1, x9 // synchronised by eret to el0
#endif /* CONFIG_ARM64_SVE */
el0_svc_naked: // compat entry point el0_svc_naked: // compat entry point
stp x0, xscno, [sp, #S_ORIG_X0] // save the original x0 and syscall number stp x0, xscno, [sp, #S_ORIG_X0] // save the original x0 and syscall number
enable_daif enable_daif
ct_user_exit 1 ct_user_exit 1
ldr x16, [tsk, #TSK_TI_FLAGS] // check for syscall hooks tst x16, #_TIF_SYSCALL_WORK // check for syscall hooks
tst x16, #_TIF_SYSCALL_WORK
b.ne __sys_trace b.ne __sys_trace
cmp wscno, wsc_nr // check upper syscall limit cmp wscno, wsc_nr // check upper syscall limit
b.hs ni_sys b.hs ni_sys
......
This diff is collapsed.
...@@ -49,6 +49,7 @@ ...@@ -49,6 +49,7 @@
#include <linux/notifier.h> #include <linux/notifier.h>
#include <trace/events/power.h> #include <trace/events/power.h>
#include <linux/percpu.h> #include <linux/percpu.h>
#include <linux/thread_info.h>
#include <asm/alternative.h> #include <asm/alternative.h>
#include <asm/compat.h> #include <asm/compat.h>
...@@ -273,11 +274,27 @@ void release_thread(struct task_struct *dead_task) ...@@ -273,11 +274,27 @@ void release_thread(struct task_struct *dead_task)
{ {
} }
void arch_release_task_struct(struct task_struct *tsk)
{
fpsimd_release_task(tsk);
}
/*
* src and dst may temporarily have aliased sve_state after task_struct
* is copied. We cannot fix this properly here, because src may have
* live SVE state and dst's thread_info may not exist yet, so tweaking
* either src's or dst's TIF_SVE is not safe.
*
* The unaliasing is done in copy_thread() instead. This works because
* dst is not schedulable or traceable until both of these functions
* have been called.
*/
int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
{ {
if (current->mm) if (current->mm)
fpsimd_preserve_current_state(); fpsimd_preserve_current_state();
*dst = *src; *dst = *src;
return 0; return 0;
} }
...@@ -290,6 +307,13 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, ...@@ -290,6 +307,13 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start,
memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context)); memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context));
/*
* Unalias p->thread.sve_state (if any) from the parent task
* and disable discard SVE state for p:
*/
clear_tsk_thread_flag(p, TIF_SVE);
p->thread.sve_state = NULL;
if (likely(!(p->flags & PF_KTHREAD))) { if (likely(!(p->flags & PF_KTHREAD))) {
*childregs = *current_pt_regs(); *childregs = *current_pt_regs();
childregs->regs[0] = 0; childregs->regs[0] = 0;
......
...@@ -311,8 +311,8 @@ static int call_undef_hook(struct pt_regs *regs) ...@@ -311,8 +311,8 @@ static int call_undef_hook(struct pt_regs *regs)
return fn ? fn(regs, instr) : 1; return fn ? fn(regs, instr) : 1;
} }
static void force_signal_inject(int signal, int code, struct pt_regs *regs, void force_signal_inject(int signal, int code, struct pt_regs *regs,
unsigned long address) unsigned long address)
{ {
siginfo_t info; siginfo_t info;
void __user *pc = (void __user *)instruction_pointer(regs); void __user *pc = (void __user *)instruction_pointer(regs);
...@@ -326,7 +326,7 @@ static void force_signal_inject(int signal, int code, struct pt_regs *regs, ...@@ -326,7 +326,7 @@ static void force_signal_inject(int signal, int code, struct pt_regs *regs,
desc = "illegal memory access"; desc = "illegal memory access";
break; break;
default: default:
desc = "bad mode"; desc = "unknown or unrecoverable error";
break; break;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment