Commit 190f1ca8 authored by Ard Biesheuvel's avatar Ard Biesheuvel

arm64: add support for kernel mode NEON in interrupt context

This patch modifies kernel_neon_begin() and kernel_neon_end(), so
they may be called from any context. To address the case where only
a couple of registers are needed, kernel_neon_begin_partial(u32) is
introduced which takes as a parameter the number of bottom 'n' NEON
q-registers required. To mark the end of such a partial section, the
regular kernel_neon_end() should be used.
Signed-off-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
parent 005f78cd
...@@ -41,6 +41,17 @@ struct fpsimd_state { ...@@ -41,6 +41,17 @@ struct fpsimd_state {
unsigned int cpu; unsigned int cpu;
}; };
/*
* Struct for stacking the bottom 'n' FP/SIMD registers.
*/
struct fpsimd_partial_state {
u32 fpsr;
u32 fpcr;
u32 num_regs;
__uint128_t vregs[32];
};
#if defined(__KERNEL__) && defined(CONFIG_COMPAT) #if defined(__KERNEL__) && defined(CONFIG_COMPAT)
/* Masks for extracting the FPSR and FPCR from the FPSCR */ /* Masks for extracting the FPSR and FPCR from the FPSCR */
#define VFP_FPSCR_STAT_MASK 0xf800009f #define VFP_FPSCR_STAT_MASK 0xf800009f
...@@ -66,6 +77,10 @@ extern void fpsimd_update_current_state(struct fpsimd_state *state); ...@@ -66,6 +77,10 @@ extern void fpsimd_update_current_state(struct fpsimd_state *state);
extern void fpsimd_flush_task_state(struct task_struct *target); extern void fpsimd_flush_task_state(struct task_struct *target);
extern void fpsimd_save_partial_state(struct fpsimd_partial_state *state,
u32 num_regs);
extern void fpsimd_load_partial_state(struct fpsimd_partial_state *state);
#endif #endif
#endif #endif
...@@ -62,3 +62,38 @@ ...@@ -62,3 +62,38 @@
ldr w\tmpnr, [\state, #16 * 2 + 4] ldr w\tmpnr, [\state, #16 * 2 + 4]
msr fpcr, x\tmpnr msr fpcr, x\tmpnr
.endm .endm
.altmacro
.macro fpsimd_save_partial state, numnr, tmpnr1, tmpnr2
mrs x\tmpnr1, fpsr
str w\numnr, [\state, #8]
mrs x\tmpnr2, fpcr
stp w\tmpnr1, w\tmpnr2, [\state]
adr x\tmpnr1, 0f
add \state, \state, x\numnr, lsl #4
sub x\tmpnr1, x\tmpnr1, x\numnr, lsl #1
br x\tmpnr1
.irp qa, 30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0
.irp qb, %(qa + 1)
stp q\qa, q\qb, [\state, # -16 * \qa - 16]
.endr
.endr
0:
.endm
.macro fpsimd_restore_partial state, tmpnr1, tmpnr2
ldp w\tmpnr1, w\tmpnr2, [\state]
msr fpsr, x\tmpnr1
msr fpcr, x\tmpnr2
adr x\tmpnr1, 0f
ldr w\tmpnr2, [\state, #8]
add \state, \state, x\tmpnr2, lsl #4
sub x\tmpnr1, x\tmpnr1, x\tmpnr2, lsl #1
br x\tmpnr1
.irp qa, 30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0
.irp qb, %(qa + 1)
ldp q\qa, q\qb, [\state, # -16 * \qa - 16]
.endr
.endr
0:
.endm
...@@ -8,7 +8,11 @@ ...@@ -8,7 +8,11 @@
* published by the Free Software Foundation. * published by the Free Software Foundation.
*/ */
#include <linux/types.h>
#define cpu_has_neon() (1) #define cpu_has_neon() (1)
void kernel_neon_begin(void); #define kernel_neon_begin() kernel_neon_begin_partial(32)
void kernel_neon_begin_partial(u32 num_regs);
void kernel_neon_end(void); void kernel_neon_end(void);
...@@ -41,3 +41,27 @@ ENTRY(fpsimd_load_state) ...@@ -41,3 +41,27 @@ ENTRY(fpsimd_load_state)
fpsimd_restore x0, 8 fpsimd_restore x0, 8
ret ret
ENDPROC(fpsimd_load_state) ENDPROC(fpsimd_load_state)
#ifdef CONFIG_KERNEL_MODE_NEON
/*
* Save the bottom n FP registers.
*
* x0 - pointer to struct fpsimd_partial_state
*/
ENTRY(fpsimd_save_partial_state)
fpsimd_save_partial x0, 1, 8, 9
ret
ENDPROC(fpsimd_load_partial_state)
/*
* Load the bottom n FP registers.
*
* x0 - pointer to struct fpsimd_partial_state
*/
ENTRY(fpsimd_load_partial_state)
fpsimd_restore_partial x0, 8, 9
ret
ENDPROC(fpsimd_load_partial_state)
#endif
...@@ -218,29 +218,45 @@ void fpsimd_flush_task_state(struct task_struct *t) ...@@ -218,29 +218,45 @@ void fpsimd_flush_task_state(struct task_struct *t)
#ifdef CONFIG_KERNEL_MODE_NEON #ifdef CONFIG_KERNEL_MODE_NEON
static DEFINE_PER_CPU(struct fpsimd_partial_state, hardirq_fpsimdstate);
static DEFINE_PER_CPU(struct fpsimd_partial_state, softirq_fpsimdstate);
/* /*
* Kernel-side NEON support functions * Kernel-side NEON support functions
*/ */
void kernel_neon_begin(void) void kernel_neon_begin_partial(u32 num_regs)
{ {
/* Avoid using the NEON in interrupt context */ if (in_interrupt()) {
BUG_ON(in_interrupt()); struct fpsimd_partial_state *s = this_cpu_ptr(
preempt_disable(); in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate);
BUG_ON(num_regs > 32);
fpsimd_save_partial_state(s, roundup(num_regs, 2));
} else {
/* /*
* Save the userland FPSIMD state if we have one and if we haven't done * Save the userland FPSIMD state if we have one and if we
* so already. Clear fpsimd_last_state to indicate that there is no * haven't done so already. Clear fpsimd_last_state to indicate
* longer userland FPSIMD state in the registers. * that there is no longer userland FPSIMD state in the
* registers.
*/ */
if (current->mm && !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE)) preempt_disable();
if (current->mm &&
!test_and_set_thread_flag(TIF_FOREIGN_FPSTATE))
fpsimd_save_state(&current->thread.fpsimd_state); fpsimd_save_state(&current->thread.fpsimd_state);
this_cpu_write(fpsimd_last_state, NULL); this_cpu_write(fpsimd_last_state, NULL);
}
} }
EXPORT_SYMBOL(kernel_neon_begin); EXPORT_SYMBOL(kernel_neon_begin_partial);
void kernel_neon_end(void) void kernel_neon_end(void)
{ {
if (in_interrupt()) {
struct fpsimd_partial_state *s = this_cpu_ptr(
in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate);
fpsimd_load_partial_state(s);
} else {
preempt_enable(); preempt_enable();
}
} }
EXPORT_SYMBOL(kernel_neon_end); EXPORT_SYMBOL(kernel_neon_end);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment