Commit 2671828c authored by James Morse's avatar James Morse Committed by Will Deacon

arm64: entry: Move ct_user_exit before any other exception

When taking an SError or Debug exception from EL0, we run the C
handler for these exceptions before updating the context tracking
code and unmasking lower priority interrupts.

When booting with nohz_full lockdep tells us we got this wrong:
| =============================
| WARNING: suspicious RCU usage
| 5.3.0-rc2-00010-gb4b5e9dcb11b-dirty #11271 Not tainted
| -----------------------------
| include/linux/rcupdate.h:643 rcu_read_unlock() used illegally wh!
|
| other info that might help us debug this:
|
|
| RCU used illegally from idle CPU!
| rcu_scheduler_active = 2, debug_locks = 1
| RCU used illegally from extended quiescent state!
| 1 lock held by a.out/432:
|  #0: 00000000c7a79515 (rcu_read_lock){....}, at: brk_handler+0x00
|
| stack backtrace:
| CPU: 1 PID: 432 Comm: a.out Not tainted 5.3.0-rc2-00010-gb4b5e9d1
| Hardware name: ARM LTD ARM Juno Development Platform/ARM Juno De8
| Call trace:
|  dump_backtrace+0x0/0x140
|  show_stack+0x14/0x20
|  dump_stack+0xbc/0x104
|  lockdep_rcu_suspicious+0xf8/0x108
|  brk_handler+0x164/0x1b0
|  do_debug_exception+0x11c/0x278
|  el0_dbg+0x14/0x20

Moving the ct_user_exit calls to be before do_debug_exception() means
they are also before trace_hardirqs_off() has been updated. Add a new
ct_user_exit_irqoff macro to avoid the context-tracking code using
irqsave/restore before we've updated trace_hardirqs_off(). To be
consistent, do this everywhere.

The C helper is called enter_from_user_mode() to match x86 in the hope
we can merge them into kernel/context_tracking.c later.

Cc: Masami Hiramatsu <mhiramat@kernel.org>
Fixes: 6c81fe79 ("arm64: enable context tracking")
Signed-off-by: default avatarJames Morse <james.morse@arm.com>
Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent 2d122942
......@@ -30,4 +30,6 @@ static inline u32 disr_to_esr(u64 disr)
return esr;
}
asmlinkage void enter_from_user_mode(void);
#endif /* __ASM_EXCEPTION_H */
......@@ -30,9 +30,9 @@
* Context tracking subsystem. Used to instrument transitions
* between user and kernel mode.
*/
.macro ct_user_exit
.macro ct_user_exit_irqoff
#ifdef CONFIG_CONTEXT_TRACKING
bl context_tracking_user_exit
bl enter_from_user_mode
#endif
.endm
......@@ -792,8 +792,8 @@ el0_cp15:
/*
* Trapped CP15 (MRC, MCR, MRRC, MCRR) instructions
*/
ct_user_exit_irqoff
enable_daif
ct_user_exit
mov x0, x25
mov x1, sp
bl do_cp15instr
......@@ -805,8 +805,8 @@ el0_da:
* Data abort handling
*/
mrs x26, far_el1
ct_user_exit_irqoff
enable_daif
ct_user_exit
clear_address_tag x0, x26
mov x1, x25
mov x2, sp
......@@ -818,11 +818,11 @@ el0_ia:
*/
mrs x26, far_el1
gic_prio_kentry_setup tmp=x0
ct_user_exit_irqoff
enable_da_f
#ifdef CONFIG_TRACE_IRQFLAGS
bl trace_hardirqs_off
#endif
ct_user_exit
mov x0, x26
mov x1, x25
mov x2, sp
......@@ -832,8 +832,8 @@ el0_fpsimd_acc:
/*
* Floating Point or Advanced SIMD access
*/
ct_user_exit_irqoff
enable_daif
ct_user_exit
mov x0, x25
mov x1, sp
bl do_fpsimd_acc
......@@ -842,8 +842,8 @@ el0_sve_acc:
/*
* Scalable Vector Extension access
*/
ct_user_exit_irqoff
enable_daif
ct_user_exit
mov x0, x25
mov x1, sp
bl do_sve_acc
......@@ -852,8 +852,8 @@ el0_fpsimd_exc:
/*
* Floating Point, Advanced SIMD or SVE exception
*/
ct_user_exit_irqoff
enable_daif
ct_user_exit
mov x0, x25
mov x1, sp
bl do_fpsimd_exc
......@@ -868,11 +868,11 @@ el0_sp_pc:
* Stack or PC alignment exception handling
*/
gic_prio_kentry_setup tmp=x0
ct_user_exit_irqoff
enable_da_f
#ifdef CONFIG_TRACE_IRQFLAGS
bl trace_hardirqs_off
#endif
ct_user_exit
mov x0, x26
mov x1, x25
mov x2, sp
......@@ -882,8 +882,8 @@ el0_undef:
/*
* Undefined instruction
*/
ct_user_exit_irqoff
enable_daif
ct_user_exit
mov x0, sp
bl do_undefinstr
b ret_to_user
......@@ -891,8 +891,8 @@ el0_sys:
/*
* System instructions, for trapped cache maintenance instructions
*/
ct_user_exit_irqoff
enable_daif
ct_user_exit
mov x0, x25
mov x1, sp
bl do_sysinstr
......@@ -902,17 +902,18 @@ el0_dbg:
* Debug exception handling
*/
tbnz x24, #0, el0_inv // EL0 only
mrs x24, far_el1
gic_prio_kentry_setup tmp=x3
mrs x0, far_el1
ct_user_exit_irqoff
mov x0, x24
mov x1, x25
mov x2, sp
bl do_debug_exception
enable_da_f
ct_user_exit
b ret_to_user
el0_inv:
ct_user_exit_irqoff
enable_daif
ct_user_exit
mov x0, sp
mov x1, #BAD_SYNC
mov x2, x25
......@@ -925,13 +926,13 @@ el0_irq:
kernel_entry 0
el0_irq_naked:
gic_prio_irq_setup pmr=x20, tmp=x0
ct_user_exit_irqoff
enable_da_f
#ifdef CONFIG_TRACE_IRQFLAGS
bl trace_hardirqs_off
#endif
ct_user_exit
#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
tbz x22, #55, 1f
bl do_el0_irq_bp_hardening
......@@ -958,13 +959,14 @@ ENDPROC(el1_error)
el0_error:
kernel_entry 0
el0_error_naked:
mrs x1, esr_el1
mrs x25, esr_el1
gic_prio_kentry_setup tmp=x2
ct_user_exit_irqoff
enable_dbg
mov x0, sp
mov x1, x25
bl do_serror
enable_da_f
ct_user_exit
b ret_to_user
ENDPROC(el0_error)
......
......@@ -7,9 +7,11 @@
*/
#include <linux/bug.h>
#include <linux/context_tracking.h>
#include <linux/signal.h>
#include <linux/personality.h>
#include <linux/kallsyms.h>
#include <linux/kprobes.h>
#include <linux/spinlock.h>
#include <linux/uaccess.h>
#include <linux/hardirq.h>
......@@ -900,6 +902,13 @@ asmlinkage void do_serror(struct pt_regs *regs, unsigned int esr)
nmi_exit();
}
asmlinkage void enter_from_user_mode(void)
{
CT_WARN_ON(ct_state() != CONTEXT_USER);
user_exit_irqoff();
}
NOKPROBE_SYMBOL(enter_from_user_mode);
void __pte_error(const char *file, int line, unsigned long val)
{
pr_err("%s:%d: bad pte %016lx.\n", file, line, val);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment