Commit df5b95be authored by Catalin Marinas's avatar Catalin Marinas

Merge branch 'arm64/vmap-stack' of...

Merge branch 'arm64/vmap-stack' of git://git.kernel.org/pub/scm/linux/kernel/git/mark/linux into for-next/core

* 'arm64/vmap-stack' of git://git.kernel.org/pub/scm/linux/kernel/git/mark/linux:
  arm64: add VMAP_STACK overflow detection
  arm64: add on_accessible_stack()
  arm64: add basic VMAP_STACK support
  arm64: use an irq stack pointer
  arm64: assembler: allow adr_this_cpu to use the stack pointer
  arm64: factor out entry stack manipulation
  efi/arm64: add EFI_KIMG_ALIGN
  arm64: move SEGMENT_ALIGN to <asm/memory.h>
  arm64: clean up irq stack definitions
  arm64: clean up THREAD_* definitions
  arm64: factor out PAGE_* and CONT_* definitions
  arm64: kernel: remove {THREAD,IRQ_STACK}_START_SP
  fork: allow arch-override of VMAP stack alignment
  arm64: remove __die()'s stack dump
parents 969ff73e 872d8327
...@@ -75,6 +75,7 @@ config ARM64 ...@@ -75,6 +75,7 @@ config ARM64
select HAVE_ARCH_SECCOMP_FILTER select HAVE_ARCH_SECCOMP_FILTER
select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_TRACEHOOK
select HAVE_ARCH_TRANSPARENT_HUGEPAGE select HAVE_ARCH_TRANSPARENT_HUGEPAGE
select HAVE_ARCH_VMAP_STACK
select HAVE_ARM_SMCCC select HAVE_ARM_SMCCC
select HAVE_EBPF_JIT select HAVE_EBPF_JIT
select HAVE_C_RECORDMCOUNT select HAVE_C_RECORDMCOUNT
......
...@@ -230,12 +230,18 @@ lr .req x30 // link register ...@@ -230,12 +230,18 @@ lr .req x30 // link register
.endm .endm
/* /*
* @dst: Result of per_cpu(sym, smp_processor_id()) * @dst: Result of per_cpu(sym, smp_processor_id()), can be SP for
* non-module code
* @sym: The name of the per-cpu variable * @sym: The name of the per-cpu variable
* @tmp: scratch register * @tmp: scratch register
*/ */
.macro adr_this_cpu, dst, sym, tmp .macro adr_this_cpu, dst, sym, tmp
#ifndef MODULE
adrp \tmp, \sym
add \dst, \tmp, #:lo12:\sym
#else
adr_l \dst, \sym adr_l \dst, \sym
#endif
mrs \tmp, tpidr_el1 mrs \tmp, tpidr_el1
add \dst, \dst, \tmp add \dst, \dst, \tmp
.endm .endm
......
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
#include <asm/boot.h> #include <asm/boot.h>
#include <asm/cpufeature.h> #include <asm/cpufeature.h>
#include <asm/io.h> #include <asm/io.h>
#include <asm/memory.h>
#include <asm/mmu_context.h> #include <asm/mmu_context.h>
#include <asm/neon.h> #include <asm/neon.h>
#include <asm/ptrace.h> #include <asm/ptrace.h>
...@@ -48,6 +49,13 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); ...@@ -48,6 +49,13 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md);
*/ */
#define EFI_FDT_ALIGN SZ_2M /* used by allocate_new_fdt_and_exit_boot() */ #define EFI_FDT_ALIGN SZ_2M /* used by allocate_new_fdt_and_exit_boot() */
/*
* In some configurations (e.g. VMAP_STACK && 64K pages), stacks built into the
* kernel need greater alignment than we require the segments to be padded to.
*/
#define EFI_KIMG_ALIGN \
(SEGMENT_ALIGN > THREAD_ALIGN ? SEGMENT_ALIGN : THREAD_ALIGN)
/* on arm64, the FDT may be located anywhere in system RAM */ /* on arm64, the FDT may be located anywhere in system RAM */
static inline unsigned long efi_get_max_fdt_addr(unsigned long dram_base) static inline unsigned long efi_get_max_fdt_addr(unsigned long dram_base)
{ {
......
#ifndef __ASM_IRQ_H #ifndef __ASM_IRQ_H
#define __ASM_IRQ_H #define __ASM_IRQ_H
#define IRQ_STACK_SIZE THREAD_SIZE
#define IRQ_STACK_START_SP THREAD_START_SP
#ifndef __ASSEMBLER__ #ifndef __ASSEMBLER__
#include <linux/percpu.h>
#include <linux/sched/task_stack.h>
#include <asm-generic/irq.h> #include <asm-generic/irq.h>
#include <asm/thread_info.h>
struct pt_regs; struct pt_regs;
DECLARE_PER_CPU(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack);
extern void set_handle_irq(void (*handle_irq)(struct pt_regs *)); extern void set_handle_irq(void (*handle_irq)(struct pt_regs *));
static inline int nr_legacy_irqs(void) static inline int nr_legacy_irqs(void)
...@@ -23,21 +14,5 @@ static inline int nr_legacy_irqs(void) ...@@ -23,21 +14,5 @@ static inline int nr_legacy_irqs(void)
return 0; return 0;
} }
static inline bool on_irq_stack(unsigned long sp)
{
unsigned long low = (unsigned long)raw_cpu_ptr(irq_stack);
unsigned long high = low + IRQ_STACK_START_SP;
return (low <= sp && sp <= high);
}
static inline bool on_task_stack(struct task_struct *tsk, unsigned long sp)
{
unsigned long low = (unsigned long)task_stack_page(tsk);
unsigned long high = low + THREAD_SIZE;
return (low <= sp && sp < high);
}
#endif /* !__ASSEMBLER__ */ #endif /* !__ASSEMBLER__ */
#endif #endif
...@@ -25,6 +25,7 @@ ...@@ -25,6 +25,7 @@
#include <linux/const.h> #include <linux/const.h>
#include <linux/types.h> #include <linux/types.h>
#include <asm/bug.h> #include <asm/bug.h>
#include <asm/page-def.h>
#include <asm/sizes.h> #include <asm/sizes.h>
/* /*
...@@ -103,6 +104,58 @@ ...@@ -103,6 +104,58 @@
#define KASAN_SHADOW_SIZE (0) #define KASAN_SHADOW_SIZE (0)
#endif #endif
#define MIN_THREAD_SHIFT 14
/*
* VMAP'd stacks are allocated at page granularity, so we must ensure that such
* stacks are a multiple of page size.
*/
#if defined(CONFIG_VMAP_STACK) && (MIN_THREAD_SHIFT < PAGE_SHIFT)
#define THREAD_SHIFT PAGE_SHIFT
#else
#define THREAD_SHIFT MIN_THREAD_SHIFT
#endif
#if THREAD_SHIFT >= PAGE_SHIFT
#define THREAD_SIZE_ORDER (THREAD_SHIFT - PAGE_SHIFT)
#endif
#define THREAD_SIZE (UL(1) << THREAD_SHIFT)
/*
* By aligning VMAP'd stacks to 2 * THREAD_SIZE, we can detect overflow by
* checking sp & (1 << THREAD_SHIFT), which we can do cheaply in the entry
* assembly.
*/
#ifdef CONFIG_VMAP_STACK
#define THREAD_ALIGN (2 * THREAD_SIZE)
#else
#define THREAD_ALIGN THREAD_SIZE
#endif
#define IRQ_STACK_SIZE THREAD_SIZE
#define OVERFLOW_STACK_SIZE SZ_4K
/*
* Alignment of kernel segments (e.g. .text, .data).
*/
#if defined(CONFIG_DEBUG_ALIGN_RODATA)
/*
* 4 KB granule: 1 level 2 entry
* 16 KB granule: 128 level 3 entries, with contiguous bit
* 64 KB granule: 32 level 3 entries, with contiguous bit
*/
#define SEGMENT_ALIGN SZ_2M
#else
/*
* 4 KB granule: 16 level 3 entries, with contiguous bit
* 16 KB granule: 4 level 3 entries, without contiguous bit
* 64 KB granule: 1 level 3 entry
*/
#define SEGMENT_ALIGN SZ_64K
#endif
/* /*
* Memory types available. * Memory types available.
*/ */
......
/*
* Based on arch/arm/include/asm/page.h
*
* Copyright (C) 1995-2003 Russell King
* Copyright (C) 2017 ARM Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __ASM_PAGE_DEF_H
#define __ASM_PAGE_DEF_H
#include <linux/const.h>
/* PAGE_SHIFT determines the page size */
/* CONT_SHIFT determines the number of pages which can be tracked together */
#define PAGE_SHIFT CONFIG_ARM64_PAGE_SHIFT
#define CONT_SHIFT CONFIG_ARM64_CONT_SHIFT
#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
#define PAGE_MASK (~(PAGE_SIZE-1))
#define CONT_SIZE (_AC(1, UL) << (CONT_SHIFT + PAGE_SHIFT))
#define CONT_MASK (~(CONT_SIZE-1))
#endif /* __ASM_PAGE_DEF_H */
...@@ -19,17 +19,7 @@ ...@@ -19,17 +19,7 @@
#ifndef __ASM_PAGE_H #ifndef __ASM_PAGE_H
#define __ASM_PAGE_H #define __ASM_PAGE_H
#include <linux/const.h> #include <asm/page-def.h>
/* PAGE_SHIFT determines the page size */
/* CONT_SHIFT determines the number of pages which can be tracked together */
#define PAGE_SHIFT CONFIG_ARM64_PAGE_SHIFT
#define CONT_SHIFT CONFIG_ARM64_CONT_SHIFT
#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
#define PAGE_MASK (~(PAGE_SIZE-1))
#define CONT_SIZE (_AC(1, UL) << (CONT_SHIFT + PAGE_SHIFT))
#define CONT_MASK (~(CONT_SIZE-1))
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
......
...@@ -159,7 +159,7 @@ extern struct task_struct *cpu_switch_to(struct task_struct *prev, ...@@ -159,7 +159,7 @@ extern struct task_struct *cpu_switch_to(struct task_struct *prev,
struct task_struct *next); struct task_struct *next);
#define task_pt_regs(p) \ #define task_pt_regs(p) \
((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1) ((struct pt_regs *)(THREAD_SIZE + task_stack_page(p)) - 1)
#define KSTK_EIP(tsk) ((unsigned long)task_pt_regs(tsk)->pc) #define KSTK_EIP(tsk) ((unsigned long)task_pt_regs(tsk)->pc)
#define KSTK_ESP(tsk) user_stack_pointer(task_pt_regs(tsk)) #define KSTK_ESP(tsk) user_stack_pointer(task_pt_regs(tsk))
......
...@@ -16,7 +16,12 @@ ...@@ -16,7 +16,12 @@
#ifndef __ASM_STACKTRACE_H #ifndef __ASM_STACKTRACE_H
#define __ASM_STACKTRACE_H #define __ASM_STACKTRACE_H
struct task_struct; #include <linux/percpu.h>
#include <linux/sched.h>
#include <linux/sched/task_stack.h>
#include <asm/memory.h>
#include <asm/ptrace.h>
struct stackframe { struct stackframe {
unsigned long fp; unsigned long fp;
...@@ -31,4 +36,57 @@ extern void walk_stackframe(struct task_struct *tsk, struct stackframe *frame, ...@@ -31,4 +36,57 @@ extern void walk_stackframe(struct task_struct *tsk, struct stackframe *frame,
int (*fn)(struct stackframe *, void *), void *data); int (*fn)(struct stackframe *, void *), void *data);
extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk); extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk);
DECLARE_PER_CPU(unsigned long *, irq_stack_ptr);
static inline bool on_irq_stack(unsigned long sp)
{
unsigned long low = (unsigned long)raw_cpu_read(irq_stack_ptr);
unsigned long high = low + IRQ_STACK_SIZE;
if (!low)
return false;
return (low <= sp && sp < high);
}
static inline bool on_task_stack(struct task_struct *tsk, unsigned long sp)
{
unsigned long low = (unsigned long)task_stack_page(tsk);
unsigned long high = low + THREAD_SIZE;
return (low <= sp && sp < high);
}
#ifdef CONFIG_VMAP_STACK
DECLARE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack);
static inline bool on_overflow_stack(unsigned long sp)
{
unsigned long low = (unsigned long)raw_cpu_ptr(overflow_stack);
unsigned long high = low + OVERFLOW_STACK_SIZE;
return (low <= sp && sp < high);
}
#else
static inline bool on_overflow_stack(unsigned long sp) { return false; }
#endif
/*
* We can only safely access per-cpu stacks from current in a non-preemptible
* context.
*/
static inline bool on_accessible_stack(struct task_struct *tsk, unsigned long sp)
{
if (on_task_stack(tsk, sp))
return true;
if (tsk != current || preemptible())
return false;
if (on_irq_stack(sp))
return true;
if (on_overflow_stack(sp))
return true;
return false;
}
#endif /* __ASM_STACKTRACE_H */ #endif /* __ASM_STACKTRACE_H */
...@@ -23,19 +23,11 @@ ...@@ -23,19 +23,11 @@
#include <linux/compiler.h> #include <linux/compiler.h>
#ifdef CONFIG_ARM64_4K_PAGES
#define THREAD_SIZE_ORDER 2
#elif defined(CONFIG_ARM64_16K_PAGES)
#define THREAD_SIZE_ORDER 0
#endif
#define THREAD_SIZE 16384
#define THREAD_START_SP (THREAD_SIZE - 16)
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
struct task_struct; struct task_struct;
#include <asm/memory.h>
#include <asm/stack_pointer.h> #include <asm/stack_pointer.h>
#include <asm/types.h> #include <asm/types.h>
......
...@@ -69,8 +69,55 @@ ...@@ -69,8 +69,55 @@
#define BAD_FIQ 2 #define BAD_FIQ 2
#define BAD_ERROR 3 #define BAD_ERROR 3
.macro kernel_entry, el, regsize = 64 .macro kernel_ventry label
.align 7
sub sp, sp, #S_FRAME_SIZE sub sp, sp, #S_FRAME_SIZE
#ifdef CONFIG_VMAP_STACK
/*
* Test whether the SP has overflowed, without corrupting a GPR.
* Task and IRQ stacks are aligned to (1 << THREAD_SHIFT).
*/
add sp, sp, x0 // sp' = sp + x0
sub x0, sp, x0 // x0' = sp' - x0 = (sp + x0) - x0 = sp
tbnz x0, #THREAD_SHIFT, 0f
sub x0, sp, x0 // x0'' = sp' - x0' = (sp + x0) - sp = x0
sub sp, sp, x0 // sp'' = sp' - x0 = (sp + x0) - x0 = sp
b \label
0:
/*
* Either we've just detected an overflow, or we've taken an exception
* while on the overflow stack. Either way, we won't return to
* userspace, and can clobber EL0 registers to free up GPRs.
*/
/* Stash the original SP (minus S_FRAME_SIZE) in tpidr_el0. */
msr tpidr_el0, x0
/* Recover the original x0 value and stash it in tpidrro_el0 */
sub x0, sp, x0
msr tpidrro_el0, x0
/* Switch to the overflow stack */
adr_this_cpu sp, overflow_stack + OVERFLOW_STACK_SIZE, x0
/*
* Check whether we were already on the overflow stack. This may happen
* after panic() re-enables interrupts.
*/
mrs x0, tpidr_el0 // sp of interrupted context
sub x0, sp, x0 // delta with top of overflow stack
tst x0, #~(OVERFLOW_STACK_SIZE - 1) // within range?
b.ne __bad_stack // no? -> bad stack pointer
/* We were already on the overflow stack. Restore sp/x0 and carry on. */
sub sp, sp, x0
mrs x0, tpidrro_el0
#endif
b \label
.endm
.macro kernel_entry, el, regsize = 64
.if \regsize == 32 .if \regsize == 32
mov w0, w0 // zero upper 32 bits of x0 mov w0, w0 // zero upper 32 bits of x0
.endif .endif
...@@ -269,8 +316,8 @@ alternative_else_nop_endif ...@@ -269,8 +316,8 @@ alternative_else_nop_endif
and x25, x25, #~(THREAD_SIZE - 1) and x25, x25, #~(THREAD_SIZE - 1)
cbnz x25, 9998f cbnz x25, 9998f
adr_this_cpu x25, irq_stack, x26 ldr_this_cpu x25, irq_stack_ptr, x26
mov x26, #IRQ_STACK_START_SP mov x26, #IRQ_STACK_SIZE
add x26, x25, x26 add x26, x25, x26
/* switch to the irq stack */ /* switch to the irq stack */
...@@ -318,34 +365,62 @@ tsk .req x28 // current thread_info ...@@ -318,34 +365,62 @@ tsk .req x28 // current thread_info
.align 11 .align 11
ENTRY(vectors) ENTRY(vectors)
ventry el1_sync_invalid // Synchronous EL1t kernel_ventry el1_sync_invalid // Synchronous EL1t
ventry el1_irq_invalid // IRQ EL1t kernel_ventry el1_irq_invalid // IRQ EL1t
ventry el1_fiq_invalid // FIQ EL1t kernel_ventry el1_fiq_invalid // FIQ EL1t
ventry el1_error_invalid // Error EL1t kernel_ventry el1_error_invalid // Error EL1t
ventry el1_sync // Synchronous EL1h kernel_ventry el1_sync // Synchronous EL1h
ventry el1_irq // IRQ EL1h kernel_ventry el1_irq // IRQ EL1h
ventry el1_fiq_invalid // FIQ EL1h kernel_ventry el1_fiq_invalid // FIQ EL1h
ventry el1_error_invalid // Error EL1h kernel_ventry el1_error_invalid // Error EL1h
ventry el0_sync // Synchronous 64-bit EL0 kernel_ventry el0_sync // Synchronous 64-bit EL0
ventry el0_irq // IRQ 64-bit EL0 kernel_ventry el0_irq // IRQ 64-bit EL0
ventry el0_fiq_invalid // FIQ 64-bit EL0 kernel_ventry el0_fiq_invalid // FIQ 64-bit EL0
ventry el0_error_invalid // Error 64-bit EL0 kernel_ventry el0_error_invalid // Error 64-bit EL0
#ifdef CONFIG_COMPAT #ifdef CONFIG_COMPAT
ventry el0_sync_compat // Synchronous 32-bit EL0 kernel_ventry el0_sync_compat // Synchronous 32-bit EL0
ventry el0_irq_compat // IRQ 32-bit EL0 kernel_ventry el0_irq_compat // IRQ 32-bit EL0
ventry el0_fiq_invalid_compat // FIQ 32-bit EL0 kernel_ventry el0_fiq_invalid_compat // FIQ 32-bit EL0
ventry el0_error_invalid_compat // Error 32-bit EL0 kernel_ventry el0_error_invalid_compat // Error 32-bit EL0
#else #else
ventry el0_sync_invalid // Synchronous 32-bit EL0 kernel_ventry el0_sync_invalid // Synchronous 32-bit EL0
ventry el0_irq_invalid // IRQ 32-bit EL0 kernel_ventry el0_irq_invalid // IRQ 32-bit EL0
ventry el0_fiq_invalid // FIQ 32-bit EL0 kernel_ventry el0_fiq_invalid // FIQ 32-bit EL0
ventry el0_error_invalid // Error 32-bit EL0 kernel_ventry el0_error_invalid // Error 32-bit EL0
#endif #endif
END(vectors) END(vectors)
#ifdef CONFIG_VMAP_STACK
/*
* We detected an overflow in kernel_ventry, which switched to the
* overflow stack. Stash the exception regs, and head to our overflow
* handler.
*/
__bad_stack:
/* Restore the original x0 value */
mrs x0, tpidrro_el0
/*
* Store the original GPRs to the new stack. The orginal SP (minus
* S_FRAME_SIZE) was stashed in tpidr_el0 by kernel_ventry.
*/
sub sp, sp, #S_FRAME_SIZE
kernel_entry 1
mrs x0, tpidr_el0
add x0, x0, #S_FRAME_SIZE
str x0, [sp, #S_SP]
/* Stash the regs for handle_bad_stack */
mov x0, sp
/* Time to die */
bl handle_bad_stack
ASM_BUG()
#endif /* CONFIG_VMAP_STACK */
/* /*
* Invalid mode handlers * Invalid mode handlers
*/ */
......
...@@ -23,15 +23,16 @@ ...@@ -23,15 +23,16 @@
#include <linux/kernel_stat.h> #include <linux/kernel_stat.h>
#include <linux/irq.h> #include <linux/irq.h>
#include <linux/memory.h>
#include <linux/smp.h> #include <linux/smp.h>
#include <linux/init.h> #include <linux/init.h>
#include <linux/irqchip.h> #include <linux/irqchip.h>
#include <linux/seq_file.h> #include <linux/seq_file.h>
#include <linux/vmalloc.h>
unsigned long irq_err_count; unsigned long irq_err_count;
/* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */ DEFINE_PER_CPU(unsigned long *, irq_stack_ptr);
DEFINE_PER_CPU(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack) __aligned(16);
int arch_show_interrupts(struct seq_file *p, int prec) int arch_show_interrupts(struct seq_file *p, int prec)
{ {
...@@ -50,8 +51,43 @@ void __init set_handle_irq(void (*handle_irq)(struct pt_regs *)) ...@@ -50,8 +51,43 @@ void __init set_handle_irq(void (*handle_irq)(struct pt_regs *))
handle_arch_irq = handle_irq; handle_arch_irq = handle_irq;
} }
#ifdef CONFIG_VMAP_STACK
static void init_irq_stacks(void)
{
int cpu;
unsigned long *p;
for_each_possible_cpu(cpu) {
/*
* To ensure that VMAP'd stack overflow detection works
* correctly, the IRQ stacks need to have the same
* alignment as other stacks.
*/
p = __vmalloc_node_range(IRQ_STACK_SIZE, THREAD_ALIGN,
VMALLOC_START, VMALLOC_END,
THREADINFO_GFP, PAGE_KERNEL,
0, cpu_to_node(cpu),
__builtin_return_address(0));
per_cpu(irq_stack_ptr, cpu) = p;
}
}
#else
/* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */
DEFINE_PER_CPU_ALIGNED(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack);
static void init_irq_stacks(void)
{
int cpu;
for_each_possible_cpu(cpu)
per_cpu(irq_stack_ptr, cpu) = per_cpu(irq_stack, cpu);
}
#endif
void __init init_IRQ(void) void __init init_IRQ(void)
{ {
init_irq_stacks();
irqchip_init(); irqchip_init();
if (!handle_arch_irq) if (!handle_arch_irq)
panic("No interrupt controller found."); panic("No interrupt controller found.");
......
...@@ -42,6 +42,7 @@ ...@@ -42,6 +42,7 @@
#include <asm/compat.h> #include <asm/compat.h>
#include <asm/debug-monitors.h> #include <asm/debug-monitors.h>
#include <asm/pgtable.h> #include <asm/pgtable.h>
#include <asm/stacktrace.h>
#include <asm/syscall.h> #include <asm/syscall.h>
#include <asm/traps.h> #include <asm/traps.h>
#include <asm/system_misc.h> #include <asm/system_misc.h>
......
...@@ -154,7 +154,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) ...@@ -154,7 +154,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
* page tables. * page tables.
*/ */
secondary_data.task = idle; secondary_data.task = idle;
secondary_data.stack = task_stack_page(idle) + THREAD_START_SP; secondary_data.stack = task_stack_page(idle) + THREAD_SIZE;
update_cpu_boot_status(CPU_MMU_OFF); update_cpu_boot_status(CPU_MMU_OFF);
__flush_dcache_area(&secondary_data, sizeof(secondary_data)); __flush_dcache_area(&secondary_data, sizeof(secondary_data));
......
...@@ -50,12 +50,7 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) ...@@ -50,12 +50,7 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame)
if (!tsk) if (!tsk)
tsk = current; tsk = current;
/* if (!on_accessible_stack(tsk, fp))
* Switching between stacks is valid when tracing current and in
* non-preemptible context.
*/
if (!(tsk == current && !preemptible() && on_irq_stack(fp)) &&
!on_task_stack(tsk, fp))
return -EINVAL; return -EINVAL;
frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp)); frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp));
......
...@@ -32,6 +32,7 @@ ...@@ -32,6 +32,7 @@
#include <linux/sched/signal.h> #include <linux/sched/signal.h>
#include <linux/sched/debug.h> #include <linux/sched/debug.h>
#include <linux/sched/task_stack.h> #include <linux/sched/task_stack.h>
#include <linux/sizes.h>
#include <linux/syscalls.h> #include <linux/syscalls.h>
#include <linux/mm_types.h> #include <linux/mm_types.h>
...@@ -41,6 +42,7 @@ ...@@ -41,6 +42,7 @@
#include <asm/esr.h> #include <asm/esr.h>
#include <asm/insn.h> #include <asm/insn.h>
#include <asm/traps.h> #include <asm/traps.h>
#include <asm/smp.h>
#include <asm/stack_pointer.h> #include <asm/stack_pointer.h>
#include <asm/stacktrace.h> #include <asm/stacktrace.h>
#include <asm/exception.h> #include <asm/exception.h>
...@@ -193,8 +195,7 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) ...@@ -193,8 +195,7 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk)
if (in_entry_text(frame.pc)) { if (in_entry_text(frame.pc)) {
stack = frame.fp - offsetof(struct pt_regs, stackframe); stack = frame.fp - offsetof(struct pt_regs, stackframe);
if (on_task_stack(tsk, stack) || if (on_accessible_stack(tsk, stack))
(tsk == current && !preemptible() && on_irq_stack(stack)))
dump_mem("", "Exception stack", stack, dump_mem("", "Exception stack", stack,
stack + sizeof(struct pt_regs)); stack + sizeof(struct pt_regs));
} }
...@@ -237,8 +238,6 @@ static int __die(const char *str, int err, struct pt_regs *regs) ...@@ -237,8 +238,6 @@ static int __die(const char *str, int err, struct pt_regs *regs)
end_of_stack(tsk)); end_of_stack(tsk));
if (!user_mode(regs)) { if (!user_mode(regs)) {
dump_mem(KERN_EMERG, "Stack: ", regs->sp,
THREAD_SIZE + (unsigned long)task_stack_page(tsk));
dump_backtrace(regs, tsk); dump_backtrace(regs, tsk);
dump_instr(KERN_EMERG, regs); dump_instr(KERN_EMERG, regs);
} }
...@@ -672,6 +671,43 @@ asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) ...@@ -672,6 +671,43 @@ asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr)
force_sig_info(info.si_signo, &info, current); force_sig_info(info.si_signo, &info, current);
} }
#ifdef CONFIG_VMAP_STACK
DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack)
__aligned(16);
asmlinkage void handle_bad_stack(struct pt_regs *regs)
{
unsigned long tsk_stk = (unsigned long)current->stack;
unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr);
unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
unsigned int esr = read_sysreg(esr_el1);
unsigned long far = read_sysreg(far_el1);
console_verbose();
pr_emerg("Insufficient stack space to handle exception!");
pr_emerg("ESR: 0x%08x -- %s\n", esr, esr_get_class_string(esr));
pr_emerg("FAR: 0x%016lx\n", far);
pr_emerg("Task stack: [0x%016lx..0x%016lx]\n",
tsk_stk, tsk_stk + THREAD_SIZE);
pr_emerg("IRQ stack: [0x%016lx..0x%016lx]\n",
irq_stk, irq_stk + THREAD_SIZE);
pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);
__show_regs(regs);
/*
* We use nmi_panic to limit the potential for recusive overflows, and
* to get a better stack trace.
*/
nmi_panic(NULL, "kernel stack overflow");
cpu_park_loop();
}
#endif
void __pte_error(const char *file, int line, unsigned long val) void __pte_error(const char *file, int line, unsigned long val)
{ {
pr_err("%s:%d: bad pte %016lx.\n", file, line, val); pr_err("%s:%d: bad pte %016lx.\n", file, line, val);
......
...@@ -72,22 +72,6 @@ PECOFF_FILE_ALIGNMENT = 0x200; ...@@ -72,22 +72,6 @@ PECOFF_FILE_ALIGNMENT = 0x200;
#define PECOFF_EDATA_PADDING #define PECOFF_EDATA_PADDING
#endif #endif
#if defined(CONFIG_DEBUG_ALIGN_RODATA)
/*
* 4 KB granule: 1 level 2 entry
* 16 KB granule: 128 level 3 entries, with contiguous bit
* 64 KB granule: 32 level 3 entries, with contiguous bit
*/
#define SEGMENT_ALIGN SZ_2M
#else
/*
* 4 KB granule: 16 level 3 entries, with contiguous bit
* 16 KB granule: 4 level 3 entries, without contiguous bit
* 64 KB granule: 1 level 3 entry
*/
#define SEGMENT_ALIGN SZ_64K
#endif
SECTIONS SECTIONS
{ {
/* /*
...@@ -192,7 +176,7 @@ SECTIONS ...@@ -192,7 +176,7 @@ SECTIONS
_data = .; _data = .;
_sdata = .; _sdata = .;
RW_DATA_SECTION(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE) RW_DATA_SECTION(L1_CACHE_BYTES, PAGE_SIZE, THREAD_ALIGN)
/* /*
* Data written with the MMU off but read with the MMU on requires * Data written with the MMU off but read with the MMU on requires
......
...@@ -11,6 +11,7 @@ ...@@ -11,6 +11,7 @@
*/ */
#include <linux/efi.h> #include <linux/efi.h>
#include <asm/efi.h> #include <asm/efi.h>
#include <asm/memory.h>
#include <asm/sections.h> #include <asm/sections.h>
#include <asm/sysreg.h> #include <asm/sysreg.h>
...@@ -81,9 +82,10 @@ efi_status_t handle_kernel_image(efi_system_table_t *sys_table_arg, ...@@ -81,9 +82,10 @@ efi_status_t handle_kernel_image(efi_system_table_t *sys_table_arg,
/* /*
* If CONFIG_DEBUG_ALIGN_RODATA is not set, produce a * If CONFIG_DEBUG_ALIGN_RODATA is not set, produce a
* displacement in the interval [0, MIN_KIMG_ALIGN) that * displacement in the interval [0, MIN_KIMG_ALIGN) that
* is a multiple of the minimal segment alignment (SZ_64K) * doesn't violate this kernel's de-facto alignment
* constraints.
*/ */
u32 mask = (MIN_KIMG_ALIGN - 1) & ~(SZ_64K - 1); u32 mask = (MIN_KIMG_ALIGN - 1) & ~(EFI_KIMG_ALIGN - 1);
u32 offset = !IS_ENABLED(CONFIG_DEBUG_ALIGN_RODATA) ? u32 offset = !IS_ENABLED(CONFIG_DEBUG_ALIGN_RODATA) ?
(phys_seed >> 32) & mask : TEXT_OFFSET; (phys_seed >> 32) & mask : TEXT_OFFSET;
......
...@@ -38,6 +38,10 @@ enum { ...@@ -38,6 +38,10 @@ enum {
#ifdef __KERNEL__ #ifdef __KERNEL__
#ifndef THREAD_ALIGN
#define THREAD_ALIGN THREAD_SIZE
#endif
#ifdef CONFIG_DEBUG_STACK_USAGE #ifdef CONFIG_DEBUG_STACK_USAGE
# define THREADINFO_GFP (GFP_KERNEL_ACCOUNT | __GFP_NOTRACK | \ # define THREADINFO_GFP (GFP_KERNEL_ACCOUNT | __GFP_NOTRACK | \
__GFP_ZERO) __GFP_ZERO)
......
...@@ -88,6 +88,7 @@ ...@@ -88,6 +88,7 @@
#include <linux/sysctl.h> #include <linux/sysctl.h>
#include <linux/kcov.h> #include <linux/kcov.h>
#include <linux/livepatch.h> #include <linux/livepatch.h>
#include <linux/thread_info.h>
#include <asm/pgtable.h> #include <asm/pgtable.h>
#include <asm/pgalloc.h> #include <asm/pgalloc.h>
...@@ -217,7 +218,7 @@ static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node) ...@@ -217,7 +218,7 @@ static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
return s->addr; return s->addr;
} }
stack = __vmalloc_node_range(THREAD_SIZE, THREAD_SIZE, stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN,
VMALLOC_START, VMALLOC_END, VMALLOC_START, VMALLOC_END,
THREADINFO_GFP, THREADINFO_GFP,
PAGE_KERNEL, PAGE_KERNEL,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment