Commit e71a4e1b authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Will Deacon

arm64: ftrace: add support for far branches to dynamic ftrace

Currently, dynamic ftrace support in the arm64 kernel assumes that all
core kernel code is within range of ordinary branch instructions that
occur in module code, which is usually the case, but is no longer
guaranteed now that we have support for module PLTs and address space
randomization.

Since on arm64, all patching of branch instructions involves function
calls to the same entry point [ftrace_caller()], we can emit the modules
with a trampoline that has unlimited range, and patch both the trampoline
itself and the branch instruction to redirect the call via the trampoline.
Signed-off-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
[will: minor clarification to smp_wmb() comment]
Signed-off-by: default avatarWill Deacon <will.deacon@arm.com>
parent f8af0b36
...@@ -982,7 +982,7 @@ config RANDOMIZE_BASE ...@@ -982,7 +982,7 @@ config RANDOMIZE_BASE
config RANDOMIZE_MODULE_REGION_FULL config RANDOMIZE_MODULE_REGION_FULL
bool "Randomize the module region independently from the core kernel" bool "Randomize the module region independently from the core kernel"
depends on RANDOMIZE_BASE && !DYNAMIC_FTRACE depends on RANDOMIZE_BASE
default y default y
help help
Randomizes the location of the module region without considering the Randomizes the location of the module region without considering the
......
...@@ -70,6 +70,9 @@ endif ...@@ -70,6 +70,9 @@ endif
ifeq ($(CONFIG_ARM64_MODULE_PLTS),y) ifeq ($(CONFIG_ARM64_MODULE_PLTS),y)
KBUILD_LDFLAGS_MODULE += -T $(srctree)/arch/arm64/kernel/module.lds KBUILD_LDFLAGS_MODULE += -T $(srctree)/arch/arm64/kernel/module.lds
ifeq ($(CONFIG_DYNAMIC_FTRACE),y)
KBUILD_LDFLAGS_MODULE += $(objtree)/arch/arm64/kernel/ftrace-mod.o
endif
endif endif
# Default value # Default value
......
...@@ -30,6 +30,9 @@ struct mod_plt_sec { ...@@ -30,6 +30,9 @@ struct mod_plt_sec {
struct mod_arch_specific { struct mod_arch_specific {
struct mod_plt_sec core; struct mod_plt_sec core;
struct mod_plt_sec init; struct mod_plt_sec init;
/* for CONFIG_DYNAMIC_FTRACE */
void *ftrace_trampoline;
}; };
#endif #endif
......
...@@ -62,3 +62,6 @@ extra-y += $(head-y) vmlinux.lds ...@@ -62,3 +62,6 @@ extra-y += $(head-y) vmlinux.lds
ifeq ($(CONFIG_DEBUG_EFI),y) ifeq ($(CONFIG_DEBUG_EFI),y)
AFLAGS_head.o += -DVMLINUX_PATH="\"$(realpath $(objtree)/vmlinux)\"" AFLAGS_head.o += -DVMLINUX_PATH="\"$(realpath $(objtree)/vmlinux)\""
endif endif
# will be included by each individual module but not by the core kernel itself
extra-$(CONFIG_DYNAMIC_FTRACE) += ftrace-mod.o
/*
* Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
.section ".text.ftrace_trampoline", "ax"
.align 3
0: .quad 0
__ftrace_trampoline:
ldr x16, 0b
br x16
ENDPROC(__ftrace_trampoline)
...@@ -10,10 +10,12 @@ ...@@ -10,10 +10,12 @@
*/ */
#include <linux/ftrace.h> #include <linux/ftrace.h>
#include <linux/module.h>
#include <linux/swab.h> #include <linux/swab.h>
#include <linux/uaccess.h> #include <linux/uaccess.h>
#include <asm/cacheflush.h> #include <asm/cacheflush.h>
#include <asm/debug-monitors.h>
#include <asm/ftrace.h> #include <asm/ftrace.h>
#include <asm/insn.h> #include <asm/insn.h>
...@@ -69,8 +71,57 @@ int ftrace_update_ftrace_func(ftrace_func_t func) ...@@ -69,8 +71,57 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{ {
unsigned long pc = rec->ip; unsigned long pc = rec->ip;
long offset = (long)pc - (long)addr;
u32 old, new; u32 old, new;
if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) &&
(offset < -SZ_128M || offset >= SZ_128M)) {
unsigned long *trampoline;
struct module *mod;
/*
* On kernels that support module PLTs, the offset between the
* branch instruction and its target may legally exceed the
* range of an ordinary relative 'bl' opcode. In this case, we
* need to branch via a trampoline in the module.
*
* NOTE: __module_text_address() must be called with preemption
* disabled, but we can rely on ftrace_lock to ensure that 'mod'
* retains its validity throughout the remainder of this code.
*/
preempt_disable();
mod = __module_text_address(pc);
preempt_enable();
if (WARN_ON(!mod))
return -EINVAL;
/*
* There is only one ftrace trampoline per module. For now,
* this is not a problem since on arm64, all dynamic ftrace
* invocations are routed via ftrace_caller(). This will need
* to be revisited if support for multiple ftrace entry points
* is added in the future, but for now, the pr_err() below
* deals with a theoretical issue only.
*/
trampoline = (unsigned long *)mod->arch.ftrace_trampoline;
if (trampoline[0] != addr) {
if (trampoline[0] != 0) {
pr_err("ftrace: far branches to multiple entry points unsupported inside a single module\n");
return -EINVAL;
}
/* point the trampoline to our ftrace entry point */
module_disable_ro(mod);
trampoline[0] = addr;
module_enable_ro(mod, true);
/* update trampoline before patching in the branch */
smp_wmb();
}
addr = (unsigned long)&trampoline[1];
}
old = aarch64_insn_gen_nop(); old = aarch64_insn_gen_nop();
new = aarch64_insn_gen_branch_imm(pc, addr, AARCH64_INSN_BRANCH_LINK); new = aarch64_insn_gen_branch_imm(pc, addr, AARCH64_INSN_BRANCH_LINK);
......
...@@ -420,8 +420,12 @@ int module_finalize(const Elf_Ehdr *hdr, ...@@ -420,8 +420,12 @@ int module_finalize(const Elf_Ehdr *hdr,
for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) { for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) {
if (strcmp(".altinstructions", secstrs + s->sh_name) == 0) { if (strcmp(".altinstructions", secstrs + s->sh_name) == 0) {
apply_alternatives((void *)s->sh_addr, s->sh_size); apply_alternatives((void *)s->sh_addr, s->sh_size);
return 0;
} }
#ifdef CONFIG_ARM64_MODULE_PLTS
if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE) &&
!strcmp(".text.ftrace_trampoline", secstrs + s->sh_name))
me->arch.ftrace_trampoline = (void *)s->sh_addr;
#endif
} }
return 0; return 0;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment