Commit f48cb8b4 authored by Steven Rostedt's avatar Steven Rostedt

powerpc/ppc64: ftrace, handle module trampolines for dyn ftrace

Impact: Allow 64 bit PowerPC to trace modules with dynamic ftrace

This adds code to handle the PPC64 module trampolines, and allows for
PPC64 to use dynamic ftrace.

Thanks to Paul Mackerras for these updates:

  - fix the mod and rec->arch.mod NULL checks.
  - fix to is_bl_op compare.

Thanks to Milton Miller for:

  - finding the nasty race with using two nops, and recommending
    instead that I use a branch 8 forward.
Signed-off-by: default avatarSteven Rostedt <srostedt@redhat.com>
parent e4486fe3
...@@ -16,7 +16,7 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) ...@@ -16,7 +16,7 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr)
} }
struct dyn_arch_ftrace { struct dyn_arch_ftrace {
/* nothing yet */ struct module *mod;
}; };
#endif /* CONFIG_DYNAMIC_FTRACE */ #endif /* CONFIG_DYNAMIC_FTRACE */
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
......
...@@ -34,6 +34,11 @@ struct mod_arch_specific { ...@@ -34,6 +34,11 @@ struct mod_arch_specific {
#ifdef __powerpc64__ #ifdef __powerpc64__
unsigned int stubs_section; /* Index of stubs section in module */ unsigned int stubs_section; /* Index of stubs section in module */
unsigned int toc_section; /* What section is the TOC? */ unsigned int toc_section; /* What section is the TOC? */
#ifdef CONFIG_DYNAMIC_FTRACE
unsigned long toc;
unsigned long tramp;
#endif
#else #else
/* Indices of PLT sections within module. */ /* Indices of PLT sections within module. */
unsigned int core_plt_section; unsigned int core_plt_section;
...@@ -68,6 +73,12 @@ struct mod_arch_specific { ...@@ -68,6 +73,12 @@ struct mod_arch_specific {
# endif /* MODULE */ # endif /* MODULE */
#endif #endif
#ifdef CONFIG_DYNAMIC_FTRACE
# ifdef MODULE
asm(".section .ftrace.tramp,\"ax\",@nobits; .align 3; .previous");
# endif /* MODULE */
#endif
struct exception_table_entry; struct exception_table_entry;
void sort_ex_table(struct exception_table_entry *start, void sort_ex_table(struct exception_table_entry *start,
......
...@@ -10,22 +10,29 @@ ...@@ -10,22 +10,29 @@
#include <linux/spinlock.h> #include <linux/spinlock.h>
#include <linux/hardirq.h> #include <linux/hardirq.h>
#include <linux/uaccess.h> #include <linux/uaccess.h>
#include <linux/module.h>
#include <linux/ftrace.h> #include <linux/ftrace.h>
#include <linux/percpu.h> #include <linux/percpu.h>
#include <linux/init.h> #include <linux/init.h>
#include <linux/list.h> #include <linux/list.h>
#include <asm/cacheflush.h> #include <asm/cacheflush.h>
#include <asm/code-patching.h>
#include <asm/ftrace.h> #include <asm/ftrace.h>
#if 0
#define DEBUGP printk
#else
#define DEBUGP(fmt , ...) do { } while (0)
#endif
static unsigned int ftrace_nop = 0x60000000; static unsigned int ftrace_nop = PPC_NOP_INSTR;
#ifdef CONFIG_PPC32 #ifdef CONFIG_PPC32
# define GET_ADDR(addr) addr # define GET_ADDR(addr) addr
#else #else
/* PowerPC64's functions are data that points to the functions */ /* PowerPC64's functions are data that points to the functions */
# define GET_ADDR(addr) *(unsigned long *)addr # define GET_ADDR(addr) (*(unsigned long *)addr)
#endif #endif
...@@ -102,6 +109,9 @@ ftrace_modify_code(unsigned long ip, unsigned char *old_code, ...@@ -102,6 +109,9 @@ ftrace_modify_code(unsigned long ip, unsigned char *old_code,
return 0; return 0;
} }
/*
* Helper functions that are the same for both PPC64 and PPC32.
*/
static int test_24bit_addr(unsigned long ip, unsigned long addr) static int test_24bit_addr(unsigned long ip, unsigned long addr)
{ {
long diff; long diff;
...@@ -119,43 +129,292 @@ static int test_24bit_addr(unsigned long ip, unsigned long addr) ...@@ -119,43 +129,292 @@ static int test_24bit_addr(unsigned long ip, unsigned long addr)
return (diff < (1 << 25)) && (diff > (-1 << 26)); return (diff < (1 << 25)) && (diff > (-1 << 26));
} }
static int is_bl_op(unsigned int op)
{
return (op & 0xfc000003) == 0x48000001;
}
static int test_offset(unsigned long offset)
{
return (offset + 0x2000000 > 0x3ffffff) || ((offset & 3) != 0);
}
static unsigned long find_bl_target(unsigned long ip, unsigned int op)
{
static int offset;
offset = (op & 0x03fffffc);
/* make it signed */
if (offset & 0x02000000)
offset |= 0xfe000000;
return ip + (long)offset;
}
static unsigned int branch_offset(unsigned long offset)
{
/* return "bl ip+offset" */
return 0x48000001 | (offset & 0x03fffffc);
}
#ifdef CONFIG_PPC64
static int
__ftrace_make_nop(struct module *mod,
struct dyn_ftrace *rec, unsigned long addr)
{
unsigned char replaced[MCOUNT_INSN_SIZE * 2];
unsigned int *op = (unsigned *)&replaced;
unsigned char jmp[8];
unsigned long *ptr = (unsigned long *)&jmp;
unsigned long ip = rec->ip;
unsigned long tramp;
int offset;
/* read where this goes */
if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
return -EFAULT;
/* Make sure that that this is still a 24bit jump */
if (!is_bl_op(*op)) {
printk(KERN_ERR "Not expected bl: opcode is %x\n", *op);
return -EINVAL;
}
/* lets find where the pointer goes */
tramp = find_bl_target(ip, *op);
/*
* On PPC64 the trampoline looks like:
* 0x3d, 0x82, 0x00, 0x00, addis r12,r2, <high>
* 0x39, 0x8c, 0x00, 0x00, addi r12,r12, <low>
* Where the bytes 2,3,6 and 7 make up the 32bit offset
* to the TOC that holds the pointer.
* to jump to.
* 0xf8, 0x41, 0x00, 0x28, std r2,40(r1)
* 0xe9, 0x6c, 0x00, 0x20, ld r11,32(r12)
* The actually address is 32 bytes from the offset
* into the TOC.
* 0xe8, 0x4c, 0x00, 0x28, ld r2,40(r12)
*/
DEBUGP("ip:%lx jumps to %lx r2: %lx", ip, tramp, mod->arch.toc);
/* Find where the trampoline jumps to */
if (probe_kernel_read(jmp, (void *)tramp, 8)) {
printk(KERN_ERR "Failed to read %lx\n", tramp);
return -EFAULT;
}
DEBUGP(" %08x %08x",
(unsigned)(*ptr >> 32),
(unsigned)*ptr);
offset = (unsigned)jmp[2] << 24 |
(unsigned)jmp[3] << 16 |
(unsigned)jmp[6] << 8 |
(unsigned)jmp[7];
DEBUGP(" %x ", offset);
/* get the address this jumps too */
tramp = mod->arch.toc + offset + 32;
DEBUGP("toc: %lx", tramp);
if (probe_kernel_read(jmp, (void *)tramp, 8)) {
printk(KERN_ERR "Failed to read %lx\n", tramp);
return -EFAULT;
}
DEBUGP(" %08x %08x\n",
(unsigned)(*ptr >> 32),
(unsigned)*ptr);
/* This should match what was called */
if (*ptr != GET_ADDR(addr)) {
printk(KERN_ERR "addr does not match %lx\n", *ptr);
return -EINVAL;
}
/*
* We want to nop the line, but the next line is
* 0xe8, 0x41, 0x00, 0x28 ld r2,40(r1)
* This needs to be turned to a nop too.
*/
if (probe_kernel_read(replaced, (void *)(ip+4), MCOUNT_INSN_SIZE))
return -EFAULT;
if (*op != 0xe8410028) {
printk(KERN_ERR "Next line is not ld! (%08x)\n", *op);
return -EINVAL;
}
/*
* Milton Miller pointed out that we can not blindly do nops.
* If a task was preempted when calling a trace function,
* the nops will remove the way to restore the TOC in r2
* and the r2 TOC will get corrupted.
*/
/*
* Replace:
* bl <tramp> <==== will be replaced with "b 1f"
* ld r2,40(r1)
* 1:
*/
op[0] = 0x48000008; /* b +8 */
if (probe_kernel_write((void *)ip, replaced, MCOUNT_INSN_SIZE))
return -EPERM;
return 0;
}
#else /* !PPC64 */
static int
__ftrace_make_nop(struct module *mod,
struct dyn_ftrace *rec, unsigned long addr)
{
/* Ignore modules for PPC32 (for now) */
return 0;
}
#endif /* PPC64 */
int ftrace_make_nop(struct module *mod, int ftrace_make_nop(struct module *mod,
struct dyn_ftrace *rec, unsigned long addr) struct dyn_ftrace *rec, unsigned long addr)
{ {
unsigned char *old, *new; unsigned char *old, *new;
unsigned long ip = rec->ip;
/* /*
* If the calling address is more that 24 bits away, * If the calling address is more that 24 bits away,
* then we had to use a trampoline to make the call. * then we had to use a trampoline to make the call.
* Otherwise just update the call site. * Otherwise just update the call site.
*/ */
if (test_24bit_addr(rec->ip, addr)) { if (test_24bit_addr(ip, addr)) {
/* within range */ /* within range */
old = ftrace_call_replace(rec->ip, addr); old = ftrace_call_replace(ip, addr);
new = ftrace_nop_replace(); new = ftrace_nop_replace();
return ftrace_modify_code(rec->ip, old, new); return ftrace_modify_code(ip, old, new);
}
#ifdef CONFIG_PPC64
/*
* Out of range jumps are called from modules.
* We should either already have a pointer to the module
* or it has been passed in.
*/
if (!rec->arch.mod) {
if (!mod) {
printk(KERN_ERR "No module loaded addr=%lx\n",
addr);
return -EFAULT;
}
rec->arch.mod = mod;
} else if (mod) {
if (mod != rec->arch.mod) {
printk(KERN_ERR
"Record mod %p not equal to passed in mod %p\n",
rec->arch.mod, mod);
return -EINVAL;
}
/* nothing to do if mod == rec->arch.mod */
} else
mod = rec->arch.mod;
#endif /* CONFIG_PPC64 */
return __ftrace_make_nop(mod, rec, addr);
}
#ifdef CONFIG_PPC64
static int
__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{
unsigned char replaced[MCOUNT_INSN_SIZE * 2];
unsigned int *op = (unsigned *)&replaced;
unsigned long ip = rec->ip;
unsigned long offset;
/* read where this goes */
if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE * 2))
return -EFAULT;
/*
* It should be pointing to two nops or
* b +8; ld r2,40(r1)
*/
if (((op[0] != 0x48000008) || (op[1] != 0xe8410028)) &&
((op[0] != PPC_NOP_INSTR) || (op[1] != PPC_NOP_INSTR))) {
printk(KERN_ERR "Expected NOPs but have %x %x\n", op[0], op[1]);
return -EINVAL;
}
/* If we never set up a trampoline to ftrace_caller, then bail */
if (!rec->arch.mod->arch.tramp) {
printk(KERN_ERR "No ftrace trampoline\n");
return -EINVAL;
}
/* now calculate a jump to the ftrace caller trampoline */
offset = rec->arch.mod->arch.tramp - ip;
if (test_offset(offset)) {
printk(KERN_ERR "REL24 %li out of range!\n",
(long int)offset);
return -EINVAL;
} }
/* Set to "bl addr" */
op[0] = branch_offset(offset);
/* ld r2,40(r1) */
op[1] = 0xe8410028;
DEBUGP("write to %lx\n", rec->ip);
if (probe_kernel_write((void *)ip, replaced, MCOUNT_INSN_SIZE * 2))
return -EPERM;
return 0; return 0;
} }
#else
static int
__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{
/* PPC32 ignores modules for now */
return 0;
}
#endif /* CONFIG_PPC64 */
int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{ {
unsigned char *old, *new; unsigned char *old, *new;
unsigned long ip = rec->ip;
/* /*
* If the calling address is more that 24 bits away, * If the calling address is more that 24 bits away,
* then we had to use a trampoline to make the call. * then we had to use a trampoline to make the call.
* Otherwise just update the call site. * Otherwise just update the call site.
*/ */
if (test_24bit_addr(rec->ip, addr)) { if (test_24bit_addr(ip, addr)) {
/* within range */ /* within range */
old = ftrace_nop_replace(); old = ftrace_nop_replace();
new = ftrace_call_replace(rec->ip, addr); new = ftrace_call_replace(ip, addr);
return ftrace_modify_code(rec->ip, old, new); return ftrace_modify_code(ip, old, new);
} }
return 0; #ifdef CONFIG_PPC64
/*
* Out of range jumps are called from modules.
* Being that we are converting from nop, it had better
* already have a module defined.
*/
if (!rec->arch.mod) {
printk(KERN_ERR "No module loaded\n");
return -EINVAL;
}
#endif
return __ftrace_make_call(rec, addr);
} }
int ftrace_update_ftrace_func(ftrace_func_t func) int ftrace_update_ftrace_func(ftrace_func_t func)
...@@ -180,4 +439,3 @@ int __init ftrace_dyn_arch_init(void *data) ...@@ -180,4 +439,3 @@ int __init ftrace_dyn_arch_init(void *data)
return 0; return 0;
} }
...@@ -20,6 +20,7 @@ ...@@ -20,6 +20,7 @@
#include <linux/moduleloader.h> #include <linux/moduleloader.h>
#include <linux/err.h> #include <linux/err.h>
#include <linux/vmalloc.h> #include <linux/vmalloc.h>
#include <linux/ftrace.h>
#include <linux/bug.h> #include <linux/bug.h>
#include <asm/module.h> #include <asm/module.h>
#include <asm/firmware.h> #include <asm/firmware.h>
...@@ -163,6 +164,11 @@ static unsigned long get_stubs_size(const Elf64_Ehdr *hdr, ...@@ -163,6 +164,11 @@ static unsigned long get_stubs_size(const Elf64_Ehdr *hdr,
} }
} }
#ifdef CONFIG_DYNAMIC_FTRACE
/* make the trampoline to the ftrace_caller */
relocs++;
#endif
DEBUGP("Looks like a total of %lu stubs, max\n", relocs); DEBUGP("Looks like a total of %lu stubs, max\n", relocs);
return relocs * sizeof(struct ppc64_stub_entry); return relocs * sizeof(struct ppc64_stub_entry);
} }
...@@ -441,5 +447,12 @@ int apply_relocate_add(Elf64_Shdr *sechdrs, ...@@ -441,5 +447,12 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
} }
} }
#ifdef CONFIG_DYNAMIC_FTRACE
me->arch.toc = my_r2(sechdrs, me);
me->arch.tramp = stub_for_addr(sechdrs,
(unsigned long)ftrace_caller,
me);
#endif
return 0; return 0;
} }
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment