Commit 675eef66 authored by Ingo Molnar's avatar Ingo Molnar

Merge branch 'tip/perf/core' of...

Merge branch 'tip/perf/core' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace into perf/core
parents 03f70388 96de37b6
...@@ -2437,6 +2437,14 @@ bytes respectively. Such letter suffixes can also be entirely omitted. ...@@ -2437,6 +2437,14 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
stacktrace [FTRACE] stacktrace [FTRACE]
Enabled the stack tracer on boot up. Enabled the stack tracer on boot up.
stacktrace_filter=[function-list]
[FTRACE] Limit the functions that the stack tracer
will trace at boot up. function-list is a comma separated
list of functions. This list can be changed at run
time by the stack_trace_filter file in the debugfs
tracing directory. Note, this enables stack tracing
and the stacktrace above is not needed.
sti= [PARISC,HW] sti= [PARISC,HW]
Format: <num> Format: <num>
Set the STI (builtin display/keyboard on the HP-PARISC Set the STI (builtin display/keyboard on the HP-PARISC
......
...@@ -50,6 +50,11 @@ ...@@ -50,6 +50,11 @@
# define inline inline __attribute__((always_inline)) # define inline inline __attribute__((always_inline))
# define __inline__ __inline__ __attribute__((always_inline)) # define __inline__ __inline__ __attribute__((always_inline))
# define __inline __inline __attribute__((always_inline)) # define __inline __inline __attribute__((always_inline))
#else
/* A lot of inline functions can cause havoc with function tracing */
# define inline inline notrace
# define __inline__ __inline__ notrace
# define __inline __inline notrace
#endif #endif
#define __deprecated __attribute__((deprecated)) #define __deprecated __attribute__((deprecated))
......
...@@ -133,6 +133,8 @@ struct ftrace_func_command { ...@@ -133,6 +133,8 @@ struct ftrace_func_command {
int ftrace_arch_code_modify_prepare(void); int ftrace_arch_code_modify_prepare(void);
int ftrace_arch_code_modify_post_process(void); int ftrace_arch_code_modify_post_process(void);
void ftrace_bug(int err, unsigned long ip);
struct seq_file; struct seq_file;
struct ftrace_probe_ops { struct ftrace_probe_ops {
...@@ -161,7 +163,6 @@ extern int ftrace_text_reserved(void *start, void *end); ...@@ -161,7 +163,6 @@ extern int ftrace_text_reserved(void *start, void *end);
enum { enum {
FTRACE_FL_ENABLED = (1 << 30), FTRACE_FL_ENABLED = (1 << 30),
FTRACE_FL_FREE = (1 << 31),
}; };
#define FTRACE_FL_MASK (0x3UL << 30) #define FTRACE_FL_MASK (0x3UL << 30)
...@@ -172,10 +173,7 @@ struct dyn_ftrace { ...@@ -172,10 +173,7 @@ struct dyn_ftrace {
unsigned long ip; /* address of mcount call-site */ unsigned long ip; /* address of mcount call-site */
struct dyn_ftrace *freelist; struct dyn_ftrace *freelist;
}; };
union { unsigned long flags;
unsigned long flags;
struct dyn_ftrace *newlist;
};
struct dyn_arch_ftrace arch; struct dyn_arch_ftrace arch;
}; };
...@@ -190,6 +188,56 @@ void ftrace_set_global_notrace(unsigned char *buf, int len, int reset); ...@@ -190,6 +188,56 @@ void ftrace_set_global_notrace(unsigned char *buf, int len, int reset);
int register_ftrace_command(struct ftrace_func_command *cmd); int register_ftrace_command(struct ftrace_func_command *cmd);
int unregister_ftrace_command(struct ftrace_func_command *cmd); int unregister_ftrace_command(struct ftrace_func_command *cmd);
enum {
FTRACE_UPDATE_CALLS = (1 << 0),
FTRACE_DISABLE_CALLS = (1 << 1),
FTRACE_UPDATE_TRACE_FUNC = (1 << 2),
FTRACE_START_FUNC_RET = (1 << 3),
FTRACE_STOP_FUNC_RET = (1 << 4),
};
enum {
FTRACE_UPDATE_IGNORE,
FTRACE_UPDATE_MAKE_CALL,
FTRACE_UPDATE_MAKE_NOP,
};
enum {
FTRACE_ITER_FILTER = (1 << 0),
FTRACE_ITER_NOTRACE = (1 << 1),
FTRACE_ITER_PRINTALL = (1 << 2),
FTRACE_ITER_DO_HASH = (1 << 3),
FTRACE_ITER_HASH = (1 << 4),
FTRACE_ITER_ENABLED = (1 << 5),
};
void arch_ftrace_update_code(int command);
struct ftrace_rec_iter;
struct ftrace_rec_iter *ftrace_rec_iter_start(void);
struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter);
struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter);
int ftrace_update_record(struct dyn_ftrace *rec, int enable);
int ftrace_test_record(struct dyn_ftrace *rec, int enable);
void ftrace_run_stop_machine(int command);
int ftrace_location(unsigned long ip);
extern ftrace_func_t ftrace_trace_function;
int ftrace_regex_open(struct ftrace_ops *ops, int flag,
struct inode *inode, struct file *file);
ssize_t ftrace_filter_write(struct file *file, const char __user *ubuf,
size_t cnt, loff_t *ppos);
ssize_t ftrace_notrace_write(struct file *file, const char __user *ubuf,
size_t cnt, loff_t *ppos);
loff_t ftrace_regex_lseek(struct file *file, loff_t offset, int origin);
int ftrace_regex_release(struct inode *inode, struct file *file);
void __init
ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable);
/* defined in arch */ /* defined in arch */
extern int ftrace_ip_converted(unsigned long ip); extern int ftrace_ip_converted(unsigned long ip);
extern int ftrace_dyn_arch_init(void *data); extern int ftrace_dyn_arch_init(void *data);
...@@ -284,6 +332,25 @@ static inline int ftrace_text_reserved(void *start, void *end) ...@@ -284,6 +332,25 @@ static inline int ftrace_text_reserved(void *start, void *end)
{ {
return 0; return 0;
} }
/*
* Again users of functions that have ftrace_ops may not
* have them defined when ftrace is not enabled, but these
* functions may still be called. Use a macro instead of inline.
*/
#define ftrace_regex_open(ops, flag, inod, file) ({ -ENODEV; })
#define ftrace_set_early_filter(ops, buf, enable) do { } while (0)
static inline ssize_t ftrace_filter_write(struct file *file, const char __user *ubuf,
size_t cnt, loff_t *ppos) { return -ENODEV; }
static inline ssize_t ftrace_notrace_write(struct file *file, const char __user *ubuf,
size_t cnt, loff_t *ppos) { return -ENODEV; }
static inline loff_t ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
{
return -ENODEV;
}
static inline int
ftrace_regex_release(struct inode *inode, struct file *file) { return -ENODEV; }
#endif /* CONFIG_DYNAMIC_FTRACE */ #endif /* CONFIG_DYNAMIC_FTRACE */
/* totally disable ftrace - can not re-enable after this */ /* totally disable ftrace - can not re-enable after this */
......
...@@ -22,11 +22,13 @@ ...@@ -22,11 +22,13 @@
#include <linux/hardirq.h> #include <linux/hardirq.h>
#include <linux/kthread.h> #include <linux/kthread.h>
#include <linux/uaccess.h> #include <linux/uaccess.h>
#include <linux/bsearch.h>
#include <linux/module.h> #include <linux/module.h>
#include <linux/ftrace.h> #include <linux/ftrace.h>
#include <linux/sysctl.h> #include <linux/sysctl.h>
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/ctype.h> #include <linux/ctype.h>
#include <linux/sort.h>
#include <linux/list.h> #include <linux/list.h>
#include <linux/hash.h> #include <linux/hash.h>
#include <linux/rcupdate.h> #include <linux/rcupdate.h>
...@@ -947,13 +949,6 @@ struct ftrace_func_probe { ...@@ -947,13 +949,6 @@ struct ftrace_func_probe {
struct rcu_head rcu; struct rcu_head rcu;
}; };
enum {
FTRACE_ENABLE_CALLS = (1 << 0),
FTRACE_DISABLE_CALLS = (1 << 1),
FTRACE_UPDATE_TRACE_FUNC = (1 << 2),
FTRACE_START_FUNC_RET = (1 << 3),
FTRACE_STOP_FUNC_RET = (1 << 4),
};
struct ftrace_func_entry { struct ftrace_func_entry {
struct hlist_node hlist; struct hlist_node hlist;
unsigned long ip; unsigned long ip;
...@@ -984,18 +979,19 @@ static struct ftrace_ops global_ops = { ...@@ -984,18 +979,19 @@ static struct ftrace_ops global_ops = {
.filter_hash = EMPTY_HASH, .filter_hash = EMPTY_HASH,
}; };
static struct dyn_ftrace *ftrace_new_addrs;
static DEFINE_MUTEX(ftrace_regex_lock); static DEFINE_MUTEX(ftrace_regex_lock);
struct ftrace_page { struct ftrace_page {
struct ftrace_page *next; struct ftrace_page *next;
struct dyn_ftrace *records;
int index; int index;
struct dyn_ftrace records[]; int size;
}; };
#define ENTRIES_PER_PAGE \ static struct ftrace_page *ftrace_new_pgs;
((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
#define ENTRY_SIZE sizeof(struct dyn_ftrace)
#define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
/* estimate from running different kernels */ /* estimate from running different kernels */
#define NR_TO_INIT 10000 #define NR_TO_INIT 10000
...@@ -1003,7 +999,10 @@ struct ftrace_page { ...@@ -1003,7 +999,10 @@ struct ftrace_page {
static struct ftrace_page *ftrace_pages_start; static struct ftrace_page *ftrace_pages_start;
static struct ftrace_page *ftrace_pages; static struct ftrace_page *ftrace_pages;
static struct dyn_ftrace *ftrace_free_records; static bool ftrace_hash_empty(struct ftrace_hash *hash)
{
return !hash || !hash->count;
}
static struct ftrace_func_entry * static struct ftrace_func_entry *
ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip) ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
...@@ -1013,7 +1012,7 @@ ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip) ...@@ -1013,7 +1012,7 @@ ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
struct hlist_head *hhd; struct hlist_head *hhd;
struct hlist_node *n; struct hlist_node *n;
if (!hash->count) if (ftrace_hash_empty(hash))
return NULL; return NULL;
if (hash->size_bits > 0) if (hash->size_bits > 0)
...@@ -1157,7 +1156,7 @@ alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash) ...@@ -1157,7 +1156,7 @@ alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
return NULL; return NULL;
/* Empty hash? */ /* Empty hash? */
if (!hash || !hash->count) if (ftrace_hash_empty(hash))
return new_hash; return new_hash;
size = 1 << hash->size_bits; size = 1 << hash->size_bits;
...@@ -1282,9 +1281,9 @@ ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip) ...@@ -1282,9 +1281,9 @@ ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
filter_hash = rcu_dereference_raw(ops->filter_hash); filter_hash = rcu_dereference_raw(ops->filter_hash);
notrace_hash = rcu_dereference_raw(ops->notrace_hash); notrace_hash = rcu_dereference_raw(ops->notrace_hash);
if ((!filter_hash || !filter_hash->count || if ((ftrace_hash_empty(filter_hash) ||
ftrace_lookup_ip(filter_hash, ip)) && ftrace_lookup_ip(filter_hash, ip)) &&
(!notrace_hash || !notrace_hash->count || (ftrace_hash_empty(notrace_hash) ||
!ftrace_lookup_ip(notrace_hash, ip))) !ftrace_lookup_ip(notrace_hash, ip)))
ret = 1; ret = 1;
else else
...@@ -1307,6 +1306,47 @@ ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip) ...@@ -1307,6 +1306,47 @@ ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
} \ } \
} }
static int ftrace_cmp_recs(const void *a, const void *b)
{
const struct dyn_ftrace *reca = a;
const struct dyn_ftrace *recb = b;
if (reca->ip > recb->ip)
return 1;
if (reca->ip < recb->ip)
return -1;
return 0;
}
/**
* ftrace_location - return true if the ip giving is a traced location
* @ip: the instruction pointer to check
*
* Returns 1 if @ip given is a pointer to a ftrace location.
* That is, the instruction that is either a NOP or call to
* the function tracer. It checks the ftrace internal tables to
* determine if the address belongs or not.
*/
int ftrace_location(unsigned long ip)
{
struct ftrace_page *pg;
struct dyn_ftrace *rec;
struct dyn_ftrace key;
key.ip = ip;
for (pg = ftrace_pages_start; pg; pg = pg->next) {
rec = bsearch(&key, pg->records, pg->index,
sizeof(struct dyn_ftrace),
ftrace_cmp_recs);
if (rec)
return 1;
}
return 0;
}
static void __ftrace_hash_rec_update(struct ftrace_ops *ops, static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
int filter_hash, int filter_hash,
bool inc) bool inc)
...@@ -1336,7 +1376,7 @@ static void __ftrace_hash_rec_update(struct ftrace_ops *ops, ...@@ -1336,7 +1376,7 @@ static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
if (filter_hash) { if (filter_hash) {
hash = ops->filter_hash; hash = ops->filter_hash;
other_hash = ops->notrace_hash; other_hash = ops->notrace_hash;
if (!hash || !hash->count) if (ftrace_hash_empty(hash))
all = 1; all = 1;
} else { } else {
inc = !inc; inc = !inc;
...@@ -1346,7 +1386,7 @@ static void __ftrace_hash_rec_update(struct ftrace_ops *ops, ...@@ -1346,7 +1386,7 @@ static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
* If the notrace hash has no items, * If the notrace hash has no items,
* then there's nothing to do. * then there's nothing to do.
*/ */
if (hash && !hash->count) if (ftrace_hash_empty(hash))
return; return;
} }
...@@ -1363,8 +1403,8 @@ static void __ftrace_hash_rec_update(struct ftrace_ops *ops, ...@@ -1363,8 +1403,8 @@ static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip)) if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
match = 1; match = 1;
} else { } else {
in_hash = hash && !!ftrace_lookup_ip(hash, rec->ip); in_hash = !!ftrace_lookup_ip(hash, rec->ip);
in_other_hash = other_hash && !!ftrace_lookup_ip(other_hash, rec->ip); in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
/* /*
* *
...@@ -1372,7 +1412,7 @@ static void __ftrace_hash_rec_update(struct ftrace_ops *ops, ...@@ -1372,7 +1412,7 @@ static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
if (filter_hash && in_hash && !in_other_hash) if (filter_hash && in_hash && !in_other_hash)
match = 1; match = 1;
else if (!filter_hash && in_hash && else if (!filter_hash && in_hash &&
(in_other_hash || !other_hash->count)) (in_other_hash || ftrace_hash_empty(other_hash)))
match = 1; match = 1;
} }
if (!match) if (!match)
...@@ -1406,40 +1446,12 @@ static void ftrace_hash_rec_enable(struct ftrace_ops *ops, ...@@ -1406,40 +1446,12 @@ static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
__ftrace_hash_rec_update(ops, filter_hash, 1); __ftrace_hash_rec_update(ops, filter_hash, 1);
} }
static void ftrace_free_rec(struct dyn_ftrace *rec)
{
rec->freelist = ftrace_free_records;
ftrace_free_records = rec;
rec->flags |= FTRACE_FL_FREE;
}
static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip) static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
{ {
struct dyn_ftrace *rec; if (ftrace_pages->index == ftrace_pages->size) {
/* We should have allocated enough */
/* First check for freed records */ if (WARN_ON(!ftrace_pages->next))
if (ftrace_free_records) {
rec = ftrace_free_records;
if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
FTRACE_WARN_ON_ONCE(1);
ftrace_free_records = NULL;
return NULL; return NULL;
}
ftrace_free_records = rec->freelist;
memset(rec, 0, sizeof(*rec));
return rec;
}
if (ftrace_pages->index == ENTRIES_PER_PAGE) {
if (!ftrace_pages->next) {
/* allocate another page */
ftrace_pages->next =
(void *)get_zeroed_page(GFP_KERNEL);
if (!ftrace_pages->next)
return NULL;
}
ftrace_pages = ftrace_pages->next; ftrace_pages = ftrace_pages->next;
} }
...@@ -1459,8 +1471,6 @@ ftrace_record_ip(unsigned long ip) ...@@ -1459,8 +1471,6 @@ ftrace_record_ip(unsigned long ip)
return NULL; return NULL;
rec->ip = ip; rec->ip = ip;
rec->newlist = ftrace_new_addrs;
ftrace_new_addrs = rec;
return rec; return rec;
} }
...@@ -1475,7 +1485,19 @@ static void print_ip_ins(const char *fmt, unsigned char *p) ...@@ -1475,7 +1485,19 @@ static void print_ip_ins(const char *fmt, unsigned char *p)
printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]); printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
} }
static void ftrace_bug(int failed, unsigned long ip) /**
* ftrace_bug - report and shutdown function tracer
* @failed: The failed type (EFAULT, EINVAL, EPERM)
* @ip: The address that failed
*
* The arch code that enables or disables the function tracing
* can call ftrace_bug() when it has detected a problem in
* modifying the code. @failed should be one of either:
* EFAULT - if the problem happens on reading the @ip address
* EINVAL - if what is read at @ip is not what was expected
* EPERM - if the problem happens on writting to the @ip address
*/
void ftrace_bug(int failed, unsigned long ip)
{ {
switch (failed) { switch (failed) {
case -EFAULT: case -EFAULT:
...@@ -1517,24 +1539,19 @@ int ftrace_text_reserved(void *start, void *end) ...@@ -1517,24 +1539,19 @@ int ftrace_text_reserved(void *start, void *end)
return 0; return 0;
} }
static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
static int
__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
{ {
unsigned long ftrace_addr;
unsigned long flag = 0UL; unsigned long flag = 0UL;
ftrace_addr = (unsigned long)FTRACE_ADDR;
/* /*
* If we are enabling tracing: * If we are updating calls:
* *
* If the record has a ref count, then we need to enable it * If the record has a ref count, then we need to enable it
* because someone is using it. * because someone is using it.
* *
* Otherwise we make sure its disabled. * Otherwise we make sure its disabled.
* *
* If we are disabling tracing, then disable all records that * If we are disabling calls, then disable all records that
* are enabled. * are enabled.
*/ */
if (enable && (rec->flags & ~FTRACE_FL_MASK)) if (enable && (rec->flags & ~FTRACE_FL_MASK))
...@@ -1542,18 +1559,72 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable) ...@@ -1542,18 +1559,72 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
/* If the state of this record hasn't changed, then do nothing */ /* If the state of this record hasn't changed, then do nothing */
if ((rec->flags & FTRACE_FL_ENABLED) == flag) if ((rec->flags & FTRACE_FL_ENABLED) == flag)
return 0; return FTRACE_UPDATE_IGNORE;
if (flag) { if (flag) {
rec->flags |= FTRACE_FL_ENABLED; if (update)
rec->flags |= FTRACE_FL_ENABLED;
return FTRACE_UPDATE_MAKE_CALL;
}
if (update)
rec->flags &= ~FTRACE_FL_ENABLED;
return FTRACE_UPDATE_MAKE_NOP;
}
/**
* ftrace_update_record, set a record that now is tracing or not
* @rec: the record to update
* @enable: set to 1 if the record is tracing, zero to force disable
*
* The records that represent all functions that can be traced need
* to be updated when tracing has been enabled.
*/
int ftrace_update_record(struct dyn_ftrace *rec, int enable)
{
return ftrace_check_record(rec, enable, 1);
}
/**
* ftrace_test_record, check if the record has been enabled or not
* @rec: the record to test
* @enable: set to 1 to check if enabled, 0 if it is disabled
*
* The arch code may need to test if a record is already set to
* tracing to determine how to modify the function code that it
* represents.
*/
int ftrace_test_record(struct dyn_ftrace *rec, int enable)
{
return ftrace_check_record(rec, enable, 0);
}
static int
__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
{
unsigned long ftrace_addr;
int ret;
ftrace_addr = (unsigned long)FTRACE_ADDR;
ret = ftrace_update_record(rec, enable);
switch (ret) {
case FTRACE_UPDATE_IGNORE:
return 0;
case FTRACE_UPDATE_MAKE_CALL:
return ftrace_make_call(rec, ftrace_addr); return ftrace_make_call(rec, ftrace_addr);
case FTRACE_UPDATE_MAKE_NOP:
return ftrace_make_nop(NULL, rec, ftrace_addr);
} }
rec->flags &= ~FTRACE_FL_ENABLED; return -1; /* unknow ftrace bug */
return ftrace_make_nop(NULL, rec, ftrace_addr);
} }
static void ftrace_replace_code(int enable) static void ftrace_replace_code(int update)
{ {
struct dyn_ftrace *rec; struct dyn_ftrace *rec;
struct ftrace_page *pg; struct ftrace_page *pg;
...@@ -1563,11 +1634,7 @@ static void ftrace_replace_code(int enable) ...@@ -1563,11 +1634,7 @@ static void ftrace_replace_code(int enable)
return; return;
do_for_each_ftrace_rec(pg, rec) { do_for_each_ftrace_rec(pg, rec) {
/* Skip over free records */ failed = __ftrace_replace_code(rec, update);
if (rec->flags & FTRACE_FL_FREE)
continue;
failed = __ftrace_replace_code(rec, enable);
if (failed) { if (failed) {
ftrace_bug(failed, rec->ip); ftrace_bug(failed, rec->ip);
/* Stop processing */ /* Stop processing */
...@@ -1576,6 +1643,78 @@ static void ftrace_replace_code(int enable) ...@@ -1576,6 +1643,78 @@ static void ftrace_replace_code(int enable)
} while_for_each_ftrace_rec(); } while_for_each_ftrace_rec();
} }
struct ftrace_rec_iter {
struct ftrace_page *pg;
int index;
};
/**
* ftrace_rec_iter_start, start up iterating over traced functions
*
* Returns an iterator handle that is used to iterate over all
* the records that represent address locations where functions
* are traced.
*
* May return NULL if no records are available.
*/
struct ftrace_rec_iter *ftrace_rec_iter_start(void)
{
/*
* We only use a single iterator.
* Protected by the ftrace_lock mutex.
*/
static struct ftrace_rec_iter ftrace_rec_iter;
struct ftrace_rec_iter *iter = &ftrace_rec_iter;
iter->pg = ftrace_pages_start;
iter->index = 0;
/* Could have empty pages */
while (iter->pg && !iter->pg->index)
iter->pg = iter->pg->next;
if (!iter->pg)
return NULL;
return iter;
}
/**
* ftrace_rec_iter_next, get the next record to process.
* @iter: The handle to the iterator.
*
* Returns the next iterator after the given iterator @iter.
*/
struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
{
iter->index++;
if (iter->index >= iter->pg->index) {
iter->pg = iter->pg->next;
iter->index = 0;
/* Could have empty pages */
while (iter->pg && !iter->pg->index)
iter->pg = iter->pg->next;
}
if (!iter->pg)
return NULL;
return iter;
}
/**
* ftrace_rec_iter_record, get the record at the iterator location
* @iter: The current iterator location
*
* Returns the record that the current @iter is at.
*/
struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
{
return &iter->pg->records[iter->index];
}
static int static int
ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec) ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
{ {
...@@ -1617,13 +1756,7 @@ static int __ftrace_modify_code(void *data) ...@@ -1617,13 +1756,7 @@ static int __ftrace_modify_code(void *data)
{ {
int *command = data; int *command = data;
/* if (*command & FTRACE_UPDATE_CALLS)
* Do not call function tracer while we update the code.
* We are in stop machine, no worrying about races.
*/
function_trace_stop++;
if (*command & FTRACE_ENABLE_CALLS)
ftrace_replace_code(1); ftrace_replace_code(1);
else if (*command & FTRACE_DISABLE_CALLS) else if (*command & FTRACE_DISABLE_CALLS)
ftrace_replace_code(0); ftrace_replace_code(0);
...@@ -1636,21 +1769,33 @@ static int __ftrace_modify_code(void *data) ...@@ -1636,21 +1769,33 @@ static int __ftrace_modify_code(void *data)
else if (*command & FTRACE_STOP_FUNC_RET) else if (*command & FTRACE_STOP_FUNC_RET)
ftrace_disable_ftrace_graph_caller(); ftrace_disable_ftrace_graph_caller();
#ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
/*
* For archs that call ftrace_test_stop_func(), we must
* wait till after we update all the function callers
* before we update the callback. This keeps different
* ops that record different functions from corrupting
* each other.
*/
__ftrace_trace_function = __ftrace_trace_function_delay;
#endif
function_trace_stop--;
return 0; return 0;
} }
/**
* ftrace_run_stop_machine, go back to the stop machine method
* @command: The command to tell ftrace what to do
*
* If an arch needs to fall back to the stop machine method, the
* it can call this function.
*/
void ftrace_run_stop_machine(int command)
{
stop_machine(__ftrace_modify_code, &command, NULL);
}
/**
* arch_ftrace_update_code, modify the code to trace or not trace
* @command: The command that needs to be done
*
* Archs can override this function if it does not need to
* run stop_machine() to modify code.
*/
void __weak arch_ftrace_update_code(int command)
{
ftrace_run_stop_machine(command);
}
static void ftrace_run_update_code(int command) static void ftrace_run_update_code(int command)
{ {
int ret; int ret;
...@@ -1659,8 +1804,31 @@ static void ftrace_run_update_code(int command) ...@@ -1659,8 +1804,31 @@ static void ftrace_run_update_code(int command)
FTRACE_WARN_ON(ret); FTRACE_WARN_ON(ret);
if (ret) if (ret)
return; return;
/*
* Do not call function tracer while we update the code.
* We are in stop machine.
*/
function_trace_stop++;
stop_machine(__ftrace_modify_code, &command, NULL); /*
* By default we use stop_machine() to modify the code.
* But archs can do what ever they want as long as it
* is safe. The stop_machine() is the safest, but also
* produces the most overhead.
*/
arch_ftrace_update_code(command);
#ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
/*
* For archs that call ftrace_test_stop_func(), we must
* wait till after we update all the function callers
* before we update the callback. This keeps different
* ops that record different functions from corrupting
* each other.
*/
__ftrace_trace_function = __ftrace_trace_function_delay;
#endif
function_trace_stop--;
ret = ftrace_arch_code_modify_post_process(); ret = ftrace_arch_code_modify_post_process();
FTRACE_WARN_ON(ret); FTRACE_WARN_ON(ret);
...@@ -1691,7 +1859,7 @@ static int ftrace_startup(struct ftrace_ops *ops, int command) ...@@ -1691,7 +1859,7 @@ static int ftrace_startup(struct ftrace_ops *ops, int command)
return -ENODEV; return -ENODEV;
ftrace_start_up++; ftrace_start_up++;
command |= FTRACE_ENABLE_CALLS; command |= FTRACE_UPDATE_CALLS;
/* ops marked global share the filter hashes */ /* ops marked global share the filter hashes */
if (ops->flags & FTRACE_OPS_FL_GLOBAL) { if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
...@@ -1743,8 +1911,7 @@ static void ftrace_shutdown(struct ftrace_ops *ops, int command) ...@@ -1743,8 +1911,7 @@ static void ftrace_shutdown(struct ftrace_ops *ops, int command)
if (ops != &global_ops || !global_start_up) if (ops != &global_ops || !global_start_up)
ops->flags &= ~FTRACE_OPS_FL_ENABLED; ops->flags &= ~FTRACE_OPS_FL_ENABLED;
if (!ftrace_start_up) command |= FTRACE_UPDATE_CALLS;
command |= FTRACE_DISABLE_CALLS;
if (saved_ftrace_func != ftrace_trace_function) { if (saved_ftrace_func != ftrace_trace_function) {
saved_ftrace_func = ftrace_trace_function; saved_ftrace_func = ftrace_trace_function;
...@@ -1766,7 +1933,7 @@ static void ftrace_startup_sysctl(void) ...@@ -1766,7 +1933,7 @@ static void ftrace_startup_sysctl(void)
saved_ftrace_func = NULL; saved_ftrace_func = NULL;
/* ftrace_start_up is true if we want ftrace running */ /* ftrace_start_up is true if we want ftrace running */
if (ftrace_start_up) if (ftrace_start_up)
ftrace_run_update_code(FTRACE_ENABLE_CALLS); ftrace_run_update_code(FTRACE_UPDATE_CALLS);
} }
static void ftrace_shutdown_sysctl(void) static void ftrace_shutdown_sysctl(void)
...@@ -1788,14 +1955,16 @@ static int ops_traces_mod(struct ftrace_ops *ops) ...@@ -1788,14 +1955,16 @@ static int ops_traces_mod(struct ftrace_ops *ops)
struct ftrace_hash *hash; struct ftrace_hash *hash;
hash = ops->filter_hash; hash = ops->filter_hash;
return !!(!hash || !hash->count); return ftrace_hash_empty(hash);
} }
static int ftrace_update_code(struct module *mod) static int ftrace_update_code(struct module *mod)
{ {
struct ftrace_page *pg;
struct dyn_ftrace *p; struct dyn_ftrace *p;
cycle_t start, stop; cycle_t start, stop;
unsigned long ref = 0; unsigned long ref = 0;
int i;
/* /*
* When adding a module, we need to check if tracers are * When adding a module, we need to check if tracers are
...@@ -1817,46 +1986,44 @@ static int ftrace_update_code(struct module *mod) ...@@ -1817,46 +1986,44 @@ static int ftrace_update_code(struct module *mod)
start = ftrace_now(raw_smp_processor_id()); start = ftrace_now(raw_smp_processor_id());
ftrace_update_cnt = 0; ftrace_update_cnt = 0;
while (ftrace_new_addrs) { for (pg = ftrace_new_pgs; pg; pg = pg->next) {
/* If something went wrong, bail without enabling anything */ for (i = 0; i < pg->index; i++) {
if (unlikely(ftrace_disabled)) /* If something went wrong, bail without enabling anything */
return -1; if (unlikely(ftrace_disabled))
return -1;
p = ftrace_new_addrs; p = &pg->records[i];
ftrace_new_addrs = p->newlist; p->flags = ref;
p->flags = ref;
/* /*
* Do the initial record conversion from mcount jump * Do the initial record conversion from mcount jump
* to the NOP instructions. * to the NOP instructions.
*/ */
if (!ftrace_code_disable(mod, p)) { if (!ftrace_code_disable(mod, p))
ftrace_free_rec(p); break;
/* Game over */
break;
}
ftrace_update_cnt++; ftrace_update_cnt++;
/* /*
* If the tracing is enabled, go ahead and enable the record. * If the tracing is enabled, go ahead and enable the record.
* *
* The reason not to enable the record immediatelly is the * The reason not to enable the record immediatelly is the
* inherent check of ftrace_make_nop/ftrace_make_call for * inherent check of ftrace_make_nop/ftrace_make_call for
* correct previous instructions. Making first the NOP * correct previous instructions. Making first the NOP
* conversion puts the module to the correct state, thus * conversion puts the module to the correct state, thus
* passing the ftrace_make_call check. * passing the ftrace_make_call check.
*/ */
if (ftrace_start_up && ref) { if (ftrace_start_up && ref) {
int failed = __ftrace_replace_code(p, 1); int failed = __ftrace_replace_code(p, 1);
if (failed) { if (failed)
ftrace_bug(failed, p->ip); ftrace_bug(failed, p->ip);
ftrace_free_rec(p);
} }
} }
} }
ftrace_new_pgs = NULL;
stop = ftrace_now(raw_smp_processor_id()); stop = ftrace_now(raw_smp_processor_id());
ftrace_update_time = stop - start; ftrace_update_time = stop - start;
ftrace_update_tot_cnt += ftrace_update_cnt; ftrace_update_tot_cnt += ftrace_update_cnt;
...@@ -1864,57 +2031,108 @@ static int ftrace_update_code(struct module *mod) ...@@ -1864,57 +2031,108 @@ static int ftrace_update_code(struct module *mod)
return 0; return 0;
} }
static int __init ftrace_dyn_table_alloc(unsigned long num_to_init) static int ftrace_allocate_records(struct ftrace_page *pg, int count)
{ {
struct ftrace_page *pg; int order;
int cnt; int cnt;
int i;
/* allocate a few pages */ if (WARN_ON(!count))
ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL); return -EINVAL;
if (!ftrace_pages_start)
return -1; order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
/* /*
* Allocate a few more pages. * We want to fill as much as possible. No more than a page
* * may be empty.
* TODO: have some parser search vmlinux before
* final linking to find all calls to ftrace.
* Then we can:
* a) know how many pages to allocate.
* and/or
* b) set up the table then.
*
* The dynamic code is still necessary for
* modules.
*/ */
while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
order--;
pg = ftrace_pages = ftrace_pages_start; again:
pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
cnt = num_to_init / ENTRIES_PER_PAGE; if (!pg->records) {
pr_info("ftrace: allocating %ld entries in %d pages\n", /* if we can't allocate this size, try something smaller */
num_to_init, cnt + 1); if (!order)
return -ENOMEM;
order >>= 1;
goto again;
}
for (i = 0; i < cnt; i++) { cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
pg->next = (void *)get_zeroed_page(GFP_KERNEL); pg->size = cnt;
/* If we fail, we'll try later anyway */ if (cnt > count)
if (!pg->next) cnt = count;
return cnt;
}
static struct ftrace_page *
ftrace_allocate_pages(unsigned long num_to_init)
{
struct ftrace_page *start_pg;
struct ftrace_page *pg;
int order;
int cnt;
if (!num_to_init)
return 0;
start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
if (!pg)
return NULL;
/*
* Try to allocate as much as possible in one continues
* location that fills in all of the space. We want to
* waste as little space as possible.
*/
for (;;) {
cnt = ftrace_allocate_records(pg, num_to_init);
if (cnt < 0)
goto free_pages;
num_to_init -= cnt;
if (!num_to_init)
break; break;
pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
if (!pg->next)
goto free_pages;
pg = pg->next; pg = pg->next;
} }
return 0; return start_pg;
free_pages:
while (start_pg) {
order = get_count_order(pg->size / ENTRIES_PER_PAGE);
free_pages((unsigned long)pg->records, order);
start_pg = pg->next;
kfree(pg);
pg = start_pg;
}
pr_info("ftrace: FAILED to allocate memory for functions\n");
return NULL;
} }
enum { static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
FTRACE_ITER_FILTER = (1 << 0), {
FTRACE_ITER_NOTRACE = (1 << 1), int cnt;
FTRACE_ITER_PRINTALL = (1 << 2),
FTRACE_ITER_HASH = (1 << 3), if (!num_to_init) {
FTRACE_ITER_ENABLED = (1 << 4), pr_info("ftrace: No functions to be traced?\n");
}; return -1;
}
cnt = num_to_init / ENTRIES_PER_PAGE;
pr_info("ftrace: allocating %ld entries in %d pages\n",
num_to_init, cnt + 1);
return 0;
}
#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
...@@ -1980,6 +2198,9 @@ static void *t_hash_start(struct seq_file *m, loff_t *pos) ...@@ -1980,6 +2198,9 @@ static void *t_hash_start(struct seq_file *m, loff_t *pos)
void *p = NULL; void *p = NULL;
loff_t l; loff_t l;
if (!(iter->flags & FTRACE_ITER_DO_HASH))
return NULL;
if (iter->func_pos > *pos) if (iter->func_pos > *pos)
return NULL; return NULL;
...@@ -2023,7 +2244,7 @@ static void * ...@@ -2023,7 +2244,7 @@ static void *
t_next(struct seq_file *m, void *v, loff_t *pos) t_next(struct seq_file *m, void *v, loff_t *pos)
{ {
struct ftrace_iterator *iter = m->private; struct ftrace_iterator *iter = m->private;
struct ftrace_ops *ops = &global_ops; struct ftrace_ops *ops = iter->ops;
struct dyn_ftrace *rec = NULL; struct dyn_ftrace *rec = NULL;
if (unlikely(ftrace_disabled)) if (unlikely(ftrace_disabled))
...@@ -2047,9 +2268,7 @@ t_next(struct seq_file *m, void *v, loff_t *pos) ...@@ -2047,9 +2268,7 @@ t_next(struct seq_file *m, void *v, loff_t *pos)
} }
} else { } else {
rec = &iter->pg->records[iter->idx++]; rec = &iter->pg->records[iter->idx++];
if ((rec->flags & FTRACE_FL_FREE) || if (((iter->flags & FTRACE_ITER_FILTER) &&
((iter->flags & FTRACE_ITER_FILTER) &&
!(ftrace_lookup_ip(ops->filter_hash, rec->ip))) || !(ftrace_lookup_ip(ops->filter_hash, rec->ip))) ||
((iter->flags & FTRACE_ITER_NOTRACE) && ((iter->flags & FTRACE_ITER_NOTRACE) &&
...@@ -2081,7 +2300,7 @@ static void reset_iter_read(struct ftrace_iterator *iter) ...@@ -2081,7 +2300,7 @@ static void reset_iter_read(struct ftrace_iterator *iter)
static void *t_start(struct seq_file *m, loff_t *pos) static void *t_start(struct seq_file *m, loff_t *pos)
{ {
struct ftrace_iterator *iter = m->private; struct ftrace_iterator *iter = m->private;
struct ftrace_ops *ops = &global_ops; struct ftrace_ops *ops = iter->ops;
void *p = NULL; void *p = NULL;
loff_t l; loff_t l;
...@@ -2101,7 +2320,8 @@ static void *t_start(struct seq_file *m, loff_t *pos) ...@@ -2101,7 +2320,8 @@ static void *t_start(struct seq_file *m, loff_t *pos)
* off, we can short cut and just print out that all * off, we can short cut and just print out that all
* functions are enabled. * functions are enabled.
*/ */
if (iter->flags & FTRACE_ITER_FILTER && !ops->filter_hash->count) { if (iter->flags & FTRACE_ITER_FILTER &&
ftrace_hash_empty(ops->filter_hash)) {
if (*pos > 0) if (*pos > 0)
return t_hash_start(m, pos); return t_hash_start(m, pos);
iter->flags |= FTRACE_ITER_PRINTALL; iter->flags |= FTRACE_ITER_PRINTALL;
...@@ -2126,12 +2346,8 @@ static void *t_start(struct seq_file *m, loff_t *pos) ...@@ -2126,12 +2346,8 @@ static void *t_start(struct seq_file *m, loff_t *pos)
break; break;
} }
if (!p) { if (!p)
if (iter->flags & FTRACE_ITER_FILTER) return t_hash_start(m, pos);
return t_hash_start(m, pos);
return NULL;
}
return iter; return iter;
} }
...@@ -2189,6 +2405,7 @@ ftrace_avail_open(struct inode *inode, struct file *file) ...@@ -2189,6 +2405,7 @@ ftrace_avail_open(struct inode *inode, struct file *file)
return -ENOMEM; return -ENOMEM;
iter->pg = ftrace_pages_start; iter->pg = ftrace_pages_start;
iter->ops = &global_ops;
ret = seq_open(file, &show_ftrace_seq_ops); ret = seq_open(file, &show_ftrace_seq_ops);
if (!ret) { if (!ret) {
...@@ -2217,6 +2434,7 @@ ftrace_enabled_open(struct inode *inode, struct file *file) ...@@ -2217,6 +2434,7 @@ ftrace_enabled_open(struct inode *inode, struct file *file)
iter->pg = ftrace_pages_start; iter->pg = ftrace_pages_start;
iter->flags = FTRACE_ITER_ENABLED; iter->flags = FTRACE_ITER_ENABLED;
iter->ops = &global_ops;
ret = seq_open(file, &show_ftrace_seq_ops); ret = seq_open(file, &show_ftrace_seq_ops);
if (!ret) { if (!ret) {
...@@ -2237,7 +2455,23 @@ static void ftrace_filter_reset(struct ftrace_hash *hash) ...@@ -2237,7 +2455,23 @@ static void ftrace_filter_reset(struct ftrace_hash *hash)
mutex_unlock(&ftrace_lock); mutex_unlock(&ftrace_lock);
} }
static int /**
* ftrace_regex_open - initialize function tracer filter files
* @ops: The ftrace_ops that hold the hash filters
* @flag: The type of filter to process
* @inode: The inode, usually passed in to your open routine
* @file: The file, usually passed in to your open routine
*
* ftrace_regex_open() initializes the filter files for the
* @ops. Depending on @flag it may process the filter hash or
* the notrace hash of @ops. With this called from the open
* routine, you can use ftrace_filter_write() for the write
* routine if @flag has FTRACE_ITER_FILTER set, or
* ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
* ftrace_regex_lseek() should be used as the lseek routine, and
* release must call ftrace_regex_release().
*/
int
ftrace_regex_open(struct ftrace_ops *ops, int flag, ftrace_regex_open(struct ftrace_ops *ops, int flag,
struct inode *inode, struct file *file) struct inode *inode, struct file *file)
{ {
...@@ -2306,8 +2540,9 @@ ftrace_regex_open(struct ftrace_ops *ops, int flag, ...@@ -2306,8 +2540,9 @@ ftrace_regex_open(struct ftrace_ops *ops, int flag,
static int static int
ftrace_filter_open(struct inode *inode, struct file *file) ftrace_filter_open(struct inode *inode, struct file *file)
{ {
return ftrace_regex_open(&global_ops, FTRACE_ITER_FILTER, return ftrace_regex_open(&global_ops,
inode, file); FTRACE_ITER_FILTER | FTRACE_ITER_DO_HASH,
inode, file);
} }
static int static int
...@@ -2317,7 +2552,7 @@ ftrace_notrace_open(struct inode *inode, struct file *file) ...@@ -2317,7 +2552,7 @@ ftrace_notrace_open(struct inode *inode, struct file *file)
inode, file); inode, file);
} }
static loff_t loff_t
ftrace_regex_lseek(struct file *file, loff_t offset, int origin) ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
{ {
loff_t ret; loff_t ret;
...@@ -2426,7 +2661,6 @@ match_records(struct ftrace_hash *hash, char *buff, ...@@ -2426,7 +2661,6 @@ match_records(struct ftrace_hash *hash, char *buff,
goto out_unlock; goto out_unlock;
do_for_each_ftrace_rec(pg, rec) { do_for_each_ftrace_rec(pg, rec) {
if (ftrace_match_record(rec, mod, search, search_len, type)) { if (ftrace_match_record(rec, mod, search, search_len, type)) {
ret = enter_record(hash, rec, not); ret = enter_record(hash, rec, not);
if (ret < 0) { if (ret < 0) {
...@@ -2871,14 +3105,14 @@ ftrace_regex_write(struct file *file, const char __user *ubuf, ...@@ -2871,14 +3105,14 @@ ftrace_regex_write(struct file *file, const char __user *ubuf,
return ret; return ret;
} }
static ssize_t ssize_t
ftrace_filter_write(struct file *file, const char __user *ubuf, ftrace_filter_write(struct file *file, const char __user *ubuf,
size_t cnt, loff_t *ppos) size_t cnt, loff_t *ppos)
{ {
return ftrace_regex_write(file, ubuf, cnt, ppos, 1); return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
} }
static ssize_t ssize_t
ftrace_notrace_write(struct file *file, const char __user *ubuf, ftrace_notrace_write(struct file *file, const char __user *ubuf,
size_t cnt, loff_t *ppos) size_t cnt, loff_t *ppos)
{ {
...@@ -2919,7 +3153,7 @@ ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len, ...@@ -2919,7 +3153,7 @@ ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
ret = ftrace_hash_move(ops, enable, orig_hash, hash); ret = ftrace_hash_move(ops, enable, orig_hash, hash);
if (!ret && ops->flags & FTRACE_OPS_FL_ENABLED if (!ret && ops->flags & FTRACE_OPS_FL_ENABLED
&& ftrace_enabled) && ftrace_enabled)
ftrace_run_update_code(FTRACE_ENABLE_CALLS); ftrace_run_update_code(FTRACE_UPDATE_CALLS);
mutex_unlock(&ftrace_lock); mutex_unlock(&ftrace_lock);
...@@ -3045,8 +3279,8 @@ static void __init set_ftrace_early_graph(char *buf) ...@@ -3045,8 +3279,8 @@ static void __init set_ftrace_early_graph(char *buf)
} }
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
static void __init void __init
set_ftrace_early_filter(struct ftrace_ops *ops, char *buf, int enable) ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
{ {
char *func; char *func;
...@@ -3059,17 +3293,16 @@ set_ftrace_early_filter(struct ftrace_ops *ops, char *buf, int enable) ...@@ -3059,17 +3293,16 @@ set_ftrace_early_filter(struct ftrace_ops *ops, char *buf, int enable)
static void __init set_ftrace_early_filters(void) static void __init set_ftrace_early_filters(void)
{ {
if (ftrace_filter_buf[0]) if (ftrace_filter_buf[0])
set_ftrace_early_filter(&global_ops, ftrace_filter_buf, 1); ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
if (ftrace_notrace_buf[0]) if (ftrace_notrace_buf[0])
set_ftrace_early_filter(&global_ops, ftrace_notrace_buf, 0); ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
#ifdef CONFIG_FUNCTION_GRAPH_TRACER #ifdef CONFIG_FUNCTION_GRAPH_TRACER
if (ftrace_graph_buf[0]) if (ftrace_graph_buf[0])
set_ftrace_early_graph(ftrace_graph_buf); set_ftrace_early_graph(ftrace_graph_buf);
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
} }
static int int ftrace_regex_release(struct inode *inode, struct file *file)
ftrace_regex_release(struct inode *inode, struct file *file)
{ {
struct seq_file *m = (struct seq_file *)file->private_data; struct seq_file *m = (struct seq_file *)file->private_data;
struct ftrace_iterator *iter; struct ftrace_iterator *iter;
...@@ -3107,7 +3340,7 @@ ftrace_regex_release(struct inode *inode, struct file *file) ...@@ -3107,7 +3340,7 @@ ftrace_regex_release(struct inode *inode, struct file *file)
orig_hash, iter->hash); orig_hash, iter->hash);
if (!ret && (iter->ops->flags & FTRACE_OPS_FL_ENABLED) if (!ret && (iter->ops->flags & FTRACE_OPS_FL_ENABLED)
&& ftrace_enabled) && ftrace_enabled)
ftrace_run_update_code(FTRACE_ENABLE_CALLS); ftrace_run_update_code(FTRACE_UPDATE_CALLS);
mutex_unlock(&ftrace_lock); mutex_unlock(&ftrace_lock);
} }
...@@ -3270,9 +3503,6 @@ ftrace_set_func(unsigned long *array, int *idx, char *buffer) ...@@ -3270,9 +3503,6 @@ ftrace_set_func(unsigned long *array, int *idx, char *buffer)
do_for_each_ftrace_rec(pg, rec) { do_for_each_ftrace_rec(pg, rec) {
if (rec->flags & FTRACE_FL_FREE)
continue;
if (ftrace_match_record(rec, NULL, search, search_len, type)) { if (ftrace_match_record(rec, NULL, search, search_len, type)) {
/* if it is in the array */ /* if it is in the array */
exists = false; exists = false;
...@@ -3381,15 +3611,62 @@ static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer) ...@@ -3381,15 +3611,62 @@ static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
return 0; return 0;
} }
static void ftrace_swap_recs(void *a, void *b, int size)
{
struct dyn_ftrace *reca = a;
struct dyn_ftrace *recb = b;
struct dyn_ftrace t;
t = *reca;
*reca = *recb;
*recb = t;
}
static int ftrace_process_locs(struct module *mod, static int ftrace_process_locs(struct module *mod,
unsigned long *start, unsigned long *start,
unsigned long *end) unsigned long *end)
{ {
struct ftrace_page *pg;
unsigned long count;
unsigned long *p; unsigned long *p;
unsigned long addr; unsigned long addr;
unsigned long flags = 0; /* Shut up gcc */ unsigned long flags = 0; /* Shut up gcc */
int ret = -ENOMEM;
count = end - start;
if (!count)
return 0;
pg = ftrace_allocate_pages(count);
if (!pg)
return -ENOMEM;
mutex_lock(&ftrace_lock); mutex_lock(&ftrace_lock);
/*
* Core and each module needs their own pages, as
* modules will free them when they are removed.
* Force a new page to be allocated for modules.
*/
if (!mod) {
WARN_ON(ftrace_pages || ftrace_pages_start);
/* First initialization */
ftrace_pages = ftrace_pages_start = pg;
} else {
if (!ftrace_pages)
goto out;
if (WARN_ON(ftrace_pages->next)) {
/* Hmm, we have free pages? */
while (ftrace_pages->next)
ftrace_pages = ftrace_pages->next;
}
ftrace_pages->next = pg;
ftrace_pages = pg;
}
p = start; p = start;
while (p < end) { while (p < end) {
addr = ftrace_call_adjust(*p++); addr = ftrace_call_adjust(*p++);
...@@ -3401,9 +3678,18 @@ static int ftrace_process_locs(struct module *mod, ...@@ -3401,9 +3678,18 @@ static int ftrace_process_locs(struct module *mod,
*/ */
if (!addr) if (!addr)
continue; continue;
ftrace_record_ip(addr); if (!ftrace_record_ip(addr))
break;
} }
/* These new locations need to be initialized */
ftrace_new_pgs = pg;
/* Make each individual set of pages sorted by ips */
for (; pg; pg = pg->next)
sort(pg->records, pg->index, sizeof(struct dyn_ftrace),
ftrace_cmp_recs, ftrace_swap_recs);
/* /*
* We only need to disable interrupts on start up * We only need to disable interrupts on start up
* because we are modifying code that an interrupt * because we are modifying code that an interrupt
...@@ -3417,32 +3703,55 @@ static int ftrace_process_locs(struct module *mod, ...@@ -3417,32 +3703,55 @@ static int ftrace_process_locs(struct module *mod,
ftrace_update_code(mod); ftrace_update_code(mod);
if (!mod) if (!mod)
local_irq_restore(flags); local_irq_restore(flags);
ret = 0;
out:
mutex_unlock(&ftrace_lock); mutex_unlock(&ftrace_lock);
return 0; return ret;
} }
#ifdef CONFIG_MODULES #ifdef CONFIG_MODULES
#define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)
void ftrace_release_mod(struct module *mod) void ftrace_release_mod(struct module *mod)
{ {
struct dyn_ftrace *rec; struct dyn_ftrace *rec;
struct ftrace_page **last_pg;
struct ftrace_page *pg; struct ftrace_page *pg;
int order;
mutex_lock(&ftrace_lock); mutex_lock(&ftrace_lock);
if (ftrace_disabled) if (ftrace_disabled)
goto out_unlock; goto out_unlock;
do_for_each_ftrace_rec(pg, rec) { /*
* Each module has its own ftrace_pages, remove
* them from the list.
*/
last_pg = &ftrace_pages_start;
for (pg = ftrace_pages_start; pg; pg = *last_pg) {
rec = &pg->records[0];
if (within_module_core(rec->ip, mod)) { if (within_module_core(rec->ip, mod)) {
/* /*
* rec->ip is changed in ftrace_free_rec() * As core pages are first, the first
* It should not between s and e if record was freed. * page should never be a module page.
*/ */
FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE); if (WARN_ON(pg == ftrace_pages_start))
ftrace_free_rec(rec); goto out_unlock;
}
} while_for_each_ftrace_rec(); /* Check if we are deleting the last page */
if (pg == ftrace_pages)
ftrace_pages = next_to_ftrace_page(last_pg);
*last_pg = pg->next;
order = get_count_order(pg->size / ENTRIES_PER_PAGE);
free_pages((unsigned long)pg->records, order);
kfree(pg);
} else
last_pg = &pg->next;
}
out_unlock: out_unlock:
mutex_unlock(&ftrace_lock); mutex_unlock(&ftrace_lock);
} }
......
...@@ -1738,11 +1738,121 @@ static int replace_system_preds(struct event_subsystem *system, ...@@ -1738,11 +1738,121 @@ static int replace_system_preds(struct event_subsystem *system,
return -ENOMEM; return -ENOMEM;
} }
static int create_filter_start(char *filter_str, bool set_str,
struct filter_parse_state **psp,
struct event_filter **filterp)
{
struct event_filter *filter;
struct filter_parse_state *ps = NULL;
int err = 0;
WARN_ON_ONCE(*psp || *filterp);
/* allocate everything, and if any fails, free all and fail */
filter = __alloc_filter();
if (filter && set_str)
err = replace_filter_string(filter, filter_str);
ps = kzalloc(sizeof(*ps), GFP_KERNEL);
if (!filter || !ps || err) {
kfree(ps);
__free_filter(filter);
return -ENOMEM;
}
/* we're committed to creating a new filter */
*filterp = filter;
*psp = ps;
parse_init(ps, filter_ops, filter_str);
err = filter_parse(ps);
if (err && set_str)
append_filter_err(ps, filter);
return err;
}
static void create_filter_finish(struct filter_parse_state *ps)
{
if (ps) {
filter_opstack_clear(ps);
postfix_clear(ps);
kfree(ps);
}
}
/**
* create_filter - create a filter for a ftrace_event_call
* @call: ftrace_event_call to create a filter for
* @filter_str: filter string
* @set_str: remember @filter_str and enable detailed error in filter
* @filterp: out param for created filter (always updated on return)
*
* Creates a filter for @call with @filter_str. If @set_str is %true,
* @filter_str is copied and recorded in the new filter.
*
* On success, returns 0 and *@filterp points to the new filter. On
* failure, returns -errno and *@filterp may point to %NULL or to a new
* filter. In the latter case, the returned filter contains error
* information if @set_str is %true and the caller is responsible for
* freeing it.
*/
static int create_filter(struct ftrace_event_call *call,
char *filter_str, bool set_str,
struct event_filter **filterp)
{
struct event_filter *filter = NULL;
struct filter_parse_state *ps = NULL;
int err;
err = create_filter_start(filter_str, set_str, &ps, &filter);
if (!err) {
err = replace_preds(call, filter, ps, filter_str, false);
if (err && set_str)
append_filter_err(ps, filter);
}
create_filter_finish(ps);
*filterp = filter;
return err;
}
/**
* create_system_filter - create a filter for an event_subsystem
* @system: event_subsystem to create a filter for
* @filter_str: filter string
* @filterp: out param for created filter (always updated on return)
*
* Identical to create_filter() except that it creates a subsystem filter
* and always remembers @filter_str.
*/
static int create_system_filter(struct event_subsystem *system,
char *filter_str, struct event_filter **filterp)
{
struct event_filter *filter = NULL;
struct filter_parse_state *ps = NULL;
int err;
err = create_filter_start(filter_str, true, &ps, &filter);
if (!err) {
err = replace_system_preds(system, ps, filter_str);
if (!err) {
/* System filters just show a default message */
kfree(filter->filter_string);
filter->filter_string = NULL;
} else {
append_filter_err(ps, filter);
}
}
create_filter_finish(ps);
*filterp = filter;
return err;
}
int apply_event_filter(struct ftrace_event_call *call, char *filter_string) int apply_event_filter(struct ftrace_event_call *call, char *filter_string)
{ {
struct filter_parse_state *ps;
struct event_filter *filter; struct event_filter *filter;
struct event_filter *tmp;
int err = 0; int err = 0;
mutex_lock(&event_mutex); mutex_lock(&event_mutex);
...@@ -1759,49 +1869,30 @@ int apply_event_filter(struct ftrace_event_call *call, char *filter_string) ...@@ -1759,49 +1869,30 @@ int apply_event_filter(struct ftrace_event_call *call, char *filter_string)
goto out_unlock; goto out_unlock;
} }
err = -ENOMEM; err = create_filter(call, filter_string, true, &filter);
ps = kzalloc(sizeof(*ps), GFP_KERNEL);
if (!ps)
goto out_unlock;
filter = __alloc_filter();
if (!filter) {
kfree(ps);
goto out_unlock;
}
replace_filter_string(filter, filter_string);
parse_init(ps, filter_ops, filter_string);
err = filter_parse(ps);
if (err) {
append_filter_err(ps, filter);
goto out;
}
err = replace_preds(call, filter, ps, filter_string, false);
if (err) {
filter_disable(call);
append_filter_err(ps, filter);
} else
call->flags |= TRACE_EVENT_FL_FILTERED;
out:
/* /*
* Always swap the call filter with the new filter * Always swap the call filter with the new filter
* even if there was an error. If there was an error * even if there was an error. If there was an error
* in the filter, we disable the filter and show the error * in the filter, we disable the filter and show the error
* string * string
*/ */
tmp = call->filter; if (filter) {
rcu_assign_pointer(call->filter, filter); struct event_filter *tmp = call->filter;
if (tmp) {
/* Make sure the call is done with the filter */ if (!err)
synchronize_sched(); call->flags |= TRACE_EVENT_FL_FILTERED;
__free_filter(tmp); else
filter_disable(call);
rcu_assign_pointer(call->filter, filter);
if (tmp) {
/* Make sure the call is done with the filter */
synchronize_sched();
__free_filter(tmp);
}
} }
filter_opstack_clear(ps);
postfix_clear(ps);
kfree(ps);
out_unlock: out_unlock:
mutex_unlock(&event_mutex); mutex_unlock(&event_mutex);
...@@ -1811,7 +1902,6 @@ int apply_event_filter(struct ftrace_event_call *call, char *filter_string) ...@@ -1811,7 +1902,6 @@ int apply_event_filter(struct ftrace_event_call *call, char *filter_string)
int apply_subsystem_event_filter(struct event_subsystem *system, int apply_subsystem_event_filter(struct event_subsystem *system,
char *filter_string) char *filter_string)
{ {
struct filter_parse_state *ps;
struct event_filter *filter; struct event_filter *filter;
int err = 0; int err = 0;
...@@ -1835,48 +1925,19 @@ int apply_subsystem_event_filter(struct event_subsystem *system, ...@@ -1835,48 +1925,19 @@ int apply_subsystem_event_filter(struct event_subsystem *system,
goto out_unlock; goto out_unlock;
} }
err = -ENOMEM; err = create_system_filter(system, filter_string, &filter);
ps = kzalloc(sizeof(*ps), GFP_KERNEL); if (filter) {
if (!ps) /*
goto out_unlock; * No event actually uses the system filter
* we can free it without synchronize_sched().
filter = __alloc_filter(); */
if (!filter) __free_filter(system->filter);
goto out; system->filter = filter;
}
/* System filters just show a default message */
kfree(filter->filter_string);
filter->filter_string = NULL;
/*
* No event actually uses the system filter
* we can free it without synchronize_sched().
*/
__free_filter(system->filter);
system->filter = filter;
parse_init(ps, filter_ops, filter_string);
err = filter_parse(ps);
if (err)
goto err_filter;
err = replace_system_preds(system, ps, filter_string);
if (err)
goto err_filter;
out:
filter_opstack_clear(ps);
postfix_clear(ps);
kfree(ps);
out_unlock: out_unlock:
mutex_unlock(&event_mutex); mutex_unlock(&event_mutex);
return err; return err;
err_filter:
replace_filter_string(filter, filter_string);
append_filter_err(ps, system->filter);
goto out;
} }
#ifdef CONFIG_PERF_EVENTS #ifdef CONFIG_PERF_EVENTS
...@@ -1894,7 +1955,6 @@ int ftrace_profile_set_filter(struct perf_event *event, int event_id, ...@@ -1894,7 +1955,6 @@ int ftrace_profile_set_filter(struct perf_event *event, int event_id,
{ {
int err; int err;
struct event_filter *filter; struct event_filter *filter;
struct filter_parse_state *ps;
struct ftrace_event_call *call; struct ftrace_event_call *call;
mutex_lock(&event_mutex); mutex_lock(&event_mutex);
...@@ -1909,33 +1969,10 @@ int ftrace_profile_set_filter(struct perf_event *event, int event_id, ...@@ -1909,33 +1969,10 @@ int ftrace_profile_set_filter(struct perf_event *event, int event_id,
if (event->filter) if (event->filter)
goto out_unlock; goto out_unlock;
filter = __alloc_filter(); err = create_filter(call, filter_str, false, &filter);
if (!filter) {
err = PTR_ERR(filter);
goto out_unlock;
}
err = -ENOMEM;
ps = kzalloc(sizeof(*ps), GFP_KERNEL);
if (!ps)
goto free_filter;
parse_init(ps, filter_ops, filter_str);
err = filter_parse(ps);
if (err)
goto free_ps;
err = replace_preds(call, filter, ps, filter_str, false);
if (!err) if (!err)
event->filter = filter; event->filter = filter;
else
free_ps:
filter_opstack_clear(ps);
postfix_clear(ps);
kfree(ps);
free_filter:
if (err)
__free_filter(filter); __free_filter(filter);
out_unlock: out_unlock:
...@@ -1954,43 +1991,6 @@ int ftrace_profile_set_filter(struct perf_event *event, int event_id, ...@@ -1954,43 +1991,6 @@ int ftrace_profile_set_filter(struct perf_event *event, int event_id,
#define CREATE_TRACE_POINTS #define CREATE_TRACE_POINTS
#include "trace_events_filter_test.h" #include "trace_events_filter_test.h"
static int test_get_filter(char *filter_str, struct ftrace_event_call *call,
struct event_filter **pfilter)
{
struct event_filter *filter;
struct filter_parse_state *ps;
int err = -ENOMEM;
filter = __alloc_filter();
if (!filter)
goto out;
ps = kzalloc(sizeof(*ps), GFP_KERNEL);
if (!ps)
goto free_filter;
parse_init(ps, filter_ops, filter_str);
err = filter_parse(ps);
if (err)
goto free_ps;
err = replace_preds(call, filter, ps, filter_str, false);
if (!err)
*pfilter = filter;
free_ps:
filter_opstack_clear(ps);
postfix_clear(ps);
kfree(ps);
free_filter:
if (err)
__free_filter(filter);
out:
return err;
}
#define DATA_REC(m, va, vb, vc, vd, ve, vf, vg, vh, nvisit) \ #define DATA_REC(m, va, vb, vc, vd, ve, vf, vg, vh, nvisit) \
{ \ { \
.filter = FILTER, \ .filter = FILTER, \
...@@ -2109,12 +2109,13 @@ static __init int ftrace_test_event_filter(void) ...@@ -2109,12 +2109,13 @@ static __init int ftrace_test_event_filter(void)
struct test_filter_data_t *d = &test_filter_data[i]; struct test_filter_data_t *d = &test_filter_data[i];
int err; int err;
err = test_get_filter(d->filter, &event_ftrace_test_filter, err = create_filter(&event_ftrace_test_filter, d->filter,
&filter); false, &filter);
if (err) { if (err) {
printk(KERN_INFO printk(KERN_INFO
"Failed to get filter for '%s', err %d\n", "Failed to get filter for '%s', err %d\n",
d->filter, err); d->filter, err);
__free_filter(filter);
break; break;
} }
......
...@@ -13,6 +13,9 @@ ...@@ -13,6 +13,9 @@
#include <linux/sysctl.h> #include <linux/sysctl.h>
#include <linux/init.h> #include <linux/init.h>
#include <linux/fs.h> #include <linux/fs.h>
#include <asm/setup.h>
#include "trace.h" #include "trace.h"
#define STACK_TRACE_ENTRIES 500 #define STACK_TRACE_ENTRIES 500
...@@ -133,7 +136,6 @@ stack_trace_call(unsigned long ip, unsigned long parent_ip) ...@@ -133,7 +136,6 @@ stack_trace_call(unsigned long ip, unsigned long parent_ip)
static struct ftrace_ops trace_ops __read_mostly = static struct ftrace_ops trace_ops __read_mostly =
{ {
.func = stack_trace_call, .func = stack_trace_call,
.flags = FTRACE_OPS_FL_GLOBAL,
}; };
static ssize_t static ssize_t
...@@ -311,6 +313,21 @@ static const struct file_operations stack_trace_fops = { ...@@ -311,6 +313,21 @@ static const struct file_operations stack_trace_fops = {
.release = seq_release, .release = seq_release,
}; };
static int
stack_trace_filter_open(struct inode *inode, struct file *file)
{
return ftrace_regex_open(&trace_ops, FTRACE_ITER_FILTER,
inode, file);
}
static const struct file_operations stack_trace_filter_fops = {
.open = stack_trace_filter_open,
.read = seq_read,
.write = ftrace_filter_write,
.llseek = ftrace_regex_lseek,
.release = ftrace_regex_release,
};
int int
stack_trace_sysctl(struct ctl_table *table, int write, stack_trace_sysctl(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, void __user *buffer, size_t *lenp,
...@@ -338,8 +355,13 @@ stack_trace_sysctl(struct ctl_table *table, int write, ...@@ -338,8 +355,13 @@ stack_trace_sysctl(struct ctl_table *table, int write,
return ret; return ret;
} }
static char stack_trace_filter_buf[COMMAND_LINE_SIZE+1] __initdata;
static __init int enable_stacktrace(char *str) static __init int enable_stacktrace(char *str)
{ {
if (strncmp(str, "_filter=", 8) == 0)
strncpy(stack_trace_filter_buf, str+8, COMMAND_LINE_SIZE);
stack_tracer_enabled = 1; stack_tracer_enabled = 1;
last_stack_tracer_enabled = 1; last_stack_tracer_enabled = 1;
return 1; return 1;
...@@ -358,6 +380,12 @@ static __init int stack_trace_init(void) ...@@ -358,6 +380,12 @@ static __init int stack_trace_init(void)
trace_create_file("stack_trace", 0444, d_tracer, trace_create_file("stack_trace", 0444, d_tracer,
NULL, &stack_trace_fops); NULL, &stack_trace_fops);
trace_create_file("stack_trace_filter", 0444, d_tracer,
NULL, &stack_trace_filter_fops);
if (stack_trace_filter_buf[0])
ftrace_set_early_filter(&trace_ops, stack_trace_filter_buf, 1);
if (stack_tracer_enabled) if (stack_tracer_enabled)
register_ftrace_function(&trace_ops); register_ftrace_function(&trace_ops);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment