Skip to content

L
linux
Commit f89b30b8 authored by Will Deacon's avatar Will Deacon
Browse files
Options

Merge tag 'trace-direct-v6.3-rc3' of... 

Merge tag 'trace-direct-v6.3-rc3' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace into for-next/ftrace

Pull in ftrace trampoline updates from Steve so that we can implement
support for direct calls for arm64 on top:

tracing: Direct trampoline updates

Updates to the direct trampoline to allow ARM64 to have direct
trampolines.
parents e8d018dd fee86a4e
Hide whitespace changes
Inline Side-by-side
Showing with 101 additions and 491 deletions
arch/s390/kernel/mcount.S
View file @ f89b30b8
......@@ -32,6 +32,11 @@ ENTRY(ftrace_stub)
BR_EX %r14
ENDPROC(ftrace_stub)
SYM_CODE_START(ftrace_stub_direct_tramp)
lgr %r1, %r0
BR_EX %r1
SYM_CODE_END(ftrace_stub_direct_tramp)
.macro ftrace_regs_entry, allregs=0
stg %r14,(__SF_GPRS+8*8)(%r15) # save traced function caller
......
arch/x86/kernel/ftrace_32.S
View file @ f89b30b8
......@@ -163,6 +163,11 @@ SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL)
jmp .Lftrace_ret
SYM_CODE_END(ftrace_regs_caller)
SYM_FUNC_START(ftrace_stub_direct_tramp)
CALL_DEPTH_ACCOUNT
RET
SYM_FUNC_END(ftrace_stub_direct_tramp)
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
SYM_CODE_START(ftrace_graph_caller)
pushl %eax
......
arch/x86/kernel/ftrace_64.S
View file @ f89b30b8
......@@ -309,6 +309,10 @@ SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL)
SYM_FUNC_END(ftrace_regs_caller)
STACK_FRAME_NON_STANDARD_FP(ftrace_regs_caller)
SYM_FUNC_START(ftrace_stub_direct_tramp)
CALL_DEPTH_ACCOUNT
RET
SYM_FUNC_END(ftrace_stub_direct_tramp)
#else /* ! CONFIG_DYNAMIC_FTRACE */
......
include/linux/ftrace.h
View file @ f89b30b8
......@@ -241,6 +241,12 @@ enum {
FTRACE_OPS_FL_DIRECT = BIT(17),
};
#ifndef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
#define FTRACE_OPS_FL_SAVE_ARGS FTRACE_OPS_FL_SAVE_REGS
#else
#define FTRACE_OPS_FL_SAVE_ARGS 0
#endif
/*
* FTRACE_OPS_CMD_* commands allow the ftrace core logic to request changes
* to a ftrace_ops. Note, the requests may fail.
......@@ -321,6 +327,9 @@ struct ftrace_ops {
unsigned long trampoline_size;
struct list_head list;
ftrace_ops_func_t ops_func;
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
unsigned long direct_call;
#endif
#endif
};
......@@ -397,64 +406,36 @@ struct ftrace_func_entry {
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
extern int ftrace_direct_func_count;
int register_ftrace_direct(unsigned long ip, unsigned long addr);
int unregister_ftrace_direct(unsigned long ip, unsigned long addr);
int modify_ftrace_direct(unsigned long ip, unsigned long old_addr, unsigned long new_addr);
struct ftrace_direct_func *ftrace_find_direct_func(unsigned long addr);
int ftrace_modify_direct_caller(struct ftrace_func_entry *entry,
struct dyn_ftrace *rec,
unsigned long old_addr,
unsigned long new_addr);
unsigned long ftrace_find_rec_direct(unsigned long ip);
int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr);
int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr);
int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr);
int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsigned long addr);
int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr);
int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr,
bool free_filters);
int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr);
int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr);
void ftrace_stub_direct_tramp(void);
#else
struct ftrace_ops;
# define ftrace_direct_func_count 0
static inline int register_ftrace_direct(unsigned long ip, unsigned long addr)
{
return -ENOTSUPP;
}
static inline int unregister_ftrace_direct(unsigned long ip, unsigned long addr)
{
return -ENOTSUPP;
}
static inline int modify_ftrace_direct(unsigned long ip,
unsigned long old_addr, unsigned long new_addr)
{
return -ENOTSUPP;
}
static inline struct ftrace_direct_func *ftrace_find_direct_func(unsigned long addr)
{
return NULL;
}
static inline int ftrace_modify_direct_caller(struct ftrace_func_entry *entry,
struct dyn_ftrace *rec,
unsigned long old_addr,
unsigned long new_addr)
{
return -ENODEV;
}
static inline unsigned long ftrace_find_rec_direct(unsigned long ip)
{
return 0;
}
static inline int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
static inline int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
{
return -ENODEV;
}
static inline int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
static inline int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr,
bool free_filters)
{
return -ENODEV;
}
static inline int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
static inline int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
{
return -ENODEV;
}
static inline int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsigned long addr)
static inline int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr)
{
return -ENODEV;
}
......
kernel/bpf/trampoline.c
View file @ f89b30b8
......@@ -45,8 +45,8 @@ static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, enum ftrace_ops_cmd
lockdep_assert_held_once(&tr->mutex);
/* Instead of updating the trampoline here, we propagate
* -EAGAIN to register_ftrace_direct_multi(). Then we can
* retry register_ftrace_direct_multi() after updating the
* -EAGAIN to register_ftrace_direct(). Then we can
* retry register_ftrace_direct() after updating the
* trampoline.
*/
if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) &&
......@@ -198,7 +198,7 @@ static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr)
int ret;
if (tr->func.ftrace_managed)
ret = unregister_ftrace_direct_multi(tr->fops, (long)old_addr);
ret = unregister_ftrace_direct(tr->fops, (long)old_addr, false);
else
ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, NULL);
......@@ -215,9 +215,9 @@ static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_ad
if (tr->func.ftrace_managed) {
if (lock_direct_mutex)
ret = modify_ftrace_direct_multi(tr->fops, (long)new_addr);
ret = modify_ftrace_direct(tr->fops, (long)new_addr);
else
ret = modify_ftrace_direct_multi_nolock(tr->fops, (long)new_addr);
ret = modify_ftrace_direct_nolock(tr->fops, (long)new_addr);
} else {
ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, new_addr);
}
......@@ -243,7 +243,7 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
if (tr->func.ftrace_managed) {
ftrace_set_filter_ip(tr->fops, (unsigned long)ip, 0, 1);
ret = register_ftrace_direct_multi(tr->fops, (long)new_addr);
ret = register_ftrace_direct(tr->fops, (long)new_addr);
} else {
ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, NULL, new_addr);
}
......
kernel/trace/Kconfig
View file @ f89b30b8
......@@ -257,7 +257,7 @@ config DYNAMIC_FTRACE_WITH_REGS
config DYNAMIC_FTRACE_WITH_DIRECT_CALLS
def_bool y
depends on DYNAMIC_FTRACE_WITH_REGS
depends on DYNAMIC_FTRACE_WITH_REGS || DYNAMIC_FTRACE_WITH_ARGS
depends on HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
config DYNAMIC_FTRACE_WITH_CALL_OPS
......
kernel/trace/ftrace.c
View file @ f89b30b8
......@@ -2583,28 +2583,13 @@ ftrace_add_rec_direct(unsigned long ip, unsigned long addr,
static void call_direct_funcs(unsigned long ip, unsigned long pip,
struct ftrace_ops *ops, struct ftrace_regs *fregs)
{
unsigned long addr;
unsigned long addr = READ_ONCE(ops->direct_call);
addr = ftrace_find_rec_direct(ip);
if (!addr)
return;
arch_ftrace_set_direct_caller(fregs, addr);
}
static struct ftrace_ops direct_ops = {
.func = call_direct_funcs,
.flags = FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_REGS
| FTRACE_OPS_FL_PERMANENT,
/*
* By declaring the main trampoline as this trampoline
* it will never have one allocated for it. Allocated
* trampolines should not call direct functions.
* The direct_ops should only be called by the builtin
* ftrace_regs_caller trampoline.
*/
.trampoline = FTRACE_REGS_ADDR,
};
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
/**
......@@ -5301,388 +5286,9 @@ struct ftrace_direct_func {
static LIST_HEAD(ftrace_direct_funcs);
/**
* ftrace_find_direct_func - test an address if it is a registered direct caller
* @addr: The address of a registered direct caller
*
* This searches to see if a ftrace direct caller has been registered
* at a specific address, and if so, it returns a descriptor for it.
*
* This can be used by architecture code to see if an address is
* a direct caller (trampoline) attached to a fentry/mcount location.
* This is useful for the function_graph tracer, as it may need to
* do adjustments if it traced a location that also has a direct
* trampoline attached to it.
*/
struct ftrace_direct_func *ftrace_find_direct_func(unsigned long addr)
{
struct ftrace_direct_func *entry;
bool found = false;
/* May be called by fgraph trampoline (protected by rcu tasks) */
list_for_each_entry_rcu(entry, &ftrace_direct_funcs, next) {
if (entry->addr == addr) {
found = true;
break;
}
}
if (found)
return entry;
return NULL;
}
static struct ftrace_direct_func *ftrace_alloc_direct_func(unsigned long addr)
{
struct ftrace_direct_func *direct;
direct = kmalloc(sizeof(*direct), GFP_KERNEL);
if (!direct)
return NULL;
direct->addr = addr;
direct->count = 0;
list_add_rcu(&direct->next, &ftrace_direct_funcs);
ftrace_direct_func_count++;
return direct;
}
static int register_ftrace_function_nolock(struct ftrace_ops *ops);
/**
* register_ftrace_direct - Call a custom trampoline directly
* @ip: The address of the nop at the beginning of a function
* @addr: The address of the trampoline to call at @ip
*
* This is used to connect a direct call from the nop location (@ip)
* at the start of ftrace traced functions. The location that it calls
* (@addr) must be able to handle a direct call, and save the parameters
* of the function being traced, and restore them (or inject new ones
* if needed), before returning.
*
* Returns:
* 0 on success
* -EBUSY - Another direct function is already attached (there can be only one)
* -ENODEV - @ip does not point to a ftrace nop location (or not supported)
* -ENOMEM - There was an allocation failure.
*/
int register_ftrace_direct(unsigned long ip, unsigned long addr)
{
struct ftrace_direct_func *direct;
struct ftrace_func_entry *entry;
struct ftrace_hash *free_hash = NULL;
struct dyn_ftrace *rec;
int ret = -ENODEV;
mutex_lock(&direct_mutex);
ip = ftrace_location(ip);
if (!ip)
goto out_unlock;
/* See if there's a direct function at @ip already */
ret = -EBUSY;
if (ftrace_find_rec_direct(ip))
goto out_unlock;
ret = -ENODEV;
rec = lookup_rec(ip, ip);
if (!rec)
goto out_unlock;
/*
* Check if the rec says it has a direct call but we didn't
* find one earlier?
*/
if (WARN_ON(rec->flags & FTRACE_FL_DIRECT))
goto out_unlock;
/* Make sure the ip points to the exact record */
if (ip != rec->ip) {
ip = rec->ip;
/* Need to check this ip for a direct. */
if (ftrace_find_rec_direct(ip))
goto out_unlock;
}
ret = -ENOMEM;
direct = ftrace_find_direct_func(addr);
if (!direct) {
direct = ftrace_alloc_direct_func(addr);
if (!direct)
goto out_unlock;
}
entry = ftrace_add_rec_direct(ip, addr, &free_hash);
if (!entry)
goto out_unlock;
ret = ftrace_set_filter_ip(&direct_ops, ip, 0, 0);
if (!ret && !(direct_ops.flags & FTRACE_OPS_FL_ENABLED)) {
ret = register_ftrace_function_nolock(&direct_ops);
if (ret)
ftrace_set_filter_ip(&direct_ops, ip, 1, 0);
}
if (ret) {
remove_hash_entry(direct_functions, entry);
kfree(entry);
if (!direct->count) {
list_del_rcu(&direct->next);
synchronize_rcu_tasks();
kfree(direct);
if (free_hash)
free_ftrace_hash(free_hash);
free_hash = NULL;
ftrace_direct_func_count--;
}
} else {
direct->count++;
}
out_unlock:
mutex_unlock(&direct_mutex);
if (free_hash) {
synchronize_rcu_tasks();
free_ftrace_hash(free_hash);
}
return ret;
}
EXPORT_SYMBOL_GPL(register_ftrace_direct);
static struct ftrace_func_entry *find_direct_entry(unsigned long *ip,
struct dyn_ftrace **recp)
{
struct ftrace_func_entry *entry;
struct dyn_ftrace *rec;
rec = lookup_rec(*ip, *ip);
if (!rec)
return NULL;
entry = __ftrace_lookup_ip(direct_functions, rec->ip);
if (!entry) {
WARN_ON(rec->flags & FTRACE_FL_DIRECT);
return NULL;
}
WARN_ON(!(rec->flags & FTRACE_FL_DIRECT));
/* Passed in ip just needs to be on the call site */
*ip = rec->ip;
if (recp)
*recp = rec;
return entry;
}
int unregister_ftrace_direct(unsigned long ip, unsigned long addr)
{
struct ftrace_direct_func *direct;
struct ftrace_func_entry *entry;
struct ftrace_hash *hash;
int ret = -ENODEV;
mutex_lock(&direct_mutex);
ip = ftrace_location(ip);
if (!ip)
goto out_unlock;
entry = find_direct_entry(&ip, NULL);
if (!entry)
goto out_unlock;
hash = direct_ops.func_hash->filter_hash;
if (hash->count == 1)
unregister_ftrace_function(&direct_ops);
ret = ftrace_set_filter_ip(&direct_ops, ip, 1, 0);
WARN_ON(ret);
remove_hash_entry(direct_functions, entry);
direct = ftrace_find_direct_func(addr);
if (!WARN_ON(!direct)) {
/* This is the good path (see the ! before WARN) */
direct->count--;
WARN_ON(direct->count < 0);
if (!direct->count) {
list_del_rcu(&direct->next);
synchronize_rcu_tasks();
kfree(direct);
kfree(entry);
ftrace_direct_func_count--;
}
}
out_unlock:
mutex_unlock(&direct_mutex);
return ret;
}
EXPORT_SYMBOL_GPL(unregister_ftrace_direct);
static struct ftrace_ops stub_ops = {
.func = ftrace_stub,
};
/**
* ftrace_modify_direct_caller - modify ftrace nop directly
* @entry: The ftrace hash entry of the direct helper for @rec
* @rec: The record representing the function site to patch
* @old_addr: The location that the site at @rec->ip currently calls
* @new_addr: The location that the site at @rec->ip should call
*
* An architecture may overwrite this function to optimize the
* changing of the direct callback on an ftrace nop location.
* This is called with the ftrace_lock mutex held, and no other
* ftrace callbacks are on the associated record (@rec). Thus,
* it is safe to modify the ftrace record, where it should be
* currently calling @old_addr directly, to call @new_addr.
*
* This is called with direct_mutex locked.
*
* Safety checks should be made to make sure that the code at
* @rec->ip is currently calling @old_addr. And this must
* also update entry->direct to @new_addr.
*/
int __weak ftrace_modify_direct_caller(struct ftrace_func_entry *entry,
struct dyn_ftrace *rec,
unsigned long old_addr,
unsigned long new_addr)
{
unsigned long ip = rec->ip;
int ret;
lockdep_assert_held(&direct_mutex);
/*
* The ftrace_lock was used to determine if the record
* had more than one registered user to it. If it did,
* we needed to prevent that from changing to do the quick
* switch. But if it did not (only a direct caller was attached)
* then this function is called. But this function can deal
* with attached callers to the rec that we care about, and
* since this function uses standard ftrace calls that take
* the ftrace_lock mutex, we need to release it.
*/
mutex_unlock(&ftrace_lock);
/*
* By setting a stub function at the same address, we force
* the code to call the iterator and the direct_ops helper.
* This means that @ip does not call the direct call, and
* we can simply modify it.
*/
ret = ftrace_set_filter_ip(&stub_ops, ip, 0, 0);
if (ret)
goto out_lock;
ret = register_ftrace_function_nolock(&stub_ops);
if (ret) {
ftrace_set_filter_ip(&stub_ops, ip, 1, 0);
goto out_lock;
}
entry->direct = new_addr;
/*
* By removing the stub, we put back the direct call, calling
* the @new_addr.
*/
unregister_ftrace_function(&stub_ops);
ftrace_set_filter_ip(&stub_ops, ip, 1, 0);
out_lock:
mutex_lock(&ftrace_lock);
return ret;
}
/**
* modify_ftrace_direct - Modify an existing direct call to call something else
* @ip: The instruction pointer to modify
* @old_addr: The address that the current @ip calls directly
* @new_addr: The address that the @ip should call
*
* This modifies a ftrace direct caller at an instruction pointer without
* having to disable it first. The direct call will switch over to the
* @new_addr without missing anything.
*
* Returns: zero on success. Non zero on error, which includes:
* -ENODEV : the @ip given has no direct caller attached
* -EINVAL : the @old_addr does not match the current direct caller
*/
int modify_ftrace_direct(unsigned long ip,
unsigned long old_addr, unsigned long new_addr)
{
struct ftrace_direct_func *direct, *new_direct = NULL;
struct ftrace_func_entry *entry;
struct dyn_ftrace *rec;
int ret = -ENODEV;
mutex_lock(&direct_mutex);
mutex_lock(&ftrace_lock);
ip = ftrace_location(ip);
if (!ip)
goto out_unlock;
entry = find_direct_entry(&ip, &rec);
if (!entry)
goto out_unlock;
ret = -EINVAL;
if (entry->direct != old_addr)
goto out_unlock;
direct = ftrace_find_direct_func(old_addr);
if (WARN_ON(!direct))
goto out_unlock;
if (direct->count > 1) {
ret = -ENOMEM;
new_direct = ftrace_alloc_direct_func(new_addr);
if (!new_direct)
goto out_unlock;
direct->count--;
new_direct->count++;
} else {
direct->addr = new_addr;
}
/*
* If there's no other ftrace callback on the rec->ip location,
* then it can be changed directly by the architecture.
* If there is another caller, then we just need to change the
* direct caller helper to point to @new_addr.
*/
if (ftrace_rec_count(rec) == 1) {
ret = ftrace_modify_direct_caller(entry, rec, old_addr, new_addr);
} else {
entry->direct = new_addr;
ret = 0;
}
if (unlikely(ret && new_direct)) {
direct->count++;
list_del_rcu(&new_direct->next);
synchronize_rcu_tasks();
kfree(new_direct);
ftrace_direct_func_count--;
}
out_unlock:
mutex_unlock(&ftrace_lock);
mutex_unlock(&direct_mutex);
return ret;
}
EXPORT_SYMBOL_GPL(modify_ftrace_direct);
#define MULTI_FLAGS (FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_REGS)
#define MULTI_FLAGS (FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_ARGS)
static int check_direct_multi(struct ftrace_ops *ops)
{
......@@ -5711,7 +5317,7 @@ static void remove_direct_functions_hash(struct ftrace_hash *hash, unsigned long
}
/**
* register_ftrace_direct_multi - Call a custom trampoline directly
* register_ftrace_direct - Call a custom trampoline directly
* for multiple functions registered in @ops
* @ops: The address of the struct ftrace_ops object
* @addr: The address of the trampoline to call at @ops functions
......@@ -5732,7 +5338,7 @@ static void remove_direct_functions_hash(struct ftrace_hash *hash, unsigned long
* -ENODEV - @ip does not point to a ftrace nop location (or not supported)
* -ENOMEM - There was an allocation failure.
*/
int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
{
struct ftrace_hash *hash, *free_hash = NULL;
struct ftrace_func_entry *entry, *new;
......@@ -5774,6 +5380,7 @@ int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
ops->func = call_direct_funcs;
ops->flags = MULTI_FLAGS;
ops->trampoline = FTRACE_REGS_ADDR;
ops->direct_call = addr;
err = register_ftrace_function_nolock(ops);
......@@ -5790,11 +5397,11 @@ int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
}
return err;
}
EXPORT_SYMBOL_GPL(register_ftrace_direct_multi);
EXPORT_SYMBOL_GPL(register_ftrace_direct);
/**
* unregister_ftrace_direct_multi - Remove calls to custom trampoline
* previously registered by register_ftrace_direct_multi for @ops object.
* unregister_ftrace_direct - Remove calls to custom trampoline
* previously registered by register_ftrace_direct for @ops object.
* @ops: The address of the struct ftrace_ops object
*
* This is used to remove a direct calls to @addr from the nop locations
......@@ -5805,7 +5412,8 @@ EXPORT_SYMBOL_GPL(register_ftrace_direct_multi);
* 0 on success
* -EINVAL - The @ops object was not properly registered.
*/
int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr,
bool free_filters)
{
struct ftrace_hash *hash = ops->func_hash->filter_hash;
int err;
......@@ -5823,12 +5431,15 @@ int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
/* cleanup for possible another register call */
ops->func = NULL;
ops->trampoline = 0;
if (free_filters)
ftrace_free_filter(ops);
return err;
}
EXPORT_SYMBOL_GPL(unregister_ftrace_direct_multi);
EXPORT_SYMBOL_GPL(unregister_ftrace_direct);
static int
__modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
__modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
{
struct ftrace_hash *hash;
struct ftrace_func_entry *entry, *iter;
......@@ -5844,6 +5455,7 @@ __modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
/* Enable the tmp_ops to have the same functions as the direct ops */
ftrace_ops_init(&tmp_ops);
tmp_ops.func_hash = ops->func_hash;
tmp_ops.direct_call = addr;
err = register_ftrace_function_nolock(&tmp_ops);
if (err)
......@@ -5865,6 +5477,8 @@ __modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
entry->direct = addr;
}
}
/* Prevent store tearing if a trampoline concurrently accesses the value */
WRITE_ONCE(ops->direct_call, addr);
mutex_unlock(&ftrace_lock);
......@@ -5875,7 +5489,7 @@ __modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
}
/**
* modify_ftrace_direct_multi_nolock - Modify an existing direct 'multi' call
* modify_ftrace_direct_nolock - Modify an existing direct 'multi' call
* to call something else
* @ops: The address of the struct ftrace_ops object
* @addr: The address of the new trampoline to call at @ops functions
......@@ -5892,19 +5506,19 @@ __modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
* Returns: zero on success. Non zero on error, which includes:
* -EINVAL - The @ops object was not properly registered.
*/
int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsigned long addr)
int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr)
{
if (check_direct_multi(ops))
return -EINVAL;
if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
return -EINVAL;
return __modify_ftrace_direct_multi(ops, addr);
return __modify_ftrace_direct(ops, addr);
}
EXPORT_SYMBOL_GPL(modify_ftrace_direct_multi_nolock);
EXPORT_SYMBOL_GPL(modify_ftrace_direct_nolock);
/**
* modify_ftrace_direct_multi - Modify an existing direct 'multi' call
* modify_ftrace_direct - Modify an existing direct 'multi' call
* to call something else
* @ops: The address of the struct ftrace_ops object
* @addr: The address of the new trampoline to call at @ops functions
......@@ -5918,7 +5532,7 @@ EXPORT_SYMBOL_GPL(modify_ftrace_direct_multi_nolock);
* Returns: zero on success. Non zero on error, which includes:
* -EINVAL - The @ops object was not properly registered.
*/
int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
{
int err;
......@@ -5928,11 +5542,11 @@ int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
return -EINVAL;
mutex_lock(&direct_mutex);
err = __modify_ftrace_direct_multi(ops, addr);
err = __modify_ftrace_direct(ops, addr);
mutex_unlock(&direct_mutex);
return err;
}
EXPORT_SYMBOL_GPL(modify_ftrace_direct_multi);
EXPORT_SYMBOL_GPL(modify_ftrace_direct);
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
/**
......
kernel/trace/trace_selftest.c
View file @ f89b30b8
......@@ -785,14 +785,7 @@ static struct fgraph_ops fgraph_ops __initdata = {
};
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
#ifndef CALL_DEPTH_ACCOUNT
#define CALL_DEPTH_ACCOUNT ""
#endif
noinline __noclone static void trace_direct_tramp(void)
{
asm(CALL_DEPTH_ACCOUNT);
}
static struct ftrace_ops direct;
#endif
/*
......@@ -870,8 +863,9 @@ trace_selftest_startup_function_graph(struct tracer *trace,
* Register direct function together with graph tracer
* and make sure we get graph trace.
*/
ret = register_ftrace_direct((unsigned long) DYN_FTRACE_TEST_NAME,
(unsigned long) trace_direct_tramp);
ftrace_set_filter_ip(&direct, (unsigned long)DYN_FTRACE_TEST_NAME, 0, 0);
ret = register_ftrace_direct(&direct,
(unsigned long)ftrace_stub_direct_tramp);
if (ret)
goto out;
......@@ -891,8 +885,9 @@ trace_selftest_startup_function_graph(struct tracer *trace,
unregister_ftrace_graph(&fgraph_ops);
ret = unregister_ftrace_direct((unsigned long) DYN_FTRACE_TEST_NAME,
(unsigned long) trace_direct_tramp);
ret = unregister_ftrace_direct(&direct,
(unsigned long)ftrace_stub_direct_tramp,
true);
if (ret)
goto out;
......
samples/Kconfig
View file @ f89b30b8
......@@ -38,7 +38,7 @@ config SAMPLE_FTRACE_DIRECT
that hooks to wake_up_process and prints the parameters.
config SAMPLE_FTRACE_DIRECT_MULTI
tristate "Build register_ftrace_direct_multi() example"
tristate "Build register_ftrace_direct() on multiple ips example"
depends on DYNAMIC_FTRACE_WITH_DIRECT_CALLS && m
depends on HAVE_SAMPLE_FTRACE_DIRECT_MULTI
help
......
samples/ftrace/ftrace-direct-modify.c
View file @ f89b30b8
......@@ -96,6 +96,8 @@ asm (
#endif /* CONFIG_S390 */
static struct ftrace_ops direct;
static unsigned long my_tramp = (unsigned long)my_tramp1;
static unsigned long tramps[2] = {
(unsigned long)my_tramp1,
......@@ -114,7 +116,7 @@ static int simple_thread(void *arg)
if (ret)
continue;
t ^= 1;
ret = modify_ftrace_direct(my_ip, my_tramp, tramps[t]);
ret = modify_ftrace_direct(&direct, tramps[t]);
if (!ret)
my_tramp = tramps[t];
WARN_ON_ONCE(ret);
......@@ -129,7 +131,9 @@ static int __init ftrace_direct_init(void)
{
int ret;
ret = register_ftrace_direct(my_ip, my_tramp);
ftrace_set_filter_ip(&direct, (unsigned long) my_ip, 0, 0);
ret = register_ftrace_direct(&direct, my_tramp);
if (!ret)
simple_tsk = kthread_run(simple_thread, NULL, "event-sample-fn");
return ret;
......@@ -138,7 +142,7 @@ static int __init ftrace_direct_init(void)
static void __exit ftrace_direct_exit(void)
{
kthread_stop(simple_tsk);
unregister_ftrace_direct(my_ip, my_tramp);
unregister_ftrace_direct(&direct, my_tramp, true);
}
module_init(ftrace_direct_init);
......
samples/ftrace/ftrace-direct-multi-modify.c
View file @ f89b30b8
......@@ -123,7 +123,7 @@ static int simple_thread(void *arg)
if (ret)
continue;
t ^= 1;
ret = modify_ftrace_direct_multi(&direct, tramps[t]);
ret = modify_ftrace_direct(&direct, tramps[t]);
if (!ret)
my_tramp = tramps[t];
WARN_ON_ONCE(ret);
......@@ -141,7 +141,7 @@ static int __init ftrace_direct_multi_init(void)
ftrace_set_filter_ip(&direct, (unsigned long) wake_up_process, 0, 0);
ftrace_set_filter_ip(&direct, (unsigned long) schedule, 0, 0);
ret = register_ftrace_direct_multi(&direct, my_tramp);
ret = register_ftrace_direct(&direct, my_tramp);
if (!ret)
simple_tsk = kthread_run(simple_thread, NULL, "event-sample-fn");
......@@ -151,13 +151,12 @@ static int __init ftrace_direct_multi_init(void)
static void __exit ftrace_direct_multi_exit(void)
{
kthread_stop(simple_tsk);
unregister_ftrace_direct_multi(&direct, my_tramp);
ftrace_free_filter(&direct);
unregister_ftrace_direct(&direct, my_tramp, true);
}
module_init(ftrace_direct_multi_init);
module_exit(ftrace_direct_multi_exit);
MODULE_AUTHOR("Jiri Olsa");
MODULE_DESCRIPTION("Example use case of using modify_ftrace_direct_multi()");
MODULE_DESCRIPTION("Example use case of using modify_ftrace_direct()");
MODULE_LICENSE("GPL");
samples/ftrace/ftrace-direct-multi.c
View file @ f89b30b8
......@@ -73,13 +73,12 @@ static int __init ftrace_direct_multi_init(void)
ftrace_set_filter_ip(&direct, (unsigned long) wake_up_process, 0, 0);
ftrace_set_filter_ip(&direct, (unsigned long) schedule, 0, 0);
return register_ftrace_direct_multi(&direct, (unsigned long) my_tramp);
return register_ftrace_direct(&direct, (unsigned long) my_tramp);
}
static void __exit ftrace_direct_multi_exit(void)
{
unregister_ftrace_direct_multi(&direct, (unsigned long) my_tramp);
ftrace_free_filter(&direct);
unregister_ftrace_direct(&direct, (unsigned long) my_tramp, true);
}
module_init(ftrace_direct_multi_init);
......
samples/ftrace/ftrace-direct-too.c
View file @ f89b30b8
......@@ -70,16 +70,18 @@ asm (
#endif /* CONFIG_S390 */
static struct ftrace_ops direct;
static int __init ftrace_direct_init(void)
{
return register_ftrace_direct((unsigned long)handle_mm_fault,
(unsigned long)my_tramp);
ftrace_set_filter_ip(&direct, (unsigned long) handle_mm_fault, 0, 0);
return register_ftrace_direct(&direct, (unsigned long) my_tramp);
}
static void __exit ftrace_direct_exit(void)
{
unregister_ftrace_direct((unsigned long)handle_mm_fault,
(unsigned long)my_tramp);
unregister_ftrace_direct(&direct, (unsigned long)my_tramp, true);
}
module_init(ftrace_direct_init);
......
samples/ftrace/ftrace-direct.c
View file @ f89b30b8
......@@ -63,16 +63,18 @@ asm (
#endif /* CONFIG_S390 */
static struct ftrace_ops direct;
static int __init ftrace_direct_init(void)
{
return register_ftrace_direct((unsigned long)wake_up_process,
(unsigned long)my_tramp);
ftrace_set_filter_ip(&direct, (unsigned long) wake_up_process, 0, 0);
return register_ftrace_direct(&direct, (unsigned long) my_tramp);
}
static void __exit ftrace_direct_exit(void)
{
unregister_ftrace_direct((unsigned long)wake_up_process,
(unsigned long)my_tramp);
unregister_ftrace_direct(&direct, (unsigned long)my_tramp, true);
}
module_init(ftrace_direct_init);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7