Commit 56ddca09 authored by Brendan Gregg's avatar Brendan Gregg Committed by GitHub

Merge pull request #763 from goldshtn/enhanced-funccount

funccount: Generalized for uprobes, tracepoints, and USDT
parents ac297c1e 367234ad
.TH funccount 8 "2015-08-18" "USER COMMANDS" .TH funccount 8 "2015-08-18" "USER COMMANDS"
.SH NAME .SH NAME
funccount \- Count kernel function calls matching a pattern. Uses Linux eBPF/bcc. funccount \- Count function, tracepoint, and USDT probe calls matching a pattern. Uses Linux eBPF/bcc.
.SH SYNOPSIS .SH SYNOPSIS
.B funccount [\-h] [\-p PID] [\-i INTERVAL] [\-T] [\-r] pattern .B funccount [\-h] [\-p PID] [\-i INTERVAL] [\-T] [\-r] [\-d] pattern
.SH DESCRIPTION .SH DESCRIPTION
This tool is a quick way to determine which kernel functions are being called, This tool is a quick way to determine which functions are being called,
and at what rate. It uses in-kernel eBPF maps to count function calls. and at what rate. It uses in-kernel eBPF maps to count function calls.
WARNING: This uses dynamic tracing of (what can be many) kernel functions, an WARNING: This uses dynamic tracing of (what can be many) functions, an
activity that has had issues on some kernel versions (risk of panics or activity that has had issues on some kernel versions (risk of panics or
freezes). Test, and know what you are doing, before use. freezes). Test, and know what you are doing, before use.
...@@ -32,6 +32,9 @@ Include timestamps on output. ...@@ -32,6 +32,9 @@ Include timestamps on output.
.TP .TP
\-r \-r
Use regular expressions for the search pattern. Use regular expressions for the search pattern.
.TP
\-d
Print the BPF program before starting (for debugging purposes).
.SH EXAMPLES .SH EXAMPLES
.TP .TP
Count kernel functions beginning with "vfs_", until Ctrl-C is hit: Count kernel functions beginning with "vfs_", until Ctrl-C is hit:
...@@ -53,19 +56,28 @@ Match kernel functions beginning with "vfs_", using regular expressions: ...@@ -53,19 +56,28 @@ Match kernel functions beginning with "vfs_", using regular expressions:
Count vfs calls for process ID 181 only: Count vfs calls for process ID 181 only:
# #
.B funccount \-p 181 'vfs_*' .B funccount \-p 181 'vfs_*'
.SH FIELDS
.TP .TP
ADDR Count calls to the sched_fork tracepoint, indicating a fork() performed:
Address of the instruction pointer that was traced (only useful if the FUNC column is suspicious and you would like to double check the translation). #
.B funccount t:sched:sched_fork
.TP
Count all GC USDT probes in the Node process:
#
.B funccount -p 185 u:node:gc*
.TP
Count all malloc() calls in libc:
#
.B funccount c:malloc
.SH FIELDS
.TP .TP
FUNC FUNC
Kernel function name Function name
.TP .TP
COUNT COUNT
Number of calls while tracing Number of calls while tracing
.SH OVERHEAD .SH OVERHEAD
This traces kernel functions and maintains in-kernel counts, which This traces functions and maintains in-kernel counts, which
are asynchronously copied to user-space. While the rate of kernel calls are asynchronously copied to user-space. While the rate of calls
be very high (>1M/sec), this is a relatively efficient way to trace these be very high (>1M/sec), this is a relatively efficient way to trace these
events, and so the overhead is expected to be small for normal workloads. events, and so the overhead is expected to be small for normal workloads.
Measure in a test environment before use. Measure in a test environment before use.
...@@ -81,6 +93,8 @@ Linux ...@@ -81,6 +93,8 @@ Linux
.SH STABILITY .SH STABILITY
Unstable - in development. Unstable - in development.
.SH AUTHOR .SH AUTHOR
Brendan Gregg Brendan Gregg, Sasha Goldshtein
.SH SEE ALSO .SH SEE ALSO
stackcount(8)
funclatency(8)
vfscount(8) vfscount(8)
...@@ -376,7 +376,8 @@ class BPF(object): ...@@ -376,7 +376,8 @@ class BPF(object):
% (dev, errstr)) % (dev, errstr))
fn.sock = sock fn.sock = sock
def _get_kprobe_functions(self, event_re): @staticmethod
def get_kprobe_functions(event_re):
with open("%s/../kprobes/blacklist" % TRACEFS) as blacklist_file: with open("%s/../kprobes/blacklist" % TRACEFS) as blacklist_file:
blacklist = set([line.rstrip().split()[1] for line in blacklist = set([line.rstrip().split()[1] for line in
blacklist_file]) blacklist_file])
...@@ -386,7 +387,6 @@ class BPF(object): ...@@ -386,7 +387,6 @@ class BPF(object):
fn = line.rstrip().split()[0] fn = line.rstrip().split()[0]
if re.match(event_re, fn) and fn not in blacklist: if re.match(event_re, fn) and fn not in blacklist:
fns.append(fn) fns.append(fn)
self._check_probe_quota(len(fns))
return fns return fns
def _check_probe_quota(self, num_new_probes): def _check_probe_quota(self, num_new_probes):
...@@ -409,7 +409,9 @@ class BPF(object): ...@@ -409,7 +409,9 @@ class BPF(object):
# allow the caller to glob multiple functions together # allow the caller to glob multiple functions together
if event_re: if event_re:
for line in self._get_kprobe_functions(event_re): matches = BPF.get_kprobe_functions(event_re)
self._check_probe_quota(len(matches))
for line in matches:
try: try:
self.attach_kprobe(event=line, fn_name=fn_name, pid=pid, self.attach_kprobe(event=line, fn_name=fn_name, pid=pid,
cpu=cpu, group_fd=group_fd) cpu=cpu, group_fd=group_fd)
...@@ -448,7 +450,7 @@ class BPF(object): ...@@ -448,7 +450,7 @@ class BPF(object):
# allow the caller to glob multiple functions together # allow the caller to glob multiple functions together
if event_re: if event_re:
for line in self._get_kprobe_functions(event_re): for line in BPF.get_kprobe_functions(event_re):
try: try:
self.attach_kretprobe(event=line, fn_name=fn_name, pid=pid, self.attach_kretprobe(event=line, fn_name=fn_name, pid=pid,
cpu=cpu, group_fd=group_fd) cpu=cpu, group_fd=group_fd)
...@@ -531,7 +533,8 @@ class BPF(object): ...@@ -531,7 +533,8 @@ class BPF(object):
res = lib.bcc_procutils_which_so(libname.encode("ascii")) res = lib.bcc_procutils_which_so(libname.encode("ascii"))
return res if res is None else res.decode() return res if res is None else res.decode()
def _get_tracepoints(self, tp_re): @staticmethod
def get_tracepoints(tp_re):
results = [] results = []
events_dir = os.path.join(TRACEFS, "events") events_dir = os.path.join(TRACEFS, "events")
for category in os.listdir(events_dir): for category in os.listdir(events_dir):
...@@ -570,7 +573,7 @@ class BPF(object): ...@@ -570,7 +573,7 @@ class BPF(object):
""" """
if tp_re: if tp_re:
for tp in self._get_tracepoints(tp_re): for tp in BPF.get_tracepoints(tp_re):
self.attach_tracepoint(tp=tp, fn_name=fn_name, pid=pid, self.attach_tracepoint(tp=tp, fn_name=fn_name, pid=pid,
cpu=cpu, group_fd=group_fd) cpu=cpu, group_fd=group_fd)
return return
...@@ -615,7 +618,13 @@ class BPF(object): ...@@ -615,7 +618,13 @@ class BPF(object):
del self.open_uprobes[name] del self.open_uprobes[name]
_num_open_probes -= 1 _num_open_probes -= 1
def _get_user_functions(self, name, sym_re): @staticmethod
def get_user_functions(name, sym_re):
return set([name for (name, _) in
BPF.get_user_functions_and_addresses(name, sym_re)])
@staticmethod
def get_user_addresses(name, sym_re):
""" """
We are returning addresses here instead of symbol names because it We are returning addresses here instead of symbol names because it
turns out that the same name may appear multiple times with different turns out that the same name may appear multiple times with different
...@@ -624,10 +633,15 @@ class BPF(object): ...@@ -624,10 +633,15 @@ class BPF(object):
it makes sense to return the unique set of addresses that are mapped to it makes sense to return the unique set of addresses that are mapped to
a symbol that matches the provided regular expression. a symbol that matches the provided regular expression.
""" """
return set([address for (_, address) in
BPF.get_user_functions_and_addresses(name, sym_re)])
@staticmethod
def get_user_functions_and_addresses(name, sym_re):
addresses = [] addresses = []
def sym_cb(sym_name, addr): def sym_cb(sym_name, addr):
if re.match(sym_re, sym_name) and addr not in addresses: if re.match(sym_re, sym_name):
addresses.append(addr) addresses.append((sym_name, addr))
return 0 return 0
res = lib.bcc_foreach_symbol(name, _SYM_CB_TYPE(sym_cb)) res = lib.bcc_foreach_symbol(name, _SYM_CB_TYPE(sym_cb))
...@@ -660,7 +674,9 @@ class BPF(object): ...@@ -660,7 +674,9 @@ class BPF(object):
name = str(name) name = str(name)
if sym_re: if sym_re:
for sym_addr in self._get_user_functions(name, sym_re): addresses = BPF.get_user_addresses(name, sym_re)
self._check_probe_quota(len(addresses))
for sym_addr in addresses:
self.attach_uprobe(name=name, addr=sym_addr, self.attach_uprobe(name=name, addr=sym_addr,
fn_name=fn_name, pid=pid, cpu=cpu, fn_name=fn_name, pid=pid, cpu=cpu,
group_fd=group_fd) group_fd=group_fd)
...@@ -711,7 +727,7 @@ class BPF(object): ...@@ -711,7 +727,7 @@ class BPF(object):
""" """
if sym_re: if sym_re:
for sym_addr in self._get_user_functions(name, sym_re): for sym_addr in BPF.get_user_addresses(name, sym_re):
self.attach_uretprobe(name=name, addr=sym_addr, self.attach_uretprobe(name=name, addr=sym_addr,
fn_name=fn_name, pid=pid, cpu=cpu, fn_name=fn_name, pid=pid, cpu=cpu,
group_fd=group_fd) group_fd=group_fd)
......
...@@ -67,6 +67,10 @@ class TestProbeQuota(TestCase): ...@@ -67,6 +67,10 @@ class TestProbeQuota(TestCase):
with self.assertRaises(Exception): with self.assertRaises(Exception):
self.b.attach_kprobe(event_re=".*", fn_name="count") self.b.attach_kprobe(event_re=".*", fn_name="count")
def test_uprobe_quota(self):
with self.assertRaises(Exception):
self.b.attach_uprobe(name="c", sym_re=".*", fn_name="count")
def tearDown(self): def tearDown(self):
self.b.cleanup() self.b.cleanup()
......
This diff is collapsed.
This diff is collapsed.
...@@ -96,10 +96,11 @@ class Probe(object): ...@@ -96,10 +96,11 @@ class Probe(object):
pid=self.pid or -1) pid=self.pid or -1)
self.matched = self.bpf.num_open_tracepoints() self.matched = self.bpf.num_open_tracepoints()
elif self.type == "u": elif self.type == "u":
pass # Nothing to do -- attach already happened in `load` pass # Nothing to do -- attach already happened in `load`
if self.matched == 0: if self.matched == 0:
raise Exception("No functions matched by pattern %s" % self.pattern) raise Exception("No functions matched by pattern %s" %
self.pattern)
def load(self): def load(self):
trace_count_text = """ trace_count_text = """
...@@ -142,11 +143,12 @@ BPF_STACK_TRACE(stack_traces, 1024); ...@@ -142,11 +143,12 @@ BPF_STACK_TRACE(stack_traces, 1024);
trace_count_text = trace_count_text.replace('GET_PID', trace_count_text = trace_count_text.replace('GET_PID',
'bpf_get_current_pid_tgid() >> 32') 'bpf_get_current_pid_tgid() >> 32')
else: else:
trace_count_text = trace_count_text.replace('GET_PID', '0xffffffff') trace_count_text = trace_count_text.replace(
'GET_PID', '0xffffffff')
stack_flags = 'BPF_F_REUSE_STACKID' stack_flags = 'BPF_F_REUSE_STACKID'
if not self.is_kernel_probe(): if not self.is_kernel_probe():
stack_flags += '| BPF_F_USER_STACK' # can't do both U *and* K stack_flags += '| BPF_F_USER_STACK' # can't do both U *and* K
trace_count_text = trace_count_text.replace('STACK_FLAGS', stack_flags) trace_count_text = trace_count_text.replace('STACK_FLAGS', stack_flags)
self.usdt = None self.usdt = None
...@@ -173,22 +175,22 @@ BPF_STACK_TRACE(stack_traces, 1024); ...@@ -173,22 +175,22 @@ BPF_STACK_TRACE(stack_traces, 1024);
if debug: if debug:
print(bpf_text) print(bpf_text)
self.bpf = BPF(text=bpf_text, usdt_contexts= self.bpf = BPF(text=bpf_text,
[self.usdt] if self.usdt else []) usdt_contexts=[self.usdt] if self.usdt else [])
class Tool(object): class Tool(object):
def __init__(self): def __init__(self):
examples = """examples: examples = """examples:
./stackcount submit_bio # count kernel stack traces for submit_bio ./stackcount submit_bio # count kernel stack traces for submit_bio
./stackcount -s ip_output # show symbol offsets ./stackcount -s ip_output # show symbol offsets
./stackcount -sv ip_output # show offsets and raw addresses (verbose) ./stackcount -sv ip_output # show offsets and raw addresses (verbose)
./stackcount 'tcp_send*' # count stacks for funcs matching tcp_send* ./stackcount 'tcp_send*' # count stacks for funcs matching tcp_send*
./stackcount -r '^tcp_send.*' # same as above, using regular expressions ./stackcount -r '^tcp_send.*' # same as above, using regular expressions
./stackcount -Ti 5 ip_output # output every 5 seconds, with timestamps ./stackcount -Ti 5 ip_output # output every 5 seconds, with timestamps
./stackcount -p 185 ip_output # count ip_output stacks for PID 185 only ./stackcount -p 185 ip_output # count ip_output stacks for PID 185 only
./stackcount -p 185 c:malloc # count stacks for malloc in PID 185 ./stackcount -p 185 c:malloc # count stacks for malloc in PID 185
./stackcount t:sched:sched_fork # count stacks for the sched_fork tracepoint ./stackcount t:sched:sched_fork # count stacks for sched_fork tracepoint
./stackcount -p 185 u:node:* # count stacks for all USDT probes in node ./stackcount -p 185 u:node:* # count stacks for all USDT probes in node
""" """
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Count events and their stack traces", description="Count events and their stack traces",
...@@ -287,4 +289,3 @@ if __name__ == "__main__": ...@@ -287,4 +289,3 @@ if __name__ == "__main__":
traceback.print_exc() traceback.print_exc()
elif sys.exc_info()[0] is not SystemExit: elif sys.exc_info()[0] is not SystemExit:
print(sys.exc_info()[1]) print(sys.exc_info()[1])
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment