bcc/syms: Fix shared library symbol lookup

Shared library addresses needed to be mapped what the address is expected in the
symbol table. The address offset of the running shared library may be different from
the one in the SO binary file. So we have to map it correctly in order for symbol
look up to work.

Often the address and file offset are the same so it works, however in Android this
is not the case a lot of times. Fix the issue by adjusting the offset with the
file offset from the ELF.

Test: stackcount 'sys_futex' -v

Before patch:
  ffffff800814bc14 sys_futex
  775d8e0db0       syscall
  775e149fe8       [unknown]
  775e14a598       [unknown]
  775e16a9e4       android::Parcel::validateReadData(unsigned long) const
  775e05d738       android::Thread::_threadLoop(void*)
  775d942470       __pthread_start(void*)
  775d8e54f8       __start_thread
    Binder:13353_5 [13353]

After patch:
  ffffff800814bc14 sys_futex
  78ba2e1db0       syscall
  78b97d4fe8       android::IPCThreadState::getAndExecuteCommand()
  78b97d5598       android::IPCThreadState::joinThreadPool(bool)
  78b97f59e4       [unknown]
  78b8dda738       android::Thread::_threadLoop(void*)
  78ba4ffbd0       android::AndroidRuntime::javaThreadShell(void*)
  78ba343470       __pthread_start(void*)
  78ba2e64f8       __start_thread
    Binder:14001_8 [14001]

Matches Perf output:
        ffffff800814bc14 sys_futex ([kernel.kallsyms])
                   1edb0 syscall (/system/lib64/libc.so)
                   45fe8 android::IPCThreadState::getAndExecuteCommand (/system/lib64/libbinder.so)
                   46598 android::IPCThreadState::joinThreadPool (/system/lib64/libbinder.so)
                   669e4 [unknown] (/system/lib64/libbinder.so)
                   10738 android::Thread::_threadLoop (/system/lib64/libutils.so)
                   7ebd0 android::AndroidRuntime::javaThreadShell (/system/lib64/libandroid_runtime.so)
                   80470 __pthread_start (/system/lib64/libc.so)
                   234f8 __start_thread (/system/lib64/libc.so)

Fixes: https://github.com/iovisor/bcc/issues/1628Signed-off-by: Joel Fernandes <joelaf@google.com>
Signed-off-by: Joel Fernandes <joel@linuxinternals.org>

src/cc/bcc_elf.c

@@ -500,6 +500,41 @@ int bcc_elf_foreach_sym(const char *path, bcc_elf_symcb callback,
       path, callback, (struct bcc_symbol_option*)option, payload, 0);
 }
 
+int bcc_elf_get_text_scn_info(const char *path, uint64_t *addr,
+				   uint64_t *offset) {
+  Elf *e = NULL;
+  int fd = -1, err;
+  Elf_Scn *section = NULL;
+  GElf_Shdr header;
+  size_t stridx;
+  char *name;
+
+  if ((err = openelf(path, &e, &fd)) < 0 ||
+      (err = elf_getshdrstrndx(e, &stridx)) < 0)
+    goto exit;
+
+  err = -1;
+  while ((section = elf_nextscn(e, section)) != 0) {
+    if (!gelf_getshdr(section, &header))
+      continue;
+
+    name = elf_strptr(e, stridx, header.sh_name);
+    if (name && !strcmp(name, ".text")) {
+      *addr = (uint64_t)header.sh_addr;
+      *offset = (uint64_t)header.sh_offset;
+      err = 0;
+      break;
+    }
+  }
+
+exit:
+  if (e)
+    elf_end(e);
+  if (fd >= 0)
+    close(fd);
+  return err;
+}
+
 int bcc_elf_foreach_load_section(const char *path,
                                  bcc_elf_load_sectioncb callback,
                                  void *payload) {

src/cc/bcc_elf.h

@@ -61,6 +61,9 @@ int bcc_elf_foreach_sym(const char *path, bcc_elf_symcb callback, void *option,
 // Returns -1 on error, and 0 on success or stopped by callback
 int bcc_elf_foreach_vdso_sym(bcc_elf_symcb callback, void *payload);
 
+int bcc_elf_get_text_scn_info(const char *path, uint64_t *addr,
+                              uint64_t *offset);
+
 int bcc_elf_get_type(const char *path);
 int bcc_elf_is_shared_obj(const char *path);
 int bcc_elf_is_exe(const char *path);

src/cc/bcc_syms.cc

@@ -158,6 +158,18 @@ int ProcSyms::_add_module(const char *modname, uint64_t start, uint64_t end,
     auto module = Module(
         modname, check_mount_ns ? ps->mount_ns_instance_.get() : nullptr,
         &ps->symbol_option_);
+
+    // pid/maps doesn't account for file_offset of text within the ELF.
+    // It only gives the mmap offset. We need the real offset for symbol
+    // lookup.
+    if (module.type_ == ModuleType::SO) {
+      if (bcc_elf_get_text_scn_info(modname, &module.elf_so_addr_,
+                                    &module.elf_so_offset_) < 0) {
+        fprintf(stderr, "WARNING: Couldn't find .text section in %s\n", modname);
+        fprintf(stderr, "WARNING: BCC can't handle sym look ups for %s", modname);
+      }
+    }
+
     if (!bcc_is_perf_map(modname) || module.type_ != ModuleType::UNKNOWN)
       // Always add the module even if we can't read it, so that we could
       // report correct module name. Unless it's a perf map that we only
@@ -251,6 +263,10 @@ ProcSyms::Module::Module(const char *name, ProcMountNS *mount_ns,
     type_ = ModuleType::PERF_MAP;
   else if (bcc_elf_is_vdso(name_.c_str()) == 1)
     type_ = ModuleType::VDSO;
+
+  // Will be stored later
+  elf_so_offset_ = 0;
+  elf_so_addr_ = 0;
 }
 
 int ProcSyms::Module::_add_symbol(const char *symname, uint64_t start,
@@ -282,13 +298,22 @@ void ProcSyms::Module::load_sym_table() {
 }
 
 bool ProcSyms::Module::contains(uint64_t addr, uint64_t &offset) const {
-  for (const auto &range : ranges_)
+  for (const auto &range : ranges_) {
     if (addr >= range.start && addr < range.end) {
-      offset = (type_ == ModuleType::SO || type_ == ModuleType::VDSO)
-                   ? (addr - range.start + range.file_offset)
-                   : addr;
+      if (type_ == ModuleType::SO || type_ == ModuleType::VDSO) {
+        // Offset within the mmap
+        offset = addr - range.start + range.file_offset;
+
+        // Offset within the ELF for SO symbol lookup
+        offset += (elf_so_addr_ - elf_so_offset_);
+      } else {
+        offset = addr;
+      }
+
       return true;
     }
+  }
+
   return false;
 }
 

src/cc/syms.h

@@ -108,6 +108,10 @@ class ProcSyms : SymbolCache {
     bcc_symbol_option *symbol_option_;
     ModuleType type_;
 
+    // The file offset within the ELF of the SO's first text section.
+    uint64_t elf_so_offset_;
+    uint64_t elf_so_addr_;
+
     std::unordered_set<std::string> symnames_;
     std::vector<Symbol> syms_;