Add TableStorage class for wrapping bpf map tracking

Adds a TableStorage class for use by language frontends to store/access
references to loaded bpf maps. Includes support for shared and
namespaced maps, in a directory-like hierarchy.

Add a FileDesc helper class to automatically wrap open file descriptors.
The object prevents implicit copying of the fd (allows only
rvalue/move()), and takes care of the close() call.

Add a reference implementation of a TableStorageImpl that performs the
current default behavior expected by BPF_TABLE_PUBLIC, which is to share
maps between BPFModules in the same-process only. A stub implementation
for bpffs is started.

Update b/clang frontends to use this new class.

Also included is a framework for extracting the type information of maps
in an extensible way. Migrate BMapDeclVisitor to use this as the first
consumer.
Signed-off-by: Brenden Blanco <bblanco@gmail.com>

.clang-format

@@ -3,4 +3,4 @@ BasedOnStyle: Google
 AllowShortIfStatementsOnASingleLine: false
 AllowShortLoopsOnASingleLine: false
 IndentCaseLabels: false
-AccessModifierOffset: -2
+AccessModifierOffset: -1

src/cc/BPF.cc

@@ -28,9 +28,10 @@
 #include "bcc_exception.h"
 #include "bcc_syms.h"
 #include "bpf_module.h"
+#include "common.h"
 #include "libbpf.h"
 #include "perf_reader.h"
-#include "common.h"
+#include "table_storage.h"
 #include "usdt.h"
 
 #include "BPF.h"

src/cc/BPF.h

@@ -26,6 +26,7 @@
 #include "bpf_module.h"
 #include "compat/linux/bpf.h"
 #include "libbpf.h"
+#include "table_storage.h"
 
 static const int DEFAULT_PERF_BUFFER_PAGE_CNT = 8;
 
@@ -38,12 +39,14 @@ struct open_probe_t {
 };
 
 class USDT;
+class TableStorage;
 
 class BPF {
 public:
   static const int BPF_MAX_STACK_DEPTH = 127;
 
-  explicit BPF(unsigned int flag = 0) : bpf_module_(new BPFModule(flag)) {}
+  explicit BPF(unsigned int flag = 0, TableStorage* ts = nullptr)
+      : bpf_module_(new BPFModule(flag, ts)), ts_(ts) {}
   StatusTuple init(const std::string& bpf_program,
                    std::vector<std::string> cflags = {},
                    std::vector<USDT> usdt = {});
@@ -90,6 +93,11 @@ public:
 
   template <class KeyType, class ValueType>
   BPFHashTable<KeyType, ValueType> get_hash_table(const std::string& name) {
+    if (ts_) {
+      TableStorage::iterator it;
+      if (ts_->Find(Path({name}), it))
+        return BPFHashTable<KeyType, ValueType>(it->second);
+    }
     return BPFHashTable<KeyType, ValueType>(bpf_module_.get(), name);
   }
 
@@ -152,6 +160,7 @@ private:
                                   uint64_t symbol_addr, bcc_symbol* output);
 
   std::unique_ptr<BPFModule> bpf_module_;
+  TableStorage* ts_;
 
   std::map<std::string, int> funcs_;
 

src/cc/BPFTable.h

@@ -28,6 +28,7 @@
 #include "bpf_module.h"
 #include "libbpf.h"
 #include "perf_reader.h"
+#include "table_desc.h"
 
 namespace ebpf {
 
@@ -44,6 +45,10 @@ protected:
     fd_ = bpf_module->table_fd(id_);
     capacity_ = bpf_module->table_max_entries(id_);
   };
+  explicit BPFTableBase(const TableDesc& desc) {
+    fd_ = desc.fd;
+    capacity_ = desc.max_entries;
+  }
 
   bool lookup(KeyType* key, ValueType* value) {
     return bpf_lookup_elem(fd_, static_cast<void*>(key),
@@ -70,7 +75,8 @@ protected:
 
 template <class KeyType, class ValueType>
 class BPFHashTable : protected BPFTableBase<KeyType, ValueType> {
-public:
+ public:
+  explicit BPFHashTable(const TableDesc& desc) : BPFTableBase<KeyType, ValueType>(desc) {}
   BPFHashTable(BPFModule* bpf_module, const std::string& name)
       : BPFTableBase<KeyType, ValueType>(bpf_module, name) {}
 

src/cc/CMakeLists.txt

@@ -35,12 +35,12 @@ if (CMAKE_COMPILER_IS_GNUCC AND LIBCLANG_ISSTATIC)
   endif()
 endif()
 
-add_library(bcc-shared SHARED bpf_common.cc bpf_module.cc libbpf.c perf_reader.c shared_table.cc exported_files.cc bcc_elf.c bcc_perf_map.c bcc_proc.c bcc_syms.cc usdt_args.cc usdt.cc common.cc BPF.cc BPFTable.cc)
+add_library(bcc-shared SHARED bpf_common.cc bpf_module.cc libbpf.c perf_reader.c table_storage.cc shared_table.cc bpffs_table.cc json_map_decl_visitor.cc exported_files.cc bcc_elf.c bcc_perf_map.c bcc_proc.c bcc_syms.cc usdt_args.cc usdt.cc common.cc BPF.cc BPFTable.cc)
 set_target_properties(bcc-shared PROPERTIES VERSION ${REVISION_LAST} SOVERSION 0)
 set_target_properties(bcc-shared PROPERTIES OUTPUT_NAME bcc)
 
 add_library(bcc-loader-static STATIC libbpf.c perf_reader.c bcc_elf.c bcc_perf_map.c bcc_proc.c)
-add_library(bcc-static STATIC bpf_common.cc bpf_module.cc shared_table.cc exported_files.cc bcc_syms.cc usdt_args.cc usdt.cc common.cc BPF.cc BPFTable.cc)
+add_library(bcc-static STATIC bpf_common.cc bpf_module.cc shared_table.cc bpffs_table.cc json_map_decl_visitor.cc table_storage.cc exported_files.cc bcc_syms.cc usdt_args.cc usdt.cc common.cc BPF.cc BPFTable.cc)
 set_target_properties(bcc-static PROPERTIES OUTPUT_NAME bcc)
 
 set(llvm_raw_libs bitwriter bpfcodegen irreader linker
@@ -67,7 +67,7 @@ target_link_libraries(bcc-static b_frontend clang_frontend bcc-loader-static ${c
 
 install(TARGETS bcc-shared LIBRARY COMPONENT libbcc
   DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES bpf_common.h bpf_module.h bcc_syms.h bcc_exception.h libbpf.h perf_reader.h BPF.h BPFTable.h shared_table.h COMPONENT libbcc
+install(FILES bpf_common.h bpf_module.h bcc_syms.h bcc_exception.h libbpf.h perf_reader.h BPF.h BPFTable.h shared_table.h table_desc.h COMPONENT libbcc
   DESTINATION include/bcc)
 install(DIRECTORY compat/linux/ COMPONENT libbcc
   DESTINATION include/bcc/compat/linux

src/cc/bpf_module.cc

@@ -101,8 +101,8 @@ class MyMemoryManager : public SectionMemoryManager {
   map<string, tuple<uint8_t *, uintptr_t>> *sections_;
 };
 
-BPFModule::BPFModule(unsigned flags)
-    : flags_(flags), ctx_(new LLVMContext) {
+BPFModule::BPFModule(unsigned flags, TableStorage *ts)
+    : flags_(flags), ctx_(new LLVMContext), ts_(ts) {
   InitializeNativeTarget();
   InitializeNativeTargetAsmPrinter();
   LLVMInitializeBPFTarget();
@@ -110,21 +110,17 @@ BPFModule::BPFModule(unsigned flags)
   LLVMInitializeBPFTargetInfo();
   LLVMInitializeBPFAsmPrinter();
   LLVMLinkInMCJIT(); /* call empty function to force linking of MCJIT */
+  if (!ts_) {
+    local_ts_ = createSharedTableStorage();
+    ts_ = &*local_ts_;
+  }
 }
 
 BPFModule::~BPFModule() {
   engine_.reset();
   rw_engine_.reset();
   ctx_.reset();
-  if (tables_) {
-    for (auto table : *tables_) {
-      if (table.is_shared) {
-        SharedTables::instance()->remove_fd(table.name);
-      } else if (!table.is_extern) {
-        close(table.fd);
-      }
-    }
-  }
+  ts_->DeletePrefix(Path({std::to_string((uintptr_t)this)}));
 }
 
 static void debug_printf(Module *mod, IRBuilder<> &B, const string &fmt, vector<Value *> args) {
@@ -326,7 +322,8 @@ unique_ptr<ExecutionEngine> BPFModule::finalize_rw(unique_ptr<Module> m) {
 // load an entire c file as a module
 int BPFModule::load_cfile(const string &file, bool in_memory, const char *cflags[], int ncflags) {
   clang_loader_ = make_unique<ClangLoader>(&*ctx_, flags_);
-  if (clang_loader_->parse(&mod_, &tables_, file, in_memory, cflags, ncflags))
+  if (clang_loader_->parse(&mod_, *ts_, file, in_memory, cflags, ncflags,
+                           std::to_string((uintptr_t)this)))
     return -1;
   return 0;
 }
@@ -338,7 +335,7 @@ int BPFModule::load_cfile(const string &file, bool in_memory, const char *cflags
 // build an ExecutionEngine.
 int BPFModule::load_includes(const string &text) {
   clang_loader_ = make_unique<ClangLoader>(&*ctx_, flags_);
-  if (clang_loader_->parse(&mod_, &tables_, text, true, nullptr, 0))
+  if (clang_loader_->parse(&mod_, *ts_, text, true, nullptr, 0, ""))
     return -1;
   return 0;
 }
@@ -352,7 +349,10 @@ int BPFModule::annotate() {
   auto m = make_unique<Module>("sscanf", *ctx_);
 
   size_t id = 0;
-  for (auto &table : *tables_) {
+  Path path({std::to_string((uintptr_t)this)});
+  for (auto it = ts_->lower_bound(path), up = ts_->upper_bound(path); it != up; ++it) {
+    TableDesc &table = it->second;
+    tables_.push_back(&it->second);
     table_names_[table.name] = id++;
     GlobalValue *gvar = mod_->getNamedValue(table.name);
     if (!gvar) continue;
@@ -507,9 +507,7 @@ unsigned BPFModule::kern_version() const {
   return *(unsigned *)get<0>(section->second);
 }
 
-size_t BPFModule::num_tables() const {
-  return tables_->size();
-}
+size_t BPFModule::num_tables() const { return tables_.size(); }
 
 size_t BPFModule::table_id(const string &name) const {
   auto it = table_names_.find(name);
@@ -522,8 +520,9 @@ int BPFModule::table_fd(const string &name) const {
 }
 
 int BPFModule::table_fd(size_t id) const {
-  if (id >= tables_->size()) return -1;
-  return (*tables_)[id].fd;
+  if (id >= tables_.size())
+    return -1;
+  return tables_[id]->fd;
 }
 
 int BPFModule::table_type(const string &name) const {
@@ -531,8 +530,9 @@ int BPFModule::table_type(const string &name) const {
 }
 
 int BPFModule::table_type(size_t id) const {
-  if (id >= tables_->size()) return -1;
-  return (*tables_)[id].type;
+  if (id >= tables_.size())
+    return -1;
+  return tables_[id]->type;
 }
 
 size_t BPFModule::table_max_entries(const string &name) const {
@@ -540,8 +540,9 @@ size_t BPFModule::table_max_entries(const string &name) const {
 }
 
 size_t BPFModule::table_max_entries(size_t id) const {
-  if (id >= tables_->size()) return 0;
-  return (*tables_)[id].max_entries;
+  if (id >= tables_.size())
+    return 0;
+  return tables_[id]->max_entries;
 }
 
 int BPFModule::table_flags(const string &name) const {
@@ -549,19 +550,22 @@ int BPFModule::table_flags(const string &name) const {
 }
 
 int BPFModule::table_flags(size_t id) const {
-  if (id >= tables_->size()) return -1;
-  return (*tables_)[id].flags;
+  if (id >= tables_.size())
+    return -1;
+  return tables_[id]->flags;
 }
 
 const char * BPFModule::table_name(size_t id) const {
-  if (id >= tables_->size()) return nullptr;
-  return (*tables_)[id].name.c_str();
+  if (id >= tables_.size())
+    return nullptr;
+  return tables_[id]->name.c_str();
 }
 
 const char * BPFModule::table_key_desc(size_t id) const {
   if (b_loader_) return nullptr;
-  if (id >= tables_->size()) return nullptr;
-  return (*tables_)[id].key_desc.c_str();
+  if (id >= tables_.size())
+    return nullptr;
+  return tables_[id]->key_desc.c_str();
 }
 
 const char * BPFModule::table_key_desc(const string &name) const {
@@ -570,24 +574,27 @@ const char * BPFModule::table_key_desc(const string &name) const {
 
 const char * BPFModule::table_leaf_desc(size_t id) const {
   if (b_loader_) return nullptr;
-  if (id >= tables_->size()) return nullptr;
-  return (*tables_)[id].leaf_desc.c_str();
+  if (id >= tables_.size())
+    return nullptr;
+  return tables_[id]->leaf_desc.c_str();
 }
 
 const char * BPFModule::table_leaf_desc(const string &name) const {
   return table_leaf_desc(table_id(name));
 }
 size_t BPFModule::table_key_size(size_t id) const {
-  if (id >= tables_->size()) return 0;
-  return (*tables_)[id].key_size;
+  if (id >= tables_.size())
+    return 0;
+  return tables_[id]->key_size;
 }
 size_t BPFModule::table_key_size(const string &name) const {
   return table_key_size(table_id(name));
 }
 
 size_t BPFModule::table_leaf_size(size_t id) const {
-  if (id >= tables_->size()) return 0;
-  return (*tables_)[id].leaf_size;
+  if (id >= tables_.size())
+    return 0;
+  return tables_[id]->leaf_size;
 }
 size_t BPFModule::table_leaf_size(const string &name) const {
   return table_leaf_size(table_id(name));
@@ -603,8 +610,9 @@ struct TableIterator {
 };
 
 int BPFModule::table_key_printf(size_t id, char *buf, size_t buflen, const void *key) {
-  if (id >= tables_->size()) return -1;
-  const TableDesc &desc = (*tables_)[id];
+  if (id >= tables_.size())
+    return -1;
+  const TableDesc &desc = *tables_[id];
   if (!desc.key_snprintf) {
     fprintf(stderr, "Key snprintf not available\n");
     return -1;
@@ -627,8 +635,9 @@ int BPFModule::table_key_printf(size_t id, char *buf, size_t buflen, const void
 }
 
 int BPFModule::table_leaf_printf(size_t id, char *buf, size_t buflen, const void *leaf) {
-  if (id >= tables_->size()) return -1;
-  const TableDesc &desc = (*tables_)[id];
+  if (id >= tables_.size())
+    return -1;
+  const TableDesc &desc = *tables_[id];
   if (!desc.leaf_snprintf) {
     fprintf(stderr, "Key snprintf not available\n");
     return -1;
@@ -651,8 +660,9 @@ int BPFModule::table_leaf_printf(size_t id, char *buf, size_t buflen, const void
 }
 
 int BPFModule::table_key_scanf(size_t id, const char *key_str, void *key) {
-  if (id >= tables_->size()) return -1;
-  const TableDesc &desc = (*tables_)[id];
+  if (id >= tables_.size())
+    return -1;
+  const TableDesc &desc = *tables_[id];
   if (!desc.key_sscanf) {
     fprintf(stderr, "Key sscanf not available\n");
     return -1;
@@ -672,8 +682,9 @@ int BPFModule::table_key_scanf(size_t id, const char *key_str, void *key) {
 }
 
 int BPFModule::table_leaf_scanf(size_t id, const char *leaf_str, void *leaf) {
-  if (id >= tables_->size()) return -1;
-  const TableDesc &desc = (*tables_)[id];
+  if (id >= tables_.size())
+    return -1;
+  const TableDesc &desc = *tables_[id];
   if (!desc.leaf_sscanf) {
     fprintf(stderr, "Key sscanf not available\n");
     return -1;
@@ -714,7 +725,8 @@ int BPFModule::load_b(const string &filename, const string &proto_filename) {
     return rc;
 
   b_loader_.reset(new BLoader(flags_));
-  if (int rc = b_loader_->parse(&*mod_, filename, proto_filename, &tables_))
+  if (int rc =
+          b_loader_->parse(&*mod_, filename, proto_filename, *ts_, std::to_string((uintptr_t)this)))
     return rc;
   if (int rc = annotate())
     return rc;

src/cc/bpf_module.h

@@ -31,7 +31,8 @@ class Type;
 }
 
 namespace ebpf {
-struct TableDesc;
+class TableDesc;
+class TableStorage;
 class BLoader;
 class ClangLoader;
 
@@ -52,7 +53,7 @@ class BPFModule {
   int kbuild_flags(const char *uname_release, std::vector<std::string> *cflags);
   int run_pass_manager(llvm::Module &mod);
  public:
-  BPFModule(unsigned flags);
+  BPFModule(unsigned flags, TableStorage *ts = nullptr);
   ~BPFModule();
   int load_b(const std::string &filename, const std::string &proto_filename);
   int load_c(const std::string &filename, const char *cflags[], int ncflags);
@@ -99,11 +100,13 @@ class BPFModule {
   std::unique_ptr<BLoader> b_loader_;
   std::unique_ptr<ClangLoader> clang_loader_;
   std::map<std::string, std::tuple<uint8_t *, uintptr_t>> sections_;
-  std::unique_ptr<std::vector<TableDesc>> tables_;
+  std::vector<TableDesc *> tables_;
   std::map<std::string, size_t> table_names_;
   std::vector<std::string> function_names_;
   std::map<llvm::Type *, llvm::Function *> readers_;
   std::map<llvm::Type *, llvm::Function *> writers_;
+  TableStorage *ts_;
+  std::unique_ptr<TableStorage> local_ts_;
 };
 
 }  // namespace ebpf

src/cc/bpffs_table.cc

+/*
+ * Copyright (c) 2017 VMware, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common.h"
+#include "table_storage_impl.h"
+
+namespace ebpf {
+
+using std::string;
+using std::unique_ptr;
+
+/// A filesystem backed table storage
+class BpfFsTableStorage : public TableStorageImpl {
+ public:
+  class iterator : public TableStorageIteratorImpl {
+   public:
+    virtual ~iterator() {}
+    virtual unique_ptr<self_type> clone() const override;
+    virtual self_type &operator++() override;
+    virtual value_type &operator*() const override;
+    virtual pointer operator->() const override;
+  };
+  virtual ~BpfFsTableStorage() {}
+  virtual bool Find(const string &name, TableStorage::iterator &result) const override;
+  virtual bool Insert(const string &name, TableDesc &&desc) override;
+  virtual bool Delete(const string &name) override;
+  virtual unique_ptr<TableStorageIteratorImpl> begin() override;
+  virtual unique_ptr<TableStorageIteratorImpl> end() override;
+  virtual unique_ptr<TableStorageIteratorImpl> lower_bound(const string &k) override;
+  virtual unique_ptr<TableStorageIteratorImpl> upper_bound(const string &k) override;
+  virtual unique_ptr<TableStorageIteratorImpl> erase(const TableStorageIteratorImpl &it) override;
+
+ private:
+};
+
+bool BpfFsTableStorage::Find(const string &name, TableStorage::iterator &result) const {
+  return false;
+}
+
+bool BpfFsTableStorage::Insert(const string &name, TableDesc &&desc) { return false; }
+
+bool BpfFsTableStorage::Delete(const string &name) { return false; }
+
+unique_ptr<TableStorageIteratorImpl> BpfFsTableStorage::begin() { return unique_ptr<iterator>(); }
+unique_ptr<TableStorageIteratorImpl> BpfFsTableStorage::end() { return unique_ptr<iterator>(); }
+unique_ptr<TableStorageIteratorImpl> BpfFsTableStorage::lower_bound(const string &k) {
+  return unique_ptr<iterator>();
+}
+unique_ptr<TableStorageIteratorImpl> BpfFsTableStorage::upper_bound(const string &k) {
+  return unique_ptr<iterator>();
+}
+unique_ptr<TableStorageIteratorImpl> BpfFsTableStorage::erase(const TableStorageIteratorImpl &it) {
+  return unique_ptr<iterator>();
+}
+
+unique_ptr<TableStorage> createBpfFsTableStorage() {
+  auto t = make_unique<TableStorage>();
+  t->Init(make_unique<BpfFsTableStorage>());
+  return t;
+}
+
+}  // namespace ebpf

src/cc/frontends/b/codegen_llvm.cc

@@ -35,10 +35,10 @@
 #include "bcc_exception.h"
 #include "codegen_llvm.h"
 #include "lexer.h"
-#include "table_desc.h"
-#include "type_helper.h"
-#include "linux/bpf.h"
 #include "libbpf.h"
+#include "linux/bpf.h"
+#include "table_storage.h"
+#include "type_helper.h"
 
 namespace ebpf {
 namespace cc {
@@ -1220,7 +1220,7 @@ StatusTuple CodegenLLVM::visit_func_decl_stmt_node(FuncDeclStmtNode *n) {
   return StatusTuple(0);
 }
 
-StatusTuple CodegenLLVM::visit(Node* root, vector<TableDesc> &tables) {
+StatusTuple CodegenLLVM::visit(Node *root, TableStorage &ts, const string &id) {
   scopes_->set_current(scopes_->top_state());
   scopes_->set_current(scopes_->top_var());
 
@@ -1239,16 +1239,12 @@ StatusTuple CodegenLLVM::visit(Node* root, vector<TableDesc> &tables) {
       map_type = BPF_MAP_TYPE_HASH;
     else if (table.first->type_id()->name_ == "INDEXED")
       map_type = BPF_MAP_TYPE_ARRAY;
-    tables.push_back({
-      table.first->id_->name_,
-      table_fds_[table.first],
-      map_type,
-      table.first->key_type_->bit_width_ >> 3,
-      table.first->leaf_type_->bit_width_ >> 3,
-      table.first->size_,
-      0,
-      "", "",
-    });
+    ts.Insert(Path({id, table.first->id_->name_}),
+              {
+                  table.first->id_->name_, table_fds_[table.first], map_type,
+                  table.first->key_type_->bit_width_ >> 3, table.first->leaf_type_->bit_width_ >> 3,
+                  table.first->size_, 0,
+              });
   }
   return StatusTuple(0);
 }

src/cc/frontends/b/codegen_llvm.h

@@ -41,7 +41,8 @@ class GlobalVariable;
 }
 
 namespace ebpf {
-struct TableDesc;
+
+class TableStorage;
 
 namespace cc {
 
@@ -63,7 +64,7 @@ class CodegenLLVM : public Visitor {
   EXPAND_NODES(VISIT)
 #undef VISIT
 
-  STATUS_RETURN visit(Node* n, std::vector<TableDesc> &tables);
+  STATUS_RETURN visit(Node *n, TableStorage &ts, const std::string &id);
 
   int get_table_fd(const std::string &name) const;
 

src/cc/frontends/b/loader.cc

@@ -18,7 +18,6 @@
 #include "type_check.h"
 #include "codegen_llvm.h"
 #include "loader.h"
-#include "table_desc.h"
 
 using std::string;
 using std::unique_ptr;
@@ -34,7 +33,7 @@ BLoader::~BLoader() {
 }
 
 int BLoader::parse(llvm::Module *mod, const string &filename, const string &proto_filename,
-                   unique_ptr<vector<TableDesc>> *tables) {
+                   TableStorage &ts, const string &id) {
   int rc;
 
   proto_parser_ = make_unique<ebpf::cc::Parser>(proto_filename);
@@ -61,10 +60,8 @@ int BLoader::parse(llvm::Module *mod, const string &filename, const string &prot
     return -1;
   }
 
-  *tables = make_unique<vector<TableDesc>>();
-
   codegen_ = ebpf::make_unique<ebpf::cc::CodegenLLVM>(mod, parser_->scopes_.get(), proto_parser_->scopes_.get());
-  ret = codegen_->visit(parser_->root_node_, **tables);
+  ret = codegen_->visit(parser_->root_node_, ts, id);
   if (ret.code() != 0 || ret.msg().size()) {
     fprintf(stderr, "Codegen error @line=%d: %s\n", ret.code(), ret.msg().c_str());
     return ret.code();

src/cc/frontends/b/loader.h

@@ -20,14 +20,14 @@
 #include <memory>
 #include <string>
 
+#include "table_storage.h"
+
 namespace llvm {
 class Module;
 }
 
 namespace ebpf {
 
-struct TableDesc;
-
 namespace cc {
 class Parser;
 class CodegenLLVM;
@@ -38,7 +38,8 @@ class BLoader {
   explicit BLoader(unsigned flags);
   ~BLoader();
   int parse(llvm::Module *mod, const std::string &filename, const std::string &proto_filename,
-            std::unique_ptr<std::vector<TableDesc>> *tables);
+            TableStorage &ts, const std::string &id);
+
  private:
   unsigned flags_;
   std::unique_ptr<cc::Parser> parser_;

src/cc/frontends/clang/b_frontend_action.cc

@@ -26,8 +26,8 @@
 #include <clang/Rewrite/Core/Rewriter.h>
 
 #include "b_frontend_action.h"
-#include "shared_table.h"
 #include "common.h"
+#include "table_storage.h"
 
 #include "libbpf.h"
 
@@ -57,6 +57,7 @@ const char **calling_conv_regs = calling_conv_regs_x86;
 #endif
 
 using std::map;
+using std::move;
 using std::set;
 using std::string;
 using std::to_string;
@@ -64,69 +65,6 @@ using std::unique_ptr;
 using std::vector;
 using namespace clang;
 
-// Encode the struct layout as a json description
-BMapDeclVisitor::BMapDeclVisitor(ASTContext &C, string &result)
-    : C(C), result_(result) {}
-bool BMapDeclVisitor::VisitFieldDecl(FieldDecl *D) {
-  result_ += "\"";
-  result_ += D->getName();
-  result_ += "\",";
-  return true;
-}
-
-bool BMapDeclVisitor::TraverseRecordDecl(RecordDecl *D) {
-  // skip children, handled in Visit...
-  if (!WalkUpFromRecordDecl(D))
-    return false;
-  return true;
-}
-bool BMapDeclVisitor::VisitRecordDecl(RecordDecl *D) {
-  result_ += "[\"";
-  result_ += D->getName();
-  result_ += "\", [";
-  for (auto F : D->getDefinition()->fields()) {
-    if (F->isAnonymousStructOrUnion()) {
-      if (const RecordType *R = dyn_cast<RecordType>(F->getType()))
-        TraverseDecl(R->getDecl());
-      result_ += ", ";
-      continue;
-    }
-    result_ += "[";
-    TraverseDecl(F);
-    if (const ConstantArrayType *T = dyn_cast<ConstantArrayType>(F->getType()))
-      result_ += ", [" + T->getSize().toString(10, false) + "]";
-    if (F->isBitField())
-      result_ += ", " + to_string(F->getBitWidthValue(C));
-    result_ += "], ";
-  }
-  if (!D->getDefinition()->field_empty())
-    result_.erase(result_.end() - 2);
-  result_ += "]";
-  if (D->isUnion())
-    result_ += ", \"union\"";
-  else if (D->isStruct())
-    result_ += ", \"struct\"";
-  result_ += "]";
-  return true;
-}
-// pointer to anything should be treated as terminal, don't recurse further
-bool BMapDeclVisitor::VisitPointerType(const PointerType *T) {
-  result_ += "\"unsigned long long\"";
-  return false;
-}
-bool BMapDeclVisitor::VisitTagType(const TagType *T) {
-  return TraverseDecl(T->getDecl()->getDefinition());
-}
-bool BMapDeclVisitor::VisitTypedefType(const TypedefType *T) {
-  return TraverseDecl(T->getDecl());
-}
-bool BMapDeclVisitor::VisitBuiltinType(const BuiltinType *T) {
-  result_ += "\"";
-  result_ += T->getName(C.getPrintingPolicy());
-  result_ += "\"";
-  return true;
-}
-
 class ProbeChecker : public RecursiveASTVisitor<ProbeChecker> {
  public:
   explicit ProbeChecker(Expr *arg, const set<Decl *> &ptregs)
@@ -272,10 +210,8 @@ DiagnosticBuilder ProbeVisitor::error(SourceLocation loc, const char (&fmt)[N])
   return C.getDiagnostics().Report(loc, diag_id);
 }
 
-
-BTypeVisitor::BTypeVisitor(ASTContext &C, Rewriter &rewriter, vector<TableDesc> &tables)
-    : C(C), diag_(C.getDiagnostics()), rewriter_(rewriter), out_(llvm::errs()), tables_(tables) {
-}
+BTypeVisitor::BTypeVisitor(ASTContext &C, BFrontendAction &fe)
+    : C(C), diag_(C.getDiagnostics()), fe_(fe), rewriter_(fe.rewriter()), out_(llvm::errs()) {}
 
 bool BTypeVisitor::VisitFunctionDecl(FunctionDecl *D) {
   // put each non-static non-inline function decl in its own section, to be
@@ -360,14 +296,16 @@ bool BTypeVisitor::VisitCallExpr(CallExpr *Call) {
                                                    Call->getArg(Call->getNumArgs() - 1)->getLocEnd())));
 
         // find the table fd, which was opened at declaration time
-        auto table_it = tables_.begin();
-        for (; table_it != tables_.end(); ++table_it)
-          if (table_it->name == Ref->getDecl()->getName()) break;
-        if (table_it == tables_.end()) {
-          error(Ref->getLocEnd(), "bpf_table %0 failed to open") << Ref->getDecl()->getName();
-          return false;
+        TableStorage::iterator desc;
+        Path local_path({fe_.id(), Ref->getDecl()->getName()});
+        Path global_path({Ref->getDecl()->getName()});
+        if (!fe_.table_storage().Find(local_path, desc)) {
+          if (!fe_.table_storage().Find(global_path, desc)) {
+            error(Ref->getLocEnd(), "bpf_table %0 failed to open") << Ref->getDecl()->getName();
+            return false;
+          }
         }
-        string fd = to_string(table_it->fd);
+        string fd = to_string(desc->second.fd);
         string prefix, suffix;
         string txt;
         auto rewrite_start = Call->getLocStart();
@@ -391,7 +329,7 @@ bool BTypeVisitor::VisitCallExpr(CallExpr *Call) {
           string lookup = "bpf_map_lookup_elem_(bpf_pseudo_fd(1, " + fd + ")";
           string update = "bpf_map_update_elem_(bpf_pseudo_fd(1, " + fd + ")";
           txt  = "({ typeof(" + name + ".key) _key = " + arg0 + "; ";
-          if (table_it->type == BPF_MAP_TYPE_HASH) {
+          if (desc->second.type == BPF_MAP_TYPE_HASH) {
             txt += "typeof(" + name + ".leaf) _zleaf; memset(&_zleaf, 0, sizeof(_zleaf)); ";
             txt += update + ", &_key, &_zleaf, BPF_NOEXIST); ";
           }
@@ -416,11 +354,12 @@ bool BTypeVisitor::VisitCallExpr(CallExpr *Call) {
             meta + ", " +
             meta_len + ");";
         } else if (memb_name == "get_stackid") {
-            if (table_it->type == BPF_MAP_TYPE_STACK_TRACE) {
-              string arg0 = rewriter_.getRewrittenText(expansionRange(Call->getArg(0)->getSourceRange()));
-              txt = "bpf_get_stackid(";
-              txt += "bpf_pseudo_fd(1, " + fd + "), " + arg0;
-              rewrite_end = Call->getArg(0)->getLocEnd();
+          if (desc->second.type == BPF_MAP_TYPE_STACK_TRACE) {
+            string arg0 =
+                rewriter_.getRewrittenText(expansionRange(Call->getArg(0)->getSourceRange()));
+            txt = "bpf_get_stackid(";
+            txt += "bpf_pseudo_fd(1, " + fd + "), " + arg0;
+            rewrite_end = Call->getArg(0)->getLocEnd();
             } else {
               error(Call->getLocStart(), "get_stackid only available on stacktrace maps");
               return false;
@@ -433,7 +372,7 @@ bool BTypeVisitor::VisitCallExpr(CallExpr *Call) {
             prefix = "bpf_map_update_elem";
             suffix = ", BPF_ANY)";
           } else if (memb_name == "insert") {
-            if (table_it->type == BPF_MAP_TYPE_ARRAY) {
+            if (desc->second.type == BPF_MAP_TYPE_ARRAY) {
               warning(Call->getLocStart(), "all element of an array already exist; insert() will have no effect");
             }
             prefix = "bpf_map_update_elem";
@@ -661,8 +600,12 @@ bool BTypeVisitor::VisitVarDecl(VarDecl *Decl) {
     }
     const RecordDecl *RD = R->getDecl()->getDefinition();
 
-    TableDesc table = {};
+    TableDesc table;
+    TableStorage::iterator table_it;
     table.name = Decl->getName();
+    Path local_path({fe_.id(), table.name});
+    Path global_path({table.name});
+    QualType key_type, leaf_type;
 
     unsigned i = 0;
     for (auto F : RD->fields()) {
@@ -673,16 +616,14 @@ bool BTypeVisitor::VisitVarDecl(VarDecl *Decl) {
           return false;
         }
         table.key_size = sz;
-        BMapDeclVisitor visitor(C, table.key_desc);
-        visitor.TraverseType(F->getType());
+        key_type = F->getType();
       } else if (F->getName() == "leaf") {
         if (sz == 0) {
           error(F->getLocStart(), "invalid zero-sized leaf");
           return false;
         }
         table.leaf_size = sz;
-        BMapDeclVisitor visitor(C, table.leaf_desc);
-        visitor.TraverseType(F->getType());
+        leaf_type = F->getType();
       } else if (F->getName() == "data") {
         table.max_entries = sz / table.leaf_size;
       } else if (F->getName() == "flags") {
@@ -713,13 +654,11 @@ bool BTypeVisitor::VisitVarDecl(VarDecl *Decl) {
     } else if (A->getName() == "maps/lpm_trie") {
       map_type = BPF_MAP_TYPE_LPM_TRIE;
     } else if (A->getName() == "maps/histogram") {
-      if (table.key_desc == "\"int\"")
+      map_type = BPF_MAP_TYPE_HASH;
+      if (key_type->isSpecificBuiltinType(BuiltinType::Int))
         map_type = BPF_MAP_TYPE_ARRAY;
-      else
-        map_type = BPF_MAP_TYPE_HASH;
-      if (table.leaf_desc != "\"unsigned long long\"") {
-        error(Decl->getLocStart(), "histogram leaf type must be u64, got %0") << table.leaf_desc;
-      }
+      if (!leaf_type->isSpecificBuiltinType(BuiltinType::ULongLong))
+        error(Decl->getLocStart(), "histogram leaf type must be u64, got %0") << leaf_type;
     } else if (A->getName() == "maps/prog") {
       map_type = BPF_MAP_TYPE_PROG_ARRAY;
     } else if (A->getName() == "maps/perf_output") {
@@ -733,24 +672,22 @@ bool BTypeVisitor::VisitVarDecl(VarDecl *Decl) {
     } else if (A->getName() == "maps/stacktrace") {
       map_type = BPF_MAP_TYPE_STACK_TRACE;
     } else if (A->getName() == "maps/extern") {
+      if (!fe_.table_storage().Find(global_path, table_it)) {
+        error(Decl->getLocStart(), "reference to undefined table");
+        return false;
+      }
+      table = table_it->second.dup();
       table.is_extern = true;
-      table.fd = SharedTables::instance()->lookup_fd(table.name);
-      table.type = SharedTables::instance()->lookup_type(table.name);
     } else if (A->getName() == "maps/export") {
       if (table.name.substr(0, 2) == "__")
         table.name = table.name.substr(2);
-      auto table_it = tables_.begin();
-      for (; table_it != tables_.end(); ++table_it)
-        if (table_it->name == table.name) break;
-      if (table_it == tables_.end()) {
+      Path local_path({fe_.id(), table.name});
+      Path global_path({table.name});
+      if (!fe_.table_storage().Find(local_path, table_it)) {
         error(Decl->getLocStart(), "reference to undefined table");
         return false;
       }
-      if (!SharedTables::instance()->insert_fd(table.name, table_it->fd, table_it->type)) {
-        error(Decl->getLocStart(), "could not export bpf map %0: %1") << table.name << "already in use";
-        return false;
-      }
-      table_it->is_shared = true;
+      fe_.table_storage().Insert(global_path, table_it->second.dup());
       return true;
     }
 
@@ -769,7 +706,8 @@ bool BTypeVisitor::VisitVarDecl(VarDecl *Decl) {
       return false;
     }
 
-    tables_.push_back(std::move(table));
+    fe_.table_storage().VisitMapType(table, C, key_type, leaf_type);
+    fe_.table_storage().Insert(local_path, move(table));
   } else if (const PointerType *P = Decl->getType()->getAs<PointerType>()) {
     // if var is a pointer to a packet type, clone the annotation into the var
     // decl so that the packet dext/dins rewriter can catch it
@@ -786,9 +724,7 @@ bool BTypeVisitor::VisitVarDecl(VarDecl *Decl) {
   return true;
 }
 
-BTypeConsumer::BTypeConsumer(ASTContext &C, Rewriter &rewriter, vector<TableDesc> &tables)
-    : visitor_(C, rewriter, tables) {
-}
+BTypeConsumer::BTypeConsumer(ASTContext &C, BFrontendAction &fe) : visitor_(C, fe) {}
 
 bool BTypeConsumer::HandleTopLevelDecl(DeclGroupRef Group) {
   for (auto D : Group)
@@ -814,9 +750,9 @@ bool ProbeConsumer::HandleTopLevelDecl(DeclGroupRef Group) {
   return true;
 }
 
-BFrontendAction::BFrontendAction(llvm::raw_ostream &os, unsigned flags)
-    : os_(os), flags_(flags), rewriter_(new Rewriter), tables_(new vector<TableDesc>) {
-}
+BFrontendAction::BFrontendAction(llvm::raw_ostream &os, unsigned flags, TableStorage &ts,
+                                 const std::string &id)
+    : os_(os), flags_(flags), ts_(ts), id_(id), rewriter_(new Rewriter) {}
 
 void BFrontendAction::EndSourceFileAction() {
   if (flags_ & DEBUG_PREPROCESSOR)
@@ -829,8 +765,8 @@ unique_ptr<ASTConsumer> BFrontendAction::CreateASTConsumer(CompilerInstance &Com
   rewriter_->setSourceMgr(Compiler.getSourceManager(), Compiler.getLangOpts());
   vector<unique_ptr<ASTConsumer>> consumers;
   consumers.push_back(unique_ptr<ASTConsumer>(new ProbeConsumer(Compiler.getASTContext(), *rewriter_)));
-  consumers.push_back(unique_ptr<ASTConsumer>(new BTypeConsumer(Compiler.getASTContext(), *rewriter_, *tables_)));
-  return unique_ptr<ASTConsumer>(new MultiplexConsumer(move(consumers)));
+  consumers.push_back(unique_ptr<ASTConsumer>(new BTypeConsumer(Compiler.getASTContext(), *this)));
+  return unique_ptr<ASTConsumer>(new MultiplexConsumer(std::move(consumers)));
 }
 
 }

src/cc/frontends/clang/b_frontend_action.h

@@ -24,7 +24,7 @@
 #include <clang/Frontend/FrontendAction.h>
 #include <clang/Rewrite/Core/Rewriter.h>
 
-#include "table_desc.h"
+#include "table_storage.h"
 
 #define DEBUG_PREPROCESSOR 0x4
 
@@ -41,21 +41,7 @@ class StringRef;
 
 namespace ebpf {
 
-// Helper visitor for constructing a string representation of a key/leaf decl
-class BMapDeclVisitor : public clang::RecursiveASTVisitor<BMapDeclVisitor> {
- public:
-  explicit BMapDeclVisitor(clang::ASTContext &C, std::string &result);
-  bool TraverseRecordDecl(clang::RecordDecl *Decl);
-  bool VisitRecordDecl(clang::RecordDecl *Decl);
-  bool VisitFieldDecl(clang::FieldDecl *Decl);
-  bool VisitBuiltinType(const clang::BuiltinType *T);
-  bool VisitTypedefType(const clang::TypedefType *T);
-  bool VisitTagType(const clang::TagType *T);
-  bool VisitPointerType(const clang::PointerType *T);
- private:
-  clang::ASTContext &C;
-  std::string &result_;
-};
+class BFrontendAction;
 
 // Type visitor and rewriter for B programs.
 // It will look for B-specific features and rewrite them into a valid
@@ -63,8 +49,7 @@ class BMapDeclVisitor : public clang::RecursiveASTVisitor<BMapDeclVisitor> {
 // and store the open handles in a map of table-to-fd's.
 class BTypeVisitor : public clang::RecursiveASTVisitor<BTypeVisitor> {
  public:
-  explicit BTypeVisitor(clang::ASTContext &C, clang::Rewriter &rewriter,
-                        std::vector<TableDesc> &tables);
+  explicit BTypeVisitor(clang::ASTContext &C, BFrontendAction &fe);
   bool TraverseCallExpr(clang::CallExpr *Call);
   bool VisitFunctionDecl(clang::FunctionDecl *D);
   bool VisitCallExpr(clang::CallExpr *Call);
@@ -82,9 +67,9 @@ class BTypeVisitor : public clang::RecursiveASTVisitor<BTypeVisitor> {
 
   clang::ASTContext &C;
   clang::DiagnosticsEngine &diag_;
+  BFrontendAction &fe_;
   clang::Rewriter &rewriter_;  /// modifications to the source go into this class
   llvm::raw_ostream &out_;  /// for debugging
-  std::vector<TableDesc> &tables_;  /// store the open FDs
   std::vector<clang::ParmVarDecl *> fn_args_;
   std::set<clang::Expr *> visited_;
   std::string current_fn_;
@@ -115,8 +100,7 @@ class ProbeVisitor : public clang::RecursiveASTVisitor<ProbeVisitor> {
 // A helper class to the frontend action, walks the decls
 class BTypeConsumer : public clang::ASTConsumer {
  public:
-  explicit BTypeConsumer(clang::ASTContext &C, clang::Rewriter &rewriter,
-                         std::vector<TableDesc> &tables);
+  explicit BTypeConsumer(clang::ASTContext &C, BFrontendAction &fe);
   bool HandleTopLevelDecl(clang::DeclGroupRef Group) override;
  private:
   BTypeVisitor visitor_;
@@ -138,7 +122,7 @@ class BFrontendAction : public clang::ASTFrontendAction {
  public:
   // Initialize with the output stream where the new source file contents
   // should be written.
-  BFrontendAction(llvm::raw_ostream &os, unsigned flags);
+  BFrontendAction(llvm::raw_ostream &os, unsigned flags, TableStorage &ts, const std::string &id);
 
   // Called by clang when the AST has been completed, here the output stream
   // will be flushed.
@@ -147,13 +131,16 @@ class BFrontendAction : public clang::ASTFrontendAction {
   std::unique_ptr<clang::ASTConsumer>
       CreateASTConsumer(clang::CompilerInstance &Compiler, llvm::StringRef InFile) override;
 
-  // take ownership of the table-to-fd mapping data structure
-  std::unique_ptr<std::vector<TableDesc>> take_tables() { return move(tables_); }
+  clang::Rewriter &rewriter() const { return *rewriter_; }
+  TableStorage &table_storage() const { return ts_; }
+  std::string id() const { return id_; }
+
  private:
   llvm::raw_ostream &os_;
   unsigned flags_;
+  TableStorage &ts_;
+  std::string id_;
   std::unique_ptr<clang::Rewriter> rewriter_;
-  std::unique_ptr<std::vector<TableDesc>> tables_;
 };
 
 }  // namespace visitor

src/cc/frontends/clang/loader.cc

@@ -104,8 +104,8 @@ std::pair<bool, string> get_kernel_path_info(const string kdir)
 
 }
 
-int ClangLoader::parse(unique_ptr<llvm::Module> *mod, unique_ptr<vector<TableDesc>> *tables,
-                       const string &file, bool in_memory, const char *cflags[], int ncflags) {
+int ClangLoader::parse(unique_ptr<llvm::Module> *mod, TableStorage &ts, const string &file,
+                       bool in_memory, const char *cflags[], int ncflags, const std::string &id) {
   using namespace clang;
 
   string main_path = "/virtual/main.c";
@@ -266,12 +266,10 @@ int ClangLoader::parse(unique_ptr<llvm::Module> *mod, unique_ptr<vector<TableDes
   // capture the rewritten c file
   string out_str1;
   llvm::raw_string_ostream os1(out_str1);
-  BFrontendAction bact(os1, flags_);
+  BFrontendAction bact(os1, flags_, ts, id);
   if (!compiler1.ExecuteAction(bact))
     return -1;
   unique_ptr<llvm::MemoryBuffer> out_buf1 = llvm::MemoryBuffer::getMemBuffer(out_str1);
-  // this contains the open FDs
-  *tables = bact.take_tables();
 
   // second pass, clear input and take rewrite buffer
   CompilerInstance compiler2;

src/cc/frontends/clang/loader.h

@@ -20,6 +20,8 @@
 #include <memory>
 #include <string>
 
+#include "table_storage.h"
+
 namespace llvm {
 class Module;
 class LLVMContext;
@@ -28,14 +30,13 @@ class MemoryBuffer;
 
 namespace ebpf {
 
-struct TableDesc;
-
 class ClangLoader {
  public:
   explicit ClangLoader(llvm::LLVMContext *ctx, unsigned flags);
   ~ClangLoader();
-  int parse(std::unique_ptr<llvm::Module> *mod, std::unique_ptr<std::vector<TableDesc>> *tables,
-            const std::string &file, bool in_memory, const char *cflags[], int ncflags);
+  int parse(std::unique_ptr<llvm::Module> *mod, TableStorage &ts, const std::string &file,
+            bool in_memory, const char *cflags[], int ncflags, const std::string &id);
+
  private:
   static std::map<std::string, std::unique_ptr<llvm::MemoryBuffer>> remapped_files_;
   llvm::LLVMContext *ctx_;

src/cc/json_map_decl_visitor.cc

+/*
+ * Copyright (c) 2017 VMware, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <memory>
+#include <string>
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/RecursiveASTVisitor.h>
+#include "common.h"
+#include "table_desc.h"
+
+namespace ebpf {
+
+using std::string;
+using std::to_string;
+using std::unique_ptr;
+using namespace clang;
+
+// Helper visitor for constructing a string representation of a key/leaf decl
+class BMapDeclVisitor : public clang::RecursiveASTVisitor<BMapDeclVisitor> {
+ public:
+  explicit BMapDeclVisitor(clang::ASTContext &C, std::string &result);
+  bool TraverseRecordDecl(clang::RecordDecl *Decl);
+  bool VisitRecordDecl(clang::RecordDecl *Decl);
+  bool VisitFieldDecl(clang::FieldDecl *Decl);
+  bool VisitBuiltinType(const clang::BuiltinType *T);
+  bool VisitTypedefType(const clang::TypedefType *T);
+  bool VisitTagType(const clang::TagType *T);
+  bool VisitPointerType(const clang::PointerType *T);
+
+ private:
+  clang::ASTContext &C;
+  std::string &result_;
+};
+
+// Encode the struct layout as a json description
+BMapDeclVisitor::BMapDeclVisitor(ASTContext &C, string &result) : C(C), result_(result) {}
+bool BMapDeclVisitor::VisitFieldDecl(FieldDecl *D) {
+  result_ += "\"";
+  result_ += D->getName();
+  result_ += "\",";
+  return true;
+}
+
+bool BMapDeclVisitor::TraverseRecordDecl(RecordDecl *D) {
+  // skip children, handled in Visit...
+  if (!WalkUpFromRecordDecl(D))
+    return false;
+  return true;
+}
+bool BMapDeclVisitor::VisitRecordDecl(RecordDecl *D) {
+  result_ += "[\"";
+  result_ += D->getName();
+  result_ += "\", [";
+  for (auto F : D->getDefinition()->fields()) {
+    if (F->isAnonymousStructOrUnion()) {
+      if (const RecordType *R = dyn_cast<RecordType>(F->getType()))
+        TraverseDecl(R->getDecl());
+      result_ += ", ";
+      continue;
+    }
+    result_ += "[";
+    TraverseDecl(F);
+    if (const ConstantArrayType *T = dyn_cast<ConstantArrayType>(F->getType()))
+      result_ += ", [" + T->getSize().toString(10, false) + "]";
+    if (F->isBitField())
+      result_ += ", " + to_string(F->getBitWidthValue(C));
+    result_ += "], ";
+  }
+  if (!D->getDefinition()->field_empty())
+    result_.erase(result_.end() - 2);
+  result_ += "]";
+  if (D->isUnion())
+    result_ += ", \"union\"";
+  else if (D->isStruct())
+    result_ += ", \"struct\"";
+  result_ += "]";
+  return true;
+}
+// pointer to anything should be treated as terminal, don't recurse further
+bool BMapDeclVisitor::VisitPointerType(const PointerType *T) {
+  result_ += "\"unsigned long long\"";
+  return false;
+}
+bool BMapDeclVisitor::VisitTagType(const TagType *T) {
+  return TraverseDecl(T->getDecl()->getDefinition());
+}
+bool BMapDeclVisitor::VisitTypedefType(const TypedefType *T) { return TraverseDecl(T->getDecl()); }
+bool BMapDeclVisitor::VisitBuiltinType(const BuiltinType *T) {
+  result_ += "\"";
+  result_ += T->getName(C.getPrintingPolicy());
+  result_ += "\"";
+  return true;
+}
+
+class JsonMapTypesVisitor : public virtual MapTypesVisitor {
+ public:
+  virtual void Visit(TableDesc &desc, clang::ASTContext &C, clang::QualType key_type,
+                     clang::QualType leaf_type) {
+    BMapDeclVisitor v1(C, desc.key_desc), v2(C, desc.leaf_desc);
+    v1.TraverseType(key_type);
+    v2.TraverseType(leaf_type);
+  }
+};
+
+unique_ptr<MapTypesVisitor> createJsonMapTypesVisitor() {
+  return make_unique<JsonMapTypesVisitor>();
+}
+
+}  // namespace ebpf

src/cc/shared_table.cc

@@ -15,51 +15,100 @@
  */
 
 #include <unistd.h>
+#include <iostream>
 
-#include "shared_table.h"
+#include "common.h"
 #include "compat/linux/bpf.h"
+#include "table_storage.h"
+#include "table_storage_impl.h"
 
 namespace ebpf {
 
 using std::string;
+using std::unique_ptr;
 
-SharedTables * SharedTables::instance_;
+/// A process-wide singleton of shared tables
+class SharedTableStorage : public TableStorageImpl {
+ public:
+  class iterator : public TableStorageIteratorImpl {
+    std::map<string, TableDesc>::iterator it_;
 
-SharedTables * SharedTables::instance() {
-  if (!instance_) {
-    instance_ = new SharedTables;
-  }
-  return instance_;
-}
+   public:
+    explicit iterator(const std::map<string, TableDesc>::iterator &it) : it_(it) {}
+    virtual ~iterator() {}
+    virtual unique_ptr<self_type> clone() const override { return make_unique<iterator>(it_); }
+    virtual self_type &operator++() override {
+      ++it_;
+      return *this;
+    }
+    virtual value_type &operator*() const override { return *it_; }
+    virtual pointer operator->() const override { return &*it_; }
+  };
+  virtual ~SharedTableStorage() {}
+  virtual bool Find(const string &name, TableStorage::iterator &result) const override;
+  virtual bool Insert(const string &name, TableDesc &&desc) override;
+  virtual bool Delete(const string &name) override;
+  virtual unique_ptr<TableStorageIteratorImpl> begin() override;
+  virtual unique_ptr<TableStorageIteratorImpl> end() override;
+  virtual unique_ptr<TableStorageIteratorImpl> lower_bound(const string &k) override;
+  virtual unique_ptr<TableStorageIteratorImpl> upper_bound(const string &k) override;
+  virtual unique_ptr<TableStorageIteratorImpl> erase(const TableStorageIteratorImpl &it) override;
 
-int SharedTables::lookup_fd(const string &name) const {
-  auto table = tables_.find(name);
-  if (table == tables_.end())
-    return -1;
-  return table->second.first;
-}
+ private:
+  static std::map<string, TableDesc> tables_;
+};
 
-int SharedTables::lookup_type(const string &name) const {
-  auto table = tables_.find(name);
-  if (table == tables_.end())
-    return BPF_MAP_TYPE_UNSPEC;
-  return table->second.second;
+bool SharedTableStorage::Find(const string &name, TableStorage::iterator &result) const {
+  auto it = tables_.find(name);
+  if (it == tables_.end())
+    return false;
+  result = TableStorage::iterator(make_unique<iterator>(it));
+  return true;
 }
 
-bool SharedTables::insert_fd(const string &name, int fd, int type) {
-  if (tables_.find(name) != tables_.end())
+bool SharedTableStorage::Insert(const string &name, TableDesc &&desc) {
+  auto it = tables_.find(name);
+  if (it != tables_.end())
     return false;
-  tables_[name] = std::make_pair(fd, type);
+  tables_[name] = std::move(desc);
   return true;
 }
 
-bool SharedTables::remove_fd(const string &name) {
-  auto table = tables_.find(name);
-  if (table == tables_.end())
+bool SharedTableStorage::Delete(const string &name) {
+  auto it = tables_.find(name);
+  if (it == tables_.end())
     return false;
-  close(table->second.first);
-  tables_.erase(table);
+  tables_.erase(it);
   return true;
 }
 
+unique_ptr<TableStorageIteratorImpl> SharedTableStorage::begin() {
+  return make_unique<iterator>(tables_.begin());
+}
+unique_ptr<TableStorageIteratorImpl> SharedTableStorage::end() {
+  return make_unique<iterator>(tables_.end());
+}
+
+unique_ptr<TableStorageIteratorImpl> SharedTableStorage::lower_bound(const string &k) {
+  return make_unique<iterator>(tables_.lower_bound(k));
+}
+unique_ptr<TableStorageIteratorImpl> SharedTableStorage::upper_bound(const string &k) {
+  return make_unique<iterator>(tables_.upper_bound(k));
+}
+unique_ptr<TableStorageIteratorImpl> SharedTableStorage::erase(const TableStorageIteratorImpl &it) {
+  auto i = tables_.find((*it).first);
+  if (i == tables_.end())
+    return unique_ptr<iterator>();
+  return make_unique<iterator>(tables_.erase(i));
+}
+
+// All maps for this process are kept in global static storage.
+std::map<string, TableDesc> SharedTableStorage::tables_;
+
+unique_ptr<TableStorage> createSharedTableStorage() {
+  auto t = make_unique<TableStorage>();
+  t->Init(make_unique<SharedTableStorage>());
+  t->AddMapTypesVisitor(createJsonMapTypesVisitor());
+  return t;
+}
 }

src/cc/shared_table.h

@@ -16,27 +16,3 @@
 
 #pragma once
 
-#include <map>
-#include <string>
-
-namespace ebpf {
-
-struct TableDesc;
-
-class SharedTables {
- public:
-  static SharedTables * instance();
-  // add an fd to the shared table, return true if successfully inserted
-  bool insert_fd(const std::string &name, int fd, int type);
-  // lookup an fd in the shared table, or -1 if not found
-  int lookup_fd(const std::string &name) const;
-  // lookup on map type in the shared table, or BPF_MAP_TYPE_UNSPEC if not found
-  int lookup_type(const std::string &name) const;
-  // close and remove a shared fd. return true if the value was found
-  bool remove_fd(const std::string &name);
- private:
-  static SharedTables *instance_;
-  std::map<std::string, std::pair<int, int>> tables_;
-};
-
-}

src/cc/table_desc.h

@@ -16,18 +16,101 @@
 
 #pragma once
 
+#include <unistd.h>
 #include <cstdint>
+#include <memory>
 #include <string>
 
 namespace llvm {
 class Function;
 }
 
+namespace clang {
+class ASTContext;
+class QualType;
+}
+
 namespace ebpf {
 
-struct TableDesc {
-  std::string name;
+class TableDesc;
+
+/// FileDesc is a helper class for managing open file descriptors. Copy is
+/// disallowed (call dup instead), and cleanup happens automatically.
+class FileDesc {
+  friend TableDesc;
+
+ private:
+  FileDesc &operator=(const FileDesc &that) {
+    fd = ::dup(that.fd);
+    return *this;
+  }
+  FileDesc(const FileDesc &that) { *this = that; }
+
+ public:
+  FileDesc(int fd = -1) : fd(fd) {}
+  FileDesc &operator=(FileDesc &&that) {
+    fd = that.fd;
+    that.fd = -1;
+    return *this;
+  }
+  FileDesc(FileDesc &&that) { *this = std::move(that); }
+  ~FileDesc() {
+    if (fd >= 0)
+      ::close(fd);
+  }
+  FileDesc dup() const { return FileDesc(*this); }
+
+  operator int() { return fd; }
+  operator int() const { return fd; }
+
+ private:
   int fd;
+};
+
+/// TableDesc uniquely stores all of the runtime state for an active bpf table.
+/// The copy constructor/assign operator are disabled since the file handles
+/// owned by this table are not implicitly copyable. One should call the dup()
+/// method if an explicit new handle is required. We define the move operators
+/// so that objects of this class can reside in stl containers.
+class TableDesc {
+ private:
+  TableDesc(const TableDesc &) = default;
+  TableDesc &operator=(const TableDesc &) = default;
+
+ public:
+  TableDesc()
+      : type(0),
+        key_size(0),
+        leaf_size(0),
+        max_entries(0),
+        flags(0),
+        key_sscanf(nullptr),
+        leaf_sscanf(nullptr),
+        key_snprintf(nullptr),
+        leaf_snprintf(nullptr),
+        is_shared(false),
+        is_extern(false) {}
+  TableDesc(const std::string &name, FileDesc &&fd, int type, size_t key_size,
+            size_t leaf_size, size_t max_entries, int flags)
+      : name(name),
+        fd(std::move(fd)),
+        type(type),
+        key_size(key_size),
+        leaf_size(leaf_size),
+        max_entries(max_entries),
+        flags(flags),
+        key_sscanf(nullptr),
+        leaf_sscanf(nullptr),
+        key_snprintf(nullptr),
+        leaf_snprintf(nullptr),
+        is_shared(false),
+        is_extern(false) {}
+  TableDesc(TableDesc &&that) = default;
+  TableDesc &operator=(TableDesc &&that) = default;
+  TableDesc dup() const { return TableDesc(*this); }
+
+  std::string name;
+  FileDesc fd;
   int type;
   size_t key_size;  // sizes are in bytes
   size_t leaf_size;
@@ -43,4 +126,17 @@ struct TableDesc {
   bool is_extern;
 };
 
+/// MapTypesVisitor gets notified of new bpf tables, and has a chance to parse
+/// the key and leaf types for their own usage. Subclass this abstract class and
+/// implement the Visit method, then add an instance of this class to the
+/// StorageTable instance to be notified of each new key/leaf type.
+class MapTypesVisitor {
+ public:
+  virtual ~MapTypesVisitor() {}
+  virtual void Visit(TableDesc &desc, clang::ASTContext &C, clang::QualType key_type,
+                     clang::QualType leaf_type) = 0;
+};
+
+std::unique_ptr<MapTypesVisitor> createJsonMapTypesVisitor();
+
 }  // namespace ebpf

src/cc/table_storage.cc

+/*
+ * Copyright (c) 2017 VMware, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <unistd.h>
+
+#include <clang/AST/Type.h>
+
+#include "table_storage_impl.h"
+
+namespace ebpf {
+
+using std::move;
+using std::string;
+using std::unique_ptr;
+
+const string Path::DELIM = "/";
+
+TableStorage::TableStorage() {}
+TableStorage::~TableStorage() {}
+void TableStorage::Init(unique_ptr<TableStorageImpl> impl) { impl_ = move(impl); }
+bool TableStorage::Find(const Path &path, TableStorage::iterator &result) const {
+  return impl_->Find(path.to_string(), result);
+}
+bool TableStorage::Insert(const Path &path, TableDesc &&desc) {
+  return impl_->Insert(path.to_string(), move(desc));
+}
+bool TableStorage::Delete(const Path &path) { return impl_->Delete(path.to_string()); }
+size_t TableStorage::DeletePrefix(const Path &path) {
+  size_t i = 0;
+  auto it = lower_bound(path);
+  auto upper = upper_bound(path);
+  while (it != upper) {
+    it = impl_->erase(*it.impl_);
+    ++i;
+  }
+  return i;
+}
+
+void TableStorage::AddMapTypesVisitor(unique_ptr<MapTypesVisitor> visitor) {
+  visitors_.push_back(move(visitor));
+}
+void TableStorage::VisitMapType(TableDesc &desc, clang::ASTContext &C, clang::QualType key_type,
+                                clang::QualType leaf_type) {
+  for (auto &v : visitors_)
+    v->Visit(desc, C, key_type, leaf_type);
+}
+
+TableStorage::iterator TableStorage::begin() { return impl_->begin(); }
+TableStorage::iterator TableStorage::end() { return impl_->end(); }
+TableStorage::iterator TableStorage::lower_bound(const Path &p) {
+  return impl_->lower_bound(p.to_string());
+}
+TableStorage::iterator TableStorage::upper_bound(const Path &p) {
+  return impl_->upper_bound(p.to_string() + "\x7f");
+}
+
+/// TableStorage::iterator implementation
+TableStorage::iterator::iterator() {}
+TableStorage::iterator::iterator(unique_ptr<TableStorageIteratorImpl> impl) : impl_(move(impl)) {}
+TableStorage::iterator::iterator(const iterator &that) : impl_(that.impl_->clone()) {}
+TableStorage::iterator::~iterator() {}
+TableStorage::iterator::iterator(iterator &&that) { *this = move(that); }
+TableStorage::iterator &TableStorage::iterator::operator=(iterator &&that) {
+  impl_ = move(that.impl_);
+  return *this;
+}
+
+TableStorage::iterator &TableStorage::iterator::operator++() {
+  ++*impl_;
+  return *this;
+}
+TableStorage::iterator TableStorage::iterator::operator++(int) {
+  iterator tmp(*this);
+  operator++();
+  return tmp;
+}
+bool TableStorage::iterator::operator==(const iterator &rhs) const {
+  // assumes that the underlying pair is stored in only one place
+  return &**impl_ == &**rhs.impl_;
+}
+bool TableStorage::iterator::operator!=(const iterator &rhs) const {
+  return &**impl_ != &**rhs.impl_;
+}
+TableStorage::iterator::reference TableStorage::iterator::operator*() const { return **impl_; }
+TableStorage::iterator::pointer TableStorage::iterator::operator->() const { return &**impl_; }
+
+}  // namespace ebpf

src/cc/table_storage.h

+/*
+ * Copyright (c) 2017 VMware, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstddef>
+#include <iterator>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "table_desc.h"
+
+namespace ebpf {
+
+class TableStorageImpl;
+class TableStorageIteratorImpl;
+
+class Path {
+ public:
+  static const std::string DELIM;
+  Path() = default;
+  Path(const Path &other) = default;
+  Path &operator=(const Path &other) = default;
+  Path(std::initializer_list<std::string> parts) {
+    size_t len = parts.size() * DELIM.size();
+    for (const auto &s : parts)
+      len += s.size();
+    path_.reserve(len);
+    for (const auto &s : parts)
+      path_ += DELIM + s;
+  }
+  const std::string &to_string() const { return path_; }
+
+ private:
+  std::string path_;
+};
+
+class TableStorage {
+ public:
+  /// iterator is an abstract class for traversing the map entries in a table
+  /// storage object.
+  class iterator {
+   private:
+    friend class TableStorage;
+    iterator(const iterator &);
+
+   public:
+    typedef std::pair<const std::string, TableDesc> value_type;
+    typedef std::ptrdiff_t difference_type;
+    typedef value_type *pointer;
+    typedef value_type &reference;
+    typedef std::forward_iterator_tag iterator_category;
+    typedef iterator self_type;
+
+    iterator();
+    iterator(std::unique_ptr<TableStorageIteratorImpl>);
+    ~iterator();
+    iterator(iterator &&);
+    iterator &operator=(iterator &&);
+    self_type &operator++();
+    self_type operator++(int);
+    bool operator==(const self_type &) const;
+    bool operator!=(const self_type &) const;
+    value_type &operator*() const;
+    pointer operator->() const;
+
+   private:
+    std::unique_ptr<TableStorageIteratorImpl> impl_;
+  };
+
+  TableStorage();
+  ~TableStorage();
+  void Init(std::unique_ptr<TableStorageImpl>);
+
+  bool Find(const Path &path, TableStorage::iterator &result) const;
+  bool Insert(const Path &path, TableDesc &&desc);
+  bool Delete(const Path &path);
+  size_t DeletePrefix(const Path &path);
+
+  void AddMapTypesVisitor(std::unique_ptr<MapTypesVisitor>);
+  void VisitMapType(TableDesc &desc, clang::ASTContext &C, clang::QualType key_type,
+                    clang::QualType leaf_type);
+  iterator begin();
+  iterator end();
+  iterator lower_bound(const Path &p);
+  iterator upper_bound(const Path &p);
+
+ private:
+  std::unique_ptr<TableStorageImpl> impl_;
+  std::vector<std::unique_ptr<MapTypesVisitor>> visitors_;
+};
+
+std::unique_ptr<TableStorage> createSharedTableStorage();
+std::unique_ptr<TableStorage> createBpfFsTableStorage();
+}

src/cc/table_storage_impl.h

+/*
+ * Copyright (c) 2017 VMware, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "table_storage.h"
+
+namespace ebpf {
+
+class TableStorageIteratorImpl {
+ public:
+  typedef std::pair<const std::string, TableDesc> value_type;
+  typedef value_type *pointer;
+  typedef value_type &reference;
+  typedef TableStorageIteratorImpl self_type;
+  virtual ~TableStorageIteratorImpl() {}
+  virtual std::unique_ptr<self_type> clone() const = 0;
+  virtual self_type &operator++() = 0;
+  virtual value_type &operator*() const = 0;
+  virtual pointer operator->() const = 0;
+
+ private:
+};
+
+class TableStorageImpl {
+ public:
+  virtual ~TableStorageImpl(){};
+  virtual bool Find(const std::string &name, TableStorage::iterator &result) const = 0;
+  virtual bool Insert(const std::string &name, TableDesc &&desc) = 0;
+  virtual bool Delete(const std::string &name) = 0;
+  virtual std::unique_ptr<TableStorageIteratorImpl> begin() = 0;
+  virtual std::unique_ptr<TableStorageIteratorImpl> end() = 0;
+  virtual std::unique_ptr<TableStorageIteratorImpl> lower_bound(const std::string &k) = 0;
+  virtual std::unique_ptr<TableStorageIteratorImpl> upper_bound(const std::string &k) = 0;
+  virtual std::unique_ptr<TableStorageIteratorImpl> erase(const TableStorageIteratorImpl &it) = 0;
+};
+
+}  // namespace ebpf