Commit 8a138aed authored by Martin KaFai Lau's avatar Martin KaFai Lau Committed by Daniel Borkmann

bpf: btf: Add BTF support to libbpf

If the ".BTF" elf section exists, libbpf will try to create
a btf_fd (through BPF_BTF_LOAD).  If that fails, it will still
continue loading the bpf prog/map without the BTF.

If the bpf_object has a BTF loaded, it will create a map with the btf_fd.
libbpf will try to figure out the btf_key_id and btf_value_id of a map by
finding the BTF type with name "<map_name>_key" and "<map_name>_value".
If they cannot be found, it will continue without using the BTF.
Signed-off-by: default avatarMartin KaFai Lau <kafai@fb.com>
Acked-by: default avatarAlexei Starovoitov <ast@fb.com>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
parent 3bd86a84
libbpf-y := libbpf.o bpf.o nlattr.o
libbpf-y := libbpf.o bpf.o nlattr.o btf.o
......@@ -73,43 +73,76 @@ static inline int sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr,
return syscall(__NR_bpf, cmd, attr, size);
}
int bpf_create_map_node(enum bpf_map_type map_type, const char *name,
int key_size, int value_size, int max_entries,
__u32 map_flags, int node)
int bpf_create_map_xattr(const struct bpf_create_map_attr *create_attr)
{
__u32 name_len = name ? strlen(name) : 0;
__u32 name_len = create_attr->name ? strlen(create_attr->name) : 0;
union bpf_attr attr;
memset(&attr, '\0', sizeof(attr));
attr.map_type = map_type;
attr.key_size = key_size;
attr.value_size = value_size;
attr.max_entries = max_entries;
attr.map_flags = map_flags;
memcpy(attr.map_name, name, min(name_len, BPF_OBJ_NAME_LEN - 1));
attr.map_type = create_attr->map_type;
attr.key_size = create_attr->key_size;
attr.value_size = create_attr->value_size;
attr.max_entries = create_attr->max_entries;
attr.map_flags = create_attr->map_flags;
memcpy(attr.map_name, create_attr->name,
min(name_len, BPF_OBJ_NAME_LEN - 1));
attr.numa_node = create_attr->numa_node;
attr.btf_fd = create_attr->btf_fd;
attr.btf_key_id = create_attr->btf_key_id;
attr.btf_value_id = create_attr->btf_value_id;
return sys_bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
}
int bpf_create_map_node(enum bpf_map_type map_type, const char *name,
int key_size, int value_size, int max_entries,
__u32 map_flags, int node)
{
struct bpf_create_map_attr map_attr = {};
map_attr.name = name;
map_attr.map_type = map_type;
map_attr.map_flags = map_flags;
map_attr.key_size = key_size;
map_attr.value_size = value_size;
map_attr.max_entries = max_entries;
if (node >= 0) {
attr.map_flags |= BPF_F_NUMA_NODE;
attr.numa_node = node;
map_attr.numa_node = node;
map_attr.map_flags |= BPF_F_NUMA_NODE;
}
return sys_bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
return bpf_create_map_xattr(&map_attr);
}
int bpf_create_map(enum bpf_map_type map_type, int key_size,
int value_size, int max_entries, __u32 map_flags)
{
return bpf_create_map_node(map_type, NULL, key_size, value_size,
max_entries, map_flags, -1);
struct bpf_create_map_attr map_attr = {};
map_attr.map_type = map_type;
map_attr.map_flags = map_flags;
map_attr.key_size = key_size;
map_attr.value_size = value_size;
map_attr.max_entries = max_entries;
return bpf_create_map_xattr(&map_attr);
}
int bpf_create_map_name(enum bpf_map_type map_type, const char *name,
int key_size, int value_size, int max_entries,
__u32 map_flags)
{
return bpf_create_map_node(map_type, name, key_size, value_size,
max_entries, map_flags, -1);
struct bpf_create_map_attr map_attr = {};
map_attr.name = name;
map_attr.map_type = map_type;
map_attr.map_flags = map_flags;
map_attr.key_size = key_size;
map_attr.value_size = value_size;
map_attr.max_entries = max_entries;
return bpf_create_map_xattr(&map_attr);
}
int bpf_create_map_in_map_node(enum bpf_map_type map_type, const char *name,
......@@ -573,3 +606,28 @@ int bpf_set_link_xdp_fd(int ifindex, int fd, __u32 flags)
close(sock);
return ret;
}
int bpf_load_btf(void *btf, __u32 btf_size, char *log_buf, __u32 log_buf_size,
bool do_log)
{
union bpf_attr attr = {};
int fd;
attr.btf = ptr_to_u64(btf);
attr.btf_size = btf_size;
retry:
if (do_log && log_buf && log_buf_size) {
attr.btf_log_level = 1;
attr.btf_log_size = log_buf_size;
attr.btf_log_buf = ptr_to_u64(log_buf);
}
fd = sys_bpf(BPF_BTF_LOAD, &attr, sizeof(attr));
if (fd == -1 && !do_log && log_buf && log_buf_size) {
do_log = true;
goto retry;
}
return fd;
}
......@@ -26,6 +26,20 @@
#include <linux/bpf.h>
#include <stddef.h>
struct bpf_create_map_attr {
const char *name;
enum bpf_map_type map_type;
__u32 map_flags;
__u32 key_size;
__u32 value_size;
__u32 max_entries;
__u32 numa_node;
__u32 btf_fd;
__u32 btf_key_id;
__u32 btf_value_id;
};
int bpf_create_map_xattr(const struct bpf_create_map_attr *create_attr);
int bpf_create_map_node(enum bpf_map_type map_type, const char *name,
int key_size, int value_size, int max_entries,
__u32 map_flags, int node);
......@@ -87,4 +101,6 @@ int bpf_obj_get_info_by_fd(int prog_fd, void *info, __u32 *info_len);
int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags,
__u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt);
int bpf_raw_tracepoint_open(const char *name, int prog_fd);
int bpf_load_btf(void *btf, __u32 btf_size, char *log_buf, __u32 log_buf_size,
bool do_log);
#endif
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2018 Facebook */
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <linux/err.h>
#include <linux/btf.h>
#include "btf.h"
#include "bpf.h"
#define elog(fmt, ...) { if (err_log) err_log(fmt, ##__VA_ARGS__); }
#define max(a, b) ((a) > (b) ? (a) : (b))
#define min(a, b) ((a) < (b) ? (a) : (b))
#define BTF_MAX_NR_TYPES 65535
static struct btf_type btf_void;
struct btf {
union {
struct btf_header *hdr;
void *data;
};
struct btf_type **types;
const char *strings;
void *nohdr_data;
uint32_t nr_types;
uint32_t types_size;
uint32_t data_size;
int fd;
};
static const char *btf_name_by_offset(const struct btf *btf, uint32_t offset)
{
if (!BTF_STR_TBL_ELF_ID(offset) &&
BTF_STR_OFFSET(offset) < btf->hdr->str_len)
return &btf->strings[BTF_STR_OFFSET(offset)];
else
return NULL;
}
static int btf_add_type(struct btf *btf, struct btf_type *t)
{
if (btf->types_size - btf->nr_types < 2) {
struct btf_type **new_types;
u32 expand_by, new_size;
if (btf->types_size == BTF_MAX_NR_TYPES)
return -E2BIG;
expand_by = max(btf->types_size >> 2, 16);
new_size = min(BTF_MAX_NR_TYPES, btf->types_size + expand_by);
new_types = realloc(btf->types, sizeof(*new_types) * new_size);
if (!new_types)
return -ENOMEM;
if (btf->nr_types == 0)
new_types[0] = &btf_void;
btf->types = new_types;
btf->types_size = new_size;
}
btf->types[++(btf->nr_types)] = t;
return 0;
}
static int btf_parse_hdr(struct btf *btf, btf_print_fn_t err_log)
{
const struct btf_header *hdr = btf->hdr;
u32 meta_left;
if (btf->data_size < sizeof(struct btf_header)) {
elog("BTF header not found\n");
return -EINVAL;
}
if (hdr->magic != BTF_MAGIC) {
elog("Invalid BTF magic:%x\n", hdr->magic);
return -EINVAL;
}
if (hdr->version != BTF_VERSION) {
elog("Unsupported BTF version:%u\n", hdr->version);
return -ENOTSUP;
}
if (hdr->flags) {
elog("Unsupported BTF flags:%x\n", hdr->flags);
return -ENOTSUP;
}
meta_left = btf->data_size - sizeof(*hdr);
if (!meta_left) {
elog("BTF has no data\n");
return -EINVAL;
}
if (meta_left < hdr->type_off) {
elog("Invalid BTF type section offset:%u\n", hdr->type_off);
return -EINVAL;
}
if (meta_left < hdr->str_off) {
elog("Invalid BTF string section offset:%u\n", hdr->str_off);
return -EINVAL;
}
if (hdr->type_off >= hdr->str_off) {
elog("BTF type section offset >= string section offset. No type?\n");
return -EINVAL;
}
if (hdr->type_off & 0x02) {
elog("BTF type section is not aligned to 4 bytes\n");
return -EINVAL;
}
btf->nohdr_data = btf->hdr + 1;
return 0;
}
static int btf_parse_str_sec(struct btf *btf, btf_print_fn_t err_log)
{
const struct btf_header *hdr = btf->hdr;
const char *start = btf->nohdr_data + hdr->str_off;
const char *end = start + btf->hdr->str_len;
if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_NAME_OFFSET ||
start[0] || end[-1]) {
elog("Invalid BTF string section\n");
return -EINVAL;
}
btf->strings = start;
return 0;
}
static int btf_parse_type_sec(struct btf *btf, btf_print_fn_t err_log)
{
struct btf_header *hdr = btf->hdr;
void *nohdr_data = btf->nohdr_data;
void *next_type = nohdr_data + hdr->type_off;
void *end_type = nohdr_data + hdr->str_off;
while (next_type < end_type) {
struct btf_type *t = next_type;
uint16_t vlen = BTF_INFO_VLEN(t->info);
int err;
next_type += sizeof(*t);
switch (BTF_INFO_KIND(t->info)) {
case BTF_KIND_INT:
next_type += sizeof(int);
break;
case BTF_KIND_ARRAY:
next_type += sizeof(struct btf_array);
break;
case BTF_KIND_STRUCT:
case BTF_KIND_UNION:
next_type += vlen * sizeof(struct btf_member);
break;
case BTF_KIND_ENUM:
next_type += vlen * sizeof(struct btf_enum);
break;
case BTF_KIND_TYPEDEF:
case BTF_KIND_PTR:
case BTF_KIND_FWD:
case BTF_KIND_VOLATILE:
case BTF_KIND_CONST:
case BTF_KIND_RESTRICT:
break;
default:
elog("Unsupported BTF_KIND:%u\n",
BTF_INFO_KIND(t->info));
return -EINVAL;
}
err = btf_add_type(btf, t);
if (err)
return err;
}
return 0;
}
static const struct btf_type *btf_type_by_id(const struct btf *btf,
uint32_t type_id)
{
if (type_id > btf->nr_types)
return NULL;
return btf->types[type_id];
}
static bool btf_type_is_void(const struct btf_type *t)
{
return t == &btf_void || BTF_INFO_KIND(t->info) == BTF_KIND_FWD;
}
static bool btf_type_is_void_or_null(const struct btf_type *t)
{
return !t || btf_type_is_void(t);
}
static int64_t btf_type_size(const struct btf_type *t)
{
switch (BTF_INFO_KIND(t->info)) {
case BTF_KIND_INT:
case BTF_KIND_STRUCT:
case BTF_KIND_UNION:
case BTF_KIND_ENUM:
return t->size;
case BTF_KIND_PTR:
return sizeof(void *);
default:
return -EINVAL;
}
}
#define MAX_RESOLVE_DEPTH 32
int64_t btf__resolve_size(const struct btf *btf, uint32_t type_id)
{
const struct btf_array *array;
const struct btf_type *t;
uint32_t nelems = 1;
int64_t size = -1;
int i;
t = btf_type_by_id(btf, type_id);
for (i = 0; i < MAX_RESOLVE_DEPTH && !btf_type_is_void_or_null(t);
i++) {
size = btf_type_size(t);
if (size >= 0)
break;
switch (BTF_INFO_KIND(t->info)) {
case BTF_KIND_TYPEDEF:
case BTF_KIND_VOLATILE:
case BTF_KIND_CONST:
case BTF_KIND_RESTRICT:
type_id = t->type;
break;
case BTF_KIND_ARRAY:
array = (const struct btf_array *)(t + 1);
if (nelems && array->nelems > UINT32_MAX / nelems)
return -E2BIG;
nelems *= array->nelems;
type_id = array->type;
break;
default:
return -EINVAL;
}
t = btf_type_by_id(btf, type_id);
}
if (size < 0)
return -EINVAL;
if (nelems && size > UINT32_MAX / nelems)
return -E2BIG;
return nelems * size;
}
int32_t btf__find_by_name(const struct btf *btf, const char *type_name)
{
uint32_t i;
if (!strcmp(type_name, "void"))
return 0;
for (i = 1; i <= btf->nr_types; i++) {
const struct btf_type *t = btf->types[i];
const char *name = btf_name_by_offset(btf, t->name);
if (name && !strcmp(type_name, name))
return i;
}
return -ENOENT;
}
void btf__free(struct btf *btf)
{
if (!btf)
return;
if (btf->fd != -1)
close(btf->fd);
free(btf->data);
free(btf->types);
free(btf);
}
struct btf *btf__new(uint8_t *data, uint32_t size,
btf_print_fn_t err_log)
{
uint32_t log_buf_size = 0;
char *log_buf = NULL;
struct btf *btf;
int err;
btf = calloc(1, sizeof(struct btf));
if (!btf)
return ERR_PTR(-ENOMEM);
btf->fd = -1;
if (err_log) {
log_buf = malloc(BPF_LOG_BUF_SIZE);
if (!log_buf) {
err = -ENOMEM;
goto done;
}
*log_buf = 0;
log_buf_size = BPF_LOG_BUF_SIZE;
}
btf->data = malloc(size);
if (!btf->data) {
err = -ENOMEM;
goto done;
}
memcpy(btf->data, data, size);
btf->data_size = size;
btf->fd = bpf_load_btf(btf->data, btf->data_size,
log_buf, log_buf_size, false);
if (btf->fd == -1) {
err = -errno;
elog("Error loading BTF: %s(%d)\n", strerror(errno), errno);
if (log_buf && *log_buf)
elog("%s\n", log_buf);
goto done;
}
err = btf_parse_hdr(btf, err_log);
if (err)
goto done;
err = btf_parse_str_sec(btf, err_log);
if (err)
goto done;
err = btf_parse_type_sec(btf, err_log);
done:
free(log_buf);
if (err) {
btf__free(btf);
return ERR_PTR(err);
}
return btf;
}
int btf__fd(const struct btf *btf)
{
return btf->fd;
}
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2018 Facebook */
#ifndef __BPF_BTF_H
#define __BPF_BTF_H
#include <stdint.h>
#define BTF_ELF_SEC ".BTF"
struct btf;
typedef int (*btf_print_fn_t)(const char *, ...)
__attribute__((format(printf, 1, 2)));
void btf__free(struct btf *btf);
struct btf *btf__new(uint8_t *data, uint32_t size, btf_print_fn_t err_log);
int32_t btf__find_by_name(const struct btf *btf, const char *type_name);
int64_t btf__resolve_size(const struct btf *btf, uint32_t type_id);
int btf__fd(const struct btf *btf);
#endif
......@@ -45,6 +45,7 @@
#include "libbpf.h"
#include "bpf.h"
#include "btf.h"
#ifndef EM_BPF
#define EM_BPF 247
......@@ -212,6 +213,8 @@ struct bpf_map {
char *name;
size_t offset;
struct bpf_map_def def;
uint32_t btf_key_id;
uint32_t btf_value_id;
void *priv;
bpf_map_clear_priv_t clear_priv;
};
......@@ -256,6 +259,8 @@ struct bpf_object {
*/
struct list_head list;
struct btf *btf;
void *priv;
bpf_object_clear_priv_t clear_priv;
......@@ -819,7 +824,15 @@ static int bpf_object__elf_collect(struct bpf_object *obj)
data->d_size);
else if (strcmp(name, "maps") == 0)
obj->efile.maps_shndx = idx;
else if (sh.sh_type == SHT_SYMTAB) {
else if (strcmp(name, BTF_ELF_SEC) == 0) {
obj->btf = btf__new(data->d_buf, data->d_size,
__pr_debug);
if (IS_ERR(obj->btf)) {
pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",
BTF_ELF_SEC, PTR_ERR(obj->btf));
obj->btf = NULL;
}
} else if (sh.sh_type == SHT_SYMTAB) {
if (obj->efile.symbols) {
pr_warning("bpf: multiple SYMTAB in %s\n",
obj->path);
......@@ -996,33 +1009,126 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
return 0;
}
static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf)
{
struct bpf_map_def *def = &map->def;
const size_t max_name = 256;
int64_t key_size, value_size;
int32_t key_id, value_id;
char name[max_name];
/* Find key type by name from BTF */
if (snprintf(name, max_name, "%s_key", map->name) == max_name) {
pr_warning("map:%s length of BTF key_type:%s_key is too long\n",
map->name, map->name);
return -EINVAL;
}
key_id = btf__find_by_name(btf, name);
if (key_id < 0) {
pr_debug("map:%s key_type:%s cannot be found in BTF\n",
map->name, name);
return key_id;
}
key_size = btf__resolve_size(btf, key_id);
if (key_size < 0) {
pr_warning("map:%s key_type:%s cannot get the BTF type_size\n",
map->name, name);
return key_size;
}
if (def->key_size != key_size) {
pr_warning("map:%s key_type:%s has BTF type_size:%ld != key_size:%u\n",
map->name, name, key_size, def->key_size);
return -EINVAL;
}
/* Find value type from BTF */
if (snprintf(name, max_name, "%s_value", map->name) == max_name) {
pr_warning("map:%s length of BTF value_type:%s_value is too long\n",
map->name, map->name);
return -EINVAL;
}
value_id = btf__find_by_name(btf, name);
if (value_id < 0) {
pr_debug("map:%s value_type:%s cannot be found in BTF\n",
map->name, name);
return value_id;
}
value_size = btf__resolve_size(btf, value_id);
if (value_size < 0) {
pr_warning("map:%s value_type:%s cannot get the BTF type_size\n",
map->name, name);
return value_size;
}
if (def->value_size != value_size) {
pr_warning("map:%s value_type:%s has BTF type_size:%ld != value_size:%u\n",
map->name, name, value_size, def->value_size);
return -EINVAL;
}
map->btf_key_id = key_id;
map->btf_value_id = value_id;
return 0;
}
static int
bpf_object__create_maps(struct bpf_object *obj)
{
struct bpf_create_map_attr create_attr = {};
unsigned int i;
int err;
for (i = 0; i < obj->nr_maps; i++) {
struct bpf_map_def *def = &obj->maps[i].def;
int *pfd = &obj->maps[i].fd;
*pfd = bpf_create_map_name(def->type,
obj->maps[i].name,
def->key_size,
def->value_size,
def->max_entries,
def->map_flags);
struct bpf_map *map = &obj->maps[i];
struct bpf_map_def *def = &map->def;
int *pfd = &map->fd;
create_attr.name = map->name;
create_attr.map_type = def->type;
create_attr.map_flags = def->map_flags;
create_attr.key_size = def->key_size;
create_attr.value_size = def->value_size;
create_attr.max_entries = def->max_entries;
create_attr.btf_fd = 0;
create_attr.btf_key_id = 0;
create_attr.btf_value_id = 0;
if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) {
create_attr.btf_fd = btf__fd(obj->btf);
create_attr.btf_key_id = map->btf_key_id;
create_attr.btf_value_id = map->btf_value_id;
}
*pfd = bpf_create_map_xattr(&create_attr);
if (*pfd < 0 && create_attr.btf_key_id) {
pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
map->name, strerror(errno), errno);
create_attr.btf_fd = 0;
create_attr.btf_key_id = 0;
create_attr.btf_value_id = 0;
map->btf_key_id = 0;
map->btf_value_id = 0;
*pfd = bpf_create_map_xattr(&create_attr);
}
if (*pfd < 0) {
size_t j;
int err = *pfd;
err = *pfd;
pr_warning("failed to create map (name: '%s'): %s\n",
obj->maps[i].name,
map->name,
strerror(errno));
for (j = 0; j < i; j++)
zclose(obj->maps[j].fd);
return err;
}
pr_debug("create map %s: fd=%d\n", obj->maps[i].name, *pfd);
pr_debug("create map %s: fd=%d\n", map->name, *pfd);
}
return 0;
......@@ -1641,6 +1747,7 @@ void bpf_object__close(struct bpf_object *obj)
bpf_object__elf_finish(obj);
bpf_object__unload(obj);
btf__free(obj->btf);
for (i = 0; i < obj->nr_maps; i++) {
zfree(&obj->maps[i].name);
......@@ -1692,6 +1799,11 @@ unsigned int bpf_object__kversion(struct bpf_object *obj)
return obj ? obj->kern_version : 0;
}
int bpf_object__btf_fd(const struct bpf_object *obj)
{
return obj->btf ? btf__fd(obj->btf) : -1;
}
int bpf_object__set_priv(struct bpf_object *obj, void *priv,
bpf_object_clear_priv_t clear_priv)
{
......@@ -1937,6 +2049,16 @@ const char *bpf_map__name(struct bpf_map *map)
return map ? map->name : NULL;
}
uint32_t bpf_map__btf_key_id(const struct bpf_map *map)
{
return map ? map->btf_key_id : 0;
}
uint32_t bpf_map__btf_value_id(const struct bpf_map *map)
{
return map ? map->btf_value_id : 0;
}
int bpf_map__set_priv(struct bpf_map *map, void *priv,
bpf_map_clear_priv_t clear_priv)
{
......
......@@ -78,6 +78,7 @@ int bpf_object__load(struct bpf_object *obj);
int bpf_object__unload(struct bpf_object *obj);
const char *bpf_object__name(struct bpf_object *obj);
unsigned int bpf_object__kversion(struct bpf_object *obj);
int bpf_object__btf_fd(const struct bpf_object *obj);
struct bpf_object *bpf_object__next(struct bpf_object *prev);
#define bpf_object__for_each_safe(pos, tmp) \
......@@ -241,6 +242,8 @@ bpf_map__next(struct bpf_map *map, struct bpf_object *obj);
int bpf_map__fd(struct bpf_map *map);
const struct bpf_map_def *bpf_map__def(struct bpf_map *map);
const char *bpf_map__name(struct bpf_map *map);
uint32_t bpf_map__btf_key_id(const struct bpf_map *map);
uint32_t bpf_map__btf_value_id(const struct bpf_map *map);
typedef void (*bpf_map_clear_priv_t)(struct bpf_map *, void *);
int bpf_map__set_priv(struct bpf_map *map, void *priv,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment