Commit 7fb4d48d authored by Andrii Nakryiko's avatar Andrii Nakryiko

Merge branch 'bpf: CO-RE support in the kernel'

Alexei Starovoitov says:

====================

From: Alexei Starovoitov <ast@kernel.org>

v4->v5:
. Reduce number of memory allocations in candidate cache logic
. Fix couple UAF issues
. Add Andrii's patch to cleanup struct bpf_core_cand
. More thorough tests
. Planned followups:
  - support -v in lskel
  - move struct bpf_core_spec out of bpf_core_apply_relo_insn to
    reduce stack usage
  - implement bpf_core_types_are_compat

v3->v4:
. complete refactor of find candidates logic.
  Now it has small permanent cache.
. Fix a bug in gen_loader related to attach_kind.
. Fix BTF log size limit.
. More tests.

v2->v3:
. addressed Andrii's feedback in every patch.
  New field in union bpf_attr changed from "core_relo" to "core_relos".
. added one more test and checkpatch.pl-ed the set.

v1->v2:
. Refactor uapi to pass 'struct bpf_core_relo' from LLVM into libbpf and further
into the kernel instead of bpf_core_apply_relo() bpf helper. Because of this
change the CO-RE algorithm has an ability to log error and debug events through
the standard bpf verifer log mechanism which was not possible with helper
approach.
. #define RELO_CORE macro was removed and replaced with btf_member_bit_offset() patch.

This set introduces CO-RE support in the kernel.
There are several reasons to add such support:
1. It's a step toward signed BPF programs.
2. It allows golang like languages that struggle to adopt libbpf
   to take advantage of CO-RE powers.
3. Currently the field accessed by 'ldx [R1 + 10]' insn is recognized
   by the verifier purely based on +10 offset. If R1 points to a union
   the verifier picks one of the fields at this offset.
   With CO-RE the kernel can disambiguate the field access.

Alexei Starovoitov (16):
  libbpf: Replace btf__type_by_id() with btf_type_by_id().
  bpf: Rename btf_member accessors.
  bpf: Prepare relo_core.c for kernel duty.
  bpf: Define enum bpf_core_relo_kind as uapi.
  bpf: Pass a set of bpf_core_relo-s to prog_load command.
  bpf: Adjust BTF log size limit.
  bpf: Add bpf_core_add_cands() and wire it into
    bpf_core_apply_relo_insn().
  libbpf: Use CO-RE in the kernel in light skeleton.
  libbpf: Support init of inner maps in light skeleton.
  libbpf: Clean gen_loader's attach kind.
  selftests/bpf: Add lskel version of kfunc test.
  selftests/bpf: Improve inner_map test coverage.
  selftests/bpf: Convert map_ptr_kern test to use light skeleton.
  selftests/bpf: Additional test for CO-RE in the kernel.
  selftests/bpf: Revert CO-RE removal in test_ksyms_weak.
  selftests/bpf: Add CO-RE relocations to verifier scale test.
====================
Signed-off-by: default avatarAndrii Nakryiko <andrii@kernel.org>
parents 64b5b97b 098dc533
......@@ -1732,6 +1732,14 @@ bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);
const struct btf_func_model *
bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
const struct bpf_insn *insn);
struct bpf_core_ctx {
struct bpf_verifier_log *log;
const struct btf *btf;
};
int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo,
int relo_idx, void *insn);
#else /* !CONFIG_BPF_SYSCALL */
static inline struct bpf_prog *bpf_prog_get(u32 ufd)
{
......
......@@ -144,6 +144,53 @@ static inline bool btf_type_is_enum(const struct btf_type *t)
return BTF_INFO_KIND(t->info) == BTF_KIND_ENUM;
}
static inline bool str_is_empty(const char *s)
{
return !s || !s[0];
}
static inline u16 btf_kind(const struct btf_type *t)
{
return BTF_INFO_KIND(t->info);
}
static inline bool btf_is_enum(const struct btf_type *t)
{
return btf_kind(t) == BTF_KIND_ENUM;
}
static inline bool btf_is_composite(const struct btf_type *t)
{
u16 kind = btf_kind(t);
return kind == BTF_KIND_STRUCT || kind == BTF_KIND_UNION;
}
static inline bool btf_is_array(const struct btf_type *t)
{
return btf_kind(t) == BTF_KIND_ARRAY;
}
static inline bool btf_is_int(const struct btf_type *t)
{
return btf_kind(t) == BTF_KIND_INT;
}
static inline bool btf_is_ptr(const struct btf_type *t)
{
return btf_kind(t) == BTF_KIND_PTR;
}
static inline u8 btf_int_offset(const struct btf_type *t)
{
return BTF_INT_OFFSET(*(u32 *)(t + 1));
}
static inline u8 btf_int_encoding(const struct btf_type *t)
{
return BTF_INT_ENCODING(*(u32 *)(t + 1));
}
static inline bool btf_type_is_scalar(const struct btf_type *t)
{
return btf_type_is_int(t) || btf_type_is_enum(t);
......@@ -184,6 +231,11 @@ static inline u16 btf_type_vlen(const struct btf_type *t)
return BTF_INFO_VLEN(t->info);
}
static inline u16 btf_vlen(const struct btf_type *t)
{
return btf_type_vlen(t);
}
static inline u16 btf_func_linkage(const struct btf_type *t)
{
return BTF_INFO_VLEN(t->info);
......@@ -194,25 +246,54 @@ static inline bool btf_type_kflag(const struct btf_type *t)
return BTF_INFO_KFLAG(t->info);
}
static inline u32 btf_member_bit_offset(const struct btf_type *struct_type,
const struct btf_member *member)
static inline u32 __btf_member_bit_offset(const struct btf_type *struct_type,
const struct btf_member *member)
{
return btf_type_kflag(struct_type) ? BTF_MEMBER_BIT_OFFSET(member->offset)
: member->offset;
}
static inline u32 btf_member_bitfield_size(const struct btf_type *struct_type,
const struct btf_member *member)
static inline u32 __btf_member_bitfield_size(const struct btf_type *struct_type,
const struct btf_member *member)
{
return btf_type_kflag(struct_type) ? BTF_MEMBER_BITFIELD_SIZE(member->offset)
: 0;
}
static inline struct btf_member *btf_members(const struct btf_type *t)
{
return (struct btf_member *)(t + 1);
}
static inline u32 btf_member_bit_offset(const struct btf_type *t, u32 member_idx)
{
const struct btf_member *m = btf_members(t) + member_idx;
return __btf_member_bit_offset(t, m);
}
static inline u32 btf_member_bitfield_size(const struct btf_type *t, u32 member_idx)
{
const struct btf_member *m = btf_members(t) + member_idx;
return __btf_member_bitfield_size(t, m);
}
static inline const struct btf_member *btf_type_member(const struct btf_type *t)
{
return (const struct btf_member *)(t + 1);
}
static inline struct btf_array *btf_array(const struct btf_type *t)
{
return (struct btf_array *)(t + 1);
}
static inline struct btf_enum *btf_enum(const struct btf_type *t)
{
return (struct btf_enum *)(t + 1);
}
static inline const struct btf_var_secinfo *btf_type_var_secinfo(
const struct btf_type *t)
{
......
......@@ -1342,8 +1342,10 @@ union bpf_attr {
/* or valid module BTF object fd or 0 to attach to vmlinux */
__u32 attach_btf_obj_fd;
};
__u32 :32; /* pad */
__u32 core_relo_cnt; /* number of bpf_core_relo */
__aligned_u64 fd_array; /* array of FDs */
__aligned_u64 core_relos;
__u32 core_relo_rec_size; /* sizeof(struct bpf_core_relo) */
};
struct { /* anonymous struct used by BPF_OBJ_* commands */
......@@ -6374,4 +6376,78 @@ enum {
BTF_F_ZERO = (1ULL << 3),
};
/* bpf_core_relo_kind encodes which aspect of captured field/type/enum value
* has to be adjusted by relocations. It is emitted by llvm and passed to
* libbpf and later to the kernel.
*/
enum bpf_core_relo_kind {
BPF_CORE_FIELD_BYTE_OFFSET = 0, /* field byte offset */
BPF_CORE_FIELD_BYTE_SIZE = 1, /* field size in bytes */
BPF_CORE_FIELD_EXISTS = 2, /* field existence in target kernel */
BPF_CORE_FIELD_SIGNED = 3, /* field signedness (0 - unsigned, 1 - signed) */
BPF_CORE_FIELD_LSHIFT_U64 = 4, /* bitfield-specific left bitshift */
BPF_CORE_FIELD_RSHIFT_U64 = 5, /* bitfield-specific right bitshift */
BPF_CORE_TYPE_ID_LOCAL = 6, /* type ID in local BPF object */
BPF_CORE_TYPE_ID_TARGET = 7, /* type ID in target kernel */
BPF_CORE_TYPE_EXISTS = 8, /* type existence in target kernel */
BPF_CORE_TYPE_SIZE = 9, /* type size in bytes */
BPF_CORE_ENUMVAL_EXISTS = 10, /* enum value existence in target kernel */
BPF_CORE_ENUMVAL_VALUE = 11, /* enum value integer value */
};
/*
* "struct bpf_core_relo" is used to pass relocation data form LLVM to libbpf
* and from libbpf to the kernel.
*
* CO-RE relocation captures the following data:
* - insn_off - instruction offset (in bytes) within a BPF program that needs
* its insn->imm field to be relocated with actual field info;
* - type_id - BTF type ID of the "root" (containing) entity of a relocatable
* type or field;
* - access_str_off - offset into corresponding .BTF string section. String
* interpretation depends on specific relocation kind:
* - for field-based relocations, string encodes an accessed field using
* a sequence of field and array indices, separated by colon (:). It's
* conceptually very close to LLVM's getelementptr ([0]) instruction's
* arguments for identifying offset to a field.
* - for type-based relocations, strings is expected to be just "0";
* - for enum value-based relocations, string contains an index of enum
* value within its enum type;
* - kind - one of enum bpf_core_relo_kind;
*
* Example:
* struct sample {
* int a;
* struct {
* int b[10];
* };
* };
*
* struct sample *s = ...;
* int *x = &s->a; // encoded as "0:0" (a is field #0)
* int *y = &s->b[5]; // encoded as "0:1:0:5" (anon struct is field #1,
* // b is field #0 inside anon struct, accessing elem #5)
* int *z = &s[10]->b; // encoded as "10:1" (ptr is used as an array)
*
* type_id for all relocs in this example will capture BTF type id of
* `struct sample`.
*
* Such relocation is emitted when using __builtin_preserve_access_index()
* Clang built-in, passing expression that captures field address, e.g.:
*
* bpf_probe_read(&dst, sizeof(dst),
* __builtin_preserve_access_index(&src->a.b.c));
*
* In this case Clang will emit field relocation recording necessary data to
* be able to find offset of embedded `a.b.c` field within `src` struct.
*
* [0] https://llvm.org/docs/LangRef.html#getelementptr-instruction
*/
struct bpf_core_relo {
__u32 insn_off;
__u32 type_id;
__u32 access_str_off;
enum bpf_core_relo_kind kind;
};
#endif /* _UAPI__LINUX_BPF_H__ */
......@@ -36,3 +36,7 @@ obj-$(CONFIG_BPF_SYSCALL) += bpf_struct_ops.o
obj-${CONFIG_BPF_LSM} += bpf_lsm.o
endif
obj-$(CONFIG_BPF_PRELOAD) += preload/
obj-$(CONFIG_BPF_SYSCALL) += relo_core.o
$(obj)/relo_core.o: $(srctree)/tools/lib/bpf/relo_core.c FORCE
$(call if_changed_rule,cc_o_c)
......@@ -165,7 +165,7 @@ void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log)
break;
}
if (btf_member_bitfield_size(t, member)) {
if (__btf_member_bitfield_size(t, member)) {
pr_warn("bit field member %s in struct %s is not supported\n",
mname, st_ops->name);
break;
......@@ -296,7 +296,7 @@ static int check_zero_holes(const struct btf_type *t, void *data)
const struct btf_type *mtype;
for_each_member(i, t, member) {
moff = btf_member_bit_offset(t, member) / 8;
moff = __btf_member_bit_offset(t, member) / 8;
if (moff > prev_mend &&
memchr_inv(data + prev_mend, 0, moff - prev_mend))
return -EINVAL;
......@@ -387,7 +387,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
struct bpf_prog *prog;
u32 moff;
moff = btf_member_bit_offset(t, member) / 8;
moff = __btf_member_bit_offset(t, member) / 8;
ptype = btf_type_resolve_ptr(btf_vmlinux, member->type, NULL);
if (ptype == module_type) {
if (*(void **)(udata + moff))
......
This diff is collapsed.
......@@ -2184,7 +2184,7 @@ static bool is_perfmon_prog_type(enum bpf_prog_type prog_type)
}
/* last field in 'union bpf_attr' used by this command */
#define BPF_PROG_LOAD_LAST_FIELD fd_array
#define BPF_PROG_LOAD_LAST_FIELD core_relo_rec_size
static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr)
{
......
......@@ -10273,6 +10273,78 @@ static int check_btf_line(struct bpf_verifier_env *env,
return err;
}
#define MIN_CORE_RELO_SIZE sizeof(struct bpf_core_relo)
#define MAX_CORE_RELO_SIZE MAX_FUNCINFO_REC_SIZE
static int check_core_relo(struct bpf_verifier_env *env,
const union bpf_attr *attr,
bpfptr_t uattr)
{
u32 i, nr_core_relo, ncopy, expected_size, rec_size;
struct bpf_core_relo core_relo = {};
struct bpf_prog *prog = env->prog;
const struct btf *btf = prog->aux->btf;
struct bpf_core_ctx ctx = {
.log = &env->log,
.btf = btf,
};
bpfptr_t u_core_relo;
int err;
nr_core_relo = attr->core_relo_cnt;
if (!nr_core_relo)
return 0;
if (nr_core_relo > INT_MAX / sizeof(struct bpf_core_relo))
return -EINVAL;
rec_size = attr->core_relo_rec_size;
if (rec_size < MIN_CORE_RELO_SIZE ||
rec_size > MAX_CORE_RELO_SIZE ||
rec_size % sizeof(u32))
return -EINVAL;
u_core_relo = make_bpfptr(attr->core_relos, uattr.is_kernel);
expected_size = sizeof(struct bpf_core_relo);
ncopy = min_t(u32, expected_size, rec_size);
/* Unlike func_info and line_info, copy and apply each CO-RE
* relocation record one at a time.
*/
for (i = 0; i < nr_core_relo; i++) {
/* future proofing when sizeof(bpf_core_relo) changes */
err = bpf_check_uarg_tail_zero(u_core_relo, expected_size, rec_size);
if (err) {
if (err == -E2BIG) {
verbose(env, "nonzero tailing record in core_relo");
if (copy_to_bpfptr_offset(uattr,
offsetof(union bpf_attr, core_relo_rec_size),
&expected_size, sizeof(expected_size)))
err = -EFAULT;
}
break;
}
if (copy_from_bpfptr(&core_relo, u_core_relo, ncopy)) {
err = -EFAULT;
break;
}
if (core_relo.insn_off % 8 || core_relo.insn_off / 8 >= prog->len) {
verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n",
i, core_relo.insn_off, prog->len);
err = -EINVAL;
break;
}
err = bpf_core_apply(&ctx, &core_relo, i,
&prog->insnsi[core_relo.insn_off / 8]);
if (err)
break;
bpfptr_add(&u_core_relo, rec_size);
}
return err;
}
static int check_btf_info(struct bpf_verifier_env *env,
const union bpf_attr *attr,
bpfptr_t uattr)
......@@ -10303,6 +10375,10 @@ static int check_btf_info(struct bpf_verifier_env *env,
if (err)
return err;
err = check_core_relo(env, attr, uattr);
if (err)
return err;
return 0;
}
......
......@@ -169,7 +169,7 @@ static u32 prog_ops_moff(const struct bpf_prog *prog)
t = bpf_tcp_congestion_ops.type;
m = &btf_type_member(t)[midx];
return btf_member_bit_offset(t, m) / 8;
return __btf_member_bit_offset(t, m) / 8;
}
static const struct bpf_func_proto *
......@@ -244,7 +244,7 @@ static int bpf_tcp_ca_init_member(const struct btf_type *t,
utcp_ca = (const struct tcp_congestion_ops *)udata;
tcp_ca = (struct tcp_congestion_ops *)kdata;
moff = btf_member_bit_offset(t, member) / 8;
moff = __btf_member_bit_offset(t, member) / 8;
switch (moff) {
case offsetof(struct tcp_congestion_ops, flags):
if (utcp_ca->flags & ~TCP_CONG_MASK)
......@@ -274,7 +274,7 @@ static int bpf_tcp_ca_init_member(const struct btf_type *t,
static int bpf_tcp_ca_check_member(const struct btf_type *t,
const struct btf_member *member)
{
if (is_unsupported(btf_member_bit_offset(t, member) / 8))
if (is_unsupported(__btf_member_bit_offset(t, member) / 8))
return -ENOTSUPP;
return 0;
}
......
......@@ -1342,8 +1342,10 @@ union bpf_attr {
/* or valid module BTF object fd or 0 to attach to vmlinux */
__u32 attach_btf_obj_fd;
};
__u32 :32; /* pad */
__u32 core_relo_cnt; /* number of bpf_core_relo */
__aligned_u64 fd_array; /* array of FDs */
__aligned_u64 core_relos;
__u32 core_relo_rec_size; /* sizeof(struct bpf_core_relo) */
};
struct { /* anonymous struct used by BPF_OBJ_* commands */
......@@ -6374,4 +6376,78 @@ enum {
BTF_F_ZERO = (1ULL << 3),
};
/* bpf_core_relo_kind encodes which aspect of captured field/type/enum value
* has to be adjusted by relocations. It is emitted by llvm and passed to
* libbpf and later to the kernel.
*/
enum bpf_core_relo_kind {
BPF_CORE_FIELD_BYTE_OFFSET = 0, /* field byte offset */
BPF_CORE_FIELD_BYTE_SIZE = 1, /* field size in bytes */
BPF_CORE_FIELD_EXISTS = 2, /* field existence in target kernel */
BPF_CORE_FIELD_SIGNED = 3, /* field signedness (0 - unsigned, 1 - signed) */
BPF_CORE_FIELD_LSHIFT_U64 = 4, /* bitfield-specific left bitshift */
BPF_CORE_FIELD_RSHIFT_U64 = 5, /* bitfield-specific right bitshift */
BPF_CORE_TYPE_ID_LOCAL = 6, /* type ID in local BPF object */
BPF_CORE_TYPE_ID_TARGET = 7, /* type ID in target kernel */
BPF_CORE_TYPE_EXISTS = 8, /* type existence in target kernel */
BPF_CORE_TYPE_SIZE = 9, /* type size in bytes */
BPF_CORE_ENUMVAL_EXISTS = 10, /* enum value existence in target kernel */
BPF_CORE_ENUMVAL_VALUE = 11, /* enum value integer value */
};
/*
* "struct bpf_core_relo" is used to pass relocation data form LLVM to libbpf
* and from libbpf to the kernel.
*
* CO-RE relocation captures the following data:
* - insn_off - instruction offset (in bytes) within a BPF program that needs
* its insn->imm field to be relocated with actual field info;
* - type_id - BTF type ID of the "root" (containing) entity of a relocatable
* type or field;
* - access_str_off - offset into corresponding .BTF string section. String
* interpretation depends on specific relocation kind:
* - for field-based relocations, string encodes an accessed field using
* a sequence of field and array indices, separated by colon (:). It's
* conceptually very close to LLVM's getelementptr ([0]) instruction's
* arguments for identifying offset to a field.
* - for type-based relocations, strings is expected to be just "0";
* - for enum value-based relocations, string contains an index of enum
* value within its enum type;
* - kind - one of enum bpf_core_relo_kind;
*
* Example:
* struct sample {
* int a;
* struct {
* int b[10];
* };
* };
*
* struct sample *s = ...;
* int *x = &s->a; // encoded as "0:0" (a is field #0)
* int *y = &s->b[5]; // encoded as "0:1:0:5" (anon struct is field #1,
* // b is field #0 inside anon struct, accessing elem #5)
* int *z = &s[10]->b; // encoded as "10:1" (ptr is used as an array)
*
* type_id for all relocs in this example will capture BTF type id of
* `struct sample`.
*
* Such relocation is emitted when using __builtin_preserve_access_index()
* Clang built-in, passing expression that captures field address, e.g.:
*
* bpf_probe_read(&dst, sizeof(dst),
* __builtin_preserve_access_index(&src->a.b.c));
*
* In this case Clang will emit field relocation recording necessary data to
* be able to find offset of embedded `a.b.c` field within `src` struct.
*
* [0] https://llvm.org/docs/LangRef.html#getelementptr-instruction
*/
struct bpf_core_relo {
__u32 insn_off;
__u32 type_id;
__u32 access_str_off;
enum bpf_core_relo_kind kind;
};
#endif /* _UAPI__LINUX_BPF_H__ */
......@@ -39,6 +39,8 @@ struct bpf_gen {
int error;
struct ksym_relo_desc *relos;
int relo_cnt;
struct bpf_core_relo *core_relos;
int core_relo_cnt;
char attach_target[128];
int attach_kind;
struct ksym_desc *ksyms;
......@@ -64,5 +66,7 @@ void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx);
void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *name, enum bpf_attach_type type);
void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak,
bool is_typeless, int kind, int insn_idx);
void bpf_gen__record_relo_core(struct bpf_gen *gen, const struct bpf_core_relo *core_relo);
void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int key, int inner_map_idx);
#endif
......@@ -454,7 +454,7 @@ const struct btf *btf__base_btf(const struct btf *btf)
}
/* internal helper returning non-const pointer to a type */
struct btf_type *btf_type_by_id(struct btf *btf, __u32 type_id)
struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id)
{
if (type_id == 0)
return &btf_void;
......
......@@ -829,6 +829,22 @@ static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo,
emit_ksym_relo_log(gen, relo, kdesc->ref);
}
void bpf_gen__record_relo_core(struct bpf_gen *gen,
const struct bpf_core_relo *core_relo)
{
struct bpf_core_relo *relos;
relos = libbpf_reallocarray(gen->core_relos, gen->core_relo_cnt + 1, sizeof(*relos));
if (!relos) {
gen->error = -ENOMEM;
return;
}
gen->core_relos = relos;
relos += gen->core_relo_cnt;
memcpy(relos, core_relo, sizeof(*relos));
gen->core_relo_cnt++;
}
static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns)
{
int insn;
......@@ -861,6 +877,15 @@ static void emit_relos(struct bpf_gen *gen, int insns)
emit_relo(gen, gen->relos + i, insns);
}
static void cleanup_core_relo(struct bpf_gen *gen)
{
if (!gen->core_relo_cnt)
return;
free(gen->core_relos);
gen->core_relo_cnt = 0;
gen->core_relos = NULL;
}
static void cleanup_relos(struct bpf_gen *gen, int insns)
{
int i, insn;
......@@ -888,6 +913,7 @@ static void cleanup_relos(struct bpf_gen *gen, int insns)
gen->relo_cnt = 0;
gen->relos = NULL;
}
cleanup_core_relo(gen);
}
void bpf_gen__prog_load(struct bpf_gen *gen,
......@@ -895,12 +921,13 @@ void bpf_gen__prog_load(struct bpf_gen *gen,
const char *license, struct bpf_insn *insns, size_t insn_cnt,
struct bpf_prog_load_opts *load_attr, int prog_idx)
{
int attr_size = offsetofend(union bpf_attr, fd_array);
int prog_load_attr, license_off, insns_off, func_info, line_info;
int prog_load_attr, license_off, insns_off, func_info, line_info, core_relos;
int attr_size = offsetofend(union bpf_attr, core_relo_rec_size);
union bpf_attr attr;
memset(&attr, 0, attr_size);
pr_debug("gen: prog_load: type %d insns_cnt %zd\n", prog_type, insn_cnt);
pr_debug("gen: prog_load: type %d insns_cnt %zd progi_idx %d\n",
prog_type, insn_cnt, prog_idx);
/* add license string to blob of bytes */
license_off = add_data(gen, license, strlen(license) + 1);
/* add insns to blob of bytes */
......@@ -924,6 +951,11 @@ void bpf_gen__prog_load(struct bpf_gen *gen,
line_info = add_data(gen, load_attr->line_info,
attr.line_info_cnt * attr.line_info_rec_size);
attr.core_relo_rec_size = sizeof(struct bpf_core_relo);
attr.core_relo_cnt = gen->core_relo_cnt;
core_relos = add_data(gen, gen->core_relos,
attr.core_relo_cnt * attr.core_relo_rec_size);
memcpy(attr.prog_name, prog_name,
min((unsigned)strlen(prog_name), BPF_OBJ_NAME_LEN - 1));
prog_load_attr = add_data(gen, &attr, attr_size);
......@@ -940,6 +972,9 @@ void bpf_gen__prog_load(struct bpf_gen *gen,
/* populate union bpf_attr with a pointer to line_info */
emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info);
/* populate union bpf_attr with a pointer to core_relos */
emit_rel_store(gen, attr_field(prog_load_attr, core_relos), core_relos);
/* populate union bpf_attr fd_array with a pointer to data where map_fds are saved */
emit_rel_store(gen, attr_field(prog_load_attr, fd_array), gen->fd_array);
......@@ -970,9 +1005,11 @@ void bpf_gen__prog_load(struct bpf_gen *gen,
debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt);
/* successful or not, close btf module FDs used in extern ksyms and attach_btf_obj_fd */
cleanup_relos(gen, insns_off);
if (gen->attach_kind)
if (gen->attach_kind) {
emit_sys_close_blob(gen,
attr_field(prog_load_attr, attach_btf_obj_fd));
gen->attach_kind = 0;
}
emit_check_err(gen);
/* remember prog_fd in the stack, if successful */
emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,
......@@ -1018,6 +1055,33 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue,
emit_check_err(gen);
}
void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slot,
int inner_map_idx)
{
int attr_size = offsetofend(union bpf_attr, flags);
int map_update_attr, key;
union bpf_attr attr;
memset(&attr, 0, attr_size);
pr_debug("gen: populate_outer_map: outer %d key %d inner %d\n",
outer_map_idx, slot, inner_map_idx);
key = add_data(gen, &slot, sizeof(slot));
map_update_attr = add_data(gen, &attr, attr_size);
move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
blob_fd_array_off(gen, outer_map_idx));
emit_rel_store(gen, attr_field(map_update_attr, key), key);
emit_rel_store(gen, attr_field(map_update_attr, value),
blob_fd_array_off(gen, inner_map_idx));
/* emit MAP_UPDATE_ELEM command */
emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size);
debug_ret(gen, "populate_outer_map outer %d key %d inner %d",
outer_map_idx, slot, inner_map_idx);
emit_check_err(gen);
}
void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx)
{
int attr_size = offsetofend(union bpf_attr, map_fd);
......
......@@ -230,13 +230,19 @@ enum reloc_type {
RELO_EXTERN_VAR,
RELO_EXTERN_FUNC,
RELO_SUBPROG_ADDR,
RELO_CORE,
};
struct reloc_desc {
enum reloc_type type;
int insn_idx;
int map_idx;
int sym_off;
union {
const struct bpf_core_relo *core_relo; /* used when type == RELO_CORE */
struct {
int map_idx;
int sym_off;
};
};
};
struct bpf_sec_def;
......@@ -4965,9 +4971,9 @@ static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map)
fd = bpf_map__fd(targ_map);
if (obj->gen_loader) {
pr_warn("// TODO map_update_elem: idx %td key %d value==map_idx %td\n",
map - obj->maps, i, targ_map - obj->maps);
return -ENOTSUP;
bpf_gen__populate_outer_map(obj->gen_loader,
map - obj->maps, i,
targ_map - obj->maps);
} else {
err = bpf_map_update_elem(map->fd, &i, &fd, 0);
}
......@@ -5179,15 +5185,18 @@ static int bpf_core_add_cands(struct bpf_core_cand *local_cand,
struct bpf_core_cand_list *cands)
{
struct bpf_core_cand *new_cands, *cand;
const struct btf_type *t;
const char *targ_name;
const struct btf_type *t, *local_t;
const char *targ_name, *local_name;
size_t targ_essent_len;
int n, i;
local_t = btf__type_by_id(local_cand->btf, local_cand->id);
local_name = btf__str_by_offset(local_cand->btf, local_t->name_off);
n = btf__type_cnt(targ_btf);
for (i = targ_start_id; i < n; i++) {
t = btf__type_by_id(targ_btf, i);
if (btf_kind(t) != btf_kind(local_cand->t))
if (btf_kind(t) != btf_kind(local_t))
continue;
targ_name = btf__name_by_offset(targ_btf, t->name_off);
......@@ -5198,12 +5207,12 @@ static int bpf_core_add_cands(struct bpf_core_cand *local_cand,
if (targ_essent_len != local_essent_len)
continue;
if (strncmp(local_cand->name, targ_name, local_essent_len) != 0)
if (strncmp(local_name, targ_name, local_essent_len) != 0)
continue;
pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
local_cand->id, btf_kind_str(local_cand->t),
local_cand->name, i, btf_kind_str(t), targ_name,
local_cand->id, btf_kind_str(local_t),
local_name, i, btf_kind_str(t), targ_name,
targ_btf_name);
new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
sizeof(*cands->cands));
......@@ -5212,8 +5221,6 @@ static int bpf_core_add_cands(struct bpf_core_cand *local_cand,
cand = &new_cands[cands->len];
cand->btf = targ_btf;
cand->t = t;
cand->name = targ_name;
cand->id = i;
cands->cands = new_cands;
......@@ -5320,18 +5327,21 @@ bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 l
struct bpf_core_cand local_cand = {};
struct bpf_core_cand_list *cands;
const struct btf *main_btf;
const struct btf_type *local_t;
const char *local_name;
size_t local_essent_len;
int err, i;
local_cand.btf = local_btf;
local_cand.t = btf__type_by_id(local_btf, local_type_id);
if (!local_cand.t)
local_cand.id = local_type_id;
local_t = btf__type_by_id(local_btf, local_type_id);
if (!local_t)
return ERR_PTR(-EINVAL);
local_cand.name = btf__name_by_offset(local_btf, local_cand.t->name_off);
if (str_is_empty(local_cand.name))
local_name = btf__name_by_offset(local_btf, local_t->name_off);
if (str_is_empty(local_name))
return ERR_PTR(-EINVAL);
local_essent_len = bpf_core_essential_name_len(local_cand.name);
local_essent_len = bpf_core_essential_name_len(local_name);
cands = calloc(1, sizeof(*cands));
if (!cands)
......@@ -5481,6 +5491,24 @@ static void *u32_as_hash_key(__u32 x)
return (void *)(uintptr_t)x;
}
static int record_relo_core(struct bpf_program *prog,
const struct bpf_core_relo *core_relo, int insn_idx)
{
struct reloc_desc *relos, *relo;
relos = libbpf_reallocarray(prog->reloc_desc,
prog->nr_reloc + 1, sizeof(*relos));
if (!relos)
return -ENOMEM;
relo = &relos[prog->nr_reloc];
relo->type = RELO_CORE;
relo->insn_idx = insn_idx;
relo->core_relo = core_relo;
prog->reloc_desc = relos;
prog->nr_reloc++;
return 0;
}
static int bpf_core_apply_relo(struct bpf_program *prog,
const struct bpf_core_relo *relo,
int relo_idx,
......@@ -5517,13 +5545,15 @@ static int bpf_core_apply_relo(struct bpf_program *prog,
return -EINVAL;
if (prog->obj->gen_loader) {
pr_warn("// TODO core_relo: prog %td insn[%d] %s kind %d\n",
const char *spec_str = btf__name_by_offset(local_btf, relo->access_str_off);
pr_debug("record_relo_core: prog %td insn[%d] %s %s %s final insn_idx %d\n",
prog - prog->obj->programs, relo->insn_off / 8,
local_name, relo->kind);
return -ENOTSUP;
btf_kind_str(local_type), local_name, spec_str, insn_idx);
return record_relo_core(prog, relo, insn_idx);
}
if (relo->kind != BPF_TYPE_ID_LOCAL &&
if (relo->kind != BPF_CORE_TYPE_ID_LOCAL &&
!hashmap__find(cand_cache, type_key, (void **)&cands)) {
cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
if (IS_ERR(cands)) {
......@@ -5725,6 +5755,9 @@ bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
case RELO_CALL:
/* handled already */
break;
case RELO_CORE:
/* will be handled by bpf_program_record_relos() */
break;
default:
pr_warn("prog '%s': relo #%d: bad relo type %d\n",
prog->name, i, relo->type);
......@@ -6165,6 +6198,35 @@ bpf_object__free_relocs(struct bpf_object *obj)
}
}
static int cmp_relocs(const void *_a, const void *_b)
{
const struct reloc_desc *a = _a;
const struct reloc_desc *b = _b;
if (a->insn_idx != b->insn_idx)
return a->insn_idx < b->insn_idx ? -1 : 1;
/* no two relocations should have the same insn_idx, but ... */
if (a->type != b->type)
return a->type < b->type ? -1 : 1;
return 0;
}
static void bpf_object__sort_relos(struct bpf_object *obj)
{
int i;
for (i = 0; i < obj->nr_programs; i++) {
struct bpf_program *p = &obj->programs[i];
if (!p->nr_reloc)
continue;
qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
}
}
static int
bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)
{
......@@ -6179,6 +6241,8 @@ bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)
err);
return err;
}
if (obj->gen_loader)
bpf_object__sort_relos(obj);
}
/* Before relocating calls pre-process relocations and mark
......@@ -6383,21 +6447,6 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj,
return 0;
}
static int cmp_relocs(const void *_a, const void *_b)
{
const struct reloc_desc *a = _a;
const struct reloc_desc *b = _b;
if (a->insn_idx != b->insn_idx)
return a->insn_idx < b->insn_idx ? -1 : 1;
/* no two relocations should have the same insn_idx, but ... */
if (a->type != b->type)
return a->type < b->type ? -1 : 1;
return 0;
}
static int bpf_object__collect_relos(struct bpf_object *obj)
{
int i, err;
......@@ -6430,14 +6479,7 @@ static int bpf_object__collect_relos(struct bpf_object *obj)
return err;
}
for (i = 0; i < obj->nr_programs; i++) {
struct bpf_program *p = &obj->programs[i];
if (!p->nr_reloc)
continue;
qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
}
bpf_object__sort_relos(obj);
return 0;
}
......@@ -6679,7 +6721,7 @@ static int bpf_object_load_prog_instance(struct bpf_object *obj, struct bpf_prog
return ret;
}
static int bpf_program__record_externs(struct bpf_program *prog)
static int bpf_program_record_relos(struct bpf_program *prog)
{
struct bpf_object *obj = prog->obj;
int i;
......@@ -6701,6 +6743,17 @@ static int bpf_program__record_externs(struct bpf_program *prog)
ext->is_weak, false, BTF_KIND_FUNC,
relo->insn_idx);
break;
case RELO_CORE: {
struct bpf_core_relo cr = {
.insn_off = relo->insn_idx * 8,
.type_id = relo->core_relo->type_id,
.access_str_off = relo->core_relo->access_str_off,
.kind = relo->core_relo->kind,
};
bpf_gen__record_relo_core(obj->gen_loader, &cr);
break;
}
default:
continue;
}
......@@ -6740,7 +6793,7 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog
prog->name, prog->instances.nr);
}
if (obj->gen_loader)
bpf_program__record_externs(prog);
bpf_program_record_relos(prog);
err = bpf_object_load_prog_instance(obj, prog,
prog->insns, prog->insns_cnt,
license, kern_ver, &fd);
......
......@@ -172,7 +172,7 @@ static inline void *libbpf_reallocarray(void *ptr, size_t nmemb, size_t size)
struct btf;
struct btf_type;
struct btf_type *btf_type_by_id(struct btf *btf, __u32 type_id);
struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id);
const char *btf_kind_str(const struct btf_type *t);
const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id);
......
This diff is collapsed.
......@@ -4,81 +4,10 @@
#ifndef __RELO_CORE_H
#define __RELO_CORE_H
/* bpf_core_relo_kind encodes which aspect of captured field/type/enum value
* has to be adjusted by relocations.
*/
enum bpf_core_relo_kind {
BPF_FIELD_BYTE_OFFSET = 0, /* field byte offset */
BPF_FIELD_BYTE_SIZE = 1, /* field size in bytes */
BPF_FIELD_EXISTS = 2, /* field existence in target kernel */
BPF_FIELD_SIGNED = 3, /* field signedness (0 - unsigned, 1 - signed) */
BPF_FIELD_LSHIFT_U64 = 4, /* bitfield-specific left bitshift */
BPF_FIELD_RSHIFT_U64 = 5, /* bitfield-specific right bitshift */
BPF_TYPE_ID_LOCAL = 6, /* type ID in local BPF object */
BPF_TYPE_ID_TARGET = 7, /* type ID in target kernel */
BPF_TYPE_EXISTS = 8, /* type existence in target kernel */
BPF_TYPE_SIZE = 9, /* type size in bytes */
BPF_ENUMVAL_EXISTS = 10, /* enum value existence in target kernel */
BPF_ENUMVAL_VALUE = 11, /* enum value integer value */
};
/* The minimum bpf_core_relo checked by the loader
*
* CO-RE relocation captures the following data:
* - insn_off - instruction offset (in bytes) within a BPF program that needs
* its insn->imm field to be relocated with actual field info;
* - type_id - BTF type ID of the "root" (containing) entity of a relocatable
* type or field;
* - access_str_off - offset into corresponding .BTF string section. String
* interpretation depends on specific relocation kind:
* - for field-based relocations, string encodes an accessed field using
* a sequence of field and array indices, separated by colon (:). It's
* conceptually very close to LLVM's getelementptr ([0]) instruction's
* arguments for identifying offset to a field.
* - for type-based relocations, strings is expected to be just "0";
* - for enum value-based relocations, string contains an index of enum
* value within its enum type;
*
* Example to provide a better feel.
*
* struct sample {
* int a;
* struct {
* int b[10];
* };
* };
*
* struct sample *s = ...;
* int x = &s->a; // encoded as "0:0" (a is field #0)
* int y = &s->b[5]; // encoded as "0:1:0:5" (anon struct is field #1,
* // b is field #0 inside anon struct, accessing elem #5)
* int z = &s[10]->b; // encoded as "10:1" (ptr is used as an array)
*
* type_id for all relocs in this example will capture BTF type id of
* `struct sample`.
*
* Such relocation is emitted when using __builtin_preserve_access_index()
* Clang built-in, passing expression that captures field address, e.g.:
*
* bpf_probe_read(&dst, sizeof(dst),
* __builtin_preserve_access_index(&src->a.b.c));
*
* In this case Clang will emit field relocation recording necessary data to
* be able to find offset of embedded `a.b.c` field within `src` struct.
*
* [0] https://llvm.org/docs/LangRef.html#getelementptr-instruction
*/
struct bpf_core_relo {
__u32 insn_off;
__u32 type_id;
__u32 access_str_off;
enum bpf_core_relo_kind kind;
};
#include <linux/bpf.h>
struct bpf_core_cand {
const struct btf *btf;
const struct btf_type *t;
const char *name;
__u32 id;
};
......
......@@ -325,9 +325,10 @@ LINKED_SKELS := test_static_linked.skel.h linked_funcs.skel.h \
linked_vars.skel.h linked_maps.skel.h
LSKELS := kfunc_call_test.c fentry_test.c fexit_test.c fexit_sleep.c \
test_ringbuf.c atomics.c trace_printk.c trace_vprintk.c
test_ringbuf.c atomics.c trace_printk.c trace_vprintk.c \
map_ptr_kern.c core_kern.c
# Generate both light skeleton and libbpf skeleton for these
LSKELS_EXTRA := test_ksyms_module.c test_ksyms_weak.c
LSKELS_EXTRA := test_ksyms_module.c test_ksyms_weak.c kfunc_call_test_subprog.c
SKEL_BLACKLIST += $$(LSKELS)
test_static_linked.skel.h-deps := test_static_linked1.o test_static_linked2.o
......
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2021 Facebook */
#include "test_progs.h"
#include "core_kern.lskel.h"
void test_core_kern_lskel(void)
{
struct core_kern_lskel *skel;
skel = core_kern_lskel__open_and_load();
ASSERT_OK_PTR(skel, "open_and_load");
core_kern_lskel__destroy(skel);
}
......@@ -4,6 +4,7 @@
#include <network_helpers.h>
#include "kfunc_call_test.lskel.h"
#include "kfunc_call_test_subprog.skel.h"
#include "kfunc_call_test_subprog.lskel.h"
static void test_main(void)
{
......@@ -49,6 +50,26 @@ static void test_subprog(void)
kfunc_call_test_subprog__destroy(skel);
}
static void test_subprog_lskel(void)
{
struct kfunc_call_test_subprog_lskel *skel;
int prog_fd, retval, err;
skel = kfunc_call_test_subprog_lskel__open_and_load();
if (!ASSERT_OK_PTR(skel, "skel"))
return;
prog_fd = skel->progs.kfunc_call_test1.prog_fd;
err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
NULL, NULL, (__u32 *)&retval, NULL);
ASSERT_OK(err, "bpf_prog_test_run(test1)");
ASSERT_EQ(retval, 10, "test1-retval");
ASSERT_NEQ(skel->data->active_res, -1, "active_res");
ASSERT_EQ(skel->data->sk_state_res, BPF_TCP_CLOSE, "sk_state_res");
kfunc_call_test_subprog_lskel__destroy(skel);
}
void test_kfunc_call(void)
{
if (test__start_subtest("main"))
......@@ -56,4 +77,7 @@ void test_kfunc_call(void)
if (test__start_subtest("subprog"))
test_subprog();
if (test__start_subtest("subprog_lskel"))
test_subprog_lskel();
}
......@@ -4,31 +4,29 @@
#include <test_progs.h>
#include <network_helpers.h>
#include "map_ptr_kern.skel.h"
#include "map_ptr_kern.lskel.h"
void test_map_ptr(void)
{
struct map_ptr_kern *skel;
struct map_ptr_kern_lskel *skel;
__u32 duration = 0, retval;
char buf[128];
int err;
int page_size = getpagesize();
skel = map_ptr_kern__open();
skel = map_ptr_kern_lskel__open();
if (!ASSERT_OK_PTR(skel, "skel_open"))
return;
err = bpf_map__set_max_entries(skel->maps.m_ringbuf, page_size);
if (!ASSERT_OK(err, "bpf_map__set_max_entries"))
goto cleanup;
skel->maps.m_ringbuf.max_entries = page_size;
err = map_ptr_kern__load(skel);
err = map_ptr_kern_lskel__load(skel);
if (!ASSERT_OK(err, "skel_load"))
goto cleanup;
skel->bss->page_size = page_size;
err = bpf_prog_test_run(bpf_program__fd(skel->progs.cg_skb), 1, &pkt_v4,
err = bpf_prog_test_run(skel->progs.cg_skb.prog_fd, 1, &pkt_v4,
sizeof(pkt_v4), buf, NULL, &retval, NULL);
if (CHECK(err, "test_run", "err=%d errno=%d\n", err, errno))
......@@ -39,5 +37,5 @@ void test_map_ptr(void)
goto cleanup;
cleanup:
map_ptr_kern__destroy(skel);
map_ptr_kern_lskel__destroy(skel);
}
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2021 Facebook */
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_core_read.h>
#define ATTR __always_inline
#include "test_jhash.h"
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__type(key, u32);
__type(value, u32);
__uint(max_entries, 256);
} array1 SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__type(key, u32);
__type(value, u32);
__uint(max_entries, 256);
} array2 SEC(".maps");
static __noinline int randmap(int v, const struct net_device *dev)
{
struct bpf_map *map = (struct bpf_map *)&array1;
int key = bpf_get_prandom_u32() & 0xff;
int *val;
if (bpf_get_prandom_u32() & 1)
map = (struct bpf_map *)&array2;
val = bpf_map_lookup_elem(map, &key);
if (val)
*val = bpf_get_prandom_u32() + v + dev->mtu;
return 0;
}
SEC("tp_btf/xdp_devmap_xmit")
int BPF_PROG(tp_xdp_devmap_xmit_multi, const struct net_device
*from_dev, const struct net_device *to_dev, int sent, int drops,
int err)
{
return randmap(from_dev->ifindex, from_dev);
}
SEC("fentry/eth_type_trans")
int BPF_PROG(fentry_eth_type_trans, struct sk_buff *skb,
struct net_device *dev, unsigned short protocol)
{
return randmap(dev->ifindex + skb->len, dev);
}
SEC("fexit/eth_type_trans")
int BPF_PROG(fexit_eth_type_trans, struct sk_buff *skb,
struct net_device *dev, unsigned short protocol)
{
return randmap(dev->ifindex + skb->len, dev);
}
volatile const int never;
struct __sk_bUfF /* it will not exist in vmlinux */ {
int len;
} __attribute__((preserve_access_index));
struct bpf_testmod_test_read_ctx /* it exists in bpf_testmod */ {
size_t len;
} __attribute__((preserve_access_index));
SEC("tc")
int balancer_ingress(struct __sk_buff *ctx)
{
void *data_end = (void *)(long)ctx->data_end;
void *data = (void *)(long)ctx->data;
void *ptr;
int ret = 0, nh_off, i = 0;
nh_off = 14;
/* pragma unroll doesn't work on large loops */
#define C do { \
ptr = data + i; \
if (ptr + nh_off > data_end) \
break; \
ctx->tc_index = jhash(ptr, nh_off, ctx->cb[0] + i++); \
if (never) { \
/* below is a dead code with unresolvable CO-RE relo */ \
i += ((struct __sk_bUfF *)ctx)->len; \
/* this CO-RE relo may or may not resolve
* depending on whether bpf_testmod is loaded.
*/ \
i += ((struct bpf_testmod_test_read_ctx *)ctx)->len; \
} \
} while (0);
#define C30 C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;C;
C30;C30;C30; /* 90 calls */
return 0;
}
char LICENSE[] SEC("license") = "GPL";
......@@ -334,9 +334,11 @@ static inline int check_lpm_trie(void)
return 1;
}
#define INNER_MAX_ENTRIES 1234
struct inner_map {
__uint(type, BPF_MAP_TYPE_ARRAY);
__uint(max_entries, 1);
__uint(max_entries, INNER_MAX_ENTRIES);
__type(key, __u32);
__type(value, __u32);
} inner_map SEC(".maps");
......@@ -348,7 +350,7 @@ struct {
__type(value, __u32);
__array(values, struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__uint(max_entries, 1);
__uint(max_entries, INNER_MAX_ENTRIES);
__type(key, __u32);
__type(value, __u32);
});
......@@ -360,8 +362,13 @@ static inline int check_array_of_maps(void)
{
struct bpf_array *array_of_maps = (struct bpf_array *)&m_array_of_maps;
struct bpf_map *map = (struct bpf_map *)&m_array_of_maps;
struct bpf_array *inner_map;
int key = 0;
VERIFY(check_default(&array_of_maps->map, map));
inner_map = bpf_map_lookup_elem(array_of_maps, &key);
VERIFY(inner_map != 0);
VERIFY(inner_map->map.max_entries == INNER_MAX_ENTRIES);
return 1;
}
......@@ -382,8 +389,13 @@ static inline int check_hash_of_maps(void)
{
struct bpf_htab *hash_of_maps = (struct bpf_htab *)&m_hash_of_maps;
struct bpf_map *map = (struct bpf_map *)&m_hash_of_maps;
struct bpf_htab *inner_map;
int key = 2;
VERIFY(check_default(&hash_of_maps->map, map));
inner_map = bpf_map_lookup_elem(hash_of_maps, &key);
VERIFY(inner_map != 0);
VERIFY(inner_map->map.max_entries == INNER_MAX_ENTRIES);
return 1;
}
......
......@@ -38,7 +38,7 @@ int pass_handler(const void *ctx)
/* tests existing symbols. */
rq = (struct rq *)bpf_per_cpu_ptr(&runqueues, 0);
if (rq)
out__existing_typed = 0;
out__existing_typed = rq->cpu;
out__existing_typeless = (__u64)&bpf_prog_active;
/* tests non-existent symbols. */
......
// SPDX-License-Identifier: GPL-2.0
// Copyright (c) 2019 Facebook
#include <linux/bpf.h>
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#define ATTR __always_inline
#include "test_jhash.h"
SEC("scale90_inline")
SEC("tc")
int balancer_ingress(struct __sk_buff *ctx)
{
void *data_end = (void *)(long)ctx->data_end;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment