Commit 7e3f977e authored by Daniel Borkmann's avatar Daniel Borkmann Committed by David S. Miller

perf, events: add non-linear data support for raw records

This patch adds support for non-linear data on raw records. It
extends raw records to have one or multiple fragments that will
be written linearly into the ring slot, where each fragment can
optionally have a custom callback handler to walk and extract
complex, possibly non-linear data.

If a callback handler is provided for a fragment, then the new
__output_custom() will be used instead of __output_copy() for
the perf_output_sample() part. perf_prepare_sample() does all
the size calculation only once, so perf_output_sample() doesn't
need to redo the same work anymore, meaning real_size and padding
will be cached in the raw record. The raw record becomes 32 bytes
in size without holes; to not increase it further and to avoid
doing unnecessary recalculations in fast-path, we can reuse
next pointer of the last fragment, idea here is borrowed from
ZERO_OR_NULL_PTR(), which should keep the perf_output_sample()
path for PERF_SAMPLE_RAW minimal.

This facility is needed for BPF's event output helper as a first
user that will, in a follow-up, add an additional perf_raw_frag
to its perf_raw_record in order to be able to more efficiently
dump skb context after a linear head meta data related to it.
skbs can be non-linear and thus need a custom output function to
dump buffers. Currently, the skb data needs to be copied twice;
with the help of __output_custom() this work only needs to be
done once. Future users could be things like XDP/BPF programs
that work on different context though and would thus also have
a different callback function.

The few users of raw records are adapted to initialize their frag
data from the raw record itself, no change in behavior for them.
The code is based upon a PoC diff provided by Peter Zijlstra [1].

  [1] http://thread.gmane.org/gmane.linux.network/421294Suggested-by: default avatarPeter Zijlstra <peterz@infradead.org>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Acked-by: default avatarAlexei Starovoitov <ast@kernel.org>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 7acef604
...@@ -979,12 +979,15 @@ static int perf_push_sample(struct perf_event *event, struct sf_raw_sample *sfr) ...@@ -979,12 +979,15 @@ static int perf_push_sample(struct perf_event *event, struct sf_raw_sample *sfr)
struct pt_regs regs; struct pt_regs regs;
struct perf_sf_sde_regs *sde_regs; struct perf_sf_sde_regs *sde_regs;
struct perf_sample_data data; struct perf_sample_data data;
struct perf_raw_record raw; struct perf_raw_record raw = {
.frag = {
.size = sfr->size,
.data = sfr,
},
};
/* Setup perf sample */ /* Setup perf sample */
perf_sample_data_init(&data, 0, event->hw.last_period); perf_sample_data_init(&data, 0, event->hw.last_period);
raw.size = sfr->size;
raw.data = sfr;
data.raw = &raw; data.raw = &raw;
/* Setup pt_regs to look like an CPU-measurement external interrupt /* Setup pt_regs to look like an CPU-measurement external interrupt
......
...@@ -655,8 +655,12 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) ...@@ -655,8 +655,12 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
} }
if (event->attr.sample_type & PERF_SAMPLE_RAW) { if (event->attr.sample_type & PERF_SAMPLE_RAW) {
raw.size = sizeof(u32) + ibs_data.size; raw = (struct perf_raw_record){
raw.data = ibs_data.data; .frag = {
.size = sizeof(u32) + ibs_data.size,
.data = ibs_data.data,
},
};
data.raw = &raw; data.raw = &raw;
} }
......
...@@ -69,9 +69,22 @@ struct perf_callchain_entry_ctx { ...@@ -69,9 +69,22 @@ struct perf_callchain_entry_ctx {
bool contexts_maxed; bool contexts_maxed;
}; };
typedef unsigned long (*perf_copy_f)(void *dst, const void *src,
unsigned long len);
struct perf_raw_frag {
union {
struct perf_raw_frag *next;
unsigned long pad;
};
perf_copy_f copy;
void *data;
u32 size;
} __packed;
struct perf_raw_record { struct perf_raw_record {
struct perf_raw_frag frag;
u32 size; u32 size;
void *data;
}; };
/* /*
...@@ -1283,6 +1296,11 @@ extern void perf_restore_debug_store(void); ...@@ -1283,6 +1296,11 @@ extern void perf_restore_debug_store(void);
static inline void perf_restore_debug_store(void) { } static inline void perf_restore_debug_store(void) { }
#endif #endif
static __always_inline bool perf_raw_frag_last(const struct perf_raw_frag *frag)
{
return frag->pad < sizeof(u64);
}
#define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x)) #define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
/* /*
......
...@@ -5553,16 +5553,26 @@ void perf_output_sample(struct perf_output_handle *handle, ...@@ -5553,16 +5553,26 @@ void perf_output_sample(struct perf_output_handle *handle,
} }
if (sample_type & PERF_SAMPLE_RAW) { if (sample_type & PERF_SAMPLE_RAW) {
if (data->raw) { struct perf_raw_record *raw = data->raw;
u32 raw_size = data->raw->size;
u32 real_size = round_up(raw_size + sizeof(u32), if (raw) {
sizeof(u64)) - sizeof(u32); struct perf_raw_frag *frag = &raw->frag;
u64 zero = 0;
perf_output_put(handle, raw->size);
perf_output_put(handle, real_size); do {
__output_copy(handle, data->raw->data, raw_size); if (frag->copy) {
if (real_size - raw_size) __output_custom(handle, frag->copy,
__output_copy(handle, &zero, real_size - raw_size); frag->data, frag->size);
} else {
__output_copy(handle, frag->data,
frag->size);
}
if (perf_raw_frag_last(frag))
break;
frag = frag->next;
} while (1);
if (frag->pad)
__output_skip(handle, NULL, frag->pad);
} else { } else {
struct { struct {
u32 size; u32 size;
...@@ -5687,14 +5697,28 @@ void perf_prepare_sample(struct perf_event_header *header, ...@@ -5687,14 +5697,28 @@ void perf_prepare_sample(struct perf_event_header *header,
} }
if (sample_type & PERF_SAMPLE_RAW) { if (sample_type & PERF_SAMPLE_RAW) {
int size = sizeof(u32); struct perf_raw_record *raw = data->raw;
int size;
if (data->raw)
size += data->raw->size; if (raw) {
else struct perf_raw_frag *frag = &raw->frag;
size += sizeof(u32); u32 sum = 0;
do {
sum += frag->size;
if (perf_raw_frag_last(frag))
break;
frag = frag->next;
} while (1);
size = round_up(sum + sizeof(u32), sizeof(u64));
raw->size = size - sizeof(u32);
frag->pad = raw->size - sum;
} else {
size = sizeof(u64);
}
header->size += round_up(size, sizeof(u64)); header->size += size;
} }
if (sample_type & PERF_SAMPLE_BRANCH_STACK) { if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
...@@ -7331,7 +7355,7 @@ static struct pmu perf_swevent = { ...@@ -7331,7 +7355,7 @@ static struct pmu perf_swevent = {
static int perf_tp_filter_match(struct perf_event *event, static int perf_tp_filter_match(struct perf_event *event,
struct perf_sample_data *data) struct perf_sample_data *data)
{ {
void *record = data->raw->data; void *record = data->raw->frag.data;
/* only top level events have filters set */ /* only top level events have filters set */
if (event->parent) if (event->parent)
...@@ -7387,8 +7411,10 @@ void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size, ...@@ -7387,8 +7411,10 @@ void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size,
struct perf_event *event; struct perf_event *event;
struct perf_raw_record raw = { struct perf_raw_record raw = {
.size = entry_size, .frag = {
.data = record, .size = entry_size,
.data = record,
},
}; };
perf_sample_data_init(&data, 0, 0); perf_sample_data_init(&data, 0, 0);
......
...@@ -123,10 +123,7 @@ static inline unsigned long perf_aux_size(struct ring_buffer *rb) ...@@ -123,10 +123,7 @@ static inline unsigned long perf_aux_size(struct ring_buffer *rb)
return rb->aux_nr_pages << PAGE_SHIFT; return rb->aux_nr_pages << PAGE_SHIFT;
} }
#define DEFINE_OUTPUT_COPY(func_name, memcpy_func) \ #define __DEFINE_OUTPUT_COPY_BODY(memcpy_func) \
static inline unsigned long \
func_name(struct perf_output_handle *handle, \
const void *buf, unsigned long len) \
{ \ { \
unsigned long size, written; \ unsigned long size, written; \
\ \
...@@ -152,6 +149,17 @@ func_name(struct perf_output_handle *handle, \ ...@@ -152,6 +149,17 @@ func_name(struct perf_output_handle *handle, \
return len; \ return len; \
} }
#define DEFINE_OUTPUT_COPY(func_name, memcpy_func) \
static inline unsigned long \
func_name(struct perf_output_handle *handle, \
const void *buf, unsigned long len) \
__DEFINE_OUTPUT_COPY_BODY(memcpy_func)
static inline unsigned long
__output_custom(struct perf_output_handle *handle, perf_copy_f copy_func,
const void *buf, unsigned long len)
__DEFINE_OUTPUT_COPY_BODY(copy_func)
static inline unsigned long static inline unsigned long
memcpy_common(void *dst, const void *src, unsigned long n) memcpy_common(void *dst, const void *src, unsigned long n)
{ {
......
...@@ -245,8 +245,10 @@ static u64 bpf_perf_event_output(u64 r1, u64 r2, u64 flags, u64 r4, u64 size) ...@@ -245,8 +245,10 @@ static u64 bpf_perf_event_output(u64 r1, u64 r2, u64 flags, u64 r4, u64 size)
struct bpf_event_entry *ee; struct bpf_event_entry *ee;
struct perf_event *event; struct perf_event *event;
struct perf_raw_record raw = { struct perf_raw_record raw = {
.size = size, .frag = {
.data = data, .size = size,
.data = data,
},
}; };
if (unlikely(flags & ~(BPF_F_INDEX_MASK))) if (unlikely(flags & ~(BPF_F_INDEX_MASK)))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment