Add AF_XDP sockets

Describe XDP option in help
Add option to link with local libbpf folder
Update wrapper server script
Fix min max functions conflicts
Compile xdp-kern.o
Add IFLAGS to CFLAGS in makefile for XDP
Add bpf_helpers and xdp_kern files
Close XDP socket before exiting program
Add tracecmd option for XDP
Print stats for XDP too
Add an active polling mode

packet-exchange/build/Makefile

@@ -27,15 +27,35 @@ CFLAGS += -MD -MP
 CFLAGS += -I $(SRCDIR)
 CFLAGS += -std=gnu99
 
-LDFLAGS = -pthread
+LLIBS = -pthread
+
+ifneq ($(WITH_XDP),)
+CFLAGS += -D WITH_XDP
+LDFLAGS += -L/usr/lib -lbpf
+else ifneq ($(WITH_GIT_XDP),)
+IFLAGS += -I ${HOME}/libbpf/include
+CFLAGS += -D WITH_XDP $(IFLAGS)
+LDIRS += -L${HOME}/libbpf/src
+LLIBS += -lelf -lz -l:libbpf.a
+endif
 
 vpath %.c $(SRCDIR)
 
+xdp_kern.o: xdp_kern.c
+	clang $(IFLAGS) -isystem /usr/include/arm-linux-gnueabihf -S -target bpf -D __BPF_TRACING__ -Wall -O2 -emit-llvm -c -g -o xdp_kern.ll $^
+	llc -march=bpf -filetype=obj -o $@ xdp_kern.ll
+
+ifneq ($(WITH_GIT_XDP),)
+$(SERVER_PROG): $(SERVER_OBJS) xdp_kern.o
+	$(CC) $(LDFLAGS) $(LDIRS) $(SERVER_OBJS) $(LLIBS) -o $@
+else
 $(SERVER_PROG): $(SERVER_OBJS)
-	$(CC) $(LDFLAGS) $^ -o $@
+	$(CC) $(LDFLAGS) $(LDIRS) $^ $(LLIBS) -o $@
+endif
 
 $(CLIENT_PROG): $(CLIENT_OBJS)
-	$(CC) $(LDFLAGS) $^ -o $@
+	$(CC) $(LDFLAGS) $(LDIRS) $^ $(LLIBS) -o $@
+
 
 -include $(subst .c,.d,$(SERVER_SRCS))
 -include $(subst .c,.d,$(CLIENT_SRCS))

packet-exchange/src/common.c

 #define _GNU_SOURCE
+#include "common.h"
+
 #include <inttypes.h>
 #include <signal.h>
 #include <stdint.h>
@@ -7,73 +9,65 @@
 #include <time.h>
 #include <unistd.h>
 
-#include "common.h"
-
 void (*previous_handlers[NSIG])(int);
 static void (*sighand)(int);
 
 uint64_t ts_to_uint(struct timespec t) {
-  return t.tv_sec * NSEC_PER_SEC + t.tv_nsec;
+	return t.tv_sec * NSEC_PER_SEC + t.tv_nsec;
 }
 
 void add_ns(struct timespec *t, uint64_t ns) {
-  t->tv_nsec += ns;
+	t->tv_nsec += ns;
 
-  while ((unsigned int)t->tv_nsec >= NSEC_PER_SEC) {
-    t->tv_sec += 1;
-    t->tv_nsec -= NSEC_PER_SEC;
-  }
+	while ((unsigned int)t->tv_nsec >= NSEC_PER_SEC) {
+		t->tv_sec += 1;
+		t->tv_nsec -= NSEC_PER_SEC;
+	}
 }
 
 uint64_t calcdiff_ns(struct timespec t1, struct timespec t2) {
-  uint64_t diff;
-  diff = NSEC_PER_SEC * (uint64_t)((int)t1.tv_sec - (int)t2.tv_sec);
-  diff += ((int)t1.tv_nsec - (int)t2.tv_nsec);
-  return diff;
+	uint64_t diff;
+	diff = NSEC_PER_SEC * (uint64_t)((int)t1.tv_sec - (int)t2.tv_sec);
+	diff += ((int)t1.tv_nsec - (int)t2.tv_nsec);
+	return diff;
 }
 
-int max(int a, int b) { return a > b ? a : b; }
-int min(int a, int b) { return a < b ? a : b; }
+int _max_(int a, int b) { return a > b ? a : b; }
+int _min_(int a, int b) { return a < b ? a : b; }
 
-int histogram_min(uint64_t * histogram, int max_value) {
-  int ret = max_value;
-  for (int i = max_value; i >= 0; i--)
-    ret = histogram[i] ? i : ret;
-  return ret;
+int histogram_min(uint64_t *histogram, int max_value) {
+	int ret = max_value;
+	for (int i = max_value; i >= 0; i--) ret = histogram[i] ? i : ret;
+	return ret;
 }
-int histogram_max(uint64_t * histogram, int max_value) {
-  int ret = 0;
-  for (int i = 0; i <= max_value; i++)
-    ret = histogram[i] ? i : ret;
-  return ret;
+int histogram_max(uint64_t *histogram, int max_value) {
+	int ret = 0;
+	for (int i = 0; i <= max_value; i++) ret = histogram[i] ? i : ret;
+	return ret;
 }
 
-
 static void sighand_wrapper(int sig) {
-
-  // If we get un unexpected signal, report it, if not print the histogram
-  if (sig == SIGINT || sig == SIGTERM)
-    (*sighand)(sig); // Will print the histogram
-  else
-    printf("Uknown signal interrupt: %s (%d)\n", strsignal(sig), sig);
-
-  // Execute the default handler
-  if (previous_handlers[sig] == SIG_DFL) {
-    signal(sig, SIG_DFL);
-    raise(sig);
-  } else if (previous_handlers[sig] == SIG_IGN) {
-    return;
-  } else {
-    (*previous_handlers[sig])(sig);
-  }
+	// If we get un unexpected signal, report it, if not print the histogram
+	if (sig == SIGINT || sig == SIGTERM)
+		(*sighand)(sig);  // Will print the histogram
+	else
+		printf("Uknown signal interrupt: %s (%d)\n", strsignal(sig),
+		       sig);
+
+	// Execute the default handler
+	if (previous_handlers[sig] == SIG_DFL) {
+		signal(sig, SIG_DFL);
+		raise(sig);
+	} else if (previous_handlers[sig] == SIG_IGN) {
+		return;
+	} else {
+		(*previous_handlers[sig])(sig);
+	}
 }
 
-void init_signals(void (*_sighand)(int), int enable_histograms) {
-
-  sighand = _sighand;
+void init_signals(void (*_sighand)(int)) {
+	sighand = _sighand;
 
-  if (enable_histograms)
-    for (int i = 0; i < NSIG; i++)
-      signal(i, sighand_wrapper);
+	for (int i = 0; i < NSIG; i++) signal(i, sighand_wrapper);
 }
 

packet-exchange/src/common.h

@@ -8,6 +8,17 @@
 #include <time.h>
 #include <unistd.h>
 
+#ifdef WITH_XDP
+#include <bpf/bpf.h>
+#include <bpf/libbpf.h>
+#include <bpf/xsk.h>
+#include <linux/if_ether.h>
+#include <linux/if_link.h>
+#include <linux/if_xdp.h>
+#include <linux/ip.h>
+#include <linux/udp.h>
+#endif
+
 #define NSEC_PER_SEC UINT64_C(1000000000)
 #define SERVER_PORT "50000"
 #define SERVER_PORT_INT 50000
@@ -19,18 +30,25 @@
 #define MAX_BUFFER_SIZE 1024
 
 #define TIMESTAMP_BUFFER_SIZE 4096
+#define err(...)                              \
+	do {                                  \
+		fprintf(stderr, __VA_ARGS__); \
+		exit(EXIT_FAILURE);           \
+	} while (0)
+
+#define err_errno(...) error(EXIT_FAILURE, errno, __VA_ARGS__);
 
 uint64_t ts_to_uint(struct timespec t);
 void add_ns(struct timespec *t, uint64_t ns);
 uint64_t calcdiff_ns(struct timespec t1, struct timespec t2);
 
-void init_signals(void (*_sighand)(int), int enable_histograms);
+void init_signals(void (*_sighand)(int));
 
-int min(int a, int b);
-int max(int a, int b);
+int _min_(int a, int b);
+int _max_(int a, int b);
 
-int histogram_min(uint64_t * histogram, int max_value);
-int histogram_max(uint64_t * histogram, int max_value);
+int histogram_min(uint64_t *histogram, int max_value);
+int histogram_max(uint64_t *histogram, int max_value);
 
 extern void (*previous_handlers[NSIG])(int);
 

packet-exchange/src/recv_packet.h

@@ -4,37 +4,61 @@
 #include "common.h"
 
 typedef struct ingress_param {
-  char network_if[16];
+	char network_if[16];
 
-  int use_timestamps;
+	int use_timestamps;
+	int xdp_polling_mode;
+	int interval;
 
-  size_t tx_buffer_len;
+	size_t tx_buffer_len;
 
 } ingress_param_t;
 
 typedef struct ingress_stat {
+	int min_kernel_latency;
+	int avg_kernel_latency;
+	int max_kernel_latency;
 
-  int min_kernel_latency;
-  int avg_kernel_latency;
-  int max_kernel_latency;
+	int min_interval;
+	int avg_interval;
+	int max_interval;
 
-  int min_interval;
-  int avg_interval;
-  int max_interval;
+	uint64_t packets_received;
+	uint64_t high_kernel_latency;
+	uint64_t high_jitter;
+	int lost_packets;
 
-  uint64_t packets_received;
-  uint64_t high_kernel_latency;
-  uint64_t high_jitter;
-  int lost_packets;
-
-  char data[MAX_BUFFER_SIZE];
+	char data[MAX_BUFFER_SIZE];
 } ingress_stat_t;
 
-void init_udp_recv(ingress_param_t *_params,
-                   ingress_stat_t *stats,
-                   int use_histogram,
-                   uint64_t *_kernel_latency_hist);
+void init_udp_recv(ingress_param_t *_params, ingress_stat_t *stats,
+		   int use_histogram, uint64_t *_kernel_latency_hist);
+
+void recv_udp_packet(void);
+
+#ifdef WITH_XDP
+#define NUM_FRAMES 4096
+#define FRAME_SIZE XSK_UMEM__DEFAULT_FRAME_SIZE
+
+struct xsk_umem_info {
+	struct xsk_ring_prod fq;
+	struct xsk_ring_cons cq;
+	struct xsk_umem *umem;
+	void *buffer;
+};
+struct xdpsock {
+	struct xsk_ring_cons rx;
+	struct xsk_ring_prod tx;
+	struct xsk_umem_info umem;
+	struct xsk_socket *xsk;
+	int fd;
+};
+#endif
 
-void recv_udp_packet();
+void init_xdp_recv(ingress_param_t * _params);
+int recv_xdp_packet(void);
+void recv_xdp_cleanup(void);
+void setup_poll_fd(void);
+void close_xdp_socket(void);
 
 #endif

packet-exchange/src/send_packet.c

@@ -222,9 +222,9 @@ static void process_error_queue() {
       ts_buf_read_index = (ts_buf_read_index + 1) % TIMESTAMP_BUFFER_SIZE;
 
       stats->min_kernel_latency =
-          min(kernel_latency, stats->min_kernel_latency);
+          _min_(kernel_latency, stats->min_kernel_latency);
       stats->max_kernel_latency =
-          max(kernel_latency, stats->max_kernel_latency);
+          _max_(kernel_latency, stats->max_kernel_latency);
       stats->avg_kernel_latency =
           (stats->max_kernel_latency * packets_sent + kernel_latency) /
           (packets_sent + 1);

packet-exchange/src/tracer.c

@@ -75,7 +75,7 @@ static void setkernvar(const char *name, char *value) {
     fprintf(stderr, "could not set %s to %s\n", name, value);
 }
 
-void open_fds() {
+void open_fds(void) {
 
   fileprefix = debugfileprefix;
 

packet-exchange/src/tracer.h

@@ -2,7 +2,7 @@
 #define TRACER_H
 
 void setup_tracer(int enable_graph);
-void open_fds();
+void open_fds(void);
 void tracing(int on);
 void tracemark(char * s);
 

packet-exchange/src/xdp_kern.c

+#define KBUILD_MODNAME "blub"
+
+#include <linux/bpf.h>
+#include <linux/if_ether.h>
+#include <linux/if_link.h>
+#include <linux/if_xdp.h>
+#include <linux/in.h>
+#include <linux/ip.h>
+#include <linux/udp.h>
+
+#include "bpf_helpers.h"
+
+#define bpf_printk(fmt, ...)                                               \
+	({                                                                 \
+		char ____fmt[] = fmt;                                      \
+		bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \
+	})
+
+struct bpf_map_def SEC("maps") xsks_map = {
+    .type = BPF_MAP_TYPE_XSKMAP,
+    .key_size = sizeof(int),
+    .value_size = sizeof(int),
+    .max_entries = 64,
+};
+
+static inline int parse_ipv4(void *data, unsigned long long nh_off,
+			     void *data_end) {
+	struct iphdr *iph = data + nh_off;
+
+	if ((void *)(iph + 1) > data_end) return 0;
+
+	return iph->protocol;
+}
+
+SEC("xdp_sock")
+int xdp_sock_prog(struct xdp_md *ctx) {
+	void *data_end = (void *)(long)ctx->data_end;
+	void *data = (void *)(long)ctx->data;
+	struct ethhdr *eth = data;
+	int idx = ctx->rx_queue_index;
+	unsigned int ipproto = 0;
+	unsigned long long nh_off;
+
+	/* Check if it's a UDP frame: If UDP -> Redirect to active xsk for user
+	 * space. If not -> pass to stack.
+	 */
+	nh_off = sizeof(*eth);
+	if (data + nh_off > data_end) return XDP_PASS;
+
+	if (eth->h_proto == __builtin_bswap16(ETH_P_IP))
+		ipproto = parse_ipv4(data, nh_off, data_end);
+
+	if (ipproto != IPPROTO_UDP) return XDP_PASS;
+
+	/* If socket bound to rx_queue then redirect to user space */
+	if (bpf_map_lookup_elem(&xsks_map, &idx))
+		return bpf_redirect_map(&xsks_map, idx, 0);
+
+	/* Else pass to Linux' network stack */
+	return XDP_PASS;
+}
+
+char _license[] SEC("license") = "GPL";
+

scripts/run-server

@@ -3,16 +3,28 @@
 script_dir=$(dirname $(realpath $0))
 
 usage() {
-  echo "Usage: $0 [[-bt] [-g INTERVAL]] | (-d NB_PACKETS [-i INTERVAL]) [-T LATENCY_THRESHOLD -G]" 1>&2;
+  echo "Usage: $0  |  " 1>&2;
+  exit 1;
+}
+usage() {
+  cat << ENDUSAGE
+Usage: $0 SERVER | TCPDUMP [TRACE_OPTS]
+   SERVER: [-bt] [(-x | -X) POLL] [-g INTERVAL]
+   TCPDUMP: -d NB_PACKETS [-i INTERVAL]
+   TRACE_OPTS: (-T LATENCY_THRESHOLD -G) | -E
+ENDUSAGE
+1>&2;
+
   exit 1;
 }
 
 # Default options
 server_options="-a -p 99 -f eth0"
+make_opts=""
 ip="10.100.21."
 tcpdump_interval=1000000
 
-while getopts "b:htd:i:g:T:G" opt; do
+while getopts "b:htx:X:d:i:g:T:GE" opt; do
   case "${opt}" in
     b )
         use_rtt=1
@@ -37,12 +49,23 @@ while getopts "b:htd:i:g:T:G" opt; do
     t )
         server_options+=" -t"
         ;;
+    x )
+        server_options+=" -x ${OPTARG}"
+	make_opts=" -e WITH_XDP=1"
+        ;;
+    X )
+        server_options+=" -x ${OPTARG}"
+	make_opts=" -e WITH_GIT_XDP=1"
+        ;;
     T )
         server_options+=" -T ${OPTARG}"
       ;;
     G )
         server_options+=" -G"
       ;;
+    E )
+        enable_xdp_events=1
+      ;;
     * )
         usage
       ;;
@@ -78,10 +101,14 @@ if [ -n "${use_tcpdump}" ]; then
   $script_dir/txtime-stats.py -f server_stats_tmp.out -i $tcpdump_interval;
 else
 
-  echo "make server";
-  cd $script_dir/../packet-exchange/build;make server;cd $script_dir
+  echo "make $make_opts server";
+  cd $script_dir/../packet-exchange/build;make $make_opts server;cd $script_dir
 
-  if [ -z "${use_histogram}" ]; then
+  if [ -n "${enable_xdp_events}" ]; then
+    tracecmd_events="-e irq -e sched -e xdp"
+    echo "trace-cmd record $tracecmd_events $script_dir/../packet-exchange/build/server $server_options";
+    trace-cmd record $tracecmd_events $script_dir/../packet-exchange/build/server $server_options;
+  elif [ -z "${use_histogram}" ]; then
     echo "server $server_options";
     $script_dir/../packet-exchange/build/server $server_options;
   else