Skip to content

T
tsn-measures
Commit 2ba307c4 authored by Joanne Hugé's avatar Joanne Hugé
Browse files
Options

Remove poll wake-up for signal emission and XPD

parent 3f5754b5
Show whitespace changes
Inline Side-by-side
Showing with 316 additions and 407 deletions
packet-exchange/src/recv_packet.c
View file @ 2ba307c4
......@@ -92,8 +92,7 @@ void init_udp_recv(ingress_param_t *_params, ingress_stat_t *_stats,
getaddrinfo_err = getaddrinfo(NULL, SERVER_PORT, &hints, &servinfo);
if (getaddrinfo_err != 0) {
fprintf(stderr, "getaddrinfo: %s\n",
gai_strerror(getaddrinfo_err));
fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(getaddrinfo_err));
exit(EXIT_FAILURE);
}
......@@ -102,8 +101,7 @@ void init_udp_recv(ingress_param_t *_params, ingress_stat_t *_stats,
sock_fd = socket(servinfo->ai_family, servinfo->ai_socktype,
servinfo->ai_protocol);
if (bind(sock_fd, servinfo_it->ai_addr,
servinfo_it->ai_addrlen) == -1) {
if (bind(sock_fd, servinfo_it->ai_addr, servinfo_it->ai_addrlen) == -1) {
close(sock_fd);
continue;
}
......@@ -116,20 +114,16 @@ void init_udp_recv(ingress_param_t *_params, ingress_stat_t *_stats,
error(EXIT_FAILURE, errno, "Couldn't create receive socket");
set_if_err = set_if();
if (set_if_err < 0)
error(EXIT_FAILURE, errno, "Couldn't set interface\n");
if (set_if_err < 0) error(EXIT_FAILURE, errno, "Couldn't set interface\n");
if (setsockopt(sock_fd, SOL_SOCKET, SO_BINDTODEVICE, params->network_if,
strlen(params->network_if)))
error(EXIT_FAILURE, errno,
"setsockopt SO_BINDTODEVICE failed\n");
error(EXIT_FAILURE, errno, "setsockopt SO_BINDTODEVICE failed\n");
if (params->use_timestamps) {
if (setsockopt(sock_fd, SOL_SOCKET, SO_TIMESTAMPING,
&so_timestamping_flags,
if (setsockopt(sock_fd, SOL_SOCKET, SO_TIMESTAMPING, &so_timestamping_flags,
sizeof(so_timestamping_flags)))
error(EXIT_FAILURE, errno,
"setsockopt SO_TIMESTAMPING failed\n");
error(EXIT_FAILURE, errno, "setsockopt SO_TIMESTAMPING failed\n");
}
}
......@@ -163,8 +157,7 @@ void recv_udp_packet() {
recvmsgerr = recvmsg(sock_fd, &msg, 0);
if (recvmsgerr < 0)
error(EXIT_FAILURE, errno, "recvmsg failed, ret value: %d\n",
recvmsgerr);
error(EXIT_FAILURE, errno, "recvmsg failed, ret value: %d\n", recvmsgerr);
if (params->use_timestamps) {
clock_gettime(CLOCK_REALTIME, &ts);
......@@ -172,30 +165,24 @@ void recv_udp_packet() {
}
if (params->use_timestamps) {
for (cmsg = CMSG_FIRSTHDR(&msg); cmsg;
cmsg = CMSG_NXTHDR(&msg, cmsg)) {
for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
if (cmsg->cmsg_level == SOL_SOCKET &&
cmsg->cmsg_type == SO_TIMESTAMPING) {
struct timespec *stamp =
(struct timespec *)CMSG_DATA(cmsg);
struct timespec *stamp = (struct timespec *)CMSG_DATA(cmsg);
if(params->enable_ts_tracemark) {
if (params->enable_ts_tracemark) {
sprintf(ts_tracemark_buf, "%" PRIu64, ts_to_uint(*stamp));
tracemark(ts_tracemark_buf);
}
else {
uint64_t kernel_latency =
post_kernel_timestamp - ts_to_uint(*stamp);
} else {
uint64_t kernel_latency = post_kernel_timestamp - ts_to_uint(*stamp);
kernel_latency /= 1000u;
stats->min_kernel_latency = _min_(
kernel_latency, stats->min_kernel_latency);
stats->max_kernel_latency = _max_(
kernel_latency, stats->max_kernel_latency);
stats->min_kernel_latency =
_min_(kernel_latency, stats->min_kernel_latency);
stats->max_kernel_latency =
_max_(kernel_latency, stats->max_kernel_latency);
stats->avg_kernel_latency =
(stats->max_kernel_latency *
(stats->packets_received) +
(stats->max_kernel_latency * (stats->packets_received) +
kernel_latency) /
(stats->packets_received + 1);
......@@ -203,16 +190,14 @@ void recv_udp_packet() {
if (kernel_latency > MAX_KERNEL_LATENCY)
stats->high_kernel_latency++;
else
kernel_latency_hist
[kernel_latency]++;
kernel_latency_hist[kernel_latency]++;
}
}
}
}
}
for(int i = 0; i < MAX_BUFFER_SIZE; i++)
stats->data[i] = rx_buffer[i];
for (int i = 0; i < MAX_BUFFER_SIZE; i++) stats->data[i] = rx_buffer[i];
}
#ifdef WITH_XDP
......@@ -233,9 +218,8 @@ static void open_xdp_socket(char *network_if) {
xsk_cfg.xdp_flags = xdp_flags;
xsk_cfg.bind_flags = 0;
ret = xsk_socket__create(&xdp_socket.xsk, network_if, 0,
xdp_socket.umem.umem, &xdp_socket.rx,
&xdp_socket.tx, &xsk_cfg);
ret = xsk_socket__create(&xdp_socket.xsk, network_if, 0, xdp_socket.umem.umem,
&xdp_socket.rx, &xdp_socket.tx, &xsk_cfg);
if (ret) err("xsk_socket__create() failed");
/* Add some buffers */
......@@ -246,17 +230,15 @@ static void open_xdp_socket(char *network_if) {
err("xsk_ring_prod__reserve() failed");
for (i = 0; i < XSK_RING_PROD__DEFAULT_NUM_DESCS; i++)
*xsk_ring_prod__fill_addr(&xdp_socket.umem.fq, idx++) =
i * FRAME_SIZE;
*xsk_ring_prod__fill_addr(&xdp_socket.umem.fq, idx++) = i * FRAME_SIZE;
xsk_ring_prod__submit(&xdp_socket.umem.fq,
XSK_RING_PROD__DEFAULT_NUM_DESCS);
xsk_ring_prod__submit(&xdp_socket.umem.fq, XSK_RING_PROD__DEFAULT_NUM_DESCS);
}
/*
* Init XDP socket
*/
void init_xdp_recv(ingress_param_t * _params, ingress_stat_t *_stats) {
void init_xdp_recv(ingress_param_t *_params, ingress_stat_t *_stats) {
int ret, prog_fd, xsks_map = 0;
struct bpf_prog_load_attr prog_load_attr = {
.prog_type = BPF_PROG_TYPE_XDP,
......@@ -291,13 +273,12 @@ void init_xdp_recv(ingress_param_t * _params, ingress_stat_t *_stats) {
if (ret) err("bpf_set_link_xdp_fd() failed");
/* Allocate user space memory for xdp frames */
ret = posix_memalign(&buffer, sysconf(_SC_PAGE_SIZE),
NUM_FRAMES * FRAME_SIZE);
ret =
posix_memalign(&buffer, sysconf(_SC_PAGE_SIZE), NUM_FRAMES * FRAME_SIZE);
if (ret) err_errno("posix_memalign() failed");
ret = xsk_umem__create(&xdp_socket.umem.umem, buffer,
NUM_FRAMES * FRAME_SIZE, &xdp_socket.umem.fq,
&xdp_socket.umem.cq, &cfg);
ret = xsk_umem__create(&xdp_socket.umem.umem, buffer, NUM_FRAMES * FRAME_SIZE,
&xdp_socket.umem.fq, &xdp_socket.umem.cq, &cfg);
if (ret) err("xsk_umem__create() failed");
xdp_socket.umem.buffer = buffer;
......@@ -313,26 +294,7 @@ void setup_poll_fd(void) {
static int received;
static uint32_t idx_rx = 0, idx;
static void poll_wakeup(struct timespec ts, int margin) {
int ret;
struct timespec ts_prev, current;
ts_prev = ts;
substract_ns(&ts_prev, margin * 1000);
ret = clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, &ts_prev, NULL);
if (ret) {
fprintf(stderr, "clock_nanosleep returned error: %d, aborting...\n", ret);
exit(EXIT_FAILURE);
}
do {
clock_gettime(CLOCK_REALTIME, &current);
} while(calcdiff_ns_signed(ts, current) > 1000);
}
static void parse_raw_packet(uint64_t addr, size_t len)
{
static void parse_raw_packet(uint64_t addr, size_t len) {
char *packet;
struct ethhdr *eth;
struct iphdr *ip;
......@@ -363,29 +325,12 @@ int recv_xdp_packet(struct timespec next) {
struct timespec next_pre, current;
int k = 0;
if (params->xdp_polling_mode == 0) {
ret = poll(fds, 1, -1);
if (ret == 0) {
return -1;
} else if (ret < 0)
if (ret != 1)
error(EXIT_FAILURE, errno, "poll failed");
received = xsk_ring_cons__peek(&xdp_socket.rx, 1, &idx_rx);
} else {
next_pre = next;
substract_ns(&next_pre, 120 * 1000);
ret = clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, &next_pre, NULL);
if (ret) {
fprintf(stderr, "clock_nanosleep returned error: %d, aborting...\n", ret);
exit(EXIT_FAILURE);
}
do {
received = xsk_ring_cons__peek(&xdp_socket.rx, 1, &idx_rx);
} while(!received);
}
if (received != 1)
error(EXIT_FAILURE, errno, "Received %d packets", received);
......@@ -408,8 +353,7 @@ void recv_xdp_cleanup(void) {
/* Add that particular buffer back to the fill queue */
if (xsk_prod_nb_free(&xdp_socket.umem.fq, received)) {
ret =
xsk_ring_prod__reserve(&xdp_socket.umem.fq, received, &idx);
ret = xsk_ring_prod__reserve(&xdp_socket.umem.fq, received, &idx);
if (ret != received) err("xsk_ring_prod__reserve() failed");
......@@ -427,10 +371,16 @@ void close_xdp_socket(void) {
}
#else
void init_xdp_recv(ingress_param_t * _params, ingress_stat_t *_stats) { (void) _params; (void) _stats; }
void init_xdp_recv(ingress_param_t *_params, ingress_stat_t *_stats) {
(void)_params;
(void)_stats;
}
void setup_poll_fd(void) {}
void close_xdp_socket(void) {}
int recv_xdp_packet(struct timespec next) { (void) next; return 0; }
int recv_xdp_packet(struct timespec next) {
(void)next;
return 0;
}
void recv_xdp_cleanup(void) {}
#endif
......@@ -446,8 +396,7 @@ static int process_socket_error_queue(void) {
struct cmsghdr *cmsg;
__u64 tstamp = 0;
struct iovec iov = {.iov_base = err_buffer,
.iov_len = sizeof(err_buffer)};
struct iovec iov = {.iov_base = err_buffer, .iov_len = sizeof(err_buffer)};
struct msghdr msg = {.msg_iov = &iov,
.msg_iovlen = 1,
.msg_control = msg_control,
......@@ -466,15 +415,13 @@ static int process_socket_error_queue(void) {
switch (serr->ee_code) {
case SO_EE_CODE_TXTIME_INVALID_PARAM:
fprintf(
stderr,
fprintf(stderr,
"packet with tstamp %llu dropped "
"due to invalid params\n",
tstamp);
return 0;
case SO_EE_CODE_TXTIME_MISSED:
fprintf(
stderr,
fprintf(stderr,
"packet with tstamp %llu dropped "
"due to missed deadline\n",
tstamp);
......
packet-exchange/src/recv_packet.h
View file @ 2ba307c4
......@@ -8,7 +8,6 @@ typedef struct ingress_param {
int use_timestamps;
int enable_ts_tracemark;
int xdp_polling_mode;
uint64_t interval;
size_t tx_buffer_len;
......
packet-exchange/src/server.c
View file @ 2ba307c4
......@@ -48,9 +48,6 @@ typedef struct thread_param {
uint64_t start_ts;
int poll;
int poll_margin;
} thread_param_t;
typedef struct main_params {
......@@ -93,7 +90,7 @@ static char ts_tracemark_buf[64];
static void help(char *argv[]) {
printf(
"Usage: %s [-f IF -a CPU -p PRIO -i USEC -r USEC] [-b CLIENT_IP] [-d "
"BUF_LEN -cgtv] [-T LATENCY_THRESHOLD -G] [-x POLL_MODE -X]\n\n"
"BUF_LEN -cgtv] [-T LATENCY_THRESHOLD -G] [-xX]\n\n"
" -h Show help\n"
" -f IF Set the network interface to be used\n"
" -a CPU Pin the real time thread to CPU\n"
......@@ -103,13 +100,11 @@ static void help(char *argv[]) {
" -d BUF_LEN Set the length of tx buffer\n"
" -c Receive timestamp and emit signal\n"
" -s NS Common start time reference\n"
" -P USEC Do polling to wakeup signal thread with specified margin\n"
" -C Receive timestamp and print difference with current time\n"
" -b CLIENT_IP Server side RTT\n"
" -g Print histograms to sdtout on exit\n"
" -t Enable timestamps\n"
" -x POLL_MODE Use AF_XDP sockets\n"
" POLL_MODE: 0 for polling, 1 for combination of both\n"
" -x Use AF_XDP sockets\n"
" -X Trace during XDP packet reception\n"
" -T THRESHOLD Enable tracing until THRESHOLD is reached\n"
" -M Send tracemark when packet is received\n"
......@@ -119,30 +114,13 @@ static void help(char *argv[]) {
argv[0]);
}
static void poll_wakeup(struct timespec ts, int margin) {
int ret;
struct timespec ts_prev, current;
ts_prev = ts;
substract_ns(&ts_prev, margin * 1000);
ret = clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, &ts_prev, NULL);
if (ret) {
fprintf(stderr, "clock_nanosleep returned error: %d, aborting...\n", ret);
exit(EXIT_FAILURE);
}
do {
clock_gettime(CLOCK_REALTIME, &current);
} while(calcdiff_ns_signed(ts, current) > 1000);
}
static void *emit_signal_thread(void *p) {
(void)p;
cpu_set_t mask;
struct timespec current;
struct timespec previous_emit, previous_ts;
int64_t emit_diff, ts_diff;
int ret;
// Set thread CPU affinity
if (thread_params.affinity_cpu) {
......@@ -157,15 +135,18 @@ static void *emit_signal_thread(void *p) {
for (int i = 0;;i++) {
pthread_cond_wait(&emit_signal_ts_received, &emit_signal_mutex);
clock_gettime(CLOCK_REALTIME, &current);
poll_wakeup(emit_signal_next, thread_params.poll_margin);
ret = clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, &emit_signal_next, NULL);
if (ret) {
fprintf(stderr, "clock_nanosleep returned error: %d, aborting...\n", ret);
exit(EXIT_FAILURE);
}
toggle_gpio();
clock_gettime(CLOCK_REALTIME, &current);
// Check if something went wrong
if(i > 0) {
clock_gettime(CLOCK_REALTIME, &current);
emit_diff = calcdiff_ns_signed(current, previous_emit);
ts_diff = calcdiff_ns_signed(emit_signal_next, previous_ts);
if((emit_diff < ((int64_t)thread_params.interval) - ERROR_MARGIN_NS) ||
......@@ -397,8 +378,6 @@ int main(int argc, char *argv[]) {
thread_params.affinity_cpu = 0;
thread_params.enable_diff_ts = 0;
thread_params.enable_receive_tracemark = 0;
thread_params.poll = 0;
thread_params.poll_margin = 75;
thread_params.start_ts = 0;
main_params.refresh_rate = 50000;
main_params.verbose = 0;
......@@ -557,7 +536,7 @@ static void process_options(int argc, char *argv[]) {
int network_if_specified = 0;
for (;;) {
int c = getopt(argc, argv, "a:b:cCs:d:f:ghi:p:r:tvx:XT:GMSP:");
int c = getopt(argc, argv, "a:b:cCs:d:f:ghi:p:r:tvxXT:GMS");
if (c == -1) break;
......@@ -613,7 +592,6 @@ static void process_options(int argc, char *argv[]) {
break;
case 'x':
tsn_task = XDP_TASK;
ingress_params.xdp_polling_mode = atoi(optarg);
break;
case 'X':
main_params.enable_xdp_tracing = 1;
......@@ -628,10 +606,6 @@ static void process_options(int argc, char *argv[]) {
case 'S':
ingress_params.enable_ts_tracemark = 1;
break;
case 'P':
thread_params.poll = 1;
thread_params.poll_margin = atoi(optarg);
break;
}
}
......
scripts/run-server
View file @ 2ba307c4
......@@ -10,18 +10,14 @@ usage() {
cat << ENDUSAGE
Usage: $0 [-h] [-I if] [SERVER] | TCPDUMP [TRACE_OPTS]
-h Show help
SERVER: -bct ((-x | -X) POLL) -g INTERVAL -a CPU
SERVER: -bctX -g INTERVAL -a CPU
Options passed to the C server program (everything here is optional)
-b Send back packets for round trip measurements
-c USEC Emit a signal on GPIO at the timestamp given in packet, specify interval
(to be used with -c option in client program)
-O USEC Do polling to wakeup signal thread with specified margin
-C USEC Measure difference between current time and timestamp sent in tx data
-t Use SO_TIMESTAMPS to see how much time packet spent in kernel
-x POLL Use XDP sockets, with a global libbpf installation
-X POLL Use XDP sockets, with libbpf located in \$HOME/libbpf folder
POLL: Polling mode used in server program, 0 to poll with poll function,
1 to do active polling
-X Use XDP sockets, with libbpf located in \$HOME/libbpf folder
-s NS Specify a CLOCK_REALTIME timestamp at which client should start
(to be used with PTP) (interval needs to be specified with -j)
-j USEC Specify interval (used with -s)
......@@ -60,7 +56,7 @@ tracecmd_events="-e irq -e sched -e net -e napi"
tracecmd_opts=""
cpu=1
while getopts "j:a:b:c:C:htx:X:s:d:i:g:I:T:E:P:B:MSQO:" opt; do
while getopts "j:a:b:c:C:htx:Xs:d:i:g:I:T:E:P:B:MSQ" opt; do
case "${opt}" in
h )
usage
......@@ -78,9 +74,6 @@ while getopts "j:a:b:c:C:htx:X:s:d:i:g:I:T:E:P:B:MSQO:" opt; do
c )
server_options+=" -c -i ${OPTARG}"
;;
O )
server_options+=" -P ${OPTARG} "
;;
C )
server_options+=" -C -i ${OPTARG}"
;;
......@@ -107,7 +100,7 @@ while getopts "j:a:b:c:C:htx:X:s:d:i:g:I:T:E:P:B:MSQO:" opt; do
interface="${OPTARG}"
;;
X )
server_options+=" -x ${OPTARG}"
server_options+=" -x "
make_opts=" -e WITH_GIT_XDP=1"
;;
s )
......
scripts/test-board-synchro
View file @ 2ba307c4
......@@ -4,13 +4,12 @@ script_dir=$(dirname $(realpath $0))
usage() {
cat << ENDUSAGE
Usage: $0 [-h] [-i USEC -c USEC -t MSEC] [-T] BOARD1_HOSTNAME BOARD2_HOSTNAME
Usage: $0 [-h] [-i USEC -c USEC -t MSEC] [-TX] BOARD1_HOSTNAME BOARD2_HOSTNAME
-h Show help
-i USEC Specify which interval to use in client
-c USEC Specify which offset to use for the timestamp in the packet
-t MSEC Set the start timestamp offset
-P USEC Do polling to wakeup signal thread with specified margin
-X POLL_MODE Use XDP with specified poll mode
-X Use XDP
-T Enable tracing on the boards
BOARD_HOSTNAME Uses /etc/hosts to find the IP address associated to the hostname
ENDUSAGE
......@@ -23,7 +22,7 @@ interval=1000
server_opts=""
ts_offset=2000
while getopts "hc:i:o:rt:TP:X:" opt; do
while getopts "hc:i:o:rt:TX" opt; do
case "${opt}" in
h )
usage
......@@ -40,11 +39,8 @@ while getopts "hc:i:o:rt:TP:X:" opt; do
T )
enable_tracing=1
;;
P )
server_opts+=" -P ${OPTARG} "
;;
X )
server_opts+=" -X ${OPTARG} "
server_opts+=" -X "
;;
* )
usage
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7