Clean up all the code, add timestamps and histograms printing to the server

packet-exchange/src/client.c

 /*
- * Real time packet sending thread
+ * Real time packet sending client
  *
  * Large portions taken from cyclictest
  *
@@ -27,7 +27,7 @@
 typedef struct thread_stat {
   int nb_cycles;
   uint64_t rtt;
-  struct packet_timestamps packet_ts;
+  packet_timestamps_t packet_ts;
 } thread_stat_t;
 
 typedef struct thread_param {
@@ -74,6 +74,9 @@ enum TSNTask { SEND_PACKET_TASK,
                RTT_TASK };
 static enum TSNTask tsn_task;
 
+struct timespec measures_start;
+struct timespec measures_end;
+
 static void help(char *argv[]) {
   printf("Usage: %s -f IF  [-abethgv] [-d BUF_LEN] [-i USEC] [-l N] [-p PRIO] [-r USEC]\n\n", argv[0]);
   printf("       -a                    Run the real time thread on CPU1\n");
@@ -99,7 +102,7 @@ static void *packet_sending_thread(void *p) {
   uint64_t next_txtime;
   struct sched_param priority;
   thread_param_t *param = (thread_param_t *)p;
-  thread_stat_t * stats = &param->stats;
+  thread_stat_t *stats = &param->stats;
   cpu_set_t mask;
 
   // Set thread CPU affinity
@@ -122,6 +125,7 @@ static void *packet_sending_thread(void *p) {
   // Send packet while thread is sleeping
   next_txtime += (param->interval) / 2;
 
+  clock_gettime(CLOCK_MONOTONIC, &measures_start);
   // Packet sending loop
   for (stats->nb_cycles = 0;; stats->nb_cycles++) {
     if (param->max_cycles)
@@ -184,7 +188,7 @@ int main(int argc, char *argv[]) {
 
   // Initialize the UDP packet receiving socket if RTT is measured
   if (tsn_task == RTT_TASK)
-    init_udp_recv();
+    init_udp_recv(0, network_config.network_if);
 
   // Create the real time thread
   if (pthread_create(&thread, NULL, packet_sending_thread, (void *)param))
@@ -201,13 +205,15 @@ int main(int argc, char *argv[]) {
 
       } else if (enable_timestamps) {
 
-        printf("(%d)\n", stats->nb_cycles);
-        printf("      Enter send_udp_packet timestamp: %" PRIu64 "\n",
-               stats->packet_ts.user_enter_send);
-        printf("      Call sendmsg timestamp         : %" PRIu64 "\n",
-               stats->packet_ts.user_call_sendmsg);
-        printf("      Leave kernel timestamp         : %" PRIu64 "\n",
-               stats->packet_ts.kernel_leave);
+        printf("(%d)  Enter send_udp_packet timestamp: %" PRIu64 "\n",
+               stats->nb_cycles,
+               stats->packet_ts.enter_user_space);
+        printf("(%d)  Call sendmsg timestamp         : %" PRIu64 "\n",
+               stats->nb_cycles,
+               stats->packet_ts.enter_kernel);
+        printf("(%d)  Leave kernel timestamp         : %" PRIu64 "\n",
+               stats->nb_cycles,
+               stats->packet_ts.leave_kernel);
       }
     }
 
@@ -237,56 +243,53 @@ static void do_tsn_task(struct thread_param *param, uint64_t next_txtime) {
   } else if (tsn_task == RTT_TASK) {
 
     clock_gettime(CLOCK_MONOTONIC, &t1);
-    send_udp_packet(enable_etf, enable_timestamps, next_txtime,
-                    network_config.ip_address, histograms);
-    recv_udp_packet();
+    send_udp_packet(0, 0, next_txtime,
+                    network_config.ip_address, NULL);
+    recv_udp_packet(0, NULL);
     clock_gettime(CLOCK_MONOTONIC, &t2);
 
     param->stats.rtt = calcdiff_ns(t2, t1);
-
   }
 }
 
 // Print histograms in .json format
 static void print_histograms() {
 
-  printf("{\"measure_type\": \"packet_timestamps\",\
+  uint64_t duration;
+  int duration_hour, duration_minutes, interval;
+
+  clock_gettime(CLOCK_MONOTONIC, &measures_end);
+
+  duration = calcdiff_ns(measures_end, measures_start);
+  duration_hour = duration / NSEC_PER_SEC / 3600;
+  duration_minutes = duration / NSEC_PER_SEC / 60 - duration_hour * 60;
+
+  interval = param->interval / 1000;
+
+  printf("{\"measure_sets\": [{\
+             \"measure_type\": \"packet_send_timestamps\",\
              \"props_names\": [\"user_space\", \"kernel_space\"],\
              \"units\": [\"us\", \"us\"],\
-           \"props\": [");
+             \"props_type\": \"histogram\",\
+             \"metadata\": {\
+                \"i\": \"%dus\", \"duration\": \"%dh%d\"\
+              },\
+             \"props\": [", interval, duration_hour, duration_minutes);
 
   int max_hist_val = 0;
-  for (int i = 0; i < NB_HISTOGRAMS; i++) {
+  for (int i = 0; i < 2; i++)
     for (int j = 0; j < MAX_HIST_VAL; j++)
       if (histograms[i][j])
         max_hist_val = j > max_hist_val ? j : max_hist_val;
-  }
-
-  for (int i = 0; i < NB_HISTOGRAMS; i++) {
 
+  for (int i = 0; i < 2; i++) {
     printf("[");
-    for (int j = 0; j < max_hist_val; j++) {
-      if (j + 1 < max_hist_val)
-        printf("%" PRIi64 ", ", histograms[i][j]);
-      else
-        printf("%" PRIi64, histograms[i][j]);
+    for (int j = 0; j < max_hist_val; j++)
+      printf("%" PRIi64 "%s", histograms[i][j], (j + 1 < max_hist_val ? ", " : ""));
+    printf("%s", (i + 1 < 2 ? "], " : "]"));
   }
-    if (i + 1 < NB_HISTOGRAMS)
-      printf("], ");
-    else
-      printf("]");
-  }
-
-  int interval = param->interval / 1000;
-  uint64_t duration = interval * param->stats.nb_cycles;
-  int duration_hour = duration / 1000000;
-  duration_hour /= 3600;
-  int duration_minutes = duration / 1000000;
-  duration_minutes /= 60;
-  duration_minutes -= duration_hour * 60;
 
-  printf("], ");
-  printf("\"props_type\": \"histogram\", \"metadata\": {\"i\": \"%dus\", \"duration\": \"%dh%d\"}}\n", interval, duration_hour, duration_minutes);
+  printf(   "]]}]}\n");
 }
 
 static void sigint_handler(int sig_num) {
@@ -359,14 +362,14 @@ static void process_options(int argc, char *argv[]) {
     }
   }
 
-  if(!network_if_specified) {
+  if (!network_if_specified) {
     fprintf(stderr, "You need to specifiy an network interface\n");
     help(argv);
     exit(EXIT_FAILURE);
   }
 
   if (argc != optind + 1) {
-    if( argc < optind + 1)
+    if (argc < optind + 1)
       fprintf(stderr, "You need to specifiy an IP address\n");
     else
       fprintf(stderr, "Too many arguments\n");
@@ -375,4 +378,3 @@ static void process_options(int argc, char *argv[]) {
   }
   strcpy(network_config.ip_address, argv[optind]);
 }
-

packet-exchange/src/recv_packet.c

@@ -2,31 +2,60 @@
 #include <arpa/inet.h>
 #include <errno.h>
 #include <error.h>
+#include <fcntl.h>
+#include <ifaddrs.h>
 #include <inttypes.h>
+#include <linux/errqueue.h>
+#include <linux/ethtool.h>
+#include <linux/net_tstamp.h>
+#include <linux/sockios.h>
+#include <net/if.h>
 #include <netdb.h>
 #include <netinet/in.h>
+#include <poll.h>
 #include <pthread.h>
 #include <sched.h>
+#include <signal.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/ioctl.h>
 #include <sys/socket.h>
+#include <sys/stat.h>
 #include <sys/types.h>
-#include <time.h>
 #include <unistd.h>
 
+#include "send_packet.h"
 #include "utilities.h"
 
-#define SERVER_PORT "50000"
+
 #define BUFFER_SIZE 1024
 
+static void process_timestamps(packet_timestamps_t *packet_ts, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]);
+
 static char rx_buffer[BUFFER_SIZE];
 static int sock_fd;
 
-int init_udp_recv() {
-  int status;
-  int sock_fd = -1;
+static int so_timestamping_flags =
+    SOF_TIMESTAMPING_RX_SOFTWARE | SOF_TIMESTAMPING_SOFTWARE;
+
+// Sets the interface
+static int set_if(char *network_if) {
+  struct ifreq ifreq;
+
+  memset(&ifreq, 0, sizeof(ifreq));
+  strncpy(ifreq.ifr_name, network_if, sizeof(ifreq.ifr_name) - 1);
+
+  if (ioctl(sock_fd, SIOCGIFINDEX, &ifreq))
+    error(EXIT_FAILURE, errno, "ioctl SIOCGIFINDEX failed\n");
+
+  return ifreq.ifr_ifindex;
+}
+
+int init_udp_recv(int use_timestamps, char *network_if) {
+  int getaddrinfo_err;
+  int set_if_err;
   struct addrinfo hints, *servinfo, *servinfo_it;
 
   memset(&hints, 0, sizeof hints);
@@ -34,9 +63,9 @@ int init_udp_recv() {
   hints.ai_socktype = SOCK_DGRAM;
   hints.ai_flags = AI_PASSIVE;
 
-  status = getaddrinfo(NULL, SERVER_PORT, &hints, &servinfo);
-  if (status != 0) {
-    fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(status));
+  getaddrinfo_err = getaddrinfo(NULL, SERVER_PORT, &hints, &servinfo);
+  if (getaddrinfo_err != 0) {
+    fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(getaddrinfo_err));
     exit(EXIT_FAILURE);
   }
 
@@ -54,23 +83,190 @@ int init_udp_recv() {
 
   freeaddrinfo(servinfo);
 
-  if(sock_fd == -1)
+  if (sock_fd == -1)
     error(EXIT_FAILURE, errno, "Couldn't create receive socket");
 
-  printf("waiting to receive...\n");
+  set_if_err = set_if(network_if);
+  if (set_if_err < 0)
+    error(EXIT_FAILURE, errno, "Couldn't set interface\n");
+
+  if (setsockopt(sock_fd, SOL_SOCKET, SO_BINDTODEVICE, network_if,
+                 strlen(network_if)))
+    error(EXIT_FAILURE, errno, "setsockopt SO_BINDTODEVICE failed\n");
+
+  if (use_timestamps) {
+    if (setsockopt(sock_fd, SOL_SOCKET, SO_TIMESTAMPING, &so_timestamping_flags,
+                   sizeof(so_timestamping_flags)))
+      error(EXIT_FAILURE, errno, "setsockopt SO_TIMESTAMPING failed\n");
+  }
 
   return sock_fd;
 }
 
-void recv_udp_packet() {
-#ifdef DEBUG
-  int bytes_received = 0;
+/*
+ * Receives udp packets
+ */
+packet_timestamps_t recv_udp_packet(int use_timestamps, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]) {
+
+  struct msghdr msg; // Message hardware, sent to the socket
+  struct iovec iov; // The iovec structures stores the RX buffer
+  struct sockaddr_in sin;
+
+  // Poll file descriptor, used to poll for timestamp messages
+  struct pollfd poll_fd = {sock_fd, POLLPRI, 0};
+  int recvmsgerr, pollerr;
+
+  packet_timestamps_t packet_ts;
+  struct timespec ts;
+
+  if (use_timestamps) {
+    clock_gettime(CLOCK_REALTIME, &ts);
+    packet_ts.enter_user_space = ts_to_uint(ts);
+  }
+
+  iov.iov_base = &rx_buffer;
+  iov.iov_len = BUFFER_SIZE - 1;
+
+  memset(&msg, 0, sizeof(msg));
+  msg.msg_name = &sin;
+  msg.msg_namelen = sizeof(sin);
+  msg.msg_iov = &iov;
+  msg.msg_iovlen = 1;
+
+  if (use_timestamps) {
+    clock_gettime(CLOCK_REALTIME, &ts);
+    packet_ts.enter_kernel = ts_to_uint(ts);
+  }
+
+  recvmsgerr = recvmsg(sock_fd, &msg, 0);
+  if (recvmsgerr < 0)
+    error(EXIT_FAILURE, errno, "recvmsg failed, ret value: %d\n", recvmsgerr);
+
+  if (use_timestamps) {
+    pollerr = poll(&poll_fd, 1, 0);
+    if (pollerr > 0)
+      process_timestamps(&packet_ts, histograms);
+    else
+      fprintf(stderr, "select failed\n");
+  }
+
+  return packet_ts;
+}
+
+static void fill_histograms(packet_timestamps_t *packet_ts, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]) {
+
+  uint64_t user_space_time = packet_ts->enter_kernel - packet_ts->enter_user_space;
+  uint64_t kernel_space_time = packet_ts->leave_kernel - packet_ts->enter_kernel;
+
+  user_space_time /= 1000u;
+  kernel_space_time /= 1000u;
+
+  if (user_space_time > MAX_HIST_VAL) {
+    fprintf(stderr, "user_space_time value too high: %" PRIu64 "us\n", user_space_time);
+    exit(EXIT_FAILURE);
+  }
+  if (kernel_space_time > MAX_HIST_VAL) {
+    fprintf(stderr, "kernel_space_time value too high: %" PRIu64 "us\n", kernel_space_time);
+    exit(EXIT_FAILURE);
+  }
+
+  histograms[0][user_space_time]++;
+  histograms[1][kernel_space_time]++;
+}
+
+static void process_timestamps(packet_timestamps_t *packet_ts, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]) {
+  char data[256];
+  struct msghdr msg;
+  struct iovec entry;
+  struct sockaddr_in from_addr;
+  struct {
+    struct cmsghdr cm;
+    char control[512];
+  } control;
+  struct cmsghdr *cmsg;
+
+  memset(&msg, 0, sizeof(msg));
+  msg.msg_iov = &entry;
+  msg.msg_iovlen = 1;
+  entry.iov_base = data;
+  entry.iov_len = sizeof(data);
+  msg.msg_name = (caddr_t)&from_addr;
+  msg.msg_namelen = sizeof(from_addr);
+  msg.msg_control = &control;
+  msg.msg_controllen = sizeof(control);
+
+  if (recvmsg(sock_fd, &msg, MSG_ERRQUEUE | MSG_DONTWAIT) == -1) {
+    fprintf(stderr, "recvmsg failed\n");
+    return;
+  }
+
+  for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
+    if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SO_TIMESTAMPING) {
+
+      struct timespec *stamp = (struct timespec *)CMSG_DATA(cmsg);
+      packet_ts->leave_kernel = ts_to_uint(*stamp);
+      fill_histograms(packet_ts, histograms);
+
+    } else {
 
-  bytes_received = recvfrom(sock_fd, rx_buffer, BUFFER_SIZE - 1, 0, NULL, NULL);
+#ifdef DEBUG
 
-  if (bytes_received == -1)
-    error(EXIT_FAILURE, errno, "Error while attempting to receive packets");
-#else
-  recvfrom(sock_fd, rx_buffer, BUFFER_SIZE - 1, 0, NULL, NULL);
+      fprintf(stderr, "process_timestamps: level %d type %d", cmsg->cmsg_level,
+              cmsg->cmsg_type);
 #endif
+
+    }
+  }
 }
+
+#ifdef DEBUG
+
+/*
+ * Code from scheduled_tx_tools
+ */
+static int process_socket_error_queue() {
+  uint8_t msg_control[CMSG_SPACE(sizeof(struct sock_extended_err))];
+  unsigned char err_buffer[256];
+  struct sock_extended_err *serr;
+  struct cmsghdr *cmsg;
+  __u64 tstamp = 0;
+
+  struct iovec iov = {.iov_base = err_buffer, .iov_len = sizeof(err_buffer)};
+  struct msghdr msg = {.msg_iov = &iov,
+                       .msg_iovlen = 1,
+                       .msg_control = msg_control,
+                       .msg_controllen = sizeof(msg_control)};
+
+  if (recvmsg(sock_fd, &msg, MSG_ERRQUEUE) == -1) {
+    fprintf(stderr, "recvmsg failed");
+    return -1;
+  }
+
+  cmsg = CMSG_FIRSTHDR(&msg);
+  while (cmsg != NULL) {
+    serr = (void *)CMSG_DATA(cmsg);
+    if (serr->ee_origin == SO_EE_ORIGIN_TXTIME) {
+      tstamp = ((__u64)serr->ee_data << 32) + serr->ee_info;
+
+      switch (serr->ee_code) {
+      case SO_EE_CODE_TXTIME_INVALID_PARAM:
+        fprintf(stderr,
+                "packet with tstamp %llu dropped due to invalid params\n",
+                tstamp);
+        return 0;
+      case SO_EE_CODE_TXTIME_MISSED:
+        fprintf(stderr,
+                "packet with tstamp %llu dropped due to missed deadline\n",
+                tstamp);
+        return 0;
+      default:
+        return -1;
+      }
+    }
+
+    cmsg = CMSG_NXTHDR(&msg, cmsg);
+  }
+
+  return 0;
+}
+#endif

packet-exchange/src/recv_packet.h

 #ifndef RECV_PACKET
 #define RECV_PACKET
 
-int init_udp_recv();
-void recv_udp_packet();
+#include "utilities.h"
+
+int init_udp_recv(int use_timestamps, char *network_if);
+packet_timestamps_t recv_udp_packet(int use_timestamps, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]);
 
 #endif

packet-exchange/src/send_packet.c

@@ -7,7 +7,6 @@
  *
  */
 
-
 #define _GNU_SOURCE
 #include <arpa/inet.h>
 #include <errno.h>
@@ -36,12 +35,12 @@
 #include <sys/types.h>
 #include <unistd.h>
 
-#include "utilities.h"
 #include "send_packet.h"
+#include "utilities.h"
 
 #define MESSAGE ((uint32_t)0x00FACADE)
 
-static void process_timestamps(struct packet_timestamps *packet_ts, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]);
+static void process_timestamps(packet_timestamps_t *packet_ts, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]);
 static void init_tx_buffer(size_t _tx_buffer_len);
 
 static int so_priority = 3;
@@ -89,7 +88,7 @@ static void init_tx_buffer(size_t _tx_buffer_len) {
  */
 void init_udp_send(int use_etf, int use_timestamps, int packet_priority,
                    char *network_if, size_t _tx_buffer_len) {
-  int index;
+  int set_if_err;
 
   struct timespec ts_mon;
   struct timespec ts_tai;
@@ -104,10 +103,12 @@ void init_udp_send(int use_etf, int use_timestamps, int packet_priority,
   so_priority = packet_priority;
 
   sock_fd = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP);
-  if (sock_fd < 0) error(EXIT_FAILURE, errno, "Socket creation failed\n");
+  if (sock_fd < 0)
+    error(EXIT_FAILURE, errno, "Socket creation failed\n");
 
-  index = set_if(network_if);
-  if (index < 0) error(EXIT_FAILURE, errno, "Couldn't set interface\n");
+  set_if_err = set_if(network_if);
+  if (set_if_err < 0)
+    error(EXIT_FAILURE, errno, "Couldn't set interface\n");
 
   if (setsockopt(sock_fd, SOL_SOCKET, SO_PRIORITY, &so_priority,
                  sizeof(so_priority)))
@@ -146,25 +147,26 @@ uint64_t get_txtime() {
 /*
  * Sends udp packets
  */
-struct packet_timestamps send_udp_packet(int use_etf, int use_timestamps,
+packet_timestamps_t send_udp_packet(int use_etf, int use_timestamps,
                                          uint64_t txtime,
                                          const char *server_ip,
                                          int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]) {
-  char control[CMSG_SPACE(sizeof(txtime))] = {};
-  struct sockaddr_in sin;
-  struct cmsghdr *cmsg;
-  struct msghdr msg;
-  struct iovec iov;
-  int sendmsgerr;
-  int res;
+  struct msghdr msg; // Message hardware, sent to the socket
+  struct cmsghdr *cmsg; // Control message hardware, for txtime
+  char control[CMSG_SPACE(sizeof(txtime))] = {}; // Stores txtime
+  struct iovec iov; // The iovec structures stores the TX buffer
+  // Poll file descriptor, used to poll for timestamp messages
   struct pollfd poll_fd = {sock_fd, POLLPRI, 0};
+  int sendmsgerr, pollerr;
+  // Server address
+  struct sockaddr_in sin;
 
-  struct packet_timestamps packet_ts;
+  packet_timestamps_t packet_ts;
   struct timespec ts;
 
   if (use_timestamps) {
     clock_gettime(CLOCK_REALTIME, &ts);
-    packet_ts.user_enter_send = ts_to_uint(ts);
+    packet_ts.enter_user_space = ts_to_uint(ts);
   }
 
   memset(&sin, 0, sizeof(sin));
@@ -196,7 +198,7 @@ struct packet_timestamps send_udp_packet(int use_etf, int use_timestamps,
 
   if (use_timestamps) {
     clock_gettime(CLOCK_REALTIME, &ts);
-    packet_ts.user_call_sendmsg = ts_to_uint(ts);
+    packet_ts.enter_kernel = ts_to_uint(ts);
   }
 
   sendmsgerr = sendmsg(sock_fd, &msg, 0);
@@ -204,8 +206,8 @@ struct packet_timestamps send_udp_packet(int use_etf, int use_timestamps,
     error(EXIT_FAILURE, errno, "sendmsg failed, ret value: %d\n", sendmsgerr);
 
   if (use_timestamps) {
-    res = poll(&poll_fd, 1, 0);
-    if (res > 0)
+    pollerr = poll(&poll_fd, 1, 0);
+    if (pollerr > 0)
       process_timestamps(&packet_ts, histograms);
     else
       fprintf(stderr, "select failed\n");
@@ -214,18 +216,19 @@ struct packet_timestamps send_udp_packet(int use_etf, int use_timestamps,
   return packet_ts;
 }
 
-static void fill_histograms(struct packet_timestamps *packet_ts, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]) {
-  uint64_t user_space_time = packet_ts->user_call_sendmsg - packet_ts->user_enter_send;
-  uint64_t kernel_space_time = packet_ts->kernel_leave - packet_ts->user_call_sendmsg;
+static void fill_histograms(packet_timestamps_t *packet_ts, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]) {
+
+  uint64_t user_space_time = packet_ts->enter_kernel - packet_ts->enter_user_space;
+  uint64_t kernel_space_time = packet_ts->leave_kernel - packet_ts->enter_kernel;
 
   user_space_time /= 1000u;
   kernel_space_time /= 1000u;
 
-  if(user_space_time > MAX_HIST_VAL) {
+  if (user_space_time > MAX_HIST_VAL) {
     fprintf(stderr, "user_space_time value too high: %" PRIu64 "us\n", user_space_time);
     exit(EXIT_FAILURE);
   }
-  if(kernel_space_time > MAX_HIST_VAL) {
+  if (kernel_space_time > MAX_HIST_VAL) {
     fprintf(stderr, "kernel_space_time value too high: %" PRIu64 "us\n", kernel_space_time);
     exit(EXIT_FAILURE);
   }
@@ -234,7 +237,7 @@ static void fill_histograms(struct packet_timestamps *packet_ts, int64_t histogr
   histograms[1][kernel_space_time]++;
 }
 
-static void process_timestamps(struct packet_timestamps *packet_ts, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]) {
+static void process_timestamps(packet_timestamps_t *packet_ts, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]) {
   char data[256];
   struct msghdr msg;
   struct iovec entry;
@@ -262,14 +265,18 @@ static void process_timestamps(struct packet_timestamps *packet_ts, int64_t hist
 
   for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
     if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SO_TIMESTAMPING) {
+
       struct timespec *stamp = (struct timespec *)CMSG_DATA(cmsg);
-      packet_ts->kernel_leave = ts_to_uint(*stamp);
+      packet_ts->leave_kernel = ts_to_uint(*stamp);
       fill_histograms(packet_ts, histograms);
+
     } else {
+
 #ifdef DEBUG
       fprintf(stderr, "process_timestamps: level %d type %d", cmsg->cmsg_level,
               cmsg->cmsg_type);
 #endif
+
     }
   }
 }

packet-exchange/src/send_packet.h

 #ifndef SEND_PACKET_H
 #define SEND_PACKET_H
 
-#include <stdint.h>
-#include <stdio.h>
-
 #include "utilities.h"
 
-struct packet_timestamps {
-  uint64_t user_enter_send;
-  uint64_t user_call_sendmsg;
-  uint64_t kernel_leave;
-};
-
-void init_udp_send(int use_etf, int use_timestamps, int so_priority, char * network_if, size_t tx_buffer_len);
-struct packet_timestamps send_udp_packet(int use_etf, int use_timestamps, uint64_t txtime, const char *server_ip, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]);
+void init_udp_send(int use_etf, int use_timestamps, int so_priority, char *network_if, size_t tx_buffer_len);
+packet_timestamps_t send_udp_packet(int use_etf, int use_timestamps, uint64_t txtime, const char *server_ip, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]);
 
 #endif

packet-exchange/src/server.c

 /*
- * Real time packet receiving thread
- *
- * Bash options:
- *
- *  -a                    Run the real time thread on CPU1
- *  -b CLIENT_IP          Server side RTT
- *  -d TX_BUFFER_LEN      Set the length of tx buffer
- *  -f IF                 Set the network interface to be used
- *  -p PRIO               Run the real time thread at priority PRIO
- *  -r USEC               Refresh the non real time main thread every USEC
- * microseconds
+ * Real time packet receiving server
  *
  * Large portions taken from cyclictest
  *
@@ -24,6 +14,7 @@
 #include <netinet/in.h>
 #include <pthread.h>
 #include <sched.h>
+#include <signal.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -39,34 +30,67 @@
 
 #define BUFFER_SIZE 1024
 
-enum TSNTask { RECV_PACKET_TASK, RTT_TASK };
+// Structs
 
 typedef struct thread_stat {
   uint64_t min_interval;
   uint64_t max_interval;
   int packets_received;
+  packet_timestamps_t packet_ts;
 } thread_stat_t;
 
 typedef struct thread_param {
   int priority;
 
-  int enable_affinity;
-  enum TSNTask tsn_task;
-
-  int sockfd;
-  char *ip_address;
-  char network_if[256];
-
   thread_stat_t stats;
 } thread_param_t;
 
+typedef struct network_config {
+  size_t tx_buffer_len;
+  char ip_address[256];
+  char network_if[256];
+  int packet_priority;
+} network_config_t;
+
 typedef struct main_param {
   int refresh_rate;
+  int verbose;
   size_t tx_buffer_len;
 } main_param_t;
 
-static void process_options(int argc, char *argv[], thread_param_t *param,
-                            main_param_t *main_param);
+static void process_options(int argc, char *argv[]);
+static void print_histograms();
+static void sigint_handler(int sig_num);
+
+// Static variables
+
+static int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL];
+
+static main_param_t main_param;
+static thread_param_t *param;
+static network_config_t network_config;
+
+static int enable_histograms;
+static int enable_affinity;
+static int enable_timestamps;
+
+enum TSNTask { RECV_PACKET_TASK,
+               RTT_TASK };
+static enum TSNTask tsn_task;
+
+struct timespec measures_start;
+struct timespec measures_end;
+
+static void help(char *argv[]) {
+  printf("Usage: %s [-a] [-b CLIENT_IP] [-d BUF_LEN] [-f IF] [-p PRIO] [-r USEC]\n\n", argv[0]);
+  printf("       -a                    Run the real time thread on CPU1\n");
+  printf("       -b CLIENT_IP          Server side RTT\n");
+  printf("       -d BUF_LEN            Set the length of tx buffer\n");
+  printf("       -f IF                 Set the network interface to be used\n");
+  printf("       -p PRIO               Run the real time thread at priority PRIO\n");
+  printf("       -r USEC               Refresh the non real time main thread every USEC microseconds\n");
+  printf("\n");
+}
 
 // Real-time thread
 // Measures intervals between packet receptions
@@ -81,12 +105,12 @@ static void *packet_receiving_thread(void *p) {
   stats->min_interval = UINT64_MAX;
   stats->max_interval = 0;
 
-  if (param->enable_affinity) {
+  if (enable_affinity) {
     // Set thread CPU affinity
     CPU_ZERO(&mask);
     CPU_SET(1, &mask);
     if (sched_setaffinity(0, sizeof(mask), &mask))
-      fprintf(stderr, "Could not set CPU affinity to CPU #1\n");
+      error(EXIT_FAILURE, errno, "Could not set CPU affinity to CPU #1\n");
   }
 
   // Set thread priority
@@ -94,20 +118,25 @@ static void *packet_receiving_thread(void *p) {
   if (sched_setscheduler(0, SCHED_FIFO, &priority))
     error(EXIT_FAILURE, errno, "Couldn't set priority");
 
+  clock_gettime(CLOCK_MONOTONIC, &measures_start);
   // Packet receiving loop
   for (stats->packets_received = 0;; stats->packets_received++) {
-    if (param->tsn_task == RTT_TASK) {
-      recv_udp_packet(param->sockfd);
-      send_udp_packet(0, 0, 0, param->ip_address, NULL);
-    } else if (param->tsn_task == RECV_PACKET_TASK) {
-      recv_udp_packet(param->sockfd);
+
+    if (tsn_task == RTT_TASK) {
+
+      recv_udp_packet(0, NULL);
+      send_udp_packet(0, 0, 0, network_config.ip_address, NULL);
+
+    } else if (tsn_task == RECV_PACKET_TASK) {
+
+      recv_udp_packet(enable_timestamps, histograms);
 
       clock_gettime(CLOCK_MONOTONIC, &current);
 
       if (stats->packets_received) {
         diff = calcdiff_ns(current, previous);
-        if (diff < stats->min_interval) stats->min_interval = diff;
-        if (diff >= stats->max_interval) stats->max_interval = diff;
+        stats->min_interval = diff < stats->min_interval ? diff : stats->min_interval;
+        stats->max_interval = diff > stats->max_interval ? diff : stats->max_interval;
       }
 
       previous = current;
@@ -121,108 +150,201 @@ static void *packet_receiving_thread(void *p) {
 // Handles the IO and creates real time threads
 int main(int argc, char *argv[]) {
   pthread_t thread;
-  thread_param_t param;
-  main_param_t main_param;
-
+  thread_stat_t *stats;
   int64_t diff;
 
-  // Default values
-  param.priority = 99;
-  param.tsn_task = RECV_PACKET_TASK;
+  // Catch breaks with sigint_handler
+  signal(SIGINT, sigint_handler);
+
+  param = malloc(sizeof(thread_param_t));
+  stats = &param->stats;
+
+  // Default configuration values
+  param->priority = 99;
+
+  enable_affinity = 0;
+  enable_timestamps = 0;
+  enable_histograms = 0;
+  tsn_task = RECV_PACKET_TASK;
+
+  network_config.tx_buffer_len = 1024;
 
   main_param.refresh_rate = 50000;
-  main_param.tx_buffer_len = 1024;
+  main_param.verbose = 0;
 
   // Process bash options
-  process_options(argc, argv, &param, &main_param);
+  process_options(argc, argv);
 
-  param.sockfd = init_udp_recv();
+  if (enable_histograms) {
+    // Init histograms
+    memset((int64_t *)histograms, 0, NB_HISTOGRAMS * MAX_HIST_VAL);
+  }
 
-  if (param.tsn_task == RTT_TASK)
-    init_udp_send(0, 0, 1, param.network_if, main_param.tx_buffer_len);
+  // Initialize the UDP packet receiving socket
+  init_udp_recv(enable_timestamps, network_config.network_if);
 
-  usleep(10000);
+  // Initialize the UDP packet sending socket if RTT is measured
+  if (tsn_task == RTT_TASK)
+    init_udp_send(0, 0, 1, network_config.network_if, network_config.tx_buffer_len);
 
-  if (pthread_create(&thread, NULL, packet_receiving_thread, (void *)&param))
+  // Create the real time thread
+  if (pthread_create(&thread, NULL, packet_receiving_thread, (void *)param))
     error(EXIT_FAILURE, errno, "Couldn't create thread");
 
+  // Verbose loop
   for (;;) {
     usleep(main_param.refresh_rate);
 
-    if (param.tsn_task == RECV_PACKET_TASK) {
-      diff = ((int64_t)param.stats.max_interval) - param.stats.min_interval;
-      printf("%" PRIi64 "  -  %" PRIu64 ", %" PRIu64 "  (%d)\n", diff,
-             param.stats.min_interval, param.stats.max_interval,
-             param.stats.packets_received);
+    if(main_param.verbose) {
+
+      if(tsn_task == RECV_PACKET_TASK) {
+        diff = ((int64_t)stats->max_interval) - stats->min_interval;
+        printf(  "(%d)  Jitter        :                  %" PRIi64 "\n",
+                 stats->packets_received,
+                 diff);
+
+        if(enable_timestamps) {
+          printf("(%d)  Enter send_udp_packet timestamp: %" PRIu64 "\n",
+                 stats->packets_received,
+                 stats->packet_ts.enter_user_space);
+          printf("(%d)  Call sendmsg timestamp         : %" PRIu64 "\n",
+                 stats->packets_received,
+                 stats->packet_ts.enter_kernel);
+          printf("(%d)  Leave kernel timestamp         : %" PRIu64 "\n",
+                 stats->packets_received,
+                 stats->packet_ts.leave_kernel);
+        }
+      }
+    }
   }
+}
+
+// Print histograms in .json format
+static void print_histograms() {
+
+  uint64_t duration;
+  int duration_hour, duration_minutes, interval;
+
+  clock_gettime(CLOCK_MONOTONIC, &measures_end);
+
+  duration = calcdiff_ns(measures_end, measures_start);
+  duration_hour = duration / NSEC_PER_SEC / 3600;
+  duration_minutes = duration / NSEC_PER_SEC / 60 - duration_hour * 60;
+
+  interval = (param->stats.min_interval + param->stats.max_interval) / 2;
+
+  printf("{\"measure_sets\": [{\
+             \"measure_type\": \"packet_recv_timestamps\",\
+             \"props_names\": [\"user_space\", \"kernel_space\"],\
+             \"units\": [\"us\", \"us\"],\
+             \"props_type\": \"histogram\",\
+             \"metadata\": {\
+                \"i\": \"%dus\", \"duration\": \"%dh%d\"\
+              },\
+             \"props\": [", interval, duration_hour, duration_minutes);
+
+  int max_hist_val = 0;
+  for (int i = 0; i < 2; i++)
+    for (int j = 0; j < MAX_HIST_VAL; j++)
+      if (histograms[i][j])
+        max_hist_val = j > max_hist_val ? j : max_hist_val;
+
+  for (int i = 0; i < 2; i++) {
+    printf("[");
+    for (int j = 0; j < max_hist_val; j++)
+      printf("%" PRIi64 "%s", histograms[i][j], (j + 1 < max_hist_val ? ", " : ""));
+    printf("%s", (i + 1 < 2 ? "], " : "]"));
   }
 
-  exit(EXIT_SUCCESS);
+  printf(   "]}, {\
+             \"measure_type\": \"packet_jitter\",\
+             \"props_names\": [\"jitter\"],\
+             \"units\": [\"us\"],\
+             \"props_type\": \"histogram\",\
+             \"metadata\": {\
+                \"i\": \"%dus\", \"duration\": \"%dh%d\"\
+              },\
+             \"props\": [[", interval, duration_hour, duration_minutes);
+
+  max_hist_val = 0;
+  for (int j = 0; j < MAX_HIST_VAL && histograms[2][j]; j++)
+    max_hist_val = j;
+
+  for (int j = 0; j < max_hist_val; j++)
+    printf("%" PRIi64 "%s", histograms[2][j], (j + 1 < max_hist_val ? ", " : ""));
+
+  printf(   "]]}]}\n");
 }
 
-static void help(char *argv[]) {
-  printf(
-      "Usage: %s [-a] [-b CLIENT_IP] [-d BUF_LEN] [-f IF] [-p PRIO] [-r "
-      "USEC]\n\n",
-      argv[0]);
-  printf("       -a                    Run the real time thread on CPU1\n");
-  printf("       -b CLIENT_IP          Server side RTT\n");
-  printf("       -d BUF_LEN            Set the length of tx buffer\n");
-  printf("       -f IF                 Set the network interface to be used\n");
-  printf(
-      "       -p PRIO               Run the real time thread at priority "
-      "PRIO\n");
-  printf(
-      "       -r USEC               Refresh the non real time main thread "
-      "every USEC microseconds\n");
-  printf("\n");
+
+static void sigint_handler(int sig_num) {
+  (void)sig_num;
+  if (enable_histograms)
+    print_histograms();
+  exit(EXIT_SUCCESS);
 }
 
-static void process_options(int argc, char *argv[], thread_param_t *param,
-                            main_param_t *main_param) {
+// Process bash options
+static void process_options(int argc, char *argv[]) {
+
+  int network_if_specified = 0;
+
   for (;;) {
-    int c = getopt(argc, argv, "ab:d:f:hp:r:");
+    int c = getopt(argc, argv, "ab:d:f:ghp:r:tv");
 
-    if (c == -1) break;
+    if (c == -1)
+      break;
 
     switch (c) {
     case 'a':
-        param->enable_affinity = 1;
+      enable_affinity = 1;
       break;
     case 'b':
-        param->tsn_task = RTT_TASK;
-        strcpy(param->ip_address, optarg);
+      tsn_task = RTT_TASK;
+      strcpy(network_config.ip_address, optarg);
       break;
     case 'd':
-        main_param->tx_buffer_len = atoi(optarg);
-        if (main_param->tx_buffer_len < 1) {
+      network_config.tx_buffer_len = atoi(optarg);
+      if (network_config.tx_buffer_len < 1) {
         fprintf(stderr, "BUF_LEN should be greater than 1\n");
         exit(EXIT_FAILURE);
       }
       break;
     case 'f':
-        strcpy(param->network_if, optarg);
+      network_if_specified = 1;
+      strcpy(network_config.network_if, optarg);
       break;
     case 'h':
       help(argv);
       exit(EXIT_SUCCESS);
       break;
+    case 'g':
+      enable_histograms = 1;
+      break;
     case 'p':
       param->priority = atoi(optarg);
       break;
     case 'r':
-        main_param->refresh_rate = atoi(optarg);
+      main_param.refresh_rate = atoi(optarg);
       break;
-      default:
-        help(argv);
-        exit(EXIT_FAILURE);
+    case 't':
+      enable_timestamps = 1;
+      break;
+    case 'v':
+      main_param.verbose = 1;
       break;
     }
   }
 
+  if (!network_if_specified) {
+    fprintf(stderr, "You need to specifiy an network interface\n");
+    help(argv);
+    exit(EXIT_FAILURE);
+  }
+
   if (argc != optind) {
+    fprintf(stderr, "Too many arguments\n");
     help(argv);
     exit(EXIT_FAILURE);
   }
 }
-

packet-exchange/src/utilities.c

@@ -28,4 +28,3 @@ uint64_t calcdiff_ns(struct timespec t1, struct timespec t2) {
 
 uint64_t max(uint64_t a, uint64_t b) { return a > b ? a : b; }
 uint64_t min(uint64_t a, uint64_t b) { return a < b ? a : b; }
-

packet-exchange/src/utilities.h

@@ -13,6 +13,12 @@
 #define MAX_HIST_VAL 1000
 #define NB_HISTOGRAMS 2
 
+typedef struct packet_timestamps {
+  uint64_t enter_user_space;
+  uint64_t enter_kernel;
+  uint64_t leave_kernel;
+} packet_timestamps_t;
+
 uint64_t ts_to_uint(struct timespec t);
 void add_ns(struct timespec *t, uint64_t ns);
 uint64_t calcdiff_ns(struct timespec t1, struct timespec t2);