Merge branch 'packet-exchange' into measure-analysis

packet-exchange/build/Makefile

@@ -6,8 +6,14 @@ CLIENT_PROG = client
 SRCDIR = ../src
 
 SERVER_SRCS  = server.c
+SERVER_SRCS  += recv_packet.c
+SERVER_SRCS  += send_packet.c
+SERVER_SRCS  += utilities.c
+
 CLIENT_SRCS  = client.c
+CLIENT_SRCS += recv_packet.c
 CLIENT_SRCS += send_packet.c
+CLIENT_SRCS += utilities.c
 
 SERVER_OBJS = $(SERVER_SRCS:%.c=%.o)
 CLIENT_OBJS = $(CLIENT_SRCS:%.c=%.o)

packet-exchange/src/client.c

 /*
  * Real time packet sending thread
  *
- * Bash options:
- *
- *  -a          Run the real time thread on CPU1
- *  -e          Set a txtime (to be used in an ETF qdisc)
- *  -f          Set the network interface to be used
- *  -i USEC     Wake up the real time thread every USEC microseconds (Default: 10ms)
- *  -l N        Wake up the real time thread N times (Default: 0)
- *  -p PRIO     Run the real time thread at priority PRIO
- *  -r USEC     Refresh the non real time main thread every USEC microseconds (Default: 50ms)
- *  -t          Enable timestamps
- *
  * Large portions taken from cyclictest
  *
  */
@@ -21,6 +10,7 @@
 #include <error.h>
 #include <pthread.h>
 #include <sched.h>
+#include <signal.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -28,37 +18,79 @@
 #include <time.h>
 #include <unistd.h>
 
+#include "recv_packet.h"
 #include "send_packet.h"
+#include "utilities.h"
 
-#define CLOCK_ID CLOCK_MONOTONIC
-#define NSEC_PER_SEC UINT64_C(1000000000)
+// Structs
 
 typedef struct thread_stat {
   int nb_cycles;
+  uint64_t rtt;
+  struct packet_timestamps packet_ts;
 } thread_stat_t;
 
 typedef struct thread_param {
   int interval;
-  int priority;
   int max_cycles;
+  int priority;
 
-  int enable_affinity;
-  int enable_etf;
-  int enable_timestamps;
-
-  const char *ip_address;
-  char network_if[256];
   thread_stat_t stats;
 } thread_param_t;
 
 typedef struct main_param {
   int refresh_rate;
-  int packet_priority;
+  int verbose;
 } main_param_t;
 
-static inline void add_ns(struct timespec *t, uint64_t ns);
-static void process_options(int argc, char *argv[], thread_param_t *param,
-                            main_param_t *main_param);
+typedef struct network_config {
+  size_t tx_buffer_len;
+  char ip_address[256];
+  char network_if[256];
+  int packet_priority;
+} network_config_t;
+
+// Static functions
+
+static void process_options(int argc, char *argv[]);
+static void do_tsn_task(struct thread_param *param, uint64_t next_txtime);
+static void print_histograms();
+static void sigint_handler(int sig_num);
+
+// Static variables
+
+static int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL];
+
+static main_param_t main_param;
+static thread_param_t *param;
+static network_config_t network_config;
+
+static int enable_histograms;
+static int enable_affinity;
+static int enable_etf;
+static int enable_timestamps;
+
+enum TSNTask { SEND_PACKET_TASK,
+               RTT_TASK };
+static enum TSNTask tsn_task;
+
+static void help(char *argv[]) {
+  printf("Usage: %s -f IF  [-abethgv] [-d BUF_LEN] [-i USEC] [-l N] [-p PRIO] [-r USEC]\n\n", argv[0]);
+  printf("       -a                    Run the real time thread on CPU1\n");
+  printf("       -b                    Measure RTT\n");
+  printf("       -d BUF_LEN            Set the length of tx buffer\n");
+  printf("       -e                    Set a txtime (to be used in an ETF qdisc)\n");
+  printf("       -f IF                 Set the network interface to be used\n");
+  printf("       -g                    Print histograms to sdtout on exit\n");
+  printf("       -h                    Show help\n");
+  printf("       -i USEC               Wake up the real time thread every USEC microseconds (Default: 10ms)\n");
+  printf("       -l N                  Wake up the real time thread N times (Default: 0)\n");
+  printf("       -p PRIO               Run the real time thread at priority PRIO\n");
+  printf("       -r USEC               Refresh the non real time main thread every USEC microseconds (Default: 50ms)\n");
+  printf("       -t                    Enable timestamps\n");
+  printf("       -v                    Verbose\n");
+  printf("\n");
+}
 
 // Real-time thread
 // Sends packets at a regular intervall
@@ -67,14 +99,15 @@ static void *packet_sending_thread(void *p) {
   uint64_t next_txtime;
   struct sched_param priority;
   thread_param_t *param = (thread_param_t *)p;
+  thread_stat_t * stats = &param->stats;
   cpu_set_t mask;
 
-  if (param->enable_affinity) {
   // Set thread CPU affinity
+  if (enable_affinity) {
     CPU_ZERO(&mask);
     CPU_SET(1, &mask);
     if (sched_setaffinity(0, sizeof(mask), &mask))
-      fprintf(stderr, "Could not set CPU affinity to CPU #1\n");
+      error(EXIT_FAILURE, errno, "Could not set CPU affinity to CPU #1\n");
   }
 
   // Set thread priority
@@ -82,7 +115,7 @@ static void *packet_sending_thread(void *p) {
   if (sched_setscheduler(0, SCHED_FIFO, &priority))
     error(EXIT_FAILURE, errno, "Couldn't set priority");
 
-  clock_gettime(CLOCK_ID, &next);
+  clock_gettime(CLOCK_MONOTONIC, &next);
   next_txtime = next.tv_sec * NSEC_PER_SEC + next.tv_nsec;
   // Wait around 1 second
   next_txtime += (10 * NSEC_PER_SEC / param->interval) * param->interval;
@@ -90,16 +123,17 @@ static void *packet_sending_thread(void *p) {
   next_txtime += (param->interval) / 2;
 
   // Packet sending loop
-  for (param->stats.nb_cycles = 0;; param->stats.nb_cycles++) {
+  for (stats->nb_cycles = 0;; stats->nb_cycles++) {
     if (param->max_cycles)
-      if (param->stats.nb_cycles >= param->max_cycles) break;
+      if (stats->nb_cycles >= param->max_cycles)
+        break;
 
-    send_udp_packet(param->enable_etf, param->enable_timestamps, next_txtime, param->ip_address);
+    do_tsn_task(param, next_txtime);
 
     add_ns(&next, param->interval);
     next_txtime += (param->interval) / 2;
 
-    clock_nanosleep(CLOCK_ID, TIMER_ABSTIME, &next, NULL);
+    clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &next, NULL);
   }
 
   return NULL;
@@ -109,60 +143,197 @@ static void *packet_sending_thread(void *p) {
 // Handles the IO and creates real time threads
 int main(int argc, char *argv[]) {
   pthread_t thread;
-  thread_param_t param;
-  main_param_t main_param;
+  thread_stat_t *stats;
+
+  // Catch breaks with sigint_handler
+  signal(SIGINT, sigint_handler);
+
+  param = malloc(sizeof(thread_param_t));
+  stats = &param->stats;
 
-  // Default values
-  param.interval = 100000 * 1000;
-  param.max_cycles = 0;
-  param.priority = 99;
-  param.enable_affinity = 0;
-  param.enable_etf = 0;
-  param.enable_timestamps = 0;
+  // Default configuration values
+  param->interval = 100000 * 1000;
+  param->max_cycles = 0;
+  param->priority = 99;
+
+  enable_affinity = 0;
+  enable_etf = 0;
+  enable_timestamps = 0;
+  enable_histograms = 0;
+  tsn_task = SEND_PACKET_TASK;
+
+  network_config.packet_priority = 3;
+  network_config.tx_buffer_len = 1024;
 
   main_param.refresh_rate = 50000;
-  main_param.packet_priority = 3;
+  main_param.verbose = 0;
 
   // Process bash options
-  process_options(argc, argv, &param, &main_param);
+  process_options(argc, argv);
+
+  if (enable_histograms) {
+    // Init histograms
+    memset((int64_t *)histograms, 0, NB_HISTOGRAMS * MAX_HIST_VAL);
+  }
 
-  init_udp_etf(param.enable_etf, param.enable_timestamps, main_param.packet_priority, param.network_if);
+  // Initialize the UDP packet sending socket
+  init_udp_send(enable_etf, enable_timestamps,
+                network_config.packet_priority,
+                network_config.network_if,
+                network_config.tx_buffer_len);
 
-  usleep(10000);
+  // Initialize the UDP packet receiving socket if RTT is measured
+  if (tsn_task == RTT_TASK)
+    init_udp_recv();
 
-  if (pthread_create(&thread, NULL, packet_sending_thread, (void *)&param))
+  // Create the real time thread
+  if (pthread_create(&thread, NULL, packet_sending_thread, (void *)param))
     error(EXIT_FAILURE, errno, "Couldn't create thread");
 
+  // Verbose loop
   for (;;) {
     usleep(main_param.refresh_rate);
 
-#ifdef DEBUG_ENABLE
-    printf("Nb cycles: %d\n", param.stats.nb_cycles);
-#endif
+    if (main_param.verbose) {
+      if (tsn_task == RTT_TASK) {
 
-    if (param.max_cycles)
-      if (param.max_cycles == param.stats.nb_cycles) break;
+        printf("RTT: %" PRIu64 " (%d)\n", stats->rtt, stats->nb_cycles);
+
+      } else if (enable_timestamps) {
+
+        printf("(%d)\n", stats->nb_cycles);
+        printf("      Enter send_udp_packet timestamp: %" PRIu64 "\n",
+               stats->packet_ts.user_enter_send);
+        printf("      Call sendmsg timestamp         : %" PRIu64 "\n",
+               stats->packet_ts.user_call_sendmsg);
+        printf("      Leave kernel timestamp         : %" PRIu64 "\n",
+               stats->packet_ts.kernel_leave);
+      }
+    }
+
+    if (param->max_cycles)
+      if (param->max_cycles == stats->nb_cycles)
+        break;
   }
 
+  if (enable_histograms)
+    print_histograms();
+
   exit(EXIT_SUCCESS);
 }
 
-static void process_options(int argc, char *argv[], thread_param_t *param,
-                            main_param_t *main_param) {
+// Critical TSN task
+static void do_tsn_task(struct thread_param *param, uint64_t next_txtime) {
+  struct timespec t1, t2;
+
+  // One way packet sending
+  if (tsn_task == SEND_PACKET_TASK) {
+
+    param->stats.packet_ts = send_udp_packet(
+        enable_etf, enable_timestamps, next_txtime,
+        network_config.ip_address, histograms);
+
+  // Round Trip Time measurement
+  } else if (tsn_task == RTT_TASK) {
+
+    clock_gettime(CLOCK_MONOTONIC, &t1);
+    send_udp_packet(enable_etf, enable_timestamps, next_txtime,
+                    network_config.ip_address, histograms);
+    recv_udp_packet();
+    clock_gettime(CLOCK_MONOTONIC, &t2);
+
+    param->stats.rtt = calcdiff_ns(t2, t1);
+
+  }
+}
+
+// Print histograms in .json format
+static void print_histograms() {
+
+  printf("{\"measure_type\": \"packet_timestamps\",\
+           \"props_names\": [\"user_space\", \"kernel_space\"],\
+           \"units\": [\"us\", \"us\"],\
+           \"props\": [");
+
+  int max_hist_val = 0;
+  for (int i = 0; i < NB_HISTOGRAMS; i++) {
+    for (int j = 0; j < MAX_HIST_VAL; j++)
+      if (histograms[i][j])
+        max_hist_val = j > max_hist_val ? j : max_hist_val;
+  }
+
+  for (int i = 0; i < NB_HISTOGRAMS; i++) {
+
+    printf("[");
+    for (int j = 0; j < max_hist_val; j++) {
+      if (j + 1 < max_hist_val)
+        printf("%" PRIi64 ", ", histograms[i][j]);
+      else
+        printf("%" PRIi64, histograms[i][j]);
+    }
+    if (i + 1 < NB_HISTOGRAMS)
+      printf("], ");
+    else
+      printf("]");
+  }
+
+  int interval = param->interval / 1000;
+  uint64_t duration = interval * param->stats.nb_cycles;
+  int duration_hour = duration / 1000000;
+  duration_hour /= 3600;
+  int duration_minutes = duration / 1000000;
+  duration_minutes /= 60;
+  duration_minutes -= duration_hour * 60;
+
+  printf("], ");
+  printf("\"props_type\": \"histogram\", \"metadata\": {\"i\": \"%dus\", \"duration\": \"%dh%d\"}}\n", interval, duration_hour, duration_minutes);
+}
+
+static void sigint_handler(int sig_num) {
+  (void)sig_num;
+  if (enable_histograms)
+    print_histograms();
+  exit(EXIT_SUCCESS);
+}
+
+// Process bash options
+static void process_options(int argc, char *argv[]) {
+
+  int network_if_specified = 0;
+
   for (;;) {
-    int c = getopt(argc, argv, "aef:i:l:p:q:r:t");
+    int c = getopt(argc, argv, "abd:ef:ghi:l:p:q:r:tv");
 
-    if (c == -1) break;
+    if (c == -1)
+      break;
 
     switch (c) {
     case 'a':
-        param->enable_affinity = 1;
+      enable_affinity = 1;
+      break;
+    case 'b':
+      tsn_task = RTT_TASK;
+      break;
+    case 'd':
+      network_config.tx_buffer_len = atoi(optarg);
+      if (network_config.tx_buffer_len < 1) {
+        fprintf(stderr, "BUF_LEN should be greater than 1\n");
+        exit(EXIT_FAILURE);
+      }
       break;
     case 'e':
-        param->enable_etf = 1;
+      enable_etf = 1;
       break;
     case 'f':
-        strcpy(param->network_if, optarg);
+      network_if_specified = 1;
+      strcpy(network_config.network_if, optarg);
+      break;
+    case 'g':
+      enable_histograms = 1;
+      break;
+    case 'h':
+      help(argv);
+      exit(EXIT_SUCCESS);
       break;
     case 'i':
       param->interval = atoi(optarg) * 1000;
@@ -174,32 +345,34 @@ static void process_options(int argc, char *argv[], thread_param_t *param,
       param->priority = atoi(optarg);
       break;
     case 'q':
-        main_param->packet_priority = atoi(optarg);
+      network_config.packet_priority = atoi(optarg);
       break;
     case 'r':
-        main_param->refresh_rate = atoi(optarg);
+      main_param.refresh_rate = atoi(optarg);
       break;
     case 't':
-        param->enable_timestamps = 1;
+      enable_timestamps = 1;
       break;
-      default:
-        exit(EXIT_FAILURE);
+    case 'v':
+      main_param.verbose = 1;
       break;
     }
   }
 
-  if (argc != optind + 1) {
-    printf("Usage: %s server_ip\n", argv[0]);
+  if(!network_if_specified) {
+      fprintf(stderr, "You need to specifiy an network interface\n");
+      help(argv);
       exit(EXIT_FAILURE);
   }
-  param->ip_address = argv[optind];
-}
 
-static inline void add_ns(struct timespec *t, uint64_t ns) {
-  t->tv_nsec += ns;
-
-  if ((unsigned int)t->tv_nsec >= NSEC_PER_SEC) {
-    t->tv_sec += 1;
-    t->tv_nsec -= NSEC_PER_SEC;
+  if (argc != optind + 1) {
+    if( argc < optind + 1)
+      fprintf(stderr, "You need to specifiy an IP address\n");
+    else
+      fprintf(stderr, "Too many arguments\n");
+    help(argv);
+    exit(EXIT_FAILURE);
   }
+  strcpy(network_config.ip_address, argv[optind]);
 }
+

packet-exchange/src/recv_packet.c

+#define _GNU_SOURCE
+#include <arpa/inet.h>
+#include <errno.h>
+#include <error.h>
+#include <inttypes.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <pthread.h>
+#include <sched.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
+
+#include "utilities.h"
+
+#define SERVER_PORT "50000"
+#define BUFFER_SIZE 1024
+
+static char rx_buffer[BUFFER_SIZE];
+static int sock_fd;
+
+int init_udp_recv() {
+  int status;
+  int sock_fd = -1;
+  struct addrinfo hints, *servinfo, *servinfo_it;
+
+  memset(&hints, 0, sizeof hints);
+  hints.ai_family = AF_UNSPEC;
+  hints.ai_socktype = SOCK_DGRAM;
+  hints.ai_flags = AI_PASSIVE;
+
+  status = getaddrinfo(NULL, SERVER_PORT, &hints, &servinfo);
+  if (status != 0) {
+    fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(status));
+    exit(EXIT_FAILURE);
+  }
+
+  for (servinfo_it = servinfo; servinfo_it;
+       servinfo_it = servinfo_it->ai_next) {
+    sock_fd = socket(servinfo->ai_family, servinfo->ai_socktype,
+                servinfo->ai_protocol);
+
+    if (bind(sock_fd, servinfo_it->ai_addr, servinfo_it->ai_addrlen) == -1) {
+      close(sock_fd);
+      continue;
+    }
+    break;
+  }
+
+  freeaddrinfo(servinfo);
+
+  if(sock_fd == -1)
+    error(EXIT_FAILURE, errno, "Couldn't create receive socket");
+
+  printf("waiting to receive...\n");
+
+  return sock_fd;
+}
+
+void recv_udp_packet() {
+#ifdef DEBUG
+  int bytes_received = 0;
+
+  bytes_received = recvfrom(sock_fd, rx_buffer, BUFFER_SIZE - 1, 0, NULL, NULL);
+
+  if (bytes_received == -1)
+    error(EXIT_FAILURE, errno, "Error while attempting to receive packets");
+#else
+  recvfrom(sock_fd, rx_buffer, BUFFER_SIZE - 1, 0, NULL, NULL);
+#endif
+}

packet-exchange/src/recv_packet.h

+#ifndef RECV_PACKET
+#define RECV_PACKET
+
+int init_udp_recv();
+void recv_udp_packet();
+
+#endif

packet-exchange/src/send_packet.c

@@ -7,8 +7,8 @@
  *
  */
 
-#include "send_packet.h"
 
+#define _GNU_SOURCE
 #include <arpa/inet.h>
 #include <errno.h>
 #include <error.h>
@@ -36,20 +36,19 @@
 #include <sys/types.h>
 #include <unistd.h>
 
-#define SERVER_PORT "50000"
-#define SERVER_PORT_INT 50000
-#define CLOCK_ID CLOCK_TAI
+#include "utilities.h"
+#include "send_packet.h"
+
 #define MESSAGE ((uint32_t)0x00FACADE)
-#define NSEC_PER_SEC ((uint64_t)1000000000)
 
-static void print_timestamps(struct msghdr *msg, uint64_t txtime);
-static void process_timestamps(uint64_t txtime);
+static void process_timestamps(struct packet_timestamps *packet_ts, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]);
+static void init_tx_buffer(size_t _tx_buffer_len);
 
 static int so_priority = 3;
 static struct sock_txtime sk_txtime;
-static unsigned char tx_buffer[1024] = "Hi";
-static size_t tx_buffer_len = sizeof(tx_buffer);
-static int fd;
+static unsigned char *tx_buffer;
+static size_t tx_buffer_len;
+static int sock_fd;
 
 static int64_t tai_offset;
 
@@ -63,51 +62,71 @@ static int set_if(char *network_if) {
   memset(&ifreq, 0, sizeof(ifreq));
   strncpy(ifreq.ifr_name, network_if, sizeof(ifreq.ifr_name) - 1);
 
-  if (ioctl(fd, SIOCGIFINDEX, &ifreq))
+  if (ioctl(sock_fd, SIOCGIFINDEX, &ifreq))
     error(EXIT_FAILURE, errno, "ioctl SIOCGIFINDEX failed\n");
 
   return ifreq.ifr_ifindex;
 }
 
+static void init_tx_buffer(size_t _tx_buffer_len) {
+  if (_tx_buffer_len < 1) {
+    fprintf(stderr, "tx buffer length should be greater than 1\n");
+    exit(EXIT_FAILURE);
+  }
+
+  tx_buffer_len = _tx_buffer_len;
+  tx_buffer = malloc(tx_buffer_len);
+
+  for (int i = 0; i < (((int)tx_buffer_len) - 1); i++) {
+    tx_buffer[i] = (unsigned char)i;
+  }
+
+  tx_buffer[tx_buffer_len - 1] = '\0';
+}
+
 /*
  * Init UDP socket
  */
-void init_udp_etf(int use_etf, int use_timestamps, int packet_priority,
-                  char *network_if) {
+void init_udp_send(int use_etf, int use_timestamps, int packet_priority,
+                   char *network_if, size_t _tx_buffer_len) {
   int index;
 
   struct timespec ts_mon;
   struct timespec ts_tai;
+
+  init_tx_buffer(_tx_buffer_len);
+
   clock_gettime(CLOCK_MONOTONIC, &ts_mon);
   clock_gettime(CLOCK_TAI, &ts_tai);
-  tai_offset = (ts_mon.tv_sec - ts_tai.tv_sec) * NSEC_PER_SEC + (ts_mon.tv_nsec - ts_tai.tv_nsec);
+  tai_offset = (ts_mon.tv_sec - ts_tai.tv_sec) * NSEC_PER_SEC +
+               (ts_mon.tv_nsec - ts_tai.tv_nsec);
 
   so_priority = packet_priority;
 
-  fd = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP);
-  if (fd < 0) error(EXIT_FAILURE, errno, "Socket creation failed\n");
+  sock_fd = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP);
+  if (sock_fd < 0) error(EXIT_FAILURE, errno, "Socket creation failed\n");
 
   index = set_if(network_if);
   if (index < 0) error(EXIT_FAILURE, errno, "Couldn't set interface\n");
 
-  if (setsockopt(fd, SOL_SOCKET, SO_PRIORITY, &so_priority,
+  if (setsockopt(sock_fd, SOL_SOCKET, SO_PRIORITY, &so_priority,
                  sizeof(so_priority)))
     error(EXIT_FAILURE, errno, "Couldn't set socket priority\n");
 
-  if (setsockopt(fd, SOL_SOCKET, SO_BINDTODEVICE, network_if,
+  if (setsockopt(sock_fd, SOL_SOCKET, SO_BINDTODEVICE, network_if,
                  strlen(network_if)))
     error(EXIT_FAILURE, errno, "setsockopt SO_BINDTODEVICE failed\n");
 
   if (use_etf) {
-    sk_txtime.clockid = CLOCK_ID;
+    sk_txtime.clockid = CLOCK_TAI;
     sk_txtime.flags = 0;
 
-    if (setsockopt(fd, SOL_SOCKET, SO_TXTIME, &sk_txtime, sizeof(sk_txtime)))
+    if (setsockopt(sock_fd, SOL_SOCKET, SO_TXTIME, &sk_txtime, sizeof(sk_txtime)))
       error(EXIT_FAILURE, errno, "setsockopt SO_TXTIME failed\n");
   }
 
   if (use_timestamps) {
-    if (setsockopt(fd, SOL_SOCKET, SO_TIMESTAMPING, &so_timestamping_flags,
+    if (setsockopt(sock_fd, SOL_SOCKET, SO_TIMESTAMPING, &so_timestamping_flags,
                    sizeof(so_timestamping_flags)))
       error(EXIT_FAILURE, errno, "setsockopt SO_TIMESTAMPING failed\n");
   }
@@ -125,10 +144,12 @@ uint64_t get_txtime() {
 }
 
 /*
- * Sends udp packets using the ETF qdisc
+ * Sends udp packets
  */
-void send_udp_packet(int use_etf, int use_timestamps, uint64_t txtime,
-                     const char *server_ip) {
+struct packet_timestamps send_udp_packet(int use_etf, int use_timestamps,
+                                         uint64_t txtime,
+                                         const char *server_ip,
+                                         int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]) {
   char control[CMSG_SPACE(sizeof(txtime))] = {};
   struct sockaddr_in sin;
   struct cmsghdr *cmsg;
@@ -136,7 +157,15 @@ void send_udp_packet(int use_etf, int use_timestamps, uint64_t txtime,
   struct iovec iov;
   int sendmsgerr;
   int res;
-  struct pollfd poll_fd = {fd, POLLPRI, 0};
+  struct pollfd poll_fd = {sock_fd, POLLPRI, 0};
+
+  struct packet_timestamps packet_ts;
+  struct timespec ts;
+
+  if (use_timestamps) {
+    clock_gettime(CLOCK_REALTIME, &ts);
+    packet_ts.user_enter_send = ts_to_uint(ts);
+  }
 
   memset(&sin, 0, sizeof(sin));
   sin.sin_family = AF_INET;
@@ -165,20 +194,47 @@ void send_udp_packet(int use_etf, int use_timestamps, uint64_t txtime,
     msg.msg_controllen = cmsg->cmsg_len;
   }
 
-  sendmsgerr = sendmsg(fd, &msg, 0);
+  if (use_timestamps) {
+    clock_gettime(CLOCK_REALTIME, &ts);
+    packet_ts.user_call_sendmsg = ts_to_uint(ts);
+  }
+
+  sendmsgerr = sendmsg(sock_fd, &msg, 0);
   if (sendmsgerr < 0)
     error(EXIT_FAILURE, errno, "sendmsg failed, ret value: %d\n", sendmsgerr);
 
   if (use_timestamps) {
     res = poll(&poll_fd, 1, 0);
     if (res > 0)
-      process_timestamps(txtime);
+      process_timestamps(&packet_ts, histograms);
     else
       fprintf(stderr, "select failed\n");
   }
+
+  return packet_ts;
+}
+
+static void fill_histograms(struct packet_timestamps *packet_ts, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]) {
+  uint64_t user_space_time = packet_ts->user_call_sendmsg - packet_ts->user_enter_send;
+  uint64_t kernel_space_time = packet_ts->kernel_leave - packet_ts->user_call_sendmsg;
+
+  user_space_time /= 1000u;
+  kernel_space_time /= 1000u;
+
+  if(user_space_time > MAX_HIST_VAL) {
+    fprintf(stderr, "user_space_time value too high: %" PRIu64 "us\n", user_space_time);
+    exit(EXIT_FAILURE);
+  }
+  if(kernel_space_time > MAX_HIST_VAL) {
+    fprintf(stderr, "kernel_space_time value too high: %" PRIu64 "us\n", kernel_space_time);
+    exit(EXIT_FAILURE);
+  }
+
+  histograms[0][user_space_time]++;
+  histograms[1][kernel_space_time]++;
 }
 
-static void process_timestamps(uint64_t txtime) {
+static void process_timestamps(struct packet_timestamps *packet_ts, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]) {
   char data[256];
   struct msghdr msg;
   struct iovec entry;
@@ -187,6 +243,7 @@ static void process_timestamps(uint64_t txtime) {
     struct cmsghdr cm;
     char control[512];
   } control;
+  struct cmsghdr *cmsg;
 
   memset(&msg, 0, sizeof(msg));
   msg.msg_iov = &entry;
@@ -198,54 +255,33 @@ static void process_timestamps(uint64_t txtime) {
   msg.msg_control = &control;
   msg.msg_controllen = sizeof(control);
 
-  if (recvmsg(fd, &msg, MSG_ERRQUEUE | MSG_DONTWAIT) == -1) {
+  if (recvmsg(sock_fd, &msg, MSG_ERRQUEUE | MSG_DONTWAIT) == -1) {
     fprintf(stderr, "recvmsg failed\n");
-  } else {
-    print_timestamps(&msg, txtime);
+    return;
   }
-}
 
-static void print_timestamps(struct msghdr *msg, uint64_t txtime) {
-  struct cmsghdr *cmsg;
-
-  for (cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
-    switch (cmsg->cmsg_level) {
-      case SOL_SOCKET:
-        printf("SOL_SOCKET ");
-        switch (cmsg->cmsg_type) {
-          case SO_TIMESTAMPING: {
+  for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
+    if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SO_TIMESTAMPING) {
       struct timespec *stamp = (struct timespec *)CMSG_DATA(cmsg);
-            uint64_t timestamp_ns =
-                stamp->tv_sec * NSEC_PER_SEC + stamp->tv_nsec;
-            printf("SO_TIMESTAMPING ");
-            printf("SW %" PRIu64 " - %" PRIu64 " (%" PRIi64 ") ", timestamp_ns,
-                   txtime, ((int64_t)timestamp_ns) - txtime - tai_offset);
-            break;
-          }
-          default:
-#ifdef DEBUG
-            printf("type %d", cmsg->cmsg_type);
-#endif
-            break;
-        }
-        break;
-      default:
+      packet_ts->kernel_leave = ts_to_uint(*stamp);
+      fill_histograms(packet_ts, histograms);
+    } else {
 #ifdef DEBUG
-        printf("level %d type %d", cmsg->cmsg_level, cmsg->cmsg_type);
+      fprintf(stderr, "process_timestamps: level %d type %d", cmsg->cmsg_level,
+              cmsg->cmsg_type);
 #endif
-        break;
     }
-    printf("\n");
   }
 }
 
 #ifdef DEBUG
+
 /*
  * Code from scheduled_tx_tools
  */
 static int process_socket_error_queue() {
   uint8_t msg_control[CMSG_SPACE(sizeof(struct sock_extended_err))];
-  unsigned char err_buffer[sizeof(tx_buffer)];
+  unsigned char err_buffer[256];
   struct sock_extended_err *serr;
   struct cmsghdr *cmsg;
   __u64 tstamp = 0;
@@ -256,7 +292,7 @@ static int process_socket_error_queue() {
                        .msg_control = msg_control,
                        .msg_controllen = sizeof(msg_control)};
 
-  if (recvmsg(fd, &msg, MSG_ERRQUEUE) == -1) {
+  if (recvmsg(sock_fd, &msg, MSG_ERRQUEUE) == -1) {
     fprintf(stderr, "recvmsg failed");
     return -1;
   }

packet-exchange/src/send_packet.h

@@ -2,8 +2,17 @@
 #define SEND_PACKET_H
 
 #include <stdint.h>
+#include <stdio.h>
 
-void init_udp_etf(int use_etf, int use_timestamps, int so_priority, char * network_if);
-void send_udp_packet(int use_etf, int use_timestamps, uint64_t txtime, const char *server_ip);
+#include "utilities.h"
+
+struct packet_timestamps {
+  uint64_t user_enter_send;
+  uint64_t user_call_sendmsg;
+  uint64_t kernel_leave;
+};
+
+void init_udp_send(int use_etf, int use_timestamps, int so_priority, char * network_if, size_t tx_buffer_len);
+struct packet_timestamps send_udp_packet(int use_etf, int use_timestamps, uint64_t txtime, const char *server_ip, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]);
 
 #endif

packet-exchange/src/server.c

@@ -4,8 +4,12 @@
  * Bash options:
  *
  *  -a                    Run the real time thread on CPU1
+ *  -b CLIENT_IP          Server side RTT
+ *  -d TX_BUFFER_LEN      Set the length of tx buffer
+ *  -f IF                 Set the network interface to be used
  *  -p PRIO               Run the real time thread at priority PRIO
- *  -r USEC     Refresh the non real time main thread every USEC microseconds
+ *  -r USEC               Refresh the non real time main thread every USEC
+ * microseconds
  *
  * Large portions taken from cyclictest
  *
@@ -29,11 +33,14 @@
 #include <time.h>
 #include <unistd.h>
 
-#define NSEC_PER_SEC UINT64_C(1000000000)
+#include "recv_packet.h"
+#include "send_packet.h"
+#include "utilities.h"
 
-#define SERVER_PORT "50000"
 #define BUFFER_SIZE 1024
 
+enum TSNTask { RECV_PACKET_TASK, RTT_TASK };
+
 typedef struct thread_stat {
   uint64_t min_interval;
   uint64_t max_interval;
@@ -42,24 +49,24 @@ typedef struct thread_stat {
 
 typedef struct thread_param {
   int priority;
-  thread_stat_t stats;
 
   int enable_affinity;
+  enum TSNTask tsn_task;
 
   int sockfd;
+  char *ip_address;
+  char network_if[256];
+
+  thread_stat_t stats;
 } thread_param_t;
 
 typedef struct main_param {
   int refresh_rate;
+  size_t tx_buffer_len;
 } main_param_t;
 
-static inline uint64_t calcdiff_ns(struct timespec t1, struct timespec t2);
-static inline uint64_t max(uint64_t a, uint64_t b);
-static inline uint64_t min(uint64_t a, uint64_t b);
-
 static void process_options(int argc, char *argv[], thread_param_t *param,
                             main_param_t *main_param);
-void init_server(thread_param_t *param);
 
 // Real-time thread
 // Measures intervals between packet receptions
@@ -71,13 +78,6 @@ static void *packet_receiving_thread(void *p) {
   uint64_t diff = 0;
   cpu_set_t mask;
 
-  char buf[BUFFER_SIZE];
-  int bytes_received = 0;
-  struct sockaddr_storage client_addr;
-  socklen_t addr_len;
-
-  addr_len = sizeof client_addr;
-
   stats->min_interval = UINT64_MAX;
   stats->max_interval = 0;
 
@@ -96,12 +96,13 @@ static void *packet_receiving_thread(void *p) {
 
   // Packet receiving loop
   for (stats->packets_received = 0;; stats->packets_received++) {
-    bytes_received = recvfrom(param->sockfd, buf, BUFFER_SIZE - 1, 0,
-                              (struct sockaddr *)&client_addr, &addr_len);
-    clock_gettime(CLOCK_MONOTONIC, &current);
+    if (param->tsn_task == RTT_TASK) {
+      recv_udp_packet(param->sockfd);
+      send_udp_packet(0, 0, 0, param->ip_address, NULL);
+    } else if (param->tsn_task == RECV_PACKET_TASK) {
+      recv_udp_packet(param->sockfd);
 
-    if (bytes_received == -1)
-      error(EXIT_FAILURE, errno, "Error while attempting to receive packets");
+      clock_gettime(CLOCK_MONOTONIC, &current);
 
       if (stats->packets_received) {
         diff = calcdiff_ns(current, previous);
@@ -111,6 +112,7 @@ static void *packet_receiving_thread(void *p) {
 
       previous = current;
     }
+  }
 
   return NULL;
 }
@@ -122,14 +124,22 @@ int main(int argc, char *argv[]) {
   thread_param_t param;
   main_param_t main_param;
 
+  int64_t diff;
+
   // Default values
   param.priority = 99;
+  param.tsn_task = RECV_PACKET_TASK;
+
   main_param.refresh_rate = 50000;
+  main_param.tx_buffer_len = 1024;
 
   // Process bash options
   process_options(argc, argv, &param, &main_param);
 
-  init_server(&param);
+  param.sockfd = init_udp_recv();
+
+  if (param.tsn_task == RTT_TASK)
+    init_udp_send(0, 0, 1, param.network_if, main_param.tx_buffer_len);
 
   usleep(10000);
 
@@ -139,19 +149,39 @@ int main(int argc, char *argv[]) {
   for (;;) {
     usleep(main_param.refresh_rate);
 
-    printf("%" PRIu64 "  -  %" PRIu64 ", %" PRIu64 "  (%d)\n",
-           param.stats.max_interval - param.stats.min_interval,
+    if (param.tsn_task == RECV_PACKET_TASK) {
+      diff = ((int64_t)param.stats.max_interval) - param.stats.min_interval;
+      printf("%" PRIi64 "  -  %" PRIu64 ", %" PRIu64 "  (%d)\n", diff,
              param.stats.min_interval, param.stats.max_interval,
              param.stats.packets_received);
     }
+  }
 
   exit(EXIT_SUCCESS);
 }
 
+static void help(char *argv[]) {
+  printf(
+      "Usage: %s [-a] [-b CLIENT_IP] [-d BUF_LEN] [-f IF] [-p PRIO] [-r "
+      "USEC]\n\n",
+      argv[0]);
+  printf("       -a                    Run the real time thread on CPU1\n");
+  printf("       -b CLIENT_IP          Server side RTT\n");
+  printf("       -d BUF_LEN            Set the length of tx buffer\n");
+  printf("       -f IF                 Set the network interface to be used\n");
+  printf(
+      "       -p PRIO               Run the real time thread at priority "
+      "PRIO\n");
+  printf(
+      "       -r USEC               Refresh the non real time main thread "
+      "every USEC microseconds\n");
+  printf("\n");
+}
+
 static void process_options(int argc, char *argv[], thread_param_t *param,
                             main_param_t *main_param) {
   for (;;) {
-    int c = getopt(argc, argv, "ap:r:");
+    int c = getopt(argc, argv, "ab:d:f:hp:r:");
 
     if (c == -1) break;
 
@@ -159,6 +189,24 @@ static void process_options(int argc, char *argv[], thread_param_t *param,
       case 'a':
         param->enable_affinity = 1;
         break;
+      case 'b':
+        param->tsn_task = RTT_TASK;
+        strcpy(param->ip_address, optarg);
+        break;
+      case 'd':
+        main_param->tx_buffer_len = atoi(optarg);
+        if (main_param->tx_buffer_len < 1) {
+          fprintf(stderr, "BUF_LEN should be greater than 1\n");
+          exit(EXIT_FAILURE);
+        }
+        break;
+      case 'f':
+        strcpy(param->network_if, optarg);
+        break;
+      case 'h':
+        help(argv);
+        exit(EXIT_SUCCESS);
+        break;
       case 'p':
         param->priority = atoi(optarg);
         break;
@@ -166,52 +214,15 @@ static void process_options(int argc, char *argv[], thread_param_t *param,
         main_param->refresh_rate = atoi(optarg);
         break;
       default:
+        help(argv);
         exit(EXIT_FAILURE);
         break;
     }
   }
-}
-
-void init_server(thread_param_t *param) {
-  int status;
-  struct addrinfo hints, *servinfo, *servinfo_it;
 
-  memset(&hints, 0, sizeof hints);
-  hints.ai_family = AF_UNSPEC;
-  hints.ai_socktype = SOCK_DGRAM;
-  hints.ai_flags = AI_PASSIVE;
-
-  status = getaddrinfo(NULL, SERVER_PORT, &hints, &servinfo);
-  if (status != 0) {
-    fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(status));
+  if (argc != optind) {
+    help(argv);
     exit(EXIT_FAILURE);
   }
-
-  for (servinfo_it = servinfo; servinfo_it;
-       servinfo_it = servinfo_it->ai_next) {
-    param->sockfd = socket(servinfo->ai_family, servinfo->ai_socktype,
-                           servinfo->ai_protocol);
-
-    if (bind(param->sockfd, servinfo_it->ai_addr, servinfo_it->ai_addrlen) ==
-        -1) {
-      close(param->sockfd);
-      continue;
-    }
-    break;
-  }
-
-  freeaddrinfo(servinfo);
-
-  printf("waiting to receive...\n");
-}
-
-static inline uint64_t calcdiff_ns(struct timespec t1, struct timespec t2) {
-  uint64_t diff;
-  diff = NSEC_PER_SEC * (uint64_t)((int)t1.tv_sec - (int)t2.tv_sec);
-  diff += ((int)t1.tv_nsec - (int)t2.tv_nsec);
-  return diff;
 }
 
-static inline uint64_t max(uint64_t a, uint64_t b) { return a > b ? a : b; }
-static inline uint64_t min(uint64_t a, uint64_t b) { return a < b ? a : b; }
-

packet-exchange/src/utilities.c

+#define _GNU_SOURCE
+#include <inttypes.h>
+#include <stdint.h>
+#include <time.h>
+#include <unistd.h>
+
+#include "utilities.h"
+
+uint64_t ts_to_uint(struct timespec t) {
+  return t.tv_sec * NSEC_PER_SEC + t.tv_nsec;
+}
+
+void add_ns(struct timespec *t, uint64_t ns) {
+  t->tv_nsec += ns;
+
+  while ((unsigned int)t->tv_nsec >= NSEC_PER_SEC) {
+    t->tv_sec += 1;
+    t->tv_nsec -= NSEC_PER_SEC;
+  }
+}
+
+uint64_t calcdiff_ns(struct timespec t1, struct timespec t2) {
+  uint64_t diff;
+  diff = NSEC_PER_SEC * (uint64_t)((int)t1.tv_sec - (int)t2.tv_sec);
+  diff += ((int)t1.tv_nsec - (int)t2.tv_nsec);
+  return diff;
+}
+
+uint64_t max(uint64_t a, uint64_t b) { return a > b ? a : b; }
+uint64_t min(uint64_t a, uint64_t b) { return a < b ? a : b; }
+

packet-exchange/src/utilities.h

+#ifndef UTILITIES_H
+#define UTILITIES_H
+
+#define _GNU_SOURCE
+#include <inttypes.h>
+#include <stdint.h>
+#include <time.h>
+#include <unistd.h>
+
+#define NSEC_PER_SEC UINT64_C(1000000000)
+#define SERVER_PORT "50000"
+#define SERVER_PORT_INT 50000
+#define MAX_HIST_VAL 1000
+#define NB_HISTOGRAMS 2
+
+uint64_t ts_to_uint(struct timespec t);
+void add_ns(struct timespec *t, uint64_t ns);
+uint64_t calcdiff_ns(struct timespec t1, struct timespec t2);
+uint64_t max(uint64_t a, uint64_t b);
+uint64_t min(uint64_t a, uint64_t b);
+
+#endif

scripts/run_client

@@ -3,7 +3,7 @@
 script_dir=$(dirname $(realpath $0))
 
 usage() {
-  echo "Usage: $0 -e delta|-p [-i INTERVAL] [-t]" 1>&2;
+  echo "Usage: $0 -e delta | -p [-b] [-i INTERVAL] [-t] [-d TX_BUFFER_LEN]" 1>&2;
   exit 1;
 }
 
@@ -14,13 +14,23 @@ interval=100000
 client_options="-a -p 99 -f eth0"
 qdisc_options=""
 
-while getopts "e:i:pt" opt; do
+while getopts "bd:e:hi:pt" opt; do
   case "${opt}" in
+    b )
+        client_options+=" -b"
+        ;;
+    d )
+        client_options+=" -d ${OPTARG}"
+        ;;
     e )
         use_etf=1
         client_options+=" -e -q 7"
         qdisc_options+="-e ${OPTARG}"
         ;;
+    h )
+        usage
+        exit 1
+        ;;
     i )
         interval=${OPTARG}
         ;;
@@ -46,8 +56,8 @@ if [ -n "${use_etf}" ] && [ -n "${use_pfast}" ]; then
   usage
 fi
 
-echo "$script_dir/create_qdisc $qdisc_options";
+echo "create_qdisc $qdisc_options";
 $script_dir/create_qdisc $qdisc_options;
 
-echo "$script_dir/../packet-exchange/build/client_arm $client_options -i $interval 192.168.99.25";
+echo "client_arm $client_options -i $interval 192.168.99.25";
 $script_dir/../packet-exchange/build/client_arm $client_options -i $interval 192.168.99.25;

scripts/run_server

@@ -3,12 +3,22 @@
 script_dir=$(dirname $(realpath $0))
 
 usage() {
-  echo "Usage: $0 [-t NB_PACKETS]" 1>&2;
+  echo "Usage: $0 [[-b] | -t NB_PACKETS]" 1>&2;
   exit 1;
 }
 
-while getopts "t:" opt; do
+server_options="-a -p 99"
+server_options="-a -p 99"
+
+while getopts "bht:" opt; do
   case "${opt}" in
+    b )
+        server_options+=" -b 192.168.99.26"
+        ;;
+    h )
+        usage
+        exit 1
+        ;;
     t )
         use_tcpdump=1
         nb_packets=${OPTARG}
@@ -26,9 +36,9 @@ if [ -n "${use_tcpdump}" ]; then
   tcpdump -c $nb_packets -i eth0 -w tmp.pcap -tt --time-stamp-precision=nano udp port 50000;
   echo "tshark -r tmp.pcap --disable-protocol dcp-etsi --disable-protocol dcp-pft -t e -E separator=, -T fields -e frame.number -e frame.time_epoch -e data.data > tmp.out";
   tshark -r tmp.pcap --disable-protocol dcp-etsi --disable-protocol dcp-pft -t e -E separator=, -T fields -e frame.number -e frame.time_epoch -e data.data > tmp.out;
-  echo "$script_dir/txtime_stats.py -f tmp.out";
+  echo "txtime_stats.py -f tmp.out";
   $script_dir/txtime_stats.py -f tmp.out;
 else
-  echo "$script_dir/../packet-exchange/build/server_arm -a -p 99";
-  $script_dir/../packet-exchange/build/server_arm -a -p 99;
+  echo "server_arm $server_options";
+  $script_dir/../packet-exchange/build/server_arm $server_options;
 fi