Merge branch 'packet-exchange' into dev

packet-exchange/build/Makefile

@@ -6,11 +6,13 @@ SERVER_SRCS  = server.c
 SERVER_SRCS  += recv_packet.c
 SERVER_SRCS  += send_packet.c
 SERVER_SRCS  += common.c
+SERVER_SRCS  += tracer.c
 
 CLIENT_SRCS  = client.c
 CLIENT_SRCS += recv_packet.c
 CLIENT_SRCS += send_packet.c
 CLIENT_SRCS += common.c
+CLIENT_SRCS += tracer.c
 
 SERVER_OBJS = $(SERVER_SRCS:%.c=%.o)
 CLIENT_OBJS = $(CLIENT_SRCS:%.c=%.o)

packet-exchange/src/client.c

@@ -16,12 +16,14 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/mman.h>
 #include <time.h>
 #include <unistd.h>
 
 #include "common.h"
 #include "recv_packet.h"
 #include "send_packet.h"
+#include "tracer.h"
 
 // Structs
 
@@ -39,12 +41,15 @@ typedef struct thread_param {
   unsigned int max_cycles;
   int priority;
   int etf_offset;
+  uint64_t latency_threshold;
 
 } thread_param_t;
 
 typedef struct main_param {
   int refresh_rate;
   int verbose;
+  int enable_tracing;
+  int enable_graph;
 } main_param_t;
 
 // Static functions
@@ -67,7 +72,8 @@ static egress_param_t egress_params;
 static ingress_param_t ingress_params;
 
 static rtt_stat_t rtt_stats = {.min_rtt = INT_MAX};
-static egress_stat_t egress_stats = {.min_kernel_latency = INT_MAX};
+static egress_stat_t egress_stats = {.min_kernel_latency = INT_MAX,
+                                     .min_interval = INT_MAX};
 static ingress_stat_t ingress_stats = {.min_kernel_latency = INT_MAX,
                                        .min_interval = INT_MAX};
 
@@ -85,35 +91,33 @@ static char send_data[MAX_BUFFER_SIZE];
 
 static void help(char *argv[]) {
   printf(
-      "Usage: %s -f IF  [-abthgv] [-e ETF_OFFSET] [-d BUF_LEN] [-i USEC] [-l "
-      "N] [-p PRIO] [-q PACKET_PRIO] [-r USEC]\n\n",
-      argv[0]);
-  printf("       -a                    Run the real time thread on CPU1\n");
-  printf("       -b                    Measure RTT\n");
-  printf("       -d BUF_LEN            Set the length of tx buffer\n");
-  printf(
+      "Usage: %s -f IF  [-abthgv] [-e ETF_OFFSET] [-d BUF_LEN] [-i USEC] [-l N]"
+      "[-p PRIO] [-q PACKET_PRIO] [-r USEC] [-T LATENCY_THRESHOLD -G]\n\n",
+      "       -a                    Run the real time thread on CPU1\n"
+      "       -b                    Measure RTT\n"
+      "       -d BUF_LEN            Set the length of tx buffer\n"
       "       -e ETF_OFFSET         Set a txtime with an offset of ETF_OFFSET "
-      "us (to be used in an ETF qdisc)\n");
-  printf("       -f IF                 Set the network interface to be used\n");
-  printf("       -g                    Print histograms to sdtout on exit\n");
-  printf("       -h                    Show help\n");
-  printf(
+      "us (to be used in an ETF qdisc)\n"
+      "       -f IF                 Set the network interface to be used\n"
+      "       -g                    Print histograms to sdtout on exit\n"
+      "       -h                    Show help\n"
       "       -i USEC               Wake up the real time thread every USEC "
-      "microseconds (Default: 10ms)\n");
-  printf(
+      "microseconds (Default: 10ms)\n"
       "       -l N                  Wake up the real time thread N times "
-      "(Default: 0)\n");
-  printf(
+      "(Default: 0)\n"
       "       -p PRIO               Run the real time thread at priority "
-      "PRIO\n");
-  printf(
-      "       -q PACKET_PRIO        Send packets with PACKET_PRIO priority\n");
-  printf(
+      "PRIO\n"
+      "       -q PACKET_PRIO        Send packets with PACKET_PRIO priority\n"
       "       -r USEC               Refresh the non real time main thread "
-      "every USEC microseconds (Default: 50ms)\n");
-  printf("       -t                    Enable timestamps\n");
-  printf("       -v                    Verbose\n");
-  printf("\n");
+      "every USEC microseconds (Default: 50ms)\n"
+      "       -t                    Enable timestamps\n"
+      "       -v                    Verbose\n"
+      "       -T LATENCY_THRESHOLD  Enable tracing until LATENCY_THRESHOLD is "
+      "reached\n"
+      "       -G                    Enable function_graph tracer, used with "
+      "-T\n"
+      "\n",
+      argv[0]);
 }
 
 /*
@@ -121,9 +125,8 @@ static void help(char *argv[]) {
  */
 static void *packet_sending_thread(void *p) {
   (void)p;
-  struct timespec next;
+  struct timespec next, current, previous;
   uint64_t next_txtime;
-  struct sched_param priority;
   cpu_set_t mask;
 
   // Set thread CPU affinity
@@ -134,11 +137,6 @@ static void *packet_sending_thread(void *p) {
       error(EXIT_FAILURE, errno, "Could not set CPU affinity to CPU #1\n");
   }
 
-  // Set thread priority
-  priority.sched_priority = thread_params.priority;
-  if (sched_setscheduler(0, SCHED_FIFO, &priority))
-    error(EXIT_FAILURE, errno, "Couldn't set priority");
-
   if (enable_etf) {
     // Measure from CLOCK_TAI to generate timestamp
     clock_gettime(CLOCK_TAI, &next);
@@ -148,6 +146,9 @@ static void *packet_sending_thread(void *p) {
     next_txtime = 0;
   }
 
+  // Start tracing
+  if (main_params.enable_tracing) tracing(1);
+
   clock_gettime(CLOCK_MONOTONIC, &next);
   clock_gettime(CLOCK_MONOTONIC, &measures_start);
 
@@ -157,6 +158,8 @@ static void *packet_sending_thread(void *p) {
         nb_cycles >= ((unsigned int)thread_params.max_cycles))
       break;
 
+    clock_gettime(CLOCK_MONOTONIC, &current);
+
     sprintf(send_data, "%d", (int)(nb_cycles % 1000));
     do_tsn_task(send_data, next_txtime);
 
@@ -164,6 +167,25 @@ static void *packet_sending_thread(void *p) {
 
     if (enable_etf) next_txtime += thread_params.interval;
 
+    if (nb_cycles) {
+      int interval_us = calcdiff_ns(current, previous) / 1000;
+
+      egress_stats.min_interval = min(interval_us, egress_stats.min_interval);
+      egress_stats.max_interval = max(interval_us, egress_stats.max_interval);
+      egress_stats.avg_interval =
+          (egress_stats.avg_interval * nb_cycles + interval_us) /
+          (nb_cycles + 1);
+    }
+
+    // If the latency hits the tracing threshold, stop tracing
+    if (main_params.enable_tracing &&
+        (egress_stats.max_interval > thread_params.latency_threshold)) {
+      tracing(0);
+      break;
+    }
+
+    previous = current;
+
     clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &next, NULL);
   }
 
@@ -176,6 +198,8 @@ static void *packet_sending_thread(void *p) {
  */
 int main(int argc, char *argv[]) {
   pthread_t thread;
+  struct sched_param param;
+  pthread_attr_t attr;
 
   // Default configuration values
   thread_params.interval = 100000 * 1000;
@@ -184,6 +208,8 @@ int main(int argc, char *argv[]) {
 
   main_params.refresh_rate = 50000;
   main_params.verbose = 0;
+  main_params.enable_tracing = 0;
+  main_params.enable_graph = 0;
 
   enable_affinity = 0;
   enable_etf = 0;
@@ -194,6 +220,12 @@ int main(int argc, char *argv[]) {
   egress_params.packet_priority = 3;
   egress_params.tx_buffer_len = 1024;
 
+  /* Lock all current and future pages from preventing of being paged to swap */
+  if (mlockall(MCL_CURRENT | MCL_FUTURE)) {
+    perror("mlockall failed");
+    /* exit(-1) or do error handling */
+  }
+
   // Process bash options
   process_options(argc, argv);
 
@@ -206,6 +238,11 @@ int main(int argc, char *argv[]) {
     memset(rtt_hist, 0, sizeof(rtt_hist));
   }
 
+  if (main_params.enable_tracing) {
+    // Enable ftrace
+    setup_tracer(main_params.enable_graph);
+  }
+
   // Catch breaks with sighand to print the histograms
   init_signals(sighand, enable_histograms);
 
@@ -218,8 +255,34 @@ int main(int argc, char *argv[]) {
     init_udp_recv(&ingress_params, &ingress_stats, enable_histograms,
                   kernel_latency_hist);
 
+  /* Initialize pthread attributes (default values) */
+  if (pthread_attr_init(&attr)) {
+    fprintf(stderr, "init pthread attributes failed\n");
+    exit(EXIT_FAILURE);
+  }
+  /* Set a specific stack size  */
+  if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN)) {
+    fprintf(stderr, "pthread setstacksize failed\n");
+    exit(EXIT_FAILURE);
+  }
+  /* Set scheduler policy and priority of pthread */
+  if (pthread_attr_setschedpolicy(&attr, SCHED_FIFO)) {
+    fprintf(stderr, "pthread setschedpolicy failed\n");
+    exit(EXIT_FAILURE);
+  }
+  param.sched_priority = thread_params.priority;
+  if (pthread_attr_setschedparam(&attr, &param)) {
+    fprintf(stderr, "pthread setschedparam failed\n");
+    exit(EXIT_FAILURE);
+  }
+  /* Use scheduling parameters of attr */
+  if (pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED)) {
+    fprintf(stderr, "pthread setinheritsched failed\n");
+    exit(EXIT_FAILURE);
+  }
+
   // Create the real time thread
-  if (pthread_create(&thread, NULL, packet_sending_thread, NULL))
+  if (pthread_create(&thread, &attr, packet_sending_thread, NULL))
     error(EXIT_FAILURE, errno, "Couldn't create packet sending thread");
 
   // Verbose loop
@@ -232,9 +295,11 @@ int main(int argc, char *argv[]) {
                rtt_stats.avg_rtt, rtt_stats.max_rtt);
         printf("\033[%dA", 1);
       } else {
-        printf("%9" PRIu64 ": [%4d, %4d]", nb_cycles,
+        printf("%9" PRIu64 ": [%4d, %4d], I (10us): %3d %3d %3d", nb_cycles,
                (int)egress_stats.invalid_parameter,
-               (int)egress_stats.missed_deadline);
+               (int)egress_stats.missed_deadline,
+               egress_stats.min_interval / 10, egress_stats.avg_interval / 10,
+               egress_stats.max_interval / 10);
 
         if (enable_timestamps) {
           printf(", K: %4d %4d %4d [%4d]\n", egress_stats.min_kernel_latency,
@@ -340,7 +405,7 @@ static void process_options(int argc, char *argv[]) {
   int network_if_specified = 0;
 
   for (;;) {
-    int c = getopt(argc, argv, "abd:e:f:ghi:l:p:q:r:tv");
+    int c = getopt(argc, argv, "abd:e:f:ghi:l:p:q:r:tvT:G");
 
     if (c == -1) break;
 
@@ -394,6 +459,13 @@ static void process_options(int argc, char *argv[]) {
       case 'v':
         main_params.verbose = 1;
         break;
+      case 'T':
+        main_params.enable_tracing = 1;
+        thread_params.latency_threshold = atoi(optarg);
+        break;
+      case 'G':
+        main_params.enable_graph = 1;
+        break;
     }
   }
 

packet-exchange/src/common.h

@@ -18,7 +18,7 @@
 
 #define MAX_BUFFER_SIZE 1024
 
-#define TIMESTAMP_BUFFER_SIZE 64
+#define TIMESTAMP_BUFFER_SIZE 4096
 
 uint64_t ts_to_uint(struct timespec t);
 void add_ns(struct timespec *t, uint64_t ns);

packet-exchange/src/send_packet.c

@@ -7,6 +7,8 @@
  */
 
 #define _GNU_SOURCE
+#include "send_packet.h"
+
 #include <arpa/inet.h>
 #include <errno.h>
 #include <error.h>
@@ -35,7 +37,6 @@
 #include <unistd.h>
 
 #include "common.h"
-#include "send_packet.h"
 
 static void *poll_thread(void *p);
 static void process_error_queue();
@@ -56,18 +57,15 @@ static struct sock_txtime sk_txtime;
 static char *tx_buffer;
 static int sock_fd;
 
-static uint64_t timestamps_buffer[TIMESTAMP_BUFFER_SIZE];
+static int64_t timestamps_buffer[TIMESTAMP_BUFFER_SIZE];
 static int ts_buf_read_index = 0;
 static int ts_buf_write_index = 0;
 
 /*
  * Init UDP socket
  */
-void init_udp_send(egress_param_t *_params,
-                   egress_stat_t *_stats,
-                   int _use_histogram,
-                   uint64_t *_kernel_latency_hist) {
-
+void init_udp_send(egress_param_t *_params, egress_stat_t *_stats,
+                   int _use_histogram, uint64_t *_kernel_latency_hist) {
   int set_if_err;
   pthread_t thread;
 
@@ -79,12 +77,10 @@ void init_udp_send(egress_param_t *_params,
   init_tx_buffer();
 
   sock_fd = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP);
-  if (sock_fd < 0)
-    error(EXIT_FAILURE, errno, "Socket creation failed\n");
+  if (sock_fd < 0) error(EXIT_FAILURE, errno, "Socket creation failed\n");
 
   set_if_err = set_if();
-  if (set_if_err < 0)
-    error(EXIT_FAILURE, errno, "Couldn't set interface\n");
+  if (set_if_err < 0) error(EXIT_FAILURE, errno, "Couldn't set interface\n");
 
   if (setsockopt(sock_fd, SOL_SOCKET, SO_PRIORITY, &params->packet_priority,
                  sizeof(params->packet_priority)))
@@ -98,7 +94,8 @@ void init_udp_send(egress_param_t *_params,
     sk_txtime.clockid = CLOCK_TAI;
     sk_txtime.flags = SOF_TXTIME_REPORT_ERRORS;
 
-    if (setsockopt(sock_fd, SOL_SOCKET, SO_TXTIME, &sk_txtime, sizeof(sk_txtime)))
+    if (setsockopt(sock_fd, SOL_SOCKET, SO_TXTIME, &sk_txtime,
+                   sizeof(sk_txtime)))
       error(EXIT_FAILURE, errno, "setsockopt SO_TXTIME failed\n");
   }
 
@@ -113,12 +110,11 @@ void init_udp_send(egress_param_t *_params,
     error(EXIT_FAILURE, errno, "Couldn't create poll thread");
 }
 
+
 /*
  * Sends udp packets
  */
-void send_udp_packet(char *data,
-                     uint64_t txtime) {
-
+void send_udp_packet(char *data, uint64_t txtime) {
   struct msghdr msg;     // Message hardware, sent to the socket
   struct cmsghdr *cmsg;  // Control message hardware, for txtime
   char control[CMSG_SPACE(sizeof(txtime))] = {};  // Stores txtime
@@ -170,7 +166,6 @@ void send_udp_packet(char *data,
 }
 
 static void *poll_thread(void *p) {
-
   (void)p;
   // Poll file descriptor
   struct pollfd poll_fd = {.fd = sock_fd};
@@ -192,16 +187,15 @@ static void process_error_queue() {
 
   // IO vector
   unsigned char data_buffer[256];  // Buffer in io vector
-  struct iovec iov = {
-      .iov_base = data_buffer,
-      .iov_len = sizeof(data_buffer)};
+  struct iovec iov = {.iov_base = data_buffer, .iov_len = sizeof(data_buffer)};
 
   // Control data, will store error or timestamps
-  unsigned char msg_control[CMSG_SPACE(sizeof(struct sock_extended_err)) + CMSG_SPACE(sizeof(struct timespec))];
+  unsigned char msg_control[CMSG_SPACE(sizeof(struct sock_extended_err)) +
+                            CMSG_SPACE(sizeof(struct timespec))];
 
-  // Message hardware structure, containts IO vector and control message hardware
-  struct msghdr msg = {
-      .msg_iov = &iov,
+  // Message hardware structure, containts IO vector and control message
+  // hardware
+  struct msghdr msg = {.msg_iov = &iov,
                        .msg_iovlen = 1,
                        .msg_control = msg_control,
                        .msg_controllen = sizeof(msg_control)};
@@ -218,15 +212,19 @@ static void process_error_queue() {
   for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
     // If a timestamp was received
     if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SO_TIMESTAMPING) {
-
       struct timespec *stamp = (struct timespec *)CMSG_DATA(cmsg);
 
-      int kernel_latency = (ts_to_uint(*stamp) - timestamps_buffer[ts_buf_read_index]) / 1000;
+      int kernel_latency =
+          (ts_to_uint(*stamp) - timestamps_buffer[ts_buf_read_index]) / 1000;
       ts_buf_read_index = (ts_buf_read_index + 1) % TIMESTAMP_BUFFER_SIZE;
 
-      stats->min_kernel_latency = min(kernel_latency, stats->min_kernel_latency);
-      stats->max_kernel_latency = max(kernel_latency, stats->max_kernel_latency);
-      stats->avg_kernel_latency = (stats->max_kernel_latency * packets_sent + kernel_latency) / (packets_sent + 1);
+      stats->min_kernel_latency =
+          min(kernel_latency, stats->min_kernel_latency);
+      stats->max_kernel_latency =
+          max(kernel_latency, stats->max_kernel_latency);
+      stats->avg_kernel_latency =
+          (stats->max_kernel_latency * packets_sent + kernel_latency) /
+          (packets_sent + 1);
 
       if (use_histogram) {
         if (kernel_latency > MAX_KERNEL_LATENCY)
@@ -238,11 +236,9 @@ static void process_error_queue() {
     }
     // If an error was received
     else {
-
       struct sock_extended_err *serr = (void *)CMSG_DATA(cmsg);
 
-      if (serr->ee_origin != SO_EE_ORIGIN_TXTIME)
-        continue;
+      if (serr->ee_origin != SO_EE_ORIGIN_TXTIME) continue;
 
       switch (serr->ee_code) {
         case SO_EE_CODE_TXTIME_INVALID_PARAM:

packet-exchange/src/send_packet.h

@@ -24,6 +24,10 @@ typedef struct egress_stat {
   int avg_kernel_latency;
   int max_kernel_latency;
 
+  int min_interval;
+  int avg_interval;
+  int max_interval;
+
 } egress_stat_t;
 
 void init_udp_send(egress_param_t *_params,

packet-exchange/src/server.c

@@ -20,6 +20,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/mman.h>
 #include <sys/socket.h>
 #include <sys/types.h>
 #include <time.h>
@@ -28,6 +29,7 @@
 #include "common.h"
 #include "recv_packet.h"
 #include "send_packet.h"
+#include "tracer.h"
 
 // Structs
 
@@ -36,12 +38,15 @@ enum TSNTask { RECV_PACKET_TASK, RTT_TASK };
 typedef struct thread_param {
   int interval;
   int priority;
+  uint64_t latency_threshold;
 
 } thread_param_t;
 
 typedef struct main_params {
   int refresh_rate;
   int verbose;
+  int enable_tracing;
+  int enable_graph;
 } main_param_t;
 
 static void process_options(int argc, char *argv[]);
@@ -72,19 +77,29 @@ static struct timespec measures_start;
 static struct timespec measures_end;
 
 static void help(char *argv[]) {
-  printf("Usage: %s [-aghtv] [-b CLIENT_IP] [-d BUF_LEN] [-f IF] [-i USEC] [-p PRIO] [-r USEC]\n\n", argv[0]);
-  printf("       -a                    Run the real time thread on CPU1\n");
-  printf("       -b CLIENT_IP          Server side RTT\n");
-  printf("       -d BUF_LEN            Set the length of tx buffer\n");
-  printf("       -f IF                 Set the network interface to be used\n");
-  printf("       -g                    Print histograms to sdtout on exit\n");
-  printf("       -h                    Show help\n");
-  printf("       -i USEC               Wake up the real time thread every USEC microseconds (Default: 10ms)\n");
-  printf("       -p PRIO               Run the real time thread at priority PRIO\n");
-  printf("       -r USEC               Refresh the non real time main thread every USEC microseconds\n");
-  printf("       -t                    Enable timestamps\n");
-  printf("       -v                    Verbose\n");
-  printf("\n");
+  printf(
+      "Usage: %s [-aghtv] [-b CLIENT_IP] [-d BUF_LEN] [-f IF] [-i USEC] [-p "
+      "PRIO] [-T LATENCY_THRESHOLD -G]"
+      " [-r USEC]\n\n",
+      "       -a                    Run the real time thread on CPU1\n"
+      "       -b CLIENT_IP          Server side RTT\n"
+      "       -d BUF_LEN            Set the length of tx buffer\n"
+      "       -f IF                 Set the network interface to be used\n"
+      "       -g                    Print histograms to sdtout on exit\n"
+      "       -h                    Show help\n"
+      "       -i USEC               Wake up the real time thread every USEC "
+      "microseconds (Default: 10ms)\n"
+      "       -p PRIO               Run the real time thread at priority PRIO\n"
+      "       -r USEC               Refresh the non real time main thread "
+      "every USEC microseconds\n"
+      "       -t                    Enable timestamps\n"
+      "       -v                    Verbose\n"
+      "       -T LATENCY_THRESHOLD  Enable tracing until LATENCY_THRESHOLD is "
+      "reached\n"
+      "       -G                    Enable function_graph tracer, used with "
+      "-T\n"
+      "\n",
+      argv[0]);
 }
 
 // Real-time thread
@@ -92,7 +107,6 @@ static void help(char *argv[]) {
 static void *packet_receiving_thread(void *p) {
   (void)p;
   struct timespec current, previous;
-  struct sched_param priority;
   cpu_set_t mask;
   int prev_packet_id = 0;
 
@@ -104,23 +118,18 @@ static void *packet_receiving_thread(void *p) {
       error(EXIT_FAILURE, errno, "Could not set CPU affinity to CPU #1\n");
   }
 
-  // Set thread priority
-  priority.sched_priority = thread_params.priority;
-  if (sched_setscheduler(0, SCHED_FIFO, &priority))
-    error(EXIT_FAILURE, errno, "Couldn't set priority");
+  // Start tracing
+  if (main_params.enable_tracing) tracing(1);
 
   clock_gettime(CLOCK_MONOTONIC, &measures_start);
 
   // Packet receiving loop
   for (ingress_stats.packets_received = 0;; ingress_stats.packets_received++) {
-
     if (tsn_task == RTT_TASK) {
-
       recv_udp_packet();
       send_udp_packet("", 0);
 
     } else if (tsn_task == RECV_PACKET_TASK) {
-
       int current_packet_id;
 
       recv_udp_packet();
@@ -132,12 +141,18 @@ static void *packet_receiving_thread(void *p) {
       if (ingress_stats.packets_received) {
         int interval_us = calcdiff_ns(current, previous) / 1000;
 
-        ingress_stats.min_interval = min(interval_us, ingress_stats.min_interval);
-        ingress_stats.max_interval = max(interval_us, ingress_stats.max_interval);
-        ingress_stats.avg_interval = (ingress_stats.avg_interval * ingress_stats.packets_received + interval_us) / (ingress_stats.packets_received + 1);
+        ingress_stats.min_interval =
+            min(interval_us, ingress_stats.min_interval);
+        ingress_stats.max_interval =
+            max(interval_us, ingress_stats.max_interval);
+        ingress_stats.avg_interval =
+            (ingress_stats.avg_interval * ingress_stats.packets_received +
+             interval_us) /
+            (ingress_stats.packets_received + 1);
 
         // Check if packets were lost
-        ingress_stats.lost_packets += (current_packet_id - prev_packet_id - 1) % 1000;
+        ingress_stats.lost_packets +=
+            (current_packet_id - prev_packet_id - 1) % 1000;
 
         if (enable_histograms) {
           int dist_to_interval = interval_us - (thread_params.interval / 1000);
@@ -150,6 +165,13 @@ static void *packet_receiving_thread(void *p) {
         }
       }
 
+      // If the latency hits the tracing threshold, stop tracing
+      if (main_params.enable_tracing &&
+          (ingress_stats.max_interval > thread_params.latency_threshold)) {
+        tracing(0);
+        break;
+      }
+
       previous = current;
       prev_packet_id = current_packet_id;
     }
@@ -162,6 +184,8 @@ static void *packet_receiving_thread(void *p) {
 // Handles the IO and creates real time threads
 int main(int argc, char *argv[]) {
   pthread_t thread;
+  struct sched_param param;
+  pthread_attr_t attr;
 
   ingress_stats.min_interval = INT_MAX;
   ingress_stats.avg_interval = 0;
@@ -178,6 +202,8 @@ int main(int argc, char *argv[]) {
   thread_params.priority = 99;
   main_params.refresh_rate = 50000;
   main_params.verbose = 0;
+  main_params.enable_tracing = 0;
+  main_params.enable_graph = 0;
 
   enable_affinity = 0;
   enable_timestamps = 0;
@@ -197,24 +223,50 @@ int main(int argc, char *argv[]) {
     memset(jitter_hist, 0, sizeof(jitter_hist));
   }
 
+  if (main_params.enable_tracing) {
+    // Enable ftrace
+    setup_tracer(main_params.enable_graph);
+  }
+
   // Catch breaks with sighand to print the histograms
   init_signals(sighand, enable_histograms);
 
   // Initialize the UDP packet receiving socket
-  init_udp_recv(&ingress_params,
-                &ingress_stats,
-                enable_histograms,
+  init_udp_recv(&ingress_params, &ingress_stats, enable_histograms,
                 kernel_latency_hist);
 
   // Initialize the UDP packet sending socket if RTT is measured
   if (tsn_task == RTT_TASK)
-    init_udp_send(&egress_params,
-                  &egress_stats,
-                  0,
-                  NULL);
+    init_udp_send(&egress_params, &egress_stats, 0, NULL);
+
+  /* Initialize pthread attributes (default values) */
+  if (pthread_attr_init(&attr)) {
+    fprintf(stderr, "init pthread attributes failed\n");
+    exit(EXIT_FAILURE);
+  }
+  /* Set a specific stack size  */
+  if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN)) {
+    fprintf(stderr, "pthread setstacksize failed\n");
+    exit(EXIT_FAILURE);
+  }
+  /* Set scheduler policy and priority of pthread */
+  if (pthread_attr_setschedpolicy(&attr, SCHED_FIFO)) {
+    fprintf(stderr, "pthread setschedpolicy failed\n");
+    exit(EXIT_FAILURE);
+  }
+  param.sched_priority = thread_params.priority;
+  if (pthread_attr_setschedparam(&attr, &param)) {
+    fprintf(stderr, "pthread setschedparam failed\n");
+    exit(EXIT_FAILURE);
+  }
+  /* Use scheduling parameters of attr */
+  if (pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED)) {
+    fprintf(stderr, "pthread setinheritsched failed\n");
+    exit(EXIT_FAILURE);
+  }
 
   // Create the real time thread
-  if (pthread_create(&thread, NULL, packet_receiving_thread, NULL))
+  if (pthread_create(&thread, &attr, packet_receiving_thread, NULL))
     error(EXIT_FAILURE, errno, "Couldn't create packet receiving thread");
 
   // Verbose loop
@@ -222,27 +274,20 @@ int main(int argc, char *argv[]) {
     usleep(main_params.refresh_rate);
 
     if (main_params.verbose && ingress_stats.packets_received > 1) {
-
       if (tsn_task == RECV_PACKET_TASK) {
-
         int jitter = ingress_stats.max_interval - ingress_stats.min_interval;
 
-        printf("%9" PRIu64 ": J: %5d, I (ms): %3d %3d %3d [%3d,%3d]",
-               ingress_stats.packets_received,
-               jitter,
-               ingress_stats.min_interval / 1000,
-               ingress_stats.avg_interval / 1000,
-               ingress_stats.max_interval / 1000,
-               (int) ingress_stats.high_jitter,
-               (int) ingress_stats.lost_packets);
+        printf("%9" PRIu64 ": J: %5d, I (10us): %3d %3d %3d [%3d,%3d]",
+               ingress_stats.packets_received, jitter,
+               ingress_stats.min_interval / 10, ingress_stats.avg_interval / 10,
+               ingress_stats.max_interval / 10, (int)ingress_stats.high_jitter,
+               (int)ingress_stats.lost_packets);
 
         if (enable_timestamps) {
-
-          printf(", K: %4d %4d %4d [%3d]\n",
-                 ingress_stats.min_kernel_latency,
+          printf(", K: %4d %4d %4d [%3d]\n", ingress_stats.min_kernel_latency,
                  ingress_stats.avg_kernel_latency,
                  ingress_stats.max_kernel_latency,
-                 (int) ingress_stats.high_kernel_latency);
+                 (int)ingress_stats.high_kernel_latency);
         } else {
           printf("\n");
         }
@@ -250,12 +295,15 @@ int main(int argc, char *argv[]) {
         printf("\033[%dA", 1);
       }
     }
+
+    if (main_params.enable_tracing &&
+        (ingress_stats.max_interval >= thread_params.latency_threshold))
+      break;
   }
 }
 
 // Print histograms in .json format
 static void print_histograms() {
-
   uint64_t duration;
   int duration_hour, duration_minutes, interval;
   int max_latency, max_jitter, min_jitter;
@@ -269,7 +317,8 @@ static void print_histograms() {
   interval = thread_params.interval / 1000;
 
   if (enable_timestamps) {
-    printf("{\"measure_sets\": [{"
+    printf(
+        "{\"measure_sets\": [{"
         "\"measure_type\": \"packet_recv_timestamps\","
         "\"props_names\": [\"kernel_space\"],"
         "\"units\": [\"us\"],"
@@ -277,12 +326,14 @@ static void print_histograms() {
         "\"metadata\": {\"i\": \"%dus\", \"duration\": \"%dh%d\","
         "\"lost_packets\": \"%d\"},"
         "\"props\": [[",
-           interval, duration_hour, duration_minutes, ingress_stats.lost_packets);
+        interval, duration_hour, duration_minutes,
+        ingress_stats.lost_packets + ((int)ingress_stats.high_jitter));
 
     max_latency = histogram_max(kernel_latency_hist, MAX_KERNEL_LATENCY - 1);
 
     for (int j = 0; j < max_latency; j++)
-      printf("%" PRIi64 "%s", kernel_latency_hist[j], (j + 1 < max_latency ? ", " : ""));
+      printf("%" PRIi64 "%s", kernel_latency_hist[j],
+             (j + 1 < max_latency ? ", " : ""));
     printf("]]");
   }
 
@@ -291,7 +342,8 @@ static void print_histograms() {
 
   printf("%s", enable_timestamps ? "}, {" : "{\"measure_sets\": [{");
 
-  printf("\"measure_type\": \"packet_jitter\","
+  printf(
+      "\"measure_type\": \"packet_jitter\","
       "\"props_names\": [\"jitter\"],"
       "\"units\": [\"us\"],"
       "\"props_type\": \"histogram\","
@@ -300,7 +352,7 @@ static void print_histograms() {
       "\"lost_packets\": \"%d\"},"
       "\"props\": [[",
       MAX_JITTER / 2 - min_jitter, interval, duration_hour, duration_minutes,
-         ingress_stats.lost_packets);
+      ingress_stats.lost_packets + ((int)ingress_stats.high_jitter));
 
   for (int j = min_jitter; j < max_jitter; j++)
     printf("%" PRIi64 "%s", jitter_hist[j], (j + 1 < max_jitter ? ", " : ""));
@@ -320,14 +372,12 @@ static void sighand(int sig_num) {
 
 // Process bash options
 static void process_options(int argc, char *argv[]) {
-
   int network_if_specified = 0;
 
   for (;;) {
-    int c = getopt(argc, argv, "ab:d:f:ghi:p:r:tv");
+    int c = getopt(argc, argv, "ab:d:f:ghi:p:r:tvT:G");
 
-    if (c == -1)
-      break;
+    if (c == -1) break;
 
     switch (c) {
       case 'a':
@@ -370,6 +420,13 @@ static void process_options(int argc, char *argv[]) {
       case 'v':
         main_params.verbose = 1;
         break;
+      case 'T':
+        main_params.enable_tracing = 1;
+        thread_params.latency_threshold = atoi(optarg);
+        break;
+      case 'G':
+        main_params.enable_graph = 1;
+        break;
     }
   }
 

packet-exchange/src/tracer.c

+#include "tracer.h"
+
+#define _GNU_SOURCE
+#include <errno.h>
+#include <error.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <limits.h>
+#include <linux/unistd.h>
+#include <pthread.h>
+#include <sched.h>
+#include <signal.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/prctl.h>
+#include <sys/resource.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/sysinfo.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/utsname.h>
+#include <time.h>
+#include <unistd.h>
+
+/*
+ * From cyclictest code source
+ */
+
+#define KVARS 32
+#define KVARNAMELEN 32
+#define KVALUELEN 32
+
+#define MAX_PATH 255
+
+static char *fileprefix;
+static char *procfileprefix = "/proc/sys/kernel/";
+static char *debugfileprefix = "/sys/kernel/debug/tracing/";
+
+static int trace_fd = -1;
+static int tracemark_fd = -1;
+
+static int kernvar(int mode, const char *name, char *value,
+                   size_t sizeofvalue) {
+  char filename[128];
+  int retval = 1;
+  int path;
+  size_t len_prefix = strlen(fileprefix), len_name = strlen(name);
+
+  memcpy(filename, fileprefix, len_prefix);
+  memcpy(filename + len_prefix, name, len_name + 1);
+
+  path = open(filename, mode);
+  if (path >= 0) {
+    if (mode == O_RDONLY) {
+      int got;
+      if ((got = read(path, value, sizeofvalue)) > 0) {
+        retval = 0;
+        value[got - 1] = '\0';
+      }
+    } else if (mode == O_WRONLY) {
+      if (write(path, value, sizeofvalue) == sizeofvalue) retval = 0;
+    }
+    close(path);
+  }
+  return retval;
+}
+
+static void setkernvar(const char *name, char *value) {
+  if (kernvar(O_WRONLY, name, value, strlen(value)))
+    fprintf(stderr, "could not set %s to %s\n", name, value);
+}
+
+void setup_tracer(int enable_graph) {
+
+  char trace_path[MAX_PATH];
+  char tracemark_path[MAX_PATH];
+
+  fileprefix = procfileprefix;
+  setkernvar("ftrace_enabled", "1");
+
+  fileprefix = debugfileprefix;
+  // Clear old traces by setting tracer to nop first
+  setkernvar("current_tracer", "nop");
+  if(enable_graph)
+    setkernvar("current_tracer", "function_graph");
+  else
+    setkernvar("current_tracer", "function");
+
+  // Open tracing_on file
+  strcpy(trace_path, fileprefix);
+  strcat(trace_path, "tracing_on");
+  if ((trace_fd = open(trace_path, O_WRONLY)) == -1)
+    printf("unable to open %s for tracing", trace_path);
+
+  // Open trace mark file
+  strcpy(tracemark_path, fileprefix);
+  strcat(tracemark_path, "trace_marker");
+  if ((tracemark_fd = open(tracemark_path, O_WRONLY)) == -1)
+    printf("unable to open %s for tracing", tracemark_path);
+
+  tracing(0);
+}
+
+void tracing(int on) {
+  if (on)
+    write(trace_fd, "1", 1);
+  else
+    write(trace_fd, "0", 1);
+}
+
+void tracemark(char * s) {
+  write(tracemark_fd, s, strlen(s));
+}

packet-exchange/src/tracer.h

+#ifndef TRACER_H
+#define TRACER_H
+
+void setup_tracer(int enable_graph);
+void tracing(int on);
+void tracemark(char * s);
+
+#endif
+

scripts/multissh

+#!/bin/bash
+
+usage() {
+  cat << ENDUSAGE
+Usage: $0 CMD
+       $0 sudo CMD
+ENDUSAGE
+  exit 1;
+}
+
+if [ -z "$1" ]; then
+  usage
+fi
+
+if [ $1 == "sudo" ]; then
+  if [ -z "$2" ]; then
+    usage
+  fi
+  ./sudossh emerald "$2";
+  ./sudossh slate "$2";
+  ./sudossh onyx "$2";
+else
+  ssh emerald $1&
+  ssh onyx $1&
+  ssh slate $1
+fi

scripts/run-client

@@ -3,7 +3,7 @@
 script_dir=$(dirname $(realpath $0))
 
 usage() {
-  echo "Usage: $0 (-e delta [-o etf_offset] | -p) [-bgt] [-i INTERVAL] [-d TX_BUFFER_LEN] [emerald|slate|onyx]" 1>&2;
+  echo "Usage: $0 (-e delta [-o etf_offset] | -p) [-bgt] [-i INTERVAL] [-d TX_BUFFER_LEN] [-T LATENCY_THRESHOLD -G] [emerald|slate|onyx]" 1>&2;
   exit 1;
 }
 
@@ -16,7 +16,7 @@ qdisc_options=""
 ip="192.168.99."
 etf_offset=500
 
-while getopts "bd:e:o:ghi:pt" opt; do
+while getopts "bd:e:o:ghi:ptT:G" opt; do
   case "${opt}" in
     b )
         client_options+=" -b"
@@ -52,6 +52,12 @@ while getopts "bd:e:o:ghi:pt" opt; do
         use_timestamps=1
         client_options+=" -t"
       ;;
+    T )
+        client_options+=" -T ${OPTARG}"
+      ;;
+    G )
+        client_options+=" -G"
+      ;;
     * )
         usage
       ;;
@@ -102,8 +108,8 @@ fi
 
 client_options+=" -i $interval"
 
-echo "create_qdisc $qdisc_options";
-$script_dir/create_qdisc $qdisc_options;
+echo "create-qdisc $qdisc_options";
+$script_dir/create-qdisc $qdisc_options;
 
 echo "make client";
 cd $script_dir/../packet-exchange/build;

scripts/run-server

@@ -3,7 +3,7 @@
 script_dir=$(dirname $(realpath $0))
 
 usage() {
-  echo "Usage: $0 [[-bt] [-g INTERVAL]] | (-d NB_PACKETS [-i INTERVAL])" 1>&2;
+  echo "Usage: $0 [[-bt] [-g INTERVAL]] | (-d NB_PACKETS [-i INTERVAL]) [-T LATENCY_THRESHOLD -G]" 1>&2;
   exit 1;
 }
 
@@ -12,7 +12,7 @@ server_options="-a -p 99 -f eth0"
 ip="192.168.99."
 tcpdump_interval=1000000
 
-while getopts "b:htd:i:g:" opt; do
+while getopts "b:htd:i:g:T:G" opt; do
   case "${opt}" in
     b )
         use_rtt=1
@@ -37,6 +37,12 @@ while getopts "b:htd:i:g:" opt; do
     t )
         server_options+=" -t"
         ;;
+    T )
+        client_options+=" -T ${OPTARG}"
+      ;;
+    G )
+        client_options+=" -G"
+      ;;
     * )
         usage
       ;;
@@ -73,8 +79,8 @@ if [ -n "${use_tcpdump}" ]; then
   echo "tshark -r tmp.pcap --disable-protocol dcp-etsi --disable-protocol dcp-pft -t e -E separator=, -T fields -e frame.number -e frame.time_epoch -e data.data > tmp.out";
   tshark -r server_stats_tmp.pcap --disable-protocol dcp-etsi --disable-protocol dcp-pft -t e -E separator=, -T fields -e frame.number -e frame.time_epoch -e data.data > server_stats_tmp.out;
 
-  echo "txtime_stats.py -f server_pcap_stats -i $tcpdump_interval";
-  $script_dir/txtime_stats.py -f server_stats_tmp.out -i $tcpdump_interval;
+  echo "txtime-stats.py -f server_pcap_stats -i $tcpdump_interval";
+  $script_dir/txtime-stats.py -f server_stats_tmp.out -i $tcpdump_interval;
 else
 
   echo "make server";

scripts/sudossh

+#!/bin/expect
+
+spawn ssh [lindex $argv 0]
+
+expect "oli@"
+send -- "sudo [lindex $argv 1]\r"
+expect "assword"
+send -- "olimex\r"
+expect {
+      "Do you want to continue?" {
+            send "Y\r"
+            exp_continue
+      }
+      "oli@" {
+            exit
+      }
+}