software/simulation_generator: Add simulation_generator SR

Provide :
* a Python with selenium egg installed
* a Python script to run simulations
* a Fluentd instance
* instance parameters to send generated logs to a Wendelin instance through Fluentd

software/simulation_generator/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+[instance-selenium]
+filename = instance.cfg
+md5sum = 80c32e1a77cfab9e6df77fd4c145deeb

software/simulation_generator/flight_script.js

+/*jslint nomen: true, indent: 2, maxerr: 3, maxlen: 80 */
+/*global console, me*/
+
+(function (console, me) {
+  "use strict";
+
+  var ALTITUDE = 100,
+    EPSILON = 9,
+    CHECKPOINT_LIST = [],
+    DIRECTION_LIST = [
+      {
+        distance: 1053,
+        bearing: 293
+      },
+      {
+        distance: 800,
+        bearing: 104
+      },
+      {
+        distance: 669,
+        bearing: 352
+      },
+      {
+        distance: 925,
+        bearing: 162
+      },
+      {
+        distance: 690,
+        bearing: 47
+      },
+      {
+        distance: 935,
+        bearing: 166
+      },
+      {
+        distance: 1129,
+        bearing: 289
+      },
+      {
+        distance: 520,
+        bearing: 94
+      }
+    ],
+    R = 6371e3; // meters
+
+  function to_deg(rad) {
+    return rad * 180 / Math.PI;
+  }
+
+  function to_rad(deg) {
+    return deg * Math.PI / 180;
+  }
+
+  function set_checkpoints(lat, lon) {
+    var bearing_rad,
+      lat_end,
+      lon_end,
+      lat_start = to_rad(lat),
+      lon_start = to_rad(lon),
+      relative_d;
+
+    DIRECTION_LIST.forEach(function (e) {
+      bearing_rad = to_rad(e.bearing);
+      relative_d = e.distance / R;
+
+      lat_end = Math.asin(Math.sin(lat_start) * Math.cos(relative_d) +
+        Math.cos(lat_start) * Math.sin(relative_d) * Math.cos(bearing_rad));
+      lon_end = lon_start + Math.atan2(
+        Math.sin(bearing_rad) * Math.sin(relative_d) * Math.cos(lat_start),
+        Math.cos(relative_d) - Math.sin(lat_start) * Math.sin(lon_start));
+
+      CHECKPOINT_LIST.push({
+        latitude: to_deg(lat_end),
+        longitude: to_deg(lon_end)
+      });
+
+      lat_start = lat_end;
+      lon_start = lon_end;
+    });
+  }
+
+  function distance(lat1, lon1, lat2, lon2) {
+    var la1 = lat1 * Math.PI / 180, // lat, lon in radians
+      la2 = lat2 * Math.PI / 180,
+      lo1 = lon1 * Math.PI / 180,
+      lo2 = lon2 * Math.PI / 180,
+      haversine_phi = Math.pow(Math.sin((la2 - la1) / 2), 2),
+      sin_lon = Math.sin((lo2 - lo1) / 2),
+      h = haversine_phi + Math.cos(la1) * Math.cos(la2) * sin_lon * sin_lon;
+    return 2 * R * Math.asin(Math.sqrt(h));
+  }
+
+  me.onStart = function (timestamp) {
+    me.direction_set = false;
+    me.next_checkpoint = 0;
+    me.takeOff();
+  };
+
+  me.onUpdate = function (timestamp) {
+    if (!me.isReadyToFly()) {
+      return;
+    }
+
+    me.current_position = me.getCurrentPosition();
+    if (CHECKPOINT_LIST.length === 0) {
+      set_checkpoints(me.current_position.latitude,
+                      me.current_position.longitude);
+    }
+
+    if (!me.direction_set) {
+      if (me.next_checkpoint < CHECKPOINT_LIST.length) {
+        me.setTargetCoordinates(
+          CHECKPOINT_LIST[me.next_checkpoint].latitude,
+          CHECKPOINT_LIST[me.next_checkpoint].longitude,
+          ALTITUDE + ALTITUDE * me.id,
+          5
+        );
+        console.log("[DEMO] Going to Checkpoint", me.next_checkpoint);
+      }
+      me.direction_set = true;
+      return;
+    }
+
+    if (me.next_checkpoint < CHECKPOINT_LIST.length) {
+      me.current_position = me.getCurrentPosition();
+      me.distance = distance(
+        me.current_position.latitude,
+        me.current_position.longitude,
+        CHECKPOINT_LIST[me.next_checkpoint].latitude,
+        CHECKPOINT_LIST[me.next_checkpoint].longitude
+      );
+      if (me.distance <= EPSILON) {
+        console.log("[DEMO] Reached Checkpoint %d", me.next_checkpoint);
+        me.next_checkpoint += 1;
+        me.direction_set = false;
+      }
+      return;
+    }
+
+    if (!me.isLanding()) {
+      me.land();
+    }
+    if (me.getCurrentPosition().altitude <= 0) {
+      me.exit(0);
+    }
+  };
+}(console, me));
\ No newline at end of file

software/simulation_generator/generate_simulations.py

+"""Run the web application through remote selenium browsers."""
+
+import argparse
+import itertools
+import logging
+import os
+import traceback
+from datetime import datetime
+from multiprocessing import Queue
+from threading import Thread
+from urllib3 import PoolManager
+from urllib3.exceptions import ProtocolError
+
+from selenium import webdriver
+from selenium.common.exceptions import TimeoutException, WebDriverException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.remote.remote_connection import RemoteConnection
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+SELENIUM_INSTANCE_DICT= {
+#  'https://selenium:PASSWORD@[IPV6]:9443/wd/hub': 1,
+}
+
+TIMEOUT = 30
+NUMBER_OF_DRONES = 1
+GRANULARITY = 5
+GAME_INPUT_DICT = {
+  'simulation_speed': '100000',
+  'number_of_drones': str(NUMBER_OF_DRONES),
+}
+DRONE_INPUT_DICT = {
+  'maxAcceleration': (1, 15),
+  'maxDeceleration': (1, 15),
+  'maxRoll': (1, 30),
+  'speedFactor': (0.5, 2.5),
+  'minApproachSpeed': (0.01, 1),
+  'takeOffFactor': (1, 30),
+  'yawAcceptance': (0.1, 5),
+}
+
+parser = argparse.ArgumentParser(
+  prog='run_web_application',
+  description='use selenium to run a web application and collect the logged results',
+)
+parser.add_argument(
+  '-d', '--directory', type=str, default='.', help='location to store results')
+parser.add_argument('-m', '--map', type=str, help='map location')
+parser.add_argument('-s', '--script', type=str, help='flight script location')
+parser.add_argument(
+  '-u', '--url', type=str, default='https://flock.app.officejs.com/', help='simulator url')
+args = parser.parse_args()
+os.makedirs(args.directory, exist_ok=True)
+
+def createDriver(capabilities, selenium_url):
+  # XXX we are using a self signed certificate, but selenium 3.141.0 does
+  # not expose API to ignore certificate verification
+  executor = RemoteConnection(selenium_url, keep_alive=True)
+  executor._conn = PoolManager(cert_reqs='CERT_NONE', ca_certs=None)
+  return webdriver.Remote(
+    command_executor=executor,
+    desired_capabilities=capabilities,
+    keep_alive=True,
+  )
+
+def chromeCapabilities():
+  options = webdriver.ChromeOptions()
+  options.headless = True
+  capabilities = options.to_capabilities()
+  capabilities['version'] = '120.0.6099.109'
+  return capabilities
+
+def downloadLogs(driver, combination, result_dir):
+  """Collect results from application run"""
+  for i in range(NUMBER_OF_DRONES):
+    id_s = "log_result_" + str(i)
+    result_log = driver.find_element(
+      By.XPATH, '//textarea[@id="log_result_%s"]' % i)
+    with open(
+      os.path.join(
+        result_dir,
+        'simulation_log_%s.log' % '_'.join(('%s_%s' % (k, v) for k, v in zip(DRONE_INPUT_DICT, combination)),
+      )
+    ), 'w') as f:
+      f.write(result_log.get_attribute('value'))
+
+def write_into_CodeMirror(driver, text):
+  WebDriverWait(driver, TIMEOUT).until(
+    EC.presence_of_element_located((By.XPATH, '//iframe'))
+  )
+  frame = driver.find_element(By.XPATH, '//iframe')
+  driver.switch_to.frame(frame)
+  driver.execute_script(
+    'return document.getElementsByClassName("CodeMirror")[0].CodeMirror.setValue("' + text.replace("\n", "\\n").replace('"', "'") + '")'
+  )
+  driver.switch_to.default_content()
+
+def selenium_task(driver, app_url, combination, data_queue, error_logger):
+  try:
+    try:
+      driver.set_window_size(1000, 4080)
+      driver.get(app_url)
+
+      WebDriverWait(driver, TIMEOUT).until(
+        EC.presence_of_element_located((By.XPATH, '//canvas'))
+      )
+
+      if args.map is not None:
+        driver.find_element(By.XPATH, '//button[text()="Map"]').click()
+        with open(args.map, 'r') as map_file:
+          write_into_CodeMirror(driver, map_file.read())
+
+      driver.find_element(By.XPATH, '//button[text()="Parameters"]').click()
+      WebDriverWait(driver, TIMEOUT).until(
+        EC.presence_of_element_located((By.ID, 'simulation_speed'))
+      )
+
+      for key, value in GAME_INPUT_DICT.items():
+        driver.execute_script(
+          'arguments[0].value = ' + value,
+          driver.find_element(By.ID, key),
+        )
+      for i, key in enumerate(DRONE_INPUT_DICT):
+        driver.execute_script(
+          'arguments[0].value = ' + str(combination[i]),
+          driver.find_element(By.ID, key),
+        )
+
+      if args.script is not None:
+        driver.find_element(By.XPATH, '//button[text()="Script"]').click()
+        with open(args.script, 'r') as script:
+          write_into_CodeMirror(driver, script.read())
+
+      driver.find_element(By.XPATH, '//button[text()="Run"]').click()
+      WebDriverWait(driver, TIMEOUT).until(
+        EC.presence_of_element_located((By.XPATH, '//div//a[contains(text(), "Download Simulation LOG")]'))
+      )
+      downloadLogs(driver, combination, args.directory)
+      if not data_queue.empty():
+        driver.refresh()
+    except (TimeoutException, WebDriverException):
+      driver.refresh()
+      raise
+    except ProtocolError:
+      driver = createDriver(dict(DesiredCapabilities.CHROME, version='120'), driver.command_executor._url)
+      raise
+  except Exception as e:
+    error_logger.error(driver.command_executor._url)
+    error_logger.error(e)
+    error_logger.error(traceback.format_exc())
+    data_queue.put(combination)
+
+def selenium_queue_listener(data_queue, worker_queue, selenium_worker_list, app_url, combination_nb, error_logger):
+  while not data_queue.empty():
+    current_data = data_queue.get()
+    worker_id = worker_queue.get()
+    selenium_task(selenium_worker_list[worker_id], app_url, current_data, data_queue, error_logger)
+    worker_queue.put(worker_id)
+    update_progress((combination_nb - data_queue.qsize()) / combination_nb)
+  return
+
+def values_in_range(start, end, n):
+  if n == 1:
+    return [start]
+  d = (end - start) / (n - 1)
+  return [start + i*d for i in range(n)]
+
+def update_progress(progress):
+  bar_length = 20
+  if isinstance(progress, int):
+    progress = float(progress)
+  if not isinstance(progress, float):
+    progress = 0
+  if progress < 0:
+    progress = 0
+  if progress >= 1:
+    progress = 1
+
+  block = int(round(bar_length * progress))
+  text = "Progress: [{0}] {1:.1f}%".format( "#" * block + "-" * (bar_length - block), progress * 100)
+  print(text, end='\r')
+
+##### MAIN #####
+worker_queue = Queue()
+selenium_worker_list = []
+if not SELENIUM_INSTANCE_DICT:
+  raise ValueError('SELENIUM_INSTANCE_DICT is empty')
+print('Creating web drivers ' + datetime.now().strftime('%d/%m/%Y %H:%M:%S'))
+for i, url in enumerate(
+  (url for url, slot_quantity in SELENIUM_INSTANCE_DICT.items() for _ in range(slot_quantity))
+):
+  selenium_worker_list.append(createDriver(chromeOptions(), url))
+  worker_queue.put(i)
+
+drone_input_value_tuple = (values_in_range(start, end, GRANULARITY) for start, end in DRONE_INPUT_DICT.values())
+combination_list = [combination for combination in itertools.product(*drone_input_value_tuple)]
+combination_nb = len(combination_list)
+print('Total combinations: ' + str(combination_nb))
+selenium_data_queue = Queue()
+for d in combination_list:
+  selenium_data_queue.put(d)
+
+creation_time = datetime.now()
+print('Starting selenium background processes ' + creation_time.strftime('%d/%m/%Y %H:%M:%S'))
+error_logger = logging.getLogger(__name__)
+logging.basicConfig(filename='error.log')
+selenium_processes = [
+  Thread(target=selenium_queue_listener,
+         args=(selenium_data_queue, worker_queue, selenium_worker_list, args.url, combination_nb, error_logger))
+  for _ in range(len(selenium_worker_list))
+]
+update_progress(0)
+for p in selenium_processes:
+  p.daemon = True
+  p.start()
+for p in selenium_processes:
+  p.join()
+elapsed_time = datetime.now() - creation_time
+print('\nTotal time for runs: %s seconds.' % elapsed_time.seconds)
+
+for b in selenium_worker_list:
+  try:
+    b.quit()
+  except WebDriverException as e:
+    error_logger.error(e)
+    error_logger.error(traceback.format_exc())

software/simulation_generator/instance-input-schema.json

+{
+  "$schema": "https://json-schema.org/draft-06/schema#",
+  "type": "object",
+  "description": "Parameters to instantiate Selenium client",
+  "additionalProperties": false,
+  "properties": {
+    "flight-log-dir": {
+      "title": "Fligh log directory",
+      "description": "Directory where Fluentd will find the flight logs to send to Wendelin.",
+      "type": "string",
+      "default": "BUILDOUT_DIRECTORY/etc/flight_log_dir"
+    },
+    "simulation-log-dir": {
+      "title": "Simulation log directory",
+      "description": "Directory where Fluentd will find the simulation logs to send to Wendelin.",
+      "type": "string",
+      "default": "BUILDOUT_DIRECTORY/etc/flight_log_dir/simulation_log_dir"
+    },
+    "wendelin-password": {
+      "title": "Wendelin passwor",
+      "description": "Password for Wendelin instance",
+      "type": "string",
+      "default": ""
+    },
+    "wendelin-url": {
+      "title": "Wendelin url",
+      "description": "Url of the wendelin instance to send logs to",
+      "type": "string",
+      "default": ""
+    },
+    "wendelin-user": {
+      "title": "Wendelin user",
+      "description": "Username for Wendelin instance",
+      "type": "string",
+      "default": ""
+    }
+  }
+}

software/simulation_generator/instance-output-schema.json

+{
+  "$schema": "https://json-schema.org/draft-06/schema#",
+  "description": "Values returned by Selenium client instantiation",
+  "additionalProperties": false,
+  "properties": {
+    "python-with-eggs": {
+      "description": "Path to a Python interpreter with selenium egg installed",
+      "type": "array"
+    }
+  },
+  "type": "object"
+}

software/simulation_generator/instance.cfg

+[buildout]
+extends =
+  ${instance-fluentd:output}
+parts +=
+  publish-connection-information
+
+[dynamic-template-fluentd]
+context =
+  key fluentd_agent_conf selenium-client-conf:output
+  key port_list fluentd-conf:port-list
+  section directory directory
+
+[selenium-client-conf]
+recipe = slapos.recipe.template:jinja2
+output = $${buildout:directory}/selenium-conf.cfg
+context =
+  key slapparameter_dict slap-configuration:configuration
+  key simulation_log_dir directory:simulation-log-dir
+  key flight_log_dir directory:flight-log-dir
+  key tmp_dir directory:tmp
+inline =
+  {% set wendelin_url = slapparameter_dict.get('wendelin-url', '') -%}
+  {% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%}
+  {% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%}
+  {% set simulation_log_dir = slapparameter_dict.get('simulation-log-dir', simulation_log_dir) -%}
+  {% set flight_log_dir = slapparameter_dict.get('flight-log-dir', flight_log_dir) -%}
+  <source>
+    @type tail
+    read_from_head true
+    path {{ simulation_log_dir }}/*.log
+    pos_file {{ tmp_dir }}/drone_simulation.pos
+    tag drone_simulation.sample-drone-raw-data
+    format none
+    path_key filepath
+  </source>
+
+  <match drone_simulation.sample-drone-raw-data>
+    @type wendelin
+    streamtool_uri {{ wendelin_url }}/erp5/portal_ingestion_policies/default
+    user {{ wendelin_user }}
+    password {{ wendelin_password }}
+    <buffer>
+      @type memory
+      flush_mode interval
+      flush_interval 1s
+    </buffer>
+  </match>
+
+  <source>
+    @type tail
+    read_from_head true
+    path {{ flight_log_dir }}/*.log
+    pos_file {{ tmp_dir }}/drone_real.pos
+    tag drone_real.bouncy_flight_mavsdk_raw_data
+    format none
+    path_key filepath
+  </source>
+
+  <match drone_real.bouncy_flight_mavsdk_raw_data>
+    @type wendelin
+    streamtool_uri {{ wendelin_url }}/erp5/portal_ingestion_policies/default
+    user {{ wendelin_user }}
+    password {{ wendelin_password }}
+    <buffer>
+      @type memory
+      flush_mode interval
+      flush_interval 1s
+    </buffer>
+  </match>
+
+[directory]
+tmp = $${:home}/tmp
+simulation-log-dir = $${:etc}/simulation_log_dir
+flight-log-dir = $${:etc}/flight_log_dir
+
+[publish-connection-information]
+recipe = slapos.cookbook:publish
+python-with-eggs = ${python-with-eggs:executable}

software/simulation_generator/map.json

+{
+  "height": 100,
+  "start_AMSL": 15,
+  "min_lat": 45.12,
+  "max_lat": 45.15,
+  "min_lon": 5.67,
+  "max_lon": 5.7,
+  "flag_list": [],
+  "obstacle_list": [],
+  "enemy_list": [],
+  "initial_position": {
+      "latitude": 45.1333,
+      "longitude": 5.6816,
+      "altitude": 15
+  }
+}

software/simulation_generator/software.cfg

+[buildout]
+extends =
+  buildout.hash.cfg
+  ../../component/selenium/buildout.cfg
+  ../fluentd/software.cfg
+
+parts +=
+  python-with-eggs
+
+[python-with-eggs]
+recipe = zc.recipe.egg
+interpreter = ${:_buildout_section_name_}
+scripts = ${:interpreter}
+executable = ${buildout:bin-directory}/${:interpreter}
+eggs = ${selenium:egg}
+
+[instance-profile]
+url = ${:_profile_base_location_}/${instance-selenium:filename}
+md5sum = ${instance-selenium:md5sum}
+
+[instance-fluentd]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/../fluentd/instance.cfg
+output = ${buildout:directory}/template-fluentd-profile.cfg

software/simulation_generator/software.cfg.json

+{
+  "name": "Selenium client",
+  "description": "Selenium client",
+  "serialisation": "xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "description": "Selenium client",
+      "request": "instance-input-schema.json",
+      "response": "instance-output-schema.json",
+      "index": 0
+    }
+  }
+}