software/selenium_client: Add selenium_client SR

Provide :
* a Python with selenium egg installed
* a Fluentd instance
* instance parameters to send generated logs to a Wendelin instance through Fluentd

software/selenium_client/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+[instance-selenium]
+filename = instance.cfg
+md5sum = e9414172af42c1e8a483286716ed7370

software/selenium_client/example.py

+"""Run the web application through remote selenium browsers."""
+
+import argparse
+import itertools
+import logging
+import os
+import traceback
+from datetime import datetime
+from multiprocessing import Queue
+from threading import Thread
+from urllib3 import PoolManager
+from urllib3.exceptions import ProtocolError
+
+from selenium import webdriver
+from selenium.common.exceptions import TimeoutException, WebDriverException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.remote.remote_connection import RemoteConnection
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+SELENIUM_INSTANCE_DICT= {
+#  'https://selenium:PASSWORD@[IPV6]:9443/wd/hub': 1,
+}
+
+TIMEOUT = 30
+NUMBER_OF_DRONES = 1
+GRANULARITY = 4
+GAME_INPUT_DICT = {
+  'simulation_speed': '100000',
+  'number_of_drones': str(NUMBER_OF_DRONES),
+}
+DRONE_INPUT_DICT = {
+  "maxDeceleration": (1, 15),
+  "speedFactor": (0.5, 2.5)
+}
+
+parser = argparse.ArgumentParser(
+  prog='run_web_application',
+  description='use selenium to run a web application and collect the logged results',
+)
+parser.add_argument(
+  '-d', '--directory', type=str, default='.', help='location to store results')
+parser.add_argument(
+  '-u', '--url', type=str, default='https://dronesimulator.app.officejs.com/', help='simulator url')
+args = parser.parse_args()
+os.makedirs(args.directory, exist_ok=True)
+
+def createDriver(options, selenium_url):
+  # XXX we are using a self signed certificate, but selenium 3.141.0 does
+  # not expose API to ignore certificate verification
+  executor = RemoteConnection(selenium_url, keep_alive=True)
+  executor._conn = PoolManager(cert_reqs='CERT_NONE', ca_certs=None)
+  return webdriver.Remote(
+    command_executor=executor,
+    desired_capabilities=options.to_capabilities(),
+    keep_alive=True,
+  )
+
+def chromeOptions():
+  options = webdriver.ChromeOptions()
+  options.add_argument('--headless=new')
+  options.browser_version = '120'
+  return options
+
+def downloadLogs(driver, combination, result_dir):
+  """Collect results from application run"""
+  for i in range(NUMBER_OF_DRONES):
+    id_s = "log_result_" + str(i)
+    result_log = driver.find_element(
+      By.XPATH, '//textarea[@id="log_result_%s"]' % i)
+    with open(
+      os.path.join(
+        result_dir,
+        'simulation_log_%s.log' % '_'.join(('%s_%s' % (k, v) for k, v in zip(DRONE_INPUT_DICT, combination)),
+      )
+    ), 'w') as f:
+      f.write(result_log.get_attribute('value'))
+
+def selenium_task(driver, app_url, combination, data_queue, error_logger):
+  try:
+    try:
+      driver.set_window_size(1000, 4080)
+      driver.get(app_url)
+
+      WebDriverWait(driver, TIMEOUT).until(
+        EC.presence_of_element_located((By.XPATH, '//iframe'))
+      )
+
+      for key, value in GAME_INPUT_DICT.items():
+        driver.execute_script(
+          'arguments[0].value = ' + value,
+          driver.find_element(By.ID, key),
+        )
+      for i, key in enumerate(DRONE_INPUT_DICT):
+        driver.execute_script(
+          'arguments[0].value = ' + str(combination[i]),
+          driver.find_element(By.ID, key),
+        )
+
+      driver.find_element(
+        By.XPATH, '//input[@type="submit" and @name="action_run"]').click()
+      WebDriverWait(driver, TIMEOUT).until(
+        EC.presence_of_element_located((By.XPATH, '//div[@class="container"]//a[contains(text(), "Download Simulation LOG")]'))
+      )
+      downloadLogs(driver, combination, args.directory)
+      if not data_queue.empty():
+        driver.refresh()
+    except (TimeoutException, WebDriverException):
+      driver.refresh()
+      raise
+    except ProtocolError:
+      driver = createDriver(chromeOptions(), driver.command_executor._url)
+      raise
+  except Exception as e:
+    error_logger.error(driver.command_executor._url)
+    error_logger.error(e)
+    error_logger.error(traceback.format_exc())
+    data_queue.put(combination)
+
+def selenium_queue_listener(data_queue, worker_queue, selenium_worker_list, app_url, combination_nb, error_logger):
+  while not data_queue.empty():
+    current_data = data_queue.get()
+    worker_id = worker_queue.get()
+    selenium_task(selenium_worker_list[worker_id], app_url, current_data, data_queue, error_logger)
+    worker_queue.put(worker_id)
+    update_progress((combination_nb - data_queue.qsize()) / combination_nb)
+  return
+
+def values_in_range(start, end, n):
+  if n == 1:
+    return [start]
+  d = (end - start) / (n - 1)
+  return [start + i*d for i in range(n)]
+
+def update_progress(progress):
+  bar_length = 20
+  if isinstance(progress, int):
+    progress = float(progress)
+  if not isinstance(progress, float):
+    progress = 0
+  if progress < 0:
+    progress = 0
+  if progress >= 1:
+    progress = 1
+
+  block = int(round(bar_length * progress))
+  text = "Progress: [{0}] {1:.1f}%".format( "#" * block + "-" * (bar_length - block), progress * 100)
+  print(text, end='\r')
+
+##### MAIN #####
+worker_queue = Queue()
+selenium_worker_list = []
+if not SELENIUM_INSTANCE_DICT:
+  raise ValueError('SELENIUM_INSTANCE_DICT is empty')
+print('Creating web drivers ' + datetime.now().strftime('%d/%m/%Y %H:%M:%S'))
+for i, url in enumerate(
+  (url for url, slot_quantity in SELENIUM_INSTANCE_DICT.items() for _ in range(slot_quantity))
+):
+  selenium_worker_list.append(createDriver(chromeOptions(), url))
+  worker_queue.put(i)
+
+drone_input_value_tuple = (values_in_range(start, end, GRANULARITY) for start, end in DRONE_INPUT_DICT.values())
+combination_list = [combination for combination in itertools.product(*drone_input_value_tuple)]
+combination_nb = len(combination_list)
+print('Total combinations: ' + str(combination_nb))
+selenium_data_queue = Queue()
+for d in combination_list:
+  selenium_data_queue.put(d)
+
+creation_time = datetime.now()
+print('Starting selenium background processes ' + creation_time.strftime('%d/%m/%Y %H:%M:%S'))
+error_logger = logging.getLogger(__name__)
+logging.basicConfig(filename='error.log')
+selenium_processes = [
+  Thread(target=selenium_queue_listener,
+         args=(selenium_data_queue, worker_queue, selenium_worker_list, args.url, combination_nb, error_logger))
+  for _ in range(len(selenium_worker_list))
+]
+update_progress(0)
+for p in selenium_processes:
+  p.daemon = True
+  p.start()
+for p in selenium_processes:
+  p.join()
+elapsed_time = datetime.now() - creation_time
+print('\nTotal time for runs: %s seconds.' % elapsed_time.seconds)
+
+for b in selenium_worker_list:
+  try:
+    b.quit()
+  except WebDriverException as e:
+    error_logger.error(e)
+    error_logger.error(traceback.format_exc())

software/selenium_client/instance-input-schema.json

+{
+  "$schema": "https://json-schema.org/draft-06/schema#",
+  "type": "object",
+  "description": "Parameters to instantiate Selenium client",
+  "additionalProperties": false,
+  "properties": {
+    "flight-log-dir": {
+      "title": "Fligh log directory",
+      "description": "Directory where Fluentd will find the flight logs to send to Wendelin.",
+      "type": "string",
+      "default": "BUILDOUT_DIRECTORY/etc/flight_log_dir"
+    },
+    "simulation-log-dir": {
+      "title": "Simulation log directory",
+      "description": "Directory where Fluentd will find the simulation logs to send to Wendelin.",
+      "type": "string",
+      "default": "BUILDOUT_DIRECTORY/etc/flight_log_dir/simulation_log_dir"
+    },
+    "wendelin-password": {
+      "title": "Wendelin passwor",
+      "description": "Password for Wendelin instance",
+      "type": "string",
+      "default": ""
+    },
+    "wendelin-url": {
+      "title": "Wendelin url",
+      "description": "Url of the wendelin instance to send logs to",
+      "type": "string",
+      "default": ""
+    },
+    "wendelin-user": {
+      "title": "Wendelin user",
+      "description": "Username for Wendelin instance",
+      "type": "string",
+      "default": ""
+    }
+  }
+}

software/selenium_client/instance-output-schema.json

+{
+  "$schema": "https://json-schema.org/draft-06/schema#",
+  "description": "Values returned by Selenium client instantiation",
+  "additionalProperties": false,
+  "properties": {
+    "python-with-eggs": {
+      "description": "Path to a Python interpreter with selenium egg installed",
+      "type": "array"
+    }
+  },
+  "type": "object"
+}

software/selenium_client/instance.cfg

+[buildout]
+extends =
+  ${instance-fluentd:output}
+parts +=
+  publish-connection-information
+
+[dynamic-template-fluentd]
+context =
+  key fluentd_agent_conf selenium-client-conf:output
+  key port_list fluentd-conf:port-list
+  section directory directory
+
+[selenium-client-conf]
+recipe = slapos.recipe.template:jinja2
+output = $${buildout:directory}/selenium-conf.cfg
+context =
+  key slapparameter_dict slap-configuration:configuration
+  key simulation_log_dir directory:simulation-log-dir
+  key flight_log_dir directory:flight-log-dir
+inline =
+  {% set wendelin_url = slapparameter_dict.get('wendelin-url', '') -%}
+  {% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%}
+  {% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%}
+  {% set simulation_log_dir = slapparameter_dict.get('simulation-log-dir', simulation_log_dir) -%}
+  {% set flight_log_dir = slapparameter_dict.get('flight-log-dir', flight_log_dir) -%}
+  <source>
+    @type tail
+    read_from_head true
+    path {{ simulation_log_dir }}/*.log
+    tag drone_simulation.sample-drone-raw-data
+    format none
+    path_key filepath
+  </source>
+
+  <match drone_simulation.sample-drone-raw-data>
+    @type wendelin
+    streamtool_uri {{ wendelin_url }}/erp5/portal_ingestion_policies/default
+    user {{ wendelin_user }}
+    password {{ wendelin_password }}
+    <buffer>
+      @type memory
+      flush_mode interval
+      flush_interval 1s
+    </buffer>
+  </match>
+
+  <source>
+    @type tail
+    read_from_head true
+    path {{ flight_log_dir }}/*.log
+    tag drone_real.bouncy_flight_mavsdk_raw_data
+    format none
+    path_key filepath
+  </source>
+
+  <match drone_real.bouncy_flight_mavsdk_raw_data>
+    @type wendelin
+    streamtool_uri {{ wendelin_url }}/erp5/portal_ingestion_policies/default
+    user {{ wendelin_user }}
+    password {{ wendelin_password }}
+    <buffer>
+      @type memory
+      flush_mode interval
+      flush_interval 1s
+    </buffer>
+  </match>
+
+[directory]
+simulation-log-dir = $${:etc}/simulation_log_dir
+flight-log-dir = $${:etc}/flight_log_dir
+
+[publish-connection-information]
+recipe = slapos.cookbook:publish
+python-with-eggs = ${python-with-eggs:executable}

software/selenium_client/software.cfg

+[buildout]
+extends =
+  buildout.hash.cfg
+  ../../component/selenium/buildout.cfg
+  ../fluentd/software.cfg
+
+parts +=
+  python-with-eggs
+
+[python-with-eggs]
+recipe = zc.recipe.egg
+interpreter = ${:_buildout_section_name_}
+scripts = ${:interpreter}
+executable = ${buildout:bin-directory}/${:interpreter}
+eggs = ${selenium:egg}
+
+[instance-profile]
+url = ${:_profile_base_location_}/${instance-selenium:filename}
+md5sum = ${instance-selenium:md5sum}
+
+[instance-fluentd]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/../fluentd/instance.cfg
+output = ${buildout:directory}/template-fluentd-profile.cfg

software/selenium_client/software.cfg.json

+{
+  "name": "Selenium client",
+  "description": "Selenium client",
+  "serialisation": "xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "description": "Selenium client",
+      "request": "instance-input-schema.json",
+      "response": "instance-output-schema.json",
+      "index": 0
+    }
+  }
+}