testsuite/deploy-test: Deploy script testing

Software release for testing deploy scripts and Ansible playbooks
provided in slapos.package repository.

It is used by Test Suite and ERP5TestNode and runs tests configured in
Test Suite against VM built with deploy script.

software/erp5testnode/testsuite/deploy-test/README.rst

+deply-test
+==========
+
+Introduction
+------------
+
+This is software release to run tests on VMs, but without direct access (like
+ssh) to the VM.
+
+It is supposed to be used as backend for ERP5TestNode.
+
+Characteristics and limitations:
+
+ * partitions share the same user (as they access files directly)
+ * this SR will be installed and instantiated only from local file system
+   (like git clone)

software/erp5testnode/testsuite/deploy-test/deploy-script-controller

+#!/bin/bash
+
+# Script for controlling deploy script
+#
+# Assumption: deployment script returning with code 0
+#             run correctly and everything is done
+#
+# This script is run inside of KVM, by passing its
+# URL with bootstrap-script-url
+# The script configuration happens by passing
+# it in text form with data-to-vm
+#
+# Format of data-to-vm is shell script:
+#  URL=<url>\nWAITTIME=<seconds>\nTRIES=<amount>
+#
+# Expected values in configuration:
+#  * URL - the url of the script to download and test
+#  * WAITTIME - waiting time, before next try
+#  * TRIES - amount of tries
+
+# Possible TODOs:
+#  * post results on each try
+#  * use function + trap to assure posting on exit
+
+LOG_FILE=/var/log/test-script-deployment.log
+
+wget -O /tmp/test-script.cfg.$$ -q http://10.0.2.100/data
+
+source /tmp/test-script.cfg.$$
+
+if [ -z "$LOG_FILE" ] ; then
+  echo "Output log file is missing"
+  exit 1
+fi
+
+if [ -z "$URL" ] ; then
+  echo "URL is missing" >> $LOG_FILE 2>&1
+  exit 1
+fi
+
+if [ -z "$WAITTIME" ] ; then
+  echo "WAITTIME missing" >> $LOG_FILE 2>&1
+  exit 1
+fi
+
+if [ -z "$TRIES" ] ; then
+  echo "TRIES missing" >> $LOG_FILE 2>&1
+  exit 1
+fi
+
+DEPLOYMENT_SCRIPT=/tmp/test-script-deployment.bash.$$
+wget -O $DEPLOYMENT_SCRIPT -q $URL
+
+if [[ ! -s "$DEPLOYMENT_SCRIPT" ]] ; then
+  echo "exit 1" > $DEPLOYMENT_SCRIPT
+fi
+
+function upload ()
+{
+  try=$1
+  LOG_FILE=$2
+  t=`date '+%Y%m%d%H%S'`
+  mv $LOG_FILE ${LOG_FILE}.$t
+  # just to be sure flush all disk operations before uploading
+  flush
+  curl -q -X POST --data-urlencode "path=test-script-result/log-file.log.$t" --data-urlencode "content@${LOG_FILE}.$t" http://10.0.2.100/
+}
+
+try=1
+while true; do
+  echo "$0: Try $try. Running '/bin/bash $DEPLOYMENT_SCRIPT'" >> $LOG_FILE 2>&1
+  export TEST_YML_PATH
+  /bin/bash $DEPLOYMENT_SCRIPT >> $LOG_FILE 2>&1
+  result=$?
+  if [ $result == 0 ] ; then
+    echo "$0: Try $try. Script executed successfully." >> $LOG_FILE 2>&1
+    upload $try $LOG_FILE
+    break
+  fi
+  if (( try > TRIES )) ; then
+    echo "$0: Try $try. Amount of tries $TRIES exceeded, giving up." >> $LOG_FILE 2>&1
+    upload $try $LOG_FILE
+    break
+  fi
+  # wait WAITTIME before checking the state
+  echo "$0: Try $try. Sleeping $WAITTIME before retry." >> $LOG_FILE 2>&1
+  upload $try $LOG_FILE
+  sleep $WAITTIME
+  ((try++))
+done
+
+exit $result

software/erp5testnode/testsuite/deploy-test/instance-deploy-test.cfg.jinja2

+{# Get test type, default to script-from-url, as defined in instance-input-schema.json -#}
+{% set test_type = slapparameter_dict.get('test-type', 'script-from-url') -%}
+{# Choose parameters according to test type -#}
+{% if test_type == 'script-from-url' -%}
+{% set script_url = slapparameter_dict.get('script-to-test-url') -%}
+{% set test_yml_path = '/not/required' -%}
+{% elif test_type == 'cloned-playbook' -%}
+{% set script_url = 'http://10.0.2.100/standalone-local-playbook' -%}
+{% set test_yml_path = slapparameter_dict.get('yml-path-to-test') -%}
+{% else -%}
+The test_type = "{{ test_type }}" is unsupported.
+{% endif -%}
+[buildout]
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+parts =
+  request-kvm
+
+[request-kvm]
+<= slap-connection
+
+recipe = slapos.cookbook:request
+software-url = ${slap-connection:software-release-url}
+software-type = kvm
+name = test-kvm
+sla-computer_guid = ${slap-connection:computer-id}
+
+# Tested image
+# Passed by request
+config-virtual-hard-drive-url = {{ slapparameter_dict.get('image-to-test-url') }}
+config-virtual-hard-drive-md5sum = {{ slapparameter_dict.get('image-to-test-md5sum') }}
+
+# The test script
+config-bootstrap-script-url = {{ in_vm_test_script }}#{{ in_vm_test_script_md5 }}
+
+# Script configuration
+config-data-to-vm =
+  URL={{ script_url }}
+  WAITTIME={{ waittime }}
+  TRIES={{ tries }}
+  TEST_YML_PATH={{ test_yml_path }}
+
+# require HTTP server
+config-enable-http-server = true
+
+# VM options
+config-ram-size = 2048
+config-cpu-count = 2
+
+# await for system to be ready
+return =
+  url
+  backend-url
+  ipv6
+  nat-rule-port-443
+  nat-rule-port-80
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc/
+var = ${:home}/var/
+srv = ${:home}/srv/
+bin = ${:home}/bin/
+tmp = ${:home}/tmp/
+log = ${:var}/log/
+services = ${:etc}/service/
+scripts = ${:etc}/run/

software/erp5testnode/testsuite/deploy-test/instance-input-schema.json

+{
+  "type": "object",
+  "$schema": "http://json-schema.org/draft-04/schema",
+
+  "title": "Input Parameters",
+  "properties": {
+    "image-to-test-url": {
+      "title": "Image To Test (URL)",
+      "description": "Absolute URL to QCOW2 vm-bootstrap compatible image.",
+      "type": "string"
+    },
+    "image-to-test-md5sum": {
+      "title": "MD5 checksum of Image To Test.",
+      "description": "MD5 checksum of QCOW2 vm-bootstrap compatible image.",
+      "type": "string"
+    },
+    "test-relative-directory": {
+      "title": "Relative directory where tests are found.",
+      "description": "The base of this directory is [slapos-package].",
+      "type": "string"
+    },
+    "test-type": {
+      "title": "Type of the test.",
+      "type": "string",
+      "default": "script-from-url",
+      "enum" : ["script-from-url", "cloned-playbook"]
+    },
+    "script-to-test-url": {
+      "title": "Optional URL of script to test, used for test-type=script-from-url.",
+      "description": "Optional URL of script to test, which will be injected into the VM.\nThis parameter will NOT use local git clone of [slapos-package] part, it will test fully provided script.",
+      "type": "string"
+    },
+    "yml-path-to-test": {
+      "title": "Optional YML path to test from the playbook, used for test-type=cloned-playbook.",
+      "description": "Optional YML path to test from the playbook, provided as relative path to playbook root.\nThis parameter will use local git clone of [slapos-package] part, from which playbook will be constructed.",
+      "type": "string"
+    }
+  },
+  "required": ["image-to-test-url", "image-to-test-md5sum", "test-type"]
+}

software/erp5testnode/testsuite/deploy-test/instance.cfg.in

+[buildout]
+extends = ${template:output}
+parts +=
+  runTestSuite
+  copy-test-data
+
+[copy-test-data]
+# Note: This is a hack, as such while requesting KVM it is not possible
+#       to fill correctly http directory
+# Note: This will only work inside of non-user related environment
+#       dest shell variable will evaluate to requested partition's
+#       srv/public directory
+#       The "if" statement makes this script be run only in partition
+#       *without* srv/public - so according to current knowledge, the
+#       one which requests KVM backend for testing
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = $${:command}
+command =
+  if [ ! -d srv/public ] ; then
+    dest=`echo ../*/srv/public`
+    cp ${playbook:output} $dest/playbook.tar.gz &&
+    cp ${standalone-local-playbook:location} $dest/standalone-local-playbook
+  fi
+
+[runTestSuite]
+recipe = slapos.recipe.template:jinja2
+rendered = $${buildout:directory}/bin/$${:_buildout_section_name_}
+template = inline:
+  #!/bin/sh
+  export PATH=${python-with-eggs:location}:$PATH
+  exec ${buildout:bin-directory}/${runTestSuite_py:interpreter} ${:_profile_base_location_}/runTestSuite.py --partition_ipv4 {{ list(partition_ipv4)[0] }} --partition_path $${buildout:directory} --test_reference "{{ slapparameter_dict.get('image-to-test-url') }} {{ slapparameter_dict.get('script-to-test-url')}}" --test_location "${test-location:base}/{{ slapparameter_dict.get('test-relative-directory')}}" "$@"
+mode = 0755
+context =
+    key slapparameter_dict slap-configuration:configuration
+    key partition_ipv4 slap-configuration:ipv4
+
+[switch_softwaretype]
+default = $${:deploy-test}
+deploy-test = $${dynamic-template-deploy-test:rendered}
+
+[dynamic-template-deploy-test]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance-deploy-test.cfg.jinja2
+rendered = $${buildout:directory}/template-deploy-test.cfg
+context =
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    key slapparameter_dict slap-configuration:configuration
+    raw bin_directory ${buildout:bin-directory}
+    raw in_vm_test_script ${deploy-script-controller-script:location}
+    raw in_vm_test_script_md5 ${deploy-script-controller-script:md5sum}
+    raw waittime ${deploy-script-controller-script:waittime}
+    raw tries ${deploy-script-controller-script:tries}
+mode = 0644

software/erp5testnode/testsuite/deploy-test/runTestSuite.py

+from __future__ import print_function
+import argparse
+import os
+import glob
+from time import gmtime, strftime, time, sleep
+from erp5.util import taskdistribution, testsuite
+import logging
+import sys
+import tempfile
+import json
+
+
+SLEEP_TIME = 10
+TRY_AMOUNT = 3600
+
+
+def waitForSite(partition_path):
+  status_dict = {'command': 'file not found'}
+
+  # find test result, wait 10h
+  try_num = 1
+  start = time()
+  result_found = False
+  while 1:
+    finished = False
+    try_info = 'Try %s/%s: ' % (try_num, TRY_AMOUNT)
+    test_result_glob = os.path.join(
+        partition_path,
+        '..',
+        '*',
+        'srv',
+        'public',
+        'test-script-result',
+    )
+    print(try_info + 'Waiting for data in %r.' % (test_result_glob,))
+    result_list = glob.glob(test_result_glob)
+    if len(result_list) > 0:
+      result_path = result_list[0]
+      print(try_info + 'Data directory %r found, looking for results.' % (
+            result_path,))
+      result_file_list = list((
+          os.path.join(dirname, filename)
+          for dirname, dirnames, filenames in os.walk(result_path)
+          for filename in filenames
+      ))
+      if len(result_file_list):
+        print(try_info + 'No result posted, will check next try.')
+      for result_file in result_file_list:
+        print(try_info + 'Data found.')
+        result_found = True
+        result_file = os.path.abspath(result_file)
+        status_dict['command'] = result_file
+        result = open(result_file).read()
+        # remove result, as it is not required anymore
+        os.unlink(result_file)
+        print(try_info + 'Analysis of result %r:' % (result_file,))
+        print(try_info + result)
+        status_dict['stderr'] = 'Last result:\n%s' % (result,)
+        if 'FATAL: all hosts have already failed -- aborting' in result:
+          # failed
+          status_dict.update(
+              success=False
+          )
+          finished = False
+          status_dict['stdout'] = try_info + 'Build not yet successful.'
+          print(try_info + '%r: Found not yet finished run.' % (result_file,))
+        elif "\"msg\": \"[u'Build successful, connect to:', u'" in result:
+          # success
+          status_dict.update(
+              success=True
+          )
+          finished = True
+          print(try_info + '%r: Found finished successful run.' % (
+              result_file,))
+          status_dict['stdout'] = try_info + 'Build successful.'
+          break
+        else:
+          # unknown
+          status_dict.update(
+              success=False
+
+          )
+          status_dict['stdout'] = \
+              try_info + 'Cannot find success nor failure result in the output'
+          print(try_info + '%r: Found unknown run.' % (result_file,))
+          finished = False
+    if finished:
+      break
+    if try_num >= TRY_AMOUNT:
+      msg = try_info + 'Time exceeded, success not found.'
+      print(msg)
+      status_dict.setdefault('stdout', '')
+      status_dict['stdout'] = '\n'.join([status_dict['stdout'], msg])
+      break
+    try_num += 1
+    print(try_info + 'Sleeping for %ss.' % (SLEEP_TIME,))
+    sleep(SLEEP_TIME)
+  if not result_found:
+    status_dict['stdout'] = try_info + 'Test timed out and no result found.'
+    status_dict.update(
+        success=False
+
+    )
+  end = time()
+  status_dict.update(
+      date=strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
+      duration=end - start,
+  )
+  return status_dict
+
+
+def main():
+  logger = logging.getLogger()
+  logger.addHandler(logging.StreamHandler(sys.stdout))
+  logger.setLevel(logging.DEBUG)
+  parser = argparse.ArgumentParser(description='Run a test suite.')
+  parser.add_argument('--test_suite', help='The test suite name')
+  parser.add_argument('--test_suite_title', help='The test suite title')
+  parser.add_argument('--test_node_title', help='The test node title')
+  parser.add_argument('--project_title', help='The project title')
+  parser.add_argument('--revision', help='The revision to test',
+                      default='dummy_revision')
+  parser.add_argument('--node_quantity', type=int,
+                      help='Number of CPUs to use for the VM')
+  parser.add_argument('--master_url',
+                      help='The Url of Master controlling test suites')
+  # SlapOS and deploy test specific
+  parser.add_argument(
+      '--partition_path',
+      help="Path of a partition",
+      default=os.path.abspath(os.getcwd()))
+  parser.add_argument(
+      '--test_reference',
+      help="Reference of the test",
+      default="missing"
+  )
+  parser.add_argument(
+      '--partition_ipv4',
+      help="IPv4 of a partition"
+  )
+  parser.add_argument(
+      '--test_location',
+      help="Location of the tests"
+  )
+
+  args = parser.parse_args()
+
+  revision = args.revision
+  test_suite_title = args.test_suite_title or args.test_suite
+  os.environ['SOURCE_CODE_TO_TEST'] = args.test_location
+  suite = testsuite.EggTestSuite(
+      1, test_suite=args.test_suite, node_quantity=args.node_quantity,
+      revision=revision)
+  access_url_http = None
+  access_url_https = None
+  if args.partition_ipv4:
+    access_url_http = 'http://%s:10080' % (args.partition_ipv4,)
+    access_url_https = 'https://%s:10443' % (args.partition_ipv4,)
+    os.environ['TEST_ACCESS_URL_HTTP'] = access_url_http
+    os.environ['TEST_ACCESS_URL_HTTPS'] = access_url_https
+  tool = taskdistribution.TaskDistributionTool(
+      args.master_url,
+      logger=logger)
+  test_result = tool.createTestResult(
+      revision, suite.getTestList(), args.test_node_title,
+      suite.allow_restart, test_suite_title, args.project_title)
+  if test_result is None:
+    return
+
+  # Create the site
+  status_dict = waitForSite(args.partition_path)
+
+  status_file = tempfile.NamedTemporaryFile()
+  status_file.write(json.dumps(status_dict))
+  status_file.flush()
+  os.fsync(status_file.fileno())
+  os.environ['TEST_SITE_STATUS_JSON'] = status_file.name
+
+  assert revision == test_result.revision, (revision, test_result.revision)
+  while suite.acquire():
+    test = test_result.start(suite.running.keys())
+    if test is not None:
+      suite.start(test.name, lambda status_dict,
+                  __test=test: __test.stop(**status_dict))
+    elif not suite.running:
+      break
+  return
+
+if __name__ == "__main__":
+    main()

software/erp5testnode/testsuite/deploy-test/software.cfg

+[buildout]
+extends = ../../../kvm/software.cfg
+
+parts =
+  eggs
+  template-deploy-test
+  runTestSuite_py
+  playbook
+  erp5.util-dev
+
+[runTestSuite_py]
+recipe = zc.recipe.egg
+eggs =
+  ${erp5.util-dev:egg}
+interpreter = ${:_buildout_section_name_}
+
+[python-with-eggs]
+recipe = plone.recipe.command
+
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+stop-on-error = true
+command =
+  rm -fr ${:location} &&
+  mkdir -p ${:location} &&
+  ln -s ${buildout:bin-directory}/pythonwitheggs ${:location}/python &&
+  ln -s ${buildout:bin-directory}/pythonwitheggs ${:location}/python2.7
+update-command = ${:command}
+
+[playbook]
+recipe = plone.recipe.command
+
+stop-on-error = true
+environment = export PATH=${tar:location}/bin:${gzip:location}/bin:$PATH
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+output = ${:location}/playbook.tar.gz
+command =
+  ${:environment}
+  rm -fr ${:location}
+  mkdir -p ${:location}
+  cd ${slapos-package:location}/playbook
+  tar czf ${:output} .
+update-command = ${:command}
+
+[erp5.util-repository]
+recipe = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/erp5.git
+branch = taskdistribution-xmlrpc-binary
+git-executable = ${git:location}/bin/git
+develop = true
+
+[erp5.util-dev]
+recipe = zc.recipe.egg:develop
+egg = erp5.util
+setup = ${erp5.util-repository:location}
+
+[test-location]
+base = ${slapos-package:location}
+
+[slapos-package]
+recipe = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/slapos.package.git
+branch = cdn-test
+git-executable = ${git:location}/bin/git
+
+[template]
+output = ${buildout:directory}/template-original.kvm.cfg
+
+[deploy-script-controller-script]
+filename = deploy-script-controller
+location = ${:_profile_base_location_}/${:filename}
+md5sum = d2b92f45257a52e5a7ff5c311d46d4ae
+# configuration
+waittime = 360
+tries = 80
+
+[standalone-local-playbook]
+filename = standalone-local-playbook
+location = ${:_profile_base_location_}/${:filename}
+
+[template-deploy-test]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg.in
+output = ${buildout:directory}/template.cfg
+mode = 0644
+
+[versions]
+# erp5.util is used from a given branch
+erp5.util =

software/erp5testnode/testsuite/deploy-test/software.cfg.json

+{
+    "name": "Deploy Test",
+    "description": "Deploy Testing software release",
+    "serialisation": "xml",
+    "software-type": {
+        "default": {
+            "title": "Default",
+            "description": "Standard ERP5TestNode hooked",
+            "request": "instance-input-schema.json",
+            "index": 0
+        }
+    }
+}