software/end-to-end-testing: Write minimal e2e PoC

Use runTestSuite interface to launch tests that request instance on
actual SlapOS cloud, using certificate passed as parameter.

Parameters are:
{
  'client.crt': <crt>,
  'client.key': <key>
  'master-url': <url>
}

Such a json of parameters can be generated from existing SlapOS client
configuration using the `generate_parameters` script.

The test currently requests a KVM and examines the connection dict as
a demo.

software/end-to-end-testing/buildout.hash.cfg

+[instance.cfg]
+filename = instance.cfg.in
+md5sum = 562e123cefa9e39cbc78300e4643f7b3
+
+[runTestSuite.in]
+filename = runTestSuite.in
+md5sum = 524531d759d4e517a993246b4e3f6a27

software/end-to-end-testing/generate_parameters.py

+import argparse
+import configparser
+import json
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--cfg', required=True)
+  parser.add_argument('-o', '--output', required=True)
+  args = parser.parse_args()
+
+  configp = configparser.ConfigParser()
+  configp.read(args.cfg)
+
+  with open(configp.get('slapconsole', 'cert_file')) as f:
+    crt = f.read()
+
+  with open(configp.get('slapconsole', 'key_file')) as f:
+    key = f.read()
+
+  url = configp.get('slapos', 'master_url')
+
+  with open(args.output, 'w') as f:
+    json.dump({
+      'client.crt': crt,
+      'client.key': key,
+      'master-url': url
+    }, f, indent=2)
+
+
+if __name__ == '__main__':
+  main()

software/end-to-end-testing/instance.cfg.in

+[buildout]
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+
+parts =
+  runTestSuite
+  client.crt
+  client.key
+  slapos-client.cfg
+
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = $${buildout:directory}
+bin = $${buildout:directory}/bin
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+cfg = $${buildout:directory}/.slapos
+tmp = $${buildout:directory}/tmp
+nxdtest = $${:var}/nxdtest
+
+
+[runTestSuite]
+recipe = slapos.recipe.template
+output = $${directory:bin}/runTestSuite
+url = ${runTestSuite.in:target}
+python_for_test = ${python_for_test:executable}
+
+
+[client.crt]
+recipe = slapos.recipe.template
+output = $${directory:cfg}/client.crt
+inline = $${slap-configuration:configuration.client.crt}
+
+
+[client.key]
+recipe = slapos.recipe.template
+output = $${directory:cfg}/client.key
+inline = $${slap-configuration:configuration.client.key}
+
+
+[slapos-client.cfg]
+recipe = slapos.recipe.template
+output = $${directory:cfg}/slapos-client.cfg
+inline =
+  [slapos]
+  master_url = $${slap-configuration:configuration.master-url}
+
+  [slapconsole]
+  cert_file = $${client.crt:output}
+  key_file = $${client.key:output}

software/end-to-end-testing/runTestSuite.in

+#!${:python_for_test}
+
+import argparse
+import configparser
+import importlib
+import json
+import logging
+import time
+import unittest
+
+import erp5.util.taskdistribution
+
+import slapos.client
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--test_suite')
+  parser.add_argument('--test_suite_title')
+  parser.add_argument('--test_node_title')
+  parser.add_argument('--project_title')
+  parser.add_argument('--revision')
+  parser.add_argument('--master_url')
+  args, _ = parser.parse_known_args() # ignore other arguments
+
+  if not args.master_url:
+    unittest.main() # exits
+
+  module = importlib.import_module(__name__)
+  suite = unittest.defaultTestLoader.loadTestsFromModule(module)
+  all_tests = [t for s in suite for t in s]
+
+  task_distributor = erp5.util.taskdistribution.TaskDistributor(
+    portal_url=args.master_url
+  )
+  test_result = task_distributor.createTestResult(
+    revision = args.revision,
+    test_name_list = [t.id() for t in all_tests],
+    node_title = args.test_node_title,
+    test_title = args.test_suite_title,
+    project_title = args.project_title,
+  )
+
+  runner = unittest.TextTestRunner()
+  result = runner.run(suite)
+
+  errors = dict(result.errors)
+  failures = dict(result.failures)
+  skipped = dict(result.skipped)
+
+  print(errors)
+  print(failures)
+  print(skipped)
+
+  for t in all_tests:
+    kind = [errors, failures, skipped]
+    count = [0, 0, 0]
+    output = 'OK'
+    for i in range(len(kind)):
+      try:
+        output = kind[i][t]
+        count[i] = 1
+        break
+      except KeyError:
+        pass
+    print(t.id())
+    print(count)
+    print(output)
+    test_result_line = test_result.start()
+    test_result_line.stop(
+      test_count = 1,
+      error_count = count[0],
+      failure_count = count[1],
+      skip_count = count[2],
+      duration = 0,
+      command = '',
+      stdout = output,
+      stderr = '',
+      html_test_result = '',
+    )
+
+
+class EndToEndTestCase(unittest.TestCase):
+  @classmethod
+  def setUpClass(cls):
+    configp = configparser.ConfigParser()
+    configp.read('${slapos-client.cfg:output}')
+    args = type("empty_args", (), {})()
+    conf = slapos.client.ClientConfig(args, configp)
+    local = slapos.client.init(conf, logging.getLogger(__name__))
+    cls.slap = local['slap']
+    cls.supply = staticmethod(local['supply'])
+    cls._request = staticmethod(local['request'])
+    cls.product = staticmethod(local['product'])
+    cls._requested = {}
+
+  @classmethod
+  def tearDownClass(cls):
+    for args, kw in cls._requested.values():
+      kw['state']  = 'stopped'
+      cls._request(*args, **kw)
+
+  @classmethod
+  def request(cls, *args, **kw):
+    instance_name = args[1]
+    cls._requested[instance_name] = (args, kw)
+    partition = cls._request(*args, **kw)
+    return cls.unwrapConnectionDict(partition.getConnectionParameterDict())
+
+  @staticmethod
+  def unwrapConnectionDict(connection_dict):
+    try:
+      connection_json = connection_dict['_']
+    except KeyError:
+      return connection_dict
+    return json.loads(connection_json)
+
+  @classmethod
+  def getInstanceInfos(cls, instance_name):
+    # adapated from cli/info
+    infos = cls.slap.registerOpenOrder().getInformation(instance_name)
+    class Infos:
+      def __init__(self, **kw):
+        self.__dict__.update(kw)
+    connection_dict = {
+      e['connection_key'] : e['connection_value']
+      for e in infos._connection_dict
+    }
+    return Infos(
+      software_url = infos._software_release_url,
+      software_type = infos._source_reference,
+      shared = bool(infos._root_slave),
+      requested_state = infos._requested_state,
+      parameter_dict = infos._parameter_dict,
+      connection_dict = cls.unwrapConnectionDict(connection_dict),
+      news = infos._news,
+    )
+
+  @classmethod
+  def getInstanceNews(cls, instance_name):
+    try:
+      news = cls.slap.registerOpenOrder().getInformation(instance_name)._news
+    except Exception:
+      return ()
+    return news['instance']
+
+  @classmethod
+  def getInstanceStatus(cls, instance_name):
+    # adapated from cli/info
+    status = 0b00
+    for e in cls.getInstanceNews(instance_name):
+      text = e.get('text', '')
+      if text.startswith('#access'):
+        status |= 0b01
+      elif text.startswith('#error'):
+        status |= 0b10
+      if status == 0b11:
+        break
+    return ('none', 'green', 'red', 'orange')[status]
+
+  @classmethod
+  def waitUntilGreen(cls, instance_name, timeout=80):
+    t0 = time.time()
+    while (status := cls.getInstanceStatus(instance_name)) != 'green':
+      msg = 'Instance %s status is still %s' % (instance_name, status)
+      print(msg)
+      if (time.time() - t0) > 60 * timeout:
+        raise TimeoutError(msg)
+      time.sleep(60)
+
+
+class KvmTest(EndToEndTestCase):
+  def test(self):
+    # instance_name = time.strftime('e2e-test-kvm-%Y-%B-%d-%H:%M:%S')
+    instance_name = 'e2e-kvm-test' # avoid timestamp to reuse instance
+    self.request(self.product.kvm, instance_name)
+    self.waitUntilGreen(instance_name)
+    connection_dict = self.request(self.product.kvm, instance_name)
+    self.assertIn('url', connection_dict)
+
+
+class Fail(EndToEndTestCase):
+  def test(self):
+    self.assertEqual(0, 1)
+
+
+if __name__ == '__main__':
+  main()

software/end-to-end-testing/software.cfg

+[buildout]
+extends =
+  ../../stack/slapos.cfg
+  buildout.hash.cfg
+
+parts =
+  instance.cfg
+  slapos-cookbook
+
+
+[instance.cfg]
+recipe = slapos.recipe.template
+output = ${buildout:directory}/instance.cfg
+url = ${:_profile_base_location_}/${:filename}
+
+
+[runTestSuite.in]
+recipe = slapos.recipe.build:download
+output = ${buildout:directory}/${:filename}
+url = ${:_profile_base_location_}/${:filename}
+
+
+[test_one.py]
+recipe = slapos.recipe.build:download
+output = ${buildout:directory}/${:filename}
+url = ${:_profile_base_location_}/${:filename}
+
+
+[python_for_test]
+recipe = zc.recipe.egg
+interpreter = python_for_test
+scripts = ${:interpreter}
+executable = ${buildout:bin-directory}/${:interpreter}
+depends = ${lxml-python:egg}
+eggs =
+  slapos.core
+  erp5.util

software/end-to-end-testing/software.cfg.json

+{
+  "name": "End-To-End-Testing",
+  "description": "End-To-End Testing on SlapOS Cloud",
+  "serialisation": "xml",
+  "software-type": {
+    "default": {
+      "title": "default",
+      "software-type": "default",
+      "description": "default"
+    }
+  }
+}