rina: new SR to build packages and test 'rina-tools' component

This is implement as a test suite so that it can be automatically on any commit
to rina/slapos repositories.

Currently, the Debian packaging of the kernel only supports Debian Jessie.

component/qemu-kvm/buildout.cfg

@@ -44,14 +44,30 @@ environment =
   PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${gnutls:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig
   LDFLAGS=-L${pixman:location}/lib -Wl,-rpath=${pixman:location}/lib
 
-[debian-amd64-netinst.iso]
-# Download the installer of Debian 8 (Jessie)
+[debian-netinst-base]
 recipe = hexagonit.recipe.download
 ignore-existing = true
 filename = ${:_buildout_section_name_}
-url = http://cdimage.debian.org/debian-cd/${:version}/amd64/iso-cd/debian-${:version}-amd64-netinst.iso
-version = 8.6.0
-md5sum = e9f61bf327db6d8f7cee05a99f2353cc
+url = http://cdimage.debian.org/cdimage/${:release}/${:arch}/iso-cd/debian-${:version}-${:arch}-netinst.iso
+release = archive/${:version}
 download-only = true
 mode = 0644
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
+[debian-amd64-netinst-base]
+<= debian-netinst-base
+arch = amd64
+
+[debian-amd64-netinst.iso]
+# Download the installer of Debian 8 (Jessie)
+<= debian-amd64-netinst-base
+release = release/current
+version = 8.6.0
+md5sum = e9f61bf327db6d8f7cee05a99f2353cc
+
+[debian-amd64-testing-netinst.iso]
+# Download the installer of Debian Stretch
+<= debian-amd64-netinst-base
+release = stretch_di_alpha7
+version = stretch-DI-alpha7
+md5sum = 3fe53635b904553b26588491e1473e99

component/rina-tools/buildout.cfg

@@ -10,7 +10,7 @@ parts = rina-tools
 [rina-tools-repository]
 recipe = slapos.recipe.build:gitclone
 repository = https://github.com/jmuchemb/irati-stack.git
-revision = 47e2b6a5f32d1090ec75206ccdb797b78a1a3330
+revision = af3cd8350ec43ab2d309c785daf0dd16d3835005
 git-executable = ${git:location}/bin/git
 
 [rina-tools]

component/vm-img/debian.cfg

+[buildout]
+extends =
+  ../file/buildout.cfg
+  ../openssh/buildout.cfg
+  ../p7zip/buildout.cfg
+  ../qemu-kvm/buildout.cfg
+
+parts = vm-debian
+
+[vm-run-environment]
+PATH = ${openssh:location}/bin:${qemu:location}/bin:%(PATH)s
+
+[vm-install-environment]
+PATH = ${file:location}/bin:${p7zip:location}/bin:${vm-run-environment:PATH}
+
+[vm-run-base]
+recipe = slapos.recipe.build:vm.run
+environment = vm-run-environment
+vm = ${vm-debian:location}
+dist = ${vm-debian:dists}
+
+[vm-debian]
+recipe = slapos.recipe.build:vm.install-debian
+environment = vm-install-environment
+dists = debian-jessie
+size = 1Gi
+late-command =
+# rdnssd causes too much trouble with recent QEMU, because the latter acts as
+# a DNS proxy on both IPv4 and IPv6 without translating queries to what the
+# host supports.
+  dpkg -P rdnssd
+debconf.debconf =
+  debconf/frontend noninteractive
+  debconf/priority critical
+# minimal size
+preseed.recommends = false
+preseed.tasks =
+packages = localepurge ssh
+
+[debian-jessie]
+x86_64.iso = debian-amd64-netinst.iso
+x86_64.kernel = install.amd/vmlinuz
+x86_64.initrd = install.amd/initrd.gz
+
+[debian-stretch]
+<= debian-jessie
+x86_64.iso = debian-amd64-testing-netinst.iso

software/build-rina/build.cfg

+[buildout]
+extends =
+  ../../component/vm-img/debian.cfg
+parts =
+  debuild-rina-tools
+  test-rina
+
+[vm-run-base]
+smp = 4
+# estimate the of RAM needed depending on the number CPUs
+mem = 2048
+apt-sources =
+apt-update =
+  for x in ${:apt-sources}
+  do echo deb [trusted=yes] file:`map $x` ./
+  done >>/etc/apt/sources.list
+  apt-get update
+
+[debuild-rina-base]
+<= vm-run-base
+mount.rina = ${rina-tools-repository:location}
+cd-source =
+  git clone -snc core.sparseCheckout=true /mnt/rina /rina
+  cd /rina
+  echo /${:component} > .git/info/sparse-checkout
+  git reset --hard
+  cd ${:component}
+# following lines are for debchange
+  export NAME=`git show -s --format=%cn`
+  export EMAIL=`git show -s --format=%ce`
+  dist=${:suite}
+  [ "$dist" ] && dist=`lsb_release -sc`/$dist || dist=UNRELEASED
+prepare =
+build-deps =
+  mk-build-deps -irt 'apt-get -y'
+build =
+  dpkg-buildpackage -uc -b -jauto
+finalize =
+  df
+  cd ..
+  mv *.changes *.deb $PARTDIR
+  cd $PARTDIR
+  dpkg-scanpackages -m . /dev/null |gzip --best >Packages.gz
+  r=`apt-ftparchive release .`
+  echo "$r" > Release
+command =
+  ${:apt-update}
+  ${:cd-source}
+  ( ${:prepare}
+    ${:build-deps}
+    ${:build}
+  ) 2>&1 |tee $PARTDIR/build.log
+  ${:finalize}
+
+[debuild-rina-kernel]
+<= debuild-rina-base
+component = linux
+prepare =
+  debian/dch-snapshot --force-distribution -D $dist
+  apt-get -y install kernel-wedge quilt python-six
+  QUILT_PATCHES=$PWD/debian/patches QUILT_PC=.pc quilt push --quiltrc - -a -q --fuzz=0
+  debian/rules debian/rules.gen || :
+build =
+  arch=`dpkg-architecture -qDEB_HOST_ARCH`
+  make -j${:smp} -f debian/rules.gen binary-arch_$${arch}_none binary-libc-dev_$${arch}
+  dpkg-genchanges -b -UBinary -UDescription \
+    >../linux_`dpkg-parsechangelog -S Version`_$${arch}.changes
+
+[debuild-librina]
+<= debuild-rina-base
+component = librina
+apt-sources = ${debuild-rina-kernel:location}
+prepare =
+  debian/rules DIST=$dist
+
+[debuild-rinad]
+<= debuild-librina
+component = rinad
+apt-sources = ${debuild-librina:location}
+
+[rina-tools-deps]
+apt-sources = ${debuild-librina:location} ${debuild-rinad:location}
+
+[debuild-rina-tools]
+<= debuild-librina
+  rina-tools-deps
+component = rina-tools
+# check that the built packages install
+finalize =
+  dpkg -i ../*.deb
+  ${debuild-librina:finalize}
+
+[test-rina]
+<= vm-run-base
+  rina-tools-deps
+mount.rina = ${rina-tools-repository:location}
+mount.slapos = ${:_profile_base_location_}/../..
+mount.slapos.package = ${slapos.package-repository:location}
+commands = install test
+install =
+  ${:apt-update}
+  apt-get -y install librinad-dev rinad
+  : || {
+  mkdir rina-tools
+  cd rina-tools
+  cat <<EOF >buildout.cfg
+  [buildout]
+  extends = /mnt/slapos/software/hellorina/software.cfg
+  develop = /mnt/slapos
+  [slapos-cookbook-develop]
+  recipe =
+  setup =
+  [rina-tools-repository]
+  repository = /mnt/rina
+  [versions]
+  slapos.cookbook =
+  EOF
+  sed s,${buildout:directory}/,/mnt/buildout/, /mnt/buildout/bin/buildout |
+  python - bootstrap
+  MAKEFLAGS=-j${:smp} bin/buildout
+  }
+
+  /mnt/slapos.package/playbook/roles/rina/gen-ipcm-conf
+  systemctl enable ipcm-re6st
+  arch=`dpkg-architecture -qDEB_HOST_ARCH`
+  dpkg -i `map ${debuild-rina-kernel:location}`/linux-image-*-$${arch}_*.deb
+  reboot
+test = ( set -x
+# TODO: more tests
+  grep re6st/1// /sys/rina/ipcps/1/name
+  ) 2>&1 |tee $PARTDIR/result

software/build-rina/instance-build-rina-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Parameters to build/publish RINA packages and test hellorina SR.",
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "publish": {
+      "description": "Upload built packages automatically to a Debian repository when successful.",
+      "type": "object",
+      "required": ["suite", "host", "key"],
+      "properties": {
+        "suite": {
+          "type": "string"
+        },
+        "host": {
+          "type": "string"
+        },
+        "port": {
+          "type": "integer",
+          "default": 22
+        },
+        "key": {
+          "type": "string"
+        }
+      }
+    }
+  }
+}

software/build-rina/runTestSuite.in

+#!{{parameter_dict['runTestSuite_py']}}
+from __future__ import print_function
+import argparse, errno, os, shutil, subprocess, sys, traceback
+from time import gmtime, strftime, time
+# These are the 2 modules to reuse when using ERP5 for managing test bots.
+# What we do here is currently too new to reuse more from testsuite.
+from erp5.util import taskdistribution
+from erp5.util.testsuite import format_command
+
+{% set vm = parameter_dict['vm'] -%}
+
+dist_list = {{vm['dists'].split()}}
+publish = {{slapparameter_dict.get('publish')}}
+
+BUILDDIR = 'build'
+BUILDOUT = os.path.realpath(os.path.join('software_release', 'bin', 'buildout'))
+
+STAT_MAP = dict(
+  TOTAL = 'test_count',
+  PASS  = None,
+  SKIP  = 'skip_count',
+  # Same as for NEO. ERP5 must be changed to only distinguish
+  # - expected (-> 'failure' for now) and
+  # - unexpected (-> 'error') failures.
+  XFAIL = 'failure_count',
+  FAIL  = 'error_count',
+  XPASS = 'error_count',
+  ERROR = 'error_count',
+)
+
+class DummyTestResult:
+
+  class DummyTestResultLine:
+
+    def stop(self, duration, stdout='', **kw):
+      print('\n' + stdout)
+      print('Ran in %.3fs' % duration)
+
+  done = 0
+
+  def __init__(self, test_name_list):
+    self.test_name_list = test_name_list
+
+  def start(self):
+    test_result_line = self.DummyTestResultLine()
+    try:
+      test_result_line.name = self.test_name_list[self.done]
+    except IndexError:
+      return
+    self.done += 1
+    return test_result_line
+
+
+def main():
+  os.environ.update({k: v.strip() % os.environ
+    for k, v in {{parameter_dict['environment'].items()}}})
+
+  parser = argparse.ArgumentParser(description='Run a test suite.')
+  parser.add_argument('--test_suite', help='The test suite name')
+  parser.add_argument('--test_suite_title', help='The test suite title')
+  parser.add_argument('--test_node_title', help='The test node title')
+  parser.add_argument('--project_title', help='The project title')
+  parser.add_argument('--revision', help='The revision to test',
+                      default='dummy_revision')
+  parser.add_argument('--node_quantity', type=int,
+                      help='Number of CPUs to use for the VM')
+  parser.add_argument('--master_url',
+                      help='The Url of Master controling many suites')
+
+  args = parser.parse_args()
+
+  test_title = args.test_suite_title or args.test_suite
+  if args.master_url:
+    tool = taskdistribution.TaskDistributionTool(args.master_url)
+    test_result = tool.createTestResult(args.revision,
+                                        dist_list,
+                                        args.test_node_title,
+                                        test_title=test_title,
+                                        project_title=args.project_title)
+    if test_result is None:
+      return
+  else:
+    test_result = DummyTestResult(dist_list)
+
+  if os.path.exists(BUILDDIR):
+    shutil.rmtree(BUILDDIR)
+  os.mkdir(BUILDDIR)
+  oldpwd = os.getcwd()
+  os.chdir(BUILDDIR)
+
+  with open('buildout.cfg', 'w') as f:
+    f.write("""\
+[buildout]
+extends = {{parameter_dict['profile_base_location']}}/build.cfg
+offline = true
+develop-eggs-directory = {{buildout['develop-eggs-directory']}}
+eggs-directory = {{buildout['eggs-directory']}}
+
+[vm-run-base]
+environment =
+vm = {{vm['location']}}
+
+[rina-tools-repository]
+location = {{parameter_dict['rina_tools']}}
+
+[slapos.package-repository]
+location = {{parameter_dict['slapos_package']}}
+""")
+  librina_log = os.path.join('parts', 'debuild-librina', 'build.log')
+  stderr_write = sys.stderr.write
+
+  while 1:
+    test_result_line = test_result.start()
+    if not test_result_line:
+      break
+    dist = test_result_line.name
+
+    cmd = [BUILDOUT,
+      'vm-run-base:dist=' + dist,
+      'debuild-rina-base:suite=' + (publish['suite'] if publish else ''),
+    ]
+    if args.node_quantity:
+      cmd.append('vm-run-base:smp=%s' % args.node_quantity)
+    status_dict = {'command': format_command(*cmd)}
+    print('$', status_dict['command'])
+
+    # Wanted on test result lines:
+    #  status: UNKNOWN in case of buildout failure
+    #          (even if the test suite could be run)
+    #  output: test suite summary if any
+    #  error: buildout traceback or test suite log
+
+    start = time()
+    try:
+      p = subprocess.Popen(cmd, stderr=subprocess.PIPE)
+      stderr = []
+      while 1:
+        line = p.stderr.readline()
+        if not line:
+          break
+        stderr_write(line)
+        stderr.append(line)
+      returncode = p.wait()
+      end = time()
+      del p
+
+      if returncode:
+        iter_err = enumerate(reversed(stderr), 1)
+        for i, line in iter_err:
+          if line == "Traceback (most recent call last):\n":
+            for i, line in iter_err:
+              if line == '\n':
+                break
+            for i, line in iter_err:
+              if line[0] != ' ':
+                break
+            break
+        if line == "While:\n":
+          del stderr[:-i]
+        status_dict['stderr'] = ''.join(stderr)
+
+      with open(librina_log) as f:
+        log = f.readlines()
+      del log[:log.index('make  check-TESTS\n')]
+      for i, line in enumerate(log):
+        if line.startswith('Testsuite summary'):
+          del log[log.index(log[i+1], i+2):]
+          status_dict['stdout'] = ''.join(log[i:])
+          stat = {}
+          for line in log[i+2:]:
+            k, v = line[2:].split(':')
+            k = STAT_MAP[k]
+            if k:
+              stat[k] = stat.get(k, 0) + int(v.strip())
+          if not returncode:
+            status_dict.update(stat)
+          status_dict.setdefault('stderr', ''.join(log[:i-1]))
+          break
+    except Exception:
+      status_dict.setdefault('stderr', traceback.format_exc())
+
+    test_result_line.stop(
+      date = strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
+      duration = end - start,
+      **status_dict)
+
+    # TODO: upload packages if 'publish' parameter is given
+
+  if args.master_url:
+    os.chdir(oldpwd)
+    shutil.rmtree(BUILDDIR)
+
+
+if __name__ == "__main__":
+    main()

software/build-rina/software.cfg

+[buildout]
+extends =
+  ../../stack/slapos.cfg
+  ../../component/vm-img/debian.cfg
+  ../../component/rina-tools/buildout.cfg
+parts =
+  slapos-cookbook
+  template
+
+[template]
+recipe = slapos.recipe.template:jinja2
+# XXX: "template.cfg" is hardcoded in instanciation recipe
+rendered = ${buildout:directory}/template.cfg
+template = inline:
+  [buildout]
+  eggs-directory = ${buildout:eggs-directory}
+  develop-eggs-directory = ${buildout:develop-eggs-directory}
+  parts = runTestSuite
+  [slap-configuration]
+  recipe = slapos.cookbook:slapconfiguration.serialised
+  computer = $${slap-connection:computer-id}
+  partition = $${slap-connection:partition-id}
+  url = $${slap-connection:server-url}
+  key = $${slap-connection:key-file}
+  cert = $${slap-connection:cert-file}
+  [directory]
+  recipe = slapos.cookbook:mkdirectory
+  bin = $${buildout:directory}/bin
+  [runTestSuite]
+  recipe = slapos.recipe.template:jinja2
+  rendered = $${directory:bin}/$${:_buildout_section_name_}
+  template = ${:_profile_base_location_}/$${:_buildout_section_name_}.in
+  mode = 0755
+  context =
+  {##} section buildout buildout
+  {##} section parameter_dict runTestSuite-parameters
+  {##} key slapparameter_dict slap-configuration:configuration
+  [runTestSuite-parameters]
+  environment = {{dumps(environment)}}
+  vm = {{dumps(vm)}}
+  runTestSuite_py = ${buildout:bin-directory}/${runTestSuite_py:interpreter}
+  profile_base_location = ${:_profile_base_location_}
+  rina_tools = ${rina-tools-repository:location}
+  slapos_package = ${slapos.package-repository:location}
+context =
+  section environment vm-run-environment
+  section vm vm-debian
+
+[runTestSuite_py]
+recipe = zc.recipe.egg
+eggs = erp5.util
+interpreter = ${:_buildout_section_name_}
+
+[rina-tools-repository]
+sparse-checkout = /.gitignore
+
+[slapos.package-repository]
+recipe = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/slapos.package.git
+git-executable = ${git:location}/bin/git
+sparse-checkout = /playbook/roles/rina
+
+[vm-debian]
+# building a generic Debian kernel uses a lot of space
+size = 16Gi
+packages +=
+# generic (another SR that build packages automatically would use the same list)
+  apt-utils build-essential devscripts equivs lsb-release
+# specific
+  git ca-certificates python
+# biggest and common build-deps for RINA
+  dh-autoreconf pkg-config doxygen maven xmlto

software/test-slave-instance-deployment/software.cfg

@@ -58,7 +58,7 @@ eggs = collective.recipe.template
 [versions]
 collective.recipe.template = 1.11
 plone.recipe.command = 1.1
-slapos.recipe.build = 0.24
+slapos.recipe.build = 0.25
 slapos.recipe.template = 2.7
 
 # Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.

stack/slapos.cfg

@@ -137,7 +137,7 @@ slapos.cookbook = 1.0.35
 slapos.core = 1.3.16
 slapos.extension.strip = 0.1
 slapos.libnetworkcache = 0.14.5
-slapos.recipe.build = 0.24
+slapos.recipe.build = 0.25
 slapos.recipe.cmmi = 0.2
 stevedore = 1.17.1
 unicodecsv = 0.14.1