Update Release Candidate

component/ZEO/buildout.cfg

@@ -3,22 +3,37 @@
 
 [buildout]
 extends =
+    ../ZODB/buildout.cfg
     ../git/buildout.cfg
 
 parts = ZEO/scripts
 
 
-# ZEO allows to use either ZEO4 or ZEO5
-# To select which version to use users should do:
+# ZEO provides ZEO<X> depending on ZODB major version.
 #
-#   [ZEO]
-#   egg = ${ZEO<version>:egg}
-#
-# By default ZEO4 is used.
+# - ZEO4 works only with ZODB4
+#   https://github.com/zopefoundation/ZEO/blob/4.3.1-1-g47d3fbe8/setup.py#L122
+# - ZEO5 works only with ZODB5
+#   https://github.com/zopefoundation/ZEO/blob/5.2.2-1-g3d90ed42/setup.py#L20
 [ZEO]
-recipe  = zc.recipe.egg:eggs
-egg     = ${ZEO4:egg}
-eggs    = ${:egg}
+recipe  = slapos.recipe.build
+depends = ${ZODB:egg}
+init =
+  # link/depend to ZEO<ZODB.major>
+  zodb = self.buildout['ZODB']
+  zmajor = zodb['major']
+  zeo_x = self.buildout['ZEO'+zmajor]
+  options['depends'] += '$${%s:egg}' % zeo_x.name
+  options['egg'] = zeo_x['egg']
+
+  # update [versions] from what is needed by ZEO<X>
+  self.buildout.parse('[_ZEO-versions]\n' + zeo_x['egg-versions'])
+  versions = self.buildout['versions']
+  versions.update(self.buildout['_ZEO-versions'])
+  # propagate updated [versions] -> easy_install
+  # (buildout does this in Buildout constructor)
+  import zc.buildout.easy_install
+  zc.buildout.easy_install.default_versions(versions)
 
 
 # ZEO/scripts installs scripts from ZEO
@@ -32,6 +47,7 @@ eggs    = ${ZEO:egg}
 recipe  = zc.recipe.egg:develop
 setup   = ${ZEO4-repository:location}
 egg     = ZEO
+egg-versions =
 
 [ZEO4-repository]
 recipe  = slapos.recipe.build:gitclone
@@ -47,3 +63,18 @@ git-executable = ${git:location}/bin/git
 recipe  = zc.recipe.egg:eggs
 egg     = ZEO
 eggs    = ${:egg}
+
+egg-versions =
+  ZEO = 5.2.2
+  trollius = 2.2.post1
+  futures = 3.2.0
+
+
+# ZEO4-wc2 is ZEO4 version with patches for wendelin.core 2 to work correctly.
+# The patches are backports of what is in ZEO master (ZEO5), but since upstream
+# considers 4 branch to be "dead", we have to maintain it by ourselves.
+# See https://github.com/zopefoundation/ZEO/pull/161 for the reference.
+[ZEO4-wc2]
+<= ZEO4
+revisions = bf80d23d3506
+location = ${buildout:parts-directory}/ZEO4-wc2

component/ZODB/buildout.cfg

+# SlapOS component for ZODB.
+# https://zodb.org/
+
+[buildout]
+extends =
+    ../git/buildout.cfg
+    ../python-cffi/buildout.cfg
+
+parts = ZODB/scripts
+
+
+# ZODB allows to use either ZODB4, ZODB4-wc2 or ZODB5.
+# To select which version to use users should do:
+#
+#   [ZODB]
+#   major = <ZODB-version-major>
+#
+# By default ZODB4 is used.
+[ZODB]
+recipe = slapos.recipe.build
+major  = 4
+init =
+  # link/depend ZODB -> ZODB<X>
+  zodb_x = 'ZODB'+options['major']
+  zodb_x = self.buildout[zodb_x]
+  options['depends'] = '$${%s:egg}' % zodb_x.name
+
+  options['egg'] = zodb_x['egg']
+
+  # update [versions] from what is needed by ZODB<X>
+  self.buildout.parse('[_ZODB-versions]\n' + zodb_x['egg-versions'])
+  versions = self.buildout['versions']
+  versions.update(self.buildout['_ZODB-versions'])
+  # propagate updated [versions] -> easy_install
+  # (buildout does this in Buildout constructor)
+  import zc.buildout.easy_install
+  zc.buildout.easy_install.default_versions(versions)
+
+
+# ZODB/scripts installs scripts from ZODB
+[ZODB/scripts]
+recipe  = zc.recipe.egg:scripts
+eggs    = ${ZODB:egg}
+
+# ZODB4 and ZODB5 are plain upstream eggs
+[_ZODB]
+recipe  = zc.recipe.egg:eggs
+egg     = ZODB
+eggs    = ${:egg}
+depends = ${persistent:egg} ${BTrees:egg}
+
+[ZODB4]
+<= _ZODB
+egg-versions =
+  ZODB = 4.4.5
+  transaction = 1.7.0
+
+[ZODB5]
+<= _ZODB
+egg-versions =
+  ZODB = 5.6.0
+  transaction = 2.4.0
+
+
+# ZODB4-wc2 is ZODB4 version with patches for wendelin.core 2 to work correctly.
+# The main change is backport of the way MVCC is handled by always calling
+# loadBefore instead of load.
+# See https://lab.nexedi.com/nexedi/ZODB/commit/8e7eab33 for details.
+[ZODB4-wc2]
+<= ZODB4
+recipe = zc.recipe.egg:develop
+setup = ${ZODB4-wc2-repository:location}
+
+[ZODB4-wc2-repository]
+recipe  = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/ZODB.git
+git-executable = ${git:location}/bin/git
+branch = 4-nxd+MVCC-load-before
+revision = 4.4.5-9-g8e7eab330
+location = ${buildout:parts-directory}/ZODB4-wc2
+
+
+# XXX: buildout fails to install properly eggs with setup_requires
+[BTrees]
+recipe = zc.recipe.egg:custom
+egg = ${:_buildout_section_name_}
+setup-eggs = ${persistent:egg}
+
+[persistent]
+recipe = zc.recipe.egg:custom
+egg = ${:_buildout_section_name_}
+setup-eggs = ${python-cffi:egg}
+
+
+# eggs that are common to ZODB4 and ZODB5.
+[versions]
+BTrees = 4.5.1
+persistent = 4.6.4
+zodbpickle = 1.0.4
+
+# Provide ZODB3 for those eggs that still care about ZODB3 compatibility -
+# for example wendelin.core. ZODB3 3.11 is just a dependency egg on _latest_
+# ZODB, persistent, BTrees and ZEO.
+ZODB3 = 3.11.0

component/ZODB/test-zodb4-wc2.cfg

+# SlapOS software release to test ZODB4-wc2 on Nexedi testing infrastructure.
+
+[buildout]
+extends =
+# test<X>.cfg configures ZODB.major=<X>.
+
+    ../../stack/nxdtest.cfg
+    ../pygolang/buildout.cfg
+    ../python-manuel/buildout.cfg
+    buildout.cfg
+    ../ZEO/buildout.cfg
+
+parts =
+    ZODB
+    .nxdtest
+
+#   for instance
+    ZODB-python
+    slapos-cookbook
+    instance.cfg
+
+[ZODB]
+major = 4-wc2
+[ZODB4-wc2-repository]
+revision =
+
+# we need persistent to be a git checkout because persistent tests want to
+# discover in-tree files that are not present in persistent egg when it is
+# installed in non-development mode:
+#
+#   https://erp5.nexedi.net/test_result_module/20201123-3F859E35/7
+#   (look for "AssertionError: could not find my setup.py")
+#
+#   https://github.com/zopefoundation/persistent/blob/4.6.4-0-g7ed95cf/persistent/tests/test_docs.py#L37-L43
+[persistent]
+recipe = zc.recipe.egg:develop
+setup = ${persistent-repository:location}
+
+[persistent-repository]
+recipe  = slapos.recipe.build:gitclone
+location = ${buildout:parts-directory}/persistent
+repository = https://github.com/zopefoundation/persistent.git
+branch = master
+revision = 4.6.4-0-g7ed95cf
+git-executable = ${git:location}/bin/git
+
+
+# test-dependent eggs that must come through in-tree recipes.
+[ZODB]
+depends += ${manuel:egg}
+
+
+# bin/python is preinstalled with sys.path to ZODB & friends.
+[ZODB-python]
+<= python-interpreter
+eggs +=
+  ${ZODB:egg}[test]
+  ${ZEO:egg}[test]
+  transaction[test]
+
+# env.sh for ZODB's python to be on $PATH.
+[ZODB-env.sh]
+recipe   = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/${:_buildout_section_name_}
+template = inline:
+  export PS1="(ZODB-env) $PS1"
+  export PATH=${buildout:bin-directory}:$PATH
+
+# .nxdtest to run ZODB tests
+[.nxdtest]
+<= jinja2-template
+template =
+  inline:
+  # determine where ZODB & friends were installed
+  from subprocess import check_output
+  where = check_output(['python', '-c', """if 1:
+        from os.path import dirname
+        for m in "ZODB", "persistent", "BTrees", "transaction", "ZEO":
+            print(dirname(__import__(m).__file__))
+  """])
+  ZODB, persistent, BTrees, transaction, ZEO = where.split()
+
+  # run tests for whole ZODB stack
+  TestCase('ZODB/unit',       ['python', '-m', 'zope.testrunner', '-uv', '--package-path', ZODB, 'ZODB'])
+  TestCase('ZODB/functional', ['python', '-m', 'zope.testrunner', '-fv', '--package-path', ZODB, 'ZODB'])
+
+  TestCase('ZEO/unit',       ['python', '-m', 'zope.testrunner', '-uv', '--package-path', ZEO, 'ZEO'])
+  TestCase('ZEO/functional', ['python', '-m', 'zope.testrunner', '-fv', '--package-path', ZEO, 'ZEO'])
+
+  TestCase('persistent', ['python', '-m', 'zope.testrunner', '-v', '--package-path', persistent, 'persistent'])
+  TestCase('BTrees',     ['python', '-m', 'zope.testrunner', '-v', '--package-path', BTrees, 'BTrees'])
+
+  # transaction uses unittest instead of zope.testrunner
+  import os.path
+  def P(path): return os.path.join(path, '..')
+  TestCase('transaction',['python', '-m', 'unittest', 'discover', '-s', P(transaction)])
+
+
+# instance to run nxdtest.
+[instance.cfg]
+<= jinja2-template
+template = inline:
+  [buildout]
+  extends = ${nxdtest-instance.cfg:rendered}
+
+  [runTestSuite]
+  env.sh  = ${ZODB-env.sh:rendered}
+  workdir = ${buildout:directory}
+
+
+[versions]
+mock = 3.0.5
+random2 = 1.0.1
+zope.testing = 4.7
+zope.testrunner = 5.2
+zope.exceptions = 4.4

component/defaults.cfg

@@ -49,11 +49,11 @@ init =
     current = subprocess.check_output(('gcc', '-dumpversion'),
                                       universal_newlines=True).strip()
   self.system_version = current
-  # If we're still going to use the same GCC,
-  # the conditions have no impact on the dependant parts.
+# If we're still going to use the same GCC,
+# the conditions have no impact on the dependant parts.
   min_version = options.pop('min_version', None)
   max_version = options.pop('max_version', None)
-  ###
+###
   if (parse_version(min_version or current)
       <= parse_version(current)
       <= parse_version(max_version or current)):
@@ -61,7 +61,7 @@ init =
     for path in os.getenv('PATH', '').split(os.pathsep): # PY3: shutil.which
       gcc = os.path.join(path, 'gcc')
       if os.access(gcc, os.X_OK) and not os.path.isdir(gcc):
-        options['prefix'] = os.path.dirname(path)
+        options['prefix'] = os.path.dirname(os.path.normpath(path))
         break
   else:
     options['prefix'] = self.buildout[options['part']]['location']
@@ -72,6 +72,7 @@ update =
     env = os.environ
     env['PATH'] = os.pathsep.join((
       os.path.join(options['prefix'], 'bin'),
+      os.path.join(self.buildout['binutils']['location'], 'bin'),
       env['PATH']
     ))
   else:

component/file/buildout.cfg

@@ -4,21 +4,15 @@
 [buildout]
 parts = file
 extends =
-  ../patch/buildout.cfg
   ../zlib/buildout.cfg
 
 [file]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://ftp.icm.edu.pl/packages/file/file-5.37.tar.gz
-md5sum = 80c29aca745466c6c24d11f059329075
+url = http://ftp.icm.edu.pl/packages/file/file-5.39.tar.gz
+md5sum = 1c450306053622803a25647d88f80f25
 configure-options =
   --disable-static
 environment =
   CPPFLAGS=-I${zlib:location}/include
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
-
-patch-binary = ${patch:location}/bin/patch
-patch-options = -p1
-patches =
-  https://git.in-ulm.de/cbiedl/file/raw/debian/1%255.37-6/debian/patches/cherry-pick.FILE5_37-67-g46a8443f.limit-the-number-of-elements-in-a-vector-found-by-oss-fuzz.patch#fb6f7d32ce89573bf4b4b302c812e394

component/firewalld/buildout.cfg

@@ -19,12 +19,15 @@ md5sum = a120286a2c2bc12ece24d719ef2879b5
 python-egg = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/python${python:version}/site-packages
 location = @@LOCATION@@
 sbin-dir = sbin-orig
+etc-root = ${:location}/etc
+etc-dir = ${:etc-root}/firewalld
 pre-configure =
   sed -i -E 's,(^SUBDIRS *=.*) doc(\b.*),\1\2,' Makefile.am
   aclocal -I${pkgconfig:location}/share/aclocal -I${gettext:location}/share/aclocal -I${libtool:location}/share/aclocal
   ./autogen.sh
 configure-options =
   --sbindir=${:location}/${:sbin-dir}
+  --sysconfdir=${:etc-root}
   --disable-docs
   --with-iptables=${iptables:location}/sbin/iptables-legacy
   --with-iptables-restore=${iptables:location}/sbin/iptables-legacy-restore
@@ -110,28 +113,30 @@ md5sum = 893752ba2e93a1e96334dfee19f884ad
 [firewalld-patch]
 recipe = plone.recipe.command
 python = ${buildout:bin-directory}/${firewalld-eggs:interpreter}
+stop-on-error = True
 command =
+  set -e
   sed -i 's#python2.7#${:python}#' ${firewalld:location}/bin/firewall-applet
   sed -i 's#python2.7#${:python}#' ${firewalld:location}/bin/firewall-cmd
   sed -i 's#python2.7#${:python}#' ${firewalld:location}/bin/firewall-config
   sed -i 's#python2.7#${:python}#' ${firewalld:location}/bin/firewall-offline-cmd
   sed -i 's#python2.7#${:python}#' ${firewalld:location}/${firewalld:sbin-dir}/firewalld
 
-  sed -i 's#python2.7 /usr/bin/firewall-config#${:python} ${firewalld:location}/bin/firewall-config#' ${firewalld:location}/etc/firewalld/lockdown-whitelist.xml
+  sed -i 's#python2.7 /usr/bin/firewall-config#${:python} ${firewalld:location}/bin/firewall-config#' ${firewalld:etc-dir}/lockdown-whitelist.xml
 
-  sed -i 's#DefaultZone=public#DefaultZone=trusted#' ${firewalld:location}/etc/firewalld/firewalld.conf
+  sed -i 's#DefaultZone=public#DefaultZone=trusted#' ${firewalld:etc-dir}/firewalld.conf
+  sed -i 's#FirewallBackend=nftables#FirewallBackend=iptables#' ${firewalld:etc-dir}/firewalld.conf
 
-  sed -i 's#/etc/firewalld#${firewalld:location}/etc/firewalld#'     ${firewalld:python-egg}/firewall/config/__init__.py
-  sed -i 's#/usr/lib/firewalld#${firewalld:location}/lib/firewalld#' ${firewalld:python-egg}/firewall/config/__init__.py
-  sed -i 's#/usr/share/#${firewalld:location}/share#'                ${firewalld:python-egg}/firewall/config/__init__.py
+  sed -i "s#'/etc/firewalld'#'${firewalld:etc-dir}'#"     ${firewalld:python-egg}/firewall/config/__init__.py
+  sed -i "s#'/usr/lib/firewalld'#'${firewalld:location}/lib/firewalld'#" ${firewalld:python-egg}/firewall/config/__init__.py
+  sed -i "s#'/usr/share/'#'${firewalld:location}/share'#"                ${firewalld:python-egg}/firewall/config/__init__.py
 
   sed -i "s#import sys#import sys, os\n\nos.environ['GI_TYPELIB_PATH'] = '${gobject-introspection:location}/lib/girepository-1.0/'#" ${:python}
-  sed -i 's#<syslog/>#<!-- no syslog -->#' ${dbus:location}/etc/dbus-1/system.conf
   sed -i 's#<user>messagebus</user>#<user>slapsoft</user>#' ${dbus:location}/share/dbus-1/system.conf
-  cp -f ${firewalld:location}/lib/firewalld/zones/trusted.xml ${firewalld:location}/etc/firewalld/zones/
+  cp -f ${firewalld:location}/lib/firewalld/zones/trusted.xml ${firewalld:etc-dir}/zones/
   cp -f ${firewalld:location}/share/dbus-1/system.d/FirewallD.conf ${dbus:location}/share/dbus-1/system.d/
   mkdir -p ${firewalld:location}/sbin
-  echo -n '#!/bin/sh\nLD_LIBRARY_PATH=${nftables:location}/lib ${firewalld:location}/${firewalld:sbin-dir}/firewalld "$@"' > ${firewalld:location}/sbin/firewalld
+  echo -n '#!/bin/sh\nLD_LIBRARY_PATH=${nftables:location}/lib exec ${firewalld:location}/${firewalld:sbin-dir}/firewalld "$@"' > ${firewalld:location}/sbin/firewalld
   chmod a+x ${firewalld:location}/sbin/firewalld
 
 update-command = ${:command}

component/golang/buildout.cfg

@@ -42,28 +42,10 @@ md5sum = 6b607fc795391dc609ffd79ebf41f080
 environment-extra =
   GOROOT_BOOTSTRAP=${golang14:location}
 
-[golang1.13]
-<= golang-common
-url = https://golang.org/dl/go1.13.15.src.tar.gz
-md5sum = 4f4af14d88352a62761a9dcedf863ac0
-
-# go1.13 needs go1.4 to bootstrap
-environment-extra =
-  GOROOT_BOOTSTRAP=${golang14:location}
-
-[golang1.14]
-<= golang-common
-url = https://golang.org/dl/go1.14.15.src.tar.gz
-md5sum = 1065b705a0d330136084f4e1628baf82
-
-# go1.14 needs go1.4 to bootstrap
-environment-extra =
-  GOROOT_BOOTSTRAP=${golang14:location}
-
 [golang1.15]
 <= golang-common
-url = https://golang.org/dl/go1.15.8.src.tar.gz
-md5sum = 7e4b8580bfe25ffc51501b1dc3d2d8ce
+url = https://golang.org/dl/go1.15.10.src.tar.gz
+md5sum = 898a4aea4bf5d4adc924154e347acfa3
 
 # go1.15 needs go1.4 to bootstrap
 environment-extra =
@@ -71,8 +53,8 @@ environment-extra =
 
 [golang1.16]
 <= golang-common
-url = https://golang.org/dl/go1.16.src.tar.gz
-md5sum = b767f7e81d886b4208874ea953db2ce3
+url = https://golang.org/dl/go1.16.2.src.tar.gz
+md5sum = 516f63e11c64b1984045c035d4c2f65d
 
 # go1.16 needs go1.4 to bootstrap
 environment-extra =
@@ -139,7 +121,7 @@ bin	= ${gowork.dir:bin}
 depends = ${gowork.goinstall:recipe}
 
 # go version used for the workspace (possible to override in applications)
-golang  = ${golang1.15:location}
+golang  = ${golang1.16:location}
 
 # no special build flags by default
 buildflags =

component/helloweb/buildout.cfg

@@ -30,8 +30,6 @@ revision      = a7c788ae71
 install =
     ${helloweb:location}/go:./...
 
-golang  = ${golang1.16:location}
-
 # -*- go -*-
 [helloweb-go]
 # we already have gowork/bin/helloweb with helloweb Go build.

component/patch/buildout.cfg

@@ -7,7 +7,7 @@ parts =
 [patch]
 recipe = slapos.recipe.cmmi
 shared = true
-url = ftp://ftp.gnu.org/gnu/patch/patch-2.7.5.tar.xz
-md5sum = e3da7940431633fb65a01b91d3b7a27a
+url = http://ftp.gnu.org/gnu/patch/patch-2.7.6.tar.xz
+md5sum = 78ad9937e4caadcba1526ef1853730d5
 environment =
   PATH=${xz-utils:location}/bin:%(PATH)s

component/wendelin.core/activate-WC2-preview.cfg

+# SlapOS extension that switches Wendelin.core to Wendelin.core 2 preview.
+#
+# Should go away once wendelin.core 2 lands to nexedi/wendelin.core@master.
+# Must be extended from last, for example:
+# 
+# [buildout]
+# extends =
+#   .../stack/erp5/buildout.cfg
+#   .../component/wendelin.core/activate-WC2-preview.cfg
+
+[wendelin.core-repository]
+repository = https://lab.nexedi.com/kirr/wendelin.core.git
+branch   = t
+revison  =

component/wendelin.core/buildout.cfg

@@ -3,6 +3,7 @@
 [buildout]
 extends =
   ../pygolang/buildout.cfg
+  ../ZODB/buildout.cfg
   ../zodbtools/buildout.cfg
   ../numpy/buildout.cfg
   ../golang/buildout.cfg
@@ -24,6 +25,7 @@ environment = wendelin.core-env
 
 # dependent eggs that must come through in-tree recipes
 depends =
+    ${ZODB:egg}
     ${zodbtools:egg}
     ${numpy:egg}
 

component/wendelin.core/test-zodb4-wc2.cfg

+# SlapOS software release to test wendelin.core/ZODB4-wc2 on Nexedi testing infrastructure.
+
+[buildout]
+extends = test.cfg
+
+[ZODB]
+major = 4-wc2

component/xz-utils/buildout.cfg

@@ -5,7 +5,7 @@ parts =
 [xz-utils]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://tukaani.org/xz/xz-5.2.2.tar.bz2
-md5sum = f90c9a0c8b259aee2234c4e0d7fd70af
+url = http://tukaani.org/xz/xz-5.2.5.tar.bz2
+md5sum = 33ab3ef79aa1146b83b778210e7b0a54
 configure-options =
   --disable-static

component/zodbtools/buildout.cfg

@@ -4,6 +4,7 @@
 [buildout]
 extends =
     ../pygolang/buildout.cfg
+    ../ZODB/buildout.cfg
     ../ZEO/buildout.cfg
 
 parts =
@@ -19,6 +20,7 @@ eggs    =
 # dependent eggs that must come through in-tree recipes
 depends =
     ${pygolang:egg}
+    ${ZODB:egg}
 # ZEO comes through zodbtools -> zodburi -> ZEO
     ${ZEO:egg}
 

component/zodbtools/test-common.cfg

@@ -3,8 +3,7 @@
 
 [buildout]
 extends =
-# test*.cfg first extend from neoppod/software<ZODB-flabour>.cfg to use
-# appropriate ZODB and versions of other components.
+# test<X>.cfg configures ZODB.major=<X>.
 
     ../../stack/nxdtest.cfg
     ../pytest/buildout.cfg
@@ -45,4 +44,5 @@ template = inline:
 
 
 [versions]
+mock = 3.0.5
 freezegun = 1.0.0

component/zodbtools/test-zodb4-wc2.cfg

+# SlapOS software release to test zodbtools/ZODB4-wc2 on Nexedi testing infrastructure.
+
+[buildout]
+extends = test-common.cfg
+
+[ZODB]
+major = 4-wc2

component/zodbtools/test-zodb5.cfg

 # SlapOS software release to test zodbtools/ZODB5 on Nexedi testing infrastructure.
 
 [buildout]
-extends =
-    ../../software/neoppod/software-zodb5.cfg
-    test-common.cfg
+extends = test-common.cfg
+
+[ZODB]
+major = 5

component/zodbtools/test.cfg

 # SlapOS software release to test zodbtools on Nexedi testing infrastructure.
 
 [buildout]
-extends =
-    ../../software/neoppod/software.cfg
-    test-common.cfg
+extends = test-common.cfg

component/zstd/buildout.cfg

@@ -10,8 +10,8 @@ parts =
 recipe = slapos.recipe.cmmi
 shared = true
 url = https://github.com/facebook/zstd/releases/download/v${:version}/zstd-${:version}.tar.gz
-version = 1.4.5
-md5sum = dd0b53631303b8f972dafa6fd34beb0c
+version = 1.4.9
+md5sum = eb718b8aae0302cabe20f968e500534d
 location = @@LOCATION@@
 configure-command = :
 environment =

software/erp5/test/test/test_balancer.py

@@ -949,10 +949,10 @@ class TestPathBasedRouting(BalancerTestCase):
         expected_path,
       )
     # Trailing slash presence is preserved.
-    assertRoutingEqual('default', '/foo/bar',       prefix + '/erp5/boo/far/faz' + vhr + '/_vh_foo/bar')
-    assertRoutingEqual('default', '/foo/bar/',      prefix + '/erp5/boo/far/faz' + vhr + '/_vh_foo/bar/')
+    assertRoutingEqual('default', '/foo/bar',       prefix + '/erp5/boo/far/faz' + vhr + '/_vh_foo/_vh_bar')
+    assertRoutingEqual('default', '/foo/bar/',      prefix + '/erp5/boo/far/faz' + vhr + '/_vh_foo/_vh_bar/')
     # Subpaths are preserved.
-    assertRoutingEqual('default', '/foo/bar/hey',   prefix + '/erp5/boo/far/faz' + vhr + '/_vh_foo/bar/hey')
+    assertRoutingEqual('default', '/foo/bar/hey',   prefix + '/erp5/boo/far/faz' + vhr + '/_vh_foo/_vh_bar/hey')
     # Rule precedence: later less-specific rules are applied.
     assertRoutingEqual('default', '/foo',           prefix + '/erp5/somewhere' + vhr + '/_vh_foo')
     assertRoutingEqual('default', '/foo/',          prefix + '/erp5/somewhere' + vhr + '/_vh_foo/')

software/erp5testnode/buildout.hash.cfg

@@ -18,4 +18,4 @@ md5sum = 307663d73ef3ef94b02567ecd322252e
 
 [template-default]
 filename = instance-default.cfg
-md5sum = e553b582cd51db90fcd5634bd055f042
+md5sum = d10958c62d0be8d0d051300d695f4f44

software/erp5testnode/instance-default.cfg

@@ -258,5 +258,5 @@ config-url = $${testnode-log-frontend:connection-secure_access}
 node-quantity = 1
 test-suite-master-url =
 instance-dict =
-software-path-list = ["https://lab.nexedi.com/nexedi/slapos/raw/1.0.154/software/seleniumrunner/software.cfg"]
+software-path-list = ["https://lab.nexedi.com/nexedi/slapos/raw/1.0.181/software/seleniumrunner/software.cfg"]
 keep-log-days = 15

software/galene/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [instance-cfg]
 filename = instance.cfg.in
-md5sum = e54af429aba9b395c13471de254c4621
+md5sum = cfb389ef50e1d5e359fef4686dac097d

software/galene/instance.cfg.in

@@ -61,8 +61,7 @@ template = inline:{
     "public":true,
     "op": [{"username":"$${admin-password:username}","password":"$${admin-password:passwd}"}],
     "other": [],
-    "presenter": [{"username": "", "password": "nexedi"}],
-    "max-users":100
+    "presenter": [{"username": "", "password": "nexedi"}]
   }
 rendered = $${directory:groups}/public.json
 
@@ -74,8 +73,8 @@ template =
   inline:#!/bin/sh
   
   ulimit -n $(ulimit -Hn)
-  exec ${galene:location}/bin/galene \
-    -static ${galene-repository:location}/static \
+  exec ${gowork:bin}/galene \
+    -static ${galene:location}/static \
     -recordings $${directory:recordings} \
     -groups $${directory:groups} \
     -data $${directory:data} \

software/galene/software.cfg

@@ -7,7 +7,6 @@ extends =
 
 parts =
   slapos-cookbook
-  galene
   eggs
   instance-cfg
 
@@ -19,25 +18,18 @@ eggs    =
     collective.recipe.template
 
 
-[galene-repository]
-<= git-repository
+[galene]
+<= go-git-package
+go.importpath = lab.nexedi.com/nexedi/galene
 repository = https://lab.nexedi.com/nexedi/galene.git
 revision = galene-0.3
-location = ${buildout:parts-directory}/galene-repository
 
-[galene]
-recipe = slapos.recipe.cmmi
-path = ${galene-repository:location}
-location = @@LOCATION@@
-golang  = ${golang1.13:location}
-configure-command = true
-make-binary = 
-make-targets =
-  go install -ldflags='-s -w'
+[gowork]
+install =
+    ${galene:location}:./...
 environment =
-  PATH=${:location}/bin:${:golang}/bin:${git:location}/bin:${pkgconfig:location}/bin:$PATH
-  GOPATH=${:location}
-  CGO_ENABLED=0
+   CGO_ENABLED = 0
+buildflags = -ldflags='-s -w'
 
 [instance-cfg]
 recipe = slapos.recipe.template

software/html5as/test/README.md

+Tests for html5as software release

software/html5as/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.html5as'
+with open("README.md") as f:
+  long_description = f.read()
+
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS' HTML5AS",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        'requests',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)

software/html5as/test/test.py

+##############################################################################
+#
+# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import requests
+from urlparse import urlparse
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class HTML5ASTestCase(SlapOSInstanceTestCase):
+  """
+  Common class for testing html5as.
+  It inherit from SlapOSInstanceTestCase which:
+    * Install the software release.
+    * Checks it compile without issue.
+    * Deploy the instance
+    * Check deployement works and promise pass
+  For testing the deployement a different testing class will need to be set up
+  per each variation of parameters the instance needs to be given.
+  """
+
+  def checkUrlAndGetResponse(self, url):
+    """
+    Common class to check an url and return the response
+    """
+    response = requests.get(url)
+    self.assertEqual(requests.codes['OK'], response.status_code)
+    return response
+
+
+class TestEmptyDeploy(HTML5ASTestCase):
+  """
+  This class test the instance with no parameters.
+  """
+
+  def test_deploy_with_no_paramater(self):
+    url = self.requestDefaultInstance().getConnectionParameterDict()['server_url']
+    response = self.checkUrlAndGetResponse(url)
+    result = response.text
+    self.assertFalse("<h1>" in result)
+    self.assertTrue("<p>Hello World</p>" in result)
+
+class TestDeployWithTitle(HTML5ASTestCase):
+  """
+  This class test an instance with the parameter "title"
+  """
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'title': 'Test1',
+    }
+
+  def test_deploy_with_title_parameter(self):
+    connection_parameter_dict = self.computer_partition.getConnectionParameterDict()
+    self.assertEqual(connection_parameter_dict["title"], "Title Test1!")
+    url = connection_parameter_dict['server_url']
+    response = self.checkUrlAndGetResponse(url)
+    result = response.text
+    self.assertTrue("<h1>Test1</h1>" in result)
+    self.assertTrue("<p>Hello World</p>" in result)
+
+class TestGracefulWithPortChange(HTML5ASTestCase):
+  """
+  This class test the instance with the parameter "port"
+  """
+
+  instance_parameter_dict = {
+    'port': 8087
+  }
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return cls.instance_parameter_dict
+
+  def test_change_port_parameter(self):
+    """
+    This test test port change and its application with graceful restart
+    """
+    # Check initial connection parameter match expected port
+    url = self.computer_partition.getConnectionParameterDict()['server_url']
+    self.assertEqual(urlparse(url).port, 8087)
+    # Check port is listening even thought it is duplicated with the promise:
+    # "port-listening-promise"
+    self.checkUrlAndGetResponse(url)
+
+    # Update port parameter
+    self.instance_parameter_dict.update({
+      'port': 8086,
+
+    })
+    # Request instance with the new port parameter
+    self.requestDefaultInstance()
+    # Reprocess the instance to apply new port and run promises
+    self.slap.waitForInstance(self.instance_max_retry)
+    # Rerequest instance to get update connection parameter
+    url = self.requestDefaultInstance().getConnectionParameterDict()['server_url']
+    # Make sure the new port is the one being used
+    self.assertEqual(urlparse(url).port, 8086)
+
+    # Check port is listening even thought it is duplicated with the promise:
+    # "port-listening-promise"
+    self.checkUrlAndGetResponse(url)
+
+

software/neoppod/software-common.cfg

@@ -18,6 +18,7 @@ extends =
     ../../component/python-mysqlclient/buildout.cfg
     ../../component/python-cryptography/buildout.cfg
     ../../component/pycurl/buildout.cfg
+    ../../component/ZODB/buildout.cfg
     ../../component/ZEO/buildout.cfg
     ../../component/zodbtools/buildout.cfg
 
@@ -47,27 +48,14 @@ eggs = neoppod[admin, ctl, master]
   ${cython-zstd:egg}
   ${msgpack-python:egg}
   ${:adapter-egg}
-  ${BTrees:egg}
+  ${ZODB:egg}
+  ${zodbtools:egg}
   psutil
-  ZODB
   zope.testing
-  ${zodbtools:egg}
   coverage
   setproctitle
 adapter-egg = ${python-mysqlclient:egg}
 
-# XXX: buildout fails to install properly eggs with setup_requires
-[BTrees]
-recipe = zc.recipe.egg:custom
-egg = ${:_buildout_section_name_}
-setup-eggs = ${persistent:egg}
-
-[persistent]
-recipe = zc.recipe.egg:custom
-egg = ${:_buildout_section_name_}
-setup-eggs = ${python-cffi:egg}
-###
-
 [download-base-neo]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:filename}
@@ -136,18 +124,13 @@ template =
   {% endraw %}
 
 [versions]
-BTrees = 4.5.1
-ZODB = 4.4.5
 coverage = 4.5.1
 mock = 3.0.5
 ecdsa = 0.13
 mysqlclient = 1.3.12
-persistent = 4.6.4
 pycrypto = 2.6.1
 pycurl = 7.43.0
 setproctitle = 1.1.10
-transaction = 1.7.0
-zodbpickle = 1.0.4
 cython-zstd = 0.2
 python-dateutil = 2.7.3
 

software/neoppod/software-zodb5.cfg

 [buildout]
 extends = software.cfg
 
+[ZODB]
+major = 5
+
 [neoppod]
 eggs += mock
-
-[ZEO]
-egg = ${ZEO5:egg}
-
-[versions]
-ZODB = 5.6.0
-ZEO = 5.2.2
-transaction = 2.4.0
-
-# Required by:
-# ZEO==5.2.0
-# trollius==2.2.post1
-futures = 3.2.0
-
-# Required by:
-# ZEO==5.2.0
-trollius = 2.2.post1

software/neoppod/software.cfg

@@ -34,8 +34,6 @@ interpreter = ${:_buildout_section_name_}
 
 [versions]
 # To match ERP5
-ZConfig = 2.9.3
-zc.lockfile = 1.0.2
 zope.event = 3.5.2
 zope.exceptions = 3.6.2
 zope.testing = 3.9.7

software/neotest/software.cfg

@@ -131,20 +131,4 @@ eggs    =
 
 # ping eggs versions
 [versions]
-ZODB3 = 3.11.0
 zope.testing = 4.6.2
-
-# Required by:
-# ZEO==4.3.1
-# ZODB==4.4.5
-# zodburi==2.3.0
-ZConfig = 3.2.0
-
-# Required by:
-# ZEO==4.3.1
-# ZODB==4.4.5
-zc.lockfile = 1.2.1
-
-# Required by:
-# neoppod==1.8.1
-python-dateutil = 2.7.1

software/slapos-master/software.cfg

@@ -6,6 +6,15 @@ extends =
 parts +=
   vifib-fix-products-paths
 
+
+# SlapOS Master needs to use Wendelin.core 2.
+# Wendelin.core 2 works with either ZODB4-wc2, or ZODB5, but not with ZODB4.
+# ERP5 does not currently work with ZODB5.
+# -> this leaves ZODB4-wc2 as the only choice.
+[ZODB]
+major = 4-wc2
+
+
 [local-bt5-repository]
 # Same as bt5-repository, but only local repository.
 # Used to generate bt5lists.

software/slapos-sr-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 9639d0c0e161c094454808fb95fc9781
+md5sum = 5d1320a9b1d2828f2870e3096f1c2bfb

software/slapos-sr-testing/instance.cfg

 [buildout]
 extends = {{ nxdtest_instance }}
-parts =
-  runTestSuite
+parts +=
   publish
 
-eggs-directory = {{ buildout['eggs-directory'] }}
-develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
-offline = true
-
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration
 computer = ${slap-connection:computer-id}
@@ -36,7 +31,7 @@ nxdtest-working-dir = ${:var}/nxdtest
 recipe = slapos.recipe.template:jinja2
 rendered = ${directory:etc}/${:_buildout_section_name_}
 template = inline:
-  export PATH={{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:{{ faketime_location }}/bin/:{{ openssl_location }}/bin/:/usr/bin/:/bin
+  export PATH={{ buildout['bin-directory'] }}:{{ curl_location }}/bin:{{ faketime_location }}/bin:{{ openssl_location }}/bin:/usr/bin:/bin
   export SLAPOS_TEST_IPV4=${slap-configuration:ipv4-random}
   export SLAPOS_TEST_IPV6=${slap-configuration:ipv6-random}
   export SLAPOS_TEST_WORKING_DIR=${directory:test-working-dir}

software/slapos-sr-testing/software.cfg

@@ -119,6 +119,11 @@ setup = ${slapos-repository:location}/software/metabase/test/
 egg = slapos.test.helloworld
 setup = ${slapos-repository:location}/software/helloworld/test/
 
+[slapos.test.html5as-setup]
+<= setup-develop-egg
+egg = slapos.test.html5as
+setup = ${slapos-repository:location}/software/html5as/test/
+
 [slapos.test.jupyter-setup]
 <= setup-develop-egg
 egg = slapos.test.jupyter
@@ -224,6 +229,7 @@ extra-eggs =
   ${slapos.test.repman-setup:egg}
   ${slapos.test.caucase-setup:egg}
   ${slapos.test.jscrawler-setup:egg}
+  ${slapos.test.html5as-setup:egg}
 
 # We don't name this interpreter `python`, so that when we run slapos node
 # software, installation scripts running `python` use a python without any
@@ -295,6 +301,7 @@ extra =
   dream ${slapos.test.dream-setup:setup}
   caucase ${slapos.test.caucase-setup:setup}
   jscrawler ${slapos.test.jscrawler-setup:setup}
+  html5as ${slapos.test.html5as-setup:setup}
 
 [versions]
 # slapos.core is used from the clone always

software/slapos-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 3b1b386f6ad4c9ac50ab1f1e1384e751
+md5sum = a6091981a78cb2b10a97e431adc7615c

software/slapos-testing/instance.cfg

 [buildout]
 extends = ${nxdtest-instance.cfg:rendered}
-parts =
-  runTestSuite
+parts +=
   publish
 
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-offline = true
-
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration
 computer = $${slap-connection:computer-id}

software/slaprunner/buildout.hash.cfg

@@ -50,7 +50,7 @@ md5sum = 7a2f5e25a818cb29964666ada7852a5c
 
 [template-slapos-cfg]
 filename = template/slapos.cfg.in
-md5sum = fa90fc9a9010ce4bb1478ae37da9c56d
+md5sum = 3b228b26a170f3c07c50a0f2ae056daf
 
 [template-slapformat-definition.cfg]
 filename = template/slapformat-definition.cfg.in

software/slaprunner/template/slapos.cfg.in

@@ -25,7 +25,7 @@ input_definition_file = {{ slaprunner['slapformat-definition.cfg'] }}
 partition_amount = {{ slaprunner['partition-amount'] }}
 alter_user = false
 alter_network = false
-create_tab = false
+create_tap = false
 create_tun = false
 computer_xml = {{ slaprunner['etc_dir'] }}/slapos.xml
 

software/theia/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = 4f119df2b2d71e5ec1fd04051fd8fff8
+md5sum = a82e4fdc4efd60df699ee072399e6973
 
 [yarn.lock]
 filename = yarn.lock

software/theia/generate_download_plugins_cfg.py

@@ -78,7 +78,8 @@ for plugin_and_version in '''\
     EditorConfig/EditorConfig/latest
     dbaeumer/vscode-eslint/latest
     ms-vscode/references-view/latest
-    golang/Go/0.16.2
+# golang.Go removed because it overwrites the PATH in theia shell
+# golang/Go/0.16.2
     vscjava/vscode-java-debug/0.29.0
     redhat/java/0.61.0
     vscjava/vscode-java-test/0.26.0

software/theia/instance-input-schema.json

@@ -4,8 +4,8 @@
   "additionalProperties": false,
   "properties": {
     "autorun": {
-      "title": "autorun",
-      "description": "State of the autorun service",
+      "title": "Automatically Run Sofware/Instance",
+      "description": "The option used to pilot automatic build and run of software and instances hosted inside Theia. When 'running', build and run is done automatically in the background. When 'stopped' build and run processes are stopped and need to be run manually. When 'disabled', the user can manage it directly in the local supervisord. ",
       "type": "string",
       "enum": [
         "running",

software/theia/instance-output-schema.json

@@ -13,6 +13,10 @@
     "password": {
       "description": "Theia password",
       "type": "string"
+    },
+    "backend-url": {
+      "description": "Theia Backend URL",
+      "type": "string"
     }
   },
   "type": "object"

software/theia/instance.cfg.in

@@ -7,6 +7,7 @@ theia-environment-parts =
   slapos-repository
   runner-link
   settings.json
+  request-script
 
 theia-parts =
   frontend-reload
@@ -41,6 +42,7 @@ recipe = slapos.cookbook:publish
 url = $${apache-frontend:connection-secure_access}
 username = $${frontend-instance-password:username}
 password = $${frontend-instance-password:passwd}
+backend-url = $${frontend-instance:url}
 
 
 [directory]
@@ -136,7 +138,7 @@ return = domain secure_access
 
 [frontend-instance-password]
 recipe = slapos.cookbook:generate.password
-username = node
+username = admin
 bytes = 12
 
 [frontend-instance-port]
@@ -277,6 +279,23 @@ location = $${directory:frontend-static}/$${:filename}
 filename = $${:_buildout_section_name_}
 
 
+# Common Environment
+# ------------------
+
+[common-environment]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
+mode = 0777
+template =
+  inline:
+  #!/bin/sh
+  . ${gowork:env.sh}
+  # reset PS1 from gowork
+  export PS1='$ '
+  export HOME=$${buildout:directory}
+  export PATH=${python-language-server:location}/bin:${java-jdk:location}/bin:${cli-utilities:PATH}:$HOME/.cargo/bin:$PATH
+
+
 # Theia Backend
 # -------------
 
@@ -298,13 +317,14 @@ template =
   export THEIA_OPEN_EDITOR_TOKEN=$(${openssl:location}/bin/openssl rand -hex 32)
   export THEIA_URL=$${:base-url}
   export THEIA_SHELL=$${theia-shell:rendered}
-  export HOME=$${buildout:directory}
   export TMP=$${directory:tmp}
   export TEMP=$TMP
   export LC_ALL=C.UTF-8
   export TERMINFO=${ncurses:location}/lib/terminfo/
   export EDITOR="${python-language-server:location}/bin/python -m theia_open --wait"
-  exec ${theia-wrapper:rendered} $@
+  export THEIA_DEFAULT_PLUGINS="local-dir:${theia-plugins:location}"
+  . $${common-environment:rendered}
+  exec ${theia-wrapper:rendered} "$@"
 ip =  $${theia-service-port:ip}
 port = $${theia-service-port:port}
 base-url = http://$${:ip}:$${:port}/
@@ -326,16 +346,22 @@ recipe = slapos.recipe.template:jinja2
 rendered = $${directory:bin}/$${:_buildout_section_name_}
 mode = 0700
 template = inline:
-  #!${python:location}/bin/python
-  import os
-  import sys
-  import time
-  args = sys.argv[1:]
+  #!{{ bash }}
+  SHELL=$BASH
   # when running interactively, activate slapos configuration and reset GIT_EXEC_PATH to workaround https://github.com/eclipse-theia/theia/issues/7555
-  if not args: args = ["-c", ". $${slapos-standalone-activate:rendered} && exec env GIT_EXEC_PATH= ${bash:location}/bin/bash --rcfile $${theia-bashrc:rendered}", ]
+  if [ $# = 0 ]; then
+    . {{ activate }}
+    unset GIT_EXEC_PATH
+    set -- --rcfile {{ bashrc }}
   # otherwise, assume this shell is running task and add an artificial delay to workaround https://github.com/eclipse-theia/theia/issues/2961
-  else: time.sleep(1)
-  os.execv('${bash:location}/bin/bash', ['${bash:location}/bin/bash']  + args)
+  else
+    sleep 1
+  fi
+  exec "$SHELL" "$@"
+context =
+  raw bash ${bash:location}/bin/bash
+  key activate slapos-standalone-activate:rendered
+  key bashrc theia-bashrc:rendered
 
 [theia-bashrc]
 recipe = slapos.recipe.template:jinja2
@@ -376,9 +402,8 @@ computer-id = slaprunner
 [slapos-standalone-activate]
 recipe = slapos.recipe.template:jinja2
 rendered = $${directory:bin}/$${:_buildout_section_name_}
-mode = 0700
 template =
-  inline:#!/bin/sh
+  inline:
   export PATH=${buildout:bin-directory}:$PATH
   export SLAPOS_CONFIGURATION=$${slapos-standalone-config:slapos-configuration}
   export SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION
@@ -390,7 +415,10 @@ rendered = $${directory:bin}/$${:_buildout_section_name_}
 mode = 0700
 template =
   inline:#!/bin/sh
-  export PATH=${buildout:bin-directory}:$PATH
+  . $${common-environment:rendered}
+  #XXX find out where the extra nodejs in theia's PATH comes from
+  export PATH=${nodejs:location}/bin/:$PATH
+  . $${slapos-standalone-activate:rendered}
   exec ${slapos-standalone:script-path} \
       $${directory:runner} \
       $${slapos-standalone-config:ipv4} \
@@ -524,3 +552,18 @@ template =
 recipe = slapos.cookbook:symbolic.link
 target-directory = $${directory:project}
 link-binary = $${directory:runner}
+
+[request-script]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:project}/$${:_buildout_section_name_}.sh
+mode = 0700
+template =
+  inline:#!/bin/sh
+  software_name=html5as-base #replace the software name
+  software_release_uri=~/srv/project/slapos/software/$software_name/software.cfg
+  # slapos supply is used to add the software to the software list to be supplied to a node.
+  slapos supply $software_release_uri slaprunner
+  # slapos request the allocation of an instance to the master. 
+  # slapos request also gets status and parameters of the instance if it has any 
+  # (slapos request is meant to be run multiple time until you get the status).
+  slapos request $software_name'_1' $software_release_uri

software/theia/software.cfg

@@ -299,7 +299,8 @@ install +=
 
 
 [cli-utilities]
-PATH = ${nodejs:location}/bin/:${bash:location}/bin/:${fish-shell:location}/bin/:${tig:location}/bin/:${vim:location}/bin/:${tmux:location}/bin/:${git:location}/bin/:${curl:location}/bin:${python2.7:location}/bin/:${buildout:bin-directory}
+PATH = ${nodejs:location}/bin:${bash:location}/bin:${fish-shell:location}/bin:${tig:location}/bin:${vim:location}/bin:${tmux:location}/bin:${git:location}/bin:${curl:location}/bin:${python2.7:location}/bin:${buildout:bin-directory}
+
 
 [theia-wrapper]
 recipe = slapos.recipe.template:jinja2
@@ -307,14 +308,9 @@ rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
 mode = 0777
 template =
   inline:
-  #!/bin/bash
-  . ${gowork:env.sh}
-  export PATH=${python-language-server:location}/bin/:${java-jdk:location}/bin/:${cli-utilities:PATH}:$HOME/.cargo/bin:$PATH
-  export THEIA_DEFAULT_PLUGINS="local-dir:${theia-plugins:location}"
-  # reset PS1 from gowork
-  export PS1='$ '
+  #!/bin/sh
   cd ${theia:location}
-  exec ${yarn:location}/bin/yarn theia start $@
+  exec ${yarn:location}/bin/yarn theia start "$@"
 
 
 [instance]

software/theia/test/dummy/README.md

+# Dummy Software
+
+A very simple SR that dumps the environment in JSON format during installation.

software/theia/test/dummy/software.cfg

+[buildout]
+
+newest = false
+
+versions = versions
+
+parts =
+  log-env
+
+[log-env]
+recipe = slapos.recipe.build
+init =
+  import json
+  import os
+  with open("${buildout:directory}/../env.json", 'w') as f:
+    json.dump(dict(os.environ), f)
+
+[versions]
+slapos.recipe.build = 0.46

software/theia/test/test.py

@@ -33,6 +33,7 @@ import subprocess
 import tempfile
 import time
 import re
+import json
 from six.moves.urllib.parse import urlparse, urljoin
 
 import pexpect
@@ -50,11 +51,36 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
         os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
 
 
-class TestTheia(SlapOSInstanceTestCase):
-  __partition_reference__ = 'T' # for sockets in included slapos
+class TheiaTestCase(SlapOSInstanceTestCase):
+  __partition_reference__ = 'T' # for supervisord sockets in included slapos
+
+  @classmethod
+  def _getSlapos(cls):
+    partition_root = cls.computer_partition_root_path
+    slapos = os.path.join(partition_root, 'srv', 'runner', 'bin', 'slapos')
+    return slapos
+
+
+class TestTheia(TheiaTestCase):
   def setUp(self):
     self.connection_parameters = self.computer_partition.getConnectionParameterDict()
 
+  def test_backend_http_get(self):
+    resp = requests.get(self.connection_parameters['backend-url'], verify=False)
+    self.assertEqual(requests.codes.unauthorized, resp.status_code)
+
+    # with login/password, this is allowed
+    parsed_url = urlparse(self.connection_parameters['backend-url'])
+    authenticated_url = parsed_url._replace(
+        netloc='{}:{}@[{}]:{}'.format(
+            self.connection_parameters['username'],
+            self.connection_parameters['password'],
+            parsed_url.hostname,
+            parsed_url.port,
+        )).geturl()
+    resp = requests.get(authenticated_url, verify=False)
+    self.assertEqual(requests.codes.ok, resp.status_code)
+
   def test_http_get(self):
     resp = requests.get(self.connection_parameters['url'], verify=False)
     self.assertEqual(requests.codes.unauthorized, resp.status_code)
@@ -152,10 +178,17 @@ class TestTheia(SlapOSInstanceTestCase):
     ])
     self.assertTrue(os.path.exists(test_file))
 
+  def test_theia_request_script(self):
+    script_path = os.path.join(
+      self.computer_partition_root_path,
+      'srv',
+      'project',
+      'request-script.sh',
+    )
+    self.assertTrue(os.path.exists(script_path))
 
-class TestTheiaEmbeddedSlapOSShutdown(SlapOSInstanceTestCase):
-  __partition_reference__ = 'T' # for sockets in included slapos
 
+class TestTheiaEmbeddedSlapOSShutdown(TheiaTestCase):
   def test_stopping_instance_stops_embedded_slapos(self):
     embedded_slapos_supervisord_socket = _getSupervisordSocketPath(
         os.path.join(
@@ -187,57 +220,95 @@ class TestTheiaEmbeddedSlapOSShutdown(SlapOSInstanceTestCase):
     self.assertFalse(embedded_slapos_process.is_running())
 
 
-class SQLiteDB(object):
-  def __init__(self, sqlitedb_file):
-    self.sqlitedb_file = sqlitedb_file
-
-  def select(self, fields, table, where={}):
-    connection = sqlite3.connect(self.sqlitedb_file)
-
-    def dict_factory(cursor, row):
-      d = {}
-      for idx, col in enumerate(cursor.description):
-        d[col[0]] = row[idx]
-      return d
-    connection.row_factory = dict_factory
-    cursor = connection.cursor()
-
-    condition = " AND ".join("%s='%s'" % (k, v) for k, v in where.items())
-    cursor.execute(
-      "SELECT %s FROM %s%s"
-      % (
-        ", ".join(fields),
-        table,
-        " WHERE %s" % condition if where else "",
-      )
-    )
-    return cursor.fetchall()
+class TestTheiaWithSR(TheiaTestCase):
+  sr_url = 'bogus/software.cfg'
+  sr_type = 'bogus_type'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'embedded-sr': cls.sr_url,
+      'embedded-sr-type': cls.sr_type,
+    }
 
+  def test(self):
+    slapos = self._getSlapos()
+    info = subprocess.check_output((slapos, 'proxy', 'show'))
+    instance_name = "Embedded Instance"
+
+    self.assertIsNotNone(re.search(r"%s\s+slaprunner\s+available" % (self.sr_url,), info), info)
+    self.assertIsNotNone(re.search(r"%s\s+%s\s+%s" % (self.sr_url, self.sr_type, instance_name), info), info)
 
-class TestTheiaWithSR(SlapOSInstanceTestCase):
-  __partition_reference__ = 'T' # for sockets in included slapos
 
-  srurl = 'bogus/software.cfg'
-  srtype = 'bogus'
+class TestTheiaEnv(TheiaTestCase):
+  dummy_software_path = os.path.abspath('dummy/software.cfg')
 
   @classmethod
   def getInstanceParameterDict(cls):
     return {
-      'embedded-sr': cls.srurl,
-      'embedded-sr-type': cls.srtype,
+      'embedded-sr': cls.dummy_software_path,
+      'autorun': 'stopped',
     }
 
-  def test(self):
-    db = SQLiteDB(os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'var', 'proxy.db'))
-    supplied = db.select(
-      fields=["*"],
-      table = "software14",
-      where={'url': self.srurl}
-    )
-    self.assertEqual(len(supplied), 1)
-    requested = db.select(
-      fields=["*"],
-      table = "partition14",
-      where={'software_release': self.srurl, 'software_type': self.srtype}
-    )
-    self.assertEqual(len(requested), 1)
+  def test_theia_env(self):
+    # The path of the env.json file expected to be generated by building the dummy software release
+    env_json_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'software', 'env.json')
+
+    # Get the pid of the theia process from the test node's instance-supervisord
+    with self.slap.instance_supervisor_rpc as supervisor:
+      all_process_info = supervisor.getAllProcessInfo()
+      for p in all_process_info:
+        if p['name'].startswith('theia-instance'):
+          theia_process = p
+          break
+      else:
+        self.fail("Could not find theia process")
+    theia_pid = theia_process['pid']
+
+    # Get the environment of the theia process
+    theia_env = psutil.Process(theia_pid).environ()
+
+    # Start a theia shell that inherits the environment of the theia process
+    # This simulates the environment of a shell launched from the browser application
+    theia_shell_process = pexpect.spawnu('{}/bin/theia-shell'.format(self.computer_partition_root_path), env=theia_env)
+    theia_shell_process.expect_exact('Standalone SlapOS for computer `slaprunner` activated')
+
+    # Launch slapos node software from theia shell
+    theia_shell_process.sendline('slapos node software')
+    theia_shell_process.expect('Installing software release %s' % self.dummy_software_path)
+    theia_shell_process.expect('Finished software releases.')
+
+    # Get the theia shell environment
+    with open(env_json_path) as f:
+      theia_shell_env = json.load(f)
+
+    # Remove the env.json file to later be sure that a new one has been generated
+    os.remove(env_json_path)
+
+    # Launch slapos-node-software from the embedded supervisord
+    embedded_run_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'var', 'run')
+    embedded_supervisord_socket_path = _getSupervisordSocketPath(embedded_run_path, self.logger)
+    with getSupervisorRPC(embedded_supervisord_socket_path) as embedded_supervisor:
+      previous_stop_time = embedded_supervisor.getProcessInfo('slapos-node-software')['stop']
+      embedded_supervisor.startProcess('slapos-node-software')
+      for _retries in range(20):
+        time.sleep(1)
+        if embedded_supervisor.getProcessInfo('slapos-node-software')['stop'] != previous_stop_time:
+          break
+      else:
+        self.fail("the supervisord service 'slapos-node-software' takes too long to finish")
+
+    # Get the supervisord environment
+    with open(env_json_path) as f:
+      supervisord_env = json.load(f)
+
+    # Compare relevant variables from both environments
+    self.maxDiff = None
+    self.assertEqual(theia_shell_env['PATH'].split(':'), supervisord_env['PATH'].split(':'))
+    self.assertEqual(theia_shell_env['SLAPOS_CONFIGURATION'], supervisord_env['SLAPOS_CONFIGURATION'])
+    self.assertEqual(theia_shell_env['SLAPOS_CLIENT_CONFIGURATION'], supervisord_env['SLAPOS_CLIENT_CONFIGURATION'])
+    self.assertEqual(theia_shell_env['HOME'], supervisord_env['HOME'])
+
+    # Cleanup the theia shell process
+    theia_shell_process.terminate()
+    theia_shell_process.wait()

stack/caucase/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [caucase-jinja2-library]
 filename = caucase.jinja2.library
-md5sum = 2e7e61bb0cf41c28d6d811a0283cf03e
+md5sum = b8a5a93b8a7bacd6ebd4b6df24c7c828

stack/caucase/caucase.jinja2.library

@@ -88,7 +88,10 @@ rendered = ${ {{- prefix }}-directory:data-dir}/provided.csr.pem
 {%-   else -%}
 [{{ prefix }}-csr]
 recipe = plone.recipe.command
-command = '{{ openssl }}' req -newkey rsa:2048 -batch -new -nodes -subj /CN=example.com -keyout '{{ key_path or crt_path }}' -out '${:csr}'
+command =
+  if [ ! -f '{{ key_path or crt_path }}' ] && [ ! -f '${:csr}' ] ; then
+    '{{ openssl }}' req -newkey rsa:2048 -batch -new -nodes -subj /CN=example.com -keyout '{{ key_path or crt_path }}' -out '${:csr}'
+  fi
 {%-   endif %}
 csr = ${ {{- prefix }}-directory:data-dir}/good.csr.pem
 {%- endif %}
@@ -124,7 +127,7 @@ recipe = plone.recipe.command
 checksum-file = '{{ template }}.md5'
 command =
   set -e
-  md5_current=$(${buildout:executable} -c "import hashlib ; hashlib.md5(open('{{ template }}').read()).hexdigest()")
+  md5_current=$(${buildout:executable} -c "import hashlib ; hashlib.md5(open('{{ template }}', 'rb').read()).hexdigest()")
   md5_old=$([ -f ${:checksum-file} ] && cat ${:checksum-file} || echo none)
   if [ "$md5_current" != "$md5_old" ] || [ ! -f '{{ csr }}' ] || [ ! -f '{{ key }}' ] ; then
     '{{ buildout_bin_directory }}/caucase-rerequest' --template '{{ template }}' --csr '{{ csr }}' --key '{{ key }}'

stack/erp5/buildout.cfg

@@ -260,6 +260,7 @@ link-binary =
   ${poppler:location}/bin/pdfinfo
   ${poppler:location}/bin/pdftohtml
   ${poppler:location}/bin/pdftotext
+  ${python2.7:location}/bin/2to3
   ${sed:location}/bin/sed
   ${tesseract:location}/bin/tesseract
   ${w3m:location}/bin/w3m
@@ -664,7 +665,6 @@ PyXML = 0.8.5
 Pympler = 0.4.3
 StructuredText = 2.11.1
 WSGIUtils = 0.7
-ZODB3 = 3.11.0
 # astroid 1.4.1 breaks testDynamicClassGeneration
 astroid = 1.3.8
 erp5diff = 0.8.1.7

stack/erp5/buildout.hash.cfg

@@ -90,7 +90,7 @@ md5sum = 8d3694226b6cbed961f6d608b6d6d294
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in
-md5sum = 452c502fabd5a6066c9dee533dfb1c77
+md5sum = 3f4f7e49c504cbf610fc5dc462713dfc
 
 [template-rsyslogd-cfg]
 filename = rsyslogd.cfg.in

stack/erp5/haproxy.cfg.in

@@ -176,7 +176,7 @@ listen family_{{ name }}
 
 {%   for outer_prefix, inner_prefix in family_path_routing_dict.get(name, []) + path_routing_list %}
   {%   set outer_prefix = outer_prefix.strip('/') -%}
-  http-request replace-path ^(/+VirtualHostBase/+[^/]+/+[^/]+)/+VirtualHostRoot/+({% if outer_prefix %}{{ outer_prefix }}($|/.*){% else %}.*{% endif %}) \1/{{ inner_prefix.strip('/') }}/VirtualHostRoot/{% if outer_prefix %}_vh_{% endif %}\2
+  http-request replace-path ^(/+VirtualHostBase/+[^/]+/+[^/]+)/+VirtualHostRoot/+{% if outer_prefix %}{{ outer_prefix }}($|/.*){% else %}(.*){% endif %} \1/{{ inner_prefix.strip('/') }}/VirtualHostRoot/{% if outer_prefix %}_vh_{{ outer_prefix.replace('/', '/_vh_') }}{% endif %}\2
 {%  endfor %}
 
 {%   set has_webdav = [] -%}

stack/slapos.cfg

@@ -156,6 +156,7 @@ Importing = 1.10
 MarkupSafe = 1.0
 PyYAML = 5.4.1
 Werkzeug = 1.0.1
+ZConfig = 2.9.3
 asn1crypto = 1.3.0
 atomicwrites = 1.4.0
 backports.functools-lru-cache = 1.6.1
@@ -201,7 +202,7 @@ setuptools-dso = 1.7
 rubygemsrecipe  = 0.3.0
 six = 1.12.0
 slapos.cookbook = 1.0.171
-slapos.core = 1.6.6
+slapos.core = 1.6.8
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.20
@@ -216,6 +217,7 @@ unicodecsv = 0.14.1
 wcwidth = 0.2.5
 wheel = 0.35.1
 xml-marshaller = 1.0.2
+zc.lockfile = 1.0.2
 zdaemon = 4.2.0
 zipp = 1.2.0
 zodburi = 2.4.0