Update Release Candidate

component/curl/buildout.cfg

@@ -8,6 +8,7 @@ extends =
   ../pkgconfig/buildout.cfg
   ../xz-utils/buildout.cfg
   ../zlib/buildout.cfg
+  ../nghttp2/buildout.cfg
 parts =
   curl
 
@@ -42,9 +43,9 @@ configure-options =
   --without-libssh2
   --without-librtmp
   --without-libidn
-  --without-nghttp2
+  --with-nghttp2=${nghttp2:location}
 
 environment =
   PATH=${perl:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig
-  LDFLAGS=-Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib
+  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${nghttp2:location}/lib/pkgconfig
+  LDFLAGS=-Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${nghttp2:location}/lib

component/cython-zstd/buildout.cfg

+[buildout]
+extends =
+  ../zstd/buildout.cfg
+
+parts =
+  cython-zstd
+
+[cython-zstd]
+recipe = zc.recipe.egg:custom
+egg = ${:_buildout_section_name_}
+include-dirs =
+  ${zstd:location}/include
+library-dirs =
+  ${zstd:location}/lib
+rpath =
+  ${:library-dirs}

component/messagepack/buildout.cfg

 [buildout]
 parts = messagepack
 
+# XXX: Do not reuse. It should be trashed along with kumofs.
+# This project was moved to https://github.com/msgpack/msgpack-c
+# A new 'msgpack' component should be created if needed.
+
 [messagepack]
 recipe = slapos.recipe.cmmi
 url = http://downloads.sourceforge.net/project/msgpack/msgpack/cpp/msgpack-0.5.4.tar.gz

component/nghttp2/buildout.cfg

@@ -20,4 +20,4 @@ pre-configure =
   automake
   autoconf
 environment =
-  PATH=${autoconf:location}/bin:${automake:location}/bin:${git:location}/bin:${libtool:location}/bin:${m4:location}/bin:${python2.7:location}/bin:%(PATH)s
+  PATH=${autoconf:location}/bin:${automake:location}/bin:${libtool:location}/bin:${m4:location}/bin:${python2.7:location}/bin:%(PATH)s

component/nodejs/buildout.cfg

@@ -76,7 +76,7 @@ configure-options =
   --openssl-libpath=${openssl:location}/lib
 environment =
   HOME=${buildout:parts-directory}/${:_buildout_section_name_}
-  PATH=${pkgconfig:location}/bin:${patch:location}/bin:%(PATH)s
+  PATH=${pkgconfig:location}/bin:${patch:location}/bin:${python2.7:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig/
   CPPFLAGS=-I${zlib:location}/include
   LDFLAGS=-Wl,-rpath=${openssl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib

component/proftpd/buildout.cfg

+# http://www.proftpd.org/ - Highly configurable GPL-licensed FTP server software
+#
+# Because it uses collective.recipe.environment this components needs slapos.core >= 1.4.7
+# ( grid: do not hide `$USER` when running buildout )
+#
+[buildout]
+extends = 
+  ../openssl/buildout.cfg
+  ../perl/buildout.cfg
+  ../zlib/buildout.cfg
+  ../curl/buildout.cfg
+  ../libtool/buildout.cfg
+  ../git/buildout.cfg
+
+# proftpd server
+[proftpd-environment]
+recipe = collective.recipe.environment
+
+[proftpd-grp]
+recipe = collective.recipe.grp
+
+[proftpd]
+recipe = slapos.recipe.cmmi
+md5sum = 13270911c42aac842435f18205546a1b
+url = ftp://ftp.proftpd.org/distrib/source/proftpd-1.3.6.tar.gz
+configure-options =
+  --enable-openssl
+  --enable-nls
+  --enable-ctrls
+  --enable-dso
+  --with-modules=mod_sftp:mod_ban
+  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
+environment = 
+  CPPFLAGS=${:cppflags}
+  LDFLAGS=${:ldflags}
+  install_user=${proftpd-environment:USER}
+  install_group=${proftpd-grp:GROUP}
+cppflags=-I${zlib:location}/include -I${openssl:location}/include
+ldflags=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib
+
+
+# mod_auth_web: a proftpd module to authenticate users against an HTTP service
+[proftpd-mod_auth_web-repository]
+recipe = slapos.recipe.build:gitclone
+#repository = https://github.com/proftpd/mod_auth_web
+# XXX until https://github.com/proftpd/mod_auth_web/pull/1 gets merged, we use
+# the source repository of this PR directly
+repository = https://github.com/jwm/mod_auth_web
+revision = dec090bd0e287544a34be156ee17f715bd4286f9
+git-executable = ${git:location}/bin/git
+
+[proftpd-mod_auth_web]
+recipe = plone.recipe.command
+stop-on-error = true
+command =
+  LIBTOOL=${libtool:location}/bin/libtool \
+  LDFLAGS="-Wl,-rpath=${curl:location}/lib ${proftpd:ldflags}" \
+  ${proftpd-output:prxs} -c -i \
+  -I ${curl:location}/include ${proftpd:cppflags} \
+  -L ${curl:location}/lib/ \
+  -l curl \
+  -d ${proftpd-mod_auth_web-repository:location}/mod_auth_web.c
+location=${proftpd:location}/libexec/mod_auth_web.so
+
+
+# ftpasswod: a perl script to manage a proftpd AuthUserFile
+[ftpasswd]
+recipe = slapos.recipe.build:download
+url = https://raw.githubusercontent.com/proftpd/proftpd/v1.3.6/contrib/ftpasswd
+md5sum = 4a47df2cab86d8de7077a445bb416f31
+download-only = true
+mode = 0700
+
+
+[proftpd-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:proftpd} -a -x ${:ftpasswd}
+modules-deps =
+  ${proftpd-mod_auth_web:recipe}
+
+perl = ${perl:location}/bin/perl
+ftpasswd = ${ftpasswd:target}
+proftpd = ${proftpd:location}/sbin/proftpd
+prxs = ${proftpd:location}/bin/prxs
+ftpdctl = ${proftpd:location}/bin/ftpdctl

component/unixodbc/buildout.cfg

@@ -6,3 +6,4 @@ parts=
 recipe = slapos.recipe.cmmi
 url = http://www.unixodbc.org/unixODBC-2.3.5.tar.gz
 md5sum = abf14cf943f1f8c5e63a24cb26d54fd9
+post-install = ln -s ${buildout:parts-directory}/${:_buildout_section_name_}/lib/libodbcinst.so ${buildout:parts-directory}/${:_buildout_section_name_}/lib/libodbcinst.so.1

slapos/recipe/erp5_test/__init__.py

@@ -69,6 +69,7 @@ class Recipe(GenericBaseRecipe):
     )
     common_list = (
       '--conversion_server_url=' + cloudooo_url,
+      '--conversion_server_retry_count=%s' % self.options['cloudooo-retry-count'],
       # BBB: We still have test suites that only accept the following 2 options.
       '--conversion_server_hostname=%s' % cloudooo_parsed.hostname,
       '--conversion_server_port=%s' % cloudooo_parsed.port,

slapos/recipe/userinfo.py

@@ -10,13 +10,13 @@ class Recipe(object):
     def __init__(self, buildout, name, options):
         pinfo = pwd.getpwuid(os.getuid())
         options['pw-name'] = pinfo.pw_name
-        options['pw-uid'] = pinfo.pw_uid
-        options['pw-gid'] = pinfo.pw_gid
+        options['pw-uid'] = str(pinfo.pw_uid)
+        options['pw-gid'] = str(pinfo.pw_gid)
         options['pw-dir'] = pinfo.pw_dir
         options['pw-shell'] = pinfo.pw_shell
 
         ginfo = grp.getgrgid(os.getgid())
         options['gr-name'] = ginfo.gr_name
-        options['gr-gid'] = ginfo.gr_gid
+        options['gr-gid'] = str(ginfo.gr_gid)
 
     install = update = lambda self: []

slapos/recipe/wrapper.py

@@ -36,6 +36,7 @@ class Recipe(GenericBaseRecipe):
     :param str wrapper-path: absolute path to file's destination
 
     :param lines wait-for-files: list of files to wait for
+    :param lines hash-files: list of files to be checked by hash
     :param str pidfile: path to pidfile ensure exclusivity for the process
     :param str private-dev-shm: size of private /dev/shm, using user namespaces
     :param bool reserve-cpu: command will ask for an exclusive CPU core
@@ -44,6 +45,7 @@ class Recipe(GenericBaseRecipe):
         args = shlex.split(self.options['command-line'])
         wrapper_path = self.options['wrapper-path']
         wait_files = self.options.get('wait-for-files')
+        hash_files = self.options.get('hash-files')
         pidfile = self.options.get('pidfile')
         private_dev_shm = self.options.get('private-dev-shm')
 
@@ -63,5 +65,20 @@ class Recipe(GenericBaseRecipe):
           kw['private_dev_shm'] = private_dev_shm
         if self.isTrueValue(self.options.get('reserve-cpu')):
           kw['reserve_cpu'] = True
+        if hash_files:
+          hash_file_list = hash_files.split()
+          hash = self.generateHashFromFiles(hash_file_list)
+          wrapper_path = "%s-%s" % (wrapper_path, hash)
 
         return self.createWrapper(wrapper_path, args, environment, **kw)
+
+    def generateHashFromFiles(self, file_list):
+      import hashlib
+      hasher = hashlib.md5()
+      for path in file_list:
+        with open(path, 'r') as afile:
+          buf = afile.read()
+        hasher.update("%s\n" % len(buf))
+        hasher.update(buf)
+      hash = hasher.hexdigest()
+      return hash

slapos/test/recipe/test_userinfo.py

+import unittest
+import zc.buildout.testing
+
+class UserInfoTest(unittest.TestCase):
+  def setUp(self):
+    self.buildout = buildout = zc.buildout.testing.Buildout()
+    buildout['userinfo'] = {}
+    from slapos.recipe import userinfo
+    self.recipe = userinfo.Recipe(
+        buildout,
+        'userinfo',
+        buildout['userinfo'])
+
+  def test_options(self):
+    buildout = self.buildout
+    self.assertTrue(buildout['userinfo']['pw-name'])
+    self.assertTrue(buildout['userinfo']['pw-uid'])
+    self.assertTrue(buildout['userinfo']['pw-gid'])
+    self.assertTrue(buildout['userinfo']['pw-dir'])
+    self.assertTrue(buildout['userinfo']['pw-shell'])
+
+    self.assertTrue(buildout['userinfo']['gr-name'])
+    self.assertTrue(buildout['userinfo']['gr-gid'])
+
+  def test_install(self):
+    self.assertEqual(self.recipe.install(), [])

software/apache-frontend/buildout.hash.cfg

@@ -23,15 +23,15 @@ md5sum = b6a2c860ea1cd4bc9d185c7108c52d0a
 
 [template-apache-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = c55506cfb92d87f915d50d32fb129a28
+md5sum = 9e76028df7e93d3e32982884d5dc0913
 
 [template-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = 4010f7366262d00c33f9e7f53f63d42e
+md5sum = 24e514ad6f15859229db46f24a8cd280
 
 [template-slave-configuration]
 filename = templates/custom-virtualhost.conf.in
-md5sum = d103143e5d50682bd5ad43117d82e2fa
+md5sum = b3d5a3573fa381fec954ad5cbf1575a0
 
 [template-replicate-publish-slave-information]
 filename = templates/replicate-publish-slave-information.cfg.in
@@ -43,7 +43,7 @@ md5sum = a56045e7b53ff00ab34d2a8f911fc1a1
 
 [template-custom-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = 4010f7366262d00c33f9e7f53f63d42e
+md5sum = 24e514ad6f15859229db46f24a8cd280
 
 [template-not-found-html]
 filename = templates/notfound.html
@@ -55,7 +55,7 @@ md5sum = d98a01182f38868612948c87d5231428
 
 [template-default-slave-virtualhost]
 filename = templates/default-virtualhost.conf.in
-md5sum = 7f38084af107034bedefba971abe165c
+md5sum = 4a2db0b69d2941cdf08ebf55575d8ad8
 
 [template-cached-slave-virtualhost]
 filename = templates/cached-virtualhost.conf.in

software/apache-frontend/instance-apache-replicate.cfg.in

@@ -114,19 +114,13 @@ sla-{{ parameter }} = {{ value }}
 
 
 [publish-information]
+<= monitor-publish
 recipe = slapos.cookbook:publish
 domain = {{ slapparameter_dict.get('domain') }}
 slave-amount = {{ slave_instance_list | length }}
 accepted-slave-amount = {{ authorized_slave_list | length }}
 rejected-slave-amount = {{ rejected_slave_list | length }}
 rejected-slave-list = {{ json_module.dumps(rejected_slave_list) }}
-monitor-base-url = ${monitor-conf-parameters:base-url}
-monitor-url = ${:monitor-base-url}/public/feeds
-monitor-user = ${monitor-instance-parameter:username}
-monitor-password = ${monitor-instance-parameter:password}
-{% set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
-monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${:monitor-url}
-
 
 #----------------------------
 #--

software/apache-frontend/templates/apache-custom-slave-list.cfg.in

@@ -81,7 +81,7 @@ crl = {{ custom_ssl_directory }}/crl/
 
 {# Set slave domain if none was defined #}
 {%   if slave_instance.get('custom_domain', None) == None -%}
-{%     set domain_prefix = slave_instance.get('slave_reference').replace("-", "").lower() -%}
+{%     set domain_prefix = slave_instance.get('slave_reference').replace("-", "").replace("_", "").lower() -%}
 {%     if slave_type in NGINX_TYPE_LIST -%}
 {%       do slave_instance.__setitem__('custom_domain', "%s.%s" % (domain_prefix, slapparameter_dict.get('nginx-domain', slapparameter_dict.get('domain')))) -%}
 {%     else -%}

software/apache-frontend/templates/custom-virtualhost.conf.in

@@ -3,5 +3,5 @@
 </VirtualHost>
 
 <VirtualHost *:{{ http_port }}>
-{{ slave_parameter.get('apache_custom_https', '') }}
+{{ slave_parameter.get('apache_custom_http', '') }}
 </VirtualHost>

software/apache-frontend/templates/default-virtualhost.conf.in

@@ -159,7 +159,7 @@
   # First, we check if we have a zope backend server
   # If so, let's use Virtual Host Daemon rewrite
   # We suppose that Apache listens to 80 (even indirectly thanks to things like iptables)
-  RewriteRule ^/(.*)$ {{ slave_parameter.get('url', '') }}/VirtualHostBase/http/%{SERVER_NAME}:{{ slave_parameter.get('virtualhostroot-http-port', '80') }}/{{ slave_parameter.get('path', '') }}/VirtualHostRoot/$1 [L,P]
+  RewriteRule ^/(.*)$ {{ slave_parameter.get('url', '') }}/VirtualHostBase/http//%{SERVER_NAME}:{{ slave_parameter.get('virtualhostroot-http-port', '80') }}/{{ slave_parameter.get('path', '') }}/VirtualHostRoot/$1 [L,P]
 {% else -%}
   {% if 'default-path' in slave_parameter %}
   RewriteRule ^/?$ {{ slave_parameter.get('default-path') }} [R=301,L]

software/erp5/instance-erp5-input-schema.json

@@ -194,6 +194,11 @@
       },
       "type": "object"
     },
+    "cloudooo-retry-count": {
+      "description": "Define retry count for cloudooo in network error case in test",
+      "type": "integer",
+      "default": 2
+    },
     "smtp": {
       "description": "Mail queuing and relay service",
       "additionalProperties": {

software/erp5testnode/testsuite/proftpd/README.md

+# ProFTPd test
+
+This software release is simply to run the test suite from `../proftpd/test/setup.py`
+
+Nexedi staff can see the results of this test from the test suite
+`SLAPOS-PROFTPD-TEST` in test result module.
+

software/erp5testnode/testsuite/proftpd/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     But avoid directories, they are not portable.
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[template]
+filename = instance.cfg
+md5sum = 9b4cc727c43d0daaec5b01cf4b9f7a15
+

software/erp5testnode/testsuite/proftpd/instance.cfg

+[buildout]
+parts =
+  slapos-test-runner
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[download-source]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+
+[slapos]
+<= download-source
+repository = ${slapos-repository:location}
+
+[create-directory]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+
+[slapos-test-runner]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${create-directory:bin}/runTestSuite
+command-line =
+  ${buildout:bin-directory}/runTestSuite
+  --python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
+  --source_code_path_list=$${slapos:location}/software/proftpd/test
+
+
+# XXX slapos.cookbook:wrapper does not allow extending env, so we add some default $PATH entries ( not sure they are needed )
+environment =
+  PATH=${buildout:bin-directory}:/usr/bin/:/bin/
+  LOCAL_IPV4=$${slap-configuration:ipv4-random}
+  GLOBAL_IPV6=$${slap-configuration:ipv6-random}

software/erp5testnode/testsuite/proftpd/software.cfg

+[buildout]
+
+extends =
+  ../../../../component/git/buildout.cfg
+  ../../../../stack/slapos.cfg
+  ./buildout.hash.cfg
+
+parts =
+  slapos-cookbook
+  eggs
+  template
+
+[setup-develop-egg]
+recipe = zc.recipe.egg:develop
+
+[slapos.test.proftpd-setup]
+<= setup-develop-egg
+egg = slapos.test.proftpd
+setup = ${slapos-repository:location}/software/proftpd/test/
+
+[erp5.util-setup]
+<= setup-develop-egg
+egg = erp5.util[testnode]
+setup = ${erp5.util-repository:location}
+
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${slapos.test.proftpd-setup:egg}
+  ${erp5.util-setup:egg}
+  slapos.core
+entry-points =
+  runTestSuite=erp5.util.testsuite:runTestSuite
+scripts =
+  runTestSuite
+  slapos
+interpreter=
+  python_for_test
+
+[git-clone-repository]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+forbid-download-cache = true
+branch = master
+
+[slapos-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/slapos.git
+
+# XXX we need an unreleased ( 0.4.51 ) version of erp5.util runTestSuite 
+# later we can stop fetching it from git and just use egg
+[erp5.util-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/erp5.git
+revision = 69013fa0fb67501089c776ab5e75d7bbf2e0e3bc
+
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/template.cfg
+mode = 640
+
+[versions]
+# clear the version of tested eggs, to make sure we installed the developped ones
+slapos.test.proftpd =
+erp5.util =
+#erp5.util = 0.4.51

software/kvm/common.cfg

@@ -99,7 +99,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
 mode = 644
-md5sum = c348188a6dcb15430bea0bad51c249fc
+md5sum = e59ea29533d7f989ec676e14b0f29839
 download-only = true
 on-update = true
 

software/kvm/instance-kvm.cfg.jinja2

@@ -218,7 +218,7 @@ input = inline:#!/bin/sh
   # Return code 0 is "OK"
   # Return code 3 is "found leaks, but image is OK"
   # http://git.qemu.org/?p=qemu.git;a=blob;f=qemu-img.c;h=4e9a7f5741c9cb863d978225829e68fefcae3947;hb=HEAD#l702
-  ${kvm-parameter-dict:qemu-img-path} check ${kvm-parameter-dict:disk-path}
+  ${kvm-parameter-dict:qemu-img-path} check -U ${kvm-parameter-dict:disk-path}
   RETURN_CODE=$?
   if [ $RETURN_CODE -eq 0 ] || [ $RETURN_CODE -eq 3 ]; then
     exit 0

software/neoppod/instance-neo-input-schema.json

@@ -2,7 +2,7 @@
   "$schema": "http://json-schema.org/draft-04/schema#",
   "description": "Parameters to instantiate a NEO cluster. See https://lab.nexedi.com/nexedi/neoppod/blob/master/neo.conf for more information.",
   "additionalProperties": false,
-  "required": ["cluster"],
+  "require": ["cluster"],
   "properties": {
     "cluster": {
       "description": "Cluster unique identifier. Your last line of defense against mixing up NEO clusters and corrupting your data. Choose a unique value for each of your cluster.",

software/neoppod/software-common.cfg

@@ -8,6 +8,7 @@
 extends =
     ../../stack/slapos.cfg
     ../../stack/logrotate/buildout.cfg
+    ../../component/cython-zstd/buildout.cfg
 #LXML
     ../../component/lxml-python/buildout.cfg
 #END LXML
@@ -41,6 +42,7 @@ environment = neoppod-setup-env
 [neoppod]
 recipe = zc.recipe.egg
 eggs = neoppod[admin, ctl, master, storage-mysqldb]
+  ${cython-zstd:egg}
   ${msgpack-python:egg}
   ${python-mysqlclient:egg}
   psutil
@@ -116,7 +118,7 @@ ZODB = 4.4.5
 apache-libcloud = 1.5.0
 ecdsa = 0.13
 gitdb2 = 2.0.0
-msgpack-python = 0.4.8
+msgpack-python = 0.5.6
 mysqlclient = 1.3.12
 persistent = 4.2.3
 pycrypto = 2.6.1
@@ -127,6 +129,7 @@ smmap2 = 2.0.1
 transaction = 1.7.0
 zodbpickle = 0.6.0
 zodbtools = 0.0.0.dev4
+cython-zstd = 0.2
 
 # Required by:
 # slapos.toolbox==0.71

software/proftpd/README.md

+Proftpd with sftp and virtual users
+
+http://www.proftpd.org/docs/
+
+# Features
+
+ * sftp only is enabled
+ * partially uploadloaded are not visible thanks to [`HiddenStores`](http://proftpd.org/docs/directives/linked/config_ref_HiddenStores.html) ( in fact they are, but name starts with `.` )
+ * 5 failed login attempts will cause the host to be temporary banned
+
+
+# TODO
+
+ * only password login is enabled. enabling [`SFTPAuthorizedUserKeys`](http://www.proftpd.org/docs/contrib/mod_sftp.html#SFTPAuthorizedUserKeys) seems to break password only login
+ * log rotation
+ * make sure SFTPLog is useful (seems very verbose and does not contain more than stdout)
+ * make it easier to manage users ( using `mod_auth_web` against an ERP5 endpoint or accepting a list of user/password as instance parameter )
+ * allow configuring webhooks when new file is uploaded
+
+

software/proftpd/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     But avoid directories, they are not portable.
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[instance-profile]
+filename = instance.cfg.in
+md5sum = 8ed5b4a7940db47ccb386c2f4e3e7273
+
+[instance-default]
+filename = instance-default.cfg.in
+md5sum = 89e145188fe2a223f22f57e5c0c242e4
+
+[proftpd-config-file]
+filename = proftpd-config-file.cfg.in
+md5sum = 46cbe4ea65b445822d3da32c76c7d67d
+

software/proftpd/instance-default.cfg.in

+[buildout]
+parts =
+  promises
+  publish-connection-parameter
+
+
+[instance-parameter]
+# TODO: this is not needed
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+log = ${:var}/log
+srv = ${:home}/srv
+service = ${:etc}/service
+promise = ${:etc}/promise
+proftpd-dir = ${:srv}/proftpd/
+ssh-authorized-keys-dir = ${:etc}/authorized_keys/
+
+[config-file]
+recipe = slapos.recipe.template:jinja2
+template = {{ software_parts_directory }}/${:_buildout_section_name_}/${:_buildout_section_name_}.cfg.in
+rendered = ${directory:etc}/${:_buildout_section_name_}.cfg
+mode = 0644
+extensions = jinja2.ext.do
+
+[check-port-listening-promise]
+recipe = slapos.cookbook:check_port_listening
+path = ${directory:promise}/${:_buildout_section_name_}
+
+[proftpd-userinfo]
+recipe = slapos.cookbook:userinfo
+
+
+[proftpd-password]
+recipe = slapos.cookbook:generate.password
+username = proftpd
+bytes = 12
+
+[proftpd]
+ipv6 = ${instance-parameter:ipv6-random}
+ipv4 = ${instance-parameter:ipv4-random}
+host = ${:ipv6}
+sftp-port = {{ slapparameter_dict.get('port', 8022) }}
+url = sftp://[${:host}]:${:sftp-port}
+data-dir = ${directory:proftpd-dir}
+user=${proftpd-userinfo:pw-name}
+group=${proftpd-userinfo:gr-name}
+sftp-log=${directory:log}/proftpd-sftp.log
+xfer-log=${directory:log}/proftpd-xfer.log
+ban-log=${directory:log}/proftpd-ban.log
+ssh-host-rsa-key=${ssh-host-rsa-key:output}
+ssh-host-dsa-key=${ssh-host-dsa-key:output}
+ssh-host-ecdsa-key=${ssh-host-ecdsa-key:output}
+ssh-authorized-keys-dir = ${directory:ssh-authorized-keys-dir}
+ban-table=${directory:srv}/proftpd-ban-table
+auth-user-file=${auth-user-file:output}
+
+recipe = slapos.cookbook:wrapper
+command-line =
+   {{ proftpd_bin }} --nodaemon --config ${proftpd-config-file:rendered}
+wrapper-path = ${directory:service}/proftpd
+
+[proftpd-listen-promise]
+<= check-port-listening-promise
+hostname = ${proftpd:ipv6}
+port = ${proftpd:sftp-port}
+
+
+[ftpasswd]
+# command line to add a user, invoke with:
+#   ftpasswd --name=bob
+# to prompt for password, or --stdin to read password from stdin
+recipe = slapos.recipe.template:jinja2
+mode = 0700
+extensions = jinja2.ext.do
+rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
+wrapper-path = ${:rendered}
+# XXX: to workaround bug that pw-uid and gr-gir slapos.cookbook:userinfo cannot
+# be used in a buildout part (as they are returned as int and not strings), we
+# use slapos.recipe.template:jinja2 and not a simple slapos.cookbook:wrapper,
+# because only the former is compatible with userinfo.
+# Because we are already in a jinja template, we have to escape jinja template
+# with {% raw %} / {% endraw %}
+template = inline:
+  #!/bin/bash
+  {{ perl_bin }} {{ ftpasswd_bin }} --passwd --home=${proftpd:data-dir} --shell=/bin/false --uid={% raw %}{{ pw_uid }} --gid={{ gr_gid }}{% endraw %} --file ${auth-user-file:output} $@
+context =
+  key pw_uid proftpd-userinfo:pw-uid
+  key gr_gid proftpd-userinfo:gr-gid
+
+[auth-user-file]
+recipe = plone.recipe.command
+output = ${directory:etc}/ftpd.passwd
+command =
+  echo ${proftpd-password:passwd} | ${ftpasswd:wrapper-path} --name=${proftpd-password:username} --stdin
+update-command = ${:command}
+
+
+[ssh-keygen-base]
+recipe = plone.recipe.command
+output = ${directory:etc}/${:_buildout_section_name_}
+command = {{ ssh_keygen_bin }} -f ${:output} -N '' ${:extra-args}
+
+[ssh-host-rsa-key]
+<=ssh-keygen-base
+extra-args=-t rsa
+[ssh-host-dsa-key]
+<=ssh-keygen-base
+extra-args=-t dsa
+[ssh-host-ecdsa-key]
+<=ssh-keygen-base
+extra-args=-t ecdsa -b 521
+
+
+[proftpd-config-file]
+<= config-file
+context =
+  section proftpd proftpd
+  key slapparameter_dict slap-configuration:configuration
+
+
+[promises]
+recipe =
+instance-promises =
+  ${proftpd-listen-promise:path}
+
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+url = ${proftpd:url}
+username = ${proftpd-password:username}
+password = ${proftpd-password:passwd}

software/proftpd/instance-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Parameters to instantiate PoFTPd",
+  "additionalProperties": false,
+  "properties": {
+    "port": {
+      "description": "Port number to listen to - default to 8022",
+      "type": "number"
+    }
+  }
+}

software/proftpd/instance-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by ProFTPd instantiation",
+  "additionalProperties": false,
+  "properties": {
+    "url": {
+      "description": "URL of the SFTP service",
+      "pattern": "^sftp://",
+      "type": "string"
+    },
+    "username": {
+      "description": "Default username",
+      "type": "string",
+      "optional": true
+    },
+    "password": {
+      "description": "Password for default username",
+      "type": "string",
+      "optional": true
+    }
+  },
+  "type": "object"
+}

software/proftpd/instance.cfg.in

+[buildout]
+parts = switch-softwaretype
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[instance-template]
+recipe = slapos.recipe.template:jinja2
+filename = ${:_buildout_section_name_}.cfg
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
+mode = 0644
+extensions = jinja2.ext.do
+context =
+  key slapparameter_dict slap-configuration:configuration
+  raw software_parts_directory {{ buildout['parts-directory'] }}
+  raw proftpd_bin {{ proftpd_bin }}
+  raw ftpasswd_bin {{ ftpasswd_bin }}
+  raw ftpdctl_bin {{ ftpdctl_bin }}
+  raw ssh_keygen_bin {{ ssh_keygen_bin }}
+  raw perl_bin {{ perl_bin }}
+
+[instance-default]
+<= instance-template
+template = {{ instance_default }}
+
+[switch-softwaretype]
+recipe = slapos.cookbook:switch-softwaretype
+# XXX Jerome: what is this override for ?
+override = {{ dumps(override_switch_softwaretype |default) }}
+default = instance-default:rendered
+# BBB
+RootSoftwareInstance = ${:default}

software/proftpd/proftpd-config-file.cfg.in

+ServerName "ProFTPD SlapOS"
+ServerType standalone
+MaxInstances  30
+DefaultAddress {{ proftpd['ipv6'] }} {{ proftpd['ipv4']}}
+SocketBindTight on
+DefaultServer off
+Port {{ proftpd['sftp-port'] }}
+
+User {{ proftpd['user'] }}
+Group {{ proftpd['group'] }}
+
+Umask 022
+AllowOverwrite on
+
+# SFTP
+SFTPEngine on
+SFTPHostKey {{ proftpd['ssh-host-rsa-key'] }}
+SFTPHostKey {{ proftpd['ssh-host-dsa-key'] }}
+SFTPHostKey {{ proftpd['ssh-host-ecdsa-key'] }}
+#SFTPAuthorizedUserKeys file:{{ proftpd['ssh-authorized-keys-dir'] }}%u
+
+
+# Logging
+TransferLog {{ proftpd['xfer-log'] }}
+SFTPLog {{ proftpd['sftp-log'] }}
+BanLog {{ proftpd['ban-log'] }}
+
+
+# Virtual users
+RequireValidShell off
+AuthUserFile {{ proftpd['auth-user-file'] }}
+
+
+# Prevent partially uploaded files to be visible
+HiddenStores on
+DeleteAbortedStores on
+AllowStoreRestart off
+
+
+# Limitations of running as non-root user
+WtmpLog off
+# ( we cannot use DefaultRoot ~ )
+
+# Prevent user for escaping their home
+<Directory />
+  <Limit ALL>
+    DenyAll
+  </Limit>
+</Directory>
+
+<Directory {{ proftpd['data-dir'] }}>
+  <Limit ALL>
+    AllowAll
+  </Limit>
+</Directory>
+
+
+
+# Ban failed logins
+MaxLoginAttempts 1
+BanEngine on
+BanTable {{ proftpd['ban-table'] }}
+# 5 failed login attemps in 5 minutes -> ban for 20 minutes
+BanOnEvent MaxLoginAttempts 5/00:05:00 00:20:00 "Too many Failed Login Attempts"
+BanControlsACLs all allow user {{ proftpd['user'] }}
+
+

software/proftpd/software.cfg

+[buildout]
+extends =
+  ../../stack/slapos.cfg
+  ../../component/openssh/buildout.cfg
+  ../../component/proftpd/buildout.cfg
+  buildout.hash.cfg
+
+versions = versions
+parts =
+  slapos-cookbook
+  proftpd-config-file
+  instance-profile
+
+
+[download-file-base]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:filename}
+download-only = true
+mode = 0644
+
+[proftpd-config-file]
+<= download-file-base
+
+[instance-default]
+<= download-file-base
+
+[instance-profile]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/${:filename}
+rendered = ${buildout:directory}/instance.cfg
+mode = 0644
+extensions = jinja2.ext.do
+context =
+  section buildout buildout
+  key instance_default instance-default:target
+  key proftpd_bin proftpd-output:proftpd
+  key ftpasswd_bin proftpd-output:ftpasswd
+  key ftpdctl_bin proftpd-output:ftpdctl
+  key ssh_keygen_bin openssh-output:keygen
+  key perl_bin proftpd-output:perl
+
+
+[versions]
+collective.recipe.environment = 1.1.0
+collective.recipe.grp = 1.1.0
+slapos.recipe.template = 4.3
+plone.recipe.command = 1.1

software/proftpd/software.cfg.json

+{
+  "name": "ProFTPd",
+  "description": "ProFTPd as a SFTP server with virtual users",
+  "serialisation": "json-in-xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "description": "ProFTPd, with a default user and users added with command line",
+      "request": "instance-input-schema.json",
+      "response": "instance-output-schema.json",
+      "index": 0
+    }
+  }
+}

software/proftpd/test/README.md

+Tests for ProFTPd software release

software/proftpd/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+import glob
+import os
+
+version = '0.0.1.dev0'
+name = 'slapos.test.proftpd'
+long_description = open("README.md").read()
+
+setup(name=name,
+      version=version,
+      description="Test for SlapOS' ProFTPd",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+        'slapos.core',
+        'erp5.util',
+        'pysftp',
+        ],
+      zip_safe=True,
+      test_suite='test',
+    )

software/proftpd/test/test.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import shutil
+import urlparse
+import tempfile
+import StringIO
+import subprocess
+
+import pysftp
+from paramiko.ssh_exception import SSHException
+from paramiko.ssh_exception import AuthenticationException
+
+import utils
+
+
+# for development: debugging logs and install Ctrl+C handler
+if os.environ.get('DEBUG'):
+  import logging
+  logging.basicConfig(level=logging.DEBUG)
+  import unittest
+  unittest.installHandler()
+
+
+class ProFTPdTestCase(utils.SlapOSInstanceTestCase):
+  @classmethod
+  def getSoftwareURLList(cls):
+    return (os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'software.cfg')), )
+
+  def _getConnection(self, username=None, password=None, hostname=None):
+    """Returns a pysftp connection connected to the SFTP
+
+    username and password can be specified and default to the ones from
+    instance connection parameters.
+    another hostname can also be passed.
+    """
+    # this tells paramiko not to verify host key
+    cnopts = pysftp.CnOpts()
+    cnopts.hostkeys = None
+
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    sftp_url = urlparse.urlparse(parameter_dict['url'])
+
+    return pysftp.Connection(
+        hostname or sftp_url.hostname,
+        port=sftp_url.port,
+        cnopts=cnopts,
+        username=username or parameter_dict['username'],
+        password=password or parameter_dict['password'])
+
+
+class TestSFTPListen(ProFTPdTestCase):
+  def test_listen_on_ipv4(self):
+    self.assertTrue(self._getConnection(hostname=self.config['ipv4_address']))
+
+  def test_does_not_listen_on_all_ip(self):
+    with self.assertRaises(SSHException):
+      self._getConnection(hostname='0.0.0.0')
+
+
+class TestSFTPOperations(ProFTPdTestCase):
+  """Tests upload / download features we expect in SFTP server.
+  """
+  def setUp(self):
+    self.upload_dir = os.path.join(
+        self.computer_partition_root_path, 'srv', 'proftpd')
+
+  def tearDown(self):
+    for name in os.listdir(self.upload_dir):
+      path = os.path.join(self.upload_dir, name)
+      if os.path.isfile(path) or os.path.islink(path):
+        os.remove(path)
+      else:
+        shutil.rmtree(path)
+
+  def test_simple_sftp_session(self):
+    with self._getConnection() as sftp:
+      # put a file
+      with tempfile.NamedTemporaryFile() as f:
+        f.write("Hello FTP !")
+        f.flush()
+        sftp.put(f.name, remotepath='testfile')
+
+      # it's visible in listdir()
+      self.assertEqual(['testfile'], sftp.listdir())
+
+      # and also in the server filesystem
+      self.assertEqual(['testfile'], os.listdir(self.upload_dir))
+
+      # download the file again, it should have same content
+      tempdir = tempfile.mkdtemp()
+      self.addCleanup(lambda : shutil.rmtree(tempdir))
+      local_file = os.path.join(tempdir, 'testfile')
+      retrieve_same_file = sftp.get('testfile', local_file)
+      with open(local_file) as f:
+        self.assertEqual(f.read(), "Hello FTP !")
+
+  def test_uploaded_file_not_visible_until_fully_uploaded(self):
+    test_self = self
+    class PartialFile(StringIO.StringIO):
+      def read(self, *args):
+        # file is not visible yet
+        test_self.assertNotIn('destination', os.listdir(test_self.upload_dir))
+        # it's just a hidden file
+        test_self.assertEqual(['.in.destination.'], os.listdir(test_self.upload_dir))
+        return StringIO.StringIO.read(self, *args)
+
+    with self._getConnection() as sftp:
+      sftp.sftp_client.putfo(PartialFile("content"), "destination")
+
+    # now file is visible
+    self.assertEqual(['destination'], os.listdir(self.upload_dir))
+
+  def test_partial_upload_are_deleted(self):
+    test_self = self
+    with self._getConnection() as sftp:
+      class ErrorFile(StringIO.StringIO):
+        def read(self, *args):
+          # at this point, file is already created on server
+          test_self.assertEqual(['.in.destination.'], os.listdir(test_self.upload_dir))
+          # simulate a connection closed
+          sftp.sftp_client.close()
+          return "something that will not be sent to server"
+      with self.assertRaises(IOError):
+        sftp.sftp_client.putfo(ErrorFile(), "destination")
+    # no half uploaded file is kept
+    self.assertEqual([], os.listdir(self.upload_dir))
+
+  def test_user_cannot_escape_home(self):
+    with self._getConnection() as sftp:
+      with self.assertRaisesRegexp(IOError, 'Permission denied'):
+        sftp.listdir('..')
+      with self.assertRaisesRegexp(IOError, 'Permission denied'):
+        sftp.listdir('/')
+      with self.assertRaisesRegexp(IOError, 'Permission denied'):
+        sftp.listdir('/tmp/')
+
+
+class TestUserManagement(ProFTPdTestCase):
+  def test_user_can_be_added_from_script(self):
+    with self.assertRaisesRegexp(AuthenticationException, 'Authentication failed'):
+      self._getConnection(username='bob', password='secret')
+
+    subprocess.check_call(
+      'echo secret | %s/bin/ftpasswd --name=bob --stdin' % self.computer_partition_root_path,
+      shell=True)
+    self.assertTrue(self._getConnection(username='bob', password='secret'))
+
+
+class TestBan(ProFTPdTestCase):
+  def test_client_are_banned_after_5_wrong_passwords(self):
+    # Simulate failed 5 login attempts
+    for i in range(5):
+      with self.assertRaisesRegexp(AuthenticationException, 'Authentication failed'):
+        self._getConnection(password='wrong')
+
+    # after that, even with a valid password we cannot connect
+    with self.assertRaisesRegexp(SSHException, 'Connection reset by peer'):
+      self._getConnection()
+
+    # ban event is logged
+    with open(os.path.join(
+        self.computer_partition_root_path, 'var', 'log', 'proftpd-ban.log')) as ban_log_file:
+      self.assertRegexpMatches(
+        ban_log_file.readlines()[-1],
+        'login from host .* denied due to host ban')
+
+
+class TestInstanceParameterPort(ProFTPdTestCase):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    cls.free_port = utils.findFreeTCPPort(cls.config['ipv4_address'])
+    return {'port': cls.free_port}
+
+  def test_instance_parameter_port(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    sftp_url = urlparse.urlparse(parameter_dict['url'])
+    self.assertEqual(self.free_port, sftp_url.port)
+    self.assertTrue(self._getConnection())
+
+

software/proftpd/test/utils.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import unittest
+import os
+import socket
+from contextlib import closing
+import logging
+
+from erp5.util.testnode.SlapOSControler import SlapOSControler
+from erp5.util.testnode.ProcessManager import ProcessManager
+import slapos
+
+
+def findFreeTCPPort(ip=''):
+  """Find a free TCP port to listen to.
+  """
+  family = socket.AF_INET6 if ':' in ip else socket.AF_INET
+  with closing(socket.socket(family, socket.SOCK_STREAM)) as s:
+    s.bind((ip, 0))
+    return s.getsockname()[1]
+
+
+class SlapOSInstanceTestCase(unittest.TestCase):
+  @classmethod
+  def getSoftwareURLList(cls):
+    """Return URL of software releases to install.
+
+    To be defined by subclasses.
+    """
+    raise NotImplementedError()
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    """Return instance parameters
+
+    To be defined by subclasses if they need to request instance with specific
+    parameters.
+    """
+    return {}
+
+ # TODO: allow subclasses to request a specific software type ?
+
+
+  @classmethod
+  def setUpClass(cls):
+
+    working_directory = os.environ.get(
+        'SLAPOS_TEST_WORKING_DIR',
+        os.path.join(os.path.dirname(__file__), '.slapos'))
+    # To prevent error: Cannot open an HTTP server: socket.error reported
+    # AF_UNIX path too long This `working_directory` should not be too deep.
+    # Socket path is 108 char max on linux
+    # https://github.com/torvalds/linux/blob/3848ec5/net/unix/af_unix.c#L234-L238
+    if len(working_directory + '/inst/supervisord.socket') > 108:
+      raise RuntimeError('working directory too deep, try setting SLAPOS_TEST_WORKING_DIR')
+
+    if not os.path.exists(working_directory):
+      os.mkdir(working_directory)
+
+    cls.config = config = {
+      "working_directory": working_directory,
+      "slapos_directory": working_directory,
+      "log_directory": working_directory,
+      "computer_id": 'slapos.test', # XXX
+      'proxy_database': os.path.join(working_directory, 'proxy.db'),
+      'partition_reference': cls.__name__,
+      # "proper" slapos command must be in $PATH
+      'slapos_binary': 'slapos',
+    }
+
+    # Some tests are expecting that local IP is not set to 127.0.0.1
+    ipv4_address = os.environ.get('LOCAL_IPV4', '127.0.1.1')
+    ipv6_address = os.environ['GLOBAL_IPV6']
+
+    config['proxy_host'] = config['ipv4_address'] = ipv4_address
+    config['ipv6_address'] = ipv6_address
+    config['proxy_port'] = findFreeTCPPort(ipv4_address)
+    config['master_url'] = 'http://{proxy_host}:{proxy_port}'.format(**config)
+
+    cls._process_manager = process_manager = ProcessManager()
+
+    # XXX this code is copied from testnode code
+    slapos_controler = SlapOSControler(
+        working_directory,
+        config
+    )
+
+    slapproxy_log = os.path.join(config['log_directory'], 'slapproxy.log')
+    logger = logging.getLogger(__name__)
+    logger.debug('Configured slapproxy log to %r', slapproxy_log)
+
+    software_url_list = cls.getSoftwareURLList()
+    slapos_controler.initializeSlapOSControler(
+        slapproxy_log=slapproxy_log,
+        process_manager=process_manager,
+        reset_software=False,
+        software_path_list=software_url_list)
+
+    process_manager.supervisord_pid_file = os.path.join(
+           slapos_controler.instance_root, 'var', 'run', 'supervisord.pid')
+
+    software_status_dict = slapos_controler.runSoftwareRelease(config, environment=os.environ)
+    # TODO: log more details in this case
+    assert software_status_dict['status_code'] == 0
+
+    instance_parameter_dict = cls.getInstanceParmeterDict()
+    instance_status_dict = slapos_controler.runComputerPartition(
+        config,
+        cluster_configuration=instance_parameter_dict,
+        environment=os.environ)
+    # TODO: log more details in this case
+    assert instance_status_dict['status_code'] == 0
+
+    # FIXME: similar to test node, only one (root) partition is really supported for now.
+    computer_partition_list = []
+    for i in range(len(software_url_list)):
+      computer_partition_list.append(
+          slapos_controler.slap.registerOpenOrder().request(
+          software_url_list[i],
+          # This is how testnode's SlapOSControler name created partitions
+          partition_reference='testing partition {i}'.format(i=i, **config),
+          partition_parameter_kw=instance_parameter_dict))
+
+    # expose some class attributes so that tests can use them:
+    # the ComputerPartition instances, to getInstanceParmeterDict
+    cls.computer_partition = computer_partition_list[0]
+
+    # the path of the instance on the filesystem, for low level inspection
+    cls.computer_partition_root_path = os.path.join(
+        config['working_directory'],
+        'inst',
+        cls.computer_partition.getId())
+
+
+  @classmethod
+  def tearDownClass(cls):
+    # FIXME: if setUpClass fail, this is not called and leaks zombie processes
+    cls._process_manager.killPreviousRun()
+

software/re6stnet/software.cfg

@@ -120,6 +120,9 @@ pycurl = 7.43.0
 slapos.recipe.template = 4.3
 slapos.toolbox = 0.76
 smmap = 0.9.0
+dnspython = 1.15.0
+erp5.util = 0.4.50
+passlib = 1.7.1
 
 # Required by:
 # slapos.toolbox==0.71

software/slapos-master/buildout.hash.cfg

@@ -15,11 +15,11 @@
 # not need these here).
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = f539acb8da805ce2de0787769283869e
+md5sum = ab91539deb74e0fdb0680f35d246bd3a
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 257467e676f74c1601fd09d33968fbc3
+md5sum = 697e43a020af5edee2151987abc8e7b4
 
 [template-apache-backend-conf]
 filename = apache-backend.conf.in

software/slapos-master/instance-balancer.cfg.in

@@ -306,7 +306,7 @@ apachedex = ${monitor-directory:private}/apachedex
 [{{ section('monitor-generate-apachedex-report') }}]
 recipe = slapos.cookbook:wrapper
 wrapper-path = ${monitor-directory:reports}/${:command}
-command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --config "${apachedex-parameters:configuration}"
+command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration "${apachedex-parameters:configuration}"
 command = apachedex_every_23_hour
 
 [apachedex-parameters]

software/slapos-master/instance-erp5.cfg.in

@@ -166,6 +166,7 @@ config-mysql-url-list = ${request-mariadb:connection-database-list}
 config-site-id = {{ dumps(site_id) }}
 config-smtp-url = ${request-smtp:connection-url}
 config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
+config-cloudooo-retry-count = {{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
 config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }}
 config-ca-path = ${directory:ca-dir}
 config-zodb-dict = {{ dumps(zodb_dict) }}
@@ -211,7 +212,7 @@ config-thread-amount = {{ dumps(zope_parameter_dict.get('thread-amount', 4)) }}
 config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }}
 config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }}
 config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }}
-config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', '10MB')) }}
+config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "10MB")) }}
 config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
 config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }}
 {% endfor -%}
@@ -334,6 +335,7 @@ config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configurat
 [request-frontend-base]
 {% if has_frontend -%}
 <= request-common
+recipe = slapos.cookbook:request
 software-url = {{ dumps(frontend_dict['software-url']) }}
 software-type = {{ dumps(frontend_dict.get('software-type', 'RootSoftwareInstance')) }}
 {{ root_common.sla('frontend', True) }}
@@ -344,6 +346,12 @@ slave = true
 {%   if frontend_dict.get('domain') -%}
 {%     do config_dict.__setitem__('custom_domain', frontend_dict['domain']) -%}
 {%   endif -%}
+{%   if frontend_dict.get('virtualhostroot-http-port') -%}
+{%     do config_dict.__setitem__('virtualhostroot-http-port', frontend_dict['virtualhostroot-http-port']) -%}
+{%   endif -%}
+{%   if frontend_dict.get('virtualhostroot-https-port') -%}
+{%     do config_dict.__setitem__('virtualhostroot-https-port', frontend_dict['virtualhostroot-https-port']) -%}
+{%   endif -%}
 {%   for name, value in config_dict.items() -%}
 config-{{ name }} = {{ value }}
 {%   endfor -%}

software/wendelin/software.cfg

@@ -85,5 +85,5 @@ revision = 56123b8d75bb7827b3f70864d4f9dca19d136ad6
 revision = f785ac079b3ed602692c1b3a275d6d51a01eb453
 
 [versions]
-msgpack-python = 0.4.8
+msgpack-python = 0.5.6
 wendelin.core = 0.12

stack/cloudooo.cfg

@@ -8,7 +8,6 @@ extends =
   ../component/dash/buildout.cfg
   ../component/dbus/buildout.cfg
   ../component/dcron/buildout.cfg
-  ../component/ffmpeg/buildout.cfg
   ../component/file/buildout.cfg
   ../component/fonts/buildout.cfg
   ../component/git/buildout.cfg
@@ -69,7 +68,6 @@ parts =
   wkhtmltopdf
   file
   poppler
-  ffmpeg
 
   rdiff-backup
   apache

stack/erp5/buildout.hash.cfg

@@ -79,7 +79,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 5eb5ff7491b9e47c647ecfd381a2e143
+md5sum = 2431422c66082a6fb6a6be236ebacd5c
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -87,12 +87,12 @@ md5sum = d1f33d406d528ae27d973e2dd0efb1ba
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = 3fdfbc82c450c35a9a444dfbc0a1c196
+md5sum = 7fb40624bd0dcd96a1df03395da64ebd
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 29cfe35d37cf615378574707978e5f75
+md5sum = 55fce4e96d844cc418c586d8d9a67c29
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in
-md5sum = 3defd473e2cea17ae36bba7752494858
+md5sum = 13f1f731ec941f4ba941d6fa8834a5cc

stack/erp5/haproxy.cfg.in

@@ -12,7 +12,6 @@ defaults
   option redispatch
   maxconn 2000
   cookie SERVERID rewrite
-  http-send-name-header X-Balancer-Current-Server
   balance roundrobin
   stats uri /haproxy
   stats realm Global\ statistics

stack/erp5/instance-balancer.cfg.in

@@ -161,6 +161,7 @@ extensions = jinja2.ext.do
 recipe = slapos.cookbook:wrapper
 wrapper-path = ${directory:services}/haproxy
 command-line = "{{ parameter_dict['haproxy'] }}/sbin/haproxy" -f "${haproxy-cfg:rendered}"
+hash-files = ${haproxy-cfg:rendered}
 
 {# TODO: build socat and wrap it as "${directory:bin}/haproxy-ctl" to connect to "${haproxy-cfg-parameter-dict:socket-path}" #}
 

stack/erp5/instance-erp5.cfg.in

@@ -166,6 +166,7 @@ config-mysql-url-list = ${request-mariadb:connection-database-list}
 config-site-id = {{ dumps(site_id) }}
 config-smtp-url = ${request-smtp:connection-url}
 config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
+config-cloudooo-retry-count = {{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
 config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }}
 config-zodb-dict = {{ dumps(zodb_dict) }}
 {% for server_type, server_dict in storage_dict.iteritems() -%}

stack/erp5/instance-zope.cfg.in

@@ -28,6 +28,7 @@ mysql-url-list = {{ dumps(slapparameter_dict['mysql-test-url-list']) }}
 kumofs-url = {{ dumps(slapparameter_dict['kumofs-url']) }}
 memcached-url = {{ dumps(slapparameter_dict['memcached-url']) }}
 cloudooo-url = {{ dumps(slapparameter_dict['cloudooo-url']) }}
+cloudooo-retry-count = {{ slapparameter_dict['cloudooo-retry-count'] }}
 test-instance-path = ${directory:unit-test-path}
 prepend-path = ${buildout:bin-directory}
 run-unit-test = ${buildout:bin-directory}/runUnitTest.real

stack/resilient/buildout.hash.cfg

@@ -31,7 +31,7 @@ md5sum = 85e777bd349a5c881d36e747882aee8a
 
 [template-replicated]
 filename = template-replicated.cfg.in
-md5sum = 55ef107f499e5e604b8d277098618fed
+md5sum = 5369b039600e1df8b471ac4b41d5de60
 
 [template-parts]
 filename = template-parts.cfg.in

stack/resilient/template-replicated.cfg.in

@@ -140,8 +140,8 @@ takeover-{{namebase}}-{{id}}-password = ${request-{{namebase}}-pseudo-replicatin
 # XXX: maybe we should consider empty values to be non-nexistent.
 recipe = collective.recipe.template
 # XXX: don't use system executable
-input = inline:#!/bin/sh
-  PUBLIC_KEY_CONTENT="${request-{{namebase}}:connection-ssh-public-key})"
+input = inline:#!/bin/bash
+  PUBLIC_KEY_CONTENT="${request-{{namebase}}:connection-ssh-public-key}"
   if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then
     exit 1
   fi
@@ -155,8 +155,8 @@ mode = 700
 # XXX: maybe we should consider empty values to be non-nexistent.
 recipe = collective.recipe.template
 # XXX: don't use system executable
-input = inline:#!/bin/sh
-  PUBLIC_KEY_CONTENT="${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-public-key})"
+input = inline:#!/bin/bash
+  PUBLIC_KEY_CONTENT="${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-public-key}"
   if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then
     exit 1
   fi
@@ -217,8 +217,8 @@ sla-{{ key }} = {{ value }}
 # XXX: maybe we should consider empty values to be non-nexistent.
 recipe = collective.recipe.template
 # XXX: don't use system executable
-input = inline:#!/bin/sh
-  PUBLIC_KEY_CONTENT="${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}:connection-ssh-key})"
+input = inline:#!/bin/bash
+  PUBLIC_KEY_CONTENT="${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}"
   if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then
     exit 1
   fi