Stop using erp5 test recipe

The goal is to drop slapos.cookbook:erp5.test

No need for a recipe to do a bit of url parsing and generating 2 wrappers.
Also, drop unused & empty "host-common" section.

- stack/erp5: Add path to all binary links in PATH
- stack/erp5: Add single quotation marks to not break parameters in many lines
- stack/erp5: Fix path to avoid one section to overwrite the file from another section
- stack/erp5: pass cloudooo host instead of the IP address. The IP address and cloudooo hostname are well in $SLAPPART/etc/hosts
- stack/erp5: Use use-ipv6 instance parameter to control where caucased listens when locally deployed.
  Drop incomplete 6tunnel usage on balancer toward zopes.

Tests are somehow happy test_result_module/20180626-62AD32BB test_result_module/20180626-70AC16FA

If no objections I would like to merge tomorrow but feel free to press merge for me :)

The goal is rebase !318 to fix the conflict and merge after tomorrow.

/cc @rafael @vpelletier 

/reviewed-on !347

setup.py

@@ -101,7 +101,6 @@ setup(name=name,
           'duplicity = slapos.recipe.duplicity:Recipe',
           'equeue = slapos.recipe.equeue:Recipe',
           'erp5.promise = slapos.recipe.erp5_promise:Recipe',
-          'erp5.test = slapos.recipe.erp5_test:Recipe',
           'erp5scalabilitytestbed = slapos.recipe.erp5scalabilitytestbed:Recipe',
           'erp5testnode = slapos.recipe.erp5testnode:Recipe',
           'firefox = slapos.recipe.firefox:Recipe',

slapos/recipe/erp5_test/__init__.py

@@ -25,74 +25,10 @@
 #
 ##############################################################################
 from slapos.recipe.librecipe import GenericBaseRecipe
-import urlparse
 
 # The follow recipes should be unified somehow in order to improve
 # code mantainence.
 
-class Recipe(GenericBaseRecipe):
-  def install(self):
-    testinstance = self.options['test-instance-path']
-    mysql_connection_string_list = []
-    path_list = []
-    # XXX: assume existence of 100 test databases, because slaves are not
-    # functional yet in slapos: testdb_0...testdb_100, with testuser_N
-    mysql_template = "%s@%s:%s %s %s"
-    mysql_url_list = self.options.get('mysql-url-list')
-    if mysql_url_list is None:
-      mysql_parsed = urlparse.urlparse(self.options['mysql-url'])
-      for i in range(0, 100):
-        mysql_connection_string_list.append(mysql_template % ('testdb_%s'% i,
-          mysql_parsed.hostname, mysql_parsed.port, 'testuser_%s'% i, mysql_parsed.password))
-      mysql_connection_string = mysql_template % ('erp5_test', mysql_parsed.hostname,
-        mysql_parsed.port, 'erp5_test', mysql_parsed.password)
-    else:
-      for mysql_url in mysql_url_list:
-        mysql_parsed = urlparse.urlparse(mysql_url)
-        mysql_connection_string_list.append(mysql_template % (
-          mysql_parsed.path.lstrip('/'),
-          mysql_parsed.hostname,
-          mysql_parsed.port,
-          mysql_parsed.username,
-          mysql_parsed.password,
-        ))
-      mysql_connection_string = mysql_connection_string_list.pop()
-    cloudooo_url = self.options['cloudooo-url']
-    cloudooo_parsed = urlparse.urlparse(cloudooo_url)
-    memcached_parsed = urlparse.urlparse(self.options['memcached-url'])
-    kumofs_parsed = urlparse.urlparse(self.options['kumofs-url'])
-    common_dict = dict(
-        instance_home=testinstance,
-        prepend_path=self.options['prepend-path'],
-        openssl_binary=self.options['openssl-binary'],
-        test_ca_path=self.options['certificate-authority-path'],
-    )
-    common_list = (
-      '--conversion_server_url=' + cloudooo_url,
-      '--conversion_server_retry_count=%s' % self.options.get('cloudooo-retry-count', 2),
-      # BBB: We still have test suites that only accept the following 2 options.
-      '--conversion_server_hostname=%s' % cloudooo_parsed.hostname,
-      '--conversion_server_port=%s' % cloudooo_parsed.port,
-      '--volatile_memcached_server_hostname=%s' % memcached_parsed.hostname,
-      '--volatile_memcached_server_port=%s' % memcached_parsed.port,
-      '--persistent_memcached_server_hostname=%s' % kumofs_parsed.hostname,
-      '--persistent_memcached_server_port=%s' % kumofs_parsed.port,
-    )
-    path_list.append(self.createPythonScript(self.options['run-unit-test'],
-        __name__ + '.test.runUnitTest',
-        ((self.options['run-unit-test-binary'],
-          '--erp5_sql_connection_string', mysql_connection_string,
-          '--extra_sql_connection_string_list', ','.join(
-            mysql_connection_string_list),
-          ) + common_list, common_dict)))
-    path_list.append(self.createPythonScript(self.options['run-test-suite'],
-        __name__ + '.test.runTestSuite',
-        ((self.options['run-test-suite-binary'],
-          '--db_list', ','.join(mysql_connection_string_list),
-          ) + common_list, common_dict)))
-
-    return path_list
-
 class CloudoooRecipe(GenericBaseRecipe):
   def install(self):
     path_list = []

stack/erp5/buildout.hash.cfg

@@ -82,7 +82,7 @@ md5sum = 3e650915959ff31c9c13c84069bbcd35
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = 7fb40624bd0dcd96a1df03395da64ebd
+md5sum = a4dda17c1671ead45ac92b7decc32228
 
 [template-balancer]
 filename = instance-balancer.cfg.in

stack/erp5/instance-zope.cfg.in

@@ -16,36 +16,45 @@ partition. No more (undefined result), no less (IndexError).
 -#}
 {% set ipv4 = (ipv4_set | list)[0] -%}
 
+{% set hosts_dict = {} -%}
+{% set port_dict = {} -%}
+{% for alias, url in (
+    ('erp5-memcached-volatile', slapparameter_dict['memcached-url']),
+    ('erp5-memcached-persistent', slapparameter_dict['kumofs-url']),
+    ('erp5-cloudooo', slapparameter_dict['cloudooo-url']),
+    ('erp5-smtp', slapparameter_dict['smtp-url']),
+  ) -%}
+{%   set parsed_url = urlparse.urlparse(url) -%}
+{%   do port_dict.__setitem__(alias, parsed_url.port)  -%}
+{%   do hosts_dict.__setitem__(alias, parsed_url.hostname)  -%}
+{%- endfor %}
+{% for i, url in enumerate(slapparameter_dict['mysql-url-list']) -%}
+{%   do hosts_dict.__setitem__(
+       'erp5-catalog-' ~ i,
+       urlparse.urlparse(url).hostname,
+     ) -%}
+{%- endfor %}
+{% do hosts_dict.update(slapparameter_dict['hosts-dict']) -%}
+
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
 mode = 644
 
-{% if slapparameter_dict['mysql-test-url-list'] -%}
-[{{ section('test-runner') }}]
-recipe = slapos.cookbook:erp5.test
-certificate-authority-path = ${test-certificate-authority:ca-dir}
-mysql-url-list = {{ dumps(slapparameter_dict['mysql-test-url-list']) }}
-kumofs-url = {{ dumps(slapparameter_dict['kumofs-url']) }}
-memcached-url = {{ dumps(slapparameter_dict['memcached-url']) }}
-cloudooo-url = {{ dumps(slapparameter_dict['cloudooo-url']) }}
-cloudooo-retry-count = {{ slapparameter_dict['cloudooo-retry-count'] }}
-test-instance-path = ${directory:unit-test-path}
-prepend-path = ${buildout:bin-directory}
-run-unit-test = ${buildout:bin-directory}/runUnitTest.real
-run-test-suite = ${buildout:bin-directory}/runTestSuite.real
-openssl-binary = ${test-certificate-authority:openssl-binary}
-run-unit-test-binary = {{ parameter_dict['bin-directory'] }}/runUnitTest
-run-test-suite-binary = {{ parameter_dict['bin-directory'] }}/runTestSuite
-
-[{{ section('run-unit-test-userhosts-wrapper') }}]
+[run-common]
 <= userhosts-wrapper-base
-wrapped-command-line = ${test-runner:run-unit-test}
-wrapper-path = ${buildout:bin-directory}/runUnitTest
-
-[{{ section('run-test-suite-userhosts-wrapper') }}]
-<= userhosts-wrapper-base
-wrapped-command-line = ${test-runner:run-test-suite}
-wrapper-path = ${buildout:bin-directory}/runTestSuite
+environment-extra =
+environment +=
+  TMP=${directory:tmp}
+  TMPDIR=${directory:tmp}
+  HOME=${buildout:directory}
+  PATH=${binary-link:target-directory}:{{ parameter_dict['coreutils'] }}/bin
+  TZ={{ slapparameter_dict['timezone'] }}
+  MATPLOTLIBRC={{ parameter_dict['matplotlibrc'] }}
+  INSTANCE_HOME=${:instance-home}
+{% if slapparameter_dict.get('wendelin-core-zblk-fmt') %}
+  WENDELIN_CORE_ZBLK_FMT={{ slapparameter_dict['wendelin-core-zblk-fmt'] }}
+{% endif %}
+  ${:environment-extra}
 
 [test-certificate-authority]
 recipe = slapos.cookbook:certificate_authority
@@ -57,6 +66,64 @@ ca-private = ${directory:test-ca-private}
 ca-certs = ${directory:test-ca-certs}
 ca-newcerts = ${directory:test-ca-newcerts}
 ca-crl = ${directory:test-ca-crl}
+
+{% if slapparameter_dict['mysql-test-url-list'] -%}
+[{{ section('run-unit-test-userhosts-wrapper') }}]
+<= userhosts-wrapper-base
+wrapped-command-line = ${runUnitTest:wrapper-path}
+wrapper-path = ${buildout:bin-directory}/runUnitTest
+
+[{{ section('run-test-suite-userhosts-wrapper') }}]
+<= userhosts-wrapper-base
+wrapped-command-line = ${runTestSuite:wrapper-path}
+wrapper-path = ${buildout:bin-directory}/runTestSuite
+
+{% set connection_string_list = [] -%}
+{% for url in slapparameter_dict['mysql-test-url-list'] -%}
+{%   set parsed_url = urlparse.urlparse(url) -%}
+{%   do connection_string_list.append(
+       '%s@%s:%s %s %s' % (
+         parsed_url.path.lstrip('/'),
+         parsed_url.hostname,
+         parsed_url.port,
+         parsed_url.username,
+         parsed_url.password,
+       ),
+     ) -%}
+{% endfor -%}
+[run-test-common]
+< = run-common
+environment-extra =
+  REAL_INSTANCE_HOME=${:instance-home}
+  OPENSSL_BINARY=${test-certificate-authority:openssl-binary}
+  TEST_CA_PATH=${test-certificate-authority:ca-dir}
+instance-home = ${directory:unit-test-path}
+wrapper-path = ${directory:bin}/${:command-name}.real
+command-line =
+  '{{ parameter_dict['bin-directory'] }}/${:command-name}'
+  ${:command-line-extra}
+  --conversion_server_url={{ slapparameter_dict['cloudooo-url'] }}
+  --conversion_server_retry_count={{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
+{#- BBB: We still have test suites that only accept the following 2 options. #}
+  --conversion_server_hostname=erp5-cloudooo
+  --conversion_server_port={{ port_dict['erp5-cloudooo'] }}
+  --volatile_memcached_server_hostname=erp5-memcached-volatile
+  --volatile_memcached_server_port={{ port_dict['erp5-memcached-volatile'] }}
+  --persistent_memcached_server_hostname=erp5-memcached-persistent
+  --persistent_memcached_server_port={{ port_dict['erp5-memcached-persistent'] }}
+
+[{{ section('runUnitTest') }}]
+< = run-test-common
+command-name = runUnitTest
+command-line-extra =
+  --erp5_sql_connection_string '{{ connection_string_list[0] }}'
+  --extra_sql_connection_string_list '{{ ','.join(connection_string_list[1:]) }}'
+
+[{{ section('runTestSuite') }}]
+< = run-test-common
+command-name = runTestSuite
+command-line-extra =
+  --db_list '{{ ','.join(connection_string_list) }}'
 {%- endif %}
 
 [directory]
@@ -127,27 +194,6 @@ ipv4 = {{ ipv4 }}
 ipv6 = {{ ipv6 }}
 {% endif -%}
 
-{% set hosts_dict = {} -%}
-{% for alias, url in (
-    ('erp5-memcached-volatile', slapparameter_dict['memcached-url']),
-    ('erp5-memcached-persistent', slapparameter_dict['kumofs-url']),
-    ('erp5-cloudooo', slapparameter_dict['cloudooo-url']),
-    ('erp5-smtp', slapparameter_dict['smtp-url']),
-  ) -%}
-{%   do hosts_dict.__setitem__(
-       alias,
-       urlparse.urlparse(url).hostname,
-     )  -%}
-{%- endfor %}
-{% for i, url in enumerate(slapparameter_dict['mysql-url-list']) -%}
-{%   do hosts_dict.__setitem__(
-       'erp5-catalog-' ~ i,
-       urlparse.urlparse(url).hostname,
-     ) -%}
-{%- endfor %}
-{% do hosts_dict.update(slapparameter_dict['hosts-dict']) -%}
-[host-common]
-
 [hosts-parameter]
 # Used for both hosts and hostaliases sections.
 host-dict = {{ dumps(hosts_dict) }}
@@ -208,18 +254,8 @@ pem = {{dumps(storage_dict.pop(k))}}
 {# endhack -#}
 
 [runzope-base]
-<= userhosts-wrapper-base
-environment +=
-  TMP=${directory:tmp}
-  TMPDIR=${directory:tmp}
-  HOME=${buildout:directory}
-  PATH=${directory:bin}:{{ parameter_dict['coreutils'] }}/bin
-  TZ={{ slapparameter_dict['timezone'] }}
-  INSTANCE_HOME=${directory:instance}
-  MATPLOTLIBRC={{ parameter_dict['matplotlibrc'] }}
-{% if slapparameter_dict.get('wendelin-core-zblk-fmt') %}
-  WENDELIN_CORE_ZBLK_FMT={{ slapparameter_dict['wendelin-core-zblk-fmt'] }}
-{% endif %}
+<= run-common
+instance-home = ${directory:instance}
 wrapped-command-line = '{{ bin_directory }}/runzope' -C '${:configuration-file}'
 private-dev-shm = {{ slapparameter_dict['private-dev-shm'] }}