diff --git a/component/apache/apache-backend.conf.in b/component/apache/apache-backend.conf.in index 65daaa8c3c6fd9fc9638f5713b6a1a8301a5f613..32886fc860b3e7faca4575252d6707dc836384e7 100644 --- a/component/apache/apache-backend.conf.in +++ b/component/apache/apache-backend.conf.in @@ -46,17 +46,42 @@ # # # The list of backends which apache should redirect to. # "backend-list": [ - # # (port, unused, internal_scheme) - # (8000, _, "http://10.0.0.10:8001"), - # (8002, _, "http://10.0.0.10:8003"), + # # (port, unused, internal_scheme, enable_authentication) + # (8000, _, "http://10.0.0.10:8001", True), + # (8002, _, "http://10.0.0.10:8003", False), # ], + # + # # The mapping of zope paths this apache should redirect to. + # # This is a Zope specific feature. + # # `enable_authentication` has same meaning as for `backend-list`. + # "zope-virtualhost-monster-backend-dict": { + # # {(ip, port): ( enable_authentication, {frontend_path: ( internal_scheme ) }, ) } + # ('[::1]', 8004): ( + # True, { + # 'zope-1': 'http://10.0.0.10:8001', + # 'zope-2': 'http://10.0.0.10:8002', + # }, + # ), + # }, # } # - # This sample of `parameter_dict` will make apache listening to : - # - 0.0.0.0:8000 redirecting internaly to http://10.0.0.10:8001 - # - [::1]:8000 redirecting internaly to http://10.0.0.10:8001 - # - 0.0.0.0:8002 redirecting internaly to http://10.0.0.10:8003 - # - [::1]:8002 redirecting internaly to http://10.0.0.10:8003 + # This sample of `parameter_dict` will make apache listening to : + # From to `backend-list`: + # - 0.0.0.0:8000 redirecting internaly to http://10.0.0.10:8001 and + # - [::1]:8000 redirecting internaly to http://10.0.0.10:8001 + # only accepting requests from clients who provide a valid SSL certificate trusted in `ca-cert`. + # - 0.0.0.0:8002 redirecting internaly to http://10.0.0.10:8003 + # - [::1]:8002 redirecting internaly to http://10.0.0.10:8003 + # accepting requests from any client. + # + # From zope-virtualhost-monster-backend-dict`: + # - [::1]:8004 with some path based rewrite-rules redirecting to: + # * http://10.0.0.10/8001 when path matches /zope-1(.*) + # * http://10.0.0.10/8002 when path matches /zope-2(.*) + # with some VirtualHostMonster rewrite rules so zope writes URLs with + # [::1]:8004 as server name. + # For more details, refer to + # https://docs.zope.org/zope2/zope2book/VirtualHosting.html#using-virtualhostroot-and-virtualhostbase-together -#} LoadModule unixd_module modules/mod_unixd.so LoadModule access_compat_module modules/mod_access_compat.so @@ -153,3 +178,28 @@ Listen {{ ip }}:{{ port }} RewriteRule ^/(.*) {{ backend }}/$1 [L,P] </VirtualHost> {% endfor -%} + + +{% for (ip, port), (enable_authentication, path_mapping) in parameter_dict.get('zope-virtualhost-monster-backend-dict', {}).items() -%} +Listen {{ ip }}:{{ port }} +<VirtualHost {{ ip }}:{{ port }}> + SSLEngine on +{% if enable_authentication and parameter_dict['ca-cert'] and parameter_dict['crl'] -%} + SSLVerifyClient require + SSLCACertificateFile {{ parameter_dict['ca-cert'] }} + SSLCARevocationCheck chain + SSLCARevocationFile {{ parameter_dict['crl'] }} + + LogFormat "%h %l %{REMOTE_USER}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined + + # We would like to separate the the authentificated logs. + # XXX filename ? is it log-rotated ? + ErrorLog "{{ parameter_dict['log-dir'] }}/apache-service-virtual-host-error.log" + CustomLog "{{ parameter_dict['log-dir'] }}/apache-service-virtual-host-access.log" combined +{% endif -%} + +{% for path, backend in path_mapping.items() %} + RewriteRule ^/{{path}}(.*) {{ backend }}/VirtualHostBase/https/{{ ip }}:{{ port }}/VirtualHostRoot/_vh_{{ path }}$1 [L,P] +{% endfor -%} +</VirtualHost> +{% endfor -%} \ No newline at end of file diff --git a/component/apache/buildout.cfg b/component/apache/buildout.cfg index c6039af637a7ba67eb8e5276d5b52492a46fb109..cf1f00b0eefe106256f9deb44d0a00104679ab72 100644 --- a/component/apache/buildout.cfg +++ b/component/apache/buildout.cfg @@ -16,6 +16,7 @@ extends = ../pkgconfig/buildout.cfg ../sqlite3/buildout.cfg ../zlib/buildout.cfg + ./buildout.hash.cfg [apr] recipe = slapos.recipe.build:download-unpacked @@ -196,6 +197,5 @@ make-targets = [template-apache-backend-conf] recipe = slapos.recipe.build:download shared = false -url = ${:_profile_base_location_}/apache-backend.conf.in -md5sum = 8f1f92ad308ab6bad973c790ee583428 +url = ${:_profile_base_location_}/${:filename} mode = 640 diff --git a/component/apache/buildout.hash.cfg b/component/apache/buildout.hash.cfg new file mode 100644 index 0000000000000000000000000000000000000000..089abcf6ab4329e31d2b4385caaa20bb937198e3 --- /dev/null +++ b/component/apache/buildout.hash.cfg @@ -0,0 +1,18 @@ +# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax. +# The only allowed lines here are (regexes): +# - "^#" comments, copied verbatim +# - "^[" section beginings, copied verbatim +# - lines containing an "=" sign which must fit in the following categorie. +# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file +# Copied verbatim. +# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported +# by the re-generation script. +# Re-generated. +# - other lines are copied verbatim +# Substitution (${...:...}), extension ([buildout] extends = ...) and +# section inheritance (< = ...) are NOT supported (but you should really +# not need these here). +[template-apache-backend-conf] +filename = apache-backend.conf.in +md5sum = 3b430ca726a2707e1b6a2ae41a6c8e21 + diff --git a/software/erp5/README.rst b/software/erp5/README.rst index e00c095663f6cc6df34b03d7ad54432d17ae80c4..084908d8999eed025d460de117554f25d0207c74 100644 --- a/software/erp5/README.rst +++ b/software/erp5/README.rst @@ -3,7 +3,7 @@ Available ``software-type`` values - ``default`` - Recommended for developemnt and production use. Automatic creation of + Recommended for development and production use. Automatic creation of erp5-site. Notes @@ -100,7 +100,7 @@ Zope partitions should be assigned port ranges starting at 2200, incrementing by some value which depends on how many zope process you want per partition (see the ``port-base`` parameter in ``zope-partition-dict``). -Notes to the Software Release developper: These ranges are not strictly +Notes to the Software Release developer: These ranges are not strictly defined. Not each port is actually used so one may reduce alread-assigned ranges if needed (ex: memcached partitions use actually fewer ports). There should be enough room for evolution (as between smtp and mariadb types). It is diff --git a/software/erp5/instance-erp5-input-schema.json b/software/erp5/instance-erp5-input-schema.json index ffccb44000fb02a5ed27988dfb6b7b067d48f0d1..caebfd3753b5949589c874e3ae981a765ab9da53 100644 --- a/software/erp5/instance-erp5-input-schema.json +++ b/software/erp5/instance-erp5-input-schema.json @@ -345,5 +345,26 @@ }, "type": "object" } + }, + "test-runner": { + "description": "Test runner parameters.", + "properties": { + "enabled": { + "description": "Generate helper scripts to run test suite.", + "default": true, + "type": "boolean" + }, + "node-count": { + "description": "Number of tests this instance can execute in parrallel. This must be at least equal to the number of nodes configured on testnode running the test", + "default": 3, + "type": "integer" + }, + "extra-database-count": { + "description": "Number of extra databases this instance tests will need.", + "default": 3, + "type": "integer" + } + }, + "type": "object" } } diff --git a/software/erp5/instance-mariadb-schema.json b/software/erp5/instance-mariadb-schema.json index c1fb3d36d613192af1ba0718d93975e9de5fc8b4..85ebe4215ec5339796458f2e2b4ed5477b2d8336 100644 --- a/software/erp5/instance-mariadb-schema.json +++ b/software/erp5/instance-mariadb-schema.json @@ -48,12 +48,6 @@ }, "type": "array" }, - "test-database-amount": { - "description": "The number of test databases to create, adding auto-generated entries to database-list", - "default": 1, - "minimum": 0, - "type": "integer" - }, "catalog-backup": { "description": "Backup control knobs", "properties": { diff --git a/stack/erp5/buildout.hash.cfg b/stack/erp5/buildout.hash.cfg index 5ae2c7206e0531e9a5b259fbc7b8e38d30b3e556..eebdb94290cdb6340120d11275c7575b646fe374 100644 --- a/stack/erp5/buildout.hash.cfg +++ b/stack/erp5/buildout.hash.cfg @@ -74,7 +74,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e [template-erp5] filename = instance-erp5.cfg.in -md5sum = 83e0f627633ec3e394dc27d06c627b3b +md5sum = 64fd49395de02e6c2107afd1dd9cbdc9 [template-zeo] filename = instance-zeo.cfg.in @@ -82,11 +82,11 @@ md5sum = 3e650915959ff31c9c13c84069bbcd35 [template-zope] filename = instance-zope.cfg.in -md5sum = e73678921067506e710ae11e41f0a9a8 +md5sum = 408c8f358935d3167448ea3732adc994 [template-balancer] filename = instance-balancer.cfg.in -md5sum = 89872075cd2f31d824369447aaf0532f +md5sum = dccdf2a8cbfa57e49fa04be9ba4ff595 [template-haproxy-cfg] filename = haproxy.cfg.in diff --git a/stack/erp5/instance-balancer.cfg.in b/stack/erp5/instance-balancer.cfg.in index a47d696601654e96bdcd29f0bbdb6cfe66f740e5..d2203f683fd1f39f786bb6d9206ae692678148b0 100644 --- a/stack/erp5/instance-balancer.cfg.in +++ b/stack/erp5/instance-balancer.cfg.in @@ -97,10 +97,13 @@ ipv4 = {{ ipv4 }} {% endif -%} {% set haproxy_dict = {} -%} {% set apache_dict = {} -%} +{% set zope_virtualhost_monster_backend_dict = {} %} +{% set test_runner_url_dict = {} %} {# family_name => list of apache URLs #} {% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%} {% for family_name, parameter_id_list in sorted( slapparameter_dict['zope-family-dict'].iteritems()) -%} {% set zope_family_address_list = [] -%} +{% set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%} {% set has_webdav = [] -%} {% for parameter_id in parameter_id_list -%} {% set zope_address_list = slapparameter_dict[parameter_id] -%} @@ -121,8 +124,27 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }} {% set zope_effective_address = zope_address -%} {% endif -%} {% do zope_family_address_list.append((zope_effective_address, maxconn, webdav)) -%} + +{# # Generate entries with rewrite rule for test runnners #} +{% set test_runner_backend_mapping = {} %} +{% set test_runner_apache_url_list = [] %} +{% set test_runner_external_port = next_port() %} +{% for i, (test_runner_internal_ip, test_runner_internal_port) in + enumerate(slapparameter_dict[parameter_id ~ '-test-runner-address-list']) %} +{% do test_runner_backend_mapping.__setitem__( + 'unit_test_' ~ i, + 'http://' ~ test_runner_internal_ip ~ ':' ~ test_runner_internal_port ) %} +{% do test_runner_apache_url_list.append( + 'https://' ~ ipv4 ~ ':' ~ test_runner_external_port ~ '/unit_test_' ~ i ~ '/' ) %} +{% endfor %} +{% do zope_virtualhost_monster_backend_dict.__setitem__( + (ipv4, test_runner_external_port), + ( ssl_authentication, test_runner_backend_mapping ) ) -%} +{% do test_runner_url_dict.__setitem__(family_name, test_runner_apache_url_list) -%} + {% endfor -%} {% endfor -%} + {# Make rendering fail artificially if any family has no known backend. # This is useful as haproxy's hot-reconfiguration mechanism is # supervisord-incompatible. @@ -140,7 +162,6 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }} {% set internal_scheme = 'http' -%} {% set external_scheme = 'https' -%} {% endif -%} -{% set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%} {% do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ backend_path, ssl_authentication)) -%} {% endfor -%} @@ -173,6 +194,7 @@ crl = ${directory:apache-conf}/crl.pem [apache-conf-parameter-dict] backend-list = {{ dumps(apache_dict.values()) }} +zope-virtualhost-monster-backend-dict = {{ dumps(zope_virtualhost_monster_backend_dict) }} ip-list = {{ dumps(apache_ip_list) }} pid-file = ${directory:run}/apache.pid log-dir = ${directory:log} @@ -225,6 +247,10 @@ recipe = slapos.cookbook:publish.serialised {{ family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %} {{ family_name }} = {{ scheme ~ '://' ~ ipv4 ~ ':' ~ apache_port }} {% endfor -%} +{% for family_name, test_runner_url_list in test_runner_url_dict.items() -%} +{{ family_name ~ '-test-runner-url-list' }} = {{ dumps(test_runner_url_list) }} +{% endfor -%} + monitor-base-url = ${monitor-publish-parameters:monitor-base-url} [apache-ssl] diff --git a/stack/erp5/instance-erp5.cfg.in b/stack/erp5/instance-erp5.cfg.in index fa3853a71474275133d10c42bef9ad5e18ec080d..de856c696203d70e2aa4f7ff61c3f555ffae6c07 100644 --- a/stack/erp5/instance-erp5.cfg.in +++ b/stack/erp5/instance-erp5.cfg.in @@ -8,6 +8,16 @@ {% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%} {% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%} {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%} +{% set test_runner_enabled = slapparameter_dict.get('test-runner', {}).get('enabled', True) -%} +{% set test_runner_node_count = slapparameter_dict.get('test-runner', {}).get('node-count', 3) -%} +{% set test_runner_extra_database_count = slapparameter_dict.get('test-runner', {}).get('extra-database-count', 3) -%} +{% set test_runner_total_database_count = test_runner_node_count * (1 + test_runner_extra_database_count) -%} +{# Backward compatibility for mariadb.test-database-amount #} +{% set mariadb_test_database_amount = slapparameter_dict.get('mariadb', {}).get('test-database-amount') -%} +{% if mariadb_test_database_amount is not none -%} +{% set test_runner_total_database_count = mariadb_test_database_amount %} +{% set test_runner_enabled = mariadb_test_database_amount > 0 %} +{% endif -%} {% set monitor_base_url_dict = {} -%} {% set caucase_url = slapparameter_dict.get('caucase', {}).pop('url', '') -%} {% set monitor_dict = slapparameter_dict.get('monitor', {}) %} @@ -44,7 +54,7 @@ config-name = {{ name }} {{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }} {{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }} -{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099, 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', '') }, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }} +{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099, 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', ''), 'test-database-amount': test_runner_total_database_count}, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }} {% if has_posftix -%} {{ request('smtp', 'postfix', 'smtp', {'tcpv4-port': 2025, 'smtpd-sasl-user': 'erp5@nowhere'}, key_config={'smtpd-sasl-password': 'publish-early:smtpd-sasl-password'}) }} {%- else %} @@ -93,44 +103,6 @@ connection-http-url = {{ caucase_url }} {% endif -%} {% endfor -%} -[publish-early] -recipe = slapos.cookbook:publish-early --init = - inituser-password gen-password:passwd - deadlock-debugger-password gen-deadlock-debugger-password:passwd -{%- if has_posftix %} - smtpd-sasl-password gen-smtpd-sasl-password:passwd -{%- endif %} -{%- if neo %} - neo-cluster gen-neo-cluster:name -{%- if neo[0] %} -neo-cluster = {{ dumps(neo[0]) }} -{%- endif %} -{%- endif %} -{%- set inituser_password = slapparameter_dict.get('inituser-password') %} -{%- if inituser_password %} -inituser-password = {{ dumps(inituser_password) }} -{%- endif %} -{%- set deadlock_debugger_password = slapparameter_dict.get('deadlock-debugger-password') -%} -{%- if deadlock_debugger_password %} -deadlock-debugger-password = {{ dumps(deadlock_debugger_password) }} -{%- endif %} - -[gen-password] -recipe = slapos.cookbook:generate.password -storage-path = - -[gen-deadlock-debugger-password] -<= gen-password - -[gen-neo-cluster-base] -<= gen-password - -[gen-neo-cluster] -name = neo-${gen-neo-cluster-base:passwd} - -[gen-smtpd-sasl-password] -< = gen-password {% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%} {% set zope_address_list_id_dict = {} -%} @@ -142,6 +114,9 @@ return = zope-address-list hosts-dict monitor-base-url +{%- if test_runner_enabled %} + test-runner-address-list +{% endif %} {% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_run_my_doc' -%} {% if has_jupyter -%} {% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%} @@ -169,6 +144,8 @@ config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }} config-cloudooo-retry-count = {{ slapparameter_dict.get('cloudooo-retry-count', 2) }} config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }} config-zodb-dict = {{ dumps(zodb_dict) }} +config-test-runner-enabled = {{ dumps(test_runner_enabled) }} +config-test-runner-node-count = {{ dumps(test_runner_node_count) }} {% for server_type, server_dict in storage_dict.iteritems() -%} {% if server_type == 'neo' -%} config-neo-cluster = ${publish-early:neo-cluster} @@ -183,6 +160,7 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port} software-type = zope {% set zope_family_dict = {} -%} +{% set zope_family_name_list = [] -%} {% set zope_backend_path_dict = {} -%} {% set ssl_authentication_dict = {} -%} {% set jupyter_zope_family_default = [] -%} @@ -190,6 +168,7 @@ software-type = zope {% set partition_name = 'zope-' ~ custom_name -%} {% set section_name = 'request-' ~ partition_name -%} {% set zope_family = zope_parameter_dict.get('family', 'default') -%} +{% do zope_family_name_list.append(zope_family) %} {% set backend_path = zope_parameter_dict.get('backend-path', '/') % {'site-id': site_id} %} {# # default jupyter zope family is first zope family. -#} {# # use list.append() to update it, because in jinja2 set changes only local scope. -#} @@ -214,6 +193,7 @@ config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longreques config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "10MB")) }} config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }} config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }} +config-test-runner-apache-url-list = ${publish-early:{{ zope_family }}-test-runner-url-list} {% endfor -%} {# if not explicitly configured, connect jupyter to first zope family, which -#} @@ -268,6 +248,9 @@ return = {%- for family in zope_family_dict %} {{ family }} {{ family }}-v6 + {% if test_runner_enabled %} + {{ family }}-test-runner-url-list + {% endif %} {% endfor -%} {% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%} @@ -275,6 +258,9 @@ config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }} config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }} {% for zope_section_id, name in zope_address_list_id_dict.items() -%} config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list}' }} +{% if test_runner_enabled -%} +config-{{ name }}-test-runner-address-list = {{ ' ${' ~ zope_section_id ~ ':connection-test-runner-address-list}' }} +{% endif -%} {% endfor -%} # XXX: should those really be same for all families ? config-haproxy-server-check-path = {{ dumps(balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) }} @@ -335,6 +321,57 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts- {% for name, value in publish_dict.items() -%} {{ name }} = {{ value }} {% endfor -%} +{% for zope_family_name in zope_family_name_list -%} +{{ zope_family_name }}-test-runner-url-list = ${request-balancer:connection-{{ zope_family_name }}-test-runner-url-list} +{% endfor -%} + + +[publish-early] +recipe = slapos.cookbook:publish-early +-init = + inituser-password gen-password:passwd + deadlock-debugger-password gen-deadlock-debugger-password:passwd +{%- if has_posftix %} + smtpd-sasl-password gen-smtpd-sasl-password:passwd +{%- endif %} +{% for zope_family_name in zope_family_name_list %} + {{ zope_family_name }}-test-runner-url-list default-balancer-test-runner-url-list:default +{% endfor -%} +{%- if neo %} + neo-cluster gen-neo-cluster:name +{%- if neo[0] %} +neo-cluster = {{ dumps(neo[0]) }} +{%- endif %} +{%- endif %} +{%- set inituser_password = slapparameter_dict.get('inituser-password') %} +{%- if inituser_password %} +inituser-password = {{ dumps(inituser_password) }} +{%- endif %} +{%- set deadlock_debugger_password = slapparameter_dict.get('deadlock-debugger-password') -%} +{%- if deadlock_debugger_password %} +deadlock-debugger-password = {{ dumps(deadlock_debugger_password) }} +{%- endif %} + + +[default-balancer-test-runner-url-list] +recipe = +default = not-ready + +[gen-password] +recipe = slapos.cookbook:generate.password +storage-path = + +[gen-deadlock-debugger-password] +<= gen-password + +[gen-neo-cluster-base] +<= gen-password + +[gen-neo-cluster] +name = neo-${gen-neo-cluster-base:passwd} + +[gen-smtpd-sasl-password] +< = gen-password [monitor-instance-parameter] monitor-httpd-port = 8386 diff --git a/stack/erp5/instance-zope.cfg.in b/stack/erp5/instance-zope.cfg.in index 3864c7f9a498fbf1589ac78ba7377d5e2356a59f..cd198017e5e288bdb04a1e4eb8482f665d6f0354 100644 --- a/stack/erp5/instance-zope.cfg.in +++ b/stack/erp5/instance-zope.cfg.in @@ -8,6 +8,9 @@ {% set node_id_index_format = '-%%0%ii' % (len(str(instance_index_list[-1])), ) -%} {% set part_list = [] -%} {% set publish_list = [] -%} +{% set test_runner_address_list = [] -%} +{% set test_runner_enabled = slapparameter_dict['test-runner-enabled'] -%} +{% set test_runner_node_count = slapparameter_dict['test-runner-node-count'] -%} {% set longrequest_logger_base_path = buildout_directory ~ '/var/log/longrequest_logger_' -%} {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%} {% set bin_directory = parameter_dict['buildout-bin-directory'] -%} @@ -68,84 +71,6 @@ ca-certs = ${directory:test-ca-certs} ca-newcerts = ${directory:test-ca-newcerts} ca-crl = ${directory:test-ca-crl} -{% if saucelabs_dict -%} -[test-zelenium-runner-parameter] -configuration = {{ dumps(saucelabs_dict) }} -user = {{ dumps(slapparameter_dict['inituser-login']) }} -password = {{ dumps(slapparameter_dict['inituser-password']) }} -bin-path = {{ bin_directory }}/{{ parameter_dict['egg-interpreter'] }} - -[{{ section('test-zelenium-runner') }}] -<= jinja2-template-base -template = {{ parameter_dict['run-zelenium-template'] }} -rendered = ${directory:bin}/runTestSuite -mode = 755 -context = - import json_module json - key configuration test-zelenium-runner-parameter:configuration - key user test-zelenium-runner-parameter:user - key password test-zelenium-runner-parameter:password - key bin_path test-zelenium-runner-parameter:bin-path -{% else -%} -{% if slapparameter_dict['mysql-test-url-list'] -%} -[{{ section('run-unit-test-userhosts-wrapper') }}] -<= userhosts-wrapper-base -wrapped-command-line = ${runUnitTest:wrapper-path} -wrapper-path = ${buildout:bin-directory}/runUnitTest - -[{{ section('run-test-suite-userhosts-wrapper') }}] -<= userhosts-wrapper-base -wrapped-command-line = ${runTestSuite:wrapper-path} -wrapper-path = ${buildout:bin-directory}/runTestSuite - -{% set connection_string_list = [] -%} -{% for url in slapparameter_dict['mysql-test-url-list'] -%} -{% set parsed_url = urlparse.urlparse(url) -%} -{% do connection_string_list.append( - '%s@%s:%s %s %s' % ( - parsed_url.path.lstrip('/'), - parsed_url.hostname, - parsed_url.port, - parsed_url.username, - parsed_url.password, - ), - ) -%} -{% endfor -%} -[run-test-common] -< = run-common -environment-extra = - REAL_INSTANCE_HOME=${:instance-home} - OPENSSL_BINARY=${test-certificate-authority:openssl-binary} - TEST_CA_PATH=${test-certificate-authority:ca-dir} -instance-home = ${directory:unit-test-path} -wrapper-path = ${directory:bin}/${:command-name}.real -command-line = - '{{ parameter_dict['bin-directory'] }}/${:command-name}' - ${:command-line-extra} - --conversion_server_url={{ slapparameter_dict['cloudooo-url'] }} - --conversion_server_retry_count={{ slapparameter_dict.get('cloudooo-retry-count', 2) }} -{#- BBB: We still have test suites that only accept the following 2 options. #} - --conversion_server_hostname=erp5-cloudooo - --conversion_server_port={{ port_dict['erp5-cloudooo'] }} - --volatile_memcached_server_hostname=erp5-memcached-volatile - --volatile_memcached_server_port={{ port_dict['erp5-memcached-volatile'] }} - --persistent_memcached_server_hostname=erp5-memcached-persistent - --persistent_memcached_server_port={{ port_dict['erp5-memcached-persistent'] }} - -[{{ section('runUnitTest') }}] -< = run-test-common -command-name = runUnitTest -command-line-extra = - --erp5_sql_connection_string '{{ connection_string_list[0] }}' - --extra_sql_connection_string_list '{{ ','.join(connection_string_list[1:]) }}' - -[{{ section('runTestSuite') }}] -< = run-test-common -command-name = runTestSuite -command-line-extra = - --db_list '{{ ','.join(connection_string_list) }}' -{%- endif %} -{%- endif %} [directory] recipe = slapos.cookbook:mkdirectory @@ -472,6 +397,111 @@ wrapper-path = ${buildout:bin-directory}/${:_buildout_section_name_} [{{ section("wait_activities") }}] <= watch_activities + +{% if saucelabs_dict -%} +[test-zelenium-runner-parameter] +configuration = {{ dumps(saucelabs_dict) }} +user = {{ dumps(slapparameter_dict['inituser-login']) }} +password = {{ dumps(slapparameter_dict['inituser-password']) }} +bin-path = {{ bin_directory }}/{{ parameter_dict['egg-interpreter'] }} + +[{{ section('test-zelenium-runner') }}] +<= jinja2-template-base +template = {{ parameter_dict['run-zelenium-template'] }} +rendered = ${directory:bin}/runTestSuite +mode = 755 +context = + import json_module json + key configuration test-zelenium-runner-parameter:configuration + key user test-zelenium-runner-parameter:user + key password test-zelenium-runner-parameter:password + key bin_path test-zelenium-runner-parameter:bin-path +{% else -%} +{% if test_runner_enabled and test_runner_node_count -%} +{% for _ in range(test_runner_node_count) %} +{% do test_runner_address_list.append((ipv4, next_port())) %} +{% endfor %} + +[{{ section('run-unit-test-userhosts-wrapper') }}] +<= userhosts-wrapper-base +wrapped-command-line = ${runUnitTest:wrapper-path} +wrapper-path = ${buildout:bin-directory}/runUnitTest + +[{{ section('run-test-suite-userhosts-wrapper') }}] +<= userhosts-wrapper-base +wrapped-command-line = ${runTestSuite:wrapper-path} +wrapper-path = ${buildout:bin-directory}/runTestSuite + +{% set connection_string_list = [] -%} +{% for url in slapparameter_dict['mysql-test-url-list'] -%} +{% set parsed_url = urlparse.urlparse(url) -%} +{% do connection_string_list.append( + '%s@%s:%s %s %s' % ( + parsed_url.path.lstrip('/'), + parsed_url.hostname, + parsed_url.port, + parsed_url.username, + parsed_url.password, + ), + ) -%} +{% endfor -%} +[run-test-common] +< = run-common +environment-extra = + REAL_INSTANCE_HOME=${:instance-home} + OPENSSL_BINARY=${test-certificate-authority:openssl-binary} + TEST_CA_PATH=${test-certificate-authority:ca-dir} +instance-home = ${directory:unit-test-path} +wrapper-path = ${directory:bin}/${:command-name}.real +command-line = + '{{ parameter_dict['bin-directory'] }}/${:command-name}' + ${:command-line-extra} + --conversion_server_url={{ slapparameter_dict['cloudooo-url'] }} + --conversion_server_retry_count={{ slapparameter_dict.get('cloudooo-retry-count', 2) }} +{#- BBB: We still have test suites that only accept the following 2 options. #} + --conversion_server_hostname=erp5-cloudooo + --conversion_server_port={{ port_dict['erp5-cloudooo'] }} + --volatile_memcached_server_hostname=erp5-memcached-volatile + --volatile_memcached_server_port={{ port_dict['erp5-memcached-volatile'] }} + --persistent_memcached_server_hostname=erp5-memcached-persistent + --persistent_memcached_server_port={{ port_dict['erp5-memcached-persistent'] }} + +[{{ section('runUnitTest') }}] +< = run-test-common +command-name = runUnitTest +command-line-extra = + --erp5_sql_connection_string '{{ connection_string_list[0] }}' + --extra_sql_connection_string_list '{{ ','.join(connection_string_list[1:]) }}' + --zserver {{ test_runner_address_list[0][0] ~ ':' ~ test_runner_address_list[0][1] }} + --zserver_frontend_url {{ slapparameter_dict['test-runner-apache-url-list'][0] }} + +[{{ section('runTestSuite') }}] +< = run-test-common +command-name = runTestSuite +command-line-extra = + --db_list '{{ ','.join(connection_string_list) }}' +environment-extra += + {#- turn a list of (ip, port) in a list of 'ip:port' #} + {% set zserver_address_list = [] -%} + {% for ip, port in test_runner_address_list %} + {% do zserver_address_list.append(ip ~ ':' ~ port) %} + {% endfor -%} + zserver_address_list={{ ','.join(zserver_address_list) }} + zserver_frontend_url_list={{ ','.join(slapparameter_dict['test-runner-apache-url-list']) }} + +[{{ section("promise-test-runner-apache-url") }}] +# promise to wait for apache partition to have returned the parameter +recipe = slapos.cookbook:check_parameter +value = {{č½slapparameter_dict['test-runner-apache-url-list'] }} +expected-not-value = not-ready +path = ${directory:promises}/${:_buildout_section_name_} +expected-value = + +{%- endif %} +{%- endif %} + + + [publish] recipe = slapos.cookbook:publish.serialised zope-address-list = {{ dumps(publish_list) }} @@ -484,6 +514,7 @@ hard to guess. -#} hosts-dict = {{ dumps(hosts_dict) }} monitor-base-url = ${monitor-publish-parameters:monitor-base-url} +test-runner-address-list = {{ dumps(test_runner_address_list) }} [monitor-instance-parameter] monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}