Commit a729a677 authored by Jérome Perrin's avatar Jérome Perrin

stack/erp5: also use backup cloudooo by default

We have two public cloudooo instances, https:///cloudooo.erp5.net/ and
https://cloudooo1.erp5.net/ but the software release uses only the first
one. This changes to use both instances by default, in a random order.
This happens for the ERP5 instance and also for the test runner.

This also includes a breaking change: the parameter is renamed from
`cloudooo-url` into `cloudooo-url-list` and instead of being a coma
delimited list of URLs, this is now a proper list of URLs.

Some obsolete code and comments about the old `erp5-cloudooo` partition
that used to part of ERP5 software release have also been removed.

On testing side, ZopeSkinsMixin._setUpClass is changed be made more
robust, by waiting that all activities are processed and using a new
xmlrpc client in every iteration ( to prevent issues like the one from
https://erp5js.nexedi.net/#/test_result_module/20230107-548523A7/10 )
parent 99b209eb
......@@ -145,7 +145,7 @@
"type": "object"
},
"hosts-dict": {
"description": "Host entries to be used in addition to and/or overriding auto-generated ones (erp5-catalog-0, erp5-cloudooo, erp5-memcached-persistent, erp5-memcached-volatile and erp5-smtp)",
"description": "Host entries to be used in addition to and/or overriding auto-generated ones (erp5-catalog-0, erp5-memcached-persistent, erp5-memcached-volatile and erp5-smtp)",
"patternProperties": {
".*": {
"description": "An IP or domain name to which current entry will resolve",
......@@ -294,11 +294,14 @@
],
"type": "object"
},
"cloudooo-url": {
"description": "Format conversion service URL",
"pattern": "^https?://",
"type": "string",
"format": "uri"
"cloudooo-url-list": {
"description": "Format conversion service URLs",
"type": "array",
"items": {
"pattern": "^https?://",
"type": "string",
"format": "uri"
}
},
"cloudooo-retry-count": {
"description": "Define retry count for cloudooo in network error case in test",
......
......@@ -28,6 +28,7 @@
import contextlib
import glob
import http.client
import json
import os
import shutil
......@@ -450,17 +451,20 @@ class ZopeSkinsMixin:
def _setUpClass(cls):
super()._setUpClass()
param_dict = cls.getRootPartitionConnectionParameterDict()
with cls.getXMLRPCClient() as erp5_xmlrpc_client:
# wait for ERP5 to be ready (TODO: this should probably be a promise)
for _ in range(120):
time.sleep(1)
# wait for ERP5 to be ready and have processed all activities
# from initial setup
for _ in range(120):
with cls.getXMLRPCClient() as erp5_xmlrpc_client:
try:
erp5_xmlrpc_client.getTitle()
if erp5_xmlrpc_client.portal_activities.countMessage() == 0:
break
except (xmlrpc.client.ProtocolError,
xmlrpc.client.Fault):
xmlrpc.client.Fault,
http.client.HTTPException):
pass
else:
break
time.sleep(5)
else:
raise AssertionError("ERP5 is not ready")
@classmethod
def _getAuthenticatedZopeUrl(cls, path, family_name='default'):
......@@ -924,3 +928,65 @@ class TestZopePublisherTimeout(ZopeSkinsMixin, ERP5InstanceTestCase):
self._getAuthenticatedZopeUrl('ERP5Site_doSlowRequest', family_name='no-timeout'),
verify=False,
timeout=6)
class TestCloudooo(ZopeSkinsMixin, ERP5InstanceTestCase):
"""Test ERP5 can be instantiated with cloudooo parameters
"""
__partition_reference__ = 'c'
@classmethod
def getInstanceParameterDict(cls):
return {
'_':
json.dumps({
'cloudooo-url-list': [
'https://cloudooo1.example.com/',
'https://cloudooo2.example.com/',
],
'cloudooo-retry-count': 123,
})
}
def test_cloudooo_url_list_preference(self):
self.assertEqual(
requests.get(
self._getAuthenticatedZopeUrl(
'portal_preferences/getPreferredDocumentConversionServerUrlList'),
verify=False).text,
"['https://cloudooo1.example.com/', 'https://cloudooo2.example.com/']")
@unittest.expectedFailure # setting "retry" is not implemented
def test_cloudooo_retry_count_preference(self):
self.assertEqual(
requests.get(
self._getAuthenticatedZopeUrl(
'portal_preferences/getPreferredDocumentConversionServerRetry'),
verify=False).text,
"123")
class TestCloudoooDefaultParameter(ZopeSkinsMixin, ERP5InstanceTestCase):
"""Test default ERP5 cloudooo parameters
"""
__partition_reference__ = 'cd'
def test_cloudooo_url_list_preference(self):
self.assertIn(
requests.get(
self._getAuthenticatedZopeUrl(
'portal_preferences/getPreferredDocumentConversionServerUrlList'),
verify=False).text,
[
"['https://cloudooo1.erp5.net/', 'https://cloudooo.erp5.net/']",
"['https://cloudooo.erp5.net/', 'https://cloudooo1.erp5.net/']",
])
@unittest.expectedFailure # default value of "retry" does not match schema
def test_cloudooo_retry_count_preference(self):
self.assertEqual(
requests.get(
self._getAuthenticatedZopeUrl(
'portal_preferences/getPreferredDocumentConversionServerRetry'),
verify=False).text,
"2")
......@@ -14,7 +14,7 @@
# not need these here).
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 6d9b94b50ea46b0b98c9019d74780876
md5sum = 2491969d49f8bf9b172e89b1c0e9d98e
[template-balancer]
filename = instance-balancer.cfg.in
......
......@@ -21,6 +21,10 @@
{% set test_runner_total_database_count = mariadb_test_database_amount %}
{% set test_runner_enabled = mariadb_test_database_amount > 0 %}
{% endif -%}
{# Backward compatibility for cloudooo-url #}
{% if slapparameter_dict.get('cloudooo-url') -%}
{% do slapparameter_dict.setdefault('cloudooo-url-list', slapparameter_dict['cloudooo-url'].split(',')) %}
{% endif -%}
{% set test_runner_random_activity_priority = slapparameter_dict.get('test-runner', {}).get('random-activity-priority') -%}
{% set monitor_base_url_dict = {} -%}
{% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
......@@ -231,7 +235,7 @@ return =
{% endif -%}
config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }}
config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }}
config-cloudooo-url-list = {{ dumps(slapparameter_dict.get('cloudooo-url-list', default_cloudooo_url_list)) }}
config-caucase-url = {{ dumps(caucase_url) }}
config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
......
......@@ -174,7 +174,7 @@ context =
key dash_location dash:location
key bash_location bash:location
key dcron_location dcron:location
key default_cloudooo_url erp5-defaults:cloudooo-connection-url
key default_cloudooo_url_list erp5-defaults:cloudooo-connection-url-list
key erp5_location erp5:location
key findutils_location findutils:location
key gzip_location gzip:location
......@@ -331,7 +331,11 @@ repository_id_list = erp5 erp5-bin erp5-doc
# ERP5 defaults, which can be overridden in inheriting recipes (e.g. wendelin)
[erp5-defaults]
cloudooo-connection-url = https://cloudooo.erp5.net/
# Default cloudooo is https://cloudooo.erp5.net/ and https://cloudooo1.erp5.net/ in random order.
# The random order is applied in instance profile
cloudooo-connection-url-list =
https://cloudooo.erp5.net/
https://cloudooo1.erp5.net/
# Jupyter is by default disabled in ERP5
jupyter-enable-default = false
# WCFS is by default disabled in ERP5
......
......@@ -70,11 +70,11 @@ md5sum = b95084ae9eed95a68eada45e28ef0c04
[template]
filename = instance.cfg.in
md5sum = 74979d6743cdc7ce3d2f17680115458d
md5sum = 3f7b28085ceff321a3cb785db60f7c3e
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 52dfc0c9a4547b1e928707ef5caeecdb
md5sum = 30a1e738a8211887e75a5e75820e5872
[template-zeo]
filename = instance-zeo.cfg.in
......@@ -86,7 +86,7 @@ md5sum = 0ac4b74436f554cd677f19275d18d880
[template-zope]
filename = instance-zope.cfg.in
md5sum = 57a19c90d281b8cbf3e36207db48f0bd
md5sum = 7d3b3769b60f0cc2883beeb05cdf82d7
[template-balancer]
filename = instance-balancer.cfg.in
......
......@@ -21,6 +21,10 @@
{% set test_runner_total_database_count = mariadb_test_database_amount %}
{% set test_runner_enabled = mariadb_test_database_amount > 0 %}
{% endif -%}
{# Backward compatibility for cloudooo-url #}
{% if slapparameter_dict.get('cloudooo-url') -%}
{% do slapparameter_dict.setdefault('cloudooo-url-list', slapparameter_dict['cloudooo-url'].split(',')) %}
{% endif -%}
{% set test_runner_random_activity_priority = slapparameter_dict.get('test-runner', {}).get('random-activity-priority') -%}
{% set monitor_base_url_dict = {} -%}
{% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
......@@ -234,7 +238,7 @@ return =
{% endif -%}
config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }}
config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }}
config-cloudooo-url-list = {{ dumps(slapparameter_dict.get('cloudooo-url-list', default_cloudooo_url_list)) }}
config-caucase-url = {{ dumps(caucase_url) }}
config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
......
......@@ -33,7 +33,6 @@ partition. No more (undefined result), no less (IndexError).
{% for alias, url in (
('erp5-memcached-volatile', slapparameter_dict['memcached-url']),
('erp5-memcached-persistent', slapparameter_dict['kumofs-url']),
('erp5-cloudooo', slapparameter_dict['cloudooo-url']),
('erp5-smtp', slapparameter_dict['smtp-url']),
) -%}
{% set parsed_url = urllib_parse.urlparse(url) -%}
......@@ -282,12 +281,7 @@ bt5 = {{ slapparameter_dict['bt5'] }}
bt5-repository-url = {{ slapparameter_dict['bt5-repository-url'] }}
id-store-interval = {{ dumps(slapparameter_dict['id-store-interval']) }}
home = ${buildout:directory}
# We only want to change the hostname to 'erp5-cloudooo' if we use the internal
# cloudooo. We plan to remove the ability to have an internal one, so this
# heuristic is enough.
{% set cloudooo = urllib_parse.urlsplit(slapparameter_dict['cloudooo-url']) -%}
cloudooo-url = {{ (cloudooo if cloudooo.port == None else
cloudooo._replace(netloc='erp5-cloudooo:%s' % cloudooo.port)).geturl() }}
cloudooo-url = {{ ",".join(slapparameter_dict['cloudooo-url-list']) }}
{% endif -%}
developer-list = {{ dumps(slapparameter_dict['developer-list']) }}
......@@ -506,11 +500,8 @@ wrapper-path = ${directory:bin}/${:command-name}.real
command-line =
'{{ parameter_dict['bin-directory'] }}/${:command-name}'
${:command-line-extra}
--conversion_server_url={{ slapparameter_dict['cloudooo-url'] }}
--conversion_server_url={{ ",".join(slapparameter_dict['cloudooo-url-list']) }}
--conversion_server_retry_count={{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
{#- BBB: We still have test suites that only accept the following 2 options. #}
--conversion_server_hostname=erp5-cloudooo
--conversion_server_port={{ port_dict['erp5-cloudooo'] }}
--volatile_memcached_server_hostname=erp5-memcached-volatile
--volatile_memcached_server_port={{ port_dict['erp5-memcached-volatile'] }}
--persistent_memcached_server_hostname=erp5-memcached-persistent
......
......@@ -26,8 +26,21 @@ extra-context =
section parameter_dict dynamic-template-postfix-parameters
import urllib urllib
[default-cloudooo-url-list]
recipe = slapos.recipe.build
default-url-list =
{{ default_cloudooo_url_list | indent }}
init =
# Expose the default cloudooo URLs, sorted using a pseudo-random seed,
# so that the same instance keep the same order
import six, random
seed = repr(sorted(six.iteritems(self.buildout['slap-connection'])))
default_cloudooo_url_list = [url.strip() for url in self.options['default-url-list'].splitlines()]
random.Random(seed).shuffle(default_cloudooo_url_list)
self.options['url-list'] = default_cloudooo_url_list
[dynamic-template-erp5-parameters]
default-cloudooo-url = {{ dumps(default_cloudooo_url) }}
default-cloudooo-url-list = ${default-cloudooo-url-list:url-list}
jupyter-enable-default = {{ jupyter_enable_default }}
wcfs-enable-default = {{ wcfs_enable_default }}
local-bt5-repository = {{ ' '.join(local_bt5_repository.split()) }}
......@@ -42,7 +55,7 @@ template-zodb-base = {{ template_zodb_base }}
url = {{ template_erp5 }}
filename = instance-erp5.cfg
extra-context =
key default_cloudooo_url dynamic-template-erp5-parameters:default-cloudooo-url
key default_cloudooo_url_list dynamic-template-erp5-parameters:default-cloudooo-url-list
key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
key wcfs_enable_default dynamic-template-erp5-parameters:wcfs-enable-default
key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment