diff --git a/component/qt/buildout.cfg b/component/qt/buildout.cfg index cbef59ba91c1f1fc4a78887b5eec62d9f18613aa..2bdafc08253b5e3d7347ac8a772da89c18cdfb62 100644 --- a/component/qt/buildout.cfg +++ b/component/qt/buildout.cfg @@ -43,7 +43,7 @@ post-install = # qmake binary can be reached directly from ${qt:location}/bin/qmake if [qt] is fully built recipe = slapos.recipe.cmmi location = ${buildout:parts-directory}/${:_buildout_section_name_} -url = http://download.qt.io/official_releases/qt/4.8/4.8.7/qt-everywhere-opensource-src-4.8.7.tar.gz +url = http://download.qt.io/archive/qt/4.8/4.8.7/qt-everywhere-opensource-src-4.8.7.tar.gz md5sum = d990ee66bf7ab0c785589776f35ba6ad # see https://github.com/NixOS/nixpkgs/blob/3e387c3e005c87566b5403d24c86f71f4945a79b/pkgs/development/libraries/qt-4.x/4.8/default.nix#L101 pre-configure = diff --git a/component/tomcat/buildout.cfg b/component/tomcat/buildout.cfg index 547bf4954d55c482f7efd37da04c8b8837c55d5a..63750260d4712de6b95d8a715e434b20bfdfacbb 100644 --- a/component/tomcat/buildout.cfg +++ b/component/tomcat/buildout.cfg @@ -20,8 +20,8 @@ md5sum = 3dde098fd0b3a08d3f2867e4a95591ba recipe = hexagonit.recipe.download ignore-existing = true strip-top-level-dir = true -url = http://www-us.apache.org/dist/tomcat/tomcat-7/v7.0.99/bin/apache-tomcat-7.0.99.tar.gz -md5sum = ab39c15461f2a99493528b4a5819bc56 +url = http://www-us.apache.org/dist/tomcat/tomcat-7/v7.0.100/bin/apache-tomcat-7.0.100.tar.gz +md5sum = 79be4ba5a6e770730a4be3d5cb3c7862 [tomcat9] recipe = hexagonit.recipe.download diff --git a/slapos/recipe/simplehttpserver/__init__.py b/slapos/recipe/simplehttpserver/__init__.py index 7f8bcaf2a4946d3359278d2b03f0431e7e8bf74e..bbb6f06b39aab5d6b155260fdc2c60e8bc167670 100644 --- a/slapos/recipe/simplehttpserver/__init__.py +++ b/slapos/recipe/simplehttpserver/__init__.py @@ -27,6 +27,7 @@ from slapos.recipe.librecipe import GenericBaseRecipe import string, random import os +from six.moves import range class Recipe(GenericBaseRecipe): @@ -35,7 +36,7 @@ class Recipe(GenericBaseRecipe): base_path = options['base-path'] if options.get('use-hash-url', 'True') in ['true', 'True']: pool = string.letters + string.digits - hash_string = ''.join(random.choice(pool) for i in xrange(64)) + hash_string = ''.join(random.choice(pool) for i in range(64)) path = os.path.join(base_path, hash_string) if os.path.exists(base_path): diff --git a/slapos/recipe/simplehttpserver/simplehttpserver.py b/slapos/recipe/simplehttpserver/simplehttpserver.py index 4503f0deb5fd06b4236bc6bfa7cf930e588b2667..ba978d6be3053ffc4752429647f6ac6d37262771 100644 --- a/slapos/recipe/simplehttpserver/simplehttpserver.py +++ b/slapos/recipe/simplehttpserver/simplehttpserver.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from SimpleHTTPServer import SimpleHTTPRequestHandler -from BaseHTTPServer import HTTPServer +from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler +from six.moves.BaseHTTPServer import HTTPServer import ssl import os import logging @@ -8,6 +8,9 @@ from netaddr import valid_ipv4, valid_ipv6 import socket import cgi, errno +from slapos.util import str2bytes + + class ServerHandler(SimpleHTTPRequestHandler): document_path = '' @@ -22,7 +25,7 @@ class ServerHandler(SimpleHTTPRequestHandler): if self.restrict_root_folder and self.path and self.path == '/': # no access to root path self.respond(403) - self.wfile.write("Forbidden") + self.wfile.write(b"Forbidden") return True return False @@ -46,11 +49,11 @@ class ServerHandler(SimpleHTTPRequestHandler): name = form['path'].value content = form['content'].value method = 'a' - if form.has_key('clear') and form['clear'].value == '1': + if 'clear' in form and form['clear'].value == '1': method = 'w' self.writeFile(name, content, method) self.respond(200, type=self.headers['Content-Type']) - self.wfile.write("Content written to %s" % name) + self.wfile.write(b"Content written to %s" % str2bytes(name)) def writeFile(self, filename, content, method='a'): file_path = os.path.join(self.document_path, filename) @@ -97,7 +100,7 @@ def run(args): httpd = server((host, port), Handler) scheme = 'http' - if args.has_key('cert-file') and args.has_key('key-file') and \ + if 'cert-file' in args and 'key-file' in args and \ os.path.exists(args['cert-file']) and os.path.exists(args['key-file']): scheme = 'https' httpd.socket = ssl.wrap_socket (httpd.socket, diff --git a/software/backupserver/buildout.hash.cfg b/software/backupserver/buildout.hash.cfg index 21d03691dd39507c76f888a428017035d876e719..583d02c8aacf17c8bf15da66caa6c47e9c58f0c3 100644 --- a/software/backupserver/buildout.hash.cfg +++ b/software/backupserver/buildout.hash.cfg @@ -18,7 +18,7 @@ md5sum = 5c94d952305552dcbeaeaeb27dd28f3b [template-nginx-configuration] filename = template-nginx.cfg.in -md5sum = c54d36f55ba71c897505ed61213e104a +md5sum = a7012aac9b47ddd4e3aa3197aee1ffac [template-dcron-service] filename = template-dcron-service.sh.in diff --git a/software/backupserver/template-nginx.cfg.in b/software/backupserver/template-nginx.cfg.in index fd62366ae664860a139810d819980ca57520fc60..84227e87ae7bf1b2bdc5957ded882b9846894872 100644 --- a/software/backupserver/template-nginx.cfg.in +++ b/software/backupserver/template-nginx.cfg.in @@ -48,7 +48,7 @@ http { gzip_comp_level 6; gzip_buffers 16 8k; gzip_http_version 1.1; - gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; + gzip_types text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript; server { listen [$${nginx-configuration:ip}]:$${nginx-configuration:port}; diff --git a/software/erp5testnode/testsuite/deploy-test/runTestSuite.py b/software/erp5testnode/testsuite/deploy-test/runTestSuite.py index 5c80373e81784260f960adb64629e21daf87438f..bce7cbf33bcb315c9d826099ba449341221efc6b 100644 --- a/software/erp5testnode/testsuite/deploy-test/runTestSuite.py +++ b/software/erp5testnode/testsuite/deploy-test/runTestSuite.py @@ -10,7 +10,7 @@ import tempfile import json -SLEEP_TIME = 10 +SLEEP_TIME = 15 TRY_AMOUNT = 3600 diff --git a/software/erp5testnode/testsuite/instance.cfg.in b/software/erp5testnode/testsuite/instance.cfg.in index 9c8db282f573931f1f60dd8f660c1c7dc465187b..d8da4c656959cb96f766ea5294dc5074ff606505 100644 --- a/software/erp5testnode/testsuite/instance.cfg.in +++ b/software/erp5testnode/testsuite/instance.cfg.in @@ -6,7 +6,7 @@ default = $${:test} RootSoftwareInstance = $${:test} # Used for the test of resiliency. The system wants a "test" software_type. -test = $${dynamic-template-resilient-test:rendered} +test = dynamic-template-resilient-test:rendered [dynamic-template-resilient-test] recipe = slapos.recipe.template:jinja2 diff --git a/software/erp5testnode/testsuite/testsuite.cfg b/software/erp5testnode/testsuite/testsuite.cfg index b9338266e92340244ec39a37982b8e64c9bc1db7..cdf50653c45250ba6f05d3941af4ba51d8696648 100644 --- a/software/erp5testnode/testsuite/testsuite.cfg +++ b/software/erp5testnode/testsuite/testsuite.cfg @@ -1,7 +1,7 @@ [template-erp5testnode] recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance.cfg.in -md5sum = e5adcd511bca060bfeccec48b57c635c +md5sum = d864a32edab3c4f7207a7d4fe6bb7e30 output = ${buildout:directory}/template.cfg mode = 0644 diff --git a/software/jstestnode/buildout.hash.cfg b/software/jstestnode/buildout.hash.cfg index c07c3784b6448580d834395f6b4a99cdd5eb0c13..2bc9cd7f02db2d125993e47a5d4a2e3dad2850a2 100644 --- a/software/jstestnode/buildout.hash.cfg +++ b/software/jstestnode/buildout.hash.cfg @@ -15,7 +15,7 @@ [instance] filename = instance.cfg.in -md5sum = 1d6bb3263f642a115982ddf245cb1cf0 +md5sum = e6203cba0084289edec2176b185d3427 [template-nginx-service] filename = template-nginx-service.sh.in @@ -23,7 +23,7 @@ md5sum = 529532e1240a66bdf39e3cbbef90ba87 [template-nginx-configuration] filename = template-nginx.cfg.in -md5sum = 9f22db89a2679534aa8fd37dbca86782 +md5sum = 98faa5ad8cfb23a11d97a459078a1d05 [template-runTestSuite] filename = runTestSuite.in diff --git a/software/jstestnode/instance.cfg.in b/software/jstestnode/instance.cfg.in index 0924015c68187e08a055154794b7cc8630bba5bb..ff11202e5f0430f49f46741fee788836181fa568 100644 --- a/software/jstestnode/instance.cfg.in +++ b/software/jstestnode/instance.cfg.in @@ -95,3 +95,4 @@ partition = $${slap_connection:partition_id} url = $${slap_connection:server_url} key = $${slap_connection:key_file} cert = $${slap_connection:cert_file} +configuration._ = {} \ No newline at end of file diff --git a/software/jstestnode/template-nginx.cfg.in b/software/jstestnode/template-nginx.cfg.in index 189fce180badb1b2e7ed141612f5cbdf521cf67d..ba66fbf14bca95b5b8cc4927dcecfd5ebbdc2462 100644 --- a/software/jstestnode/template-nginx.cfg.in +++ b/software/jstestnode/template-nginx.cfg.in @@ -48,7 +48,7 @@ http { gzip_comp_level 6; gzip_buffers 16 8k; gzip_http_version 1.1; - gzip_types text/html text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; + gzip_types text/html text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript; server { listen [$${nginx-configuration:ip}]:$${nginx-configuration:port}; diff --git a/software/jstestnode/test/README.md b/software/jstestnode/test/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6fb7f3e5ed5749ce59930f0dcf13150f127ed017 --- /dev/null +++ b/software/jstestnode/test/README.md @@ -0,0 +1 @@ +Tests for jstestnode software release diff --git a/software/jstestnode/test/setup.py b/software/jstestnode/test/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..014baacac490d653ee4a1e4e3cbcd004ba4b2a1a --- /dev/null +++ b/software/jstestnode/test/setup.py @@ -0,0 +1,50 @@ +############################################################################## +# +# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +from setuptools import setup, find_packages + +version = '0.0.1.dev0' +name = 'slapos.test.jstestnode' +long_description = open("README.md").read() + +setup(name=name, + version=version, + description="Test for SlapOS' jstestnode", + long_description=long_description, + long_description_content_type='text/markdown', + maintainer="Nexedi", + maintainer_email="info@nexedi.com", + url="https://lab.nexedi.com/nexedi/slapos", + packages=find_packages(), + install_requires=[ + 'slapos.core', + 'slapos.libnetworkcache', + 'erp5.util', + 'requests', + ], + zip_safe=True, + test_suite='test', + ) diff --git a/software/jstestnode/test/test.py b/software/jstestnode/test/test.py new file mode 100644 index 0000000000000000000000000000000000000000..956b10eb963f68bd9adf56fe6c9cd98e709a233e --- /dev/null +++ b/software/jstestnode/test/test.py @@ -0,0 +1,88 @@ +############################################################################## +# +# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## + + +import httplib +import json +import os +import requests + +from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass + +setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass( + os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))) + + +class TestJSTestNode(InstanceTestCase): + + def test(self): + parameter_dict = self.computer_partition.getConnectionParameterDict() + self.assertTrue('_' in parameter_dict) + try: + connection_dict = json.loads(parameter_dict['_']) + except Exception as e: + self.fail("Can't parse json in %s, error %s" % (parameter_dict['_'], e)) + + self.assertEqual( + { + 'nginx': 'http://[%s]:9443/' % (self._ipv6_address, ) + }, + connection_dict + ) + + # jio tests + result = requests.get( + '%sjio/test/tests.html' % (connection_dict['nginx'], ), allow_redirects=False) + self.assertEqual( + [httplib.OK, False], + [result.status_code, result.is_redirect] + ) + + # rjs tests + result = requests.get( + '%srenderjs/test/' % (connection_dict['nginx'], ), allow_redirects=False) + self.assertEqual( + [httplib.OK, False], + [result.status_code, result.is_redirect] + ) + + # rsvp tests + result = requests.get( + '%srsvp/test/index.html' % (connection_dict['nginx'], ), allow_redirects=False) + self.assertEqual( + [httplib.OK, False], + [result.status_code, result.is_redirect] + ) + + # Default access + result = requests.get( + 'http://[%s]:9443' % (self._ipv6_address, ), allow_redirects=False) + self.assertEqual( + [httplib.FORBIDDEN, False], + [result.status_code, result.is_redirect] + ) diff --git a/software/kvm/buildout.hash.cfg b/software/kvm/buildout.hash.cfg index 0b48b06fe2e7b20ef338696f12a89d94e4b1b9c6..f8d76077accb951a0b32fc192d2209cc405c999a 100644 --- a/software/kvm/buildout.hash.cfg +++ b/software/kvm/buildout.hash.cfg @@ -15,15 +15,15 @@ [template] filename = instance.cfg.in -md5sum = a236b719aaac61ac342ada0ce569151a +md5sum = c9d4356b5148ed8ff8c4f3da63c137ce [template-kvm] filename = instance-kvm.cfg.jinja2 -md5sum = fd0b26e5ae200ce2f7ee2a169731998c +md5sum = 0db4ed796808b892a2e8a0aea704b13e [template-kvm-cluster] filename = instance-kvm-cluster.cfg.jinja2.in -md5sum = 2e743132ba4e001f784791311df9ba6a +md5sum = 6f4c60f4366728021a6e438ad3dc6956 [template-kvm-resilient] filename = instance-kvm-resilient.cfg.jinja2 @@ -31,7 +31,7 @@ md5sum = 7de5756f59ef7d823cd8ed33e6d15230 [template-kvm-import] filename = instance-kvm-import.cfg.jinja2.in -md5sum = 97a8ff8a5891678274b14481dfc5214c +md5sum = bd7e5db872b0dbe7716ec49c3907c401 [template-kvm-import-script] filename = template/kvm-import.sh.jinja2 @@ -39,7 +39,7 @@ md5sum = cd0008f1689dfca9b77370bc4d275b70 [template-kvm-export] filename = instance-kvm-export.cfg.jinja2 -md5sum = 00ce5e6da3c833d9d9d1825311f11a81 +md5sum = f12df4256eb5bd31a01c0ddc4b3897bb [template-kvm-export-script] filename = template/kvm-export.sh.jinja2 @@ -47,7 +47,7 @@ md5sum = b617d64de73de1eed518185f310bbc82 [template-nbd] filename = instance-nbd.cfg.jinja2 -md5sum = 003112c44ef8d536861a46e3dfc67a37 +md5sum = 6ea26f88252bf899c966d0f5675e7176 [template-ansible-promise] filename = template/ansible-promise.in @@ -55,11 +55,11 @@ md5sum = 2036bf145f472f62ef8dee5e729328fd [template-kvm-run] filename = template/template-kvm-run.in -md5sum = 08af4ed0e2a53e76c844e3d7325aac09 +md5sum = c319ca536b6bac5425245fae1684ca49 [template-kvm-controller] filename = template/kvm-controller-run.in -md5sum = b96cba47c97f277c857176d69e086a12 +md5sum = 3827b4f7624de190cf5f5d37e3b72e86 [template-apache-conf] filename = template/apache.conf.in diff --git a/software/kvm/instance-kvm-cluster.cfg.jinja2.in b/software/kvm/instance-kvm-cluster.cfg.jinja2.in index 72e5d6da48e420a1969ec7aa5508f3426e09461d..820b93cf63149f336a012464dccc5f2c63c1bdde 100644 --- a/software/kvm/instance-kvm-cluster.cfg.jinja2.in +++ b/software/kvm/instance-kvm-cluster.cfg.jinja2.in @@ -19,6 +19,14 @@ config-{{ name }} = {{ dumps(value) }} {% endif -%} {% endmacro -%} +[slap-network-information] +global-ipv6 = {{ ipv6 }} + +[slap-parameter] +{% for k, v in slapparameter_dict.items() -%} +{{ k }} = {{ v }} +{% endfor -%} + [request-common] recipe = slapos.cookbook:request software-url = ${slap-connection:software-release-url} diff --git a/software/kvm/instance-kvm-export.cfg.jinja2 b/software/kvm/instance-kvm-export.cfg.jinja2 index dd8a099e58d0a73bfb70d32c68eb905de23ce1b3..d881e1e441cdeb12adfd818f8f1e5b5c82d072ce 100644 --- a/software/kvm/instance-kvm-export.cfg.jinja2 +++ b/software/kvm/instance-kvm-export.cfg.jinja2 @@ -20,6 +20,11 @@ parts += # monitor parts monitor-base +[slap-parameter] +{% for k, v in slapparameter_dict.items() -%} +{{ k }} = {{ v }} +{% endfor -%} + # Create the exporter executable, which is a simple shell script [exporter] recipe = slapos.recipe.template:jinja2 diff --git a/software/kvm/instance-kvm-import.cfg.jinja2.in b/software/kvm/instance-kvm-import.cfg.jinja2.in index cab0a651da9d3faaa62a33f682f765b42221883a..189d46c5fc3d40b4a9751186020bdb8c5266ef31 100644 --- a/software/kvm/instance-kvm-import.cfg.jinja2.in +++ b/software/kvm/instance-kvm-import.cfg.jinja2.in @@ -12,6 +12,11 @@ parts += extends = {{ pbsready_import_template }} +[slap-parameter] +{% for k, v in slapparameter_dict.items() -%} +{{ k }} = {{ v }} +{% endfor -%} + [resilient-publish-connection-parameter] monitor-base-url = ${monitor-publish-parameters:monitor-base-url} monitor-url = ${monitor-publish-parameters:monitor-url} diff --git a/software/kvm/instance-kvm.cfg.jinja2 b/software/kvm/instance-kvm.cfg.jinja2 index 51f079bec65d6fdb2cea71104dbae95e7214f7b0..8dd39d59c17ac57b7eac53185c6adf078c09fb89 100644 --- a/software/kvm/instance-kvm.cfg.jinja2 +++ b/software/kvm/instance-kvm.cfg.jinja2 @@ -31,6 +31,10 @@ {% do extends_list.append(template_monitor) -%} {% do extends_list.append(logrotate_cfg) -%} +[slap-network-information] +local-ipv4 = {{ slap_configuration['ipv4-random'] }} +global-ipv6 = {{ slap_configuration['ipv6-random'] }} + [directory] recipe = slapos.cookbook:mkdirectory etc = ${buildout:directory}/etc @@ -454,38 +458,6 @@ interface-url = {{ slapparameter_dict.get('monitor-interface-url', 'https://moni [helper] blank-line = -[frontend-port-execute-base] -recipe = plone.recipe.command -command = - set -e - port=$(echo '${request-slave-frontend:connection-secure_access}' | cut -d ':' -f 3 | cut -d '/' -f 1) - [ -z $port ] && port=443 - echo $port > ${:output} -update-command = ${:command} -stop-on-error = True - -[frontend-port-execute] -<= frontend-port-execute-base -secure_access = ${request-slave-frontend:connection-secure_access} -output = ${directory:var}/frontend_port.txt - -[frontend-port] -recipe = collective.recipe.shelloutput -filename = ${frontend-port-execute:output} -commands = - port = [ -f '${:filename}' ] && cat '${:filename}' || echo "NotReady" - -{% if additional_frontend %} -[frontend-additional-port-execute] -<= frontend-port-execute-base -secure_access = ${request-slave-frontend-additional:connection-secure_access} -output = ${directory:var}/frontend_additional_port.txt - -[frontend-additional-port] -<= frontend-port -filename = ${frontend-additional-port-execute:output} -{% endif %} - [publish-connection-information] <= monitor-publish recipe = slapos.cookbook:publish @@ -493,7 +465,7 @@ ipv6 = ${slap-network-information:global-ipv6} backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc.html?auto=1&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd} url = ${request-slave-frontend:connection-secure_access}/vnc.html?auto=1&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd} {% if additional_frontend %} -url-additional = ${request-slave-frontend-additional:connection-secure_access}/vnc.html?host=${request-slave-frontend-additional:connection-domain}&port=${frontend-additional-port:port}&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd} +url-additional = ${request-slave-frontend-additional:connection-secure_access}/vnc.html?auto=1&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd} {% endif %} {% set disk_number = len(storage_dict) -%} maximum-extra-disk-amount = {{ disk_number }} @@ -781,6 +753,10 @@ data-to-vm = # Change keyboard layout language (Change to en-us if you face some bad bihaviors) keyboard-layout-language = fr +{% for k, v in slapparameter_dict.items() -%} +{{ k }} = {{ v }} +{% endfor -%} + ############################# # # Instanciate kvm (Buildout Section) diff --git a/software/kvm/instance-nbd.cfg.jinja2 b/software/kvm/instance-nbd.cfg.jinja2 index 96a0187c1dbf543ed3d5390afc67cde48c4252ea..e1c3dde0731b3529970258e38f824f901f411002 100644 --- a/software/kvm/instance-nbd.cfg.jinja2 +++ b/software/kvm/instance-nbd.cfg.jinja2 @@ -9,9 +9,8 @@ parts = onetimeupload-promise publish-connection-information extends = {{ template_monitor }} -eggs-directory = {{ eggs_directory }} -develop-eggs-directory = {{ develop_eggs_directory }} -offline = true + +{% set ipv6 = slap_configuration['ipv6-random'] -%} [rootdirectory] recipe = slapos.cookbook:mkdirectory @@ -27,7 +26,7 @@ watched-services = ${rootdirectory:etc}/service [nbd-instance] recipe = slapos.cookbook:nbdserver -ip = ${slap-network-information:global-ipv6} +ip = {{ ipv6 }} port = 1024 image-path = ${onetimeupload-instance:image-path} qemu-path = {{ qemu_nbd_executable_location }} @@ -55,7 +54,7 @@ bytes = 24 [onetimeupload-instance] recipe = slapos.cookbook:onetimeupload -ip = ${slap-network-information:global-ipv6} +ip = {{ ipv6 }} port = {{ slapparameter_dict.get('otu-port', 8080) }} image-path = ${rootdirectory:srv}/cdrom.iso log-path = ${rootdirectory:log}/onetimeupload.log @@ -80,6 +79,10 @@ upload_key = ${onetimeupload-instance:key} status_message = ${detect-if-cdrom-present:status} [detect-if-cdrom-present] -recipe = collective.recipe.shelloutput -commands = - status = [ -f ${onetimeupload-instance:image-path} ] && echo "image already uploaded, you can't upload it again" || echo "WARNING: no image yet, the NBD server doesn't work" +recipe = slapos.recipe.build +init = + import os + options['status'] = ( + "image already uploaded, you can't upload it again" + if os.path.isfile("${onetimeupload-instance:image-path}") + else "WARNING: no image yet, the NBD server doesn't work") diff --git a/software/kvm/instance.cfg.in b/software/kvm/instance.cfg.in index 64abad8cc134389f85660023a10121c19f6dde89..907747c539a6e77cce86952285213acf8f471e38 100644 --- a/software/kvm/instance.cfg.in +++ b/software/kvm/instance.cfg.in @@ -8,18 +8,21 @@ develop-eggs-directory = ${buildout:develop-eggs-directory} extends = ${template-resilient-templates:output} [switch_softwaretype] -recipe = slapos.cookbook:softwaretype +recipe = slapos.cookbook:switch-softwaretype default = $${:kvm} -kvm-cluster = $${dynamic-template-kvm-cluster:rendered} -kvm = $${dynamic-template-kvm:rendered} -nbd = $${dynamic-template-nbd:rendered} +kvm-cluster = dynamic-template-kvm-cluster:rendered +kvm = dynamic-template-kvm:rendered +nbd = dynamic-template-nbd:rendered -kvm-resilient = $${dynamic-template-kvm-resilient:rendered} -kvm-import = $${dynamic-template-kvm-import:rendered} -kvm-export = $${dynamic-template-kvm-export:rendered} +kvm-resilient = dynamic-template-kvm-resilient:rendered +kvm-import = dynamic-template-kvm-import:rendered +kvm-export = dynamic-template-kvm-export:rendered -frozen = ${instance-frozen:output} -pull-backup = ${template-pull-backup:output} +frozen = instance-frozen:rendered +pull-backup = template-pull-backup:rendered + +# BBB +RootSoftwareInstance = $${:default} # XXX - If this configuration is not generated by slapgrid, use empty values [storage-configuration] diff --git a/software/kvm/software-py3.cfg b/software/kvm/software-py3.cfg new file mode 100644 index 0000000000000000000000000000000000000000..245ed3bd51bf39d421ab9e491b74111c73b48d3f --- /dev/null +++ b/software/kvm/software-py3.cfg @@ -0,0 +1,10 @@ +[buildout] +extends = + ../../component/python3/buildout.cfg + software.cfg + +python = python3 + +# Ignore these for now +common-parts -= + rdiff-backup diff --git a/software/kvm/software.cfg b/software/kvm/software.cfg index 954540116acf80c88264ef07fbe3118815500e3d..afc8ac3e0506ca31f77d2e6b0a981ad3aa13d1a4 100644 --- a/software/kvm/software.cfg +++ b/software/kvm/software.cfg @@ -192,8 +192,7 @@ context = raw template_apache_conf ${template-apache-conf:location}/${template-apache-conf:filename} [versions] -# XXX - use websockify = 0.5.1 for compatibility with kvm frontend -websockify = 0.5.1 +websockify = 0.9.0 collective.recipe.environment = 0.2.0 gitdb = 0.6.4 diff --git a/software/kvm/template/kvm-controller-run.in b/software/kvm/template/kvm-controller-run.in index daf18d854721ae6c536a1cdc0526ab3481b2e1f3..07436d673fcb7b4c54c5dd9ecad0aa8dece97e69 100644 --- a/software/kvm/template/kvm-controller-run.in +++ b/software/kvm/template/kvm-controller-run.in @@ -57,7 +57,7 @@ def update(): 'nslot': 128, 'canreboot': 1 }) - except Exception, e: + except Exception as e: write(str(e)) raise diff --git a/software/kvm/template/template-kvm-run.in b/software/kvm/template/template-kvm-run.in index 9fc377d18c90bf4ba7541ac59c3ca86b8889dff6..edda9e897e00cb3d92e1895280193eac491943da 100644 --- a/software/kvm/template/template-kvm-run.in +++ b/software/kvm/template/template-kvm-run.in @@ -6,7 +6,10 @@ import hashlib import os import socket import subprocess -import urllib +try: + from urllib.request import FancyURLopener +except ImportError: + from urllib import FancyURLopener import gzip import shutil from random import shuffle @@ -96,9 +99,9 @@ enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lowe logfile = '{{ parameter_dict.get("log-file") }}' if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false': - opener = urllib.FancyURLopener(context=ssl._create_unverified_context()) + opener = FancyURLopener(context=ssl._create_unverified_context()) else: - opener = urllib.FancyURLopener({}) + opener = FancyURLopener({}) def md5Checksum(file_path): with open(file_path, 'rb') as fh: @@ -112,22 +115,16 @@ def md5Checksum(file_path): def getSocketStatus(host, port): s = None - for res in socket.getaddrinfo(host, port, - socket.AF_UNSPEC, socket.SOCK_STREAM): - af, socktype, proto, canonname, sa = res + for af, socktype, proto, canonname, sa in socket.getaddrinfo( + host, port, socket.AF_UNSPEC, socket.SOCK_STREAM): try: s = socket.socket(af, socktype, proto) - except socket.error, msg: - s = None - continue - try: s.connect(sa) - except socket.error, msg: - s.close() - s = None - continue - break - return s + return s + except socket.error: + if s: + s.close() + s = None def getMapStorageList(disk_storage_dict, external_disk_number): map_disk_file = os.path.join(etc_directory, '.data-disk-ids') @@ -319,7 +316,7 @@ for numa in numa_list: kvm_argument_list += numa_parameter if tap_network_parameter == [] and nat_network_parameter == []: - print 'Warning : No network interface defined.' + print('Warning : No network interface defined.') else: kvm_argument_list += nat_network_parameter + tap_network_parameter if language in language_list: @@ -361,7 +358,7 @@ for nbd_ip, nbd_port in nbd_list: s = getSocketStatus(nbd_ip, nbd_port) if s is None: # NBD is not available : launch kvm without it - print 'Warning : Nbd is not available.' + print('Warning : Nbd is not available.') else: # NBD is available # We close the NBD socket else qemu won't be able to use it apparently @@ -375,5 +372,5 @@ else: '-drive', 'file=%s,media=cdrom' % default_cdrom_iso ]) -print 'Starting KVM: \n %s' % ' '.join(kvm_argument_list) +print('Starting KVM: \n %s' % ' '.join(kvm_argument_list)) os.execv(qemu_path, kvm_argument_list) diff --git a/software/kvm/test/setup.py b/software/kvm/test/setup.py index 88f69708e3153626d9d7e22904ef5fef33da1ced..ad2d335e920999eaa1f56e03589131a6be143520 100644 --- a/software/kvm/test/setup.py +++ b/software/kvm/test/setup.py @@ -47,6 +47,7 @@ setup(name=name, 'erp5.util', 'supervisor', 'psutil', + 'six', ], zip_safe=True, test_suite='test', diff --git a/software/kvm/test/test.py b/software/kvm/test/test.py index 5b3227be69823c27cc271518e85b9d3d2fae0e41..b72966d5f5c3ae73c7ca2db1104c02c1f027294a 100644 --- a/software/kvm/test/test.py +++ b/software/kvm/test/test.py @@ -25,13 +25,14 @@ # ############################################################################## -import httplib +import six.moves.http_client as httplib import json import os import requests +import six import slapos.util import sqlite3 -import urlparse +from six.moves.urllib.parse import parse_qs, urlparse import unittest from slapos.recipe.librecipe import generateHashFromFiles @@ -43,7 +44,8 @@ skipUnlessKvm = unittest.skipUnless(has_kvm, 'kvm not loaded or not allowed') if has_kvm: setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass( os.path.abspath( - os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))) + os.path.join(os.path.dirname(__file__), '..', + 'software%s.cfg' % ("-py3" if six.PY3 else "")))) else: setUpModule, InstanceTestCase = None, unittest.TestCase @@ -118,7 +120,7 @@ class MonitorAccessMixin(object): monitor_setup_url = connection_parameter_dict['monitor-setup-url'] monitor_url_with_auth = 'https' + monitor_setup_url.split('https')[2] - auth = urlparse.parse_qs(urlparse.urlparse(monitor_url_with_auth).path) + auth = parse_qs(urlparse(monitor_url_with_auth).path) # check that monitor-base-url for all partitions in the tree are accessible # with published username and password @@ -128,7 +130,7 @@ class MonitorAccessMixin(object): if not connection_xml: continue connection_dict = slapos.util.xml2dict( - partition_information['connection_xml'].encode('utf-8')) + connection_xml if six.PY3 else connection_xml.encode('utf-8')) monitor_base_url = connection_dict.get('monitor-base-url') if not monitor_base_url: continue @@ -162,8 +164,8 @@ class TestAccessDefault(MonitorAccessMixin, InstanceTestCase): httplib.OK, result.status_code ) - self.assertTrue('<title>noVNC</title>' in result.text) - self.assertFalse('url-additional' in connection_parameter_dict) + self.assertIn('<title>noVNC</title>', result.text) + self.assertNotIn('url-additional', connection_parameter_dict) @skipUnlessKvm @@ -186,7 +188,7 @@ class TestAccessDefaultAdditional(MonitorAccessMixin, InstanceTestCase): httplib.OK, result.status_code ) - self.assertTrue('<title>noVNC</title>' in result.text) + self.assertIn('<title>noVNC</title>', result.text) result = requests.get( connection_parameter_dict['url-additional'], verify=False) @@ -194,7 +196,7 @@ class TestAccessDefaultAdditional(MonitorAccessMixin, InstanceTestCase): httplib.OK, result.status_code ) - self.assertTrue('<title>noVNC</title>' in result.text) + self.assertIn('<title>noVNC</title>', result.text) @skipUnlessKvm @@ -219,13 +221,13 @@ class TestAccessKvmCluster(MonitorAccessMixin, InstanceTestCase): def test(self): connection_parameter_dict = self.computer_partition\ .getConnectionParameterDict() - result = requests.get(connection_parameter_dict['kvm0-url'], verify=False) + result = requests.get(connection_parameter_dict['KVM0-url'], verify=False) self.assertEqual( httplib.OK, result.status_code ) - self.assertTrue('<title>noVNC</title>' in result.text) - self.assertFalse('kvm0-url-additional' in connection_parameter_dict) + self.assertIn('<title>noVNC</title>', result.text) + self.assertNotIn('KVM0-url-additional', connection_parameter_dict) @skipUnlessKvm @@ -253,20 +255,20 @@ class TestAccessKvmClusterAdditional(MonitorAccessMixin, InstanceTestCase): def test(self): connection_parameter_dict = self.computer_partition\ .getConnectionParameterDict() - result = requests.get(connection_parameter_dict['kvm0-url'], verify=False) + result = requests.get(connection_parameter_dict['KVM0-url'], verify=False) self.assertEqual( httplib.OK, result.status_code ) - self.assertTrue('<title>noVNC</title>' in result.text) + self.assertIn('<title>noVNC</title>', result.text) result = requests.get( - connection_parameter_dict['kvm0-url-additional'], verify=False) + connection_parameter_dict['KVM0-url-additional'], verify=False) self.assertEqual( httplib.OK, result.status_code ) - self.assertTrue('<title>noVNC</title>' in result.text) + self.assertIn('<title>noVNC</title>', result.text) @skipUnlessKvm diff --git a/software/slapos-sr-testing/buildout.hash.cfg b/software/slapos-sr-testing/buildout.hash.cfg index ba9ad6a257cff6d278110da3c4539f375b5b48ca..15f092c09e3236acc2bb18bc9b1890b4d84b41bf 100644 --- a/software/slapos-sr-testing/buildout.hash.cfg +++ b/software/slapos-sr-testing/buildout.hash.cfg @@ -15,4 +15,4 @@ [template] filename = instance.cfg -md5sum = 3ad1b06673000d9f424a1e7187c6a1fa +md5sum = b21b2a9ac7f027a044a897c6eacbba56 diff --git a/software/slapos-sr-testing/instance.cfg b/software/slapos-sr-testing/instance.cfg index 65aa841c9b62132a3ed810195b776977d5afe6f3..17cf7b878db9b01672a7e47baa7ef74112293e9a 100644 --- a/software/slapos-sr-testing/instance.cfg +++ b/software/slapos-sr-testing/instance.cfg @@ -2,44 +2,41 @@ parts = slapos-test-runner -eggs-directory = ${buildout:eggs-directory} -develop-eggs-directory = ${buildout:develop-eggs-directory} +eggs-directory = {{ buildout['eggs-directory'] }} +develop-eggs-directory = {{ buildout['develop-eggs-directory'] }} offline = true [slap-configuration] recipe = slapos.cookbook:slapconfiguration -computer = $${slap-connection:computer-id} -partition = $${slap-connection:partition-id} -url = $${slap-connection:server-url} -key = $${slap-connection:key-file} -cert = $${slap-connection:cert-file} +computer = ${slap-connection:computer-id} +partition = ${slap-connection:partition-id} +url = ${slap-connection:server-url} +key = ${slap-connection:key-file} +cert = ${slap-connection:cert-file} [download-source] recipe = slapos.recipe.build:gitclone -git-executable = ${git:location}/bin/git +git-executable = {{ git_location }}/bin/git [slapos] <= download-source -repository = ${slapos-repository:location} +repository = {{ slapos_location }} [directory] recipe = slapos.cookbook:mkdirectory -bin = $${buildout:directory}/bin -working-dir = $${buildout:directory}/tmp - -[test-list] -path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup},${slapos.test.theia-setup:setup},${slapos.test.grafana-setup:setup},${slapos.test.gitlab-setup:setup} +bin = ${buildout:directory}/bin +working-dir = ${buildout:directory}/tmp [slapos-test-runner] recipe = slapos.cookbook:wrapper -wrapper-path = $${directory:bin}/runTestSuite +wrapper-path = ${directory:bin}/runTestSuite command-line = - ${buildout:bin-directory}/runTestSuite - --python_interpreter=${buildout:bin-directory}/${eggs:interpreter} - --source_code_path_list=$${test-list:path_list} + {{ buildout['bin-directory'] }}/runTestSuite + --python_interpreter={{ buildout['bin-directory'] }}/{{ interpreter }} + --source_code_path_list={{ ','.join(tests.splitlines()) }} environment = - PATH=${buildout:bin-directory}:${quic_client-bin:location}:${curl:location}/bin/:/usr/bin/:/bin - SLAPOS_TEST_IPV4=$${slap-configuration:ipv4-random} - SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random} - SLAPOS_TEST_WORKING_DIR=$${directory:working-dir} + PATH={{ buildout['bin-directory'] }}:{{ quic_client_location }}:{{ curl_location }}/bin/:/usr/bin/:/bin + SLAPOS_TEST_IPV4=${slap-configuration:ipv4-random} + SLAPOS_TEST_IPV6=${slap-configuration:ipv6-random} + SLAPOS_TEST_WORKING_DIR=${directory:working-dir} diff --git a/software/slapos-sr-testing/software-py3.cfg b/software/slapos-sr-testing/software-py3.cfg new file mode 100644 index 0000000000000000000000000000000000000000..5f442accb904882243bc389ce8e429ba3b9de3e2 --- /dev/null +++ b/software/slapos-sr-testing/software-py3.cfg @@ -0,0 +1,15 @@ +[buildout] + +extends = + ../../component/python3/buildout.cfg + software.cfg + +python = python3 + +[eggs] +eggs -= +# plantuml is not Py3-compatible + ${slapos.test.plantuml-setup:egg} + +[template] +extra = diff --git a/software/slapos-sr-testing/software.cfg b/software/slapos-sr-testing/software.cfg index 095f929657e66db7487e8c385b7bcacd66a18076..7545420c9c077226d5b5cc37afcdaffaa92169b7 100644 --- a/software/slapos-sr-testing/software.cfg +++ b/software/slapos-sr-testing/software.cfg @@ -27,6 +27,11 @@ recipe = zc.recipe.egg:develop egg = slapos.cookbook[test] setup = ${slapos-repository:location}/ +[slapos.test.backupserver-setup] +<= setup-develop-egg +egg = slapos.test.backupserver +setup = ${slapos-repository:location}/software/backupserver/test/ + [slapos.test.caddy-frontend-setup] <= setup-develop-egg egg = slapos.test.caddy-frontend @@ -37,11 +42,21 @@ setup = ${slapos-repository:location}/software/caddy-frontend/test/ egg = slapos.test.erp5 setup = ${slapos-repository:location}/software/erp5/test/ +[slapos.test.htmlvalidatorserver-setup] +<= setup-develop-egg +egg = slapos.test.htmlvalidatorserver +setup = ${slapos-repository:location}/software/htmlvalidatorserver/test/ + [slapos.test.slapos-master-setup] <= setup-develop-egg egg = slapos.test.slapos-master setup = ${slapos-repository:location}/software/slapos-master/test/ +[slapos.test.jstestnode-setup] +<= setup-develop-egg +egg = slapos.test.jstestnode +setup = ${slapos-repository:location}/software/jstestnode/test/ + [slapos.test.kvm-setup] <= setup-develop-egg egg = slapos.test.kvm @@ -140,9 +155,12 @@ eggs = slapos.libnetworkcache supervisor ${slapos.cookbook-setup:egg} + ${slapos.test.backupserver-setup:egg} ${slapos.test.caddy-frontend-setup:egg} ${slapos.test.erp5-setup:egg} + ${slapos.test.htmlvalidatorserver-setup:egg} ${slapos.test.slapos-master-setup:egg} + ${slapos.test.jstestnode-setup:egg} ${slapos.test.kvm-setup:egg} ${slapos.test.monitor-setup:egg} ${slapos.test.plantuml-setup:egg} @@ -176,10 +194,43 @@ repository = https://lab.nexedi.com/nexedi/slapos.git branch = master [template] -recipe = slapos.recipe.template -url = ${:_profile_base_location_}/${:filename} -output = ${buildout:directory}/template.cfg +recipe = slapos.recipe.template:jinja2 +template = ${:_profile_base_location_}/${:filename} +rendered = ${buildout:directory}/template.cfg mode = 640 +context = + section buildout buildout + key git_location git:location + key slapos_location slapos-repository:location + key interpreter eggs:interpreter + key quic_client_location quic_client-bin:location + key curl_location curl:location + key tests :tests +tests = + ${slapos.test.kvm-setup:setup} + ${slapos.test.slaprunner-setup:setup} + ${:extra} +extra = + ${slapos.cookbook-setup:setup} + ${slapos.test.backupserver-setup:setup} + ${slapos.test.caddy-frontend-setup:setup} + ${slapos.test.erp5-setup:setup} + ${slapos.test.htmlvalidatorserver-setup:setup} + ${slapos.test.slapos-master-setup:setup} + ${slapos.test.monitor-setup:setup} + ${slapos.test.plantuml-setup:setup} + ${slapos.test.powerdns-setup:setup} + ${slapos.test.proftpd-setup:setup} + ${slapos.test.re6stnet-setup:setup} + ${slapos.test.seleniumserver-setup:setup} + ${slapos.test.helloworld-setup:setup} + ${slapos.test.jstestnode-setup:setup} + ${slapos.test.jupyter-setup:setup} + ${slapos.test.nextcloud-setup:setup} + ${slapos.test.turnserver-setup:setup} + ${slapos.test.theia-setup:setup} + ${slapos.test.grafana-setup:setup} + ${slapos.test.gitlab-setup:setup} [versions] # slapos.core is used from the clone always diff --git a/software/slaprunner/buildout.hash.cfg b/software/slaprunner/buildout.hash.cfg index fb07a29ef8b4e611f62e6947e52208749d2fc1cf..38e304b136a89c6705020529861d1039dc4f7ae0 100644 --- a/software/slaprunner/buildout.hash.cfg +++ b/software/slaprunner/buildout.hash.cfg @@ -14,11 +14,11 @@ # not need these here). [template] filename = instance.cfg -md5sum = 8b78e32b877d591400746ec7fd68ed4c +md5sum = ec70348dd71b319590a5c5837f3d2e45 [template-runner] filename = instance-runner.cfg -md5sum = 84b81c18c77af08d7e97b0b6da4ee105 +md5sum = b20894f378530e79c847ddfb61782cc5 [template-runner-import-script] filename = template/runner-import.sh.jinja2 @@ -34,7 +34,7 @@ md5sum = b992bb3391de9d6d422bfa8011d8ffc4 [template-resilient] filename = instance-resilient.cfg.jinja2 -md5sum = 2271c829b94542b7b2d9c589376ae538 +md5sum = 105ed7f54f251c64a2d34559360a5604 [template_nginx_conf] filename = nginx_conf.in @@ -62,7 +62,7 @@ md5sum = 7645048216fcf957f7773534cd0408dc [template-supervisord] filename = template/supervisord.conf.in -md5sum = d294d0dafd265048399de6da8c96345f +md5sum = 28f69b57c2835bddfcfbe312563eef51 [template-listener-slapgrid] filename = template/listener_slapgrid.py.in diff --git a/software/slaprunner/instance-resilient.cfg.jinja2 b/software/slaprunner/instance-resilient.cfg.jinja2 index d9eaae96449cfb87bdd8484137f5f9cbec1c2741..2763a14791797ee75d2d51d0cea1e3572fbb2e1d 100644 --- a/software/slaprunner/instance-resilient.cfg.jinja2 +++ b/software/slaprunner/instance-resilient.cfg.jinja2 @@ -19,7 +19,7 @@ {% set monitor_interface_url = slapparameter_dict.pop('monitor-interface-url', 'https://monitor.app.officejs.com') -%} {% import 'parts' as parts %} -{% import 'replicated' as replicated %} +{% import 'replicated' as replicated with context %} [buildout] eggs-directory = {{ eggs_directory }} diff --git a/software/slaprunner/instance-runner.cfg b/software/slaprunner/instance-runner.cfg index e6647b35f06f432d34d905bbdc03be4f111abeec..62534ab8fae5f667569dd12e842d62c9b1ad9d75 100644 --- a/software/slaprunner/instance-runner.cfg +++ b/software/slaprunner/instance-runner.cfg @@ -51,9 +51,9 @@ extends = ${monitor2-template:rendered} ${template-logrotate-base:rendered} -eggs-directory = ${buildout:eggs-directory} -develop-eggs-directory = ${buildout:develop-eggs-directory} -offline = true +[slap-network-information] +local-ipv4 = $${slap-configuration:ipv4-random} +global-ipv6 = $${slap-configuration:ipv6-random} {% if slapparameter_dict.get('custom-frontend-backend-url') -%} [request-custom-frontend] @@ -287,35 +287,29 @@ extra-args=-t rsa <=runner-sshd-ssh-keygen-base extra-args=-t ecdsa -b 521 -[runner-sshd-publickey-fingerprint-shelloutput] -recipe = collective.recipe.shelloutput -# XXX because collective.recipe.shelloutput ignore errors, we run the same -# command in a plone.recipe.command so that if fails if something goes wrong. -commands = - fingerprint = bash -o pipefail -c "${openssh-output:keygen} -lf $${runner-sshd-ssh-host-ecdsa-key:output} | cut -f 2 -d\ | sed 's/+/%2B/g' | sed 's/\//%2F/g' | sed 's/SHA256://'" - [runner-sshd-publickey-fingerprint] # fingerprint for ssh url, see # https://tools.ietf.org/id/draft-salowey-secsh-uri-00.html#connparam # https://winscp.net/eng/docs/session_url#hostkey - -_fingerprint = $${runner-sshd-publickey-fingerprint-shelloutput:fingerprint} - -# format is host-key-alg-fingerprint, but we know that -# $${runner-sshkeys-sshd:public-key} is rsa so for host-key-alg -# we just use use rsa. -fingerprint = ssh-rsa-$${:_fingerprint} - -# XXX because collective.recipe.shelloutput ignore errors and capture output -# "Error ...", we use a plone.recipe.command to check that this command did -# not fail. -# This command will always fail on first buildout run, because -# collective.recipe.shelloutput is evaluated at buildout recipes __init__ step, -# but the key file is created later at install step. -recipe = plone.recipe.command -stop-on-error = true -command = echo "$${:_fingerprint}" | ( grep ^Error || exit 0 && exit 1 ) - +recipe = slapos.recipe.build +init = + import os + import subprocess + from six.moves.urllib.parse import quote + keyfile = self.buildout['runner-sshd-ssh-host-ecdsa-key']['output'] + if os.path.isfile(keyfile): + x = subprocess.check_output(('${openssh-output:keygen}', '-lf', keyfile)) + x = x.split()[1] + assert x.startswith(b'SHA256:'), x + # format is host-key-alg-fingerprint, but we know that + # $${runner-sshd-ssh-host-ecdsa-key:output} is rsa so for host-key-alg + # we just use use rsa. + options['fingerprint'] = "ssh-rsa-" + quote(x[7:], safe='') + else: + # This command will always fail on first buildout run, because it is + # evaluated at buildout recipes __init__ step, but the key file is created + # later at install step. + options['fingerprint'] = "NotReady" #--------------------------- #-- @@ -665,6 +659,9 @@ monitor-cors-domains = monitor-interface-url = monitor-httpd-port = 8386 buildout-shared-folder = $${runnerdirectory:home}/shared +{% for k, v in slapparameter_dict.items() -%} +{{ k }} = {{ v }} +{% endfor -%} [slapos-cfg] recipe = slapos.recipe.template:jinja2 @@ -745,15 +742,6 @@ recipe = plone.recipe.command stop-on-error = true command = SR=$${slap-parameter:slapos-software} && if [ -n "$SR" ] && [ ! -f "$${directory:etc}/.project" ]; then echo workspace/slapos/$${slap-parameter:slapos-software}/ > $${directory:etc}/.project; fi -[slap-configuration] -recipe = slapos.cookbook:slapconfiguration.serialised -computer = $${slap-connection:computer-id} -partition = $${slap-connection:partition-id} -url = $${slap-connection:server-url} -key = $${slap-connection:key-file} -cert = $${slap-connection:cert-file} - - [minishell-cwd] recipe = plone.recipe.command command = if [ ! -f $${slaprunner:minishell_cwd_file} ]; then echo $${runnerdirectory:home} > $${slaprunner:minishell_cwd_file}; fi @@ -820,7 +808,6 @@ template = ${template-supervisord:location}/${template-supervisord:filename} rendered = $${directory:etc}/supervisord.conf context = import multiprocessing multiprocessing - import builtin __builtin__ section supervisord supervisord key slapparameter_dict slap-configuration:configuration key listener_slapgrid listener-slapgrid-bin:rendered diff --git a/software/slaprunner/instance.cfg b/software/slaprunner/instance.cfg index de1389256b2e7bbf2ee804d390052c12c9934e66..2547a1b47e93c93e0477f17cf33786ffa893597f 100644 --- a/software/slaprunner/instance.cfg +++ b/software/slaprunner/instance.cfg @@ -9,15 +9,18 @@ extends = ${template-resilient-templates:output} [switch_softwaretype] -recipe = slapos.cookbook:softwaretype -default = $${instance-base-runner:rendered} -resilient = $${instance-resilient:rendered} -runner = $${instance-base-runner:rendered} -runner-import = $${template-runner-import:rendered} -runner-export = $${template-runner-export:rendered} +recipe = slapos.cookbook:switch-softwaretype +default = $${:runner} +resilient = instance-resilient:rendered +runner = instance-base-runner:rendered +runner-import = template-runner-import:rendered +runner-export = template-runner-export:rendered -frozen = ${instance-frozen:output} -pull-backup = ${template-pull-backup:output} +frozen = instance-frozen:rendered +pull-backup = template-pull-backup:rendered + +# BBB +RootSoftwareInstance = $${:default} [instance-base-runner] recipe = slapos.recipe.template:jinja2 diff --git a/software/slaprunner/software-py3.cfg b/software/slaprunner/software-py3.cfg new file mode 100644 index 0000000000000000000000000000000000000000..681c06f1d3a2f0ed0b25760a8147d13137e68c29 --- /dev/null +++ b/software/slaprunner/software-py3.cfg @@ -0,0 +1,15 @@ +[buildout] +extends = + ../../component/python3/buildout.cfg + software.cfg + +python = python3 + +# Ignore these for now +common-parts -= + rdiff-backup + +[eggs] +eggs -= +# futures is a backport of Py3's concurrent.futures module + futures diff --git a/software/slaprunner/software.cfg b/software/slaprunner/software.cfg index 13332aed60987791f097ef9046e99a5ede93deae..ea19c64e536a5836540bbffa8368627ab5384691 100644 --- a/software/slaprunner/software.cfg +++ b/software/slaprunner/software.cfg @@ -150,10 +150,10 @@ eggs = erp5.util lock-file plone.recipe.command - collective.recipe.shelloutput slapos.recipe.build slapos.toolbox[flask_auth] - gunicorn==19.7.1 + gunicorn +# for gunicorn[gthread] futures ${slapos-cookbook:eggs} slapos.core # listed explicitly for scripts generation @@ -169,7 +169,7 @@ Flask-Auth = 0.85 cns.recipe.symlink = 0.2.3 futures = 3.0.5 gitdb = 0.6.4 -gunicorn = 19.7.1 +gunicorn = 19.10.0 prettytable = 0.7.2 pycurl = 7.43.0 slapos.recipe.template = 4.4 diff --git a/software/slaprunner/template/supervisord.conf.in b/software/slaprunner/template/supervisord.conf.in index d9b46f111510b55c263bd8609e4ef30756e71be2..18336617dab4333ca6e7285a9985d20634d4dab0 100644 --- a/software/slaprunner/template/supervisord.conf.in +++ b/software/slaprunner/template/supervisord.conf.in @@ -27,7 +27,7 @@ stdout_logfile = {{ supervisord['no_logfile'] }} stderr_logfile = {{ supervisord['no_logfile'] }} directory = {{ supervisord['directory'] }} {# how many parallel build jobs to spawn when compiling software -#} -{% set njobs = builtin.max(1, (multiprocessing.cpu_count() // builtin.int(slapparameter_dict.get('cpu-usage-ratio', 4)))) -%} +{% set njobs = max(1, (multiprocessing.cpu_count() // int(slapparameter_dict.get('cpu-usage-ratio', 4)))) -%} environment = PATH="{{- supervisord['path'] -}}",MAKEFLAGS="-j{{ njobs }}",NPY_NUM_BUILD_JOBS="{{ njobs }}",BUNDLE_JOBS="{{ njobs }}" [program:{{- supervisord['slapgrid-cp'] -}}] diff --git a/software/slaprunner/test/test.py b/software/slaprunner/test/test.py index ff7a75af4e2e45bc01103ff1c9e7a834fe609986..a2f718d9c9e6e3224e73de5a3810edf707cfca03 100644 --- a/software/slaprunner/test/test.py +++ b/software/slaprunner/test/test.py @@ -38,13 +38,16 @@ from six.moves.urllib.parse import quote from six.moves.urllib.parse import urljoin from six.moves.configparser import ConfigParser import requests +import six from slapos.recipe.librecipe import generateHashFromFiles from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass +from slapos.util import bytes2str setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass( os.path.abspath( - os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))) + os.path.join(os.path.dirname(__file__), '..', + 'software%s.cfg' % ("-py3" if six.PY3 else "")))) class SlaprunnerTestCase(SlapOSInstanceTestCase): @@ -185,7 +188,7 @@ class TestSSH(SlaprunnerTestCase): channel.settimeout(30) received = '' while True: - r = channel.recv(1024) + r = bytes2str(channel.recv(1024)) self.logger.debug("received >%s<", r) if not r: break @@ -197,7 +200,7 @@ class TestSSH(SlaprunnerTestCase): # simple commands can also be executed ( this would be like `ssh bash -c 'pwd'` ) self.assertEqual( self.computer_partition_root_path, - client.exec_command("pwd")[1].read(1000).strip()) + bytes2str(client.exec_command("pwd")[1].read(1000)).strip()) class TestSlapOS(SlaprunnerTestCase): @@ -211,7 +214,7 @@ class TestSlapOS(SlaprunnerTestCase): 'show', ), env={}) - self.assertIn('slaprunner', proxy_show_output) + self.assertIn(b'slaprunner', proxy_show_output) def test_shared_part_list(self): # this slapos used shared_part_list diff --git a/software/theia/buildout.hash.cfg b/software/theia/buildout.hash.cfg index 6bb72b0710dfab8f11e461a565d17bdeb35921d0..1357a4ee6d3edcc9a413d5767faf887afd73c3dc 100644 --- a/software/theia/buildout.hash.cfg +++ b/software/theia/buildout.hash.cfg @@ -23,4 +23,4 @@ md5sum = b63a993a13e7c3b16b66c25fac5ac8b9 [python-language-server-requirements.txt] filename = python-language-server-requirements.txt -md5sum = ce87c9818d64a16fce49c84ed5dba8fc +md5sum = 6c940b7015f45de6e679fc44807e5220 diff --git a/software/theia/python-language-server-requirements.txt b/software/theia/python-language-server-requirements.txt index 403862b25182b4d4e24bd970d6d87cbe370252ac..afd320b8b1fded6dd7fb86502586803800509fbb 100644 --- a/software/theia/python-language-server-requirements.txt +++ b/software/theia/python-language-server-requirements.txt @@ -19,5 +19,4 @@ six==1.12.0 snowballstemmer==1.2.1 toml==0.10.0 typed-ast==1.2.0 -typeshed==0.0.1 yapf==0.28.0 diff --git a/stack/resilient/buildout.hash.cfg b/stack/resilient/buildout.hash.cfg index ed1310bb9176c04ea6e5e3baa3f261c04ad4ee30..42a381889b1e827c5e766d8cec2da6404303ee3a 100644 --- a/stack/resilient/buildout.hash.cfg +++ b/stack/resilient/buildout.hash.cfg @@ -14,7 +14,7 @@ # not need these here). [pbsready] filename = pbsready.cfg.in -md5sum = 5e0dcd4c290f0b46cb2d316dc1c9c011 +md5sum = 66331047b7dbf2513c5726d5d1647320 [pbsready-import] filename = pbsready-import.cfg.in @@ -22,15 +22,15 @@ md5sum = d813c43ed00eff868fb13bc75b045336 [pbsready-export] filename = pbsready-export.cfg.in -md5sum = 2e804e06b5203c3f127c31a1704c48bd +md5sum = 2b0c71b085cfe8017f28098c160b1f49 [template-pull-backup] filename = instance-pull-backup.cfg.in -md5sum = 0bbe16f3d805afd880a251a4f40ecaf1 +md5sum = 555d528b198564f0ce1e94db1160ebf3 [template-replicated] filename = template-replicated.cfg.in -md5sum = 290b380fe3da8736642bc10a8b1163d1 +md5sum = 815fd8f7c42b9cf59b286b0fe77fa76d [template-parts] filename = template-parts.cfg.in @@ -38,7 +38,7 @@ md5sum = 071b1034ee8f5cc14f79b16fdeba2813 [template-resilient-templates] filename = template-resilient-templates.cfg.in -md5sum = 41e82859dc6b65e94a300a006d51536e +md5sum = 097a14371efde11465ab4bd08ef3131b [instance-frozen] filename = instance-frozen.cfg.in @@ -46,7 +46,7 @@ md5sum = d21472f0e58f928fb827f2cbf22c4d4a [resilient-web-takeover-cgi-script-download] filename = resilient-web-takeover-cgi-script.py.in -md5sum = 60d4912fdf5e8dafaba9d9f333aa9e36 +md5sum = 675ac9e1cf49ccc8f8eddb541a62d899 [template-wrapper] filename = templates/wrapper.in @@ -54,9 +54,9 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8 [notifier-feed-promise-template] filename = templates/notifier-feed-promise.py.in -md5sum = d75346911dbc4cfcdb39a21e56cd5016 +md5sum = fa6521daaa02fef4dd2ce06d29ef90be [template-monitor-check-resilient-feed] filename = templates/monitor-check-resilient-feed.in -md5sum = 19ee9055de961acf402e2dfe5b9581d2 +md5sum = af9787f8440fef19924b2e765372b20f diff --git a/stack/resilient/instance-pull-backup.cfg.in b/stack/resilient/instance-pull-backup.cfg.in index e2c522ebfc011667df6bfde50e29df76ccdf07c6..ec272966cba51b1cb55f11662c71e22bfb4a57f0 100644 --- a/stack/resilient/instance-pull-backup.cfg.in +++ b/stack/resilient/instance-pull-backup.cfg.in @@ -19,10 +19,6 @@ parts = extends = ${monitor2-template:rendered} ${template-logrotate-base:rendered} -eggs-directory = ${buildout:eggs-directory} -develop-eggs-directory = ${buildout:develop-eggs-directory} -offline = true - #---------------- #-- @@ -90,7 +86,7 @@ rendered = $${:wrapper} wrapper = $${basedirectory:services}/notifier mode = 0700 command = ${buildout:bin-directory}/pubsubserver --callbacks $${directory:notifier-callbacks} --feeds $${directory:notifier-feeds} --equeue-socket $${equeue:socket} --logfile $${basedirectory:log}/notifier.log $${:host} $${:port} -host = $${slap-network-information:global-ipv6} +host = {{ ipv6 }} port = $${notifier-port:port} context = key content notifier:command @@ -128,8 +124,8 @@ run-directory = $${basedirectory:run} pull-push-maximum-run = 5 # XXX: this should be named "notifier-host" notifier-url = http://[$${notifier:host}]:$${notifier:port} -slave-instance-list = $${slap-parameter:slave_instance_list} -ignore-known-hosts-file = $${slap-parameter:ignore-known-hosts-file} +slave-instance-list = {{ slapparameter_dict.get('slave_instance_list', '[]') }} +ignore-known-hosts-file = {{ slapparameter_dict.get('ignore-known-hosts-file', 'false') }} # To get a verbose feed about PBS state instance-root-name = $${monitor-instance-parameter:root-instance-title} log-url = $${monitor-publish-parameters:monitor-base-url}/private/notifier/ @@ -194,20 +190,6 @@ identity-file = $${:home}/id_rsa command-line = ${openssh:location}/bin/ssh -T -o "UserKnownHostsFile $${pbs:known-hosts}" -i $${:identity-file} wrapper-path = $${rootdirectory:bin}/ssh - -#---------------- -#-- -#-- Slave instance list (empty default). - -[slap-parameter] -slave_instance_list = [] -ignore-known-hosts-file = false -monitor-cors-domains = -monitor-httpd-port = 8070 -monitor-title = PBS Instance -monitor-password = $${monitor-htpasswd:passwd} -monitor-username = admin - #---------------- #-- #-- Resiliency promises. @@ -269,11 +251,11 @@ monitor-password = $${monitor-publish-parameters:monitor-password} #-- Monitor [monitor-instance-parameter] -monitor-httpd-port = $${slap-parameter:monitor-httpd-port} -monitor-title = $${slap-parameter:monitor-title} -cors-domains = $${slap-parameter:monitor-cors-domains} -username = $${slap-parameter:monitor-username} -password = $${slap-parameter:monitor-password} +monitor-httpd-port = {{ slapparameter_dict.get('monitor-httpd-port', 8070) }} +monitor-title = {{ slapparameter_dict.get('monitor-title', 'PBS Instance') }} +cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', '') }} +username = {{ slapparameter_dict.get('monitor-username', 'admin') }} +password = {{ slapparameter_dict.get('monitor-password', '$${monitor-htpasswd:passwd}') }} [monitor-conf-parameters] private-path-list += diff --git a/stack/resilient/pbsready-export.cfg.in b/stack/resilient/pbsready-export.cfg.in index 9f7a9d60b07277965dd0a51d4bb2c8df63131102..f906eaa6c64b3d4bcacb01b57be4bd7db0c3317c 100644 --- a/stack/resilient/pbsready-export.cfg.in +++ b/stack/resilient/pbsready-export.cfg.in @@ -39,7 +39,7 @@ name = exporter title = Dumping ${slap-parameter:namebase} executable = ${exporter:wrapper} wrapper = ${rootdirectory:bin}/exporter -notify = ${slap-parameter:notify} +notify = {{ slapparameter_dict.get('notify', '') }} pidfile = ${resilient-directory:pid}/${:name}.pid max-run = 3 @@ -85,8 +85,3 @@ recipe = slapos.cookbook:random.time recipe = slapos.cookbook:publish-early -init = resiliency-backup-periodicity gen-resiliency-backup-periodicity:time - -[slap-parameter] -# In cron.d format (i.e things like */15 * * * * are accepted). -resiliency-backup-periodicity = -notify = \ No newline at end of file diff --git a/stack/resilient/pbsready.cfg.in b/stack/resilient/pbsready.cfg.in index 9b1557361da6756034b4dad481128ece0cc45ad3..f4d265a8f0833b6edda674fb5d8a7c0402b4a156 100644 --- a/stack/resilient/pbsready.cfg.in +++ b/stack/resilient/pbsready.cfg.in @@ -22,6 +22,10 @@ parts += extends = ${monitor2-template:rendered} +[slap-network-information] +local-ipv4 = $${slap-configuration:ipv4-random} +global-ipv6 = $${slap-configuration:ipv6-random} + #---------------- #-- #-- Creation of all needed directories. diff --git a/stack/resilient/resilient-web-takeover-cgi-script.py.in b/stack/resilient/resilient-web-takeover-cgi-script.py.in index c9d200596578dac416f9ca0a6dfe8b4a60023ed1..83ebaa46dabfd53d0f4a507508e47ea885af8a00 100644 --- a/stack/resilient/resilient-web-takeover-cgi-script.py.in +++ b/stack/resilient/resilient-web-takeover-cgi-script.py.in @@ -1,5 +1,6 @@ #!${buildout:executable} +from __future__ import print_function equeue_database = '${equeue:database}' equeue_lockfile = '${equeue:lockfile}' takeover_script = '${resiliency-takeover-script:wrapper-takeover}' @@ -8,7 +9,11 @@ import atexit import cgi import cgitb import datetime -import gdbm +try: + import dbm.gnu as gdbm +except ImportError: + import gdbm + import os import shutil import subprocess @@ -39,11 +44,12 @@ def getLatestBackupDate(): # Usually, there is only one callback (so only one key # in the db), but if there are several: # Take the "oldest" one (oldest value). - if not db.keys(): + db_keys = db.keys() + if not db_keys: result = False else: - last_backup = db[db.keys()[-1]] - for callback in db.keys(): + last_backup = db[db_keys[-1]] + for callback in db_keys: timestamp = float(db[callback]) if timestamp < last_backup: last_backup = timestamp @@ -79,12 +85,11 @@ if latest_backup_date == False: else: latest_backup_message = latest_backup_date.strftime('%Y-%m-%d %H:%M:%S') -print "Content-Type: text/html" -print +print("Content-Type: text/html\n") form = cgi.FieldStorage() if "password" not in form: - print """<html> + print("""<html> <body> <h1>This is takeover web interface.</h1> <p>Calling takeover will stop and freeze the current main instance, and make this clone instance the new main instance, replacing the old one.</p> @@ -100,15 +105,15 @@ if "password" not in form: <input type="submit" value="Take over" style="background: red;"> </form> </body> -</html>""" % (latest_backup_message, isBackupInProgress(), getSoftwareReleaseInformationFormatted()) +</html>""" % (latest_backup_message, isBackupInProgress(), getSoftwareReleaseInformationFormatted())) sys.exit(0) if form['password'].value != '${:password}': - print "<H1>Error</H1>" - print "Password is invalid." + print("<H1>Error</H1>") + print("Password is invalid.") sys.exit(1) # XXX hardcoded location result = subprocess.check_output([takeover_script], stderr=subprocess.STDOUT) -print 'Success.' -print '<pre>%s</pre>' % result +print('Success.') +print('<pre>%s</pre>' % result) diff --git a/stack/resilient/template-replicated.cfg.in b/stack/resilient/template-replicated.cfg.in index 6a990ae361e3adb4165b2daabb7083e187a6aa3b..f5fab9935d061c6f82edc869ff09665440b45255 100644 --- a/stack/resilient/template-replicated.cfg.in +++ b/stack/resilient/template-replicated.cfg.in @@ -41,7 +41,7 @@ config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}} config-name = {{namebase}}0 # Bubble up all the instance parameters to the requested export instance. {% if slapparameter_dict is defined %} -{% for parameter_name, parameter_value in slapparameter_dict.items() %} +{% for parameter_name, parameter_value in six.iteritems(slapparameter_dict) %} {% if parameter_value is string %} config-{{parameter_name}} = {{ parameter_value.split('\n') | join('\n ') }} {% else %} @@ -49,7 +49,7 @@ config-{{parameter_name}} = {{ parameter_value }} {% endif %} {% endfor %} {% endif %} -{% for key, value in monitor_dict.iteritems() -%} +{% for key, value in six.iteritems(monitor_dict) -%} config-{{ key }} = {{ value }} {% endfor -%} {% if sla_parameter_dict == {} -%} @@ -67,7 +67,7 @@ sla-mode = unique_by_network {% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%} {% endif -%} {% endfor -%} -{% for key, value in sla_dict.iteritems() -%} +{% for key, value in six.iteritems(sla_dict) -%} sla-{{ key }} = {{ value }} {% endfor -%} {% endif -%} @@ -101,7 +101,7 @@ config-number = {{id}} config-name = {{namebase}}{{id}} config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key} config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id} -{% for key, value in monitor_dict.iteritems() -%} +{% for key, value in six.iteritems(monitor_dict) -%} config-{{ key }} = {{ value }} {% endfor -%} {% if sla_parameter_dict == {} -%} @@ -119,7 +119,7 @@ sla-mode = unique_by_network {% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%} {% endif -%} {% endfor -%} -{% for key, value in sla_dict.iteritems() -%} +{% for key, value in six.iteritems(sla_dict) -%} sla-{{ key }} = {{ value }} {% endfor -%} {% endif %} @@ -202,7 +202,7 @@ software-type = pull-backup name = PBS ({{namebase}} / {{id}}) config-ignore-known-hosts-file = ${slap-parameter:ignore-known-hosts-file} config-monitor-title = PBS ${slap-connection:computer-id}-{{namebase}}-{{id}} -{% for key, value in monitor_dict.iteritems() -%} +{% for key, value in six.iteritems(monitor_dict) -%} config-{{ key }} = {{ value }} {% endfor -%} return = ssh-key notification-url feeds-url {{ monitor_return }} @@ -222,7 +222,7 @@ sla-mode = unique_by_network {% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%} {% endif -%} {% endfor -%} -{% for key, value in sla_dict.iteritems() -%} +{% for key, value in six.iteritems(sla_dict) -%} sla-{{ key }} = {{ value }} {% endfor %} {% endif %} diff --git a/stack/resilient/template-resilient-templates.cfg.in b/stack/resilient/template-resilient-templates.cfg.in index 4524e839a282d7a112f2fa53063f67d584d3ab28..ddb2a265f4ef2a606cd0670eb60436a0e5012ec6 100644 --- a/stack/resilient/template-resilient-templates.cfg.in +++ b/stack/resilient/template-resilient-templates.cfg.in @@ -8,3 +8,20 @@ context = raw pbsready_template_path ${pbsready:output} raw bash_executable_location ${bash:location}/bin/bash raw logrotate_executable_location ${logrotate:location}/usr/sbin/logrotate + +[instance-frozen] +recipe = slapos.recipe.template:jinja2 +template = ${instance-frozen:output} +rendered = $${buildout:directory}/template-frozen.cfg +extensions = jinja2.ext.do +context = + key slapparameter_dict slap-configuration:configuration + +[template-pull-backup] +recipe = slapos.recipe.template:jinja2 +template = ${template-pull-backup:output} +rendered = $${buildout:directory}/template-pull-backup.cfg +extensions = jinja2.ext.do +context = + key slapparameter_dict slap-configuration:configuration + key ipv6 slap-configuration:ipv6-random diff --git a/stack/resilient/templates/monitor-check-resilient-feed.in b/stack/resilient/templates/monitor-check-resilient-feed.in index de3f6bffaa5a7e895f2c9a35750bd1e6fa498447..e8a1e0db00120d0cd51b39789e2091680f83886c 100644 --- a/stack/resilient/templates/monitor-check-resilient-feed.in +++ b/stack/resilient/templates/monitor-check-resilient-feed.in @@ -1,7 +1,12 @@ #!{{ python_executable }} +from __future__ import print_function import os -import urllib2 +try: + from urllib2 import HTTPError, urlopen +except ImportError: + from urllib.error import HTTPError + from urllib.request import urlopen import sys input_feed_directory = '{{ input_feed_directory }}' @@ -12,12 +17,14 @@ feed_file_list = os.listdir(input_feed_directory) rss_ok = True for feed_file_name in feed_file_list: - print "Getting %s" % feed_file_name + print("Getting", feed_file_name) url = base_url + feed_file_name try: - feed = urllib2.urlopen(url) + feed = urlopen(url) body = feed.read() - open(os.path.join(monitor_feed_directory, feed_file_name + '.rss'), 'w').write(body) - print "FEED is ok" - except urllib2.HTTPError as e: + with open(os.path.join(monitor_feed_directory, feed_file_name + '.rss'), + 'wb') as f: + f.write(body) + print("FEED is ok") + except HTTPError as e: sys.exit("%s is unvailable: %s" % (feed_file_name, e)) diff --git a/stack/resilient/templates/notifier-feed-promise.py.in b/stack/resilient/templates/notifier-feed-promise.py.in index fb021dabe572a4e7980b369556a2ae04810d3fe2..e694760fbcabcde48f4f45ca342fb8ab1eb94cc5 100644 --- a/stack/resilient/templates/notifier-feed-promise.py.in +++ b/stack/resilient/templates/notifier-feed-promise.py.in @@ -2,7 +2,11 @@ import csv import os import sys -import urllib2 +try: + from urllib2 import HTTPError, urlopen +except ImportError: + from urllib.error import HTTPError + from urllib.request import urlopen csv.field_size_limit(sys.maxsize) @@ -15,9 +19,9 @@ for feed_file_name in feed_file_list: url = base_url + feed_file_name # Try feed consistency try: - feed = urllib2.urlopen(url) + feed = urlopen(url) body = feed.read() - except urllib2.HTTPError as e: + except HTTPError as e: sys.exit("%s is unavailable: %s" % (feed_file_name, e)) with open(os.path.join(notifier_feed_directory, feed_file_name)) as feed_file: reader = csv.reader(feed_file) diff --git a/stack/slapos.cfg b/stack/slapos.cfg index f901addf8cb028d21cda410a4fe8f4506f9da813..586c9995e748acf7a427175ee59086e5c96b3b75 100644 --- a/stack/slapos.cfg +++ b/stack/slapos.cfg @@ -145,7 +145,7 @@ slapos.libnetworkcache = 0.20 slapos.rebootstrap = 4.4 slapos.recipe.build = 0.43 slapos.recipe.cmmi = 0.12 -slapos.toolbox = 0.108 +slapos.toolbox = 0.109 stevedore = 1.21.0 subprocess32 = 3.5.3 unicodecsv = 0.14.1 diff --git a/stack/lapp/README.rst b/stack/unstable/lapp/README.rst similarity index 100% rename from stack/lapp/README.rst rename to stack/unstable/lapp/README.rst diff --git a/stack/lapp/apache/instance-apache-export.cfg.jinja2 b/stack/unstable/lapp/apache/instance-apache-export.cfg.jinja2 similarity index 100% rename from stack/lapp/apache/instance-apache-export.cfg.jinja2 rename to stack/unstable/lapp/apache/instance-apache-export.cfg.jinja2 diff --git a/stack/lapp/apache/instance-apache-import.cfg.in b/stack/unstable/lapp/apache/instance-apache-import.cfg.in similarity index 100% rename from stack/lapp/apache/instance-apache-import.cfg.in rename to stack/unstable/lapp/apache/instance-apache-import.cfg.in diff --git a/stack/lapp/apache/instance-apache-php.cfg.in b/stack/unstable/lapp/apache/instance-apache-php.cfg.in similarity index 100% rename from stack/lapp/apache/instance-apache-php.cfg.in rename to stack/unstable/lapp/apache/instance-apache-php.cfg.in diff --git a/stack/lapp/buildout.cfg b/stack/unstable/lapp/buildout.cfg similarity index 100% rename from stack/lapp/buildout.cfg rename to stack/unstable/lapp/buildout.cfg diff --git a/stack/lapp/instance-resilient.cfg.jinja2 b/stack/unstable/lapp/instance-resilient.cfg.jinja2 similarity index 100% rename from stack/lapp/instance-resilient.cfg.jinja2 rename to stack/unstable/lapp/instance-resilient.cfg.jinja2 diff --git a/stack/lapp/instance.cfg.in b/stack/unstable/lapp/instance.cfg.in similarity index 100% rename from stack/lapp/instance.cfg.in rename to stack/unstable/lapp/instance.cfg.in diff --git a/stack/lapp/postgres/instance-postgres-export.cfg.in b/stack/unstable/lapp/postgres/instance-postgres-export.cfg.in similarity index 100% rename from stack/lapp/postgres/instance-postgres-export.cfg.in rename to stack/unstable/lapp/postgres/instance-postgres-export.cfg.in diff --git a/stack/lapp/postgres/instance-postgres-import.cfg.in b/stack/unstable/lapp/postgres/instance-postgres-import.cfg.in similarity index 100% rename from stack/lapp/postgres/instance-postgres-import.cfg.in rename to stack/unstable/lapp/postgres/instance-postgres-import.cfg.in diff --git a/stack/lapp/postgres/instance-postgres.cfg.in b/stack/unstable/lapp/postgres/instance-postgres.cfg.in similarity index 100% rename from stack/lapp/postgres/instance-postgres.cfg.in rename to stack/unstable/lapp/postgres/instance-postgres.cfg.in