Commit 075b0ba2 authored by Bryton Lacquement's avatar Bryton Lacquement 🚪

kvm/slaprunner: add support for Python 3

TODO: rdiff-backup
parents ea1eda99 04c0d5f7
...@@ -27,6 +27,7 @@ ...@@ -27,6 +27,7 @@
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
import string, random import string, random
import os import os
from six.moves import range
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
...@@ -35,7 +36,7 @@ class Recipe(GenericBaseRecipe): ...@@ -35,7 +36,7 @@ class Recipe(GenericBaseRecipe):
base_path = options['base-path'] base_path = options['base-path']
if options.get('use-hash-url', 'True') in ['true', 'True']: if options.get('use-hash-url', 'True') in ['true', 'True']:
pool = string.letters + string.digits pool = string.letters + string.digits
hash_string = ''.join(random.choice(pool) for i in xrange(64)) hash_string = ''.join(random.choice(pool) for i in range(64))
path = os.path.join(base_path, hash_string) path = os.path.join(base_path, hash_string)
if os.path.exists(base_path): if os.path.exists(base_path):
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from SimpleHTTPServer import SimpleHTTPRequestHandler from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler
from BaseHTTPServer import HTTPServer from six.moves.BaseHTTPServer import HTTPServer
import ssl import ssl
import os import os
import logging import logging
...@@ -8,6 +8,9 @@ from netaddr import valid_ipv4, valid_ipv6 ...@@ -8,6 +8,9 @@ from netaddr import valid_ipv4, valid_ipv6
import socket import socket
import cgi, errno import cgi, errno
from slapos.util import str2bytes
class ServerHandler(SimpleHTTPRequestHandler): class ServerHandler(SimpleHTTPRequestHandler):
document_path = '' document_path = ''
...@@ -22,7 +25,7 @@ class ServerHandler(SimpleHTTPRequestHandler): ...@@ -22,7 +25,7 @@ class ServerHandler(SimpleHTTPRequestHandler):
if self.restrict_root_folder and self.path and self.path == '/': if self.restrict_root_folder and self.path and self.path == '/':
# no access to root path # no access to root path
self.respond(403) self.respond(403)
self.wfile.write("Forbidden") self.wfile.write(b"Forbidden")
return True return True
return False return False
...@@ -46,11 +49,11 @@ class ServerHandler(SimpleHTTPRequestHandler): ...@@ -46,11 +49,11 @@ class ServerHandler(SimpleHTTPRequestHandler):
name = form['path'].value name = form['path'].value
content = form['content'].value content = form['content'].value
method = 'a' method = 'a'
if form.has_key('clear') and form['clear'].value == '1': if 'clear' in form and form['clear'].value == '1':
method = 'w' method = 'w'
self.writeFile(name, content, method) self.writeFile(name, content, method)
self.respond(200, type=self.headers['Content-Type']) self.respond(200, type=self.headers['Content-Type'])
self.wfile.write("Content written to %s" % name) self.wfile.write(b"Content written to %s" % str2bytes(name))
def writeFile(self, filename, content, method='a'): def writeFile(self, filename, content, method='a'):
file_path = os.path.join(self.document_path, filename) file_path = os.path.join(self.document_path, filename)
...@@ -97,7 +100,7 @@ def run(args): ...@@ -97,7 +100,7 @@ def run(args):
httpd = server((host, port), Handler) httpd = server((host, port), Handler)
scheme = 'http' scheme = 'http'
if args.has_key('cert-file') and args.has_key('key-file') and \ if 'cert-file' in args and 'key-file' in args and \
os.path.exists(args['cert-file']) and os.path.exists(args['key-file']): os.path.exists(args['cert-file']) and os.path.exists(args['key-file']):
scheme = 'https' scheme = 'https'
httpd.socket = ssl.wrap_socket (httpd.socket, httpd.socket = ssl.wrap_socket (httpd.socket,
......
...@@ -15,15 +15,15 @@ ...@@ -15,15 +15,15 @@
[template] [template]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = a236b719aaac61ac342ada0ce569151a md5sum = c9d4356b5148ed8ff8c4f3da63c137ce
[template-kvm] [template-kvm]
filename = instance-kvm.cfg.jinja2 filename = instance-kvm.cfg.jinja2
md5sum = fd0b26e5ae200ce2f7ee2a169731998c md5sum = 0db4ed796808b892a2e8a0aea704b13e
[template-kvm-cluster] [template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in filename = instance-kvm-cluster.cfg.jinja2.in
md5sum = 2e743132ba4e001f784791311df9ba6a md5sum = 6f4c60f4366728021a6e438ad3dc6956
[template-kvm-resilient] [template-kvm-resilient]
filename = instance-kvm-resilient.cfg.jinja2 filename = instance-kvm-resilient.cfg.jinja2
...@@ -31,7 +31,7 @@ md5sum = 7de5756f59ef7d823cd8ed33e6d15230 ...@@ -31,7 +31,7 @@ md5sum = 7de5756f59ef7d823cd8ed33e6d15230
[template-kvm-import] [template-kvm-import]
filename = instance-kvm-import.cfg.jinja2.in filename = instance-kvm-import.cfg.jinja2.in
md5sum = 97a8ff8a5891678274b14481dfc5214c md5sum = bd7e5db872b0dbe7716ec49c3907c401
[template-kvm-import-script] [template-kvm-import-script]
filename = template/kvm-import.sh.jinja2 filename = template/kvm-import.sh.jinja2
...@@ -39,7 +39,7 @@ md5sum = cd0008f1689dfca9b77370bc4d275b70 ...@@ -39,7 +39,7 @@ md5sum = cd0008f1689dfca9b77370bc4d275b70
[template-kvm-export] [template-kvm-export]
filename = instance-kvm-export.cfg.jinja2 filename = instance-kvm-export.cfg.jinja2
md5sum = 00ce5e6da3c833d9d9d1825311f11a81 md5sum = f12df4256eb5bd31a01c0ddc4b3897bb
[template-kvm-export-script] [template-kvm-export-script]
filename = template/kvm-export.sh.jinja2 filename = template/kvm-export.sh.jinja2
...@@ -47,7 +47,7 @@ md5sum = b617d64de73de1eed518185f310bbc82 ...@@ -47,7 +47,7 @@ md5sum = b617d64de73de1eed518185f310bbc82
[template-nbd] [template-nbd]
filename = instance-nbd.cfg.jinja2 filename = instance-nbd.cfg.jinja2
md5sum = 003112c44ef8d536861a46e3dfc67a37 md5sum = 6ea26f88252bf899c966d0f5675e7176
[template-ansible-promise] [template-ansible-promise]
filename = template/ansible-promise.in filename = template/ansible-promise.in
...@@ -55,11 +55,11 @@ md5sum = 2036bf145f472f62ef8dee5e729328fd ...@@ -55,11 +55,11 @@ md5sum = 2036bf145f472f62ef8dee5e729328fd
[template-kvm-run] [template-kvm-run]
filename = template/template-kvm-run.in filename = template/template-kvm-run.in
md5sum = 08af4ed0e2a53e76c844e3d7325aac09 md5sum = c319ca536b6bac5425245fae1684ca49
[template-kvm-controller] [template-kvm-controller]
filename = template/kvm-controller-run.in filename = template/kvm-controller-run.in
md5sum = b96cba47c97f277c857176d69e086a12 md5sum = 3827b4f7624de190cf5f5d37e3b72e86
[template-apache-conf] [template-apache-conf]
filename = template/apache.conf.in filename = template/apache.conf.in
......
...@@ -19,6 +19,14 @@ config-{{ name }} = {{ dumps(value) }} ...@@ -19,6 +19,14 @@ config-{{ name }} = {{ dumps(value) }}
{% endif -%} {% endif -%}
{% endmacro -%} {% endmacro -%}
[slap-network-information]
global-ipv6 = {{ ipv6 }}
[slap-parameter]
{% for k, v in slapparameter_dict.items() -%}
{{ k }} = {{ v }}
{% endfor -%}
[request-common] [request-common]
recipe = slapos.cookbook:request recipe = slapos.cookbook:request
software-url = ${slap-connection:software-release-url} software-url = ${slap-connection:software-release-url}
......
...@@ -20,6 +20,11 @@ parts += ...@@ -20,6 +20,11 @@ parts +=
# monitor parts # monitor parts
monitor-base monitor-base
[slap-parameter]
{% for k, v in slapparameter_dict.items() -%}
{{ k }} = {{ v }}
{% endfor -%}
# Create the exporter executable, which is a simple shell script # Create the exporter executable, which is a simple shell script
[exporter] [exporter]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
......
...@@ -12,6 +12,11 @@ parts += ...@@ -12,6 +12,11 @@ parts +=
extends = extends =
{{ pbsready_import_template }} {{ pbsready_import_template }}
[slap-parameter]
{% for k, v in slapparameter_dict.items() -%}
{{ k }} = {{ v }}
{% endfor -%}
[resilient-publish-connection-parameter] [resilient-publish-connection-parameter]
monitor-base-url = ${monitor-publish-parameters:monitor-base-url} monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
monitor-url = ${monitor-publish-parameters:monitor-url} monitor-url = ${monitor-publish-parameters:monitor-url}
......
...@@ -31,6 +31,10 @@ ...@@ -31,6 +31,10 @@
{% do extends_list.append(template_monitor) -%} {% do extends_list.append(template_monitor) -%}
{% do extends_list.append(logrotate_cfg) -%} {% do extends_list.append(logrotate_cfg) -%}
[slap-network-information]
local-ipv4 = {{ slap_configuration['ipv4-random'] }}
global-ipv6 = {{ slap_configuration['ipv6-random'] }}
[directory] [directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
etc = ${buildout:directory}/etc etc = ${buildout:directory}/etc
...@@ -454,38 +458,6 @@ interface-url = {{ slapparameter_dict.get('monitor-interface-url', 'https://moni ...@@ -454,38 +458,6 @@ interface-url = {{ slapparameter_dict.get('monitor-interface-url', 'https://moni
[helper] [helper]
blank-line = blank-line =
[frontend-port-execute-base]
recipe = plone.recipe.command
command =
set -e
port=$(echo '${request-slave-frontend:connection-secure_access}' | cut -d ':' -f 3 | cut -d '/' -f 1)
[ -z $port ] && port=443
echo $port > ${:output}
update-command = ${:command}
stop-on-error = True
[frontend-port-execute]
<= frontend-port-execute-base
secure_access = ${request-slave-frontend:connection-secure_access}
output = ${directory:var}/frontend_port.txt
[frontend-port]
recipe = collective.recipe.shelloutput
filename = ${frontend-port-execute:output}
commands =
port = [ -f '${:filename}' ] && cat '${:filename}' || echo "NotReady"
{% if additional_frontend %}
[frontend-additional-port-execute]
<= frontend-port-execute-base
secure_access = ${request-slave-frontend-additional:connection-secure_access}
output = ${directory:var}/frontend_additional_port.txt
[frontend-additional-port]
<= frontend-port
filename = ${frontend-additional-port-execute:output}
{% endif %}
[publish-connection-information] [publish-connection-information]
<= monitor-publish <= monitor-publish
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
...@@ -493,7 +465,7 @@ ipv6 = ${slap-network-information:global-ipv6} ...@@ -493,7 +465,7 @@ ipv6 = ${slap-network-information:global-ipv6}
backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc.html?auto=1&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd} backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc.html?auto=1&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
url = ${request-slave-frontend:connection-secure_access}/vnc.html?auto=1&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd} url = ${request-slave-frontend:connection-secure_access}/vnc.html?auto=1&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
{% if additional_frontend %} {% if additional_frontend %}
url-additional = ${request-slave-frontend-additional:connection-secure_access}/vnc.html?host=${request-slave-frontend-additional:connection-domain}&port=${frontend-additional-port:port}&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd} url-additional = ${request-slave-frontend-additional:connection-secure_access}/vnc.html?auto=1&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
{% endif %} {% endif %}
{% set disk_number = len(storage_dict) -%} {% set disk_number = len(storage_dict) -%}
maximum-extra-disk-amount = {{ disk_number }} maximum-extra-disk-amount = {{ disk_number }}
...@@ -781,6 +753,10 @@ data-to-vm = ...@@ -781,6 +753,10 @@ data-to-vm =
# Change keyboard layout language (Change to en-us if you face some bad bihaviors) # Change keyboard layout language (Change to en-us if you face some bad bihaviors)
keyboard-layout-language = fr keyboard-layout-language = fr
{% for k, v in slapparameter_dict.items() -%}
{{ k }} = {{ v }}
{% endfor -%}
############################# #############################
# #
# Instanciate kvm (Buildout Section) # Instanciate kvm (Buildout Section)
......
...@@ -9,9 +9,8 @@ parts = ...@@ -9,9 +9,8 @@ parts =
onetimeupload-promise onetimeupload-promise
publish-connection-information publish-connection-information
extends = {{ template_monitor }} extends = {{ template_monitor }}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }} {% set ipv6 = slap_configuration['ipv6-random'] -%}
offline = true
[rootdirectory] [rootdirectory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
...@@ -27,7 +26,7 @@ watched-services = ${rootdirectory:etc}/service ...@@ -27,7 +26,7 @@ watched-services = ${rootdirectory:etc}/service
[nbd-instance] [nbd-instance]
recipe = slapos.cookbook:nbdserver recipe = slapos.cookbook:nbdserver
ip = ${slap-network-information:global-ipv6} ip = {{ ipv6 }}
port = 1024 port = 1024
image-path = ${onetimeupload-instance:image-path} image-path = ${onetimeupload-instance:image-path}
qemu-path = {{ qemu_nbd_executable_location }} qemu-path = {{ qemu_nbd_executable_location }}
...@@ -55,7 +54,7 @@ bytes = 24 ...@@ -55,7 +54,7 @@ bytes = 24
[onetimeupload-instance] [onetimeupload-instance]
recipe = slapos.cookbook:onetimeupload recipe = slapos.cookbook:onetimeupload
ip = ${slap-network-information:global-ipv6} ip = {{ ipv6 }}
port = {{ slapparameter_dict.get('otu-port', 8080) }} port = {{ slapparameter_dict.get('otu-port', 8080) }}
image-path = ${rootdirectory:srv}/cdrom.iso image-path = ${rootdirectory:srv}/cdrom.iso
log-path = ${rootdirectory:log}/onetimeupload.log log-path = ${rootdirectory:log}/onetimeupload.log
...@@ -80,6 +79,10 @@ upload_key = ${onetimeupload-instance:key} ...@@ -80,6 +79,10 @@ upload_key = ${onetimeupload-instance:key}
status_message = ${detect-if-cdrom-present:status} status_message = ${detect-if-cdrom-present:status}
[detect-if-cdrom-present] [detect-if-cdrom-present]
recipe = collective.recipe.shelloutput recipe = slapos.recipe.build
commands = init =
status = [ -f ${onetimeupload-instance:image-path} ] && echo "image already uploaded, you can't upload it again" || echo "WARNING: no image yet, the NBD server doesn't work" import os
options['status'] = (
"image already uploaded, you can't upload it again"
if os.path.isfile("${onetimeupload-instance:image-path}")
else "WARNING: no image yet, the NBD server doesn't work")
...@@ -8,18 +8,21 @@ develop-eggs-directory = ${buildout:develop-eggs-directory} ...@@ -8,18 +8,21 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
extends = ${template-resilient-templates:output} extends = ${template-resilient-templates:output}
[switch_softwaretype] [switch_softwaretype]
recipe = slapos.cookbook:softwaretype recipe = slapos.cookbook:switch-softwaretype
default = $${:kvm} default = $${:kvm}
kvm-cluster = $${dynamic-template-kvm-cluster:rendered} kvm-cluster = dynamic-template-kvm-cluster:rendered
kvm = $${dynamic-template-kvm:rendered} kvm = dynamic-template-kvm:rendered
nbd = $${dynamic-template-nbd:rendered} nbd = dynamic-template-nbd:rendered
kvm-resilient = $${dynamic-template-kvm-resilient:rendered} kvm-resilient = dynamic-template-kvm-resilient:rendered
kvm-import = $${dynamic-template-kvm-import:rendered} kvm-import = dynamic-template-kvm-import:rendered
kvm-export = $${dynamic-template-kvm-export:rendered} kvm-export = dynamic-template-kvm-export:rendered
frozen = ${instance-frozen:output} frozen = instance-frozen:rendered
pull-backup = ${template-pull-backup:output} pull-backup = template-pull-backup:rendered
# BBB
RootSoftwareInstance = $${:default}
# XXX - If this configuration is not generated by slapgrid, use empty values # XXX - If this configuration is not generated by slapgrid, use empty values
[storage-configuration] [storage-configuration]
......
[buildout]
extends =
../../component/python3/buildout.cfg
software.cfg
python = python3
# Ignore these for now
common-parts -=
rdiff-backup
...@@ -192,8 +192,7 @@ context = ...@@ -192,8 +192,7 @@ context =
raw template_apache_conf ${template-apache-conf:location}/${template-apache-conf:filename} raw template_apache_conf ${template-apache-conf:location}/${template-apache-conf:filename}
[versions] [versions]
# XXX - use websockify = 0.5.1 for compatibility with kvm frontend websockify = 0.9.0
websockify = 0.5.1
collective.recipe.environment = 0.2.0 collective.recipe.environment = 0.2.0
gitdb = 0.6.4 gitdb = 0.6.4
......
...@@ -57,7 +57,7 @@ def update(): ...@@ -57,7 +57,7 @@ def update():
'nslot': 128, 'nslot': 128,
'canreboot': 1 'canreboot': 1
}) })
except Exception, e: except Exception as e:
write(str(e)) write(str(e))
raise raise
......
...@@ -6,7 +6,10 @@ import hashlib ...@@ -6,7 +6,10 @@ import hashlib
import os import os
import socket import socket
import subprocess import subprocess
import urllib try:
from urllib.request import FancyURLopener
except ImportError:
from urllib import FancyURLopener
import gzip import gzip
import shutil import shutil
from random import shuffle from random import shuffle
...@@ -96,9 +99,9 @@ enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lowe ...@@ -96,9 +99,9 @@ enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lowe
logfile = '{{ parameter_dict.get("log-file") }}' logfile = '{{ parameter_dict.get("log-file") }}'
if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false': if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
opener = urllib.FancyURLopener(context=ssl._create_unverified_context()) opener = FancyURLopener(context=ssl._create_unverified_context())
else: else:
opener = urllib.FancyURLopener({}) opener = FancyURLopener({})
def md5Checksum(file_path): def md5Checksum(file_path):
with open(file_path, 'rb') as fh: with open(file_path, 'rb') as fh:
...@@ -112,22 +115,16 @@ def md5Checksum(file_path): ...@@ -112,22 +115,16 @@ def md5Checksum(file_path):
def getSocketStatus(host, port): def getSocketStatus(host, port):
s = None s = None
for res in socket.getaddrinfo(host, port, for af, socktype, proto, canonname, sa in socket.getaddrinfo(
socket.AF_UNSPEC, socket.SOCK_STREAM): host, port, socket.AF_UNSPEC, socket.SOCK_STREAM):
af, socktype, proto, canonname, sa = res
try: try:
s = socket.socket(af, socktype, proto) s = socket.socket(af, socktype, proto)
except socket.error, msg:
s = None
continue
try:
s.connect(sa) s.connect(sa)
except socket.error, msg: return s
s.close() except socket.error:
s = None if s:
continue s.close()
break s = None
return s
def getMapStorageList(disk_storage_dict, external_disk_number): def getMapStorageList(disk_storage_dict, external_disk_number):
map_disk_file = os.path.join(etc_directory, '.data-disk-ids') map_disk_file = os.path.join(etc_directory, '.data-disk-ids')
...@@ -319,7 +316,7 @@ for numa in numa_list: ...@@ -319,7 +316,7 @@ for numa in numa_list:
kvm_argument_list += numa_parameter kvm_argument_list += numa_parameter
if tap_network_parameter == [] and nat_network_parameter == []: if tap_network_parameter == [] and nat_network_parameter == []:
print 'Warning : No network interface defined.' print('Warning : No network interface defined.')
else: else:
kvm_argument_list += nat_network_parameter + tap_network_parameter kvm_argument_list += nat_network_parameter + tap_network_parameter
if language in language_list: if language in language_list:
...@@ -361,7 +358,7 @@ for nbd_ip, nbd_port in nbd_list: ...@@ -361,7 +358,7 @@ for nbd_ip, nbd_port in nbd_list:
s = getSocketStatus(nbd_ip, nbd_port) s = getSocketStatus(nbd_ip, nbd_port)
if s is None: if s is None:
# NBD is not available : launch kvm without it # NBD is not available : launch kvm without it
print 'Warning : Nbd is not available.' print('Warning : Nbd is not available.')
else: else:
# NBD is available # NBD is available
# We close the NBD socket else qemu won't be able to use it apparently # We close the NBD socket else qemu won't be able to use it apparently
...@@ -375,5 +372,5 @@ else: ...@@ -375,5 +372,5 @@ else:
'-drive', 'file=%s,media=cdrom' % default_cdrom_iso '-drive', 'file=%s,media=cdrom' % default_cdrom_iso
]) ])
print 'Starting KVM: \n %s' % ' '.join(kvm_argument_list) print('Starting KVM: \n %s' % ' '.join(kvm_argument_list))
os.execv(qemu_path, kvm_argument_list) os.execv(qemu_path, kvm_argument_list)
...@@ -47,6 +47,7 @@ setup(name=name, ...@@ -47,6 +47,7 @@ setup(name=name,
'erp5.util', 'erp5.util',
'supervisor', 'supervisor',
'psutil', 'psutil',
'six',
], ],
zip_safe=True, zip_safe=True,
test_suite='test', test_suite='test',
......
...@@ -25,13 +25,14 @@ ...@@ -25,13 +25,14 @@
# #
############################################################################## ##############################################################################
import httplib import six.moves.http_client as httplib
import json import json
import os import os
import requests import requests
import six
import slapos.util import slapos.util
import sqlite3 import sqlite3
import urlparse from six.moves.urllib.parse import parse_qs, urlparse
import unittest import unittest
from slapos.recipe.librecipe import generateHashFromFiles from slapos.recipe.librecipe import generateHashFromFiles
...@@ -43,7 +44,8 @@ skipUnlessKvm = unittest.skipUnless(has_kvm, 'kvm not loaded or not allowed') ...@@ -43,7 +44,8 @@ skipUnlessKvm = unittest.skipUnless(has_kvm, 'kvm not loaded or not allowed')
if has_kvm: if has_kvm:
setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass( setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath( os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))) os.path.join(os.path.dirname(__file__), '..',
'software%s.cfg' % ("-py3" if six.PY3 else ""))))
else: else:
setUpModule, InstanceTestCase = None, unittest.TestCase setUpModule, InstanceTestCase = None, unittest.TestCase
...@@ -118,7 +120,7 @@ class MonitorAccessMixin(object): ...@@ -118,7 +120,7 @@ class MonitorAccessMixin(object):
monitor_setup_url = connection_parameter_dict['monitor-setup-url'] monitor_setup_url = connection_parameter_dict['monitor-setup-url']
monitor_url_with_auth = 'https' + monitor_setup_url.split('https')[2] monitor_url_with_auth = 'https' + monitor_setup_url.split('https')[2]
auth = urlparse.parse_qs(urlparse.urlparse(monitor_url_with_auth).path) auth = parse_qs(urlparse(monitor_url_with_auth).path)
# check that monitor-base-url for all partitions in the tree are accessible # check that monitor-base-url for all partitions in the tree are accessible
# with published username and password # with published username and password
...@@ -128,7 +130,7 @@ class MonitorAccessMixin(object): ...@@ -128,7 +130,7 @@ class MonitorAccessMixin(object):
if not connection_xml: if not connection_xml:
continue continue
connection_dict = slapos.util.xml2dict( connection_dict = slapos.util.xml2dict(
partition_information['connection_xml'].encode('utf-8')) connection_xml if six.PY3 else connection_xml.encode('utf-8'))
monitor_base_url = connection_dict.get('monitor-base-url') monitor_base_url = connection_dict.get('monitor-base-url')
if not monitor_base_url: if not monitor_base_url:
continue continue
...@@ -162,8 +164,8 @@ class TestAccessDefault(MonitorAccessMixin, InstanceTestCase): ...@@ -162,8 +164,8 @@ class TestAccessDefault(MonitorAccessMixin, InstanceTestCase):
httplib.OK, httplib.OK,
result.status_code result.status_code
) )
self.assertTrue('<title>noVNC</title>' in result.text) self.assertIn('<title>noVNC</title>', result.text)
self.assertFalse('url-additional' in connection_parameter_dict) self.assertNotIn('url-additional', connection_parameter_dict)
@skipUnlessKvm @skipUnlessKvm
...@@ -186,7 +188,7 @@ class TestAccessDefaultAdditional(MonitorAccessMixin, InstanceTestCase): ...@@ -186,7 +188,7 @@ class TestAccessDefaultAdditional(MonitorAccessMixin, InstanceTestCase):
httplib.OK, httplib.OK,
result.status_code result.status_code
) )
self.assertTrue('<title>noVNC</title>' in result.text) self.assertIn('<title>noVNC</title>', result.text)
result = requests.get( result = requests.get(
connection_parameter_dict['url-additional'], verify=False) connection_parameter_dict['url-additional'], verify=False)
...@@ -194,7 +196,7 @@ class TestAccessDefaultAdditional(MonitorAccessMixin, InstanceTestCase): ...@@ -194,7 +196,7 @@ class TestAccessDefaultAdditional(MonitorAccessMixin, InstanceTestCase):
httplib.OK, httplib.OK,
result.status_code result.status_code
) )
self.assertTrue('<title>noVNC</title>' in result.text) self.assertIn('<title>noVNC</title>', result.text)
@skipUnlessKvm @skipUnlessKvm
...@@ -219,13 +221,13 @@ class TestAccessKvmCluster(MonitorAccessMixin, InstanceTestCase): ...@@ -219,13 +221,13 @@ class TestAccessKvmCluster(MonitorAccessMixin, InstanceTestCase):
def test(self): def test(self):
connection_parameter_dict = self.computer_partition\ connection_parameter_dict = self.computer_partition\
.getConnectionParameterDict() .getConnectionParameterDict()
result = requests.get(connection_parameter_dict['kvm0-url'], verify=False) result = requests.get(connection_parameter_dict['KVM0-url'], verify=False)
self.assertEqual( self.assertEqual(
httplib.OK, httplib.OK,
result.status_code result.status_code
) )
self.assertTrue('<title>noVNC</title>' in result.text) self.assertIn('<title>noVNC</title>', result.text)
self.assertFalse('kvm0-url-additional' in connection_parameter_dict) self.assertNotIn('KVM0-url-additional', connection_parameter_dict)
@skipUnlessKvm @skipUnlessKvm
...@@ -253,20 +255,20 @@ class TestAccessKvmClusterAdditional(MonitorAccessMixin, InstanceTestCase): ...@@ -253,20 +255,20 @@ class TestAccessKvmClusterAdditional(MonitorAccessMixin, InstanceTestCase):
def test(self): def test(self):
connection_parameter_dict = self.computer_partition\ connection_parameter_dict = self.computer_partition\
.getConnectionParameterDict() .getConnectionParameterDict()
result = requests.get(connection_parameter_dict['kvm0-url'], verify=False) result = requests.get(connection_parameter_dict['KVM0-url'], verify=False)
self.assertEqual( self.assertEqual(
httplib.OK, httplib.OK,
result.status_code result.status_code
) )
self.assertTrue('<title>noVNC</title>' in result.text) self.assertIn('<title>noVNC</title>', result.text)
result = requests.get( result = requests.get(
connection_parameter_dict['kvm0-url-additional'], verify=False) connection_parameter_dict['KVM0-url-additional'], verify=False)
self.assertEqual( self.assertEqual(
httplib.OK, httplib.OK,
result.status_code result.status_code
) )
self.assertTrue('<title>noVNC</title>' in result.text) self.assertIn('<title>noVNC</title>', result.text)
@skipUnlessKvm @skipUnlessKvm
......
...@@ -15,4 +15,4 @@ ...@@ -15,4 +15,4 @@
[template] [template]
filename = instance.cfg filename = instance.cfg
md5sum = 3ad1b06673000d9f424a1e7187c6a1fa md5sum = b21b2a9ac7f027a044a897c6eacbba56
...@@ -2,44 +2,41 @@ ...@@ -2,44 +2,41 @@
parts = parts =
slapos-test-runner slapos-test-runner
eggs-directory = ${buildout:eggs-directory} eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true offline = true
[slap-configuration] [slap-configuration]
recipe = slapos.cookbook:slapconfiguration recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id} computer = ${slap-connection:computer-id}
partition = $${slap-connection:partition-id} partition = ${slap-connection:partition-id}
url = $${slap-connection:server-url} url = ${slap-connection:server-url}
key = $${slap-connection:key-file} key = ${slap-connection:key-file}
cert = $${slap-connection:cert-file} cert = ${slap-connection:cert-file}
[download-source] [download-source]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git git-executable = {{ git_location }}/bin/git
[slapos] [slapos]
<= download-source <= download-source
repository = ${slapos-repository:location} repository = {{ slapos_location }}
[directory] [directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
bin = $${buildout:directory}/bin bin = ${buildout:directory}/bin
working-dir = $${buildout:directory}/tmp working-dir = ${buildout:directory}/tmp
[test-list]
path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup},${slapos.test.theia-setup:setup},${slapos.test.grafana-setup:setup},${slapos.test.gitlab-setup:setup}
[slapos-test-runner] [slapos-test-runner]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:bin}/runTestSuite wrapper-path = ${directory:bin}/runTestSuite
command-line = command-line =
${buildout:bin-directory}/runTestSuite {{ buildout['bin-directory'] }}/runTestSuite
--python_interpreter=${buildout:bin-directory}/${eggs:interpreter} --python_interpreter={{ buildout['bin-directory'] }}/{{ interpreter }}
--source_code_path_list=$${test-list:path_list} --source_code_path_list={{ ','.join(tests.splitlines()) }}
environment = environment =
PATH=${buildout:bin-directory}:${quic_client-bin:location}:${curl:location}/bin/:/usr/bin/:/bin PATH={{ buildout['bin-directory'] }}:{{ quic_client_location }}:{{ curl_location }}/bin/:/usr/bin/:/bin
SLAPOS_TEST_IPV4=$${slap-configuration:ipv4-random} SLAPOS_TEST_IPV4=${slap-configuration:ipv4-random}
SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random} SLAPOS_TEST_IPV6=${slap-configuration:ipv6-random}
SLAPOS_TEST_WORKING_DIR=$${directory:working-dir} SLAPOS_TEST_WORKING_DIR=${directory:working-dir}
[buildout]
extends =
../../component/python3/buildout.cfg
software.cfg
python = python3
[eggs]
eggs -=
# plantuml is not Py3-compatible
${slapos.test.plantuml-setup:egg}
[template]
extra =
...@@ -176,10 +176,40 @@ repository = https://lab.nexedi.com/nexedi/slapos.git ...@@ -176,10 +176,40 @@ repository = https://lab.nexedi.com/nexedi/slapos.git
branch = master branch = master
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template:jinja2
url = ${:_profile_base_location_}/${:filename} template = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template.cfg rendered = ${buildout:directory}/template.cfg
mode = 640 mode = 640
context =
section buildout buildout
key git_location git:location
key slapos_location slapos-repository:location
key interpreter eggs:interpreter
key quic_client_location quic_client-bin:location
key curl_location curl:location
key tests :tests
tests =
${slapos.test.kvm-setup:setup}
${slapos.test.slaprunner-setup:setup}
${:extra}
extra =
${slapos.cookbook-setup:setup}
${slapos.test.caddy-frontend-setup:setup}
${slapos.test.erp5-setup:setup}
${slapos.test.slapos-master-setup:setup}
${slapos.test.monitor-setup:setup}
${slapos.test.plantuml-setup:setup}
${slapos.test.powerdns-setup:setup}
${slapos.test.proftpd-setup:setup}
${slapos.test.re6stnet-setup:setup}
${slapos.test.seleniumserver-setup:setup}
${slapos.test.helloworld-setup:setup}
${slapos.test.jupyter-setup:setup}
${slapos.test.nextcloud-setup:setup}
${slapos.test.turnserver-setup:setup}
${slapos.test.theia-setup:setup}
${slapos.test.grafana-setup:setup}
${slapos.test.gitlab-setup:setup}
[versions] [versions]
# slapos.core is used from the clone always # slapos.core is used from the clone always
......
...@@ -14,11 +14,11 @@ ...@@ -14,11 +14,11 @@
# not need these here). # not need these here).
[template] [template]
filename = instance.cfg filename = instance.cfg
md5sum = 8b78e32b877d591400746ec7fd68ed4c md5sum = ec70348dd71b319590a5c5837f3d2e45
[template-runner] [template-runner]
filename = instance-runner.cfg filename = instance-runner.cfg
md5sum = 84b81c18c77af08d7e97b0b6da4ee105 md5sum = b20894f378530e79c847ddfb61782cc5
[template-runner-import-script] [template-runner-import-script]
filename = template/runner-import.sh.jinja2 filename = template/runner-import.sh.jinja2
...@@ -34,7 +34,7 @@ md5sum = b992bb3391de9d6d422bfa8011d8ffc4 ...@@ -34,7 +34,7 @@ md5sum = b992bb3391de9d6d422bfa8011d8ffc4
[template-resilient] [template-resilient]
filename = instance-resilient.cfg.jinja2 filename = instance-resilient.cfg.jinja2
md5sum = 2271c829b94542b7b2d9c589376ae538 md5sum = 105ed7f54f251c64a2d34559360a5604
[template_nginx_conf] [template_nginx_conf]
filename = nginx_conf.in filename = nginx_conf.in
...@@ -62,7 +62,7 @@ md5sum = 7645048216fcf957f7773534cd0408dc ...@@ -62,7 +62,7 @@ md5sum = 7645048216fcf957f7773534cd0408dc
[template-supervisord] [template-supervisord]
filename = template/supervisord.conf.in filename = template/supervisord.conf.in
md5sum = d294d0dafd265048399de6da8c96345f md5sum = 28f69b57c2835bddfcfbe312563eef51
[template-listener-slapgrid] [template-listener-slapgrid]
filename = template/listener_slapgrid.py.in filename = template/listener_slapgrid.py.in
......
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
{% set monitor_interface_url = slapparameter_dict.pop('monitor-interface-url', 'https://monitor.app.officejs.com') -%} {% set monitor_interface_url = slapparameter_dict.pop('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
{% import 'parts' as parts %} {% import 'parts' as parts %}
{% import 'replicated' as replicated %} {% import 'replicated' as replicated with context %}
[buildout] [buildout]
eggs-directory = {{ eggs_directory }} eggs-directory = {{ eggs_directory }}
......
...@@ -51,9 +51,9 @@ extends = ...@@ -51,9 +51,9 @@ extends =
${monitor2-template:rendered} ${monitor2-template:rendered}
${template-logrotate-base:rendered} ${template-logrotate-base:rendered}
eggs-directory = ${buildout:eggs-directory} [slap-network-information]
develop-eggs-directory = ${buildout:develop-eggs-directory} local-ipv4 = $${slap-configuration:ipv4-random}
offline = true global-ipv6 = $${slap-configuration:ipv6-random}
{% if slapparameter_dict.get('custom-frontend-backend-url') -%} {% if slapparameter_dict.get('custom-frontend-backend-url') -%}
[request-custom-frontend] [request-custom-frontend]
...@@ -287,35 +287,29 @@ extra-args=-t rsa ...@@ -287,35 +287,29 @@ extra-args=-t rsa
<=runner-sshd-ssh-keygen-base <=runner-sshd-ssh-keygen-base
extra-args=-t ecdsa -b 521 extra-args=-t ecdsa -b 521
[runner-sshd-publickey-fingerprint-shelloutput]
recipe = collective.recipe.shelloutput
# XXX because collective.recipe.shelloutput ignore errors, we run the same
# command in a plone.recipe.command so that if fails if something goes wrong.
commands =
fingerprint = bash -o pipefail -c "${openssh-output:keygen} -lf $${runner-sshd-ssh-host-ecdsa-key:output} | cut -f 2 -d\ | sed 's/+/%2B/g' | sed 's/\//%2F/g' | sed 's/SHA256://'"
[runner-sshd-publickey-fingerprint] [runner-sshd-publickey-fingerprint]
# fingerprint for ssh url, see # fingerprint for ssh url, see
# https://tools.ietf.org/id/draft-salowey-secsh-uri-00.html#connparam # https://tools.ietf.org/id/draft-salowey-secsh-uri-00.html#connparam
# https://winscp.net/eng/docs/session_url#hostkey # https://winscp.net/eng/docs/session_url#hostkey
recipe = slapos.recipe.build
_fingerprint = $${runner-sshd-publickey-fingerprint-shelloutput:fingerprint} init =
import os
# format is host-key-alg-fingerprint, but we know that import subprocess
# $${runner-sshkeys-sshd:public-key} is rsa so for host-key-alg from six.moves.urllib.parse import quote
# we just use use rsa. keyfile = self.buildout['runner-sshd-ssh-host-ecdsa-key']['output']
fingerprint = ssh-rsa-$${:_fingerprint} if os.path.isfile(keyfile):
x = subprocess.check_output(('${openssh-output:keygen}', '-lf', keyfile))
# XXX because collective.recipe.shelloutput ignore errors and capture output x = x.split()[1]
# "Error ...", we use a plone.recipe.command to check that this command did assert x.startswith(b'SHA256:'), x
# not fail. # format is host-key-alg-fingerprint, but we know that
# This command will always fail on first buildout run, because # $${runner-sshd-ssh-host-ecdsa-key:output} is rsa so for host-key-alg
# collective.recipe.shelloutput is evaluated at buildout recipes __init__ step, # we just use use rsa.
# but the key file is created later at install step. options['fingerprint'] = "ssh-rsa-" + quote(x[7:], safe='')
recipe = plone.recipe.command else:
stop-on-error = true # This command will always fail on first buildout run, because it is
command = echo "$${:_fingerprint}" | ( grep ^Error || exit 0 && exit 1 ) # evaluated at buildout recipes __init__ step, but the key file is created
# later at install step.
options['fingerprint'] = "NotReady"
#--------------------------- #---------------------------
#-- #--
...@@ -665,6 +659,9 @@ monitor-cors-domains = ...@@ -665,6 +659,9 @@ monitor-cors-domains =
monitor-interface-url = monitor-interface-url =
monitor-httpd-port = 8386 monitor-httpd-port = 8386
buildout-shared-folder = $${runnerdirectory:home}/shared buildout-shared-folder = $${runnerdirectory:home}/shared
{% for k, v in slapparameter_dict.items() -%}
{{ k }} = {{ v }}
{% endfor -%}
[slapos-cfg] [slapos-cfg]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -745,15 +742,6 @@ recipe = plone.recipe.command ...@@ -745,15 +742,6 @@ recipe = plone.recipe.command
stop-on-error = true stop-on-error = true
command = SR=$${slap-parameter:slapos-software} && if [ -n "$SR" ] && [ ! -f "$${directory:etc}/.project" ]; then echo workspace/slapos/$${slap-parameter:slapos-software}/ > $${directory:etc}/.project; fi command = SR=$${slap-parameter:slapos-software} && if [ -n "$SR" ] && [ ! -f "$${directory:etc}/.project" ]; then echo workspace/slapos/$${slap-parameter:slapos-software}/ > $${directory:etc}/.project; fi
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[minishell-cwd] [minishell-cwd]
recipe = plone.recipe.command recipe = plone.recipe.command
command = if [ ! -f $${slaprunner:minishell_cwd_file} ]; then echo $${runnerdirectory:home} > $${slaprunner:minishell_cwd_file}; fi command = if [ ! -f $${slaprunner:minishell_cwd_file} ]; then echo $${runnerdirectory:home} > $${slaprunner:minishell_cwd_file}; fi
...@@ -820,7 +808,6 @@ template = ${template-supervisord:location}/${template-supervisord:filename} ...@@ -820,7 +808,6 @@ template = ${template-supervisord:location}/${template-supervisord:filename}
rendered = $${directory:etc}/supervisord.conf rendered = $${directory:etc}/supervisord.conf
context = context =
import multiprocessing multiprocessing import multiprocessing multiprocessing
import builtin __builtin__
section supervisord supervisord section supervisord supervisord
key slapparameter_dict slap-configuration:configuration key slapparameter_dict slap-configuration:configuration
key listener_slapgrid listener-slapgrid-bin:rendered key listener_slapgrid listener-slapgrid-bin:rendered
......
...@@ -9,15 +9,18 @@ extends = ...@@ -9,15 +9,18 @@ extends =
${template-resilient-templates:output} ${template-resilient-templates:output}
[switch_softwaretype] [switch_softwaretype]
recipe = slapos.cookbook:softwaretype recipe = slapos.cookbook:switch-softwaretype
default = $${instance-base-runner:rendered} default = $${:runner}
resilient = $${instance-resilient:rendered} resilient = instance-resilient:rendered
runner = $${instance-base-runner:rendered} runner = instance-base-runner:rendered
runner-import = $${template-runner-import:rendered} runner-import = template-runner-import:rendered
runner-export = $${template-runner-export:rendered} runner-export = template-runner-export:rendered
frozen = ${instance-frozen:output} frozen = instance-frozen:rendered
pull-backup = ${template-pull-backup:output} pull-backup = template-pull-backup:rendered
# BBB
RootSoftwareInstance = $${:default}
[instance-base-runner] [instance-base-runner]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
......
[buildout]
extends =
../../component/python3/buildout.cfg
software.cfg
python = python3
# Ignore these for now
common-parts -=
rdiff-backup
[eggs]
eggs -=
# futures is a backport of Py3's concurrent.futures module
futures
...@@ -150,10 +150,10 @@ eggs = ...@@ -150,10 +150,10 @@ eggs =
erp5.util erp5.util
lock-file lock-file
plone.recipe.command plone.recipe.command
collective.recipe.shelloutput
slapos.recipe.build slapos.recipe.build
slapos.toolbox[flask_auth] slapos.toolbox[flask_auth]
gunicorn==19.7.1 gunicorn
# for gunicorn[gthread]
futures futures
${slapos-cookbook:eggs} ${slapos-cookbook:eggs}
slapos.core # listed explicitly for scripts generation slapos.core # listed explicitly for scripts generation
...@@ -169,7 +169,7 @@ Flask-Auth = 0.85 ...@@ -169,7 +169,7 @@ Flask-Auth = 0.85
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
futures = 3.0.5 futures = 3.0.5
gitdb = 0.6.4 gitdb = 0.6.4
gunicorn = 19.7.1 gunicorn = 19.10.0
prettytable = 0.7.2 prettytable = 0.7.2
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.4 slapos.recipe.template = 4.4
......
...@@ -27,7 +27,7 @@ stdout_logfile = {{ supervisord['no_logfile'] }} ...@@ -27,7 +27,7 @@ stdout_logfile = {{ supervisord['no_logfile'] }}
stderr_logfile = {{ supervisord['no_logfile'] }} stderr_logfile = {{ supervisord['no_logfile'] }}
directory = {{ supervisord['directory'] }} directory = {{ supervisord['directory'] }}
{# how many parallel build jobs to spawn when compiling software -#} {# how many parallel build jobs to spawn when compiling software -#}
{% set njobs = builtin.max(1, (multiprocessing.cpu_count() // builtin.int(slapparameter_dict.get('cpu-usage-ratio', 4)))) -%} {% set njobs = max(1, (multiprocessing.cpu_count() // int(slapparameter_dict.get('cpu-usage-ratio', 4)))) -%}
environment = PATH="{{- supervisord['path'] -}}",MAKEFLAGS="-j{{ njobs }}",NPY_NUM_BUILD_JOBS="{{ njobs }}",BUNDLE_JOBS="{{ njobs }}" environment = PATH="{{- supervisord['path'] -}}",MAKEFLAGS="-j{{ njobs }}",NPY_NUM_BUILD_JOBS="{{ njobs }}",BUNDLE_JOBS="{{ njobs }}"
[program:{{- supervisord['slapgrid-cp'] -}}] [program:{{- supervisord['slapgrid-cp'] -}}]
......
...@@ -38,13 +38,16 @@ from six.moves.urllib.parse import quote ...@@ -38,13 +38,16 @@ from six.moves.urllib.parse import quote
from six.moves.urllib.parse import urljoin from six.moves.urllib.parse import urljoin
from six.moves.configparser import ConfigParser from six.moves.configparser import ConfigParser
import requests import requests
import six
from slapos.recipe.librecipe import generateHashFromFiles from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.util import bytes2str
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass( setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath( os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))) os.path.join(os.path.dirname(__file__), '..',
'software%s.cfg' % ("-py3" if six.PY3 else ""))))
class SlaprunnerTestCase(SlapOSInstanceTestCase): class SlaprunnerTestCase(SlapOSInstanceTestCase):
...@@ -185,7 +188,7 @@ class TestSSH(SlaprunnerTestCase): ...@@ -185,7 +188,7 @@ class TestSSH(SlaprunnerTestCase):
channel.settimeout(30) channel.settimeout(30)
received = '' received = ''
while True: while True:
r = channel.recv(1024) r = bytes2str(channel.recv(1024))
self.logger.debug("received >%s<", r) self.logger.debug("received >%s<", r)
if not r: if not r:
break break
...@@ -197,7 +200,7 @@ class TestSSH(SlaprunnerTestCase): ...@@ -197,7 +200,7 @@ class TestSSH(SlaprunnerTestCase):
# simple commands can also be executed ( this would be like `ssh bash -c 'pwd'` ) # simple commands can also be executed ( this would be like `ssh bash -c 'pwd'` )
self.assertEqual( self.assertEqual(
self.computer_partition_root_path, self.computer_partition_root_path,
client.exec_command("pwd")[1].read(1000).strip()) bytes2str(client.exec_command("pwd")[1].read(1000)).strip())
class TestSlapOS(SlaprunnerTestCase): class TestSlapOS(SlaprunnerTestCase):
...@@ -211,7 +214,7 @@ class TestSlapOS(SlaprunnerTestCase): ...@@ -211,7 +214,7 @@ class TestSlapOS(SlaprunnerTestCase):
'show', 'show',
), ),
env={}) env={})
self.assertIn('slaprunner', proxy_show_output) self.assertIn(b'slaprunner', proxy_show_output)
def test_shared_part_list(self): def test_shared_part_list(self):
# this slapos used shared_part_list # this slapos used shared_part_list
......
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[pbsready] [pbsready]
filename = pbsready.cfg.in filename = pbsready.cfg.in
md5sum = 5e0dcd4c290f0b46cb2d316dc1c9c011 md5sum = 66331047b7dbf2513c5726d5d1647320
[pbsready-import] [pbsready-import]
filename = pbsready-import.cfg.in filename = pbsready-import.cfg.in
...@@ -22,15 +22,15 @@ md5sum = d813c43ed00eff868fb13bc75b045336 ...@@ -22,15 +22,15 @@ md5sum = d813c43ed00eff868fb13bc75b045336
[pbsready-export] [pbsready-export]
filename = pbsready-export.cfg.in filename = pbsready-export.cfg.in
md5sum = 2e804e06b5203c3f127c31a1704c48bd md5sum = 2b0c71b085cfe8017f28098c160b1f49
[template-pull-backup] [template-pull-backup]
filename = instance-pull-backup.cfg.in filename = instance-pull-backup.cfg.in
md5sum = 0bbe16f3d805afd880a251a4f40ecaf1 md5sum = 555d528b198564f0ce1e94db1160ebf3
[template-replicated] [template-replicated]
filename = template-replicated.cfg.in filename = template-replicated.cfg.in
md5sum = 290b380fe3da8736642bc10a8b1163d1 md5sum = 815fd8f7c42b9cf59b286b0fe77fa76d
[template-parts] [template-parts]
filename = template-parts.cfg.in filename = template-parts.cfg.in
...@@ -38,7 +38,7 @@ md5sum = 071b1034ee8f5cc14f79b16fdeba2813 ...@@ -38,7 +38,7 @@ md5sum = 071b1034ee8f5cc14f79b16fdeba2813
[template-resilient-templates] [template-resilient-templates]
filename = template-resilient-templates.cfg.in filename = template-resilient-templates.cfg.in
md5sum = 41e82859dc6b65e94a300a006d51536e md5sum = 097a14371efde11465ab4bd08ef3131b
[instance-frozen] [instance-frozen]
filename = instance-frozen.cfg.in filename = instance-frozen.cfg.in
...@@ -46,7 +46,7 @@ md5sum = d21472f0e58f928fb827f2cbf22c4d4a ...@@ -46,7 +46,7 @@ md5sum = d21472f0e58f928fb827f2cbf22c4d4a
[resilient-web-takeover-cgi-script-download] [resilient-web-takeover-cgi-script-download]
filename = resilient-web-takeover-cgi-script.py.in filename = resilient-web-takeover-cgi-script.py.in
md5sum = 60d4912fdf5e8dafaba9d9f333aa9e36 md5sum = 675ac9e1cf49ccc8f8eddb541a62d899
[template-wrapper] [template-wrapper]
filename = templates/wrapper.in filename = templates/wrapper.in
...@@ -54,9 +54,9 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8 ...@@ -54,9 +54,9 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
[notifier-feed-promise-template] [notifier-feed-promise-template]
filename = templates/notifier-feed-promise.py.in filename = templates/notifier-feed-promise.py.in
md5sum = d75346911dbc4cfcdb39a21e56cd5016 md5sum = fa6521daaa02fef4dd2ce06d29ef90be
[template-monitor-check-resilient-feed] [template-monitor-check-resilient-feed]
filename = templates/monitor-check-resilient-feed.in filename = templates/monitor-check-resilient-feed.in
md5sum = 19ee9055de961acf402e2dfe5b9581d2 md5sum = af9787f8440fef19924b2e765372b20f
...@@ -19,10 +19,6 @@ parts = ...@@ -19,10 +19,6 @@ parts =
extends = extends =
${monitor2-template:rendered} ${monitor2-template:rendered}
${template-logrotate-base:rendered} ${template-logrotate-base:rendered}
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
#---------------- #----------------
#-- #--
...@@ -90,7 +86,7 @@ rendered = $${:wrapper} ...@@ -90,7 +86,7 @@ rendered = $${:wrapper}
wrapper = $${basedirectory:services}/notifier wrapper = $${basedirectory:services}/notifier
mode = 0700 mode = 0700
command = ${buildout:bin-directory}/pubsubserver --callbacks $${directory:notifier-callbacks} --feeds $${directory:notifier-feeds} --equeue-socket $${equeue:socket} --logfile $${basedirectory:log}/notifier.log $${:host} $${:port} command = ${buildout:bin-directory}/pubsubserver --callbacks $${directory:notifier-callbacks} --feeds $${directory:notifier-feeds} --equeue-socket $${equeue:socket} --logfile $${basedirectory:log}/notifier.log $${:host} $${:port}
host = $${slap-network-information:global-ipv6} host = {{ ipv6 }}
port = $${notifier-port:port} port = $${notifier-port:port}
context = context =
key content notifier:command key content notifier:command
...@@ -128,8 +124,8 @@ run-directory = $${basedirectory:run} ...@@ -128,8 +124,8 @@ run-directory = $${basedirectory:run}
pull-push-maximum-run = 5 pull-push-maximum-run = 5
# XXX: this should be named "notifier-host" # XXX: this should be named "notifier-host"
notifier-url = http://[$${notifier:host}]:$${notifier:port} notifier-url = http://[$${notifier:host}]:$${notifier:port}
slave-instance-list = $${slap-parameter:slave_instance_list} slave-instance-list = {{ slapparameter_dict.get('slave_instance_list', '[]') }}
ignore-known-hosts-file = $${slap-parameter:ignore-known-hosts-file} ignore-known-hosts-file = {{ slapparameter_dict.get('ignore-known-hosts-file', 'false') }}
# To get a verbose feed about PBS state # To get a verbose feed about PBS state
instance-root-name = $${monitor-instance-parameter:root-instance-title} instance-root-name = $${monitor-instance-parameter:root-instance-title}
log-url = $${monitor-publish-parameters:monitor-base-url}/private/notifier/ log-url = $${monitor-publish-parameters:monitor-base-url}/private/notifier/
...@@ -194,20 +190,6 @@ identity-file = $${:home}/id_rsa ...@@ -194,20 +190,6 @@ identity-file = $${:home}/id_rsa
command-line = ${openssh:location}/bin/ssh -T -o "UserKnownHostsFile $${pbs:known-hosts}" -i $${:identity-file} command-line = ${openssh:location}/bin/ssh -T -o "UserKnownHostsFile $${pbs:known-hosts}" -i $${:identity-file}
wrapper-path = $${rootdirectory:bin}/ssh wrapper-path = $${rootdirectory:bin}/ssh
#----------------
#--
#-- Slave instance list (empty default).
[slap-parameter]
slave_instance_list = []
ignore-known-hosts-file = false
monitor-cors-domains =
monitor-httpd-port = 8070
monitor-title = PBS Instance
monitor-password = $${monitor-htpasswd:passwd}
monitor-username = admin
#---------------- #----------------
#-- #--
#-- Resiliency promises. #-- Resiliency promises.
...@@ -269,11 +251,11 @@ monitor-password = $${monitor-publish-parameters:monitor-password} ...@@ -269,11 +251,11 @@ monitor-password = $${monitor-publish-parameters:monitor-password}
#-- Monitor #-- Monitor
[monitor-instance-parameter] [monitor-instance-parameter]
monitor-httpd-port = $${slap-parameter:monitor-httpd-port} monitor-httpd-port = {{ slapparameter_dict.get('monitor-httpd-port', 8070) }}
monitor-title = $${slap-parameter:monitor-title} monitor-title = {{ slapparameter_dict.get('monitor-title', 'PBS Instance') }}
cors-domains = $${slap-parameter:monitor-cors-domains} cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', '') }}
username = $${slap-parameter:monitor-username} username = {{ slapparameter_dict.get('monitor-username', 'admin') }}
password = $${slap-parameter:monitor-password} password = {{ slapparameter_dict.get('monitor-password', '$${monitor-htpasswd:passwd}') }}
[monitor-conf-parameters] [monitor-conf-parameters]
private-path-list += private-path-list +=
......
...@@ -39,7 +39,7 @@ name = exporter ...@@ -39,7 +39,7 @@ name = exporter
title = Dumping ${slap-parameter:namebase} title = Dumping ${slap-parameter:namebase}
executable = ${exporter:wrapper} executable = ${exporter:wrapper}
wrapper = ${rootdirectory:bin}/exporter wrapper = ${rootdirectory:bin}/exporter
notify = ${slap-parameter:notify} notify = {{ slapparameter_dict.get('notify', '') }}
pidfile = ${resilient-directory:pid}/${:name}.pid pidfile = ${resilient-directory:pid}/${:name}.pid
max-run = 3 max-run = 3
...@@ -85,8 +85,3 @@ recipe = slapos.cookbook:random.time ...@@ -85,8 +85,3 @@ recipe = slapos.cookbook:random.time
recipe = slapos.cookbook:publish-early recipe = slapos.cookbook:publish-early
-init = -init =
resiliency-backup-periodicity gen-resiliency-backup-periodicity:time resiliency-backup-periodicity gen-resiliency-backup-periodicity:time
[slap-parameter]
# In cron.d format (i.e things like */15 * * * * are accepted).
resiliency-backup-periodicity =
notify =
\ No newline at end of file
...@@ -22,6 +22,10 @@ parts += ...@@ -22,6 +22,10 @@ parts +=
extends = extends =
${monitor2-template:rendered} ${monitor2-template:rendered}
[slap-network-information]
local-ipv4 = $${slap-configuration:ipv4-random}
global-ipv6 = $${slap-configuration:ipv6-random}
#---------------- #----------------
#-- #--
#-- Creation of all needed directories. #-- Creation of all needed directories.
......
#!${buildout:executable} #!${buildout:executable}
from __future__ import print_function
equeue_database = '${equeue:database}' equeue_database = '${equeue:database}'
equeue_lockfile = '${equeue:lockfile}' equeue_lockfile = '${equeue:lockfile}'
takeover_script = '${resiliency-takeover-script:wrapper-takeover}' takeover_script = '${resiliency-takeover-script:wrapper-takeover}'
...@@ -8,7 +9,11 @@ import atexit ...@@ -8,7 +9,11 @@ import atexit
import cgi import cgi
import cgitb import cgitb
import datetime import datetime
import gdbm try:
import dbm.gnu as gdbm
except ImportError:
import gdbm
import os import os
import shutil import shutil
import subprocess import subprocess
...@@ -39,11 +44,12 @@ def getLatestBackupDate(): ...@@ -39,11 +44,12 @@ def getLatestBackupDate():
# Usually, there is only one callback (so only one key # Usually, there is only one callback (so only one key
# in the db), but if there are several: # in the db), but if there are several:
# Take the "oldest" one (oldest value). # Take the "oldest" one (oldest value).
if not db.keys(): db_keys = db.keys()
if not db_keys:
result = False result = False
else: else:
last_backup = db[db.keys()[-1]] last_backup = db[db_keys[-1]]
for callback in db.keys(): for callback in db_keys:
timestamp = float(db[callback]) timestamp = float(db[callback])
if timestamp < last_backup: if timestamp < last_backup:
last_backup = timestamp last_backup = timestamp
...@@ -79,12 +85,11 @@ if latest_backup_date == False: ...@@ -79,12 +85,11 @@ if latest_backup_date == False:
else: else:
latest_backup_message = latest_backup_date.strftime('%Y-%m-%d %H:%M:%S') latest_backup_message = latest_backup_date.strftime('%Y-%m-%d %H:%M:%S')
print "Content-Type: text/html" print("Content-Type: text/html\n")
print
form = cgi.FieldStorage() form = cgi.FieldStorage()
if "password" not in form: if "password" not in form:
print """<html> print("""<html>
<body> <body>
<h1>This is takeover web interface.</h1> <h1>This is takeover web interface.</h1>
<p>Calling takeover will stop and freeze the current main instance, and make this clone instance the new main instance, replacing the old one.</p> <p>Calling takeover will stop and freeze the current main instance, and make this clone instance the new main instance, replacing the old one.</p>
...@@ -100,15 +105,15 @@ if "password" not in form: ...@@ -100,15 +105,15 @@ if "password" not in form:
<input type="submit" value="Take over" style="background: red;"> <input type="submit" value="Take over" style="background: red;">
</form> </form>
</body> </body>
</html>""" % (latest_backup_message, isBackupInProgress(), getSoftwareReleaseInformationFormatted()) </html>""" % (latest_backup_message, isBackupInProgress(), getSoftwareReleaseInformationFormatted()))
sys.exit(0) sys.exit(0)
if form['password'].value != '${:password}': if form['password'].value != '${:password}':
print "<H1>Error</H1>" print("<H1>Error</H1>")
print "Password is invalid." print("Password is invalid.")
sys.exit(1) sys.exit(1)
# XXX hardcoded location # XXX hardcoded location
result = subprocess.check_output([takeover_script], stderr=subprocess.STDOUT) result = subprocess.check_output([takeover_script], stderr=subprocess.STDOUT)
print 'Success.' print('Success.')
print '<pre>%s</pre>' % result print('<pre>%s</pre>' % result)
...@@ -41,7 +41,7 @@ config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}} ...@@ -41,7 +41,7 @@ config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}
config-name = {{namebase}}0 config-name = {{namebase}}0
# Bubble up all the instance parameters to the requested export instance. # Bubble up all the instance parameters to the requested export instance.
{% if slapparameter_dict is defined %} {% if slapparameter_dict is defined %}
{% for parameter_name, parameter_value in slapparameter_dict.items() %} {% for parameter_name, parameter_value in six.iteritems(slapparameter_dict) %}
{% if parameter_value is string %} {% if parameter_value is string %}
config-{{parameter_name}} = {{ parameter_value.split('\n') | join('\n ') }} config-{{parameter_name}} = {{ parameter_value.split('\n') | join('\n ') }}
{% else %} {% else %}
...@@ -49,7 +49,7 @@ config-{{parameter_name}} = {{ parameter_value }} ...@@ -49,7 +49,7 @@ config-{{parameter_name}} = {{ parameter_value }}
{% endif %} {% endif %}
{% endfor %} {% endfor %}
{% endif %} {% endif %}
{% for key, value in monitor_dict.iteritems() -%} {% for key, value in six.iteritems(monitor_dict) -%}
config-{{ key }} = {{ value }} config-{{ key }} = {{ value }}
{% endfor -%} {% endfor -%}
{% if sla_parameter_dict == {} -%} {% if sla_parameter_dict == {} -%}
...@@ -67,7 +67,7 @@ sla-mode = unique_by_network ...@@ -67,7 +67,7 @@ sla-mode = unique_by_network
{% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%} {% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
{% endif -%} {% endif -%}
{% endfor -%} {% endfor -%}
{% for key, value in sla_dict.iteritems() -%} {% for key, value in six.iteritems(sla_dict) -%}
sla-{{ key }} = {{ value }} sla-{{ key }} = {{ value }}
{% endfor -%} {% endfor -%}
{% endif -%} {% endif -%}
...@@ -101,7 +101,7 @@ config-number = {{id}} ...@@ -101,7 +101,7 @@ config-number = {{id}}
config-name = {{namebase}}{{id}} config-name = {{namebase}}{{id}}
config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key} config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id} config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
{% for key, value in monitor_dict.iteritems() -%} {% for key, value in six.iteritems(monitor_dict) -%}
config-{{ key }} = {{ value }} config-{{ key }} = {{ value }}
{% endfor -%} {% endfor -%}
{% if sla_parameter_dict == {} -%} {% if sla_parameter_dict == {} -%}
...@@ -119,7 +119,7 @@ sla-mode = unique_by_network ...@@ -119,7 +119,7 @@ sla-mode = unique_by_network
{% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%} {% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
{% endif -%} {% endif -%}
{% endfor -%} {% endfor -%}
{% for key, value in sla_dict.iteritems() -%} {% for key, value in six.iteritems(sla_dict) -%}
sla-{{ key }} = {{ value }} sla-{{ key }} = {{ value }}
{% endfor -%} {% endfor -%}
{% endif %} {% endif %}
...@@ -202,7 +202,7 @@ software-type = pull-backup ...@@ -202,7 +202,7 @@ software-type = pull-backup
name = PBS ({{namebase}} / {{id}}) name = PBS ({{namebase}} / {{id}})
config-ignore-known-hosts-file = ${slap-parameter:ignore-known-hosts-file} config-ignore-known-hosts-file = ${slap-parameter:ignore-known-hosts-file}
config-monitor-title = PBS ${slap-connection:computer-id}-{{namebase}}-{{id}} config-monitor-title = PBS ${slap-connection:computer-id}-{{namebase}}-{{id}}
{% for key, value in monitor_dict.iteritems() -%} {% for key, value in six.iteritems(monitor_dict) -%}
config-{{ key }} = {{ value }} config-{{ key }} = {{ value }}
{% endfor -%} {% endfor -%}
return = ssh-key notification-url feeds-url {{ monitor_return }} return = ssh-key notification-url feeds-url {{ monitor_return }}
...@@ -222,7 +222,7 @@ sla-mode = unique_by_network ...@@ -222,7 +222,7 @@ sla-mode = unique_by_network
{% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%} {% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
{% endif -%} {% endif -%}
{% endfor -%} {% endfor -%}
{% for key, value in sla_dict.iteritems() -%} {% for key, value in six.iteritems(sla_dict) -%}
sla-{{ key }} = {{ value }} sla-{{ key }} = {{ value }}
{% endfor %} {% endfor %}
{% endif %} {% endif %}
......
...@@ -8,3 +8,20 @@ context = ...@@ -8,3 +8,20 @@ context =
raw pbsready_template_path ${pbsready:output} raw pbsready_template_path ${pbsready:output}
raw bash_executable_location ${bash:location}/bin/bash raw bash_executable_location ${bash:location}/bin/bash
raw logrotate_executable_location ${logrotate:location}/usr/sbin/logrotate raw logrotate_executable_location ${logrotate:location}/usr/sbin/logrotate
[instance-frozen]
recipe = slapos.recipe.template:jinja2
template = ${instance-frozen:output}
rendered = $${buildout:directory}/template-frozen.cfg
extensions = jinja2.ext.do
context =
key slapparameter_dict slap-configuration:configuration
[template-pull-backup]
recipe = slapos.recipe.template:jinja2
template = ${template-pull-backup:output}
rendered = $${buildout:directory}/template-pull-backup.cfg
extensions = jinja2.ext.do
context =
key slapparameter_dict slap-configuration:configuration
key ipv6 slap-configuration:ipv6-random
#!{{ python_executable }} #!{{ python_executable }}
from __future__ import print_function
import os import os
import urllib2 try:
from urllib2 import HTTPError, urlopen
except ImportError:
from urllib.error import HTTPError
from urllib.request import urlopen
import sys import sys
input_feed_directory = '{{ input_feed_directory }}' input_feed_directory = '{{ input_feed_directory }}'
...@@ -12,12 +17,14 @@ feed_file_list = os.listdir(input_feed_directory) ...@@ -12,12 +17,14 @@ feed_file_list = os.listdir(input_feed_directory)
rss_ok = True rss_ok = True
for feed_file_name in feed_file_list: for feed_file_name in feed_file_list:
print "Getting %s" % feed_file_name print("Getting", feed_file_name)
url = base_url + feed_file_name url = base_url + feed_file_name
try: try:
feed = urllib2.urlopen(url) feed = urlopen(url)
body = feed.read() body = feed.read()
open(os.path.join(monitor_feed_directory, feed_file_name + '.rss'), 'w').write(body) with open(os.path.join(monitor_feed_directory, feed_file_name + '.rss'),
print "FEED is ok" 'wb') as f:
except urllib2.HTTPError as e: f.write(body)
print("FEED is ok")
except HTTPError as e:
sys.exit("%s is unvailable: %s" % (feed_file_name, e)) sys.exit("%s is unvailable: %s" % (feed_file_name, e))
...@@ -2,7 +2,11 @@ ...@@ -2,7 +2,11 @@
import csv import csv
import os import os
import sys import sys
import urllib2 try:
from urllib2 import HTTPError, urlopen
except ImportError:
from urllib.error import HTTPError
from urllib.request import urlopen
csv.field_size_limit(sys.maxsize) csv.field_size_limit(sys.maxsize)
...@@ -15,9 +19,9 @@ for feed_file_name in feed_file_list: ...@@ -15,9 +19,9 @@ for feed_file_name in feed_file_list:
url = base_url + feed_file_name url = base_url + feed_file_name
# Try feed consistency # Try feed consistency
try: try:
feed = urllib2.urlopen(url) feed = urlopen(url)
body = feed.read() body = feed.read()
except urllib2.HTTPError as e: except HTTPError as e:
sys.exit("%s is unavailable: %s" % (feed_file_name, e)) sys.exit("%s is unavailable: %s" % (feed_file_name, e))
with open(os.path.join(notifier_feed_directory, feed_file_name)) as feed_file: with open(os.path.join(notifier_feed_directory, feed_file_name)) as feed_file:
reader = csv.reader(feed_file) reader = csv.reader(feed_file)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment