Commit 4f842a7c authored by Rafael Monnerat's avatar Rafael Monnerat

Update Release Candidate

parents db5ee6df 5bb74740
[buildout]
extends =
../sqlite3/buildout.cfg
../openssl/buildout.cfg
../libevent/buildout.cfg
parts = coturn
[coturn]
recipe = slapos.recipe.cmmi
url = https://github.com/coturn/coturn/archive/4.5.1.1.tar.gz
md5sum = ef20628b026d666be24df056f20a86ea
environment =
PATH=${sqlite3:location}/bin:%(PATH)s
CPPFLAGS=-I${openssl:location}/include -I${sqlite3:location}/include -I${libevent2:location}/include
CFLAGS=-I${libevent2:location}/include
LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${openssl:location}/lib -L${sqlite3:location}/lib -L${libevent2:location}/lib -Wl,-rpath -Wl,${libevent2:location}/lib
...@@ -14,7 +14,6 @@ recipe = slapos.recipe.cmmi ...@@ -14,7 +14,6 @@ recipe = slapos.recipe.cmmi
shared = true shared = true
url = https://github.com/downloads/etolabo/kumofs/kumofs-0.4.13.tar.gz url = https://github.com/downloads/etolabo/kumofs/kumofs-0.4.13.tar.gz
md5sum = 46148e9536222d0ad2ef36777c55714d md5sum = 46148e9536222d0ad2ef36777c55714d
pre-configure-hook = ${:_profile_base_location_}/kumo-hooks.py#958a595a02de75624728f8d65e39d800:pre_configure_hook
patches = patches =
${:_profile_base_location_}/kumofs-0.4.13_ipv6support_multiiplistenfix.patch#53af9f1f1375940841c589a6cbe11425 ${:_profile_base_location_}/kumofs-0.4.13_ipv6support_multiiplistenfix.patch#53af9f1f1375940841c589a6cbe11425
${:_profile_base_location_}/kumofs-0.4.13_fix_gcc-4.9_ftbfs.patch#c09e04c620ce11c3fdd4afc3459cd355 ${:_profile_base_location_}/kumofs-0.4.13_fix_gcc-4.9_ftbfs.patch#c09e04c620ce11c3fdd4afc3459cd355
......
import os
import sys
import traceback
from shutil import copy
from subprocess import Popen, PIPE
CONFIGURE_PATH = os.path.join('configure')
CONFIGURE_BACKUP_PATH = CONFIGURE_PATH + '_disabled'
# Fake configure, generating a fake Makefile which will create a marker file
# instead of actually installing anything.
# This is needed (?) to fetch --prefix from configure parameters, so we know
# where to tell Makefile to put the dummy file.
FAKE_CONFIGURE = '''#!%(python)s -S
import os
import sys
print 'Configuration is disabled on this host because %%s'
print 'Original configure file available at %(backup)s'
prefix = None
next = False
for arg in sys.argv:
if next:
prefix = arg
break
if arg.startswith('--prefix'):
if arg.startswith('--prefix='):
_, prefix = arg.split('=', 1)
break
next = True
if prefix is None:
raise '--prefix parameter not found'
# Generate Makefile with proper prefix
open('Makefile', 'w').write("""all:
\techo 'make disabled, see configure'
install:
\ttouch %%%%s""" %%%% (
os.path.join(prefix, 'BUILD_DISABLED_BY_BUILDOUT'),
))
sys.exit(0)
''' % {
'backup': CONFIGURE_BACKUP_PATH,
'python': sys.executable,
}
def pre_configure_hook(options, buildout):
gcc_executable = os.getenv('CC', 'gcc')
try:
gcc = Popen([gcc_executable, '-v'], stdout=PIPE, stderr=PIPE,
close_fds=True)
except OSError, (errno, _):
if errno == 2:
# No gcc installed, nothing to check
pass
else:
print 'Unexpected failure trying to detect gcc version'
traceback.print_exc()
else:
gcc.wait()
# Considered innocent until proven guilty.
error = None
for line in '\n'.join((gcc.stdout.read(), gcc.stderr.read())).splitlines():
if line.startswith('gcc version'):
if '4.1.1' in line and 'prerelease' in line:
# There is a bug in 4.1.1 prerelease (ie, as of mandriva
# 2007.0) g++ preventing kumo compilation from succeeding.
error = 'broken GCC version: %s' % (line, )
break
else:
print >>sys.stderr, 'GCC version could not be detected, ' \
'building anyway'
if error is not None:
print 'Disabling build, with reason:', error
# Copy to preserver permission
copy(CONFIGURE_PATH, CONFIGURE_BACKUP_PATH)
open(CONFIGURE_PATH, 'w').write(FAKE_CONFIGURE % (error, ))
[buildout] [buildout]
parts = libevent parts = libevent
extends =
../openssl/buildout.cfg
[libevent] [libevent]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
...@@ -7,3 +9,11 @@ url = http://www.monkey.org/~provos/libevent-1.4.13-stable.tar.gz ...@@ -7,3 +9,11 @@ url = http://www.monkey.org/~provos/libevent-1.4.13-stable.tar.gz
md5sum = 0b3ea18c634072d12b3c1ee734263664 md5sum = 0b3ea18c634072d12b3c1ee734263664
configure-options = configure-options =
--disable-static --disable-static
[libevent2]
<= libevent
url = https://github.com/libevent/libevent/releases/download/release-2.1.10-stable/libevent-2.1.10-stable.tar.gz
md5sum = 999caf86f52943af2363bc8077f00167
environment =
CPPFLAGS=-I${openssl:location}/include
LDFLAGS=-L${openssl:location}/lib
\ No newline at end of file
...@@ -93,7 +93,7 @@ make-binary= ...@@ -93,7 +93,7 @@ make-binary=
${:extra-env} PERL5LIB="${:inc}:${:install-inc}:${:perl-PERL5LIB}" make ${:extra-env} PERL5LIB="${:inc}:${:install-inc}:${:perl-PERL5LIB}" make
# this post-make-hook is same for all users of the macro. # this post-make-hook is same for all users of the macro.
post-make-hook = ${:_profile_base_location_}/../../component/perl/perl-CPAN-package-create-wrapper.py#d012f7ba3300b14b2c6b173351d410be:post_make_hook post-make-hook = ${:_profile_base_location_}/../../component/perl/perl-CPAN-package-create-wrapper.py#f28c45a0f473ae050ca3ebaed5c39a4e:post_make_hook
perl_location = ${perl:location} perl_location = ${perl:location}
......
...@@ -17,7 +17,7 @@ def post_make_hook(options, buildout, environmet): ...@@ -17,7 +17,7 @@ def post_make_hook(options, buildout, environmet):
export PERL5LIB="{site_perl}:$PERL5LIB" export PERL5LIB="{site_perl}:$PERL5LIB"
exec {perl_location}/bin/perl "$@" exec {perl_location}/bin/perl "$@"
'''.format(**locals())) '''.format(**locals()))
os.chmod(perl_wrapper_path, 0755) os.chmod(perl_wrapper_path, 0o755)
# create a wrapper for each scripts installed in perl-bin # create a wrapper for each scripts installed in perl-bin
for script_path in glob.glob(os.path.join(prefix, 'perl-bin', '*')): for script_path in glob.glob(os.path.join(prefix, 'perl-bin', '*')):
...@@ -28,4 +28,4 @@ exec {perl_location}/bin/perl "$@" ...@@ -28,4 +28,4 @@ exec {perl_location}/bin/perl "$@"
export PERL5LIB="{site_perl}:$PERL5LIB" export PERL5LIB="{site_perl}:$PERL5LIB"
exec {perl_location}/bin/perl {script_path} "$@" exec {perl_location}/bin/perl {script_path} "$@"
'''.format(**locals())) '''.format(**locals()))
os.chmod(wrapper_path, 0755) os.chmod(wrapper_path, 0o755)
...@@ -2,7 +2,6 @@ ...@@ -2,7 +2,6 @@
extends = extends =
../fontconfig/buildout.cfg ../fontconfig/buildout.cfg
../libexpat/buildout.cfg ../libexpat/buildout.cfg
../dash/buildout.cfg
parts = parts =
phantomjs phantomjs
...@@ -19,17 +18,19 @@ x86 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-i686 ...@@ -19,17 +18,19 @@ x86 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-i686
x86-64 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-x86_64.tar.bz2 f278996c3edd0e8d8ec4893807f27d71 x86-64 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-x86_64.tar.bz2 f278996c3edd0e8d8ec4893807f27d71
script = script =
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ') if not self.options.get('url'):
extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum'))) self.options['url'], self.options['md5sum'] = \
self.options[guessPlatform()].split(' ')
extract_dir = self.extract(self.download(self.options['url'],
self.options.get('md5sum')))
workdir = guessworkdir(extract_dir) workdir = guessworkdir(extract_dir)
self.copyTree(workdir, "%(location)s") self.copyTree(workdir, "%(location)s")
wrapper_location = os.path.join("%(location)s", "phantomjs-slapos") wrapper_location = os.path.join("%(location)s", "phantomjs-slapos")
wrapper = open(wrapper_location, 'w') with open(wrapper_location, 'w') as wrapper:
wrapper.write("""#!${dash:location}/bin/dash wrapper.write("""#!/bin/sh
cd %(location)s cd %(location)s
export LD_LIBRARY_PATH=%(location)s:${freetype:location}/lib/:${fontconfig:location}/lib/:${libexpat:location}/lib export LD_LIBRARY_PATH=%(location)s:${freetype:location}/lib/:${fontconfig:location}/lib/:${libexpat:location}/lib
export PATH=${fontconfig:location}/bin:$PATH export PATH=${fontconfig:location}/bin:$PATH
exec %(location)s/bin/phantomjs $*""") exec %(location)s/bin/phantomjs "$@"
wrapper.flush() """)
wrapper.close() os.chmod(wrapper_location, 0o755)
os.chmod(wrapper_location, 0755)
...@@ -24,6 +24,8 @@ ...@@ -24,6 +24,8 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
from __future__ import print_function
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
import zc.buildout import zc.buildout
import sys import sys
...@@ -101,17 +103,17 @@ class Recipe(GenericBaseRecipe): ...@@ -101,17 +103,17 @@ class Recipe(GenericBaseRecipe):
# TODO factor # TODO factor
if delete != []: if delete != []:
print "Creating lampconfigure with 'delete' arguments" print("Creating lampconfigure with 'delete' arguments")
command = argument + delete command = argument + delete
if rename != []: if rename != []:
for parameters in rename: for parameters in rename:
print "Creating lampconfigure with 'rename' arguments" print("Creating lampconfigure with 'rename' arguments")
command = argument + rename command = argument + rename
if chmod != []: if chmod != []:
print "Creating lampconfigure with 'chmod' arguments" print("Creating lampconfigure with 'chmod' arguments")
command = argument + chmod command = argument + chmod
if data != []: if data != []:
print "Creating lampconfigure with 'run' arguments" print("Creating lampconfigure with 'run' arguments")
command = argument + data command = argument + data
......
...@@ -94,7 +94,7 @@ class Recipe(GenericBaseRecipe): ...@@ -94,7 +94,7 @@ class Recipe(GenericBaseRecipe):
dict(ip_address=self.options['ipv6'].strip(), dict(ip_address=self.options['ipv6'].strip(),
project=project, project=project,
middleware=type))) middleware=type)))
os.chmod(config_info_file, 0744) os.chmod(config_info_file, 0o744)
path_list.append(config_info) path_list.append(config_info)
update = install update = install
......
...@@ -26,11 +26,11 @@ ...@@ -26,11 +26,11 @@
############################################################################## ##############################################################################
import os import os
import hashlib import hashlib
import ConfigParser from six.moves import configparser
import tempfile import tempfile
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
from certificate_authority import popenCommunicate from .certificate_authority import popenCommunicate
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
...@@ -119,7 +119,7 @@ class Request(Recipe): ...@@ -119,7 +119,7 @@ class Request(Recipe):
open(certificate, 'w').write(cert_content) open(certificate, 'w').write(cert_content)
request_needed = False request_needed = False
else: else:
parser = ConfigParser.RawConfigParser() parser = configparser.RawConfigParser()
parser.add_section('certificate') parser.add_section('certificate')
parser.set('certificate', 'name', name) parser.set('certificate', 'name', name)
parser.set('certificate', 'key_file', key) parser.set('certificate', 'key_file', key)
......
from __future__ import print_function
import os import os
import subprocess import subprocess
import time import time
import ConfigParser from six.moves import configparser
import uuid import uuid
...@@ -95,12 +97,12 @@ class CertificateAuthority: ...@@ -95,12 +97,12 @@ class CertificateAuthority:
def checkRequestDir(self): def checkRequestDir(self):
for request_file in os.listdir(self.request_dir): for request_file in os.listdir(self.request_dir):
parser = ConfigParser.RawConfigParser() parser = configparser.RawConfigParser()
parser.readfp(open(os.path.join(self.request_dir, request_file), 'r')) parser.readfp(open(os.path.join(self.request_dir, request_file), 'r'))
if self._checkCertificate(parser.get('certificate', 'name'), if self._checkCertificate(parser.get('certificate', 'name'),
parser.get('certificate', 'key_file'), parser.get('certificate', parser.get('certificate', 'key_file'), parser.get('certificate',
'certificate_file')): 'certificate_file')):
print 'Created certificate %r' % parser.get('certificate', 'name') print('Created certificate %r' % parser.get('certificate', 'name'))
def runCertificateAuthority(*args): def runCertificateAuthority(*args):
ca = CertificateAuthority(*args) ca = CertificateAuthority(*args)
......
...@@ -29,7 +29,6 @@ import os ...@@ -29,7 +29,6 @@ import os
import subprocess import subprocess
import zc.buildout import zc.buildout
import filecmp import filecmp
import urlparse
import shutil import shutil
import re import re
import json import json
...@@ -130,9 +129,9 @@ class Recipe(GenericBaseRecipe): ...@@ -130,9 +129,9 @@ class Recipe(GenericBaseRecipe):
#create condor binary launcher for slapos #create condor binary launcher for slapos
if not os.path.exists(self.wrapper_bin): if not os.path.exists(self.wrapper_bin):
os.makedirs(self.wrapper_bin, int('0744', 8)) os.makedirs(self.wrapper_bin, int('0o744', 8))
if not os.path.exists(self.wrapper_sbin): if not os.path.exists(self.wrapper_sbin):
os.makedirs(self.wrapper_sbin, int('0744', 8)) os.makedirs(self.wrapper_sbin, int('0o744', 8))
#generate script for each file in prefix/bin #generate script for each file in prefix/bin
for binary in os.listdir(self.prefix+'/bin'): for binary in os.listdir(self.prefix+'/bin'):
wrapper_location = os.path.join(self.wrapper_bin, binary) wrapper_location = os.path.join(self.wrapper_bin, binary)
...@@ -153,7 +152,7 @@ class Recipe(GenericBaseRecipe): ...@@ -153,7 +152,7 @@ class Recipe(GenericBaseRecipe):
wrapper.write(content) wrapper.write(content)
wrapper.close() wrapper.close()
path_list.append(wrapper_location) path_list.append(wrapper_location)
os.chmod(wrapper_location, 0744) os.chmod(wrapper_location, 0o744)
#generate script for each file in prefix/sbin #generate script for each file in prefix/sbin
for binary in os.listdir(self.prefix+'/sbin'): for binary in os.listdir(self.prefix+'/sbin'):
...@@ -175,7 +174,7 @@ class Recipe(GenericBaseRecipe): ...@@ -175,7 +174,7 @@ class Recipe(GenericBaseRecipe):
wrapper.write(content) wrapper.write(content)
wrapper.close() wrapper.close()
path_list.append(wrapper_location) path_list.append(wrapper_location)
os.chmod(wrapper_location, 0744) os.chmod(wrapper_location, 0o744)
#generate script for start condor #generate script for start condor
wrapper = self.createPythonScript( wrapper = self.createPythonScript(
...@@ -228,7 +227,7 @@ class AppSubmit(GenericBaseRecipe): ...@@ -228,7 +227,7 @@ class AppSubmit(GenericBaseRecipe):
for file in file_list: for file in file_list:
if file and (file.startswith('http') or file.startswith('ftp')): if file and (file.startswith('http') or file.startswith('ftp')):
file_list[file] = self.download(file_list[file]) file_list[file] = self.download(file_list[file])
os.chmod(file_list[file], 0600) os.chmod(file_list[file], 0o600)
else: else:
app_list[app]['files'] = {} app_list[app]['files'] = {}
...@@ -236,11 +235,11 @@ class AppSubmit(GenericBaseRecipe): ...@@ -236,11 +235,11 @@ class AppSubmit(GenericBaseRecipe):
if executable and (executable.startswith('http') or executable.startswith('ftp')): if executable and (executable.startswith('http') or executable.startswith('ftp')):
app_list[app]['executable'] = self.download(executable, app_list[app]['executable'] = self.download(executable,
app_list[app]['executable-name']) app_list[app]['executable-name'])
os.chmod(app_list[app]['executable-name'], 0700) os.chmod(app_list[app]['executable-name'], 0o700)
submit_file = app_list[app].get('description-file', '') submit_file = app_list[app].get('description-file', '')
if submit_file and (submit_file.startswith('http') or submit_file.startswith('ftp')): if submit_file and (submit_file.startswith('http') or submit_file.startswith('ftp')):
app_list[app]['description-file'] = self.download(submit_file, 'submit') app_list[app]['description-file'] = self.download(submit_file, 'submit')
os.chmod(app_list[app]['description-file'], 0600) os.chmod(app_list[app]['description-file'], 0o600)
return app_list return app_list
......
...@@ -25,7 +25,7 @@ ...@@ -25,7 +25,7 @@
# #
############################################################################## ##############################################################################
import subprocess import subprocess
import httplib from six.moves import http_client as httplib
import base64 import base64
import os import os
import shutil import shutil
......
...@@ -29,6 +29,8 @@ import os ...@@ -29,6 +29,8 @@ import os
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
from zc.buildout import UserError from zc.buildout import UserError
from six.moves import map
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
def install(self): def install(self):
...@@ -124,7 +126,7 @@ def systemd_to_cron(spec): ...@@ -124,7 +126,7 @@ def systemd_to_cron(spec):
x = spec[i] x = spec[i]
if x != '*': if x != '*':
for x in x.split(','): for x in x.split(','):
x = map(int, x.split('/', 1)) x = list(map(int, x.split('/', 1)))
a = x[0] - y a = x[0] - y
if 0 <= a < z: if 0 <= a < z:
if len(x) == 1: if len(x) == 1:
......
...@@ -26,7 +26,7 @@ ...@@ -26,7 +26,7 @@
############################################################################## ##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
import ConfigParser from six.moves import configparser
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
""" """
...@@ -34,7 +34,7 @@ class Recipe(GenericBaseRecipe): ...@@ -34,7 +34,7 @@ class Recipe(GenericBaseRecipe):
""" """
def install(self): def install(self):
promise_parser = ConfigParser.RawConfigParser() promise_parser = configparser.RawConfigParser()
for section_name, option_id_list in ( for section_name, option_id_list in (
('portal_templates', ( ('portal_templates', (
('repository', 'bt5-repository-url'), ('repository', 'bt5-repository-url'),
......
...@@ -24,10 +24,10 @@ ...@@ -24,10 +24,10 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
import ConfigParser from six.moves import configparser
import io
import json import json
import os import os
import StringIO
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
...@@ -40,13 +40,13 @@ class Recipe(GenericBaseRecipe): ...@@ -40,13 +40,13 @@ class Recipe(GenericBaseRecipe):
CONFIG['PATH'] = os.environ['PATH'] CONFIG['PATH'] = os.environ['PATH']
if self.options['instance-dict']: if self.options['instance-dict']:
config_instance_dict = ConfigParser.ConfigParser() config_instance_dict = configparser.ConfigParser()
config_instance_dict.add_section('instance_dict') config_instance_dict.add_section('instance_dict')
instance_dict = json.loads(self.options['instance-dict']) instance_dict = json.loads(self.options['instance-dict'])
for k ,v in instance_dict.iteritems(): for k ,v in instance_dict.iteritems():
config_instance_dict.set('instance_dict', k, v) config_instance_dict.set('instance_dict', k, v)
value = StringIO.StringIO() value = io.StringIO()
config_instance_dict.write(value) config_instance_dict.write(value)
CONFIG['instance_dict'] = value.getvalue() CONFIG['instance_dict'] = value.getvalue()
......
...@@ -25,7 +25,7 @@ ...@@ -25,7 +25,7 @@
# #
############################################################################## ##############################################################################
import ConfigParser from six.moves import configparser
import os import os
import netaddr import netaddr
import socket import socket
...@@ -48,7 +48,7 @@ class Recipe(object): ...@@ -48,7 +48,7 @@ class Recipe(object):
# If this check isn't done, a new port would be picked for every upgrade # If this check isn't done, a new port would be picked for every upgrade
# of the software release # of the software release
try: try:
parser = ConfigParser.RawConfigParser() parser = configparser.RawConfigParser()
if os.path.exists(buildout['buildout']['installed']): if os.path.exists(buildout['buildout']['installed']):
with open(buildout['buildout']['installed']) as config_file: with open(buildout['buildout']['installed']) as config_file:
parser.readfp(config_file) parser.readfp(config_file)
...@@ -59,7 +59,7 @@ class Recipe(object): ...@@ -59,7 +59,7 @@ class Recipe(object):
if port != '0': if port != '0':
self.options['port'] = port self.options['port'] = port
return return
except (IOError, ConfigParser.NoSectionError, ConfigParser.NoOptionError): except (IOError, configparser.NoSectionError, configparser.NoOptionError):
pass pass
# Otherwise, let's find one # Otherwise, let's find one
......
...@@ -24,9 +24,11 @@ ...@@ -24,9 +24,11 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
from functools import cmp_to_key
import zc.buildout import zc.buildout
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
@cmp_to_key
def compareMimetypeEntryPair(a, b): def compareMimetypeEntryPair(a, b):
""" """
Like comparing strings, but here the star `*` is stronger than any other Like comparing strings, but here the star `*` is stronger than any other
...@@ -115,7 +117,7 @@ class Recipe(GenericBaseRecipe): ...@@ -115,7 +117,7 @@ class Recipe(GenericBaseRecipe):
if l and not l.isspace() if l and not l.isspace()
] ]
mimetype_entry_list.extend(default_mimetype_entry_list) mimetype_entry_list.extend(default_mimetype_entry_list)
mimetype_entry_list.sort(compareMimetypeEntryPair) mimetype_entry_list.sort(key=compareMimetypeEntryPair)
conversion_server_dict['MIMETYPE_ENTRY_LIST'] = \ conversion_server_dict['MIMETYPE_ENTRY_LIST'] = \
"\n".join([" " + l for l in mimetype_entry_list]) "\n".join([" " + l for l in mimetype_entry_list])
config_file = self.createFile(self.options['configuration-file'], config_file = self.createFile(self.options['configuration-file'],
......
...@@ -31,7 +31,7 @@ import pkg_resources ...@@ -31,7 +31,7 @@ import pkg_resources
import zc.buildout import zc.buildout
import sys import sys
import zc.recipe.egg import zc.recipe.egg
import urlparse from six.moves.urllib.parse import urlparse
# Warning : this recipe is deprecated and has been replaced by apachephp. # Warning : this recipe is deprecated and has been replaced by apachephp.
...@@ -264,7 +264,7 @@ class Request(BaseRecipe): ...@@ -264,7 +264,7 @@ class Request(BaseRecipe):
mysql = self.request(self.options['mariadb-software-url'], mysql = self.request(self.options['mariadb-software-url'],
software_type, 'MariaDB Server', partition_parameter_kw=parameters software_type, 'MariaDB Server', partition_parameter_kw=parameters
).getConnectionParameter('url') ).getConnectionParameter('url')
mysql_parsed = urlparse.urlparse(mysql) mysql_parsed = urlparse(mysql)
mysql_host, mysql_port = mysql_parsed.hostname, mysql_parsed.port mysql_host, mysql_port = mysql_parsed.hostname, mysql_parsed.port
if mysql_parsed.scheme == 'mysqls': # Listen over stunnel if mysql_parsed.scheme == 'mysqls': # Listen over stunnel
......
...@@ -57,7 +57,7 @@ class Recipe(BaseSlapRecipe): ...@@ -57,7 +57,7 @@ class Recipe(BaseSlapRecipe):
try: try:
self.slave_partition_configuration_dict_list.append( self.slave_partition_configuration_dict_list.append(
self._installSlavePartition(slave_partition)) self._installSlavePartition(slave_partition))
except SlavePartitionError, e: except SlavePartitionError as e:
self.logger.warning('Slave Parttion %r not installed, issue: %r'%( self.logger.warning('Slave Parttion %r not installed, issue: %r'%(
slave_partition.getId(), e)) slave_partition.getId(), e))
# Installs wrappers # Installs wrappers
......
...@@ -34,13 +34,13 @@ import stat ...@@ -34,13 +34,13 @@ import stat
import netaddr import netaddr
import time import time
import re import re
import urlparse from six.moves.urllib.parse import urlunparse
import json import json
# Use to do from slapos.recipe.librecipe import GenericBaseRecipe # Use to do from slapos.recipe.librecipe import GenericBaseRecipe
from generic import GenericBaseRecipe from .generic import GenericBaseRecipe
from genericslap import GenericSlapRecipe from .genericslap import GenericSlapRecipe
from filehash import filehash from .filehash import filehash
# Utility functions to (de)serialise live python objects in order to send them # Utility functions to (de)serialise live python objects in order to send them
# to master. # to master.
...@@ -324,7 +324,7 @@ class BaseSlapRecipe: ...@@ -324,7 +324,7 @@ class BaseSlapRecipe:
if port is not None: if port is not None:
netloc += ':%s' % port netloc += ':%s' % port
url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment)) url = urlunparse((scheme, netloc, path, params, query, fragment))
return url return url
from __future__ import print_function
import sys import sys
import os import os
import signal import signal
...@@ -5,6 +7,8 @@ import subprocess ...@@ -5,6 +7,8 @@ import subprocess
from collections import defaultdict from collections import defaultdict
from inotify_simple import INotify, flags from inotify_simple import INotify, flags
import six
def _wait_files_creation(file_list): def _wait_files_creation(file_list):
# Establish a list of directory and subfiles. # Establish a list of directory and subfiles.
# and test existence before watching, so that we don't miss an event. # and test existence before watching, so that we don't miss an event.
...@@ -14,7 +18,7 @@ def _wait_files_creation(file_list): ...@@ -14,7 +18,7 @@ def _wait_files_creation(file_list):
directories[dirname][filename] = os.path.lexists(f) directories[dirname][filename] = os.path.lexists(f)
def all_files_exists(): def all_files_exists():
return all(all(files.itervalues()) for files in directories.itervalues()) return all(all(six.itervalues(files)) for files in six.itervalues(directories))
with INotify() as inotify: with INotify() as inotify:
watchdescriptors = {inotify.add_watch(dirname, watchdescriptors = {inotify.add_watch(dirname,
...@@ -101,7 +105,7 @@ def generic_exec(args, extra_environ=None, wait_list=None, ...@@ -101,7 +105,7 @@ def generic_exec(args, extra_environ=None, wait_list=None,
child_pg = None child_pg = None
def sig_handler(sig, frame): def sig_handler(sig, frame):
print 'Received signal %r, killing children and exiting' % sig print('Received signal %r, killing children and exiting' % sig)
if child_pg is not None: if child_pg is not None:
os.killpg(child_pg, signal.SIGHUP) os.killpg(child_pg, signal.SIGHUP)
os.killpg(child_pg, signal.SIGTERM) os.killpg(child_pg, signal.SIGTERM)
...@@ -116,7 +120,7 @@ def execute_with_signal_translation(args): ...@@ -116,7 +120,7 @@ def execute_with_signal_translation(args):
child = subprocess.Popen(args, close_fds=True, preexec_fn=os.setsid) child = subprocess.Popen(args, close_fds=True, preexec_fn=os.setsid)
child_pg = child.pid child_pg = child.pid
try: try:
print 'Process %r started' % (args, ) print('Process %r started' % (args, ))
signal.pause() signal.pause()
finally: finally:
os.killpg(child_pg, signal.SIGHUP) os.killpg(child_pg, signal.SIGHUP)
......
...@@ -24,6 +24,7 @@ ...@@ -24,6 +24,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
from __future__ import print_function
import hashlib import hashlib
import shutil import shutil
import os import os
...@@ -91,6 +92,6 @@ if __name__ == '__main__': ...@@ -91,6 +92,6 @@ if __name__ == '__main__':
if len(sys.argv) == 1: if len(sys.argv) == 1:
raise ValueError("Not enough command line arguments") raise ValueError("Not enough command line arguments")
if len(sys.argv) == 2: if len(sys.argv) == 2:
print sys.argv[1], '-', pathhash(sys.argv[1]) print(sys.argv[1], '-', pathhash(sys.argv[1]))
else: else:
print sys.argv[2], '-', pathhash(sys.argv[2], sys.argv[1]) print(sys.argv[2], '-', pathhash(sys.argv[2], sys.argv[1]))
...@@ -35,8 +35,12 @@ import inspect ...@@ -35,8 +35,12 @@ import inspect
import re import re
import shutil import shutil
import stat import stat
import urllib from six.moves.urllib.parse import quote
import urlparse import itertools
import six
from six.moves import map
from six.moves.urllib.parse import urlunparse
import pkg_resources import pkg_resources
import zc.buildout import zc.buildout
...@@ -90,7 +94,7 @@ class GenericBaseRecipe(object): ...@@ -90,7 +94,7 @@ class GenericBaseRecipe(object):
"""Options Hook method. This method can be overriden in child classes""" """Options Hook method. This method can be overriden in child classes"""
return return
def createFile(self, name, content, mode=0600): def createFile(self, name, content, mode=0o600):
"""Create a file with content """Create a file with content
The parent directory should exists, else it would raise IOError""" The parent directory should exists, else it would raise IOError"""
...@@ -117,7 +121,7 @@ class GenericBaseRecipe(object): ...@@ -117,7 +121,7 @@ class GenericBaseRecipe(object):
f.write(content) f.write(content)
return os.path.abspath(name) return os.path.abspath(name)
def createExecutable(self, name, content, mode=0700): def createExecutable(self, name, content, mode=0o700):
return self.createFile(name, content, mode) return self.createFile(name, content, mode)
def addLineToFile(self, filepath, line, encoding='utf8'): def addLineToFile(self, filepath, line, encoding='utf8'):
...@@ -148,9 +152,9 @@ class GenericBaseRecipe(object): ...@@ -148,9 +152,9 @@ class GenericBaseRecipe(object):
module, function = function module, function = function
path, filename = os.path.split(os.path.abspath(name)) path, filename = os.path.split(os.path.abspath(name))
assert not isinstance(args, (basestring, dict)), args assert not isinstance(args, (six.string_types, dict)), args
args = map(repr, args) args = itertools.chain(map(repr, args),
args += map('%s=%r'.__mod__, kw.iteritems()) map('%s=%r'.__mod__, six.iteritems(kw)))
return zc.buildout.easy_install.scripts( return zc.buildout.easy_install.scripts(
[(filename, module, function)], self._ws, sys.executable, [(filename, module, function)], self._ws, sys.executable,
...@@ -173,12 +177,12 @@ class GenericBaseRecipe(object): ...@@ -173,12 +177,12 @@ class GenericBaseRecipe(object):
lines = ['#!/bin/sh'] lines = ['#!/bin/sh']
if env: if env:
for k, v in sorted(env.iteritems()): for k, v in sorted(six.iteritems(env)):
lines.append('export %s=%s' % (k, shlex.quote(v))) lines.append('export %s=%s' % (k, shlex.quote(v)))
lines.append('exec') lines.append('exec')
args = map(shlex.quote, args) args = list(map(shlex.quote, args))
args.append('"$@"') args.append('"$@"')
for arg in args: for arg in args:
if len(lines[-1]) < 40: if len(lines[-1]) < 40:
...@@ -188,9 +192,9 @@ class GenericBaseRecipe(object): ...@@ -188,9 +192,9 @@ class GenericBaseRecipe(object):
lines.append('\t' + arg) lines.append('\t' + arg)
lines.append('') lines.append('')
return self.createFile(path, '\n'.join(lines), 0700) return self.createFile(path, '\n'.join(lines), 0o700)
def createDirectory(self, parent, name, mode=0700): def createDirectory(self, parent, name, mode=0o700):
path = os.path.join(parent, name) path = os.path.join(parent, name)
if not os.path.exists(path): if not os.path.exists(path):
os.mkdir(path, mode) os.mkdir(path, mode)
...@@ -240,9 +244,9 @@ class GenericBaseRecipe(object): ...@@ -240,9 +244,9 @@ class GenericBaseRecipe(object):
netloc = '' netloc = ''
if auth is not None: if auth is not None:
auth = tuple(auth) auth = tuple(auth)
netloc = urllib.quote(str(auth[0])) # Login netloc = quote(str(auth[0])) # Login
if len(auth) > 1: if len(auth) > 1:
netloc += ':%s' % urllib.quote(auth[1]) # Password netloc += ':%s' % quote(auth[1]) # Password
netloc += '@' netloc += '@'
# host is an ipv6 address whithout brackets # host is an ipv6 address whithout brackets
...@@ -254,7 +258,7 @@ class GenericBaseRecipe(object): ...@@ -254,7 +258,7 @@ class GenericBaseRecipe(object):
if port is not None: if port is not None:
netloc += ':%s' % port netloc += ':%s' % port
url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment)) url = urlunparse((scheme, netloc, path, params, query, fragment))
return url return url
......
...@@ -27,7 +27,7 @@ ...@@ -27,7 +27,7 @@
from slapos import slap from slapos import slap
import time import time
from generic import GenericBaseRecipe from .generic import GenericBaseRecipe
CONNECTION_CACHE = {} CONNECTION_CACHE = {}
......
...@@ -27,6 +27,7 @@ ...@@ -27,6 +27,7 @@
import os import os
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
import six
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
...@@ -36,7 +37,7 @@ class Recipe(GenericBaseRecipe): ...@@ -36,7 +37,7 @@ class Recipe(GenericBaseRecipe):
self.mode = int(self.directory.pop('mode', '0777'), 8) self.mode = int(self.directory.pop('mode', '0777'), 8)
def install(self): def install(self):
for path in sorted(self.directory.itervalues()): for path in sorted(six.itervalues(self.directory)):
if path and not os.path.isdir(path): if path and not os.path.isdir(path):
os.makedirs(path, self.mode) os.makedirs(path, self.mode)
# WARNING: This recipe is currently used to create directories that will # WARNING: This recipe is currently used to create directories that will
......
...@@ -25,6 +25,8 @@ ...@@ -25,6 +25,8 @@
# #
############################################################################## ##############################################################################
from __future__ import print_function
import sys import sys
import pkg_resources import pkg_resources
from logging import Formatter from logging import Formatter
...@@ -41,7 +43,7 @@ class NoSQLTestBed(BaseSlapRecipe): ...@@ -41,7 +43,7 @@ class NoSQLTestBed(BaseSlapRecipe):
testbed = plugin_class() testbed = plugin_class()
except: except:
print Formatter().formatException(sys.exc_info()) print(Formatter().formatException(sys.exc_info()))
return None return None
software_type = self.parameter_dict.get('slap_software_type', 'default') software_type = self.parameter_dict.get('slap_software_type', 'default')
......
...@@ -25,12 +25,14 @@ ...@@ -25,12 +25,14 @@
# #
############################################################################## ##############################################################################
from __future__ import print_function
import json import json
import os import os
import subprocess import subprocess
import sys import sys
import textwrap import textwrap
import urlparse from six.moves.urllib.parse import urlparse
from slapos.recipe.librecipe import GenericSlapRecipe from slapos.recipe.librecipe import GenericSlapRecipe
from slapos.recipe.dropbear import KnownHostsFile from slapos.recipe.dropbear import KnownHostsFile
...@@ -208,7 +210,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback): ...@@ -208,7 +210,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
# This behavior has been removed to accelerate deployment of the # This behavior has been removed to accelerate deployment of the
# Software Release. The buildout, instead of failing, can process # Software Release. The buildout, instead of failing, can process
# other sections, which will return parameters to the main instance faster # other sections, which will return parameters to the main instance faster
parsed_url = urlparse.urlparse(url) parsed_url = urlparse(url)
slave_type = entry['type'] slave_type = entry['type']
if not slave_type in ['pull', 'push']: if not slave_type in ['pull', 'push']:
...@@ -216,7 +218,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback): ...@@ -216,7 +218,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
slave_id = entry['notification-id'] slave_id = entry['notification-id']
print 'Processing PBS slave %s with type %s' % (slave_id, slave_type) print('Processing PBS slave %s with type %s' % (slave_id, slave_type))
path_list.append(self.createPythonScript( path_list.append(self.createPythonScript(
os.path.join(self.options['promises-directory'], "ssh-to-%s" % slave_id), os.path.join(self.options['promises-directory'], "ssh-to-%s" % slave_id),
......
...@@ -25,7 +25,7 @@ ...@@ -25,7 +25,7 @@
# #
############################################################################## ##############################################################################
import md5 import hashlib
import os import os
import subprocess import subprocess
import textwrap import textwrap
...@@ -195,7 +195,7 @@ class Recipe(GenericBaseRecipe): ...@@ -195,7 +195,7 @@ class Recipe(GenericBaseRecipe):
password = self.options['password'] password = self.options['password']
# encrypt the password to avoid storing in the logs # encrypt the password to avoid storing in the logs
enc_password = 'md5' + md5.md5(password+user).hexdigest() enc_password = 'md5' + hashlib.md5(password+user).hexdigest()
self.runPostgresCommand(cmd="""ALTER USER "%s" ENCRYPTED PASSWORD '%s'""" % (user, enc_password)) self.runPostgresCommand(cmd="""ALTER USER "%s" ENCRYPTED PASSWORD '%s'""" % (user, enc_password))
......
...@@ -24,9 +24,11 @@ ...@@ -24,9 +24,11 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
from __future__ import print_function
import zc.buildout import zc.buildout
from slapos.recipe.librecipe import wrap from slapos.recipe.librecipe import wrap
from slapos.recipe.librecipe import GenericSlapRecipe from slapos.recipe.librecipe import GenericSlapRecipe
import six
CONNECTION_PARAMETER_STRING = 'connection-' CONNECTION_PARAMETER_STRING = 'connection-'
...@@ -77,9 +79,9 @@ class PublishSection(GenericSlapRecipe): ...@@ -77,9 +79,9 @@ class PublishSection(GenericSlapRecipe):
for section in self.options['section-list'].strip().split(): for section in self.options['section-list'].strip().split():
section = section.strip() section = section.strip()
options = self.buildout[section].copy() options = self.buildout[section].copy()
for k, v in options.iteritems(): for k, v in six.iteritems(options):
if k.startswith(CONNECTION_PARAMETER_STRING): if k.startswith(CONNECTION_PARAMETER_STRING):
print k, v print(k, v)
publish_dict[k.lstrip(CONNECTION_PARAMETER_STRING)] = v publish_dict[k.lstrip(CONNECTION_PARAMETER_STRING)] = v
self.setConnectionDict(publish_dict) self.setConnectionDict(publish_dict)
return [] return []
......
...@@ -25,8 +25,9 @@ ...@@ -25,8 +25,9 @@
# #
############################################################################## ##############################################################################
from collections import defaultdict from collections import OrderedDict
from .librecipe import unwrap, wrap, GenericSlapRecipe from .librecipe import unwrap, wrap, GenericSlapRecipe
import six
def volatileOptions(options, volatile): def volatileOptions(options, volatile):
def copy(): def copy():
...@@ -65,6 +66,9 @@ class Recipe(GenericSlapRecipe): ...@@ -65,6 +66,9 @@ class Recipe(GenericSlapRecipe):
the published value 'foo' if it exists. If its __init__ modifies 'x', the new the published value 'foo' if it exists. If its __init__ modifies 'x', the new
value is published. To prevent [gen-foo] from being accessed too early, 'x' value is published. To prevent [gen-foo] from being accessed too early, 'x'
is then removed and the value can only be accessed with ${publish-early:foo}. is then removed and the value can only be accessed with ${publish-early:foo}.
Init sections are processed in the order of first appearance in the '-init'
section, so that a init section can access a value that is generated by a
previous one (above, [gen-bar] can access ${publish-early:foo}).
Generated values don't end up in the buildout installed file, which is good Generated values don't end up in the buildout installed file, which is good
if they're secret. Note however that buildout won't detect if values change if they're secret. Note however that buildout won't detect if values change
...@@ -75,13 +79,16 @@ class Recipe(GenericSlapRecipe): ...@@ -75,13 +79,16 @@ class Recipe(GenericSlapRecipe):
""" """
def __init__(self, buildout, name, options): def __init__(self, buildout, name, options):
GenericSlapRecipe.__init__(self, buildout, name, options) GenericSlapRecipe.__init__(self, buildout, name, options)
init = defaultdict(dict) init = OrderedDict()
for line in options['-init'].splitlines(): for line in options['-init'].splitlines():
if line: if line:
k, v = line.split() k, v = line.split()
if k not in options: if k not in options:
section, v = v.split(':') section, v = v.split(':')
try:
init[section][k] = v init[section][k] = v
except KeyError:
init[section] = {k: v}
if init: if init:
self.slap.initializeConnection(self.server_url, self.key_file, self.slap.initializeConnection(self.server_url, self.key_file,
self.cert_file) self.cert_file)
...@@ -109,9 +116,9 @@ class Recipe(GenericSlapRecipe): ...@@ -109,9 +116,9 @@ class Recipe(GenericSlapRecipe):
publish = False publish = False
publish_dict = {} publish_dict = {}
try: try:
for init_section, init in init.iteritems(): for init_section, init in six.iteritems(init):
override = {} override = {}
for k, v in init.iteritems(): for k, v in six.iteritems(init):
try: try:
override[v] = published_dict[k] override[v] = published_dict[k]
except KeyError: except KeyError:
...@@ -120,9 +127,9 @@ class Recipe(GenericSlapRecipe): ...@@ -120,9 +127,9 @@ class Recipe(GenericSlapRecipe):
init_section = buildout[init_section] init_section = buildout[init_section]
assert buildout.Options is Options assert buildout.Options is Options
new = {} new = {}
for k, v in init.iteritems(): for k, v in six.iteritems(init):
try: try:
publish_dict[k] = new[v] = init_section.pop(v) options[k] = publish_dict[k] = new[v] = init_section.pop(v)
except KeyError: except KeyError:
pass pass
if new != override: if new != override:
...@@ -138,6 +145,5 @@ class Recipe(GenericSlapRecipe): ...@@ -138,6 +145,5 @@ class Recipe(GenericSlapRecipe):
publish += publish_dict publish += publish_dict
publish_dict['-publish'] = ' '.join(publish) publish_dict['-publish'] = ' '.join(publish)
volatileOptions(options, list(publish_dict)) volatileOptions(options, list(publish_dict))
options.update(publish_dict)
install = update = lambda self: None install = update = lambda self: None
...@@ -172,7 +172,7 @@ class Password(object): ...@@ -172,7 +172,7 @@ class Password(object):
raise raise
fd = os.open(self.storage_path, fd = os.open(self.storage_path,
os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_TRUNC, 0600) os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_TRUNC, 0o600)
try: try:
os.write(fd, self.passwd) os.write(fd, self.passwd)
finally: finally:
......
...@@ -123,7 +123,7 @@ class Recipe(GenericBaseRecipe): ...@@ -123,7 +123,7 @@ class Recipe(GenericBaseRecipe):
token_dict[reference] = new_token token_dict[reference] = new_token
to_add_dict[reference] = new_token to_add_dict[reference] = new_token
for reference in token_dict.keys(): for reference in list(token_dict):
if not reference in reference_list: if not reference in reference_list:
# This slave instance is destroyed ? # This slave instance is destroyed ?
to_remove_dict[reference] = token_dict.pop(reference) to_remove_dict[reference] = token_dict.pop(reference)
......
...@@ -81,5 +81,5 @@ def promise(host, port, unixsocket): ...@@ -81,5 +81,5 @@ def promise(host, port, unixsocket):
r = Redis(host=host, port=port, unix_socket_path=unixsocket, db=0) r = Redis(host=host, port=port, unix_socket_path=unixsocket, db=0)
r.publish("Promise-Service","SlapOS Promise") r.publish("Promise-Service","SlapOS Promise")
r.connection_pool.disconnect() r.connection_pool.disconnect()
except Exception, e: except Exception as e:
sys.exit(e) sys.exit(e)
...@@ -33,6 +33,8 @@ from slapos.slap import SoftwareProductCollection ...@@ -33,6 +33,8 @@ from slapos.slap import SoftwareProductCollection
import slapos.recipe.librecipe.generic as librecipe import slapos.recipe.librecipe.generic as librecipe
import traceback import traceback
import six
SOFTWARE_PRODUCT_NAMESPACE = "product." SOFTWARE_PRODUCT_NAMESPACE = "product."
DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance' DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance'
...@@ -110,10 +112,10 @@ class Recipe(object): ...@@ -110,10 +112,10 @@ class Recipe(object):
raise UserError("'config' & 'sla' options are obsolete." raise UserError("'config' & 'sla' options are obsolete."
" Clean up your software release.") " Clean up your software release.")
filter_kw = {k[4:]: v filter_kw = {k[4:]: v
for k, v in options.iteritems() for k, v in six.iteritems(options)
if k.startswith('sla-') and v} if k.startswith('sla-') and v}
partition_parameter_kw = self._filterForStorage({k[7:]: v partition_parameter_kw = self._filterForStorage({k[7:]: v
for k, v in options.iteritems() for k, v in six.iteritems(options)
if k.startswith('config-')}) if k.startswith('config-')})
slave = options.get('slave', 'false').lower() in \ slave = options.get('slave', 'false').lower() in \
librecipe.GenericBaseRecipe.TRUE_VALUES librecipe.GenericBaseRecipe.TRUE_VALUES
...@@ -196,7 +198,7 @@ class Recipe(object): ...@@ -196,7 +198,7 @@ class Recipe(object):
except KeyError: except KeyError:
if self.failed is None: if self.failed is None:
self.failed = param self.failed = param
if isinstance(value, unicode): if six.PY2 and isinstance(value, unicode):
value = value.encode('UTF-8') value = value.encode('UTF-8')
options['connection-%s' % param] = value options['connection-%s' % param] = value
...@@ -310,12 +312,12 @@ class RequestEdge(Recipe): ...@@ -310,12 +312,12 @@ class RequestEdge(Recipe):
self.request_dict[country] = Recipe(buildout, name, local_options) self.request_dict[country] = Recipe(buildout, name, local_options)
# "Bubble" all connection parameters # "Bubble" all connection parameters
for option, value in local_options.iteritems(): for option, value in six.iteritems(local_options):
if option.startswith(CONNECTION_PARAMETER_STRING): if option.startswith(CONNECTION_PARAMETER_STRING):
self.options['%s-%s' % (option, country)] = value self.options['%s-%s' % (option, country)] = value
def install(self): def install(self):
for country, request in self.request_dict.iteritems(): for country, request in six.iteritems(self.request_dict):
request.install() request.install()
return [] return []
......
...@@ -26,8 +26,6 @@ ...@@ -26,8 +26,6 @@
############################################################################## ##############################################################################
import os import os
import urllib
import urllib2
import pkg_resources import pkg_resources
from slapos.recipe.librecipe import BaseSlapRecipe from slapos.recipe.librecipe import BaseSlapRecipe
......
...@@ -31,7 +31,8 @@ import os ...@@ -31,7 +31,8 @@ import os
import slapos.slap import slapos.slap
from slapos.recipe.librecipe import unwrap from slapos.recipe.librecipe import unwrap
from ConfigParser import RawConfigParser import six
from six.moves.configparser import RawConfigParser
from netaddr import valid_ipv4, valid_ipv6 from netaddr import valid_ipv4, valid_ipv6
from slapos.util import mkdir_p from slapos.util import mkdir_p
from slapos import format as slapformat from slapos import format as slapformat
...@@ -115,7 +116,7 @@ class Recipe(object): ...@@ -115,7 +116,7 @@ class Recipe(object):
buildout['buildout']['directory']) buildout['buildout']['directory'])
match = self.OPTCRE_match match = self.OPTCRE_match
for key, value in parameter_dict.iteritems(): for key, value in six.iteritems(parameter_dict):
if match(key) is not None: if match(key) is not None:
continue continue
options['configuration.' + key] = value options['configuration.' + key] = value
...@@ -157,11 +158,10 @@ class Recipe(object): ...@@ -157,11 +158,10 @@ class Recipe(object):
options[his_key.replace('_', '-')] = value options[his_key.replace('_', '-')] = value
# Get Instance and root instance title or return UNKNOWN if not set # Get Instance and root instance title or return UNKNOWN if not set
options['instance-title'] = parameter_dict.pop('instance_title', options['instance-title'] = parameter_dict.pop('instance_title',
'UNKNOWN Instance').encode('UTF-8') 'UNKNOWN Instance')
options['root-instance-title'] = parameter_dict.pop('root_instance_title', options['root-instance-title'] = parameter_dict.pop('root_instance_title',
'UNKNOWN').encode('UTF-8') 'UNKNOWN')
options['instance-guid'] = computer_partition.getInstanceGuid() \ options['instance-guid'] = computer_partition.getInstanceGuid()
.encode('UTF-8')
ipv4_set = set() ipv4_set = set()
v4_add = ipv4_set.add v4_add = ipv4_set.add
...@@ -204,9 +204,9 @@ class Recipe(object): ...@@ -204,9 +204,9 @@ class Recipe(object):
# also export single ip values for those recipes that don't support sets. # also export single ip values for those recipes that don't support sets.
if ipv4_set: if ipv4_set:
options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8') options['ipv4-random'] = min(ipv4_set)
if ipv6_set: if ipv6_set:
options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8') options['ipv6-random'] = min(ipv6_set)
storage_home = options.get('storage-home') storage_home = options.get('storage-home')
storage_dict = {} storage_dict = {}
...@@ -240,7 +240,7 @@ class Recipe(object): ...@@ -240,7 +240,7 @@ class Recipe(object):
# be very careful with overriding master's information # be very careful with overriding master's information
for key, value in flatten_dict(partition_params).items(): for key, value in flatten_dict(partition_params).items():
if key not in options: if key not in options:
if isinstance(value, unicode): if six.PY2 and isinstance(value, unicode):
value = value.encode('UTF-8') value = value.encode('UTF-8')
options[key] = value options[key] = value
# print out augmented options to see what we are passing # print out augmented options to see what we are passing
...@@ -265,8 +265,11 @@ class JsonDump(Recipe): ...@@ -265,8 +265,11 @@ class JsonDump(Recipe):
def __init__(self, buildout, name, options): def __init__(self, buildout, name, options):
parameter_dict = self.fetch_parameter_dict(options) parameter_dict = self.fetch_parameter_dict(options)
self._json_output = options['json-output'] self._json_output = options['json-output']
with os.fdopen(os.open(self._json_output, os.O_WRONLY | os.O_CREAT, 0600), 'w') as fout: # XXX: do not touch file if there's no change to avoid excessive IO
fout.write(json.dumps(parameter_dict, indent=2, sort_keys=True)) # (see https://lab.nexedi.com/nexedi/slapos.recipe.template/commit/14d26bc8c77a1940f389026bdbd3a9b229b241f4
# for an example to fix this)
with os.fdopen(os.open(self._json_output, os.O_WRONLY | os.O_CREAT, 0o600), 'w') as fout:
json.dump(parameter_dict, fout, indent=2, sort_keys=True)
def install(self): def install(self):
return [self._json_output] return [self._json_output]
......
...@@ -36,7 +36,7 @@ def kill(pid_file, sig=signal.SIGUSR1): ...@@ -36,7 +36,7 @@ def kill(pid_file, sig=signal.SIGUSR1):
pid = int(f.read().strip()) pid = int(f.read().strip())
try: try:
os.kill(pid, sig) os.kill(pid, sig)
except OSError, e: except OSError as e:
if e.errno != errno.ESRCH: # No such process if e.errno != errno.ESRCH: # No such process
raise e raise e
os.unlink(pid_file) os.unlink(pid_file)
......
...@@ -76,9 +76,9 @@ class Recipe(GenericBaseRecipe): ...@@ -76,9 +76,9 @@ class Recipe(GenericBaseRecipe):
import hashlib import hashlib
hasher = hashlib.md5() hasher = hashlib.md5()
for path in file_list: for path in file_list:
with open(path, 'r') as afile: with open(path, 'rb') as afile:
buf = afile.read() buf = afile.read()
hasher.update("%s\n" % len(buf)) hasher.update(b"%u\n" % len(buf))
hasher.update(buf) hasher.update(buf)
hash = hasher.hexdigest() hash = hasher.hexdigest()
return hash return hash
...@@ -69,7 +69,7 @@ export JAVA_OPTS="${JAVA_OPTS} -Djava.awt.headless=true" ...@@ -69,7 +69,7 @@ export JAVA_OPTS="${JAVA_OPTS} -Djava.awt.headless=true"
bindir = os.path.join(tomcat_home, 'bin') bindir = os.path.join(tomcat_home, 'bin')
for f in os.listdir(bindir): for f in os.listdir(bindir):
if f.endswith('.sh'): if f.endswith('.sh'):
os.chmod(os.path.join(bindir, f), 0755) os.chmod(os.path.join(bindir, f), 0o755)
tomcat_wrapper = self.createRunningWrapper('xwiki', """#!/bin/sh tomcat_wrapper = self.createRunningWrapper('xwiki', """#!/bin/sh
export JRE_HOME=%(java_home)s export JRE_HOME=%(java_home)s
exec %(catalina)s run exec %(catalina)s run
......
...@@ -33,7 +33,6 @@ import hashlib ...@@ -33,7 +33,6 @@ import hashlib
import sys import sys
import zc.buildout import zc.buildout
import zc.recipe.egg import zc.recipe.egg
import ConfigParser
class Recipe(BaseSlapRecipe): class Recipe(BaseSlapRecipe):
def installLogrotate(self): def installLogrotate(self):
......
...@@ -5,6 +5,8 @@ import sys ...@@ -5,6 +5,8 @@ import sys
import tempfile import tempfile
import unittest import unittest
import six
class PBSTest(unittest.TestCase): class PBSTest(unittest.TestCase):
...@@ -22,7 +24,7 @@ class PBSTest(unittest.TestCase): ...@@ -22,7 +24,7 @@ class PBSTest(unittest.TestCase):
def test_push(self): def test_push(self):
recipe = self.new_recipe() recipe = self.new_recipe()
with tempfile.NamedTemporaryFile() as rdiff_wrapper: with tempfile.NamedTemporaryFile('w+') as rdiff_wrapper:
recipe.wrapper_push(remote_schema='TEST_REMOTE_SCHEMA', recipe.wrapper_push(remote_schema='TEST_REMOTE_SCHEMA',
local_dir='TEST_LOCAL_DIR', local_dir='TEST_LOCAL_DIR',
remote_dir='TEST_REMOTE_DIR', remote_dir='TEST_REMOTE_DIR',
...@@ -35,7 +37,7 @@ class PBSTest(unittest.TestCase): ...@@ -35,7 +37,7 @@ class PBSTest(unittest.TestCase):
def test_pull(self): def test_pull(self):
recipe = self.new_recipe() recipe = self.new_recipe()
with tempfile.NamedTemporaryFile() as rdiff_wrapper: with tempfile.NamedTemporaryFile('w+') as rdiff_wrapper:
recipe.wrapper_pull(remote_schema='TEST_REMOTE_SCHEMA', recipe.wrapper_pull(remote_schema='TEST_REMOTE_SCHEMA',
local_dir='TEST_LOCAL_DIR', local_dir='TEST_LOCAL_DIR',
remote_dir='TEST_REMOTE_DIR', remote_dir='TEST_REMOTE_DIR',
...@@ -104,22 +106,22 @@ class PBSTest(unittest.TestCase): ...@@ -104,22 +106,22 @@ class PBSTest(unittest.TestCase):
recipe._install() recipe._install()
self.assertItemsEqual(os.listdir(promises_directory), six.assertCountEqual(self, os.listdir(promises_directory),
['ssh-to-pulltest', 'ssh-to-pushtest']) ['ssh-to-pulltest', 'ssh-to-pushtest'])
self.assertItemsEqual(os.listdir(wrappers_directory), six.assertCountEqual(self, os.listdir(wrappers_directory),
['pulltest_raw', 'pulltest', 'pushtest_raw', 'pushtest']) ['pulltest_raw', 'pulltest', 'pushtest_raw', 'pushtest'])
self.assertItemsEqual(os.listdir(directory), six.assertCountEqual(self, os.listdir(directory),
['TEST_NAME']) ['TEST_NAME'])
self.assertItemsEqual(os.listdir(feeds_directory), six.assertCountEqual(self, os.listdir(feeds_directory),
['pulltest', 'pushtest']) ['pulltest', 'pushtest'])
self.assertItemsEqual(os.listdir(run_directory), six.assertCountEqual(self, os.listdir(run_directory),
[]) [])
self.assertItemsEqual(os.listdir(cron_directory), six.assertCountEqual(self, os.listdir(cron_directory),
['pulltest', 'pushtest']) ['pulltest', 'pushtest'])
shutil.rmtree(promises_directory) shutil.rmtree(promises_directory)
......
...@@ -3,6 +3,7 @@ from slapos.recipe import promise_plugin ...@@ -3,6 +3,7 @@ from slapos.recipe import promise_plugin
from slapos.test.utils import makeRecipe from slapos.test.utils import makeRecipe
from pprint import pformat from pprint import pformat
import stat, json import stat, json
import six
class TestPromisePlugin(unittest.TestCase): class TestPromisePlugin(unittest.TestCase):
...@@ -68,7 +69,7 @@ in multi line ...@@ -68,7 +69,7 @@ in multi line
with open(self.output) as f: with open(self.output) as f:
content = f.read() content = f.read()
self.assertIn("from slapos.promise.plugin.check_site_available import RunPromise", content) self.assertIn("from slapos.promise.plugin.check_site_available import RunPromise", content)
self.assertIn('extra_config_dict = { }', content) self.assertIn('extra_config_dict = %s' % ('{}' if six.PY3 else '{ }'), content)
def test_bad_parameters(self): def test_bad_parameters(self):
...@@ -99,7 +100,7 @@ in multi line ...@@ -99,7 +100,7 @@ in multi line
with self.assertRaises(ValueError) as p: with self.assertRaises(ValueError) as p:
recipe.install() recipe.install()
self.assertEqual(p.exception.message, "Import path %r is not a valid" % self.options['import']) self.assertEqual(str(p.exception), "Import path %r is not a valid" % self.options['import'])
def test_bad_content(self): def test_bad_content(self):
self.options['content'] = 'from slapos.plugin.check_site_available import toto; print "toto"' self.options['content'] = 'from slapos.plugin.check_site_available import toto; print "toto"'
...@@ -110,5 +111,5 @@ in multi line ...@@ -110,5 +111,5 @@ in multi line
with self.assertRaises(ValueError) as p: with self.assertRaises(ValueError) as p:
recipe.install() recipe.install()
self.assertEqual(p.exception.message, "Promise content %r is not valid" % self.options['content']) self.assertEqual(str(p.exception), "Promise content %r is not valid" % self.options['content'])
...@@ -6,6 +6,8 @@ import tempfile ...@@ -6,6 +6,8 @@ import tempfile
import unittest import unittest
from slapos.slap.slap import NotFoundError, ConnectionError from slapos.slap.slap import NotFoundError, ConnectionError
import six
class Re6stnetTest(unittest.TestCase): class Re6stnetTest(unittest.TestCase):
...@@ -88,7 +90,7 @@ class Re6stnetTest(unittest.TestCase): ...@@ -88,7 +90,7 @@ class Re6stnetTest(unittest.TestCase):
recipe.generateCertificate() recipe.generateCertificate()
self.assertItemsEqual(os.listdir(self.ssl_dir), six.assertCountEqual(self, os.listdir(self.ssl_dir),
['cert.key', 'cert.crt', 'dh.pem']) ['cert.key', 'cert.crt', 'dh.pem'])
last_time = time.ctime(os.stat(self.options['key-file'])[7]) last_time = time.ctime(os.stat(self.options['key-file'])[7])
...@@ -143,10 +145,10 @@ class Re6stnetTest(unittest.TestCase): ...@@ -143,10 +145,10 @@ class Re6stnetTest(unittest.TestCase):
token_dict = recipe.loadJsonFile(token_file) token_dict = recipe.loadJsonFile(token_file)
self.assertEqual(len(token_dict), 2) self.assertEqual(len(token_dict), 2)
self.assertTrue(token_dict.has_key('SOFTINST-58770')) self.assertIn('SOFTINST-58770', token_dict)
self.assertTrue(token_dict.has_key('SOFTINST-58778')) self.assertIn('SOFTINST-58778', token_dict)
self.assertItemsEqual(os.listdir(self.token_dir), six.assertCountEqual(self, os.listdir(self.token_dir),
['SOFTINST-58770.add', 'SOFTINST-58778.add']) ['SOFTINST-58770.add', 'SOFTINST-58778.add'])
first_add = recipe.readFile(os.path.join(self.token_dir, 'SOFTINST-58770.add')) first_add = recipe.readFile(os.path.join(self.token_dir, 'SOFTINST-58770.add'))
...@@ -173,7 +175,7 @@ class Re6stnetTest(unittest.TestCase): ...@@ -173,7 +175,7 @@ class Re6stnetTest(unittest.TestCase):
self.assertEqual(len(token_dict), 1) self.assertEqual(len(token_dict), 1)
self.assertEqual(token_dict['SOFTINST-58770'], first_add) self.assertEqual(token_dict['SOFTINST-58770'], first_add)
self.assertItemsEqual(os.listdir(self.token_dir), six.assertCountEqual(self, os.listdir(self.token_dir),
['SOFTINST-58770.add', 'SOFTINST-58778.remove']) ['SOFTINST-58770.add', 'SOFTINST-58778.remove'])
second_remove = recipe.readFile(os.path.join(self.token_dir, 'SOFTINST-58778.remove')) second_remove = recipe.readFile(os.path.join(self.token_dir, 'SOFTINST-58778.remove'))
...@@ -193,7 +195,7 @@ class Re6stnetTest(unittest.TestCase): ...@@ -193,7 +195,7 @@ class Re6stnetTest(unittest.TestCase):
token_content = recipe.readFile(token_file) token_content = recipe.readFile(token_file)
self.assertEqual(token_content, '{}') self.assertEqual(token_content, '{}')
self.assertItemsEqual(os.listdir(self.options['token-dir']), []) six.assertCountEqual(self, os.listdir(self.options['token-dir']), [])
self.checkWrapper(os.path.join(self.base_dir, 'manager_wrapper')) self.checkWrapper(os.path.join(self.base_dir, 'manager_wrapper'))
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
""" """
import sys import sys
import os.path import os.path
from ConfigParser import ConfigParser from zc.buildout.configparser import parse
import logging import logging
...@@ -52,18 +52,17 @@ def makeRecipe(recipe_class, options, name='test', slap_connection=None): ...@@ -52,18 +52,17 @@ def makeRecipe(recipe_class, options, name='test', slap_connection=None):
buildout_cfg = os.path.join(base_directory, 'buildout.cfg') buildout_cfg = os.path.join(base_directory, 'buildout.cfg')
if os.path.exists(buildout_cfg): if os.path.exists(buildout_cfg):
parser = ConfigParser() with open(buildout_cfg) as f:
parser.readfp(open(buildout_cfg)) parsed_cfg = parse(f, buildout_cfg)
if parser.has_option('buildout', 'eggs-directory'):
# when buildout_cfg is an instance buildout (like in SLAPOS-EGG-TEST), # When buildout_cfg is an instance buildout (like in SLAPOS-EGG-TEST),
# there's a ${buildout:eggs-directory} we can use. # there's a ${buildout:eggs-directory} we can use.
eggs_directory = parser.get('buildout', 'eggs-directory') # When buildout_cfg is a software buildout, we can only guess the
develop_eggs_directory = parser.get('buildout', 'develop-eggs-directory')
else:
# when when buildout_cfg is a software buildout, we can only guess the
# standard eggs directories. # standard eggs directories.
eggs_directory = os.path.join(base_directory, 'eggs') eggs_directory = parsed_cfg['buildout'].get(
develop_eggs_directory = os.path.join(base_directory, 'develop-eggs') 'eggs-directory', os.path.join(base_directory, 'eggs'))
develop_eggs_directory = parsed_cfg['buildout'].get(
'develop-eggs-directory', os.path.join(base_directory, 'develop-eggs'))
logging.getLogger(__name__).info( logging.getLogger(__name__).info(
'Using eggs-directory (%s) and develop-eggs-directory (%s) from buildout at %s', 'Using eggs-directory (%s) and develop-eggs-directory (%s) from buildout at %s',
......
...@@ -147,6 +147,11 @@ template = ...@@ -147,6 +147,11 @@ template =
--datadir="$datadir.new" --datadir="$datadir.new"
mv -v "$datadir.new" "$datadir" mv -v "$datadir.new" "$datadir"
} }
{%- if environ is defined %}
{%- for variable in environ.splitlines() %}
{{ variable }} \
{%- endfor %}
{%- endif %}
exec '${mariadb:location}/bin/mysqld' \ exec '${mariadb:location}/bin/mysqld' \
--defaults-file='{{defaults_file}}' \ --defaults-file='{{defaults_file}}' \
"$@" "$@"
......
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 7f3318aec3b682c3379609ccdd659411 md5sum = 001affafc204b638615deea04c95cfdf
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
......
...@@ -213,7 +213,7 @@ config-test-runner-node-count = {{ dumps(test_runner_node_count) }} ...@@ -213,7 +213,7 @@ config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
{% if server_type == 'neo' -%} {% if server_type == 'neo' -%}
config-neo-cluster = ${publish-early:neo-cluster} config-neo-cluster = ${publish-early:neo-cluster}
config-neo-name = {{ server_dict.keys()[0] }} config-neo-name = {{ server_dict.keys()[0] }}
config-neo-masters = ${neo-0-final:connection-masters} config-neo-masters = ${publish-early:neo-masters}
{% else -%} {% else -%}
config-zodb-zeo = ${request-zodb:connection-storage-dict} config-zodb-zeo = ${request-zodb:connection-storage-dict}
config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip} config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
...@@ -411,10 +411,6 @@ return = site_url ...@@ -411,10 +411,6 @@ return = site_url
<= monitor-publish <= monitor-publish
recipe = slapos.cookbook:publish.serialised recipe = slapos.cookbook:publish.serialised
-extends = publish-early -extends = publish-early
{% if 'neo' in storage_dict -%}
neo-masters = ${neo-0-final:connection-masters}
neo-admins = ${neo-0-final:connection-admins}
{% endif -%}
{% if zope_address_list_id_dict -%} {% if zope_address_list_id_dict -%}
{# {#
Pick any published hosts-dict, they are expected to be identical - and there is Pick any published hosts-dict, they are expected to be identical - and there is
...@@ -431,7 +427,6 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts- ...@@ -431,7 +427,6 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-
{% endfor -%} {% endfor -%}
{% endif -%} {% endif -%}
[publish-early] [publish-early]
recipe = slapos.cookbook:publish-early recipe = slapos.cookbook:publish-early
-init = -init =
...@@ -447,6 +442,8 @@ recipe = slapos.cookbook:publish-early ...@@ -447,6 +442,8 @@ recipe = slapos.cookbook:publish-early
{%- endif %} {%- endif %}
{%- if neo %} {%- if neo %}
neo-cluster gen-neo-cluster:name neo-cluster gen-neo-cluster:name
neo-admins neo-cluster:admins
neo-masters neo-cluster:masters
{%- if neo[0] %} {%- if neo[0] %}
neo-cluster = {{ dumps(neo[0]) }} neo-cluster = {{ dumps(neo[0]) }}
{%- endif %} {%- endif %}
......
...@@ -15,4 +15,4 @@ ...@@ -15,4 +15,4 @@
[template] [template]
filename = instance.cfg filename = instance.cfg
md5sum = 81b01eb048e0db93bf0152504bf8a04d md5sum = 3d12778d8dc910180ff3a7fcae8b37f9
...@@ -28,7 +28,7 @@ bin = $${buildout:directory}/bin ...@@ -28,7 +28,7 @@ bin = $${buildout:directory}/bin
working-dir = $${buildout:directory}/tmp working-dir = $${buildout:directory}/tmp
[test-list] [test-list]
path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup} path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup}
[slapos-test-runner] [slapos-test-runner]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
......
...@@ -97,6 +97,11 @@ setup = ${slapos-repository:location}/software/jupyter/test/ ...@@ -97,6 +97,11 @@ setup = ${slapos-repository:location}/software/jupyter/test/
egg = slapos.test.nextcloud egg = slapos.test.nextcloud
setup = ${slapos-repository:location}/software/nextcloud/test/ setup = ${slapos-repository:location}/software/nextcloud/test/
[slapos.test.turnserver-setup]
<= setup-develop-egg
egg = slapos.test.nextcloud
setup = ${slapos-repository:location}/software/turnserver/test/
[eggs] [eggs]
recipe = zc.recipe.egg recipe = zc.recipe.egg
eggs = eggs =
...@@ -121,6 +126,7 @@ eggs = ...@@ -121,6 +126,7 @@ eggs =
${slapos.test.slaprunner-setup:egg} ${slapos.test.slaprunner-setup:egg}
${slapos.test.jupyter-setup:egg} ${slapos.test.jupyter-setup:egg}
${slapos.test.nextcloud-setup:egg} ${slapos.test.nextcloud-setup:egg}
${slapos.test.turnserver-setup:egg}
${backports.lzma:egg} ${backports.lzma:egg}
entry-points = entry-points =
......
[buildout]
extends =
../../component/python3/buildout.cfg
software.cfg
python = python3.5
[nghttp2]
environment =
PATH=${autoconf:location}/bin:${automake:location}/bin:${libtool:location}/bin:${m4:location}/bin:${python3.5:location}/bin:%(PATH)s
[supervisor-repository]
<= git-clone-repository
repository = https://github.com/Supervisor/supervisor.git
[supervisor-develop]
recipe = zc.recipe.egg:develop
egg = supervisor
setup = ${supervisor-repository:location}
[eggs]
recipe = zc.recipe.egg
eggs += ${supervisor-develop:egg}
[versions]
supervisor =
{% set part_list = [] -%}
{% set server_name = slapparameter_dict.get('server-name', 'turn.example.com') -%}
[directory]
recipe = slapos.cookbook:mkdirectory
etc = ${buildout:directory}/etc
bin = ${buildout:directory}/bin
srv = ${buildout:directory}/srv
var = ${buildout:directory}/var
run = ${:var}/run
log = ${:var}/log
scripts = ${:etc}/run
services = ${:etc}/service
promises = ${:etc}/promise
plugins = ${:etc}/plugin
ssl = ${:etc}/ssl
[file-base]
recipe = slapos.recipe.template:jinja2
template = inline:{{ '{{ content }}' }}
{% macro simplefile(section_name, file_path, content, mode='') -%}
{% set content_section_name = section_name ~ '-content' -%}
[{{ content_section_name }}]
content = {{ dumps(content) }}
[{{ section_name }}]
< = file-base
rendered = {{ file_path }}
context = key content {{ content_section_name }}:content
mode = {{ mode }}
{% do part_list.append(section_name) -%}
{%- endmacro %}
{% if slapparameter_dict.get('ssl-key') and slapparameter_dict.get('ssl-crt') -%}
{{ simplefile('ssl-certificate', '${turnserver-ssl:certificate}', slapparameter_dict.get('ssl-crt')) }}
{{ simplefile('ssl-key', '${turnserver-ssl:key}', slapparameter_dict.get('ssl-key'), 600) }}
{% else -%}
{% do part_list.append('gen-certificate') -%}
[gen-certificate]
recipe = plone.recipe.command
command = "{{ parameter_dict['openssl'] }}/bin/openssl" req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout "${turnserver-ssl:key}" -out "${turnserver-ssl:certificate}"
{% endif -%}
[turnserver-ssl]
recipe = plone.recipe.command
certificate = ${directory:ssl}/cert.pem
key = ${directory:ssl}/key.pem
dhparam = ${directory:ssl}/dhparam.pem
command =
if [ ! -s "${directory:ssl}//dhparam.pem" ]; then
"{{ parameter_dict['openssl'] }}/bin/openssl" dhparam -out ${:dhparam} 4096
fi
[gen-secret]
recipe = plone.recipe.command
secret-file = ${directory:etc}/.turnsecret
command =
if [ ! -s "${:secret-file}" ]; then
cat <<EOF > ${:secret-file}
[turnserver]
secret = $("{{ parameter_dict['openssl'] }}/bin/openssl" rand -hex 32)
EOF
fi
chmod 600 ${:secret-file}
[read-secret]
recipe = slapos.cookbook:zero-knowledge.read
file-path = ${gen-secret:secret-file}
secret =
{% set turn_port = slapparameter_dict.get('port', 3478) -%}
{% set turn_tls_port = slapparameter_dict.get('tls-port', 5349) -%}
{% set listining_ip = slapparameter_dict.get('listening-ip', (ipv4 | list)[0]) -%}
[turnserver-config]
recipe = collective.recipe.template
input = inline:
listening-port={{ turn_port }}
tls-listening-port={{ turn_tls_port }}
fingerprint
lt-cred-mech
use-auth-secret
static-auth-secret=${read-secret:secret}
listening-ip={{ listining_ip }}
{% if slapparameter_dict.get('external-ip', '') %}
external-ip={{ slapparameter_dict['external-ip'] }}
{% endif %}
server-name={{ server_name }}
realm={{ server_name }}
total-quota=100
bps-capacity=0
stale-nonce=600
cert=${turnserver-ssl:certificate}
pkey=${turnserver-ssl:key}
dh-file=${turnserver-ssl:dhparam}
cipher-list="ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AES:RSA+3DES:!ADH:!AECDH:!MD5"
no-loopback-peers
no-multicast-peers
mobility
no-tlsv1
no-tlsv1_1
no-stdout-log
log-file=${directory:log}/turnserver.log
userdb=${directory:srv}/turndb
pidfile=${directory:run}/turnserver.pid
verbose
output = ${directory:etc}/turnserver.conf
mode = 644
[turnserver-wrapper]
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['turnserver-location'] }}/bin/turnserver
-c ${turnserver-config:output}
wrapper-path = ${directory:services}/turnserver
hash-files = ${buildout:directory}/software_release/buildout.cfg
[promise-check-turnserver-port]
<= monitor-promise-base
module = check_port_listening
name = turnserver-port-listening.py
config-hostname = {{ listining_ip }}
config-port = {{ turn_port }}
[promise-check-turnserver-tls-port]
<= monitor-promise-base
module = check_port_listening
name = turnserver-tls-port-listening.py
config-hostname = {{ listining_ip }}
config-port = {{ turn_tls_port }}
[publish-connection-information]
<= monitor-publish
recipe = slapos.cookbook:publish
turn-url = {{ server_name ~ ':' ~ turn_port }}
turn-tls-url = {{ server_name ~ ':' ~ turn_tls_port }}
secret = ${read-secret:secret}
[buildout]
extends = {{ template_monitor }}
parts =
publish-connection-information
# Complete parts with sections
{{ part_list | join('\n ') }}
# turn server
turnserver-wrapper
promise-check-turnserver-tls-port
promise-check-turnserver-port
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
\ No newline at end of file
[buildout]
parts = switch-softwaretype
# std stuff for slapos instance
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${dynamic-template-turnserver:rendered}
RootSoftwareInstance = $${:default}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = $${buildout:directory}/$${:filename}
extensions = jinja2.ext.do
mode = 0644
extra-context =
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key ipv4 slap-configuration:ipv4
key ipv6 slap-configuration:ipv6
key global_ipv4_prefix network-information:global-ipv4-network
key slapparameter_dict slap-configuration:configuration
key computer_id slap-configuration:computer
raw template_monitor ${monitor2-template:rendered}
raw logrotate_cfg ${template-logrotate-base:rendered}
$${:extra-context}
[dynamic-template-turnserver-parameters]
openssl = ${openssl:location}
turnserver-location = ${coturn:location}
[dynamic-template-turnserver]
<= jinja2-template-base
template = ${template-turnserver:location}/${template-turnserver:filename}
filename = instance-turnserver.cfg
extra-context =
section parameter_dict dynamic-template-turnserver-parameters
[buildout]
extends =
../../component/coturn/buildout.cfg
../../component/openssl/buildout.cfg
# ../../component/6tunnel/buildout.cfg
../../component/socat/buildout.cfg
../../stack/monitor/buildout.cfg
../../stack/slapos.cfg
parts +=
slapos-cookbook
coturn
instance-cfg
[download-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
mode = 644
[instance-cfg]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
output = ${buildout:directory}/instance.cfg
md5sum = d027a2dccaf15ae6e7d3a28cc02d70c3
[template-turnserver]
<= download-base
filename = instance-turnserver.cfg.jinja2.in
md5sum = 02bddf180519f7649d3b1f997a496eed
[versions]
slapos.recipe.template = 4.3
\ No newline at end of file
Tests for TurnServer Software Release
\ No newline at end of file
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
import glob
import os
version = '0.0.1.dev0'
name = 'slapos.test.turnserver'
long_description = open("README.md").read()
setup(name=name,
version=version,
description="Test for TurnServer Software Release",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'erp5.util',
'supervisor',
'psutil',
],
zip_safe=True,
test_suite='test',
)
\ No newline at end of file
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import subprocess
import json
import glob
import ConfigParser
import utils
# for development: debugging logs and install Ctrl+C handler
if os.environ.get('SLAPOS_TEST_DEBUG'):
import logging
logging.basicConfig(level=logging.DEBUG)
import unittest
unittest.installHandler()
def subprocess_status_output(*args, **kwargs):
prc = subprocess.Popen(
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
*args,
**kwargs)
out, err = prc.communicate()
return prc.returncode, out
class InstanceTestCase(utils.SlapOSInstanceTestCase):
@classmethod
def getSoftwareURLList(cls):
return (os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'software.cfg')), )
class ServicesTestCase(InstanceTestCase):
@staticmethod
def generateHash(file_list):
import hashlib
hasher = hashlib.md5()
for path in file_list:
with open(path, 'r') as afile:
buf = afile.read()
hasher.update("%s\n" % len(buf))
hasher.update(buf)
hash = hasher.hexdigest()
return hash
def test_process_list(self):
hash_list = [
'software_release/buildout.cfg',
]
expected_process_names = [
'bootstrap-monitor',
'turnserver-{hash}-on-watch',
'certificate_authority-{hash}-on-watch',
'crond-{hash}-on-watch',
'monitor-httpd-{hash}-on-watch',
'monitor-httpd-graceful',
]
supervisor = self.getSupervisorRPCServer().supervisor
process_name_list = [process['name']
for process in supervisor.getAllProcessInfo()]
hash_file_list = [os.path.join(self.computer_partition_root_path, path)
for path in hash_list]
for name in expected_process_names:
h = ServicesTestCase.generateHash(hash_file_list)
expected_process_name = name.format(hash=h)
self.assertIn(expected_process_name, process_name_list)
def test_default_deployment(self):
partition_path_list = glob.glob(os.path.join(self.instance_path, '*'))
instance_folder = None
for partition_path in partition_path_list:
if os.path.exists(os.path.join(partition_path, 'etc/turnserver.conf')):
instance_folder = partition_path
break
secret_file = os.path.join(instance_folder, 'etc/.turnsecret')
self.assertTrue(os.path.exists(instance_folder))
self.assertTrue(os.path.exists(secret_file))
config = ConfigParser.ConfigParser()
config.readfp(open(secret_file))
secret = config.get('turnserver', 'secret')
expected_config = """listening-port=3478
tls-listening-port=5349
fingerprint
lt-cred-mech
use-auth-secret
static-auth-secret=%(secret)s
listening-ip=%(ipv4)s
server-name=turn.example.com
realm=turn.example.com
total-quota=100
bps-capacity=0
stale-nonce=600
cert=%(instance_path)s/etc/ssl/cert.pem
pkey=%(instance_path)s/etc/ssl/key.pem
dh-file=%(instance_path)s/etc/ssl/dhparam.pem
cipher-list="ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AES:RSA+3DES:!ADH:!AECDH:!MD5"
no-loopback-peers
no-multicast-peers
mobility
no-tlsv1
no-tlsv1_1
no-stdout-log
log-file=%(instance_path)s/var/log/turnserver.log
userdb=%(instance_path)s/srv/turndb
pidfile=%(instance_path)s/var/run/turnserver.pid
verbose""" % {'instance_path': instance_folder, 'secret': secret, 'ipv4': self.config['ipv4_address']}
with open(os.path.join(instance_folder, 'etc/turnserver.conf')) as f:
current_config = f.read().strip()
self.assertEqual(current_config, expected_config)
def test_turnserver_promises(self):
partition_path_list = glob.glob(os.path.join(self.instance_path, '*'))
instance_folder = None
for partition_path in partition_path_list:
if os.path.exists(os.path.join(partition_path, 'etc/turnserver.conf')):
instance_folder = partition_path
break
self.assertTrue(os.path.exists(instance_folder))
promise_path_list = glob.glob(os.path.join(instance_folder, 'etc/plugin/*.py'))
promise_name_list = [x for x in
os.listdir(os.path.join(instance_folder, 'etc/plugin'))
if not x.endswith('.pyc')]
partition_name = os.path.basename(instance_folder.rstrip('/'))
self.assertEqual(sorted(promise_name_list),
sorted([
"__init__.py",
"check-free-disk-space.py",
"monitor-http-frontend.py",
"buildout-%s-status.py" % partition_name,
"monitor-bootstrap-status.py",
"monitor-httpd-listening-on-tcp.py",
"turnserver-port-listening.py",
"turnserver-tls-port-listening.py",
]))
ignored_plugin_list = [
'__init__.py',
'monitor-http-frontend.py',
]
runpromise_bin = os.path.join(
self.software_path, 'bin', 'monitor.runpromise')
monitor_conf = os.path.join(instance_folder, 'etc', 'monitor.conf')
msg = []
status = 0
for plugin_path in promise_path_list:
plugin_name = os.path.basename(plugin_path)
if plugin_name in ignored_plugin_list:
continue
plugin_status, plugin_result = subprocess_status_output([
runpromise_bin,
'-c', monitor_conf,
'--run-only', plugin_name,
'--force',
'--check-anomaly'
])
status += plugin_status
if plugin_status == 1:
msg.append(plugin_result)
# sanity check
if 'Checking promise %s' % plugin_name not in plugin_result:
plugin_status = 1
msg.append(plugin_result)
msg = ''.join(msg).strip()
self.assertEqual(status, 0, msg)
class ParametersTestCase(InstanceTestCase):
@classmethod
def getInstanceParameterDict(cls):
return {
'server-name': "turn.site.com",
'port': 3488,
'tls-port': 5369,
'external-ip': '127.0.0.1',
'listening-ip': '127.0.0.1'
}
def test_turnserver_with_parameters(self):
partition_path_list = glob.glob(os.path.join(self.instance_path, '*'))
instance_folder = None
for partition_path in partition_path_list:
if os.path.exists(os.path.join(partition_path, 'etc/turnserver.conf')):
instance_folder = partition_path
break
secret_file = os.path.join(instance_folder, 'etc/.turnsecret')
self.assertTrue(os.path.exists(instance_folder))
self.assertTrue(os.path.exists(secret_file))
config = ConfigParser.ConfigParser()
config.readfp(open(secret_file))
secret = config.get('turnserver', 'secret')
expected_config = """listening-port=%(port)s
tls-listening-port=%(tls_port)s
fingerprint
lt-cred-mech
use-auth-secret
static-auth-secret=%(secret)s
listening-ip=%(ipv4)s
external-ip=%(external_ip)s
server-name=%(name)s
realm=%(name)s
total-quota=100
bps-capacity=0
stale-nonce=600
cert=%(instance_path)s/etc/ssl/cert.pem
pkey=%(instance_path)s/etc/ssl/key.pem
dh-file=%(instance_path)s/etc/ssl/dhparam.pem
cipher-list="ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AES:RSA+3DES:!ADH:!AECDH:!MD5"
no-loopback-peers
no-multicast-peers
mobility
no-tlsv1
no-tlsv1_1
no-stdout-log
log-file=%(instance_path)s/var/log/turnserver.log
userdb=%(instance_path)s/srv/turndb
pidfile=%(instance_path)s/var/run/turnserver.pid
verbose""" % {'instance_path': instance_folder,
'secret': secret,
'ipv4': '127.0.0.1',
'name': 'turn.site.com',
'external_ip': '127.0.0.1',
'port': 3488,
'tls_port': 5369,}
with open(os.path.join(instance_folder, 'etc/turnserver.conf')) as f:
current_config = f.read().strip()
self.assertEqual(current_config, expected_config)
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import unittest
import os
import socket
from contextlib import closing
import logging
import StringIO
import xmlrpclib
import supervisor.xmlrpc
from erp5.util.testnode.SlapOSControler import SlapOSControler
from erp5.util.testnode.ProcessManager import ProcessManager
# Utility functions
def findFreeTCPPort(ip=''):
"""Find a free TCP port to listen to.
"""
family = socket.AF_INET6 if ':' in ip else socket.AF_INET
with closing(socket.socket(family, socket.SOCK_STREAM)) as s:
s.bind((ip, 0))
return s.getsockname()[1]
# TODO:
# - allow requesting multiple instances ?
class SlapOSInstanceTestCase(unittest.TestCase):
"""Install one slapos instance.
This test case install software(s) and request one instance during `setUpClass`
and destroy the instance during `tearDownClass`.
Software Release URL, Instance Software Type and Instance Parameters can be defined
on the class.
All tests from the test class will run with the same instance.
The following class attributes are available:
* `computer_partition`: the computer partition instance, implementing
`slapos.slap.interface.slap.IComputerPartition`.
* `computer_partition_root_path`: the path of the instance root directory.
"""
# Methods to be defined by subclasses.
@classmethod
def getSoftwareURLList(cls):
"""Return URL of software releases to install.
To be defined by subclasses.
"""
raise NotImplementedError()
@classmethod
def getInstanceParameterDict(cls):
"""Return instance parameters
To be defined by subclasses if they need to request instance with specific
parameters.
"""
return {}
@classmethod
def getInstanceSoftwareType(cls):
"""Return software type for instance, default "default"
To be defined by subclasses if they need to request instance with specific
software type.
"""
return "default"
# Utility methods.
def getSupervisorRPCServer(self):
"""Returns a XML-RPC connection to the supervisor used by slapos node
Refer to http://supervisord.org/api.html for details of available methods.
"""
# xmlrpc over unix socket https://stackoverflow.com/a/11746051/7294664
return xmlrpclib.ServerProxy(
'http://slapos-supervisor',
transport=supervisor.xmlrpc.SupervisorTransport(
None,
None,
# XXX hardcoded socket path
serverurl="unix://{working_directory}/inst/supervisord.socket".format(
**self.config)))
# Unittest methods
@classmethod
def setUpClass(cls):
"""Setup the class, build software and request an instance.
If you have to override this method, do not forget to call this method on
parent class.
"""
try:
cls.setUpWorkingDirectory()
cls.setUpConfig()
cls.setUpSlapOSController()
cls.runSoftwareRelease()
# XXX instead of "runSoftwareRelease", it would be better to be closer to slapos usage:
# cls.supplySoftwares()
# cls.installSoftwares()
cls.runComputerPartition()
# XXX instead of "runComputerPartition", it would be better to be closer to slapos usage:
# cls.requestInstances()
# cls.createInstances()
# cls.requestInstances()
except Exception:
cls.stopSlapOSProcesses()
raise
cls.instance_path = os.path.join(
cls.config['working_directory'],
'inst')
cls.software_path = os.path.realpath(os.path.join(
cls.computer_partition_root_path, 'software_release'))
@classmethod
def tearDownClass(cls):
"""Tear down class, stop the processes and destroy instance.
"""
cls.stopSlapOSProcesses()
# Implementation
@classmethod
def stopSlapOSProcesses(cls):
if hasattr(cls, '_process_manager'):
cls._process_manager.killPreviousRun()
@classmethod
def setUpWorkingDirectory(cls):
"""Initialise the directories"""
cls.working_directory = os.environ.get(
'SLAPOS_TEST_WORKING_DIR',
os.path.join(os.path.dirname(__file__), '.slapos'))
# To prevent error: Cannot open an HTTP server: socket.error reported
# AF_UNIX path too long This `working_directory` should not be too deep.
# Socket path is 108 char max on linux
# https://github.com/torvalds/linux/blob/3848ec5/net/unix/af_unix.c#L234-L238
# Supervisord socket name contains the pid number, which is why we add
# .xxxxxxx in this check.
if len(cls.working_directory + '/inst/supervisord.socket.xxxxxxx') > 108:
raise RuntimeError('working directory ( {} ) is too deep, try setting '
'SLAPOS_TEST_WORKING_DIR'.format(cls.working_directory))
if not os.path.exists(cls.working_directory):
os.mkdir(cls.working_directory)
@classmethod
def setUpConfig(cls):
"""Create slapos configuration"""
cls.config = {
"working_directory": cls.working_directory,
"slapos_directory": cls.working_directory,
"log_directory": cls.working_directory,
"computer_id": 'slapos.test', # XXX
'proxy_database': os.path.join(cls.working_directory, 'proxy.db'),
'partition_reference': cls.__name__,
# "proper" slapos command must be in $PATH
'slapos_binary': 'slapos',
'node_quantity': '3',
}
# Some tests are expecting that local IP is not set to 127.0.0.1
ipv4_address = os.environ.get('SLAPOS_TEST_IPV4', '127.0.1.1')
ipv6_address = os.environ['SLAPOS_TEST_IPV6']
cls.config['proxy_host'] = cls.config['ipv4_address'] = ipv4_address
cls.config['ipv6_address'] = ipv6_address
cls.config['proxy_port'] = findFreeTCPPort(ipv4_address)
cls.config['master_url'] = 'http://{proxy_host}:{proxy_port}'.format(
**cls.config)
@classmethod
def setUpSlapOSController(cls):
"""Create the a "slapos controller" and supply softwares from `getSoftwareURLList`.
This is equivalent to:
slapos proxy start
for sr in getSoftwareURLList; do
slapos supply $SR $COMP
done
"""
cls._process_manager = ProcessManager()
# XXX this code is copied from testnode code
cls.slapos_controler = SlapOSControler(
cls.working_directory,
cls.config
)
slapproxy_log = os.path.join(cls.config['log_directory'], 'slapproxy.log')
logger = logging.getLogger(__name__)
logger.debug('Configured slapproxy log to %r', slapproxy_log)
cls.software_url_list = cls.getSoftwareURLList()
cls.slapos_controler.initializeSlapOSControler(
slapproxy_log=slapproxy_log,
process_manager=cls._process_manager,
reset_software=False,
software_path_list=cls.software_url_list)
# XXX we should check *earlier* if that pidfile exist and if supervisord
# process still running, because if developer started supervisord (or bugs?)
# then another supervisord will start and starting services a second time
# will fail.
cls._process_manager.supervisord_pid_file = os.path.join(
cls.slapos_controler.instance_root, 'var', 'run', 'supervisord.pid')
@classmethod
def runSoftwareRelease(cls):
"""Run all the software releases that were supplied before.
This is the equivalent of `slapos node software`.
The tests will be marked file if software building fail.
"""
logger = logging.getLogger()
logger.level = logging.DEBUG
stream = StringIO.StringIO()
stream_handler = logging.StreamHandler(stream)
logger.addHandler(stream_handler)
try:
cls.software_status_dict = cls.slapos_controler.runSoftwareRelease(
cls.config, environment=os.environ)
stream.seek(0)
stream.flush()
message = ''.join(stream.readlines()[-100:])
assert cls.software_status_dict['status_code'] == 0, message
finally:
logger.removeHandler(stream_handler)
del stream
@classmethod
def runComputerPartition(cls):
"""Instanciate the software.
This is the equivalent of doing:
slapos request --type=getInstanceSoftwareType --parameters=getInstanceParameterDict
slapos node instance
and return the slapos request instance parameters.
This can be called by tests to simulate re-request with different parameters.
"""
logger = logging.getLogger()
logger.level = logging.DEBUG
stream = StringIO.StringIO()
stream_handler = logging.StreamHandler(stream)
logger.addHandler(stream_handler)
if cls.getInstanceSoftwareType() != 'default':
raise NotImplementedError
instance_parameter_dict = cls.getInstanceParameterDict()
try:
cls.instance_status_dict = cls.slapos_controler.runComputerPartition(
cls.config,
cluster_configuration=instance_parameter_dict,
environment=os.environ)
stream.seek(0)
stream.flush()
message = ''.join(stream.readlines()[-100:])
assert cls.instance_status_dict['status_code'] == 0, message
finally:
logger.removeHandler(stream_handler)
del stream
# FIXME: similar to test node, only one (root) partition is really
# supported for now.
computer_partition_list = []
for i in range(len(cls.software_url_list)):
computer_partition_list.append(
cls.slapos_controler.slap.registerOpenOrder().request(
cls.software_url_list[i],
# This is how testnode's SlapOSControler name created partitions
partition_reference='testing partition {i}'.format(
i=i, **cls.config),
partition_parameter_kw=instance_parameter_dict))
# expose some class attributes so that tests can use them:
# the ComputerPartition instances, to getInstanceParameterDict
cls.computer_partition = computer_partition_list[0]
# the path of the instance on the filesystem, for low level inspection
cls.computer_partition_root_path = os.path.join(
cls.config['working_directory'],
'inst',
cls.computer_partition.getId())
...@@ -294,6 +294,7 @@ context = ...@@ -294,6 +294,7 @@ context =
key template_mariadb template-mariadb:target key template_mariadb template-mariadb:target
key template_mariadb_initial_setup template-mariadb-initial-setup:target key template_mariadb_initial_setup template-mariadb-initial-setup:target
key template_my_cnf template-my-cnf:target key template_my_cnf template-my-cnf:target
key template_mysqld_wrapper template-mysqld-wrapper:rendered
key template_postfix template-postfix:target key template_postfix template-postfix:target
key template_postfix_aliases template-postfix-aliases:target key template_postfix_aliases template-postfix-aliases:target
key template_postfix_main_cf template-postfix-main-cf:target key template_postfix_main_cf template-postfix-main-cf:target
......
...@@ -26,7 +26,7 @@ md5sum = d95e8500bdc72d1f40b97cc414656e7e ...@@ -26,7 +26,7 @@ md5sum = d95e8500bdc72d1f40b97cc414656e7e
[template-mariadb] [template-mariadb]
filename = instance-mariadb.cfg.in filename = instance-mariadb.cfg.in
md5sum = 5a93ff00ee4eb60cc065d51fb437877a md5sum = 591fe60e2f615c7690fa5078473b1b0a
[template-kumofs] [template-kumofs]
filename = instance-kumofs.cfg.in filename = instance-kumofs.cfg.in
...@@ -70,7 +70,7 @@ md5sum = c64f35f825200fe35328641b2b8e0fdd ...@@ -70,7 +70,7 @@ md5sum = c64f35f825200fe35328641b2b8e0fdd
[template] [template]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = f81f4c9881bf868882cd56fe6a769666 md5sum = f6c6820f9b3653d0d5c29708606fc591
[monitor-template-dummy] [monitor-template-dummy]
filename = dummy.cfg filename = dummy.cfg
......
...@@ -126,7 +126,7 @@ command = {{ parameter_dict['findutils-location'] }}/bin/find "${directory:maria ...@@ -126,7 +126,7 @@ command = {{ parameter_dict['findutils-location'] }}/bin/find "${directory:maria
ip = {{ ip }} ip = {{ ip }}
port = {{ port }} port = {{ port }}
socket = ${directory:run}/mariadb.sock socket = ${directory:run}/mariadb.sock
data-directory = ${directory:mariadb-data} data-directory = ${directory:srv}/mariadb
tmp-directory = ${directory:tmp} tmp-directory = ${directory:tmp}
etc-directory = ${directory:etc} etc-directory = ${directory:etc}
pid-file = ${directory:run}/mariadb.pid pid-file = ${directory:run}/mariadb.pid
...@@ -178,25 +178,18 @@ mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path} ...@@ -178,25 +178,18 @@ mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
[{{ section('mysqld') }}] [{{ section('mysqld') }}]
< = jinja2-template-executable < = jinja2-template-executable
# Note: all rendering is done when this file is rendered, not when the mysqld
# section is installed - so I only use jinja2 as a fancy way to write an
# executable file with partition-dependent but instance-parameters independent
# content.
template = inline:#!{{ dash }}
'{{ parameter_dict['mariadb-location'] }}/scripts/mysql_install_db' \
--defaults-file='${my-cnf:rendered}' \
--skip-name-resolve \
--datadir='${my-cnf-parameters:data-directory}' \
--basedir='{{ parameter_dict['mariadb-location'] }}' \
&& ODBCSYSINI='${my-cnf-parameters:etc-directory}' \
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:{{ parameter_dict['unixodbc-location'] }}/lib \
{% for variable in slapparameter_dict.get('environment-variables', ()) %}
{{ variable }} \
{% endfor %}
exec '{{ parameter_dict['mariadb-location'] }}/bin/mysqld' \
--defaults-file='${my-cnf:rendered}' \
"$@"
rendered = ${directory:services}/mariadb rendered = ${directory:services}/mariadb
template = {{ parameter_dict['template-mysqld-wrapper'] }}
context =
key defaults_file my-cnf:rendered
key datadir my-cnf-parameters:data-directory
key environ :environ
environ =
ODBCSYSINI='${my-cnf-parameters:etc-directory}'
LD_LIBRARY_PATH=$${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}'{{ parameter_dict['unixodbc-location'] }}/lib'
{%- for variable in slapparameter_dict.get('environment-variables', ()) %}
{{ variable }}
{%- endfor %}
[odbc-ini-text] [odbc-ini-text]
text = {{ dumps(slapparameter_dict.get('odbc-ini', '').encode('base64')) }} text = {{ dumps(slapparameter_dict.get('odbc-ini', '').encode('base64')) }}
...@@ -265,7 +258,6 @@ tmp = ${buildout:directory}/tmp ...@@ -265,7 +258,6 @@ tmp = ${buildout:directory}/tmp
backup = ${:srv}/backup backup = ${:srv}/backup
mariadb-backup-full = ${:backup}/mariadb-full mariadb-backup-full = ${:backup}/mariadb-full
mariadb-backup-incremental = ${:backup}/mariadb-incremental mariadb-backup-incremental = ${:backup}/mariadb-incremental
mariadb-data = ${:srv}/mariadb
mariadb-ssl = ${:etc}/mariadb-ssl mariadb-ssl = ${:etc}/mariadb-ssl
var = ${buildout:directory}/var var = ${buildout:directory}/var
log = ${:var}/log log = ${:var}/log
...@@ -275,7 +267,7 @@ slowquery = ${monitor-directory:private}/slowquery_digest ...@@ -275,7 +267,7 @@ slowquery = ${monitor-directory:private}/slowquery_digest
[{{ section('resiliency-exclude-file') }}] [{{ section('resiliency-exclude-file') }}]
# Generate rdiff exclude file in case of resiliency # Generate rdiff exclude file in case of resiliency
< = jinja2-template-base < = jinja2-template-base
template = {{ 'inline:{{ "${directory:mariadb-data}/**\\n${directory:mariadb-backup-incremental}/**\\n${directory:log}/**\\n${directory:tmp}/**\\n" }}' }} template = {{ 'inline:{{ "${my-cnf-parameters:data-directory}/**\\n${directory:mariadb-backup-incremental}/**\\n${directory:log}/**\\n${directory:tmp}/**\\n" }}' }}
rendered = ${directory:srv}/exporter.exclude rendered = ${directory:srv}/exporter.exclude
[{{ section("resiliency-identity-signature-script")}}] [{{ section("resiliency-identity-signature-script")}}]
...@@ -295,7 +287,7 @@ rendered = ${directory:bin}/start-clone-from-backup ...@@ -295,7 +287,7 @@ rendered = ${directory:bin}/start-clone-from-backup
context = context =
key dash dash:dash key dash dash:dash
key client binary-wrap-mysql:wrapper-path key client binary-wrap-mysql:wrapper-path
key data_directory directory:mariadb-data key data_directory my-cnf-parameters:data-directory
key pid_file my-cnf-parameters:pid-file key pid_file my-cnf-parameters:pid-file
key server mysqld:rendered key server mysqld:rendered
key update update-mysql:output key update update-mysql:output
...@@ -309,7 +301,7 @@ rendered = ${directory:bin}/restore-from-backup ...@@ -309,7 +301,7 @@ rendered = ${directory:bin}/restore-from-backup
context = context =
key dash dash:dash key dash dash:dash
key mysql_executable binary-wrap-mysql:wrapper-path key mysql_executable binary-wrap-mysql:wrapper-path
key mariadb_data_directory directory:mariadb-data key mariadb_data_directory my-cnf-parameters:data-directory
key mariadb_backup_directory directory:mariadb-backup-full key mariadb_backup_directory directory:mariadb-backup-full
key pid_file my-cnf-parameters:pid-file key pid_file my-cnf-parameters:pid-file
key binlog_path my-cnf-parameters:binlog-path key binlog_path my-cnf-parameters:binlog-path
......
...@@ -141,6 +141,7 @@ gzip-location = {{ gzip_location }} ...@@ -141,6 +141,7 @@ gzip-location = {{ gzip_location }}
mariadb-location = {{ mariadb_location }} mariadb-location = {{ mariadb_location }}
template-my-cnf = {{ template_my_cnf }} template-my-cnf = {{ template_my_cnf }}
template-mariadb-initial-setup = {{ template_mariadb_initial_setup }} template-mariadb-initial-setup = {{ template_mariadb_initial_setup }}
template-mysqld-wrapper = {{ template_mysqld_wrapper }}
link-binary = {{ dumps(mariadb_link_binary) }} link-binary = {{ dumps(mariadb_link_binary) }}
mariadb-resiliency-after-import-script = {{ mariadb_resiliency_after_import_script }} mariadb-resiliency-after-import-script = {{ mariadb_resiliency_after_import_script }}
mariadb-slow-query-report-script = {{ mariadb_slow_query_report_script }} mariadb-slow-query-report-script = {{ mariadb_slow_query_report_script }}
......
...@@ -148,7 +148,7 @@ slapos.toolbox = 0.94 ...@@ -148,7 +148,7 @@ slapos.toolbox = 0.94
stevedore = 1.21.0 stevedore = 1.21.0
subprocess32 = 3.5.3 subprocess32 = 3.5.3
unicodecsv = 0.14.1 unicodecsv = 0.14.1
xml-marshaller = 0.9.7 xml-marshaller = 1.0.2
paramiko = 2.1.3 paramiko = 2.1.3
CacheControl = 0.12.5 CacheControl = 0.12.5
msgpack = 0.6.1 msgpack = 0.6.1
...@@ -195,7 +195,7 @@ enum34 = 1.1.6 ...@@ -195,7 +195,7 @@ enum34 = 1.1.6
# Required by: # Required by:
# slapos.toolbox==0.94 # slapos.toolbox==0.94
erp5.util = 0.4.59.1 erp5.util = 0.4.60
# Required by: # Required by:
# slapos.toolbox==0.94 # slapos.toolbox==0.94
...@@ -228,7 +228,7 @@ lockfile = 0.12.2 ...@@ -228,7 +228,7 @@ lockfile = 0.12.2
# Required by: # Required by:
# slapos.core==1.4.26 # slapos.core==1.4.26
# XXX 'slapos node format' raises an exception with netifaces 0.10.5. # XXX 'slapos node format' raises an exception with netifaces 0.10.5.
netifaces = 0.10.4 netifaces = 0.10.7
# Required by: # Required by:
# cryptography==1.8.1 # cryptography==1.8.1
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment