Commit 430cbc77 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Update Release Candidate

parents 93f7f494 85a521b6
[buildout]
extends = ../xz-utils/buildout.cfg
parts = bash-completion
[bash-completion]
recipe = slapos.recipe.cmmi
shared = true
url = https://github.com/scop/bash-completion/releases/download/2.11/bash-completion-2.11.tar.xz
md5sum = 2514c6772d0de6254758b98c53f91861
environment =
PATH=${xz-utils:location}/bin:%(PATH)s
...@@ -52,13 +52,23 @@ environment-extra = ...@@ -52,13 +52,23 @@ environment-extra =
[golang1.14] [golang1.14]
<= golang-common <= golang-common
url = https://golang.org/dl/go1.14.10.src.tar.gz url = https://golang.org/dl/go1.14.12.src.tar.gz
md5sum = c0b485e8ac8b1978b5164e3e200150fa md5sum = ae124560e0185a82883b57cc9662aa73
# go1.14 needs go1.4 to bootstrap # go1.14 needs go1.4 to bootstrap
environment-extra = environment-extra =
GOROOT_BOOTSTRAP=${golang14:location} GOROOT_BOOTSTRAP=${golang14:location}
[golang1.15]
<= golang-common
url = https://golang.org/dl/go1.15.5.src.tar.gz
md5sum = 12a60d319ff58c03c323e8dcc125ae81
# go1.15 needs go1.4 to bootstrap
environment-extra =
GOROOT_BOOTSTRAP=${golang14:location}
# ---- infrastructure to build Go workspaces / projects ---- # ---- infrastructure to build Go workspaces / projects ----
# gowork is a top-level section representing workspace # gowork is a top-level section representing workspace
......
...@@ -25,7 +25,7 @@ parts = ...@@ -25,7 +25,7 @@ parts =
install = install =
lab.nexedi.com/nexedi/helloweb/go/... lab.nexedi.com/nexedi/helloweb/go/...
golang = ${golang1.14:location} golang = ${golang1.15:location}
# -*- go -*- # -*- go -*-
[helloweb-go] [helloweb-go]
......
[buildout]
extends =
../pygolang/buildout.cfg
../git/buildout.cfg
[nxdtest]
recipe = zc.recipe.egg:scripts
eggs = ${pygolang:egg}
${nxdtest-egg:egg}
scripts = nxdtest
# convenience for nxdtest users
exe = ${buildout:bin-directory}/nxdtest
[nxdtest-egg]
recipe = zc.recipe.egg:develop
setup = ${nxdtest-repository:location}
egg = nxdtest
[nxdtest-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/nxdtest.git
revision = 40e2c4ab
location = ${buildout:parts-directory}/nxdtest
git-executable = ${git:location}/bin/git
# SlapOS software release to test nxdtest on Nexedi testing infrastructure.
[buildout]
extends =
../defaults.cfg
test.cfg
[python]
part = python3
# SlapOS software release to test nxdtest on Nexedi testing infrastructure.
[buildout]
extends =
../../stack/nxdtest.cfg
../pytest/buildout.cfg
parts =
python-interpreter
slapos-cookbook
instance.cfg
# override to install test extra
[nxdtest-egg]
egg = nxdtest[test]
[python-interpreter]
eggs +=
${pytest:eggs}
${nxdtest:eggs}
# env.sh for interpreter to be on $PATH.
[nxdtest-env.sh]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:_buildout_section_name_}
template = inline:
export PS1="(nxdtest-env) $PS1"
export PATH=${buildout:bin-directory}:$PATH
# instance to run nxdtest.
[instance.cfg]
<= jinja2-template
template = inline:
[buildout]
extends = ${nxdtest-instance.cfg:rendered}
[runTestSuite]
env.sh = ${nxdtest-env.sh:rendered}
workdir = ${nxdtest-repository:location}
...@@ -37,6 +37,14 @@ interpreter = python ...@@ -37,6 +37,14 @@ interpreter = python
# interpreter code that buildout generates cannot process `-m pytest --<pytest-option>` # interpreter code that buildout generates cannot process `-m pytest --<pytest-option>`
# -> use pymain from gpython to workaround that. # -> use pymain from gpython to workaround that.
initialization = initialization =
# minimal support for -O argument.
if '-O' in sys.argv:
import os
with open(sys.argv[0]) as f:
executable = f.readlines()[0][2:-1]
sys.argv.remove('-O')
os.execve(executable, sys.argv, dict(os.environ, PYTHONOPTIMIZE='1'))
# set sys.executable to self, so that subprocess and friends go through us # set sys.executable to self, so that subprocess and friends go through us
# and this way spawn children with correct sys.path where all eggs that # and this way spawn children with correct sys.path where all eggs that
# parent have are present. TODO consider migrating this into pymain # parent have are present. TODO consider migrating this into pymain
......
# SlapOS software release to test pygolang/py3 on Nexedi testing infrastructure.
[buildout]
extends = test.cfg
[python]
part = python3
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
[buildout] [buildout]
extends = extends =
../pygolang/buildout.cfg ../pygolang/buildout.cfg
../numpy/buildout.cfg
../git/buildout.cfg ../git/buildout.cfg
parts = parts =
...@@ -27,6 +28,11 @@ setup-eggs = ...@@ -27,6 +28,11 @@ setup-eggs =
${pygolang:egg}[pyx.build] ${pygolang:egg}[pyx.build]
environment = wendelin.core-env environment = wendelin.core-env
# dependent eggs that must come through in-tree recipes
depends =
${numpy:egg}
[wendelin.core-env] [wendelin.core-env]
# wendelin.core needs git(dev) to build # wendelin.core needs git(dev) to build
PATH = ${git:location}/bin:%(PATH)s PATH = ${git:location}/bin:%(PATH)s
......
...@@ -36,7 +36,7 @@ class Recipe(GenericBaseRecipe): ...@@ -36,7 +36,7 @@ class Recipe(GenericBaseRecipe):
self.path_list = [] self.path_list = []
options = self.options.copy() options = self.options.copy()
del options['recipe'] del options['recipe']
CONFIG = {k.replace('-', '_'): v for k, v in options.iteritems()} CONFIG = {k.replace('-', '_'): v for k, v in options.items()}
CONFIG['PATH'] = os.environ['PATH'] CONFIG['PATH'] = os.environ['PATH']
if self.options['instance-dict']: if self.options['instance-dict']:
...@@ -44,7 +44,7 @@ class Recipe(GenericBaseRecipe): ...@@ -44,7 +44,7 @@ class Recipe(GenericBaseRecipe):
config_instance_dict.add_section('instance_dict') config_instance_dict.add_section('instance_dict')
instance_dict = json.loads(self.options['instance-dict']) instance_dict = json.loads(self.options['instance-dict'])
for k ,v in instance_dict.iteritems(): for k ,v in instance_dict.items():
config_instance_dict.set('instance_dict', k, v) config_instance_dict.set('instance_dict', k, v)
value = io.StringIO() value = io.StringIO()
config_instance_dict.write(value) config_instance_dict.write(value)
......
import functools
import os
import shutil
import tempfile
import unittest
from six.moves import configparser
import zc.buildout.testing
class UserInfoTest(unittest.TestCase):
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.tmp_dir)
self.get_temp_path = functools.partial(os.path.join, self.tmp_dir)
self.buildout = buildout = zc.buildout.testing.Buildout()
buildout['slap-connection'] = {
'computer-id': 'computer-id',
'server-url': 'https://slapos.example.com',
}
# dummy defaults
buildout['erp5testnode'] = {
'apache-binary': '/bin/httpd',
'apache-htpasswd': '/bin/htpasswd',
'apache-mime-file': '/etc/mime.types',
'apache-modules-dir': '/srv/modules',
'frontend-url': 'https://example.com/',
'git-binary': '/bin/git',
'httpd-cert-file': 'etc/httpd-public.crt',
'httpd-key-file': 'etc/httpd-public.key',
'httpd-conf-file': 'etc/httpd.conf',
'httpd-ip': '::1',
'httpd-lock-file': 'var/run/httpd.lock',
'httpd-log-directory': 'var/log',
'httpd-pid-file': 'var/run/httpd.pid',
'httpd-port': '9080',
'httpd-software-access-port': '9081',
'httpd-software-directory': 'srv/software',
'httpd-wrapper': 'bin/httpd',
'instance-dict': '',
'ipv4-address': '127.0.0.1',
'ipv6-address': '::1',
'keep-log-days': '3',
'log-directory': 'var/log/testnode',
'log-file': 'var/log/erp5testnode.log',
'log-frontend-url': 'https://example.com',
'node-quantity': '2',
'proxy-host': '127.0.0.1',
'proxy-port': '5000',
'recipe': 'slapos.cookbook:erp5testnode',
'run-directory': 'var/run/testnode',
'shared-part-list': 'srv/shared',
'slapos-binary': '/bin/slapos',
'slapos-directory': 'srv/slapos',
'software-directory': 'srv/software',
'software-path-list': '[""]',
'srv-directory': 'srv',
'test-node-title': 'TEST NODE TITLE',
'test-suite-directory': 'srv/test_suite',
'test-suite-master-url': 'https://testnode.example.com',
'testnode': '/bin/testnode',
'working-directory': 'srv/testnode',
'wrapper': 'bin/erp5testnode-service',
'zip-binary': '/bin/zip',
}
# values for test
buildout['erp5testnode']['configuration-file'] = self.get_temp_path(
'configuration-file')
buildout['erp5testnode']['wrapper'] = self.get_temp_path('wrapper')
buildout['erp5testnode']['httpd-conf-file'] = self.get_temp_path(
'httpd-conf-file')
buildout['erp5testnode']['httpd-wrapper'] = self.get_temp_path(
'httpd-wrapper')
buildout['erp5testnode']['log-directory'] = self.get_temp_path(
'log-directory')
os.mkdir(self.get_temp_path('log-directory'))
# software URLs are given as a json encoded string
buildout['erp5testnode'][
'software-path-list'] = '["https://example.com/slapos/software.cfg", "https://example.com/slapos/another-software.cfg"]'
from slapos.recipe import erp5testnode
self.recipe = erp5testnode.Recipe(
buildout,
'erp5testnode',
buildout['erp5testnode'],
)
def test_installed_paths(self):
self.assertEqual(
sorted(self.recipe.install()),
sorted([
self.get_temp_path('configuration-file'),
self.get_temp_path('wrapper'),
self.get_temp_path('httpd-conf-file'),
self.get_temp_path('httpd-wrapper'),
]))
def test_configuration_file(self):
self.recipe.install()
parser = configparser.ConfigParser()
parser.read(self.get_temp_path('configuration-file'))
# this is generally a valid configparser file.
self.assertEqual(parser.get('testnode', 'slapos_directory'), 'srv/slapos')
# software URLs are specified comma separated (XXX not sure it was
# good idea, but it's like this)
self.assertEqual(parser.get('software_list', 'path_list'),
"https://example.com/slapos/software.cfg,https://example.com/slapos/another-software.cfg")
def test_log_directory_apache(self):
self.recipe.install()
self.assertTrue(
os.path.exists(self.get_temp_path('log-directory', 'index.html')))
# apache to expose log directory
with open(self.get_temp_path('httpd-conf-file')) as f:
self.assertIn(
'DocumentRoot "{}"'.format(self.get_temp_path('log-directory')),
f.read())
# wrapper references the config file
with open(self.get_temp_path('httpd-wrapper')) as f:
self.assertIn(self.get_temp_path('httpd-conf-file'), f.read())
...@@ -14,4 +14,4 @@ ...@@ -14,4 +14,4 @@
# not need these here). # not need these here).
[template-instance] [template-instance]
filename = instance.cfg filename = instance.cfg
md5sum = aa91245be4b946d0537f6e007f0d72ea md5sum = 0974248c0b0ad5da45670386a5301e47
# Jinja2 template of a buildout file to create runnable scripts # Jinja2 template of a buildout file to create runnable scripts
# #
# Received variables # Received variables
# - software_dir: location where this repository was cloned to # - project_dir: location where this repository was cloned to
# - instance_dir: location where this 'instance.cfg' is gettin rendered (same as ${buildout:directory}) # - software_dir: location where this 'instance.cfg' is gettin rendered (same as ${buildout:directory})
[buildout] [buildout]
parts = parts =
...@@ -12,6 +12,7 @@ parts = ...@@ -12,6 +12,7 @@ parts =
template-full-build-script template-full-build-script
template-sudo-wrapper template-sudo-wrapper
template-cros-sdk-wrapper template-cros-sdk-wrapper
resiliency-exclude-file
extends = {{ monitor_template }} extends = {{ monitor_template }}
# standard declaration of eggs directories # standard declaration of eggs directories
...@@ -30,7 +31,7 @@ git-executable = {{ git_path }}/bin/git ...@@ -30,7 +31,7 @@ git-executable = {{ git_path }}/bin/git
[customize-path] [customize-path]
# add depot tools directory (for cros_sdk binary among others) and git directory to the path # add depot tools directory (for cros_sdk binary among others) and git directory to the path
command = command =
export PATH="${directory:wrapper_dir}":"{{ git_path }}/bin":"{{ curl_path }}/bin":"${depot-tools:location}":"{{thin_provisioning_tools}}/sbin":"$PATH"; export PATH="${directory:wrapper_dir}":"{{ software_dir }}/bin":"{{ git_path }}/bin":"{{ curl_path }}/bin":"${depot-tools:location}":"{{thin_provisioning_tools}}/sbin":"$PATH";
[nayuos-ebuilds] [nayuos-ebuilds]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
...@@ -56,8 +57,10 @@ cert = ${slap_connection:cert_file} ...@@ -56,8 +57,10 @@ cert = ${slap_connection:cert_file}
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
log = ${buildout:directory}/var/log log = ${buildout:directory}/var/log
run = ${buildout:directory}/etc/run run = ${buildout:directory}/etc/run
backup = ${buildout:directory}/srv/backup
wrapper_dir = ${buildout:directory}/wrapper_bin wrapper_dir = ${buildout:directory}/wrapper_bin
cros_location = ${buildout:directory}/parts/chromiumos cros_location = ${buildout:directory}/parts/chromiumos
images = ${:cros_location}/images
[bin] [bin]
# dummy section to hold references to frequently used binaries # dummy section to hold references to frequently used binaries
...@@ -79,7 +82,7 @@ config-command = ${promise-sudo-on-host-bin:wrapper-path} ...@@ -79,7 +82,7 @@ config-command = ${promise-sudo-on-host-bin:wrapper-path}
[template-sudo-wrapper] [template-sudo-wrapper]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = {{ software_dir }}/scripts/wrapper_sudo.in template = {{ project_dir }}/scripts/wrapper_sudo.in
rendered = ${directory:wrapper_dir}/sudo rendered = ${directory:wrapper_dir}/sudo
md5sum = ded5a92be4e37ec32eb9d3087d3e19bd md5sum = ded5a92be4e37ec32eb9d3087d3e19bd
mode = 0700 mode = 0700
...@@ -90,7 +93,7 @@ context = ...@@ -90,7 +93,7 @@ context =
[template-cros-sdk-wrapper] [template-cros-sdk-wrapper]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = {{ software_dir }}/scripts/wrapper_cros_sdk.in template = {{ project_dir }}/scripts/wrapper_cros_sdk.in
rendered = ${bin:wrapper_cros_sdk} rendered = ${bin:wrapper_cros_sdk}
md5sum = 7159fe3d5b85a283733cf686c4ee0a74 md5sum = 7159fe3d5b85a283733cf686c4ee0a74
mode = 0700 mode = 0700
...@@ -104,25 +107,32 @@ context = ...@@ -104,25 +107,32 @@ context =
# create the rendered script in the buildout-directory/etc/run # create the rendered script in the buildout-directory/etc/run
# (use jinja for templating) # (use jinja for templating)
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = {{ software_dir }}/scripts/cros_full_build.in template = {{ project_dir }}/scripts/cros_full_build.in
rendered = ${directory:run}/cros_full_build rendered = ${directory:run}/cros_full_build
md5sum = f3b3b5408b69f752f2221d3965caf590 #md5sum = f3b3b5408b69f752f2221d3965caf590
mode = 0700 mode = 0700
context = context =
key bash_path bin:bash key bash_path bin:bash
key instance_log_dir directory:log key instance_log_dir directory:log
key instance_dir buildout:directory
key cros_location directory:cros_location key cros_location directory:cros_location
key export_path_cmd customize-path:command key export_path_cmd customize-path:command
key branch parameters:configuration.branch key branch parameters:configuration.branch
key boards_list parameters:configuration.boards key boards_list parameters:configuration.boards
key keep_cache parameters:configuration.keep_cache key keep_cache parameters:configuration.keep_cache
key ebuilds_dir nayuos-ebuilds:location key ebuilds_dir nayuos-ebuilds:location
raw scripts_dir {{ software_dir }}/scripts raw scripts_dir {{ project_dir }}/scripts
raw logo_dir {{ software_dir }}/logo raw logo_dir {{ project_dir }}/logo
raw wallpaper_dir {{ software_dir }}/wallpaper raw wallpaper_dir {{ project_dir }}/wallpaper
raw patch_dir {{ software_dir }}/patch raw patch_dir {{ project_dir }}/patch
# packages to be copied from our overlay into ChromiuOS' overlay # packages to be copied from our overlay into ChromiuOS' overlay
# those packages will be marked as dependency of root filesystem # those packages will be marked as dependency of root filesystem
raw nayu_dev_rootfs_packages app-misc/nayuos-chromium-policy raw nayu_dev_rootfs_packages app-misc/nayuos-chromium-policy
# those packages will be marked as dependency of the chromium build itself # those packages will be marked as dependency of the chromium build itself
raw nayu_dev_packages dev-vcs/git dev-python/flask dev-python/virtualenv sys-fs/cryptsetup raw nayu_dev_packages dev-vcs/git dev-python/flask dev-python/virtualenv sys-fs/cryptsetup
[resiliency-exclude-file]
# Generate rdiff exclude file for resiliency
recipe = collective.recipe.template
input = inline: parts/chromiumos/release*
output = ${directory:srv}/exporter.exclude
...@@ -8,19 +8,29 @@ extends = ...@@ -8,19 +8,29 @@ extends =
../../component/libexpat/buildout.cfg ../../component/libexpat/buildout.cfg
../../component/libaio/buildout.cfg ../../component/libaio/buildout.cfg
../../component/boost-lib/buildout.cfg ../../component/boost-lib/buildout.cfg
../../component/rsync/buildout.cfg
parts += parts +=
eggs
slapos-cookbook slapos-cookbook
template-instance template-instance
git git
curl curl
thin-provisioning-tools thin-provisioning-tools
versions = versions [gcc]
min_version = 0
[versions] [python]
slapos.recipe.template = 4.4 # we need python3 for repo/cros_sdk command
part = python3
[eggs]
# name is python3 so that cros_sdk uses it automatically through "/usr/bin/env python3"
recipe = zc.recipe.egg
interpreter = python3
eggs =
setuptools
[template-instance] [template-instance]
# jinja2 render instance.cfg file in the buildout directory # jinja2 render instance.cfg file in the buildout directory
...@@ -29,10 +39,11 @@ template = ${:_profile_base_location_}/${:filename} ...@@ -29,10 +39,11 @@ template = ${:_profile_base_location_}/${:filename}
rendered = ${buildout:directory}/${:filename} rendered = ${buildout:directory}/${:filename}
mode = 0644 mode = 0644
context = context =
key software_dir :_profile_base_location_ key project_dir :_profile_base_location_
key instance_dir buildout:directory key software_dir buildout:directory
key curl_path curl:location key curl_path curl:location
key git_path git:location key git_path git:location
key rsync_path rsync:location
key eggs_directory buildout:eggs-directory key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory key develop_eggs_directory buildout:develop-eggs-directory
key thin_provisioning_tools thin-provisioning-tools:location key thin_provisioning_tools thin-provisioning-tools:location
...@@ -56,3 +67,6 @@ dependencies = ...@@ -56,3 +67,6 @@ dependencies =
libexpat libexpat
libaio libaio
boost-lib boost-lib
[versions]
slapos.recipe.template = 4.4
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[neotest-env.sh] [neotest-env.sh]
filename = neotest-env.sh.in filename = neotest-env.sh.in
md5sum = bd46e95f1cea62c3b0082fe8c0c9c90b md5sum = dd41bd736561b7d59b5ec7e0ab7cde98
[neotest] [neotest]
filename = neotest.in filename = neotest.in
......
...@@ -9,24 +9,7 @@ PATH="${ioping:location}/bin:$PATH" ...@@ -9,24 +9,7 @@ PATH="${ioping:location}/bin:$PATH"
PATH="${lmbench:location}/bin:$PATH" PATH="${lmbench:location}/bin:$PATH"
PATH="${mariadb:location}/bin:$PATH" PATH="${mariadb:location}/bin:$PATH"
PATH="${mariadb:location}/scripts:$PATH" PATH="${mariadb:location}/scripts:$PATH"
PATH="${python:location}/bin:$PATH" PATH="${buildout:bin-directory}/:$PATH" # for python with eggs
# add all eggs and develop-eggs to py path
#
# XXX better leverage something like zc.recipe.egg to generate eggs path buildout way
# XXX see e.g. https://lab.nexedi.com/nexedi/slapos/merge_requests/242#note_49644 for why
# TODO(kirr) try doing so
PYTHONPATH=${buildout:develop-eggs-directory}:$PYTHONPATH # for .egg-link to be found by setuptools
for egglink in ${buildout:develop-eggs-directory}/*.egg-link; do
PYTHONPATH=`cat $egglink |head -1`:$PYTHONPATH
done
for egg in ${buildout:develop-eggs-directory}/*.egg ${buildout:eggs-directory}/*.egg; do
export PYTHONPATH=$egg:$PYTHONPATH
done
export PATH export PATH
export PYTHONPATH
export PS1="(neotest-`basename ${buildout:directory}`) $PS1" export PS1="(neotest-`basename ${buildout:directory}`) $PS1"
...@@ -18,7 +18,6 @@ extends = ...@@ -18,7 +18,6 @@ extends =
../../component/coreutils/buildout.cfg ../../component/coreutils/buildout.cfg
../../component/util-linux/buildout.cfg ../../component/util-linux/buildout.cfg
../../component/pygolang/buildout.cfg ../../component/pygolang/buildout.cfg
../../component/numpy/buildout.cfg
../../stack/nxdtest.cfg ../../stack/nxdtest.cfg
...@@ -36,8 +35,12 @@ parts = ...@@ -36,8 +35,12 @@ parts =
neoppod-develop neoppod-develop
neoppod neoppod
wendelin.core-dev wendelin.core-dev
eggs scripts
# for ZEO scripts (runzeo)
ZEO/scripts
neotest-python
neotest-env.sh neotest-env.sh
neotest neotest
...@@ -100,23 +103,21 @@ template = inline: ...@@ -100,23 +103,21 @@ template = inline:
# eggs: # bin/python is preinstalled with sys.path to wendelin.core & friends.
[eggs] [neotest-python]
recipe = zc.recipe.egg <= python-interpreter
eggs = eggs +=
# wendelin.core still requires ZODB3 but having ZODB4 or 5 installed satisfies latest ZODB3 ${wendelin.core-dev:egg}
ZODB3 ${neoppod:eggs}
# also for wc
${numpy:egg}
# to install not only wendelin.core modules but also scripts
wendelin.core
# for ZEO scripts (runzeo)
${ZEO:egg}
# for e.g. tcpu.py
${pygolang:egg}
# for instance # for instance
plone.recipe.command plone.recipe.command
[scripts]
recipe = zc.recipe.egg:scripts
eggs =
# to install not only wendelin.core modules but also scripts
${wendelin.core-dev:egg}
# wendelin.core: latest not yet released # wendelin.core: latest not yet released
[wendelin.core-repository] [wendelin.core-repository]
......
...@@ -18,15 +18,15 @@ md5sum = fddea033e1aa9d6147a1a47bd7cc4b62 ...@@ -18,15 +18,15 @@ md5sum = fddea033e1aa9d6147a1a47bd7cc4b62
[template-powerdns] [template-powerdns]
filename = instance-powerdns.cfg filename = instance-powerdns.cfg
md5sum = 9cd4e436fa432f37b9f8f4de8350581b md5sum = 0920200cb05a68b1b4a161a927d9488f
[template-pdns-configuration] [template-pdns-configuration]
_update_hash_filename_ = template/pdns.conf.jinja2 _update_hash_filename_ = template/pdns.conf.jinja2
md5sum = e45d72de87b4adb89c195ba463be4077 md5sum = 20c37ea06a8fa405bc02470d5115fd11
[template-dns-replicate] [template-dns-replicate]
_update_hash_filename_ = instance-powerdns-replicate.cfg.jinja2 _update_hash_filename_ = instance-powerdns-replicate.cfg.jinja2
md5sum = a23e241a236f90ae1afbb5bd5ba0b32d md5sum = c2bd424f588ad57d37f4cf1329734fb6
[iso-list] [iso-list]
_update_hash_filename_ = template/zz.countries.nexedi.dk.rbldnsd _update_hash_filename_ = template/zz.countries.nexedi.dk.rbldnsd
......
...@@ -43,6 +43,7 @@ context = ...@@ -43,6 +43,7 @@ context =
{% set dns_name = 'ns%s' % i -%} {% set dns_name = 'ns%s' % i -%}
{% set dns_domain = dns_name_template_string % i %} {% set dns_domain = dns_name_template_string % i %}
{% set request_section_title = 'request-%s' % dns_name -%} {% set request_section_title = 'request-%s' % dns_name -%}
{% set promise_section_title = 'promise-listen-port-%s' % dns_name -%}
{% set sla_key = "-sla-%s-" % i -%} {% set sla_key = "-sla-%s-" % i -%}
{% set sla_key_length = sla_key | length %} {% set sla_key_length = sla_key | length %}
{% set sla_parameters = [] %} {% set sla_parameters = [] %}
...@@ -55,6 +56,7 @@ context = ...@@ -55,6 +56,7 @@ context =
{% do dns_domain_list.append(dns_domain) -%} {% do dns_domain_list.append(dns_domain) -%}
{% do dns_section_list.append(request_section_title) -%} {% do dns_section_list.append(request_section_title) -%}
{% do part_list.append(request_section_title) -%} {% do part_list.append(request_section_title) -%}
{% do part_list.append(promise_section_title) -%}
[{{request_section_title}}] [{{request_section_title}}]
<= replicate <= replicate
...@@ -69,6 +71,15 @@ config-soa = {{ "%s,%s" % (dns_domain, server_admin) }} ...@@ -69,6 +71,15 @@ config-soa = {{ "%s,%s" % (dns_domain, server_admin) }}
sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }} sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
{% endfor -%} {% endfor -%}
[{{promise_section_title}}]
<= monitor-promise-base
module = check_port_listening
name = pdns-port-listening.py
{% set ipv6 = '${' ~ request_section_title ~ ':connection-powerdns-ipv6}' -%}
config-hostname = {{ipv6}}
{% set port = '${' ~ request_section_title ~ ':connection-powerdns-port}' -%}
config-port = {{port}}
{% do monitor_url_list.append('${' ~ request_section_title ~ ':connection-monitor-base-url}') -%} {% do monitor_url_list.append('${' ~ request_section_title ~ ':connection-monitor-base-url}') -%}
{% endfor -%} {% endfor -%}
...@@ -101,7 +112,7 @@ software-url = {{ slapparameter_dict.pop(dns_software_url_key) }} ...@@ -101,7 +112,7 @@ software-url = {{ slapparameter_dict.pop(dns_software_url_key) }}
software-url = ${slap-connection:software-release-url} software-url = ${slap-connection:software-release-url}
{% endif %} {% endif %}
software-type = {{dns_type}} software-type = {{dns_type}}
return = private-ipv4 public-ipv4 slave-instance-information-list monitor-base-url return = slave-instance-information-list monitor-base-url powerdns-ipv6 powerdns-port powerdns-ipv4
config-server-admin = {{ server_admin }} config-server-admin = {{ server_admin }}
config-ns-record = {{ ns_record }} config-ns-record = {{ ns_record }}
{% for parameter, value in slapparameter_dict.items() -%} {% for parameter, value in slapparameter_dict.items() -%}
...@@ -114,12 +125,17 @@ config-monitor-password = ${monitor-htpasswd:passwd} ...@@ -114,12 +125,17 @@ config-monitor-password = ${monitor-htpasswd:passwd}
[publish-information] [publish-information]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
domain = {{ slapparameter_dict.get('domain') }}
slave-amount = {{ slave_instance_list | length }} slave-amount = {{ slave_instance_list | length }}
ns-record = {{ ns_record }} ns-record = {{ ns_record }}
monitor-url = ${monitor-publish-parameters:monitor-url} monitor-url = ${monitor-publish-parameters:monitor-url}
monitor-user = ${monitor-publish-parameters:monitor-user} monitor-user = ${monitor-publish-parameters:monitor-user}
monitor-password = ${monitor-publish-parameters:monitor-password} monitor-password = ${monitor-publish-parameters:monitor-password}
{% for dns_name, dns_section in zip(dns_list, dns_section_list) -%}
{% set dns_ipv6 = '${' ~ dns_section ~ ':connection-powerdns-ipv6}' -%}
{% set dns_port = '${' ~ dns_section ~ ':connection-powerdns-port}' -%}
{{ dns_name }}-ipv6 = {{ dns_ipv6 }}
{{ dns_name }}-port = {{ dns_port }}
{% endfor -%}
{% set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') -%} {% set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
{% if monitor_interface_url -%} {% if monitor_interface_url -%}
monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url} monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}
......
...@@ -55,7 +55,7 @@ context = ...@@ -55,7 +55,7 @@ context =
# #
[pdns] [pdns]
configuration = $${pdns-directory:configuration}/pdns.conf configuration = $${pdns-directory:configuration}/pdns.conf
local-ipv4 = $${instance-parameter:ipv4-random} ipv4 = $${instance-parameter:ipv4-random}
ipv6 = $${instance-parameter:ipv6-random} ipv6 = $${instance-parameter:ipv6-random}
port = 5353 port = 5353
socket-directory = $${pdns-directory:socket-directory} socket-directory = $${pdns-directory:socket-directory}
...@@ -139,7 +139,7 @@ extra-context = ...@@ -139,7 +139,7 @@ extra-context =
<= monitor-promise-base <= monitor-promise-base
module = check_port_listening module = check_port_listening
name = pdns-port-listening.py name = pdns-port-listening.py
config-hostname = $${pdns:local-ipv4} config-hostname = $${pdns:ipv4}
config-port = $${pdns:port} config-port = $${pdns:port}
[publish-connection-informations] [publish-connection-informations]
...@@ -147,6 +147,9 @@ recipe = slapos.cookbook:publish ...@@ -147,6 +147,9 @@ recipe = slapos.cookbook:publish
monitor-url = $${monitor-publish-parameters:monitor-url} monitor-url = $${monitor-publish-parameters:monitor-url}
monitor-user = $${monitor-publish-parameters:monitor-user} monitor-user = $${monitor-publish-parameters:monitor-user}
monitor-password = $${monitor-publish-parameters:monitor-password} monitor-password = $${monitor-publish-parameters:monitor-password}
powerdns-ipv4 = $${pdns:ipv4}
powerdns-ipv6 = $${pdns:ipv6}
powerdns-port = $${pdns:port}
[buildout] [buildout]
parts = parts =
......
...@@ -55,7 +55,6 @@ PyRSS2Gen = 1.1 ...@@ -55,7 +55,6 @@ PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.4 slapos.recipe.template = 4.4
dnspython = 1.15.0
passlib = 1.7.1 passlib = 1.7.1
GitPython = 2.1.11 GitPython = 2.1.11
lockfile = 0.12.2 lockfile = 0.12.2
......
# ------------------------------------------------------------------------- # -------------------------------------------------------------------------
# Configure ip/port binding # Configure ip/port binding
local-address={{ pdns.get('local-ipv4') }} local-address={{ pdns.get('ipv4') }}
local-ipv6={{ pdns.get('ipv6') }} local-ipv6={{ pdns.get('ipv6') }}
local-port={{ pdns.get('port') }} local-port={{ pdns.get('port') }}
...@@ -10,7 +10,7 @@ socket-dir={{ pdns.get('socket-directory') }} ...@@ -10,7 +10,7 @@ socket-dir={{ pdns.get('socket-directory') }}
# Monitoring # Monitoring
webserver=yes webserver=yes
webserver-address={{ pdns.get('local-ipv4') }} webserver-address={{ pdns.get('ipv4') }}
webserver-port={{ pdns.get('webserver-port') }} webserver-port={{ pdns.get('webserver-port') }}
# These totally disable query+packet caching for all zones. This is necessary # These totally disable query+packet caching for all zones. This is necessary
......
...@@ -47,6 +47,7 @@ setup(name=name, ...@@ -47,6 +47,7 @@ setup(name=name,
'erp5.util', 'erp5.util',
'supervisor', 'supervisor',
'psutil', 'psutil',
'dnspython',
], ],
zip_safe=True, zip_safe=True,
test_suite='test', test_suite='test',
......
...@@ -25,7 +25,12 @@ ...@@ -25,7 +25,12 @@
# #
############################################################################## ##############################################################################
import dns.edns
import dns.message
import dns.query
import http.client
import os import os
import requests
from slapos.recipe.librecipe import generateHashFromFiles from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
...@@ -35,10 +40,64 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass( ...@@ -35,10 +40,64 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath( os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))) os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
DNS_PORT = 5353
AFRICAN_SUBNET = '41.0.0.0'
CHINA_TELECOM_SUBNET = '1.0.32.0'
CHINA_UNICOM_SUBNET = '116.181.0.0'
CHINA_MOBILE_SUBNET = '112.21.42.5'
EAST_ASIAN_SUBNET = '1.11.0.0'
EUROPEAN_SUBNET = '5.42.160.0'
HONG_KONG_SUBNET = '1.32.192.0'
JAPANESE_SUBNET = '1.0.16.0'
NORTH_AMERICAN_SUBNET = '3.0.0.0'
OCEANIAN_SUBNET = '1.120.0.0'
SOUTH_AMERICAN_SUBNET = '45.70.188.0'
WEST_ASIAN_SUBNET = '46.70.0.0'
class PowerDNSTestCase(SlapOSInstanceTestCase): class PowerDNSTestCase(SlapOSInstanceTestCase):
# power dns uses sockets and need shorter paths on test nodes. # power dns uses sockets and need shorter paths on test nodes.
__partition_reference__ = 'pdns' __partition_reference__ = 'pdns'
default_zone = 'domain.com'
# focus to test connexion parameters only depending on PowerDNS
def getPowerDNSParameterDict(self, parameter_dict):
new_parameter_dict = {}
for key, value in parameter_dict.items():
if key in [
'ns-record',
'ns1-port',
'ns1-ipv6',
'slave-amount',
]:
new_parameter_dict[key] = value
return new_parameter_dict
def getPowerDNSConnexionParameterDict(self):
return self.getPowerDNSParameterDict(
self.requestDefaultInstance().getConnectionParameterDict()
)
def _test_parameter_dict(self, zone=None, dns_quantity=1, slave_amount=0):
if zone is None:
zone = self.default_zone
parameter_dict = self.getPowerDNSConnexionParameterDict()
expected_dict = {
'ns-record': '',
}
ns_record = ''
for replicate_nb in range(1, dns_quantity + 1):
prefix = 'ns%s' % replicate_nb
ns_record += prefix + '.%s' % zone
expected_dict[prefix + '-port'] = str(DNS_PORT)
expected_dict[prefix + '-ipv6'] = self._ipv6_address
expected_dict['ns-record'] = ns_record
expected_dict['slave-amount'] = str(slave_amount)
self.assertEqual(expected_dict, parameter_dict)
class ServicesTestCase(PowerDNSTestCase): class ServicesTestCase(PowerDNSTestCase):
...@@ -63,3 +122,194 @@ class ServicesTestCase(PowerDNSTestCase): ...@@ -63,3 +122,194 @@ class ServicesTestCase(PowerDNSTestCase):
expected_process_name = name.format(hash=h) expected_process_name = name.format(hash=h)
self.assertIn(expected_process_name, process_names) self.assertIn(expected_process_name, process_names)
class TestMonitorAccess(PowerDNSTestCase):
def test(self):
connection_parameter_dict = self.requestDefaultInstance()\
.getConnectionParameterDict()
monitor_base_url = connection_parameter_dict.get('monitor-url')
result = requests.get(
monitor_base_url, verify=False, auth=(
connection_parameter_dict.get('monitor-user'),
connection_parameter_dict.get('monitor-password')
)
)
self.assertEqual(
http.client.OK,
result.status_code
)
class TestMasterRequest(PowerDNSTestCase):
def test(self):
self._test_parameter_dict()
class TestMasterRequestDomain(PowerDNSTestCase):
@classmethod
def getInstanceParameterDict(cls):
return {
'zone': 'toto.example.com',
}
def test(self):
self._test_parameter_dict(zone=self.getInstanceParameterDict()['zone'])
class PowerDNSSlaveTestCase(PowerDNSTestCase):
@classmethod
def requestDefaultInstance(cls):
default_instance = super(PowerDNSSlaveTestCase, cls)\
.requestDefaultInstance()
cls.requestSlaves()
return default_instance
@classmethod
def requestSlaves(cls):
software_url = cls.getSoftwareURL()
software_type = cls.getInstanceSoftwareType()
for slave_reference, partition_parameter_kw in cls\
.getSlaveParameterDictDict().items():
cls.logger.debug(
'requesting slave "%s" software:%s parameters:%s',
slave_reference, software_url, partition_parameter_kw)
cls.slap.request(
software_release=software_url,
software_type=software_type,
partition_reference=slave_reference,
partition_parameter_kw=partition_parameter_kw,
shared=True
)
@classmethod
def getSlaveConnectionParameterDictList(cls):
parameter_dict_list = []
for slave_reference, partition_parameter_kw in cls\
.getSlaveParameterDictDict().items():
parameter_dict_list.append(cls.slap.request(
software_release=cls.getSoftwareURL(),
software_type=cls.getInstanceSoftwareType(),
partition_reference=slave_reference,
partition_parameter_kw=partition_parameter_kw,
shared=True
).getConnectionParameterDict())
return parameter_dict_list
@classmethod
def getSlaveParameterDictDict(cls):
return {
'slave-pdns1': {
'record': 'test1',
'origin': 'nexedi.com',
'default': 'test1.com.',
'africa': 'test1africa.com.',
'china-telecom': 'test1china-telecom.com.',
'china-unicom': 'test1china-unicom.com.',
'china-mobile': 'test1china-mobile.com.',
'east-asia': 'test1east-asia.com.',
'europe': 'test1europe.com.',
'hong-kong': 'test1hong-kong.com.',
'japan': 'test1japan.com.',
'north-america': 'test1north-america.com.',
'oceania': 'test1oceania.com.',
'south-america': 'test1south-america.com.',
'west-asia': 'test1west-asia.com.',
},
'slave-pdns2': {
'record': 'test2',
'origin': 'nexedi.com',
'default': 'test2.com.',
'china-telecom': 'test2china-telecom.com.',
'europe': 'test2europe.com.',
'japan': 'test2japan.com.',
}
}
def dns_query(self, domain_name, subnet):
message = dns.message.make_query(domain_name, 'A')
client_subnet_option = dns.edns.ECSOption(subnet)
message.use_edns(options=[client_subnet_option])
answer = dns.query.udp(message, self._ipv6_address, port=DNS_PORT)
return answer.find_rrset(
dns.message.ANSWER,
dns.name.from_text(domain_name),
dns.rdataclass.IN,
dns.rdatatype.CNAME
).to_text().split()[-1]
def _test_dns_resolver(self, zone):
slave_parameter_dict_dict = self.getSlaveParameterDictDict()
subnet_dict = {
'africa': AFRICAN_SUBNET,
'china-telecom': CHINA_TELECOM_SUBNET,
'china-unicom': CHINA_UNICOM_SUBNET,
'china-mobile': CHINA_MOBILE_SUBNET,
'east-asia': EAST_ASIAN_SUBNET,
'europe': EUROPEAN_SUBNET,
'hong-kong': HONG_KONG_SUBNET,
'japan': JAPANESE_SUBNET,
'north-america': NORTH_AMERICAN_SUBNET,
'oceania': OCEANIAN_SUBNET,
'south-america': SOUTH_AMERICAN_SUBNET,
'west-asia': WEST_ASIAN_SUBNET,
}
default_rr_dict = {
'europe': 'eu',
'africa': 'af',
'south-america': 'sa',
'north-america': 'na',
'china-telecom': 'cn-t',
'china-unicom': 'cn-u',
'china-mobile': 'cn-m',
'japan': 'jp',
'hong-kong': 'hk',
'east-asia': 'as',
'west-asia': 'eu',
'oceania': 'oc',
}
for slave_name in slave_parameter_dict_dict:
slave_parameter_dict = slave_parameter_dict_dict[slave_name]
domain_name = '%s.%s' % (slave_parameter_dict['record'], zone)
for region in subnet_dict:
self.assertEqual(
slave_parameter_dict.pop(
region,
'%s.%s.' % (default_rr_dict[region], slave_parameter_dict['origin'])
),
self.dns_query(domain_name, subnet_dict[region])
)
def _test(self, zone=None):
if zone is None:
zone = self.default_zone
self._test_parameter_dict(
zone=zone,
slave_amount=len(self.getSlaveParameterDictDict())
)
self._test_dns_resolver(zone)
class TestSlaveRequest(PowerDNSSlaveTestCase):
def test(self):
self._test()
class TestSlaveRequestDomain(PowerDNSSlaveTestCase):
@classmethod
def getInstanceParameterDict(cls):
return {
'zone': 'toto.example.com',
}
def test(self):
self._test(zone=self.getInstanceParameterDict()['zone'])
...@@ -31,15 +31,15 @@ slapos request --node=computer_guid=$COMP $INSTANCE_NAME $SR ...@@ -31,15 +31,15 @@ slapos request --node=computer_guid=$COMP $INSTANCE_NAME $SR
# and load this script to set environment variables # and load this script to set environment variables
source ( environment-script from step above ) source ( environment-script from step above )
# Clone a working copy somewhere # The source code is a git clone working copy on the instance
cd ~/srv/runner/project/ cd ~/srv/runner/instance/slappartXXX/parts/slapos/
git clone https://lab.nexedi.com/nexedi/slapos.git slapos_work
# change directory to the directory containing test for this software # change directory to the directory containing test for this software
cd ~/srv/runner/project/slapos_work/software/helloworld/test/ cd ./software/helloworld/test/
# make changes to test code or profile
# run test (with debugging features activated) # run test for helloworld software release (with debugging features activated)
SLAPOS_TEST_DEBUG= 1 python_for_test setup.py test SLAPOS_TEST_DEBUG=1 python_for_test -m unittest discover -v
``` ```
## Environment variables ## Environment variables
......
...@@ -15,4 +15,4 @@ ...@@ -15,4 +15,4 @@
[template] [template]
filename = instance.cfg filename = instance.cfg
md5sum = 4246cde0a27138e057ba1635cc621edf md5sum = 9639d0c0e161c094454808fb95fc9781
[buildout] [buildout]
extends = {{ nxdtest_instance }}
parts = parts =
slapos-test-runner runTestSuite
publish publish
eggs-directory = {{ buildout['eggs-directory'] }} eggs-directory = {{ buildout['eggs-directory'] }}
...@@ -26,46 +27,53 @@ repository = {{ slapos_location }} ...@@ -26,46 +27,53 @@ repository = {{ slapos_location }}
[directory] [directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin bin = ${buildout:directory}/bin
working-dir = ${buildout:directory}/tmp
etc = ${buildout:directory}/etc etc = ${buildout:directory}/etc
var = ${buildout:directory}/var
test-working-dir = ${buildout:directory}/tmp
nxdtest-working-dir = ${:var}/nxdtest
[slapos-test-runner] [slapos-test-runner-nxdtest-environment.sh]
recipe = slapos.cookbook:wrapper recipe = slapos.recipe.template:jinja2
wrapper-path = ${directory:bin}/runTestSuite rendered = ${directory:etc}/${:_buildout_section_name_}
command-line = template = inline:
{{ buildout['bin-directory'] }}/runTestSuite export PATH={{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:{{ faketime_location }}/bin/:{{ openssl_location }}/bin/:/usr/bin/:/bin
--python_interpreter={{ buildout['bin-directory'] }}/{{ interpreter }} export SLAPOS_TEST_IPV4=${slap-configuration:ipv4-random}
--source_code_path_list={{ ','.join(tests.splitlines()) }} export SLAPOS_TEST_IPV6=${slap-configuration:ipv6-random}
export SLAPOS_TEST_WORKING_DIR=${directory:test-working-dir}
environment = [slapos-test-runner-dot-nxdtest]
PATH=${slapos-test-runner-environment:PATH} recipe = slapos.recipe.template:jinja2
SLAPOS_TEST_IPV4=${slapos-test-runner-environment:SLAPOS_TEST_IPV4} rendered = ${:workdir}/.nxdtest
SLAPOS_TEST_IPV6=${slapos-test-runner-environment:SLAPOS_TEST_IPV6} workdir = ${directory:nxdtest-working-dir}
SLAPOS_TEST_WORKING_DIR=${slapos-test-runner-environment:SLAPOS_TEST_WORKING_DIR} template = inline:
{% for test in tests.splitlines() %}
TestCase(
{{ repr(test.split()[0]) }},
[ {{ repr(interpreter) }}, '-m', 'unittest', 'discover', '-v'],
cwd={{ repr(test.split()[1]) }},
summaryf=UnitTest.summary,
)
{% endfor %}
[slapos-test-runner-environment] [runTestSuite]
PATH = {{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:{{ faketime_location }}/bin/:{{ openssl_location }}/bin/:/usr/bin/:/bin env.sh = ${slapos-test-runner-nxdtest-environment.sh:rendered}
SLAPOS_TEST_IPV4 = ${slap-configuration:ipv4-random} workdir = ${slapos-test-runner-dot-nxdtest:workdir}
SLAPOS_TEST_IPV6 = ${slap-configuration:ipv6-random}
SLAPOS_TEST_WORKING_DIR = ${directory:working-dir}
[slapos-local-development-environment.sh] [slapos-local-development-environment.sh]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
rendered = ${directory:etc}/${:_buildout_section_name_} rendered = ${directory:etc}/${:_buildout_section_name_}
template = inline: template = inline:
export PATH=${slapos-test-runner-environment:PATH}:$PATH source ${slapos-test-runner-nxdtest-environment.sh:rendered}
export SLAPOS_TEST_IPV4=${slapos-test-runner-environment:SLAPOS_TEST_IPV4}
export SLAPOS_TEST_IPV6=${slapos-test-runner-environment:SLAPOS_TEST_IPV6}
export SLAPOS_TEST_WORKING_DIR=${slapos-test-runner-environment:SLAPOS_TEST_WORKING_DIR}
{% set shared_part_list = [] %} {% set shared_part_list = [] %}
{% for shared_part in buildout['shared-part-list'].splitlines() -%} {% for shared_part in buildout['shared-part-list'].splitlines() -%}
{% do shared_part_list.append(shared_part) %} {% do shared_part_list.append(shared_part) %}
{%- endfor %} {%- endfor %}
export SLAPOS_TEST_SHARED_PART_LIST={{ os.pathsep.join(shared_part_list) }} export SLAPOS_TEST_SHARED_PART_LIST={{ os.pathsep.join(shared_part_list) }}
echo "Environment loaded." echo "Environment loaded."
echo "The embedded SlapOS is in $SLAPOS_TEST_WORKING_DIR"
echo "To work on a test, execute:" echo "To work on a test, execute:"
echo " SLAPOS_TEST_DEBUG=1 {{ interpreter }} setup.py test" echo " SLAPOS_TEST_DEBUG=1 {{ interpreter }} -m unittest discover -v"
echo "from test folder" echo "from a folder containing software release test."
echo echo
[publish] [publish]
......
...@@ -12,9 +12,9 @@ eggs += ...@@ -12,9 +12,9 @@ eggs +=
[template] [template]
extra = extra =
${slapos.test.helloworld-setup:setup} helloworld ${slapos.test.helloworld-setup:setup}
${slapos.test.monitor-setup:setup} monitor ${slapos.test.monitor-setup:setup}
${slapos.test.plantuml-setup:setup} plantuml ${slapos.test.plantuml-setup:setup}
${slapos.test.powerdns-setup:setup} powerdns ${slapos.test.powerdns-setup:setup}
${slapos.test.proftpd-setup:setup} proftpd ${slapos.test.proftpd-setup:setup}
${slapos.test.repman-setup:setup} repman ${slapos.test.repman-setup:setup}
...@@ -13,11 +13,12 @@ extends = ...@@ -13,11 +13,12 @@ extends =
../../component/python-backports-lzma/buildout.cfg ../../component/python-backports-lzma/buildout.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
../../stack/nxdtest.cfg
./buildout.hash.cfg ./buildout.hash.cfg
parts = parts =
eggs eggs/scripts
slapos-cookbook slapos-cookbook
template template
...@@ -167,8 +168,8 @@ egg = slapos.core ...@@ -167,8 +168,8 @@ egg = slapos.core
setup = ${slapos.core-repository:location} setup = ${slapos.core-repository:location}
[eggs] [eggs]
recipe = zc.recipe.egg <= python-interpreter
eggs = eggs +=
${lxml-python:egg} ${lxml-python:egg}
${slapos.core-setup:egg} ${slapos.core-setup:egg}
${pillow-python:egg} ${pillow-python:egg}
...@@ -202,21 +203,24 @@ eggs = ...@@ -202,21 +203,24 @@ eggs =
${slapos.test.dream-setup:egg} ${slapos.test.dream-setup:egg}
${slapos.test.metabase-setup:egg} ${slapos.test.metabase-setup:egg}
${slapos.test.repman-setup:egg} ${slapos.test.repman-setup:egg}
entry-points =
runTestSuite=erp5.util.testsuite:runTestSuite # We don't name this interpreter `python`, so that when we run slapos node
scripts = # software, installation scripts running `python` use a python without any
runTestSuite # custom eggs pre-installed, not our special python interpreter.
slapos interpreter = python_for_test
supervisorctl
supervisord
interpreter=
python_for_test
# patches for eggs # patches for eggs
patch-binary = ${patch:location}/bin/patch patch-binary = ${patch:location}/bin/patch
PyPDF2-patches = ${:_profile_base_location_}/../../component/egg-patch/PyPDF2/0001-Custom-implementation-of-warnings.formatwarning-remo.patch#d25bb0f5dde7f3337a0a50c2f986f5c8 PyPDF2-patches = ${:_profile_base_location_}/../../component/egg-patch/PyPDF2/0001-Custom-implementation-of-warnings.formatwarning-remo.patch#d25bb0f5dde7f3337a0a50c2f986f5c8
PyPDF2-patch-options = -p1 PyPDF2-patch-options = -p1
[eggs/scripts]
recipe = zc.recipe.egg
eggs = ${eggs:eggs}
scripts =
slapos
supervisord
[git-clone-repository] [git-clone-repository]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
...@@ -236,6 +240,7 @@ mode = 640 ...@@ -236,6 +240,7 @@ mode = 640
context = context =
section buildout buildout section buildout buildout
import os os import os os
key nxdtest_instance nxdtest-instance.cfg:rendered
key git_location git:location key git_location git:location
key slapos_location slapos-repository:location key slapos_location slapos-repository:location
key interpreter eggs:interpreter key interpreter eggs:interpreter
...@@ -243,29 +248,31 @@ context = ...@@ -243,29 +248,31 @@ context =
key openssl_location openssl-output:bin key openssl_location openssl-output:bin
key faketime_location faketime:location key faketime_location faketime:location
key tests :tests key tests :tests
tests = tests =
${slapos.test.kvm-setup:setup} json-schemas ${slapos.cookbook-setup:setup}
${slapos.test.slaprunner-setup:setup} kvm ${slapos.test.kvm-setup:setup}
${slapos.test.metabase-setup:setup} slaprunner ${slapos.test.slaprunner-setup:setup}
metabase ${slapos.test.metabase-setup:setup}
${:extra} ${:extra}
extra = extra =
${slapos.cookbook-setup:setup} backupserver ${slapos.test.backupserver-setup:setup}
${slapos.test.backupserver-setup:setup} caddy-frontend ${slapos.test.caddy-frontend-setup:setup}
${slapos.test.caddy-frontend-setup:setup} erp5 ${slapos.test.erp5-setup:setup}
${slapos.test.erp5-setup:setup} htmlvalidatorserver ${slapos.test.htmlvalidatorserver-setup:setup}
${slapos.test.htmlvalidatorserver-setup:setup} slapos-master ${slapos.test.slapos-master-setup:setup}
${slapos.test.slapos-master-setup:setup} re6stnet ${slapos.test.re6stnet-setup:setup}
${slapos.test.re6stnet-setup:setup} seleniumserver ${slapos.test.seleniumserver-setup:setup}
${slapos.test.seleniumserver-setup:setup} jstestnode ${slapos.test.jstestnode-setup:setup}
${slapos.test.jstestnode-setup:setup} jupyter ${slapos.test.jupyter-setup:setup}
${slapos.test.jupyter-setup:setup} nextcloud ${slapos.test.nextcloud-setup:setup}
${slapos.test.nextcloud-setup:setup} turnserver ${slapos.test.turnserver-setup:setup}
${slapos.test.turnserver-setup:setup} theia ${slapos.test.theia-setup:setup}
${slapos.test.theia-setup:setup} grafana ${slapos.test.grafana-setup:setup}
${slapos.test.grafana-setup:setup} gitlab ${slapos.test.gitlab-setup:setup}
${slapos.test.gitlab-setup:setup} cloudooo ${slapos.test.cloudooo-setup:setup}
${slapos.test.cloudooo-setup:setup} dream ${slapos.test.dream-setup:setup}
${slapos.test.dream-setup:setup} repman ${slapos.test.repman-setup:setup}
[versions] [versions]
# slapos.core is used from the clone always # slapos.core is used from the clone always
......
...@@ -2,13 +2,11 @@ ...@@ -2,13 +2,11 @@
This software release is used to run unit test of slapos eggs. This software release is used to run unit test of slapos eggs.
The approach is to use setuptools' integrated test runner, `python setup.py test`, to run tests. The approach is to use nxdtest test runner, which will run tests for each
projects, as described in `.nxdtest` file.
The `python` used in this command will be a `zc.recipe.egg` interpreter with The results of this test suite running on Nexedi ERP5 are published as
all eggs pre-installed by this software release. `SlapOS.Eggs.UnitTest-Master.Python3` and `SlapOS.Eggs.UnitTest-Master.Python2`.
The results of this test suite running on Nexedi ERP5 are published as `SlapOS.Eggs.UnitTest-Master.Python3`
and `SlapOS.Eggs.UnitTest-Master.Python2`.
Here's an example session of how a developer could use this software release in Here's an example session of how a developer could use this software release in
...@@ -23,21 +21,27 @@ INSTANCE_NAME=$COMP ...@@ -23,21 +21,27 @@ INSTANCE_NAME=$COMP
slapos supply $SR $COMP slapos supply $SR $COMP
slapos node software slapos node software
slapos request --node=node=$COMP $INSTANCE_NAME $SR slapos request --node=computer_guid=$COMP $INSTANCE_NAME $SR
slapos node instance slapos node instance
# The path of a an environment script was published by slapos parameters, as
# "environment-script"
slapos request --node=computer_guid=$COMP $INSTANCE_NAME $SR
# sourcing the script in the shell configure all environment variables and
# print a message explaining how to run tests
source ( environment script from step above )
# The source code is a git clone working copy on the instance # The source code is a git clone working copy on the instance
cd ~/srv/runner/instance/slappart0/parts/slapos.core/ cd ~/srv/runner/instance/slappartXXX/parts/slapos.core/
# make some changes to the code # make some changes to the code
vim slapos/tests/client.py vim slapos/tests/client.py
# run tests, using bundled python intepreter with pre-installed eggs dependencies # run slapos.core tests
SLAPOS_TEST_IPV6=::1 \ runTestSuite --run slapos.core
SLAPOS_TEST_IPV4=127.0.0.1 \ # ... or run all eggs tests
SLAPOS_TEST_VERBOSE=1 \ runTestSuite
SLAPOS_TEST_DEBUG=1 \
~/srv/runner/instance/slappart0/software_release/bin/python_for_test setup.py test
# when satified, commit changes # when satified, commit changes
git add -p && git commit git add -p && git commit
......
...@@ -15,4 +15,4 @@ ...@@ -15,4 +15,4 @@
[template] [template]
filename = instance.cfg filename = instance.cfg
md5sum = 2df601dd3ccb3ba38b3aee7243b7f8e5 md5sum = 3b1b386f6ad4c9ac50ab1f1e1384e751
[buildout] [buildout]
extends = ${nxdtest-instance.cfg:rendered}
parts = parts =
phantomjs-wrapper runTestSuite
slapos-test-runner publish
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
...@@ -21,6 +22,8 @@ bin = $${buildout:directory}/bin ...@@ -21,6 +22,8 @@ bin = $${buildout:directory}/bin
etc = $${buildout:directory}/etc etc = $${buildout:directory}/etc
services = $${:etc}/run services = $${:etc}/run
srv = $${buildout:directory}/srv srv = $${buildout:directory}/srv
var = $${buildout:directory}/var
nxdtest-working-dir = $${:var}/nxdtest
[download-source] [download-source]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
...@@ -75,30 +78,127 @@ repository = ${slapos.rebootstrap-repository:location} ...@@ -75,30 +78,127 @@ repository = ${slapos.rebootstrap-repository:location}
repository = ${rubygemsrecipe-repository:location} repository = ${rubygemsrecipe-repository:location}
[slapos-test-runner] [slapos-test-runner-nxdtest-environment.sh]
recipe = slapos.cookbook:wrapper recipe = slapos.recipe.template:jinja2
wrapper-path = $${create-directory:bin}/runTestSuite rendered = $${create-directory:etc}/$${:_buildout_section_name_}
command-line = template = inline:
${buildout:bin-directory}/runTestSuite export PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${socat:location}/bin:${lmsensors:location}/bin:${rsync:location}/bin/:${buildout:bin-directory}:$PATH
--python_interpreter=${buildout:bin-directory}/${eggs:interpreter} export SLAPOS_TEST_IPV4=$${slap-configuration:ipv4-random}
--source_code_path_list=$${kedifa:location},$${caucase:location},$${erp5.util:location},$${slapos.cookbook:location},$${slapos.core:location},$${slapos.recipe.build:location},$${slapos.recipe.cmmi:location},$${slapos.recipe.template:location},$${slapos.toolbox:location},$${slapos.libnetworkcache:location},$${slapos.rebootstrap:location},$${rubygemsrecipe:location} export SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random}
# Notes about environment:
# * slapos.cookbook:wrapper does not seem to allow "extending" PATH. Tests [slapos-test-runner-dot-nxdtest]
# needs ping, which is a setuid binary that cannot be installed via slapos recipe = slapos.recipe.template:jinja2
# way of building software without root access, so we keep "standard" rendered = $${:workdir}/.nxdtest
# /usr/bin and /bin in $PATH workdir = $${create-directory:nxdtest-working-dir}
# * SLAPOS_TEST_environment is safe to be used by tests, but there is no
# guarantee about free ports on IPV4 and IPV6 template = inline:
# * LOCAL_IPV4 is backward compatible, to be migrated, SLAPOS_TEST_IPV4 TestCase(
environment = "kedifa",
PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${socat:location}/bin:${lmsensors:location}/bin:${rsync:location}/bin/:${buildout:bin-directory}:/usr/bin/:/bin/ ['python', '-m', 'unittest', 'discover', '-v'],
LOCAL_IPV4=$${slap-configuration:ipv4-random} cwd="""$${kedifa:location}""",
SLAPOS_TEST_IPV4=$${slap-configuration:ipv4-random} summaryf=UnitTest.summary,
SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random} )
TestCase(
"caucase",
[phantomjs-wrapper] # XXX caucase uses 2to3 dynamically in setup.py, so it only supports
recipe = slapos.cookbook:wrapper # runnning tests with python setup.py test
command-line = ${phantomjs:location}/phantomjs-slapos ['python', 'setup.py', 'test'],
wrapper-path = $${create-directory:bin}/phantomjs cwd="""$${caucase:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"erp5.util",
['python', '-m', 'unittest', 'discover', '-v', '--start', 'erp5/tests/'],
cwd="""$${erp5.util:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.cookbook",
# slapos/test contain both tests for recipes and tests for
# json schemas, we only care about recipe tests here, json
# schemas are tested in software/slapos-sr-testing
['python', '-m', 'unittest', 'discover', '-v', '--start', 'slapos/test/recipe'],
cwd="""$${slapos.cookbook:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.core",
# ['python', '-m', 'unittest', 'discover', '-v'],
# XXX some test fail when running with unittest (slapos*.cfg.example
# files cannot be found with pkg_resources.resource_string), so we keep
# using setup.py test for now.
['python', 'setup.py', 'test'],
cwd="""$${slapos.core:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.recipe.build",
['python', '-m', 'unittest', 'discover', '-v'],
cwd="""$${slapos.recipe.build:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.recipe.cmmi",
['python', '-m', 'unittest', 'discover', '-v'],
cwd="""$${slapos.recipe.cmmi:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.recipe.template",
# ['python', '-m', 'unittest', 'slapos.recipe.template.tests.test_suite'],
# XXX some test fail when running with unittest, so we keep using setup.py test
['python', 'setup.py', 'test'],
cwd="""$${slapos.recipe.template:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.toolbox",
# ['python', '-m', 'unittest', 'discover', '-v'],
# XXX We can not just run unittest discover here, since slapos/lamp
# imports MySQLDb and we currently don't have installed
# slapos.toolbox[lampconfigure] and on python3 discovery make a
# failing test for this import error.
# Currently slapos/lamp is not tested, but if it it is still used,
# the TODO seem to also install slapos.toolbox[lampconfigure] here.
['python', '-m', 'unittest', 'discover', '-v', '--start', 'slapos/test', '--top-level-directory', '.'],
cwd="""$${slapos.toolbox:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.libnetworkcache",
['python', '-m', 'unittest', '-v', 'slapos.libnetworkcachetests'],
cwd="""$${slapos.libnetworkcache:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.rebootstrap",
['python', '-m', 'unittest', '-v', 'slapos.rebootstrap.tests.test_suite'],
cwd="""$${slapos.rebootstrap:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"rubygemsrecipe",
['python', '-m', 'unittest', 'discover', '-v'],
cwd="""$${rubygemsrecipe:location}""",
summaryf=UnitTest.summary,
)
[runTestSuite]
env.sh = $${slapos-test-runner-nxdtest-environment.sh:rendered}
workdir = $${slapos-test-runner-dot-nxdtest:workdir}
[slapos-local-development-environment.sh]
recipe = slapos.recipe.template:jinja2
rendered = $${create-directory:etc}/$${:_buildout_section_name_}
template = inline:
source $${slapos-test-runner-nxdtest-environment.sh:rendered}
echo "Environment loaded."
echo "To work on a test, execute:"
echo " $${runTestSuite:wrapper-path} -k test_name"
echo "replacing test_name by the name of the test."
echo
[publish]
recipe = slapos.cookbook:publish
environment-script = $${slapos-local-development-environment.sh:rendered}
...@@ -14,11 +14,12 @@ extends = ...@@ -14,11 +14,12 @@ extends =
../../component/lmsensors/buildout.cfg ../../component/lmsensors/buildout.cfg
../../component/rsync/buildout.cfg ../../component/rsync/buildout.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
../../stack/nxdtest.cfg
./buildout.hash.cfg ./buildout.hash.cfg
parts = parts =
bootstrap-slapos.recipe.cmmi bootstrap-slapos.recipe.cmmi
eggs eggs/scripts
phantomjs phantomjs
template template
...@@ -103,8 +104,8 @@ egg = rubygemsrecipe[test] ...@@ -103,8 +104,8 @@ egg = rubygemsrecipe[test]
setup = ${rubygemsrecipe-repository:location} setup = ${rubygemsrecipe-repository:location}
[eggs] [eggs]
recipe = zc.recipe.egg <= python-interpreter
eggs = eggs +=
${lxml-python:egg} ${lxml-python:egg}
${python-cryptography:egg} ${python-cryptography:egg}
${backports.lzma:egg} ${backports.lzma:egg}
...@@ -126,14 +127,13 @@ eggs = ...@@ -126,14 +127,13 @@ eggs =
${rubygemsrecipe-setup:egg} ${rubygemsrecipe-setup:egg}
zope.testing zope.testing
supervisor supervisor
entry-points =
runTestSuite=erp5.util.testsuite:runTestSuite [eggs/scripts]
recipe = zc.recipe.egg
eggs = ${eggs:eggs}
scripts = scripts =
runTestSuite
slapos slapos
supervisord supervisord
interpreter=
python_for_test
[git-clone-repository] [git-clone-repository]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
......
...@@ -18,7 +18,7 @@ md5sum = 8d6878ff1d2e75010c50a1a2b0c13b24 ...@@ -18,7 +18,7 @@ md5sum = 8d6878ff1d2e75010c50a1a2b0c13b24
[template-runner] [template-runner]
filename = instance-runner.cfg filename = instance-runner.cfg
md5sum = 6e279c46b07bf56b7b037a8ee2c6587e md5sum = 2582723c31166244ff25cb3d8c839ffa
[template-runner-import-script] [template-runner-import-script]
filename = template/runner-import.sh.jinja2 filename = template/runner-import.sh.jinja2
......
...@@ -46,6 +46,7 @@ common-runner-parts = ...@@ -46,6 +46,7 @@ common-runner-parts =
custom-frontend-promise custom-frontend-promise
{% endif %} {% endif %}
{% endif %} {% endif %}
project-link
## Monitoring part ## Monitoring part
monitor-base monitor-base
monitor-check-webrunner-internal-instance monitor-check-webrunner-internal-instance
...@@ -159,6 +160,11 @@ software-test = $${:test}/software ...@@ -159,6 +160,11 @@ software-test = $${:test}/software
instance-test = $${:test}/instance instance-test = $${:test}/instance
sessions = $${buildout:directory}/.sessions sessions = $${buildout:directory}/.sessions
[project-link]
recipe = slapos.cookbook:symbolic.link
target-directory = $${directory:srv}
link-binary = $${runnerdirectory:project}
[slaprunner] [slaprunner]
slaprunner = ${buildout:bin-directory}/slaprunner slaprunner = ${buildout:bin-directory}/slaprunner
slapos = ${buildout:bin-directory}/slapos slapos = ${buildout:bin-directory}/slapos
......
...@@ -15,11 +15,11 @@ ...@@ -15,11 +15,11 @@
[instance] [instance]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = 2ceb9389281c00261abd864fc8ed566f md5sum = 1c60191f8724854f979a17d2624e65d8
[yarn.lock] [yarn.lock]
filename = yarn.lock filename = yarn.lock
md5sum = c7aa84922a1b80fd8a4c3d96f6ac7e25 md5sum = 5a89742266a9f9d4115efa6d641fd5bb
[python-language-server-requirements.txt] [python-language-server-requirements.txt]
filename = python-language-server-requirements.txt filename = python-language-server-requirements.txt
......
...@@ -173,7 +173,7 @@ template = ...@@ -173,7 +173,7 @@ template =
], ],
"options": { "options": {
"env": { "env": {
"SLAPOS_CONFIGURATION": "$${slapos-standalone-activate:slapos-configuration}", "SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
"GIT_EXEC_PATH": "" "GIT_EXEC_PATH": ""
} }
}, },
...@@ -197,7 +197,7 @@ template = ...@@ -197,7 +197,7 @@ template =
], ],
"options": { "options": {
"env": { "env": {
"SLAPOS_CONFIGURATION": "$${slapos-standalone-activate:slapos-configuration}", "SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
"GIT_EXEC_PATH": "" "GIT_EXEC_PATH": ""
} }
}, },
...@@ -227,6 +227,7 @@ template = ...@@ -227,6 +227,7 @@ template =
export TMP=$${directory:tmp} export TMP=$${directory:tmp}
export TEMP=$TMP export TEMP=$TMP
export LC_ALL=C.UTF-8 export LC_ALL=C.UTF-8
export TERMINFO=${ncurses:location}/lib/terminfo/
export EDITOR="${python-language-server:location}/bin/python -m theia_open --wait" export EDITOR="${python-language-server:location}/bin/python -m theia_open --wait"
exec ${theia-wrapper:rendered} $@ exec ${theia-wrapper:rendered} $@
ip = $${instance-parameter:ipv4-random} ip = $${instance-parameter:ipv4-random}
...@@ -256,36 +257,76 @@ template = inline: ...@@ -256,36 +257,76 @@ template = inline:
import time import time
args = sys.argv[1:] args = sys.argv[1:]
# when running interactively, activate slapos configuration and reset GIT_EXEC_PATH to workaround https://github.com/eclipse-theia/theia/issues/7555 # when running interactively, activate slapos configuration and reset GIT_EXEC_PATH to workaround https://github.com/eclipse-theia/theia/issues/7555
if not args: args = ["-c", ". $${slapos-standalone-activate:rendered} && exec env GIT_EXEC_PATH= ${bash:location}/bin/bash", ] if not args: args = ["-c", ". $${slapos-standalone-activate:rendered} && exec env GIT_EXEC_PATH= ${bash:location}/bin/bash --rcfile $${theia-bashrc:rendered}", ]
# otherwise, assume this shell is running task and add an artificial delay to workaround https://github.com/eclipse-theia/theia/issues/2961 # otherwise, assume this shell is running task and add an artificial delay to workaround https://github.com/eclipse-theia/theia/issues/2961
else: time.sleep(1) else: time.sleep(1)
os.execv('${bash:location}/bin/bash', ['${bash:location}/bin/bash'] + args) os.execv('${bash:location}/bin/bash', ['${bash:location}/bin/bash'] + args)
[theia-bashrc]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:etc}/$${:_buildout_section_name_}
template =
inline:
# enable bash completion
. ${bash-completion:location}/etc/profile.d/bash_completion.sh
# source user's .bashrc
[ -f ~/.bashrc ] && . ~/.bashrc
depends =
$${shell-setup-completion:recipe}
[shell-setup-completion]
recipe = plone.recipe.command
stop-on-error = true
command =
${buildout:bin-directory}/slapos complete > $${directory:bash-completions}/slapos
${buildout:bin-directory}/slapos complete --shell fish > $${directory:fish-completions}/slapos.fish
[slapos-standalone-config]
ipv4 = $${instance-parameter:ipv4-random}
ipv6 = $${instance-parameter:ipv6-random}
port = 4000
slapos-configuration = $${directory:runner}/etc/slapos.cfg
computer-id = slaprunner
[slapos-standalone-activate] [slapos-standalone-activate]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_} rendered = $${directory:bin}/$${:_buildout_section_name_}
mode = 0700 mode = 0700
# XXX maybe standalone slapos should provide an activate script like virtualenv is doing?
template = template =
inline:#!/bin/sh inline:#!/bin/sh
export PATH=${buildout:bin-directory}:$PATH export PATH=${buildout:bin-directory}:$PATH
${slapos-standalone:script-path} \ export SLAPOS_CONFIGURATION=$${slapos-standalone-config:slapos-configuration}
$${directory:slapos} \ export SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION
$${:ipv4} \ echo 'Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` activated'
$${:ipv6} \
$${:port} \ [slapos-standalone]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
mode = 0700
template =
inline:#!/bin/sh
export PATH=${buildout:bin-directory}:$PATH
exec ${slapos-standalone:script-path} \
$${directory:runner} \
$${slapos-standalone-config:ipv4} \
$${slapos-standalone-config:ipv6} \
$${slapos-standalone-config:port} \
$${slapos-standalone-config:computer-id} \
$${slap-connection:server-url} \ $${slap-connection:server-url} \
$${slap-connection:computer-id} \ $${slap-connection:computer-id} \
$${slap-connection:partition-id} \ $${slap-connection:partition-id} \
--key='$${slap-connection:key-file}' \ --key='$${slap-connection:key-file}' \
--cert='$${slap-connection:cert-file}' --cert='$${slap-connection:cert-file}'
export SLAPOS_CONFIGURATION=$${:slapos-configuration}
export SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION
ipv4 = $${instance-parameter:ipv4-random} [slapos-standalone-instance]
ipv6 = $${instance-parameter:ipv6-random} recipe = slapos.cookbook:wrapper
port = 4000 wrapper-path = $${directory:services}/$${:_buildout_section_name_}
slapos-configuration = $${directory:slapos}/etc/slapos.cfg command-line = $${slapos-standalone:rendered}
hash-files =
$${slapos-standalone:rendered}
hostname = $${slapos-standalone-config:ipv4}
port = $${slapos-standalone-config:port}
[promises] [promises]
recipe = recipe =
...@@ -293,6 +334,7 @@ instance-promises = ...@@ -293,6 +334,7 @@ instance-promises =
$${theia-listen-promise:name} $${theia-listen-promise:name}
$${frontend-listen-promise:name} $${frontend-listen-promise:name}
$${apache-frontend-url-available-promise:name} $${apache-frontend-url-available-promise:name}
$${slapos-standalone-listen-promise:name}
[theia-listen-promise] [theia-listen-promise]
<= monitor-promise-base <= monitor-promise-base
...@@ -315,6 +357,14 @@ name = $${:_buildout_section_name_}.py ...@@ -315,6 +357,14 @@ name = $${:_buildout_section_name_}.py
config-url = $${apache-frontend:connection-secure_access} config-url = $${apache-frontend:connection-secure_access}
config-check-secure = 1 config-check-secure = 1
[slapos-standalone-listen-promise]
<= monitor-promise-base
module = check_port_listening
# XXX promise plugins can not contain "slapos" in their names
name = standalone-listen-promise.py
config-hostname = $${slapos-standalone-instance:hostname}
config-port = $${slapos-standalone-instance:port}
[apache-frontend] [apache-frontend]
<= slap-connection <= slap-connection
recipe = slapos.cookbook:requestoptional recipe = slapos.cookbook:requestoptional
...@@ -353,8 +403,11 @@ dot-theia = $${buildout:directory}/.theia/ ...@@ -353,8 +403,11 @@ dot-theia = $${buildout:directory}/.theia/
pidfiles = $${:var}/run pidfiles = $${:var}/run
services = $${:etc}/service services = $${:etc}/service
runner = $${:srv}/runner
project = $${:srv}/project project = $${:srv}/project
slapos = $${:srv}/slapos
frontend-static = $${:srv}/frontend-static frontend-static = $${:srv}/frontend-static
frontend-static-public = $${:frontend-static}/public frontend-static-public = $${:frontend-static}/public
frontend-static-css = $${:frontend-static}/css frontend-static-css = $${:frontend-static}/css
bash-completions = $${buildout:directory}/.local/share/bash-completion/completions/
fish-completions = $${buildout:directory}/.config/fish/completions/
...@@ -4,6 +4,7 @@ extends = ...@@ -4,6 +4,7 @@ extends =
../../component/caddy/buildout.cfg ../../component/caddy/buildout.cfg
../../component/git/buildout.cfg ../../component/git/buildout.cfg
../../component/bash/buildout.cfg ../../component/bash/buildout.cfg
../../component/bash-completion/buildout.cfg
../../component/fish-shell/buildout.cfg ../../component/fish-shell/buildout.cfg
../../component/tmux/buildout.cfg ../../component/tmux/buildout.cfg
../../component/tig/buildout.cfg ../../component/tig/buildout.cfg
...@@ -55,9 +56,11 @@ entry-points = ...@@ -55,9 +56,11 @@ entry-points =
${:scripts}=not_used:main ${:scripts}=not_used:main
initialization = initialization =
import argparse import argparse
import glob
import os.path import os.path
import sys import sys
import glob import signal
import time
import slapos.slap.standalone import slapos.slap.standalone
...@@ -66,6 +69,7 @@ initialization = ...@@ -66,6 +69,7 @@ initialization =
parser.add_argument('ipv4') parser.add_argument('ipv4')
parser.add_argument('ipv6') parser.add_argument('ipv6')
parser.add_argument('server_port', type=int) parser.add_argument('server_port', type=int)
parser.add_argument('computer_id')
forwarded_arguments = parser.add_argument_group('forwarded') forwarded_arguments = parser.add_argument_group('forwarded')
forwarded_arguments.add_argument('master_url') forwarded_arguments.add_argument('master_url')
forwarded_arguments.add_argument('computer') forwarded_arguments.add_argument('computer')
...@@ -91,7 +95,10 @@ initialization = ...@@ -91,7 +95,10 @@ initialization =
args.base_directory, args.base_directory,
args.ipv4, args.ipv4,
args.server_port, args.server_port,
computer_id=args.computer_id,
shared_part_list=shared_part_list, shared_part_list=shared_part_list,
software_root="%s/software" % args.base_directory,
instance_root="%s/instance" % args.base_directory,
partition_forward_configuration=partition_forward_configuration, partition_forward_configuration=partition_forward_configuration,
) )
standalone.start() standalone.start()
...@@ -104,7 +111,26 @@ initialization = ...@@ -104,7 +111,26 @@ initialization =
args.ipv4, args.ipv4,
args.ipv6 args.ipv6
) )
print ("Standalone SlapOS for computer `{}` activated".format(standalone.computer._computer_id)) print("Standalone SlapOS for computer `{}` started".format(args.computer_id))
# Run instance at least once, to start the supervisor managing instances.
try:
standalone.waitForInstance(max_retry=0)
except slapos.slap.standalone.SlapOSNodeCommandError as e:
print("Error instanciating: {}".format(e))
quit_requested = []
def signal_handler(signum, frame):
print("Signal {signum} received".format(signum=signum))
quit_requested.append(True)
signal.signal(signal.SIGTERM, signal_handler)
print("Standalone SlapOS ready")
while not quit_requested:
time.sleep(.1)
print("Stopping standalone subsystem")
standalone.stop()
print("Exiting")
sys.exit(0) sys.exit(0)
needs-these-eggs-scripts-in-path = needs-these-eggs-scripts-in-path =
...@@ -343,7 +369,7 @@ command = ...@@ -343,7 +369,7 @@ command =
[cli-utilities] [cli-utilities]
PATH = ${nodejs:location}/bin/:${bash:location}/bin/:${fish-shell:location}/bin/:${tig:location}/bin/:${vim:location}/bin/:${tmux:location}/bin/:${git:location}/bin/:${curl:location}/bin:${python2.7:location}/bin/ PATH = ${nodejs:location}/bin/:${bash:location}/bin/:${fish-shell:location}/bin/:${tig:location}/bin/:${vim:location}/bin/:${tmux:location}/bin/:${git:location}/bin/:${curl:location}/bin:${python2.7:location}/bin/:${buildout:bin-directory}
[theia-wrapper] [theia-wrapper]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
......
...@@ -36,9 +36,13 @@ import re ...@@ -36,9 +36,13 @@ import re
from six.moves.urllib.parse import urlparse, urljoin from six.moves.urllib.parse import urlparse, urljoin
import pexpect import pexpect
import psutil
import requests import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.grid.svcbackend import getSupervisorRPC
from slapos.grid.svcbackend import _getSupervisordSocketPath
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass( setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath( os.path.abspath(
...@@ -110,12 +114,11 @@ class TestTheia(SlapOSInstanceTestCase): ...@@ -110,12 +114,11 @@ class TestTheia(SlapOSInstanceTestCase):
pass pass
process.logfile = DebugLogFile() process.logfile = DebugLogFile()
process.expect_exact('Standalone SlapOS: Formatting 20 partitions') process.expect_exact('Standalone SlapOS for computer `slaprunner` activated')
process.expect_exact('Standalone SlapOS for computer `local` activated')
# try to supply and install a software to check that this slapos is usable # try to supply and install a software to check that this slapos is usable
process.sendline( process.sendline(
'slapos supply https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg local' 'slapos supply https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg slaprunner'
) )
process.expect( process.expect(
'Requesting software installation of https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg...' 'Requesting software installation of https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg...'
...@@ -135,12 +138,6 @@ class TestTheia(SlapOSInstanceTestCase): ...@@ -135,12 +138,6 @@ class TestTheia(SlapOSInstanceTestCase):
# interrupt this, we don't want to actually wait for software installation # interrupt this, we don't want to actually wait for software installation
process.sendcontrol('c') process.sendcontrol('c')
# shutdown this slapos
process.sendline(
'supervisorctl -c {}/srv/slapos/etc/supervisord.conf shutdown'.format(
self.computer_partition_root_path))
process.expect('Shut down')
process.terminate() process.terminate()
process.wait() process.wait()
...@@ -153,3 +150,37 @@ class TestTheia(SlapOSInstanceTestCase): ...@@ -153,3 +150,37 @@ class TestTheia(SlapOSInstanceTestCase):
'touch "{}"'.format(test_file) 'touch "{}"'.format(test_file)
]) ])
self.assertTrue(os.path.exists(test_file)) self.assertTrue(os.path.exists(test_file))
class TestTheiaEmbeddedSlapOSShutdown(SlapOSInstanceTestCase):
__partition_reference__ = 'T' # for sockets in included slapos
def test_stopping_instance_stops_embedded_slapos(self):
embedded_slapos_supervisord_socket = _getSupervisordSocketPath(
os.path.join(
self.computer_partition_root_path,
'srv',
'slapos',
'inst',
), self.logger)
# Wait a bit for this supervisor to be started.
for _ in range(20):
if os.path.exists(embedded_slapos_supervisord_socket):
break
time.sleep(1)
# get the pid of the supervisor used to manage instances
with getSupervisorRPC(embedded_slapos_supervisord_socket) as embedded_slapos_supervisor:
embedded_slapos_process = psutil.Process(embedded_slapos_supervisor.getPID())
# Stop theia's services
with self.slap.instance_supervisor_rpc as instance_supervisor:
process_info, = [
p for p in instance_supervisor.getAllProcessInfo()
if p['name'].startswith('slapos-standalone-instance-')
]
instance_supervisor.stopProcessGroup(process_info['group'])
# the supervisor controlling instances is also stopped
self.assertFalse(embedded_slapos_process.is_running())
This diff is collapsed.
...@@ -32,7 +32,7 @@ ...@@ -32,7 +32,7 @@
[buildout] [buildout]
extends = extends =
slapos.cfg slapos.cfg
../component/git/buildout.cfg ../component/nxdtest/buildout.cfg
nxdtest/buildout.hash.cfg nxdtest/buildout.hash.cfg
[jinja2-template] [jinja2-template]
...@@ -52,25 +52,5 @@ context = ...@@ -52,25 +52,5 @@ context =
section nxdtest nxdtest section nxdtest nxdtest
[nxdtest]
recipe = zc.recipe.egg:scripts
eggs = ${nxdtest-egg:egg}
scripts = nxdtest
# convenience for nxdtest users
exe = ${buildout:bin-directory}/nxdtest
[nxdtest-egg]
recipe = zc.recipe.egg:develop
setup = ${nxdtest-repository:location}
egg = nxdtest
[nxdtest-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/nxdtest.git
revision = bd91f6f1579a
location = ${buildout:parts-directory}/nxdtest
git-executable = ${git:location}/bin/git
[versions] [versions]
slapos.recipe.template = 4.4 slapos.recipe.template = 4.4
...@@ -164,6 +164,7 @@ dateparser = 0.7.6 ...@@ -164,6 +164,7 @@ dateparser = 0.7.6
decorator = 4.3.0 decorator = 4.3.0
funcsigs = 1.0.2 funcsigs = 1.0.2
gevent = 20.9.0 gevent = 20.9.0
geventmp = 0.0.1
greenlet = 0.4.17 greenlet = 0.4.17
idna = 2.9 idna = 2.9
importlib-metadata = 1.7.0 importlib-metadata = 1.7.0
...@@ -191,7 +192,7 @@ setuptools-dso = 1.7 ...@@ -191,7 +192,7 @@ setuptools-dso = 1.7
rubygemsrecipe = 0.3.0 rubygemsrecipe = 0.3.0
six = 1.12.0 six = 1.12.0
slapos.cookbook = 1.0.167 slapos.cookbook = 1.0.167
slapos.core = 1.6.2 slapos.core = 1.6.3
slapos.extension.strip = 0.4 slapos.extension.strip = 0.4
slapos.extension.shared = 1.0 slapos.extension.shared = 1.0
slapos.libnetworkcache = 0.20 slapos.libnetworkcache = 0.20
...@@ -247,7 +248,7 @@ croniter = 0.3.25 ...@@ -247,7 +248,7 @@ croniter = 0.3.25
# Required by: # Required by:
# slapos.toolbox==0.94 # slapos.toolbox==0.94
dnspython = 1.15.0 dnspython = 1.16.0
# Required by: # Required by:
# cryptography==1.8.1 # cryptography==1.8.1
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment