Commit b22afdd8 authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki

Merge remote-tracking branch 'origin/master' into erp5-component

parents 38b77eef c56da5c4
...@@ -19,7 +19,7 @@ environment = ...@@ -19,7 +19,7 @@ environment =
recipe = plone.recipe.command recipe = plone.recipe.command
stop-on-error = true stop-on-error = true
update-command = ${:command} update-command = ${:command}
command = ${:test} -x ${:test} -a -x ${:cat} -a -x ${:rm} -a -x ${:echo} -a -x ${:date} -a -x ${:md5sum} -a -x ${:basename} command = ${:test} -x ${:test} -a -x ${:cat} -a -x ${:rm} -a -x ${:echo} -a -x ${:date} -a -x ${:md5sum} -a -x ${:basename} -a -x ${:sort} -a -x ${:tail}
test = ${coreutils:location}/bin/test test = ${coreutils:location}/bin/test
cat = ${coreutils:location}/bin/cat cat = ${coreutils:location}/bin/cat
rm = ${coreutils:location}/bin/rm rm = ${coreutils:location}/bin/rm
...@@ -27,3 +27,5 @@ echo = ${coreutils:location}/bin/echo ...@@ -27,3 +27,5 @@ echo = ${coreutils:location}/bin/echo
date = ${coreutils:location}/bin/date date = ${coreutils:location}/bin/date
md5sum = ${coreutils:location}/bin/md5sum md5sum = ${coreutils:location}/bin/md5sum
basename = ${coreutils:location}/bin/basename basename = ${coreutils:location}/bin/basename
sort = ${coreutils:location}/bin/sort
tail = ${coreutils:location}/bin/tail
[buildout]
extends =
../coreutils/buildout.cfg
parts =
findutils-output
[findutils]
recipe = slapos.recipe.cmmi
url = http://ftp.gnu.org/gnu/findutils/findutils-4.4.2.tar.gz
md5sum = 351cc4adb07d54877fa15f75fb77d39f
[findutils-output]
# Shared binary location to ease migration
recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command = ${coreutils-output:test} -x ${:find} -a -x ${:xargs}
find = ${findutils:location}/bin/find
xargs = ${findutils:location}/bin/xargs
...@@ -19,6 +19,27 @@ configure-options= ...@@ -19,6 +19,27 @@ configure-options=
--with-ld-opt="-L ${zlib:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib" --with-ld-opt="-L ${zlib:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib"
--with-cc-opt="-I ${pcre:location}/include -I ${openssl:location}/include -I ${zlib:location}/include" --with-cc-opt="-I ${pcre:location}/include -I ${openssl:location}/include -I ${zlib:location}/include"
[webdav-module]
recipe = hexagonit.recipe.download
url = https://github.com/arut/nginx-dav-ext-module/archive/master.zip
strip-top-level-dir = true
#md5sum =
mode = 0644
[nginx-webdav]
recipe = slapos.recipe.cmmi
url = http://nginx.org/download/nginx-1.5.3.tar.gz
md5sum = 1e735dd6a6ade2b5c20e924b67c3d355
configure-options =
--with-ipv6
--with-http_ssl_module
--with-mail
--with-mail_ssl_module
--with-ld-opt="-L ${zlib:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib -L ${libexpat:location}/lib"
--with-cc-opt="-I ${pcre:location}/include -I ${openssl:location}/include -I ${zlib:location}/include -I ${libexpat:location}/include"
--with-http_dav_module
--add-module='${webdav-module:location}'
[nginx-output] [nginx-output]
# Shared binary location to ease migration # Shared binary location to ease migration
recipe = plone.recipe.command recipe = plone.recipe.command
......
[buildout]
extends =
../libffi/buildout.cfg
../pkgconfig/buildout.cfg
parts =
python-cffi
[python-cffi]
recipe = zc.recipe.egg:custom
egg = cffi
environment = python-cffi-env
library-dirs =
${libffi:location}/lib/
rpath =
${libffi:location}/lib/
[python-cffi-env]
PATH = ${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH = ${libffi:location}/lib/pkgconfig
[buildout]
extends =
../libffi/buildout.cfg
../openssl/buildout.cfg
../pkgconfig/buildout.cfg
parts =
python-cryptography
[python-cryptography]
recipe = zc.recipe.egg:custom
egg = cryptography
environment = python-cryptography-env
[python-cryptography-env]
PATH = ${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH = ${libffi:location}/lib/pkgconfig
LD_LIBRARY_PATH = ${libffi:location}/lib
CPATH = ${openssl:location}/include
[buildout]
extends =
../openssl/buildout.cfg
parts =
python-openssl
[python-openssl]
recipe = zc.recipe.egg:custom
egg = pyOpenSSL
include-dirs =
${openssl:location}/include/
library-dirs =
${openssl:location}/lib/
rpath =
${openssl:location}/lib/
...@@ -17,7 +17,8 @@ extends = ...@@ -17,7 +17,8 @@ extends =
../pkgconfig/buildout.cfg ../pkgconfig/buildout.cfg
../popt/buildout.cfg ../popt/buildout.cfg
../python-2.7/buildout.cfg ../python-2.7/buildout.cfg
../python-openssl/buildout.cfg ../python-cffi/buildout.cfg
../python-cryptography/buildout.cfg
../readline/buildout.cfg ../readline/buildout.cfg
../sqlite3/buildout.cfg ../sqlite3/buildout.cfg
../swig/buildout.cfg ../swig/buildout.cfg
...@@ -91,14 +92,19 @@ output = ${buildout:directory}/environment.sh ...@@ -91,14 +92,19 @@ output = ${buildout:directory}/environment.sh
[lxml-python] [lxml-python]
python = python2.7 python = python2.7
[python-openssl] [python-cffi]
python = python2.7
[python-cryptography]
python = python2.7 python = python2.7
[slapos] [slapos]
recipe = z3c.recipe.scripts recipe = z3c.recipe.scripts
python = python2.7 python = python2.7
eggs = eggs =
${python-openssl:egg} ${python-cffi:egg}
${python-cryptography:egg}
pyOpenSSL
slapos.libnetworkcache slapos.libnetworkcache
zc.buildout zc.buildout
${lxml-python:egg} ${lxml-python:egg}
...@@ -150,14 +156,16 @@ MarkupSafe = 0.18 ...@@ -150,14 +156,16 @@ MarkupSafe = 0.18
Pygments = 1.6 Pygments = 1.6
Werkzeug = 0.9.4 Werkzeug = 0.9.4
buildout-versions = 1.7 buildout-versions = 1.7
cffi = 0.8.1
cmd2 = 0.6.7 cmd2 = 0.6.7
collective.recipe.template = 1.10 collective.recipe.template = 1.10
cryptography = 0.2.1
itsdangerous = 0.23 itsdangerous = 0.23
lxml = 3.2.3 lxml = 3.2.3
meld3 = 0.6.10 meld3 = 0.6.10
netaddr = 0.7.10 netaddr = 0.7.10
prettytable = 0.7.2 prettytable = 0.7.2
pyOpenSSL = 0.13.1 pyOpenSSL = 0.14
pyparsing = 2.0.1 pyparsing = 2.0.1
setuptools = 1.1.6 setuptools = 1.1.6
slapos.libnetworkcache = 0.13.4 slapos.libnetworkcache = 0.13.4
......
...@@ -207,6 +207,8 @@ setup(name=name, ...@@ -207,6 +207,8 @@ setup(name=name,
'zabbixagent = slapos.recipe.zabbixagent:Recipe', 'zabbixagent = slapos.recipe.zabbixagent:Recipe',
'zimbra.kvm = slapos.recipe.zimbra_kvm:Recipe', 'zimbra.kvm = slapos.recipe.zimbra_kvm:Recipe',
'zeo = slapos.recipe.zeo:Recipe', 'zeo = slapos.recipe.zeo:Recipe',
'zeroknown.read = slapos.recipe.zeroknown:ReadRecipe',
'zeroknown.write = slapos.recipe.zeroknown:WriteRecipe'
], ],
'slapos.recipe.nosqltestbed.plugin': [ 'slapos.recipe.nosqltestbed.plugin': [
'kumo = slapos.recipe.nosqltestbed.kumo:KumoTestBed', 'kumo = slapos.recipe.nosqltestbed.kumo:KumoTestBed',
......
These recipes provide the ability to save some buildout parameters and their value in a custom file, inside the instance folder.
In both recipes, you HAVE TO give a filename, which will be stored at the root of the instance folder
WriteRecipe :
-------------
* Is used to create a section (named according to the buildout section_name).
* You can give then as much parameters you wish, with their default values.
* Whenever you run buildout, if the parameter has yet been saved in the config file, it will do nothing.
* If the parameter's value has changed in the config file, it won't be overwritten
* /!\ If you decide to change the default value of one parameter, ALL other parameters will be reseted in the config file, even if you changed it manually. Explanation : The default values aren't expected to change, except while development purposes.
ReadRecipe :
* It fills its own section with all the options in all the sections of the config file.
...@@ -71,7 +71,7 @@ class ExportRecipe(GenericBaseRecipe): ...@@ -71,7 +71,7 @@ class ExportRecipe(GenericBaseRecipe):
done done
} }
sync_element %(srv-directory)s/runner %(backup-directory)s/runner/ instance project proxy.db sync_element %(srv-directory)s/runner %(backup-directory)s/runner/ instance project proxy.db
sync_element %(etc-directory)s %(backup-directory)s/etc/ .rcode .project .users .htpasswd ssh sync_element %(etc-directory)s %(backup-directory)s/etc/ .rcode .project .users .htpasswd config.json ssh
if [ -d %(backup-directory)s/runner/software ]; then if [ -d %(backup-directory)s/runner/software ]; then
rm %(backup-directory)s/runner/software/* rm %(backup-directory)s/runner/software/*
fi fi
...@@ -120,7 +120,7 @@ class ImportRecipe(GenericBaseRecipe): ...@@ -120,7 +120,7 @@ class ImportRecipe(GenericBaseRecipe):
done done
} }
restore_element %(backup-directory)s/runner/ %(srv-directory)s/runner instance project proxy.db restore_element %(backup-directory)s/runner/ %(srv-directory)s/runner instance project proxy.db
restore_element %(backup-directory)s/etc/ %(etc-directory)s .rcode .project .users .htpasswd ssh restore_element %(backup-directory)s/etc/ %(etc-directory)s .rcode .project .users .htpasswd config.json ssh
ifs=$IFS IFS=';' ifs=$IFS IFS=';'
read user pass remaining < %(etc-directory)s/.users read user pass remaining < %(etc-directory)s/.users
IFS=$ifs IFS=$ifs
......
##############################################################################
#
# Copyright (c) 2013 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import ConfigParser
import os
import zc.buildout
from slapos.recipe.librecipe import GenericBaseRecipe
class WriteRecipe(GenericBaseRecipe):
"""
"""
def __init__(self, buildout, name, options):
if not "filename" in options:
raise zc.buildout.UserError("You have to provide the parameter \"filename\"")
self.filename = options['filename'].strip()
self.path = os.path.join(buildout['buildout']['directory'], self.filename)
self.name = name
self.options = options.copy()
del self.options['filename']
del self.options['recipe']
# Set up the parser, and write config file if needed
self.parser = ConfigParser.ConfigParser()
try:
self.parser.read(self.path)
#clean_options(options)
for key in self.options:
if key not in self.parser.options(self.name):
self.parser.set(self.name, key, self.options[key])
with open(self.path, 'w') as file:
self.parser.write(file)
# If the file or section do not exist
except (ConfigParser.NoSectionError, IOError) as e:
self.full_install()
install = update = lambda self: []
def full_install(self):
"""XXX-Nicolas : when some parameter's value is changed in
buildout profile, this will override custom user defined values"""
self.parser.read(self.path)
if self.parser.has_section(self.name):
self.parser.remove_section(self.name)
self.parser.add_section(self.name)
for key in self.options:
self.parser.set(self.name, key, self.options[key])
with open(self.path, 'w') as file:
self.parser.write(file)
class ReadRecipe(GenericBaseRecipe):
"""
"""
def __init__(self, buildout, name, options):
if not "filename" in options:
raise zc.buildout.UserError("You have to provide the parameter \"filename\"")
self.filename = options['filename'].strip()
self.path = os.path.join(buildout['buildout']['directory'], self.filename)
# Set up the parser, and write config file if needed
self.parser = ConfigParser.ConfigParser()
if os.path.exists(self.path):
self.parser.read(self.path)
for section in self.parser.sections():
for key ,value in self.parser.items(section):
options[key] = value
install = update = lambda self: []
...@@ -23,6 +23,7 @@ crontabs = $${:etc}/crontabs ...@@ -23,6 +23,7 @@ crontabs = $${:etc}/crontabs
cronstamps = $${:etc}/cronstamps cronstamps = $${:etc}/cronstamps
backup = $${:srv}/backup backup = $${:srv}/backup
status = $${:srv}/status status = $${:srv}/status
statistic = $${:srv}/statistic
backupscript = $${:etc}/backup backupscript = $${:etc}/backup
www = $${:srv}/www www = $${:srv}/www
home = $${:etc}/home home = $${:etc}/home
...@@ -111,7 +112,9 @@ include = {{ include_string }} ...@@ -111,7 +112,9 @@ include = {{ include_string }}
exclude_string = {{ exclude_string }} exclude_string = {{ exclude_string }}
remote_schema = {{ remote_schema }} remote_schema = {{ remote_schema }}
status_name = {{ slave_reference }}_status.txt status_name = {{ slave_reference }}_status.txt
statistic_name = {{ slave_reference }}_statistic.txt
status_log = $${directory:status}/$${:status_name} status_log = $${directory:status}/$${:status_name}
statistic_log = $${directory:statistic}/$${:statistic_name}
[{{ slave_reference }}-backup-crontab-line] [{{ slave_reference }}-backup-crontab-line]
recipe = slapos.recipe.template recipe = slapos.recipe.template
......
...@@ -12,6 +12,7 @@ extends = ...@@ -12,6 +12,7 @@ extends =
../../component/rsync/buildout.cfg ../../component/rsync/buildout.cfg
../../component/dropbear/buildout.cfg ../../component/dropbear/buildout.cfg
../../component/grep/buildout.cfg ../../component/grep/buildout.cfg
../../component/findutils/buildout.cfg
# ../../stack/flask.cfg # ../../stack/flask.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
...@@ -196,7 +197,7 @@ mode = 0644 ...@@ -196,7 +197,7 @@ mode = 0644
[template-backup-script] [template-backup-script]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-backup-script.sh.in url = ${:_profile_base_location_}/template-backup-script.sh.in
md5sum = 8a076962fc4df7f154572543899328e3 md5sum = 3c4ad4b9a1ecef88f35a84ceb84a3ec6
output = ${buildout:directory}/template-backup-script.sh.in output = ${buildout:directory}/template-backup-script.sh.in
mode = 0644 mode = 0644
...@@ -217,7 +218,7 @@ mode = 0644 ...@@ -217,7 +218,7 @@ mode = 0644
[status2rss] [status2rss]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/status2rss.py url = ${:_profile_base_location_}/status2rss.py
md5sum = 138c96e0836f2b06414b98ba2643f21c md5sum = 0cd1cf97b199dd18fc0168c7281890ea
output = ${buildout:directory}/status2rss.py output = ${buildout:directory}/status2rss.py
mode = 0644 mode = 0644
...@@ -234,7 +235,7 @@ mode = 0644 ...@@ -234,7 +235,7 @@ mode = 0644
[template-pullrdiffbackup] [template-pullrdiffbackup]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pullrdiffbackup.cfg.in url = ${:_profile_base_location_}/instance-pullrdiffbackup.cfg.in
md5sum = 9bf3a34fa41ae6fe57b183293b3ff377 md5sum = 061b98d001b501c9e1beb424e8802d3d
output = ${buildout:directory}/template-pullrdiffbackup.cfg output = ${buildout:directory}/template-pullrdiffbackup.cfg
mode = 0644 mode = 0644
...@@ -327,3 +328,12 @@ cliff = 1.4 ...@@ -327,3 +328,12 @@ cliff = 1.4
cmd2 = 0.6.5.1 cmd2 = 0.6.5.1
prettytable = 0.7.2 prettytable = 0.7.2
requests = 1.2.3 requests = 1.2.3
cffi = 0.8.1
cryptography = 0.2.1
pyOpenSSL = 0.14
six = 1.5.2
# Required by:
# cffi==0.8.1
pycparser = 2.10
...@@ -23,11 +23,12 @@ while 1: ...@@ -23,11 +23,12 @@ while 1:
if not line: if not line:
break break
time, desc = line.split(',', 1) time, statistic, desc = line.split(',', 2)
rss_item = PyRSS2Gen.RSSItem( rss_item = PyRSS2Gen.RSSItem(
title = desc, title = desc,
description = "%s, %s" % (time, desc), description = "<p>%s</p>" % "<br/>".join(("%s, %s\n<a href='http://www.nongnu.org/rdiff-backup/FAQ.html#statistics'>Lastest statistic</a>\n%s" % (time, desc,
open(statistic).read())).split("\n")),
link = LINK, link = LINK,
pubDate = datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz(time))), pubDate = datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz(time))),
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc))) guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc)))
......
...@@ -6,9 +6,11 @@ export HOME=$${directory:home} ...@@ -6,9 +6,11 @@ export HOME=$${directory:home}
# Clean status file (no history needed) # Clean status file (no history needed)
${coreutils-output:rm} -f $${:status_log} ${coreutils-output:rm} -f $${:status_log}
${coreutils-output:rm} -f $${:statistic_log}
# Inform about beginning of backup # Inform about beginning of backup
${coreutils-output:echo} "`${coreutils-output:date} -u`, $${:hostname} backup running" >> $${:status_log} ${coreutils-output:echo} "`${coreutils-output:date} -u`, $${:hostname} backup running" >> $${:status_log}
${coreutils-output:echo} "Available only if backup succeed." >> $${:statistic_log}
# set -e # set -e
cd $${:datadirectory} cd $${:datadirectory}
...@@ -25,9 +27,11 @@ RESULT=$? ...@@ -25,9 +27,11 @@ RESULT=$?
${coreutils-output:rm} -f $${:status_log} ${coreutils-output:rm} -f $${:status_log}
if [ $RESULT -eq 0 ] if [ $RESULT -eq 0 ]
then then
${coreutils-output:echo} "`${coreutils-output:date} -u`, $${:hostname} backup success" >> $${:status_log} ${coreutils-output:echo} "`${coreutils-output:date} -u`,$${:statistic_log},$${:hostname} backup success" >> $${:status_log}
${coreutils-output:rm} -f $${:statistic_log}
${findutils-output:find} rdiff-backup-data/ -name "session_statistic*" | ${coreutils-output:sort} | ${coreutils-output:tail} -n 1 | ${findutils-output:xargs} ${rdiff-backup-output:rdiff-backup} --calculate-average >> $${:statistic_log}
else else
${coreutils-output:echo} "`${coreutils-output:date} -u`, $${:hostname} backup failed" >> $${:status_log} ${coreutils-output:echo} "`${coreutils-output:date} -u`,$${:statistic_log},$${:hostname} backup failed" >> $${:status_log}
fi fi
# python scripts/verify_with_sudo.py ./ $${:connection}:/ # python scripts/verify_with_sudo.py ./ $${:connection}:/
......
...@@ -6,6 +6,8 @@ find-links += http://dist.repoze.org ...@@ -6,6 +6,8 @@ find-links += http://dist.repoze.org
extends = extends =
../../stack/slapos.cfg ../../stack/slapos.cfg
../../component/python-2.7/buildout.cfg ../../component/python-2.7/buildout.cfg
../../component/python-cffi/buildout.cfg
../../component/python-cryptography/buildout.cfg
../../component/git/buildout.cfg ../../component/git/buildout.cfg
../../component/lxml-python/buildout.cfg ../../component/lxml-python/buildout.cfg
../../component/zip/buildout.cfg ../../component/zip/buildout.cfg
...@@ -67,6 +69,12 @@ python = python2.7 ...@@ -67,6 +69,12 @@ python = python2.7
[lxml-python] [lxml-python]
python = python2.7 python = python2.7
[python-cffi]
python = python2.7
[python-cryptography]
python = python2.7
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg url = ${:_profile_base_location_}/instance.cfg
...@@ -196,7 +204,9 @@ PyXML = 0.8.5 ...@@ -196,7 +204,9 @@ PyXML = 0.8.5
Pygments = 1.6 Pygments = 1.6
Werkzeug = 0.9.4 Werkzeug = 0.9.4
buildout-versions = 1.7 buildout-versions = 1.7
cffi = 0.8.1
cmd2 = 0.6.7 cmd2 = 0.6.7
cryptography = 0.2.1
erp5.util = 0.4.36 erp5.util = 0.4.36
inotifyx = 0.2.0-1 inotifyx = 0.2.0-1
itsdangerous = 0.23 itsdangerous = 0.23
...@@ -205,7 +215,7 @@ meld3 = 0.6.10 ...@@ -205,7 +215,7 @@ meld3 = 0.6.10
netaddr = 0.7.10 netaddr = 0.7.10
plone.recipe.command = 1.1 plone.recipe.command = 1.1
psutil = 1.2.1 psutil = 1.2.1
pyOpenSSL = 0.13.1 pyOpenSSL = 0.14
pyparsing = 2.0.1 pyparsing = 2.0.1
pytz = 2013.9 pytz = 2013.9
slapos.core = 1.0.2.1 slapos.core = 1.0.2.1
......
#############################
#
# Deploy officejs instance
#
#############################
[buildout]
parts =
test-runner
phantomjs-wrapper
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
# Create all needed directories, depending on your needs
[directory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
bin = $${buildout:directory}/bin/
[download-source]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
# Local development
[officejs]
<= download-source
repository = ${officejs-repository:location}
[test-runner]
recipe = slapos.cookbook:egg_test
run-test-suite = $${directory:bin}/runTestSuite
run-test-suite-binary = ${buildout:bin-directory}/runTestSuite
test-list =
$${officejs:location}
prepend-path = ${git:location}/bin:${libxslt:location}/bin:${python2.7:location}/bin:${buildout:bin-directory}
environment = environment
[environment]
CPPFLAGS = -I${python2.7:location}/include/python2.7 -I${libxml2:location}/include -I${libxslt:location}/include
LDFLAGS = -L${python2.7:location}/lib -L${libxml2:location}/lib -L${libxslt:location}/lib -L${zlib:location}/lib -L${fontconfig:location}/lib -L${libexpat:location}/lib -L${freetype:location}/lib
PYTHONPATH = ${python-setuptools:location}
LD_LIBRARY_PATH = ${libxslt:location}/lib:${libxml2:location}/lib:${zlib:location}/lib:${fontconfig:location}/lib:${libexpat:location}/lib:${freetype:location}/lib
[phantomjs-wrapper]
recipe = slapos.cookbook:wrapper
command-line = ${phantomjs:location}/phantomjs-slapos
wrapper-path = $${directory:bin}/phantomjs
# XXX we must use "parameters-extra" (- instead of _)
# when new slapos.cookbook will be used
parameters_extra = true
[buildout]
extends =
../../stack/slapos.cfg
../../component/git/buildout.cfg
../../component/phantomjs/buildout.cfg
../../component/git/buildout.cfg
../../component/python-2.7/buildout.cfg
../../component/python-setuptools/buildout.cfg
parts =
template
phantomjs
eggs
[eggs]
recipe = zc.recipe.egg
eggs =
${lxml-python:egg}
slapos.cookbook
erp5.util
collective.recipe.template
entry-points =
runTestSuite=erp5.util.testsuite:runTestSuite
scripts =
runTestSuite
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
output = ${buildout:directory}/instance.cfg
# MD5 checksum can be skipped for development, but must be filled for production
md5sum = bad9eca44660b819a69816e53c7d5428
mode = 0644
[officejs-repository]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
forbid-download-cache = true
repository = http://git.erp5.org/repos/officejs.git
branch = jqs
[versions]
Jinja2 = 2.6
Werkzeug = 0.8.3
buildout-versions = 1.7
collective.recipe.template = 1.10
erp5.util = 0.4.34
hexagonit.recipe.cmmi = 2.0
lxml = 3.1.1
meld3 = 0.6.10
plone.recipe.command = 1.1
psutil = 0.6.1
slapos.cookbook = 0.78.1
slapos.recipe.build = 0.12
slapos.recipe.template = 2.4.2
# Required by:
# slapos.core==0.35.1
Flask = 0.9
# Required by:
# slapos.cookbook==0.76.0
inotifyx = 0.2.0
# Required by:
# slapos.cookbook==0.76.0
netaddr = 0.7.10
# Required by:
# slapos.core==0.35.1
netifaces = 0.8
# Required by:
# slapos.core==0.35.1
pyflakes = 0.6.1
# Required by:
# slapos.cookbook==0.76.0
pytz = 2013b
# Required by:
# collective.recipe.template==1.10
# erp5.util==0.4.33
# hexagonit.recipe.download==1.6nxd002
# slapos.cookbook==0.76.0
# slapos.core==0.35.1
# supervisor==3.0b1
# zc.buildout==1.6.0-dev-SlapOS-010
# zope.interface==4.0.5
setuptools = 0.6c12dev-r88846
# Required by:
# slapos.cookbook==0.76.0
slapos.core = 0.35.1
# Required by:
# slapos.core==0.35.1
supervisor = 3.0b1
# Required by:
# slapos.core==0.35.1
unittest2 = 0.5.1
# Required by:
# slapos.cookbook==0.76.0
xml-marshaller = 0.9.7
# Required by:
# slapos.core==0.35.1
zope.interface = 4.0.5
...@@ -5,6 +5,8 @@ extends = ...@@ -5,6 +5,8 @@ extends =
../../component/xorg/buildout.cfg ../../component/xorg/buildout.cfg
../../component/lxml-python/buildout.cfg ../../component/lxml-python/buildout.cfg
../../component/python-2.7/buildout.cfg ../../component/python-2.7/buildout.cfg
../../component/python-cffi/buildout.cfg
../../component/python-cryptography/buildout.cfg
../../component/firefox/buildout.cfg ../../component/firefox/buildout.cfg
../../component/dash/buildout.cfg ../../component/dash/buildout.cfg
../../stack/shacache-client.cfg ../../stack/shacache-client.cfg
...@@ -23,6 +25,15 @@ parts = ...@@ -23,6 +25,15 @@ parts =
unzip = true unzip = true
[lxml-python]
python = python2.7
[python-cffi]
python = python2.7
[python-cryptography]
python = python2.7
[instance-recipe] [instance-recipe]
egg = slapos.cookbook egg = slapos.cookbook
module = seleniumrunner module = seleniumrunner
...@@ -54,70 +65,90 @@ output = ${buildout:directory}/template-selenium.cfg ...@@ -54,70 +65,90 @@ output = ${buildout:directory}/template-selenium.cfg
mode = 0644 mode = 0644
[versions] [versions]
# Use SlapOS patched zc.buildout # pin version of setuptools
zc.buildout = 1.6.0-dev-SlapOS-004 setuptools = 2.2
Jinja2 = 2.6 Jinja2 = 2.7.2
Werkzeug = 0.8.3 MarkupSafe = 0.18
Werkzeug = 0.9.4
buildout-versions = 1.7 buildout-versions = 1.7
hexagonit.recipe.cmmi = 1.5.0 cffi = 0.8.1
meld3 = 0.6.8 cmd2 = 0.6.7
slapos.cookbook = 0.46 cryptography = 0.2.1
slapos.recipe.build = 0.7 itsdangerous = 0.23
slapos.recipe.template = 2.2 meld3 = 0.6.10
pyOpenSSL = 0.14
pyparsing = 2.0.1
slapos.cookbook = 0.85
slapos.recipe.build = 0.12
slapos.recipe.cmmi = 0.2
slapos.recipe.template = 2.5
# Required by:
# slapos.core==1.0.3
Flask = 0.10.1
# Required by: # Required by:
# slapos.core==0.23 # slapos.core==1.0.3
Flask = 0.8 cliff = 1.5.2
# Required by: # Required by:
# slapos.cookbook==0.42 # slapos.cookbook==0.85
PyXML = 0.8.4 inotifyx = 0.2.0-1
# Required by: # Required by:
# slapos.cookbook==0.42 # slapos.cookbook==0.85
inotifyx = 0.2.0 lock-file = 2.0
# Required by: # Required by:
# slapos.cookbook==0.42 # slapos.cookbook==0.85
# slapos.core==0.23 # slapos.core==1.0.3
# xml-marshaller==0.9.7 # xml-marshaller==0.9.7
lxml = 2.3.4 lxml = 3.3.1
# Required by: # Required by:
# slapos.cookbook==0.42 # slapos.cookbook==0.85
netaddr = 0.7.6 netaddr = 0.7.10
# Required by: # Required by:
# slapos.core==0.23 # slapos.core==1.0.3
netifaces = 0.8 netifaces = 0.8-1
# Required by: # Required by:
# slapos.cookbook==0.42 # cffi==0.8.1
# slapos.core==0.23 pycparser = 2.10
# zc.buildout==1.5.3-dev-SlapOS-010
# zc.recipe.egg==1.3.2
setuptools = 0.6c12dev-r88846
# Required by: # Required by:
# slapos.cookbook==0.42 # slapos.cookbook==0.85
slapos.core = 0.23 pytz = 2013.9
# Required by: # Required by:
# slapos.core==0.23 # slapos.core==1.0.3
supervisor = 3.0a12 requests = 2.2.1
# Required by: # Required by:
# slapos.cookbook==0.42 # cliff==1.5.2
xml-marshaller = 0.9.7 six = 1.5.2
# Required by:
# slapos.cookbook==0.85
slapos.core = 1.0.3
# Required by:
# cliff==1.5.2
stevedore = 0.14.1
# Required by:
# slapos.core==1.0.3
supervisor = 3.0
# Required by: # Required by:
# slapos.cookbook==0.42 # slapos.cookbook==0.85
zc.recipe.egg = 1.3.2 xml-marshaller = 0.9.7
# Required by: # Required by:
# slapos.core==0.23 # slapos.core==1.0.3
zope.interface = 3.8.0 zope.interface = 4.1.0
[networkcache] [networkcache]
# signature certificates of the following uploaders. # signature certificates of the following uploaders.
......
[buildout] [buildout]
parts = parts =
slapos.core-setup slapos.core-setup
erp5.util-setup
phantomjs-wrapper
slapos-test-runner slapos-test-runner
sh-environment sh-environment
...@@ -30,6 +32,9 @@ recipe = plone.recipe.command ...@@ -30,6 +32,9 @@ recipe = plone.recipe.command
command = echo "Updating setup...";cd $${slapos.core:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; python setup.py test -n; python setup.py test -n; python setup.py test -n; python setup.py test -n command = echo "Updating setup...";cd $${slapos.core:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; python setup.py test -n; python setup.py test -n; python setup.py test -n; python setup.py test -n
update-command = $${:command} update-command = $${:command}
[slapos.package]
<= download-source
repository = ${slapos.package-repository:location}
[slapos.recipe.template] [slapos.recipe.template]
<= download-source <= download-source
...@@ -39,6 +44,23 @@ repository = ${slapos.recipe.template-repository:location} ...@@ -39,6 +44,23 @@ repository = ${slapos.recipe.template-repository:location}
<= download-source <= download-source
repository = ${slapos.recipe.build-repository:location} repository = ${slapos.recipe.build-repository:location}
[erp5-util]
<= download-source
repository = ${erp5-util-repository:location}
[erp5.util-setup]
recipe = plone.recipe.command
command = echo "Updating setup...";cd $${erp5-util:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; python setup.py test -n; python setup.py test -n; python setup.py test -n; python setup.py test -n; python setup.py test -n
update-command = $${:command}
[officejs]
<= download-source
repository = ${officejs-repository:location}
[jio]
<= download-source
repository = ${jio-repository:location}
[slapos-test-runner] [slapos-test-runner]
recipe = slapos.cookbook:egg_test recipe = slapos.cookbook:egg_test
run-test-suite = $${create-directory:bin}/runTestSuite run-test-suite = $${create-directory:bin}/runTestSuite
...@@ -48,8 +70,12 @@ run-test-suite-binary = ${buildout:bin-directory}/runTestSuite ...@@ -48,8 +70,12 @@ run-test-suite-binary = ${buildout:bin-directory}/runTestSuite
#python-list = $${} #python-list = $${}
test-list = test-list =
$${slapos.core:location} $${slapos.core:location}
$${slapos.package:location}
$${slapos.recipe.template:location} $${slapos.recipe.template:location}
$${slapos.recipe.build:location} $${slapos.recipe.build:location}
$${erp5-util:location}
$${jio:location}
$${officejs:location}
prepend-path = ${git:location}/bin:${libxslt:location}/bin:${python2.7:location}/bin prepend-path = ${git:location}/bin:${libxslt:location}/bin:${python2.7:location}/bin
environment = environment environment = environment
...@@ -71,3 +97,9 @@ input = inline: ...@@ -71,3 +97,9 @@ input = inline:
export PS1="[slapos-testing env Active] $PS1" export PS1="[slapos-testing env Active] $PS1"
output = $${create-directory:bin}/environment.sh output = $${create-directory:bin}/environment.sh
mode = 755 mode = 755
[phantomjs-wrapper]
recipe = slapos.cookbook:wrapper
command-line = ${phantomjs:location}/phantomjs-slapos
wrapper-path = $${create-directory:bin}/phantomjs
parameters-extra = true
...@@ -8,14 +8,20 @@ extends = ...@@ -8,14 +8,20 @@ extends =
../../component/python-2.7/buildout.cfg ../../component/python-2.7/buildout.cfg
../../component/python-setuptools/buildout.cfg ../../component/python-setuptools/buildout.cfg
../../component/zlib/buildout.cfg ../../component/zlib/buildout.cfg
../../component/phantomjs/buildout.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
parts = parts =
slapos.cookbook-repository slapos.cookbook-repository
slapos.core-repository slapos.core-repository
slapos.package-repository
slapos.recipe.template-repository slapos.recipe.template-repository
slapos.recipe.build-repository slapos.recipe.build-repository
erp5-util-repository
officejs-repository
jio-repository
eggs eggs
phantomjs
template template
[eggs] [eggs]
...@@ -45,6 +51,13 @@ forbid-download-cache = true ...@@ -45,6 +51,13 @@ forbid-download-cache = true
repository = http://git.erp5.org/repos/slapos.core.git repository = http://git.erp5.org/repos/slapos.core.git
branch = master branch = master
[slapos.package-repository]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
forbid-download-cache = true
repository = http://git.erp5.org/repos/slapos.package.git
branch = master
[slapos.recipe.template-repository] [slapos.recipe.template-repository]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
...@@ -59,10 +72,31 @@ forbid-download-cache = true ...@@ -59,10 +72,31 @@ forbid-download-cache = true
repository = http://git.erp5.org/repos/slapos.recipe.build.git repository = http://git.erp5.org/repos/slapos.recipe.build.git
branch = master branch = master
[erp5-util-repository]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
forbid-download-cache = true
repository = http://git.erp5.org/repos/erp5.git
branch = master
[officejs-repository]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
forbid-download-cache = true
repository = http://git.erp5.org/repos/officejs.git
branch = jqs
[jio-repository]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
forbid-download-cache = true
repository = http://git.erp5.org/repos/jio.git
branch = master
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg url = ${:_profile_base_location_}/instance.cfg
md5sum = 1330c8ccd1dab485aadba5e688d2b2e1 md5sum = b065539ca58a3400b0ed30093ab020ac
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
mode = 640 mode = 640
......
...@@ -29,9 +29,18 @@ Optional parameter. ...@@ -29,9 +29,18 @@ Optional parameter.
Default: 10 Default: 10
Public-directory
----------------
You can serve static files with the webrunner. For that, just put your data in "srv/runner/public". All these files will be served throught the url of the webrunner + "/public/". Useful for developping your own static website.
AUTO-DEPLOYMENT AUTO-DEPLOYMENT
--------------- ---------------
for software
~~~~~~~~~~~~
You can automatically deploy a software release while deploying the webrunner itself, using the paramater XML. You can automatically deploy a software release while deploying the webrunner itself, using the paramater XML.
To do this, you only need to pass as a parameter named "slapos-software" : "AAA/BBB", where AAA is the folder of slapos.git where is located your BBB software. To do this, you only need to pass as a parameter named "slapos-software" : "AAA/BBB", where AAA is the folder of slapos.git where is located your BBB software.
...@@ -43,6 +52,23 @@ It is also possible to download you own git repository, by providing the url in ...@@ -43,6 +52,23 @@ It is also possible to download you own git repository, by providing the url in
Last but not least, it is also possible to switch the branch with the parameter "slapos-reference" (by default pointing on master) Last but not least, it is also possible to switch the branch with the parameter "slapos-reference" (by default pointing on master)
for instance
~~~~~~~~~~~~
The parameter "auto-deploy-instance" can be explicitly set to allow or prevent the runner to deploy the instance at START TIME (if you manually restart the runner, or if the server reboots). Values : "true" or "false". Default value is "true", except for the instances of import (while type is resilient or test) which is "false"
There also exists the parameter "autorun", which will build&run your software if set to true. For this, you need "auto_deploy" to true, and set the parameter "slapos-software" to the software you want to deploy. Do not hesitate to clone a different repo than "slapos", or to change the tag/branch to use your custom Software Release. (see "slapos-repository" and "slapos-reference" in previous section).
To deploy the instance with some parameters, just give to the runner parameters starting with "parameter-", they will be correctly forwarded to the instance, which will use them for its configuration. For example, if you want to send to the sofware helloworld the parameter "name" with the value "nicolas", here is how to configure the parameter.xml of the webrunner for auto-depolyment :
<?xml version='1.0' encoding='utf-8'?>
<instance>
<parameter id="slapos-software">software/helloworld</parameter>
<parameter id="auto_deploy">true</parameter>
<parameter id="autorun">true</parameter>
<parameter id="parameter-name">nicolas</parameter>
</instance>
Resilience : Resilience :
------------ ------------
...@@ -95,7 +121,32 @@ Example : ...@@ -95,7 +121,32 @@ Example :
<parameter id="custom-frontend-basic-auth">true</parameter> <parameter id="custom-frontend-basic-auth">true</parameter>
</instance> </instance>
Git repositories :
------------------
It is easy to give access to your git repository/ies to everyone, or to clone it on your own computer. For this, there are 2 urls to remember:
- For read only, you can clone : https://[IPV6]:PORT/git-public/YourRepo.git/
- For read and write access, using your runner account : https://[IPV6]:PORT/git/YourRepo.git/
To create the repo, go in the folder srv/runner/project and initiate a new git repo (git init/clone --bare XXX).
For the moment, the PORT is the port of monitoring, which is 9685.
Things to notice for the nex developer : Things to notice for the nex developer :
---------------------------------------- ----------------------------------------
As you can see in instance-runner-*.cfg, the buildout section extends a hard-coded template file. If one day you need to modify the filename, do not forget to modify it in instance.cfg, but also in these files ! (the problem is that the content of instance.cfg is not known by buildout while the deployment of the software release) As you can see in instance-runner-*.cfg, the buildout section extends a hard-coded template file. If one day you need to modify the filename, do not forget to modify it in instance.cfg, but also in these files ! (the problem is that the content of instance.cfg is not known by buildout while the deployment of the software release)
List of ports used by the webrunner:
------------------------------------
8602 : slapproxy, while running tests
8080 : shellinabox
9684 : apache (monitoring of slaprunner, git access)
22222 : dropbear
50000 : slapproxy
50005 : webrunner (flask app), webdav access
Tips:
-----
You can use shellinabox in fullscreen, by accessing : https://[IPV6]:8080
- resilient sr: Cloned instances should not launch slapgrid-sr if it was not launched on export instance - resilient sr: Cloned instances should not launch slapgrid-sr if it was not launched on export instance
- shellinabox password should be the same in all the resilient instances - shellinabox password should be the same in all the resilient instances
- add test for parameter auto-deploy-instance
...@@ -19,12 +19,12 @@ extends = ...@@ -19,12 +19,12 @@ extends =
../../stack/flask.cfg ../../stack/flask.cfg
../../stack/shacache-client.cfg ../../stack/shacache-client.cfg
../../stack/resilient/buildout.cfg ../../stack/resilient/buildout.cfg
../../stack/monitor/buildout.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
parts = parts =
template template
eggs eggs
nginx
instance-runner-import instance-runner-import
instance-runner-export instance-runner-export
slapos-cookbook slapos-cookbook
...@@ -33,39 +33,38 @@ parts = ...@@ -33,39 +33,38 @@ parts =
rdiff-backup rdiff-backup
collective.recipe.template-egg collective.recipe.template-egg
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
md5sum = 8a47421ac6158b4ee476acab212c67d9 #md5sum = 8a47421ac6158b4ee476acab212c67d9
mode = 0644 mode = 0644
[template-runner] [template-runner]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner.cfg url = ${:_profile_base_location_}/instance-runner.cfg
output = ${buildout:directory}/template-runner.cfg.in output = ${buildout:directory}/template-runner.cfg.in
md5sum = b3288126400c4ca8469c255ef130dec9 md5sum = c66f18b74b958805bb2196fb1c54683e
mode = 0644 mode = 0644
[instance-runner-import] [instance-runner-import]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner-import.cfg.in url = ${:_profile_base_location_}/instance-runner-import.cfg.in
output = ${buildout:directory}/instance-runner-import.cfg output = ${buildout:directory}/instance-runner-import.cfg
md5sum = 82f2d0be111617eac9849cb7b8baac5d md5sum = e1769596ef62f6db6bbfca4391e740c0
mode = 0644 mode = 0644
[instance-runner-export] [instance-runner-export]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner-export.cfg.in url = ${:_profile_base_location_}/instance-runner-export.cfg.in
output = ${buildout:directory}/instance-runner-export.cfg output = ${buildout:directory}/instance-runner-export.cfg
md5sum = 9a8cdf65bca8a562accb710a7c3b6595 md5sum = b7664995a6dd954f4f214252ffeaaa61
mode = 0644 mode = 0644
[template-resilient] [template-resilient]
recipe = slapos.recipe.download recipe = slapos.recipe.download
url = ${:_profile_base_location_}/instance-resilient.cfg.jinja2 url = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
md5sum = f8758a5a2f886d649ad02157d76550ba md5sum = a9647ae3c99308b85dd72fae76ef071f
filename = instance-resilient.cfg.jinja2 filename = instance-resilient.cfg.jinja2
mode = 0644 mode = 0644
...@@ -73,7 +72,7 @@ mode = 0644 ...@@ -73,7 +72,7 @@ mode = 0644
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/instance-resilient-test.cfg.jinja2 url = ${:_profile_base_location_}/instance-resilient-test.cfg.jinja2
download-only = true download-only = true
md5sum = b231383bb63edc0009fb6e0ff5c54d8e #md5sum = b231383bb63edc0009fb6e0ff5c54d8e
filename = instance-resilient-test.cfg.jinja2 filename = instance-resilient-test.cfg.jinja2
mode = 0644 mode = 0644
...@@ -81,14 +80,22 @@ mode = 0644 ...@@ -81,14 +80,22 @@ mode = 0644
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/nginx_conf.in url = ${:_profile_base_location_}/nginx_conf.in
download-only = true download-only = true
md5sum = 67d2ce92b4ad6dca6eaf69eeb2c2734c md5sum = fa66988e96bb712c0580b5fb90f95700
filename = nginx_conf.in filename = nginx_conf.in
mode = 0644 mode = 0644
[template_httpd_conf]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/httpd_conf.in
download-only = true
md5sum = 0b63a652d1d51e0894a780896fce9893
filename = httpd_conf.in
mode = 0644
[template_launcher] [template_launcher]
recipe = slapos.recipe.download recipe = slapos.recipe.download
url = ${:_profile_base_location_}/launcher.in url = ${:_profile_base_location_}/launcher.in
md5sum = c7f8b6e9ae84aa94686a9cbaaa3dd693 #md5sum = c7f8b6e9ae84aa94686a9cbaaa3dd693
filename = launcher.in filename = launcher.in
mode = 0644 mode = 0644
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
...@@ -96,12 +103,21 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_} ...@@ -96,12 +103,21 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
[slapos-cfg-template] [slapos-cfg-template]
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/${:filename} url = ${:_profile_base_location_}/template/${:filename}
md5sum = 8207e74c7b97a4a3f3be390ea4f97ae1 md5sum = d31d1c51a51bc4b2b8dda197777a4bcb
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
filename = slapos.cfg.in filename = slapos.cfg.in
download-only = true download-only = true
mode = 0644 mode = 0644
[parameters-template]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename}
md5sum = f8446fcf254b4929eb828a9a1d7e5f62
location = ${buildout:parts-directory}/${:_buildout_section_name_}
filename = parameters.xml.in
download-only = true
mode = 0644
[eggs] [eggs]
recipe = z3c.recipe.scripts recipe = z3c.recipe.scripts
eggs = eggs =
...@@ -112,6 +128,7 @@ eggs = ...@@ -112,6 +128,7 @@ eggs =
hexagonit.recipe.download hexagonit.recipe.download
inotifyx inotifyx
lock-file lock-file
lxml
netaddr netaddr
plone.recipe.command plone.recipe.command
pytz pytz
......
...@@ -10,8 +10,8 @@ extends = common.cfg ...@@ -10,8 +10,8 @@ extends = common.cfg
parts += parts +=
slapos.cookbook-repository slapos.cookbook-repository
slapos.toolbox-repository # slapos.toolbox-repository
erp5.util-repository # erp5.util-repository
check-recipe check-recipe
# slapos.core-repository # slapos.core-repository
...@@ -19,20 +19,21 @@ parts += ...@@ -19,20 +19,21 @@ parts +=
develop = develop =
${:parts-directory}/slapos.toolbox-repository ${:parts-directory}/slapos.toolbox-repository
${:parts-directory}/slapos.cookbook-repository ${:parts-directory}/slapos.cookbook-repository
${:parts-directory}/erp5.util-repository # ${:parts-directory}/erp5.util-repository
# ${:parts-directory}/slapos.core-repository # ${:parts-directory}/slapos.core-repository
[slapos.toolbox-repository] [slapos.toolbox-repository]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/slapos.toolbox.git repository = http://git.erp5.org/repos/slapos.toolbox.git
branch = slaprunner-paas #branch = slaprunner-paas
revision = 399dc9f60967593bad0a16dcccf3bec487fc0a8f
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
[slapos.cookbook-repository] [slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/slapos.git repository = http://git.erp5.org/repos/slapos.git
branch = slaprunner-paas-test branch = master
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
# Used for resiliency tests only # Used for resiliency tests only
...@@ -55,7 +56,7 @@ stop-on-error = true ...@@ -55,7 +56,7 @@ stop-on-error = true
update-command = ${:command} update-command = ${:command}
command = command =
grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link && grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link && grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
grep parts ${buildout:develop-eggs-directory}/erp5.util.egg-link # grep parts ${buildout:develop-eggs-directory}/erp5.util.egg-link
# grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link && # grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link &&
PidFile "{{ parameters.path_pid }}"
ServerName example.com
ServerAdmin someone@email
<IfDefine !MonitorPort>
Listen [{{ parameters.global_ip }}]:{{ parameters.monitor_port }}
Define MonitorPort
</IfDefine>
LoadModule unixd_module modules/mod_unixd.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule auth_basic_module modules/mod_auth_basic.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule authz_user_module modules/mod_authz_user.so
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule authn_core_module modules/mod_authn_core.so
LoadModule authn_file_module modules/mod_authn_file.so
LoadModule mime_module modules/mod_mime.so
LoadModule cgid_module modules/mod_cgid.so
LoadModule ssl_module modules/mod_ssl.so
LoadModule alias_module modules/mod_alias.so
LoadModule env_module modules/mod_env.so
LoadModule rewrite_module modules/mod_rewrite.so
LoadModule log_config_module modules/mod_log_config.so
ErrorLog "{{ parameters.path_error_log }}"
LogFormat "%h %l %u %t \"%r\" %>s %b" common
CustomLog "{{ parameters.path_access_log }}" common
# SSL Configuration
Define SSLConfigured
SSLCertificateFile {{ parameters.cert_file }}
SSLCertificateKeyFile {{ parameters.key_file }}
SSLRandomSeed startup builtin
SSLRandomSeed connect builtin
SSLRandomSeed startup /dev/urandom 256
SSLRandomSeed connect builtin
SSLProtocol -ALL +SSLv3 +TLSv1
SSLHonorCipherOrder On
SSLCipherSuite RC4-SHA:HIGH:!ADH
SSLEngine On
ScriptSock {{ parameters.path_pid }}
SetEnv GIT_PROJECT_ROOT {{ parameters.project_folder }}
SetEnv GIT_HTTP_EXPORT_ALL
ScriptAlias /git/ {{ parameters.git_http_backend }}/
ScriptAlias /git-public/ {{ parameters.git_http_backend }}/
RewriteCond %{QUERY_STRING} service=git-receive-pack [OR]
RewriteCond %{REQUEST_URI} /git-receive-pack$
RewriteRule ^/git/ - [E=AUTHREQUIRED:yes]
RewriteRule ^/git-public/ - [E=AUTHREQUIRED:yes]
<LocationMatch "^/git-public/">
Order Deny,Allow
Deny from env=AUTHREQUIRED
AuthType Basic
AuthName "Git Access"
AuthUserFile "{{ parameters.etc_dir }}/.htpasswd"
Require valid-user
Satisfy any
</LocationMatch>
<LocationMatch "^/git/">
Order Deny,Allow
Deny from env=AUTHREQUIRED
AuthType Basic
AuthName "Git Access"
AuthUserFile "{{ parameters.etc_dir }}/.htpasswd"
Require valid-user
</LocationMatch>
include {{ parameters.cgi_httpd_conf }}
...@@ -17,17 +17,20 @@ parts += ...@@ -17,17 +17,20 @@ parts +=
# Bubble up the parameters # Bubble up the parameters
[request-runner] [request-runner]
return = url ssh-public-key ssh-url notification-id ip backend_url url ssh_command password_recovery_code shell_password access_url 1_info return = url ssh-public-key ssh-url notification-id ip backend_url url ssh_command access_url 1_info 2_info monitor_url IMPORTANT_monitor_info webdav_url public_url
[publish-connection-informations] [publish-connection-informations]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
1_info = ${request-runner:connection-1_info} 1_info = ${request-runner:connection-1_info}
2_info = ${request-runner:connection-2_info}
backend_url = ${request-runner:connection-backend_url} backend_url = ${request-runner:connection-backend_url}
access_url = ${request-runner:connection-access_url} access_url = ${request-runner:connection-access_url}
url = ${request-runner:connection-url} url = ${request-runner:connection-url}
ssh_command = ${request-runner:connection-ssh_command} ssh_command = ${request-runner:connection-ssh_command}
password_recovery_code = ${request-runner:connection-password_recovery_code} monitor_url = ${request-runner:connection-monitor_url}
shell_password = ${request-runner:connection-shell_password} IMPORTANT_monitor_info = ${request-runner:connection-IMPORTANT_monitor_info}
webdav_url = ${request-runner:connection-webdav_url}
public_url = ${request-runner:connection-public_url}
[slap-parameter] [slap-parameter]
# Default parameters for distributed deployment # Default parameters for distributed deployment
......
...@@ -18,13 +18,36 @@ parts += ...@@ -18,13 +18,36 @@ parts +=
slaprunner-promise slaprunner-promise
slaprunner-frontend-promise slaprunner-frontend-promise
dropbear-promise dropbear-promise
private
shellinabox-promise shellinabox-promise
symlinks symlinks
shellinabox shellinabox
slapos-cfg slapos-cfg
slapos-repo-config slapos-repo-config
prepare-software
cron-entry-backup cron-entry-backup
cron-entry-prepare-software
deploy-instance-parameters
###Parts to add for monitoring
slap-parameters
certificate-authority
cron
cron-entry-monitor
cron-entry-rss
deploy-index
deploy-index-template
deploy-monitor-script
deploy-rss-script
deploy-settings-cgi
deploy-status-cgi
make-rss
monitor-frontend
monitor-promise
setup-static-files
certificate-authority
public
zero-parameters
cgi-httpd-wrapper
public-symlink
[exporter] [exporter]
recipe = slapos.cookbook:slaprunner.export recipe = slapos.cookbook:slaprunner.export
...@@ -35,6 +58,8 @@ backup-directory = $${directory:backup} ...@@ -35,6 +58,8 @@ backup-directory = $${directory:backup}
shell-binary = ${dash:location}/bin/dash shell-binary = ${dash:location}/bin/dash
rsync-binary = ${rsync:location}/bin/rsync rsync-binary = ${rsync:location}/bin/rsync
[monitor-promise]
url = $${monitor-frontend:config-url}/$${deploy-index:filename}
# Extends publish section with resilient parameters # Extends publish section with resilient parameters
[publish-connection-informations] [publish-connection-informations]
......
...@@ -16,14 +16,36 @@ parts += ...@@ -16,14 +16,36 @@ parts +=
sshkeys-authority sshkeys-authority
slaprunner-promise slaprunner-promise
dropbear-promise dropbear-promise
private
shellinabox-promise shellinabox-promise
shellinabox shellinabox
symlinks symlinks
slapos-cfg slapos-cfg
slapos-repo-config slapos-repo-config
prepare-software cron-entry-prepare-software
deploy-instance-parameters
# have to repeat the next one, as it's not inherited from pbsready-import # have to repeat the next one, as it's not inherited from pbsready-import
import-on-notification import-on-notification
###Parts to add for monitoring
slap-parameters
certificate-authority
cron
cron-entry-monitor
cron-entry-rss
deploy-index
deploy-index-template
deploy-monitor-script
deploy-rss-script
deploy-settings-cgi
deploy-status-cgi
make-rss
monitor-promise
setup-static-files
certificate-authority
public
zero-parameters
cgi-httpd-wrapper
public-symlink
[importer] [importer]
recipe = slapos.cookbook:slaprunner.import recipe = slapos.cookbook:slaprunner.import
...@@ -35,3 +57,6 @@ shell-binary = ${dash:location}/bin/dash ...@@ -35,3 +57,6 @@ shell-binary = ${dash:location}/bin/dash
rsync-binary = ${rsync:location}/bin/rsync rsync-binary = ${rsync:location}/bin/rsync
curl-binary = ${curl:location}/bin/curl curl-binary = ${curl:location}/bin/curl
backend-url = $${slaprunner:access-url} backend-url = $${slaprunner:access-url}
[slap-parameter]
auto-deploy-instance = false
...@@ -15,15 +15,39 @@ parts = ...@@ -15,15 +15,39 @@ parts =
slaprunner-promise slaprunner-promise
slaprunner-frontend-promise slaprunner-frontend-promise
dropbear-promise dropbear-promise
private
shellinabox-promise shellinabox-promise
symlinks symlinks
shellinabox shellinabox
slapos-cfg slapos-cfg
slapos-repo-config slapos-repo-config
prepare-software cron-entry-prepare-software
deploy-instance-parameters
{% if slapparameter_dict.get('custom-frontend-backend-url') %} {% if slapparameter_dict.get('custom-frontend-backend-url') %}
custom-frontend-promise custom-frontend-promise
{% endif %} {% endif %}
###Parts to add for monitoring
slap-parameters
certificate-authority
cron
cron-entry-monitor
cron-entry-rss
deploy-index
deploy-index-template
deploy-monitor-script
deploy-rss-script
deploy-settings-cgi
deploy-status-cgi
make-rss
monitor-promise
setup-static-files
certificate-authority
public
zero-parameters
cgi-httpd-wrapper
public-symlink
extends = ${monitor-template:output}
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
...@@ -91,13 +115,14 @@ promises = $${:etc}/promise/ ...@@ -91,13 +115,14 @@ promises = $${:etc}/promise/
test = $${:etc}/test/ test = $${:etc}/test/
nginx-data = $${directory:srv}/nginx nginx-data = $${directory:srv}/nginx
ca-dir = $${:srv}/ssl ca-dir = $${:srv}/ssl
project = $${:srv}/runner/project/ project = $${:srv}/runner/project
[runnerdirectory] [runnerdirectory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
home = $${directory:srv}/runner/ home = $${directory:srv}/runner/
test = $${directory:srv}/test/ test = $${directory:srv}/test/
project = $${:home}/project project = $${:home}/project
public = $${:home}/public
software-root = $${:home}/software software-root = $${:home}/software
instance-root = $${:home}/instance instance-root = $${:home}/instance
project-test = $${:test}/project project-test = $${:test}/project
...@@ -123,6 +148,8 @@ working-directory = $${runnerdirectory:home} ...@@ -123,6 +148,8 @@ working-directory = $${runnerdirectory:home}
project-directory = $${runnerdirectory:project} project-directory = $${runnerdirectory:project}
software-directory = $${runnerdirectory:software-root} software-directory = $${runnerdirectory:software-root}
instance-directory = $${runnerdirectory:instance-root} instance-directory = $${runnerdirectory:instance-root}
#XXX-Nico hardcoded default port because overridden by this buildout config
instance-monitor-url = https://[$${:ipv6}]:9685
etc_dir = $${directory:etc} etc_dir = $${directory:etc}
log_dir = $${directory:log} log_dir = $${directory:log}
run_dir = $${directory:run} run_dir = $${directory:run}
...@@ -230,13 +257,13 @@ path_tmp = $${directory:tmp}/ ...@@ -230,13 +257,13 @@ path_tmp = $${directory:tmp}/
# Config files # Config files
path_nginx_conf = $${directory:etc}/nginx.conf path_nginx_conf = $${directory:etc}/nginx.conf
# Executables # Executables
bin_nginx = ${nginx:location}/sbin/nginx bin_nginx = ${nginx-webdav:location}/sbin/nginx
bin_launcher = $${directory:bin}/launcher bin_launcher = $${directory:bin}/launcher
# Utils # Utils
path_shell = ${dash:location}/bin/dash path_shell = ${dash:location}/bin/dash
# Misc. # Misc.
etc_dir = $${directory:etc} etc_dir = $${directory:etc}
work_dir = $${slaprunner:working-directory}
[nginx_conf] [nginx_conf]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -256,6 +283,37 @@ mode = 700 ...@@ -256,6 +283,37 @@ mode = 700
context = context =
section param_nginx_frontend nginx-frontend section param_nginx_frontend nginx-frontend
[httpd-parameters]
path_pid = $${directory:run}/httpd.pid
path_error_log = $${directory:log}/httpd-error.log
path_access_log = $${directory:log}/httpd-access.log
key_file = $${ca-httpd:key-file}
cert_file = $${ca-httpd:cert-file}
global_ip = $${slap-network-information:global-ipv6}
global_port = $${slaprunner:runner_port}
monitor_port = $${monitor-parameters:port}
monitor_index = $${deploy-index:rendered}
working_directory = $${slaprunner:working-directory}
dav_lock = $${directory:var}/DavLock
etc_dir = $${directory:etc}
document_root = $${directory:www}
project_folder = $${directory:project}
git_http_backend = ${git:location}/libexec/git-core/git-http-backend
cgi_httpd_conf = $${cgi-httpd-configuration-file:output}
[httpd-conf]
recipe = slapos.recipe.template:jinja2
template = ${template_httpd_conf:location}/${template_httpd_conf:filename}
rendered = $${directory:etc}/httpd.conf
context =
section parameters httpd-parameters
[cgi-httpd-wrapper]
recipe = slapos.cookbook:wrapper
apache-executable = ${apache:location}/bin/httpd
wrapper-path = $${ca-httpd:executable}
command-line = $${:apache-executable} -f $${httpd-conf:rendered} -DFOREGROUND
#-------------------- #--------------------
#-- #--
#-- WSGI #-- WSGI
...@@ -335,19 +393,33 @@ config-url = $${slaprunner:access-url} ...@@ -335,19 +393,33 @@ config-url = $${slaprunner:access-url}
config-domain = $${slap-parameter:frontend-domain} config-domain = $${slap-parameter:frontend-domain}
return = site_url domain return = site_url domain
[monitor-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Monitor Frontend
# XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config = url domain
config-url = https://[$${cgi-httpd-configuration-file:listening-ip}]:$${monitor-parameters:port}
config-domain = $${slap-parameter:frontend-domain}
return = site_url domain
#-------------------------------------- #--------------------------------------
#-- #--
#-- Send informations to SlapOS Master #-- Send informations to SlapOS Master
[publish-connection-informations] [publish-connection-informations]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
1_info = On your first run, Use "access_url" to setup you account.Then you can use both "url" or "access_url". Or "backend_url" if you want to use ipv6. 1_info = On your first run, Use "access_url" to setup you account. Then you can use both "url" or "access_url". Or "backend_url" if you want to use ipv6. Set up your account in the webrunner in order to use webdav, and being able to clone your git repositories from the runner.
2_info = In order to set up your account, get the recovery-code from the monitoring interface. Before read the notification on monitor_info.
backend_url = $${slaprunner:access-url} backend_url = $${slaprunner:access-url}
access_url = $${:url}/login access_url = $${:url}/login
url = https://$${request-frontend:connection-domain} url = https://$${request-frontend:connection-domain}
ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port} ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port}
shell_password = $${shellinabox:password} webdav_url = https://$${request-frontend:connection-domain}/share
password_recovery_code = $${recovery-code:passwd} monitor_url = https://$${monitor-frontend:connection-domain}
public_url = $${:url}/public/
#--------------------------- #---------------------------
#-- #--
...@@ -393,7 +465,13 @@ frontend-domain = ...@@ -393,7 +465,13 @@ frontend-domain =
slapos-repository = http://git.erp5.org/repos/slapos.git slapos-repository = http://git.erp5.org/repos/slapos.git
slapos-software = slapos-software =
slapos-reference = master slapos-reference = master
auto-deploy = true auto-deploy = false
auto-deploy-instance = true
autorun = false
monitor-port = 9684
[monitor-parameters]
port = $${slap-parameter:monitor-port}
[slapos-cfg] [slapos-cfg]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -403,30 +481,35 @@ mode = 700 ...@@ -403,30 +481,35 @@ mode = 700
context = context =
key software_root runnerdirectory:software-root key software_root runnerdirectory:software-root
key instance_root runnerdirectory:instance-root key instance_root runnerdirectory:instance-root
key auto_deploy slap-parameter:auto-deploy
key auto_deploy_instance slap-parameter:auto-deploy-instance
key autorun slap-parameter:autorun
key console slaprunner:console
key debug slaprunner:debug
key etc_dir slaprunner:etc_dir
key git slaprunner:git-binary
key instance_root runnerdirectory:instance-root
key instance_monitoring_url slaprunner:instance-monitor-url
key ipv4 slaprunner:ipv4 key ipv4 slaprunner:ipv4
key proxy_port slaprunner:proxy_port key ipv6 slaprunner:ipv6
key log_dir slaprunner:log_dir
key partition_amount slaprunner:partition-amount key partition_amount slaprunner:partition-amount
key slapgrid_sr slaprunner:slapgrid_sr key private_key slaprunner:private_key
key proxy_database slaprunner:prox-db-var
key proxy_host slaprunner:ipv4
key proxy_port slaprunner:proxy_port
key public_key slaprunner:public_key
key run_dir slaprunner:run_dir
key runner_port slaprunner:runner_port
key runner_workdir slaprunner:working-directory
key slapgrid_cp slaprunner:slapgrid_cp key slapgrid_cp slaprunner:slapgrid_cp
key slapgrid_sr slaprunner:slapgrid_sr
key slapproxy slaprunner:slapproxy key slapproxy slaprunner:slapproxy
key software_root runnerdirectory:software-root
key ssh_client slaprunner:ssh_client
key supervisor slaprunner:supervisor key supervisor slaprunner:supervisor
key supervisord_config slaprunner:supd-temp-var key supervisord_config slaprunner:supd-temp-var
key runner_workdir slaprunner:working-directory
key runner_port slaprunner:runner_port
key ipv6 slaprunner:ipv6
key etc_dir slaprunner:etc_dir
key proxy_host slaprunner:ipv4
key proxy_database slaprunner:prox-db-var
key ssh_client slaprunner:ssh_client
key public_key slaprunner:public_key
key private_key slaprunner:private_key
key git slaprunner:git-binary
key run_dir slaprunner:run_dir
key log_dir slaprunner:log_dir
key console slaprunner:console
key verbose slaprunner:verbose key verbose slaprunner:verbose
key debug slaprunner:debug
key auto_deploy slap-parameter:auto-deploy
[shellinabox] [shellinabox]
recipe = slapos.cookbook:shellinabox recipe = slapos.cookbook:shellinabox
...@@ -435,7 +518,7 @@ port = 8080 ...@@ -435,7 +518,7 @@ port = 8080
shell = $${shell:wrapper} shell = $${shell:wrapper}
wrapper = $${directory:bin}/shellinaboxd wrapper = $${directory:bin}/shellinaboxd
shellinabox-binary = ${shellinabox:location}/bin/shellinaboxd shellinabox-binary = ${shellinabox:location}/bin/shellinaboxd
password = $${shellinabox-code:passwd} password = $${zero-parameters:shell-password}
directory = $${runnerdirectory:home} directory = $${runnerdirectory:home}
login-shell = $${directory:bin}/login login-shell = $${directory:bin}/login
certificate-directory = $${cadirectory:certs} certificate-directory = $${cadirectory:certs}
...@@ -468,12 +551,50 @@ location = $${directory:project}/slapos ...@@ -468,12 +551,50 @@ location = $${directory:project}/slapos
[slapos-repo-config] [slapos-repo-config]
recipe = plone.recipe.command recipe = plone.recipe.command
stop-on-error = true stop-on-error = true
reference = $${slap-parameter:slapos-reference} command = cd $${slapos-repo:location} && ${git:location}/bin/git checkout $${slap-parameter:slapos-reference} && SR=$${slap-parameter:slapos-software} && if [ -n "$SR" ] && [ ! -f "$${directory:etc}/.project" ]; then echo workspace/slapos/$${slap-parameter:slapos-software}/ > $${directory:etc}/.project; fi
location = $${slapos-repo:location} update-command = true
command = cd $${:location} && ${git:location}/bin/git checkout $${:reference} && echo "$${directory:etc}/.project" && SR=$${slap-parameter:slapos-software} && if [ -n "$SR" ] && [ ! -f "$${directory:etc}/.project" ]; then echo workspace/slapos/$${slap-parameter:slapos-software}/ > $${directory:etc}/.project; fi
[prepare-software] [prepare-software]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
command-line = ${curl:location}/bin/curl -g https://[$${slaprunner:ipv6}]:$${slaprunner:runner_port}/isSRReady --max-time 1 --insecure command-line = ${curl:location}/bin/curl -g https://[$${slaprunner:ipv6}]:$${slaprunner:runner_port}/isSRReady --max-time 1 --insecure
wrapper-path = $${directory:scripts}/prepareSoftware wrapper-path = $${directory:scripts}/prepareSoftware
[cron-entry-prepare-software]
<= cron
recipe = slapos.cookbook:cron.d
name = prepare-software
frequency = */2 * * * *
command = $${prepare-software:wrapper-path}
[instance-parameters]
recipe = slapos.recipe.template:jinja2
extensions = jinja2.ext.do
template = ${parameters-template:location}/${parameters-template:filename}
rendered = $${directory:etc}/.parameter.xml.default
mode = 0644
context =
key slapparameter_dict slap-configuration:configuration
[deploy-instance-parameters]
recipe = plone.recipe.command
stop-on-error = true
parameter-xml = $${directory:etc}/.parameter.xml
command = if [ ! -f $${:parameter-xml} ]; then cp $${instance-parameters:rendered} $${:parameter-xml}; fi
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[public]
shell-password = $${shellinabox-code:passwd}
[private]
recipe = slapos.cookbook:zeroknown.write
filename = knowledge0.cfg
recovery-code = $${recovery-code:passwd}
[zero-parameters]
...@@ -64,5 +64,22 @@ http { ...@@ -64,5 +64,22 @@ http {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $http_host; proxy_set_header X-Forwarded-Host $http_host;
} }
location /public/ {
root {{ param_nginx_frontend['work_dir'] }};
index index.html index.htm;
}
location /share {
rewrite ^/share / break;
auth_basic "Restricted";
auth_basic_user_file {{ param_nginx_frontend['etc_dir'] }}/.htpasswd;
dav_methods PUT DELETE MKCOL COPY MOVE;
dav_ext_methods PROPFIND OPTIONS;
root {{ param_nginx_frontend['work_dir'] }};
create_full_put_path on;
}
} }
} }
{% set inst_parameter_dict = {} -%}
{% if slapparameter_dict is defined -%}
{% for key in slapparameter_dict.keys() -%}
{% if key.startswith('parameter-') -%}
{% do inst_parameter_dict.__setitem__(key[10:], slapparameter_dict.pop(key)) -%}
{% endif -%}
{% endfor -%}
{% endif -%}
<?xml version='1.0' encoding='utf-8'?>
<instance>
{% if slapparameter_dict is defined %}
{% for parameter_name in inst_parameter_dict.keys() %}
<parameter id="{{ parameter_name }}">{{ inst_parameter_dict[parameter_name] }}</parameter>
{% endfor %}
{% endif %}
</instance>
{
"$schema": "http://json-schema.org/draft-04/schema#",
"properties": {
"authorized-key": {
"description" : "SSH public key in order to connect to the SSH server of this runner instance",
"type": "string"
},
"instance-amount": {
"description": "number of slappart to deploy inside the runner",
"type": "integer",
"default": 10
},
"slapos-software": {
"description": "a relative path from the slapos git repo to a folder containing a software release, which will be automaticaly deployed while the runner instanciation, and only if the parameter auto-deploy is set to 'true'. For example: 'software/helloworld",
"type": "string"
}, "auto-deploy": { "description": "authorizes the software declared with 'slapos-software' to be automatically deployed, or not",
"type": "boolean",
"default": "true"
},
"slapos-repository": {
"description": "url of the default git repository that will be download by the runner while its instanciation. Will be cloned in a directory named 'slapos'",
"type": "string",
"default": "http://git.erp5.org/repos/slapos.git"
},
"slapos-reference": {
"description": "commit reference on which the default repository will checkout",
"type": "string",
"default": "master"
},
"auto-deploy-instance": {
"description": "prevent the runner from deploying and starting instances",
"type": "boolean",
"default": "true, but is set to false for instances of type 'import' in resiliency"
},
"autorun": {
"description": "let automaticaly build and run a declared software with 'slapos-software'. Only works if 'slapos-software' is set, and 'auto-deploy' is true",
"type": "boolean",
"default": "false"
},
"parameter-*": {
"description": "'*' is a parameter which will be used to configure the instance inside the runner.",
"type": "string"
},
"custom-frontend-backend-url": {
"description": "return an ipv4 frontend of the given ipv6(+optional port)",
"type": "string"
},
"custom-frontend-basic-auth": {
"description": "if the ip given with 'custom-frontend-backend-url' is secure, set it to true for the promise do not fail",
"type": "boolean",
"default": "false"
}
"monitor-port": {
"description": "allow to manually change the port on wich the apache server running monitoring interface is listening. The default value for the webrunner is different from the default value of the standalone stack-monitor server",
"type": "integer",
"default": 9684
}
}
}
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
# 2/ Define list of trusted certificates for the cache. # 2/ Define list of trusted certificates for the cache.
[buildout] [buildout]
extends = common.cfg extends = development.cfg
[networkcache] [networkcache]
# signature certificates of the following uploaders. # signature certificates of the following uploaders.
...@@ -165,19 +165,19 @@ signature-certificate-list = ...@@ -165,19 +165,19 @@ signature-certificate-list =
+uZ16u1DbO9rYoKgWqjLk1GfiLw5v86pd5+wZd5I9QJ0/Sbz2vZk5S4ciMIGwArc +uZ16u1DbO9rYoKgWqjLk1GfiLw5v86pd5+wZd5I9QJ0/Sbz2vZk5S4ciMIGwArc
m711+GzlW5xe6GyH9SZaGOPAdUbI6JTDwLzEgA== m711+GzlW5xe6GyH9SZaGOPAdUbI6JTDwLzEgA==
-----END CERTIFICATE----- -----END CERTIFICATE-----
-----BEGIN CERTIFICATE----- # -----BEGIN CERTIFICATE-----
MIIB+DCCAWGgAwIBAgIJAJAOvB8P2YYMMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV # MIIB+DCCAWGgAwIBAgIJAJAOvB8P2YYMMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV
BAMMCUNPTVAtMTg1MjAgFw0xMzEyMDMxMjU5NDJaGA8yMTEzMTEwOTEyNTk0Mlow # BAMMCUNPTVAtMTg1MjAgFw0xMzEyMDMxMjU5NDJaGA8yMTEzMTEwOTEyNTk0Mlow
FDESMBAGA1UEAwwJQ09NUC0xODUyMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB # FDESMBAGA1UEAwwJQ09NUC0xODUyMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB
gQDgR2qKVURmOUmzc7KVL2Gk2VezWlHlJy3V3k1WVYahhFsSjf3O4VyTtroHbIGu # gQDgR2qKVURmOUmzc7KVL2Gk2VezWlHlJy3V3k1WVYahhFsSjf3O4VyTtroHbIGu
YMJaHLSvGvvvkVJIz/DKqOvl1/6435t+coYECimvzqzTBDWFlEKJbEixz/3nPnXe # YMJaHLSvGvvvkVJIz/DKqOvl1/6435t+coYECimvzqzTBDWFlEKJbEixz/3nPnXe
CFqVzuCQ7e3nRIrIGLml6jZnXwPlzx8ANn1LO8pwKHj+xwIDAQABo1AwTjAdBgNV # CFqVzuCQ7e3nRIrIGLml6jZnXwPlzx8ANn1LO8pwKHj+xwIDAQABo1AwTjAdBgNV
HQ4EFgQUvo7qkSq82Ax5/vnxHoJOhySQSGowHwYDVR0jBBgwFoAUvo7qkSq82Ax5 # HQ4EFgQUvo7qkSq82Ax5/vnxHoJOhySQSGowHwYDVR0jBBgwFoAUvo7qkSq82Ax5
/vnxHoJOhySQSGowDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQCbNxtq # /vnxHoJOhySQSGowDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQCbNxtq
BckDSWb92gy/p+JbkMXLjRSSWgeKafvWQXMXpPHW847CPzUDB7zT0rT8a8SRjVuL # BckDSWb92gy/p+JbkMXLjRSSWgeKafvWQXMXpPHW847CPzUDB7zT0rT8a8SRjVuL
Vt/oo2VlJ9xbKEz/qppX8w1TwEZhEJk6h8ky51s2EyyITUy8LHESJes9l46DRbr8 # Vt/oo2VlJ9xbKEz/qppX8w1TwEZhEJk6h8ky51s2EyyITUy8LHESJes9l46DRbr8
Rm6WS5sk5oiCd146YkWhfZuOsQHXSG/+WC/GPQ== # Rm6WS5sk5oiCd146YkWhfZuOsQHXSG/+WC/GPQ==
-----END CERTIFICATE----- # -----END CERTIFICATE-----
[versions] [versions]
Flask-Auth = 0.85 Flask-Auth = 0.85
...@@ -212,7 +212,7 @@ slapos.recipe.build = 0.11.5 ...@@ -212,7 +212,7 @@ slapos.recipe.build = 0.11.5
slapos.recipe.cmmi = 0.2 slapos.recipe.cmmi = 0.2
slapos.recipe.download = 1.0.dev-r4053 slapos.recipe.download = 1.0.dev-r4053
slapos.recipe.template = 2.4.2 slapos.recipe.template = 2.4.2
slapos.toolbox = 0.38.1 slapos.toolbox = 0.39
smmap = 0.8.2 smmap = 0.8.2
xml-marshaller = 0.9.7 xml-marshaller = 0.9.7
z3c.recipe.scripts = 1.0.1 z3c.recipe.scripts = 1.0.1
......
...@@ -17,6 +17,7 @@ supervisord_config = {{ supervisord_config }} ...@@ -17,6 +17,7 @@ supervisord_config = {{ supervisord_config }}
runner_workdir = {{ runner_workdir }} runner_workdir = {{ runner_workdir }}
runner_host = {{ ipv4 }} runner_host = {{ ipv4 }}
runner_port = {{ runner_port }} runner_port = {{ runner_port }}
instance_monitoring_url = {{ instance_monitoring_url }}
ipv4_address = {{ ipv4 }} ipv4_address = {{ ipv4 }}
ipv6_address = {{ ipv6 }} ipv6_address = {{ ipv6 }}
etc_dir = {{ etc_dir }} etc_dir = {{ etc_dir }}
...@@ -26,6 +27,8 @@ console = {{ console }} ...@@ -26,6 +27,8 @@ console = {{ console }}
verbose = {{ verbose }} verbose = {{ verbose }}
debug = {{ debug }} debug = {{ debug }}
auto_deploy = {{ auto_deploy }} auto_deploy = {{ auto_deploy }}
auto_deploy_instance = {{ auto_deploy_instance }}
autorun = {{ autorun }}
[slapproxy] [slapproxy]
host = {{ proxy_host }} host = {{ proxy_host }}
......
* This stack has for purpose to know if all promises, services, custom monitoring scripts went/are ok.
* The second purpose of this stack is to implement a zero-knowledge feature : it means you can use its control interface to provide the user with sensible data. It can also let the user change some parameters
* It also provides a web interface, to see which promises, services and custom scripts failed. It also provide a rss feed to easily know the actual state of your instance, and to know when it started to went bad. You can also add your own monitoring scripts, or cgi files (or just files) that you would want to check easily using a web interface.
Implementation :
----------------
1/ In the software.cfg of your Software Release, extends the stack
2/ In the template that will be copied for the buildout in the instance folder (instance.cfg ?), you have to add these parts:
###Parts to add for monitoring
slap-parameters
certificate-authority
cron
cron-entry-monitor
cron-entry-rss
deploy-index
deploy-index-template
deploy-monitor-script
deploy-rss-script
deploy-settings-cgi
deploy-status-cgi
make-rss
monitor-promise
setup-static-files
certificate-authority
public
zero-parameters
cgi-httpd-wrappers
public-symlink
* If you want to add a custom monitoring script, you can write it (in whatever language you wish) and save it in YOUR_INSTANCE_FOLDER/etc/monitor.
The only thing to know, is that if your script successfully passed, do not return or print nothing. If there is a problem, you can print the explanation on stdout or stderr
* Here are 2 promises that you can add to your instance buildout, to see if it is working (one is ok, not the other) :
[google-promise]
recipe = slapos.cookbook:check_url_available
path = $${directory:promise}/google
url = http://www.google.com
dash_path = ${dash:location}/bin/dash
curl_path = ${curl:location}/bin/curl
[failing-promise]
recipe = slapos.cookbook:check_url_available
path = $${directory:promise}/fail
url = http://127.0.0.2
dash_path = ${dash:location}/bin/dash
curl_path = ${curl:location}/bin/curl
CGI Scripts:
------------
This stack also provides a web interface, in wich you can execute custom cgi scripts, or just print files. The web link is provided in the published parameters, as for the password that you have to change as soon as possible
In that interface you will have access to the previous scripts and the RSS feed. You can also add your files/scripts.
For that, there exists a folder /var/cgi-bin. You should see that directory as a tree having of deep 2. In /var/cgi-bin, you must create only folders, which are called categories. In each category, you can then add your own files.
The backend system will automatically render the webpage according to the inside structure of the cgi-bin directory. Moreover, it will also let you access to your scripts only if you are logged in : you do not need do do your own authentication system !
Notice :
--------
* /!\A default password is set up at the installation : "passwordtochange". It has to be rewritten in the control interface by the user itself
* /!\ If you use the recipe zeroknown, never name a parameter "recipe" or "password".
* The control interface will let you change the values of the options declared in the [public] section of the config file (see zeroknown recipe). Other section's values will just be printed. These values won't be overwritten by buildout.
* If you want to allow a user to change a parameter, use the recipe zeroknown, with the buildout section name : "[public]"
* If you manually change a parameter, it could take some time for the modifications to be applied (at least 1 or 2 slapgrid-cp)
* If you need to change the port of the web interface of the monitoring stack, just create in your software release file a part called [monitor-parameters] and give the new port value to the parameter "port".
[buildout]
extends =
../../component/apache/buildout.cfg
../../component/curl/buildout.cfg
../../component/dash/buildout.cfg
../../component/dcron/buildout.cfg
../../component/openssl/buildout.cfg
../../component/nginx/buildout.cfg
parts +=
backup-script-template
collective.recipe.template-egg
dcron
download-static-files
eggs
extra-eggs
make-rss
monitor-bin
monitor-template
nginx
rss-bin
slapos-cookbook
static-folder
template-nginx-conf
[collective.recipe.template-egg]
recipe = zc.recipe.egg
eggs = collective.recipe.template
PyRSS2Gen
[extra-eggs]
recipe = zc.recipe.egg
interpreter = pythonwitheggs
eggs =
PyRSS2Gen
Jinja2
[make-rss-script]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/make-rss.sh.in
md5sum = 8d3a4b212784b591316b8b93d6bd163e
output = ${buildout:directory}/make-rss.sh.in
mode = 0644
[monitor-template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/monitor.cfg.in
output = ${buildout:directory}/monitor.cfg
filename = monitor.cfg
md5sum = feeca02cd409457b7ee61697f2eb3eef
mode = 0644
[monitor-bin]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename}
download-only = true
md5sum = 05bbb70f6f69dc4f3fa83dc0f8c9960e
filename = monitor.py.in
mode = 0644
[index]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/webfiles/${:filename}
download-only = true
md5sum = 91ac749f86aecc0c383d93e51e15a572
filename = index.cgi.in
mode = 0644
[index-template]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/webfiles/${:filename}
download-only = true
md5sum = aa375a4225e2587b22f68c28cafd7871
filename = index.html.jinja2
mode = 0644
[status-cgi]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/webfiles/${:filename}
download-only = true
md5sum = 4e5b7fc5c5c237836c7c81fe3e3bb903
filename = status.cgi.in
mode = 0644
[settings-cgi]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/webfiles/${:filename}
download-only = true
md5sum = 18574b804da0c65d8670959f9e7c4774
filename = settings.cgi.in
mode = 0644
[rss-bin]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename}
download-only = true
md5sum = 916f37f083b1ef391adea2f7a717bf8a
filename = status2rss.py
mode = 0644
[dcron-service]
recipe = slapos.recipe.template
url = ${template-dcron-service:output}
output = $${directory:services}/crond
mode = 0700
logfile = $${directory:log}/crond.log
[download-static-files]
recipe = hexagonit.recipe.download
url = https://github.com/Sebatyne/staticForMonitoring/blob/3f0e93cec706c7ad311ddbf1ebf996965ce2f0a3/static-files.tar.gz?raw=true
download-only = true
md5sum = e98585b85634de48240b2c215e946769
filename = static-files.tar.gz
mode = 0644
[eggs]
recipe = z3c.recipe.scripts
eggs =
slapos.cookbook
PyRSS2Gen
[versions]
PyRSS2Gen = 1.1
Jinja2 = 2.6
#!${dash-output:dash}
STATUS=$${monitor-parameters:result-dir}
RSS_FILE=$${monitor-parameters:rss-path}
PYTHON=${buildout:directory}/bin/${extra-eggs:interpreter}
STATUS2RSS=${rss-bin:location}/${rss-bin:filename}
NAME=`basename $STATUS`
cat $STATUS/* | $PYTHON $STATUS2RSS "Monitoring RSS feed" "https://[$${slap-parameters:ipv6-random}]:$${monitor-parameters:port}/$${deploy-index:filename}" > $RSS_FILE
[slap-parameters]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[monitor-parameters]
monitor-dir = $${directory:var}/monitor
result-dir = $${:monitor-dir}/bool
json-filename = monitor.json
json-path = $${:monitor-dir}/$${:json-filename}
rss-path = $${:public-cgi}/$${:rss-filename}
rss-filename = rssfeed.html
executable = $${directory:bin}/monitor.py
cgi-bin = $${directory:cgi-bin}
monitoring-cgi = $${directory:monitoring-cgi}
knowledge0-cgi = $${directory:knowledge0-cgi}
public-cgi = $${directory:public-cgi}
port = 9685
[directory]
home = $${buildout:directory}
etc = $${:home}/etc
bin = $${:home}/bin
srv = $${:home}/srv
var = $${:home}/var
promises = $${:etc}/promise
ca-dir = $${:srv}/ssl
cgi-bin = $${:var}/cgi-bin
monitoring-cgi = $${:cgi-bin}/monitoring
knowledge0-cgi = $${:cgi-bin}/zero-knowledge
cron-entries = $${:etc}/cron.d
crontabs = $${:etc}/crontabs
cronstamps = $${:etc}/cronstamps
log = $${:var}/log
monitor = $${:etc}/monitor
monitor-result = $${monitor-parameters:monitor-dir}
monitor-result-bool = $${monitor-parameters:result-dir}
promise = $${:etc}/promise
public-cgi = $${:cgi-bin}/public
run = $${:var}/run
service = $${:etc}/service/
tmp = $${:home}/tmp
www = $${:var}/www
[public-symlink]
recipe = cns.recipe.symlink
symlink = $${monitor-parameters:public-cgi} = $${directory:www}/public
autocreate = true
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:wrapper}
binary = $${directory:service}/crond
# Add log to cron
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${directory:bin}/cron_simplelogger
log = $${directory:log}/cron.log
[cron-entry-monitor]
<= cron
recipe = slapos.cookbook:cron.d
name = launch-monitor
frequency = */5 * * * *
command = $${monitor-parameters:executable} -a
[cron-entry-rss]
<= cron
recipe = slapos.cookbook:cron.d
name = build-rss
frequency = */5 * * * *
command = $${make-rss:output}
[setup-static-files]
recipe = hexagonit.recipe.download
url = ${download-static-files:destination}/${download-static-files:filename}
md5sum = e98585b85634de48240b2c215e946769
filename = static
destination = $${directory:www}
ignore-existing = true
mode = 0644
[deploy-index]
recipe = slapos.recipe.template:jinja2
template = ${index:location}/${index:filename}
rendered = $${directory:www}/$${:filename}
filename = index.cgi
mode = 0744
context =
key cgi_directory monitor-parameters:cgi-bin
raw index_template $${deploy-index-template:location}/$${deploy-index-template:filename}
key password zero-parameters:monitor-password
raw extra_eggs_interpreter ${buildout:directory}/bin/${extra-eggs:interpreter}
raw default_page /index.cgi?script=$${monitor-parameters:knowledge0-cgi}%2F$${deploy-settings-cgi:filename}
[deploy-index-template]
recipe = hexagonit.recipe.download
url = ${index-template:location}/$${:filename}
destination = $${directory:www}
filename = ${index-template:filename}
download-only = true
md5sum = aa375a4225e2587b22f68c28cafd7871
mode = 0644
[deploy-status-cgi]
recipe = slapos.recipe.template:jinja2
template = ${status-cgi:location}/${status-cgi:filename}
rendered = $${monitor-parameters:monitoring-cgi}/$${:filename}
filename = status.cgi
mode = 0744
context =
key json_file monitor-parameters:json-path
raw python_executable ${buildout:executable}
[deploy-settings-cgi]
recipe = slapos.recipe.template:jinja2
template = ${settings-cgi:location}/${settings-cgi:filename}
rendered = $${monitor-parameters:knowledge0-cgi}/$${:filename}
filename = settings.cgi
mode = 0744
context =
raw config_cfg $${buildout:directory}/knowledge0.cfg
raw timestamp $${buildout:directory}/.timestamp
raw python_executable ${buildout:executable}
key pwd monitor-parameters:knowledge0-cgi
key this_file :filename
[deploy-monitor-script]
recipe = slapos.recipe.template:jinja2
template = ${monitor-bin:location}/${monitor-bin:filename}
rendered = $${monitor-parameters:executable}
mode = 0744
context =
section directory directory
key monitoring_file_json monitor-parameters:json-path
key monitoring_folder_bool monitor-parameters:result-dir
raw python_executable ${buildout:executable}
[deploy-rss-script]
recipe = hexagonit.recipe.download
url = ${rss-bin:destination}/${rss-bin:filename}
destination = $${directory:bin}
filename = ${rss-bin:filename}
md5sum = 916f37f083b1ef391adea2f7a717bf8a
mode = 0744
download-only = true
[make-rss]
recipe = slapos.recipe.template
url = ${make-rss-script:output}
output = $${directory:bin}/make-rss.sh
md5sum = 8d3a4b212784b591316b8b93d6bd163e
mode = 0744
[cadirectory]
recipe = slapos.cookbook:mkdirectory
requests = $${directory:ca-dir}/requests/
private = $${directory:ca-dir}/private/
certs = $${directory:ca-dir}/certs/
newcerts = $${directory:ca-dir}/newcerts/
crl = $${directory:ca-dir}/crl/
[certificate-authority]
recipe = slapos.cookbook:certificate_authority
openssl-binary = ${openssl:location}/bin/openssl
ca-dir = $${directory:ca-dir}
requests-directory = $${cadirectory:requests}
wrapper = $${directory:service}/certificate_authority
ca-private = $${cadirectory:private}
ca-certs = $${cadirectory:certs}
ca-newcerts = $${cadirectory:newcerts}
ca-crl = $${cadirectory:crl}
[ca-httpd]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
key-file = $${cadirectory:certs}/httpd.key
cert-file = $${cadirectory:certs}/httpd.crt
executable = $${directory:bin}/cgi-httpd
wrapper = $${directory:service}/cgi-httpd
# Put domain name
name = example.com
###########
# Deploy a webserver running cgi scripts for monitoring
###########
[public]
recipe = slapos.cookbook:zeroknown.write
filename = knowledge0.cfg
monitor-password = passwordtochange
[zero-parameters]
recipe = slapos.cookbook:zeroknown.read
filename = $${public:filename}
# XXX could it be something lighter?
[cgi-httpd-configuration-file]
recipe = collective.recipe.template
input = inline:
PidFile "$${:pid-file}"
ServerName example.com
ServerAdmin someone@email
<IfDefine !MonitorPort>
Listen [$${:listening-ip}]:$${monitor-parameters:port}
Define MonitorPort
</IfDefine>
DocumentRoot "$${:document-root}"
ErrorLog "$${:error-log}"
LoadModule unixd_module modules/mod_unixd.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule mime_module modules/mod_mime.so
LoadModule cgid_module modules/mod_cgid.so
LoadModule dir_module modules/mod_dir.so
LoadModule ssl_module modules/mod_ssl.so
# SSL Configuration
<IfDefine !SSLConfigured>
Define SSLConfigured
SSLCertificateFile $${ca-httpd:cert-file}
SSLCertificateKeyFile $${ca-httpd:key-file}
SSLRandomSeed startup builtin
SSLRandomSeed connect builtin
SSLRandomSeed startup /dev/urandom 256
SSLRandomSeed connect builtin
SSLProtocol -ALL +SSLv3 +TLSv1
SSLHonorCipherOrder On
SSLCipherSuite RC4-SHA:HIGH:!ADH
</IfDefine>
SSLEngine On
ScriptSock $${:cgid-pid-file}
<Directory $${:document-root}>
SSLVerifyDepth 1
SSLRequireSSL
SSLOptions +StrictRequire
# XXX: security????
Options +ExecCGI
AddHandler cgi-script .cgi
DirectoryIndex $${deploy-index:filename}
</Directory>
output = $${directory:etc}/cgi-httpd.conf
# md5sum =
listening-ip = $${slap-parameters:ipv6-random}
# XXX: randomize-me
htdocs = $${directory:www}
pid-file = $${directory:run}/cgi-httpd.pid
cgid-pid-file = $${directory:run}/cgi-httpd-cgid.pid
document-root = $${directory:www}
error-log = $${directory:log}/cgi-httpd-error-log
[cgi-httpd-wrapper]
recipe = slapos.cookbook:wrapper
apache-executable = ${apache:location}/bin/httpd
command-line = $${:apache-executable} -f $${cgi-httpd-configuration-file:output} -DFOREGROUND
wrapper-path = $${ca-httpd:executable}
[monitor-promise]
recipe = slapos.cookbook:check_url_available
path = $${directory:promises}/monitor
url = https://[$${cgi-httpd-configuration-file:listening-ip}]:$${monitor-parameters:port}/$${deploy-index:filename}
check-secure = 1
dash_path = ${dash:location}/bin/dash
curl_path = ${curl:location}/bin/curl
[publish-connection-informations]
recipe = slapos.cookbook:publish
monitor_url = https://[$${cgi-httpd-configuration-file:listening-ip}]:$${monitor-parameters:port}
IMPORTANT_monitor_info = Change the monitor_password as soon as possible ! Default is : $${public:monitor-password}
#!{{ python_executable }}
import datetime
import json
import os
import subprocess
import sys
import time
from optparse import OptionParser, make_option
instance_path = "{{ directory['home'] }}"
monitor_dir = "{{ directory['monitor'] }}"
pid_dir = "{{ directory['run'] }}"
promise_dir = "{{ directory['promise'] }}"
monitoring_file_json = "{{ monitoring_file_json }}"
monitoring_folder_bool = "{{ monitoring_folder_bool }}"
option_list = [
make_option("-a", "--all", action="store_true", dest="all",
help="test everything : promises, services, customs"),
make_option("-n", "--no-write", action="store_true", dest="only_stdout",
help="just show the json output on stdout"),
make_option("-m", "--monitors", action="store_true", dest="monitor",
help="add the custom monitoring file to the files to monitor"),
make_option("-p", "--promises", action="store_true", dest="promise",
help="add the promises\'file to the files to monitor"),
make_option("-s", "--services", action="store_true", dest="service",
help="add the file containing services\'pid to the files to monitor")
]
def getListOfScripts(directory):
scripts = []
if os.path.exists(directory) and os.path.isdir(directory):
for file in os.listdir(directory):
scripts.append(os.path.join(directory, file))
else:
exit("There is a problem in your directories" \
"of monitoring. Please check them")
return scripts
def runServices(directory):
services = getListOfScripts(directory)
result = {}
for service in services:
service_path = os.path.join(pid_dir, service)
service_name = os.path.basename(service_path)
try:
pid = int(open(service_path).read())
### because apache (or others) can write sockets
except IOError:
continue
try:
os.kill(pid, 0)
result[service_name] = ''
except OSError:
result[service_name] = "This service is not running anymore"
return result
def runScripts(directory):
scripts = getListOfScripts(directory)
script_timeout = 3
result = {}
for script in scripts:
command = [os.path.join(promise_dir, script)]
script = os.path.basename(command[0])
result[script] = ''
process_handler = subprocess.Popen(command,
cwd=instance_path,
env=None if sys.platform == 'cygwin' else {},
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE)
process_handler.stdin.flush()
process_handler.stdin.close()
process_handler.stdin = None
time.sleep(script_timeout)
if process_handler.poll() is None:
process_handler.terminate()
result[script] = "Time Out"
elif process_handler.poll() != 0:
stderr = process_handler.communicate()[1]
if stderr is not None:
result[script] = stderr.strip()
return result
def writeFiles(monitors):
fail = False
for i in monitors.values():
if i != "" :
fail = True
if fail:
message = "FAILURE : something went wrong\n"
else:
message = "SUCCESS : everything is ok\n"
date = datetime.datetime.now().ctime()
monitors['datetime'] = date
file_bool = os.path.join(monitoring_folder_bool, str(time.time()))
open(file_bool, "w+").write(date + "," + message)
open(monitoring_file_json, "w+").write(json.dumps(monitors))
if __name__ == "__main__":
parser = OptionParser(option_list=option_list)
monitors = {}
(options, args) = parser.parse_args()
if not (options.monitor or options.promise
or options.service or options.all):
exit("Please provide at list one arg in : -a, -m, -p, -s")
if options.monitor or options.all:
monitors.update(runScripts(monitor_dir))
if options.promise or options.all:
monitors.update(runScripts(promise_dir))
if options.service or options.all:
monitors.update(runServices(pid_dir))
if options.only_stdout:
print json.dumps(monitors)
else:
writeFiles(monitors)
if len(monitors) == 0:
exit(0)
else:
exit(1)
import datetime
import uuid
import PyRSS2Gen
import sys
from email.utils import parsedate_tz, mktime_tz
import base64
# Based on http://thehelpfulhacker.net/2011/03/27/a-rss-feed-for-your-crontabs/
# ### Defaults
TITLE = sys.argv[1]
LINK = sys.argv[2]
DESCRIPTION = TITLE
items = []
while 1:
try:
line = sys.stdin.readline()
except KeyboardInterrupt:
break
if not line:
break
time, desc = line.split(',', 1)
rss_item = PyRSS2Gen.RSSItem(
title = desc,
description = "%s, %s" % (time, desc),
link = LINK,
pubDate = datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz(time))),
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc)))
)
items.append(rss_item)
### Build the rss feed
rss_feed = PyRSS2Gen.RSS2 (
title = TITLE,
link = LINK,
description = DESCRIPTION,
lastBuildDate = datetime.datetime.utcnow(),
items = items
)
print rss_feed.to_xml()
\ No newline at end of file
#!{{ extra_eggs_interpreter }}
import cgi
import cgitb
import Cookie
import jinja2
import json
import os
import subprocess
import sys
import urllib
cgitb.enable(display=0, logdir="/tmp/cgi.log")
form = cgi.FieldStorage()
cookie = Cookie.SimpleCookie()
cgi_path = "{{ cgi_directory }}"
def forward_form():
command = os.path.join(cgi_path, form['posting-script'].value)
params_dict = {}
for f in form:
params_dict[f] = form[f].value
del params_dict['posting-script']
os.environ['QUERY_STRING'] = urllib.urlencode(params_dict)
try:
if os.access(command, os.X_OK):
print '\n', subprocess.check_output([command])
except subprocess.CalledProcessError:
print "There is a problem with sub-process"
pass
def return_document():
command = os.path.join(cgi_path, form['script'].value)
#XXX this functions should be called only for display,
#so a priori it doesn't need form data
os.environ['QUERY_STRING'] = ''
try:
if os.access(command, os.X_OK):
print '\n', subprocess.check_output([command])
elif os.access(command, os.R_OK):
print open(command).read()
else:
raise OSError
except (subprocess.CalledProcessError, OSError):
print "<p>File cannot be found</p>"
def make_menu():
# Transform deep-2 tree in json
folder_list = {}
for folder in os.listdir(cgi_path):
if os.path.isdir(os.path.join(cgi_path, folder)):
folder_list[folder] = []
for folder in folder_list:
for file in os.listdir(os.path.join(cgi_path, folder)):
if os.path.isfile(os.path.join(cgi_path, folder, file)):
folder_list[folder].append(file)
return folder_list
# Beginning of response
print "Content-Type: text/html"
# Check if user is logged
if "password" in form:
password = form['password'].value
if password == '{{ password }}' :
cookie['password'] = password
print cookie, "; Path=/; HttpOnly"
else:
cookie_string = os.environ.get('HTTP_COOKIE')
if cookie_string:
cookie.load(cookie_string)
try:
password = cookie['password'].value
except KeyError:
password = None
else:
password = None
print '\n'
if not password or password != '{{ password }}':
print "<html><head>"
print """
<link rel="stylesheet" href="pure-min.css">
<link rel="stylesheet" href="/style.css">"""
print "</head><body>"
if password is None:
print "<h1>This is the monitoring interface</h1>"
else:
print "<h1>Error</h1><p>Wrong password</p>"
print """
<p>Please enter the monitor_password in the next field to access the data</p>
<form action="/index.cgi" method="post" class="pure-form-aligned">
Password : <input type="password" name="password">
<button type="submit" class="pure-button pure-button-primary">Access</button>
</form>
</body></html>"""
# redirection to the required script/page
else:
print
if "posting-script" in form:
forward_form()
elif "script" in form:
return_document()
else:
html_base = jinja2.Template(open('{{ index_template }}').read())
print
print html_base.render(tree=make_menu(), default_page="{{ default_page }}")
<html>
<head>
<title>Monitoring Interface</title>
<link rel="stylesheet" href="pure-min.css">
<link rel="stylesheet" href="/style.css">
<script src="jquery-1.10.2.min.js"></script>
<script src="script.js"></script>
</head>
<body>
<div id="div-menu">
<h1>Monitoring</h1>
<div id="script-categories" class="pure-menu pure-menu-open">
<ul>
{% for category in tree %}
<li class="pure-menu-heading category">{{ category }}</li>
{% for script in tree[category] %}
<li><a href="{{ category }}/{{ script }}" class="script">{{ script }}</a></li>
{% endfor %}
{% endfor %}
</ul>
</div>
</div>
<div id="content">
<iframe src="{{ default_page }}">
</iframe>
</div>
</body>
</html>
#!{{ python_executable }}
import cgi
import cgitb
import ConfigParser
import os
cgitb.enable()
form = cgi.FieldStorage()
print "<html><head>"
print "<link rel=\"stylesheet\" href=\"pure-min.css\">"
print "<link rel=\"stylesheet\" href=\"/style.css\">"
print "</head><body>"
config_file = "{{ config_cfg }}"
if not os.path.exists(config_file):
print "Your software does <b>not</b> embed 0-knowledge. \
This interface is useless in this case"
exit(0)
parser = ConfigParser.ConfigParser()
parser.read(config_file)
for name in form:
parser.set('public', name, form[name].value)
with open(config_file, 'w') as file:
parser.write(file)
if len(form) > 0:
try:
os.remove("{{ timestamp }}")
except OSError:
pass
print "<h1>Values that can be defined :</h1>"
print "<form action=\"/index.cgi\" method=\"post\" class=\"pure-form-aligned\">"
print "<input type=\"hidden\" name=\"posting-script\" value=\"{{ pwd }}/{{ this_file }}\">"
for option in parser.options("public"):
print "<div class=\"pure-control-group\">"
print "<label for=\"%s\">%s</label>"%(option, option)
print "<input type=\"text\" name=\"%s\" value=\"%s\">"%(option, parser.get('public', option))
print "</div>"
print "<div class=\"pure-controls\"><button type=\"submit\" class=\"pure-button \
pure-button-primary\">Save</button></div></form>"
print "<br><h1>Other values :</h1>"
print "<form class=\"pure-form-aligned\">"
for section in parser.sections():
if section != 'public':
for option in parser.options(section):
print "<div class=\"pure-control-group\">"
print "<label for=\"%s\">%s</label>"%(option, option)
print "<input type=\"text\" name=\"%s\" value=\"%s\" readonly>"%(option, parser.get(section, option))
print "</div>"
print "</form>"
print "</body></html>"
#!{{ python_executable }}
import cgitb
import json
cgitb.enable(display=0, logdir="/tmp/cgi.log")
json_file = "{{ json_file }}"
result = json.load(open(json_file))
print "<html><head>"
print "<link rel=\"stylesheet\" href=\"pure-min.css\">"
print "<link rel=\"stylesheet\" href=\"/style.css\">"
print "</head><body>"
print "<h1>Monitoring :</h1>"
print "<p><em>Last time of monitoring process : %s</em></p>" % (result['datetime'])
del result['datetime']
print "<br/>"
print "<h2>These scripts and promises have failed :</h2>"
for r in result:
if result[r] != '':
print "<h3>%s</h3><p style=\"padding-left:30px;\">%s</p>" % (r, result[r])
print "<br/>"
print "<h2>These scripts and promises were successful :</h2>"
print "<ul>"
for r in result:
if result[r] == '':
print "<li>%s</li>" % (r)
print "</ul>"
print "</body></html>"
...@@ -15,7 +15,8 @@ extensions += ...@@ -15,7 +15,8 @@ extensions +=
# Use shacache and lxml # Use shacache and lxml
extends = extends =
../component/lxml-python/buildout.cfg ../component/lxml-python/buildout.cfg
../component/python-openssl/buildout.cfg ../component/python-cffi/buildout.cfg
../component/python-cryptography/buildout.cfg
# Separate from site eggs # Separate from site eggs
allowed-eggs-from-site-packages = allowed-eggs-from-site-packages =
...@@ -63,7 +64,9 @@ networkcache-section = networkcache ...@@ -63,7 +64,9 @@ networkcache-section = networkcache
recipe = zc.recipe.egg recipe = zc.recipe.egg
eggs = eggs =
${lxml-python:egg} ${lxml-python:egg}
${python-openssl:egg} ${python-cffi:egg}
${python-cryptography:egg}
pyOpenSSL
slapos.cookbook slapos.cookbook
cliff cliff
hexagonit.recipe.download hexagonit.recipe.download
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment