diff --git a/component/file/buildout.cfg b/component/file/buildout.cfg index 0a0a392f422d8c0c983f5572cef58355366766b4..8e0909e6e404a02c94d02fb1e027264d66efec20 100644 --- a/component/file/buildout.cfg +++ b/component/file/buildout.cfg @@ -8,8 +8,8 @@ extends = [file] recipe = slapos.recipe.cmmi -url = ftp://ftp.astron.com/pub/file/file-5.18.tar.gz -md5sum = d420d8f2990cd344673acfbf8d76ff5a +url = ftp://ftp.astron.com/pub/file/file-5.19.tar.gz +md5sum = e3526f59023f3f7d1ffa4d541335edab configure-options = --disable-static environment = diff --git a/component/gcc/buildout.cfg b/component/gcc/buildout.cfg index f91596c732b70309924a2da06186fcfa02767075..3a3493a509d516e9e675b1227fb9a0296e930586 100644 --- a/component/gcc/buildout.cfg +++ b/component/gcc/buildout.cfg @@ -14,21 +14,21 @@ recipe = slapos.recipe.cmmi url = http://www.mpfr.org/mpfr-3.0.1/mpfr-3.0.1.tar.bz2 md5sum = bfbecb2eacb6d48432ead5cfc3f7390a configure-options = - --with-gmp=${gmp:location} + --with-gmp=${gmp-4:location} --disable-static environment = - LDFLAGS=-Wl,-rpath=${gmp:location}/lib + LDFLAGS=-Wl,-rpath=${gmp-4:location}/lib [mpc] recipe = slapos.recipe.cmmi url = http://www.multiprecision.org/mpc/download/mpc-0.9.tar.gz md5sum = 0d6acab8d214bd7d1fbbc593e83dd00d configure-options = - --with-gmp=${gmp:location} + --with-gmp=${gmp-4:location} --with-mpfr=${mpfr:location} --disable-static environment = - LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp:location}/lib + LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp-4:location}/lib [ppl] recipe = slapos.recipe.cmmi @@ -36,24 +36,24 @@ recipe = slapos.recipe.cmmi url = http://bugseng.com/products/ppl/download/ftp/releases/0.10.2/ppl-0.10.2.tar.bz2 md5sum = 5667111f53150618b0fa522ffc53fc3e configure-options = - --with-libgmp-prefix=${gmp:location} - --with-libgmpxx-prefix=${gmp:location} + --with-libgmp-prefix=${gmp-4:location} + --with-libgmpxx-prefix=${gmp-4:location} --disable-static environment = PATH=${m4:location}/bin:%(PATH)s - LDFLAGS=-Wl,-rpath=${gmp:location}/lib + LDFLAGS=-Wl,-rpath=${gmp-4:location}/lib [cloog-ppl] recipe = slapos.recipe.cmmi url = ftp://gcc.gnu.org/pub/gcc/infrastructure/cloog-ppl-0.15.9.tar.gz md5sum = 806e001d1b1a6b130069ff6274900af5 configure-options = - --with-gmp=${gmp:location} + --with-gmp=${gmp-4:location} --with-ppl=${ppl:location} --disable-static environment = PATH=${m4:location}/bin:%(PATH)s - LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${ppl:location}/lib + LDFLAGS=-Wl,-rpath=${gmp-4:location}/lib -Wl,-rpath=${ppl:location}/lib [gcc-download] hack-revision = ${gcc-interconnection-workaround:hack-revision} @@ -89,14 +89,14 @@ configure-options = --disable-bootstrap --enable-languages="c,c++" --disable-multilib - --with-gmp=${gmp:location} + --with-gmp=${gmp-4:location} --with-mpfr=${mpfr:location} --with-mpc=${mpc:location} --with-ppl=${ppl:location} --with-cloog=${cloog-ppl:location} --prefix=${buildout:parts-directory}/${:_buildout_section_name_} environment = - LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${ppl:location}/lib -Wl,-rpath=${cloog-ppl:location}/lib + LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp-4:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${ppl:location}/lib -Wl,-rpath=${cloog-ppl:location}/lib # make install does not work when several core are used make-targets = install -j1 @@ -111,13 +111,13 @@ configure-options = --disable-bootstrap --enable-languages=c --disable-multilib - --with-gmp=${gmp:location} + --with-gmp=${gmp-4:location} --with-mpfr=${mpfr:location} --with-mpc=${mpc:location} --without-ppl --without-cloog environment = - LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${mpc:location}/lib + LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp-4:location}/lib -Wl,-rpath=${mpc:location}/lib # make install does not work when several core are used make-targets = install -j1 diff --git a/component/gmp/buildout.cfg b/component/gmp/buildout.cfg index 3a90c862575f60cd59a588f75207b30565340387..90a8d9fffef23f1064582aca14deea0e970c55a9 100644 --- a/component/gmp/buildout.cfg +++ b/component/gmp/buildout.cfg @@ -1,13 +1,14 @@ [buildout] extends = ../m4/buildout.cfg + ../xz-utils/buildout.cfg parts = gmp -[gmp] -recipe = slapos.recipe.cmmi # we should use version 4.x for ppl-0.10 +[gmp-4] +recipe = slapos.recipe.cmmi url = ftp://ftp.gmplib.org/pub/gmp-4.3.2/gmp-4.3.2.tar.bz2 md5sum = dd60683d7057917e34630b4a787932e8 # GMP does not correctly detect achitecture so it have to be given @@ -20,4 +21,13 @@ configure-options = environment = PATH=${m4:location}/bin:%(PATH)s - +[gmp] +recipe = slapos.recipe.cmmi +version = 6.0.0a +url = ftp://ftp.gmplib.org/pub/gmp-6.0.0/gmp-${:version}.tar.xz +md5sum = 1e6da4e434553d2811437aa42c7f7c76 +configure-options = + --enable-cxx + --disable-static +environment = + PATH=${m4:location}/bin:${xz-utils:location}/bin:%(PATH)s diff --git a/component/gnutls/buildout.cfg b/component/gnutls/buildout.cfg index 0996e8e528a16a3798dfbdc33c83992660c97193..d3f79944634f62112af5ade20648d58109d3f685 100644 --- a/component/gnutls/buildout.cfg +++ b/component/gnutls/buildout.cfg @@ -28,6 +28,8 @@ environment = recipe = slapos.recipe.cmmi url = ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.15.tar.xz md5sum = ec3b06f80e312137386c5d322183ca5a +depends = + ${gmp:version} configure-options = --disable-static --disable-libdane diff --git a/component/haproxy/buildout.cfg b/component/haproxy/buildout.cfg index 252eca35f222e77c303032f9e2ab3db931b7b090..a6da5507db62a4bf17afd41c84e3e1eb4b168a98 100644 --- a/component/haproxy/buildout.cfg +++ b/component/haproxy/buildout.cfg @@ -11,8 +11,8 @@ parts = haproxy [haproxy] recipe = slapos.recipe.cmmi -url = http://haproxy.1wt.eu/download/1.4/src/haproxy-1.5.1.tar.gz -md5sum = 49640cf3ddd793a05fbd3394481a1ed4 +url = http://www.haproxy.org/download/1.5/src/haproxy-1.5.2.tar.gz +md5sum = e854fed32ea751d6db7f366cb910225a configure-command = true # If the system is running on Linux 2.6, we use "linux26" as the TARGET, # otherwise use "generic". diff --git a/component/nettle/buildout.cfg b/component/nettle/buildout.cfg index e3fc207ffdf4cc25175413489c2fb51066bed4e2..2afbd72be2599369c0f50860b261431c1d15db2b 100644 --- a/component/nettle/buildout.cfg +++ b/component/nettle/buildout.cfg @@ -14,6 +14,8 @@ md5sum = 41dd0ce2a73487929bdc637b75dd62c9 recipe = slapos.recipe.cmmi url = http://www.lysator.liu.se/~nisse/archive/nettle-2.7.1.tar.gz md5sum = 003d5147911317931dd453520eb234a5 +depends = + ${gmp:version} patches = ${nettle-lib-location.patch:location}/${nettle-lib-location.patch:filename} configure-option = diff --git a/component/nginx/buildout.cfg b/component/nginx/buildout.cfg index fac104bc55bf81d931a904b0183bbe2cd84301a4..fd02cd2984c849a5b72f825c4399cecd3052f241 100644 --- a/component/nginx/buildout.cfg +++ b/component/nginx/buildout.cfg @@ -4,13 +4,14 @@ extends = ../zlib/buildout.cfg ../openssl/buildout.cfg ../coreutils/buildout.cfg + ../git/buildout.cfg parts = nginx-output [nginx] recipe = slapos.recipe.cmmi -url = http://nginx.org/download/nginx-1.5.3.tar.gz -md5sum = 1e735dd6a6ade2b5c20e924b67c3d355 +url = http://nginx.org/download/nginx-1.7.2.tar.gz +md5sum = 68949a7a0bad4615e9b737fc4e6047f2 configure-options= --with-ipv6 --with-http_ssl_module @@ -49,6 +50,7 @@ command = ${coreutils-output:test} -x ${:nginx} -a -f ${:mime} nginx = ${nginx:location}/sbin/nginx mime = ${nginx:location}/conf/mime.types + [nginx-unstable] <= nginx url = http://nginx.org/download/nginx-1.3.15.tar.gz @@ -70,3 +72,23 @@ configure-options= --with-cc-opt="-I ${pcre:location}/include -I ${openssl:location}/include -I ${zlib:location}/include" --add-module=${hexaglobe-nginx-module:location}/sub_module # --add-module=${hexaglobe-nginx-module:location}/nginx-upstream-fair + +[nginx-push-stream-module] +recipe = slapos.recipe.build:gitclone +repository = https://github.com/wandenberg/nginx-push-stream-module.git +revision = 3d3a204177d3a7ab8a2858e04e792a6d11bf133f +git-executable = ${git:location}/bin/git + +[nginx-push-stream] +<= nginx +configure-options= + --with-ipv6 + --with-http_ssl_module + --with-ld-opt="-L ${zlib:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib" + --with-cc-opt="-I ${pcre:location}/include -I ${openssl:location}/include -I ${zlib:location}/include" + --add-module=${nginx-push-stream-module:location} + +[nginx-push-stream-output] +<= nginx-output +nginx = ${nginx-push-stream:location}/sbin/nginx +mime = ${nginx-push-stream:location}/conf/mime.types diff --git a/component/percona-toolkit/buildout.cfg b/component/percona-toolkit/buildout.cfg index 77098661e0c7e7a59c92c10e78af3dbf40e23221..4e861898e442b098d3e4b5464459f733b914cb77 100644 --- a/component/percona-toolkit/buildout.cfg +++ b/component/percona-toolkit/buildout.cfg @@ -10,8 +10,8 @@ parts = recipe = slapos.recipe.cmmi depends = ${perl:version} -version = 2.2.8 +version = 2.2.9 url = http://www.percona.com/redir/downloads/percona-toolkit/${:version}/percona-toolkit-${:version}.tar.gz -md5sum = c7fe343308898ad7f0c456c8394f2cbf +md5sum = f0a24d89e56bd01c6c8c5fdfdb5cad44 configure-command = ${perl:location}/bin/perl Makefile.PL diff --git a/component/pycrypto-python/buildout.cfg b/component/pycrypto-python/buildout.cfg index 2555803c4e556c03b73249178e295264aca3b563..3ea551ac5fb7e9ef512fb49caa8f2941d8505c07 100644 --- a/component/pycrypto-python/buildout.cfg +++ b/component/pycrypto-python/buildout.cfg @@ -11,6 +11,8 @@ versions = versions recipe = zc.recipe.egg:custom egg = pycrypto environment = pycrypto-python-env +depends = + ${gmp:version} include-dirs = ${gmp:location}/include library-dirs = diff --git a/component/screen/buildout.cfg b/component/screen/buildout.cfg index 1fb191585a0bdb789e5f924254744f13f6523791..0f19f721214c5c645684865949ae2ab2bcc3373f 100644 --- a/component/screen/buildout.cfg +++ b/component/screen/buildout.cfg @@ -12,4 +12,8 @@ url = http://ftp.gnu.org/gnu/screen/screen-4.0.3.tar.gz md5sum = 8506fd205028a96c741e4037de6e3c42 environment= CFLAGS=-I${ncurses:location}/include - LDFLAGS=-L${ncurses:location}/lib/ -Wl,-rpath=${ncurses:location}/lib/ \ No newline at end of file + LDFLAGS=-L${ncurses:location}/lib/ -Wl,-rpath=${ncurses:location}/lib/ +# Compatibility with CentOS +patch-options = -p1 -b +patches = + ${:_profile_base_location_}/screen-4.0.3-stropts.patch#5677917353067ffbcb08a23a59605aa4 \ No newline at end of file diff --git a/component/screen/screen-4.0.3-stropts.patch b/component/screen/screen-4.0.3-stropts.patch new file mode 100644 index 0000000000000000000000000000000000000000..d6cbed74e326deba5423967e497d330ced3f21fc --- /dev/null +++ b/component/screen/screen-4.0.3-stropts.patch @@ -0,0 +1,48 @@ +diff -up screen-4.0.3/tty.sh.stropts screen-4.0.3/tty.sh +--- screen-4.0.3/tty.sh.stropts 2003-09-08 16:24:25.000000000 +0200 ++++ screen-4.0.3/tty.sh 2008-04-07 11:28:34.000000000 +0200 +@@ -76,7 +76,7 @@ exit 0 + #endif + + #include "config.h" +-#ifdef SVR4 ++#if 0 + #include <sys/stropts.h> /* for I_POP */ + #endif + +diff -up screen-4.0.3/screen.c.stropts screen-4.0.3/screen.c +--- screen-4.0.3/screen.c.stropts 2008-04-07 11:25:21.000000000 +0200 ++++ screen-4.0.3/screen.c 2008-04-07 11:29:14.000000000 +0200 +@@ -50,7 +50,7 @@ + + #include "config.h" + +-#ifdef SVR4 ++#if 0 + # include <sys/stropts.h> + #endif + +diff -up screen-4.0.3/process.c.stropts screen-4.0.3/process.c +--- screen-4.0.3/process.c.stropts 2003-09-18 14:53:54.000000000 +0200 ++++ screen-4.0.3/process.c 2008-04-07 11:29:47.000000000 +0200 +@@ -37,7 +37,7 @@ + #include "config.h" + + /* for solaris 2.1, Unixware (SVR4.2) and possibly others: */ +-#ifdef SVR4 ++#if 0 + # include <sys/stropts.h> + #endif + +diff -up screen-4.0.3/pty.c.stropts screen-4.0.3/pty.c +--- screen-4.0.3/pty.c.stropts 2003-09-08 16:26:18.000000000 +0200 ++++ screen-4.0.3/pty.c 2008-04-07 11:30:07.000000000 +0200 +@@ -34,7 +34,7 @@ + #endif + + /* for solaris 2.1, Unixware (SVR4.2) and possibly others */ +-#ifdef HAVE_SVR4_PTYS ++#if 0 + # include <sys/stropts.h> + #endif + diff --git a/setup.py b/setup.py index 69ef0ebfbb046cf56444dc7afa58b4e1a891eb0e..c220063c4e460c88c7585ed28a50b517d42b86c6 100755 --- a/setup.py +++ b/setup.py @@ -184,9 +184,6 @@ setup(name=name, 'slapmonitor = slapos.recipe.slapmonitor:MonitorRecipe', 'slapmonitor-xml = slapos.recipe.slapmonitor:MonitorXMLRecipe', 'slapreport = slapos.recipe.slapreport:Recipe', - 'slaprunner.test = slapos.recipe.slaprunner:Test', - 'slaprunner.export = slapos.recipe.slaprunner.backup:ExportRecipe', - 'slaprunner.import = slapos.recipe.slaprunner.backup:ImportRecipe', 'softwaretype = slapos.recipe.softwaretype:Recipe', 'sphinx= slapos.recipe.sphinx:Recipe', 'squid = slapos.recipe.squid:Recipe', diff --git a/slapos/recipe/request.py b/slapos/recipe/request.py index 739b71940362cee068dcf060caf1e080c31d54a9..ae5976a6dcc4857d51a0479131418d7282ba8a19 100644 --- a/slapos/recipe/request.py +++ b/slapos/recipe/request.py @@ -153,6 +153,7 @@ class Recipe(object): # XXX: deprecated, to be removed options['instance_guid'] = self.instance.getInstanceGuid() options['instance-state'] = self.instance.getState() + options['instance-status'] = self.instance.getStatus() except (slapmodule.ResourceNotReady, AttributeError): # Backward compatibility. Old SlapOS master and core don't know this. self.logger.warning("Impossible to fetch instance GUID nor state.") diff --git a/slapos/recipe/slaprunner/__init__.py b/slapos/recipe/slaprunner/__init__.py deleted file mode 100644 index 6804b73d92b78a4192badd9975feb5330d189628..0000000000000000000000000000000000000000 --- a/slapos/recipe/slaprunner/__init__.py +++ /dev/null @@ -1,98 +0,0 @@ -############################################################################## -# -# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. -# -# WARNING: This program as such is intended to be used by professional -# programmers who take the whole responsibility of assessing all potential -# consequences resulting from its eventual inadequacies and bugs -# End users who are looking for a ready-to-use solution with commercial -# guarantees and support are strongly adviced to contract a Free Software -# Service Company -# -# This program is Free Software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 3 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# -############################################################################## -from slapos.recipe.librecipe import GenericBaseRecipe -import os - -#XXX-Nicolas This recipe has to be deleted as soon as possible -#No changes allowed, except full-replacement using buildout -class Test(GenericBaseRecipe): - def _options(self, options): - self.ipv4 = options['ipv4'].strip() - self.ipv6 = options['ipv6'].strip() - self.proxy_port = options['proxy_port'].strip() - self.runner_port = options['runner_port'].strip() - self.workdir = options['working-directory'].strip() - self.software_directory = options['software-directory'].strip() - self.instance_directory = options['instance-directory'].strip() - self.partition_amount = options['partition-amount'].strip() - self.cloud9_url = options.get('cloud9-url', '').strip() - # Set slaprunner access URL - options['access-url'] = 'http://[%s]:%s' % (self.ipv6, self.runner_port) - - def install(self): - path_list = [] - - configuration = dict( - software_root=self.software_directory, - instance_root=self.instance_directory, - master_url='http://%s:%s/' % (self.ipv4, self.proxy_port), - computer_id='slaprunner', - partition_amount=self.partition_amount, - slapgrid_sr=self.options['slapgrid_sr'], - slapgrid_cp=self.options['slapgrid_cp'], - slapproxy=self.options['slapproxy'], - supervisor=self.options['supervisor'], - supervisord_config=os.path.join(self.instance_directory, 'etc', - 'supervisord.conf'), - runner_workdir=self.workdir, - etc_dir=self.options['etc_dir'], - run_dir=self.options['etc_dir'], - log_dir=self.workdir, - runner_host=self.ipv4, - runner_port=self.runner_port, - ipv4_address=self.ipv4, - ipv6_address=self.ipv6, - proxy_host=self.ipv4, - proxy_port=self.proxy_port, - proxy_database=os.path.join(self.workdir, 'proxy.db'), - git=self.options['git-binary'], - ssh_client=self.options['ssh_client'], - public_key=self.options['public_key'], - private_key=self.options['private_key'], - cloud9_url=self.cloud9_url - ) - - config_file = self.createFile(self.options['slapos.cfg'], - self.substituteTemplate(self.getTemplateFilename('slapos.cfg.in'), - configuration)) - path_list.append(config_file) - - environment = dict( - PATH=os.path.dirname( - self.options['git-binary']) + ':' + os.environ['PATH'], - GIT_SSH=self.options['ssh_client'], - CONFIG_FILE_PATH=config_file - ) - launch_args = [self.options['slaprunnertest'].strip()] - - wrapper = self.createPythonScript(self.options['wrapper'], - 'slapos.recipe.librecipe.execute.executee', - (launch_args, environment) - ) - path_list.append(wrapper) - - return path_list diff --git a/slapos/recipe/slaprunner/backup.py b/slapos/recipe/slaprunner/backup.py deleted file mode 100644 index 06ecc885275025528fc45c727f96ba5c489df988..0000000000000000000000000000000000000000 --- a/slapos/recipe/slaprunner/backup.py +++ /dev/null @@ -1,131 +0,0 @@ -############################################################################## -# -# Copyright (c) 2013 Vifib SARL and Contributors. All Rights Reserved. -# -# WARNING: This program as such is intended to be used by professional -# programmers who take the whole responsibility of assessing all potential -# consequences resulting from its eventual inadequacies and bugs -# End users who are looking for a ready-to-use solution with commercial -# guarantees and support are strongly adviced to contract a Free Software -# Service Company -# -# This program is Free Software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 3 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# -############################################################################## - -import textwrap - -from slapos.recipe.librecipe import GenericBaseRecipe - - - -class ExportRecipe(GenericBaseRecipe): - """\ - This recipe creates an exporter script for using with the resilient stack. - - Required options: - backup-directory - folder that will contain the dump file. - srv-directory - folder that contain the runner directory. - wrapper - full path of the exporter script to create. - """ - - def install(self): - wrapper = self.options['wrapper'] - self.createBackupScript(wrapper) - return [wrapper] - - - def createBackupScript(self, wrapper): - """\ - Create a script to backup the database in 'custom' format. - """ - content = textwrap.dedent("""\ - #!%(shell-binary)s - umask 077 - sync_element () { - path=$1 - backup_path=$2 - shift 2 - element_list=$* - for element in $element_list - do - cd $path; - if [ -f $element ] || [ -d $element ]; then - %(rsync-binary)s -avz --safe-links --delete $element $backup_path; - fi - done - } - sync_element %(srv-directory)s/runner %(backup-directory)s/runner/ instance project proxy.db - # We sync .* appart - cp -r %(etc-directory)s/.??* %(backup-directory)s/etc/ - sync_element %(etc-directory)s %(backup-directory)s/etc/ config.json ssh - if [ -d %(backup-directory)s/runner/software ]; then - rm %(backup-directory)s/runner/software/* - fi - """ % self.options) - self.createExecutable(wrapper, content=content) - - - -class ImportRecipe(GenericBaseRecipe): - """\ - This recipe creates an importer script for using with the resilient stack. - - Required options: - backup-directory - folder that will contain the dump file. - srv-directory - folder that contain the runner directory. - wrapper - full path of the exporter script to create. - """ - - def install(self): - wrapper = self.options['wrapper'] - self.createRestoreScript(wrapper) - return [wrapper] - - - def createRestoreScript(self, wrapper): - """\ - Create a script to restore the database from 'custom' format. - """ - content = textwrap.dedent("""\ - #!%(shell-binary)s - umask 077 - restore_element () { - backup_path=$1 - restore_path=$2 - shift 2 - element_list=$* - for element in $element_list - do - cd $backup_path; - if [ -f $element ] || [ -d $element ]; then - %(rsync-binary)s -avz --delete $backup_path/$element $restore_path; - fi - done - } - restore_element %(backup-directory)s/runner/ %(srv-directory)s/runner instance project proxy.db - restore_element %(backup-directory)s/etc/ %(etc-directory)s config.json ssh - cp -r %(backup-directory)s/etc/.??* %(etc-directory)s; - %(curl-binary)s --insecure -vg6L --max-time 5 %(backend-url)s/isSRReady; - """ % self.options) - self.createExecutable(wrapper, content=content) - - diff --git a/slapos/recipe/slaprunner/template/slapos.cfg.in b/slapos/recipe/slaprunner/template/slapos.cfg.in deleted file mode 100644 index 7bc2d22c63a7b7f5847b275bf5020f8a6693b3d0..0000000000000000000000000000000000000000 --- a/slapos/recipe/slaprunner/template/slapos.cfg.in +++ /dev/null @@ -1,40 +0,0 @@ -[slapos] -software_root = %(software_root)s -instance_root = %(instance_root)s -master_url = %(master_url)s -computer_id = %(computer_id)s -maximal_delay = 0 - -[slapformat] -partition_amount = %(partition_amount)s - -[slaprunner] -slapgrid_sr = %(slapgrid_sr)s -slapgrid_cp = %(slapgrid_cp)s -slapproxy = %(slapproxy)s -supervisor = %(supervisor)s -supervisord_config = %(supervisord_config)s -runner_workdir = %(runner_workdir)s -runner_host = %(runner_host)s -runner_port = %(runner_port)s -ipv4_address = %(ipv4_address)s -ipv6_address = %(ipv6_address)s -etc_dir = %(etc_dir)s -run_dir = %(run_dir)s -log_dir = %(log_dir)s - -[slapproxy] -host = %(proxy_host)s -port = %(proxy_port)s -database_uri = %(proxy_database)s - -[sshkeys_authority] -ssh_client = %(ssh_client)s -public_key = %(public_key)s -private_key = %(private_key)s - -[gitclient] -git = %(git)s - -[cloud9_IDE] -cloud9 = %(cloud9_url)s \ No newline at end of file diff --git a/software/html5ide/instance-html5ide.cfg b/software/html5ide/instance-html5ide.cfg new file mode 100644 index 0000000000000000000000000000000000000000..7b56412ab8e1ae1deeacf4c88c10a46dd022f5e8 --- /dev/null +++ b/software/html5ide/instance-html5ide.cfg @@ -0,0 +1,184 @@ +[buildout] +parts = + httpd-launcher + httpd-graceful + httpd-dav-htaccess + httpd-make-dummy-index-html + promise-httpd-ipv6 + publish-connection-information + +## Monitoring part +###Parts to add for monitoring + certificate-authority + cron-entry-monitor + cron-entry-rss + deploy-index + deploy-settings-cgi + deploy-status-cgi + deploy-status-history-cgi + setup-static-files + zero-parameters + public-symlink + cgi-httpd-wrapper + cgi-httpd-graceful-wrapper + monitor-promise + monitor-instance-log-access +## Monitor for apache + +extends = ${monitor-template:output} + +eggs-directory = ${buildout:eggs-directory} +develop-eggs-directory = ${buildout:develop-eggs-directory} +offline = true + +# Create all needed directories +[directory] +recipe = slapos.cookbook:mkdirectory +etc = $${buildout:directory}/etc/ +var = $${buildout:directory}/var/ +srv = $${buildout:directory}/srv +bin = $${buildout:directory}/bin/ +tmp = $${buildout:directory}/tmp/ + +service = $${:etc}/service/ +etc-run = $${:etc}/run/ +log = $${:var}/log/ +run = $${:var}/run/ +backup = $${:srv}/backup/ +promise = $${:etc}/promise/ + +# Jinja Common + +[jinja2-template-base] +recipe = slapos.recipe.template:jinja2 +rendered = $${buildout:directory}/$${:filename} +extra-context = +context = + import json_module json + key eggs_directory buildout:eggs-directory + key develop_eggs_directory buildout:develop-eggs-directory + key slap_software_type instance-parameter:slap-software-type + key slapparameter_dict instance-parameter:configuration + section directory directory + $${:extra-context} + +########################### +# Apache configuration +# + +[apache-directory] +recipe = slapos.cookbook:mkdirectory +www = $${directory:srv}/www +dav = $${directory:srv}/dav +slave-configuration = $${directory:etc}/apache-slave-conf.d/ +ssl = $${directory:etc}/ssl + +[apache-configuration] +httpd-launcher = $${directory:service}/httpd +httpd-graceful = $${directory:etc-run}/httpd-graceful +frontend-configuration = $${directory:etc}/httpd.conf +access-log = $${directory:log}/httpd-access.log +error-log = $${directory:log}/httpd-error.log +pid-file = $${directory:run}/httpd.pid +httpd-graceful-command = kill -USR1 $(cat $${:pid-file}) + +# Configuration +[httpd-parameter] +path_pid = $${directory:run}/httpd.pid +global_ip = $${slap-network-information:global-ipv6} +global_port = 6666 +path_error_log = $${apache-configuration:error-log} +path_access_log = $${apache-configuration:access-log} +key_file = $${ca-httpd:key-file} +cert_file = $${ca-httpd:cert-file} +document_root = $${apache-directory:www} +dav_lock = $${directory:var}/DavLock +dav_home = $${apache-directory:dav} +htpassword_path = $${directory:etc}/.htpasswd +etc_dir = $${directory:etc} +var_dir = $${directory:var} + +[httpd-conf] +recipe = slapos.recipe.template:jinja2 +template = ${template_httpd_conf:location}/${template_httpd_conf:filename} +rendered = $${apache-configuration:frontend-configuration} +context = + section parameter httpd-parameter + +# Htaccess for dav storage +[httpd-dav-htaccess] +recipe = plone.recipe.command +stop-on-error = true +htaccess-path = $${httpd-parameter:htpassword_path} +command = ${apache:location}/bin/htpasswd -cb $${:htaccess-path} $${:user} $${:password} +user = admin +password = admin + +# Launcher and Graceful +[httpd-launcher] +recipe = slapos.cookbook:wrapper +apache-executable = ${apache:location}/bin/httpd +wrapper-path = $${apache-configuration:httpd-launcher} +command-line = $${:apache-executable} -f $${httpd-conf:rendered} -DFOREGROUND + +[httpd-graceful] +recipe = slapos.recipe.template:jinja2 +template = ${template-wrapper:output} +rendered = $${apache-configuration:httpd-graceful} +mode = 0700 +context = + key content apache-configuration:httpd-graceful-command + +# Create Dummu index.html +[httpd-make-dummy-index-html] +recipe = plone.recipe.command +command = if [ ! -f $${httpd-parameter:document_root}/index.html ]; then echo "Hello World!!!" > $${httpd-parameter:document_root}/index.html ; fi + +# Promise +[promise-httpd-ipv6] +recipe = slapos.cookbook:check_port_listening +path = $${directory:promise}/httpd_ipv6 +hostname = $${httpd-parameter:global_ip} +port = $${httpd-parameter:global_port} + +########################### +# Certificate Authority +# + +[cadirectory] +recipe = slapos.cookbook:mkdirectory +ca-dir = $${directory:srv}/ssl +requests = $${:ca-dir}/requests/ +private = $${:ca-dir}/private/ +certs = $${:ca-dir}/certs/ +newcerts = $${:ca-dir}/newcerts/ +crl = $${:ca-dir}/crl/ + +[certificate-authority] +recipe = slapos.cookbook:certificate_authority +openssl-binary = ${openssl:location}/bin/openssl +ca-dir = $${cadirectory:ca-dir} +requests-directory = $${cadirectory:requests} +wrapper = $${directory:service}/certificate_authority +ca-private = $${cadirectory:private} +ca-certs = $${cadirectory:certs} +ca-newcerts = $${cadirectory:newcerts} +ca-crl = $${cadirectory:crl} + +[ca-httpd] +<= certificate-authority +recipe = slapos.cookbook:certificate_authority.request +key-file = $${cadirectory:certs}/httpd.key +cert-file = $${cadirectory:certs}/httpd.crt +executable = $${directory:service}/ca-certificate +wrapper = $${directory:service}/ca-certificate +# Put domain name +name = example.com + +########################### +# Monitor Part +# + +[publish-connection-information] +recipe = slapos.cookbook:publish +server_url = https://[$${httpd-parameter:global_ip}]:$${httpd-parameter:global_port} diff --git a/software/html5ide/instance.cfg b/software/html5ide/instance.cfg new file mode 100644 index 0000000000000000000000000000000000000000..0fd963138b33c6a09df53f1135689087a834f037 --- /dev/null +++ b/software/html5ide/instance.cfg @@ -0,0 +1,11 @@ +[buildout] +parts = + switch-softwaretype + +eggs-directory = ${buildout:eggs-directory} +develop-eggs-directory = ${buildout:develop-eggs-directory} +offline = true + +[switch-softwaretype] +recipe = slapos.cookbook:softwaretype +default = ${template-html5-ide:output} diff --git a/software/html5ide/software.cfg b/software/html5ide/software.cfg new file mode 100644 index 0000000000000000000000000000000000000000..ad5ee4f1978ce4d560c49af1f0ef607dfac3d66c --- /dev/null +++ b/software/html5ide/software.cfg @@ -0,0 +1,97 @@ +[buildout] +extends = + ../../stack/slapos.cfg + ../../stack/monitor/buildout.cfg + ../../component/apache/buildout.cfg + ../../component/curl/buildout.cfg + ../../component/dash/buildout.cfg + ../../component/nodejs/buildout.cfg + +parts = + monitor-eggs + slapos-cookbook + nodejs + template + template_httpd_conf + +[template] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/instance.cfg +md5sum = 8fd9378f81c26be62e5c93bf52c837a5 +output = ${buildout:directory}/template.cfg +mode = 0644 + +[template-html5-ide] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/instance-html5ide.cfg +md5sum = e4d8adb5b7f0e5bf6ea12e47c5a13f8e +output = ${buildout:directory}/template-html5ide.cfg +mode = 0644 + +[template_httpd_conf] +recipe = hexagonit.recipe.download +url = ${:_profile_base_location_}/template/httpd.conf.jinja2 +download-only = true +md5sum = 0c9e75bcbaf5ed97f7b33d472107b634 +filename = httpd_conf.in +mode = 0644 + +[template-wrapper] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/template/wrapper.in +output = ${buildout:directory}/template-wrapper.cfg +mode = 0644 +md5sum = 8cde04bfd0c0e9bd56744b988275cfd8 + +[versions] +Jinja2 = 2.7.3 +MarkupSafe = 0.23 +PyRSS2Gen = 1.1 +Werkzeug = 0.9.6 +buildout-versions = 1.7 +cffi = 0.8.6 +cliff = 1.6.1 +cmd2 = 0.6.7 +cns.recipe.symlink = 0.2.3 +collective.recipe.template = 1.11 +cryptography = 0.5.1 +inotifyx = 0.2.0-1 +itsdangerous = 0.24 +lxml = 3.3.5 +meld3 = 1.0.0 +mr.developer = 1.30 +netaddr = 0.7.12 +netifaces = 0.10.4 +plone.recipe.command = 1.1 +pyOpenSSL = 0.14 +pyparsing = 2.0.2 +pytz = 2014.4 +requests = 2.3.0 +six = 1.7.3 +slapos.cookbook = 0.87 +slapos.core = 1.1.2 +slapos.recipe.cmmi = 0.2 +slapos.recipe.template = 2.5 +stevedore = 0.15 +supervisor = 3.0 +xml-marshaller = 0.9.7 + +# Required by: +# slapos.core==1.1.2 +Flask = 0.10.1 + +# Required by: +# slapos.cookbook==0.87 +lock-file = 2.0 + +# Required by: +# slapos.core==1.1.2 +psutil = 2.1.1 + +# Required by: +# cffi==0.8.6 +pycparser = 2.10 + +# Required by: +# slapos.core==1.1.2 +zope.interface = 4.1.1 \ No newline at end of file diff --git a/software/html5ide/template/httpd.conf.jinja2 b/software/html5ide/template/httpd.conf.jinja2 new file mode 100644 index 0000000000000000000000000000000000000000..6f93120d292e6fafa4617edd7f1a815fcc9124cc --- /dev/null +++ b/software/html5ide/template/httpd.conf.jinja2 @@ -0,0 +1,63 @@ +PidFile "{{ parameter.path_pid }}" +ServerName example.com +ServerAdmin someone@email + +Listen [{{ parameter.global_ip }}]:{{ parameter.global_port }} + +LoadModule unixd_module modules/mod_unixd.so +LoadModule access_compat_module modules/mod_access_compat.so +LoadModule auth_basic_module modules/mod_auth_basic.so +LoadModule authz_core_module modules/mod_authz_core.so +LoadModule authz_user_module modules/mod_authz_user.so +LoadModule authz_host_module modules/mod_authz_host.so +LoadModule authn_core_module modules/mod_authn_core.so +LoadModule authn_file_module modules/mod_authn_file.so +LoadModule mime_module modules/mod_mime.so +LoadModule cgid_module modules/mod_cgid.so +LoadModule ssl_module modules/mod_ssl.so +LoadModule alias_module modules/mod_alias.so +LoadModule env_module modules/mod_env.so +LoadModule rewrite_module modules/mod_rewrite.so +LoadModule log_config_module modules/mod_log_config.so +LoadModule dav_module modules/mod_dav.so +LoadModule dav_fs_module modules/mod_dav_fs.so +LoadModule dir_module modules/mod_dir.so + +ErrorLog "{{ parameter.path_error_log }}" +LogFormat "%h %l %u %t \"%r\" %>s %b" common +CustomLog "{{ parameter.path_access_log }}" common + +# SSL Configuration +Define SSLConfigured +SSLCertificateFile {{ parameter.cert_file }} +SSLCertificateKeyFile {{ parameter.key_file }} +SSLRandomSeed startup builtin +SSLRandomSeed connect builtin +SSLRandomSeed startup /dev/urandom 256 +SSLRandomSeed connect builtin +SSLProtocol -ALL +SSLv3 +TLSv1 +SSLHonorCipherOrder On +SSLCipherSuite RC4-SHA:HIGH:!ADH +SSLEngine On + + +DavLockDB {{ parameter.dav_lock }} +Alias /share {{ parameter.dav_home }} +<Directory {{ parameter.dav_home }}> + DirectoryIndex disabled + DAV On + Options Indexes FollowSymLinks + AuthType Basic + AuthName "webdav" + AuthUserFile "{{ parameter.htpassword_path }}" + Require valid-user +</Directory> + +Alias / {{ parameter.document_root }}/ +<Directory {{ parameter.document_root }}/> + Order Allow,Deny + Allow from all + Satisfy Any + DirectoryIndex index.html + Options FollowSymLinks +</Directory> \ No newline at end of file diff --git a/software/html5ide/template/wrapper.in b/software/html5ide/template/wrapper.in new file mode 100644 index 0000000000000000000000000000000000000000..6fa3fa47bee33d0636478a1b203e0cb1246377b7 --- /dev/null +++ b/software/html5ide/template/wrapper.in @@ -0,0 +1,2 @@ +#!${dash-output:dash} +{{ content }} \ No newline at end of file diff --git a/software/nginx-push-stream/instance-nginx.cfg.in b/software/nginx-push-stream/instance-nginx.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..8fd70153b2bc0ed73c57b592bdda2a5b6d243712 --- /dev/null +++ b/software/nginx-push-stream/instance-nginx.cfg.in @@ -0,0 +1,57 @@ +{% set part_list = [] -%} +{% set crontab_line_list = [] -%} +[directory] +recipe = slapos.cookbook:mkdirectory +etc = $${buildout:directory}/etc +bin = $${buildout:directory}/bin +srv = $${buildout:directory}/srv +var = $${buildout:directory}/var +run = $${:var}/run +log = $${:var}/log +varnginx = $${:var}/nginx +services = $${:etc}/service +cron-entries = $${:etc}/cron.d +www = $${:srv}/www +ssl = $${:etc}/ssl + +################################# +# Nginx service +################################# +[nginx-service] +recipe = slapos.recipe.template +url = ${template-nginx-service:output} +output = $${directory:services}/nginx +mode = 0700 +virtual-depends = + $${nginx-configuration:ip} + +[nginx-configuration] +recipe = slapos.recipe.template +url = ${template-nginx-configuration:output} +output = $${directory:etc}/nginx.cfg +mode = 0600 +access_log = $${directory:log}/nginx-access.log +error_log = $${directory:log}/nginx-error.log +ip = $${slap-network-information:global-ipv6} +local_ip = $${slap-network-information:local-ipv4} +port = 9443 +publisher_push_stream_store_messages = off +publisher_client_max_body_size = 16k +publisher_client_body_buffer_size = 16k +subscriber_allow_origin = '*' +# Prevent to use credential if origin is star +subscriber_allow_credential = 'false' +subscriber_allow_methods = 'GET, HEAD, OPTIONS' +subscriber_allow_headers = 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since' + +# Add parts generated by template +[buildout] +parts = + nginx-service +{% for part in part_list -%} +{{ ' %s' % part }} +{% endfor -%} + +eggs-directory = ${buildout:eggs-directory} +develop-eggs-directory = ${buildout:develop-eggs-directory} +offline = true diff --git a/software/nginx-push-stream/instance.cfg.in b/software/nginx-push-stream/instance.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..fdd64a5f15bf054692f5d2179ebb9c6c9409ac20 --- /dev/null +++ b/software/nginx-push-stream/instance.cfg.in @@ -0,0 +1,36 @@ +[buildout] +parts = + switch-softwaretype + +eggs-directory = ${buildout:eggs-directory} +develop-eggs-directory = ${buildout:develop-eggs-directory} +offline = true + +[switch-softwaretype] +recipe = slapos.cookbook:softwaretype +default = $${:nginx} +nginx = $${dynamic-template-nginx:rendered} + +[dynamic-template-nginx] +recipe = slapos.recipe.template:jinja2 +template = ${template-nginx:output} +rendered = $${buildout:parts-directory}/$${:_buildout_section_name_}/$${:filename} +filename = instance-nginx.cfg + +[slap-connection] +computer-id = $${slap_connection:computer_id} +partition-id = $${slap_connection:partition_id} +server-url = $${slap_connection:server_url} +software-release-url = $${slap_connection:software_release_url} +key-file = $${slap_connection:key_file} +cert-file = $${slap_connection:cert_file} + +[instance-parameter] +# Fetches parameters defined in SlapOS Master for this instance. +# Always the same. +recipe = slapos.cookbook:slapconfiguration.serialised +computer = $${slap_connection:computer_id} +partition = $${slap_connection:partition_id} +url = $${slap_connection:server_url} +key = $${slap_connection:key_file} +cert = $${slap_connection:cert_file} diff --git a/software/nginx-push-stream/software.cfg b/software/nginx-push-stream/software.cfg new file mode 100644 index 0000000000000000000000000000000000000000..b2d39997f06dbb85d45bb3ffd418bceafe1ae675 --- /dev/null +++ b/software/nginx-push-stream/software.cfg @@ -0,0 +1,223 @@ +[buildout] + +extends = + ../../stack/slapos.cfg + ../../component/dash/buildout.cfg + ../../component/nginx/buildout.cfg + +parts = + slapos-cookbook + nginx-push-stream-module + nginx-push-stream + template + template-nginx-service + template-nginx + +[template] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/instance.cfg.in +md5sum = eb4c69df9a8dbb94fb76d0a6c11e360f +output = ${buildout:directory}/template.cfg +mode = 0644 + +[template-nginx-service] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/template-nginx-service.sh.in +md5sum = 90ba5a247c489261d3304528cba56e06 +output = ${buildout:directory}/template-nginx-service.sh.in +mode = 0644 + +[template-nginx-configuration] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/template-nginx.cfg.in +md5sum = 58cd6a59089ab93bb3ec0f5d12888500 +output = ${buildout:directory}/template-nginx.cfg.in +mode = 0644 + +[template-nginx] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/instance-nginx.cfg.in +md5sum = 478085d0abb0ffc0b259c038bddae205 +output = ${buildout:directory}/instance-nginx.cfg.in +mode = 0644 + +[networkcache] +# signature certificates of the following uploaders. +# Romain Courteaud +# Sebastien Robin +# Kazuhiko Shiozaki +# Cedric de Saint Martin +# Yingjie Xu +# Gabriel Monnerat +# 脜聛ukasz Nowak +# Test Agent (Automatic update from tests) +# Aur脙漏lien Calonne +signature-certificate-list = + -----BEGIN CERTIFICATE----- + MIIB4DCCAUkCADANBgkqhkiG9w0BAQsFADA5MQswCQYDVQQGEwJGUjEZMBcGA1UE + CBMQRGVmYXVsdCBQcm92aW5jZTEPMA0GA1UEChMGTmV4ZWRpMB4XDTExMDkxNTA5 + MDAwMloXDTEyMDkxNTA5MDAwMlowOTELMAkGA1UEBhMCRlIxGTAXBgNVBAgTEERl + ZmF1bHQgUHJvdmluY2UxDzANBgNVBAoTBk5leGVkaTCBnzANBgkqhkiG9w0BAQEF + AAOBjQAwgYkCgYEApYZv6OstoqNzxG1KI6iE5U4Ts2Xx9lgLeUGAMyfJLyMmRLhw + boKOyJ9Xke4dncoBAyNPokUR6iWOcnPHtMvNOsBFZ2f7VA28em3+E1JRYdeNUEtX + Z0s3HjcouaNAnPfjFTXHYj4um1wOw2cURSPuU5dpzKBbV+/QCb5DLheynisCAwEA + ATANBgkqhkiG9w0BAQsFAAOBgQBCZLbTVdrw3RZlVVMFezSHrhBYKAukTwZrNmJX + mHqi2tN8tNo6FX+wmxUUAf3e8R2Ymbdbn2bfbPpcKQ2fG7PuKGvhwMG3BlF9paEC + q7jdfWO18Zp/BG7tagz0jmmC4y/8akzHsVlruo2+2du2freE8dK746uoMlXlP93g + QUUGLQ== + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIB8jCCAVugAwIBAgIJAPu2zchZ2BxoMA0GCSqGSIb3DQEBBQUAMBIxEDAOBgNV + BAMMB3RzeGRldjMwHhcNMTExMDE0MTIxNjIzWhcNMTIxMDEzMTIxNjIzWjASMRAw + DgYDVQQDDAd0c3hkZXYzMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCrPbh+ + YGmo6mWmhVb1vTqX0BbeU0jCTB8TK3i6ep3tzSw2rkUGSx3niXn9LNTFNcIn3MZN + XHqbb4AS2Zxyk/2tr3939qqOrS4YRCtXBwTCuFY6r+a7pZsjiTNddPsEhuj4lEnR + L8Ax5mmzoi9nE+hiPSwqjRwWRU1+182rzXmN4QIDAQABo1AwTjAdBgNVHQ4EFgQU + /4XXREzqBbBNJvX5gU8tLWxZaeQwHwYDVR0jBBgwFoAU/4XXREzqBbBNJvX5gU8t + LWxZaeQwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQA07q/rKoE7fAda + FED57/SR00OvY9wLlFEF2QJ5OLu+O33YUXDDbGpfUSF9R8l0g9dix1JbWK9nQ6Yd + R/KCo6D0sw0ZgeQv1aUXbl/xJ9k4jlTxmWbPeiiPZEqU1W9wN5lkGuLxV4CEGTKU + hJA/yXa1wbwIPGvX3tVKdOEWPRXZLg== + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIB7jCCAVegAwIBAgIJAJWA0jQ4o9DGMA0GCSqGSIb3DQEBBQUAMA8xDTALBgNV + BAMMBHg2MXMwIBcNMTExMTI0MTAyNDQzWhgPMjExMTEwMzExMDI0NDNaMA8xDTAL + BgNVBAMMBHg2MXMwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANdJNiFsRlkH + vq2kHP2zdxEyzPAWZH3CQ3Myb3F8hERXTIFSUqntPXDKXDb7Y/laqjMXdj+vptKk + 3Q36J+8VnJbSwjGwmEG6tym9qMSGIPPNw1JXY1R29eF3o4aj21o7DHAkhuNc5Tso + 67fUSKgvyVnyH4G6ShQUAtghPaAwS0KvAgMBAAGjUDBOMB0GA1UdDgQWBBSjxFUE + RfnTvABRLAa34Ytkhz5vPzAfBgNVHSMEGDAWgBSjxFUERfnTvABRLAa34Ytkhz5v + PzAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4GBAFLDS7zNhlrQYSQO5KIj + z2RJe3fj4rLPklo3TmP5KLvendG+LErE2cbKPqnhQ2oVoj6u9tWVwo/g03PMrrnL + KrDm39slYD/1KoE5kB4l/p6KVOdeJ4I6xcgu9rnkqqHzDwI4v7e8/D3WZbpiFUsY + vaZhjNYKWQf79l6zXfOvphzJ + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIB9jCCAV+gAwIBAgIJAO4V/jiMoICoMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV + BAMMCENPTVAtMjMyMCAXDTEyMDIxNjExMTAyM1oYDzIxMTIwMTIzMTExMDIzWjAT + MREwDwYDVQQDDAhDT01QLTIzMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA + wi/3Z8W9pUiegUXIk/AiFDQ0UJ4JFAwjqr+HSRUirlUsHHT+8DzH/hfcTDX1I5BB + D1ADk+ydXjMm3OZrQcXjn29OUfM5C+g+oqeMnYQImN0DDQIOcUyr7AJc4xhvuXQ1 + P2pJ5NOd3tbd0kexETa1LVhR6EgBC25LyRBRae76qosCAwEAAaNQME4wHQYDVR0O + BBYEFMDmW9aFy1sKTfCpcRkYnP6zUd1cMB8GA1UdIwQYMBaAFMDmW9aFy1sKTfCp + cRkYnP6zUd1cMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAskbFizHr + b6d3iIyN+wffxz/V9epbKIZVEGJd/6LrTdLiUfJPec7FaxVCWNyKBlCpINBM7cEV + Gn9t8mdVQflNqOlAMkOlUv1ZugCt9rXYQOV7rrEYJBWirn43BOMn9Flp2nibblby + If1a2ZoqHRxoNo2yTmm7TSYRORWVS+vvfjY= + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIB9jCCAV+gAwIBAgIJAIlBksrZVkK8MA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV + BAMMCENPTVAtMzU3MCAXDTEyMDEyNjEwNTUyOFoYDzIxMTIwMTAyMTA1NTI4WjAT + MREwDwYDVQQDDAhDT01QLTM1NzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA + ts+iGUwi44vtIfwXR8DCnLtHV4ydl0YTK2joJflj0/Ws7mz5BYkxIU4fea/6+VF3 + i11nwBgYgxQyjNztgc9u9O71k1W5tU95yO7U7bFdYd5uxYA9/22fjObaTQoC4Nc9 + mTu6r/VHyJ1yRsunBZXvnk/XaKp7gGE9vNEyJvPn2bkCAwEAAaNQME4wHQYDVR0O + BBYEFKuGIYu8+6aEkTVg62BRYaD11PILMB8GA1UdIwQYMBaAFKuGIYu8+6aEkTVg + 62BRYaD11PILMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAMoTRpBxK + YLEZJbofF7gSrRIcrlUJYXfTfw1QUBOKkGFFDsiJpEg4y5pUk1s5Jq9K3SDzNq/W + it1oYjOhuGg3al8OOeKFrU6nvNTF1BAvJCl0tr3POai5yXyN5jlK/zPfypmQYxE+ + TaqQSGBJPVXYt6lrq/PRD9ciZgKLOwEqK8w= + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIB9jCCAV+gAwIBAgIJAPHoWu90gbsgMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV + BAMMCXZpZmlibm9kZTAeFw0xMjAzMTkyMzIwNTVaFw0xMzAzMTkyMzIwNTVaMBQx + EjAQBgNVBAMMCXZpZmlibm9kZTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA + ozBijpO8PS5RTeKTzA90vi9ezvv4vVjNaguqT4UwP9+O1+i6yq1Y2W5zZxw/Klbn + oudyNzie3/wqs9VfPmcyU9ajFzBv/Tobm3obmOqBN0GSYs5fyGw+O9G3//6ZEhf0 + NinwdKmrRX+d0P5bHewadZWIvlmOupcnVJmkks852BECAwEAAaNQME4wHQYDVR0O + BBYEFF9EtgfZZs8L2ZxBJxSiY6eTsTEwMB8GA1UdIwQYMBaAFF9EtgfZZs8L2ZxB + JxSiY6eTsTEwMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAc43YTfc6 + baSemaMAc/jz8LNLhRE5dLfLOcRSoHda8y0lOrfe4lHT6yP5l8uyWAzLW+g6s3DA + Yme/bhX0g51BmI6gjKJo5DoPtiXk/Y9lxwD3p7PWi+RhN+AZQ5rpo8UfwnnN059n + yDuimQfvJjBFMVrdn9iP6SfMjxKaGk6gVmI= + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIB9jCCAV+gAwIBAgIJAMNZBmoIOXPBMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV + BAMMCENPTVAtMTMyMCAXDTEyMDUwMjEyMDQyNloYDzIxMTIwNDA4MTIwNDI2WjAT + MREwDwYDVQQDDAhDT01QLTEzMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA + 6peZQt1sAmMAmSG9BVxxcXm8x15kE9iAplmANYNQ7z2YO57c10jDtlYlwVfi/rct + xNUOKQtc8UQtV/fJWP0QT0GITdRz5X/TkWiojiFgkopza9/b1hXs5rltYByUGLhg + 7JZ9dZGBihzPfn6U8ESAKiJzQP8Hyz/o81FPfuHCftsCAwEAAaNQME4wHQYDVR0O + BBYEFNuxsc77Z6/JSKPoyloHNm9zF9yqMB8GA1UdIwQYMBaAFNuxsc77Z6/JSKPo + yloHNm9zF9yqMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAl4hBaJy1 + cgiNV2+Z5oNTrHgmzWvSY4duECOTBxeuIOnhql3vLlaQmo0p8Z4c13kTZq2s3nhd + Loe5mIHsjRVKvzB6SvIaFUYq/EzmHnqNdpIGkT/Mj7r/iUs61btTcGUCLsUiUeci + Vd0Ozh79JSRpkrdI8R/NRQ2XPHAo+29TT70= + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIB9jCCAV+gAwIBAgIJAKRvzcy7OH0UMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV + BAMMCENPTVAtNzcyMCAXDTEyMDgxMDE1NDI1MVoYDzIxMTIwNzE3MTU0MjUxWjAT + MREwDwYDVQQDDAhDT01QLTc3MjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA + o7aipd6MbnuGDeR1UJUjuMLQUariAyQ2l2ZDS6TfOwjHiPw/mhzkielgk73kqN7A + sUREx41eTcYCXzTq3WP3xCLE4LxLg1eIhd4nwNHj8H18xR9aP0AGjo4UFl5BOMa1 + mwoyBt3VtfGtUmb8whpeJgHhqrPPxLoON+i6fIbXDaUCAwEAAaNQME4wHQYDVR0O + BBYEFEfjy3OopT2lOksKmKBNHTJE2hFlMB8GA1UdIwQYMBaAFEfjy3OopT2lOksK + mKBNHTJE2hFlMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAaNRx6YN2 + M/p3R8/xS6zvH1EqJ3FFD7XeAQ52WuQnKSREzuw0dsw12ClxjcHiQEFioyTiTtjs + 5pW18Ry5Ie7iFK4cQMerZwWPxBodEbAteYlRsI6kePV7Gf735Y1RpuN8qZ2sYL6e + x2IMeSwJ82BpdEI5niXxB+iT0HxhmR+XaMI= + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIB+DCCAWGgAwIBAgIJAKGd0vpks6T/MA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV + BAMMCUNPTVAtMTU4NDAgFw0xMzA2MjAxMjE5MjBaGA8yMTEzMDUyNzEyMTkyMFow + FDESMBAGA1UEAwwJQ09NUC0xNTg0MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB + gQDZTH9etPUC+wMZQ3UIiOwyyCfHsJ+7duCFYjuo1uZrhtDt/fp8qb8qK9ob+df3 + EEYgA0IgI2j/9jNUEnKbc5+OrfKznzXjrlrH7zU8lKBVNCLzQuqBKRNajZ+UvO8R + nlqK2jZCXP/p3HXDYUTEwIR5W3tVCEn/Vda4upTLcPVE5wIDAQABo1AwTjAdBgNV + HQ4EFgQU7KXaNDheQWoy5uOU01tn1M5vNkEwHwYDVR0jBBgwFoAU7KXaNDheQWoy + 5uOU01tn1M5vNkEwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQASmqCU + Znbvu6izdicvjuE3aKnBa7G++Fdp2bdne5VCwVbVLYCQWatB+n4crKqGdnVply/u + +uZ16u1DbO9rYoKgWqjLk1GfiLw5v86pd5+wZd5I9QJ0/Sbz2vZk5S4ciMIGwArc + m711+GzlW5xe6GyH9SZaGOPAdUbI6JTDwLzEgA== + -----END CERTIFICATE----- + +[versions] +Jinja2 = 2.7.3 +MarkupSafe = 0.23 +Werkzeug = 0.9.6 +buildout-versions = 1.7 +cffi = 0.8.6 +cliff = 1.6.1 +cmd2 = 0.6.7 +cryptography = 0.5.1 +inotifyx = 0.2.0-1 +itsdangerous = 0.24 +lxml = 3.3.5 +meld3 = 1.0.0 +mr.developer = 1.30 +netaddr = 0.7.12 +netifaces = 0.10.4 +plone.recipe.command = 1.1 +pyOpenSSL = 0.14 +pyparsing = 2.0.2 +pytz = 2014.4 +requests = 2.3.0 +six = 1.7.3 +slapos.cookbook = 0.87 +slapos.core = 1.1.2 +slapos.recipe.build = 0.12 +slapos.recipe.cmmi = 0.2 +slapos.recipe.template = 2.5 +stevedore = 0.15 +supervisor = 3.0 +xml-marshaller = 0.9.7 + +# Required by: +# slapos.core==1.1.2 +Flask = 0.10.1 + +# Required by: +# slapos.cookbook==0.87 +lock-file = 2.0 + +# Required by: +# slapos.core==1.1.2 +psutil = 2.1.1 + +# Required by: +# cffi==0.8.6 +pycparser = 2.10 + +# Required by: +# slapos.core==1.1.2 +zope.interface = 4.1.1 diff --git a/software/nginx-push-stream/template-nginx-service.sh.in b/software/nginx-push-stream/template-nginx-service.sh.in new file mode 100644 index 0000000000000000000000000000000000000000..6737e2eaf1462d17e18222b9db78e73314b8d026 --- /dev/null +++ b/software/nginx-push-stream/template-nginx-service.sh.in @@ -0,0 +1,6 @@ +#!${dash-output:dash} +# BEWARE: This file is operated by slapgrid +# BEWARE: It will be overwritten automatically + +exec ${nginx-push-stream-output:nginx} \ + -c $${nginx-configuration:output} diff --git a/software/nginx-push-stream/template-nginx.cfg.in b/software/nginx-push-stream/template-nginx.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..e682f253d1bc27c9d5e7cc5317f5d38a92ebe00f --- /dev/null +++ b/software/nginx-push-stream/template-nginx.cfg.in @@ -0,0 +1,107 @@ +daemon off; # run in the foreground so supervisord can look after it + +worker_processes 4; +pid $${directory:run}/nginx.pid; + +events { + worker_connections 768; + # multi_accept on; +} + +error_log $${nginx-configuration:error_log}; + +http { + + ## + # Basic Settings + ## + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + server_tokens off; + + # server_names_hash_bucket_size 64; + # server_name_in_redirect off; + + default_type application/octet-stream; + include ${nginx-push-stream-output:mime}; + + ## + # Logging Settings + ## + + access_log $${nginx-configuration:access_log}; + error_log $${nginx-configuration:error_log}; + + ## + # Gzip Settings + ## + + gzip on; + gzip_disable "msie6"; + + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_buffers 16 8k; + gzip_http_version 1.1; + gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; + ## + # Push stream Settings + ## + push_stream_shared_memory_size 32m; + +server { + listen [$${nginx-configuration:ip}]:$${nginx-configuration:port}; + listen $${nginx-configuration:local_ip}:$${nginx-configuration:port}; + + fastcgi_temp_path $${directory:varnginx} 1 2; + uwsgi_temp_path $${directory:varnginx} 1 2; + scgi_temp_path $${directory:varnginx} 1 2; + + client_body_temp_path $${directory:varnginx} 1 2; + proxy_temp_path $${directory:varnginx} 1 2; + + ## Serve an error 204 (No Content) for favicon.ico + location = /favicon.ico { + return 204; + } + + location /pub { + push_stream_publisher; + + push_stream_channels_path $arg_id; + # store messages in memory + push_stream_store_messages $${nginx-configuration:publisher_push_stream_store_messages}; + + # Message size limit + # client_max_body_size MUST be equal to client_body_buffer_size or + # you will be sorry. + client_max_body_size $${nginx-configuration:publisher_client_max_body_size}; + client_body_buffer_size $${nginx-configuration:publisher_client_body_buffer_size}; + + } + + location ~ /sub/(.*) { + # activate subscriber mode for this location + add_header "Access-Control-Allow-Origin" $${nginx-configuration:subscriber_allow_origin}; + add_header 'Access-Control-Allow-Credentials' $${nginx-configuration:subscriber_allow_credential}; + add_header 'Access-Control-Allow-Methods' $${nginx-configuration:subscriber_allow_methods}; + add_header 'Access-Control-Allow-Headers' $${nginx-configuration:subscriber_allow_headers}; + + push_stream_subscriber eventsource; + # positional channel path + push_stream_channels_path $1; + + # content-type + default_type "text/event-stream; charset=utf-8"; + } + + location / { + root $${directory:www}; + } +} +} diff --git a/software/slaprunner/common.cfg b/software/slaprunner/common.cfg index 26aedfbfae0d4c635cec2babd9e6e57353a4f0fc..3c24b353756d9e0a65de3b74d6e48c9120ecd874 100644 --- a/software/slaprunner/common.cfg +++ b/software/slaprunner/common.cfg @@ -44,27 +44,43 @@ mode = 0644 recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance-runner.cfg output = ${buildout:directory}/template-runner.cfg.in -md5sum = b30d71f2f0abfecc2638fe6b804f3c43 +md5sum = 1c084d9aa33202ccba58be750f0698a5 +mode = 0644 + +[template-runner-import-script] +recipe = hexagonit.recipe.download +url = ${:_profile_base_location_}/template/runner-import.sh.jinja2 +download-only = true +md5sum = b003f6a6963f7787948be85c3d479ccd +filename = runner-import.sh.jinja2 mode = 0644 [instance-runner-import] recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance-runner-import.cfg.in output = ${buildout:directory}/instance-runner-import.cfg -md5sum = 202c40bcca67d70a3c79d218be853b10 +md5sum = 354ff83819e8379f5e4a2e2985e34e9e +mode = 0644 + +[template-runner-export-script] +recipe = hexagonit.recipe.download +url = ${:_profile_base_location_}/template/runner-export.sh.jinja2 +download-only = true +md5sum = ea478351883e6e716c044771f69a09ac +filename = runner-export.sh.jinja2 mode = 0644 [instance-runner-export] recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance-runner-export.cfg.in output = ${buildout:directory}/instance-runner-export.cfg -md5sum = a8815402e2d65c0a9cda53492b70d175 +md5sum = cf43eb1d6e9eea545858a86ea790a1cb mode = 0644 [template-resilient] recipe = slapos.recipe.download url = ${:_profile_base_location_}/instance-resilient.cfg.jinja2 -md5sum = e5c7e445d0103b5e73aedb43fac681dd +md5sum = ab564b9a9de3e44884adf072fc740d23 filename = instance-resilient.cfg.jinja2 mode = 0644 diff --git a/software/slaprunner/instance-resilient.cfg.jinja2 b/software/slaprunner/instance-resilient.cfg.jinja2 index bd68b1cdb90ecc5f5e3b06d499213de4d346b68b..d3863723ecc58e585a1e7e0a40cf29645d63adef 100644 --- a/software/slaprunner/instance-resilient.cfg.jinja2 +++ b/software/slaprunner/instance-resilient.cfg.jinja2 @@ -23,7 +23,7 @@ parts += # Bubble up the parameters [request-runner] -return = url ssh-public-key ssh-url notification-id ip backend_url url ssh_command access_url 1_info 2_info monitor_url IMPORTANT_monitor_info webdav_url public_url +return = url ssh-public-key ssh-url notification-id ip backend_url url ssh_command access_url 1_info 2_info monitor_url webdav_url public_url [publish-connection-informations] recipe = slapos.cookbook:publish @@ -34,7 +34,6 @@ access_url = ${request-runner:connection-access_url} url = ${request-runner:connection-url} ssh_command = ${request-runner:connection-ssh_command} monitor_url = ${request-runner:connection-monitor_url} -IMPORTANT_monitor_info = ${request-runner:connection-IMPORTANT_monitor_info} webdav_url = ${request-runner:connection-webdav_url} public_url = ${request-runner:connection-public_url} diff --git a/software/slaprunner/instance-runner-export.cfg.in b/software/slaprunner/instance-runner-export.cfg.in index 0933548eed78cfdd3ce7d7969c928eaa8bc2bee1..a297b44393f10fac209ab584122dfd5c380be974 100644 --- a/software/slaprunner/instance-runner-export.cfg.in +++ b/software/slaprunner/instance-runner-export.cfg.in @@ -26,40 +26,54 @@ parts += cron-entry-backup cron-entry-prepare-software deploy-instance-parameters +## Monitoring part ###Parts to add for monitoring - slap-parameters - certificate-authority cron + certificate-authority cron-entry-monitor cron-entry-rss deploy-index - deploy-index-template - deploy-monitor-script - deploy-rss-script deploy-settings-cgi deploy-status-cgi - make-rss - monitor-frontend - monitor-promise + deploy-status-history-cgi setup-static-files certificate-authority - public zero-parameters - cgi-httpd-wrapper public-symlink + cgi-httpd-wrapper + cgi-httpd-graceful-wrapper + monitor-promise + monitor-instance-log-access +## Monitor for runner + monitor-current-log-access + [exporter] -recipe = slapos.cookbook:slaprunner.export -wrapper = $${directory:bin}/$${slap-parameter:namebase}-exporter -srv-directory = $${directory:srv} -etc-directory = $${directory:etc} -backup-directory = $${directory:backup} -shell-binary = ${dash:location}/bin/dash -rsync-binary = ${rsync:location}/bin/rsync +recipe = slapos.recipe.template:jinja2 +template = ${template-runner-export-script:location}/${template-runner-export-script:filename} +rendered = $${directory:bin}/$${slap-parameter:namebase}-exporter +# backward compatibility for resilient stack +wrapper = $${:rendered} +mode = 700 +context = + section directory directory + raw shell_binary ${dash:location}/bin/dash + raw rsync_binary ${rsync:location}/bin/rsync [monitor-promise] -url = $${monitor-frontend:config-url}/$${deploy-index:filename} +url = $${monitor-frontend:config-url}/$${deploy-index-template:filename} # Extends publish section with resilient parameters [publish-connection-informations] <= resilient-publish-connection-parameter + +[monitor-check-resilient-feed-file] +recipe = slapos.recipe.template:jinja2 +template = ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename} +rendered = $${monitor-directory:monitor-custom-scripts}/check-create-resilient-feed-files.py +mode = 700 +context = + key input_feed_directory directory:notifier-feeds + key monitor_feed_directory monitor-directory:public-cgi + raw base_url http://[$${notifier:host}]:$${notifier:port}/get/ + raw python_executable ${buildout:executable} diff --git a/software/slaprunner/instance-runner-import.cfg.in b/software/slaprunner/instance-runner-import.cfg.in index b1c36e96c7f4f89184829291d69049cc4358452e..d37e2e813895260c82fa6767513b415205adf8e2 100644 --- a/software/slaprunner/instance-runner-import.cfg.in +++ b/software/slaprunner/instance-runner-import.cfg.in @@ -23,39 +23,63 @@ parts += slapos-repo-config cron-entry-prepare-software deploy-instance-parameters + instance-software-type # have to repeat the next one, as it's not inherited from pbsready-import import-on-notification +## Monitoring part ###Parts to add for monitoring - slap-parameters - certificate-authority cron + certificate-authority cron-entry-monitor cron-entry-rss deploy-index - deploy-index-template - deploy-monitor-script - deploy-rss-script deploy-settings-cgi deploy-status-cgi - make-rss - monitor-promise + deploy-status-history-cgi setup-static-files certificate-authority - public zero-parameters - cgi-httpd-wrapper public-symlink + cgi-httpd-wrapper + cgi-httpd-graceful-wrapper + monitor-promise + monitor-instance-log-access +## Monitor for runner + monitor-current-log-access + monitor-backup-log-access +## Monitor for import runner + monitor-latest-restored-backup [importer] -recipe = slapos.cookbook:slaprunner.import -wrapper = $${directory:bin}/$${slap-parameter:namebase}-importer -srv-directory = $${directory:srv} -etc-directory = $${directory:etc} -backup-directory = $${directory:backup} -shell-binary = ${dash:location}/bin/dash -rsync-binary = ${rsync:location}/bin/rsync -curl-binary = ${curl:location}/bin/curl -backend-url = $${slaprunner:access-url} +recipe = slapos.recipe.template:jinja2 +template = ${template-runner-import-script:location}/${template-runner-import-script:filename} +rendered = $${directory:bin}/$${slap-parameter:namebase}-importer +# backward compatibility for resilient stack +wrapper = $${:rendered} +mode = 700 +context = + key backend_url slaprunner:access-url + section directory directory + raw shell_binary ${dash:location}/bin/dash + raw rsync_binary ${rsync:location}/bin/rsync + raw curl_binary ${curl:location}/bin/curl [slap-parameter] auto-deploy-instance = false +auto-deploy = true + +[resilient-publish-connection-parameter] +monitor-url = $${monitor-parameters:url} + +[monitor-backup-log-access] +< = monitor-directory-access +source = $${directory:logrotate-backup} + +[monitor-latest-restored-backup] +recipe = slapos.recipe.template:jinja2 +command = if [ -f $${directory:etc}/.resilient-timestamp ]; then echo "$(date -d @$(cat $${directory:etc}/.resilient-timestamp) +%c)"; else echo "No backup timestamp found"; fi +rendered = $${monitor-directory:monitoring-cgi}/latest-restored-backup +template = ${template-wrapper:output} +mode = 744 +context = + key content :command \ No newline at end of file diff --git a/software/slaprunner/instance-runner.cfg b/software/slaprunner/instance-runner.cfg index 3760640a445db50d338dcf957f03a30b99355db4..fc661a638a63f299ea66b67acb6bcf9565c46f6a 100644 --- a/software/slaprunner/instance-runner.cfg +++ b/software/slaprunner/instance-runner.cfg @@ -22,29 +22,30 @@ parts = slapos-repo-config cron-entry-prepare-software deploy-instance-parameters + instance-software-type {% if slapparameter_dict.get('custom-frontend-backend-url') %} custom-frontend-promise {% endif %} +## Monitoring part ###Parts to add for monitoring - slap-parameters - certificate-authority cron + certificate-authority cron-entry-monitor cron-entry-rss deploy-index - deploy-index-template - deploy-monitor-script - deploy-rss-script deploy-settings-cgi deploy-status-cgi - make-rss - monitor-promise + deploy-status-history-cgi setup-static-files certificate-authority - public zero-parameters - cgi-httpd-wrapper public-symlink + cgi-httpd-wrapper + cgi-httpd-graceful-wrapper + monitor-promise + monitor-instance-log-access +## Monitor for runner + monitor-current-log-access extends = ${monitor-template:output} @@ -309,7 +310,6 @@ working_directory = $${slaprunner:working-directory} dav_lock = $${directory:var}/DavLock etc_dir = $${directory:etc} var_dir = $${directory:var} -document_root = $${directory:www} project_folder = $${directory:project} runner_home = $${runnerdirectory:home} git_http_backend = ${git:location}/libexec/git-core/git-http-backend @@ -480,6 +480,7 @@ debug = false frontend-domain = slapos-repository = http://git.erp5.org/repos/slapos.git slapos-software = +slapos-software-type = slapos-reference = master auto-deploy = false auto-deploy-instance = true @@ -575,6 +576,13 @@ stop-on-error = true parameter-xml = $${directory:etc}/.parameter.xml command = if [ ! -f $${:parameter-xml} ]; then cp $${instance-parameters:rendered} $${:parameter-xml}; fi +[instance-software-type] +recipe = plone.recipe.command +stop-on-error = true +# XXX It should not be named with .xml as it is not xml +software-type-path = $${directory:etc}/.software_type.xml +command = if [ ! -f $${:software-type-path} -a "$${slap-parameter:slapos-software-type}" != "" ]; then echo "$${slap-parameter:slapos-software-type}" > $${:software-type-path}; fi + [slap-configuration] recipe = slapos.cookbook:slapconfiguration.serialised computer = $${slap-connection:computer-id} @@ -588,3 +596,8 @@ shell-password = $${shellinabox-code:passwd} recovery-code = $${recovery-code:passwd} [zero-parameters] + + +[monitor-current-log-access] +< = monitor-directory-access +source = $${directory:log} diff --git a/software/slaprunner/schema.json b/software/slaprunner/schema.json index b40fe23247213df21b9eb1231befbab02f9ac3b1..ee164c51ce03f8f1677318f401935a3c505d50ab 100644 --- a/software/slaprunner/schema.json +++ b/software/slaprunner/schema.json @@ -38,6 +38,10 @@ "type": "boolean", "default": "false" }, + "slapos-software-type": { + "description": "Software type of your instance inside the runner", + "type": "string" + }, "parameter-*": { "description": "'*' is a parameter which will be used to configure the instance inside the runner.", "type": "string" diff --git a/software/slaprunner/software.cfg b/software/slaprunner/software.cfg index eb3a7c23ab0a21cab69862f148b5b7d41fb8fa07..2cdb9eff3ef48158e756642eb0bbdba0f80248ab 100644 --- a/software/slaprunner/software.cfg +++ b/software/slaprunner/software.cfg @@ -194,10 +194,11 @@ collective.recipe.template = 1.10 erp5.util = 0.4.36 gitdb = 0.5.4 hexagonit.recipe.cmmi = 1.6 -inotifyx = 0.2.0-1 +inotifyx = 0.2.0 lock-file = 2.0 lxml = 3.0.1 meld3 = 0.6.10 +mr.developer = 1.30 netaddr = 0.7.10 plone.recipe.command = 1.1 prettytable = 0.7.2 @@ -212,7 +213,7 @@ slapos.recipe.build = 0.11.5 slapos.recipe.cmmi = 0.2 slapos.recipe.download = 1.0.dev-r4053 slapos.recipe.template = 2.4.2 -slapos.toolbox = 0.39.2 +slapos.toolbox = 0.39.3 smmap = 0.8.2 xml-marshaller = 0.9.7 z3c.recipe.scripts = 1.0.1 diff --git a/software/slaprunner/template/runner-export.sh.jinja2 b/software/slaprunner/template/runner-export.sh.jinja2 new file mode 100644 index 0000000000000000000000000000000000000000..adc1e0ccdf790fc6f54d1b2b3ab0d509e757b76f --- /dev/null +++ b/software/slaprunner/template/runner-export.sh.jinja2 @@ -0,0 +1,23 @@ +#!{{ shell_binary }} +umask 077 +sync_element () { + path=$1 + backup_path=$2 + shift 2 + element_list=$* + for element in $element_list + do + cd $path; + if [ -f $element ] || [ -d $element ]; then + {{ rsync_binary }} -avz --safe-links --delete $element $backup_path; + fi + done +} +sync_element {{ directory['srv'] }}/runner {{ directory['backup'] }}/runner/ instance project proxy.db +# We sync .* appart +date +%s -u > {{ directory['etc'] }}/.resilient-timestamp +cp -r {{ directory['etc'] }}/.??* {{ directory['backup'] }}/etc/ +sync_element {{ directory['etc'] }} {{ directory['backup'] }}/etc/ config.json ssh +if [ -d {{ directory['backup'] }}/runner/software ]; then + rm {{ directory['backup'] }}/runner/software/* +fi diff --git a/software/slaprunner/template/runner-import.sh.jinja2 b/software/slaprunner/template/runner-import.sh.jinja2 new file mode 100644 index 0000000000000000000000000000000000000000..79a0dcc25bb93f202005715030bc2f6fac22b716 --- /dev/null +++ b/software/slaprunner/template/runner-import.sh.jinja2 @@ -0,0 +1,19 @@ +#!{{ shell_binary }} +umask 077 +restore_element () { + backup_path=$1 + restore_path=$2 + shift 2 + element_list=$* + for element in $element_list + do + cd $backup_path; + if [ -f $element ] || [ -d $element ]; then + {{ rsync_binary }} -avz --delete $backup_path/$element $restore_path; + fi + done +} +restore_element {{ directory['backup'] }}/runner/ {{ directory['srv'] }}/runner instance project proxy.db +restore_element {{ directory['backup'] }}/etc/ {{ directory['etc'] }} config.json ssh +cp -r {{ directory['backup'] }}/etc/.??* {{ directory['etc'] }}; +{{ curl_binary }} --insecure -vg6L --max-time 5 {{ backend_url }}/isSRReady; diff --git a/stack/erp5/buildout.cfg b/stack/erp5/buildout.cfg index 8501be37bb40002da5122da8231fa3fe73afa759..8df39db899db3c2aff2a5351fca193f72ff3b638 100644 --- a/stack/erp5/buildout.cfg +++ b/stack/erp5/buildout.cfg @@ -682,7 +682,7 @@ Pympler = 0.3.1 StructuredText = 2.11.1 WSGIUtils = 0.7 Werkzeug = 0.9.6 -apache-libcloud = 0.15.0 +apache-libcloud = 0.15.1 astroid = 1.1.1 async = 0.6.1 buildout-versions = 1.7 @@ -692,7 +692,7 @@ csp-eventlet = 0.7.0 elementtree = 1.2.7-20070827-preview erp5.recipe.cmmiforcei686 = 0.1.3 erp5diff = 0.8.1.5 -eventlet = 0.14.0 +eventlet = 0.15.0 five.formlib = 1.0.4 five.localsitemanager = 2.0.5 gitdb = 0.5.4 @@ -705,16 +705,16 @@ ipdb = 0.8 ipython = 2.1.0 itsdangerous = 0.24 lock-file = 2.0 -logilab-common = 0.61.0 +logilab-common = 0.62.0 meld3 = 1.0.0 minitage.paste = 1.4.6 minitage.recipe.egg = 1.107 -netaddr = 0.7.11 +netaddr = 0.7.12 plone.recipe.command = 1.1 ply = 3.4 polib = 1.0.4 pprofile = 1.7.2 -pycountry = 1.7 +pycountry = 1.8 pyflakes = 0.8.1 pylint = 1.2.1 pyparsing = 2.0.2 @@ -728,7 +728,7 @@ rtjp-eventlet = 0.3.2 slapos.recipe.build = 0.12 slapos.recipe.cmmi = 0.2 slapos.recipe.template = 2.5 -slapos.toolbox = 0.39.2 +slapos.toolbox = 0.39.3 smmap = 0.8.2 socketpool = 0.5.3 spyne = 2.11.0 @@ -738,17 +738,17 @@ threadframe = 0.2 timerserver = 2.0.2 urlnorm = 1.1.2 uuid = 1.30 -validictory = 0.9.3 +validictory = 1.0.0a1 xupdate-processor = 0.4 xfw = 0.10 # Required by: # slapos.core==1.1.2 -# slapos.toolbox==0.39.2 +# slapos.toolbox==0.39.3 Flask = 0.10.1 # Required by: -# slapos.toolbox==0.39.2 +# slapos.toolbox==0.39.3 GitPython = 0.3.2.RC1 # Required by: @@ -756,7 +756,7 @@ GitPython = 0.3.2.RC1 Products.ZSQLMethods = 2.13.4 # Required by: -# slapos.toolbox==0.39.2 +# slapos.toolbox==0.39.3 atomize = 0.2.0 # Required by: @@ -772,7 +772,7 @@ ecdsa = 0.11 erp5.util = 0.4.38 # Required by: -# slapos.toolbox==0.39.2 +# slapos.toolbox==0.39.3 feedparser = 5.1.3 # Required by: @@ -809,12 +809,12 @@ netifaces = 0.10.4 ordereddict = 1.1 # Required by: -# slapos.toolbox==0.39.2 +# slapos.toolbox==0.39.3 paramiko = 1.14.0 # Required by: # slapos.core==1.1.2 -# slapos.toolbox==0.39.2 +# slapos.toolbox==0.39.3 psutil = 2.1.1 # Required by: @@ -830,7 +830,7 @@ six = 1.7.3 supervisor = 3.0 # Required by: -# slapos.toolbox==0.39.2 +# slapos.toolbox==0.39.3 xml-marshaller = 0.9.7 # Required by: diff --git a/stack/monitor/buildout.cfg b/stack/monitor/buildout.cfg index cace35c61cd1ba22491613200d63e700eb59a5b2..f8f30adb50ee9f09e0b9e38ac3661b5d9d031d60 100644 --- a/stack/monitor/buildout.cfg +++ b/stack/monitor/buildout.cfg @@ -41,7 +41,7 @@ recipe = slapos.recipe.template url = ${:_profile_base_location_}/monitor.cfg.in output = ${buildout:directory}/monitor.cfg filename = monitor.cfg -md5sum = 0fe5b78cd16df190ec8838116954669e +md5sum = fe76a9e0619f276e9de3dacf9e3e01ec mode = 0644 [monitor-bin] @@ -84,7 +84,7 @@ mode = 0644 recipe = hexagonit.recipe.download url = ${:_profile_base_location_}/webfile-directory/${:filename} download-only = true -md5sum = fa8ec5aa5395b1bc6b9c7e898d6c5ec5 +#md5sum = 4fb26753ee669b8ac90ffe33dbd12e8f destination = ${buildout:directory}/parts/monitor-template-status-history-cgi filename = status-history.cgi.in mode = 0644 diff --git a/stack/monitor/monitor.cfg.in b/stack/monitor/monitor.cfg.in index f0a04739a85047c8d11dc7b1eeaceb6592736b3b..401620e7167b0ed9e79fc730d461caf21e68d8f4 100644 --- a/stack/monitor/monitor.cfg.in +++ b/stack/monitor/monitor.cfg.in @@ -45,7 +45,7 @@ www = $${:var}/www cgi-bin = $${:var}/cgi-bin monitoring-cgi = $${:cgi-bin}/monitoring knowledge0-cgi = $${:cgi-bin}/zero-knowledge -public-cgi = $${:cgi-bin}/public +public-cgi = $${:cgi-bin}/monitor-public monitor-custom-scripts = $${:etc}/monitor monitor-result = $${:var}/monitor @@ -54,7 +54,7 @@ private-directory = $${:srv}/monitor-private [public-symlink] recipe = cns.recipe.symlink -symlink = $${monitor-directory:public-cgi} = $${monitor-directory:www}/public +symlink = $${monitor-directory:public-cgi} = $${monitor-directory:www}/monitor-public autocreate = true [cron] diff --git a/stack/resilient/buildout.cfg b/stack/resilient/buildout.cfg index cf0e60a1b988c12a31430b761ffb32080aeb8570..969c90339655e0fed1381dd0bdf3692f822deed9 100644 --- a/stack/resilient/buildout.cfg +++ b/stack/resilient/buildout.cfg @@ -6,6 +6,7 @@ extends = ../../component/gzip/buildout.cfg ../../component/rdiff-backup/buildout.cfg ../../component/rsync/buildout.cfg + ../monitor/buildout.cfg parts = collective.recipe.template-egg @@ -37,7 +38,7 @@ eggs = collective.recipe.template recipe = slapos.recipe.template url = ${:_profile_base_location_}/pbsready.cfg.in output = ${buildout:directory}/pbsready.cfg -md5sum = 02a5f1741d6b732519c06b522dbe0d66 +md5sum = 28e9bc354cc146d1bec31c8cac263ee2 mode = 0644 [pbsready-import] @@ -62,7 +63,7 @@ mode = 0644 recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance-pull-backup.cfg.in output = ${buildout:directory}/instance-pull-backup.cfg -md5sum = c49e5911b94078d87b94507fb4efc93b +md5sum = 08b80d1b7beb1610077151904526a2ec mode = 0644 [template-replicated] @@ -95,6 +96,26 @@ md5sum = e6262c5cf9b1c4d1ea4d959fdcbe3070 mode = 0644 destination = ${buildout:directory}/resilient-web-takeover-cgi-script.py.in +# Provide an empty wrapper +[template-wrapper] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/templates/wrapper.in +output = ${buildout:directory}/template-wrapper.cfg +mode = 0644 +md5sum = 8cde04bfd0c0e9bd56744b988275cfd8 + +################## +# Monitor element +# + +[template-monitor-check-resilient-feed] +recipe = hexagonit.recipe.download +url = ${:_profile_base_location_}/templates/monitor-check-resilient-feed.in +download-only = true +md5sum = 19ee9055de961acf402e2dfe5b9581d2 +filename = monitor-check-resilient-feed.in +mode = 0644 + [versions] # Pin Jinja2 to 2.6, as 2.7 breaks current code Jinja2 = 2.6 diff --git a/stack/resilient/instance-pull-backup.cfg.in b/stack/resilient/instance-pull-backup.cfg.in index 5cca1f4d30b6277bc286ed7621b6e3b127badde9..cb1784c52dfa46f666934b1ed3e5ab38d48cd048 100644 --- a/stack/resilient/instance-pull-backup.cfg.in +++ b/stack/resilient/instance-pull-backup.cfg.in @@ -1,14 +1,36 @@ [buildout] parts = - connection-dict + publish-connection-informations pbs logrotate cron cron-entry-logrotate sshkeys-authority sshkeys-dropbear - +## Monitoring part +###Parts to add for monitoring + certificate-authority + cron-entry-monitor + cron-entry-rss + deploy-index + deploy-settings-cgi + deploy-status-cgi + deploy-status-history-cgi + setup-static-files + certificate-authority + zero-parameters + public-symlink + cgi-httpd-wrapper + cgi-httpd-graceful-wrapper + monitor-promise + monitor-instance-log-access +## Monitor for pbs + monitor-current-log-access + monitor-backup-log-access + monitor-check-resilient-feed-file + +extends = ${monitor-template:output} eggs-directory = ${buildout:eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory} offline = true @@ -69,16 +91,17 @@ equeue-binary = ${buildout:bin-directory}/equeue # notifier.notify adds the [exporter, notifier] to the execution queue # notifier.notify.callback sets up a callback [notifier] -recipe = slapos.cookbook:notifier -feeds = $${directory:notifier-feeds} -callbacks = $${directory:notifier-callbacks} -id-file = $${rootdirectory:etc}/notifier.id -equeue-socket = $${equeue:socket} +recipe = slapos.recipe.template:jinja2 +template = ${template-wrapper:output} +rendered = $${:wrapper} +wrapper = $${basedirectory:services}/notifier +mode = 0700 +command = ${buildout:bin-directory}/pubsubserver --callbacks $${directory:notifier-callbacks} --feeds $${directory:notifier-feeds} --equeue-socket $${equeue:socket} --logfile $${basedirectory:log}/notifier.log $${:host} $${:port} host = $${slap-network-information:global-ipv6} port = 8088 -wrapper = $${basedirectory:services}/notifier -server-binary = ${buildout:bin-directory}/pubsubserver -notifier-binary = ${buildout:bin-directory}/pubsubnotifier +context = + key content notifier:command + [logrotate-entry-equeue] <= logrotate @@ -98,6 +121,10 @@ rotate-num = 30 <= notifier recipe = slapos.cookbook:pbs client = true +feeds = $${directory:notifier-feeds} +callbacks = $${directory:notifier-callbacks} +equeue-socket = $${equeue:socket} +notifier-binary = ${buildout:bin-directory}/pubsubnotifier rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup sshclient-binary = $${dropbear-client:wrapper} known-hosts = $${directory:dot-ssh}/known_hosts @@ -215,9 +242,32 @@ slave_instance_list = [] #-- #-- Publish instance parameters. -[connection-dict] +[publish-connection-informations] recipe = slapos.cookbook:publish ssh-key = $${sshkeys-dropbear:public-key-value} notification-url = http://[$${notifier:host}]:$${notifier:port}/notify feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/ + +#---------------- +#-- +#-- Monitor + +[monitor-backup-log-access] +< = monitor-directory-access +source = $${directory:logrotate-backup} + +[monitor-current-log-access] +< = monitor-directory-access +source = $${basedirectory:log} + +[monitor-check-resilient-feed-file] +recipe = slapos.recipe.template:jinja2 +template = ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename} +rendered = $${monitor-directory:monitor-custom-scripts}/check-create-resilient-feed-files.py +mode = 700 +context = + key input_feed_directory directory:notifier-feeds + key monitor_feed_directory monitor-directory:public-cgi + key base_url publish-connection-informations:feeds-url + raw python_executable ${buildout:executable} diff --git a/stack/resilient/pbsready.cfg.in b/stack/resilient/pbsready.cfg.in index 9fa421f92cb6bb6c5b9f09ca376673dbbc63aaa6..58e90dab95ea9d1719ee3f616afe476587ea5ed2 100644 --- a/stack/resilient/pbsready.cfg.in +++ b/stack/resilient/pbsready.cfg.in @@ -169,16 +169,19 @@ equeue-binary = ${buildout:bin-directory}/equeue # notifier.notify adds the [exporter, notifier] to the execution queue # notifier.notify.callback sets up a callback [notifier] -recipe = slapos.cookbook:notifier +recipe = slapos.recipe.template:jinja2 +template = ${template-wrapper:output} +rendered = $${:wrapper} +wrapper = $${basedirectory:services}/notifier +mode = 0700 feeds = $${directory:notifier-feeds} callbacks = $${directory:notifier-callbacks} -id-file = $${rootdirectory:etc}/notifier.id -equeue-socket = $${equeue:socket} +command = ${buildout:bin-directory}/pubsubserver --callbacks $${directory:notifier-callbacks} --feeds $${directory:notifier-feeds} --equeue-socket $${equeue:socket} --logfile $${basedirectory:log}/notifier.log $${:host} $${:port} +notifier-binary = ${buildout:bin-directory}/pubsubnotifier host = $${slap-network-information:global-ipv6} port = 65534 -wrapper = $${basedirectory:services}/notifier -server-binary = ${buildout:bin-directory}/pubsubserver -notifier-binary = ${buildout:bin-directory}/pubsubnotifier +context = + key content notifier:command #---------------- diff --git a/stack/resilient/templates/monitor-check-resilient-feed.in b/stack/resilient/templates/monitor-check-resilient-feed.in new file mode 100644 index 0000000000000000000000000000000000000000..de3f6bffaa5a7e895f2c9a35750bd1e6fa498447 --- /dev/null +++ b/stack/resilient/templates/monitor-check-resilient-feed.in @@ -0,0 +1,23 @@ +#!{{ python_executable }} + +import os +import urllib2 +import sys + +input_feed_directory = '{{ input_feed_directory }}' +monitor_feed_directory = '{{ monitor_feed_directory }}' +base_url = "{{ base_url }}" +feed_file_list = os.listdir(input_feed_directory) + +rss_ok = True + +for feed_file_name in feed_file_list: + print "Getting %s" % feed_file_name + url = base_url + feed_file_name + try: + feed = urllib2.urlopen(url) + body = feed.read() + open(os.path.join(monitor_feed_directory, feed_file_name + '.rss'), 'w').write(body) + print "FEED is ok" + except urllib2.HTTPError as e: + sys.exit("%s is unvailable: %s" % (feed_file_name, e)) diff --git a/stack/resilient/templates/wrapper.in b/stack/resilient/templates/wrapper.in new file mode 100644 index 0000000000000000000000000000000000000000..6fa3fa47bee33d0636478a1b203e0cb1246377b7 --- /dev/null +++ b/stack/resilient/templates/wrapper.in @@ -0,0 +1,2 @@ +#!${dash-output:dash} +{{ content }} \ No newline at end of file