Merge remote-tracking branch 'origin/master' into erp5-component

component/screen/buildout.cfg

@@ -12,4 +12,8 @@ url = http://ftp.gnu.org/gnu/screen/screen-4.0.3.tar.gz
 md5sum = 8506fd205028a96c741e4037de6e3c42
 environment=
     CFLAGS=-I${ncurses:location}/include
-    LDFLAGS=-L${ncurses:location}/lib/ -Wl,-rpath=${ncurses:location}/lib/
\ No newline at end of file
+    LDFLAGS=-L${ncurses:location}/lib/ -Wl,-rpath=${ncurses:location}/lib/
+# Compatibility with CentOS
+patch-options = -p1 -b
+patches =
+  ${:_profile_base_location_}/screen-4.0.3-stropts.patch#5677917353067ffbcb08a23a59605aa4
\ No newline at end of file

component/screen/screen-4.0.3-stropts.patch

+diff -up screen-4.0.3/tty.sh.stropts screen-4.0.3/tty.sh
+--- screen-4.0.3/tty.sh.stropts	2003-09-08 16:24:25.000000000 +0200
++++ screen-4.0.3/tty.sh	2008-04-07 11:28:34.000000000 +0200
+@@ -76,7 +76,7 @@ exit 0
+ #endif
+ 
+ #include "config.h"
+-#ifdef SVR4
++#if 0
+ #include <sys/stropts.h>	/* for I_POP */
+ #endif
+ 
+diff -up screen-4.0.3/screen.c.stropts screen-4.0.3/screen.c
+--- screen-4.0.3/screen.c.stropts	2008-04-07 11:25:21.000000000 +0200
++++ screen-4.0.3/screen.c	2008-04-07 11:29:14.000000000 +0200
+@@ -50,7 +50,7 @@
+ 
+ #include "config.h"
+ 
+-#ifdef SVR4
++#if 0
+ # include <sys/stropts.h>
+ #endif
+ 
+diff -up screen-4.0.3/process.c.stropts screen-4.0.3/process.c
+--- screen-4.0.3/process.c.stropts	2003-09-18 14:53:54.000000000 +0200
++++ screen-4.0.3/process.c	2008-04-07 11:29:47.000000000 +0200
+@@ -37,7 +37,7 @@
+ #include "config.h"
+ 
+ /* for solaris 2.1, Unixware (SVR4.2) and possibly others: */
+-#ifdef SVR4
++#if 0 
+ # include <sys/stropts.h>
+ #endif
+ 
+diff -up screen-4.0.3/pty.c.stropts screen-4.0.3/pty.c
+--- screen-4.0.3/pty.c.stropts	2003-09-08 16:26:18.000000000 +0200
++++ screen-4.0.3/pty.c	2008-04-07 11:30:07.000000000 +0200
+@@ -34,7 +34,7 @@
+ #endif
+ 
+ /* for solaris 2.1, Unixware (SVR4.2) and possibly others */
+-#ifdef HAVE_SVR4_PTYS
++#if 0 
+ # include <sys/stropts.h>
+ #endif
+ 

setup.py

@@ -184,9 +184,6 @@ setup(name=name,
           'slapmonitor = slapos.recipe.slapmonitor:MonitorRecipe',
           'slapmonitor-xml = slapos.recipe.slapmonitor:MonitorXMLRecipe',
           'slapreport = slapos.recipe.slapreport:Recipe',
-          'slaprunner.test = slapos.recipe.slaprunner:Test',
-          'slaprunner.export = slapos.recipe.slaprunner.backup:ExportRecipe',
-          'slaprunner.import = slapos.recipe.slaprunner.backup:ImportRecipe',
           'softwaretype = slapos.recipe.softwaretype:Recipe',
           'sphinx= slapos.recipe.sphinx:Recipe',
           'squid = slapos.recipe.squid:Recipe',

slapos/recipe/request.py

@@ -153,6 +153,7 @@ class Recipe(object):
           # XXX: deprecated, to be removed
           options['instance_guid'] = self.instance.getInstanceGuid()
           options['instance-state'] = self.instance.getState()
+          options['instance-status'] = self.instance.getStatus()
         except (slapmodule.ResourceNotReady, AttributeError):
           # Backward compatibility. Old SlapOS master and core don't know this.
           self.logger.warning("Impossible to fetch instance GUID nor state.")

slapos/recipe/slaprunner/__init__.py

-##############################################################################
-#
-# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-from slapos.recipe.librecipe import GenericBaseRecipe
-import os
-
-#XXX-Nicolas This recipe has to be deleted as soon as possible
-#No changes allowed, except full-replacement using buildout
-class Test(GenericBaseRecipe):
-  def _options(self, options):
-    self.ipv4 = options['ipv4'].strip()
-    self.ipv6 = options['ipv6'].strip()
-    self.proxy_port = options['proxy_port'].strip()
-    self.runner_port = options['runner_port'].strip()
-    self.workdir = options['working-directory'].strip()
-    self.software_directory = options['software-directory'].strip()
-    self.instance_directory = options['instance-directory'].strip()
-    self.partition_amount = options['partition-amount'].strip()
-    self.cloud9_url = options.get('cloud9-url', '').strip()
-    # Set slaprunner access URL
-    options['access-url'] = 'http://[%s]:%s' % (self.ipv6, self.runner_port)
-
-  def install(self):
-    path_list = []
-
-    configuration = dict(
-        software_root=self.software_directory,
-        instance_root=self.instance_directory,
-        master_url='http://%s:%s/' % (self.ipv4, self.proxy_port),
-        computer_id='slaprunner',
-        partition_amount=self.partition_amount,
-        slapgrid_sr=self.options['slapgrid_sr'],
-        slapgrid_cp=self.options['slapgrid_cp'],
-        slapproxy=self.options['slapproxy'],
-        supervisor=self.options['supervisor'],
-        supervisord_config=os.path.join(self.instance_directory, 'etc',
-          'supervisord.conf'),
-        runner_workdir=self.workdir,
-        etc_dir=self.options['etc_dir'],
-        run_dir=self.options['etc_dir'],
-        log_dir=self.workdir,
-        runner_host=self.ipv4,
-        runner_port=self.runner_port,
-        ipv4_address=self.ipv4,
-        ipv6_address=self.ipv6,
-        proxy_host=self.ipv4,
-        proxy_port=self.proxy_port,
-        proxy_database=os.path.join(self.workdir, 'proxy.db'),
-        git=self.options['git-binary'],
-        ssh_client=self.options['ssh_client'],
-        public_key=self.options['public_key'],
-        private_key=self.options['private_key'],
-        cloud9_url=self.cloud9_url
-    )
-
-    config_file = self.createFile(self.options['slapos.cfg'],
-        self.substituteTemplate(self.getTemplateFilename('slapos.cfg.in'),
-        configuration))
-    path_list.append(config_file)
-
-    environment = dict(
-        PATH=os.path.dirname(
-            self.options['git-binary']) + ':' + os.environ['PATH'],
-        GIT_SSH=self.options['ssh_client'],
-        CONFIG_FILE_PATH=config_file
-    )
-    launch_args = [self.options['slaprunnertest'].strip()]
-
-    wrapper = self.createPythonScript(self.options['wrapper'],
-        'slapos.recipe.librecipe.execute.executee',
-        (launch_args, environment)
-    )
-    path_list.append(wrapper)
-
-    return path_list

slapos/recipe/slaprunner/backup.py

-##############################################################################
-#
-# Copyright (c) 2013 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-
-import textwrap
-
-from slapos.recipe.librecipe import GenericBaseRecipe
-
-
-
-class ExportRecipe(GenericBaseRecipe):
-    """\
-    This recipe creates an exporter script for using with the resilient stack.
-
-    Required options:
-        backup-directory
-            folder that will contain the dump file.
-        srv-directory
-            folder that contain the runner directory.
-        wrapper
-            full path of the exporter script to create.
-    """
-
-    def install(self):
-        wrapper = self.options['wrapper']
-        self.createBackupScript(wrapper)
-        return [wrapper]
-
-
-    def createBackupScript(self, wrapper):
-        """\
-        Create a script to backup the database in 'custom' format.
-        """
-        content = textwrap.dedent("""\
-                #!%(shell-binary)s
-                umask 077
-                sync_element () {
-                  path=$1
-                  backup_path=$2
-                  shift 2
-                  element_list=$*
-                  for element in $element_list
-                  do
-                    cd $path;
-                    if [ -f $element ] || [ -d $element ]; then
-                       %(rsync-binary)s -avz --safe-links --delete $element  $backup_path;
-                    fi
-                  done
-                }
-                sync_element %(srv-directory)s/runner  %(backup-directory)s/runner/ instance project  proxy.db
-                # We sync .* appart
-                cp -r %(etc-directory)s/.??* %(backup-directory)s/etc/
-                sync_element %(etc-directory)s  %(backup-directory)s/etc/ config.json ssh
-                if [ -d %(backup-directory)s/runner/software ]; then
-                  rm %(backup-directory)s/runner/software/*
-                fi
-                """ % self.options)
-        self.createExecutable(wrapper, content=content)
-
-
-
-class ImportRecipe(GenericBaseRecipe):
-    """\
-    This recipe creates an importer script for using with the resilient stack.
-
-    Required options:
-        backup-directory
-            folder that will contain the dump file.
-        srv-directory
-            folder that contain the runner directory.
-        wrapper
-            full path of the exporter script to create.
-    """
-
-    def install(self):
-        wrapper = self.options['wrapper']
-        self.createRestoreScript(wrapper)
-        return [wrapper]
-
-
-    def createRestoreScript(self, wrapper):
-        """\
-        Create a script to restore the database from 'custom' format.
-        """
-        content = textwrap.dedent("""\
-                #!%(shell-binary)s
-                umask 077
-                restore_element () {
-                  backup_path=$1
-                  restore_path=$2
-                  shift 2
-                  element_list=$*
-                  for element in $element_list
-                  do
-                    cd $backup_path;
-                    if [ -f $element ] || [ -d $element ]; then
-                       %(rsync-binary)s -avz --delete $backup_path/$element $restore_path;
-                    fi
-                  done
-                }
-                restore_element %(backup-directory)s/runner/ %(srv-directory)s/runner  instance project  proxy.db
-                restore_element  %(backup-directory)s/etc/ %(etc-directory)s config.json ssh
-                cp -r %(backup-directory)s/etc/.??* %(etc-directory)s;
-                %(curl-binary)s --insecure -vg6L --max-time 5  %(backend-url)s/isSRReady;
-                """ % self.options)
-        self.createExecutable(wrapper, content=content)
-
-

slapos/recipe/slaprunner/template/slapos.cfg.in

-[slapos]
-software_root = %(software_root)s
-instance_root = %(instance_root)s
-master_url = %(master_url)s
-computer_id = %(computer_id)s
-maximal_delay = 0
-
-[slapformat]
-partition_amount = %(partition_amount)s
-
-[slaprunner]
-slapgrid_sr = %(slapgrid_sr)s
-slapgrid_cp = %(slapgrid_cp)s
-slapproxy = %(slapproxy)s
-supervisor = %(supervisor)s
-supervisord_config = %(supervisord_config)s
-runner_workdir = %(runner_workdir)s
-runner_host = %(runner_host)s
-runner_port = %(runner_port)s
-ipv4_address = %(ipv4_address)s
-ipv6_address = %(ipv6_address)s
-etc_dir = %(etc_dir)s
-run_dir = %(run_dir)s
-log_dir = %(log_dir)s
-
-[slapproxy]
-host = %(proxy_host)s
-port = %(proxy_port)s
-database_uri = %(proxy_database)s
-
-[sshkeys_authority]
-ssh_client = %(ssh_client)s
-public_key = %(public_key)s
-private_key = %(private_key)s
-
-[gitclient]
-git = %(git)s
-
-[cloud9_IDE]
-cloud9 = %(cloud9_url)s
\ No newline at end of file

software/slaprunner/common.cfg

@@ -44,27 +44,43 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner.cfg
 output = ${buildout:directory}/template-runner.cfg.in
-md5sum = b30d71f2f0abfecc2638fe6b804f3c43
+md5sum = 1c084d9aa33202ccba58be750f0698a5
+mode = 0644
+
+[template-runner-import-script]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/template/runner-import.sh.jinja2
+download-only = true
+md5sum = b003f6a6963f7787948be85c3d479ccd
+filename = runner-import.sh.jinja2
 mode = 0644
 
 [instance-runner-import]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner-import.cfg.in
 output = ${buildout:directory}/instance-runner-import.cfg
-md5sum = 202c40bcca67d70a3c79d218be853b10
+md5sum = 354ff83819e8379f5e4a2e2985e34e9e
+mode = 0644
+
+[template-runner-export-script]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/template/runner-export.sh.jinja2
+download-only = true
+md5sum = ea478351883e6e716c044771f69a09ac
+filename = runner-export.sh.jinja2
 mode = 0644
 
 [instance-runner-export]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner-export.cfg.in
 output = ${buildout:directory}/instance-runner-export.cfg
-md5sum = a8815402e2d65c0a9cda53492b70d175
+md5sum = cf43eb1d6e9eea545858a86ea790a1cb
 mode = 0644
 
 [template-resilient]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
-md5sum = e5c7e445d0103b5e73aedb43fac681dd
+md5sum = ab564b9a9de3e44884adf072fc740d23
 filename = instance-resilient.cfg.jinja2
 mode = 0644
 

software/slaprunner/instance-resilient.cfg.jinja2

@@ -23,7 +23,7 @@ parts +=
 
 # Bubble up the parameters
 [request-runner]
-return = url ssh-public-key ssh-url notification-id ip backend_url url ssh_command access_url 1_info 2_info monitor_url IMPORTANT_monitor_info webdav_url public_url
+return = url ssh-public-key ssh-url notification-id ip backend_url url ssh_command access_url 1_info 2_info monitor_url webdav_url public_url
 
 [publish-connection-informations]
 recipe = slapos.cookbook:publish
@@ -34,7 +34,6 @@ access_url = ${request-runner:connection-access_url}
 url = ${request-runner:connection-url}
 ssh_command = ${request-runner:connection-ssh_command}
 monitor_url = ${request-runner:connection-monitor_url}
-IMPORTANT_monitor_info = ${request-runner:connection-IMPORTANT_monitor_info}
 webdav_url = ${request-runner:connection-webdav_url}
 public_url = ${request-runner:connection-public_url}
 

software/slaprunner/instance-runner-export.cfg.in

@@ -26,40 +26,54 @@ parts +=
   cron-entry-backup
   cron-entry-prepare-software
   deploy-instance-parameters
+## Monitoring part
 ###Parts to add for monitoring
-  slap-parameters
-  certificate-authority
   cron
+  certificate-authority
   cron-entry-monitor
   cron-entry-rss
   deploy-index
-  deploy-index-template
-  deploy-monitor-script
-  deploy-rss-script
   deploy-settings-cgi
   deploy-status-cgi
-  make-rss
-  monitor-frontend
-  monitor-promise
+  deploy-status-history-cgi
   setup-static-files
   certificate-authority
-  public
   zero-parameters
-  cgi-httpd-wrapper
   public-symlink
+  cgi-httpd-wrapper
+  cgi-httpd-graceful-wrapper
+  monitor-promise
+  monitor-instance-log-access
+## Monitor for runner
+  monitor-current-log-access
+
 
 [exporter]
-recipe = slapos.cookbook:slaprunner.export
-wrapper = $${directory:bin}/$${slap-parameter:namebase}-exporter
-srv-directory = $${directory:srv}
-etc-directory = $${directory:etc}
-backup-directory = $${directory:backup}
-shell-binary = ${dash:location}/bin/dash
-rsync-binary = ${rsync:location}/bin/rsync
+recipe = slapos.recipe.template:jinja2
+template = ${template-runner-export-script:location}/${template-runner-export-script:filename}
+rendered = $${directory:bin}/$${slap-parameter:namebase}-exporter
+# backward compatibility for resilient stack
+wrapper = $${:rendered}
+mode = 700
+context =
+  section directory directory
+  raw  shell_binary ${dash:location}/bin/dash
+  raw  rsync_binary ${rsync:location}/bin/rsync
 
 [monitor-promise]
-url = $${monitor-frontend:config-url}/$${deploy-index:filename}
+url = $${monitor-frontend:config-url}/$${deploy-index-template:filename}
 
 # Extends publish section with resilient parameters
 [publish-connection-informations]
 <= resilient-publish-connection-parameter
+
+[monitor-check-resilient-feed-file]
+recipe = slapos.recipe.template:jinja2
+template = ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
+rendered = $${monitor-directory:monitor-custom-scripts}/check-create-resilient-feed-files.py
+mode = 700
+context =
+  key input_feed_directory directory:notifier-feeds
+  key monitor_feed_directory monitor-directory:public-cgi
+  raw base_url http://[$${notifier:host}]:$${notifier:port}/get/
+  raw python_executable ${buildout:executable}

software/slaprunner/instance-runner-import.cfg.in

@@ -23,39 +23,63 @@ parts +=
   slapos-repo-config
   cron-entry-prepare-software
   deploy-instance-parameters
+  instance-software-type
 # have to repeat the next one, as it's not inherited from pbsready-import
   import-on-notification
+## Monitoring part
 ###Parts to add for monitoring
-  slap-parameters
-  certificate-authority
   cron
+  certificate-authority
   cron-entry-monitor
   cron-entry-rss
   deploy-index
-  deploy-index-template
-  deploy-monitor-script
-  deploy-rss-script
   deploy-settings-cgi
   deploy-status-cgi
-  make-rss
-  monitor-promise
+  deploy-status-history-cgi
   setup-static-files
   certificate-authority
-  public
   zero-parameters
-  cgi-httpd-wrapper
   public-symlink
+  cgi-httpd-wrapper
+  cgi-httpd-graceful-wrapper
+  monitor-promise
+  monitor-instance-log-access
+## Monitor for runner
+  monitor-current-log-access
+  monitor-backup-log-access
+## Monitor for import runner
+  monitor-latest-restored-backup
 
 [importer]
-recipe = slapos.cookbook:slaprunner.import
-wrapper = $${directory:bin}/$${slap-parameter:namebase}-importer
-srv-directory = $${directory:srv}
-etc-directory = $${directory:etc}
-backup-directory = $${directory:backup}
-shell-binary = ${dash:location}/bin/dash
-rsync-binary = ${rsync:location}/bin/rsync
-curl-binary = ${curl:location}/bin/curl
-backend-url = $${slaprunner:access-url}
+recipe = slapos.recipe.template:jinja2
+template = ${template-runner-import-script:location}/${template-runner-import-script:filename}
+rendered = $${directory:bin}/$${slap-parameter:namebase}-importer
+# backward compatibility for resilient stack
+wrapper = $${:rendered}
+mode = 700
+context =
+  key backend_url slaprunner:access-url
+  section directory directory
+  raw  shell_binary ${dash:location}/bin/dash
+  raw  rsync_binary ${rsync:location}/bin/rsync
+  raw  curl_binary ${curl:location}/bin/curl
 
 [slap-parameter]
 auto-deploy-instance = false
+auto-deploy = true
+
+[resilient-publish-connection-parameter]
+monitor-url = $${monitor-parameters:url}
+
+[monitor-backup-log-access]
+< = monitor-directory-access
+source = $${directory:logrotate-backup}
+
+[monitor-latest-restored-backup]
+recipe = slapos.recipe.template:jinja2
+command = if [ -f $${directory:etc}/.resilient-timestamp ]; then echo "$(date -d @$(cat $${directory:etc}/.resilient-timestamp) +%c)"; else echo "No backup timestamp found"; fi
+rendered = $${monitor-directory:monitoring-cgi}/latest-restored-backup
+template = ${template-wrapper:output}
+mode = 744
+context =
+  key content :command
\ No newline at end of file

software/slaprunner/instance-runner.cfg

@@ -22,29 +22,30 @@ parts =
   slapos-repo-config
   cron-entry-prepare-software
   deploy-instance-parameters
+  instance-software-type
 {% if slapparameter_dict.get('custom-frontend-backend-url') %}
   custom-frontend-promise
 {% endif %}
+## Monitoring part
 ###Parts to add for monitoring
-  slap-parameters
-  certificate-authority
   cron
+  certificate-authority
   cron-entry-monitor
   cron-entry-rss
   deploy-index
-  deploy-index-template
-  deploy-monitor-script
-  deploy-rss-script
   deploy-settings-cgi
   deploy-status-cgi
-  make-rss
-  monitor-promise
+  deploy-status-history-cgi
   setup-static-files
   certificate-authority
-  public
   zero-parameters
-  cgi-httpd-wrapper
   public-symlink
+  cgi-httpd-wrapper
+  cgi-httpd-graceful-wrapper
+  monitor-promise
+  monitor-instance-log-access
+## Monitor for runner
+  monitor-current-log-access
 
 extends = ${monitor-template:output}
 
@@ -309,7 +310,6 @@ working_directory = $${slaprunner:working-directory}
 dav_lock = $${directory:var}/DavLock
 etc_dir = $${directory:etc}
 var_dir = $${directory:var}
-document_root = $${directory:www}
 project_folder = $${directory:project}
 runner_home = $${runnerdirectory:home}
 git_http_backend = ${git:location}/libexec/git-core/git-http-backend
@@ -480,6 +480,7 @@ debug = false
 frontend-domain =
 slapos-repository = http://git.erp5.org/repos/slapos.git
 slapos-software =
+slapos-software-type =
 slapos-reference = master
 auto-deploy = false
 auto-deploy-instance = true
@@ -575,6 +576,13 @@ stop-on-error = true
 parameter-xml = $${directory:etc}/.parameter.xml
 command = if [ ! -f $${:parameter-xml} ]; then cp $${instance-parameters:rendered} $${:parameter-xml}; fi
 
+[instance-software-type]
+recipe = plone.recipe.command
+stop-on-error = true
+# XXX It should not be named with .xml as it is not xml
+software-type-path = $${directory:etc}/.software_type.xml
+command = if [ ! -f $${:software-type-path} -a "$${slap-parameter:slapos-software-type}" != "" ]; then echo "$${slap-parameter:slapos-software-type}" > $${:software-type-path}; fi
+
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration.serialised
 computer = $${slap-connection:computer-id}
@@ -588,3 +596,8 @@ shell-password = $${shellinabox-code:passwd}
 recovery-code = $${recovery-code:passwd}
 
 [zero-parameters]
+
+
+[monitor-current-log-access]
+< = monitor-directory-access
+source = $${directory:log}

software/slaprunner/schema.json

@@ -38,6 +38,10 @@
                                 "type": "boolean",
                                 "default": "false"
                         },
+                        "slapos-software-type": {
+                                "description": "Software type of your instance inside the runner",
+                                "type": "string"
+                        },
                         "parameter-*": {
                                 "description": "'*' is a parameter which will be used to configure the instance inside the runner.",
                                 "type": "string"

software/slaprunner/software.cfg

@@ -194,10 +194,11 @@ collective.recipe.template = 1.10
 erp5.util = 0.4.36
 gitdb = 0.5.4
 hexagonit.recipe.cmmi = 1.6
-inotifyx = 0.2.0-1
+inotifyx = 0.2.0
 lock-file = 2.0
 lxml = 3.0.1
 meld3 = 0.6.10
+mr.developer = 1.30
 netaddr = 0.7.10
 plone.recipe.command = 1.1
 prettytable = 0.7.2
@@ -212,7 +213,7 @@ slapos.recipe.build = 0.11.5
 slapos.recipe.cmmi = 0.2
 slapos.recipe.download = 1.0.dev-r4053
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.39.2
+slapos.toolbox = 0.39.3
 smmap = 0.8.2
 xml-marshaller = 0.9.7
 z3c.recipe.scripts = 1.0.1

software/slaprunner/template/runner-export.sh.jinja2

+#!{{ shell_binary }}
+umask 077
+sync_element () {
+  path=$1
+  backup_path=$2
+  shift 2
+  element_list=$*
+  for element in $element_list
+  do
+    cd $path;
+    if [ -f $element ] || [ -d $element ]; then
+       {{ rsync_binary }} -avz --safe-links --delete $element  $backup_path;
+    fi
+  done
+}
+sync_element {{ directory['srv'] }}/runner  {{ directory['backup'] }}/runner/ instance project  proxy.db
+# We sync .* appart
+date +%s -u > {{ directory['etc'] }}/.resilient-timestamp
+cp -r {{ directory['etc'] }}/.??* {{ directory['backup'] }}/etc/
+sync_element {{ directory['etc'] }}  {{ directory['backup'] }}/etc/ config.json ssh
+if [ -d {{ directory['backup'] }}/runner/software ]; then
+  rm {{ directory['backup'] }}/runner/software/*
+fi

software/slaprunner/template/runner-import.sh.jinja2

+#!{{ shell_binary }}
+umask 077
+restore_element () {
+  backup_path=$1
+  restore_path=$2
+  shift 2
+  element_list=$*
+  for element in $element_list
+  do
+    cd $backup_path;
+    if [ -f $element ] || [ -d $element ]; then
+       {{ rsync_binary }} -avz --delete $backup_path/$element $restore_path;
+    fi
+  done
+}
+restore_element {{ directory['backup'] }}/runner/ {{ directory['srv'] }}/runner  instance project  proxy.db
+restore_element  {{ directory['backup'] }}/etc/ {{ directory['etc'] }} config.json ssh
+cp -r {{ directory['backup'] }}/etc/.??* {{ directory['etc'] }};
+{{ curl_binary }} --insecure -vg6L --max-time 5  {{ backend_url }}/isSRReady;

stack/monitor/buildout.cfg

@@ -41,7 +41,7 @@ recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/monitor.cfg.in
 output = ${buildout:directory}/monitor.cfg
 filename = monitor.cfg
-md5sum = 0fe5b78cd16df190ec8838116954669e
+md5sum = fe76a9e0619f276e9de3dacf9e3e01ec
 mode = 0644
 
 [monitor-bin]
@@ -84,7 +84,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/webfile-directory/${:filename}
 download-only = true
-md5sum = fa8ec5aa5395b1bc6b9c7e898d6c5ec5
+#md5sum = 4fb26753ee669b8ac90ffe33dbd12e8f
 destination = ${buildout:directory}/parts/monitor-template-status-history-cgi
 filename = status-history.cgi.in
 mode = 0644

stack/monitor/monitor.cfg.in

@@ -45,7 +45,7 @@ www = $${:var}/www
 cgi-bin = $${:var}/cgi-bin
 monitoring-cgi = $${:cgi-bin}/monitoring
 knowledge0-cgi = $${:cgi-bin}/zero-knowledge
-public-cgi = $${:cgi-bin}/public
+public-cgi = $${:cgi-bin}/monitor-public
 
 monitor-custom-scripts = $${:etc}/monitor
 monitor-result = $${:var}/monitor
@@ -54,7 +54,7 @@ private-directory = $${:srv}/monitor-private
 
 [public-symlink]
 recipe = cns.recipe.symlink
-symlink = $${monitor-directory:public-cgi} = $${monitor-directory:www}/public
+symlink = $${monitor-directory:public-cgi} = $${monitor-directory:www}/monitor-public
 autocreate = true
 
 [cron]

stack/resilient/buildout.cfg

@@ -6,6 +6,7 @@ extends =
   ../../component/gzip/buildout.cfg
   ../../component/rdiff-backup/buildout.cfg
   ../../component/rsync/buildout.cfg
+  ../monitor/buildout.cfg
 
 parts =
   collective.recipe.template-egg
@@ -37,7 +38,7 @@ eggs = collective.recipe.template
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready.cfg.in
 output = ${buildout:directory}/pbsready.cfg
-md5sum = 02a5f1741d6b732519c06b522dbe0d66
+md5sum = 28e9bc354cc146d1bec31c8cac263ee2
 mode = 0644
 
 [pbsready-import]
@@ -62,7 +63,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-pull-backup.cfg.in
 output = ${buildout:directory}/instance-pull-backup.cfg
-md5sum = c49e5911b94078d87b94507fb4efc93b
+md5sum = 08b80d1b7beb1610077151904526a2ec
 mode = 0644
 
 [template-replicated]
@@ -95,6 +96,26 @@ md5sum = e6262c5cf9b1c4d1ea4d959fdcbe3070
 mode = 0644
 destination = ${buildout:directory}/resilient-web-takeover-cgi-script.py.in
 
+# Provide an empty wrapper
+[template-wrapper]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/templates/wrapper.in
+output = ${buildout:directory}/template-wrapper.cfg
+mode = 0644
+md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
+
+##################
+# Monitor element
+#
+
+[template-monitor-check-resilient-feed]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/templates/monitor-check-resilient-feed.in
+download-only = true
+md5sum = 19ee9055de961acf402e2dfe5b9581d2
+filename = monitor-check-resilient-feed.in
+mode = 0644
+
 [versions]
 # Pin Jinja2 to 2.6, as 2.7 breaks current code
 Jinja2 = 2.6

stack/resilient/instance-pull-backup.cfg.in

 [buildout]
 
 parts =
-  connection-dict
+  publish-connection-informations
   pbs
   logrotate
   cron
   cron-entry-logrotate
   sshkeys-authority
   sshkeys-dropbear
-
+## Monitoring part
+###Parts to add for monitoring
+  certificate-authority
+  cron-entry-monitor
+  cron-entry-rss
+  deploy-index
+  deploy-settings-cgi
+  deploy-status-cgi
+  deploy-status-history-cgi
+  setup-static-files
+  certificate-authority
+  zero-parameters
+  public-symlink
+  cgi-httpd-wrapper
+  cgi-httpd-graceful-wrapper
+  monitor-promise
+  monitor-instance-log-access
+## Monitor for pbs
+  monitor-current-log-access
+  monitor-backup-log-access
+  monitor-check-resilient-feed-file
+
+extends = ${monitor-template:output}
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
@@ -69,16 +91,17 @@ equeue-binary = ${buildout:bin-directory}/equeue
 # notifier.notify adds the [exporter, notifier] to the execution queue
 # notifier.notify.callback sets up a callback
 [notifier]
-recipe = slapos.cookbook:notifier
-feeds = $${directory:notifier-feeds}
-callbacks = $${directory:notifier-callbacks}
-id-file = $${rootdirectory:etc}/notifier.id
-equeue-socket = $${equeue:socket}
+recipe = slapos.recipe.template:jinja2
+template = ${template-wrapper:output}
+rendered = $${:wrapper}
+wrapper = $${basedirectory:services}/notifier
+mode = 0700
+command = ${buildout:bin-directory}/pubsubserver --callbacks $${directory:notifier-callbacks} --feeds $${directory:notifier-feeds} --equeue-socket $${equeue:socket} --logfile $${basedirectory:log}/notifier.log $${:host} $${:port}
 host = $${slap-network-information:global-ipv6}
 port = 8088
-wrapper = $${basedirectory:services}/notifier
-server-binary = ${buildout:bin-directory}/pubsubserver
-notifier-binary = ${buildout:bin-directory}/pubsubnotifier
+context =
+  key content notifier:command
+
 
 [logrotate-entry-equeue]
 <= logrotate
@@ -98,6 +121,10 @@ rotate-num = 30
 <= notifier
 recipe = slapos.cookbook:pbs
 client = true
+feeds = $${directory:notifier-feeds}
+callbacks = $${directory:notifier-callbacks}
+equeue-socket = $${equeue:socket}
+notifier-binary = ${buildout:bin-directory}/pubsubnotifier
 rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
 sshclient-binary = $${dropbear-client:wrapper}
 known-hosts = $${directory:dot-ssh}/known_hosts
@@ -215,9 +242,32 @@ slave_instance_list = []
 #--
 #-- Publish instance parameters.
 
-[connection-dict]
+[publish-connection-informations]
 recipe = slapos.cookbook:publish
 ssh-key = $${sshkeys-dropbear:public-key-value}
 notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
 feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/
 
+
+#----------------
+#--
+#-- Monitor
+
+[monitor-backup-log-access]
+< = monitor-directory-access
+source = $${directory:logrotate-backup}
+
+[monitor-current-log-access]
+< = monitor-directory-access
+source = $${basedirectory:log}
+
+[monitor-check-resilient-feed-file]
+recipe = slapos.recipe.template:jinja2
+template = ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
+rendered = $${monitor-directory:monitor-custom-scripts}/check-create-resilient-feed-files.py
+mode = 700
+context =
+  key input_feed_directory directory:notifier-feeds
+  key monitor_feed_directory monitor-directory:public-cgi
+  key base_url publish-connection-informations:feeds-url
+  raw python_executable ${buildout:executable}

stack/resilient/pbsready.cfg.in

@@ -169,16 +169,19 @@ equeue-binary = ${buildout:bin-directory}/equeue
 # notifier.notify adds the [exporter, notifier] to the execution queue
 # notifier.notify.callback sets up a callback
 [notifier]
-recipe = slapos.cookbook:notifier
+recipe = slapos.recipe.template:jinja2
+template = ${template-wrapper:output}
+rendered = $${:wrapper}
+wrapper = $${basedirectory:services}/notifier
+mode = 0700
 feeds = $${directory:notifier-feeds}
 callbacks = $${directory:notifier-callbacks}
-id-file = $${rootdirectory:etc}/notifier.id
-equeue-socket = $${equeue:socket}
+command = ${buildout:bin-directory}/pubsubserver --callbacks $${directory:notifier-callbacks} --feeds $${directory:notifier-feeds} --equeue-socket $${equeue:socket} --logfile $${basedirectory:log}/notifier.log $${:host} $${:port}
+notifier-binary = ${buildout:bin-directory}/pubsubnotifier
 host = $${slap-network-information:global-ipv6}
 port = 65534
-wrapper = $${basedirectory:services}/notifier
-server-binary = ${buildout:bin-directory}/pubsubserver
-notifier-binary = ${buildout:bin-directory}/pubsubnotifier
+context =
+  key content notifier:command
 
 
 #----------------

stack/resilient/templates/monitor-check-resilient-feed.in

+#!{{ python_executable }}
+
+import os
+import urllib2
+import sys
+
+input_feed_directory = '{{ input_feed_directory }}'
+monitor_feed_directory = '{{ monitor_feed_directory }}'
+base_url = "{{ base_url }}"
+feed_file_list = os.listdir(input_feed_directory)
+
+rss_ok = True
+
+for feed_file_name in feed_file_list:
+  print "Getting %s" % feed_file_name
+  url = base_url + feed_file_name
+  try:
+    feed = urllib2.urlopen(url)
+    body = feed.read()
+    open(os.path.join(monitor_feed_directory, feed_file_name + '.rss'), 'w').write(body)
+    print "FEED is ok"
+  except urllib2.HTTPError as e:
+    sys.exit("%s is unvailable: %s" % (feed_file_name, e))

stack/resilient/templates/wrapper.in

+#!${dash-output:dash}
+{{ content }}
\ No newline at end of file