Merge branch 'lapp-resilient4'

slapos/recipe/apachephp/__init__.py

@@ -24,17 +24,13 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-
 import shutil
 import os
 import signal
-import subprocess
-
 from binascii import b2a_uu as uuencode
 
 from slapos.recipe.librecipe import GenericBaseRecipe
 
-
 class Recipe(GenericBaseRecipe):
 
   def install(self):
@@ -122,16 +118,12 @@ class Recipe(GenericBaseRecipe):
         self.substituteTemplate(self.options['template'], application_conf))
       path_list.append(config)
 
-    # Reload apache configuration.
-    # notez-bien: a graceful restart or a SIGUSR1 can somehow hang the apache threads.
-
-    subprocess.call([
-                        self.options['httpd-binary'],
-                        '-f',
-                        self.options['httpd-conf'],
-                        '-k',
-                        'graceful'
-                    ])
-
+    #if os.path.exists(self.options['pid-file']):
+    #  # Reload apache configuration
+    #  with open(self.options['pid-file']) as pid_file:
+    #    pid = int(pid_file.read().strip(), 10)
+    #  try:
+    #    os.kill(pid, signal.SIGUSR1) # Graceful restart
+    #  except OSError:
+    #    pass
     return path_list
-

software/slaprunner/README.txt

@@ -21,3 +21,10 @@ Example of parameter XML::
   <instance>
   <parameter id="authorized-key">ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCdNp7qZMVAzNc07opHshiIFDmJpYXQpetfcSgUj39a409d42PpsJElp7WsAE/x0nN6gUIoWIl7UiAlMzf6bKEJGJVSOZEPTmiJVlgK1Gp+kE0x9yNcncYg7p38Jny0daVA/NkkpAFyRsAm5kLGzyLtaCcktSvy0cJuy7WSSHU05pd1f8Y8thofE9g5t+/JA2VZvipxPkRfkFAG3aOAGLULlTImTSDFSDFGSDFG5F6mMnl7yvY2d6vEHVBu+K+aKmAwZVfCUwtSpa/tq3i2Lppjrw3UfrxbQSFHZCkzefr+u+l4YYe+tJrX7rYJYXD7LIfZfdSeFTlHFaN/yI1 user@host.local</parameter>
   </instance>
+
+instance-amount
+~~~~~~~~~~~~~~~
+
+Optional parameter.
+
+Default: 10

stack/lamp/apache/instance-apache-backup.cfg.in

-[buildout]
-
-parts =
-  urls
-  apache-proxy
-  logrotate
-  logrotate-entry-apache
-  cron
-  cron-entry-logrotate
-  sshkeys-authority
-  sshkeys-dropbear
-  dropbear-server
-  dropbear-server-pbs-authorized-key
-
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-offline = true
-
-
-#----------------
-#--
-#-- Creation of all needed directories.
-
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc
-var = $${buildout:directory}/var
-srv = $${buildout:directory}/srv
-bin = $${buildout:directory}/bin
-tmp = $${buildout:directory}/tmp
-
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-log = $${rootdirectory:var}/log
-services = $${rootdirectory:etc}/run
-run = $${rootdirectory:var}/run
-backup = $${rootdirectory:srv}/backup
-promises = $${rootdirectory:etc}/promise
-
-[directory]
-recipe = slapos.cookbook:mkdirectory
-htdocs = $${rootdirectory:srv}/www
-logrotate-entries = $${rootdirectory:etc}/logrotate.d
-logrotate-backup = $${basedirectory:backup}/logrotate
-cronstamps = $${rootdirectory:etc}/cronstamps
-cron-entries = $${rootdirectory:etc}/cron.d
-crontabs = $${rootdirectory:etc}/crontabs
-ssh = $${rootdirectory:etc}/ssh
-sshkeys = $${rootdirectory:srv}/sshkeys
-httpd-log = $${basedirectory:log}/apache
-
-
-#----------------
-#--
-#-- Deploy cron.
-
-[cron]
-recipe = slapos.cookbook:cron
-dcrond-binary = ${dcron:location}/sbin/crond
-cron-entries = $${directory:cron-entries}
-crontabs = $${directory:crontabs}
-cronstamps = $${directory:cronstamps}
-catcher = $${cron-simplelogger:wrapper}
-binary = $${basedirectory:services}/crond
-
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = $${rootdirectory:bin}/cron_simplelogger
-log = $${basedirectory:log}/crond.log
-
-
-#----------------
-#--
-#-- Deploy logrotate.
-
-[cron-entry-logrotate]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = logrotate
-frequency = 0 0 * * *
-command = $${logrotate:wrapper}
-
-[logrotate]
-recipe = slapos.cookbook:logrotate
-# Binaries
-logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
-gzip-binary = ${gzip:location}/bin/gzip
-gunzip-binary = ${gzip:location}/bin/gunzip
-# Directories
-wrapper = $${rootdirectory:bin}/logrotate
-conf = $${rootdirectory:etc}/logrotate.conf
-logrotate-entries = $${directory:logrotate-entries}
-backup = $${directory:logrotate-backup}
-state-file = $${rootdirectory:srv}/logrotate.status
-
-
-#----------------
-#--
-#-- sshkeys
-
-[sshkeys-directory]
-recipe = slapos.cookbook:mkdirectory
-requests = $${directory:sshkeys}/requests
-keys = $${directory:sshkeys}/keys
-
-[sshkeys-authority]
-recipe = slapos.cookbook:sshkeys_authority
-request-directory = $${sshkeys-directory:requests}
-keys-directory = $${sshkeys-directory:keys}
-wrapper = $${basedirectory:services}/sshkeys_authority
-keygen-binary = ${dropbear:location}/bin/dropbearkey
-
-[sshkeys-dropbear]
-<= sshkeys-authority
-recipe = slapos.cookbook:sshkeys_authority.request
-name = dropbear
-type = rsa
-executable = $${dropbear-server:wrapper}
-public-key = $${dropbear-server:rsa-keyfile}.pub
-private-key = $${dropbear-server:rsa-keyfile}
-wrapper = $${basedirectory:services}/sshd
-
-
-#----------------
-#--
-#-- Dropbear.
-
-[dropbear-server]
-recipe = slapos.cookbook:dropbear
-host = $${slap-network-information:global-ipv6}
-port = 2222
-home = $${directory:ssh}
-wrapper = $${rootdirectory:bin}/raw_sshd
-shell = $${rdiff-backup-server:wrapper}
-rsa-keyfile = $${directory:ssh}/server_key.rsa
-dropbear-binary = ${dropbear:location}/sbin/dropbear
-
-[dropbear-server-pbs-authorized-key]
-<= dropbear-server
-recipe = slapos.cookbook:dropbear.add_authorized_key
-key = $${slap-parameter:authorized-key}
-
-
-#----------------
-#--
-#-- rdiff
-
-[rdiff-backup-server]
-recipe = slapos.cookbook:pbs
-client = false
-path = $${directory:htdocs}
-wrapper = $${rootdirectory:bin}/rdiffbackup-server
-rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
-
-
-#----------------
-#--
-#-- Apache Proxy.
-
-[apache-proxy]
-recipe = slapos.cookbook:apacheproxy
-url = $${slap-parameter:proxy-url}
-pid-file = $${basedirectory:run}/apache.pid
-lock-file = $${basedirectory:run}/apache.lock
-ip = $${slap-network-information:global-ipv6}
-port = 8080
-error-log = $${directory:httpd-log}/error.log
-access-log = $${directory:httpd-log}/access.log
-httpd-conf = $${rootdirectory:etc}/apache.conf
-wrapper = $${basedirectory:services}/apache
-
-promise = $${basedirectory:promises}/apache
-
-httpd-binary = ${apache:location}/bin/httpd
-
-[logrotate-entry-apache]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = apache
-log = $${apache-proxy:error-log} $${apache-proxy:access-log}
-frequency = daily
-rotate-num = 30
-sharedscripts = true
-notifempty = true
-create = true
-
-
-#----------------
-#--
-#-- Publish instance parameters.
-
-[urls]
-recipe = slapos.cookbook:publish
-url = http://[$${apache-proxy:ip}]:$${apache-proxy:port}/
-ssh-public-key = $${sshkeys-dropbear:public-key-value}
-ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
-

stack/lamp/apache/instance-apache-export.cfg.jinja2

+# This file is responsible of two things:
+# 1/ Act as "Apache exporter"
+# 2/ Act as "Mariadb backup infrastructure requester"
+
+{% import 'parts' as parts %}
+{% import 'replicated' as replicated %}
+
+[buildout]
+extends = {{templateapache}}
+          {{templatepbsreadyexport}}
+
+
+parts +=
+    {{ parts.replicate("mariadb", "3") }}
+# have to repeat the next one, as it's not inherited from pbsready-export
+    cron-entry-backup
+
+{{ replicated.replicate("mariadb", "3", "mariadb-export", "mariadb-import") }}
+
+# Nothing to do for the exporter. Just dummy part that does nothing.
+# For httpd instance, PBS will directly pull data from srv/www.
+[exporter]
+wrapper = /usr/bin/true
+
+# State that we want to backup srv/www directory, not srv/backup.
+[rdiff-backup-server]
+path = ${directory:www}
+
+# XXX-Cedric: resilient uses "urls" part to publish, but apache doesn't.
+[urls]
+<= publish-connection-informations
+
+# XXX-Cedric: resilient overwrites what's returned from request-mariadb
+[request-mariadb]
+return = ssh-public-key ssh-url notification-id ip url

stack/lamp/apache/instance-apache-import.cfg

-[buildout]
-extends = ${instance-apache-php:output}
-          ${pbsready-export:output}
-
-parts =
-  apache-proxy
-  logrotate
-  logrotate-entry-apache
-  cron
-  cron-entry-logrotate
-  sshkeys-authority
-  sshkeys-dropbear
-  dropbear-server
-  dropbear-server-pbs-authorized-key
-
-
-[apache-proxy]
-recipe = slapos.cookbook:apacheproxy
-url = $${slap-parameter:proxy-url}
-pid-file = $${basedirectory:run}/apache.pid
-lock-file = $${basedirectory:run}/apache.lock
-ip = $${slap-network-information:global-ipv6}
-port = 8080
-error-log = $${directory:httpd-log}/error.log
-access-log = $${directory:httpd-log}/access.log
-httpd-conf = $${rootdirectory:etc}/apache.conf
-wrapper = $${basedirectory:services}/apache
-
-promise = $${basedirectory:promises}/apache
-
-httpd-binary = ${apache:location}/bin/httpd
-
-
-#----------------
-#--
-#-- Creation of all needed directories.
-
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc
-var = $${buildout:directory}/var
-srv = $${buildout:directory}/srv
-bin = $${buildout:directory}/bin
-tmp = $${buildout:directory}/tmp
-
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-log = $${rootdirectory:var}/log
-services = $${rootdirectory:etc}/run
-run = $${rootdirectory:var}/run
-backup = $${rootdirectory:srv}/backup
-promises = $${rootdirectory:etc}/promise
-
-[directory]
-recipe = slapos.cookbook:mkdirectory
-htdocs = $${rootdirectory:srv}/www
-logrotate-entries = $${rootdirectory:etc}/logrotate.d
-logrotate-backup = $${basedirectory:backup}/logrotate
-cronstamps = $${rootdirectory:etc}/cronstamps
-cron-entries = $${rootdirectory:etc}/cron.d
-crontabs = $${rootdirectory:etc}/crontabs
-ssh = $${rootdirectory:etc}/ssh
-sshkeys = $${rootdirectory:srv}/sshkeys
-httpd-log = $${basedirectory:log}/apache
-
-
-#----------------
-#--
-#-- Deploy cron.
-
-[cron]
-recipe = slapos.cookbook:cron
-dcrond-binary = ${dcron:location}/sbin/crond
-cron-entries = $${directory:cron-entries}
-crontabs = $${directory:crontabs}
-cronstamps = $${directory:cronstamps}
-catcher = $${cron-simplelogger:wrapper}
-binary = $${basedirectory:services}/crond
-
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = $${rootdirectory:bin}/cron_simplelogger
-log = $${basedirectory:log}/crond.log
-
-
-#----------------
-#--
-#-- Deploy logrotate.
-
-[cron-entry-logrotate]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = logrotate
-frequency = 0 0 * * *
-command = $${logrotate:wrapper}
-
-[logrotate]
-recipe = slapos.cookbook:logrotate
-# Binaries
-logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
-gzip-binary = ${gzip:location}/bin/gzip
-gunzip-binary = ${gzip:location}/bin/gunzip
-# Directories
-wrapper = $${rootdirectory:bin}/logrotate
-conf = $${rootdirectory:etc}/logrotate.conf
-logrotate-entries = $${directory:logrotate-entries}
-backup = $${directory:logrotate-backup}
-state-file = $${rootdirectory:srv}/logrotate.status
-
-[logrotate-entry-apache]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = apache
-log = $${apache-proxy:error-log} $${apache-proxy:access-log}
-frequency = daily
-rotate-num = 30
-sharedscripts = true
-notifempty = true
-create = true
-
-
-#----------------
-#--
-#-- sshkeys
-
-[sshkeys-directory]
-recipe = slapos.cookbook:mkdirectory
-requests = $${directory:sshkeys}/requests
-keys = $${directory:sshkeys}/keys
-
-[sshkeys-authority]
-recipe = slapos.cookbook:sshkeys_authority
-request-directory = $${sshkeys-directory:requests}
-keys-directory = $${sshkeys-directory:keys}
-wrapper = $${basedirectory:services}/sshkeys_authority
-keygen-binary = ${dropbear:location}/bin/dropbearkey
-
-[sshkeys-dropbear]
-<= sshkeys-authority
-recipe = slapos.cookbook:sshkeys_authority.request
-name = dropbear
-type = rsa
-executable = $${dropbear-server:wrapper}
-public-key = $${dropbear-server:rsa-keyfile}.pub
-private-key = $${dropbear-server:rsa-keyfile}
-wrapper = $${basedirectory:services}/sshd
-
-
-#----------------
-#--
-#-- Dropbear.
-
-[dropbear-server]
-recipe = slapos.cookbook:dropbear
-host = $${slap-network-information:global-ipv6}
-port = 2222
-home = $${directory:ssh}
-wrapper = $${rootdirectory:bin}/raw_sshd
-shell = $${rdiff-backup-server:wrapper}
-rsa-keyfile = $${directory:ssh}/server_key.rsa
-dropbear-binary = ${dropbear:location}/sbin/dropbear
-
-[dropbear-server-pbs-authorized-key]
-<= dropbear-server
-recipe = slapos.cookbook:dropbear.add_authorized_key
-key = $${slap-parameter:authorized-key}
-
-
-#----------------
-#--
-#-- rdiff
-
-[rdiff-backup-server]
-recipe = slapos.cookbook:pbs
-client = false
-path = $${directory:htdocs}
-wrapper = $${rootdirectory:bin}/rdiffbackup-server
-rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
-

stack/lamp/apache/instance-apache-import.cfg.in

+[buildout]
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+# XXX-Cedric: is instance-apache-php needed?
+#extends = ${instance-apache-php:output}
+#          ${pbsready-import:output}
+extends = ${pbsready-import:output}
+
+parts +=
+# have to repeat the next one, as it's not inherited from pbsready-import
+    import-on-notification
+
+[directory]
+srv = $${buildout:directory}/srv
+www = $${:srv}/www/
+
+# Nothing to do for the import. Just dummy part that does nothing.
+# For httpd instance, PBS will directly push data to srv/www.
+# XXX-Cedric: is it safe?
+[exporter]
+wrapper = /usr/bin/true
+
+# XXX-Cedric: resilient uses "urls" part to publish, but apache doesn't.
+[urls]
+recipe = slapos.cookbook:publish
+
+
+# State that we want to push to srv/www directory, not srv/backup.
+[rdiff-backup-server]
+path = $${directory:www}

stack/lamp/apache/instance-apache-php.cfg.in

@@ -55,6 +55,7 @@ logrotate-backup = $${basedirectory:backup}/logrotate
 report = $${rootdirectory:etc}/report
 stunnel-conf = $${rootdirectory:etc}/stunnel
 xml-report = $${rootdirectory:var}/xml_report
+www = $${rootdirectory:srv}/www/
 
 [cadirectory]
 recipe = slapos.cookbook:mkdirectory
@@ -194,7 +195,7 @@ source = ${application:location}
 template = ${application-template:location}/${application-template:filename}
 configuration = ${application-configuration:location}
 
-htdocs = $${rootdirectory:srv}/www/
+htdocs = $${directory:www}
 pid-file = $${basedirectory:run}/apache.pid
 lock-file = $${basedirectory:run}/apache.lock
 ip = $${slap-network-information:global-ipv6}

stack/lamp/buildout.cfg

@@ -20,7 +20,9 @@ parts =
   instance-mariadb-import
   instance-mariadb-export
 
-  instance-default-root
+#Contains the importer and exporter recipes for apache
+  instance-apache-import
+  instance-apache-export
 
 
 extends =
@@ -64,44 +66,57 @@ strip-top-level-dir = true
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
 output = ${buildout:directory}/instance.cfg
-md5sum = 38bdcf0a8263d4a19bd6a35c0cd00340
+md5sum = 650cd2527158734fd6ccd9ec374b5e69
 mode = 0644
 
 [instance-apache-php]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in
 output = ${buildout:directory}/instance-apache-php.cfg
-md5sum = bf3219cd7d318e02e39c462fbb530715
+md5sum = 0538b05d27d8a64e89f898c09eb57716
 mode = 0644
 
-[instance-apache-backup]
+[instance-apache-import]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg.in
-output = ${buildout:directory}/instance-apache-backup.cfg
-md5sum = 48f969d82319a9d145570f5f0fd27672
+url = ${:_profile_base_location_}/apache/instance-apache-import.cfg.in
+output = ${buildout:directory}/instance-apache-import.cfg
+md5sum = 7c5bb18f1dccb38ad043c33e224798e6
 mode = 0644
 
-[template-resilient-lamp]
+[instance-apache-export]
 recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/template-resilient.cfg.in
-rendered = ${buildout:directory}/instance-resilient.cfg
+template = ${:_profile_base_location_}/apache/instance-apache-export.cfg.jinja2
+rendered = ${buildout:directory}/instance-apache-export.cfg
 
 context = key templateapache instance-apache-php:output
-          key dropbear dropbear:location
-          key buildout buildout:bin-directory
+          key templatepbsreadyexport pbsready-export:output
+
+import-list = file parts template-parts:destination
+              file replicated template-replicated:destination
+
+md5sum = 6e544e4d6f8093fee1724b231e1e8f34
+mode = 0644
+
+[instance-resilient]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
+rendered = ${buildout:directory}/instance-resilient.cfg
+
+context = key buildout buildout:bin-directory
+          key develop_eggs_directory buildout:develop-eggs-directory
+          key eggs_directory buildout:eggs-directory
 
 import-list = file parts template-parts:destination
               file replicated template-replicated:destination
 
-md5sum = 5605ad8896c2718854bf26148c4ae940
+md5sum = efcf17ce8e1544738a64cc4f7025cec3
 mode = 0644
- 
 
 [instance-mariadb]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/mariadb/instance-mariadb.cfg.in
 output = ${buildout:directory}/instance-mariadb.cfg
-md5sum = fafb873323c0084c522360f94ebe1df7
+md5sum = b214f56ede1a9d10f243c245c804d3a5
 mode = 0644
 
 [instance-mariadb-import]
@@ -119,14 +134,6 @@ md5sum = 0513bf25fbb20cccd57f285c40d96498
 mode = 0644
 
 
-[instance-default-root]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-default-root.cfg.in
-output = ${buildout:directory}/instance-default-root.cfg
-md5sum = 53c9020f7a0b5203f976e069e455787b
-mode = 0644
-
-
 #----------------
 #-- Dummy parts in case no application configuration file is needed
 

stack/lamp/instance-default-root.cfg.in

-[buildout]
-parts = request-apache
-
-[request-apache]
-<= slap-connection
-recipe = slapos.cookbook:request
-software-url = $${slap-connection:software-release-url}
-software-type = apache
-name = Apache
-

stack/lamp/instance-resilient.cfg.jinja2

+# vim: set ft=cfg:
+
+{% import 'parts' as parts %}
+{% import 'replicated' as replicated %}
+
+[buildout]
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
+parts +=
+  {{ parts.replicate("apache", "3") }}
+  publish-connection-informations
+
+{{ replicated.replicate("apache", "3", "apache-export", "apache-import") }}
+
+# Bubble up the parameters
+[request-apache]
+return = url ssh-public-key ssh-url notification-id ip url backend_url
+
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+backend_url = ${request-apache:connection-backend_url}
+url = ${request-apache:connection-url}
+

stack/lamp/instance.cfg.in

@@ -10,15 +10,12 @@ offline = true
 [switch_softwaretype]
 recipe = slapos.cookbook:softwaretype
 default = ${instance-apache-php:output}
-resilient = ${template-resilient-lamp:rendered}
+resilient = ${instance-resilient:rendered}
 mariadb = ${instance-mariadb:output}
 mariadb-import = ${instance-mariadb-import:output}
 mariadb-export = ${instance-mariadb-export:output}
-apache-backup = ${instance-apache-backup:output}
-
-# To prepend an empty root instance:
-#default = ${instance-default-root:output}
-#apache = ${instance-apache-php:output}
+apache-import = ${instance-apache-import:output}
+apache-export = ${instance-apache-export:rendered}
 
 #frozen creates a syntax error, meaning it can keep its data.
 #It's dirty as hell, it needs to be replaced.

stack/lamp/mariadb/instance-mariadb.cfg.in

@@ -14,8 +14,6 @@ parts =
   slapmonitor
   slapmonitor-xml
 
-gzip-binary = ${gzip:location}/bin/gzip
-
 # Define egg directories to be the one from Software Release
 # (/opt/slapgrid/...)
 eggs-directory = ${buildout:eggs-directory}
@@ -91,7 +89,7 @@ command = $${logrotate:wrapper}
 recipe = slapos.cookbook:logrotate
 # Binaries
 logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
-gzip-binary = $${buildout:gzip-binary}
+gzip-binary = ${gzip:location}/bin/gzip
 gunzip-binary = ${gzip:location}/bin/gunzip
 # Directories
 wrapper = $${rootdirectory:bin}/logrotate

stack/lamp/template-resilient.cfg.in

-{% import 'parts' as parts %}
-{% import 'replicated' as replicated %}
-
-[buildout]
-
-extends =
-   {{templateapache}}
-
-# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
-parts +=
-  {{ parts.replicate("mariadb","3") }}
-  request-apache-backup-1
-  request-apache-backup-2
-
-  request-pull-backup-server-apache-1
-  request-pull-backup-server-apache-backup-1
-
-  request-pull-backup-server-apache-2
-  request-pull-backup-server-apache-backup-2
-
-  publish-connection-informations
-  apache-php
-  stunnel
-  certificate-authority
-  ca-stunnel
-  logrotate
-  logrotate-entry-apache
-  logrotate-entry-stunnel
-  cron
-  cron-entry-logrotate
-  dropbear-server
-  sshkeys-authority
-  dropbear-server-pbs-authorized-key
-
-  request-pull-backup-server
-
-{{ replicated.replicate("mariadb", "3", "mariadb-export", "mariadb-import") }}
-
-
-[request-pull-backup-server]
-<= request-pbs-common
-name = PBS (Pull Backup Server)
-return = ssh-key notification-url feeds-url
-slave = false
-
-[sshkeys-directory]
-recipe = slapos.cookbook:mkdirectory
-requests = ${directory:sshkeys}/requests
-keys = ${directory:sshkeys}/keys
-
-[sshkeys-authority]
-recipe = slapos.cookbook:sshkeys_authority
-request-directory = ${sshkeys-directory:requests}
-keys-directory = ${sshkeys-directory:keys}
-wrapper = ${basedirectory:services}/sshkeys_authority
-keygen-binary = {{dropbear}}/bin/dropbearkey
-
-[sshkeys-dropbear]
-<= sshkeys-authority
-recipe = slapos.cookbook:sshkeys_authority.request
-name = dropbear
-type = rsa
-executable = ${dropbear-server:wrapper}
-public-key = ${dropbear-server:rsa-keyfile}.pub
-private-key = ${dropbear-server:rsa-keyfile}
-wrapper = ${basedirectory:services}/sshd
-
-[dropbear-server]
-recipe = slapos.cookbook:dropbear
-host = ${slap-network-information:global-ipv6}
-port = 2222
-home = ${directory:ssh}
-wrapper = ${rootdirectory:bin}/raw_sshd
-shell = ${rdiff-backup-server:wrapper}
-rsa-keyfile = ${directory:ssh}/server_key.rsa
-dropbear-binary = {{dropbear}}/sbin/dropbear
-
-[dropbear-server-pbs-authorized-key]
-<= dropbear-server
-recipe = slapos.cookbook:dropbear.add_authorized_key
-key = ${request-pull-backup-server:connection-ssh-key}
-
-[rdiff-backup-server]
-<= apache-php
-recipe = slapos.cookbook:pbs
-client = false
-path = ${apache-php:htdocs}
-wrapper = ${rootdirectory:bin}/rdiffbackup-server
-rdiffbackup-binary = {{buildout}}/rdiff-backup
-
-[request-apache-backup-1]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = Apache Backup 1
-software-url = ${slap-connection:software-release-url}
-software-type = apache-backup
-return = url ssh-url ssh-public-key
-config = authorized-key proxy-url
-config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
-config-proxy-url = ${publish-connection-informations:url}
-
-[request-apache-backup-2]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = Apache Backup 2
-software-url = ${slap-connection:software-release-url}
-software-type = apache-backup
-return = url ssh-url ssh-public-key
-config = authorized-key proxy-url
-config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
-config-proxy-url = ${publish-connection-informations:url}
-
-[request-pull-backup-server-apache-1]
-<= request-pbs-common
-name = PBS pulling from Apache 1
-config = url name type server-key notify notification-id frequency
-config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
-config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
-config-type = pull
-config-server-key = ${sshkeys-dropbear:public-key-value}
-config-notify = ${request-pull-backup-server:connection-notification-url}
-config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
-config-frequency = 30 * * * *
-slave = true
-sla = instance_guid
-sla-instance_guid = ${request-pull-backup-server:instance_guid}
-
-[request-pull-backup-server-apache-2]
-<= request-pbs-common
-name = PBS pulling from Apache 2
-config = url name type server-key notify notification-id frequency
-config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
-config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
-config-type = pull
-config-server-key = ${sshkeys-dropbear:public-key-value}
-config-notify = ${request-pull-backup-server:connection-notification-url}
-config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
-config-frequency = 30 * * * *
-slave = true
-sla = instance_guid
-sla-instance_guid = ${request-pull-backup-server:instance_guid}
-
-
-[request-pull-backup-server-apache-backup-1]
-<= request-pbs-common
-name = PBS pushing to ${request-apache-backup-1:name}
-config = url name type server-key on-notification
-config-url = ${request-apache-backup-1:connection-ssh-url}
-config-name = ${request-pull-backup-server-apache-1:config-name}
-config-type = push
-config-server-key = ${request-apache-backup-1:connection-ssh-public-key}
-config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id}
-slave = true
-sla = instance_guid
-sla-instance_guid = ${request-pull-backup-server:instance_guid}
-
-[request-pull-backup-server-apache-backup-2]
-<= request-pbs-common
-name = PBS pushing to ${request-apache-backup-2:name}
-config = url name type server-key on-notification
-config-url = ${request-apache-backup-2:connection-ssh-url}
-config-name = ${request-pull-backup-server-apache-2:config-name}
-config-type = push
-config-server-key = ${request-apache-backup-2:connection-ssh-public-key}
-config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id}
-slave = true
-sla = instance_guid
-sla-instance_guid = ${request-pull-backup-server:instance_guid}
-
-
-[directory]
-ssh = ${rootdirectory:etc}/ssh
-sshkeys = ${rootdirectory:srv}/sshkeys

stack/lapp/buildout.cfg

@@ -50,7 +50,7 @@ strip-top-level-dir = true
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
 output = ${buildout:directory}/instance.cfg
-md5sum = 1aaf3ea7b14e09e66904bdb80e3cfe2f
+md5sum = 7e09e857ea14fee4a43a771afeafb677
 mode = 0644
 
 [instance-apache-php]
@@ -67,9 +67,9 @@ output = ${buildout:directory}/instance-apache-backup.cfg
 md5sum = db879141c0b6a77ef8b3b7e699f5583a
 mode = 0644
 
-[template-resilient-lapp]
+[instance-resilient]
 recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/template-resilient.cfg.in
+template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
 rendered = ${buildout:directory}/instance-resilient.cfg
 
 context = key templateapache instance-apache-php:output
@@ -79,7 +79,7 @@ context = key templateapache instance-apache-php:output
 import-list = file parts template-parts:destination
               file replicated template-replicated:destination
 
-md5sum = 525f50e60d0a96557a552de6afa4ab88
+md5sum = 46c7d8f691bd37d84e0bd03b83e51d14
 mode = 0644
  
 

stack/lapp/template-resilient.cfg.in → stack/lapp/instance-resilient.cfg.jinja2

+# vim: set ft=cfg:
+
 {% import 'parts' as parts %}
 {% import 'replicated' as replicated %}
 

stack/lapp/instance.cfg.in

@@ -11,7 +11,7 @@ offline = true
 recipe = slapos.cookbook:softwaretype
 default = ${instance-default-root:output}
 apache = ${instance-apache-php:output}
-resilient = ${template-resilient-lapp:rendered}
+resilient = ${instance-resilient:rendered}
 postgres = ${instance-postgres:output}
 postgres-import = ${instance-postgres-import:output}
 postgres-export = ${instance-postgres-export:output}

stack/resilient/buildout.cfg

@@ -22,7 +22,7 @@ parts =
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready.cfg.in
 output = ${buildout:directory}/pbsready.cfg
-md5sum = e602f6746cbe5a1da82b90097719c4a1
+md5sum = d337ecde38c368a17a74c647b2598b29
 mode = 0644
 
 [pbsready-import]
@@ -53,7 +53,7 @@ mode = 0644
 [template-replicated]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/template-replicated.cfg.in
-md5sum = 63b5649f3cf1c9a77315382793d9593f
+md5sum = 15ef411b2678fd2a14e0a4ccb6520310
 mode = 0644
 destination = ${buildout:directory}/template-replicated.cfg.in
 

stack/resilient/pbsready.cfg.in

@@ -3,11 +3,7 @@
 parts +=
   resiliency
   urls
-  stunnel
-  certificate-authority
-  ca-stunnel
   logrotate
-  logrotate-entry-stunnel
   logrotate-entry-cron
   logrotate-entry-equeue
   cron
@@ -23,17 +19,110 @@ parts +=
 #--
 #-- Creation of all needed directories.
 
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+
 [basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/service
+run = $${rootdirectory:var}/run
+script = $${rootdirectory:etc}/script
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
 services = $${rootdirectory:etc}/service
 cache = $${rootdirectory:var}/cache
 notifier = $${rootdirectory:etc}/notifier
 
 [directory]
+recipe = slapos.cookbook:mkdirectory
 backup = $${basedirectory:backup}/$${slap-parameter:namebase}
 ssh = $${rootdirectory:etc}/ssh/
 sshkeys = $${rootdirectory:srv}/sshkeys
 notifier-feeds = $${basedirectory:notifier}/feeds
 notifier-callbacks = $${basedirectory:notifier}/callbacks
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+cronstamps = $${rootdirectory:etc}/cronstamps
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+logrotate-backup = $${basedirectory:backup}/logrotate
+
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+
+#----------------
+#--
+#-- Deploy logrotate.
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+[logrotate-entry-mariadb]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = mariadb
+log = $${mariadb:error-log}
+frequency = daily
+rotate-num = 30
+post = $${mariadb:logrotate-post}
+sharedscripts = true
+notifempty = true
+create = true
+
+[logrotate-entry-cron]
+<= logrotate
+recipe =slapos.cookbook:logrotate.d
+name = crond
+log = $${cron-simplelogger:log}
+frequency = daily
+rotate-num = 30
+notifempty = true
+create = true
+
+[logrotate-entry-equeue]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = equeue
+log = $${equeue:log}
+frequency = daily
+rotate-num = 30
 
 
 #----------------
@@ -59,6 +148,7 @@ etc = $${rootdirectory:etc}
 [urls]
 ssh-public-key = $${sshkeys-dropbear:public-key-value}
 ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
+ip = $${slap-network-information:global-ipv6}
 
 [rdiff-backup-server]
 recipe = slapos.cookbook:pbs
@@ -94,14 +184,6 @@ wrapper = $${basedirectory:services}/notifier
 server-binary = ${buildout:bin-directory}/pubsubserver
 notifier-binary = ${buildout:bin-directory}/pubsubnotifier
 
-[logrotate-entry-equeue]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = equeue
-log = $${equeue:log}
-frequency = daily
-rotate-num = 30
-
 
 #----------------
 #--

stack/resilient/template-replicated.cfg.in

@@ -2,7 +2,9 @@
 
 ## Tells the Backupable recipe that we want a backup
 [resilient]
+recipe = slapos.cookbook:request
 config-namebase = {{namebase}}
+software-url = ${slap-connection:software-release-url}
 
 ## Every request is double to provide the 3 IPs.
 [request-{{namebase}}]
@@ -11,7 +13,7 @@ config-namebase = {{namebase}}
    {{heriteLeader}}
 software-type = {{typeexport}}
 name = {{namebase}}0
-return = url ssh-public-key ssh-url notification-id ip
+return = ssh-public-key ssh-url notification-id ip
 
 config = number authorized-key notify ip-list namebase
 config-number = 0
@@ -30,7 +32,7 @@ name = {{namebase}}{{id}}
 
 software-url = ${slap-connection:software-release-url}
 software-type = {{typeimport}}
-return = url ssh-public-key ssh-url notification-url ip
+return = ssh-public-key ssh-url notification-url ip
 
 pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
 
@@ -56,7 +58,7 @@ name = {{namebase}}0
 
 software-url = ${slap-connection:software-release-url}
 software-type = {{typeexport}}
-return = url ssh-public-key ssh-url notification-id ip
+return = ssh-public-key ssh-url notification-id ip
 
 config = number authorized-key notify ip-list namebase
 config-number = 0
@@ -75,7 +77,7 @@ name = {{namebase}}{{id}}
 
 software-url = ${slap-connection:software-release-url}
 software-type = {{typeimport}}
-return = url ssh-public-key ssh-url notification-url
+return = ssh-public-key ssh-url notification-url
 
 pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push