Publish notification based lamp stack.

stack/lamp/buildout.cfg

@@ -90,7 +90,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-backuped.cfg
 output = ${buildout:directory}/template-backuped.cfg
-md5sum = 4190baa5e3f87a02e8721aeac3b70942
+md5sum = 0fedb9ed84137b2d5518ffbaaa6d76eb
 mode = 0644
 
 [template-use-backup]
@@ -111,34 +111,35 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-mariadb-pbsready.cfg
 output = ${buildout:directory}/template-mariadb-pbsready.cfg
-md5sum = cfc87bb3bb1d4aa4ffd11e4b6dfcabe7
+md5sum = d2a580dcd7efdd528be45c5ffadfe760
 mode = 0644
 
 [template-mariadb-pbsready-import]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-mariadb-pbsready-import.cfg
 output = ${buildout:directory}/template-mariadb-pbsready-import.cfg
-md5sum = 483796d0e0ebdc86fb6fe6589bb8ed7e
+md5sum = 4a96ff02da3898fef7077fa8baec81ac
 mode = 0644
 
 [template-mariadb-pbsready-export]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-mariadb-pbsready-export.cfg
 output = ${buildout:directory}/template-mariadb-pbsready-export.cfg
-md5sum = 112f259b613eefa94f8cc83a0e2aa9f3
+md5sum = 11a9e45e8bc590bc11bfdd304b07a4a5
 mode = 0644
 
 [template-pull-backup]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-pull-backup.cfg
 output = ${buildout:directory}/template-pull-backup.cfg
-md5sum = 46bd46a0b46f39937ab3d604eeb20441
+md5sum = c7099cdf45605da4e9595c6e7b913327
 mode = 0644
 
 [eggs]
 recipe = zc.recipe.egg
 eggs =
   ${lxml-python:egg}
+  slapos.toolbox
 
 [versions]
 setuptools = 0.6c12dev-r88795

stack/lamp/instance-backuped.cfg

@@ -26,14 +26,15 @@ recipe = slapos.cookbook:request
 name = PBS (Pull Backup Server)
 software-url = $${slap-connection:software-release-url}
 software-type = pull-backup
-return = ssh-key
+return = ssh-key notification-url feeds-url
 slave = false
 
 [request-mariadb]
 software-type = mariadb-pbsready-export
-config = authorized-key
+config = authorized-key notify
 config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
-return = url ssh-public-key ssh-url
+config-notify = $${request-pull-backup-server:connection-notification-url}
+return = url ssh-public-key ssh-url notification-id
 
 [request-mariadb-pseudo-replicating]
 <= slap-connection
@@ -41,30 +42,38 @@ recipe = slapos.cookbook:request
 name = MariaDB (backup)
 software-url = $${slap-connection:software-release-url}
 software-type = mariadb-pbsready-import
-return = url ssh-public-key ssh-url
-config = authorized-key
+return = url ssh-public-key ssh-url notification-url
+config = authorized-key on-notification
 config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
+pbs-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb-push
+config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${:pbs-notification-id}
 
 [request-pull-backup-server-mariadb]
 <= request-pull-backup-server
    slap-connection
 name = PBS pulling from MariaDB (backuped)
-config = url name type server-key frequency
+config = url name type server-key on-notification notify notification-id title
 config-url = $${request-mariadb:connection-ssh-url}
 config-name = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb
 config-type = pull
 config-server-key = $${request-mariadb:connection-ssh-public-key}
-config-frequency = 15 0 * * *
+config-on-notification = $${request-mariadb:connection-notification-id}
+config-notify = $${request-pull-backup-server:connection-notification-url}
+config-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb-pull
+config-title = Pulling from MariaDB
 slave = true
 
 [request-pull-backup-server-mariadb-backup]
 <= request-pull-backup-server
    slap-connection
 name = PBS pushing on MariaDB (ready to use backup)
-config = url name type server-key frequency
+config = url name type server-key on-notification notify notification-id title
 config-url = $${request-mariadb-pseudo-replicating:connection-ssh-url}
 config-name = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb
 config-type = push
 config-server-key = $${request-mariadb-pseudo-replicating:connection-ssh-public-key}
-config-frequency = 45 0 * * *
+config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${request-pull-backup-server-mariadb:config-notification-id}
+config-notify = $${request-mariadb-pseudo-replicating:connection-notification-url}
+config-notification-id = $${request-mariadb-pseudo-replicating:pbs-notification-id}
+config-title = Pushing to MariaDB backup
 slave = true

stack/lamp/instance-mariadb-pbsready-export.cfg

@@ -3,6 +3,9 @@ extends = ${template-mariadb-pbsready:output}
 
 parts += cron-entry-mariadb-backup
 
+[urls]
+notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-mydumper:name}
+
 [mydumper]
 recipe = slapos.cookbook:mydumper
 wrapper = $${rootdirectory:bin}/raw_mydumper
@@ -13,15 +16,18 @@ mydumper-binary = ${mydumper:location}/bin/mydumper
 database = $${mariadb:database}
 import = false
 
-[mydumper-locked]
-recipe = slapos.cookbook:lockfile
-lock-file = $${basedirectory:run}/mysql-backup.lock
-binary = $${mydumper:wrapper}
+[notifier-mydumper]
+<= notifier
+recipe = slapos.cookbook:notifier.notify
+name = mydumper
+title = Dumping MariaDB Database
+executable = $${mydumper:wrapper}
 wrapper = $${rootdirectory:bin}/mydumper
+notify = $${slap-parameter:notify}
 
 [cron-entry-mariadb-backup]
 <= cron
 recipe = slapos.cookbook:cron.d
 name = backup
 frequency = 0 * * * *
-command = $${mydumper-locked:wrapper}
+command = $${notifier-mydumper:wrapper}

stack/lamp/instance-mariadb-pbsready-import.cfg

@@ -2,27 +2,14 @@
 
 extends = ${template-mariadb-pbsready:output}
 
-parts = 
-  urls
-  mariadb
-  stunnel
-  certificate-authority
-  ca-stunnel
-  logrotate
-  logrotate-entry-mariadb
-  logrotate-entry-stunnel
-  logrotate-entry-cron
-  cron
-  cron-entry-logrotate
-  sshkeys-authority
-  dropbear-server
-  sshkeys-dropbear
-  dropbear-server-pbs-authorized-key
-  cron-entry-mariadb-import
+parts += mariadb-import-on-notification
+
+[urls]
+notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
 
 [mydumper-import]
 recipe = slapos.cookbook:mydumper
-wrapper = $${rootdirectory:bin}/raw_myloader
+wrapper = $${rootdirectory:bin}/myloader
 backup-directory = $${directory:mariadb-backup}
 socket = $${mariadb:socket}
 user = root
@@ -30,16 +17,8 @@ myloader-binary = ${mydumper:location}/bin/myloader
 database = $${mariadb:database}
 import = true
 
-[mydumper-import-locked]
-recipe = slapos.cookbook:lockfile
-lock-file = $${basedirectory:run}/import.lock
-wrapper = $${rootdirectory:bin}/myloader
-binary = $${mydumper-import:wrapper}
-wait = true
-
-[cron-entry-mariadb-import]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = mariadb_import
-frequency = 0 * * * *
-command = $${mydumper-import-locked:wrapper}
+[mariadb-import-on-notification]
+<= notifier
+recipe = slapos.cookbook:notifier.callback
+on-notification-id = $${slap-parameter:on-notification}
+callback = $${mydumper-import:wrapper}

stack/lamp/instance-mariadb-pbsready.cfg

@@ -12,12 +12,14 @@ parts =
   logrotate-entry-mariadb
   logrotate-entry-stunnel
   logrotate-entry-cron
+  logrotate-entry-equeue
   cron
   cron-entry-logrotate
   sshkeys-authority
   dropbear-server
   sshkeys-dropbear
   dropbear-server-pbs-authorized-key
+  notifier
 
 [urls]
 ssh-public-key = $${sshkeys-dropbear:public-key-value}
@@ -61,16 +63,48 @@ recipe = slapos.cookbook:dropbear.add_authorized_key
 key = $${slap-parameter:authorized-key}
 
 [rdiff-backup-server]
-recipe = slapos.cookbook:rdiffbackup
+recipe = slapos.cookbook:pbs
 client = false
 path = $${directory:mariadb-backup}
 wrapper = $${rootdirectory:bin}/rdiffbackup-server
 rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
 
+[logrotate-entry-equeue]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = equeue
+log = $${equeue:log}
+frequency = daily
+rotate-num = 30
+
+[equeue]
+recipe = slapos.cookbook:equeue
+socket = $${basedirectory:run}/equeue.sock
+log = $${basedirectory:log}/equeue.log
+database = $${rootdirectory:srv}/equeue.db
+wrapper = $${basedirectory:services}/equeue
+equeue-binary = ${buildout:bin-directory}/equeue
+
+[notifier]
+recipe = slapos.cookbook:notifier
+feeds = $${directory:notifier-feeds}
+callbacks = $${directory:notifier-callbacks}
+id-file = $${rootdirectory:etc}/notifier.id
+equeue-socket = $${equeue:socket}
+host = $${slap-network-information:global-ipv6}
+port = 8080
+wrapper = $${basedirectory:services}/notifier
+server-binary = ${buildout:bin-directory}/pubsubserver
+notifier-binary = ${buildout:bin-directory}/pubsubnotifier
+
+
 [basedirectory]
 cache = $${rootdirectory:var}/cache/
+notifier = $${rootdirectory:etc}/notifier/
 
 [directory]
 mariadb-backup = $${basedirectory:backup}/mariadb/
 ssh = $${rootdirectory:etc}/ssh/
 sshkeys = $${rootdirectory:srv}/sshkeys
+notifier-feeds = $${basedirectory:notifier}/feeds/
+notifier-callbacks = $${basedirectory:notifier}/callbacks/

stack/lamp/instance-pull-backup.cfg

@@ -2,7 +2,7 @@
 
 parts =
     connection-dict
-    rdiff-backup
+    pbs
     cron
     cron-entry-logrotate
     logrotate
@@ -16,6 +16,28 @@ offline = true
 [connection-dict]
 recipe = slapos.cookbook:publish
 ssh-key = $${sshkeys-dropbear:public-key-value}
+notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
+feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/
+
+[equeue]
+recipe = slapos.cookbook:equeue
+socket = $${basedirectory:run}/equeue.sock
+log = $${basedirectory:log}/equeue.log
+database = $${rootdirectory:srv}/equeue.db
+wrapper = $${basedirectory:services}/equeue
+equeue-binary = ${buildout:bin-directory}/equeue
+
+[notifier]
+recipe = slapos.cookbook:notifier
+feeds = $${directory:notifier-feeds}
+callbacks = $${directory:notifier-callbacks}
+id-file = $${rootdirectory:etc}/notifier.id
+equeue-socket = $${equeue:socket}
+host = $${slap-network-information:global-ipv6}
+port = 8080
+wrapper = $${basedirectory:services}/notifier
+server-binary = ${buildout:bin-directory}/pubsubserver
+notifier-binary = ${buildout:bin-directory}/pubsubnotifier
 
 [dropbear-client]
 recipe = slapos.cookbook:dropbear.client
@@ -39,23 +61,25 @@ wrapper = $${basedirectory:services}/sshkeys_authority
 [sshkeys-dropbear]
 <= sshkeys-authority
 recipe = slapos.cookbook:sshkeys_authority.request
-name = rdiff-backup
+name = pbs
 type = rsa
 executable = $${dropbear-client:wrapper}
 public-key = $${dropbear-client:identity-file}.pub
 private-key = $${dropbear-client:identity-file}
 wrapper = $${rootdirectory:bin}/do_backup
 
-[rdiff-backup]
-recipe = slapos.cookbook:rdiffbackup
+[pbs]
+<= notifier
+recipe = slapos.cookbook:pbs
 client = true
 rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
 sshclient-binary = $${dropbear-client:wrapper}
 known-hosts = $${directory:dot-ssh}/known_hosts
 promises-directory = $${basedirectory:promises}
-directory = $${directory:backups}
+directory = $${directory:pbs-backup}
 cron-entries = $${cron:cron-entries}
 wrappers-directory = $${directory:pbs-wrappers}
+notifier-url = http://[$${notifier:host}]:$${notifier:port}/
 
 [cron-simplelogger]
 recipe = slapos.cookbook:simplelogger
@@ -91,6 +115,23 @@ logrotate-entries = $${directory:logrotate-entries}
 backup = $${directory:logrotate-backup}
 state-file = $${rootdirectory:srv}/logrotate.status
 
+[logrotate-entry-equeue]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = equeue
+log = $${equeue:log}
+frequency = daily
+rotate-num = 30
+
+[logrotate-entry-equeue]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = cron
+log = $${cron-simplelogger:log}
+frequency = daily
+rotate-num = 30
+
+
 [rootdirectory]
 recipe = slapos.cookbook:mkdirectory
 etc = $${buildout:directory}/etc/
@@ -113,17 +154,18 @@ cronstamps = $${rootdirectory:etc}/cronstamps/
 cron-entries = $${rootdirectory:etc}/cron.d/
 crontabs = $${rootdirectory:etc}/crontabs/
 cronoutput = $${basedirectory:log}/cron-ouput/
-rdiff-backup-backup = $${basedirectory:backup}/rdiff-backup/
+pbs-backup = $${basedirectory:backup}/pbs/
 logrotate-entries = $${rootdirectory:etc}/logrotate.d/
 logrotate-backup = $${basedirectory:backup}/logrotate/
-rdiff-backup = $${rootdirectory:etc}/rdiff-backup/
 sshkeys = $${rootdirectory:srv}/sshkeys
-backups = $${basedirectory:backup}/backups/
-pbs-wrappers = $${rootdirectory:bin}/rdiff-backup/
+pbs-wrappers = $${rootdirectory:bin}/pbs/
 dot-ssh = $${basedirectory:ssh-home}/.ssh/
+notifier-feeds = $${basedirectory:notifier}/feeds/
+notifier-callbacks = $${basedirectory:notifier}/callbacks/
 
 [basedirectory]
 ssh-home = $${rootdirectory:home}/ssh
+notifier = $${rootdirectory:etc}/notifier/
 
 [rootdirectory]
 home = $${buildout:directory}/home/