implemented new instance structure from lamp stack

stack/lapp/apache/instance-apache-export.cfg.jinja2

+# This file is responsible of three things:
+# 1/ Act as "Apache exporter"
+# 2/ Act as "Mariadb backup infrastructure requester"
+# 3/ Act as "Apache" instance
+
+{% import 'parts' as parts %}
+{% import 'replicated' as replicated %}
+
+[buildout]
+extends = {{templateapache}}
+          {{templatepbsreadyexport}}
+
+parts +=
+    {{ parts.replicate("mariadb", "3") }}
+
+# Repeating parts from instance-apache-php.
+# XXX-Cedric: how to simplify this?
+    certificate-authority
+    ca-stunnel
+    logrotate
+    logrotate-entry-apache
+    logrotate-entry-stunnel
+    cron
+    cron-entry-logrotate
+    promise
+    frontend-promise
+    content-promise
+    publish-connection-informations
+
+{{ replicated.replicate("mariadb", "3", "mariadb-export", "mariadb-import") }}
+
+# Nothing to do for the exporter. Just dummy part that does nothing.
+# For httpd instance, PBS will directly pull data from srv/www.
+# XXX-Cedric: write a real backup system.
+[exporter]
+wrapper = /bin/true
+
+# State that we want to backup srv/www directory, not srv/backup.
+# XXX-Cedric: works well, but doesn't work with big data.
+[rdiff-backup-server]
+path = ${directory:www}
+
+# Add "exporter" parameters to list of published connection parameters
+[publish-connection-informations]
+# XXX-Cedric: Long term goal: could be a recipe that requests an instance and
+# bubbles ALL
+# parameters of the requested instance. Requirement: aggregated publish.
+<= resilient-publish-connection-parameter
+
+# XXX-Cedric: resilient overwrites what's returned from request-mariadb
+# XXX-Cedric: change the request method to return everything from
+#             getConnectionParameterDict()
+[request-mariadb]
+return = ssh-public-key ssh-url notification-id ip url

stack/lapp/apache/instance-apache-import.cfg.in

+[buildout]
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+extends = ${pbsready-import:output}
+
+[directory]
+srv = $${buildout:directory}/srv
+www = $${:srv}/www/
+
+# Nothing to do for the import. Just dummy part that does nothing.
+# For httpd instance, PBS will directly push data to srv/www.
+# XXX-Cedric: write a real backup system.
+[importer]
+wrapper = /bin/true
+
+# State that we want to push to srv/www directory, not srv/backup.
+[rdiff-backup-server]
+path = $${directory:www}

stack/lapp/apache/instance-apache-php.cfg.in

@@ -57,6 +57,7 @@ logrotate-backup = $${basedirectory:backup}/logrotate
 report = $${rootdirectory:etc}/report
 stunnel-conf = $${rootdirectory:etc}/stunnel
 xml-report = $${rootdirectory:var}/xml_report
+www = $${rootdirectory:srv}/www/
 
 [cadirectory]
 recipe = slapos.cookbook:mkdirectory
@@ -195,7 +196,7 @@ url = $${request-postgres:connection-url}
 recipe = slapos.cookbook:apachephp
 source = ${application:location}
 
-htdocs = $${rootdirectory:srv}/www/
+htdocs = $${directory:www}
 pid-file = $${basedirectory:run}/apache.pid
 lock-file = $${basedirectory:run}/apache.lock
 ip = $${slap-network-information:global-ipv6}

stack/lapp/buildout.cfg

 [buildout]
+
+ignore-existing = true
+
 parts =
   slapos-cookbook
   apache-php-postgres
@@ -15,11 +18,14 @@ parts =
   instance-postgres-import
   instance-postgres-export
 
+#Contains the importer and exporter recipes for apache
+  instance-apache-import
+  instance-apache-export
+
 
 extends =
-  ../resilient/buildout.cfg
-  ../../component/apache/buildout.cfg
   ../../component/apache-php/buildout.cfg
+  ../../component/apache/buildout.cfg
   ../../component/dash/buildout.cfg
   ../../component/dcron/buildout.cfg
   ../../component/gzip/buildout.cfg
@@ -29,6 +35,7 @@ extends =
   ../../component/rdiff-backup/buildout.cfg
   ../../component/stunnel/buildout.cfg
   ../../component/dropbear/buildout.cfg
+  ../resilient/buildout.cfg
   ../slapos.cfg
 
 
@@ -48,21 +55,35 @@ strip-top-level-dir = true
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
 output = ${buildout:directory}/instance.cfg
-md5sum = a482fa0e72839b4bd75b169ac1460d64
+md5sum = 25d07b5101d5f566398686642ada4cee
 mode = 0644
 
 [instance-apache-php]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in
 output = ${buildout:directory}/instance-apache-php.cfg
-md5sum = 72b70452d1c077cfcd0f268181506b8e
+md5sum = 823257dda6f3068a38c6b69c771cf307
 mode = 0644
 
-[instance-apache-backup]
+[instance-apache-import]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg.in
-output = ${buildout:directory}/instance-apache-backup.cfg
-md5sum = db879141c0b6a77ef8b3b7e699f5583a
+url = ${:_profile_base_location_}/apache/instance-apache-import.cfg.in
+output = ${buildout:directory}/instance-apache-import.cfg
+md5sum = f1dc2a71d362b5d2d36481ffefdd2293
+mode = 0644
+
+[instance-apache-export]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/apache/instance-apache-export.cfg.jinja2
+rendered = ${buildout:directory}/instance-apache-export.cfg
+
+context = key templateapache instance-apache-php:output
+          key templatepbsreadyexport pbsready-export:output
+
+import-list = file parts template-parts:destination
+              file replicated template-replicated:destination
+
+md5sum = 4704f2788f096c7494694db72a9f6193
 mode = 0644
 
 [instance-resilient]
@@ -70,16 +91,15 @@ recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
 rendered = ${buildout:directory}/instance-resilient.cfg
 
-context = key templateapache instance-apache-php:output
-          key dropbear dropbear:location
-          key buildout buildout:bin-directory
+context = key buildout buildout:bin-directory
+          key develop_eggs_directory buildout:develop-eggs-directory
+          key eggs_directory buildout:eggs-directory
 
 import-list = file parts template-parts:destination
               file replicated template-replicated:destination
 
-md5sum = 46c7d8f691bd37d84e0bd03b83e51d14
+md5sum = 224a7e73db84168cd5aedec10ddd50f2
 mode = 0644
- 
 
 [instance-postgres]
 recipe = slapos.recipe.template
@@ -155,7 +175,6 @@ eggs =
   ${psycopg2:egg}
   slapos.toolbox
 
-
 [networkcache]
 # Romain Courteaud + Sebastien Robin + Alain Takoudjou
 # + Cedric de Saint Martin signature certificate

stack/lapp/instance-resilient.cfg.jinja2

@@ -4,172 +4,44 @@
 {% import 'replicated' as replicated %}
 
 [buildout]
-
-extends =
-   {{templateapache}}
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
 
 # += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
 parts +=
-  {{ parts.replicate("postgres","3") }}
-  request-apache-backup-1
-  request-apache-backup-2
-
-  request-pull-backup-server-apache-1
-  request-pull-backup-server-apache-backup-1
-
-  request-pull-backup-server-apache-2
-  request-pull-backup-server-apache-backup-2
-
+  {{ parts.replicate("apache", "3") }}
   publish-connection-informations
-  apache-php
-  stunnel
-  certificate-authority
-  ca-stunnel
-  logrotate
-  logrotate-entry-apache
-  logrotate-entry-stunnel
-  cron
-  cron-entry-logrotate
-  dropbear-server
-  sshkeys-authority
-  dropbear-server-pbs-authorized-key
-
-  request-pull-backup-server
-
-{{ replicated.replicate("postgres", "3", "postgres-export", "postgres-import") }}
-
-
-[request-pull-backup-server]
-<= request-pbs-common
-name = PBS (Pull Backup Server)
-return = ssh-key notification-url feeds-url
-slave = false
-
-[sshkeys-directory]
-recipe = slapos.cookbook:mkdirectory
-requests = ${directory:sshkeys}/requests
-keys = ${directory:sshkeys}/keys
-
-[sshkeys-authority]
-recipe = slapos.cookbook:sshkeys_authority
-request-directory = ${sshkeys-directory:requests}
-keys-directory = ${sshkeys-directory:keys}
-wrapper = ${basedirectory:services}/sshkeys_authority
-keygen-binary = {{dropbear}}/bin/dropbearkey
-
-[sshkeys-dropbear]
-<= sshkeys-authority
-recipe = slapos.cookbook:sshkeys_authority.request
-name = dropbear
-type = rsa
-executable = ${dropbear-server:wrapper}
-public-key = ${dropbear-server:rsa-keyfile}.pub
-private-key = ${dropbear-server:rsa-keyfile}
-wrapper = ${basedirectory:services}/sshd
-
-[dropbear-server]
-recipe = slapos.cookbook:dropbear
-host = ${slap-network-information:global-ipv6}
-port = 2222
-home = ${directory:ssh}
-wrapper = ${rootdirectory:bin}/raw_sshd
-shell = ${rdiff-backup-server:wrapper}
-rsa-keyfile = ${directory:ssh}/server_key.rsa
-dropbear-binary = {{dropbear}}/sbin/dropbear
-
-[dropbear-server-pbs-authorized-key]
-<= dropbear-server
-recipe = slapos.cookbook:dropbear.add_authorized_key
-key = ${request-pull-backup-server:connection-ssh-key}
-
-[rdiff-backup-server]
-<= apache-php
-recipe = slapos.cookbook:pbs
-client = false
-path = ${apache-php:htdocs}
-wrapper = ${rootdirectory:bin}/rdiffbackup-server
-rdiffbackup-binary = {{buildout}}/rdiff-backup
-
-[request-apache-backup-1]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = Apache Backup 1
-software-url = ${slap-connection:software-release-url}
-software-type = apache-backup
-return = url ssh-url ssh-public-key
-config = authorized-key proxy-url
-config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
-config-proxy-url = ${publish-connection-informations:url}
-
-[request-apache-backup-2]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = Apache Backup 2
-software-url = ${slap-connection:software-release-url}
-software-type = apache-backup
-return = url ssh-url ssh-public-key
-config = authorized-key proxy-url
-config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
-config-proxy-url = ${publish-connection-informations:url}
-
-[request-pull-backup-server-apache-1]
-<= request-pbs-common
-name = PBS pulling from Apache 1
-config = url name type server-key notify notification-id frequency
-config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
-config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
-config-type = pull
-config-server-key = ${sshkeys-dropbear:public-key-value}
-config-notify = ${request-pull-backup-server:connection-notification-url}
-config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
-config-frequency = 30 * * * *
-slave = true
-sla = instance_guid
-sla-instance_guid = ${request-pull-backup-server:instance_guid}
-
-[request-pull-backup-server-apache-2]
-<= request-pbs-common
-name = PBS pulling from Apache 2
-config = url name type server-key notify notification-id frequency
-config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
-config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
-config-type = pull
-config-server-key = ${sshkeys-dropbear:public-key-value}
-config-notify = ${request-pull-backup-server:connection-notification-url}
-config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
-config-frequency = 30 * * * *
-slave = true
-sla = instance_guid
-sla-instance_guid = ${request-pull-backup-server:instance_guid}
-
-
-[request-pull-backup-server-apache-backup-1]
-<= request-pbs-common
-name = PBS pushing to ${request-apache-backup-1:name}
-config = url name type server-key on-notification
-config-url = ${request-apache-backup-1:connection-ssh-url}
-config-name = ${request-pull-backup-server-apache-1:config-name}
-config-type = push
-config-server-key = ${request-apache-backup-1:connection-ssh-public-key}
-config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id}
-slave = true
-sla = instance_guid
-sla-instance_guid = ${request-pull-backup-server:instance_guid}
-
-[request-pull-backup-server-apache-backup-2]
-<= request-pbs-common
-name = PBS pushing to ${request-apache-backup-2:name}
-config = url name type server-key on-notification
-config-url = ${request-apache-backup-2:connection-ssh-url}
-config-name = ${request-pull-backup-server-apache-2:config-name}
-config-type = push
-config-server-key = ${request-apache-backup-2:connection-ssh-public-key}
-config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id}
-slave = true
-sla = instance_guid
-sla-instance_guid = ${request-pull-backup-server:instance_guid}
-
 
-[directory]
-ssh = ${rootdirectory:etc}/ssh
-sshkeys = ${rootdirectory:srv}/sshkeys
+{{ replicated.replicate("apache", "3", "apache-export", "apache-import") }}
+
+# Bubble up the parameters
+[request-apache]
+return = url ssh-public-key ssh-url notification-id ip url backend_url
+# XXX: hardcoded values
+config = domain number authorized-key notify ip-list namebase postgres1-computer-guid pbs-postgres1-computer-guid postgres2-computer-guid pbs-postgres2-computer-guid postgres3-computer-guid pbs-postgres3-computer-guid
+config-postgres1-computer-guid = ${slap-parameter:postgres1-computer-guid}
+config-pbs-postgres1-computer-guid = ${slap-parameter:pbs-postgres1-computer-guid}
+config-postgres2-computer-guid = ${slap-parameter:postgres2-computer-guid}
+config-pbs-postgres2-computer-guid = ${slap-parameter:pbs-postgres2-computer-guid}
+config-postgres3-computer-guid = ${slap-parameter:postgres3-computer-guid}
+config-pbs-postgres3-computer-guid = ${slap-parameter:pbs-postgres3-computer-guid}
+config-domain = ${slap-parameter:domain}
+
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+backend_url = ${request-apache:connection-backend_url}
+url = ${request-apache:connection-url}
+
+[slap-parameter]
+# Default parameters for distributed deployment
+# I.e state "backup1 of maria should go there, ..."
+# XXX-Cedric: Hardcoded number of backups. Should be dynamically generated.
+postgres1-computer-guid =
+pbs-postgres1-computer-guid =
+postgres2-computer-guid =
+pbs-postgres2-computer-guid =
+postgres3-computer-guid =
+pbs-postgres3-computer-guid =
+# XXX-Cedric: Hardcoded parameters. Should be dynamically generated.
+domain =

stack/lapp/instance.cfg.in

@@ -14,7 +14,8 @@ resilient = ${instance-resilient:rendered}
 postgres = ${instance-postgres:output}
 postgres-import = ${instance-postgres-import:output}
 postgres-export = ${instance-postgres-export:output}
-apache-backup = ${instance-apache-backup:output}
+apache-import = ${instance-apache-import:output}
+apache-export = ${instance-apache-export:rendered}
 
 #frozen creates a syntax error, meaning it can keep its data.
 #It's dirty as hell, it needs to be replaced.