diff --git a/stack/lapp/apache/instance-apache-export.cfg.jinja2 b/stack/lapp/apache/instance-apache-export.cfg.jinja2 new file mode 100644 index 0000000000000000000000000000000000000000..ac2c24505c5d86beb50fbe4e49e25dab7fcfe998 --- /dev/null +++ b/stack/lapp/apache/instance-apache-export.cfg.jinja2 @@ -0,0 +1,54 @@ +# This file is responsible of three things: +# 1/ Act as "Apache exporter" +# 2/ Act as "Mariadb backup infrastructure requester" +# 3/ Act as "Apache" instance + +{% import 'parts' as parts %} +{% import 'replicated' as replicated %} + +[buildout] +extends = {{templateapache}} + {{templatepbsreadyexport}} + +parts += + {{ parts.replicate("mariadb", "3") }} + +# Repeating parts from instance-apache-php. +# XXX-Cedric: how to simplify this? + certificate-authority + ca-stunnel + logrotate + logrotate-entry-apache + logrotate-entry-stunnel + cron + cron-entry-logrotate + promise + frontend-promise + content-promise + publish-connection-informations + +{{ replicated.replicate("mariadb", "3", "mariadb-export", "mariadb-import") }} + +# Nothing to do for the exporter. Just dummy part that does nothing. +# For httpd instance, PBS will directly pull data from srv/www. +# XXX-Cedric: write a real backup system. +[exporter] +wrapper = /bin/true + +# State that we want to backup srv/www directory, not srv/backup. +# XXX-Cedric: works well, but doesn't work with big data. +[rdiff-backup-server] +path = ${directory:www} + +# Add "exporter" parameters to list of published connection parameters +[publish-connection-informations] +# XXX-Cedric: Long term goal: could be a recipe that requests an instance and +# bubbles ALL +# parameters of the requested instance. Requirement: aggregated publish. +<= resilient-publish-connection-parameter + +# XXX-Cedric: resilient overwrites what's returned from request-mariadb +# XXX-Cedric: change the request method to return everything from +# getConnectionParameterDict() +[request-mariadb] +return = ssh-public-key ssh-url notification-id ip url diff --git a/stack/lapp/apache/instance-apache-import.cfg.in b/stack/lapp/apache/instance-apache-import.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..009a48922ee045ab2ffbb4fd16746d0b511cbd68 --- /dev/null +++ b/stack/lapp/apache/instance-apache-import.cfg.in @@ -0,0 +1,20 @@ +[buildout] +eggs-directory = ${buildout:eggs-directory} +develop-eggs-directory = ${buildout:develop-eggs-directory} +offline = true + +extends = ${pbsready-import:output} + +[directory] +srv = $${buildout:directory}/srv +www = $${:srv}/www/ + +# Nothing to do for the import. Just dummy part that does nothing. +# For httpd instance, PBS will directly push data to srv/www. +# XXX-Cedric: write a real backup system. +[importer] +wrapper = /bin/true + +# State that we want to push to srv/www directory, not srv/backup. +[rdiff-backup-server] +path = $${directory:www} diff --git a/stack/lapp/apache/instance-apache-php.cfg.in b/stack/lapp/apache/instance-apache-php.cfg.in index 5521da7e8b0f3cb27178d2a8807a1f7fb0205e5f..cd425c71109592d3dc27f1b896467a68c8138ab9 100644 --- a/stack/lapp/apache/instance-apache-php.cfg.in +++ b/stack/lapp/apache/instance-apache-php.cfg.in @@ -57,6 +57,7 @@ logrotate-backup = $${basedirectory:backup}/logrotate report = $${rootdirectory:etc}/report stunnel-conf = $${rootdirectory:etc}/stunnel xml-report = $${rootdirectory:var}/xml_report +www = $${rootdirectory:srv}/www/ [cadirectory] recipe = slapos.cookbook:mkdirectory @@ -195,7 +196,7 @@ url = $${request-postgres:connection-url} recipe = slapos.cookbook:apachephp source = ${application:location} -htdocs = $${rootdirectory:srv}/www/ +htdocs = $${directory:www} pid-file = $${basedirectory:run}/apache.pid lock-file = $${basedirectory:run}/apache.lock ip = $${slap-network-information:global-ipv6} diff --git a/stack/lapp/buildout.cfg b/stack/lapp/buildout.cfg index b642c4b228450b9c6516f6aa73f950f560c28d38..c410d62daf0b53c987b0d0deea6652af8eb119d3 100644 --- a/stack/lapp/buildout.cfg +++ b/stack/lapp/buildout.cfg @@ -1,4 +1,7 @@ [buildout] + +ignore-existing = true + parts = slapos-cookbook apache-php-postgres @@ -15,11 +18,14 @@ parts = instance-postgres-import instance-postgres-export +#Contains the importer and exporter recipes for apache + instance-apache-import + instance-apache-export + extends = - ../resilient/buildout.cfg - ../../component/apache/buildout.cfg ../../component/apache-php/buildout.cfg + ../../component/apache/buildout.cfg ../../component/dash/buildout.cfg ../../component/dcron/buildout.cfg ../../component/gzip/buildout.cfg @@ -29,6 +35,7 @@ extends = ../../component/rdiff-backup/buildout.cfg ../../component/stunnel/buildout.cfg ../../component/dropbear/buildout.cfg + ../resilient/buildout.cfg ../slapos.cfg @@ -48,21 +55,35 @@ strip-top-level-dir = true recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance.cfg.in output = ${buildout:directory}/instance.cfg -md5sum = a482fa0e72839b4bd75b169ac1460d64 +md5sum = 25d07b5101d5f566398686642ada4cee mode = 0644 [instance-apache-php] recipe = slapos.recipe.template url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in output = ${buildout:directory}/instance-apache-php.cfg -md5sum = 72b70452d1c077cfcd0f268181506b8e +md5sum = 823257dda6f3068a38c6b69c771cf307 mode = 0644 -[instance-apache-backup] +[instance-apache-import] recipe = slapos.recipe.template -url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg.in -output = ${buildout:directory}/instance-apache-backup.cfg -md5sum = db879141c0b6a77ef8b3b7e699f5583a +url = ${:_profile_base_location_}/apache/instance-apache-import.cfg.in +output = ${buildout:directory}/instance-apache-import.cfg +md5sum = f1dc2a71d362b5d2d36481ffefdd2293 +mode = 0644 + +[instance-apache-export] +recipe = slapos.recipe.template:jinja2 +template = ${:_profile_base_location_}/apache/instance-apache-export.cfg.jinja2 +rendered = ${buildout:directory}/instance-apache-export.cfg + +context = key templateapache instance-apache-php:output + key templatepbsreadyexport pbsready-export:output + +import-list = file parts template-parts:destination + file replicated template-replicated:destination + +md5sum = 4704f2788f096c7494694db72a9f6193 mode = 0644 [instance-resilient] @@ -70,16 +91,15 @@ recipe = slapos.recipe.template:jinja2 template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2 rendered = ${buildout:directory}/instance-resilient.cfg -context = key templateapache instance-apache-php:output - key dropbear dropbear:location - key buildout buildout:bin-directory +context = key buildout buildout:bin-directory + key develop_eggs_directory buildout:develop-eggs-directory + key eggs_directory buildout:eggs-directory import-list = file parts template-parts:destination file replicated template-replicated:destination -md5sum = 46c7d8f691bd37d84e0bd03b83e51d14 +md5sum = 224a7e73db84168cd5aedec10ddd50f2 mode = 0644 - [instance-postgres] recipe = slapos.recipe.template @@ -155,7 +175,6 @@ eggs = ${psycopg2:egg} slapos.toolbox - [networkcache] # Romain Courteaud + Sebastien Robin + Alain Takoudjou # + Cedric de Saint Martin signature certificate diff --git a/stack/lapp/instance-resilient.cfg.jinja2 b/stack/lapp/instance-resilient.cfg.jinja2 index 022a37f951fb6ab88e577fd9e08f6ba8b69ce09d..fb80a0fce621b9a8c09b8e80f955cd3511cf171e 100644 --- a/stack/lapp/instance-resilient.cfg.jinja2 +++ b/stack/lapp/instance-resilient.cfg.jinja2 @@ -4,172 +4,44 @@ {% import 'replicated' as replicated %} [buildout] - -extends = - {{templateapache}} +eggs-directory = {{ eggs_directory }} +develop-eggs-directory = {{ develop_eggs_directory }} +offline = true # += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended parts += - {{ parts.replicate("postgres","3") }} - request-apache-backup-1 - request-apache-backup-2 - - request-pull-backup-server-apache-1 - request-pull-backup-server-apache-backup-1 - - request-pull-backup-server-apache-2 - request-pull-backup-server-apache-backup-2 - + {{ parts.replicate("apache", "3") }} publish-connection-informations - apache-php - stunnel - certificate-authority - ca-stunnel - logrotate - logrotate-entry-apache - logrotate-entry-stunnel - cron - cron-entry-logrotate - dropbear-server - sshkeys-authority - dropbear-server-pbs-authorized-key - - request-pull-backup-server - -{{ replicated.replicate("postgres", "3", "postgres-export", "postgres-import") }} - - -[request-pull-backup-server] -<= request-pbs-common -name = PBS (Pull Backup Server) -return = ssh-key notification-url feeds-url -slave = false - -[sshkeys-directory] -recipe = slapos.cookbook:mkdirectory -requests = ${directory:sshkeys}/requests -keys = ${directory:sshkeys}/keys - -[sshkeys-authority] -recipe = slapos.cookbook:sshkeys_authority -request-directory = ${sshkeys-directory:requests} -keys-directory = ${sshkeys-directory:keys} -wrapper = ${basedirectory:services}/sshkeys_authority -keygen-binary = {{dropbear}}/bin/dropbearkey - -[sshkeys-dropbear] -<= sshkeys-authority -recipe = slapos.cookbook:sshkeys_authority.request -name = dropbear -type = rsa -executable = ${dropbear-server:wrapper} -public-key = ${dropbear-server:rsa-keyfile}.pub -private-key = ${dropbear-server:rsa-keyfile} -wrapper = ${basedirectory:services}/sshd - -[dropbear-server] -recipe = slapos.cookbook:dropbear -host = ${slap-network-information:global-ipv6} -port = 2222 -home = ${directory:ssh} -wrapper = ${rootdirectory:bin}/raw_sshd -shell = ${rdiff-backup-server:wrapper} -rsa-keyfile = ${directory:ssh}/server_key.rsa -dropbear-binary = {{dropbear}}/sbin/dropbear - -[dropbear-server-pbs-authorized-key] -<= dropbear-server -recipe = slapos.cookbook:dropbear.add_authorized_key -key = ${request-pull-backup-server:connection-ssh-key} - -[rdiff-backup-server] -<= apache-php -recipe = slapos.cookbook:pbs -client = false -path = ${apache-php:htdocs} -wrapper = ${rootdirectory:bin}/rdiffbackup-server -rdiffbackup-binary = {{buildout}}/rdiff-backup - -[request-apache-backup-1] -<= slap-connection -recipe = slapos.cookbook:request -name = Apache Backup 1 -software-url = ${slap-connection:software-release-url} -software-type = apache-backup -return = url ssh-url ssh-public-key -config = authorized-key proxy-url -config-authorized-key = ${request-pull-backup-server:connection-ssh-key} -config-proxy-url = ${publish-connection-informations:url} - -[request-apache-backup-2] -<= slap-connection -recipe = slapos.cookbook:request -name = Apache Backup 2 -software-url = ${slap-connection:software-release-url} -software-type = apache-backup -return = url ssh-url ssh-public-key -config = authorized-key proxy-url -config-authorized-key = ${request-pull-backup-server:connection-ssh-key} -config-proxy-url = ${publish-connection-informations:url} - -[request-pull-backup-server-apache-1] -<= request-pbs-common -name = PBS pulling from Apache 1 -config = url name type server-key notify notification-id frequency -config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path} -config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache -config-type = pull -config-server-key = ${sshkeys-dropbear:public-key-value} -config-notify = ${request-pull-backup-server:connection-notification-url} -config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull -config-frequency = 30 * * * * -slave = true -sla = instance_guid -sla-instance_guid = ${request-pull-backup-server:instance_guid} - -[request-pull-backup-server-apache-2] -<= request-pbs-common -name = PBS pulling from Apache 2 -config = url name type server-key notify notification-id frequency -config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path} -config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache -config-type = pull -config-server-key = ${sshkeys-dropbear:public-key-value} -config-notify = ${request-pull-backup-server:connection-notification-url} -config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull -config-frequency = 30 * * * * -slave = true -sla = instance_guid -sla-instance_guid = ${request-pull-backup-server:instance_guid} - - -[request-pull-backup-server-apache-backup-1] -<= request-pbs-common -name = PBS pushing to ${request-apache-backup-1:name} -config = url name type server-key on-notification -config-url = ${request-apache-backup-1:connection-ssh-url} -config-name = ${request-pull-backup-server-apache-1:config-name} -config-type = push -config-server-key = ${request-apache-backup-1:connection-ssh-public-key} -config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id} -slave = true -sla = instance_guid -sla-instance_guid = ${request-pull-backup-server:instance_guid} - -[request-pull-backup-server-apache-backup-2] -<= request-pbs-common -name = PBS pushing to ${request-apache-backup-2:name} -config = url name type server-key on-notification -config-url = ${request-apache-backup-2:connection-ssh-url} -config-name = ${request-pull-backup-server-apache-2:config-name} -config-type = push -config-server-key = ${request-apache-backup-2:connection-ssh-public-key} -config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id} -slave = true -sla = instance_guid -sla-instance_guid = ${request-pull-backup-server:instance_guid} - -[directory] -ssh = ${rootdirectory:etc}/ssh -sshkeys = ${rootdirectory:srv}/sshkeys +{{ replicated.replicate("apache", "3", "apache-export", "apache-import") }} + +# Bubble up the parameters +[request-apache] +return = url ssh-public-key ssh-url notification-id ip url backend_url +# XXX: hardcoded values +config = domain number authorized-key notify ip-list namebase postgres1-computer-guid pbs-postgres1-computer-guid postgres2-computer-guid pbs-postgres2-computer-guid postgres3-computer-guid pbs-postgres3-computer-guid +config-postgres1-computer-guid = ${slap-parameter:postgres1-computer-guid} +config-pbs-postgres1-computer-guid = ${slap-parameter:pbs-postgres1-computer-guid} +config-postgres2-computer-guid = ${slap-parameter:postgres2-computer-guid} +config-pbs-postgres2-computer-guid = ${slap-parameter:pbs-postgres2-computer-guid} +config-postgres3-computer-guid = ${slap-parameter:postgres3-computer-guid} +config-pbs-postgres3-computer-guid = ${slap-parameter:pbs-postgres3-computer-guid} +config-domain = ${slap-parameter:domain} + +[publish-connection-informations] +recipe = slapos.cookbook:publish +backend_url = ${request-apache:connection-backend_url} +url = ${request-apache:connection-url} + +[slap-parameter] +# Default parameters for distributed deployment +# I.e state "backup1 of maria should go there, ..." +# XXX-Cedric: Hardcoded number of backups. Should be dynamically generated. +postgres1-computer-guid = +pbs-postgres1-computer-guid = +postgres2-computer-guid = +pbs-postgres2-computer-guid = +postgres3-computer-guid = +pbs-postgres3-computer-guid = +# XXX-Cedric: Hardcoded parameters. Should be dynamically generated. +domain = diff --git a/stack/lapp/instance.cfg.in b/stack/lapp/instance.cfg.in index 596a9dfb03c9981c7e2aefb3e0c17f4d56e0f216..910502c6647e81ac1208ade4978f18f4decacaa6 100644 --- a/stack/lapp/instance.cfg.in +++ b/stack/lapp/instance.cfg.in @@ -14,7 +14,8 @@ resilient = ${instance-resilient:rendered} postgres = ${instance-postgres:output} postgres-import = ${instance-postgres-import:output} postgres-export = ${instance-postgres-export:output} -apache-backup = ${instance-apache-backup:output} +apache-import = ${instance-apache-import:output} +apache-export = ${instance-apache-export:rendered} #frozen creates a syntax error, meaning it can keep its data. #It's dirty as hell, it needs to be replaced.