repman: rework software release architecture, add nginx frontend and some promises

component/replication-manager/buildout.cfg

@@ -3,12 +3,13 @@ extends =
   ../../component/golang/buildout.cfg
 
 parts =
-  repman
+  gowork
 
 [gowork]
 golang  = ${golang1.12:location}
 install =
 buildflags =  -v --tags server --ldflags "-extldflags 'static' -w -s -X main.GoOS=linux -X main.GoArch=amd64 -X main.Version=2.1 -X main.FullVersion=$FULLVERSION -X main.Build=$(date +%FT%T%z) -X main.WithProvisioning=ON -X main.WithOpenSVC=OFF -X main.WithHaproxy=ON -X main.WithMaxscale=ON  -X main.WithMariadbshardproxy=ON -X  main.WithProxysql=ON -X  main.WithSphinx=ON -X main.WithArbitration=OFF -X main.WithArbitrationClient=ON -X main.WithMonitoring=ON -X main.WithHttp=ON -X main.WithBackup=ON -X main.WithMail=ON -X main.WithEnforce=ON -X main.WithDeprecate=ON"
+gcc-bin-directory = ${golang1.12:gcc-bin-directory}
 
 [gowork.goinstall]
 depends_gitfetch  =
@@ -21,23 +22,9 @@ command = . ${gowork:env.sh} &&
   go build ${gowork:buildflags} -o ${gowork:bin}/replication-manager  &&
   chmod -R u+w .
 
-#  export VERSION=$(git describe --abbrev=0 --tags) &&
 
 [git.signal18.io_signal18_repman]
 <= go-git-package
 go.importpath = github.com/signal18/replication-manager
 repository =  https://github.com/signal18/replication-manager
 branch = 2.1
\ No newline at end of file
-
-[repman]
-# revision and repository can be used to control which repman version is used
-depend = ${gowork:recipe}
-recipe  = collective.recipe.template
-# Do something useless to pass
-output  = ${buildout:bin-directory}/${:_buildout_section_name_}
-mode    = 0755
-input   = inline:
-    #!/bin/sh
-    ${gowork:bin}/replication-manager
-    --monitoring-basedir=system --monitoring-sharedir=software_release/src/share --http-root=software_release/src/dashboard monitor
-# --monitoring-datadir=$${directory:var}/lib --config=$${directory:etc}/config.toml --log-file=$${:logfile}

software/repman/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+[instance.cfg]
+filename = instance.cfg.in
+md5sum = 4373aa04326e85ae6c3136d12367646e
+
+[instance-repman.cfg]
+_update_hash_filename_ = instance-repman.cfg.jinja2.in
+md5sum = d5b3bbd9317e8ce60acdf03730fca1e5
+
+[config-toml.in]
+_update_hash_filename_ = templates/config.toml.in
+md5sum = aa7f9fc85cef6061b87b080882d3d0d7
+
+[nginx.conf.in]
+_update_hash_filename_ = templates/nginx.conf.in
+md5sum = 6d555e5e296654dc3e28d2ad5fa1bb8d
\ No newline at end of file

software/repman/instance-repman.cfg.jinja2.in

+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+run = ${:var}/run
+script = ${:etc}/run
+service = ${:etc}/service
+promise = ${:etc}/promise
+log = ${:var}/log
+data = ${:var}/lib
+nginx-prefix = ${:var}/nginx
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[instance-parameter]
+<= slap-configuration
+# repman monitor seems to use a fixed port
+repman-port = 10005
+nginx-port  = 10006
+
+
+[repman]
+recipe   = slapos.cookbook:mkdirectory
+etc      = ${directory:etc}/repman
+data-dir = ${directory:var}/lib
+root-dir = ${directory:srv}/repman
+
+[nginx-parameter]
+ipv6 = ${instance-parameter:ipv6-random}
+port = ${instance-parameter:nginx-port}
+ssl-certificate = ${ca-nginx:cert-file}
+ssl-key = ${ca-nginx:key-file}
+pid-file = ${directory:run}/nginx.pid
+access-log = ${directory:log}/nginx_access.log
+error-log = ${directory:log}/nginx_error.log
+repman-url = https://${repman-parameter:ipv4}:${repman-parameter:port}
+config-file = ${directory:etc}/nginx.conf
+backend-url = https://[${:ipv6}]:${:port}
+
+[repman-password]
+recipe = slapos.cookbook:generate.password
+bytes = 12
+
+[repman-parameter]
+log = ${directory:log}/repman.log
+http-root = ${repman:root-dir}/dashboard
+share-dir = ${repman:root-dir}/share
+port = ${instance-parameter:repman-port}
+ipv4 = ${instance-parameter:ipv4-random}
+username = admin
+password = ${repman-password:passwd}
+
+[repman-config-folder]
+recipe = plone.recipe.command
+repman-location = {{ repman_src_location }}
+command = 
+  cd ${:repman-location}
+  cp -r share ${repman-parameter:share-dir}
+  cp -r dashboard ${repman-parameter:http-root}
+
+# XXX - this need to be checked/optimized in case of upgrade.
+update-command =
+  
+
+[replication-manager]
+recipe = slapos.cookbook:wrapper
+command-line =
+   {{ gowork_bin }}/replication-manager
+   --monitoring-basedir=${repman:root-dir}
+   --monitoring-sharedir=${repman-parameter:share-dir}
+   --http-root=${repman-parameter:http-root}
+   --monitoring-datadir=${repman:data-dir}
+   --config=${repman-config.toml:rendered}
+   --log-file=${repman-parameter:log}
+   monitor
+
+wrapper-path = ${directory:service}/replication-manager
+# setup repman instance folder
+depends =
+  ${repman-config-folder:recipe}
+
+
+[repman-config.toml]
+recipe = slapos.recipe.template:jinja2
+template = {{ config_toml_in }}
+rendered = ${repman:etc}/config.toml
+extra-context =
+context =
+  section parameter_dict repman-parameter
+
+[repman-listen-promise]
+<= monitor-promise-base
+module = check_port_listening
+name = repman_service_listen.py
+config-hostname = ${repman-parameter:ipv4}
+config-port = ${repman-parameter:port}
+
+[nginx-conf]
+recipe = slapos.recipe.template:jinja2
+template = {{ nginx_conf_in }}
+rendered = ${nginx-parameter:config-file}
+context =
+    section parameter_dict nginx-parameter
+
+[nginx-launcher]
+recipe = slapos.cookbook:wrapper
+command-line =
+  {{ nginx_bin }}
+  -p ${directory:nginx-prefix}
+  -c ${nginx-conf:rendered}
+wrapper-path = ${directory:bin}/nginx-start
+wait-for-files =
+  ${ca-directory:certs}/nginx.key
+  ${ca-directory:certs}/nginx.crt
+  ${nginx-graceful-wrapper:wrapper-path}
+
+[nginx-graceful-wrapper]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:scripts}/nginx-graceful
+command-line =
+  kill -USR1 $(cat ${nginx-parameter:pid-file})
+
+
+[ca-nginx]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+cert-file = ${ca-directory:certs}/nginx.crt
+key-file = ${ca-directory:certs}/nginx.key
+executable = ${nginx-launcher:wrapper-path}
+wrapper = ${directory:bin}/ca-nginx
+
+[ca-nginx-service]
+recipe = slapos.cookbook:wrapper
+command-line = ${ca-nginx:wrapper}
+wrapper-path = ${directory:services}/nginx
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+
+[logrotate-entry-nginx]
+<= logrotate-entry-base
+name = nginx
+log = ${nginx-parameter:access-log} ${nginx-parameter:error-log}
+post = kill -USR1 $(cat ${nginx-parameter:pid-file})
+
+[publish-connection-parameter]
+<= monitor-publish
+recipe = slapos.cookbook:publish
+backend-url = ${nginx-parameter:backend-url}
+url = ${repman-frontend:connection-secure_access}
+username = ${repman-parameter:username}
+password = ${repman-parameter:password}
+
+[repman-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Replication Manager Frontend
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config-url = ${nginx-parameter:backend-url}
+config-domain = 
+return = domain secure_access
+
+[repman-frontend-promise]
+<= monitor-promise-base
+module = check_url_available
+name = check_repman_frontend.py
+config-url = https://${repman-frontend:connection-domain}
+config-check-secure = 1
+
+[repman-backend-promise]
+<= monitor-promise-base
+module = check_url_available
+name = check_repman_frontend.py
+config-url = ${nginx-parameter:backend-url}
+config-check-secure = 1
+
+
+#############################
+#
+# Deploy replication-manager instance
+#
+#############################
+[buildout]
+extends =
+  {{ template_monitor_cfg }}
+parts =
+  replication-manager
+  repman-listen-promise
+  monitor-base
+  logrotate-entry-nginx
+  ca-nginx-service
+  publish-connection-parameter
+  repman-frontend-promise
+  repman-backend-promise
+
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true

software/repman/instance.cfg.in

-#############################
-#
-# Deploy replication-manager instance
-#
-#############################
+# GitLab "switch-softwaretype" instance
 [buildout]
-parts =
-  directory
-  publish-connection-parameter
-  instance-parameter
-  template-jinja2-base
-  replication-manager
-  repman-promise
+parts = switch-softwaretype
 
-
-# Define egg directories to be the one from Software Release
-# (/opt/slapgrid/...)
-# Always the same.
+# std stuff for slapos instance
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
 
-[instance-parameter]
-# Fetch arbitrary parameters defined by the user in SlapOS Master for his instance.
-# We use the slapconfiguration recipe with a few parameters (partition id,
-# computer id, certificate, etc).
-# It will then authenticate to SlapOS Master and fetch the instance parameters.
-# The parameters are accessible from $${instance-parameter:configuration.name-of-parameter}
-# Always the same. Just copy/paste.
-# See docstring of slapos.cookbook:slapconfiguration for more information.
-recipe = slapos.cookbook:slapconfiguration
-computer = $${slap-connection:computer-id}
-partition = $${slap-connection:partition-id}
-url = $${slap-connection:server-url}
-key = $${slap-connection:key-file}
-cert = $${slap-connection:cert-file}
-
-
-[slap-configuration]
-# apache-frontend reads from  a part named [slap-configuration]
-recipe = slapos.cookbook:slapconfiguration.serialised
-computer = $${slap-connection:computer-id}
-partition = $${slap-connection:partition-id}
-url = $${slap-connection:server-url}
-key = $${slap-connection:key-file}
-cert = $${slap-connection:cert-file}
-
-
-
-# Create all needed directories, depending on your needs
-[directory]
-recipe = slapos.cookbook:mkdirectory
-home = $${buildout:directory}
-etc = $${:home}/etc
-var = $${:home}/var
-# Executables put here will be started but not monitored (for startup scripts)
-script = $${:etc}/run/
-# Executables put here will be started and monitored (for daemons)
-service = $${:etc}/service
-# Executables put here will be launched after buildout has completed to see
-# if instance is running
-promise = $${:etc}/promise/
-# Path of the log directory used by our service (see [helloweb])
-log = $${:var}/log
-data = $${:var}/lib
-
-[replication-manager-credential]
-recipe = slapos.cookbook:generate.password
-username = admin
-bytes = 12
-
-[replication-manager]
-# repman service is listening on:
-# - global IPv6 address, and
-# - fixed port
-#
-# NOTE because every computer partition is allocated its own global IPv6
-# address, it is ok to fix the port - different hello-world instances will have
-# different IPv6 addresses and they all will be accessible at the same time.
-port = 10005
-ipv6 = $${instance-parameter:ipv6-random}
-# full URL - for convenience
-url = https://[$${:ipv6}]:$${:port}
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+repman  = $${template-instance-repman.cfg:rendered}
+default = $${:repman}
 
-# the service will log here
-logfile = $${directory:log}/repman.log
 
-# Actual script that starts the service:
-# This recipe will try to "exec" the command-line after separating parameters.
-recipe = slapos.cookbook:wrapper
-command-line =
-   ${gowork:bin}/replication-manager  --monitoring-basedir=system --monitoring-sharedir=software_release/src/share --http-root=software_release/src/dashboard --monitoring-datadir=$${directory:var}/lib --config=$${directory:etc}/config.toml --log-file=$${:logfile} monitor
-
-# Put this shell script in the "etc/service" directory. Each executable of this
-# repository will be started and monitored by supervisord. If a service
-# exits/crashes, it will trigger a "bang" and cause a re-run of the instance.
-wrapper-path = $${directory:service}/replication-manager
+# macro: render instance-*.cfg from instance-*.cfg.in
+[instance-cfg]
+recipe  = slapos.recipe.template:jinja2
+mode    = 0644
+rendered= $${buildout:directory}/$${:_buildout_section_name_}
+context =
 
-name = config.toml
+    key bin_directory           buildout:bin-directory
+    key eggs_directory          buildout:eggs-directory
+    key develop_eggs_directory  buildout:develop-eggs-directory
 
+# program binaries
+    raw bash_bin                    ${bash:location}/bin/bash
+    raw gowork_bin                  ${gowork:bin}
+    raw nginx_bin                   ${nginx:location}/sbin/nginx
+    raw repman_src_location         ${git.signal18.io_signal18_repman:location}
+    raw template_monitor_cfg        ${monitor2-template:rendered}
 
+# config files
+    raw config_toml_in             ${config-toml.in:target}
+    raw nginx_conf_in              ${nginx.conf.in:target}
 
-[template-jinja2-base]
-recipe = slapos.recipe.template:jinja2
-template = ${download-config-toml:destination}
-rendered =$${directory:etc}/config.toml
+    $${:extra-context}
 extra-context =
-context =
-  key service_ipv6 replication-manager:ipv6
-  key service_user replication-manager-credential:username
-  key service_pwd replication-manager-credential:passwd
-# promise, that checks that helloweb service is alive
-[repman-promise]
-recipe = slapos.cookbook:check_port_listening
-path = $${directory:promise}/replication-manager-ipv6
-hostname= $${replication-manager:ipv6}
-port    = 10005
-
 
-# Publish all the parameters needed for the user to connect to the instance.
-# It can be anything: URL(s), password(s), or arbitrary parameters.
-# Here we'll just echo back the entered name as instance parameter
-[publish-connection-parameter]
-recipe = slapos.cookbook:publish
-url = $${replication-manager:url}
-username = $${replication-manager-credential:username}
-password = $${replication-manager-credential:passwd}
+[template-instance-repman.cfg]
+<= instance-cfg
+template= ${instance-repman.cfg:target}
+extra-context =

software/repman/software.cfg

 [buildout]
 
 extends =
-  ../../stack/slapos.cfg
+  buildout.hash.cfg
   ../../stack/monitor/buildout.cfg
   ../../component/replication-manager/buildout.cfg
-  ../../component/git/buildout.cfg
+  ../../component/mariadb/buildout.cfg
+  ../../component/nginx/buildout.cfg
+  ../../stack/monitor/buildout.cfg
+  ../../component/logrotate/buildout.cfg
+  ../../stack/slapos.cfg
 
 parts =
   slapos-cookbook
-  download-config-toml
-  template
-  repman
-  repman-repository
+  instance.cfg
+  mariadb
 
 
-[template]
+[instance.cfg]
 recipe  = slapos.recipe.template
 url     = ${:_profile_base_location_}/${:filename}
-mode = 0644
-output = ${buildout:directory}/template.cfg
-filename = instance.cfg.in
-
+output  = ${buildout:directory}/instance.cfg
 
-[download-config-toml]
+[download-file]
 recipe  = slapos.recipe.build:download
-url = ${:_profile_base_location_}/templates/config.toml.in
-destination = ${buildout:directory}/config.toml.in
-mode = 644
+url     = ${:_profile_base_location_}/${:_update_hash_filename_}
+destination = ${buildout:directory}/${:_buildout_section_name_}
+
+[config-toml.in]
+<= download-file
+
+[instance-repman.cfg]
+<= download-file
+
+[nginx.conf.in]
+<= download-file
 
 # Pin versions of eggs used that are not already pinned by stack/slapos.cfg
 [versions]
 slapos.recipe.template = 4.3
 rubygemsrecipe  = 0.2.2+slapos001
\ No newline at end of file
-
-[repman-repository]
-recipe  = slapos.recipe.build:gitclone
-git-executable = ${git:location}/bin/git
-location = ${buildout:directory}/src
-branch = 2.1
-repository = https://github.com/signal18/replication-manager

software/repman/templates/config.toml.in

 [Default]
-api-bind = "[{{service_ipv6}}]"
-http-bind-address = "[{{service_ipv6}}]"
+api-bind = "[{{ parameter_dict['ipv4'] }}]"
+http-bind-address = "[{{ parameter_dict['ipv4'] }}]"
 monitoring-save-config = true
 api-https-bind = true
-api-credentials = "{{service_user}}:{{service_pwd}}"
+api-credentials = "{{ parameter_dict['username'] }}:{{ parameter_dict['password'] }}"
 
 [cluster1]
 title = "cluster1"

software/repman/templates/nginx.conf.in

+pid {{ parameter_dict['pid-file'] }};
+error_log {{ parameter_dict['error-log'] }};
+
+daemon off;
+
+events {
+  worker_connections 1024;
+  accept_mutex off;
+}
+
+http {
+     default_type application/octet-stream;
+     access_log {{ parameter_dict['access-log'] }} combined;
+     client_max_body_size 10M;
+     map $http_upgrade $connection_upgrade {
+        default upgrade;
+        ''      close;
+     }
+
+     server {
+        listen [{{ parameter_dict['ipv6'] }}]:{{ parameter_dict['port'] }} ssl;
+        server_name _;
+        ssl_certificate     {{ parameter_dict['ssl-certificate'] }};
+        ssl_certificate_key {{ parameter_dict['ssl-key'] }};
+        ssl_protocols       TLSv1 TLSv1.1 TLSv1.2;
+        ssl_ciphers         ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5;
+        ssl_prefer_server_ciphers on;
+        keepalive_timeout 90s;
+
+        location / {
+            proxy_redirect off;
+            proxy_set_header   X-Forwarded-Proto $scheme;
+            proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
+            proxy_set_header   X-Forwarded-Host  $http_host;
+            proxy_connect_timeout 200;
+
+            proxy_pass {{ parameter_dict['repman-url'] }};
+        }
+    }
+}