Skip to content

slapos
slapos
Commit 71d5e8ff authored by Alain Takoudjou's avatar Alain Takoudjou
Browse files
Options

repman: rework software release architecture, add nginx frontend and some promises

parent 42e91bdf
Hide whitespace changes
Inline Side-by-side
Showing with 336 additions and 157 deletions
component/replication-manager/buildout.cfg
View file @ 71d5e8ff
......@@ -3,12 +3,13 @@ extends =
../../component/golang/buildout.cfg
parts =
repman
gowork
[gowork]
golang = ${golang1.12:location}
install =
buildflags = -v --tags server --ldflags "-extldflags 'static' -w -s -X main.GoOS=linux -X main.GoArch=amd64 -X main.Version=2.1 -X main.FullVersion=$FULLVERSION -X main.Build=$(date +%FT%T%z) -X main.WithProvisioning=ON -X main.WithOpenSVC=OFF -X main.WithHaproxy=ON -X main.WithMaxscale=ON -X main.WithMariadbshardproxy=ON -X main.WithProxysql=ON -X main.WithSphinx=ON -X main.WithArbitration=OFF -X main.WithArbitrationClient=ON -X main.WithMonitoring=ON -X main.WithHttp=ON -X main.WithBackup=ON -X main.WithMail=ON -X main.WithEnforce=ON -X main.WithDeprecate=ON"
gcc-bin-directory = ${golang1.12:gcc-bin-directory}
[gowork.goinstall]
depends_gitfetch =
......@@ -21,23 +22,9 @@ command = . ${gowork:env.sh} &&
go build ${gowork:buildflags} -o ${gowork:bin}/replication-manager &&
chmod -R u+w .
# export VERSION=$(git describe --abbrev=0 --tags) &&
[git.signal18.io_signal18_repman]
<= go-git-package
go.importpath = github.com/signal18/replication-manager
repository = https://github.com/signal18/replication-manager
branch = 2.1
[repman]
# revision and repository can be used to control which repman version is used
depend = ${gowork:recipe}
recipe = collective.recipe.template
# Do something useless to pass
output = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
input = inline:
#!/bin/sh
${gowork:bin}/replication-manager
--monitoring-basedir=system --monitoring-sharedir=software_release/src/share --http-root=software_release/src/dashboard monitor
# --monitoring-datadir=$${directory:var}/lib --config=$${directory:etc}/config.toml --log-file=$${:logfile}
branch = 2.1
\ No newline at end of file
software/repman/buildout.hash.cfg 0 → 100644
View file @ 71d5e8ff
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance.cfg]
filename = instance.cfg.in
md5sum = 4373aa04326e85ae6c3136d12367646e
[instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = d5b3bbd9317e8ce60acdf03730fca1e5
[config-toml.in]
_update_hash_filename_ = templates/config.toml.in
md5sum = aa7f9fc85cef6061b87b080882d3d0d7
[nginx.conf.in]
_update_hash_filename_ = templates/nginx.conf.in
md5sum = 6d555e5e296654dc3e28d2ad5fa1bb8d
\ No newline at end of file
software/repman/instance-repman.cfg.jinja2.in 0 → 100644
View file @ 71d5e8ff
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
run = ${:var}/run
script = ${:etc}/run
service = ${:etc}/service
promise = ${:etc}/promise
log = ${:var}/log
data = ${:var}/lib
nginx-prefix = ${:var}/nginx
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[instance-parameter]
<= slap-configuration
# repman monitor seems to use a fixed port
repman-port = 10005
nginx-port = 10006
[repman]
recipe = slapos.cookbook:mkdirectory
etc = ${directory:etc}/repman
data-dir = ${directory:var}/lib
root-dir = ${directory:srv}/repman
[nginx-parameter]
ipv6 = ${instance-parameter:ipv6-random}
port = ${instance-parameter:nginx-port}
ssl-certificate = ${ca-nginx:cert-file}
ssl-key = ${ca-nginx:key-file}
pid-file = ${directory:run}/nginx.pid
access-log = ${directory:log}/nginx_access.log
error-log = ${directory:log}/nginx_error.log
repman-url = https://${repman-parameter:ipv4}:${repman-parameter:port}
config-file = ${directory:etc}/nginx.conf
backend-url = https://[${:ipv6}]:${:port}
[repman-password]
recipe = slapos.cookbook:generate.password
bytes = 12
[repman-parameter]
log = ${directory:log}/repman.log
http-root = ${repman:root-dir}/dashboard
share-dir = ${repman:root-dir}/share
port = ${instance-parameter:repman-port}
ipv4 = ${instance-parameter:ipv4-random}
username = admin
password = ${repman-password:passwd}
[repman-config-folder]
recipe = plone.recipe.command
repman-location = {{ repman_src_location }}
command =
cd ${:repman-location}
cp -r share ${repman-parameter:share-dir}
cp -r dashboard ${repman-parameter:http-root}
# XXX - this need to be checked/optimized in case of upgrade.
update-command =
[replication-manager]
recipe = slapos.cookbook:wrapper
command-line =
{{ gowork_bin }}/replication-manager
--monitoring-basedir=${repman:root-dir}
--monitoring-sharedir=${repman-parameter:share-dir}
--http-root=${repman-parameter:http-root}
--monitoring-datadir=${repman:data-dir}
--config=${repman-config.toml:rendered}
--log-file=${repman-parameter:log}
monitor
wrapper-path = ${directory:service}/replication-manager
# setup repman instance folder
depends =
${repman-config-folder:recipe}
[repman-config.toml]
recipe = slapos.recipe.template:jinja2
template = {{ config_toml_in }}
rendered = ${repman:etc}/config.toml
extra-context =
context =
section parameter_dict repman-parameter
[repman-listen-promise]
<= monitor-promise-base
module = check_port_listening
name = repman_service_listen.py
config-hostname = ${repman-parameter:ipv4}
config-port = ${repman-parameter:port}
[nginx-conf]
recipe = slapos.recipe.template:jinja2
template = {{ nginx_conf_in }}
rendered = ${nginx-parameter:config-file}
context =
section parameter_dict nginx-parameter
[nginx-launcher]
recipe = slapos.cookbook:wrapper
command-line =
{{ nginx_bin }}
-p ${directory:nginx-prefix}
-c ${nginx-conf:rendered}
wrapper-path = ${directory:bin}/nginx-start
wait-for-files =
${ca-directory:certs}/nginx.key
${ca-directory:certs}/nginx.crt
${nginx-graceful-wrapper:wrapper-path}
[nginx-graceful-wrapper]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:scripts}/nginx-graceful
command-line =
kill -USR1 $(cat ${nginx-parameter:pid-file})
[ca-nginx]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
cert-file = ${ca-directory:certs}/nginx.crt
key-file = ${ca-directory:certs}/nginx.key
executable = ${nginx-launcher:wrapper-path}
wrapper = ${directory:bin}/ca-nginx
[ca-nginx-service]
recipe = slapos.cookbook:wrapper
command-line = ${ca-nginx:wrapper}
wrapper-path = ${directory:services}/nginx
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[logrotate-entry-nginx]
<= logrotate-entry-base
name = nginx
log = ${nginx-parameter:access-log} ${nginx-parameter:error-log}
post = kill -USR1 $(cat ${nginx-parameter:pid-file})
[publish-connection-parameter]
<= monitor-publish
recipe = slapos.cookbook:publish
backend-url = ${nginx-parameter:backend-url}
url = ${repman-frontend:connection-secure_access}
username = ${repman-parameter:username}
password = ${repman-parameter:password}
[repman-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Replication Manager Frontend
# XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-url = ${nginx-parameter:backend-url}
config-domain =
return = domain secure_access
[repman-frontend-promise]
<= monitor-promise-base
module = check_url_available
name = check_repman_frontend.py
config-url = https://${repman-frontend:connection-domain}
config-check-secure = 1
[repman-backend-promise]
<= monitor-promise-base
module = check_url_available
name = check_repman_frontend.py
config-url = ${nginx-parameter:backend-url}
config-check-secure = 1
#############################
#
# Deploy replication-manager instance
#
#############################
[buildout]
extends =
{{ template_monitor_cfg }}
parts =
replication-manager
repman-listen-promise
monitor-base
logrotate-entry-nginx
ca-nginx-service
publish-connection-parameter
repman-frontend-promise
repman-backend-promise
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
software/repman/instance.cfg.in
View file @ 71d5e8ff
#############################
#
# Deploy replication-manager instance
#
#############################
# GitLab "switch-softwaretype" instance
[buildout]
parts =
directory
publish-connection-parameter
instance-parameter
template-jinja2-base
replication-manager
repman-promise
parts = switch-softwaretype
# Define egg directories to be the one from Software Release
# (/opt/slapgrid/...)
# Always the same.
# std stuff for slapos instance
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[instance-parameter]
# Fetch arbitrary parameters defined by the user in SlapOS Master for his instance.
# We use the slapconfiguration recipe with a few parameters (partition id,
# computer id, certificate, etc).
# It will then authenticate to SlapOS Master and fetch the instance parameters.
# The parameters are accessible from $${instance-parameter:configuration.name-of-parameter}
# Always the same. Just copy/paste.
# See docstring of slapos.cookbook:slapconfiguration for more information.
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[slap-configuration]
# apache-frontend reads from a part named [slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
# Create all needed directories, depending on your needs
[directory]
recipe = slapos.cookbook:mkdirectory
home = $${buildout:directory}
etc = $${:home}/etc
var = $${:home}/var
# Executables put here will be started but not monitored (for startup scripts)
script = $${:etc}/run/
# Executables put here will be started and monitored (for daemons)
service = $${:etc}/service
# Executables put here will be launched after buildout has completed to see
# if instance is running
promise = $${:etc}/promise/
# Path of the log directory used by our service (see [helloweb])
log = $${:var}/log
data = $${:var}/lib
[replication-manager-credential]
recipe = slapos.cookbook:generate.password
username = admin
bytes = 12
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
repman = $${template-instance-repman.cfg:rendered}
default = $${:repman}
[replication-manager]
# repman service is listening on:
# - global IPv6 address, and
# - fixed port
#
# NOTE because every computer partition is allocated its own global IPv6
# address, it is ok to fix the port - different hello-world instances will have
# different IPv6 addresses and they all will be accessible at the same time.
port = 10005
ipv6 = $${instance-parameter:ipv6-random}
# full URL - for convenience
url = https://[$${:ipv6}]:$${:port}
# the service will log here
logfile = $${directory:log}/repman.log
# Actual script that starts the service:
# This recipe will try to "exec" the command-line after separating parameters.
recipe = slapos.cookbook:wrapper
command-line =
${gowork:bin}/replication-manager --monitoring-basedir=system --monitoring-sharedir=software_release/src/share --http-root=software_release/src/dashboard --monitoring-datadir=$${directory:var}/lib --config=$${directory:etc}/config.toml --log-file=$${:logfile} monitor
# Put this shell script in the "etc/service" directory. Each executable of this
# repository will be started and monitored by supervisord. If a service
# exits/crashes, it will trigger a "bang" and cause a re-run of the instance.
wrapper-path = $${directory:service}/replication-manager
# macro: render instance-*.cfg from instance-*.cfg.in
[instance-cfg]
recipe = slapos.recipe.template:jinja2
mode = 0644
rendered= $${buildout:directory}/$${:_buildout_section_name_}
context =
name = config.toml
key bin_directory buildout:bin-directory
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
# program binaries
raw bash_bin ${bash:location}/bin/bash
raw gowork_bin ${gowork:bin}
raw nginx_bin ${nginx:location}/sbin/nginx
raw repman_src_location ${git.signal18.io_signal18_repman:location}
raw template_monitor_cfg ${monitor2-template:rendered}
# config files
raw config_toml_in ${config-toml.in:target}
raw nginx_conf_in ${nginx.conf.in:target}
[template-jinja2-base]
recipe = slapos.recipe.template:jinja2
template = ${download-config-toml:destination}
rendered =$${directory:etc}/config.toml
$${:extra-context}
extra-context =
context =
key service_ipv6 replication-manager:ipv6
key service_user replication-manager-credential:username
key service_pwd replication-manager-credential:passwd
# promise, that checks that helloweb service is alive
[repman-promise]
recipe = slapos.cookbook:check_port_listening
path = $${directory:promise}/replication-manager-ipv6
hostname= $${replication-manager:ipv6}
port = 10005
# Publish all the parameters needed for the user to connect to the instance.
# It can be anything: URL(s), password(s), or arbitrary parameters.
# Here we'll just echo back the entered name as instance parameter
[publish-connection-parameter]
recipe = slapos.cookbook:publish
url = $${replication-manager:url}
username = $${replication-manager-credential:username}
password = $${replication-manager-credential:passwd}
[template-instance-repman.cfg]
<= instance-cfg
template= ${instance-repman.cfg:target}
extra-context =
software/repman/software.cfg
View file @ 71d5e8ff
[buildout]
extends =
../../stack/slapos.cfg
buildout.hash.cfg
../../stack/monitor/buildout.cfg
../../component/replication-manager/buildout.cfg
../../component/git/buildout.cfg
../../component/mariadb/buildout.cfg
../../component/nginx/buildout.cfg
../../stack/monitor/buildout.cfg
../../component/logrotate/buildout.cfg
../../stack/slapos.cfg
parts =
slapos-cookbook
download-config-toml
template
repman
repman-repository
instance.cfg
mariadb
[instance.cfg]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/instance.cfg
[download-file]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
destination = ${buildout:directory}/${:_buildout_section_name_}
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
mode = 0644
output = ${buildout:directory}/template.cfg
filename = instance.cfg.in
[config-toml.in]
<= download-file
[instance-repman.cfg]
<= download-file
[download-config-toml]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/config.toml.in
destination = ${buildout:directory}/config.toml.in
mode = 644
[nginx.conf.in]
<= download-file
# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
[versions]
slapos.recipe.template = 4.3
rubygemsrecipe = 0.2.2+slapos001
[repman-repository]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
location = ${buildout:directory}/src
branch = 2.1
repository = https://github.com/signal18/replication-manager
rubygemsrecipe = 0.2.2+slapos001
\ No newline at end of file
software/repman/templates/config.toml.in
View file @ 71d5e8ff
[Default]
api-bind = "[{{service_ipv6}}]"
http-bind-address = "[{{service_ipv6}}]"
api-bind = "[{{ parameter_dict['ipv4'] }}]"
http-bind-address = "[{{ parameter_dict['ipv4'] }}]"
monitoring-save-config = true
api-https-bind = true
api-credentials = "{{service_user}}:{{service_pwd}}"
api-credentials = "{{ parameter_dict['username'] }}:{{ parameter_dict['password'] }}"
[cluster1]
title = "cluster1"
......
software/repman/templates/nginx.conf.in 0 → 100644
View file @ 71d5e8ff
pid {{ parameter_dict['pid-file'] }};
error_log {{ parameter_dict['error-log'] }};
daemon off;
events {
worker_connections 1024;
accept_mutex off;
}
http {
default_type application/octet-stream;
access_log {{ parameter_dict['access-log'] }} combined;
client_max_body_size 10M;
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
server {
listen [{{ parameter_dict['ipv6'] }}]:{{ parameter_dict['port'] }} ssl;
server_name _;
ssl_certificate {{ parameter_dict['ssl-certificate'] }};
ssl_certificate_key {{ parameter_dict['ssl-key'] }};
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
keepalive_timeout 90s;
location / {
proxy_redirect off;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $http_host;
proxy_connect_timeout 200;
proxy_pass {{ parameter_dict['repman-url'] }};
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7