Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
slapos
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Cédric Le Ninivin
slapos
Commits
71d5e8ff
Commit
71d5e8ff
authored
Mar 23, 2020
by
Alain Takoudjou
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
repman: rework software release architecture, add nginx frontend and some promises
parent
42e91bdf
Changes
7
Hide whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
336 additions
and
157 deletions
+336
-157
component/replication-manager/buildout.cfg
component/replication-manager/buildout.cfg
+3
-16
software/repman/buildout.hash.cfg
software/repman/buildout.hash.cfg
+29
-0
software/repman/instance-repman.cfg.jinja2.in
software/repman/instance-repman.cfg.jinja2.in
+206
-0
software/repman/instance.cfg.in
software/repman/instance.cfg.in
+30
-113
software/repman/software.cfg
software/repman/software.cfg
+25
-25
software/repman/templates/config.toml.in
software/repman/templates/config.toml.in
+3
-3
software/repman/templates/nginx.conf.in
software/repman/templates/nginx.conf.in
+40
-0
No files found.
component/replication-manager/buildout.cfg
View file @
71d5e8ff
...
@@ -3,12 +3,13 @@ extends =
...
@@ -3,12 +3,13 @@ extends =
../../component/golang/buildout.cfg
../../component/golang/buildout.cfg
parts =
parts =
repman
gowork
[gowork]
[gowork]
golang = ${golang1.12:location}
golang = ${golang1.12:location}
install =
install =
buildflags = -v --tags server --ldflags "-extldflags 'static' -w -s -X main.GoOS=linux -X main.GoArch=amd64 -X main.Version=2.1 -X main.FullVersion=$FULLVERSION -X main.Build=$(date +%FT%T%z) -X main.WithProvisioning=ON -X main.WithOpenSVC=OFF -X main.WithHaproxy=ON -X main.WithMaxscale=ON -X main.WithMariadbshardproxy=ON -X main.WithProxysql=ON -X main.WithSphinx=ON -X main.WithArbitration=OFF -X main.WithArbitrationClient=ON -X main.WithMonitoring=ON -X main.WithHttp=ON -X main.WithBackup=ON -X main.WithMail=ON -X main.WithEnforce=ON -X main.WithDeprecate=ON"
buildflags = -v --tags server --ldflags "-extldflags 'static' -w -s -X main.GoOS=linux -X main.GoArch=amd64 -X main.Version=2.1 -X main.FullVersion=$FULLVERSION -X main.Build=$(date +%FT%T%z) -X main.WithProvisioning=ON -X main.WithOpenSVC=OFF -X main.WithHaproxy=ON -X main.WithMaxscale=ON -X main.WithMariadbshardproxy=ON -X main.WithProxysql=ON -X main.WithSphinx=ON -X main.WithArbitration=OFF -X main.WithArbitrationClient=ON -X main.WithMonitoring=ON -X main.WithHttp=ON -X main.WithBackup=ON -X main.WithMail=ON -X main.WithEnforce=ON -X main.WithDeprecate=ON"
gcc-bin-directory = ${golang1.12:gcc-bin-directory}
[gowork.goinstall]
[gowork.goinstall]
depends_gitfetch =
depends_gitfetch =
...
@@ -21,23 +22,9 @@ command = . ${gowork:env.sh} &&
...
@@ -21,23 +22,9 @@ command = . ${gowork:env.sh} &&
go build ${gowork:buildflags} -o ${gowork:bin}/replication-manager &&
go build ${gowork:buildflags} -o ${gowork:bin}/replication-manager &&
chmod -R u+w .
chmod -R u+w .
# export VERSION=$(git describe --abbrev=0 --tags) &&
[git.signal18.io_signal18_repman]
[git.signal18.io_signal18_repman]
<= go-git-package
<= go-git-package
go.importpath = github.com/signal18/replication-manager
go.importpath = github.com/signal18/replication-manager
repository = https://github.com/signal18/replication-manager
repository = https://github.com/signal18/replication-manager
branch = 2.1
branch = 2.1
\ No newline at end of file
[repman]
# revision and repository can be used to control which repman version is used
depend = ${gowork:recipe}
recipe = collective.recipe.template
# Do something useless to pass
output = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
input = inline:
#!/bin/sh
${gowork:bin}/replication-manager
--monitoring-basedir=system --monitoring-sharedir=software_release/src/share --http-root=software_release/src/dashboard monitor
# --monitoring-datadir=$${directory:var}/lib --config=$${directory:etc}/config.toml --log-file=$${:logfile}
software/repman/buildout.hash.cfg
0 → 100644
View file @
71d5e8ff
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance.cfg]
filename = instance.cfg.in
md5sum = 4373aa04326e85ae6c3136d12367646e
[instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = d5b3bbd9317e8ce60acdf03730fca1e5
[config-toml.in]
_update_hash_filename_ = templates/config.toml.in
md5sum = aa7f9fc85cef6061b87b080882d3d0d7
[nginx.conf.in]
_update_hash_filename_ = templates/nginx.conf.in
md5sum = 6d555e5e296654dc3e28d2ad5fa1bb8d
\ No newline at end of file
software/repman/instance-repman.cfg.jinja2.in
0 → 100644
View file @
71d5e8ff
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
run = ${:var}/run
script = ${:etc}/run
service = ${:etc}/service
promise = ${:etc}/promise
log = ${:var}/log
data = ${:var}/lib
nginx-prefix = ${:var}/nginx
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[instance-parameter]
<= slap-configuration
# repman monitor seems to use a fixed port
repman-port = 10005
nginx-port = 10006
[repman]
recipe = slapos.cookbook:mkdirectory
etc = ${directory:etc}/repman
data-dir = ${directory:var}/lib
root-dir = ${directory:srv}/repman
[nginx-parameter]
ipv6 = ${instance-parameter:ipv6-random}
port = ${instance-parameter:nginx-port}
ssl-certificate = ${ca-nginx:cert-file}
ssl-key = ${ca-nginx:key-file}
pid-file = ${directory:run}/nginx.pid
access-log = ${directory:log}/nginx_access.log
error-log = ${directory:log}/nginx_error.log
repman-url = https://${repman-parameter:ipv4}:${repman-parameter:port}
config-file = ${directory:etc}/nginx.conf
backend-url = https://[${:ipv6}]:${:port}
[repman-password]
recipe = slapos.cookbook:generate.password
bytes = 12
[repman-parameter]
log = ${directory:log}/repman.log
http-root = ${repman:root-dir}/dashboard
share-dir = ${repman:root-dir}/share
port = ${instance-parameter:repman-port}
ipv4 = ${instance-parameter:ipv4-random}
username = admin
password = ${repman-password:passwd}
[repman-config-folder]
recipe = plone.recipe.command
repman-location = {{ repman_src_location }}
command =
cd ${:repman-location}
cp -r share ${repman-parameter:share-dir}
cp -r dashboard ${repman-parameter:http-root}
# XXX - this need to be checked/optimized in case of upgrade.
update-command =
[replication-manager]
recipe = slapos.cookbook:wrapper
command-line =
{{ gowork_bin }}/replication-manager
--monitoring-basedir=${repman:root-dir}
--monitoring-sharedir=${repman-parameter:share-dir}
--http-root=${repman-parameter:http-root}
--monitoring-datadir=${repman:data-dir}
--config=${repman-config.toml:rendered}
--log-file=${repman-parameter:log}
monitor
wrapper-path = ${directory:service}/replication-manager
# setup repman instance folder
depends =
${repman-config-folder:recipe}
[repman-config.toml]
recipe = slapos.recipe.template:jinja2
template = {{ config_toml_in }}
rendered = ${repman:etc}/config.toml
extra-context =
context =
section parameter_dict repman-parameter
[repman-listen-promise]
<= monitor-promise-base
module = check_port_listening
name = repman_service_listen.py
config-hostname = ${repman-parameter:ipv4}
config-port = ${repman-parameter:port}
[nginx-conf]
recipe = slapos.recipe.template:jinja2
template = {{ nginx_conf_in }}
rendered = ${nginx-parameter:config-file}
context =
section parameter_dict nginx-parameter
[nginx-launcher]
recipe = slapos.cookbook:wrapper
command-line =
{{ nginx_bin }}
-p ${directory:nginx-prefix}
-c ${nginx-conf:rendered}
wrapper-path = ${directory:bin}/nginx-start
wait-for-files =
${ca-directory:certs}/nginx.key
${ca-directory:certs}/nginx.crt
${nginx-graceful-wrapper:wrapper-path}
[nginx-graceful-wrapper]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:scripts}/nginx-graceful
command-line =
kill -USR1 $(cat ${nginx-parameter:pid-file})
[ca-nginx]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
cert-file = ${ca-directory:certs}/nginx.crt
key-file = ${ca-directory:certs}/nginx.key
executable = ${nginx-launcher:wrapper-path}
wrapper = ${directory:bin}/ca-nginx
[ca-nginx-service]
recipe = slapos.cookbook:wrapper
command-line = ${ca-nginx:wrapper}
wrapper-path = ${directory:services}/nginx
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[logrotate-entry-nginx]
<= logrotate-entry-base
name = nginx
log = ${nginx-parameter:access-log} ${nginx-parameter:error-log}
post = kill -USR1 $(cat ${nginx-parameter:pid-file})
[publish-connection-parameter]
<= monitor-publish
recipe = slapos.cookbook:publish
backend-url = ${nginx-parameter:backend-url}
url = ${repman-frontend:connection-secure_access}
username = ${repman-parameter:username}
password = ${repman-parameter:password}
[repman-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Replication Manager Frontend
# XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-url = ${nginx-parameter:backend-url}
config-domain =
return = domain secure_access
[repman-frontend-promise]
<= monitor-promise-base
module = check_url_available
name = check_repman_frontend.py
config-url = https://${repman-frontend:connection-domain}
config-check-secure = 1
[repman-backend-promise]
<= monitor-promise-base
module = check_url_available
name = check_repman_frontend.py
config-url = ${nginx-parameter:backend-url}
config-check-secure = 1
#############################
#
# Deploy replication-manager instance
#
#############################
[buildout]
extends =
{{ template_monitor_cfg }}
parts =
replication-manager
repman-listen-promise
monitor-base
logrotate-entry-nginx
ca-nginx-service
publish-connection-parameter
repman-frontend-promise
repman-backend-promise
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
software/repman/instance.cfg.in
View file @
71d5e8ff
#############################
# GitLab "switch-softwaretype" instance
#
# Deploy replication-manager instance
#
#############################
[buildout]
[buildout]
parts =
parts = switch-softwaretype
directory
publish-connection-parameter
instance-parameter
template-jinja2-base
replication-manager
repman-promise
# std stuff for slapos instance
# Define egg directories to be the one from Software Release
# (/opt/slapgrid/...)
# Always the same.
eggs-directory = ${buildout:eggs-directory}
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
offline = true
[instance-parameter]
[switch-softwaretype]
# Fetch arbitrary parameters defined by the user in SlapOS Master for his instance.
recipe = slapos.cookbook:softwaretype
# We use the slapconfiguration recipe with a few parameters (partition id,
repman = $${template-instance-repman.cfg:rendered}
# computer id, certificate, etc).
default = $${:repman}
# It will then authenticate to SlapOS Master and fetch the instance parameters.
# The parameters are accessible from $${instance-parameter:configuration.name-of-parameter}
# Always the same. Just copy/paste.
# See docstring of slapos.cookbook:slapconfiguration for more information.
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[slap-configuration]
# apache-frontend reads from a part named [slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
# Create all needed directories, depending on your needs
[directory]
recipe = slapos.cookbook:mkdirectory
home = $${buildout:directory}
etc = $${:home}/etc
var = $${:home}/var
# Executables put here will be started but not monitored (for startup scripts)
script = $${:etc}/run/
# Executables put here will be started and monitored (for daemons)
service = $${:etc}/service
# Executables put here will be launched after buildout has completed to see
# if instance is running
promise = $${:etc}/promise/
# Path of the log directory used by our service (see [helloweb])
log = $${:var}/log
data = $${:var}/lib
[replication-manager-credential]
recipe = slapos.cookbook:generate.password
username = admin
bytes = 12
[replication-manager]
# repman service is listening on:
# - global IPv6 address, and
# - fixed port
#
# NOTE because every computer partition is allocated its own global IPv6
# address, it is ok to fix the port - different hello-world instances will have
# different IPv6 addresses and they all will be accessible at the same time.
port = 10005
ipv6 = $${instance-parameter:ipv6-random}
# full URL - for convenience
url = https://[$${:ipv6}]:$${:port}
# the service will log here
# macro: render instance-*.cfg from instance-*.cfg.in
logfile = $${directory:log}/repman.log
[instance-cfg]
recipe = slapos.recipe.template:jinja2
# Actual script that starts the service:
mode = 0644
# This recipe will try to "exec" the command-line after separating parameters.
rendered= $${buildout:directory}/$${:_buildout_section_name_}
recipe = slapos.cookbook:wrapper
context =
command-line =
${gowork:bin}/replication-manager --monitoring-basedir=system --monitoring-sharedir=software_release/src/share --http-root=software_release/src/dashboard --monitoring-datadir=$${directory:var}/lib --config=$${directory:etc}/config.toml --log-file=$${:logfile} monitor
# Put this shell script in the "etc/service" directory. Each executable of this
# repository will be started and monitored by supervisord. If a service
# exits/crashes, it will trigger a "bang" and cause a re-run of the instance.
wrapper-path = $${directory:service}/replication-manager
name = config.toml
key bin_directory buildout:bin-directory
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
# program binaries
raw bash_bin ${bash:location}/bin/bash
raw gowork_bin ${gowork:bin}
raw nginx_bin ${nginx:location}/sbin/nginx
raw repman_src_location ${git.signal18.io_signal18_repman:location}
raw template_monitor_cfg ${monitor2-template:rendered}
# config files
raw config_toml_in ${config-toml.in:target}
raw nginx_conf_in ${nginx.conf.in:target}
[template-jinja2-base]
$${:extra-context}
recipe = slapos.recipe.template:jinja2
template = ${download-config-toml:destination}
rendered =$${directory:etc}/config.toml
extra-context =
extra-context =
context =
key service_ipv6 replication-manager:ipv6
key service_user replication-manager-credential:username
key service_pwd replication-manager-credential:passwd
# promise, that checks that helloweb service is alive
[repman-promise]
recipe = slapos.cookbook:check_port_listening
path = $${directory:promise}/replication-manager-ipv6
hostname= $${replication-manager:ipv6}
port = 10005
# Publish all the parameters needed for the user to connect to the instance.
[template-instance-repman.cfg]
# It can be anything: URL(s), password(s), or arbitrary parameters.
<= instance-cfg
# Here we'll just echo back the entered name as instance parameter
template= ${instance-repman.cfg:target}
[publish-connection-parameter]
extra-context =
recipe = slapos.cookbook:publish
url = $${replication-manager:url}
username = $${replication-manager-credential:username}
password = $${replication-manager-credential:passwd}
software/repman/software.cfg
View file @
71d5e8ff
[buildout]
[buildout]
extends =
extends =
../../stack/slapos
.cfg
buildout.hash
.cfg
../../stack/monitor/buildout.cfg
../../stack/monitor/buildout.cfg
../../component/replication-manager/buildout.cfg
../../component/replication-manager/buildout.cfg
../../component/git/buildout.cfg
../../component/mariadb/buildout.cfg
../../component/nginx/buildout.cfg
../../stack/monitor/buildout.cfg
../../component/logrotate/buildout.cfg
../../stack/slapos.cfg
parts =
parts =
slapos-cookbook
slapos-cookbook
download-config-toml
instance.cfg
template
mariadb
repman
repman-repository
[instance.cfg]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/instance.cfg
[download-file]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
destination = ${buildout:directory}/${:_buildout_section_name_}
[template]
[config-toml.in]
recipe = slapos.recipe.template
<= download-file
url = ${:_profile_base_location_}/${:filename}
mode = 0644
output = ${buildout:directory}/template.cfg
filename = instance.cfg.in
[instance-repman.cfg]
<= download-file
[download-config-toml]
[nginx.conf.in]
recipe = slapos.recipe.build:download
<= download-file
url = ${:_profile_base_location_}/templates/config.toml.in
destination = ${buildout:directory}/config.toml.in
mode = 644
# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
[versions]
[versions]
slapos.recipe.template = 4.3
slapos.recipe.template = 4.3
rubygemsrecipe = 0.2.2+slapos001
rubygemsrecipe = 0.2.2+slapos001
\ No newline at end of file
[repman-repository]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
location = ${buildout:directory}/src
branch = 2.1
repository = https://github.com/signal18/replication-manager
software/repman/templates/config.toml.in
View file @
71d5e8ff
[Default]
[Default]
api-bind = "[{{
service_ipv6
}}]"
api-bind = "[{{
parameter_dict['ipv4']
}}]"
http-bind-address = "[{{
service_ipv6
}}]"
http-bind-address = "[{{
parameter_dict['ipv4']
}}]"
monitoring-save-config = true
monitoring-save-config = true
api-https-bind = true
api-https-bind = true
api-credentials = "{{
service_user}}:{{service_pwd
}}"
api-credentials = "{{
parameter_dict['username'] }}:{{ parameter_dict['password']
}}"
[cluster1]
[cluster1]
title = "cluster1"
title = "cluster1"
...
...
software/repman/templates/nginx.conf.in
0 → 100644
View file @
71d5e8ff
pid {{ parameter_dict['pid-file'] }};
error_log {{ parameter_dict['error-log'] }};
daemon off;
events {
worker_connections 1024;
accept_mutex off;
}
http {
default_type application/octet-stream;
access_log {{ parameter_dict['access-log'] }} combined;
client_max_body_size 10M;
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
server {
listen [{{ parameter_dict['ipv6'] }}]:{{ parameter_dict['port'] }} ssl;
server_name _;
ssl_certificate {{ parameter_dict['ssl-certificate'] }};
ssl_certificate_key {{ parameter_dict['ssl-key'] }};
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
keepalive_timeout 90s;
location / {
proxy_redirect off;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $http_host;
proxy_connect_timeout 200;
proxy_pass {{ parameter_dict['repman-url'] }};
}
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment