Commit 27ce939c authored by Alain Takoudjou's avatar Alain Takoudjou

repman: Use caucase for repman and mariadb ssl certificate

parent 3d49e677
...@@ -14,11 +14,11 @@ ...@@ -14,11 +14,11 @@
# not need these here). # not need these here).
[instance.cfg] [instance.cfg]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = 9aeb37bd4590aa7fa746ef70d78cc0d0 md5sum = af2fc4a7a0f782fed2cb1112ef3cb397
[instance-repman.cfg] [instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in _update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = e5b2d17e6ef5448e851521fee612f0c4 md5sum = d9151fe9f86fb1700c6c33d30a750837
[config-toml.in] [config-toml.in]
_update_hash_filename_ = templates/config.toml.in _update_hash_filename_ = templates/config.toml.in
...@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30 ...@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
[template-mariadb.cfg] [template-mariadb.cfg]
_update_hash_filename_ = instance-mariadb.cfg.jinja2.in _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
md5sum = 284cf030eeba4bdcc884908b2d730e47 md5sum = cfbbf574c1f755855022b1bcd6e6ff64
[template-my-cnf] [template-my-cnf]
_update_hash_filename_ = templates/my.cnf.in _update_hash_filename_ = templates/my.cnf.in
......
...@@ -21,6 +21,7 @@ database-host = {{ host }}:{{ port }} ...@@ -21,6 +21,7 @@ database-host = {{ host }}:{{ port }}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url} monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
partition-path = ${buildout:directory} partition-path = ${buildout:directory}
receiver-port = ${dbjob-parameter:socat-port} receiver-port = ${dbjob-parameter:socat-port}
csr-id = ${caucase-csr-id:csr-id}
[jinja2-template-base] [jinja2-template-base]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -62,6 +63,50 @@ context = key content {{content_section_name}}:content ...@@ -62,6 +63,50 @@ context = key content {{content_section_name}}:content
mode = {{ mode }} mode = {{ mode }}
{%- endmacro %} {%- endmacro %}
{% import "caucase" as caucase with context %}
{{ caucase.updater(
prefix='caucase-updater',
buildout_bin_directory=buildout_bin_directory,
updater_path='${directory:services}/caucase-updater',
url=slapparameter_dict['caucase-url'],
data_dir='${directory:srv}/caucase-updater',
crt_path='${directory:ssl}/mariadb-cert.crt',
ca_path='${directory:srv}/caucase-updater/ca.crt',
crl_path='${directory:srv}/caucase-updater/crl.pem',
key_path='${directory:ssl}/mariadb-cert.key',
on_renew=None,
max_sleep=None,
template_csr_pem=None,
openssl=openssl_bin,
)}}
{% do part_list.append('caucase-updater') -%}
{% do part_list.append('caucase-updater-promise') -%}
[get-csr-id]
recipe = plone.recipe.command
output = ${directory:tmp}/csr_id
command =
if [ -s "${directory:ssl}/mariadb-cert.crt" ]; then
RESULT="None";
else
if [ -f "${caucase-updater-csr:csr}" ]; then
RESULT=$({{ caucase_bin_client }} --ca-url {{ slapparameter_dict['caucase-url'] }} --send-csr ${caucase-updater-csr:csr} | cut -d ' ' -f1)
if [ ! $? -eq 0 ]; then
RESULT="None";
fi
fi
fi
cat <<EOF > ${:output}
[caucase]
csr-id = $(echo $RESULT)
EOF
update-command = ${:command}
[caucase-csr-id]
recipe = slapos.cookbook:zero-knowledge.read
file-path = ${get-csr-id:output}
csr-id =
[my-cnf-parameters] [my-cnf-parameters]
socket = ${directory:run}/mariadb.sock socket = ${directory:run}/mariadb.sock
ip = {{ ip }} ip = {{ ip }}
...@@ -75,9 +120,9 @@ innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0 ...@@ -75,9 +120,9 @@ innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0
innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }} innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }} innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }} relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
ssl-crt = ${directory:mariadb-ssl}/crt.pem ssl-crt = ${directory:ssl}/mariadb-cert.crt
ssl-key = ${directory:mariadb-ssl}/key.pem ssl-key = ${directory:ssl}/mariadb-cert.key
ssl-ca-crt = ${certificate-authority:ca-dir}/cacert.pem ssl-ca-crt = ${directory:srv}/caucase-updater/ca.crt
[my-cnf] [my-cnf]
< = jinja2-template-base < = jinja2-template-base
...@@ -152,18 +197,17 @@ environ = ...@@ -152,18 +197,17 @@ environ =
{{ variable }} {{ variable }}
{%- endfor %} {%- endfor %}
[ca-mysqld] [mysqld-launcher]
<= certificate-authority recipe = slapos.cookbook:wrapper
recipe = slapos.cookbook:certificate_authority.request command-line = ${mysqld:rendered}
key-file = ${my-cnf-parameters:ssl-key} wrapper-path = ${directory:controller}/mariadb
cert-file = ${my-cnf-parameters:ssl-crt} wait-for-files =
executable = ${mysqld:rendered} ${directory:ssl}/mariadb-cert.crt
wrapper = ${directory:controller}/mariadb
{% import "supervisord_lib" as supervisord_lib with context %} {% import "supervisord_lib" as supervisord_lib with context %}
{{ supervisord_lib.supervisord("mariadb-ctl", buildout_bin_directory, supervisord_conf, use_service_hash=False) }} {{ supervisord_lib.supervisord("mariadb-ctl", buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
{% do part_list.append("supervisord-mariadb-ctl") -%} {% do part_list.append("supervisord-mariadb-ctl") -%}
{% set maradb_program_dict = {"name": "mariadb", "command": "${ca-mysqld:wrapper}", {% set maradb_program_dict = {"name": "mariadb", "command": "${mysqld-launcher:wrapper-path}",
"stopwaitsecs": 86400, "environment": [], "stopwaitsecs": 86400, "environment": [],
"stdout_logfile": "${directory:log}/mariadb_stdout.log", "stdout_logfile": "${directory:log}/mariadb_stdout.log",
"stderr_logfile": "${directory:log}/mariadb_stdout.log" } %} "stderr_logfile": "${directory:log}/mariadb_stdout.log" } %}
...@@ -262,6 +306,7 @@ log = ${:var}/log ...@@ -262,6 +306,7 @@ log = ${:var}/log
run = ${:var}/run run = ${:var}/run
config-tmp = ${:tmp}/config config-tmp = ${:tmp}/config
custom = ${directory:etc}/mysql/custom custom = ${directory:etc}/mysql/custom
ssl = ${:etc}/ssl
[dbjob-parameter] [dbjob-parameter]
bash-bin = {{ bash_bin }} bash-bin = {{ bash_bin }}
......
...@@ -12,6 +12,10 @@ ...@@ -12,6 +12,10 @@
{% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%} {% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
{% set database_slave_list = [] -%} {% set database_slave_list = [] -%}
{% set db_name_list = [] -%} {% set db_name_list = [] -%}
{% set count = namespace(value=1) %}
{% set caucase_bind = '[' ~ ip ~ ']:8890' -%}
{% set caucase_url = 'http://' ~ caucase_bind -%}
{% set csrid_list = [] -%}
{% macro password(name) -%} {% macro password(name) -%}
[{{ name }}-password] [{{ name }}-password]
...@@ -34,6 +38,9 @@ log = ${:var}/log ...@@ -34,6 +38,9 @@ log = ${:var}/log
data = ${:var}/lib data = ${:var}/lib
nginx-prefix = ${:var}/nginx nginx-prefix = ${:var}/nginx
tmp = ${:home}/tmp tmp = ${:home}/tmp
backup-caucased = ${:srv}/backup-caucased
caucased = ${:srv}/caucased
ssl = ${:etc}/ssl
{% import "supervisord_lib" as supervisord_lib with context %} {% import "supervisord_lib" as supervisord_lib with context %}
{% set proxysql_controller = "proxysql-ctl" -%} {% set proxysql_controller = "proxysql-ctl" -%}
...@@ -48,6 +55,7 @@ key-file = ${slap-connection:key-file} ...@@ -48,6 +55,7 @@ key-file = ${slap-connection:key-file}
cert-file = ${slap-connection:cert-file} cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id} computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id} partition-id = ${slap-connection:partition-id}
config-caucase-url = {{ caucase_url }}
[download-proxy-config] [download-proxy-config]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -100,6 +108,7 @@ mode = 755 ...@@ -100,6 +108,7 @@ mode = 755
{% set db_amount = 2 -%} {% set db_amount = 2 -%}
{% endif -%} {% endif -%}
{% set count.value = count.value + db_amount %}
{% for i in range(0, db_amount) -%} {% for i in range(0, db_amount) -%}
{% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%} {% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%}
{% set section = 'request-mariadb-' ~ i -%} {% set section = 'request-mariadb-' ~ i -%}
...@@ -130,12 +139,14 @@ return = ...@@ -130,12 +139,14 @@ return =
receiver-port receiver-port
monitor-base-url monitor-base-url
partition-path partition-path
csr-id
{% do part_list.append(section) -%} {% do part_list.append(section) -%}
{% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%} {% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%}
{% do receiver_port_list.append('${' ~ section ~ ':connection-receiver-port}') -%} {% do receiver_port_list.append('${' ~ section ~ ':connection-receiver-port}') -%}
{% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%} {% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%}
{% do monitor_base_url_dict.__setitem__('mariadb' ~ i, '${' ~ section ~ ':connection-monitor-base-url}') -%} {% do monitor_base_url_dict.__setitem__('mariadb' ~ i, '${' ~ section ~ ':connection-monitor-base-url}') -%}
{% do csrid_list.append('${' ~ section ~ ':connection-csr-id}') -%}
{% endfor -%} {% endfor -%}
...@@ -299,6 +310,84 @@ context = ...@@ -299,6 +310,84 @@ context =
{% do part_list.append(name ~ '-publish-slave-information') -%} {% do part_list.append(name ~ '-publish-slave-information') -%}
{% endfor -%} {% endfor -%}
# deploy caucase
{% import "caucase" as caucase with context %}
{{ caucase.caucased(
prefix='caucased',
buildout_bin_directory=buildout_bin_directory,
caucased_path='${directory:service}/caucased',
backup_dir='${directory:backup-caucased}',
data_dir='${directory:caucased}',
netloc=caucase_bind,
tmp='${directory:tmp}',
service_auto_approve_count=count.value,
user_auto_approve_count=1,
key_len=2048,
)}}
{% do part_list.append('caucased') -%}
{% do part_list.append('caucased-promise') -%}
{% do publish_dict.__setitem__('caucase-http-url', caucase_url) -%}
{{ caucase.updater(
prefix='caucase-updater',
buildout_bin_directory=buildout_bin_directory,
updater_path='${directory:services}/caucase-updater',
url=caucase_url,
data_dir='${directory:srv}/caucase-updater',
crt_path='${directory:ssl}/repman-cert.crt',
ca_path='${directory:srv}/caucase-updater/ca.crt',
crl_path='${directory:srv}/caucase-updater/crl.pem',
key_path='${directory:ssl}/repman-cert.key',
on_renew=None,
max_sleep=None,
template_csr_pem=None,
openssl=openssl_bin,
)}}
{% do part_list.append('caucase-updater') -%}
{% do part_list.append('caucase-updater-promise') -%}
#caucase user certificate
{{ caucase.updater(
prefix='caucase-user-updater',
buildout_bin_directory=buildout_bin_directory,
updater_path='${directory:services}/caucase-user-updater',
url=caucase_url,
data_dir='${directory:srv}/caucase-user-updater',
crt_path='${directory:ssl}/caucase.user.crt',
ca_path='${directory:srv}/caucase-user-updater/ca.crt',
crl_path='${directory:srv}/caucase-user-updater/crl.pem',
key_path='${directory:ssl}/caucase.user.key',
on_renew=None,
max_sleep=None,
template_csr_pem=None,
openssl=openssl_bin,
mode='user',
)}}
{% do part_list.append('caucase-user-updater') -%}
{% do part_list.append('caucase-user-updater-promise') -%}
[caucase-sign-csr]
recipe = slapos.recipe.template:jinja2
mode = 755
rendered = ${directory:bin}/caucase-sign
template =
inline:#!/bin/sh
cp ${directory:ssl}/caucase.user.key ${directory:ssl}/caucase-full.key
cat ${directory:ssl}/caucase.user.crt >> ${directory:ssl}/caucase-full.key
for csr_id in {{ csrid_list | join(' ') }}; do
if [ "$csr_id" = "None" ] || [ -z "$csr_id"]; then
continue
fi
{{ buildout_bin_directory }}/caucase --ca-url {{ caucase_url }} --user-key ${directory:ssl}/caucase-full.key --sign-csr $csr_id
done
[cron-caucase-sign-csr]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = caucase-sign-csr
frequency = * * * * *
command = ${caucase-sign-csr:rendered}
{% do part_list.append('cron-caucase-sign-csr') -%}
[slap-configuration] [slap-configuration]
recipe = slapos.cookbook:slapconfiguration recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id} computer = ${slap-connection:computer-id}
...@@ -332,8 +421,9 @@ proxies-log = ${directory:log}/proxy ...@@ -332,8 +421,9 @@ proxies-log = ${directory:log}/proxy
ipv6 = ${instance-parameter:ipv6-random} ipv6 = ${instance-parameter:ipv6-random}
port = ${instance-parameter:nginx-port} port = ${instance-parameter:nginx-port}
ssl-port = ${instance-parameter:nginx-ssl-port} ssl-port = ${instance-parameter:nginx-ssl-port}
ssl-certificate = ${ca-nginx:cert-file} ssl-certificate = ${directory:ssl}/repman-cert.crt
ssl-key = ${ca-nginx:key-file} ca-certificate = ${directory:srv}/caucase-updater/ca.crt
ssl-key = ${directory:ssl}/repman-cert.key
pid-file = ${directory:run}/nginx.pid pid-file = ${directory:run}/nginx.pid
access-log = ${directory:log}/nginx_access.log access-log = ${directory:log}/nginx_access.log
error-log = ${directory:log}/nginx_error.log error-log = ${directory:log}/nginx_error.log
...@@ -464,11 +554,12 @@ command-line = ...@@ -464,11 +554,12 @@ command-line =
{{ nginx_bin }} {{ nginx_bin }}
-p ${directory:nginx-prefix} -p ${directory:nginx-prefix}
-c ${nginx-conf:rendered} -c ${nginx-conf:rendered}
wrapper-path = ${directory:bin}/nginx-start wrapper-path = ${directory:services}/nginx
wait-for-files = wait-for-files =
${ca-directory:certs}/nginx.key ${directory:ssl}/repman-cert.crt
${ca-directory:certs}/nginx.crt ${directory:srv}/caucase-updater/ca.crt
${nginx-graceful-wrapper:rendered} ${directory:ssl}/repman-cert.key
[nginx-graceful-wrapper] [nginx-graceful-wrapper]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -478,20 +569,6 @@ rendered = ${directory:scripts}/nginx-graceful ...@@ -478,20 +569,6 @@ rendered = ${directory:scripts}/nginx-graceful
context = context =
mode = 755 mode = 755
[ca-nginx]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
cert-file = ${ca-directory:certs}/nginx.crt
key-file = ${ca-directory:certs}/nginx.key
executable = ${nginx-launcher:wrapper-path}
wrapper = ${directory:bin}/ca-nginx
[ca-nginx-service]
recipe = slapos.cookbook:wrapper
command-line = ${ca-nginx:wrapper}
wrapper-path = ${directory:services}/nginx
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[logrotate-entry-nginx] [logrotate-entry-nginx]
<= logrotate-entry-base <= logrotate-entry-base
name = nginx name = nginx
...@@ -599,7 +676,7 @@ parts = ...@@ -599,7 +676,7 @@ parts =
replication-manager replication-manager
monitor-base monitor-base
logrotate-entry-nginx logrotate-entry-nginx
ca-nginx-service nginx-launcher
publish-connection-parameter publish-connection-parameter
repman-frontend-promise repman-frontend-promise
repman-backend-promise repman-backend-promise
......
...@@ -28,8 +28,10 @@ mode = 0644 ...@@ -28,8 +28,10 @@ mode = 0644
extensions = jinja2.ext.do extensions = jinja2.ext.do
rendered= ${buildout:directory}/${:_buildout_section_name_} rendered= ${buildout:directory}/${:_buildout_section_name_}
supervisord-lib = {{ supervisord_lib }} supervisord-lib = {{ supervisord_lib }}
causace-lib = {{ caucase_library }}
import-list = import-list =
file supervisord_lib :supervisord-lib file supervisord_lib :supervisord-lib
file caucase :causace-lib
context = context =
key slapparameter_dict slap-configuration:configuration key slapparameter_dict slap-configuration:configuration
key computer_id slap-configuration:computer key computer_id slap-configuration:computer
...@@ -48,6 +50,8 @@ context = ...@@ -48,6 +50,8 @@ context =
raw bash_bin {{ bash_location }}/bin/bash raw bash_bin {{ bash_location }}/bin/bash
raw jq_bin {{ jq_location }}/bin/jq raw jq_bin {{ jq_location }}/bin/jq
raw curl_bin {{ curl_location }}/bin/curl raw curl_bin {{ curl_location }}/bin/curl
raw openssl_bin {{ openssl_location }}/bin/openssl
raw caucase_bin_client {{ caucase_bin_client }}
${:extra-context} ${:extra-context}
extra-context = extra-context =
......
...@@ -18,8 +18,10 @@ extends = ...@@ -18,8 +18,10 @@ extends =
../../component/socat/buildout.cfg ../../component/socat/buildout.cfg
../../component/rsync/buildout.cfg ../../component/rsync/buildout.cfg
../../component/jq/buildout.cfg ../../component/jq/buildout.cfg
../../component/openssl/buildout.cfg
../../stack/supervisord/buildout.cfg ../../stack/supervisord/buildout.cfg
../../stack/monitor/buildout.cfg ../../stack/monitor/buildout.cfg
../../stack/caucase/buildout.cfg
parts = parts =
slapos-cookbook slapos-cookbook
...@@ -27,6 +29,7 @@ parts = ...@@ -27,6 +29,7 @@ parts =
instance.cfg instance.cfg
template-mariadb.cfg template-mariadb.cfg
template-mysqld-wrapper template-mysqld-wrapper
caucase-eggs
gowork gowork
[mariadb] [mariadb]
...@@ -39,7 +42,6 @@ part = python3 ...@@ -39,7 +42,6 @@ part = python3
# replication-manager does not build on golang 1.17 # replication-manager does not build on golang 1.17
golang = ${golang1.16:location} golang = ${golang1.16:location}
[template-mysqld-wrapper] [template-mysqld-wrapper]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/mysqld.in rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/mysqld.in
...@@ -76,6 +78,7 @@ mode = 0644 ...@@ -76,6 +78,7 @@ mode = 0644
context = context =
key bash_location bash:location key bash_location bash:location
key bin_directory buildout:bin-directory key bin_directory buildout:bin-directory
key caucase_library caucase-jinja2-library:target
key config_toml_in config-toml.in:target key config_toml_in config-toml.in:target
key config_cluster_toml_in config-cluster-toml.in:target key config_cluster_toml_in config-cluster-toml.in:target
key coreutils_location coreutils:location key coreutils_location coreutils:location
...@@ -101,6 +104,7 @@ context = ...@@ -101,6 +104,7 @@ context =
key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir
key nginx_conf_in nginx.conf.in:target key nginx_conf_in nginx.conf.in:target
key nginx_location nginx:location key nginx_location nginx:location
key openssl_location openssl:location
key percona_toolkit_location percona-toolkit:location key percona_toolkit_location percona-toolkit:location
key proxy_need_stop_start_template proxy-need-start-stop.sh.in:target key proxy_need_stop_start_template proxy-need-start-stop.sh.in:target
key repman_src_location git.signal18.io_signal18_repman:location key repman_src_location git.signal18.io_signal18_repman:location
...@@ -123,7 +127,7 @@ context = ...@@ -123,7 +127,7 @@ context =
key unixodbc_location unixodbc:location key unixodbc_location unixodbc:location
key sysbench_location sysbench:location key sysbench_location sysbench:location
key proxysql_location proxysql:location key proxysql_location proxysql:location
raw caucase_bin_client ${buildout:bin-directory}/caucase
[download-file] [download-file]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment