From ff51927cbab9d7272c16977a2859aa8719315384 Mon Sep 17 00:00:00 2001
From: Alain Takoudjou <alain.takoudjou@nexedi.com>
Date: Wed, 13 Jul 2022 10:39:25 +0200
Subject: [PATCH] repman: fix issues when creating additional clusters
It should be possible to add 2 or more cluster in the same instance
---
software/repman/buildout.hash.cfg | 6 +-
.../instance-repman-slave-input-schema.json | 6 ++
software/repman/instance-repman.cfg.jinja2.in | 66 ++++++++++++-------
.../publish-database-slave-parameters.cfg.in | 3 +
.../repman/templates/repman-manager.sh.in | 36 ++++++++--
5 files changed, 85 insertions(+), 32 deletions(-)
diff --git a/software/repman/buildout.hash.cfg b/software/repman/buildout.hash.cfg
index c7d828641..807f71dd2 100644
--- a/software/repman/buildout.hash.cfg
+++ b/software/repman/buildout.hash.cfg
@@ -18,7 +18,7 @@ md5sum = 55c7fd4dd6a39b31878889fbfb00f995
[instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in
-md5sum = 529411c1c6233f0a0d2740c9dd6be4f0
+md5sum = 0240cb0f25cda84c37e1f93cc8cf8900
[config-toml.in]
_update_hash_filename_ = templates/config.toml.in
@@ -46,7 +46,7 @@ md5sum = 47b76144e1b116580c8acf08274af976
[template-publish-slave-information]
_update_hash_filename_ = publish-database-slave-parameters.cfg.in
-md5sum = 9616477ff993c55c2b43bf8797db6994
+md5sum = 2e9f3afde619e20f08803013a4225e94
[mariadb-init-root-sql]
_update_hash_filename_ = templates/mariadb_init_root.sql.in
@@ -58,7 +58,7 @@ md5sum = c203f40a58386310a433b58fd345a341
[repman-manager-sh.in]
_update_hash_filename_ = templates/repman-manager.sh.in
-md5sum = 5d22f599b39c25c285d0e9c02e7074a3
+md5sum = 70201395116d79b5bb4aabef13079c10
[dbjobs-in]
_update_hash_filename_ = templates/dbjobs.in
diff --git a/software/repman/instance-repman-slave-input-schema.json b/software/repman/instance-repman-slave-input-schema.json
index 687549859..61908d463 100644
--- a/software/repman/instance-repman-slave-input-schema.json
+++ b/software/repman/instance-repman-slave-input-schema.json
@@ -36,6 +36,12 @@
"ascii_bin"
],
"default": ""
+ },
+ "cluster_id": {
+ "title": "Replication Manager Cluster Name",
+ "description": "When a cluster name is given, try to request the slave in the specified cluster name. If cluster name is not provided or does not exist, the slave is requested in the first cluster.",
+ "type": "string",
+ "default": ""
}
},
"title": "Input Parameters",
diff --git a/software/repman/instance-repman.cfg.jinja2.in b/software/repman/instance-repman.cfg.jinja2.in
index c0ee61534..c9c6d93a3 100644
--- a/software/repman/instance-repman.cfg.jinja2.in
+++ b/software/repman/instance-repman.cfg.jinja2.in
@@ -2,9 +2,6 @@
{% set part_list = [] -%}
{% set monitor_base_url_dict = {} -%}
{% set mariadb_dict = {} -%}
-{% set mariadb_server_list = [] -%}
-{% set receiver_port_list = [] -%}
-{% set mariadb_path_list = [] -%}
{% set ip = (ipv6_set | list)[0] -%}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% set cluster_list = [] -%}
@@ -12,12 +9,17 @@
"pfs", "linux", "readonly", "diskmonitor", "sqlerror", "compressbinlog", "bm4ci",
"mroonga", "utctime", "readcommitted", "nohandshake", "ssl"] -%}
{% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
-{% set database_slave_list = [] -%}
-{% set db_name_list = [] -%}
{% set count = namespace(value=2) %}
{% set caucase_bind = '[' ~ ip ~ ']:8890' -%}
{% set caucase_url = 'http://' ~ caucase_bind -%}
{% set csrid_list = [] -%}
+{% set default_parameter_dict = {"cluster1": {"db-prefered-master": "",
+ "database-amount": 2, "proxysql-user": "external",
+ "logical-backup-cron": "0 21 * * *", "physical-backup-cron": "0 1 * * *"}} -%}
+{% set cluster_name_list = slapparameter_dict.get('repman-cluster-dict', default_parameter_dict).keys() -%}
+{% set database_slave_dict = {} -%}
+{% set db_name_dict = {} -%}
+{% set default_slave_cluster_id = list(cluster_name_list)[0] -%}
{% macro password(name) -%}
[{{ name }}-password]
@@ -46,8 +48,14 @@ ssl = ${:etc}/ssl
[proxysql-directory]
recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}/proxy
config = ${directory:etc}/proxysql
-ssl = ${:config}/ssl
+ssl = ${:config}/default-ssl
+{% for cluster in cluster_name_list -%}
+{{ cluster }} = ${:home}/{{ cluster }}
+{% do database_slave_dict.__setitem__(cluster, []) -%}
+{% do db_name_dict.__setitem__(cluster, []) -%}
+{% endfor -%}
{% import "supervisord_lib" as supervisord_lib with context %}
{% set proxysql_controller = "proxysql-ctl" -%}
@@ -81,13 +89,17 @@ inline =
if [ -z "$CONFIG" ]; then
CONFIG="${proxysql-directory:config}/proxysql.cnf"
fi
+ ETCDIR=`dirname $CONFIG`
mkdir -p ${repman:config-tmp}/proxies
+ mkdir -p $ETCDIR/proxysql
+ ln -sf ${proxysql-directory:ssl} $ETCDIR/proxysql/ssl
cd ${repman:config-tmp}/proxies
{{ curl_bin }} -o proxies-$NAME.tar.gz ${nginx-parameter:repman-url}/api/clusters/$NAME/servers/$HOST/$PORT/config
tar -xzf proxies-$NAME.tar.gz
cp etc/proxysql/proxysql.cnf $CONFIG
output = ${directory:bin}/update-proxysql-config
+{% set slave_information_list = [] -%}
{% for instance_dict in slave_instance_list -%}
{% set slave_dict = {
'name': 'db_%s' % instance_dict['slave_reference'].replace('-', '_').lower(),
@@ -95,26 +107,26 @@ output = ${directory:bin}/update-proxysql-config
'password': instance_dict.get('db_password', '${' ~ instance_dict['slave_reference'] ~ '-password:passwd}'),
'slave_reference': instance_dict['slave_reference'],
'charset': instance_dict.get('db_charset', ''),
+ 'cluster_id': instance_dict.get('cluster_id', default_slave_cluster_id),
'require_ssl': True
} -%}
-{% do database_slave_list.append(slave_dict) -%}
-{% do db_name_list.append([slave_dict['name'], slave_dict['user']]) -%}
+{% do slave_information_list.append(slave_dict) -%}
+{% do database_slave_dict[slave_dict['cluster_id']].append(slave_dict) -%}
+{% do db_name_dict[slave_dict['cluster_id']].append([slave_dict['name'], slave_dict['user']]) -%}
{{ password(instance_dict['slave_reference']) }}
{% endfor %}
+# set each database dict which will be used to publish slave information
[database-slave-information]
-{% for slave_dict in database_slave_list -%}
-{{ slave_dict['name'] }} = !py!{{ slave_dict }}
+{% for slave_dict in slave_information_list -%}
+{{ slave_dict['name'] }} = !py!{{ slave_dict }}
{% endfor %}
-{% set db_list = db_name_list -%}
{% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%}
-{% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "",
- "database-amount": 2, "proxysql-user": "external",
- "logical-backup-cron": "0 21 * * *", "physical-backup-cron": "0 1 * * *"}} -%}
{% for name, parameter_dict in slapparameter_dict.get('repman-cluster-dict', default_parameter_dict).items() -%}
{% do mariadb_dict.__setitem__('innodb-file-per-table', parameter_dict.get('innodb-file-per-table', 1)) -%}
{% do mariadb_dict.__setitem__('use-ipv6', parameter_dict.get('use-ipv6', True)) -%}
+{% set db_list = db_name_dict[name] -%}
# Request mariadb instances
{% set db_amount = parameter_dict.get('database-amount', 2) -%}
@@ -123,16 +135,19 @@ output = ${directory:bin}/update-proxysql-config
{% endif -%}
{% set count.value = count.value + db_amount %}
+{% set mariadb_path_list = [] -%}
+{% set mariadb_server_list = [] -%}
+{% set receiver_port_list = [] -%}
{% for i in range(0, db_amount) -%}
{% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%}
-{% set section = 'request-mariadb-' ~ i -%}
+{% set section = 'request-mariadb-' ~ i ~ '-' ~ name -%}
{% set dbname = 'Mariadb-' ~ i %}
-
+{% set instance_name = dbname ~ '-' ~ name-%}
[{{ section }}]
<= request-common
software-type = mariadb
-name = {{ dbname ~ '-' ~ name}}
+name = {{ instance_name }}
sla-computer_guid = {{ dumps(parameter_dict.get('-sla-' ~ i ~'-computer_guid', '')) }}
{% for key, value in mariadb_dict.items() -%}
config-{{ key }} = {{ dumps(value) }}
@@ -145,8 +160,8 @@ config-repman-passwd = ${repman-parameter:password}
config-repman-url = ${nginx-parameter:backend-url}
config-repman-secure-url = ${nginx-parameter:backend-ssl-url}
config-cluster = {{ name }}
-config-name = {{ dbname }}
-config-database-list = !py!{{ database_slave_list }}
+config-name = {{ instance_name }}
+config-database-list = !py!{{ database_slave_dict[name] }}
config-database-name = {{ dumps(db_list) }}
config-require-ssl = {{ dumps(slapparameter_dict.get('require-ssl', False)) }}
return =
@@ -160,7 +175,7 @@ return =
{% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%}
{% do receiver_port_list.append('${' ~ section ~ ':connection-receiver-port}') -%}
{% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%}
-{% do monitor_base_url_dict.__setitem__('mariadb' ~ i, '${' ~ section ~ ':connection-monitor-base-url}') -%}
+{% do monitor_base_url_dict.__setitem__(instance_name, '${' ~ section ~ ':connection-monitor-base-url}') -%}
{% do csrid_list.append('${' ~ section ~ ':connection-csr-id}') -%}
{% endfor -%}
@@ -205,7 +220,7 @@ db-prefered-master = {{ mariadb_server_list[0] }}
proxysql-servers = {{ ipv4 }}
proxysql-servers-ipv6 = [{{ ip }}]
password = ${repman-parameter:password}
-proxysql-partition = ${buildout:directory}
+proxysql-partition = {{ '${proxysql-directory:' ~ name ~ '}' }}
receiver-port-list = {{ receiver_port_list | join(',') }}
enabled-tags = {{ parameter_dict.get("tag-list", tag_list) | join(',') }}
proxy-tags = {{ parameter_dict.get("proxy-tags", ["pkg", "masterslave", "linux", "noreadwritesplit", "ssl"]) | join(',') }}
@@ -248,7 +263,7 @@ context =
recipe = plone.recipe.command
# if Repman is not started, cannot download config from server
stop-on-error = false
-config = ${proxysql-directory:config}/proxysql.cnf
+config = ${proxysql-directory:home}/{{ name }}/etc/proxysql.cnf
data = ${repman:proxy-data}/{{ name }}
command =
mkdir -p ${:data} &&
@@ -313,7 +328,7 @@ RootSoftwareInstance = ${:default}
[dynamic-{{ name}}-publish-slave-information]
recipe = slapos.recipe.template:jinja2
-output = ${buildout:directory}/database-publish-slave-information.cfg
+output = ${buildout:directory}/database-{{ name}}-publish-slave-information.cfg
extensions = jinja2.ext.do
url = {{ template_publish_slave_information }}
context =
@@ -321,6 +336,7 @@ context =
key proxy_port {{ name }}-cluster-parameter:proxy-port
raw eggs_directory {{ eggs_directory }}
raw develop_eggs_directory {{ develop_eggs_directory }}
+ raw cluster {{ name }}
raw ipv6 {{ ip }}
raw ipv4 {{ ipv4 }}
section slave_dict database-slave-information
@@ -617,7 +633,7 @@ recipe = slapos.cookbook:publish-early
monitor-password monitor-htpasswd:passwd
db-root-password gen-root-password:passwd
repman-password repman-password:passwd
-database-list = {{ db_list }}
+database-dict = {{ db_name_dict }}
[publish-connection-parameter]
<= monitor-publish
@@ -626,7 +642,7 @@ recipe = slapos.cookbook:publish
backend-url = ${nginx-parameter:backend-ssl-url}
url = ${repman-frontend:connection-secure_access}
username = ${repman-parameter:username}
-database-list = {{ db_list }}
+database-dict = {{ db_name_dict }}
{% for name, value in publish_dict.items() -%}
{{ name }} = {{ value }}
{% endfor %}
diff --git a/software/repman/publish-database-slave-parameters.cfg.in b/software/repman/publish-database-slave-parameters.cfg.in
index 7a95f5594..6b1cb73d3 100644
--- a/software/repman/publish-database-slave-parameters.cfg.in
+++ b/software/repman/publish-database-slave-parameters.cfg.in
@@ -11,6 +11,7 @@ cert_file = ${slap-connection:cert-file}
{% for _, database in slave_dict.items() -%}
+{% if database['cluster_id'] == cluster -%}
{% set slave_reference = database['slave_reference'] -%}
{% do part_list.append('publish-slave-' ~ slave_reference) -%}
@@ -24,6 +25,8 @@ url = {{ "mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ipv
url = {{ "mysql://" ~ ipv4 ~ ":" ~ proxy_port ~ database['name'] }}
url_v6 = {{ "mysql://[" ~ ip ~ "]:" ~ proxy_port ~ database['name'] }}
{% endif -%}
+cluster = {{ database['cluster_id'] }}
+{% endif -%}
{% endfor %}
[buildout]
diff --git a/software/repman/templates/repman-manager.sh.in b/software/repman/templates/repman-manager.sh.in
index 0df77cf6e..30b89a8fc 100644
--- a/software/repman/templates/repman-manager.sh.in
+++ b/software/repman/templates/repman-manager.sh.in
@@ -59,14 +59,42 @@ activate_proxy () {
fi
}
+check_add () {
+ NAME=$1
+ STATUS=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/$NAME)
+ if [ $STATUS != 200 ]; then
+ # if cluster doesn't exists we add it
+ curl -H "Authorization: Bearer ${TOKEN}" \
+ {{ secure_url }}/api/clusters/actions/add/$NAME
+ fi
+}
+
+reload_config () {
+ # check if etc/repman/cluster.d/config-clusterXX.toml has changed and call reload
+ NAME=$1
+ FILE="{{ parameter_dict['config-tmp'] }}/$NAME-config-cheksum.txt"
+ CHECKSUM=$(md5sum {{ parameter_dict['clusters'] }}/config-$NAME.toml)
+ if [ -s $FILE ]; then
+ OLDSUM=$(cat $FILE);
+ fi
+ echo "$CHECKSUM" > $FILE
+ if [ "$CHECKSUM" != "$OLDSUM" ]; then
+ echo "Reloading settings for $NAME to apply changes..."
+ curl -H "Authorization: Bearer ${TOKEN}" \
+ {{ secure_url }}/api/clusters/$NAME/settings/actions/reload
+ else
+ echo "$NAME config is up to date."
+ fi
+}
+
TOKEN=$(get_token | {{ jq_bin }} -r '.token')
# Always reload cluster configuration to apply recent changes
{% for name in cluster_name_list -%}
-# reload {{ name }} settings
-echo "Reloading settings for {{ name }}..."
-curl -H "Authorization: Bearer ${TOKEN}" \
- {{ secure_url }}/api/clusters/{{ name }}/settings/actions/reload
+
+check_add {{ name }}
+
+reload_config {{ name }}
# Start Replication on {{ name }}
if [ ! -f "{{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped" ]; then
--
2.30.9