repman: include restic for backup, add more parameter for instance and json parameter schema

software/repman/buildout.hash.cfg

@@ -14,19 +14,19 @@
 # not need these here).
 [instance.cfg]
 filename = instance.cfg.in
-md5sum = 158e4c884b04442aff9726704b3f4d60
+md5sum = b38fb4c5cdba2223c84d62ef60513e49
 
 [instance-repman.cfg]
 _update_hash_filename_ = instance-repman.cfg.jinja2.in
-md5sum = 0a17bad0a5e7f472641c8a2984ae9400
+md5sum = e50536613b175a2e25a041c61adeb59f
 
 [config-toml.in]
 _update_hash_filename_ = templates/config.toml.in
-md5sum = 5114d1bb43025cdc5357572634871da7
+md5sum = 1efa65af63b14ca2dd1b8c692b2af80e
 
 [config-cluster-toml.in]
 _update_hash_filename_ = templates/cluster-config.toml.in
-md5sum = d883337a75140c08853960cd6a8fdc5f
+md5sum = 39fb83a9b542b9809948cab98aefa2e7
 
 [nginx.conf.in]
 _update_hash_filename_ = templates/nginx.conf.in
@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
 
 [template-mariadb.cfg]
 _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
-md5sum = 57ffd0fc1a7bca46b07cdc3121bbf36f
+md5sum = d32a50db737a9131de7f3eb27f704232
 
 [template-my-cnf]
 _update_hash_filename_ = templates/my.cnf.in
@@ -42,11 +42,11 @@ md5sum = 04a12d3271857fcf29cb010fd7e38a5c
 
 [template-mariadb-initial-setup]
 _update_hash_filename_ = templates/mariadb_initial_setup.sql.in
-md5sum = 6c88f4a98668d7c5c0aec14ae662686e
+md5sum = c31618ef115ecd72b30ae59d0440a3ec
 
 [mariadb-init-root-sql]
 _update_hash_filename_ = templates/mariadb_init_root.sql.in
-md5sum = bd39fb1e20212c820e5c6b74102b72e8
+md5sum = 864d419ea506c9ac543409ba7abb22b0
 
 [init-root-wrapper-in]
 _update_hash_filename_ = templates/init_root_wrapper.in
@@ -54,7 +54,7 @@ md5sum = 3ba02b99c3fcbc24b3659917980b310f
 
 [repman-manager-sh.in]
 _update_hash_filename_ = templates/repman-manager.sh.in
-md5sum = 8ad6565e7b26e57b41bda4023a9d29ed
+md5sum = 99324b56192003254081ef336dcee94c
 
 [dbjobs-in]
 _update_hash_filename_ = templates/dbjobs.in

software/repman/instance-mariadb.cfg.jinja2.in

 {% set part_list = [] -%}
 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
-{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
+{% set use_ipv6 = slapparameter_dict.get('use-ipv6', True) -%}
 {% set port = slapparameter_dict['tcp-port'] %}
 {% set host = (ipv4_set | list)[0] -%}
 {% if use_ipv6 -%}
 {%   set ip = (ipv6_set | list)[0] -%}
-{%   set host = '[' ~ (ipv4_set | list)[0] ~ ']' -%}
+{%   set host = '[' ~ ip ~ ']' -%}
 {% else -%}
 {%   set ip = (ipv4_set | list)[0] -%}
 {% endif -%}
 {% set dash = parameter_dict['dash-location'] ~ '/bin/dash' %}
 {% set database_list = slapparameter_dict.get('database-list', [{'name': 'repdb', 'user': 'user', 'password': 'insecure'}]) -%}
 
+# XXX- TODO: add cron with check db need restart then restart
+# API check restard needed: http://repman/api/clusters/{clusterName}/servers/{serverName}/{serverPort}/need-restart
+
 [{{ section('publish') }}]
 recipe = slapos.cookbook:publish.serialised
 -extends = publish-early
@@ -274,7 +277,7 @@ cluster = cluster1
 config = ${directory:etc}/mysql/my.cnf
 command =
   cd ${directory:config-tmp} &&
-  {{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ ip }}/{{ port }}/config &&
+  {{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ host }}/{{ port }}/config &&
   tar -xzf config.tar.gz &&
   cp -r data/.system ${directory:mysql} &&
   rm -rf ${directory:etc}/mysql &&

software/repman/instance-repman-output-schema.json

+{
+  "name": "Output Parameters",
+  "properties": {
+    "backend-url": {
+      "title": "Backend URL",
+      "description": "URL used to connect directly to backend without frontend. Requires IPv6.",
+      "type": "string",
+      "format": "uri",
+      "require": true
+    },
+    "url": {
+      "title": "URL",
+      "description": "URL used to connect to the service.",
+      "type": "string",
+      "format": "uri",
+      "require": false
+    },
+    "repman-password": {
+      "title": "Repman password",
+      "description": "Password for Replication Manager service.",
+      "type": "string",
+      "require": false
+    }
+  }
+}

software/repman/instance.cfg.in

@@ -60,6 +60,7 @@ extra-context =
     raw config_cluster_toml_in     {{ config_cluster_toml_in }}
     raw nginx_conf_in              {{ nginx_conf_in }}
     raw rsync_location             {{ rsync_location }}
+    raw restic_bin_location        {{ restic_bin_location }}
     raw sysbench_location          {{ sysbench_location }}
     raw proxysql_location          {{ proxysql_location }}
     raw template_repman_manager_sh {{ template_repman_manager_sh }}

software/repman/software.cfg

@@ -4,6 +4,7 @@ extends =
   buildout.hash.cfg
   ../neoppod/software-common.cfg
   ../../stack/monitor/buildout.cfg
+  ../../component/restic/buildout.cfg
   ../../component/replication-manager/buildout.cfg
   ../../component/mariadb/buildout.cfg
   ../../component/nginx/buildout.cfg
@@ -27,6 +28,8 @@ parts =
   instance.cfg
   template-mariadb.cfg
   template-mysqld-wrapper
+  gowork
+  restic
 
 [mariadb]
 # Compile dir is for plugins, there's no plugin yet
@@ -62,6 +65,7 @@ context =
     key percona_toolkit_location percona-toolkit:location
     key repman_src_location git.signal18.io_signal18_repman:location
     key rsync_location rsync:location
+    key restic_bin_location restic:location
     key socat_location socat:location
     key template_repman_manager_sh repman-manager-sh.in:target
     key template_mariadb template-mariadb.cfg:target

software/repman/software.cfg.json

+{
+  "name": "Replication Manager",
+  "description": "Replication Manager",
+  "serialisation": "xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "description": "Replication Manager",
+      "request": "instance-repman-input-schema.json",
+      "response": "instance-repman-output-schema.json",
+      "index": 0
+    }
+  }
+}

software/repman/templates/cluster-config.toml.in

+{% macro setbool(value) -%}
+{% if value in ['true', 'True'] %} true {% else %} false {% endif -%}
+{% endmacro -%}
 [{{ parameter_dict['name'] }}]
 title = "{{ parameter_dict['name'] }}"
+monitoring-save-config = true
 db-servers-hosts = "{{ parameter_dict['db-list'] }}"
 db-servers-prefered-master = "{{ parameter_dict['db-prefered-master'] }}"
 db-servers-credential = "{{ parameter_dict['db-credential'] }}"
@@ -16,22 +20,48 @@ proxysql-bootstrap = true
 proxysql-admin-port = {{ parameter_dict['proxy-admin-port'] }}
 proxysql-password = "{{ parameter_dict['password'] }}"
 
-prov-proxy-tags = "pkg,masterslave,linux,noreadwritesplit"
+prov-proxy-tags = "{{ parameter_dict['proxy-tags'] }}"
 
 monitoring-scheduler = true
 scheduler-db-servers-logical-backup  = true
-scheduler-db-servers-logical-backup-cron= "0 0 1 * * 6"
+scheduler-db-servers-logical-backup-cron = "0 {{ parameter_dict['logical-backup-cron'] }}"
 scheduler-db-servers-logs   =  true
 scheduler-db-servers-logs-cron = "0 0 * * * *"
-#scheduler-db-servers-logs-table-keep = 12
 scheduler-db-servers-logs-table-keep = 4
 scheduler-db-servers-logs-table-rotate  = true
-#scheduler-db-servers-logs-table-rotate-cron = "0 0 0/6 * * *"
 scheduler-db-servers-logs-table-rotate-cron = "0 0 23 * * *"
 scheduler-db-servers-optimize  = true
 scheduler-db-servers-optimize-cron = "0 0 3 1 * 5"
 scheduler-db-servers-physical-backup = true
-scheduler-db-servers-physical-backup-cron = "0 0 0 * * *"
+scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backup-cron'] }}"
 backup-physical-type = "mariabackup"
 backup-logical-type = "mysqldump"
-# scheduler-db-servers-receiver-ports="4444,4445"
+# scheduler-db-servers-receiver-ports="4444,4445"
\ No newline at end of file
+
+prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }}
+prov-proxy-memory = {{ parameter_dict['proxy-memory'] }}
+prov-db-cpu-cores = {{ parameter_dict['db-cpu-cores'] }}
+prov-db-disk-iops = {{ parameter_dict['db-disk-iops'] }}
+prov-db-memory = {{ parameter_dict['db-memory'] }}
+prov-db-memory-shared-pct = "{{ parameter_dict['db-memory-shared-pct'] }}"
+prov-db-memory-threaded-pct = "{{ parameter_dict['db-memory-threaded-pct'] }}"
+
+test-inject-traffic = true
+# failover
+failover-mode = "{{ parameter_dict['failover-mode'] }}"
+failover-limit = {{ parameter_dict['failover-limit'] }}
+failover-falsepositive-heartbeat = {{ setbool(parameter_dict['failover-falsepositive-heartbeat']) }}
+failover-falsepositive-heartbeat-timeout = {{ parameter_dict['failover-falsepositive-heartbeat-timeout'] }}
+failover-falsepositive-ping-counter = {{ parameter_dict['failover-falsepositive-ping-counter'] }}
+failover-max-slave-delay = {{ parameter_dict['failover-max-slave-delay'] }}
+failover-readonly-state = {{ setbool(parameter_dict['failover-readonly-state']) }}
+failover-restart-unsafe = {{ setbool(parameter_dict['failover-restart-unsafe']) }}
+failover-time-limit = {{ parameter_dict['failover-time-limit'] }}
+#switchover
+switchover-at-equal-gtid = {{ setbool(parameter_dict['switchover-at-equal-gtid']) }}
+switchover-slave-wait-catch = {{ setbool(parameter_dict['switchover-slave-wait-catch']) }}
+switchover-wait-kill = {{ parameter_dict['switchover-wait-kill'] }}
+switchover-wait-trx = {{ parameter_dict['switchover-wait-trx'] }}
+switchover-wait-write-query = {{ parameter_dict['switchover-wait-write-query'] }}
+
+

software/repman/templates/config.toml.in

 [Default]
 api-bind = "{{ parameter_dict['ipv4'] }}"
 http-bind-address = "{{ parameter_dict['ipv4'] }}"
+http-server = true
+http-session-lifetime = {{ parameter_dict['http-session-lifetime'] }}
+http-refresh-interval = {{ int(parameter_dict['http-refresh-interval'])*1000 }}
 monitoring-save-config = false
 api-https-bind = true
 api-credentials = "{{ parameter_dict['username'] }}:{{ parameter_dict['password'] }}"
@@ -10,11 +13,10 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
 # Database list of hosts to ignore in election
 #db-servers-ignored-hosts =
 # Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
-# XXX - check if mandatory (as present in cluster)
-#db-servers-hosts =
+monitoring-address = "{{ parameter_dict['ipv4'] }}"
 
 #haproxy = true
-haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
+#haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
 # HaProxy input bind address for read (default "0.0.0.0")
 #haproxy-ip-read-bind =
 # HaProxy input bind address for write (default "0.0.0.0")
@@ -28,6 +30,17 @@ haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
 #HaProxy read-write port to leader (default 3306)
 #haproxy-write-port =
 
+# Use restic to archive and restore backups
+backup = true
+backup-restic = true
+backup-restic-binary-path = "{{ parameter_dict['restic-bin'] }}"
+backup-restic-aws = false
+backup-restic-password = "{{ parameter_dict['password'] }}"
+
+backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
+backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
+backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
+
 # Mail configuration
 # Alert email sender (default "mrm@localhost")
 mail-from = "{{ parameter_dict['mail-from'] }}"
@@ -38,12 +51,8 @@ mail-smtp-user = "{{ parameter_dict['mail-smtp-user'] }}"
 # Alert email recipients, separated by commas
 mail-to = "{{ parameter_dict['mail-to'] }}"
 
-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
-
 prov-orchestrator = "slapos"
-prov-db-tags="gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog"
+prov-db-tags = "{{ parameter_dict['enabled-tags'] }}"
 
 sysbench-binary-path = "{{ parameter_dict['sysbench-bin'] }}"
 # Number of threads to run benchmark (default 4)

software/repman/templates/mariadb_init_root.sql.in

@@ -6,5 +6,7 @@ CREATE USER 'repman'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}
 GRANT ALL ON *.* TO 'repman'@'localhost' WITH GRANT OPTION ;
 CREATE USER 'repman'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
 GRANT ALL ON *.* TO 'repman'@'%' WITH GRANT OPTION ;
+CREATE USER 'repman'@'::' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
+GRANT ALL ON *.* TO 'repman'@'::' WITH GRANT OPTION ;
 DROP DATABASE IF EXISTS test ;
 FLUSH PRIVILEGES ;
\ No newline at end of file

software/repman/templates/mariadb_initial_setup.sql.in

@@ -14,6 +14,7 @@ CREATE DATABASE IF NOT EXISTS `{{ name }}`;
 {% if user -%}
 GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
 GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
+GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@'::' IDENTIFIED BY '{{ password }}';
 {%- endif %}
 {% endmacro -%}
 

software/repman/templates/repman-manager.sh.in

@@ -17,14 +17,26 @@ wait_database () {
     else
       if [ $CODE -eq 200 ]; then
         break;
-      else:
-        echo ">> [$retry] waitdatabases returned code $CODE...";
       fi
+      echo ">> [$retry] waitdatabases returned code $CODE...";
     fi
     sleep 1
   done
 }
 
+activate_proxy () {
+  NAME=$1
+  URL="{{ secure_url }}/api/clusters/$NAME/settings/actions/switch/database-hearbeat"
+  echo ">> Calling $URL...";
+  CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" $URL)
+  if [ $CODE -eq 200 ]; then
+    return 0;
+  else
+    echo ">> ERROR: failed to activate proxy: $URL returned code $CODE"
+    return 1;
+  fi
+}
+
 TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login | {{ jq_bin }} -r '.token')
 
 # Always reload cluster configuration to apply recent changes
@@ -45,10 +57,14 @@ if [ ! -f "{{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped" ]; then
   curl -H "Authorization: Bearer ${TOKEN}" \
     {{ secure_url }}/api/clusters/{{ name }}/actions/replication/cleanup
   CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/{{ name }}/actions/replication/bootstrap/master-slave)
+  SUCCESS=0
   if [ $CODE -eq 200 ]; then
-    # Mark boostrap done!
-    echo "Cluster {{ name }} replication bootstrapped"
-    echo "DO NOT REMOVE THIS FILE" > {{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped
+    activate_proxy {{ name }}
+    if [ $? -eq 0 ]; then
+      # Mark boostrap done!
+      echo "Cluster {{ name }} replication bootstrapped"
+      echo "DO NOT REMOVE THIS FILE" > {{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped
+    fi
   else
     echo "ERROR: Failed to bootstrap cluster {{ name }}... http_code $CODE"
   fi