repman: support for ssl secure connection with repman, proxysql and mariadb

Communication between repman and mariadb can now use ssl for
secure connection.
Proxysql is also configured to work with ssl, user access
database with secure connection.

software/repman/buildout.hash.cfg

@@ -18,15 +18,15 @@ md5sum = af2fc4a7a0f782fed2cb1112ef3cb397
 
 [instance-repman.cfg]
 _update_hash_filename_ = instance-repman.cfg.jinja2.in
-md5sum = d9151fe9f86fb1700c6c33d30a750837
+md5sum = c6e41783b8e3e844b6319953a8d665ec
 
 [config-toml.in]
 _update_hash_filename_ = templates/config.toml.in
-md5sum = 28ab31bbf073f777221f90902e9d435e
+md5sum = dc9fdcf6c3c775da6b0b72f83c4a4105
 
 [config-cluster-toml.in]
 _update_hash_filename_ = templates/cluster-config.toml.in
-md5sum = c78c6be9537d3dffbb526b0e819bacb5
+md5sum = ff44b81dc1177e32019ddbf4f1c6b616
 
 [nginx.conf.in]
 _update_hash_filename_ = templates/nginx.conf.in
@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
 
 [template-mariadb.cfg]
 _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
-md5sum = 53734d9527308a255ddb35677d143579
+md5sum = 938f1e8087395757f5ae861d20500658
 
 [template-my-cnf]
 _update_hash_filename_ = templates/my.cnf.in
@@ -42,7 +42,7 @@ md5sum = f3661b788099bb31d71ba6e7d36836d9
 
 [template-mariadb-initial-setup]
 _update_hash_filename_ = templates/mariadb_initial_setup.sql.in
-md5sum = 28fdc24f85b82242751b6c3f40267b39
+md5sum = 47b76144e1b116580c8acf08274af976
 
 [template-publish-slave-information]
 _update_hash_filename_ = publish-database-slave-parameters.cfg.in
@@ -50,7 +50,7 @@ md5sum = 9616477ff993c55c2b43bf8797db6994
 
 [mariadb-init-root-sql]
 _update_hash_filename_ = templates/mariadb_init_root.sql.in
-md5sum = d8a0a7d72b02f911dafda655941b805d
+md5sum = 6e5c02542568f250aacd84694249efc0
 
 [init-root-wrapper-in]
 _update_hash_filename_ = templates/init_root_wrapper.in

software/repman/instance-mariadb.cfg.jinja2.in

@@ -69,11 +69,11 @@ mode = {{ mode }}
      buildout_bin_directory=buildout_bin_directory,
      updater_path='${directory:services}/caucase-updater',
      url=slapparameter_dict['caucase-url'],
-     data_dir='${directory:srv}/caucase-updater',
-     crt_path='${directory:ssl}/mariadb-cert.crt',
-     ca_path='${directory:srv}/caucase-updater/ca.crt',
-     crl_path='${directory:srv}/caucase-updater/crl.pem',
-     key_path='${directory:ssl}/mariadb-cert.key',
+     data_dir='${mysql-directory:ssl}',
+     crt_path='${mysql-directory:ssl}/server-cert.pem',
+     ca_path='${mysql-directory:ssl}/ca-cert.pem',
+     crl_path='${mysql-directory:ssl}/crl.pem',
+     key_path='${mysql-directory:ssl}/server-key.pem',
      on_renew=None,
      max_sleep=None,
      template_csr_pem=None,
@@ -86,7 +86,7 @@ mode = {{ mode }}
 recipe = plone.recipe.command
 output = ${directory:tmp}/csr_id
 command = 
-  if [ -s "${directory:ssl}/mariadb-cert.crt" ]; then
+  if [ -s "${mysql-directory:ssl}/server-cert.pem" ]; then
     RESULT="None";
   else
     if [ -f "${caucase-updater-csr:csr}" ]; then
@@ -110,7 +110,7 @@ csr-id =
 [my-cnf-parameters]
 socket = ${directory:run}/mariadb.sock
 ip = {{ ip }}
-data-directory = ${directory:mysql}
+data-directory = ${mysql-directory:data}
 pid-file = ${directory:run}/mariadb.pid
 plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }}
 groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }}
@@ -120,9 +120,6 @@ innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0
 innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
 innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
 relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
-ssl-crt = ${directory:ssl}/mariadb-cert.crt
-ssl-key = ${directory:ssl}/mariadb-cert.key
-ssl-ca-crt = ${directory:srv}/caucase-updater/ca.crt
 
 [my-cnf]
 < = jinja2-template-base
@@ -143,6 +140,7 @@ database-list = {{ dumps(database_list) }}
 mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }}
 root-user = repman
 heartbeat-user = {{ slapparameter_dict['heartbeat-user'] }}
+require-ssl = {{ dumps(slapparameter_dict['require-ssl']) }}
 
 [init-script]
 < = jinja2-template-executable
@@ -202,14 +200,14 @@ recipe = slapos.cookbook:wrapper
 command-line = ${mysqld:rendered}
 wrapper-path = ${directory:controller}/mariadb
 wait-for-files =
-  ${directory:etc}/mysql/my.cnf
-  ${directory:ssl}/mariadb-cert.crt
+  ${mysql-directory:mysql}/my.cnf
+  ${mysql-directory:ssl}/server-cert.pem
 
 {% import "supervisord_lib" as supervisord_lib with context %}
 {{ supervisord_lib.supervisord("mariadb-ctl", buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
 {% do part_list.append("supervisord-mariadb-ctl") -%}
 {% set maradb_program_dict = {"name": "mariadb", "command": "${mysqld-launcher:wrapper-path}",
-  "stopwaitsecs": 86400, "environment": [],
+  "stopwaitsecs": 86400, "startretries": 10, "autorestart": True, "environment": [],
   "stdout_logfile": "${directory:log}/mariadb_stdout.log",
   "stderr_logfile": "${directory:log}/mariadb_stdout.log" } %}
 
@@ -297,25 +295,26 @@ plugin = ${:etc}/plugin
 srv = ${buildout:directory}/srv
 tmp = ${buildout:directory}/tmp
 backup = ${:srv}/backup
-mariadb-backup-full = ${:backup}/mariadb-full
-mariadb-backup-incremental = ${:backup}/mariadb-incremental
-mariadb-ssl = ${:etc}/mariadb-ssl
 var = ${buildout:directory}/var
 lib = ${:var}/lib
-mysql = ${:lib}/mysql
 log = ${:var}/log
 run = ${:var}/run
 config-tmp = ${:tmp}/config
-custom = ${directory:etc}/mysql/custom
-ssl = ${:etc}/ssl
+
+[mysql-directory]
+recipe = slapos.cookbook:mkdirectory
+mysql = ${directory:etc}/mysql
+ssl = ${:mysql}/ssl
+data = ${directory:lib}/mysql
+custom = ${:mysql}/custom.d
 
 [dbjob-parameter]
 bash-bin = {{ bash_bin }}
 db-user = ${init-script-parameters:root-user}
 db-password = ${init-script-parameters:password}
-mysql-dir = ${directory:mysql}
-dbjob-cnf = ${directory:etc}/mysql/dbjob.cnf
-log-dir = ${directory:mysql}/.system/logs
+mysql-dir = ${mysql-directory:data}
+dbjob-cnf = ${mysql-directory:mysql}/dbjob.cnf
+log-dir = ${mysql-directory:data}/.system/logs
 tmp-dir = ${directory:tmp}
 mysqld-socket  = ${my-cnf-parameters:socket}
 mysql-client = ${binary-wrap-mysql:wrapper-path}
@@ -382,11 +381,11 @@ template = inline:#!{{ bash_bin }}
   cd ${directory:config-tmp} &&
   {{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ host }}/{{ port }}/config
   tar -xzf config.tar.gz
-  cp -r data/.system ${directory:mysql}
-  rm -rf ${directory:etc}/mysql
-  cp -r etc/mysql ${directory:etc}
-  ln -sf ${directory:mysql}/.system ${directory:var}/system
-  ln -sf ${my-cnf:rendered} ${directory:etc}/mysql/custom.d/01_mariadb.cnf
+  rm -r etc/mysql/ssl  # we don't need generated ssl files
+  cp -r data/.system ${mysql-directory:data}
+  cp -r etc/mysql/* ${mysql-directory:mysql}
+  ln -sf ${mysql-directory:data}/.system ${directory:var}/system
+  ln -sf ${my-cnf:rendered} ${mysql-directory:custom}/01_mariadb.cnf
 
 [{{ section('install-mysql-config') }}]
 recipe = plone.recipe.command

software/repman/instance-repman-input-schema.json

@@ -113,7 +113,8 @@
         "mroonga",
         "utctime",
         "readcommitted",
-        "nohandshake"
+        "nohandshake",
+        "ssl"
       ]
     },
     "http-session-lifetime": {
@@ -140,6 +141,12 @@
       "type": "boolean",
       "default": true
     },
+    "require-ssl": {
+      "title": "Enable REQUIRE SSL for repman database users",
+      "description": "Enable secure connection between repman and mariadb databases. This option is an initial value, used only when creating database users.",
+      "type": "boolean",
+      "default": false
+    },
     "repman-cluster-dict": {
       "title": "Replication Manager clusters definition",
       "description": "Replication Manager clusters definition",
@@ -197,7 +204,7 @@
             },
             "proxy-tags": {
               "title": "Proxy tag list",
-              "description": "playbook configuration tags. Default: [\"pkg\", \"masterslave\", \"linux\", \"noreadwritesplit\"]",
+              "description": "playbook configuration tags. Default: [\"pkg\", \"masterslave\", \"linux\", \"noreadwritesplit\", \"ssl\"]",
               "type": "array",
               "items": {
                 "type": "string"
@@ -206,7 +213,8 @@
                 "pkg",
                 "masterslave",
                 "linux",
-                "noreadwritesplit"
+                "noreadwritesplit",
+                "ssl"
               ]
             },
             "logical-backup-cron": {

software/repman/instance-repman.cfg.jinja2.in

@@ -8,11 +8,13 @@
 {% set ip = (ipv6_set | list)[0] -%}
 {% set ipv4 = (ipv4_set | list)[0] -%}
 {% set cluster_list = [] -%}
-{% set tag_list = ["gtidstrict", "bind", "pkg", "innodb", "noquerycache", "slow", "pfs", "linux", "readonly", "diskmonitor", "sqlerror", "compressbinlog", "bm4ci", "mroonga", "utctime", "readcommitted", "nohandshake"] -%}
+{% set tag_list = ["gtidstrict", "bind", "pkg", "innodb", "noquerycache", "slow",
+  "pfs", "linux", "readonly", "diskmonitor", "sqlerror", "compressbinlog", "bm4ci",
+  "mroonga", "utctime", "readcommitted", "nohandshake", "ssl"] -%}
 {% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
 {% set database_slave_list = [] -%}
 {% set db_name_list = [] -%}
-{% set count = namespace(value=1) %}
+{% set count = namespace(value=2) %}
 {% set caucase_bind = '[' ~ ip ~ ']:8890' -%}
 {% set caucase_url = 'http://' ~ caucase_bind -%}
 {% set csrid_list = [] -%}
@@ -42,10 +44,21 @@ backup-caucased = ${:srv}/backup-caucased
 caucased = ${:srv}/caucased
 ssl = ${:etc}/ssl
 
+[proxysql-directory]
+recipe = slapos.cookbook:mkdirectory
+config = ${directory:etc}/proxysql
+ssl = ${:config}/ssl
+
 {% import "supervisord_lib" as supervisord_lib with context %}
 {% set proxysql_controller = "proxysql-ctl" -%}
 {{ supervisord_lib.supervisord(proxysql_controller, buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
+
+[proxysql-controller-update]
+recipe = slapos.cookbook:wrapper
+command-line = ${proxysql-ctl-bin:wrapper-path} update
+wrapper-path = ${directory:scripts}/proxysql-ctl-update
 {% do part_list.append("supervisord-proxysql-ctl") -%}
+{% do part_list.append("proxysql-controller-update") -%}
 
 [request-common]
 recipe = slapos.cookbook:request.serialised
@@ -65,13 +78,13 @@ template = inline:#!{{ bash_bin }}
   PORT=$3
   CONFIG=$4
   if [ -z "$CONFIG" ]; then
-    CONFIG="${repman:proxies}/proxysql-$NAME.cnf"
+    CONFIG="${proxysql-directory:config}/proxysql.cnf"
   fi
   mkdir -p ${repman:config-tmp}/proxies
   cd ${repman:config-tmp}/proxies
   {{ curl_bin }} -o proxies-$NAME.tar.gz ${nginx-parameter:repman-url}/api/clusters/$NAME/servers/$HOST/$PORT/config
   tar -xzf proxies-$NAME.tar.gz
-  cp etc/proxysql.cnf $CONFIG
+  cp etc/proxysql/proxysql.cnf $CONFIG
 rendered = ${directory:bin}/update-proxysql-config
 mode = 755
 
@@ -81,7 +94,8 @@ mode = 755
   'user': instance_dict.get('db_user', 'user'),
   'password': instance_dict.get('db_password', '${' ~ instance_dict['slave_reference'] ~ '-password:passwd}'),
   'slave_reference': instance_dict['slave_reference'],
-  'charset': instance_dict.get('db_charset', '')
+  'charset': instance_dict.get('db_charset', ''),
+  'require_ssl': True
   } -%}
 {% do database_slave_list.append(slave_dict) -%}
 {% do db_name_list.append(slave_dict['name']) -%}
@@ -134,6 +148,7 @@ config-cluster = {{ name }}
 config-name = {{ dbname }}
 config-database-list = !py!{{ database_slave_list }}
 config-database-name = {{ dumps(db_list) }}
+config-require-ssl = {{ dumps(slapparameter_dict.get('require-ssl', False)) }}
 return =
   database-host
   receiver-port
@@ -168,6 +183,7 @@ maximum = 7132
 {% for key, value in parameter_dict.items() -%}
 {{ key }} = {{ value }}
 {% endfor -%}
+name = {{ name }}
 proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }}
 proxy-port = {{ '${' ~ name ~ '-port:port}' }}
 proxy-admin-port = {{ '${' ~ name ~ '-admin-port:port}' }}
@@ -190,7 +206,7 @@ password = ${repman-parameter:password}
 proxysql-partition = ${buildout:directory}
 receiver-port-list = {{ receiver_port_list | join(',') }}
 enabled-tags = {{ slapparameter_dict.get("tag-list", tag_list) | join(',') }}
-proxy-tags = {{ parameter_dict.get("proxy-tags", ["pkg", "masterslave", "linux", "noreadwritesplit"]) | join(',') }}
+proxy-tags = {{ parameter_dict.get("proxy-tags", ["pkg", "masterslave", "linux", "noreadwritesplit", "ssl"]) | join(',') }}
 logical-backup-cron = {{ parameter_dict.get("logical-backup-cron", "0 22 * * *") }}
 physical-backup-cron = {{ parameter_dict.get("physical-backup-cron", "0 0 * * *") }}
 proxy-cpu-cores = {{ parameter_dict.get("proxy-cpu-cores", 2) }}
@@ -230,7 +246,7 @@ context =
 recipe = plone.recipe.command
 # if Repman is not started, cannot download config from server
 stop-on-error = false
-config = ${repman:proxies}/proxysql-{{ name }}.cnf
+config = ${proxysql-directory:config}/proxysql.cnf
 data = ${repman:proxy-data}/{{ name }}
 command =
   mkdir -p ${:data} &&
@@ -277,7 +293,7 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
 
 {% set service_name = "proxysql-" ~ name -%}
 {% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}",
-  "stopwaitsecs": 60, "environment": [],
+  "stopwaitsecs": 60, "startretries": 10, "autorestart": True, "environment": [],
   "stdout_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log",
   "stderr_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log" } %}
 
@@ -365,6 +381,25 @@ context =
 {% do part_list.append('caucase-user-updater') -%}
 {% do part_list.append('caucase-user-updater-promise') -%}
 
+#caucase proxysql certificate
+{{ caucase.updater(
+     prefix='caucase-proxysql-updater',
+     buildout_bin_directory=buildout_bin_directory,
+     updater_path='${directory:services}/caucase-proxysql-updater',
+     url=caucase_url,
+     data_dir='${proxysql-directory:ssl}',
+     crt_path='${proxysql-directory:ssl}/client-cert.pem',
+     ca_path='${proxysql-directory:ssl}/ca-cert.pem',
+     crl_path='${proxysql-directory:ssl}/crl.pem',
+     key_path='${proxysql-directory:ssl}/client-key.pem',
+     on_renew=None,
+     max_sleep=None,
+     template_csr_pem=None,
+     openssl=openssl_bin,
+)}}
+{% do part_list.append('caucase-proxysql-updater') -%}
+{% do part_list.append('caucase-proxysql-updater-promise') -%}
+
 [caucase-sign-csr]
 recipe = slapos.recipe.template:jinja2
 mode = 755
@@ -411,7 +446,6 @@ etc      = ${directory:etc}/repman
 data-dir = ${directory:var}/lib
 root-dir = ${directory:srv}/repman
 clusters = ${:etc}/cluster.d
-proxies  = ${:etc}/proxy
 proxy-data = ${:data-dir}/proxy
 config-tmp = ${directory:tmp}/config
 bootstrap = ${:etc}/bootstrap
@@ -471,6 +505,8 @@ mail-to = {{ slapparameter_dict.get("mail-to", "") }}
 
 http-session-lifetime = {{ slapparameter_dict.get("http-session-lifetime", 86400) }}
 http-refresh-interval = {{ slapparameter_dict.get("http-refresh-interval", 4) }}
+ssl-cert = ${directory:ssl}/repman-cert.crt
+ssl-key = ${directory:ssl}/repman-cert.key
 
 [repman-config-folder]
 recipe = plone.recipe.command
@@ -518,6 +554,9 @@ depends =
   ${replication-manager-reload:recipe}
   ${repman-listen-promise:recipe}
   ${repman-listen-ssl-promise:recipe}
+wait-for-files =
+  ${directory:ssl}/repman-cert.crt
+  ${directory:ssl}/repman-cert.key
 
 [repman-config.toml]
 recipe = slapos.recipe.template:jinja2

software/repman/templates/cluster-config.toml.in

@@ -40,6 +40,7 @@ scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backu
 backup-physical-type = "mariabackup"
 backup-logical-type = "mysqldump"
 scheduler-db-servers-receiver-ports= "{{ parameter_dict['receiver-port-list'] }}"
+#scheduler-db-servers-receiver-use-ssl   ## set to true when ssl enabled on socat
 
 prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }}
 prov-proxy-memory = {{ parameter_dict['proxy-memory'] }}
@@ -49,6 +50,7 @@ prov-db-memory = {{ parameter_dict['db-memory'] }}
 prov-db-memory-shared-pct = "{{ parameter_dict['db-memory-shared-pct'] }}"
 prov-db-memory-threaded-pct = "{{ parameter_dict['db-memory-threaded-pct'] }}"
 
+replication-use-ssl = true
 test-inject-traffic = true
 # failover
 failover-mode = "{{ parameter_dict['failover-mode'] }}"

software/repman/templates/config.toml.in

@@ -22,6 +22,8 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
 # Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
 monitoring-address = "{{ parameter_dict['ipv4'] }}"
 monitoring-wait-retry = 1200
+monitoring-ssl-cert = "{{ parameter_dict['ssl-cert'] }}"
+monitoring-ssl-key = "{{ parameter_dict['ssl-key'] }}"
 
 #haproxy = true
 #haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"

software/repman/templates/mariadb_init_root.sql.in

@@ -2,16 +2,18 @@
 --  or products like mysql-fabric won't work
 SET @@SESSION.SQL_LOG_BIN=0;
 
+# {{ parameter_dict["require-ssl"] }}
+{% set ssl = parameter_dict["require-ssl"] -%}
 CREATE USER '{{ parameter_dict["root-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
 GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'localhost' WITH GRANT OPTION ;
-CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
+CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' {% if ssl %} REQUIRE SSL{% endif %};
 GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'%' WITH GRANT OPTION ;
 CREATE USER '{{ parameter_dict["root-user"] }}'@'::' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
 GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'::' WITH GRANT OPTION ;
 
 CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
 GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'localhost' WITH GRANT OPTION ;
-CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
+CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' {% if ssl %} REQUIRE SSL{% endif %};
 GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'%' WITH GRANT OPTION ;
 
 DROP DATABASE IF EXISTS test ;

software/repman/templates/mariadb_initial_setup.sql.in

 SET @@SESSION.SQL_LOG_BIN=0;
 
 USE mysql;
-{% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
-{% if mroonga %}
-#SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
-{% endif %}
 DROP FUNCTION IF EXISTS sphinx_snippets;
 #CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
 
@@ -36,5 +32,5 @@ GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@'::' IDENTIFIED BY '{{ pa
 {% endmacro -%}
 
 {% for entry in parameter_dict['database-list'] -%}
-{{   database(entry['name'], entry.get('user'), entry.get('password'), entry.get('require_ssl', True), entry.get('charset')) }}
+{{   database(entry['name'], entry.get('user'), entry.get('password'), entry.get('require_ssl', True) ~ '', entry.get('charset')) }}
 {% endfor -%}