repman: switch to mariadb 10.4 and fix database backup failure

software/repman/buildout.hash.cfg

@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
 
 [template-mariadb.cfg]
 _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
-md5sum = 6af0fb36ebdc5f1730ed3a9319eafdad
+md5sum = 6d26b96a25d927452f2d41b3844e7be0
 
 [template-my-cnf]
 _update_hash_filename_ = templates/my.cnf.in
@@ -42,7 +42,7 @@ md5sum = f3661b788099bb31d71ba6e7d36836d9
 
 [template-mariadb-initial-setup]
 _update_hash_filename_ = templates/mariadb_initial_setup.sql.in
-md5sum = 96c151c376600a90fe2a44825f2de550
+md5sum = 5a070090fc5b266bf866855c9c54dadf
 
 [template-publish-slave-information]
 _update_hash_filename_ = publish-database-slave-parameters.cfg.in
@@ -54,7 +54,7 @@ md5sum = d8a0a7d72b02f911dafda655941b805d
 
 [init-root-wrapper-in]
 _update_hash_filename_ = templates/init_root_wrapper.in
-md5sum = b3dffa3c8b9b2cc86b536d411cbde9d4
+md5sum = 6d5e15e586810520f7d4a873ec9d3cc3
 
 [repman-manager-sh.in]
 _update_hash_filename_ = templates/repman-manager.sh.in
@@ -62,7 +62,7 @@ md5sum = 852dfab6d798aa1382eec4de2fd624f9
 
 [dbjobs-in]
 _update_hash_filename_ = templates/dbjobs.in
-md5sum = bf3a3d91e3152d5dfd807f35890d89c6
+md5sum = da15968c75e7c856662099edaea5ef46
 
 [mysqld-need-start.sh.in]
 _update_hash_filename_ = templates/mysqld-need-start.sh.in
@@ -74,4 +74,4 @@ md5sum = 455aaf369bf5141758dc57f2c0e67b08
 
 [slave-db-manage.in]
 _update_hash_filename_ = templates/slave-db-manage.in
-md5sum = 6af818d4513561eea7def0f4f8d42164
+md5sum = b9220b1867974d4164abcb48ed01872f

software/repman/instance-mariadb.cfg.jinja2.in

@@ -131,8 +131,9 @@ template = {{ parameter_dict['template-init-root-wrapper'] }}
 [mysql-slave-db-cleanup]
 < = jinja2-template-script-base
 rendered = ${directory:bin}/manage_slave_db
+db-name = {{ slapparameter_dict['database-name'] }}
 extra-context =
-  raw database_name      {{ slapparameter_dict['database-name'] }}
+  key database_name      :db-name
 template = {{ parameter_dict['template-manage-db'] }}
 
 [mysqld]
@@ -193,13 +194,18 @@ target-directory = ${directory:bin}
 link-binary = {{ dumps(parameter_dict['link-binary']) }}
 
 [binary-wrap-base]
-recipe = slapos.cookbook:wrapper
 # Note: --defaults-file must be the first argument, otherwise wrapped binary
 # will reject it.
-command-line = 
-  "{{ parameter_dict['mariadb-location'] }}/bin/${:command}"
-  --defaults-file="${directory:etc}/mysql/my.cnf" --protocol=socket ${:extra-args}
-wrapper-path = ${directory:bin}/${:command}
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:bin}/${:command}
+mode = 755
+template =
+  inline:#!/bin/sh
+  exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
+  --defaults-file="${directory:etc}/mysql/my.cnf" \
+  --protocol=socket --user=$USER \
+  ${:extra-args} "$@"
+wrapper-path = ${:rendered}
 extra-args =
 
 [binary-wrap-mysql]
@@ -223,6 +229,11 @@ command = mysqladmin
 <= binary-wrap-base
 command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --skip-write-binlog
 command = mysql_tzinfo_to_sql
+template =
+  inline:#!/bin/sh
+  exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
+  --defaults-file="${directory:etc}/mysql/my.cnf" \
+  --skip-write-binlog "$@"
 
 [binary-wrap-pt-digest]
 <= binary-wrap-base

software/repman/software.cfg

@@ -20,7 +20,6 @@ extends =
   ../../component/jq/buildout.cfg
   ../../stack/supervisord/buildout.cfg
   ../../stack/monitor/buildout.cfg
-  ../neoppod/software-common.cfg
 
 parts =
   slapos-cookbook
@@ -29,10 +28,45 @@ parts =
   template-mariadb.cfg
   template-mysqld-wrapper
   gowork
-  
+
+[mariadb]
+location = ${mariadb-10.4:location}
+
 [python]
 part = python2.7
 
+[template-mysqld-wrapper]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/mysqld.in
+mode = 644
+template =
+  inline:{% raw %}#!/bin/sh -e
+  basedir='${mariadb:location}'
+  datadir='{{datadir}}'
+  marker=$datadir/.slapos_initializing
+  [ -d "$datadir/mysql" ] && [ ! -f "$marker" ] || {
+    find "$datadir/" -mindepth 1 ! -path $marker -delete || mkdir "$datadir"
+    touch "$marker"
+    "$basedir/scripts/mysql_install_db" \
+      --defaults-file='{{defaults_file}}' \
+      --skip-name-resolve \
+      --auth-root-authentication-method=socket \
+      --cross-bootstrap \
+      --auth-root-socket-user=$USER \
+      --user=$USER \
+      --basedir="$basedir" --plugin_dir="$basedir/lib/plugin" \
+      --datadir="$datadir"
+    rm "$marker"
+  }
+  {%- if environ is defined %}
+  {%-   for variable in environ.splitlines() %}
+  {{ variable }} \
+  {%-   endfor %}
+  {%- endif %}
+  exec "$basedir/bin/mysqld" --defaults-file='{{defaults_file}}' "$@"
+  {% endraw %}
+
+
 [instance.cfg]
 recipe = slapos.recipe.template:jinja2
 rendered = ${buildout:directory}/instance.cfg

software/repman/templates/dbjobs.in

 #!/bin/bash
 
-USER={{ parameter_dict['db-user'] }}
+DBUSER={{ parameter_dict['db-user'] }}
 PASSWORD={{ parameter_dict['db-password'] }}
 ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
 SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow
@@ -54,7 +54,7 @@ partialRestore()
     mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/
     $MYSQL_CLIENT  -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
  done
- cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk  -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER
+ cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk  -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER
  $MYSQL_CLIENT -e "flush privileges;start slave;"
 }
 
@@ -63,7 +63,7 @@ kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
 for job in "${JOBS[@]}"
 do
 
- TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER -N))
+ TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER -N))
 
  ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }'))
  ID=($(echo $TASK | awk -F@ '{ print $1 }'))
@@ -78,13 +78,13 @@ do
       reseedmysqldump)
        echo "Waiting backup." >  {{ parameter_dict['log-dir'] }}/dbjob.out
        pauseJob
-       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
+       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
        $MYSQL_CLIENT -e 'start slave;'
       ;;
       flashbackmysqldump)
        echo "Waiting backup." >  {{ parameter_dict['log-dir'] }}/dbjob.out
        pauseJob
-       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
+       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
        $MYSQL_CLIENT  -e 'start slave;'
       ;;
       reseedmariabackup)
@@ -108,7 +108,7 @@ do
       ;;
       mariabackup)
        cd {{ parameter_dict['tmp-dir'] }}
-       mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}'  --no-version-check  --user=$USER --password=$PASSWORD --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
+       mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}'  --no-version-check  --user=$USER --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
       ;;
       error)
        cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out

software/repman/templates/init_root_wrapper.in

@@ -2,7 +2,7 @@
 
 run_mysql () {
   {{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \
-  --protocol=socket -uroot -hlocalhost "$@"
+  --protocol=socket -u$USER -hlocalhost "$@"
 }
 
 if [ ! -f "{{ init_password_done }}" ]; then

software/repman/templates/mariadb_initial_setup.sql.in

@@ -3,7 +3,7 @@ SET @@SESSION.SQL_LOG_BIN=0;
 USE mysql;
 {% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
 {% if mroonga %}
-SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
+#SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
 {% endif %}
 DROP FUNCTION IF EXISTS sphinx_snippets;
 #CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
@@ -22,9 +22,9 @@ CREATE DATABASE IF NOT EXISTS `repman_slave_definition`;
                        'ascii_general_ci': 'ascii',
                        'ascii_bin': 'ascii'} -%}
 {% if charset %}
-{%   set charset_collate = "CHARACTER SET %s" % charset -%}
+{%   set charset_collate = "DEFAULT CHARACTER SET %s" % charset -%}
 {%   if charset_dict.get(charset) -%}
-{%     set charset_collate = "CHARACTER SET %s COLLATE %s" % (charset_dict[charset], charset) -%}
+{%     set charset_collate = "DEFAULT CHARACTER SET %s COLLATE %s" % (charset_dict[charset], charset) -%}
 {%   endif -%}
 {% endif -%}
 CREATE DATABASE IF NOT EXISTS `{{ name }}` {{ charset_collate }};

software/repman/templates/slave-db-manage.in

@@ -10,7 +10,7 @@ get_token () {
 
 run_mysql () {
   {{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \
-  --protocol=socket -uroot -hlocalhost "$@"
+  --protocol=socket -u$USER -hlocalhost "$@"
 }
 
 TOKEN=$(get_token | {{ jq_bin }} -r '.token')
@@ -23,10 +23,10 @@ if [ "$DATADIR" = "{{ partition_dir }}" ]; then
 use repman_slave_definition;
 
 CREATE TABLE IF NOT EXISTS \`slave\` (
-  \`name\` varchar(20) NOT NULL,
+  \`name\` varchar(80) NOT NULL,
   \`state\` tinyint(1) DEFAULT NULL,
   PRIMARY KEY (\`name\`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
 UPDATE \`slave\` set \`state\`=false;
 {% for name in database_name.split(' ') -%}