MDEV-18426: Most of the mtr tests in the galera_3nodes suite fail

Most of the mtr tests in the galera_3nodes suite fail
for a variety of reasons with a variety of errors.
Some tests simply need to add the missing "connection"
lines to the result files, but many of them fail due
to substantial errors that require reworking test files.

This patch adds the missing "connection" lines to
the result files and fixes several substantial flaws
in the galera_3nodes suite tests and in the mtr framework
service files, adapting the tests from galera_3nodes
for the current version of MariaDB.

https://jira.mariadb.org/browse/MDEV-18426

mysql-test/suite/galera/include/galera_resume.inc → mysql-test/include/galera_resume.inc

@@ -3,7 +3,7 @@
         my $pid_filename = $ENV{'_SUSPEND_NODE_PIDFILE'};
         my $mysqld_pid = `cat $pid_filename`;
         chomp($mysqld_pid);
-        system("kill -18 $mysqld_pid");
+        system("kill -SIGCONT $mysqld_pid");
         exit(0);
 EOF
 

mysql-test/suite/galera_3nodes/include/galera_suspend.inc → mysql-test/include/galera_suspend.inc

@@ -9,6 +9,6 @@
         my $pid_filename = $ENV{'_SUSPEND_NODE_PIDFILE'};
         my $mysqld_pid = `cat $pid_filename`;
         chomp($mysqld_pid);
-        system("kill -19 $mysqld_pid");
+        system("kill -SIGSTOP $mysqld_pid");
         exit(0);
 EOF

mysql-test/include/mtr_check.sql

@@ -35,6 +35,7 @@ BEGIN
      AND variable_name != 'INNODB_BUFFER_POOL_LOAD_AT_STARTUP'
      AND variable_name not like 'GTID%POS'
      AND variable_name != 'GTID_BINLOG_STATE'
+     AND variable_name != 'AUTO_INCREMENT_INCREMENT'
    ORDER BY variable_name;
 
   -- Dump all databases, there should be none

mysql-test/suite/galera_3nodes/r/galera_evs_suspect_timeout.result

+connection node_1;
 SET GLOBAL wsrep_provider_options = 'evs.inactive_timeout=PT100M; evs.suspect_timeout=PT1S';
+connection node_2;
 SET GLOBAL wsrep_provider_options = 'evs.inactive_timeout=PT100M; evs.suspect_timeout=PT1S';
+connection node_3;
 Suspending node ...
+connection node_1;
 SET SESSION wsrep_sync_wait = 0;
 SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
 VARIABLE_VALUE = 2
 1
 CREATE TABLE t1 (f1 INTEGER);
 INSERT INTO t1 VALUES (1);
+connection node_2;
 SET SESSION wsrep_sync_wait = 0;
 SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
 VARIABLE_VALUE = 2
@@ -17,4 +22,5 @@ COUNT(*) = 1
 1
 DROP TABLE t1;
 Resuming node ...
+connection node_3;
 CALL mtr.add_suppression("WSREP: gcs_caused() returned -1 \\(Operation not permitted\\)");

mysql-test/suite/galera_3nodes/r/galera_garbd.result

 Killing node #3 to free ports for garbd ...
+connection node_1;
 Starting garbd ...
 CREATE TABLE t1 (f1 INTEGER);
 INSERT INTO t1 VALUES (1);
+connection node_2;
 SELECT COUNT(*) = 1 FROM t1;
 COUNT(*) = 1
 1
 Killing garbd ...
+connection node_1;
 INSERT INTO t1 VALUES (2);
+connection node_2;
 SELECT COUNT(*) = 2 FROM t1;
 COUNT(*) = 2
 1
 DROP TABLE t1;
 Restarting node #3 to satisfy MTR's end-of-test checks
+connection node_3;
+connection node_1;
 CALL mtr.add_suppression("WSREP: Protocol violation\. JOIN message sender 1\.0 \(.*\) is not in state transfer \(SYNCED\)");
+connection node_2;
 CALL mtr.add_suppression("WSREP: Protocol violation\. JOIN message sender 1\.0 \(.*\) is not in state transfer \(SYNCED\)");
+connection node_3;
 CALL mtr.add_suppression("WSREP: Protocol violation\. JOIN message sender 1\.0 \(.*\) is not in state transfer \(SYNCED\)");

mysql-test/suite/galera_3nodes/r/galera_ist_gcache_rollover.result

 connection node_1;
 connection node_2;
 connection node_3;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY);
 INSERT INTO t1 VALUES (01), (02), (03), (04), (05);
 connection node_2;

mysql-test/suite/galera_3nodes/r/galera_pc_bootstrap.result

 CREATE TABLE t1 (f1 INTEGER);
+connection node_1;
 SET GLOBAL wsrep_provider_options = 'gmcast.isolate=1';
+connection node_2;
 SET GLOBAL wsrep_provider_options = 'gmcast.isolate=1';
+connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3;
+connection node_3;
 SET GLOBAL wsrep_provider_options = 'gmcast.isolate=1';
 SET SESSION wsrep_sync_wait = 0;
+connection node_2;
 SET GLOBAL wsrep_provider_options = 'pc.bootstrap=1';
 SHOW STATUS LIKE 'wsrep_cluster_size';
 Variable_name	Value
@@ -11,15 +16,21 @@ SHOW STATUS LIKE 'wsrep_cluster_status';
 Variable_name	Value
 wsrep_cluster_status	Primary
 INSERT INTO t1 VALUES (1);
+connection node_2;
 SET GLOBAL wsrep_provider_options = 'gmcast.isolate=0';
+connection node_1;
 SET GLOBAL wsrep_provider_options = 'gmcast.isolate=0';
+connection node_3;
 SET GLOBAL wsrep_provider_options = 'gmcast.isolate=0';
+connection node_1;
 SELECT COUNT(*) FROM t1;
 COUNT(*)
 1
+connection node_2;
 SELECT COUNT(*) FROM t1;
 COUNT(*)
 1
+connection node_3;
 SELECT COUNT(*) FROM t1;
 COUNT(*)
 1

mysql-test/suite/galera_3nodes/r/galera_pc_weight.result

+connection node_1;
 SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_weight';
 VARIABLE_VALUE = 3
 SET GLOBAL wsrep_provider_options = 'pc.weight=3';
 SELECT VARIABLE_VALUE = 5 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_weight';
 VARIABLE_VALUE = 5
 SET GLOBAL wsrep_provider_options = 'gmcast.isolate=1';
+connection node_2;
 SET SESSION wsrep_sync_wait=0;
 SET SESSION wsrep_on=OFF;
 SET SESSION wsrep_on=ON;
@@ -27,6 +29,7 @@ wsrep_local_state	0
 SHOW STATUS LIKE 'wsrep_local_state_comment';
 Variable_name	Value
 wsrep_local_state_comment	Initialized
+connection node_3;
 SET SESSION wsrep_sync_wait=0;
 SET SESSION wsrep_on=OFF;
 SET SESSION wsrep_on=ON;
@@ -50,6 +53,7 @@ wsrep_local_state	0
 SHOW STATUS LIKE 'wsrep_local_state_comment';
 Variable_name	Value
 wsrep_local_state_comment	Initialized
+connection node_1;
 SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_weight';
 VARIABLE_VALUE = 3
 SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
@@ -70,7 +74,11 @@ VARIABLE_VALUE = 'Synced'
 SET GLOBAL wsrep_provider_options = 'pc.weight=1';
 SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_weight';
 VARIABLE_VALUE = 1
+connection node_1;
 SET GLOBAL wsrep_provider_options = 'gmcast.isolate=0';
+connection node_2;
+connection node_3;
+connection node_1;
 SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
 VARIABLE_VALUE = 3
 1
@@ -91,6 +99,7 @@ VARIABLE_VALUE = 4
 SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment';
 VARIABLE_VALUE = 'Synced'
 1
+connection node_2;
 SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
 VARIABLE_VALUE = 3
 1
@@ -111,6 +120,7 @@ VARIABLE_VALUE = 4
 SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment';
 VARIABLE_VALUE = 'Synced'
 1
+connection node_3;
 SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
 VARIABLE_VALUE = 3
 1
@@ -131,12 +141,15 @@ VARIABLE_VALUE = 4
 SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment';
 VARIABLE_VALUE = 'Synced'
 1
+connection node_1;
 SET GLOBAL wsrep_provider_options = 'pc.weight=1';
 CALL mtr.add_suppression('WSREP: gcs_caused\\(\\) returned -1');
+connection node_2;
 CALL mtr.add_suppression('overriding reported weight for');
 CALL mtr.add_suppression('SYNC message from member');
 CALL mtr.add_suppression('user message in state LEAVING');
 CALL mtr.add_suppression('sending install message failed: (Transport endpoint is not connected|Socket is not connected)');
+connection node_3;
 CALL mtr.add_suppression('WSREP: user message in state LEAVING');
 CALL mtr.add_suppression('sending install message failed: (Transport endpoint is not connected|Socket is not connected)');
 CALL mtr.add_suppression('overriding reported weight for');

mysql-test/suite/galera_3nodes/r/galera_safe_to_bootstrap.result

+connection node_1;
+connection node_2;
+connection node_3;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
+connection node_2;
+connection node_1;
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
+connection node_3;
+connection node_1;
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 1']
+connection node_2;
+connection node_1;
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
+connection node_2;
+connection node_1;
 SET SESSION wsrep_on = OFF;
 Killing server ...
 safe_to_bootstrap: 1
 safe_to_bootstrap: 0
 safe_to_bootstrap: 0
+connection node_1;
+connection node_2;
+connection node_3;
+connection node_2;
+CALL mtr.add_suppression("WSREP: no nodes coming from prim view, prim not possible");
+CALL mtr.add_suppression("WSREP: It may not be safe to bootstrap the cluster from this node");
+CALL mtr.add_suppression("WSREP: wsrep::connect(.*) failed: 7");
+CALL mtr.add_suppression("Aborting");
+CALL mtr.add_suppression("WSREP: moving position backwards: [0-9]+ -> 0");
 CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
+connection node_3;
+CALL mtr.add_suppression("WSREP: no nodes coming from prim view, prim not possible");
+CALL mtr.add_suppression("WSREP: It may not be safe to bootstrap the cluster from this node");
+CALL mtr.add_suppression("WSREP: wsrep::connect(.*) failed: 7");
+CALL mtr.add_suppression("Aborting");
+CALL mtr.add_suppression("WSREP: moving position backwards: [0-9]+ -> 0");
 CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
 SHOW CREATE TABLE t1;
 Table	Create Table

mysql-test/suite/galera_3nodes/r/galera_var_dirty_reads2.result

 CREATE TABLE t1 (f1 INTEGER);
 INSERT INTO t1 VALUES (1);
+connection node_2;
 SET GLOBAL wsrep_provider_options='gmcast.isolate=1';
+connection node_1;
+connection node_2;
 SET SESSION wsrep_sync_wait = 0;
 SET SESSION wsrep_dirty_reads = 1;
 SELECT f1 FROM t1;
@@ -45,4 +48,6 @@ SELECT COUNT(*) > 0 FROM INFORMATION_SCHEMA.PROCESSLIST;
 COUNT(*) > 0
 1
 SET GLOBAL wsrep_provider_options='gmcast.isolate=0';
+connection node_1;
+connection node_2;
 DROP TABLE t1;

mysql-test/suite/galera_3nodes/t/galera_evs_suspect_timeout.test

@@ -56,8 +56,8 @@ DROP TABLE t1;
 
 # Reconnect node #3 so that MTR's end-of-test checks can run
 
---connection node_3
 --source include/galera_resume.inc
+--connection node_3
 --source include/wait_until_connected_again.inc
 
 CALL mtr.add_suppression("WSREP: gcs_caused() returned -1 \\(Operation not permitted\\)");

mysql-test/suite/galera_3nodes/t/galera_garbd.test

@@ -11,6 +11,8 @@
 --let $galera_connection_name = node_3
 --let $galera_server_number = 3
 --source include/galera_connect.inc
+--let $gp3 = `SELECT SUBSTR(@@wsrep_provider_options, LOCATE('base_port =', @@wsrep_provider_options) + LENGTH('base_port = '))`
+--let $galera_port_3 = `SELECT SUBSTR('$gp3', 1, LOCATE(';', '$gp3') - 1)`
 --source include/shutdown_mysqld.inc
 
 --connection node_1
@@ -18,7 +20,9 @@
 --source include/wait_condition.inc
 
 --echo Starting garbd ...
---exec `dirname $WSREP_PROVIDER`/garb/garbd --address "gcomm://127.0.0.1:$NODE_GALERAPORT_1" --group my_wsrep_cluster --options 'base_port=$NODE_GALERAPORT_3' > $MYSQL_TMP_DIR/garbd.log 2>&1 &
+--let $gp1 = `SELECT SUBSTR(@@wsrep_provider_options, LOCATE('base_port =', @@wsrep_provider_options) + LENGTH('base_port = '))`
+--let $galera_port_1 = `SELECT SUBSTR('$gp1', 1, LOCATE(';', '$gp1') - 1)`
+--exec `dirname $WSREP_PROVIDER`/../../bin/garb/garbd --address "gcomm://127.0.0.1:$galera_port_1" --group my_wsrep_cluster --options 'base_port=$galera_port_3' > $MYSQL_TMP_DIR/garbd.log 2>&1 &
 
 --sleep 5
 
@@ -32,7 +36,7 @@ INSERT INTO t1 VALUES (1);
 SELECT COUNT(*) = 1 FROM t1;
 
 --echo Killing garbd ...
---exec pkill --oldest --full garbd.*$NODE_GALERAPORT_3
+--exec pkill --oldest --full garbd.*$galera_port_3
 
 --sleep 5
 

mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.cnf

 !include ../galera_3nodes.cnf
 
 [mysqld.1]
-wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true;gcache.size=1M'
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.1.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.ignore_sb=true;gcache.size=1M'
 
 [mysqld.2]
-wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true;gcache.size=1M'
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.2.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.ignore_sb=true;gcache.size=1M'
 
 [mysqld.3]
-wsrep_provider_options='base_port=@mysqld.3.#galera_port;pc.ignore_sb=true;gcache.size=1M'
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.3.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.ignore_sb=true;gcache.size=1M'
 

mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.test

@@ -23,6 +23,7 @@
 --let $node_3=node_3
 --source ../galera/include/auto_increment_offset_save.inc
 
+--connection node_1
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY);
 INSERT INTO t1 VALUES (01), (02), (03), (04), (05);
 

mysql-test/suite/galera_3nodes/t/galera_safe_to_bootstrap.test

 #
 # Test the safe_to_bootstrap in grastate.dat 
 #
-
 --source include/galera_cluster.inc
+
+--let $galera_connection_name = node_3
+--let $galera_server_number = 3
+--source include/galera_connect.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--let $node_3=node_3
+--source ../galera/include/auto_increment_offset_save.inc
+
+--connection node_1
 CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 
 #
@@ -47,8 +58,6 @@ CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 #
 # Shut down one more node
 #
-
---connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3
 --connection node_3
 --source include/shutdown_mysqld.inc
 
@@ -129,9 +138,9 @@ SET SESSION wsrep_on = OFF;
 #
 
 --error 1
---exec $MYSQLD --defaults-group-suffix=.2 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --wsrep-new-cluster | grep 'This node is not safe to bootstrap the cluster'
+--exec $MYSQLD --defaults-group-suffix=.2 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --wsrep-new-cluster --wsrep-cluster-address='gcomm://' | grep 'This node is not safe to bootstrap the cluster'
 --error 1
---exec $MYSQLD --defaults-group-suffix=.3 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --wsrep-new-cluster | grep 'This node is not safe to bootstrap the cluster'
+--exec $MYSQLD --defaults-group-suffix=.3 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --wsrep-new-cluster --wsrep-cluster-address='gcomm://' | grep 'This node is not safe to bootstrap the cluster'
 
 #
 # Attempt to bootstrap starting from node #1, should succeed
@@ -154,10 +163,23 @@ SET SESSION wsrep_on = OFF;
 --source include/wait_condition.inc
 
 --connection node_2
+CALL mtr.add_suppression("WSREP: no nodes coming from prim view, prim not possible");
+CALL mtr.add_suppression("WSREP: It may not be safe to bootstrap the cluster from this node");
+CALL mtr.add_suppression("WSREP: wsrep::connect(.*) failed: 7");
+CALL mtr.add_suppression("Aborting");
+CALL mtr.add_suppression("WSREP: moving position backwards: [0-9]+ -> 0");
 CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
 
 --connection node_3
+CALL mtr.add_suppression("WSREP: no nodes coming from prim view, prim not possible");
+CALL mtr.add_suppression("WSREP: It may not be safe to bootstrap the cluster from this node");
+CALL mtr.add_suppression("WSREP: wsrep::connect(.*) failed: 7");
+CALL mtr.add_suppression("Aborting");
+CALL mtr.add_suppression("WSREP: moving position backwards: [0-9]+ -> 0");
 CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
 SHOW CREATE TABLE t1;
 
 DROP TABLE t1;
+
+# Restore original auto_increment_offset values.
+--source ../galera/include/auto_increment_offset_restore.inc