MDEV-18426: Most of the mtr tests in the galera_3nodes suite fail

Most of the mtr tests in the galera_3nodes suite fail
for a variety of reasons with a variety of errors.
Some tests simply need to add the missing "connection"
lines to the result files, but many of them fail due
to substantial errors that require reworking test files.

This patch adds the missing "connection" lines to
the result files and fixes several substantial flaws
in the galera_3nodes suite tests and in the mtr framework
service files, adapting the tests from galera_3nodes
for the current version of MariaDB.

https://jira.mariadb.org/browse/MDEV-18426

mysql-test/suite/galera/include/galera_resume.inc → mysql-test/include/galera_resume.inc

@@ -3,7 +3,7 @@
         my $pid_filename = $ENV{'_SUSPEND_NODE_PIDFILE'};
         my $mysqld_pid = `cat $pid_filename`;
         chomp($mysqld_pid);
-        system("kill -18 $mysqld_pid");
+        system("kill -SIGCONT $mysqld_pid");
         exit(0);
 EOF
 

mysql-test/suite/galera_3nodes/include/galera_suspend.inc → mysql-test/include/galera_suspend.inc

@@ -9,6 +9,6 @@
         my $pid_filename = $ENV{'_SUSPEND_NODE_PIDFILE'};
         my $mysqld_pid = `cat $pid_filename`;
         chomp($mysqld_pid);
-        system("kill -19 $mysqld_pid");
+        system("kill -SIGSTOP $mysqld_pid");
         exit(0);
 EOF

mysql-test/include/mtr_check.sql

@@ -35,6 +35,7 @@ BEGIN
      AND variable_name != 'INNODB_BUFFER_POOL_LOAD_AT_STARTUP'
      AND variable_name not like 'GTID%POS'
      AND variable_name != 'GTID_BINLOG_STATE'
+     AND variable_name != 'AUTO_INCREMENT_INCREMENT'
    ORDER BY variable_name;
 
   -- Dump all databases, there should be none

mysql-test/suite/galera_3nodes/r/galera_evs_suspect_timeout.result

+connection node_1;
 SET GLOBAL wsrep_provider_options = 'evs.inactive_timeout=PT100M; evs.suspect_timeout=PT1S';
+connection node_2;
 SET GLOBAL wsrep_provider_options = 'evs.inactive_timeout=PT100M; evs.suspect_timeout=PT1S';
+connection node_3;
 Suspending node ...
+connection node_1;
 SET SESSION wsrep_sync_wait = 0;
 SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
 VARIABLE_VALUE = 2
 1
 CREATE TABLE t1 (f1 INTEGER);
 INSERT INTO t1 VALUES (1);
+connection node_2;
 SET SESSION wsrep_sync_wait = 0;
 SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
 VARIABLE_VALUE = 2
@@ -17,4 +22,5 @@ COUNT(*) = 1
 1
 DROP TABLE t1;
 Resuming node ...
+connection node_3;
 CALL mtr.add_suppression("WSREP: gcs_caused() returned -1 \\(Operation not permitted\\)");

mysql-test/suite/galera_3nodes/r/galera_garbd.result

+connection node_1;
+connection node_2;
+connection node_3;
 Killing node #3 to free ports for garbd ...
+connection node_3;
+connection node_1;
 Starting garbd ...
 CREATE TABLE t1 (f1 INTEGER);
 INSERT INTO t1 VALUES (1);
+connection node_2;
 SELECT COUNT(*) = 1 FROM t1;
 COUNT(*) = 1
 1
 Killing garbd ...
+connection node_1;
 INSERT INTO t1 VALUES (2);
+connection node_2;
 SELECT COUNT(*) = 2 FROM t1;
 COUNT(*) = 2
 1
 DROP TABLE t1;
 Restarting node #3 to satisfy MTR's end-of-test checks
+connection node_3;
+connection node_1;
 CALL mtr.add_suppression("WSREP: Protocol violation\. JOIN message sender 1\.0 \(.*\) is not in state transfer \(SYNCED\)");
+connection node_2;
 CALL mtr.add_suppression("WSREP: Protocol violation\. JOIN message sender 1\.0 \(.*\) is not in state transfer \(SYNCED\)");
+connection node_3;
 CALL mtr.add_suppression("WSREP: Protocol violation\. JOIN message sender 1\.0 \(.*\) is not in state transfer \(SYNCED\)");

mysql-test/suite/galera_3nodes/r/galera_ist_gcache_rollover.result

 connection node_1;
 connection node_2;
 connection node_3;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY);
 INSERT INTO t1 VALUES (01), (02), (03), (04), (05);
 connection node_2;

mysql-test/suite/galera_3nodes/r/galera_safe_to_bootstrap.result

+connection node_1;
+connection node_2;
+connection node_3;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
@@ -7,7 +11,6 @@ connection node_1;
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
-connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3;
 connection node_3;
 connection node_1;
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
@@ -29,8 +32,18 @@ connection node_1;
 connection node_2;
 connection node_3;
 connection node_2;
+CALL mtr.add_suppression("WSREP: no nodes coming from prim view, prim not possible");
+CALL mtr.add_suppression("WSREP: It may not be safe to bootstrap the cluster from this node");
+CALL mtr.add_suppression("WSREP: wsrep::connect(.*) failed: 7");
+CALL mtr.add_suppression("Aborting");
+CALL mtr.add_suppression("WSREP: moving position backwards: [0-9]+ -> 0");
 CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
 connection node_3;
+CALL mtr.add_suppression("WSREP: no nodes coming from prim view, prim not possible");
+CALL mtr.add_suppression("WSREP: It may not be safe to bootstrap the cluster from this node");
+CALL mtr.add_suppression("WSREP: wsrep::connect(.*) failed: 7");
+CALL mtr.add_suppression("Aborting");
+CALL mtr.add_suppression("WSREP: moving position backwards: [0-9]+ -> 0");
 CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
 SHOW CREATE TABLE t1;
 Table	Create Table

mysql-test/suite/galera_3nodes/r/galera_var_dirty_reads2.result

+connection node_1;
+connection node_2;
+connection node_3;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER);
 INSERT INTO t1 VALUES (1);
 connection node_2;

mysql-test/suite/galera_3nodes/t/galera_evs_suspect_timeout.test

@@ -56,8 +56,8 @@ DROP TABLE t1;
 
 # Reconnect node #3 so that MTR's end-of-test checks can run
 
---connection node_3
 --source include/galera_resume.inc
+--connection node_3
 --source include/wait_until_connected_again.inc
 
 CALL mtr.add_suppression("WSREP: gcs_caused() returned -1 \\(Operation not permitted\\)");

mysql-test/suite/galera_3nodes/t/galera_garbd.test

@@ -7,10 +7,20 @@
 --source include/have_innodb.inc
 --source include/big_test.inc
 
---echo Killing node #3 to free ports for garbd ...
 --let $galera_connection_name = node_3
 --let $galera_server_number = 3
 --source include/galera_connect.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--let $node_3=node_3
+--source ../galera/include/auto_increment_offset_save.inc
+
+--echo Killing node #3 to free ports for garbd ...
+--connection node_3
+--let $gp3 = `SELECT SUBSTR(@@wsrep_provider_options, LOCATE('base_port =', @@wsrep_provider_options) + LENGTH('base_port = '))`
+--let $galera_port_3 = `SELECT SUBSTR('$gp3', 1, LOCATE(';', '$gp3') - 1)`
 --source include/shutdown_mysqld.inc
 
 --connection node_1
@@ -18,7 +28,9 @@
 --source include/wait_condition.inc
 
 --echo Starting garbd ...
---exec `dirname $WSREP_PROVIDER`/garb/garbd --address "gcomm://127.0.0.1:$NODE_GALERAPORT_1" --group my_wsrep_cluster --options 'base_port=$NODE_GALERAPORT_3' > $MYSQL_TMP_DIR/garbd.log 2>&1 &
+--let $gp1 = `SELECT SUBSTR(@@wsrep_provider_options, LOCATE('base_port =', @@wsrep_provider_options) + LENGTH('base_port = '))`
+--let $galera_port_1 = `SELECT SUBSTR('$gp1', 1, LOCATE(';', '$gp1') - 1)`
+--exec `dirname $WSREP_PROVIDER`/../../bin/garb/garbd --address "gcomm://127.0.0.1:$galera_port_1" --group my_wsrep_cluster --options 'base_port=$galera_port_3' > $MYSQL_TMP_DIR/garbd.log 2>&1 &
 
 --sleep 5
 
@@ -32,7 +44,7 @@ INSERT INTO t1 VALUES (1);
 SELECT COUNT(*) = 1 FROM t1;
 
 --echo Killing garbd ...
---exec pkill --oldest --full garbd.*$NODE_GALERAPORT_3
+--exec pkill --oldest --full garbd.*$galera_port_3
 
 --sleep 5
 
@@ -51,6 +63,8 @@ DROP TABLE t1;
 --connection node_3
 --source include/start_mysqld.inc
 
+# Restore original auto_increment_offset values.
+--source ../galera/include/auto_increment_offset_restore.inc
 
 # Workaround for galera#101
 

mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.cnf

 !include ../galera_3nodes.cnf
 
 [mysqld.1]
-wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true;gcache.size=1M'
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.1.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.ignore_sb=true;gcache.size=1M'
 
 [mysqld.2]
-wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true;gcache.size=1M'
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.2.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.ignore_sb=true;gcache.size=1M'
 
 [mysqld.3]
-wsrep_provider_options='base_port=@mysqld.3.#galera_port;pc.ignore_sb=true;gcache.size=1M'
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.3.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.ignore_sb=true;gcache.size=1M'
 

mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.test

@@ -23,6 +23,7 @@
 --let $node_3=node_3
 --source ../galera/include/auto_increment_offset_save.inc
 
+--connection node_1
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY);
 INSERT INTO t1 VALUES (01), (02), (03), (04), (05);
 

mysql-test/suite/galera_3nodes/t/galera_safe_to_bootstrap.test

 #
 # Test the safe_to_bootstrap in grastate.dat 
 #
-
 --source include/galera_cluster.inc
+
+--let $galera_connection_name = node_3
+--let $galera_server_number = 3
+--source include/galera_connect.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--let $node_3=node_3
+--source ../galera/include/auto_increment_offset_save.inc
+
+--connection node_1
 CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 
 #
@@ -47,8 +58,6 @@ CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 #
 # Shut down one more node
 #
-
---connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3
 --connection node_3
 --source include/shutdown_mysqld.inc
 
@@ -129,9 +138,9 @@ SET SESSION wsrep_on = OFF;
 #
 
 --error 1
---exec $MYSQLD --defaults-group-suffix=.2 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --wsrep-new-cluster | grep 'This node is not safe to bootstrap the cluster'
+--exec $MYSQLD --defaults-group-suffix=.2 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --wsrep-new-cluster --wsrep-cluster-address='gcomm://' | grep 'This node is not safe to bootstrap the cluster'
 --error 1
---exec $MYSQLD --defaults-group-suffix=.3 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --wsrep-new-cluster | grep 'This node is not safe to bootstrap the cluster'
+--exec $MYSQLD --defaults-group-suffix=.3 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --wsrep-new-cluster --wsrep-cluster-address='gcomm://' | grep 'This node is not safe to bootstrap the cluster'
 
 #
 # Attempt to bootstrap starting from node #1, should succeed
@@ -154,10 +163,23 @@ SET SESSION wsrep_on = OFF;
 --source include/wait_condition.inc
 
 --connection node_2
+CALL mtr.add_suppression("WSREP: no nodes coming from prim view, prim not possible");
+CALL mtr.add_suppression("WSREP: It may not be safe to bootstrap the cluster from this node");
+CALL mtr.add_suppression("WSREP: wsrep::connect(.*) failed: 7");
+CALL mtr.add_suppression("Aborting");
+CALL mtr.add_suppression("WSREP: moving position backwards: [0-9]+ -> 0");
 CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
 
 --connection node_3
+CALL mtr.add_suppression("WSREP: no nodes coming from prim view, prim not possible");
+CALL mtr.add_suppression("WSREP: It may not be safe to bootstrap the cluster from this node");
+CALL mtr.add_suppression("WSREP: wsrep::connect(.*) failed: 7");
+CALL mtr.add_suppression("Aborting");
+CALL mtr.add_suppression("WSREP: moving position backwards: [0-9]+ -> 0");
 CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
 SHOW CREATE TABLE t1;
 
 DROP TABLE t1;
+
+# Restore original auto_increment_offset values.
+--source ../galera/include/auto_increment_offset_restore.inc

mysql-test/suite/galera_3nodes/t/galera_var_dirty_reads2.test

@@ -5,6 +5,17 @@
 --source include/galera_cluster.inc
 --source include/have_innodb.inc
 
+--let $galera_connection_name = node_3
+--let $galera_server_number = 3
+--source include/galera_connect.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--let $node_3=node_3
+--source ../galera/include/auto_increment_offset_save.inc
+
+--connection node_1
 CREATE TABLE t1 (f1 INTEGER);
 INSERT INTO t1 VALUES (1);
 
@@ -110,3 +121,6 @@ SET GLOBAL wsrep_provider_options='gmcast.isolate=0';
 --source include/wait_condition.inc
 
 DROP TABLE t1;
+
+# Restore original auto_increment_offset values.
+--source ../galera/include/auto_increment_offset_restore.inc