Galera MTR Tests: Test for GCF-942 - safe_to_bootstrap flag in grastate.dat

mysql-test/include/assert_grep.inc

+# ==== Purpose ====
+#
+# Grep a file for a pattern, produce a single string out of the
+# matching lines, and assert that the string matches a given regular
+# expression.
+#
+# ==== Usage ====
+#
+# --let $assert_text= TEXT
+# --let $assert_file= FILE
+# --let $assert_select= REGEX
+# [--let $assert_match= REGEX | --let $assert_count= NUMBER]
+# [--let $assert_only_after= REGEX]
+# --source include/assert_grep.inc
+#
+# Parameters:
+#
+#   $assert_text
+#     Text that describes what is being checked. This text is written to
+#     the query log so it should not contain non-deterministic elements.
+#
+#   $assert_file
+#     File to search.
+#
+#   $assert_select
+#     All lines matching this text will be checked.
+#
+#   $assert_match
+#     The script will find all lines that match $assert_select,
+#     concatenate them to a long string, and assert that it matches
+#     $assert_match.
+#
+#   $assert_count
+#     Instead of asserting that the selected lines match
+#     $assert_match, assert that there were exactly $assert_count
+#     matching lines.
+#
+#   $assert_only_after
+#     Reset all the lines matched and the counter when finding this pattern.
+#     It is useful for searching things in the mysqld.err log file just
+#     after the last server restart for example (discarding the log content
+#     of previous server executions).
+
+
+if (!$assert_text)
+{
+  --die !!!ERROR IN TEST: you must set $assert_text
+}
+if (!$assert_file)
+{
+  --die !!!ERROR IN TEST: you must set $assert_file
+}
+if (!$assert_select)
+{
+  --die !!!ERROR IN TEST: you must set $assert_select
+}
+if ($assert_match == '')
+{
+  if ($assert_count == '')
+  {
+    --die !!!ERROR IN TEST: you must set either $assert_match or $assert_count
+  }
+}
+if ($assert_match != '')
+{
+  if ($assert_count != '')
+  {
+    --echo assert_text='$assert_text' assert_count='$assert_count'
+    --die !!!ERROR IN TEST: you must set only one of $assert_match or $assert_count
+  }
+}
+
+
+--let $include_filename= assert_grep.inc [$assert_text]
+--source include/begin_include_file.inc
+
+
+--let _AG_ASSERT_TEXT= $assert_text
+--let _AG_ASSERT_FILE= $assert_file
+--let _AG_ASSERT_SELECT= $assert_select
+--let _AG_ASSERT_MATCH= $assert_match
+--let _AG_ASSERT_COUNT= $assert_count
+--let _AG_OUT= `SELECT CONCAT('$MYSQLTEST_VARDIR/tmp/_ag_', UUID())`
+--let _AG_ASSERT_ONLY_AFTER= $assert_only_after
+
+
+--perl
+  use strict;
+  use warnings;
+  my $file= $ENV{'_AG_ASSERT_FILE'};
+  my $assert_select= $ENV{'_AG_ASSERT_SELECT'};
+  my $assert_match= $ENV{'_AG_ASSERT_MATCH'};
+  my $assert_count= $ENV{'_AG_ASSERT_COUNT'};
+  my $assert_only_after= $ENV{'_AG_ASSERT_ONLY_AFTER'};
+  my $out= $ENV{'_AG_OUT'};
+
+  my $result= '';
+  my $count= 0;
+  open(FILE, "$file") or die("Error $? opening $file: $!\n");
+  while (<FILE>) {
+    my $line = $_;
+    if ($assert_only_after && $line =~ /$assert_only_after/) {
+      $result = "";
+      $count = 0;
+    }
+    if ($line =~ /$assert_select/) {
+      if ($assert_count ne '') {
+        $count++;
+      }
+      else {
+        $result .= $line;
+      }
+    }
+  }
+  close(FILE) or die("Error $? closing $file: $!");
+  open OUT, "> $out" or die("Error $? opening $out: $!");
+  if ($assert_count ne '' && ($count != $assert_count)) {
+    print OUT ($count) or die("Error $? writing $out: $!");
+  }
+  elsif ($assert_count eq '' && $result !~ /$assert_match/) {
+    print OUT ($result) or die("Error $? writing $out: $!");
+  }
+  else {
+    print OUT ("assert_grep.inc ok");
+  }
+  close OUT or die("Error $? closing $out: $!");
+EOF
+
+
+--let $_ag_outcome= `SELECT LOAD_FILE('$_AG_OUT')`
+if ($_ag_outcome != 'assert_grep.inc ok')
+{
+  --source include/show_rpl_debug_info.inc
+  --echo include/assert_grep.inc failed!
+  --echo assert_text: '$assert_text'
+  --echo assert_file: '$assert_file'
+  --echo assert_select: '$assert_select'
+  --echo assert_match: '$assert_match'
+  --echo assert_count: '$assert_count'
+  --echo assert_only_after: '$assert_only_after'
+  if ($assert_match != '')
+  {
+    --echo matching lines: '$_ag_outcome'
+  }
+  if ($assert_count != '')
+  {
+    --echo number of matching lines: $_ag_outcome
+  }
+  --die assert_grep.inc failed.
+}
+
+
+--let $include_filename= include/assert_grep.inc [$assert_text]
+--source include/end_include_file.inc

mysql-test/suite/galera_3nodes/disabled.def

@@ -5,3 +5,4 @@ galera_slave_options_do :MDEV-8798
 galera_slave_options_ignore : MDEV-8798
 galera_pc_bootstrap : TODO: Investigate: Timeout in wait_condition.inc
 galera_pc_weight : Test times out
+galera_safe_to_bootstrap : I Really dont know :(

mysql-test/suite/galera_3nodes/r/galera_safe_to_bootstrap.result

+CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
+include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
+include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
+include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
+include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
+include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
+include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
+include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
+include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
+include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 1']
+include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
+include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
+include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
+SET SESSION wsrep_on = OFF;
+Killing server ...
+safe_to_bootstrap: 1
+safe_to_bootstrap: 0
+safe_to_bootstrap: 0
+CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
+CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `f1` int(11) DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+DROP TABLE t1;

mysql-test/suite/galera_3nodes/t/galera_safe_to_bootstrap.test

+#
+# Test the safe_to_bootstrap in grastate.dat 
+#
+
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
+
+#
+# At start, all grastate.dat files have safe_to_boostrap: 0
+#
+
+--let $assert_text= grastate.dat does not have 'safe_to_bootstrap: 0'
+--let $assert_select= safe_to_bootstrap: 0
+--let $assert_count= 1
+
+--let $assert_file= $MYSQLTEST_VARDIR/mysqld.1/data/grastate.dat
+--source include/assert_grep.inc
+
+--let $assert_file= $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat
+--source include/assert_grep.inc
+
+--let $assert_file= $MYSQLTEST_VARDIR/mysqld.3/data/grastate.dat
+--source include/assert_grep.inc
+
+#
+# Shut down one node
+#
+
+--connection node_2
+--source include/shutdown_mysqld.inc
+
+--connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
+
+# Still, all grastate.dat files should have safe_to_boostrap: 0
+
+--let $assert_file= $MYSQLTEST_VARDIR/mysqld.1/data/grastate.dat
+--source include/assert_grep.inc
+
+--let $assert_file= $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat
+--source include/assert_grep.inc
+
+--let $assert_file= $MYSQLTEST_VARDIR/mysqld.3/data/grastate.dat
+--source include/assert_grep.inc
+
+#
+# Shut down one more node
+#
+
+--connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3
+--connection node_3
+--source include/shutdown_mysqld.inc
+
+--connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
+
+# Now, nodes 2,3 should have safe_to_boostrap: 0
+
+--let $assert_file= $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat
+--source include/assert_grep.inc
+
+--let $assert_file= $MYSQLTEST_VARDIR/mysqld.3/data/grastate.dat
+--source include/assert_grep.inc
+
+# But node #1 should have safe_to_boostrap: 1
+
+--let $assert_text= grastate.dat does not have 'safe_to_bootstrap: 1'
+--let $assert_select= safe_to_bootstrap: 1
+
+--let $assert_file= $MYSQLTEST_VARDIR/mysqld.1/data/grastate.dat
+--source include/assert_grep.inc
+
+# Restart one node
+
+--connection node_2
+--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
+--source include/start_mysqld.inc
+
+--connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
+
+# All nodes should be back to 'safe_to_bootstrap: 0'
+
+--let $assert_text= grastate.dat does not have 'safe_to_bootstrap: 0'
+--let $assert_select= safe_to_bootstrap: 0
+
+--let $assert_file= $MYSQLTEST_VARDIR/mysqld.1/data/grastate.dat
+--source include/assert_grep.inc
+
+--let $assert_file= $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat
+--source include/assert_grep.inc
+
+--let $assert_file= $MYSQLTEST_VARDIR/mysqld.3/data/grastate.dat
+--source include/assert_grep.inc
+
+#
+# Kill the cluster
+#
+
+--connection node_2
+--source include/shutdown_mysqld.inc
+
+--connection node_1
+SET SESSION wsrep_on = OFF;
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
+
+--source include/kill_galera.inc
+
+#
+# Only node #1 should have safe_to_bootstrap: 1
+# include/assert_grep.inc requires a running server, so we revert to simple grep
+#
+
+--error 0
+--exec grep 'safe_to_bootstrap: 1' $MYSQLTEST_VARDIR/mysqld.1/data/grastate.dat
+
+--error 0
+--exec grep 'safe_to_bootstrap: 0' $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat
+
+--error 0
+--exec grep 'safe_to_bootstrap: 0' $MYSQLTEST_VARDIR/mysqld.3/data/grastate.dat
+
+#
+# Attempt to bootstrap nodes #2, #3, should fail 
+#
+
+--error 1
+--exec $MYSQLD --defaults-group-suffix=.2 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --wsrep-new-cluster | grep 'This node is not safe to bootstrap the cluster'
+--error 1
+--exec $MYSQLD --defaults-group-suffix=.3 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --wsrep-new-cluster | grep 'This node is not safe to bootstrap the cluster'
+
+#
+# Attempt to bootstrap starting from node #1, should succeed
+#
+
+--connection node_1
+--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--source include/start_mysqld.inc
+--source include/wait_until_connected_again.inc
+
+--connection node_2
+--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
+--source include/start_mysqld.inc
+
+--connection node_3
+--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.3.expect
+--source include/start_mysqld.inc
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
+
+--connection node_2
+CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
+
+--connection node_3
+CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
+SHOW CREATE TABLE t1;
+
+DROP TABLE t1;