Galera MTR Tests: MW-308 , MW-307, GCF-992

* a dedicated test for wsrep_retry_autocommit
* some galera_toi_* tests were only passing because wsrep_retry_autocommit
  was in effect. The tests were changed to do not use autocommit
* higher timeout values in galera_2nodes.cnf , galera_3nodes.cnf
Signed-off-by: Sachin Setiya <sachin.setiya@mariadb.com>

mysql-test/suite/galera/galera_2nodes.cnf

@@ -25,7 +25,14 @@ wsrep_sst_receive_address='127.0.0.1:@mysqld.1.#sst_port'
 #ist_port=@OPT.port
 #sst_port=@OPT.port
 wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.1.#galera_port'
-wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.2.#galera_port;gcache.size=10M;evs.suspect_timeout=PT10S'
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.2.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S'
+
+# enforce read-committed characteristics across the cluster
+wsrep_causal_reads=ON
+wsrep_sync_wait = 7
+
+wsrep_node_address=127.0.0.1
+wsrep_sst_receive_address=127.0.0.2:@mysqld.2.#sst_port
 wsrep_node_incoming_address=127.0.0.1:@mysqld.2.port
 wsrep_sst_receive_address='127.0.0.1:@mysqld.2.#sst_port'
 

mysql-test/suite/galera/r/galera_defaults.result

@@ -50,7 +50,7 @@ WSREP_SST_DONOR
 WSREP_SST_DONOR_REJECTS_QUERIES	OFF
 WSREP_SST_METHOD	rsync
 WSREP_SYNC_WAIT	7
-<BASE_DIR>; <BASE_HOST>; <BASE_PORT>; cert.log_conflicts = no; debug = no; evs.auto_evict = 0; evs.causal_keepalive_period = PT1S; evs.debug_log_mask = 0x1; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT15S; evs.info_log_mask = 0; evs.install_timeout = PT7.5S; evs.join_retrans_period = PT1S; evs.keepalive_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 4; evs.stats_report_period = PT1M; evs.suspect_timeout = PT10S; evs.use_aggregate = true; evs.user_send_window = 2; evs.version = 0; evs.view_forget_timeout = P1D; <GCACHE_DIR>; gcache.keep_pages_size = 0; gcache.mem_size = 0; <GCACHE_NAME>; gcache.page_size = 128M; gcache.size = 10M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1.0; gcs.fc_limit = 16; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; <GCS_RECV_Q_HARD_LIMIT>; gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; <GMCAST_LISTEN_ADDR>; gmcast.mcast_addr = ; gmcast.mcast_ttl = 1; gmcast.peer_timeout = PT3S; gmcast.segment = 0; gmcast.time_wait = PT5S; gmcast.version = 0; <IST_RECV_ADDR>; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.linger = PT20S; pc.npvo = false; pc.recovery = true; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = P30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT90S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 7; socket.checksum = 2; socket.recv_buf_size = 212992; 
+<BASE_DIR>; <BASE_HOST>; <BASE_PORT>; cert.log_conflicts = no; debug = no; evs.auto_evict = 0; evs.causal_keepalive_period = PT1S; evs.debug_log_mask = 0x1; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT30S; evs.info_log_mask = 0; evs.install_timeout = PT15S; evs.join_retrans_period = PT1S; evs.keepalive_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 4; evs.stats_report_period = PT1M; evs.suspect_timeout = PT10S; evs.use_aggregate = true; evs.user_send_window = 2; evs.version = 0; evs.view_forget_timeout = P1D; <GCACHE_DIR>; gcache.keep_pages_size = 0; gcache.mem_size = 0; <GCACHE_NAME>; gcache.page_size = 128M; gcache.size = 128M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1.0; gcs.fc_limit = 16; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; <RECV_Q_HARD_LIMIT>;gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; <GMCAST_LISTEN_ADDR>; gmcast.mcast_addr = ; gmcast.mcast_ttl = 1; gmcast.peer_timeout = PT3S; gmcast.segment = 0; gmcast.time_wait = PT5S; gmcast.version = 0; <IST_RECV_ADDR>; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.linger = PT20S; pc.npvo = false; pc.recovery = true; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = P30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT90S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 7; socket.checksum = 2; socket.recv_buf_size = 212992; 
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.GLOBAL_STATUS
 WHERE VARIABLE_NAME LIKE 'wsrep_%'
 AND VARIABLE_NAME != 'wsrep_debug_sync_waiters';

mysql-test/suite/galera/r/galera_toi_ddl_online.result

-CREATE TABLE ten (f1 INTEGER);
-INSERT INTO ten VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
-CREATE TABLE t1 (f1 INTEGER PRIMARY KEY AUTO_INCREMENT, f2 INTEGER) ENGINE=InnoDB;
-INSERT INTO t1 (f2) SELECT 1 FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4, ten AS a5;
-INSERT INTO t1 (f2) SELECT 1 FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4, ten AS a5;;
-CREATE INDEX i1 ON t1 (f2);;
-ALTER TABLE t1 ADD COLUMN f3 INTEGER;;
-SELECT COUNT(*) = 200000 FROM t1;
-COUNT(*) = 200000
-1
-SELECT COUNT(*) = 3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
-COUNT(*) = 3
-1
-SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = 't1';
-COUNT(*) = 2
-1
-SELECT COUNT(*) = 200000 FROM t1;
-COUNT(*) = 200000
-1
-SELECT COUNT(*) = 3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
-COUNT(*) = 3
-1
-SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = 't1';
-COUNT(*) = 2
-1
-DROP TABLE t1;
-DROP TABLE ten;

mysql-test/suite/galera/r/galera_toi_drop_database.result

@@ -4,12 +4,14 @@ CREATE TABLE ten (f1 INTEGER) ENGINE=InnoDB;
 INSERT INTO ten VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10);
 CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 CREATE TABLE t2 (f1 INTEGER) ENGINE=InnoDB;
+SET SESSION wsrep_retry_autocommit = 0;
 INSERT INTO t1 (f1) SELECT 1 FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4, ten AS a5;;
 USE database1;
+SET SESSION wsrep_retry_autocommit = 0;
 INSERT INTO t2 (f1) SELECT 1 FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4, ten AS a5;;
 DROP DATABASE database1;;
-ERROR 42S02: Table 'database1.t1' doesn't exist
-ERROR 42S02: Table 'database1.t2' doesn't exist
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
 SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = 'database1';
 COUNT(*) = 0
 1

mysql-test/suite/galera/r/galera_toi_truncate.result

 CREATE TABLE ten (f1 INTEGER) ENGINE=InnoDB;
 INSERT INTO ten VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10);
 CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
+SET SESSION wsrep_retry_autocommit = 0;
 INSERT INTO t1 (f1) SELECT 1 FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4, ten AS a5, ten AS a6;;
 TRUNCATE TABLE t1;;
-SELECT COUNT(*) = 1000000 FROM t1;
-COUNT(*) = 1000000
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+SELECT COUNT(*) = 0 FROM t1;
+COUNT(*) = 0
 1
-DROP TABLE t1;
-CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
-INSERT INTO t1 (f1) SELECT 1 FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4, ten AS a5, ten AS a6;;
-TRUNCATE TABLE t1;;
-SELECT COUNT(*) = 1000000 FROM t1;
-COUNT(*) = 1000000
+SELECT COUNT(*) = 0 FROM t1;
+COUNT(*) = 0
 1
 DROP TABLE t1;
 DROP TABLE ten;

mysql-test/suite/galera/r/galera_var_retry_autocommit.result

+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=InnoDB;
+CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW SET NEW.f2 = SLEEP(5);
+SET SESSION wsrep_retry_autocommit = 0;
+INSERT INTO t1 (f1) VALUES (1),(2);;
+TRUNCATE TABLE t1;
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+SET SESSION wsrep_retry_autocommit = 1;
+INSERT INTO t1 (f1) VALUES (3),(4);;
+TRUNCATE TABLE t1;
+SELECT * FROM test.t1;
+f1	f2
+3	0
+4	0
+CREATE PROCEDURE repeated_truncate ()
+BEGIN
+DECLARE i INT;
+DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;
+SET i = 0;
+WHILE i <= 1000 DO
+TRUNCATE TABLE t1;
+SET i = i + 1;
+END WHILE;
+END|
+CALL repeated_truncate();
+SET SESSION wsrep_retry_autocommit = 1;
+INSERT INTO t1 (f1) VALUES (5),(6);
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+SET SESSION wsrep_retry_autocommit = 1024;
+INSERT INTO t1 (f1) VALUES (7),(8);;
+include/diff_servers.inc [servers=1 2]
+DROP TABLE t1;
+DROP PROCEDURE repeated_truncate;

mysql-test/suite/galera/t/galera_toi_ddl_online.test

---source include/galera_cluster.inc
---source include/have_innodb.inc
-
-#
-# In this test, we run concurrent INSERT against a table against which we have issued concurrent DDL statements that are fully online,
-# that is, DDL statements that allow for the DML to proceed non-blocking while the DDL is in progress
-#
-
-CREATE TABLE ten (f1 INTEGER);
-INSERT INTO ten VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
-
-CREATE TABLE t1 (f1 INTEGER PRIMARY KEY AUTO_INCREMENT, f2 INTEGER) ENGINE=InnoDB;
-INSERT INTO t1 (f2) SELECT 1 FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4, ten AS a5;
-
---connection node_1
---send INSERT INTO t1 (f2) SELECT 1 FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4, ten AS a5;
-
---let $galera_connection_name = node_1a
---let $galera_server_number = 1
---source include/galera_connect.inc
-
---connection node_1a
---send CREATE INDEX i1 ON t1 (f2);
-
-
---let $galera_connection_name = node_1b
---let $galera_server_number = 1
---source include/galera_connect.inc
-
---connection node_1b
---send ALTER TABLE t1 ADD COLUMN f3 INTEGER;
-
---connection node_1
---reap
-
---connection node_1a
---reap
-
---connection node_1b
---reap
-
---connection node_2
-SELECT COUNT(*) = 200000 FROM t1;
-SELECT COUNT(*) = 3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
-SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = 't1';
-
---connection node_1
-SELECT COUNT(*) = 200000 FROM t1;
-SELECT COUNT(*) = 3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
-SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = 't1';
-
-DROP TABLE t1;
-DROP TABLE ten;

mysql-test/suite/galera/t/galera_toi_drop_database.test

@@ -2,7 +2,6 @@
 # Test the operation of DDLs that affect multiple database objects
 #
 
---source include/big_test.inc
 --source include/galera_cluster.inc
 --source include/have_innodb.inc
 
@@ -20,11 +19,14 @@ INSERT INTO ten VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10);
 CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 CREATE TABLE t2 (f1 INTEGER) ENGINE=InnoDB;
 
-# Insert 1M rows
+# Make sure autocommit retrying does not kick in as this will mask the error we expect to get
+SET SESSION wsrep_retry_autocommit = 0;
+# Attemp to insert 1M rows
 --send INSERT INTO t1 (f1) SELECT 1 FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4, ten AS a5;
 
 --connection node_1a
 USE database1;
+SET SESSION wsrep_retry_autocommit = 0;
 --send INSERT INTO t2 (f1) SELECT 1 FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4, ten AS a5;
 
 --connection node_2
@@ -32,12 +34,12 @@ USE database1;
 --send DROP DATABASE database1;
 
 --connection node_1
---sleep 30
---error ER_NO_SUCH_TABLE
+--sleep 1
+--error ER_LOCK_DEADLOCK
 --reap
 
 --connection node_1a
---error ER_NO_SUCH_TABLE
+--error ER_LOCK_DEADLOCK
 --reap
 
 --connection node_2

mysql-test/suite/galera/t/galera_toi_truncate.test

 #
-# Test the operation of TRUNCATE with concurrent DML. Even in the face of a concurrent INSERT,
-# the TRUNCATE will complete first and be recorded in the history before the INSERT.
+# Test the operation of TRUNCATE with concurrent DML.
+# The DML should be BF-aborted if the DDL arrives from another node
 #
 
---source include/big_test.inc
 --source include/galera_cluster.inc
 --source include/have_innodb.inc
 
@@ -19,6 +18,8 @@ CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 
 # Insert 100K rows
 --connection node_2
+# Prevent autocommit retring from masking the deadlock error we expect to get
+SET SESSION wsrep_retry_autocommit = 0;
 --send INSERT INTO t1 (f1) SELECT 1 FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4, ten AS a5, ten AS a6;
 
 --connection node_1
@@ -29,49 +30,14 @@ CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 --reap
 
 --connection node_2
+--error ER_LOCK_DEADLOCK
 --reap
 
 --connection node_2
-SELECT COUNT(*) = 1000000 FROM t1;
+SELECT COUNT(*) = 0 FROM t1;
 
 --connection node_1
---let $wait_condition = SELECT COUNT(*) = 1000000 FROM t1;
---source include/wait_condition.inc
-
-DROP TABLE t1;
-
-#
-# INSERT AND TRUNCATE on same node
-# 
-
---let $galera_connection_name = node_1a
---let $galera_server_number = 1
---source include/galera_connect.inc
-
---connection node_1
-
-CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
-
-# Insert 100K rows
---connection node_1
---send INSERT INTO t1 (f1) SELECT 1 FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4, ten AS a5, ten AS a6;
-
---connection node_1a
---sleep 1
---send TRUNCATE TABLE t1;
-
---connection node_1
---reap
-
---connection node_1a
---reap
-
---connection node_1
-SELECT COUNT(*) = 1000000 FROM t1;
-
---connection node_2
---let $wait_condition = SELECT COUNT(*) = 1000000 FROM t1;
---source include/wait_condition.inc
+SELECT COUNT(*) = 0 FROM t1;
 
 DROP TABLE t1;
 DROP TABLE ten;

mysql-test/suite/galera/t/galera_var_retry_autocommit.test

+#
+# Test that the wsrep_retry_autocommit variable is respected. We use an INSERT that
+# proceeds very slowly due to extra SLEEP() in a trigger
+#
+
+--source include/galera_cluster.inc
+
+--connection node_1
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=InnoDB;
+CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW SET NEW.f2 = SLEEP(5);
+
+#
+# With wsrep_retry_autocommit = 0, error is certain
+#
+
+--connection node_1
+SET SESSION wsrep_retry_autocommit = 0;
+--send INSERT INTO t1 (f1) VALUES (1),(2);
+
+--connection node_2
+--sleep 1
+TRUNCATE TABLE t1;
+
+--connection node_1
+--error ER_LOCK_DEADLOCK
+--reap
+
+#
+# With wsrep_retry_autocommit = 1, success against one TRUNCATE
+#
+
+--connection node_1
+SET SESSION wsrep_retry_autocommit = 1;
+--send INSERT INTO t1 (f1) VALUES (3),(4);
+
+--connection node_2
+--sleep 1
+TRUNCATE TABLE t1;
+
+--connection node_1
+--error 0
+--reap
+SELECT * FROM test.t1;
+
+#
+# With wsrep_retry_autocommit = 1, failure against multiple TRUNCATEs
+#
+
+--connection node_2
+DELIMITER |;
+CREATE PROCEDURE repeated_truncate ()
+BEGIN
+	DECLARE i INT;
+	DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;
+
+	SET i = 0;
+        WHILE i <= 1000 DO
+		TRUNCATE TABLE t1;
+		SET i = i + 1;
+        END WHILE;
+END|
+DELIMITER ;|
+
+# Begin streaming TRUNCATEs
+--let $truncate_connection_id = `SELECT CONNECTION_ID()`
+--send CALL repeated_truncate()
+
+--connection node_1
+SET SESSION wsrep_retry_autocommit = 1;
+--sleep 1
+--error ER_LOCK_DEADLOCK
+INSERT INTO t1 (f1) VALUES (5),(6);
+
+#
+# With wsrep_retry_autocommit = 1024, success against multiple TRUNCATEs
+#
+
+--connection node_1
+SET SESSION wsrep_retry_autocommit = 1024;
+--send INSERT INTO t1 (f1) VALUES (7),(8);
+
+--sleep 6
+
+# Once he stream of TRUNCATEs is complete
+--connection node_2
+--reap
+
+# the INSERT will eventually be sucessfull
+--connection node_1
+--error 0
+--reap
+
+--let $diff_servers = 1 2
+--source include/diff_servers.inc
+
+DROP TABLE t1;
+DROP PROCEDURE repeated_truncate;

mysql-test/suite/galera_3nodes/galera_3nodes.cnf

@@ -27,7 +27,9 @@ wsrep_sst_receive_address='127.0.0.1:@mysqld.1.#sst_port'
 #ist_port=@OPT.port
 #sst_port=@OPT.port
 wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.1.#galera_port'
-wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=10M;evs.suspect_timeout=PT10S'
+wsrep_provider_options='base_port=@mysqld.2.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S'
+
+wsrep_sst_receive_address=127.0.0.2:@mysqld.2.#sst_port
 wsrep_node_incoming_address=127.0.0.1:@mysqld.2.port
 wsrep_sst_receive_address='127.0.0.1:@mysqld.2.#sst_port'
 
@@ -36,7 +38,9 @@ wsrep_sst_receive_address='127.0.0.1:@mysqld.2.#sst_port'
 #ist_port=@OPT.port
 #sst_port=@OPT.port
 wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.1.#galera_port'
-wsrep_provider_options='base_port=@mysqld.3.#galera_port;gcache.size=10M;evs.suspect_timeout=PT10S'
+wsrep_provider_options='base_port=@mysqld.3.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S'
+
+wsrep_sst_receive_address=127.0.0.2:@mysqld.3.#sst_port
 wsrep_node_incoming_address=127.0.0.1:@mysqld.3.port
 wsrep_sst_receive_address='127.0.0.1:@mysqld.3.#sst_port'