MDEV-16543 Replicating to spider is fragile without retries (#1259)

mysql-test/suite/sys_vars/r/slave_transaction_retry_errors.result

 select @@global.slave_transaction_retry_errors;
 @@global.slave_transaction_retry_errors
-1213,1205,10,20,400
+1158,1159,1160,1161,1205,1213,1429,2013,12701,10,20,400
 select @@session.slave_transaction_retry_errors;
 ERROR HY000: Variable 'slave_transaction_retry_errors' is a GLOBAL variable
 show global variables like 'slave_transaction_retry_errors';
 Variable_name	Value
-slave_transaction_retry_errors	1213,1205,10,20,400
+slave_transaction_retry_errors	1158,1159,1160,1161,1205,1213,1429,2013,12701,10,20,400
 show session variables like 'slave_transaction_retry_errors';
 Variable_name	Value
-slave_transaction_retry_errors	1213,1205,10,20,400
+slave_transaction_retry_errors	1158,1159,1160,1161,1205,1213,1429,2013,12701,10,20,400
 select * from information_schema.global_variables where variable_name='slave_transaction_retry_errors';
 VARIABLE_NAME	VARIABLE_VALUE
-SLAVE_TRANSACTION_RETRY_ERRORS	1213,1205,10,20,400
+SLAVE_TRANSACTION_RETRY_ERRORS	1158,1159,1160,1161,1205,1213,1429,2013,12701,10,20,400
 select * from information_schema.session_variables where variable_name='slave_transaction_retry_errors';
 VARIABLE_NAME	VARIABLE_VALUE
-SLAVE_TRANSACTION_RETRY_ERRORS	1213,1205,10,20,400
+SLAVE_TRANSACTION_RETRY_ERRORS	1158,1159,1160,1161,1205,1213,1429,2013,12701,10,20,400
 set global slave_transaction_retry_errors=1;
 ERROR HY000: Variable 'slave_transaction_retry_errors' is a read only variable
 set session slave_transaction_retry_errors=1;

sql/slave.cc

@@ -995,6 +995,8 @@ static void make_slave_transaction_retry_errors_printable(void)
 }
 
 
+#define DEFAULT_SLAVE_RETRY_ERRORS 9
+
 bool init_slave_transaction_retry_errors(const char* arg)
 {
   const char *p;
@@ -1006,7 +1008,7 @@ bool init_slave_transaction_retry_errors(const char* arg)
   if (!arg)
     arg= "";
 
-  slave_transaction_retry_error_length= 2;
+  slave_transaction_retry_error_length= DEFAULT_SLAVE_RETRY_ERRORS;
   for (;my_isspace(system_charset_info,*arg);++arg)
     /* empty */;
   for (p= arg; *p; )
@@ -1029,11 +1031,18 @@ bool init_slave_transaction_retry_errors(const char* arg)
     currently, InnoDB deadlock detected by InnoDB or lock
     wait timeout (innodb_lock_wait_timeout exceeded
   */
-  slave_transaction_retry_errors[0]= ER_LOCK_DEADLOCK;
-  slave_transaction_retry_errors[1]= ER_LOCK_WAIT_TIMEOUT;
+  slave_transaction_retry_errors[0]= ER_NET_READ_ERROR;
+  slave_transaction_retry_errors[1]= ER_NET_READ_INTERRUPTED;
+  slave_transaction_retry_errors[2]= ER_NET_ERROR_ON_WRITE;
+  slave_transaction_retry_errors[3]= ER_NET_WRITE_INTERRUPTED;
+  slave_transaction_retry_errors[4]= ER_LOCK_WAIT_TIMEOUT;
+  slave_transaction_retry_errors[5]= ER_LOCK_DEADLOCK;
+  slave_transaction_retry_errors[6]= ER_CONNECT_TO_FOREIGN_DATA_SOURCE;
+  slave_transaction_retry_errors[7]= 2013; /* CR_SERVER_LOST */
+  slave_transaction_retry_errors[8]= 12701; /* ER_SPIDER_REMOTE_SERVER_GONE_AWAY_NUM */
 
   /* Add user codes after this */
-  for (p= arg, i= 2; *p; )
+  for (p= arg, i= DEFAULT_SLAVE_RETRY_ERRORS; *p; )
   {
     if (!(p= str2int(p, 10, 0, LONG_MAX, &err_code)))
       break;

sql/sys_vars.cc

@@ -5223,8 +5223,10 @@ static Sys_var_ulonglong Sys_read_binlog_speed_limit(
 static Sys_var_charptr Sys_slave_transaction_retry_errors(
        "slave_transaction_retry_errors", "Tells the slave thread to retry "
        "transaction for replication when a query event returns an error from "
-       "the provided list. Deadlock and elapsed lock wait timeout errors are "
-       "automatically added to this list",
+       "the provided list. Deadlock error, elapsed lock wait timeout, "
+       "net read error, net read timeout, net write error, net write timeout, "
+       "connect error and 2 types of lost connection error are automatically "
+       "added to this list",
        READ_ONLY GLOBAL_VAR(opt_slave_transaction_retry_errors), CMD_LINE(REQUIRED_ARG),
        IN_SYSTEM_CHARSET, DEFAULT(0));
 

storage/spider/mysql-test/spider/feature/include/slave_transaction_retry_errors_deinit.inc

+--connection slave1_1
+--disable_warnings
+--disable_query_log
+--disable_result_log
+--source ../include/deinit_spider.inc
+--source ../t/slave_test_deinit.inc
+--source ../t/test_deinit.inc
+--enable_result_log
+--enable_query_log
+--enable_warnings

storage/spider/mysql-test/spider/feature/include/slave_transaction_retry_errors_init.inc

+--disable_warnings
+--disable_query_log
+--disable_result_log
+--source ../t/test_init.inc
+--source ../t/slave_test_init.inc
+--connection slave1_1
+--source ../include/init_spider.inc
+--enable_result_log
+--enable_query_log
+--enable_warnings

storage/spider/mysql-test/spider/feature/my.cnf

+!include include/default_mysqld.cnf
+!include my_1_1.cnf

storage/spider/mysql-test/spider/feature/my_1_1.cnf

+[mysqld.1.1]
+log-bin=                  master-bin
+loose_handlersocket_port=                20000
+loose_handlersocket_port_wr=             20001
+loose_handlersocket_threads=             2
+loose_handlersocket_threads_wr=          1
+loose_handlersocket_support_merge_table= 0
+loose_handlersocket_direct_update_mode=  2
+loose_handlersocket_unlimited_boundary=  65536
+loose_handlersocket_bulk_insert=         0
+loose_handlersocket_bulk_insert_timeout= 0
+loose_handlersocket_general_log=         1
+loose_handlersocket_timeout=             30
+loose_handlersocket_close_table_interval=2
+open_files_limit=                        4096
+loose_partition=                         1
+
+[ENV]
+USE_GEOMETRY_TEST=        1
+USE_FULLTEXT_TEST=        1
+USE_HA_TEST=              1
+USE_GENERAL_LOG=          1
+USE_REPLICATION=          1
+MASTER_1_MYPORT=          @mysqld.1.1.port
+MASTER_1_HSRPORT=         20000
+MASTER_1_HSWPORT=         20001
+MASTER_1_MYSOCK=          @mysqld.1.1.socket
+MASTER_1_ENGINE_TYPE=     Spider
+#MASTER_1_ENGINE_TYPE=     MyISAM
+MASTER_1_ENGINE=          ENGINE=Spider
+MASTER_1_CHARSET=         DEFAULT CHARSET=utf8
+MASTER_1_ENGINE2=         ENGINE=MyISAM
+MASTER_1_CHARSET2=        DEFAULT CHARSET=utf8
+MASTER_1_CHARSET3=        DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+
+STR_SEMICOLON=            ;
+
+#The followings are set in include/init_xxx.inc files
+# MASTER_1_COMMENT_2_1
+# MASTER_1_COMMENT2_2_1
+# MASTER_1_COMMENT3_2_1
+# MASTER_1_COMMENT4_2_1
+# MASTER_1_COMMENT5_2_1
+# MASTER_1_COMMENT_P_2_1

storage/spider/mysql-test/spider/feature/my_2_1.cnf

+[mysqld.2.1]
+loose_handlersocket_port=                20002
+loose_handlersocket_port_wr=             20003
+loose_handlersocket_threads=             2
+loose_handlersocket_threads_wr=          1
+loose_handlersocket_support_merge_table= 0
+loose_handlersocket_direct_update_mode=  2
+loose_handlersocket_unlimited_boundary=  65536
+loose_handlersocket_bulk_insert=         0
+loose_handlersocket_bulk_insert_timeout= 0
+loose_handlersocket_general_log=         1
+loose_handlersocket_timeout=             30
+loose_handlersocket_close_table_interval=2
+open_files_limit=                        4096
+
+[ENV]
+USE_CHILD_GROUP2=         1
+OUTPUT_CHILD_GROUP2=      0
+CHILD2_1_MYPORT=          @mysqld.2.1.port
+CHILD2_1_HSRPORT=         20002
+CHILD2_1_HSWPORT=         20003
+CHILD2_1_MYSOCK=          @mysqld.2.1.socket
+CHILD2_1_ENGINE_TYPE=     InnoDB
+CHILD2_1_ENGINE=          ENGINE=InnoDB
+CHILD2_1_CHARSET=         DEFAULT CHARSET=utf8
+CHILD2_1_CHARSET2=        DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+CHILD2_1_FT_MYPORT=       @mysqld.2.1.port
+CHILD2_1_FT_MYSOCK=       @mysqld.2.1.socket
+CHILD2_1_FT_ENGINE_TYPE=  MyISAM
+CHILD2_1_FT_ENGINE=       ENGINE=MyISAM
+CHILD2_1_FT_CHARSET=      DEFAULT CHARSET=utf8
+CHILD2_1_GM_MYPORT=       @mysqld.2.1.port
+CHILD2_1_GM_MYSOCK=       @mysqld.2.1.socket
+CHILD2_1_GM_ENGINE_TYPE=  MyISAM
+CHILD2_1_GM_ENGINE=       ENGINE=MyISAM
+CHILD2_1_GM_CHARSET=      DEFAULT CHARSET=utf8
+
+#The followings are set in include/init_xxx.inc files
+# CHILD2_1_DROP_TABLES
+# CHILD2_1_CREATE_TABLES
+# CHILD2_1_SELECT_TABLES
+# CHILD2_1_DROP_TABLES2
+# CHILD2_1_CREATE_TABLES2
+# CHILD2_1_SELECT_TABLES2
+# CHILD2_1_DROP_TABLES3
+# CHILD2_1_CREATE_TABLES3
+# CHILD2_1_SELECT_TABLES3
+# CHILD2_1_DROP_TABLES4
+# CHILD2_1_CREATE_TABLES4
+# CHILD2_1_SELECT_TABLES4
+# CHILD2_1_DROP_TABLES5
+# CHILD2_1_CREATE_TABLES5
+# CHILD2_1_SELECT_TABLES5
+# CHILD2_1_DROP_TABLES6
+# CHILD2_1_CREATE_TABLES6
+# CHILD2_1_SELECT_TABLES6

storage/spider/mysql-test/spider/feature/my_2_2.cnf

+[mysqld.2.2]
+loose_handlersocket_port=                20004
+loose_handlersocket_port_wr=             20005
+loose_handlersocket_threads=             2
+loose_handlersocket_threads_wr=          1
+loose_handlersocket_support_merge_table= 0
+loose_handlersocket_direct_update_mode=  2
+loose_handlersocket_unlimited_boundary=  65536
+loose_handlersocket_bulk_insert=         0
+loose_handlersocket_bulk_insert_timeout= 0
+loose_handlersocket_general_log=         1
+loose_handlersocket_timeout=             30
+loose_handlersocket_close_table_interval=2
+open_files_limit=                        4096
+
+[ENV]
+CHILD2_2_MYPORT=          @mysqld.2.2.port
+CHILD2_2_HSRPORT=         20004
+CHILD2_2_HSWPORT=         20005
+CHILD2_2_MYSOCK=          @mysqld.2.2.socket
+CHILD2_2_ENGINE_TYPE=     InnoDB
+CHILD2_2_ENGINE=          ENGINE=InnoDB
+CHILD2_2_CHARSET=         DEFAULT CHARSET=utf8
+CHILD2_2_FT_MYPORT=       @mysqld.2.2.port
+CHILD2_2_FT_MYSOCK=       @mysqld.2.2.socket
+CHILD2_2_FT_ENGINE_TYPE=  MyISAM
+CHILD2_2_FT_ENGINE=       ENGINE=MyISAM
+CHILD2_2_FT_CHARSET=      DEFAULT CHARSET=utf8
+CHILD2_2_GM_MYPORT=       @mysqld.2.2.port
+CHILD2_2_GM_MYSOCK=       @mysqld.2.2.socket
+CHILD2_2_GM_ENGINE_TYPE=  MyISAM
+CHILD2_2_GM_ENGINE=       ENGINE=MyISAM
+CHILD2_2_GM_CHARSET=      DEFAULT CHARSET=utf8
+
+#The followings are set in include/init_xxx.inc files
+# CHILD2_2_DROP_TABLES
+# CHILD2_2_CREATE_TABLES
+# CHILD2_2_SELECT_TABLES

storage/spider/mysql-test/spider/feature/my_2_3.cnf

+[mysqld.2.3]
+
+[ENV]
+CHILD2_3_MYPORT=          @mysqld.2.3.port
+CHILD2_3_MYSOCK=          @mysqld.2.3.socket
+CHILD2_3_ENGINE_TYPE=     InnoDB
+CHILD2_3_ENGINE=          ENGINE=InnoDB
+CHILD2_3_CHARSET=         DEFAULT CHARSET=utf8

storage/spider/mysql-test/spider/feature/my_3_1.cnf

+[mysqld.3.1]
+loose_partition=                         1
+
+[ENV]
+USE_CHILD_GROUP3=         1
+OUTPUT_CHILD_GROUP3=      0
+CHILD3_1_MYPORT=          @mysqld.3.1.port
+CHILD3_1_MYSOCK=          @mysqld.3.1.socket
+CHILD3_1_ENGINE_TYPE=     InnoDB
+CHILD3_1_ENGINE=          ENGINE=InnoDB
+CHILD3_1_CHARSET=         DEFAULT CHARSET=utf8

storage/spider/mysql-test/spider/feature/my_3_2.cnf

+[mysqld.3.2]
+loose_partition=                         1
+
+[ENV]
+CHILD3_2_MYPORT=          @mysqld.3.2.port
+CHILD3_2_MYSOCK=          @mysqld.3.2.socket
+CHILD3_2_ENGINE_TYPE=     InnoDB
+CHILD3_2_ENGINE=          ENGINE=InnoDB
+CHILD3_2_CHARSET=         DEFAULT CHARSET=utf8

storage/spider/mysql-test/spider/feature/my_3_3.cnf

+[mysqld.3.3]
+loose_partition=                         1
+
+[ENV]
+CHILD3_3_MYPORT=          @mysqld.3.3.port
+CHILD3_3_MYSOCK=          @mysqld.3.3.socket
+CHILD3_3_ENGINE_TYPE=     InnoDB
+CHILD3_3_ENGINE=          ENGINE=InnoDB
+CHILD3_3_CHARSET=         DEFAULT CHARSET=utf8

storage/spider/mysql-test/spider/feature/my_4_1.cnf

+[mysqld.4.1]
+loose_partition=                         1
+
+[ENV]
+SLAVE1_1_MYPORT=          @mysqld.4.1.port
+SLAVE1_1_MYSOCK=          @mysqld.4.1.socket
+SLAVE1_1_ENGINE_TYPE=     MyISAM
+SLAVE1_1_ENGINE=          ENGINE=MyISAM
+SLAVE1_1_CHARSET=         DEFAULT CHARSET=utf8

storage/spider/mysql-test/spider/feature/r/slave_transaction_retry_errors.result

+for master_1
+for child2
+child2_1
+child2_2
+child2_3
+for child3
+for slave1_1
+
+connection slave1_1;
+SHOW VARIABLES LIKE 'slave_transaction_retry_errors';
+Variable_name	Value
+slave_transaction_retry_errors	1158,1159,1160,1161,1205,1213,1429,2013,12701
+connection slave1_1;
+for slave1_1
+for master_1
+for child2
+child2_1
+child2_2
+child2_3
+for child3
+
+end of test

storage/spider/mysql-test/spider/feature/suite.opt

+--loose-innodb --loose-skip-performance-schema

storage/spider/mysql-test/spider/feature/suite.pm

+package My::Suite::Spider;
+
+@ISA = qw(My::Suite);
+
+return "No Spider engine" unless $ENV{HA_SPIDER_SO};
+return "Not run for embedded server" if $::opt_embedded_server;
+return "Test needs --big-test" unless $::opt_big_test;
+
+sub is_default { 1 }
+
+bless { };
+

storage/spider/mysql-test/spider/feature/t/slave_transaction_retry_errors.cnf

+!include include/default_mysqld.cnf
+!include ../my_1_1.cnf
+!include ../my_2_1.cnf
+!include ../my_4_1.cnf

storage/spider/mysql-test/spider/feature/t/slave_transaction_retry_errors.test

+--source ../include/slave_transaction_retry_errors_init.inc
+--echo
+
+--connection slave1_1
+SHOW VARIABLES LIKE 'slave_transaction_retry_errors';
+
+--source ../include/slave_transaction_retry_errors_deinit.inc
+--echo
+--echo end of test