MDEV-26223 Galera cluster node consider old server_id value even after...

MDEV-26223 Galera cluster node consider old server_id value even after modification of server_id [wsrep_gtid_mode=ON]

Variable `wsrep_new_cluster` now will be TRUE also when there is only `gcomm://` used
in configuration. This configuration, even without --wsrep-new-cluster,
is considered to bootstrap new cluster.

Updated galera GTID test to ignore warning message when non bootstrap
node have server-id different thant one cluster is initialized with.
Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>

mysql-test/suite/galera/r/MDEV-10715.result

@@ -26,3 +26,5 @@ wsrep_last_written_gtid()
 1-1-0
 connection node_1;
 drop table t1;
+connection node_2;
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");

mysql-test/suite/galera/r/galera-features#117.result

 connection node_2;
 connection node_1;
+connection node_1;
+connection node_2;
 connection node_2;
 SET SESSION wsrep_on=OFF;
 CREATE TABLE test.t1 (f2 INTEGER);
@@ -35,3 +37,4 @@ Killing server ...
 CALL mtr.add_suppression("Inconsistent by consensus.");
 CALL mtr.add_suppression("WSREP: Failed to execute TOI action");
 CALL mtr.add_suppression("WSREP: TO isolation end failed");
+connection node_1;

mysql-test/suite/galera/r/galera_gtid_server_id.result

+connection node_2;
+connection node_1;
+connection node_1;
+select @@gtid_domain_id, @@server_id, @@wsrep_gtid_domain_id,@@wsrep_gtid_mode;
+@@gtid_domain_id	@@server_id	@@wsrep_gtid_domain_id	@@wsrep_gtid_mode
+0	11	1	1
+connection node_2;
+call mtr.add_suppression("WSREP: Ignoring server id for non bootstrap node.");
+select @@gtid_domain_id, @@server_id, @@wsrep_gtid_domain_id,@@wsrep_gtid_mode;
+@@gtid_domain_id	@@server_id	@@wsrep_gtid_domain_id	@@wsrep_gtid_mode
+0	12	1	1
+connection node_1;
+CREATE TABLE t1(id int not null primary key) engine=innodb;
+INSERT INTO t1 values (1);
+show global variables  like '%gtid%';
+Variable_name	Value
+gtid_binlog_pos	1-11-2
+gtid_binlog_state	1-11-2
+gtid_cleanup_batch_size	64
+gtid_current_pos	1-11-2
+gtid_domain_id	0
+gtid_ignore_duplicates	OFF
+gtid_pos_auto_engines	
+gtid_slave_pos	
+gtid_strict_mode	OFF
+wsrep_gtid_domain_id	1
+wsrep_gtid_mode	ON
+connection node_2;
+SET SESSION wsrep_sync_wait=15;
+show global variables  like '%gtid%';
+Variable_name	Value
+gtid_binlog_pos	0-12-1,1-11-2
+gtid_binlog_state	0-12-1,1-11-2
+gtid_cleanup_batch_size	64
+gtid_current_pos	0-12-1
+gtid_domain_id	0
+gtid_ignore_duplicates	OFF
+gtid_pos_auto_engines	
+gtid_slave_pos	
+gtid_strict_mode	OFF
+wsrep_gtid_domain_id	1
+wsrep_gtid_mode	ON
+SET GLOBAL server_id=200;
+ERROR 42000: Variable 'server_id' can't be set to the value of '200'
+SHOW WARNINGS;
+Level	Code	Message
+Warning	1231	Can't change server_id because wsrep and wsrep_gtid_mode is set. You can set server_id only with wsrep_new_cluster. 
+Error	1231	Variable 'server_id' can't be set to the value of '200'
+SET SESSION server_id=200;
+ERROR 42000: Variable 'server_id' can't be set to the value of '200'
+SHOW WARNINGS;
+Level	Code	Message
+Warning	1231	Can't change server_id because wsrep and wsrep_gtid_mode is set. You can set server_id only with wsrep_new_cluster. 
+Error	1231	Variable 'server_id' can't be set to the value of '200'
+INSERT INTO t1 values(2);
+show global variables  like '%gtid%';
+Variable_name	Value
+gtid_binlog_pos	0-12-1,1-11-3
+gtid_binlog_state	0-12-1,1-11-3
+gtid_cleanup_batch_size	64
+gtid_current_pos	0-12-1
+gtid_domain_id	0
+gtid_ignore_duplicates	OFF
+gtid_pos_auto_engines	
+gtid_slave_pos	
+gtid_strict_mode	OFF
+wsrep_gtid_domain_id	1
+wsrep_gtid_mode	ON
+connection node_1;
+show global variables  like '%gtid%';
+Variable_name	Value
+gtid_binlog_pos	1-11-3
+gtid_binlog_state	1-11-3
+gtid_cleanup_batch_size	64
+gtid_current_pos	1-11-3
+gtid_domain_id	0
+gtid_ignore_duplicates	OFF
+gtid_pos_auto_engines	
+gtid_slave_pos	
+gtid_strict_mode	OFF
+wsrep_gtid_domain_id	1
+wsrep_gtid_mode	ON
+DROP TABLE t1;

mysql-test/suite/galera/r/galera_gtid_slave.result

@@ -41,3 +41,5 @@ reset master;
 SET GLOBAL wsrep_on=ON;
 connection node_3;
 reset master;
+connection node_2;
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");

mysql-test/suite/galera/r/galera_gtid_slave_sst_rsync.result

@@ -166,3 +166,5 @@ set global wsrep_on=ON;
 #Connection 3
 connection node_3;
 reset master;
+connection node_2;
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");

mysql-test/suite/galera/r/galera_gtid_trx_conflict.result

@@ -42,3 +42,4 @@ SELECT @@gtid_binlog_state;
 @@gtid_binlog_state
 1-1-101
 DROP TABLE t1;
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");

mysql-test/suite/galera/r/galera_last_committed_id.result

@@ -33,3 +33,5 @@ wsrep_last_written_id_advanced
 1
 SET AUTOCOMMIT=ON;
 DROP TABLE t1;
+connection node_2;
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");

mysql-test/suite/galera/r/galera_sync_wait_upto.result

@@ -23,3 +23,5 @@ WSREP_SYNC_WAIT_UPTO
 1
 connection node_1;
 DROP TABLE t1;
+connection node_2;
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");

mysql-test/suite/galera/r/galera_var_gtid_domain_id.result

@@ -89,6 +89,8 @@ connection node_2;
 set global wsrep_on=OFF;
 reset master;
 set global wsrep_on=ON;
+connection node_2;
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");
 disconnect node_2;
 disconnect node_1;
 # End of test

mysql-test/suite/galera/r/mdev_10518.result

@@ -89,6 +89,8 @@ connection node_2;
 set global wsrep_on=OFF;
 reset master;
 set global wsrep_on=ON;
+connection node_2;
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");
 disconnect node_2;
 disconnect node_1;
 # End of test

mysql-test/suite/galera/t/MDEV-10715.test

@@ -17,3 +17,6 @@ select wsrep_last_written_gtid();
 
 --connection node_1
 drop table t1;
+
+--connection node_2
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");
\ No newline at end of file

mysql-test/suite/galera/t/galera-features#117.test

@@ -4,6 +4,11 @@
 --source include/galera_cluster.inc
 --source include/have_innodb.inc
 
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--source ../galera/include/auto_increment_offset_save.inc
+
 --connection node_2
 SET SESSION wsrep_on=OFF;
 CREATE TABLE test.t1 (f2 INTEGER);
@@ -36,3 +41,7 @@ SHOW TABLES IN test;
 CALL mtr.add_suppression("Inconsistent by consensus.");
 CALL mtr.add_suppression("WSREP: Failed to execute TOI action");
 CALL mtr.add_suppression("WSREP: TO isolation end failed");
+
+--connection node_1
+# Restore original auto_increment_offset values.
+--source ../galera/include/auto_increment_offset_restore.inc

mysql-test/suite/galera/t/galera_gtid_server_id.cnf

+!include ../galera_2nodes.cnf
+
+[mysqld.1]
+wsrep_gtid_domain_id=1
+wsrep_gtid_mode=ON
+server-id=11
+log_slave_updates
+log_bin
+
+[mysqld.2]
+wsrep_gtid_domain_id=1
+wsrep_gtid_mode=ON
+server-id=12
+log_slave_updates
+log_bin
+

mysql-test/suite/galera/t/galera_gtid_server_id.test

+--source include/galera_cluster.inc
+
+--connection node_1
+select @@gtid_domain_id, @@server_id, @@wsrep_gtid_domain_id,@@wsrep_gtid_mode;
+--connection node_2
+call mtr.add_suppression("WSREP: Ignoring server id for non bootstrap node.");
+select @@gtid_domain_id, @@server_id, @@wsrep_gtid_domain_id,@@wsrep_gtid_mode;
+
+--connection node_1
+CREATE TABLE t1(id int not null primary key) engine=innodb;
+INSERT INTO t1 values (1);
+show global variables  like '%gtid%';
+--connection node_2
+SET SESSION wsrep_sync_wait=15;
+show global variables  like '%gtid%';
+# This should give error
+--error ER_WRONG_VALUE_FOR_VAR
+SET GLOBAL server_id=200;
+SHOW WARNINGS;
+# This should give error
+--error ER_WRONG_VALUE_FOR_VAR
+SET SESSION server_id=200;
+SHOW WARNINGS;
+INSERT INTO t1 values(2);
+show global variables  like '%gtid%';
+--connection node_1
+show global variables  like '%gtid%';
+DROP TABLE t1;

mysql-test/suite/galera/t/galera_gtid_slave.test

@@ -90,3 +90,6 @@ SET GLOBAL wsrep_on=ON;
 
 --connection node_3
 reset master;
+
+--connection node_2
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");

mysql-test/suite/galera/t/galera_gtid_slave_sst_rsync.test

@@ -197,3 +197,6 @@ set global wsrep_on=ON;
 --echo #Connection 3
 --connection node_3
 reset master;
+
+--connection node_2
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");

mysql-test/suite/galera/t/galera_gtid_trx_conflict.test

@@ -51,6 +51,6 @@ SELECT @@gtid_binlog_state;
 
 --connection node_2
 SELECT @@gtid_binlog_state;
-
-
 DROP TABLE t1;
+
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");

mysql-test/suite/galera/t/galera_last_committed_id.test

@@ -64,3 +64,6 @@ COMMIT;
 SET AUTOCOMMIT=ON;
 
 DROP TABLE t1;
+
+--connection node_2
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");

mysql-test/suite/galera/t/galera_sync_wait_upto.test

@@ -70,3 +70,6 @@ INSERT INTO t1 VALUES (2);
 
 --connection node_1
 DROP TABLE t1;
+
+--connection node_2
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");

mysql-test/suite/galera/t/galera_var_gtid_domain_id.test

@@ -59,6 +59,8 @@ set global wsrep_on=OFF;
 reset master;
 set global wsrep_on=ON;
 
+--connection node_2
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");
+
 --source include/galera_end.inc
 --echo # End of test
-

mysql-test/suite/galera/t/mdev_10518.test

@@ -59,6 +59,8 @@ set global wsrep_on=OFF;
 reset master;
 set global wsrep_on=ON;
 
+--connection node_2
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");
+
 --source include/galera_end.inc
 --echo # End of test
-

mysql-test/suite/galera_3nodes/r/GCF-363.result

 connection node_2;
 connection node_1;
-CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 CHAR(1)) ENGINE=InnoDB;
 connection node_1;
+connection node_1;
+connection node_2;
+connection node_1;
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 CHAR(1)) ENGINE=InnoDB;
 SET GLOBAL wsrep_on=OFF;
 INSERT INTO t1 VALUES (1, 'a');
 SET GLOBAL wsrep_on=ON;
@@ -44,3 +47,4 @@ connection node_3;
 CALL mtr.add_suppression("WSREP: Vote 0 \\\(success\\\) on (.*) is inconsistent with group. Leaving cluster.");
 CALL mtr.add_suppression("WSREP: Inconsistency detected: Inconsistent by consensus on ");
 CALL mtr.add_suppression("Plugin 'InnoDB' will be forced to shutdown");
+connection node_1;

mysql-test/suite/galera_3nodes/r/galera_2_cluster.result

@@ -75,15 +75,19 @@ connection node_2;
 SET GLOBAL wsrep_on = OFF;
 RESET MASTER;
 SET GLOBAL wsrep_on = ON;
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");
 connection node_3;
 SET GLOBAL wsrep_on = OFF;
 RESET MASTER;
 SET GLOBAL wsrep_on = ON;
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");
 connection node_5;
 SET GLOBAL wsrep_on = OFF;
 RESET MASTER;
 SET GLOBAL wsrep_on = ON;
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");
 connection node_6;
 SET GLOBAL wsrep_on = OFF;
 RESET MASTER;
 SET GLOBAL wsrep_on = ON;
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");

mysql-test/suite/galera_3nodes/t/GCF-363.test

@@ -5,9 +5,15 @@
 --source include/galera_cluster.inc
 --source include/have_innodb.inc
 
-CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 CHAR(1)) ENGINE=InnoDB;
+--connection node_1
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--source ../galera/include/auto_increment_offset_save.inc
 
 --connection node_1
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 CHAR(1)) ENGINE=InnoDB;
+
 SET GLOBAL wsrep_on=OFF;
 INSERT INTO t1 VALUES (1, 'a');
 SET GLOBAL wsrep_on=ON;
@@ -63,4 +69,7 @@ CALL mtr.add_suppression("WSREP: Vote 0 \\\(success\\\) on (.*) is inconsistent
 CALL mtr.add_suppression("WSREP: Inconsistency detected: Inconsistent by consensus on ");
 CALL mtr.add_suppression("Plugin 'InnoDB' will be forced to shutdown");
 
+--connection node_1
+# Restore original auto_increment_offset values.
+--source ../galera/include/auto_increment_offset_restore.inc
 

mysql-test/suite/galera_3nodes/t/galera_2_cluster.test

@@ -129,20 +129,28 @@ SET GLOBAL wsrep_on = OFF;
 RESET MASTER;
 SET GLOBAL wsrep_on = ON;
 
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");
+
 --connection node_3
 
 SET GLOBAL wsrep_on = OFF;
 RESET MASTER;
 SET GLOBAL wsrep_on = ON;
 
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");
+
 --connection node_5
 
 SET GLOBAL wsrep_on = OFF;
 RESET MASTER;
 SET GLOBAL wsrep_on = ON;
 
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");
+
 --connection node_6
 
 SET GLOBAL wsrep_on = OFF;
 RESET MASTER;
 SET GLOBAL wsrep_on = ON;
+
+CALL mtr.add_suppression("Ignoring server id for non bootstrap node");

mysql-test/suite/galera_sr/r/mdev_18631.result

@@ -5,6 +5,7 @@ connection node_1;
 CREATE TABLE t1(f1 INT PRIMARY KEY) ENGINE=INNODB;
 INSERT INTO t1 VALUES (1), (2), (3);
 connection node_2;
+call mtr.add_suppression("WSREP: Ignoring server id for non bootstrap node.");
 SELECT * FROM  t1;
 f1
 1

mysql-test/suite/galera_sr/t/mdev_18631.test

@@ -14,6 +14,7 @@ CREATE TABLE t1(f1 INT PRIMARY KEY) ENGINE=INNODB;
 INSERT INTO t1 VALUES (1), (2), (3);
 
 --connection node_2
+call mtr.add_suppression("WSREP: Ignoring server id for non bootstrap node.");
 SELECT * FROM  t1;
 
 --connection node_1

sql/sys_vars.cc

 /* Copyright (c) 2002, 2015, Oracle and/or its affiliates.
-   Copyright (c) 2012, 2021, MariaDB Corporation.
+   Copyright (c) 2012, 2022, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -3223,6 +3223,21 @@ static Sys_var_charptr_fscs Sys_secure_file_priv(
        PREALLOCATED READ_ONLY GLOBAL_VAR(opt_secure_file_priv),
        CMD_LINE(REQUIRED_ARG), DEFAULT(0));
 
+static bool check_server_id(sys_var *self, THD *thd, set_var *var)
+{
+#ifdef WITH_WSREP
+  if (WSREP_ON && WSREP_PROVIDER_EXISTS && !wsrep_new_cluster && wsrep_gtid_mode)
+  {
+    push_warning(thd, Sql_condition::WARN_LEVEL_WARN,
+                 ER_WRONG_VALUE_FOR_VAR,
+                 "Can't change server_id because wsrep and wsrep_gtid_mode is set."
+                 " You can set server_id only with wsrep_new_cluster. ");
+    return true;
+  }
+#endif /* WITH_WSREP */
+  return false;
+}
+
 static bool fix_server_id(sys_var *self, THD *thd, enum_var_type type)
 {
   if (type == OPT_GLOBAL)
@@ -3247,7 +3262,7 @@ Sys_server_id(
        "replication partners",
        SESSION_VAR(server_id), CMD_LINE(REQUIRED_ARG, OPT_SERVER_ID),
        VALID_RANGE(1, UINT_MAX32), DEFAULT(1), BLOCK_SIZE(1), NO_MUTEX_GUARD,
-       NOT_IN_BINLOG, ON_CHECK(0), ON_UPDATE(fix_server_id));
+       NOT_IN_BINLOG, ON_CHECK(check_server_id), ON_UPDATE(fix_server_id));
 
 static Sys_var_on_access_global<Sys_var_mybool,
                           PRIV_SET_SYSTEM_GLOBAL_VAR_SLAVE_COMPRESSED_PROTOCOL>

sql/wsrep_mysqld.cc

-/* Copyright 2008-2021 Codership Oy <http://www.codership.com>
-   Copyright (c) 2020, 2021, MariaDB
+/* Copyright (c) 2008, 2022 Codership Oy <http://www.codership.com>
+   Copyright (c) 2020, 2022, MariaDB
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -802,7 +802,6 @@ int wsrep_init_server()
 
 void wsrep_init_globals()
 {
-  wsrep_gtid_server.domain_id= wsrep_gtid_domain_id;
   wsrep_init_sidno(Wsrep_server_state::instance().connected_gtid().id());
   /* Recover last written wsrep gtid */
   wsrep_init_gtid();
@@ -817,6 +816,13 @@ void wsrep_init_globals()
     wsrep_get_binlog_gtid_seqno(new_gtid);
     wsrep_gtid_server.gtid(new_gtid);
   }
+  else
+  {
+    if (wsrep_gtid_mode && wsrep_gtid_server.server_id != global_system_variables.server_id)
+    {
+      WSREP_WARN("Ignoring server id for non bootstrap node.");
+    }
+  }
   wsrep_init_schema();
   if (WSREP_ON)
   {
@@ -1143,6 +1149,15 @@ bool wsrep_start_replication(const char *wsrep_cluster_address)
 
   DBUG_ASSERT(wsrep_cluster_address[0]);
 
+  // --wsrep-new-cluster flag is not used, checking wsrep_cluster_address
+  // it should match gcomm:// only to be considered as bootstrap node.
+  // This logic is used in galera.
+  if (!wsrep_new_cluster && (strlen(wsrep_cluster_address) == 8) &&
+      !strncmp(wsrep_cluster_address, "gcomm://", 8))
+  {
+    wsrep_new_cluster= true;
+  }
+
   bool const bootstrap(TRUE == wsrep_new_cluster);
 
   WSREP_INFO("Start replication");