MDEV#6206: wsrep_slave_threads subtracts from max_connections

Decoupled wsrep thread count from connection count. By doing so,
the number of wsrep threads (applier/rollbacker) would no longer
affect the threads_connected status variable and thus maximum
allowable user connections limit would be @@max_connections.

Also introduced a new status variable 'wsrep_thread_count' to hold
the number of wsrep applier/rollbacker threads.

Added a test case.

mysql-test/include/mtr_check.sql

@@ -32,6 +32,7 @@ BEGIN
      AND variable_name not like "Last_IO_Err*"
      AND variable_name != 'INNODB_IBUF_MAX_SIZE'
      AND variable_name != 'INNODB_USE_NATIVE_AIO'
+     AND variable_name != 'WSREP_DATA_HOME_DIR'
    ORDER BY variable_name;
 
   -- Dump all databases, there should be none

mysql-test/suite/wsrep/r/variables.result

@@ -11,4 +11,69 @@ SET SESSION wsrep_replicate_myisam= ON;
 ERROR HY000: Variable 'wsrep_replicate_myisam' is a GLOBAL variable and should be set with SET GLOBAL
 SET GLOBAL wsrep_replicate_myisam= ON;
 SET GLOBAL wsrep_replicate_myisam= OFF;
+
+#
+# MDEV#6206: wsrep_slave_threads subtracts from max_connections
+#
+call mtr.add_suppression("safe_mutex: Found wrong usage of mutex 'LOCK_wsrep_slave_threads' and 'LOCK_global_system_variables'");
+call mtr.add_suppression("WSREP: Failed to get provider options");
+SELECT @@global.wsrep_provider;
+@@global.wsrep_provider
+none
+SELECT @@global.wsrep_slave_threads;
+@@global.wsrep_slave_threads
+1
+SELECT @@global.wsrep_cluster_address;
+@@global.wsrep_cluster_address
+
+SHOW STATUS LIKE 'threads_connected';
+Variable_name	Value
+Threads_connected	1
+SHOW STATUS LIKE 'wsrep_thread_count';
+Variable_name	Value
+wsrep_thread_count	0
+
+SET GLOBAL wsrep_provider= '/usr/lib/galera/libgalera_smm.so';
+SELECT @@global.wsrep_provider;
+@@global.wsrep_provider
+/usr/lib/galera/libgalera_smm.so
+SELECT @@global.wsrep_cluster_address;
+@@global.wsrep_cluster_address
+NULL
+SHOW STATUS LIKE 'threads_connected';
+Variable_name	Value
+Threads_connected	1
+SHOW STATUS LIKE 'wsrep_thread_count';
+Variable_name	Value
+wsrep_thread_count	0
+
+# Setting wsrep_cluster_address triggers the creation of
+# applier/rollbacker threads.
+SET GLOBAL wsrep_cluster_address= 'gcomm://';
+# Wait for applier threads to get created.
+SELECT @@global.wsrep_provider;
+@@global.wsrep_provider
+/usr/lib/galera/libgalera_smm.so
+SELECT @@global.wsrep_cluster_address;
+@@global.wsrep_cluster_address
+gcomm://
+SHOW STATUS LIKE 'threads_connected';
+Variable_name	Value
+Threads_connected	1
+SHOW STATUS LIKE 'wsrep_thread_count';
+Variable_name	Value
+wsrep_thread_count	2
+
+SET @wsrep_slave_threads_saved= @@global.wsrep_slave_threads;
+SET GLOBAL wsrep_slave_threads= 10;
+# Wait for applier threads to get created.
+SHOW STATUS LIKE 'threads_connected';
+Variable_name	Value
+Threads_connected	1
+SHOW STATUS LIKE 'wsrep_thread_count';
+Variable_name	Value
+wsrep_thread_count	11
+SET GLOBAL wsrep_slave_threads= @wsrep_slave_threads_saved;
+SET GLOBAL wsrep_cluster_address= '';
+SET GLOBAL wsrep_provider= 'none';
 # End of test.

mysql-test/suite/wsrep/t/variables.test

@@ -16,4 +16,48 @@ SET GLOBAL wsrep_replicate_myisam= ON;
 # Reset it back.
 SET GLOBAL wsrep_replicate_myisam= OFF;
 
+--echo
+--echo #
+--echo # MDEV#6206: wsrep_slave_threads subtracts from max_connections
+--echo #
+call mtr.add_suppression("safe_mutex: Found wrong usage of mutex 'LOCK_wsrep_slave_threads' and 'LOCK_global_system_variables'");
+call mtr.add_suppression("WSREP: Failed to get provider options");
+SELECT @@global.wsrep_provider;
+SELECT @@global.wsrep_slave_threads;
+SELECT @@global.wsrep_cluster_address;
+SHOW STATUS LIKE 'threads_connected';
+SHOW STATUS LIKE 'wsrep_thread_count';
+--echo
+
+eval SET GLOBAL wsrep_provider= '$WSREP_PROVIDER';
+SELECT @@global.wsrep_provider;
+SELECT @@global.wsrep_cluster_address;
+SHOW STATUS LIKE 'threads_connected';
+SHOW STATUS LIKE 'wsrep_thread_count';
+--echo
+
+--echo # Setting wsrep_cluster_address triggers the creation of
+--echo # applier/rollbacker threads.
+SET GLOBAL wsrep_cluster_address= 'gcomm://';
+--echo # Wait for applier threads to get created.
+sleep 3; 
+SELECT @@global.wsrep_provider;
+SELECT @@global.wsrep_cluster_address;
+SHOW STATUS LIKE 'threads_connected';
+SHOW STATUS LIKE 'wsrep_thread_count';
+--echo
+
+SET @wsrep_slave_threads_saved= @@global.wsrep_slave_threads;
+SET GLOBAL wsrep_slave_threads= 10;
+--echo # Wait for applier threads to get created.
+sleep 3; 
+SHOW STATUS LIKE 'threads_connected';
+SHOW STATUS LIKE 'wsrep_thread_count';
+
+# reset (for mtr internal checks)
+SET GLOBAL wsrep_slave_threads= @wsrep_slave_threads_saved;
+SET GLOBAL wsrep_cluster_address= '';
+SET GLOBAL wsrep_provider= 'none';
+
 --echo # End of test.
+

sql/mysqld.cc

@@ -2604,9 +2604,19 @@ void thd_cleanup(THD *thd)
 
 void dec_connection_count(THD *thd)
 {
-  mysql_mutex_lock(&LOCK_connection_count);
-  (*thd->scheduler->connection_count)--;
-  mysql_mutex_unlock(&LOCK_connection_count);
+#ifdef WITH_WSREP
+  /*
+    Do not decrement when its wsrep system thread. wsrep_applier is set for
+    applier as well as rollbacker threads.
+  */
+  if (!thd->wsrep_applier)
+#endif /* WITH_WSREP */
+  {
+    DBUG_ASSERT(*thd->scheduler->connection_count > 0);
+    mysql_mutex_lock(&LOCK_connection_count);
+    (*thd->scheduler->connection_count)--;
+    mysql_mutex_unlock(&LOCK_connection_count);
+  }
 }
 
 
@@ -4879,10 +4889,6 @@ pthread_handler_t start_wsrep_THD(void *arg)
   thd->set_time();
   thd->init_for_queries();
 
-  mysql_mutex_lock(&LOCK_connection_count);
-  ++connection_count;
-  mysql_mutex_unlock(&LOCK_connection_count);
-
   mysql_mutex_lock(&LOCK_thread_count);
   wsrep_running_threads++;
   mysql_cond_broadcast(&COND_thread_count);
@@ -7938,6 +7944,7 @@ SHOW_VAR status_vars[]= {
   {"wsrep_provider_name",      (char*) &wsrep_provider_name,     SHOW_CHAR_PTR},
   {"wsrep_provider_version",   (char*) &wsrep_provider_version,  SHOW_CHAR_PTR},
   {"wsrep_provider_vendor",    (char*) &wsrep_provider_vendor,   SHOW_CHAR_PTR},
+  {"wsrep_thread_count",       (char*) &wsrep_running_threads,   SHOW_LONG_NOFLUSH},
   {"wsrep",                    (char*) &wsrep_show_status,       SHOW_FUNC},
 #endif
   {NullS, NullS, SHOW_LONG}