MDEV-19313 Threadpool : provide information schema tables for internals of generic threadpool

Added thread_pool_groups, thread_pool_queues, thread_pool_waits and
thread_pool_stats tables to information_schema.

sql/CMakeLists.txt

@@ -164,6 +164,7 @@ IF ((CMAKE_SYSTEM_NAME MATCHES "Linux" OR
  ENDIF()
  SET(SQL_SOURCE ${SQL_SOURCE} threadpool_generic.cc)
  SET(SQL_SOURCE ${SQL_SOURCE} threadpool_common.cc)
+ MYSQL_ADD_PLUGIN(thread_pool_info thread_pool_info.cc DEFAULT STATIC_ONLY NOT_EMBEDDED)
 ENDIF()
 
 IF(WIN32)

sql/mysqld.cc

@@ -7461,7 +7461,7 @@ static int debug_status_func(THD *thd, SHOW_VAR *var, char *buff,
 #endif
 
 #ifdef HAVE_POOL_OF_THREADS
-int show_threadpool_idle_threads(THD *thd, SHOW_VAR *var, char *buff,
+static int show_threadpool_idle_threads(THD *thd, SHOW_VAR *var, char *buff,
                                  enum enum_var_type scope)
 {
   var->type= SHOW_INT;

sql/thread_pool_info.cc

+/* Copyright(C) 2019 MariaDB
+
+This program is free software; you can redistribute itand /or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111 - 1301 USA*/
+
+#include <mysql_version.h>
+#include <mysql/plugin.h>
+
+#include <my_global.h>
+#include <sql_class.h>
+#include <table.h>
+#include <mysql/plugin.h>
+#include <sql_show.h>
+#include <threadpool_generic.h>
+
+static ST_FIELD_INFO groups_fields_info[] =
+{
+  {"GROUP_ID", 6, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+  {"CONNECTIONS", 6, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+  {"THREADS", 6, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+  {"ACTIVE_THREADS",6, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+  {"STANDBY_THREADS", 6, MYSQL_TYPE_LONG, 0, 0, 0,0},
+  {"QUEUE_LENGTH", 6, MYSQL_TYPE_LONG, 0,0, 0, 0},
+  {"HAS_LISTENER",1,MYSQL_TYPE_TINY, 0, 0, 0, 0},
+  {"IS_STALLED",1,MYSQL_TYPE_TINY, 0, 0, 0, 0},
+  {0, 0, MYSQL_TYPE_STRING, 0, 0, 0, 0}
+};
+
+static int groups_fill_table(THD* thd, TABLE_LIST* tables, COND*)
+{
+  if (!all_groups)
+    return 0;
+
+  TABLE* table = tables->table;
+  for (uint i = 0; i < threadpool_max_size && all_groups[i].pollfd != INVALID_HANDLE_VALUE; i++)
+  {
+    thread_group_t* group = &all_groups[i];
+    /* ID */
+    table->field[0]->store(i, true);
+    /* CONNECTION_COUNT */
+    table->field[1]->store(group->connection_count, true);
+    /* THREAD_COUNT */
+    table->field[2]->store(group->thread_count, true);
+    /* ACTIVE_THREAD_COUNT */
+    table->field[3]->store(group->active_thread_count, true);
+    /* STANDBY_THREAD_COUNT */
+    table->field[4]->store(group->waiting_threads.elements(), true);
+    /* QUEUE LENGTH */
+    uint queue_len = group->queues[TP_PRIORITY_LOW].elements()
+      + group->queues[TP_PRIORITY_HIGH].elements();
+    table->field[5]->store(queue_len, true);
+    /* HAS_LISTENER */
+    table->field[6]->store((longlong)(group->listener != 0), true);
+    /* IS_STALLED */
+    table->field[7]->store(group->stalled, true);
+
+    if (schema_table_store_record(thd, table))
+      return 1;
+  }
+  return 0;
+}
+
+
+static int groups_init(void* p)
+{
+  ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*)p;
+  schema->fields_info = groups_fields_info;
+  schema->fill_table = groups_fill_table;
+  return 0;
+}
+
+
+static ST_FIELD_INFO queues_field_info[] =
+{
+  {"GROUP_ID", 6, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+  {"POSITION",6,MYSQL_TYPE_LONG, 0, 0, 0, 0},
+  {"PRIORITY", 1, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+  {"CONNECTION_ID", 19, MYSQL_TYPE_LONGLONG, MY_I_S_UNSIGNED, 0, 0, 0},
+  {"QUEUEING_TIME_MICROSECONDS", 19, MYSQL_TYPE_LONGLONG, 0, 0, 0, 0},
+  {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0}
+};
+
+typedef connection_queue_t::Iterator connection_queue_iterator;
+
+static int queues_fill_table(THD* thd, TABLE_LIST* tables, COND*)
+{
+  if (!all_groups)
+    return 0;
+
+  TABLE* table = tables->table;
+  for (uint group_id = 0;
+    group_id < threadpool_max_size && all_groups[group_id].pollfd != INVALID_HANDLE_VALUE;
+    group_id++)
+  {
+    thread_group_t* group = &all_groups[group_id];
+
+    mysql_mutex_lock(&group->mutex);
+    bool err = false;
+    int pos = 0;
+    ulonglong now = microsecond_interval_timer();
+    for (uint prio = 0; prio < NQUEUES && !err; prio++)
+    {
+      connection_queue_iterator it(group->queues[prio]);
+      TP_connection_generic* c;
+      while ((c = it++) != 0)
+      {
+        /* GROUP_ID */
+        table->field[0]->store(group_id, true);
+        /* POSITION */
+        table->field[1]->store(pos++, true);
+        /* PRIORITY */
+        table->field[2]->store(prio, true);
+        /* CONNECTION_ID */
+        table->field[3]->store(c->thd->thread_id, true);
+        /* QUEUEING_TIME */
+        table->field[4]->store(now - c->enqueue_time, true);
+
+        err = schema_table_store_record(thd, table);
+        if (err)
+          break;
+      }
+    }
+    mysql_mutex_unlock(&group->mutex);
+    if (err)
+      return 1;
+  }
+  return 0;
+}
+
+static int queues_init(void* p)
+{
+  ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*)p;
+  schema->fields_info = queues_field_info;
+  schema->fill_table = queues_fill_table;
+  return 0;
+}
+
+static ST_FIELD_INFO stats_fields_info[] =
+{
+  {"GROUP_ID", 6, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+  {"THREAD_CREATIONS",19,MYSQL_TYPE_LONGLONG,0,0, 0,0},
+  {"THREAD_CREATIONS_DUE_TO_STALL",19,MYSQL_TYPE_LONGLONG,0,0, 0,0},
+  {"WAKES",19,MYSQL_TYPE_LONGLONG,0,0, 0,0},
+  {"WAKES_DUE_TO_STALL",19,MYSQL_TYPE_LONGLONG,0,0, 0,0},
+  {"THROTTLES",19,MYSQL_TYPE_LONGLONG,0,0, 0,0},
+  {"STALLS",19,MYSQL_TYPE_LONGLONG,0,0, 0, 0},
+  {"POLLS_BY_LISTENER",19,MYSQL_TYPE_LONGLONG,0,0, 0, 0},
+  {"POLLS_BY_WORKER",19,MYSQL_TYPE_LONGLONG,0,0, 0, 0},
+  {"DEQUEUES_BY_LISTENER",19,MYSQL_TYPE_LONGLONG,0,0, 0, 0},
+  {"DEQUEUES_BY_WORKER",19,MYSQL_TYPE_LONGLONG,0,0, 0, 0},
+  {0, 0, MYSQL_TYPE_STRING, 0, 0, 0, 0}
+};
+
+static int stats_fill_table(THD* thd, TABLE_LIST* tables, COND*)
+{
+  if (!all_groups)
+    return 0;
+
+  TABLE* table = tables->table;
+  for (uint i = 0; i < threadpool_max_size && all_groups[i].pollfd != INVALID_HANDLE_VALUE; i++)
+  {
+    table->field[0]->store(i, true);
+    thread_group_t* group = &all_groups[i];
+
+    mysql_mutex_lock(&group->mutex);
+    thread_group_counters_t* counters = &group->counters;
+    table->field[1]->store(counters->thread_creations, true);
+    table->field[2]->store(counters->thread_creations_due_to_stall, true);
+    table->field[3]->store(counters->wakes, true);
+    table->field[4]->store(counters->wakes_due_to_stall, true);
+    table->field[5]->store(counters->throttles, true);
+    table->field[6]->store(counters->stalls, true);
+    table->field[7]->store(counters->polls_by_listener, true);
+    table->field[8]->store(counters->polls_by_worker, true);
+    table->field[9]->store(counters->dequeues_by_listener, true);
+    table->field[10]->store(counters->dequeues_by_worker, true);
+    mysql_mutex_unlock(&group->mutex);
+    if (schema_table_store_record(thd, table))
+      return 1;
+  }
+  return 0;
+}
+
+static int stats_reset_table()
+{
+  for (uint i = 0; i < threadpool_max_size && all_groups[i].pollfd != INVALID_HANDLE_VALUE; i++)
+  {
+    thread_group_t* group = &all_groups[i];
+    mysql_mutex_lock(&group->mutex);
+    memset(&group->counters, 0, sizeof(group->counters));
+    mysql_mutex_unlock(&group->mutex);
+  }
+  return 0;
+}
+
+static int stats_init(void* p)
+{
+  ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*)p;
+  schema->fields_info = stats_fields_info;
+  schema->fill_table = stats_fill_table;
+  schema->reset_table = stats_reset_table;
+  return 0;
+}
+
+
+
+static ST_FIELD_INFO waits_fields_info[] =
+{
+  {"REASON", 16, MYSQL_TYPE_STRING, 0, 0, 0, 0},
+  {"COUNT",19,MYSQL_TYPE_LONGLONG,0,0, 0,0},
+  {0, 0, MYSQL_TYPE_STRING, 0, 0, 0, 0}
+};
+
+/* See thd_wait_type enum for explanation*/
+static const LEX_CSTRING wait_reasons[THD_WAIT_LAST] =
+{
+  {STRING_WITH_LEN("UNKNOWN")},
+  {STRING_WITH_LEN("SLEEP")},
+  {STRING_WITH_LEN("DISKIO")},
+  {STRING_WITH_LEN("ROW_LOCK")},
+  {STRING_WITH_LEN("GLOBAL_LOCK")},
+  {STRING_WITH_LEN("META_DATA_LOCK")},
+  {STRING_WITH_LEN("TABLE_LOCK")},
+  {STRING_WITH_LEN("USER_LOCK")},
+  {STRING_WITH_LEN("BINLOG")},
+  {STRING_WITH_LEN("GROUP_COMMIT")},
+  {STRING_WITH_LEN("SYNC")},
+  {STRING_WITH_LEN("NET")}
+};
+
+extern Atomic_counter<unsigned long long> tp_waits[THD_WAIT_LAST];
+
+static int waits_fill_table(THD* thd, TABLE_LIST* tables, COND*)
+{
+  if (!all_groups)
+    return 0;
+
+  TABLE* table = tables->table;
+  for (auto i = 0; i < THD_WAIT_LAST; i++)
+  {
+    table->field[0]->store(wait_reasons[i].str, wait_reasons[i].length, system_charset_info);
+    table->field[1]->store(tp_waits[i], true);
+    if (schema_table_store_record(thd, table))
+      return 1;
+  }
+  return 0;
+}
+
+static int waits_reset_table()
+{
+  for (auto i = 0; i < THD_WAIT_LAST; i++)
+    tp_waits[i] = 0;
+
+  return 0;
+}
+
+static int waits_init(void* p)
+{
+  ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*)p;
+  schema->fields_info = waits_fields_info;
+  schema->fill_table = waits_fill_table;
+  schema->reset_table = waits_reset_table;
+  return 0;
+}
+
+static struct st_mysql_information_schema plugin_descriptor =
+{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION };
+
+maria_declare_plugin(thread_pool_info)
+{
+  MYSQL_INFORMATION_SCHEMA_PLUGIN,
+  &plugin_descriptor,
+  "THREAD_POOL_GROUPS",
+  "Vladislav Vaintroub",
+  "Provides information about threadpool groups.",
+  PLUGIN_LICENSE_GPL,
+  groups_init,
+  0,
+  0x0100,
+  NULL,
+  NULL,
+  "1.0",
+  MariaDB_PLUGIN_MATURITY_STABLE
+},
+{
+  MYSQL_INFORMATION_SCHEMA_PLUGIN,
+  &plugin_descriptor,
+  "THREAD_POOL_QUEUES",
+  "Vladislav Vaintroub",
+  "Provides information about threadpool queues.",
+  PLUGIN_LICENSE_GPL,
+  queues_init,
+  0,
+  0x0100,
+  NULL,
+  NULL,
+  "1.0",
+  MariaDB_PLUGIN_MATURITY_STABLE
+},
+{
+  MYSQL_INFORMATION_SCHEMA_PLUGIN,
+  &plugin_descriptor,
+  "THREAD_POOL_STATS",
+  "Vladislav Vaintroub",
+  "Provides performance counter information for threadpool.",
+  PLUGIN_LICENSE_GPL,
+  stats_init,
+  0,
+  0x0100,
+  NULL,
+  NULL,
+  "1.0",
+  MariaDB_PLUGIN_MATURITY_STABLE
+},
+{
+  MYSQL_INFORMATION_SCHEMA_PLUGIN,
+  &plugin_descriptor,
+  "THREAD_POOL_WAITS",
+  "Vladislav Vaintroub",
+  "Provides wait counters for threadpool.",
+  PLUGIN_LICENSE_GPL,
+  waits_init,
+  0,
+  0x0100,
+  NULL,
+  NULL,
+  "1.0",
+  MariaDB_PLUGIN_MATURITY_STABLE
+}
+maria_declare_plugin_end;

sql/threadpool.h

@@ -64,8 +64,6 @@ extern int tp_get_thread_count();
 /* Activate threadpool scheduler */
 extern void tp_scheduler(void);
 
-extern int show_threadpool_idle_threads(THD *thd, SHOW_VAR *var, char *buff,
-                                        enum enum_var_type scope);
 
 enum  TP_PRIORITY {
   TP_PRIORITY_HIGH,
@@ -88,6 +86,8 @@ enum TP_STATE
   inside threadpool_win.cc and threadpool_unix.cc
 */
 
+class CONNECT;
+
 struct TP_connection
 {
   THD*        thd;

sql/threadpool_common.cc

@@ -23,7 +23,7 @@
 #include <sql_audit.h>
 #include <debug_sync.h>
 #include <threadpool.h>
-
+#include <my_counter.h>
 
 /* Threadpool parameters */
 
@@ -153,9 +153,8 @@ static TP_PRIORITY get_priority(TP_connection *c)
   DBUG_ASSERT(c->thd == current_thd);
   TP_PRIORITY prio= (TP_PRIORITY)c->thd->variables.threadpool_priority;
   if (prio == TP_PRIORITY_AUTO)
-  {
-    return c->thd->transaction.is_active() ? TP_PRIORITY_HIGH : TP_PRIORITY_LOW;
-  }
+    prio= c->thd->transaction.is_active() ? TP_PRIORITY_HIGH : TP_PRIORITY_LOW;
+
   return prio;
 }
 
@@ -463,12 +462,17 @@ void tp_timeout_handler(TP_connection *c)
   mysql_mutex_unlock(&thd->LOCK_thd_kill);
 }
 
+MY_ALIGNED(CPU_LEVEL1_DCACHE_LINESIZE) Atomic_counter<unsigned long long> tp_waits[THD_WAIT_LAST];
 
 static void tp_wait_begin(THD *thd, int type)
 {
   TP_connection *c = get_TP_connection(thd);
   if (c)
+  {
+    DBUG_ASSERT(type > 0 && type < THD_WAIT_LAST);
+    tp_waits[type]++;
     c->wait_begin(type);
+  }
 }
 
 

sql/threadpool_generic.h

+/* Copyright(C) 2019 MariaDB
+ *
+ * This program is free software; you can redistribute itand /or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111 - 1301 USA*/
+
+#if defined (HAVE_POOL_OF_THREADS)
+#include <my_global.h>
+#include <sql_plist.h>
+#include <my_pthread.h>
+#include <threadpool.h>
+#include <mysqld.h>
+#include <violite.h>
+
+#ifdef _WIN32
+#include <windows.h>
+/* AIX may define this, too ?*/
+#define HAVE_IOCP 
+#endif
+
+
+#ifdef _WIN32
+typedef HANDLE TP_file_handle;
+#else
+typedef int TP_file_handle;
+#define  INVALID_HANDLE_VALUE -1
+#endif
+
+#ifdef __linux__
+#include <sys/epoll.h>
+typedef struct epoll_event native_event;
+#elif defined(HAVE_KQUEUE)
+#include <sys/event.h>
+typedef struct kevent native_event;
+#elif defined (__sun)
+#include <port.h>
+typedef port_event_t native_event;
+#elif defined (HAVE_IOCP)
+typedef OVERLAPPED_ENTRY native_event;
+#else
+#error threadpool is not available on this platform
+#endif
+
+struct thread_group_t;
+
+/* Per-thread structure for workers */
+struct worker_thread_t
+{
+  ulonglong  event_count; /* number of request handled by this thread */
+  thread_group_t* thread_group;
+  worker_thread_t* next_in_list;
+  worker_thread_t** prev_in_list;
+  mysql_cond_t  cond;
+  bool          woken;
+};
+
+typedef I_P_List<worker_thread_t, I_P_List_adapter<worker_thread_t,
+  & worker_thread_t::next_in_list,
+  & worker_thread_t::prev_in_list>,
+  I_P_List_counter
+>
+worker_list_t;
+
+struct TP_connection_generic :public TP_connection
+{
+  TP_connection_generic(CONNECT* c);
+  ~TP_connection_generic();
+
+  virtual int init() { return 0; };
+  virtual void set_io_timeout(int sec);
+  virtual int  start_io();
+  virtual void wait_begin(int type);
+  virtual void wait_end();
+
+  thread_group_t* thread_group;
+  TP_connection_generic* next_in_queue;
+  TP_connection_generic** prev_in_queue;
+  ulonglong abs_wait_timeout;
+  ulonglong enqueue_time;
+  TP_file_handle fd;
+  bool bound_to_poll_descriptor;
+  int waiting;
+#ifdef HAVE_IOCP
+  OVERLAPPED overlapped;
+#endif
+#ifdef _WIN32
+  enum_vio_type vio_type;
+#endif
+};
+
+
+typedef I_P_List<TP_connection_generic,
+  I_P_List_adapter<TP_connection_generic,
+  & TP_connection_generic::next_in_queue,
+  & TP_connection_generic::prev_in_queue>,
+  I_P_List_counter,
+  I_P_List_fast_push_back<TP_connection_generic> >
+  connection_queue_t;
+
+const int NQUEUES = 2; /* We have high and low priority queues*/
+
+struct thread_group_counters_t
+{
+  ulonglong thread_creations;
+  ulonglong thread_creations_due_to_stall;
+  ulonglong wakes;
+  ulonglong wakes_due_to_stall;
+  ulonglong throttles;
+  ulonglong stalls;
+  ulonglong dequeues_by_worker;
+  ulonglong dequeues_by_listener;
+  ulonglong polls_by_listener;
+  ulonglong polls_by_worker;
+};
+
+struct MY_ALIGNED(CPU_LEVEL1_DCACHE_LINESIZE) thread_group_t
+{
+  mysql_mutex_t mutex;
+  connection_queue_t queues[NQUEUES];
+  worker_list_t waiting_threads;
+  worker_thread_t* listener;
+  pthread_attr_t* pthread_attr;
+  TP_file_handle  pollfd;
+  int  thread_count;
+  int  active_thread_count;
+  int  connection_count;
+  /* Stats for the deadlock detection timer routine.*/
+  int io_event_count;
+  int queue_event_count;
+  ulonglong last_thread_creation_time;
+  int  shutdown_pipe[2];
+  bool shutdown;
+  bool stalled;
+  thread_group_counters_t counters;
+};
+
+#define TP_INCREMENT_GROUP_COUNTER(group,var) group->counters.var++;
+
+extern thread_group_t* all_groups;
+#endif
+