MDEV-7914: spider/bg.ha, spider/bg.ha_part crash server sporadically in buildbot

The crash occurs when a thread that is closing its connection attempts to
access Spider transaction information when another thread has freed that memory
while processing Spider plugin deinit.  This occurs because Spider does not
adjust the plugin's reference count when it sets a transaction information
pointer for the plugin.

The fix I implemented changes the way Spider sets the transaction information
pointer to use thd_set_ha_data() so that Spider's plugin reference counter is
adjusted as well.

Author:
  Jacob Mathew.

Reviewer:
  Kentoku Shiba.

Merged From:
  bb-10.3-MDEV-7914

storage/spider/ha_spider.cc

@@ -725,7 +725,7 @@ int ha_spider::close()
     }
   }
 
-  if (!thd || !*thd_ha_data(thd, spider_hton_ptr))
+  if (!thd || !thd_get_ha_data(thd, spider_hton_ptr))
   {
     for (roop_count = 0; roop_count < (int) share->link_count; roop_count++)
       conns[roop_count] = NULL;

storage/spider/spd_malloc.h

@@ -19,7 +19,7 @@
 #define spider_bulk_malloc(A,B,C,...) \
   spider_bulk_alloc_mem(A,B,__func__,__FILE__,__LINE__,C,__VA_ARGS__)
 #define spider_current_trx \
-  (current_thd ? ((SPIDER_TRX *) *thd_ha_data(current_thd, spider_hton_ptr)) : NULL)
+  (current_thd ? ((SPIDER_TRX *) thd_get_ha_data(current_thd, spider_hton_ptr)) : NULL)
 
 #define init_calc_mem(A) init_mem_calc(A,__func__,__FILE__,__LINE__)
 

storage/spider/spd_table.cc

@@ -75,7 +75,7 @@ void destroy_thd(MYSQL_THD thd)
   delete thd;
 }
 #endif
-inline MYSQL_THD spider_create_thd(SPIDER_THREAD *thread)
+inline MYSQL_THD spider_create_sys_thd(SPIDER_THREAD *thread)
 {
   THD *thd = create_thd();
   if (thd)
@@ -86,10 +86,30 @@ inline MYSQL_THD spider_create_thd(SPIDER_THREAD *thread)
   }
   return thd;
 }
-inline void spider_destroy_thd(MYSQL_THD thd)
+inline void spider_destroy_sys_thd(MYSQL_THD thd)
 {
   destroy_thd(thd);
 }
+inline MYSQL_THD spider_create_thd()
+{
+  THD *thd;
+  my_thread_init();
+  if (!(thd = new THD(next_thread_id())))
+    my_thread_end();
+  else
+  {
+#ifdef HAVE_PSI_INTERFACE
+    mysql_thread_set_psi_id(thd->thread_id);
+#endif
+    thd->thread_stack = (char *) &thd;
+    thd->store_globals();
+  }
+  return thd;
+}
+inline void spider_destroy_thd(MYSQL_THD thd)
+{
+  delete thd;
+}
 
 #ifdef SPIDER_XID_USES_xid_cache_iterate
 #else
@@ -5714,6 +5734,8 @@ int spider_free_share(
 ) {
   DBUG_ENTER("spider_free_share");
   pthread_mutex_lock(&spider_tbl_mutex);
+  bool do_delete_thd = false;
+  THD *thd = current_thd;
   if (!--share->use_count)
   {
 #ifndef WITHOUT_SPIDER_BG_SEARCH
@@ -5735,8 +5757,16 @@ int spider_free_share(
       share->sts_init &&
       spider_param_store_last_sts(share->store_last_sts)
     ) {
+      if (!thd)
+      {
+        /* Create a thread for Spider system table update */
+        thd = spider_create_thd();
+        if (!thd)
+          DBUG_RETURN(HA_ERR_OUT_OF_MEM);
+        do_delete_thd = TRUE;
+      }
       spider_sys_insert_or_update_table_sts(
-        current_thd,
+        thd,
         share->lgtm_tblhnd_share->table_name,
         share->lgtm_tblhnd_share->table_name_length,
         &share->data_file_length,
@@ -5754,8 +5784,16 @@ int spider_free_share(
       share->crd_init &&
       spider_param_store_last_crd(share->store_last_crd)
     ) {
+      if (!thd)
+      {
+        /* Create a thread for Spider system table update */
+        thd = spider_create_thd();
+        if (!thd)
+          DBUG_RETURN(HA_ERR_OUT_OF_MEM);
+        do_delete_thd = TRUE;
+      }
       spider_sys_insert_or_update_table_crd(
-        current_thd,
+        thd,
         share->lgtm_tblhnd_share->table_name,
         share->lgtm_tblhnd_share->table_name_length,
         share->cardinality,
@@ -5777,6 +5815,8 @@ int spider_free_share(
     free_root(&share->mem_root, MYF(0));
     spider_free(spider_current_trx, share, MYF(0));
   }
+  if (do_delete_thd)
+    spider_destroy_thd(thd);
   pthread_mutex_unlock(&spider_tbl_mutex);
   DBUG_RETURN(0);
 }
@@ -6501,7 +6541,7 @@ int spider_close_connection(
   SPIDER_CONN *conn;
   SPIDER_TRX *trx;
   DBUG_ENTER("spider_close_connection");
-  if (!(trx = (SPIDER_TRX*) *thd_ha_data(thd, spider_hton_ptr)))
+  if (!(trx = (SPIDER_TRX*) thd_get_ha_data(thd, spider_hton_ptr)))
     DBUG_RETURN(0); /* transaction is not started */
 
   trx->tmp_spider->conns = &conn;
@@ -6556,6 +6596,7 @@ int spider_db_done(
   void *p
 ) {
   int roop_count;
+  bool do_delete_thd;
   THD *thd = current_thd, *tmp_thd;
   SPIDER_CONN *conn;
   SPIDER_INIT_ERROR_TABLE *spider_init_error_table;
@@ -6563,6 +6604,18 @@ int spider_db_done(
   SPIDER_LGTM_TBLHND_SHARE *lgtm_tblhnd_share;
   DBUG_ENTER("spider_db_done");
 
+  /* Begin Spider plugin deinit */
+  if (thd)
+    do_delete_thd = FALSE;
+  else
+  {
+    /* Create a thread for Spider plugin deinit */
+    thd = spider_create_thd();
+    if (!thd)
+      DBUG_RETURN(HA_ERR_OUT_OF_MEM);
+    do_delete_thd = TRUE;
+  }
+
 #ifndef WITHOUT_SPIDER_BG_SEARCH
   spider_free_trx(spider_global_trx, TRUE);
 #endif
@@ -6619,21 +6672,22 @@ int spider_db_done(
     pthread_mutex_destroy(&spider_udf_table_mon_mutexes[roop_count]);
   spider_free(NULL, spider_udf_table_mon_mutexes, MYF(0));
 
-  if (thd && thd_sql_command(thd) == SQLCOM_UNINSTALL_PLUGIN) {
-    pthread_mutex_lock(&spider_allocated_thds_mutex);
-    while ((tmp_thd = (THD *) my_hash_element(&spider_allocated_thds, 0)))
+  pthread_mutex_lock(&spider_allocated_thds_mutex);
+  while ((tmp_thd = (THD *) my_hash_element(&spider_allocated_thds, 0)))
+  {
+    SPIDER_TRX *trx = (SPIDER_TRX *)
+                      thd_get_ha_data(tmp_thd, spider_hton_ptr);
+    if (trx)
     {
-      SPIDER_TRX *trx = (SPIDER_TRX *) *thd_ha_data(tmp_thd, spider_hton_ptr);
-      if (trx)
-      {
-        DBUG_ASSERT(tmp_thd == trx->thd);
-        spider_free_trx(trx, FALSE);
-        *thd_ha_data(tmp_thd, spider_hton_ptr) = (void *) NULL;
-      } else
-        my_hash_delete(&spider_allocated_thds, (uchar *) tmp_thd);
+      DBUG_ASSERT(tmp_thd == trx->thd);
+      spider_free_trx(trx, FALSE);
+      thd_set_ha_data(tmp_thd, spider_hton_ptr, NULL);
     }
-    pthread_mutex_unlock(&spider_allocated_thds_mutex);
+    else
+      my_hash_delete(&spider_allocated_thds, (uchar *) tmp_thd);
   }
+  pthread_mutex_unlock(&spider_allocated_thds_mutex);
+
 #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET)
   pthread_mutex_lock(&spider_hs_w_conn_mutex);
   while ((conn = (SPIDER_CONN*) my_hash_element(&spider_hs_w_conn_hash, 0)))
@@ -6784,6 +6838,11 @@ int spider_db_done(
         spider_current_alloc_mem[roop_count] ? "NG" : "OK"
       ));
   }
+
+  /* End Spider plugin deinit */
+  if (do_delete_thd)
+    spider_destroy_thd(thd);
+
 /*
 DBUG_ASSERT(0);
 */
@@ -9865,7 +9924,7 @@ void *spider_table_bg_sts_action(
   DBUG_ENTER("spider_table_bg_sts_action");
   /* init start */
   pthread_mutex_lock(&thread->mutex);
-  if (!(thd = spider_create_thd(thread)))
+  if (!(thd = spider_create_sys_thd(thread)))
   {
     thread->thd_wait = FALSE;
     thread->killed = FALSE;
@@ -9880,7 +9939,7 @@ void *spider_table_bg_sts_action(
   thd_proc_info(thd, "Spider table background statistics action handler");
   if (!(trx = spider_get_trx(NULL, FALSE, &error_num)))
   {
-    spider_destroy_thd(thd);
+    spider_destroy_sys_thd(thd);
     thread->thd_wait = FALSE;
     thread->killed = FALSE;
     pthread_mutex_unlock(&thread->mutex);
@@ -9901,7 +9960,7 @@ void *spider_table_bg_sts_action(
       DBUG_PRINT("info",("spider bg sts kill start"));
       trx->thd = NULL;
       spider_free_trx(trx, TRUE);
-      spider_destroy_thd(thd);
+      spider_destroy_sys_thd(thd);
       pthread_cond_signal(&thread->sync_cond);
       pthread_mutex_unlock(&thread->mutex);
 #if !defined(MYSQL_DYNAMIC_PLUGIN) || !defined(_WIN32)
@@ -10017,7 +10076,7 @@ void *spider_table_bg_crd_action(
   DBUG_ENTER("spider_table_bg_crd_action");
   /* init start */
   pthread_mutex_lock(&thread->mutex);
-  if (!(thd = spider_create_thd(thread)))
+  if (!(thd = spider_create_sys_thd(thread)))
   {
     thread->thd_wait = FALSE;
     thread->killed = FALSE;
@@ -10032,7 +10091,7 @@ void *spider_table_bg_crd_action(
   thd_proc_info(thd, "Spider table background cardinality action handler");
   if (!(trx = spider_get_trx(NULL, FALSE, &error_num)))
   {
-    spider_destroy_thd(thd);
+    spider_destroy_sys_thd(thd);
     thread->thd_wait = FALSE;
     thread->killed = FALSE;
     pthread_mutex_unlock(&thread->mutex);
@@ -10053,7 +10112,7 @@ void *spider_table_bg_crd_action(
       DBUG_PRINT("info",("spider bg crd kill start"));
       trx->thd = NULL;
       spider_free_trx(trx, TRUE);
-      spider_destroy_thd(thd);
+      spider_destroy_sys_thd(thd);
       pthread_cond_signal(&thread->sync_cond);
       pthread_mutex_unlock(&thread->mutex);
 #if !defined(MYSQL_DYNAMIC_PLUGIN) || !defined(_WIN32)

storage/spider/spd_trx.cc

@@ -1195,7 +1195,7 @@ SPIDER_TRX *spider_get_trx(
 
   if (
     !thd ||
-    !(trx = (SPIDER_TRX*) *thd_ha_data(thd, spider_hton_ptr))
+    !(trx = (SPIDER_TRX*) thd_get_ha_data(thd, spider_hton_ptr))
   ) {
     DBUG_PRINT("info",("spider create new trx"));
     if (!(trx = (SPIDER_TRX *)
@@ -1233,7 +1233,7 @@ SPIDER_TRX *spider_get_trx(
       goto error_init_hash;
     spider_alloc_calc_mem_init(trx->trx_conn_hash, 151);
     spider_alloc_calc_mem(
-      thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+      thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
       trx->trx_conn_hash,
       trx->trx_conn_hash.array.max_element *
       trx->trx_conn_hash.array.size_of_element);
@@ -1245,7 +1245,7 @@ SPIDER_TRX *spider_get_trx(
       goto error_init_another_hash;
     spider_alloc_calc_mem_init(trx->trx_another_conn_hash, 152);
     spider_alloc_calc_mem(
-      thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+      thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
       trx->trx_another_conn_hash,
       trx->trx_another_conn_hash.array.max_element *
       trx->trx_another_conn_hash.array.size_of_element);
@@ -1258,7 +1258,7 @@ SPIDER_TRX *spider_get_trx(
       goto error_hs_r_init_hash;
     spider_alloc_calc_mem_init(trx->trx_hs_r_conn_hash, 153);
     spider_alloc_calc_mem(
-      thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+      thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
       trx->trx_hs_r_conn_hash,
       trx->trx_hs_r_conn_hash.array.max_element *
       trx->trx_hs_r_conn_hash.array.size_of_element);
@@ -1270,7 +1270,7 @@ SPIDER_TRX *spider_get_trx(
       goto error_hs_w_init_hash;
     spider_alloc_calc_mem_init(trx->trx_hs_w_conn_hash, 154);
     spider_alloc_calc_mem(
-      thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+      thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
       trx->trx_hs_w_conn_hash,
       trx->trx_hs_w_conn_hash.array.max_element *
       trx->trx_hs_w_conn_hash.array.size_of_element);
@@ -1284,7 +1284,7 @@ SPIDER_TRX *spider_get_trx(
       goto error_direct_hs_r_init_hash;
     spider_alloc_calc_mem_init(trx->trx_direct_hs_r_conn_hash, 155);
     spider_alloc_calc_mem(
-      thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+      thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
       trx->trx_direct_hs_r_conn_hash,
       trx->trx_direct_hs_r_conn_hash.array.max_element *
       trx->trx_direct_hs_r_conn_hash.array.size_of_element);
@@ -1296,7 +1296,7 @@ SPIDER_TRX *spider_get_trx(
       goto error_direct_hs_w_init_hash;
     spider_alloc_calc_mem_init(trx->trx_direct_hs_w_conn_hash, 156);
     spider_alloc_calc_mem(
-      thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+      thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
       trx->trx_direct_hs_w_conn_hash,
       trx->trx_direct_hs_w_conn_hash.array.max_element *
       trx->trx_direct_hs_w_conn_hash.array.size_of_element);
@@ -1309,7 +1309,7 @@ SPIDER_TRX *spider_get_trx(
       goto error_init_alter_hash;
     spider_alloc_calc_mem_init(trx->trx_alter_table_hash, 157);
     spider_alloc_calc_mem(
-      thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+      thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
       trx->trx_alter_table_hash,
       trx->trx_alter_table_hash.array.max_element *
       trx->trx_alter_table_hash.array.size_of_element);
@@ -1321,7 +1321,7 @@ SPIDER_TRX *spider_get_trx(
       goto error_init_trx_ha_hash;
     spider_alloc_calc_mem_init(trx->trx_ha_hash, 158);
     spider_alloc_calc_mem(
-      thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+      thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
       trx->trx_ha_hash,
       trx->trx_ha_hash.array.max_element *
       trx->trx_ha_hash.array.size_of_element);
@@ -1443,7 +1443,7 @@ SPIDER_TRX *spider_get_trx(
         pthread_mutex_unlock(&spider_allocated_thds_mutex);
         trx->registed_allocated_thds = TRUE;
       }
-      *thd_ha_data(thd, spider_hton_ptr) = (void *) trx;
+      thd_set_ha_data(thd, spider_hton_ptr, trx);
     }
   }
 
@@ -1489,7 +1489,7 @@ SPIDER_TRX *spider_get_trx(
   my_hash_free(&trx->trx_ha_hash);
 error_init_trx_ha_hash:
   spider_free_mem_calc(
-    thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+    thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
     trx->trx_alter_table_hash_id,
     trx->trx_alter_table_hash.array.max_element *
     trx->trx_alter_table_hash.array.size_of_element);
@@ -1497,14 +1497,14 @@ SPIDER_TRX *spider_get_trx(
 error_init_alter_hash:
 #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET)
   spider_free_mem_calc(
-    thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+    thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
     trx->trx_direct_hs_w_conn_hash_id,
     trx->trx_direct_hs_w_conn_hash.array.max_element *
     trx->trx_direct_hs_w_conn_hash.array.size_of_element);
   my_hash_free(&trx->trx_direct_hs_w_conn_hash);
 error_direct_hs_w_init_hash:
   spider_free_mem_calc(
-    thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+    thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
     trx->trx_direct_hs_r_conn_hash_id,
     trx->trx_direct_hs_r_conn_hash.array.max_element *
     trx->trx_direct_hs_r_conn_hash.array.size_of_element);
@@ -1513,14 +1513,14 @@ SPIDER_TRX *spider_get_trx(
 #endif
 #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET)
   spider_free_mem_calc(
-    thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+    thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
     trx->trx_hs_w_conn_hash_id,
     trx->trx_hs_w_conn_hash.array.max_element *
     trx->trx_hs_w_conn_hash.array.size_of_element);
   my_hash_free(&trx->trx_hs_w_conn_hash);
 error_hs_w_init_hash:
   spider_free_mem_calc(
-    thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+    thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
     trx->trx_hs_r_conn_hash_id,
     trx->trx_hs_r_conn_hash.array.max_element *
     trx->trx_hs_r_conn_hash.array.size_of_element);
@@ -1528,14 +1528,14 @@ SPIDER_TRX *spider_get_trx(
 error_hs_r_init_hash:
 #endif
   spider_free_mem_calc(
-    thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+    thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
     trx->trx_another_conn_hash_id,
     trx->trx_another_conn_hash.array.max_element *
     trx->trx_another_conn_hash.array.size_of_element);
   my_hash_free(&trx->trx_another_conn_hash);
 error_init_another_hash:
   spider_free_mem_calc(
-    thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+    thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
     trx->trx_conn_hash_id,
     trx->trx_conn_hash.array.max_element *
     trx->trx_conn_hash.array.size_of_element);
@@ -1574,7 +1574,7 @@ int spider_free_trx(
       if (need_lock)
         pthread_mutex_unlock(&spider_allocated_thds_mutex);
     }
-    *thd_ha_data(trx->thd, spider_hton_ptr) = (void *) NULL;
+    thd_set_ha_data(trx->thd, spider_hton_ptr, NULL);
   }
   spider_free_trx_alloc(trx);
   spider_merge_mem_calc(trx, TRUE);
@@ -3374,7 +3374,7 @@ int spider_commit(
   SPIDER_CONN *conn;
   DBUG_ENTER("spider_commit");
 
-  if (!(trx = (SPIDER_TRX*) *thd_ha_data(thd, spider_hton_ptr)))
+  if (!(trx = (SPIDER_TRX*) thd_get_ha_data(thd, spider_hton_ptr)))
     DBUG_RETURN(0); /* transaction is not started */
 
 #ifdef HA_CAN_BULK_ACCESS
@@ -3466,7 +3466,7 @@ int spider_rollback(
   SPIDER_CONN *conn;
   DBUG_ENTER("spider_rollback");
 
-  if (!(trx = (SPIDER_TRX*) *thd_ha_data(thd, spider_hton_ptr)))
+  if (!(trx = (SPIDER_TRX*) thd_get_ha_data(thd, spider_hton_ptr)))
     DBUG_RETURN(0); /* transaction is not started */
 
 #ifdef HA_CAN_BULK_ACCESS
@@ -3543,7 +3543,7 @@ int spider_xa_prepare(
 
   if (all || (!thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)))
   {
-    if (!(trx = (SPIDER_TRX*) *thd_ha_data(thd, spider_hton_ptr)))
+    if (!(trx = (SPIDER_TRX*) thd_get_ha_data(thd, spider_hton_ptr)))
       DBUG_RETURN(0); /* transaction is not started */
 
     DBUG_PRINT("info",("spider trx_start=%s",