MDEV-7914 spider/bg.ha, spider/bg.ha_part fail sporadically in buildbot

Fixed the problem by adding a Spider shutdown indicator and a Spider memory
lock.  Spider shutdown acquires the lock for write access and
all other requestors acquire the lock for read access.

storage/spider/ha_spider.cc

@@ -55,6 +55,10 @@
 #define SPIDER_CAN_BG_UPDATE (LL(1) << 39)
 #endif
 
+extern bool is_spider_shutdown();
+extern int  spider_memory_rdlock();
+extern int  spider_memory_unlock();
+
 extern handlerton *spider_hton_ptr;
 extern SPIDER_DBTON spider_dbton[SPIDER_DBTON_SIZE];
 #ifdef SPIDER_HAS_HASH_VALUE_TYPE
@@ -343,6 +347,10 @@ int ha_spider::open(
 
   dup_key_idx = (uint) -1;
   conn_kinds = SPIDER_CONN_KIND_MYSQL;
+
+  if (spider_memory_rdlock())
+    DBUG_RETURN(ER_PLUGIN_IS_NOT_LOADED);
+
   if (!spider_get_share(name, table, thd, this, &error_num))
     goto error_get_share;
   thr_lock_data_init(&share->lock,&lock,NULL);
@@ -584,6 +592,7 @@ int ha_spider::open(
     goto error_reset;
   }
 
+  spider_memory_unlock();
   DBUG_RETURN(0);
 
 error_reset:
@@ -636,6 +645,8 @@ int ha_spider::open(
     spider_free(spider_current_trx, conn_keys, MYF(0));
     conn_keys = NULL;
   }
+
+  spider_memory_unlock();
   DBUG_RETURN(error_num);
 }
 
@@ -650,6 +661,9 @@ int ha_spider::close()
   DBUG_ENTER("ha_spider::close");
   DBUG_PRINT("info",("spider this=%p", this));
 
+  if (spider_memory_rdlock())
+    DBUG_RETURN(ER_PLUGIN_IS_NOT_LOADED);
+
 #ifdef HA_MRR_USE_DEFAULT_IMPL
   if (multi_range_keys)
   {
@@ -822,6 +836,7 @@ int ha_spider::close()
   trx = NULL;
   conns = NULL;
 
+  spider_memory_unlock();
   DBUG_RETURN(error_num);
 }
 
@@ -1245,7 +1260,7 @@ int ha_spider::external_lock(
   }
 
   DBUG_PRINT("info",("spider sql_command=%d", sql_command));
-  DBUG_ASSERT(trx == spider_get_trx(thd, TRUE, &error_num));
+
 #ifdef HA_CAN_BULK_ACCESS
   external_lock_cnt++;
 #endif
@@ -1254,8 +1269,23 @@ int ha_spider::external_lock(
     sql_command != SQLCOM_UNLOCK_TABLES
   )
     DBUG_RETURN(0);
+
+  if (spider_memory_rdlock())
+    DBUG_RETURN(ER_PLUGIN_IS_NOT_LOADED);
+
+  SPIDER_TRX *tmp_trx = spider_get_trx(thd, TRUE, &error_num);
+  if (error_num)
+  {
+    spider_memory_unlock();
+    DBUG_RETURN(error_num);
+  }
+  DBUG_ASSERT(trx == tmp_trx);
+
   if (store_error_num)
+  {
+    spider_memory_unlock();
     DBUG_RETURN(store_error_num);
+  }
 #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET)
   if ((conn_kinds & SPIDER_CONN_KIND_MYSQL))
   {
@@ -1271,10 +1301,12 @@ int ha_spider::external_lock(
           ER_SPIDER_ALTER_BEFORE_UNLOCK_STR, MYF(0));
         DBUG_RETURN(ER_SPIDER_ALTER_BEFORE_UNLOCK_NUM);
       }
+      spider_memory_unlock();
       DBUG_RETURN(0);
     }
     if (!conns[search_link_idx])
     {
+      spider_memory_unlock();
       my_message(ER_SPIDER_REMOTE_SERVER_GONE_AWAY_NUM,
         ER_SPIDER_REMOTE_SERVER_GONE_AWAY_STR, MYF(0));
       DBUG_RETURN(ER_SPIDER_REMOTE_SERVER_GONE_AWAY_NUM);
@@ -1289,7 +1321,10 @@ int ha_spider::external_lock(
         SPIDER_LINK_STATUS_RECOVERY)
     ) {
       if (sql_command == SQLCOM_TRUNCATE)
+      {
+        spider_memory_unlock();
         DBUG_RETURN(0);
+      }
       else if (sql_command != SQLCOM_UNLOCK_TABLES)
       {
         DBUG_PRINT("info",("spider conns[%d]->join_trx=%u",
@@ -1319,6 +1354,7 @@ int ha_spider::external_lock(
                 TRUE
               );
           }
+          spider_memory_unlock();
           DBUG_RETURN(check_error_mode(error_num));
         }
         result_list.lock_type = lock_type;
@@ -1372,6 +1408,7 @@ int ha_spider::external_lock(
                     TRUE
                   );
               }
+              spider_memory_unlock();
               DBUG_RETURN(check_error_mode(error_num));
             }
           }
@@ -1405,6 +1442,7 @@ int ha_spider::external_lock(
               );
           }
           conns[roop_count]->table_lock = 0;
+          spider_memory_unlock();
           DBUG_RETURN(check_error_mode(error_num));
         }
         if (conns[roop_count]->table_lock == 2)
@@ -1439,6 +1477,7 @@ int ha_spider::external_lock(
                   TRUE
                 );
             }
+            spider_memory_unlock();
             DBUG_RETURN(check_error_mode(error_num));
           }
         }
@@ -1533,6 +1572,8 @@ int ha_spider::external_lock(
     }
   }
 #endif
+
+  spider_memory_unlock();
   DBUG_RETURN(0);
 }
 
@@ -10594,6 +10635,10 @@ int ha_spider::create(
     sql_command == SQLCOM_DROP_INDEX
   )
     DBUG_RETURN(0);
+
+  if (spider_memory_rdlock())
+    DBUG_RETURN(ER_PLUGIN_IS_NOT_LOADED);
+
   if (!(trx = spider_get_trx(thd, TRUE, &error_num)))
     goto error_get_trx;
   if (
@@ -10746,6 +10791,8 @@ int ha_spider::create(
   if (tmp_share.static_key_cardinality)
     spider_free(spider_current_trx, tmp_share.static_key_cardinality, MYF(0));
   spider_free_share_alloc(&tmp_share);
+
+  spider_memory_unlock();
   DBUG_RETURN(0);
 
 error:
@@ -10759,6 +10806,7 @@ int ha_spider::create(
   spider_free_share_alloc(&tmp_share);
 error_alter_before_unlock:
 error_get_trx:
+  spider_memory_unlock();
   DBUG_RETURN(error_num);
 }
 
@@ -10817,6 +10865,10 @@ int ha_spider::rename_table(
     sql_command == SQLCOM_DROP_INDEX
   )
     DBUG_RETURN(0);
+
+  if (spider_memory_rdlock())
+    DBUG_RETURN(ER_PLUGIN_IS_NOT_LOADED);
+
   if (!(trx = spider_get_trx(thd, TRUE, &error_num)))
     goto error;
   if (
@@ -10982,6 +11034,8 @@ int ha_spider::rename_table(
   }
   pthread_mutex_unlock(&spider_lgtm_tblhnd_share_mutex);
   spider_delete_init_error_table(from);
+
+  spider_memory_unlock();
   DBUG_RETURN(0);
 
 error:
@@ -10999,6 +11053,8 @@ int ha_spider::rename_table(
   if (to_lgtm_tblhnd_share)
     spider_free_lgtm_tblhnd_share_alloc(to_lgtm_tblhnd_share, TRUE);
   pthread_mutex_unlock(&spider_lgtm_tblhnd_share_mutex);
+
+  spider_memory_unlock();
   DBUG_RETURN(error_num);
 }
 
@@ -11025,6 +11081,10 @@ int ha_spider::delete_table(
     sql_command == SQLCOM_DROP_INDEX
   )
     DBUG_RETURN(0);
+
+  if (spider_memory_rdlock())
+    DBUG_RETURN(ER_PLUGIN_IS_NOT_LOADED);
+
   if (!(trx = spider_get_trx(thd, TRUE, &error_num)))
     goto error;
   if (
@@ -11064,8 +11124,10 @@ int ha_spider::delete_table(
         (uchar*) name, name_len)) &&
 #endif
       alter_table->now_create
-    )
+    ) {
+      spider_memory_unlock();
       DBUG_RETURN(0);
+    }
 
     DBUG_PRINT("info",
       ("spider alter_info.flags=%u", thd->lex->alter_info.flags));
@@ -11117,12 +11179,16 @@ int ha_spider::delete_table(
   }
 
   spider_delete_init_error_table(name);
+
+  spider_memory_unlock();
   DBUG_RETURN(0);
 
 error:
   if (table_tables)
     spider_close_sys_table(current_thd, table_tables,
       &open_tables_backup, need_lock);
+
+  spider_memory_unlock();
   DBUG_RETURN(error_num);
 }
 

storage/spider/spd_conn.cc

@@ -37,6 +37,10 @@
 #include "spd_ping_table.h"
 #include "spd_malloc.h"
 
+extern bool is_spider_shutdown();
+extern int  spider_memory_rdlock();
+extern int  spider_memory_unlock();
+
 extern handlerton *spider_hton_ptr;
 extern SPIDER_DBTON spider_dbton[SPIDER_DBTON_SIZE];
 pthread_mutex_t spider_conn_id_mutex;
@@ -2251,6 +2255,7 @@ void *spider_bg_conn_action(
   void *arg
 ) {
   int error_num;
+  bool do_kill = FALSE;
   SPIDER_CONN *conn = (SPIDER_CONN*) arg;
   SPIDER_TRX *trx;
   ha_spider *spider;
@@ -2272,12 +2277,18 @@ void *spider_bg_conn_action(
 #endif
   thd->thread_stack = (char*) &thd;
   thd->store_globals();
-  if (!(trx = spider_get_trx(thd, FALSE, &error_num)))
-  {
+  if (spider_memory_rdlock())
+    do_kill = TRUE;
+  if (
+    do_kill ||
+    !(trx = spider_get_trx(thd, FALSE, &error_num))
+  ) {
     delete thd;
     pthread_mutex_lock(&conn->bg_conn_sync_mutex);
     pthread_cond_signal(&conn->bg_conn_sync_cond);
     pthread_mutex_unlock(&conn->bg_conn_sync_mutex);
+    if (!do_kill)
+      spider_memory_unlock();
 #if !defined(MYSQL_DYNAMIC_PLUGIN) || !defined(_WIN32)
     my_pthread_setspecific_ptr(THR_THD, NULL);
 #endif
@@ -2301,7 +2312,10 @@ void *spider_bg_conn_action(
       conn->bg_conn_chain_mutex_ptr = NULL;
     }
     thd->clear_error();
+    spider_memory_unlock();
     pthread_cond_wait(&conn->bg_conn_cond, &conn->bg_conn_mutex);
+    if (spider_memory_rdlock())
+      do_kill = TRUE;
     DBUG_PRINT("info",("spider bg roop start"));
 #ifndef DBUG_OFF
     DBUG_PRINT("info",("spider conn->thd=%p", conn->thd));
@@ -2310,6 +2324,8 @@ void *spider_bg_conn_action(
       DBUG_PRINT("info",("spider query_id=%lld", conn->thd->query_id));
     }
 #endif
+    if (!do_kill)
+    {
       if (conn->bg_caller_sync_wait)
       {
         pthread_mutex_lock(&conn->bg_conn_sync_mutex);
@@ -2327,7 +2343,8 @@ void *spider_bg_conn_action(
           }
         }
       }
-    if (conn->bg_kill)
+    }
+    if (conn->bg_kill || do_kill)
     {
       DBUG_PRINT("info",("spider bg kill start"));
       if (conn->bg_conn_chain_mutex_ptr)
@@ -2342,6 +2359,8 @@ void *spider_bg_conn_action(
       pthread_cond_signal(&conn->bg_conn_sync_cond);
       pthread_mutex_unlock(&conn->bg_conn_mutex);
       pthread_mutex_unlock(&conn->bg_conn_sync_mutex);
+      if (!do_kill)
+        spider_memory_unlock();
 #if !defined(MYSQL_DYNAMIC_PLUGIN) || !defined(_WIN32)
       my_pthread_setspecific_ptr(THR_THD, NULL);
 #endif
@@ -2700,6 +2719,7 @@ void *spider_bg_sts_action(
   SPIDER_SHARE *share = (SPIDER_SHARE*) arg;
   SPIDER_TRX *trx;
   int error_num = 0, roop_count;
+  bool do_kill = FALSE;
   ha_spider spider;
 #if defined(_MSC_VER) || defined(__SUNPRO_CC)
   int *need_mons;
@@ -2782,13 +2802,19 @@ void *spider_bg_sts_action(
 #endif
   thd->thread_stack = (char*) &thd;
   thd->store_globals();
-  if (!(trx = spider_get_trx(thd, FALSE, &error_num)))
-  {
+  if (spider_memory_rdlock())
+    do_kill = TRUE;
+  if (
+    do_kill ||
+    !(trx = spider_get_trx(thd, FALSE, &error_num))
+  ) {
     delete thd;
     share->bg_sts_thd_wait = FALSE;
     share->bg_sts_kill = FALSE;
     share->bg_sts_init = FALSE;
     pthread_mutex_unlock(&share->sts_mutex);
+    if (!do_kill)
+      spider_memory_unlock();
 #if !defined(MYSQL_DYNAMIC_PLUGIN) || !defined(_WIN32)
     my_pthread_setspecific_ptr(THR_THD, NULL);
 #endif
@@ -2863,12 +2889,15 @@ void *spider_bg_sts_action(
 #endif
     DBUG_RETURN(NULL);
   }
+  spider_memory_unlock();
   /* init end */
 
   while (TRUE)
   {
     DBUG_PRINT("info",("spider bg sts roop start"));
-    if (share->bg_sts_kill)
+    if (spider_memory_rdlock())
+      do_kill = TRUE;
+    if (share->bg_sts_kill || do_kill)
     {
       DBUG_PRINT("info",("spider bg sts kill start"));
       for (roop_count = SPIDER_DBTON_SIZE - 1; roop_count >= 0; --roop_count)
@@ -2885,6 +2914,8 @@ void *spider_bg_sts_action(
       delete thd;
       pthread_cond_signal(&share->bg_sts_sync_cond);
       pthread_mutex_unlock(&share->sts_mutex);
+      if (!do_kill)
+        spider_memory_unlock();
 #if !defined(MYSQL_DYNAMIC_PLUGIN) || !defined(_WIN32)
       my_pthread_setspecific_ptr(THR_THD, NULL);
 #endif
@@ -2995,6 +3026,7 @@ void *spider_bg_sts_action(
     }
     memset(need_mons, 0, sizeof(int) * share->link_count);
     share->bg_sts_thd_wait = TRUE;
+    spider_memory_unlock();
     pthread_cond_wait(&share->bg_sts_cond, &share->sts_mutex);
   }
 }
@@ -3078,6 +3110,7 @@ void *spider_bg_crd_action(
   SPIDER_SHARE *share = (SPIDER_SHARE*) arg;
   SPIDER_TRX *trx;
   int error_num = 0, roop_count;
+  bool do_kill = FALSE;
   ha_spider spider;
   TABLE table;
 #if defined(_MSC_VER) || defined(__SUNPRO_CC)
@@ -3161,13 +3194,19 @@ void *spider_bg_crd_action(
 #endif
   thd->thread_stack = (char*) &thd;
   thd->store_globals();
-  if (!(trx = spider_get_trx(thd, FALSE, &error_num)))
-  {
+  if (spider_memory_rdlock())
+    do_kill = TRUE;
+  if (
+    do_kill ||
+    !(trx = spider_get_trx(thd, FALSE, &error_num))
+  ) {
     delete thd;
     share->bg_crd_thd_wait = FALSE;
     share->bg_crd_kill = FALSE;
     share->bg_crd_init = FALSE;
     pthread_mutex_unlock(&share->crd_mutex);
+    if (!do_kill)
+      spider_memory_unlock();
 #if !defined(MYSQL_DYNAMIC_PLUGIN) || !defined(_WIN32)
     my_pthread_setspecific_ptr(THR_THD, NULL);
 #endif
@@ -3237,6 +3276,7 @@ void *spider_bg_crd_action(
     share->bg_crd_kill = FALSE;
     share->bg_crd_init = FALSE;
     pthread_mutex_unlock(&share->crd_mutex);
+    spider_memory_unlock();
 #if !defined(MYSQL_DYNAMIC_PLUGIN) || !defined(_WIN32)
     my_pthread_setspecific_ptr(THR_THD, NULL);
 #endif
@@ -3246,12 +3286,15 @@ void *spider_bg_crd_action(
 #endif
     DBUG_RETURN(NULL);
   }
+  spider_memory_unlock();
   /* init end */
 
   while (TRUE)
   {
     DBUG_PRINT("info",("spider bg crd roop start"));
-    if (share->bg_crd_kill)
+    if (spider_memory_rdlock())
+      do_kill = TRUE;
+    if (share->bg_crd_kill || do_kill)
     {
       DBUG_PRINT("info",("spider bg crd kill start"));
       for (roop_count = SPIDER_DBTON_SIZE - 1; roop_count >= 0; --roop_count)
@@ -3268,6 +3311,8 @@ void *spider_bg_crd_action(
       delete thd;
       pthread_cond_signal(&share->bg_crd_sync_cond);
       pthread_mutex_unlock(&share->crd_mutex);
+      if (!do_kill)
+        spider_memory_unlock();
 #if !defined(MYSQL_DYNAMIC_PLUGIN) || !defined(_WIN32)
       my_pthread_setspecific_ptr(THR_THD, NULL);
 #endif
@@ -3378,6 +3423,7 @@ void *spider_bg_crd_action(
     }
     memset(need_mons, 0, sizeof(int) * share->link_count);
     share->bg_crd_thd_wait = TRUE;
+    spider_memory_unlock();
     pthread_cond_wait(&share->bg_crd_cond, &share->crd_mutex);
   }
 }
@@ -3628,6 +3674,7 @@ void *spider_bg_mon_action(
   SPIDER_SHARE *share = link_pack->share;
   SPIDER_TRX *trx;
   int error_num, link_idx = link_pack->link_idx;
+  bool do_kill = FALSE;
   THD *thd;
   my_thread_init();
   DBUG_ENTER("spider_bg_mon_action");
@@ -3647,13 +3694,19 @@ void *spider_bg_mon_action(
 #endif
   thd->thread_stack = (char*) &thd;
   thd->store_globals();
-  if (!(trx = spider_get_trx(thd, FALSE, &error_num)))
-  {
+  if (spider_memory_rdlock())
+    do_kill = TRUE;
+  if (
+    do_kill ||
+    !(trx = spider_get_trx(thd, FALSE, &error_num))
+  ) {
     delete thd;
     share->bg_mon_kill = FALSE;
     share->bg_mon_init = FALSE;
     pthread_cond_signal(&share->bg_mon_conds[link_idx]);
     pthread_mutex_unlock(&share->bg_mon_mutexes[link_idx]);
+    if (!do_kill)
+      spider_memory_unlock();
 #if !defined(MYSQL_DYNAMIC_PLUGIN) || !defined(_WIN32)
     my_pthread_setspecific_ptr(THR_THD, NULL);
 #endif
@@ -3665,13 +3718,16 @@ void *spider_bg_mon_action(
 /*
   pthread_mutex_unlock(&share->bg_mon_mutexes[link_idx]);
 */
+  spider_memory_unlock();
   /* init end */
 
   while (TRUE)
   {
     DBUG_PRINT("info",("spider bg mon sleep %lld",
       share->monitoring_bg_interval[link_idx]));
-    if (!share->bg_mon_kill)
+    if (spider_memory_rdlock())
+      do_kill = TRUE;
+    if (!share->bg_mon_kill && !do_kill)
     {
       struct timespec abstime;
       set_timespec_nsec(abstime,
@@ -3683,7 +3739,7 @@ void *spider_bg_mon_action(
 */
     }
     DBUG_PRINT("info",("spider bg mon roop start"));
-    if (share->bg_mon_kill)
+    if (share->bg_mon_kill || do_kill)
     {
       DBUG_PRINT("info",("spider bg mon kill start"));
 /*
@@ -3693,6 +3749,8 @@ void *spider_bg_mon_action(
       pthread_mutex_unlock(&share->bg_mon_mutexes[link_idx]);
       spider_free_trx(trx, TRUE);
       delete thd;
+      if (!do_kill)
+        spider_memory_unlock();
 #if !defined(MYSQL_DYNAMIC_PLUGIN) || !defined(_WIN32)
       my_pthread_setspecific_ptr(THR_THD, NULL);
 #endif
@@ -3719,6 +3777,7 @@ void *spider_bg_mon_action(
       );
       lex_end(thd->lex);
     }
+    spider_memory_unlock();
   }
 }
 #endif

storage/spider/spd_copy_tables.cc

@@ -40,6 +40,10 @@
 #include "spd_udf.h"
 #include "spd_malloc.h"
 
+extern bool is_spider_shutdown();
+extern int  spider_memory_rdlock();
+extern int  spider_memory_unlock();
+
 extern handlerton *spider_hton_ptr;
 extern SPIDER_DBTON spider_dbton[SPIDER_DBTON_SIZE];
 
@@ -835,6 +839,7 @@ long long spider_copy_tables_body(
   char *error
 ) {
   int error_num, roop_count, all_link_cnt = 0, use_table_charset;
+  bool do_spider_memory_unlock = FALSE;
   SPIDER_COPY_TABLES *copy_tables = NULL;
   THD *thd = current_thd;
   TABLE_LIST *table_list = NULL;
@@ -913,6 +918,10 @@ long long spider_copy_tables_body(
     goto error;
   }
   
+  if (spider_memory_rdlock())
+    goto error;
+  do_spider_memory_unlock = TRUE;
+
   if (!(copy_tables = (SPIDER_COPY_TABLES *)
     spider_bulk_malloc(spider_current_trx, 27, MYF(MY_WME | MY_ZEROFILL),
       &copy_tables, sizeof(SPIDER_COPY_TABLES),
@@ -1239,6 +1248,8 @@ long long spider_copy_tables_body(
     delete [] tmp_sql;
   spider_udf_free_copy_tables_alloc(copy_tables);
 
+  if (do_spider_memory_unlock)
+    spider_memory_unlock();
   DBUG_RETURN(1);
 
 error_db_udf_copy_tables:
@@ -1306,6 +1317,8 @@ long long spider_copy_tables_body(
   {
     spider_udf_free_copy_tables_alloc(copy_tables);
   }
+  if (do_spider_memory_unlock)
+    spider_memory_unlock();
   *error = 1;
   DBUG_RETURN(0);
 }

storage/spider/spd_direct_sql.cc

@@ -44,6 +44,10 @@
 #define SPIDER_NEED_INIT_ONE_TABLE_FOR_FIND_TEMPORARY_TABLE
 #endif
 
+extern bool is_spider_shutdown();
+extern int  spider_memory_rdlock();
+extern int  spider_memory_unlock();
+
 extern const char **spd_defaults_extra_file;
 extern const char **spd_defaults_file;
 
@@ -1529,6 +1533,7 @@ long long spider_direct_sql_body(
   my_bool bg
 ) {
   int error_num, roop_count;
+  bool do_spider_memory_unlock = FALSE;
   SPIDER_DIRECT_SQL *direct_sql = NULL, *tmp_direct_sql;
   THD *thd = current_thd;
   SPIDER_TRX *trx;
@@ -1543,6 +1548,11 @@ long long spider_direct_sql_body(
 #endif
   DBUG_ENTER("spider_direct_sql_body");
   SPIDER_BACKUP_DASTATUS;
+
+  if (spider_memory_rdlock())
+    goto error;
+  do_spider_memory_unlock = TRUE;
+
   if (!(direct_sql = (SPIDER_DIRECT_SQL *)
     spider_bulk_malloc(spider_current_trx, 34, MYF(MY_WME | MY_ZEROFILL),
       &direct_sql, sizeof(SPIDER_DIRECT_SQL),
@@ -1737,6 +1747,9 @@ long long spider_direct_sql_body(
 #ifndef WITHOUT_SPIDER_BG_SEARCH
   }
 #endif
+
+  if (do_spider_memory_unlock)
+    spider_memory_unlock();
   DBUG_RETURN(1);
 
 error:
@@ -1748,10 +1761,15 @@ long long spider_direct_sql_body(
     ) {
       SPIDER_RESTORE_DASTATUS;
       spider_udf_free_direct_sql_alloc(direct_sql, bg);
+      if (do_spider_memory_unlock)
+        spider_memory_unlock();
       DBUG_RETURN(1);
     }
     spider_udf_free_direct_sql_alloc(direct_sql, bg);
   }
+
+  if (do_spider_memory_unlock)
+    spider_memory_unlock();
   *error = 1;
   DBUG_RETURN(0);
 }

storage/spider/spd_table.cc

@@ -54,6 +54,137 @@ const char **spd_defaults_extra_file;
 const char **spd_defaults_file;
 bool volatile *spd_abort_loop;
 
+// Set to TRUE when Spider plugin uninstall begins
+longlong spider_shutdown_thd;
+
+void set_spider_shutdown(THD *thd)
+{
+  my_atomic_store64(&spider_shutdown_thd, (longlong) thd);
+}
+
+void clear_spider_shutdown()
+{
+  my_atomic_store64(&spider_shutdown_thd, (longlong) NULL);
+}
+
+THD *get_spider_shutdown_thd()
+{
+  return (THD *) my_atomic_load64(&spider_shutdown_thd);
+}
+
+bool is_spider_shutdown()
+{
+  return (bool) my_atomic_load64(&spider_shutdown_thd);
+}
+
+int spider_memory_implicit_rdlock_count;
+
+/*
+  Lock for accessing allocated Spider memory
+
+  Spider plugin uninstall acquires a WR lock
+  Other threads acquire a RD lock
+*/
+mysql_rwlock_t spider_memory_lock;
+#ifdef HAVE_PSI_INTERFACE
+PSI_rwlock_key spider_memory_lock_key;
+static PSI_rwlock_info spider_memory_lock_info =
+  { &spider_memory_lock_key, "spider_memory_lock", PSI_FLAG_GLOBAL };
+#endif
+
+int spider_memory_lock_init()
+{
+  clear_spider_shutdown();
+  my_atomic_store32(&spider_memory_implicit_rdlock_count, 0);
+#ifdef HAVE_PSI_INTERFACE
+  mysql_rwlock_register("spider", &spider_memory_lock_info, 1);
+#endif
+  return mysql_rwlock_init(spider_memory_lock_key, &spider_memory_lock);
+}
+
+int spider_memory_unlock()
+{
+  THD *shutdown_thd = get_spider_shutdown_thd();
+  if (shutdown_thd)
+  {
+    if (shutdown_thd == current_thd)
+    {
+      // Current thread is the Spider shutdown thread
+      if (my_atomic_load32(&spider_memory_implicit_rdlock_count))
+      {
+        // Release an implicit read-lock
+        my_atomic_add32(&spider_memory_implicit_rdlock_count, -1);
+        return 0;
+      }
+    }
+  }
+  return mysql_rwlock_unlock(&spider_memory_lock);
+}
+
+int spider_memory_rdlock()
+{
+  THD *shutdown_thd = get_spider_shutdown_thd();
+  if (shutdown_thd)
+  {
+    if (shutdown_thd == current_thd)
+    {
+      /*
+        Current thread is the Spider shutdown thread and
+        already holds a write lock for Spider shutdown,
+        so grant an implicit read-lock
+      */
+      my_atomic_add32(&spider_memory_implicit_rdlock_count, 1);
+      return 0;
+    }
+    // Spider shutdown is in progress, so deny the lock request
+    return ER_PLUGIN_IS_NOT_LOADED;
+  }
+
+  // Acquire the read-lock
+  int result = mysql_rwlock_rdlock(&spider_memory_lock);
+  if (!result)
+  {
+    if (is_spider_shutdown())
+    {
+      spider_memory_unlock();
+      result = ER_PLUGIN_IS_NOT_LOADED;
+    }
+  }
+  return result;
+}
+
+int spider_memory_wrlock()
+{
+  return mysql_rwlock_wrlock(&spider_memory_lock);
+}
+
+int spider_memory_lock_destroy()
+{
+  return mysql_rwlock_destroy(&spider_memory_lock);
+}
+
+int spider_shutdown_lock(THD *thd)
+{
+  set_spider_shutdown(thd);
+
+  // Acquire the write-lock
+  int result = spider_memory_wrlock();
+  if (result)
+  {
+    fprintf(stderr, "\nWrite-lock request on spider memory failed\n");
+    fflush(stderr);
+  }
+  return result;
+}
+
+void spider_shutdown_unlock()
+{
+  spider_memory_unlock();
+  // Do not destroy the spider memory lock as another thread
+  // may try to acquire it after completion of plugin deinit
+  // spider_memory_lock_destroy();
+}
+
 handlerton *spider_hton_ptr;
 SPIDER_DBTON spider_dbton[SPIDER_DBTON_SIZE];
 extern SPIDER_DBTON spider_dbton_mysql;
@@ -5989,19 +6120,41 @@ int spider_close_connection(
   handlerton* hton,
   THD* thd
 ) {
-  int roop_count = 0;
-  SPIDER_CONN *conn;
   SPIDER_TRX *trx;
   DBUG_ENTER("spider_close_connection");
+
+  if (spider_memory_rdlock())
+    DBUG_RETURN(ER_PLUGIN_IS_NOT_LOADED);
+
   if (!(trx = (SPIDER_TRX*) *thd_ha_data(thd, spider_hton_ptr)))
+  {
+    spider_memory_unlock();
     DBUG_RETURN(0); /* transaction is not started */
+  }
+
+  DBUG_ASSERT(thd == trx->thd);
+  spider_close_trx_connection(trx);
+  spider_free_trx(trx, TRUE);
+
+  spider_memory_unlock();
+  DBUG_RETURN(0);
+}
+
+int spider_close_trx_connection(
+  SPIDER_TRX *trx
+) {
+  THD* thd = trx->thd;
+  SPIDER_CONN *conn;
+  DBUG_ENTER("spider_close_trx_connection");
 
   trx->tmp_spider->conns = &conn;
-  while ((conn = (SPIDER_CONN*) my_hash_element(&trx->trx_conn_hash,
-    roop_count)))
+  for (ulong i = 0; i < trx->trx_conn_hash.records; i++)
+  {
+    conn = (SPIDER_CONN*) my_hash_element(&trx->trx_conn_hash, i);
+    if (conn)
     {
       SPIDER_BACKUP_DASTATUS;
-    DBUG_PRINT("info",("spider conn->table_lock=%d", conn->table_lock));
+      DBUG_PRINT("info", ("spider conn->table_lock=%d", conn->table_lock));
       if (conn->table_lock > 0)
       {
         if (!conn->trx_start)
@@ -6012,13 +6165,11 @@ int spider_close_connection(
         }
         conn->table_lock = 0;
       }
-    roop_count++;
       SPIDER_CONN_RESTORE_DASTATUS;
     }
+  }
 
   spider_rollback(spider_hton_ptr, thd, TRUE);
-  spider_free_trx(trx, TRUE);
-
   DBUG_RETURN(0);
 }
 
@@ -6048,6 +6199,8 @@ int spider_db_done(
   void *p
 ) {
   int roop_count;
+  int error_num;
+  bool do_delete_thd;
   THD *thd = current_thd, *tmp_thd;
   SPIDER_CONN *conn;
   SPIDER_INIT_ERROR_TABLE *spider_init_error_table;
@@ -6055,6 +6208,29 @@ int spider_db_done(
   SPIDER_LGTM_TBLHND_SHARE *lgtm_tblhnd_share;
   DBUG_ENTER("spider_db_done");
 
+  if (thd)
+    do_delete_thd = FALSE;
+  else
+  {
+    do_delete_thd = TRUE;
+    my_thread_init();
+    if (!(thd = new THD(next_thread_id())))
+    {
+      my_thread_end();
+      DBUG_RETURN(HA_ERR_OUT_OF_MEM);
+    }
+#ifdef HAVE_PSI_INTERFACE
+    mysql_thread_set_psi_id(thd->thread_id);
+#endif
+    thd->thread_stack = (char*) &thd;
+    thd->store_globals();
+  }
+
+  // Begin Spider plugin deinit
+  error_num = spider_shutdown_lock(thd);
+  if (error_num)
+    DBUG_RETURN(error_num);
+
 #ifndef WITHOUT_SPIDER_BG_SEARCH
   spider_free_trx(spider_global_trx, TRUE);
 #endif
@@ -6097,21 +6273,28 @@ int spider_db_done(
     pthread_mutex_destroy(&spider_udf_table_mon_mutexes[roop_count]);
   spider_free(NULL, spider_udf_table_mon_mutexes, MYF(0));
 
-  if (thd && thd_sql_command(thd) == SQLCOM_UNINSTALL_PLUGIN) {
   pthread_mutex_lock(&spider_allocated_thds_mutex);
-    while ((tmp_thd = (THD *) my_hash_element(&spider_allocated_thds, 0)))
+  for (ulong i = 0; i < spider_allocated_thds.records;)
+  {
+    tmp_thd = (THD *) my_hash_element(&spider_allocated_thds, i);
+    if (tmp_thd)
     {
       SPIDER_TRX *trx = (SPIDER_TRX *) *thd_ha_data(tmp_thd, spider_hton_ptr);
       if (trx)
       {
         DBUG_ASSERT(tmp_thd == trx->thd);
+        spider_close_trx_connection(trx);
         spider_free_trx(trx, FALSE);
         *thd_ha_data(tmp_thd, spider_hton_ptr) = (void *) NULL;
-      } else
+      }
+      else
         my_hash_delete(&spider_allocated_thds, (uchar *) tmp_thd);
     }
-    pthread_mutex_unlock(&spider_allocated_thds_mutex);
+    else
+      i++;
   }
+  pthread_mutex_unlock(&spider_allocated_thds_mutex);
+
 #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET)
   pthread_mutex_lock(&spider_hs_w_conn_mutex);
   while ((conn = (SPIDER_CONN*) my_hash_element(&spider_hs_w_conn_hash, 0)))
@@ -6260,10 +6443,16 @@ int spider_db_done(
         spider_current_alloc_mem[roop_count] ? "NG" : "OK"
       ));
   }
+
+  // End Spider plugin deinit
+  spider_shutdown_unlock();
+  if (do_delete_thd)
+    delete thd;
+
 /*
 DBUG_ASSERT(0);
 */
-  DBUG_RETURN(0);
+  DBUG_RETURN(error_num);
 }
 
 int spider_panic(
@@ -6281,6 +6470,12 @@ int spider_db_init(
   uint dbton_id = 0;
   handlerton *spider_hton = (handlerton *)p;
   DBUG_ENTER("spider_db_init");
+  if (spider_memory_lock_init())
+  {
+    error_num = HA_ERR_OUT_OF_MEM;
+    DBUG_RETURN(error_num);
+  }
+
   spider_hton_ptr = spider_hton;
 
   spider_hton->state = SHOW_OPTION_YES;

storage/spider/spd_table.h

@@ -265,6 +265,10 @@ int spider_close_connection(
   THD* thd
 );
 
+int spider_close_trx_connection(
+  SPIDER_TRX *trx
+);
+
 void spider_drop_database(
   handlerton *hton,
   char* path

storage/spider/spd_trx.cc

@@ -57,6 +57,10 @@ ulonglong spider_thread_id = 1;
 extern PSI_mutex_key spd_key_mutex_udf_table;
 #endif
 
+extern bool is_spider_shutdown();
+extern int  spider_memory_rdlock();
+extern int  spider_memory_unlock();
+
 extern HASH spider_allocated_thds;
 extern uint spider_allocated_thds_id;
 extern const char *spider_allocated_thds_func_name;
@@ -1145,6 +1149,14 @@ SPIDER_TRX *spider_get_trx(
   pthread_mutex_t *udf_table_mutexes;
   DBUG_ENTER("spider_get_trx");
 
+  if (spider_memory_rdlock())
+  {
+    *error_num = ER_PLUGIN_IS_NOT_LOADED;
+    DBUG_RETURN(NULL);
+  }
+
+  *error_num = 0;
+
   if (
     !thd ||
     !(trx = (SPIDER_TRX*) *thd_ha_data(thd, spider_hton_ptr))
@@ -1399,6 +1411,7 @@ SPIDER_TRX *spider_get_trx(
     }
   }
 
+  spider_memory_unlock();
   DBUG_PRINT("info",("spider trx=%p", trx));
   DBUG_RETURN(trx);
 
@@ -1502,6 +1515,7 @@ SPIDER_TRX *spider_get_trx(
   free_root(&trx->mem_root, MYF(0));
   spider_free(NULL, trx, MYF(0));
 error_alloc_trx:
+  spider_memory_unlock();
   *error_num = HA_ERR_OUT_OF_MEM;
   DBUG_RETURN(NULL);
 }
@@ -1511,7 +1525,8 @@ int spider_free_trx(
   bool need_lock
 ) {
   DBUG_ENTER("spider_free_trx");
-  if (trx->thd)
+  THD *thd = trx->thd;
+  if (thd)
   {
     if (trx->registed_allocated_thds)
     {
@@ -1519,9 +1534,9 @@ int spider_free_trx(
         pthread_mutex_lock(&spider_allocated_thds_mutex);
 #ifdef HASH_UPDATE_WITH_HASH_VALUE
       my_hash_delete_with_hash_value(&spider_allocated_thds,
-        trx->thd_hash_value, (uchar*) trx->thd);
+        trx->thd_hash_value, (uchar*) thd);
 #else
-      my_hash_delete(&spider_allocated_thds, (uchar*) trx->thd);
+      my_hash_delete(&spider_allocated_thds, (uchar*) thd);
 #endif
       if (need_lock)
         pthread_mutex_unlock(&spider_allocated_thds_mutex);
@@ -1793,6 +1808,9 @@ int spider_internal_start_trx(
   time_t tmp_time = (time_t) time((time_t*) 0);
   DBUG_ENTER("spider_internal_start_trx");
 
+  if (spider_memory_rdlock())
+    DBUG_RETURN(ER_PLUGIN_IS_NOT_LOADED);
+
   if (
     conn->server_lost ||
     difftime(tmp_time, conn->ping_time) >= ping_interval_at_trx_start
@@ -1967,11 +1985,14 @@ int spider_internal_start_trx(
     conn->c_big = NULL;
     trx->join_trx_top = conn;
   }
+
+  spider_memory_unlock();
   DBUG_RETURN(0);
 
 error:
   if (xa_lock)
     spider_xa_unlock(&trx->internal_xid_state);
+  spider_memory_unlock();
   DBUG_RETURN(error_num);
 }
 
@@ -3299,8 +3320,14 @@ int spider_commit(
   SPIDER_CONN *conn;
   DBUG_ENTER("spider_commit");
 
+  if (spider_memory_rdlock())
+    DBUG_RETURN(ER_PLUGIN_IS_NOT_LOADED);
+
   if (!(trx = (SPIDER_TRX*) *thd_ha_data(thd, spider_hton_ptr)))
+  {
+    spider_memory_unlock();
     DBUG_RETURN(0); /* transaction is not started */
+  }
 
 #ifdef HA_CAN_BULK_ACCESS
   DBUG_PRINT("info",("spider trx->bulk_access_conn_first=%p",
@@ -3329,6 +3356,7 @@ int spider_commit(
 /*
             }
 */
+            spider_memory_unlock();
             DBUG_RETURN(error_num);
           }
           trx->trx_xa_prepared = TRUE;
@@ -3376,6 +3404,8 @@ int spider_commit(
     trx->trx_consistent_snapshot = FALSE;
   }
   spider_merge_mem_calc(trx, FALSE);
+
+  spider_memory_unlock();
   DBUG_RETURN(error_num);
 }
 
@@ -3389,8 +3419,14 @@ int spider_rollback(
   SPIDER_CONN *conn;
   DBUG_ENTER("spider_rollback");
 
+  if (spider_memory_rdlock())
+    DBUG_RETURN(ER_PLUGIN_IS_NOT_LOADED);
+
   if (!(trx = (SPIDER_TRX*) *thd_ha_data(thd, spider_hton_ptr)))
+  {
+    spider_memory_unlock();
     DBUG_RETURN(0); /* transaction is not started */
+  }
 
 #ifdef HA_CAN_BULK_ACCESS
   DBUG_PRINT("info",("spider trx->bulk_access_conn_first=%p",
@@ -3448,6 +3484,8 @@ int spider_rollback(
   }
 
   spider_merge_mem_calc(trx, FALSE);
+
+  spider_memory_unlock();
   DBUG_RETURN(error_num);
 }
 
@@ -3572,6 +3610,10 @@ int spider_end_trx(
 ) {
   int error_num = 0, need_mon = 0;
   DBUG_ENTER("spider_end_trx");
+
+  if (spider_memory_rdlock())
+    DBUG_RETURN(ER_PLUGIN_IS_NOT_LOADED);
+
   if (conn->table_lock == 3)
   {
     trx->tmp_spider->conns = &conn;
@@ -3608,6 +3650,8 @@ int spider_end_trx(
   conn->semi_trx_isolation = -2;
   conn->semi_trx_isolation_chk = FALSE;
   conn->semi_trx_chk = FALSE;
+
+  spider_memory_unlock();
   DBUG_RETURN(error_num);
 }