wt: comments, OOM checks, test case for deadlock detection

include/waiting_threads.h:
  make wt_thd_dontwait private
mysql-test/r/maria.result:
  deadlock example
mysql-test/t/maria.test:
  deadlock example
mysys/waiting_threads.c:
  comments, OOM checks
sql/mysqld.cc:
  fix variables
sql/sql_class.cc:
  move wt_lazy_init to THD constructor
sql/sql_class.h:
  move wt_lazy_init to THD constructor
storage/maria/ha_maria.cc:
  backport from 6.0
storage/maria/ma_write.c:
  poset-review fixes, set thd->proc_info
storage/maria/trnman.c:
  bugfixing
storage/myisam/mi_check.c:
  warnings
storage/myisam/mi_page.c:
  warnings
storage/myisam/mi_search.c:
  warnings
storage/myisammrg/myrg_create.c:
  warnings
unittest/mysys/waiting_threads-t.c:
  fixes

include/waiting_threads.h

@@ -154,7 +154,6 @@ void wt_end(void);
 void wt_thd_lazy_init(WT_THD *, ulong *, ulong *, ulong *, ulong *);
 void wt_thd_destroy(WT_THD *);
 int wt_thd_will_wait_for(WT_THD *, WT_THD *, WT_RESOURCE_ID *);
-int wt_thd_dontwait(WT_THD *);
 int wt_thd_cond_timedwait(WT_THD *, pthread_mutex_t *);
 void wt_thd_release(WT_THD *, WT_RESOURCE_ID *);
 #define wt_thd_release_all(THD) wt_thd_release((THD), 0)

mysql-test/r/maria.result

@@ -1900,3 +1900,18 @@ check table t2 extended;
 Table	Op	Msg_type	Msg_text
 test.t2	check	status	OK
 drop table t2;
+set session  deadlock_timeout_long=60000000;
+create table t1 (a int unique) transactional=1;
+insert t1 values (1);
+lock table t1 write concurrent;
+insert t1 values (2);
+set session  deadlock_timeout_long=60000000;
+lock table t1 write concurrent;
+insert t1 values (3);
+insert t1 values (2);
+insert t1 values (3);
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+unlock tables;
+ERROR 23000: Duplicate entry '2' for key 'a'
+unlock tables;
+drop table t1;

mysql-test/t/maria.test

@@ -1186,6 +1186,33 @@ insert into t2 values (repeat('x',28)), (repeat('p',21)), (repeat('k',241)),
 check table t2 extended;
 drop table t2;
 
+#
+# an example of a deadlock
+#
+set session  deadlock_timeout_long=60000000;
+create table t1 (a int unique) transactional=1;
+insert t1 values (1);
+lock table t1 write concurrent;
+insert t1 values (2);
+connect(con_d,localhost,root,,);
+set session  deadlock_timeout_long=60000000;
+lock table t1 write concurrent;
+insert t1 values (3);
+send insert t1 values (2);
+connection default;
+let $wait_condition=select count(*) = 1 from information_schema.processlist where state="waiting for a resource";
+--source include/wait_condition.inc
+--error ER_LOCK_DEADLOCK
+insert t1 values (3);
+unlock tables;
+connection con_d;
+--error ER_DUP_ENTRY
+reap;
+unlock tables;
+disconnect con_d;
+connection default;
+drop table t1;
+
 --disable_result_log
 --disable_query_log
 eval set global storage_engine=$default_engine, maria_page_checksum=$default_checksum;

sql/mysqld.cc

@@ -5703,22 +5703,22 @@ struct my_option my_long_options[] =
   {"deadlock-search-depth-short", OPT_DEADLOCK_SEARCH_DEPTH_SHORT,
    "Short search depth for the two-step deadlock detection",
    (uchar**) &global_system_variables.wt_deadlock_search_depth_short,
-   (uchar**) &global_system_variables.wt_deadlock_search_depth_short,
+   (uchar**) &max_system_variables.wt_deadlock_search_depth_short,
    0, GET_ULONG, REQUIRED_ARG, 4, 0, 32, 0, 0, 0},
   {"deadlock-search-depth-long", OPT_DEADLOCK_SEARCH_DEPTH_LONG,
    "Long search depth for the two-step deadlock detection",
    (uchar**) &global_system_variables.wt_deadlock_search_depth_long,
-   (uchar**) &global_system_variables.wt_deadlock_search_depth_long,
+   (uchar**) &max_system_variables.wt_deadlock_search_depth_long,
    0, GET_ULONG, REQUIRED_ARG, 15, 0, 33, 0, 0, 0},
   {"deadlock-timeout-short", OPT_DEADLOCK_TIMEOUT_SHORT,
    "Short timeout for the two-step deadlock detection (in microseconds)",
    (uchar**) &global_system_variables.wt_timeout_short,
-   (uchar**) &global_system_variables.wt_timeout_short,
+   (uchar**) &max_system_variables.wt_timeout_short,
    0, GET_ULONG, REQUIRED_ARG, 100, 0, ULONG_MAX, 0, 0, 0},
   {"deadlock-timeout-long", OPT_DEADLOCK_TIMEOUT_LONG,
    "Long timeout for the two-step deadlock detection (in microseconds)",
    (uchar**) &global_system_variables.wt_timeout_long,
-   (uchar**) &global_system_variables.wt_timeout_long,
+   (uchar**) &max_system_variables.wt_timeout_long,
    0, GET_ULONG, REQUIRED_ARG, 10000, 0, ULONG_MAX, 0, 0, 0},
 #ifndef DBUG_OFF
   {"debug", '#', "Debug log.", (uchar**) &default_dbug_option,

sql/sql_class.cc

@@ -586,6 +586,10 @@ THD::THD()
   peer_port= 0;					// For SHOW PROCESSLIST
   transaction.m_pending_rows_event= 0;
   transaction.on= 1;
+  wt_thd_lazy_init(&transaction.wt, &variables.wt_deadlock_search_depth_short,
+                                    &variables.wt_timeout_short,
+                                    &variables.wt_deadlock_search_depth_long,
+                                    &variables.wt_timeout_long);
 #ifdef SIGNAL_WITH_VIO_CLOSE
   active_vio = 0;
 #endif

sql/sql_class.h

@@ -1352,14 +1352,9 @@ class THD :public Statement,
     st_transactions()
     {
 #ifdef USING_TRANSACTIONS
-      THD *thd=current_thd;
       bzero((char*)this, sizeof(*this));
       xid_state.xid.null();
       init_sql_alloc(&mem_root, ALLOC_ROOT_MIN_BLOCK_SIZE, 0);
-      wt_thd_lazy_init(&wt, &thd->variables.wt_deadlock_search_depth_short,
-                            &thd->variables.wt_timeout_short,
-                            &thd->variables.wt_deadlock_search_depth_long,
-                            &thd->variables.wt_timeout_long);
 #else
       xid_state.xa_state= XA_NOTR;
 #endif

storage/maria/ha_maria.cc

@@ -2346,7 +2346,8 @@ int ha_maria::external_lock(THD *thd, int lock_type)
             This is a bit excessive, ACID requires this only if there are some
             changes to commit (rollback shouldn't be tested).
           */
-          DBUG_ASSERT(!thd->main_da.is_sent);
+          DBUG_ASSERT(!thd->main_da.is_sent ||
+                      thd->killed == THD::KILL_CONNECTION);
           /* autocommit ? rollback a transaction */
 #ifdef MARIA_CANNOT_ROLLBACK
           if (ma_commit(trn))

storage/maria/ma_write.c

@@ -182,23 +182,20 @@ int maria_write(MARIA_HA *info, uchar *record)
       {
         while (keyinfo->ck_insert(info,
                                   (*keyinfo->make_key)(info, &int_key, i,
-                                                        buff, record, filepos,
-                                                        info->trn->trid)))
+                                                       buff, record, filepos,
+                                                       info->trn->trid)))
         {
           TRN *blocker;
           DBUG_PRINT("error",("Got error: %d on write",my_errno));
           /*
-            explicit check for our own trid, because temp tables
-            aren't transactional and don't have a proper TRN so the code
-            below doesn't work for them
-            XXX a better test perhaps ?
+            explicit check to filter out temp tables, they aren't
+            transactional and don't have a proper TRN so the code
+            below doesn't work for them.
+            Also, filter out non-thread maria use, and table modified in
+            the same transaction.
           */
-          if (info->dup_key_trid == info->trn->trid)
-          {
-            if (local_lock_tree)
-              rw_unlock(&keyinfo->root_lock);
+          if (!local_lock_tree || info->dup_key_trid == info->trn->trid)
             goto err;
-          }
           blocker= trnman_trid_to_trn(info->trn, info->dup_key_trid);
           /*
             if blocker TRN was not found, it means that the conflicting
@@ -206,16 +203,16 @@ int maria_write(MARIA_HA *info, uchar *record)
             aborted, as it would have to wait on the key tree lock
             to remove the conflicting key it has inserted.
           */
-          if (local_lock_tree)
+          if (!blocker || blocker->commit_trid != ~(TrID)0)
+          { /* committed */
+            if (blocker)
+              pthread_mutex_unlock(& blocker->state_lock);
             rw_unlock(&keyinfo->root_lock);
-          if (!blocker)
-            goto err;
-          if (blocker->commit_trid != ~(TrID)0)
-          { /* committed, albeit recently */
-            pthread_mutex_unlock(& blocker->state_lock);
             goto err;
           }
-          { /* running. now we wait */
+          rw_unlock(&keyinfo->root_lock);
+          {
+            /* running. now we wait */
             WT_RESOURCE_ID rc;
             int res;
 
@@ -225,15 +222,26 @@ int maria_write(MARIA_HA *info, uchar *record)
             if (res != WT_OK)
             {
               pthread_mutex_unlock(& blocker->state_lock);
+              my_errno= HA_ERR_LOCK_DEADLOCK;
               goto err;
             }
-            res=wt_thd_cond_timedwait(info->trn->wt, & blocker->state_lock);
+            {
+              const char *old_proc_info= proc_info_hook(0,
+                    "waiting for a resource", __func__, __FILE__, __LINE__);
+
+              res= wt_thd_cond_timedwait(info->trn->wt, & blocker->state_lock);
+
+              proc_info_hook(0, old_proc_info, __func__, __FILE__, __LINE__);
+            }
             pthread_mutex_unlock(& blocker->state_lock);
             if (res != WT_OK)
+            {
+              my_errno= res == WT_TIMEOUT ? HA_ERR_LOCK_WAIT_TIMEOUT
+                                          : HA_ERR_LOCK_DEADLOCK;
               goto err;
+            }
           }
-          if (local_lock_tree)
-            rw_wrlock(&keyinfo->root_lock);
+          rw_wrlock(&keyinfo->root_lock);
         }
       }
 
@@ -643,7 +651,7 @@ static int w_search(register MARIA_HA *info, uint32 comp_flag, MARIA_KEY *key,
     {
       DBUG_PRINT("warning", ("Duplicate key"));
       /*
-        FIXME
+        TODO
         When the index will support true versioning - with multiple
         identical values in the UNIQUE index, invisible to each other -
         the following should be changed to "continue inserting keys, at the

storage/maria/trnman.c

@@ -89,6 +89,7 @@ static void wt_thd_release_self(TRN *trn)
     rc.type= &ma_rc_dup_unique;
     rc.value.ptr= trn;
     wt_thd_release(trn->wt, & rc);
+    trn->wt= 0;
   }
 }
 
@@ -296,8 +297,8 @@ TRN *trnman_new_trn(WT_THD *wt)
     }
     trnman_allocated_transactions++;
     pthread_mutex_init(&trn->state_lock, MY_MUTEX_INIT_FAST);
-    trn->wt= wt;
   }
+  trn->wt= wt;
   trn->pins= lf_hash_get_pins(&trid_to_trn);
   if (!trn->pins)
   {
@@ -415,17 +416,17 @@ my_bool trnman_end_trn(TRN *trn, my_bool commit)
     }
   }
 
+  pthread_mutex_lock(&trn->state_lock);
+  trn->commit_trid= global_trid_generator;
+  wt_thd_release_self(trn);
+  pthread_mutex_unlock(&trn->state_lock);
+
   /*
     if transaction is committed and it was not the only active transaction -
     add it to the committed list
   */
   if (commit && active_list_min.next != &active_list_max)
   {
-    pthread_mutex_lock(&trn->state_lock);
-    trn->commit_trid= global_trid_generator;
-    wt_thd_release_self(trn);
-    pthread_mutex_unlock(&trn->state_lock);
-
     trn->next= &committed_list_max;
     trn->prev= committed_list_max.prev;
     trnman_committed_transactions++;
@@ -440,6 +441,7 @@ my_bool trnman_end_trn(TRN *trn, my_bool commit)
                                active_list_min.next != &active_list_max))
     res= -1;
   trnman_active_transactions--;
+
   pthread_mutex_unlock(&LOCK_trn_list);
 
   /* the rest is done outside of a critical section */
@@ -492,8 +494,6 @@ void trnman_free_trn(TRN *trn)
 
   pthread_mutex_lock(&trn->state_lock);
   trn->short_id= 0;
-  wt_thd_release_self(trn);
-  trn->wt= 0; /* just in case */
   pthread_mutex_unlock(&trn->state_lock);
 
   tmp.trn= pool;

storage/myisam/mi_check.c

@@ -803,9 +803,9 @@ static int chk_index(HA_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
 	(flag=ha_key_cmp(keyinfo->seg,info->lastkey,key,key_length,
 			 comp_flag, diff_pos)) >=0)
     {
-      DBUG_DUMP("old",(uchar*) info->lastkey, info->lastkey_length);
-      DBUG_DUMP("new",(uchar*) key, key_length);
-      DBUG_DUMP("new_in_page",(char*) old_keypos,(uint) (keypos-old_keypos));
+      DBUG_DUMP("old",info->lastkey, info->lastkey_length);
+      DBUG_DUMP("new",key, key_length);
+      DBUG_DUMP("new_in_page",old_keypos,(uint) (keypos-old_keypos));
 
       if (comp_flag & SEARCH_FIND && flag == 0)
 	mi_check_print_error(param,"Found duplicated key at page %s",llstr(page,llbuff));
@@ -874,8 +874,8 @@ static int chk_index(HA_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
       DBUG_PRINT("test",("page: %s  record: %s  filelength: %s",
 			 llstr(page,llbuff),llstr(record,llbuff2),
 			 llstr(info->state->data_file_length,llbuff3)));
-      DBUG_DUMP("key",(uchar*) key,key_length);
-      DBUG_DUMP("new_in_page",(char*) old_keypos,(uint) (keypos-old_keypos));
+      DBUG_DUMP("key",key,key_length);
+      DBUG_DUMP("new_in_page",old_keypos,(uint) (keypos-old_keypos));
       goto err;
     }
     param->record_checksum+=(ha_checksum) record;
@@ -4026,7 +4026,7 @@ static int sort_insert_key(MI_SORT_PARAM *sort_param,
       DBUG_RETURN(1);
     }
     a_length=2+nod_flag;
-    key_block->end_pos= (char*) anc_buff+2;
+    key_block->end_pos= anc_buff+2;
     lastkey=0;					/* No previous key in block */
   }
   else

storage/myisam/mi_page.c

@@ -49,7 +49,7 @@ uchar *_mi_fetch_keypage(register MI_INFO *info, MI_KEYDEF *keyinfo,
   {
     DBUG_PRINT("error",("page %lu had wrong page length: %u",
 			(ulong) page, page_size));
-    DBUG_DUMP("page", (char*) tmp, keyinfo->block_length);
+    DBUG_DUMP("page",tmp, keyinfo->block_length);
     info->last_keypage = HA_OFFSET_ERROR;
     mi_print_error(info->s, HA_ERR_CRASHED);
     my_errno = HA_ERR_CRASHED;

storage/myisam/mi_search.c

@@ -816,7 +816,7 @@ uint _mi_get_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag,
 	    DBUG_PRINT("error",
                        ("Found too long null packed key: %u of %u at 0x%lx",
                         length, keyseg->length, (long) *page_pos));
-	    DBUG_DUMP("key",(char*) *page_pos,16);
+	    DBUG_DUMP("key",*page_pos,16);
             mi_print_error(keyinfo->share, HA_ERR_CRASHED);
 	    my_errno=HA_ERR_CRASHED;
 	    return 0;
@@ -873,7 +873,7 @@ uint _mi_get_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag,
       {
         DBUG_PRINT("error",("Found too long packed key: %u of %u at 0x%lx",
                             length, keyseg->length, (long) *page_pos));
-        DBUG_DUMP("key",(char*) *page_pos,16);
+        DBUG_DUMP("key",*page_pos,16);
         mi_print_error(keyinfo->share, HA_ERR_CRASHED);
         my_errno=HA_ERR_CRASHED;
         return 0;                               /* Error */
@@ -945,7 +945,7 @@ uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag,
       DBUG_PRINT("error",
                  ("Found too long binary packed key: %u of %u at 0x%lx",
                   length, keyinfo->maxlength, (long) *page_pos));
-      DBUG_DUMP("key",(char*) *page_pos,16);
+      DBUG_DUMP("key",*page_pos,16);
       mi_print_error(keyinfo->share, HA_ERR_CRASHED);
       my_errno=HA_ERR_CRASHED;
       DBUG_RETURN(0);                                 /* Wrong key */

storage/myisammrg/myrg_create.c

@@ -46,7 +46,7 @@ int myrg_create(const char *name, const char **table_names,
 	fn_same(buff,name,4);
       *(end=strend(buff))='\n';
       end[1]=0;
-      if (my_write(file,(char*) buff,(uint) (end-buff+1),
+      if (my_write(file,(uchar*) buff,(uint) (end-buff+1),
 		   MYF(MY_WME | MY_NABP)))
 	goto err;
     }

unittest/mysys/waiting_threads-t.c

@@ -214,12 +214,9 @@ void do_tests()
     ok_wait(1,2,1);
     ok_deadlock(2,0,2);
 
-    // FIXME remove wt_thd_dontwait calls below
-    wt_thd_dontwait(& thds[0].thd);
-    wt_thd_dontwait(& thds[1].thd);
-    wt_thd_dontwait(& thds[2].thd);
-    wt_thd_dontwait(& thds[3].thd);
     pthread_mutex_lock(&lock);
+    wt_thd_cond_timedwait(& thds[0].thd, &lock);
+    wt_thd_cond_timedwait(& thds[1].thd, &lock);
     wt_thd_release_all(& thds[0].thd);
     wt_thd_release_all(& thds[1].thd);
     wt_thd_release_all(& thds[2].thd);
@@ -252,7 +249,10 @@ void do_tests()
   do_one_test();
 
   test_kill_strategy(LATEST);
-  test_kill_strategy(RANDOM);
+  SKIP_BIG_TESTS(1)
+  {
+    test_kill_strategy(RANDOM);
+  }
   test_kill_strategy(YOUNGEST);
   test_kill_strategy(LOCKS);