Fix for MDEV-5629: Failing assertion: state == TRX_STATE_NOT_STARTED on...

Fix for MDEV-5629: Failing assertion: state == TRX_STATE_NOT_STARTED on concurrent CREATE OR REPLACE and transactional UPDATE

mysql-test/suite/innodb/r/row_lock.result:
  Test case for MDEV-5629
mysql-test/suite/innodb/t/row_lock.test:
  Test case for MDEV-5629
sql/filesort.cc:
  Don't call unlock_row() in case of errors

mysql-test/suite/innodb/r/row_lock.result

+CREATE TABLE t1 (a INT, b INT) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1,1),(2,2);
+CREATE TABLE t2 (c INT, d INT) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (1,1), (2,2);
+CREATE TABLE t3 (e INT) ENGINE=InnoDB;
+CREATE TABLE t4 ENGINE=InnoDB AS SELECT * FROM t2;
+BEGIN;
+UPDATE t1 SET a = 0 WHERE a = ( SELECT e FROM t3 );
+SET DEBUG_SYNC='srv_suspend_mysql_thread_enter SIGNAL waiting';
+UPDATE t4 SET d = 1 WHERE d in ( SELECT a FROM t1 ) ORDER BY c LIMIT 6;
+SET DEBUG_SYNC='now WAIT_FOR waiting';
+SET DEBUG_SYNC='RESET';
+UPDATE t4 SET d = 9;
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+commit;
+drop table t1,t2,t3,t4;

mysql-test/suite/innodb/t/row_lock.test

+#
+# Test of wrong call to unlock_row
+# MDEV-5629 Failing assertion: state == TRX_STATE_NOT_STARTED on
+# concurrent CREATE OR REPLACE and transactional UPDATE
+#
+
+--source include/have_innodb.inc
+--source include/have_debug_sync.inc
+
+CREATE TABLE t1 (a INT, b INT) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1,1),(2,2);
+
+CREATE TABLE t2 (c INT, d INT) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (1,1), (2,2);
+
+CREATE TABLE t3 (e INT) ENGINE=InnoDB;
+CREATE TABLE t4 ENGINE=InnoDB AS SELECT * FROM t2;
+
+--connect (con11,localhost,root,,test)
+
+BEGIN;
+UPDATE t1 SET a = 0 WHERE a = ( SELECT e FROM t3 );
+
+--connect (con12,localhost,root,,test)
+
+SET DEBUG_SYNC='srv_suspend_mysql_thread_enter SIGNAL waiting';
+
+--send 
+UPDATE t4 SET d = 1 WHERE d in ( SELECT a FROM t1 ) ORDER BY c LIMIT 6;
+
+--connection con11
+
+# Wait for con12 to start waiting for con11
+
+SET DEBUG_SYNC='now WAIT_FOR waiting';
+SET DEBUG_SYNC='RESET';
+
+UPDATE t4 SET d = 9;
+
+--connection con12
+--error ER_LOCK_DEADLOCK
+--reap
+--connection con11
+commit;
+--connection default
+--disconnect con12
+--disconnect con11
+drop table t1,t2,t3,t4;

sql/filesort.cc

@@ -652,12 +652,17 @@ static ha_rows find_all_keys(SORTPARAM *param, SQL_SELECT *select,
       make_sortkey(param, next_sort_key, ref_pos);
       next_sort_key+= param->rec_length;
     }
-    else
-      file->unlock_row();
 
     /* It does not make sense to read more keys in case of a fatal error */
     if (thd->is_error())
       break;
+
+    /*
+      We need to this after checking the error as the transaction may have
+      rolled back in case of a deadlock
+    */
+    if (!write_record)
+      file->unlock_row();
   }
   if (!quick_select)
   {