Bug #25357789 INNODB: LATCH ORDER VIOLATION DURING TRUNCATE TABLE IF INNODB_SYNC_DEBUG ENABLED

Analysis:
========

(1) During TRUNCATE of file_per_table tablespace, dict_operation_lock is
released before eviction of dirty pages of a tablespace from the buffer
pool. After eviction, we try to re-acquire
dict_operation_lock (higher level latch) but we already hold lower
level latch (index->lock). This causes latch order violation

(2) Deadlock issue is present if child table is being truncated and it
holds index lock. At the same time, cascade dml happens and it took
dict_operation_lock and waiting for index lock.

Fix:
====
1) Release the indexes lock before releasing the dict operation lock.

2) Ignore the cascading dml operation on the parent table, for the
cascading foreign key, if the child table is truncated or if it is
in the process of being truncated.
Reviewed-by: Jimmy Yang <jimmy.yang@oracle.com>
Reviewed-by: Kevin Lewis <kevin.lewis@oracle.com>
RB: 16122

storage/innobase/fil/fil0fil.cc

@@ -3292,15 +3292,17 @@ fil_prepare_for_truncate(
 
 /** Reinitialize the original tablespace header with the same space id
 for single tablespace
-@param[in]      id              space id of the tablespace
+@param[in]      table		table belongs to tablespace
 @param[in]      size            size in blocks
 @param[in]      trx             Transaction covering truncate */
 void
-fil_reinit_space_header(
-	ulint		id,
+fil_reinit_space_header_for_table(
+	dict_table_t*	table,
 	ulint		size,
 	trx_t*		trx)
 {
+	ulint	id = table->space;
+
 	ut_a(!is_system_tablespace(id));
 
 	/* Invalidate in the buffer pool all pages belonging
@@ -3309,6 +3311,9 @@ fil_reinit_space_header(
 	and the dict operation lock during the scan and aquire
 	it again after the buffer pool scan.*/
 
+	/* Release the lock on the indexes too. So that
+	they won't violate the latch ordering. */
+	dict_table_x_unlock_indexes(table);
 	row_mysql_unlock_data_dictionary(trx);
 
 	/* Lock the search latch in shared mode to prevent user
@@ -3317,8 +3322,11 @@ fil_reinit_space_header(
 	DEBUG_SYNC_C("buffer_pool_scan");
 	buf_LRU_flush_or_remove_pages(id, BUF_REMOVE_ALL_NO_WRITE, 0);
 	btr_search_s_unlock_all();
+
 	row_mysql_lock_data_dictionary(trx);
 
+	dict_table_x_lock_indexes(table);
+
 	/* Remove all insert buffer entries for the tablespace */
 	ibuf_delete_for_discarded_space(id);
 

storage/innobase/include/fil0fil.h

@@ -963,12 +963,12 @@ fil_prepare_for_truncate(
 
 /** Reinitialize the original tablespace header with the same space id
 for single tablespace
-@param[in]	id		space id of the tablespace
+@param[in]	table		table belongs to the tablespace
 @param[in]	size            size in blocks
 @param[in]	trx		Transaction covering truncate */
 void
-fil_reinit_space_header(
-	ulint		id,
+fil_reinit_space_header_for_table(
+	dict_table_t*	table,
 	ulint		size,
 	trx_t*		trx);
 

storage/innobase/row/row0trunc.cc

@@ -2018,7 +2018,7 @@ row_truncate_table_for_mysql(
 			space_size -= ib_vector_size(table->fts->indexes);
 		}
 
-		fil_reinit_space_header(table->space, space_size, trx);
+		fil_reinit_space_header_for_table(table, space_size, trx);
 	}
 
 	DBUG_EXECUTE_IF("ib_trunc_crash_with_intermediate_log_checkpoint",

storage/innobase/row/row0upd.cc

 /*****************************************************************************
 
-Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2015, 2016, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -256,6 +256,9 @@ row_upd_check_references_constraints(
 		row_mysql_freeze_data_dictionary(trx);
 	}
 
+	DEBUG_SYNC_C_IF_THD(thr_get_trx(thr)->mysql_thd,
+			    "foreign_constraint_check_for_insert");
+
 	for (dict_foreign_set::iterator it = table->referenced_set.begin();
 	     it != table->referenced_set.end();
 	     ++it) {
@@ -283,6 +286,34 @@ row_upd_check_references_constraints(
 					FALSE, FALSE, DICT_ERR_IGNORE_NONE);
 			}
 
+			/* dict_operation_lock is held both here
+			(UPDATE or DELETE with FOREIGN KEY) and by TRUNCATE
+			TABLE operations.
+			If a TRUNCATE TABLE operation is in progress,
+			there can be 2 possible conditions:
+			1) row_truncate_table_for_mysql() is not yet called.
+			2) Truncate releases dict_operation_lock
+			during eviction of pages from buffer pool
+			for a file-per-table tablespace.
+
+			In case of (1), truncate will wait for FK operation
+			to complete.
+			In case of (2), truncate will be rolled forward even
+			if it is interrupted. So if the foreign table is
+			undergoing a truncate, ignore the FK check. */
+
+			if (foreign_table) {
+				mutex_enter(&fil_system->mutex);
+				const fil_space_t* space = fil_space_get_by_id(
+					foreign_table->space);
+				const bool being_truncated = space
+					&& space->is_being_truncated;
+				mutex_exit(&fil_system->mutex);
+				if (being_truncated) {
+					continue;
+				}
+			}
+
 			/* NOTE that if the thread ends up waiting for a lock
 			we will release dict_operation_lock temporarily!
 			But the counter on the table protects 'foreign' from