Fixed BUG#15653, BUG#16582.

  Applied innodb-4.1-ss20 snapshot.


innobase/btr/btr0sea.c:
  Applied innodb-4.1-ss20 snapshot.
    Account for a race condition when dropping the adaptive
    hash index for a B-tree page (Bug #16582).
    btr_search_drop_page_hash_index(): Retry the operation if a hash
    index with different parameters was built meanwhile. Add
    diagnostics for the case that hash node pointers to the page
    remain. This fix is from Heikki.
    btr_search_info_update_hash(), btr_search_info_update_slow():
    Document the parameter "info" as in/out.
innobase/fil/fil0fil.c:
  Applied innodb-4.1-ss20 snapshot.
    Keep track on unflushed modifications to file spaces. When
    there are tens of thousnads of file spaces, flushing all files
    in fil_flush_file_spaces() would be very slow (Bug #16582).
    fil_flush_file_spaces(): Only flush unflushed file spaces.
    fil_space_t, fil_system_t: Add a list of unflushed spaces.
innobase/include/btr0sea.ic:
  Applied innodb-4.1-ss20 snapshot.
    btr_search_info_update_hash(), btr_search_info_update_slow():
    Document the parameter "info" as in/out.

innobase/btr/btr0sea.c

@@ -191,7 +191,7 @@ static
 void
 btr_search_info_update_hash(
 /*========================*/
-	btr_search_t*	info,	/* in: search info */
+	btr_search_t*	info,	/* in/out: search info */
 	btr_cur_t*	cursor)	/* in: cursor which was just positioned */
 {
 	dict_index_t*	index;
@@ -443,7 +443,7 @@ Updates the search info. */
 void
 btr_search_info_update_slow(
 /*========================*/
-	btr_search_t*	info,	/* in: search info */
+	btr_search_t*	info,	/* in/out: search info */
 	btr_cur_t*	cursor)	/* in: cursor which was just positioned */
 {
 	buf_block_t*	block;
@@ -931,7 +931,7 @@ btr_search_drop_page_hash_index(
 	ut_ad(!rw_lock_own(&btr_search_latch, RW_LOCK_SHARED));
 	ut_ad(!rw_lock_own(&btr_search_latch, RW_LOCK_EX));
 #endif /* UNIV_SYNC_DEBUG */
-
+retry:
 	rw_lock_s_lock(&btr_search_latch);
 
 	block = buf_block_align(page);
@@ -1007,6 +1007,24 @@ btr_search_drop_page_hash_index(
 
 	rw_lock_x_lock(&btr_search_latch);
 
+	if (!block->is_hashed) {
+		/* Someone else has meanwhile dropped the hash index */
+
+		goto cleanup;
+	}
+
+	if (block->curr_n_fields != n_fields
+	    || block->curr_n_bytes != n_bytes) {
+
+		/* Someone else has meanwhile built a new hash index on the
+		page, with different parameters */
+
+		rw_lock_x_unlock(&btr_search_latch);
+
+		mem_free(folds);
+		goto retry;
+	}
+
 	for (i = 0; i < n_cached; i++) {
 
 		ha_remove_all_nodes_to_page(table, folds[i], page);
@@ -1014,7 +1032,22 @@ btr_search_drop_page_hash_index(
 
 	block->is_hashed = FALSE;
 
-	rw_lock_x_unlock(&btr_search_latch);
+cleanup:
+	if (block->n_pointers) {
+		/* Corruption */
+		ut_print_timestamp(stderr);
+		fprintf(stderr,
+"  InnoDB: Corruption of adaptive hash index. After dropping\n"
+"InnoDB: the hash index to a page of %lu %lu, still %lu hash nodes remain.\n",
+			(ulong) ut_dulint_get_high(tree_id),
+			(ulong) ut_dulint_get_low(tree_id),
+			(ulong) block->n_pointers);
+		rw_lock_x_unlock(&btr_search_latch);
+
+		btr_search_validate();
+	} else {
+		rw_lock_x_unlock(&btr_search_latch);
+	}
 
 	mem_free(folds);
 }

innobase/fil/fil0fil.c

@@ -179,6 +179,11 @@ struct fil_space_struct {
 	hash_node_t	name_hash;/* hash chain the name_hash table */
 	rw_lock_t	latch;	/* latch protecting the file space storage
 				allocation */
+	UT_LIST_NODE_T(fil_space_t) unflushed_spaces;
+				/* list of spaces with at least one unflushed
+				file we have written to */
+	ibool		is_in_unflushed_spaces; /* TRUE if this space is
+				currently in the list above */
 	UT_LIST_NODE_T(fil_space_t) space_list;
 				/* list of all spaces */
 	ibuf_data_t*	ibuf_data;
@@ -211,6 +216,12 @@ struct fil_system_struct {
 					not put to this list: they are opened
 					after the startup, and kept open until
 					shutdown */
+	UT_LIST_BASE_NODE_T(fil_space_t) unflushed_spaces;
+					/* base node for the list of those
+					tablespaces whose files contain
+					unflushed writes; those spaces have
+					at least one file node where
+					modification_counter > flush_counter */
 	ulint		n_open;		/* number of files currently open */
 	ulint		max_n_open;	/* n_open is not allowed to exceed
 					this */
@@ -387,6 +398,36 @@ fil_space_get_ibuf_data(
 	return(space->ibuf_data);
 }
 
+/**************************************************************************
+Checks if all the file nodes in a space are flushed. The caller must hold
+the fil_system mutex. */
+static
+ibool
+fil_space_is_flushed(
+/*=================*/
+				/* out: TRUE if all are flushed */
+	fil_space_t*	space)	/* in: space */
+{
+	fil_node_t*	node;
+
+#ifdef UNIV_SYNC_DEBUG
+	ut_ad(mutex_own(&(fil_system->mutex)));
+#endif /* UNIV_SYNC_DEBUG */
+
+	node = UT_LIST_GET_FIRST(space->chain);
+
+	while (node) {
+		if (node->modification_counter > node->flush_counter) {
+
+			return(FALSE);
+		}
+
+		node = UT_LIST_GET_NEXT(chain, node);
+	}
+
+	return(TRUE);
+}
+
 /***********************************************************************
 Appends a new file to the chain of files of a space. File must be closed. */
 
@@ -517,7 +558,7 @@ fil_node_open_file(
 		if (size_bytes < FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE) {
 	        	fprintf(stderr,
 "InnoDB: Error: the size of single-table tablespace file %s\n"
-"InnoDB: is only %lu %lu, should be at least %lu!", node->name,
+"InnoDB: is only %lu %lu, should be at least %lu!\n", node->name,
 			(ulong) size_high,
 			(ulong) size_low, (ulong) (4 * UNIV_PAGE_SIZE));
 			
@@ -687,8 +728,8 @@ fil_try_to_close_file_in_LRU(
 			ut_print_filename(stderr, node->name);
 			fprintf(stderr,
 				", because mod_count %ld != fl_count %ld\n",
-				(ulong) node->modification_counter,
-				(ulong) node->flush_counter);
+				(long) node->modification_counter,
+				(long) node->flush_counter);
 		}
 
 		node = UT_LIST_GET_PREV(LRU, node);
@@ -839,6 +880,16 @@ fil_node_free(
 
 		node->modification_counter = node->flush_counter;
 
+		if (space->is_in_unflushed_spaces
+		    && fil_space_is_flushed(space)) {
+
+			space->is_in_unflushed_spaces = FALSE;
+
+			UT_LIST_REMOVE(unflushed_spaces,
+					system->unflushed_spaces,
+					space);
+		}
+
 		fil_node_close_file(node, system);
 	}
 
@@ -1002,6 +1053,8 @@ fil_space_create(
 
 	HASH_INSERT(fil_space_t, name_hash, system->name_hash,
 						ut_fold_string(name), space);
+	space->is_in_unflushed_spaces = FALSE;
+
 	UT_LIST_ADD_LAST(space_list, system->space_list, space);
 				
 	mutex_exit(&(system->mutex));
@@ -1097,6 +1150,13 @@ fil_space_free(
 	HASH_DELETE(fil_space_t, name_hash, system->name_hash,
 					   ut_fold_string(space->name), space);
 
+	if (space->is_in_unflushed_spaces) {
+		space->is_in_unflushed_spaces = FALSE;
+
+		UT_LIST_REMOVE(unflushed_spaces, system->unflushed_spaces,
+								space);
+	}
+
 	UT_LIST_REMOVE(space_list, system->space_list, space);
 
 	ut_a(space->magic_n == FIL_SPACE_MAGIC_N);
@@ -1248,6 +1308,7 @@ fil_system_create(
 
 	system->tablespace_version = 0;
 
+	UT_LIST_INIT(system->unflushed_spaces);
 	UT_LIST_INIT(system->space_list);
 
 	return(system);
@@ -2612,12 +2673,12 @@ fil_open_single_table_tablespace(
 		fputs("!\n"
 "InnoDB: Have you moved InnoDB .ibd files around without using the\n"
 "InnoDB: commands DISCARD TABLESPACE and IMPORT TABLESPACE?\n"
-"InnoDB: It is also possible that this is a table created with\n"
-"InnoDB: CREATE TEMPORARY TABLE, and MySQL removed the .ibd file for this.\n"
+"InnoDB: It is also possible that this is a temporary table #sql...,\n"
+"InnoDB: and MySQL removed the .ibd file for this.\n"
 "InnoDB: Please refer to\n"
 "InnoDB:"
 " http://dev.mysql.com/doc/mysql/en/InnoDB_troubleshooting_datadict.html\n"
-"InnoDB: how to resolve the issue.\n", stderr);
+"InnoDB: for how to resolve the issue.\n", stderr);
 
 		mem_free(filepath);
 
@@ -2657,7 +2718,7 @@ fil_open_single_table_tablespace(
 "InnoDB: Please refer to\n"
 "InnoDB:"
 " http://dev.mysql.com/doc/mysql/en/InnoDB_troubleshooting_datadict.html\n"
-"InnoDB: how to resolve the issue.\n", (ulong) space_id, (ulong) id);
+"InnoDB: for how to resolve the issue.\n", (ulong) space_id, (ulong) id);
 
 		ret = FALSE;
 
@@ -3292,7 +3353,7 @@ fil_space_for_table_exists_in_mem(
 			ut_print_filename(stderr, name);
 			fprintf(stderr, "\n"
 "InnoDB: in InnoDB data dictionary has tablespace id %lu,\n"
-"InnoDB: but tablespace with that id does not exist. There is\n"
+"InnoDB: but a tablespace with that id does not exist. There is\n"
 "InnoDB: a tablespace of name %s and id %lu, though. Have\n"
 "InnoDB: you deleted or moved .ibd files?\n",
 				(ulong) id, namespace->name,
@@ -3303,7 +3364,7 @@ fil_space_for_table_exists_in_mem(
 "InnoDB: Please refer to\n"
 "InnoDB:"
 " http://dev.mysql.com/doc/mysql/en/InnoDB_troubleshooting_datadict.html\n"
-"InnoDB: how to resolve the issue.\n", stderr);
+"InnoDB: for how to resolve the issue.\n", stderr);
 
 		mem_free(path);
 		mutex_exit(&(system->mutex));
@@ -3317,7 +3378,7 @@ fil_space_for_table_exists_in_mem(
 		ut_print_filename(stderr, name);
 		fprintf(stderr, "\n"
 "InnoDB: in InnoDB data dictionary has tablespace id %lu,\n"
-"InnoDB: but tablespace with that id has name %s.\n"
+"InnoDB: but the tablespace with that id has name %s.\n"
 "InnoDB: Have you deleted or moved .ibd files?\n", (ulong) id, space->name);
 
 		if (namespace != NULL) {
@@ -3732,6 +3793,14 @@ fil_node_complete_io(
 	if (type == OS_FILE_WRITE) {
 		system->modification_counter++;
 		node->modification_counter = system->modification_counter;
+
+		if (!node->space->is_in_unflushed_spaces) {
+
+			node->space->is_in_unflushed_spaces = TRUE;
+			UT_LIST_ADD_FIRST(unflushed_spaces,
+					system->unflushed_spaces,
+					node->space);
+		}
 	}
 	
 	if (node->n_pending == 0 && node->space->purpose == FIL_TABLESPACE
@@ -4145,6 +4214,16 @@ fil_flush(
 skip_flush:
 			if (node->flush_counter < old_mod_counter) {
 				node->flush_counter = old_mod_counter;
+
+				if (space->is_in_unflushed_spaces
+				    && fil_space_is_flushed(space)) {
+
+					space->is_in_unflushed_spaces = FALSE;
+
+					UT_LIST_REMOVE(unflushed_spaces,
+						system->unflushed_spaces,
+						space);
+				}
 			}
 
 			if (space->purpose == FIL_TABLESPACE) {
@@ -4176,7 +4255,7 @@ fil_flush_file_spaces(
 
 	mutex_enter(&(system->mutex));
 
-	space = UT_LIST_GET_FIRST(system->space_list);
+	space = UT_LIST_GET_FIRST(system->unflushed_spaces);
 
 	while (space) {
 		if (space->purpose == purpose && !space->is_being_deleted) {
@@ -4192,7 +4271,7 @@ fil_flush_file_spaces(
 
 			space->n_pending_flushes--;
 		}
-		space = UT_LIST_GET_NEXT(space_list, space);
+		space = UT_LIST_GET_NEXT(unflushed_spaces, space);
 	}
 	
 	mutex_exit(&(system->mutex));

innobase/include/btr0sea.ic

@@ -16,7 +16,7 @@ Updates the search info. */
 void
 btr_search_info_update_slow(
 /*========================*/
-	btr_search_t*	info,	/* in: search info */
+	btr_search_t*	info,	/* in/out: search info */
 	btr_cur_t*	cursor);/* in: cursor which was just positioned */
 
 /************************************************************************