Bug#14628410 - ASSERTION `! IS_SET()' FAILED IN DIAGNOSTICS_AREA::SET_OK_STATUS

The error code returned from Merge file/Temp file creation functions are
ignored.

Use the return codes of the row_merge_file_create() and innobase_mysql_tmpfile()
to return the error to caller if file creation fails.

Approved by Marko. rb#1618

storage/innodb_plugin/handler/ha_innodb.cc

@@ -929,6 +929,8 @@ convert_error_code_to_mysql(
 #endif /* HA_ERR_TOO_MANY_CONCURRENT_TRXS */
 	case DB_UNSUPPORTED:
 		return(HA_ERR_UNSUPPORTED);
+	case DB_OUT_OF_MEMORY:
+		return(HA_ERR_OUT_OF_MEM);
 	}
 }
 
@@ -1138,6 +1140,8 @@ innobase_mysql_tmpfile(void)
 
 	DBUG_ENTER("innobase_mysql_tmpfile");
 
+	DBUG_EXECUTE_IF("innobase_tmpfile_creation_failure", return(-1););
+
 	tmpdir = my_tmpdir(&mysql_tmpdir_list);
 
 	/* The tmpdir parameter can not be NULL for GetTempFileName. */
@@ -1200,6 +1204,9 @@ innobase_mysql_tmpfile(void)
 {
 	int	fd2 = -1;
 	File	fd = mysql_tmpfile("ib");
+
+	DBUG_EXECUTE_IF("innobase_tmpfile_creation_failure", return(-1););
+
 	if (fd >= 0) {
 		/* Copy the file descriptor, so that the additional resources
 		allocated by create_temp_file() can be freed by invoking

storage/innodb_plugin/row/row0merge.c

@@ -702,6 +702,8 @@ row_merge_read(
 	ib_uint64_t	ofs = ((ib_uint64_t) offset) * sizeof *buf;
 	ibool		success;
 
+	DBUG_EXECUTE_IF("row_merge_read_failure", return(FALSE););
+
 #ifdef UNIV_DEBUG
 	if (row_merge_print_block_read) {
 		fprintf(stderr, "row_merge_read fd=%d ofs=%lu\n",
@@ -738,6 +740,8 @@ row_merge_write(
 	ib_uint64_t	ofs = ((ib_uint64_t) offset)
 		* sizeof(row_merge_block_t);
 
+	DBUG_EXECUTE_IF("row_merge_write_failure", return(FALSE););
+
 #ifdef UNIV_DEBUG
 	if (row_merge_print_block_write) {
 		fprintf(stderr, "row_merge_write fd=%d ofs=%lu\n",
@@ -2189,9 +2193,10 @@ row_merge_drop_temp_indexes(void)
 }
 
 /*********************************************************************//**
-Create a merge file. */
-static
-void
+Create a merge file.
+@return file descriptor, or -1 on failure */
+static __attribute__((nonnull, warn_unused_result))
+int
 row_merge_file_create(
 /*==================*/
 	merge_file_t*	merge_file)	/*!< out: merge file structure */
@@ -2199,6 +2204,7 @@ row_merge_file_create(
 	merge_file->fd = innobase_mysql_tmpfile();
 	merge_file->offset = 0;
 	merge_file->n_rec = 0;
+	return(merge_file->fd);
 }
 
 /*********************************************************************//**
@@ -2619,7 +2625,7 @@ row_merge_build_indexes(
 	ulint			block_size;
 	ulint			i;
 	ulint			error;
-	int			tmpfd;
+	int			tmpfd = -1;
 
 	ut_ad(trx);
 	ut_ad(old_table);
@@ -2638,11 +2644,21 @@ row_merge_build_indexes(
 
 	for (i = 0; i < n_indexes; i++) {
 
-		row_merge_file_create(&merge_files[i]);
+		if (row_merge_file_create(&merge_files[i]) < 0)
+		{
+			error = DB_OUT_OF_MEMORY;
+			goto func_exit;
+		}
 	}
 
 	tmpfd = innobase_mysql_tmpfile();
 
+	if (tmpfd < 0)
+	{
+		error = DB_OUT_OF_MEMORY;
+		goto func_exit;
+	}
+
 	/* Reset the MySQL row buffer that is used when reporting
 	duplicate keys. */
 	innobase_rec_reset(table);