Merge 10.2 into 10.3

mysql-test/suite/innodb/r/ibuf_not_empty.result

+SET GLOBAL innodb_purge_rseg_truncate_frequency=1;
 CREATE TABLE t1(
 a INT AUTO_INCREMENT PRIMARY KEY,
 b CHAR(1),
@@ -5,6 +6,7 @@ c INT,
 INDEX(b))
 ENGINE=InnoDB STATS_PERSISTENT=0;
 SET GLOBAL innodb_change_buffering_debug = 1;
+BEGIN;
 INSERT INTO t1 VALUES(0,'x',1);
 INSERT INTO t1 SELECT 0,b,c FROM t1;
 INSERT INTO t1 SELECT 0,b,c FROM t1;
@@ -18,11 +20,11 @@ INSERT INTO t1 SELECT 0,b,c FROM t1;
 INSERT INTO t1 SELECT 0,b,c FROM t1;
 INSERT INTO t1 SELECT 0,b,c FROM t1;
 INSERT INTO t1 SELECT 0,b,c FROM t1;
+COMMIT;
+InnoDB		0 transactions not purged
 check table t1;
 Table	Op	Msg_type	Msg_text
 test.t1	check	Warning	InnoDB: Index 'b' contains #### entries, should be 4096.
 test.t1	check	error	Corrupt
-SET GLOBAL innodb_purge_rseg_truncate_frequency=1;
-InnoDB		0 transactions not purged
 SET GLOBAL innodb_fast_shutdown=0;
 DROP TABLE t1;

mysql-test/suite/innodb/t/ibuf_not_empty.test

@@ -6,6 +6,7 @@
 # The test is not big enough to use change buffering with larger page size.
 --source include/have_innodb_max_16k.inc
 
+SET GLOBAL innodb_purge_rseg_truncate_frequency=1;
 --disable_query_log
 call mtr.add_suppression("InnoDB: Failed to find tablespace for table `test`\\.`t1` in the cache\\. Attempting to load the tablespace with space id");
 call mtr.add_suppression("InnoDB: Allocated tablespace ID \\d+ for test.t1, old maximum was");
@@ -29,6 +30,7 @@ SET GLOBAL innodb_change_buffering_debug = 1;
 # Create enough rows for the table, so that the change buffer will be
 # used for modifying the secondary index page. There must be multiple
 # index pages, because changes to the root page are never buffered.
+BEGIN;
 INSERT INTO t1 VALUES(0,'x',1);
 INSERT INTO t1 SELECT 0,b,c FROM t1;
 INSERT INTO t1 SELECT 0,b,c FROM t1;
@@ -42,9 +44,13 @@ INSERT INTO t1 SELECT 0,b,c FROM t1;
 INSERT INTO t1 SELECT 0,b,c FROM t1;
 INSERT INTO t1 SELECT 0,b,c FROM t1;
 INSERT INTO t1 SELECT 0,b,c FROM t1;
+COMMIT;
 let MYSQLD_DATADIR=`select @@datadir`;
 let PAGE_SIZE=`select @@innodb_page_size`;
 
+# Ensure that purge will not access the truncated .ibd file
+--source include/wait_all_purged.inc
+
 --source include/shutdown_mysqld.inc
 
 # Corrupt the change buffer bitmap, to claim that pages are clean
@@ -75,14 +81,32 @@ EOF
 --replace_regex /contains \d+ entries/contains #### entries/
 check table t1;
 
---let $restart_parameters=
---source include/restart_mysqld.inc
+--source include/shutdown_mysqld.inc
 
-# Ensure that the slow shutdown will not time out due to running purge.
-SET GLOBAL innodb_purge_rseg_truncate_frequency=1;
---source include/wait_all_purged.inc
-# The change buffer merge for the injected corruption must complete
-# without exceeding the 60-second shutdown_server timeout.
+# Truncate the file to 5 pages, as if it were empty
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/crc32.pl";
+my $file = "$ENV{MYSQLD_DATADIR}/test/t1.ibd";
+open(FILE, "+<$file") || die "Unable to open $file";
+binmode FILE;
+my $ps= $ENV{PAGE_SIZE};
+my $pages=5;
+my $page;
+die "Unable to read $file" unless sysread(FILE, $page, $ps) == $ps;
+substr($page,46,4)=pack("N", $pages);
+my $polynomial = 0x82f63b78; # CRC-32C
+my $ck= pack("N",mycrc32(substr($page, 4, 22), 0, $polynomial) ^
+		 mycrc32(substr($page, 38, $ps - 38 - 8), 0, $polynomial));
+substr($page,0,4)=$ck;
+substr($page,$ps-8,4)=$ck;
+sysseek(FILE, 0, 0) || die "Unable to rewind $file\n";
+syswrite(FILE, $page, $ps)==$ps || die "Unable to write $file\n";
+truncate(FILE, $ps * $pages);
+close(FILE) || die "Unable to close $file";
+EOF
+
+--let $restart_parameters=
+--source include/start_mysqld.inc
 SET GLOBAL innodb_fast_shutdown=0;
 --source include/restart_mysqld.inc
 

storage/innobase/buf/buf0rea.cc

@@ -819,11 +819,8 @@ buf_read_ibuf_merge_pages(
 #endif
 
 	for (ulint i = 0; i < n_stored; i++) {
-		bool			found;
-		const page_size_t	page_size(fil_space_get_page_size(
-			space_ids[i], &found));
-
-		if (!found) {
+		fil_space_t* space = fil_space_acquire_silent(space_ids[i]);
+		if (!space) {
 tablespace_deleted:
 			/* The tablespace was not found: remove all
 			entries for it */
@@ -835,6 +832,19 @@ buf_read_ibuf_merge_pages(
 			continue;
 		}
 
+		if (UNIV_UNLIKELY(page_nos[i] >= space->size)) {
+			do {
+				ibuf_delete_recs(page_id_t(space_ids[i],
+							   page_nos[i]));
+			} while (++i < n_stored
+				 && space_ids[i - 1] == space_ids[i]
+				 && page_nos[i] >= space->size);
+			i--;
+next:
+			space->release();
+			continue;
+		}
+
 		const page_id_t	page_id(space_ids[i], page_nos[i]);
 
 		buf_pool_t*	buf_pool = buf_pool_get(page_id);
@@ -849,8 +859,8 @@ buf_read_ibuf_merge_pages(
 		buf_read_page_low(&err,
 				  sync && (i + 1 == n_stored),
 				  0,
-				  BUF_READ_ANY_PAGE, page_id, page_size,
-				  true, true /* ignore_missing_space */);
+				  BUF_READ_ANY_PAGE, page_id,
+				  page_size_t(space->flags), true);
 
 		switch(err) {
 		case DB_SUCCESS:
@@ -858,15 +868,20 @@ buf_read_ibuf_merge_pages(
 		case DB_ERROR:
 			break;
 		case DB_TABLESPACE_DELETED:
+			space->release();
 			goto tablespace_deleted;
 		case DB_PAGE_CORRUPTED:
 		case DB_DECRYPTION_FAILED:
-			ib::error() << "Failed to read or decrypt " << page_id
-				<< " for change buffer merge";
+			ib::error() << "Failed to read or decrypt page "
+				    << page_nos[i]
+				    << " of '" << space->chain.start->name
+				    << "' for change buffer merge";
 			break;
 		default:
 			ut_error;
 		}
+
+		goto next;
 	}
 
 	os_aio_simulated_wake_handler_threads();

storage/innobase/dict/dict0crea.cc

@@ -919,17 +919,13 @@ dict_create_index_tree_in_mem(
 /** Drop the index tree associated with a row in SYS_INDEXES table.
 @param[in,out]	rec	SYS_INDEXES record
 @param[in,out]	pcur	persistent cursor on rec
+@param[in,out]	trx	dictionary transaction
 @param[in,out]	mtr	mini-transaction
 @return	whether freeing the B-tree was attempted */
-bool
-dict_drop_index_tree(
-	rec_t*		rec,
-	btr_pcur_t*	pcur,
-	mtr_t*		mtr)
+bool dict_drop_index_tree(rec_t* rec, btr_pcur_t* pcur, trx_t* trx, mtr_t* mtr)
 {
 	const byte*	ptr;
 	ulint		len;
-	ulint		space;
 	ulint		root_page_no;
 
 	ut_ad(mutex_own(&dict_sys->mutex));
@@ -956,7 +952,14 @@ dict_drop_index_tree(
 
 	ut_ad(len == 4);
 
-	space = mtr_read_ulint(ptr, MLOG_4BYTES, mtr);
+	const uint32_t space_id = mach_read_from_4(ptr);
+	ut_ad(space_id < SRV_TMP_SPACE_ID);
+	if (space_id != TRX_SYS_SPACE
+	    && trx_get_dict_operation(trx) == TRX_DICT_OP_TABLE) {
+		/* We are about to delete the entire .ibd file;
+		do not bother to free pages inside it. */
+		return false;
+	}
 
 	ptr = rec_get_nth_field_old(
 		rec, DICT_FLD__SYS_INDEXES__ID, &len);
@@ -964,7 +967,7 @@ dict_drop_index_tree(
 	ut_ad(len == 8);
 
 	bool			found;
-	const page_size_t	page_size(fil_space_get_page_size(space,
+	const page_size_t	page_size(fil_space_get_page_size(space_id,
 								  &found));
 
 	if (!found) {
@@ -977,11 +980,11 @@ dict_drop_index_tree(
 	/* If tablespace is scheduled for truncate, do not try to drop
 	the indexes in that tablespace. There is a truncate fixup action
 	which will take care of it. */
-	if (srv_is_tablespace_truncated(space)) {
+	if (srv_is_tablespace_truncated(space_id)) {
 		return(false);
 	}
 
-	btr_free_if_exists(page_id_t(space, root_page_no), page_size,
+	btr_free_if_exists(page_id_t(space_id, root_page_no), page_size,
 			   mach_read_from_8(ptr), mtr);
 
 	return(true);

storage/innobase/ibuf/ibuf0ibuf.cc

@@ -4319,7 +4319,7 @@ This prevents an infinite loop on slow shutdown
 in the case where the change buffer bitmap claims that no buffered
 changes exist, while entries exist in the change buffer tree.
 @param page_id  page number for which there should be no unbuffered changes */
-ATTRIBUTE_COLD static void ibuf_delete_recs(const page_id_t page_id)
+ATTRIBUTE_COLD void ibuf_delete_recs(const page_id_t page_id)
 {
 	ulint dops[IBUF_OP_COUNT];
 	mtr_t mtr;

storage/innobase/include/dict0crea.h

 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, 2018, MariaDB Corporation.
+Copyright (c) 2017, 2019, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -123,13 +123,11 @@ dict_recreate_index_tree(
 /** Drop the index tree associated with a row in SYS_INDEXES table.
 @param[in,out]	rec	SYS_INDEXES record
 @param[in,out]	pcur	persistent cursor on rec
+@param[in,out]	trx	dictionary transaction
 @param[in,out]	mtr	mini-transaction
 @return	whether freeing the B-tree was attempted */
-bool
-dict_drop_index_tree(
-	rec_t*		rec,
-	btr_pcur_t*	pcur,
-	mtr_t*		mtr);
+bool dict_drop_index_tree(rec_t* rec, btr_pcur_t* pcur, trx_t* trx, mtr_t* mtr)
+	MY_ATTRIBUTE((nonnull));
 
 /***************************************************************//**
 Creates an index tree for the index if it is not a member of a cluster.

storage/innobase/include/ibuf0ibuf.h

@@ -323,6 +323,14 @@ ibuf_insert(
 	const page_size_t&	page_size,
 	que_thr_t*		thr);
 
+/**
+Delete any buffered entries for a page.
+This prevents an infinite loop on slow shutdown
+in the case where the change buffer bitmap claims that no buffered
+changes exist, while entries exist in the change buffer tree.
+@param page_id  page number for which there should be no unbuffered changes */
+ATTRIBUTE_COLD void ibuf_delete_recs(const page_id_t page_id);
+
 /** When an index page is read from a disk to the buffer pool, this function
 applies any buffered operations to the page and deletes the entries from the
 insert buffer. If the page is not read, but created in the buffer pool, this

storage/innobase/row/row0trunc.cc

@@ -448,14 +448,10 @@ class DropIndex : public Callback {
 	Constructor
 
 	@param[in,out]	table	Table to truncate
+	@param[in,out]	trx	dictionary transaction
 	@param[in]	noredo	whether to disable redo logging */
-	DropIndex(dict_table_t* table, bool noredo)
-		:
-		Callback(table->id, noredo),
-		m_table(table)
-	{
-		/* No op */
-	}
+	DropIndex(dict_table_t* table, trx_t* trx, bool noredo)
+		: Callback(table->id, noredo), m_trx(trx), m_table(table) {}
 
 	/**
 	@param mtr	mini-transaction covering the read
@@ -464,8 +460,10 @@ class DropIndex : public Callback {
 	dberr_t operator()(mtr_t* mtr, btr_pcur_t* pcur) const;
 
 private:
+	/** dictionary transaction */
+	trx_t* const		m_trx;
 	/** Table to be truncated */
-	dict_table_t*		m_table;
+	dict_table_t* const	m_table;
 };
 
 /** Callback to create the indexes during TRUNCATE */
@@ -553,7 +551,7 @@ DropIndex::operator()(mtr_t* mtr, btr_pcur_t* pcur) const
 {
 	rec_t*	rec = btr_pcur_get_rec(pcur);
 
-	bool	freed = dict_drop_index_tree(rec, pcur, mtr);
+	bool	freed = dict_drop_index_tree(rec, pcur, m_trx, mtr);
 
 #ifdef UNIV_DEBUG
 	{

storage/innobase/row/row0uins.cc

@@ -128,7 +128,8 @@ row_undo_ins_remove_clust_rec(
 		ut_ad(node->rec_type == TRX_UNDO_INSERT_REC);
 
 		dict_drop_index_tree(
-			btr_pcur_get_rec(&node->pcur), &(node->pcur), &mtr);
+			btr_pcur_get_rec(&node->pcur), &node->pcur, node->trx,
+			&mtr);
 
 		mtr.commit();
 

storage/innobase/row/row0upd.cc

@@ -3134,7 +3134,7 @@ row_upd_clust_step(
 		ut_ad(!dict_index_is_online_ddl(index));
 
 		dict_drop_index_tree(
-			btr_pcur_get_rec(pcur), pcur, &mtr);
+			btr_pcur_get_rec(pcur), pcur, trx, &mtr);
 
 		mtr.commit();