MDEV-18543 IMPORT TABLESPACE fails after instant DROP COLUMN

ALTER TABLE IMPORT doesn't properly handle instant alter metadata.
This patch makes IMPORT read, parse and apply instant alter metadata at the
very beginning of operation. So, cases when source table has some metadata
and destination table doesn't have it now works fine.

DISCARD already removes instant metadata so importing normal table into
instant table worked fine before this patch.

decrypt_decompress(): decrypts and decompresses page if needed

handle_instant_metadata(): this should be the first thing to read source
table. Basically, it applies instant metadata to a destination
dict_table_t object. This is the first thing to read FSP flags so
all possible checks of it were moved to this function.

PageConverter::update_index_page(): it doesn't now read instant metadata.
This logic were moved into handle_instant_metadata()

row_import::match_flags(): this is a first part row_import::match_schema().
As a separate function it's used by handle_instant_metadata().

fil_space_t::is_full_crc32_compressed(): added convenient function

ha_innobase::discard_or_import_tablespace(): do not reload table definition
to read instant metadata because handle_instant_metadata() does it better.
The reverted code was originally added in
4e7ee166

ANONYMOUS_VAR: this is a handy thing to use along with make_scope_exit()

full_crc32_import.test shows different results, because no
dict_table_close() and dict_table_open_on_id() happens.
Thus, SHOW CREATE TABLE shows a little bit older table definition.

include/scope.h

@@ -64,3 +64,11 @@ make_scope_exit(Callable &&f)
   return detail::scope_exit<typename std::decay<Callable>::type>(
       std::forward<Callable>(f));
 }
+
+#define CONCAT_IMPL(x, y) x##y
+
+#define CONCAT(x, y) CONCAT_IMPL(x, y)
+
+#define ANONYMOUS_VARIABLE CONCAT(_anonymous_variable, __LINE__)
+
+#define SCOPE_EXIT auto ANONYMOUS_VARIABLE= make_scope_exit

mysql-test/suite/encryption/r/instant_alter_import.result

+SET GLOBAL innodb_encrypt_tables = ON;
+SET GLOBAL innodb_encryption_threads = 4;
+CREATE TABLE t1 (id INT PRIMARY KEY AUTO_INCREMENT, i2 INT, i1 INT)
+ENGINE=INNODB ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+INSERT INTO t1 (i2) SELECT 4 FROM seq_1_to_1024;
+ALTER TABLE t1 DROP COLUMN i2, ALGORITHM=INSTANT;
+CREATE TABLE t2 LIKE t1;
+ALTER TABLE t2 DISCARD TABLESPACE;
+FLUSH TABLE t1 FOR EXPORT;
+UNLOCK TABLES;
+ALTER TABLE t2 IMPORT TABLESPACE;
+DROP TABLE t2, t1;
+CREATE TABLE t1 (id INT PRIMARY KEY AUTO_INCREMENT, i2 INT, i1 INT)
+ENGINE=INNODB ENCRYPTED=YES ENCRYPTION_KEY_ID=4 PAGE_COMPRESSED=1;
+INSERT INTO t1 (i2) SELECT 4 FROM seq_1_to_1024;
+ALTER TABLE t1 DROP COLUMN i2, ALGORITHM=INSTANT;
+CREATE TABLE t2 LIKE t1;
+ALTER TABLE t2 DISCARD TABLESPACE;
+FLUSH TABLE t1 FOR EXPORT;
+UNLOCK TABLES;
+ALTER TABLE t2 IMPORT TABLESPACE;
+DROP TABLE t2, t1;
+SET GLOBAL innodb_encrypt_tables = OFF;
+SET GLOBAL innodb_encryption_threads = 0;

mysql-test/suite/encryption/t/instant_alter_import.test

+--source include/have_innodb.inc
+--source include/have_sequence.inc
+--source include/have_file_key_management_plugin.inc
+
+SET GLOBAL innodb_encrypt_tables = ON;
+SET GLOBAL innodb_encryption_threads = 4;
+
+
+--let $MYSQLD_DATADIR= `SELECT @@datadir`
+
+CREATE TABLE t1 (id INT PRIMARY KEY AUTO_INCREMENT, i2 INT, i1 INT)
+  ENGINE=INNODB ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+
+INSERT INTO t1 (i2) SELECT 4 FROM seq_1_to_1024;
+ALTER TABLE t1 DROP COLUMN i2, ALGORITHM=INSTANT;
+
+CREATE TABLE t2 LIKE t1;
+ALTER TABLE t2 DISCARD TABLESPACE;
+
+FLUSH TABLE t1 FOR EXPORT;
+
+--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_DATADIR/test/t2.ibd
+--copy_file $MYSQLD_DATADIR/test/t1.cfg $MYSQLD_DATADIR/test/t2.cfg
+
+UNLOCK TABLES;
+ALTER TABLE t2 IMPORT TABLESPACE;
+
+DROP TABLE t2, t1;
+
+
+CREATE TABLE t1 (id INT PRIMARY KEY AUTO_INCREMENT, i2 INT, i1 INT)
+  ENGINE=INNODB ENCRYPTED=YES ENCRYPTION_KEY_ID=4 PAGE_COMPRESSED=1;
+
+INSERT INTO t1 (i2) SELECT 4 FROM seq_1_to_1024;
+ALTER TABLE t1 DROP COLUMN i2, ALGORITHM=INSTANT;
+
+CREATE TABLE t2 LIKE t1;
+ALTER TABLE t2 DISCARD TABLESPACE;
+
+FLUSH TABLE t1 FOR EXPORT;
+
+--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_DATADIR/test/t2.ibd
+--copy_file $MYSQLD_DATADIR/test/t1.cfg $MYSQLD_DATADIR/test/t2.cfg
+
+UNLOCK TABLES;
+ALTER TABLE t2 IMPORT TABLESPACE;
+
+DROP TABLE t2, t1;
+
+
+SET GLOBAL innodb_encrypt_tables = OFF;
+SET GLOBAL innodb_encryption_threads = 0;

mysql-test/suite/innodb/r/default_row_format_compatibility.result

+call mtr.add_suppression("Index for table 'tab' is corrupt; try to repair it");
 SET @row_format = @@GLOBAL.innodb_default_row_format;
 # ###########################################################
 # Check with Import/Export tablespace with Default_row_format
@@ -38,7 +39,7 @@ tab	InnoDB	#	Compact	#	#	#	#	#	#	NULL	#	NULL	NULL	latin1_swedish_ci	NULL			0	N
 ALTER TABLE tab DISCARD TABLESPACE;
 call mtr.add_suppression("InnoDB: Tried to read .* bytes at offset 0");
 ALTER TABLE tab IMPORT TABLESPACE;
-ERROR HY000: Internal error: Cannot reset LSNs in table `test`.`tab` : I/O error
+ERROR HY000: Index for table 'tab' is corrupt; try to repair it
 ALTER TABLE tab IMPORT TABLESPACE;
 SELECT * FROM tab;
 a

mysql-test/suite/innodb/r/full_crc32_import.result

@@ -46,7 +46,7 @@ t1	CREATE TABLE `t1` (
   `b` blob DEFAULT NULL,
   `c` blob DEFAULT NULL,
   PRIMARY KEY (`a`)
-) ENGINE=InnoDB AUTO_INCREMENT=46 DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC
+) ENGINE=InnoDB AUTO_INCREMENT=57 DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC
 UPDATE t1 set b = repeat("de", 100) where b = repeat("cd", 200);
 explain SELECT a FROM t1 where b = repeat("de", 100);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
@@ -126,7 +126,7 @@ t1	CREATE TABLE `t1` (
   `c2` point NOT NULL,
   `c3` linestring NOT NULL,
   PRIMARY KEY (`c1`)
-) ENGINE=InnoDB AUTO_INCREMENT=14325 DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC
+) ENGINE=InnoDB AUTO_INCREMENT=16372 DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC
 UPDATE t1 SET C2 = ST_GeomFromText('POINT(0 0)');
 SELECT COUNT(*) FROM t1;
 COUNT(*)

mysql-test/suite/innodb/r/innodb-wl5522-debug.result

@@ -951,7 +951,7 @@ ERROR HY000: Tablespace has been discarded for table `t1`
 restore: t1 .ibd and .cfg files
 SET SESSION debug_dbug="+d,fsp_flags_is_valid_failure";
 ALTER TABLE t1 IMPORT TABLESPACE;
-ERROR HY000: Internal error: Cannot reset LSNs in table `test`.`t1` : Data structure corruption
+ERROR HY000: Index for table 't1' is corrupt; try to repair it
 SET SESSION debug_dbug=@saved_debug_dbug;
 DROP TABLE t1;
 unlink: t1.ibd

mysql-test/suite/innodb/r/instant_alter_import.result

@@ -61,13 +61,10 @@ alter table t1 discard tablespace;
 flush tables t2 for export;
 unlock tables;
 alter table t1 import tablespace;
-ERROR HY000: Schema mismatch (Index field count 4 doesn't match tablespace metadata file value 5)
 select * from t1;
-ERROR HY000: Tablespace has been discarded for table `t1`
-alter table t1 import tablespace;
-ERROR HY000: Internal error: Cannot reset LSNs in table `test`.`t1` : Unsupported
-select * from t1;
-ERROR HY000: Tablespace has been discarded for table `t1`
+z
+42
+41
 drop table t2;
 drop table t1;
 CREATE TABLE t1 (id INT PRIMARY KEY AUTO_INCREMENT, i1 INT) ENGINE=INNODB;
@@ -78,3 +75,46 @@ FLUSH TABLE t1 FOR EXPORT;
 UNLOCK TABLES;
 ALTER TABLE t2 IMPORT TABLESPACE;
 DROP TABLE t2, t1;
+CREATE TABLE t1 (id INT PRIMARY KEY, i2 INT, i1 INT) ENGINE=INNODB;
+INSERT INTO t1 VALUES (1, 1, 1);
+ALTER TABLE t1 MODIFY COLUMN i2 INT AFTER i1, ALGORITHM=INSTANT;
+CREATE TABLE t2 LIKE t1;
+ALTER TABLE t2 DISCARD TABLESPACE;
+FLUSH TABLE t1 FOR EXPORT;
+UNLOCK TABLES;
+ALTER TABLE t2 IMPORT TABLESPACE;
+SELECT * FROM t2;
+id	i1	i2
+1	1	1
+DROP TABLE t2, t1;
+CREATE TABLE t1 (id INT PRIMARY KEY, i2 INT, i1 INT) ENGINE=INNODB;
+INSERT INTO t1 VALUES (1, 1, 1);
+ALTER TABLE t1 DROP COLUMN i2, ALGORITHM=INSTANT;
+CREATE TABLE t2 LIKE t1;
+ALTER TABLE t2 DISCARD TABLESPACE;
+FLUSH TABLE t1 FOR EXPORT;
+UNLOCK TABLES;
+ALTER TABLE t2 IMPORT TABLESPACE;
+SELECT * FROM t2;
+id	i1
+1	1
+DROP TABLE t2, t1;
+CREATE TABLE t1 (id INT PRIMARY KEY, i2 INT, i1 INT)
+ENGINE=INNODB PAGE_COMPRESSED=1;
+INSERT INTO t1 VALUES (1, 1, 1);
+ALTER TABLE t1 DROP COLUMN i2, ALGORITHM=INSTANT;
+CREATE TABLE t2 LIKE t1;
+ALTER TABLE t2 DISCARD TABLESPACE;
+FLUSH TABLE t1 FOR EXPORT;
+UNLOCK TABLES;
+ALTER TABLE t2 IMPORT TABLESPACE;
+DROP TABLE t2, t1;
+CREATE TABLE t1 (id INT PRIMARY KEY AUTO_INCREMENT, i2 INT, i1 INT) ENGINE=INNODB;
+INSERT INTO t1 (i2) SELECT 4 FROM seq_1_to_1024;
+ALTER TABLE t1 DROP COLUMN i2, ALGORITHM=INSTANT;
+CREATE TABLE t2 LIKE t1;
+ALTER TABLE t2 DISCARD TABLESPACE;
+FLUSH TABLE t1 FOR EXPORT;
+UNLOCK TABLES;
+ALTER TABLE t2 IMPORT TABLESPACE;
+DROP TABLE t2, t1;

mysql-test/suite/innodb/t/default_row_format_compatibility.test

 --source include/have_innodb.inc
 
+call mtr.add_suppression("Index for table 'tab' is corrupt; try to repair it");
+
+
 SET @row_format = @@GLOBAL.innodb_default_row_format;
 
 # set the variables
@@ -79,7 +82,7 @@ ALTER TABLE tab DISCARD TABLESPACE;
 
 call mtr.add_suppression("InnoDB: Tried to read .* bytes at offset 0");
 
---error ER_INTERNAL_ERROR
+--error ER_NOT_KEYFILE
 ALTER TABLE tab IMPORT TABLESPACE;
 --remove_file $MYSQLD_DATADIR/test/tab.ibd
 --move_file $MYSQLD_DATADIR/tab.ibd $MYSQLD_DATADIR/test/tab.ibd

mysql-test/suite/innodb/t/innodb-wl5522-debug.test

@@ -1375,7 +1375,7 @@ SET SESSION debug_dbug="+d,fsp_flags_is_valid_failure";
 
 --replace_regex /'.*t1.cfg'/'t1.cfg'/
 
---error ER_INTERNAL_ERROR
+--error ER_NOT_KEYFILE
 ALTER TABLE t1 IMPORT TABLESPACE;
 
 SET SESSION debug_dbug=@saved_debug_dbug;

mysql-test/suite/innodb/t/instant_alter_import.test

 --source include/have_innodb.inc
+--source include/have_sequence.inc
+--source include/innodb_checksum_algorithm.inc
+
 set default_storage_engine=innodb;
 
 --echo #
@@ -70,14 +73,7 @@ flush tables t2 for export;
 --copy_file $MYSQLD_DATADIR/test/t2.ibd $MYSQLD_DATADIR/test/t1.ibd
 unlock tables;
 
---error ER_TABLE_SCHEMA_MISMATCH
-alter table t1 import tablespace;
---error ER_TABLESPACE_DISCARDED
-select * from t1;
---remove_file $MYSQLD_DATADIR/test/t1.cfg
---error ER_INTERNAL_ERROR
 alter table t1 import tablespace;
---error ER_TABLESPACE_DISCARDED
 select * from t1;
 
 drop table t2;
@@ -101,3 +97,86 @@ UNLOCK TABLES;
 ALTER TABLE t2 IMPORT TABLESPACE;
 
 DROP TABLE t2, t1;
+
+
+CREATE TABLE t1 (id INT PRIMARY KEY, i2 INT, i1 INT) ENGINE=INNODB;
+
+INSERT INTO t1 VALUES (1, 1, 1);
+ALTER TABLE t1 MODIFY COLUMN i2 INT AFTER i1, ALGORITHM=INSTANT;
+
+CREATE TABLE t2 LIKE t1;
+ALTER TABLE t2 DISCARD TABLESPACE;
+
+
+FLUSH TABLE t1 FOR EXPORT;
+
+--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_DATADIR/test/t2.ibd
+--copy_file $MYSQLD_DATADIR/test/t1.cfg $MYSQLD_DATADIR/test/t2.cfg
+
+UNLOCK TABLES;
+ALTER TABLE t2 IMPORT TABLESPACE;
+
+SELECT * FROM t2;
+
+DROP TABLE t2, t1;
+
+
+CREATE TABLE t1 (id INT PRIMARY KEY, i2 INT, i1 INT) ENGINE=INNODB;
+
+INSERT INTO t1 VALUES (1, 1, 1);
+ALTER TABLE t1 DROP COLUMN i2, ALGORITHM=INSTANT;
+
+CREATE TABLE t2 LIKE t1;
+ALTER TABLE t2 DISCARD TABLESPACE;
+
+
+FLUSH TABLE t1 FOR EXPORT;
+
+--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_DATADIR/test/t2.ibd
+--copy_file $MYSQLD_DATADIR/test/t1.cfg $MYSQLD_DATADIR/test/t2.cfg
+
+UNLOCK TABLES;
+ALTER TABLE t2 IMPORT TABLESPACE;
+
+SELECT * FROM t2;
+
+DROP TABLE t2, t1;
+
+
+CREATE TABLE t1 (id INT PRIMARY KEY, i2 INT, i1 INT)
+  ENGINE=INNODB PAGE_COMPRESSED=1;
+
+INSERT INTO t1 VALUES (1, 1, 1);
+ALTER TABLE t1 DROP COLUMN i2, ALGORITHM=INSTANT;
+
+CREATE TABLE t2 LIKE t1;
+ALTER TABLE t2 DISCARD TABLESPACE;
+
+FLUSH TABLE t1 FOR EXPORT;
+
+--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_DATADIR/test/t2.ibd
+--copy_file $MYSQLD_DATADIR/test/t1.cfg $MYSQLD_DATADIR/test/t2.cfg
+
+UNLOCK TABLES;
+ALTER TABLE t2 IMPORT TABLESPACE;
+
+DROP TABLE t2, t1;
+
+
+CREATE TABLE t1 (id INT PRIMARY KEY AUTO_INCREMENT, i2 INT, i1 INT) ENGINE=INNODB;
+
+INSERT INTO t1 (i2) SELECT 4 FROM seq_1_to_1024;
+ALTER TABLE t1 DROP COLUMN i2, ALGORITHM=INSTANT;
+
+CREATE TABLE t2 LIKE t1;
+ALTER TABLE t2 DISCARD TABLESPACE;
+
+FLUSH TABLE t1 FOR EXPORT;
+
+--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_DATADIR/test/t2.ibd
+--copy_file $MYSQLD_DATADIR/test/t1.cfg $MYSQLD_DATADIR/test/t2.cfg
+
+UNLOCK TABLES;
+ALTER TABLE t2 IMPORT TABLESPACE;
+
+DROP TABLE t2, t1;

mysql-test/suite/innodb_zip/r/wl5522_debug_zip.result

@@ -407,7 +407,7 @@ ERROR HY000: Tablespace has been discarded for table `t1`
 restore: t1 .ibd and .cfg files
 SET SESSION debug_dbug="+d,fsp_flags_is_valid_failure";
 ALTER TABLE t1 IMPORT TABLESPACE;
-ERROR HY000: Internal error: Cannot reset LSNs in table `test`.`t1` : Data structure corruption
+ERROR HY000: Index for table 't1' is corrupt; try to repair it
 SET SESSION debug_dbug=@saved_debug_dbug;
 DROP TABLE t1;
 unlink: t1.ibd

mysql-test/suite/innodb_zip/t/wl5522_debug_zip.test

@@ -624,7 +624,7 @@ EOF
 
 SET SESSION debug_dbug="+d,fsp_flags_is_valid_failure";
 
---error ER_INTERNAL_ERROR
+--error ER_NOT_KEYFILE
 ALTER TABLE t1 IMPORT TABLESPACE;
 
 SET SESSION debug_dbug=@saved_debug_dbug;

storage/innobase/handler/ha_innodb.cc

@@ -13117,25 +13117,6 @@ ha_innobase::discard_or_import_tablespace(
 				    err, m_prebuilt->table->flags, NULL));
 	}
 
-	/* Evict and reload the table definition in order to invoke
-	btr_cur_instant_init(). */
-	table_id_t id = m_prebuilt->table->id;
-	ut_ad(id);
-	mutex_enter(&dict_sys.mutex);
-	dict_table_close(m_prebuilt->table, TRUE, FALSE);
-	dict_sys.remove(m_prebuilt->table);
-	m_prebuilt->table = dict_table_open_on_id(id, TRUE,
-						  DICT_TABLE_OP_NORMAL);
-	mutex_exit(&dict_sys.mutex);
-	if (!m_prebuilt->table) {
-		err = DB_TABLE_NOT_FOUND;
-	} else {
-		if (const Field* ai = table->found_next_number_field) {
-			initialize_auto_increment(m_prebuilt->table, ai);
-		}
-		dict_stats_init(m_prebuilt->table);
-	}
-
 	if (dict_stats_is_persistent_enabled(m_prebuilt->table)) {
 		dberr_t		ret;
 

storage/innobase/include/fil0fil.h

@@ -316,6 +316,18 @@ struct fil_space_t
 	static bool full_crc32(ulint flags) {
 		return flags & FSP_FLAGS_FCRC32_MASK_MARKER;
 	}
+  /** Determine if full_crc32 is used along with compression */
+  static bool is_full_crc32_compressed(ulint flags)
+  {
+    if (full_crc32(flags))
+    {
+      ulint algo= FSP_FLAGS_FCRC32_GET_COMPRESSED_ALGO(flags);
+      DBUG_ASSERT(algo <= PAGE_ALGORITHM_LAST);
+      return algo > 0;
+    }
+
+    return false;
+  }
 	/** @return whether innodb_checksum_algorithm=full_crc32 is active */
 	bool full_crc32() const { return full_crc32(flags); }
 	/** Determine the logical page size.
@@ -379,16 +391,10 @@ struct fil_space_t
 	unsigned physical_size() const { return physical_size(flags); }
   /** Check whether the compression enabled in tablespace.
   @param[in]	flags	tablespace flags */
-	static bool is_compressed(ulint flags) {
-
-		if (full_crc32(flags)) {
-			ulint algo = FSP_FLAGS_FCRC32_GET_COMPRESSED_ALGO(
-				flags);
-			DBUG_ASSERT(algo <= PAGE_ALGORITHM_LAST);
-			return algo > 0;
-		}
-
-		return FSP_FLAGS_HAS_PAGE_COMPRESSION(flags);
+  static bool is_compressed(ulint flags)
+  {
+    return is_full_crc32_compressed(flags)
+        || FSP_FLAGS_HAS_PAGE_COMPRESSION(flags);
   }
 	/** @return whether the compression enabled for the tablespace. */
 	bool is_compressed() const { return is_compressed(flags); }