MDEV-31000 Assertion failed on ALTER TABLE...page_compressed=1

ha_innobase::check_if_supported_inplace_alter(): On ALTER_OPTIONS,
if innodb_file_per_table=1 and the table resides in the system tablespace,
require that the table be rebuilt (and moved to an .ibd file).

Reviewed by: Thirunarayanan Balathandayuthapani
Tested by: Matthias Leich

mysql-test/suite/innodb/r/alter_table.result

@@ -117,5 +117,16 @@ ERROR 42000: Incorrect column specifier for column 'c'
 CREATE TABLE t1 (c DATETIME AUTO_INCREMENT UNIQUE) ENGINE=InnoDB;
 ERROR 42000: Incorrect column specifier for column 'c'
 #
+# MDEV-31000 Assertion failed on ALTER TABLE...page_compressed=1
+#
+SET @save_file_per_table=@@GLOBAL.innodb_file_per_table;
+SET GLOBAL innodb_file_per_table=0;
+CREATE TABLE t (c INT PRIMARY KEY) ENGINE=INNODB;
+SET GLOBAL innodb_file_per_table=1;
+ALTER TABLE t page_compressed=1;
+SET GLOBAL innodb_file_per_table=@save_file_per_table;
+SELECT space>0 FROM information_schema.innodb_sys_tables WHERE name='test/t';
+space>0
+1
+DROP TABLE t;
 # End of 10.4 tests
-#

mysql-test/suite/innodb/t/alter_table.test

@@ -121,6 +121,16 @@ CREATE TABLE t1 (c TIMESTAMP AUTO_INCREMENT UNIQUE) ENGINE=InnoDB;
 CREATE TABLE t1 (c DATETIME AUTO_INCREMENT UNIQUE) ENGINE=InnoDB;
 
 --echo #
---echo # End of 10.4 tests
+--echo # MDEV-31000 Assertion failed on ALTER TABLE...page_compressed=1
 --echo #
 
+SET @save_file_per_table=@@GLOBAL.innodb_file_per_table;
+SET GLOBAL innodb_file_per_table=0;
+CREATE TABLE t (c INT PRIMARY KEY) ENGINE=INNODB;
+SET GLOBAL innodb_file_per_table=1;
+ALTER TABLE t page_compressed=1;
+SET GLOBAL innodb_file_per_table=@save_file_per_table;
+SELECT space>0 FROM information_schema.innodb_sys_tables WHERE name='test/t';
+DROP TABLE t;
+
+--echo # End of 10.4 tests

storage/innobase/handler/handler0alter.cc

@@ -2155,12 +2155,16 @@ ha_innobase::check_if_supported_inplace_alter(
 		}
 	}
 
+	bool need_rebuild = false;
+
 	switch (ha_alter_info->handler_flags & ~INNOBASE_INPLACE_IGNORE) {
 	case ALTER_OPTIONS:
-		if (alter_options_need_rebuild(ha_alter_info, table)) {
+		if ((srv_file_per_table && !m_prebuilt->table->space_id)
+		    || alter_options_need_rebuild(ha_alter_info, table)) {
 			reason_rebuild = my_get_err_msg(
 				ER_ALTER_OPERATION_TABLE_OPTIONS_NEED_REBUILD);
 			ha_alter_info->unsupported_reason = reason_rebuild;
+			need_rebuild= true;
 			break;
 		}
 		/* fall through */
@@ -2272,7 +2276,7 @@ ha_innobase::check_if_supported_inplace_alter(
 
 	/* We should be able to do the operation in-place.
 	See if we can do it online (LOCK=NONE) or without rebuild. */
-	bool online = true, need_rebuild = false;
+	bool online = true;
 	const uint fulltext_indexes = innobase_fulltext_exist(altered_table);
 
 	/* Fix the key parts. */