MDEV-15333 MariaDB (still) slow start

This performance regression was introduced in the MariaDB 10.1
file format incompatibility bug fix MDEV-11623 (MariaDB 10.1.21
and MariaDB 10.2.4) and partially fixed in MariaDB 10.1.25 in
MDEV-12610 without adding a regression test case.

On a normal startup (without crash recovery), InnoDB should not read
every .ibd data file, because this is slow. Like in MySQL, for now,
InnoDB will still open every data file (without reading), and it
will read every .ibd file for which an .isl file exists, or the
DATA DIRECTORY attribute has been specified for the table.

The test case shuts down InnoDB, moves data files, replaces them
with garbage, and then restarts InnoDB, expecting no messages to
be issued for the garbage files. (Some messages will for now be
issued for the table that uses the DATA DIRECTORY attribute.)
Finally, the test shuts down the server, restores the old data files,
and restarts again to drop the tables.

fil_open_single_table_tablespace(): Remove the condition on flags,
and only call fsp_flags_try_adjust() if validate==true
(reading the first page has been requested). The only caller with
validate==false is at server startup when we are processing all
records from SYS_TABLES. The flags passed to this function are
actually derived from SYS_TABLES.TYPE and SYS_TABLES.N_COLS,
and there never was any problem with SYS_TABLES in MariaDB 10.1.
The problem that MDEV-11623 was that incorrect tablespace flags
were computed and written to FSP_SPACE_FLAGS.

mysql-test/suite/innodb/r/restart.result

+SET GLOBAL innodb_file_format=Barracuda;
+SET GLOBAL innodb_file_per_table=1;
+#
+# MDEV-15333 MariaDB (still) slow start
+#
+# FIXME: Unlike MySQL, maybe MariaDB should not read the .ibd files
+# of tables with .isl file or DATA DIRECTORY attribute.
+call mtr.add_suppression("\\[ERROR\\] InnoDB: Invalid flags 0x7a207879 in tablespace 2048948345");
+call mtr.add_suppression("\\[ERROR\\] InnoDB: Error invalid tablespace flags in file '.*td\\.ibd'");
+CREATE TABLE tr(a INT)ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+CREATE TABLE tc(a INT)ENGINE=InnoDB ROW_FORMAT=COMPACT
+PAGE_COMPRESSED=1 PAGE_COMPRESSION_LEVEL=9;
+CREATE TABLE td(a INT)ENGINE=InnoDB ROW_FORMAT=DYNAMIC
+STATS_PERSISTENT=0 DATA DIRECTORY='MYSQL_TMP_DIR';
+SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
+WHERE engine = 'innodb'
+AND support IN ('YES', 'DEFAULT', 'ENABLED');
+COUNT(*)
+1
+SELECT * FROM tr;
+a
+SELECT * FROM tc;
+a
+SELECT * FROM td;
+a
+DROP TABLE tr,tc,td;

mysql-test/suite/innodb/t/restart.test

+--source include/innodb_page_size.inc
+--source include/not_embedded.inc
+
+let datadir= `select @@datadir`;
+let page_size= `select @@innodb_page_size`;
+
+SET GLOBAL innodb_file_format=Barracuda;
+SET GLOBAL innodb_file_per_table=1;
+
+--echo #
+--echo # MDEV-15333 MariaDB (still) slow start
+--echo #
+
+# Ensure that on normal startup, no data files are read.
+# Note: just like in MySQL, all .ibd files will still be
+# opened at least once.
+
+--echo # FIXME: Unlike MySQL, maybe MariaDB should not read the .ibd files
+--echo # of tables with .isl file or DATA DIRECTORY attribute.
+call mtr.add_suppression("\\[ERROR\\] InnoDB: Invalid flags 0x7a207879 in tablespace 2048948345");
+call mtr.add_suppression("\\[ERROR\\] InnoDB: Error invalid tablespace flags in file '.*td\\.ibd'");
+
+CREATE TABLE tr(a INT)ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+CREATE TABLE tc(a INT)ENGINE=InnoDB ROW_FORMAT=COMPACT
+PAGE_COMPRESSED=1 PAGE_COMPRESSION_LEVEL=9;
+--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR
+eval CREATE TABLE td(a INT)ENGINE=InnoDB ROW_FORMAT=DYNAMIC
+STATS_PERSISTENT=0 DATA DIRECTORY='$MYSQL_TMP_DIR';
+
+--source include/shutdown_mysqld.inc
+
+--move_file $datadir/test/tr.ibd $datadir/test/tr0.ibd
+--move_file $datadir/test/tc.ibd $datadir/test/tc0.ibd
+--move_file $MYSQL_TMP_DIR/test/td.ibd $datadir/test/td0.ibd
+# TODO: test that MariaDB does not even attempt to open the files
+#--mkdir $datadir/test/tr.ibd
+#--mkdir $datadir/test/tc.ibd
+#--mkdir $MYSQL_TMP_DIR/test/td.ibd
+
+perl;
+die unless open OUT, ">", "$ENV{datadir}/test/tr.ibd";
+print OUT "foo " x $ENV{page_size};
+close OUT or die;
+die unless open OUT, ">", "$ENV{datadir}/test/tc.ibd";
+print OUT "bar " x $ENV{page_size};
+close OUT or die;
+die unless open OUT, ">", "$ENV{MYSQL_TMP_DIR}/test/td.ibd";
+print OUT "xyz " x $ENV{page_size};
+close OUT or die;
+EOF
+
+--let $restart_parameters= --skip-innodb-buffer-pool-load-at-startup
+--source include/start_mysqld.inc
+--let $restart_parameters=
+
+SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
+WHERE engine = 'innodb'
+AND support IN ('YES', 'DEFAULT', 'ENABLED');
+
+--source include/shutdown_mysqld.inc
+
+# TODO: test that MariaDB does not even attempt to open the files
+#--rmdir $datadir/test/tr.ibd
+#--rmdir $datadir/test/tc.ibd
+#--rmdir $MYSQL_TMP_DIR/test/td.ibd
+--remove_file $datadir/test/tr.ibd
+--remove_file $datadir/test/tc.ibd
+--remove_file $MYSQL_TMP_DIR/test/td.ibd
+
+--move_file $datadir/test/tr0.ibd $datadir/test/tr.ibd
+--move_file $datadir/test/tc0.ibd $datadir/test/tc.ibd
+--move_file $datadir/test/td0.ibd $MYSQL_TMP_DIR/test/td.ibd
+
+--source include/start_mysqld.inc
+SELECT * FROM tr;
+SELECT * FROM tc;
+SELECT * FROM td;
+DROP TABLE tr,tc,td;

storage/innobase/fil/fil0fil.cc

 /*****************************************************************************
 
 Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2014, 2017, MariaDB Corporation.
+Copyright (c) 2014, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -4278,17 +4278,7 @@ fil_open_single_table_tablespace(
 
 	mem_free(def.filepath);
 
-	/* We need to check fsp flags when no errors has happened and
-	server was not started on read only mode and tablespace validation
-	was requested or flags contain other table options except
-	low order bits to FSP_FLAGS_POS_PAGE_SSIZE position.
-	Note that flag comparison is pessimistic. Adjust is required
-	only when flags contain buggy MariaDB 10.1.0 -
-	MariaDB 10.1.20 flags.  */
-	if (err == DB_SUCCESS
-	    && !srv_read_only_mode
-	    && (validate
-		|| flags >= (1U << FSP_FLAGS_POS_PAGE_SSIZE))) {
+	if (err == DB_SUCCESS && validate && !srv_read_only_mode) {
 		fsp_flags_try_adjust(id, flags & ~FSP_FLAGS_MEM_MASK);
 	}
 

storage/xtradb/fil/fil0fil.cc

@@ -4468,17 +4468,7 @@ fil_open_single_table_tablespace(
 
 	mem_free(def.filepath);
 
-	/* We need to check fsp flags when no errors has happened and
-	server was not started on read only mode and tablespace validation
-	was requested or flags contain other table options except
-	low order bits to FSP_FLAGS_POS_PAGE_SSIZE position.
-	Note that flag comparison is pessimistic. Adjust is required
-	only when flags contain buggy MariaDB 10.1.0 -
-	MariaDB 10.1.20 flags.  */
-	if (err == DB_SUCCESS
-	    && !srv_read_only_mode
-	    && (validate
-		|| flags >= (1U << FSP_FLAGS_POS_PAGE_SSIZE))) {
+	if (err == DB_SUCCESS && validate && !srv_read_only_mode) {
 		fsp_flags_try_adjust(id, flags & ~FSP_FLAGS_MEM_MASK);
 	}