MDEV-34529 Shrink the system tablespace when system tablespace contains...

MDEV-34529  Shrink the system tablespace when system tablespace contains MDEV-30671 leaked undo pages

- InnoDB fails to shrink the system tablespace when it contains
the leaked undo log pages caused by MDEV-30671.

- InnoDB does free the unused segment in system tablespace
before shrinking the tablespace.

inode_info: Structure to store the inode page and offsets.

fil_space_t::garbage_collect(): Frees the system tablespace
unused segment

fsp_free_unused_seg(): Frees the unused segment

fsp_get_sys_used_segment(): Iterates through all default
file segment and index segment present in system tablespace.

fseg_inode_free(): Frees the extents, fragment pages for the
given index node and ignores any error similar to
trx_purge_free_segment()

trx_sys_t::reset_page(): Retain the TRX_SYS_FSEG_HEADER value
in trx_sys page while resetting the page.

mysql-test/suite/innodb/r/undo_leak.result

+# restart: --debug_dbug=d,undo_segment_leak
+SET GLOBAL INNODB_FILE_PER_TABLE=0;
+Warnings:
+Warning	1287	'@@innodb_file_per_table' is deprecated and will be removed in a future release
+CREATE TABLE t1(f1 INT NOT NULL, f2 INT NOT NULL)ENGINE=InnoDB;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+UPDATE t1 SET f1 = f1 + 1 WHERE f1 > 1000;
+UPDATE t1 SET f2 = f2 + 1 WHERE f1 > 1000;
+UPDATE t1 SET f1 = f2 + 1 WHERE f1 > 1000;
+UPDATE t1 SET f2 = f1 + 1 WHERE f1 > 1000;
+DELETE FROM t1;
+DROP TABLE t1;
+set GLOBAL innodb_fast_shutdown=0;
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+NAME	FILE_SIZE
+innodb_system	79691776
+# restart
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+NAME	FILE_SIZE
+innodb_system	12582912

mysql-test/suite/innodb/r/undo_leak_fail.result

+call mtr.add_suppression("InnoDB: Can't free the unused segments in system tablespace because a previous shutdown was not with innodb_fast_shutdown=0");
+call mtr.add_suppression("InnoDB: Cannot free the unused segment in system tablespace due to Data structure corruption. Ignorning the autoshrink option");
+call mtr.add_suppression("InnoDB: Failed to free the unused segment");
+# restart: --debug_dbug=d,undo_segment_leak
+SET GLOBAL INNODB_FILE_PER_TABLE=0;
+Warnings:
+Warning	1287	'@@innodb_file_per_table' is deprecated and will be removed in a future release
+CREATE TABLE t1(f1 INT NOT NULL, f2 INT NOT NULL)ENGINE=InnoDB;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+UPDATE t1 SET f1 = f1 + 1 WHERE f1 > 1000;
+UPDATE t1 SET f2 = f2 + 1 WHERE f1 > 1000;
+UPDATE t1 SET f1 = f2 + 1 WHERE f1 > 1000;
+UPDATE t1 SET f2 = f1 + 1 WHERE f1 > 1000;
+DELETE FROM t1;
+DROP TABLE t1;
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+NAME	FILE_SIZE
+innodb_system	79691776
+# restart: --debug_dbug=d,unused_undo_free_fail_1
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+NAME	FILE_SIZE
+innodb_system	79691776
+# restart: --debug_dbug=d,unused_undo_free_fail_2
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+NAME	FILE_SIZE
+innodb_system	79691776
+# restart: --debug_dbug=d,unused_undo_free_fail_3
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+NAME	FILE_SIZE
+innodb_system	79691776
+# restart: --debug_dbug=d,unused_undo_free_fail_4
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+NAME	FILE_SIZE
+innodb_system	79691776
+# restart: --debug_dbug=d,unused_undo_free_fail_5
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+NAME	FILE_SIZE
+innodb_system	79691776
+# restart: --debug_dbug=d,unused_undo_free_fail_6
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+NAME	FILE_SIZE
+innodb_system	79691776
+# restart
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+NAME	FILE_SIZE
+innodb_system	79691776

mysql-test/suite/innodb/t/undo_leak.opt

+--innodb_undo_tablespaces=0
+--innodb_sys_tablespaces

mysql-test/suite/innodb/t/undo_leak.test

+--source include/have_innodb.inc
+--source include/have_sequence.inc
+--source include/have_debug.inc
+
+let $restart_parameters=--debug_dbug=d,undo_segment_leak;
+--source include/restart_mysqld.inc
+SET GLOBAL INNODB_FILE_PER_TABLE=0;
+CREATE TABLE t1(f1 INT NOT NULL, f2 INT NOT NULL)ENGINE=InnoDB;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+UPDATE t1 SET f1 = f1 + 1 WHERE f1 > 1000;
+UPDATE t1 SET f2 = f2 + 1 WHERE f1 > 1000;
+UPDATE t1 SET f1 = f2 + 1 WHERE f1 > 1000;
+UPDATE t1 SET f2 = f1 + 1 WHERE f1 > 1000;
+DELETE FROM t1;
+DROP TABLE t1;
+set GLOBAL innodb_fast_shutdown=0;
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+let $restart_parameters=;
+--source include/restart_mysqld.inc
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;

mysql-test/suite/innodb/t/undo_leak_fail.opt

+--innodb_undo_tablespaces=0
+--innodb_sys_tablespaces

mysql-test/suite/innodb/t/undo_leak_fail.test

+--source include/have_innodb.inc
+--source include/have_sequence.inc
+--source include/have_debug.inc
+
+call mtr.add_suppression("InnoDB: Can't free the unused segments in system tablespace because a previous shutdown was not with innodb_fast_shutdown=0");
+call mtr.add_suppression("InnoDB: Cannot free the unused segment in system tablespace due to Data structure corruption. Ignorning the autoshrink option");
+call mtr.add_suppression("InnoDB: Failed to free the unused segment");
+
+let $restart_parameters=--debug_dbug=d,undo_segment_leak;
+--source include/restart_mysqld.inc
+SET GLOBAL INNODB_FILE_PER_TABLE=0;
+CREATE TABLE t1(f1 INT NOT NULL, f2 INT NOT NULL)ENGINE=InnoDB;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096;
+UPDATE t1 SET f1 = f1 + 1 WHERE f1 > 1000;
+UPDATE t1 SET f2 = f2 + 1 WHERE f1 > 1000;
+UPDATE t1 SET f1 = f2 + 1 WHERE f1 > 1000;
+UPDATE t1 SET f2 = f1 + 1 WHERE f1 > 1000;
+DELETE FROM t1;
+DROP TABLE t1;
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+let $restart_parameters=--debug_dbug=d,unused_undo_free_fail_1;
+--source include/restart_mysqld.inc
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+
+let $restart_parameters=--debug_dbug=d,unused_undo_free_fail_2;
+--source include/restart_mysqld.inc
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+
+let $restart_parameters=--debug_dbug=d,unused_undo_free_fail_3;
+--source include/restart_mysqld.inc
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+
+let $restart_parameters=--debug_dbug=d,unused_undo_free_fail_4;
+--source include/restart_mysqld.inc
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+
+let $restart_parameters=--debug_dbug=d,unused_undo_free_fail_5;
+--source include/restart_mysqld.inc
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+
+let $restart_parameters=--debug_dbug=d,unused_undo_free_fail_6;
+--source include/restart_mysqld.inc
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+
+let $restart_parameters=;
+--source include/restart_mysqld.inc
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;

storage/innobase/dict/dict0load.cc

@@ -172,36 +172,24 @@ name_of_col_is(
 }
 #endif /* UNIV_DEBUG */
 
-/********************************************************************//**
-This function gets the next system table record as it scans the table.
-@return the next record if found, NULL if end of scan */
-static
 const rec_t*
-dict_getnext_system_low(
-/*====================*/
-	btr_pcur_t*	pcur,		/*!< in/out: persistent cursor to the
-					record*/
-	mtr_t*		mtr)		/*!< in: the mini-transaction */
+dict_getnext_system_low(btr_pcur_t *pcur, mtr_t *mtr)
 {
-	rec_t*	rec = NULL;
-
-	while (!rec) {
-		btr_pcur_move_to_next_user_rec(pcur, mtr);
-
-		rec = btr_pcur_get_rec(pcur);
-
-		if (!btr_pcur_is_on_user_rec(pcur)) {
-			/* end of index */
-			btr_pcur_close(pcur);
-
-			return(NULL);
-		}
-	}
-
-	/* Get a record, let's save the position */
-	btr_pcur_store_position(pcur, mtr);
-
-	return(rec);
+  rec_t *rec = NULL;
+  while (!rec)
+  {
+    btr_pcur_move_to_next_user_rec(pcur, mtr);
+    rec = btr_pcur_get_rec(pcur);
+    if (!btr_pcur_is_on_user_rec(pcur))
+    {
+      /* end of index */
+      btr_pcur_close(pcur);
+      return NULL;
+    }
+  }
+  /* Get a record, let's save the position */
+  btr_pcur_store_position(pcur, mtr);
+  return rec;
 }
 
 /********************************************************************//**

storage/innobase/include/dict0load.h

@@ -213,4 +213,12 @@ dict_process_sys_foreign_col_rec(
 					in referenced table */
 	ulint*		pos);		/*!< out: column position */
 
+/** This function gets the next system table record as it scans
+the table.
+@param pcur persistent cursor
+@param mtr  mini-transaction
+@return the next record if found
+@retval nullptr at the end of the table */
+const rec_t*
+dict_getnext_system_low(btr_pcur_t *pcur, mtr_t *mtr);
 #endif

storage/innobase/include/fil0fil.h

@@ -1009,6 +1009,10 @@ struct fil_space_t final
   /** Update the data structures on write completion */
   void complete_write();
 
+  /** Free the unused segment for the tablespace
+  @param shutdown called during slow shutdown
+  @return error code */
+  dberr_t garbage_collect(bool shutdown);
 private:
   /** @return whether the file is usable for io() */
   ATTRIBUTE_COLD bool prepare_acquired();

storage/innobase/include/fsp0fsp.h

@@ -555,8 +555,9 @@ inline void fsp_init_file_page(
 	mtr->init(block);
 }
 
-/** Truncate the system tablespace */
-void fsp_system_tablespace_truncate();
+/** Truncate the system tablespace
+@param shutdown Called during shutdown */
+void fsp_system_tablespace_truncate(bool shutdown);
 
 #ifndef UNIV_DEBUG
 # define fsp_init_file_page(space, block, mtr) fsp_init_file_page(block, mtr)

storage/innobase/srv/srv0srv.cc

@@ -1579,6 +1579,6 @@ void srv_purge_shutdown()
     srv_shutdown_purge_tasks();
     if (!srv_fast_shutdown && !high_level_read_only && srv_was_started &&
         !opt_bootstrap && srv_operation == SRV_OPERATION_NORMAL)
-      fsp_system_tablespace_truncate();
+      fsp_system_tablespace_truncate(true);
   }
 }

storage/innobase/srv/srv0start.cc

@@ -333,6 +333,15 @@ inline dberr_t trx_sys_t::reset_page(mtr_t *mtr)
 
   if (!sys_header) return err;
 
+  if (mach_read_from_4(sys_header->page.frame + TRX_SYS +
+                       TRX_SYS_FSEG_HEADER) != TRX_SYS_SPACE)
+    return DB_CORRUPTION;
+
+  /* Store the TRX_SYS_FSEG_HEADER page, offset */
+  char fseg_addr[6];
+  memcpy(fseg_addr,
+         sys_header->page.frame + TRX_SYS + TRX_SYS_FSEG_HEADER + 4, 6);
+
   const bool dblwr_enabled=
     mach_read_from_4(TRX_SYS_DOUBLEWRITE_MAGIC + TRX_SYS_DOUBLEWRITE +
                      sys_header->page.frame)
@@ -347,6 +356,9 @@ inline dberr_t trx_sys_t::reset_page(mtr_t *mtr)
   mtr->write<2>(*sys_header, FIL_PAGE_TYPE + sys_header->page.frame,
                 FIL_PAGE_TYPE_TRX_SYS);
 
+  mtr->memcpy(*sys_header,
+              sys_header->page.frame + TRX_SYS + TRX_SYS_FSEG_HEADER + 4,
+              fseg_addr, 6);
   mtr->write<4>(*sys_header,
                 TRX_SYS + TRX_SYS_RSEGS + TRX_SYS_RSEG_PAGE_NO +
                 sys_header->page.frame, FSP_FIRST_RSEG_PAGE_NO);
@@ -1783,7 +1795,7 @@ dberr_t srv_start(bool create_new_db)
 
 		if (!high_level_read_only
 		    && srv_sys_space.can_auto_shrink()) {
-			fsp_system_tablespace_truncate();
+			fsp_system_tablespace_truncate(false);
 			DBUG_EXECUTE_IF("crash_after_sys_truncate",
 					return srv_init_abort(DB_ERROR););
 		}

storage/innobase/trx/trx0purge.cc

@@ -482,6 +482,7 @@ inline dberr_t purge_sys_t::iterator::free_history_rseg(trx_rseg_t &rseg) const
     free_segment:
       ut_ad(rseg.curr_size >= seg_size);
       rseg.curr_size-= seg_size;
+      DBUG_EXECUTE_IF("undo_segment_leak", goto skip_purge_free;);
       trx_purge_free_segment(rseg_hdr, b, mtr);
       break;
     case TRX_UNDO_CACHED:
@@ -507,7 +508,9 @@ inline dberr_t purge_sys_t::iterator::free_history_rseg(trx_rseg_t &rseg) const
       goto free_segment;
     }
   }
-
+#ifndef DBUG_OFF
+skip_purge_free:
+#endif /* !DBUG_OFF */
   hdr_addr= prev_hdr_addr;
 
   mtr.commit();