MDEV-12353: Replace MLOG_*_END_COPY_CREATED

Instead of writing the high-level redo log records
MLOG_LIST_END_COPY_CREATED, MLOG_COMP_LIST_END_COPY_CREATED
write log for each individual insert of a record.

page_copy_rec_list_end_to_created_page(): Remove.

This will improve the fill factor of some pages.
Adjust some tests accordingly.

PageBulk::init(), PageBulk::finish(): Avoid setting bogus limits
to PAGE_HEAP_TOP and PAGE_N_DIR_SLOTS. Avoid accessor functions
that would enforce these limits before the correct ones are set
at the end of PageBulk::finish().

mysql-test/suite/encryption/r/innodb-discard-import-change.result

@@ -100,5 +100,5 @@ NOT FOUND /verysecretmessage/ in t3.ibd
 # t4 page compressed and encrypted expecting NOT FOUND
 NOT FOUND /verysecretmessage/ in t4.ibd
 # t5 normal expecting FOUND
-FOUND 289 /verysecretmessage/ in t5.ibd
+FOUND 256 /verysecretmessage/ in t5.ibd
 DROP TABLE t1,t2,t3,t4,t5,t6;

mysql-test/suite/innodb/r/index_tree_operation.result

@@ -16,9 +16,10 @@ information_schema.innodb_buffer_page s2
 WHERE s1.space = s2.space AND name = 'test/t1'
 AND page_type = "INDEX" ORDER BY page_number;
 page_number	number_records
-3	2
+3	3
 4	3
-5	3
+5	1
+6	2
 INSERT INTO t1 VALUES (999, REPEAT('a', 4096));
 SELECT page_number, number_records
 FROM information_schema.innodb_sys_tablespaces s1,
@@ -28,8 +29,8 @@ AND page_type = "INDEX" ORDER BY page_number;
 page_number	number_records
 3	3
 4	3
-5	3
-6	1
+5	1
+6	3
 INSERT INTO t1 VALUES (998, REPEAT('a', 4096));
 SELECT page_number, number_records
 FROM information_schema.innodb_sys_tablespaces s1,
@@ -37,10 +38,11 @@ information_schema.innodb_buffer_page s2
 WHERE s1.space = s2.space AND name = 'test/t1'
 AND page_type = "INDEX" ORDER BY page_number;
 page_number	number_records
-3	3
+3	4
 4	3
-5	3
-6	2
+5	1
+6	3
+7	1
 INSERT INTO t1 VALUES (997, REPEAT('a', 4096));
 SELECT page_number, number_records
 FROM information_schema.innodb_sys_tablespaces s1,
@@ -48,8 +50,9 @@ information_schema.innodb_buffer_page s2
 WHERE s1.space = s2.space AND name = 'test/t1'
 AND page_type = "INDEX" ORDER BY page_number;
 page_number	number_records
-3	3
+3	4
 4	3
-5	3
+5	1
 6	3
+7	2
 DROP TABLE t1;

mysql-test/suite/innodb/r/innodb-change-buffer-recovery.result

@@ -13,6 +13,7 @@ c INT,
 INDEX(b))
 ENGINE=InnoDB STATS_PERSISTENT=0;
 SET GLOBAL innodb_change_buffering_debug = 1;
+SET GLOBAL innodb_limit_optimistic_insert_debug=700;
 INSERT INTO t1 SELECT 0,'x',1 FROM seq_1_to_8192;
 BEGIN;
 SELECT b FROM t1 LIMIT 3;
@@ -50,5 +51,5 @@ Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK
 SHOW ENGINE INNODB STATUS;
 Type	Name	Status
-InnoDB		insert 79, delete mark 1
+InnoDB		insert 139, delete mark 1
 DROP TABLE t1;

mysql-test/suite/innodb/r/innodb-page_compression_default.result

@@ -36,7 +36,7 @@ select count(*) from innodb_page_compressed9;
 count(*)
 10000
 # innodb_normal expected FOUND
-FOUND 24084 /AaAaAaAa/ in innodb_normal.ibd
+FOUND 24000 /AaAaAaAa/ in innodb_normal.ibd
 # innodb_page_compressed1 page compressed expected NOT FOUND
 NOT FOUND /AaAaAaAa/ in innodb_page_compressed1.ibd
 # innodb_page_compressed2 page compressed expected NOT FOUND

mysql-test/suite/innodb/r/innodb-page_compression_snappy.result

@@ -37,7 +37,7 @@ select count(*) from innodb_page_compressed9;
 count(*)
 10000
 # innodb_normal expected FOUND
-FOUND 24084 /AaAaAaAa/ in innodb_normal.ibd
+FOUND 24000 /AaAaAaAa/ in innodb_normal.ibd
 # innodb_page_compressed1 page compressed expected NOT FOUND
 NOT FOUND /AaAaAaAa/ in innodb_page_compressed1.ibd
 # innodb_page_compressed2 page compressed expected NOT FOUND

mysql-test/suite/innodb/r/innodb_defragment_fill_factor.result

@@ -115,4 +115,5 @@ insert into t2 values (201, REPEAT('A', 16), REPEAT('B', 32));
 insert into t2 values (202, REPEAT('A', 16), REPEAT('B', 32));
 insert into t2 values (203, REPEAT('A', 16), REPEAT('B', 32));
 insert into t2 values (204, REPEAT('A', 16), REPEAT('B', 32));
+Too little space is reserved on second index.
 DROP TABLE t2;

mysql-test/suite/innodb/r/instant_alter,32k.rdiff

@@ -44,7 +44,7 @@
  WHERE name = 'test/big';
  clust_index_size
 -7
-+5
++4
  connection default;
  ROLLBACK;
  CHECKSUM TABLE big;
@@ -71,7 +71,7 @@
  WHERE name = 'test/big';
  clust_index_size
 -7
-+5
++4
  connection default;
  ROLLBACK;
  CHECKSUM TABLE big;
@@ -98,7 +98,7 @@
  WHERE name = 'test/big';
  clust_index_size
 -7
-+5
++4
  connection default;
  ROLLBACK;
  CHECKSUM TABLE big;

mysql-test/suite/innodb/r/instant_alter,4k.rdiff

@@ -117,7 +117,7 @@
  WHERE name = 'test/big';
  clust_index_size
 -7
-+28
++27
  connection default;
  ROLLBACK;
  CHECKSUM TABLE big;

mysql-test/suite/innodb/r/instant_alter,8k.rdiff

@@ -41,7 +41,7 @@
  WHERE name = 'test/big';
  clust_index_size
 -3
-+5
++4
  connection default;
  ALTER TABLE big ADD COLUMN
  (d1 INT DEFAULT 0, d2 VARCHAR(20) DEFAULT 'abcde',
@@ -50,7 +50,7 @@
  WHERE name = 'test/big';
  clust_index_size
 -7
-+13
++12
  connection default;
  ROLLBACK;
  CHECKSUM TABLE big;
@@ -59,7 +59,7 @@
  WHERE name = 'test/big';
  clust_index_size
 -3
-+5
++4
  connection default;
  InnoDB		0 transactions not purged
  DROP TABLE t1,t2,t3,big;
@@ -104,7 +104,7 @@
  WHERE name = 'test/big';
  clust_index_size
 -3
-+5
++4
  connection default;
  ALTER TABLE big ADD COLUMN
  (d1 INT DEFAULT 0, d2 VARCHAR(20) DEFAULT 'abcde',
@@ -113,7 +113,7 @@
  WHERE name = 'test/big';
  clust_index_size
 -7
-+13
++12
  connection default;
  ROLLBACK;
  CHECKSUM TABLE big;
@@ -122,7 +122,7 @@
  WHERE name = 'test/big';
  clust_index_size
 -3
-+5
++4
  connection default;
  InnoDB		0 transactions not purged
  DROP TABLE t1,t2,t3,big;
@@ -167,7 +167,7 @@
  WHERE name = 'test/big';
  clust_index_size
 -3
-+5
++4
  connection default;
  ALTER TABLE big ADD COLUMN
  (d1 INT DEFAULT 0, d2 VARCHAR(20) DEFAULT 'abcde',
@@ -176,7 +176,7 @@
  WHERE name = 'test/big';
  clust_index_size
 -7
-+13
++12
  connection default;
  ROLLBACK;
  CHECKSUM TABLE big;
@@ -185,7 +185,7 @@
  WHERE name = 'test/big';
  clust_index_size
 -3
-+5
++4
  connection default;
  InnoDB		0 transactions not purged
  DROP TABLE t1,t2,t3,big;

mysql-test/suite/innodb/r/leaf_page_corrupted_during_recovery.result

@@ -4,17 +4,23 @@ INSERT INTO t1 VALUES(1, 'sql'), (2, 'server'), (3, 'mariadb'),
 (4, 'mariadb'), (5, 'test1'), (6, 'test2'), (7, 'test3'),
 (8, 'test4'), (9, 'test5'), (10, 'test6'), (11, 'test7'),
 (12, 'test8');
+FLUSH TABLE t1 FOR EXPORT;
+UNLOCK TABLES;
+SET GLOBAL innodb_log_checkpoint_now=ON;
 SELECT COUNT(*) FROM t1;
 COUNT(*)
 12
 UPDATE t1 SET c='best8' WHERE pk=12;
 # Kill the server
 # Corrupt the pages
+# restart
 SELECT * FROM t1 WHERE PK = 1;
 ERROR 42000: Unknown storage engine 'InnoDB'
+# restart: --innodb-force-recovery=1
 SELECT * FROM t1 WHERE PK = 1;
 pk	c
 1	sql
 SELECT * FROM t1 WHERE pk = 12;
-ERROR HY000: Index for table 't1' is corrupt; try to repair it
+ERROR HY000: Got error 168 "Unknown (generic) error from engine" from storage engine InnoDB
 DROP TABLE t1;
+# restart

mysql-test/suite/innodb/t/innodb-change-buffer-recovery.test

@@ -35,9 +35,16 @@ ENGINE=InnoDB STATS_PERSISTENT=0;
 SET GLOBAL innodb_change_buffering_debug = 1;
 let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err;
 
+# The removed function page_copy_rec_list_end_to_created_page() would create
+# less dense pages than the remaining part of page_copy_rec_list_end().
+# For the INSERT after the DELETE to be buffered, the leftmost page
+# must not be too densely packed.
+SET GLOBAL innodb_limit_optimistic_insert_debug=700;
+
 # Create enough rows for the table, so that the change buffer will be
 # used for modifying the secondary index page. There must be multiple
 # index pages, because changes to the root page are never buffered.
+
 INSERT INTO t1 SELECT 0,'x',1 FROM seq_1_to_8192;
 
 BEGIN;

mysql-test/suite/innodb/t/leaf_page_corrupted_during_recovery.test

@@ -3,8 +3,8 @@
 
 --disable_query_log
 call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed file read of tablespace test/t1 page ");
-call mtr.add_suppression("InnoDB: Background Page read failed to read or decrypt \\[page id: space=\\d+, page number=19\\]");
-call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read file '.*test.t1\\.ibd' at offset 19: Page read from tablespace is corrupted\\.");
+call mtr.add_suppression("InnoDB: Background Page read failed to read or decrypt \\[page id: space=\\d+, page number=14\\]");
+call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read file '.*test.t1\\.ibd' at offset 14: Page read from tablespace is corrupted\\.");
 call mtr.add_suppression("\\[ERROR\\] InnoDB: Plugin initialization aborted at srv0start\\.cc.* with error Data structure corruption");
 call mtr.add_suppression("\\[ERROR\\] Plugin 'InnoDB' (init function|registration)");
 call mtr.add_suppression("\\[ERROR\\] InnoDB: We detected index corruption");
@@ -20,8 +20,12 @@ INSERT INTO t1 VALUES(1, 'sql'), (2, 'server'), (3, 'mariadb'),
 	(8, 'test4'), (9, 'test5'), (10, 'test6'), (11, 'test7'),
 	(12, 'test8');
 
-let $restart_noprint=2;
---source include/restart_mysqld.inc
+# Flush all pages of the table, and perform log checkpoint,
+# so that no page initialization can be replayed from the redo log
+# to undo the page corruption that we are going to inject.
+FLUSH TABLE t1 FOR EXPORT;
+UNLOCK TABLES;
+SET GLOBAL innodb_log_checkpoint_now=ON;
 
 let INNODB_PAGE_SIZE=`select @@innodb_page_size`;
 let MYSQLD_DATADIR=`select @@datadir`;
@@ -38,7 +42,7 @@ perl;
 my $file = "$ENV{MYSQLD_DATADIR}/test/t1.ibd";
 open(FILE, "+<$file") || die "Unable to open $file";
 binmode FILE;
-seek (FILE, $ENV{INNODB_PAGE_SIZE} * 19 + 38, SEEK_SET) or die "seek";
+seek (FILE, $ENV{INNODB_PAGE_SIZE} * 14 + 38, SEEK_SET) or die "seek";
 print FILE "junk";
 close FILE or die "close";
 EOF
@@ -50,7 +54,7 @@ SELECT * FROM t1 WHERE PK = 1;
 let $restart_parameters=--innodb-force-recovery=1;
 --source include/restart_mysqld.inc
 SELECT * FROM t1 WHERE PK = 1;
---error ER_NOT_KEYFILE
+--error ER_GET_ERRNO
 SELECT * FROM t1 WHERE pk = 12;
 
 DROP TABLE t1;

mysql-test/suite/innodb_zip/r/wl6347_comp_indx_stat.result

@@ -962,7 +962,7 @@ AND compress_ops BETWEEN @inl_val AND 1000
 AND table_name='tab5' AND database_name='test'
 AND index_name like 'idx%' ;
 compress_stat	1
-The size of the tab5.ibd file: 163840
+The size of the tab5.ibd file: 159744
 # fetch the compressed page and check the stats
 ===============
 Fetch Records
@@ -986,7 +986,7 @@ AND compress_ops BETWEEN @inl_val AND 1000
 AND table_name='tab5' AND database_name='test'
 AND index_name like 'idx%' ;
 compress_stat	1
-The size of the tab5.ibd file: 163840
+The size of the tab5.ibd file: 159744
 # fetch the compressed same page once again and check the stats
 # the stat figures should be same as above query
 ===============
@@ -1011,7 +1011,7 @@ AND compress_ops BETWEEN @inl_val AND 1000
 AND table_name='tab5' AND database_name='test'
 AND index_name like 'idx%' ;
 compress_stat	1
-The size of the tab5.ibd file: 163840
+The size of the tab5.ibd file: 159744
 DROP TABLE tab5;
 #******************************************************************
 # Test 1-8K: innodb_cmp_per_index_enabled=ON and innodb_compression_level=0 with page size 8K

storage/innobase/include/page0cur.h

@@ -217,6 +217,7 @@ IMPORTANT: The caller will have to update IBUF_BITMAP_FREE
 if this is a compressed leaf page in a secondary index.
 This has to be done either within the same mini-transaction,
 or by invoking ibuf_reset_free_bits() before mtr_commit(). */
+ATTRIBUTE_COLD /* only used when crash-upgrading */
 void
 page_copy_rec_list_end_to_created_page(
 /*===================================*/

storage/innobase/page/page0cur.cc

@@ -2041,6 +2041,7 @@ IMPORTANT: The caller will have to update IBUF_BITMAP_FREE
 if this is a compressed leaf page in a secondary index.
 This has to be done either within the same mini-transaction,
 or by invoking ibuf_reset_free_bits() before mtr_commit(). */
+ATTRIBUTE_COLD /* only used when crash-upgrading */
 void
 page_copy_rec_list_end_to_created_page(
 /*===================================*/

storage/innobase/page/page0page.cc

@@ -599,37 +599,30 @@ page_copy_rec_list_end(
 	/* Here, "ret" may be pointing to a user record or the
 	predefined supremum record. */
 
-	mtr_log_t	log_mode = MTR_LOG_NONE;
-
-	if (new_page_zip) {
-		log_mode = mtr_set_log_mode(mtr, MTR_LOG_NONE);
-	}
+	const mtr_log_t log_mode = new_page_zip
+		? mtr->set_log_mode(MTR_LOG_NONE) : MTR_LOG_NONE;
+	ut_d(const bool was_empty = page_dir_get_n_heap(new_page)
+	     == PAGE_HEAP_NO_USER_LOW);
 
-	if (page_dir_get_n_heap(new_page) == PAGE_HEAP_NO_USER_LOW) {
-		page_copy_rec_list_end_to_created_page(new_page, rec,
-						       index, mtr);
-	} else {
-		if (dict_index_is_spatial(index)) {
-			ulint	max_to_move = page_get_n_recs(
-						buf_block_get_frame(block));
-			heap = mem_heap_create(256);
+	if (index->is_spatial()) {
+		ulint	max_to_move = page_get_n_recs(
+			buf_block_get_frame(block));
+		heap = mem_heap_create(256);
 
-			rec_move = static_cast<rtr_rec_move_t*>(mem_heap_alloc(
-					heap,
-					sizeof (*rec_move) * max_to_move));
+		rec_move = static_cast<rtr_rec_move_t*>(
+			mem_heap_alloc(heap, max_to_move * sizeof *rec_move));
 
-			/* For spatial index, we need to insert recs one by one
-			to keep recs ordered. */
-			rtr_page_copy_rec_list_end_no_locks(new_block,
-							    block, rec, index,
-							    heap, rec_move,
-							    max_to_move,
-							    &num_moved,
-							    mtr);
-		} else {
-			page_copy_rec_list_end_no_locks(new_block, block, rec,
-							index, mtr);
-		}
+		/* For spatial index, we need to insert recs one by one
+		to keep recs ordered. */
+		rtr_page_copy_rec_list_end_no_locks(new_block,
+						    block, rec, index,
+						    heap, rec_move,
+						    max_to_move,
+						    &num_moved,
+						    mtr);
+	} else {
+		page_copy_rec_list_end_no_locks(new_block, block, rec,
+						index, mtr);
 	}
 
 	/* Update PAGE_MAX_TRX_ID on the uncompressed page.
@@ -642,6 +635,9 @@ page_copy_rec_list_end(
 	if (dict_index_is_sec_or_ibuf(index)
 	    && page_is_leaf(page)
 	    && !index->table->is_temporary()) {
+		ut_ad(!was_empty || page_dir_get_n_heap(new_page)
+		      == PAGE_HEAP_NO_USER_LOW
+		      + page_header_get_field(new_page, PAGE_N_RECS));
 		page_update_max_trx_id(new_block, NULL,
 				       page_get_max_trx_id(page), mtr);
 	}