branches/innodb+: Merge revisions 2774:2799 from branches/zip:

  ------------------------------------------------------------------------
  r2781 | marko | 2008-10-13 13:40:57 +0300 (Mon, 13 Oct 2008) | 1 line

  branches/zip: page_cur_delete_rec(): Call page_zip_validate_low().
  ------------------------------------------------------------------------
  r2783 | vasil | 2008-10-13 18:34:34 +0300 (Mon, 13 Oct 2008) | 9 lines

  branches/zip:

  Remove mysql-test/patches/bug37312.diff because MySQL "fixed"
  Bug#37312 by removing the test.

  http://bugs.mysql.com/37312
  http://lists.mysql.com/commits/54462
  ------------------------------------------------------------------------
  r2784 | marko | 2008-10-13 21:35:30 +0300 (Mon, 13 Oct 2008) | 1 line

  branches/zip: Add missing NULL check to the assertion added in r2781.
  ------------------------------------------------------------------------
  r2785 | marko | 2008-10-13 22:29:12 +0300 (Mon, 13 Oct 2008) | 2 lines

  branches/zip: page_cur_delete_rec(): Remove the bogus page_zip_validate_low()
  assertion that was added in r2781 and explain why it was bogus.
  ------------------------------------------------------------------------
  r2786 | calvin | 2008-10-14 19:14:47 +0300 (Tue, 14 Oct 2008) | 7 lines

  branches/zip: fix Mantis issue #96 Problem compiling ha_innodb.cc
  on 64-bit Windows

  Change the definition of srv_replication_delay from ulint to ulong.
  ulint is 64-bit on Win64.

  Approved by:	Heikki (on IM)
  ------------------------------------------------------------------------
  r2787 | calvin | 2008-10-14 19:19:41 +0300 (Tue, 14 Oct 2008) | 7 lines

  branches/zip: fix compiler warning

  Change the definition of add_on from ulint to ullint, to eliminate
  the warning in .\btr\btr0cur.c:
  conversion from 'ullint' to 'ulint', possible loss of data

  Approved by:	Heikki (on IM)
  ------------------------------------------------------------------------
  r2793 | marko | 2008-10-15 10:00:06 +0300 (Wed, 15 Oct 2008) | 2 lines

  branches/zip: row_create_table_for_mysql(), row_create_index_for_mysql():
  Note that the dictionary object will be freed.
  ------------------------------------------------------------------------
  r2794 | marko | 2008-10-15 10:32:40 +0300 (Wed, 15 Oct 2008) | 9 lines

  branches/zip: When invoking page_zip_copy_recs(), update the lock table
  and the adaptive hash index.  This should fix Issue #95 and Issue #87.

  page_zip_copy_recs(): Copy PAGE_MAX_TRX_ID as well, to have similar behavior
  to page_copy_rec_list_start() and page_copy_rec_list_end().

  btr_root_raise_and_insert(), btr_page_split_and_insert(), btr_lift_page_up():
  Update the lock table and the adaptive hash index.
  ------------------------------------------------------------------------
  r2797 | marko | 2008-10-15 13:21:54 +0300 (Wed, 15 Oct 2008) | 3 lines

  branches/zip: Introduce UNIV_ZIP_COPY for invoking page_zip_copy_recs()
  more often in B-tree operations.
  ------------------------------------------------------------------------
  r2799 | marko | 2008-10-15 14:27:42 +0300 (Wed, 15 Oct 2008) | 25 lines

  branches/zip: When the server crashes while freeing an externally stored
  column of a compressed table, the BTR_EXTERN_LEN field in the BLOB pointer
  will be written as 0.  Tolerate this in the functions that deal with
  externally stored columns.  This fixes Issue #80 and was posted at rb://26.

  Note that the clustered index record is always deleted or purged last,
  after any secondary index records referring to it have been deleted.

  btr_free_externally_stored_field(): On an uncompressed table, zero out
  the BTR_EXTERN_LEN, so that half-deleted BLOBs can be detected after
  crash recovery.

  btr_copy_externally_stored_field_prefix(): Return 0 if the BLOB has been
  half-deleted.

  row_upd_ext_fetch(): Assert that the externally stored column exists.

  row_ext_cache_fill(): Allow btr_copy_externally_stored_field_prefix()
  to return 0.

  row_sel_sec_rec_is_for_blob(): Return FALSE if the BLOB has been half-deleted.
  This is correct, because the clustered index record would have been deleted
  or purged last, after any secondary index records referring to it had been
  deleted.
  ------------------------------------------------------------------------

btr/btr0btr.c

@@ -1191,7 +1191,11 @@ btr_root_raise_and_insert(
 
 	/* Copy the records from root to the new page one by one. */
 
-	if (UNIV_UNLIKELY
+	if (0
+#ifdef UNIV_ZIP_COPY
+	    || new_page_zip
+#endif /* UNIV_ZIP_COPY */
+	    || UNIV_UNLIKELY
 	    (!page_copy_rec_list_end(new_block, root_block,
 				     page_get_infimum_rec(root),
 				     index, mtr))) {
@@ -1200,6 +1204,14 @@ btr_root_raise_and_insert(
 		/* Copy the page byte for byte. */
 		page_zip_copy_recs(new_page_zip, new_page,
 				   root_page_zip, root, index, mtr);
+
+		/* Update the lock table and possible hash index. */
+
+		lock_move_rec_list_end(new_block, root_block,
+				       page_get_infimum_rec(root));
+
+		btr_search_move_or_delete_hash_entries(new_block, root_block,
+						       index);
 	}
 
 	/* If this is a pessimistic insert which is actually done to
@@ -1953,7 +1965,11 @@ btr_page_split_and_insert(
 	if (direction == FSP_DOWN) {
 		/*		fputs("Split left\n", stderr); */
 
-		if (UNIV_UNLIKELY
+		if (0
+#ifdef UNIV_ZIP_COPY
+		    || page_zip
+#endif /* UNIV_ZIP_COPY */
+		    || UNIV_UNLIKELY
 		    (!page_move_rec_list_start(new_block, block, move_limit,
 					       cursor->index, mtr))) {
 			/* For some reason, compressing new_page failed,
@@ -1969,6 +1985,18 @@ btr_page_split_and_insert(
 						 new_block, cursor->index,
 						 ULINT_UNDEFINED,
 						 ULINT_UNDEFINED, mtr);
+
+			/* Update the lock table and possible hash index. */
+
+			lock_move_rec_list_start(
+				new_block, block, move_limit,
+				new_page + PAGE_NEW_INFIMUM);
+
+			btr_search_move_or_delete_hash_entries(
+				new_block, block, cursor->index);
+
+			/* Delete the records from the source page. */
+
 			page_delete_rec_list_start(move_limit, block,
 						   cursor->index, mtr);
 		}
@@ -1980,7 +2008,11 @@ btr_page_split_and_insert(
 	} else {
 		/*		fputs("Split right\n", stderr); */
 
-		if (UNIV_UNLIKELY
+		if (0
+#ifdef UNIV_ZIP_COPY
+		    || page_zip
+#endif /* UNIV_ZIP_COPY */
+		    || UNIV_UNLIKELY
 		    (!page_move_rec_list_end(new_block, block, move_limit,
 					     cursor->index, mtr))) {
 			/* For some reason, compressing new_page failed,
@@ -1995,6 +2027,16 @@ btr_page_split_and_insert(
 			page_delete_rec_list_start(move_limit - page
 						   + new_page, new_block,
 						   cursor->index, mtr);
+
+			/* Update the lock table and possible hash index. */
+
+			lock_move_rec_list_end(new_block, block, move_limit);
+
+			btr_search_move_or_delete_hash_entries(
+				new_block, block, cursor->index);
+
+			/* Delete the records from the source page. */
+
 			page_delete_rec_list_end(move_limit, block,
 						 cursor->index,
 						 ULINT_UNDEFINED,
@@ -2343,7 +2385,11 @@ btr_lift_page_up(
 	btr_page_set_level(father_page, father_page_zip, page_level, mtr);
 
 	/* Copy the records to the father page one by one. */
-	if (UNIV_UNLIKELY
+	if (0
+#ifdef UNIV_ZIP_COPY
+	    || father_page_zip
+#endif /* UNIV_ZIP_COPY */
+	    || UNIV_UNLIKELY
 	    (!page_copy_rec_list_end(father_block, block,
 				     page_get_infimum_rec(page),
 				     index, mtr))) {
@@ -2355,6 +2401,14 @@ btr_lift_page_up(
 		/* Copy the page byte for byte. */
 		page_zip_copy_recs(father_page_zip, father_page,
 				   page_zip, page, index, mtr);
+
+		/* Update the lock table and possible hash index. */
+
+		lock_move_rec_list_end(father_block, block,
+				       page_get_infimum_rec(page));
+
+		btr_search_move_or_delete_hash_entries(father_block, block,
+						       index);
 	}
 
 	lock_update_copy_and_discard(father_block, block);

btr/btr0cur.c

@@ -3291,7 +3291,7 @@ btr_estimate_number_of_different_key_vals(
 	ulint		total_external_size = 0;
 	ulint		i;
 	ulint		j;
-	ulint		add_on;
+	ullint		add_on;
 	mtr_t		mtr;
 	mem_heap_t*	heap		= NULL;
 	ulint		offsets_rec_[REC_OFFS_NORMAL_SIZE];
@@ -4351,14 +4351,8 @@ btr_free_externally_stored_field(
 						 MLOG_4BYTES, &mtr);
 			}
 		} else {
-			ulint	extern_len	= mach_read_from_4(
-				field_ref + BTR_EXTERN_LEN + 4);
-			ulint	part_len	= btr_blob_get_part_len(
-				page + FIL_PAGE_DATA);
-
 			ut_a(fil_page_get_type(page) == FIL_PAGE_TYPE_BLOB);
 			ut_a(!page_zip);
-			ut_a(extern_len >= part_len);
 
 			next_page_no = mach_read_from_4(
 				page + FIL_PAGE_DATA
@@ -4376,16 +4370,14 @@ btr_free_externally_stored_field(
 			mlog_write_ulint(field_ref + BTR_EXTERN_PAGE_NO,
 					 next_page_no,
 					 MLOG_4BYTES, &mtr);
+			/* Zero out the BLOB length.  If the server
+			crashes during the execution of this function,
+			trx_rollback_or_clean_all_recovered() could
+			dereference the half-deleted BLOB, fetching a
+			wrong prefix for the BLOB. */
 			mlog_write_ulint(field_ref + BTR_EXTERN_LEN + 4,
-					 extern_len - part_len,
+					 0,
 					 MLOG_4BYTES, &mtr);
-			if (next_page_no == FIL_NULL) {
-				ut_a(extern_len - part_len == 0);
-			}
-
-			if (extern_len - part_len == 0) {
-				ut_a(next_page_no == FIL_NULL);
-			}
 		}
 
 		/* Commit mtr and release the BLOB block to save memory. */
@@ -4723,7 +4715,9 @@ UNIV_INTERN
 ulint
 btr_copy_externally_stored_field_prefix(
 /*====================================*/
-				/* out: the length of the copied field */
+				/* out: the length of the copied field,
+				or 0 if the column was being or has been
+				deleted */
 	byte*		buf,	/* out: the field, or a prefix of it */
 	ulint		len,	/* in: length of buf, in bytes */
 	ulint		zip_size,/* in: nonzero=compressed BLOB page size,
@@ -4752,6 +4746,14 @@ btr_copy_externally_stored_field_prefix(
 
 	ut_a(memcmp(data, field_ref_zero, BTR_EXTERN_FIELD_REF_SIZE));
 
+	if (!mach_read_from_4(data + BTR_EXTERN_LEN + 4)) {
+		/* The externally stored part of the column has been
+		(partially) deleted.  Signal the half-deleted BLOB
+		to the caller. */
+
+		return(0);
+	}
+
 	space_id = mach_read_from_4(data + BTR_EXTERN_SPACE_ID);
 
 	page_no = mach_read_from_4(data + BTR_EXTERN_PAGE_NO);

include/btr0cur.h

@@ -519,7 +519,9 @@ UNIV_INTERN
 ulint
 btr_copy_externally_stored_field_prefix(
 /*====================================*/
-				/* out: the length of the copied field */
+				/* out: the length of the copied field,
+				or 0 if the column is being or has been
+				deleted */
 	byte*		buf,	/* out: the field, or a prefix of it */
 	ulint		len,	/* in: length of buf, in bytes */
 	ulint		zip_size,/* in: nonzero=compressed BLOB page size,

include/page0zip.h

@@ -387,7 +387,8 @@ page_zip_reorganize(
 /**************************************************************************
 Copy the records of a page byte for byte.  Do not copy the page header
 or trailer, except those B-tree header fields that are directly
-related to the storage of records. */
+related to the storage of records.  Also copy PAGE_MAX_TRX_ID.
+NOTE: The caller must update the lock table and the adaptive hash index. */
 UNIV_INTERN
 void
 page_zip_copy_recs(

include/row0mysql.h

@@ -339,7 +339,8 @@ int
 row_create_table_for_mysql(
 /*=======================*/
 					/* out: error code or DB_SUCCESS */
-	dict_table_t*	table,		/* in: table definition */
+	dict_table_t*	table,		/* in, own: table definition
+					(will be freed) */
 	trx_t*		trx);		/* in: transaction handle */
 /*************************************************************************
 Does an index creation operation for MySQL. TODO: currently failure
@@ -350,7 +351,8 @@ int
 row_create_index_for_mysql(
 /*=======================*/
 					/* out: error number or DB_SUCCESS */
-	dict_index_t*	index,		/* in: index definition */
+	dict_index_t*	index,		/* in, own: index definition
+					(will be freed) */
 	trx_t*		trx,		/* in: transaction handle */
 	const ulint*	field_lengths); /* in: if not NULL, must contain
 					dict_index_get_n_fields(index)

include/srv0srv.h

@@ -145,7 +145,7 @@ extern int	srv_query_thread_priority;
 extern ulong	srv_max_buf_pool_modified_pct;
 extern ulong	srv_max_purge_lag;
 
-extern ulint	srv_replication_delay;
+extern ulong	srv_replication_delay;
 /*-------------------------------------------*/
 
 extern ulint	srv_n_rows_inserted;

include/univ.i

@@ -158,6 +158,8 @@ operations (very slow); also UNIV_DEBUG must be defined */
 						printing B-trees */
 #define UNIV_ZIP_DEBUG				/* extensive consistency checks
 						for compressed pages */
+#define UNIV_ZIP_COPY				/* call page_zip_copy_recs()
+						more often */
 #endif
 
 #define UNIV_BTR_DEBUG				/* check B-tree links */

mysql-test/patches/bug37312.diff

---- mysql-test/extra/binlog_tests/innodb_stat.test.orig	2008-06-10 15:12:02.000000000 +0300
-+++ mysql-test/extra/binlog_tests/innodb_stat.test	2008-06-10 15:12:06.000000000 +0300
-@@ -41,6 +41,7 @@
- 
- # Test for testable InnoDB status variables. This test
- # uses previous ones(pages_created, rows_deleted, ...).
-+-- replace_regex /51[12]/51_/
- show status like "Innodb_buffer_pool_pages_total";
- show status like "Innodb_page_size";
- show status like "Innodb_rows_deleted";
---- mysql-test/suite/binlog/r/binlog_row_innodb_stat.result.orig	2008-06-10 15:29:44.000000000 +0300
-+++ mysql-test/suite/binlog/r/binlog_row_innodb_stat.result	2008-06-10 15:30:04.000000000 +0300
-@@ -24,7 +24,7 @@
- drop table t1;
- show status like "Innodb_buffer_pool_pages_total";
- Variable_name	Value
--Innodb_buffer_pool_pages_total	512
-+Innodb_buffer_pool_pages_total	51_
- show status like "Innodb_page_size";
- Variable_name	Value
- Innodb_page_size	16384
---- mysql-test/suite/binlog/r/binlog_stm_innodb_stat.result.orig	2008-06-10 15:33:43.000000000 +0300
-+++ mysql-test/suite/binlog/r/binlog_stm_innodb_stat.result	2008-06-10 15:33:55.000000000 +0300
-@@ -24,7 +24,7 @@
- drop table t1;
- show status like "Innodb_buffer_pool_pages_total";
- Variable_name	Value
--Innodb_buffer_pool_pages_total	512
-+Innodb_buffer_pool_pages_total	51_
- show status like "Innodb_page_size";
- Variable_name	Value
- Innodb_page_size	16384

page/page0cur.c

@@ -1766,8 +1766,15 @@ page_cur_delete_rec(
 
 	page = page_cur_get_page(cursor);
 	page_zip = page_cur_get_page_zip(cursor);
-	/* page_zip_validate() may fail here when
-	btr_cur_pessimistic_delete() invokes btr_set_min_rec_mark(). */
+
+	/* page_zip_validate() will fail here when
+	btr_cur_pessimistic_delete() invokes btr_set_min_rec_mark().
+	Then, both "page_zip" and "page" would have the min-rec-mark
+	set on the smallest user record, but "page" would additionally
+	have it set on the smallest-but-one record.  Because sloppy
+	page_zip_validate_low() only ignores min-rec-flag differences
+	in the smallest user record, it cannot be used here either. */
+
 	current_rec = cursor->rec;
 	ut_ad(rec_offs_validate(current_rec, index, offsets));
 	ut_ad(!!page_is_comp(page) == dict_table_is_comp(index->table));

page/page0zip.c

@@ -4403,7 +4403,8 @@ page_zip_reorganize(
 /**************************************************************************
 Copy the records of a page byte for byte.  Do not copy the page header
 or trailer, except those B-tree header fields that are directly
-related to the storage of records. */
+related to the storage of records.  Also copy PAGE_MAX_TRX_ID.
+NOTE: The caller must update the lock table and the adaptive hash index. */
 UNIV_INTERN
 void
 page_zip_copy_recs(
@@ -4438,18 +4439,18 @@ page_zip_copy_recs(
 	UNIV_MEM_ASSERT_RW(src_zip->data, page_zip_get_size(page_zip));
 
 	/* Copy those B-tree page header fields that are related to
-	the records stored in the page.  Do not copy the field
+	the records stored in the page.  Also copy the field
 	PAGE_MAX_TRX_ID.  Skip the rest of the page header and
 	trailer.  On the compressed page, there is no trailer. */
 #if PAGE_MAX_TRX_ID + 8 != PAGE_HEADER_PRIV_END
 # error "PAGE_MAX_TRX_ID + 8 != PAGE_HEADER_PRIV_END"
 #endif
 	memcpy(PAGE_HEADER + page, PAGE_HEADER + src,
-	       PAGE_MAX_TRX_ID);
+	       PAGE_HEADER_PRIV_END);
 	memcpy(PAGE_DATA + page, PAGE_DATA + src,
 	       UNIV_PAGE_SIZE - PAGE_DATA - FIL_PAGE_DATA_END);
 	memcpy(PAGE_HEADER + page_zip->data, PAGE_HEADER + src_zip->data,
-	       PAGE_MAX_TRX_ID);
+	       PAGE_HEADER_PRIV_END);
 	memcpy(PAGE_DATA + page_zip->data, PAGE_DATA + src_zip->data,
 	       page_zip_get_size(page_zip) - PAGE_DATA);
 

row/row0ext.c

@@ -40,10 +40,13 @@ row_ext_cache_fill(
 		ext->len[i] = 0;
 	} else {
 		/* Fetch at most REC_MAX_INDEX_COL_LEN of the column.
-		The column must be non-empty. */
+		The column should be non-empty.  However,
+		trx_rollback_or_clean_all_recovered() may try to
+		access a half-deleted BLOB if the server previously
+		crashed during the execution of
+		btr_free_externally_stored_field(). */
 		ext->len[i] = btr_copy_externally_stored_field_prefix(
 			buf, REC_MAX_INDEX_COL_LEN, zip_size, field, f_len);
-		ut_a(ext->len[i]);
 	}
 }
 

row/row0mysql.c

@@ -1709,7 +1709,8 @@ int
 row_create_table_for_mysql(
 /*=======================*/
 				/* out: error code or DB_SUCCESS */
-	dict_table_t*	table,	/* in: table definition */
+	dict_table_t*	table,	/* in, own: table definition
+				(will be freed) */
 	trx_t*		trx)	/* in: transaction handle */
 {
 	tab_node_t*	node;
@@ -1903,7 +1904,8 @@ int
 row_create_index_for_mysql(
 /*=======================*/
 					/* out: error number or DB_SUCCESS */
-	dict_index_t*	index,		/* in: index definition */
+	dict_index_t*	index,		/* in, own: index definition
+					(will be freed) */
 	trx_t*		trx,		/* in: transaction handle */
 	const ulint*	field_lengths)	/* in: if not NULL, must contain
 					dict_index_get_n_fields(index)

row/row0sel.c

@@ -86,6 +86,16 @@ row_sel_sec_rec_is_for_blob(
 	len = btr_copy_externally_stored_field_prefix(buf, sizeof buf,
 						      zip_size,
 						      clust_field, clust_len);
+
+	if (UNIV_UNLIKELY(len == 0)) {
+		/* The BLOB was being deleted as the server crashed.
+		There should not be any secondary index records
+		referring to this clustered index record, because
+		btr_free_externally_stored_field() is called after all
+		secondary index entries of the row have been purged. */
+		return(FALSE);
+	}
+
 	len = dtype_get_at_most_n_mbchars(prtype, mbminlen, mbmaxlen,
 					  sec_len, len, (const char*) buf);
 

row/row0upd.c

@@ -861,6 +861,8 @@ row_upd_ext_fetch(
 	*len = btr_copy_externally_stored_field_prefix(buf, *len,
 						       zip_size,
 						       data, local_len);
+	/* We should never update records containing a half-deleted BLOB. */
+	ut_a(*len);
 
 	return(buf);
 }

srv/srv0srv.c

@@ -311,7 +311,7 @@ UNIV_INTERN ibool	srv_use_checksums = TRUE;
 UNIV_INTERN ibool	srv_set_thread_priorities = TRUE;
 UNIV_INTERN int	srv_query_thread_priority = 0;
 
-UNIV_INTERN ulint	srv_replication_delay		= 0;
+UNIV_INTERN ulong	srv_replication_delay		= 0;
 
 /*-------------------------------------------*/
 UNIV_INTERN ulong	srv_n_spin_wait_rounds	= 20;