MDEV-7477: Make innochecksum work on compressed tables

This patch ports the work that facebook has performed
to make innochecksum handle compressed tables.
the basic idea is to use actual innodb-code to perform
checksum verification rather than duplicating in innochecksum.cc.
to make this work, innodb code has been annotated with
lots of #ifndef UNIV_INNOCHECKSUM so that it can be
compiled outside of storage/innobase.

A new testcase is also added that verifies that innochecksum
works on compressed/non-compressed tables.

Merged from commit fabc79d2ea976c4ff5b79bfe913e6bc03ef69d42 
from https://code.google.com/p/google-mysql/

The actual steps to produce this patch are:

    take innochecksum from 5.6.14
    apply changes in innodb from facebook patches needed to make innochecksum compile
    apply changes in innochecksum from facebook patches
    add handcrafted testcase

The referenced facebook patches used are:

    https://github.com/facebook/mysql-5.6/commit/91e25120e75272db4cdbc07d0e45877d9dea5715
    https://github.com/facebook/mysql-5.6/commit/847fe76ea5239b09fa361b023c56e6be76d32046
    https://github.com/facebook/mysql-5.6/commit/1135628a5a9b3412621b93233478f3804bcef51a
    https://github.com/facebook/mysql-5.6/commit/4dbf7c240ce2f08b7d6572d9452c9779ce90641c

extra/CMakeLists.txt

@@ -75,11 +75,29 @@ ENDIF()
 MYSQL_ADD_EXECUTABLE(replace replace.c COMPONENT Server)
 TARGET_LINK_LIBRARIES(replace mysys)
 IF(UNIX)
-  MYSQL_ADD_EXECUTABLE(innochecksum innochecksum.c)
-
   MYSQL_ADD_EXECUTABLE(resolve_stack_dump resolve_stack_dump.c)
   TARGET_LINK_LIBRARIES(resolve_stack_dump mysys)
 
   MYSQL_ADD_EXECUTABLE(mysql_waitpid mysql_waitpid.c COMPONENT Client)
   TARGET_LINK_LIBRARIES(mysql_waitpid mysys)
 ENDIF()
+
+
+  # Add path to the InnoDB headers
+  INCLUDE_DIRECTORIES(
+    ${CMAKE_SOURCE_DIR}/storage/innobase/include
+    ${CMAKE_SOURCE_DIR}/sql)
+
+  # We use the InnoDB code directly in case the code changes.
+  ADD_DEFINITIONS("-DUNIV_INNOCHECKSUM")
+  SET(INNOBASE_SOURCES
+      ../storage/innobase/buf/buf0checksum.cc
+      ../storage/innobase/ut/ut0crc32.cc
+      ../storage/innobase/ut/ut0ut.cc
+      ../storage/innobase/page/page0zip.cc
+      )
+
+  MYSQL_ADD_EXECUTABLE(innochecksum innochecksum.cc ${INNOBASE_SOURCES})
+  TARGET_LINK_LIBRARIES(innochecksum mysys mysys_ssl)
+  ADD_DEPENDENCIES(innochecksum GenError)
+

extra/innochecksum.c

-/*
-   Copyright (c) 2005, 2011, Oracle and/or its affiliates
-
-   This program is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation; version 2 of the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
-*/
-
-/*
-  InnoDB offline file checksum utility.  85% of the code in this file
-  was taken wholesale fron the InnoDB codebase.
-
-  The final 15% was originally written by Mark Smith of Danga
-  Interactive, Inc. <junior@danga.com>
-
-  Published with a permission.
-*/
-
-#include <my_global.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <time.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-
-/* all of these ripped from InnoDB code from MySQL 4.0.22 */
-#define UT_HASH_RANDOM_MASK     1463735687
-#define UT_HASH_RANDOM_MASK2    1653893711
-#define FIL_PAGE_LSN          16 
-#define FIL_PAGE_FILE_FLUSH_LSN 26
-#define FIL_PAGE_OFFSET     4
-#define FIL_PAGE_DATA       38
-#define FIL_PAGE_END_LSN_OLD_CHKSUM 8
-#define FIL_PAGE_SPACE_OR_CHKSUM 0
-#define UNIV_PAGE_SIZE          (2 * 8192)
-
-/* command line argument to do page checks (that's it) */
-/* another argument to specify page ranges... seek to right spot and go from there */
-
-typedef unsigned long int ulint;
-
-/* innodb function in name; modified slightly to not have the ASM version (lots of #ifs that didn't apply) */
-ulint mach_read_from_4(uchar *b)
-{
-  return( ((ulint)(b[0]) << 24)
-          + ((ulint)(b[1]) << 16)
-          + ((ulint)(b[2]) << 8)
-          + (ulint)(b[3])
-          );
-}
-
-ulint
-ut_fold_ulint_pair(
-/*===============*/
-            /* out: folded value */
-    ulint   n1, /* in: ulint */
-    ulint   n2) /* in: ulint */
-{
-    return(((((n1 ^ n2 ^ UT_HASH_RANDOM_MASK2) << 8) + n1)
-                        ^ UT_HASH_RANDOM_MASK) + n2);
-}
-
-ulint
-ut_fold_binary(
-/*===========*/
-            /* out: folded value */
-    uchar*   str,    /* in: string of bytes */
-    ulint   len)    /* in: length */
-{
-    ulint   i;
-    ulint   fold= 0;
-
-    for (i= 0; i < len; i++)
-    {
-      fold= ut_fold_ulint_pair(fold, (ulint)(*str));
-
-      str++;
-    }
-
-    return(fold);
-}
-
-ulint
-buf_calc_page_new_checksum(
-/*=======================*/
-               /* out: checksum */
-    uchar*    page) /* in: buffer page */
-{
-    ulint checksum;
-
-    /* Since the fields FIL_PAGE_FILE_FLUSH_LSN and ..._ARCH_LOG_NO
-    are written outside the buffer pool to the first pages of data
-    files, we have to skip them in the page checksum calculation.
-    We must also skip the field FIL_PAGE_SPACE_OR_CHKSUM where the
-    checksum is stored, and also the last 8 bytes of page because
-    there we store the old formula checksum. */
-
-    checksum= ut_fold_binary(page + FIL_PAGE_OFFSET,
-                             FIL_PAGE_FILE_FLUSH_LSN - FIL_PAGE_OFFSET)
-            + ut_fold_binary(page + FIL_PAGE_DATA,
-                             UNIV_PAGE_SIZE - FIL_PAGE_DATA
-                             - FIL_PAGE_END_LSN_OLD_CHKSUM);
-    checksum= checksum & 0xFFFFFFFF;
-
-    return(checksum);
-}
-
-ulint
-buf_calc_page_old_checksum(
-/*=======================*/
-               /* out: checksum */
-    uchar*    page) /* in: buffer page */
-{
-    ulint checksum;
-
-    checksum= ut_fold_binary(page, FIL_PAGE_FILE_FLUSH_LSN);
-
-    checksum= checksum & 0xFFFFFFFF;
-
-    return(checksum);
-}
-
-
-int main(int argc, char **argv)
-{
-  FILE *f;                     /* our input file */
-  uchar *p;                     /* storage of pages read */
-  int bytes;                   /* bytes read count */
-  ulint ct;                    /* current page number (0 based) */
-  int now;                     /* current time */
-  int lastt;                   /* last time */
-  ulint oldcsum, oldcsumfield, csum, csumfield, logseq, logseqfield; /* ulints for checksum storage */
-  struct stat st;              /* for stat, if you couldn't guess */
-  unsigned long long int size; /* size of file (has to be 64 bits) */
-  ulint pages;                 /* number of pages in file */
-  ulint start_page= 0, end_page= 0, use_end_page= 0; /* for starting and ending at certain pages */
-  off_t offset= 0;
-  int just_count= 0;          /* if true, just print page count */
-  int verbose= 0;
-  int debug= 0;
-  int c;
-  int fd;
-
-  /* remove arguments */
-  while ((c= getopt(argc, argv, "cvds:e:p:")) != -1)
-  {
-    switch (c)
-    {
-    case 'v':
-      verbose= 1;
-      break;
-    case 'c':
-      just_count= 1;
-      break;
-    case 's':
-      start_page= atoi(optarg);
-      break;
-    case 'e':
-      end_page= atoi(optarg);
-      use_end_page= 1;
-      break;
-    case 'p':
-      start_page= atoi(optarg);
-      end_page= atoi(optarg);
-      use_end_page= 1;
-      break;
-    case 'd':
-      debug= 1;
-      break;
-    case ':':
-      fprintf(stderr, "option -%c requires an argument\n", optopt);
-      return 1;
-      break;
-    case '?':
-      fprintf(stderr, "unrecognized option: -%c\n", optopt);
-      return 1;
-      break;
-    }
-  }
-
-  /* debug implies verbose... */
-  if (debug) verbose= 1;
-
-  /* make sure we have the right arguments */
-  if (optind >= argc)
-  {
-    printf("InnoDB offline file checksum utility.\n");
-    printf("usage: %s [-c] [-s <start page>] [-e <end page>] [-p <page>] [-v] [-d] <filename>\n", argv[0]);
-    printf("\t-c\tprint the count of pages in the file\n");
-    printf("\t-s n\tstart on this page number (0 based)\n");
-    printf("\t-e n\tend at this page number (0 based)\n");
-    printf("\t-p n\tcheck only this page (0 based)\n");
-    printf("\t-v\tverbose (prints progress every 5 seconds)\n");
-    printf("\t-d\tdebug mode (prints checksums for each page)\n");
-    return 1;
-  }
-
-  /* stat the file to get size and page count */
-  if (stat(argv[optind], &st))
-  {
-    perror("error statting file");
-    return 1;
-  }
-  size= st.st_size;
-  pages= size / UNIV_PAGE_SIZE;
-  if (just_count)
-  {
-    printf("%lu\n", pages);
-    return 0;
-  }
-  else if (verbose)
-  {
-    printf("file %s = %llu bytes (%lu pages)...\n", argv[optind], size, pages);
-    printf("checking pages in range %lu to %lu\n", start_page, use_end_page ? end_page : (pages - 1));
-  }
-
-  /* open the file for reading */
-  f= fopen(argv[optind], "r");
-  if (!f)
-  {
-    perror("error opening file");
-    return 1;
-  }
-
-  /* seek to the necessary position */
-  if (start_page)
-  {
-    fd= fileno(f);
-    if (!fd)
-    {
-      perror("unable to obtain file descriptor number");
-      return 1;
-    }
-
-    offset= (off_t)start_page * (off_t)UNIV_PAGE_SIZE;
-
-    if (lseek(fd, offset, SEEK_SET) != offset)
-    {
-      perror("unable to seek to necessary offset");
-      return 1;
-    }
-  }
-
-  /* allocate buffer for reading (so we don't realloc every time) */
-  p= (uchar *)malloc(UNIV_PAGE_SIZE);
-
-  /* main checksumming loop */
-  ct= start_page;
-  lastt= 0;
-  while (!feof(f))
-  {
-    bytes= fread(p, 1, UNIV_PAGE_SIZE, f);
-    if (!bytes && feof(f)) return 0;
-    if (bytes != UNIV_PAGE_SIZE)
-    {
-      fprintf(stderr, "bytes read (%d) doesn't match universal page size (%d)\n", bytes, UNIV_PAGE_SIZE);
-      return 1;
-    }
-
-    /* check the "stored log sequence numbers" */
-    logseq= mach_read_from_4(p + FIL_PAGE_LSN + 4);
-    logseqfield= mach_read_from_4(p + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + 4);
-    if (debug)
-      printf("page %lu: log sequence number: first = %lu; second = %lu\n", ct, logseq, logseqfield);
-    if (logseq != logseqfield)
-    {
-      fprintf(stderr, "page %lu invalid (fails log sequence number check)\n", ct);
-      return 1;
-    }
-
-    /* check old method of checksumming */
-    oldcsum= buf_calc_page_old_checksum(p);
-    oldcsumfield= mach_read_from_4(p + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
-    if (debug)
-      printf("page %lu: old style: calculated = %lu; recorded = %lu\n", ct, oldcsum, oldcsumfield);
-    if (oldcsumfield != mach_read_from_4(p + FIL_PAGE_LSN) && oldcsumfield != oldcsum)
-    {
-      fprintf(stderr, "page %lu invalid (fails old style checksum)\n", ct);
-      return 1;
-    }
-
-    /* now check the new method */
-    csum= buf_calc_page_new_checksum(p);
-    csumfield= mach_read_from_4(p + FIL_PAGE_SPACE_OR_CHKSUM);
-    if (debug)
-      printf("page %lu: new style: calculated = %lu; recorded = %lu\n", ct, csum, csumfield);
-    if (csumfield != 0 && csum != csumfield)
-    {
-      fprintf(stderr, "page %lu invalid (fails new style checksum)\n", ct);
-      return 1;
-    }
-
-    /* end if this was the last page we were supposed to check */
-    if (use_end_page && (ct >= end_page))
-      return 0;
-
-    /* do counter increase and progress printing */
-    ct++;
-    if (verbose)
-    {
-      if (ct % 64 == 0)
-      {
-        now= time(0);
-        if (!lastt) lastt= now;
-        if (now - lastt >= 1)
-        {
-          printf("page %lu okay: %.3f%% done\n", (ct - 1), (float) ct / pages * 100);
-          lastt= now;
-        }
-      }
-    }
-  }
-  return 0;
-}
-

mysql-test/mysql-test-run.pl

@@ -2491,6 +2491,17 @@ sub environment_setup {
                                  "$bindir/sql$opt_vs_config/mysql_tzinfo_to_sql");
   $ENV{'MYSQL_TZINFO_TO_SQL'}= native_path($exe_mysql_tzinfo_to_sql);
 
+  # ----------------------------------------------------
+  # innochecksum
+  # ----------------------------------------------------
+  my $exe_innochecksum=
+    mtr_exe_maybe_exists("$bindir/extra$opt_vs_config/innochecksum",
+		         "$path_client_bindir/innochecksum");
+  if ($exe_innochecksum)
+  {
+    $ENV{'INNOCHECKSUM'}= native_path($exe_innochecksum);
+  }
+
   # Create an environment variable to make it possible
   # to detect that valgrind is being used from test cases
   $ENV{'VALGRIND_TEST'}= $opt_valgrind;

mysql-test/suite/innodb/r/innochecksum.result

+# Create and populate a table
+CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB;
+INSERT INTO t1 (b) VALUES ('corrupt me');
+INSERT INTO t1 (b) VALUES ('corrupt me');
+CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT)
+ROW_FORMAT=COMPRESSED ENGINE=InnoDB ;
+INSERT INTO t2(b) SELECT b from t1;
+CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT)
+ROW_FORMAT=COMPRESSED ENGINE=InnoDB KEY_BLOCK_SIZE=16;
+INSERT INTO t3(b) SELECT b from t1;
+# Write file to make mysql-test-run.pl expect the "crash", but don't
+# start it until it's told to
+# We give 30 seconds to do a clean shutdown because we do not want
+# to redo apply the pages of t1.ibd at the time of recovery.
+# We want SQL to initiate the first access to t1.ibd.
+# Wait until disconnected.
+# Run innochecksum on t1
+InnoDB offline file checksum utility.
+Table is uncompressed
+Page size is 16384
+# Run innochecksum on t2
+InnoDB offline file checksum utility.
+Table is compressed
+Key block size is 8192
+# Run innochecksum on t3
+InnoDB offline file checksum utility.
+Table is compressed
+Key block size is 16384
+# Write file to make mysql-test-run.pl start up the server again
+# Cleanup
+DROP TABLE t1, t2, t3;

mysql-test/suite/innodb/t/innochecksum.opt

+--innodb_file_per_table=1
+--innodb_file_format=Barracuda

mysql-test/suite/innodb/t/innochecksum.test

+#
+# Test innochecksum
+#
+
+# Don't test under embedded
+source include/not_embedded.inc;
+# Require InnoDB
+source include/have_innodb.inc;
+
+if (!$INNOCHECKSUM) {
+  --echo Need innochecksum binary
+  --die Need innochecksum binary
+}
+
+--echo # Create and populate a table
+CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB;
+INSERT INTO t1 (b) VALUES ('corrupt me');
+--disable_query_log
+--let $i = 1000
+while ($i)
+{
+  INSERT INTO t1 (b) VALUES (REPEAT('abcdefghijklmnopqrstuvwxyz', 100));
+  dec $i;
+}
+--enable_query_log
+INSERT INTO t1 (b) VALUES ('corrupt me');
+
+CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT)
+ROW_FORMAT=COMPRESSED ENGINE=InnoDB ;
+
+INSERT INTO t2(b) SELECT b from t1;
+
+CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT)
+ROW_FORMAT=COMPRESSED ENGINE=InnoDB KEY_BLOCK_SIZE=16;
+
+INSERT INTO t3(b) SELECT b from t1;
+
+let $MYSQLD_DATADIR=`select @@datadir`;
+let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd;
+let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd;
+let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd;
+
+--echo # Write file to make mysql-test-run.pl expect the "crash", but don't
+--echo # start it until it's told to
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+
+--echo # We give 30 seconds to do a clean shutdown because we do not want
+--echo # to redo apply the pages of t1.ibd at the time of recovery.
+--echo # We want SQL to initiate the first access to t1.ibd.
+shutdown_server 30;
+
+--echo # Wait until disconnected.
+--source include/wait_until_disconnected.inc
+
+--echo # Run innochecksum on t1
+--exec $INNOCHECKSUM $t1_IBD
+
+--echo # Run innochecksum on t2
+--exec $INNOCHECKSUM $t2_IBD
+
+--echo # Run innochecksum on t3
+--exec $INNOCHECKSUM $t3_IBD
+
+--echo # Write file to make mysql-test-run.pl start up the server again
+--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+--echo # Cleanup
+DROP TABLE t1, t2, t3;

storage/innobase/buf/buf0checksum.cc

@@ -27,20 +27,21 @@ Created Aug 11, 2011 Vasil Dimov
 #include "fil0fil.h" /* FIL_* */
 #include "ut0crc32.h" /* ut_crc32() */
 #include "ut0rnd.h" /* ut_fold_binary() */
+#include "buf0checksum.h"
 
 #ifndef UNIV_INNOCHECKSUM
 
 #include "srv0srv.h" /* SRV_CHECKSUM_* */
 #include "buf0types.h"
 
+#endif /* !UNIV_INNOCHECKSUM */
+
 /** the macro MYSQL_SYSVAR_ENUM() requires "long unsigned int" and if we
 use srv_checksum_algorithm_t here then we get a compiler error:
 ha_innodb.cc:12251: error: cannot convert 'srv_checksum_algorithm_t*' to
   'long unsigned int*' in initialization */
 UNIV_INTERN ulong	srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB;
 
-#endif /* !UNIV_INNOCHECKSUM */
-
 /********************************************************************//**
 Calculates a page CRC32 which is stored to the page when it is written
 to a file. Note that we must be careful to calculate the same value on

storage/innobase/include/buf0buf.h

@@ -96,9 +96,6 @@ extern buf_block_t*	back_block1;	/*!< first block, for --apply-log */
 extern buf_block_t*	back_block2;	/*!< second block, for page reorganize */
 #endif /* !UNIV_HOTBACKUP */
 
-/** Magic value to use instead of checksums when they are disabled */
-#define BUF_NO_CHECKSUM_MAGIC 0xDEADBEEFUL
-
 /** @brief States of a control block
 @see buf_page_t
 

storage/innobase/include/buf0checksum.h

@@ -28,11 +28,10 @@ Created Aug 11, 2011 Vasil Dimov
 
 #include "univ.i"
 
-#ifndef UNIV_INNOCHECKSUM
-
 #include "buf0types.h"
 
-#endif /* !UNIV_INNOCHECKSUM */
+/** Magic value to use instead of checksums when they are disabled */
+#define BUF_NO_CHECKSUM_MAGIC 0xDEADBEEFUL
 
 /********************************************************************//**
 Calculates a page CRC32 which is stored to the page when it is written
@@ -70,8 +69,6 @@ buf_calc_page_old_checksum(
 /*=======================*/
 	const byte*	page);	/*!< in: buffer page */
 
-#ifndef UNIV_INNOCHECKSUM
-
 /********************************************************************//**
 Return a printable string describing the checksum algorithm.
 @return	algorithm name */
@@ -83,6 +80,4 @@ buf_checksum_algorithm_name(
 
 extern ulong	srv_checksum_algorithm;
 
-#endif /* !UNIV_INNOCHECKSUM */
-
 #endif /* buf0checksum_h */

storage/innobase/include/fil0fil.h

@@ -65,11 +65,16 @@ of the address is FIL_NULL, the address is considered undefined. */
 
 typedef	byte	fil_faddr_t;	/*!< 'type' definition in C: an address
 				stored in a file page is a string of bytes */
+
+#endif /* !UNIV_INNOCHECKSUM */
+
 #define FIL_ADDR_PAGE	0	/* first in address is the page offset */
 #define	FIL_ADDR_BYTE	4	/* then comes 2-byte byte offset within page*/
 
 #define	FIL_ADDR_SIZE	6	/* address size is 6 bytes */
 
+#ifndef UNIV_INNOCHECKSUM
+
 /** File space address */
 struct fil_addr_t{
 	ulint	page;		/*!< page number within a space */
@@ -140,8 +145,6 @@ extern fil_addr_t	fil_addr_null;
 #define FIL_PAGE_DATA_END	8	/*!< size of the page trailer */
 /* @} */
 
-#ifndef UNIV_INNOCHECKSUM
-
 /** File page types (values of FIL_PAGE_TYPE) @{ */
 #define FIL_PAGE_INDEX		17855	/*!< B-tree node */
 #define FIL_PAGE_UNDO_LOG	2	/*!< Undo log page */
@@ -166,6 +169,8 @@ extern fil_addr_t	fil_addr_null;
 #define FIL_LOG			502	/*!< redo log */
 /* @} */
 
+#ifndef UNIV_INNOCHECKSUM
+
 /** The number of fsyncs done to the log */
 extern ulint	fil_n_log_flushes;
 

storage/innobase/include/mach0data.ic

@@ -73,6 +73,8 @@ mach_write_to_2(
 	b[1] = (byte)(n);
 }
 
+#endif /* !UNIV_INNOCHECKSUM */
+
 /********************************************************//**
 The following function is used to fetch data from 2 consecutive
 bytes. The most significant byte is at the lowest address.
@@ -86,6 +88,8 @@ mach_read_from_2(
 	return(((ulint)(b[0]) << 8) | (ulint)(b[1]));
 }
 
+#ifndef UNIV_INNOCHECKSUM
+
 /********************************************************//**
 The following function is used to convert a 16-bit data item
 to the canonical format, for fast bytewise equality test
@@ -295,6 +299,8 @@ mach_write_to_8(
 	mach_write_to_4(static_cast<byte*>(b) + 4, (ulint) n);
 }
 
+#endif /* !UNIV_INNOCHECKSUM */
+
 /********************************************************//**
 The following function is used to fetch data from 8 consecutive
 bytes. The most significant byte is at the lowest address.
@@ -313,6 +319,8 @@ mach_read_from_8(
 	return(ull);
 }
 
+#ifndef UNIV_INNOCHECKSUM
+
 /*******************************************************//**
 The following function is used to store data in 7 consecutive
 bytes. We store the most significant byte to the lowest address. */

storage/innobase/include/page0page.h

@@ -29,6 +29,7 @@ Created 2/2/1994 Heikki Tuuri
 #include "univ.i"
 
 #include "page0types.h"
+#ifndef UNIV_INNOCHECKSUM
 #include "fil0fil.h"
 #include "buf0buf.h"
 #include "data0data.h"
@@ -42,6 +43,8 @@ Created 2/2/1994 Heikki Tuuri
 #define UNIV_INLINE
 #endif
 
+#endif /* !UNIV_INNOCHECKSUM */
+
 /*			PAGE HEADER
 			===========
 
@@ -117,6 +120,8 @@ typedef	byte		page_header_t;
 				a new-style compact page */
 /*-----------------------------*/
 
+#ifndef UNIV_INNOCHECKSUM
+
 /* Heap numbers */
 #define PAGE_HEAP_NO_INFIMUM	0	/* page infimum */
 #define PAGE_HEAP_NO_SUPREMUM	1	/* page supremum */
@@ -343,6 +348,7 @@ page_cmp_dtuple_rec_with_match(
 				matched; when function returns contains the
 				value for current comparison */
 #endif /* !UNIV_HOTBACKUP */
+#endif /* !UNIV_INNOCHECKSUM */
 /*************************************************************//**
 Gets the page number.
 @return	page number */
@@ -351,6 +357,7 @@ ulint
 page_get_page_no(
 /*=============*/
 	const page_t*	page);	/*!< in: page */
+#ifndef UNIV_INNOCHECKSUM
 /*************************************************************//**
 Gets the tablespace identifier.
 @return	space id */
@@ -359,6 +366,7 @@ ulint
 page_get_space_id(
 /*==============*/
 	const page_t*	page);	/*!< in: page */
+#endif /* !UNIV_INNOCHECKSUM */
 /*************************************************************//**
 Gets the number of user records on page (the infimum and supremum records
 are not user records).
@@ -368,6 +376,7 @@ ulint
 page_get_n_recs(
 /*============*/
 	const page_t*	page);	/*!< in: index page */
+#ifndef UNIV_INNOCHECKSUM
 /***************************************************************//**
 Returns the number of records before the given record in chain.
 The number includes infimum and supremum records.
@@ -516,6 +525,7 @@ ulint
 page_rec_get_heap_no(
 /*=================*/
 	const rec_t*	rec);	/*!< in: the physical record */
+#endif /* !UNIV_INNOCHECKSUM */
 /************************************************************//**
 Determine whether the page is a B-tree leaf.
 @return	true if the page is a B-tree leaf (PAGE_LEVEL = 0) */
@@ -525,6 +535,7 @@ page_is_leaf(
 /*=========*/
 	const page_t*	page)	/*!< in: page */
 	__attribute__((nonnull, pure));
+#ifndef UNIV_INNOCHECKSUM
 /************************************************************//**
 Determine whether the page is empty.
 @return	true if the page is empty (PAGE_N_RECS = 0) */
@@ -1115,8 +1126,11 @@ page_find_rec_max_not_deleted(
 #define UNIV_INLINE  UNIV_INLINE_ORIGINAL
 #endif
 
+#endif /* !UNIV_INNOCHECKSUM */
+
 #ifndef UNIV_NONINL
 #include "page0page.ic"
 #endif
 
+
 #endif

storage/innobase/include/page0page.ic

@@ -23,6 +23,8 @@ Index page routines
 Created 2/2/1994 Heikki Tuuri
 *******************************************************/
 
+#ifndef UNIV_INNOCHECKSUM
+
 #include "mach0data.h"
 #ifdef UNIV_DEBUG
 # include "log0recv.h"
@@ -38,6 +40,7 @@ Created 2/2/1994 Heikki Tuuri
 #define UNIV_INLINE
 #endif
 
+#endif /* !UNIV_INNOCHECKSUM */
 /************************************************************//**
 Gets the start of a page.
 @return	start of the page */
@@ -49,6 +52,7 @@ page_align(
 {
 	return((page_t*) ut_align_down(ptr, UNIV_PAGE_SIZE));
 }
+#ifndef UNIV_INNOCHECKSUM
 /************************************************************//**
 Gets the offset within a page.
 @return	offset from the start of the page */
@@ -103,6 +107,7 @@ page_update_max_trx_id(
 	}
 }
 
+#endif /* !UNIV_INNOCHECKSUM */
 /*************************************************************//**
 Reads the given header field. */
 UNIV_INLINE
@@ -118,6 +123,7 @@ page_header_get_field(
 	return(mach_read_from_2(page + PAGE_HEADER + field));
 }
 
+#ifndef UNIV_INNOCHECKSUM
 /*************************************************************//**
 Sets the given header field. */
 UNIV_INLINE
@@ -223,6 +229,7 @@ page_header_reset_last_insert(
 }
 #endif /* !UNIV_HOTBACKUP */
 
+#endif /* !UNIV_INNOCHECKSUM */
 /************************************************************//**
 Determine whether the page is in new-style compact format.
 @return nonzero if the page is in compact format, zero if it is in
@@ -236,6 +243,7 @@ page_is_comp(
 	return(page_header_get_field(page, PAGE_N_HEAP) & 0x8000);
 }
 
+#ifndef UNIV_INNOCHECKSUM
 /************************************************************//**
 TRUE if the record is on a page in compact format.
 @return	nonzero if in compact format */
@@ -264,6 +272,7 @@ page_rec_get_heap_no(
 	}
 }
 
+#endif /* !UNIV_INNOCHECKSUM */
 /************************************************************//**
 Determine whether the page is a B-tree leaf.
 @return	true if the page is a B-tree leaf (PAGE_LEVEL = 0) */
@@ -276,6 +285,7 @@ page_is_leaf(
 	return(!*(const uint16*) (page + (PAGE_HEADER + PAGE_LEVEL)));
 }
 
+#ifndef UNIV_INNOCHECKSUM
 /************************************************************//**
 Determine whether the page is empty.
 @return	true if the page is empty (PAGE_N_RECS = 0) */
@@ -529,6 +539,7 @@ page_cmp_dtuple_rec_with_match(
 }
 #endif /* !UNIV_HOTBACKUP */
 
+#endif /* !UNIV_INNOCHECKSUM */
 /*************************************************************//**
 Gets the page number.
 @return	page number */
@@ -542,6 +553,7 @@ page_get_page_no(
 	return(mach_read_from_4(page + FIL_PAGE_OFFSET));
 }
 
+#ifndef UNIV_INNOCHECKSUM
 /*************************************************************//**
 Gets the tablespace identifier.
 @return	space id */
@@ -555,6 +567,7 @@ page_get_space_id(
 	return(mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID));
 }
 
+#endif /* !UNIV_INNOCHECKSUM */
 /*************************************************************//**
 Gets the number of user records on page (infimum and supremum records
 are not user records).
@@ -568,6 +581,7 @@ page_get_n_recs(
 	return(page_header_get_field(page, PAGE_N_RECS));
 }
 
+#ifndef UNIV_INNOCHECKSUM
 /*************************************************************//**
 Gets the number of dir slots in directory.
 @return	number of slots */
@@ -958,6 +972,7 @@ page_rec_get_base_extra_size(
 	return(REC_N_NEW_EXTRA_BYTES + (ulint) !page_rec_is_comp(rec));
 }
 
+#endif /* !UNIV_INNOCHECKSUM */
 /************************************************************//**
 Returns the sum of the sizes of the records in the record list, excluding
 the infimum and supremum records.
@@ -981,7 +996,7 @@ page_get_data_size(
 	return(ret);
 }
 
-
+#ifndef UNIV_INNOCHECKSUM
 /************************************************************//**
 Allocates a block of memory from the free list of an index page. */
 UNIV_INLINE
@@ -1170,6 +1185,8 @@ page_mem_free(
 	}
 }
 
+#endif /* !UNIV_INNOCHECKSUM */
+
 #ifdef UNIV_MATERIALIZE
 #undef UNIV_INLINE
 #define UNIV_INLINE	UNIV_INLINE_ORIGINAL

storage/innobase/include/page0types.h

@@ -33,6 +33,8 @@ using namespace std;
 #include "univ.i"
 #include "dict0types.h"
 #include "mtr0types.h"
+#include "sync0types.h"
+#include "os0thread.h"
 
 /** Eliminates a name collision on HP-UX */
 #define page_t	   ib_page_t

storage/innobase/include/page0zip.h

+
 /*****************************************************************************
 
 Copyright (c) 2005, 2013, Oracle and/or its affiliates. All Rights Reserved.
@@ -32,13 +33,15 @@ Created June 2005 by Marko Makela
 # define UNIV_INLINE
 #endif
 
-#include "mtr0types.h"
 #include "page0types.h"
 #include "buf0types.h"
+#ifndef UNIV_INNOCHECKSUM
+#include "mtr0types.h"
 #include "dict0types.h"
 #include "srv0srv.h"
 #include "trx0types.h"
 #include "mem0mem.h"
+#endif /* !UNIV_INNOCHECKSUM */
 
 /* Compression level to be used by zlib. Settable by user. */
 extern uint	page_zip_level;
@@ -50,6 +53,7 @@ extern uint	page_zip_level;
 compression algorithm changes in zlib. */
 extern my_bool	page_zip_log_pages;
 
+#ifndef UNIV_INNOCHECKSUM
 /**********************************************************************//**
 Determine the size of a compressed page in bytes.
 @return	size in bytes */
@@ -112,6 +116,7 @@ page_zip_set_alloc(
 /*===============*/
 	void*		stream,		/*!< in/out: zlib stream */
 	mem_heap_t*	heap);		/*!< in: memory heap to use */
+#endif /* !UNIV_INNOCHECKSUM */
 
 /**********************************************************************//**
 Compress a page.
@@ -147,6 +152,7 @@ page_zip_decompress(
 				after page creation */
 	__attribute__((nonnull(1,2)));
 
+#ifndef UNIV_INNOCHECKSUM
 #ifdef UNIV_DEBUG
 /**********************************************************************//**
 Validate a compressed page descriptor.
@@ -158,6 +164,7 @@ page_zip_simple_validate(
 	const page_zip_des_t*	page_zip);	/*!< in: compressed page
 						descriptor */
 #endif /* UNIV_DEBUG */
+#endif /* !UNIV_INNOCHECKSUM */
 
 #ifdef UNIV_ZIP_DEBUG
 /**********************************************************************//**
@@ -185,6 +192,7 @@ page_zip_validate(
 	__attribute__((nonnull(1,2)));
 #endif /* UNIV_ZIP_DEBUG */
 
+#ifndef UNIV_INNOCHECKSUM
 /**********************************************************************//**
 Determine how big record can be inserted without recompressing the page.
 @return a positive number indicating the maximum size of a record
@@ -418,6 +426,8 @@ page_zip_reorganize(
 	dict_index_t*	index,	/*!< in: index of the B-tree node */
 	mtr_t*		mtr)	/*!< in: mini-transaction */
 	__attribute__((nonnull));
+#endif /* !UNIV_INNOCHECKSUM */
+
 #ifndef UNIV_HOTBACKUP
 /**********************************************************************//**
 Copy the records of a page byte for byte.  Do not copy the page header
@@ -474,6 +484,8 @@ page_zip_verify_checksum(
 /*=====================*/
 	const void*	data,	/*!< in: compressed page */
 	ulint		size);	/*!< in: size of compressed page */
+
+#ifndef UNIV_INNOCHECKSUM
 /**********************************************************************//**
 Write a log record of compressing an index page without the data on the page. */
 UNIV_INLINE
@@ -506,6 +518,8 @@ void
 page_zip_reset_stat_per_index();
 /*===========================*/
 
+#endif /* !UNIV_INNOCHECKSUM */
+
 #ifndef UNIV_HOTBACKUP
 /** Check if a pointer to an uncompressed page matches a compressed page.
 When we IMPORT a tablespace the blocks and accompanying frames are allocted
@@ -531,8 +545,10 @@ from outside the buffer pool.
 # define UNIV_INLINE	UNIV_INLINE_ORIGINAL
 #endif
 
+#ifndef UNIV_INNOCHECKSUM
 #ifndef UNIV_NONINL
 # include "page0zip.ic"
 #endif
+#endif /* !UNIV_INNOCHECKSUM */
 
 #endif /* page0zip_h */

storage/innobase/include/rem0rec.h

@@ -26,11 +26,13 @@ Created 5/30/1994 Heikki Tuuri
 #ifndef rem0rec_h
 #define rem0rec_h
 
+#ifndef UNIV_INNOCHECKSUM
 #include "univ.i"
 #include "data0data.h"
 #include "rem0types.h"
 #include "mtr0types.h"
 #include "page0types.h"
+#endif /* !UNIV_INNOCHECKSUM */
 
 /* Info bit denoting the predefined minimum record: this bit is set
 if and only if the record is the first user record on a non-leaf
@@ -88,6 +90,7 @@ offsets[] array, first passed to rec_get_offsets() */
 #define REC_OFFS_NORMAL_SIZE	100
 #define REC_OFFS_SMALL_SIZE	10
 
+#ifndef UNIV_INNOCHECKSUM
 /******************************************************//**
 The following function is used to get the pointer of the next chained record
 on the same page.
@@ -985,4 +988,5 @@ two upmost bits in a two byte offset for special purposes */
 #include "rem0rec.ic"
 #endif
 
+#endif /* !UNIV_INNOCHECKSUM */
 #endif

storage/innobase/include/srv0srv.h

@@ -361,7 +361,6 @@ extern my_bool			srv_stats_sample_traditional;
 
 extern ibool	srv_use_doublewrite_buf;
 extern ulong	srv_doublewrite_batch_size;
-extern ulong	srv_checksum_algorithm;
 
 extern ibool	srv_use_atomic_writes;
 #ifdef HAVE_POSIX_FALLOCATE

storage/innobase/include/trx0undo.h

@@ -26,6 +26,8 @@ Created 3/26/1996 Heikki Tuuri
 #ifndef trx0undo_h
 #define trx0undo_h
 
+#ifndef UNIV_INNOCHECKSUM
+
 #include "univ.i"
 #include "trx0types.h"
 #include "mtr0mtr.h"
@@ -385,6 +387,8 @@ trx_undo_mem_free(
 /*==============*/
 	trx_undo_t*	undo);		/* in: the undo object to be freed */
 
+#endif /* !UNIV_INNOCHECKSUM */
+
 /* Types of an undo log segment */
 #define	TRX_UNDO_INSERT		1	/* contains undo entries for inserts */
 #define	TRX_UNDO_UPDATE		2	/* contains undo entries for updates
@@ -403,6 +407,7 @@ trx_undo_mem_free(
 					prepared transaction */
 
 #ifndef UNIV_HOTBACKUP
+#ifndef UNIV_INNOCHECKSUM
 /** Transaction undo log memory object; this is protected by the undo_mutex
 in the corresponding transaction object */
 
@@ -461,6 +466,7 @@ struct trx_undo_t{
 					/*!< undo log objects in the rollback
 					segment are chained into lists */
 };
+#endif /* !UNIV_INNOCHECKSUM */
 #endif /* !UNIV_HOTBACKUP */
 
 /** The offset of the undo log page header on pages of the undo log */
@@ -588,8 +594,10 @@ quite a large overhead. */
 					with the XA XID */
 /* @} */
 
+#ifndef UNIV_INNOCHECKSUM
 #ifndef UNIV_NONINL
 #include "trx0undo.ic"
 #endif
+#endif /* !UNIV_INNOCHECKSUM */
 
 #endif

storage/innobase/include/univ.i

@@ -105,7 +105,7 @@ if we are compiling on Windows. */
 /* Include the header file generated by GNU autoconf */
 # ifndef __WIN__
 #  ifndef UNIV_HOTBACKUP
-#   include "config.h"
+#   include "my_config.h"
 #  endif /* UNIV_HOTBACKUP */
 # endif
 

storage/innobase/include/ut0ut.h

@@ -36,6 +36,8 @@ Created 1/20/1994 Heikki Tuuri
 # include "os0sync.h" /* for HAVE_ATOMIC_BUILTINS */
 #endif /* UNIV_HOTBACKUP */
 
+#endif /* !UNIV_INNOCHECKSUM */
+
 #include <time.h>
 #ifndef MYSQL_SERVER
 #include <ctype.h>
@@ -64,6 +66,7 @@ private:
 	F& f;
 };
 
+#ifndef UNIV_INNOCHECKSUM
 #ifndef UNIV_HOTBACKUP
 # if defined(HAVE_PAUSE_INSTRUCTION)
    /* According to the gcc info page, asm volatile means that the
@@ -162,6 +165,7 @@ ut_pair_cmp(
 	ulint	a2,	/*!< in: less significant part of first pair */
 	ulint	b1,	/*!< in: more significant part of second pair */
 	ulint	b2);	/*!< in: less significant part of second pair */
+#endif /* !UNIV_INNOCHECKSUM */
 /*************************************************************//**
 Determines if a number is zero or a power of two.
 @param n	in: number
@@ -192,6 +196,7 @@ when m is a power of two.  In other words, rounds n up to m * k.
 @param m	in: alignment, must be a power of two
 @return		n rounded up to the smallest possible integer multiple of m */
 #define ut_calc_align(n, m) (((n) + ((m) - 1)) & ~((m) - 1))
+#ifndef UNIV_INNOCHECKSUM
 /*************************************************************//**
 Calculates fast the 2-logarithm of a number, rounded upward to an
 integer.

storage/innobase/page/page0zip.cc

@@ -36,6 +36,10 @@ using namespace std;
 # include "page0zip.ic"
 #endif
 #undef THIS_MODULE
+#include "fil0fil.h"
+#include "buf0checksum.h"
+#include "mach0data.h"
+#ifndef UNIV_INNOCHECKSUM
 #include "page0page.h"
 #include "mtr0log.h"
 #include "ut0sort.h"
@@ -43,15 +47,18 @@ using namespace std;
 #include "btr0cur.h"
 #include "page0types.h"
 #include "log0recv.h"
+#endif /* !UNIV_INNOCHECKSUM */
 #include "zlib.h"
 #ifndef UNIV_HOTBACKUP
+#ifndef UNIV_INNOCHECKSUM
 # include "buf0buf.h"
-# include "buf0lru.h"
 # include "btr0sea.h"
 # include "dict0boot.h"
 # include "lock0lock.h"
-# include "srv0mon.h"
 # include "srv0srv.h"
+#endif /* !UNIV_INNOCHECKSUM */
+# include "buf0lru.h"
+# include "srv0mon.h"
 # include "ut0crc32.h"
 #else /* !UNIV_HOTBACKUP */
 # include "buf0checksum.h"
@@ -60,6 +67,7 @@ using namespace std;
 #endif /* !UNIV_HOTBACKUP */
 
 #ifndef UNIV_HOTBACKUP
+#ifndef UNIV_INNOCHECKSUM
 /** Statistics on compression, indexed by page_zip_des_t::ssize - 1 */
 UNIV_INTERN page_zip_stat_t		page_zip_stat[PAGE_ZIP_SSIZE_MAX];
 /** Statistics on compression, indexed by index->id */
@@ -69,6 +77,7 @@ UNIV_INTERN ib_mutex_t			page_zip_stat_per_index_mutex;
 #ifdef HAVE_PSI_INTERFACE
 UNIV_INTERN mysql_pfs_key_t		page_zip_stat_per_index_mutex_key;
 #endif /* HAVE_PSI_INTERFACE */
+#endif /* !UNIV_INNOCHECKSUM */
 #endif /* !UNIV_HOTBACKUP */
 
 /* Compression level to be used by zlib. Settable by user. */
@@ -117,6 +126,7 @@ Compare at most sizeof(field_ref_zero) bytes.
 
 /* Enable some extra debugging output.  This code can be enabled
 independently of any UNIV_ debugging conditions. */
+#ifndef UNIV_INNOCHECKSUM
 #if defined UNIV_DEBUG || defined UNIV_ZIP_DEBUG
 # include <stdarg.h>
 __attribute__((format (printf, 1, 2)))
@@ -149,7 +159,9 @@ page_zip_fail_func(
 @param fmt_args	ignored: printf(3) format string and arguments */
 # define page_zip_fail(fmt_args) /* empty */
 #endif /* UNIV_DEBUG || UNIV_ZIP_DEBUG */
+#endif /* !UNIV_INNOCHECKSUM */
 
+#ifndef UNIV_INNOCHECKSUM
 #ifndef UNIV_HOTBACKUP
 /**********************************************************************//**
 Determine the guaranteed free space on an empty page.
@@ -4838,6 +4850,7 @@ corrupt:
 
 	return(ptr + 8 + size + trailer_size);
 }
+#endif /* !UNIV_INNOCHECKSUM */
 
 /**********************************************************************//**
 Calculate the compressed page checksum.
@@ -4913,6 +4926,10 @@ page_zip_verify_checksum(
 	stored = static_cast<ib_uint32_t>(mach_read_from_4(
 		static_cast<const unsigned char*>(data) + FIL_PAGE_SPACE_OR_CHKSUM));
 
+#ifndef UNIV_INNOCHECKSUM
+	/* innochecksum doesn't compile with ut_d. Since we don't
+	need to check for empty pages when running innochecksum,
+	just don't include this code. */
 	/* declare empty pages non-corrupted */
 	if (stored == 0) {
 		/* make sure that the page is really empty */
@@ -4925,6 +4942,7 @@ page_zip_verify_checksum(
 
 		return(TRUE);
 	}
+#endif
 
 	calc = static_cast<ib_uint32_t>(page_zip_calc_checksum(
 		data, size, static_cast<srv_checksum_algorithm_t>(