WL#2645 (CHECK TABLE FOR UPGRADE)

necessary implementation in the server
mysql_upgrade script added


client/mysqlcheck.c:
  --check-upgrade option added
include/my_base.h:
  errcode added
include/myisam.h:
  option added
scripts/Makefile.am:
  mysql_upgrade script added
sql/handler.cc:
  checks for old types/bugs added
sql/handler.h:
  declarations regarding checks for upgrade
sql/lex.h:
  sym added
sql/share/errmsg.txt:
  error message added
sql/slave.cc:
  now ha_repair is for public use
sql/sql_table.cc:
  upgrade in ha_repair implemented
sql/sql_yacc.yy:
  CHECK ... FOR UPGRADE added to syntax

client/mysqlcheck.c

@@ -34,7 +34,7 @@ static my_bool opt_alldbs = 0, opt_check_only_changed = 0, opt_extended = 0,
                opt_compress = 0, opt_databases = 0, opt_fast = 0,
                opt_medium_check = 0, opt_quick = 0, opt_all_in_1 = 0,
                opt_silent = 0, opt_auto_repair = 0, ignore_errors = 0,
-               tty_password = 0, opt_frm = 0;
+               tty_password = 0, opt_frm = 0, opt_upgrade= 0;
 static uint verbose = 0, opt_mysql_port=0;
 static my_string opt_mysql_unix_port = 0;
 static char *opt_password = 0, *current_user = 0, 
@@ -78,6 +78,9 @@ static struct my_option my_long_options[] =
   {"check-only-changed", 'C',
    "Check only tables that have changed since last check or haven't been closed properly.",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"check-upgrade", 'g',
+   "Check tables for version dependent changes.May be used with auto-repair to correct tables requiring version dependent updates.",
+   0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"compress", OPT_COMPRESS, "Use compression in server/client protocol.",
    (gptr*) &opt_compress, (gptr*) &opt_compress, 0, GET_BOOL, NO_ARG, 0, 0, 0,
    0, 0, 0},
@@ -268,6 +271,10 @@ get_one_option(int optid, const struct my_option *opt __attribute__((unused)),
   case 'r':
     what_to_do = DO_REPAIR;
     break;
+  case 'g':
+    what_to_do= DO_CHECK;
+    opt_upgrade= 1;
+    break;
   case 'W':
 #ifdef __WIN__
     opt_protocol = MYSQL_PROTOCOL_PIPE;
@@ -525,6 +532,7 @@ static int handle_request_for_tables(char *tables, uint length)
     if (opt_medium_check)       end = strmov(end, " MEDIUM"); /* Default */
     if (opt_extended)           end = strmov(end, " EXTENDED");
     if (opt_check_only_changed) end = strmov(end, " CHANGED");
+    if (opt_upgrade)            end = strmov(end, " FOR UPGRADE");
     break;
   case DO_REPAIR:
     op = "REPAIR";

include/my_base.h

@@ -346,8 +346,9 @@ enum ha_base_keytype {
 #define HA_ERR_NO_CONNECTION     157  /* Could not connect to storage engine */
 #define HA_ERR_NULL_IN_SPATIAL   158  /* NULLs are not supported in spatial index */
 #define HA_ERR_TABLE_DEF_CHANGED 159  /* The table changed in storage engine */
+#define HA_ERR_TABLE_NEEDS_UPGRADE 160  /* The table changed in storage engine */
 
-#define HA_ERR_LAST              159  /*Copy last error nr.*/
+#define HA_ERR_LAST              160  /*Copy last error nr.*/
 /* Add error numbers before HA_ERR_LAST and change it accordingly. */
 #define HA_ERR_ERRORS            (HA_ERR_LAST - HA_ERR_FIRST + 1)
 

include/myisam.h

@@ -368,6 +368,7 @@ extern uint mi_get_pointer_length(ulonglong file_length, uint def);
 */
 
 #define TT_USEFRM               1
+#define TT_FOR_UPGRADE          2
 
 #define O_NEW_INDEX	1		/* Bits set in out_flag */
 #define O_NEW_DATA	2

scripts/Makefile.am

@@ -32,6 +32,7 @@ bin_SCRIPTS =		@server_scripts@ \
 			mysqldumpslow \
 			mysql_explain_log \
 			mysql_tableinfo \
+			mysql_upgrade \
 			mysqld_multi \
 			mysql_create_system_tables
 
@@ -59,6 +60,7 @@ EXTRA_SCRIPTS =		make_binary_distribution.sh \
 			mysql_explain_log.sh \
 			mysqld_multi.sh \
 			mysql_tableinfo.sh \
+			mysql_upgrade.sh \
 			mysqld_safe.sh \
 			mysql_create_system_tables.sh
 
@@ -87,6 +89,7 @@ CLEANFILES =		@server_scripts@ \
 			mysqldumpslow \
 			mysql_explain_log \
 			mysql_tableinfo \
+			mysql_upgrade \
 			mysqld_multi \
 			make_win_src_distribution \
 			mysql_create_system_tables

scripts/mysql_upgrade.sh

+#!/bin/sh
+# Copyright (C) 2002-2003 MySQL AB
+# For a more info consult the file COPYRIGHT distributed with this file.
+
+# Runs mysqlcheck --check-upgrade in case it has not been done on this
+# major MySQL version
+
+# This script should always be run when upgrading from one major version
+# to another (ie: 4.1 -> 5.0 -> 5.1)
+
+#
+# Note that in most cases one have to use '--password' as
+# arguments as these needs to be passed on to the mysqlcheck command
+
+
+user=root
+
+case "$1" in
+    --no-defaults|--defaults-file=*|--defaults-extra-file=*)
+      defaults="$1"; shift
+      ;;
+esac
+
+parse_arguments() {
+  # We only need to pass arguments through to the server if we don't
+  # handle them here.  So, we collect unrecognized options (passed on
+  # the command line) into the args variable.
+  pick_args=
+  if test "$1" = PICK-ARGS-FROM-ARGV
+  then
+    pick_args=1
+    shift
+  fi
+
+  for arg do
+    case "$arg" in
+      --basedir=*) MY_BASEDIR_VERSION=`echo "$arg" | sed -e 's/^[^=]*=//'` ;;
+      --user=*) user=`echo "$arg" | sed -e 's/^[^=]*=//'` ;;
+      --ldata=*|--data=*|--datadir=*) DATADIR=`echo "$arg" | sed -e 's/^[^=]*=//'` ;;
+      --force) force=1 ;;
+      --verbose) verbose=1 ;;
+      *)
+        if test -n "$pick_args"
+        then
+          # This sed command makes sure that any special chars are quoted,
+          # so the arg gets passed exactly to the server.
+          args="$args "`echo "$arg" | sed -e 's,\([^a-zA-Z0-9_.=-]\),\\\\\1,g'`
+        fi
+        ;;
+    esac
+  done
+}
+
+#
+# Find where my_print_defaults is
+#
+
+find_my_print_defaults () {
+  if test -x ./bin/my_print_defaults
+  then
+    print_defaults="./bin/my_print_defaults"
+  elif test -x ./extra/my_print_defaults
+  then
+    print_defaults="./extra/my_print_defaults"
+  elif test -x @bindir@/my_print_defaults
+  then
+    print_defaults="@bindir@/my_print_defaults"
+  elif test -x @bindir@/mysql_print_defaults
+  then
+    print_defaults="@bindir@/mysql_print_defaults"
+  else
+    print_defaults="my_print_defaults"
+  fi
+}
+
+find_my_print_defaults
+
+# Get first arguments from the my.cfg file, groups [mysqld] and
+# [mysql_upgrade], and then merge with the command line arguments
+
+args=
+DATADIR=
+bindir=
+MY_BASEDIR_VERSION=
+verbose=0
+force=0
+
+parse_arguments `$print_defaults $defaults mysqld mysql_upgrade`
+parse_arguments PICK-ARGS-FROM-ARGV "$@"
+
+#
+# Try to find where binaries are installed
+#
+
+MY_PWD=`pwd`
+# Check for the directories we would expect from a binary release install
+if test -z "$MY_BASEDIR_VERSION"
+then
+  if test -f ./share/mysql/english/errmsg.sys -a -x ./bin/mysqld
+  then
+    MY_BASEDIR_VERSION=$MY_PWD            # Where bin, share and data are
+    bindir="$MY_BASEDIR_VERSION/bin"
+  # Check for the directories we would expect from a source install
+  elif test -f ./share/mysql/english/errmsg.sys -a -x ./libexec/mysqld
+  then
+    MY_BASEDIR_VERSION=$MY_PWD            # Where libexec, share and var are
+    bindir="$MY_BASEDIR_VERSION/bin"
+# Since we didn't find anything, used the compiled-in defaults
+  else
+    MY_BASEDIR_VERSION=@prefix@
+    bindir=@bindir@
+  fi
+else
+  bindir="$MY_BASEDIR_VERSION/bin"
+fi
+
+#
+# Try to find the data directory
+#
+
+if test -z "$DATADIR"
+then
+  # Try where the binary installs put it
+  if test -d $MY_BASEDIR_VERSION/data/mysql
+  then
+    DATADIR=$MY_BASEDIR_VERSION/data
+  # Next try where the source installs put it
+  elif test -d $MY_BASEDIR_VERSION/var/mysql
+  then
+    DATADIR=$MY_BASEDIR_VERSION/var
+  # Or just give up and use our compiled-in default
+  else
+    DATADIR=@localstatedir@
+  fi
+fi
+
+if test ! -x "$bindir/mysqlcheck"
+then
+  echo "Can't find program '$bindir/mysqlcheck'"
+  echo "Please restart with --basedir=mysql-install-directory"
+  exit 1
+fi
+
+if test ! -f "$DATADIR/mysql/user.frm"
+then
+  echo "Can't find data directory. Please restart with --datadir=path-to-data-dir"
+  exit 1
+fi
+
+CHECK_FILE=$DATADIR/mysql_upgrade.info
+
+if test -f $CHECK_FILE -a $force = 0
+then
+  version=`cat $CHECK_FILE`
+  if test "$version" = "@MYSQL_BASE_VERSION@"
+  then
+    if test $verbose = 1
+    then
+       echo "mysql_upgrade already done for this version"
+    fi
+    $bindir/mysql_fix_privilege_tables --silent $args
+    exit 0
+  fi
+fi
+
+#
+# Run the upgrade
+#
+
+check_args="--check-upgrade --all-databases --auto-repair --user=$user"
+
+if test $verbose = 1
+then
+  echo "Running $bindir/mysqlcheck $args $check_args"
+fi
+
+$bindir/mysqlcheck $check_args $args
+if [ $? = 0 ]
+then
+  # Remember base version so that we don't run this script again on the
+  # same base version
+  echo "@MYSQL_BASE_VERSION@" > $CHECK_FILE
+fi
+
+$bindir/mysql_fix_privilege_tables --silent --user=$user $args

sql/handler.cc

@@ -425,6 +425,7 @@ static int ha_init_errors(void)
   SETMSG(HA_ERR_TABLE_EXIST,            ER(ER_TABLE_EXISTS_ERROR));
   SETMSG(HA_ERR_NO_CONNECTION,          "Could not connect to storage engine");
   SETMSG(HA_ERR_TABLE_DEF_CHANGED,      ER(ER_TABLE_DEF_CHANGED));
+  SETMSG(HA_ERR_TABLE_NEEDS_UPGRADE,    ER(ER_TABLE_NEEDS_UPGRADE));
 
   /* Register the error messages for use with my_error(). */
   return my_error_register(errmsgs, HA_ERR_FIRST, HA_ERR_LAST);
@@ -1795,6 +1796,9 @@ void handler::print_error(int error, myf errflag)
     my_error(ER_NO_SUCH_TABLE, MYF(0), db, table->alias);
     break;
   }
+  case HA_ERR_TABLE_NEEDS_UPGRADE:
+    textno=ER_TABLE_NEEDS_UPGRADE;
+    break;
   default:
     {
       /* The error was "unknown" to this function.
@@ -1836,6 +1840,103 @@ bool handler::get_error_message(int error, String* buf)
 }
 
 
+int handler::ha_check_for_upgrade(HA_CHECK_OPT *check_opt)
+{
+  KEY *keyinfo, *keyend;
+  KEY_PART_INFO *keypart, *keypartend;
+
+  if (!table->s->mysql_version)
+  {
+    /* check for blob-in-key error */
+    keyinfo= table->key_info;
+    keyend= table->key_info + table->s->keys;
+    for (; keyinfo < keyend; keyinfo++)
+    {
+      keypart= keyinfo->key_part;
+      keypartend= keypart + keyinfo->key_parts;
+      for (; keypart < keypartend; keypart++)
+      {
+        if (!keypart->fieldnr)
+          continue;
+        Field *field= table->field[keypart->fieldnr-1];
+        if (field->type() == FIELD_TYPE_BLOB)
+        {
+          if (check_opt->sql_flags & TT_FOR_UPGRADE)
+            check_opt->flags= T_MEDIUM;
+          return HA_ADMIN_NEEDS_CHECK;
+        }
+      }
+    }
+  }
+  return check_for_upgrade(check_opt);
+}
+
+
+int handler::check_old_types()
+{
+  Field** field;
+
+  if (!table->s->mysql_version)
+  {
+    /* check for bad DECIMAL field */
+    for (field= table->field; (*field); field++)
+    {
+      if ((*field)->type() == FIELD_TYPE_NEWDECIMAL)
+      {
+        return HA_ADMIN_NEEDS_ALTER;
+      }
+    }
+  }
+  return 0;
+}
+
+
+static bool update_frm_version(TABLE *table, bool needs_lock)
+{
+  char path[FN_REFLEN];
+  File file;
+  int result= 1;
+  DBUG_ENTER("update_frm_version");
+
+  if (table->s->mysql_version != MYSQL_VERSION_ID)
+    DBUG_RETURN(0);
+
+  strxnmov(path, sizeof(path)-1, mysql_data_home, "/", table->s->db, "/",
+           table->s->table_name, reg_ext, NullS);
+  if (!unpack_filename(path, path))
+    DBUG_RETURN(1);
+
+  if (needs_lock)
+    pthread_mutex_lock(&LOCK_open);
+
+  if ((file= my_open(path, O_RDWR|O_BINARY, MYF(MY_WME))) >= 0)
+  {
+    uchar version[4];
+    char *key= table->s->table_cache_key;
+    uint key_length= table->s->key_length;
+    TABLE *entry;
+    HASH_SEARCH_STATE state;
+
+    int4store(version, MYSQL_VERSION_ID);
+
+    if ((result= my_pwrite(file,(byte*) version,4,51L,MYF_RW)))
+      goto err;
+
+    for (entry=(TABLE*) hash_first(&open_cache,(byte*) key,key_length, &state);
+         entry;
+         entry= (TABLE*) hash_next(&open_cache,(byte*) key,key_length, &state))
+      entry->s->mysql_version= MYSQL_VERSION_ID;
+  }
+err:
+  if (file >= 0)
+    VOID(my_close(file,MYF(MY_WME)));
+  if (needs_lock)
+    pthread_mutex_unlock(&LOCK_open);
+  DBUG_RETURN(result);
+}
+
+
+
 /* Return key if error because of duplicated keys */
 
 uint handler::get_dup_key(int error)
@@ -1903,6 +2004,57 @@ int handler::rename_table(const char * from, const char * to)
   return error;
 }
 
+
+/*
+   Performs checks upon the table.
+
+   SYNOPSIS
+   check()
+   thd                thread doing CHECK TABLE operation
+   check_opt          options from the parser
+
+   NOTES
+
+   RETURN
+   HA_ADMIN_OK                 Successful upgrade
+   HA_ADMIN_NEEDS_UPGRADE      Table has structures requiring upgrade
+   HA_ADMIN_NEEDS_ALTER        Table has structures requiring ALTER TABLE
+   HA_ADMIN_NOT_IMPLEMENTED
+*/
+
+int handler::ha_check(THD *thd, HA_CHECK_OPT *check_opt)
+{
+  int error;
+
+  if ((table->s->mysql_version >= MYSQL_VERSION_ID) &&
+      (check_opt->sql_flags & TT_FOR_UPGRADE))
+    return 0;
+
+  if (table->s->mysql_version < MYSQL_VERSION_ID)
+  {
+    if ((error= check_old_types()))
+      return error;
+    error= ha_check_for_upgrade(check_opt);
+    if (error && (error != HA_ADMIN_NEEDS_CHECK))
+      return error;
+    if (!error && (check_opt->sql_flags & TT_FOR_UPGRADE))
+      return 0;
+  }
+  if ((error= check(thd, check_opt)))
+    return error;
+  return update_frm_version(table, 0);
+}
+
+
+int handler::ha_repair(THD* thd, HA_CHECK_OPT* check_opt)
+{
+  int result;
+  if ((result= repair(thd, check_opt)))
+    return result;
+  return update_frm_version(table, 0);
+}
+
+
 /*
   Tell the storage engine that it is allowed to "disable transaction" in the
   handler. It is a hint that ACID is not required - it is used in NDB for

sql/handler.h

@@ -46,6 +46,9 @@
 #define HA_ADMIN_TRY_ALTER       -7
 #define HA_ADMIN_WRONG_CHECKSUM  -8
 #define HA_ADMIN_NOT_BASE_TABLE  -9
+#define HA_ADMIN_NEEDS_UPGRADE  -10
+#define HA_ADMIN_NEEDS_ALTER    -11
+#define HA_ADMIN_NEEDS_CHECK    -12
 
 /* Bits in table_flags() to show what database can do */
 
@@ -702,10 +705,26 @@ class handler :public Sql_alloc
   { return HA_ERR_WRONG_COMMAND; }
 
   virtual void update_create_info(HA_CREATE_INFO *create_info) {}
+protected:
+  /* to be implemented in handlers */
 
   /* admin commands - called from mysql_admin_table */
   virtual int check(THD* thd, HA_CHECK_OPT* check_opt)
   { return HA_ADMIN_NOT_IMPLEMENTED; }
+
+  /*
+     in these two methods check_opt can be modified
+     to specify CHECK option to use to call check()
+     upon the table
+  */
+  virtual int check_for_upgrade(HA_CHECK_OPT *check_opt)
+  { return 0; }
+public:
+  int ha_check_for_upgrade(HA_CHECK_OPT *check_opt);
+  int check_old_types();
+  /* to be actually called to get 'check()' functionality*/
+  int ha_check(THD *thd, HA_CHECK_OPT *check_opt);
+   
   virtual int backup(THD* thd, HA_CHECK_OPT* check_opt)
   { return HA_ADMIN_NOT_IMPLEMENTED; }
   /*
@@ -714,8 +733,11 @@ class handler :public Sql_alloc
   */
   virtual int restore(THD* thd, HA_CHECK_OPT* check_opt)
   { return HA_ADMIN_NOT_IMPLEMENTED; }
+protected:
   virtual int repair(THD* thd, HA_CHECK_OPT* check_opt)
   { return HA_ADMIN_NOT_IMPLEMENTED; }
+public:
+  int ha_repair(THD* thd, HA_CHECK_OPT* check_opt);
   virtual int optimize(THD* thd, HA_CHECK_OPT* check_opt)
   { return HA_ADMIN_NOT_IMPLEMENTED; }
   virtual int analyze(THD* thd, HA_CHECK_OPT* check_opt)

sql/lex.h

@@ -514,6 +514,7 @@ static SYMBOL symbols[] = {
   { "UNSIGNED",		SYM(UNSIGNED)},
   { "UNTIL",		SYM(UNTIL_SYM)},
   { "UPDATE",		SYM(UPDATE_SYM)},
+  { "UPGRADE",          SYM(UPGRADE_SYM)},
   { "USAGE",		SYM(USAGE)},
   { "USE",		SYM(USE_SYM)},
   { "USER",		SYM(USER)},

sql/share/errmsg.txt

@@ -5607,3 +5607,5 @@ ER_SP_PROC_TABLE_CORRUPT
 	eng "Failed to load routine %s. The table mysql.proc is missing, corrupt, or contains bad data (internal code %d)"
 ER_SP_WRONG_NAME 42000
 	eng "Incorrect routine name '%-.64s'"
+ER_TABLE_NEEDS_UPGRADE
+         eng "Table upgrade required. Please do \"REPAIR TABLE `%-.32s`\" to fix it!"

sql/slave.cc

@@ -1622,7 +1622,7 @@ static int create_table_from_dump(THD* thd, MYSQL *mysql, const char* db,
   save_vio = thd->net.vio;
   thd->net.vio = 0;
   /* Rebuild the index file from the copied data file (with REPAIR) */
-  error=file->repair(thd,&check_opt) != 0;
+  error=file->ha_repair(thd,&check_opt) != 0;
   thd->net.vio = save_vio;
   if (error)
     my_error(ER_INDEX_REBUILD, MYF(0), tables.table->s->table_name);

sql/sql_table.cc

@@ -2328,7 +2328,7 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
       open_for_modify= 0;
     }
 
-    if (table->table->s->crashed && operator_func == &handler::check)
+    if (table->table->s->crashed && operator_func == &handler::ha_check)
     {
       protocol->prepare_for_resend();
       protocol->store(table_name, system_charset_info);
@@ -2340,6 +2340,21 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
         goto err;
     }
 
+    if (operator_func == &handler::ha_repair)
+    {
+      if ((table->table->file->check_old_types() == HA_ADMIN_NEEDS_ALTER) ||
+          (table->table->file->ha_check_for_upgrade(check_opt) ==
+           HA_ADMIN_NEEDS_ALTER))
+      {
+        close_thread_tables(thd);
+        tmp_disable_binlog(thd); // binlogging is done by caller if wanted
+        result_code= mysql_recreate_table(thd, table, 0);
+        reenable_binlog(thd);
+        goto send_result;
+      }
+
+    }
+
     result_code = (table->table->file->*operator_func)(thd, check_opt);
 
 send_result:
@@ -2466,6 +2481,19 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
       break;
     }
 
+    case HA_ADMIN_NEEDS_UPGRADE:
+    case HA_ADMIN_NEEDS_ALTER:
+    {
+      char buf[ERRMSGSIZE];
+      uint length;
+
+      protocol->store(STRING_WITH_LEN("error"), system_charset_info);
+      length=my_snprintf(buf, ERRMSGSIZE, ER(ER_TABLE_NEEDS_UPGRADE), table->table_name);
+      protocol->store(buf, length, system_charset_info);
+      fatal_error=1;
+      break;
+    }
+
     default:				// Probably HA_ADMIN_INTERNAL_ERROR
       {
         char buf[ERRMSGSIZE+20];
@@ -2535,7 +2563,7 @@ bool mysql_repair_table(THD* thd, TABLE_LIST* tables, HA_CHECK_OPT* check_opt)
                                 test(check_opt->sql_flags & TT_USEFRM),
                                 HA_OPEN_FOR_REPAIR,
 				&prepare_for_repair,
-				&handler::repair, 0));
+				&handler::ha_repair, 0));
 }
 
 
@@ -2847,7 +2875,7 @@ bool mysql_check_table(THD* thd, TABLE_LIST* tables,HA_CHECK_OPT* check_opt)
   DBUG_RETURN(mysql_admin_table(thd, tables, check_opt,
 				"check", lock_type,
 				0, HA_OPEN_FOR_REPAIR, 0, 0,
-				&handler::check, &view_checksum));
+				&handler::ha_check, &view_checksum));
 }
 
 

sql/sql_yacc.yy

@@ -628,6 +628,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
 %token  UNTIL_SYM
 %token  UPDATE_SYM
 %token  UPDATE_SYM
+%token  UPGRADE_SYM
 %token  USAGE
 %token  USER
 %token  USE_FRM
@@ -3836,7 +3837,8 @@ mi_check_type:
 	| FAST_SYM { Lex->check_opt.flags|= T_FAST; }
 	| MEDIUM_SYM { Lex->check_opt.flags|= T_MEDIUM; }
 	| EXTENDED_SYM { Lex->check_opt.flags|= T_EXTEND; }
-	| CHANGED  { Lex->check_opt.flags|= T_CHECK_ONLY_CHANGED; };
+	| CHANGED  { Lex->check_opt.flags|= T_CHECK_ONLY_CHANGED; }
+        | FOR_SYM UPGRADE_SYM { Lex->check_opt.sql_flags|= TT_FOR_UPGRADE; };
 
 optimize:
 	OPTIMIZE opt_no_write_to_binlog table_or_tables