Merge branch '5.5' into 10.0

5.5 without InnoDB/XtraDB changes

client/mysqlslap.c

 /*
-   Copyright (c) 2005, 2012, Oracle and/or its affiliates.
+   Copyright (c) 2005, 2015, Oracle and/or its affiliates.
+   Copyright (c) 2010, 2015, MariaDB
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -242,7 +243,7 @@ void print_conclusions_csv(conclusions *con);
 void generate_stats(conclusions *con, option_string *eng, stats *sptr);
 uint parse_comma(const char *string, uint **range);
 uint parse_delimiter(const char *script, statement **stmt, char delm);
-uint parse_option(const char *origin, option_string **stmt, char delm);
+int parse_option(const char *origin, option_string **stmt, char delm);
 static int drop_schema(MYSQL *mysql, const char *db);
 uint get_random_string(char *buf);
 static statement *build_table_string(void);
@@ -1264,7 +1265,13 @@ get_options(int *argc,char ***argv)
   if (num_int_cols_opt)
   {
     option_string *str;
-    parse_option(num_int_cols_opt, &str, ',');
+    if(parse_option(num_int_cols_opt, &str, ',') == -1)
+    {
+      fprintf(stderr, "Invalid value specified for the option "
+              "'number-int-cols'\n");
+      option_cleanup(str);
+      return 1;
+    }
     num_int_cols= atoi(str->string);
     if (str->option)
       num_int_cols_index= atoi(str->option);
@@ -1275,7 +1282,13 @@ get_options(int *argc,char ***argv)
   if (num_char_cols_opt)
   {
     option_string *str;
-    parse_option(num_char_cols_opt, &str, ',');
+    if(parse_option(num_char_cols_opt, &str, ',') == -1)
+    {
+      fprintf(stderr, "Invalid value specified for the option "
+              "'number-char-cols'\n");
+      option_cleanup(str);
+      return 1;
+    }
     num_char_cols= atoi(str->string);
     if (str->option)
       num_char_cols_index= atoi(str->option);
@@ -1512,7 +1525,13 @@ get_options(int *argc,char ***argv)
     printf("Parsing engines to use.\n");
 
   if (default_engine)
-    parse_option(default_engine, &engine_options, ',');
+  {
+    if(parse_option(default_engine, &engine_options, ',') == -1)
+    {
+      fprintf(stderr, "Invalid value specified for the option 'engine'\n");
+      return 1;
+    }
+  }
 
   if (tty_password)
     opt_password= get_tty_password(NullS);
@@ -1989,7 +2008,7 @@ pthread_handler_t run_task(void *p)
   DBUG_RETURN(0);
 }
 
-uint
+int
 parse_option(const char *origin, option_string **stmt, char delm)
 {
   char *retstr;
@@ -2014,6 +2033,13 @@ parse_option(const char *origin, option_string **stmt, char delm)
     char buffer[HUGE_STRING_LENGTH]= "";
     char *buffer_ptr;
 
+    /*
+      Return an error if the length of the any of the comma seprated value
+      exceeds HUGE_STRING_LENGTH.
+    */
+    if ((size_t)(retstr - ptr) > HUGE_STRING_LENGTH)
+      return -1;
+
     count++;
     strncpy(buffer, ptr, (size_t)(retstr - ptr));
     /*
@@ -2053,6 +2079,13 @@ parse_option(const char *origin, option_string **stmt, char delm)
   {
     char *origin_ptr;
 
+    /*
+      Return an error if the length of the any of the comma seprated value
+      exceeds HUGE_STRING_LENGTH.
+    */
+    if (strlen(ptr) > HUGE_STRING_LENGTH)
+      return -1;
+
     if ((origin_ptr= strchr(ptr, ':')))
     {
       char *option_ptr;
@@ -2063,13 +2096,13 @@ parse_option(const char *origin, option_string **stmt, char delm)
       option_ptr= (char *)ptr + 1 + tmp->length;
 
       /* Move past the : and the first string */
-      tmp->option_length= (size_t)((ptr + length) - option_ptr);
+      tmp->option_length= strlen(option_ptr);
       tmp->option= my_strndup(option_ptr, tmp->option_length,
                               MYF(MY_FAE));
     }
     else
     {
-      tmp->length= (size_t)((ptr + length) - ptr);
+      tmp->length= strlen(ptr);
       tmp->string= my_strndup(ptr, tmp->length, MYF(MY_FAE));
     }
 

mysql-test/r/merge_recover.result → mysql-test/r/myisam_recover.result

 #
-# Test of MyISAM MRG tables with corrupted children.
+# Tests for corrupted MyISAM tables and MyISAMMRG tables with corrupted
+# children..
+#
 # Run with --myisam-recover=force option.
 #
 # Preparation: we need to make sure that the merge parent
@@ -44,20 +46,20 @@ drop procedure p_create;
 # Switching to connection 'default'
 #
 #
-# We have to disable the ps-protocol, to avoid 
+# We have to disable the ps-protocol, to avoid
 # "Prepared statement needs to be re-prepared" errors
 # -- table def versions change all the time with full table cache.
-# 
+#
 drop table if exists t1, t1_mrg, t1_copy;
 #
 # Prepare a MERGE engine table, that refers to a corrupted
 # child.
-# 
+#
 create table t1 (a int, key(a)) engine=myisam;
 create table t1_mrg (a int) union (t1) engine=merge;
 #
 # Create a table with a corrupted index file:
-# save an old index file, insert more rows, 
+# save an old index file, insert more rows,
 # overwrite the new index file with the old one.
 #
 insert into  t1 (a) values (1), (2), (3);
@@ -101,3 +103,48 @@ execute stmt;
 deallocate prepare stmt;
 set @@global.table_definition_cache=default;
 set @@global.table_open_cache=default;
+#
+# 18075170 - sql node restart required to avoid deadlock after
+#            restore
+#
+# Check that auto-repair for MyISAM tables can now happen in the
+# middle of transaction, without aborting it.
+create table t1 (a int, key(a)) engine=myisam;
+create table t2 (a int);
+insert into t2 values (1);
+# Create a table with a corrupted index file:
+# save an old index file, insert more rows,
+# overwrite the new index file with the old one.
+insert into  t1 (a) values (1);
+flush table t1;
+insert into  t1 (a) values (4);
+flush table t1;
+# Check table is needed to mark the table as crashed.
+check table t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	warning	Size of datafile is: 14       Should be: 7
+test.t1	check	error	Record-count is not ok; is 2   Should be: 1
+test.t1	check	warning	Found 2 key parts. Should be: 1
+test.t1	check	error	Corrupt
+# At this point we have a corrupt t1
+set autocommit = 0;
+select * from t2;
+a
+1
+# Without fix select from t1 will break the transaction. After the fix
+# transaction should be active and should hold lock on table t2. Alter
+# table from con2 will wait only if the transaction is not broken.
+select * from t1;
+a
+1
+4
+Warnings:
+Error	145	Table 't1' is marked as crashed and should be repaired
+Error	1194	Table 't1' is marked as crashed and should be repaired
+Error	1034	Number of rows changed from 1 to 2
+ALTER TABLE t2 ADD val INT;
+# With fix we should have alter table waiting for t2 lock here.
+ROLLBACK;
+SET autocommit = 1;
+# Cleanup
+drop table t1, t2;

mysql-test/r/view.result

@@ -5427,6 +5427,21 @@ View	Create View	character_set_client	collation_connection
 v2	CREATE ALGORITHM=MERGE DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v2` AS select (`t1`.`b` + 1) AS `c` from `t1`	latin1	latin1_swedish_ci
 drop view v2;
 drop table t1;
+#
+# MDEV-8554: Server crashes in base_list_iterator::next_fast on 1st execution of PS with a multi-table update
+#
+CREATE TABLE t1 (a INT) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (1),(2);
+CREATE TABLE t2 (b INT) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (3),(4);
+CREATE TABLE t3 (c INT) ENGINE=MyISAM;
+INSERT INTO t3 VALUES (5),(6);
+CREATE OR REPLACE ALGORITHM=MERGE VIEW v3 AS SELECT * FROM t3;
+PREPARE stmt FROM 'UPDATE t1, t2 SET a = 1 WHERE a IN ( SELECT 0 FROM t3 )';
+UPDATE t1, t2 SET a = 1 WHERE a IN ( SELECT 0 FROM v3 );
+EXECUTE stmt;
+DROP TABLE t1, t2, t3;
+DROP VIEW v3;
 # -----------------------------------------------------------------
 # -- End of 5.5 tests.
 # -----------------------------------------------------------------

mysql-test/suite/innodb/r/add_constraint.result

+#
+# Bug #20762798 FK DDL: CRASH IN DICT_FOREIGN_REMOVE_FROM_CACHE
+#
+create table t1(a int, b int, key(a),key(b))engine=innodb;
+create table t2(a int, b int, key(a),key(b))engine=innodb;
+alter table t2 add constraint b foreign key (b) references t1(a);
+alter table t1 add constraint b1 foreign key (b) references t2(a);
+alter table t2 add constraint b1 foreign key (b) references t1(a);
+ERROR HY000: Can't create table `test`.`#sql-temporary` (errno: 121 "Duplicate key on write or update")
+alter table t2 drop foreign key b;
+alter table t1 drop foreign key b1;
+drop table t2;
+drop table t1;

mysql-test/suite/innodb/t/add_constraint.test

+--source include/have_innodb.inc
+
+--echo #
+--echo # Bug #20762798 FK DDL: CRASH IN DICT_FOREIGN_REMOVE_FROM_CACHE
+--echo #
+
+create table t1(a int, b int, key(a),key(b))engine=innodb;
+create table t2(a int, b int, key(a),key(b))engine=innodb;
+
+alter table t2 add constraint b foreign key (b) references t1(a);
+alter table t1 add constraint b1 foreign key (b) references t2(a);
+
+--replace_regex /#sql-[0-9a-f_]*/#sql-temporary/
+--error ER_CANT_CREATE_TABLE
+alter table t2 add constraint b1 foreign key (b) references t1(a);
+
+alter table t2 drop foreign key b;
+alter table t1 drop foreign key b1;
+
+drop table t2;
+drop table t1;

mysql-test/suite/ndb/r/ndb_restore_discover.result

+#
+# 18075170 - sql node restart required to avoid deadlock after
+#            restore
+#
+CREATE TABLE t1 (id INT) ENGINE=NDBCluster;
+CREATE TABLE t2 (id INT) ENGINE=NDBCluster;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1);
+DROP TABLE t1;
+DROP TABLE t2;
+SET autocommit = 0;
+SELECT * FROM t1;
+id
+1
+SELECT * FROM t2;
+id
+1
+ROLLBACK;
+SET autocommit = 1;
+drop table t1;
+drop table t2;
+SET autocommit = 0;
+SELECT * FROM t1;
+id
+1
+SELECT * FROM t2;
+id
+1
+ALTER TABLE t1 ADD val INT;
+ROLLBACK;
+SET autocommit = 1;
+drop table t1;
+drop table t2;

mysql-test/suite/ndb/t/ndb_restore_discover.test

+-- source include/have_ndb.inc
+-- source include/count_sessions.inc
+
+--echo #
+--echo # 18075170 - sql node restart required to avoid deadlock after
+--echo #            restore
+--echo #
+# Test Auto Discover option within a transaction
+# and make sure the transaction is not broken.
+CREATE TABLE t1 (id INT) ENGINE=NDBCluster;
+CREATE TABLE t2 (id INT) ENGINE=NDBCluster;
+
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1);
+
+-- source include/ndb_backup.inc
+
+DROP TABLE t1;
+DROP TABLE t2;
+
+-- source include/ndb_restore_master.inc
+
+SET autocommit = 0;
+SELECT * FROM t1;
+
+# Without fix below select was resulting in DEADLOCK error. With fix select
+# should succeed.
+SELECT * FROM t2;
+ROLLBACK;
+SET autocommit = 1;
+
+drop table t1;
+drop table t2;
+
+#
+# Checking lock preservation in transaction
+#
+# Using existing backup to create the scenario. Tables are deleted as part of
+# above test cleanup. Thus restoring the backup will bring the system to
+# required state.
+-- source include/ndb_restore_master.inc
+
+SET autocommit = 0;
+SELECT * FROM t1;
+SELECT * FROM t2;
+
+connect(con2, localhost, root);
+--SEND ALTER TABLE t1 ADD val INT
+
+connection default;
+# Alter from con2 will be in waiting state as there is a lock on t1 from
+# default connection due to active transaction. We check for this condition
+# then releasing the lock by rollbacking active transaction.
+let $wait_condition=
+  SELECT count(*) = 1 FROM information_schema.processlist WHERE state
+  LIKE "Waiting%" AND info = "ALTER TABLE t1 ADD val INT";
+--source include/wait_condition.inc
+ROLLBACK;
+SET autocommit = 1;
+
+connection con2;
+--REAP
+
+disconnect con2;
+connection default;
+drop table t1;
+drop table t2;
+
+# Wait till all disconnects are completed
+-- source include/wait_until_count_sessions.inc

mysql-test/t/merge_recover.test → mysql-test/t/myisam_recover.test

+--source include/count_sessions.inc
+
+--echo #
+--echo # Tests for corrupted MyISAM tables and MyISAMMRG tables with corrupted
+--echo # children..
 --echo #
---echo # Test of MyISAM MRG tables with corrupted children.
 --echo # Run with --myisam-recover=force option.
 --echo #
 --echo # Preparation: we need to make sure that the merge parent
@@ -57,10 +61,10 @@ eval $lock;
 --echo #
 connection default;
 --echo #
---echo # We have to disable the ps-protocol, to avoid 
+--echo # We have to disable the ps-protocol, to avoid
 --echo # "Prepared statement needs to be re-prepared" errors
 --echo # -- table def versions change all the time with full table cache.
---echo # 
+--echo #
 --disable_ps_protocol
 --disable_warnings
 drop table if exists t1, t1_mrg, t1_copy;
@@ -69,12 +73,12 @@ let $MYSQLD_DATADIR=`select @@datadir`;
 --echo #
 --echo # Prepare a MERGE engine table, that refers to a corrupted
 --echo # child.
---echo # 
+--echo #
 create table t1 (a int, key(a)) engine=myisam;
 create table t1_mrg (a int) union (t1) engine=merge;
 --echo #
 --echo # Create a table with a corrupted index file:
---echo # save an old index file, insert more rows, 
+--echo # save an old index file, insert more rows,
 --echo # overwrite the new index file with the old one.
 --echo #
 insert into  t1 (a) values (1), (2), (3);
@@ -111,3 +115,66 @@ set @@global.table_open_cache=default;
 disconnect con1;
 connection default;
 --enable_ps_protocol
+
+--echo #
+--echo # 18075170 - sql node restart required to avoid deadlock after
+--echo #            restore
+--echo #
+--echo # Check that auto-repair for MyISAM tables can now happen in the
+--echo # middle of transaction, without aborting it.
+--enable_prepare_warnings
+
+connection default;
+
+create table t1 (a int, key(a)) engine=myisam;
+create table t2 (a int);
+insert into t2 values (1);
+
+--echo # Create a table with a corrupted index file:
+--echo # save an old index file, insert more rows,
+--echo # overwrite the new index file with the old one.
+insert into  t1 (a) values (1);
+flush table t1;
+--copy_file $MYSQLD_DATADIR/test/t1.MYI $MYSQLD_DATADIR/test/t1_copy.MYI
+insert into  t1 (a) values (4);
+flush table t1;
+--remove_file $MYSQLD_DATADIR/test/t1.MYI
+--copy_file $MYSQLD_DATADIR/test/t1_copy.MYI $MYSQLD_DATADIR/test/t1.MYI
+--remove_file $MYSQLD_DATADIR/test/t1_copy.MYI
+
+--echo # Check table is needed to mark the table as crashed.
+check table t1;
+
+--echo # At this point we have a corrupt t1
+set autocommit = 0;
+select * from t2;
+--echo # Without fix select from t1 will break the transaction. After the fix
+--echo # transaction should be active and should hold lock on table t2. Alter
+--echo # table from con2 will wait only if the transaction is not broken.
+--replace_regex /'.*[\/\\]/'/
+select * from t1;
+
+connect(con2, localhost, root);
+--SEND ALTER TABLE t2 ADD val INT
+
+connection default;
+--echo # With fix we should have alter table waiting for t2 lock here.
+let $wait_condition=
+  SELECT count(*) = 1 FROM information_schema.processlist WHERE state
+  LIKE "Waiting%" AND info = "ALTER TABLE t2 ADD val INT";
+
+--source include/wait_condition.inc
+ROLLBACK;
+SET autocommit = 1;
+
+connection con2;
+--REAP
+
+connection default;
+disconnect con2;
+
+--echo # Cleanup
+drop table t1, t2;
+
+# Wait till all disconnects are completed
+-- source include/wait_until_count_sessions.inc

mysql-test/t/view.test

@@ -5380,6 +5380,27 @@ show create view v2;
 drop view v2;
 drop table t1;
 
+--echo #
+--echo # MDEV-8554: Server crashes in base_list_iterator::next_fast on 1st execution of PS with a multi-table update
+--echo #
+CREATE TABLE t1 (a INT) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (1),(2); # Not necessary, the table can be empty
+
+CREATE TABLE t2 (b INT) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (3),(4); # Not necessary, the table can be empty
+
+CREATE TABLE t3 (c INT) ENGINE=MyISAM;
+INSERT INTO t3 VALUES (5),(6); # Not necessary, the table can be empty
+
+CREATE OR REPLACE ALGORITHM=MERGE VIEW v3 AS SELECT * FROM t3;
+
+PREPARE stmt FROM 'UPDATE t1, t2 SET a = 1 WHERE a IN ( SELECT 0 FROM t3 )';
+UPDATE t1, t2 SET a = 1 WHERE a IN ( SELECT 0 FROM v3 );
+EXECUTE stmt;
+
+DROP TABLE t1, t2, t3;
+DROP VIEW v3;
+
 --echo # -----------------------------------------------------------------
 --echo # -- End of 5.5 tests.
 --echo # -----------------------------------------------------------------

mysys/ptr_cmp.c

-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -91,6 +91,7 @@ static int ptr_compare(size_t *compare_length, uchar **a, uchar **b)
   reg3 int length= *compare_length;
   reg1 uchar *first,*last;
 
+  DBUG_ASSERT(length > 0);
   first= *a; last= *b;
   while (--length)
   {

sql/item_sum.cc

@@ -3531,9 +3531,10 @@ bool Item_func_group_concat::setup(THD *thd)
   {
     uint n_elems= arg_count_order + all_fields.elements;
     ref_pointer_array= static_cast<Item**>(thd->alloc(sizeof(Item*) * n_elems));
+    if (!ref_pointer_array)
+      DBUG_RETURN(TRUE);
     memcpy(ref_pointer_array, args, arg_count * sizeof(Item*));
-    if (!ref_pointer_array ||
-        setup_order(thd, ref_pointer_array, context->table_list, list,
+    if (setup_order(thd, ref_pointer_array, context->table_list, list,
                     all_fields, *order))
       DBUG_RETURN(TRUE);
   }

sql/sql_base.cc

@@ -3409,10 +3409,11 @@ request_backoff_action(enum_open_table_action action_arg,
     * We met a broken table that needs repair, or a table that
       is not present on this MySQL server and needs re-discovery.
       To perform the action, we need an exclusive metadata lock on
-      the table. Acquiring an X lock while holding other shared
-      locks is very deadlock-prone. If this is a multi- statement
-      transaction that holds metadata locks for completed
-      statements, we don't do it, and report an error instead.
+      the table. Acquiring X lock while holding other shared
+      locks can easily lead to deadlocks. We rely on MDL deadlock
+      detector to discover them. If this is a multi-statement
+      transaction that holds metadata locks for completed statements,
+      we should keep these locks after discovery/repair.
       The action type in this case is OT_DISCOVER or OT_REPAIR.
     * Our attempt to acquire an MDL lock lead to a deadlock,
       detected by the MDL deadlock detector. The current
@@ -3453,7 +3454,7 @@ request_backoff_action(enum_open_table_action action_arg,
       keep tables open between statements and a livelock
       is not possible.
   */
-  if (action_arg != OT_REOPEN_TABLES && m_has_locks)
+  if (action_arg == OT_BACKOFF_AND_RETRY && m_has_locks)
   {
     my_error(ER_LOCK_DEADLOCK, MYF(0));
     m_thd->mark_transaction_to_rollback(true);
@@ -3481,6 +3482,32 @@ request_backoff_action(enum_open_table_action action_arg,
 }
 
 
+/**
+  An error handler to mark transaction to rollback on DEADLOCK error
+  during DISCOVER / REPAIR.
+*/
+class MDL_deadlock_discovery_repair_handler : public Internal_error_handler
+{
+public:
+  virtual bool handle_condition(THD *thd,
+                                  uint sql_errno,
+                                  const char* sqlstate,
+                                  Sql_condition::enum_warning_level level,
+                                  const char* msg,
+                                  Sql_condition ** cond_hdl)
+  {
+    if (sql_errno == ER_LOCK_DEADLOCK)
+    {
+      thd->mark_transaction_to_rollback(true);
+    }
+    /*
+      We have marked this transaction to rollback. Return false to allow
+      error to be reported or handled by other handlers.
+    */
+    return false;
+  }
+};
+
 /**
    Recover from failed attempt of open table by performing requested action.
 
@@ -3495,6 +3522,12 @@ bool
 Open_table_context::recover_from_failed_open()
 {
   bool result= FALSE;
+  MDL_deadlock_discovery_repair_handler handler;
+  /*
+    Install error handler to mark transaction to rollback on DEADLOCK error.
+  */
+  m_thd->push_internal_handler(&handler);
+
   /* Execute the action. */
   switch (m_action)
   {
@@ -3530,7 +3563,12 @@ Open_table_context::recover_from_failed_open()
             result= FALSE;
         }
 
-        m_thd->mdl_context.release_transactional_locks();
+        /*
+          Rollback to start of the current statement to release exclusive lock
+          on table which was discovered but preserve locks from previous statements
+          in current transaction.
+        */
+        m_thd->mdl_context.rollback_to_savepoint(start_of_statement_svp());
         break;
       }
     case OT_REPAIR:
@@ -3543,12 +3581,18 @@ Open_table_context::recover_from_failed_open()
                          m_failed_table->table_name, FALSE);
 
         result= auto_repair_table(m_thd, m_failed_table);
-        m_thd->mdl_context.release_transactional_locks();
+        /*
+          Rollback to start of the current statement to release exclusive lock
+          on table which was discovered but preserve locks from previous statements
+          in current transaction.
+        */
+        m_thd->mdl_context.rollback_to_savepoint(start_of_statement_svp());
         break;
       }
     default:
       DBUG_ASSERT(0);
   }
+  m_thd->pop_internal_handler();
   /*
     Reset the pointers to conflicting MDL request and the
     TABLE_LIST element, set when we need auto-discovery or repair,

sql/sql_parse.cc

@@ -6269,6 +6269,8 @@ void THD::reset_for_next_command()
   thd->reset_current_stmt_binlog_format_row();
   thd->binlog_unsafe_warning_flags= 0;
 
+  thd->save_prep_leaf_list= false;
+
   DBUG_PRINT("debug",
              ("is_current_stmt_binlog_format_row(): %d",
               thd->is_current_stmt_binlog_format_row()));