merge from 5.5

client/CMakeLists.txt

@@ -64,7 +64,10 @@ MYSQL_ADD_EXECUTABLE(mysqlslap mysqlslap.c)
 SET_SOURCE_FILES_PROPERTIES(mysqlslap.c PROPERTIES COMPILE_FLAGS "-DTHREADS")
 TARGET_LINK_LIBRARIES(mysqlslap mysqlclient)
 
-ADD_EXECUTABLE(echo echo.c)
+# "WIN32" also covers 64 bit. "echo" is used in some files below "mysql-test/".
+IF(WIN32)
+  MYSQL_ADD_EXECUTABLE(echo echo.c)
+ENDIF(WIN32)
 
 SET_TARGET_PROPERTIES (mysqlcheck mysqldump mysqlimport mysql_upgrade mysqlshow mysqlslap 
 PROPERTIES HAS_CXX TRUE)

client/sql_string.cc

@@ -31,9 +31,12 @@
 ** String functions
 *****************************************************************************/
 
-bool String::real_alloc(uint32 arg_length)
+bool String::real_alloc(uint32 length)
 {
-  arg_length=ALIGN_SIZE(arg_length+1);
+  uint32 arg_length= ALIGN_SIZE(length + 1);
+  DBUG_ASSERT(arg_length > length);
+  if (arg_length <= length)
+    return TRUE;                                 /* Overflow */
   str_length=0;
   if (Alloced_length < arg_length)
   {
@@ -56,6 +59,9 @@ bool String::real_alloc(uint32 arg_length)
 bool String::realloc(uint32 alloc_length)
 {
   uint32 len=ALIGN_SIZE(alloc_length+1);
+  DBUG_ASSERT(len > alloc_length);
+  if (len <= alloc_length)
+    return TRUE;                                 /* Overflow */
   if (Alloced_length < len)
   {
     char *new_ptr;

configure.in

@@ -27,7 +27,7 @@ dnl
 dnl When changing the major version number please also check the switch
 dnl statement in mysqlbinlog::check_master_version().  You may also need
 dnl to update version.c in ndb.
-AC_INIT([MySQL Server], [5.5.7-m3], [], [mysql])
+AC_INIT([MySQL Server], [5.5.7-rc], [], [mysql])
 
 AC_CONFIG_SRCDIR([sql/mysqld.cc])
 AC_CANONICAL_SYSTEM

include/CMakeLists.txt

@@ -54,6 +54,7 @@ SET(HEADERS
   keycache.h
   m_ctype.h
   my_attribute.h 
+  my_compiler.h
   ${HEADERS_GEN_CONFIGURE}
 )
 

include/Makefile.am

@@ -32,8 +32,9 @@ pkginclude_HEADERS =	$(HEADERS_ABI) my_dbug.h m_string.h my_sys.h \
 			decimal.h errmsg.h my_global.h my_net.h \
 			my_getopt.h sslopt-longopts.h my_dir.h \
 			sslopt-vars.h sslopt-case.h sql_common.h keycache.h \
-			m_ctype.h my_attribute.h $(HEADERS_GEN_CONFIGURE) \
-			$(HEADERS_GEN_MAKE) probes_mysql.h probes_mysql_nodtrace.h
+			m_ctype.h my_attribute.h my_compiler.h \
+			$(HEADERS_GEN_CONFIGURE) $(HEADERS_GEN_MAKE) \
+			probes_mysql.h probes_mysql_nodtrace.h
 
 noinst_HEADERS =	lf.h my_bit.h \
 			heap.h my_bitmap.h my_uctype.h password.h \
@@ -47,7 +48,7 @@ noinst_HEADERS =	lf.h my_bit.h \
 			my_user.h my_atomic.h atomic/nolock.h \
 			atomic/rwlock.h atomic/x86-gcc.h atomic/generic-msvc.h \
 			atomic/gcc_builtins.h my_libwrap.h my_stacktrace.h \
-			atomic/solaris.h mysql/innodb_priv.h my_compiler.h
+			atomic/solaris.h mysql/innodb_priv.h
 
 pkgpsiinclude_HEADERS = mysql/psi/psi.h mysql/psi/mysql_thread.h \
 			mysql/psi/mysql_file.h

include/m_ctype.h

@@ -539,6 +539,11 @@ size_t my_strnxfrm_unicode(CHARSET_INFO *,
                            uchar *dst, size_t dstlen,
                            const uchar *src, size_t srclen);
 
+size_t my_strnxfrm_unicode_full_bin(CHARSET_INFO *,
+                                    uchar *dst, size_t dstlen,
+                                    const uchar *src, size_t srclen);
+size_t  my_strnxfrmlen_unicode_full_bin(CHARSET_INFO *, size_t); 
+
 int my_wildcmp_unicode(CHARSET_INFO *cs,
                        const char *str, const char *str_end,
                        const char *wildstr, const char *wildend,

mysql-test/include/ctype_filesort2.inc

+#
+# Testing filesort for full Unicode character sets
+# with supplementary characters.
+#
+
+--echo #
+--echo # Bug#55980 Character sets: supplementary character _bin ordering is wrong
+--echo #
+CREATE TABLE t1 AS SELECT REPEAT('a',1) AS a LIMIT 0;
+SHOW CREATE TABLE t1;
+INSERT INTO t1 VALUES (_utf8mb4 0xEFBE9D),(_utf8mb4 0xF0908E84);
+INSERT INTO t1 VALUES (_utf8mb4 0xCE85),(_utf8mb4 0xF4808080);
+SELECT HEX(a), HEX(CONVERT(a USING utf8mb4)) FROM t1 ORDER BY a;
+ALTER TABLE t1 ADD KEY(a);
+SELECT HEX(a), HEX(CONVERT(a USING utf8mb4)) FROM t1 ORDER BY a;
+DROP TABLE IF EXISTS t1;

mysql-test/r/ctype_utf16.result

@@ -611,6 +611,31 @@ utf16_bin	00610009
 utf16_bin	0061
 utf16_bin	00610020
 drop table t1;
+#
+# Bug#55980 Character sets: supplementary character _bin ordering is wrong
+#
+CREATE TABLE t1 AS SELECT REPEAT('a',1) AS a LIMIT 0;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` varchar(1) CHARACTER SET utf16 COLLATE utf16_bin NOT NULL DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES (_utf8mb4 0xEFBE9D),(_utf8mb4 0xF0908E84);
+INSERT INTO t1 VALUES (_utf8mb4 0xCE85),(_utf8mb4 0xF4808080);
+SELECT HEX(a), HEX(CONVERT(a USING utf8mb4)) FROM t1 ORDER BY a;
+HEX(a)	HEX(CONVERT(a USING utf8mb4))
+0385	CE85
+D800DF84	F0908E84
+DBC0DC00	F4808080
+FF9D	EFBE9D
+ALTER TABLE t1 ADD KEY(a);
+SELECT HEX(a), HEX(CONVERT(a USING utf8mb4)) FROM t1 ORDER BY a;
+HEX(a)	HEX(CONVERT(a USING utf8mb4))
+0385	CE85
+D800DF84	F0908E84
+DBC0DC00	F4808080
+FF9D	EFBE9D
+DROP TABLE IF EXISTS t1;
 select @@collation_connection;
 @@collation_connection
 utf16_bin

mysql-test/r/ctype_utf32.result

@@ -610,6 +610,31 @@ utf32_bin	0000006100000009
 utf32_bin	00000061
 utf32_bin	0000006100000020
 drop table t1;
+#
+# Bug#55980 Character sets: supplementary character _bin ordering is wrong
+#
+CREATE TABLE t1 AS SELECT REPEAT('a',1) AS a LIMIT 0;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` varchar(1) CHARACTER SET utf32 COLLATE utf32_bin NOT NULL DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES (_utf8mb4 0xEFBE9D),(_utf8mb4 0xF0908E84);
+INSERT INTO t1 VALUES (_utf8mb4 0xCE85),(_utf8mb4 0xF4808080);
+SELECT HEX(a), HEX(CONVERT(a USING utf8mb4)) FROM t1 ORDER BY a;
+HEX(a)	HEX(CONVERT(a USING utf8mb4))
+00000385	CE85
+0000FF9D	EFBE9D
+00010384	F0908E84
+00100000	F4808080
+ALTER TABLE t1 ADD KEY(a);
+SELECT HEX(a), HEX(CONVERT(a USING utf8mb4)) FROM t1 ORDER BY a;
+HEX(a)	HEX(CONVERT(a USING utf8mb4))
+00000385	CE85
+0000FF9D	EFBE9D
+00010384	F0908E84
+00100000	F4808080
+DROP TABLE IF EXISTS t1;
 select @@collation_connection;
 @@collation_connection
 utf32_bin

mysql-test/r/ctype_utf8mb4.result

@@ -987,6 +987,31 @@ utf8mb4_bin	6109
 utf8mb4_bin	61
 utf8mb4_bin	6120
 drop table t1;
+#
+# Bug#55980 Character sets: supplementary character _bin ordering is wrong
+#
+CREATE TABLE t1 AS SELECT REPEAT('a',1) AS a LIMIT 0;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` varchar(1) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES (_utf8mb4 0xEFBE9D),(_utf8mb4 0xF0908E84);
+INSERT INTO t1 VALUES (_utf8mb4 0xCE85),(_utf8mb4 0xF4808080);
+SELECT HEX(a), HEX(CONVERT(a USING utf8mb4)) FROM t1 ORDER BY a;
+HEX(a)	HEX(CONVERT(a USING utf8mb4))
+CE85	CE85
+EFBE9D	EFBE9D
+F0908E84	F0908E84
+F4808080	F4808080
+ALTER TABLE t1 ADD KEY(a);
+SELECT HEX(a), HEX(CONVERT(a USING utf8mb4)) FROM t1 ORDER BY a;
+HEX(a)	HEX(CONVERT(a USING utf8mb4))
+CE85	CE85
+EFBE9D	EFBE9D
+F0908E84	F0908E84
+F4808080	F4808080
+DROP TABLE IF EXISTS t1;
 select @@collation_connection;
 @@collation_connection
 utf8mb4_bin

mysql-test/r/func_if.result

@@ -186,3 +186,13 @@ MAX(IFNULL(CAST(c AS UNSIGNED), 0))
 12345678901234567890
 DROP TABLE t1;
 End of 5.0 tests
+#
+# Bug#55077: Assertion failed: width > 0 && to != ((void *)0), file .\dtoa.c
+# 
+CREATE TABLE t1 (a LONGBLOB, b DOUBLE);
+INSERT INTO t1 VALUES (NULL, 0), (NULL, 1);
+SELECT IF(b, (SELECT a FROM t1 LIMIT 1), b) c FROM t1 GROUP BY c;
+c
+NULL
+0
+DROP TABLE t1;

mysql-test/r/sp-destruct.result

@@ -134,3 +134,19 @@ Warning	1405	Failed to revoke all privileges to dropped routine
 # Restore the procs_priv table
 RENAME TABLE procs_priv_backup TO mysql.procs_priv;
 FLUSH TABLE mysql.procs_priv;
+#
+# Bug #56137 "Assertion `thd->lock == 0' failed on upgrading from
+#             5.1.50 to 5.5.6".
+#
+drop database if exists mysqltest;
+# Backup mysql.proc.
+flush table mysql.proc;
+create database mysqltest;
+# Corrupt mysql.proc to make it unusable by current version of server.
+alter table mysql.proc drop column type;
+# The below statement should not cause assertion failure.
+drop database mysqltest;
+Warnings:
+Error	1547	Column count of mysql.proc is wrong. Expected 20, found 19. The table is probably corrupted
+# Restore mysql.proc.
+drop table mysql.proc;

mysql-test/t/ctype_utf16.test

@@ -326,6 +326,7 @@ SET collation_connection='utf16_general_ci';
 SET NAMES latin1;
 SET collation_connection='utf16_bin';
 -- source include/ctype_filesort.inc
+-- source include/ctype_filesort2.inc
 -- source include/ctype_like_escape.inc
 
 #

mysql-test/t/ctype_utf32.test

@@ -328,6 +328,7 @@ SET collation_connection='utf32_general_ci';
 SET NAMES latin1;
 SET collation_connection='utf32_bin';
 -- source include/ctype_filesort.inc
+-- source include/ctype_filesort2.inc
 -- source include/ctype_like_escape.inc
 
 #

mysql-test/t/ctype_utf8mb4.test

@@ -733,6 +733,7 @@ SET collation_connection='utf8mb4_general_ci';
 -- source include/ctype_german.inc
 SET collation_connection='utf8mb4_bin';
 -- source include/ctype_filesort.inc
+-- source include/ctype_filesort2.inc
 -- source include/ctype_like_escape.inc
 
 #

mysql-test/t/func_if.test

@@ -165,3 +165,15 @@ DROP TABLE t1;
 
 
 --echo End of 5.0 tests
+
+
+--echo #
+--echo # Bug#55077: Assertion failed: width > 0 && to != ((void *)0), file .\dtoa.c
+--echo # 
+
+CREATE TABLE t1 (a LONGBLOB, b DOUBLE);
+INSERT INTO t1 VALUES (NULL, 0), (NULL, 1);
+
+SELECT IF(b, (SELECT a FROM t1 LIMIT 1), b) c FROM t1 GROUP BY c;
+
+DROP TABLE t1;

mysql-test/t/sp-destruct.test

@@ -222,3 +222,33 @@ SHOW WARNINGS;
 --echo # Restore the procs_priv table
 RENAME TABLE procs_priv_backup TO mysql.procs_priv;
 FLUSH TABLE mysql.procs_priv;
+
+
+--echo #
+--echo # Bug #56137 "Assertion `thd->lock == 0' failed on upgrading from
+--echo #             5.1.50 to 5.5.6".
+--echo #
+--disable_warnings
+drop database if exists mysqltest;
+--enable_warnings
+--echo # Backup mysql.proc.
+flush table mysql.proc;
+let $MYSQLD_DATADIR= `select @@datadir`;
+--copy_file $MYSQLD_DATADIR/mysql/proc.frm $MYSQLTEST_VARDIR/tmp/proc.frm
+--copy_file $MYSQLD_DATADIR/mysql/proc.MYD $MYSQLTEST_VARDIR/tmp/proc.MYD
+--copy_file $MYSQLD_DATADIR/mysql/proc.MYI $MYSQLTEST_VARDIR/tmp/proc.MYI
+
+create database mysqltest;
+--echo # Corrupt mysql.proc to make it unusable by current version of server.
+alter table mysql.proc drop column type;
+--echo # The below statement should not cause assertion failure.
+drop database mysqltest;
+
+--echo # Restore mysql.proc.
+drop table mysql.proc;
+--copy_file $MYSQLTEST_VARDIR/tmp/proc.frm $MYSQLD_DATADIR/mysql/proc.frm
+--copy_file $MYSQLTEST_VARDIR/tmp/proc.MYD $MYSQLD_DATADIR/mysql/proc.MYD
+--copy_file $MYSQLTEST_VARDIR/tmp/proc.MYI $MYSQLD_DATADIR/mysql/proc.MYI 
+--remove_file $MYSQLTEST_VARDIR/tmp/proc.frm
+--remove_file $MYSQLTEST_VARDIR/tmp/proc.MYD
+--remove_file $MYSQLTEST_VARDIR/tmp/proc.MYI

scripts/CMakeLists.txt

@@ -139,6 +139,7 @@ ELSE()
 ENDIF()
 
 SET(HOSTNAME "hostname")
+SET(MYSQLD_USER "mysql")
 
 # Required for mysqlbug until autotools are deprecated, once done remove these
 # and expand default cmake variables

scripts/make_win_bin_dist

@@ -260,6 +260,7 @@ cp include/mysql.h \
    include/keycache.h \
    include/m_ctype.h \
    include/my_attribute.h \
+   include/my_compiler.h \
    include/mysqld_error.h \
    include/sql_state.h \
    include/mysqld_ername.h \

sql/CMakeLists.txt

@@ -271,8 +271,7 @@ IF(WIN32 AND MYSQLD_EXECUTABLE)
      COMMAND ${CMAKE_COMMAND}
      ${CONFIG_PARAM} -P ${CMAKE_CURRENT_BINARY_DIR}/create_initial_db.cmake
      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/data
-     COMMAND  ${CMAKE_COMMAND} -E touch initdb.dep
-     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+     COMMAND  ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/initdb.dep
      DEPENDS mysqld
   )
   ADD_CUSTOM_TARGET(initial_database  

sql/field.cc

@@ -4189,6 +4189,7 @@ String *Field_float::val_str(String *val_buffer,
 			     String *val_ptr __attribute__((unused)))
 {
   ASSERT_COLUMN_MARKED_FOR_READ;
+  DBUG_ASSERT(field_length <= MAX_FIELD_CHARLENGTH);
   float nr;
 #ifdef WORDS_BIGENDIAN
   if (table->s->db_low_byte_first)
@@ -4199,8 +4200,13 @@ String *Field_float::val_str(String *val_buffer,
 #endif
     memcpy(&nr, ptr, sizeof(nr));
 
-  uint to_length=max(field_length,70);
-  val_buffer->alloc(to_length);
+  uint to_length= 70;
+  if (val_buffer->alloc(to_length))
+  {
+    my_error(ER_OUT_OF_RESOURCES, MYF(0));
+    return val_buffer;
+  }
+
   char *to=(char*) val_buffer->ptr();
   size_t len;
 
@@ -4209,7 +4215,7 @@ String *Field_float::val_str(String *val_buffer,
   else
   {
     /*
-      We are safe here because the buffer length is >= 70, and
+      We are safe here because the buffer length is 70, and
       fabs(float) < 10^39, dec < NOT_FIXED_DEC. So the resulting string
       will be not longer than 69 chars + terminating '\0'.
     */
@@ -4506,6 +4512,7 @@ String *Field_double::val_str(String *val_buffer,
 			      String *val_ptr __attribute__((unused)))
 {
   ASSERT_COLUMN_MARKED_FOR_READ;
+  DBUG_ASSERT(field_length <= MAX_FIELD_CHARLENGTH);
   double nr;
 #ifdef WORDS_BIGENDIAN
   if (table->s->db_low_byte_first)
@@ -4515,9 +4522,13 @@ String *Field_double::val_str(String *val_buffer,
   else
 #endif
     doubleget(nr,ptr);
+  uint to_length= DOUBLE_TO_STRING_CONVERSION_BUFFER_SIZE;
+  if (val_buffer->alloc(to_length))
+  {
+    my_error(ER_OUT_OF_RESOURCES, MYF(0));
+    return val_buffer;
+  }
 
-  uint to_length=max(field_length, DOUBLE_TO_STRING_CONVERSION_BUFFER_SIZE);
-  val_buffer->alloc(to_length);
   char *to=(char*) val_buffer->ptr();
   size_t len;
 

sql/item_cmpfunc.cc

@@ -2560,27 +2560,30 @@ Item_func_if::fix_length_and_dec()
     cached_result_type= arg2_type;
     collation.set(args[2]->collation.collation);
     cached_field_type= args[2]->field_type();
+    max_length= args[2]->max_length;
+    return;
   }
-  else if (null2)
+
+  if (null2)
   {
     cached_result_type= arg1_type;
     collation.set(args[1]->collation.collation);
     cached_field_type= args[1]->field_type();
+    max_length= args[1]->max_length;
+    return;
+  }
+
+  agg_result_type(&cached_result_type, args + 1, 2);
+  if (cached_result_type == STRING_RESULT)
+  {
+    if (agg_arg_charsets_for_string_result(collation, args + 1, 2))
+      return;
   }
   else
   {
-    agg_result_type(&cached_result_type, args+1, 2);
-    if (cached_result_type == STRING_RESULT)
-    {
-      if (agg_arg_charsets_for_string_result(collation, args + 1, 2))
-        return;
-    }
-    else
-    {
-      collation.set_numeric(); // Number
-    }
-    cached_field_type= agg_field_type(args + 1, 2);
+    collation.set_numeric(); // Number
   }
+  cached_field_type= agg_field_type(args + 1, 2);
 
   uint32 char_length;
   if ((cached_result_type == DECIMAL_RESULT )

sql/mdl.cc

@@ -124,7 +124,6 @@ class Deadlock_detection_visitor: public MDL_wait_for_graph_visitor
   Deadlock_detection_visitor(MDL_context *start_node_arg)
     : m_start_node(start_node_arg),
       m_victim(NULL),
-      m_current_search_depth(0),
       m_found_deadlock(FALSE)
   {}
   virtual bool enter_node(MDL_context *node);
@@ -133,6 +132,8 @@ class Deadlock_detection_visitor: public MDL_wait_for_graph_visitor
   virtual bool inspect_edge(MDL_context *dest);
 
   MDL_context *get_victim() const { return m_victim; }
+
+  void abort_traversal(MDL_context *node);
 private:
   /**
     Change the deadlock victim to a new one if it has lower deadlock
@@ -147,13 +148,6 @@ class Deadlock_detection_visitor: public MDL_wait_for_graph_visitor
   MDL_context *m_start_node;
   /** If a deadlock is found, the context that identifies the victim. */
   MDL_context *m_victim;
-  /** Set to the 0 at start. Increased whenever
-    we descend into another MDL context (aka traverse to the next
-    wait-for graph node). When MAX_SEARCH_DEPTH is reached, we
-    assume that a deadlock is found, even if we have not found a
-    loop.
-  */
-  uint m_current_search_depth;
   /** TRUE if we found a deadlock. */
   bool m_found_deadlock;
   /**
@@ -187,7 +181,7 @@ class Deadlock_detection_visitor: public MDL_wait_for_graph_visitor
 
 bool Deadlock_detection_visitor::enter_node(MDL_context *node)
 {
-  m_found_deadlock= ++m_current_search_depth >= MAX_SEARCH_DEPTH;
+  m_found_deadlock= m_current_search_depth >= MAX_SEARCH_DEPTH;
   if (m_found_deadlock)
   {
     DBUG_ASSERT(! m_victim);
@@ -207,7 +201,6 @@ bool Deadlock_detection_visitor::enter_node(MDL_context *node)
 
 void Deadlock_detection_visitor::leave_node(MDL_context *node)
 {
-  --m_current_search_depth;
   if (m_found_deadlock)
     opt_change_victim_to(node);
 }
@@ -251,6 +244,21 @@ Deadlock_detection_visitor::opt_change_victim_to(MDL_context *new_victim)
 }
 
 
+/**
+  Abort traversal of a wait-for graph and report a deadlock.
+
+  @param node Node which we were about to visit when abort
+              was initiated.
+*/
+
+void Deadlock_detection_visitor::abort_traversal(MDL_context *node)
+{
+  DBUG_ASSERT(! m_victim);
+  m_found_deadlock= TRUE;
+  opt_change_victim_to(node);
+}
+
+
 /**
   Get a bit corresponding to enum_mdl_type value in a granted/waiting bitmaps
   and compatibility matrices.
@@ -2056,8 +2064,13 @@ bool MDL_lock::visit_subgraph(MDL_ticket *waiting_ticket,
     are visiting it but this is OK: in the worst case we might do some
     extra work and one more context might be chosen as a victim.
   */
+  ++gvisitor->m_current_search_depth;
+
   if (gvisitor->enter_node(src_ctx))
+  {
+    --gvisitor->m_current_search_depth;
     goto end;
+  }
 
   /*
     We do a breadth-first search first -- that is, inspect all
@@ -2114,6 +2127,7 @@ bool MDL_lock::visit_subgraph(MDL_ticket *waiting_ticket,
 
 end_leave_node:
   gvisitor->leave_node(src_ctx);
+  --gvisitor->m_current_search_depth;
 
 end:
   mysql_prlock_unlock(&m_rwlock);

sql/mdl.h

@@ -385,7 +385,10 @@ class MDL_wait_for_graph_visitor
 
   virtual bool inspect_edge(MDL_context *dest) = 0;
   virtual ~MDL_wait_for_graph_visitor();
-  MDL_wait_for_graph_visitor() :m_lock_open_count(0) {}
+  MDL_wait_for_graph_visitor() :m_lock_open_count(0),
+                                m_current_search_depth(0)
+  { }
+  virtual void abort_traversal(MDL_context *node) = 0;
 public:
   /**
    XXX, hack: During deadlock search, we may need to
@@ -396,6 +399,17 @@ class MDL_wait_for_graph_visitor
    LOCK_open since it has significant performance impacts.
   */
   uint m_lock_open_count;
+  /**
+    Set to the 0 at start. Increased whenever
+    we descend into another MDL context (aka traverse to the next
+    wait-for graph node). When MAX_SEARCH_DEPTH is reached, we
+    assume that a deadlock is found, even if we have not found a
+    loop.
+
+    XXX: This member belongs to this class only temporarily until
+         bug #56405 is fixed.
+  */
+  uint m_current_search_depth;
 };
 
 /**

sql/sp.cc

@@ -440,6 +440,7 @@ static TABLE *open_proc_table_for_update(THD *thd)
 {
   TABLE_LIST table_list;
   TABLE *table;
+  MDL_ticket *mdl_savepoint= thd->mdl_context.mdl_savepoint();
   DBUG_ENTER("open_proc_table_for_update");
 
   table_list.init_one_table("mysql", 5, "proc", 4, "proc", TL_WRITE);
@@ -450,6 +451,9 @@ static TABLE *open_proc_table_for_update(THD *thd)
   if (!proc_table_intact.check(table, &proc_table_def))
     DBUG_RETURN(table);
 
+  close_thread_tables(thd);
+  thd->mdl_context.rollback_to_savepoint(mdl_savepoint);
+
   DBUG_RETURN(NULL);
 }
 

sql/sql_base.cc

@@ -100,6 +100,8 @@ bool No_such_table_error_handler::safely_trapped_errors()
   TABLE_SHAREs, refresh_version and the table id counter.
 */
 mysql_mutex_t LOCK_open;
+mysql_mutex_t LOCK_dd_owns_lock_open;
+uint dd_owns_lock_open= 0;
 
 #ifdef HAVE_PSI_INTERFACE
 static PSI_mutex_key key_LOCK_open;
@@ -298,6 +300,7 @@ bool table_def_init(void)
   init_tdc_psi_keys();
 #endif
   mysql_mutex_init(key_LOCK_open, &LOCK_open, MY_MUTEX_INIT_FAST);
+  mysql_mutex_init(NULL, &LOCK_dd_owns_lock_open, MY_MUTEX_INIT_FAST);
   oldest_unused_share= &end_of_unused_share;
   end_of_unused_share.prev= &oldest_unused_share;
 
@@ -341,6 +344,7 @@ void table_def_free(void)
     table_def_inited= 0;
     /* Free table definitions. */
     my_hash_free(&table_def_cache);
+    mysql_mutex_destroy(&LOCK_dd_owns_lock_open);
     mysql_mutex_destroy(&LOCK_open);
   }
   DBUG_VOID_RETURN;

sql/sql_base.h

@@ -71,6 +71,8 @@ enum enum_tdc_remove_table_type {TDC_RT_REMOVE_ALL, TDC_RT_REMOVE_NOT_OWN,
 
 bool check_dup(const char *db, const char *name, TABLE_LIST *tables);
 extern mysql_mutex_t LOCK_open;
+extern mysql_mutex_t LOCK_dd_owns_lock_open;
+extern uint dd_owns_lock_open;
 bool table_cache_init(void);
 void table_cache_free(void);
 bool table_def_init(void);

sql/sql_string.cc

@@ -31,9 +31,12 @@
 ** String functions
 *****************************************************************************/
 
-bool String::real_alloc(uint32 arg_length)
+bool String::real_alloc(uint32 length)
 {
-  arg_length=ALIGN_SIZE(arg_length+1);
+  uint32 arg_length= ALIGN_SIZE(length + 1);
+  DBUG_ASSERT(arg_length > length);
+  if (arg_length <= length)
+    return TRUE;                                 /* Overflow */
   str_length=0;
   if (Alloced_length < arg_length)
   {
@@ -56,6 +59,9 @@ bool String::real_alloc(uint32 arg_length)
 bool String::realloc(uint32 alloc_length)
 {
   uint32 len=ALIGN_SIZE(alloc_length+1);
+  DBUG_ASSERT(len > alloc_length);
+  if (len <= alloc_length)
+    return TRUE;                                 /* Overflow */
   if (Alloced_length < len)
   {
     char *new_ptr;

sql/table.cc

@@ -3085,7 +3085,30 @@ bool TABLE_SHARE::visit_subgraph(Wait_for_flush *wait_for_flush,
     holding a write-lock on MDL_lock::m_rwlock.
   */
   if (gvisitor->m_lock_open_count++ == 0)
+  {
+    /*
+      To circumvent bug #56405 "Deadlock in the MDL deadlock detector"
+      we don't try to lock LOCK_open mutex if some thread doing
+      deadlock detection already owns it and current search depth is
+      greater than 0. Instead we report a deadlock.
+
+      TODO/FIXME: The proper fix for this bug is to use rwlocks for
+                  protection of table shares/instead of LOCK_open.
+                  Unfortunately it requires more effort/has significant
+                  performance effect.
+    */
+    mysql_mutex_lock(&LOCK_dd_owns_lock_open);
+    if (gvisitor->m_current_search_depth > 0 && dd_owns_lock_open > 0)
+    {
+      mysql_mutex_unlock(&LOCK_dd_owns_lock_open);
+      --gvisitor->m_lock_open_count;
+      gvisitor->abort_traversal(src_ctx);
+      return TRUE;
+    }
+    ++dd_owns_lock_open;
+    mysql_mutex_unlock(&LOCK_dd_owns_lock_open);
     mysql_mutex_lock(&LOCK_open);
+  }
 
   I_P_List_iterator <TABLE, TABLE_share> tables_it(used_tables);
 
@@ -3100,8 +3123,12 @@ bool TABLE_SHARE::visit_subgraph(Wait_for_flush *wait_for_flush,
     goto end;
   }
 
+  ++gvisitor->m_current_search_depth;
   if (gvisitor->enter_node(src_ctx))
+  {
+    --gvisitor->m_current_search_depth;
     goto end;
+  }
 
   while ((table= tables_it++))
   {
@@ -3124,10 +3151,16 @@ bool TABLE_SHARE::visit_subgraph(Wait_for_flush *wait_for_flush,
 
 end_leave_node:
   gvisitor->leave_node(src_ctx);
+  --gvisitor->m_current_search_depth;
 
 end:
   if (gvisitor->m_lock_open_count-- == 1)
+  {
     mysql_mutex_unlock(&LOCK_open);
+    mysql_mutex_lock(&LOCK_dd_owns_lock_open);
+    --dd_owns_lock_open;
+    mysql_mutex_unlock(&LOCK_dd_owns_lock_open);
+  }
 
   return result;
 }

strings/ctype-ucs2.c

@@ -1469,7 +1469,7 @@ my_strnncoll_utf16_bin(CHARSET_INFO *cs,
     }
     if (s_wc != t_wc)
     {
-      return  s_wc > t_wc ? 1 : -1;
+      return  my_bincmp(s, s + s_res, t, t + t_res);
     }
     
     s+= s_res;
@@ -1511,7 +1511,7 @@ my_strnncollsp_utf16_bin(CHARSET_INFO *cs,
 
     if (s_wc != t_wc)
     {
-      return s_wc > t_wc ? 1 : -1;
+      return my_bincmp(s, s + s_res, t, t + t_res);
     }
 
     s+= s_res;
@@ -1684,8 +1684,8 @@ static MY_COLLATION_HANDLER my_collation_utf16_bin_handler =
   NULL,                /* init */
   my_strnncoll_utf16_bin,
   my_strnncollsp_utf16_bin,
-  my_strnxfrm_unicode,
-  my_strnxfrmlen_simple,
+  my_strnxfrm_unicode_full_bin,
+  my_strnxfrmlen_unicode_full_bin,
   my_like_range_utf16,
   my_wildcmp_utf16_bin,
   my_strcasecmp_mb2_or_mb4,
@@ -2711,8 +2711,8 @@ static MY_COLLATION_HANDLER my_collation_utf32_bin_handler =
   NULL, /* init */
   my_strnncoll_utf32_bin,
   my_strnncollsp_utf32_bin,
-  my_strnxfrm_unicode,
-  my_strnxfrmlen_utf32,
+  my_strnxfrm_unicode_full_bin,
+  my_strnxfrmlen_unicode_full_bin,
   my_like_range_utf32,
   my_wildcmp_utf32_bin,
   my_strcasecmp_mb2_or_mb4,

strings/ctype-utf8.c

@@ -1893,7 +1893,13 @@ my_wildcmp_unicode(CHARSET_INFO *cs,
 
 
 /*
-  This function is shared between utf8mb3/utf8mb4/ucs2/utf16/utf32
+  Store sorting weights using 2 bytes per character.
+
+  This function is shared between
+  - utf8mb3_general_ci, utf8_bin, ucs2_general_ci, ucs2_bin
+    which support BMP only (U+0000..U+FFFF).
+  - utf8mb4_general_ci, utf16_general_ci, utf32_general_ci,
+    which map all supplementary characters to weight 0xFFFD.
 */
 size_t
 my_strnxfrm_unicode(CHARSET_INFO *cs,
@@ -1937,6 +1943,70 @@ my_strnxfrm_unicode(CHARSET_INFO *cs,
 }
 
 
+/*
+  Store sorting weights using 3 bytes per character.
+  This function is shared between utf8mb4_bin, utf16_bin, utf32_bin.
+*/
+size_t
+my_strnxfrm_unicode_full_bin(CHARSET_INFO *cs,
+                             uchar *dst, size_t dstlen,
+                             const uchar *src, size_t srclen)
+{
+  my_wc_t wc;
+  uchar *de= dst + dstlen;
+  uchar *de_beg= de - 2; /* The beginning of the last chunk */
+  const uchar *se = src + srclen;
+
+  LINT_INIT(wc);
+  DBUG_ASSERT(src);
+  DBUG_ASSERT(cs->state & MY_CS_BINSORT);
+
+  while (dst < de_beg)
+  {
+    int res;
+    if ((res= cs->cset->mb_wc(cs, &wc, src, se)) <= 0)
+      break;
+    src+= res;
+    if (cs->mbminlen == 2) /* utf16_bin */
+    {
+      /*
+        Reorder code points to weights as follows:
+        U+0000..U+D7FF    -> [00][00][00]..[00][D7][FF] BMP part #1
+        U+10000..U+10FFFF -> [01][00][00]..[10][FF][FF] Supplementary
+        U+E000..U+FFFF    -> [20][E0][00]..[20][FF][FF] BMP part #2
+      */
+      if (wc >= 0xE000 && wc <= 0xFFFF)
+        wc+= 0x200000;
+    }
+    *dst++= (uchar) (wc >> 16);
+    *dst++= (uchar) ((wc >> 8) & 0xFF);
+    *dst++= (uchar) (wc & 0xFF);
+  }
+
+  while (dst < de_beg) /* Fill the tail with keys for space character */
+  {
+    *dst++= 0x00;
+    *dst++= 0x00;
+    *dst++= 0x20;
+  }
+
+  /* Clear the last one or two bytes, if "dstlen" was not divisible by 3 */
+  if (dst < de)
+  {
+    *dst++= 0x00;
+    if (dst < de)
+      *dst= 0x00;
+  }
+
+  return dstlen;
+}
+
+
+size_t
+my_strnxfrmlen_unicode_full_bin(CHARSET_INFO *cs, size_t len)
+{
+  return ((len + 3) / cs->mbmaxlen) * 3;
+}
 #endif /* HAVE_UNIDATA */
 
 
@@ -5067,8 +5137,8 @@ static MY_COLLATION_HANDLER my_collation_utf8mb4_bin_handler =
     NULL,		/* init */
     my_strnncoll_mb_bin,
     my_strnncollsp_mb_bin,
-    my_strnxfrm_unicode,
-    my_strnxfrmlen_utf8mb4,
+    my_strnxfrm_unicode_full_bin,
+    my_strnxfrmlen_unicode_full_bin,
     my_like_range_mb,
     my_wildcmp_mb_bin,
     my_strcasecmp_mb_bin,