BUG#14303860 - EXECUTING A SELECT QUERY WITH TOO 
MANY WILDCARDS CAUSES A SEGFAULT
      Back port from 5.6 and trunk

include/m_ctype.h

@@ -149,6 +149,8 @@ enum my_lex_states
 struct charset_info_st;
 
 
+extern int (*my_string_stack_guard)(int);
+
 /* See strings/CHARSET_INFO.txt for information about this structure  */
 typedef struct my_collation_handler_st
 {

regex/my_regex.h

@@ -28,7 +28,7 @@ typedef struct {
 
 
 /* === regcomp.c === */
-typedef int (*my_regex_stack_check_t)();
+typedef int (*my_regex_stack_check_t)(int);
 extern int my_regcomp(my_regex_t *, const char *, int, CHARSET_INFO *charset);
 #define	REG_BASIC	0000
 #define	REG_EXTENDED	0001

regex/regcomp.c

@@ -227,7 +227,7 @@ int stop;			/* character this ERE should end at */
 		while (MORE() && (c = PEEK()) != '|' && c != stop)
 		{
 		  if (my_regex_enough_mem_in_stack &&
-		      my_regex_enough_mem_in_stack())
+		      my_regex_enough_mem_in_stack(0))
 		  {
 		    SETERROR(REG_ESPACE);
 		    return;

sql/mysqld.cc

@@ -2881,14 +2881,25 @@ sizeof(load_default_groups)/sizeof(load_default_groups[0]);
 
 
 #ifndef EMBEDDED_LIBRARY
-static
-int
-check_enough_stack_size()
+/**
+  This function is used to check for stack overrun for pathological
+  cases of  regular expressions and 'like' expressions.
+  The call to current_thd is  quite expensive, so we try to avoid it
+  for the normal cases.
+  The size of  each stack frame for the wildcmp() routines is ~128 bytes,
+  so checking  *every* recursive call is not necessary.
+ */
+extern "C" int
+check_enough_stack_size(int recurse_level)
 {
   uchar stack_top;
+  if (recurse_level % 16 != 0)
+    return 0;
 
-  return check_stack_overrun(current_thd, STACK_MIN_SIZE,
-                             &stack_top);
+  THD *my_thd= current_thd;
+  if (my_thd != NULL)
+    return check_stack_overrun(my_thd, STACK_MIN_SIZE * 2, &stack_top);
+  return 0;
 }
 #endif
 
@@ -3367,6 +3378,7 @@ static int init_common_variables()
   item_init();
 #ifndef EMBEDDED_LIBRARY
   my_regex_init(&my_charset_latin1, check_enough_stack_size);
+  my_string_stack_guard= check_enough_stack_size;
 #else
   my_regex_init(&my_charset_latin1, NULL);
 #endif

strings/ctype-bin.c

@@ -318,13 +318,16 @@ void my_hash_sort_bin(CHARSET_INFO *cs __attribute__((unused)),
 #define INC_PTR(cs,A,B) (A)++
 
 
-int my_wildcmp_bin(CHARSET_INFO *cs,
+static
+int my_wildcmp_bin_impl(CHARSET_INFO *cs,
                         const char *str,const char *str_end,
                         const char *wildstr,const char *wildend,
-                   int escape, int w_one, int w_many)
+                        int escape, int w_one, int w_many, int recurse_level)
 {
   int result= -1;			/* Not found, using wildcards */
 
+  if (my_string_stack_guard && my_string_stack_guard(recurse_level))
+    return 1;
   while (wildstr != wildend)
   {
     while (*wildstr != w_many && *wildstr != w_one)
@@ -383,8 +386,8 @@ int my_wildcmp_bin(CHARSET_INFO *cs,
 	if (str++ == str_end)
 	  return(-1);
 	{
-	  int tmp=my_wildcmp_bin(cs,str,str_end,wildstr,wildend,escape,w_one,
-				 w_many);
+	  int tmp=my_wildcmp_bin_impl(cs,str,str_end,wildstr,wildend,escape,w_one,
+                                      w_many, recurse_level + 1);
 	  if (tmp <= 0)
 	    return(tmp);
 	}
@@ -395,6 +398,16 @@ int my_wildcmp_bin(CHARSET_INFO *cs,
   return(str != str_end ? 1 : 0);
 }
 
+int my_wildcmp_bin(CHARSET_INFO *cs,
+                   const char *str,const char *str_end,
+                   const char *wildstr,const char *wildend,
+                   int escape, int w_one, int w_many)
+{
+  return my_wildcmp_bin_impl(cs, str, str_end,
+                             wildstr, wildend,
+                             escape, w_one, w_many, 1);
+}
+
 
 static size_t my_strnxfrm_bin(CHARSET_INFO *cs __attribute__((unused)),
                               uchar *dest, size_t dstlen,

strings/ctype-mb.c

@@ -251,13 +251,16 @@ int my_strcasecmp_mb(CHARSET_INFO * cs,const char *s, const char *t)
 
 #define likeconv(s,A) (uchar) (s)->sort_order[(uchar) (A)]
 
-int my_wildcmp_mb(CHARSET_INFO *cs,
+static
+int my_wildcmp_mb_impl(CHARSET_INFO *cs,
                        const char *str,const char *str_end,
                        const char *wildstr,const char *wildend,
-		  int escape, int w_one, int w_many)
+                       int escape, int w_one, int w_many, int recurse_level)
 {
   int result= -1;				/* Not found, using wildcards */
 
+  if (my_string_stack_guard && my_string_stack_guard(recurse_level))
+    return 1;
   while (wildstr != wildend)
   {
     while (*wildstr != w_many && *wildstr != w_one)
@@ -346,8 +349,8 @@ int my_wildcmp_mb(CHARSET_INFO *cs,
           INC_PTR(cs,str, str_end);
         }
 	{
-	  int tmp=my_wildcmp_mb(cs,str,str_end,wildstr,wildend,escape,w_one,
-                                w_many);
+	  int tmp=my_wildcmp_mb_impl(cs,str,str_end,wildstr,wildend,escape,w_one,
+                                     w_many, recurse_level + 1);
 	  if (tmp <= 0)
 	    return (tmp);
 	}
@@ -358,6 +361,16 @@ int my_wildcmp_mb(CHARSET_INFO *cs,
   return (str != str_end ? 1 : 0);
 }
 
+int my_wildcmp_mb(CHARSET_INFO *cs,
+                  const char *str,const char *str_end,
+                  const char *wildstr,const char *wildend,
+                  int escape, int w_one, int w_many)
+{
+  return my_wildcmp_mb_impl(cs, str, str_end,
+                            wildstr, wildend,
+                            escape, w_one, w_many, 1);
+}
+
 
 size_t my_numchars_mb(CHARSET_INFO *cs __attribute__((unused)),
 		      const char *pos, const char *end)
@@ -988,14 +1001,15 @@ pad_min_max:
 }
 
 
-int
-my_wildcmp_mb_bin(CHARSET_INFO *cs,
+static int my_wildcmp_mb_bin_impl(CHARSET_INFO *cs,
                                   const char *str,const char *str_end,
                                   const char *wildstr,const char *wildend,
-                  int escape, int w_one, int w_many)
+                                  int escape, int w_one, int w_many, int recurse_level)
 {
   int result= -1;				/* Not found, using wildcards */
 
+  if (my_string_stack_guard && my_string_stack_guard(recurse_level))
+    return 1;
   while (wildstr != wildend)
   {
     while (*wildstr != w_many && *wildstr != w_one)
@@ -1082,7 +1096,9 @@ my_wildcmp_mb_bin(CHARSET_INFO *cs,
           INC_PTR(cs,str, str_end);
         }
 	{
-	  int tmp=my_wildcmp_mb_bin(cs,str,str_end,wildstr,wildend,escape,w_one,w_many);
+	  int tmp=my_wildcmp_mb_bin_impl(cs,str,str_end,
+                                         wildstr,wildend,escape,
+                                         w_one,w_many, recurse_level+1);
 	  if (tmp <= 0)
 	    return (tmp);
 	}
@@ -1093,6 +1109,17 @@ my_wildcmp_mb_bin(CHARSET_INFO *cs,
   return (str != str_end ? 1 : 0);
 }
 
+int
+my_wildcmp_mb_bin(CHARSET_INFO *cs,
+                  const char *str,const char *str_end,
+                  const char *wildstr,const char *wildend,
+                  int escape, int w_one, int w_many)
+{
+  return my_wildcmp_mb_bin_impl(cs, str, str_end,
+                                wildstr, wildend,
+                                escape, w_one, w_many, 1);
+}
+
 
 /*
   Data was produced from EastAsianWidth.txt 

strings/ctype-simple.c

@@ -841,13 +841,16 @@ cnv:
 #define INC_PTR(cs,A,B) (A)++
 
 
-int my_wildcmp_8bit(CHARSET_INFO *cs,
+static
+int my_wildcmp_8bit_impl(CHARSET_INFO *cs,
                          const char *str,const char *str_end,
                          const char *wildstr,const char *wildend,
-		    int escape, int w_one, int w_many)
+                         int escape, int w_one, int w_many, int recurse_level)
 {
   int result= -1;			/* Not found, using wildcards */
 
+  if (my_string_stack_guard && my_string_stack_guard(recurse_level))
+    return 1;
   while (wildstr != wildend)
   {
     while (*wildstr != w_many && *wildstr != w_one)
@@ -907,8 +910,9 @@ int my_wildcmp_8bit(CHARSET_INFO *cs,
 	  str++;
 	if (str++ == str_end) return(-1);
 	{
-	  int tmp=my_wildcmp_8bit(cs,str,str_end,wildstr,wildend,escape,w_one,
-				  w_many);
+	  int tmp=my_wildcmp_8bit_impl(cs,str,str_end,
+                                       wildstr,wildend,escape,w_one,
+                                       w_many, recurse_level+1);
 	  if (tmp <= 0)
 	    return(tmp);
 	}
@@ -919,6 +923,16 @@ int my_wildcmp_8bit(CHARSET_INFO *cs,
   return(str != str_end ? 1 : 0);
 }
 
+int my_wildcmp_8bit(CHARSET_INFO *cs,
+                    const char *str,const char *str_end,
+                    const char *wildstr,const char *wildend,
+                    int escape, int w_one, int w_many)
+{
+  return my_wildcmp_8bit_impl(cs, str, str_end,
+                              wildstr, wildend,
+                              escape, w_one, w_many, 1);
+}
+
 
 /*
 ** Calculate min_str and max_str that ranges a LIKE string.

strings/ctype-uca.c

@@ -7394,10 +7394,10 @@ static int my_uca_charcmp(CHARSET_INFO *cs, my_wc_t wc1, my_wc_t wc2)
 */
 
 static
-int my_wildcmp_uca(CHARSET_INFO *cs,
+int my_wildcmp_uca_impl(CHARSET_INFO *cs,
                         const char *str,const char *str_end,
                         const char *wildstr,const char *wildend,
-		   int escape, int w_one, int w_many)
+                        int escape, int w_one, int w_many, int recurse_level)
 {
   int result= -1;			/* Not found, using wildcards */
   my_wc_t s_wc, w_wc;
@@ -7406,6 +7406,8 @@ int my_wildcmp_uca(CHARSET_INFO *cs,
                const uchar *, const uchar *);
   mb_wc= cs->cset->mb_wc;
 
+  if (my_string_stack_guard && my_string_stack_guard(recurse_level))
+    return 1;
   while (wildstr != wildend)
   {
     while (1)
@@ -7512,8 +7514,8 @@ int my_wildcmp_uca(CHARSET_INFO *cs,
         if (str == str_end)
           return -1;
         
-        result= my_wildcmp_uca(cs, str, str_end, wildstr, wildend,
-        		       escape, w_one, w_many);
+        result= my_wildcmp_uca_impl(cs, str, str_end, wildstr, wildend,
+                                    escape, w_one, w_many, recurse_level+1);
         
         if (result <= 0)
           return result;
@@ -7525,6 +7527,16 @@ int my_wildcmp_uca(CHARSET_INFO *cs,
   return (str != str_end ? 1 : 0);
 }
 
+int my_wildcmp_uca(CHARSET_INFO *cs,
+                   const char *str,const char *str_end,
+                   const char *wildstr,const char *wildend,
+                   int escape, int w_one, int w_many)
+{
+  return my_wildcmp_uca_impl(cs, str, str_end,
+                             wildstr, wildend,
+                             escape, w_one, w_many, 1);
+}
+
 
 /*
   Collation language is implemented according to

strings/ctype-utf8.c

@@ -1921,12 +1921,12 @@ my_tosort_unicode(MY_UNICASE_INFO **uni_plane, my_wc_t *wc)
 **	 1 if matched with wildcard
 */
 
-int
-my_wildcmp_unicode(CHARSET_INFO *cs,
+static
+int my_wildcmp_unicode_impl(CHARSET_INFO *cs,
                             const char *str,const char *str_end,
                             const char *wildstr,const char *wildend,
                             int escape, int w_one, int w_many,
-                   MY_UNICASE_INFO **weights)
+                            MY_UNICASE_INFO **weights, int recurse_level)
 {
   int result= -1;                             /* Not found, using wildcards */
   my_wc_t s_wc, w_wc;
@@ -1935,6 +1935,8 @@ my_wildcmp_unicode(CHARSET_INFO *cs,
                const uchar *, const uchar *);
   mb_wc= cs->cset->mb_wc;
 
+  if (my_string_stack_guard && my_string_stack_guard(recurse_level))
+    return 1;
   while (wildstr != wildend)
   {
     while (1)
@@ -2056,9 +2058,9 @@ my_wildcmp_unicode(CHARSET_INFO *cs,
           return -1;
         
         str+= scan;
-        result= my_wildcmp_unicode(cs, str, str_end, wildstr, wildend,
+        result= my_wildcmp_unicode_impl(cs, str, str_end, wildstr, wildend,
                                         escape, w_one, w_many,
-                                   weights);
+                                        weights, recurse_level+1);
         if (result <= 0)
           return result;
       } 
@@ -2068,6 +2070,17 @@ my_wildcmp_unicode(CHARSET_INFO *cs,
 }
 
 
+int
+my_wildcmp_unicode(CHARSET_INFO *cs,
+                   const char *str,const char *str_end,
+                   const char *wildstr,const char *wildend,
+                   int escape, int w_one, int w_many,
+                   MY_UNICASE_INFO **weights)
+{
+  return my_wildcmp_unicode_impl(cs, str, str_end,
+                                 wildstr, wildend,
+                                 escape, w_one, w_many, weights, 1);
+}
 /*
   Store sorting weights using 2 bytes per character.
 

strings/ctype.c

@@ -40,6 +40,8 @@
   
 */
 
+int (*my_string_stack_guard)(int)= NULL;
+
 static char *mstr(char *str,const char *src,size_t l1,size_t l2)
 {
   l1= l1<l2 ? l1 : l2;