Bug#38296 (low memory crash with many conditions in a query)

This fix is for 5.0 only : back porting the 6.0 patch manually

The parser code in sql/sql_yacc.yy needs to be more robust to out of
memory conditions, so that when parsing a query fails due to OOM,
the thread gracefully returns an error.

Before this fix, a new/alloc returning NULL could:
- cause a crash, if dereferencing the NULL pointer,
- produce a corrupted parsed tree, containing NULL nodes,
- alter the semantic of a query, by silently dropping token values or nodes

With this fix:
- C++ constructors are *not* executed with a NULL "this" pointer
when operator new fails.
This is achieved by declaring "operator new" with a "throw ()" clause,
so that a failed new gracefully returns NULL on OOM conditions.

- calls to new/alloc are tested for a NULL result,

- The thread diagnostic area is set to an error status when OOM occurs.
This ensures that a request failing in the server properly returns an
ER_OUT_OF_RESOURCES error to the client.

- OOM conditions cause the parser to stop immediately (MYSQL_YYABORT).
This prevents causing further crashes when using a partially built parsed
tree in further rules in the parser.

No test scripts are provided, since automating OOM failures is not
instrumented in the server.
Tested under the debugger, to verify that an error in alloc_root cause the
thread to returns gracefully all the way to the client application, with
an ER_OUT_OF_RESOURCES error.

mysys/my_alloc.c

@@ -202,7 +202,7 @@ gptr alloc_root(MEM_ROOT *mem_root,unsigned int Size)
     {
       if (mem_root->error_handler)
 	(*mem_root->error_handler)();
-      return((gptr) 0);				/* purecov: inspected */
+      DBUG_RETURN((gptr) 0);			/* purecov: inspected */
     }
     mem_root->block_num++;
     next->next= *prev;

sql/field.h

@@ -48,7 +48,8 @@ class Field
   Field(const Item &);				/* Prevent use of these */
   void operator=(Field &);
 public:
-  static void *operator new(size_t size) {return (void*) sql_alloc((uint) size); }
+  static void *operator new(size_t size) throw ()
+  { return (void*) sql_alloc((uint) size); }
   static void operator delete(void *ptr_arg, size_t size) { TRASH(ptr_arg, size); }
 
   char		*ptr;			// Position to field in record

sql/item.h

@@ -439,9 +439,9 @@ class Item {
   Item(const Item &);			/* Prevent use of these */
   void operator=(Item &);
 public:
-  static void *operator new(size_t size)
+  static void *operator new(size_t size) throw ()
   { return (void*) sql_alloc((uint) size); }
-  static void *operator new(size_t size, MEM_ROOT *mem_root)
+  static void *operator new(size_t size, MEM_ROOT *mem_root) throw ()
   { return (void*) alloc_root(mem_root, (uint) size); }
   static void operator delete(void *ptr,size_t size) { TRASH(ptr, size); }
   static void operator delete(void *ptr, MEM_ROOT *mem_root) {}

sql/sp_head.cc

@@ -446,7 +446,7 @@ sp_head::operator new(size_t size) throw()
   init_sql_alloc(&own_root, MEM_ROOT_BLOCK_SIZE, MEM_ROOT_PREALLOC);
   sp= (sp_head *) alloc_root(&own_root, size);
   if (sp == NULL)
-    return NULL;
+    DBUG_RETURN(NULL);
   sp->main_mem_root= own_root;
   DBUG_PRINT("info", ("mem_root 0x%lx", (ulong) &sp->mem_root));
   DBUG_RETURN(sp);

sql/sql_lex.h

@@ -331,11 +331,11 @@ class st_select_lex_node {
   bool no_table_names_allowed; /* used for global order by */
   bool no_error; /* suppress error message (convert it to warnings) */
 
-  static void *operator new(size_t size)
+  static void *operator new(size_t size) throw ()
   {
     return (void*) sql_alloc((uint) size);
   }
-  static void *operator new(size_t size, MEM_ROOT *mem_root)
+  static void *operator new(size_t size, MEM_ROOT *mem_root) throw ()
   { return (void*) alloc_root(mem_root, (uint) size); }
   static void operator delete(void *ptr,size_t size) { TRASH(ptr, size); }
   static void operator delete(void *ptr, MEM_ROOT *mem_root) {}

sql/sql_list.h

@@ -27,7 +27,7 @@ class Sql_alloc
   {
     return (void*) sql_alloc((uint) size);
   }
-  static void *operator new[](size_t size)
+  static void *operator new[](size_t size) throw ()
   {
     return (void*) sql_alloc((uint) size);
   }
@@ -466,7 +466,7 @@ template <class T> class List_iterator_fast :public base_list_iterator
 struct ilink
 {
   struct ilink **prev,*next;
-  static void *operator new(size_t size)
+  static void *operator new(size_t size) throw ()
   {
     return (void*)my_malloc((uint)size, MYF(MY_WME | MY_FAE));
   }

sql/sql_string.h

@@ -78,7 +78,7 @@ class String
     Alloced_length=str.Alloced_length; alloced=0; 
     str_charset=str.str_charset;
   }
-  static void *operator new(size_t size, MEM_ROOT *mem_root)
+  static void *operator new(size_t size, MEM_ROOT *mem_root) throw ()
   { return (void*) alloc_root(mem_root, (uint) size); }
   static void operator delete(void *ptr_arg,size_t size)
   { TRASH(ptr_arg, size); }

sql/thr_malloc.cc

@@ -21,10 +21,35 @@
 extern "C" {
   void sql_alloc_error_handler(void)
   {
-    THD *thd=current_thd;
-    if (thd)					// QQ;  To be removed
-      thd->fatal_error();			/* purecov: inspected */
     sql_print_error(ER(ER_OUT_OF_RESOURCES));
+
+    THD *thd=current_thd;
+    if (thd)
+    {
+      /*
+        This thread is Out Of Memory.
+        An OOM condition is a fatal error.
+        It should not be caught by error handlers in stored procedures.
+        Also, recording that SQL condition in the condition area could
+        cause more memory allocations, which in turn could raise more
+        OOM conditions, causing recursion in the error handling code itself.
+        As a result, my_error() should not be invoked, and the
+        thread diagnostics area is set to an error status directly.
+        The visible result for a client application will be:
+        - a query fails with an ER_OUT_OF_RESOURCES error,
+        returned in the error packet.
+        - SHOW ERROR/SHOW WARNINGS may be empty.
+      */
+
+      NET *net= &thd->net;
+      thd->fatal_error();
+      if (!net->last_error[0])                  // Return only first message
+      {
+        strmake(net->last_error, ER(ER_OUT_OF_RESOURCES),
+                sizeof(net->last_error)-1);
+        net->last_errno= ER_OUT_OF_RESOURCES;
+      }
+    }
   }
 }