MDEV-10694 - SIGFPE and/or huge memory allocation in maria_create ...

The issue was that JOIN::rollup_write_data() used
JOIN::tmp_table_param::[start_]recinfo, which had uninitialized data.

These fields have uninitialized data, because JOIN::tmp_table_param
currently only stores some grouping-related data fields.  The data about
the work (temporary) tables themselves is stored in
join->join_tab[...].tmp_table_param.

The fix is to make JOIN::rollup_write_data follow this convention
and look at the right TMP_TABLE_PARAM object

mysql-test/r/group_by.result

@@ -2771,3 +2771,12 @@ SELECT 1 IN ( SELECT COUNT( DISTINCT f2 ) FROM t1 WHERE f1 <= 4 );
 1 IN ( SELECT COUNT( DISTINCT f2 ) FROM t1 WHERE f1 <= 4 )
 0
 drop table t1;
+#
+# MDEV-10694 - SIGFPE and/or huge memory allocation in maria_create with distinct/group by/ rollup
+#
+create table t1 (a int,b int) ;
+insert into t1 values(-126,7),(1,1),(0,0),(-1,1),(351,65534);
+select distinct 1 from t1 group by a,b with rollup limit 1;
+1
+1
+drop table t1;

mysql-test/t/group_by.test

@@ -1876,3 +1876,11 @@ INSERT INTO t1 VALUES (0,'foo'),(1,'bar');
 SELECT 1 IN ( SELECT COUNT( DISTINCT f2 ) FROM t1 WHERE f1 <= 4 );
 drop table t1;
 
+--echo #
+--echo # MDEV-10694 - SIGFPE and/or huge memory allocation in maria_create with distinct/group by/ rollup
+--echo #
+create table t1 (a int,b int) ;
+insert into t1 values(-126,7),(1,1),(0,0),(-1,1),(351,65534);
+select distinct 1 from t1 group by a,b with rollup limit 1;
+drop table t1;
+

sql/sql_select.cc

@@ -20037,8 +20037,11 @@ end_write_group(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)),
         }
         if (join->rollup.state != ROLLUP::STATE_NONE)
 	{
-	  if (join->rollup_write_data((uint) (idx+1), table))
+          if (join->rollup_write_data((uint) (idx+1),
+                                      join_tab->tmp_table_param, table))
+          {
 	    DBUG_RETURN(NESTED_LOOP_ERROR);
+          }
 	}
 	if (end_of_records)
 	  goto end;
@@ -23844,7 +23847,7 @@ int JOIN::rollup_send_data(uint idx)
     1   if write_data_failed()
 */
 
-int JOIN::rollup_write_data(uint idx, TABLE *table_arg)
+int JOIN::rollup_write_data(uint idx, TMP_TABLE_PARAM *tmp_table_param_arg, TABLE *table_arg)
 {
   uint i;
   for (i= send_group_parts ; i-- > idx ; )
@@ -23865,8 +23868,8 @@ int JOIN::rollup_write_data(uint idx, TABLE *table_arg)
       if ((write_error= table_arg->file->ha_write_tmp_row(table_arg->record[0])))
       {
 	if (create_internal_tmp_table_from_heap(thd, table_arg, 
-                                                tmp_table_param.start_recinfo,
-                                                &tmp_table_param.recinfo,
+                                                tmp_table_param_arg->start_recinfo,
+                                                &tmp_table_param_arg->recinfo,
                                                 write_error, 0, NULL))
 	  return 1;		     
       }

sql/sql_select.h

@@ -1559,7 +1559,7 @@ class JOIN :public Sql_alloc
   bool rollup_make_fields(List<Item> &all_fields, List<Item> &fields,
 			  Item_sum ***func);
   int rollup_send_data(uint idx);
-  int rollup_write_data(uint idx, TABLE *table);
+  int rollup_write_data(uint idx, TMP_TABLE_PARAM *tmp_table_param, TABLE *table);
   void join_free();
   /** Cleanup this JOIN, possibly for reuse */
   void cleanup(bool full);