Fix LP BUG#680058

  
Analysis:
The send_data method of the result sink class used to collect
data statistics about materialized subqueries incorrectly assumed
that duplicate rows are removed prior to calling send_data. As
a result the collected statistics was wrong, which resulted in
an incorrect maximal number of keys in the Ordered_key buffer.
  
Solution:
Try to insert each row into the materialized temp table before
collecting statistics, and if the insertion results in a duplicate
row, do not count the current row.

mysql-test/r/subselect_partial_match.result

@@ -48,3 +48,18 @@ select * from t1 where (a1, a2) not in (select a1, a2 from t1);
 a1	a2
 drop table t1;
 set @@optimizer_switch=@save_optimizer_switch;
+#
+# LP BUG#680058 void Ordered_key::add_key(rownum_t):
+# Assertion `key_buff_elements && cur_key_idx < key_buff_elements' failed
+#
+create table t1 (f1 char(1), f2 char(1));
+insert into t1 values ('t', '0'), ('0', 't');
+create table t2 (f3 char(1), f4 char(1));
+insert into t2 values ('t', NULL), ('t', NULL), ('d', 'y');
+set @save_optimizer_switch=@@optimizer_switch;
+SET @@optimizer_switch='materialization=on,partial_match_rowid_merge=on,partial_match_table_scan=off,semijoin=off';
+select * from t1 where (f1, f2) not in (select * from t2);
+f1	f2
+0	t
+set @@optimizer_switch=@save_optimizer_switch;
+drop table t1, t2;

mysql-test/t/subselect_partial_match.test

@@ -54,3 +54,19 @@ explain select * from t1 where (a1, a2) not in (select a1, a2 from t1);
 select * from t1 where (a1, a2) not in (select a1, a2 from t1);
 drop table t1;
 set @@optimizer_switch=@save_optimizer_switch;
+
+--echo #
+--echo # LP BUG#680058 void Ordered_key::add_key(rownum_t):
+--echo # Assertion `key_buff_elements && cur_key_idx < key_buff_elements' failed
+--echo #
+
+create table t1 (f1 char(1), f2 char(1));
+insert into t1 values ('t', '0'), ('0', 't');
+create table t2 (f3 char(1), f4 char(1));
+insert into t2 values ('t', NULL), ('t', NULL), ('d', 'y');
+
+set @save_optimizer_switch=@@optimizer_switch;
+SET @@optimizer_switch='materialization=on,partial_match_rowid_merge=on,partial_match_table_scan=off,semijoin=off';
+select * from t1 where (f1, f2) not in (select * from t2);
+set @@optimizer_switch=@save_optimizer_switch;
+drop table t1, t2;

sql/sql_class.cc

@@ -3057,6 +3057,13 @@ bool select_materialize_with_stats::send_data(List<Item> &items)
   Column_statistics *cur_col_stat= col_stat;
   uint nulls_in_row= 0;
 
+  if (select_union::send_data(items))
+    return 1;
+  /* Skip duplicate rows. */
+  if (write_err == HA_ERR_FOUND_DUPP_KEY ||
+      write_err == HA_ERR_FOUND_DUPP_UNIQUE)
+    return 0;
+
   ++count_rows;
 
   while ((cur_item= item_it++))
@@ -3074,7 +3081,7 @@ bool select_materialize_with_stats::send_data(List<Item> &items)
   if (nulls_in_row > max_nulls_in_row)
     max_nulls_in_row= nulls_in_row;
 
-  return select_union::send_data(items);
+  return 0;
 }
 
 

sql/sql_class.h

@@ -2978,10 +2978,11 @@ class select_union :public select_result_interceptor
 {
 protected:
   TMP_TABLE_PARAM tmp_table_param;
+  int write_err; /* Error code from the last send_data->ha_write_row call. */
 public:
   TABLE *table;
 
-  select_union() :table(0) { tmp_table_param.init(); }
+  select_union() :write_err(0),table(0) { tmp_table_param.init(); }
   int prepare(List<Item> &list, SELECT_LEX_UNIT *u);
   bool send_data(List<Item> &items);
   bool send_eof();

sql/sql_union.cc

@@ -51,7 +51,6 @@ int select_union::prepare(List<Item> &list, SELECT_LEX_UNIT *u)
 
 bool select_union::send_data(List<Item> &values)
 {
-  int error= 0;
   if (unit->offset_limit_cnt)
   {						// using limit offset,count
     unit->offset_limit_cnt--;
@@ -61,14 +60,14 @@ bool select_union::send_data(List<Item> &values)
   if (thd->is_error())
     return 1;
 
-  if ((error= table->file->ha_write_row(table->record[0])))
+  if ((write_err= table->file->ha_write_row(table->record[0])))
   {
     /* create_internal_tmp_table_from_heap will generate error if needed */
-    if (table->file->is_fatal_error(error, HA_CHECK_DUP) &&
+    if (table->file->is_fatal_error(write_err, HA_CHECK_DUP) &&
         create_internal_tmp_table_from_heap(thd, table,
                                             tmp_table_param.start_recinfo, 
-                                            &tmp_table_param.recinfo, error,
-                                            1))
+                                            &tmp_table_param.recinfo,
+                                            write_err, 1))
       return 1;
   }
   return 0;