Automerge with 5.3-subqueries

mysql-test/r/subselect_no_mat.result

 show variables like 'optimizer_switch';
 Variable_name	Value
-optimizer_switch	index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=on,table_elimination=on
+optimizer_switch	index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=on
 set optimizer_switch='materialization=off';
 drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t11,t12;
 select (select 2);
@@ -1360,9 +1360,9 @@ a
 3
 explain extended select * from t2 where t2.a in (select t1.a from t1,t3 where t1.b=t3.a);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	PRIMARY	t2	index	a	a	5	NULL	4	100.00	Using where; Using index
-1	PRIMARY	t3	index	a	a	5	NULL	3	100.00	Using where; Using index
-1	PRIMARY	t1	ref	a	a	10	test.t2.a,test.t3.a	116	100.00	Using index; FirstMatch(t2)
+1	PRIMARY	t2	index	a	a	5	NULL	4	100.00	Using index
+1	PRIMARY	t3	index	a	a	5	NULL	3	100.00	Using index
+1	PRIMARY	t1	ref	a	a	10	test.t2.a,test.t3.a	116	100.61	Using index; FirstMatch(t2)
 Warnings:
 Note	1003	select `test`.`t2`.`a` AS `a` from `test`.`t2` semi join (`test`.`t1` join `test`.`t3`) where ((`test`.`t1`.`a` = `test`.`t2`.`a`) and (`test`.`t1`.`b` = `test`.`t3`.`a`))
 insert into t1 values (3,31);
@@ -4811,4 +4811,4 @@ End of 5.1 tests.
 set optimizer_switch=default;
 show variables like 'optimizer_switch';
 Variable_name	Value
-optimizer_switch	index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=on,table_elimination=on
+optimizer_switch	index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=on

mysql-test/r/subselect_no_opts.result

 show variables like 'optimizer_switch';
 Variable_name	Value
-optimizer_switch	index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=on,table_elimination=on
+optimizer_switch	index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=on
 set optimizer_switch='materialization=off,semijoin=off';
 drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t11,t12;
 select (select 2);
@@ -4811,4 +4811,4 @@ End of 5.1 tests.
 set optimizer_switch=default;
 show variables like 'optimizer_switch';
 Variable_name	Value
-optimizer_switch	index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=on,table_elimination=on
+optimizer_switch	index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=on

mysql-test/r/subselect_no_semijoin.result

 show variables like 'optimizer_switch';
 Variable_name	Value
-optimizer_switch	index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=on,table_elimination=on
+optimizer_switch	index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=on
 set optimizer_switch='semijoin=off';
 drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t11,t12;
 select (select 2);
@@ -4811,4 +4811,4 @@ End of 5.1 tests.
 set optimizer_switch=default;
 show variables like 'optimizer_switch';
 Variable_name	Value
-optimizer_switch	index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=on,table_elimination=on
+optimizer_switch	index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=on

mysql-test/r/subselect_sj.result

@@ -202,39 +202,39 @@ BUG#37120 optimizer_switch allowable values not according to specification
 
 select @@optimizer_switch;
 @@optimizer_switch
-index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=on,table_elimination=on
+index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=on
 set optimizer_switch='default,materialization=off';
 select @@optimizer_switch;
 @@optimizer_switch
-index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=off,semijoin=on,table_elimination=on
+index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=off,semijoin=on
 set optimizer_switch='default,semijoin=off';
 select @@optimizer_switch;
 @@optimizer_switch
-index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=off,table_elimination=on
+index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=off
 set optimizer_switch='default,loosescan=off';
 select @@optimizer_switch;
 @@optimizer_switch
-index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=off,materialization=on,semijoin=on,table_elimination=on
+index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=off,materialization=on,semijoin=on
 set optimizer_switch='default,semijoin=off,materialization=off';
 select @@optimizer_switch;
 @@optimizer_switch
-index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=off,semijoin=off,table_elimination=on
+index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=off,semijoin=off
 set optimizer_switch='default,materialization=off,semijoin=off';
 select @@optimizer_switch;
 @@optimizer_switch
-index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=off,semijoin=off,table_elimination=on
+index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=off,semijoin=off
 set optimizer_switch='default,semijoin=off,materialization=off,loosescan=off';
 select @@optimizer_switch;
 @@optimizer_switch
-index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=off,materialization=off,semijoin=off,table_elimination=on
+index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=off,materialization=off,semijoin=off
 set optimizer_switch='default,semijoin=off,loosescan=off';
 select @@optimizer_switch;
 @@optimizer_switch
-index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=off,materialization=on,semijoin=off,table_elimination=on
+index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=off,materialization=on,semijoin=off
 set optimizer_switch='default,materialization=off,loosescan=off';
 select @@optimizer_switch;
 @@optimizer_switch
-index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=off,materialization=off,semijoin=on,table_elimination=on
+index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=off,materialization=off,semijoin=on
 set optimizer_switch=default;
 drop table t0, t1, t2;
 drop table t10, t11, t12;

mysql-test/r/subselect_sj_jcl6.result

@@ -206,39 +206,39 @@ BUG#37120 optimizer_switch allowable values not according to specification
 
 select @@optimizer_switch;
 @@optimizer_switch
-index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=on,table_elimination=on
+index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=on
 set optimizer_switch='default,materialization=off';
 select @@optimizer_switch;
 @@optimizer_switch
-index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=off,semijoin=on,table_elimination=on
+index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=off,semijoin=on
 set optimizer_switch='default,semijoin=off';
 select @@optimizer_switch;
 @@optimizer_switch
-index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=off,table_elimination=on
+index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=on,semijoin=off
 set optimizer_switch='default,loosescan=off';
 select @@optimizer_switch;
 @@optimizer_switch
-index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=off,materialization=on,semijoin=on,table_elimination=on
+index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=off,materialization=on,semijoin=on
 set optimizer_switch='default,semijoin=off,materialization=off';
 select @@optimizer_switch;
 @@optimizer_switch
-index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=off,semijoin=off,table_elimination=on
+index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=off,semijoin=off
 set optimizer_switch='default,materialization=off,semijoin=off';
 select @@optimizer_switch;
 @@optimizer_switch
-index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=off,semijoin=off,table_elimination=on
+index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=on,materialization=off,semijoin=off
 set optimizer_switch='default,semijoin=off,materialization=off,loosescan=off';
 select @@optimizer_switch;
 @@optimizer_switch
-index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=off,materialization=off,semijoin=off,table_elimination=on
+index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=off,materialization=off,semijoin=off
 set optimizer_switch='default,semijoin=off,loosescan=off';
 select @@optimizer_switch;
 @@optimizer_switch
-index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=off,materialization=on,semijoin=off,table_elimination=on
+index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=off,materialization=on,semijoin=off
 set optimizer_switch='default,materialization=off,loosescan=off';
 select @@optimizer_switch;
 @@optimizer_switch
-index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=off,materialization=off,semijoin=on,table_elimination=on
+index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_condition_pushdown=on,firstmatch=on,loosescan=off,materialization=off,semijoin=on
 set optimizer_switch=default;
 drop table t0, t1, t2;
 drop table t10, t11, t12;

mysql-test/t/subselect_no_mat.test

 # 
 # Run subselect.test without semi-join optimization (test materialize)
 #
+--replace_regex /,table_elimination=on//
 show variables like 'optimizer_switch';
 set optimizer_switch='materialization=off';
 
 --source t/subselect.test
 
 set optimizer_switch=default;
+--replace_regex /,table_elimination=on//
 show variables like 'optimizer_switch';
 

mysql-test/t/subselect_no_opts.test

 # 
 # Run subselect.test without semi-join optimization (test materialize)
 #
+--replace_regex /,table_elimination=on//
 show variables like 'optimizer_switch';
 set optimizer_switch='materialization=off,semijoin=off';
 
 --source t/subselect.test
 
 set optimizer_switch=default;
+--replace_regex /,table_elimination=on//
 show variables like 'optimizer_switch';
 

mysql-test/t/subselect_no_semijoin.test

 # 
 # Run subselect.test without semi-join optimization (test materialize)
 #
+--replace_regex /,table_elimination=on//
 show variables like 'optimizer_switch';
 set optimizer_switch='semijoin=off';
 
 --source t/subselect.test
 
 set optimizer_switch=default;
+--replace_regex /,table_elimination=on//
 show variables like 'optimizer_switch';
 

mysql-test/t/subselect_sj.test

@@ -96,30 +96,39 @@ explain extended select * from t1 where a in (select pk from t10 where pk<3);
 --echo BUG#37120 optimizer_switch allowable values not according to specification
 --echo
 
+--replace_regex /,table_elimination=on//
 select @@optimizer_switch; 
 
 set optimizer_switch='default,materialization=off';
+--replace_regex /,table_elimination=on//
 select @@optimizer_switch; 
 
 set optimizer_switch='default,semijoin=off';
+--replace_regex /,table_elimination=on//
 select @@optimizer_switch; 
 
 set optimizer_switch='default,loosescan=off';
+--replace_regex /,table_elimination=on//
 select @@optimizer_switch; 
 
 set optimizer_switch='default,semijoin=off,materialization=off';
+--replace_regex /,table_elimination=on//
 select @@optimizer_switch; 
 
 set optimizer_switch='default,materialization=off,semijoin=off';
+--replace_regex /,table_elimination=on//
 select @@optimizer_switch; 
 
 set optimizer_switch='default,semijoin=off,materialization=off,loosescan=off';
+--replace_regex /,table_elimination=on//
 select @@optimizer_switch; 
 
 set optimizer_switch='default,semijoin=off,loosescan=off';
+--replace_regex /,table_elimination=on//
 select @@optimizer_switch; 
 
 set optimizer_switch='default,materialization=off,loosescan=off';
+--replace_regex /,table_elimination=on//
 select @@optimizer_switch; 
 set optimizer_switch=default;
 

sql/item.cc

@@ -1959,7 +1959,7 @@ void Item_field::reset_field(Field *f)
 bool Item_field::enumerate_field_refs_processor(uchar *arg)
 {
   Field_enumerator *fe= (Field_enumerator*)arg;
-  fe->visit_field(field);
+  fe->visit_field(this);
   return FALSE;
 }
 
@@ -5779,6 +5779,35 @@ Item_ref::Item_ref(Name_resolution_context *context_arg,
     set_properties();
 }
 
+/*
+  A Field_enumerator-compatible class that invokes mark_as_dependent() for
+  each field that is a reference to some ancestor of current_select.
+*/
+class Dependency_marker: public Field_enumerator
+{
+public:
+  THD *thd;
+  st_select_lex *current_select;
+  virtual void visit_field(Item_field *item)
+  {
+    // Find which select the field is in. This is achieved by walking up 
+    // the select tree and looking for the table of interest.
+    st_select_lex *sel;
+    for (sel= current_select; sel; sel= sel->outer_select())
+    {
+      TABLE_LIST *tbl;
+      for (tbl= sel->leaf_tables; tbl; tbl= tbl->next_leaf)
+      {
+        if (tbl->table == item->field->table)
+        {
+          if (sel != current_select)
+            mark_as_dependent(thd, sel, current_select, item, item);
+          return;
+        }
+      }
+    }
+  }
+};
 
 /**
   Resolve the name of a reference to a column reference.
@@ -6038,6 +6067,20 @@ bool Item_ref::fix_fields(THD *thd, Item **reference)
                       last_checked_context->select_lex->nest_level);
     }
   }
+  else
+  {
+    ;
+    /*
+      It could be that we're referring to something that's in ancestor selects.
+      We must make an appropriate mark_as_dependent() call for each such
+      outside reference.
+    */
+    Dependency_marker dep_marker;
+    dep_marker.current_select= current_sel;
+    dep_marker.thd= thd;
+    (*ref)->walk(&Item::enumerate_field_refs_processor, FALSE,
+                 (uchar*)&dep_marker);
+  }
 
   DBUG_ASSERT(*ref);
   /*

sql/item.h

@@ -1134,7 +1134,7 @@ public:
 class Field_enumerator
 {
 public:
-  virtual void visit_field(Field *field)= 0;
+  virtual void visit_field(Item_field *field)= 0;
   virtual ~Field_enumerator() {}; /* purecov: inspected */
 };
 
@@ -2378,7 +2378,12 @@ public:
     return ref ? (*ref)->real_item() : this;
   }
   bool walk(Item_processor processor, bool walk_subquery, uchar *arg)
-  { return (*ref)->walk(processor, walk_subquery, arg); }
+  { 
+    if (ref && *ref)
+      return (*ref)->walk(processor, walk_subquery, arg); 
+    else
+      return FALSE;
+  }
   bool enumerate_field_refs_processor(uchar *arg)
   { return (*ref)->enumerate_field_refs_processor(arg); }
   virtual void print(String *str, enum_query_type query_type);

sql/item_subselect.cc

@@ -168,6 +168,11 @@ bool Item_subselect::fix_fields(THD *thd_param, Item **ref)
   {
     done_first_fix_fields= TRUE;
     inside_first_fix_fields= TRUE;
+    upper_refs.empty();
+    /*
+      psergey-todo: remove _first_fix_fields calls, we need changes on every
+      execution
+    */
   }
 
   eliminated= FALSE;
@@ -182,6 +187,7 @@ bool Item_subselect::fix_fields(THD *thd_param, Item **ref)
   changed= 1;
   inside_first_fix_fields= FALSE;
 
+
   if (!res)
   {
     // all transformation is done (used by prepared statements)
@@ -212,12 +218,14 @@ bool Item_subselect::fix_fields(THD *thd_param, Item **ref)
       if (!(*ref)->fixed)
 	ret= (*ref)->fix_fields(thd, ref);
       thd->where= save_where;
+  done_first_fix_fields= FALSE;
       return ret;
     }
     // Is it one field subselect?
     if (engine->cols() > max_columns)
     {
       my_error(ER_OPERAND_COLUMNS, MYF(0), 1);
+  done_first_fix_fields= FALSE;
       return TRUE;
     }
     fix_length_and_dec();
@@ -234,6 +242,7 @@ bool Item_subselect::fix_fields(THD *thd_param, Item **ref)
   fixed= 1;
 
 err:
+  done_first_fix_fields= FALSE;
   thd->where= save_where;
   return res;
 }
@@ -276,6 +285,7 @@ bool Item_subselect::mark_as_dependent(THD *thd, st_select_lex *select,
   return FALSE;
 }
 
+
 /*
   Adjust attributes after our parent select has been merged into grandparent
 
@@ -304,18 +314,19 @@ void Item_subselect::fix_after_pullout(st_select_lex *new_parent, Item **ref)
   parent_select= new_parent;
 }
 
+
 class Field_fixer: public Field_enumerator
 {
 public:
   table_map used_tables; /* Collect used_tables here */
   st_select_lex *new_parent; /* Select we're in */
-  virtual void visit_field(Field *field)
+  virtual void visit_field(Item_field *item)
   {
     //for (TABLE_LIST *tbl= new_parent->leaf_tables; tbl; tbl= tbl->next_local)
     //{
     //  if (tbl->table == field->table)
     //  {
-        used_tables|= field->table->map;
+        used_tables|= item->field->table->map;
     //    return;
     //  }
     //}

sql/opt_table_elimination.cc

@@ -922,8 +922,9 @@ public:
   Field_dependency_recorder(Dep_analysis_context *ctx_arg): ctx(ctx_arg)
   {}
   
-  void visit_field(Field *field)
+  void visit_field(Item_field *item)
   {
+    Field *field= item->field;
     Dep_value_table *tbl_dep;
     if ((tbl_dep= ctx->table_deps[field->table->tablenr]))
     {