MDEV-11588 Support for ONLY_FULL_GROUP_BY functional dependency

Support functional dependencies usage when ONLY_FULL_GROUP_BY
SQL_MODE is enabled.

It is allowed to use fields that are either GROUP BY fields or
fields that are functionally dependent on GROUP BY fields in
SELECT list, HAVING clause and ORDER BY clause.
Functionally dependent fields can be extracted from the WHERE clause
(for the most outer JOIN tables) and ON expressions (for JOIN weak part
tables) equalities.

For these equalities such rules should hold:
1. Equalities should be of the form (1):

   F2 = g(H11,..,H1n), where

   (H11, ..., H1n)  are some functions of GROUP BY fields and/or
                    GROUP BY fields and/or constants.
    g               is some function. It can be identity function.
    F2              is some non GROUP BY field.

    F2 here can be extracted as a new functionally dependent field.
2. If F2 is from the left part of LEFT JOIN it can’t be extracted
3. If F2 is from the right part of the LEFT JOIN either:
   2.1. H11,...,H1n should be fields from the right part
   2.2  n=1 and H1 is from the left part of the considered JOIN
4. If (1) is used in ON expression this ON expression can't depend
   on non GROUP BY fields of the left part of the LEFT JOIN.
5. If (1) is used in ON expression this ON expression can't depend
   on the right JOIN part fields only.
6. 4 and 5 doesn't work if F2 table has at least one field used in
   GROUP BY.

mysql-test/main/derived_cond_pushdown.result

@@ -16611,7 +16611,7 @@ EXPLAIN EXTENDED
 SELECT * FROM v1 JOIN v2 ON v1.f = v2.f;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE noticed after reading const tables
-3	LATERAL DERIVED	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	no matching row in const table
+3	LATERAL DERIVED	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE noticed after reading const tables
 Warnings:
 Note	1003	/* select#1 */ select NULL AS `f`,`v2`.`f` AS `f` from `test`.`t1` `a` straight_join `test`.`t1` `b` join `test`.`v2` where 0
 DROP VIEW v1,v2;

mysql-test/main/group_by.result

@@ -992,7 +992,7 @@ a + 1
 6
 7
 SELECT a + b FROM t1 GROUP BY b;
-ERROR 42000: 'test.t1.a' isn't in GROUP BY
+ERROR 42000: Non-grouping field 'test.t1.a' is used in SELECT list
 SELECT (SELECT t1_outer.a FROM t1 AS t1_inner GROUP BY b LIMIT 1) 
 FROM t1 AS t1_outer;
 (SELECT t1_outer.a FROM t1 AS t1_inner GROUP BY b LIMIT 1)
@@ -1013,7 +1013,7 @@ HAVING (SELECT t1_outer.a FROM t1 AS t1_inner GROUP BY b LIMIT 1);
 1
 SELECT (SELECT t1_outer.a FROM t1 AS t1_inner LIMIT 1) 
 FROM t1 AS t1_outer GROUP BY t1_outer.b;
-ERROR 42000: 'test.t1_outer.a' isn't in GROUP BY
+ERROR 42000: Non-grouping field 'test.t1_outer.a' is used in SELECT list
 SELECT 1 FROM t1 as t1_outer GROUP BY a 
 HAVING (SELECT t1_outer.b FROM t1 AS t1_inner LIMIT 1);
 ERROR 42S22: Unknown column 'test.t1_outer.b' in 'field list'
@@ -1034,7 +1034,10 @@ FROM t1 AS t1_outer;
 3
 SELECT (SELECT SUM(t1_outer.a) FROM t1 AS t1_inner LIMIT 1) 
 FROM t1 AS t1_outer GROUP BY t1_outer.b;
-ERROR 42000: 'test.t1_outer.a' isn't in GROUP BY
+(SELECT SUM(t1_outer.a) FROM t1 AS t1_inner LIMIT 1)
+3
+7
+11
 SELECT 1 FROM t1 as t1_outer 
 WHERE (SELECT t1_outer.b FROM t1 AS t1_inner GROUP BY t1_inner.b LIMIT 1);
 1
@@ -1063,19 +1066,20 @@ ERROR HY000: Invalid use of group function
 SELECT b FROM t1 AS t1_outer GROUP BY a HAVING t1_outer.a IN 
 (SELECT SUM(t1_inner.b)+t1_outer.b FROM t1 AS t1_inner GROUP BY t1_inner.a
 HAVING SUM(t1_inner.b)+t1_outer.b > 5);
-ERROR 42000: 'test.t1_outer.b' isn't in GROUP BY
+b
+3
 DROP TABLE t1;
 SET SQL_MODE = '';
 SET SQL_MODE = 'ONLY_FULL_GROUP_BY';
 create table t1(f1 int, f2 int);
 select * from t1 group by f1;
-ERROR 42000: 'test.t1.f2' isn't in GROUP BY
+ERROR 42000: Non-grouping field 'test.t1.f2' is used in SELECT list
 select * from t1 group by f2;
-ERROR 42000: 'test.t1.f1' isn't in GROUP BY
+ERROR 42000: Non-grouping field 'test.t1.f1' is used in SELECT list
 select * from t1 group by f1, f2;
 f1	f2
 select t1.f1,t.* from t1, t1 t group by 1;
-ERROR 42000: 'test.t.f1' isn't in GROUP BY
+ERROR 42000: Non-grouping field 'test.t.f1' is used in SELECT list
 drop table t1;
 SET SQL_MODE = '';
 CREATE TABLE t1(
@@ -1723,7 +1727,7 @@ c	(SELECT a FROM t1 WHERE b = c)
 SELECT b c, (SELECT a FROM t1 WHERE b = c)
 FROM t1 
 HAVING b = 10;
-ERROR 42000: Non-grouping field 'b' is used in HAVING clause
+ERROR 42000: Non-grouping field 'test.t1.b' is used in SELECT list
 SELECT MAX(b) c, (SELECT a FROM t1 WHERE b = c)
 FROM t1 
 HAVING b = 10;

mysql-test/main/group_by.test

@@ -743,13 +743,13 @@ SELECT CEILING(MIN(a)) FROM t1 GROUP BY b;
 SELECT CASE WHEN AVG(a)>=0 THEN 'Positive' ELSE 'Negative' END FROM t1 
  GROUP BY b;
 SELECT a + 1 FROM t1 GROUP BY a;
---error ER_WRONG_FIELD_WITH_GROUP 
+--error ER_NON_GROUPING_FIELD_USED
 SELECT a + b FROM t1 GROUP BY b;
 SELECT (SELECT t1_outer.a FROM t1 AS t1_inner GROUP BY b LIMIT 1) 
   FROM t1 AS t1_outer;
 SELECT 1 FROM t1 as t1_outer GROUP BY a 
   HAVING (SELECT t1_outer.a FROM t1 AS t1_inner GROUP BY b LIMIT 1);
---error ER_WRONG_FIELD_WITH_GROUP 
+--error ER_NON_GROUPING_FIELD_USED
 SELECT (SELECT t1_outer.a FROM t1 AS t1_inner LIMIT 1) 
   FROM t1 AS t1_outer GROUP BY t1_outer.b;
 --error ER_BAD_FIELD_ERROR 
@@ -759,7 +759,6 @@ SELECT (SELECT SUM(t1_inner.a) FROM t1 AS t1_inner LIMIT 1)
   FROM t1 AS t1_outer GROUP BY t1_outer.b;
 SELECT (SELECT SUM(t1_inner.a) FROM t1 AS t1_inner GROUP BY t1_inner.b LIMIT 1)
   FROM t1 AS t1_outer;
---error ER_WRONG_FIELD_WITH_GROUP 
 SELECT (SELECT SUM(t1_outer.a) FROM t1 AS t1_inner LIMIT 1) 
   FROM t1 AS t1_outer GROUP BY t1_outer.b;
 
@@ -775,7 +774,6 @@ SELECT 1 FROM t1 GROUP BY b HAVING ROW (b,b) = ROW (1,1);
 SELECT 1 FROM t1 GROUP BY b HAVING a = 2;
 --error ER_INVALID_GROUP_FUNC_USE
 SELECT 1 FROM t1 GROUP BY SUM(b);
---error ER_WRONG_FIELD_WITH_GROUP 
 SELECT b FROM t1 AS t1_outer GROUP BY a HAVING t1_outer.a IN 
   (SELECT SUM(t1_inner.b)+t1_outer.b FROM t1 AS t1_inner GROUP BY t1_inner.a
    HAVING SUM(t1_inner.b)+t1_outer.b > 5);
@@ -786,12 +784,12 @@ SET SQL_MODE = '';
 #
 SET SQL_MODE = 'ONLY_FULL_GROUP_BY';
 create table t1(f1 int, f2 int);
---error 1055
+--error ER_NON_GROUPING_FIELD_USED
 select * from t1 group by f1;
---error 1055
+--error ER_NON_GROUPING_FIELD_USED
 select * from t1 group by f2;
 select * from t1 group by f1, f2;
---error 1055
+--error ER_NON_GROUPING_FIELD_USED
 select t1.f1,t.* from t1, t1 t group by 1;
 drop table t1;
 SET SQL_MODE = '';

mysql-test/main/having.result

@@ -419,7 +419,7 @@ select f1 from t1 group by f1 having max(f1)=f1;
 f1
 set session sql_mode='ONLY_FULL_GROUP_BY';
 select f1 from t1 having max(f1)=f1;
-ERROR 42000: Non-grouping field 'f1' is used in HAVING clause
+ERROR 42000: Mixing of GROUP columns (MIN(),MAX(),COUNT(),...) with no GROUP columns is illegal if there is no GROUP BY clause
 select f1 from t1 group by f1 having max(f1)=f1;
 f1
 set session sql_mode='';

mysql-test/main/having.test

@@ -427,7 +427,7 @@ create table t1(f1 int);
 select f1 from t1 having max(f1)=f1;
 select f1 from t1 group by f1 having max(f1)=f1;
 set session sql_mode='ONLY_FULL_GROUP_BY';
---error ER_NON_GROUPING_FIELD_USED
+--error ER_MIX_OF_GROUP_FUNC_AND_FIELDS
 select f1 from t1 having max(f1)=f1;
 select f1 from t1 group by f1 having max(f1)=f1;
 set session sql_mode='';

sql/CMakeLists.txt

@@ -142,6 +142,7 @@ SET (SQL_SOURCE
                opt_split.cc
                rowid_filter.cc rowid_filter.h
                opt_trace.cc
+               sql_func_dep.cc
 	       ${WSREP_SOURCES}
                table_cache.cc encryption.cc temporary_tables.cc
                proxy_protocol.cc backup.cc xa.cc

sql/field.cc

@@ -11240,3 +11240,66 @@ bool Field::val_str_nopad(MEM_ROOT *mem_root, LEX_CSTRING *to)
 
   return rc;
 }
+
+
+bool Field::is_outer_select_field(st_select_lex *sl)
+{
+  return (table->pos_in_table_list->select_lex->select_number !=
+          sl->select_number);
+}
+
+
+static
+bool outer_select_field_is_in_where(Field *fld,
+                                    st_select_lex *curr_sl)
+{
+  if (!curr_sl)
+    return false;
+
+  /* Field is used in some parent select of sl */
+  if (fld->is_outer_select_field(curr_sl) &&
+      curr_sl->context_analysis_place != NO_MATTER)
+  {
+    st_select_lex *sl= curr_sl;
+    while ((sl->master_unit()->outer_select()->select_number !=
+            fld->table->pos_in_table_list->select_lex->select_number))
+      sl= sl->master_unit()->outer_select();
+    if (sl->context_analysis_place == IN_WHERE)
+      return true;
+  }
+  return false;
+}
+
+
+bool Field::excl_func_dep_on_grouping_fields(st_select_lex *sl,
+                                             List<Item> *gb_items,
+                                             bool in_where,
+                                             Item **err_item)
+{
+  if (sl && in_where && !is_outer_select_field(sl))
+    return true;
+  if (outer_select_field_is_in_where(this, sl))
+    return true;
+  if (vcol_info &&
+      vcol_info->expr->excl_func_dep_on_grouping_fields(sl, gb_items,
+                                                        in_where, err_item))
+    return true;
+  return bitmap_is_set(&table->tmp_set, field_index);
+}
+
+
+bool Field::check_usage_in_fd_field_extraction(st_select_lex *sl,
+                                               List<Field> *fields,
+                                               Item **err_item)
+{
+  if (outer_select_field_is_in_where(this, sl))
+    return true;
+  if (vcol_info)
+  {
+    List<Field> vcol_fields;
+    if (vcol_info->expr->check_usage_in_fd_field_extraction(sl, &vcol_fields,
+                                                            err_item))
+      return true;
+  }
+  return bitmap_is_set(&table->tmp_set, field_index);
+}

sql/field.h

@@ -1680,6 +1680,14 @@ class Field: public Value_source
     return NULL;
   }
   virtual bool sp_prepare_and_store_item(THD *thd, Item **value);
+  bool is_outer_select_field(st_select_lex *sl);
+  bool excl_func_dep_on_grouping_fields(st_select_lex *sl,
+                                        List<Item> *gb_items,
+                                        bool in_where,
+                                        Item **err_item);
+  bool check_usage_in_fd_field_extraction(st_select_lex *sl,
+                                          List<Field> *fields,
+                                          Item **err_item);
 
   friend int cre_myisam(char * name, TABLE *form, uint options,
 			ulonglong auto_increment_value);

sql/item.cc

@@ -5224,19 +5224,6 @@ resolve_ref_in_select_and_group(THD *thd, Item_ident *ref, SELECT_LEX *select)
     }
   }
 
-  if (thd->variables.sql_mode & MODE_ONLY_FULL_GROUP_BY &&
-      select->having_fix_field  &&
-      select_ref != not_found_item && !group_by_ref &&
-      !ref->alias_name_used)
-  {
-    /*
-      Report the error if fields was found only in the SELECT item list and
-      the strict mode is enabled.
-    */
-    my_error(ER_NON_GROUPING_FIELD_USED, MYF(0),
-             ref->name.str, "HAVING");
-    return NULL;
-  }
   if (select_ref != not_found_item || group_by_ref)
   {
     if (select_ref != not_found_item && !ambiguous_fields)
@@ -9073,6 +9060,41 @@ Item_field::excl_dep_on_grouping_fields(st_select_lex *sel)
 }
 
 
+bool Item_field::has_outer_nogb_field_processor(void *arg)
+{
+  table_map curr_level_tabs= *((table_map *)arg);
+  return !(field->table->map & curr_level_tabs) &&
+         !bitmap_is_set(&field->table->tmp_set, field->field_index);
+}
+
+
+bool Item_field::check_usage_in_fd_field_extraction(st_select_lex *sl,
+                                                    List<Field> *fields,
+                                                    Item **err_item)
+{
+  if (field->cmp_type() == STRING_RESULT &&
+      (!((field->charset()->state & MY_CS_BINSORT) &&
+      (field->charset()->state & MY_CS_NOPAD))))
+  {
+    fields->empty();
+    return false;
+  }
+  if (fields->push_back(field, sl->join->thd->mem_root))
+    return false;
+
+  if (!field->check_usage_in_fd_field_extraction(sl, fields, err_item))
+  {
+    if (field->is_outer_select_field(sl))
+    {
+      *err_item= this;
+      fields->empty();
+    }
+    return false;
+  }
+  return true;
+}
+
+
 bool Item_direct_view_ref::excl_dep_on_table(table_map tab_map)
 {
   table_map used= used_tables();
@@ -10350,6 +10372,7 @@ void Item_ref::update_used_tables()
     (*ref)->update_used_tables();
 }
 
+
 void Item_direct_view_ref::update_used_tables()
 {
   set_null_ref_table();

sql/item_row.h

@@ -150,6 +150,11 @@ class Item_row: public Item_fixed_hybrid,
     return Item_args::excl_dep_on_in_subq_left_part(subq_pred);
   }
 
+  bool excl_func_dep_on_grouping_fields(st_select_lex *sl,
+                                        List<Item> *gb_items,
+                                        bool in_where,
+                                        Item **err_item);
+
   bool check_vcol_func_processor(void *arg) {return FALSE; }
   Item *get_copy(THD *thd)
   { return get_item_copy<Item_row>(thd, this); }