MDEV-31983 jointable materialization subquery optimization ignoring

...errors, then failing ASSERT.

UPDATE queries treat warnings as errors. In this case, an invalid
condition "datetime_key_col >= '2012-01'" caused warning-as-error inside
SQL_SELECT::test_quick_select().

The code that called test_quick_select() ignored this error and continued
join optimization. Then it eventually reached a thd->is_error() check
and failed to setup SJ-Materialization which failed an assert.

Fixed this by making SQL_SELECT::test_quick_select() return error in
its return value, and making any code that calls it to check for error
condition and abort the query if the error is returned.

Places in the code that didn't check for errors from
SQL_SELECT::test_quick_select but now do:
- get_quick_record_count() call in make_join_statistics(),
- test_if_skip_sort_order(),
- "Range checked for each record" code.

Extra error handling fixes and commit text wording by Sergei Petrunia,

Reviewed-by: Sergei Petrunia, Oleg Smirnov

mysql-test/main/subselect_mat.result

@@ -2687,6 +2687,19 @@ i1	i2
 1	4
 2	6
 DROP TABLE t1;
+#
+#     MDEV-31983 jointable materialization subquery optimization ignoring errors, then failing ASSERT.
+#
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 VALUES (1),(2);
+CREATE TABLE t2 (b INT);
+INSERT INTO t2 VALUES (3),(4);
+CREATE TABLE t3 (c DATETIME, d INT, KEY(c));
+INSERT INTO t3 VALUES ('2012-11-11',5),('2012-12-12',6);
+UPDATE t1, t2 SET t1.a = 26 WHERE t2.b IN (SELECT MIN(d) FROM t3 WHERE c >= '2012-01');
+ERROR 22007: Incorrect datetime value: '2012-01' for column `test`.`t3`.`c` at row 1
+DROP TABLE t1, t2, t3;
+# end of 10.6 tests
 set @subselect_mat_test_optimizer_switch_value=null;
 set @@optimizer_switch='materialization=on,in_to_exists=off,semijoin=off';
 set optimizer_switch='mrr=on,mrr_sort_keys=on,index_condition_pushdown=on';

mysql-test/main/subselect_sj_mat.result

@@ -2729,3 +2729,16 @@ i1	i2
 1	4
 2	6
 DROP TABLE t1;
+#
+#     MDEV-31983 jointable materialization subquery optimization ignoring errors, then failing ASSERT.
+#
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 VALUES (1),(2);
+CREATE TABLE t2 (b INT);
+INSERT INTO t2 VALUES (3),(4);
+CREATE TABLE t3 (c DATETIME, d INT, KEY(c));
+INSERT INTO t3 VALUES ('2012-11-11',5),('2012-12-12',6);
+UPDATE t1, t2 SET t1.a = 26 WHERE t2.b IN (SELECT MIN(d) FROM t3 WHERE c >= '2012-01');
+ERROR 22007: Incorrect datetime value: '2012-01' for column `test`.`t3`.`c` at row 1
+DROP TABLE t1, t2, t3;
+# end of 10.6 tests

mysql-test/main/subselect_sj_mat.test

@@ -2419,3 +2419,21 @@ WHERE alias1.i1 IN (
 );
 DROP TABLE t1;
 
+--echo #
+--echo #     MDEV-31983 jointable materialization subquery optimization ignoring errors, then failing ASSERT.
+--echo #
+
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 VALUES (1),(2);
+CREATE TABLE t2 (b INT);
+INSERT INTO t2 VALUES (3),(4);
+CREATE TABLE t3 (c DATETIME, d INT, KEY(c));
+INSERT INTO t3 VALUES ('2012-11-11',5),('2012-12-12',6);
+
+--error ER_TRUNCATED_WRONG_VALUE
+UPDATE t1, t2 SET t1.a = 26 WHERE t2.b IN (SELECT MIN(d) FROM t3 WHERE c >= '2012-01');
+
+# Cleanup
+DROP TABLE t1, t2, t3;
+
+--echo # end of 10.6 tests

sql/opt_range.cc

@@ -2671,24 +2671,34 @@ static int fill_used_fields_bitmap(PARAM *param)
      force_quick_range is really needed.
 
   RETURN
-   -1 if error or impossible select (i.e. certainly no rows will be selected)
-    0 if can't use quick_select
-    1 if found usable ranges and quick select has been successfully created.
+    SQL_SELECT::
+      IMPOSSIBLE_RANGE,
+        impossible select (i.e. certainly no rows will be selected)
+      ERROR,
+        an error occurred, either memory or in evaluating conditions
+      OK = 1,
+        either
+          found usable ranges and quick select has been successfully created.
+          or can't use quick_select
 */
 
-int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
-				  table_map prev_tables,
-				  ha_rows limit, bool force_quick_range, 
-                                  bool ordered_output,
-                                  bool remove_false_parts_of_where,
-                                  bool only_single_index_range_scan,
-                                  bool suppress_unusable_key_notes)
+quick_select_return
+SQL_SELECT::test_quick_select(THD *thd,
+                              key_map keys_to_use,
+                              table_map prev_tables,
+                              ha_rows limit, bool force_quick_range,
+                              bool ordered_output,
+                              bool remove_false_parts_of_where,
+                              bool only_single_index_range_scan,
+                              bool suppress_unusable_key_notes)
 {
   uint idx;
   double scan_time;
   Item *notnull_cond= NULL;
   TABLE_READ_PLAN *best_trp= NULL;
   SEL_ARG **backup_keys= 0;
+  quick_select_return returnval= OK;
+
   DBUG_ENTER("SQL_SELECT::test_quick_select");
   DBUG_PRINT("enter",("keys_to_use: %lu  prev_tables: %lu  const_tables: %lu",
 		      (ulong) keys_to_use.to_ulonglong(), (ulong) prev_tables,
@@ -2701,7 +2711,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
   head->with_impossible_ranges.clear_all();
   DBUG_ASSERT(!head->is_filled_at_execution());
   if (keys_to_use.is_clear_all() || head->is_filled_at_execution())
-    DBUG_RETURN(0);
+    DBUG_RETURN(OK);
   records= head->stat_records();
   notnull_cond= head->notnull_cond;
   if (!records)
@@ -2754,7 +2764,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
     bool force_group_by = false;
 
     if (check_stack_overrun(thd, 2*STACK_MIN_SIZE + sizeof(PARAM), buff))
-      DBUG_RETURN(0);                           // Fatal error flag is set
+      DBUG_RETURN(ERROR);               // Fatal error flag is set
 
     /* set up parameter that is passed to all functions */
     bzero((void*) &param, sizeof(param));
@@ -2790,7 +2800,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
     {
       thd->no_errors=0;
       free_root(&alloc,MYF(0));			// Return memory & allocator
-      DBUG_RETURN(-1);				// Error
+      DBUG_RETURN(ERROR);
     }
     key_parts= param.key_parts;
 
@@ -2858,7 +2868,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
     {
       thd->no_errors=0;
       free_root(&alloc,MYF(0));			// Return memory & allocator
-      DBUG_RETURN(-1);				// Error
+      DBUG_RETURN(ERROR);
     }
 
     thd->mem_root= &alloc;
@@ -2910,7 +2920,8 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
       {
         if (tree->type == SEL_TREE::IMPOSSIBLE)
         {
-          records=0L;                      /* Return -1 from this function. */
+          records=0L;
+          returnval= IMPOSSIBLE_RANGE;
           read_time= (double) HA_POS_ERROR;
           trace_range.add("impossible_range", true);
           goto free_mem;
@@ -2930,7 +2941,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
         thd->no_errors=0;
         thd->mem_root= param.old_root;
         free_root(&alloc, MYF(0));
-        DBUG_RETURN(-1);
+        DBUG_RETURN(ERROR);
       }
     }
 
@@ -3081,9 +3092,14 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
         delete quick;
         quick= NULL;
       }
+      if (quick && records)
+        returnval= OK;
     }
     possible_keys= param.possible_keys;
 
+  if (!records)
+    returnval= IMPOSSIBLE_RANGE;
+
   free_mem:
     if (unlikely(quick && best_trp && thd->trace_started()))
     {
@@ -3109,7 +3125,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
     Assume that if the user is using 'limit' we will only need to scan
     limit rows if we are using a key
   */
-  DBUG_RETURN(records ? MY_TEST(quick) : -1);
+  DBUG_RETURN(returnval);
 }
 
 /****************************************************************************

sql/opt_range.h

@@ -1712,13 +1712,20 @@ class SQL_SELECT :public Sql_alloc {
   ~SQL_SELECT();
   void cleanup();
   void set_quick(QUICK_SELECT_I *new_quick) { delete quick; quick= new_quick; }
+
+  /*
+    @return
+      true  - for ERROR and IMPOSSIBLE_RANGE
+      false   - Ok
+  */
   bool check_quick(THD *thd, bool force_quick_range, ha_rows limit)
   {
     key_map tmp;
     tmp.set_all();
     return test_quick_select(thd, tmp, 0, limit, force_quick_range,
-                             FALSE, FALSE, FALSE) < 0;
+                             FALSE, FALSE, FALSE) != OK;
   }
+
   /* 
     RETURN
       0   if record must be skipped <-> (cond && cond->val_int() == 0)
@@ -1732,13 +1739,25 @@ class SQL_SELECT :public Sql_alloc {
       rc= -1;
     return rc;
   }
-  int test_quick_select(THD *thd, key_map keys, table_map prev_tables,
-			ha_rows limit, bool force_quick_range, 
-                        bool ordered_output, bool remove_false_parts_of_where,
-                        bool only_single_index_range_scan,
-                        bool suppress_unusable_key_notes = 0);
+
+  enum quick_select_return_type {
+    IMPOSSIBLE_RANGE = -1,
+    ERROR,
+    OK
+  };
+
+  enum quick_select_return_type
+  test_quick_select(THD *thd, key_map keys, table_map prev_tables,
+                    ha_rows limit,
+                    bool force_quick_range,
+                    bool ordered_output,
+                    bool remove_false_parts_of_where,
+                    bool only_single_index_range_scan,
+                    bool suppress_unusable_key_notes = 0);
 };
 
+typedef enum SQL_SELECT::quick_select_return_type quick_select_return;
+
 
 class SQL_SELECT_auto
 {