MDEV-32324: Server crashes inside filesort at my_decimal::to_binary

A subquery in form "(SELECT not_null_value LIMIT 1 OFFSET 1)" will
produce no rows which will translate into scalar SQL NULL value.

The code in Item_singlerow_subselect::fix_length_and_dec() failed to
take the LIMIT/OFFSET clause into account and used to set
item_subselect->maybe_null=0, despite that SQL NULL will be produced.

If such subselect was used in ORDER BY, this would cause a crash in
filesort() code when it would get a NULL value for a not-nullable item.

also made subselect_engine::no_tables() const function.

mysql-test/main/order_by.result

@@ -3700,4 +3700,10 @@ Note	1003	select `test`.`t4`.`a` AS `a`,`test`.`t4`.`b` AS `b`,`test`.`t4`.`c` A
 set histogram_size=@tmp_h, histogram_type=@tmp_ht, use_stat_tables=@tmp_u,
 optimizer_use_condition_selectivity=@tmp_o;
 drop table t1,t2,t3,t4;
+#
+# MDEV-32324: Server crashes inside filesort at my_decimal::to_binary
+#
+SELECT 1.000000 two UNION SELECT 1 ORDER BY ( SELECT two LIMIT 1 OFFSET 1 ) ;
+two
+1.000000
 # End of 10.4 tests

mysql-test/main/order_by.test

@@ -2447,4 +2447,9 @@ set histogram_size=@tmp_h, histogram_type=@tmp_ht, use_stat_tables=@tmp_u,
 
 drop table t1,t2,t3,t4;
 
+--echo #
+--echo # MDEV-32324: Server crashes inside filesort at my_decimal::to_binary
+--echo #
+SELECT 1.000000 two UNION SELECT 1 ORDER BY ( SELECT two LIMIT 1 OFFSET 1 ) ;
+
 --echo # End of 10.4 tests

sql/item_subselect.cc

@@ -1240,18 +1240,10 @@ bool Item_singlerow_subselect::fix_length_and_dec()
   }
   unsigned_flag= value->unsigned_flag;
   /*
-    If the subquery has no tables (1) and is not a UNION (2), like:
-
-      (SELECT subq_value)
-
+    If the subquery always returns a row, like "(SELECT subq_value)"
     then its NULLability is the same as subq_value's NULLability.
-
-    (1): A subquery that uses a table will return NULL when the table is empty.
-    (2): A UNION subquery will return NULL if it produces a "Subquery returns
-         more than one row" error.
   */
-  if (engine->no_tables() &&
-      engine->engine_type() != subselect_engine::UNION_ENGINE)
+  if (engine->always_returns_one_row())
     maybe_null= engine->may_be_null();
   else
   {
@@ -1262,6 +1254,32 @@ bool Item_singlerow_subselect::fix_length_and_dec()
 }
 
 
+
+/*
+  @brief
+     Check if we can guarantee that this engine will always produce exactly one
+     row.
+
+  @detail
+    Check if the subquery is just
+
+      (SELECT value)
+
+    Then we can guarantee we always return one row.
+    Selecting from tables may produce more than one row.
+    HAVING, WHERE or ORDER BY/LIMIT clauses may cause no rows to be produced.
+*/
+
+bool subselect_single_select_engine::always_returns_one_row() const
+{
+  st_select_lex *params= select_lex->master_unit()->global_parameters();
+  return no_tables() &&
+         !params->select_limit &&
+         !params->offset_limit &&
+         !select_lex->where &&
+         !select_lex->having;
+}
+
 /**
   Add an expression cache for this subquery if it is needed
 
@@ -4705,7 +4723,7 @@ subselect_uniquesubquery_engine::change_result(Item_subselect *si,
   @retval
     FALSE there are some tables in subquery
 */
-bool subselect_single_select_engine::no_tables()
+bool subselect_single_select_engine::no_tables() const
 {
   return(select_lex->table_list.elements == 0);
 }
@@ -4735,7 +4753,7 @@ bool subselect_single_select_engine::may_be_null()
   @retval
     FALSE there are some tables in subquery
 */
-bool subselect_union_engine::no_tables()
+bool subselect_union_engine::no_tables() const
 {
   for (SELECT_LEX *sl= unit->first_select(); sl; sl= sl->next_select())
   {
@@ -4755,7 +4773,7 @@ bool subselect_union_engine::no_tables()
     FALSE there are some tables in subquery
 */
 
-bool subselect_uniquesubquery_engine::no_tables()
+bool subselect_uniquesubquery_engine::no_tables() const
 {
   /* returning value is correct, but this method should never be called */
   DBUG_ASSERT(FALSE);
@@ -5750,7 +5768,7 @@ void subselect_hash_sj_engine::exclude()
   DBUG_ASSERT(FALSE);
 }
 
-bool subselect_hash_sj_engine::no_tables()
+bool subselect_hash_sj_engine::no_tables() const
 {
   DBUG_ASSERT(FALSE);
   return FALSE;

sql/item_subselect.h

@@ -867,7 +867,11 @@ class subselect_engine: public Sql_alloc,
   virtual bool change_result(Item_subselect *si,
                              select_result_interceptor *result,
                              bool temp= FALSE)= 0;
-  virtual bool no_tables()= 0;
+  virtual bool no_tables() const = 0;
+  /*
+    Return true we can guarantee that the subquery will always return one row.
+  */
+  virtual bool always_returns_one_row() const { return false; }
   virtual bool is_executed() const { return FALSE; }
   /* Check if subquery produced any rows during last query execution */
   virtual bool no_rows() = 0;
@@ -900,7 +904,8 @@ class subselect_single_select_engine: public subselect_engine
   bool change_result(Item_subselect *si,
                      select_result_interceptor *result,
                      bool temp);
-  bool no_tables();
+  bool no_tables() const override;
+  bool always_returns_one_row() const override;
   bool may_be_null();
   bool is_executed() const { return executed; }
   bool no_rows();
@@ -937,7 +942,7 @@ class subselect_union_engine: public subselect_engine
   bool change_result(Item_subselect *si,
                      select_result_interceptor *result,
                      bool temp= FALSE);
-  bool no_tables();
+  bool no_tables() const override;
   bool is_executed() const;
   void force_reexecution();
   bool no_rows();
@@ -995,7 +1000,7 @@ class subselect_uniquesubquery_engine: public subselect_engine
   bool change_result(Item_subselect *si,
                      select_result_interceptor *result,
                      bool temp= FALSE);
-  bool no_tables();
+  bool no_tables() const override;
   int index_lookup(); /* TIMOUR: this method needs refactoring. */
   int scan_table();
   bool copy_ref_key(bool skip_constants);
@@ -1140,7 +1145,7 @@ class subselect_hash_sj_engine : public subselect_engine
   bool change_result(Item_subselect *si,
                      select_result_interceptor *result,
                      bool temp= FALSE);
-  bool no_tables();//=>base class
+  bool no_tables() const override;//=>base class
 
 protected:
   /* The engine used to compute the IN predicate. */
@@ -1416,7 +1421,7 @@ class subselect_partial_match_engine : public subselect_engine
                      select_result_interceptor*,
                      bool temp= FALSE)
   { DBUG_ASSERT(FALSE); return false; }
-  bool no_tables() { return false; }
+  bool no_tables() const override { return false; }
   bool no_rows()
   {
     /*