MDEV-34041 Display additional information for materialized subqueries in...

MDEV-34041 Display additional information for materialized subqueries in EXPLAIN/ANALYZE FORMAT=JSON

This commits adds the "materialization" block to the output of
EXPLAIN/ANALYZE FORMAT=JSON when materialized subqueries are involved
into processing. In the case of ANALYZE additional runtime information
is displayed, such as:
  - chosen strategy of materialization
  - number of partial match/index lookup loops
  - sizes of partial match buffers

mysql-test/include/analyze-format.inc

 # The time on ANALYSE FORMAT=JSON is rather variable
 
---replace_regex /("(r_[a-z_]*_time_ms|r_buffer_size)": )[^, \n]*/\1"REPLACED"/
+--replace_regex /("(r_[a-z_]*_time_ms|r_buffer_size|r_partial_match_buffer_size)": )[^, \n]*/\1"REPLACED"/

mysql-test/main/analyze_format_json.result

@@ -659,6 +659,9 @@ ANALYZE
     },
     "subqueries": [
       {
+        "materialization": {
+          "r_strategy": "index_lookup",
+          "r_loops": 2,
           "query_block": {
             "select_id": 2,
             "r_loops": 1,
@@ -676,6 +679,7 @@ ANALYZE
             }
           }
         }
+      }
     ]
   }
 }

mysql-test/main/derived_cond_pushdown.result

@@ -7671,6 +7671,7 @@ EXPLAIN
       "filtered": 100,
       "materialized": {
         "unique": 1,
+        "materialization": {
           "query_block": {
             "select_id": 2,
             "table": {
@@ -7678,6 +7679,7 @@ EXPLAIN
             }
           }
         }
+      }
     },
     "read_sorted_file": {
       "filesort": {
@@ -7731,6 +7733,7 @@ EXPLAIN
       "filtered": 100,
       "materialized": {
         "unique": 1,
+        "materialization": {
           "query_block": {
             "select_id": 2,
             "table": {
@@ -7738,6 +7741,7 @@ EXPLAIN
             }
           }
         }
+      }
     },
     "table": {
       "table_name": "<derived3>",
@@ -7815,6 +7819,7 @@ EXPLAIN
       "filtered": 100,
       "materialized": {
         "unique": 1,
+        "materialization": {
           "query_block": {
             "select_id": 3,
             "table": {
@@ -7822,6 +7827,7 @@ EXPLAIN
             }
           }
         }
+      }
     },
     "table": {
       "table_name": "<derived2>",
@@ -7872,6 +7878,7 @@ EXPLAIN
       "filtered": 100,
       "materialized": {
         "unique": 1,
+        "materialization": {
           "query_block": {
             "select_id": 3,
             "table": {
@@ -7879,6 +7886,7 @@ EXPLAIN
             }
           }
         }
+      }
     },
     "table": {
       "table_name": "<derived2>",
@@ -7924,6 +7932,7 @@ EXPLAIN
       "filtered": 100,
       "materialized": {
         "unique": 1,
+        "materialization": {
           "query_block": {
             "select_id": 3,
             "table": {
@@ -7931,6 +7940,7 @@ EXPLAIN
             }
           }
         }
+      }
     },
     "table": {
       "table_name": "<derived2>",
@@ -7976,6 +7986,7 @@ EXPLAIN
       "filtered": 100,
       "materialized": {
         "unique": 1,
+        "materialization": {
           "query_block": {
             "select_id": 3,
             "table": {
@@ -7983,6 +7994,7 @@ EXPLAIN
             }
           }
         }
+      }
     },
     "table": {
       "table_name": "<derived2>",
@@ -8030,6 +8042,7 @@ EXPLAIN
       "filtered": 100,
       "materialized": {
         "unique": 1,
+        "materialization": {
           "query_block": {
             "select_id": 3,
             "table": {
@@ -8037,6 +8050,7 @@ EXPLAIN
             }
           }
         }
+      }
     },
     "table": {
       "table_name": "<derived2>",
@@ -8082,6 +8096,7 @@ EXPLAIN
       "filtered": 100,
       "materialized": {
         "unique": 1,
+        "materialization": {
           "query_block": {
             "select_id": 3,
             "table": {
@@ -8089,6 +8104,7 @@ EXPLAIN
             }
           }
         }
+      }
     },
     "table": {
       "table_name": "<derived2>",
@@ -8134,6 +8150,7 @@ EXPLAIN
       "filtered": 100,
       "materialized": {
         "unique": 1,
+        "materialization": {
           "query_block": {
             "select_id": 3,
             "table": {
@@ -8141,6 +8158,7 @@ EXPLAIN
             }
           }
         }
+      }
     },
     "table": {
       "table_name": "<derived2>",
@@ -8223,6 +8241,7 @@ EXPLAIN
     },
     "subqueries": [
       {
+        "materialization": {
           "query_block": {
             "select_id": 2,
             "table": {
@@ -8230,6 +8249,7 @@ EXPLAIN
             }
           }
         }
+      }
     ]
   }
 }
@@ -9294,6 +9314,7 @@ EXPLAIN
       "filtered": 100,
       "materialized": {
         "unique": 1,
+        "materialization": {
           "query_block": {
             "select_id": 2,
             "temporary_table": {
@@ -9326,6 +9347,7 @@ EXPLAIN
         }
       }
     }
+  }
 }
 SELECT * FROM t1
 WHERE (t1.a,t1.b) IN
@@ -9396,6 +9418,7 @@ EXPLAIN
       "filtered": 100,
       "materialized": {
         "unique": 1,
+        "materialization": {
           "query_block": {
             "select_id": 2,
             "temporary_table": {
@@ -9428,6 +9451,7 @@ EXPLAIN
         }
       }
     }
+  }
 }
 DROP TABLE t1,t2;
 #
@@ -10826,6 +10850,7 @@ EXPLAIN
       "attached_condition": "t4.c = `<subquery2>`.`sum(b)`",
       "materialized": {
         "unique": 1,
+        "materialization": {
           "query_block": {
             "select_id": 2,
             "having_condition": "`f1(a)` > 1 and `sum(b)` > 123",
@@ -10842,6 +10867,7 @@ EXPLAIN
         }
       }
     }
+  }
 }
 drop view v2;
 drop function f1;

mysql-test/main/explain_json.result

@@ -574,6 +574,7 @@ EXPLAIN
       "filtered": 100,
       "materialized": {
         "unique": 1,
+        "materialization": {
           "query_block": {
             "select_id": 2,
             "temporary_table": {
@@ -588,6 +589,7 @@ EXPLAIN
         }
       }
     }
+  }
 }
 #
 # Semi-join Materialization
@@ -906,6 +908,7 @@ EXPLAIN
     },
     "subqueries": [
       {
+        "materialization": {
           "query_block": {
             "select_id": 2,
             "table": {
@@ -916,6 +919,7 @@ EXPLAIN
             }
           }
         }
+      }
     ]
   }
 }

mysql-test/main/having_cond_pushdown.result

@@ -1134,6 +1134,7 @@ EXPLAIN
           "filtered": 100,
           "materialized": {
             "unique": 1,
+            "materialization": {
               "query_block": {
                 "select_id": 2,
                 "temporary_table": {
@@ -1151,6 +1152,7 @@ EXPLAIN
         }
       }
     }
+  }
 }
 set statement optimizer_switch='condition_pushdown_from_having=off' for explain format=json SELECT * FROM t1
 WHERE
@@ -1183,6 +1185,7 @@ EXPLAIN
           "filtered": 100,
           "materialized": {
             "unique": 1,
+            "materialization": {
               "query_block": {
                 "select_id": 3,
                 "temporary_table": {
@@ -1200,6 +1203,7 @@ EXPLAIN
         }
       }
     }
+  }
 }
 # conjunctive subformula : pushdown into materialized IN subquery
 #                          HAVING clause
@@ -1257,6 +1261,7 @@ EXPLAIN
           "filtered": 100,
           "materialized": {
             "unique": 1,
+            "materialization": {
               "query_block": {
                 "select_id": 2,
                 "having_condition": "`MAX(t2.y)` < 14",
@@ -1275,6 +1280,7 @@ EXPLAIN
         }
       }
     }
+  }
 }
 set statement optimizer_switch='condition_pushdown_from_having=off' for explain format=json SELECT * FROM t1
 WHERE
@@ -1307,6 +1313,7 @@ EXPLAIN
           "filtered": 100,
           "materialized": {
             "unique": 1,
+            "materialization": {
               "query_block": {
                 "select_id": 3,
                 "having_condition": "`MAX(t2.y)` < 14",
@@ -1325,6 +1332,7 @@ EXPLAIN
         }
       }
     }
+  }
 }
 # non-standard allowed queries
 # conjunctive subformula

mysql-test/main/subselect_mat_analyze_json.test

+set @save_optimizer_switch=@@optimizer_switch;
+
+create table t1 (a int);
+create table t2 (b int);
+insert into t1 values (null), (1), (2), (3);
+insert into t2 values (3), (4);
+
+set @@optimizer_switch = "materialization=on,in_to_exists=off,semijoin=off";
+
+explain format=json select * from t1 where a in (select b from t2);
+--echo # "Complete match" execution strategy
+--source include/analyze-format.inc
+analyze format=json select * from t1 where a in (select b from t2);
+
+--echo # "Partial match" is used due to NOT IN
+--echo # Force rowid-merge partial partial matching
+set @@optimizer_switch="partial_match_rowid_merge=on,partial_match_table_scan=off";
+--source include/analyze-format.inc
+analyze format=json select * from t1 where a not in (select b from t2);
+
+--echo # Force table scan partial matching
+set @@optimizer_switch="partial_match_rowid_merge=off,partial_match_table_scan=on";
+--source include/analyze-format.inc
+analyze format=json select * from t1 where a not in (select b from t2);
+
+--echo # Subselect in GROUP BY
+--source include/analyze-format.inc
+analyze format=json select a from t1 group by a in (select b from t2);
+
+set @@optimizer_switch="partial_match_rowid_merge=on,partial_match_table_scan=off";
+--source include/analyze-format.inc
+analyze format=json select a from t1 group by a not in (select b from t2);
+
+set @@optimizer_switch="partial_match_rowid_merge=on,partial_match_table_scan=on";
+--echo # Subselect in ORDER BY
+--source include/analyze-format.inc
+analyze format=json select a from t1 order by a in (select b from t2);
+
+--echo # Subselect in HAVING
+--source include/analyze-format.inc
+analyze format=json select a from t1 having a not in (select b from t2);
+
+--echo # Nested IN
+--source include/analyze-format.inc
+analyze format=json select a from t1 where a in (select a from t1 where a in (select b from t2));
+
+create table t3 (c int);
+insert into t3 (c) values (3), (null), (4);
+
+--echo # Subquery in ON-clause of outer join
+--source include/analyze-format.inc
+analyze format=json select a from t1 left join t2 on a not in (select c from t3);
+
+--source include/analyze-format.inc
+analyze format=json
+select (b, b + 1, b + 2) not in
+       (select count(distinct a), a + 1, a + 2 from t1 group by a + 1, a + 2)
+from t2;
+
+drop table t1, t2, t3;
+
+
+--echo #
+--echo # Tables with more than one column
+--echo #
+create table t1 (a1 char(1), a2 char(1));
+insert into t1 values (null, 'b');
+create table t2 (b1 char(1), b2 char(2));
+insert into t2 values ('a','b'), ('c', 'd'), (null, 'e'), ('f', 'g');
+
+set @@optimizer_switch="partial_match_rowid_merge=on,partial_match_table_scan=off";
+explain format=json select * from t1 where (a1, a2) not in (select b1, b2 from t2);
+--source include/analyze-format.inc
+analyze format=json select * from t1 where (a1, a2) not in (select b1, b2 from t2);
+
+set @@optimizer_switch="partial_match_rowid_merge=off,partial_match_table_scan=on";
+--source include/analyze-format.inc
+analyze format=json select * from t1 where (a1, a2) not in (select b1, b2 from t2);
+
+--echo # Subquery in SELECT list
+explain format=json select t1.*, (a1, a2) in (select * from t2) as in_res from t1;
+
+--source include/analyze-format.inc
+analyze format=json select t1.*, (a1, a2) in (select * from t2) as in_res from t1;
+
+--source include/analyze-format.inc
+analyze format=json select t1.*, (a1, a2) not in (select * from t2) as in_res from t1;
+
+set @@optimizer_switch="partial_match_rowid_merge=on,partial_match_table_scan=off";
+--source include/analyze-format.inc
+analyze format=json select t1.*, (a1, a2) in (select * from t2) as in_res from t1;
+
+--source include/analyze-format.inc
+analyze format=json select t1.*, (a1, a2) not in (select * from t2) as in_res from t1;
+
+drop table t1,t2;
+
+
+set @@optimizer_switch=@save_optimizer_switch;

sql/item_subselect.cc

@@ -42,6 +42,7 @@
 #include "sql_parse.h"                          // check_stack_overrun
 #include "sql_cte.h"
 #include "sql_test.h"
+#include "my_json_writer.h"
 
 double get_post_group_estimate(JOIN* join, double join_op_rows);
 
@@ -192,6 +193,7 @@ void Item_in_subselect::cleanup()
     in_strategy&= ~SUBS_STRATEGY_CHOSEN;
   */
   first_execution= TRUE;
+  materialization_tracker= NULL;
   pushed_cond_guards= NULL;
   Item_subselect::cleanup();
   DBUG_VOID_RETURN;
@@ -1602,6 +1604,7 @@ Item_in_subselect::Item_in_subselect(THD *thd, Item * left_exp,
 				     st_select_lex *select_lex):
   Item_exists_subselect(thd), left_expr_cache(0), first_execution(TRUE),
   in_strategy(SUBS_NOT_TRANSFORMED),
+  materialization_tracker(NULL),
   pushed_cond_guards(NULL), do_not_convert_to_sj(FALSE), is_jtbm_merged(FALSE),
   is_jtbm_const_tab(FALSE), is_flattenable_semijoin(FALSE),
   is_registered_semijoin(FALSE),
@@ -3648,6 +3651,26 @@ bool Item_in_subselect::init_cond_guards()
   return FALSE;
 }
 
+/**
+  Initialize the tracker which will be used to provide information for
+  the output of EXPLAIN and ANALYZE
+*/
+void Item_in_subselect::init_subq_materialization_tracker(THD *thd)
+{
+  if (test_strategy(SUBS_MATERIALIZATION | SUBS_PARTIAL_MATCH_ROWID_MERGE |
+                    SUBS_PARTIAL_MATCH_TABLE_SCAN))
+  {
+    Explain_query *qw= thd->lex->explain;
+    DBUG_ASSERT(qw);
+    Explain_node *node= qw->get_node(unit->first_select()->select_number);
+    if (!node)
+      return;
+    node->subq_materialization= new(qw->mem_root)
+        Explain_subq_materialization(qw->mem_root);
+    materialization_tracker= node->subq_materialization->get_tracker();
+  }
+}
+
 
 bool
 Item_allany_subselect::select_transformer(JOIN *join)
@@ -4245,11 +4268,13 @@ int subselect_uniquesubquery_engine::exec()
   empty_result_set= TRUE;
   table->status= 0;
   Item_in_subselect *in_subs= item->get_IN_subquery();
+  Subq_materialization_tracker *tracker= in_subs->get_materialization_tracker();
   DBUG_ASSERT(in_subs);
 
   if (!tab->preread_init_done && tab->preread_init())
     DBUG_RETURN(1);
- 
+  if (tracker)
+    tracker->increment_loops_count();
   if (in_subs->left_expr_has_null())
   {
     /*
@@ -5002,6 +5027,9 @@ subselect_hash_sj_engine::choose_partial_match_strategy(
                                         partial_match_key_parts_arg);
     if (pm_buff_size > thd->variables.rowid_merge_buff_size)
       strategy= PARTIAL_MATCH_SCAN;
+    else
+      item->get_IN_subquery()->get_materialization_tracker()->
+          report_partial_match_buffer_size(pm_buff_size);
   }
 }
 
@@ -5777,6 +5805,7 @@ int subselect_hash_sj_engine::exec()
     }
   }
 
+  item_in->get_materialization_tracker()->report_exec_strategy(strategy);
   if (pm_engine)
     lookup_engine= pm_engine;
   item_in->change_engine(lookup_engine);
@@ -6247,6 +6276,9 @@ int subselect_partial_match_engine::exec()
   DBUG_ASSERT(!(item_in->left_expr_has_null() &&
                 item_in->is_top_level_item()));
 
+  Subq_materialization_tracker *tracker= item_in->get_materialization_tracker();
+  tracker->increment_loops_count();
+
   if (!item_in->left_expr_has_null())
   {
     /* Try to find a matching row by index lookup. */
@@ -6260,6 +6292,7 @@ int subselect_partial_match_engine::exec()
     else
     {
       /* Search for a complete match. */
+      tracker->increment_index_lookups();
       if ((lookup_res= lookup_engine->index_lookup()))
       {
         /* An error occurred during lookup(). */
@@ -6304,6 +6337,7 @@ int subselect_partial_match_engine::exec()
   if (tmp_table->file->inited)
     tmp_table->file->ha_index_end();
 
+  tracker->increment_partial_matches();
   if (partial_match())
   {
     /* The result of IN is UNKNOWN. */
@@ -6510,6 +6544,8 @@ subselect_rowid_merge_engine::init(MY_BITMAP *non_null_key_parts,
                  0, 0))
     return TRUE;
 
+  item->get_IN_subquery()->get_materialization_tracker()->
+      report_partial_merge_keys(merge_keys, merge_keys_count);
   return FALSE;
 }
 
@@ -6966,3 +7002,12 @@ void Item_subselect::init_expr_cache_tracker(THD *thd)
   DBUG_ASSERT(expr_cache->type() == Item::EXPR_CACHE_ITEM);
   node->cache_tracker= ((Item_cache_wrapper *)expr_cache)->init_tracker(qw->mem_root);
 }
+
+
+void Subq_materialization_tracker::report_partial_merge_keys(
+    Ordered_key **merge_keys, uint merge_keys_count)
+{
+  partial_match_array_sizes.resize(merge_keys_count, 0);
+  for (uint i= 0; i < merge_keys_count; i++)
+    partial_match_array_sizes[i]= merge_keys[i]->get_key_buff_elements();
+}

sql/item_subselect.h

@@ -18,6 +18,7 @@
 
 /* subselect Item */
 
+#include "item.h"
 #ifdef USE_PRAGMA_INTERFACE
 #pragma interface			/* gcc class implementation */
 #endif
@@ -43,6 +44,8 @@ typedef class st_select_lex SELECT_LEX;
 */
 typedef Comp_creator* (*chooser_compare_func_creator)(bool invert);
 class Cached_item;
+class Subq_materialization_tracker;
+class Explain_subq_materialization;
 
 /* base class for subselects */
 
@@ -501,6 +504,8 @@ class Item_in_subselect :public Item_exists_subselect
   bool was_null;
   /* A bitmap of possible execution strategies for an IN predicate. */
   uchar in_strategy;
+  /* Tracker collecting execution parameters of a materialized subquery */
+  Subq_materialization_tracker *materialization_tracker;
 protected:
   /* Used to trigger on/off conditions that were pushed down to subselect */
   bool *pushed_cond_guards;
@@ -525,6 +530,7 @@ class Item_in_subselect :public Item_exists_subselect
     left_expr could later be changed to something on the execution arena.
   */
   Item *left_expr_orig;
+
 public:
   /* Priority of this predicate in the convert-to-semi-join-nest process. */
   int sj_convert_priority;
@@ -626,7 +632,7 @@ class Item_in_subselect :public Item_exists_subselect
   Item_in_subselect(THD *thd_arg, Item * left_expr, st_select_lex *select_lex);
   Item_in_subselect(THD *thd_arg):
     Item_exists_subselect(thd_arg), left_expr_cache(0), first_execution(TRUE),
-    in_strategy(SUBS_NOT_TRANSFORMED),
+    in_strategy(SUBS_NOT_TRANSFORMED), materialization_tracker(NULL),
     pushed_cond_guards(NULL), func(NULL), do_not_convert_to_sj(FALSE),
     is_jtbm_merged(FALSE), is_jtbm_const_tab(FALSE), upper_item(0),
     converted_from_in_predicate(FALSE) {}
@@ -778,6 +784,9 @@ class Item_in_subselect :public Item_exists_subselect
   { return left_expr; }
   inline Item* left_exp_orig() const
   { return left_expr_orig; }
+  void init_subq_materialization_tracker(THD *thd);
+  Subq_materialization_tracker *get_materialization_tracker() const
+  { return materialization_tracker; }
 
   friend class Item_ref_null_helper;
   friend class Item_is_not_null_test;
@@ -1159,6 +1168,15 @@ class subselect_hash_sj_engine : public subselect_engine
                      select_result_interceptor *result,
                      bool temp= FALSE) override;
   bool no_tables() const override;//=>base class
+  /* Possible execution strategies that can be used to compute hash semi-join.*/
+  enum exec_strategy {
+    UNDEFINED= 0,
+    COMPLETE_MATCH, /* Use regular index lookups. */
+    PARTIAL_MATCH,  /* Use some partial matching strategy. */
+    PARTIAL_MATCH_MERGE, /* Use partial matching through index merging. */
+    PARTIAL_MATCH_SCAN,  /* Use partial matching through table scan. */
+    IMPOSSIBLE      /* Subquery materialization is not applicable. */
+  };
 
 protected:
   /* The engine used to compute the IN predicate. */
@@ -1170,15 +1188,6 @@ class subselect_hash_sj_engine : public subselect_engine
   uint count_partial_match_columns;
   uint count_null_only_columns;
   uint count_columns_with_nulls;
-  /* Possible execution strategies that can be used to compute hash semi-join.*/
-  enum exec_strategy {
-    UNDEFINED,
-    COMPLETE_MATCH, /* Use regular index lookups. */
-    PARTIAL_MATCH,  /* Use some partial matching strategy. */
-    PARTIAL_MATCH_MERGE, /* Use partial matching through index merging. */
-    PARTIAL_MATCH_SCAN,  /* Use partial matching through table scan. */
-    IMPOSSIBLE      /* Subquery materialization is not applicable. */
-  };
   /* The chosen execution strategy. Computed after materialization. */
   exec_strategy strategy;
   exec_strategy get_strategy_using_schema();
@@ -1311,6 +1320,7 @@ class Ordered_key : public Sql_alloc
   rownum_t get_max_null_row() { return max_null_row; }
   MY_BITMAP * get_null_key() { return &null_key; }
   ha_rows get_null_count() { return null_count; }
+  ha_rows get_key_buff_elements() { return key_buff_elements; }
   /*
     Get the search key element that corresponds to the i-th key part of this
     index.
@@ -1548,4 +1558,93 @@ class subselect_table_scan_engine: public subselect_partial_match_engine
   void cleanup() override;
   enum_engine_type engine_type() override { return TABLE_SCAN_ENGINE; }
 };
+
+/**
+  @brief Subquery materialization tracker
+
+  @details
+  Used to track various parameters of the materialized subquery execution,
+  such as the execution strategy, sizes of buffers employed, etc
+*/
+class Subq_materialization_tracker
+{
+public:
+  using Strategy = subselect_hash_sj_engine::exec_strategy;
+
+  Subq_materialization_tracker(MEM_ROOT *mem_root)
+    : exec_strategy(Strategy::UNDEFINED),
+      partial_match_buffer_size(0),
+      partial_match_array_sizes(mem_root),
+      loops_count(0),
+      index_lookups_count(0),
+      partial_matches_count(0)
+  {}
+
+  void report_partial_merge_keys(Ordered_key **merge_keys,
+                                 uint merge_keys_count);
+
+  void report_exec_strategy(Strategy es)
+  {
+    exec_strategy= es;
+  }
+
+  void report_partial_match_buffer_size(longlong sz)
+  {
+    partial_match_buffer_size= sz;
+  }
+
+  void increment_loops_count()
+  {
+    loops_count++;
+  }
+
+  void increment_index_lookups()
+  {
+    index_lookups_count++;
+  }
+
+  void increment_partial_matches()
+  {
+    partial_matches_count++;
+  }
+
+  void print_json_members(Json_writer *writer) const;
+private:
+  Strategy exec_strategy;
+  ulonglong partial_match_buffer_size;
+  Dynamic_array<ha_rows> partial_match_array_sizes;
+
+  /* Number of times subquery predicate was evaluated */
+  ulonglong loops_count;
+
+  /*
+    Number of times we made a lookup in the materialized temptable
+    (we do this when all parts of left_expr are not NULLs)
+  */
+  ulonglong index_lookups_count;
+
+  /*
+    Number of times we had to check for a partial match (either by
+    scanning the materialized subquery or by doing a merge)
+  */
+  ulonglong partial_matches_count;
+
+  const char *get_exec_strategy() const
+  {
+    switch (exec_strategy)
+    {
+      case Strategy::UNDEFINED:
+        return "undefined";
+      case Strategy::COMPLETE_MATCH:
+        return "index_lookup";
+      case Strategy::PARTIAL_MATCH_MERGE:
+        return "index_lookup;array merge for partial match";
+      case Strategy::PARTIAL_MATCH_SCAN:
+        return "index_lookup;full scan for partial match";
+      default:
+        return "unsupported";
+    }
+  }
+};
+
 #endif /* ITEM_SUBSELECT_INCLUDED */

sql/sql_array.h

@@ -140,12 +140,24 @@ template <class Elem> class Dynamic_array
     DBUG_ASSERT(idx < array.elements);
     return *(((Elem*)array.buffer) + idx);
   }
+
   /// Const variant of at(), which cannot change data
   const Elem& at(size_t idx) const
   {
     return *(((Elem*)array.buffer) + idx);
   }
 
+  Elem& operator[](size_t idx)
+  {
+    return at(idx);
+  }
+
+  /// Const variant of operator[]
+  const Elem& operator[](size_t idx) const
+  {
+    return at(idx);
+  }
+
   /// @returns pointer to first element
   Elem *front()
   {

sql/sql_explain.cc

@@ -24,6 +24,7 @@
 #include "my_json_writer.h"
 #include "opt_range.h"
 #include "sql_expression_cache.h"
+#include "item_subselect.h"
 
 const char * STR_DELETING_ALL_ROWS= "Deleting all rows";
 const char * STR_IMPOSSIBLE_WHERE= "Impossible WHERE";
@@ -782,6 +783,18 @@ bool Explain_node::print_explain_json_cache(Json_writer *writer,
 }
 
 
+bool Explain_node::print_explain_json_subq_materialization(Json_writer *writer,
+                                                           bool is_analyze)
+{
+  if (subq_materialization)
+  {
+    subq_materialization->print_explain_json(writer, is_analyze);
+    return true;
+  }
+  return false;
+}
+
+
 Explain_basic_join::~Explain_basic_join()
 {
   if (join_tabs)
@@ -929,6 +942,8 @@ void Explain_select::print_explain_json(Explain_query *query,
   Json_writer_nesting_guard guard(writer);
   
   bool started_cache= print_explain_json_cache(writer, is_analyze);
+  bool started_subq_mat= print_explain_json_subq_materialization(writer,
+                                                                 is_analyze);
 
   if (message ||
       select_type == pushed_derived_text ||
@@ -1032,6 +1047,8 @@ void Explain_select::print_explain_json(Explain_query *query,
     writer->end_object();
   }
 
+  if (started_subq_mat)
+    writer->end_object();
   if (started_cache)
     writer->end_object();
 }
@@ -2693,3 +2710,41 @@ void Explain_range_checked_fer::print_json(Json_writer *writer,
     writer->end_object();
   }
 }
+
+
+void Explain_subq_materialization::print_explain_json(Json_writer *writer,
+                                                      bool is_analyze)
+{
+  writer->add_member("materialization").start_object();
+  if (is_analyze)
+    tracker.print_json_members(writer);
+}
+
+
+void Subq_materialization_tracker::print_json_members(Json_writer *writer) const
+{
+  writer->add_member("r_strategy").add_str(get_exec_strategy());
+  if (loops_count)
+    writer->add_member("r_loops").add_ull(loops_count);
+
+  if (index_lookups_count)
+    writer->add_member("r_index_lookups").add_ull(index_lookups_count);
+
+  if (partial_matches_count)
+    writer->add_member("r_partial_matches").add_ull(partial_matches_count);
+
+  if (partial_match_buffer_size)
+  {
+    writer->add_member("r_partial_match_buffer_size").
+            add_size(partial_match_buffer_size);
+  }
+
+  if (partial_match_array_sizes.elements())
+  {
+    writer->add_member("r_partial_match_array_sizes").start_array();
+    for(size_t i= 0; i < partial_match_array_sizes.elements(); i++)
+      writer->add_ull(partial_match_array_sizes[i]);
+    writer->end_array();
+  }
+}
+

sql/sql_explain.h

@@ -86,6 +86,7 @@ class Explain_node : public Sql_alloc
 public:
   Explain_node(MEM_ROOT *root) :
     cache_tracker(NULL),
+    subq_materialization(NULL),
     connection_type(EXPLAIN_NODE_OTHER),
     children(root)
   {}
@@ -115,6 +116,12 @@ class Explain_node : public Sql_alloc
   */
   Expression_cache_tracker* cache_tracker;
 
+  /**
+    If not NULL, this node is a SELECT (or UNION) in a materialized
+    IN-subquery.
+  */
+  Explain_subq_materialization* subq_materialization;
+
   /*
     How this node is connected to its parent.
     (NOTE: EXPLAIN_NODE_NON_MERGED_SJ is set very late currently)
@@ -143,6 +150,8 @@ class Explain_node : public Sql_alloc
   void print_explain_json_for_children(Explain_query *query,
                                        Json_writer *writer, bool is_analyze);
   bool print_explain_json_cache(Json_writer *writer, bool is_analyze);
+  bool print_explain_json_subq_materialization(Json_writer *writer,
+                                               bool is_analyze);
   virtual ~Explain_node() = default;
 };
 
@@ -1003,4 +1012,26 @@ class Explain_delete: public Explain_update
 };
 
 
+/*
+  EXPLAIN data structure for subquery materialization.
+
+  All decisions are made at execution time so here we just store the tracker
+  that has all the info.
+*/
+
+class Explain_subq_materialization : public Sql_alloc
+{
+public:
+  Explain_subq_materialization(MEM_ROOT *mem_root)
+    : tracker(mem_root)
+  {}
+
+  Subq_materialization_tracker *get_tracker() { return &tracker; }
+
+  void print_explain_json(Json_writer *writer, bool is_analyze);
+
+private:
+  Subq_materialization_tracker tracker;
+};
+
 #endif //SQL_EXPLAIN_INCLUDED

sql/sql_select.cc

@@ -1729,6 +1729,11 @@ bool JOIN::build_explain()
         curr_tab->tracker= tmp->get_using_temporary_read_tracker();
     }
   }
+  if (is_in_subquery())
+  {
+    Item_in_subselect *subq= unit->item->get_IN_subquery();
+    subq->init_subq_materialization_tracker(thd);
+  }
   DBUG_RETURN(0);
 }