Adding support for the Vertical Partition Engine

Contains Spiral patches:
- Spiral Patch 021: 021_mariadb-10.2.0.merge_table.diff MDEV-7719
  - Changes for identifying MyISAM Merge child tables that can be merged.
  - This patch has the following differences compared to the original patch:
    - Changed bit positions for handlerton flags to eliminate conflicts
      with flags merged from MySQL.
- Spiral Patch 048: 048_mariadb-10.2.0.vp_partition.diff MDEV-7744
  - Check and set the partition bitmap.
- Spiral Patch 054: 054_mariadb-10.2.0.for_vp_pruning.diff MDEV-7750
  - Support for vertical partition pruning.
- Spiral Patch 055: 055_mariadb-10.2.0.for_vp_same_columns.diff MDEV-13000
  - Support for MERGE tables in the vertical partition engine.
- Spiral Patch 056: 056_mariadb-10.2.0.partition_top_table_fields.diff
                    MDEV-12970
  - Push down to each partition the table and fields of a vertical partition
    that are mapped to local table fields.
- Spiral Patch 060: 060_mariadb-10.2.0.partition_reset_top_table_fields.diff
                    MDEV-12971
  - Completion of functionality to push down to each partition the table and
    fields of a vertical partition that are mapped to local table fields.

include/my_base.h

@@ -196,6 +196,7 @@ enum ha_extra_function {
   */
   HA_EXTRA_ADD_CHILDREN_LIST,
   HA_EXTRA_ATTACH_CHILDREN,
+  HA_EXTRA_INIT_AFTER_ATTACH_CHILDREN,
   HA_EXTRA_IS_ATTACHED_CHILDREN,
   HA_EXTRA_DETACH_CHILDREN,
   HA_EXTRA_DETACH_CHILD,

sql/ha_partition.h

@@ -353,6 +353,8 @@ class ha_partition :public handler
      m_part_info= part_info;
      m_is_sub_partitioned= part_info->is_sub_partitioned();
   }
+  virtual Field **get_full_part_fields();
+  virtual int choose_partition_from_column_value(uchar *buf);
   /*
     -------------------------------------------------------------------------
     MODULE create/delete handler object
@@ -1339,6 +1341,9 @@ class ha_partition :public handler
   */
     virtual const COND *cond_push(const COND *cond);
     virtual void cond_pop();
+    virtual int set_top_table_and_fields(TABLE *top_table,
+                                         Field **top_table_field,
+                                         uint top_table_fields);
     virtual void clear_top_table_fields();
 
     private:
@@ -1369,6 +1374,7 @@ class ha_partition :public handler
     virtual int assign_to_keycache(THD* thd, HA_CHECK_OPT *check_opt);
     virtual int preload_keys(THD* thd, HA_CHECK_OPT* check_opt);
     virtual TABLE_LIST *get_next_global_for_child();
+    virtual void check_and_set_bitmap_for_update(bool rnd);
 
   /*
     -------------------------------------------------------------------------

sql/handler.cc

@@ -5935,8 +5935,10 @@ int handler::ha_reset()
   mark_trx_read_write_done= check_table_binlog_row_based_done=
     check_table_binlog_row_based_result= 0;
   /* Reset information about pushed engine conditions */
-  cancel_pushed_idx_cond();
   /* Reset information about pushed index conditions */
+  cancel_pushed_idx_cond();
+  /* Reset information about pushed top table and fields */
+  clear_top_table_fields();
   DBUG_RETURN(reset());
 }
 

sql/handler.h

@@ -1399,9 +1399,6 @@ handlerton *ha_default_tmp_handlerton(THD *thd);
 #define HTON_TEMPORARY_NOT_SUPPORTED (1 << 6) //Having temporary tables not supported
 #define HTON_SUPPORT_LOG_TABLES      (1 << 7) //Engine supports log tables
 #define HTON_NO_PARTITION            (1 << 8) //Not partition of these tables
-#define HTON_CAN_MULTISTEP_MERGE     (1 << 9) //You can merge mearged tables
-// Engine needs to access the main connect string in partitions
-#define HTON_CAN_READ_CONNECT_STRING_IN_PARTITION (1 << 10)
 
 /*
   This flag should be set when deciding that the engine does not allow
@@ -1422,6 +1419,10 @@ handlerton *ha_default_tmp_handlerton(THD *thd);
 // MySQL compatibility. Unused.
 #define HTON_SUPPORTS_FOREIGN_KEYS   (1 << 0) //Foreign key constraint supported.
 
+#define HTON_CAN_MERGE               (1 <<11) //Merge type table
+// Engine needs to access the main connect string in partitions
+#define HTON_CAN_READ_CONNECT_STRING_IN_PARTITION (1 <<12)
+
 class Ha_trx_info;
 
 struct THD_TRANS
@@ -3456,6 +3457,8 @@ class handler :public Sql_alloc
     return 0;
   }
   virtual void set_part_info(partition_info *part_info) {return;}
+  virtual Field **get_full_part_fields() { return NULL; }
+  virtual int choose_partition_from_column_value(uchar *buf) { return 0; }
 
   virtual ulong index_flags(uint idx, uint part, bool all_parts) const =0;
 
@@ -3727,6 +3730,7 @@ class handler :public Sql_alloc
 
  /* Needed for partition / spider */
   virtual TABLE_LIST *get_next_global_for_child() { return NULL; }
+  virtual void check_and_set_bitmap_for_update(bool rnd) { return; }
 
  /**
    Part of old, deprecated in-place ALTER API.

sql/sql_admin.cc

@@ -182,7 +182,7 @@ static int prepare_for_repair(THD *thd, TABLE_LIST *table_list,
     goto end;					// No data file
 
   /* A MERGE table must not come here. */
-  DBUG_ASSERT(table->file->ht->db_type != DB_TYPE_MRG_MYISAM);
+  DBUG_ASSERT(!(table->file->ht->flags & HTON_CAN_MERGE));
 
   // Name of data file
   strxmov(from, table->s->normalized_path.str, ext[1], NullS);

sql/sql_base.cc

@@ -974,7 +974,7 @@ TABLE_LIST* find_dup_table(THD *thd, TABLE_LIST *table, TABLE_LIST *table_list,
   if (table->table)
   {
     /* All MyISAMMRG children are plain MyISAM tables. */
-    DBUG_ASSERT(table->table->file->ht->db_type != DB_TYPE_MRG_MYISAM);
+    DBUG_ASSERT(!(table->table->file->ht->flags & HTON_CAN_MERGE));
 
     table= table->find_underlying_table(table->table);
     /*
@@ -1080,7 +1080,8 @@ unique_table(THD *thd, TABLE_LIST *table, TABLE_LIST *table_list,
   table= table->find_table_for_update();
 
   if (table->table &&
-      table->table->file->ha_table_flags() & HA_CAN_MULTISTEP_MERGE)
+      ((table->table->file->ht->flags & HTON_CAN_MERGE) ||
+       (table->table->file->ha_table_flags() & HA_CAN_MULTISTEP_MERGE)))
   {
     TABLE_LIST *child;
     dup= NULL;
@@ -1089,7 +1090,8 @@ unique_table(THD *thd, TABLE_LIST *table, TABLE_LIST *table_list,
          child= child->next_global)
     {
       if (child->table &&
-          child->table->file->ha_table_flags() & HA_CAN_MULTISTEP_MERGE)
+          ((child->table->file->ht->flags & HTON_CAN_MERGE) ||
+           (child->table->file->ha_table_flags() & HA_CAN_MULTISTEP_MERGE)))
         continue;
 
       /*
@@ -4059,7 +4061,8 @@ bool open_tables(THD *thd, const DDL_options_st &options,
       continue;
 
     /* Schema tables may not have a TABLE object here. */
-    if (tbl->file->ha_table_flags() & HA_CAN_MULTISTEP_MERGE)
+    if ((tbl->file->ht->flags & HTON_CAN_MERGE) ||
+        (tbl->file->ha_table_flags() & HA_CAN_MULTISTEP_MERGE))
     {
       /* MERGE tables need to access parent and child TABLE_LISTs. */
       DBUG_ASSERT(tbl->pos_in_table_list == tables);
@@ -4604,7 +4607,8 @@ TABLE *open_ltable(THD *thd, TABLE_LIST *table_list, thr_lock_type lock_type,
     */
     DBUG_ASSERT(table_list->table);
     table= table_list->table;
-    if (table->file->ha_table_flags() & HA_CAN_MULTISTEP_MERGE)
+    if ((table->file->ht->flags & HTON_CAN_MERGE) ||
+        (table->file->ha_table_flags() & HA_CAN_MULTISTEP_MERGE))
     {
       /* A MERGE table must not come here. */
       /* purecov: begin tested */

storage/myisammrg/ha_myisammrg.cc

@@ -1755,7 +1755,7 @@ static int myisammrg_init(void *p)
   myisammrg_hton->db_type= DB_TYPE_MRG_MYISAM;
   myisammrg_hton->create= myisammrg_create_handler;
   myisammrg_hton->panic= myisammrg_panic;
-  myisammrg_hton->flags= HTON_NO_PARTITION;
+  myisammrg_hton->flags= HTON_NO_PARTITION | HTON_CAN_MERGE;
   myisammrg_hton->tablefile_extensions= ha_myisammrg_exts;
 
   return 0;