MDEV 7701 extra() calls for VP engine

Added Spider patches:
003_mariadb-10.0.15.vp.diff
060_mariadb-10.2.0.partition_reset_top_table_fields.diff

- Support HA_EXTRA_ADD_CHILDREN_LIST,HA_EXTRA_ATTACH_CHILDREN,
  HA_EXTRA_IS_ATTACHED_CHILDREN and HA_EXTRA_DETACH_CHILDREN
  in partition handler for handlers that has HA_CAN_MULTISTEPL_MERGE flag
- Added HA_CAN_MULTISTEPL_MERGE to MERGE handler.
- Added handler::get_child_handlers()
- Change m_num_lock to contain total number of locks. This was needed as
  we now adjust number of locks when extra(HA_EXTRA_ATTACH_CHILDREN) is
  called.

sql/ha_partition.cc

@@ -3536,6 +3536,13 @@ int ha_partition::open(const char *name, int mode, uint test_if_locked)
       name_buffer_ptr+= strlen(name_buffer_ptr) + 1;
     } while (*(++file));
   }
+  /*
+    We want to know the upper bound for locks, to allocate enough memory.
+    There is no performance lost if we simply return in lock_count() the
+    maximum number locks needed, only some minor over allocation of memory
+    in get_lock_data().
+  */
+  m_num_locks*= m_tot_parts;
   
   file= m_file;
   ref_length= (*file)->ref_length;
@@ -3977,25 +3984,14 @@ int ha_partition::start_stmt(THD *thd, thr_lock_type lock_type)
   @returns Number of locks returned in call to store_lock
 
   @desc
-    Returns the number of store locks needed in call to store lock.
-    We return number of partitions we will lock multiplied with number of
-    locks needed by each partition. Assists the above functions in allocating
-    sufficient space for lock structures.
+    Returns the maxinum possible number of store locks needed in call to
+    store lock.
 */
 
 uint ha_partition::lock_count() const
 {
   DBUG_ENTER("ha_partition::lock_count");
-  /*
-    The caller want to know the upper bound, to allocate enough memory.
-    There is no performance lost if we simply return maximum number locks
-    needed, only some minor over allocation of memory in get_lock_data().
-
-    Also notice that this may be called for another thread != table->in_use,
-    when mysql_lock_abort_for_thread() is called. So this is more safe, then
-    using number of partitions after pruning.
-  */
-  DBUG_RETURN(m_tot_parts * m_num_locks);
+  DBUG_RETURN(m_num_locks);
 }
 
 
@@ -7253,19 +7249,36 @@ int ha_partition::extra(enum ha_extra_function operation)
   }
     /* Category 9) Operations only used by MERGE */
   case HA_EXTRA_ADD_CHILDREN_LIST:
+    DBUG_RETURN(loop_extra(operation));
   case HA_EXTRA_ATTACH_CHILDREN:
-  case HA_EXTRA_IS_ATTACHED_CHILDREN:
-  case HA_EXTRA_DETACH_CHILDREN:
   {
-    /* Special actions for MERGE tables. Ignore. */
+    int result;
+    uint num_locks= 0;
+    handler **file;
+    if ((result = loop_extra(operation)))
+      DBUG_RETURN(result);
+
+    /* Recalculate lock count as each child may have different set of locks */
+    num_locks = 0;
+    file = m_file;
+    do
+    {
+      num_locks+= (*file)->lock_count();
+    } while (*(++file));
+
+    m_num_locks= num_locks;
     break;
   }
+  case HA_EXTRA_IS_ATTACHED_CHILDREN:
+    DBUG_RETURN(loop_extra(operation));
+  case HA_EXTRA_DETACH_CHILDREN:
+    DBUG_RETURN(loop_extra(operation));
+  case HA_EXTRA_MARK_AS_LOG_TABLE:
   /*
     http://dev.mysql.com/doc/refman/5.1/en/partitioning-limitations.html
     says we no longer support logging to partitioned tables, so we fail
     here.
   */
-  case HA_EXTRA_MARK_AS_LOG_TABLE:
     DBUG_RETURN(ER_UNSUPORTED_LOG_ENGINE);
   default:
   {
@@ -9139,6 +9152,22 @@ const COND *ha_partition::cond_push(const COND *cond)
   COND *res_cond = NULL;
   DBUG_ENTER("ha_partition::cond_push");
 
+  if (set_top_table_fields)
+  {
+    /*
+      We want to do this in a separate loop to not come into a situation
+      where we have only done cond_push() to some of the tables
+    */
+    do
+    {
+      if (((*file)->set_top_table_and_fields(top_table,
+                                             top_table_field,
+                                             top_table_fields)))
+        DBUG_RETURN(cond);                      // Abort cond push, no error
+    } while (*(++file));
+    file= m_file;
+  }
+
   do
   {
     if ((*file)->pushed_cond != cond)
@@ -9165,6 +9194,23 @@ void ha_partition::cond_pop()
   DBUG_VOID_RETURN;
 }
 
+void ha_partition::clear_top_table_fields()
+{
+  handler **file;
+  DBUG_ENTER("ha_partition::clear_top_table_fields");
+
+  if (set_top_table_fields)
+  {
+    set_top_table_fields= FALSE;
+    top_table= NULL;
+    top_table_field= NULL;
+    top_table_fields= 0;
+    for (file= m_file; *file; file++)
+      (*file)->clear_top_table_fields();
+  }
+  DBUG_VOID_RETURN;
+}
+
 
 struct st_mysql_storage_engine partition_storage_engine=
 { MYSQL_HANDLERTON_INTERFACE_VERSION };

sql/ha_partition.h

@@ -271,6 +271,10 @@ class ha_partition :public handler
   MY_BITMAP m_key_not_found_partitions;
   bool m_key_not_found;
 public:
+  handler **get_child_handlers()
+  {
+    return m_file;
+  }
   Partition_share *get_part_share() { return part_share; }
   handler *clone(const char *name, MEM_ROOT *mem_root);
   virtual void set_part_info(partition_info *part_info)
@@ -1205,6 +1209,7 @@ class ha_partition :public handler
   */
     virtual const COND *cond_push(const COND *cond);
     virtual void cond_pop();
+    virtual void clear_top_table_fields();
 
     private:
     int handle_opt_partitions(THD *thd, HA_CHECK_OPT *check_opt, uint flags);

sql/handler.cc

@@ -6000,6 +6000,7 @@ int handler::ha_reset()
   /* Reset information about pushed engine conditions */
   cancel_pushed_idx_cond();
   /* Reset information about pushed index conditions */
+  clear_top_table_fields();
   DBUG_RETURN(reset());
 }
 

sql/handler.h

@@ -286,6 +286,9 @@ enum enum_alter_inplace_result {
  */
 #define HA_BINLOG_FLAGS (HA_BINLOG_ROW_CAPABLE | HA_BINLOG_STMT_CAPABLE)
 
+/* The following is for partition handler */
+#define HA_CAN_MULTISTEP_MERGE (1LL << 53)
+
 /* bits in index_flags(index_number) for what you can do with index */
 #define HA_READ_NEXT            1       /* TODO really use this flag */
 #define HA_READ_PREV            2       /* supports ::index_prev */
@@ -1413,6 +1416,7 @@ handlerton *ha_default_tmp_handlerton(THD *thd);
 #define HTON_TEMPORARY_NOT_SUPPORTED (1 << 6) //Having temporary tables not supported
 #define HTON_SUPPORT_LOG_TABLES      (1 << 7) //Engine supports log tables
 #define HTON_NO_PARTITION            (1 << 8) //Not partition of these tables
+#define HTON_CAN_MULTISTEP_MERGE     (1 << 9) //You can merge mearged tables
 
 /*
   This flag should be set when deciding that the engine does not allow
@@ -2774,6 +2778,11 @@ class handler :public Sql_alloc
   virtual void unbind_psi();
   virtual void rebind_psi();
 
+  bool set_top_table_fields;
+  struct TABLE *top_table;
+  Field **top_table_field;
+  uint top_table_fields;
+
 private:
   /**
     The lock type set by when calling::ha_external_lock(). This is 
@@ -2808,7 +2817,9 @@ class handler :public Sql_alloc
     pushed_idx_cond(NULL),
     pushed_idx_cond_keyno(MAX_KEY),
     auto_inc_intervals_count(0),
-    m_psi(NULL), m_lock_type(F_UNLCK), ha_share(NULL)
+    m_psi(NULL), set_top_table_fields(FALSE), top_table(0),
+    top_table_field(0), top_table_fields(0),
+    m_lock_type(F_UNLCK), ha_share(NULL)
   {
     DBUG_PRINT("info",
                ("handler created F_UNLCK %d F_RDLCK %d F_WRLCK %d",
@@ -3657,6 +3668,36 @@ class handler :public Sql_alloc
  */
  virtual void cond_pop() { return; };
 
+ /**
+    This function is used to get correlating of a parent (table/column)
+    and children (table/column). When conditions are pushed down to child
+    table (like child of myisam_merge), child table needs to know about
+    which table/column is my parent for understanding conditions.
+ */
+ virtual int set_top_table_and_fields(TABLE *top_table,
+                                      Field **top_table_field,
+                                      uint top_table_fields)
+ {
+   if (!set_top_table_fields)
+   {
+     set_top_table_fields= TRUE;
+     this->top_table= top_table;
+     this->top_table_field= top_table_field;
+     this->top_table_fields= top_table_fields;
+   }
+   return 0;
+ }
+ virtual void clear_top_table_fields()
+ {
+   if (set_top_table_fields)
+   {
+     set_top_table_fields= FALSE;
+     top_table= NULL;
+     top_table_field= NULL;
+     top_table_fields= 0;
+   }
+ }
+
  /**
    Push down an index condition to the handler.
 

sql/sql_base.cc

@@ -1096,14 +1096,32 @@ unique_table(THD *thd, TABLE_LIST *table, TABLE_LIST *table_list,
 
   table= table->find_table_for_update();
 
-  if (table->table && table->table->file->ht->db_type == DB_TYPE_MRG_MYISAM)
+  if (table->table &&
+      table->table->file->ha_table_flags() & HA_CAN_MULTISTEP_MERGE)
   {
     TABLE_LIST *child;
     dup= NULL;
     /* Check duplicates of all merge children. */
-    for (child= table->next_global; child && child->parent_l == table;
+    for (child= table->next_global; child;
          child= child->next_global)
     {
+      if (child->table &&
+          child->table->file->ha_table_flags() & HA_CAN_MULTISTEP_MERGE)
+        continue;
+
+      /*
+        Ensure that the child has one parent that is the table that is
+        updated.
+      */
+      TABLE_LIST *tmp_parent= child;
+      while ((tmp_parent= tmp_parent->parent_l))
+      {
+        if (tmp_parent == table)
+          break;
+      }
+      if (!tmp_parent)
+        break;
+
       if ((dup= find_dup_table(thd, child, child->next_global, check_alias)))
         break;
     }
@@ -1112,6 +1130,8 @@ unique_table(THD *thd, TABLE_LIST *table, TABLE_LIST *table_list,
     dup= find_dup_table(thd, table, table_list, check_alias);
   return dup;
 }
+
+
 /*
   Issue correct error message in case we found 2 duplicate tables which
   prevent some update operation
@@ -4089,7 +4109,7 @@ bool open_tables(THD *thd, const DDL_options_st &options,
       continue;
 
     /* Schema tables may not have a TABLE object here. */
-    if (tbl->file->ht->db_type == DB_TYPE_MRG_MYISAM)
+    if (tbl->file->ha_table_flags() & HA_CAN_MULTISTEP_MERGE)
     {
       /* MERGE tables need to access parent and child TABLE_LISTs. */
       DBUG_ASSERT(tbl->pos_in_table_list == tables);
@@ -4636,7 +4656,7 @@ TABLE *open_ltable(THD *thd, TABLE_LIST *table_list, thr_lock_type lock_type,
     */
     DBUG_ASSERT(table_list->table);
     table= table_list->table;
-    if (table->file->ht->db_type == DB_TYPE_MRG_MYISAM)
+    if (table->file->ha_table_flags() & HA_CAN_MULTISTEP_MERGE)
     {
       /* A MERGE table must not come here. */
       /* purecov: begin tested */

storage/myisammrg/ha_myisammrg.h

@@ -91,7 +91,7 @@ class ha_myisammrg: public handler
             HA_ANY_INDEX_MAY_BE_UNIQUE | HA_CAN_BIT_FIELD |
             HA_HAS_RECORDS | HA_CAN_EXPORT |
             HA_NO_COPY_ON_ALTER |
-            HA_DUPLICATE_POS);
+            HA_DUPLICATE_POS | HA_CAN_MULTISTEP_MERGE);
   }
   ulong index_flags(uint inx, uint part, bool all_parts) const
   {