MDEV-28413 System versioning is incorrect in Spider

Basic functioning of system versioning in Spider: UPDATE, DELETE,
DELETE HISTORY, AS OF, FROM .. TO, BETWEEN .. AND.

More testing should be done for:
LOAD DATA, joins, derived tables, multi-update/delete and what else?

Note that we are still using HA_CAN_DIRECT_UPDATE_AND_DELETE, but not
for periods in DELETE (fwiw there is condition
!table_list->has_period(), but it is missing in UDPATE).

storage/spider/mysql-test/spider/r/versioning.result

+install plugin spider soname 'ha_spider';
+set spider_same_server_link= on;
+create server s foreign data wrapper mysql options
+(host '127.0.0.1', database 'test', user 'root', port MASTER_1_MYPORT);
+create table t1 (
+x int, y int
+) with system versioning;
+create table t2 (
+x int, y int,
+row_start timestamp(6) as row start invisible,
+row_end timestamp(6) as row end invisible,
+period for system_time (row_start, row_end)
+) with system versioning;
+create or replace table t1_sp (
+x int, y int)
+engine=spider comment='wrapper "mysql", srv "s", table "t1"'
+with system versioning;
+create or replace table t2_sp (
+x int, y int,
+row_start timestamp(6) as row start invisible,
+row_end timestamp(6) as row end invisible,
+period for system_time (row_start, row_end))
+engine=spider comment='wrapper "mysql", srv "s", table "t2"'
+with system versioning;
+# Timestamps are not propagated (subject of MDEV-16546, but not for implicit system fields)
+set timestamp= unix_timestamp('2000-01-01 00:00:00');
+insert into t1_sp values (1, 1), (2, 2);
+insert into t2_sp values (1, 1), (2, 2);
+set timestamp= unix_timestamp('2000-01-01 00:11:11');
+# check dml generating history
+delete from t1_sp where x = 1;
+delete from t2_sp where x = 1;
+select row_end into @t1 from t2 for system_time all where x = 1;
+update t1_sp set y= y + 1;
+select row_start into @t2 from t1 where x = 2;
+update t2_sp set y= y + 1;
+select * from t1_sp;
+x	y
+2	3
+select * from t2_sp;
+x	y
+2	3
+select * from t1_sp for system_time as of timestamp @t1;
+x	y
+2	2
+select * from t2_sp for system_time as of timestamp @t1;
+x	y
+2	2
+select * from t1_sp for system_time as of timestamp @t2;
+x	y
+2	3
+select * from t2_sp for system_time as of timestamp @t2;
+x	y
+2	2
+select * from t1_sp for system_time from timestamp @t1 to timestamp @t2;
+x	y
+2	2
+select * from t2_sp for system_time from timestamp @t1 to timestamp @t2;
+x	y
+2	2
+select * from t1_sp for system_time between timestamp @t1 and timestamp @t2;
+x	y
+2	3
+2	2
+select * from t2_sp for system_time between timestamp @t1 and timestamp @t2;
+x	y
+2	2
+# spider cannot call functions, it fails in spider_create_group_by_handler()
+# but somehow it doesn't fail the query and continues without FOR SYSTEM_TIME ALL clause
+select *, check_row(row_start, row_end) from t1_sp for system_time all order by row_end, y, x;
+x	y	check_row(row_start, row_end)
+2	3	CURRENT ROW
+select *, check_row(row_start, row_end) from t2_sp for system_time all order by row_end, y, x;
+x	y	check_row(row_start, row_end)
+2	3	CURRENT ROW
+# SELECT from base table works as usual:
+select *, check_row(row_start, row_end) from t1 for system_time all order by row_end, y, x;
+x	y	check_row(row_start, row_end)
+1	1	HISTORICAL ROW
+2	2	HISTORICAL ROW
+2	3	CURRENT ROW
+select *, check_row(row_start, row_end) from t2 for system_time all order by row_end, y, x;
+x	y	check_row(row_start, row_end)
+1	1	HISTORICAL ROW
+2	2	HISTORICAL ROW
+2	3	CURRENT ROW
+# here it works nice (append_table_list() has the chance to add FOR SYSTEM_TIME ALL)
+select *, row_start, row_end from t1_sp for system_time all order by row_end, y, x;
+x	y	row_start	row_end
+1	1	TIMESTAMP	TIMESTAMP
+2	2	TIMESTAMP	TIMESTAMP
+2	3	TIMESTAMP	TIMESTAMP
+select *, row_start, row_end from t2_sp for system_time all order by row_end, y, x;
+x	y	row_start	row_end
+1	1	TIMESTAMP	TIMESTAMP
+2	2	TIMESTAMP	TIMESTAMP
+2	3	TIMESTAMP	TIMESTAMP
+delete history from t1_sp before system_time @t1;
+delete history from t2_sp before system_time @t1;
+select *, row_start, row_end from t1_sp for system_time all order by row_end, y, x;
+x	y	row_start	row_end
+2	2	TIMESTAMP	TIMESTAMP
+2	3	TIMESTAMP	TIMESTAMP
+select *, row_start, row_end from t2_sp for system_time all order by row_end, y, x;
+x	y	row_start	row_end
+1	1	TIMESTAMP	TIMESTAMP
+2	2	TIMESTAMP	TIMESTAMP
+2	3	TIMESTAMP	TIMESTAMP
+select * from t1 for system_time all;
+x	y
+2	3
+2	2
+select * from t2 for system_time all;
+x	y
+1	1
+2	3
+2	2
+delete history from t1_sp;
+delete history from t2_sp;
+select *, row_start, row_end from t1_sp for system_time all order by row_end, y, x;
+x	y	row_start	row_end
+2	3	TIMESTAMP	TIMESTAMP
+select *, row_start, row_end from t2_sp for system_time all order by row_end, y, x;
+x	y	row_start	row_end
+2	3	TIMESTAMP	TIMESTAMP
+select * from t1 for system_time all;
+x	y
+2	3
+select * from t2 for system_time all;
+x	y
+2	3
+drop tables t1, t2, t1_sp, t2_sp;
+set timestamp= default;
+uninstall plugin spider;
+Warnings:
+Warning	1620	Plugin is busy and will be uninstalled on shutdown
+drop table mysql.spider_link_failed_log, mysql.spider_link_mon_servers, mysql.spider_tables, mysql.spider_table_crd, mysql.spider_table_position_for_recovery, mysql.spider_table_sts, mysql.spider_xa, mysql.spider_xa_failed_log, mysql.spider_xa_member;
+drop function spider_direct_sql;
+drop function spider_bg_direct_sql;
+drop function spider_ping_table;
+drop function spider_copy_tables;
+drop function spider_flush_table_mon_cache;

storage/spider/mysql-test/spider/t/versioning.test

+--source include/have_innodb.inc
+--source suite/versioning/common.inc
+
+install plugin spider soname 'ha_spider';
+set spider_same_server_link= on;
+
+--replace_result $MASTER_1_MYPORT MASTER_1_MYPORT
+eval create server s foreign data wrapper mysql options
+(host '127.0.0.1', database 'test', user 'root', port $MASTER_1_MYPORT);
+
+
+create table t1 (
+  x int, y int
+) with system versioning;
+
+create table t2 (
+  x int, y int,
+  row_start timestamp(6) as row start invisible,
+  row_end timestamp(6) as row end invisible,
+  period for system_time (row_start, row_end)
+) with system versioning;
+
+create or replace table t1_sp (
+  x int, y int)
+engine=spider comment='wrapper "mysql", srv "s", table "t1"'
+with system versioning;
+
+create or replace table t2_sp (
+  x int, y int,
+  row_start timestamp(6) as row start invisible,
+  row_end timestamp(6) as row end invisible,
+  period for system_time (row_start, row_end))
+engine=spider comment='wrapper "mysql", srv "s", table "t2"'
+with system versioning;
+
+--echo # Timestamps are not propagated (subject of MDEV-16546, but not for implicit system fields)
+set timestamp= unix_timestamp('2000-01-01 00:00:00');
+insert into t1_sp values (1, 1), (2, 2);
+insert into t2_sp values (1, 1), (2, 2);
+--sleep 0.01
+
+set timestamp= unix_timestamp('2000-01-01 00:11:11');
+--echo # check dml generating history
+delete from t1_sp where x = 1;
+--sleep 0.01
+delete from t2_sp where x = 1;
+--sleep 0.01
+
+select row_end into @t1 from t2 for system_time all where x = 1;
+
+update t1_sp set y= y + 1;
+select row_start into @t2 from t1 where x = 2;
+--sleep 0.01
+update t2_sp set y= y + 1;
+
+select * from t1_sp;
+select * from t2_sp;
+select * from t1_sp for system_time as of timestamp @t1;
+select * from t2_sp for system_time as of timestamp @t1;
+select * from t1_sp for system_time as of timestamp @t2;
+select * from t2_sp for system_time as of timestamp @t2;
+select * from t1_sp for system_time from timestamp @t1 to timestamp @t2;
+select * from t2_sp for system_time from timestamp @t1 to timestamp @t2;
+select * from t1_sp for system_time between timestamp @t1 and timestamp @t2;
+select * from t2_sp for system_time between timestamp @t1 and timestamp @t2;
+
+--echo  # spider cannot call functions, it fails in spider_create_group_by_handler()
+        # at:
+        #
+        # 1555            if (spider_db_print_item_type(item, null, spider, null, null, 0,
+        # 1556              roop_count, true, fields_arg))
+        # 1557            {
+        # 1558              dbug_print("info",("spider dbton_id=%d can't create select", roop_count));
+        # 1559              spider_clear_bit(dbton_bitmap, roop_count);
+        # 1560              keep_going = false;
+        # 1561              break;
+        # 1562            }
+        #
+--echo  # but somehow it doesn't fail the query and continues without FOR SYSTEM_TIME ALL clause
+select *, check_row(row_start, row_end) from t1_sp for system_time all order by row_end, y, x;
+select *, check_row(row_start, row_end) from t2_sp for system_time all order by row_end, y, x;
+--echo # SELECT from base table works as usual:
+select *, check_row(row_start, row_end) from t1 for system_time all order by row_end, y, x;
+select *, check_row(row_start, row_end) from t2 for system_time all order by row_end, y, x;
+
+--echo # here it works nice (append_table_list() has the chance to add FOR SYSTEM_TIME ALL)
+
+# Usage: SHOW_TIMESTAMPS=1 mtr versioning.spider
+if ($SHOW_TIMESTAMPS)
+{
+  select @t1, @t2;
+}
+if (!$SHOW_TIMESTAMPS)
+{
+--replace_regex /\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d.\d\d\d\d\d\d/TIMESTAMP/
+}
+select *, row_start, row_end from t1_sp for system_time all order by row_end, y, x;
+
+if (!$SHOW_TIMESTAMPS)
+{
+--replace_regex /\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d.\d\d\d\d\d\d/TIMESTAMP/
+}
+select *, row_start, row_end from t2_sp for system_time all order by row_end, y, x;
+
+delete history from t1_sp before system_time @t1;
+delete history from t2_sp before system_time @t1;
+
+if (!$SHOW_TIMESTAMPS)
+{
+--replace_regex /\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d.\d\d\d\d\d\d/TIMESTAMP/
+}
+select *, row_start, row_end from t1_sp for system_time all order by row_end, y, x;
+
+if (!$SHOW_TIMESTAMPS)
+{
+--replace_regex /\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d.\d\d\d\d\d\d/TIMESTAMP/
+}
+select *, row_start, row_end from t2_sp for system_time all order by row_end, y, x;
+
+select * from t1 for system_time all;
+select * from t2 for system_time all;
+
+delete history from t1_sp;
+delete history from t2_sp;
+
+if (!$SHOW_TIMESTAMPS)
+{
+--replace_regex /\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d.\d\d\d\d\d\d/TIMESTAMP/
+}
+select *, row_start, row_end from t1_sp for system_time all order by row_end, y, x;
+
+if (!$SHOW_TIMESTAMPS)
+{
+--replace_regex /\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d.\d\d\d\d\d\d/TIMESTAMP/
+}
+select *, row_start, row_end from t2_sp for system_time all order by row_end, y, x;
+
+select * from t1 for system_time all;
+select * from t2 for system_time all;
+
+drop tables t1, t2, t1_sp, t2_sp;
+set timestamp= default;
+--source suite/versioning/common_finish.inc
+
+uninstall plugin spider;
+drop table mysql.spider_link_failed_log, mysql.spider_link_mon_servers, mysql.spider_tables, mysql.spider_table_crd, mysql.spider_table_position_for_recovery, mysql.spider_table_sts, mysql.spider_xa, mysql.spider_xa_failed_log, mysql.spider_xa_member;
+drop function spider_direct_sql;
+drop function spider_bg_direct_sql;
+drop function spider_ping_table;
+drop function spider_copy_tables;
+drop function spider_flush_table_mon_cache;

storage/spider/spd_db_conn.cc

@@ -1515,9 +1515,8 @@ int spider_db_append_key_where_internal(
 ) {
   SPIDER_RESULT_LIST *result_list = &spider->result_list;
   SPIDER_SHARE *share = spider->share;
-#ifndef DBUG_OFF
   TABLE *table = spider->get_table();
-#endif
+  const auto &vers_conditions= table->pos_in_table_list->vers_conditions;
   KEY *key_info = result_list->key_info;
   int error_num;
   uint key_name_length;
@@ -1538,6 +1537,22 @@ int spider_db_append_key_where_internal(
   spider_db_handler *dbton_hdl = spider->dbton_handler[dbton_id];
   spider_db_share *dbton_share = share->dbton_share[dbton_id];
   DBUG_ENTER("spider_db_append_key_where_internal");
+  if (sql_type == SPIDER_SQL_TYPE_DELETE_SQL &&
+      vers_conditions.delete_history)
+  {
+    if (vers_conditions.orig_type == SYSTEM_TIME_BEFORE)
+    {
+      auto *start= vers_conditions.start.item->val_str();
+      if (str->append(" before system_time '"))
+        DBUG_RETURN(HA_ERR_OUT_OF_MEM);
+      if (str->append(*start))
+        DBUG_RETURN(HA_ERR_OUT_OF_MEM);
+      if (str->append("'"))
+        DBUG_RETURN(HA_ERR_OUT_OF_MEM);
+    }
+    DBUG_RETURN(0);
+  }
+
   switch (sql_type)
   {
     case SPIDER_SQL_TYPE_HANDLER:

storage/spider/spd_db_include.h

@@ -413,6 +413,7 @@ class spider_string
     uint32 arg_length,
     const String &to
   );
+  // FIXME: not defined inline?
   inline bool append(
     char chr
   );

storage/spider/spd_db_mysql.cc

@@ -6278,7 +6278,9 @@ int spider_db_mbase_util::append_table_list(spider_fields *fields,
     if (int error_num= db_share->append_table_name(
           str, spd->conn_link_idx[dbton_hdl->first_link_idx]))
       DBUG_RETURN(error_num);
-  if (str->append(" ") ||
+  if (str->append((table->vers_conditions.orig_type == SYSTEM_TIME_ALL ||
+                   table->vers_conditions.type == SYSTEM_TIME_ALL) ?
+                  " for system_time all " : " ") ||
       str->append(table_holder->alias->ptr(),
                   /* Don't append the trailing dot */
                   table_holder->alias->length() - 1))
@@ -8518,6 +8520,14 @@ int spider_mbase_handler::append_delete(
   if (str->reserve(SPIDER_SQL_DELETE_LEN))
     DBUG_RETURN(HA_ERR_OUT_OF_MEM);
   str->q_append(SPIDER_SQL_DELETE_STR, SPIDER_SQL_DELETE_LEN);
+  const bool delete_history=
+    spider->get_table()->pos_in_table_list->vers_conditions.delete_history;
+  if (delete_history)
+  {
+    str->append("history");
+    DBUG_RETURN(0);
+  }
+
   if (spider->wide_handler->low_priority)
   {
     if (str->reserve(SPIDER_SQL_LOW_PRIORITY_LEN))
@@ -11413,6 +11423,16 @@ int spider_mbase_handler::append_insert_values(
   str->q_append(SPIDER_SQL_OPEN_PAREN_STR, SPIDER_SQL_OPEN_PAREN_LEN);
   for (field = table->field; *field; field++)
   {
+    /*
+        Note: as of MDEV-16546 this condition should look like:
+
+        if ((*field)->invisible >= INVISIBLE_SYSTEM)
+          continue;
+
+        But @@system_versioning_insert_history=1 must be propagated for the above.
+    */
+    if ((*field)->vers_sys_field())
+      continue;
     DBUG_PRINT("info",("spider field_index=%u", (*field)->field_index));
     if (
       bitmap_is_set(table->write_set, (*field)->field_index) ||
@@ -11516,6 +11536,9 @@ int spider_mbase_handler::append_into(
   str->q_append(SPIDER_SQL_OPEN_PAREN_STR, SPIDER_SQL_OPEN_PAREN_LEN);
   for (field = table->field; *field; field++)
   {
+    /* Note: see above comment for MDEV-16546 */
+    if ((*field)->vers_sys_field())
+      continue;
     if (
       bitmap_is_set(table->write_set, (*field)->field_index) ||
       bitmap_is_set(table->read_set, (*field)->field_index)