MDEV-20077 Warning on full history partition is delayed until next DML statement

Moved LIMIT warning from vers_set_hist_part() to new call
vers_check_limit() at table unlock phase. At that point
read_partitions bitmap is already pruned by DML code (see
prune_partitions(), find_used_partitions()) so we have to set
corresponding bits for working history partition.

Also we don't do my_error(ME_WARNING|ME_ERROR_LOG), because at that
point it doesn't update warnings number, so command reports 0 warnings
(but warning list is still updated). Instead we do
push_warning_printf() and sql_print_warning() separately.

Under LOCK TABLES external_lock(F_UNLCK) is not executed. There is
start_stmt(), but no corresponding "stop_stmt()". So for that mode we
call vers_check_limit() directly from close_thread_tables().

Test result has been changed according to new LIMIT and warning
printing algorithm. For convenience all LIMIT warnings are marked with
"You see warning above ^".

TODO MDEV-20345 fixed. Now vers_history_generating() contains
fine-grained list of DML-commands that can generate history (and TODO
mechanism worked well).

mysql-test/suite/versioning/r/delete_history.result

@@ -65,6 +65,9 @@ partition pn current);
 insert into t values (1);
 update t set a= 2;
 update t set a= 3;
+Warnings:
+Warning	4114	Versioned table `test`.`t`: partition `p1` is full, add more HISTORY partitions
+# You see warning above ^
 delete history from t;
 select * from t for system_time all;
 a

mysql-test/suite/versioning/t/delete_history.test

@@ -65,6 +65,7 @@ partition by system_time limit 1 (
 insert into t values (1);
 update t set a= 2;
 update t set a= 3;
+--echo # You see warning above ^
 delete history from t;
 select * from t for system_time all;
 

mysql-test/suite/versioning/t/partition.test

@@ -228,12 +228,14 @@ insert into t1 values (1), (2), (3), (4), (5), (6);
 select * from t1 partition (pn);
 delete from t1 where x < 4;
 delete from t1;
+--echo # You see warning above ^
 select * from t1 partition (p0);
 select * from t1 partition (p1);
 
 insert into t1 values (7), (8);
 --echo ### warn about full partition
 delete from t1;
+--echo # You see warning above ^
 select * from t1 partition (p1) order by x;
 
 --echo ### Assertion in ALTER on warning from partitioning LIMIT [#446]
@@ -283,7 +285,9 @@ select * from t1 partition (pnsp1);
 --echo ### warn about full partition
 delete from t1 where x < 3;
 delete from t1;
+--echo # You see warning above ^
 delete from t1;
+--echo # You see warning above ^ (no matter if nothing was deleted)
 select * from t1 partition (p0sp0);
 select * from t1 partition (p0sp1);
 select * from t1 partition (p1sp0);
@@ -381,7 +385,9 @@ alter table t1 partition by system_time limit 1 (
     partition p2 history,
     partition pn current);
 delete from t1 where x = 1;
+--echo # You see warning above ^
 delete from t1 where x = 2;
+--echo # You see warning above ^
 
 --echo # MDEV-14923 Assertion upon INSERT into locked versioned partitioned table
 create or replace table t1 (x int) with system versioning
@@ -529,7 +535,9 @@ create or replace table t1 (x int) with system versioning partition by system_ti
 lock tables t1 write;
 insert into t1 values (0), (1), (2), (3);
 delete from t1 where x < 3;
+--echo # You see warning above ^
 delete from t1;
+--echo # You see warning above ^
 unlock tables;
 
 --echo #
@@ -640,6 +648,7 @@ partition by system_time limit 1
 insert into t1 values (null);
 update t1 set f= 'foo';
 update t1 set f= 'bar';
+--echo # You see warning above ^
 
 create or replace view v1 as select * from t1 for system_time all;
 --error ER_TABLE_NOT_LOCKED_FOR_WRITE
@@ -830,22 +839,19 @@ create table t1 (x int) with system versioning
 partition by system_time limit 1 (
   partition p0 history,
   partition p1 history,
+  partition p2 history, # p2 just disables warning about p1 partition full
   partition pn current);
 
 insert into t1 values (0);
 update t1 set x= x + 1;
 update t1 set x= x + 1;
-update t1 set x= x + 1;
-update t1 set x= x + 1;
 
 select * from t1 partition (p0);
 select * from t1 partition (p1);
 select * from t1 partition (pn);
 
 --echo # TRUNCATE TABLE deletes history and current data
---disable_warnings
 truncate table t1;
---enable_warnings
 select * from t1 partition (p0);
 select * from t1 partition (p1);
 select * from t1 partition (pn);
@@ -853,8 +859,6 @@ select * from t1 partition (pn);
 insert into t1 values (0);
 update t1 set x= x + 1;
 update t1 set x= x + 1;
-update t1 set x= x + 1;
-update t1 set x= x + 1;
 
 --echo # TRUNCATE PARTITION ALL does the same
 alter table t1 truncate partition all;
@@ -865,8 +869,6 @@ select * from t1 partition (pn);
 insert into t1 values (0);
 update t1 set x= x + 1;
 update t1 set x= x + 1;
-update t1 set x= x + 1;
-update t1 set x= x + 1;
 
 --echo # TRUNCATE PARTITION deletes data from HISTORY partition
 alter table t1 truncate partition p1;
@@ -882,6 +884,202 @@ select * from t1 partition (pn);
 
 drop table t1;
 
+--echo #
+--echo # MDEV-20077 Warning on full history partition is delayed until next DML statement
+--echo #
+--echo # DELETE
+create table t1 (x int) with system versioning
+partition by system_time limit 100 (
+  partition p0 history,
+  partition p1 history,
+  partition pn current);
+
+insert into t1 select seq from seq_0_to_200;
+
+--echo # p0 is filled with 100 records (no warnings):
+delete from t1 where x <= 99;
+--echo # p1 is filled with 1 + 100 records (warning is printed):
+delete from t1 where x <= 100;
+delete from t1;
+--echo # You see warning above ^
+
+select count(*) from t1 partition (p0);
+select count(*) from t1 partition (p1);
+drop table t1;
+
+--echo # DELETE under LOCK TABLES
+create table t1 (x int) with system versioning
+partition by system_time limit 100 (
+  partition p0 history,
+  partition p1 history,
+  partition pn current);
+
+insert into t1 select seq from seq_0_to_200;
+
+lock tables t1 write;
+--echo # (LOCK TABLES) p0 is filled with 100 records (no warnings):
+delete from t1 where x <= 99;
+--echo # (LOCK TABLES) p1 is filled with 1 + 100 records (warning is printed):
+delete from t1 where x <= 100;
+delete from t1;
+--echo # You see warning above ^
+unlock tables;
+
+select count(*) from t1 partition (p0);
+select count(*) from t1 partition (p1);
+drop table t1;
+
+--echo # DELETE multitable
+create table t1 (x int) with system versioning
+partition by system_time limit 100 (
+  partition p0 history,
+  partition p1 history,
+  partition pn current);
+
+create table t2 (y int);
+
+insert into t1 select seq from seq_0_to_200;
+insert into t2 select seq from seq_0_to_3;
+delete t1, t2 from t1 join t2 where x < 50 and y = 0;
+delete t1, t2 from t1 join t2 where x < 100 and y = 1;
+delete t1, t2 from t1 join t2 where x < 150 and y = 2;
+delete t1, t2 from t1 join t2;
+--echo # You see warning above ^
+
+select count(*) from t1 partition (p0);
+select count(*) from t1 partition (p1);
+drop table t1;
+
+--echo # UDPATE
+create table t1 (x int) with system versioning
+partition by system_time limit 100 (
+  partition p0 history,
+  partition p1 history,
+  partition pn current);
+
+insert into t1 select seq from seq_0_to_49;
+
+update t1 set x= x + 1;
+update t1 set x= x + 1;
+update t1 set x= x + 1;
+update t1 set x= x + 1;
+--echo # You see warning above ^
+
+select count(*) from t1 partition (p0);
+select count(*) from t1 partition (p1);
+drop tables t1, t2;
+
+--echo # UPDATE multitable
+create table t1 (x int) with system versioning
+partition by system_time limit 100 (
+  partition p0 history,
+  partition p1 history,
+  partition pn current);
+
+create table t2 (y int);
+
+insert into t1 select seq from seq_0_to_49;
+insert into t2 values (5);
+
+update t1, t2 set x= x + 1;
+update t1, t2 set x= x + 1;
+update t1, t2 set x= x + 1;
+update t1, t2 set x= x + 1;
+--echo # You see warning above ^
+
+select count(*) from t1 partition (p0);
+select count(*) from t1 partition (p1);
+drop tables t1, t2;
+
+--echo # INSERT .. ON DUPLICATE KEY UPDATE (ODKU)
+create table t1 (x int primary key) with system versioning
+partition by system_time limit 100 (
+  partition p0 history,
+  partition p1 history,
+  partition pn current);
+
+insert into t1 select seq from seq_0_to_100;
+
+delete from t1 where x <= 99;
+insert into t1 values (100) on duplicate key update x= 400;
+
+select count(*) from t1 partition (p0);
+select count(*) from t1 partition (p1);
+drop table t1;
+
+--echo # INSERT .. SELECT .. ON DUPLICATE KEY UPDATE (ODKU)
+create table t1 (x int primary key) with system versioning
+partition by system_time limit 100 (
+  partition p0 history,
+  partition p1 history,
+  partition pn current);
+
+create table t2 (y int);
+insert into t2 values (100);
+insert into t1 select seq from seq_0_to_100;
+
+delete from t1 where x <= 99;
+insert into t1 select * from t2 on duplicate key update x= 500;
+
+select count(*) from t1 partition (p0);
+select count(*) from t1 partition (p1);
+drop tables t1, t2;
+
+--echo # REPLACE
+create table t1 (x int primary key) with system versioning
+partition by system_time limit 100 (
+  partition p0 history,
+  partition p1 history,
+  partition pn current);
+
+insert into t1 select seq from seq_0_to_100;
+
+delete from t1 where x < 99;
+replace t1 values (100);
+replace t1 values (100);
+
+select count(*) from t1 partition (p0);
+select count(*) from t1 partition (p1);
+drop table t1;
+
+--echo # LOAD DATA .. REPLACE
+create table t1 (x int primary key) with system versioning
+partition by system_time limit 100 (
+  partition p0 history,
+  partition p1 history,
+  partition pn current);
+
+insert into t1 select seq from seq_0_to_49;
+select x into outfile 'MDEV-20077.data' from t1;
+
+load data infile 'MDEV-20077.data' replace into table t1 (x);
+load data infile 'MDEV-20077.data' replace into table t1 (x);
+load data infile 'MDEV-20077.data' replace into table t1 (x);
+load data infile 'MDEV-20077.data' replace into table t1 (x);
+--echo # You see warning above ^
+
+select count(*) from t1 partition (p0);
+select count(*) from t1 partition (p1);
+drop table t1;
+--remove_file $datadir/test/MDEV-20077.data
+
+--echo # REPLACE .. SELECT
+create table t1 (x int primary key) with system versioning
+partition by system_time limit 100 (
+  partition p0 history,
+  partition p1 history,
+  partition pn current);
+
+insert into t1 select seq from seq_0_to_49;
+replace t1 select * from t1;
+replace t1 select * from t1;
+replace t1 select * from t1;
+replace t1 select * from t1;
+--echo # You see warning above ^
+
+select count(*) from t1 partition (p0);
+select count(*) from t1 partition (p1);
+drop table t1;
 --echo # End of 10.3 tests
 
 set global innodb_stats_persistent= @save_persistent;

sql/ha_partition.cc

@@ -4008,6 +4008,8 @@ int ha_partition::external_lock(THD *thd, int lock_type)
   if (lock_type == F_UNLCK)
   {
     bitmap_clear_all(used_partitions);
+    if (m_lock_type == F_WRLCK && m_part_info->vers_require_hist_part(thd))
+      m_part_info->vers_check_limit(thd);
   }
   else
   {
@@ -4028,14 +4030,7 @@ int ha_partition::external_lock(THD *thd, int lock_type)
   {
     if (m_part_info->part_expr)
       m_part_info->part_expr->walk(&Item::register_field_in_read_map, 1, 0);
-    if (m_part_info->part_type == VERSIONING_PARTITION &&
-      /* TODO: MDEV-20345 exclude more inapproriate commands like INSERT
-         These commands may be excluded because working history partition is needed
-         only for versioned DML. */
-      thd->lex->sql_command != SQLCOM_SELECT &&
-      thd->lex->sql_command != SQLCOM_INSERT_SELECT &&
-      thd->lex->sql_command != SQLCOM_ALTER_TABLE &&
-      (error= m_part_info->vers_set_hist_part(thd)))
+    if ((error= m_part_info->vers_set_hist_part(thd)))
       goto err_handler;
   }
   DBUG_RETURN(0);
@@ -4188,11 +4183,7 @@ int ha_partition::start_stmt(THD *thd, thr_lock_type lock_type)
   case TL_WRITE_ONLY:
     if (m_part_info->part_expr)
       m_part_info->part_expr->walk(&Item::register_field_in_read_map, 1, 0);
-    if (m_part_info->part_type == VERSIONING_PARTITION &&
-      // TODO: MDEV-20345 (see above)
-      thd->lex->sql_command != SQLCOM_SELECT &&
-      thd->lex->sql_command != SQLCOM_INSERT_SELECT)
-      error= m_part_info->vers_set_hist_part(thd);
+    error= m_part_info->vers_set_hist_part(thd);
   default:;
   }
   DBUG_RETURN(error);

sql/partition_info.cc

@@ -835,6 +835,9 @@ bool partition_info::has_unique_name(partition_element *element)
 
 int partition_info::vers_set_hist_part(THD *thd)
 {
+  if (!vers_require_hist_part(thd))
+    return 0;
+
   if (table->pos_in_table_list &&
       table->pos_in_table_list->partition_names)
   {
@@ -856,12 +859,8 @@ int partition_info::vers_set_hist_part(THD *thd)
       vers_info->hist_part= next;
       records= next_records;
     }
-    if (records > vers_info->limit)
-    {
-      if (next == vers_info->now_part)
-        goto warn;
+    if (records >= vers_info->limit && next != vers_info->now_part)
       vers_info->hist_part= next;
-    }
     return 0;
   }
 
@@ -883,11 +882,45 @@ int partition_info::vers_set_hist_part(THD *thd)
     }
   }
   return 0;
-warn:
-  my_error(WARN_VERS_PART_FULL, MYF(ME_WARNING|ME_ERROR_LOG),
-           table->s->db.str, table->s->table_name.str,
-           vers_info->hist_part->partition_name);
-  return 0;
+}
+
+
+/**
+  Warn at the end of DML command if the last history partition is out of LIMIT.
+*/
+void partition_info::vers_check_limit(THD *thd)
+{
+  if (!vers_info->limit ||
+      vers_info->hist_part->id + 1 < vers_info->now_part->id)
+    return;
+
+  /*
+    NOTE: at this point read_partitions bitmap is already pruned by DML code,
+    we have to set read bits for working history partition. We could use
+    bitmap_set_all(), but this is not optimal since there can be quite a number
+    of partitions.
+  */
+  const uint32 sub_factor= num_subparts ? num_subparts : 1;
+  uint32 part_id= vers_info->hist_part->id * sub_factor;
+  const uint32 part_id_end= part_id + sub_factor;
+  DBUG_ASSERT(part_id_end <= num_parts * sub_factor);
+  for (; part_id < part_id_end; ++part_id)
+    bitmap_set_bit(&read_partitions, part_id);
+
+  ha_partition *hp= (ha_partition*)(table->file);
+  ha_rows hist_rows= hp->part_records(vers_info->hist_part);
+  if (hist_rows >= vers_info->limit)
+  {
+    push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
+                        WARN_VERS_PART_FULL,
+                        ER_THD(thd, WARN_VERS_PART_FULL),
+                        table->s->db.str, table->s->table_name.str,
+                        vers_info->hist_part->partition_name);
+
+    sql_print_warning(ER_THD(thd, WARN_VERS_PART_FULL),
+                      table->s->db.str, table->s->table_name.str,
+                      vers_info->hist_part->partition_name);
+  }
 }
 
 

sql/partition_info.h

@@ -421,7 +421,13 @@ class partition_info : public Sql_alloc
     vers_info->limit= limit;
     return !limit;
   }
+  bool vers_require_hist_part(THD *thd) const
+  {
+    return part_type == VERSIONING_PARTITION &&
+      thd->lex->vers_history_generating();
+  }
   int vers_set_hist_part(THD *thd);
+  void vers_check_limit(THD *thd);
   bool vers_setup_expression(THD *thd, uint32 alter_add= 0); /* Stage 1. */
   partition_element *get_partition(uint part_id)
   {

sql/sql_base.cc

@@ -750,7 +750,12 @@ void close_thread_tables(THD *thd)
     DBUG_PRINT("tcache", ("table: '%s'  query_id: %lu",
                           table->s->table_name.str, (ulong) table->query_id));
     if (thd->locked_tables_mode)
+    {
+      if (table->part_info && table->part_info->vers_require_hist_part(thd) &&
+          !thd->stmt_arena->is_stmt_prepare())
+        table->part_info->vers_check_limit(thd);
       table->vcol_cleanup_expr(thd);
+    }
     if (thd->locked_tables_mode <= LTM_LOCK_TABLES ||
         table->query_id == thd->query_id)
     {

sql/sql_lex.h

@@ -4084,6 +4084,28 @@ struct LEX: public Query_tables_list
   {
     return create_info.vers_info;
   }
+  /* The list of history-generating DML commands */
+  bool vers_history_generating() const
+  {
+    switch (sql_command)
+    {
+      case SQLCOM_DELETE:
+        return !vers_conditions.delete_history;
+      case SQLCOM_UPDATE:
+      case SQLCOM_UPDATE_MULTI:
+      case SQLCOM_DELETE_MULTI:
+      case SQLCOM_REPLACE:
+      case SQLCOM_REPLACE_SELECT:
+        return true;
+      case SQLCOM_INSERT:
+      case SQLCOM_INSERT_SELECT:
+        return duplicates == DUP_UPDATE;
+      case SQLCOM_LOAD:
+        return duplicates == DUP_REPLACE;
+      default:
+        return false;
+    }
+  }
   sp_package *get_sp_package() const;
 
   /**