Part2: MDEV-18156 Assertion `0' failed or `btr_validate_index(index, 0,...

Part2: MDEV-18156 Assertion `0' failed or `btr_validate_index(index, 0, false)' in row_upd_sec_index_entry or error code 126: Index is corrupted upon DELETE with PAD_CHAR_TO_FULL_LENGTH

This patch allows the server to open old tables that have
"bad" generated columns (i.e. indexed virtual generated columns,
persistent generated columns) that depend on sql_mode,
for general things like SELECT, INSERT, DROP, etc.
Warning are issued in such cases.

Only these commands are now disallowed and return an error:
- CREATE TABLE introducing a "bad" generated column
- ALTER TABLE introducing a "bad" generated column
- CREATE INDEX introdicing a "bad" generated column
  (i.e. adding an index on a virtual generated column
   that depends on sql_mode).

Note, these commands are allowed:
- ALTER TABLE removing a "bad" generate column
- ALTER TABLE removing an index from a "bad" virtual generated column
- DROP INDEX removing an index from a "bad" virtual generated column
but only if the table does not have any "bad" columns as a result.

mysql-test/suite/vcol/r/vcol_sql_mode.result

@@ -400,6 +400,12 @@ SHOW WARNINGS;
 Level	Code	Message
 Error	1901	Function or expression '`c`' cannot be used in the GENERATED ALWAYS AS clause of `v`
 Warning	1105	Expression depends on the @@sql_mode value PAD_CHAR_TO_FULL_LENGTH
+CREATE INDEX v ON t1 (v);
+ERROR HY000: Function or expression '`c`' cannot be used in the GENERATED ALWAYS AS clause of `v`
+SHOW WARNINGS;
+Level	Code	Message
+Error	1901	Function or expression '`c`' cannot be used in the GENERATED ALWAYS AS clause of `v`
+Warning	1105	Expression depends on the @@sql_mode value PAD_CHAR_TO_FULL_LENGTH
 DROP TABLE t1;
 # A virtual column on the second position in an index - cannot depend on sql_mode
 CREATE TABLE t1 (id int, a CHAR(5), v TEXT AS (a) VIRTUAL, KEY(id, v(100)));

mysql-test/suite/vcol/t/vcol_sql_mode.test

@@ -294,6 +294,9 @@ CREATE TABLE t1 (
 --error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED
 ALTER TABLE t1 ADD KEY(v);
 SHOW WARNINGS;
+--error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED
+CREATE INDEX v ON t1 (v);
+SHOW WARNINGS;
 DROP TABLE t1;
 
 

mysql-test/suite/vcol/t/vcol_sql_mode_upgrade.test

+-- source include/mysql_upgrade_preparation.inc
+call mtr.add_suppression("Table rebuild required");
+
+let $MYSQLD_DATADIR= `select @@datadir`;
+
+--enable_prepare_warnings
+
+--echo #
+--echo # Opening a Maria-10.2.26 table with a stored VARCHAR column
+--echo #
+
+--echo # Copying maria100226_char_to_vchar_stored.* to MYSQLD_DATADIR
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_stored.frm $MYSQLD_DATADIR/test/t1.frm
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_stored.MYD $MYSQLD_DATADIR/test/t1.MYD
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_stored.MYI $MYSQLD_DATADIR/test/t1.MYI
+
+SHOW CREATE TABLE t1;
+SHOW CREATE TABLE t1;
+FLUSH TABLES;
+SHOW CREATE TABLE t1;
+
+FLUSH TABLES;
+SELECT * FROM t1;
+SELECT * FROM t1;
+FLUSH TABLES;
+SELECT * FROM t1;
+
+--error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED
+CREATE TABLE t2 LIKE t1;
+FLUSH TABLES;
+--error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED
+CREATE TABLE t2 LIKE t1;
+
+SHOW CREATE TABLE t1;
+--error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED
+ALTER TABLE t1 ADD b INT DEFAULT a;
+SHOW CREATE TABLE t1;
+SELECT * FROM t1;
+FLUSH TABLES;
+--error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED
+ALTER TABLE t1 ADD c INT DEFAULT a;
+SELECT * FROM t1;
+DROP TABLE t1;
+
+
+--echo #
+--echo # Fixing a Maria-10.2.26 table with a stored VARCHAR column
+--echo #
+
+--echo # Fixing by dropping the generated stored column
+--echo # Copying maria100226_char_to_vchar_stored.* to MYSQLD_DATADIR
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_stored.frm $MYSQLD_DATADIR/test/t1.frm
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_stored.MYD $MYSQLD_DATADIR/test/t1.MYD
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_stored.MYI $MYSQLD_DATADIR/test/t1.MYI
+
+ALTER TABLE t1 DROP v;
+SELECT * FROM t1;
+DROP TABLE t1;
+
+--echo # Fixing by altering the generation expression of the stored column
+--echo # Copying maria100226_char_to_vchar_stored.* to MYSQLD_DATADIR
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_stored.frm $MYSQLD_DATADIR/test/t1.frm
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_stored.MYD $MYSQLD_DATADIR/test/t1.MYD
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_stored.MYI $MYSQLD_DATADIR/test/t1.MYI
+
+SHOW CREATE TABLE t1;
+ALTER TABLE t1 MODIFY v VARCHAR(5) AS (RTRIM(a)) PERSISTENT;
+SHOW CREATE TABLE t1;
+DROP TABLE t1;
+
+
+
+--echo #
+--echo # Opening a Maria-10.2.26 table with a virtual VARCHAR column
+--echo #
+
+--echo # Copying maria100226_char_to_vchar_virtual.* to MYSQLD_DATADIR
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_virtual.frm $MYSQLD_DATADIR/test/t1.frm
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_virtual.MYD $MYSQLD_DATADIR/test/t1.MYD
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_virtual.MYI $MYSQLD_DATADIR/test/t1.MYI
+
+SHOW CREATE TABLE t1;
+SHOW CREATE TABLE t1;
+FLUSH TABLES;
+SHOW CREATE TABLE t1;
+
+FLUSH TABLES;
+SELECT * FROM t1;
+SELECT * FROM t1;
+FLUSH TABLES;
+SELECT * FROM t1;
+
+--error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED
+CREATE TABLE t2 LIKE t1;
+FLUSH TABLES;
+--error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED
+CREATE TABLE t2 LIKE t1;
+
+
+SHOW CREATE TABLE t1;
+--error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED
+ALTER TABLE t1 ADD b INT DEFAULT a;
+FLUSH TABLES;
+--error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED
+ALTER TABLE t1 ADD c INT DEFAULT a;
+SELECT * FROM t1;
+DROP TABLE t1;
+
+
+--echo #
+--echo # Fixing a Maria-10.2.26 table with a virtual VARCHAR column
+--echo #
+
+--echo # Fixing by dropping the virtual column
+--echo # Copying maria100226_char_to_vchar_virtual.* to MYSQLD_DATADIR
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_virtual.frm $MYSQLD_DATADIR/test/t1.frm
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_virtual.MYD $MYSQLD_DATADIR/test/t1.MYD
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_virtual.MYI $MYSQLD_DATADIR/test/t1.MYI
+
+SHOW CREATE TABLE t1;
+ALTER TABLE t1 DROP v;
+SELECT * FROM t1;
+DROP TABLE t1;
+
+--echo # Fixing by dropping a key on a virtual column, using ALTER TABLE
+--echo # Copying maria100226_char_to_vchar_virtual.* to MYSQLD_DATADIR
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_virtual.frm $MYSQLD_DATADIR/test/t1.frm
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_virtual.MYD $MYSQLD_DATADIR/test/t1.MYD
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_virtual.MYI $MYSQLD_DATADIR/test/t1.MYI
+
+SHOW CREATE TABLE t1;
+ALTER TABLE t1 DROP KEY v;
+SHOW CREATE TABLE t1;
+SELECT * FROM t1;
+DROP TABLE t1;
+
+
+--echo # Fixing by dropping a key on a virtual column, using DROP INDEX
+--echo # Copying maria100226_char_to_vchar_virtual.* to MYSQLD_DATADIR
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_virtual.frm $MYSQLD_DATADIR/test/t1.frm
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_virtual.MYD $MYSQLD_DATADIR/test/t1.MYD
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_virtual.MYI $MYSQLD_DATADIR/test/t1.MYI
+
+DROP INDEX v ON t1;
+SHOW CREATE TABLE t1;
+SELECT * FROM t1;
+DROP TABLE t1;
+
+
+--echo # Fixing by altering the generation expression of a virtual column
+--echo # Copying maria100226_char_to_vchar_virtual.* to MYSQLD_DATADIR
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_virtual.frm $MYSQLD_DATADIR/test/t1.frm
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_virtual.MYD $MYSQLD_DATADIR/test/t1.MYD
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_virtual.MYI $MYSQLD_DATADIR/test/t1.MYI
+
+SHOW CREATE TABLE t1;
+ALTER TABLE t1 MODIFY v VARCHAR(5) AS(RTRIM(a)) VIRTUAL;
+SHOW CREATE TABLE t1;
+SELECT * FROM t1;
+CREATE TABLE t2 LIKE t1;
+SHOW CREATE TABLE t2;
+DROP TABLE t2;
+DROP TABLE t1;
+
+
+--echo #
+--echo # Upgrading a Maria-10.2.26 table with a stored column
+--echo #
+
+--echo # Copying maria100226_char_to_varchar.* to MYSQLD_DATADIR
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_stored.frm $MYSQLD_DATADIR/test/t1.frm
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_stored.MYD $MYSQLD_DATADIR/test/t1.MYD
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_stored.MYI $MYSQLD_DATADIR/test/t1.MYI
+
+CHECK TABLE t1 FOR UPGRADE;
+FLUSH TABLES;
+CHECK TABLE t1 FOR UPGRADE;
+DROP TABLE t1;
+
+
+--echo #
+--echo # Upgrading a Maria-10.2.26 table with a virtual column
+--echo #
+
+--echo # Copying maria100226_char_to_varchar.* to MYSQLD_DATADIR
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_virtual.frm $MYSQLD_DATADIR/test/t1.frm
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_virtual.MYD $MYSQLD_DATADIR/test/t1.MYD
+--copy_file std_data/vcol_upgrade/maria100226_char_to_vchar_virtual.MYI $MYSQLD_DATADIR/test/t1.MYI
+
+CHECK TABLE t1 FOR UPGRADE;
+FLUSH TABLES;
+CHECK TABLE t1 FOR UPGRADE;
+DROP TABLE t1;

sql/field.cc

@@ -1428,14 +1428,16 @@ void Field::load_data_set_value(const char *pos, uint length,
 }
 
 
-void Field::error_generated_column_function_is_not_allowed(THD *thd) const
+void Field::error_generated_column_function_is_not_allowed(THD *thd,
+                                                           bool error) const
 {
   StringBuffer<64> tmp;
   vcol_info->expr->print(&tmp, (enum_query_type)
                                (QT_TO_SYSTEM_CHARSET |
                                 QT_ITEM_IDENT_SKIP_DB_NAMES |
                                 QT_ITEM_IDENT_SKIP_TABLE_NAMES));
-  my_error(ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED, MYF(0),
+  my_error(ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED,
+           MYF(error ? 0 : ME_JUST_WARNING),
            tmp.c_ptr(), vcol_info->get_vcol_type_name(),
            const_cast<const char*>(field_name));
 }
@@ -1446,7 +1448,7 @@ void Field::error_generated_column_function_is_not_allowed(THD *thd) const
   that it cannot handle.
   See sql_mode.h for details.
 */
-bool Field::check_vcol_sql_mode_dependency(THD *thd) const
+bool Field::check_vcol_sql_mode_dependency(THD *thd, vcol_init_mode mode) const
 {
   DBUG_ASSERT(vcol_info);
   if ((flags & PART_KEY_FLAG) != 0 || stored_in_db())
@@ -1456,9 +1458,10 @@ bool Field::check_vcol_sql_mode_dependency(THD *thd) const
         Sql_mode_dependency(~0, ~can_handle_sql_mode_dependency_on_store());
     if (dep)
     {
-      error_generated_column_function_is_not_allowed(thd);
+      bool error= (mode & VCOL_INIT_DEPENDENCY_FAILURE_IS_ERROR) != 0;
+      error_generated_column_function_is_not_allowed(thd, error);
       dep.push_dependency_warnings(thd);
-      return true;
+      return error;
     }
   }
   return false;

sql/field.h

@@ -697,7 +697,8 @@ class Field: public Value_source
     val_str(&result);
     return to->store(result.ptr(), result.length(), charset());
   }
-  void error_generated_column_function_is_not_allowed(THD *thd) const;
+  void error_generated_column_function_is_not_allowed(THD *thd, bool error)
+                                                      const;
   static void do_field_int(Copy_field *copy);
   static void do_field_real(Copy_field *copy);
   static void do_field_string(Copy_field *copy);
@@ -1175,7 +1176,7 @@ class Field: public Value_source
   }
 
   bool stored_in_db() const { return !vcol_info || vcol_info->stored_in_db; }
-  bool check_vcol_sql_mode_dependency(THD *) const;
+  bool check_vcol_sql_mode_dependency(THD *, vcol_init_mode mode) const;
 
   virtual sql_mode_t value_depends_on_sql_mode() const
   {

sql/sql_insert.cc

@@ -2529,7 +2529,8 @@ TABLE *Delayed_insert::get_local_table(THD* client_thd)
                                                       sizeof(MY_BITMAP))))
       goto error;
 
-    if (parse_vcol_defs(client_thd, client_thd->mem_root, copy, &error_reported))
+    if (parse_vcol_defs(client_thd, client_thd->mem_root, copy, &error_reported,
+                        VCOL_INIT_DEPENDENCY_FAILURE_IS_WARNING))
       goto error;
   }
 

sql/table.cc

@@ -989,7 +989,7 @@ static void mysql57_calculate_null_position(TABLE_SHARE *share,
     expression
 */
 bool parse_vcol_defs(THD *thd, MEM_ROOT *mem_root, TABLE *table,
-                     bool *error_reported)
+                     bool *error_reported, vcol_init_mode mode)
 {
   CHARSET_INFO *save_character_set_client= thd->variables.character_set_client;
   CHARSET_INFO *save_collation= thd->variables.collation_connection;
@@ -1073,7 +1073,7 @@ bool parse_vcol_defs(THD *thd, MEM_ROOT *mem_root, TABLE *table,
       vcol= unpack_vcol_info_from_frm(thd, mem_root, table, &expr_str,
                                     &((*field_ptr)->vcol_info), error_reported);
       *(vfield_ptr++)= *field_ptr;
-      if (vcol && field_ptr[0]->check_vcol_sql_mode_dependency(thd))
+      if (vcol && field_ptr[0]->check_vcol_sql_mode_dependency(thd, mode))
       {
         DBUG_ASSERT(thd->is_error());
         *error_reported= true;
@@ -3223,7 +3223,23 @@ enum open_frm_error open_table_from_share(THD *thd, TABLE_SHARE *share,
     if (share->table_check_constraints || share->field_check_constraints)
       outparam->check_constraints= check_constraint_ptr;
 
-    if (parse_vcol_defs(thd, &outparam->mem_root, outparam, &error_reported))
+    vcol_init_mode mode= VCOL_INIT_DEPENDENCY_FAILURE_IS_WARNING;
+    switch (thd->lex->sql_command)
+    {
+    case SQLCOM_CREATE_TABLE:
+      mode= VCOL_INIT_DEPENDENCY_FAILURE_IS_ERROR;
+      break;
+    case SQLCOM_ALTER_TABLE:
+    case SQLCOM_CREATE_INDEX:
+    case SQLCOM_DROP_INDEX:
+      if ((ha_open_flags & HA_OPEN_FOR_ALTER) == 0)
+        mode= VCOL_INIT_DEPENDENCY_FAILURE_IS_ERROR;
+      break;
+    default:
+      break;
+    }
+    if (parse_vcol_defs(thd, &outparam->mem_root, outparam,
+                        &error_reported, mode))
     {
       error= OPEN_FRM_CORRUPTED;
       goto err;

sql/table.h

@@ -325,6 +325,20 @@ enum tmp_table_type
 };
 enum release_type { RELEASE_NORMAL, RELEASE_WAIT_FOR_DROP };
 
+
+enum vcol_init_mode
+{
+  VCOL_INIT_DEPENDENCY_FAILURE_IS_WARNING= 1,
+  VCOL_INIT_DEPENDENCY_FAILURE_IS_ERROR= 2
+  /*
+    There may be new flags here.
+    e.g. to automatically remove sql_mode dependency:
+      GENERATED ALWAYS AS (char_col) ->
+      GENERATED ALWAYS AS (RTRIM(char_col))
+  */
+};
+
+
 enum enum_vcol_update_mode
 {
   VCOL_UPDATE_FOR_READ= 0,
@@ -2736,7 +2750,7 @@ bool fix_session_vcol_expr(THD *thd, Virtual_column_info *vcol);
 bool fix_session_vcol_expr_for_read(THD *thd, Field *field,
                                     Virtual_column_info *vcol);
 bool parse_vcol_defs(THD *thd, MEM_ROOT *mem_root, TABLE *table,
-                     bool *error_reported);
+                     bool *error_reported, vcol_init_mode expr);
 TABLE_SHARE *alloc_table_share(const char *db, const char *table_name,
                                const char *key, uint key_length);
 void init_tmp_table_share(THD *thd, TABLE_SHARE *share, const char *key,