MDEV-34740 mariadb-import: create secondary indexes after data load

Parse CREATE TABLE statement, to perform data load without
secondary indexes. i.e temporarily drop secondary indexes and constraints
and recreate them after LOAD DATA INFILE.

This reduces random IO, and LOADING time, in many cases.

The behavior is guarded  by the new innodb_optimize_indexes option(default)

client/CMakeLists.txt

@@ -56,7 +56,7 @@ TARGET_LINK_LIBRARIES(mariadb-check ${CLIENT_LIB})
 MYSQL_ADD_EXECUTABLE(mariadb-dump mysqldump.cc ../sql-common/my_user.c connection_pool.cc)
 TARGET_LINK_LIBRARIES(mariadb-dump ${CLIENT_LIB})
 
-MYSQL_ADD_EXECUTABLE(mariadb-import mysqlimport.cc)
+MYSQL_ADD_EXECUTABLE(mariadb-import mysqlimport.cc import_util.cc import_util.h)
 target_include_directories(mariadb-import PRIVATE ${CMAKE_SOURCE_DIR}/tpool)
 target_link_libraries(mariadb-import PRIVATE tpool ${CLIENT_LIB})
 

client/import_util.cc

+/*
+   Copyright (c) 2024, MariaDB
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335  USA
+*/
+
+/*
+  This file contains some routines to do client-side parsing of CREATE TABLE
+  statements. The goal is to extract the primary key, constraints, and
+  secondary key. his is useful for optimizing the import process, to delay
+  secondary index creation until after the data has been loaded.
+*/
+
+#include <string>
+#include <vector>
+#include <regex>
+
+#include "import_util.h"
+/**
+ * Extract the first CREATE TABLE statement from a script.
+ *
+ * @param script The input script containing SQL statements.
+ * @return std::string The first CREATE TABLE statement found, or an empty
+ * string if not found.
+ */
+std::string extract_first_create_table(const std::string &script)
+{
+  std::regex create_table_regex(R"((CREATE\s+TABLE\s+[^;]+;)\s*\n)",
+                                std::regex::icase);
+  std::smatch match;
+  if (std::regex_search(script, match, create_table_regex))
+  {
+    return match[1];
+  }
+  return "";
+}
+
+TableDDLInfo::TableDDLInfo(const std::string &create_table_stmt)
+{
+  std::regex primary_key_regex(R"(^\s*(PRIMARY\s+KEY\s+(.*?)),?\n)",
+                               std::regex::icase);
+
+  std::regex constraint_regex(
+      R"(^\s*(CONSTRAINT\s+(`?(?:[^`]|``)+`?)\s+.*?),?\n)", std::regex::icase);
+
+  std::regex index_regex(
+      R"(^\s*((UNIQUE\s+)?(INDEX|KEY)\s+(`?(?:[^`]|``)+`?)\s+.*?),?\n)",
+      std::regex::icase);
+
+  std::regex engine_regex(R"(\bENGINE\s*=\s*(\w+))", std::regex::icase);
+  std::smatch match;
+  auto search_start= create_table_stmt.cbegin();
+
+  // Extract primary key
+  if (std::regex_search(search_start, create_table_stmt.cend(), match,
+                        primary_key_regex))
+  {
+    primary_key= {KeyOrConstraintType::PRIMARY_KEY, match[0], match[2]};
+    search_start= match.suffix().first;
+  }
+
+  // Extract constraints and foreign keys
+  search_start= create_table_stmt.cbegin();
+  while (std::regex_search(search_start, create_table_stmt.cend(), match,
+                           constraint_regex))
+  {
+    auto type= KeyOrConstraintType::CONSTRAINT;
+    auto name= match[2].matched ? match[2].str() : "";
+    auto definition= match[1];
+    constraints.push_back({type, definition, name});
+    search_start= match.suffix().first;
+  }
+
+  // Extract secondary indexes
+  search_start= create_table_stmt.cbegin();
+  while (std::regex_search(search_start, create_table_stmt.cend(), match,
+                           index_regex))
+  {
+    auto type= KeyOrConstraintType::INDEX;
+    auto name= match[4].matched ? match[4].str() : "";
+    auto definition= match[1];
+    secondary_indexes.push_back({type, definition, name});
+    search_start= match.suffix().first;
+  }
+
+  // Extract storage engine
+  if (std::regex_search(create_table_stmt, match, engine_regex))
+  {
+    storage_engine= match[1];
+  }
+
+  std::regex table_name_regex(R"(CREATE\s+TABLE\s+(`?(?:[^`]|``)+`?)\s*\()",
+                              std::regex::icase);
+  std::smatch table_name_match;
+  if (std::regex_search(create_table_stmt, table_name_match, table_name_regex))
+  {
+    table_name= table_name_match[1];
+  }
+}
+
+/**
+ Convert a KeyOrConstraintDefinitionType enum value to its
+ corresponding string representation.
+
+ @param type The KeyOrConstraintDefinitionType enum value.
+ @return std::string The string representation of the
+  KeyOrConstraintDefinitionType.
+*/
+static std::string definition_type_to_string(KeyOrConstraintType type)
+{
+  switch (type)
+  {
+  case KeyOrConstraintType::PRIMARY_KEY:
+    return "PRIMARY KEY";
+  case KeyOrConstraintType::UNIQUE_KEY:
+    return "UNIQUE KEY";
+  case KeyOrConstraintType::FOREIGN_KEY:
+    return "FOREIGN KEY";
+  case KeyOrConstraintType::CONSTRAINT:
+    return "CONSTRAINT";
+  case KeyOrConstraintType::INDEX:
+    return "INDEX";
+  case KeyOrConstraintType::KEY:
+    return "KEY";
+  default:
+    return "UNKNOWN";
+  }
+}
+
+std::string TableDDLInfo::generate_alter_add(
+    const std::vector<KeyDefinition> &definitions) const
+{
+  if (definitions.empty())
+  {
+    return "";
+  }
+
+  std::string sql= "ALTER TABLE " + table_name + " ";
+  bool need_comma= false;
+  for (const auto &definition : definitions)
+  {
+    if (need_comma)
+      sql+= ", ";
+    else
+      need_comma= true;
+    sql+= "ADD " + definition.definition;
+  }
+  return sql;
+}
+
+std::string TableDDLInfo::generate_alter_drop(
+    const std::vector<KeyDefinition> &definitions) const
+{
+  if (definitions.empty())
+  {
+    return "";
+  }
+
+  std::string sql= "ALTER TABLE " + table_name + " ";
+  bool need_comma= false;
+  for (const auto &definition : definitions)
+  {
+    if (need_comma)
+      sql+= ", ";
+    else
+      need_comma= true;
+    sql+= "DROP " + definition_type_to_string(definition.type) + " " +
+          definition.name;
+  }
+  return sql;
+}
+
+#ifdef MAIN
+int main()
+{
+  std::string script= R"(
+        -- Some SQL script
+         CREATE TABLE `book` (
+  `id` mediumint(8) unsigned NOT NULL AUTO_INCREMENT,
+  `title` varchar(200) NOT NULL,
+  `author_id` smallint(5) unsigned NOT NULL,
+  PRIMARY KEY (`id`),
+  KEY `fk_book_author` (`author_id`),
+  CONSTRAINT `fk_book_author` FOREIGN KEY (`author_id`) REFERENCES `author` (`id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci;
+    )";
+
+  auto create_table_stmt= extract_first_create_table(script);
+  if (create_table_stmt.empty())
+  {
+    std::cerr << "No CREATE TABLE statement found in the script." << std::endl;
+    return 1;
+  }
+
+  TableDDLInfo table_definitions(create_table_stmt);
+
+  std::cout << "Primary Key:" << std::endl;
+  std::cout << "Type: "
+            << definition_type_to_string(table_definitions.primary_key.type)
+            << std::endl;
+  std::cout << "Definition: " << table_definitions.primary_key.definition
+            << std::endl;
+  std::cout << "Name: " << table_definitions.primary_key.name << std::endl;
+
+  std::cout << "\nConstraints and Foreign Keys:" << std::endl;
+  for (const auto &entry : table_definitions.constraints)
+  {
+    std::cout << "Type: " << definition_type_to_string(entry.type)
+              << std::endl;
+    std::cout << "Definition: " << entry.definition << std::endl;
+    std::cout << "Name: " << entry.name << std::endl;
+  }
+
+  std::cout << "\nSecondary Indexes:" << std::endl;
+  for (const auto &entry : table_definitions.secondary_indexes)
+  {
+    std::cout << "Type: " << definition_type_to_string(entry.type)
+              << std::endl;
+    std::cout << "Definition: " << entry.definition << std::endl;
+    std::cout << "Name: " << entry.name << std::endl;
+  }
+
+  std::cout << "\nStorage Engine: " << table_definitions.storage_engine
+            << std::endl;
+
+  std::cout << "\nTable Name: " << table_definitions.table_name << std::endl;
+  std::cout << "\nDrop Constraints SQL: "
+            << table_definitions.drop_constraints_sql() << std::endl;
+  std::cout << "\nCreate Constraints SQL: "
+            << table_definitions.create_constraints_sql() << std::endl;
+  std::cout << "\nDrop Indexes SQL: "
+            << table_definitions.drop_secondary_indexes_sql() << std::endl;
+  std::cout << "\nCreate Indexes SQL: "
+            << table_definitions.create_secondary_indexes_sql() << std::endl;
+  return 0;
+}
+#endif

client/import_util.h

+/*
+   Copyright (c) 2024, MariaDB
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335  USA
+*/
+#include <string>
+#include <vector>
+
+/* TABLE DDL INFO - representation of parsed CREATE TABLE Statement */
+
+enum class KeyOrConstraintType
+{
+  PRIMARY_KEY,
+  UNIQUE_KEY,
+  FOREIGN_KEY,
+  CONSTRAINT,
+  INDEX,
+  KEY,
+  UNKNOWN
+};
+
+/**
+ *  Struct representing a table key definition
+ */
+struct KeyDefinition
+{
+  KeyOrConstraintType type; /* e.g PRIMARY_KEY, UNIQUE_KEY */
+  std::string definition;   /* The full key or constraint definition string, e.g
+                                UNIQUE KEY `uniq_idx` (`col`)
+                             */
+  std::string name;         /*The name of key or constraint */
+};
+
+/**
+   Extracts of index information from a database table.
+ */
+class TableDDLInfo
+{
+public:
+  TableDDLInfo(const std::string &create_table_stmt);
+
+  KeyDefinition primary_key;
+  std::vector<KeyDefinition> constraints;
+  std::vector<KeyDefinition> secondary_indexes;
+  std::string storage_engine;
+  std::string table_name;
+  std::string
+  generate_alter_add(const std::vector<KeyDefinition> &definitions) const;
+
+  std::string
+  generate_alter_drop(const std::vector<KeyDefinition> &definitions) const;
+
+public:
+ 
+  std::string drop_constraints_sql() const
+  {
+    return generate_alter_drop(constraints);
+  }
+  std::string create_constraints_sql() const
+  {
+    return generate_alter_add(constraints);
+  }
+  std::string drop_secondary_indexes_sql() const
+  {
+    return generate_alter_drop(secondary_indexes);
+  }
+  std::string create_secondary_indexes_sql() const
+  {
+    return generate_alter_add(secondary_indexes);
+  }
+};
+std::string extract_first_create_table(const std::string &script);

client/mysqlimport.cc

@@ -67,6 +67,7 @@ static char * opt_mysql_unix_port=0;
 static char *opt_plugin_dir= 0, *opt_default_auth= 0;
 static longlong opt_ignore_lines= -1;
 static char *opt_dir;
+static char opt_innodb_optimize_keys;
 
 #include <sslopt-vars.h>
 
@@ -169,6 +170,9 @@ static struct my_option my_long_options[] =
    "--ignore-table=database.table.  Only takes effect when used together with "
    "--dir option",
    0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+   {"innodb-optimize-keys", 0, "Create secondary indexes after data load (Innodb only).",
+   &opt_innodb_optimize_keys, &opt_innodb_optimize_keys, 0, GET_BOOL, NO_ARG,
+   1, 0, 0, 0, 0, 0},
   {"lines-terminated-by", 0, 
    "Lines in the input file are terminated by the given string.",
    &lines_terminated, &lines_terminated, 0, GET_STR,
@@ -575,6 +579,9 @@ static int create_db_if_not_exists(MYSQL *mysql, const char *dbname)
   return 0;
 }
 
+
+#include "import_util.h"
+
 static int handle_one_table(const table_load_params *params, MYSQL *mysql)
 {
   char tablename[FN_REFLEN], hard_path[FN_REFLEN],
@@ -613,9 +620,10 @@ static int handle_one_table(const table_load_params *params, MYSQL *mysql)
   bool tz_utc= false;
   std::string engine;
   std::vector<std::string> triggers;
+  std::string sql_text;
   if (!params->sql_file.empty())
   {
-    std::string sql_text= parse_sql_script(params->sql_file.c_str(), &tz_utc, &triggers,&engine);
+    sql_text= parse_sql_script(params->sql_file.c_str(), &tz_utc, &triggers,&engine);
     if (execute_sql_batch(mysql, sql_text.c_str(),params->sql_file.c_str()))
       DBUG_RETURN(1);
     if (params->data_file.empty())
@@ -645,6 +653,35 @@ static int handle_one_table(const table_load_params *params, MYSQL *mysql)
     if (exec_sql(mysql, sql_statement))
       DBUG_RETURN(1);
   }
+
+  std::string create_secondary_keys_sql, create_constraints_sql;
+  if (!sql_text.empty() && opt_innodb_optimize_keys)
+  {
+    std::string create_stmt= extract_first_create_table(sql_text);
+    TableDDLInfo ddl_info(create_stmt);
+
+    // std::string engine= ddl_info.get_engine();
+
+    if (engine == "InnoDB")
+    {
+      std::string drop_constraints_sql= ddl_info.drop_constraints_sql();
+      std::string drop_secondary_keys_sql=
+          ddl_info.drop_secondary_indexes_sql();
+      create_secondary_keys_sql= ddl_info.create_secondary_indexes_sql();
+      create_constraints_sql= ddl_info.create_constraints_sql();
+      if (!drop_constraints_sql.empty())
+      {
+        if (exec_sql(mysql, drop_constraints_sql))
+          DBUG_RETURN(1);
+      }
+      if (!drop_secondary_keys_sql.empty())
+      {
+        if (exec_sql(mysql, drop_secondary_keys_sql))
+          DBUG_RETURN(1);
+      }
+    }
+  }
+
   to_unix_path(hard_path);
   if (verbose)
   {
@@ -693,6 +730,21 @@ static int handle_one_table(const table_load_params *params, MYSQL *mysql)
       fprintf(stdout, "%s.%s: %s\n", db, tablename, info);
   }
 
+  if (!create_secondary_keys_sql.empty())
+  {
+    if (verbose)
+    {
+      fprintf(stdout,"Creating secondary keys for table %s\n", tablename);
+    }
+    if (exec_sql(mysql, create_secondary_keys_sql))
+     DBUG_RETURN(1);
+  }
+  if (!create_constraints_sql.empty())
+  {
+    if (exec_sql(mysql, create_constraints_sql))
+      DBUG_RETURN(1);
+  }
+
   /* Create triggers after loading data */
   for (const auto &trigger: triggers)
   {

mysql-test/main/mariadb-import.result

@@ -171,13 +171,35 @@ PRIMARY KEY (id)
 CREATE TABLE child (
 id INT,
 parent_id INT,
+c CHAR(4),
 INDEX par_ind (parent_id),
+UNIQUE INDEX(c),
 FOREIGN KEY (parent_id)
 REFERENCES parent(id)
-ON DELETE CASCADE
+ON DELETE CASCADE,
+CHECK (c >= 'a')
 ) ENGINE=INNODB;
 insert into parent values(1),(2);
-insert into child values (1,1),(1,2),(2,1),(2,2);
+insert into child values (1,1,'a'),(1,2,'b'),(2,1,'c'),(2,2,'d');
+CREATE TABLE offices (
+id int NOT NULL AUTO_INCREMENT,
+PRIMARY KEY (id)
+) ENGINE=InnoDB;
+CREATE TABLE users (
+id int NOT NULL AUTO_INCREMENT,
+office_id int DEFAULT NULL,
+slogan text GENERATED ALWAYS AS (concat('Hello world #',office_id)) STORED,
+PRIMARY KEY (id),
+KEY office_id (office_id),
+CONSTRAINT users_ibfk_1 FOREIGN KEY (office_id) REFERENCES offices (id)
+) ENGINE=InnoDB;
+insert into offices values();
+insert into offices values();
+insert into offices values();
+insert into offices values();
+insert into users (office_id) values (1);
+insert into users (office_id) values (2);
+insert into users (office_id) values (3);
 drop database db2;
 use db2;
 select * from parent;
@@ -185,11 +207,28 @@ id
 1
 2
 select * from child;
-id	parent_id
-1	1
-1	2
-2	1
-2	2
+id	parent_id	c
+1	1	a
+1	2	b
+2	1	c
+2	2	d
+show create table parent;
+Table	Create Table
+parent	CREATE TABLE `parent` (
+  `id` int(11) NOT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci
+show create table child;
+Table	Create Table
+child	CREATE TABLE `child` (
+  `id` int(11) DEFAULT NULL,
+  `parent_id` int(11) DEFAULT NULL,
+  `c` char(4) DEFAULT NULL,
+  UNIQUE KEY `c` (`c`),
+  KEY `par_ind` (`parent_id`),
+  CONSTRAINT `child_ibfk_1` FOREIGN KEY (`parent_id`) REFERENCES `parent` (`id`) ON DELETE CASCADE,
+  CONSTRAINT `CONSTRAINT_1` CHECK (`c` >= 'a')
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci
 drop table child;
 drop table parent;
 CREATE TABLE animals (id mediumint(9)
@@ -215,9 +254,16 @@ animals
 use test;
 drop database db2;
 Connecting to localhost
+Executing SQL script vardir/tmp/dump/db2/users.sql
+Loading data from LOCAL file: vardir/tmp/dump/db2/users.txt into users
+db2.users: Records: 3  Deleted: 0  Skipped: 0  Warnings: 0
+Creating secondary keys for table users
 Executing SQL script vardir/tmp/dump/db2/animals.sql
 Loading data from LOCAL file: vardir/tmp/dump/db2/animals.txt into animals
 db2.animals: Records: 2  Deleted: 0  Skipped: 0  Warnings: 0
+Executing SQL script vardir/tmp/dump/db2/offices.sql
+Loading data from LOCAL file: vardir/tmp/dump/db2/offices.txt into offices
+db2.offices: Records: 4  Deleted: 0  Skipped: 0  Warnings: 0
 Executing SQL script vardir/tmp/dump/db2/animal_count.sql
 Loading data from LOCAL file: vardir/tmp/dump/db2/animal_count.txt into animal_count
 db2.animal_count: Records: 1  Deleted: 0  Skipped: 0  Warnings: 0
@@ -238,6 +284,13 @@ create view a1 as select * from t1;
 use test;
 drop database db2;
 Connecting to localhost
+Executing SQL script vardir/tmp/dump/db2/users.sql
+Loading data from LOCAL file: vardir/tmp/dump/db2/users.txt into users
+db2.users: Records: 3  Deleted: 0  Skipped: 0  Warnings: 0
+Creating secondary keys for table users
+Executing SQL script vardir/tmp/dump/db2/offices.sql
+Loading data from LOCAL file: vardir/tmp/dump/db2/offices.txt into offices
+db2.offices: Records: 4  Deleted: 0  Skipped: 0  Warnings: 0
 Executing SQL script vardir/tmp/dump/db2/t1.sql
 Loading data from LOCAL file: vardir/tmp/dump/db2/t1.txt into t1
 db2.t1: Records: 1  Deleted: 0  Skipped: 0  Warnings: 0

mysql-test/main/mariadb-import.test

@@ -45,13 +45,40 @@ CREATE TABLE parent (
 CREATE TABLE child (
     id INT,
     parent_id INT,
+    c CHAR(4),
     INDEX par_ind (parent_id),
+    UNIQUE INDEX(c),
     FOREIGN KEY (parent_id)
         REFERENCES parent(id)
-        ON DELETE CASCADE
+        ON DELETE CASCADE,
+    CHECK (c >= 'a')
 ) ENGINE=INNODB;
 insert into parent values(1),(2);
-insert into child values (1,1),(1,2),(2,1),(2,2);
+insert into child values (1,1,'a'),(1,2,'b'),(2,1,'c'),(2,2,'d');
+
+# Example from https://github.com/mydumper/mydumper/issues/395  (can't repeat now)
+CREATE TABLE offices (
+  id int NOT NULL AUTO_INCREMENT,
+  PRIMARY KEY (id)
+) ENGINE=InnoDB;
+
+CREATE TABLE users (
+  id int NOT NULL AUTO_INCREMENT,
+  office_id int DEFAULT NULL,
+  slogan text GENERATED ALWAYS AS (concat('Hello world #',office_id)) STORED,
+  PRIMARY KEY (id),
+  KEY office_id (office_id),
+  CONSTRAINT users_ibfk_1 FOREIGN KEY (office_id) REFERENCES offices (id)
+) ENGINE=InnoDB;
+
+insert into offices values();
+insert into offices values();
+insert into offices values();
+insert into offices values();
+
+insert into users (office_id) values (1);
+insert into users (office_id) values (2);
+insert into users (office_id) values (3);
 
 --mkdir $MYSQLTEST_VARDIR/tmp/dump
 --exec $MYSQL_DUMP --dir=$MYSQLTEST_VARDIR/tmp/dump --all-databases
@@ -61,6 +88,8 @@ drop database db2;
 use db2;
 select * from parent;
 select * from child;
+show create table parent;
+show create table child;
 drop table child;
 drop table parent;
 --rmdir $MYSQLTEST_VARDIR/tmp/dump