Merge branch '11.4' into bb-11.4-release

client/CMakeLists.txt

@@ -53,7 +53,7 @@ SET_TARGET_PROPERTIES(mariadb-test PROPERTIES ENABLE_EXPORTS TRUE)
 MYSQL_ADD_EXECUTABLE(mariadb-check mysqlcheck.c)
 TARGET_LINK_LIBRARIES(mariadb-check ${CLIENT_LIB})
 
-MYSQL_ADD_EXECUTABLE(mariadb-dump mysqldump.c ../sql-common/my_user.c)
+MYSQL_ADD_EXECUTABLE(mariadb-dump mysqldump.cc ../sql-common/my_user.c connection_pool.cc)
 TARGET_LINK_LIBRARIES(mariadb-dump ${CLIENT_LIB})
 
 MYSQL_ADD_EXECUTABLE(mariadb-import mysqlimport.c)

client/connection_pool.cc

+/*
+   Copyright (c) 2023, MariaDB.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335
+   USA
+*/
+
+/*
+  Connection pool, with parallel query execution
+  Does not use threads, but IO multiplexing via mysql_send_query and
+  poll/iocp to wait for completions
+*/
+#include <my_global.h>
+#ifdef _WIN32
+#include <winsock2.h>
+#else
+#include <poll.h>
+#endif
+#include "connection_pool.h"
+#include <my_compiler.h>
+
+namespace async_pool
+{
+static ATTRIBUTE_NORETURN void die(const char *format, ...)
+{
+  va_list args;
+  va_start(args, format);
+  vfprintf(stderr, format, args);
+  va_end(args);
+  abort();
+}
+
+pooled_connection *connection_pool::get_connection()
+{
+  while (free_connections.empty())
+    wait_for_completions();
+  auto c= free_connections.front();
+  free_connections.pop();
+  return c;
+}
+
+#ifdef _WIN32
+void connection_pool::add_to_pollset(pooled_connection *c)
+{
+  DWORD err= ERROR_SUCCESS;
+  static char ch;
+  WSABUF buf;
+  buf.len= 0;
+  buf.buf= &ch;
+  if (!c->is_pipe)
+  {
+    /* Do async io (sockets). */
+    DWORD flags= 0;
+    if (WSARecv((SOCKET) c->handle, &buf, 1, 0, &flags, c, NULL))
+      err= WSAGetLastError();
+  }
+  else
+  {
+    /* Do async read (named pipe) */
+    if (!ReadFile(c->handle, buf.buf, buf.len, 0, c))
+      err= GetLastError();
+  }
+
+  if (err && err != ERROR_IO_PENDING)
+    die("%s failed: %d\n", c->is_pipe ? "ReadFile" : "WSARecv", err);
+}
+
+/*
+  Wait for completions of queries.Uses IOCP on windows to wait for completions.
+  (ReadFile/WSARecv with 0 bytes serves as readiness notification)
+*/
+void connection_pool::wait_for_completions()
+{
+  ULONG n;
+  OVERLAPPED_ENTRY events[32];
+  if (!GetQueuedCompletionStatusEx(iocp, events, array_elements(events), &n, INFINITE,
+                                   FALSE))
+  {
+    die("GetQueuedCompletionStatusEx failed: %d\n", GetLastError());
+  }
+
+  for (ULONG i= 0; i < n; i++)
+  {
+    auto c= (pooled_connection *) events[i].lpOverlapped;
+    if (!c)
+      die("GetQueuedCompletionStatusEx unexpected return");
+    DBUG_ASSERT(c->mysql);
+    DBUG_ASSERT(!events[i].lpCompletionKey);
+    DBUG_ASSERT(!events[i].dwNumberOfBytesTransferred);
+    complete_query(c);
+  }
+}
+#else /* !_WIN32 */
+void connection_pool::add_to_pollset(pooled_connection *c)
+{
+  size_t idx= c - &all_connections[0];
+  pollfd *pfd= &pollset[idx];
+  pfd->fd= c->fd;
+  pfd->events= POLLIN;
+  pfd->revents= 0;
+}
+
+/* something Linux-ish, can be returned for POLLIN event*/
+#ifndef POLLRDHUP
+#define POLLRDHUP 0
+#endif
+
+void connection_pool::wait_for_completions()
+{
+  int n;
+  while ((n= poll(pollset.data(), pollset.size(), -1)) <= 0)
+  {
+    if (errno == EINTR)
+      continue;
+    die("poll failed: %d\n", errno);
+  }
+
+  for (uint i= 0; n > 0 && i < pollset.size(); i++)
+  {
+    pollfd* pfd= &pollset[i];
+    if (pfd->revents &
+        (POLLIN | POLLPRI | POLLHUP | POLLRDHUP| POLLERR | POLLNVAL))
+    {
+      pfd->events= 0;
+      pfd->revents= 0;
+      pfd->fd= -1;
+      complete_query(&all_connections[i]);
+      n--;
+    }
+  }
+  if (n)
+    die("poll() failed to find free connection: %d\n");
+}
+#endif
+
+void connection_pool::complete_query(pooled_connection *c)
+{
+  int err= mysql_read_query_result(c->mysql);
+  if (c->on_completion)
+    c->on_completion(c->mysql, c->query.c_str(), !err, c->context);
+  if (c->release_connection)
+  {
+    c->in_use= false;
+    free_connections.push(c);
+  }
+}
+
+connection_pool::~connection_pool()
+{
+  close();
+}
+
+void connection_pool::init(MYSQL *con[], size_t n)
+{
+#ifdef _WIN32
+  iocp= CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
+  if (!iocp)
+    die("CreateIoCompletionPort failed: %d\n", GetLastError());
+#else
+  pollset.resize(n);
+  for (auto &pfd : pollset)
+    pfd.fd= -1;
+#endif
+
+
+  for (size_t i= 0; i < n; i++)
+    all_connections.emplace_back(con[i]);
+
+  for (auto &con : all_connections)
+  {
+    free_connections.push(&con);
+#ifdef _WIN32
+    if (!CreateIoCompletionPort(con.handle, iocp, 0, 0))
+      die("CreateIoCompletionPort failed: %d\n", GetLastError());
+#endif
+  }
+}
+
+int connection_pool::execute_async(const char *query,
+                                   query_completion_handler on_completion,
+                                   void *context, bool release_connecton)
+{
+  auto c= get_connection();
+  c->context= context;
+  c->on_completion= on_completion;
+  c->release_connection= release_connecton;
+  c->query= query;
+
+  int ret= mysql_send_query(c->mysql, query, (unsigned long) c->query.size());
+  if (ret)
+  {
+    free_connections.push(c);
+    return ret;
+  }
+
+  c->in_use= true;
+  add_to_pollset(c);
+  return 0;
+}
+
+/*
+  Wait until all queries are completed and all 
+  connections are idle.
+*/
+void connection_pool::wait_all()
+{
+  while (free_connections.size() != all_connections.size())
+    wait_for_completions();
+}
+
+void connection_pool::for_each_connection(void(*f)(MYSQL *mysql))
+{
+  for (auto &c : all_connections)
+   f(c.mysql);
+}
+
+int connection_pool::close()
+{
+  for (auto &c : all_connections)
+    mysql_close(c.mysql);
+
+  all_connections.clear();
+  while (!free_connections.empty())
+    free_connections.pop();
+#ifdef _WIN32
+  if (iocp)
+  {
+    CloseHandle(iocp);
+    iocp= nullptr;
+  }
+#endif
+  return 0;
+}
+
+pooled_connection::pooled_connection(MYSQL *c)
+{
+  mysql= c;
+#ifdef _WIN32
+  OVERLAPPED *ov= static_cast<OVERLAPPED *>(this);
+  memset(ov, 0, sizeof(OVERLAPPED));
+  mysql_protocol_type protocol;
+  if (c->host && !strcmp(c->host, "."))
+    protocol= MYSQL_PROTOCOL_PIPE;
+  else
+    protocol= (mysql_protocol_type) c->options.protocol;
+  is_pipe= protocol == MYSQL_PROTOCOL_PIPE;
+  handle= (HANDLE) mysql_get_socket(c);
+#else
+  fd= mysql_get_socket(c);
+#endif
+}
+
+} // namespace async_pool

client/connection_pool.h

+/*
+   Copyright (c) 2023, MariaDB.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335
+   USA
+*/
+
+#pragma once
+
+#include <mysql.h>
+#include <vector>
+#include <queue>
+#include <string>
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <poll.h>
+#endif
+
+/*
+  Implementation of asynchronous mariadb connection pool.
+
+  This pool consists of set of MYSQL* connections, created by C API
+  function. The intention is that all connections have the same state
+  same server, by the same user etc.
+
+  The "asynchronous" means the queries are executed on the server
+  without waiting for the server reply. The queries are submitted
+  with mysql_send_query(), and completions are picked by poll/IOCP.
+*/
+
+namespace async_pool
+{
+typedef void (*query_completion_handler)(MYSQL *mysql, const char *query, bool success, void *context);
+
+struct pooled_connection
+#ifdef _WIN32
+    : OVERLAPPED
+#endif
+{
+  MYSQL *mysql;
+  query_completion_handler on_completion=NULL;
+  void *context=NULL;
+  std::string query;
+  bool in_use=false;
+  bool release_connection=false;
+#ifdef _WIN32
+  bool is_pipe;
+  HANDLE handle;
+#else
+  int fd;
+#endif
+  pooled_connection(MYSQL *mysql);
+};
+
+
+struct connection_pool
+{
+private:
+  std::vector<pooled_connection> all_connections;
+  std::queue<pooled_connection *> free_connections;
+  pooled_connection *get_connection();
+  void wait_for_completions();
+  void complete_query(pooled_connection *c);
+  void add_to_pollset(pooled_connection *c);
+
+#ifdef _WIN32
+  HANDLE iocp=nullptr;
+#else
+  std::vector<pollfd> pollset;
+#endif
+public:
+  ~connection_pool();
+
+  /**
+    Add connections to the connection pool
+
+    @param con  - connections
+    @param n_connections - number of connections
+  */
+  void init(MYSQL *con[], size_t n_connections);
+
+  /**
+  Send query to the connection pool
+  Executes query on a connection in the pool, using mysql_send_query
+
+  @param query         - query string
+  @param on_completion - callback function to be called on completion
+  @param context       - user context that will be passed to the callback function
+  @param release_connecton - if true, the connection should be released to the
+         pool after the query is executed. If you execute another
+         mysql_send_query() on the same connection, set this to false.
+
+  Note: the function will block if there are no free connections in the pool.
+
+  @return return code of mysql_send_query
+  */
+  int execute_async(const char *query, query_completion_handler on_completion, void *context, bool release_connecton=true);
+
+  /** Waits for all outstanding queries to complete.*/
+  void wait_all();
+
+  /** Execute callback for each connection in the pool. */
+  void for_each_connection(void (*f)(MYSQL *mysql));
+
+  /**
+    Closes all connections in pool and frees all resources.
+    Does not wait for pending queries to complete
+    (use wait_all() for that)
+  */
+  int close();
+};
+
+} // namespace async_pool

client/mysqlimport.c

@@ -148,6 +148,9 @@ static struct my_option my_long_options[] =
   {"pipe", 'W', "Use named pipes to connect to server.", 0, 0, 0, GET_NO_ARG,
    NO_ARG, 0, 0, 0, 0, 0, 0},
 #endif
+  {"parallel", 'j', "Number of LOAD DATA jobs executed in parallel",
+   &opt_use_threads, &opt_use_threads, 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0,
+   0, 0},
   {"plugin_dir", OPT_PLUGIN_DIR, "Directory for client-side plugins.",
    &opt_plugin_dir, &opt_plugin_dir, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
@@ -170,10 +173,8 @@ static struct my_option my_long_options[] =
    &opt_mysql_unix_port, &opt_mysql_unix_port, 0, GET_STR,
    REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
 #include <sslopt-longopts.h>
-  {"use-threads", OPT_USE_THREADS,
-   "Load files in parallel. The argument is the number "
-   "of threads to use for loading data.",
-   &opt_use_threads, &opt_use_threads, 0, 
+  {"use-threads", OPT_USE_THREADS, "Synonym for --parallel option",
+   &opt_use_threads, &opt_use_threads, 0,
    GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
 #ifndef DONT_ALLOW_USER_CHANGE
   {"user", 'u', "User for login if not current user.", &current_user,

mysql-test/main/mariadb-dump-debug.result

+set @save_debug_dbug=@@global.debug_dbug;
+set global debug_dbug='+d,select_export_kill';
+mariadb-dump: Couldn't execute async query 'SELECT /*!40001 SQL_NO_CACHE */ `Host`, `User`, `Priv` INTO OUTFILE 'MYSQLTEST_VARDIR/tmp/global_priv.txt' /*!50138 CHARACTER SET binary */ FROM `mysql`.`global_priv`' (Query execution was interrupted)
+set global debug_dbug=@save_debug_dbug;

mysql-test/main/mariadb-dump-debug.test

+--source include/have_debug.inc
+--source include/not_embedded.inc
+# need to dump "mysql" schema
+--source include/have_innodb.inc
+
+# inject error on SELECT INTO OUTFILE
+set @save_debug_dbug=@@global.debug_dbug;
+set global debug_dbug='+d,select_export_kill';
+# This one succeeds
+--replace_result .\exe '' $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--error 2
+--exec $MYSQL_DUMP --tab=$MYSQLTEST_VARDIR/tmp/ --parallel=2 mysql global_priv 2>&1
+
+set global debug_dbug=@save_debug_dbug;
+

mysql-test/main/mysqldump-win.opt

+--loose-enable-named-pipe

mysql-test/main/mysqldump-win.result

+# Test mysqldump specific features (pipe connection, with parallel)
+CREATE TABLE t1 (i INT);
+INSERT INTO t1 VALUES(1);
+DROP TABLE t1;
+test.t1: Records: 1  Deleted: 0  Skipped: 0  Warnings: 0
+SELECT * FROM t1;
+i
+1
+DROP TABLE t1;

mysql-test/main/mysqldump-win.test

+--source include/windows.inc
+--source include/not_embedded.inc
+
+--echo # Test mysqldump specific features (pipe connection, with parallel)
+CREATE TABLE t1 (i INT);
+INSERT INTO t1 VALUES(1);
+
+--exec $MYSQL_DUMP --host=. --tab=$MYSQLTEST_VARDIR/tmp/ test --parallel=2
+DROP TABLE t1;
+--exec $MYSQL test  < $MYSQLTEST_VARDIR/tmp/t1.sql
+--exec $MYSQL_IMPORT  test $MYSQLTEST_VARDIR/tmp/t1.txt
+SELECT * FROM t1;
+DROP TABLE t1;
+

mysql-test/main/mysqldump.result

@@ -5531,7 +5531,6 @@ proc
 one
 DROP DATABASE bug25717383;
 mariadb-dump: Got error: 2005: "Unknown server host 'unknownhost'" when trying to connect
-mariadb-dump: Couldn't execute 'SHOW SLAVE STATUS': Server has gone away (2006)
 Usage: mariadb-dump [OPTIONS] database [tables]
 OR     mariadb-dump [OPTIONS] --databases DB1 [DB2 DB3...]
 OR     mariadb-dump [OPTIONS] --all-databases
@@ -6659,3 +6658,12 @@ drop table t1;
 # End of 10.4 tests
 #
 mariadb-dump: --xml can't be used with --tab.
+select @@max_connections into @save_max_connections;
+set global max_connections=10;
+mariadb-dump: Got error: 1040: "Too many connections" when trying to connect
+set global max_connections=300;
+mariadb-dump: Too many connections
+set global max_connections=@save_max_connections;
+#
+# End of 11.4 tests
+#

mysql-test/main/mysqldump.test

@@ -2482,12 +2482,12 @@ INSERT INTO t2 VALUES (3), (4);
 SELECT * FROM t1;
 SELECT * FROM t2;
 
---exec $MYSQL_DUMP --default-character-set=utf8mb4 --tab=$MYSQLTEST_VARDIR/tmp/ db_20772273
+--exec $MYSQL_DUMP --default-character-set=utf8mb4 --tab=$MYSQLTEST_VARDIR/tmp/ --parallel=2 db_20772273
 --exec $MYSQL db_20772273 < $MYSQLTEST_VARDIR/tmp/t1.sql
 --exec $MYSQL db_20772273 < $MYSQLTEST_VARDIR/tmp/t2.sql
 
 # Test mysqlimport with multiple threads
---exec $MYSQL_IMPORT --silent --use-threads=2 db_20772273 $MYSQLTEST_VARDIR/tmp/t1.txt $MYSQLTEST_VARDIR/tmp/t2.txt
+--exec $MYSQL_IMPORT --silent --parallel=2 db_20772273 $MYSQLTEST_VARDIR/tmp/t1.txt $MYSQLTEST_VARDIR/tmp/t2.txt
 
 SELECT * FROM t1;
 SELECT * FROM t2;
@@ -3023,3 +3023,24 @@ drop table t1;
 --replace_result mariadb-dump.exe mariadb-dump
 --error 1
 --exec $MYSQL_DUMP --xml --tab=$MYSQLTEST_VARDIR/tmp 2>&1
+
+#
+# MDEV-32589 parallel-mysqldump - test "too many connections"
+#
+select @@max_connections into @save_max_connections;
+set global max_connections=10;
+--replace_result mariadb-dump.exe mariadb-dump
+--error 2
+--exec $MYSQL_DUMP --tab=$MYSQLTEST_VARDIR/tmp/ --parallel=20 mysql 2>&1
+#
+# MDEV-32589 test builtin 256 connections limit
+#
+set global max_connections=300;
+--replace_result mariadb-dump.exe mariadb-dump
+--error 1
+--exec $MYSQL_DUMP --tab=$MYSQLTEST_VARDIR/tmp/ --parallel=280 mysql 2>&1
+set global max_connections=@save_max_connections;
+
+--echo #
+--echo # End of 11.4 tests
+--echo #

sql/sql_class.cc

@@ -3450,6 +3450,9 @@ int select_export::send_data(List<Item> &items)
   uint used_length=0,items_left=items.elements;
   List_iterator_fast<Item> li(items);
 
+  DBUG_EXECUTE_IF("select_export_kill", {
+    thd->killed= KILL_QUERY;
+  });
   if (my_b_write(&cache,(uchar*) exchange->line_start->ptr(),
 		 exchange->line_start->length()))
     goto err;