MDEV-25282 Auto-shutdown on idle when socket-activated

Adds max_idle_execution system variable that corresponds
to the time in seconds under which the mariadbd executable will
run in an idle state (since last query) with no connections.

Under systemd socket activation this variable will get a 10 minute
default value, otherwise 0, representing no timeout. This will
enable a service to be activated on connection and fall back to
a shutdown state after 10 minutes of no queries. The systemd
socket activation can restart the service on the next connection
transparently.

The maximum max_idle_execution is given by the following
factors given its dependence on the OS system calls.

Windows WaitForMultipleObjects takes a DWORD (unsigned)
measure in milliseconds.  Poll takes a signed int milliseconds,
and negative values are treated as infinite so we can't overflow.
Select, the fall back if poll isn't there, takes a seconds value
in a timeval.time_t structure. As such the interface maximums are:
Windows: UINT_MAX / 1000
Poll: INT_MAX / 1000
Select: UINT_MAX (or higher)

As even the smallest value here, INT_MAX(32) / 1000 is ~25 days,
sufficient for the typical use case, its used in all environment for
simplicity of documentation and test cases.

A (non-exposed) global variable of server_last_activity is updated
on accepted connections (when max_idle_execution !=0)
and when the connection count (standard or extra) is down
to <= 1 to keep the number of updates on a single variable
low.

When the main accept loop times out on the max_idle_execution
seconds, and then the server_last_activity is checked along
with if current connection count (standard + extra) is 0
(in case a recently started connection hasn't finished
a query).

To make this neater, in non-Windows main accept loop moved
code to handle_new_socket_connection that encompasses accepting
a connection and the timeout mechanism has been separated too.

Changed when looping though possible connections, loop until
the end of the connection list and hereby assume two connection can
occur on the same poll/select call and both will be accepted.

The interactive_timeout and wait_timeout inherit the
max_idle_execution time (if set) compared to their previous defaults.

Thanks Sergei for the review.

include/my_service_manager.h

@@ -33,11 +33,12 @@
   sd_notifyf(0, "STATUS=" FMTSTR "\nEXTEND_TIMEOUT_USEC=%u\n", ##__VA_ARGS__, INTERVAL * 1000000)
 /* sd_listen_fds_with_names added v227 however RHEL/Centos7 has v219, fallback to sd_listen_fds */
 #ifndef HAVE_SYSTEMD_SD_LISTEN_FDS_WITH_NAMES
-#define sd_listen_fds_with_names(FD, NAMES) sd_listen_fds(FD)
+#define sd_listen_fds_with_names(UNSET, NAMES) sd_listen_fds(UNSET)
 #endif
 
 #else
-#define sd_listen_fds_with_names(FD, NAMES) (0)
+#define sd_listen_fds(UNSET) (0)
+#define sd_listen_fds_with_names(UNSET, NAMES) (0)
 #define sd_is_socket_unix(FD, TYPE, LISTENING, PATH, SIZE) (0)
 #define sd_is_socket_inet(FD, FAMILY, TYPE, LISTENING, PORT) (0)
 #define SD_LISTEN_FDS_START (0)

mysql-test/main/max_idle_execution.cnf

+!include include/default_my.cnf
+
+[mysqld.1]
+extra-port=        @ENV.MASTER_EXTRA_PORT
+extra-max-connections=1
+max-idle-execution=5
+
+[ENV]
+MASTER_EXTRA_PORT= @OPT.port

mysql-test/main/max_idle_execution.result

+disconnect default;
+'allow server to time out'
+'should have timed out'
+connect con0,localhost,root,,test;
+SELECT VARIABLE_VALUE AS THREAD_CONNECTED FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='THREADS_CONNECTED';
+THREAD_CONNECTED
+1
+SELECT 'we are back';
+we are back
+we are back
+SELECT VARIABLE_VALUE < 5 AS UPTIME_LESS_THAN_5_SECONDS FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='UPTIME';
+UPTIME_LESS_THAN_5_SECONDS
+1
+SELECT 'still here because the connection is open, but disconnecting now';
+still here because the connection is open, but disconnecting now
+still here because the connection is open, but disconnecting now
+disconnect con0;
+connect con2,127.0.0.1,root,,test,$MASTER_EXTRA_PORT,;
+SELECT "extra connection just created: still here, only 3 seconds since last query";
+extra connection just created: still here, only 3 seconds since last query
+extra connection just created: still here, only 3 seconds since last query
+SELECT VARIABLE_VALUE > 5 AS UPTIME_GREATER_THAN_5_SECONDS FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='UPTIME';
+UPTIME_GREATER_THAN_5_SECONDS
+1
+SELECT 'active extra connection kept the server up';
+active extra connection kept the server up
+active extra connection kept the server up
+SELECT @@interactive_timeout, @@wait_timeout;
+@@interactive_timeout	@@wait_timeout
+5	5
+disconnect con2;
+'allow server to time out'
+'should have timed out'

mysql-test/main/max_idle_execution.test

+--source include/not_embedded.inc
+
+--let $_server_id= `SELECT @@server_id`
+--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.expect
+--append_file $_expect_file_name
+wait
+EOF
+disable_reconnect;
+disconnect default;
+
+--echo 'allow server to time out'
+--sleep 7
+--echo 'should have timed out'
+--source include/wait_until_disconnected.inc
+
+--append_file $_expect_file_name
+restart
+EOF
+--sleep 1
+--connect con0,localhost,root,,test
+
+--source include/wait_until_connected_again.inc
+SELECT VARIABLE_VALUE AS THREAD_CONNECTED FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='THREADS_CONNECTED';
+SELECT 'we are back';
+SELECT VARIABLE_VALUE < 5 AS UPTIME_LESS_THAN_5_SECONDS FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='UPTIME';
+SELECT 'still here because the connection is open, but disconnecting now';
+disconnect con0;
+--sleep 3
+
+--connect con2,127.0.0.1,root,,test,$MASTER_EXTRA_PORT,
+SELECT "extra connection just created: still here, only 3 seconds since last query";
+
+--sleep 3
+SELECT VARIABLE_VALUE > 5 AS UPTIME_GREATER_THAN_5_SECONDS FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='UPTIME';
+SELECT 'active extra connection kept the server up';
+SELECT @@interactive_timeout, @@wait_timeout;
+disconnect con2;
+
+--echo 'allow server to time out'
+--sleep 7
+--echo 'should haved timed out'
+--source include/wait_until_disconnected.inc

mysql-test/main/mysqld--help.result

@@ -408,7 +408,8 @@ The following specify which files/extra groups are read (specified before remain
  the SQL thread starts
  --interactive-timeout=# 
  The number of seconds the server waits for activity on an
- interactive connection before closing it
+ interactive connection before closing it (Automatically
+ configured unless set explicitly)
  --join-buffer-size=# 
  The size of the buffer that is used for joins
  --join-buffer-space-limit=# 
@@ -602,6 +603,12 @@ The following specify which files/extra groups are read (specified before remain
  --max-error-count=# Max number of errors/warnings to store for a statement
  --max-heap-table-size=# 
  Don't allow creation of heap tables bigger than this
+ --max-idle-execution=# 
+ If no new connections or running queries within this time
+ (in seconds) shutdown the server. Defaults to 10 minutes
+ when started as a systemd socket activated service. 0
+ represents disabled. (Automatically configured unless set
+ explicitly)
  --max-join-size=#   Joins that are probably going to read more than
  max_join_size records return an error
  --max-length-for-sort-data=# 
@@ -1543,7 +1550,8 @@ The following specify which files/extra groups are read (specified before remain
  -V, --version[=name] 
  Output version information and exit.
  --wait-timeout=#    The number of seconds the server waits for activity on a
- connection before closing it
+ connection before closing it (Automatically configured
+ unless set explicitly)
 
 Variables (--variable-name=value)
 allow-suspicious-udfs FALSE
@@ -1700,6 +1708,7 @@ max-delayed-threads 20
 max-digest-length 1024
 max-error-count 64
 max-heap-table-size 16777216
+max-idle-execution 0
 max-join-size 18446744073709551615
 max-length-for-sort-data 1024
 max-password-errors 18446744073709551615

mysql-test/suite/sys_vars/r/sysvars_server_embedded.result

@@ -1912,6 +1912,16 @@ NUMERIC_BLOCK_SIZE	1024
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
+VARIABLE_NAME	MAX_IDLE_EXECUTION
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	INT UNSIGNED
+VARIABLE_COMMENT	If no new connections or running queries within this time (in seconds) shutdown the server. Defaults to 10 minutes when started as a systemd socket activated service. 0 represents disabled.
+NUMERIC_MIN_VALUE	0
+NUMERIC_MAX_VALUE	2147483
+NUMERIC_BLOCK_SIZE	1
+ENUM_VALUE_LIST	NULL
+READ_ONLY	NO
+COMMAND_LINE_ARGUMENT	REQUIRED
 VARIABLE_NAME	MAX_INSERT_DELAYED_THREADS
 VARIABLE_SCOPE	SESSION
 VARIABLE_TYPE	BIGINT UNSIGNED

mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result

@@ -2072,6 +2072,16 @@ NUMERIC_BLOCK_SIZE	1024
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
+VARIABLE_NAME	MAX_IDLE_EXECUTION
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	INT UNSIGNED
+VARIABLE_COMMENT	If no new connections or running queries within this time (in seconds) shutdown the server. Defaults to 10 minutes when started as a systemd socket activated service. 0 represents disabled.
+NUMERIC_MIN_VALUE	0
+NUMERIC_MAX_VALUE	2147483
+NUMERIC_BLOCK_SIZE	1
+ENUM_VALUE_LIST	NULL
+READ_ONLY	NO
+COMMAND_LINE_ARGUMENT	REQUIRED
 VARIABLE_NAME	MAX_INSERT_DELAYED_THREADS
 VARIABLE_SCOPE	SESSION
 VARIABLE_TYPE	BIGINT UNSIGNED

sql/handle_connections_win.cc

@@ -28,6 +28,7 @@
 /* From mysqld.cc */
 extern HANDLE hEventShutdown;
 extern Dynamic_array<MYSQL_SOCKET> listen_sockets;
+extern Atomic_counter<uint> connection_count;
 #ifdef HAVE_POOL_OF_THREADS
 extern PTP_CALLBACK_ENVIRON get_threadpool_win_callback_environ();
 extern void tp_win_callback_prolog();
@@ -631,12 +632,28 @@ void handle_connections_win()
   {
     DBUG_ASSERT(wait_events.size() <= MAXIMUM_WAIT_OBJECTS);
     DWORD idx = WaitForMultipleObjects((DWORD)wait_events.size(),
-                                       wait_events.data(), FALSE, INFINITE);
-    DBUG_ASSERT((int)idx >= 0 && (int)idx < (int)wait_events.size());
+                                       wait_events.data(), FALSE,
+                                       max_idle_execution ?
+                                         max_idle_execution * 1000 : INFINITE);
+    DBUG_ASSERT(idx == WAIT_TIMEOUT ||
+                  ((int)idx >= 0 && (int)idx < (int)wait_events.size()));
 
     if (idx == SHUTDOWN_IDX)
       break;
 
+    if (max_idle_execution)
+    {
+      if (idx == WAIT_TIMEOUT)
+      {
+        if (handle_max_idle_execution_timeout())
+          break;
+
+        continue;
+      }
+      else
+        server_last_activity= microsecond_interval_timer();
+    }
+
     all_listeners[idx - LISTENER_START_IDX]->completion_callback();
   }
 

sql/mysqld.cc

@@ -481,6 +481,8 @@ ulong specialflag=0;
 ulong binlog_cache_use= 0, binlog_cache_disk_use= 0;
 ulong binlog_stmt_cache_use= 0, binlog_stmt_cache_disk_use= 0;
 ulong max_connections, max_connect_errors;
+uint max_idle_execution;
+Atomic_relaxed<ulonglong> server_last_activity;
 uint max_password_errors;
 ulong extra_max_connections;
 uint max_digest_length= 0;
@@ -4246,6 +4248,23 @@ static int init_common_variables()
     SYSVAR_AUTOSIZE(back_log, MY_MIN(900, (50 + max_connections / 5)));
   }
 
+  /*
+    max_idle_execution, defaults to 10mins under systemd socket activation,
+    otherwise 0 representing disabled.
+  */
+  if (IS_SYSVAR_AUTOSIZE(&max_idle_execution))
+    SYSVAR_AUTOSIZE(max_idle_execution, sd_listen_fds(0) ? 600 : 0);
+
+  if (IS_SYSVAR_AUTOSIZE(&global_system_variables.net_wait_timeout) &&
+      max_idle_execution &&
+      global_system_variables.net_wait_timeout > max_idle_execution)
+    SYSVAR_AUTOSIZE(global_system_variables.net_wait_timeout, max_idle_execution);
+
+  if (IS_SYSVAR_AUTOSIZE(&global_system_variables.net_interactive_timeout) &&
+      max_idle_execution &&
+      global_system_variables.net_interactive_timeout > max_idle_execution)
+    SYSVAR_AUTOSIZE(global_system_variables.net_interactive_timeout, max_idle_execution);
+
   unireg_init(opt_specialflag); /* Set up extern variabels */
   if (!(my_default_lc_messages=
         my_locale_by_name(lc_messages)))
@@ -6275,18 +6294,75 @@ static void set_non_blocking_if_supported(MYSQL_SOCKET sock)
 #endif
 }
 
+/*
+  Terminate the server if the server_last_activity exceeds the
+  max_idle_exection time
+  Returns true if a termination is progressing, false otherwise
+*/
 
-void handle_connections_sockets()
+bool handle_max_idle_execution_timeout()
+{
+  /*
+    The server_last_activity is set when one of the connection counts
+    is greater than 0, and just before it decrements. So the race condition
+    of getting all conditions here set because of memory re(ordering) effects
+    isn't possible.
+  */
+  if (connection_count == 0 && extra_connection_count == 0 &&
+      microsecond_interval_timer() > (server_last_activity
+        + max_idle_execution * 1000000UL))
+  {
+    sql_print_information("max_idle_execution time reached starting shutdown");
+    break_connect_loop();
+    return true;
+  }
+  return false;
+}
+
+
+static void handle_new_socket_connection(MYSQL_SOCKET sock)
 {
-  MYSQL_SOCKET sock= mysql_socket_invalid();
-  uint error_count=0;
   struct sockaddr_storage cAddr;
+  uint error_count= 0;
+
+  for (uint retry=0; retry < MAX_ACCEPT_RETRY && !abort_loop; retry++)
+  {
+    size_socket length= sizeof(struct sockaddr_storage);
+    MYSQL_SOCKET new_sock;
+
+    new_sock= mysql_socket_accept(key_socket_client_connection, sock,
+                                  (struct sockaddr *)(&cAddr),
+                                  &length);
+    if (mysql_socket_getfd(new_sock) != INVALID_SOCKET)
+      handle_accepted_socket(new_sock, sock);
+    else if (socket_errno != SOCKET_EINTR && socket_errno != SOCKET_EAGAIN)
+    {
+      /*
+        accept(2) failed on the listening port.
+        There is not much details to report about the client,
+        increment the server global status variable.
+      */
+      statistic_increment(connection_errors_accept, &LOCK_status);
+      if ((error_count++ & 255) == 0) // This can happen often
+        sql_perror("Error in accept");
+      if (socket_errno == SOCKET_ENFILE || socket_errno == SOCKET_EMFILE)
+        sleep(1); // Give other threads some time
+      break;
+    }
+  }
+}
+
+
+void handle_connections_sockets()
+{
+  MYSQL_SOCKET sock;
   int retval;
 #ifdef HAVE_POLL
   // for ip_sock, unix_sock and extra_ip_sock
   Dynamic_array<struct pollfd> fds(PSI_INSTRUMENT_MEM);
 #else
   fd_set readFDs,clientFDs;
+  struct timespec timeout;
 #endif
 
   DBUG_ENTER("handle_connections_sockets");
@@ -6311,6 +6387,7 @@ void handle_connections_sockets()
   }
 #endif
 
+  server_last_activity= microsecond_interval_timer();
   sd_notify(0, "READY=1\n"
             "STATUS=Taking your SQL requests now...\n");
 
@@ -6318,10 +6395,14 @@ void handle_connections_sockets()
   while (!abort_loop)
   {
 #ifdef HAVE_POLL
-    retval= poll(fds.get_pos(0), fds.size(), -1);
+    /* poll timeout in milliseconds */
+    retval= poll(fds.get_pos(0), fds.size(),
+                 max_idle_execution ? max_idle_execution * 1000 : -1);
 #else
+    timeout= { max_idle_execution, 0};
     readFDs=clientFDs;
-    retval= select(FD_SETSIZE, &readFDs, NULL, NULL, NULL);
+    retval= select(FD_SETSIZE, &readFDs, NULL, NULL,
+                   max_idle_execution ? &timeout : NULL);
 #endif
 
     if (retval < 0)
@@ -6344,50 +6425,26 @@ void handle_connections_sockets()
       break;
 
     /* Is this a new connection request ? */
-#ifdef HAVE_POLL
-    for (size_t i= 0; i < fds.size(); ++i)
+    sock= mysql_socket_invalid();
+    for (size_t i= 0; i < listen_sockets.size(); i++)
     {
+#ifdef HAVE_POLL
       if (fds.at(i).revents & POLLIN)
-      {
-        sock= listen_sockets.at(i);
-        break;
-      }
-    }
 #else  // HAVE_POLL
-    for (size_t i=0; i < listen_sockets.size(); i++)
-    {
       if (FD_ISSET(mysql_socket_getfd(listen_sockets.at(i)), &readFDs))
+#endif // HAVE_POLL
       {
         sock= listen_sockets.at(i);
-        break;
+        handle_new_socket_connection(sock);
       }
     }
-#endif // HAVE_POLL
-
-    for (uint retry=0; retry < MAX_ACCEPT_RETRY && !abort_loop; retry++)
+    /* timeout */
+    if (max_idle_execution)
     {
-      size_socket length= sizeof(struct sockaddr_storage);
-      MYSQL_SOCKET new_sock;
-
-      new_sock= mysql_socket_accept(key_socket_client_connection, sock,
-                                    (struct sockaddr *)(&cAddr),
-                                    &length);
-      if (mysql_socket_getfd(new_sock) != INVALID_SOCKET)
-        handle_accepted_socket(new_sock, sock);
-      else if (socket_errno != SOCKET_EINTR && socket_errno != SOCKET_EAGAIN)
-      {
-        /*
-          accept(2) failed on the listening port.
-          There is not much details to report about the client,
-          increment the server global status variable.
-        */
-        statistic_increment(connection_errors_accept, &LOCK_status);
-        if ((error_count++ & 255) == 0) // This can happen often
-          sql_perror("Error in accept");
-        if (socket_errno == SOCKET_ENFILE || socket_errno == SOCKET_EMFILE)
-          sleep(1); // Give other threads some time
-        break;
-      }
+      if (mysql_socket_getfd(sock) == INVALID_SOCKET)
+        handle_max_idle_execution_timeout();
+      else
+        server_last_activity= microsecond_interval_timer();
     }
   }
   sd_notify(0, "STOPPING=1\n"

sql/mysqld.h

@@ -74,6 +74,7 @@ enum enum_slave_parallel_mode {
 };
 
 /* Function prototypes */
+bool handle_max_idle_execution_timeout();
 void kill_mysql(THD *thd);
 void close_connection(THD *thd, uint sql_errno= 0);
 void handle_connection_in_main_thread(CONNECT *thd);
@@ -229,6 +230,7 @@ extern ulong slow_launch_threads, slow_launch_time;
 extern MYSQL_PLUGIN_IMPORT ulong max_connections;
 extern uint max_digest_length;
 extern ulong max_connect_errors, connect_timeout;
+extern uint max_idle_execution;
 extern uint max_password_errors;
 extern my_bool slave_allow_batching;
 extern my_bool allow_slave_start;

sql/sql_class.cc

@@ -2003,6 +2003,14 @@ void THD::disconnect()
     vio_close(net.vio);
   net.thd= 0;                                   // Don't collect statistics
 
+#ifndef EMBEDDED_LIBRARY
+  /* If we're tracking idle execution, and we're down to the last connection */
+  if (max_idle_execution && *scheduler->connection_count <= 1)
+  {
+    server_last_activity= utime_after_query;
+    handle_max_idle_execution_timeout();
+  }
+#endif
   mysql_mutex_unlock(&LOCK_thd_data);
 }
 

sql/sql_connect.h

@@ -22,7 +22,10 @@
 #include <mysql/psi/mysql_socket.h>
 #include <hash.h>
 #include "violite.h"
+#include "my_atomic_wrapper.h"
 
+
+extern Atomic_relaxed<ulonglong> server_last_activity;
 /*
   Object to hold connect information to be given to the newly created thread
 */

sql/sys_vars.cc

@@ -1365,7 +1365,7 @@ static Sys_var_ulong Sys_interactive_timeout(
        "interactive_timeout",
        "The number of seconds the server waits for activity on an interactive "
        "connection before closing it",
-       NO_SET_STMT SESSION_VAR(net_interactive_timeout),
+       NO_SET_STMT AUTO_SET SESSION_VAR(net_interactive_timeout),
        CMD_LINE(REQUIRED_ARG),
        VALID_RANGE(1, LONG_TIMEOUT), DEFAULT(NET_WAIT_TIMEOUT), BLOCK_SIZE(1));
 
@@ -1771,6 +1771,15 @@ Sys_max_connect_errors(
        VALID_RANGE(1, UINT_MAX), DEFAULT(MAX_CONNECT_ERRORS),
        BLOCK_SIZE(1));
 
+static Sys_var_uint Sys_max_idle_execution(
+       "max_idle_execution",
+       "If no new connections or running queries within this time (in seconds) "
+       "shutdown the server. Defaults to 10 minutes when started as a systemd "
+       "socket activated service. 0 represents disabled.",
+       AUTO_SET GLOBAL_VAR(max_idle_execution), CMD_LINE(REQUIRED_ARG),
+       VALID_RANGE(0, INT_MAX32 / 1000), DEFAULT(0),
+       BLOCK_SIZE(1));
+
 static Sys_var_on_access_global<Sys_var_uint,
                                 PRIV_SET_SYSTEM_GLOBAL_VAR_MAX_PASSWORD_ERRORS>
 Sys_max_password_errors(
@@ -4347,7 +4356,7 @@ static Sys_var_ulong Sys_net_wait_timeout(
        "wait_timeout",
        "The number of seconds the server waits for activity on a "
        "connection before closing it",
-       NO_SET_STMT SESSION_VAR(net_wait_timeout), CMD_LINE(REQUIRED_ARG),
+       NO_SET_STMT AUTO_SET SESSION_VAR(net_wait_timeout), CMD_LINE(REQUIRED_ARG),
        VALID_RANGE(1, IF_WIN(INT_MAX32/1000, LONG_TIMEOUT)),
        DEFAULT(NET_WAIT_TIMEOUT), BLOCK_SIZE(1));