Merge branch 'preview-10.8-MDEV-26713-Windows-i18-support' into 10.8

client/mysql.cc

@@ -88,9 +88,7 @@ extern "C" {
 #endif /* defined(HAVE_CURSES_H) && defined(HAVE_TERM_H) */
 
 #undef bcmp				// Fix problem with new readline
-#if defined(_WIN32)
-#include <conio.h>
-#else
+#if !defined(_WIN32)
 # ifdef __APPLE__
 #  include <editline/readline.h>
 # else
@@ -104,6 +102,98 @@ extern "C" {
 #define USE_POPEN
 }
 
+static CHARSET_INFO *charset_info= &my_charset_latin1;
+
+#if defined(_WIN32)
+/*
+  Set console mode for the whole duration of the client session.
+
+  We need for input
+    - line input (i.e read lines from console)
+    - echo typed characters
+    - "cooked" mode, i.e we do not want to handle all keystrokes,
+      like DEL etc ourselves, yet. We might want handle keystrokes
+      in the future, to implement tab completion, and better
+      (multiline) history.
+
+ Disable VT escapes for the output.We do not know what kind of escapes SELECT would return.
+*/
+struct Console_mode
+{
+  HANDLE in= GetStdHandle(STD_INPUT_HANDLE);
+  HANDLE out= GetStdHandle(STD_OUTPUT_HANDLE);
+  DWORD mode_in=0;
+  DWORD mode_out=0;
+
+  enum {STDIN_CHANGED = 1, STDOUT_CHANGED = 2};
+  int changes=0;
+
+  Console_mode()
+  {
+    if (in && in != INVALID_HANDLE_VALUE && GetConsoleMode(in, &mode_in))
+    {
+      SetConsoleMode(in, ENABLE_ECHO_INPUT|ENABLE_LINE_INPUT|ENABLE_PROCESSED_INPUT);
+      changes |= STDIN_CHANGED;
+    }
+
+    if (out && out != INVALID_HANDLE_VALUE && GetConsoleMode(out, &mode_out))
+    {
+#ifdef ENABLE_VIRTUAL_TERMINAL_INPUT
+      SetConsoleMode(out, mode_out & ~ENABLE_VIRTUAL_TERMINAL_INPUT);
+      changes |= STDOUT_CHANGED;
+#endif
+    }
+  }
+
+  ~Console_mode()
+  {
+    if (changes & STDIN_CHANGED)
+      SetConsoleMode(in, mode_in);
+
+    if(changes & STDOUT_CHANGED)
+      SetConsoleMode(out, mode_out);
+  }
+};
+
+static Console_mode my_conmode;
+
+#define MAX_CGETS_LINE_LEN 65535
+/** Read line from console, chomp EOL*/
+static char *win_readline()
+{
+  static wchar_t wstrbuf[MAX_CGETS_LINE_LEN];
+  static char strbuf[MAX_CGETS_LINE_LEN * 4];
+
+  DWORD nchars= 0;
+  uint len= 0;
+  SetLastError(0);
+  if (!ReadConsoleW(GetStdHandle(STD_INPUT_HANDLE), wstrbuf, MAX_CGETS_LINE_LEN-1,
+                    &nchars, NULL))
+    goto err;
+  if (nchars == 0 && GetLastError() == ERROR_OPERATION_ABORTED)
+    goto err;
+
+  for (;nchars > 0; nchars--)
+  {
+    if (wstrbuf[nchars - 1] != '\n' && wstrbuf[nchars - 1] != '\r')
+      break;
+  }
+
+  if (nchars > 0)
+  {
+    uint errors;
+    len= my_convert(strbuf, sizeof(strbuf), charset_info,
+                    (const char *) wstrbuf, nchars * sizeof(wchar_t),
+                    &my_charset_utf16le_bin, &errors);
+  }
+  strbuf[len]= 0;
+  return strbuf;
+err:
+  return NULL;
+}
+#endif
+
+
 #ifdef HAVE_VIDATTR
 static int have_curses= 0;
 static void my_vidattr(chtype attrs)
@@ -208,7 +298,6 @@ unsigned short terminal_width= 80;
 
 static uint opt_protocol=0;
 static const char *opt_protocol_type= "";
-static CHARSET_INFO *charset_info= &my_charset_latin1;
 
 static uint protocol_to_force= MYSQL_PROTOCOL_DEFAULT;
 
@@ -1353,6 +1442,46 @@ sig_handler mysql_end(int sig)
   exit(status.exit_status);
 }
 
+#ifdef _WIN32
+#define CNV_BUFSIZE 1024
+
+/**
+ Convert user,database,and password to requested charset.
+
+ This is done in the single case when user connects with non-UTF8
+ default-character-set, on UTF8 capable Windows.
+
+ User, password, and database are UTF8 encoded, prior to the function,
+ this needs to be fixed, in case they contain non-ASCIIs.
+
+ Mostly a workaround, to allow existng users with non-ASCII password
+ to survive upgrade without losing connectivity.
+*/
+static void maybe_convert_charset(const char **user, const char **password,
+                                  const char **database, const char *csname)
+{
+  if (GetACP() != CP_UTF8 || !strncmp(csname, "utf8", 4))
+    return;
+  static char bufs[3][CNV_BUFSIZE];
+  const char **from[]= {user, password, database};
+  CHARSET_INFO *cs= get_charset_by_csname(csname, MY_CS_PRIMARY,
+                                         MYF(MY_UTF8_IS_UTF8MB3 | MY_WME));
+  if (!cs)
+    return;
+  for (int i= 0; i < 3; i++)
+  {
+    const char *str= *from[i];
+    if (!str)
+      continue;
+    uint errors;
+    uint len= my_convert(bufs[i], CNV_BUFSIZE, cs, str, (uint32) strlen(str),
+                         &my_charset_utf8mb4_bin, &errors);
+    bufs[i][len]= 0;
+    *from[i]= bufs[i];
+  }
+}
+#endif
+
 /*
   set connection-specific options and call mysql_real_connect
 */
@@ -1384,6 +1513,10 @@ static bool do_connect(MYSQL *mysql, const char *host, const char *user,
   mysql_options(mysql, MYSQL_OPT_CONNECT_ATTR_RESET, 0);
   mysql_options4(mysql, MYSQL_OPT_CONNECT_ATTR_ADD,
                  "program_name", "mysql");
+#ifdef _WIN32
+  maybe_convert_charset(&user, &password, &database,default_charset);
+#endif
+
   return mysql_real_connect(mysql, host, user, password, database,
                             opt_mysql_port, opt_mysql_unix_port, flags);
 }
@@ -2033,11 +2166,6 @@ static int get_options(int argc, char **argv)
 
 static int read_and_execute(bool interactive)
 {
-#if defined(_WIN32)
-  String tmpbuf;
-  String buffer;
-#endif
-
   char	*line= NULL;
   char	in_string=0;
   ulong line_number=0;
@@ -2115,26 +2243,7 @@ static int read_and_execute(bool interactive)
 
 #if defined(_WIN32)
       tee_fputs(prompt, stdout);
-      if (!tmpbuf.is_alloced())
-        tmpbuf.alloc(65535);
-      tmpbuf.length(0);
-      buffer.length(0);
-      size_t clen;
-      do
-      {
-	line= my_cgets((char*)tmpbuf.ptr(), tmpbuf.alloced_length()-1, &clen);
-        buffer.append(line, clen);
-        /* 
-           if we got buffer fully filled than there is a chance that
-           something else is still in console input buffer
-        */
-      } while (tmpbuf.alloced_length() <= clen);
-      /* 
-        An empty line is returned from my_cgets when there's error reading :
-        Ctrl-c for example
-      */
-      if (line)
-        line= buffer.c_ptr();
+      line= win_readline();
 #else
       if (opt_outfile)
 	fputs(prompt, OUTFILE);
@@ -2201,10 +2310,7 @@ static int read_and_execute(bool interactive)
     }
   }
 
-#if defined(_WIN32)
-  buffer.free();
-  tmpbuf.free();
-#else
+#if !defined(_WIN32)
   if (interactive)
     /*
       free the last entered line.
@@ -3242,6 +3348,21 @@ com_clear(String *buffer,char *line __attribute__((unused)))
   return 0;
 }
 
+static void adjust_console_codepage(const char *name __attribute__((unused)))
+{
+#ifdef _WIN32
+  if (my_set_console_cp(name) < 0)
+  {
+    char buf[128];
+    snprintf(buf, sizeof(buf),
+      "WARNING: Could not determine Windows codepage for charset '%s',"
+      "continue using codepage %u", name, GetConsoleOutputCP());
+    put_info(buf, INFO_INFO);
+  }
+#endif
+}
+
+
 	/* ARGSUSED */
 static int
 com_charset(String *buffer __attribute__((unused)), char *line)
@@ -3263,6 +3384,7 @@ com_charset(String *buffer __attribute__((unused)), char *line)
     mysql_set_character_set(&mysql, charset_info->cs_name.str);
     default_charset= (char *)charset_info->cs_name.str;
     put_info("Charset changed", INFO_INFO);
+    adjust_console_codepage(charset_info->cs_name.str);
   }
   else put_info("Charset is not found", INFO_INFO);
   return 0;
@@ -4806,6 +4928,7 @@ sql_real_connect(char *host,char *database,char *user,char *password,
     put_info(buff, INFO_ERROR);
     return 1;
   }
+  adjust_console_codepage(charset_info->cs_name.str);
   connected=1;
 #ifndef EMBEDDED_LIBRARY
   mysql_options(&mysql, MYSQL_OPT_RECONNECT, &debug_info_flag);

client/mysqladmin.cc

@@ -438,6 +438,7 @@ int main(int argc,char *argv[])
     mysql_options(&mysql,MYSQL_OPT_PROTOCOL,(char*)&opt_protocol);
   if (!strcmp(default_charset,MYSQL_AUTODETECT_CHARSET_NAME))
     default_charset= (char *)my_default_csname();
+  my_set_console_cp(default_charset);
   mysql_options(&mysql, MYSQL_SET_CHARSET_NAME, default_charset);
   error_flags= (myf)(opt_nobeep ? 0 : ME_BELL);
 

client/mysqlcheck.c

@@ -503,6 +503,7 @@ static int get_options(int *argc, char ***argv)
     printf("Unsupported character set: %s\n", default_charset);
     DBUG_RETURN(1);
   }
+  my_set_console_cp(default_charset);
   if (*argc > 0 && opt_alldbs)
   {
     printf("You should give only options, no arguments at all, with option\n");

client/mysqlimport.c

@@ -525,6 +525,7 @@ static MYSQL *db_connect(char *host, char *database,
     mysql_options(mysql, MYSQL_DEFAULT_AUTH, opt_default_auth);
   if (!strcmp(default_charset,MYSQL_AUTODETECT_CHARSET_NAME))
     default_charset= (char *)my_default_csname();
+  my_set_console_cp(default_charset);
   mysql_options(mysql, MYSQL_SET_CHARSET_NAME, my_default_csname());
   mysql_options(mysql, MYSQL_OPT_CONNECT_ATTR_RESET, 0);
   mysql_options4(mysql, MYSQL_OPT_CONNECT_ATTR_ADD,

client/mysqlshow.c

@@ -147,6 +147,7 @@ int main(int argc, char **argv)
 
   if (!strcmp(default_charset,MYSQL_AUTODETECT_CHARSET_NAME))
     default_charset= (char *)my_default_csname();
+  my_set_console_cp(default_charset);
   mysql_options(&mysql, MYSQL_SET_CHARSET_NAME, default_charset);
 
   if (opt_plugin_dir && *opt_plugin_dir)

client/mysqltest.cc

@@ -3258,6 +3258,47 @@ static int replace(DYNAMIC_STRING *ds_str,
   return 0;
 }
 
+#ifdef _WIN32
+/**
+  Check if background execution of command was requested.
+  Like in Unix shell, we assume background execution of the last
+  character in command is a ampersand (we do not tokenize though)
+*/
+static bool is_background_command(const DYNAMIC_STRING *ds)
+{
+  for (size_t i= ds->length - 1; i > 1; i--)
+  {
+    char c= ds->str[i];
+    if (!isspace(c))
+      return (c == '&');
+  }
+  return false;
+}
+
+/**
+  Execute OS command in background. We assume that the last character
+  is ampersand, i.e is_background_command() returned
+*/
+#include <string>
+static int execute_in_background(char *cmd)
+{
+  STARTUPINFO s{};
+  PROCESS_INFORMATION pi{};
+  char *end= strrchr(cmd, '&');
+  DBUG_ASSERT(end);
+  *end =0;
+  std::string scmd("cmd /c ");
+  scmd.append(cmd);
+  BOOL ok=
+   CreateProcess(0, (char *)scmd.c_str(), 0, 0, 0, CREATE_NO_WINDOW, 0, 0, &s, &pi);
+  *end= '&';
+  if (!ok)
+    return (int) GetLastError();
+  CloseHandle(pi.hProcess);
+  CloseHandle(pi.hThread);
+  return 0;
+}
+#endif
 
 /*
   Execute given command.
@@ -3332,6 +3373,14 @@ void do_exec(struct st_command *command)
   DBUG_PRINT("info", ("Executing '%s' as '%s'",
                       command->first_argument, ds_cmd.str));
 
+#ifdef _WIN32
+  if (is_background_command(&ds_cmd))
+  {
+    error= execute_in_background(ds_cmd.str);
+    goto end;
+  }
+#endif
+
   if (!(res_file= my_popen(ds_cmd.str, "r")))
   {
     dynstr_free(&ds_cmd);
@@ -3358,7 +3407,9 @@ void do_exec(struct st_command *command)
     dynstr_append_sorted(&ds_res, &ds_sorted, 0);
     dynstr_free(&ds_sorted);
   }
-
+#ifdef _WIN32
+end:
+#endif
   if (error)
   {
     uint status= WEXITSTATUS(error);

cmake/os/Windows.cmake

@@ -60,8 +60,8 @@ ENDIF()
 
 ADD_DEFINITIONS(-D_CRT_SECURE_NO_DEPRECATE)
 ADD_DEFINITIONS(-D_WIN32_WINNT=0x0A00)
-# We do not want the windows.h macros min/max
-ADD_DEFINITIONS(-DNOMINMAX)
+# We do not want the windows.h , or winsvc.h macros min/max
+ADD_DEFINITIONS(-DNOMINMAX -DNOSERVICE)
 # Speed up build process excluding unused header files
 ADD_DEFINITIONS(-DWIN32_LEAN_AND_MEAN)
   

cmake/win_compatibility.manifest

@@ -19,4 +19,9 @@
 
     </application>
   </compatibility>
+  <application>
+    <windowsSettings>
+      <activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
+    </windowsSettings>
+  </application>
 </asmv1:assembly>

include/my_global.h

@@ -29,6 +29,14 @@
 #pragma GCC poison __WIN__
 #endif
 
+#if defined(_MSC_VER)
+/*
+  Following functions have bugs, when used with UTF-8 active codepage.
+  #include <winservice.h> will use the non-buggy wrappers
+*/
+#pragma deprecated("CreateServiceA", "OpenServiceA", "ChangeServiceConfigA")
+#endif
+
 /*
   InnoDB depends on some MySQL internals which other plugins should not
   need.  This is because of InnoDB's foreign key support, "safe" binlog

include/my_sys.h

@@ -1087,6 +1087,9 @@ extern char *get_tty_password(const char *opt_message);
 #define BACKSLASH_MBTAIL
 /* File system character set */
 extern CHARSET_INFO *fs_character_set(void);
+extern int my_set_console_cp(const char *name);
+#else
+#define my_set_console_cp(A) do {} while (0)
 #endif
 extern const char *my_default_csname(void);
 extern size_t escape_quotes_for_mysql(CHARSET_INFO *charset_info,
@@ -1098,13 +1101,6 @@ extern void thd_increment_bytes_sent(void *thd, size_t length);
 extern void thd_increment_bytes_received(void *thd, size_t length);
 extern void thd_increment_net_big_packet_count(void *thd, size_t length);
 
-#ifdef _WIN32
-
-/* implemented in my_conio.c */
-char* my_cgets(char *string, size_t clen, size_t* plen);
-
-#endif
-
 #include <mysql/psi/psi.h>
 
 #ifdef HAVE_PSI_INTERFACE

mysql-test/include/check_utf8_cli.inc

+# Check if utf8 can be used on the command line for  --exec
+# The real check is done in the suite.pm
+#

mysql-test/include/check_windows_admin.inc

+# Check if current user is Windows admin
+# Used for testing services with mysql_install_db.exe
+# Actual value is set by suite.pm

mysql-test/include/no_utf8_cli.inc

+# Check if utf8 can't be used on the command line for  --exec
+# The real check is done in the suite.pm
+#

mysql-test/main/charset_client_win.test

 --source include/windows.inc
+--source include/no_utf8_cli.inc
 --exec chcp 1257 > NUL && $MYSQL --default-character-set=auto -e "select @@character_set_client"

mysql-test/main/charset_client_win_utf8mb4.result

+@@character_set_client
+utf8mb4
+ERROR 1045 (28000): Access denied for user 'u'@'localhost' (using password: YES)
+2
+2
+DROP user u;

mysql-test/main/charset_client_win_utf8mb4.test

+--source include/windows.inc
+--source include/check_utf8_cli.inc
+--exec $MYSQL --default-character-set=auto -e "select @@character_set_client"
+
+# Test that a user with old, non-UTF8 password can still connect
+# by setting setting non-auto --default-character-set
+# This is important for backward compatibility
+
+# Emulate creating password in an interactive client session, with older clients
+# which communicates with the server using with something like cp850
+
+exec chcp 850 > NUL && echo CREATE USER 'u' IDENTIFIED by 'ü' | $MYSQL --default-character-set=cp850;
+
+
+# Can't connect with UTF8
+--error 1
+exec $MYSQL --default-character-set=auto --user=u --password=ü  -e "select 1" 2>&1;
+
+# Can connect with tweaked --default-character-set
+exec $MYSQL --default-character-set=cp850 --user=u --password=ü -e "select 2";
+DROP user u;
+

mysql-test/main/grant_not_windows.test → mysql-test/main/grant_utf8_cli.test

- # UTF8 parameters to mysql client do not work on Windows
---source include/not_windows.inc
 --source include/not_embedded.inc
+--source include/check_utf8_cli.inc
 
 #
 # Bug#21432 Database/Table name limited to 64 bytes, not chars, problems with multi-byte

mysql-test/main/mysql_install_db_win_utf8.result

+Running bootstrap
+Creating my.ini file
+Removing default user
+Allowing remote access for user root
+Setting root password
+Creation of the database was successful
+# Kill the server
+connect  con1,localhost,root,パスワード,mysql;
+SELECT @@datadir;
+@@datadir
+DATADIR/
+# Kill the server
+connection default;
+# restart

mysql-test/main/mysql_install_db_win_utf8.test

+--source include/windows.inc
+--source include/check_utf8_cli.inc
+
+# Create database  in tmp directory using mysql_install_db.exe,
+# and start server from this directory.
+let $ddir= $MYSQLTEST_VARDIR/tmp/датадир;
+--error 0,1
+rmdir $ddir;
+exec $MYSQL_INSTALL_DB_EXE --datadir=$ddir --password=パスワード -R;
+--source include/kill_mysqld.inc
+
+# Note "restart" via MTR does not work, if server's command line has 
+# non-ASCII characters used (or, characters outside of ANSI codepage).
+# This is a perl limitation, which is worked around in this test - 
+# the server started in background, via exec $MYSQLD
+
+--replace_result $MYSQLD MYSQLD $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+exec $MYSQLD  --defaults-file=$MYSQLTEST_VARDIR/my.cnf --defaults-group-suffix=.1 --datadir=$ddir --loose-innodb > NUL 2>&1 &;
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+--disable_reconnect
+
+connect (con1,localhost,root,パスワード,mysql);
+
+# Smoke test - check that we're actually using datadir
+# we've created (i.e restart_parameters worked)
+--replace_result  $ddir DATADIR
+SELECT @@datadir;
+# restart in the original datadir again
+--source include/kill_mysqld.inc
+rmdir $ddir;
+
+connection default;
+--source include/start_mysqld.inc
+

mysql-test/main/winservice.inc

+source include/check_windows_admin.inc;
+
+# The test uses return code from sc.exe utility, which are as follows
+let $ERROR_SERVICE_DOES_NOT_EXIST= 1060;
+let $ERROR_SERVICE_CANNOT_ACCEPT_CTRL=1061;# intermediate, during start or stop
+let $ERROR_SERVICE_NOT_ACTIVE=1062;# service stopped
+
+let $sc_exe= C:\Windows\System32\sc.exe;
+let $ddir= $MYSQLTEST_VARDIR/tmp/$datadir_name;
+let $service_name=$service_name_prefix$MASTER_MYPORT;
+
+
+error 0,1;
+rmdir $ddir;
+
+--disable_result_log
+error 0,$ERROR_SERVICE_DOES_NOT_EXIST;
+exec $sc_exe delete $service_name;
+--enable_result_log
+
+source include/kill_mysqld.inc;
+echo # run mysql_install_db with --service parameter;
+--disable_result_log
+exec $MYSQL_INSTALL_DB_EXE --datadir=$ddir --port=$MASTER_MYPORT --password=$password --service=$service_name -R;
+--enable_result_log
+
+echo # Start service;
+--disable_result_log
+exec $sc_exe start $service_name;
+--enable_result_log
+
+enable_reconnect;
+source include/wait_until_connected_again.inc;
+disable_reconnect;
+
+echo # Connect with root user password=$password;
+connect (con1,localhost,root,$password,mysql);
+
+# Smoke test - check that we're actually using datadir
+# we've created (i.e restart_parameters worked)
+replace_result $ddir DATADIR;
+select @@datadir;
+
+echo # Stop service and wait until it is down;
+
+# stop service
+--disable_result_log
+exec $sc_exe stop  $service_name;
+# Wait until stopped
+let $sys_errno=0;
+while($sys_errno != $ERROR_SERVICE_NOT_ACTIVE)
+{
+  --error 0,$ERROR_SERVICE_CANNOT_ACCEPT_CTRL,$ERROR_SERVICE_NOT_ACTIVE
+  exec $sc_exe stop $service_name;
+  if($sys_errno != $ERROR_SERVICE_NOT_ACTIVE)
+  {
+    --real_sleep 0.1
+  }
+}
+--enable_result_log
+
+echo # Delete service;
+let $sys_errno=0;
+--disable_result_log
+exec $sc_exe delete $service_name;
+--enable_result_log
+
+# Cleanup
+source include/wait_until_disconnected.inc;
+rmdir $ddir;
+
+#restart original server
+connection default;
+source include/start_mysqld.inc;
+

mysql-test/main/winservice_basic.result

+# Kill the server
+# run mysql_install_db with --service parameter
+# Start service
+# Connect with root user password=password
+connect  con1,localhost,root,$password,mysql;
+select @@datadir;
+@@datadir
+DATADIR/
+# Stop service and wait until it is down
+# Delete service
+connection default;
+# restart

mysql-test/main/winservice_basic.test

+source include/windows.inc;
+let $datadir_name=data;
+let $service_name_prefix=mariadb;
+let $password=password;
+source winservice.inc;

mysql-test/main/winservice_i18n.result

+# Kill the server
+# run mysql_install_db with --service parameter
+# Start service
+# Connect with root user password=パスワード
+connect  con1,localhost,root,$password,mysql;
+select @@datadir;
+@@datadir
+DATADIR/
+# Stop service and wait until it is down
+# Delete service
+connection default;
+# restart

mysql-test/main/winservice_i18n.test

+source include/windows.inc;
+source include/check_utf8_cli.inc;
+
+let $datadir_name=датадир;
+let $service_name_prefix=mariadb_sörvis;
+let $password=パスワード;
+source winservice.inc;

mysql-test/suite.pm

@@ -87,6 +87,31 @@ sub skip_combinations {
   $skip{'main/ssl_verify_ip.test'} = 'x509v3 support required'
     unless $openssl_ver ge "1.0.2";
 
+  sub utf8_command_line_ok() {
+   if (IS_WINDOWS) {
+     # Can use UTF8 on command line since Windows 10 1903 (10.0.18362)
+     # or if OS codepage is set to UTF8
+     my($os_name, $os_major, $os_minor, $os_build, $os_id) = Win32::GetOSVersion();
+     if($os_major lt 10){
+       return 0;
+     } elsif($os_major gt 10 or $os_minor gt 0 or $os_build ge 18362){
+       return 1;
+     } elsif(Win32::GetACP() eq 65001) {
+       return 1;
+     }
+     return 0;
+   }
+   return 1;
+  }
+
+  $skip{'include/check_utf8_cli.inc'} = 'No utf8 command line support'
+    unless utf8_command_line_ok();
+
+  $skip{'include/no_utf8_cli.inc'} = 'Not tested with utf8 command line support'
+    unless !utf8_command_line_ok();
+  
+  $skip{'include/check_windows_admin.inc'} = 'Requires admin privileges'
+    unless IS_WINDOWS and Win32::IsAdminUser();
 
   %skip;
 }

mysys/CMakeLists.txt

@@ -54,7 +54,6 @@ IF (WIN32)
     my_wincond.c
     my_winerr.c
     my_winfile.c
-    my_conio.c
     my_minidump.cc
     my_win_popen.cc)
 ENDIF()

mysys/charset.c

@@ -1209,30 +1209,17 @@ size_t escape_string_for_mysql(CHARSET_INFO *charset_info,
 
 
 #ifdef BACKSLASH_MBTAIL
-static CHARSET_INFO *fs_cset_cache= NULL;
-
 CHARSET_INFO *fs_character_set()
 {
-  if (!fs_cset_cache)
-  {
-    char buf[10]= "cp";
-    GetLocaleInfo(LOCALE_SYSTEM_DEFAULT, LOCALE_IDEFAULTANSICODEPAGE,
-                  buf+2, sizeof(buf)-3);
-    /*
-      We cannot call get_charset_by_name here
-      because fs_character_set() is executed before
-      LOCK_THD_charset mutex initialization, which
-      is used inside get_charset_by_name.
-      As we're now interested in cp932 only,
-      let's just detect it using strcmp().
-    */
-    fs_cset_cache= 
-                #ifdef HAVE_CHARSET_cp932
-                        !strcmp(buf, "cp932") ? &my_charset_cp932_japanese_ci : 
-                #endif
-                        &my_charset_bin;
-  }
-  return fs_cset_cache;
+  static CHARSET_INFO *fs_cset_cache;
+  if (fs_cset_cache)
+    return fs_cset_cache;
+#ifdef HAVE_CHARSET_cp932
+  else if (GetACP() == 932)
+    return fs_cset_cache= &my_charset_cp932_japanese_ci;
+#endif
+  else
+    return fs_cset_cache= &my_charset_bin;
 }
 #endif
 
@@ -1393,8 +1380,8 @@ static const MY_CSET_OS_NAME charsets[] =
 #ifdef UNCOMMENT_THIS_WHEN_WL_WL_4024_IS_DONE
   {"cp54936",        "gb18030",  my_cs_exact},
 #endif
-  {"cp65001",        "utf8",     my_cs_exact},
-
+  {"cp65001",        "utf8mb4",  my_cs_exact},
+  {"cp65001",        "utf8mb3",  my_cs_approx},
 #else /* not Windows */
 
   {"646",            "latin1",   my_cs_approx}, /* Default on Solaris */
@@ -1517,9 +1504,15 @@ const char* my_default_csname()
   const char* csname = NULL;
 #ifdef _WIN32
   char cpbuf[64];
-  int cp = GetConsoleCP();
-  if (cp == 0)
-    cp = GetACP();
+  UINT cp;
+  if (GetACP() == CP_UTF8)
+    cp= CP_UTF8;
+  else
+  {
+    cp= GetConsoleCP();
+    if (cp == 0)
+      cp= GetACP();
+  }
   snprintf(cpbuf, sizeof(cpbuf), "cp%d", (int)cp);
   csname = my_os_charset_to_mysql_charset(cpbuf);
 #elif defined(HAVE_SETLOCALE) && defined(HAVE_NL_LANGINFO)
@@ -1528,3 +1521,90 @@ const char* my_default_csname()
 #endif
   return csname ? csname : MYSQL_DEFAULT_CHARSET_NAME;
 }
+
+
+#ifdef _WIN32
+/**
+  Extract codepage number from "cpNNNN" string,
+  and check that this codepage is supported.
+
+  @return 0 - invalid codepage(or unsupported)
+          > 0 - valid codepage number.
+*/
+static UINT get_codepage(const char *s)
+{
+  UINT cp;
+  if (s[0] != 'c' || s[1] != 'p')
+  {
+    DBUG_ASSERT(0);
+    return 0;
+  }
+  cp= strtoul(s + 2, NULL, 10);
+  if (!IsValidCodePage(cp))
+  {
+    /*
+     Can happen also with documented CP, i.e 51936
+     Perhaps differs from one machine to another.
+    */
+    return 0;
+  }
+  return cp;
+}
+
+static UINT mysql_charset_to_codepage(const char *my_cs_name)
+{
+  const MY_CSET_OS_NAME *csp;
+  UINT cp=0,tmp;
+  for (csp= charsets; csp->os_name; csp++)
+  {
+    if (!strcasecmp(csp->my_name, my_cs_name))
+    {
+      switch (csp->param)
+      {
+      case my_cs_exact:
+        tmp= get_codepage(csp->os_name);
+        if (tmp)
+          return tmp;
+        break;
+      case my_cs_approx:
+        /*
+          don't return just yet, perhaps there is a better
+          (exact) match later.
+        */
+        if (!cp)
+          cp= get_codepage(csp->os_name);
+        continue;
+
+      default:
+        return 0;
+      }
+    }
+  }
+  return cp;
+}
+
+/** Set console codepage for MariaDB's charset name */
+int my_set_console_cp(const char *csname)
+{
+  UINT cp;
+  if (fileno(stdout) < 0 || !isatty(fileno(stdout)))
+    return 0;
+  cp= mysql_charset_to_codepage(csname);
+  if (!cp)
+  {
+    /* No compatible os charset.*/
+    return -1;
+  }
+
+  if (GetConsoleOutputCP() != cp && !SetConsoleOutputCP(cp))
+  {
+    return -1;
+  }
+
+  if (GetConsoleCP() != cp && !SetConsoleCP(cp))
+  {
+    return -1;
+  }
+  return 0;
+}
+#endif

mysys/get_password.c

@@ -62,35 +62,58 @@
 
 char *get_tty_password(const char *opt_message)
 {
-  char to[80];
-  char *pos=to,*end=to+sizeof(to)-1;
+  wchar_t wbuf[80];
+  char *to;
+  int to_len;
+  UINT cp;
+  wchar_t *pos=wbuf,*end=wbuf + array_elements(wbuf)-1;
   DBUG_ENTER("get_tty_password");
   _cputs(opt_message ? opt_message : "Enter password: ");
   for (;;)
   {
-    char tmp;
-    tmp=_getch();
-    if (tmp == '\b' || (int) tmp == 127)
+    int wc;
+    wc=_getwch();
+    if (wc == '\b' || wc == 127)
     {
-      if (pos != to)
+      if (pos != wbuf)
       {
-	_cputs("\b \b");
-	pos--;
-	continue;
+        _cputs("\b \b");
+        pos--;
+        continue;
       }
     }
-    if (tmp == '\n' || tmp == '\r' || tmp == 3)
+    if (wc == '\n' || wc == '\r' || wc == 3 || pos == end)
       break;
-    if (iscntrl(tmp) || pos == end)
+    if (iswcntrl(wc))
       continue;
-    _cputs("*");
-    *(pos++) = tmp;
+
+    /* Do not print '*' for half-unicode char(high surrogate)*/
+    if (wc < 0xD800 || wc > 0xDBFF)
+    {
+      _cputs("*");
+    }
+    *(pos++)= (wchar_t)wc;
   }
-  while (pos != to && isspace(pos[-1]) == ' ')
-    pos--;					/* Allow dummy space at end */
   *pos=0;
   _cputs("\n");
-  DBUG_RETURN(my_strdup(PSI_INSTRUMENT_ME, to,MYF(MY_FAE)));
+
+  /*
+    Allocate output string, and convert UTF16 password to output codepage.
+  */
+  cp= GetACP() == CP_UTF8 ? CP_UTF8 : GetConsoleCP();
+
+  if (!(to_len= WideCharToMultiByte(cp, 0, wbuf, -1, NULL, 0, NULL, NULL)))
+    DBUG_RETURN(NULL);
+
+  if (!(to= my_malloc(PSI_INSTRUMENT_ME, to_len, MYF(MY_FAE))))
+    DBUG_RETURN(NULL);
+
+  if (!WideCharToMultiByte(cp, 0, wbuf, -1, to, to_len, NULL, NULL))
+  {
+    my_free(to);
+    DBUG_RETURN(NULL);
+  }
+  DBUG_RETURN(to);
 }
 
 #else

mysys/my_conio.c

-/* Copyright (c) 2000, 2005, 2007 MySQL AB
-   Use is subject to license terms
-
-   This program is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation; version 2 of the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335  USA */
-
-
-#include "mysys_priv.h"
-
-#ifdef _WIN32
-
-static HANDLE my_coninpfh= 0;     /* console input */
-
-/*
-  functions my_pthread_auto_mutex_lock & my_pthread_auto_mutex_free
-  are experimental at this moment, they are intended to bring
-  ability of protecting code sections without necessity to explicitly
-  initialize synchronization object in one of threads
-
-  if found useful they are to be exported in mysys
-*/
-
-
-/*
-  int my_pthread_auto_mutex_lock(HANDLE* ph, const char* name, 
-                                 int id, int time)
-  NOTES
-    creates a mutex with given name and tries to lock it time msec.
-    mutex name is appended with id to allow system wide or process wide
-    locks. Handle to created mutex returned in ph argument.
-
-  RETURN
-    0	              thread owns mutex
-    <>0	            error
-*/
-
-static
-int my_pthread_auto_mutex_lock(HANDLE* ph, const char* name, int id, int time)
-{
-  DWORD res;
-  char tname[FN_REFLEN];
-  
-  sprintf(tname, "%s-%08X", name, id);
-  
-  *ph= CreateMutex(NULL, FALSE, tname);
-  if (*ph == NULL)
-    return GetLastError();
-
-  res= WaitForSingleObject(*ph, time);
-  
-  if (res == WAIT_TIMEOUT)
-    return ERROR_SEM_TIMEOUT;
-
-  if (res == WAIT_FAILED)
-    return GetLastError();
-
-  return 0;
-}
-
-/*
-  int my_pthread_auto_mutex_free(HANDLE* ph)
-
-  NOTES
-    releases a mutex.
-
-  RETURN
-    0	              thread released mutex
-    <>0	            error
-
-*/
-static
-int my_pthread_auto_mutex_free(HANDLE* ph)
-{
-  if (*ph)
-  {
-    ReleaseMutex(*ph);
-    CloseHandle(*ph);
-    *ph= NULL;
-  }
-
-  return 0;
-}
-
-
-#define pthread_auto_mutex_decl(name)                           \
-  HANDLE __h##name= NULL;
-
-#define pthread_auto_mutex_lock(name, proc, time)               \
-  my_pthread_auto_mutex_lock(&__h##name, #name, (proc), (time))
-
-#define pthread_auto_mutex_free(name)                           \
-  my_pthread_auto_mutex_free(&__h##name)
-
-
-/*
-  char* my_cgets()
-
-  NOTES
-    Replaces _cgets from libc to support input of more than 255 chars.
-    Reads from the console via ReadConsole into buffer which 
-    should be at least clen characters.
-    Actual length of string returned in plen.
-
-  WARNING
-    my_cgets() does NOT check the pushback character buffer (i.e., _chbuf).
-    Thus, my_cgets() will not return any character that is pushed back by 
-    the _ungetch() call.
-
-  RETURN
-    string pointer	ok
-    NULL	          Error
-
-*/
-
-char* my_cgets(char *buffer, size_t clen, size_t* plen)
-{
-  ULONG state;
-  char *result;
-  DWORD plen_res;
-  CONSOLE_SCREEN_BUFFER_INFO csbi;
-  
-  pthread_auto_mutex_decl(my_conio_cs);
- 
-  /* lock the console for the current process*/
-  if (pthread_auto_mutex_lock(my_conio_cs, GetCurrentProcessId(), INFINITE))
-  {
-    /* can not lock console */
-    pthread_auto_mutex_free(my_conio_cs);  
-    return NULL;
-  }
-
-  /* init console input */
-  if (my_coninpfh == 0)
-  {
-    /* same handle will be used until process termination */
-    my_coninpfh= CreateFile("CONIN$", GENERIC_READ | GENERIC_WRITE,
-                            FILE_SHARE_READ | FILE_SHARE_WRITE,
-                            NULL, OPEN_EXISTING, 0, NULL);
-  }
-
-  if (my_coninpfh == INVALID_HANDLE_VALUE) 
-  {
-    /* unlock the console */
-    pthread_auto_mutex_free(my_conio_cs);  
-    return(NULL);
-  }
-
-  GetConsoleMode((HANDLE)my_coninpfh, &state);
-  SetConsoleMode((HANDLE)my_coninpfh, ENABLE_LINE_INPUT | 
-                 ENABLE_PROCESSED_INPUT | ENABLE_ECHO_INPUT);
-
-  GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi);
-
-  /* 
-    there is no known way to determine allowed buffer size for input
-    though it is known it should not be more than 64K               
-    so we cut 64K and try first size of screen buffer               
-    if it is still to large we cut half of it and try again         
-    later we may want to cycle from MY_MIN(clen, 65535) to allowed size
-    with small decrement to determine exact allowed buffer           
-  */
-  clen= MY_MIN(clen, 65535);
-  do
-  {
-    clen= MY_MIN(clen, (size_t) csbi.dwSize.X*csbi.dwSize.Y);
-    if (!ReadConsole((HANDLE)my_coninpfh, (LPVOID)buffer, (DWORD) clen - 1, &plen_res,
-                     NULL))
-    {
-      result= NULL;
-      clen>>= 1;
-    }
-    else
-    {
-      result= buffer;
-      break;
-    }
-  }
-  while (GetLastError() == ERROR_NOT_ENOUGH_MEMORY);
-  *plen= plen_res;
-
-  /* We go here on error reading the string (Ctrl-C for example) */
-  if (!*plen)
-    result= NULL;                              /* purecov: inspected */
-
-  if (result != NULL)
-  {
-    if (*plen > 1 && buffer[*plen - 2] == '\r')
-    {
-      *plen= *plen - 2;
-    }
-    else 
-    {
-      if (*plen > 0 && buffer[*plen - 1] == '\r')
-      {
-        char tmp[3];
-        DWORD  tmplen= (DWORD)sizeof(tmp);
-
-        *plen= *plen - 1;
-        /* read /n left in the buffer */
-        ReadConsole((HANDLE)my_coninpfh, (LPVOID)tmp, tmplen, &tmplen, NULL);
-      }
-    }
-    buffer[*plen]= '\0';
-  }
-
-  SetConsoleMode((HANDLE)my_coninpfh, state);
-  /* unlock the console */
-  pthread_auto_mutex_free(my_conio_cs);  
-
-  return result;
-}
-
-#endif /* _WIN32 */

mysys/my_getopt.c

@@ -38,7 +38,7 @@ static double getopt_double(char *arg, const struct my_option *optp, int *err);
 static void init_variables(const struct my_option *, init_func_p);
 static void init_one_value(const struct my_option *, void *, longlong);
 static void fini_one_value(const struct my_option *, void *, longlong);
-static int setval(const struct my_option *, void *, char *, my_bool);
+static int setval(const struct my_option *, void *, char *, my_bool, const char *);
 static char *check_struct_option(char *cur_arg, char *key_name);
 
 /*
@@ -133,6 +133,45 @@ double getopt_ulonglong2double(ulonglong v)
   return u.dbl;
 }
 
+#ifdef _WIN32
+/**
+
+ On Windows, if program is running in UTF8 mode, but some arguments are not UTF8.
+
+ This will mostly likely be a sign of old "ANSI" my.ini, and it is likely that
+ something will go wrong, e.g file access error.
+*/
+static void validate_value(const char *key, const char *value,
+                           const char *filename)
+{
+  MY_STRCOPY_STATUS status;
+  const struct charset_info_st *cs= &my_charset_utf8mb4_bin;
+  size_t len;
+  if (GetACP() != CP_UTF8)
+    return;
+  if (!(len= strlen(value)))
+    return;
+  cs->cset->well_formed_char_length(cs, value, value + len, len, &status);
+  if (!status.m_well_formed_error_pos)
+    return;
+  if (filename && *filename)
+  {
+    my_getopt_error_reporter(WARNING_LEVEL,
+      "%s: invalid (non-UTF8) characters found for option '%s'"
+      " in file '%s'",
+      my_progname, key, filename);
+  }
+  else
+  {
+    my_getopt_error_reporter(
+    WARNING_LEVEL, "%s: invalid (non-UTF8) characters for option %s",
+      my_progname, key);
+  }
+}
+#else
+#define validate_value(key, value, filename) (void)filename
+#endif
+
 /**
   Handle command line options.
   Sort options.
@@ -564,7 +603,7 @@ int handle_options(int *argc, char ***argv, const struct my_option *longopts,
 		}
 	      }
 	      if ((error= setval(optp, optp->value, argument,
-				 set_maximum_value)))
+                             set_maximum_value,filename)))
 		DBUG_RETURN(error);
               if (get_one_option(optp, argument, filename))
                 DBUG_RETURN(EXIT_UNSPECIFIED_ERROR);
@@ -610,7 +649,7 @@ int handle_options(int *argc, char ***argv, const struct my_option *longopts,
 	continue;
       }
       if ((!option_is_autoset) &&
-	  ((error= setval(optp, value, argument, set_maximum_value))) &&
+        ((error= setval(optp, value, argument, set_maximum_value,filename))) &&
           !option_is_loose)
 	DBUG_RETURN(error);
       if (get_one_option(optp, argument, filename))
@@ -711,7 +750,7 @@ static my_bool get_bool_argument(const struct my_option *opts,
 */
 
 static int setval(const struct my_option *opts, void *value, char *argument,
-		  my_bool set_maximum_value)
+                  my_bool set_maximum_value, const char *option_file)
 {
   int err= 0, res= 0;
   DBUG_ENTER("setval");
@@ -858,6 +897,7 @@ static int setval(const struct my_option *opts, void *value, char *argument,
       goto ret;
     };
   }
+  validate_value(opts->name, argument, option_file);
   DBUG_RETURN(0);
 
 ret:

mysys/my_init.c

@@ -34,6 +34,7 @@
 #endif
 static void my_win_init(void);
 static my_bool win32_init_tcp_ip();
+static void setup_codepages();
 #else
 #define my_win_init()
 #endif
@@ -67,6 +68,69 @@ static ulong atoi_octal(const char *str)
 MYSQL_FILE *mysql_stdin= NULL;
 static MYSQL_FILE instrumented_stdin;
 
+#ifdef _WIN32
+static UINT orig_console_cp, orig_console_output_cp;
+
+static void reset_console_cp(void)
+{
+  /*
+    We try not to call SetConsoleCP unnecessarily, to workaround a bug on
+    older Windows 10 (1803), which could switch truetype console fonts to
+    raster, eventhough SetConsoleCP would be a no-op (switch from UTF8 to UTF8).
+  */
+  if (GetConsoleCP() != orig_console_cp)
+    SetConsoleCP(orig_console_cp);
+  if (GetConsoleOutputCP() != orig_console_output_cp)
+    SetConsoleOutputCP(orig_console_output_cp);
+}
+
+/*
+  The below fixes discrepancies in console output and
+  command line parameter encoding. command line is in
+  ANSI codepage, output to console by default is in OEM, but
+  we like them to be in the same encoding.
+
+  We do this only if current codepage is UTF8, i.e when we
+  know we're on Windows that can handle UTF8 well.
+*/
+static void setup_codepages()
+{
+  UINT acp;
+  BOOL is_a_tty= fileno(stdout) >= 0 && isatty(fileno(stdout));
+
+  if (is_a_tty)
+  {
+    /*
+      Save console codepages, in case we change them,
+      to restore them on exit.
+    */
+    orig_console_cp= GetConsoleCP();
+    orig_console_output_cp= GetConsoleOutputCP();
+    if (orig_console_cp && orig_console_output_cp)
+      atexit(reset_console_cp);
+  }
+
+  if ((acp= GetACP()) != CP_UTF8)
+    return;
+
+  /*
+    Use setlocale to make mbstowcs/mkdir/getcwd behave, see
+    https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setlocale-wsetlocale
+  */
+  setlocale(LC_ALL, "en_US.UTF8");
+
+  if (is_a_tty && (orig_console_cp != acp || orig_console_output_cp != acp))
+  {
+    /*
+      If ANSI codepage is UTF8, we actually want to switch console
+      to it as well.
+    */
+    SetConsoleCP(acp);
+    SetConsoleOutputCP(acp);
+  }
+}
+#endif
+
 /**
   Initialize my_sys functions, resources and variables
 
@@ -337,6 +401,17 @@ static void my_win_init(void)
 
   _tzset();
 
+  /*
+   We do not want text translation (LF->CRLF)
+   when stdout is console/terminal, it is buggy
+  */
+  if (fileno(stdout) >= 0 && isatty(fileno(stdout)))
+    (void)setmode(fileno(stdout), O_BINARY);
+
+  if (fileno(stderr) >= 0 && isatty(fileno(stderr)))
+    (void) setmode(fileno(stderr), O_BINARY);
+
+  setup_codepages();
   DBUG_VOID_RETURN;
 }
 

sql/CMakeLists.txt

@@ -482,10 +482,11 @@ IF(WIN32)
   MYSQL_ADD_EXECUTABLE(mariadb-install-db
     mysql_install_db.cc
     ${CMAKE_CURRENT_BINARY_DIR}/mysql_bootstrap_sql.c
+    password.c
     COMPONENT Server
   )
   SET_TARGET_PROPERTIES(mariadb-install-db PROPERTIES COMPILE_FLAGS -DINSTALL_PLUGINDIR=${INSTALL_PLUGINDIR})
-  TARGET_LINK_LIBRARIES(mariadb-install-db mysys shlwapi)
+  TARGET_LINK_LIBRARIES(mariadb-install-db mysys mysys_ssl shlwapi)
 
   ADD_LIBRARY(winservice STATIC winservice.c)
   TARGET_LINK_LIBRARIES(winservice shell32)

sql/mysql_install_db.cc

@@ -21,6 +21,7 @@
 #include "mariadb.h"
 #include <my_getopt.h>
 #include <m_string.h>
+#include <password.h>
 
 #include <windows.h>
 #include <shellapi.h>
@@ -30,6 +31,7 @@
 #include <sddl.h>
 struct IUnknown;
 #include <shlwapi.h>
+#include <winservice.h>
 
 #include <string>
 
@@ -442,16 +444,14 @@ static int create_myini()
 }
 
 
-static const char update_root_passwd_part1[]=
+static constexpr const char* update_root_passwd=
   "UPDATE mysql.global_priv SET priv=json_set(priv,"
   "'$.password_last_changed', UNIX_TIMESTAMP(),"
   "'$.plugin','mysql_native_password',"
-  "'$.authentication_string',PASSWORD(";
-static const char update_root_passwd_part2[]=
-  ")) where User='root';\n";
-static const char remove_default_user_cmd[]=
+  "'$.authentication_string','%s') where User='root';\n";
+static constexpr char remove_default_user_cmd[]=
   "DELETE FROM mysql.user where User='';\n";
-static const char allow_remote_root_access_cmd[]=
+static constexpr char allow_remote_root_access_cmd[]=
   "CREATE TEMPORARY TABLE tmp_user LIKE global_priv;\n"
   "INSERT INTO tmp_user SELECT * from global_priv where user='root' "
     " AND host='localhost';\n"
@@ -870,18 +870,10 @@ static int create_db_instance(const char *datadir)
   /* Change root password if requested. */
   if (opt_password && opt_password[0])
   {
-    verbose("Setting root password",remove_default_user_cmd);
-    fputs(update_root_passwd_part1, in);
-
-    /* Use hex encoding for password, to avoid escaping problems.*/
-    fputc('0', in);
-    fputc('x', in);
-    for(int i= 0; opt_password[i]; i++)
-    {
-      fprintf(in,"%02x",opt_password[i]);
-    }
-
-    fputs(update_root_passwd_part2, in);
+    verbose("Setting root password");
+    char buf[2 * MY_SHA1_HASH_SIZE + 2];
+    my_make_scrambled_password(buf, opt_password, strlen(opt_password));
+    fprintf(in, update_root_passwd, buf);
     fflush(in);
   }
 
@@ -916,7 +908,7 @@ static int create_db_instance(const char *datadir)
     auto sc_manager = OpenSCManager(NULL, NULL, SC_MANAGER_ALL_ACCESS);
     if (sc_manager)
     {
-      auto sc_handle= OpenServiceA(sc_manager,opt_service, DELETE);
+      auto sc_handle= OpenService(sc_manager,opt_service, DELETE);
       if (sc_handle)
       {
         DeleteService(sc_handle);

sql/mysql_upgrade_service.cc

@@ -374,13 +374,17 @@ static void change_service_config()
     Write datadir to my.ini, after converting  backslashes to 
     unix style slashes.
   */
-  strcpy_s(buf, MAX_PATH, service_properties.datadir);
-  for(i= 0; buf[i]; i++)
+  if (service_properties.datadir[0])
   {
-    if (buf[i] == '\\')
-      buf[i]= '/';
+    strcpy_s(buf, MAX_PATH, service_properties.datadir);
+    for (i= 0; buf[i]; i++)
+    {
+      if (buf[i] == '\\')
+        buf[i]= '/';
+    }
+    WritePrivateProfileString("mysqld", "datadir", buf,
+                              service_properties.inifile);
   }
-  WritePrivateProfileString("mysqld", "datadir",buf, service_properties.inifile);
 
   /*
     Remove basedir from defaults file, otherwise the service wont come up in 
@@ -465,13 +469,8 @@ int main(int argc, char **argv)
     }
   }
 
-  old_mysqld_exe_exists = (GetFileAttributes(service_properties.mysqld_exe) != INVALID_FILE_ATTRIBUTES);
-  log("Phase %d/%d: Fixing server config file%s", ++phase, max_phases, my_ini_exists ? "" : "(skipped)");
-
-  snprintf(my_ini_bck, sizeof(my_ini_bck), "%s.BCK", service_properties.inifile);
-  CopyFile(service_properties.inifile, my_ini_bck, FALSE);
-  upgrade_config_file(service_properties.inifile);
-
+  old_mysqld_exe_exists= (GetFileAttributes(service_properties.mysqld_exe) !=
+                          INVALID_FILE_ATTRIBUTES);
   bool do_start_stop_server = old_mysqld_exe_exists && initial_service_state != SERVICE_RUNNING;
 
   log("Phase %d/%d: Start and stop server in the old version, to avoid crash recovery %s", ++phase, max_phases,
@@ -526,6 +525,14 @@ int main(int argc, char **argv)
       start_duration_ms += 500;
     }
   }
+
+  log("Phase %d/%d: Fixing server config file%s", ++phase, max_phases,
+      my_ini_exists ? "" : "(skipped)");
+  snprintf(my_ini_bck, sizeof(my_ini_bck), "%s.BCK",
+           service_properties.inifile);
+  CopyFile(service_properties.inifile, my_ini_bck, FALSE);
+  upgrade_config_file(service_properties.inifile);
+
   /* 
     Start mysqld.exe as non-service skipping privileges (so we do not 
     care about the password). But disable networking and enable pipe 

sql/mysqld.cc

@@ -127,6 +127,7 @@
 #ifdef _WIN32
 #include <handle_connections_win.h>
 #include <sddl.h>
+#include <winservice.h> /* SERVICE_STOPPED, SERVICE_RUNNING etc */
 #endif
 
 #include <my_service_manager.h>

sql/upgrade_conf_file.cc

@@ -25,6 +25,8 @@
   Note : the list below only includes the default-compiled server and none of the
   loadable plugins.
 */
+#include <my_global.h>
+#include <my_sys.h>
 #include <windows.h>
 #include <initializer_list>
 #include <stdlib.h>
@@ -158,51 +160,159 @@ static int cmp_strings(const void* a, const void *b)
   return strcmp((const char *)a, *(const char **)b);
 }
 
-/**
-  Convert file from a previous version, by removing
-*/
-int upgrade_config_file(const char *myini_path)
+
+#define MY_INI_SECTION_SIZE 32 * 1024 + 3
+
+static bool is_utf8_str(const char *s)
+{
+  MY_STRCOPY_STATUS status;
+  const struct charset_info_st *cs= &my_charset_utf8mb4_bin;
+  size_t len= strlen(s);
+  if (!len)
+    return true;
+  cs->cset->well_formed_char_length(cs, s, s + len, len, &status);
+  return status.m_well_formed_error_pos == nullptr;
+}
+
+
+static UINT get_system_acp()
 {
-#define MY_INI_SECTION_SIZE 32*1024 +3
+  static DWORD system_acp;
+  if (system_acp)
+    return system_acp;
+
+  char str_cp[10];
+  int cch= GetLocaleInfo(GetSystemDefaultLCID(), LOCALE_IDEFAULTANSICODEPAGE,
+                         str_cp, sizeof(str_cp));
+
+  system_acp= cch > 0 ? atoi(str_cp) : 1252;
+
+  return system_acp;
+}
+
+
+static char *ansi_to_utf8(const char *s)
+{
+#define MAX_STR_LEN MY_INI_SECTION_SIZE
+  static wchar_t utf16_buf[MAX_STR_LEN];
+  static char utf8_buf[MAX_STR_LEN];
+  if (MultiByteToWideChar(get_system_acp(), 0, s, -1, utf16_buf, MAX_STR_LEN))
+  {
+    if (WideCharToMultiByte(CP_UTF8, 0, utf16_buf, -1, utf8_buf, MAX_STR_LEN,
+                            0, 0))
+      return utf8_buf;
+  }
+  return 0;
+}
+
+int fix_section(const char *myini_path, const char *section_name,
+                bool is_server)
+{
+  if (!is_server && GetACP() != CP_UTF8)
+    return 0;
+
   static char section_data[MY_INI_SECTION_SIZE];
-  for (const char *section_name : { "mysqld","server","mariadb" })
+  DWORD size= GetPrivateProfileSection(section_name, section_data,
+                                       MY_INI_SECTION_SIZE, myini_path);
+  if (size == MY_INI_SECTION_SIZE - 2)
   {
-    DWORD size = GetPrivateProfileSection(section_name, section_data, MY_INI_SECTION_SIZE, myini_path);
-    if (size == MY_INI_SECTION_SIZE - 2)
-    {
-      return -1;
-    }
+    return -1;
+  }
 
-    for (char *keyval = section_data; *keyval; keyval += strlen(keyval) + 1)
+  for (char *keyval= section_data; *keyval; keyval += strlen(keyval)+1)
+  {
+    char varname[256];
+    char *value;
+    char *key_end= strchr(keyval, '=');
+    if (!key_end)
+      key_end= keyval + strlen(keyval);
+
+    if (key_end - keyval > sizeof(varname))
+      continue;
+
+    value= key_end + 1;
+    if (GetACP() == CP_UTF8 && !is_utf8_str(value))
     {
-      char varname[256];
-      char *key_end = strchr(keyval, '=');
-      if (!key_end)
-        key_end = keyval+ strlen(keyval);
-
-      if (key_end - keyval > sizeof(varname))
-        continue;
-      // copy and normalize (convert dash to underscore) to  variable names
-      for (char *p = keyval, *q = varname;; p++,q++)
+      /*Convert a value, if it is not already UTF-8*/
+      char *new_val= ansi_to_utf8(value);
+      if (new_val)
       {
-        if (p == key_end)
-        {
-          *q = 0;
-          break;
-        }
-        *q = (*p == '-') ? '_' : *p;
+        *key_end= 0;
+        fprintf(stdout, "Fixing variable '%s' charset, value=%s\n", keyval,
+                new_val);
+        WritePrivateProfileString(section_name, keyval, new_val, myini_path);
+        *key_end= '=';
       }
-      const char *v = (const char *)bsearch(varname, removed_variables, sizeof(removed_variables) / sizeof(removed_variables[0]),
-        sizeof(char *), cmp_strings);
+    }
+    if (!is_server)
+      continue;
 
-      if (v)
+    // Check if variable should be removed from config.
+    // First, copy and normalize (convert dash to underscore) to  variable
+    // names
+    for (char *p= keyval, *q= varname;; p++, q++)
+    {
+      if (p == key_end)
       {
-        fprintf(stdout, "Removing variable '%s' from config file\n", varname);
-        // delete variable
-        *key_end = 0;
-        WritePrivateProfileString(section_name, keyval, 0, myini_path);
+        *q= 0;
+        break;
       }
+      *q= (*p == '-') ? '_' : *p;
+    }
+    const char *v= (const char *) bsearch(varname, removed_variables, sizeof(removed_variables) / sizeof(removed_variables[0]),
+                                          sizeof(char *), cmp_strings);
+
+    if (v)
+    {
+      fprintf(stdout, "Removing variable '%s' from config file\n", varname);
+      // delete variable
+      *key_end= 0;
+      WritePrivateProfileString(section_name, keyval, 0, myini_path);
     }
   }
   return 0;
 }
+
+static bool is_mariadb_section(const char *name, bool *is_server)
+{
+  if (strncmp(name, "mysql", 5)
+      && strncmp(name, "mariadb", 7)
+      && strcmp(name, "client")
+      && strcmp(name, "client-server")
+      && strcmp(name, "server"))
+  {
+    return false;
+  }
+
+  for (const char *section_name : {"mysqld", "server", "mariadb"})
+    if (*is_server= !strcmp(section_name, name))
+      break;
+
+  return true;
+}
+
+
+/**
+  Convert file from a previous version, by removing obsolete variables
+  Also, fix values to be UTF8, if MariaDB is running in utf8 mode
+*/
+int upgrade_config_file(const char *myini_path)
+{
+  static char all_sections[MY_INI_SECTION_SIZE];
+  int sz= GetPrivateProfileSectionNamesA(all_sections, MY_INI_SECTION_SIZE,
+                                         myini_path);
+  if (!sz)
+    return 0;
+  if (sz > MY_INI_SECTION_SIZE - 2)
+  {
+    fprintf(stderr, "Too many sections in config file\n");
+    return -1;
+  }
+  for (char *section= all_sections; *section; section+= strlen(section) + 1)
+  {
+    bool is_server_section;
+    if (is_mariadb_section(section, &is_server_section))
+      fix_section(myini_path, section, is_server_section);
+  }
+  return 0;
+}

sql/winmain.cc

@@ -55,6 +55,7 @@
 #include <windows.h>
 #include <string>
 #include <cassert>
+#include <winservice.h>
 
 static SERVICE_STATUS svc_status{SERVICE_WIN32_OWN_PROCESS};
 static SERVICE_STATUS_HANDLE svc_status_handle;

sql/winservice.c

@@ -134,6 +134,20 @@ static void get_datadir_from_ini(const char *ini, char *service_name, char *data
 }
 
 
+static int fix_and_check_datadir(mysqld_service_properties *props)
+{
+  normalize_path(props->datadir, MAX_PATH);
+  /* Check if datadir really exists */
+  if (GetFileAttributes(props->datadir) != INVALID_FILE_ATTRIBUTES)
+    return 0;
+  /*
+    It is possible, that datadir contains some unconvertable character.
+    We just pretend not to know what's the data directory
+  */
+  props->datadir[0]= 0;
+  return 0;
+}
+
 /*
   Retrieve some properties from windows mysqld service binary path.
   We're interested in ini file location and datadir, and also in version of
@@ -284,16 +298,9 @@ int get_mysql_service_properties(const wchar_t *bin_path,
     }
   }
 
-  if (props->datadir[0])
-  {
-    normalize_path(props->datadir, MAX_PATH);
-    /* Check if datadir really exists */
-    if (GetFileAttributes(props->datadir) == INVALID_FILE_ATTRIBUTES)
-      goto end;
-  }
-  else
+  if (props->datadir[0] == 0 || fix_and_check_datadir(props))
   {
-    /* There is no datadir in ini file,  bail out.*/
+    /* There is no datadir in ini file, or non-existing dir, bail out.*/
     goto end;
   }
 

sql/winservice.h

@@ -17,11 +17,22 @@
 /*
   Extract properties of a windows service binary path
 */
+#pragma once
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include <windows.h> 
+#include <windows.h>
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable : 4995)
+#endif
+#include <winsvc.h>
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
 typedef struct mysqld_service_properties_st
 {
   char mysqld_exe[MAX_PATH];
@@ -32,9 +43,171 @@ typedef struct mysqld_service_properties_st
   int  version_patch;
 } mysqld_service_properties;
 
-extern int get_mysql_service_properties(const wchar_t *bin_path, 
+extern int get_mysql_service_properties(const wchar_t *bin_path,
   mysqld_service_properties *props);
 
+
+#if !defined(UNICODE)
+/*
+  The following wrappers workaround Windows bugs
+  with CreateService/OpenService with ANSI codepage UTF8.
+
+  Apparently, these function in ANSI mode, for this codepage only
+  do *not* behave as expected (as-if string parameters were
+  converted to UTF16 and "wide" function were called)
+*/
+#include <malloc.h>
+static inline wchar_t* awstrdup(const char *str)
+{
+  if (!str)
+    return NULL;
+  size_t len= strlen(str) + 1;
+  wchar_t *wstr= (wchar_t *) malloc(sizeof(wchar_t)*len);
+  if (MultiByteToWideChar(GetACP(), 0, str, (int)len, wstr, (int)len) == 0)
+  {
+    free(wstr);
+    return NULL;
+  }
+  return wstr;
+}
+
+#define AWSTRDUP(dest, src)                                                   \
+  dest= awstrdup(src);                                                        \
+  if (src && !dest)                                                           \
+  {                                                                           \
+    ok= FALSE;                                                                \
+    last_error = ERROR_OUTOFMEMORY;                                           \
+    goto end;                                                                 \
+  }
+
+static inline SC_HANDLE my_OpenService(SC_HANDLE hSCManager, LPCSTR lpServiceName, DWORD dwDesiredAccess)
+{
+  wchar_t *w_ServiceName= NULL;
+  BOOL ok=TRUE;
+  DWORD last_error=0;
+  SC_HANDLE sch=NULL;
+
+  AWSTRDUP(w_ServiceName, lpServiceName);
+  sch= OpenServiceW(hSCManager, w_ServiceName, dwDesiredAccess);
+  if (!sch)
+  {
+    ok= FALSE;
+    last_error= GetLastError();
+  }
+
+end:
+  free(w_ServiceName);
+  if (!ok)
+    SetLastError(last_error);
+  return sch;
+}
+
+static inline SC_HANDLE my_CreateService(SC_HANDLE hSCManager,
+  LPCSTR lpServiceName, LPCSTR lpDisplayName,
+  DWORD dwDesiredAccess,  DWORD dwServiceType,
+  DWORD dwStartType, DWORD dwErrorControl,
+  LPCSTR lpBinaryPathName, LPCSTR lpLoadOrderGroup,
+  LPDWORD lpdwTagId, LPCSTR lpDependencies,
+  LPCSTR lpServiceStartName, LPCSTR lpPassword)
+{
+  wchar_t *w_ServiceName= NULL;
+  wchar_t *w_DisplayName= NULL;
+  wchar_t *w_BinaryPathName= NULL;
+  wchar_t *w_LoadOrderGroup= NULL;
+  wchar_t *w_Dependencies= NULL;
+  wchar_t *w_ServiceStartName= NULL;
+  wchar_t *w_Password= NULL;
+  SC_HANDLE sch = NULL;
+  DWORD last_error=0;
+  BOOL ok= TRUE;
+
+  AWSTRDUP(w_ServiceName,lpServiceName);
+  AWSTRDUP(w_DisplayName,lpDisplayName);
+  AWSTRDUP(w_BinaryPathName, lpBinaryPathName);
+  AWSTRDUP(w_LoadOrderGroup, lpLoadOrderGroup);
+  AWSTRDUP(w_Dependencies, lpDependencies);
+  AWSTRDUP(w_ServiceStartName, lpServiceStartName);
+  AWSTRDUP(w_Password, lpPassword);
+
+  sch= CreateServiceW(
+      hSCManager, w_ServiceName, w_DisplayName, dwDesiredAccess, dwServiceType,
+      dwStartType, dwErrorControl, w_BinaryPathName, w_LoadOrderGroup,
+      lpdwTagId, w_Dependencies, w_ServiceStartName, w_Password);
+  if(!sch)
+  {
+    ok= FALSE;
+    last_error= GetLastError();
+  }
+
+end:
+  free(w_ServiceName);
+  free(w_DisplayName);
+  free(w_BinaryPathName);
+  free(w_LoadOrderGroup);
+  free(w_Dependencies);
+  free(w_ServiceStartName);
+  free(w_Password);
+
+  if (!ok)
+    SetLastError(last_error);
+  return sch;
+}
+
+static inline BOOL my_ChangeServiceConfig(SC_HANDLE hService, DWORD dwServiceType,
+   DWORD dwStartType, DWORD dwErrorControl,
+   LPCSTR lpBinaryPathName, LPCSTR lpLoadOrderGroup,
+   LPDWORD lpdwTagId, LPCSTR lpDependencies,
+   LPCSTR lpServiceStartName, LPCSTR lpPassword,
+   LPCSTR lpDisplayName)
+{
+  wchar_t *w_DisplayName= NULL;
+  wchar_t *w_BinaryPathName= NULL;
+  wchar_t *w_LoadOrderGroup= NULL;
+  wchar_t *w_Dependencies= NULL;
+  wchar_t *w_ServiceStartName= NULL;
+  wchar_t *w_Password= NULL;
+  SC_HANDLE sch = NULL;
+  DWORD last_error=0;
+  BOOL ok= TRUE;
+
+  AWSTRDUP(w_DisplayName, lpDisplayName);
+  AWSTRDUP(w_BinaryPathName, lpBinaryPathName);
+  AWSTRDUP(w_LoadOrderGroup, lpLoadOrderGroup);
+  AWSTRDUP(w_Dependencies, lpDependencies);
+  AWSTRDUP(w_ServiceStartName, lpServiceStartName);
+  AWSTRDUP(w_Password, lpPassword);
+
+  ok= ChangeServiceConfigW(
+      hService, dwServiceType, dwStartType, dwErrorControl, w_BinaryPathName,
+      w_LoadOrderGroup, lpdwTagId, w_Dependencies, w_ServiceStartName,
+      w_Password, w_DisplayName);
+  if (!ok)
+  {
+    last_error= GetLastError();
+  }
+
+end:
+  free(w_DisplayName);
+  free(w_BinaryPathName);
+  free(w_LoadOrderGroup);
+  free(w_Dependencies);
+  free(w_ServiceStartName);
+  free(w_Password);
+
+  if (last_error)
+    SetLastError(last_error);
+  return ok;
+}
+#undef AWSTRDUP
+
+#undef OpenService
+#define OpenService my_OpenService
+#undef ChangeServiceConfig
+#define ChangeServiceConfig my_ChangeServiceConfig
+#undef CreateService
+#define CreateService my_CreateService
+#endif
+
 #ifdef __cplusplus
 }
 #endif

win/packaging/extra.wxs.in

@@ -464,7 +464,7 @@
             Section="mysqld"
             Name="my.ini"
             Key="character-set-server"
-            Value="utf8" />
+            Value="utf8mb4" />
       </Component>
 
       <!-- Shortcuts in program menu (mysql client etc) -->

win/upgrade_wizard/CMakeLists.txt

@@ -5,6 +5,7 @@ ENDIF()
 # We need MFC
 # /permissive- flag does not play well with MFC, disable it.
 STRING(REPLACE "/permissive-"  "" CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS}")
+REMOVE_DEFINITIONS(-DNOSERVICE) # fixes "already defined" warning in an AFX header
 
 FIND_PACKAGE(MFC)
 IF(NOT MFC_FOUND)

win/upgrade_wizard/upgradeDlg.cpp

@@ -15,6 +15,7 @@
 #include <vector>
 
 #include <winservice.h>
+#include <locale.h>
 
 using namespace std;
 
@@ -141,24 +142,24 @@ void CUpgradeDlg::PopulateServicesList()
     ErrorExit("OpenSCManager failed");
   }
 
-  static BYTE buf[64*1024];
+  static BYTE buf[2*64*1024];
   static BYTE configBuffer[8*1024];
 
   DWORD bufsize= sizeof(buf);
   DWORD bufneed;
   DWORD num_services;
-  BOOL ok= EnumServicesStatusEx(scm, SC_ENUM_PROCESS_INFO,  SERVICE_WIN32,
+  BOOL ok= EnumServicesStatusExW(scm, SC_ENUM_PROCESS_INFO,  SERVICE_WIN32,
     SERVICE_STATE_ALL,  buf, bufsize,  &bufneed, &num_services, NULL, NULL);
   if(!ok) 
     ErrorExit("EnumServicesStatusEx failed");
 
 
-  LPENUM_SERVICE_STATUS_PROCESS info =
-    (LPENUM_SERVICE_STATUS_PROCESS)buf;
+  LPENUM_SERVICE_STATUS_PROCESSW info =
+    (LPENUM_SERVICE_STATUS_PROCESSW)buf;
   int index=-1;
   for (ULONG i=0; i < num_services; i++)
   {
-    SC_HANDLE service= OpenService(scm, info[i].lpServiceName, 
+    SC_HANDLE service= OpenServiceW(scm, info[i].lpServiceName,
       SERVICE_QUERY_CONFIG);
     if (!service)
       continue;
@@ -187,7 +188,11 @@ void CUpgradeDlg::PopulateServicesList()
         ServiceProperties props;
         props.myini= service_props.inifile;
         props.datadir= service_props.datadir;
-        props.servicename = info[i].lpServiceName;
+        char service_name_buf[1024];
+        WideCharToMultiByte(GetACP(), 0, info[i].lpServiceName, -1,
+                            service_name_buf, sizeof(service_name_buf),
+                            0, 0);
+        props.servicename= service_name_buf;
         if (service_props.version_major)
         {
           char ver[64];
@@ -198,7 +203,7 @@ void CUpgradeDlg::PopulateServicesList()
         else
           props.version= "<unknown>";
 
-        index = m_Services.AddString(info[i].lpServiceName);
+        index = m_Services.AddString(service_name_buf);
         services.resize(index+1);
         services[index] = props;
       }
@@ -267,6 +272,11 @@ BOOL CUpgradeDlg::OnInitDialog()
 
   m_Progress.ShowWindow(SW_HIDE);
   m_Ok.EnableWindow(FALSE);
+  if (GetACP() == CP_UTF8)
+  {
+    /* Required for mbstowcs, used in some functions.*/
+    setlocale(LC_ALL, "en_US.UTF8");
+  }
   PopulateServicesList();
   return TRUE;  // return TRUE  unless you set the focus to a control
 }