MDEV-27090 Windows client - ReadConsoleA does not work correctly with UTF8 codepage

Corresponding Windows bug  https://github.com/microsoft/terminal/issues/4551

Use ReadConsoleW instead and convert to console's input codepage, to
workaround.

Also, disable VT sequences in the console output, as we do not knows what
type of data comes with SELECT, we do not want VT escapes there.

Remove my_cgets()

client/mysql.cc

@@ -88,9 +88,7 @@ extern "C" {
 #endif /* defined(HAVE_CURSES_H) && defined(HAVE_TERM_H) */
 
 #undef bcmp				// Fix problem with new readline
-#if defined(_WIN32)
-#include <conio.h>
-#else
+#if !defined(_WIN32)
 # ifdef __APPLE__
 #  include <editline/readline.h>
 # else
@@ -104,6 +102,98 @@ extern "C" {
 #endif
 }
 
+static CHARSET_INFO *charset_info= &my_charset_latin1;
+
+#if defined(_WIN32)
+/*
+  Set console mode for the whole duration of the client session.
+
+  We need for input
+    - line input (i.e read lines from console)
+    - echo typed characters
+    - "cooked" mode, i.e we do not want to handle all keystrokes,
+      like DEL etc ourselves, yet. We might want handle keystrokes
+      in the future, to implement tab completion, and better
+      (multiline) history.
+
+ Disable VT escapes for the output.We do not know what kind of escapes SELECT would return.
+*/
+struct Console_mode
+{
+  HANDLE in= GetStdHandle(STD_INPUT_HANDLE);
+  HANDLE out= GetStdHandle(STD_OUTPUT_HANDLE);
+  DWORD mode_in=0;
+  DWORD mode_out=0;
+
+  enum {STDIN_CHANGED = 1, STDOUT_CHANGED = 2};
+  int changes=0;
+
+  Console_mode()
+  {
+    if (in && in != INVALID_HANDLE_VALUE && GetConsoleMode(in, &mode_in))
+    {
+      SetConsoleMode(in, ENABLE_ECHO_INPUT|ENABLE_LINE_INPUT|ENABLE_PROCESSED_INPUT);
+      changes |= STDIN_CHANGED;
+    }
+
+    if (out && out != INVALID_HANDLE_VALUE && GetConsoleMode(out, &mode_out))
+    {
+#ifdef ENABLE_VIRTUAL_TERMINAL_INPUT
+      SetConsoleMode(out, mode_out & ~ENABLE_VIRTUAL_TERMINAL_INPUT);
+      changes |= STDOUT_CHANGED;
+#endif
+    }
+  }
+
+  ~Console_mode()
+  {
+    if (changes & STDIN_CHANGED)
+      SetConsoleMode(in, mode_in);
+
+    if(changes & STDOUT_CHANGED)
+      SetConsoleMode(out, mode_out);
+  }
+};
+
+static Console_mode my_conmode;
+
+#define MAX_CGETS_LINE_LEN 65535
+/** Read line from console, chomp EOL*/
+static char *win_readline()
+{
+  static wchar_t wstrbuf[MAX_CGETS_LINE_LEN];
+  static char strbuf[MAX_CGETS_LINE_LEN * 4];
+
+  DWORD nchars= 0;
+  uint len= 0;
+  SetLastError(0);
+  if (!ReadConsoleW(GetStdHandle(STD_INPUT_HANDLE), wstrbuf, MAX_CGETS_LINE_LEN-1,
+                    &nchars, NULL))
+    goto err;
+  if (nchars == 0 && GetLastError() == ERROR_OPERATION_ABORTED)
+    goto err;
+
+  for (;nchars > 0; nchars--)
+  {
+    if (wstrbuf[nchars - 1] != '\n' && wstrbuf[nchars - 1] != '\r')
+      break;
+  }
+
+  if (nchars > 0)
+  {
+    uint errors;
+    len= my_convert(strbuf, sizeof(strbuf), charset_info,
+                    (const char *) wstrbuf, nchars * sizeof(wchar_t),
+                    &my_charset_utf16le_bin, &errors);
+  }
+  strbuf[len]= 0;
+  return strbuf;
+err:
+  return NULL;
+}
+#endif
+
+
 #ifdef HAVE_VIDATTR
 static int have_curses= 0;
 static void my_vidattr(chtype attrs)
@@ -208,7 +298,6 @@ unsigned short terminal_width= 80;
 
 static uint opt_protocol=0;
 static const char *opt_protocol_type= "";
-static CHARSET_INFO *charset_info= &my_charset_latin1;
 
 static uint protocol_to_force= MYSQL_PROTOCOL_DEFAULT;
 
@@ -2033,11 +2122,6 @@ static int get_options(int argc, char **argv)
 
 static int read_and_execute(bool interactive)
 {
-#if defined(_WIN32)
-  String tmpbuf;
-  String buffer;
-#endif
-
   char	*line= NULL;
   char	in_string=0;
   ulong line_number=0;
@@ -2115,26 +2199,7 @@ static int read_and_execute(bool interactive)
 
 #if defined(_WIN32)
       tee_fputs(prompt, stdout);
-      if (!tmpbuf.is_alloced())
-        tmpbuf.alloc(65535);
-      tmpbuf.length(0);
-      buffer.length(0);
-      size_t clen;
-      do
-      {
-	line= my_cgets((char*)tmpbuf.ptr(), tmpbuf.alloced_length()-1, &clen);
-        buffer.append(line, clen);
-        /* 
-           if we got buffer fully filled than there is a chance that
-           something else is still in console input buffer
-        */
-      } while (tmpbuf.alloced_length() <= clen);
-      /* 
-        An empty line is returned from my_cgets when there's error reading :
-        Ctrl-c for example
-      */
-      if (line)
-        line= buffer.c_ptr();
+      line= win_readline();
 #else
       if (opt_outfile)
 	fputs(prompt, OUTFILE);
@@ -2201,10 +2266,7 @@ static int read_and_execute(bool interactive)
     }
   }
 
-#if defined(_WIN32)
-  buffer.free();
-  tmpbuf.free();
-#else
+#if !defined(_WIN32)
   if (interactive)
     /*
       free the last entered line.

include/my_sys.h

@@ -1097,13 +1097,6 @@ extern void thd_increment_bytes_sent(void *thd, size_t length);
 extern void thd_increment_bytes_received(void *thd, size_t length);
 extern void thd_increment_net_big_packet_count(void *thd, size_t length);
 
-#ifdef _WIN32
-
-/* implemented in my_conio.c */
-char* my_cgets(char *string, size_t clen, size_t* plen);
-
-#endif
-
 #include <mysql/psi/psi.h>
 
 #ifdef HAVE_PSI_INTERFACE

mysys/CMakeLists.txt

@@ -54,7 +54,6 @@ IF (WIN32)
     my_wincond.c
     my_winerr.c
     my_winfile.c
-    my_conio.c
     my_minidump.cc
     my_win_popen.cc)
 ENDIF()

mysys/my_conio.c

-/* Copyright (c) 2000, 2005, 2007 MySQL AB
-   Use is subject to license terms
-
-   This program is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation; version 2 of the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335  USA */
-
-
-#include "mysys_priv.h"
-
-#ifdef _WIN32
-
-static HANDLE my_coninpfh= 0;     /* console input */
-
-/*
-  functions my_pthread_auto_mutex_lock & my_pthread_auto_mutex_free
-  are experimental at this moment, they are intended to bring
-  ability of protecting code sections without necessity to explicitly
-  initialize synchronization object in one of threads
-
-  if found useful they are to be exported in mysys
-*/
-
-
-/*
-  int my_pthread_auto_mutex_lock(HANDLE* ph, const char* name, 
-                                 int id, int time)
-  NOTES
-    creates a mutex with given name and tries to lock it time msec.
-    mutex name is appended with id to allow system wide or process wide
-    locks. Handle to created mutex returned in ph argument.
-
-  RETURN
-    0	              thread owns mutex
-    <>0	            error
-*/
-
-static
-int my_pthread_auto_mutex_lock(HANDLE* ph, const char* name, int id, int time)
-{
-  DWORD res;
-  char tname[FN_REFLEN];
-  
-  sprintf(tname, "%s-%08X", name, id);
-  
-  *ph= CreateMutex(NULL, FALSE, tname);
-  if (*ph == NULL)
-    return GetLastError();
-
-  res= WaitForSingleObject(*ph, time);
-  
-  if (res == WAIT_TIMEOUT)
-    return ERROR_SEM_TIMEOUT;
-
-  if (res == WAIT_FAILED)
-    return GetLastError();
-
-  return 0;
-}
-
-/*
-  int my_pthread_auto_mutex_free(HANDLE* ph)
-
-  NOTES
-    releases a mutex.
-
-  RETURN
-    0	              thread released mutex
-    <>0	            error
-
-*/
-static
-int my_pthread_auto_mutex_free(HANDLE* ph)
-{
-  if (*ph)
-  {
-    ReleaseMutex(*ph);
-    CloseHandle(*ph);
-    *ph= NULL;
-  }
-
-  return 0;
-}
-
-
-#define pthread_auto_mutex_decl(name)                           \
-  HANDLE __h##name= NULL;
-
-#define pthread_auto_mutex_lock(name, proc, time)               \
-  my_pthread_auto_mutex_lock(&__h##name, #name, (proc), (time))
-
-#define pthread_auto_mutex_free(name)                           \
-  my_pthread_auto_mutex_free(&__h##name)
-
-
-/*
-  char* my_cgets()
-
-  NOTES
-    Replaces _cgets from libc to support input of more than 255 chars.
-    Reads from the console via ReadConsole into buffer which 
-    should be at least clen characters.
-    Actual length of string returned in plen.
-
-  WARNING
-    my_cgets() does NOT check the pushback character buffer (i.e., _chbuf).
-    Thus, my_cgets() will not return any character that is pushed back by 
-    the _ungetch() call.
-
-  RETURN
-    string pointer	ok
-    NULL	          Error
-
-*/
-
-char* my_cgets(char *buffer, size_t clen, size_t* plen)
-{
-  ULONG state;
-  char *result;
-  DWORD plen_res;
-  CONSOLE_SCREEN_BUFFER_INFO csbi;
-  
-  pthread_auto_mutex_decl(my_conio_cs);
- 
-  /* lock the console for the current process*/
-  if (pthread_auto_mutex_lock(my_conio_cs, GetCurrentProcessId(), INFINITE))
-  {
-    /* can not lock console */
-    pthread_auto_mutex_free(my_conio_cs);  
-    return NULL;
-  }
-
-  /* init console input */
-  if (my_coninpfh == 0)
-  {
-    /* same handle will be used until process termination */
-    my_coninpfh= CreateFile("CONIN$", GENERIC_READ | GENERIC_WRITE,
-                            FILE_SHARE_READ | FILE_SHARE_WRITE,
-                            NULL, OPEN_EXISTING, 0, NULL);
-  }
-
-  if (my_coninpfh == INVALID_HANDLE_VALUE) 
-  {
-    /* unlock the console */
-    pthread_auto_mutex_free(my_conio_cs);  
-    return(NULL);
-  }
-
-  GetConsoleMode((HANDLE)my_coninpfh, &state);
-  SetConsoleMode((HANDLE)my_coninpfh, ENABLE_LINE_INPUT | 
-                 ENABLE_PROCESSED_INPUT | ENABLE_ECHO_INPUT);
-
-  GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi);
-
-  /* 
-    there is no known way to determine allowed buffer size for input
-    though it is known it should not be more than 64K               
-    so we cut 64K and try first size of screen buffer               
-    if it is still to large we cut half of it and try again         
-    later we may want to cycle from MY_MIN(clen, 65535) to allowed size
-    with small decrement to determine exact allowed buffer           
-  */
-  clen= MY_MIN(clen, 65535);
-  do
-  {
-    clen= MY_MIN(clen, (size_t) csbi.dwSize.X*csbi.dwSize.Y);
-    if (!ReadConsole((HANDLE)my_coninpfh, (LPVOID)buffer, (DWORD) clen - 1, &plen_res,
-                     NULL))
-    {
-      result= NULL;
-      clen>>= 1;
-    }
-    else
-    {
-      result= buffer;
-      break;
-    }
-  }
-  while (GetLastError() == ERROR_NOT_ENOUGH_MEMORY);
-  *plen= plen_res;
-
-  /* We go here on error reading the string (Ctrl-C for example) */
-  if (!*plen)
-    result= NULL;                              /* purecov: inspected */
-
-  if (result != NULL)
-  {
-    if (*plen > 1 && buffer[*plen - 2] == '\r')
-    {
-      *plen= *plen - 2;
-    }
-    else 
-    {
-      if (*plen > 0 && buffer[*plen - 1] == '\r')
-      {
-        char tmp[3];
-        DWORD  tmplen= (DWORD)sizeof(tmp);
-
-        *plen= *plen - 1;
-        /* read /n left in the buffer */
-        ReadConsole((HANDLE)my_coninpfh, (LPVOID)tmp, tmplen, &tmplen, NULL);
-      }
-    }
-    buffer[*plen]= '\0';
-  }
-
-  SetConsoleMode((HANDLE)my_coninpfh, state);
-  /* unlock the console */
-  pthread_auto_mutex_free(my_conio_cs);  
-
-  return result;
-}
-
-#endif /* _WIN32 */