Bug#27759871: BACKRONYM ISSUE IS STILL IN MYSQL 5.7

Description:- Client applications establishes connection to
server, which does not support SSL, via TCP even when SSL is
enforced via MYSQL_OPT_SSL_MODE or MYSQL_OPT_SSL_ENFORCE or
MYSQL_OPT_SSL_VERIFY_SERVER_CERT.

Analysis:- There exist no error handling for catching client
applications which enforces SSL connection to connect to a
server which does not support SSL.

Fix:- Error handling is done to catch above mentioned
scenarios.

include/sql_common.h

 #ifndef SQL_COMMON_INCLUDED
 #define SQL_COMMON_INCLUDED
 
-/* Copyright (c) 2003, 2017, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
    
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -96,6 +96,9 @@ void set_stmt_error(MYSQL_STMT *stmt, int errcode, const char *sqlstate,
 void set_mysql_error(MYSQL *mysql, int errcode, const char *sqlstate);
 void set_mysql_extended_error(MYSQL *mysql, int errcode, const char *sqlstate,
                               const char *format, ...);
+#ifdef EMBEDDED_LIBRARY
+int embedded_ssl_check(MYSQL *mysql);
+#endif
 
 /* client side of the pluggable authentication */
 struct st_plugin_vio_info;

libmysqld/libmysqld.c

-/* Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -173,6 +173,9 @@ mysql_real_connect(MYSQL *mysql,const char *host, const char *user,
   if (db)
     client_flag|=CLIENT_CONNECT_WITH_DB;
 
+  if (embedded_ssl_check(mysql))
+    goto error;
+
   mysql->info_buffer= my_malloc(MYSQL_ERRMSG_SIZE, MYF(0));
   mysql->thd= create_embedded_thd(client_flag);
 

sql-common/client.c

@@ -2020,6 +2020,34 @@ static int ssl_verify_server_cert(Vio *vio, const char* server_hostname, const c
 #endif /* HAVE_OPENSSL && !EMBEDDED_LIBRARY */
 
 
+/**
+  Checks if any SSL option is set for libmysqld embedded server.
+
+  @param  mysql   the connection handle
+  @retval 0       success
+  @retval 1       failure
+*/
+#ifdef EMBEDDED_LIBRARY
+int embedded_ssl_check(MYSQL *mysql)
+{
+  if (mysql->options.ssl_key || mysql->options.ssl_cert ||
+      mysql->options.ssl_ca || mysql->options.ssl_capath ||
+      mysql->options.ssl_cipher ||
+      mysql->options.client_flag & CLIENT_SSL_VERIFY_SERVER_CERT ||
+      (mysql->options.extension &&
+        mysql->options.extension->ssl_mode == SSL_MODE_REQUIRED))
+  {
+     set_mysql_extended_error(mysql, CR_SSL_CONNECTION_ERROR, unknown_sqlstate,
+                              ER(CR_SSL_CONNECTION_ERROR),
+                              "Embedded server libmysqld library doesn't support "
+                              "SSL connections");
+     return 1;
+  }
+  return 0;
+}
+#endif
+
+
 /*
   Note that the mysql argument must be initialized with mysql_init()
   before calling mysql_real_connect !
@@ -3592,6 +3620,11 @@ CLI_MYSQL_REAL_CONNECT(MYSQL *mysql,const char *host, const char *user,
 
   mysql->client_flag= client_flag;
 
+#ifdef EMBEDDED_LIBRARY
+  if (embedded_ssl_check(mysql))
+    goto error;
+#endif
+
   /*
     Part 2: invoke the plugin to send the authentication data to the server
   */
@@ -4271,10 +4304,14 @@ mysql_options(MYSQL *mysql,enum mysql_option option, const void *arg)
     mysql->reconnect= *(my_bool *) arg;
     break;
   case MYSQL_OPT_SSL_VERIFY_SERVER_CERT:
+#if defined(HAVE_OPENSSL) && !defined(EMBEDDED_LIBRARY)
     if (*(my_bool*) arg)
       mysql->options.client_flag|= CLIENT_SSL_VERIFY_SERVER_CERT;
     else
       mysql->options.client_flag&= ~CLIENT_SSL_VERIFY_SERVER_CERT;
+#elif defined(EMBEDDED_LIBRARY)
+    DBUG_RETURN(1);
+#endif
     break;
   case MYSQL_PLUGIN_DIR:
     EXTENSION_SET_STRING(&mysql->options, plugin_dir, arg);
@@ -4288,11 +4325,15 @@ mysql_options(MYSQL *mysql,enum mysql_option option, const void *arg)
       (*(my_bool*) arg) ? TRUE : FALSE;
     break;
   case MYSQL_OPT_SSL_MODE:
+#if defined(HAVE_OPENSSL) && !defined(EMBEDDED_LIBRARY)
     if (*(uint *) arg == SSL_MODE_REQUIRED)
     {
       ENSURE_EXTENSIONS_PRESENT(&mysql->options);
       mysql->options.extension->ssl_mode= SSL_MODE_REQUIRED;
     }
+#elif defined(EMBEDDED_LIBRARY)
+    DBUG_RETURN(1);
+#endif
     break;
   default:
     DBUG_RETURN(1);