MDEV-6561 libedit detection is broken

fix readline/libedit detection:
* search in readline/, editline/ and edit/readline/
* fix typos CMAKE_REQUIRES_LIBRARIES -> CMAKE_REQUIRED_LIBRARIES
* use correct libedit API
* use different cmake variables for libedit and readline

client/mysql.cc

@@ -89,7 +89,7 @@ extern "C" {
 #if defined(__WIN__)
 #include <conio.h>
 #else
-#include <readline/readline.h>
+#include <readline.h>
 #define HAVE_READLINE
 #define USE_POPEN
 #endif

cmake/build_configurations/mysql_release.cmake

@@ -121,12 +121,6 @@ ENDIF()
 
 IF(UNIX)
   SET(WITH_EXTRA_CHARSETS all CACHE STRING "")
-  IF(EXISTS "${CMAKE_SOURCE_DIR}/COPYING")
-    OPTION(WITH_READLINE  "" ON)
-  ELSE()
-    OPTION(WITH_LIBEDIT  "" ON)
-  ENDIF()
-
 
   IF(CMAKE_SYSTEM_NAME STREQUAL "Linux")
 

cmake/readline.cmake

@@ -116,24 +116,23 @@ ENDMACRO()
 MACRO (MYSQL_USE_BUNDLED_READLINE)
   SET(USE_NEW_READLINE_INTERFACE 1)
   SET(HAVE_HIST_ENTRY 0 CACHE INTERNAL "" FORCE)
-  SET(READLINE_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/cmd-line-utils)
-  SET(READLINE_LIBRARY readline)
+  SET(MY_READLINE_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/cmd-line-utils/readline)
+  SET(MY_READLINE_LIBRARY readline)
   ADD_SUBDIRECTORY(${CMAKE_SOURCE_DIR}/cmd-line-utils/readline)
 ENDMACRO()
 
 MACRO (MYSQL_FIND_SYSTEM_READLINE)
   
-  FIND_PATH(READLINE_INCLUDE_DIR readline/readline.h )
+  FIND_PATH(READLINE_INCLUDE_DIR readline.h PATH_SUFFIXES readline)
   FIND_LIBRARY(READLINE_LIBRARY NAMES readline)
   MARK_AS_ADVANCED(READLINE_INCLUDE_DIR READLINE_LIBRARY)
 
-  SET(CMAKE_REQUIRES_LIBRARIES ${READLINE_LIBRARY} ${CURSES_LIBRARY})
-
   IF(READLINE_LIBRARY AND READLINE_INCLUDE_DIR)
     SET(CMAKE_REQUIRED_LIBRARIES ${READLINE_LIBRARY} ${CURSES_LIBRARY})
+    SET(CMAKE_REQUIRED_INCLUDES ${READLINE_INCLUDE_DIR})
     CHECK_CXX_SOURCE_COMPILES("
     #include <stdio.h>
-    #include <readline/readline.h>
+    #include <readline.h>
     int main(int argc, char **argv)
     {
       rl_completion_func_t *func1= (rl_completion_func_t*)0;
@@ -141,19 +140,9 @@ MACRO (MYSQL_FIND_SYSTEM_READLINE)
     }"
     NEW_READLINE_INTERFACE)
 
-    CHECK_CXX_SOURCE_COMPILES("
-    #include <stdio.h>
-    #include <readline/readline.h>
-    int main(int argc, char **argv)
-    {
-       HIST_ENTRY entry;
-       return 0;
-    }"
-    HAVE_HIST_ENTRY)
-    
     CHECK_C_SOURCE_COMPILES("
     #include <stdio.h>
-    #include <readline/readline.h>
+    #include <readline.h>
     #if RL_VERSION_MAJOR > 5
     #error
     #endif
@@ -176,30 +165,27 @@ MACRO (MYSQL_FIND_SYSTEM_READLINE)
       ENDIF(READLINE_V5)
     ENDIF(NEW_READLINE_INTERFACE)
   ENDIF()
-  SET(CMAKE_REQUIRES_LIBRARIES )
 ENDMACRO()
 
 MACRO (MYSQL_FIND_SYSTEM_LIBEDIT)
-  
-  FIND_PATH(READLINE_INCLUDE_DIR readline/readline.h )
-  FIND_LIBRARY(READLINE_LIBRARY NAMES readline)
-  MARK_AS_ADVANCED(READLINE_INCLUDE_DIR READLINE_LIBRARY)
-
-  SET(CMAKE_REQUIRES_LIBRARIES ${READLINE_LIBRARY})
+  FIND_PATH(LIBEDIT_INCLUDE_DIR readline.h PATH_SUFFIXES editline edit/readline)
+  FIND_LIBRARY(LIBEDIT_LIBRARY edit)
+  MARK_AS_ADVANCED(LIBEDIT_INCLUDE_DIR LIBEDIT_LIBRARY)
 
-  IF(READLINE_LIBRARY AND READLINE_INCLUDE_DIR)
+  IF(LIBEDIT_LIBRARY AND LIBEDIT_INCLUDE_DIR)
+    SET(CMAKE_REQUIRED_LIBRARIES ${LIBEDIT_LIBRARY})
+    SET(CMAKE_REQUIRED_INCLUDES ${LIBEDIT_INCLUDE_DIR})
     CHECK_CXX_SOURCE_COMPILES("
     #include <stdio.h>
-    #include <readline/readline.h>
+    #include <readline.h>
     int main(int argc, char **argv)
     {
-      char res= *(*rl_completion_entry_function)(0,0);
+      int res= (*rl_completion_entry_function)(0,0);
       completion_matches(0,0);
     }"
     LIBEDIT_INTERFACE)
     SET(USE_LIBEDIT_INTERFACE ${LIBEDIT_INTERFACE})
   ENDIF()
-  SET(CMAKE_REQUIRES_LIBRARIES)
 ENDMACRO()
 
 
@@ -216,15 +202,33 @@ MACRO (MYSQL_CHECK_READLINE)
       IF (NOT APPLE)
         MYSQL_FIND_SYSTEM_READLINE()
       ENDIF()
-      IF(NOT USE_NEW_READLINE_INTERFACE)
+      IF(USE_NEW_READLINE_INTERFACE)
+        SET(MY_READLINE_INCLUDE_DIR ${READLINE_INCLUDE_DIR})
+        SET(MY_READLINE_LIBRARY ${READLINE_LIBRARY} ${CURSES_LIBRARY})
+      ELSE()
         MYSQL_FIND_SYSTEM_LIBEDIT()
-        IF(NOT USE_LIBEDIT_INTERFACE)
+        IF(USE_LIBEDIT_INTERFACE)
+          SET(MY_READLINE_INCLUDE_DIR ${LIBEDIT_INCLUDE_DIR})
+          SET(MY_READLINE_LIBRARY ${LIBEDIT_LIBRARY} ${CURSES_LIBRARY})
+        ELSE()
           MYSQL_USE_BUNDLED_READLINE()
         ENDIF()
       ENDIF()
     ENDIF()
-    SET(MY_READLINE_INCLUDE_DIR ${READLINE_INCLUDE_DIR})
-    SET(MY_READLINE_LIBRARY ${READLINE_LIBRARY} ${CURSES_LIBRARY})
+
+    SET(CMAKE_REQUIRED_LIBRARIES ${MY_READLINE_LIBRARY})
+    SET(CMAKE_REQUIRED_INCLUDES ${MY_READLINE_INCLUDE_DIR})
+    CHECK_CXX_SOURCE_COMPILES("
+    #include <stdio.h>
+    #include <readline.h>
+    int main(int argc, char **argv)
+    {
+       HIST_ENTRY entry;
+       return 0;
+    }"
+    HAVE_HIST_ENTRY)
+    SET(CMAKE_REQUIRED_LIBRARIES)
+    SET(CMAKE_REQUIRED_INCLUDES)
   ENDIF(NOT WIN32)
 ENDMACRO()
 

cmd-line-utils/readline/CMakeLists.txt

@@ -13,8 +13,7 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 
-INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include
-                    ${CMAKE_SOURCE_DIR}/cmd-line-utils)
+INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR})
 
 ADD_DEFINITIONS(-DHAVE_CONFIG_H -DNO_KILL_INTR)
 

cmd-line-utils/readline/history.h

@@ -32,8 +32,8 @@ extern "C" {
 #  include "rlstdc.h"
 #  include "rltypedefs.h"
 #else
-#  include <readline/rlstdc.h>
-#  include <readline/rltypedefs.h>
+#  include <rlstdc.h>
+#  include <rltypedefs.h>
 #endif
 
 #ifdef __STDC__

cmd-line-utils/readline/keymaps.h

@@ -32,9 +32,9 @@ extern "C" {
 #  include "chardefs.h"
 #  include "rltypedefs.h"
 #else
-#  include <readline/rlstdc.h>
-#  include <readline/chardefs.h>
-#  include <readline/rltypedefs.h>
+#  include <rlstdc.h>
+#  include <chardefs.h>
+#  include <rltypedefs.h>
 #endif
 
 /* A keymap contains one entry for each key in the ASCII set.

cmd-line-utils/readline/readline.h

@@ -33,10 +33,10 @@ extern "C" {
 #  include "keymaps.h"
 #  include "tilde.h"
 #else
-#  include <readline/rlstdc.h>
-#  include <readline/rltypedefs.h>
-#  include <readline/keymaps.h>
-#  include <readline/tilde.h>
+#  include <rlstdc.h>
+#  include <rltypedefs.h>
+#  include <keymaps.h>
+#  include <tilde.h>
 #endif
 
 /* Hex-encoded Readline version number. */

cmd-line-utils/readline/xmalloc.h

@@ -26,7 +26,7 @@
 #if defined (READLINE_LIBRARY)
 #  include "rlstdc.h"
 #else
-#  include <readline/rlstdc.h>
+#  include <rlstdc.h>
 #endif
 
 #ifndef PTR_T