Merge remote-tracking branch 'connect/10.2' 10.2

storage/connect/CMakeLists.txt

@@ -81,6 +81,19 @@ ELSE(NOT UNIX)
 ENDIF(UNIX)
 
 
+#
+# BSON: the new handling of JSON data included temporarily for testing
+#
+
+OPTION(CONNECT_WITH_BSON "Compile CONNECT storage engine with BSON support" ON)
+
+IF(CONNECT_WITH_BSON)
+  SET(CONNECT_SOURCES ${CONNECT_SOURCES}
+ bson.cpp bsonudf.cpp tabbson.cpp bson.h bsonudf.h tabbson.h)
+  add_definitions(-DBSON_SUPPORT)
+ENDIF(CONNECT_WITH_BSON)
+
+
 #
 # VCT: the VEC format might be not supported in future versions
 #
@@ -318,29 +331,29 @@ ENDIF(CONNECT_WITH_MONGO)
 OPTION(CONNECT_WITH_REST "Compile CONNECT storage engine with REST support" ON)
 
 IF(CONNECT_WITH_REST)
-  MESSAGE_ONCE(CONNECT_WITH_REST "REST support is ON")
+# MESSAGE(STATUS "=====> REST support is ON")
   SET(CONNECT_SOURCES ${CONNECT_SOURCES} tabrest.cpp tabrest.h)
   add_definitions(-DREST_SUPPORT)
-  FIND_PACKAGE(cpprestsdk QUIET)
-  IF (cpprestsdk_FOUND)
-	IF(UNIX)
-#     INCLUDE_DIRECTORIES(${CPPRESTSDK_INCLUDE_DIR})
-#     If needed edit next line to set the path to libcpprest.so
-      SET(REST_LIBRARY -lcpprest)
-      MESSAGE (STATUS ${REST_LIBRARY})
-	ELSE(NOT UNIX)
-#     Next line sets debug compile mode matching cpprest_2_10d.dll
-#     when it was binary installed (can be change later in Visual Studio)
-#     Comment it out if not needed depending on your cpprestsdk installation.
-	  SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MDd")
-	ENDIF(UNIX)
-#   IF(REST_LIBRARY)     why this? how about Windows
-    SET(CONNECT_SOURCES ${CONNECT_SOURCES} restget.cpp)
-    add_definitions(-DREST_SOURCE)
-#   ENDIF()
-  ELSE(NOT cpprestsdk_FOUND)
-#    MESSAGE(STATUS "=====> cpprestsdk package not found")
-  ENDIF (cpprestsdk_FOUND)
+# FIND_PACKAGE(cpprestsdk QUIET)
+# IF (cpprestsdk_FOUND)
+#	IF(UNIX)
+##    INCLUDE_DIRECTORIES(${CPPRESTSDK_INCLUDE_DIR})
+##    If needed edit next line to set the path to libcpprest.so
+#     SET(REST_LIBRARY -lcpprest)
+#     MESSAGE (STATUS ${REST_LIBRARY})
+#	ELSE(NOT UNIX)
+##    Next line sets debug compile mode matching cpprest_2_10d.dll
+##    when it was binary installed (can be change later in Visual Studio)
+##    Comment it out if not needed depending on your cpprestsdk installation.
+#	  SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MDd")
+#	ENDIF(UNIX)
+##  IF(REST_LIBRARY)     why this? how about Windows
+#   SET(CONNECT_SOURCES ${CONNECT_SOURCES} restget.cpp)
+#   add_definitions(-DREST_SOURCE)
+##  ENDIF()
+##ELSE(NOT cpprestsdk_FOUND)
+##  MESSAGE(STATUS "=====> cpprestsdk package not found")
+# ENDIF (cpprestsdk_FOUND)
 ENDIF(CONNECT_WITH_REST)
 
 #

storage/connect/block.h

 /**************** Block H Declares Source Code File (.H) ***************/
-/*  Name: BLOCK.H     Version 2.0                                      */
+/*  Name: BLOCK.H     Version 2.1                                      */
 /*                                                                     */
-/*  (C) Copyright to the author Olivier BERTRAND          1998         */
+/*  (C) Copyright to the author Olivier BERTRAND          1998 - 2020  */
 /*                                                                     */
 /*  This file contains the BLOCK pure virtual class definition.        */
 /*---------------------------------------------------------------------*/
 /*  Note: one of the main purpose of this base class is to take care   */
-/*  of the very specific way Plug handles memory allocation.           */
+/*  of the very specific way Connect handles memory allocation.        */
 /*  Instead of allocating small chunks of storage via new or malloc    */
-/*  Plug works in its private memory pool in which it does the sub-    */
+/*  Connect works in its private memory pool in which it does the sub- */
 /*  allocation using the function PlugSubAlloc. These are never freed  */
 /*  separately but when a transaction is terminated, the entire pool   */
 /*  is set to empty, resulting in a very fast and efficient allocate   */
 /*  process, no garbage collection problem, and an automatic recovery  */
-/*  procedure (via LongJump) when the memory is exhausted.             */
+/*  procedure (via throw) when the memory is exhausted.                */
 /*  For this to work new must be given two parameters, first the       */
 /*  global pointer of the Plug application, and an optional pointer to */
 /*  the memory pool to use, defaulting to NULL meaning using the Plug  */
@@ -37,21 +37,25 @@ typedef class BLOCK *PBLOCK;
 
 class DllExport BLOCK {
  public:
-  void * operator new(size_t size, PGLOBAL g, void *p = NULL) {
-	  xtrc(256, "New BLOCK: size=%d g=%p p=%p\n", size, g, p);
-    return (PlugSubAlloc(g, p, size));
+  void *operator new(size_t size, PGLOBAL g, void *mp = NULL) {
+	  xtrc(256, "New BLOCK: size=%d g=%p p=%p\n", size, g, mp);
+		return PlugSubAlloc(g, mp, size);
+  } // end of new
+
+	void* operator new(size_t size, long long mp) {
+		xtrc(256, "Realloc at: mp=%lld\n", mp);
+		return (void*)mp;
 	} // end of new
 
 	virtual void Printf(PGLOBAL, FILE *, uint) {}   // Produce file desc
   virtual void Prints(PGLOBAL, char *, uint) {}   // Produce string desc
   
-#if !defined(__BORLANDC__)
-  // Avoid warning C4291 by defining a matching dummy delete operator
-  void operator delete(void *, PGLOBAL, void *) {}
-  void operator delete(void *, size_t) {}
-#endif
-  virtual ~BLOCK() {}
+  // Avoid gcc errors by defining matching dummy delete operators
+  void operator delete(void*, PGLOBAL, void *) {}
+	void operator delete(void*, long long) {}
+	void operator delete(void*) {}
 
-  }; // end of class BLOCK
+  virtual ~BLOCK() {}
+}; // end of class BLOCK
 
 #endif   // !BLOCK_DEFINED

storage/connect/bson.h

+/**************** bson H Declares Source Code File (.H) ****************/
+/*  Name: bson.h   Version 1.0                                         */
+/*                                                                     */
+/*  (C) Copyright to the author Olivier BERTRAND          2020         */
+/*                                                                     */
+/*  This file contains the BSON classe declares.                       */
+/***********************************************************************/
+#pragma once
+#include <mysql_com.h>
+#include "json.h"
+#include "xobject.h"
+
+#if defined(_DEBUG)
+#define X  assert(false);
+#else
+#define X												  
+#endif
+
+#define ARGS    MY_MIN(24,(int)len-i),s+MY_MAX(i-3,0)
+
+class BDOC;
+class BOUT;
+class BJSON;
+
+typedef class BDOC* PBDOC;
+typedef class BJSON* PBJSON;
+typedef uint  OFFSET;
+
+/***********************************************************************/
+/* Structure BVAL. Binary representation of a JVALUE.                  */
+/***********************************************************************/
+typedef struct _jvalue {
+	union {
+		OFFSET To_Val;    // Offset to a value
+		int    N;         // An integer value
+		float  F;				  // A float value
+		bool   B;				  // A boolean value True or false (0)
+	};
+	short    Nd;				// Number of decimals
+	char     Type;      // The value type
+	char     Ktp;       // The key type
+	OFFSET   Next;      // Offset to the next value in array
+} BVAL, *PBVAL;  // end of struct BVALUE
+
+/***********************************************************************/
+/* Structure BPAIR. The pairs of a json Object.                        */
+/***********************************************************************/
+typedef struct _jpair {
+	OFFSET Key;    // Offset to this pair key name
+	BVAL   Vlp;    // The value of the pair
+} BPAIR, *PBPR;  // end of struct BPAIR
+
+char* NextChr(PSZ s, char sep);
+char* GetJsonNull(void);
+const char* GetFmt(int type, bool un);														 
+
+DllExport bool IsNum(PSZ s);
+
+/***********************************************************************/
+/* Class BJSON. The class handling all BJSON operations.               */
+/***********************************************************************/
+class BJSON : public BLOCK {
+public:
+	// Constructor
+	BJSON(PGLOBAL g, PBVAL vp = NULL)
+	  { G = g, Base = G->Sarea; Bvp = vp; Throw = true; }
+
+	// Utility functions
+	inline OFFSET   MOF(void  *p) {return MakeOff(Base, p);}
+	inline void     *MP(OFFSET o) {return MakePtr(Base, o);} 
+	inline PBPR     MPP(OFFSET o) {return (PBPR)MakePtr(Base, o);}
+	inline PBVAL    MVP(OFFSET o) {return (PBVAL)MakePtr(Base, o);}
+	inline PSZ      MZP(OFFSET o) {return (PSZ)MakePtr(Base, o);}
+	inline longlong LLN(OFFSET o) {return *(longlong*)MakePtr(Base, o);}
+	inline double   DBL(OFFSET o) {return *(double*)MakePtr(Base, o);}
+
+	void  Reset(void) {Base = G->Sarea;}
+	void* GetBase(void) { return Base; }
+	void  SubSet(bool b = false);
+	void  MemSave(void) {G->Saved_Size = ((PPOOLHEADER)G->Sarea)->To_Free;}
+	void  MemSet(size_t size);
+	void  GetMsg(PGLOBAL g) { if (g != G) strcpy(g->Message, G->Message); }
+
+	// SubAlloc functions
+	void* BsonSubAlloc(size_t size);
+	PBPR  NewPair(OFFSET key, int type = TYPE_NULL);
+	OFFSET NewPair(PSZ key, int type = TYPE_NULL)
+				{return MOF(NewPair(DupStr(key), type));}
+	PBVAL NewVal(int type = TYPE_NULL);
+	PBVAL NewVal(PVAL valp);
+	PBVAL SubAllocVal(OFFSET toval, int type = TYPE_NULL, short nd = 0);
+	PBVAL SubAllocVal(PBVAL toval, int type = TYPE_NULL, short nd = 0)
+				{return SubAllocVal(MOF(toval), type, nd);}
+	PBVAL SubAllocStr(OFFSET str, short nd = 0);
+	PBVAL SubAllocStr(PSZ str, short nd = 0)
+				{return SubAllocStr(DupStr(str), nd);}
+	PBVAL DupVal(PBVAL bvp);
+	OFFSET DupStr(PSZ str) { return MOF(NewStr(str)); }
+	PSZ   NewStr(PSZ str);
+
+	// Array functions
+	inline PBVAL GetArray(PBVAL vlp) {return MVP(vlp->To_Val);}
+	int   GetArraySize(PBVAL bap, bool b = false);
+	PBVAL GetArrayValue(PBVAL bap, int i);
+  PSZ   GetArrayText(PGLOBAL g, PBVAL bap, PSTRG text);
+	void  MergeArray(PBVAL bap1,PBVAL bap2);
+	bool  DeleteValue(PBVAL bap, int n);
+	void  AddArrayValue(PBVAL bap, OFFSET nvp = NULL, int* x = NULL);
+	inline void AddArrayValue(PBVAL bap, PBVAL nvp = NULL, int* x = NULL)
+				{AddArrayValue(bap, MOF(nvp), x);}
+	void  SetArrayValue(PBVAL bap, PBVAL nvp, int n);
+	bool  IsArrayNull(PBVAL bap);
+
+	// Object functions
+	inline PBPR GetObject(PBVAL bop) {return MPP(bop->To_Val);}
+	inline PBPR	GetNext(PBPR brp) { return MPP(brp->Vlp.Next); }
+	void  SetPairValue(PBPR brp, PBVAL bvp);
+	int   GetObjectSize(PBVAL bop, bool b = false);
+  PSZ   GetObjectText(PGLOBAL g, PBVAL bop, PSTRG text);
+	PBVAL MergeObject(PBVAL bop1, PBVAL bop2);
+	PBVAL AddPair(PBVAL bop, PSZ key, int type = TYPE_NULL);
+	PSZ   GetKey(PBPR prp) {return prp ? MZP(prp->Key) : NULL;}
+	PBVAL GetTo_Val(PBPR prp) {return prp ? MVP(prp->Vlp.To_Val) : NULL;}
+	PBVAL GetVlp(PBPR prp) {return prp ? (PBVAL)&prp->Vlp : NULL;}
+	PBVAL GetKeyValue(PBVAL bop, PSZ key);
+	PBVAL GetKeyList(PBVAL bop);
+	PBVAL GetObjectValList(PBVAL bop);
+	void  SetKeyValue(PBVAL bop, OFFSET bvp, PSZ key);
+	inline void SetKeyValue(PBVAL bop, PBVAL vlp, PSZ key)
+				{SetKeyValue(bop, MOF(vlp), key);}
+	bool  DeleteKey(PBVAL bop, PCSZ k);
+	bool  IsObjectNull(PBVAL bop);
+
+	// Value functions
+	int   GetSize(PBVAL vlp, bool b = false);
+	PBVAL GetNext(PBVAL vlp) {return MVP(vlp->Next);}
+	//PJSON GetJsp(void) { return (DataType == TYPE_JSON ? Jsp : NULL); }
+	PSZ   GetValueText(PGLOBAL g, PBVAL vlp, PSTRG text);
+	PBVAL GetBson(PBVAL bvp);
+	PSZ   GetString(PBVAL vp, char* buff = NULL);
+	int   GetInteger(PBVAL vp);
+	long long GetBigint(PBVAL vp);
+	double GetDouble(PBVAL vp);
+	PVAL  GetValue(PGLOBAL g, PBVAL vp);
+	void  SetValueObj(PBVAL vlp, PBVAL bop);
+	void  SetValueArr(PBVAL vlp, PBVAL bap);
+	void  SetValueVal(PBVAL vlp, PBVAL vp);
+	PBVAL SetValue(PBVAL vlp, PVAL valp);
+	void  SetString(PBVAL vlp, PSZ s, int ci = 0);
+	void  SetInteger(PBVAL vlp, int n);
+	void  SetBigint(PBVAL vlp, longlong ll);
+	void  SetFloat(PBVAL vlp, double f, int nd = 16);
+	void  SetFloat(PBVAL vlp, PSZ s);
+	void  SetBool(PBVAL vlp, bool b);
+	void  Clear(PBVAL vlp) { vlp->N = 0; vlp->Nd = 0; vlp->Next = 0; }
+	bool  IsValueNull(PBVAL vlp);
+	bool  IsJson(PBVAL vlp)	{return vlp ? vlp->Type == TYPE_JAR ||
+		                                    vlp->Type == TYPE_JOB ||
+		                                    vlp->Type == TYPE_JVAL : false;}
+
+	// Members
+	PGLOBAL G;
+	PBVAL   Bvp;
+	void   *Base;
+	bool    Throw;
+
+protected:
+	// Default constructor not to be used
+	BJSON(void) {}
+}; // end of class BJSON
+
+/***********************************************************************/
+/* Class JDOC. The class for parsing and serializing json documents.   */
+/***********************************************************************/
+class BDOC : public BJSON {
+public:
+	BDOC(PGLOBAL G);
+
+	bool  GetComma(void) { return comma; }
+	int   GetPretty(void) { return pretty; }
+	void  SetPretty(int pty) { pretty = pty; }
+
+	// Methods
+	PBVAL ParseJson(PGLOBAL g, char* s, size_t n);
+	PSZ   Serialize(PGLOBAL g, PBVAL bvp, char* fn, int pretty);
+
+protected:
+	OFFSET ParseArray(int& i);
+	OFFSET ParseObject(int& i);
+	PBVAL  ParseValue(int& i, PBVAL bvp);
+	OFFSET ParseString(int& i);
+	void   ParseNumeric(int& i, PBVAL bvp);
+	OFFSET ParseAsArray(int& i);
+	bool   SerializeArray(OFFSET arp, bool b);
+	bool   SerializeObject(OFFSET obp);
+	bool   SerializeValue(PBVAL vp, bool b = false);
+
+	// Members used when parsing and serializing
+	JOUT* jp;						 // Used with serialize
+	char* s;						 // The Json string to parse
+	int   len;					 // The Json string length
+	int   pretty;				 // The pretty style of the file to parse
+	bool  pty[3];				 // Used to guess what pretty is
+	bool  comma;				 // True if Pretty = 1
+
+	// Default constructor not to be used
+	BDOC(void) {}
+}; // end of class BDOC

storage/connect/cmgfam.cpp

 /************** CMGFAM C++ Program Source Code File (.CPP) *************/
 /* PROGRAM NAME: cmgfam.cpp                                            */
 /* -------------                                                       */
-/*  Version 1.4                                                        */
+/*  Version 1.5                                                        */
 /*                                                                     */
 /* COPYRIGHT:                                                          */
 /* ----------                                                          */
-/*  (C) Copyright to the author Olivier BERTRAND          20017        */
+/*  (C) Copyright to the author Olivier BERTRAND          20017 - 2020 */
 /*                                                                     */
 /* WHAT THIS PROGRAM DOES:                                             */
 /* -----------------------                                             */
@@ -29,7 +29,11 @@
 #include "reldef.h"
 #include "filamtxt.h"
 #include "tabdos.h"
+#if defined(BSON_SUPPORT)
+#include "tabbson.h"
+#else
 #include "tabjson.h"
+#endif   // BSON_SUPPORT
 #include "cmgfam.h"
 
 #if defined(UNIX) || defined(UNIV_LINUX)
@@ -53,6 +57,7 @@ CMGFAM::CMGFAM(PJDEF tdp) : DOSFAM((PDOSDEF)NULL)
 		Pcg.Options = tdp->Options;
 		Pcg.Filter = tdp->Filter;
 		Pcg.Pipe = tdp->Pipe && tdp->Options != NULL;
+		Lrecl = tdp->Lrecl + tdp->Ending;
 	} else {
 		Pcg.Uristr = NULL;
 		Pcg.Db_name = NULL;
@@ -60,21 +65,55 @@ CMGFAM::CMGFAM(PJDEF tdp) : DOSFAM((PDOSDEF)NULL)
 		Pcg.Options = NULL;
 		Pcg.Filter = NULL;
 		Pcg.Pipe = false;
+		Lrecl = 0;
 	} // endif tdp
 
 	To_Fbt = NULL;
 	Mode = MODE_ANY;
 	Done = false;
+} // end of CMGFAM standard constructor
+ 
+#if defined(BSON_SUPPORT)
+	/***********************************************************************/
+/*  Constructors.                                                      */
+/***********************************************************************/
+CMGFAM::CMGFAM(PBDEF tdp) : DOSFAM((PDOSDEF)NULL)
+{
+	Cmgp = NULL;
+	Pcg.Tdbp = NULL;
+
+	if (tdp) {
+		Pcg.Uristr = tdp->Uri;
+		Pcg.Db_name = tdp->Schema;
+		Pcg.Coll_name = tdp->Collname;
+		Pcg.Options = tdp->Options;
+		Pcg.Filter = tdp->Filter;
+		Pcg.Pipe = tdp->Pipe && tdp->Options != NULL;
 		Lrecl = tdp->Lrecl + tdp->Ending;
+	} else {
+		Pcg.Uristr = NULL;
+		Pcg.Db_name = NULL;
+		Pcg.Coll_name = NULL;
+		Pcg.Options = NULL;
+		Pcg.Filter = NULL;
+		Pcg.Pipe = false;
+		Lrecl = 0;
+	} // endif tdp
+
+	To_Fbt = NULL;
+	Mode = MODE_ANY;
+	Done = false;
 } // end of CMGFAM standard constructor
+#endif    // BSON_SUPPORT
 
- CMGFAM::CMGFAM(PCMGFAM tdfp) : DOSFAM(tdfp)
+CMGFAM::CMGFAM(PCMGFAM tdfp) : DOSFAM(tdfp)
 {
+	Cmgp = tdfp->Cmgp;
 	Pcg = tdfp->Pcg;
 	To_Fbt = tdfp->To_Fbt;
 	Mode = tdfp->Mode;
 	Done = tdfp->Done;
- } // end of CMGFAM copy constructor
+} // end of CMGFAM copy constructor
 
 /***********************************************************************/
 /*  Reset: reset position values at the beginning of file.             */

storage/connect/cmgfam.h

 /*************** CMGFam H Declares Source Code File (.H) ***************/
-/*  Name: cmgfam.h    Version 1.5                                      */
+/*  Name: cmgfam.h    Version 1.6                                      */
 /*                                                                     */
-/*  (C) Copyright to the author Olivier BERTRAND          2017         */
+/*  (C) Copyright to the author Olivier BERTRAND          2017 - 2020  */
 /*                                                                     */
 /*  This file contains the MongoDB access method classes declares.     */
 /***********************************************************************/
@@ -20,6 +20,9 @@ class DllExport CMGFAM : public DOSFAM {
 public:
 	// Constructor
 	CMGFAM(PJDEF tdp);
+#if defined(BSON_SUPPORT)
+	CMGFAM(PBDEF tdp);
+#endif   // BSON_SUPPORT
 	CMGFAM(PCMGFAM txfp);
 
 	// Implementation

storage/connect/connect.cc

@@ -73,8 +73,7 @@ PGLOBAL CntExit(PGLOBAL g)
 			g->Activityp = NULL;
 		}	// endif Activityp
 
-    PlugExit(g);
-    g= NULL;
+    g= PlugExit(g);
     } // endif g
 
   return g;

storage/connect/filamap.cpp

@@ -5,7 +5,7 @@
 /*                                                                     */
 /* COPYRIGHT:                                                          */
 /* ----------                                                          */
-/*  (C) Copyright to the author Olivier BERTRAND          2005-2017    */
+/*  (C) Copyright to the author Olivier BERTRAND          2005-2020    */
 /*                                                                     */
 /* WHAT THIS PROGRAM DOES:                                             */
 /* -----------------------                                             */
@@ -102,7 +102,7 @@ int MAPFAM::GetFileLength(PGLOBAL g)
 bool MAPFAM::OpenTableFile(PGLOBAL g)
   {
   char    filename[_MAX_PATH];
-  int     len;
+  size_t  len;
   MODE    mode = Tdbp->GetMode();
   PFBLOCK fp;
   PDBUSER dbuserp = (PDBUSER)g->Activityp->Aptr;
@@ -170,13 +170,18 @@ bool MAPFAM::OpenTableFile(PGLOBAL g)
         htrc("CreateFileMap: %s\n", g->Message);
 
       return (mode == MODE_READ && rc == ENOENT)
-              ? PushWarning(g, Tdbp) : true;
+        ? false : true;
+//      ? PushWarning(g, Tdbp) : true; --> assert fails into MariaDB
       } // endif hFile
 
     /*******************************************************************/
-    /*  Get the file size (assuming file is smaller than 4 GB)         */
+    /*  Get the file size.                                             */
     /*******************************************************************/
-    len = mm.lenL;
+		len = (size_t)mm.lenL;
+		
+		if (mm.lenH)
+			len += ((size_t)mm.lenH * 0x000000001LL);
+
     Memory = (char *)mm.memory;
 
     if (!len) {              // Empty or deleted file

storage/connect/filamtxt.h

 /************** FilAMTxt H Declares Source Code File (.H) **************/
-/*  Name: FILAMTXT.H    Version 1.3                                    */
+/*  Name: FILAMTXT.H    Version 1.4                                    */
 /*                                                                     */
-/*  (C) Copyright to the author Olivier BERTRAND          2005-2014    */
+/*  (C) Copyright to the author Olivier BERTRAND          2005-2020    */
 /*                                                                     */
 /*  This file contains the file access method classes declares.        */
 /***********************************************************************/
@@ -15,6 +15,7 @@
 typedef class TXTFAM *PTXF;
 typedef class DOSFAM *PDOSFAM;
 typedef class BLKFAM *PBLKFAM;
+typedef class BINFAM *PBINFAM;
 typedef class DOSDEF *PDOSDEF;
 typedef class TDBDOS *PTDBDOS;
 
@@ -210,4 +211,44 @@ class DllExport BLKFAM : public DOSFAM {
   bool  Closing;              // True when closing on Update
   }; // end of class BLKFAM
 
+/***********************************************************************/
+/*  This is the DOS/UNIX Access Method class declaration for binary    */
+/*  files with variable record format (BJSON)                          */
+/***********************************************************************/
+class DllExport BINFAM : public DOSFAM {
+public:
+	// Constructor
+	BINFAM(PDOSDEF tdp) : DOSFAM(tdp) {Recsize = 0;}
+	BINFAM(PBINFAM txfp) : DOSFAM(txfp) {Recsize = txfp->Recsize;}
+
+	// Implementation
+	virtual AMT   GetAmType(void) {return TYPE_AM_BIN;}
+//virtual int   GetPos(void);
+//virtual int   GetNextPos(void);
+	virtual PTXF  Duplicate(PGLOBAL g) { return (PTXF)new(g) BINFAM(this); }
+
+	// Methods
+//virtual void  Reset(void) {TXTFAM::Reset();}
+//virtual int   GetFileLength(PGLOBAL g);
+//virtual int   Cardinality(PGLOBAL g);
+	virtual int   MaxBlkSize(PGLOBAL g, int s) {return s;}
+	virtual bool  AllocateBuffer(PGLOBAL g);
+//virtual int   GetRowID(void);
+//virtual bool  RecordPos(PGLOBAL g);
+//virtual bool  SetPos(PGLOBAL g, int recpos);
+	virtual int   SkipRecord(PGLOBAL g, bool header) {return RC_OK;}
+//virtual bool  OpenTableFile(PGLOBAL g);
+	virtual int   ReadBuffer(PGLOBAL g);
+	virtual int   WriteBuffer(PGLOBAL g);
+//virtual int   DeleteRecords(PGLOBAL g, int irc);
+//virtual void  CloseTableFile(PGLOBAL g, bool abort);
+//virtual void  Rewind(void);
+
+//protected:
+//virtual int   InitDelete(PGLOBAL g, int fpos, int spos);
+
+	// Members
+	size_t Recsize;		// Length of last read or next written record
+}; // end of class BINFAM
+
 #endif // __FILAMTXT_H

storage/connect/filamvct.cpp

@@ -5,7 +5,7 @@
 /*                                                                     */
 /* COPYRIGHT:                                                          */
 /* ----------                                                          */
-/*  (C) Copyright to the author Olivier BERTRAND          2005-2017    */
+/*  (C) Copyright to the author Olivier BERTRAND          2005-2020    */
 /*                                                                     */
 /* WHAT THIS PROGRAM DOES:                                             */
 /* -----------------------                                             */
@@ -1327,7 +1327,7 @@ VCMFAM::VCMFAM(PVCMFAM txfp) : VCTFAM(txfp)
 bool VCMFAM::OpenTableFile(PGLOBAL g)
   {
   char    filename[_MAX_PATH];
-  int    len;
+  size_t  len;
   MODE    mode = Tdbp->GetMode();
   PFBLOCK fp = NULL;
   PDBUSER dbuserp = (PDBUSER)g->Activityp->Aptr;
@@ -1421,9 +1421,13 @@ bool VCMFAM::OpenTableFile(PGLOBAL g)
       } // endif hFile
 
     /*******************************************************************/
-    /*  Get the file size (assuming file is smaller than 4 GB)         */
+    /*  Get the file size.                                             */
     /*******************************************************************/
-    len = mm.lenL;
+		len = (size_t)mm.lenL;
+
+		if (mm.lenH)
+			len += ((size_t)mm.lenH * 0x000000001LL);
+
 		Memory = (char *)mm.memory;
 
     if (!len) {             // Empty or deleted file
@@ -2762,7 +2766,7 @@ bool VMPFAM::OpenTableFile(PGLOBAL g)
 bool VMPFAM::MapColumnFile(PGLOBAL g, MODE mode, int i)
   {
   char    filename[_MAX_PATH];
-  int    len;
+  size_t  len;
   HANDLE  hFile;
   MEMMAP  mm;
   PFBLOCK fp;
@@ -2816,7 +2820,11 @@ bool VMPFAM::MapColumnFile(PGLOBAL g, MODE mode, int i)
     /*****************************************************************/
     /*  Get the file size (assuming file is smaller than 4 GB)       */
     /*****************************************************************/
-    len = mm.lenL;
+		len = (size_t)mm.lenL;
+
+		if (mm.lenH)
+			len += ((size_t)mm.lenH * 0x000000001LL);
+
 		Memcol[i] = (char *)mm.memory;
 
     if (!len) {             // Empty or deleted file

storage/connect/global.h

@@ -185,7 +185,7 @@ typedef struct _global {            /* Global structure                */
   size_t    Sarea_Size;             /* Work area size                  */
 	PACTIVITY Activityp;
   char      Message[MAX_STR];				/* Message (result, error, trace)  */
-	ulong     More;										/* Used by jsonudf                 */
+	size_t    More;										/* Used by jsonudf                 */
 	size_t    Saved_Size;             /* Saved work area to_free         */
 	bool      Createas;               /* To pass multi to ext tables     */
   void     *Xchk;                   /* indexes in create/alter         */
@@ -208,7 +208,7 @@ DllExport char   *PlugGetMessage(PGLOBAL, int);
 DllExport short   GetLineLength(PGLOBAL);   // Console line length
 #endif   // __WIN__
 DllExport PGLOBAL PlugInit(LPCSTR, size_t); // Plug global initialization
-DllExport int     PlugExit(PGLOBAL);        // Plug global termination
+DllExport PGLOBAL PlugExit(PGLOBAL);        // Plug global termination
 DllExport LPSTR   PlugRemoveType(LPSTR, LPCSTR);
 DllExport LPCSTR  PlugSetPath(LPSTR to, LPCSTR prefix, LPCSTR name, LPCSTR dir);
 DllExport BOOL    PlugIsAbsolutePath(LPCSTR path);
@@ -220,30 +220,11 @@ DllExport char   *PlugDup(PGLOBAL g, const char *str);
 DllExport void    htrc(char const *fmt, ...);
 DllExport void    xtrc(uint, char const* fmt, ...);
 DllExport uint    GetTraceValue(void);
+DllExport void*   MakePtr(void* memp, size_t offset);
+DllExport size_t  MakeOff(void* memp, void* ptr);
 
 #if defined(__cplusplus)
 } // extern "C"
 #endif
 
-/***********************************************************************/
-/*  Inline routine definitions.                                        */
-/***********************************************************************/
-/***********************************************************************/
-/* This routine makes a pointer from an offset to a memory pointer.    */
-/***********************************************************************/
-inline void* MakePtr(void* memp, size_t offset) {
-	// return ((offset == 0) ? NULL : &((char*)memp)[offset]);
-	return (!offset) ? NULL : (char *)memp + offset;
-} /* end of MakePtr */
-
-/***********************************************************************/
-/* This routine makes an offset from a pointer new format.             */
-/***********************************************************************/
-inline size_t MakeOff(void* memp, void* ptr) {
-#if defined(_DEBUG)
-	assert(ptr > memp);
-#endif   // _DEBUG
-	return ((!ptr) ? 0 : (size_t)((char*)ptr - (size_t)memp));
-} /* end of MakeOff */
-
 /*-------------------------- End of Global.H --------------------------*/

storage/connect/jdbconn.cpp

@@ -766,6 +766,7 @@ void JDBConn::AddJars(PSTRG jpop, char sep)
 /***********************************************************************/
 bool JDBConn::Connect(PJPARM sop)
 {
+	int      irc = RC_FX;
 	bool		 err = false;
 	jint     rc;
 	jboolean jt = (trace(1));

storage/connect/jmgfam.cpp

 /************ JMONGO FAM C++ Program Source Code File (.CPP) ***********/
 /* PROGRAM NAME: jmgfam.cpp                                            */
 /* -------------                                                       */
-/*  Version 1.0                                                        */
+/*  Version 1.1                                                        */
 /*                                                                     */
 /* COPYRIGHT:                                                          */
 /* ----------                                                          */
-/*  (C) Copyright to the author Olivier BERTRAND          20017        */
+/*  (C) Copyright to the author Olivier BERTRAND          20017 - 2020 */
 /*                                                                     */
 /* WHAT THIS PROGRAM DOES:                                             */
 /* -----------------------                                             */
@@ -49,7 +49,11 @@
 #include "reldef.h"
 #include "filamtxt.h"
 #include "tabdos.h"
+#if defined(BSON_SUPPORT)
+#include "tabbson.h"
+#else
 #include "tabjson.h"
+#endif   // BSON_SUPPORT
 #include "jmgfam.h"
 
 #if defined(UNIX) || defined(UNIV_LINUX)
@@ -92,10 +96,38 @@ JMGFAM::JMGFAM(PJDEF tdp) : DOSFAM((PDOSDEF)NULL)
 	Version = tdp->Version;
 	Lrecl = tdp->Lrecl + tdp->Ending;
 	Curpos = 0;
-} // end of JMGFAM standard constructor
+} // end of JMGFAM Json standard constructor
+
+#if defined(BSON_SUPPORT)
+JMGFAM::JMGFAM(PBDEF tdp) : DOSFAM((PDOSDEF)NULL)
+{
+	Jcp = NULL;
+	Ops.Driver = tdp->Schema;
+	Ops.Url = tdp->Uri;
+	Ops.User = NULL;
+	Ops.Pwd = NULL;
+	Ops.Scrollable = false;
+	Ops.Fsize = 0;
+	Ops.Version = tdp->Version;
+	To_Fbt = NULL;
+	Mode = MODE_ANY;
+	Uristr = tdp->Uri;
+	Db_name = tdp->Schema;
+	Coll_name = tdp->Collname;
+	Options = tdp->Options;
+	Filter = tdp->Filter;
+	Wrapname = tdp->Wrapname;
+	Done = false;
+	Pipe = tdp->Pipe;
+	Version = tdp->Version;
+	Lrecl = tdp->Lrecl + tdp->Ending;
+	Curpos = 0;
+} // end of JMGFAM Bson standard constructor
+#endif   // BSON_SUPPORT
 
 JMGFAM::JMGFAM(PJMGFAM tdfp) : DOSFAM(tdfp)
 {
+	Jcp = tdfp->Jcp;
 	//Client = tdfp->Client;
 	//Database = NULL;
 	//Collection = tdfp->Collection;
@@ -114,6 +146,7 @@ JMGFAM::JMGFAM(PJMGFAM tdfp) : DOSFAM(tdfp)
 	Done = tdfp->Done;
 	Pipe = tdfp->Pipe;
 	Version = tdfp->Version;
+	Curpos = tdfp->Curpos;
 } // end of JMGFAM copy constructor
 
 /***********************************************************************/

storage/connect/jmgfam.h

 /************** MongoFam H Declares Source Code File (.H) **************/
-/*  Name: jmgfam.h    Version 1.0                                      */
+/*  Name: jmgfam.h    Version 1.1                                      */
 /*                                                                     */
-/*  (C) Copyright to the author Olivier BERTRAND          2017         */
+/*  (C) Copyright to the author Olivier BERTRAND          2017 - 2020  */
 /*                                                                     */
 /*  This file contains the JAVA MongoDB access method classes declares */
 /***********************************************************************/
@@ -25,6 +25,9 @@ class DllExport JMGFAM : public DOSFAM {
 public:
 	// Constructor
 	JMGFAM(PJDEF tdp);
+#if defined(BSON_SUPPORT)
+	JMGFAM(PBDEF tdp);
+#endif   // BSON_SUPPORT
 	JMGFAM(PJMGFAM txfp);
 
 	// Implementation

storage/connect/jmgoconn.cpp

@@ -121,7 +121,7 @@ JMgoConn::JMgoConn(PGLOBAL g, PCSZ collname, PCSZ wrapper)
 /***********************************************************************/
 void JMgoConn::AddJars(PSTRG jpop, char sep)
 {
-#if defined(DEVELOPMENT)
+#if defined(BSON_SUPPORT)
 	if (m_Version == 2) {
 		jpop->Append(sep);
 //	jpop->Append("C:/Eclipse/workspace/MongoWrap2/bin");
@@ -134,7 +134,7 @@ void JMgoConn::AddJars(PSTRG jpop, char sep)
 		jpop->Append(sep);
 		jpop->Append("C:/mongo-java-driver/mongo-java-driver-3.4.2.jar");
 	} // endif m_Version
-#endif   // DEVELOPMENT
+#endif   // BSON_SUPPORT
 } // end of AddJars
 
 /***********************************************************************/

storage/connect/jsonudf.h

 /******************** tabjson H Declares Source Code File (.H) *******************/
-/*  Name: jsonudf.h   Version 1.3                                                */
+/*  Name: jsonudf.h   Version 1.4                                                */
 /*                                                                               */
-/*  (C) Copyright to the author Olivier BERTRAND          2015-2017              */
+/*  (C) Copyright to the author Olivier BERTRAND          2015-2020              */
 /*                                                                               */
 /*  This file contains the JSON UDF function and class declares.                 */
 /*********************************************************************************/
+#pragma once
 #include "global.h"
 #include "plgdbsem.h"
 #include "block.h"
@@ -15,6 +16,27 @@
 #define UDF_EXEC_ARGS \
   UDF_INIT*, UDF_ARGS*, char*, unsigned long*, char*, char*
 
+// BSON size should be equal on Linux and Windows
+#define BMX 255
+typedef struct BSON* PBSON;
+
+/***********************************************************************/
+/*  Structure used to return binary json to Json UDF functions.        */
+/***********************************************************************/
+struct BSON {
+	char    Msg[BMX + 1];
+	char   *Filename;
+	PGLOBAL G;
+	int     Pretty;
+	ulong   Reslen;
+	my_bool Changed;
+	PJSON   Top;
+	PJSON   Jsp;
+	PBSON   Bsp;
+}; // end of struct BSON
+
+PBSON JbinAlloc(PGLOBAL g, UDF_ARGS* args, ulong len, PJSON jsp);
+
 /*********************************************************************************/
 /*  The JSON tree node. Can be an Object or an Array.                     	  	 */
 /*********************************************************************************/
@@ -29,9 +51,29 @@ typedef struct _jnode {
 } JNODE, *PJNODE;
 
 typedef class JSNX     *PJSNX;
-typedef class JOUTPATH *PJTP;
-typedef class JOUTALL  *PJTA;
 
+/*********************************************************************************/
+/*  The JSON utility functions.                     	  	                       */
+/*********************************************************************************/
+bool    IsNum(PSZ s);
+char   *NextChr(PSZ s, char sep);
+char   *GetJsonNull(void);
+uint    GetJsonGrpSize(void);
+my_bool JsonSubSet(PGLOBAL g, my_bool b = false);
+my_bool CalcLen(UDF_ARGS* args, my_bool obj, unsigned long& reslen,
+	              unsigned long& memlen, my_bool mod = false);
+my_bool JsonInit(UDF_INIT* initid, UDF_ARGS* args, char* message, my_bool mbn,
+								 unsigned long reslen, unsigned long memlen,
+								 unsigned long more = 0);
+my_bool CheckMemory(PGLOBAL g, UDF_INIT* initid, UDF_ARGS* args, uint n,
+	                  my_bool m, my_bool obj = false, my_bool mod = false);
+PSZ     MakePSZ(PGLOBAL g, UDF_ARGS* args, int i);
+int     IsJson(UDF_ARGS* args, uint i, bool b = false);
+char   *GetJsonFile(PGLOBAL g, char* fn);
+
+/*********************************************************************************/
+/*  The JSON UDF functions.                                               	  	 */
+/*********************************************************************************/
 extern "C" {
 	DllExport my_bool jsonvalue_init(UDF_INIT*, UDF_ARGS*, char*);
 	DllExport char *jsonvalue(UDF_EXEC_ARGS);
@@ -132,7 +174,7 @@ extern "C" {
 	DllExport void jsonget_real_deinit(UDF_INIT*);
 
 	DllExport my_bool jsoncontains_init(UDF_INIT*, UDF_ARGS*, char*);
-	DllExport long long jsoncontains(UDF_EXEC_ARGS);
+	DllExport long long jsoncontains(UDF_INIT*, UDF_ARGS*, char*, char*);
 	DllExport void jsoncontains_deinit(UDF_INIT*);
 
 	DllExport my_bool jsonlocate_init(UDF_INIT*, UDF_ARGS*, char*);
@@ -144,7 +186,7 @@ extern "C" {
 	DllExport void json_locate_all_deinit(UDF_INIT*);
 
 	DllExport my_bool jsoncontains_path_init(UDF_INIT*, UDF_ARGS*, char*);
-	DllExport long long jsoncontains_path(UDF_EXEC_ARGS);
+	DllExport long long jsoncontains_path(UDF_INIT*, UDF_ARGS*, char*, char*);
 	DllExport void jsoncontains_path_deinit(UDF_INIT*);
 
 	DllExport my_bool json_set_item_init(UDF_INIT*, UDF_ARGS*, char*);
@@ -239,6 +281,10 @@ extern "C" {
 	DllExport char* jfile_convert(UDF_EXEC_ARGS);
 	DllExport void jfile_convert_deinit(UDF_INIT*);
 
+	DllExport my_bool jfile_bjson_init(UDF_INIT*, UDF_ARGS*, char*);
+	DllExport char* jfile_bjson(UDF_EXEC_ARGS);
+	DllExport void jfile_bjson_deinit(UDF_INIT*);
+
 	DllExport my_bool envar_init(UDF_INIT*, UDF_ARGS*, char*);
 	DllExport char *envar(UDF_EXEC_ARGS);
 
@@ -248,7 +294,7 @@ extern "C" {
 #endif   // DEVELOPMENT
 
 	DllExport my_bool countin_init(UDF_INIT*, UDF_ARGS*, char*);
-	DllExport long long countin(UDF_EXEC_ARGS);
+	DllExport long long countin(UDF_INIT*, UDF_ARGS*, char*, char*);
 } // extern "C"																											  
 
 
@@ -256,7 +302,7 @@ extern "C" {
 /*  Structure JPN. Used to make the locate path.                                 */
 /*********************************************************************************/
 typedef struct _jpn {
-	enum JTYP Type;
+	int  Type;
 	PCSZ Key;
 	int  N;
 } JPN, *PJPN;
@@ -290,15 +336,16 @@ class JSNX : public BLOCK {
 	PVAL    ExpandArray(PGLOBAL g, PJAR arp, int n);
 	PVAL    CalculateArray(PGLOBAL g, PJAR arp, int n);
 	PVAL    MakeJson(PGLOBAL g, PJSON jsp);
-	void    SetJsonValue(PGLOBAL g, PVAL vp, PJVAL val, int n);
+	void    SetJsonValue(PGLOBAL g, PVAL vp, PJVAL val);
 	PJSON   GetRow(PGLOBAL g);
-	my_bool LocateArray(PJAR jarp);
-	my_bool LocateObject(PJOB jobp);
-	my_bool LocateValue(PJVAL jvp);
-	my_bool LocateArrayAll(PJAR jarp);
-	my_bool LocateObjectAll(PJOB jobp);
-	my_bool LocateValueAll(PJVAL jvp);
-	my_bool CompareTree(PJSON jp1, PJSON jp2);
+	my_bool CompareValues(PJVAL v1, PJVAL v2);
+	my_bool LocateArray(PGLOBAL g, PJAR jarp);
+	my_bool LocateObject(PGLOBAL g, PJOB jobp);
+	my_bool LocateValue(PGLOBAL g, PJVAL jvp);
+	my_bool LocateArrayAll(PGLOBAL g, PJAR jarp);
+	my_bool LocateObjectAll(PGLOBAL g, PJOB jobp);
+	my_bool LocateValueAll(PGLOBAL g, PJVAL jvp);
+	my_bool CompareTree(PGLOBAL g, PJSON jp1, PJSON jp2);
 	my_bool AddPath(void);
 
 	// Default constructor not to be used
@@ -355,11 +402,10 @@ class JUP : public BLOCK {
 	void  CopyNumeric(PGLOBAL g);
 
 	// Members
-	FILE* fs;
-	char* s;
-	char* buff;
-	int len;
-	int recl;
-	int i, k;
+	FILE  *fs;
+	char  *s;
+	char  *buff;
+	size_t len;
+	uint   i;
+	int    k, recl;
 }; // end of class JUP
-

storage/connect/mycat.cc

@@ -16,9 +16,9 @@
 /*************** Mycat CC Program Source Code File (.CC) ***************/
 /* PROGRAM NAME: MYCAT                                                 */
 /* -------------                                                       */
-/*  Version 1.7                                                        */
+/*  Version 1.8                                                        */
 /*                                                                     */
-/*  Author: Olivier Bertrand                       2012 - 2019         */
+/*  Author: Olivier Bertrand                       2012 - 2020         */
 /*                                                                     */
 /* WHAT THIS PROGRAM DOES:                                             */
 /* -----------------------                                             */
@@ -82,7 +82,11 @@
 #endif   // JAVA_SUPPORT
 #include "tabpivot.h"
 #include "tabvir.h"
+#if defined(BSON_SUPPORT)
+#include "tabbson.h"
+#else
 #include "tabjson.h"
+#endif   // BSON_SUPPORT
 #include "ha_connect.h"
 #if defined(XML_SUPPORT)
 #include "tabxml.h"
@@ -107,6 +111,9 @@ extern "C" HINSTANCE s_hModule;           // Saved module handle
 #if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT)
 bool MongoEnabled(void);
 #endif   // JAVA_SUPPORT || CMGO_SUPPORT
+#if defined(BSON_SUPPORT)
+bool Force_Bson(void);
+#endif   // BSON_SUPPORT
 
 /***********************************************************************/
 /*  Get the plugin directory.                                          */
@@ -130,25 +137,25 @@ TABTYPE GetTypeID(const char *type)
                  : (!stricmp(type, "DBF"))   ? TAB_DBF
 #if defined(XML_SUPPORT)
                  : (!stricmp(type, "XML"))   ? TAB_XML
-#endif
+#endif   // XML_SUPPORT
                  : (!stricmp(type, "INI"))   ? TAB_INI
                  : (!stricmp(type, "VEC"))   ? TAB_VEC
 #if defined(ODBC_SUPPORT)
                  : (!stricmp(type, "ODBC"))  ? TAB_ODBC
-#endif
+#endif   // ODBC_SUPPORT
 #if defined(JAVA_SUPPORT)
                  : (!stricmp(type, "JDBC"))  ? TAB_JDBC
-#endif
+#endif   // JAVA_SUPPORT
 #if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT)
                  : (!stricmp(type, "MONGO") && MongoEnabled()) ? TAB_MONGO
-#endif
+#endif   // JAVA_SUPPORT  ||         CMGO_SUPPORT
                  : (!stricmp(type, "MYSQL")) ? TAB_MYSQL
                  : (!stricmp(type, "MYPRX")) ? TAB_MYSQL
                  : (!stricmp(type, "DIR"))   ? TAB_DIR
 #if defined(__WIN__)
                  : (!stricmp(type, "MAC"))   ? TAB_MAC
                  : (!stricmp(type, "WMI"))   ? TAB_WMI
-#endif
+#endif   // __WIN__
                  : (!stricmp(type, "TBL"))   ? TAB_TBL
                  : (!stricmp(type, "XCOL"))  ? TAB_XCL
                  : (!stricmp(type, "OCCUR")) ? TAB_OCCUR
@@ -157,9 +164,12 @@ TABTYPE GetTypeID(const char *type)
                  : (!stricmp(type, "PIVOT")) ? TAB_PIVOT
                  : (!stricmp(type, "VIR"))   ? TAB_VIR
                  : (!stricmp(type, "JSON"))  ? TAB_JSON
+#if defined(BSON_SUPPORT)
+                 : (!stricmp(type, "BSON"))  ? TAB_BSON
+#endif   // BSON_SUPPORT
 #if defined(ZIP_SUPPORT)
                  : (!stricmp(type, "ZIP"))   ? TAB_ZIP
-#endif
+#endif   // ZIP_SUPPORT
                  : (!stricmp(type, "OEM"))   ? TAB_OEM : TAB_NIY;
   } // end of GetTypeID
 
@@ -181,6 +191,9 @@ bool IsFileType(TABTYPE type)
     case TAB_INI:
     case TAB_VEC:
     case TAB_JSON:
+#if defined(BSON_SUPPORT)
+    case TAB_BSON:
+#endif   // BSON_SUPPORT
     case TAB_REST:
  // case TAB_ZIP:
       isfile= true;
@@ -276,6 +289,9 @@ bool IsTypeIndexable(TABTYPE type)
     case TAB_VEC:
     case TAB_DBF:
     case TAB_JSON:
+#if defined(BSON_SUPPORT)
+    case TAB_BSON:
+#endif   // BSON_SUPPORT
       idx= true;
       break;
     default:
@@ -302,6 +318,9 @@ int GetIndexType(TABTYPE type)
     case TAB_VEC:
     case TAB_DBF:
     case TAB_JSON:
+#if defined(BSON_SUPPORT)
+    case TAB_BSON:
+#endif   // BSON_SUPPORT
       xtyp= 1;
       break;
     case TAB_MYSQL:
@@ -445,7 +464,7 @@ PTABDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am)
     case TAB_XML: tdp= new(g) XMLDEF;   break;
 #endif   // XML_SUPPORT
 #if defined(VCT_SUPPORT)
-    case TAB_VEC: tdp = new(g) VCTDEF;  break;
+    case TAB_VEC: tdp= new(g) VCTDEF;  break;
 #endif   // VCT_SUPPORT
 #if defined(ODBC_SUPPORT)
     case TAB_ODBC: tdp= new(g) ODBCDEF; break;
@@ -465,9 +484,20 @@ PTABDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am)
     case TAB_MYSQL: tdp= new(g) MYSQLDEF; break;
     case TAB_PIVOT: tdp= new(g) PIVOTDEF; break;
     case TAB_VIR: tdp= new(g) VIRDEF;   break;
-    case TAB_JSON: tdp= new(g) JSONDEF; break;
+    case TAB_JSON:
+#if defined(BSON_SUPPORT)
+      if (Force_Bson())
+        tdp= new(g) BSONDEF;
+      else
+#endif   // BSON_SUPPORT
+        tdp= new(g) JSONDEF;
+
+      break;
+#if defined(BSON_SUPPORT)
+    case TAB_BSON: tdp= new(g) BSONDEF; break;
+#endif   // BSON_SUPPORT
 #if defined(ZIP_SUPPORT)
-    case TAB_ZIP: tdp = new(g) ZIPDEF;   break;
+    case TAB_ZIP: tdp= new(g) ZIPDEF;   break;
 #endif   // ZIP_SUPPORT
 #if defined(REST_SUPPORT)
     case TAB_REST: tdp= new (g) RESTDEF; break;

storage/connect/mysql-test/connect/disabled.def

@@ -16,9 +16,12 @@ jdbc_postgresql : Variable settings depend on machine configuration
 json_mongo_c    : Need MongoDB running and its C Driver installed
 json_java_2     : Need MongoDB running and its Java Driver installed
 json_java_3     : Need MongoDB running and its Java Driver installed
+bson_mongo_c    : Need MongoDB running and its C Driver installed
+bson_java_2     : Need MongoDB running and its Java Driver installed
+bson_java_3     : Need MongoDB running and its Java Driver installed
 mongo_c         : Need MongoDB running and its C Driver installed
 mongo_java_2    : Need MongoDB running and its Java Driver installed
 mongo_java_3    : Need MongoDB running and its Java Driver installed
 tbl_thread      : Bug MDEV-9844,10179,14214 03/01/2018 OB Option THREAD removed
-grant2          : Until fixed
+#bson            : Development
 #vcol            : Different error code on different versions

storage/connect/mysql-test/connect/r/alter_xml.result

@@ -54,7 +54,7 @@ line
 </t1>
 # NOTE: The first (ignored) row is due to the remaining HEADER=1 option.
 # Testing field option modification
-ALTER TABLE t1 MODIFY d CHAR(10) NOT NULL FIELD_FORMAT='@', HEADER=0;
+ALTER TABLE t1 MODIFY d CHAR(10) NOT NULL XPATH='@', HEADER=0;
 SELECT * FROM t1;
 c	d
 1	One
@@ -64,7 +64,7 @@ SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
   `c` int(11) NOT NULL,
-  `d` char(10) NOT NULL `FIELD_FORMAT`='@'
+  `d` char(10) NOT NULL `XPATH`='@'
 ) ENGINE=CONNECT DEFAULT CHARSET=latin1 `QUOTED`=1 `TABLE_TYPE`=XML `TABNAME`=t1 `OPTION_LIST`='xmlsup=domdoc,rownode=row' `HEADER`=0
 SELECT * FROM t2;
 line

storage/connect/mysql-test/connect/r/alter_xml2.result

@@ -56,7 +56,7 @@ line
 </t1>
 # NOTE: The first (ignored) row is due to the remaining HEADER=1 option.
 # Testing field option modification
-ALTER TABLE t1 MODIFY d CHAR(10) NOT NULL FIELD_FORMAT='@', HEADER=0;
+ALTER TABLE t1 MODIFY d CHAR(10) NOT NULL XPATH='@', HEADER=0;
 SELECT * FROM t1;
 c	d
 1	One
@@ -66,7 +66,7 @@ SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
   `c` int(11) NOT NULL,
-  `d` char(10) NOT NULL `FIELD_FORMAT`='@'
+  `d` char(10) NOT NULL `XPATH`='@'
 ) ENGINE=CONNECT DEFAULT CHARSET=latin1 `QUOTED`=1 `TABLE_TYPE`=XML `TABNAME`=t1 `OPTION_LIST`='xmlsup=libxml2,rownode=row' `HEADER`=0
 SELECT * FROM t2;
 line

storage/connect/mysql-test/connect/r/jdbc_oracle.result

@@ -3,7 +3,7 @@ command varchar(128) not null,
 number int(5) not null flag=1,
 message varchar(255) flag=2)
 ENGINE=CONNECT TABLE_TYPE=JDBC CONNECTION='jdbc:oracle:thin:@localhost:1521:xe'
-OPTION_LIST='User=system,Password=manager,Execsrc=1';
+OPTION_LIST='User=system,Password=Choupy01,Execsrc=1';
 SELECT * FROM t2 WHERE command = 'drop table employee';
 command	number	message
 drop table employee	0	Execute: java.sql.SQLSyntaxErrorException: ORA-00942: table or view does not exist
@@ -23,14 +23,14 @@ Warnings:
 Warning	1105	Affected rows
 CREATE TABLE t1 ENGINE=CONNECT TABLE_TYPE=JDBC CATFUNC=tables
 CONNECTION='jdbc:oracle:thin:@localhost:1521:xe'
-OPTION_LIST='User=system,Password=manager';
+OPTION_LIST='User=system,Password=Choupy01';
 SELECT * FROM t1 WHERE table_name='employee';
 Table_Cat	Table_Schema	Table_Name	Table_Type	Remark
 NULL	SYSTEM	EMPLOYEE	TABLE	NULL
 DROP TABLE t1;
 CREATE TABLE t1 ENGINE=CONNECT TABLE_TYPE=JDBC TABNAME='EMPLOYEE' CATFUNC=columns
 CONNECTION='jdbc:oracle:thin:@localhost:1521:xe'
-OPTION_LIST='User=system,Password=manager';
+OPTION_LIST='User=system,Password=Choupy01';
 SELECT * FROM t1;
 Table_Cat	Table_Schema	Table_Name	Column_Name	Data_Type	Type_Name	Column_Size	Buffer_Length	Decimal_Digits	Radix	Nullable	Remarks
 NULL	SYSTEM	EMPLOYEE	ID	3	NUMBER	38	0	0	10	0	NULL
@@ -42,7 +42,7 @@ CREATE SERVER 'oracle' FOREIGN DATA WRAPPER 'oracle.jdbc.driver.OracleDriver' OP
 HOST 'jdbc:oracle:thin:@localhost:1521:xe',
 DATABASE 'SYSTEM',
 USER 'system',
-PASSWORD 'manager',
+PASSWORD 'Choupy01',
 PORT 0,
 SOCKET '',
 OWNER 'SYSTEM');

storage/connect/mysql-test/connect/r/json_udf.result

@@ -187,11 +187,11 @@ DATEPUB int(4)
 ) ENGINE=CONNECT TABLE_TYPE=JSON FILE_NAME='biblio.json';
 SELECT Json_Make_Array(AUTHOR, TITLE, DATEPUB) FROM t2;
 Json_Make_Array(AUTHOR, TITLE, DATEPUB)
-["Jean-Christophe Bernadac","Construire une application XML",1999]
+[" Jean-Christophe Bernadac, Franois Knab","Construire une application XML",1999]
 ["William J. Pardi","XML en Action",1999]
 SELECT Json_Make_Object(AUTHOR, TITLE, DATEPUB) FROM t2;
 Json_Make_Object(AUTHOR, TITLE, DATEPUB)
-{"AUTHOR":"Jean-Christophe Bernadac","TITLE":"Construire une application XML","DATEPUB":1999}
+{"AUTHOR":" Jean-Christophe Bernadac, Franois Knab","TITLE":"Construire une application XML","DATEPUB":1999}
 {"AUTHOR":"William J. Pardi","TITLE":"XML en Action","DATEPUB":1999}
 SELECT Json_Array_Grp(TITLE, DATEPUB) FROM t2;
 ERROR HY000: Can't initialize function 'json_array_grp'; This function can only accept 1 argument
@@ -610,7 +610,7 @@ JsonGet_String(Json_File('test/fx.json'), '1.*')
 {"_id":6,"type":"car","item":"roadster","mileage":56000,"ratings":[6,9]}
 SELECT JsonGet_String(Json_File('test/fx.json'), '1');
 JsonGet_String(Json_File('test/fx.json'), '1')
-6 car roadster 56000 6 9
+6 car roadster 56000 (6, 9)
 SELECT JsonGet_Int(Json_File('test/fx.json'), '1.mileage') AS Mileage;
 Mileage
 56000

storage/connect/mysql-test/connect/r/json_udf_bin.result

@@ -87,7 +87,7 @@ Json_Get_Item(Jbin_File('gloss.json'),'$.glossary.GlossDiv')
 {"title":"S","GlossList":{"GlossEntry":{"ID":"SGML","SortAs":"SGML","GlossTerm":"Standard Generalized Markup Language","Acronym":"SGML","Abbrev":"ISO 8879:1986","GlossDef":{"para":"A meta-markup language, used to create markup languages such as DocBook.","GlossSeeAlso":["GML","XML"]},"GlossSee":"markup"}}}
 SELECT JsonGet_String(Json_File('gloss.json'),'$.glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso') lang;
 lang
-GML
+GML, XML
 SELECT Json_Get_Item(Jbin_File('gloss.json'),'$.glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso') "See also";
 See also
 ["GML","XML"]

storage/connect/mysql-test/connect/r/rest.result

+#
+# Testing REST query
+#
+CREATE TABLE t1
+ENGINE=CONNECT DATA_CHARSET=utf8 TABLE_TYPE=JSON FILE_NAME='users.json'
+HTTP='http://jsonplaceholder.typicode.com/users';
+SELECT * FROM t1;
+id	name	username	email	address_street	address_suite	address_city	address_zipcode	address_geo_lat	address_geo_lng	phone	website	company_name	company_catchPhrase	company_bs
+1	Leanne Graham	Bret	Sincere@april.biz	Kulas Light	Apt. 556	Gwenborough	92998-3874	-37.3159	81.1496	1-770-736-8031 x56442	hildegard.org	Romaguera-Crona	Multi-layered client-server neural-net	harness real-time e-markets
+2	Ervin Howell	Antonette	Shanna@melissa.tv	Victor Plains	Suite 879	Wisokyburgh	90566-7771	-43.9509	-34.4618	010-692-6593 x09125	anastasia.net	Deckow-Crist	Proactive didactic contingency	synergize scalable supply-chains
+3	Clementine Bauch	Samantha	Nathan@yesenia.net	Douglas Extension	Suite 847	McKenziehaven	59590-4157	-68.6102	-47.0653	1-463-123-4447	ramiro.info	Romaguera-Jacobson	Face to face bifurcated interface	e-enable strategic applications
+4	Patricia Lebsack	Karianne	Julianne.OConner@kory.org	Hoeger Mall	Apt. 692	South Elvis	53919-4257	29.4572	-164.2990	493-170-9623 x156	kale.biz	Robel-Corkery	Multi-tiered zero tolerance productivity	transition cutting-edge web services
+5	Chelsey Dietrich	Kamren	Lucio_Hettinger@annie.ca	Skiles Walks	Suite 351	Roscoeview	33263	-31.8129	62.5342	(254)954-1289	demarco.info	Keebler LLC	User-centric fault-tolerant solution	revolutionize end-to-end systems
+6	Mrs. Dennis Schulist	Leopoldo_Corkery	Karley_Dach@jasper.info	Norberto Crossing	Apt. 950	South Christy	23505-1337	-71.4197	71.7478	1-477-935-8478 x6430	ola.org	Considine-Lockman	Synchronised bottom-line interface	e-enable innovative applications
+7	Kurtis Weissnat	Elwyn.Skiles	Telly.Hoeger@billy.biz	Rex Trail	Suite 280	Howemouth	58804-1099	24.8918	21.8984	210.067.6132	elvis.io	Johns Group	Configurable multimedia task-force	generate enterprise e-tailers
+8	Nicholas Runolfsdottir V	Maxime_Nienow	Sherwood@rosamond.me	Ellsworth Summit	Suite 729	Aliyaview	45169	-14.3990	-120.7677	586.493.6943 x140	jacynthe.com	Abernathy Group	Implemented secondary concept	e-enable extensible e-tailers
+9	Glenna Reichert	Delphine	Chaim_McDermott@dana.io	Dayna Park	Suite 449	Bartholomebury	76495-3109	24.6463	-168.8889	(775)976-6794 x41206	conrad.com	Yost and Sons	Switchable contextually-based project	aggregate real-time technologies
+10	Clementina DuBuque	Moriah.Stanton	Rey.Padberg@karina.biz	Kattie Turnpike	Suite 198	Lebsackbury	31428-2261	-38.2386	57.2232	024-648-3804	ambrose.net	Hoeger LLC	Centralized empowering task-force	target end-to-end models
+DROP TABLE t1;

storage/connect/mysql-test/connect/r/xml.result

@@ -374,7 +374,7 @@ INSERT INTO t1 VALUES (_cp1251 0xC0C1C2C3);
 Warnings:
 Level	Warning
 Code	1105
-Message	Com error: Unable to save character to 'iso-8859-1' encoding.
+Message	Com error: Impossible d'enregistrer le caractre dans le codage iso-8859-1.
 
 INSERT INTO t1 VALUES ('&<>"\'');
 SELECT node, hex(node) FROM t1;

storage/connect/mysql-test/connect/r/xml2_html.result

@@ -5,9 +5,9 @@ SET NAMES utf8;
 # Testing HTML like XML file
 #
 CREATE TABLE beers (
-`Name` CHAR(16) FIELD_FORMAT='brandName',
-`Origin` CHAR(16) FIELD_FORMAT='origin',
-`Description` CHAR(32) FIELD_FORMAT='details')
+`Name` CHAR(16) XPATH='brandName',
+`Origin` CHAR(16) XPATH='origin',
+`Description` CHAR(32) XPATH='details')
 ENGINE=CONNECT TABLE_TYPE=XML FILE_NAME='beers.xml'
 TABNAME='table' OPTION_LIST='xmlsup=libxml2,rownode=tr,colnode=td';
 SELECT * FROM beers;

storage/connect/mysql-test/connect/r/xml2_mult.result

@@ -5,9 +5,9 @@ SET NAMES utf8;
 # Testing expanded values
 #
 CREATE TABLE `bookstore` (
-`category` CHAR(16) NOT NULL FIELD_FORMAT='@',
+`category` CHAR(16) NOT NULL XPATH='@',
 `title` VARCHAR(50) NOT NULL,
-`lang` char(2) NOT NULL FIELD_FORMAT='title/@',
+`lang` char(2) NOT NULL XPATH='title/@',
 `author` VARCHAR(24) NOT NULL,
 `year` INT(4) NOT NULL,
 `price` DOUBLE(8,2) NOT NULL)