Merge XPand Storage Engine (rebased)

mysql-test/suite/xpand/basics.result

+CREATE DATABASE xpd;
+USE xpd;
+DROP TABLE IF EXISTS cx1, t1, t2;
+CREATE TABLE cx1(i BIGINT)ENGINE=xpand;
+CREATE TABLE cx1(i BIGINT)ENGINE=xpand;
+ERROR 42S01: Table 'cx1' already exists
+INSERT INTO cx1 VALUES (42);
+SELECT * FROM cx1;
+i
+42
+DROP TABLE cx1;
+SHOW CREATE TABLE cx1;
+ERROR 42S02: Table 'xpd.cx1' doesn't exist
+DROP TABLE IF EXISTS intandtext;
+Warnings:
+Note	1051	Unknown table 'xpd.intandtext'
+CREATE TABLE intandtext(i bigint, t text)ENGINE=xpand;
+INSERT INTO intandtext VALUES(10, 'someqwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq');
+SELECT i,t FROM intandtext;
+i	t
+10	someqwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq
+EXPLAIN SELECT i,t FROM intandtext;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PUSHED SELECT	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL
+SET @@optimizer_switch='derived_merge=OFF';
+SET xpand_select_handler=OFF;
+SELECT i,t FROM (SELECT i,t FROM intandtext) t;
+i	t
+10	someqwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq
+EXPLAIN SELECT i,t FROM (SELECT i,t FROM intandtext) t;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	<derived2>	ALL	NULL	NULL	NULL	NULL	10000	
+2	PUSHED DERIVED	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL
+SET xpand_derived_handler=OFF;
+SELECT i,t FROM intandtext;
+i	t
+10	someqwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq
+SELECT i,t FROM (SELECT i,t FROM intandtext) t;
+i	t
+10	someqwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq
+EXPLAIN SELECT i,t FROM (SELECT i,t FROM intandtext) t;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	<derived2>	ALL	NULL	NULL	NULL	NULL	10000	
+2	DERIVED	intandtext	ALL	NULL	NULL	NULL	NULL	10000	
+DROP TABLE intandtext;
+set 
+optimizer_switch=default,
+xpand_derived_handler= default,
+xpand_select_handler=default;
+#
+# CLX-77: INSERT ... SELECT returns rows to the client instead of inserting
+#
+drop table if exists t1,t2;
+create table t1 (a int) engine=xpand;
+insert into t1 values (1);
+select a into @var from t1;
+select @var;
+@var
+1
+# This must not emit output to the client:
+select a into outfile 'tmpfile1' from t1;
+create table t2 (a int) engine=myisam;
+insert into t2 select * from t1;
+select * from t2;
+a
+1
+drop table t1,t2;
+# 
+# CLX-55: Prepared statement support: 
+#   Implement "Direct Update" by printing the statement
+# 
+create table t1 (a int primary key, b int) engine=xpand;
+insert into t1 values (1,1),(2,2),(3,3);
+prepare s from 'update t1 set b=b+? where a=?';
+execute s using 10000, 2;
+select * from t1;
+a	b
+1	1
+2	10002
+3	3
+drop table t1;
+#
+# CLX-80: ALTER TABLE t ENGINE=CLUSTRIX fails with an error
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (1),(2),(3);
+alter table t1 engine=xpand;
+select * from t1;
+a
+1
+2
+3
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(11) DEFAULT NULL
+) ENGINE=XPAND DEFAULT CHARSET=utf8
+# Try a RENAME TABLE too since the patch touches the code
+alter table t1 rename t2;
+show create table t2;
+Table	Create Table
+t2	CREATE TABLE `t2` (
+  `a` int(11) DEFAULT NULL
+) ENGINE=XPAND DEFAULT CHARSET=utf8
+drop table t2;
+USE test;
+DROP DATABASE xpd;

mysql-test/suite/xpand/basics.test

+CREATE DATABASE xpd;
+USE xpd;
+
+--disable_warnings
+DROP TABLE IF EXISTS cx1, t1, t2;
+--enable_warnings
+
+CREATE TABLE cx1(i BIGINT)ENGINE=xpand;
+--error ER_TABLE_EXISTS_ERROR
+CREATE TABLE cx1(i BIGINT)ENGINE=xpand;
+
+INSERT INTO cx1 VALUES (42);
+
+SELECT * FROM cx1;
+
+DROP TABLE cx1;
+
+--error ER_NO_SUCH_TABLE
+SHOW CREATE TABLE cx1;
+
+DROP TABLE IF EXISTS intandtext;
+CREATE TABLE intandtext(i bigint, t text)ENGINE=xpand;
+INSERT INTO intandtext VALUES(10, 'someqwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq');
+
+SELECT i,t FROM intandtext;
+EXPLAIN SELECT i,t FROM intandtext;
+
+SET @@optimizer_switch='derived_merge=OFF';
+SET xpand_select_handler=OFF;
+SELECT i,t FROM (SELECT i,t FROM intandtext) t;
+EXPLAIN SELECT i,t FROM (SELECT i,t FROM intandtext) t;
+
+SET xpand_derived_handler=OFF;
+SELECT i,t FROM intandtext;
+SELECT i,t FROM (SELECT i,t FROM intandtext) t;
+EXPLAIN SELECT i,t FROM (SELECT i,t FROM intandtext) t;
+
+DROP TABLE intandtext;
+
+set 
+  optimizer_switch=default,
+  xpand_derived_handler= default,
+  xpand_select_handler=default;
+
+--echo #
+--echo # CLX-77: INSERT ... SELECT returns rows to the client instead of inserting
+--echo #
+--disable_warnings
+drop table if exists t1,t2;
+--enable_warnings
+
+create table t1 (a int) engine=xpand;
+insert into t1 values (1);
+
+select a into @var from t1;
+select @var;
+
+--echo # This must not emit output to the client:
+select a into outfile 'tmpfile1' from t1;
+let $file=`select concat(@@datadir,'/xpd/tmpfile1')`;
+
+--remove_file $file
+
+create table t2 (a int) engine=myisam;
+insert into t2 select * from t1;
+select * from t2;
+
+drop table t1,t2;
+
+--echo # 
+--echo # CLX-55: Prepared statement support: 
+--echo #   Implement "Direct Update" by printing the statement
+--echo # 
+create table t1 (a int primary key, b int) engine=xpand;
+insert into t1 values (1,1),(2,2),(3,3);
+
+prepare s from 'update t1 set b=b+? where a=?';
+execute s using 10000, 2;
+--sorted_result
+select * from t1;
+drop table t1;
+
+--echo #
+--echo # CLX-80: ALTER TABLE t ENGINE=CLUSTRIX fails with an error
+--echo #
+create table t1 (a int) engine=myisam;
+insert into t1 values (1),(2),(3);
+alter table t1 engine=xpand;
+--sorted_result
+select * from t1;
+show create table t1;
+
+--echo # Try a RENAME TABLE too since the patch touches the code
+alter table t1 rename t2;
+show create table t2;
+
+drop table t2;
+
+USE test;
+DROP DATABASE xpd;

mysql-test/suite/xpand/my.cnf

+!include include/default_my.cnf
+
+[mysqld.1]
+socket= /tmp/mysqld42.sock

mysql-test/suite/xpand/pushdown_conditions.result

+CREATE DATABASE xpd;
+USE xpd;
+DROP TABLE IF EXISTS cx1;
+CREATE TABLE cx1(i BIGINT, i2 BIGINT, t TEXT)ENGINE=xpand;
+INSERT INTO cx1 VALUES (41, 43, 'some1'), (42, 42, 'some2'), (43, 41, 'some3');
+SELECT * FROM cx1 ORDER BY i;
+i	i2	t
+41	43	some1
+42	42	some2
+43	41	some3
+SET xpand_select_handler=OFF;
+SELECT * FROM cx1 WHERE i>41 AND i2>41;
+i	i2	t
+42	42	some2
+EXPLAIN SELECT * FROM cx1 WHERE i>41 AND i2>41;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	cx1	ALL	NULL	NULL	NULL	NULL	10000	Using where with pushed condition
+SELECT * FROM cx1 WHERE i>41 AND i2>41 AND t='some2';
+i	i2	t
+42	42	some2
+EXPLAIN SELECT * FROM cx1 WHERE i>41 AND i2>41 AND t='some2';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	cx1	ALL	NULL	NULL	NULL	NULL	10000	Using where with pushed condition
+SELECT * FROM cx1 WHERE i>i2+1;
+i	i2	t
+43	41	some3
+EXPLAIN SELECT * FROM cx1 WHERE i>i2+1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	cx1	ALL	NULL	NULL	NULL	NULL	10000	Using where with pushed condition
+SET @@optimizer_switch='derived_merge=OFF';
+SELECT * FROM (SELECT * FROM cx1 WHERE i>i2+1) a1 ORDER BY i;
+i	i2	t
+43	41	some3
+EXPLAIN SELECT * FROM (SELECT * FROM cx1 WHERE i>i2+1) a1 ORDER BY i;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	<derived2>	ALL	NULL	NULL	NULL	NULL	10000	Using filesort
+2	PUSHED DERIVED	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL
+SET xpand_derived_handler=OFF;
+SELECT * FROM (SELECT * FROM cx1 WHERE i>i2+1) a1 ORDER BY i;
+i	i2	t
+43	41	some3
+EXPLAIN SELECT * FROM (SELECT * FROM cx1 WHERE i>i2+1) a1 ORDER BY i;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	<derived2>	ALL	NULL	NULL	NULL	NULL	10000	Using filesort
+2	DERIVED	cx1	ALL	NULL	NULL	NULL	NULL	10000	Using where with pushed condition
+USE test;
+DROP DATABASE xpd;

mysql-test/suite/xpand/pushdown_conditions.test

+CREATE DATABASE xpd;
+USE xpd;
+
+--disable_warnings
+DROP TABLE IF EXISTS cx1;
+--enable_warnings
+CREATE TABLE cx1(i BIGINT, i2 BIGINT, t TEXT)ENGINE=xpand;
+
+INSERT INTO cx1 VALUES (41, 43, 'some1'), (42, 42, 'some2'), (43, 41, 'some3');
+
+SELECT * FROM cx1 ORDER BY i;
+SET xpand_select_handler=OFF;
+
+SELECT * FROM cx1 WHERE i>41 AND i2>41;
+EXPLAIN SELECT * FROM cx1 WHERE i>41 AND i2>41;
+SELECT * FROM cx1 WHERE i>41 AND i2>41 AND t='some2';
+EXPLAIN SELECT * FROM cx1 WHERE i>41 AND i2>41 AND t='some2';
+SELECT * FROM cx1 WHERE i>i2+1;
+EXPLAIN SELECT * FROM cx1 WHERE i>i2+1;
+
+# The plugin doesn't use pushdown conditions for DH as of 10.5.1
+# but it is worth to test memory leaks.
+SET @@optimizer_switch='derived_merge=OFF';
+SELECT * FROM (SELECT * FROM cx1 WHERE i>i2+1) a1 ORDER BY i;
+EXPLAIN SELECT * FROM (SELECT * FROM cx1 WHERE i>i2+1) a1 ORDER BY i;
+SET xpand_derived_handler=OFF;
+SELECT * FROM (SELECT * FROM cx1 WHERE i>i2+1) a1 ORDER BY i;
+EXPLAIN SELECT * FROM (SELECT * FROM cx1 WHERE i>i2+1) a1 ORDER BY i;
+
+# SELECT * FROM (SELECT i FROM cx1 WHERE i=42)a1,(SELECT i FROM cx1 WHERE i =42)a2 WHERE a1.i=a2.i;
+# EXPLAIN SELECT * FROM (SELECT i FROM cx1 WHERE i=42)a1,(SELECT i FROM cx1 WHERE i =42)a2 WHERE a1.i=a2.i;
+
+USE test;
+DROP DATABASE xpd;

mysql-test/suite/xpand/suite.opt

+--plugin-load=xpand=ha_xpand.so 
+--core-file 
+--xpand_port=3306 
+--plugin-maturity=unknown

mysql-test/suite/xpand/update.result

+CREATE DATABASE IF NOT EXISTS `db1`;
+connect  con1,localhost,root,,test;
+connection con1;
+USE `db1`;
+DROP TABLE IF EXISTS `t1`;
+CREATE TABLE `t1`(i BIGINT, t TEXT)ENGINE=xpand;
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `i` bigint(20) DEFAULT NULL,
+  `t` text DEFAULT NULL
+) ENGINE=XPAND DEFAULT CHARSET=utf8
+set character_set_client=utf8;
+set collation_connection=utf8_bin;
+set character_set_results=utf8;
+INSERT INTO `t1` (i, t) VALUES (42, 'один');
+INSERT INTO `t1` (i, t) VALUES (42, 'ноль');
+SELECT * FROM `t1` ORDER BY `i` DESC, `t` DESC;
+i	t
+42	один
+42	ноль
+# examine the data on the backend:
+# This should show that the SELECT has been pushed to the backend:
+explain
+select i, hex(t) from t1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PUSHED SELECT	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL
+select i, hex(t) from t1;
+i	hex(t)
+42	D0BDD0BED0BBD18C
+42	D0BED0B4D0B8D0BD
+# The above should match:
+select hex('один') as row1, hex('ноль') as row2;
+row1	row2
+D0BED0B4D0B8D0BD	D0BDD0BED0BBD18C
+UPDATE `t1` SET i=i+1,t='два' WHERE t='один';
+SELECT * FROM `t1` ORDER BY `i` DESC, `t` DESC;
+i	t
+43	два
+42	ноль
+USE test;
+UPDATE `db1`.`t1` SET i=i+1,t='три' WHERE t='два';
+SELECT * FROM `db1`.`t1` ORDER BY `i` DESC, `t` DESC;
+i	t
+44	три
+42	ноль
+disconnect con1;
+connection default;
+DROP TABLE `db1`.`t1`;
+USE test;
+DROP DATABASE `db1`;

mysql-test/suite/xpand/update.test

+CREATE DATABASE IF NOT EXISTS `db1`;
+
+# Do the test in another connection so that we don't have to clean up
+connect (con1,localhost,root,,test);
+connection con1;
+
+USE `db1`;
+--disable_warnings
+DROP TABLE IF EXISTS `t1`;
+--enable_warnings
+
+CREATE TABLE `t1`(i BIGINT, t TEXT)ENGINE=xpand;
+show create table t1;
+
+set character_set_client=utf8;
+set collation_connection=utf8_bin;
+set character_set_results=utf8;
+INSERT INTO `t1` (i, t) VALUES (42, 'один');
+INSERT INTO `t1` (i, t) VALUES (42, 'ноль');
+SELECT * FROM `t1` ORDER BY `i` DESC, `t` DESC;
+
+--echo # examine the data on the backend:
+--echo # This should show that the SELECT has been pushed to the backend:
+explain
+select i, hex(t) from t1;
+--sorted_result
+select i, hex(t) from t1;
+--echo # The above should match:
+select hex('один') as row1, hex('ноль') as row2;
+
+UPDATE `t1` SET i=i+1,t='два' WHERE t='один';
+SELECT * FROM `t1` ORDER BY `i` DESC, `t` DESC;
+
+USE test;
+UPDATE `db1`.`t1` SET i=i+1,t='три' WHERE t='два';
+SELECT * FROM `db1`.`t1` ORDER BY `i` DESC, `t` DESC;
+
+disconnect con1;
+connection default;
+
+DROP TABLE `db1`.`t1`;
+
+USE test;
+DROP DATABASE `db1`;

mysql-test/suite/xpand/upsert.result

+CREATE DATABASE IF NOT EXISTS `db1`;
+USE `db1`;
+DROP TABLE IF EXISTS `ins_duplicate`;
+Warnings:
+Note	1051	Unknown table 'db1.ins_duplicate'
+CREATE TABLE `ins_duplicate`(`id` INT PRIMARY KEY, `animal` VARCHAR(30)) ENGINE=xpand;
+INSERT INTO `ins_duplicate` VALUES (1,'Aardvark'), (2,'Cheetah'), (3,'Zebra');
+SELECT * FROM `ins_duplicate` ORDER BY `id`;
+id	animal
+1	Aardvark
+2	Cheetah
+3	Zebra
+INSERT INTO ins_duplicate VALUES (1,'Antelope');
+ERROR 23000: Can't write; duplicate key in table 'ins_duplicate'
+INSERT INTO ins_duplicate VALUES (1,'Antelope') ON DUPLICATE KEY UPDATE animal='Banana';
+SELECT * FROM `ins_duplicate` ORDER BY `id`;
+id	animal
+1	Banana
+2	Cheetah
+3	Zebra
+INSERT INTO ins_duplicate VALUES (1,'Antelope'),(2,'Cheetah');
+ERROR 23000: Can't write; duplicate key in table 'ins_duplicate'
+INSERT INTO ins_duplicate VALUES (1,'Antelope'),(2,'Cheetah') ON DUPLICATE KEY UPDATE animal='hybrid';
+SELECT * FROM `ins_duplicate` ORDER BY `id`;
+id	animal
+1	hybrid
+2	hybrid
+3	Zebra
+BEGIN;
+SELECT * FROM `ins_duplicate` ORDER BY `id`;
+id	animal
+1	hybrid
+2	hybrid
+3	Zebra
+INSERT INTO ins_duplicate VALUES (1,'Antelope');
+ERROR 23000: Can't write; duplicate key in table 'ins_duplicate'
+INSERT INTO ins_duplicate VALUES (1,'Antelope') ON DUPLICATE KEY UPDATE animal='Vegetable';
+SELECT * FROM `ins_duplicate` ORDER BY `id`;
+id	animal
+1	Vegetable
+2	hybrid
+3	Zebra
+INSERT INTO ins_duplicate VALUES (1,'Antelope'),(2,'Cheetah');
+ERROR 23000: Can't write; duplicate key in table 'ins_duplicate'
+INSERT INTO ins_duplicate VALUES (1,'Antelope'),(2,'Cheetah') ON DUPLICATE KEY UPDATE animal='hybrid2';
+COMMIT;
+BEGIN;
+SELECT * FROM `ins_duplicate` ORDER BY `id`;
+id	animal
+1	hybrid2
+2	hybrid2
+3	Zebra
+INSERT INTO ins_duplicate VALUES (1,'Antelope');
+ERROR 23000: Can't write; duplicate key in table 'ins_duplicate'
+INSERT INTO ins_duplicate VALUES (1,'Antelope') ON DUPLICATE KEY UPDATE animal='Vegetable';
+SELECT * FROM `ins_duplicate` ORDER BY `id`;
+id	animal
+1	Vegetable
+2	hybrid2
+3	Zebra
+INSERT INTO ins_duplicate VALUES (1,'Antelope'),(2,'Cheetah');
+ERROR 23000: Can't write; duplicate key in table 'ins_duplicate'
+INSERT INTO ins_duplicate VALUES (1,'Antelope'),(2,'Cheetah') ON DUPLICATE KEY UPDATE animal='hybrid3';
+ROLLBACK;
+SELECT * FROM `ins_duplicate` ORDER BY `id`;
+id	animal
+1	hybrid2
+2	hybrid2
+3	Zebra
+DROP TABLE `db1`.`ins_duplicate`;
+USE test;
+DROP DATABASE `db1`;

mysql-test/suite/xpand/upsert.test

+CREATE DATABASE IF NOT EXISTS `db1`;
+USE `db1`;
+DROP TABLE IF EXISTS `ins_duplicate`;
+CREATE TABLE `ins_duplicate`(`id` INT PRIMARY KEY, `animal` VARCHAR(30)) ENGINE=xpand;
+INSERT INTO `ins_duplicate` VALUES (1,'Aardvark'), (2,'Cheetah'), (3,'Zebra');
+SELECT * FROM `ins_duplicate` ORDER BY `id`;
+
+--error ER_DUP_KEY
+INSERT INTO ins_duplicate VALUES (1,'Antelope');
+INSERT INTO ins_duplicate VALUES (1,'Antelope') ON DUPLICATE KEY UPDATE animal='Banana';
+SELECT * FROM `ins_duplicate` ORDER BY `id`;
+
+--error ER_DUP_KEY
+INSERT INTO ins_duplicate VALUES (1,'Antelope'),(2,'Cheetah');
+INSERT INTO ins_duplicate VALUES (1,'Antelope'),(2,'Cheetah') ON DUPLICATE KEY UPDATE animal='hybrid';
+SELECT * FROM `ins_duplicate` ORDER BY `id`;
+
+BEGIN;
+SELECT * FROM `ins_duplicate` ORDER BY `id`;
+
+--error ER_DUP_KEY
+INSERT INTO ins_duplicate VALUES (1,'Antelope');
+INSERT INTO ins_duplicate VALUES (1,'Antelope') ON DUPLICATE KEY UPDATE animal='Vegetable';
+SELECT * FROM `ins_duplicate` ORDER BY `id`;
+
+--error ER_DUP_KEY
+INSERT INTO ins_duplicate VALUES (1,'Antelope'),(2,'Cheetah');
+INSERT INTO ins_duplicate VALUES (1,'Antelope'),(2,'Cheetah') ON DUPLICATE KEY UPDATE animal='hybrid2';
+COMMIT;
+
+BEGIN;
+SELECT * FROM `ins_duplicate` ORDER BY `id`;
+
+--error ER_DUP_KEY
+INSERT INTO ins_duplicate VALUES (1,'Antelope');
+INSERT INTO ins_duplicate VALUES (1,'Antelope') ON DUPLICATE KEY UPDATE animal='Vegetable';
+SELECT * FROM `ins_duplicate` ORDER BY `id`;
+
+--error ER_DUP_KEY
+INSERT INTO ins_duplicate VALUES (1,'Antelope'),(2,'Cheetah');
+INSERT INTO ins_duplicate VALUES (1,'Antelope'),(2,'Cheetah') ON DUPLICATE KEY UPDATE animal='hybrid3';
+ROLLBACK;
+
+SELECT * FROM `ins_duplicate` ORDER BY `id`;
+
+DROP TABLE `db1`.`ins_duplicate`;
+
+USE test;
+DROP DATABASE `db1`;

storage/federatedx/federatedx_pushdown.cc

@@ -181,6 +181,16 @@ create_federatedx_select_handler(THD* thd, SELECT_LEX *sel)
     else if (ht != tbl->table->file->partition_ht())
       return 0;
   }
+  
+  /*
+    Currently, ha_federatedx_select_handler::init_scan just takes the 
+    thd->query and sends it to the backend.
+    This obviously won't work if the SELECT uses an "INTO @var" or
+    "INTO OUTFILE". It is also unlikely to work if the select has some
+    other kind of side effect.
+  */
+  if (sel->uncacheable & UNCACHEABLE_SIDEEFFECT)
+    return NULL;
 
   /*
     Currently, ha_federatedx_select_handler::init_scan just takes the

storage/xpand/CMakeLists.txt

+#*****************************************************************************
+# Copyright (c) 2019, 2020, MariaDB Corporation.
+#****************************************************************************/
+
+SET(XPAND_SOURCES ha_xpand.cc xpand_connection.cc ha_xpand_pushdown.cc)
+MYSQL_ADD_PLUGIN(xpand ${XPAND_SOURCES} STORAGE_ENGINE COMPONENT xpand-engine)

storage/xpand/ha_xpand.h

+/*****************************************************************************
+Copyright (c) 2019, 2020, MariaDB Corporation.
+*****************************************************************************/
+
+#ifndef _ha_xpand_h
+#define _ha_xpand_h
+
+#ifdef USE_PRAGMA_INTERFACE
+#pragma interface     /* gcc class implementation */
+#endif
+
+#define MYSQL_SERVER 1
+#include "xpand_connection.h"
+#include "my_bitmap.h"
+#include "table.h"
+#include "rpl_rli.h"
+#include "handler.h"
+#include "sql_class.h"
+#include "sql_show.h"
+#include "mysql.h"
+#include "../../sql/rpl_record.h"
+
+size_t estimate_row_size(TABLE *table);
+xpand_connection *get_trx(THD *thd, int *error_code);
+bool get_enable_sh(THD* thd);
+void add_current_table_to_rpl_table_list(rpl_group_info **_rgi, THD *thd,
+                                         TABLE *table);
+void remove_current_table_from_rpl_table_list(rpl_group_info *rgi);
+int unpack_row_to_buf(rpl_group_info *rgi, TABLE *table, uchar *data,
+                      uchar const *const row_data, MY_BITMAP const *cols,
+                      uchar const *const row_end);
+void xpand_mark_tables_for_discovery(LEX *lex);
+ulonglong *xpand_extract_table_oids(THD *thd, LEX *lex);
+
+
+class Xpand_share : public Handler_share {
+public:
+  Xpand_share(): xpand_table_oid(0), rediscover_table(false) {}
+
+  std::atomic<ulonglong> xpand_table_oid;
+  std::atomic<bool> rediscover_table;
+};
+
+class ha_xpand : public handler
+{
+private:
+  // TODO: do we need this here or one in share would be sufficient?
+  ulonglong xpand_table_oid;
+  rpl_group_info *rgi;
+
+  Field *auto_inc_field;
+  ulonglong auto_inc_value;
+
+  bool has_hidden_key;
+  ulonglong last_hidden_key;
+  xpand_connection_cursor *scan_cur;
+  bool is_scan;
+  MY_BITMAP scan_fields;
+  bool sorted_scan;
+  xpand_lock_mode_t xpd_lock_type;
+
+  uint last_dup_errkey;
+
+  typedef enum xpand_upsert_flags {
+    XPAND_HAS_UPSERT= 1,
+    XPAND_BULK_UPSERT= 2,
+    XPAND_UPSERT_SENT= 4
+  } xpd_upsert_flags_t;
+  int upsert_flag;
+
+  List<COND> pushdown_cond_list;
+
+  Xpand_share *get_share(); ///< Get the share
+
+public:
+  ha_xpand(handlerton *hton, TABLE_SHARE *table_arg);
+  ~ha_xpand();
+  int create(const char *name, TABLE *form, HA_CREATE_INFO *info) override;
+  int delete_table(const char *name) override;
+  int rename_table(const char* from, const char* to) override;
+  int open(const char *name, int mode, uint test_if_locked) override;
+  int close(void) override;
+  int reset() override;
+  int extra(enum ha_extra_function operation) override;
+  int write_row(const uchar *buf) override;
+  // start_bulk_update exec_bulk_update
+  int update_row(const uchar *old_data, const uchar *new_data) override;
+  // start_bulk_delete exec_bulk_delete
+  int delete_row(const uchar *buf) override;
+  int direct_update_rows_init(List<Item> *update_fields) override;
+  int direct_update_rows(ha_rows *update_rows, ha_rows *found_rows) override;
+  void start_bulk_insert(ha_rows rows, uint flags = 0) override;
+  int end_bulk_insert() override;
+
+  Table_flags table_flags(void) const override;
+  ulong index_flags(uint idx, uint part, bool all_parts) const override;
+  uint max_supported_keys() const override { return MAX_KEY; }
+
+  ha_rows records() override;
+  ha_rows records_in_range(uint inx, key_range *min_key,
+                           key_range *max_key) override;
+
+  int info(uint flag) override; // see my_base.h for full description
+
+  // multi_read_range
+  // read_range
+  int index_init(uint idx, bool sorted) override;
+  int index_read(uchar * buf, const uchar * key, uint key_len,
+                 enum ha_rkey_function find_flag) override;
+  int index_first(uchar *buf) override;
+  int index_prev(uchar *buf) override;
+  int index_last(uchar *buf) override;
+  int index_next(uchar *buf) override;
+  //int index_next_same(uchar *buf, const uchar *key, uint keylen) override;
+  int index_end() override;
+
+  int rnd_init(bool scan) override;
+  int rnd_next(uchar *buf) override;
+  int rnd_pos(uchar * buf, uchar *pos) override;
+  int rnd_end() override;
+
+  void position(const uchar *record) override;
+  uint lock_count(void) const override;
+  THR_LOCK_DATA **store_lock(THD *thd,
+                             THR_LOCK_DATA **to,
+                             enum thr_lock_type lock_type) override;
+  int external_lock(THD *thd, int lock_type) override;
+
+  uint8 table_cache_type() override
+  {
+    return(HA_CACHE_TBL_NOCACHE);
+  }
+
+  const COND *cond_push(const COND *cond) override;
+  void cond_pop() override;
+  int info_push(uint info_type, void *info) override;
+
+  ulonglong get_table_oid();
+private:
+  void build_key_packed_row(uint index, const uchar *buf,
+                            uchar *packed_key, size_t *packed_key_len);
+};
+
+bool select_handler_setting(THD* thd);
+bool derived_handler_setting(THD* thd);
+uint row_buffer_setting(THD* thd);
+#endif  // _ha_xpand_h

storage/xpand/ha_xpand_pushdown.h

+/*****************************************************************************
+Copyright (c) 2019, 2020, MariaDB Corporation.
+*****************************************************************************/
+#ifndef _ha_xpand_pushdown_h
+#define _ha_xpand_pushdown_h
+
+#include "select_handler.h"
+#include "derived_handler.h"
+#include "sql_select.h"
+
+/*@brief base_handler class*/
+/***********************************************************
+ * DESCRIPTION:
+ * To be described
+ ************************************************************/
+class ha_xpand_base_handler
+{
+  // To simulate abstract class
+protected:
+  ha_xpand_base_handler(): thd__(0),table__(0) {}
+  ~ha_xpand_base_handler() {}
+
+  // Copies of pushdown handlers attributes
+  // to use them in shared methods.
+  THD *thd__;
+  TABLE *table__;
+  // The bitmap used to sent
+  MY_BITMAP scan_fields;
+  // Structures to unpack RBR rows from XPD BE
+  rpl_group_info *rgi;
+  // XPD BE scan operation reference
+  xpand_connection_cursor *scan;
+};
+
+/*@brief select_handler class*/
+/***********************************************************
+ * DESCRIPTION:
+ *  select_handler API methods. Could be used by the server
+ *  tp pushdown the whole query described by SELECT_LEX.
+ *  More details in server/sql/select_handler.h
+ *  sel semantic tree for the query in SELECT_LEX.
+ ************************************************************/
+class ha_xpand_select_handler:
+    private ha_xpand_base_handler,
+    public select_handler
+{
+public:
+  ha_xpand_select_handler(THD* thd_arg, SELECT_LEX* sel,
+                          xpand_connection_cursor *scan);
+  ~ha_xpand_select_handler();
+
+  int init_scan() override;
+  int next_row() override;
+  int end_scan() override;
+  void print_error(int, unsigned long) override {}
+};
+
+/*@brief derived_handler class*/
+/***********************************************************
+ * DESCRIPTION:
+ *  derived_handler API methods. Could be used by the server
+ *  tp pushdown the whole query described by SELECT_LEX.
+ *  More details in server/sql/derived_handler.h
+ *  sel semantic tree for the query in SELECT_LEX.
+ ************************************************************/
+class ha_xpand_derived_handler:
+    private ha_xpand_base_handler,
+    public derived_handler
+{
+public:
+  ha_xpand_derived_handler(THD* thd_arg, SELECT_LEX* sel,
+                           xpand_connection_cursor *scan);
+  ~ha_xpand_derived_handler();
+
+  int init_scan() override;
+  int next_row() override;
+  int end_scan() override;
+  void print_error(int, unsigned long) override {}
+};
+
+select_handler *create_xpand_select_handler(THD* thd, SELECT_LEX* select_lex);
+derived_handler *create_xpand_derived_handler(THD* thd, TABLE_LIST *derived);
+
+#endif

storage/xpand/xpand_connection.h

+/*****************************************************************************
+Copyright (c) 2019, 2020, MariaDB Corporation.
+*****************************************************************************/
+
+#ifndef _xpand_connection_h
+#define _xpand_connection_h
+
+#ifdef USE_PRAGMA_INTERFACE
+#pragma interface     /* gcc class implementation */
+#endif
+
+#define MYSQL_SERVER 1
+#include "my_global.h"
+#include "m_string.h"
+#include "mysql.h"
+#include "sql_common.h"
+#include "my_base.h"
+#include "mysqld_error.h"
+#include "my_bitmap.h"
+#include "handler.h"
+
+#define XPAND_SERVER_REQUEST 30
+
+enum xpand_lock_mode_t {
+    XPAND_NO_LOCKS,
+    XPAND_SHARED,
+    XPAND_EXCLUSIVE,
+};
+
+enum xpand_balance_algorithm_enum {
+  XPAND_BALANCE_FIRST,
+  XPAND_BALANCE_ROUND_ROBIN
+};
+
+class xpand_connection_cursor;
+class xpand_connection
+{
+private:
+  MYSQL xpand_net;
+  uchar *command_buffer;
+  size_t command_buffer_length;
+  size_t command_length;
+
+  int trans_state;
+  int trans_flags;
+  int allocate_cursor(MYSQL *xpand_net, ulong buffer_size,
+                      xpand_connection_cursor **scan);
+public:
+  xpand_connection();
+  ~xpand_connection();
+
+  inline bool is_connected()
+  {
+    return xpand_net.net.vio;
+  }
+  int connect();
+  int connect_direct(char *host);
+  void disconnect(bool is_destructor = FALSE);
+
+  bool has_open_transaction();
+  int commit_transaction();
+  int rollback_transaction();
+  int begin_transaction_next();
+  int new_statement_next();
+  int rollback_statement_next(); // also starts new statement
+  void auto_commit_next();
+  void auto_commit_closed();
+
+  int run_query(String &stmt);
+  int write_row(ulonglong xpand_table_oid, uchar *packed_row,
+                size_t packed_size, ulonglong *last_insert_id);
+  int key_update(ulonglong xpand_table_oid,
+                 uchar *packed_key, size_t packed_key_length,
+                 MY_BITMAP *update_set,
+                 uchar *packed_new_data, size_t packed_new_length);
+  int key_delete(ulonglong xpand_table_oid,
+                 uchar *packed_key, size_t packed_key_length);
+  int key_read(ulonglong xpand_table_oid, uint index,
+               xpand_lock_mode_t lock_mode, MY_BITMAP *read_set,
+               uchar *packed_key, ulong packed_key_length, uchar **rowdata,
+               ulonglong *rowdata_length);
+  enum sort_order {SORT_NONE = 0, SORT_ASC = 1, SORT_DESC = 2};
+  enum scan_type {
+    READ_KEY_OR_NEXT,  /* rows with key and greater */
+    READ_KEY_OR_PREV,  /* rows with key and less. */
+    READ_AFTER_KEY,    /* rows with keys greater than key */
+    READ_BEFORE_KEY,   /* rows with keys less than key */
+    READ_FROM_START,   /* rows with forwards from first key. */
+    READ_FROM_LAST,    /* rows with backwards from last key. */
+  };
+  int scan_table(ulonglong xpand_table_oid,
+                 xpand_lock_mode_t lock_mode,
+                 MY_BITMAP *read_set, ushort row_req,
+                 xpand_connection_cursor **scan, String* pushdown_cond_sql);
+  int scan_query(String &stmt, uchar *fieldtype, uint fields, uchar *null_bits,
+                 uint null_bits_size, uchar *field_metadata,
+                 uint field_metadata_size, ushort row_req, ulonglong *oids,
+                 xpand_connection_cursor **scan);
+  int update_query(String &stmt, LEX_CSTRING &dbname, ulonglong *oids,
+                   ulonglong *affected_rows);
+  int scan_from_key(ulonglong xpand_table_oid, uint index,
+                    xpand_lock_mode_t lock_mode,
+                    enum scan_type scan_dir, int no_key_cols, bool sorted_scan,
+                    MY_BITMAP *read_set, uchar *packed_key,
+                    ulong packed_key_length, ushort row_req,
+                    xpand_connection_cursor **scan);
+  int scan_next(xpand_connection_cursor *scan, uchar **rowdata,
+                ulong *rowdata_length);
+  int scan_end(xpand_connection_cursor *scan);
+
+  int populate_table_list(LEX_CSTRING *db, handlerton::discovered_list *result);
+  int get_table_oid(const char *db, size_t db_len, const char *name,
+                    size_t name_len, ulonglong *oid, TABLE_SHARE *share);
+  int discover_table_details(LEX_CSTRING *db, LEX_CSTRING *name, THD *thd,
+                             TABLE_SHARE *share);
+
+private:
+  int expand_command_buffer(size_t add_length);
+  int add_command_operand_uchar(uchar value);
+  int add_command_operand_ushort(ushort value);
+  int add_command_operand_uint(uint value);
+  int add_command_operand_ulonglong(ulonglong value);
+  int add_command_operand_lcb(ulonglong value);
+  int add_command_operand_str(const uchar *str, size_t length);
+  int add_command_operand_vlstr(const uchar *str, size_t length);
+  int add_command_operand_lex_string(LEX_CSTRING str);
+  int add_command_operand_bitmap(MY_BITMAP *bitmap);
+  int add_status_vars();
+  int begin_command(uchar command);
+  int send_command();
+  int read_query_response();
+};
+
+static const int max_host_count = 128;
+class xpand_host_list {
+private:
+  char *strtok_buf;
+public:
+  int hosts_len;
+  char *hosts[max_host_count];
+
+  int fill(const char *hosts);
+  void empty();
+};
+
+#endif  // _xpand_connection_h