Merge branch 'bb-10.2-myrocks-merge' into 10.2

storage/rocksdb/.clang-format

----
+# Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License, version 2.0,
+# as published by the Free Software Foundation.
+#
+# This program is also distributed with certain software (including
+# but not limited to OpenSSL) that is licensed under separate terms,
+# as designated in a particular file or component or in included license
+# documentation.  The authors of MySQL hereby grant you an additional
+# permission to link the program and your derivative works with the
+# separately licensed software that they have included with MySQL.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License, version 2.0, for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
+
+# This is the output of clang-format-5.0 --style=google --dump-config,
+# except for changes mentioned below. We lock the style so that any newer
+# version of clang-format will give the same result; as time goes, we may
+# update this list, requiring newer versions of clang-format.
+
 Language:        Cpp
-# BasedOnStyle:  LLVM
-AccessModifierOffset: -2
+# BasedOnStyle:  Google
+AccessModifierOffset: -1
 AlignAfterOpenBracket: Align
 AlignConsecutiveAssignments: false
 AlignConsecutiveDeclarations: false
-AlignEscapedNewlinesLeft: false
+AlignEscapedNewlines: Left
 AlignOperands:   true
 AlignTrailingComments: true
 AllowAllParametersOfDeclarationOnNextLine: true
 AllowShortBlocksOnASingleLine: false
 AllowShortCaseLabelsOnASingleLine: false
 AllowShortFunctionsOnASingleLine: All
-AllowShortIfStatementsOnASingleLine: false
-AllowShortLoopsOnASingleLine: false
+AllowShortIfStatementsOnASingleLine: true
+AllowShortLoopsOnASingleLine: true
 AlwaysBreakAfterDefinitionReturnType: None
 AlwaysBreakAfterReturnType: None
-AlwaysBreakBeforeMultilineStrings: false
-AlwaysBreakTemplateDeclarations: false
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
 BinPackArguments: true
 BinPackParameters: true
 BraceWrapping:   
@@ -32,62 +58,80 @@ BraceWrapping:
   BeforeCatch:     false
   BeforeElse:      false
   IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
 BreakBeforeBinaryOperators: None
 BreakBeforeBraces: Attach
+BreakBeforeInheritanceComma: false
 BreakBeforeTernaryOperators: true
 BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
 BreakAfterJavaFieldAnnotations: false
 BreakStringLiterals: true
 ColumnLimit:     80
 CommentPragmas:  '^ IWYU pragma:'
-ConstructorInitializerAllOnOneLineOrOnePerLine: false
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
 ConstructorInitializerIndentWidth: 4
 ContinuationIndentWidth: 4
 Cpp11BracedListStyle: true
-DerivePointerAlignment: false
 DisableFormat:   false
 ExperimentalAutoDetectBinPacking: false
-ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
+FixNamespaceComments: true
+ForEachMacros:   
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
 IncludeCategories: 
-  - Regex:           '^"(llvm|llvm-c|clang|clang-c)/'
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+  - Regex:           '^<.*'
     Priority:        2
-  - Regex:           '^(<|"(gtest|isl|json)/)'
-    Priority:        3
   - Regex:           '.*'
-    Priority:        1
-IncludeIsMainRegex: '$'
-IndentCaseLabels: false
+    Priority:        3
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IndentCaseLabels: true
 IndentWidth:     2
 IndentWrappedFunctionNames: false
-KeepEmptyLinesAtTheStartOfBlocks: true
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
 MacroBlockBegin: ''
 MacroBlockEnd:   ''
 MaxEmptyLinesToKeep: 1
 NamespaceIndentation: None
 ObjCBlockIndentWidth: 2
 ObjCSpaceAfterProperty: false
-ObjCSpaceBeforeProtocolList: true
-PenaltyBreakBeforeFirstCallParameter: 19
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 1
 PenaltyBreakComment: 300
 PenaltyBreakFirstLessLess: 120
 PenaltyBreakString: 1000
 PenaltyExcessCharacter: 1000000
-PenaltyReturnTypeOnItsOwnLine: 60
-PointerAlignment: Right
+PenaltyReturnTypeOnItsOwnLine: 200
 ReflowComments:  true
 SortIncludes:    true
+SortUsingDeclarations: true
 SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
 SpaceBeforeAssignmentOperators: true
 SpaceBeforeParens: ControlStatements
 SpaceInEmptyParentheses: false
-SpacesBeforeTrailingComments: 1
+SpacesBeforeTrailingComments: 2
 SpacesInAngles:  false
 SpacesInContainerLiterals: true
 SpacesInCStyleCastParentheses: false
 SpacesInParentheses: false
 SpacesInSquareBrackets: false
-Standard:        Cpp11
 TabWidth:        8
 UseTab:          Never
-JavaScriptQuotes: Leave
-...
+
+# We declare one specific pointer style since right alignment is dominant in
+# the MySQL code base (default --style=google has DerivePointerAlignment true).
+DerivePointerAlignment: false
+PointerAlignment: Right
+
+# MySQL source code is allowed to use C++11 features.
+Standard:        Cpp11

storage/rocksdb/CMakeLists.txt

@@ -100,6 +100,8 @@ SET(ROCKSDB_SE_SOURCES
   rdb_psi.cc
   rdb_sst_info.cc
   rdb_sst_info.h
+  rdb_converter.cc
+  rdb_converter.h
 )
 
 # MariaDB: the following is added in build_rocksdb.cmake, when appropriate:
@@ -149,6 +151,7 @@ ADD_CONVENIENCE_LIBRARY(rocksdb_aux_lib
   rdb_perf_context.h
   rdb_buff.h
   rdb_mariadb_port.h
+  nosql_access.cc nosql_access.h
 )
 
 ADD_DEPENDENCIES(rocksdb_aux_lib GenError)
@@ -159,6 +162,27 @@ if (UNIX AND NOT APPLE)
   TARGET_LINK_LIBRARIES(rocksdb_aux_lib -lrt)
 endif()
 
+# IF (WITH_JEMALLOC)
+#  FIND_LIBRARY(JEMALLOC_LIBRARY
+#    NAMES libjemalloc${PIC_EXT}.a jemalloc
+#    HINTS ${WITH_JEMALLOC}/lib)
+#  SET(rocksdb_static_libs ${rocksdb_static_libs}
+#    ${JEMALLOC_LIBRARY})
+#  ADD_DEFINITIONS(-DROCKSDB_JEMALLOC)
+#  ADD_DEFINITIONS(-DROCKSDB_MALLOC_USABLE_SIZE)
+# ENDIF()
+
+# MariaDB: Q: why does the upstream add libunwind for a particular 
+# storage engine?
+#IF (WITH_UNWIND)
+#  FIND_LIBRARY(UNWIND_LIBRARY
+#    NAMES libunwind${PIC_EXT}.a unwind
+#    HINTS ${WITH_UNWIND}/lib)
+#  SET(rocksdb_static_libs ${rocksdb_static_libs}
+#    ${UNWIND_LIBRARY})
+#ENDIF()
+
+
 TARGET_LINK_LIBRARIES(rocksdb rocksdb_aux_lib)
   FIND_LIBRARY(LZ4_LIBRARY
     NAMES liblz4${PIC_EXT}.a lz4
@@ -187,6 +211,8 @@ ENDIF()
 CHECK_FUNCTION_EXISTS(sched_getcpu  HAVE_SCHED_GETCPU)
 IF(HAVE_SCHED_GETCPU)
   ADD_DEFINITIONS(-DHAVE_SCHED_GETCPU=1)
+# MariaDB: don't do this:
+#  ADD_DEFINITIONS(-DZSTD_STATIC_LINKING_ONLY)
 ENDIF()
 
 IF (WITH_TBB)

storage/rocksdb/event_listener.cc

@@ -34,9 +34,9 @@
 
 namespace myrocks {
 
-static std::vector<Rdb_index_stats>
-extract_index_stats(const std::vector<std::string> &files,
-                    const rocksdb::TablePropertiesCollection &props) {
+static std::vector<Rdb_index_stats> extract_index_stats(
+    const std::vector<std::string> &files,
+    const rocksdb::TablePropertiesCollection &props) {
   std::vector<Rdb_index_stats> ret;
   for (auto fn : files) {
     const auto it = props.find(fn);
@@ -87,10 +87,11 @@ void Rdb_event_listener::OnExternalFileIngested(
 void Rdb_event_listener::OnBackgroundError(
     rocksdb::BackgroundErrorReason reason, rocksdb::Status *status) {
   rdb_log_status_error(*status, "Error detected in background");
+  // NO_LINT_DEBUG
   sql_print_error("RocksDB: BackgroundErrorReason: %d", (int)reason);
   if (status->IsCorruption()) {
     rdb_persist_corruption_marker();
     abort();
   }
 }
-} // namespace myrocks
+}  // namespace myrocks

storage/rocksdb/event_listener.h

@@ -22,7 +22,7 @@ namespace myrocks {
 class Rdb_ddl_manager;
 
 class Rdb_event_listener : public rocksdb::EventListener {
-public:
+ public:
   Rdb_event_listener(const Rdb_event_listener &) = delete;
   Rdb_event_listener &operator=(const Rdb_event_listener &) = delete;
 
@@ -40,10 +40,10 @@ class Rdb_event_listener : public rocksdb::EventListener {
   void OnBackgroundError(rocksdb::BackgroundErrorReason reason,
                          rocksdb::Status *status) override;
 
-private:
+ private:
   Rdb_ddl_manager *m_ddl_manager;
 
   void update_index_stats(const rocksdb::TableProperties &props);
 };
 
-} // namespace myrocks
+}  // namespace myrocks

storage/rocksdb/ha_rocksdb_proto.h

@@ -78,7 +78,7 @@ Rdb_cf_manager &rdb_get_cf_manager();
 const rocksdb::BlockBasedTableOptions &rdb_get_table_options();
 bool rdb_is_ttl_enabled();
 bool rdb_is_ttl_read_filtering_enabled();
-#ifndef NDEBUG
+#ifndef DBUG_OFF
 int rdb_dbug_set_ttl_rec_ts();
 int rdb_dbug_set_ttl_snapshot_ts();
 int rdb_dbug_set_ttl_read_filter_ts();
@@ -100,4 +100,4 @@ Rdb_ddl_manager *rdb_get_ddl_manager(void)
 class Rdb_binlog_manager;
 Rdb_binlog_manager *rdb_get_binlog_manager(void)
     MY_ATTRIBUTE((__warn_unused_result__));
-} // namespace myrocks
+}  // namespace myrocks

storage/rocksdb/logger.h

@@ -22,7 +22,7 @@
 namespace myrocks {
 
 class Rdb_logger : public rocksdb::Logger {
-public:
+ public:
   explicit Rdb_logger(const rocksdb::InfoLogLevel log_level =
                           rocksdb::InfoLogLevel::ERROR_LEVEL)
       : m_mysql_log_level(log_level) {}
@@ -77,9 +77,9 @@ class Rdb_logger : public rocksdb::Logger {
     m_mysql_log_level = log_level;
   }
 
-private:
+ private:
   std::shared_ptr<rocksdb::Logger> m_logger;
   rocksdb::InfoLogLevel m_mysql_log_level;
 };
 
-} // namespace myrocks
+}  // namespace myrocks

storage/rocksdb/myrocks_hotbackup.py

@@ -45,12 +45,14 @@ class Writer(object):
 class StreamWriter(Writer):
   stream_cmd= ''
 
-  def __init__(self, stream_option):
+  def __init__(self, stream_option, direct = 0):
     super(StreamWriter, self).__init__()
     if stream_option == 'tar':
       self.stream_cmd= 'tar chf -'
     elif stream_option == 'xbstream':
       self.stream_cmd= 'xbstream -c'
+      if direct:
+        self.stream_cmd = self.stream_cmd + ' -d'
     else:
       raise Exception("Only tar or xbstream is supported as streaming option.")
 
@@ -342,6 +344,13 @@ class MySQLUtil:
     row = cur.fetchone()
     return row[0]
 
+  @staticmethod
+  def is_directio_enabled(dbh):
+    sql = "SELECT @@global.rocksdb_use_direct_reads"
+    cur = dbh.cursor()
+    cur.execute(sql)
+    row = cur.fetchone()
+    return row[0]
 
 class BackupRunner:
   datadir = None
@@ -363,9 +372,7 @@ class BackupRunner:
     try:
       signal.signal(signal.SIGINT, signal_handler)
       w = None
-      if opts.output_stream:
-        w = StreamWriter(opts.output_stream)
-      else:
+      if not opts.output_stream:
         raise Exception("Currently only streaming backup is supported.")
 
       snapshot_dir = opts.checkpoint_directory + '/' + str(backup_round)
@@ -373,6 +380,11 @@ class BackupRunner:
                               opts.mysql_password,
                               opts.mysql_port,
                               opts.mysql_socket)
+      direct = MySQLUtil.is_directio_enabled(dbh)
+      logger.info("Direct I/O: %d", direct)
+
+      w = StreamWriter(opts.output_stream, direct)
+
       if not self.datadir:
         self.datadir = MySQLUtil.get_datadir(dbh)
         logger.info("Set datadir: %s", self.datadir)

storage/rocksdb/mysql-test/rocksdb/combinations

@@ -3,4 +3,3 @@ loose-rocksdb_write_policy=write_committed
 
 [write_prepared]
 loose-rocksdb_write_policy=write_prepared
-loose-rocksdb_commit_time_batch_for_recovery=on

storage/rocksdb/mysql-test/rocksdb/include/bulk_load.inc

@@ -89,20 +89,32 @@ EOF
 # Make sure a snapshot held by another user doesn't block the bulk load
 connect (other,localhost,root,,);
 set session transaction isolation level repeatable read;
-select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
 start transaction with consistent snapshot;
-select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
+
+# Assert that there is a pending snapshot
+select VALUE > 0 as 'Has opened snapshots' from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
 
 connection default;
+
+# Update CF to smaller value to create multiple SST in ingestion
+eval SET @@GLOBAL.ROCKSDB_UPDATE_CF_OPTIONS=
+    '$pk_cf_name={write_buffer_size=8m;target_file_size_base=1m};';
+
 set rocksdb_bulk_load=1;
 set rocksdb_bulk_load_size=100000;
 --disable_query_log
 --echo LOAD DATA INFILE <input_file> INTO TABLE t1;
 eval LOAD DATA INFILE '$file' INTO TABLE t1;
+# There should be no SST being ingested
+select * from t1;
 --echo LOAD DATA INFILE <input_file> INTO TABLE t2;
 eval LOAD DATA INFILE '$file' INTO TABLE t2;
+# There should be no SST being ingested
+select * from t2;
 --echo LOAD DATA INFILE <input_file> INTO TABLE t3;
 eval LOAD DATA INFILE '$file' INTO TABLE t3;
+# There should be no SST being ingested
+select * from t3;
 --enable_query_log
 set rocksdb_bulk_load=0;
 

storage/rocksdb/mysql-test/rocksdb/include/bulk_load_unsorted.inc

@@ -100,9 +100,10 @@ EOF
 # Make sure a snapshot held by another user doesn't block the bulk load
 connect (other,localhost,root,,);
 set session transaction isolation level repeatable read;
-select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
 start transaction with consistent snapshot;
-select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
+
+# Assert that there is a pending snapshot
+select VALUE > 0 as 'Has opened snapshots' from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
 
 connection default;
 set rocksdb_bulk_load=1;

storage/rocksdb/mysql-test/rocksdb/include/have_direct_io.inc

+# Common test pattern for options that control direct i/o
+#
+# Required input:
+#   $io_option - name and assignment to enable on server command line
+
+--perl
+use Cwd 'abs_path';
+
+open(FILE, ">", "$ENV{MYSQL_TMP_DIR}/data_in_shm.inc") or die;
+my $real_path= abs_path($ENV{'MYSQLTEST_VARDIR'});
+my $in_shm= (index($real_path, "/dev/shm") != -1) ||
+            (index($real_path, "/run/shm") != -1);
+print FILE "let \$DATA_IN_SHM= $in_shm;\n";
+close FILE;
+EOF
+
+--source $MYSQL_TMP_DIR/data_in_shm.inc
+--remove_file $MYSQL_TMP_DIR/data_in_shm.inc
+
+if ($DATA_IN_SHM)
+{
+  --skip DATADIR is in /{dev|run}/shm, possibly due to --mem
+}

storage/rocksdb/mysql-test/rocksdb/include/locking_issues_case3.inc

@@ -52,8 +52,9 @@ let $ID = `SELECT connection_id()`;
 send SELECT * FROM t0 WHERE value > 0 FOR UPDATE;
 
 connection con2;
-let $wait_condition = SELECT 1 FROM information_schema.processlist
-                      WHERE id = $ID AND state = "Sending data";
+let $wait_condition =
+    SELECT 1 FROM information_schema.processlist
+    WHERE (id = $ID/* OR srv_id = $ID*/) AND state = "Sending data";
 --source include/wait_condition.inc
 eval SET SESSION TRANSACTION ISOLATION LEVEL $isolation_level;
 UPDATE t0 SET VALUE=VALUE+1 WHERE id=190000;

storage/rocksdb/mysql-test/rocksdb/include/locking_issues_case4.inc

@@ -52,8 +52,9 @@ let $ID = `SELECT connection_id()`;
 send SELECT * FROM t0 WHERE value > 0 FOR UPDATE;
 
 connection con2;
-let $wait_condition = SELECT 1 FROM information_schema.processlist
-                      WHERE id = $ID AND state = "Sending data";
+let $wait_condition =
+    SELECT 1 FROM information_schema.processlist
+    WHERE (id = $ID/* OR srv_id = $ID*/) AND state = "Sending data";
 --source include/wait_condition.inc
 eval SET SESSION TRANSACTION ISOLATION LEVEL $isolation_level;
 INSERT INTO t0 VALUES(200001,1), (-1,1);

storage/rocksdb/mysql-test/rocksdb/include/locking_issues_case5.inc

@@ -55,8 +55,9 @@ let $ID = `SELECT connection_id()`;
 send SELECT * FROM t0 WHERE value > 0 FOR UPDATE;
 
 connection con2;
-let $wait_condition = SELECT 1 FROM information_schema.processlist
-                      WHERE id = $ID AND state = "Sending data";
+let $wait_condition =
+    SELECT 1 FROM information_schema.processlist
+    WHERE (id = $ID /* OR srv_id = $ID*/) AND state = "Sending data";
 --source include/wait_condition.inc
 eval SET SESSION TRANSACTION ISOLATION LEVEL $isolation_level;
 BEGIN;

storage/rocksdb/mysql-test/rocksdb/include/locking_issues_case6.inc

@@ -55,8 +55,9 @@ let $ID = `SELECT connection_id()`;
 send SELECT * FROM t0 WHERE value > 0 FOR UPDATE;
 
 connection con2;
-let $wait_condition = SELECT 1 FROM information_schema.processlist
-                      WHERE id = $ID AND state = "Sending data";
+let $wait_condition =
+   SELECT 1 FROM information_schema.processlist
+   WHERE (id = $ID/* OR srv_id = $ID*/) AND state = "Sending data";
 --source include/wait_condition.inc
 eval SET SESSION TRANSACTION ISOLATION LEVEL $isolation_level;
 BEGIN;

storage/rocksdb/mysql-test/rocksdb/include/use_direct_io_option.inc

+# Common test pattern for options that control direct i/o
+#
+# Required input:
+#   $io_option - name and assignment to enable on server command line
+
+--source include/have_direct_io.inc
+
+--echo Checking direct reads
+--let $_mysqld_option=$io_option
+--source include/restart_mysqld_with_option.inc
+
+CREATE TABLE t1 (pk INT PRIMARY KEY DEFAULT '0', a INT(11), b CHAR(8)) ENGINE=rocksdb;
+SHOW CREATE TABLE t1;
+INSERT INTO t1 VALUES (1, 1,'a');
+INSERT INTO t1 (a,b) VALUES (2,'b');
+set global rocksdb_force_flush_memtable_now=1;
+--sorted_result
+SELECT a,b FROM t1;
+DROP TABLE t1;
+
+# cleanup
+--let _$mysqld_option=
+--source include/restart_mysqld.inc

storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_sstfilewriter.result

@@ -17,7 +17,7 @@ ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE;
 set @tmp= @@rocksdb_max_row_locks;
 set session rocksdb_max_row_locks=1000;
 ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY;
-ERROR HY000: Got error 10 'Operation aborted: Failed to acquire lock due to max_num_locks limit' from ROCKSDB
+ERROR HY000: Got error 10 'Operation aborted: Failed to acquire lock due to rocksdb_max_row_locks limit' from ROCKSDB
 set session rocksdb_bulk_load=1;
 ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY;
 set session rocksdb_bulk_load=0;

storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars.result

@@ -176,3 +176,24 @@ INSERT INTO t1 (a) VALUES (1);
 UPDATE t1 SET pk = 3;
 ALTER TABLE t1 AUTO_INCREMENT 2;
 DROP TABLE t1;
+#----------------------------------
+# Issue #902 Debug assert in autoincrement with small field type
+#----------------------------------
+SET auto_increment_increment=100, auto_increment_offset=10;
+CREATE TABLE t1(i INT AUTO_INCREMENT PRIMARY KEY) ENGINE=ROCKSDB AUTO_INCREMENT=18446744073709551615;
+INSERT INTO t1 VALUES (NULL);
+ERROR HY000: Failed to read auto-increment value from storage engine
+SELECT * FROM t1;
+i
+ALTER TABLE t1 AUTO_INCREMENT=1;
+INSERT INTO t1 VALUES (NULL);
+SELECT * FROM t1;
+i
+10
+ALTER TABLE t1 AUTO_INCREMENT=18446744073709551615;
+INSERT INTO t1 VALUES (NULL);
+ERROR HY000: Failed to read auto-increment value from storage engine
+SELECT * FROM t1;
+i
+10
+DROP TABLE t1;

storage/rocksdb/mysql-test/rocksdb/r/blind_delete_without_tx_api.result → storage/rocksdb/mysql-test/rocksdb/r/blind_delete_rc.result

@@ -3,6 +3,7 @@ Warnings:
 Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
 Note	####	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
 [connection master]
+SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED;
 set @save_rocksdb_blind_delete_primary_key=@@session.rocksdb_blind_delete_primary_key;
 set @save_rocksdb_master_skip_tx_api=@@session.rocksdb_master_skip_tx_api;
 DROP TABLE IF EXISTS t1,t2;
@@ -17,6 +18,7 @@ SELECT count(*) FROM t1;
 count(*)
 9000
 include/sync_slave_sql_with_master.inc
+SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED;
 SELECT count(*) FROM t1;
 count(*)
 9000
@@ -71,14 +73,14 @@ count(*)
 call mtr.add_suppression("Slave SQL.*Could not execute Delete_rows event on table test.t1.*Error_code.*");
 call mtr.add_suppression("Slave: Can't find record in 't1'.*");
 include/wait_for_slave_sql_error.inc [errno=1032]
-set @save_rocksdb_read_free_rpl_tables=@@global.rocksdb_read_free_rpl_tables;
-set global rocksdb_read_free_rpl_tables="t.*";
+set @save_rocksdb_read_free_rpl=@@global.rocksdb_read_free_rpl;
+set global rocksdb_read_free_rpl=PK_SK;
 START SLAVE;
 include/sync_slave_sql_with_master.inc
 SELECT count(*) FROM t1;
 count(*)
 7000
-set global rocksdb_read_free_rpl_tables=@save_rocksdb_read_free_rpl_tables;
+set global rocksdb_read_free_rpl=@save_rocksdb_read_free_rpl;
 SET session rocksdb_blind_delete_primary_key=@save_rocksdb_blind_delete_primary_key;
 SET session rocksdb_master_skip_tx_api=@save_rocksdb_master_skip_tx_api;
 DROP TABLE t1, t2;

storage/rocksdb/mysql-test/rocksdb/r/blind_delete_rr.result

+include/master-slave.inc
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
+[connection master]
+SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ;
+set @save_rocksdb_blind_delete_primary_key=@@session.rocksdb_blind_delete_primary_key;
+set @save_rocksdb_master_skip_tx_api=@@session.rocksdb_master_skip_tx_api;
+DROP TABLE IF EXISTS t1,t2;
+create table t1 (id int primary key, value int, value2 varchar(200)) engine=rocksdb;
+create table t2 (id int primary key, value int, value2 varchar(200), index(value)) engine=rocksdb;
+SET session rocksdb_blind_delete_primary_key=1;
+select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_deleted_blind';
+select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_deleted_blind';
+variable_value-@c
+1000
+SELECT count(*) FROM t1;
+count(*)
+9000
+include/sync_slave_sql_with_master.inc
+SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ;
+SELECT count(*) FROM t1;
+count(*)
+9000
+select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_deleted_blind';
+select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_deleted_blind';
+variable_value-@c
+0
+SELECT count(*) FROM t2;
+count(*)
+9000
+SET session rocksdb_master_skip_tx_api=1;
+select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_deleted_blind';
+select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_deleted_blind';
+variable_value-@c
+1000
+SELECT count(*) FROM t1;
+count(*)
+8000
+SELECT count(*) FROM t2;
+count(*)
+8000
+include/sync_slave_sql_with_master.inc
+SELECT count(*) FROM t1;
+count(*)
+8000
+SELECT count(*) FROM t2;
+count(*)
+8000
+select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_deleted_blind';
+DELETE FROM t1 WHERE id BETWEEN 3001 AND 4000;
+DELETE FROM t2 WHERE id BETWEEN 3001 AND 4000;
+select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_deleted_blind';
+variable_value-@c
+0
+SELECT count(*) FROM t1;
+count(*)
+7000
+SELECT count(*) FROM t2;
+count(*)
+7000
+include/sync_slave_sql_with_master.inc
+SELECT count(*) FROM t1;
+count(*)
+7000
+SELECT count(*) FROM t2;
+count(*)
+7000
+DELETE FROM t1 WHERE id = 10;
+SELECT count(*) FROM t1;
+count(*)
+7000
+call mtr.add_suppression("Slave SQL.*Could not execute Delete_rows event on table test.t1.*Error_code.*");
+call mtr.add_suppression("Slave: Can't find record in 't1'.*");
+include/wait_for_slave_sql_error.inc [errno=1032]
+set @save_rocksdb_read_free_rpl=@@global.rocksdb_read_free_rpl;
+set global rocksdb_read_free_rpl=PK_SK;
+START SLAVE;
+include/sync_slave_sql_with_master.inc
+SELECT count(*) FROM t1;
+count(*)
+7000
+set global rocksdb_read_free_rpl=@save_rocksdb_read_free_rpl;
+SET session rocksdb_blind_delete_primary_key=@save_rocksdb_blind_delete_primary_key;
+SET session rocksdb_master_skip_tx_api=@save_rocksdb_master_skip_tx_api;
+DROP TABLE t1, t2;
+include/rpl_end.inc

storage/rocksdb/mysql-test/rocksdb/r/bloomfilter3.result

@@ -20,6 +20,24 @@ id1	id2	link_type	visibility	data	time	version
 select case when variable_value-@c > 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_bloom_filter_prefix_checked';
 case when variable_value-@c > 0 then 'true' else 'false' end
 true
+# MariaDB: we don't have optimizer_force_index_for_range, but we can use EITS
+#          to get the query plan we want.
+set @tmp_use_stat_tables= @@use_stat_tables;
+set use_stat_tables='preferably';
+analyze table linktable persistent for all;
+Table	Op	Msg_type	Msg_text
+test.linktable	analyze	status	Engine-independent statistics collected
+test.linktable	analyze	status	OK
+flush tables;
+explain select * from linktable;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	linktable	ALL	NULL	NULL	NULL	NULL	10000	
+# This must use range(id1_type2), key_len=24
+explain
+select id1, id2, link_type, visibility, data, time, version from linktable
+FORCE INDEX(`id1_type2`)  where id1 = 100 and link_type = 1 and time >= 0 and time <= 9223372036854775807 order by time desc;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	linktable	range	id1_type2	id1_type2	24	NULL	1000	Using where; Using index
 select variable_value into @c from information_schema.global_status where variable_name='rocksdb_bloom_filter_prefix_checked';
 select id1, id2, link_type, visibility, data, time, version from linktable FORCE INDEX(`id1_type2`)  where id1 = 100 and link_type = 1 and time >= 0 and time <= 9223372036854775807 order by time desc;
 id1	id2	link_type	visibility	data	time	version

storage/rocksdb/mysql-test/rocksdb/r/bloomfilter5.result

@@ -59,4 +59,27 @@ insert into t4 values (1, 0xFFFF, 0xFFF, 12345);
 # This must not fail an assert:
 select * from t4 force index(kp1) where kp1=0xFFFFFFFF and kp2<=0xFFFFFFFF order by  kp2 desc;
 pk	kp1	kp2	col1
-drop table t1,t2,t3,t4;
+#
+#  Issue #881: Issue #809 still occurs for reverse scans on forward cfs
+#
+create table t5 (
+id1 bigint not null,
+id2 bigint not null,
+id3 varchar(100) not null,
+id4 int not null,
+id5 int not null,
+value bigint,
+value2 varchar(100),
+primary key (id1, id2, id3, id4) COMMENT 'bf5_1'
+) engine=ROCKSDB;
+insert into t5 select * from t1;
+set global rocksdb_force_flush_memtable_now=1;
+# An index scan starting from the end of the table:
+explain
+select * from t5 order by id1 desc,id2 desc, id3 desc, id4 desc limit 1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t5	index	NULL	PRIMARY	122	NULL	1	
+select * from t5 order by id1 desc,id2 desc, id3 desc, id4 desc limit 1;
+id1	id2	id3	id4	id5	value	value2
+1000	2000	2000	10000	10000	1000	aaabbbccc
+drop table t1,t2,t3,t4,t5;

storage/rocksdb/mysql-test/rocksdb/r/bulk_load.result

@@ -22,19 +22,20 @@ KEY(a)
 ) ENGINE=ROCKSDB COLLATE 'latin1_bin' PARTITION BY KEY() PARTITIONS 4;
 connect  other,localhost,root,,;
 set session transaction isolation level repeatable read;
-select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
-STAT_TYPE	VALUE
-DB_NUM_SNAPSHOTS	0
 start transaction with consistent snapshot;
-select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
-STAT_TYPE	VALUE
-DB_NUM_SNAPSHOTS	1
-connection default;
+select VALUE > 0 as 'Has opened snapshots' from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
+Has opened snapshots
+1
+SET @@GLOBAL.ROCKSDB_UPDATE_CF_OPTIONS=
+'cf1={write_buffer_size=8m;target_file_size_base=1m};';
 set rocksdb_bulk_load=1;
 set rocksdb_bulk_load_size=100000;
 LOAD DATA INFILE <input_file> INTO TABLE t1;
+pk	a	b
 LOAD DATA INFILE <input_file> INTO TABLE t2;
+pk	a	b
 LOAD DATA INFILE <input_file> INTO TABLE t3;
+pk	a	b
 set rocksdb_bulk_load=0;
 SHOW TABLE STATUS WHERE name LIKE 't%';
 Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	Index_length	Data_free	Auto_increment	Create_time	Update_time	Check_time	Collation	Checksum	Create_options	Comment

storage/rocksdb/mysql-test/rocksdb/r/bulk_load_rev_cf.result

@@ -22,19 +22,20 @@ KEY(a)
 ) ENGINE=ROCKSDB COLLATE 'latin1_bin' PARTITION BY KEY() PARTITIONS 4;
 connect  other,localhost,root,,;
 set session transaction isolation level repeatable read;
-select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
-STAT_TYPE	VALUE
-DB_NUM_SNAPSHOTS	0
 start transaction with consistent snapshot;
-select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
-STAT_TYPE	VALUE
-DB_NUM_SNAPSHOTS	1
-connection default;
+select VALUE > 0 as 'Has opened snapshots' from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
+Has opened snapshots
+1
+SET @@GLOBAL.ROCKSDB_UPDATE_CF_OPTIONS=
+'cf1={write_buffer_size=8m;target_file_size_base=1m};';
 set rocksdb_bulk_load=1;
 set rocksdb_bulk_load_size=100000;
 LOAD DATA INFILE <input_file> INTO TABLE t1;
+pk	a	b
 LOAD DATA INFILE <input_file> INTO TABLE t2;
+pk	a	b
 LOAD DATA INFILE <input_file> INTO TABLE t3;
+pk	a	b
 set rocksdb_bulk_load=0;
 SHOW TABLE STATUS WHERE name LIKE 't%';
 Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	Index_length	Data_free	Auto_increment	Create_time	Update_time	Check_time	Collation	Checksum	Create_options	Comment

storage/rocksdb/mysql-test/rocksdb/r/bulk_load_rev_cf_and_data.result

@@ -22,19 +22,20 @@ KEY(a)
 ) ENGINE=ROCKSDB COLLATE 'latin1_bin' PARTITION BY KEY() PARTITIONS 4;
 connect  other,localhost,root,,;
 set session transaction isolation level repeatable read;
-select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
-STAT_TYPE	VALUE
-DB_NUM_SNAPSHOTS	0
 start transaction with consistent snapshot;
-select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
-STAT_TYPE	VALUE
-DB_NUM_SNAPSHOTS	1
-connection default;
+select VALUE > 0 as 'Has opened snapshots' from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
+Has opened snapshots
+1
+SET @@GLOBAL.ROCKSDB_UPDATE_CF_OPTIONS=
+'cf1={write_buffer_size=8m;target_file_size_base=1m};';
 set rocksdb_bulk_load=1;
 set rocksdb_bulk_load_size=100000;
 LOAD DATA INFILE <input_file> INTO TABLE t1;
+pk	a	b
 LOAD DATA INFILE <input_file> INTO TABLE t2;
+pk	a	b
 LOAD DATA INFILE <input_file> INTO TABLE t3;
+pk	a	b
 set rocksdb_bulk_load=0;
 SHOW TABLE STATUS WHERE name LIKE 't%';
 Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	Index_length	Data_free	Auto_increment	Create_time	Update_time	Check_time	Collation	Checksum	Create_options	Comment

storage/rocksdb/mysql-test/rocksdb/r/bulk_load_rev_data.result

@@ -22,19 +22,20 @@ KEY(a)
 ) ENGINE=ROCKSDB COLLATE 'latin1_bin' PARTITION BY KEY() PARTITIONS 4;
 connect  other,localhost,root,,;
 set session transaction isolation level repeatable read;
-select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
-STAT_TYPE	VALUE
-DB_NUM_SNAPSHOTS	0
 start transaction with consistent snapshot;
-select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
-STAT_TYPE	VALUE
-DB_NUM_SNAPSHOTS	1
-connection default;
+select VALUE > 0 as 'Has opened snapshots' from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
+Has opened snapshots
+1
+SET @@GLOBAL.ROCKSDB_UPDATE_CF_OPTIONS=
+'cf1={write_buffer_size=8m;target_file_size_base=1m};';
 set rocksdb_bulk_load=1;
 set rocksdb_bulk_load_size=100000;
 LOAD DATA INFILE <input_file> INTO TABLE t1;
+pk	a	b
 LOAD DATA INFILE <input_file> INTO TABLE t2;
+pk	a	b
 LOAD DATA INFILE <input_file> INTO TABLE t3;
+pk	a	b
 set rocksdb_bulk_load=0;
 SHOW TABLE STATUS WHERE name LIKE 't%';
 Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	Index_length	Data_free	Auto_increment	Create_time	Update_time	Check_time	Collation	Checksum	Create_options	Comment

storage/rocksdb/mysql-test/rocksdb/r/bulk_load_unsorted.result

@@ -59,13 +59,10 @@ CREATE TABLE t3(a INT, b INT, PRIMARY KEY(a) COMMENT "cf1")
 ENGINE=ROCKSDB PARTITION BY KEY() PARTITIONS 4;
 connect  other,localhost,root,,;
 set session transaction isolation level repeatable read;
-select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
-STAT_TYPE	VALUE
-DB_NUM_SNAPSHOTS	0
 start transaction with consistent snapshot;
-select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
-STAT_TYPE	VALUE
-DB_NUM_SNAPSHOTS	1
+select VALUE > 0 as 'Has opened snapshots' from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
+Has opened snapshots
+1
 connection default;
 set rocksdb_bulk_load=1;
 set rocksdb_bulk_load_size=100000;

storage/rocksdb/mysql-test/rocksdb/r/bulk_load_unsorted_rev.result

@@ -59,13 +59,10 @@ CREATE TABLE t3(a INT, b INT, PRIMARY KEY(a) COMMENT "rev:cf1")
 ENGINE=ROCKSDB PARTITION BY KEY() PARTITIONS 4;
 connect  other,localhost,root,,;
 set session transaction isolation level repeatable read;
-select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
-STAT_TYPE	VALUE
-DB_NUM_SNAPSHOTS	0
 start transaction with consistent snapshot;
-select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
-STAT_TYPE	VALUE
-DB_NUM_SNAPSHOTS	1
+select VALUE > 0 as 'Has opened snapshots' from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
+Has opened snapshots
+1
 connection default;
 set rocksdb_bulk_load=1;
 set rocksdb_bulk_load_size=100000;

storage/rocksdb/mysql-test/rocksdb/r/check_flags.result

+set debug_sync='RESET';
+set global rocksdb_debug_ttl_read_filter_ts = -10;
+connect  conn1, localhost, root,,;
+connection default;
+CREATE TABLE t1 (id INT, value INT, KEY (id), KEY (value)) ENGINE=ROCKSDB;
+CREATE TABLE t2 (id INT, value INT) ENGINE=ROCKSDB;
+CREATE TABLE t3 (id INT, kp1 INT, PRIMARY KEY (id), KEY(kp1)) ENGINE=ROCKSDB COMMENT='ttl_duration=1';
+INSERT INTO t1 VALUES (1,1), (2,2), (3,3), (4,4), (5,5);
+INSERT INTO t2 SELECT * FROM t1;
+INSERT INTO t3 SELECT * FROM t1;
+connection conn1;
+set debug_sync='rocksdb.check_flags_rmi SIGNAL parked WAIT_FOR go';
+SELECT value FROM t1 WHERE value = 3;
+connection default;
+set debug_sync='now WAIT_FOR parked';
+KILL QUERY $conn1_id;
+set debug_sync='now SIGNAL go';
+connection conn1;
+ERROR 70100: Query execution was interrupted
+set debug_sync='RESET';
+connection conn1;
+set debug_sync='rocksdb.check_flags_rmi_scan SIGNAL parked WAIT_FOR go';
+SELECT DISTINCT(id) FROM t1 WHERE value = 5 AND id IN (1, 3, 5);
+connection default;
+set debug_sync='now WAIT_FOR parked';
+KILL QUERY $conn1_id;
+set debug_sync='now SIGNAL go';
+connection conn1;
+ERROR 70100: Query execution was interrupted
+set debug_sync='RESET';
+connection conn1;
+set debug_sync='rocksdb.check_flags_inwd SIGNAL parked WAIT_FOR go';
+SELECT value FROM t1 WHERE value > 3;
+connection default;
+set debug_sync='now WAIT_FOR parked';
+KILL QUERY $conn1_id;
+set debug_sync='now SIGNAL go';
+connection conn1;
+ERROR 70100: Query execution was interrupted
+set debug_sync='RESET';
+connection conn1;
+set debug_sync='rocksdb.check_flags_rnwd SIGNAL parked WAIT_FOR go';
+SELECT id FROM t2;
+connection default;
+set debug_sync='now WAIT_FOR parked';
+KILL QUERY $conn1_id;
+set debug_sync='now SIGNAL go';
+connection conn1;
+ERROR 70100: Query execution was interrupted
+set debug_sync='RESET';
+connection conn1;
+set debug_sync='rocksdb.check_flags_ser SIGNAL parked WAIT_FOR go';
+SELECT kp1 FROM t3 ORDER BY kp1;
+connection default;
+set debug_sync='now WAIT_FOR parked';
+KILL QUERY $conn1_id;
+set debug_sync='now SIGNAL go';
+connection conn1;
+ERROR 70100: Query execution was interrupted
+connection default;
+disconnect conn1;
+set debug_sync='RESET';
+set global rocksdb_debug_ttl_read_filter_ts = DEFAULT;
+DROP TABLE t1;
+DROP TABLE t2;
+DROP TABLE t3;

storage/rocksdb/mysql-test/rocksdb/r/create_no_primary_key_table.result

@@ -8,7 +8,7 @@ ERROR HY000: Table without primary key cannot be created outside mysql schema.
 CREATE TABLE IF NOT EXISTS mysql_table_2 (a INT) ENGINE=ROCKSDB;
 ERROR HY000: Table without primary key cannot be created outside mysql schema.
 CREATE TABLE mysql_table_no_cols ENGINE=ROCKSDB;
-ERROR HY000: Table without primary key cannot be created outside mysql schema.
+ERROR 42000: A table must have at least 1 column
 CREATE TABLE mysql.mysql_table_2 (a INT) ENGINE=ROCKSDB;
 CREATE TABLE mysql_primkey (a INT PRIMARY KEY, b INT, c INT, d INT, INDEX (c)) ENGINE=ROCKSDB;
 ALTER TABLE mysql_primkey DROP b, DROP a, ADD (f INT PRIMARY KEY);
@@ -29,10 +29,24 @@ DROP INDEX `PRIMARY` ON mysql_primkey4;
 ERROR HY000: Table without primary key cannot be created outside mysql schema.
 ALTER TABLE mysql.mysql_table ADD PRIMARY KEY (a);
 ALTER TABLE mysql.mysql_table DROP PRIMARY KEY;
+SET default_storage_engine=ROCKSDB;
+CREATE TABLE mysql_noeng(a INT, b INT);
+ERROR HY000: Table without primary key cannot be created outside mysql schema.
+SET sql_mode="";
+CREATE TABLE mysql_noeng_sub(a INT, b INT) ENGINE=BOGUS_ENGINE;
+ERROR HY000: Table without primary key cannot be created outside mysql schema.
+CREATE TABLE mysql_primkey5 LIKE mysql_primkey;
+SET @@global.block_create_no_primary_key = false;
+CREATE TABLE mysql_no_primkey (a INT) ENGINE=ROCKSDB;
+SET @@global.block_create_no_primary_key = true;
+CREATE TABLE mysql_block_no_primkey LIKE mysql_no_primkey;
+ERROR HY000: Table without primary key cannot be created outside mysql schema.
 DROP TABLE mysql_primkey;
 DROP TABLE mysql_primkey2;
 DROP TABLE mysql_primkey3;
 DROP TABLE mysql_primkey4;
+DROP TABLE mysql_primkey5;
+DROP TABLE mysql_no_primkey;
 USE mysql;
 DROP TABLE mysql_table;
 DROP TABLE mysql_table_2;

storage/rocksdb/mysql-test/rocksdb/t/cons_snapshot_read_committed.opt

+--rocksdb_default_cf_options=disable_auto_compactions=true
\ No newline at end of file