MDEV-14455: rocksdb.2pc_group_commit failed in buildbot

Use RocksDB debug sync points to introduce a sync delay. This
commits to get grouped even when the datadir is on ramdisk.

For some unclear reason the effect is visible on write_prepared
but not write_committed, so run the test only with write_prepared.

storage/rocksdb/ha_rocksdb.cc

@@ -71,6 +71,9 @@
 #include "util/stop_watch.h"
 #include "./rdb_source_revision.h"
 
+// MariaRocks: this is needed to access RocksDB debug syncpoints:
+#include "test_util/sync_point.h"
+
 /* MyRocks includes */
 #include "./event_listener.h"
 #include "./ha_rocksdb_proto.h"
@@ -7789,6 +7792,28 @@ int ha_rocksdb::create(const char *const name, TABLE *const table_arg,
     }
   }
 
+  // The below adds/clears hooks in RocksDB sync points. There's no reason for
+  // this code to be in ::create() but it needs to be somewhere where it is
+  // away from any tight loops and where one can invoke it from mtr:
+  DBUG_EXECUTE_IF("rocksdb_enable_delay_commits",
+    {
+      auto syncpoint= rocksdb::SyncPoint::GetInstance();
+      syncpoint->SetCallBack("DBImpl::WriteImpl:BeforeLeaderEnters",
+                             [&](void* /*arg*/) {my_sleep(500);} );
+      syncpoint->EnableProcessing();
+      push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, ER_WRONG_ARGUMENTS,
+                        "enable_delay_commits_mode ON");
+
+    });
+  DBUG_EXECUTE_IF("rocksdb_disable_delay_commits",
+    {
+      auto syncpoint= rocksdb::SyncPoint::GetInstance();
+      syncpoint->ClearCallBack("DBImpl::WriteImpl:BeforeLeaderEnters");
+      syncpoint->DisableProcessing();
+      push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, ER_WRONG_ARGUMENTS,
+                        "enable_delay_commits_mode OFF");
+    });
+
   DBUG_RETURN(create_table(str, table_arg, create_info->auto_increment_value));
 }
 

storage/rocksdb/mysql-test/rocksdb/include/have_write_prepared.inc

+if (`select count(*) = 0 from information_schema.session_variables where variable_name = 'rocksdb_write_policy' and variable_value = 'write_prepared';`) {
+  --skip Test requires write_prepared policy
+}

storage/rocksdb/mysql-test/rocksdb/r/2pc_group_commit.result

@@ -27,6 +27,12 @@ select IF(variable_value - @b3 between 1000 and 1500, 'OK', variable_value - @b3
 from information_schema.global_status where variable_name='Rocksdb_wal_synced';
 Rocksdb_wal_synced
 OK
+set debug_dbug='+d,rocksdb_enable_delay_commits';
+create table dummy10(a int) engine=rocksdb;
+Warnings:
+Warning	1210	enable_delay_commits_mode ON
+drop table dummy10;
+set debug_dbug='-d,rocksdb_enable_delay_commits';
 ## 
 ## 2PC + durability + group commit
 ## 
@@ -45,6 +51,12 @@ select IF(variable_value - @b3 between 1 and 9000, 'OK', variable_value - @b3)
 from information_schema.global_status where variable_name='Rocksdb_wal_synced';
 IF(variable_value - @b3 between 1 and 9000, 'OK', variable_value - @b3)
 OK
+set debug_dbug='+d,rocksdb_disable_delay_commits';
+create table dummy10(a int) engine=rocksdb;
+Warnings:
+Warning	1210	enable_delay_commits_mode OFF
+drop table dummy10;
+set debug_dbug='-d,rocksdb_disable_delay_commits';
 ## 
 # 2PC enabled, MyRocks durability disabled, single thread
 ## 

storage/rocksdb/mysql-test/rocksdb/t/2pc_group_commit.test

@@ -4,6 +4,13 @@
 --echo # Disable for valgrind because this takes too long
 --source include/not_valgrind.inc
 
+# MariaDB: tooling to slowdown commits (also when running on ramdisk, we need 
+# write_prepared for some reason, this is set in the .opt file)
+--source include/have_debug.inc
+--source include/have_debug_sync.inc
+--source include/have_write_prepared.inc
+
+
 --disable_warnings
 DROP DATABASE IF EXISTS mysqlslap;
 --enable_warnings
@@ -15,6 +22,7 @@ CREATE TABLE t1(id BIGINT AUTO_INCREMENT, value BIGINT, PRIMARY KEY(id)) ENGINE=
 SET @save_rocksdb_enable_2pc= @@rocksdb_enable_2pc;
 SET @save_rocksdb_flush_log_at_trx_commit= @@rocksdb_flush_log_at_trx_commit;
 
+
 #
 # In MariaDB, regular group commit operation does not cause increment of
 # rocksdb_wal_group_syncs. 
@@ -39,6 +47,19 @@ from information_schema.global_status where variable_name='Binlog_group_commits'
 select IF(variable_value - @b3 between 1000 and 1500, 'OK', variable_value - @b3) as Rocksdb_wal_synced
 from information_schema.global_status where variable_name='Rocksdb_wal_synced';
 
+# SQL layer solution is sufficient for Binlog counts but not RocksDB.
+#set @tmp_bcwc= @@binlog_commit_wait_count;
+#set @tmp_bcwu= @@binlog_commit_wait_usec;
+#set global binlog_commit_wait_count=30;
+#set global binlog_commit_wait_usec=500*1000;
+
+# RocksDB-side solution:
+
+set debug_dbug='+d,rocksdb_enable_delay_commits';
+create table dummy10(a int) engine=rocksdb;
+drop table dummy10;
+set debug_dbug='-d,rocksdb_enable_delay_commits';
+
 --echo ## 
 --echo ## 2PC + durability + group commit
 --echo ## 
@@ -55,6 +76,14 @@ from information_schema.global_status where variable_name='Binlog_group_commits'
 select IF(variable_value - @b3 between 1 and 9000, 'OK', variable_value - @b3)
 from information_schema.global_status where variable_name='Rocksdb_wal_synced';
 
+#set global binlog_commit_wait_count= @tmp_bcwc;
+#set global binlog_commit_wait_usec=  @tmp_bcwu;
+
+set debug_dbug='+d,rocksdb_disable_delay_commits';
+create table dummy10(a int) engine=rocksdb;
+drop table dummy10;
+set debug_dbug='-d,rocksdb_disable_delay_commits';
+
 --echo ## 
 --echo # 2PC enabled, MyRocks durability disabled, single thread
 --echo ##