MDEV-4991: GTID binlog indexing

Improve the performance of slave connect using B+-Tree indexes on each binlog
file. The index allows fast lookup of a GTID position to the corresponding
offset in the binlog file, as well as lookup of a position to find the
corresponding GTID position.

This eliminates a costly sequential scan of the starting binlog file
to find the GTID starting position when a slave connects. This is
especially costly if the binlog file is not cached in memory (IO
cost), or if it is encrypted or a lot of slaves connect simultaneously
(CPU cost).

The size of the index files is generally less than 1% of the binlog data, so
not expected to be an issue.

Most of the work writing the index is done as a background task, in
the binlog background thread. This minimises the performance impact on
transaction commit. A simple global mutex is used to protect index
reads and (background) index writes; this is fine as slave connect is
a relatively infrequent operation.

Here are the user-visible options and status variables. The feature is on by
default and is expected to need no tuning or configuration for most users.

binlog_gtid_index
  On by default. Can be used to disable the indexes for testing purposes.

binlog_gtid_index_page_size (default 4096)
  Page size to use for the binlog GTID index. This is the size of the nodes
  in the B+-tree used internally in the index. A very small page-size (64 is
  the minimum) will be less efficient, but can be used to stress the
  BTree-code during testing.

binlog_gtid_index_span_min (default 65536)
  Control sparseness of the binlog GTID index. If set to N, at most one
  index record will be added for every N bytes of binlog file written.
  This can be used to reduce the number of records in the index, at
  the cost only of having to scan a few more events in the binlog file
  before finding the target position

Two status variables are available to monitor the use of the GTID indexes:

  Binlog_gtid_index_hit
  Binlog_gtid_index_miss

The "hit" status increments for each successful lookup in a GTID index.
The "miss" increments when a lookup is not possible. This indicates that the
index file is missing (eg. binlog written by old server version
without GTID index support), or corrupt.
Signed-off-by: Kristian Nielsen <knielsen@knielsen-hq.org>

libmysqld/CMakeLists.txt

@@ -126,7 +126,7 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc
            ../sql/sql_expression_cache.cc
            ../sql/my_apc.cc ../sql/my_apc.h
            ../sql/my_json_writer.cc ../sql/my_json_writer.h
-	   ../sql/rpl_gtid.cc
+	   ../sql/rpl_gtid.cc ../sql/gtid_index.cc
            ../sql/sql_explain.cc ../sql/sql_explain.h
            ../sql/sql_analyze_stmt.cc ../sql/sql_analyze_stmt.h
            ../sql/compat56.cc

mysql-test/main/mysqld--help.result

@@ -98,6 +98,17 @@ The following specify which files/extra groups are read (specified before remain
  involve user-defined functions (i.e. UDFs) or the UUID()
  function; for those, row-based binary logging is
  automatically used.
+ --binlog-gtid-index Enable the creation of a GTID index for every binlog
+ file, and the use of such index for speeding up GTID
+ lookup in the binlog.
+ (Defaults to on; use --skip-binlog-gtid-index to disable.)
+ --binlog-gtid-index-page-size=# 
+ Page size to use for the binlog GTID index.
+ --binlog-gtid-index-span-min=# 
+ Control sparseness of the binlog GTID index. If set to N,
+ at most one index record will be added for every N bytes
+ of binlog file written, to reduce the size of the index.
+ Normally does not need tuning.
  --binlog-ignore-db=name 
  Tells the master that updates to the given database
  should not be logged to the binary log.
@@ -1597,6 +1608,9 @@ binlog-direct-non-transactional-updates FALSE
 binlog-expire-logs-seconds 0
 binlog-file-cache-size 16384
 binlog-format MIXED
+binlog-gtid-index TRUE
+binlog-gtid-index-page-size 4096
+binlog-gtid-index-span-min 65536
 binlog-legacy-event-pos FALSE
 binlog-optimize-thread-scheduling TRUE
 binlog-row-event-max-size 8192

mysql-test/suite/binlog/r/binlog_gtid_index.result

+SET GLOBAL binlog_gtid_index= 0;
+SET GLOBAL binlog_gtid_index= 1;
+SET @gtid1= @@gtid_binlog_pos;
+CREATE TABLE t1 (a INT PRIMARY KEY);
+SET @gtid2= @@gtid_binlog_pos;
+INSERT INTO t1 VALUES (1);
+SET @gtid3= @@gtid_binlog_pos;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t1 VALUES (3);
+INSERT INTO t1 VALUES (4);
+SET @gtid4= @@gtid_binlog_pos;
+INSERT INTO t1 VALUES (5);
+SET @gtid5= @@gtid_binlog_pos;
+SET @gtid6= @@gtid_binlog_pos;
+INSERT INTO t1 VALUES (106);
+INSERT INTO t1 VALUES (107);
+Ok
+1
+Ok
+1
+Ok
+1
+Ok
+1
+Ok
+1
+Ok
+1
+FLUSH BINARY LOGS;
+Ok
+1
+Ok
+1
+Ok
+1
+Ok
+1
+Ok
+1
+Ok
+1
+*** Test that purge deletes the gtid index files. ***
+FLUSH BINARY LOGS;
+INSERT INTO t1 VALUES (200);
+FLUSH BINARY LOGS;
+INSERT INTO t1 VALUES (201);
+FLUSH BINARY LOGS;
+INSERT INTO t1 VALUES (202);
+PURGE BINARY LOGS TO 'FILE';
+*** Test missed index lookup due to missing or corrupt index file.
+FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
+INSERT INTO t1 VALUES (301);
+INSERT INTO t1 VALUES (302);
+INSERT INTO t1 VALUES (303);
+SET @gtid_pos= @@GLOBAL.gtid_binlog_pos;
+INSERT INTO t1 VALUES (304);
+INSERT INTO t1 VALUES (305);
+FLUSH NO_WRITE_TO_BINLOG STATUS;
++++ Initial status:
+SHOW STATUS LIKE 'binlog_gtid_index_%';
+Variable_name	Value
+Binlog_gtid_index_hit	0
+Binlog_gtid_index_miss	0
++++ GTID Lookup in good index.
+Gtid_Lookup_Ok
+1
+SHOW STATUS LIKE 'binlog_gtid_index_%';
+Variable_name	Value
+Binlog_gtid_index_hit	1
+Binlog_gtid_index_miss	0
++++ GTID Lookup, index file is missing.
+Gtid_Lookup_Ok
+1
+SHOW STATUS LIKE 'binlog_gtid_index_%';
+Variable_name	Value
+Binlog_gtid_index_hit	1
+Binlog_gtid_index_miss	1
+FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
+INSERT INTO t1 VALUES (306);
+SET @gtid_pos= @@GLOBAL.gtid_binlog_pos;
+INSERT INTO t1 VALUES (307);
+INSERT INTO t1 VALUES (308);
+FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
++++ GTID Lookup, first page of index is corrupt.
+Gtid_Lookup_Ok
+1
+SHOW STATUS LIKE 'binlog_gtid_index_%';
+Variable_name	Value
+Binlog_gtid_index_hit	1
+Binlog_gtid_index_miss	2
+SET @old_page_size= @@GLOBAL.binlog_gtid_index_page_size;
+SET @old_span_min= @@GLOBAL.binlog_gtid_index_span_min;
+SET GLOBAL binlog_gtid_index_page_size= 64;
+SET GLOBAL binlog_gtid_index_span_min= 1;
+FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
+INSERT INTO t1 VALUES (310);
+INSERT INTO t1 VALUES (311);
+INSERT INTO t1 VALUES (312);
+SET @gtid_pos= @@GLOBAL.gtid_binlog_pos;
+INSERT INTO t1 VALUES (313);
+INSERT INTO t1 VALUES (314);
+INSERT INTO t1 VALUES (315);
+INSERT INTO t1 VALUES (316);
+FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
+SET GLOBAL binlog_gtid_index_page_size= @old_page_size;
+SET GLOBAL binlog_gtid_index_span_min= @old_span_min;
++++ GTID Lookup, root page of index is corrupt.
+Gtid_Lookup_Ok
+1
+SHOW STATUS LIKE 'binlog_gtid_index_%';
+Variable_name	Value
+Binlog_gtid_index_hit	1
+Binlog_gtid_index_miss	3
+*** Test BINLOG_GTID_POS() with too-large offset.
+FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
+INSERT INTO t1 VALUES (401);
+INSERT INTO t1 VALUES (402);
++++ Test the hot index.
+SELECT BINLOG_GTID_POS('FILE', 100000000);
+BINLOG_GTID_POS('FILE', 100000000)
+NULL
+SHOW STATUS LIKE 'binlog_gtid_index_%';
+Variable_name	Value
+Binlog_gtid_index_hit	2
+Binlog_gtid_index_miss	3
+FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
++++ Test the cold index.
+SELECT BINLOG_GTID_POS('FILE', 100000000);
+BINLOG_GTID_POS('FILE', 100000000)
+NULL
+SHOW STATUS LIKE 'binlog_gtid_index_%';
+Variable_name	Value
+Binlog_gtid_index_hit	3
+Binlog_gtid_index_miss	3
+DROP TABLE t1;

mysql-test/suite/binlog/r/binlog_gtid_index_crash.result

+*** Test that binlog GTID index is recovered after a crash.
+CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB;
+Ok
+1
+Ok
+1
+Ok
+1
+SHOW STATUS LIKE 'binlog_gtid_index_%';
+Variable_name	Value
+Binlog_gtid_index_hit	3
+Binlog_gtid_index_miss	0
+*** Crash the server, check that GTID index can be used after restart.
+SET debug_dbug="d,crash_shutdown";
+shutdown;
+ERROR HY000: Lost connection to server during query
+FLUSH NO_WRITE_TO_BINLOG STATUS;
+Ok
+1
+Ok
+1
+Ok
+1
+SHOW STATUS LIKE 'binlog_gtid_index_%';
+Variable_name	Value
+Binlog_gtid_index_hit	3
+Binlog_gtid_index_miss	0
+DROP TABLE t1;

mysql-test/suite/binlog/t/binlog_gtid_index.test

+--source include/have_binlog_format_mixed.inc
+
+SET GLOBAL binlog_gtid_index= 0;
+SET GLOBAL binlog_gtid_index= 1;
+
+--let $file= query_get_value(SHOW MASTER STATUS, File, 1)
+--let $pos1= query_get_value(SHOW MASTER STATUS, Position, 1)
+SET @gtid1= @@gtid_binlog_pos;
+CREATE TABLE t1 (a INT PRIMARY KEY);
+--let $pos2= query_get_value(SHOW MASTER STATUS, Position, 1)
+SET @gtid2= @@gtid_binlog_pos;
+INSERT INTO t1 VALUES (1);
+--let $pos3= query_get_value(SHOW MASTER STATUS, Position, 1)
+SET @gtid3= @@gtid_binlog_pos;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t1 VALUES (3);
+INSERT INTO t1 VALUES (4);
+--let $pos4= query_get_value(SHOW MASTER STATUS, Position, 1)
+SET @gtid4= @@gtid_binlog_pos;
+INSERT INTO t1 VALUES (5);
+--let $pos5= query_get_value(SHOW MASTER STATUS, Position, 1)
+SET @gtid5= @@gtid_binlog_pos;
+
+--disable_query_log
+--let $i=0
+while ($i < 100) {
+  eval INSERT INTO t1 VALUES (6 + $i);
+  inc $i;
+}
+--enable_query_log
+--let $pos6= query_get_value(SHOW MASTER STATUS, Position, 1)
+SET @gtid6= @@gtid_binlog_pos;
+
+INSERT INTO t1 VALUES (106);
+INSERT INTO t1 VALUES (107);
+
+# Test first the hot and then the cold index.
+--let $i= 0
+while ($i < 2) {
+  --disable_query_log
+  eval SELECT BINLOG_GTID_POS('$file', $pos1) = @gtid1 AS Ok;
+  eval SELECT BINLOG_GTID_POS('$file', $pos2) = @gtid2 AS Ok;
+  eval SELECT BINLOG_GTID_POS('$file', $pos3) = @gtid3 AS Ok;
+  eval SELECT BINLOG_GTID_POS('$file', $pos4) = @gtid4 AS Ok;
+  eval SELECT BINLOG_GTID_POS('$file', $pos5) = @gtid5 AS Ok;
+  eval SELECT BINLOG_GTID_POS('$file', $pos6) = @gtid6 AS Ok;
+  --enable_query_log
+
+  inc $i;
+  if ($i == 1) {
+    FLUSH BINARY LOGS;
+  }
+}
+
+--echo *** Test that purge deletes the gtid index files. ***
+FLUSH BINARY LOGS;
+INSERT INTO t1 VALUES (200);
+--let $file2= query_get_value(SHOW MASTER STATUS, File, 1)
+FLUSH BINARY LOGS;
+INSERT INTO t1 VALUES (201);
+--let $file3= query_get_value(SHOW MASTER STATUS, File, 1)
+FLUSH BINARY LOGS;
+INSERT INTO t1 VALUES (202);
+--let $file4= query_get_value(SHOW MASTER STATUS, File, 1)
+
+--replace_result $file3 FILE
+eval PURGE BINARY LOGS TO '$file3';
+
+--let $MYSQLD_DATADIR= `select @@datadir`
+--error 1
+--file_exists $MYSQLD_DATADIR/$file.idx
+--error 1
+--file_exists $MYSQLD_DATADIR/$file2.idx
+--file_exists $MYSQLD_DATADIR/$file3.idx
+--file_exists $MYSQLD_DATADIR/$file4.idx
+
+--echo *** Test missed index lookup due to missing or corrupt index file.
+FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
+--let $file= query_get_value(SHOW MASTER STATUS, File, 1)
+INSERT INTO t1 VALUES (301);
+INSERT INTO t1 VALUES (302);
+INSERT INTO t1 VALUES (303);
+--let $pos= query_get_value(SHOW MASTER STATUS, Position, 1)
+SET @gtid_pos= @@GLOBAL.gtid_binlog_pos;
+INSERT INTO t1 VALUES (304);
+INSERT INTO t1 VALUES (305);
+
+# BINLOG_GTID_POS() has a side effect: it increments binlog_gtid_index_hit
+--disable_ps2_protocol
+FLUSH NO_WRITE_TO_BINLOG STATUS;
+--echo +++ Initial status:
+SHOW STATUS LIKE 'binlog_gtid_index_%';
+--echo +++ GTID Lookup in good index.
+--disable_query_log
+eval SELECT BINLOG_GTID_POS('$file', $pos) = @gtid_pos AS Gtid_Lookup_Ok;
+--enable_query_log
+SHOW STATUS LIKE 'binlog_gtid_index_%';
+--remove_file $MYSQLD_DATADIR/$file.idx
+--echo +++ GTID Lookup, index file is missing.
+--disable_query_log
+eval SELECT BINLOG_GTID_POS('$file', $pos) = @gtid_pos AS Gtid_Lookup_Ok;
+--enable_query_log
+SHOW STATUS LIKE 'binlog_gtid_index_%';
+
+FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
+--let $file= query_get_value(SHOW MASTER STATUS, File, 1)
+INSERT INTO t1 VALUES (306);
+--let $pos= query_get_value(SHOW MASTER STATUS, Position, 1)
+SET @gtid_pos= @@GLOBAL.gtid_binlog_pos;
+INSERT INTO t1 VALUES (307);
+INSERT INTO t1 VALUES (308);
+# Rotate again so we hit an on-disk index file, not the "hot" index.
+FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
+
+# Corrupt the flag byte of the first page with an unused bit.
+--let FILE_TO_CORRUPT= $MYSQLD_DATADIR/$file.idx
+--perl
+use strict;
+use warnings;
+use Fcntl qw(:DEFAULT :seek);
+sysopen F, $ENV{FILE_TO_CORRUPT}, O_RDWR
+  or die "Cannot open file $ENV{FILE_TO_CORRUPT}: $!\n";
+# Corrupt the flag byte with an unused flag.
+sysseek(F, 16, SEEK_SET)
+  or die "Cannot seek file: $!\n";
+my $buf;
+sysread(F, $buf, 1)
+  or die "Cannot read file: $!\n";
+$buf= chr(ord($buf) | 0x80);
+sysseek(F, 16, SEEK_SET)
+  or die "Cannot seek file: $!\n";
+syswrite(F, $buf, 1) == 1
+  or die "Cannot write file: $!\n";
+close F;
+EOF
+
+--echo +++ GTID Lookup, first page of index is corrupt.
+--disable_query_log
+eval SELECT BINLOG_GTID_POS('$file', $pos) = @gtid_pos AS Gtid_Lookup_Ok;
+--enable_query_log
+SHOW STATUS LIKE 'binlog_gtid_index_%';
+
+# Corrupt the last byte of the root page.
+# Set a small page-size so we test corruption in something not the header page.
+SET @old_page_size= @@GLOBAL.binlog_gtid_index_page_size;
+SET @old_span_min= @@GLOBAL.binlog_gtid_index_span_min;
+SET GLOBAL binlog_gtid_index_page_size= 64;
+SET GLOBAL binlog_gtid_index_span_min= 1;
+FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
+--let $file= query_get_value(SHOW MASTER STATUS, File, 1)
+INSERT INTO t1 VALUES (310);
+INSERT INTO t1 VALUES (311);
+INSERT INTO t1 VALUES (312);
+--let $pos= query_get_value(SHOW MASTER STATUS, Position, 1)
+SET @gtid_pos= @@GLOBAL.gtid_binlog_pos;
+INSERT INTO t1 VALUES (313);
+INSERT INTO t1 VALUES (314);
+INSERT INTO t1 VALUES (315);
+INSERT INTO t1 VALUES (316);
+FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
+SET GLOBAL binlog_gtid_index_page_size= @old_page_size;
+SET GLOBAL binlog_gtid_index_span_min= @old_span_min;
+
+--let FILE_TO_CORRUPT= $MYSQLD_DATADIR/$file.idx
+--perl
+use strict;
+use warnings;
+use Fcntl qw(:DEFAULT :seek);
+sysopen F, $ENV{FILE_TO_CORRUPT}, O_RDWR
+  or die "Cannot open file $ENV{FILE_TO_CORRUPT}: $!\n";
+# Tricky: The index is written asynchroneously, it may still be incomplete.
+# So wait for the file to be written completely with a root node at the end.
+my $count= 0;
+for (;;) {
+  my $end= sysseek(F, 0, SEEK_END);
+  if ($end > 0 && ($end % 64) == 0) {
+    # The index file is non-empty with a full page at the end, test if the
+    # root page has been fully written. This is seen as bit 2 (PAGE_FLAG_LAST)
+    # and bit 3 (PAGE_FLAG_ROOT) being set (0xc).
+    my $flag;
+    if (sysseek(F, -64, SEEK_CUR) &&
+        sysread(F, $flag, 1) &&
+	(ord($flag) & 0xc) == 0xc) {
+      last;
+    }
+  }
+  die "Timeout waiting for GTID index to be non-empty\n"
+    if ++$count >= 500;
+  # Simple way to do sub-second sleep.
+  select(undef, undef, undef, 0.050);
+}
+# Corrupt the flag byte with an unused flag.
+sysseek(F, -2, SEEK_END)
+  or die "Cannot seek file: $!\n";
+my $buf;
+sysread(F, $buf, 1)
+  or die "Cannot read file: $!\n";
+$buf= chr(ord($buf) ^ 0x4);
+sysseek(F, -2, SEEK_END)
+  or die "Cannot seek file: $!\n";
+syswrite(F, $buf, 1) == 1
+  or die "Cannot write file: $!\n";
+close F;
+EOF
+
+--echo +++ GTID Lookup, root page of index is corrupt.
+--disable_query_log
+eval SELECT BINLOG_GTID_POS('$file', $pos) = @gtid_pos AS Gtid_Lookup_Ok;
+--enable_query_log
+SHOW STATUS LIKE 'binlog_gtid_index_%';
+
+--echo *** Test BINLOG_GTID_POS() with too-large offset.
+# New binlog to skip the now corrupted one.
+FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
+--let $file= query_get_value(SHOW MASTER STATUS, File, 1)
+INSERT INTO t1 VALUES (401);
+INSERT INTO t1 VALUES (402);
+--echo +++ Test the hot index.
+--replace_result $file FILE
+eval SELECT BINLOG_GTID_POS('$file', 100000000);
+SHOW STATUS LIKE 'binlog_gtid_index_%';
+FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
+--echo +++ Test the cold index.
+--replace_result $file FILE
+eval SELECT BINLOG_GTID_POS('$file', 100000000);
+SHOW STATUS LIKE 'binlog_gtid_index_%';
+--enable_ps2_protocol
+
+DROP TABLE t1;

mysql-test/suite/binlog/t/binlog_gtid_index_crash.opt

+--binlog-gtid-index-page-size=128 --binlog-gtid-index-span-min=1

mysql-test/suite/binlog/t/binlog_gtid_index_crash.test

+--source include/have_innodb.inc
+# Don't test this under valgrind, memory leaks will occur
+--source include/not_valgrind.inc
+# Avoid CrashReporter popup on Mac
+--source include/not_crashrep.inc
+# Binary must be compiled with debug for crash to occur
+--source include/have_debug.inc
+--source include/have_binlog_format_row.inc
+
+# We have an .opt file that sets a small page size and disables sparseness,
+# so we get something non-trivial in the GTID index even with a small amount
+# of binlogged events.
+
+--echo *** Test that binlog GTID index is recovered after a crash.
+CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB;
+
+--disable_query_log
+INSERT INTO t1 VALUES (0, 0);
+INSERT INTO t1 VALUES (1, 0);
+INSERT INTO t1 VALUES (2, 0);
+--let $i= 10
+while ($i < 20) {
+  eval INSERT INTO t1 VALUES ($i, 0);
+  inc $i;
+}
+--let $file= query_get_value(SHOW MASTER STATUS, File, 1)
+--let $pos1= query_get_value(SHOW MASTER STATUS, Position, 1)
+--let $gtid1= `SELECT @@gtid_binlog_pos`
+while ($i < 30) {
+  eval INSERT INTO t1 VALUES ($i, 0);
+  inc $i;
+}
+--let $pos2= query_get_value(SHOW MASTER STATUS, Position, 1)
+--let $gtid2= `SELECT @@gtid_binlog_pos`
+while ($i < 40) {
+  eval INSERT INTO t1 VALUES ($i, 0);
+  inc $i;
+}
+--let $pos3= query_get_value(SHOW MASTER STATUS, Position, 1)
+--let $gtid3= `SELECT @@gtid_binlog_pos`
+INSERT INTO t1 VALUES (50, 0);
+INSERT INTO t1 VALUES (51, 0);
+
+--disable_ps2_protocol
+FLUSH NO_WRITE_TO_BINLOG STATUS;
+eval SELECT BINLOG_GTID_POS('$file', $pos1) = "$gtid1" AS Ok;
+eval SELECT BINLOG_GTID_POS('$file', $pos2) = "$gtid2" AS Ok;
+eval SELECT BINLOG_GTID_POS('$file', $pos3) = "$gtid3" AS Ok;
+--enable_query_log
+SHOW STATUS LIKE 'binlog_gtid_index_%';
+--enable_ps2_protocol
+
+--echo *** Crash the server, check that GTID index can be used after restart.
+--source include/crash_mysqld.inc
+
+--disable_ps2_protocol
+FLUSH NO_WRITE_TO_BINLOG STATUS;
+--disable_query_log
+eval SELECT BINLOG_GTID_POS('$file', $pos1) = "$gtid1" AS Ok;
+eval SELECT BINLOG_GTID_POS('$file', $pos2) = "$gtid2" AS Ok;
+eval SELECT BINLOG_GTID_POS('$file', $pos3) = "$gtid3" AS Ok;
+--enable_query_log
+SHOW STATUS LIKE 'binlog_gtid_index_%';
+--enable_ps2_protocol
+
+DROP TABLE t1;

mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted_gtid.result

@@ -6,6 +6,7 @@ connection server_2;
 include/stop_slave.inc
 CHANGE MASTER TO MASTER_USE_GTID=SLAVE_POS;
 call mtr.add_suppression(" Got fatal error 1236 from master when reading data from binary log: 'Could not set up decryption for binlog.'");
+call mtr.add_suppression(" Got fatal error 1236 from master when reading data from binary log: 'Could not decrypt binlog: encryption key error");
 #####################################################
 # Part 1: unencrypted master
 #####################################################
@@ -58,10 +59,11 @@ INSERT INTO table3_no_encryption SELECT NULL,NOW(),b FROM table3_no_encryption;
 connection server_2;
 start slave;
 include/wait_for_slave_io_error.inc [errno=1236]
-# Ensuring slave was unable to replicate any transactions..
+# Ensuring slave was unable to replicate any encrypted transactions..
 # ..success
 SHOW TABLES;
 Tables_in_test
+table1_no_encryption
 include/stop_slave_sql.inc
 reset slave;
 ##########
@@ -80,5 +82,7 @@ COUNT(*)
 4
 DROP TABLE table1_no_encryption, table2_to_encrypt, table3_no_encryption;
 connection server_2;
+RESET MASTER;
+SET GLOBAL gtid_slave_pos= '';
 include/start_slave.inc
 include/rpl_end.inc

mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted_gtid.test

@@ -36,6 +36,7 @@
 CHANGE MASTER TO MASTER_USE_GTID=SLAVE_POS;
 --enable_connect_log
 call mtr.add_suppression(" Got fatal error 1236 from master when reading data from binary log: 'Could not set up decryption for binlog.'");
+call mtr.add_suppression(" Got fatal error 1236 from master when reading data from binary log: 'Could not decrypt binlog: encryption key error");
 
 --echo #####################################################
 --echo # Part 1: unencrypted master
@@ -55,6 +56,7 @@ FLUSH BINARY LOGS;
 SET binlog_format=ROW;
 INSERT INTO table1_no_encryption SELECT NULL,NOW(),b FROM table1_no_encryption;
 INSERT INTO table1_no_encryption SELECT NULL,NOW(),b FROM table1_no_encryption;
+--let $last_unencrypted_gtid= `SELECT @@gtid_binlog_pos`
 
 # Make sure that binary logs are not encrypted
 
@@ -120,11 +122,11 @@ start slave;
 --let $slave_io_errno= 1236
 --source include/wait_for_slave_io_error.inc
 
---echo # Ensuring slave was unable to replicate any transactions..
+--echo # Ensuring slave was unable to replicate any encrypted transactions..
 --let $gsp= `SELECT @@global.gtid_slave_pos`
-if (`SELECT strcmp("$gsp","")`)
+if (`SELECT strcmp("$gsp","$last_unencrypted_gtid")`)
 {
-  die Slave without encryption configured should fail to read encrypted binlog;
+  die Slave without encryption configured should fail to read encrypted binlog (expected $last_unencrypted_gtid but got $gsp);
 }
 --echo # ..success
 
@@ -151,5 +153,7 @@ DROP TABLE table1_no_encryption, table2_to_encrypt, table3_no_encryption;
 
 --connection server_2
 --disable_connect_log
+RESET MASTER;
+SET GLOBAL gtid_slave_pos= '';
 --source include/start_slave.inc
 --source include/rpl_end.inc

mysql-test/suite/galera/r/galera_gtid_server_id.result

@@ -14,6 +14,9 @@ CREATE TABLE t1(id int not null primary key) engine=innodb;
 INSERT INTO t1 values (1);
 show global variables  like '%gtid%';
 Variable_name	Value
+binlog_gtid_index	ON
+binlog_gtid_index_page_size	4096
+binlog_gtid_index_span_min	65536
 gtid_binlog_pos	1-11-2
 gtid_binlog_state	1-11-2
 gtid_cleanup_batch_size	64
@@ -29,6 +32,9 @@ connection node_2;
 SET SESSION wsrep_sync_wait=15;
 show global variables  like '%gtid%';
 Variable_name	Value
+binlog_gtid_index	ON
+binlog_gtid_index_page_size	4096
+binlog_gtid_index_span_min	65536
 gtid_binlog_pos	0-12-1,1-11-2
 gtid_binlog_state	0-12-1,1-11-2
 gtid_cleanup_batch_size	64
@@ -55,6 +61,9 @@ Error	1231	Variable 'server_id' can't be set to the value of '200'
 INSERT INTO t1 values(2);
 show global variables  like '%gtid%';
 Variable_name	Value
+binlog_gtid_index	ON
+binlog_gtid_index_page_size	4096
+binlog_gtid_index_span_min	65536
 gtid_binlog_pos	0-12-1,1-11-3
 gtid_binlog_state	0-12-1,1-11-3
 gtid_cleanup_batch_size	64
@@ -69,6 +78,9 @@ wsrep_gtid_mode	ON
 connection node_1;
 show global variables  like '%gtid%';
 Variable_name	Value
+binlog_gtid_index	ON
+binlog_gtid_index_page_size	4096
+binlog_gtid_index_span_min	65536
 gtid_binlog_pos	1-11-3
 gtid_binlog_state	1-11-3
 gtid_cleanup_batch_size	64

mysql-test/suite/perfschema/r/dml_setup_instruments.result

@@ -8,12 +8,12 @@ wait/synch/mutex/sql/Ack_receiver::mutex	YES	YES
 wait/synch/mutex/sql/Cversion_lock	YES	YES
 wait/synch/mutex/sql/Delayed_insert::mutex	YES	YES
 wait/synch/mutex/sql/Event_scheduler::LOCK_scheduler_state	YES	YES
+wait/synch/mutex/sql/Gtid_index_writer::gtid_index_mutex	YES	YES
 wait/synch/mutex/sql/gtid_waiting::LOCK_gtid_waiting	YES	YES
 wait/synch/mutex/sql/hash_filo::lock	YES	YES
 wait/synch/mutex/sql/HA_DATA_PARTITION::LOCK_auto_inc	YES	YES
 wait/synch/mutex/sql/LOCK_active_mi	YES	YES
 wait/synch/mutex/sql/LOCK_after_binlog_sync	YES	YES
-wait/synch/mutex/sql/LOCK_audit_mask	YES	YES
 select * from performance_schema.setup_instruments
 where name like 'Wait/Synch/Rwlock/sql/%'
   and name not in (

mysql-test/suite/perfschema/r/relaylog.result

@@ -23,6 +23,7 @@ from performance_schema.file_summary_by_instance
 where file_name like "%master-%" order by file_name;
 FILE_NAME	EVENT_NAME	COUNT_READ	COUNT_WRITE	SUM_NUMBER_OF_BYTES_READ	SUM_NUMBER_OF_BYTES_WRITE
 master-bin.000001	wait/io/file/sql/binlog	MANY	MANY	MANY	MANY
+master-bin.000001.idx	wait/io/file/sql/gtid_index	NONE	MANY	NONE	MANY
 master-bin.index	wait/io/file/sql/binlog_index	MANY	MANY	MANY	MANY
 select * from performance_schema.file_summary_by_instance
 where file_name like "%slave-%" order by file_name;
@@ -112,6 +113,7 @@ where file_name like "%slave-%"
 order by file_name;
 FILE_NAME	EVENT_NAME	COUNT_READ	COUNT_WRITE	SUM_NUMBER_OF_BYTES_READ	SUM_NUMBER_OF_BYTES_WRITE
 slave-bin.000001	wait/io/file/sql/binlog	MANY	MANY	MANY	MANY
+slave-bin.000001.idx	wait/io/file/sql/gtid_index	NONE	MANY	NONE	MANY
 slave-bin.index	wait/io/file/sql/binlog_index	MANY	MANY	MANY	MANY
 slave-relay-bin.000001	wait/io/file/sql/relaylog	MANY	MANY	MANY	MANY
 slave-relay-bin.000002	wait/io/file/sql/relaylog	MANY	MANY	MANY	MANY

mysql-test/suite/rpl/include/rpl_gtid_index.inc

+# Include file for main test rpl.rpl_gtid_index.
+# Test GTID indexes with given parameters.
+#
+# Parameters:
+#   $NUM_POS             Number of GTIDs/binlog positions to create
+#   $NUM_DOMAIN          Number of different domains to use
+#   $NUM_SERVER          Number of different server_id to use
+#   $NUM_SLAVE_CONNECTS  How many GTID slave connect positions to test
+#   $RND_SEED            Random seed
+
+
+--echo *** Testing $NUM_POS GTIDs with $NUM_SLAVE_CONNECTS test connects
+
+--connection master
+DELETE FROM t1 WHERE a >= 1000;
+# Rotate binlogs to make new GTID index settings take effect.
+FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
+
+# Prepare some random values, but deterministic between test runs.
+CREATE TABLE rand_data(idx INT PRIMARY KEY, domain_id INT, server_id INT)
+  ENGINE=InnoDB;
+INSERT INTO rand_data(idx, domain_id, server_id) VALUES (0, 0, 1);
+eval
+INSERT INTO rand_data(idx, domain_id, server_id)
+SELECT seq,
+       @tmp:=floor($NUM_DOMAIN*POW(rand($RND_SEED),2)),
+       100 + $NUM_SERVER*@tmp + floor($NUM_SERVER*rand($RND_SEED))
+  FROM seq_1_to_$NUM_POS;
+# Let's check that the test data is deterministic.
+# If this changes due to some server changes, it's fine, the .result can just
+# be updated. But we want it to be identical between test runs on same code,
+# to facilitate debugging test failures.
+SELECT COUNT(*), SUM(domain_id), SUM(server_id) FROM rand_data;
+
+# Create some data for the binlog (and GTID index), recording the correct
+# binlog positions and GTIDs.
+CREATE TABLE gtid_data(
+  idx INT PRIMARY KEY,
+  gtid VARCHAR(44),
+  gtid_pos VARCHAR(255),
+  file VARCHAR(100),
+  pos INT,
+  row_count INT,
+  KEY(file, pos)) ENGINE=InnoDB;
+--let $gtid= `SELECT @@last_gtid`
+
+--source include/save_master_gtid.inc
+
+--connection slave
+--source include/sync_with_master_gtid.inc
+--source include/stop_slave.inc
+
+--connection master
+SET @orig_domain_id= @@gtid_domain_id;
+SET @orig_server_id= @@server_id;
+--let $i= 0
+--let $rotate_point= `SELECT floor($NUM_POS/2)`
+--let $base_count= `SELECT COUNT(*) FROM t1`
+--disable_query_log
+while ($i < $NUM_POS) {
+  --let $file= query_get_value(SHOW MASTER STATUS, File, 1)
+  --let $pos= query_get_value(SHOW MASTER STATUS, Position, 1)
+  --let $gtid_pos= `SELECT @@gtid_binlog_pos`
+  --let $row_count= `SELECT $base_count + $i`
+  eval SET gtid_domain_id= (SELECT domain_id FROM rand_data WHERE idx=$i+1);
+  eval SET server_id= (SELECT server_id FROM rand_data WHERE idx=$i+1);
+  BEGIN;
+  eval INSERT INTO gtid_data(idx, gtid, gtid_pos, file, pos, row_count)
+       VALUES ($i, '$gtid', '$gtid_pos', '$file', $pos, $row_count);
+  eval INSERT INTO t1 VALUES ($i + 1000, 0);
+  COMMIT;
+--let $gtid= `SELECT @@last_gtid`
+  inc $i;
+  if ($i==$rotate_point) {
+    FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
+  }
+}
+--enable_query_log
+SET gtid_domain_id= @orig_domain_id;
+SET server_id= @orig_server_id;
+
+SELECT COUNT(*) FROM gtid_data;
+
+# Test that BINLOG_GTID_POS returns correct positions for every GTID position.
+--echo *** The result should be empty, otherwise some result is wrong:
+SELECT idx, gtid_pos, BINLOG_GTID_POS(file, pos)
+  FROM gtid_data
+ WHERE NOT gtid_eq(CONVERT(gtid_pos USING utf8),BINLOG_GTID_POS(file, pos)) 
+ ORDER BY idx;
+
+# Prepare to rewind the slave to this point to test again on same binlog.
+--connection slave
+SET @orig_pos= @@GLOBAL.gtid_slave_pos;
+SET @orig_t1_limit= (SELECT MAX(a) FROM t1);
+
+--echo *** Now connect the slave to each position in turn, and test that
+--echo *** the right amount of data is replicated at each point.
+--let $old_silent= $keep_include_silent
+--let $keep_include_silent= 1
+--let $i= 0
+--disable_query_log
+while ($i < $NUM_POS) {
+  --connection master
+  --let $gtid_pos= `SELECT gtid_pos FROM gtid_data WHERE idx=$i`
+  --let $master_count= `SELECT row_count FROM gtid_data WHERE idx=$i`
+  --connection slave
+  --disable_result_log
+  eval START SLAVE UNTIL master_gtid_pos='$gtid_pos';
+  --enable_result_log
+  --let $res= `SELECT MASTER_GTID_WAIT('$gtid_pos')`
+  if ($res != 0) {
+    --die "FAIL: MASTER_GTID_WAIT($gtid_pos) returned $res, should have been 0"
+  }
+  --source include/wait_for_slave_to_stop.inc
+  --let $slave_count = `SELECT COUNT(*) FROM t1`
+  if ($master_count != $slave_count) {
+    SELECT * FROM gtid_data ORDER BY file, pos;
+    SELECT * FROM t1 ORDER BY a;
+    --die "Not all rows replicated. $master_count on master but $slave_count on slave."
+  }
+  --let $i= `SELECT $i + ceil($NUM_POS / $NUM_SLAVE_CONNECTS)`
+}
+--enable_query_log
+
+--echo *** Test slave connecting to some GTID positions where the position in
+--echo *** the master's binlog is different between the different domains.
+--echo *** Revind the slave and test on the same binlog data from the master as before.
+--connection slave
+SET sql_log_bin= 0;
+TRUNCATE gtid_data;
+DELETE FROM t1 WHERE a > @orig_t1_limit;
+SET sql_log_bin= 1;
+SET GLOBAL gtid_slave_pos= @orig_pos;
+
+--let $i= 0
+--disable_query_log
+while ($i <= $NUM_DOMAIN) {
+  # Build a GTID position from GTIDs that are picked at different locations
+  # in the gtid_data table for each domain.
+  --connection master
+  let $until_pos=`
+    SELECT GROUP_CONCAT(gtid SEPARATOR ',')
+      FROM gtid_data
+     WHERE idx IN (
+       SELECT MAX(gtid_data.idx) AS pick
+	 FROM gtid_data
+	 INNER JOIN rand_data ON (rand_data.idx = gtid_data.idx)
+	WHERE gtid_data.idx*$NUM_DOMAIN <= (domain_id + $i)*$NUM_POS
+	GROUP BY domain_id
+      )`;
+  --connection slave
+  --disable_result_log
+  eval START SLAVE UNTIL master_gtid_pos='$until_pos';
+  --enable_result_log
+  --let $res= `SELECT MASTER_GTID_WAIT('$until_pos')`
+  if ($res != 0) {
+    --die "FAIL: MASTER_GTID_WAIT($until_pos) returned $res, should have been 0"
+  }
+  --source include/wait_for_slave_to_stop.inc
+
+  inc $i;
+}
+--enable_query_log
+--let $keep_include_silent= $old_silent
+
+# Check that everything was replicated (nothing skipped).
+# We have one less row on the slave since the last UNTIL is the one before
+# the master inserted the last row.
+--connection master
+--let $master_count= `SELECT COUNT(*)-1 FROM t1`
+--connection slave
+--let $slave_count= `SELECT COUNT(*) FROM t1`
+if ($master_count != $slave_count) {
+  SELECT * FROM gtid_data ORDER BY file, pos;
+  SELECT * FROM t1 ORDER BY a;
+  --die "Not all rows replicated. $master_count on master but $slave_count on slave."
+}
+
+--connection master
+DROP TABLE gtid_data, rand_data;
+--source include/save_master_gtid.inc
+
+--connection slave
+--source include/start_slave.inc
+--source include/sync_with_master_gtid.inc
+
+--connection master

mysql-test/suite/rpl/r/rpl_gtid_glle_no_terminate.result

@@ -28,7 +28,6 @@ include/show_events.inc
 Log_name	Pos	Event_type	Server_id	End_log_pos	Info
 slave-relay-bin.000002	#	Rotate	#	#	master-bin.000001;pos=POS
 slave-relay-bin.000002	#	Format_desc	#	#	SERVER_VERSION, BINLOG_VERSION
-slave-relay-bin.000002	#	Gtid_list	#	#	[]
 slave-relay-bin.000002	#	Binlog_checkpoint	#	#	master-bin.000001
 slave-relay-bin.000002	#	Gtid	#	#	GTID #-#-#
 slave-relay-bin.000002	#	Gtid_list	#	#	[#-#-#]

mysql-test/suite/rpl/t/rpl_gtid_glle_no_terminate.test

@@ -24,6 +24,7 @@ CHANGE MASTER TO MASTER_USE_GTID=slave_pos;
 --echo #
 --echo # Initialize test data
 --connection master
+--source include/wait_for_binlog_checkpoint.inc
 create table t1 (a int);
 SET @@session.server_id= 3;
 create table t2 (a int);

mysql-test/suite/rpl/t/rpl_gtid_index.test

+--source include/have_sequence.inc
+--source include/have_innodb.inc
+--source include/master-slave.inc
+--source include/have_binlog_format_mixed.inc
+
+--connection slave
+--source include/stop_slave.inc
+CHANGE MASTER TO master_use_gtid= slave_pos;
+--source include/start_slave.inc
+
+--connection master
+CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (0, 0);
+
+
+--echo *** Test looking up a lot of different event positions and GTIDs.
+
+# A function for comparing GTID positions.
+# Handles that the domain_id order is different in the two strings.
+# Works by repeatedly removing one GTID from each string. If the strings have
+# the same length and nothing is left at the end, then they are identical.
+delimiter //;
+CREATE FUNCTION gtid_eq(a VARCHAR(255), b VARCHAR(255)) RETURNS BOOLEAN DETERMINISTIC
+BEGIN
+  DECLARE g VARCHAR(255);
+  IF a IS NULL OR b IS NULL OR LENGTH(a) != LENGTH(b) THEN
+    RETURN FALSE;
+  END IF;
+  SET a= CONCAT(a, ',');
+  SET b= CONCAT(',', b, ',');
+  WHILE LENGTH(a) > 0 DO
+    SET g= REGEXP_SUBSTR(a, '^[^,]+,');
+    SET a= SUBSTRING(a, LENGTH(g)+1);
+    SET b= REPLACE(b, CONCAT(',', g), ',');
+  END WHILE;
+  RETURN b = ',';
+END //
+delimiter ;//
+
+SET @old_page_size= @@GLOBAL.binlog_gtid_index_page_size;
+SET @old_span_min= @@GLOBAL.binlog_gtid_index_span_min;
+
+--echo *** A fair amount of work with default GTID index settings.
+--let $NUM_POS= 200
+--let $NUM_DOMAIN= 5
+--let $NUM_SERVER= 5
+--let $NUM_SLAVE_CONNECTS= 50
+--let $RND_SEED= 42
+--source suite/rpl/include/rpl_gtid_index.inc
+
+--echo *** A lot of GTIDs with small btree pages to stress the Btree code.
+--let $NUM_POS= 1000
+--let $NUM_DOMAIN= 10
+--let $RND_SEED= 150
+SET GLOBAL binlog_gtid_index_page_size= 64;
+SET GLOBAL binlog_gtid_index_span_min= 1;
+--source suite/rpl/include/rpl_gtid_index.inc
+
+--echo *** Small page size with sparse index.
+--let $NUM_POS= 200
+--let $RND_SEED= 666
+SET GLOBAL binlog_gtid_index_page_size= 64;
+SET GLOBAL binlog_gtid_index_span_min= 2048;
+--source suite/rpl/include/rpl_gtid_index.inc
+
+--echo *** Medium page size.
+--let $NUM_POS= 200
+--let $RND_SEED= 1024
+SET GLOBAL binlog_gtid_index_page_size= 512;
+SET GLOBAL binlog_gtid_index_span_min= 512;
+--source suite/rpl/include/rpl_gtid_index.inc
+
+--echo *** Large page size.
+--let $NUM_POS= 200
+--let $RND_SEED= 12345
+SET GLOBAL binlog_gtid_index_page_size= 16384;
+SET GLOBAL binlog_gtid_index_span_min= 1;
+--source suite/rpl/include/rpl_gtid_index.inc
+
+
+# Cleanup.
+--connection master
+SET GLOBAL binlog_gtid_index_page_size= @old_page_size;
+SET GLOBAL binlog_gtid_index_span_min= @old_span_min;
+
+DROP TABLE t1;
+DROP FUNCTION gtid_eq;
+
+--source include/rpl_end.inc

mysql-test/suite/sys_vars/r/sysvars_server_embedded.result

@@ -432,6 +432,36 @@ NUMERIC_BLOCK_SIZE	NULL
 ENUM_VALUE_LIST	MIXED,STATEMENT,ROW
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
+VARIABLE_NAME	BINLOG_GTID_INDEX
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	BOOLEAN
+VARIABLE_COMMENT	Enable the creation of a GTID index for every binlog file, and the use of such index for speeding up GTID lookup in the binlog.
+NUMERIC_MIN_VALUE	NULL
+NUMERIC_MAX_VALUE	NULL
+NUMERIC_BLOCK_SIZE	NULL
+ENUM_VALUE_LIST	OFF,ON
+READ_ONLY	NO
+COMMAND_LINE_ARGUMENT	OPTIONAL
+VARIABLE_NAME	BINLOG_GTID_INDEX_PAGE_SIZE
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	INT UNSIGNED
+VARIABLE_COMMENT	Page size to use for the binlog GTID index.
+NUMERIC_MIN_VALUE	64
+NUMERIC_MAX_VALUE	16777216
+NUMERIC_BLOCK_SIZE	1
+ENUM_VALUE_LIST	NULL
+READ_ONLY	NO
+COMMAND_LINE_ARGUMENT	REQUIRED
+VARIABLE_NAME	BINLOG_GTID_INDEX_SPAN_MIN
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	INT UNSIGNED
+VARIABLE_COMMENT	Control sparseness of the binlog GTID index. If set to N, at most one index record will be added for every N bytes of binlog file written, to reduce the size of the index. Normally does not need tuning.
+NUMERIC_MIN_VALUE	1
+NUMERIC_MAX_VALUE	1073741824
+NUMERIC_BLOCK_SIZE	1
+ENUM_VALUE_LIST	NULL
+READ_ONLY	NO
+COMMAND_LINE_ARGUMENT	REQUIRED
 VARIABLE_NAME	BINLOG_OPTIMIZE_THREAD_SCHEDULING
 VARIABLE_SCOPE	GLOBAL
 VARIABLE_TYPE	BOOLEAN

mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result

@@ -452,6 +452,36 @@ NUMERIC_BLOCK_SIZE	NULL
 ENUM_VALUE_LIST	MIXED,STATEMENT,ROW
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
+VARIABLE_NAME	BINLOG_GTID_INDEX
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	BOOLEAN
+VARIABLE_COMMENT	Enable the creation of a GTID index for every binlog file, and the use of such index for speeding up GTID lookup in the binlog.
+NUMERIC_MIN_VALUE	NULL
+NUMERIC_MAX_VALUE	NULL
+NUMERIC_BLOCK_SIZE	NULL
+ENUM_VALUE_LIST	OFF,ON
+READ_ONLY	NO
+COMMAND_LINE_ARGUMENT	OPTIONAL
+VARIABLE_NAME	BINLOG_GTID_INDEX_PAGE_SIZE
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	INT UNSIGNED
+VARIABLE_COMMENT	Page size to use for the binlog GTID index.
+NUMERIC_MIN_VALUE	64
+NUMERIC_MAX_VALUE	16777216
+NUMERIC_BLOCK_SIZE	1
+ENUM_VALUE_LIST	NULL
+READ_ONLY	NO
+COMMAND_LINE_ARGUMENT	REQUIRED
+VARIABLE_NAME	BINLOG_GTID_INDEX_SPAN_MIN
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	INT UNSIGNED
+VARIABLE_COMMENT	Control sparseness of the binlog GTID index. If set to N, at most one index record will be added for every N bytes of binlog file written, to reduce the size of the index. Normally does not need tuning.
+NUMERIC_MIN_VALUE	1
+NUMERIC_MAX_VALUE	1073741824
+NUMERIC_BLOCK_SIZE	1
+ENUM_VALUE_LIST	NULL
+READ_ONLY	NO
+COMMAND_LINE_ARGUMENT	REQUIRED
 VARIABLE_NAME	BINLOG_IGNORE_DB
 VARIABLE_SCOPE	GLOBAL
 VARIABLE_TYPE	VARCHAR

sql/CMakeLists.txt

@@ -165,7 +165,7 @@ SET (SQL_SOURCE
                gcalc_slicescan.cc gcalc_tools.cc
                my_apc.cc mf_iocache_encr.cc item_jsonfunc.cc
                my_json_writer.cc json_schema.cc json_schema_helper.cc
-               rpl_gtid.cc rpl_parallel.cc
+               rpl_gtid.cc gtid_index.cc rpl_parallel.cc
                semisync.cc semisync_master.cc semisync_slave.cc
                semisync_master_ack_receiver.cc
                sp_instr.cc

sql/log.h

@@ -21,8 +21,10 @@
 #include "rpl_constants.h"
 
 class Relay_log_info;
+class Gtid_index_writer;
 
 class Format_description_log_event;
+class Gtid_log_event;
 
 bool reopen_fstreams(const char *filename, FILE *outstream, FILE *errstream);
 void setup_log_handling();
@@ -240,6 +242,7 @@ extern TC_LOG_DUMMY tc_log_dummy;
 #define LOG_CLOSE_TO_BE_OPENED	2
 #define LOG_CLOSE_STOP_EVENT	4
 #define LOG_CLOSE_DELAYED_CLOSE 8
+#define LOG_CLOSE_SYNC_GTID_INDEX 16
 
 /* 
   Maximum unique log filename extension.
@@ -711,6 +714,9 @@ class MYSQL_BIN_LOG: public TC_LOG, private Event_log
   ulonglong group_commit_trigger_count, group_commit_trigger_timeout;
   ulonglong group_commit_trigger_lock_wait;
 
+  /* Binlog GTID index. */
+  Gtid_index_writer *gtid_index;
+
   /* pointer to the sync period variable, for binlog this will be
      sync_binlog_period, for relay log this will be
      sync_relay_log_period
@@ -720,6 +726,13 @@ class MYSQL_BIN_LOG: public TC_LOG, private Event_log
   bool state_file_deleted;
   bool binlog_state_recover_done;
 
+  Gtid_index_writer *recover_gtid_index_start(const char *base_name,
+                                              my_off_t offset);
+  void recover_gtid_index_process(Gtid_index_writer *gi, my_off_t offset,
+                                  const rpl_gtid *gtid);
+  void recover_gtid_index_end(Gtid_index_writer *gi);
+  void recover_gtid_index_abort(Gtid_index_writer *gi);
+
   inline uint get_sync_period()
   {
     return *sync_period_ptr;
@@ -739,6 +752,8 @@ class MYSQL_BIN_LOG: public TC_LOG, private Event_log
   bool write_transaction_to_binlog_events(group_commit_entry *entry);
   void trx_group_commit_leader(group_commit_entry *leader);
   bool is_xidlist_idle_nolock();
+  void update_gtid_index(uint32 offset, rpl_gtid gtid);
+
 public:
   int new_file_without_locking();
   /*
@@ -759,11 +774,8 @@ class MYSQL_BIN_LOG: public TC_LOG, private Event_log
     ulong binlog_id;
     /* Total prepared XIDs and pending checkpoint requests in this binlog. */
     long xid_count;
-    long notify_count;
-    /* For linking in requests to the binlog background thread. */
-    xid_count_per_binlog *next_in_queue;
     xid_count_per_binlog(char *log_file_name, uint log_file_name_len)
-      :binlog_id(0), xid_count(0), notify_count(0)
+      :binlog_id(0), xid_count(0)
     {
       binlog_name_len= log_file_name_len;
       binlog_name= (char *) my_malloc(PSI_INSTRUMENT_ME, binlog_name_len, MYF(MY_ZEROFILL));

sql/mysqld.cc

@@ -83,6 +83,7 @@
 #include "wsrep_server_state.h"
 #endif /* WITH_WSREP */
 #include "proxy_protocol.h"
+#include "gtid_index.h"
 
 #include "sql_callback.h"
 #include "threadpool.h"
@@ -443,6 +444,9 @@ my_bool sp_automatic_privileges= 1;
 
 ulong opt_binlog_rows_event_max_size;
 ulong binlog_row_metadata;
+my_bool opt_binlog_gtid_index= TRUE;
+uint opt_binlog_gtid_index_page_size= 4096;
+uint opt_binlog_gtid_index_span_min= 65536;
 my_bool opt_master_verify_checksum= 0;
 my_bool opt_slave_sql_verify_checksum= 1;
 const char *binlog_format_names[]= {"MIXED", "STATEMENT", "ROW", NullS};
@@ -491,6 +495,7 @@ ulong malloc_calls;
 ulong specialflag=0;
 ulong binlog_cache_use= 0, binlog_cache_disk_use= 0;
 ulong binlog_stmt_cache_use= 0, binlog_stmt_cache_disk_use= 0;
+ulong binlog_gtid_index_hit= 0, binlog_gtid_index_miss= 0;
 ulong max_connections, max_connect_errors;
 uint max_password_errors;
 ulong extra_max_connections;
@@ -896,7 +901,7 @@ PSI_file_key key_file_binlog,  key_file_binlog_cache, key_file_binlog_index,
 PSI_file_key key_file_query_log, key_file_slow_log;
 PSI_file_key key_file_relaylog, key_file_relaylog_index,
              key_file_relaylog_cache, key_file_relaylog_index_cache;
-PSI_file_key key_file_binlog_state;
+PSI_file_key key_file_binlog_state, key_file_gtid_index;
 
 #ifdef HAVE_PSI_INTERFACE
 #ifdef HAVE_MMAP
@@ -921,6 +926,7 @@ PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_xid_list,
   key_LOCK_status, key_LOCK_temp_pool,
   key_LOCK_system_variables_hash, key_LOCK_thd_data, key_LOCK_thd_kill,
   key_LOCK_user_conn, key_LOCK_uuid_short_generator, key_LOG_LOCK_log,
+  key_gtid_index_lock,
   key_master_info_data_lock, key_master_info_run_lock,
   key_master_info_sleep_lock, key_master_info_start_stop_lock,
   key_master_info_start_alter_lock,
@@ -1007,6 +1013,7 @@ static PSI_mutex_info all_server_mutexes[]=
   { &key_LOCK_user_conn, "LOCK_user_conn", PSI_FLAG_GLOBAL},
   { &key_LOCK_uuid_short_generator, "LOCK_uuid_short_generator", PSI_FLAG_GLOBAL},
   { &key_LOG_LOCK_log, "LOG::LOCK_log", 0},
+  { &key_gtid_index_lock, "Gtid_index_writer::gtid_index_mutex", 0},
   { &key_master_info_data_lock, "Master_info::data_lock", 0},
   { &key_master_info_start_stop_lock, "Master_info::start_stop_lock", 0},
   { &key_master_info_run_lock, "Master_info::run_lock", 0},
@@ -2011,6 +2018,7 @@ static void clean_up(bool print_message)
 
   injector::free_instance();
   mysql_bin_log.cleanup();
+  Gtid_index_writer::gtid_index_cleanup();
 
   my_tz_free();
   my_dboptions_cache_free();
@@ -3962,6 +3970,7 @@ static int init_common_variables()
     inited before MY_INIT(). So we do it here.
   */
   mysql_bin_log.init_pthread_objects();
+  Gtid_index_writer::gtid_index_init();
 
   /* TODO: remove this when my_time_t is 64 bit compatible */
   if (!IS_TIME_T_VALID_FOR_TIMESTAMP(server_start_time))
@@ -7396,6 +7405,8 @@ SHOW_VAR status_vars[]= {
   {"Binlog_bytes_written",     (char*) offsetof(STATUS_VAR, binlog_bytes_written), SHOW_LONGLONG_STATUS},
   {"Binlog_cache_disk_use",    (char*) &binlog_cache_disk_use,  SHOW_LONG},
   {"Binlog_cache_use",         (char*) &binlog_cache_use,       SHOW_LONG},
+  {"Binlog_gtid_index_hit",    (char*) &binlog_gtid_index_hit, SHOW_LONG},
+  {"Binlog_gtid_index_miss",   (char*) &binlog_gtid_index_miss, SHOW_LONG},
   {"Binlog_stmt_cache_disk_use",(char*) &binlog_stmt_cache_disk_use,  SHOW_LONG},
   {"Binlog_stmt_cache_use",    (char*) &binlog_stmt_cache_use,       SHOW_LONG},
   {"Busy_time",                (char*) offsetof(STATUS_VAR, busy_time), SHOW_DOUBLE_STATUS},
@@ -7821,6 +7832,7 @@ static int mysql_init_variables(void)
   delayed_insert_errors= thread_created= 0;
   specialflag= 0;
   binlog_cache_use=  binlog_cache_disk_use= 0;
+  binlog_gtid_index_hit= binlog_gtid_index_miss= 0;
   max_used_connections= slow_launch_threads = 0;
   max_used_connections_time= 0;
   mysqld_user= mysqld_chroot= opt_init_file= opt_bin_logname = 0;
@@ -9219,7 +9231,8 @@ static PSI_file_info all_server_files[]=
   { &key_file_trg, "trigger_name", 0},
   { &key_file_trn, "trigger", 0},
   { &key_file_init, "init", 0},
-  { &key_file_binlog_state, "binlog_state", 0}
+  { &key_file_binlog_state, "binlog_state", 0},
+  { &key_file_gtid_index, "gtid_index", 0}
 };
 #endif /* HAVE_PSI_INTERFACE */
 
@@ -9413,6 +9426,7 @@ PSI_memory_key key_memory_acl_cache;
 PSI_memory_key key_memory_acl_mem;
 PSI_memory_key key_memory_acl_memex;
 PSI_memory_key key_memory_binlog_cache_mngr;
+PSI_memory_key key_memory_binlog_gtid_index;
 PSI_memory_key key_memory_binlog_pos;
 PSI_memory_key key_memory_binlog_recover_exec;
 PSI_memory_key key_memory_binlog_statement_buffer;
@@ -9652,6 +9666,7 @@ static PSI_memory_info all_server_memory[]=
 //  { &key_memory_Slave_job_group_group_relay_log_name, "Slave_job_group::group_relay_log_name", 0},
   { &key_memory_Relay_log_info_group_relay_log_name, "Relay_log_info::group_relay_log_name", 0},
   { &key_memory_binlog_cache_mngr, "binlog_cache_mngr", 0},
+  { &key_memory_binlog_gtid_index, "binlog_gtid_index", 0},
   { &key_memory_Row_data_memory_memory, "Row_data_memory::memory", 0},
 //  { &key_memory_Gtid_set_to_string, "Gtid_set::to_string", 0},
 //  { &key_memory_Gtid_state_to_string, "Gtid_state::to_string", 0},

sql/mysqld.h

@@ -217,6 +217,7 @@ extern ulonglong thd_startup_options;
 extern my_thread_id global_thread_id;
 extern ulong binlog_cache_use, binlog_cache_disk_use;
 extern ulong binlog_stmt_cache_use, binlog_stmt_cache_disk_use;
+extern ulong binlog_gtid_index_hit, binlog_gtid_index_miss;
 extern ulong aborted_threads, aborted_connects, aborted_connects_preauth;
 extern ulong delayed_insert_timeout;
 extern ulong delayed_insert_limit, delayed_queue_size;
@@ -249,6 +250,9 @@ extern ulonglong slave_max_statement_time;
 extern double slave_max_statement_time_double;
 extern ulong opt_binlog_rows_event_max_size;
 extern ulong binlog_row_metadata;
+extern my_bool opt_binlog_gtid_index;
+extern uint opt_binlog_gtid_index_page_size;
+extern uint opt_binlog_gtid_index_span_min;
 extern ulong thread_cache_size;
 extern ulong stored_program_cache_size;
 extern ulong opt_slave_parallel_threads;
@@ -333,7 +337,7 @@ extern PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_xid_list,
   key_LOCK_rpl_status, key_LOCK_server_started,
   key_LOCK_status, key_LOCK_optimizer_costs,
   key_LOCK_thd_data, key_LOCK_thd_kill,
-  key_LOCK_user_conn, key_LOG_LOCK_log,
+  key_LOCK_user_conn, key_LOG_LOCK_log, key_gtid_index_lock,
   key_master_info_data_lock, key_master_info_run_lock,
   key_master_info_sleep_lock, key_master_info_start_stop_lock,
   key_master_info_start_alter_lock,
@@ -411,7 +415,7 @@ extern PSI_file_key key_file_relaylog, key_file_relaylog_index,
                     key_file_relaylog_cache, key_file_relaylog_index_cache;
 extern PSI_socket_key key_socket_tcpip, key_socket_unix,
   key_socket_client_connection;
-extern PSI_file_key key_file_binlog_state;
+extern PSI_file_key key_file_binlog_state, key_file_gtid_index;
 
 #ifdef HAVE_PSI_INTERFACE
 void init_server_psi_keys();
@@ -456,6 +460,7 @@ extern PSI_memory_key key_memory_user_var_entry_value;
 extern PSI_memory_key key_memory_Slave_job_group_group_relay_log_name;
 extern PSI_memory_key key_memory_Relay_log_info_group_relay_log_name;
 extern PSI_memory_key key_memory_binlog_cache_mngr;
+extern PSI_memory_key key_memory_binlog_gtid_index;
 extern PSI_memory_key key_memory_Row_data_memory_memory;
 extern PSI_memory_key key_memory_errmsgs;
 extern PSI_memory_key key_memory_Event_queue_element_for_exec_names;

sql/privilege.h

@@ -376,6 +376,15 @@ constexpr privilege_t PRIV_SET_SYSTEM_GLOBAL_VAR_BINLOG_ROW_METADATA=
 constexpr privilege_t PRIV_SET_SYSTEM_GLOBAL_VAR_BINLOG_LEGACY_EVENT_POS=
   SUPER_ACL | BINLOG_ADMIN_ACL;
 
+constexpr privilege_t PRIV_SET_SYSTEM_GLOBAL_VAR_BINLOG_GTID_INDEX=
+  BINLOG_ADMIN_ACL;
+
+constexpr privilege_t PRIV_SET_SYSTEM_GLOBAL_VAR_BINLOG_GTID_INDEX_PAGE_SIZE=
+  BINLOG_ADMIN_ACL;
+
+constexpr privilege_t PRIV_SET_SYSTEM_GLOBAL_VAR_BINLOG_GTID_INDEX_SPAN_MIN=
+  BINLOG_ADMIN_ACL;
+
 constexpr privilege_t PRIV_SET_SYSTEM_GLOBAL_VAR_EXPIRE_LOGS_DAYS=
   BINLOG_ADMIN_ACL;
 

sql/rpl_gtid.h

@@ -26,6 +26,11 @@
 extern const LEX_CSTRING rpl_gtid_slave_state_table_name;
 
 class String;
+#ifdef MYSQL_SERVER
+struct TABLE;
+#endif
+struct slave_connection_state;
+
 #define PARAM_GTID(G) G.domain_id, G.server_id, G.seq_no
 
 #define GTID_MAX_STR_LENGTH (10+1+10+1+20)
@@ -296,8 +301,13 @@ struct rpl_slave_state
   to know where to start when a master is changed to a slave. As a side
   effect, it also allows to skip a hash lookup in the very common case of
   logging a new GTID with same server id as last GTID.
+
+  The base class rpl_binlog_state_base contains just be basic data operations
+  to insert/update GTIDs, and is used eg. from Gtid_index_*. The main class
+  rpl_binlog_state builds server logic on top of that like mutex locking,
+  gtid_strict_mode handling, etc.
 */
-struct rpl_binlog_state
+struct rpl_binlog_state_base
 {
   struct element {
     uint32 domain_id;
@@ -309,29 +319,45 @@ struct rpl_binlog_state
 
     int update_element(const rpl_gtid *gtid);
   };
+
   /* Mapping from domain_id to collection of elements. */
   HASH hash;
+  my_bool initialized;
+
+  rpl_binlog_state_base() : initialized(0) {}
+  ~rpl_binlog_state_base();
+  void init();
+  void reset_nolock();
+  void free();
+  bool load_nolock(struct rpl_gtid *list, uint32 count);
+  bool load_nolock(rpl_binlog_state_base *orig_state);
+  int update_nolock(const struct rpl_gtid *gtid);
+  int alloc_element_nolock(const rpl_gtid *gtid);
+  uint32 count_nolock();
+  int get_gtid_list_nolock(rpl_gtid *gtid_list, uint32 list_size);
+  rpl_gtid *find_nolock(uint32 domain_id, uint32 server_id);
+  bool is_before_pos(slave_connection_state *pos);
+};
+
+struct rpl_binlog_state : public rpl_binlog_state_base
+{
   /* Mutex protecting access to the state. */
   mysql_mutex_t LOCK_binlog_state;
-  my_bool initialized;
 
   /* Auxiliary buffer to sort gtid list. */
   DYNAMIC_ARRAY gtid_sort_array;
 
-   rpl_binlog_state() :initialized(0) {}
+  rpl_binlog_state() {}
   ~rpl_binlog_state();
 
   void init();
-  void reset_nolock();
   void reset();
   void free();
   bool load(struct rpl_gtid *list, uint32 count);
   bool load(rpl_slave_state *slave_pos);
-  int update_nolock(const struct rpl_gtid *gtid, bool strict);
   int update(const struct rpl_gtid *gtid, bool strict);
   int update_with_next_gtid(uint32 domain_id, uint32 server_id,
                              rpl_gtid *gtid);
-  int alloc_element_nolock(const rpl_gtid *gtid);
   bool check_strict_sequence(uint32 domain_id, uint32 server_id, uint64 seq_no,
                              bool no_error= false);
   int bump_seq_no_if_needed(uint32 domain_id, uint64 seq_no);
@@ -342,7 +368,6 @@ struct rpl_binlog_state
   int get_most_recent_gtid_list(rpl_gtid **list, uint32 *size);
   bool append_pos(String *str);
   bool append_state(String *str);
-  rpl_gtid *find_nolock(uint32 domain_id, uint32 server_id);
   rpl_gtid *find(uint32 domain_id, uint32 server_id);
   rpl_gtid *find_most_recent(uint32 domain_id);
   const char* drop_domain(DYNAMIC_ARRAY *ids, Gtid_list_log_event *glev, char*);

sql/rpl_rli.cc

@@ -1547,7 +1547,7 @@ Relay_log_info::update_relay_log_state(rpl_gtid *gtid_list, uint32 count)
   int res= 0;
   while (count)
   {
-    if (relay_log_state.update_nolock(gtid_list, false))
+    if (relay_log_state.update_nolock(gtid_list))
       res= 1;
     ++gtid_list;
     --count;

sql/sys_vars.cc

@@ -6844,6 +6844,36 @@ Sys_binlog_row_metadata(
        ON_UPDATE(NULL));
 
 
+static Sys_var_on_access_global<Sys_var_mybool,
+                           PRIV_SET_SYSTEM_GLOBAL_VAR_BINLOG_GTID_INDEX>
+Sys_binlog_gtid_index(
+       "binlog_gtid_index",
+       "Enable the creation of a GTID index for every binlog file, and the use "
+       "of such index for speeding up GTID lookup in the binlog.",
+       GLOBAL_VAR(opt_binlog_gtid_index), CMD_LINE(OPT_ARG),
+       DEFAULT(TRUE));
+
+
+static Sys_var_on_access_global<Sys_var_uint,
+                        PRIV_SET_SYSTEM_GLOBAL_VAR_BINLOG_GTID_INDEX_PAGE_SIZE>
+Sys_binlog_gtid_index_page_size(
+       "binlog_gtid_index_page_size",
+       "Page size to use for the binlog GTID index.",
+       GLOBAL_VAR(opt_binlog_gtid_index_page_size), CMD_LINE(REQUIRED_ARG),
+       VALID_RANGE(64, 1<<24), DEFAULT(4096), BLOCK_SIZE(1));
+
+
+static Sys_var_on_access_global<Sys_var_uint,
+                        PRIV_SET_SYSTEM_GLOBAL_VAR_BINLOG_GTID_INDEX_SPAN_MIN>
+Sys_binlog_gtid_index_span_min(
+       "binlog_gtid_index_span_min",
+       "Control sparseness of the binlog GTID index. If set to N, at most one "
+       "index record will be added for every N bytes of binlog file written, "
+       "to reduce the size of the index. Normally does not need tuning.",
+       GLOBAL_VAR(opt_binlog_gtid_index_span_min), CMD_LINE(REQUIRED_ARG),
+       VALID_RANGE(1, 1024*1024L*1024L), DEFAULT(65536), BLOCK_SIZE(1));
+
+
 static bool check_pseudo_slave_mode(sys_var *self, THD *thd, set_var *var)
 {
   longlong previous_val= thd->variables.pseudo_slave_mode;