MDEV-12972 Random and Frequent Segfault

Problem was a memory overflow in MRR

Reviewed by Sergei Petruna. Testcase by Elena

sql/multi_range_read.cc

@@ -1231,28 +1231,18 @@ bool DsMrr_impl::setup_buffer_sharing(uint key_size_in_keybuf,
   
   ptrdiff_t bytes_for_keys= (full_buf_end - full_buf) - bytes_for_rowids;
 
-  if (bytes_for_keys < key_buff_elem_size + 1)
-  {
-    ptrdiff_t add= key_buff_elem_size + 1 - bytes_for_keys;
-    bytes_for_keys= key_buff_elem_size + 1;
-    bytes_for_rowids -= add;
-  }
-
-  if (bytes_for_rowids < (ptrdiff_t)rowid_buf_elem_size + 1)
-  {
-    ptrdiff_t add= (ptrdiff_t)(rowid_buf_elem_size + 1 - bytes_for_rowids);
-    bytes_for_rowids= (ptrdiff_t)rowid_buf_elem_size + 1;
-    bytes_for_keys -= add;
-  }
+  if (bytes_for_keys < key_buff_elem_size + 1 ||
+      bytes_for_rowids < (ptrdiff_t)rowid_buf_elem_size + 1)
+    return TRUE; /* Failed to provide minimum space for one of the buffers */
 
   rowid_buffer_end= full_buf + bytes_for_rowids;
   rowid_buffer.set_buffer_space(full_buf, rowid_buffer_end);
   key_buffer= &backward_key_buf;
   key_buffer->set_buffer_space(rowid_buffer_end, full_buf_end); 
 
-  if (!key_buffer->have_space_for(key_buff_elem_size) ||
-      !rowid_buffer.have_space_for((size_t)rowid_buf_elem_size))
-    return TRUE; /* Failed to provide minimum space for one of the buffers */
+  /* The above code guarantees that the buffers are big enough */
+  DBUG_ASSERT(key_buffer->have_space_for(key_buff_elem_size) &&
+              rowid_buffer.have_space_for((size_t)rowid_buf_elem_size));
 
   return FALSE;
 }

storage/tokudb/mysql-test/tokudb_mariadb/r/mdev12972.result

+CREATE TABLE t1 (pk INT PRIMARY KEY, i1 INT, c1 VARCHAR(1), KEY(c1)) ENGINE=TokuDB;
+INSERT INTO t1 VALUES 
+(1,2,NULL),(2,5,'x'),(3,3,'i'),(4,1,'e'),(5,4,'p');
+CREATE TABLE t2 (i2 INT) ENGINE=TokuDB;
+INSERT INTO t2 VALUES (1),(2);
+CREATE TABLE t3 (i3 INT, c3 VARCHAR(1), KEY(i3)) ENGINE=TokuDB;
+INSERT INTO t3 VALUES 
+(1,'e'),(1,'z'),(1,'i'),(1,'q'),(1,'i'),(1,'f'),(1,'m'),(2,'c'),(1,'d'),
+(2,'n'),(1,'t'),(2,'e'),(1,'w'),(2,'y'),(1,'j'),(2,'i'),(1,'f'),(2,'f'),
+(1,'s'),(2,'y');
+SET join_cache_level = 8;
+SET optimizer_switch ='mrr=on,mrr_sort_keys=on,optimize_join_buffer_size=on';
+SELECT * FROM t1 AS t1_outer WHERE EXISTS ( SELECT * FROM t2 WHERE i2 IN ( SELECT i3 FROM t3 INNER JOIN t1 AS t1_inner ON (t1_inner.c1 = c3 ) WHERE t1_inner.i1 < t1_outer.i1 ) );
+pk	i1	c1
+1	2	NULL
+2	5	x
+3	3	i
+5	4	p
+DROP TABLE t1, t2, t3;

storage/tokudb/mysql-test/tokudb_mariadb/t/mdev12972.test

+#
+# MDEV-12972 Random and Frequent Segfault (SIG 11) During Runtime
+#
+# This was actually a memory overrun in MRR that TokuDB triggered
+#
+
+CREATE TABLE t1 (pk INT PRIMARY KEY, i1 INT, c1 VARCHAR(1), KEY(c1)) ENGINE=TokuDB;
+INSERT INTO t1 VALUES
+(1,2,NULL),(2,5,'x'),(3,3,'i'),(4,1,'e'),(5,4,'p');
+
+CREATE TABLE t2 (i2 INT) ENGINE=TokuDB;
+INSERT INTO t2 VALUES (1),(2);
+
+CREATE TABLE t3 (i3 INT, c3 VARCHAR(1), KEY(i3)) ENGINE=TokuDB;
+INSERT INTO t3 VALUES
+(1,'e'),(1,'z'),(1,'i'),(1,'q'),(1,'i'),(1,'f'),(1,'m'),(2,'c'),(1,'d'),
+(2,'n'),(1,'t'),(2,'e'),(1,'w'),(2,'y'),(1,'j'),(2,'i'),(1,'f'),(2,'f'),
+(1,'s'),(2,'y');
+
+SET join_cache_level = 8;
+SET optimizer_switch ='mrr=on,mrr_sort_keys=on,optimize_join_buffer_size=on';
+
+--sorted_result
+SELECT * FROM t1 AS t1_outer WHERE EXISTS ( SELECT * FROM t2 WHERE i2 IN ( SELECT i3 FROM t3 INNER JOIN t1 AS t1_inner ON (t1_inner.c1 = c3 ) WHERE t1_inner.i1 < t1_outer.i1 ) );
+
+DROP TABLE t1, t2, t3;