MDEV-33533: Crash at execution of DELETE when trying to use rowid filter

(Based on original patch by Oleksandr Byelkin)

Multi-table DELETE can execute via "buffered" mode: at phase #1 it collects
rowids of rows to be deleted, then at phase #2 in multi_delete::do_deletes()
it calls handler->rnd_pos() to read rows to be deleted and deletes them.

The problem occurred when phase #1 used Rowid Filter on the table that
phase #2 would be deleting from.
In InnoDB, h->rnd_init(scan=false) and h->rnd_pos() is an index scan over PK
under the hood. So, at phase #2 ha_innobase::rnd_init() would try to use the
Rowid Filter and hit an assertion inside ha_innobase::rnd_init().

Note that multi-table UPDATE works similarly but was not affected, because
patch for MDEV-7487 added code to disable rowid filter for phase #2 in
multi_update::do_updates().

This patch changes the approach:
- It makes InnoDB not use Rowid Filter in rnd_pos() scans: it is disabled in
  ha_innobase::rnd_init() and enabled back in ha_innobase::rnd_end().
- multi_update::do_updates() no longer disables Rowid Filter for phase#2 as
  it is no longer necessary.

mysql-test/main/delete_innodb.test

@@ -49,4 +49,175 @@ analyze DELETE FROM t1 WHERE c1 IN (select c2 from t2) ORDER BY c1 limit 1;
 select * from t1;
 
 DROP TABLE t1, t2;
+
+--echo #
+--echo # MDEV-33533: multi-delete using rowid filter
+--echo #
+
+set @save_default_storage_engine=@@default_storage_engine;
+set default_storage_engine=InnoDB;
+
+CREATE DATABASE dbt3_s001;
+
+use dbt3_s001;
+
+--disable_query_log
+--disable_result_log
+--disable_warnings
+--source include/dbt3_s001.inc
+--enable_warnings
+--enable_result_log
+--enable_query_log
+
+create index i_n_name on nation(n_name);
+analyze table
+  nation, lineitem, customer, orders, part, supplier, partsupp, region
+persistent for all;
+
+let $c1=
+        o_orderDATE between '1992-01-01' and '1992-06-30' and
+        o_custkey =  c_custkey and
+        c_nationkey = n_nationkey and
+        n_name='PERU';
+
+eval
+explain
+select o_orderkey, o_totalprice from orders, customer, nation where $c1;
+--source include/explain-no-costs.inc
+eval
+explain format=json
+select o_orderkey, o_totalprice from orders, customer, nation where $c1;
+--sorted_result
+eval
+select o_orderkey, o_totalprice from orders, customer, nation where $c1;
+eval
+create table t as
+select orders.* from orders, customer, nation where $c1;
+
+eval
+explain
+delete from orders using orders, customer, nation where $c1;
+--source include/explain-no-costs.inc
+eval
+explain format=json
+delete from orders using orders, customer, nation where $c1;
+eval
+delete from orders using orders, customer, nation where $c1;
+eval
+select o_orderkey, o_totalprice from orders, customer, nation where $c1;
+
+insert into orders select * from t;
+--sorted_result
+eval
+select o_orderkey, o_totalprice from orders, customer, nation where $c1;
+
+eval
+prepare stmt from "
+delete from orders using orders, customer, nation where $c1;
+";
+
+execute stmt;
+--sorted_result
+eval
+select o_orderkey, o_totalprice from orders, customer, nation where $c1;
+insert into orders select * from t;
+--sorted_result
+eval
+select o_orderkey, o_totalprice from orders, customer, nation where $c1;
+
+execute stmt;
+--sorted_result
+eval
+select o_orderkey, o_totalprice from orders, customer, nation where $c1;
+insert into orders select * from t;
+--sorted_result
+eval
+select o_orderkey, o_totalprice from orders, customer, nation where $c1;
+
+deallocate prepare stmt;
+
+drop table t;
+
+let $c1=
+        o_orderDATE between '1992-01-01' and '1992-06-30' and
+        o_custkey in (select c_custkey from customer
+                        where c_nationkey in (select n_nationkey from nation
+                                                where n_name='PERU'));
+
+eval
+explain
+select o_orderkey, o_totalprice from orders where $c1;
+--source include/explain-no-costs.inc
+eval
+explain format=json
+select o_orderkey, o_totalprice from orders where $c1;
+--sorted_result
+eval
+select o_orderkey, o_totalprice from orders where $c1;
+eval
+create table t as
+select * from orders where $c1;
+
+eval
+explain
+delete from orders where $c1;
+--source include/explain-no-costs.inc
+eval
+explain format=json
+delete from orders where $c1;
+eval
+delete from orders where $c1;
+eval
+select o_orderkey, o_totalprice from orders where $c1;
+
+insert into orders select * from t;
+--sorted_result
+eval
+select o_orderkey, o_totalprice from orders where $c1;
+drop table t;
+
+DROP DATABASE dbt3_s001;
+
+set default_storage_engine=@save_default_storage_engine;
+
+--echo #
+--echo # Additional tests of first table and rowid filter
+--echo #
+
+CREATE DATABASE dbt3_s001;
+
+use dbt3_s001;
+
+set @save_default_storage_engine=@@default_storage_engine;
+set default_storage_engine=InnoDB;
+
+--disable_query_log
+--disable_result_log
+--disable_warnings
+--source include/dbt3_s001.inc
+--enable_warnings
+--enable_result_log
+--enable_query_log
+
+CREATE INDEX i_l_quantity ON lineitem(l_quantity);
+CREATE INDEX i_o_totalprice ON orders(o_totalprice);
+ANALYZE TABLE lineitem, orders;
+set optimizer_use_condition_selectivity=2;
+create table second(s_receiptDATE date, filler char(100), key(s_receiptDATE)) ;
+insert into second select date_add(l_receiptDATE, interval 1 day), 'helllo' from lineitem ;
+
+select count(*) from lineitem, second   WHERE  l_shipdate BETWEEN '1997-01-01' AND '1997-06-30' AND l_quantity > 47 and second.s_receiptDATE = date_add(l_receiptDATE, interval 1 day);
+
+--echo # lineitem should be first and with "Using rowid filter"
+explain delete lineitem FROM lineitem, second   WHERE  l_shipdate BETWEEN '1997-01-01' AND '1997-06-30' AND l_quantity > 47 and second.s_receiptDATE = date_add(l_receiptDATE, interval 1 day);
+
+delete lineitem FROM lineitem, second   WHERE  l_shipdate BETWEEN '1997-01-01' AND '1997-06-30' AND l_quantity > 47 and second.s_receiptDATE = date_add(l_receiptDATE, interval 1 day);
+
+--echo # Should be 0
+select count(*) from lineitem, second   WHERE  l_shipdate BETWEEN '1997-01-01' AND '1997-06-30' AND l_quantity > 47 and second.s_receiptDATE = date_add(l_receiptDATE, interval 1 day);
+
+drop database  dbt3_s001;
+set default_storage_engine=@save_default_storage_engine;
+
+
 --echo End of 11.1 tests

mysql-test/main/update_innodb.result

@@ -143,3 +143,111 @@ connection default;
 disconnect con2;
 drop table t1,t2;
 # End of 10.4 tests
+#
+# MDEV-33533: multi-delete using rowid filter
+#
+set @save_default_storage_engine=@@default_storage_engine;
+set default_storage_engine=InnoDB;
+CREATE DATABASE dbt3_s001;
+use dbt3_s001;
+create index i_n_name on nation(n_name);
+analyze table
+nation, lineitem, customer, orders, part, supplier, partsupp, region
+persistent for all;
+Table	Op	Msg_type	Msg_text
+dbt3_s001.nation	analyze	status	Engine-independent statistics collected
+dbt3_s001.nation	analyze	status	OK
+dbt3_s001.lineitem	analyze	status	Engine-independent statistics collected
+dbt3_s001.lineitem	analyze	status	OK
+dbt3_s001.customer	analyze	status	Engine-independent statistics collected
+dbt3_s001.customer	analyze	status	OK
+dbt3_s001.orders	analyze	status	Engine-independent statistics collected
+dbt3_s001.orders	analyze	status	OK
+dbt3_s001.part	analyze	status	Engine-independent statistics collected
+dbt3_s001.part	analyze	status	OK
+dbt3_s001.supplier	analyze	status	Engine-independent statistics collected
+dbt3_s001.supplier	analyze	status	OK
+dbt3_s001.partsupp	analyze	status	Engine-independent statistics collected
+dbt3_s001.partsupp	analyze	status	OK
+dbt3_s001.region	analyze	status	Engine-independent statistics collected
+dbt3_s001.region	analyze	status	OK
+explain
+update  orders, customer, nation set orders.o_comment = "+++" where o_orderDATE between '1992-01-01' and '1992-06-30' and
+o_custkey =  c_custkey and c_nationkey = n_nationkey and n_name='PERU';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	nation	ref	PRIMARY,i_n_name	i_n_name	26	const	1	Using where; Using index
+1	SIMPLE	customer	ref	PRIMARY,i_c_nationkey	i_c_nationkey	5	dbt3_s001.nation.n_nationkey	6	Using index
+1	SIMPLE	orders	ref|filter	i_o_orderdate,i_o_custkey	i_o_custkey|i_o_orderdate	5|4	dbt3_s001.customer.c_custkey	15 (8%)	Using where; Using rowid filter
+explain format=json
+update  orders, customer, nation set orders.o_comment = "+++" where o_orderDATE between '1992-01-01' and '1992-06-30' and
+o_custkey =  c_custkey and c_nationkey = n_nationkey and n_name='PERU';
+EXPLAIN
+{
+  "query_block": {
+    "select_id": 1,
+    "cost": "COST_REPLACED",
+    "nested_loop": [
+      {
+        "table": {
+          "table_name": "nation",
+          "access_type": "ref",
+          "possible_keys": ["PRIMARY", "i_n_name"],
+          "key": "i_n_name",
+          "key_length": "26",
+          "used_key_parts": ["n_name"],
+          "ref": ["const"],
+          "loops": 1,
+          "rows": 1,
+          "cost": "COST_REPLACED",
+          "filtered": 100,
+          "attached_condition": "nation.n_name = 'PERU'",
+          "using_index": true
+        }
+      },
+      {
+        "table": {
+          "table_name": "customer",
+          "access_type": "ref",
+          "possible_keys": ["PRIMARY", "i_c_nationkey"],
+          "key": "i_c_nationkey",
+          "key_length": "5",
+          "used_key_parts": ["c_nationkey"],
+          "ref": ["dbt3_s001.nation.n_nationkey"],
+          "loops": 1,
+          "rows": 6,
+          "cost": "COST_REPLACED",
+          "filtered": 100,
+          "using_index": true
+        }
+      },
+      {
+        "table": {
+          "table_name": "orders",
+          "access_type": "ref",
+          "possible_keys": ["i_o_orderdate", "i_o_custkey"],
+          "key": "i_o_custkey",
+          "key_length": "5",
+          "used_key_parts": ["o_custkey"],
+          "ref": ["dbt3_s001.customer.c_custkey"],
+          "rowid_filter": {
+            "range": {
+              "key": "i_o_orderdate",
+              "used_key_parts": ["o_orderDATE"]
+            },
+            "rows": 119,
+            "selectivity_pct": 7.933333333
+          },
+          "loops": 6,
+          "rows": 15,
+          "cost": "COST_REPLACED",
+          "filtered": 7.933333397,
+          "attached_condition": "orders.o_orderDATE between '1992-01-01' and '1992-06-30'"
+        }
+      }
+    ]
+  }
+}
+update  orders, customer, nation set orders.o_comment = "+++" where o_orderDATE between '1992-01-01' and '1992-06-30' and
+o_custkey =  c_custkey and c_nationkey = n_nationkey and n_name='PERU';
+DROP DATABASE dbt3_s001;
+set default_storage_engine=@save_default_storage_engine;

mysql-test/main/update_innodb.test

@@ -162,3 +162,50 @@ drop table t1,t2;
 --enable_view_protocol
 
 --echo # End of 10.4 tests
+
+--echo #
+--echo # MDEV-33533: multi-delete using rowid filter
+--echo #
+
+set @save_default_storage_engine=@@default_storage_engine;
+set default_storage_engine=InnoDB;
+
+CREATE DATABASE dbt3_s001;
+
+use dbt3_s001;
+
+--disable_query_log
+--disable_result_log
+--disable_warnings
+--source include/dbt3_s001.inc
+--enable_warnings
+--enable_result_log
+--enable_query_log
+
+create index i_n_name on nation(n_name);
+analyze table
+  nation, lineitem, customer, orders, part, supplier, partsupp, region
+persistent for all;
+
+let $c1=
+        o_orderDATE between '1992-01-01' and '1992-06-30' and
+        o_custkey =  c_custkey and
+        c_nationkey = n_nationkey and
+        n_name='PERU';
+
+
+explain
+update  orders, customer, nation set orders.o_comment = "+++" where o_orderDATE between '1992-01-01' and '1992-06-30' and
+ o_custkey =  c_custkey and c_nationkey = n_nationkey and n_name='PERU';
+--source include/explain-no-costs.inc
+explain format=json
+update  orders, customer, nation set orders.o_comment = "+++" where o_orderDATE between '1992-01-01' and '1992-06-30' and
+ o_custkey =  c_custkey and c_nationkey = n_nationkey and n_name='PERU';
+
+update  orders, customer, nation set orders.o_comment = "+++" where o_orderDATE between '1992-01-01' and '1992-06-30' and
+ o_custkey =  c_custkey and c_nationkey = n_nationkey and n_name='PERU';
+
+DROP DATABASE dbt3_s001;
+
+set default_storage_engine=@save_default_storage_engine;
+

sql/sql_update.cc

@@ -2502,8 +2502,6 @@ int multi_update::do_updates()
     table = cur_table->table;
     if (table == table_to_update)
       continue;					// Already updated
-    if (table->file->pushed_rowid_filter)
-      table->file->disable_pushed_rowid_filter();
     org_updated= updated;
     tmp_table= tmp_tables[cur_table->shared];
     tmp_table->file->extra(HA_EXTRA_CACHE);	// Change to read cache
@@ -2697,9 +2695,7 @@ int multi_update::do_updates()
     (void) tmp_table->file->ha_rnd_end();
     check_opt_it.rewind();
     while (TABLE *tbl= check_opt_it++)
-        tbl->file->ha_rnd_end();
-    if (table->file->save_pushed_rowid_filter)
-      table->file->enable_pushed_rowid_filter();
+      tbl->file->ha_rnd_end();
   }
   DBUG_RETURN(0);
 
@@ -2710,8 +2706,6 @@ int multi_update::do_updates()
   }
 
 err2:
-  if (table->file->save_pushed_rowid_filter)
-    table->file->enable_pushed_rowid_filter();
   if (table->file->inited)
     (void) table->file->ha_rnd_end();
   if (tmp_table->file->inited)

storage/innobase/handler/ha_innodb.cc

@@ -5814,6 +5814,7 @@ ha_innobase::open(const char* name, int, uint)
 	/* Will be allocated if it is needed in ::update_row() */
 	m_upd_buf = NULL;
 	m_upd_buf_size = 0;
+	m_disable_rowid_filter = false;
 
 	char*	is_part = is_partition(norm_name);
 	THD*	thd = ha_thd();
@@ -7286,7 +7287,8 @@ ha_innobase::build_template(
 	/* Below we check column by column if we need to access
 	the clustered index. */
 
-	if (pushed_rowid_filter && rowid_filter_is_active) {
+	if (pushed_rowid_filter && rowid_filter_is_active
+	    && !m_disable_rowid_filter) {
 		fetch_primary_key_cols = TRUE;
 		m_prebuilt->pk_filter = this;
 	} else {
@@ -7343,7 +7345,8 @@ ha_innobase::build_template(
 	simplified to handle both. It should handle the issues. */
 
 	const bool pushed_down = active_index != MAX_KEY
-				 && active_index == pushed_idx_cond_keyno;
+				 && active_index == pushed_idx_cond_keyno
+				 && !m_disable_rowid_filter;
 
 	m_prebuilt->idx_cond = pushed_down ? this : nullptr;
 
@@ -9375,6 +9378,11 @@ ha_innobase::rnd_init(
 {
 	int		err;
 
+	/* Don't use rowid filter when doing full table scan or rnd_pos calls.*/
+	if (!scan) {
+		m_disable_rowid_filter = true;
+	}
+
 	/* Store the active index value so that we can restore the original
 	value after a scan */
 
@@ -9384,6 +9392,12 @@ ha_innobase::rnd_init(
 		err = change_active_index(m_primary_key);
 	}
 
+	if (err && !scan) {
+		/* Restore the original value in case of error */
+		m_disable_rowid_filter = false;
+	}
+
+
 	/* Don't use semi-consistent read in random row reads (by position).
 	This means we must disable semi_consistent_read if scan is false */
 
@@ -9404,6 +9418,7 @@ int
 ha_innobase::rnd_end(void)
 /*======================*/
 {
+	m_disable_rowid_filter = false;
 	return(index_end());
 }
 
@@ -9455,6 +9470,7 @@ ha_innobase::rnd_pos(
 	/* Note that we assume the length of the row reference is fixed
 	for the table, and it is == ref_length */
 
+	DBUG_ASSERT(m_disable_rowid_filter == true);
 	int	error = index_read(buf, pos, (uint)ref_length, HA_READ_KEY_EXACT);
 
 	if (error != 0) {

storage/innobase/handler/ha_innodb.h

@@ -522,6 +522,10 @@ class ha_innobase final : public handler
 
         /** If mysql has locked with external_lock() */
         bool                    m_mysql_has_locked;
+
+	/** If true, disable the Rowid Filter. It is disabled when
+	the enigne is intialized for making rnd_pos() calls */
+	bool                    m_disable_rowid_filter;
 };