MDEV-26803 PA unsafety with FK cascade delete operation

This commit has a mtr test where two two transactions delete a row from
two separate tables, which will cascade a FK delete for the same row in
a third table. Second replica node is configured with 2 applier threads,
and the test will fail if these two transactions are applied in parallel.

The actual fix, in this commit, is to mark a transaction as unsafe for
parallel applying when it traverses into cascade delete operation.
Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>

include/mysql/service_wsrep.h

@@ -88,6 +88,7 @@ extern struct wsrep_service_st {
                                                                 unsigned long long trx_id);
   void                        (*wsrep_thd_kill_LOCK_func)(const MYSQL_THD thd);
   void                        (*wsrep_thd_kill_UNLOCK_func)(const MYSQL_THD thd);
+  void                        (*wsrep_thd_set_wsrep_PA_unsafe_func)(MYSQL_THD thd);
 } *wsrep_service;
 
 #define MYSQL_SERVICE_WSREP_INCLUDED
@@ -131,6 +132,7 @@ extern struct wsrep_service_st {
 #define wsrep_thd_is_applying(T) wsrep_service->wsrep_thd_is_applying_func(T)
 #define wsrep_thd_set_wsrep_aborter(T) wsrep_service->wsrep_thd_set_wsrep_aborter_func(T1, T2)
 #define wsrep_report_bf_lock_wait(T,I) wsrep_service->wsrep_report_bf_lock_wait(T,I)
+#define wsrep_thd_set_PA_unsafe(T) wsrep_service->wsrep_thd_set_PA_unsafe_func(T)
 #else
 
 #define MYSQL_SERVICE_WSREP_STATIC_INCLUDED
@@ -229,5 +231,7 @@ extern "C" my_bool wsrep_thd_is_applying(const MYSQL_THD thd);
 extern "C" bool wsrep_thd_set_wsrep_aborter(MYSQL_THD bf_thd, MYSQL_THD victim_thd);
 extern "C" void wsrep_report_bf_lock_wait(const THD *thd,
                                           unsigned long long trx_id);
+/* declare parallel applying unsafety for the THD */
+extern "C" void wsrep_thd_set_PA_unsafe(MYSQL_THD thd);
 #endif
 #endif /* MYSQL_SERVICE_WSREP_INCLUDED */

mysql-test/suite/galera/r/galera_fk_cascade_delete.result

@@ -48,3 +48,113 @@ id	parent_id
 DROP TABLE child;
 DROP TABLE parent;
 DROP TABLE grandparent;
+
+Scenario 2, testing PA applying with FK cascade delete
+
+CREATE TABLE p1 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE p2 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE c (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER,
+f2 INTEGER,
+CONSTRAINT fk_1 FOREIGN KEY (p1_id) REFERENCES p1 (f1)
+ON DELETE CASCADE,
+CONSTRAINT fk_2 FOREIGN KEY (p2_id) REFERENCES p2 (f1)
+ON DELETE CASCADE);
+connection node_2;
+set global wsrep_slave_threads=DEFAULT;
+SELECT * FROM p1;
+f1	f2
+SELECT * FROM p2;
+f1	f2
+SELECT * FROM c;
+f1	p1_id	p2_id	f2
+connection node_1;
+DROP TABLE c;
+DROP TABLE p1,p2;
+
+Scenario 4, testing PA applying with FK cascade delete on
+more than one level
+
+CREATE TABLE gp1 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE gp2 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE p1 (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER,
+f2 INTEGER,
+CONSTRAINT pfk_3 FOREIGN KEY (p1_id) REFERENCES gp1 (f1)
+ON DELETE CASCADE
+) ENGINE=INNODB;
+CREATE TABLE p2 (f1 INTEGER PRIMARY KEY,p1_id INTEGER, p2_id INTEGER,
+f2 INTEGER,
+CONSTRAINT pfk_4 FOREIGN KEY (p1_id) REFERENCES gp2 (f1)
+ON DELETE CASCADE
+) ENGINE=INNODB;
+CREATE TABLE c (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER,
+f2 INTEGER,
+CONSTRAINT fk_1 FOREIGN KEY (p1_id) REFERENCES p1 (f1)
+ON DELETE CASCADE,
+CONSTRAINT fk_2 FOREIGN KEY (p2_id) REFERENCES p2 (f1)
+ON DELETE CASCADE) ENGINE=INNODB;
+connection node_2;
+set global wsrep_slave_threads=DEFAULT;
+SELECT * FROM gp1;
+f1	f2
+SELECT * FROM gp2;
+f1	f2
+SELECT * FROM p1;
+f1	p1_id	p2_id	f2
+SELECT * FROM p2;
+f1	p1_id	p2_id	f2
+SELECT * FROM c;
+f1	p1_id	p2_id	f2
+connection node_1;
+DROP TABLE c;
+DROP TABLE p1,p2;
+DROP TABLE gp1,gp2;
+
+Scenario 3, testing PA applying with FK cascade delete on
+more than one level in a diamond topology
+
+CREATE TABLE ggp1 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE gp1 (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER,
+f2 INTEGER,
+CONSTRAINT pfk_6 FOREIGN KEY (p1_id) REFERENCES ggp1 (f1)
+ON DELETE CASCADE
+) ENGINE=INNODB;
+CREATE TABLE gp2 (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER,
+f2 INTEGER,
+CONSTRAINT pfk_5 FOREIGN KEY (p1_id) REFERENCES ggp1 (f1)
+ON DELETE CASCADE
+) ENGINE=INNODB;
+CREATE TABLE p1 (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER,
+f2 INTEGER,
+CONSTRAINT pfk_3 FOREIGN KEY (p1_id) REFERENCES gp1 (f1)
+ON DELETE CASCADE
+) ENGINE=INNODB;
+CREATE TABLE p2 (f1 INTEGER PRIMARY KEY,p1_id INTEGER, p2_id INTEGER,
+f2 INTEGER,
+CONSTRAINT pfk_4 FOREIGN KEY (p1_id) REFERENCES gp2 (f1)
+ON DELETE CASCADE
+) ENGINE=INNODB;
+CREATE TABLE c (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER,
+f2 INTEGER,
+CONSTRAINT fk_1 FOREIGN KEY (p1_id) REFERENCES p1 (f1)
+ON DELETE CASCADE,
+CONSTRAINT fk_2 FOREIGN KEY (p2_id) REFERENCES p2 (f1)
+ON DELETE CASCADE) ENGINE=INNODB;
+connection node_2;
+set global wsrep_slave_threads=DEFAULT;
+SELECT * FROM ggp1;
+f1	f2
+SELECT * FROM gp2;
+f1	p1_id	p2_id	f2
+SELECT * FROM gp1;
+f1	p1_id	p2_id	f2
+SELECT * FROM p1;
+f1	p1_id	p2_id	f2
+SELECT * FROM p2;
+f1	p1_id	p2_id	f2
+SELECT * FROM c;
+f1	p1_id	p2_id	f2
+connection node_1;
+DROP TABLE c;
+DROP TABLE p1,p2;
+DROP TABLE gp1,gp2;
+DROP TABLE ggp1;

mysql-test/suite/galera/t/galera_fk_cascade_delete.test

@@ -68,3 +68,189 @@ SELECT * FROM child;
 DROP TABLE child;
 DROP TABLE parent;
 DROP TABLE grandparent;
+
+--echo
+--echo Scenario 2, testing PA applying with FK cascade delete
+--echo
+
+CREATE TABLE p1 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE p2 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE c (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER,
+                f2 INTEGER,
+                CONSTRAINT fk_1 FOREIGN KEY (p1_id) REFERENCES p1 (f1)
+                    ON DELETE CASCADE,
+                CONSTRAINT fk_2 FOREIGN KEY (p2_id) REFERENCES p2 (f1)
+                    ON DELETE CASCADE);
+
+--let $count = 100
+--disable_query_log
+while ($count)
+{
+    --eval INSERT INTO p1 VALUES ($count, 0);
+    --eval INSERT INTO p2 VALUES ($count, 0);
+    --eval INSERT INTO c VALUES ($count, $count, $count, 0);
+    --dec $count
+}
+
+--connection node_2
+set global wsrep_slave_threads=2;
+
+--connection node_1
+--let $count = 100
+while ($count)
+{
+    --eval DELETE FROM p2 WHERE f1=$count;
+    --eval DELETE FROM p1 WHERE f1=$count;
+
+--dec $count
+}
+--enable_query_log
+
+--connection node_2
+set global wsrep_slave_threads=DEFAULT;
+
+
+SELECT * FROM p1;
+SELECT * FROM p2;
+SELECT * FROM c;
+
+--connection node_1
+DROP TABLE c;
+DROP TABLE p1,p2;
+
+--echo
+--echo Scenario 4, testing PA applying with FK cascade delete on
+--echo more than one level
+--echo
+CREATE TABLE gp1 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE gp2 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE p1 (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER,
+                 f2 INTEGER,
+                CONSTRAINT pfk_3 FOREIGN KEY (p1_id) REFERENCES gp1 (f1)
+                    ON DELETE CASCADE
+                ) ENGINE=INNODB;
+CREATE TABLE p2 (f1 INTEGER PRIMARY KEY,p1_id INTEGER, p2_id INTEGER,
+                 f2 INTEGER,
+                CONSTRAINT pfk_4 FOREIGN KEY (p1_id) REFERENCES gp2 (f1)
+                    ON DELETE CASCADE
+		 ) ENGINE=INNODB;
+CREATE TABLE c (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER,
+                f2 INTEGER,
+                CONSTRAINT fk_1 FOREIGN KEY (p1_id) REFERENCES p1 (f1)
+                    ON DELETE CASCADE,
+                CONSTRAINT fk_2 FOREIGN KEY (p2_id) REFERENCES p2 (f1)
+                    ON DELETE CASCADE) ENGINE=INNODB;
+
+--let $count = 100
+--disable_query_log
+while ($count)
+{
+    --eval INSERT INTO gp1 VALUES ($count, 0);
+    --eval INSERT INTO gp2 VALUES ($count, 0);
+    --eval INSERT INTO p1 VALUES ($count, $count, $count, 0);
+    --eval INSERT INTO p2 VALUES ($count, $count, $count, 0);
+    --eval INSERT INTO c VALUES ($count, $count, $count, 0);
+    --dec $count
+}
+
+--connection node_2
+set global wsrep_slave_threads=2;
+
+--connection node_1
+--let $count = 100
+while ($count)
+{
+    --eval DELETE FROM gp1 WHERE f1=$count;
+    --eval DELETE FROM gp2 WHERE f1=$count;
+
+--dec $count
+}
+--enable_query_log
+
+--connection node_2
+set global wsrep_slave_threads=DEFAULT;
+
+SELECT * FROM gp1;
+SELECT * FROM gp2;
+SELECT * FROM p1;
+SELECT * FROM p2;
+SELECT * FROM c;
+
+--connection node_1
+DROP TABLE c;
+DROP TABLE p1,p2;
+DROP TABLE gp1,gp2;
+
+--echo
+--echo Scenario 3, testing PA applying with FK cascade delete on
+--echo more than one level in a diamond topology
+--echo
+CREATE TABLE ggp1 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE gp1 (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER,
+                  f2 INTEGER,
+                  CONSTRAINT pfk_6 FOREIGN KEY (p1_id) REFERENCES ggp1 (f1)
+                  ON DELETE CASCADE
+		  ) ENGINE=INNODB;
+CREATE TABLE gp2 (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER,
+                  f2 INTEGER,
+                  CONSTRAINT pfk_5 FOREIGN KEY (p1_id) REFERENCES ggp1 (f1)
+                  ON DELETE CASCADE
+		  ) ENGINE=INNODB;
+CREATE TABLE p1 (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER,
+                 f2 INTEGER,
+                CONSTRAINT pfk_3 FOREIGN KEY (p1_id) REFERENCES gp1 (f1)
+                    ON DELETE CASCADE
+                ) ENGINE=INNODB;
+CREATE TABLE p2 (f1 INTEGER PRIMARY KEY,p1_id INTEGER, p2_id INTEGER,
+                 f2 INTEGER,
+                CONSTRAINT pfk_4 FOREIGN KEY (p1_id) REFERENCES gp2 (f1)
+                    ON DELETE CASCADE
+		 ) ENGINE=INNODB;
+CREATE TABLE c (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER,
+                f2 INTEGER,
+                CONSTRAINT fk_1 FOREIGN KEY (p1_id) REFERENCES p1 (f1)
+                    ON DELETE CASCADE,
+                CONSTRAINT fk_2 FOREIGN KEY (p2_id) REFERENCES p2 (f1)
+                    ON DELETE CASCADE) ENGINE=INNODB;
+
+--let $count = 100
+--disable_query_log
+while ($count)
+{
+    --eval INSERT INTO ggp1 VALUES ($count, 0);
+    --eval INSERT INTO gp1 VALUES ($count, $count, $count, 0);
+    --eval INSERT INTO gp2 VALUES ($count,  $count, $count, 0);
+    --eval INSERT INTO p1 VALUES ($count, $count, $count, 0);
+    --eval INSERT INTO p2 VALUES ($count, $count, $count, 0);
+    --eval INSERT INTO c VALUES ($count, $count, $count, 0);
+    --dec $count
+}
+
+--connection node_2
+set global wsrep_slave_threads=2;
+
+--connection node_1
+--let $count = 100
+while ($count)
+{
+    --eval DELETE FROM ggp1 WHERE f1=$count;
+
+--dec $count
+}
+--enable_query_log
+
+--connection node_2
+set global wsrep_slave_threads=DEFAULT;
+
+SELECT * FROM ggp1;
+SELECT * FROM gp2;
+SELECT * FROM gp1;
+SELECT * FROM p1;
+SELECT * FROM p2;
+SELECT * FROM c;
+
+--connection node_1
+DROP TABLE c;
+DROP TABLE p1,p2;
+DROP TABLE gp1,gp2;
+DROP TABLE ggp1;

sql/service_wsrep.cc

@@ -381,3 +381,11 @@ extern "C" void wsrep_report_bf_lock_wait(const THD *thd,
                 wsrep_thd_query(thd));
   }
 }
+
+extern "C" void  wsrep_thd_set_PA_unsafe(THD *thd)
+{
+  if (thd && thd->wsrep_cs().mark_transaction_pa_unsafe())
+  {
+    WSREP_DEBUG("session does not have active transaction, can not mark as PA unsafe");
+  }
+}

sql/sql_plugin_services.ic

@@ -176,7 +176,8 @@ static struct wsrep_service_st wsrep_handler = {
   wsrep_thd_set_wsrep_aborter,
   wsrep_report_bf_lock_wait,
   wsrep_thd_kill_LOCK,
-  wsrep_thd_kill_UNLOCK
+  wsrep_thd_kill_UNLOCK,
+  wsrep_thd_set_PA_unsafe
 };
 
 static struct thd_specifics_service_st thd_specifics_handler=

sql/wsrep_dummy.cc

@@ -147,3 +147,7 @@ bool wsrep_thd_set_wsrep_aborter(THD*, THD*)
 void wsrep_report_bf_lock_wait(const THD*,
                                unsigned long long)
 {}
+
+void wsrep_thd_set_PA_unsafe(THD*)
+{}
+

sql/wsrep_thd.h

@@ -90,8 +90,6 @@ void wsrep_create_rollbacker();
 bool wsrep_bf_abort(THD* bf_thd, THD* victim_thd);
 int  wsrep_abort_thd(THD *bf_thd_ptr, THD *victim_thd_ptr, my_bool signal);
 
-extern void  wsrep_thd_set_PA_safe(void *thd_ptr, my_bool safe);
-
 /*
   Helper methods to deal with thread local storage.
   The purpose of these methods is to hide the details of thread

storage/innobase/handler/ha_innodb.cc

@@ -10220,12 +10220,20 @@ wsrep_append_foreign_key(
 	dict_foreign_t*	foreign,	/*!< in: foreign key constraint */
 	const rec_t*	rec,		/*!<in: clustered index record */
 	dict_index_t*	index,		/*!<in: clustered index */
-	ibool		referenced,	/*!<in: is check for referenced table */
+	bool		referenced,	/*!<in: is check for
+					referenced table */
+	upd_node_t*	upd_node,	/*<!in: update node */
+	bool		pa_disable,	/*<!in: disable parallel apply ?*/
 	Wsrep_service_key_type	key_type)	/*!< in: access type of this key
 					(shared, exclusive, reference...) */
 {
-	if (!trx->is_wsrep() || !wsrep_thd_is_local(trx->mysql_thd)) {
+	ut_ad(trx->is_wsrep());
+
+	if (!wsrep_thd_is_local(trx->mysql_thd))
 		return DB_SUCCESS;
+
+	if (upd_node && wsrep_protocol_version < 4) {
+		key_type = WSREP_SERVICE_KEY_SHARED;
 	}
 
 	THD* thd = trx->mysql_thd;
@@ -10286,8 +10294,7 @@ wsrep_append_foreign_key(
 		WSREP_WARN("FK: %s missing in query: %s",
 			   (!foreign->referenced_table) ?
 			   "referenced table" : "foreign table",
-			   (wsrep_thd_query(thd)) ?
-			   wsrep_thd_query(thd) : "void");
+			   wsrep_thd_query(thd));
 		return DB_ERROR;
 	}
 
@@ -10365,20 +10372,24 @@ wsrep_append_foreign_key(
 		wkey_part,
 		(size_t*)&wkey.key_parts_num)) {
 		WSREP_WARN("key prepare failed for cascaded FK: %s",
-			   (wsrep_thd_query(thd)) ?
-			    wsrep_thd_query(thd) : "void");
+			   wsrep_thd_query(thd));
 		return DB_ERROR;
 	}
+
 	rcode = wsrep_thd_append_key(thd, &wkey, 1, key_type);
+
 	if (rcode) {
-		DBUG_PRINT("wsrep", ("row key failed: " ULINTPF, rcode));
 		WSREP_ERROR("Appending cascaded fk row key failed: %s, "
 			    ULINTPF,
-			    (wsrep_thd_query(thd)) ?
-			    wsrep_thd_query(thd) : "void", rcode);
+			    wsrep_thd_query(thd),
+			    rcode);
 		return DB_ERROR;
 	}
 
+	if (pa_disable) {
+		wsrep_thd_set_PA_unsafe(trx->mysql_thd);
+	}
+
 	return DB_SUCCESS;
 }
 

storage/innobase/row/row0ins.cc

@@ -970,7 +970,9 @@ dberr_t wsrep_append_foreign_key(trx_t *trx,
 			       dict_foreign_t*	foreign,
 			       const rec_t*	clust_rec,
 			       dict_index_t*	clust_index,
-			       ibool		referenced,
+			       bool		referenced,
+			       upd_node_t*	upd_node,
+			       bool		pa_disable,
 			       Wsrep_service_key_type	key_type);
 #endif /* WITH_WSREP */
 
@@ -1322,12 +1324,14 @@ row_ins_foreign_check_on_constraint(
 	}
 
 #ifdef WITH_WSREP
+	if (trx->is_wsrep()) {
 		err = wsrep_append_foreign_key(trx, foreign, clust_rec, clust_index,
-				       FALSE, WSREP_SERVICE_KEY_EXCLUSIVE);
+					       false, NULL, true,
+					       WSREP_SERVICE_KEY_EXCLUSIVE);
 		if (err != DB_SUCCESS) {
-		ib::info() << "WSREP: foreign key append failed: " <<  err;
 			goto nonstandard_exit_func;
 		}
+	}
 #endif /* WITH_WSREP */
 	mtr_commit(mtr);
 
@@ -1722,19 +1726,16 @@ row_ins_check_foreign_constraint(
 				if (check_ref) {
 					err = DB_SUCCESS;
 #ifdef WITH_WSREP
+					if (trx->is_wsrep()) {
 						err = wsrep_append_foreign_key(
 							thr_get_trx(thr),
 							foreign,
 							rec,
 							check_index,
 							check_ref,
-						(upd_node != NULL
-						 && wsrep_protocol_version < 4)
-						? WSREP_SERVICE_KEY_SHARED
-						: WSREP_SERVICE_KEY_REFERENCE);
-					if (err != DB_SUCCESS) {
-						fprintf(stderr,
-							"WSREP: foreign key append failed: %d\n", err);
+							upd_node,
+						        false,
+						        WSREP_SERVICE_KEY_REFERENCE);
 					}
 #endif /* WITH_WSREP */
 					goto end_scan;