Commit 8a22d1ad authored by Marko Mäkelä's avatar Marko Mäkelä

Bug #11766513 - 59641: Prepared XA transaction in system after hard crash

causes future shutdown hang

InnoDB would hang on shutdown if any XA transactions exist in the
system in the PREPARED state. This has been masked by the fact that
MySQL would roll back any PREPARED transaction on shutdown, in the
spirit of Bug #12161 Xa recovery and client disconnection.

[mysql-test-run] do_shutdown_server: Interpret --shutdown_server 0 as
a request to kill the server immediately without initiating a
shutdown procedure.

xid_cache_insert(): Initialize XID_STATE::rm_error in order to avoid a
bogus error message on XA ROLLBACK of a recovered PREPARED transaction.

innobase_commit_by_xid(), innobase_rollback_by_xid(): Free the InnoDB
transaction object after rolling back a PREPARED transaction.

trx_get_trx_by_xid(): Only consider transactions whose
trx->is_prepared flag is set. The MySQL layer seems to prevent
attempts to roll back connected transactions that are in the PREPARED
state from another connection, but it is better to play it safe. The
is_prepared flag was introduced in the InnoDB Plugin.

trx_n_prepared: A new counter, counting the number of InnoDB
transactions in the PREPARED state.

logs_empty_and_mark_files_at_shutdown(): On shutdown, allow
trx_n_prepared transactions to exist in the system.

trx_undo_free_prepared(), trx_free_prepared(): New functions, to free
the memory objects of PREPARED transactions on shutdown. This is not
needed in the built-in InnoDB, because it would collect all allocated
memory on shutdown. The InnoDB Plugin needs this because of
innodb_use_sys_malloc.

trx_sys_close(): Invoke trx_free_prepared() on all remaining
transactions.
parent 88cc4706
......@@ -4461,13 +4461,14 @@ static int my_kill(int pid, int sig)
command called command
DESCRIPTION
shutdown [<timeout>]
shutdown_server [<timeout>]
*/
void do_shutdown_server(struct st_command *command)
{
int timeout=60, pid;
long timeout=60;
int pid;
DYNAMIC_STRING ds_pidfile_name;
MYSQL* mysql = &cur_con->mysql;
static DYNAMIC_STRING ds_timeout;
......@@ -4482,8 +4483,9 @@ void do_shutdown_server(struct st_command *command)
if (ds_timeout.length)
{
timeout= atoi(ds_timeout.str);
if (timeout == 0)
char* endptr;
timeout= strtol(ds_timeout.str, &endptr, 10);
if (*endptr != '\0')
die("Illegal argument for timeout: '%s'", ds_timeout.str);
}
dynstr_free(&ds_timeout);
......@@ -4525,7 +4527,7 @@ void do_shutdown_server(struct st_command *command)
DBUG_PRINT("info", ("Process %d does not exist anymore", pid));
DBUG_VOID_RETURN;
}
DBUG_PRINT("info", ("Sleeping, timeout: %d", timeout));
DBUG_PRINT("info", ("Sleeping, timeout: %ld", timeout));
my_sleep(1000000L);
}
......
CREATE TABLE t(a INT PRIMARY KEY, b INT)ENGINE=InnoDB;
INSERT INTO t VALUES(2,2),(4,4),(8,8),(16,16),(32,32);
COMMIT;
XA START '123';
INSERT INTO t VALUES(1,1);
XA END '123';
XA PREPARE '123';
XA START '456';
INSERT INTO t VALUES(3,47),(5,67);
UPDATE t SET b=2*b WHERE a BETWEEN 5 AND 8;
XA END '456';
XA PREPARE '456';
XA START '789';
UPDATE t SET b=4*a WHERE a=32;
XA END '789';
XA PREPARE '789';
call mtr.add_suppression("Found 3 prepared XA transactions");
SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
SELECT * FROM t;
a b
1 1
2 2
3 47
4 4
5 134
8 16
16 16
32 128
COMMIT;
SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
SELECT * FROM t;
a b
1 1
2 2
3 47
4 4
5 134
8 16
16 16
32 128
COMMIT;
XA RECOVER;
formatID gtrid_length bqual_length data
1 3 0 789
1 3 0 456
1 3 0 123
XA ROLLBACK '123';
XA ROLLBACK '456';
XA COMMIT '789';
SELECT * FROM t;
a b
2 2
4 4
8 8
16 16
32 128
DROP TABLE t;
# Bug #59641 Prepared XA transaction causes shutdown hang after a crash
-- source include/not_embedded.inc
-- source include/have_innodb.inc
CREATE TABLE t(a INT PRIMARY KEY, b INT)ENGINE=InnoDB;
INSERT INTO t VALUES(2,2),(4,4),(8,8),(16,16),(32,32);
COMMIT;
XA START '123';
INSERT INTO t VALUES(1,1);
XA END '123';
XA PREPARE '123';
CONNECT (con1,localhost,root,,);
CONNECTION con1;
XA START '456';
INSERT INTO t VALUES(3,47),(5,67);
UPDATE t SET b=2*b WHERE a BETWEEN 5 AND 8;
XA END '456';
XA PREPARE '456';
CONNECT (con2,localhost,root,,);
CONNECTION con2;
XA START '789';
UPDATE t SET b=4*a WHERE a=32;
XA END '789';
XA PREPARE '789';
# The server would issue this warning on restart.
call mtr.add_suppression("Found 3 prepared XA transactions");
# Kill the server without sending a shutdown command
-- exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
-- shutdown_server 0
-- source include/wait_until_disconnected.inc
# Restart the server.
-- exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
-- enable_reconnect
-- source include/wait_until_connected_again.inc
SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
SELECT * FROM t;
COMMIT;
# Shut down the server. This would hang because of the bug.
-- exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
-- shutdown_server
-- source include/wait_until_disconnected.inc
# Restart the server.
-- exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
-- enable_reconnect
-- source include/wait_until_connected_again.inc
SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
SELECT * FROM t;
COMMIT;
XA RECOVER;
XA ROLLBACK '123';
XA ROLLBACK '456';
XA COMMIT '789';
SELECT * FROM t;
DROP TABLE t;
CREATE TABLE t(a INT PRIMARY KEY, b INT)ENGINE=InnoDB;
INSERT INTO t VALUES(2,2),(4,4),(8,8),(16,16),(32,32);
COMMIT;
XA START '123';
INSERT INTO t VALUES(1,1);
XA END '123';
XA PREPARE '123';
XA START '456';
INSERT INTO t VALUES(3,47),(5,67);
UPDATE t SET b=2*b WHERE a BETWEEN 5 AND 8;
XA END '456';
XA PREPARE '456';
XA START '789';
UPDATE t SET b=4*a WHERE a=32;
XA END '789';
XA PREPARE '789';
call mtr.add_suppression("Found 3 prepared XA transactions");
SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
SELECT * FROM t;
a b
1 1
2 2
3 47
4 4
5 134
8 16
16 16
32 128
COMMIT;
SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
SELECT * FROM t;
a b
1 1
2 2
3 47
4 4
5 134
8 16
16 16
32 128
COMMIT;
XA RECOVER;
formatID gtrid_length bqual_length data
1 3 0 789
1 3 0 456
1 3 0 123
XA ROLLBACK '123';
XA ROLLBACK '456';
XA COMMIT '789';
SELECT * FROM t;
a b
2 2
4 4
8 8
16 16
32 128
DROP TABLE t;
# Bug #59641 Prepared XA transaction causes shutdown hang after a crash
-- source include/not_embedded.inc
-- source include/have_innodb_plugin.inc
let $innodb_file_format_check_orig=`select @@innodb_file_format_check`;
CREATE TABLE t(a INT PRIMARY KEY, b INT)ENGINE=InnoDB;
INSERT INTO t VALUES(2,2),(4,4),(8,8),(16,16),(32,32);
COMMIT;
XA START '123';
INSERT INTO t VALUES(1,1);
XA END '123';
XA PREPARE '123';
CONNECT (con1,localhost,root,,);
CONNECTION con1;
XA START '456';
INSERT INTO t VALUES(3,47),(5,67);
UPDATE t SET b=2*b WHERE a BETWEEN 5 AND 8;
XA END '456';
XA PREPARE '456';
CONNECT (con2,localhost,root,,);
CONNECTION con2;
XA START '789';
UPDATE t SET b=4*a WHERE a=32;
XA END '789';
XA PREPARE '789';
# The server would issue this warning on restart.
call mtr.add_suppression("Found 3 prepared XA transactions");
# Kill the server without sending a shutdown command
-- exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
-- shutdown_server 0
-- source include/wait_until_disconnected.inc
# Restart the server.
-- exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
-- enable_reconnect
-- source include/wait_until_connected_again.inc
SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
SELECT * FROM t;
COMMIT;
# Shut down the server. This would hang because of the bug.
-- exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
-- shutdown_server
-- source include/wait_until_disconnected.inc
# Restart the server.
-- exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
-- enable_reconnect
-- source include/wait_until_connected_again.inc
SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
SELECT * FROM t;
COMMIT;
XA RECOVER;
XA ROLLBACK '123';
XA ROLLBACK '456';
XA COMMIT '789';
SELECT * FROM t;
DROP TABLE t;
--disable_query_log
eval set global innodb_file_format_check=$innodb_file_format_check_orig;
......@@ -3383,6 +3383,7 @@ bool xid_cache_insert(XID *xid, enum xa_states xa_state)
xs->xa_state=xa_state;
xs->xid.set(xid);
xs->in_thd=0;
xs->rm_error=0;
res=my_hash_insert(&xid_cache, (uchar*)xs);
}
pthread_mutex_unlock(&LOCK_xid_cache);
......
......@@ -8565,7 +8565,7 @@ innobase_commit_by_xid(
if (trx) {
innobase_commit_low(trx);
trx_free_for_background(trx);
return(XA_OK);
} else {
return(XAER_NOTA);
......@@ -8588,7 +8588,9 @@ innobase_rollback_by_xid(
trx = trx_get_trx_by_xid(xid);
if (trx) {
return(innobase_rollback_trx(trx));
int ret = innobase_rollback_trx(trx);
trx_free_for_background(trx);
return(ret);
} else {
return(XAER_NOTA);
}
......
......@@ -19,7 +19,12 @@ Created 3/26/1996 Heikki Tuuri
#include "dict0types.h"
#include "trx0xa.h"
/* Number of transactions currently allocated for MySQL: protected by
the kernel mutex */
extern ulint trx_n_mysql_transactions;
/* Number of transactions currently in the XA PREPARED state: protected by
the kernel mutex */
extern ulint trx_n_prepared;
/************************************************************************
Releases the search latch if trx has reserved it. */
......
......@@ -3052,12 +3052,13 @@ logs_empty_and_mark_files_at_shutdown(void)
goto loop;
}
/* Check that there are no longer transactions. We need this wait even
for the 'very fast' shutdown, because the InnoDB layer may have
committed or prepared transactions and we don't want to lose them. */
/* Check that there are no longer transactions, except for
PREPARED ones. We need this wait even for the 'very fast'
shutdown, because the InnoDB layer may have committed or
prepared transactions and we don't want to lose them. */
if (trx_n_mysql_transactions > 0
|| UT_LIST_GET_LEN(trx_sys->trx_list) > 0) {
|| UT_LIST_GET_LEN(trx_sys->trx_list) > trx_n_prepared) {
mutex_exit(&kernel_mutex);
......
......@@ -41,6 +41,9 @@ sess_t* trx_dummy_sess = NULL;
/* Number of transactions currently allocated for MySQL: protected by
the kernel mutex */
ulint trx_n_mysql_transactions = 0;
/* Number of transactions currently in the XA PREPARED state: protected by
the kernel mutex */
ulint trx_n_prepared = 0;
/*****************************************************************
Starts the transaction if it is not yet started. */
......@@ -480,6 +483,7 @@ trx_lists_init_at_db_start(void)
if (srv_force_recovery == 0) {
trx->conc_state = TRX_PREPARED;
trx_n_prepared++;
} else {
fprintf(stderr,
"InnoDB: Since"
......@@ -558,6 +562,7 @@ trx_lists_init_at_db_start(void)
trx->conc_state
= TRX_PREPARED;
trx_n_prepared++;
} else {
fprintf(stderr,
"InnoDB: Since"
......@@ -832,6 +837,11 @@ trx_commit_off_kernel(
|| trx->conc_state == TRX_PREPARED);
ut_ad(mutex_own(&kernel_mutex));
if (UNIV_UNLIKELY(trx->conc_state == TRX_PREPARED)) {
ut_a(trx_n_prepared > 0);
trx_n_prepared--;
}
/* The following assignment makes the transaction committed in memory
and makes its changes to data visible to other transactions.
NOTE that there is a small discrepancy from the strict formal
......@@ -1882,6 +1892,7 @@ trx_prepare_off_kernel(
/*--------------------------------------*/
trx->conc_state = TRX_PREPARED;
trx_n_prepared++;
/*--------------------------------------*/
if (must_flush_log) {
......
2011-04-07 The InnoDB Team
* handler/ha_innodb.cc, include/trx0trx.h, include/trx0undo.h,
log/log0log.c, trx/trx0sys.c, trx/trx0trx.c, trx/trx0undo.c:
Fix Bug #59641 Prepared XA transaction in system after hard crash
causes future shutdown hang
2011-03-30 The InnoDB Team
* srv/srv0srv.c, sync/sync0arr.h, sync/sync0arr.c:
......
......@@ -9998,7 +9998,7 @@ innobase_commit_by_xid(
if (trx) {
innobase_commit_low(trx);
trx_free_for_background(trx);
return(XA_OK);
} else {
return(XAER_NOTA);
......@@ -10024,7 +10024,9 @@ innobase_rollback_by_xid(
trx = trx_get_trx_by_xid(xid);
if (trx) {
return(innobase_rollback_trx(trx));
int ret = innobase_rollback_trx(trx);
trx_free_for_background(trx);
return(ret);
} else {
return(XAER_NOTA);
}
......
......@@ -44,6 +44,9 @@ extern sess_t* trx_dummy_sess;
/** Number of transactions currently allocated for MySQL: protected by
the kernel mutex */
extern ulint trx_n_mysql_transactions;
/** Number of transactions currently in the XA PREPARED state: protected by
the kernel mutex */
extern ulint trx_n_prepared;
/********************************************************************//**
Releases the search latch if trx has reserved it. */
......@@ -108,6 +111,14 @@ trx_free(
/*=====*/
trx_t* trx); /*!< in, own: trx object */
/********************************************************************//**
At shutdown, frees a transaction object that is in the PREPARED state. */
UNIV_INTERN
void
trx_free_prepared(
/*==============*/
trx_t* trx) /*!< in, own: trx object */
__attribute__((nonnull));
/********************************************************************//**
Frees a transaction object for MySQL. */
UNIV_INTERN
void
......
......@@ -298,6 +298,15 @@ void
trx_undo_insert_cleanup(
/*====================*/
trx_t* trx); /*!< in: transaction handle */
/********************************************************************//**
At shutdown, frees the undo logs of a PREPARED transaction. */
UNIV_INTERN
void
trx_undo_free_prepared(
/*===================*/
trx_t* trx) /*!< in/out: PREPARED transaction */
__attribute__((nonnull));
#endif /* !UNIV_HOTBACKUP */
/***********************************************************//**
Parses the redo log entry of an undo log page initialization.
......
......@@ -3085,12 +3085,13 @@ logs_empty_and_mark_files_at_shutdown(void)
goto loop;
}
/* Check that there are no longer transactions. We need this wait even
for the 'very fast' shutdown, because the InnoDB layer may have
committed or prepared transactions and we don't want to lose them. */
/* Check that there are no longer transactions, except for
PREPARED ones. We need this wait even for the 'very fast'
shutdown, because the InnoDB layer may have committed or
prepared transactions and we don't want to lose them. */
if (trx_n_mysql_transactions > 0
|| UT_LIST_GET_LEN(trx_sys->trx_list) > 0) {
|| UT_LIST_GET_LEN(trx_sys->trx_list) > trx_n_prepared) {
mutex_exit(&kernel_mutex);
......
......@@ -37,6 +37,7 @@ Created 3/26/1996 Heikki Tuuri
#include "trx0rseg.h"
#include "trx0undo.h"
#include "srv0srv.h"
#include "srv0start.h"
#include "trx0purge.h"
#include "log0log.h"
#include "os0file.h"
......@@ -1548,10 +1549,12 @@ void
trx_sys_close(void)
/*===============*/
{
trx_t* trx;
trx_rseg_t* rseg;
read_view_t* view;
ut_ad(trx_sys != NULL);
ut_ad(srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS);
/* Check that all read views are closed except read view owned
by a purge. */
......@@ -1583,6 +1586,13 @@ trx_sys_close(void)
mem_free(trx_doublewrite);
trx_doublewrite = NULL;
/* Only prepared transactions may be left in the system. Free them. */
ut_a(UT_LIST_GET_LEN(trx_sys->trx_list) == trx_n_prepared);
while ((trx = UT_LIST_GET_FIRST(trx_sys->trx_list)) != NULL) {
trx_free_prepared(trx);
}
/* There can't be any active transactions. */
rseg = UT_LIST_GET_FIRST(trx_sys->rseg_list);
......
......@@ -50,6 +50,9 @@ UNIV_INTERN sess_t* trx_dummy_sess = NULL;
/** Number of transactions currently allocated for MySQL: protected by
the kernel mutex */
UNIV_INTERN ulint trx_n_mysql_transactions = 0;
/* Number of transactions currently in the XA PREPARED state: protected by
the kernel mutex */
UNIV_INTERN ulint trx_n_prepared = 0;
/*************************************************************//**
Set detailed error message for the transaction. */
......@@ -333,6 +336,60 @@ trx_free(
mem_free(trx);
}
/********************************************************************//**
At shutdown, frees a transaction object that is in the PREPARED state. */
UNIV_INTERN
void
trx_free_prepared(
/*==============*/
trx_t* trx) /*!< in, own: trx object */
{
ut_ad(mutex_own(&kernel_mutex));
ut_a(trx->conc_state == TRX_PREPARED);
ut_a(trx->magic_n == TRX_MAGIC_N);
/* Prepared transactions are sort of active; they allow
ROLLBACK and COMMIT operations. Because the system does not
contain any other transactions than prepared transactions at
the shutdown stage and because a transaction cannot become
PREPARED while holding locks, it is safe to release the locks
held by PREPARED transactions here at shutdown.*/
lock_release_off_kernel(trx);
trx_undo_free_prepared(trx);
mutex_free(&trx->undo_mutex);
if (trx->undo_no_arr) {
trx_undo_arr_free(trx->undo_no_arr);
}
ut_a(UT_LIST_GET_LEN(trx->signals) == 0);
ut_a(UT_LIST_GET_LEN(trx->reply_signals) == 0);
ut_a(trx->wait_lock == NULL);
ut_a(UT_LIST_GET_LEN(trx->wait_thrs) == 0);
ut_a(!trx->has_search_latch);
ut_a(trx->dict_operation_lock_mode == 0);
if (trx->lock_heap) {
mem_heap_free(trx->lock_heap);
}
if (trx->global_read_view_heap) {
mem_heap_free(trx->global_read_view_heap);
}
ut_a(ib_vector_is_empty(trx->autoinc_locks));
ib_vector_free(trx->autoinc_locks);
UT_LIST_REMOVE(trx_list, trx_sys->trx_list, trx);
mem_free(trx);
}
/********************************************************************//**
Frees a transaction object for MySQL. */
UNIV_INTERN
......@@ -463,6 +520,7 @@ trx_lists_init_at_db_start(void)
if (srv_force_recovery == 0) {
trx->conc_state = TRX_PREPARED;
trx_n_prepared++;
} else {
fprintf(stderr,
"InnoDB: Since"
......@@ -541,6 +599,7 @@ trx_lists_init_at_db_start(void)
trx->conc_state
= TRX_PREPARED;
trx_n_prepared++;
} else {
fprintf(stderr,
"InnoDB: Since"
......@@ -820,6 +879,11 @@ trx_commit_off_kernel(
|| trx->conc_state == TRX_PREPARED);
ut_ad(mutex_own(&kernel_mutex));
if (UNIV_UNLIKELY(trx->conc_state == TRX_PREPARED)) {
ut_a(trx_n_prepared > 0);
trx_n_prepared--;
}
/* The following assignment makes the transaction committed in memory
and makes its changes to data visible to other transactions.
NOTE that there is a small discrepancy from the strict formal
......@@ -1857,6 +1921,7 @@ trx_prepare_off_kernel(
/*--------------------------------------*/
trx->conc_state = TRX_PREPARED;
trx_n_prepared++;
/*--------------------------------------*/
if (lsn) {
......@@ -2031,10 +2096,11 @@ trx_get_trx_by_xid(
while (trx) {
/* Compare two X/Open XA transaction id's: their
length should be the same and binary comparison
of gtrid_lenght+bqual_length bytes should be
of gtrid_length+bqual_length bytes should be
the same */
if (trx->conc_state == TRX_PREPARED
if (trx->is_recovered
&& trx->conc_state == TRX_PREPARED
&& xid->gtrid_length == trx->xid.gtrid_length
&& xid->bqual_length == trx->xid.bqual_length
&& memcmp(xid->data, trx->xid.data,
......
......@@ -36,6 +36,7 @@ Created 3/26/1996 Heikki Tuuri
#include "trx0rseg.h"
#include "trx0trx.h"
#include "srv0srv.h"
#include "srv0start.h"
#include "trx0rec.h"
#include "trx0purge.h"
......@@ -1976,4 +1977,31 @@ trx_undo_insert_cleanup(
mutex_exit(&(rseg->mutex));
}
/********************************************************************//**
At shutdown, frees the undo logs of a PREPARED transaction. */
UNIV_INTERN
void
trx_undo_free_prepared(
/*===================*/
trx_t* trx) /*!< in/out: PREPARED transaction */
{
mutex_enter(&trx->rseg->mutex);
ut_ad(srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS);
if (trx->update_undo) {
ut_a(trx->update_undo->state == TRX_UNDO_PREPARED);
UT_LIST_REMOVE(undo_list, trx->rseg->update_undo_list,
trx->update_undo);
trx_undo_mem_free(trx->update_undo);
}
if (trx->insert_undo) {
ut_a(trx->insert_undo->state == TRX_UNDO_PREPARED);
UT_LIST_REMOVE(undo_list, trx->rseg->insert_undo_list,
trx->insert_undo);
trx_undo_mem_free(trx->insert_undo);
}
mutex_exit(&trx->rseg->mutex);
}
#endif /* !UNIV_HOTBACKUP */
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment