diff options
Diffstat (limited to 'mysql-test/suite/rpl/t/rpl_gtid_crash.test')
| -rw-r--r-- | mysql-test/suite/rpl/t/rpl_gtid_crash.test | 668 |
1 files changed, 668 insertions, 0 deletions
diff --git a/mysql-test/suite/rpl/t/rpl_gtid_crash.test b/mysql-test/suite/rpl/t/rpl_gtid_crash.test new file mode 100644 index 00000000..28329831 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_gtid_crash.test @@ -0,0 +1,668 @@ +--source include/have_innodb.inc +--source include/have_debug.inc +# Valgrind does not work well with test that crashes the server +--source include/not_valgrind.inc + +--let $rpl_topology=1->2 +--source include/rpl_init.inc + +--echo *** Test crashing master, causing slave IO thread to reconnect while SQL thread is running *** + +--connection server_1 +call mtr.add_suppression("Checking table:"); +call mtr.add_suppression("client is using or hasn't closed the table properly"); +call mtr.add_suppression("Table .* is marked as crashed and should be repaired"); +flush tables; + +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1, 0); +--save_master_pos + +--connection server_2 +--sync_with_master +SET sql_log_bin=0; +call mtr.add_suppression('Master command COM_REGISTER_SLAVE failed: failed registering on master, reconnecting to try again'); +SET sql_log_bin=1; +--source include/stop_slave.inc +--replace_result $MASTER_MYPORT MASTER_PORT +eval CHANGE MASTER TO master_host = '127.0.0.1', master_port = $MASTER_MYPORT, + MASTER_USE_GTID=SLAVE_POS; + +--connection server_1 +INSERT INTO t1 VALUES (2,1); +INSERT INTO t1 VALUES (3,1); + +--connection server_2 +--source include/start_slave.inc + +--connection server_1 + +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +wait-rpl_gtid_crash.test +EOF + +let $1=200; +--disable_query_log +while ($1) +{ + eval INSERT INTO t1 VALUES ($1 + 10, 2); + dec $1; +} +--enable_query_log +--source include/save_master_gtid.inc + +SET SESSION debug_dbug="+d,crash_dispatch_command_before"; +--error 2006,2013 +SELECT 1; +--source include/wait_until_disconnected.inc + +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +restart-rpl_gtid_crash.test +EOF + +--enable_reconnect +--source include/wait_until_connected_again.inc + +--connection server_2 +--source include/sync_with_master_gtid.inc + +--connection server_1 +INSERT INTO t1 VALUES (1000, 3); +--source include/save_master_gtid.inc + +--connection server_2 +--source include/sync_with_master_gtid.inc + +--connection server_1 +DROP TABLE t1; +--save_master_pos + +--echo *** Test crashing the master mysqld and check that binlog state is recovered. *** +--connection server_2 +--sync_with_master +--source include/stop_slave.inc +RESET MASTER; +SET GLOBAL gtid_slave_pos=''; + +--connection server_1 +RESET MASTER; +--replace_column 2 # 4 # 5 # +SHOW BINLOG EVENTS IN 'master-bin.000001' LIMIT 1,1; +CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB; + +--connection server_2 +--source include/start_slave.inc + +--connection server_1 +SET gtid_domain_id= 1; +INSERT INTO t1 VALUES (1); +INSERT INTO t1 VALUES (2); +FLUSH LOGS; +SET gtid_domain_id= 2; +INSERT INTO t1 VALUES (3); +FLUSH LOGS; +--source include/show_binary_logs.inc +--replace_column 2 # 4 # 5 # 6 # +SHOW BINLOG EVENTS IN 'master-bin.000003' LIMIT 1,1; + +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +wait +EOF + +SET SESSION debug_dbug="+d,crash_dispatch_command_before"; +--error 2006,2013 +SELECT 1; +--source include/wait_until_disconnected.inc + +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +restart +EOF + +--enable_reconnect +--source include/wait_until_connected_again.inc + +--source include/show_binary_logs.inc +--replace_column 2 # 4 # 5 # 6 # +SHOW BINLOG EVENTS IN 'master-bin.000004' LIMIT 1,1; +--save_master_pos + +--connection server_2 +--sync_with_master +SELECT * FROM t1 ORDER BY a; + + +--echo *** Test crashing slave at various points and check that it recovers crash-safe. *** + +# Crash the slave just before updating mysql.gtid_slave_pos table. +--source include/stop_slave.inc +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +wait +EOF +# We do not have to save @@GLOBAL.debug_dbug, it is reset when slave crashes. +SET GLOBAL debug_dbug="+d,inject_crash_before_write_rpl_slave_state"; +START SLAVE; + +--connection server_1 +INSERT INTO t1 VALUES (4); +--source include/save_master_gtid.inc + +--connection server_2 +--source include/wait_until_disconnected.inc + +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +restart: --skip-slave-start=0 +EOF + +--enable_reconnect +--source include/wait_until_connected_again.inc + +--source include/sync_with_master_gtid.inc + +# Crash the slave just before committing. +--source include/stop_slave.inc +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +wait +EOF +START SLAVE; +--error 0,2006,2013 +SET GLOBAL debug_dbug="+d,crash_commit_before"; + +--connection server_1 +INSERT INTO t1 VALUES (5); +--source include/save_master_gtid.inc + +--connection server_2 +--source include/wait_until_disconnected.inc + +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +restart: --skip-slave-start=0 +EOF + +--enable_reconnect +--source include/wait_until_connected_again.inc + +--source include/sync_with_master_gtid.inc + +# Crash the slave just after committing. +--source include/stop_slave.inc +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +wait +EOF +START SLAVE; +--error 0,2006,2013 +SET GLOBAL debug_dbug="+d,crash_commit_after"; + +--connection server_1 +INSERT INTO t1 VALUES (6); +--source include/save_master_gtid.inc + +--connection server_2 +--source include/wait_until_disconnected.inc + +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +restart: --skip-slave-start=0 +EOF + +--enable_reconnect +--source include/wait_until_connected_again.inc + +--source include/sync_with_master_gtid.inc + +# Crash the slave just before updating relay-log.info +--source include/stop_slave.inc +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +wait +EOF +SET GLOBAL debug_dbug="+d,inject_crash_before_flush_rli"; +--error 0,2006,2013 +START SLAVE; + +--connection server_1 +INSERT INTO t1 VALUES (7); +--source include/save_master_gtid.inc + +--connection server_2 +--source include/wait_until_disconnected.inc + +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +restart: --skip-slave-start=0 +EOF + +--enable_reconnect +--source include/wait_until_connected_again.inc + +--source include/sync_with_master_gtid.inc + +# Crash the slave just after updating relay-log.info +--source include/stop_slave.inc +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +wait +EOF +SET GLOBAL debug_dbug="+d,inject_crash_after_flush_rli"; +--error 0,2006,2013 +START SLAVE; + +--connection server_1 +INSERT INTO t1 VALUES (8); +--source include/save_master_gtid.inc + +--connection server_2 +--source include/wait_until_disconnected.inc + +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +restart: --skip-slave-start=0 +EOF + +--enable_reconnect +--source include/wait_until_connected_again.inc + +--source include/sync_with_master_gtid.inc + + +# Check that everything was replicated correctly. +SELECT * FROM t1 ORDER BY a; + + +--echo *** MDEV-4725: Incorrect recovery when crash in the middle of writing an event group *** + +--connection server_2 +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +wait +EOF + +--let $old_gtid_strict= `SELECT @@gtid_strict_mode` +SET GLOBAL debug_dbug="+d,crash_before_writing_xid"; + +--connection server_1 +INSERT INTO t1 VALUES (9), (10); +--let $saved_gtid=`SELECT @@last_gtid` +--save_master_pos + +--connection server_2 +--source include/wait_until_disconnected.inc + +# The bug was that during crash recovery we would update the binlog state +# with the GTID of the half-written event group at the end of the slaves +# binlog, even though this event group was not committed. +# We restart the server with gtid_strict_mode; this way, we get an error +# about duplicate gtid when the slave re-executes the event group, if the +# binlog crash recovery is incorrect. +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +restart: --gtid_strict_mode=1 +EOF + +--enable_reconnect +--source include/wait_until_connected_again.inc + +SHOW VARIABLES like 'gtid_strict_mode'; +--source include/start_slave.inc +--sync_with_master +--disable_query_log +eval SET GLOBAL gtid_strict_mode= $old_gtid_strict; +--enable_query_log + + +--echo *** MDEV-6462: Incorrect recovery on a slave reconnecting to crashed master *** + +--connection server_1 +set sql_log_bin= 0; +call mtr.add_suppression("Error writing file 'master-bin'"); +set sql_log_bin= 1; +--connection server_2 +set sql_log_bin= 0; +call mtr.add_suppression("The server_id of master server changed in the middle of GTID"); +call mtr.add_suppression("Unexpected change of master binlog file name in the middle of GTID"); +set sql_log_bin= 1; + +--connection server_1 +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +wait +EOF + +SET GLOBAL debug_dbug="+d,inject_error_writing_xid"; +BEGIN; +INSERT INTO t1 VALUES (11); +--error ER_ERROR_ON_WRITE +COMMIT; +SET GLOBAL debug_dbug="+d,crash_dispatch_command_before"; +--error 2006,2013 +COMMIT; + +--source include/wait_until_disconnected.inc + +# Simulate that we reconnect to a different server (new server_id). +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +restart: --server-id=3 +EOF + +--enable_reconnect +--source include/wait_until_connected_again.inc + +SELECT @@GLOBAL.server_id; +SELECT * from t1 WHERE a > 10 ORDER BY a; +--disable_query_log +eval SELECT IF(INSTR(@@gtid_binlog_pos, '$saved_gtid'), "Binlog pos ok", CONCAT("Unexpected binlog pos: ", @@gtid_binlog_pos, "; does not contain the GTID $saved_gtid.")) AS gtid_check; +--enable_query_log + +--echo # Wait 60 seconds for SQL thread to catch up with IO thread +--connection server_2 +--let $wait_timeout= 600 +while ($wait_timeout != 0) +{ + --let $read_log_pos= query_get_value('SHOW SLAVE STATUS', Read_Master_Log_Pos, 1) + --let $exec_log_pos= query_get_value('SHOW SLAVE STATUS', Exec_Master_Log_Pos, 1) + if ($read_log_pos == $exec_log_pos) + { + --let $wait_timeout= 0 + } + if ($read_log_pos != $exec_log_pos) + { + --sleep 0.1 + --dec $wait_timeout + } +} +if ($read_log_pos != $exec_log_pos) +{ + --die Timeout wait for SQL thread to catch up with IO thread +} + +SELECT * from t1 WHERE a > 10 ORDER BY a; +--disable_query_log +eval SELECT IF(INSTR(@@gtid_binlog_pos, '$saved_gtid'), "Binlog pos ok", CONCAT("Unexpected binlog pos: ", @@gtid_binlog_pos, "; does not contain the GTID $saved_gtid.")) AS gtid_check; +eval SELECT IF(INSTR(@@gtid_slave_pos, '$saved_gtid'), "Slave pos ok", CONCAT("Unexpected slave pos: ", @@gtid_slave_pos, "; does not contain the GTID $saved_gtid.")) AS gtid_check; +eval SELECT IF(INSTR(@@gtid_current_pos, '$saved_gtid'), "Current pos ok", CONCAT("Unexpected current pos: ", @@gtid_current_pos, "; does not contain the GTID $saved_gtid.")) AS gtid_check; +--enable_query_log + +--echo # Repeat this with additional transactions on the master + +--connection server_1 +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +wait +EOF + +SET GLOBAL debug_dbug="+d,inject_error_writing_xid"; +BEGIN; +INSERT INTO t1 VALUES (12); +--error ER_ERROR_ON_WRITE +COMMIT; +SET GLOBAL debug_dbug="+d,crash_dispatch_command_before"; +--error 2006,2013 +COMMIT; + +--source include/wait_until_disconnected.inc + +# Simulate that we reconnect to a different server (new server_id). +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +restart: --server-id=1 +EOF + +--enable_reconnect +--source include/wait_until_connected_again.inc + +SELECT @@GLOBAL.server_id; +INSERT INTO t1 VALUES (13); +INSERT INTO t1 VALUES (14); +--let $saved_gtid=`SELECT @@last_gtid` +SELECT * from t1 WHERE a > 10 ORDER BY a; +--source include/save_master_gtid.inc + +--connection server_2 +--source include/sync_with_master_gtid.inc +SELECT * from t1 WHERE a > 10 ORDER BY a; + +--connection server_1 +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +wait +EOF + +SET GLOBAL debug_dbug="+d,inject_error_writing_xid"; +BEGIN; +INSERT INTO t1 VALUES (21); +--error ER_ERROR_ON_WRITE +COMMIT; +SET GLOBAL debug_dbug="+d,crash_dispatch_command_before"; +--error 2006,2013 +COMMIT; + +--source include/wait_until_disconnected.inc + +# Simulate that we reconnect to the same server (same server_id). +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +restart +EOF + +--enable_reconnect +--source include/wait_until_connected_again.inc + +SELECT @@GLOBAL.server_id; +SELECT * from t1 WHERE a > 10 ORDER BY a; +--disable_query_log +eval SELECT IF(INSTR(@@gtid_binlog_pos, '$saved_gtid'), "Binlog pos ok", CONCAT("Unexpected binlog pos: ", @@gtid_binlog_pos, "; does not contain the GTID $saved_gtid.")) AS gtid_check; +eval SELECT IF(INSTR(@@gtid_current_pos, '$saved_gtid'), "Current pos ok", CONCAT("Unexpected current pos: ", @@gtid_current_pos, "; does not contain the GTID $saved_gtid.")) AS gtid_check; +--enable_query_log + +--echo # Wait 60 seconds for SQL thread to catch up with IO thread +--connection server_2 +--let $wait_timeout= 600 +while ($wait_timeout != 0) +{ + --let $read_log_pos= query_get_value('SHOW SLAVE STATUS', Read_Master_Log_Pos, 1) + --let $exec_log_pos= query_get_value('SHOW SLAVE STATUS', Exec_Master_Log_Pos, 1) + if ($read_log_pos == $exec_log_pos) + { + --let $wait_timeout= 0 + } + if ($read_log_pos != $exec_log_pos) + { + --sleep 0.1 + --dec $wait_timeout + } +} +if ($read_log_pos != $exec_log_pos) +{ + --die Timeout wait for SQL thread to catch up with IO thread +} + +SELECT * from t1 WHERE a > 10 ORDER BY a; +--disable_query_log +eval SELECT IF(INSTR(@@gtid_binlog_pos, '$saved_gtid'), "Binlog pos ok", CONCAT("Unexpected binlog pos: ", @@gtid_binlog_pos, "; does not contain the GTID $saved_gtid.")) AS gtid_check; +eval SELECT IF(INSTR(@@gtid_slave_pos, '$saved_gtid'), "Slave pos ok", CONCAT("Unexpected slave pos: ", @@gtid_slave_pos, "; does not contain the GTID $saved_gtid.")) AS gtid_check; +eval SELECT IF(INSTR(@@gtid_current_pos, '$saved_gtid'), "Current pos ok", CONCAT("Unexpected current pos: ", @@gtid_current_pos, "; does not contain the GTID $saved_gtid.")) AS gtid_check; +--enable_query_log + +--echo # Repeat this with additional transactions on the master + +--connection server_1 +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +wait +EOF + +SET GLOBAL debug_dbug="+d,inject_error_writing_xid"; +BEGIN; +INSERT INTO t1 VALUES (22); +--error ER_ERROR_ON_WRITE +COMMIT; +SET GLOBAL debug_dbug="+d,crash_dispatch_command_before"; +--error 2006,2013 +COMMIT; + +--source include/wait_until_disconnected.inc + +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +restart +EOF + +--enable_reconnect +--source include/wait_until_connected_again.inc + +INSERT INTO t1 VALUES (23); +INSERT INTO t1 VALUES (24); +--let $saved_gtid=`SELECT @@last_gtid` +SELECT * from t1 WHERE a > 10 ORDER BY a; +--source include/save_master_gtid.inc + +--connection server_2 +--source include/sync_with_master_gtid.inc +SELECT * from t1 WHERE a > 10 ORDER BY a; + +--echo # Repeat this with slave restart + +--connection server_1 +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +wait +EOF + +SET GLOBAL debug_dbug="+d,inject_error_writing_xid"; +BEGIN; +INSERT INTO t1 VALUES (25); +--error ER_ERROR_ON_WRITE +COMMIT; +SET GLOBAL debug_dbug="+d,crash_dispatch_command_before"; +--error 2006,2013 +COMMIT; + +--source include/wait_until_disconnected.inc + +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +restart +EOF + +--connection server_1 +--enable_reconnect +--source include/wait_until_connected_again.inc + +--connection server_2 +--echo # Wait 60 seconds for IO thread to connect and SQL thread to catch up +--echo # with IO thread. +--let $wait_timeout= 600 +while ($wait_timeout != 0) +{ + --let $connected=`SELECT COUNT(*) > 0 FROM information_schema.processlist WHERE State = 'Waiting for master to send event'` + if ($connected) + { + --let $read_log_pos= query_get_value('SHOW SLAVE STATUS', Read_Master_Log_Pos, 1) + --let $exec_log_pos= query_get_value('SHOW SLAVE STATUS', Exec_Master_Log_Pos, 1) + if ($read_log_pos == $exec_log_pos) + { + --let $wait_timeout= 0 + } + if ($read_log_pos != $exec_log_pos) + { + --sleep 0.1 + --dec $wait_timeout + } + } + if (!$connected) + { + --sleep 0.1 + --dec $wait_timeout + } +} +if (`SELECT NOT $connected OR $read_log_pos != $exec_log_pos`) +{ + --die Timeout wait for IO thread to connect and SQL thread to catch up with IO thread +} + +--source include/stop_slave.inc + +--connection server_1 +--disable_query_log +eval SELECT IF(INSTR(@@gtid_binlog_pos, '$saved_gtid'), "Binlog pos ok", CONCAT("Unexpected binlog pos: ", @@gtid_binlog_pos, "; does not contain the GTID $saved_gtid.")) AS gtid_check; +eval SELECT IF(INSTR(@@gtid_current_pos, '$saved_gtid'), "Current pos ok", CONCAT("Unexpected current pos: ", @@gtid_current_pos, "; does not contain the GTID $saved_gtid.")) AS gtid_check; +--enable_query_log +INSERT INTO t1 VALUES (26); +INSERT INTO t1 VALUES (27); +SELECT * from t1 WHERE a > 10 ORDER BY a; +--source include/save_master_gtid.inc + +--connection server_2 +--disable_query_log +eval SELECT IF(INSTR(@@gtid_binlog_pos, '$saved_gtid'), "Binlog pos ok", CONCAT("Unexpected binlog pos: ", @@gtid_binlog_pos, "; does not contain the GTID $saved_gtid.")) AS gtid_check; +eval SELECT IF(INSTR(@@gtid_slave_pos, '$saved_gtid'), "Slave pos ok", CONCAT("Unexpected slave pos: ", @@gtid_slave_pos, "; does not contain the GTID $saved_gtid.")) AS gtid_check; +eval SELECT IF(INSTR(@@gtid_current_pos, '$saved_gtid'), "Current pos ok", CONCAT("Unexpected current pos: ", @@gtid_current_pos, "; does not contain the GTID $saved_gtid.")) AS gtid_check; +--enable_query_log +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +SELECT * from t1 WHERE a > 10 ORDER BY a; + + +--echo *** MDEV-6391: GTID binlog state not recovered if mariadb-bin.state is removed *** + +--connection server_2 +--source include/stop_slave.inc + +# Do some misc. transactions, stop the master, drop the master-bin.state file. +# Start the master back up, check that binlog state is correct. + +--connection server_1 + +INSERT INTO t1 VALUES (30); +SET @old_server_id= @@server_id; +SET @old_domain_id= @@gtid_domain_id; + +SET SESSION server_id= 10; +INSERT INTO t1 VALUES (31); +INSERT INTO t1 VALUES (32); +SET SESSION gtid_domain_id= 1; +SET SESSION server_id=11; +INSERT INTO t1 VALUES (33); +SET SESSION gtid_domain_id= 2; +INSERT INTO t1 VALUES (34); +SET SESSION server_id= 10; +INSERT INTO t1 VALUES (35); +INSERT INTO t1 VALUES (36); +SET SESSION gtid_domain_id= 0; +SET SESSION server_id= 12; +INSERT INTO t1 VALUES (37); +SET SESSION gtid_domain_id= @old_domain_id; +SET SESSION server_id= @old_server_id; +INSERT INTO t1 VALUES (38); +INSERT INTO t1 VALUES (39); +SELECT * FROM t1 WHERE a >= 30 ORDER BY a; +--source include/save_master_gtid.inc + +--let OLD_STATE= `SELECT @@gtid_binlog_state` + +--let $datadir= `SELECT @@datadir` + +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +wait +EOF +shutdown_server; +--source include/wait_until_disconnected.inc + +--remove_file $datadir/master-bin.state + +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +restart +EOF +--enable_reconnect +--source include/wait_until_connected_again.inc + +--let NEW_STATE= `SELECT @@gtid_binlog_state` + +--perl +my $old= $ENV{'OLD_STATE'}; +my $new= $ENV{'NEW_STATE'}; +# Make them order-independent, for easy comparison. +$old= join(",", sort(split(",", $old))); +$new= join(",", sort(split(",", $new))); +die "ERROR: new binlog state '$new' differs from old '$old'\n" + unless $old eq $new; +EOF + +--connection server_2 +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 WHERE a >= 30 ORDER BY a; + + +--connection server_1 +DROP TABLE t1; + +--connection default +--enable_reconnect +--source include/wait_until_connected_again.inc + +--source include/rpl_end.inc |