summaryrefslogtreecommitdiffstats
path: root/storage/maria/unittest/ma_test_recovery.pl
diff options
context:
space:
mode:
Diffstat (limited to 'storage/maria/unittest/ma_test_recovery.pl')
-rwxr-xr-xstorage/maria/unittest/ma_test_recovery.pl496
1 files changed, 496 insertions, 0 deletions
diff --git a/storage/maria/unittest/ma_test_recovery.pl b/storage/maria/unittest/ma_test_recovery.pl
new file mode 100755
index 00000000..fdc7ee68
--- /dev/null
+++ b/storage/maria/unittest/ma_test_recovery.pl
@@ -0,0 +1,496 @@
+#!/usr/bin/env perl
+
+use Getopt::Long;
+use File::Copy;
+use File::Compare;
+use File::Basename;
+use Digest::MD5;
+
+$|= 1;
+$^W = 1; # warnings, because env cannot parse 'perl -w'
+$VER= "1.2";
+
+$opt_version= 0;
+$opt_help= 0;
+$opt_verbose= 0;
+$opt_abort_on_error=0;
+
+my $silent= "-s";
+my $maria_path; # path to "storage/maria"
+my $maria_exe_path; # path to executables (ma_test1, aria_chk etc)
+my $tmp= "./tmp";
+my $my_progname= $0;
+my $suffix;
+my $zerofilled_tables= 0;
+
+$my_progname=~ s/.*[\/]//;
+$maria_path= dirname($0) . "/..";
+
+main();
+
+####
+#### main function
+####
+
+sub main
+{
+ my ($res, $table);
+
+ if (!GetOptions("abort-on-error", "help", "version", "verbose"))
+ {
+ $flag_exit= 1;
+ }
+ if ($opt_version)
+ {
+ print "$my_progname version $VER\n";
+ exit(0);
+ }
+ usage() if ($opt_help || $flag_exit);
+
+ $suffix= ( $^O =~ /win/i && $^O !~ /darwin/i ) ? ".exe" : "";
+ $maria_exe_path= "$maria_path/release";
+ # we use -f, sometimes -x is unexpectedly false in Cygwin
+ if ( ! -f "$maria_exe_path/ma_test1$suffix" )
+ {
+ $maria_exe_path= "$maria_path/relwithdebinfo";
+ if ( ! -f "$maria_exe_path/ma_test1$suffix" )
+ {
+ $maria_exe_path= "$maria_path/debug";
+ if ( ! -f "$maria_exe_path/ma_test1$suffix" )
+ {
+ $maria_exe_path= $maria_path;
+ if ( ! -f "$maria_exe_path/ma_test1$suffix" )
+ {
+ die("Cannot find ma_test1 executable\n");
+ }
+ }
+ }
+ }
+
+ # test data is always put in the current directory or a tmp subdirectory
+ # of it
+
+ if (! -d "$tmp")
+ {
+ mkdir $tmp;
+ }
+ print "ARIA RECOVERY TESTS\n";
+
+ # To not flood the screen, we redirect all the commands below to a text file
+ # and just give a final error if their output is not as expected
+
+ open (MY_LOG, ">$tmp/ma_test_recovery.output") or die "Can't open log file\n";
+ print MY_LOG "Testing the REDO PHASE ALONE\n";
+
+ # runs a program inserting/deleting rows, then moves the resulting table
+ # elsewhere; applies the log and checks that the data file is
+ # identical to the saved original.
+
+ my @t= ("ma_test1$suffix $silent -M -T -c",
+ "ma_test2$suffix $silent -L -K -W -P -M -T -c -d500",
+ "ma_test2$suffix $silent -M -T -c -b65000",
+ "ma_test2$suffix $silent -M -T -c -b65000 -d800",
+ "ma_test1$suffix $silent -M -T -c -C",
+ "ma_test2$suffix $silent -L -K -W -P -M -T -c -d500 -C",
+ #"ma_rt_test$suffix $silent -M -T -c -C",
+ # @todo: also add to @t2
+ );
+
+ foreach my $prog (@t)
+ {
+ unlink <aria_log.* aria_log_control>;
+ my $prog_no_suffix= $prog;
+ $prog_no_suffix=~ s/$suffix// if ($suffix);
+ print MY_LOG "TEST WITH $prog_no_suffix\n";
+ $res= my_exec("$maria_exe_path/$prog");
+ print MY_LOG $res;
+ # derive table's name from program's name
+ if ($prog =~ m/^ma_(\S+)\s.*/)
+ {
+ $table= $1;
+ }
+ else
+ {
+ die("can't guess table name");
+ }
+ $com= "$maria_exe_path/aria_chk$suffix -dvv $table ";
+ $com.= "| grep -v \"Creation time:\" | grep -v \"recover time:\" | grep -v \"file length\" | grep -v \"LSNs:\" | grep -v \"UUID:\"";
+ $com.= "> $tmp/aria_chk_message.good.txt 2>&1";
+ my_exec($com);
+ my $checksum= my_exec("$maria_exe_path/aria_chk$suffix -dss $table");
+ move("$table.MAD", "$tmp/$table-good.MAD") ||
+ die "Can't move $table.MAD to $tmp/$table-good.MAD\n";
+ move("$table.MAI", "$tmp/$table-good.MAI") ||
+ die "Can't move $table.MAI to $tmp/$table-good.MAI\n";
+ apply_log($table, "shouldnotchangelog");
+ check_table_is_same($table, $checksum);
+ $res= physical_cmp($table, "$tmp/$table-good");
+ print MY_LOG $res;
+ print MY_LOG "testing idempotency\n";
+ apply_log($table, "shouldnotchangelog");
+ check_table_is_same($table, $checksum);
+ $res= physical_cmp($table, "$tmp/$table-good");
+ print MY_LOG $res;
+ }
+
+ print MY_LOG "Testing the REDO AND UNDO PHASE\n";
+ # The test programs look like:
+ # work; commit (time T1); work; exit-without-commit (time T2)
+ # We first run the test program and let it exit after T1's commit.
+ # Then we run it again and let it exit at T2. Then we compare
+ # and expect identity.
+
+ my @take_checkpoints= ("no", "yes");
+ my @blobs= ("", "-b32768");
+ my @test_undo= (1, 2, 3, 4);
+ my @t2= ("ma_test1$suffix $silent -M -T -c -N blob -H1",
+ "--testflag=1",
+ "--testflag=2 --test-undo=",
+ "ma_test1$suffix $silent -M -T -c -N blob -H2",
+ "--testflag=3",
+ "--testflag=4 --test-undo=",
+ "ma_test1$suffix $silent -M -T -c -N blob -H2 --versioning",
+ "--testflag=3",
+ "--testflag=4 --test-undo=",
+ "ma_test1$suffix $silent -M -T -c -N blob -H2",
+ "--testflag=2",
+ "--testflag=3 --test-undo=",
+ "ma_test2$suffix $silent -L -K -W -P -M -T -c blob -H1",
+ "-t1",
+ "-t2 -A",
+ "ma_test2$suffix $silent -L -K -W -P -M -T -c blob -H1",
+ "-t1",
+ "-t6 -A");
+
+ foreach my $take_checkpoint (@take_checkpoints)
+ {
+ my ($i, $j, $k, $commit_run_args, $abort_run_args);
+ # we test table without blobs and then table with blobs
+ for ($i= 0; defined($blobs[$i]); $i++)
+ {
+ for ($j= 0; defined($test_undo[$j]); $j++)
+ {
+ # first iteration tests rollback of insert, second tests rollback of delete
+ # -N (create NULL fields) is needed because --test-undo adds it anyway
+ for ($k= 0; defined($t2[$k]); $k+= 3)
+ {
+ $prog= $t2[$k];
+ $prog=~ s/blob/$blobs[$i]/;
+ if ("$take_checkpoint" eq "no") {
+ $prog=~ s/\s+\-H[0-9]+//;
+ }
+ $commit_run_args= $t2[$k + 1];
+ $abort_run_args= $t2[$k + 2];
+ unlink <aria_log.* aria_log_control>;
+ my $prog_no_suffix= $prog;
+ $prog_no_suffix=~ s/$suffix// if ($suffix);
+ print MY_LOG "TEST WITH $prog_no_suffix $commit_run_args (commit at end)\n";
+ $res= my_exec("$maria_exe_path/$prog $commit_run_args");
+ print MY_LOG $res;
+ # derive table's name from program's name
+ if ($prog =~ m/^ma_(\S+)\s.*/)
+ {
+ $table= $1;
+ }
+ else
+ {
+ die("can't guess table name");
+ }
+ $com= "$maria_exe_path/aria_chk$suffix -dvv $table ";
+ $com.= "| grep -v \"Creation time:\" | grep -v \"recover time:\" | grep -v \"recover time:\" |grep -v \"file length\" | grep -v \"LSNs:\" | grep -v \"UUID:\" ";
+ $com.= "> $tmp/aria_chk_message.good.txt 2>&1";
+ $res= my_exec($com);
+ print MY_LOG $res;
+ $checksum= my_exec("$maria_exe_path/aria_chk$suffix -dss $table");
+ move("$table.MAD", "$tmp/$table-good.MAD") ||
+ die "Can't move $table.MAD to $tmp/$table-good.MAD\n";
+ move("$table.MAI", "$tmp/$table-good.MAI") ||
+ die "Can't move $table.MAI to $tmp/$table-good.MAI\n";
+ unlink <aria_log.* aria_log_control>;
+ print MY_LOG "TEST WITH $prog_no_suffix $abort_run_args$test_undo[$j] (additional aborted work)\n";
+ $res= my_exec("$maria_exe_path/$prog $abort_run_args$test_undo[$j]");
+ print MY_LOG $res;
+ copy("$table.MAD", "$tmp/$table-before_undo.MAD") ||
+ die "Can't copy $table.MAD to $tmp/$table-before_undo.MAD\n";
+ copy("$table.MAI", "$tmp/$table-before_undo.MAI") ||
+ die "Can't copy $table.MAI to $tmp/$table-before_undo.MAI\n";
+
+ # The lines below seem unneeded, will be removed soon
+ # We have to copy and restore logs, as running aria_read_log will
+ # change the aria_control_file
+ # rm -f $tmp/aria_log.* $tmp/aria_log_control
+ # cp $maria_path/aria_log* $tmp
+
+ if ($test_undo[$j] != 3) {
+ apply_log($table, "shouldchangelog"); # should undo aborted work
+ } else {
+ # probably nothing to undo went to log or data file
+ apply_log($table, "dontknow");
+ }
+ copy("$table.MAD", "$tmp/$table-after_undo.MAD") ||
+ die "Can't copy $table.MAD to $tmp/$table-after_undo.MAD\n";
+ copy("$table.MAI", "$tmp/$table-after_undo.MAI") ||
+ die "Can't copy $table.MAI to $tmp/$table-after_undo.MAI\n";
+
+ # It is impossible to do a "cmp" between .good and .after_undo,
+ # because the UNDO phase generated log
+ # records whose LSN tagged pages. Another reason is that rolling back
+ # INSERT only marks the rows free, does not empty them
+ # (optimization), so traces of the INSERT+rollback remain.
+
+ check_table_is_same($table, $checksum);
+ print MY_LOG "testing idempotency\n";
+ apply_log($table, "shouldnotchangelog");
+ check_table_is_same($table, $checksum);
+ $res= physical_cmp($table, "$tmp/$table-after_undo");
+ print MY_LOG $res;
+ print MY_LOG "testing applying of CLRs to recreate table\n";
+ unlink <$table.MA?>;
+ # cp $tmp/aria_log* $maria_path #unneeded
+ apply_log($table, "shouldnotchangelog");
+ check_table_is_same($table, $checksum);
+ $res= physical_cmp($table, "$tmp/$table-after_undo");
+ print MY_LOG $res;
+ }
+ unlink <$table.* $tmp/$table* $tmp/aria_chk_*.txt $tmp/aria_read_log_$table.txt>;
+ }
+ }
+ }
+
+ if ($? >> 8) {
+ print "Some test failed\n";
+ exit(1);
+ }
+
+ close(MY_LOG);
+ # also note that aria_chk -dvv shows differences for ma_test2 in UNDO phase,
+ # this is normal: removing records does not shrink the data/key file,
+ # does not put back the "analyzed,optimized keys"(etc) index state.
+ `diff -b $maria_path/unittest/ma_test_recovery.expected $tmp/ma_test_recovery.output`;
+ if ($? >> 8) {
+ print "UNEXPECTED OUTPUT OF TESTS, FAILED";
+ print " (zerofilled $zerofilled_tables tables)\n";
+ print "For more info, do diff -b $maria_path/unittest/ma_test_recovery.expected ";
+ print "$tmp/ma_test_recovery.output\n";
+ exit(1);
+ }
+ print "ALL RECOVERY TESTS OK (zerofilled $zerofilled_tables tables)\n";
+}
+
+####
+#### check_table_is_same
+####
+
+sub check_table_is_same
+{
+ my ($table, $checksum)= @_;
+ my ($com, $checksum2, $res);
+
+ # Computes checksum of new table and compares to checksum of old table
+ # Shows any difference in table's state (info from the index's header)
+ # Data/key file length is random in ma_test2 (as it uses srand() which
+ # may differ between machines).
+
+ if ($opt_verbose)
+ {
+ print "checking if table $table has changed\n";
+ }
+
+ $com= "$maria_exe_path/aria_chk$suffix -dvv $table | grep -v \"Creation time:\" | grep -v \"recover time:\"";
+ $com.= "| grep -v \"file length\" | grep -v \"LSNs:\" | grep -v \"UUID:\" > $tmp/aria_chk_message.txt 2>&1";
+ $res= my_exec2($com);
+ print MY_LOG $res;
+ $res= my_exec2("$maria_exe_path/aria_chk$suffix -ss -e --read-only $table");
+ print MY_LOG $res;
+ $checksum2= my_exec2("$maria_exe_path/aria_chk$suffix -dss $table");
+ if ("$checksum" ne "$checksum2")
+ {
+ print MY_LOG "checksum differs for $table before and after recovery\n";
+ return 1;
+ }
+
+ $com= "diff $tmp/aria_chk_message.good.txt $tmp/aria_chk_message.txt ";
+ $com.= "> $tmp/aria_chk_diff.txt || true";
+ $res= my_exec2($com);
+ print MY_LOG $res;
+
+ if (-s "$tmp/aria_chk_diff.txt")
+ {
+ print MY_LOG "Differences in aria_chk -dvv, recovery not yet perfect !\n";
+ print MY_LOG "========DIFF START=======\n";
+ open(MY_FILE, "<$tmp/aria_chk_diff.txt") || die "Can't open file aria_chk_diff.txt\n";
+ while (<MY_FILE>)
+ {
+ print MY_LOG $_;
+ }
+ close(MY_FILE);
+ print MY_LOG "========DIFF END=======\n";
+ }
+}
+
+####
+#### apply_log
+####
+
+sub apply_log
+{
+ my ($table, $shouldchangelog)= @_;
+ my ($log_md5, $log_md5_2);
+
+ # applies log, can verify if applying did write to log or not
+
+ if ("$shouldchangelog" ne "shouldnotchangelog" &&
+ "$shouldchangelog" ne "shouldchangelog" &&
+ "$shouldchangelog" ne "dontknow" )
+ {
+ print MY_LOG "bad argument '$shouldchangelog'\n";
+ return 1;
+ }
+ foreach (<aria_log.*>)
+ {
+ $log_md5.= md5_conv($_);
+ }
+ print MY_LOG "applying log\n";
+ my_exec("$maria_exe_path/aria_read_log$suffix -a > $tmp/aria_read_log_$table.txt");
+ foreach (<aria_log.*>)
+ {
+ $log_md5_2.= md5_conv($_);
+ }
+ if ("$log_md5" ne "$log_md5_2" )
+ {
+ if ("$shouldchangelog" eq "shouldnotchangelog")
+ {
+ print MY_LOG "aria_read_log should not have modified the log\n";
+ return 1;
+ }
+ }
+ elsif ("$shouldchangelog" eq "shouldchangelog")
+ {
+ print MY_LOG "aria_read_log should have modified the log\n";
+ return 1;
+ }
+}
+
+####
+#### md5_conv
+####
+
+sub md5_conv
+{
+ my ($file)= @_;
+
+ open(FILE, $file) or die "Can't open '$file': $!\n";
+ binmode(FILE);
+ my $md5= Digest::MD5->new;
+ $md5->addfile(FILE);
+ close (FILE);
+ return $md5->hexdigest . "\n";
+}
+
+####
+#### physical_cmp: compares two tables (MAI and MAD) physically;
+#### uses zerofill-keep-lsn to reduce irrelevant differences.
+####
+
+sub physical_cmp
+{
+ my ($table1, $table2)= @_;
+ my ($zerofilled, $ret_text)= (0, "");
+ #return `cmp $table1.MAD $table2.MAD`.`cmp $table1.MAI $table2.MAI`;
+ foreach my $file_suffix ("MAD", "MAI")
+ {
+ my $file1= "$table1.$file_suffix";
+ my $file2= "$table2.$file_suffix";
+ my $res= File::Compare::compare($file1, $file2);
+ die() if ($res == -1);
+ if ($res == 1 # they differ
+ and !$zerofilled)
+ {
+ # let's try with --zerofill-keep-lsn
+ $zerofilled= 1; # but no need to do it twice
+ $zerofilled_tables= $zerofilled_tables + 1;
+ my $table_no= 1;
+ foreach my $table ($table1, $table2)
+ {
+ # save original tables to restore them later
+ copy("$table.MAD", "$tmp/before_zerofill$table_no.MAD") || die();
+ copy("$table.MAI", "$tmp/before_zerofill$table_no.MAI") || die();
+ $com= "$maria_exe_path/aria_chk$suffix -ss --zerofill-keep-lsn --skip-update-state $table";
+ $res= `$com`;
+ print MY_LOG $res;
+ $table_no= $table_no + 1;
+ }
+ $res= File::Compare::compare($file1, $file2);
+ die() if ($res == -1);
+ }
+ $ret_text.= "$file1 and $file2 differ\n" if ($res != 0);
+ }
+ if ($zerofilled)
+ {
+ my $table_no= 1;
+ foreach my $table ($table1, $table2)
+ {
+ move("$tmp/before_zerofill$table_no.MAD", "$table.MAD") || die();
+ move("$tmp/before_zerofill$table_no.MAI", "$table.MAI") || die();
+ $table_no= $table_no + 1;
+ }
+ }
+ return $ret_text;
+}
+
+
+sub my_exec
+{
+ my($command)= @_;
+ my $res;
+ if ($opt_verbose)
+ {
+ print "$command\n";
+ }
+ $res= `$command`;
+ if ($? != 0 && $opt_abort_on_error)
+ {
+ exit(1);
+ }
+ return $res;
+}
+
+sub my_exec2
+{
+ my($command)= @_;
+ my $res, $err;
+ $res= `$command`;
+ if ($? != 0 && $opt_abort_on_error)
+ {
+ $err= $?;
+ print "$command\n";
+ print "failed with error: $err\n";
+ exit(1);
+ }
+ return $res;
+}
+
+
+####
+#### usage
+####
+
+sub usage
+{
+ print <<EOF;
+$my_progname version $VER
+
+Description:
+
+Run various Aria recovery tests and print the results
+
+Options
+--help Show this help and exit.
+
+--abort-on-error Abort at once in case of error.
+--verbose Show commands while there are executing.
+--version Show version number and exit.
+
+EOF
+ exit(0);
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-08-07 05:19:37 +0000