diff options
Diffstat (limited to 'contrib/amcheck/t')
| -rw-r--r-- | contrib/amcheck/t/001_verify_heapam.pl | 286 | ||||
| -rw-r--r-- | contrib/amcheck/t/002_cic.pl | 64 | ||||
| -rw-r--r-- | contrib/amcheck/t/003_cic_2pc.pl | 187 | ||||
| -rw-r--r-- | contrib/amcheck/t/005_pitr.pl | 83 |
4 files changed, 620 insertions, 0 deletions
diff --git a/contrib/amcheck/t/001_verify_heapam.pl b/contrib/amcheck/t/001_verify_heapam.pl new file mode 100644 index 0000000..019eed3 --- /dev/null +++ b/contrib/amcheck/t/001_verify_heapam.pl @@ -0,0 +1,286 @@ + +# Copyright (c) 2021-2022, PostgreSQL Global Development Group + +use strict; +use warnings; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; + +use Test::More; + +my ($node, $result); + +# +# Test set-up +# +$node = PostgreSQL::Test::Cluster->new('test'); +$node->init; +$node->append_conf('postgresql.conf', 'autovacuum=off'); +$node->start; +$node->safe_psql('postgres', q(CREATE EXTENSION amcheck)); + +# +# Check a table with data loaded but no corruption, freezing, etc. +# +fresh_test_table('test'); +check_all_options_uncorrupted('test', 'plain'); + +# +# Check a corrupt table +# +fresh_test_table('test'); +corrupt_first_page('test'); +detects_heap_corruption("verify_heapam('test')", "plain corrupted table"); +detects_heap_corruption( + "verify_heapam('test', skip := 'all-visible')", + "plain corrupted table skipping all-visible"); +detects_heap_corruption( + "verify_heapam('test', skip := 'all-frozen')", + "plain corrupted table skipping all-frozen"); +detects_heap_corruption( + "verify_heapam('test', check_toast := false)", + "plain corrupted table skipping toast"); +detects_heap_corruption( + "verify_heapam('test', startblock := 0, endblock := 0)", + "plain corrupted table checking only block zero"); + +# +# Check a corrupt table with all-frozen data +# +fresh_test_table('test'); +$node->safe_psql('postgres', q(VACUUM (FREEZE, DISABLE_PAGE_SKIPPING) test)); +detects_no_corruption("verify_heapam('test')", + "all-frozen not corrupted table"); +corrupt_first_page('test'); +detects_heap_corruption("verify_heapam('test')", + "all-frozen corrupted table"); +detects_no_corruption( + "verify_heapam('test', skip := 'all-frozen')", + "all-frozen corrupted table skipping all-frozen"); + +# +# Check a sequence with no corruption. The current implementation of sequences +# doesn't require its own test setup, since sequences are really just heap +# tables under-the-hood. To guard against future implementation changes made +# without remembering to update verify_heapam, we create and exercise a +# sequence, checking along the way that it passes corruption checks. +# +fresh_test_sequence('test_seq'); +check_all_options_uncorrupted('test_seq', 'plain'); +advance_test_sequence('test_seq'); +check_all_options_uncorrupted('test_seq', 'plain'); +set_test_sequence('test_seq'); +check_all_options_uncorrupted('test_seq', 'plain'); +reset_test_sequence('test_seq'); +check_all_options_uncorrupted('test_seq', 'plain'); + +# Returns the filesystem path for the named relation. +sub relation_filepath +{ + my ($relname) = @_; + + my $pgdata = $node->data_dir; + my $rel = $node->safe_psql('postgres', + qq(SELECT pg_relation_filepath('$relname'))); + die "path not found for relation $relname" unless defined $rel; + return "$pgdata/$rel"; +} + +# Returns the fully qualified name of the toast table for the named relation +sub get_toast_for +{ + my ($relname) = @_; + + return $node->safe_psql( + 'postgres', qq( + SELECT 'pg_toast.' || t.relname + FROM pg_catalog.pg_class c, pg_catalog.pg_class t + WHERE c.relname = '$relname' + AND c.reltoastrelid = t.oid)); +} + +# (Re)create and populate a test table of the given name. +sub fresh_test_table +{ + my ($relname) = @_; + + return $node->safe_psql( + 'postgres', qq( + DROP TABLE IF EXISTS $relname CASCADE; + CREATE TABLE $relname (a integer, b text); + ALTER TABLE $relname SET (autovacuum_enabled=false); + ALTER TABLE $relname ALTER b SET STORAGE external; + INSERT INTO $relname (a, b) + (SELECT gs, repeat('b',gs*10) FROM generate_series(1,1000) gs); + BEGIN; + SAVEPOINT s1; + SELECT 1 FROM $relname WHERE a = 42 FOR UPDATE; + UPDATE $relname SET b = b WHERE a = 42; + RELEASE s1; + SAVEPOINT s1; + SELECT 1 FROM $relname WHERE a = 42 FOR UPDATE; + UPDATE $relname SET b = b WHERE a = 42; + COMMIT; + )); +} + +# Create a test sequence of the given name. +sub fresh_test_sequence +{ + my ($seqname) = @_; + + return $node->safe_psql( + 'postgres', qq( + DROP SEQUENCE IF EXISTS $seqname CASCADE; + CREATE SEQUENCE $seqname + INCREMENT BY 13 + MINVALUE 17 + START WITH 23; + SELECT nextval('$seqname'); + SELECT setval('$seqname', currval('$seqname') + nextval('$seqname')); + )); +} + +# Call SQL functions to increment the sequence +sub advance_test_sequence +{ + my ($seqname) = @_; + + return $node->safe_psql( + 'postgres', qq( + SELECT nextval('$seqname'); + )); +} + +# Call SQL functions to set the sequence +sub set_test_sequence +{ + my ($seqname) = @_; + + return $node->safe_psql( + 'postgres', qq( + SELECT setval('$seqname', 102); + )); +} + +# Call SQL functions to reset the sequence +sub reset_test_sequence +{ + my ($seqname) = @_; + + return $node->safe_psql( + 'postgres', qq( + ALTER SEQUENCE $seqname RESTART WITH 51 + )); +} + +# Stops the test node, corrupts the first page of the named relation, and +# restarts the node. +sub corrupt_first_page +{ + my ($relname) = @_; + my $relpath = relation_filepath($relname); + + $node->stop; + + my $fh; + open($fh, '+<', $relpath) + or BAIL_OUT("open failed: $!"); + binmode $fh; + + # Corrupt some line pointers. The values are chosen to hit the + # various line-pointer-corruption checks in verify_heapam.c + # on both little-endian and big-endian architectures. + sysseek($fh, 32, 0) + or BAIL_OUT("sysseek failed: $!"); + syswrite( + $fh, + pack("L*", + 0xAAA15550, 0xAAA0D550, 0x00010000, + 0x00008000, 0x0000800F, 0x001e8000) + ) or BAIL_OUT("syswrite failed: $!"); + close($fh) + or BAIL_OUT("close failed: $!"); + + $node->start; +} + +sub detects_heap_corruption +{ + local $Test::Builder::Level = $Test::Builder::Level + 1; + + my ($function, $testname) = @_; + + detects_corruption( + $function, + $testname, + qr/line pointer redirection to item at offset \d+ precedes minimum offset \d+/, + qr/line pointer redirection to item at offset \d+ exceeds maximum offset \d+/, + qr/line pointer to page offset \d+ is not maximally aligned/, + qr/line pointer length \d+ is less than the minimum tuple header size \d+/, + qr/line pointer to page offset \d+ with length \d+ ends beyond maximum page offset \d+/, + ); +} + +sub detects_corruption +{ + local $Test::Builder::Level = $Test::Builder::Level + 1; + + my ($function, $testname, @re) = @_; + + my $result = $node->safe_psql('postgres', qq(SELECT * FROM $function)); + like($result, $_, $testname) for (@re); +} + +sub detects_no_corruption +{ + local $Test::Builder::Level = $Test::Builder::Level + 1; + + my ($function, $testname) = @_; + + my $result = $node->safe_psql('postgres', qq(SELECT * FROM $function)); + is($result, '', $testname); +} + +# Check various options are stable (don't abort) and do not report corruption +# when running verify_heapam on an uncorrupted test table. +# +# The relname *must* be an uncorrupted table, or this will fail. +# +# The prefix is used to identify the test, along with the options, +# and should be unique. +sub check_all_options_uncorrupted +{ + local $Test::Builder::Level = $Test::Builder::Level + 1; + + my ($relname, $prefix) = @_; + + for my $stop (qw(true false)) + { + for my $check_toast (qw(true false)) + { + for my $skip ("'none'", "'all-frozen'", "'all-visible'") + { + for my $startblock (qw(NULL 0)) + { + for my $endblock (qw(NULL 0)) + { + my $opts = + "on_error_stop := $stop, " + . "check_toast := $check_toast, " + . "skip := $skip, " + . "startblock := $startblock, " + . "endblock := $endblock"; + + detects_no_corruption( + "verify_heapam('$relname', $opts)", + "$prefix: $opts"); + } + } + } + } + } +} + +done_testing(); diff --git a/contrib/amcheck/t/002_cic.pl b/contrib/amcheck/t/002_cic.pl new file mode 100644 index 0000000..32e4e4a --- /dev/null +++ b/contrib/amcheck/t/002_cic.pl @@ -0,0 +1,64 @@ + +# Copyright (c) 2021-2022, PostgreSQL Global Development Group + +# Test CREATE INDEX CONCURRENTLY with concurrent modifications +use strict; +use warnings; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; + +use Test::More; + +my ($node, $result); + +# +# Test set-up +# +$node = PostgreSQL::Test::Cluster->new('CIC_test'); +$node->init; +$node->append_conf('postgresql.conf', + 'lock_timeout = ' . (1000 * $PostgreSQL::Test::Utils::timeout_default)); +$node->start; +$node->safe_psql('postgres', q(CREATE EXTENSION amcheck)); +$node->safe_psql('postgres', q(CREATE TABLE tbl(i int))); +$node->safe_psql('postgres', q(CREATE INDEX idx ON tbl(i))); + +# +# Stress CIC with pgbench. +# +# pgbench might try to launch more than one instance of the CIC +# transaction concurrently. That would deadlock, so use an advisory +# lock to ensure only one CIC runs at a time. +# +$node->pgbench( + '--no-vacuum --client=5 --transactions=100', + 0, + [qr{actually processed}], + [qr{^$}], + 'concurrent INSERTs and CIC', + { + '002_pgbench_concurrent_transaction' => q( + BEGIN; + INSERT INTO tbl VALUES(0); + COMMIT; + ), + '002_pgbench_concurrent_transaction_savepoints' => q( + BEGIN; + SAVEPOINT s1; + INSERT INTO tbl VALUES(0); + COMMIT; + ), + '002_pgbench_concurrent_cic' => q( + SELECT pg_try_advisory_lock(42)::integer AS gotlock \gset + \if :gotlock + DROP INDEX CONCURRENTLY idx; + CREATE INDEX CONCURRENTLY idx ON tbl(i); + SELECT bt_index_check('idx',true); + SELECT pg_advisory_unlock(42); + \endif + ) + }); + +$node->stop; +done_testing(); diff --git a/contrib/amcheck/t/003_cic_2pc.pl b/contrib/amcheck/t/003_cic_2pc.pl new file mode 100644 index 0000000..1a2ccea --- /dev/null +++ b/contrib/amcheck/t/003_cic_2pc.pl @@ -0,0 +1,187 @@ + +# Copyright (c) 2021-2022, PostgreSQL Global Development Group + +# Test CREATE INDEX CONCURRENTLY with concurrent prepared-xact modifications +use strict; +use warnings; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; + +use Test::More; + +Test::More->builder->todo_start('filesystem bug') + if PostgreSQL::Test::Utils::has_wal_read_bug; + +my ($node, $result); + +# +# Test set-up +# +$node = PostgreSQL::Test::Cluster->new('CIC_2PC_test'); +$node->init; +$node->append_conf('postgresql.conf', 'max_prepared_transactions = 10'); +$node->append_conf('postgresql.conf', + 'lock_timeout = ' . (1000 * $PostgreSQL::Test::Utils::timeout_default)); +$node->start; +$node->safe_psql('postgres', q(CREATE EXTENSION amcheck)); +$node->safe_psql('postgres', q(CREATE TABLE tbl(i int))); + + +# +# Run 3 overlapping 2PC transactions with CIC +# +# We have two concurrent background psql processes: $main_h for INSERTs and +# $cic_h for CIC. Also, we use non-background psql for some COMMIT PREPARED +# statements. +# + +my $main_in = ''; +my $main_out = ''; +my $main_timer = IPC::Run::timeout($PostgreSQL::Test::Utils::timeout_default); + +my $main_h = + $node->background_psql('postgres', \$main_in, \$main_out, + $main_timer, on_error_stop => 1); +$main_in .= q( +BEGIN; +INSERT INTO tbl VALUES(0); +\echo syncpoint1 +); +pump $main_h until $main_out =~ /syncpoint1/ || $main_timer->is_expired; + +my $cic_in = ''; +my $cic_out = ''; +my $cic_timer = IPC::Run::timeout($PostgreSQL::Test::Utils::timeout_default); +my $cic_h = + $node->background_psql('postgres', \$cic_in, \$cic_out, + $cic_timer, on_error_stop => 1); +$cic_in .= q( +\echo start +CREATE INDEX CONCURRENTLY idx ON tbl(i); +); +pump $cic_h until $cic_out =~ /start/ || $cic_timer->is_expired; + +$main_in .= q( +PREPARE TRANSACTION 'a'; +); + +$main_in .= q( +BEGIN; +INSERT INTO tbl VALUES(0); +\echo syncpoint2 +); +pump $main_h until $main_out =~ /syncpoint2/ || $main_timer->is_expired; + +$node->safe_psql('postgres', q(COMMIT PREPARED 'a';)); + +$main_in .= q( +PREPARE TRANSACTION 'b'; +BEGIN; +INSERT INTO tbl VALUES(0); +\echo syncpoint3 +); +pump $main_h until $main_out =~ /syncpoint3/ || $main_timer->is_expired; + +$node->safe_psql('postgres', q(COMMIT PREPARED 'b';)); + +$main_in .= q( +PREPARE TRANSACTION 'c'; +COMMIT PREPARED 'c'; +); +$main_h->pump_nb; + +$main_h->finish; +$cic_h->finish; + +$result = $node->psql('postgres', q(SELECT bt_index_check('idx',true))); +is($result, '0', 'bt_index_check after overlapping 2PC'); + + +# +# Server restart shall not change whether prepared xact blocks CIC +# + +$node->safe_psql( + 'postgres', q( +BEGIN; +INSERT INTO tbl VALUES(0); +PREPARE TRANSACTION 'spans_restart'; +BEGIN; +CREATE TABLE unused (); +PREPARE TRANSACTION 'persists_forever'; +)); +$node->restart; + +my $reindex_in = ''; +my $reindex_out = ''; +my $reindex_timer = + IPC::Run::timeout($PostgreSQL::Test::Utils::timeout_default); +my $reindex_h = + $node->background_psql('postgres', \$reindex_in, \$reindex_out, + $reindex_timer, on_error_stop => 1); +$reindex_in .= q( +\echo start +DROP INDEX CONCURRENTLY idx; +CREATE INDEX CONCURRENTLY idx ON tbl(i); +); +pump $reindex_h until $reindex_out =~ /start/ || $reindex_timer->is_expired; + +$node->safe_psql('postgres', "COMMIT PREPARED 'spans_restart'"); +$reindex_h->finish; +$result = $node->psql('postgres', q(SELECT bt_index_check('idx',true))); +is($result, '0', 'bt_index_check after 2PC and restart'); + + +# +# Stress CIC+2PC with pgbench +# +# pgbench might try to launch more than one instance of the CIC +# transaction concurrently. That would deadlock, so use an advisory +# lock to ensure only one CIC runs at a time. + +# Fix broken index first +$node->safe_psql('postgres', q(REINDEX TABLE tbl;)); + +# Run pgbench. +$node->pgbench( + '--no-vacuum --client=5 --transactions=100', + 0, + [qr{actually processed}], + [qr{^$}], + 'concurrent INSERTs w/ 2PC and CIC', + { + '003_pgbench_concurrent_2pc' => q( + BEGIN; + INSERT INTO tbl VALUES(0); + PREPARE TRANSACTION 'c:client_id'; + COMMIT PREPARED 'c:client_id'; + ), + '003_pgbench_concurrent_2pc_savepoint' => q( + BEGIN; + SAVEPOINT s1; + INSERT INTO tbl VALUES(0); + PREPARE TRANSACTION 'c:client_id'; + COMMIT PREPARED 'c:client_id'; + ), + '003_pgbench_concurrent_cic' => q( + SELECT pg_try_advisory_lock(42)::integer AS gotlock \gset + \if :gotlock + DROP INDEX CONCURRENTLY idx; + CREATE INDEX CONCURRENTLY idx ON tbl(i); + SELECT bt_index_check('idx',true); + SELECT pg_advisory_unlock(42); + \endif + ), + '004_pgbench_concurrent_ric' => q( + SELECT pg_try_advisory_lock(42)::integer AS gotlock \gset + \if :gotlock + REINDEX INDEX CONCURRENTLY idx; + SELECT bt_index_check('idx',true); + SELECT pg_advisory_unlock(42); + \endif + ) + }); + +$node->stop; +done_testing(); diff --git a/contrib/amcheck/t/005_pitr.pl b/contrib/amcheck/t/005_pitr.pl new file mode 100644 index 0000000..c8c8cf0 --- /dev/null +++ b/contrib/amcheck/t/005_pitr.pl @@ -0,0 +1,83 @@ +# Copyright (c) 2021-2023, PostgreSQL Global Development Group + +# Test integrity of intermediate states by PITR to those states +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# origin node: generate WAL records of interest. +my $origin = PostgreSQL::Test::Cluster->new('origin'); +$origin->init(has_archiving => 1, allows_streaming => 1); +$origin->append_conf('postgresql.conf', 'autovacuum = off'); +$origin->start; +$origin->backup('my_backup'); +# Create a table with each of 6 PK values spanning 1/4 of a block. Delete the +# first four, so one index leaf is eligible for deletion. Make a replication +# slot just so pg_walinspect will always have access to later WAL. +my $setup = <<EOSQL; +BEGIN; +CREATE EXTENSION amcheck; +CREATE EXTENSION pg_walinspect; +CREATE TABLE not_leftmost (c text); +ALTER TABLE not_leftmost ALTER c SET STORAGE PLAIN; +INSERT INTO not_leftmost + SELECT repeat(n::text, database_block_size / 4) + FROM generate_series(1,6) t(n), pg_control_init(); +ALTER TABLE not_leftmost ADD CONSTRAINT not_leftmost_pk PRIMARY KEY (c); +DELETE FROM not_leftmost WHERE c ~ '^[1-4]'; +SELECT pg_create_physical_replication_slot('for_walinspect', true, false); +COMMIT; +EOSQL +$origin->safe_psql('postgres', $setup); +my $before_vacuum_lsn = + $origin->safe_psql('postgres', "SELECT pg_current_wal_lsn()"); +# VACUUM to delete the aforementioned leaf page. Force an XLogFlush() by +# dropping a permanent table. That way, the XLogReader infrastructure can +# always see VACUUM's records, even under synchronous_commit=off. Finally, +# find the LSN of that VACUUM's last UNLINK_PAGE record. +my $vacuum = <<EOSQL; +SET synchronous_commit = off; +VACUUM (VERBOSE, INDEX_CLEANUP ON) not_leftmost; +CREATE TABLE XLogFlush (); +DROP TABLE XLogFlush; +SELECT max(start_lsn) + FROM pg_get_wal_records_info('$before_vacuum_lsn', pg_current_wal_flush_lsn()) + WHERE resource_manager = 'Btree' AND record_type = 'UNLINK_PAGE'; +EOSQL +my $unlink_lsn = $origin->safe_psql('postgres', $vacuum); +$origin->stop; +die "did not find UNLINK_PAGE record" unless $unlink_lsn; + +# replica node: amcheck at notable points in the WAL stream +my $replica = PostgreSQL::Test::Cluster->new('replica'); +$replica->init_from_backup($origin, 'my_backup', has_restoring => 1); +$replica->append_conf('postgresql.conf', + "recovery_target_lsn = '$unlink_lsn'"); +$replica->append_conf('postgresql.conf', 'recovery_target_inclusive = off'); +$replica->append_conf('postgresql.conf', 'recovery_target_action = promote'); +$replica->start; +$replica->poll_query_until('postgres', "SELECT pg_is_in_recovery() = 'f';") + or die "Timed out while waiting for PITR promotion"; +# recovery done; run amcheck +my $debug = "SET client_min_messages = 'debug1'"; +my ($rc, $stderr); +$rc = $replica->psql( + 'postgres', + "$debug; SELECT bt_index_parent_check('not_leftmost_pk', true)", + stderr => \$stderr); +print STDERR $stderr, "\n"; +is($rc, 0, "bt_index_parent_check passes"); +like( + $stderr, + qr/interrupted page deletion detected/, + "bt_index_parent_check: interrupted page deletion detected"); +$rc = $replica->psql( + 'postgres', + "$debug; SELECT bt_index_check('not_leftmost_pk', true)", + stderr => \$stderr); +print STDERR $stderr, "\n"; +is($rc, 0, "bt_index_check passes"); + +done_testing(); |