Adding upstream version 14.5.upstream/14.5upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

3 files changed, 471 insertions, 0 deletions

diff --git a/contrib/amcheck/t/001_verify_heapam.pl b/contrib/amcheck/t/001_verify_heapam.pl
new file mode 100644
index 0000000..64ba64d
--- /dev/null
+++ b/contrib/amcheck/t/001_verify_heapam.pl
@@ -0,0 +1,218 @@
+
+# Copyright (c) 2021, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgresNode;
+use TestLib;
+
+use Test::More tests => 80;
+
+my ($node, $result);
+
+#
+# Test set-up
+#
+$node = get_new_node('test');
+$node->init;
+$node->append_conf('postgresql.conf', 'autovacuum=off');
+$node->start;
+$node->safe_psql('postgres', q(CREATE EXTENSION amcheck));
+
+#
+# Check a table with data loaded but no corruption, freezing, etc.
+#
+fresh_test_table('test');
+check_all_options_uncorrupted('test', 'plain');
+
+#
+# Check a corrupt table
+#
+fresh_test_table('test');
+corrupt_first_page('test');
+detects_heap_corruption("verify_heapam('test')", "plain corrupted table");
+detects_heap_corruption(
+	"verify_heapam('test', skip := 'all-visible')",
+	"plain corrupted table skipping all-visible");
+detects_heap_corruption(
+	"verify_heapam('test', skip := 'all-frozen')",
+	"plain corrupted table skipping all-frozen");
+detects_heap_corruption(
+	"verify_heapam('test', check_toast := false)",
+	"plain corrupted table skipping toast");
+detects_heap_corruption(
+	"verify_heapam('test', startblock := 0, endblock := 0)",
+	"plain corrupted table checking only block zero");
+
+#
+# Check a corrupt table with all-frozen data
+#
+fresh_test_table('test');
+$node->safe_psql('postgres', q(VACUUM (FREEZE, DISABLE_PAGE_SKIPPING) test));
+detects_no_corruption("verify_heapam('test')",
+	"all-frozen not corrupted table");
+corrupt_first_page('test');
+detects_heap_corruption("verify_heapam('test')",
+	"all-frozen corrupted table");
+detects_no_corruption(
+	"verify_heapam('test', skip := 'all-frozen')",
+	"all-frozen corrupted table skipping all-frozen");
+
+# Returns the filesystem path for the named relation.
+sub relation_filepath
+{
+	my ($relname) = @_;
+
+	my $pgdata = $node->data_dir;
+	my $rel    = $node->safe_psql('postgres',
+		qq(SELECT pg_relation_filepath('$relname')));
+	die "path not found for relation $relname" unless defined $rel;
+	return "$pgdata/$rel";
+}
+
+# Returns the fully qualified name of the toast table for the named relation
+sub get_toast_for
+{
+	my ($relname) = @_;
+
+	return $node->safe_psql(
+		'postgres', qq(
+		SELECT 'pg_toast.' || t.relname
+			FROM pg_catalog.pg_class c, pg_catalog.pg_class t
+			WHERE c.relname = '$relname'
+			  AND c.reltoastrelid = t.oid));
+}
+
+# (Re)create and populate a test table of the given name.
+sub fresh_test_table
+{
+	my ($relname) = @_;
+
+	return $node->safe_psql(
+		'postgres', qq(
+		DROP TABLE IF EXISTS $relname CASCADE;
+		CREATE TABLE $relname (a integer, b text);
+		ALTER TABLE $relname SET (autovacuum_enabled=false);
+		ALTER TABLE $relname ALTER b SET STORAGE external;
+		INSERT INTO $relname (a, b)
+			(SELECT gs, repeat('b',gs*10) FROM generate_series(1,1000) gs);
+		BEGIN;
+		SAVEPOINT s1;
+		SELECT 1 FROM $relname WHERE a = 42 FOR UPDATE;
+		UPDATE $relname SET b = b WHERE a = 42;
+		RELEASE s1;
+		SAVEPOINT s1;
+		SELECT 1 FROM $relname WHERE a = 42 FOR UPDATE;
+		UPDATE $relname SET b = b WHERE a = 42;
+		COMMIT;
+	));
+}
+
+# Stops the test node, corrupts the first page of the named relation, and
+# restarts the node.
+sub corrupt_first_page
+{
+	my ($relname) = @_;
+	my $relpath = relation_filepath($relname);
+
+	$node->stop;
+
+	my $fh;
+	open($fh, '+<', $relpath)
+	  or BAIL_OUT("open failed: $!");
+	binmode $fh;
+
+	# Corrupt some line pointers.  The values are chosen to hit the
+	# various line-pointer-corruption checks in verify_heapam.c
+	# on both little-endian and big-endian architectures.
+	sysseek($fh, 32, 0)
+	  or BAIL_OUT("sysseek failed: $!");
+	syswrite(
+		$fh,
+		pack("L*",
+			0xAAA15550, 0xAAA0D550, 0x00010000,
+			0x00008000, 0x0000800F, 0x001e8000)
+	) or BAIL_OUT("syswrite failed: $!");
+	close($fh)
+	  or BAIL_OUT("close failed: $!");
+
+	$node->start;
+}
+
+sub detects_heap_corruption
+{
+	local $Test::Builder::Level = $Test::Builder::Level + 1;
+
+	my ($function, $testname) = @_;
+
+	detects_corruption(
+		$function,
+		$testname,
+		qr/line pointer redirection to item at offset \d+ precedes minimum offset \d+/,
+		qr/line pointer redirection to item at offset \d+ exceeds maximum offset \d+/,
+		qr/line pointer to page offset \d+ is not maximally aligned/,
+		qr/line pointer length \d+ is less than the minimum tuple header size \d+/,
+		qr/line pointer to page offset \d+ with length \d+ ends beyond maximum page offset \d+/,
+	);
+}
+
+sub detects_corruption
+{
+	local $Test::Builder::Level = $Test::Builder::Level + 1;
+
+	my ($function, $testname, @re) = @_;
+
+	my $result = $node->safe_psql('postgres', qq(SELECT * FROM $function));
+	like($result, $_, $testname) for (@re);
+}
+
+sub detects_no_corruption
+{
+	local $Test::Builder::Level = $Test::Builder::Level + 1;
+
+	my ($function, $testname) = @_;
+
+	my $result = $node->safe_psql('postgres', qq(SELECT * FROM $function));
+	is($result, '', $testname);
+}
+
+# Check various options are stable (don't abort) and do not report corruption
+# when running verify_heapam on an uncorrupted test table.
+#
+# The relname *must* be an uncorrupted table, or this will fail.
+#
+# The prefix is used to identify the test, along with the options,
+# and should be unique.
+sub check_all_options_uncorrupted
+{
+	local $Test::Builder::Level = $Test::Builder::Level + 1;
+
+	my ($relname, $prefix) = @_;
+
+	for my $stop (qw(true false))
+	{
+		for my $check_toast (qw(true false))
+		{
+			for my $skip ("'none'", "'all-frozen'", "'all-visible'")
+			{
+				for my $startblock (qw(NULL 0))
+				{
+					for my $endblock (qw(NULL 0))
+					{
+						my $opts =
+						    "on_error_stop := $stop, "
+						  . "check_toast := $check_toast, "
+						  . "skip := $skip, "
+						  . "startblock := $startblock, "
+						  . "endblock := $endblock";
+
+						detects_no_corruption(
+							"verify_heapam('$relname', $opts)",
+							"$prefix: $opts");
+					}
+				}
+			}
+		}
+	}
+}
diff --git a/contrib/amcheck/t/002_cic.pl b/contrib/amcheck/t/002_cic.pl
new file mode 100644
index 0000000..3587807
--- /dev/null
+++ b/contrib/amcheck/t/002_cic.pl
@@ -0,0 +1,65 @@
+
+# Copyright (c) 2021, PostgreSQL Global Development Group
+
+# Test CREATE INDEX CONCURRENTLY with concurrent modifications
+use strict;
+use warnings;
+
+use Config;
+use PostgresNode;
+use TestLib;
+
+use Test::More tests => 3;
+
+my ($node, $result);
+
+#
+# Test set-up
+#
+$node = get_new_node('CIC_test');
+$node->init;
+$node->append_conf('postgresql.conf',
+	'lock_timeout = ' . (1000 * $TestLib::timeout_default));
+$node->start;
+$node->safe_psql('postgres', q(CREATE EXTENSION amcheck));
+$node->safe_psql('postgres', q(CREATE TABLE tbl(i int)));
+$node->safe_psql('postgres', q(CREATE INDEX idx ON tbl(i)));
+
+#
+# Stress CIC with pgbench.
+#
+# pgbench might try to launch more than one instance of the CIC
+# transaction concurrently.  That would deadlock, so use an advisory
+# lock to ensure only one CIC runs at a time.
+#
+$node->pgbench(
+	'--no-vacuum --client=5 --transactions=100',
+	0,
+	[qr{actually processed}],
+	[qr{^$}],
+	'concurrent INSERTs and CIC',
+	{
+		'002_pgbench_concurrent_transaction' => q(
+			BEGIN;
+			INSERT INTO tbl VALUES(0);
+			COMMIT;
+		  ),
+		'002_pgbench_concurrent_transaction_savepoints' => q(
+			BEGIN;
+			SAVEPOINT s1;
+			INSERT INTO tbl VALUES(0);
+			COMMIT;
+		  ),
+		'002_pgbench_concurrent_cic' => q(
+			SELECT pg_try_advisory_lock(42)::integer AS gotlock \gset
+			\if :gotlock
+				DROP INDEX CONCURRENTLY idx;
+				CREATE INDEX CONCURRENTLY idx ON tbl(i);
+				SELECT bt_index_check('idx',true);
+				SELECT pg_advisory_unlock(42);
+			\endif
+		  )
+	});
+
+$node->stop;
+done_testing();
diff --git a/contrib/amcheck/t/003_cic_2pc.pl b/contrib/amcheck/t/003_cic_2pc.pl
new file mode 100644
index 0000000..445aaba
--- /dev/null
+++ b/contrib/amcheck/t/003_cic_2pc.pl
@@ -0,0 +1,188 @@
+
+# Copyright (c) 2021, PostgreSQL Global Development Group
+
+# Test CREATE INDEX CONCURRENTLY with concurrent prepared-xact modifications
+use strict;
+use warnings;
+
+use Config;
+use PostgresNode;
+use TestLib;
+
+use Test::More tests => 5;
+
+Test::More->builder->todo_start('filesystem bug')
+  if TestLib::has_wal_read_bug;
+
+my ($node, $result);
+
+#
+# Test set-up
+#
+$node = get_new_node('CIC_2PC_test');
+$node->init;
+$node->append_conf('postgresql.conf', 'max_prepared_transactions = 10');
+$node->append_conf('postgresql.conf',
+	'lock_timeout = ' . (1000 * $TestLib::timeout_default));
+$node->start;
+$node->safe_psql('postgres', q(CREATE EXTENSION amcheck));
+$node->safe_psql('postgres', q(CREATE TABLE tbl(i int)));
+
+
+#
+# Run 3 overlapping 2PC transactions with CIC
+#
+# We have two concurrent background psql processes: $main_h for INSERTs and
+# $cic_h for CIC.  Also, we use non-background psql for some COMMIT PREPARED
+# statements.
+#
+
+my $main_in    = '';
+my $main_out   = '';
+my $main_timer = IPC::Run::timeout($TestLib::timeout_default);
+
+my $main_h =
+  $node->background_psql('postgres', \$main_in, \$main_out,
+	$main_timer, on_error_stop => 1);
+$main_in .= q(
+BEGIN;
+INSERT INTO tbl VALUES(0);
+\echo syncpoint1
+);
+pump $main_h until $main_out =~ /syncpoint1/ || $main_timer->is_expired;
+
+my $cic_in    = '';
+my $cic_out   = '';
+my $cic_timer = IPC::Run::timeout($TestLib::timeout_default);
+my $cic_h =
+  $node->background_psql('postgres', \$cic_in, \$cic_out,
+	$cic_timer, on_error_stop => 1);
+$cic_in .= q(
+\echo start
+CREATE INDEX CONCURRENTLY idx ON tbl(i);
+);
+pump $cic_h until $cic_out =~ /start/ || $cic_timer->is_expired;
+
+$main_in .= q(
+PREPARE TRANSACTION 'a';
+);
+
+$main_in .= q(
+BEGIN;
+INSERT INTO tbl VALUES(0);
+\echo syncpoint2
+);
+pump $main_h until $main_out =~ /syncpoint2/ || $main_timer->is_expired;
+
+$node->safe_psql('postgres', q(COMMIT PREPARED 'a';));
+
+$main_in .= q(
+PREPARE TRANSACTION 'b';
+BEGIN;
+INSERT INTO tbl VALUES(0);
+\echo syncpoint3
+);
+pump $main_h until $main_out =~ /syncpoint3/ || $main_timer->is_expired;
+
+$node->safe_psql('postgres', q(COMMIT PREPARED 'b';));
+
+$main_in .= q(
+PREPARE TRANSACTION 'c';
+COMMIT PREPARED 'c';
+);
+$main_h->pump_nb;
+
+$main_h->finish;
+$cic_h->finish;
+
+$result = $node->psql('postgres', q(SELECT bt_index_check('idx',true)));
+is($result, '0', 'bt_index_check after overlapping 2PC');
+
+
+#
+# Server restart shall not change whether prepared xact blocks CIC
+#
+
+$node->safe_psql(
+	'postgres', q(
+BEGIN;
+INSERT INTO tbl VALUES(0);
+PREPARE TRANSACTION 'spans_restart';
+BEGIN;
+CREATE TABLE unused ();
+PREPARE TRANSACTION 'persists_forever';
+));
+$node->restart;
+
+my $reindex_in  = '';
+my $reindex_out = '';
+my $reindex_timer =
+  IPC::Run::timeout($TestLib::timeout_default);
+my $reindex_h =
+  $node->background_psql('postgres', \$reindex_in, \$reindex_out,
+	$reindex_timer, on_error_stop => 1);
+$reindex_in .= q(
+\echo start
+DROP INDEX CONCURRENTLY idx;
+CREATE INDEX CONCURRENTLY idx ON tbl(i);
+);
+pump $reindex_h until $reindex_out =~ /start/ || $reindex_timer->is_expired;
+
+$node->safe_psql('postgres', "COMMIT PREPARED 'spans_restart'");
+$reindex_h->finish;
+$result = $node->psql('postgres', q(SELECT bt_index_check('idx',true)));
+is($result, '0', 'bt_index_check after 2PC and restart');
+
+
+#
+# Stress CIC+2PC with pgbench
+#
+# pgbench might try to launch more than one instance of the CIC
+# transaction concurrently.  That would deadlock, so use an advisory
+# lock to ensure only one CIC runs at a time.
+
+# Fix broken index first
+$node->safe_psql('postgres', q(REINDEX TABLE tbl;));
+
+# Run pgbench.
+$node->pgbench(
+	'--no-vacuum --client=5 --transactions=100',
+	0,
+	[qr{actually processed}],
+	[qr{^$}],
+	'concurrent INSERTs w/ 2PC and CIC',
+	{
+		'003_pgbench_concurrent_2pc' => q(
+			BEGIN;
+			INSERT INTO tbl VALUES(0);
+			PREPARE TRANSACTION 'c:client_id';
+			COMMIT PREPARED 'c:client_id';
+		  ),
+		'003_pgbench_concurrent_2pc_savepoint' => q(
+			BEGIN;
+			SAVEPOINT s1;
+			INSERT INTO tbl VALUES(0);
+			PREPARE TRANSACTION 'c:client_id';
+			COMMIT PREPARED 'c:client_id';
+		  ),
+		'003_pgbench_concurrent_cic' => q(
+			SELECT pg_try_advisory_lock(42)::integer AS gotlock \gset
+			\if :gotlock
+				DROP INDEX CONCURRENTLY idx;
+				CREATE INDEX CONCURRENTLY idx ON tbl(i);
+				SELECT bt_index_check('idx',true);
+				SELECT pg_advisory_unlock(42);
+			\endif
+		  ),
+		'004_pgbench_concurrent_ric' => q(
+			SELECT pg_try_advisory_lock(42)::integer AS gotlock \gset
+			\if :gotlock
+				REINDEX INDEX CONCURRENTLY idx;
+				SELECT bt_index_check('idx',true);
+				SELECT pg_advisory_unlock(42);
+			\endif
+		  )
+	});
+
+$node->stop;
+done_testing();