Adding upstream version 15.4.upstream/15.4upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

10 files changed, 1201 insertions, 0 deletions

diff --git a/src/bin/pg_verifybackup/t/001_basic.pl b/src/bin/pg_verifybackup/t/001_basic.pl
new file mode 100644
index 0000000..16febf9
--- /dev/null
+++ b/src/bin/pg_verifybackup/t/001_basic.pl
@@ -0,0 +1,38 @@
+
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir;
+
+program_help_ok('pg_verifybackup');
+program_version_ok('pg_verifybackup');
+program_options_handling_ok('pg_verifybackup');
+
+command_fails_like(
+	['pg_verifybackup'],
+	qr/no backup directory specified/,
+	'target directory must be specified');
+command_fails_like(
+	[ 'pg_verifybackup', $tempdir ],
+	qr/could not open file.*\/backup_manifest\"/,
+	'pg_verifybackup requires a manifest');
+command_fails_like(
+	[ 'pg_verifybackup', $tempdir, $tempdir ],
+	qr/too many command-line arguments/,
+	'multiple target directories not allowed');
+
+# create fake manifest file
+open(my $fh, '>', "$tempdir/backup_manifest") || die "open: $!";
+close($fh);
+
+# but then try to use an alternate, nonexisting manifest
+command_fails_like(
+	[ 'pg_verifybackup', '-m', "$tempdir/not_the_manifest", $tempdir ],
+	qr/could not open file.*\/not_the_manifest\"/,
+	'pg_verifybackup respects -m flag');
+
+done_testing();
diff --git a/src/bin/pg_verifybackup/t/002_algorithm.pl b/src/bin/pg_verifybackup/t/002_algorithm.pl
new file mode 100644
index 0000000..ca2c0f0
--- /dev/null
+++ b/src/bin/pg_verifybackup/t/002_algorithm.pl
@@ -0,0 +1,61 @@
+
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Verify that we can take and verify backups with various checksum types.
+
+use strict;
+use warnings;
+use File::Path qw(rmtree);
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $primary = PostgreSQL::Test::Cluster->new('primary');
+$primary->init(allows_streaming => 1);
+$primary->start;
+
+for my $algorithm (qw(bogus none crc32c sha224 sha256 sha384 sha512))
+{
+	my $backup_path = $primary->backup_dir . '/' . $algorithm;
+	my @backup      = (
+		'pg_basebackup', '-D', $backup_path,
+		'--manifest-checksums', $algorithm, '--no-sync', '-cfast');
+	my @verify = ('pg_verifybackup', '-e', $backup_path);
+
+	# A backup with a bogus algorithm should fail.
+	if ($algorithm eq 'bogus')
+	{
+		$primary->command_fails(\@backup,
+			"backup fails with algorithm \"$algorithm\"");
+		next;
+	}
+
+	# A backup with a valid algorithm should work.
+	$primary->command_ok(\@backup, "backup ok with algorithm \"$algorithm\"");
+
+	# We expect each real checksum algorithm to be mentioned on every line of
+	# the backup manifest file except the first and last; for simplicity, we
+	# just check that it shows up lots of times. When the checksum algorithm
+	# is none, we just check that the manifest exists.
+	if ($algorithm eq 'none')
+	{
+		ok(-f "$backup_path/backup_manifest", "backup manifest exists");
+	}
+	else
+	{
+		my $manifest = slurp_file("$backup_path/backup_manifest");
+		my $count_of_algorithm_in_manifest =
+		  (() = $manifest =~ /$algorithm/mig);
+		cmp_ok($count_of_algorithm_in_manifest,
+			'>', 100, "$algorithm is mentioned many times in the manifest");
+	}
+
+	# Make sure that it verifies OK.
+	$primary->command_ok(\@verify,
+		"verify backup with algorithm \"$algorithm\"");
+
+	# Remove backup immediately to save disk space.
+	rmtree($backup_path);
+}
+
+done_testing();
diff --git a/src/bin/pg_verifybackup/t/003_corruption.pl b/src/bin/pg_verifybackup/t/003_corruption.pl
new file mode 100644
index 0000000..3dba7d8
--- /dev/null
+++ b/src/bin/pg_verifybackup/t/003_corruption.pl
@@ -0,0 +1,292 @@
+
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Verify that various forms of corruption are detected by pg_verifybackup.
+
+use strict;
+use warnings;
+use File::Path qw(rmtree);
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $primary = PostgreSQL::Test::Cluster->new('primary');
+$primary->init(allows_streaming => 1);
+$primary->start;
+
+# Include a user-defined tablespace in the hopes of detecting problems in that
+# area.
+my $source_ts_path   = PostgreSQL::Test::Utils::tempdir_short();
+my $source_ts_prefix = $source_ts_path;
+$source_ts_prefix =~ s!(^[A-Z]:/[^/]*)/.*!$1!;
+
+$primary->safe_psql('postgres', <<EOM);
+CREATE TABLE x1 (a int);
+INSERT INTO x1 VALUES (111);
+CREATE TABLESPACE ts1 LOCATION '$source_ts_path';
+CREATE TABLE x2 (a int) TABLESPACE ts1;
+INSERT INTO x1 VALUES (222);
+EOM
+
+my @scenario = (
+	{
+		'name'     => 'extra_file',
+		'mutilate' => \&mutilate_extra_file,
+		'fails_like' =>
+		  qr/extra_file.*present on disk but not in the manifest/
+	},
+	{
+		'name'     => 'extra_tablespace_file',
+		'mutilate' => \&mutilate_extra_tablespace_file,
+		'fails_like' =>
+		  qr/extra_ts_file.*present on disk but not in the manifest/
+	},
+	{
+		'name'     => 'missing_file',
+		'mutilate' => \&mutilate_missing_file,
+		'fails_like' =>
+		  qr/pg_xact\/0000.*present in the manifest but not on disk/
+	},
+	{
+		'name'     => 'missing_tablespace',
+		'mutilate' => \&mutilate_missing_tablespace,
+		'fails_like' =>
+		  qr/pg_tblspc.*present in the manifest but not on disk/
+	},
+	{
+		'name'       => 'append_to_file',
+		'mutilate'   => \&mutilate_append_to_file,
+		'fails_like' => qr/has size \d+ on disk but size \d+ in the manifest/
+	},
+	{
+		'name'       => 'truncate_file',
+		'mutilate'   => \&mutilate_truncate_file,
+		'fails_like' => qr/has size 0 on disk but size \d+ in the manifest/
+	},
+	{
+		'name'       => 'replace_file',
+		'mutilate'   => \&mutilate_replace_file,
+		'fails_like' => qr/checksum mismatch for file/
+	},
+	{
+		'name'       => 'bad_manifest',
+		'mutilate'   => \&mutilate_bad_manifest,
+		'fails_like' => qr/manifest checksum mismatch/
+	},
+	{
+		'name'            => 'open_file_fails',
+		'mutilate'        => \&mutilate_open_file_fails,
+		'fails_like'      => qr/could not open file/,
+		'skip_on_windows' => 1
+	},
+	{
+		'name'            => 'open_directory_fails',
+		'mutilate'        => \&mutilate_open_directory_fails,
+		'cleanup'         => \&cleanup_open_directory_fails,
+		'fails_like'      => qr/could not open directory/,
+		'skip_on_windows' => 1
+	},
+	{
+		'name'            => 'search_directory_fails',
+		'mutilate'        => \&mutilate_search_directory_fails,
+		'cleanup'         => \&cleanup_search_directory_fails,
+		'fails_like'      => qr/could not stat file or directory/,
+		'skip_on_windows' => 1
+	});
+
+for my $scenario (@scenario)
+{
+	my $name = $scenario->{'name'};
+
+  SKIP:
+	{
+		skip "unix-style permissions not supported on Windows", 4
+		  if $scenario->{'skip_on_windows'} && $windows_os;
+
+		# Take a backup and check that it verifies OK.
+		my $backup_path    = $primary->backup_dir . '/' . $name;
+		my $backup_ts_path = PostgreSQL::Test::Utils::tempdir_short();
+		# The tablespace map parameter confuses Msys2, which tries to mangle
+		# it. Tell it not to.
+		# See https://www.msys2.org/wiki/Porting/#filesystem-namespaces
+		local $ENV{MSYS2_ARG_CONV_EXCL} = $source_ts_prefix;
+		$primary->command_ok(
+			[
+				'pg_basebackup', '-D', $backup_path, '--no-sync', '-cfast',
+				'-T', "${source_ts_path}=${backup_ts_path}"
+			],
+			"base backup ok");
+		command_ok([ 'pg_verifybackup', $backup_path ],
+			"intact backup verified");
+
+		# Mutilate the backup in some way.
+		$scenario->{'mutilate'}->($backup_path);
+
+		# Now check that the backup no longer verifies.
+		command_fails_like(
+			[ 'pg_verifybackup', $backup_path ],
+			$scenario->{'fails_like'},
+			"corrupt backup fails verification: $name");
+
+		# Run cleanup hook, if provided.
+		$scenario->{'cleanup'}->($backup_path)
+		  if exists $scenario->{'cleanup'};
+
+		# Finally, use rmtree to reclaim space.
+		rmtree($backup_path);
+	}
+}
+
+sub create_extra_file
+{
+	my ($backup_path, $relative_path) = @_;
+	my $pathname = "$backup_path/$relative_path";
+	open(my $fh, '>', $pathname) || die "open $pathname: $!";
+	print $fh "This is an extra file.\n";
+	close($fh);
+	return;
+}
+
+# Add a file into the root directory of the backup.
+sub mutilate_extra_file
+{
+	my ($backup_path) = @_;
+	create_extra_file($backup_path, "extra_file");
+	return;
+}
+
+# Add a file inside the user-defined tablespace.
+sub mutilate_extra_tablespace_file
+{
+	my ($backup_path) = @_;
+	my ($tsoid) =
+	  grep { $_ ne '.' && $_ ne '..' } slurp_dir("$backup_path/pg_tblspc");
+	my ($catvdir) = grep { $_ ne '.' && $_ ne '..' }
+	  slurp_dir("$backup_path/pg_tblspc/$tsoid");
+	my ($tsdboid) = grep { $_ ne '.' && $_ ne '..' }
+	  slurp_dir("$backup_path/pg_tblspc/$tsoid/$catvdir");
+	create_extra_file($backup_path,
+		"pg_tblspc/$tsoid/$catvdir/$tsdboid/extra_ts_file");
+	return;
+}
+
+# Remove a file.
+sub mutilate_missing_file
+{
+	my ($backup_path) = @_;
+	my $pathname = "$backup_path/pg_xact/0000";
+	unlink($pathname) || die "$pathname: $!";
+	return;
+}
+
+# Remove the symlink to the user-defined tablespace.
+sub mutilate_missing_tablespace
+{
+	my ($backup_path) = @_;
+	my ($tsoid) =
+	  grep { $_ ne '.' && $_ ne '..' } slurp_dir("$backup_path/pg_tblspc");
+	my $pathname = "$backup_path/pg_tblspc/$tsoid";
+	if ($windows_os)
+	{
+		# rmdir works on some windows setups, unlink on others.
+		# Instead of trying to implement precise rules, just try one and then
+		# the other.
+		unless (rmdir($pathname))
+		{
+			my $err = $!;
+			unlink($pathname) || die "$pathname: rmdir: $err, unlink: $!";
+		}
+	}
+	else
+	{
+		unlink($pathname) || die "$pathname: $!";
+	}
+	return;
+}
+
+# Append an additional bytes to a file.
+sub mutilate_append_to_file
+{
+	my ($backup_path) = @_;
+	append_to_file "$backup_path/global/pg_control", 'x';
+	return;
+}
+
+# Truncate a file to zero length.
+sub mutilate_truncate_file
+{
+	my ($backup_path) = @_;
+	my $pathname = "$backup_path/global/pg_control";
+	open(my $fh, '>', $pathname) || die "open $pathname: $!";
+	close($fh);
+	return;
+}
+
+# Replace a file's contents without changing the length of the file. This is
+# not a particularly efficient way to do this, so we pick a file that's
+# expected to be short.
+sub mutilate_replace_file
+{
+	my ($backup_path) = @_;
+	my $pathname      = "$backup_path/PG_VERSION";
+	my $contents      = slurp_file($pathname);
+	open(my $fh, '>', $pathname) || die "open $pathname: $!";
+	print $fh 'q' x length($contents);
+	close($fh);
+	return;
+}
+
+# Corrupt the backup manifest.
+sub mutilate_bad_manifest
+{
+	my ($backup_path) = @_;
+	append_to_file "$backup_path/backup_manifest", "\n";
+	return;
+}
+
+# Create a file that can't be opened. (This is skipped on Windows.)
+sub mutilate_open_file_fails
+{
+	my ($backup_path) = @_;
+	my $pathname = "$backup_path/PG_VERSION";
+	chmod(0, $pathname) || die "chmod $pathname: $!";
+	return;
+}
+
+# Create a directory that can't be opened. (This is skipped on Windows.)
+sub mutilate_open_directory_fails
+{
+	my ($backup_path) = @_;
+	my $pathname = "$backup_path/pg_subtrans";
+	chmod(0, $pathname) || die "chmod $pathname: $!";
+	return;
+}
+
+# restore permissions on the unreadable directory we created.
+sub cleanup_open_directory_fails
+{
+	my ($backup_path) = @_;
+	my $pathname = "$backup_path/pg_subtrans";
+	chmod(0700, $pathname) || die "chmod $pathname: $!";
+	return;
+}
+
+# Create a directory that can't be searched. (This is skipped on Windows.)
+sub mutilate_search_directory_fails
+{
+	my ($backup_path) = @_;
+	my $pathname = "$backup_path/base";
+	chmod(0400, $pathname) || die "chmod $pathname: $!";
+	return;
+}
+
+# rmtree can't cope with a mode 400 directory, so change back to 700.
+sub cleanup_search_directory_fails
+{
+	my ($backup_path) = @_;
+	my $pathname = "$backup_path/base";
+	chmod(0700, $pathname) || die "chmod $pathname: $!";
+	return;
+}
+
+done_testing();
diff --git a/src/bin/pg_verifybackup/t/004_options.pl b/src/bin/pg_verifybackup/t/004_options.pl
new file mode 100644
index 0000000..8cda66c
--- /dev/null
+++ b/src/bin/pg_verifybackup/t/004_options.pl
@@ -0,0 +1,108 @@
+
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Verify the behavior of assorted pg_verifybackup options.
+
+use strict;
+use warnings;
+use File::Path qw(rmtree);
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# Start up the server and take a backup.
+my $primary = PostgreSQL::Test::Cluster->new('primary');
+$primary->init(allows_streaming => 1);
+$primary->start;
+my $backup_path = $primary->backup_dir . '/test_options';
+$primary->command_ok(
+	[ 'pg_basebackup', '-D', $backup_path, '--no-sync', '-cfast' ],
+	"base backup ok");
+
+# Verify that pg_verifybackup -q succeeds and produces no output.
+my $stdout;
+my $stderr;
+my $result = IPC::Run::run [ 'pg_verifybackup', '-q', $backup_path ],
+  '>', \$stdout, '2>', \$stderr;
+ok($result, "-q succeeds: exit code 0");
+is($stdout, '', "-q succeeds: no stdout");
+is($stderr, '', "-q succeeds: no stderr");
+
+# Corrupt the PG_VERSION file.
+my $version_pathname = "$backup_path/PG_VERSION";
+my $version_contents = slurp_file($version_pathname);
+open(my $fh, '>', $version_pathname) || die "open $version_pathname: $!";
+print $fh 'q' x length($version_contents);
+close($fh);
+
+# Verify that pg_verifybackup -q now fails.
+command_fails_like(
+	[ 'pg_verifybackup', '-q', $backup_path ],
+	qr/checksum mismatch for file \"PG_VERSION\"/,
+	'-q checksum mismatch');
+
+# Since we didn't change the length of the file, verification should succeed
+# if we ignore checksums. Check that we get the right message, too.
+command_like(
+	[ 'pg_verifybackup', '-s', $backup_path ],
+	qr/backup successfully verified/,
+	'-s skips checksumming');
+
+# Validation should succeed if we ignore the problem file.
+command_like(
+	[ 'pg_verifybackup', '-i', 'PG_VERSION', $backup_path ],
+	qr/backup successfully verified/,
+	'-i ignores problem file');
+
+# PG_VERSION is already corrupt; let's try also removing all of pg_xact.
+rmtree($backup_path . "/pg_xact");
+
+# We're ignoring the problem with PG_VERSION, but not the problem with
+# pg_xact, so verification should fail here.
+command_fails_like(
+	[ 'pg_verifybackup', '-i', 'PG_VERSION', $backup_path ],
+	qr/pg_xact.*is present in the manifest but not on disk/,
+	'-i does not ignore all problems');
+
+# If we use -i twice, we should be able to ignore all of the problems.
+command_like(
+	[ 'pg_verifybackup', '-i', 'PG_VERSION', '-i', 'pg_xact', $backup_path ],
+	qr/backup successfully verified/,
+	'multiple -i options work');
+
+# Verify that when -i is not used, both problems are reported.
+$result = IPC::Run::run [ 'pg_verifybackup', $backup_path ],
+  '>', \$stdout, '2>', \$stderr;
+ok(!$result, "multiple problems: fails");
+like(
+	$stderr,
+	qr/pg_xact.*is present in the manifest but not on disk/,
+	"multiple problems: missing files reported");
+like(
+	$stderr,
+	qr/checksum mismatch for file \"PG_VERSION\"/,
+	"multiple problems: checksum mismatch reported");
+
+# Verify that when -e is used, only the problem detected first is reported.
+$result = IPC::Run::run [ 'pg_verifybackup', '-e', $backup_path ],
+  '>', \$stdout, '2>', \$stderr;
+ok(!$result, "-e reports 1 error: fails");
+like(
+	$stderr,
+	qr/pg_xact.*is present in the manifest but not on disk/,
+	"-e reports 1 error: missing files reported");
+unlike(
+	$stderr,
+	qr/checksum mismatch for file \"PG_VERSION\"/,
+	"-e reports 1 error: checksum mismatch not reported");
+
+# Test valid manifest with nonexistent backup directory.
+command_fails_like(
+	[
+		'pg_verifybackup',              '-m',
+		"$backup_path/backup_manifest", "$backup_path/fake"
+	],
+	qr/could not open directory/,
+	'nonexistent backup directory');
+
+done_testing();
diff --git a/src/bin/pg_verifybackup/t/005_bad_manifest.pl b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
new file mode 100644
index 0000000..b9573c5
--- /dev/null
+++ b/src/bin/pg_verifybackup/t/005_bad_manifest.pl
@@ -0,0 +1,209 @@
+
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Test the behavior of pg_verifybackup when the backup manifest has
+# problems.
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir;
+
+test_bad_manifest('input string ended unexpectedly',
+	qr/could not parse backup manifest: parsing failed/, <<EOM);
+{
+EOM
+
+test_parse_error('unexpected object end', <<EOM);
+{}
+EOM
+
+test_parse_error('unexpected array start', <<EOM);
+[]
+EOM
+
+test_parse_error('expected version indicator', <<EOM);
+{"not-expected": 1}
+EOM
+
+test_parse_error('unexpected manifest version', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": "phooey"}
+EOM
+
+test_parse_error('unexpected scalar', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "Files": true}
+EOM
+
+test_parse_error('unrecognized top-level field', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "Oops": 1}
+EOM
+
+test_parse_error('unexpected object start', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "Files": {}}
+EOM
+
+test_parse_error('missing path name', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "Files": [{}]}
+EOM
+
+test_parse_error('both path name and encoded path name', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "Files": [
+    {"Path": "x", "Encoded-Path": "1234"}
+]}
+EOM
+
+test_parse_error('unexpected file field', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "Files": [
+    {"Oops": 1}
+]}
+EOM
+
+test_parse_error('missing size', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "Files": [
+    {"Path": "x"}
+]}
+EOM
+
+test_parse_error('file size is not an integer', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "Files": [
+    {"Path": "x", "Size": "Oops"}
+]}
+EOM
+
+test_parse_error('could not decode file name', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "Files": [
+    {"Encoded-Path": "123", "Size": 0}
+]}
+EOM
+
+test_fatal_error('duplicate path name in backup manifest', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "Files": [
+    {"Path": "x", "Size": 0},
+    {"Path": "x", "Size": 0}
+]}
+EOM
+
+test_parse_error('checksum without algorithm', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "Files": [
+    {"Path": "x", "Size": 100, "Checksum": "Oops"}
+]}
+EOM
+
+test_fatal_error('unrecognized checksum algorithm', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "Files": [
+    {"Path": "x", "Size": 100, "Checksum-Algorithm": "Oops", "Checksum": "00"}
+]}
+EOM
+
+test_fatal_error('invalid checksum for file', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "Files": [
+    {"Path": "x", "Size": 100, "Checksum-Algorithm": "CRC32C", "Checksum": "0"}
+]}
+EOM
+
+test_parse_error('missing start LSN', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "WAL-Ranges": [
+    {"Timeline": 1}
+]}
+EOM
+
+test_parse_error('missing end LSN', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "WAL-Ranges": [
+    {"Timeline": 1, "Start-LSN": "0/0"}
+]}
+EOM
+
+test_parse_error('unexpected WAL range field', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "WAL-Ranges": [
+    {"Oops": 1}
+]}
+EOM
+
+test_parse_error('missing timeline', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "WAL-Ranges": [
+    {}
+]}
+EOM
+
+test_parse_error('unexpected object end', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "WAL-Ranges": [
+    {"Timeline": 1, "Start-LSN": "0/0", "End-LSN": "0/0"}
+]}
+EOM
+
+test_parse_error('timeline is not an integer', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "WAL-Ranges": [
+    {"Timeline": true, "Start-LSN": "0/0", "End-LSN": "0/0"}
+]}
+EOM
+
+test_parse_error('could not parse start LSN', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "WAL-Ranges": [
+    {"Timeline": 1, "Start-LSN": "oops", "End-LSN": "0/0"}
+]}
+EOM
+
+test_parse_error('could not parse end LSN', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "WAL-Ranges": [
+    {"Timeline": 1, "Start-LSN": "0/0", "End-LSN": "oops"}
+]}
+EOM
+
+test_parse_error('expected at least 2 lines', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "Files": [], "Manifest-Checksum": null}
+EOM
+
+my $manifest_without_newline = <<EOM;
+{"PostgreSQL-Backup-Manifest-Version": 1,
+ "Files": [],
+ "Manifest-Checksum": null}
+EOM
+chomp($manifest_without_newline);
+test_parse_error('last line not newline-terminated',
+	$manifest_without_newline);
+
+test_fatal_error('invalid manifest checksum', <<EOM);
+{"PostgreSQL-Backup-Manifest-Version": 1, "Files": [],
+ "Manifest-Checksum": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01234567890-"}
+EOM
+
+sub test_parse_error
+{
+	local $Test::Builder::Level = $Test::Builder::Level + 1;
+
+	my ($test_name, $manifest_contents) = @_;
+
+	test_bad_manifest($test_name,
+		qr/could not parse backup manifest: $test_name/,
+		$manifest_contents);
+	return;
+}
+
+sub test_fatal_error
+{
+	local $Test::Builder::Level = $Test::Builder::Level + 1;
+
+	my ($test_name, $manifest_contents) = @_;
+
+	test_bad_manifest($test_name, qr/error: $test_name/, $manifest_contents);
+	return;
+}
+
+sub test_bad_manifest
+{
+	local $Test::Builder::Level = $Test::Builder::Level + 1;
+
+	my ($test_name, $regexp, $manifest_contents) = @_;
+
+	open(my $fh, '>', "$tempdir/backup_manifest") || die "open: $!";
+	print $fh $manifest_contents;
+	close($fh);
+
+	command_fails_like([ 'pg_verifybackup', $tempdir ], $regexp, $test_name);
+	return;
+}
+
+done_testing();
diff --git a/src/bin/pg_verifybackup/t/006_encoding.pl b/src/bin/pg_verifybackup/t/006_encoding.pl
new file mode 100644
index 0000000..210f269
--- /dev/null
+++ b/src/bin/pg_verifybackup/t/006_encoding.pl
@@ -0,0 +1,34 @@
+
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Verify that pg_verifybackup handles hex-encoded filenames correctly.
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $primary = PostgreSQL::Test::Cluster->new('primary');
+$primary->init(allows_streaming => 1);
+$primary->start;
+my $backup_path = $primary->backup_dir . '/test_encoding';
+$primary->command_ok(
+	[
+		'pg_basebackup', '-D',
+		$backup_path,    '--no-sync',
+		'-cfast',        '--manifest-force-encode'
+	],
+	"backup ok with forced hex encoding");
+
+my $manifest = slurp_file("$backup_path/backup_manifest");
+my $count_of_encoded_path_in_manifest = (() = $manifest =~ /Encoded-Path/mig);
+cmp_ok($count_of_encoded_path_in_manifest,
+	'>', 100, "many paths are encoded in the manifest");
+
+command_like(
+	[ 'pg_verifybackup', '-s', $backup_path ],
+	qr/backup successfully verified/,
+	'backup with forced encoding verified');
+
+done_testing();
diff --git a/src/bin/pg_verifybackup/t/007_wal.pl b/src/bin/pg_verifybackup/t/007_wal.pl
new file mode 100644
index 0000000..6e9fafc
--- /dev/null
+++ b/src/bin/pg_verifybackup/t/007_wal.pl
@@ -0,0 +1,80 @@
+
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Test pg_verifybackup's WAL verification.
+
+use strict;
+use warnings;
+use File::Path qw(rmtree);
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# Start up the server and take a backup.
+my $primary = PostgreSQL::Test::Cluster->new('primary');
+$primary->init(allows_streaming => 1);
+$primary->start;
+my $backup_path = $primary->backup_dir . '/test_wal';
+$primary->command_ok(
+	[ 'pg_basebackup', '-D', $backup_path, '--no-sync', '-cfast' ],
+	"base backup ok");
+
+# Rename pg_wal.
+my $original_pg_wal  = $backup_path . '/pg_wal';
+my $relocated_pg_wal = $primary->backup_dir . '/relocated_pg_wal';
+rename($original_pg_wal, $relocated_pg_wal) || die "rename pg_wal: $!";
+
+# WAL verification should fail.
+command_fails_like(
+	[ 'pg_verifybackup', $backup_path ],
+	qr/WAL parsing failed for timeline 1/,
+	'missing pg_wal causes failure');
+
+# Should work if we skip WAL verification.
+command_ok(
+	[ 'pg_verifybackup', '-n', $backup_path ],
+	'missing pg_wal OK if not verifying WAL');
+
+# Should also work if we specify the correct WAL location.
+command_ok([ 'pg_verifybackup', '-w', $relocated_pg_wal, $backup_path ],
+	'-w can be used to specify WAL directory');
+
+# Move directory back to original location.
+rename($relocated_pg_wal, $original_pg_wal) || die "rename pg_wal back: $!";
+
+# Get a list of files in that directory that look like WAL files.
+my @walfiles = grep { /^[0-9A-F]{24}$/ } slurp_dir($original_pg_wal);
+
+# Replace the contents of one of the files with garbage of equal length.
+my $wal_corruption_target = $original_pg_wal . '/' . $walfiles[0];
+my $wal_size              = -s $wal_corruption_target;
+open(my $fh, '>', $wal_corruption_target)
+  || die "open $wal_corruption_target: $!";
+print $fh 'w' x $wal_size;
+close($fh);
+
+# WAL verification should fail.
+command_fails_like(
+	[ 'pg_verifybackup', $backup_path ],
+	qr/WAL parsing failed for timeline 1/,
+	'corrupt WAL file causes failure');
+
+# Check that WAL-Ranges has correct values with a history file and
+# a timeline > 1.  Rather than plugging in a new standby, do a
+# self-promotion of this node.
+$primary->stop;
+$primary->append_conf('standby.signal', '');
+$primary->start;
+$primary->promote;
+$primary->safe_psql('postgres', 'SELECT pg_switch_wal()');
+my $backup_path2 = $primary->backup_dir . '/test_tli';
+# The base backup run below does a checkpoint, that removes the first segment
+# of the current timeline.
+$primary->command_ok(
+	[ 'pg_basebackup', '-D', $backup_path2, '--no-sync', '-cfast' ],
+	"base backup 2 ok");
+command_ok(
+	[ 'pg_verifybackup', $backup_path2 ],
+	'valid base backup with timeline > 1');
+
+done_testing();
diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl
new file mode 100644
index 0000000..4c49595
--- /dev/null
+++ b/src/bin/pg_verifybackup/t/008_untar.pl
@@ -0,0 +1,127 @@
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# This test case aims to verify that server-side backups and server-side
+# backup compression work properly, and it also aims to verify that
+# pg_verifybackup can verify a base backup that didn't start out in plain
+# format.
+
+use strict;
+use warnings;
+use File::Path qw(rmtree);
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $primary = PostgreSQL::Test::Cluster->new('primary');
+$primary->init(allows_streaming => 1);
+$primary->start;
+
+my $backup_path  = $primary->backup_dir . '/server-backup';
+my $extract_path = $primary->backup_dir . '/extracted-backup';
+
+my @test_configuration = (
+	{
+		'compression_method' => 'none',
+		'backup_flags'       => [],
+		'backup_archive'     => 'base.tar',
+		'enabled'            => 1
+	},
+	{
+		'compression_method' => 'gzip',
+		'backup_flags'       => [ '--compress', 'server-gzip' ],
+		'backup_archive'     => 'base.tar.gz',
+		'decompress_program' => $ENV{'GZIP_PROGRAM'},
+		'decompress_flags'   => ['-d'],
+		'enabled'            => check_pg_config("#define HAVE_LIBZ 1")
+	},
+	{
+		'compression_method' => 'lz4',
+		'backup_flags'       => [ '--compress', 'server-lz4' ],
+		'backup_archive'     => 'base.tar.lz4',
+		'decompress_program' => $ENV{'LZ4'},
+		'decompress_flags'   => [ '-d', '-m' ],
+		'enabled'            => check_pg_config("#define USE_LZ4 1")
+	},
+	{
+		'compression_method' => 'zstd',
+		'backup_flags'       => [ '--compress', 'server-zstd' ],
+		'backup_archive'     => 'base.tar.zst',
+		'decompress_program' => $ENV{'ZSTD'},
+		'decompress_flags'   => ['-d'],
+		'enabled'            => check_pg_config("#define USE_ZSTD 1")
+	});
+
+for my $tc (@test_configuration)
+{
+	my $method = $tc->{'compression_method'};
+
+  SKIP:
+	{
+		skip "$method compression not supported by this build", 3
+		  if !$tc->{'enabled'};
+		skip "no decompressor available for $method", 3
+		  if exists $tc->{'decompress_program'}
+		  && (!defined $tc->{'decompress_program'}
+			|| $tc->{'decompress_program'} eq '');
+
+		# Take a server-side backup.
+		my @backup = (
+			'pg_basebackup',       '--no-sync',
+			'-cfast',              '--target',
+			"server:$backup_path", '-Xfetch');
+		push @backup, @{ $tc->{'backup_flags'} };
+		$primary->command_ok(\@backup,
+			"server side backup, compression $method");
+
+
+		# Verify that the we got the files we expected.
+		my $backup_files = join(',',
+			sort grep { $_ ne '.' && $_ ne '..' } slurp_dir($backup_path));
+		my $expected_backup_files =
+		  join(',', sort ('backup_manifest', $tc->{'backup_archive'}));
+		is($backup_files, $expected_backup_files,
+			"found expected backup files, compression $method");
+
+		# Decompress.
+		if (exists $tc->{'decompress_program'})
+		{
+			my @decompress = ($tc->{'decompress_program'});
+			push @decompress, @{ $tc->{'decompress_flags'} }
+			  if $tc->{'decompress_flags'};
+			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
+			system_or_bail(@decompress);
+		}
+
+	  SKIP:
+		{
+			my $tar = $ENV{TAR};
+			# don't check for a working tar here, to accommodate various odd
+			# cases such as AIX. If tar doesn't work the init_from_backup below
+			# will fail.
+			skip "no tar program available", 1
+			  if (!defined $tar || $tar eq '');
+
+			# Untar.
+			mkdir($extract_path);
+			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
+				'-C', $extract_path);
+
+			# Verify.
+			$primary->command_ok(
+				[
+					'pg_verifybackup', '-n',
+					'-m', "$backup_path/backup_manifest",
+					'-e', $extract_path
+				],
+				"verify backup, compression $method");
+		}
+
+		# Cleanup.
+		unlink($backup_path . '/backup_manifest');
+		unlink($backup_path . '/base.tar');
+		unlink($backup_path . '/' . $tc->{'backup_archive'});
+		rmtree($extract_path);
+	}
+}
+
+done_testing();
diff --git a/src/bin/pg_verifybackup/t/009_extract.pl b/src/bin/pg_verifybackup/t/009_extract.pl
new file mode 100644
index 0000000..56889e1
--- /dev/null
+++ b/src/bin/pg_verifybackup/t/009_extract.pl
@@ -0,0 +1,97 @@
+
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# This test aims to verify that the client can decompress and extract
+# a backup which was compressed by the server.
+
+use strict;
+use warnings;
+use File::Path qw(rmtree);
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+my $primary = PostgreSQL::Test::Cluster->new('primary');
+$primary->init(allows_streaming => 1);
+$primary->start;
+
+my @test_configuration = (
+	{
+		'compression_method' => 'none',
+		'backup_flags'       => [],
+		'enabled'            => 1
+	},
+	{
+		'compression_method' => 'gzip',
+		'backup_flags'       => [ '--compress', 'server-gzip:5' ],
+		'enabled'            => check_pg_config("#define HAVE_LIBZ 1")
+	},
+	{
+		'compression_method' => 'lz4',
+		'backup_flags'       => [ '--compress', 'server-lz4:5' ],
+		'enabled'            => check_pg_config("#define USE_LZ4 1")
+	},
+	{
+		'compression_method' => 'zstd',
+		'backup_flags'       => [ '--compress', 'server-zstd:5' ],
+		'enabled'            => check_pg_config("#define USE_ZSTD 1")
+	},
+	{
+		'compression_method' => 'parallel zstd',
+		'backup_flags'       => [ '--compress', 'server-zstd:workers=3' ],
+		'enabled'            => check_pg_config("#define USE_ZSTD 1"),
+		'possibly_unsupported' =>
+		  qr/could not set compression worker count to 3: Unsupported parameter/
+	});
+
+for my $tc (@test_configuration)
+{
+	my $backup_path = $primary->backup_dir . '/' . 'extract_backup';
+	my $method      = $tc->{'compression_method'};
+
+  SKIP:
+	{
+		skip "$method compression not supported by this build", 2
+		  if !$tc->{'enabled'};
+
+		# Take backup with server compression enabled.
+		my @backup = (
+			'pg_basebackup', '-D', $backup_path,
+			'-Xfetch', '--no-sync', '-cfast', '-Fp');
+		push @backup, @{ $tc->{'backup_flags'} };
+
+		my @verify = ('pg_verifybackup', '-e', $backup_path);
+
+		# A backup with a valid compression method should work.
+		my $backup_stdout = '';
+		my $backup_stderr = '';
+		my $backup_result = $primary->run_log(\@backup, '>', \$backup_stdout,
+			'2>', \$backup_stderr);
+		if ($backup_stdout ne '')
+		{
+			print "# standard output was:\n$backup_stdout";
+		}
+		if ($backup_stderr ne '')
+		{
+			print "# standard error was:\n$backup_stderr";
+		}
+		if (  !$backup_result
+			&& $tc->{'possibly_unsupported'}
+			&& $backup_stderr =~ /$tc->{'possibly_unsupported'}/)
+		{
+			skip "compression with $method not supported by this build", 2;
+		}
+		else
+		{
+			ok($backup_result, "backup done, compression $method");
+		}
+
+		# Make sure that it verifies OK.
+		$primary->command_ok(\@verify,
+			"backup verified, compression method \"$method\"");
+	}
+
+	# Remove backup immediately to save disk space.
+	rmtree($backup_path);
+}
+
+done_testing();
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
new file mode 100644
index 0000000..77cb503
--- /dev/null
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -0,0 +1,155 @@
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# This test case aims to verify that client-side backup compression work
+# properly, and it also aims to verify that pg_verifybackup can verify a base
+# backup that didn't start out in plain format.
+
+use strict;
+use warnings;
+use File::Path qw(rmtree);
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $primary = PostgreSQL::Test::Cluster->new('primary');
+$primary->init(allows_streaming => 1);
+$primary->start;
+
+my $backup_path  = $primary->backup_dir . '/client-backup';
+my $extract_path = $primary->backup_dir . '/extracted-backup';
+
+my @test_configuration = (
+	{
+		'compression_method' => 'none',
+		'backup_flags'       => [],
+		'backup_archive'     => 'base.tar',
+		'enabled'            => 1
+	},
+	{
+		'compression_method' => 'gzip',
+		'backup_flags'       => [ '--compress', 'client-gzip:5' ],
+		'backup_archive'     => 'base.tar.gz',
+		'decompress_program' => $ENV{'GZIP_PROGRAM'},
+		'decompress_flags'   => ['-d'],
+		'enabled'            => check_pg_config("#define HAVE_LIBZ 1")
+	},
+	{
+		'compression_method' => 'lz4',
+		'backup_flags'       => [ '--compress', 'client-lz4:5' ],
+		'backup_archive'     => 'base.tar.lz4',
+		'decompress_program' => $ENV{'LZ4'},
+		'decompress_flags'   => ['-d'],
+		'output_file'        => 'base.tar',
+		'enabled'            => check_pg_config("#define USE_LZ4 1")
+	},
+	{
+		'compression_method' => 'zstd',
+		'backup_flags'       => [ '--compress', 'client-zstd:5' ],
+		'backup_archive'     => 'base.tar.zst',
+		'decompress_program' => $ENV{'ZSTD'},
+		'decompress_flags'   => ['-d'],
+		'enabled'            => check_pg_config("#define USE_ZSTD 1")
+	},
+	{
+		'compression_method' => 'parallel zstd',
+		'backup_flags'       => [ '--compress', 'client-zstd:workers=3' ],
+		'backup_archive'     => 'base.tar.zst',
+		'decompress_program' => $ENV{'ZSTD'},
+		'decompress_flags'   => ['-d'],
+		'enabled'            => check_pg_config("#define USE_ZSTD 1"),
+		'possibly_unsupported' =>
+		  qr/could not set compression worker count to 3: Unsupported parameter/
+	});
+
+for my $tc (@test_configuration)
+{
+	my $method = $tc->{'compression_method'};
+
+  SKIP:
+	{
+		skip "$method compression not supported by this build", 3
+		  if !$tc->{'enabled'};
+		skip "no decompressor available for $method", 3
+		  if exists $tc->{'decompress_program'}
+		  && (!defined $tc->{'decompress_program'}
+			|| $tc->{'decompress_program'} eq '');
+
+		# Take a client-side backup.
+		my @backup = (
+			'pg_basebackup', '-D', $backup_path,
+			'-Xfetch', '--no-sync', '-cfast', '-Ft');
+		push @backup, @{ $tc->{'backup_flags'} };
+		my $backup_stdout = '';
+		my $backup_stderr = '';
+		my $backup_result = $primary->run_log(\@backup, '>', \$backup_stdout,
+			'2>', \$backup_stderr);
+		if ($backup_stdout ne '')
+		{
+			print "# standard output was:\n$backup_stdout";
+		}
+		if ($backup_stderr ne '')
+		{
+			print "# standard error was:\n$backup_stderr";
+		}
+		if (  !$backup_result
+			&& $tc->{'possibly_unsupported'}
+			&& $backup_stderr =~ /$tc->{'possibly_unsupported'}/)
+		{
+			skip "compression with $method not supported by this build", 3;
+		}
+		else
+		{
+			ok($backup_result, "client side backup, compression $method");
+		}
+
+		# Verify that the we got the files we expected.
+		my $backup_files = join(',',
+			sort grep { $_ ne '.' && $_ ne '..' } slurp_dir($backup_path));
+		my $expected_backup_files =
+		  join(',', sort ('backup_manifest', $tc->{'backup_archive'}));
+		is($backup_files, $expected_backup_files,
+			"found expected backup files, compression $method");
+
+		# Decompress.
+		if (exists $tc->{'decompress_program'})
+		{
+			my @decompress = ($tc->{'decompress_program'});
+			push @decompress, @{ $tc->{'decompress_flags'} }
+			  if $tc->{'decompress_flags'};
+			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
+			push @decompress, $backup_path . '/' . $tc->{'output_file'}
+			  if $tc->{'output_file'};
+			system_or_bail(@decompress);
+		}
+
+	  SKIP:
+		{
+			my $tar = $ENV{TAR};
+			# don't check for a working tar here, to accommodate various odd
+			# cases such as AIX. If tar doesn't work the init_from_backup below
+			# will fail.
+			skip "no tar program available", 1
+			  if (!defined $tar || $tar eq '');
+
+			# Untar.
+			mkdir($extract_path);
+			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
+				'-C', $extract_path);
+
+			# Verify.
+			$primary->command_ok(
+				[
+					'pg_verifybackup', '-n',
+					'-m', "$backup_path/backup_manifest",
+					'-e', $extract_path
+				],
+				"verify backup, compression $method");
+		}
+
+		# Cleanup.
+		rmtree($extract_path);
+		rmtree($backup_path);
+	}
+}
+
+done_testing();