Adding upstream version 16.2.upstream/16.2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

5 files changed, 573 insertions, 0 deletions

diff --git a/contrib/basebackup_to_shell/.gitignore b/contrib/basebackup_to_shell/.gitignore
new file mode 100644
index 0000000..5dcb3ff
--- /dev/null
+++ b/contrib/basebackup_to_shell/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/contrib/basebackup_to_shell/Makefile b/contrib/basebackup_to_shell/Makefile
new file mode 100644
index 0000000..58bd269
--- /dev/null
+++ b/contrib/basebackup_to_shell/Makefile
@@ -0,0 +1,24 @@
+# contrib/basebackup_to_shell/Makefile
+
+MODULE_big = basebackup_to_shell
+OBJS = \
+	$(WIN32RES) \
+	basebackup_to_shell.o
+
+PGFILEDESC = "basebackup_to_shell - target basebackup to shell command"
+
+TAP_TESTS = 1
+
+export GZIP_PROGRAM=$(GZIP)
+export TAR
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = contrib/basebackup_to_shell
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/contrib/basebackup_to_shell/basebackup_to_shell.c b/contrib/basebackup_to_shell/basebackup_to_shell.c
new file mode 100644
index 0000000..57ed587
--- /dev/null
+++ b/contrib/basebackup_to_shell/basebackup_to_shell.c
@@ -0,0 +1,373 @@
+/*-------------------------------------------------------------------------
+ *
+ * basebackup_to_shell.c
+ *	  target base backup files to a shell command
+ *
+ * Copyright (c) 2016-2023, PostgreSQL Global Development Group
+ *
+ *	  contrib/basebackup_to_shell/basebackup_to_shell.c
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/xact.h"
+#include "backup/basebackup_target.h"
+#include "common/percentrepl.h"
+#include "miscadmin.h"
+#include "storage/fd.h"
+#include "utils/acl.h"
+#include "utils/guc.h"
+
+PG_MODULE_MAGIC;
+
+typedef struct bbsink_shell
+{
+	/* Common information for all types of sink. */
+	bbsink		base;
+
+	/* User-supplied target detail string. */
+	char	   *target_detail;
+
+	/* Shell command pattern being used for this backup. */
+	char	   *shell_command;
+
+	/* The command that is currently running. */
+	char	   *current_command;
+
+	/* Pipe to the running command. */
+	FILE	   *pipe;
+} bbsink_shell;
+
+static void *shell_check_detail(char *target, char *target_detail);
+static bbsink *shell_get_sink(bbsink *next_sink, void *detail_arg);
+
+static void bbsink_shell_begin_archive(bbsink *sink,
+									   const char *archive_name);
+static void bbsink_shell_archive_contents(bbsink *sink, size_t len);
+static void bbsink_shell_end_archive(bbsink *sink);
+static void bbsink_shell_begin_manifest(bbsink *sink);
+static void bbsink_shell_manifest_contents(bbsink *sink, size_t len);
+static void bbsink_shell_end_manifest(bbsink *sink);
+
+static const bbsink_ops bbsink_shell_ops = {
+	.begin_backup = bbsink_forward_begin_backup,
+	.begin_archive = bbsink_shell_begin_archive,
+	.archive_contents = bbsink_shell_archive_contents,
+	.end_archive = bbsink_shell_end_archive,
+	.begin_manifest = bbsink_shell_begin_manifest,
+	.manifest_contents = bbsink_shell_manifest_contents,
+	.end_manifest = bbsink_shell_end_manifest,
+	.end_backup = bbsink_forward_end_backup,
+	.cleanup = bbsink_forward_cleanup
+};
+
+static char *shell_command = "";
+static char *shell_required_role = "";
+
+void
+_PG_init(void)
+{
+	DefineCustomStringVariable("basebackup_to_shell.command",
+							   "Shell command to be executed for each backup file.",
+							   NULL,
+							   &shell_command,
+							   "",
+							   PGC_SIGHUP,
+							   0,
+							   NULL, NULL, NULL);
+
+	DefineCustomStringVariable("basebackup_to_shell.required_role",
+							   "Backup user must be a member of this role to use shell backup target.",
+							   NULL,
+							   &shell_required_role,
+							   "",
+							   PGC_SIGHUP,
+							   0,
+							   NULL, NULL, NULL);
+
+	MarkGUCPrefixReserved("basebackup_to_shell");
+
+	BaseBackupAddTarget("shell", shell_check_detail, shell_get_sink);
+}
+
+/*
+ * We choose to defer sanity checking until shell_get_sink(), and so
+ * just pass the target detail through without doing anything. However, we do
+ * permissions checks here, before any real work has been done.
+ */
+static void *
+shell_check_detail(char *target, char *target_detail)
+{
+	if (shell_required_role[0] != '\0')
+	{
+		Oid			roleid;
+
+		StartTransactionCommand();
+		roleid = get_role_oid(shell_required_role, true);
+		if (!has_privs_of_role(GetUserId(), roleid))
+			ereport(ERROR,
+					(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+					 errmsg("permission denied to use basebackup_to_shell")));
+		CommitTransactionCommand();
+	}
+
+	return target_detail;
+}
+
+/*
+ * Set up a bbsink to implement this base backup target.
+ *
+ * This is also a convenient place to sanity check that a target detail was
+ * given if and only if %d is present.
+ */
+static bbsink *
+shell_get_sink(bbsink *next_sink, void *detail_arg)
+{
+	bbsink_shell *sink;
+	bool		has_detail_escape = false;
+	char	   *c;
+
+	/*
+	 * Set up the bbsink.
+	 *
+	 * We remember the current value of basebackup_to_shell.shell_command to
+	 * be certain that it can't change under us during the backup.
+	 */
+	sink = palloc0(sizeof(bbsink_shell));
+	*((const bbsink_ops **) &sink->base.bbs_ops) = &bbsink_shell_ops;
+	sink->base.bbs_next = next_sink;
+	sink->target_detail = detail_arg;
+	sink->shell_command = pstrdup(shell_command);
+
+	/* Reject an empty shell command. */
+	if (sink->shell_command[0] == '\0')
+		ereport(ERROR,
+				errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				errmsg("shell command for backup is not configured"));
+
+	/* Determine whether the shell command we're using contains %d. */
+	for (c = sink->shell_command; *c != '\0'; ++c)
+	{
+		if (c[0] == '%' && c[1] != '\0')
+		{
+			if (c[1] == 'd')
+				has_detail_escape = true;
+			++c;
+		}
+	}
+
+	/* There should be a target detail if %d was used, and not otherwise. */
+	if (has_detail_escape && sink->target_detail == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("a target detail is required because the configured command includes %%d"),
+				 errhint("Try \"pg_basebackup --target shell:DETAIL ...\"")));
+	else if (!has_detail_escape && sink->target_detail != NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("a target detail is not permitted because the configured command does not include %%d")));
+
+	/*
+	 * Since we're passing the string provided by the user to popen(), it will
+	 * be interpreted by the shell, which is a potential security
+	 * vulnerability, since the user invoking this module is not necessarily a
+	 * superuser. To stay out of trouble, we must disallow any shell
+	 * metacharacters here; to be conservative and keep things simple, we
+	 * allow only alphanumerics.
+	 */
+	if (sink->target_detail != NULL)
+	{
+		char	   *d;
+		bool		scary = false;
+
+		for (d = sink->target_detail; *d != '\0'; ++d)
+		{
+			if (*d >= 'a' && *d <= 'z')
+				continue;
+			if (*d >= 'A' && *d <= 'Z')
+				continue;
+			if (*d >= '0' && *d <= '9')
+				continue;
+			scary = true;
+			break;
+		}
+
+		if (scary)
+			ereport(ERROR,
+					errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+					errmsg("target detail must contain only alphanumeric characters"));
+	}
+
+	return &sink->base;
+}
+
+/*
+ * Construct the exact shell command that we're actually going to run,
+ * making substitutions as appropriate for escape sequences.
+ */
+static char *
+shell_construct_command(const char *base_command, const char *filename,
+						const char *target_detail)
+{
+	return replace_percent_placeholders(base_command, "basebackup_to_shell.command",
+										"df", target_detail, filename);
+}
+
+/*
+ * Finish executing the shell command once all data has been written.
+ */
+static void
+shell_finish_command(bbsink_shell *sink)
+{
+	int			pclose_rc;
+
+	/* There should be a command running. */
+	Assert(sink->current_command != NULL);
+	Assert(sink->pipe != NULL);
+
+	/* Close down the pipe we opened. */
+	pclose_rc = ClosePipeStream(sink->pipe);
+	if (pclose_rc == -1)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close pipe to external command: %m")));
+	else if (pclose_rc != 0)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
+				 errmsg("shell command \"%s\" failed",
+						sink->current_command),
+				 errdetail_internal("%s", wait_result_to_str(pclose_rc))));
+	}
+
+	/* Clean up. */
+	sink->pipe = NULL;
+	pfree(sink->current_command);
+	sink->current_command = NULL;
+}
+
+/*
+ * Start up the shell command, substituting %f in for the current filename.
+ */
+static void
+shell_run_command(bbsink_shell *sink, const char *filename)
+{
+	/* There should not be anything already running. */
+	Assert(sink->current_command == NULL);
+	Assert(sink->pipe == NULL);
+
+	/* Construct a suitable command. */
+	sink->current_command = shell_construct_command(sink->shell_command,
+													filename,
+													sink->target_detail);
+
+	/* Run it. */
+	sink->pipe = OpenPipeStream(sink->current_command, PG_BINARY_W);
+	if (sink->pipe == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not execute command \"%s\": %m",
+						sink->current_command)));
+}
+
+/*
+ * Send accumulated data to the running shell command.
+ */
+static void
+shell_send_data(bbsink_shell *sink, size_t len)
+{
+	/* There should be a command running. */
+	Assert(sink->current_command != NULL);
+	Assert(sink->pipe != NULL);
+
+	/* Try to write the data. */
+	if (fwrite(sink->base.bbs_buffer, len, 1, sink->pipe) != 1 ||
+		ferror(sink->pipe))
+	{
+		if (errno == EPIPE)
+		{
+			/*
+			 * The error we're about to throw would shut down the command
+			 * anyway, but we may get a more meaningful error message by doing
+			 * this. If not, we'll fall through to the generic error below.
+			 */
+			shell_finish_command(sink);
+			errno = EPIPE;
+		}
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not write to shell backup program: %m")));
+	}
+}
+
+/*
+ * At start of archive, start up the shell command and forward to next sink.
+ */
+static void
+bbsink_shell_begin_archive(bbsink *sink, const char *archive_name)
+{
+	bbsink_shell *mysink = (bbsink_shell *) sink;
+
+	shell_run_command(mysink, archive_name);
+	bbsink_forward_begin_archive(sink, archive_name);
+}
+
+/*
+ * Send archive contents to command's stdin and forward to next sink.
+ */
+static void
+bbsink_shell_archive_contents(bbsink *sink, size_t len)
+{
+	bbsink_shell *mysink = (bbsink_shell *) sink;
+
+	shell_send_data(mysink, len);
+	bbsink_forward_archive_contents(sink, len);
+}
+
+/*
+ * At end of archive, shut down the shell command and forward to next sink.
+ */
+static void
+bbsink_shell_end_archive(bbsink *sink)
+{
+	bbsink_shell *mysink = (bbsink_shell *) sink;
+
+	shell_finish_command(mysink);
+	bbsink_forward_end_archive(sink);
+}
+
+/*
+ * At start of manifest, start up the shell command and forward to next sink.
+ */
+static void
+bbsink_shell_begin_manifest(bbsink *sink)
+{
+	bbsink_shell *mysink = (bbsink_shell *) sink;
+
+	shell_run_command(mysink, "backup_manifest");
+	bbsink_forward_begin_manifest(sink);
+}
+
+/*
+ * Send manifest contents to command's stdin and forward to next sink.
+ */
+static void
+bbsink_shell_manifest_contents(bbsink *sink, size_t len)
+{
+	bbsink_shell *mysink = (bbsink_shell *) sink;
+
+	shell_send_data(mysink, len);
+	bbsink_forward_manifest_contents(sink, len);
+}
+
+/*
+ * At end of manifest, shut down the shell command and forward to next sink.
+ */
+static void
+bbsink_shell_end_manifest(bbsink *sink)
+{
+	bbsink_shell *mysink = (bbsink_shell *) sink;
+
+	shell_finish_command(mysink);
+	bbsink_forward_end_manifest(sink);
+}
diff --git a/contrib/basebackup_to_shell/meson.build b/contrib/basebackup_to_shell/meson.build
new file mode 100644
index 0000000..a5488c3
--- /dev/null
+++ b/contrib/basebackup_to_shell/meson.build
@@ -0,0 +1,30 @@
+# Copyright (c) 2022-2023, PostgreSQL Global Development Group
+
+basebackup_to_shell_sources = files(
+  'basebackup_to_shell.c',
+)
+
+if host_system == 'windows'
+  basebackup_to_shell_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
+    '--NAME', 'basebackup_to_shell',
+    '--FILEDESC', 'basebackup_to_shell - target basebackup to shell command',])
+endif
+
+basebackup_to_shell = shared_module('basebackup_to_shell',
+  basebackup_to_shell_sources,
+  kwargs: contrib_mod_args,
+)
+contrib_targets += basebackup_to_shell
+
+tests += {
+  'name': 'basebackup_to_shell',
+  'sd': meson.current_source_dir(),
+  'bd': meson.current_build_dir(),
+  'tap': {
+    'tests': [
+      't/001_basic.pl',
+    ],
+    'env': {'GZIP_PROGRAM': gzip.path(),
+            'TAR': tar.path()},
+  },
+}
diff --git a/contrib/basebackup_to_shell/t/001_basic.pl b/contrib/basebackup_to_shell/t/001_basic.pl
new file mode 100644
index 0000000..e2cdd2e
--- /dev/null
+++ b/contrib/basebackup_to_shell/t/001_basic.pl
@@ -0,0 +1,142 @@
+# Copyright (c) 2021-2023, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# For testing purposes, we just want basebackup_to_shell to write standard
+# input to a file.  However, Windows doesn't have "cat" or any equivalent, so
+# we use "gzip" for this purpose.
+my $gzip = $ENV{'GZIP_PROGRAM'};
+if (!defined $gzip || $gzip eq '')
+{
+	plan skip_all => 'gzip not available';
+}
+
+# to ensure path can be embedded in postgresql.conf
+$gzip =~ s{\\}{/}g if ($PostgreSQL::Test::Utils::windows_os);
+
+my $node = PostgreSQL::Test::Cluster->new('primary');
+
+# Make sure pg_hba.conf is set up to allow connections from backupuser.
+# This is only needed on Windows machines that don't use UNIX sockets.
+$node->init(
+	'allows_streaming' => 1,
+	'auth_extra' => [ '--create-role', 'backupuser' ]);
+
+$node->append_conf('postgresql.conf',
+	"shared_preload_libraries = 'basebackup_to_shell'");
+$node->start;
+$node->safe_psql('postgres', 'CREATE USER backupuser REPLICATION');
+$node->safe_psql('postgres', 'CREATE ROLE trustworthy');
+
+# For nearly all pg_basebackup invocations some options should be specified,
+# to keep test times reasonable. Using @pg_basebackup_defs as the first
+# element of the array passed to IPC::Run interpolate the array (as it is
+# not a reference to an array)...
+my @pg_basebackup_defs = ('pg_basebackup', '--no-sync', '-cfast');
+
+# This particular test module generally wants to run with -Xfetch, because
+# -Xstream is not supported with a backup target, and with -U backupuser.
+my @pg_basebackup_cmd = (@pg_basebackup_defs, '-U', 'backupuser', '-Xfetch');
+
+# Can't use this module without setting basebackup_to_shell.command.
+$node->command_fails_like(
+	[ @pg_basebackup_cmd, '--target', 'shell' ],
+	qr/shell command for backup is not configured/,
+	'fails if basebackup_to_shell.command is not set');
+
+# Configure basebackup_to_shell.command and reload the configuration file.
+my $backup_path = PostgreSQL::Test::Utils::tempdir;
+my $escaped_backup_path = $backup_path;
+$escaped_backup_path =~ s{\\}{\\\\}g
+  if ($PostgreSQL::Test::Utils::windows_os);
+my $shell_command =
+  $PostgreSQL::Test::Utils::windows_os
+  ? qq{"$gzip" --fast > "$escaped_backup_path\\\\%f.gz"}
+  : qq{"$gzip" --fast > "$escaped_backup_path/%f.gz"};
+$node->append_conf('postgresql.conf',
+	"basebackup_to_shell.command='$shell_command'");
+$node->reload();
+
+# Should work now.
+$node->command_ok(
+	[ @pg_basebackup_cmd, '--target', 'shell' ],
+	'backup with no detail: pg_basebackup');
+verify_backup('', $backup_path, "backup with no detail");
+
+# Should fail with a detail.
+$node->command_fails_like(
+	[ @pg_basebackup_cmd, '--target', 'shell:foo' ],
+	qr/a target detail is not permitted because the configured command does not include %d/,
+	'fails if detail provided without %d');
+
+# Reconfigure to restrict access and require a detail.
+$shell_command =
+  $PostgreSQL::Test::Utils::windows_os
+  ? qq{"$gzip" --fast > "$escaped_backup_path\\\\%d.%f.gz"}
+  : qq{"$gzip" --fast > "$escaped_backup_path/%d.%f.gz"};
+$node->append_conf('postgresql.conf',
+	"basebackup_to_shell.command='$shell_command'");
+$node->append_conf('postgresql.conf',
+	"basebackup_to_shell.required_role='trustworthy'");
+$node->reload();
+
+# Should fail due to lack of permission.
+$node->command_fails_like(
+	[ @pg_basebackup_cmd, '--target', 'shell' ],
+	qr/permission denied to use basebackup_to_shell/,
+	'fails if required_role not granted');
+
+# Should fail due to lack of a detail.
+$node->safe_psql('postgres', 'GRANT trustworthy TO backupuser');
+$node->command_fails_like(
+	[ @pg_basebackup_cmd, '--target', 'shell' ],
+	qr/a target detail is required because the configured command includes %d/,
+	'fails if %d is present and detail not given');
+
+# Should work.
+$node->command_ok([ @pg_basebackup_cmd, '--target', 'shell:bar' ],
+	'backup with detail: pg_basebackup');
+verify_backup('bar.', $backup_path, "backup with detail");
+
+done_testing();
+
+sub verify_backup
+{
+	my ($prefix, $backup_dir, $test_name) = @_;
+
+	ok( -f "$backup_dir/${prefix}backup_manifest.gz",
+		"$test_name: backup_manifest.gz was created");
+	ok( -f "$backup_dir/${prefix}base.tar.gz",
+		"$test_name: base.tar.gz was created");
+
+  SKIP:
+	{
+		my $tar = $ENV{TAR};
+		skip "no tar program available", 1 if (!defined $tar || $tar eq '');
+
+		# Decompress.
+		system_or_bail($gzip, '-d',
+			$backup_dir . '/' . $prefix . 'backup_manifest.gz');
+		system_or_bail($gzip, '-d',
+			$backup_dir . '/' . $prefix . 'base.tar.gz');
+
+		# Untar.
+		my $extract_path = PostgreSQL::Test::Utils::tempdir;
+		system_or_bail($tar, 'xf', $backup_dir . '/' . $prefix . 'base.tar',
+			'-C', $extract_path);
+
+		# Verify.
+		$node->command_ok(
+			[
+				'pg_verifybackup', '-n',
+				'-m', "${backup_dir}/${prefix}backup_manifest",
+				'-e', $extract_path
+			],
+			"$test_name: backup verifies ok");
+	}
+}