summaryrefslogtreecommitdiffstats
path: root/src/bin/pg_basebackup/pg_receivewal.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/bin/pg_basebackup/pg_receivewal.c')
-rw-r--r--src/bin/pg_basebackup/pg_receivewal.c778
1 files changed, 778 insertions, 0 deletions
diff --git a/src/bin/pg_basebackup/pg_receivewal.c b/src/bin/pg_basebackup/pg_receivewal.c
new file mode 100644
index 0000000..cd05f5f
--- /dev/null
+++ b/src/bin/pg_basebackup/pg_receivewal.c
@@ -0,0 +1,778 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_receivewal.c - receive streaming WAL data and write it
+ * to a local file.
+ *
+ * Author: Magnus Hagander <magnus@hagander.net>
+ *
+ * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/bin/pg_basebackup/pg_receivewal.c
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include <dirent.h>
+#include <signal.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "access/xlog_internal.h"
+#include "common/file_perm.h"
+#include "common/logging.h"
+#include "getopt_long.h"
+#include "libpq-fe.h"
+#include "receivelog.h"
+#include "streamutil.h"
+
+/* Time to sleep between reconnection attempts */
+#define RECONNECT_SLEEP_TIME 5
+
+/* Global options */
+static char *basedir = NULL;
+static int verbose = 0;
+static int compresslevel = 0;
+static int noloop = 0;
+static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
+static volatile bool time_to_stop = false;
+static bool do_create_slot = false;
+static bool slot_exists_ok = false;
+static bool do_drop_slot = false;
+static bool do_sync = true;
+static bool synchronous = false;
+static char *replication_slot = NULL;
+static XLogRecPtr endpos = InvalidXLogRecPtr;
+
+
+static void usage(void);
+static DIR *get_destination_dir(char *dest_folder);
+static void close_destination_dir(DIR *dest_dir, char *dest_folder);
+static XLogRecPtr FindStreamingStart(uint32 *tli);
+static void StreamLog(void);
+static bool stop_streaming(XLogRecPtr segendpos, uint32 timeline,
+ bool segment_finished);
+
+static void
+disconnect_atexit(void)
+{
+ if (conn != NULL)
+ PQfinish(conn);
+}
+
+/* Routines to evaluate segment file format */
+#define IsCompressXLogFileName(fname) \
+ (strlen(fname) == XLOG_FNAME_LEN + strlen(".gz") && \
+ strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \
+ strcmp((fname) + XLOG_FNAME_LEN, ".gz") == 0)
+#define IsPartialCompressXLogFileName(fname) \
+ (strlen(fname) == XLOG_FNAME_LEN + strlen(".gz.partial") && \
+ strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \
+ strcmp((fname) + XLOG_FNAME_LEN, ".gz.partial") == 0)
+
+static void
+usage(void)
+{
+ printf(_("%s receives PostgreSQL streaming write-ahead logs.\n\n"),
+ progname);
+ printf(_("Usage:\n"));
+ printf(_(" %s [OPTION]...\n"), progname);
+ printf(_("\nOptions:\n"));
+ printf(_(" -D, --directory=DIR receive write-ahead log files into this directory\n"));
+ printf(_(" -E, --endpos=LSN exit after receiving the specified LSN\n"));
+ printf(_(" --if-not-exists do not error if slot already exists when creating a slot\n"));
+ printf(_(" -n, --no-loop do not loop on connection lost\n"));
+ printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
+ printf(_(" -s, --status-interval=SECS\n"
+ " time between status packets sent to server (default: %d)\n"), (standby_message_timeout / 1000));
+ printf(_(" -S, --slot=SLOTNAME replication slot to use\n"));
+ printf(_(" --synchronous flush write-ahead log immediately after writing\n"));
+ printf(_(" -v, --verbose output verbose messages\n"));
+ printf(_(" -V, --version output version information, then exit\n"));
+ printf(_(" -Z, --compress=0-9 compress logs with given compression level\n"));
+ printf(_(" -?, --help show this help, then exit\n"));
+ printf(_("\nConnection options:\n"));
+ printf(_(" -d, --dbname=CONNSTR connection string\n"));
+ printf(_(" -h, --host=HOSTNAME database server host or socket directory\n"));
+ printf(_(" -p, --port=PORT database server port number\n"));
+ printf(_(" -U, --username=NAME connect as specified database user\n"));
+ printf(_(" -w, --no-password never prompt for password\n"));
+ printf(_(" -W, --password force password prompt (should happen automatically)\n"));
+ printf(_("\nOptional actions:\n"));
+ printf(_(" --create-slot create a new replication slot (for the slot's name see --slot)\n"));
+ printf(_(" --drop-slot drop the replication slot (for the slot's name see --slot)\n"));
+ printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
+ printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
+}
+
+static bool
+stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished)
+{
+ static uint32 prevtimeline = 0;
+ static XLogRecPtr prevpos = InvalidXLogRecPtr;
+
+ /* we assume that we get called once at the end of each segment */
+ if (verbose && segment_finished)
+ pg_log_info("finished segment at %X/%X (timeline %u)",
+ (uint32) (xlogpos >> 32), (uint32) xlogpos,
+ timeline);
+
+ if (!XLogRecPtrIsInvalid(endpos) && endpos < xlogpos)
+ {
+ if (verbose)
+ pg_log_info("stopped log streaming at %X/%X (timeline %u)",
+ (uint32) (xlogpos >> 32), (uint32) xlogpos,
+ timeline);
+ time_to_stop = true;
+ return true;
+ }
+
+ /*
+ * Note that we report the previous, not current, position here. After a
+ * timeline switch, xlogpos points to the beginning of the segment because
+ * that's where we always begin streaming. Reporting the end of previous
+ * timeline isn't totally accurate, because the next timeline can begin
+ * slightly before the end of the WAL that we received on the previous
+ * timeline, but it's close enough for reporting purposes.
+ */
+ if (verbose && prevtimeline != 0 && prevtimeline != timeline)
+ pg_log_info("switched to timeline %u at %X/%X",
+ timeline,
+ (uint32) (prevpos >> 32), (uint32) prevpos);
+
+ prevtimeline = timeline;
+ prevpos = xlogpos;
+
+ if (time_to_stop)
+ {
+ if (verbose)
+ pg_log_info("received interrupt signal, exiting");
+ return true;
+ }
+ return false;
+}
+
+
+/*
+ * Get destination directory.
+ */
+static DIR *
+get_destination_dir(char *dest_folder)
+{
+ DIR *dir;
+
+ Assert(dest_folder != NULL);
+ dir = opendir(dest_folder);
+ if (dir == NULL)
+ {
+ pg_log_error("could not open directory \"%s\": %m", basedir);
+ exit(1);
+ }
+
+ return dir;
+}
+
+
+/*
+ * Close existing directory.
+ */
+static void
+close_destination_dir(DIR *dest_dir, char *dest_folder)
+{
+ Assert(dest_dir != NULL && dest_folder != NULL);
+ if (closedir(dest_dir))
+ {
+ pg_log_error("could not close directory \"%s\": %m", dest_folder);
+ exit(1);
+ }
+}
+
+
+/*
+ * Determine starting location for streaming, based on any existing xlog
+ * segments in the directory. We start at the end of the last one that is
+ * complete (size matches wal segment size), on the timeline with highest ID.
+ *
+ * If there are no WAL files in the directory, returns InvalidXLogRecPtr.
+ */
+static XLogRecPtr
+FindStreamingStart(uint32 *tli)
+{
+ DIR *dir;
+ struct dirent *dirent;
+ XLogSegNo high_segno = 0;
+ uint32 high_tli = 0;
+ bool high_ispartial = false;
+
+ dir = get_destination_dir(basedir);
+
+ while (errno = 0, (dirent = readdir(dir)) != NULL)
+ {
+ uint32 tli;
+ XLogSegNo segno;
+ bool ispartial;
+ bool iscompress;
+
+ /*
+ * Check if the filename looks like an xlog file, or a .partial file.
+ */
+ if (IsXLogFileName(dirent->d_name))
+ {
+ ispartial = false;
+ iscompress = false;
+ }
+ else if (IsPartialXLogFileName(dirent->d_name))
+ {
+ ispartial = true;
+ iscompress = false;
+ }
+ else if (IsCompressXLogFileName(dirent->d_name))
+ {
+ ispartial = false;
+ iscompress = true;
+ }
+ else if (IsPartialCompressXLogFileName(dirent->d_name))
+ {
+ ispartial = true;
+ iscompress = true;
+ }
+ else
+ continue;
+
+ /*
+ * Looks like an xlog file. Parse its position.
+ */
+ XLogFromFileName(dirent->d_name, &tli, &segno, WalSegSz);
+
+ /*
+ * Check that the segment has the right size, if it's supposed to be
+ * completed. For non-compressed segments just check the on-disk size
+ * and see if it matches a completed segment. For compressed segments,
+ * look at the last 4 bytes of the compressed file, which is where the
+ * uncompressed size is located for gz files with a size lower than
+ * 4GB, and then compare it to the size of a completed segment. The 4
+ * last bytes correspond to the ISIZE member according to
+ * http://www.zlib.org/rfc-gzip.html.
+ */
+ if (!ispartial && !iscompress)
+ {
+ struct stat statbuf;
+ char fullpath[MAXPGPATH * 2];
+
+ snprintf(fullpath, sizeof(fullpath), "%s/%s", basedir, dirent->d_name);
+ if (stat(fullpath, &statbuf) != 0)
+ {
+ pg_log_error("could not stat file \"%s\": %m", fullpath);
+ exit(1);
+ }
+
+ if (statbuf.st_size != WalSegSz)
+ {
+ pg_log_warning("segment file \"%s\" has incorrect size %d, skipping",
+ dirent->d_name, (int) statbuf.st_size);
+ continue;
+ }
+ }
+ else if (!ispartial && iscompress)
+ {
+ int fd;
+ char buf[4];
+ int bytes_out;
+ char fullpath[MAXPGPATH * 2];
+ int r;
+
+ snprintf(fullpath, sizeof(fullpath), "%s/%s", basedir, dirent->d_name);
+
+ fd = open(fullpath, O_RDONLY | PG_BINARY, 0);
+ if (fd < 0)
+ {
+ pg_log_error("could not open compressed file \"%s\": %m",
+ fullpath);
+ exit(1);
+ }
+ if (lseek(fd, (off_t) (-4), SEEK_END) < 0)
+ {
+ pg_log_error("could not seek in compressed file \"%s\": %m",
+ fullpath);
+ exit(1);
+ }
+ r = read(fd, (char *) buf, sizeof(buf));
+ if (r != sizeof(buf))
+ {
+ if (r < 0)
+ pg_log_error("could not read compressed file \"%s\": %m",
+ fullpath);
+ else
+ pg_log_error("could not read compressed file \"%s\": read %d of %zu",
+ fullpath, r, sizeof(buf));
+ exit(1);
+ }
+
+ close(fd);
+ bytes_out = (buf[3] << 24) | (buf[2] << 16) |
+ (buf[1] << 8) | buf[0];
+
+ if (bytes_out != WalSegSz)
+ {
+ pg_log_warning("compressed segment file \"%s\" has incorrect uncompressed size %d, skipping",
+ dirent->d_name, bytes_out);
+ continue;
+ }
+ }
+
+ /* Looks like a valid segment. Remember that we saw it. */
+ if ((segno > high_segno) ||
+ (segno == high_segno && tli > high_tli) ||
+ (segno == high_segno && tli == high_tli && high_ispartial && !ispartial))
+ {
+ high_segno = segno;
+ high_tli = tli;
+ high_ispartial = ispartial;
+ }
+ }
+
+ if (errno)
+ {
+ pg_log_error("could not read directory \"%s\": %m", basedir);
+ exit(1);
+ }
+
+ close_destination_dir(dir, basedir);
+
+ if (high_segno > 0)
+ {
+ XLogRecPtr high_ptr;
+
+ /*
+ * Move the starting pointer to the start of the next segment, if the
+ * highest one we saw was completed. Otherwise start streaming from
+ * the beginning of the .partial segment.
+ */
+ if (!high_ispartial)
+ high_segno++;
+
+ XLogSegNoOffsetToRecPtr(high_segno, 0, WalSegSz, high_ptr);
+
+ *tli = high_tli;
+ return high_ptr;
+ }
+ else
+ return InvalidXLogRecPtr;
+}
+
+/*
+ * Start the log streaming
+ */
+static void
+StreamLog(void)
+{
+ XLogRecPtr serverpos;
+ TimeLineID servertli;
+ StreamCtl stream;
+
+ MemSet(&stream, 0, sizeof(stream));
+
+ /*
+ * Connect in replication mode to the server
+ */
+ if (conn == NULL)
+ conn = GetConnection();
+ if (!conn)
+ /* Error message already written in GetConnection() */
+ return;
+
+ if (!CheckServerVersionForStreaming(conn))
+ {
+ /*
+ * Error message already written in CheckServerVersionForStreaming().
+ * There's no hope of recovering from a version mismatch, so don't
+ * retry.
+ */
+ exit(1);
+ }
+
+ /*
+ * Identify server, obtaining start LSN position and current timeline ID
+ * at the same time, necessary if not valid data can be found in the
+ * existing output directory.
+ */
+ if (!RunIdentifySystem(conn, NULL, &servertli, &serverpos, NULL))
+ exit(1);
+
+ /*
+ * Figure out where to start streaming.
+ */
+ stream.startpos = FindStreamingStart(&stream.timeline);
+ if (stream.startpos == InvalidXLogRecPtr)
+ {
+ stream.startpos = serverpos;
+ stream.timeline = servertli;
+ }
+
+ /*
+ * Always start streaming at the beginning of a segment
+ */
+ stream.startpos -= XLogSegmentOffset(stream.startpos, WalSegSz);
+
+ /*
+ * Start the replication
+ */
+ if (verbose)
+ pg_log_info("starting log streaming at %X/%X (timeline %u)",
+ (uint32) (stream.startpos >> 32), (uint32) stream.startpos,
+ stream.timeline);
+
+ stream.stream_stop = stop_streaming;
+ stream.stop_socket = PGINVALID_SOCKET;
+ stream.standby_message_timeout = standby_message_timeout;
+ stream.synchronous = synchronous;
+ stream.do_sync = do_sync;
+ stream.mark_done = false;
+ stream.walmethod = CreateWalDirectoryMethod(basedir, compresslevel,
+ stream.do_sync);
+ stream.partial_suffix = ".partial";
+ stream.replication_slot = replication_slot;
+
+ ReceiveXlogStream(conn, &stream);
+
+ if (!stream.walmethod->finish())
+ {
+ pg_log_info("could not finish writing WAL files: %m");
+ return;
+ }
+
+ PQfinish(conn);
+ conn = NULL;
+
+ FreeWalDirectoryMethod();
+ pg_free(stream.walmethod);
+
+ conn = NULL;
+}
+
+/*
+ * When sigint is called, just tell the system to exit at the next possible
+ * moment.
+ */
+#ifndef WIN32
+
+static void
+sigint_handler(int signum)
+{
+ time_to_stop = true;
+}
+#endif
+
+int
+main(int argc, char **argv)
+{
+ static struct option long_options[] = {
+ {"help", no_argument, NULL, '?'},
+ {"version", no_argument, NULL, 'V'},
+ {"directory", required_argument, NULL, 'D'},
+ {"dbname", required_argument, NULL, 'd'},
+ {"endpos", required_argument, NULL, 'E'},
+ {"host", required_argument, NULL, 'h'},
+ {"port", required_argument, NULL, 'p'},
+ {"username", required_argument, NULL, 'U'},
+ {"no-loop", no_argument, NULL, 'n'},
+ {"no-password", no_argument, NULL, 'w'},
+ {"password", no_argument, NULL, 'W'},
+ {"status-interval", required_argument, NULL, 's'},
+ {"slot", required_argument, NULL, 'S'},
+ {"verbose", no_argument, NULL, 'v'},
+ {"compress", required_argument, NULL, 'Z'},
+/* action */
+ {"create-slot", no_argument, NULL, 1},
+ {"drop-slot", no_argument, NULL, 2},
+ {"if-not-exists", no_argument, NULL, 3},
+ {"synchronous", no_argument, NULL, 4},
+ {"no-sync", no_argument, NULL, 5},
+ {NULL, 0, NULL, 0}
+ };
+
+ int c;
+ int option_index;
+ char *db_name;
+ uint32 hi,
+ lo;
+
+ pg_logging_init(argv[0]);
+ progname = get_progname(argv[0]);
+ set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_basebackup"));
+
+ if (argc > 1)
+ {
+ if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
+ {
+ usage();
+ exit(0);
+ }
+ else if (strcmp(argv[1], "-V") == 0 ||
+ strcmp(argv[1], "--version") == 0)
+ {
+ puts("pg_receivewal (PostgreSQL) " PG_VERSION);
+ exit(0);
+ }
+ }
+
+ while ((c = getopt_long(argc, argv, "D:d:E:h:p:U:s:S:nwWvZ:",
+ long_options, &option_index)) != -1)
+ {
+ switch (c)
+ {
+ case 'D':
+ basedir = pg_strdup(optarg);
+ break;
+ case 'd':
+ connection_string = pg_strdup(optarg);
+ break;
+ case 'h':
+ dbhost = pg_strdup(optarg);
+ break;
+ case 'p':
+ if (atoi(optarg) <= 0)
+ {
+ pg_log_error("invalid port number \"%s\"", optarg);
+ exit(1);
+ }
+ dbport = pg_strdup(optarg);
+ break;
+ case 'U':
+ dbuser = pg_strdup(optarg);
+ break;
+ case 'w':
+ dbgetpassword = -1;
+ break;
+ case 'W':
+ dbgetpassword = 1;
+ break;
+ case 's':
+ standby_message_timeout = atoi(optarg) * 1000;
+ if (standby_message_timeout < 0)
+ {
+ pg_log_error("invalid status interval \"%s\"", optarg);
+ exit(1);
+ }
+ break;
+ case 'S':
+ replication_slot = pg_strdup(optarg);
+ break;
+ case 'E':
+ if (sscanf(optarg, "%X/%X", &hi, &lo) != 2)
+ {
+ pg_log_error("could not parse end position \"%s\"", optarg);
+ exit(1);
+ }
+ endpos = ((uint64) hi) << 32 | lo;
+ break;
+ case 'n':
+ noloop = 1;
+ break;
+ case 'v':
+ verbose++;
+ break;
+ case 'Z':
+ compresslevel = atoi(optarg);
+ if (compresslevel < 0 || compresslevel > 9)
+ {
+ pg_log_error("invalid compression level \"%s\"", optarg);
+ exit(1);
+ }
+ break;
+/* action */
+ case 1:
+ do_create_slot = true;
+ break;
+ case 2:
+ do_drop_slot = true;
+ break;
+ case 3:
+ slot_exists_ok = true;
+ break;
+ case 4:
+ synchronous = true;
+ break;
+ case 5:
+ do_sync = false;
+ break;
+ default:
+
+ /*
+ * getopt_long already emitted a complaint
+ */
+ fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
+ progname);
+ exit(1);
+ }
+ }
+
+ /*
+ * Any non-option arguments?
+ */
+ if (optind < argc)
+ {
+ pg_log_error("too many command-line arguments (first is \"%s\")",
+ argv[optind]);
+ fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
+ progname);
+ exit(1);
+ }
+
+ if (do_drop_slot && do_create_slot)
+ {
+ pg_log_error("cannot use --create-slot together with --drop-slot");
+ fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
+ progname);
+ exit(1);
+ }
+
+ if (replication_slot == NULL && (do_drop_slot || do_create_slot))
+ {
+ /* translator: second %s is an option name */
+ pg_log_error("%s needs a slot to be specified using --slot",
+ do_drop_slot ? "--drop-slot" : "--create-slot");
+ fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
+ progname);
+ exit(1);
+ }
+
+ if (synchronous && !do_sync)
+ {
+ pg_log_error("cannot use --synchronous together with --no-sync");
+ fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
+ progname);
+ exit(1);
+ }
+
+ /*
+ * Required arguments
+ */
+ if (basedir == NULL && !do_drop_slot && !do_create_slot)
+ {
+ pg_log_error("no target directory specified");
+ fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
+ progname);
+ exit(1);
+ }
+
+#ifndef HAVE_LIBZ
+ if (compresslevel != 0)
+ {
+ pg_log_error("this build does not support compression");
+ exit(1);
+ }
+#endif
+
+ /*
+ * Check existence of destination folder.
+ */
+ if (!do_drop_slot && !do_create_slot)
+ {
+ DIR *dir = get_destination_dir(basedir);
+
+ close_destination_dir(dir, basedir);
+ }
+
+#ifndef WIN32
+ pqsignal(SIGINT, sigint_handler);
+#endif
+
+ /*
+ * Obtain a connection before doing anything.
+ */
+ conn = GetConnection();
+ if (!conn)
+ /* error message already written in GetConnection() */
+ exit(1);
+ atexit(disconnect_atexit);
+
+ /*
+ * Run IDENTIFY_SYSTEM to make sure we've successfully have established a
+ * replication connection and haven't connected using a database specific
+ * connection.
+ */
+ if (!RunIdentifySystem(conn, NULL, NULL, NULL, &db_name))
+ exit(1);
+
+ /*
+ * Set umask so that directories/files are created with the same
+ * permissions as directories/files in the source data directory.
+ *
+ * pg_mode_mask is set to owner-only by default and then updated in
+ * GetConnection() where we get the mode from the server-side with
+ * RetrieveDataDirCreatePerm() and then call SetDataDirectoryCreatePerm().
+ */
+ umask(pg_mode_mask);
+
+ /* determine remote server's xlog segment size */
+ if (!RetrieveWalSegSize(conn))
+ exit(1);
+
+ /*
+ * Check that there is a database associated with connection, none should
+ * be defined in this context.
+ */
+ if (db_name)
+ {
+ pg_log_error("replication connection using slot \"%s\" is unexpectedly database specific",
+ replication_slot);
+ exit(1);
+ }
+
+ /*
+ * Drop a replication slot.
+ */
+ if (do_drop_slot)
+ {
+ if (verbose)
+ pg_log_info("dropping replication slot \"%s\"", replication_slot);
+
+ if (!DropReplicationSlot(conn, replication_slot))
+ exit(1);
+ exit(0);
+ }
+
+ /* Create a replication slot */
+ if (do_create_slot)
+ {
+ if (verbose)
+ pg_log_info("creating replication slot \"%s\"", replication_slot);
+
+ if (!CreateReplicationSlot(conn, replication_slot, NULL, false, true, false,
+ slot_exists_ok))
+ exit(1);
+ exit(0);
+ }
+
+ /*
+ * Don't close the connection here so that subsequent StreamLog() can
+ * reuse it.
+ */
+
+ while (true)
+ {
+ StreamLog();
+ if (time_to_stop)
+ {
+ /*
+ * We've been Ctrl-C'ed or end of streaming position has been
+ * willingly reached, so exit without an error code.
+ */
+ exit(0);
+ }
+ else if (noloop)
+ {
+ pg_log_error("disconnected");
+ exit(1);
+ }
+ else
+ {
+ /* translator: check source for value for %d */
+ pg_log_info("disconnected; waiting %d seconds to try again",
+ RECONNECT_SLEEP_TIME);
+ pg_usleep(RECONNECT_SLEEP_TIME * 1000000);
+ }
+ }
+}