diff options
133 files changed, 16226 insertions, 0 deletions
@@ -0,0 +1,3 @@ +set expandtab +set shiftwidth=4 +set smarttab diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..07a50bd --- /dev/null +++ b/Makefile @@ -0,0 +1,64 @@ +POD2MAN=pod2man --center "Debian PostgreSQL infrastructure" -r "Debian" +POD1PROGS = pg_backupcluster.1 \ + pg_conftool.1 \ + pg_createcluster.1 \ + pg_ctlcluster.1 \ + pg_dropcluster.1 \ + pg_getwal.1 \ + pg_lsclusters.1 \ + pg_renamecluster.1 \ + pg_restorecluster.1 \ + pg_upgradecluster.1 \ + pg_wrapper.1 +POD1PROGS_POD = pg_buildext.1 \ + pg_virtualenv.1 \ + debhelper/dh_pgxs_test.1 \ + dh_make_pgxs/dh_make_pgxs.1 +POD8PROGS = pg_updatedicts.8 + +all: man sub-pgdg + +man: $(POD1PROGS) $(POD1PROGS_POD) $(POD8PROGS) + +%.1: %.pod + $(POD2MAN) --quotes=none --section 1 $< $@ + +%.1: % + $(POD2MAN) --quotes=none --section 1 $< $@ + +%.8: % + $(POD2MAN) --quotes=none --section 8 $< $@ + +clean: + rm -f *.1 *.8 debhelper/*.1 dh_make_pgxs/*.1 + +sub-pgdg: + $(MAKE) -C pgdg + +# rpm + +DPKG_VERSION=$(shell sed -ne '1s/.*(//; 1s/).*//p' debian/changelog) +RPMDIR=$(CURDIR)/rpm +TARNAME=postgresql-common_$(DPKG_VERSION).tar.xz +TARBALL=$(RPMDIR)/SOURCES/$(TARNAME) + +rpmbuild: $(TARBALL) + rpmbuild -D"%_topdir $(RPMDIR)" --define='version $(DPKG_VERSION)' -ba rpm/postgresql-common.spec + +$(TARBALL): + mkdir -p $(dir $(TARBALL)) + if test -f ../$(TARNAME); then \ + cp -v ../$(TARNAME) $(TARBALL); \ + else \ + git archive --prefix=postgresql-common-$(DPKG_VERSION)/ HEAD | xz > $(TARBALL); \ + fi + +rpminstall: + sudo yum install -y perl-JSON + sudo rpm --upgrade --replacefiles --replacepkgs -v $(RPMDIR)/RPMS/noarch/*-$(DPKG_VERSION)-*.rpm + +rpmremove: + -sudo rpm -e postgresql-common postgresql-client-common postgresql-server-dev-all + +rpmclean: + rm -rf $(RPMDIR)/*/ diff --git a/PgCommon.pm b/PgCommon.pm new file mode 100644 index 0000000..df4d060 --- /dev/null +++ b/PgCommon.pm @@ -0,0 +1,1677 @@ +=head1 NAME + +PgCommon - Common functions for the postgresql-common framework + +=head1 COPYRIGHT AND LICENSE + + (C) 2008-2009 Martin Pitt <mpitt@debian.org> + (C) 2012-2023 Christoph Berg <myon@debian.org> + +This program is free software; you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the +Free Software Foundation; either +L<version 2 of the License|https://www.gnu.org/licenses/old-licenses/gpl-2.0.html>, +or (at your option) any later version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +=cut + +package PgCommon; +use strict; +use IPC::Open3; +use Socket; +use POSIX; + +use Exporter; +our $VERSION = 1.00; +our @ISA = ('Exporter'); +our @EXPORT = qw/error user_cluster_map get_cluster_port set_cluster_port + get_cluster_socketdir set_cluster_socketdir cluster_port_running + get_cluster_start_conf set_cluster_start_conf set_cluster_pg_ctl_conf + get_program_path cluster_info validate_cluster_owner get_versions get_newest_version version_exists + get_version_clusters next_free_port cluster_exists install_file + change_ugid system_or_error config_bool replace_v_c + get_db_encoding get_db_locales get_cluster_locales get_cluster_controldata + get_cluster_databases cluster_conf_filename read_cluster_conf_file + read_pg_hba read_pidfile valid_hba_method/; +our @EXPORT_OK = qw/$confroot $binroot $rpm $have_python2 + quote_conf_value read_conf_file get_conf_value + set_conf_value set_conffile_value disable_conffile_value disable_conf_value + replace_conf_value cluster_data_directory get_file_device + check_pidfile_running/; + + +=head1 CONTENTS + +=head2 error + + Print an error message to stderr and die with exit status 1 + +=cut + +sub error { + $! = 1; # force exit code 1 + die "Error: $_[0]\n"; +} + +our $confroot = '/etc/postgresql'; +if ($ENV{'PG_CLUSTER_CONF_ROOT'}) { + ($confroot) = $ENV{'PG_CLUSTER_CONF_ROOT'} =~ /(.*)/; # untaint +} +our $common_confdir = "/etc/postgresql-common"; +if ($ENV{'PGSYSCONFDIR'}) { + ($common_confdir) = $ENV{'PGSYSCONFDIR'} =~ /(.*)/; # untaint +} +my $mapfile = "$common_confdir/user_clusters"; +our $binroot = "/usr/lib/postgresql/"; +#redhat# $binroot = "/usr/pgsql-"; +our $rpm = 0; +#redhat# $rpm = 1; +our $defaultport = 5432; +our $have_python2 = 0; # python2 removed in bullseye+ +#py2#$have_python2 = 1; + +=head2 prepare_exec, restore_exec + + Untaint the environment for executing an external program + + Optional arguments: list of additional variables + +=cut + +{ + my %saved_env; + + # untaint the environment for executing an external program + sub prepare_exec { + my @cleanvars = qw/PATH IFS ENV BASH_ENV CDPATH/; + push @cleanvars, @_; + %saved_env = (); + + foreach (@cleanvars) { + $saved_env{$_} = $ENV{$_}; + delete $ENV{$_}; + } + + $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin'; + } + + # restore the environment after prepare_exec() + sub restore_exec { + foreach (keys %saved_env) { + if (defined $saved_env{$_}) { + $ENV{$_} = $saved_env{$_}; + } else { + delete $ENV{$_}; + } + } + } +} + + +=head2 config_bool + + returns '1' if the argument is a configuration file value that stands for + true (ON, TRUE, YES, or 1, case insensitive), '0' if the argument represents + a false value (OFF, FALSE, NO, or 0, case insensitive), or undef otherwise. + +=cut + +sub config_bool { + return undef unless defined($_[0]); + return 1 if ($_[0] =~ /^(on|true|yes|1)$/i); + return 0 if ($_[0] =~ /^(off|false|no|0)$/i); + return undef; +} + + +=head2 quote_conf_value + + Quotes a value with single quotes + + Arguments: <value> + Returns: quoted string + +=cut + +sub quote_conf_value ($) { + my $value = shift; + return $value if ($value =~ /^-?[\d.]+$/); # integer or float + return $value if ($value =~ /^\w+$/); # plain word + $value =~ s/'/''/g; # else quote it + return "'$value'"; +} + + +=head2 replace_v_c + + Replaces %v and %c placeholders + + Arguments: <string> <version> <cluster> + Returns: string + +=cut + +sub replace_v_c ($$$) { + my ($str, $version, $cluster) = @_; + $str =~ s/%([vc%])/$1 eq 'v' ? $version : + $1 eq 'c' ? $cluster : '%'/eg; + return $str; +} + + +=head2 read_conf_file + + Read a 'var = value' style configuration file and return a hash with the + values. Error out if the file cannot be read. + + If the file name ends with '.conf', the keys will be normalized to + lower case (suitable for e.g. postgresql.conf), otherwise kept intact + (suitable for environment). + + Arguments: <path> + Returns: hash (empty if file does not exist) + +=cut + +sub read_conf_file { + my ($config_path) = @_; + my %conf; + local (*F); + + sub get_absolute_path { + my ($path, $parent_path) = @_; + return $path if ($path =~ m!^/!); # path is absolute + # else strip filename component from parent path + $parent_path =~ s!/[^/]*$!!; + return "$parent_path/$path"; + } + + if (open F, $config_path) { + while (<F>) { + if (/^\s*(?:#.*)?$/) { + next; + } elsif(/^\s*include_dir\s*=?\s*'([^']+)'\s*(?:#.*)?$/i) { + # read included configuration directory and merge into %conf + # files in the directory will be read in ascending order + my $path = $1; + my $absolute_path = get_absolute_path($path, $config_path); + next unless -e $absolute_path && -d $absolute_path; + my $dir; + opendir($dir, $absolute_path) or next; + foreach my $filename (sort readdir($dir) ) { + next if ($filename =~ m/^\./ or not $filename =~/\.conf$/ ); + my %include_conf = read_conf_file("$absolute_path/$filename"); + while ( my ($k, $v) = each(%include_conf) ) { + $conf{$k} = $v; + } + } + closedir($dir); + } elsif (/^\s*include(?:_if_exists)?\s*=?\s*'([^']+)'\s*(?:#.*)?$/i) { + # read included file and merge into %conf + my $path = $1; + my $absolute_path = get_absolute_path($path, $config_path); + my %include_conf = read_conf_file($absolute_path); + while ( my ($k, $v) = each(%include_conf) ) { + $conf{$k} = $v; + } + } elsif (/^\s*([a-zA-Z0-9_.-]+)\s*(?:=|\s)\s*'((?:[^']|''|(?:(?<=\\)'))*)'\s*(?:#.*)?$/i) { + # string value + my $v = $2; + my $k = $1; + $k = lc $k if $config_path =~ /\.conf$/; + $v =~ s/\\(.)/$1/g; + $v =~ s/''/'/g; + $conf{$k} = $v; + } elsif (m{^\s*([a-zA-Z0-9_.-]+)\s*(?:=|\s)\s*(-?[[:alnum:]][[:alnum:]._:/+-]*)\s*(?:\#.*)?$}i) { + # simple value (string/float) + my $v = $2; + my $k = $1; + $k = lc $k if $config_path =~ /\.conf$/; + $conf{$k} = $v; + } else { + chomp; + error "invalid line $. in $config_path: $_"; + } + } + close F; + } + + return %conf; +} + +=head2 cluster_conf_filename + + Returns path to cluster config file from a cluster configuration + directory (with /etc/postgresql-common/<file name> as fallback) + and return a hash with the values. Error out if the file cannot be read. + If config file name is postgresql.auto.conf, read from PGDATA + + Arguments: <version> <cluster> <config file name> + Returns: hash (empty if the file does not exist) + +=cut + +sub cluster_conf_filename { + my ($version, $cluster, $configfile) = @_; + if ($configfile eq 'postgresql.auto.conf') { + my $data_directory = cluster_data_directory($version, $cluster); + return "$data_directory/$configfile"; + } + my $fname = "$confroot/$version/$cluster/$configfile"; + -e $fname or $fname = "$common_confdir/$configfile"; + return $fname; +} + + +=head2 read_cluster_conf_file + +Read a 'var = value' style configuration file from a cluster configuration + +Arguments: <version> <cluster> <config file name> +Returns: hash (empty if the file does not exist) + +=cut + +sub read_cluster_conf_file { + my ($version, $cluster, $configfile) = @_; + my %conf = read_conf_file(cluster_conf_filename($version, $cluster, $configfile)); + + if ($version >= 9.4 and $configfile eq 'postgresql.conf') { # merge settings changed by ALTER SYSTEM + # data_directory cannot be changed by ALTER SYSTEM + my $data_directory = cluster_data_directory($version, $cluster, \%conf); + my %auto_conf = read_conf_file "$data_directory/postgresql.auto.conf"; + foreach my $guc (keys %auto_conf) { + next if ($guc eq 'data_directory'); # defend against pg_upgradecluster bug in 200..202 + $conf{$guc} = $auto_conf{$guc}; + } + } + + return %conf; +} + + +=head2 get_conf_value + + Return parameter from a PostgreSQL configuration file, + or undef if the parameter does not exist. + + Arguments: <version> <cluster> <config file name> <parameter name> + +=cut + +sub get_conf_value { + my %conf = (read_cluster_conf_file $_[0], $_[1], $_[2]); + return $conf{$_[3]}; +} + + +=head2 set_conffile_value + + Set parameter of a PostgreSQL configuration file. + + Arguments: <config file name> <parameter name> <value> + +=cut + +sub set_conffile_value { + my ($fname, $key, $value) = ($_[0], $_[1], quote_conf_value($_[2])); + my @lines; + + # read configuration file lines + open (F, $fname) or die "Error: could not open $fname for reading"; + push @lines, $_ while (<F>); + close F; + + my $found = 0; + # first, search for an uncommented setting + for (my $i=0; $i <= $#lines; ++$i) { + if ($lines[$i] =~ /^\s*($key)(\s*(?:=|\s)\s*)\w+\b((?:\s*#.*)?)/i or + $lines[$i] =~ /^\s*($key)(\s*(?:=|\s)\s*)'[^']*'((?:\s*#.*)?)/i) { + $lines[$i] = "$1$2$value$3\n"; + $found = 1; + last; + } + } + + # now check if the setting exists as a comment; if so, change that instead + # of appending + if (!$found) { + for (my $i=0; $i <= $#lines; ++$i) { + if ($lines[$i] =~ /^\s*#\s*($key)(\s*(?:=|\s)\s*)\w+\b((?:\s*#.*)?)$/i or + $lines[$i] =~ /^\s*#\s*($key)(\s*(?:=|\s)\s*)'[^']*'((?:\s*#.*)?)$/i) { + $lines[$i] = "$1$2$value$3\n"; + $found = 1; + last; + } + } + } + + # not found anywhere, append it + push (@lines, "$key = $value\n") unless $found; + + # write configuration file lines + open (F, ">$fname.new") or die "Error: could not open $fname.new for writing"; + foreach (@lines) { + print F $_ or die "writing $fname.new: $!"; + } + close F; + + # copy permissions + my @st = stat $fname or die "stat: $!"; + chown $st[4], $st[5], "$fname.new"; # might fail as non-root + chmod $st[2], "$fname.new" or die "chmod: $!"; + + rename "$fname.new", "$fname" or die "rename $fname.new $fname: $!"; +} + + +=head2 set_conf_value + + Set parameter of a PostgreSQL cluster configuration file. + + Arguments: <version> <cluster> <config file name> <parameter name> <value> + +=cut + +sub set_conf_value { + return set_conffile_value(cluster_conf_filename($_[0], $_[1], $_[2]), $_[3], $_[4]); +} + + +=head2 disable_conffile_value + + Disable a parameter in a PostgreSQL configuration file by prepending it + with a '#'. Appends an optional explanatory comment <reason> if given. + + Arguments: <config file name> <parameter name> <reason> + +=cut + +sub disable_conffile_value { + my ($fname, $key, $reason) = @_; + my @lines; + + # read configuration file lines + open (F, $fname) or die "Error: could not open $fname for reading"; + push @lines, $_ while (<F>); + close F; + + my $changed = 0; + for (my $i=0; $i <= $#lines; ++$i) { + if ($lines[$i] =~ /^\s*$key\s*(?:=|\s)/i) { + $lines[$i] =~ s/^/#/; + $lines[$i] =~ s/$/ #$reason/ if $reason; + $changed = 1; + last; + } + } + + # write configuration file lines + if ($changed) { + open (F, ">$fname.new") or die "Error: could not open $fname.new for writing"; + foreach (@lines) { + print F $_ or die "writing $fname.new: $!"; + } + close F; + + # copy permissions + my @st = stat $fname or die "stat: $!"; + chown $st[4], $st[5], "$fname.new"; # might fail as non-root + chmod $st[2], "$fname.new" or die "chmod: $1"; + + rename "$fname.new", "$fname"; + } +} + + +=head2 disable_conf_value + + Disable a parameter in a PostgreSQL cluster configuration file by prepending + it with a '#'. Appends an optional explanatory comment <reason> if given. + + Arguments: <version> <cluster> <config file name> <parameter name> <reason> + +=cut + +sub disable_conf_value { + return disable_conffile_value(cluster_conf_filename($_[0], $_[1], $_[2]), $_[3], $_[4]); +} + + +=head2 replace_conf_value + + Replace a parameter in a PostgreSQL configuration file. The old parameter + is prepended with a '#' and gets an optional explanatory comment <reason> + appended, if given. The new parameter is inserted directly after the old one. + + Arguments: <version> <cluster> <config file name> <old parameter name> + <reason> <new parameter name> <new value> + +=cut + +sub replace_conf_value { + my ($version, $cluster, $configfile, $oldparam, $reason, $newparam, $val) = @_; + my $fname = cluster_conf_filename($version, $cluster, $configfile); + my @lines; + + # quote $val if necessary + unless ($val =~ /^\w+$/) { + $val = "'$val'"; + } + + # read configuration file lines + open (F, $fname) or die "Error: could not open $fname for reading"; + push @lines, $_ while (<F>); + close F; + + my $found = 0; + for (my $i = 0; $i <= $#lines; ++$i) { + if ($lines[$i] =~ /^\s*$oldparam\s*(?:=|\s)/i) { + $lines[$i] = '#'.$lines[$i]; + chomp $lines[$i]; + $lines[$i] .= ' #'.$reason."\n" if $reason; + + # insert the new param + splice @lines, $i+1, 0, "$newparam = $val\n"; + ++$i; + + $found = 1; + last; + } + } + + return if !$found; + + # write configuration file lines + open (F, ">$fname.new") or die "Error: could not open $fname.new for writing"; + foreach (@lines) { + print F $_ or die "writing $fname.new: $!"; + } + close F; + + # copy permissions + my @st = stat $fname or die "stat: $!"; + chown $st[4], $st[5], "$fname.new"; # might fail as non-root + chmod $st[2], "$fname.new" or die "chmod: $1"; + + rename "$fname.new", "$fname"; +} + + +=head2 get_cluster_port + + Return the port of a particular cluster + + Arguments: <version> <cluster> + +=cut + +sub get_cluster_port { + return get_conf_value($_[0], $_[1], 'postgresql.conf', 'port') || $defaultport; +} + + +=head2 set_cluster_port + + Set the port of a particular cluster. + + Arguments: <version> <cluster> <port> + +=cut + +sub set_cluster_port { + set_conf_value $_[0], $_[1], 'postgresql.conf', 'port', $_[2]; +} + + +=head2 cluster_data_directory + + Return cluster data directory. + + Arguments: <version> <cluster name> [<config_hash>] + +=cut + +sub cluster_data_directory { + my $d; + if ($_[2]) { + $d = ${$_[2]}{'data_directory'}; + } else { + $d = get_conf_value($_[0], $_[1], 'postgresql.conf', 'data_directory'); + } + my $confdir = "$confroot/$_[0]/$_[1]"; + if (!$d) { + # fall back to /pgdata symlink (supported by earlier p-common releases) + $d = readlink "$confdir/pgdata"; + } + if (!$d and -l $confdir and -f "$confdir/PG_VERSION") { # symlink from /etc/postgresql + $d = readlink $confdir; + } + if (!$d and -f "$confdir/PG_VERSION") { # PGDATA in /etc/postgresql + $d = $confdir; + } + ($d) = $d =~ /(.*)/ if defined $d; #untaint + return $d; +} + + +=head2 get_cluster_socketdir + + Return the socket directory of a particular cluster + or undef if the cluster does not exist. + + Arguments: <version> <cluster> + +=cut + +sub get_cluster_socketdir { + # if it is explicitly configured, just return it + my $socketdir = get_conf_value($_[0], $_[1], 'postgresql.conf', + $_[0] >= 9.3 ? 'unix_socket_directories' : 'unix_socket_directory'); + $socketdir =~ s/\s*,.*// if ($socketdir); # ignore additional directories for now + return $socketdir if $socketdir; + + #redhat# return '/tmp'; # RedHat PGDG packages default to /tmp + # try to determine whether this is a postgres owned cluster and we default + # to /var/run/postgresql + $socketdir = '/var/run/postgresql'; + my @socketdirstat = stat $socketdir; + + error "Cannot stat $socketdir" unless @socketdirstat; + + if ($_[0] && $_[1]) { + my $datadir = cluster_data_directory $_[0], $_[1]; + error "Invalid data directory for cluster $_[0] $_[1]" unless $datadir; + my @datadirstat = stat $datadir; + unless (@datadirstat) { + my @p = split '/', $datadir; + my $parent = join '/', @p[0..($#p-1)]; + error "$datadir is not accessible; please fix the directory permissions ($parent/ should be world readable)" unless @datadirstat; + } + + $socketdir = '/tmp' if $socketdirstat[4] != $datadirstat[4]; + } + + return $socketdir; +} + + +=head2 set_cluster_socketdir + + Set the socket directory of a particular cluster. + + Arguments: <version> <cluster> <directory> + +=cut + +sub set_cluster_socketdir { + set_conf_value $_[0], $_[1], 'postgresql.conf', + $_[0] >= 9.3 ? 'unix_socket_directories' : 'unix_socket_directory', + $_[2]; +} + + +=head2 get_program_path + + Return the path of a program of a particular version. + + Arguments: <program name> [<version>] + +=cut + +sub get_program_path { + my ($program, $version) = @_; + return '' unless defined $program; + $version //= get_newest_version($program); + my $path = "$binroot$version/bin/$program"; + ($path) = $path =~ /(.*)/; #untaint + return $path if -x $path; + return ''; +} + + +=head2 cluster_port_running + + Check whether a postgres server is running at the specified port. + + Arguments: <version> <cluster> <port> + +=cut + +sub cluster_port_running { + die "port_running: invalid port $_[2]" if $_[2] !~ /\d+/; + my $socketdir = get_cluster_socketdir $_[0], $_[1]; + my $socketpath = "$socketdir/.s.PGSQL.$_[2]"; + return 0 unless -S $socketpath; + + socket(SRV, PF_UNIX, SOCK_STREAM, 0) or die "socket: $!"; + my $running = connect(SRV, sockaddr_un($socketpath)); + close SRV; + return $running ? 1 : 0; +} + + +=head2 get_cluster_start_conf + + Read, verify, and return the current start.conf setting. + + Arguments: <version> <cluster> + Returns: auto | manual | disabled + +=cut + +sub get_cluster_start_conf { + my $start_conf = "$confroot/$_[0]/$_[1]/start.conf"; + if (-e $start_conf) { + open F, $start_conf or error "Could not open $start_conf: $!"; + while (<F>) { + s/#.*$//; + s/^\s*//; + s/\s*$//; + next unless $_; + close F; + return $1 if (/^(auto|manual|disabled)/); + error "Invalid mode in $start_conf, must be one of auto, manual, disabled"; + } + close F; + } + return 'auto'; # default +} + + +=head2 set_cluster_start_conf + + Change start.conf setting. + + Arguments: <version> <cluster> <value> + <value> = auto | manual | disabled + +=cut + +sub set_cluster_start_conf { + my ($v, $c, $val) = @_; + + error "Invalid mode: '$val'" unless $val eq 'auto' || + $val eq 'manual' || $val eq 'disabled'; + + my $perms = 0644; + + # start.conf setting + my $start_conf = "$confroot/$_[0]/$_[1]/start.conf"; + my $text; + if (-e $start_conf) { + open F, $start_conf or error "Could not open $start_conf: $!"; + while (<F>) { + if (/^\s*(?:auto|manual|disabled)\b(.*$)/) { + $text .= $val . $1 . "\n"; + } else { + $text .= $_; + } + } + + # preserve permissions if it already exists + $perms = (stat F)[2]; + error "Could not get permissions of $start_conf: $!" unless $perms; + close F; + } else { + $text = "# Automatic startup configuration +# auto: automatically start the cluster +# manual: manual startup with pg_ctlcluster/postgresql@.service only +# disabled: refuse to start cluster +# See pg_createcluster(1) for details. When running from systemd, +# invoke 'systemctl daemon-reload' after editing this file. + +$val +"; + } + + open F, '>' . $start_conf or error "Could not open $start_conf for writing: $!"; + chmod $perms, $start_conf; + print F $text; + close F; +} + + +=head2 set_cluster_pg_ctl_conf + + Change pg_ctl.conf setting. + + Arguments: <version> <cluster> <options> + <options> = options passed to pg_ctl(1) + +=cut + +sub set_cluster_pg_ctl_conf { + my ($v, $c, $opts) = @_; + my $perms = 0644; + + # pg_ctl.conf setting + my $pg_ctl_conf = "$confroot/$v/$c/pg_ctl.conf"; + my $text = "# Automatic pg_ctl configuration +# This configuration file contains cluster specific options to be passed to +# pg_ctl(1). + +pg_ctl_options = '$opts' +"; + + open F, '>' . $pg_ctl_conf or error "Could not open $pg_ctl_conf for writing: $!"; + chmod $perms, $pg_ctl_conf; + print F $text; + close F; +} + + +=head2 read_pidfile + + Return the PID from an existing PID file or undef if it does not exist. + + Arguments: <pid file path> + +=cut + +sub read_pidfile { + return undef unless -e $_[0]; + + if (open PIDFILE, $_[0]) { + my $pid = <PIDFILE>; + close PIDFILE; + return undef unless ($pid); + chomp $pid; + ($pid) = $pid =~ /^(\d+)\s*$/; # untaint + return $pid; + } else { + return undef; + } +} + + +=head2 check_pidfile_running + + Check whether a pid file is present and belongs to a running postgres. + Returns undef if it cannot be determined + + Arguments: <pid file path> + + postgres does not clean up the PID file when it stops, and it is + not world readable, so only its absence is a definitive result; + if it is present, we need to read it and check the PID, which will + only work as root + +=cut + +sub check_pidfile_running { + return 0 if ! -e $_[0]; + + my $pid = read_pidfile $_[0]; + if (defined $pid and open CL, "/proc/$pid/cmdline") { + my $cmdline = <CL>; + close CL; + if ($cmdline and $cmdline =~ /\bpostgres\b/) { + return 1; + } else { + return 0; + } + } + return undef; +} + + +=head2 cluster_supervisor + + Determine if a cluster is managed by a supervisor (pacemaker, patroni). + Returns undef if it cannot be determined + + Arguments: <pid file path> + + postgres does not clean up the PID file when it stops, and it is + not world readable, so only its absence is a definitive result; if it + is present, we need to read it and check the PID, which will only + work as root + +=cut + +sub cluster_supervisor { + return undef if ! -e $_[0]; + + my $pid = read_pidfile $_[0]; + if (defined $pid and open(CG, "/proc/$pid/cgroup")) { + local $/; # enable localized slurp mode + my $cgroup = <CG>; + close CG; + if ($cgroup and $cgroup =~ /\b(pacemaker|patroni)\b/) { + return $1; + } + } + return undef; +} + + +=head2 cluster_info + + Return a hash with information about a specific cluster (which needs to exist). + + Arguments: <version> <cluster name> + Returns: information hash (keys: pgdata, port, running, logfile [unless it + has a custom one], configdir, owneruid, ownergid, waldir, socketdir, + config->postgresql.conf) + +=cut + +sub cluster_info { + my ($v, $c) = @_; + error 'cluster_info must be called with <version> <cluster> arguments' unless ($v and $c); + + my %result; + $result{'configdir'} = "$confroot/$v/$c"; + $result{'configuid'} = (stat "$result{configdir}/postgresql.conf")[4]; + + my %postgresql_conf = read_cluster_conf_file $v, $c, 'postgresql.conf'; + $result{'config'} = \%postgresql_conf; + $result{'pgdata'} = cluster_data_directory $v, $c, \%postgresql_conf; + return %result unless (keys %postgresql_conf); + $result{'port'} = $postgresql_conf{'port'} || $defaultport; + $result{'socketdir'} = get_cluster_socketdir $v, $c; + + # if we can determine the running status with the pid file, prefer that + if ($postgresql_conf{'external_pid_file'} && + $postgresql_conf{'external_pid_file'} ne '(none)') { + $result{'running'} = check_pidfile_running $postgresql_conf{'external_pid_file'}; + my $supervisor = cluster_supervisor($postgresql_conf{'external_pid_file'}); + $result{supervisor} = $supervisor if ($supervisor); + } + + # otherwise fall back to probing the port; this is unreliable if the port + # was changed in the configuration file in the meantime + if (!defined ($result{'running'})) { + $result{'running'} = cluster_port_running ($v, $c, $result{'port'}); + } + + if ($result{'pgdata'}) { + ($result{'owneruid'}, $result{'ownergid'}) = + (stat $result{'pgdata'})[4,5]; + if ($v >= 12) { + $result{'recovery'} = 1 if (-e "$result{'pgdata'}/recovery.signal" + or -e "$result{'pgdata'}/standby.signal"); + } else { + $result{'recovery'} = 1 if (-e "$result{'pgdata'}/recovery.conf"); + } + my $waldirname = $v >= 10 ? 'pg_wal' : 'pg_xlog'; + if (-l "$result{pgdata}/$waldirname") { # custom wal directory + ($result{waldir}) = readlink("$result{pgdata}/$waldirname") =~ /(.*)/; # untaint + } + } + $result{'start'} = get_cluster_start_conf $v, $c; + + # default log file (possibly used only for early startup messages) + my $log_symlink = $result{'configdir'} . "/log"; + if (-l $log_symlink) { + ($result{'logfile'}) = readlink ($log_symlink) =~ /(.*)/; # untaint + } else { + $result{'logfile'} = "/var/log/postgresql/postgresql-$v-$c.log"; + } + + return %result; +} + + +=head2 validate_cluster_owner + + Checks if the owner of a cluster is valid, and the owner of the config matches + the owner of the data directory. + + Arguments: cluster_info hash reference + +=cut + +sub validate_cluster_owner($) { + my $info = shift; + + unless ($info->{pgdata}) { + error "Cluster data directory is unknown"; + } + unless (-d $info->{pgdata}) { + error "$info->{pgdata} is not accessible or does not exist"; + } + unless (defined $info->{owneruid}) { + error "Could not determine owner of $info->{pgdata}"; + } + if ($info->{owneruid} == 0) { + error "Data directory $info->{pgdata} must not be owned by root"; + } + unless (getpwuid $info->{owneruid}) { + error "The cluster is owned by user id $info->{owneruid} which does not exist"; + } + unless (getgrgid $info->{ownergid}) { + error "The cluster is owned by group id $info->{ownergid} which does not exist"; + } + # owneruid and configuid need to match, unless configuid is root + if (($< == 0 or $> == 0) and $info->{configuid} != 0 and + $info->{configuid} != $info->{owneruid}) { + my $configowner = (getpwuid $info->{configuid})[0] || "(unknown)"; + my $dataowner = (getpwuid $info->{owneruid})[0]; + error "Config owner ($configowner:$info->{configuid}) and data owner ($dataowner:$info->{owneruid}) do not match, and config owner is not root"; + } +} + + +=head2 get_versions + + Return an array of all available versions (by binaries and postgresql.conf files) + + Arguments: binary to scan for (optional, defaults to postgres), maximum acceptable version (optional) + +=cut + +sub get_versions { + my $program = shift // 'postgres'; + my $max_version = shift; + my %versions = (); + + # enumerate psql versions from /usr/lib/postgresql/* (or /usr/pgsql-*) + my $dir = $binroot; + #redhat# $dir = '/usr'; + if (opendir (D, $dir)) { + my $entry; + while (defined ($entry = readdir D)) { + next if $entry eq '.' || $entry eq '..'; + my $pfx = ''; + #redhat# $pfx = "pgsql-"; + my $version; + ($version) = $entry =~ /^$pfx(\d+\.?\d+)$/; # untaint + next if ($max_version and $version > $max_version); + $versions{$version} = 1 if $version and get_program_path ($program, $version); + } + closedir D; + } + + # enumerate server versions from /etc/postgresql/* + if ($program eq 'postgres' and opendir (D, $confroot)) { + my $v; + while (defined ($v = readdir D)) { + next if $v eq '.' || $v eq '..'; + ($v) = $v =~ /^(\d+\.?\d+)$/; # untaint + next unless ($v); + next if ($max_version and $v > $max_version); + + if (opendir (C, "$confroot/$v")) { + my $c; + while (defined ($c = readdir C)) { + if (-e "$confroot/$v/$c/postgresql.conf") { + $versions{$v} = 1; + last; + } + } + closedir C; + } + } + closedir D; + } + + return sort { $a <=> $b } keys %versions; +} + + +=head2 get_newest_version + + Return the newest available version + + Arguments: binary to scan for (optional), maximum acceptable version (optional) + +=cut + +sub get_newest_version { + my $program = shift; + my $max_version = shift; + my @versions = get_versions($program, $max_version); + return undef unless (@versions); + return $versions[-1]; +} + +=head2 version_exists + + Check whether a version exists + +=cut + +sub version_exists { + my ($version) = @_; + return get_program_path ('psql', $version); +} + + +=head2 get_version_clusters + + Return an array of all available clusters of given version + + Arguments: <version> + +=cut + +sub get_version_clusters { + my $vdir = $confroot.'/'.$_[0].'/'; + my @clusters = (); + if (opendir (D, $vdir)) { + my $entry; + while (defined ($entry = readdir D)) { + next if $entry eq '.' || $entry eq '..'; + ($entry) = $entry =~ /^(.*)$/; # untaint + my $conf = "$vdir$entry/postgresql.conf"; + if (-e $conf or -l $conf) { # existing file, or dead symlink + push @clusters, $entry; + } + } + closedir D; + } + return sort @clusters; +} + + +=head2 cluster_exists + + Check if a cluster exists. + + Arguments: <version> <cluster> + +=cut + +sub cluster_exists { + for my $c (get_version_clusters $_[0]) { + return 1 if $c eq $_[1]; + } + return 0; +} + + +=head2 next_free_port + + Return the next free PostgreSQL port. + +=cut + +sub next_free_port { + # create list of already used ports + my %ports; + for my $v (get_versions) { + for my $c (get_version_clusters $v) { + $ports{ get_cluster_port ($v, $c) } = 1; + } + } + + my $port; + for ($port = $defaultport; $port < 65536; ++$port) { + # port in use by existing cluster + next if (exists $ports{$port}); + + # IPv4 port in use + my ($have_ip4, $have_ip6); + if (socket (SOCK, PF_INET, SOCK_STREAM, getprotobyname('tcp'))) { + $have_ip4 = 1; + setsockopt(SOCK, Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1) or error "setsockopt: $!"; + my $res4 = bind (SOCK, sockaddr_in($port, INADDR_ANY)) and listen (SOCK, 0); + my $err = $!; + close SOCK; + next unless ($res4); + } + + # IPv6 port in use + if (exists $Socket::{"IN6ADDR_ANY"}) { + if (socket (SOCK, PF_INET6, SOCK_STREAM, getprotobyname('tcp'))) { + $have_ip6 = 1; + setsockopt(SOCK, Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1) or error "setsockopt: $!"; + my $res6 = bind (SOCK, sockaddr_in6($port, Socket::IN6ADDR_ANY)) and listen (SOCK, 0); + my $err = $!; + close SOCK; + next unless ($res6); + } + } + + unless ($have_ip4 or $have_ip6) { + # require at least one protocol to work (PostgreSQL needs it anyway + # for the stats collector) + die "could not create socket: $!"; + } + + close SOCK; + # return port if it is available on all supported protocols + return $port; + } + + die "no free port found"; +} + + +=head2 user_cluster_map + + Return the PostgreSQL version, cluster, and database to connect to. + + Version is always set (defaulting to the version of the default port + if no matching entry is found, or finally to the latest installed version + if there are no clusters at all), cluster and database may be 'undef'. + If only one cluster exists, and no matching entry is found in the map files, + that cluster is returned. + +=cut + +sub user_cluster_map { + my ($user, $pwd, $uid, $gid) = getpwuid $>; + my $group = (getgrgid $gid)[0]; + + # check per-user configuration file + my $home = $ENV{"HOME"} || (getpwuid $>)[7]; + my $homemapfile = $home . '/.postgresqlrc'; + if (open MAP, $homemapfile) { + while (<MAP>) { + s/#.*//; + next if /^\s*$/; + my ($v,$c,$db) = split; + if (!version_exists $v) { + print "Warning: $homemapfile line $.: version $v does not exist\n"; + next; + } + if (!cluster_exists $v, $c and $c !~ /^(\S+):(\d*)$/) { + print "Warning: $homemapfile line $.: cluster $v/$c does not exist\n"; + next; + } + if ($db) { + close MAP; + return ($v, $c, ($db eq "*") ? undef : $db); + } else { + print "Warning: ignoring invalid line $. in $homemapfile\n"; + next; + } + } + close MAP; + } + + # check global map file + if (open MAP, $mapfile) { + while (<MAP>) { + s/#.*//; + next if /^\s*$/; + my ($u,$g,$v,$c,$db) = split; + if (!$db) { + print "Warning: ignoring invalid line $. in $mapfile\n"; + next; + } + if (!version_exists $v) { + print "Warning: $mapfile line $.: version $v does not exist\n"; + next; + } + if (!cluster_exists $v, $c and $c !~ /^(\S+):(\d*)$/) { + print "Warning: $mapfile line $.: cluster $v/$c does not exist\n"; + next; + } + if (($u eq "*" || $u eq $user) && ($g eq "*" || $g eq $group)) { + close MAP; + return ($v,$c, ($db eq "*") ? undef : $db); + } + } + close MAP; + } + + # if only one cluster exists, use that + my $count = 0; + my ($last_version, $last_cluster, $defaultport_version, $defaultport_cluster); + for my $v (get_versions) { + for my $c (get_version_clusters $v) { + my $port = get_cluster_port ($v, $c); + $last_version = $v; + $last_cluster = $c; + if ($port == $defaultport) { + $defaultport_version = $v; + $defaultport_cluster = $c; + } + ++$count; + } + } + return ($last_version, $last_cluster, undef) if $count == 1; + + if ($count == 0) { + # if there are no local clusters, use latest clients for accessing + # network clusters + return (get_newest_version('psql'), undef, undef); + } + + # more than one cluster exists, return cluster at default port + return ($defaultport_version, $defaultport_cluster, undef); +} + + +=head2 install_file + + Copy a file to a destination and setup permissions + + Arguments: <source file> <destination file or dir> <uid> <gid> <permissions> + +=cut + +sub install_file { + my ($source, $dest, $uid, $gid, $perm) = @_; + + if (system 'install', '-o', $uid, '-g', $gid, '-m', $perm, $source, $dest) { + error "install_file: could not install $source to $dest"; + } +} + + +=head2 change_ugid + + Change effective and real user and group id. Also activates all auxiliary + groups the user is in. Exits with an error message if user/group ID cannot + be changed. + + Arguments: <user id> <group id> + +=cut + +sub change_ugid { + my ($uid, $gid) = @_; + + # auxiliary groups + my $uname = (getpwuid $uid)[0]; + prepare_exec; + my $groups = "$gid " . `/usr/bin/id -G $uname`; + restore_exec; + + $) = $groups; + $( = $gid; + $> = $< = $uid; + error 'Could not change user id' if $< != $uid; + error 'Could not change group id' if $( != $gid; +} + + +=head2 system_or_error + + Run a command and error out if it exits with a non-zero status. + + Arguments: <command ...> + +=cut + +sub system_or_error { + my $ret = system @_; + if ($ret) { + my $message = "@_ failed with exit code $ret"; + $message .= ": $!" if ($!); + error $message; + } +} + + +=head2 get_db_encoding + + Return the encoding of a particular database in a cluster. + + This requires access privileges to that database, so this + function should be called as the cluster owner. + + Arguments: <version> <cluster> <database> + Returns: Encoding or undef if it cannot be determined. + +=cut + +sub get_db_encoding { + my ($version, $cluster, $db) = @_; + my $port = get_cluster_port $version, $cluster; + my $socketdir = get_cluster_socketdir $version, $cluster; + my $psql = get_program_path 'psql', $version; + return undef unless ($port && $socketdir && $psql); + + # try to swich to cluster owner + prepare_exec 'LC_ALL'; + $ENV{'LC_ALL'} = 'C'; + my $orig_euid = $>; + $> = (stat (cluster_data_directory $version, $cluster))[4]; + open PSQL, '-|', $psql, '-h', $socketdir, '-p', $port, '-AXtc', + 'select getdatabaseencoding()', $db or + die "Internal error: could not call $psql to determine db encoding: $!"; + my $out = <PSQL>; + close PSQL; + $> = $orig_euid; + restore_exec; + return undef if $?; + chomp $out; + ($out) = $out =~ /^([\w.-]+)$/; # untaint + return $out; +} + + +=head2 get_db_locales + + Return locale of a particular database in a cluster. This requires access + privileges to that database, so this function should be called as the cluster + owner. (For versions >= 8.4; for older versions use get_cluster_locales()). + + Arguments: <version> <cluster> <database> + Returns: (LC_CTYPE, LC_COLLATE) or (undef,undef) if it cannot be determined. + PG15 adds locale provider and icu locale to the returned values + PG16 adds icu rules + +=cut + +sub get_db_locales { + my ($version, $cluster, $db) = @_; + my $port = get_cluster_port $version, $cluster; + my $socketdir = get_cluster_socketdir $version, $cluster; + my $psql = get_program_path 'psql', $version; + return undef unless ($port && $socketdir && $psql); + my ($ctype, $collate, $locale_provider, $icu_locale, $icu_rules); + + # try to switch to cluster owner + prepare_exec 'LC_ALL'; + $ENV{'LC_ALL'} = 'C'; + my $orig_euid = $>; + $> = (stat (cluster_data_directory $version, $cluster))[4]; + + open PSQL, '-|', $psql, '-h', $socketdir, '-p', $port, '-AXtc', + "SELECT datctype, datcollate FROM pg_database where datname = current_database()", $db or + die "Internal error: could not call $psql to determine datctype and datcollate: $!"; + my $out = <PSQL> // error 'could not determine datctype and datcollate'; + close PSQL; + ($out) = $out =~ /^(.*)$/; # untaint + ($ctype, $collate) = split /\|/, $out; + + if ($version >= 15) { + open PSQL, '-|', $psql, '-h', $socketdir, '-p', $port, '-AXtc', + "SELECT CASE datlocprovider::text WHEN 'c' THEN 'libc' WHEN 'i' THEN 'icu' END, daticulocale" . + ($version >= 16 ? ", daticurules" : "") . + " FROM pg_database where datname = current_database()", $db or + die "Internal error: could not call $psql to determine datlocprovider: $!"; + $out = <PSQL> // error 'could not determine datlocprovider'; + close PSQL; + ($out) = $out =~ /^(.*)$/; # untaint + ($locale_provider, $icu_locale, $icu_rules) = split /\|/, $out; + } + + $> = $orig_euid; + restore_exec; + chomp $ctype; + chomp $collate; + return ($ctype, $collate, $locale_provider, $icu_locale, $icu_rules) unless $?; + return (undef, undef); +} + + +=head2 get_cluster_locales + + Return the CTYPE and COLLATE locales of a cluster. + + This needs to be called as root or as the cluster owner. + (For versions <= 8.3; for >= 8.4, use get_db_locales()). + + Arguments: <version> <cluster> + Returns: (LC_CTYPE, LC_COLLATE) or (undef,undef) if it cannot be determined. + +=cut + +sub get_cluster_locales { + my ($version, $cluster) = @_; + my ($lc_ctype, $lc_collate) = (undef, undef); + + if ($version >= '8.4') { + print STDERR "Error: get_cluster_locales() does not work for 8.4+\n"; + exit 1; + } + + my $pg_controldata = get_program_path 'pg_controldata', $version; + if (! -e $pg_controldata) { + print STDERR "Error: pg_controldata not found, please install postgresql-$version\n"; + exit 1; + } + prepare_exec ('LC_ALL', 'LANG', 'LANGUAGE'); + $ENV{'LC_ALL'} = 'C'; + my $result = open (CTRL, '-|', $pg_controldata, (cluster_data_directory $version, $cluster)); + restore_exec; + return (undef, undef) unless defined $result; + while (<CTRL>) { + if (/^LC_CTYPE\W*(\S+)\s*$/) { + $lc_ctype = $1; + } elsif (/^LC_COLLATE\W*(\S+)\s*$/) { + $lc_collate = $1; + } + } + close CTRL; + return ($lc_ctype, $lc_collate); +} + + +=head2 get_cluster_controldata + + Return the pg_control data for a cluster + + Arguments: <version> <cluster> + Returns: hashref + +=cut + +sub get_cluster_controldata { + my ($version, $cluster) = @_; + + my $pg_controldata = get_program_path 'pg_controldata', $version; + if (! -e $pg_controldata) { + print STDERR "Error: pg_controldata not found, please install postgresql-$version\n"; + exit 1; + } + prepare_exec ('LC_ALL', 'LANG', 'LANGUAGE'); + $ENV{'LC_ALL'} = 'C'; + my $result = open (CTRL, '-|', $pg_controldata, (cluster_data_directory $version, $cluster)); + restore_exec; + return undef unless defined $result; + my $data = {}; + while (<CTRL>) { + if (/^(.+?):\s*(.*)/) { + $data->{$1} = $2; + } else { + error "Invalid pg_controldata output: $_"; + } + } + close CTRL; + return $data; +} + + +=head2 get_cluster_databases + + Return an array with all databases of a cluster. + + This requires connection privileges to template1, so + this function should be called as the cluster owner. + + Arguments: <version> <cluster> + Returns: array of database names or undef on error. + +=cut + +sub get_cluster_databases { + my ($version, $cluster) = @_; + my $port = get_cluster_port $version, $cluster; + my $socketdir = get_cluster_socketdir $version, $cluster; + my $psql = get_program_path 'psql', $version; + return undef unless ($port && $socketdir && $psql); + + # try to swich to cluster owner + prepare_exec 'LC_ALL'; + $ENV{'LC_ALL'} = 'C'; + my $orig_euid = $>; + $> = (stat (cluster_data_directory $version, $cluster))[4]; + + my @dbs; + my @fields; + if (open PSQL, '-|', $psql, '-h', $socketdir, '-p', $port, '-AXtl') { + while (<PSQL>) { + chomp; + @fields = split '\|'; + next if $#fields < 2; # remove access privs which get line broken + push (@dbs, $fields[0]); + } + close PSQL; + } + + $> = $orig_euid; + restore_exec; + + return $? ? undef : @dbs; +} + + +=head2 get_file_device + + Return the device name a file is stored at. + + Arguments: <file path> + Returns: device name, or '' if it cannot be determined. + +=cut + +sub get_file_device { + my $dev = ''; + prepare_exec; + my $pid = open3(\*CHLD_IN, \*CHLD_OUT, \*CHLD_ERR, '/bin/df', $_[0]); + waitpid $pid, 0; # we simply ignore exit code and stderr + while (<CHLD_OUT>) { + if (/^\/dev/) { + $dev = (split)[0]; + } + } + restore_exec; + close CHLD_IN; + close CHLD_OUT; + close CHLD_ERR; + return $dev; +} + + +=head2 parse_hba_line + + Parse a single pg_hba.conf line. + + Arguments: <line> + Returns: Hash reference (or only line and type==undef for invalid lines) + +=over 4 + +=item * + +line -> the verbatim pg_hba line + +=item * + +type -> comment, local, host, hostssl, hostnossl, undef + +=item * + +db -> database name + +=item * + +user -> user name + +=item * + +method -> trust, reject, md5, crypt, password, krb5, ident, pam + +=item * + +ip -> ip address + +=item * + +mask -> network mask (either a single number as number of bits, or bit mask) + +=back + +=cut + +sub parse_hba_line { + my $l = $_[0]; + chomp $l; + + # comment line? + return { 'type' => 'comment', 'line' => $l } if ($l =~ /^\s*($|#)/); + + my $res = { 'line' => $l }; + my @tok = split /\s+/, $l; + goto error if $#tok < 3; + + $$res{'type'} = shift @tok; + $$res{'db'} = shift @tok; + $$res{'user'} = shift @tok; + + # local connection? + if ($$res{'type'} eq 'local') { + goto error if $#tok > 1; + goto error unless valid_hba_method($tok[0]); + $$res{'method'} = join (' ', @tok); + return $res; + } + + # host connection? + if ($$res{'type'} =~ /^host((no)?ssl)?$/) { + my ($i, $c) = split '/', (shift @tok); + goto error unless $i; + $$res{'ip'} = $i; + + # CIDR mask given? + if (defined $c) { + goto error if $c !~ /^(\d+)$/; + $$res{'mask'} = $c; + } else { + $$res{'mask'} = shift @tok; + } + + goto error if $#tok > 1; + goto error unless valid_hba_method($tok[0]); + $$res{'method'} = join (' ', @tok); + return $res; + } + +error: + $$res{'type'} = undef; + return $res; +} + + +=head2 read_pg_hba + + Parse given pg_hba.conf file. + + Arguments: <pg_hba.conf path> + Returns: Array with hash refs; for hash contents, see parse_hba_line(). + +=cut + +sub read_pg_hba { + open HBA, $_[0] or return undef; + my @hba; + while (<HBA>) { + my $r = parse_hba_line $_; + push @hba, $r; + } + close HBA; + return @hba; +} + + +=head2 valid_hba_method + + Check if hba method is known + + Argument: hba method + Returns: True if method is valid + +=cut + +sub valid_hba_method { + my $method = $_[0]; + + my %valid_methods = qw/trust 1 reject 1 md5 1 crypt 1 password 1 krb5 1 ident 1 pam 1/; + + return exists($valid_methods{$method}); +} + +1; diff --git a/README.md b/README.md new file mode 100644 index 0000000..5e48b28 --- /dev/null +++ b/README.md @@ -0,0 +1,114 @@ +Multi-Version/Multi-Cluster PostgreSQL architecture +=================================================== +2004, Oliver Elphick, Martin Pitt + +Solving a problem +----------------- + +When a new major version of PostgreSQL is released, it is necessary to dump and +reload the database. The old software must be used for the dump, and the new +software for the reload. + +This was a major problem for RedHat and Debian, because a dump and reload was +not required by every upgrade and by the time the need for a dump is realised, +the old software might have been deleted. Debian had certain rather unreliable +procedures to save the old software and use it to do a dump, but these +procedures often went wrong. RedHat's installation environment is so rigid that +it is not practicable for the RedHat packages to attempt an automatic upgrade. +Debian offered a debconf choice for whether to attempt automatic upgrading; if +it failed or was not allowed, a manual upgrade had to be done, either from a +pre-existing dump or by manual invocation of the postgresql-dump script. + +It is possible to run different versions of PostgreSQL simultaneously, and +indeed to run the same version on separate database clusters simultaneously. To +do so, each postgres instance must listen on a different port, so each client +must specify the correct port. By having two separate versions of the +PostgreSQL packages installed simultaneously, it is simple to do database +upgrades by dumping from the old version and uploading to the new. The +PostgreSQL client wrapper is designed to permit this. + +General Architecture idea +------------------------- + +The Debian packaging has been changed to create a new package for each major +version. The criterion for creating a new package is that initdb is required +when upgrading from the previous version. Thus, there are now source packages +`postgresql-8.1` and `postgresql-8.3` (and similarly for all the binary +packages). + +The legacy postgresql and the other existing binary package names have become +dummy packages depending on one of the versioned equivalents. Their only +purpose is now to ensure a smooth upgrade and to register the existing database +cluster to the new architecture. These packages will be removed from the +archive as soon as the next Debian release after Sarge (Etch) is released. + +Each versioned package installs into `/usr/lib/postgresql/version`. In order +to allow users easily to select the right version and cluster when working, the +`postgresql-common` package provides the `pg_wrapper` program, which reads the +per-user and system wide configuration file and forks the correct executable +with the correct library versions according to those preferences. `/usr/bin` +provides executables soft-linked to `pg_wrapper`. + +This architecture also allows separate database clusters to be maintained for +the use of different groups of users; these clusters need not all be of the +same major version. This allows much greater flexibility for those people who +need to make application software changes consequent on a PostgreSQL upgrade. + +Detailed structure +------------------ + +### Configuration hierarchy + +* `/etc/postgresql-common/user_clusters`: maps users against clusters and + default databases + +* `$HOME/.postgresqlrc`: per-user preferences for default version/cluster and + database; overrides `/etc/postgresql-common/user_clusters` + +* `/etc/postgresql/version/clustername`: cluster-specific configuration files: + + * `postgresql.conf`, `pg_hba.conf`, `pg_ident.conf` + * optionally `start.conf`: startup mode of the cluster: `auto` (start/stop in + init script), `manual` (do not start/stop in init script, but manual + control with `pg_ctlcluster` is possible), `disabled` (`pg_ctlcluster` + is not allowed). + * optionally `pg_ctl.conf`: options to be passed to `pg_ctl`. + * optionally a symbolic link `log` which points to the postgres log file. + Defaults to `/var/log/postgresql/postgresql-version-cluster.conf`. + Explicitly setting `log_directory` and/or `log_filename` in + `postgresql.conf` overrides this. + +### Per-version files and programs + +* `/usr/lib/postgresql/version` +* `/usr/share/postgresql/version` +* `/usr/share/doc/postgresql/postgresql-doc-version`: +version specific program and data files + +### Common programs + +* `/usr/share/postgresql-common/pg_wrapper`: environment chooser and program selector +* `/usr/bin/program`: symbolic links to pg_wrapper, for all client programs +* `/usr/bin/pg_lsclusters`: list all available clusters with their status and configuration +* `/usr/bin/pg_createcluster: wrapper for `initdb`, sets up the necessary configuration structure +* `/usr/bin/pg_ctlcluster`: wrapper for `pg_ctl`, control the cluster postgres server +* `/usr/bin/pg_upgradecluster`: upgrade a cluster to a newer major version +* `/usr/bin/pg_dropcluster`: remove a cluster and its configuration + +### /etc/init.d/postgresql + +This script handles the postgres server processes for each version and all +their clusters. However, most of the actual work is done by the new +`pg_ctlcluster` program. + +### pg_upgradecluster + +This program replaces postgresql-dump (a Debian specific program). + +It is used to migrate a cluster from one major version to another. + +Usage: `pg_upgradecluster [-v newversion] version name [data_dir]` + +`-v`: specifies the version to upgrade to; defaults to the newest available version. + + -- The Debian PostgreSQL maintainers @@ -0,0 +1,39 @@ +postgresql TODO +=============== + +Bugs: +- pg_createcluster with existing cluster: respect symlinks to shared + postgresql.conf files, never remove symlinks in already existing + cluster dirs +- Clean up at purging if /etc/ is there without /var, or vice versa + +Transition bugs: + +Missing selftest: +- --force option for pg_ctlcluster + +Important features: + +Wishlist: +- Add pg_conf to change startup and possibly other things +- add program for web applications, which configure pg_hba.conf: + + pg_hba add|remove|test [options] yourwebappdb yourwebappuser + + Options: + + --cluster: self-explanatory, defaults to default cluster + --ip: IP and netmask for host socket; if not given, defaults to Unix + socket (local) + --method: defaults to "md5" for TCP connections, and "ident sameuser" for + Unix socket connections + --force-ssl: If given, create a "hostssl" entry, otherwise a "host" + entry + + For remove, only --cluster is allowed; it will remove all hba + entries that refer to the given db/user pair. test checks whether the + given connection is allowed; if so, it exits with 0, otherwise it prints the + required pg_hba.conf line to stdout and exits with 1. If pg_hba.conf has a + scrambled format that cannot be parsed by pg_*_hba, the scripts exit with 2. + + Add libnet-cidr-perl dependency! @@ -0,0 +1,24 @@ +#!/bin/sh + +# Clean all traces of PostgreSQL data/configuration from the current system. +# Use with caution, for development/testing only! (This is not installed into +# packages.) +# +# (C) 2008-2009 Martin Pitt <mpitt@debian.org> +# (C) 2018 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +set -ux + +/etc/init.d/postgresql stop +killall postgres +rm -rf /etc/postgresql/* /var/lib/postgresql/* /var/log/postgresql/* /tmp/postgresql-testsuite/ /var/run/postgresql/* diff --git a/createcluster.conf b/createcluster.conf new file mode 100644 index 0000000..5e66693 --- /dev/null +++ b/createcluster.conf @@ -0,0 +1,41 @@ +# Default values for pg_createcluster(8) +# Occurrences of '%v' are replaced by the major version number, +# and '%c' by the cluster name. Use '%%' for a literal '%'. + +# Create a "main" cluster when a new postgresql-x.y server package is installed +#create_main_cluster = true + +# Default start.conf value, must be one of "auto", "manual", and "disabled". +# See pg_createcluster(8) for more documentation. +#start_conf = 'auto' + +# Default data directory. +#data_directory = '/var/lib/postgresql/%v/%c' + +# Default directory for transaction logs +# Unset by default, i.e. transaction logs remain in the data directory. +#waldir = '/var/lib/postgresql/wal/%v/%c/pg_wal' + +# Options to pass to initdb. +#initdb_options = '' + +# The following options are copied into the new cluster's postgresql.conf: + +# Enable SSL by default (using the "snakeoil" certificates installed by the +# ssl-cert package, unless configured otherwise here) +ssl = on + +# Show cluster name in process title +cluster_name = '%v/%c' + +# Put stats_temp_directory on tmpfs (PG <= 14) +stats_temp_directory = '/var/run/postgresql/%v-%c.pg_stat_tmp' + +# Add prefix to log lines +log_line_prefix = '%%m [%%p] %%q%%u@%%d ' + +# Add "include_dir" in postgresql.conf +add_include_dir = 'conf.d' + +# Directory for additional createcluster config +include_dir '/etc/postgresql-common/createcluster.d' diff --git a/debhelper/Debian/Debhelper/Buildsystem/pgxs.pm b/debhelper/Debian/Debhelper/Buildsystem/pgxs.pm new file mode 100644 index 0000000..f1270ea --- /dev/null +++ b/debhelper/Debian/Debhelper/Buildsystem/pgxs.pm @@ -0,0 +1,58 @@ +# A debhelper build system class for building PostgreSQL extension modules using PGXS +# +# Per PostgreSQL major version, a `build-$version` subdirectory is created. +# +# Copyright: © 2020 Christoph Berg +# License: GPL-2+ + +package Debian::Debhelper::Buildsystem::pgxs; + +use strict; +use warnings; +use parent qw(Debian::Debhelper::Buildsystem); +use Cwd; +use Debian::Debhelper::Dh_Lib; +use Debian::Debhelper::pgxs; + +sub DESCRIPTION { + "PGXS (PostgreSQL extensions), building in subdirectory per PostgreSQL version" +} + +sub check_auto_buildable { + my $this=shift; + unless (-e $this->get_sourcepath("debian/pgversions")) { + error("debian/pgversions is required to build with PGXS"); + } + return (-e $this->get_sourcepath("Makefile")) ? 1 : 0; +} + +sub new { + my $class=shift; + my $this=$class->SUPER::new(@_); + $this->enforce_in_source_building(); + return $this; +} + +sub build { + my $this=shift; + $this->doit_in_sourcedir(qw(pg_buildext build build-%v)); +} + +sub install { + my $this=shift; + my $pattern = package_pattern(); + $this->doit_in_sourcedir(qw(pg_buildext install build-%v), $pattern); +} + +sub test { + my $this=shift; + verbose_print("Postponing tests to install stage"); +} + +sub clean { + my $this=shift; + my $pattern = package_pattern(); + $this->doit_in_sourcedir(qw(pg_buildext clean build-%v), $pattern); +} + +1; diff --git a/debhelper/Debian/Debhelper/Buildsystem/pgxs_loop.pm b/debhelper/Debian/Debhelper/Buildsystem/pgxs_loop.pm new file mode 100644 index 0000000..716bcbf --- /dev/null +++ b/debhelper/Debian/Debhelper/Buildsystem/pgxs_loop.pm @@ -0,0 +1,33 @@ +# A debhelper build system class for building PostgreSQL extension modules using PGXS +# +# For packages not supporting building in subdirectories, the pgxs_loop variant builds +# for each PostgreSQL major version in turn in the top-level directory. +# +# Copyright: © 2020 Christoph Berg +# License: GPL-2+ + +package Debian::Debhelper::Buildsystem::pgxs_loop; + +use strict; +use warnings; +use parent qw(Debian::Debhelper::Buildsystem::pgxs); +use Cwd; +use Debian::Debhelper::Dh_Lib; +use Debian::Debhelper::pgxs; + +sub DESCRIPTION { + "PGXS (PostgreSQL extensions), building for each PostgreSQL version in top level directory" +} + +sub build { + my $this=shift; + verbose_print("Postponing build to install stage; if this package supports out-of-tree builds, replace --buildsystem=pgxs_loop by --buildsystem=pgxs to build in the build stage"); +} + +sub install { + my $this=shift; + my $pattern = package_pattern(); + $this->doit_in_sourcedir(qw(pg_buildext loop), $pattern); +} + +1; diff --git a/debhelper/Debian/Debhelper/Sequence/pgxs.pm b/debhelper/Debian/Debhelper/Sequence/pgxs.pm new file mode 100644 index 0000000..45a1ff1 --- /dev/null +++ b/debhelper/Debian/Debhelper/Sequence/pgxs.pm @@ -0,0 +1,22 @@ +#!/usr/bin/perl + +use warnings; +use strict; +use Debian::Debhelper::Dh_Lib; + +# check if debian/control needs updating from debian/control.in +insert_after("dh_clean", "pg_buildext"); +add_command_options("pg_buildext", "checkcontrol"); + +# use PGXS for clean, build, and install +add_command_options("dh_auto_clean", "--buildsystem=pgxs"); +add_command_options("dh_auto_build", "--buildsystem=pgxs"); +add_command_options("dh_auto_install", "--buildsystem=pgxs"); + +# move tests from dh_auto_test to dh_pgxs_test +remove_command("dh_auto_test"); +if (! get_buildoption("nocheck")) { + insert_after("dh_link", "dh_pgxs_test"); +} + +1; diff --git a/debhelper/Debian/Debhelper/Sequence/pgxs_loop.pm b/debhelper/Debian/Debhelper/Sequence/pgxs_loop.pm new file mode 100644 index 0000000..314085b --- /dev/null +++ b/debhelper/Debian/Debhelper/Sequence/pgxs_loop.pm @@ -0,0 +1,23 @@ +#!/usr/bin/perl + +use warnings; +use strict; +use Debian::Debhelper::Dh_Lib; + +# check if debian/control needs updating from debian/control.in +insert_after("dh_clean", "pg_buildext"); +add_command_options("pg_buildext", "checkcontrol"); + +# use PGXS for clean, build, and install +add_command_options("dh_auto_clean", "--buildsystem=pgxs_loop"); +add_command_options("dh_auto_build", "--buildsystem=pgxs_loop"); +add_command_options("dh_auto_install", "--buildsystem=pgxs_loop"); + +# move tests from dh_auto_test to dh_pgxs_test +remove_command("dh_auto_test"); +if (! get_buildoption("nocheck")) { + insert_after("dh_link", "dh_pgxs_test"); + add_command_options("dh_pgxs_test", "loop"); +} + +1; diff --git a/debhelper/Debian/Debhelper/pgxs.pm b/debhelper/Debian/Debhelper/pgxs.pm new file mode 100644 index 0000000..e3d86b2 --- /dev/null +++ b/debhelper/Debian/Debhelper/pgxs.pm @@ -0,0 +1,38 @@ +# A debhelper build system class for building PostgreSQL extension modules using PGXS +# +# Copyright: © 2020 Christoph Berg +# License: GPL-2+ + +package Debian::Debhelper::pgxs; + +use strict; +use warnings; +use Exporter 'import'; +our @EXPORT = qw(package_pattern); + +=head1 package_pattern() + +From C<debian/control.in>, look for the package name containing the +B<PGVERSION> placeholder, and return it in the format suitable for passing to +B<pg_buildext>, i.e. with B<PGVERSION> replaced by B<%v>. + +For B<Package: postgresql-PGVERSION-unit> it will return B<postgresql-%v-unit>. + +Errors out if more than one package with the B<PGVERSION> placeholder is found. + +=cut + +sub package_pattern () { + open F, "debian/control.in" or die "debian/control.in: $!"; + my $pattern; + while (<F>) { + if (/^Package: (.*)PGVERSION(.*)/) { + die "More than one Package with PGVERSION placeholder found in debian/control.in, cannot build with dh --buildsystem=pgxs. Use pg_buildext manually." if ($pattern); + $pattern = "$1%v$2"; + } + } + close F; + return $pattern; +} + +1; diff --git a/debhelper/dh_pgxs_test b/debhelper/dh_pgxs_test new file mode 100755 index 0000000..c12bacd --- /dev/null +++ b/debhelper/dh_pgxs_test @@ -0,0 +1,11 @@ +#!/usr/bin/perl + +use warnings; +use strict; +use Debian::Debhelper::Dh_Lib; +use Debian::Debhelper::pgxs; + +my $target = (@ARGV and $ARGV[0] eq 'loop') ? "." : "build-%v"; +my $pattern = package_pattern(); + +print_and_doit(qw(pg_buildext installcheck .), $target, $pattern); diff --git a/debhelper/dh_pgxs_test.pod b/debhelper/dh_pgxs_test.pod new file mode 100644 index 0000000..5b24e4b --- /dev/null +++ b/debhelper/dh_pgxs_test.pod @@ -0,0 +1,44 @@ +=head1 NAME + +dh_pgxs_test - Run testsuite during a PGXS PostgreSQL extension build + +=head1 SYNOPSIS + +B<dh_pgxs_test> [B<loop>] + +=head1 DESCRIPTION + +B<PostgreSQL> extensions need to be installed before they can be tested and +hence the usual B<debhelper> way of invoking tests from dh_auto_test(1) does +not work. + +B<dh_pgxs_test> is a dh(1) sequence point created by the B<pgxs> and +B<pgxs_loop> B<debhelper> extensions that is executed after dh_auto_install(1). +It calls B<pg_buildext installcheck> after a B<PostgreSQL> extension module has +been built and installed into the C<debian/>I<packagename/> directory. + +Users wishing to change the action called by B<dh_pgxs_test> should call +B<pg_buildext> or similar commands. + + override_dh_pgxs_test: + echo "CREATE EXTENSION foo" | pg_buildext psql . . postgresql-%v-foo + +=head1 OPTIONS + +=over 4 + +=item B<loop> + +B<dh --with pgxs> builds packages in C<build-%v> subdirectories. The B<loop> +options corresponds to B<dh --with pgxs_loop> and builds in the top-level +directory. + +=back + +=head1 SEE ALSO + +debhelper(7), dh(1), dh_make_pgxs(1), pg_buildext(1). + +=head1 AUTHOR + +Christoph Berg L<E<lt>myon@debian.orgE<gt>> diff --git a/dh_make_pgxs/debian/control.in b/dh_make_pgxs/debian/control.in new file mode 100644 index 0000000..090ac72 --- /dev/null +++ b/dh_make_pgxs/debian/control.in @@ -0,0 +1,23 @@ +Source: @SOURCE@ +Section: database +Priority: optional +Maintainer: Debian PostgreSQL Maintainers <team+postgresql@tracker.debian.org> +Uploaders: + @MAINTAINER_NAME@ <@DEBEMAIL@>, +Build-Depends: + debhelper-compat (= @COMPAT@), + postgresql-all (>= 217~), +Standards-Version: @STANDARDS_VERSION@ +Rules-Requires-Root: no +Homepage: @URL@ +Vcs-Browser: https://salsa.debian.org/postgresql/@SOURCE@ +Vcs-Git: https://salsa.debian.org/postgresql/@SOURCE@.git + +Package: postgresql-PGVERSION-@EXTNAME@ +Architecture: any +Depends: + ${misc:Depends}, + ${postgresql:Depends}, + ${shlibs:Depends}, +Description: FIXME PostgreSQL extension @SOURCE@ + FIXME long description here diff --git a/dh_make_pgxs/debian/copyright b/dh_make_pgxs/debian/copyright new file mode 100644 index 0000000..60ca5f0 --- /dev/null +++ b/dh_make_pgxs/debian/copyright @@ -0,0 +1,24 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: @NAME@ +Source: @URL@ + +Files: * +Copyright: Portions Copyright (c) 1996-@YEAR@, PostgreSQL Global Development Group + Portions Copyright (c) 1994, The Regents of the University of California +License: PostgreSQL + Permission to use, copy, modify, and distribute this software and its + documentation for any purpose, without fee, and without a written agreement + is hereby granted, provided that the above copyright notice and this + paragraph and the following two paragraphs appear in all copies. + . + IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR + DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS + DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + . + THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS + ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO + PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. diff --git a/dh_make_pgxs/debian/gitlab-ci.yml b/dh_make_pgxs/debian/gitlab-ci.yml new file mode 100644 index 0000000..67e4816 --- /dev/null +++ b/dh_make_pgxs/debian/gitlab-ci.yml @@ -0,0 +1 @@ +include: https://salsa.debian.org/postgresql/postgresql-common/raw/master/gitlab/gitlab-ci.yml diff --git a/dh_make_pgxs/debian/pgversions b/dh_make_pgxs/debian/pgversions new file mode 100644 index 0000000..0702cb5 --- /dev/null +++ b/dh_make_pgxs/debian/pgversions @@ -0,0 +1 @@ +all diff --git a/dh_make_pgxs/debian/rules b/dh_make_pgxs/debian/rules new file mode 100755 index 0000000..5c0ff40 --- /dev/null +++ b/dh_make_pgxs/debian/rules @@ -0,0 +1,36 @@ +#!/usr/bin/make -f + +%: + dh $@ --with pgxs + +override_dh_installdocs: + dh_installdocs --all README.* + +# if the package does not support building from subdirectories, use +# `--with pgxs_loop` above. +# +# change the way tests are run: +# override_dh_pgxs_test: +# +pg_buildext installcheck . . postgresql-%v-@EXTNAME@ + +# classic `pg_buildext` interface: + +#include /usr/share/postgresql-common/pgxs_debian_control.mk +# +#override_dh_auto_build: +# +pg_buildext build build-%v +# +#override_dh_auto_test: +# # nothing to do here, see debian/tests/* instead +# +#override_dh_auto_install: +# +pg_buildext install build-%v postgresql-%v-@EXTNAME@ +# +#override_dh_installdocs: +# dh_installdocs --all README.* +# +#override_dh_auto_clean: +# +pg_buildext clean build-%v +# +#%: +# dh $@ diff --git a/dh_make_pgxs/debian/source/format b/dh_make_pgxs/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/dh_make_pgxs/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/dh_make_pgxs/debian/tests/control b/dh_make_pgxs/debian/tests/control new file mode 100644 index 0000000..74b0464 --- /dev/null +++ b/dh_make_pgxs/debian/tests/control @@ -0,0 +1,5 @@ +Depends: + make, + @, +Tests: installcheck +Restrictions: allow-stderr diff --git a/dh_make_pgxs/debian/tests/installcheck b/dh_make_pgxs/debian/tests/installcheck new file mode 100755 index 0000000..5a20e78 --- /dev/null +++ b/dh_make_pgxs/debian/tests/installcheck @@ -0,0 +1,3 @@ +#!/bin/sh + +pg_buildext installcheck diff --git a/dh_make_pgxs/debian/watch b/dh_make_pgxs/debian/watch new file mode 100644 index 0000000..acfb2a8 --- /dev/null +++ b/dh_make_pgxs/debian/watch @@ -0,0 +1,2 @@ +version=4 +@URL@/tags .*/(.*).tar.gz diff --git a/dh_make_pgxs/dh_make_pgxs b/dh_make_pgxs/dh_make_pgxs new file mode 100755 index 0000000..6c59b8a --- /dev/null +++ b/dh_make_pgxs/dh_make_pgxs @@ -0,0 +1,136 @@ +#!/bin/bash + +# (C) 2015-2020 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +set -eu + +# basic variables + +template_dir="/usr/share/postgresql-common/dh_make_pgxs/debian" + +DIRECTORY="$(basename $PWD)" +NAME="${DIRECTORY%-*}" # Upstream name +VERSION="${DIRECTORY##*-}" +URL="https://FIXME/$NAME" + +# options + +while getopts "fh:n:v:" opt ; do + case $opt in + f) FORCE=yes ;; + h) URL="$OPTARG" ;; + n) NAME="$OPTARG" ;; + v) VERSION="$OPTARG" ;; + *) exit 5 ;; + esac +done +shift $((OPTIND - 1)) # shift away args + +# more variables + +SOURCE="${NAME//_/-}" # Debian name +EXTNAME="$(echo $SOURCE | sed -e 's/^\(postgresql\|pgsql\|pg\)-//')" # binary package name suffix +DHVERSION="$(dpkg-query --showformat '${Version}' --show debhelper)" +COMPAT="${DHVERSION%%.*}" +STANDARDS_VERSION="$(apt-cache show debian-policy | sed -n 's/Version: \(.*\)\..*/\1/p' | head -n1)" +USERNAME=${LOGNAME:-${USER:-root}} +MAINTAINER_NAME=$(getent passwd $USERNAME | cut -d : -f 5 | sed -e 's/,.*//') +: ${DEBEMAIL:=$USERNAME@localhost} +YEAR=$(date +%Y) + +echo "Upstream (-n): $NAME" +echo "Version (-v): $VERSION" +echo "Source: $SOURCE ($VERSION-1)" +echo "Binaries: postgresql-PGVERSION-$EXTNAME ($VERSION-1)" +echo "Uploader: $MAINTAINER_NAME <$DEBEMAIL>" +echo "Homepage (-h): $URL" +echo +if [ -t 0 ]; then + echo -n "Press Enter to continue, ^C to abort " + read +fi + +# install files + +install_dir () +{ + local directory="debian/$1" + #[ -z "$directory" ] && return + [ -d "$directory" ] && return + echo "Creating $directory/" + mkdir "$directory" +} + +install_template () +{ + local template="$1" + + if [ "${FORCE:-}" ] || ! [ -e debian/$template ]; then + echo "Installing debian/$template" + sed -e "s/@COMPAT@/$COMPAT/g" \ + -e "s/@EXTNAME@/$EXTNAME/g" \ + -e "s/@NAME@/$NAME/g" \ + -e "s/@STANDARDS_VERSION@/$STANDARDS_VERSION/g" \ + -e "s/@SOURCE@/$SOURCE/g" \ + -e "s/@MAINTAINER_NAME@/$MAINTAINER_NAME/g" \ + -e "s/@DEBEMAIL@/$DEBEMAIL/g" \ + -e "s;@URL@;$URL;g" \ + -e "s/@YEAR@/$YEAR/g" \ + "$template_dir/$template" > "debian/$template" + if [ -x $template_dir/$template ]; then + chmod +x "debian/$template" + fi + else + echo "Keeping existing debian/$template" + fi +} + +mkdir -p debian + +for template in $(find $template_dir -mindepth 1 | sort); do + case $template in + *.swp|*~) continue ;; # skip vim stuff + esac + basename=${template##$template_dir/} + if [ -d $template ]; then + install_dir "$basename" + else + install_template "$basename" + fi +done + +if [ "$COMPAT" -lt "12" ]; then + sed -i -e "s/debhelper-compat[^,]*/debhelper (>= $COMPAT)/" debian/control* + echo "$COMPAT" > debian/compat +fi + +echo "Updating debian/control from debian/control.in" +pg_buildext updatecontrol + +if [ "${FORCE:-}" ] || ! [ -e debian/changelog ]; then + rm -f debian/changelog + echo "Creating debian/changelog" + if [ -x /usr/bin/dch ]; then + dch --create --package "$SOURCE" --newversion "$VERSION-1" + else + cat > debian/changelog <<-EOT + $SOURCE ($VERSION-1) UNRELEASED; urgency=medium + + * Initial release. (Closes: #XXXXXX) + + -- $MAINTAINER_NAME <$DEBEMAIL> $(date -R) + EOT + fi +else + echo "Keeping existing debian/changelog" +fi diff --git a/dh_make_pgxs/dh_make_pgxs.pod b/dh_make_pgxs/dh_make_pgxs.pod new file mode 100644 index 0000000..8efd172 --- /dev/null +++ b/dh_make_pgxs/dh_make_pgxs.pod @@ -0,0 +1,43 @@ +=head1 NAME + +dh_make_pgxs - Create a new Debian source package for a PGXS PostgreSQL extension + +=head1 SYNOPSIS + +B<dh_make_pgxs> [B<-f>] [B<-h> I<URL>] [B<-n> I<name>] [B<-v> I<version>] + +=head1 DESCRIPTION + +B<dh_make_pgxs> creates a F<debian/> directory tree for PostgreSQL extension +packages using the PGXS build system. The B<pg_buildext> tool is used for the +build process. + +=head1 OPTIONS + +=over 4 + +=item B<-f> + +Overwrite existing files. + +=item B<-h> I<URL> + +Package upstream homepage. + +=item B<-n> I<name> + +Package name to use. Default is to extract it from the current directory's name. + +=item B<-v> I<version> + +Package version to use. Default is to extract it from the current directory's name. + +=back + +=head1 SEE ALSO + +dh_make(1), pg_buildext(1). + +=head1 AUTHOR + +Christoph Berg L<E<lt>myon@debian.orgE<gt>> diff --git a/doc/dependencies.dia b/doc/dependencies.dia Binary files differnew file mode 100644 index 0000000..ba23097 --- /dev/null +++ b/doc/dependencies.dia diff --git a/doc/dependencies.svg b/doc/dependencies.svg new file mode 100644 index 0000000..b4fd8a4 --- /dev/null +++ b/doc/dependencies.svg @@ -0,0 +1,206 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" "http://www.w3.org/TR/2001/PR-SVG-20010719/DTD/svg10.dtd"> +<svg width="65cm" height="24cm" viewBox="40 80 1300 480" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"> + <defs/> + <g id="Hintergrund"> + <rect style="fill: #d8e5e5; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2.35099e-37; stroke: #000000" x="40" y="80" width="1300" height="480" rx="0" ry="0"/> + <rect style="fill: #90ee90; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2.35099e-37; stroke: #000000" x="60" y="260" width="1260" height="200" rx="0" ry="0"/> + <g> + <rect style="fill: #84bef6; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="660" y="180" width="140" height="54" rx="0" ry="0"/> + <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="730" y="202.881"> + <tspan x="730" y="202.881">postgresql:all</tspan> + <tspan x="730" y="218.881">M-A:none</tspan> + </text> + </g> + <g> + <rect style="fill: #ffa500; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="380" y="180" width="180" height="54" rx="0" ry="0"/> + <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="470" y="202.881"> + <tspan x="470" y="202.881">postgresql-client:all</tspan> + <tspan x="470" y="218.881">M-A:foreign</tspan> + </text> + </g> + <g> + <line style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x1="470" y1="234.988" x2="470" y2="268.007"/> + <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="465,268.007 470,278.007 475,268.007 "/> + </g> + <g> + <rect style="fill: #84bef6; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="620" y="280" width="224.85" height="70" rx="0" ry="0"/> + <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="732.425" y="302.881"> + <tspan x="732.425" y="302.881">postgresql-NN:any</tspan> + <tspan x="732.425" y="318.881">M-A:none</tspan> + <tspan x="732.425" y="334.881">Provides: postgresql-contrib-NN</tspan> + </text> + </g> + <g> + <rect style="fill: #ffa500; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="360" y="280" width="220" height="60" rx="0" ry="0"/> + <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="470" y="305.881"> + <tspan x="470" y="305.881">postgresql-client-NN:any</tspan> + <tspan x="470" y="321.881">M-A:foreign</tspan> + </text> + </g> + <g> + <rect style="fill: #ffa500; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="640" y="480" width="180" height="54" rx="0" ry="0"/> + <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="730" y="502.881"> + <tspan x="730" y="502.881">postgresql-common:all</tspan> + <tspan x="730" y="518.881">M-A:foreign</tspan> + </text> + </g> + <g> + <rect style="fill: #ffa500; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="360" y="480" width="220" height="54" rx="0" ry="0"/> + <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="470" y="502.881"> + <tspan x="470" y="502.881">postgresql-client-common:all</tspan> + <tspan x="470" y="518.881">M-A:foreign</tspan> + </text> + </g> + <g> + <line style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x1="730.628" y1="234.976" x2="731.37" y2="268.012"/> + <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="726.371,268.124 731.594,278.009 736.369,267.899 "/> + </g> + <g> + <line style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x1="731.97" y1="351.006" x2="730.493" y2="468.005"/> + <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="725.493,467.942 730.366,478.004 735.492,468.068 "/> + </g> + <g> + <line style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x1="470" y1="341.004" x2="470" y2="467.996"/> + <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="465,467.996 470,477.996 475,467.996 "/> + </g> + <g> + <line style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x1="619.008" y1="312.839" x2="592.005" y2="312.325"/> + <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="592.1,307.325 582.007,312.134 591.91,317.324 "/> + </g> + <g> + <line style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x1="638.998" y1="507" x2="591.997" y2="507"/> + <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="591.997,502 581.997,507 591.997,512 "/> + </g> + <g> + <line style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x1="680.616" y1="350.994" x2="649.334" y2="372.726"/> + <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="646.481,368.62 641.121,378.432 652.186,376.833 "/> + </g> + <g> + <line style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x1="511.553" y1="341.005" x2="553.653" y2="372.418"/> + <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="550.663,376.426 561.668,378.398 556.643,368.411 "/> + </g> + <g> + <rect style="fill: #ffff00; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="540" y="380" width="120" height="54" rx="0" ry="0"/> + <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="600" y="402.881"> + <tspan x="600" y="402.881">libpq5:amd64</tspan> + <tspan x="600" y="418.881">M-A:same</tspan> + </text> + </g> + <g> + <rect style="fill: #84bef6; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="140" y="380" width="140" height="54" rx="0" ry="0"/> + <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="210" y="402.881"> + <tspan x="210" y="402.881">libpq-dev:amd64</tspan> + <tspan x="210" y="418.881">M-A:none</tspan> + </text> + </g> + <g> + <line style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x1="280.995" y1="407" x2="528.003" y2="407"/> + <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="528.003,412 538.003,407 528.003,402 "/> + </g> + <g> + <rect style="fill: #ffa500; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="1140" y="280" width="160.85" height="60" rx="0" ry="0"/> + <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1220.42" y="305.881"> + <tspan x="1220.42" y="305.881">postgresql-doc-NN:all</tspan> + <tspan x="1220.42" y="321.881">M-A:foreign</tspan> + </text> + </g> + <g> + <rect style="fill: #ffa500; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="1140" y="180" width="160" height="54" rx="0" ry="0"/> + <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1220" y="202.881"> + <tspan x="1220" y="202.881">postgresql-doc:all</tspan> + <tspan x="1220" y="218.881">M-A:foreign</tspan> + </text> + </g> + <g> + <line style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x1="1220.12" y1="234.988" x2="1220.25" y2="268.007"/> + <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="1215.25,268.028 1220.29,278.007 1225.25,267.986 "/> + </g> + <g> + <rect style="fill: #84bef6; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="840" y="180" width="180" height="54" rx="0" ry="0"/> + <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="930" y="202.881"> + <tspan x="930" y="202.881">postgresql-contrib:all</tspan> + <tspan x="930" y="218.881">M-A:none</tspan> + </text> + </g> + <g> + <line style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x1="878.822" y1="234.976" x2="807.919" y2="273.733"/> + <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="805.521,269.345 799.145,278.529 810.318,278.12 "/> + </g> + <g> + <rect style="fill: #84bef6; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="100" y="280" width="220" height="60" rx="0" ry="0"/> + <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="210" y="305.881"> + <tspan x="210" y="305.881">postgresql-server-dev-NN:any</tspan> + <tspan x="210" y="321.881">M-A:none</tspan> + </text> + </g> + <g> + <rect style="fill: #ffff00; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="100" y="180" width="220" height="54" rx="0" ry="0"/> + <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="210" y="202.881"> + <tspan x="210" y="202.881">postgresql-server-dev-all:any</tspan> + <tspan x="210" y="218.881">M-A:same</tspan> + </text> + </g> + <g> + <line style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x1="210" y1="234.988" x2="210" y2="268.007"/> + <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="205,268.007 210,278.007 215,268.007 "/> + </g> + <g> + <line style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x1="210" y1="341.005" x2="210" y2="367.996"/> + <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="205,367.996 210,377.996 215,367.996 "/> + </g> + <text font-size="19.1234" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:700" x="60" y="540"> + <tspan x="60" y="540">postgresql-common</tspan> + </text> + <text font-size="19.1234" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:end;font-family:sans-serif;font-style:normal;font-weight:700" x="1200" y="440"> + <tspan x="1200" y="440">postgresql-NN</tspan> + </text> + <text font-size="12.7998" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="210" y="207"> + <tspan x="210" y="207"></tspan> + </text> + <g> + <line style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x1="320" y1="310" x2="349.256" y2="310"/> + <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="356.756,310 346.756,315 349.256,310 346.756,305 "/> + </g> + <text font-size="12.7998" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="210" y="310"> + <tspan x="210" y="310"></tspan> + </text> + <text font-size="12.7998" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="210" y="310"> + <tspan x="210" y="310"></tspan> + </text> + <text font-size="12.7998" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="690" y="360"> + <tspan x="690" y="360"></tspan> + </text> + <g> + <rect style="fill: #84bef6; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="440" y="100" width="140" height="54" rx="0" ry="0"/> + <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="510" y="122.881"> + <tspan x="510" y="122.881">postgresql-all:all</tspan> + <tspan x="510" y="138.881">M-A:none</tspan> + </text> + </g> + <g> + <line style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x1="439.01" y1="145.931" x2="324.363" y2="176.503"/> + <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="317.117,178.436 325.491,171.028 324.363,176.503 328.067,180.69 "/> + </g> + <g> + <line style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x1="543.132" y1="155.004" x2="682.402" y2="272.719"/> + <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="688.13,277.561 677.265,274.924 682.402,272.719 683.72,267.287 "/> + </g> + <g> + <rect style="fill: #84bef6; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="880" y="280" width="224.85" height="70" rx="0" ry="0"/> + <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="992.425" y="302.881"> + <tspan x="992.425" y="302.881">postgresql-plXX:any</tspan> + <tspan x="992.425" y="318.881">M-A:none</tspan> + <tspan x="992.425" y="334.881">Provides: postgresql-plXX</tspan> + </text> + </g> + <g> + <line style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x1="879.024" y1="315" x2="855.562" y2="315"/> + <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="848.062,315 858.062,310 855.562,315 858.062,320 "/> + </g> + <g> + <polyline style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" points="580,127 1040,127 1040,270.264 "/> + <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="1040,277.764 1035,267.764 1040,270.264 1045,267.764 "/> + </g> + </g> +</svg> diff --git a/doc/postgresql-debian-packaging.md b/doc/postgresql-debian-packaging.md new file mode 100644 index 0000000..4edd9ca --- /dev/null +++ b/doc/postgresql-debian-packaging.md @@ -0,0 +1,741 @@ +Packaging PostgreSQL Extensions for Debian +========================================== +January 2024, Christoph Berg <myon@debian.org> + +Debian ships many PostgreSQL applications and extensions as packages. I often +get asked by developers how they would get their programs packaged, and ended +up writing the same reply over and over. This is a write-up intended as a more +thorough answer. + +# Anatomy of Debian packages + +Debian knows two sorts of packages: "source" packages and "binary" packages. +The latter type is the `.deb` files that get installed using apt or dpkg. The +first type is what this article is mostly about. For both sorts of packages, +there is an unpacked form and a packed form. + +## Binary packages + +Packed binary packages are a single `.deb` file: + +``` +$ ls -al postgresql-16-unit_7.7-1_amd64.deb +-rw-r--r-- 1 myon myon 136740 Jan 12 14:13 postgresql-16-unit_7.7-1_amd64.deb + +$ dpkg-deb -I postgresql-16-unit_7.7-1_amd64.deb + new Debian package, version 2.0. + size 136740 bytes: control archive=1332 bytes. + 643 bytes, 15 lines control + 2046 bytes, 25 lines md5sums + Package: postgresql-16-unit + Source: postgresql-unit + Version: 7.7-1 + Architecture: amd64 + Maintainer: Christoph Berg <myon@debian.org> + Installed-Size: 500 + Depends: postgresql-16, postgresql-16-jit-llvm (>= 16), libc6 (>= 2.14) + Section: database + Priority: optional + Homepage: https://github.com/df7cb/postgresql-unit + Description: SI Units for PostgreSQL + postgresql-unit implements a PostgreSQL datatype for SI units, plus byte. The + base units can be combined to named and unnamed derived units using operators + defined in the PostgreSQL type system. SI prefixes are used for input and + output, and quantities can be converted to arbitrary scale. + +$ dpkg-deb -c postgresql-16-unit_7.7-1_amd64.deb +drwxr-xr-x root/root 0 2023-01-06 16:34 ./ +drwxr-xr-x root/root 0 2023-01-06 16:34 ./usr/ +drwxr-xr-x root/root 0 2023-01-06 16:34 ./usr/share/ +drwxr-xr-x root/root 0 2023-01-06 16:34 ./usr/share/doc/ +drwxr-xr-x root/root 0 2023-01-06 16:34 ./usr/share/doc/postgresql-16-unit/ +-rw-r--r-- root/root 267 2023-01-06 16:34 ./usr/share/doc/postgresql-16-unit/NEWS.Debian.gz +-rw-r--r-- root/root 7202 2023-01-06 16:34 ./usr/share/doc/postgresql-16-unit/README.md.gz +-rw-r--r-- root/root 977 2023-01-06 16:34 ./usr/share/doc/postgresql-16-unit/changelog.Debian.gz +-rw-r--r-- root/root 868 2023-01-06 16:34 ./usr/share/doc/postgresql-16-unit/copyright +drwxr-xr-x root/root 0 2023-01-06 16:34 ./usr/share/postgresql/ +drwxr-xr-x root/root 0 2023-01-06 16:34 ./usr/share/postgresql/16/ +drwxr-xr-x root/root 0 2023-01-06 16:34 ./usr/share/postgresql/16/extension/ +-rw-r--r-- root/root 19259 2023-01-06 16:34 ./usr/share/postgresql/16/extension/unit--7.sql +-rw-r--r-- root/root 244 2023-01-06 16:34 ./usr/share/postgresql/16/extension/unit.control +... +``` + +The unpacked binary package is of course the files being installed on the +system. + +## Source packages + +Unpacked source packages consist of the original upstream source code, with a `debian/` directory added. + +``` +postgresql-unit/ +├── debian/ +│ ├── changelog +│ ├── control +│ ├── control.in +│ ├── copyright +│ ├── gitlab-ci.yml +│ ├── NEWS +│ ├── pgversions +│ ├── rules* +│ ├── source/ +│ │ └── format +│ ├── tests/ +│ │ ├── control +│ │ └── installcheck* +│ ├── upstream/ +│ │ └── metadata +│ └── watch +├── Makefile +├── NEWS.md +├── README.md +├── unit--7.sql.in +├── unit.c +└── unit.control +``` + +Most packages are maintained in Git, which tracks this unpacked source package +form. (Either with or without the original upstream source, more on that +later.) + +Packed source packages are actually a set of files: + +``` +-rw-rw-r-- 1 myon myon 3848 Jan 12 14:31 postgresql-unit_7.7-1.debian.tar.xz +-rw-rw-r-- 1 myon myon 1100 Jan 12 14:31 postgresql-unit_7.7-1.dsc +-rw-rw-r-- 1 myon myon 414114 Jan 12 14:12 postgresql-unit_7.7.orig.tar.gz +``` + +This transport format is used for uploading to the Debian archive and for +retrieving the source code using `apt source`. (It is not stored in Git.) + +The `.orig.tar.gz` is the original source tarball as distributed by upstream, +either as download from the upstream homepage, or for projects hosted on +GitHub, often a tarball automatically generated by GitHub from an upstream Git +tag. + +The `.debiar.tar.xz` file contains the `debian/` directory. + +The `.dsc` file is a descriptor that contains pointers to the other files in +the source package, and more meta information. + +``` +$ cat postgresql-unit_7.7-1.dsc +Format: 3.0 (quilt) +Source: postgresql-unit +Binary: postgresql-16-unit +Architecture: any +Version: 7.7-1 +Maintainer: Christoph Berg <myon@debian.org> +Homepage: https://github.com/df7cb/postgresql-unit +Standards-Version: 4.6.1 +Vcs-Browser: https://github.com/df7cb/postgresql-unit +Vcs-Git: https://github.com/df7cb/postgresql-unit.git +Testsuite: autopkgtest +Testsuite-Triggers: make +Build-Depends: bison, debhelper-compat (= 13), flex, postgresql-server-dev-all (>= 217~) +Package-List: + postgresql-16-unit deb database optional arch=any +Checksums-Sha1: + c2f81968bfbe83fed49b084b737e3aba423bf15a 414114 postgresql-unit_7.7.orig.tar.gz + b8a1917ddecb99b1441218bc74a0b7cb30752235 3848 postgresql-unit_7.7-1.debian.tar.xz +Checksums-Sha256: + 411d05beeb97e5a4abf17572bfcfbb5a68d98d1018918feff995f6ee3bb03e79 414114 postgresql-unit_7.7.orig.tar.gz + 36e89c762e50ddf997b079703200c0df6967b4fe911bde8e9482d8e82dcb6a98 3848 postgresql-unit_7.7-1.debian.tar.xz +Files: + 33a22586c8b81564ba7e9c05f430ad40 414114 postgresql-unit_7.7.orig.tar.gz + a0b31860b86c12c7173a78d6ecd525cb 3848 postgresql-unit_7.7-1.debian.tar.xz +``` + +## Building the source package + +To get started with working with a Debian package, get the unpacked source +package. This could mean invoking `apt source`, but most often checking out the +packaging Git repository is the better option as it might contain changes that +have not been uploaded yet. It also makes contributing changes easier. For +packages, that are already part of Debian, the `debcheckout` tool can automate +that (it uses the `Vcs-Git` field in the metadata). + +To build the packed source packed from the unpacked one, enter the package +directory, and invoke `dpkg-buildpackage -S --no-sign`: + +``` +postgresql-unit $ dpkg-buildpackage -S --no-sign +dpkg-buildpackage: info: source package postgresql-unit +dpkg-buildpackage: info: source version 7.7-1 +dpkg-buildpackage: info: source distribution unstable +dpkg-buildpackage: info: source changed by Christoph Berg <myon@debian.org> + dpkg-source --before-build . + debian/rules clean +dh clean --with pgxs +... + dpkg-source -b . +dpkg-source: info: using source format '3.0 (quilt)' +dpkg-source: info: building postgresql-unit using existing ./postgresql-unit_7.7.orig.tar.gz +dpkg-source: info: building postgresql-unit in postgresql-unit_7.7-1.debian.tar.xz +dpkg-source: info: building postgresql-unit in postgresql-unit_7.7-1.dsc + dpkg-genbuildinfo --build=source -O../postgresql-unit_7.7-1_source.buildinfo + dpkg-genchanges --build=source -O../postgresql-unit_7.7-1_source.changes +dpkg-genchanges: info: including full source code in upload + dpkg-source --after-build . +dpkg-buildpackage: info: source-only upload (original source is included) +``` + +Note that the artifacts produced by `dpkg-buildpackage` are always in the *parent* directory of the working directory. +To make room for that, I always create *two* levels of directory for my working copies, so a typical case looks like this: + +``` +postgresql-unit/ +├── postgresql-unit/ <--- most commands are invoked from here +│ ├── debian/ +│ │ ├── changelog +│ │ ├── control +│ │ ├── control.in +│ │ ├── copyright +│ │ ├── files +│ │ ├── gitlab-ci.yml +│ │ ├── NEWS +│ │ ├── pgversions +│ │ ├── rules* +│ │ ├── source/ +│ │ │ └── format +│ │ ├── tests/ +│ │ │ ├── control +│ │ │ └── installcheck* +│ │ ├── upstream/ +│ │ │ └── metadata +│ │ └── watch +│ ├── Makefile +│ ├── NEWS.md +│ ├── powers.c +│ ├── powers.h +│ ├── README.md +│ ├── unit--7.sql.in +│ ├── unit.c +│ └── unit.control +├── postgresql-16-unit_7.7-1_amd64.deb +├── postgresql-16-unit-dbgsym_7.7-1_amd64.deb +├── postgresql-unit_7.7-1_amd64.changes +├── postgresql-unit_7.7-1.debian.tar.xz +├── postgresql-unit_7.7-1.dsc +└── postgresql-unit_7.7.orig.tar.gz +``` + +## Building binary packages + +Binary packages are built using `dpkg-buildpackage --no-sign`. + +``` + dpkg-buildpackage --no-sign +dpkg-buildpackage: info: source package postgresql-unit +dpkg-buildpackage: info: source version 7.7-1 +dpkg-buildpackage: info: source distribution unstable +dpkg-buildpackage: info: source changed by Christoph Berg <myon@debian.org> +dpkg-buildpackage: info: host architecture amd64 + dpkg-source --before-build . + debian/rules clean +dh clean --with pgxs +... + dpkg-source -b . +dpkg-source: info: using source format '3.0 (quilt)' +dpkg-source: info: building postgresql-unit using existing ./postgresql-unit_7.7.orig.tar.gz +dpkg-source: info: building postgresql-unit in postgresql-unit_7.7-1.debian.tar.xz +dpkg-source: info: building postgresql-unit in postgresql-unit_7.7-1.dsc + debian/rules binary +dh binary --with pgxs + dh_update_autotools_config + dh_autoreconf + dh_auto_configure + dh_auto_build --buildsystem=pgxs + pg_buildext build build-%v +### PostgreSQL 16 build ### +make[1]: Entering directory '/home/myon/projects/postgresql/postgresql-unit/postgresql-unit/build-16' +gcc -Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Werror=vla -Wendif-labels -Wmissing-format-attribute -Wimplicit-fallthrough=3 -Wcast-function-type -Wshadow=compatible-local -Wformat-security -fno-strict-aliasing -fwrapv -fexcess-precision=standard -Wno-format-truncation -Wno-stringop-truncation -g -g -O2 -fstack-protector-strong -fstack-clash-protection -Wformat -Werror=format-security -fcf-protection -fno-omit-frame-pointer -g -O2 -ffile-prefix-map=/home/myon/projects/postgresql/postgresql-unit/postgresql-unit=. -fstack-protector-strong -fstack-clash-protection -Wformat -Werror=format-security -fcf-protection -fPIC -fvisibility=hidden -ffp-contract=off -I. -I/home/myon/postgresql/postgresql-unit/postgresql-unit -I/usr/include/postgresql/16/server -I/usr/include/postgresql/internal -Wdate-time -D_FORTIFY_SOURCE=2 -D_GNU_SOURCE -I/usr/include/libxml2 -c -o unit.o /home/myon/postgresql/postgresql-unit/postgresql-unit/unit.c +... +### End 16 build ### + create-stamp debian/debhelper-build-stamp + dh_prep + dh_auto_install --buildsystem=pgxs --destdir=debian/postgresql-16-unit/ + pg_buildext install build-%v postgresql-%v-unit +### PostgreSQL 16 install ### +make[1]: Entering directory '/home/myon/projects/postgresql/postgresql-unit/postgresql-unit/build-16' +/usr/bin/install -c -m 755 unit.so '/home/myon/postgresql/postgresql-unit/postgresql-unit/debian/postgresql-16-unit/usr/lib/postgresql/16/lib/unit.so' +... +### End 16 install ### + debian/rules override_dh_installdocs +make[1]: Entering directory '/home/myon/projects/postgresql/postgresql-unit/postgresql-unit' +dh_installdocs --all README.* +make[1]: Leaving directory '/home/myon/projects/postgresql/postgresql-unit/postgresql-unit' + dh_installchangelogs + dh_perl + dh_link + debian/rules override_dh_pgxs_test +make[1]: Entering directory '/home/myon/projects/postgresql/postgresql-unit/postgresql-unit' +# defer testing to autopkgtest, the data tables are not in /usr/share/postgresql yet +make[1]: Leaving directory '/home/myon/projects/postgresql/postgresql-unit/postgresql-unit' + dh_strip_nondeterminism + dh_compress + dh_fixperms + dh_missing + dh_dwz -a + dh_strip -a + dh_makeshlibs -a + dh_shlibdeps -a + dh_installdeb + dh_gencontrol + dh_md5sums + dh_builddeb +dpkg-deb: building package 'postgresql-16-unit' in '../postgresql-16-unit_7.7-1_amd64.deb'. +dpkg-deb: building package 'postgresql-16-unit-dbgsym' in '../postgresql-16-unit-dbgsym_7.7-1_amd64.deb'. + dpkg-genbuildinfo -O../postgresql-unit_7.7-1_amd64.buildinfo + dpkg-genchanges -O../postgresql-unit_7.7-1_amd64.changes +dpkg-genchanges: info: including full source code in upload + dpkg-source --after-build . +dpkg-buildpackage: info: full upload (original source is included) +``` + +Again, the resulting `.deb` files are placed in the parent directory. + +By default, this also builds the source. If this fails (often when there are +files that differ from the tarball version, more on patches later), use `-b` to +skip building the source: `dpkg-buildpackage -b --no-sign`. + +If building fails because of missing dependencies, install them using +`apt build-dep .`. + +There are various front-end utilities to automate these building steps better +(git-buildpackage, sbuild, pbuilder, debuild), but dpkg-buildpackage is just +fine. + +## The debian/ directory + +The `debian/` directory contains metadata and build instructions for the +package. "Creating a Debian package" really means editing the files in this +directory to make the package behave as desired. + +### debian/control + +The control file contains one section for the source package, followed by one +or more sections for binary packages. + +``` +$ cat debian/control +Source: postgresql-unit +Section: database +Priority: optional +Maintainer: Christoph Berg <myon@debian.org> +Build-Depends: + bison, + debhelper-compat (= 13), + flex, + postgresql-server-dev-all (>= 217~), +Standards-Version: 4.6.1 +Rules-Requires-Root: no +Vcs-Git: https://github.com/df7cb/postgresql-unit.git +Vcs-Browser: https://github.com/df7cb/postgresql-unit +Homepage: https://github.com/df7cb/postgresql-unit + +Package: postgresql-16-unit +Architecture: any +Depends: ${misc:Depends}, ${shlibs:Depends}, postgresql-16 +Description: SI Units for PostgreSQL + postgresql-unit implements a PostgreSQL datatype for SI units, plus byte. The + base units can be combined to named and unnamed derived units using operators + defined in the PostgreSQL type system. SI prefixes are used for input and + output, and quantities can be converted to arbitrary scale. +``` + +(In the case of PostgreSQL extension packages, the control file is +automatically generated from `debian/control.in`, see below. Normal packages do +not have a control.in file.) + +### debian/rules + +The rules file handles the actual binary package building steps, in Makefile +syntax. + +Almost all packages today use a helper system called `debhelper` which consists +of various building blocks called `dh_*` that handle the build steps. +Historically, rules files would consist of long lists of `dh_*` steps (similar +to the build output above), but there is a "sequencer" command `dh` that knows +how to invoke the basic steps. + +Many packages that don't need any tweaking at that level have a very short +`debian/rules` file. (Make sure the indentation is a tab, not spaces!) + +``` +#!/usr/bin/make -f + +%: + dh $@ +``` + +Or in the case of PostgreSQL extensions: + +``` +#!/usr/bin/make -f + +%: + dh $@ --with pgxs +``` + +If any of the `dh_*` steps need changes, we can override them in the rules file +by adding a `override_dh_*` target: + +``` +#!/usr/bin/make -f + +override_dh_installdocs: + dh_installdocs --all README.* + +override_dh_pgxs_test: + # defer testing to autopkgtest, the data tables are not in /usr/share/postgresql yet + +%: + dh $@ --with pgxs +``` + +### Build steps + +The most interesting build steps to hook into are: + +#### `dh_auto_configure` + +Autodetects the build system and runs `./configure`, `cmake` and the like with +a set of default options. + +To add options, do: + +``` +override_dh_auto_configure: + dh_auto_configure -- -DEXTRA_OPTION=foo --with-bar +``` + +#### `dh_auto_build` + +The main build step. If the automatically run command is wrong, just override it. + +``` +override_dh_auto_build: + $(MAKE) -C some_sub_dir world +``` + +#### `dh_auto_install` + +Runs upstreams' `make install` or equivalent. The files are installed into +`DESTDIR=debian/foo` (single-binary package) or `DESTDIR=debian/tmp` +(multi-binary package). + +#### `dh_install` and `debian/foo.install` + +If there is more than one binary package, the files installed in `debian/tmp` +need to be distributed to the individual binary packages. This is done by +directory/file lists in `debian/*.install`. + +### debian/source/format + +Should be verbatim this: + +``` +3.0 (quilt) +``` + +(More on quilt and patches below.) + +### debian/copyright + +Debian wants copyright information on *all* files in a package. For private +package, this file can be omitted, but for anything official, this file has to +list all the copyright holders, along with any copyright terms and license +texts. + +### debian/watch + +To get informed about new upstream versions, Debian runs a "watch" system that +polls upstream download locations for new package versions. The `debian/watch` +file tells the `uscan` tool where to look and (optionally) how to transform +upstream's version naming scheme into a Debian-compatible one. + +``` +version=4 +https://github.com/df7cb/postgresql-unit/tags .*/([0-9.]*).tar.gz +``` + +This is a simple example where `uscan` looks at some GitHub "tags" URL, parses +the HTML, and recognizes all links pointing to .tar.gz files as new versions. +The regexp part of the URL in parentheses is used as the version number. + +The `uscan` tool is part of the `devscripts` package which holds a bunch of +utilities useful for packaging tasks. + +## Patching the upstream source + +Debian packages are based on upstream tarballs, and `dpkg-buildpackage` does +not like changed (or new) files (except in the `debian/` directory). Changes +need to be done using patch files stored in the `debian/patches/` directory, +and applied in the order listed in `debian/patches/series`. + +At package build time, support is built-in in dpkg-buildpackage. Patches can +edited with whatever tool, with `quilt` being the easiest solution. (It's an +optional package that likely isn't preinstalled.) `quilt` needs some +configuration in `~/.quiltrc`: + +``` +QUILT_DIFF_OPTS="-p" +QUILT_REFRESH_ARGS="-p ab --no-timestamps --no-index" +QUILT_PATCHES=debian/patches +``` + +Some useful commands: + +* Apply all patches: `quilt push -a` +* Unapply all patches: `quilt pop -a` +* Create a new patch: `quilt new foo` +* Add a file to a patch: `quilt add bar` (do this *before* changing the file!) +* Add a file and edit it: `quilt edit bar` +* Show current diff: `quilt diff` +* Save a patch after editing: `quilt refresh` + +Suppose we want to fix something in the `unit.c` file: + +``` +$ quilt new unit-dllexport +Patch unit-dllexport is now on top + +$ quilt add unit.c +File unit.c added to patch unit-dllexport + +$ vi unit.c + +$ quilt diff | cat +Index: postgresql-unit/unit.c +=================================================================== +--- postgresql-unit.orig/unit.c 2023-11-08 20:51:04.343207806 +0100 ++++ postgresql-unit/unit.c 2024-01-12 16:10:23.143509535 +0100 +@@ -136,7 +136,7 @@ unit_get_definitions(void) + + PG_MODULE_MAGIC; + +-void _PG_init(void); ++void PGDLLEXPORT _PG_init(void); + + void + _PG_init(void) + +$ quilt refresh +Refreshed patch unit-dllexport +``` + +This creates the `debian/patches/` directory: + +``` +postgresql-unit/debian/patches/ +├── series +└── unit-dllexport +``` + +At build-time, `dpkg-buildpackage` will then automatically apply and un-apply +patches as needed. + +## Tests + +Package tests are run in two flavors: at build-time and later independently on +packages installed on some system. + +### Build-time package tests + +If an upstream project has a test suite, it is recommended to run it at build +time. In many cases, `dh_auto_test` will guess how to do that and no extra +configuration is needed. If it guesses incorrectly, use `override_dh_auto_test` +to provide a better command. + +### Install-time package tests + +Next to tests at build-time, we can also run tests when the binary packages are +installed on some actual system. This has the advantage that it runs when all +files are in their final location, and can also interface with other packages +and services that might not be available at build time. It can also run +periodically to spot regression, while build-time tests would usually not be +repeated. + +Debian's system to run these tests is `autopkgtest`. The TL;DR version of the +documentation is this: In the `debian/tests/` directory, provide a command +(usually a shell script) that exercises some package smoke test or more complex +scenario. Register that test in `debian/tests/control`. + +``` +postgresql-unit/debian/tests/ +├── control +└── installcheck* + +$ cat debian/tests/control +Depends: @, make +Tests: installcheck +Restrictions: allow-stderr + +$ cat debian/tests/installcheck +#!/bin/sh +pg_buildext -i '--locale=C.UTF-8' installcheck +``` + +In `Depends` the `@` is a shorthand for all binary packages built from this +source. Other packages that the tests needs (and that the binaries don't depend +on) can be listed here. `Restrictions` declare tests properties. Examples are +`allow-stderr` (don't consider stderr output to be a test failure) and +`root-needed` (run test as root instead of an unprivileged user). + +In Debian, these tests are run automatically for QA, see +https://ci.debian.net/. More details are in the autopkgtest package +documentation in /usr/share/doc/autopkgtest/. + +# PostgreSQL extension packages + +Packages for PostgreSQL extensions work as described above, but since +extensions have to be compiled for each PostgreSQL major version separately, +things are a bit more complex. + +For Debian, the process is changed to still have one source package per +upstream project, but to build separate binary packages for each PostgreSQL +major version. The naming scheme is `postgresql-NN-foo`. (In case the upstream +project is called `pg_foo`, make a judgment call if `postgresql-NN-pg-foo` or +`postgresql-NN-foo` is better.) + +In Debian, only one PostgreSQL major version is supported at a time, but in the +https://apt.postgresql.org/ repository, many major versions are supported in +parallel (currently PostgreSQL 10 everything newer, even when 10 is already +EOL). + +## `debian/control.in` + +In order not to have to edit the list of binary packages built when a new +PostgreSQL major version comes out, or when a source package is built both for +Debian and for apt.postgresql.org, the `debian/control` file is generated from +a template in `debian/control.in`. + +``` +$ cat debian/control.in +Source: postgresql-unit +Section: database +Priority: optional +Maintainer: Christoph Berg <myon@debian.org> +Build-Depends: + bison, + debhelper-compat (= 13), + flex, + postgresql-server-dev-all (>= 217~), +Standards-Version: 4.6.2 +Rules-Requires-Root: no +Vcs-Git: https://github.com/df7cb/postgresql-unit.git +Vcs-Browser: https://github.com/df7cb/postgresql-unit +Homepage: https://github.com/df7cb/postgresql-unit + +Package: postgresql-PGVERSION-unit +Architecture: any +Depends: ${misc:Depends}, ${shlibs:Depends}, ${postgresql:Depends} +Description: SI Units for PostgreSQL + postgresql-unit implements a PostgreSQL datatype for SI units, plus byte. The + base units can be combined to named and unnamed derived units using operators + defined in the PostgreSQL type system. SI prefixes are used for input and + output, and quantities can be converted to arbitrary scale. +``` + +The section with the `PGVERSION` token is duplicated for each major version +supported, with the version number filled in. + +## `debian/pgversions` + +Not every package supports all PostgreSQL major versions. The +`debian/pgversions` file is used to mark which versions are actually supported, +so apt.postgresql.org can skip building the other version. + +Unless we know the exact versions supported, we should use `all`: + +``` +$ cat debian/pgversions +all +``` + +If 14 or newer is supported: + +``` +$ cat debian/pgversions +14+ +``` + +### Supported versions + +`debian/pgversions` lists the versions supported *by the package*. The other +half of that system is the set of versions supported *by the system*. This list +is configured in `/etc/postgresql-common/supported_versions` or by setting the +`PG_SUPPORTED_VERSIONS` environment variable. (Set this variable to test +building for other major versions.) The actual build process will use the +intersection of these two lists. + +## `pg_buildext` + +The process of building PostgreSQL extensions for several major versions is +automated by the `pg_buildext` utility. It provides commands for the most +common tasks. + +Package building: + +* `pg_buildext supported-versions` - print list of supported versions. + Use this to loop over versions in `debian/rules`. +* `pg_buildext build build-%v` - build in `build-%v` directory +* `pg_buildext install build-%v postgresql-%v-unit` - invoke `make install` +* `pg_buildext installcheck build-%v postgresql-%v-unit` - invoke + `make installcheck` for build-time testing +* `pg_buildext loop postgresql-%v-unit` - use instead of + `build/install/installcheck` if the package doesn't support out-of-tree + builds in subdirectories +* `pg_buildext updatecontrol` - rebuild `debian/control` from + `debian/control.in`. Run this manually when the set of supported versions has + changed. This is not run automatically because the Debian packaging policy + forbids changing the set of binary packages at build time. (In environments + where this is not an issue, set `PG_UPDATECONTROL=yes`.) + +Package testing: + +* `pg_buildext installed-versions` - print list of installed versions. + Use this to loop over versions in `debian/tests/*`. +* `pg_buildext installcheck` - invoke `make installcheck` on installed packages + +## `dh --with pgxs` + +A debhelper extension `pgxs` is provided that adds builds steps to the `dh` +build sequence. + +``` +$ cat debian/rules +#!/usr/bin/make -f + +%: + dh $@ --with pgxs +``` + +If the package doesn't support out-of-tree builds, use +`dh $@ --with pgxs_loop`. + +``` + dh_auto_build --buildsystem=pgxs + pg_buildext build build-%v + dh_auto_install --buildsystem=pgxs + pg_buildext install build-%v postgresql-%v-unit + dh_pgxs_test + pg_buildext installcheck . build-%v postgresql-%v-unit +``` + +To override any of these steps, use `override_dh_auto_*` in `debian/rules`. + +## Package template + +To get started with a new package, the `dh_make_pgxs` tool can generate a +skeleton `debian/` directory: + +``` +$ dh_make_pgxs +``` + +If the auto-detected values are wrong, hit `^C` and add more command line +parameters. diff --git a/gitlab/gitlab-ci.yml b/gitlab/gitlab-ci.yml new file mode 100644 index 0000000..5c640aa --- /dev/null +++ b/gitlab/gitlab-ci.yml @@ -0,0 +1,13 @@ +# gitlab-ci.yml file for the Debian PostgreSQL packages +# to be used in debian/gitlab-ci.yml: +# include: https://salsa.debian.org/postgresql/postgresql-common/raw/master/gitlab/gitlab-ci.yml + +include: + - https://salsa.debian.org/salsa-ci-team/pipeline/raw/master/salsa-ci.yml + - https://salsa.debian.org/salsa-ci-team/pipeline/raw/master/pipeline-jobs.yml + +variables: + PG_UPDATECONTROL: 'yes' # tell pg_buildext to automatically generate debian/control from debian/control.in (TODO: doesn't work this way yet) + SALSA_CI_DISABLE_REPROTEST: 'yes' # too many packages fail because reprotest runs the build as root, disable for now + SALSA_CI_REPROTEST_ENABLE_DIFFOSCOPE: 'yes' + SALSA_CI_DISABLE_CROSSBUILD_ARM64: 'yes' # clang isn't ready for cross-building yet diff --git a/pg_backupcluster b/pg_backupcluster new file mode 100755 index 0000000..2959056 --- /dev/null +++ b/pg_backupcluster @@ -0,0 +1,659 @@ +#!/usr/bin/perl -wT + +# simple pg_basebackup front-end +# +# Copyright (C) 2021-2023 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use warnings; +use Fcntl qw(LOCK_EX); +use Getopt::Long; +use JSON; +use PgCommon; +use POSIX qw(strftime); + +my ($version, $cluster); + +# untaint environment +$ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin'; +delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; +chdir '/'; +umask 027; + +sub help () { + print "Syntax: $0 [options] <version> <cluster> <action> +Actions: + createdirectory Create /var/backups/version-cluster + basebackup Backup using pg_basebackup + dump Backup using pg_dump + expiredumps <N> Remove all but last N dumps + expirebasebackups <N> Remove all but last N basebackups + receivewal Launch pg_receivewal + compresswal Compress WAL files in archive + archivecleanup Remove obsolete WAL files from archive + list Show dumps, basebackups, and WAL +Options: + -c --checkpoint <spread|fast> Passed to pg_basebackup + -k --keep-on-error Keep faulty backup directory on error + -v --verbose Verbose output +"; +} + +my $checkpoint = 'spread'; +my $keep_on_error; +my $verbose; + +exit 1 unless GetOptions ( + 'c|checkpoint=s' => sub { $checkpoint = $_[1] =~ /^f/ ? "fast" : "spread" }, + 'k|keep-on-error' => \$keep_on_error, + 'v|verbose' => \$verbose, +); + +$verbose = 1 if (-t 1); # verbose output when running on terminal + +# accept both "version cluster action" and "version[-/]cluster action" +if (@ARGV >= 2 and $ARGV[0] =~ m!^(\d+\.?\d)[-/]([^/]+)$!) { + ($version, $cluster) = ($1, $2); + shift @ARGV; +} elsif (@ARGV >= 3 and $ARGV[0] =~ /^(\d+\.?\d)$/) { + $version = $1; + ($cluster) = ($ARGV[1]) =~ m!^([^/]+)$!; + shift @ARGV; + shift @ARGV; +} else { + help(); + exit 1; +} +my $action = $ARGV[0]; + +error "specified cluster $version $cluster does not exist" unless $version && $cluster && cluster_exists $version, $cluster; + +my %info = cluster_info($version, $cluster); +validate_cluster_owner \%info; + +my $rootdir = "/var/backups/postgresql"; +my $clusterdir = "$rootdir/$version-$cluster"; +my $waldir = "$clusterdir/wal"; + +# functions to be run as root + +sub create_directory() { + if (! -d $rootdir) { + my ($pg_uid, $pg_gid) = (getpwnam 'postgres')[2,3]; + ($pg_uid and $pg_gid) or error "getpwnam postgres: $!"; + mkdir $rootdir, 0755 or error "mkdir $rootdir: $!"; + chown $pg_uid, $pg_gid, $rootdir or error "chown $rootdir: $!"; + } + if (! -d $clusterdir) { + mkdir($clusterdir, 0750) or error "mkdir $clusterdir: $!"; + chown $info{owneruid}, $info{ownergid}, $clusterdir or error "chown $clusterdir: $!"; + } +} + +sub switch_to_cluster_owner() { + change_ugid $info{owneruid}, $info{ownergid}; +} + +# backup functions + +sub get_backupdir($$) { + my ($starttime, $suffix) = @_; + my $timestamp = strftime("%FT%H%M%SZ", gmtime($starttime)); + my $backupdir = "$clusterdir/$timestamp.$suffix"; + error "$backupdir already exists" if (-d $backupdir); + print "Creating $suffix in $backupdir\n" if ($verbose); + return $backupdir; +} + +sub remove_backup_on_error($) { + my $backupdir = shift; + if ($keep_on_error) { + print "Not removing $backupdir (--keep-on-error)\n"; + } else { + print "Removing $backupdir ...\n" if ($verbose); + system_or_error "rm", "-rf", $backupdir; + } +} + +sub create_basebackup($) { + my $backupdir = shift; + + system_or_error "pg_basebackup", + "--cluster", "$version/$cluster", + "--verbose", + (-t 2 ? "--progress" : ()), + "--checkpoint=$checkpoint", + "--format", "tar", "--gzip", + ($version < 10 ? "--xlog" : ()), + "-D", $backupdir; +} + +sub create_dumpall($) { + my $backupdir = shift; + mkdir($backupdir, 0750) or error "mkdir $backupdir: $!"; + + my $pg16 = $version >= 16 ? "|| ' --icu-rules ' || daticurules" : ""; + my $pg15 = $version >= 15 ? "CASE datlocprovider::text + WHEN 'c' THEN '--locale-provider libc' + WHEN 'i' THEN '--locale-provider icu --icu-locale ' || daticulocale $pg16 + END," : ""; + my $clusterquery = "SELECT + '--encoding', pg_catalog.pg_encoding_to_char(encoding), + '--lc-collate', datcollate, + '--lc-ctype', datctype, + $pg15 + CASE WHEN current_setting('data_checksums')::boolean THEN '-- --data-checksums' END +FROM pg_database WHERE datname = 'template0'"; + system_or_error "psql", + "--cluster", "$version/$cluster", + "-XAtF", " ", "-c", $clusterquery, + "-o", "$backupdir/createcluster.opts"; + + system_or_error "pg_dumpall", + "--cluster", "$version/$cluster", + "--globals-only", + "--file", "$backupdir/globals.sql"; + + $pg16 = $version >= 16 ? "|| ' ICU_RULES ' || quote_literal(coalesce(daticurules, ''))" : ""; + $pg15 = $version >= 15 ? "|| CASE datlocprovider::text + WHEN 'c' THEN 'LOCALE_PROVIDER libc' + WHEN 'i' THEN 'LOCALE_PROVIDER icu ICU_LOCALE ' || quote_literal(daticulocale) $pg16 + END" : ""; + my $dbquery = "/* database creation */ +SELECT + format('CREATE DATABASE %I WITH OWNER %I TEMPLATE template0 ENCODING %L LC_COLLATE %L LC_CTYPE %L', + datname, rolname, pg_encoding_to_char(encoding), datcollate, datctype) $pg15 || ';' AS command +FROM pg_database +LEFT JOIN pg_roles r ON r.oid = datdba +WHERE datallowconn AND datname NOT IN ('postgres', 'template1') + +UNION ALL + +/* database options */ +SELECT + CASE + WHEN rolname IS NULL THEN format('ALTER DATABASE %I', datname) + ELSE format('ALTER ROLE %I IN DATABASE %I', rolname, datname) + END || + format(' SET %I = ', match[1]) || + CASE + WHEN match[1] IN ('local_preload_libraries', 'search_path', 'session_preload_libraries', 'shared_preload_libraries', 'temp_tablespaces', 'unix_socket_directories') + THEN match[2] + ELSE quote_literal(match[2]) + END || + ';' AS command +FROM pg_db_role_setting s +JOIN pg_database d ON d.oid = setdatabase +LEFT JOIN pg_roles r ON r.oid = setrole, +unnest(setconfig) u(setting), +regexp_matches(setting, '(.+)=(.+)') m(match)"; + system_or_error "psql", + "--cluster", "$version/$cluster", + "-XAtc", $dbquery, + "-o", "$backupdir/databases.sql"; + + my $dblist = 'SELECT datname FROM pg_database WHERE datallowconn ORDER BY datname'; + my $databases = `psql --cluster '$version/$cluster' -XAtc '$dblist'`; + for my $datname ($databases =~ /(.+)/g) { + print "Dumping $datname to $backupdir/$datname.dump ...\n" if ($verbose); + system_or_error "pg_dump", + "--cluster", "$version/$cluster", + "--format=custom", + "--file", "$backupdir/$datname.dump", + $datname; + } +} + +sub create_configbackup($) { + my $backupdir = shift; + $info{configdir} or error "cluster has no configdir"; + system_or_error "tar", + "-C", $info{configdir}, + "-cz", "-f", "$backupdir/config.tar.gz", + "."; +} + +sub create_status($$$$) { + my ($type, $starttime, $backupdir, $status) = @_; + my $statusfile = "$backupdir/status"; + my $endtime = time; + my $statusjson = { + cluster => $cluster, + duration => $endtime - $starttime, + end => strftime("%FT%H%M%SZ", gmtime($endtime)), + start => strftime("%FT%H%M%SZ", gmtime($starttime)), + status => $status, + type => $type, + version => $version, + }; + if (my $hostname = `hostname`) { + chomp $hostname; + $statusjson->{hostname} = $hostname; + } + if (-e '/etc/machine-id') { + open my $fh, '/etc/machine-id'; + my $machine_id = <$fh>; + close $fh; + if ($machine_id) { + chomp $machine_id; + $statusjson->{'machine-id'} = $machine_id; + } + } + if (-e '/etc/machine-info') { + open my $fh, '/etc/machine-info'; + while (<$fh>) { + if (/^DEPLOYMENT=(.*)/) { + $statusjson->{'machine-deployment'} = $1; + } + } + close $fh; + } + + my $json = JSON->new->canonical; + open F, '>', $statusfile or error "$statusfile: $!"; + print F $json->encode($statusjson) . "\n" or error "$statusfile: $!"; + close F or error "$$statusfile: $!"; +} + +sub sync($) { + my $backupdir = shift; + system_or_error "sync $backupdir/*"; +} + +sub expire_backups($$) { + my ($suffix, $number) = @_; + my @backups = glob("$clusterdir/*.$suffix"); + my $found = 0; + for my $backup (reverse @backups) { + # iterate reversely over backups until we have found enough valid ones + if ($found >= $number) { + print "Removing $backup ...\n" if ($verbose); + $backup =~ /(.*)/; # untaint + system_or_error "rm", "-rf", $1; + } else { + if (-f "$backup/status") { + $found++; + } + } + } +} + +sub delete_broken() { + my @backups = (glob("$clusterdir/*.backup"), glob("$clusterdir/*.dump")); + for my $backup (@backups) { + if (! -f "$backup/status") { + print "Removing $backup ...\n" if ($verbose); + $backup =~ /(.*)/; # untaint + system_or_error "rm", "-rf", $1; + } + } +} + +# wal handling + +sub create_wal_directory() { + if (! -d $waldir) { + mkdir($waldir, 0750) or error "mkdir $waldir: $!"; + } +} + +sub receivewal() { + my $pg_receivewal = $version >= 10 ? 'pg_receivewal' : 'pg_receivexlog'; + + # create slot + system_or_error $pg_receivewal, "--cluster=$version/$cluster", "--slot", "pg_receivewal_service", "--create-slot", "--if-not-exists"; + + # launch pg_receivewal + $ENV{PGAPPNAME} = "pg_receivewal\@$version-$cluster.service"; + my @cmd = ($pg_receivewal, "--cluster", "$version/$cluster", + "-D", $waldir, "--slot", "pg_receivewal_service"); + push @cmd, "--compress=5" if ($version >= 10); + exec {$pg_receivewal} @cmd or error "exec $pg_receivewal: $!"; +} + +sub compresswal() { + chdir $waldir or return; # ok if not yet created + open my $lock, $waldir or error "open $waldir: $!"; # protect against concurrent runs + flock $lock, LOCK_EX or error "flock $waldir: $!"; + + for my $wal (glob "0???????????????????????") { + $wal =~ /^([0-9A-F]+)$/ or continue; + $wal = $1; # untaint + if (-f "$wal.gz") { + print STDERR "$waldir/$wal.gz already exists, skipping compression\n"; + next; + } + system_or_error "if ! gzip < $wal > $wal.tmp.gz; then rm -f $wal.tmp.gz; exit 1; fi"; + system_or_error "touch --reference=$wal $wal.tmp.gz"; + system_or_error "mv $wal.tmp.gz $wal.gz"; + system_or_error "sync", "$wal.gz"; + unlink $wal; + } + + close $lock; +} + +sub archivecleanup() { + chdir $waldir or return; # ok if not yet created + my @backups = sort glob "$clusterdir/*.backup"; + for my $backup (@backups) { + next unless (-f "$backup/status"); + + $backup =~ /(.*)/; # untaint + my $basetar = "$1/base.tar.gz"; + my $backup_label = `tar --extract --occurrence=1 --to-stdout --file '$basetar' backup_label` or error "failed to extract backup_label from $basetar"; + + # START WAL LOCATION: 0/2B000028 (file 00000001000000000000002B) + $backup_label =~ /^START WAL LOCATION: .* \(file ([0-9A-F]+)\)/ or error "no start wal location in $basetar"; + my $keep_file = $1; + system_or_error "pg_archivecleanup", "-x", ".gz", $waldir, $keep_file; + + return 0; # process first backup only + } + error "no valid basebackups found in $clusterdir"; +} + +# info functions + +sub dirsize($) { + my $dir = shift; + my $size = 0; + my $files = 0; + for my $f (glob "$dir/*") { + $size += (stat $f)[7]; + $files++; + } + return $size, $files; +} + +sub list() { + print "Cluster $version $cluster backups in $clusterdir:\n"; + my $totalsize = 0; + print "Dumps:\n"; + for my $dir (sort glob "$clusterdir/*.dump") { + my ($size, $files) = dirsize($dir); + my $status = -f "$dir/status" ? '' : ' BROKEN'; + print " $dir: $size Bytes$status\n"; + $totalsize += $size; + } + print "Basebackups:\n"; + for my $dir (sort glob "$clusterdir/*.backup") { + my ($size, $files) = dirsize($dir); + my $status = -f "$dir/status" ? '' : ' BROKEN'; + print " $dir: $size Bytes$status\n"; + $totalsize += $size; + } + if (-d "$clusterdir/wal") { + print "WAL:\n"; + my ($size, $files) = dirsize("$clusterdir/wal"); + print " $clusterdir/wal: $size Bytes, $files Files\n"; + $totalsize += $size; + } + print "Total: $totalsize Bytes\n"; +} + +# main + +if ($action eq 'createdirectory') { + create_directory(); + +} elsif ($action eq 'basebackup') { + error "basebackups of pre-9.1 servers are not supported" if ($version < 9.1); + my $starttime = time; + create_directory(); + switch_to_cluster_owner(); + my $backupdir = get_backupdir($starttime, 'backup'); + $SIG{__DIE__} = sub { remove_backup_on_error($backupdir) }; + create_basebackup($backupdir); + create_configbackup($backupdir); + create_status($action, $starttime, $backupdir, 'ok'); + $SIG{__DIE__} = undef; + sync($backupdir); + compresswal(); + +} elsif ($action eq 'dump') { + error "dumps of pre-9.3 servers are not supported" if ($version < 9.3); + my $starttime = time; + create_directory(); + switch_to_cluster_owner(); + my $backupdir = get_backupdir($starttime, 'dump'); + $SIG{__DIE__} = sub { remove_backup_on_error($backupdir) }; + create_dumpall($backupdir); + create_configbackup($backupdir); + create_status($action, $starttime, $backupdir, 'ok'); + $SIG{__DIE__} = undef; + sync($backupdir); + +} elsif ($action eq 'expiredumps' and @ARGV == 2 and $ARGV[1] =~ /^(\d+)$/) { + switch_to_cluster_owner(); + expire_backups('dump', $1); + +} elsif ($action eq 'expirebasebackups' and @ARGV == 2 and $ARGV[1] =~ /^(\d+)$/) { + switch_to_cluster_owner(); + expire_backups('backup', $1); + archivecleanup(); + compresswal(); + +} elsif ($action eq 'deletebroken') { + switch_to_cluster_owner(); + delete_broken(); + +} elsif ($action eq 'receivewal') { + create_directory(); + switch_to_cluster_owner(); + create_wal_directory(); + compresswal(); + receivewal(); + +} elsif ($action eq 'compresswal') { + switch_to_cluster_owner(); + compresswal(); + +} elsif ($action eq 'archivecleanup') { + switch_to_cluster_owner(); + archivecleanup(); + compresswal(); + +} elsif ($action eq 'list') { + switch_to_cluster_owner(); + list(); + +} else { + help(); + exit(1); +} + +__END__ + +=head1 NAME + +pg_backupcluster - simple pg_basebackup and pg_dump front-end + +=head1 SYNOPSIS + +B<pg_backupcluster> [I<options>] I<version> I<cluster> I<action> + +=head1 DESCRIPTION + +B<pg_backupcluster> provides a simple interface to create PostgreSQL cluster +backups using L<pg_basebackup(1)> and L<pg_dump(1)>. + +To ease integration with B<systemd> operation, the alternative syntax +"B<pg_basebackup> I<version>B<->I<cluster> I<action>" is also supported. + +=head1 ACTIONS + +=over 4 + +=item B<createdirectory> + +Create /var/backups and /var/backups/I<version>-I<cluster>. +This action can be run as root to create the directories required for backups. +All other actions will also attempt to create the directories when missing, but +can of course only do that when running as root. They will switch to the +cluster owner after this step. + +=item B<basebackup> + +Backup using L<pg_basebackup(1)>. The resulting basebackup contains the WAL +files required to run recovery on startup. + +=item B<dump> + +Backup using L<pg_dump(1)>. Global objects (users, tablespaces) are dumped +using L<pg_dumpall(1)> B<--globals-only>. Individual databases are dumped into +PostgreSQL's custom format. + +=item B<expirebasebackups> I<N> + +Remove all but last the I<N> basebackups. + +=item B<expiredumps> I<N> + +Remove all but last the I<N> dumps. + +=item B<receivewal> + +Launch pg_receivewal. WAL files are gzip-compressed in PG 10+. + +=item B<compresswal> + +Compress WAL files in archive. + +=item B<archivecleanup> + +Remove obsolete WAL files from archive using L<pg_archivecleanup(1)>. + +=item B<list> + +Show dumps, basebackups, and WAL, with size. + +=back + +=head1 OPTIONS + +=over 4 + +=item B<-c --checkpoint=spread|fast> + +Passed to B<pg_basebackup>. Default is B<spread>. + +=item B<-k --keep-on-error> + +Keep faulty backup directory on error. By default backups are delete on error. + +=item B<-v --verbose> + +Verbose output, even when not running on a terminal. + +=back + +=head1 FILES + +=over 4 + +=item /var/backups + +Default root directory for cluster backup directories. + +=item /var/backups/I<version>-I<cluster> + +Default directory for cluster backups. + +=item /var/backups/I<version>-I<cluster>/I<timestamp>B<.basebackup> + +Backup from B<pg_backupcluster ... basebackup>. + +=over 4 + +=item C<config.tar.gz> + +Tarball of cluster configuration directory (postgresql.conf, pg_hba.conf, ...) +in /etc/postgresql. + +=item I<tablespace>C<.tar.gz>, C<pg_wal.tar.gz>, C<backup_manifest> + +Tablespace and WAL tarballs and backup info written by B<pg_basebackup>. + +=item C<status> + +Completion timestamp of backup run. + +=back + +=item /var/backups/I<version>-I<cluster>/I<timestamp>B<.dump> + +Backup from B<pg_backupcluster ... dump>. + +=over 4 + +=item C<config.tar.gz> + +Tarball of cluster configuration directory (postgresql.conf, pg_hba.conf, ...) +in /etc/postgresql. + +=item C<createcluster.opts> + +Options (encoding, locale, data checksums) to be passed to B<pg_createcluster> +for restoring this cluster. + +=item C<globals.sql> + +Global objects (roles, tablespaces) from B<pg_dumpall --globals-only>. + +=item C<databases.sql> + +SQL commands to create databases and restore database-level options. + +=item I<database>C<.dump> + +Database dumps from B<pg_dump --format=custom>. + +=item C<status> + +Completion timestamp of backup run. + +=back + +=item /var/backups/I<version>-I<cluster>/B<wal> + +WAL files from B<pg_receivewal>. + +=back + +=head1 CAVEATS + +For dump-style backups, not all properties of the original cluster are preserved: + +=over 2 + +=item * In PostgreSQL 10 and earlier, ALTER ROLE ... IN DATABASE is not supported. + +=item * Not all B<initdb> options are carried over. Currently supported are B<--encoding>, +B<--lc-collate>, B<--lc-collate>, and B<-k --data-checksums>. + +=back + +The earliest PostgreSQL version supported for dumps is 9.3. +For basebackups, the earliest supported version is 9.1. +B<receivewal> (and hence archive recovery) are supported in 9.5 and later. + +=head1 SEE ALSO + +L<pg_restorecluster(1)>, +L<pg_dump(1)>, L<pg_dumpall(1)>, +L<pg_basebackup(1)>, L<pg_receivewal(1)>, L<pg_archivecleanup(1)>. + +=head1 AUTHOR + +Christoph Berg L<E<lt>myon@debian.orgE<gt>> diff --git a/pg_buildext b/pg_buildext new file mode 100755 index 0000000..304e8a1 --- /dev/null +++ b/pg_buildext @@ -0,0 +1,503 @@ +#!/bin/bash +# +# build a PostgreSQL module based on PGXS for given list of supported major +# versions +# +# (C) 2010 Dimitri Fontaine <dfontaine@hi-media.com> +# (C) 2011-2023 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +set -eu + +die() { + echo "$(basename $0): error: $*" >&2 + exit 1 +} + +VENVARGS="" +MAKEVARS="" +while getopts "c:i:m:o:s" opt ; do + case $opt in + c|i|o) VENVARGS="$VENVARGS -$opt $OPTARG" ;; + m) MAKEVARS="$OPTARG" ;; + s) VENVARGS="$VENVARGS -$opt" ;; + *) exit 1 ;; + esac +done +# shift away args +shift $(($OPTIND - 1)) + +# positional arguments: action [srcdir] [target [opt]] +[ "${1:-}" ] || die "no action given" +action="$1" +shift + +if [ -d "${1:-}" ] && [ "${2:-}" ]; then # optional: source directory + srcdir="${1:-}" + [ "$srcdir" = "." ] && srcdir="$PWD" + shift +else + srcdir="$PWD" +fi +target="${1:-}" +opt="${2:-}" + +prepare_env() { + local version=$1 + vtarget=`echo $target | sed -e "s:%v:$version:g"` + pgc="/usr/lib/postgresql/$version/bin/pg_config" + [ -e "$pgc" ] || die "$pgc does not exist" + if [ "${CFLAGS:-}" ]; then + export COPT="$CFLAGS" + fi +} + +configure() { + prepare_env $1 + confopts=`echo $opt | sed -e "s:%v:$1:g"` + + mkdir -p $vtarget + ( echo "calling configure in $vtarget" && + cd $vtarget && $srcdir/configure $confopts PG_CONFIG="$pgc" VPATH="$srcdir" ) || return $? +} + +build() { + prepare_env $1 + if [ "$opt" ]; then + cflags="$(echo $opt | sed -e "s:%v:$1:g")" + export COPT="${COPT:+$COPT }$cflags" + fi + + mkdir -p $vtarget + # if a Makefile was created by configure, use it, else the top level Makefile + [ -f $vtarget/Makefile ] || makefile="-f $srcdir/Makefile" + make -C $vtarget ${makefile:-} PG_CONFIG="$pgc" VPATH="$srcdir" USE_PGXS=1 $MAKEVARS || return $? +} + +substvars() { + version="$1" + package="$2" + + depends="postgresql-$version" + if [ -d "debian/$package/usr/lib/postgresql/$version/lib/bitcode" ]; then + pg_provides=$(dpkg-query --show --showformat='${Provides}' postgresql-$version | grep -o 'postgresql-[0-9]*-jit-llvm (= [0-9.]*') + llvm_version=${pg_provides#*= } + if [ "$llvm_version" ]; then # skip if server version doesn't have the Provides yet + depends="$depends, postgresql-$version-jit-llvm (>= $llvm_version)" + fi + fi + echo "postgresql:Depends=$depends" >> "debian/$package.substvars" + + if ! grep -q postgresql:Depends debian/control; then # compat: add to misc:Depends + if grep -q ^misc:Depends "debian/$package.substvars"; then + sed -i -e "s/^misc:Depends=/misc:Depends=$depends, /" "debian/$package.substvars" + else + echo "misc:Depends=$depends" >> "debian/$package.substvars" + fi + fi +} + +install() { + prepare_env $1 + package=`echo $opt | sed -e "s:%v:$1:g"` + + mkdir -p $vtarget + # if a Makefile was created by configure, use it, else the top level Makefile + [ -f $vtarget/Makefile ] || makefile="-f $srcdir/Makefile" + make -C $vtarget ${makefile:-} install DESTDIR="$PWD/debian/$package" PG_CONFIG="$pgc" VPATH="$srcdir" USE_PGXS=1 $MAKEVARS || return $? + substvars "$1" "$package" +} + +clean() { + prepare_env $1 + if [ "$vtarget" ]; then + rm -rf $vtarget + fi +} + +loop() { + prepare_env $1 + package=$(echo $target | sed -e "s:%v:$1:g") + + echo "# $1: make" + make -C "$srcdir" PG_CONFIG="$pgc" USE_PGXS=1 $MAKEVARS || return $? + echo "# $1: make install" + make -C "$srcdir" install DESTDIR="$PWD/debian/$package" PG_CONFIG="$pgc" USE_PGXS=1 $MAKEVARS || return $? + substvars "$1" "$package" + echo "# $1: make clean" + make -C "$srcdir" clean PG_CONFIG="$pgc" USE_PGXS=1 $MAKEVARS # clean errors are fatal +} + +installcheck() { + prepare_env $1 + + # ask pg_regress to output unified diffs + export PG_REGRESS_DIFF_OPTS="-U3" + + # ask pg_virtualenv to create a non-system cluster + if [ "${NONROOT-unset}" = "unset" ]; then + export NONROOT=1 + fi + + # if a package pattern is given, tell pg_virtualenv where the installed files are + if [ "$opt" ]; then + pkg=$(echo "$opt" | sed -e "s:%v:$1:g") + PKGARGS="-p $pkg" + DESTDIR="DESTDIR=$PWD/debian/$pkg" + fi + + if [ "$target" ] && [ "$target" != "." ]; then # if target is given, use it, else stay in the top source dir + # if a Makefile was created by configure, use it, else the top level Makefile + [ -f $vtarget/Makefile ] || makefile="-f $srcdir/Makefile" + if ! pg_virtualenv ${PKGARGS:-} $VENVARGS -v $1 \ + make -C $vtarget ${makefile:-} installcheck ${DESTDIR:-} \ + PG_CONFIG="$pgc" VPATH="$srcdir" USE_PGXS=1 $MAKEVARS; then + if [ -r $vtarget/regression.diffs ]; then + echo "**** $vtarget/regression.diffs ****" + cat $vtarget/regression.diffs + fi + return 1 + fi + else + if ! pg_virtualenv ${PKGARGS:-} $VENVARGS -v $1 \ + make installcheck ${DESTDIR:-} PG_CONFIG="$pgc" USE_PGXS=1 $MAKEVARS; then + if [ -r regression.diffs ]; then + echo "**** regression.diffs ****" + cat regression.diffs + fi + return 1 + fi + # since we are in the top-level directory, clean up behind us + make clean PG_CONFIG="$pgc" USE_PGXS=1 $MAKEVARS + fi +} + +run_run () { + local v=$1 + shift + + ( + echo + "${@//%v/$v}" + "${@//%v/$v}" < $PSQLTMP + ) || return $? +} +run_run_installed () { + run_run "$@" +} + +run_psql () { + prepare_env $1 + + # ask pg_virtualenv to create a non-system cluster + if [ "${NONROOT-unset}" = "unset" ]; then + export NONROOT=1 + fi + + # if a package pattern is given, tell pg_virtualenv where the installed files are + if [ "$opt" ]; then + pkg=$(echo "$opt" | sed -e "s:%v:$1:g") + PKGARGS="-p $pkg" + export DESTDIR="$PWD/debian/$pkg" + fi + + ( + if [ "$target" ] && [ "$target" != "." ]; then # if target is given, use it, else stay in the top source dir + cd $target + fi + pg_virtualenv ${PKGARGS:-} $VENVARGS -v $1 \ + psql -Xe -v ON_ERROR_STOP=1 < $PSQLTMP + ) || return $? +} + +run_virtualenv () { + prepare_env $1 + + # ask pg_virtualenv to create a non-system cluster + if [ "${NONROOT-unset}" = "unset" ]; then + export NONROOT=1 + fi + + # if a package pattern is given, tell pg_virtualenv where the installed files are + if [ "$opt" ]; then + pkg=$(echo "$opt" | sed -e "s:%v:$1:g") + PKGARGS="-p $pkg" + export DESTDIR="$PWD/debian/$pkg" + fi + + ( + if [ "$target" ] && [ "$target" != "." ]; then # if target is given, use it, else stay in the top source dir + cd $target + fi + pg_virtualenv ${PKGARGS:-} $VENVARGS -v $1 ${SHELL:-/bin/sh} -ex < $PSQLTMP + ) || return $? +} + +versions() { + [ -e /usr/share/postgresql-common/supported-versions ] || + die "/usr/share/postgresql-common/supported-versions not found" + [ -e debian/pgversions ] || die "debian/pgversions not found" + supported_versions=$(/usr/share/postgresql-common/supported-versions) + local version + while read version; do + case $version in + all) echo "$supported_versions" ;; + [1-9]*+) + for sup_version in $supported_versions; do + if dpkg --compare-versions "${version%+}" le "$sup_version"; then echo "$sup_version"; fi + done ;; + [1-9]*) + for sup_version in $supported_versions; do + if [ "$version" = "$sup_version" ]; then echo "$sup_version"; fi + done ;; + '#'*) ;; + '') ;; + *) echo "Syntax error in debian/pgversions: $version" >&2 ;; + esac + done < debian/pgversions +} + +# list of PostgreSQL versions for which packages built from this source are installed +# (not necessarily the same as listed in debian/control) +installed_versions() { + [ -e debian/control.in ] || die "debian/control.in not found" + # extract package name template(s) from control.in, split into prefix and suffix + perl -lne 'print "$1 $2" if /^Package: (.*)PGVERSION(.*)/' debian/control.in | \ + while read prefix suffix; do + # translate templates to actually installed packages, and extract version numbers + dpkg-query --showformat '${db:Status-Status} ${Package}\n' --show "$prefix*$suffix" | \ + grep '^installed' | cut -d ' ' -f 2 | sed -e "s/^$prefix//; s/$suffix\$//" | grep -E '^[0-9.]+$' + # if suffix is empty, the grep will filter out noise like '13-dbgsym' + done | sort -uV +} + +gencontrol() { + tmpcontrol=$(mktemp debian/control.XXXXXX) + if [ -f debian/tests/control.in ]; then + tmptestscontrol=$(mktemp debian/tests/control.XXXXXX) + fi + trap "rm -f $tmpcontrol ${tmptestscontrol:-}" EXIT + + export PGVERSIONS="$(versions)" + [ "$PGVERSIONS" ] || die "No current PostgreSQL versions are supported by this package" + + perl -e \ + '$/ = ""; # read paragraphs + my @pgversions = split /\s+/, $ENV{PGVERSIONS}; + my $newest_pgversion = $pgversions[-1]; + + sub replace_all ($) { + $_ = shift; + my @out; + for my $version (@pgversions) { + push @out, s/PGVERSIONS/$version/rg; + } + return join ", ", @out; + } + + while (<>) { + chomp; + if (/^Package: .*PGVERSION/) { + foreach my $version (@pgversions) { + push @out, s/PGVERSION/$version/rg; + } + } else { + s/( + [^[:space:](),]* + PGVERSIONS + [^[:space:](),]* + (?:\s*\([^(),]*\))? # optional part in parentheses + [^[:space:](),]* + )/replace_all($1)/xeg; + s/PGVERSION/$newest_pgversion/g; + push @out, $_; + } + } + print join("\n\n", @out), "\n"; + ' debian/control.in > $tmpcontrol + + if [ -f debian/tests/control.in ]; then + cp debian/tests/control.in $tmptestscontrol + # find words (package names) containing PGVERSION + REGEXP='[[:alnum:]-]*PGVERSION[[:alnum:]-]*' + for pkgpattern in $(grep -Ewo "$REGEXP" debian/tests/control.in | sort -u); do + repl="" + # build an array of replacements separated by , + for v in $(versions); do + repl="${repl:+$repl, }$(echo $pkgpattern | sed -e "s/PGVERSION/$v/g")" + done + # put array into control file + grep -q "$pkgpattern" $tmptestscontrol # assert the pattern didn't get already removed by an earlier replacement + sed -i -e "s/$pkgpattern/$repl/" $tmptestscontrol + done + fi +} + +updatecontrol() { + cat $tmpcontrol > debian/control + + if [ -f debian/tests/control.in ]; then + cat $tmptestscontrol > debian/tests/control + fi + + # re-check build-dependencies (no error raised) + ( dpkg-checkbuilddeps 2>&1 || : ) | sed -e 's/error/notice/' +} + +# when a version is included in the action, just act on this one (this is +# useful if some extra work needs to be done per version, so the loop over +# supported-versions needs to be in the script calling pg_buildext) + +case $action in + configure-*|build-*|install-*|clean-*|installcheck-*) + a=${action%%-*} + v=${action##$a-} + ret=0 + echo "### PostgreSQL $v $a ###" + if $a $v; then + echo "### End $v $a ###" + else + ret=$? + echo "### End $v $a (FAILED with exit code $ret) ###" + fi + exit $ret + ;; + + checkcontrol) + [ -f debian/control.in ] || exit 0 # silently exit if debian/control.in is not there + gencontrol + need_update= + if ! diff -u debian/control $tmpcontrol; then + if [ "${PG_UPDATECONTROL:-no}" != "no" ] || head -1 debian/changelog | grep -Eq -- '-backports|-pgdg|-pgapt'; then + echo "Notice: Updating debian/control from debian/control.in." + need_update=1 + else + echo "Error: debian/control needs updating from debian/control.in. Run 'pg_buildext updatecontrol'." + echo "If you are seeing this message in a buildd log, a sourceful upload is required." + exit 1 + fi + fi + if [ -f debian/tests/control.in ] && ! diff -u debian/tests/control $tmptestscontrol; then + echo "Notice: Updating debian/tests/control from debian/tests/control.in." + need_update=1 + fi + [ "$need_update" ] && updatecontrol + exit 0 + ;; + + updatecontrol) + [ -f debian/control.in ] || die "debian/control.in is missing, cannot update debian/control" + gencontrol + updatecontrol + exit + ;; + + clean) + if [ "$target" ]; then + pattern=$(echo "$target" | sed -e "s:%v:*:g") + echo rm -rf $pattern/ + rm -rf $pattern/ + fi + if [ "$opt" ]; then + pattern=$(echo "$opt" | sed -e "s:%v:*:g") + echo rm -rf debian/$pattern/ debian/$pattern.substvars + rm -rf debian/$pattern/ debian/$pattern.substvars + fi + if [ -f Makefile ]; then + make clean USE_PGXS=1 + fi + exit + ;; + + installed-versions) + installed_versions + exit + ;; + + installcheck) + # prefer testing installed versions over supported versions as the set + # of versions might have changed (unless a package-pattern is provided) + if [ -f debian/control.in ] && [ -z "$opt" ]; then + versions=$(installed_versions) + else + versions=$(versions) + fi + [ "$versions" ] || exit 1 + ret=0 + for v in $versions; do + echo "### PostgreSQL $v $action ###" + if $action $v; then + echo "### End $v $action ###" + else + ret=$? + echo "### End $v $action (FAILED with exit code $ret) ###" + fi + done + exit $ret + ;; + + run|run_installed|psql|virtualenv) + PSQLTMP=$(mktemp --tmpdir psql.XXXXXX) + trap "rm -f $PSQLTMP" EXIT + # if we are fed any input (and hence aren't reading from a terminal), + # store it in a tempfile + [ ! -t 0 ] && cat > $PSQLTMP + + # prefer testing installed versions over supported versions as the set + # of versions might have changed (unless a package-pattern is provided) + if [ "$action" != "run" ] && [ -f debian/control.in ] && [ -z "$opt" ]; then + versions=$(installed_versions) + else + versions=$(versions) + fi + [ "$versions" ] || exit 1 + ret=0 + for v in $versions; do + echo "### PostgreSQL $v $action ###" + if run_$action $v "$@"; then + echo "### End $v $action ###" + else + ret=$? + echo "### End $v $action (FAILED with exit code $ret) ###" + fi + done + exit $ret + ;; +esac + +# loop over versions + +ret=0 +for v in $(versions) +do + case "$action" in + "supported-versions") + echo $v + ;; + + configure|build|install|clean|loop) + [ "$target" ] || die "syntax: pg_buildext $action <target> [<srcdir>] [<opt>]" + echo "### PostgreSQL $v $action ###" + if $action $v; then + echo "### End $v $action ###" + else + ret=$? + echo "### End $v $action (FAILED with exit code $ret) ###" + fi + ;; + + *) + die "unsupported action '$action'" + ;; + esac +done + +exit $ret diff --git a/pg_buildext.pod b/pg_buildext.pod new file mode 100644 index 0000000..d76c701 --- /dev/null +++ b/pg_buildext.pod @@ -0,0 +1,358 @@ +=head1 NAME + +pg_buildext - Build and install a PostgreSQL extension + +=head1 SYNOPSIS + +B<pg_buildext> [I<options>] I<action> [I<src-dir>] [I<arguments>] + +=head1 DESCRIPTION + +B<pg_buildext> is a script that will build a PostgreSQL extension in a C<VPATH> +way, for potentially several PostgreSQL server versions in parallel. +It builds for the intersection of versions known in +C<debian/pgversions> (versions supported by the package) and in +C</usr/share/postgresql-common/supported-versions> (versions supported in this +release). + +Many PostgreSQL extension packages require no special handling at build time +and can use B<dh $@ --with pgxs> or B<dh $@ --with pgxs_loop> to +automatically execute the steps outlined below. + +=head1 USAGE + +Packages using B<pg_buildext> should be prepared to build binaries for +PostgreSQL versions that are not present in Debian unstable, e.g. for older +releases when building backports for Debian (old)stable (possibly including +backports of newer PostgreSQL releases), or for all PostgreSQL releases when +the package is built for B<apt.postgresql.org>. + +As the set of binary packages depends on the target PostgreSQL versions, +C<debian/control> is generated from a template in C<debian/control.in> when +B<pg_buildext updatecontrol> is run. +Package sections that contain B<PGVERSION> in the package name are replaced by +a list of sections, filling in the supported PostgreSQL versions. +Package sections that contain B<PGVERSION> outside the package name have the +newest supported PostgreSQL version filled in (useful for meta packages); +words containing B<PGVERSIONS> be replaced by a list of words with the +supported versions filled in, this is most useful in B<Build-Depends>. +Include +C</usr/share/postgresql-common/pgxs_debian_control.mk> in C<debian/rules> to +run a check at build time if updating debian/control is required. + +As B<pg_buildext> invokes B<make> for the B<build>, B<install>, and B<clean> +actions, invocations from C<debian/rules> (which is a makefile) should be prefixed +with B<+> so the sub-makes can talk with the make jobserver. Additional makefile +variables can be passed to B<make> via the B<-m> option. + +Many extensions support B<make installcheck> testing using B<pg_regress>. As +this needs the package to be installed, it cannot be run at build time. +Instead, the tests should be run using B<autopkgtest> from C<debian/tests/*>. + +If C<debian/tests/control.in> exists, occurrences of package names containing +B<PGVERSION> are replaced by lists of package names with the target PostgreSQL +versions filled in. (If no replacing is needed in C<debian/tests/control>, it +is fine to provide the tests control file directly.) + +=head1 OPTIONS + +=over 4 + +=item B<-cio> I<arg> + +=item B<-s> + +Passed to B<pg_virtualenv> when running B<installcheck>. + +=item B<-m> I<arg> + +Passed to B<make>. + +=back + +=head1 ACTIONS + +Most actions expect a directory name where to build the sources. It will get +created for you if it does not exist. If the I<build-dir> contains a C<%v> +sign, it will get replaced by the specific version of PostgreSQL being built +against. (Usually this parameter is C<build-%v>.) + +=over 4 + +=item B<supported-versions> + +Print effective list of supported versions, i.e. the intersection of the sets +of versions supported by the system +(from C</usr/share/postgresql-common/supported-versions>) and the package +(from C<debian/pgversions>). + +Use this when building packages. + +=item B<installed-versions> + +In the list of installed packages, look for packages matching the B<PGVERSION> +package name templates from C<debian/control.in>, and print the PostgreSQL +major version number part. + +Use this when testing packages. + +=item B<checkcontrol> + +Check if C<debian/control> needs updating from C<debian/control.in>. This is +invoked from C</usr/share/postgresql-common/pgxs_debian_control.mk>. When +building for a B<backports> or B<pgdg> suite as determined by +C<debian/changelog>, this action also updates the control file. Otherwise, +B<updatecontrol> needs to be run manually. + +=item B<updatecontrol> + +Update C<debian/control> from C<debian/control.in>, and C<debian/tests/control> +from C<debian/tests/control.in> if the latter exists. + +=item B<configure> [I<src-dir>] I<build-dir> [I<extra-configure-options>] + +For every supported version, call B<../configure> from the I<build-dir> +directory. (Most PostgreSQL extensions do not have a configure script.) + +=item B<build> [I<src-dir>] I<build-dir> [I<extra-cflags>] + +Build the extension in the I<build-dir> directory. + +=item B<install> [I<src-dir>] I<build-dir> I<package-pattern> + +Invoke B<make install> from the I<build-dir> directory. +The third parameter specifies the package name to use. Most packages +use B<postgresql-%v-pkgname>. Make will be +called with DESTDIR="$(CURDIR)/debian/I<package>". + +The B<dpkg> substitution variable B<postgresql:Depends> is set to depend +on the required PostgreSQL server package. For compatibility with previous +packaging standards, the dependency is also added to B<misc:Depends> if +postgresql:Depends is not used. + +=item B<clean> [I<src-dir>] [I<build-dir>] [I<package-pattern>] + +Clean the build directories. + +=item B<loop> [I<src-dir>] I<package-pattern> + +As a variant to calling B<build> and B<install> separately for VPATH builds, +loop over the supported PostgreSQL versions in the top source directory. This +should be used if the package does not support VPATH builds. As it also invokes +B<make install>, it should be placed were installation happens in debian/rules, +rather than where build would normally be called. + +=item B<installcheck> [I<src-dir>] [I<build-dir>] [I<package-pattern>] + +Use B<pg_virtualenv make installcheck> to run the extension regression tests. +This is meant to be run from C<debian/tests/control> using B<autopkgtest>. If +I<build-dir> is omitted, the top source directory is used. + +If I<package-pattern> is given, options are passed to B<pg_virtualenv> to set +up the temporary PostgreSQL instance to find extension files in +C<debian/package-directory/>. + +Other than the other actions which run on the "supported" versions, if C<debian/control.in> exists, this one +runs on the "installed" versions as reported by B<installed-versions> (unless +I<package-pattern> is provided, which means we are called during a build). + +=item B<psql> [I<src-dir>] [I<build-dir>] [I<package-pattern>] + +=item B<virtualenv> [I<src-dir>] [I<build-dir>] [I<package-pattern>] + +Like B<installcheck>, but invokes B<psql>, or a shell, both wrapped in +B<pg_virtualenv>. Input is read from stdin. + +=item B<run> [I<src-dir>] I<command> + +=item B<run_installed> [I<src-dir>] I<command> + +Runs I<command>, with B<%v> placeholder replaced on all supported versions, or +all installed versions, respectively. + +=back + +Sometimes it is desirable to run extra code per version before invoking the +action, in that case the loop over supported versions needs to be in the +calling script. To facilitate this mode, actions can also be called as +I<action>B<->I<version>. See the installcheck example below. + +=head1 SUPPORTED VERSIONS + +B<pg_buildext> reads C<debian/pgversions> to decide which PostgreSQL to build +modules/extensions for. This file contains one PostgreSQL version number per +line, in the following formats: + +=over 4 + +=item B<all> + +Support all versions. This is recommended unless there are known incompatibilities. + +=item I<NN> + +Support this version. + +=item I<NN>B<+> + +Support this and all greater versions. + +=item B<#>I<...> + +Comment. + +=back + +For a version to be used, it must also be listed in the output of +C</usr/share/postgresql-common/supported-versions>. See this file for how to +configure the list of supported versions on your system. + +=head1 EXAMPLE + +=over 4 + +=item B<debian/control.in:> + + Source: postgresql-foobar + Rules-Requires-Root: no + Build-Depends: + debhelper, + postgresql-all <!nocheck>, + postgresql-server-dev-all (>= 217~), + + Package: postgresql-PGVERSION-foobar + Architecture: any + Depends: + ${misc:Depends}, + ${postgresql:Depends}, + ${shlibs:Depends}, + +=item B<debian/pgversions:> + + all + + # alternatives: + #9.6 + #11+ + +=item B<debian/rules> using B<dh $@ --with pgxs>: + + #!/usr/bin/make -f + + override_dh_installdocs: + dh_installdocs --all README.* + + %: + dh $@ --with pgxs + +=item If the package does no support building from subdirectories, use B<dh $@ --with pgxs_loop>: + + #!/usr/bin/make -f + + %: + dh $@ --with pgxs_loop + +=item If the package does not use PGXS's "make installcheck" for testing: + + override_dh_pgxs_test: + +=item B<debian/rules> using B<pg_buildext> directly: + + #!/usr/bin/make -f + + include /usr/share/postgresql-common/pgxs_debian_control.mk + + # omit this if the package does not use autoconf + override_dh_auto_configure: + +pg_buildext configure build-%v "--libdir=/usr/lib/postgresql/%v/lib --datadir=/usr/share/postgresql-%v-foobar" + + override_dh_auto_build: + +pg_buildext build build-%v + + override_dh_auto_test: + # nothing to do here, see debian/tests/* instead + + override_dh_auto_install: + +pg_buildext install build-%v postgresql-%v-foobar + + override_dh_installdocs: + dh_installdocs --all README.* + + override_dh_auto_clean: + +pg_buildext clean build-%v + + %: + dh $@ + +=item B<debian/tests/control:> + + Depends: @, postgresql-server-dev-all + Tests: installcheck + Restrictions: allow-stderr + +=item B<debian/tests/control.in:> (optional) + + Depends: @, postgresql-contrib-PGVERSION, postgresql-PGVERSION-bar + Tests: installcheck + Restrictions: allow-stderr + +=item B<debian/tests/installcheck:> + + #!/bin/sh + pg_buildext installcheck + # alternatively: pg_buildext installcheck build-%v + + # Running extra code before invoking the actual action: + set -e + for v in $(pg_buildext installed-versions); do + test -L build-$v/sql || ln -s ../sql build-$v/ + test -L build-$v/expected || ln -s ../expected build-$v/ + pg_buildext installcheck-$v build-$v + done + +=back + +=head1 SOURCE DIRECTORY + +If the package source code is not in the top level directory (i.e. the directory +which has C<debian/> as subdirectory), use the I<src-dir> argument, where +I<src-dir> must be an absolute path. Example: + + override_dh_auto_build: + +pg_buildext build $(CURDIR)/postgresql-module build-%v + +=head1 COMPATIBILITY + +B<pg_buildext loop> was introduced in postgresql-server-dev-all (>= 141~). + +The usage of "all" or "NN+" in debian/pgversions was introduced in +postgresql-server-dev-all (>= 148~). + +B<pg_buildext installcheck> was introduced in postgresql-server-dev-all (>= +153~). + +B<PG_VIRTUALENV_UNSHARE=-n> was introduced in postgresql-common (>= 170~). + +Handling of C<debian/tests/control.in> with B<PGVERSION> replacement was +introduced in postgresql-common (>= 171~). + +The action B<installed-versions> was introduced in postgresql-common (>= 208~). +B<installcheck> was switched to use it in the same version. + +B<dh $@ --with pgxs> and B<pgxs_loop>, the corresponding B<--buildsystem>, and +the B<psql> and B<virtualenv> actions were introduced in postgresql-server-dev-all (>= 217~). + +The replacement of B<PGVERSIONS> (plural) in debian/control.in and +B<pg_buildext run> and B<run_installed> were introduced in +postgresql-common (>= 256~). + + +=head1 SEE ALSO + +C</usr/share/postgresql-common/supported-versions>, autopkgtest(1), +pg_virtualenv(1). + +=head1 AUTHORS + +Dimitri Fontaine L<E<lt>dim@tapoueh.orgE<gt>>, with extensions by +Christoph Berg L<E<lt>myon@debian.orgE<gt>>. diff --git a/pg_checksystem b/pg_checksystem new file mode 100755 index 0000000..bf61fef --- /dev/null +++ b/pg_checksystem @@ -0,0 +1,59 @@ +#!/usr/bin/perl -wT + +# Check various system parameters for PostgreSQL. This needs to be run as root. +# +# (C) 2005-2009 Martin Pitt <mpitt@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; + +use PgCommon; + +# Check write cache setting for given drive. +# Arguments: <device> +# Returns: 0 = disabled, 1 = enabled, -1: could not determine +sub get_device_write_cache { + open DRV, $_[0] or return -1; + sub HDIO_GET_WCACHE () {0x30e;} + my $pval = pack 'l', 0; + ioctl DRV, &HDIO_GET_WCACHE, $pval or return -1; + my ($val) = unpack 'l', $pval; + close DRV; + return $val; +} + + +error 'This command needs to be executed as root' if $> != 0; + +# collect write cache-enabled drives with databases +my %drive_wcache; +my %seen; +for my $v (get_versions) { + for my $c (get_version_clusters $v) { + my $datadir = PgCommon::cluster_data_directory $v, $c; + my $drv = PgCommon::get_file_device $datadir; + $drv =~ s/[0-9]+$//; + unless (exists $seen{$drv}) { + $drive_wcache{$drv} = 1 if (get_device_write_cache $drv) > 0; + $seen{$drv} = 1; + } + } +} +my @unsave_drives = keys %drive_wcache; + +if (@unsave_drives) { + print 'Warning: The following devices contain databases and have write +caching enabled: ', (join ' ', @unsave_drives), ' +This could destroy the integrity of your databases in the event of power +failure. Consider disabling the write cache with "hdparm -W 0 <device>". +'; +} diff --git a/pg_config b/pg_config new file mode 100755 index 0000000..efdcf6d --- /dev/null +++ b/pg_config @@ -0,0 +1,33 @@ +#!/bin/sh + +# If postgresql-server-dev-* is installed, call pg_config from the latest +# available one. Otherwise fall back to libpq-dev's version. +# +# (C) 2011 Martin Pitt <mpitt@debian.org> +# (C) 2014-2018 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +set -e +PGBINROOT="/usr/lib/postgresql/" +#redhat# PGBINROOT="/usr/pgsql-" +LATEST_SERVER_DEV=`ls -v $PGBINROOT*/bin/pg_config 2>/dev/null|tail -n1` + +if [ -n "$LATEST_SERVER_DEV" ]; then + exec "$LATEST_SERVER_DEV" "$@" +else + if [ -x /usr/bin/pg_config.libpq-dev ]; then + exec /usr/bin/pg_config.libpq-dev "$@" + else + echo "You need to install postgresql-server-dev-NN for building a server-side extension or libpq-dev for building a client-side application." >&2 + exit 1 + fi +fi diff --git a/pg_conftool b/pg_conftool new file mode 100755 index 0000000..6772b66 --- /dev/null +++ b/pg_conftool @@ -0,0 +1,233 @@ +#!/usr/bin/perl +# Read and edit PostgreSQL config files +# vim:sw=4:et: +# +# (C) 2014-2017 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use warnings; +use Getopt::Long; +use PgCommon qw(quote_conf_value user_cluster_map read_cluster_conf_file + read_conf_file set_conffile_value set_conf_value disable_conffile_value + disable_conf_value cluster_exists error config_bool); + +## option parsing + +sub help ($;$) +{ + my ($exit, $error) = @_; + print STDERR "Error: $error\n" if ($error); + print "Usage: + $0 [options] [<version> <cluster name>] [<configfile>] <command> + +Options: + -b --boolean Format output as boolean + -s --short Print only value + -v --verbose Verbose output + --help This help + +Commands: + show <parameter>|all + set <parameter> <value> + remove <parameter> + edit +"; + exit $exit; +} + +my $boolean = 0; +my $short = 0; +my $verbose = 0; + +Getopt::Long::Configure ("bundling"); +help(1) unless GetOptions ( + 'help' => sub { help(0); }, + 'b|boolean' => \$boolean, + 's|short' => \$short, + 'v|verbose' => \$verbose, +); + +# find command in argument array +my $cmdidx; +for (my $i = 0; $i < @ARGV; $i++) { + if ($ARGV[$i] =~ /^(show|get|set|remove|delete|disable|edit)$/) { + $cmdidx = $i; + last; + } +} +help(1, 'No command given') unless (defined $cmdidx); + +my ($version, $cluster, $conffile); +if ($cmdidx == 0) { # operate on postgresql.conf in default cluster + ($version, $cluster) = user_cluster_map(); + error "No default cluster found" unless ($cluster); + $conffile = 'postgresql.conf'; +} elsif ($cmdidx == 1) { # operate on given file; default cluster must exist if file is relative + $conffile = $ARGV[0]; + unless ($conffile =~ m!^/!) { + ($version, $cluster) = user_cluster_map(); + error "No default cluster found" unless ($cluster); + } +} elsif ($cmdidx == 2) { # version cluster, postgresql.conf + ($version, $cluster, $conffile) = ($ARGV[0], $ARGV[1], 'postgresql.conf'); +} elsif ($cmdidx == 3) { # version cluster conffile + ($version, $cluster, $conffile) = ($ARGV[0], $ARGV[1], $ARGV[2]); +} else { + help(1, 'Too many arguments before command'); +} + +if ($cluster) { # the cluster needs to exist unless an absolute conffile was given + error "Cluster $version $cluster does not exist" + unless (cluster_exists $version, $cluster); +} + +splice @ARGV, 0, $cmdidx; # remove everything before the command +$ARGV[0] = 'show' if ($ARGV[0] eq 'get'); # accept alternative variants of some commands +$ARGV[0] = 'remove' if ($ARGV[0] =~ /delete|disable/); + +my ($cmd, $key, $value); +if ($ARGV[0] =~ /^(edit)$/) { + help(1, "$ARGV[0] takes no argument") unless (@ARGV == 1); + ($cmd) = @ARGV; +} elsif ($ARGV[0] =~ /^(show|remove)$/) { + help(1, "$ARGV[0] needs exactly one argument") unless (@ARGV == 2); + ($cmd, $key) = @ARGV; +} else { + help(1, "$ARGV[0] needs exactly two arguments") unless (@ARGV == 3); + ($cmd, $key, $value) = @ARGV; +} +#print "$version $cluster $conffile $cmd $key $value\n"; + +## main + +if ($cmd eq 'show') { + my %conf; + if ($conffile =~ m!^/!) { + %conf = read_conf_file ($conffile); + } else { + %conf = read_cluster_conf_file ($version, $cluster, $conffile); + } + + if ($key eq 'all') { + foreach my $k (sort keys %conf) { + printf "%s = %s\n", $k, quote_conf_value($conf{$k}); + } + } elsif (exists $conf{$key}) { + $conf{$key} = config_bool($conf{$key}) ? 'on' : 'off' if ($boolean); # normalize boolean output on request + if ($short) { + printf "%s\n", $conf{$key}; + } else { + printf "%s = %s\n", $key, quote_conf_value($conf{$key}); + } + } else { + print "# $key not found in $conffile\n" if ($verbose); + exit 1; + } + +} elsif ($cmd eq 'set') { + if ($conffile =~ m!^/!) { + set_conffile_value ($conffile, $key, $value); + } else { + set_conf_value ($version, $cluster, $conffile, $key, $value); + } + +} elsif ($cmd eq 'remove') { + if ($conffile =~ m!^/!) { + disable_conffile_value ($conffile, $key); + } else { + disable_conf_value ($version, $cluster, $conffile, $key); + } + +} elsif ($cmd eq 'edit') { + my $editor = 'vi'; + if ($ENV{EDITOR}) { + ($editor) = $ENV{EDITOR} =~ /(.*)/; # untaint + } + if ($conffile =~ m!^/!) { + system $editor, $conffile; + } else { + system $editor, "$PgCommon::confroot/$version/$cluster/$conffile"; + } +} + +__END__ + +=head1 NAME + +pg_conftool - read and edit PostgreSQL cluster configuration files + +=head1 SYNOPSIS + +B<pg_conftool> [I<options>] [I<version> I<cluster>] [I<configfile>] B<command> + +=head1 DESCRIPTION + +B<pg_conftool> allows showing and setting parameters in PostgreSQL configuration +files. + +If I<version> I<cluster> is omitted, it defaults to the default cluster (see +user_clusters(5) and postgresqlrc(5)). If I<configfile> is omitted, it defaults +to B<postgresql.conf>. I<configfile> can also be a path, in which case +I<version> I<cluster> is ignored. + +=head1 OPTIONS + +=over 4 + +=item B<-b>, B<--boolean> + +Format boolean value as B<on> or B<off> (not for "show all"). + +=item B<-s>, B<--short> + +Show only the value (instead of key = value pair). + +=item B<-v>, B<--verbose> + +Verbose output. + +=item B<--help> + +Print help. + +=back + +=head1 COMMANDS + +=over 4 + +=item B<show> I<parameter>|B<all> + +Show a parameter, or all present in this config file. + +=item B<set> I<parameter> I<value> + +Set or update a parameter. + +=item B<remove> I<parameter> + +Remove (comment out) a parameter from a config file. + +=item B<edit> + +Open the config file in an editor. Unless B<$EDITOR> is set, B<vi> is used. + +=back + +=head1 SEE ALSO + +user_clusters(5), postgresqlrc(5) + +=head1 AUTHOR + +Christoph Berg L<E<lt>myon@debian.orgE<gt>> diff --git a/pg_createcluster b/pg_createcluster new file mode 100755 index 0000000..66f0b82 --- /dev/null +++ b/pg_createcluster @@ -0,0 +1,952 @@ +#!/usr/bin/perl -w + +# Create new PostgreSQL cluster or integrate an existing data directory into +# the postgresql-common infrastructure. +# +# (C) 2005-2013 Martin Pitt <mpitt@debian.org> +# (C) 2012-2021 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use warnings; +use PgCommon; +use Getopt::Long; +use POSIX qw/lchown setlocale LC_ALL LC_CTYPE/; + +$ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin'; # untaint + +my @postgres_user = getpwnam 'postgres'; +my %defaultconf; +my $explicit_auth_config = 0; +my $quiet; + +chdir '/'; + +# call initdb +sub init_db { + my ($version, $cluster, $datadir, $owneruid, $ownergid, $local_method, $host_method, $initdb_opts_from_cli) = @_; + $datadir = readlink $datadir if (-l $datadir); + + if (system 'install', '-d', '-o', $owneruid, '-g', $ownergid, $datadir) { + error 'could not create data directory; you might need to run this program with root privileges'; + } + + # disable copy-on-write semantics for PostgreSQL data on btrfs and similar; + # this fails on file systems which don't support it, so ignore errors + system "chattr +C '$datadir' 2>/dev/null"; + + my @initdb = (get_program_path 'initdb', $version); + die 'Internal error: could not determine initdb path' unless $initdb[0]; + push @initdb, ('-D', $datadir); + if (my $waldir = $defaultconf{'waldir'} || $defaultconf{'xlogdir'}) { + my $wal = $version >= 10 ? 'wal' : 'xlog'; # renamed in PG 10 + push @initdb, ("--${wal}dir", replace_v_c ($waldir, $version, $cluster)); + } + unless ($explicit_auth_config) { + if ($version >= '9.2') { + push @initdb, ('--auth-local', $local_method); + push @initdb, ('--auth-host', $host_method); + } else { + # trust is the default, but set it explicitly to mute a warning from initdb + # the actual method will be filled in by setup_pg_hba() + push @initdb, ('-A', 'trust'); + } + } + + # cluster startup message + if ($version >= 14) { + push @initdb, '--no-instructions'; + } else { + # ask initdb to print a different cluster start command (Debian patch) + $ENV{CLUSTER_START_COMMAND} = "pg_ctlcluster $version $cluster start"; + } + + # options from config and command line + if (my $options = $defaultconf{'initdb_options'}) { + push @initdb, split (/\s+/, replace_v_c ($options, $version, $cluster)); + } + push @initdb, @$initdb_opts_from_cli; + + if (fork) { + wait; + error 'initdb failed' if $?; + } else{ + change_ugid $owneruid, $ownergid; + print "@initdb\n" unless ($quiet); + close STDOUT if ($quiet); # suppress initdb output + exec @initdb; + } +} + +# move a file to a directory with defined permissions; set <postgresql.conf option> in +# postgresql.conf. +# Arguments: <source file> <target dir> <uid> <gid> <perms> <postgresql.conf option> +sub move_conffile { + my ($file, $target, $version, $cluster, $uid, $gid, $perms, $confopt) = @_; + my $realfile = $file; + while (-l $realfile) { + $realfile = readlink $realfile; + } + if (-e $file) { + install_file $realfile, $target, $uid, $gid, $perms; + unlink $file, $realfile; + + my @pathcomps = split ('/', $file); + $target .= '/' . $pathcomps[-1]; + PgCommon::set_conf_value $version, $cluster, 'postgresql.conf', $confopt, $target if $confopt; + } else { + error "move_conffile: required configuration file $realfile does not exist"; + } +} + +# Set up the default pg_hba.conf file: +# - Add a "local all" entry to pg_hba.conf for the db superuser before all +# other entries. +# - Change default authentication for host entries to md5. +sub setup_pg_hba { + my ($version, $owneruid, $confdir, $local_method, $host_method) = @_; + my $user = (getpwuid $owneruid)[0]; + my $fname = "$confdir/pg_hba.conf"; + my $su_comment = " +# DO NOT DISABLE! +# If you change this first entry you will need to make sure that the +# database superuser can access the database using some other method. +# Noninteractive access to all databases is required during automatic +# maintenance (custom daily cronjobs, replication, and similar tasks). +# +# Database administrative login by Unix domain socket +"; + + open my $hba, '<', $fname or error "could not open $fname for reading"; + my $search = 1; + my @lines; + while (my $line = <$hba>) { + # add superuser entry before column description line + if ($search && $line =~ /#.*TYPE\s+DATABASE/) { + push @lines, $su_comment; + push @lines, sprintf "%-7s %-15s %-39s %s\n", 'local', 'all', $user, $local_method; + push @lines, "\n"; + $search = 0; + } + + if ($version < '9.2' and not $explicit_auth_config) { + # default authentication for Unix socket connections + if ($line =~ /^#?local/) { + $line =~ s/trust/$local_method/; + } + + # default authentication for TCP connections + if ($line =~ /^#?host/) { + $line =~ s/trust/$host_method/; + } + } + + push @lines, $line; + } + close $hba; + + error "setup_pg_hba: did not find insert position" if $search; + + open my $new_hba, '>', $fname or error "could not open $fname for writing"; + foreach (@lines) { + print $new_hba $_; + } + close $new_hba; +} + +# +# Execution starts here +# + +# this flag gets set when we started creating the cluster, and thus we need to +# clean up on errors +my $cleanup_cruft = 0; + +# command line arguments + +my $startconf = ''; +my @pgoptions; +my $createclusterconf = "$PgCommon::common_confdir/createcluster.conf"; +my $environmentfile = "$PgCommon::common_confdir/environment"; +my ($owneruid, $ownergid, $socketdir, $datadir, $custom_logfile, $start, $port); +my ($encoding, $locale, $lc_collate, $lc_ctype, $lc_messages, $lc_monetary, $lc_numeric, $lc_time); +my ($no_status); + +exit 1 unless GetOptions ('u|user=s' => \$owneruid, 'g|group=s' => \$ownergid, + 's|socketdir=s' => \$socketdir, 'd|datadir=s' => \$datadir, + 'start' => \$start, 'e|encoding=s' => \$encoding, + 'l|logfile=s' => \$custom_logfile, 'start-conf=s' => \$startconf, + 'o|pgoption=s' => sub { $_[1] =~ /(.*?)=(.*)/ or error ("No '=' in pgoption '$_[1]'"); + push @pgoptions, [$1, $2]; + }, + 'createclusterconf=s' => \$createclusterconf, + 'environment=s' => \$environmentfile, + 'no-status' => \$no_status, + 'p|port=i' => \$port, + 'q|quiet' => \$quiet, + 'locale=s' => \$locale, + 'lc-collate=s' => \$lc_collate, 'lc-ctype=s' => \$lc_ctype, + 'lc-messages=s' => \$lc_messages, 'lc-monetary=s' => \$lc_monetary, + 'lc-numeric=s' => \$lc_numeric, 'lc-time=s' => \$lc_time); + +# read defaults from /etc/postgresql-common/createcluster.conf +%defaultconf = PgCommon::read_conf_file ($createclusterconf); +# process --pgoption parameters +foreach my $guc (@pgoptions) { + if ($guc->[1] eq '') { + delete $defaultconf{$guc->[0]}; + } else { + $defaultconf{$guc->[0]} = $guc->[1]; + } +} + +$explicit_auth_config = 1 if ($defaultconf{initdb_options} and $defaultconf{initdb_options} =~ /(^| )(-\w*A|--auth\b)/); + +# check validity of locale +unless (setlocale (LC_ALL, "")) { + my $env = join "\n", + map { " $_: $ENV{$_}" } + grep { /^(LC_|LANG)/ } sort keys %ENV; + error ("The locale requested by the environment is invalid:\n$env") +} + +if (@ARGV < 2) { + print "Usage: $0 [options] <version> <cluster name> [-- <initdb options>] + +Options: + -u <uid> cluster owner and superuser (default: 'postgres') + -g <gid> group for data files (default: primary group of owner) + -d <dir> data directory (default: + /var/lib/postgresql/<version>/<cluster name>) + -s <dir> socket directory (default: /var/run/postgresql for clusters + owned by 'postgres', /tmp for other clusters) + -l <dir> path to desired log file (default: + /var/log/postgresql/postgresql-<version>-<cluster>.log) + --locale <encoding> + set cluster locale (default: inherit from environment) + --lc-collate/ctype/messages/monetary/numeric/time <locale> + like --locale, but only set for a particular category + -e <encoding> Default encoding (default: derived from locale) + -p <port> port number (default: next free port starting from 5432) + --start start the cluster after creating it + --start-conf auto|manual|disabled + Set automatic startup behaviour in start.conf (default: 'auto') + --createclusterconf=file alternative createcluster.conf to use + --environment=file alternative environment file to use + <initdb options> other options to pass to initdb +"; + exit 1; +} + +$startconf ||= $defaultconf{'start_conf'} || 'auto'; +error "Invalid --start-conf value: $startconf" if $startconf ne 'auto' && + $startconf ne 'manual' && $startconf ne 'disabled'; + +if ($owneruid) { + $owneruid = (getpwnam $owneruid)[2] unless $owneruid =~ /^\d+$/; +} elsif ($> == 0) { + $owneruid = getpwnam 'postgres'; + error 'User postgres does not exist' unless $owneruid; +} else +{ + $owneruid = $>; +} + +if ($ownergid) { + $ownergid = (getgrnam $ownergid)[2] unless $ownergid =~ /^\d+$/; +} else { + $ownergid = (getpwuid $owneruid)[3]; +} + +error 'clusters must not be owned by root' unless $owneruid; + +my ($version) = $ARGV[0] =~ /^(\d+\.?\d+)$/; +error "invalid version '$ARGV[0]'" unless defined $version; +my ($cluster) = $ARGV[1] =~ /^([-.\w]+)$/; +error "invalid cluster name '$ARGV[1]'" unless defined $cluster; +if ($cluster =~ /-/ and -t 1) { + print "Warning: cluster names containing dashes (-) will cause problems when running from systemd. Continuing anyway\n"; +} +splice @ARGV, 0, 2; + +my @initdb_opts_from_cli; +# options passed through to initdb +push @initdb_opts_from_cli, ('--encoding', $encoding) if $encoding; +push @initdb_opts_from_cli, ('--locale', $locale) if $locale; +push @initdb_opts_from_cli, ('--lc-collate', $lc_collate) if $lc_collate; +push @initdb_opts_from_cli, ('--lc-ctype', $lc_ctype) if $lc_ctype; +push @initdb_opts_from_cli, ('--lc-messages', $lc_messages) if $lc_messages; +push @initdb_opts_from_cli, ('--lc-monetary', $lc_monetary) if $lc_monetary; +push @initdb_opts_from_cli, ('--lc-numeric', $lc_numeric) if $lc_numeric; +push @initdb_opts_from_cli, ('--lc-time', $lc_time) if $lc_time; + +# initdb options passed after -- +foreach my $argv (@ARGV) { + # the user passed an authentication method, don't mess with initdb and pg_hba.conf + if ($argv =~ /^(-\w*A|--auth\b)/) { # -A --auth --auth-host --auth-local + $explicit_auth_config = 1; + } + push @initdb_opts_from_cli, $argv =~ /(.*)/; # untaint +} + +# pg_hba.conf authentication settings +my $local_method = $version >= 9.1 ? 'peer' : + ($version >= 8.4 ? 'ident' : + 'ident sameuser'); +my $host_method = $version >= 14 ? 'scram-sha-256' : 'md5'; + +# create parent of data directory if missing +my $datadirp_created; +if (!defined $datadir) { + $datadir = replace_v_c ($defaultconf{'data_directory'} || "/var/lib/postgresql/%v/%c", $version, $cluster); + $datadir =~ s!/+$!!; + my $pd = $datadir; + $pd =~ s!/[^/]*$!!; + + # ensure that the version data dir is owned by postgres as well, so that + # it can be administrated without root permissions + if (!stat $pd) { + my @install = qw(install -d); + push @install, '-o', $postgres_user[2], '-g', $postgres_user[3] if ($> == 0); + system @install, $pd; + $datadirp_created = $pd; # clean up in case of error + } +} +my $confdirp = "$PgCommon::confroot/$version"; +my $confdir = "$confdirp/$cluster"; + +# some sanity checks +error "no initdb program for version $version found" unless get_program_path 'initdb', $version; +error 'cluster configuration already exists' + if -e "$confdir/postgresql.conf" || -e "$confdir/pg_hba.conf"; + +if (defined $port) { + error 'port must be a positive integer between 1024 and 65535' + unless $port =~ /^\d+/ && $port >= 1024 && $port <= 65535; +} else { + $port = next_free_port; +} + +# create configuration directory +if (!stat $confdirp) { + my @install = qw(install -d); + push @install, '-o', $postgres_user[2], '-g', $postgres_user[3] if ($> == 0); + system @install, $confdirp; +} +# now we created the first new directory for this cluster and start to rollback +# on error +$cleanup_cruft = 1; + +error 'could not create configuration directory; you might ' . + 'need to run this program with root privileges' if system ('install', '-d', $confdir); + +# check whether we have an already existing cluster; check version and +# determine owner in this case +my $newcluster = 0; + +if (-f "$datadir/PG_VERSION") { + open my $fh, '<', "$datadir/PG_VERSION" or error "could not open $datadir/PG_VERSION"; + chomp(my $existingver = <$fh>); + close $fh; + + ($owneruid, $ownergid) = (stat "$datadir/PG_VERSION")[4,5]; + if ($existingver == $version) { + print "Configuring already existing cluster (configuration: $confdir, data: $datadir, owner: $owneruid:$ownergid)\n"; + } else { + error "$datadir contains a version $existingver cluster, but $version was requested"; + } +} else { + print "Creating new PostgreSQL cluster $version/$cluster ...\n"; + init_db $version, $cluster, $datadir, $owneruid, $ownergid, + $local_method, $host_method, \@initdb_opts_from_cli; + $newcluster = 1; +} + +# create default "start" file +set_cluster_start_conf $version, $cluster, $startconf; + +# create default pg_ctl.conf file +set_cluster_pg_ctl_conf $version, $cluster, ''; + +# move conffiles, setup permissions +move_conffile "$datadir/postgresql.conf", $confdir, $version, $cluster, $owneruid, $ownergid, '644'; +move_conffile "$datadir/pg_hba.conf", $confdir, $version, $cluster, $owneruid, $ownergid, '640', 'hba_file'; +move_conffile "$datadir/pg_ident.conf", $confdir, $version, $cluster, $owneruid, $ownergid, '640', 'ident_file'; +foreach my $f ($datadir, $confdir, "$confdir/start.conf", "$confdir/pg_ctl.conf") { + lchown $owneruid, $ownergid, $f or error "lchown $f: $!"; +} + +PgCommon::set_conf_value $version, $cluster, 'postgresql.conf', 'data_directory', $datadir; + +# add access for database superuser +setup_pg_hba $version, $owneruid, $confdir, $local_method, $host_method + if $newcluster and not $explicit_auth_config; + +# configure socket directory +if (! $socketdir && ! -e '/var/run/postgresql') { + system 'install', '-d', '-o', $postgres_user[2], '-g', $postgres_user[3], '/var/run/postgresql'; +} +if ($socketdir && ! -e $socketdir) { + if (system 'install', '-d', '-m', '0755', '-o', $owneruid, '-g', $ownergid, $socketdir) { + error "could not create socket directory $socketdir"; + } +} +my $orig_euid = $>; +my $orig_egid = $); +$) = $ownergid; +$> = $owneruid; +unless ($socketdir) { + if ($version < 9.4 and $PgCommon::rpm) { + $socketdir = '/tmp'; # PGDG 9.3 and earlier defaults to /tmp in libpq et al. + } elsif (-w '/var/run/postgresql') { + $socketdir = '/var/run/postgresql'; + } else { + $socketdir='/tmp'; + } +} +set_cluster_socketdir $version, $cluster, $socketdir; +$> = $orig_euid; +$) = $orig_egid; + +set_cluster_port $version, $cluster, $port; + +# create log file +if (! -d '/var/log/postgresql') { + mkdir '/var/log/postgresql' or + error "could not create log directory; you might need to run this program with root privileges"; + chmod 01775, '/var/log/postgresql'; + lchown 0, $postgres_user[3], '/var/log/postgresql'; +} +my $real_logfile = $custom_logfile || "/var/log/postgresql/postgresql-$version-$cluster.log"; +error "logfile $real_logfile is a directory, not a file" if (-d $real_logfile); +if (! -e $real_logfile) { + open my $fh, '>>', $real_logfile or error "could not create log file $real_logfile"; +} +chmod 0640, $real_logfile; +my $g; +if ($owneruid < 500) { + $g = (getgrnam 'adm')[2]; +} else { + $g = $ownergid; +} +lchown $owneruid, $g, $real_logfile; +# if we are using a non-default log file, create a log symlink +if ($custom_logfile) { + symlink $real_logfile, "$confdir/log"; + lchown $owneruid, $ownergid, "$confdir/log"; +} + +# SSL configuration +my $want_ssl = PgCommon::config_bool($defaultconf{ssl} || 'on'); +# older versions (<= 9.1 as of 2019-03) do not support ssl anymore +my $postgres = get_program_path('postgres', $version); +my $ldd = `ldd $postgres 2>/dev/null`; +$want_ssl = 0 if ($ldd and $ldd !~ /libssl/); + +# Check whether we can access the SSL private key as the cluster owner +my $ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key'; +my $ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem'; +my $ssl_ca_file = "$PgCommon::common_confdir/root.crt"; +my $ssl_key_access; +my ($uid, $euid, $gid, $egid) = ($<, $>, $(, $)); +change_ugid $owneruid, $ownergid; +$ssl_key_access = -r $ssl_key_file; + +# check for stats_temp_directory access +delete $defaultconf{stats_temp_directory} if ($version < 8.4 or $version >= 15); +if ($defaultconf{stats_temp_directory}) { + my $stats_temp_directory = replace_v_c ($defaultconf{stats_temp_directory}, $version, $cluster); + $stats_temp_directory =~ s!/$!!; # strip trailing slash + my $stats_temp_parent = $stats_temp_directory; + $stats_temp_parent =~ s!/[^/]+$!!; + if (-d $stats_temp_directory) { + if (! -w $stats_temp_directory) { + print "Warning: The selected stats_temp_directory $stats_temp_directory +is not writable for the cluster owner. Not adding this setting in +postgresql.conf.\n"; + delete $defaultconf{stats_temp_directory}; + } + } elsif (! -d $stats_temp_parent) { + print "Warning: The parent $stats_temp_parent of the selected +stats_temp_directory does not exist. Not adding this setting in +postgresql.conf.\n"; + delete $defaultconf{stats_temp_directory}; + } elsif (! -w $stats_temp_parent) { + print "Warning: The parent $stats_temp_parent of the selected +stats_temp_directory is not writable for the cluster owner. Not adding this +setting in postgresql.conf.\n"; + delete $defaultconf{stats_temp_directory}; + } + # create the stats directory now. pg_ctlcluster would create it anyway, but + # when using pg_upgradecluster -m upgrade, it is not run before the cluster + # is started for the first time + if ($defaultconf{stats_temp_directory}) { + system 'install', '-d', '-m', '750', '-o', $owneruid, '-g', $ownergid, $stats_temp_directory; + } +} + +$> = $euid; +$< = $uid; +$( = $gid; +$) = $egid; +die "changing euid back: $!" if $> != $euid; +die "changing egid back: $!" if $) != $egid; + +# enable SSL if we have the snakeoil default certificate +if ($want_ssl && $newcluster && -e $ssl_cert_file && $ssl_key_access) { + if ($version >= '9.2') { + PgCommon::set_conf_value $version, $cluster, 'postgresql.conf', + 'ssl_cert_file', $ssl_cert_file; + PgCommon::set_conf_value $version, $cluster, 'postgresql.conf', + 'ssl_key_file', $ssl_key_file; + } else { + symlink $ssl_cert_file, "$datadir/server.crt"; + symlink $ssl_key_file, "$datadir/server.key"; + } + + PgCommon::set_conf_value $version, $cluster, 'postgresql.conf', 'ssl', 'on'; +} + +# SSL client certificate CA +if ($want_ssl && $newcluster && -e $ssl_ca_file) { + # check if we have a cert in there or just the boilerplate installed by our postinst + open my $fh, '<', $ssl_ca_file or error "could not open $ssl_ca_file for reading"; + my $val; + read $fh, $val, 4096; + if ($val =~ /^-----BEGIN CERTIFICATE-----/m) { + if ($version >= '9.2') { + PgCommon::set_conf_value $version, $cluster, 'postgresql.conf', + 'ssl_ca_file', $ssl_ca_file; + } else { + symlink $ssl_ca_file, $datadir.'/root.crt'; + } + } +} + +# SSL client certificate revocation list +if ($want_ssl && $newcluster && -e "$PgCommon::common_confdir/root.crl") { + if ($version >= '9.2') { + PgCommon::set_conf_value $version, $cluster, 'postgresql.conf', + 'ssl_crl_file', "$PgCommon::common_confdir/root.crl"; + } else { + symlink "$PgCommon::common_confdir/root.crl", $datadir.'/root.crl'; + } +} + +# create default (empty) environment file +my $defaultenv = "# environment variables for postgres processes +# This file has the same syntax as postgresql.conf: +# VARIABLE = simple_value +# VARIABLE2 = 'any value!' +# I. e. you need to enclose any value which does not only consist of letters, +# numbers, and '-', '_', '.' in single quotes. Shell commands are not +# evaluated. +"; +if (-e $environmentfile) { + open my $env, '<', $environmentfile or error "could not read environment file $environmentfile"; + local $/; # slurp mode + $defaultenv = <$env>; +} +$defaultenv = replace_v_c ($defaultenv, $version, $cluster); +open my $env, '>', "$confdir/environment" or error "could not create environment file $confdir/environment"; +print $env $defaultenv; +close $env; +chmod 0644, "$confdir/environment"; +lchown $owneruid, $ownergid, "$confdir/environment"; + +$cleanup_cruft = 0; + +# configure to create external PID file +if ($socketdir eq '/var/run/postgresql') { + PgCommon::set_conf_value $version, $cluster, 'postgresql.conf', 'external_pid_file', "/var/run/postgresql/$version-$cluster.pid"; +} + +# handle other createcluster.conf parameters, including --pgoption parameters +foreach my $guc (sort keys %defaultconf) { + next if $guc =~ /^(create_main_cluster|start_conf|data_directory|waldir|xlogdir|initdb_options|ssl)$/; + next if $guc eq 'logging_collector' and $version < 8.3; + next if $guc eq 'cluster_name' and $version < 9.5; + my $val = replace_v_c ($defaultconf{$guc}, $version, $cluster); + $guc =~ s/^add_include/include/; # remove harness from include directives in createcluster.conf + if ($guc eq 'include_dir') { + next if ($version < 9.3); + if ($val =~ /^[\w.]+$/ and not -e "$confdir/$val") { # create directory relative to new config directory + mkdir "$confdir/$val", 0755; + lchown $owneruid, $ownergid, "$confdir/$val"; + } + } + PgCommon::set_conf_value $version, $cluster, 'postgresql.conf', $guc, $val; +} + +# notify systemd about the new cluster +if (not exists $ENV{'PG_CLUSTER_CONF_ROOT'} and $startconf eq 'auto' and -d '/run/systemd/system') { + if ($> == 0) { + system 'systemctl daemon-reload'; + } elsif (-t 1) { + print "Warning: systemd does not know about the new cluster yet. Operations like \"service postgresql start\" will not handle it. To fix, run:\n"; + print " sudo systemctl daemon-reload\n"; + } +} + +# notify apt about the new cluster +if (not exists $ENV{'PG_CLUSTER_CONF_ROOT'} and $> == 0) { + system "/usr/share/postgresql-common/pg_updateaptconfig"; +} + +# start it if requested +if ($start) { + system 'pg_ctlcluster', $version, $cluster, 'start'; + die "Could not start cluster\n" if ($?); +} + +# finally, show the cluster we created +system 'pg_lsclusters', $version, $cluster unless ($quiet or $no_status); + +END { + # clean up cruft if something went wrong + if ($cleanup_cruft && defined $version && defined $cluster) { + system "pg_dropcluster $version $cluster 2>/dev/null"; + rmdir $datadirp_created if ($datadirp_created); # clean up after early errors which pg_dropcluster doesn't handle + exit 1; + } +} + +__END__ + +=head1 NAME + +pg_createcluster - create a new PostgreSQL cluster + +=head1 SYNOPSIS + +B<pg_createcluster> [I<options>] I<version> I<name> [B<--> I<initdb options>] + +=head1 DESCRIPTION + +B<pg_createcluster> creates a new PostgreSQL server cluster (i. e. a +collection of databases served by a L<postgres(1)> instance) and +integrates it into the multi-version/multi-cluster architecture of the +B<postgresql-common> package. + +Every cluster is uniquely identified by its version and name. The name can be +arbitrary. The default cluster that is created on installation of a server +package is C<main>. However, you might wish to create other clusters for +testing, with other superusers, a cluster for each user on a shared server, +etc. C<pg_createcluster> will abort with an error if you try to create a +cluster with a name that already exists for that version. + +For compatibility with B<systemd> service units, the cluster name should not +contain any dashes (B<->). B<pg_ctlcluster> will warn about the problem, but +succeed with the operation. + +Given a major PostgreSQL I<version> (like "8.2" or "8.3") and a cluster +I<name>, it creates the necessary configuration files in +C</etc/postgresql/>I<version>C</>I<name>C</>; in particular these are +C<postgresql.conf>, C<pg_ident.conf>, C<pg_hba.conf>, a postgresql-common +specific configuration file C<start.conf> (see B<STARTUP CONTROL> below), +C<pg_ctl.conf>, and a symbolic link C<log> which points to the log file (by +default, C</var/log/postgresql/postgresql->I<version>C<->I<name>C<.log>). + +C<postgresql.conf> is automatically adapted to use the next available port, i. +e. the first port (starting from 5432) which is not yet used by an already +existing cluster. + +If the data directory does not yet exist, PostgreSQL's L<initdb(1)> command is +used to generate a new cluster structure. If the data directory already exists, +it is integrated into the B<postgresql-common> structure by moving the +configuration file and setting the data_directory option. Please note that this +I<only> works for data directories which were created directly with B<initdb>, i. +e. all the configuration files (C<postgresql.conf> etc.) must be present in the +data directory. + +If a custom socket directory is given and it does not exist, it is created. + +If the log file does not exist, it is created. In any case the permissions are +adjusted to allow write access to the cluster owner. Please note that +C<postgresql.conf> can be customized to specify C<log_directory> and/or +C<log_filename>; if at least one of these options is present, then the symbolic +link C<log> in the cluster configuration directory is ignored. + +If the default snakeoil SSL certificate exists +(C</etc/ssl/certs/ssl-cert-snakeoil.pem> and +C</etc/ssl/private/ssl-cert-snakeoil.key>), and the C<postgres> user is in the +C<ssl-cert> Unix group, B<pg_createcluster> configures the cluster to use this +certificate, and enables SSL. Therefore all clusters will use the same SSL +certificate by default. For versions up to 9.1, symlinks in the data directory +will be created (C<server.crt> and C<server.key>); for 9.2 and later, the +appropriate C<postgresql.conf> options will be set (C<ssl_cert_file> and +C<ssl_key_file>). Of course you can replace this with a cluster specific +certificate. Similarly for C</etc/postgresql-common/root.crt> and +C</etc/postgresql-common/root.crl>, these files will be configured as client +certificate CA and revocation list, when present. (C<root.crt> is initially a +placeholder that will only be used if real certificates are added to the file.) + +=head1 OPTIONS + +=over 4 + +=item B<-u> I<user>, B<--user=>I<user> + +Set the user who owns the cluster and becomes the database superuser to the +given name or uid. By default, this is the user B<postgres>. A cluster must +not be owned by root. + +=item B<-g> I<group>, B<--group=>I<group> + +Change the group of the cluster related data files. By default this will be the +primary group of the database owner. + +=item B<-d> I<dir>, B<--datadir=>I<dir> + +Explicitly set the data directory path, which is used to store all the actual +databases and tables. This will become quite big (easily in the order of five +times the amount of actual data stored in the cluster). Defaults to +C</var/lib/postgresql/>I<version>C</>I<cluster>. + +=item B<-s> I<dir>, B<--socketdir=>I<dir> + +Explicitly set the directory where the L<postgres(1)> server stores the Unix +socket for local connections. Defaults to C</var/run/postgresql/> for clusters +owned by the user B<postgres>, and C</tmp> for clusters owned by other users. +Please be aware that C</tmp> is an unsafe directory since everybody can create +a socket there and impersonate the database server. If the given directory does +not exist, it is created with appropriate permissions. + +=item B<-l> I<path>, B<--logfile=>I<path> + +Explicitly set the path for the L<postgres(1)> server log file. Defaults to +C</var/log/postgresql/postgresql->I<version>C<->I<cluster>C<.log>. + +=item B<--locale=>I<locale> + +Set the default locale for the database cluster. If this option is not +specified, the locale is inherited from the environment that +B<pg_createcluster> runs in. + +=item B<--lc-collate=>I<locale> + +=item B<--lc-ctype=>I<locale> + +=item B<--lc-messages=>I<locale> + +=item B<--lc-monetary=>I<locale> + +=item B<--lc-numeric=>I<locale> + +=item B<--lc-time=>I<locale> + +Like B<--locale>, but only sets the locale in the specified category. + +=item B<-e> I<encoding>, B<--encoding=>I<encoding> + +Select the encoding of the template database. This will also be the default +encoding of any database you create later, unless you override it there. The +default is derived from the locale, or SQL_ASCII if that does not work. The +character sets supported by the PostgreSQL server are described in the +documentation. + +B<Note>: It is not recommended to set this option directly! Set the locale +instead. + +=item B<-p> I<port>, B<--port=>I<port> + +Select the port the new cluster listens on (for the Unix socket and the TCP +port); this must be a number between 1024 and 65535, since PostgreSQL does not +run as root and thus needs an unprivileged port number. By default the next +free port starting from 5432 is assigned. + +=item B<-q> B<--quiet> B<--no-status> + +Suppress output from B<initdb> and (or only) the cluster status message at the +end of the output. + +=item B<--start> + +Immediately start a server for the cluster after creating it (i. e. call +C<pg_ctlcluster> I<version cluster> C<start> on it). By default, the cluster is +not started. + +=item B<--start-conf=>B<auto>|B<manual>|B<disabled> + +Set the initial value in the C<start.conf> configuration file. See B<STARTUP +CONTROL> below. By default, B<auto> is used, which means that the cluster is +handled by C</etc/init.d/postgresql>, i. e. starts and stops +automatically on system boot. + +=item B<-o> I<guc>B<=>I<value>, B<--pgoption> I<guc>B<=>I<value> + +Configuration option to set in the new C<postgresql.conf> file. + +=item B<--createclusterconf=>I<file> + +Alternative B<createcluster.conf> file to use. Default is +C</etc/postgresql-common/createcluster.conf> (or +C<$PGSYSCONFDIR/createcluster.conf>). + +=item B<--environment=>I<file> + +Alternative default B<environment> file to use. Default is +C</etc/postgresql-common/environment> (or C<$PGSYSCONFDIR/environment>). +If the file is missing, a placeholder string is used. +%v and %c are replaced; see DEFAULT VALUES below. + +=item B<--> I<initdb options> + +Options passed directly to L<initdb(1)>. + +Per default, B<pg_createcluster> will update the C<pg_hba.conf> file generated +by initdb to use peer authentication on local (unix) connections, and md5 on +TCP (host) connections. If explicit authentication config is included here +(B<-A>, B<--auth>, B<--auth-host>, B<--auth-local>), the C<pg_hba.conf> file +will be left untouched. + +I<Note:> If only one of B<--auth-host> and B<--auth-local> is provided, the +other setting will default to B<trust> as per B<initdb>'s defaults, opening a +potential security risk. + +=back + +=head1 STARTUP CONTROL + +The C<start.conf> file in the cluster configuration directory controls the +start/stop behavior of that cluster's postgres process. The file can contain +comment lines (started with '#'), empty lines, and must have exactly one +line with one of the following keywords: + +=over 4 + +=item B<auto> + +The postgres process is started/stopped automatically in the init script. + +When running from B<systemd>, the cluster is started/stopped when +B<postgresql.service> is started/stopped. +This is also the default if the file is missing. + +=item B<manual> + +The postgres process is not handled by the init script, but manually +controlling the cluster with L<pg_ctlcluster(1)> is permitted. + +When running from B<systemd>, the cluster is not started automatically when +B<postgresql.service> is started. However, stopping/restarting +B<postgresql.service> will stop/restart the cluster. The cluster can be started +using B<systemctl start postgresql@>I<version>B<->I<cluster>. + +=item B<disabled> + +Neither the init script, L<pg_ctlcluster(1)>, nor B<postgresql@.service> are permitted to start/stop the +cluster. Please be aware that this will not stop the cluster owner from calling +lower level tools to control the postgres process; this option is only meant +to prevent accidents during maintenance, not more. + +=back + +When running from B<systemd>, invoke B<systemctl daemon-reload> after editing +C<start.conf>. + +The C<pg_ctl.conf> file in the cluster configuration directory can contain +additional options passed to B<pg_ctl> of that cluster. + +=head1 DEFAULT VALUES + +Some default values used by B<pg_createcluster> can be modified in +C</etc/postgresql-common/createcluster.conf>. Occurrences of B<%v> are replaced +by the major version number, and B<%c> by the cluster name. Use B<%%> for a +literal B<%>. + +=over 4 + +=item B<create_main_cluster> (Default: B<true>) + +Create a B<main> cluster when a new postgresql-NN server package is installed. + +=item B<start_conf> (Default: B<auto>) + +Default C<start.conf> value to use. + +=back + +=over 4 + +=item B<data_directory> (Default: B</var/lib/postgresql/%v/%c>) + +Default data directory. + +=item B<waldir|xlogdir> (Default: unset) + +Default directory for transaction logs. When used, B<initdb> will create a +symlink from C<pg_wal> (PostgreSQL 9.6 and earlier: C<pg_xlog>) in the data +directory to this location. Unset by default, i.e. transaction logs remain in +the data directory. Both spellings of this option are accepted, and translated +to the correct initdb invocation depending on the cluster version. + +=item B<initdb_options> (Default: unset) + +Other options to pass to B<initdb>. + +=item Other options + +All other options listed are copied into the new cluster's postgresql.conf, e.g.: + + listen_addresses = '*' + log_line_prefix = '%%t ' + +Some postgresql.conf options are treated specially: + +=over 4 + +=item B<ssl> + +Only added to postgresql.conf if the default snakeoil certificates exist and +are readable for the cluster owner as detailed above. + +=item B<stats_temp_directory> + +Only added to postgresql.conf if existing, and writable for the cluster owner, +or else if the parent directory is writable. Not used on PostgreSQL 15 or later. + +=back + +=item Include files + +=over 4 + +=item B<include> + +=item B<include_if_exists> + +=item B<include_dir> + +B<createcluster.conf> supports the same include directives as +B<postgresql.conf>. + +=item B<add_include> + +=item B<add_include_if_exists> + +=item B<add_include_dir> + +To add include directives to the new postgresql.conf file, use the B<add_*> +directives. The B<add_> prefix is removed. + +=back + +=back + +=head1 SEE ALSO + +L<initdb(1)>, L<pg_ctlcluster(8)>, L<pg_lsclusters(1)>, L<pg_wrapper(1)> + +=head1 AUTHORS + +Martin Pitt L<E<lt>mpitt@debian.orgE<gt>>, Christoph Berg L<E<lt>myon@debian.orgE<gt>> diff --git a/pg_ctlcluster b/pg_ctlcluster new file mode 100755 index 0000000..1684700 --- /dev/null +++ b/pg_ctlcluster @@ -0,0 +1,649 @@ +#!/usr/bin/perl -wT + +# multiversion/cluster aware pg_ctl wrapper; this also supplies the correct +# configuration parameters to 'start', and makes sure that postgres really +# stops on 'stop'. +# +# (C) 2005-2009 Martin Pitt <mpitt@debian.org> +# (C) 2009 Cyril Bouthors <cyril@bouthors.org> +# (C) 2013-2021 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use warnings; +use Getopt::Long; +use POSIX qw/setsid dup2 :sys_wait_h/; +use PgCommon; +use Fcntl qw(SEEK_SET O_RDWR O_CREAT O_EXCL); +use POSIX qw(lchown); + +my ($version, $cluster, $pg_ctl, $force); +my (@postgres_auxoptions, @pg_ctl_opts_from_cli); +my (%postgresql_conf, %info); +my $mode = 'fast'; # default shutdown mode +my $foreground = 0; # don't daemonize, use postgres instead of pg_ctl +my $stdlog = 0; # when run in foreground, still log to the default logfile + +# untaint environment +$ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin'; +delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; +chdir '/'; + +# If a pid file is already present, delete it if it is stale/invalid, or exit +# with a notice if it belongs to an already running postgres. +sub start_check_pid_file { + my $pidfile = $info{'pgdata'}.'/postmaster.pid'; + if (PgCommon::check_pidfile_running $pidfile) { + print "Cluster is already running.\n"; + exit 2; + } + + # Remove invalid or stale PID file + if (-e $pidfile) { + unlink $pidfile; + print "Removed stale pid file.\n"; + } +} + +# Check if a pid file is not present or it is invalid. If so, clean up/abort. +sub stop_check_pid_file { + my $pidfile = $info{'pgdata'}.'/postmaster.pid'; + my $pid = read_pidfile $pidfile; + return if (defined $pid && PgCommon::check_pidfile_running $pidfile); + if ($info{'running'}) { + error 'pid file is invalid, please manually kill the stale server process.'; + } + + # Remove invalid or stale PID file + if (-e $pidfile) { + unlink $pidfile; + print "Removed stale pid file.\n"; + } + print "Cluster is not running.\n"; + exit 2; +} + +# check if a cluster reliably connects or fails +# Arguments: <version> <cluster> <port> <socket dir> +sub cluster_port_ready { + my ($v, $c, $p, $sd) = @_; + my $psql = get_program_path('psql', $v); + error 'cluster_port_ready: could not find psql binary' unless $psql; + my $n = 0; + my $result = 0; + + # probe until we get three successful or failed connections in a row + my $nopwoption = $v >= 8.4 ? '-w' : ''; + if ($v < 8.4) { + $ENV{'PGPASSWORD'} = 'foo'; # prevent hangs if superuser cannot connect without password + } + my $out; + while ($n < ($result ? 10 : 3)) { + select undef, undef, undef, 0.5; + $out = `LC_MESSAGES=C $psql -h '$sd' --port $p $nopwoption -Xc '' template1 2>&1 >/dev/null`; + + if ($? == $result) { + $n++; + } else { + $n = 0; + } + $result = $?; + } + + if ($out =~ /FATAL:|no password supplied/) { + print STDERR "Warning: connection to the database failed, disabling startup checks:\n$out\n"; + return cluster_port_running $v, $c, $p; + } + return !$result; +} + +sub start { + start_check_pid_file; + + # check conflicting port + if (cluster_port_running $version, $cluster, $info{'port'}) { + my $sockdir = get_cluster_socketdir $version, $cluster; + error("Port conflict: another instance is already running on $sockdir with port $info{'port'}"); + } + + # prepare environment (empty except for content of 'environment', and LANG) + %ENV = read_cluster_conf_file $version, $cluster, 'environment'; + # set LANG so non-ascii chars in the server log are not replaced by '?' (affected are non-session contexts) + unless (exists $ENV{LANG}) { + my $lc_messages = PgCommon::get_conf_value $version, $cluster, 'postgresql.conf', 'lc_messages'; + $ENV{LANG} = $lc_messages if $lc_messages; + } + # 9.5 configures OOM killer using env vars + if ($version >= 9.5) { + $ENV{PG_OOM_ADJUST_FILE} = "/proc/self/oom_score_adj" unless (exists $ENV{PG_OOM_ADJUST_FILE}); + # PG_OOM_ADJUST_VALUE defaults to 0, but can be overridden here + } + + my $postgres_opts = ''; + my $usd = $version >= 9.3 ? 'unix_socket_directories' : 'unix_socket_directory'; + if (!(PgCommon::get_conf_value $version, $cluster, 'postgresql.conf', $usd)) { + $postgres_opts .= "-c $usd=\"$info{'socketdir'}\""; + } + + my $cdir = $info{'configdir'}; + $postgres_opts .= " -c config_file=\"$cdir/postgresql.conf\""; + if (!(PgCommon::get_conf_value $version, $cluster, 'postgresql.conf', 'hba_file')) { + $postgres_opts .= " -c hba_file=\"$cdir/pg_hba.conf\""; + } + if (!(PgCommon::get_conf_value $version, $cluster, 'postgresql.conf', 'ident_file')) { + $postgres_opts .= " -c ident_file=\"$cdir/pg_ident.conf\""; + } + + if ((-d '/var/run/postgresql') && !defined (PgCommon::get_conf_value $version, $cluster, 'postgresql.conf', 'external_pid_file')) { + # check whether /var/run/postgresql/ is writable as the cluster owner + my $vrp_writable; + if ($> == 0) { + change_ugid $info{'owneruid'}, $info{'ownergid'}; + $vrp_writable = -w '/var/run/postgresql'; + $< = $> = 0; + $( = $) = 0; + } else { + $vrp_writable = -w '/var/run/postgresql'; + } + if ($vrp_writable) { + $postgres_opts .= " -c external_pid_file=\"/var/run/postgresql/$version-$cluster.pid\""; + } + } + + $postgres_opts .= ' ' . (join ' ', @postgres_auxoptions); + ($postgres_opts) = $postgres_opts =~ /(.*)/; # untaint + + if ($foreground) { + if ($stdlog and $info{'logfile'}) { + my $fd = POSIX::open($info{logfile}, POSIX::O_WRONLY|POSIX::O_APPEND|POSIX::O_CREAT) or error "Could not open logfile $info{logfile}"; + dup2($fd, 1); + dup2($fd, 2); + POSIX::close($fd) or error "Could not close log fd"; + } + my $postgres = get_program_path 'postgres', $version; + error "Could not find postgres executable for version $version" unless ($postgres); + exec '/bin/sh', '-c', "exec $postgres $postgres_opts" or + error "Executing $postgres failed: $!" + } + + # only supply or default logfile path when none is given explicitly in + # postgresql.conf + my @options = ($pg_ctl, 'start', '-D', $info{'pgdata'}); + my $logsize = 0; + if ($info{'logfile'}) { + push @options, ('-l', $info{'logfile'}); + # remember current size of the log + $logsize = (stat $info{'logfile'})[7] || 0; # ignore stat errors + } + + push @options, @pg_ctl_opts_from_cli if @pg_ctl_opts_from_cli; + + my %pg_ctl_opts_from_file = read_cluster_conf_file $version, $cluster, 'pg_ctl.conf'; + push @options, split(' ', $pg_ctl_opts_from_file{'pg_ctl_options'}) + if defined $pg_ctl_opts_from_file{'pg_ctl_options'} and $pg_ctl_opts_from_file{'pg_ctl_options'} ne ''; + + push @options, ('-s', '-o', $postgres_opts); + + if (fork) { + wait; + if ($?) { + my $exit = $? >> 8; + print STDERR "Error: $pg_ctl @options exited with status $exit: $!\n"; + my $currlogsize = (stat $info{'logfile'})[7] if $info{'logfile'} && -r $info{'logfile'}; + if ($currlogsize) { + open LOG, $info{'logfile'} or + error "Could not open log file " . $info{'logfile'}; + seek LOG, $logsize, SEEK_SET; + print STDERR $_ while <LOG>; + } + exit $exit; + } + } else { + setsid or error "could not start session: $!"; + if ($info{'logfile'}) { + my $fd = POSIX::open($info{'logfile'}, POSIX::O_WRONLY|POSIX::O_APPEND|POSIX::O_CREAT) or error "Could not open logfile $info{'logfile'}"; + dup2($fd, 1); + dup2($fd, 2); + POSIX::close($fd) or error "Could not close log fd"; + } + exec $pg_ctl @options or error "could not exec $pg_ctl @options: $!"; + } + + # wait a bit until the socket exists + my $success = 0; + my $currlogsize = 0; + my $pidfile = $info{'pgdata'}.'/postmaster.pid'; + for (my $attempt = 0; $attempt < 60; $attempt++) { + select (undef, undef, undef, 0.5); + $currlogsize = (stat $info{'logfile'})[7] if $info{'logfile'} && -r $info{'logfile'}; + if (cluster_port_running $version, $cluster, $info{'port'}) { + $success = 1; + last; + } + + # if postgres wrote something, but the process does not exist any + # more, there must be a problem and we can stop immediately + last if ($currlogsize > $logsize && !PgCommon::check_pidfile_running $pidfile); + } + + # OK, the server runs, now wait until it stabilized + if ($success) { + $success = cluster_port_ready $version, $cluster, $info{'port'}, $info{'socketdir'}; + } + + if (!$success) { + if ($currlogsize) { + print STDERR "The PostgreSQL server failed to start. Please check the log output:\n"; + open LOG, $info{'logfile'} or + error "Could not open log file " . $info{'logfile'}; + seek LOG, $logsize, SEEK_SET; + print STDERR $_ while <LOG>; + } else { + print STDERR "The PostgreSQL server failed to start. Please check the log output.\n"; + } + exit 1; + } + + return 0; +} + +sub stop { + stop_check_pid_file; + my $result = 1; + + my @options = ('stop', '-s', '-w', '-D', $info{'pgdata'}); + + push @options, @pg_ctl_opts_from_cli if @pg_ctl_opts_from_cli; + + my %pg_ctl_opts_from_file = read_cluster_conf_file $version, $cluster, 'pg_ctl.conf'; + push @options, split(' ', $pg_ctl_opts_from_file{'pg_ctl_options'}) + if defined $pg_ctl_opts_from_file{'pg_ctl_options'} and $pg_ctl_opts_from_file{'pg_ctl_options'} ne ''; + + if (!fork()) { + close STDOUT; + exec $pg_ctl, @options, '-m', ($force ? 'fast' : $mode); + } else { + wait; + $result = $? >> 8; + } + + # try harder if forced and server hasn't stopped yet + if ($force and -f $info{'pgdata'}.'/postmaster.pid') { + print "(does not shutdown gracefully, now stopping immediately)\n"; + $result = system $pg_ctl, @options, '-m', 'immediate'; + } + + # external_pid_file files are currently not removed by postgres itself + if ($result == 0) { + unlink "/var/run/postgresql/$version-$cluster.pid"; + } + + return $result; +} + +sub restart { + my $result; + + if ($info{'running'}) { + $result = stop; + return $result if $result; + } + return start; +} + +sub reload { + exec $pg_ctl, '-D', $info{'pgdata'}, '-s', @pg_ctl_opts_from_cli, 'reload'; +} + +sub status { + exec $pg_ctl, '-D', $info{'pgdata'}, 'status'; +} + +sub promote { + exec $pg_ctl, '-D', $info{'pgdata'}, '-s', @pg_ctl_opts_from_cli, 'promote'; +} + +# +# main +# + +my ($skip_systemctl_redirect, $bindir); + +exit 1 unless GetOptions ('o|options=s' => \@postgres_auxoptions, + 'f|force' => \$force, + 'm|mode=s' => \$mode, + 'foreground' => \$foreground, + 'skip-systemctl-redirect' => \$skip_systemctl_redirect, + 'stdlog' => \$stdlog, + 'bindir=s' => \$bindir, +); + +if ($mode =~ /^(s(mart)?|f(ast)?|i(mmediate)?)$/) { + $mode = $1; # untaint +} else { + print "Invalid -m mode, valid are: smart fast immediate\n"; + exit 1; +} + +($bindir) = $bindir =~ /^(\/.*)$/ if $bindir; # untaint + +# parse command +my $version_re = qr/(\d+\.?\d)/; +my $cluster_re = qr/([^'"\s]+)/; +my $action_re = qr/(start|stop|restart|reload|status|promote)/; +my $action; + +if (@ARGV >= 3 and "$ARGV[0] $ARGV[1] $ARGV[2]" =~ /^$version_re $cluster_re $action_re$/) { + ($version, $cluster, $action) = ($1, $2, $3); + splice @ARGV, 0, 3; +} elsif (@ARGV >= 3 and "$ARGV[0] $ARGV[1] $ARGV[2]" =~ /^$action_re $version_re $cluster_re$/) { + ($action, $version, $cluster) = ($1, $2, $3); + splice @ARGV, 0, 3; +} elsif (@ARGV >= 2 and "$ARGV[0] $ARGV[1]" =~ m!^$version_re(?:[-/])$cluster_re $action_re$!) { + ($version, $cluster, $action) = ($1, $2, $3); + splice @ARGV, 0, 2; +} elsif (@ARGV >= 2 and "$ARGV[0] $ARGV[1]" =~ m!^$action_re $version_re(?:[-/])$cluster_re$!) { + ($action, $version, $cluster) = ($1, $2, $3); + splice @ARGV, 0, 2; +} else { + error "Usage: $0 <version> <cluster> <action> [-- <pg_ctl options>]"; +} + +@pg_ctl_opts_from_cli=(); +foreach my $argv (@ARGV) { + push @pg_ctl_opts_from_cli, $argv =~ /(.*)/; # untaint +} + +error "specified cluster '$version $cluster' does not exist" unless cluster_exists $version, $cluster; +%info = cluster_info ($version, $cluster); +validate_cluster_owner \%info; + +unless ($action eq 'stop') { + # check if cluster is disabled in start.conf + error 'Cluster is disabled' if $info{'start'} eq 'disabled'; +} + +# redirect the request through systemd +if (not $skip_systemctl_redirect and getppid() != 1 and # not run from init + -d '/run/systemd/system' and not exists $ENV{_SYSTEMCTL_SKIP_REDIRECT} and # systemd is running + not $foreground and # foreground operation not requested + not exists $ENV{'PG_CLUSTER_CONF_ROOT'} and # not handling user clusters + $action =~ /^(start|stop|restart)$/ # redirect only these actions +) { + $action = $1; # untaint + system 'systemctl', 'is-active', '-q', "postgresql\@$version-$cluster"; + my $unit_active = $? == 0; + + # if extra options are given, proceed with pg_ctlcluster with a warning + if (@postgres_auxoptions != 0 or @pg_ctl_opts_from_cli != 0 or $bindir) { # extra options given + print "Notice: extra pg_ctl/postgres options given, bypassing systemctl for $action operation\n" if (-t 1); + # if we are root, redirect to systemctl unless stopping a cluster running outside systemd + } elsif ($> == 0) { + if ($action eq 'stop' and not $unit_active) { + # proceed with pg_ctlcluster + } else { + #print "Redirecting $action request to systemctl\n" if (-t 1); + system 'systemctl', $action, "postgresql\@$version-$cluster"; + exit $? >> 8; + # program end + } + # as non-root, raise an error on restarting a cluster running from systemd + } elsif ($action eq 'restart' and $unit_active) { + error "cluster is running from systemd, can only restart it as root. Try instead:\n sudo systemctl $action postgresql\@$version-$cluster"; + # program end + # otherwise just raise a warning on start and restart as non-root + } elsif (-t 1) { + if ($action =~ /start/) { + print "Warning: the cluster will not be running as a systemd service. Consider using systemctl:\n"; + print " sudo systemctl $action postgresql\@$version-$cluster\n"; + } elsif ($unit_active) { + print "Warning: stopping the cluster using pg_ctlcluster will mark the systemd unit as failed. Consider using systemctl:\n"; + print " sudo systemctl $action postgresql\@$version-$cluster\n"; + } + } +} + +# recreate missing standard log dir +if ($> == 0 && ! -e '/var/log/postgresql' && + $info{'logfile'} =~ m!^/var/log/postgresql!) { + system 'install', '-d', '-m', '1775', '-o', 'root', '-g', 'postgres', '/var/log/postgresql'; +} + +# recreate missing log file +if ($action ne 'stop' && $info{'logfile'} && ! -e $info{'logfile'}) { + if ($> == 0) { # drop privileges; this is important if logfile + # was determined via an /etc/postgresql/.../log symlink + change_ugid $info{'owneruid'}, $info{'ownergid'}; + } + sysopen (L, $info{'logfile'}, O_RDWR|O_CREAT|O_EXCL) or + error 'Could not create log file ' . $info{'logfile'}; + close L; + chmod 0640, $info{'logfile'}; + $< = $> = 0; # will silently fail if we were not root before, that's intended + $( = $) = 0; + if ($info{'owneruid'} < 500) { + my $g = (getgrnam 'adm')[2]; + lchown $info{'owneruid'}, $g, $info{'logfile'} if (defined $g); + } +} + +if ($action ne 'stop') { + # recreate /var/run/postgresql while possibly still running as root + if (! -d '/var/run/postgresql') { + system 'install', '-d', '-m', 2775, '-o', 'postgres', '-g', 'postgres', '/var/run/postgresql'; + } + + # allow creating socket directories below /var/run/postgresql for any user + if ($info{socketdir} =~ m!^(/var)/run/postgresql/[\w_.-]+$! and ! -d $info{socketdir}) { + if (mkdir $info{socketdir}, 02775) { # don't use "install" here as it would allow stealing existing directories + chown $info{owneruid}, $info{ownergid}, $info{socketdir}; + } else { + error "Could not create $info{socketdir}: $!"; + } + } + + # allow creating stats_temp_directory below /var/run/postgresql for any user + if ($info{config}->{stats_temp_directory} and $info{config}->{stats_temp_directory}=~ m!^(/var)/run/postgresql/[\w_.-]+$! and ! -d $info{config}->{stats_temp_directory}) { + if (mkdir $info{config}->{stats_temp_directory}, 0750) { # don't use "install" here as it would allow stealing existing directories + chown $info{owneruid}, $info{ownergid}, $info{config}->{stats_temp_directory}; + } else { + error "Could not create $info{config}->{stats_temp_directory}: $!"; + } + } +} + +if ($> == 0) { + # have postgres start with increased OOM killer protection; 9.0 and + # later has builtin support for resetting the adjustment of child processes + if ($action eq 'start' and $version >= '9.0' and not $PgCommon::rpm) { + if (-w '/proc/self/oom_score_adj') { + open F, '>/proc/self/oom_score_adj'; + print F "-900\n"; + close F; + } + } + + change_ugid $info{'owneruid'}, $info{'ownergid'}; +} + +# we are running as the cluster owner now + +if( $> != $info{'owneruid'} ) { + error 'You must run this program as the cluster owner ('. + (getpwuid $info{'owneruid'})[0].') or root'; +} + +# create socket directory (if it wasn't already created in /var/run/postgresql by the code above) +if ($action ne 'stop' && ! -d $info{socketdir}) { + system 'install', '-d', '-m', 2775, $info{socketdir}; +} + +# create stats_temp_directory (if it wasn't already created in /var/run/postgresql by the code above) +if ($action ne 'stop' && $info{config}->{stats_temp_directory} && ! -d $info{config}->{stats_temp_directory}) { + system 'install', '-d', '-m', 750, $info{config}->{stats_temp_directory}; +} + +$pg_ctl = $bindir ? "$bindir/pg_ctl" : get_program_path ('pg_ctl', $version); +error "Could not find pg_ctl executable for version $version" unless ($pg_ctl); + +# do the action +no strict 'refs'; +exit &$action; + +__END__ + +=head1 NAME + +pg_ctlcluster - start/stop/restart/reload a PostgreSQL cluster + +=head1 SYNOPSIS + +B<pg_ctlcluster> [I<options>] I<cluster-version> I<cluster-name> I<action> [B<--> I<pg_ctl options>] + +where I<action> = B<start>|B<stop>|B<restart>|B<reload>|B<status>|B<promote> + +=head1 DESCRIPTION + +This program controls the B<postgres> server for a particular cluster. It +essentially wraps the L<pg_ctl(1)> command. It determines the cluster version +and data path and calls the right version of B<pg_ctl> with appropriate +configuration parameters and paths. + +You have to start this program as the user who owns the database cluster or as +root. + +To ease integration with B<systemd> operation, the alternative syntax +"B<pg_ctlcluster> I<version>B<->I<cluster> I<action>" is also supported, +as well as putting the action first (matching the ordering used by B<systemctl>). + +=head1 ACTIONS + +=over 4 + +=item B<start> + +A log file for this specific cluster is created if it does not exist yet (by +default, +C</var/log/postgresql/postgresql->I<cluster-version>C<->I<cluster-name>C<.log>), +and a PostgreSQL server process (L<postgres(1)>) is started on it. Exits with +0 on success, with 2 if the server is already running, and with 1 on other +failure conditions. + +=item B<stop> + +Stops the L<postgres(1)> server of the given cluster. By default, "fast" +shutdown mode is used. + +=item B<restart> + +Stops the server if it is running and starts it (again). + +=item B<reload> + +Causes the configuration files to be re-read without a full shutdown of the +server. + +=item B<status> + +Checks whether a server is running. If it is, the PID and the command line +options that were used to invoke it are displayed. + +=item B<promote> + +Commands a running standby server to exit recovery and begin read-write +operations. + +=back + +=head1 OPTIONS + +=over 4 + +=item B<-f>|B<--force> + +For B<stop> and B<restart>, the "fast" mode is used which rolls back all active +transactions, disconnects clients immediately and thus shuts down cleanly. If +that does not work, shutdown is attempted again in "immediate" mode, which can +leave the cluster in an inconsistent state and thus will lead to a recovery run +at the next start. If this still does not help, the B<postgres> process is +killed. Exits with 0 on success, with 2 if the server is not running, and with +1 on other failure conditions. This mode should only be used when the machine +is about to be shut down. + +=item B<-m>|B<--mode> [B<smart>|B<fast>|B<immediate>] + +Shutdown mode to use for B<stop> and B<restart> actions, default is B<fast>. +See pg_ctl(1) for documentation. + +=item B<--foreground> + +Start B<postgres> in foreground, without daemonizing via B<pg_ctl>. + +=item B<--stdlog> + +When B<--foreground> is in use, redirect stderr to the standard logfile in +C</var/log/postgresql/>. (Default when not run in foreground.) + +=item B<--skip-systemctl-redirect> + +When running as root, B<pg_ctlcluster> redirects actions to B<systemctl> so +running clusters are properly supervised by B<systemd>. This option skips the +redirect; it is used in the B<postgresql@.service> unit file. The redirect is +also skipped if additional B<postgres> or B<pg_ctl> options are provided. + +=item B<--bindir> I<directory> + +Path to B<pg_ctl>. (Default is C</usr/lib/postgresql/>I<version>C</bin>.) + +=item B<-o>|B<--options> I<option> + +Pass given I<option> as command line option to the C<postgres> process. It is +possible to specify B<-o> multiple times. See L<postgres(1)> for a +description of valid options. + +=item I<pg_ctl options> + +Pass given I<pg_ctl options> as command line options to B<pg_ctl>. See L<pg_ctl(1)> +for a description of valid options. + +=back + +=head1 FILES + +=over 4 + +=item C</etc/postgresql/>I<cluster-version>C</>I<cluster-name>C</pg_ctl.conf> + +This configuration file contains cluster specific options to be passed to +L<pg_ctl(1)>. + +=item C</etc/postgresql/>I<cluster-version>C</>I<cluster-name>C</start.conf> + +This configuration file controls the start/stop behavior of the cluster. See +section "STARTUP CONTROL" in L<pg_createcluster(8)> for details. + +=back + +=head1 BUGS + +Changing the port number on startup using B<-o -p> will not work as it breaks +the checks for running clusters. + +=head1 SEE ALSO + +L<pg_createcluster(8)>, L<pg_ctl(1)>, L<pg_wrapper(1)>, L<pg_lsclusters(1)>, +L<postgres(1)> + +=head1 AUTHOR + +Martin Pitt L<E<lt>mpitt@debian.orgE<gt>> + diff --git a/pg_dropcluster b/pg_dropcluster new file mode 100755 index 0000000..def4554 --- /dev/null +++ b/pg_dropcluster @@ -0,0 +1,226 @@ +#!/usr/bin/perl -wT + +# Completely delete a PostgreSQL cluster. Fails if there is still a server +# process attached. +# +# (C) 2005-2009 Martin Pitt <mpitt@debian.org> +# (C) 2015-2021 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use warnings; +use Getopt::Long; +use PgCommon; + +# untaint environment +$ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin'; +delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; + +my $stopserver = 0; +exit 1 unless GetOptions ('stop-server' => \$stopserver, 'stop' => \$stopserver); + +# command line options +if ($#ARGV != 1) { + print "Usage: $0 [--stop] <version> <cluster>\n"; + exit 1; +} + +my ($version) = $ARGV[0] =~ /^(\d+\.?\d+)$/; +my ($cluster) = $ARGV[1] =~ /^([-.\w]+)$/; +error 'invalid version' unless defined $version; +error 'invalid cluster name' unless defined $cluster; + +my %info; +my $c; # configuration directory +my $startconf; + +if (cluster_exists $version, $cluster) { + %info = cluster_info ($version, $cluster); + validate_cluster_owner \%info if ($info{pgdata} && -d $info{pgdata}); # ignore missing data directory + + if ($info{'running'}) { + if ($stopserver) { + if ($info{'pgdata'} && -d $info{'pgdata'}) { + if (system ('pg_ctlcluster', $version, $cluster, 'stop')) { + error 'could not stop server, aborting'; + } + } else { + print STDERR "Warning: corrupted cluster: data directory does not exist any more, but server is still running; you have to manually kill the postgres process\n"; + } + } else { + error 'This cluster is still running. Stop it or supply the --stop option'; + } + } + $c = $info{'configdir'}; + $startconf = $info{'start'}; +} else { + $c = "/etc/postgresql/$version/$cluster"; + + # check if we have a broken cluster, clean up if necessary + -d $c or error 'specified cluster does not exist'; +} + +# disable systemd-enabled clusters +if (not exists $ENV{'PG_CLUSTER_CONF_ROOT'} and -d '/run/systemd/system') { + if ($> == 0) { + system "systemctl", "disable", "postgresql\@$version-$cluster"; + system "systemctl", "disable", "--now", "pg_receivewal\@$version-$cluster"; + system "systemctl", "disable", "--now", "pg_basebackup\@$version-$cluster.timer"; + system "systemctl", "disable", "--now", "pg_dump\@$version-$cluster.timer"; + } +} + +if ($info{'pgdata'} && -d $info{'pgdata'}) { + # remove custom wal directory + if ($info{waldir} and -d $info{waldir}) { + my $walowner = (stat($info{waldir}))[4]; + if (defined $walowner and $walowner == $info{owneruid}) { + my $result = system 'rm', '-r', '--', $info{waldir}; + if ($result) { + print STDERR "Warning: could not remove wal directory $info{waldir}"; + } + } else { + print STDERR "Warning: wal directory $info{waldir} is not owned by uid $info{owneruid}, not removing\n"; + } + } + + # remove tablespace subdirectories that belong to our version + for my $link (glob "$info{pgdata}/pg_tblspc/*") { + next unless (-l $link); + my $tblspc = readlink $link; + my $tblspcowner = (stat($tblspc))[4]; + if (defined $tblspcowner and $tblspcowner == $info{owneruid}) { + if ($version >= 9.0) { + for my $dir (glob "$tblspc/PG_${version}_*") { + my $dirowner = (stat($dir))[4]; + if (defined $dirowner and $dirowner == $info{owneruid}) { + my $result = system 'rm', '-r', '--', ($dir =~ /(.*)/); # untaint + if ($result) { + print STDERR "Warning: could not remove tablespace directory $dir"; + } + } else { + print STDERR "Warning: tablespace subdirectory $dir (in tablespace linked from $link) is not owned by uid $info{owneruid}, not removing\n"; + } + } + } else { # before 9.0 + if (open my $fh, '<', "$tblspc/PG_VERSION") { + my $v = <$fh>; + chomp $v; + close $fh; + if ($v eq $version) { + $tblspc =~ /(.*)/; # untaint + my $result = system "rm -r -- $1/*"; + } else { + print STDERR "Warning: tablespace directory $tblspc (linked from $link) is from PostgreSQL version $v, not removing\n"; + } + } else { + print STDERR "Warning: tablespace directory $tblspc (linked from $link) is not a PostgreSQL directory, not removing\n"; + } + } + } else { + print STDERR "Warning: tablespace directory $tblspc (linked from $link) is not owned by uid $info{owneruid}, not considering\n"; + } + } + + # remove pgdata + my $result = system 'rm', '-r', $info{'pgdata'}; + if ($result) { + if (! -w ($info{'pgdata'} . '/..')) { + error 'you might need to run this program with root privileges'; + } + exit $result; + } +} else { + print STDERR "Warning: corrupted cluster: data directory does not exist\n"; +} + +# remove config +unlink $c.'/pg_hba.conf', $c.'/pg_ident.conf', $c.'/postgresql.conf', + $c.'/start.conf', $c.'/log', $c.'/autovacuum_log', $c.'/pgdata', + $c.'/environment', $c.'/pg_ctl.conf'; +rmdir $_ foreach (map { /(.*)/ && $1 } glob "$c/*"); # remove empty conf.d directories + +unlink $info{'logfile'} if defined ($info{'logfile'}); +if ($info{'socketdir'} and $info{'socketdir'} !~ /^(\/tmp|\/var\/run\/postgresql)\/?$/) { + rmdir $info{'socketdir'}; +} +rmdir $c; +rmdir "/etc/postgresql/$version"; +rmdir "/var/lib/postgresql/$version/$cluster"; +rmdir "/var/lib/postgresql/$version"; +unlink "/var/log/postgresql/postgresql-$version-$cluster.log"; +# remove logrotated files +foreach my $f (</var/log/postgresql/postgresql-$version-$cluster.log.[1-9]*>) { + unlink ($f =~ /(.*)/); # untaint +} + +# remove stats_temp_directory +my $statstempdir = $info{config}->{stats_temp_directory}; +if ($statstempdir) { + my $statsowner = (stat($statstempdir))[4]; + if (defined $statsowner and defined $info{owneruid} and $statsowner == $info{owneruid}) { + foreach my $f (<$statstempdir/*.stat>) { + unlink ($f =~ /(.*)/); # untaint + } + rmdir $statstempdir; + } +} + +# notify systemd when an autostarted cluster went away +if (not exists $ENV{'PG_CLUSTER_CONF_ROOT'} and $startconf and $startconf eq 'auto' and -d '/run/systemd/system') { + if ($> == 0) { + system 'systemctl daemon-reload'; + } elsif (-t 1) { + print "Warning: systemd was not informed about the removed cluster yet. Operations like \"service postgresql start\" might fail. To fix, run:\n"; + print " sudo systemctl daemon-reload\n"; + } +} + +# notify apt about the new cluster +if (not exists $ENV{'PG_CLUSTER_CONF_ROOT'} and $> == 0) { + system "/usr/share/postgresql-common/pg_updateaptconfig"; +} + +exit 0; + +__END__ + +=head1 NAME + +pg_dropcluster - completely delete a PostgreSQL cluster + +=head1 SYNOPSIS + +B<pg_dropcluster> [B<--stop>] I<cluster-version> I<cluster-name> + +=head1 DESCRIPTION + +This program removes all files that belong to a given PostgreSQL cluster; that +includes the data, wal, and tablespace directories, the log file, and all configuration files that +were created by L<pg_createcluster(1)>. If the configuration directory +(C</etc/postgresql/>I<version>C</>I<cluster>) is empty after this, it is +removed as well. +An empty socket directory other than B</var/run/postgresql> or B</tmp> is +also removed. + +Usually a cluster which still has a running server attached will not be +deleted. To override this, the B<--stop> option forces a server shutdown +before the files are removed. + +=head1 SEE ALSO + +L<pg_createcluster(1)>, L<pg_ctlcluster(1)> + +=head1 AUTHOR + +Martin Pitt L<E<lt>mpitt@debian.orgE<gt>> + diff --git a/pg_getwal b/pg_getwal new file mode 100755 index 0000000..8e64e87 --- /dev/null +++ b/pg_getwal @@ -0,0 +1,96 @@ +#!/bin/sh + +# retrieve a WAL file from a pg_receivewal archive +# +# Copyright (C) 2021-2022 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +set -eu + +binroot="/usr/lib/postgresql/" +#redhat# binroot="/usr/pgsql-" + +if [ -z "${2:-}" ]; then + echo "Syntax: $0 /path/to/wal/%f %p" >&2 + exit 1 +fi + +file="$1" +path="$2" + +# sanity-check the first argument +waldir="$(dirname $file)" +if ! [ -d "$waldir" ]; then + echo "$0: $waldir is not a directory" >&2 + exit 129 +fi + +if [ -f "$file.gz" ]; then + gunzip < "$file.gz" > "$path" || exit 129 + +elif [ -f "$file.lz4" ]; then + unlz4 < "$file.lz4" > "$path" || exit 129 + +elif [ -f "$file" ]; then + cp "$file" "$path" || exit 129 + +elif [ -f "$file.gz.partial" ] || [ -f "$file.lz4.partial" ]; then + if [ -s "$file.gz.partial" ]; then + gunzip < "$file.gz.partial" > "$path" || exit 129 + elif [ -s "$file.lz4.partial" ]; then + unlz4 < "$file.lz4.partial" > "$path" || exit 129 + else + # .gz.partial starts completely empty, gunzip doesn't like that + touch "$path" || exit 129 + fi + + # expand file to original size + version=$(cat PG_VERSION) || exit 129 + wal_file_size=$(LC_ALL=C "$binroot$version/bin/pg_controldata" . | awk '/^Bytes per WAL segment:/ { print $5 }') || exit 129 + [ "$wal_file_size" -gt 0 ] || exit 129 + truncate --size="$wal_file_size" "$path" || exit 129 + +elif [ -f "$file.partial" ]; then + cp "$file.partial" "$path" || exit 129 + +else + # file not found, exit silently in order not to spam the server log with errors + exit 1 +fi + +exit 0 + +: <<=cut + +=head1 NAME + +pg_getwal - retrieve a WAL file from a pg_receivewal archive + +=head1 SYNOPSIS + +B<pg_getwal> I</path/to/wal/%f> I<%p> + +=head1 DESCRIPTION + +B<pg_getwal> retrieves and decompresses files from a WAL archive maintained by +B<pg_receivewal> and B<pg_backupcluster>. It is put into PostgreSQL's +B<restore_command> by B<pg_restorecluster>. + +=head1 SEE ALSO + +L<pg_restorecluster(1)>, L<pg_backupcluster(1)>. + +=head1 AUTHOR + +Christoph Berg L<E<lt>myon@debian.orgE<gt>> + +=cut @@ -0,0 +1,144 @@ +#!/usr/bin/perl -w + +# Add, remove, or test a pg_hba.conf entry. +# +# (C) 2005-2009 Martin Pitt <mpitt@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use PgCommon; +use Getopt::Long; +use Net::CIDR; + +# global variables + +my $ip = ''; # default to local unix socket +my $force_ssl = 0; +my ($method, $ver_cluster, $db, $user); +my $mode; +my @hba; + +# Print an error message to stderr and exit with status 2 +sub error2 { + print STDERR 'Error: ', $_[0], "\n"; + exit 2; +} + +# Check if s1 is equal to s2 or s2 is 'all'. +# Arguments: <s1> <s2> +sub match_all { + return ($_[1] eq 'all' || $_[0] eq $_[1]); +} + +# Check if given IP matches the specification in the HBA record. +# Arguments: <ip> <ref to hba hash> +sub match_ip { + my ($ip, $hba) = @_; + + # Don't try to mix IPv4 and IPv6 addresses since that will make cidrlookup + # croak + return 0 if ((index $ip, ':') < 0) ^ ((index $$hba{'ip'}, ':') < 0); + + return Net::CIDR::cidrlookup ($ip, $$hba{'ip'}); +} + +# Check if arguments match any line +# Return: 1 if match was found, 0 otherwise. +sub mode_test { + foreach my $hbarec (@hba) { + if (!defined($$hbarec{'type'})) { + next; + } + next if $$hbarec{'type'} eq 'comment'; + next unless match_all ($user, $$hbarec{'user'}) && + match_all ($db, $$hbarec{'db'}) && + $$hbarec{'method'} eq $method; + + if ($ip) { + return 1 if + (($force_ssl && $$hbarec{'type'} eq 'hostssl') || + $$hbarec{'type'} =~ /^host/) && + match_ip ($ip, $hbarec); + } else { + return 1 if $$hbarec{'type'} eq 'local'; + } + } + + return 0; +} + +# Print hba conf. +sub mode_print { + foreach my $hbarec (@hba) { + print "$$hbarec{'line'}\n"; + } +} + +# Generate a pg_hba.conf line that matches the command line args. +sub create_hba_line { + if ($ip) { + return sprintf "%-7s %-11s %-11s %-35s %s\n", + $force_ssl ? 'hostssl' : 'host', $db, $user, $ip, $method; + } else { + return sprintf "%-7s %-11s %-47s %s\n", 'local', $db, $user, $method; + } +} + +# parse arguments + +my $ip_arg; +exit 3 unless GetOptions ( + 'cluster=s' => \$ver_cluster, + 'ip=s' => \$ip_arg, + 'method=s' => \$method, + 'force-ssl' => \$force_ssl +); + +if ($#ARGV != 2) { + print STDERR "Usage: $0 mode [options] <database> <user>\n"; + exit 2; +} +($mode, $db, $user) = @ARGV; + +error2 '--cluster must be specified' unless $ver_cluster; +my ($version, $cluster) = split ('/', $ver_cluster); +error2 'No version specified with --cluster' unless $version; +error2 'No cluster specified with --cluster' unless $cluster; +error2 'Cluster does not exist' unless cluster_exists $version, $cluster; +if (defined $ip_arg) { + $ip = Net::CIDR::cidrvalidate $ip_arg; + error2 'Invalid --ip argument' unless defined $ip; +} + +unless (defined $method) { + $method = ($ip ? 'md5' : 'ident sameuser'); +} +error2 'Invalid --method argument' unless PgCommon::valid_hba_method($method); + +# parse file + +my $hbafile = "/etc/postgresql/$version/$cluster/pg_hba.conf"; +@hba = read_pg_hba $hbafile; +error2 "Could not read $hbafile" unless $#hba; + +if ($mode eq 'pg_test_hba') { + if (mode_test) { + exit 0; + } else { + print create_hba_line(); + exit 1; + } +} elsif ($mode eq 'pg_print_hba') { + mode_print(); +} else { + error2 "Unknown mode: $mode"; +} diff --git a/pg_lsclusters b/pg_lsclusters new file mode 100755 index 0000000..fec09d7 --- /dev/null +++ b/pg_lsclusters @@ -0,0 +1,184 @@ +#!/usr/bin/perl -wT +# Show all PostgreSQL clusters in a list +# +# (C) 2005-2009 Martin Pitt <mpitt@debian.org> +# (C) 2013-2018 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use PgCommon; +use Getopt::Long; + +sub help ($) +{ + my $exit = shift; + print "Usage: $0 [-hjs] + +Options: + -h --no-header Omit column headers in output + -j --json JSON output + -s --start-conf Include start.conf information in status column + --help Print help\n"; + + exit $exit; +} + +# option handling +my $no_header; +my $json; +my $start_conf; +help(1) unless GetOptions ( + 'help' => sub { help(0); }, + 'h|no-header' => \$no_header, + 'j|json' => \$json, + 's|start-conf' => \$start_conf, +); + +my (@versions, $ls_cluster); +if (@ARGV == 1 and $ARGV[0] =~ m{^(\d+(?:\.\d+)?)[-/]([-.\w]+)$}) { + push @versions, $1; + $ls_cluster = $2; +} elsif (@ARGV >= 1) { + $ARGV[0] =~ /^(\d+(?:\.\d+)?)$/ or error "Invalid version number \"$ARGV[0]\""; + push @versions, $1; +} else { + @versions = get_versions(); +} +if (@ARGV >= 2 and $ARGV[1] =~ /^([-.\w]+)$/) { + $ls_cluster = $1; +} + +error "Cluster @versions $ls_cluster does not exist" + if ($ls_cluster and not cluster_exists(@versions, $ls_cluster)); + +# data collection +my @lines; +push @lines, ['Ver', 'Cluster', 'Port', 'Status', 'Owner', 'Data directory', 'Log file'] + unless ($no_header); +my $jsoninfo = []; + +foreach my $v (@versions) { + my $pg_log = $v >= 10 ? 'log' : 'pg_log'; # log directory in PGDATA changed in PG 10 + my @clusters = $ls_cluster ? $ls_cluster : get_version_clusters $v; + foreach my $c (@clusters) { + my %info = cluster_info $v, $c; + + my $status = $info{'running'} ? "online" : "down"; + $status .= ",recovery" if ($info{'recovery'}); + $status .= ",$info{supervisor}" if ($info{'supervisor'}); + $status .= ",$info{start}" if ($start_conf); + unless (-e "${PgCommon::binroot}$v/bin/postgres") { + $status .= ",binaries_missing"; + $info{binaries_missing} = 1; + } + + my $logfile = $info{logfile} // '<unknown>'; # default logfile in /var/log/postgresql + if (config_bool ($info{config}->{logging_collector})) { + my $path = $info{config}->{log_directory} || $pg_log; + my $file = $info{config}->{log_filename} || 'postgresql-%Y-%m-%d_%H%M%S.log'; + $logfile = "$path/$file"; + } + my $destination = $info{config}->{log_destination} || 'stderr'; + $destination =~ s/stderr/$logfile/; + my $csvlog = $logfile; + $csvlog =~ s/(?:\.log)?$/.csv/; + $destination =~ s/csvlog/$csvlog/; + + push @lines, [$v, $c, $info{'port'} // '<unknown>', $status, + defined $info{'owneruid'} ? (getpwuid $info{'owneruid'})[0] : '<unknown>', + $info{'pgdata'} || '<unknown>', $destination]; + $info{version} = $v; + $info{cluster} = $c; + push @$jsoninfo, \%info; + } +} + +# output +if ($json) { + eval { require JSON; }; + error 'Please install JSON.pm for JSON output (Debian: libjson-perl)' + if ($@); + print JSON::encode_json($jsoninfo) . "\n"; + exit 0; +} + +my @colwidth = qw(1 1 1 1 1 1 1); +foreach my $line (@lines) { + for (my $i = 0; $i < @$line - 1; $i++) { # skip adjustment for last column + my $len = length @$line[$i]; + $colwidth[$i] = $len if ($len > $colwidth[$i]); + } +} + +my $color = -t 1; # color output if stdout is a terminal +my $fmtstring = join ' ', map { "%-${_}s" } @colwidth; + +foreach my $line (@lines) { + if ($color and $line->[0] ne 'Ver') { # don't color header + printf "\033[%dm$fmtstring\033[0m\n", + ($line->[3] =~ /^online/ ? 32 : 31), # green, red + @$line; + } else { + printf "$fmtstring\n", @$line; + } +} + +__END__ + +=head1 NAME + +pg_lsclusters - show information about all PostgreSQL clusters + +=head1 SYNOPSIS + +B<pg_lsclusters> [I<options>] [I<version> [I<cluster>]] + +=head1 DESCRIPTION + +This command list the status and some configuration details of all clusters. +If a version and optionally a cluster name are given, only these are shown. + +=head1 OPTIONS + +=over 4 + +=item B<-h>, B<--no-header> + +Do not print the column header line. + +=item B<-j>, B<--json> + +Output information in JSON format. Needs JSON.pm installed. +(Debian package: libjson-perl) + +=item B<-s>, B<--start-conf> + +Include F<start.conf> information in status column. + +=item B<--help> + +Print usage help. + +=back + +=head1 NOTES + +The cluster status is shown as B<online> or B<down>. If a F<recovery.conf> file +is found in the data directory, B<,recovery> is appended. The latter needs read +access to the data directory, which only root and the cluster owner have. + +The output lines are colored green and red to indicate the cluster status +visually. + +=head1 AUTHOR + +Martin Pitt L<E<lt>mpitt@debian.orgE<gt>> diff --git a/pg_renamecluster b/pg_renamecluster new file mode 100755 index 0000000..e5f96bb --- /dev/null +++ b/pg_renamecluster @@ -0,0 +1,176 @@ +#!/usr/bin/perl -wT + +# Rename a PostgreSQL cluster +# +# (C) 2014-2021 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use warnings; +use PgCommon; +use Getopt::Long; +use POSIX; + +# untaint environment +$ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin'; +delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; + +if (@ARGV != 3) { + print "Usage: $0 [OPTIONS] <version> <old cluster name> <new cluster name>\n"; + exit 1; +} + +my ($version) = $ARGV[0] =~ /^(\d+\.?\d+)$/; +my ($oldcluster) = $ARGV[1] =~ /^([-.\w]+)$/; +my ($newcluster) = $ARGV[2] =~ /^([-.\w]+)$/; +if ($newcluster =~ /-/ and -t 1) { + print "Warning: cluster names containing dashes (-) will cause problems when running from systemd. Continuing anyway\n"; +} + +error "Old and new name must be different" + if ($oldcluster eq $newcluster); +error "specified cluster $version $oldcluster does not exist" + unless (cluster_exists $version, $oldcluster); +error "target cluster $version $newcluster already exists" + if (cluster_exists $version, $newcluster); +my %info = cluster_info ($version, $oldcluster); +validate_cluster_owner \%info; + +# stopping old cluster, so that we notice early when there are still +# connections +if ($info{'running'}) { + print "Stopping cluster $version $oldcluster ...\n"; + my @argv = ('pg_ctlcluster', $version, $oldcluster, 'stop'); + error "Could not stop cluster" if system @argv; +} + +# Arguments: <string>, <from>, <to> +sub strrepl { + my ($s, $f, $t) = @_; + $s =~ s/\b\Q$f\E\b/$t/g; + return $s; +} + +# rename config directory +my $olddir = "$PgCommon::confroot/$version/$oldcluster"; +my $newdir = "$PgCommon::confroot/$version/$newcluster"; +rename $olddir, $newdir or error "Could not rename config directory $olddir: $!"; + +# adapt paths to configuration files +my %c = read_cluster_conf_file $version, $newcluster, 'postgresql.conf'; +if ($c{hba_file}) { + PgCommon::set_conf_value $version, $newcluster, 'postgresql.conf', 'hba_file', + strrepl($c{hba_file}, $oldcluster, $newcluster); +} +if ($c{ident_file}) { + PgCommon::set_conf_value $version, $newcluster, 'postgresql.conf', 'ident_file', + strrepl($c{ident_file}, $oldcluster, $newcluster); +} +if ($c{external_pid_file}) { + PgCommon::set_conf_value $version, $newcluster, 'postgresql.conf', 'external_pid_file', + strrepl($c{external_pid_file}, $oldcluster, $newcluster); +} + +# update cluster_name +if ($c{cluster_name}) { + PgCommon::set_conf_value $version, $newcluster, 'postgresql.conf', 'cluster_name', + strrepl ($c{cluster_name}, $oldcluster, $newcluster); +} + + +# rename data directory +if ($info{pgdata}) { + my $newpgdata = strrepl ($info{pgdata}, $oldcluster, $newcluster); + if ($info{pgdata} ne $newpgdata) { + rename $info{pgdata}, $newpgdata or + error "Could not rename data directory $info{pgdata}: $!"; + PgCommon::set_conf_value $version, $newcluster, 'postgresql.conf', + 'data_directory', $newpgdata; + } +} + +# rename stats_temp_directory +my $statstempdir = $info{config}->{stats_temp_directory}; +if ($statstempdir) { + my $newstatstempdir = strrepl ($statstempdir, $oldcluster, $newcluster); + if ($statstempdir ne $newstatstempdir) { + PgCommon::set_conf_value $version, $newcluster, 'postgresql.conf', + 'stats_temp_directory', $newstatstempdir; + if (-d $statstempdir) { + rename $statstempdir, $newstatstempdir or + error "Could not rename stats temp directory $statstempdir}: $!"; + } + } +} + +# rename old log files +my $logdir = "/var/log/postgresql"; +if (opendir LOG, $logdir) { + while (my $logfile = readdir LOG) { + next unless $logfile =~ /^(\Qpostgresql-$version-$oldcluster.log\E.*)/; + $logfile = $1; # untaint + my $f = strrepl ($logfile, $oldcluster, $newcluster); + rename "$logdir/$logfile", "$logdir/$f" or error "rename $logdir/$logfile: $!"; + } + closedir LOG; +} + +# notify systemd about the new cluster +if (not exists $ENV{'PG_CLUSTER_CONF_ROOT'} and -d '/run/systemd/system') { + if ($> == 0) { + system 'systemctl daemon-reload'; + } elsif (-t 1) { + print "Warning: systemd does not know about the new cluster yet. Operations like \"service postgresql start\" will not handle it. To fix, run:\n"; + print " sudo systemctl daemon-reload\n"; + } +} + +# start cluster if it was running before +if ($info{'running'}) { + print "Starting cluster $version $newcluster ...\n"; + my @argv = ('pg_ctlcluster', $version, $newcluster, 'start'); + error "Could not start cluster" if system @argv; +} + +__END__ + +=head1 NAME + +pg_renamecluster - rename a PostgreSQL cluster + +=head1 SYNOPSIS + +B<pg_renamecluster> I<version> I<oldname> I<newname> + +=head1 DESCRIPTION + +B<pg_renamecluster> changes the name of a PostgreSQL cluster, i. e. the name of +the config directory in /etc/postgresql/I<version>/ along with the data +directory in /var/lib/postgresql/I<version>/. Existing log files in +/var/log/postgresql/ are also renamed. The cluster is stopped and started for +the operation. + +The following B<postgresql.conf> config options are updated to refer to the +changed path names: B<data_directory>, B<hba_file>, B<ident_file>, +B<external_pid_file>, B<stats_temp_directory>, B<cluster_name>. + +=head1 OPTIONS + +None. + +=head1 SEE ALSO + +L<pg_createcluster(8)>, L<pg_dropcluster(8)>, L<pg_lsclusters(1)>, L<pg_wrapper(1)> + +=head1 AUTHOR + +Christoph Berg L<E<lt>myon@debian.orgE<gt>> diff --git a/pg_restorecluster b/pg_restorecluster new file mode 100755 index 0000000..0a4af55 --- /dev/null +++ b/pg_restorecluster @@ -0,0 +1,436 @@ +#!/usr/bin/perl -wT + +# pg_restorecluster: restore from a pg_backupcluster backup +# +# Copyright (C) 2021 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use warnings; +use Cwd 'abs_path'; +use Getopt::Long; +use PgCommon; + +my ($version, $cluster); + +# untaint environment +$ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin'; +delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; +umask 022; + +sub help () { + print "Syntax: $0 [options] <version> <cluster> <backup> +Options: + -d --datadir DIR Data directory for restored cluster (default per createcluster.conf) + -p --port PORT Use port PORT for restored cluster (default is next free port) + -s --start Start cluster after restoring (default for restore from dump) + --archive Configure recovery from WAL archive + --pitr TIMEST Configure point-in-time recovery to TIMESTAMP from WAL archive + --wal-archive DIR Read WAL from archive DIR (default <backup>/../wal) +"; +} + +my $createclusterconf = "$PgCommon::common_confdir/createcluster.conf"; +my ($datadir, $port, $start, $recovery_target_time, $archive_recovery, $wal_archive); + +exit 1 unless GetOptions ( + 'd|datadir=s' => \$datadir, + 'p|port=s' => \$port, + 's|start' => \$start, + 'archive' => \$archive_recovery, + 'pitr=s' => \$recovery_target_time, + 'recovery-target-time=s' => \$recovery_target_time, + 'wal-archive=s' => \$wal_archive, + 'createclusterconf=s' => \$createclusterconf, +); +if ($recovery_target_time) { + ($recovery_target_time) = $recovery_target_time =~ /(.*)/; # untaint +} + +# accept both "version cluster" and "version[-/]cluster" +if (@ARGV >= 2 and $ARGV[0] =~ m!^(\d+\.?\d)[-/]([^/]+)$!) { + ($version, $cluster) = ($1, $2); + shift @ARGV; +} elsif (@ARGV >= 3 and $ARGV[0] =~ /^(\d+\.?\d)$/) { + $version = $1; + ($cluster) = ($ARGV[1]) =~ m!^([^/]+)$!; + shift @ARGV; + shift @ARGV; +} else { + help(); + exit 1; +} + +error "cluster $version $cluster already exists" if cluster_exists $version, $cluster; + +my %defaultconf = PgCommon::read_conf_file ($createclusterconf); + +# functions to be run as root + +sub create_cluster_directories($$$$) { + my ($owneruid, $ownergid, $configdir, $datadir) = @_; + my @postgres_user = getpwnam 'postgres'; + my ($pg_uid, $pg_gid) = (getpwnam 'postgres')[2, 3]; + + for my $pgdir ("/etc/postgresql", "/etc/postgresql/$version", "/var/lib/postgresql", "/var/lib/postgresql/$version") { + if (! -e $pgdir) { + mkdir $pgdir or error "mkdir $pgdir: $!"; + chown $pg_uid, $pg_gid, $pgdir or error "chown $pgdir: $!"; + } + } + mkdir $configdir or error "mkdir $configdir: $!"; + chown $owneruid, $ownergid, $configdir or error "chown $configdir: $!"; + mkdir $datadir, 0700 or error "mkdir $datadir: $!"; + chown $owneruid, $ownergid, $datadir or error "chown $datadir: $!"; +} + +sub create_cluster($$$$$) { + my ($backup, $owneruid, $ownergid, $configdir, $datadir) = @_; + + my @createclusteropts = (); + if (-f "$backup/createcluster.opts") { + open my $fh, "$backup/createcluster.opts" or error "$backup/createcluster.opts: $!"; + local $/; # slurp mode + my ($opts) = <$fh> =~ /(.*)/; # untaint + @createclusteropts = split /\s+/, $opts; + close $fh; + } + + system_or_error "pg_createcluster", + "--datadir", $datadir, + "--user", $owneruid, "--group", $ownergid, + $version, $cluster, + "--", + @createclusteropts; +} + +sub start_cluster() { + print "Starting cluster $version $cluster ...\n"; + system_or_error "pg_ctlcluster", $version, $cluster, "start"; +} + +sub switch_to_cluster_owner($$) { + my ($owneruid, $ownergid) = @_; + change_ugid $owneruid, $ownergid; +} + +# restore functions + +sub unpack_tar($$$) { + my ($backup, $tar, $dir) = @_; + + if (-f "$backup/$tar.gz") { + $tar = "$tar.gz"; + } elsif (-f "$backup/$tar.bz2") { + $tar = "$tar.bz2"; + } elsif (-f "$backup/$tar.xz") { + $tar = "$tar.xz"; + } elsif (-f "$backup/$tar") { + # do nothing + } else { + error "$backup/config.tar* is missing"; + } + + print "Restoring $backup/$tar to $dir ...\n"; + system_or_error "tar", "-C", $dir, "-xf", "$backup/$tar"; +} + +sub restore_config($$) { + my ($backup, $configdir) = @_; + unpack_tar($backup, "config.tar", $configdir); +} + +sub update_config($$$) { + my ($configdir, $datadir, $port) = @_; + my %settings = ( + data_directory => $datadir, + hba_file => "$configdir/pg_hba.conf", + ident_file => "$configdir/pg_ident.conf", + external_pid_file => "/var/run/postgresql/$version-$cluster.pid", + port => $port, + ); + $settings{cluster_name} = "$version/$cluster" if ($version >= 9.5); + $settings{stats_temp_directory} = "/var/run/postgresql/$version-$cluster.pg_stat_tmp" if ($version < 15); + my %config = PgCommon::read_conf_file("$configdir/postgresql.conf"); + for my $guc (sort keys %settings) { + if (not exists $config{$guc} or $config{$guc} ne $settings{$guc}) { + print "Setting $guc = $settings{$guc}\n"; + PgCommon::set_conffile_value("$configdir/postgresql.conf", $guc, $settings{$guc}); + } + } +} + +sub restore_basebackup($$) { + my ($backup, $datadir) = @_; + unpack_tar($backup, "base.tar", $datadir); +} + +sub restore_wal($$) { + my ($backup, $datadir) = @_; + return if ($version < 10); # WAL contained in base.tar.gz in PG 9.x + unpack_tar($backup, "pg_wal.tar", "$datadir/pg_wal"); +} + +sub archive_recovery_options($$$) { + my ($backup, $datadir, $wal_archive) = @_; + + $wal_archive = abs_path($wal_archive) or error "$wal_archive: $!"; + -d $wal_archive or error "$wal_archive is not a directory"; + + print "Setting archive recovery options"; + my $recovery_options = "restore_command = '/usr/share/postgresql-common/pg_getwal $wal_archive/%f %p'\n"; + if ($recovery_target_time) { + $recovery_options .= "recovery_target_time = '$recovery_target_time'\n"; + $recovery_options .= "recovery_target_action = 'promote'\n"; + } + + if ($version >= 12) { + my $autoconf = "$datadir/postgresql.auto.conf"; + open my $fh, ">>", $autoconf or error "$autoconf: $!"; + print $fh $recovery_options or error "$autoconf: $!"; + close $fh or error "$autoconf: $!"; + + my $recoverysignal = "$datadir/recovery.signal"; + open my $fh2, ">", $recoverysignal or error "$recoverysignal: $!"; + close $fh2 or error "$recoverysignal: $!"; + + } else { + my $recoveryconf = "$datadir/recovery.conf"; + open my $fh, ">>", $recoveryconf or error "$recoveryconf: $!"; + print $fh $recovery_options or error "$recoveryconf: $!"; + close $fh or error "$recoveryconf: $!"; + } +} + +sub reset_archive_recovery_options() { + if ($version >= 12) { + system_or_error "psql", + "--cluster", "$version/$cluster", + "-XAtqc", "ALTER SYSTEM RESET restore_command"; + system_or_error "psql", + "--cluster", "$version/$cluster", + "-XAtqc", "ALTER SYSTEM RESET recovery_target_time" + if ($recovery_target_time); + system_or_error "psql", + "--cluster", "$version/$cluster", + "-XAtqc", "ALTER SYSTEM RESET recovery_target_action" + if ($recovery_target_time); + } +} + +sub restore_globals($$) { + my ($backup, $owneruid) = @_; + my $owner = (getpwuid $owneruid)[0] or error "UID $owneruid has no name"; + + print "Restoring $backup/globals.sql ...\n"; + open my $globals, "$backup/globals.sql" or error "$backup/globals.sql: $!"; + open my $psql, "|-", "psql", "--cluster", "$version/$cluster", "-vON_ERROR_STOP=1", "-Xq" or error "psql: $!"; + while (my $line = <$globals>) { + next if ($line eq "CREATE ROLE $owner;\n"); + print $psql $line or error "psql: $!"; + } + close $globals; + close $psql; + error "psql failed" if ($?); +} + +sub create_databases($) { + my ($backup) = @_; + + print "Creating databases from $backup/databases.sql ...\n"; + system_or_error "psql", "--cluster", "$version/$cluster", "-vON_ERROR_STOP=1", "-Xqf", "$backup/databases.sql"; +} + +sub restore_dumps($) { + my ($backup) = @_; + + for my $dump (sort glob "$backup/*.dump") { + $dump =~ m!(.*/([^/]*).dump)$!; + $dump = $1; # untaint + my $db = $2; + print "Restoring $dump to database $db ...\n"; + system_or_error "pg_restore", "--cluster", "$version/$cluster", "-d", $db, $dump; + } +} + +sub wait_for_recovery() { + my $sleep = 1; + + print "Waiting for end of recovery ...\n"; + while (1) { + open my $psql, "-|", "psql", "--cluster", "$version/$cluster", "-XAtc", "SELECT pg_is_in_recovery()" or error "psql: $!"; + my $status = <$psql>; + error "psql: $!" unless (defined $status); + close $psql or error "psql: $!"; + last if ($status eq "f\n"); + sleep($sleep++); + } +} + +sub analyze() { + system_or_error "vacuumdb", + "--cluster", "$version/$cluster", + "--analyze-only", + ($version >= 9.4 ? "--analyze-in-stages" : ()), + "--all"; +} + +sub lscluster() { + system_or_error "pg_lsclusters", $version, $cluster; +} + +# main + +my ($backup) = $ARGV[0] =~ /(.*)/; # untaint +error "$backup is not a directory" unless (-d $backup); +$backup =~ s/\/$//; # strip trailing slash +my ($owneruid, $ownergid) = (stat $backup)[4, 5]; +my $configdir = "/etc/postgresql/$version/$cluster"; +$datadir //= replace_v_c($defaultconf{data_directory} // "/var/lib/postgresql/%v/%c", $version, $cluster); +($datadir) = $datadir =~ /(.*)/; # untaint +$wal_archive //= "$backup/../wal"; +$port //= next_free_port(); + +if ($backup =~ /\.backup$/) { + create_cluster_directories($owneruid, $ownergid, $configdir, $datadir); + if (fork == 0) { + switch_to_cluster_owner($owneruid, $ownergid); + restore_config($backup, $configdir); + update_config($configdir, $datadir, $port); + restore_basebackup($backup, $datadir); + restore_wal($backup, $datadir); + archive_recovery_options($backup, $datadir, $wal_archive) + if ($archive_recovery or $recovery_target_time); + exit(0); + } + wait; + exit(1) if ($?); + if ($start) { + print "\n"; + start_cluster(); + switch_to_cluster_owner($owneruid, $ownergid); + wait_for_recovery(); + reset_archive_recovery_options() if ($archive_recovery or $recovery_target_time); + analyze(); + } + print "\n"; + lscluster(); + +} elsif ($backup =~ /\.dump$/) { + create_cluster($backup, $owneruid, $ownergid, $configdir, $datadir); + print "\n"; + if (fork == 0) { + switch_to_cluster_owner($owneruid, $ownergid); + restore_config($backup, $configdir); + update_config($configdir, $datadir, $port); + exit(0); + } + wait; + exit(1) if ($?); + start_cluster(); + switch_to_cluster_owner($owneruid, $ownergid); + restore_globals($backup, $owneruid); + create_databases($backup); + restore_dumps($backup); + analyze(); + print "\n"; + lscluster(); + +} else { + error "$backup must end in either .backup or .dump"; +} + +__END__ + +=head1 NAME + +pg_restorecluster - Restore from a pg_backupcluster backup + +=head1 SYNOPSIS + +B<pg_restorecluster> [I<options>] I<version> I<cluster> I<backup> + +=head1 DESCRIPTION + +B<pg_restorecluster> restores a PostgreSQL cluster from a backup created by +B<pg_backupcluster>. The cluster will be newly created in the system using the +name provided on the command line; this allows renaming a cluster on restore. +The restored cluster configuration will be updated to reflect the new name and +location. + +The I<backup> name passed must end in either B<.basebackup> or B<.dump>; +usually this will be the full path to a backup directory in +C</var/backups/postgresql/version-cluster/> as reported by +B<pg_backupcluster ... list>. + +Basebackups are restored as-is. For dumps, B<pg_createcluster> is used to +create a new cluster, and schema and data are restored via B<pg_restore>. + +=head1 OPTIONS + +=over 4 + +=item B<-d --datadir> I<DIR> + +Use I<DIR> as data directory for the restored cluster (default per +createcluster.conf, by default /var/lib/postgresql/I<version>/I<cluster>). + +=item B<-p --port> I<N> + +Use port I<N> for restored cluster (default is next free port). + +=item B<-s --start> + +Start cluster after restoring (default for restore from dump; off for +basebackup restores). + +After the cluster has been started, B<ANALYZE> is run on all databases. + +=item B<--archive> + +Configure cluster for recovery from WAL archive. This sets B<restore_command> +to retrieve WAL files from I<backup>B</../wal>. + +=item B<--pitr> I<TIMESTAMP> + +=item B<--recovery-target-time> I<TIMESTAMP> + +Additionally to setting B<restore_command>, set B<recovery_target_time> to +I<TIMESTAMP> for point-in-time recovery. Also sets +B<recovery_target_action='promote'>. + +=item B<--wal-archive> I<DIR> + +For archive recovery, read WAL from archive I<DIR> (default is +I<backup>B</../wal>). + +=back + +=head1 FILES + +=over 4 + +=item /var/backups + +Default root directory for cluster backup directories. + +=back + +See L<pg_backupcluster(1)> for a description of files. + +=head1 SEE ALSO + +L<pg_backupcluster(1)>, L<pg_restore(1)>, L<vacuumdb(1)>. + +=head1 AUTHOR + +Christoph Berg L<E<lt>myon@debian.orgE<gt>> diff --git a/pg_updateaptconfig b/pg_updateaptconfig new file mode 100755 index 0000000..7e041c5 --- /dev/null +++ b/pg_updateaptconfig @@ -0,0 +1,41 @@ +#!/bin/sh + +# Tell apt which PostgreSQL versions have clusters present + +set -eu + +APTCONFDIR="/etc/apt/apt.conf.d" +[ -d "$APTCONFDIR" ] || exit 0 # skip generation on RPM systems +APTCONF="$APTCONFDIR/02autoremove-postgresql" +TMPCONF="$(mktemp --tmpdir pg_updateaptconfig.XXXXXX)" +trap "rm -f $TMPCONF" EXIT + +cat > $TMPCONF <<EOF +// DO NOT EDIT! +// File maintained by /usr/share/postgresql-common/pg_updateaptconfig. +// +// Mark all PostgreSQL packages as NeverAutoRemove for which PostgreSQL +// clusters exist. This is especially important when the "postgresql" meta +// package changes its dependencies to a new version, which might otherwise +// trigger the old postgresql-NN package to be automatically removed, rendering +// the old database cluster inaccessible. + +APT +{ + NeverAutoRemove + { +EOF + +pg_lsclusters -h | cut -d ' ' -f 1 | uniq | while read version; do + echo " \"^postgresql.*-$version\";" >> $TMPCONF +done + +cat >> $TMPCONF <<EOF + }; +}; +EOF + +if ! cmp --silent $TMPCONF $APTCONF; then + cp $TMPCONF $APTCONF + chmod 444 $APTCONF +fi diff --git a/pg_updatedicts b/pg_updatedicts new file mode 100755 index 0000000..8d8b50b --- /dev/null +++ b/pg_updatedicts @@ -0,0 +1,139 @@ +#!/usr/bin/perl -w + +# Create dictionaries and affix rules palatable for PostgreSQL, using installed +# myspell and hunspell dictionaries. +# +# (C) 2008-2009 Martin Pitt <mpitt@debian.org> +# (C) 2012-2017 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use warnings; +my @srcdirs = ('/usr/share/hunspell', '/usr/share/myspell/dicts'); +my $cachedir = '/var/cache/postgresql/dicts'; +my $pgsharedir = '/usr/share/postgresql/'; + +use PgCommon; + +# determine encoding of an .aff file +sub get_encoding { + open my $fh, '<', $_[0] or die "cannot open $_[0]: $!"; + while (<$fh>) { + if (/^SET ([\w-]+)\s*$/) { return $1; } + } + return undef; +} + +umask 022; +if ((system 'mkdir', '-p', $cachedir) != 0) { + exit 1; +} + +# keep track of all up to date files, so that we can clean up cruft +my %current; + +print "Building PostgreSQL dictionaries from installed myspell/hunspell packages...\n"; +for my $d (@srcdirs) { + for my $aff (glob "$d/*.aff") { + next if -l $aff; # ignore symlinks + my $dic = substr($aff, 0, -3) . 'dic'; + if (! -f $dic) { + print STDERR "ERROR: $aff does not have corresponding $dic, ignoring\n"; + next; + } + + my $enc = get_encoding $aff; + if (!$enc) { + print STDERR "ERROR: no encoding defined in $aff, ignoring\n"; + next; + } + + my $locale = substr ((split '/', $aff)[-1], 0, -4); + $locale =~ tr /A-Z/a-z/; + + $current{"$cachedir/$locale.affix"} = undef; + $current{"$cachedir/$locale.dict"} = undef; + + # convert to UTF-8 and write to cache dir + print " $locale\n"; + if ((system 'iconv', '-f', $enc, '-t', 'UTF-8', '-o', + "$cachedir/$locale.affix", $aff) != 0) { + unlink "$cachedir/$locale.affix"; + print STDERR "ERROR: Conversion of $aff failed\n"; + next; + } + if ((system 'iconv', '-f', $enc, '-t', 'UTF-8', '-o', + "$cachedir/$locale.dict", $dic) != 0) { + unlink "$cachedir/$locale.affix"; + unlink "$cachedir/$locale.dict"; + print STDERR "ERROR: Conversion of $dic failed\n"; + next; + } + + # install symlinks to all versions >= 8.3 + foreach my $v (get_versions) { + next if $v < '8.3'; + my $dest = "$pgsharedir/$v/tsearch_data"; + next if ! -d $dest; + $current{"$dest/$locale.affix"} = undef; + $current{"$dest/$locale.dict"} = undef; + next if -e "$dest/$locale.affix" && ! -l "$dest/$locale.affix"; + next if -e "$dest/$locale.dict" && ! -l "$dest/$locale.dict"; + unlink "$dest/$locale.affix"; + unlink "$dest/$locale.dict"; + symlink "$cachedir/$locale.affix", "$dest/$locale.affix"; + symlink "$cachedir/$locale.dict", "$dest/$locale.dict"; + } + } +} + +# clean up files for locales which do not exist any more +print "Removing obsolete dictionary files:\n"; +foreach my $f (glob "$cachedir/*") { + next if exists $current{$f}; + print " $f\n"; + unlink $f; +} +foreach my $f ((glob "$pgsharedir/*/tsearch_data/*.affix"), + (glob "$pgsharedir/*/tsearch_data/*.dict")) { + next unless -l $f; + next if exists $current{$f}; + print " $f\n"; + unlink $f; +} + +__END__ + +=head1 NAME + +pg_updatedicts - build PostgreSQL dictionaries from myspell/hunspell ones + +=head1 SYNOPSIS + +B<pg_updatedicts> + +=head1 DESCRIPTION + +B<pg_updatedicts> makes dictionaries and affix files from installed myspell +and hunspell dictionary packages available to PostgreSQL for usage with tsearch +and word stem support. In particular, it takes all I<*.dic> and I<*.aff> files +from /usr/share/myspell/dicts/, converts them to UTF-8, puts them into +/var/cache/postgresql/dicts/ with I<*.dict> and I<*.affix> suffixes, and +symlinks them into /usr/share/postgresql/I<version>/tsearch_data/, where +PostgreSQL looks for them. + +Through postgresql-common's dpkg trigger, this program is automatically run +whenever a myspell or hunspell dictionary package is installed or upgraded. + +=head1 AUTHOR + +Martin Pitt L<E<lt>mpitt@debian.orgE<gt>> diff --git a/pg_upgradecluster b/pg_upgradecluster new file mode 100755 index 0000000..06f2bf0 --- /dev/null +++ b/pg_upgradecluster @@ -0,0 +1,966 @@ +#!/usr/bin/perl -wT + +# Upgrade a PostgreSQL cluster to a newer major version. +# +# (C) 2005-2009 Martin Pitt <mpitt@debian.org> +# (C) 2013 Peter Eisentraut <petere@debian.org> +# (C) 2013-2023 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use warnings; +use PgCommon; +use File::Temp qw(tempfile); +use Getopt::Long; +use POSIX qw(lchown); + +# untaint environment +$ENV{'PATH'} = '/bin:/usr/bin'; +delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; + +# global variables +my ($version, $newversion, $cluster, $newcluster); +my (%info, %newinfo); +my ($encoding, $old_lc_ctype, $old_lc_collate); # old cluster encoding +my ($old_locale_provider, $old_icu_locale, $old_icu_rules); +my $maintenance_db = 'template1'; +my $keep_on_error = 0; + +# do not trip over cwd not being accessible to postgres superuser +chdir '/'; + +# update the new cluster's conffiles +sub adapt_conffiles { + my ($newversion, $newcluster, $configfile) = @_; + my %c = read_cluster_conf_file $newversion, $newcluster, $configfile; + + # Arguments: <ref to conf hash> <name> <comment> + my $deprecate = sub { + my ($conf, $guc, $comment) = @_; + if (defined $conf->{$guc}) { + PgCommon::disable_conf_value $newversion, $newcluster, + $configfile, $guc, $comment; + } + }; + + # Arguments: <ref to conf hash> <old name> <new name> + my $rename = sub { + my ($conf, $old, $new) = @_; + if (defined ${$conf}{$old}) { + PgCommon::replace_conf_value $newversion, $newcluster, + $configfile, $old, "deprecated in favor of $new", + $new, ${$conf}{$old}; + } + }; + + # Arguments: <config option> <value> + my $set = sub { + my ($guc, $val) = @_; + PgCommon::set_conf_value $newversion, $newcluster, $configfile, + $guc, $val; + }; + + # adapt paths to configuration files + if ($configfile eq 'postgresql.conf') { + $set->('data_directory', $newinfo{'pgdata'}); + } else { + # fix bug in pg_upgradecluster 200..202 + $deprecate->(\%c, 'data_directory', 'not valid in postgresql.auto.conf'); + } + for my $guc (qw(hba_file ident_file external_pid_file stats_temp_directory)) { + next unless (defined $c{$guc}); + my $val = $c{$guc}; + $val =~ s/\b\Q$version\E\b/$newversion/g; + $val =~ s/\b\Q$cluster\E\b/$newcluster/g if ($cluster ne $newcluster); + $set->($guc, $val); + } + + if ($newversion >= '8.2') { + # preload_libraries -> shared_preload_libraries transition + $rename->(\%c, 'preload_libraries', 'shared_preload_libraries'); + + # australian_timezones -> timezone_abbreviations transition + my $australian_timezones = config_bool $c{'australian_timezones'}; + if (defined $australian_timezones) { + PgCommon::replace_conf_value $newversion, $newcluster, $configfile, + 'australian_timezones', 'deprecated in favor of timezone_abbreviations', + 'timezone_abbreviations', ($australian_timezones ? 'Australia' : 'Default'); + } + } + + if ($newversion >= '8.3') { + $deprecate->(\%c, 'bgwriter_lru_percent', 'deprecated'); + $deprecate->(\%c, 'bgwriter_all_percent', 'deprecated'); + $deprecate->(\%c, 'bgwriter_all_maxpages', 'deprecated'); + + $rename->(\%c, 'redirect_stderr', 'logging_collector'); + + $rename->(\%c, 'stats_command_string', 'track_activities'); + $deprecate->(\%c, 'stats_start_collector', 'deprecated, always on now'); + $deprecate->(\%c, 'stats_reset_on_server_start', 'deprecated'); + + # stats_block_level and stats_row_level are merged into track_counts + if ($c{'stats_block_level'} || $c{'stats_row_level'}) { + $deprecate->(\%c, 'stats_block_level', 'deprecated in favor of track_counts'); + $deprecate->(\%c, 'stats_row_level', 'deprecated in favor of track_counts'); + $set->('track_counts', (config_bool $c{'stats_block_level'} || config_bool $c{'stats_row_level'}) ? 'on' : 'off'); + } + + # archive_command now has to be enabled explicitly + if ($c{'archive_command'}) { + $set->('archive_mode', 'on'); + } + } + + if ($newversion >= '8.4') { + $deprecate->(\%c, 'max_fsm_pages', 'not needed anymore'); + $deprecate->(\%c, 'max_fsm_relations', 'not needed anymore'); + $deprecate->(\%c, 'krb_server_hostname', 'does not exist anymore'); + $deprecate->(\%c, 'krb_realm', 'does not exist anymore'); + $rename->(\%c, 'explain_pretty_print', 'debug_pretty_print'); + } + + if ($newversion >= '9.0') { + $deprecate->(\%c, 'add_missing_from', 'does not exist anymore'); + $deprecate->(\%c, 'regex_flavor', 'does not exist anymore'); + } + + if ($newversion >= '9.2') { + $deprecate->(\%c, 'wal_sender_delay', 'does not exist anymore'); + $deprecate->(\%c, 'silent_mode', 'does not exist anymore'); + $deprecate->(\%c, 'custom_variable_classes', 'does not exist anymore'); + } + + if ($newversion >= '9.3') { + $rename->(\%c, 'unix_socket_directory', 'unix_socket_directories'); + $rename->(\%c, 'replication_timeout', 'wal_sender_timeout'); + } + + if ($newversion >= '9.4') { + $deprecate->(\%c, 'krb_srvname', 'native krb5 authentication deprecated in favor of GSSAPI'); + # grab dsmt from the new config just written by initdb + if (not $c{dynamic_shared_memory_type} and $configfile eq 'postgresql.conf') { + $set->('dynamic_shared_memory_type', ($newinfo{config}->{dynamic_shared_memory_type} || 'mmap')); + } + } + + if ($newversion >= '9.5') { + if (exists $c{checkpoint_segments}) { + my $max_wal_size = 16*$c{checkpoint_segments} . 'MB'; + $rename->(\%c, 'checkpoint_segments', 'max_wal_size'); + $set->('max_wal_size', $max_wal_size); + } + $deprecate->(\%c, 'ssl_renegotiation_limit', 'does not exist anymore'); + # adapt cluster_name + my $cluster_name = PgCommon::get_conf_value ($newversion, $newcluster, $configfile, 'cluster_name'); + if ($cluster_name) { + $cluster_name =~ s/\Q$version\E/$newversion/g; + $cluster_name =~ s/\Q$cluster\E/$newcluster/g; + $set->('cluster_name', $cluster_name); + } + } + + if ($newversion >= '10') { + $rename->(\%c, 'min_parallel_relation_size', 'min_parallel_table_scan_size'); + $deprecate->(\%c, 'sql_inheritance', 'does not exist anymore'); + } + + if ($newversion >= '11') { + $deprecate->(\%c, 'replacement_sort_tuples', 'does not exist anymore'); + } + + if ($newversion >= '13') { + if (exists $c{wal_keep_segments}) { + my $wal_keep_size = 16*$c{wal_keep_segments} . 'MB'; + $rename->(\%c, 'wal_keep_segments', 'wal_keep_size'); + $set->('wal_keep_size', $wal_keep_size); + } + } + + if ($newversion >= '14') { + $deprecate->(\%c, 'operator_precedence_warning', 'does not exist anymore'); + if ($c{password_encryption} and $c{password_encryption} =~ /^(on|off|true|false)$/) { + $deprecate->(\%c, 'password_encryption', 'password_encryption is not a boolean anymore'); + } + $deprecate->(\%c, 'vacuum_cleanup_index_scale_factor', 'does not exist anymore'); + } + + if ($newversion >= '15') { + $deprecate->(\%c, 'stats_temp_directory', 'does not exist anymore'); + } + + if ($newversion >= '16') { + $deprecate->(\%c, 'promote_trigger_file', 'does not exist anymore, use pg_promote() instead'); + $deprecate->(\%c, 'vacuum_defer_cleanup_age', 'does not exist anymore'); + $deprecate->(\%c, 'force_parallel_mode', 'does not exist anymore'); + } + + if ($newversion >= '17') { + $deprecate->(\%c, 'db_user_namespace', 'does not exist anymore'); + $deprecate->(\%c, 'old_snapshot_threshold', 'does not exist anymore'); + } +} + +sub migrate_config_files() { + # copy configuration files + print "Copying old configuration files...\n"; + install_file $info{'configdir'}.'/postgresql.conf', $newinfo{'configdir'}, + $newinfo{'owneruid'}, $newinfo{'ownergid'}, "644"; + adapt_conffiles $newversion, $newcluster, 'postgresql.conf'; + # copy auto.conf after postgresql.conf has been updated + # (otherwise read_cluster_conf_file would also read auto.conf) + if ($version >= 9.4) { + install_file $info{'pgdata'}.'/postgresql.auto.conf', $newinfo{'pgdata'}, + $newinfo{'owneruid'}, $newinfo{'ownergid'}, "600"; + adapt_conffiles $newversion, $newcluster, 'postgresql.auto.conf'; + } + install_file $info{'configdir'}.'/pg_ident.conf', $newinfo{'configdir'}, + $newinfo{'owneruid'}, $newinfo{'ownergid'}, "640"; + install_file $info{'configdir'}.'/pg_hba.conf', $newinfo{'configdir'}, + $newinfo{'owneruid'}, $newinfo{'ownergid'}, "640"; + if ($version < 8.4 and $newversion >= 8.4) { + print "Removing 'ident sameuser' from pg_hba.conf...\n"; + my $hba = "$PgCommon::confroot/$newversion/$newcluster/pg_hba.conf"; + open O, $hba or error "open $hba: $!"; + open N, ">$hba.new" or error "open $hba.new: $!"; + while (<O>) { + s/ident\s+sameuser/ident/; + print N $_; + } + close O; + close N; + lchown $newinfo{'owneruid'}, $newinfo{'ownergid'}, "$hba.new"; + chmod 0640, "$hba.new"; + rename "$hba.new", $hba or error "rename: $!"; + } + if ( -e $info{'configdir'}.'/start.conf') { + print "Copying old start.conf...\n"; + install_file $info{'configdir'}.'/start.conf', $newinfo{'configdir'}, + $newinfo{'owneruid'}, $newinfo{'ownergid'}, "644"; + } + if ( -e $info{'configdir'}.'/pg_ctl.conf') { + print "Copying old pg_ctl.conf...\n"; + install_file $info{'configdir'}.'/pg_ctl.conf', $newinfo{'configdir'}, + $newinfo{'owneruid'}, $newinfo{'ownergid'}, "644"; + } + + # copy SSL files (overwriting any file that pg_createcluster put there) + for my $file (qw/server.crt server.key root.crt root.crl/) { + if ( -e "$info{'pgdata'}/$file") { + print "Copying old $file...\n"; + if (!fork) { # we don't use install_file because that converts symlinks to files + change_ugid $info{'owneruid'}, $info{'ownergid'}; + system "cp -a $info{'pgdata'}/$file $newinfo{'pgdata'}"; + exit 0; + } + wait; + } + } + if ($newversion >= 9.2) { + # SSL certificate paths have an explicit option now, older versions use + # a symlink + for my $f (['server.crt', 'ssl_cert_file'], + ['server.key', 'ssl_key_file'], + ['root.crt', 'ssl_ca_file'], + ['root.crl', 'ssl_crl_file']) { + my $file = "$newinfo{'pgdata'}/$f->[0]"; + if (-l $file) { # migrate symlink to config entry with link target + PgCommon::set_conf_value $newversion, $newcluster, 'postgresql.conf', + $f->[1], (readlink $file); + unlink $file; + } elsif (-e $file) { # plain file in data dir, put in config + PgCommon::set_conf_value $newversion, $newcluster, 'postgresql.conf', + $f->[1], $file; + } + } + } +} + +# Write temporary pg_hba.conf. +# Arguments: <version> <cluster> <owner> <owneruid> +sub temp_hba_conf { + my ($fh, $hba) = tempfile("pg_hba.XXXXXX", TMPDIR => 1, SUFFIX => ".conf"); + + if ($_[0] >= '8.4') { + print $fh "local all $_[2] ident\n"; + } else { + print $fh "local all $_[2] ident sameuser\n"; + } + close $fh; + chmod 0400, $hba; + lchown $_[3], 0, $hba; + + return $hba; +} + +# Get encoding and locales of a running cluster +# Arguments: <version> <cluster> +sub get_encoding { + my ($version, $cluster) = @_; + $encoding = get_db_encoding $version, $cluster, $maintenance_db; + if ($version <= '8.3') { + ($old_lc_ctype, $old_lc_collate) = get_cluster_locales $version, $cluster; + } else { + ($old_lc_ctype, $old_lc_collate, $old_locale_provider, $old_icu_locale, $old_icu_rules) = get_db_locales $version, $cluster, $maintenance_db; + } + unless ($encoding && $old_lc_ctype && $old_lc_collate) { + error 'could not get cluster locales'; + } +} + +# RedHat's run-parts doesn't support any options, supply a minimalistic implementation here +# BUG: we don't care about validating the filenames yet +# Arguments: <directory> <argv to pass to scripts> +sub run_parts { + my ($dir, @argv) = @_; + for my $script (<$dir/*>) { + my ($s) = $script =~ /(.*)/; # untaint + system ($s, @argv); + error "$s failed: $?" if ($?); + } +} + +sub run_upgrade_scripts($) { + my $phase = shift; + + print "Running $phase phase upgrade hook scripts ...\n"; + if (!fork) { + change_ugid $info{'owneruid'}, $info{'ownergid'}; + + if ($PgCommon::rpm) { + run_parts ("$PgCommon::common_confdir/pg_upgradecluster.d", + $version, $newcluster, $newversion, $phase); + exit; + } + + my @argv = ('run-parts', '--lsbsysinit', '-a', $version, '-a', $newcluster, + '-a', $newversion, '-a', $phase, + "$PgCommon::common_confdir/pg_upgradecluster.d"); + error "$PgCommon::common_confdir/pg_upgradecluster.d script failed" if system @argv; + exit; + } + wait; + if ($? > 0) { + unless ($keep_on_error) { + print STDERR "Error during running upgrade hooks, removing new cluster\n"; + system 'pg_dropcluster', '--stop', $newversion, $newcluster; + } + exit 1; + } +} + +# +# Execution starts here +# + +# command line arguments + +my $newest_version = get_newest_version('postgres'); +$newversion = $newest_version; + +my $method = 'dump'; +my $link = 0; +my $clone = 0; +my $keep_port = 0; +my $start = -1; # -1 = auto + +my ($locale, $lc_collate, $lc_ctype, $lc_messages, $lc_monetary, $lc_numeric, + $lc_time, $logfile, $old_bindir, $jobs); +GetOptions ('v|version=s' => \$newversion, + 'locale=s' => \$locale, + 'lc-collate=s' => \$lc_collate, + 'lc-ctype=s' => \$lc_ctype, + 'lc-messages=s' => \$lc_messages, + 'lc-monetary=s' => \$lc_monetary, + 'lc-numeric=s' => \$lc_numeric, + 'lc-time=s' => \$lc_time, + 'logfile=s' => \$logfile, + 'm|method=s' => \$method, + 'j|jobs=s', => \$jobs, + 'k|link' => \$link, + 'clone' => \$clone, + 'keep-port' => \$keep_port, + 'rename=s' => \$newcluster, + 'old-bindir=s' => \$old_bindir, + 'maintenance-db=s' => \$maintenance_db, + 'start!' => \$start, + 'keep-on-error' => \$keep_on_error, + ) or exit 1; + +if ($method eq 'dump') { + error 'cannot use --link with --method=dump' if ($link); + error 'cannot use --clone with --method=dump' if ($clone); +} elsif ($method eq 'link') { + $method = 'upgrade'; + $link = 1; +} elsif ($method eq 'clone') { + $method = 'upgrade'; + $clone = 1; +} elsif ($method ne 'upgrade') { + error 'method must be "dump", "upgrade", "link", or "clone"'; +} + +# untaint +($newversion) = $newversion =~ /^(\d+\.?\d+)$/; +($locale) = $locale =~ /^([\w@._-]+)$/ if $locale; +($lc_collate) = $lc_collate =~ /^([\w@._-]+)$/ if $lc_collate; +($lc_ctype) = $lc_ctype =~ /^([\w@._-]+)$/ if $lc_ctype; +($lc_messages) = $lc_messages =~ /^([\w@._-]+)$/ if $lc_messages; +($lc_monetary) = $lc_monetary =~ /^([\w@._-]+)$/ if $lc_monetary; +($lc_numeric) = $lc_numeric =~ /^([\w@._-]+)$/ if $lc_numeric; +($lc_time) = $lc_time =~ /^([\w@._-]+)$/ if $lc_time; +($logfile) = $logfile =~ /^([^\n]+)$/ if $logfile; +($old_bindir) = $old_bindir =~ /^(\/.*)$/ if $old_bindir; +($maintenance_db) = $maintenance_db =~ /^([\w-]+)$/ if $maintenance_db; +if ($jobs) { + ($jobs) = $jobs =~ /^(\d+)$/; + $ENV{PGJOBS} = $jobs; # make setting available to upgrade hooks +} + +if ($#ARGV < 1) { + print "Usage: $0 [OPTIONS] <old version> <cluster name> [<new data directory>]\n"; + exit 1; +} + +($version) = $ARGV[0] =~ /^(\d+\.?\d+)$/; +($cluster) = $ARGV[1] =~ /^([-.\w]+)$/; +$newcluster ||= $cluster; # use old cluster name by default +($newcluster) = $newcluster =~ /^([-.\w]+)$/; +my $datadir; +($datadir) = $ARGV[2] =~ /(.*)/ if defined $ARGV[2]; + +error 'specified cluster does not exist' unless cluster_exists $version, $cluster; +%info = cluster_info ($version, $cluster); +validate_cluster_owner \%info; +error 'cluster is disabled' if $info{'start'} eq 'disabled'; + +error "cluster $version/$cluster is already on version $newversion. " . + "(The newest version installed on this system is $newest_version.)" + if ($version eq $newversion and $cluster eq $newcluster); + +if (cluster_exists $newversion, $newcluster) { + error "target cluster $newversion/$newcluster already exists"; +} + +my $oldcontrol = get_cluster_controldata ($version, $cluster); + +my $oldsocket = get_cluster_socketdir $version, $cluster; +my $owner = getpwuid $info{'owneruid'}; +error 'could not get name of cluster owner' unless $owner; +my $temp_hba_conf = temp_hba_conf $version, $cluster, $owner, $info{'owneruid'}; + +# stop old cluster +if ($info{'running'}) { + get_encoding $version, $cluster; + print "Stopping old cluster...\n"; + my @argv = ('pg_ctlcluster', $version, $cluster, 'stop'); + error "Could not stop old cluster" if system @argv; +} + +if ($method eq 'dump' or ($method eq 'upgrade' and not $info{'running'})) { + print "Restarting old cluster with restricted connections...\n"; + my @argv = ('pg_ctlcluster', + ($old_bindir ? ("--bindir=$old_bindir") : ()), + $version, $cluster, 'start', '-o', "-c hba_file=$temp_hba_conf"); + error "Could not restart old cluster" if system @argv; + + get_encoding $version, $cluster unless ($encoding); # if the cluster was not running before, get encoding now + + if ($method eq 'upgrade') { + print "Stopping old cluster...\n"; + @argv = ('pg_ctlcluster', $version, $cluster, 'stop'); + error "Could not stop old cluster" if system @argv; + } +} + +# in dump mode, old cluster is running now +# in upgrade mode, old cluster is stopped + +my $upgrade_port = next_free_port; + +# create new cluster, preserving encoding and locales +my @argv = ('pg_createcluster', '-u', $info{'owneruid'}, '-g', $info{'ownergid'}, + '--socketdir', $info{'socketdir'}, '--port', $upgrade_port, '--no-status', + $newversion, $newcluster); +push @argv, ('--datadir', $datadir) if $datadir; +push @argv, ('--logfile', $logfile) if $logfile; +push @argv, ('--encoding', $encoding) unless $locale or $lc_ctype; +$lc_ctype ||= $locale || $old_lc_ctype; +$lc_collate ||= $locale || $old_lc_collate; +push @argv, ('--locale', $locale) if $locale; +push @argv, ('--lc-collate', $lc_collate) if $lc_collate; +push @argv, ('--lc-ctype', $lc_ctype) if $lc_ctype; +push @argv, ('--lc-messages', $lc_messages) if $lc_messages; +push @argv, ('--lc-monetary', $lc_monetary) if $lc_monetary; +push @argv, ('--lc-numeric', $lc_numeric) if $lc_numeric; +push @argv, ('--lc-time', $lc_time) if $lc_time; +push @argv, ('--'); +push @argv, ('--locale-provider', $old_locale_provider) if $old_locale_provider; +push @argv, ('--icu-locale', $old_icu_locale) if $old_icu_locale; +push @argv, ('--icu-rules', $old_icu_rules) if $old_icu_rules; +push @argv, ('--data-checksums') if $oldcontrol->{'Data page checksum version'}; # 0 = off +push @argv, ('--encryption-key-command', $info{config}->{encryption_key_command}) if $info{config}->{encryption_key_command}; # PostgreSQL TDE + +# call pg_createcluster +delete $ENV{'LC_ALL'}; +error "Could not create target cluster" if system @argv; +print "\n"; + +# migrate config files to new cluster before running upgrade +%newinfo = cluster_info($newversion, $newcluster); +migrate_config_files(); +set_cluster_port $newversion, $newcluster, $upgrade_port; # use free port during upgrade +%newinfo = cluster_info($newversion, $newcluster); # re-read info after migrate_config_files + +if ($method eq 'dump') { + print "Starting new cluster...\n"; + @argv = ('pg_ctlcluster', $newversion, $newcluster, 'start', '-o', "-c hba_file=$temp_hba_conf"); + error "Could not start target cluster" if system @argv; +} + +my $pg_restore = get_program_path 'pg_restore', $newversion; + +# check whether upgrade scripts exist +my $upgrade_scripts = (-d "$PgCommon::common_confdir/pg_upgradecluster.d" && + ($PgCommon::rpm ? `ls $PgCommon::common_confdir/pg_upgradecluster.d` : + `run-parts --test $PgCommon::common_confdir/pg_upgradecluster.d`)); + +# Run upgrade scripts in init phase +run_upgrade_scripts('init') if ($upgrade_scripts); +print "\n"; + +# dump cluster; drop to cluster owner privileges + +if (!fork) { + change_ugid $info{'owneruid'}, $info{'ownergid'}; + my $pg_dumpall = get_program_path 'pg_dumpall', $newversion; + my $pg_dump = get_program_path 'pg_dump', $newversion; + my $psql = get_program_path 'psql'; + my $newsocket = get_cluster_socketdir $newversion, $newcluster; + + if ($method eq 'dump') { + # get list of databases (value = datallowconn) + my %databases; + open F, '-|', $psql, '-h', $oldsocket, '-p', $info{'port'}, + '-F|', '-d', $maintenance_db, '-AXtc', + 'SELECT datname, datallowconn FROM pg_database' or + error 'Could not get pg_database list'; + while (<F>) { + chomp; + my ($n, $a) = split '\|'; + $databases{$n} = ($a eq 't'); + } + close F; + error 'could not get list of databases' if $?; + + # Temporarily enable access to all DBs, so that we can upgrade them + for my $db (keys %databases) { + next if $db eq 'template0'; + + unless ($databases{$db}) { + print "Temporarily enabling access to database $db\n"; + (system $psql, '-h', $oldsocket, '-p', $info{'port'}, '-qX', + '-d', $maintenance_db, '-c', + "BEGIN READ WRITE; UPDATE pg_database SET datallowconn = 't' WHERE datname = '$db'; COMMIT") == 0 or + error 'Could not enable access to database'; + } + } + + # dump schemas + print "Roles, databases, schemas, ACLs...\n"; + open SOURCE, '-|', $pg_dumpall, '-h', $oldsocket, '-p', $info{'port'}, + '-s', '--quote-all-identifiers' or error 'Could not execute pg_dumpall for old cluster'; + my $data = ''; + my $buffer; + while (read SOURCE, $buffer, 1048576) { + $data .= $buffer; + } + close SOURCE; + ($? == 0) or exit 1; + + # remove creation of db superuser role to avoid error message + $data =~ s/^CREATE (ROLE|USER) "\Q$owner\E";\s*$//m; + + # create global objects in target cluster + open SINK, '|-', $psql, '-h', $newsocket, '-p', $newinfo{'port'}, + '-qX', '-d', $maintenance_db or + error 'Could not execute psql for new cluster'; + + # ensure that we can upgrade tables for DBs with default read-only + # transactions + print SINK "BEGIN READ WRITE; ALTER USER $owner SET default_transaction_read_only to off; COMMIT;\n"; + + print SINK $data; + + close SINK; + ($? == 0) or exit 1; + + + # Upgrade databases + for my $db (keys %databases) { + next if $db eq 'template0'; + + print "Fixing hardcoded library paths for stored procedures...\n"; + # starting from 9.0, replace() works on strings; for earlier versions it + # works on bytea + if ($version >= '9.0') { + (system $psql, '-h', $oldsocket, '-p', $info{'port'}, '-qX', '-d', + $db, '-c', "BEGIN READ WRITE; \ + UPDATE pg_proc SET probin = replace(\ + replace(probin, '/usr/lib/postgresql/lib', '\$libdir'), \ + '/usr/lib/postgresql/$version/lib', '\$libdir'); COMMIT") == 0 or + error 'Could not fix library paths'; + } else { + (system $psql, '-h', $oldsocket, '-p', $info{'port'}, '-qX', '-d', + $db, '-c', "BEGIN READ WRITE; \ + UPDATE pg_proc SET probin = decode(replace(\ + replace(encode(probin, 'escape'), '/usr/lib/postgresql/lib', '\$libdir'), \ + '/usr/lib/postgresql/$version/lib', '\$libdir'), 'escape'); COMMIT") == 0 or + error 'Could not fix library paths'; + } + + print 'Upgrading database ', $db, "...\n"; + open SOURCE, '-|', $pg_dump, '-h', $oldsocket, '-p', $info{'port'}, + '-Fc', '--quote-all-identifiers', $db or + error 'Could not execute pg_dump for old cluster'; + + # start pg_restore and copy over everything + my @restore_argv = ($pg_restore, '-h', $newsocket, '-p', + $newinfo{'port'}, '--data-only', '-d', $db, + '--disable-triggers', '--no-data-for-failed-tables'); + open SINK, '|-', @restore_argv or + error 'Could not execute pg_restore for new cluster'; + + my $buffer; + while (read SOURCE, $buffer, 1048576) { + print SINK $buffer; + } + close SOURCE; + ($? == 0) or exit 1; + close SINK; + + # clean up + unless ($databases{$db}) { + print "Disabling access to database $db again\n"; + (system $psql, '-h', $oldsocket, '-p', $info{'port'}, '-qX', + '-d', $maintenance_db, '-c', + "BEGIN READ WRITE; UPDATE pg_database SET datallowconn = 'f' where datname = '$db'; COMMIT") == 0 or + error 'Could not disable access to database in old cluster'; + (system $psql, '-h', $newsocket, '-p', $newinfo{'port'}, '-qX', + '-d', $maintenance_db, '-c', + "BEGIN READ WRITE; UPDATE pg_database SET datallowconn = 'f' where datname = '$db'; COMMIT") == 0 or + error 'Could not disable access to database in new cluster'; + } + } + + # reset owner specific override for default read-only transactions + (system $psql, '-h', $newsocket, '-p', $newinfo{'port'}, '-qX', $maintenance_db, '-c', + "BEGIN READ WRITE; ALTER USER $owner RESET default_transaction_read_only; COMMIT;\n") == 0 or + error 'Could not reset default_transaction_read_only value for superuser'; + } else { + # pg_upgrade + + use File::Temp qw(tempdir); + + my $pg_upgrade = get_program_path 'pg_upgrade', $newversion; + $pg_upgrade or error "pg_upgrade $newversion not found"; + my @argv = ($pg_upgrade, + '-b', ($old_bindir || "$PgCommon::binroot$version/bin"), + '-B', "$PgCommon::binroot$newversion/bin", + '-p', $info{'port'}, + '-P', $newinfo{'port'}, + ); + if ($version <= 9.1) { + push @argv, '-d', $info{pgdata}, '-o', "-D $info{configdir}"; # -o and -D configdir require $newversion >= 9.2 + } else { + push @argv, '-d', $info{configdir}; + } + push @argv, '-D', $newinfo{configdir}; + push @argv, "--link" if $link; + push @argv, "--clone" if $clone; + push @argv, '-j', $jobs if $jobs; + + # Make a directory for pg_upgrade to store its reports and log files. + my $logdir = tempdir("/var/log/postgresql/pg_upgradecluster-$version-$newversion-$newcluster.XXXX"); + chdir $logdir; + + # Run pg_upgrade. + print "@argv\n"; + my $status = system @argv; + + # Remove the PID file of the old cluster (normally removed by + # pg_ctlcluster, but not by pg_upgrade). + unlink "/var/run/postgresql/$version-$cluster.pid"; + + # Move output files to our log directory + if (-d (my $outdir = "$newinfo{pgdata}/pg_upgrade_output.d")) { + system mv => $outdir, $logdir + and error "Could not move $outdir to $logdir"; + } + rmdir $logdir; # remove it if it's empty + print "pg_upgradecluster: pg_upgrade output scripts are in $logdir\n" if (-d $logdir); + + exit 1 if ($status != 0); + } + + exit 0; +} + +wait; + +if ($?) { + print "\n"; + + unless ($keep_on_error) { + print STDERR "Error during cluster dumping, removing new cluster\n"; + system 'pg_dropcluster', '--stop', $newversion, $newcluster; + } + + # Restart old cluster to allow connections again (two steps because we started without systemd) + system 'pg_ctlcluster', $version, $cluster, 'stop'; # ignore errors, it might be down anyway + if ($info{running}) { + print "Starting old cluster again ...\n"; + if (system 'pg_ctlcluster', $version, $cluster, 'start') { + error 'could not start old cluster, please do that manually'; + } + } + exit 1; +} + +if ($method eq 'dump') { + print "Stopping target cluster...\n"; + @argv = ('pg_ctlcluster', $newversion, $newcluster, 'stop'); + error "Could not stop target cluster" if system @argv; + + print "Stopping old cluster...\n"; + @argv = ('pg_ctlcluster', $version, $cluster, 'stop'); + error "Could not stop old cluster" if system @argv; +} + +print "Disabling automatic startup of old cluster...\n"; +my $startconf = $info{'configdir'}.'/start.conf'; +if (open F, ">$startconf") { + print F "# This cluster was upgraded to a newer major version. The old +# cluster has been preserved for backup purposes, but is not started +# automatically. + +manual +"; + close F; +} else { + error "could not create $startconf: $!"; +} + +my $free_port = next_free_port; +unless ($keep_port) { + set_cluster_port $version, $cluster, $upgrade_port; + set_cluster_port $newversion, $newcluster, $info{port}; + $newinfo{port} = $info{port}; +} + +# notify systemd that we modified the old start.conf +if (not exists $ENV{'PG_CLUSTER_CONF_ROOT'} and -d '/run/systemd/system' and $> == 0) { + system 'systemctl daemon-reload'; +} + +# start cluster if it was running before, or upgrade scripts are present +$start = ($info{running} or $upgrade_scripts) if ($start == -1); +if ($start) { + print "Starting upgraded cluster on port $newinfo{port}...\n"; + @argv = ('pg_ctlcluster', $newversion, $newcluster, 'start'); + error "Could not start upgraded cluster; please check configuration and log files" if system @argv; +} + +# Run upgrade scripts in finish phase +if ($upgrade_scripts) { + if ($start) { + run_upgrade_scripts('finish'); + } else { + print "Warning: Skipping upgrade scripts because --no-start was given\n"; + } +} + +print "\nSuccess. Please check that the upgraded cluster works. If it does, +you can remove the old cluster with + pg_dropcluster $version $cluster\n\n"; + +system 'pg_lsclusters', $version, $cluster; +system 'pg_lsclusters', $newversion, $newcluster; + +__END__ + +=head1 NAME + +pg_upgradecluster - upgrade an existing PostgreSQL cluster to a new major version. + +=head1 SYNOPSIS + +B<pg_upgradecluster> [B<-v> I<newversion>] I<oldversion> I<name> [I<newdatadir>] + +=head1 DESCRIPTION + +B<pg_upgradecluster> upgrades an existing PostgreSQL server cluster (i. e. a +collection of databases served by a B<postgres> instance) to a new version +specified by I<newversion> (default: latest available version). The +configuration files of the old version are copied to the new cluster and +adjusted for the new version. The new cluster is set up to use data page +checksums if the old cluster uses them. + +The cluster of the old version will be configured to use a previously unused +port since the upgraded one will use the original port. The old cluster is not +automatically removed. After upgrading, please verify that the new cluster +indeed works as expected; if so, you should remove the old cluster with +L<pg_dropcluster(8)>. Please note that the old cluster is set to "manual" +startup mode, in order to avoid inadvertently changing it; this means that it +will not be started automatically on system boot, and you have to use +L<pg_ctlcluster(8)> to start/stop it. See section "STARTUP CONTROL" in +L<pg_createcluster(8)> for details. + +The I<newdatadir> argument can be used to specify a non-default data directory +of the upgraded cluster. It is passed to B<pg_createcluster>. If not specified, +this defaults to /var/lib/postgresql/I<newversion>/I<name>. + +=head1 OPTIONS + +=over 4 + +=item B<-v> I<newversion> + +Set the version to upgrade to (default: latest available). + +=item B<--logfile> I<filel> + +Set a custom log file path for the upgraded database cluster. + +=item B<--locale=>I<locale> + +Set the default locale for the upgraded database cluster. If this option is not +specified, the locale is inherited from the old cluster. + +When upgrading to PostgreSQL 11 or newer, this option no longer allows +switching the encoding of individual databases. (L<pg_dumpall(1)> was changed to +retain database encodings.) + +=item B<--lc-collate=>I<locale> + +=item B<--lc-ctype=>I<locale> + +=item B<--lc-messages=>I<locale> + +=item B<--lc-monetary=>I<locale> + +=item B<--lc-numeric=>I<locale> + +=item B<--lc-time=>I<locale> + +Like B<--locale>, but only sets the locale in the specified category. + +=item B<-m>, B<--method=>B<dump>|B<upgrade>|B<link>|B<clone> + +Specify the upgrade method. B<dump> uses L<pg_dump(1)> and +L<pg_restore(1)>, B<upgrade> uses L<pg_upgrade(1)>. The default is +B<dump>. + +B<link> and B<clone> are shorthands for B<-m upgrade --link> and B<-m upgrade --clone>, +respectively. + +=item B<-k>, B<--link> + +In pg_upgrade mode, use hard links instead of copying files to the new +cluster. This option is merely passed on to pg_upgrade. See +L<pg_upgrade(1)> for details. + +=item B<--clone> + +In pg_upgrade mode, use efficient file cloning (also known as "reflinks" +on some systems) instead of copying files to the new cluster. This option +is merely passed on to pg_upgrade. See L<pg_upgrade(1)> for details. + +=item B<-j>, B<--jobs> + +In pg_upgrade mode, number of simultaneous processes to use. This +option is passed on to pg_upgrade. See L<pg_upgrade(1)> for details. +It is also used by the B<analyze> upgrade hook (via the B<PGJOBS> environment +variable). + +=item B<--keep-port> + +By default, the old cluster is moved to a new port, and the new cluster is +moved to the original port so clients will see the upgraded cluster. This +option disables that. + +=item B<--rename=>I<new cluster name> + +Use a different name for the upgraded cluster. + +=item B<--old-bindir=>I<directory> + +Passed to B<pg_upgrade>. + +=item B<--maintenance-db=>I<database> + +Database to connect to for maintenance queries. The default is B<template1>. + +=item B<--[no-]start> + +Start the new database cluster after upgrading. The default is to start the new +cluster if the old cluster was running, or if upgrade hook scripts are present. + +=item B<--keep-on-error> + +If upgrading fails, the newly created cluster is removed. This option disables +that. + +=back + +=head1 HOOK SCRIPTS + +Some PostgreSQL extensions like PostGIS need metadata in auxiliary tables which +must not be upgraded from the old version, but rather initialized for the new +version before copying the table data. For this purpose, extensions (as well as +administrators, of course) can drop upgrade hook scripts into +C</etc/postgresql-common/pg_upgradecluster.d/>. Script file names must consist +entirely of upper and lower case letters, digits, underscores, and hyphens; in +particular, dots (i. e. file extensions) are not allowed. + +Scripts in that directory will be called with the following arguments: + +<old version> <cluster name> <new version> <phase> + +Phases: + +=over + +=item B<init> + +A virgin cluster of version I<new version> has been created, i. e. this new +cluster will already have B<template1> and B<postgres>, but no user databases. Please note that +you should not create tables in this phase, since they will be overwritten by +the dump/restore or B<pg_upgrade> operation. + +=item B<finish> + +All data from the old version cluster has been dumped/reloaded into the new +one. The old cluster still exists, but is not running. + +=back + +Failing scripts will abort the upgrade. +The scripts are called as the user who owns the database. + +=head1 SEE ALSO + +L<pg_createcluster(8)>, L<pg_dropcluster(8)>, L<pg_lsclusters(1)>, L<pg_wrapper(1)> + +=head1 AUTHORS + +Martin Pitt L<E<lt>mpitt@debian.orgE<gt>>, Christoph Berg L<E<lt>myon@debian.orgE<gt>> diff --git a/pg_upgradecluster.d/analyze b/pg_upgradecluster.d/analyze new file mode 100755 index 0000000..f410365 --- /dev/null +++ b/pg_upgradecluster.d/analyze @@ -0,0 +1,33 @@ +#!/bin/sh +# +# Run ANALYZE on all databases in the upgraded cluster + +set -eu + +oldversion="$1" +cluster="$2" +newversion="$3" +phase="$4" + +case $newversion in + 9.2|9.3) + analyze="--analyze-only" + ;; + *) + analyze="--analyze-in-stages" + ;; +esac + +case $newversion in + 9.5|9.6|[1-7]*) + [ "${PGJOBS:-}" ] && jobs="--jobs=$PGJOBS" + ;; +esac + +case $phase in + finish) + vacuumdb --cluster "$newversion/$cluster" --all $analyze ${jobs:-} + ;; +esac + +exit 0 diff --git a/pg_virtualenv b/pg_virtualenv new file mode 100755 index 0000000..f4cfe03 --- /dev/null +++ b/pg_virtualenv @@ -0,0 +1,280 @@ +#!/bin/bash + +# Create a throw-away PostgreSQL environment for running regression tests. +# This does not interfere with existing clusters. +# +# (C) 2005-2012 Martin Pitt <mpitt@debian.org> +# (C) 2012-2020 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +set -e # no -u here as that breaks PGCONF_OPTS[@] + +# wrap ourselves in newpid if requested +if [ "$PG_VIRTUALENV_NEWPID" ]; then + unset PG_VIRTUALENV_NEWPID + exec newpid $0 "$@" +fi + +# wrap ourselves in unshare if requested +if [ "$PG_VIRTUALENV_UNSHARE" ]; then + export _PG_VIRTUALENV_UNSHARE="$PG_VIRTUALENV_UNSHARE" + unset PG_VIRTUALENV_UNSHARE + exec unshare $_PG_VIRTUALENV_UNSHARE -- $0 "$@" +fi +if [ "$_PG_VIRTUALENV_UNSHARE" ]; then + unset _PG_VIRTUALENV_UNSHARE + # start localhost interface + if [ -x /bin/ip ]; then + ip link set dev lo up || true + else + ifconfig lo up || true + fi +fi + +disable_fakeroot () +{ + case ${LD_PRELOAD:-} in + *fakeroot*) LD_PRELOAD=$(echo "$LD_PRELOAD" | sed -e 's/[^ ]*fakeroot[^ ]*//g') ;; + esac +} + +help () +{ + echo "pg_virtualenv: Create throw-away PostgreSQL environment for regression tests" + echo "Syntax: $0 [options] [command]" + echo " -a use all installed server versions" + echo " -v 'version ...' list of PostgreSQL versions to run [default: latest]" + echo " -c 'options' extra options to pass to pg_createcluster" + echo " -i 'initdb opts' extra initdb options to pass to pg_createcluster" + echo " -o 'guc=value' postgresql.conf options to pass to pg_createcluster" + echo " -p 'package' set options to find extension files in debian/package/" + echo " -s open a shell when command fails" + echo " -t use a temporary cluster directory even as root" + exit ${1:-0} +} + +# option parsing +PGBINROOT="/usr/lib/postgresql/" +#redhat# PGBINROOT="/usr/pgsql-" +PG_VERSIONS="" +PGCONF_OPTS=() +while getopts "ac:i:ho:p:stv:" opt ; do + case $opt in + a) for d in $PGBINROOT*/bin/pg_ctl; do + # prepend version so latest ends up first (i.e. on port 5432) + dir=${d%%/bin/pg_ctl} + PG_VERSIONS="${dir#$PGBINROOT} ${PG_VERSIONS:-}" + done ;; + c) CREATE_OPTS="$OPTARG" ;; + i) INITDB_OPTS="$OPTARG" ;; + h) help ;; + o) PGCONF_OPTS+=("--pgoption" "$OPTARG") ;; + p) PACKAGE="$OPTARG" ;; + s) run_shell=1 ;; + t) NONROOT=1 ;; + v) PG_VERSIONS="$OPTARG" ;; + *) help 1 ;; + esac +done +if [ -z "$PG_VERSIONS" ]; then + # use latest version + d=$(ls -v $PGBINROOT*/bin/pg_ctl 2> /dev/null | tail -1) + if [ -z "$d" ]; then + echo "Could not determine PostgreSQL version, are any PostgreSQL server packages installed?" >&2 + exit 2 + fi + dir=${d%%/bin/pg_ctl} + PG_VERSIONS="${dir#$PGBINROOT}" +fi +# shift away args +shift $(($OPTIND - 1)) +# if no command is given, open a shell +[ "${1:-}" ] || set -- ${SHELL:-/bin/sh} + +# generate a password +if [ -x /usr/bin/pwgen ]; then + export PGPASSWORD=$(pwgen 20 1) +else + export PGPASSWORD=$(dd if=/dev/urandom bs=1k count=1 2>/dev/null | md5sum - | awk '{ print $1 }') +fi + +# we are not root +if [ "$(id -u)" != 0 ]; then + NONROOT=1 +fi +# we aren't really root in fakeroot (but leave it enabled for commands spawned) +case ${LD_PRELOAD:-} in + *fakeroot*) NONROOT=1 ;; +esac + +# non-root operation: create a temp dir where we store everything +if [ "${NONROOT:-}" ]; then + WORKDIR=$(mktemp -d -t pg_virtualenv.XXXXXX) + if [ $(id -u) = 0 ]; then + chown postgres:postgres $WORKDIR + umask 022 + fi + export PG_CLUSTER_CONF_ROOT="$WORKDIR/postgresql" + export PGUSER="${USER:-${LOGNAME:-$(id -un)}}" + [ "$PGUSER" = "root" ] && PGUSER="postgres" + PGSYSCONFDIR="$WORKDIR/postgresql-common" # no export yet so pg_createcluster uses the original createcluster.conf + mkdir "$PGSYSCONFDIR" "$WORKDIR/log" + PWFILE="$PGSYSCONFDIR/pwfile" + LOGDIR="$WORKDIR/log" + echo "$PGPASSWORD" > "$PWFILE" + + cleanup () { + set +e + disable_fakeroot + for v in $PG_VERSIONS; do + # don't drop existing clusters named "regress" + [ -f $PG_CLUSTER_CONF_ROOT/$v/regress/.by_pg_virtualenv ] || continue + echo "Dropping cluster $v/regress ..." + pg_ctlcluster --mode immediate $v regress stop + pg_dropcluster $v regress + done + rm -rf $WORKDIR + } + trap cleanup 0 HUP INT QUIT ILL ABRT PIPE TERM + +# root: keep everything in the standard locations +else + for v in $PG_VERSIONS; do + if [ -d /etc/postgresql/$v/regress ]; then + echo "Cluster $v/regress exists, refusing to overwrite" >&2 + exit 2 + fi + done + + : ${PGSYSCONFDIR:=/etc/postgresql-common} + pg_service="$PGSYSCONFDIR/pg_service.conf" + + export PGUSER="postgres" + PWFILE=$(mktemp -t pgpassword.XXXXXX) + echo "$PGPASSWORD" > "$PWFILE" # write password before chowning the file + chown postgres:postgres "$PWFILE" + + cleanup () { + set +e + rm -f $PWFILE $pg_service + if [ -f $pg_service.pg_virtualenv-save.$$ ]; then + mv -f $pg_service.pg_virtualenv-save.$$ $pg_service + fi + for v in $PG_VERSIONS; do + # don't drop existing clusters named "regress" + [ -f /etc/postgresql/$v/regress/.by_pg_virtualenv ] || continue + echo "Dropping cluster $v/regress ..." + rm -f /etc/postgresql/$v/regress/.by_pg_virtualenv + pg_ctlcluster --mode immediate $v regress stop + pg_dropcluster $v regress + done + } + trap cleanup 0 HUP INT QUIT ILL ABRT PIPE TERM + + if [ -f $pg_service ]; then + mv --no-clobber $pg_service $pg_service.pg_virtualenv-save.$$ + fi +fi + +# create postgres environments +for v in $PG_VERSIONS; do + # create temporary cluster + if [ "${PACKAGE:-}" ]; then + if ! grep -q 'extension_destdir' /usr/share/postgresql/$v/postgresql.conf.sample; then + echo "$0: This PostgreSQL $v installation does not support 'extension_destdir', skipping this version" + [ "$PG_VERSIONS" = "$v" ] && exit 0 # only one version requested + continue + fi + PKGARGS="--pgoption extension_destdir=$PWD/debian/$PACKAGE --pgoption dynamic_library_path=$PWD/debian/$PACKAGE/usr/lib/postgresql/$v/lib:/usr/lib/postgresql/$v/lib" + fi + # we chdir to / so programs don't throw "could not change directory to ..." + ( + cd / + case $v in + 8*|9.0|9.1|9.2) : ;; + *) NOSYNC="--nosync" ;; + esac + if [ "${NONROOT:-}" ]; then + # stats_temp_directory in /var/run/postgresql needs root (or postgres), disable it + case $v in + 8.4|9.*|1[01234]) PGCONF_OPTS+=("--pgoption" "stats_temp_directory=") ;; + esac + fi + # disable fakeroot here because we really can't run as root + # (and still switch to postgres when using fakeroot as root user) + disable_fakeroot + pg_createcluster --quiet \ + ${PGPORT:+-p "$PGPORT"} \ + ${NONROOT:+-d "$WORKDIR/data/$v/regress"} \ + ${NONROOT:+-l "$WORKDIR/log/postgresql-$v-regress.log"} \ + ${CREATE_OPTS:-} --pgoption fsync=off ${PKGARGS:-} "${PGCONF_OPTS[@]}" \ + $v regress -- \ + --username="$PGUSER" --pwfile="$PWFILE" $NOSYNC ${INITDB_OPTS:-} + # in fakeroot, the username will likely default to "postgres" otherwise + echo "This is a temporary throw-away cluster" > ${PG_CLUSTER_CONF_ROOT:-/etc/postgresql}/$v/regress/.by_pg_virtualenv + # start cluster with coredumps enabled + [ $v != 8.2 ] && ENABLE_COREDUMPS="-c" + pg_ctlcluster $v regress start -- ${ENABLE_COREDUMPS:-} + ) + port=$(pg_conftool -s $v regress show port) + + # record cluster information in service file + cat >> $PGSYSCONFDIR/pg_service.conf <<EOF +[$v] +host=localhost +port=$port +dbname=postgres +user=$PGUSER +password=$PGPASSWORD + +EOF +done + +export PGSYSCONFDIR +export PGHOST="localhost" +export PGDATABASE="postgres" +unset PGHOSTADDR PGPORT PGSERVICE # unset variables that might interfere +case $PG_VERSIONS in + *\ *) ;; # multiple versions: do not set PGPORT because that breaks --cluster + *) + export PGPORT="$port" + export PG_CONFIG="$PGBINROOT$PG_VERSIONS/bin/pg_config" + export PGVERSION="$PG_VERSIONS" + ;; +esac + +# run program +"$@" || EXIT="$?" +if [ ${EXIT:-0} -gt 0 ]; then + for log in ${LOGDIR:-/var/log/postgresql}/*.log; do + echo "*** $log (last 100 lines) ***" + tail -100 $log + done + + if command -v gdb > /dev/null; then + for v in $PG_VERSIONS; do + PGDATA=$(pg_conftool -s $v regress show data_directory) + for core in $PGDATA/core*; do + [ -f "$core" ] || continue + echo "*** $core ***" + gdb -batch -ex "bt full" /usr/lib/postgresql/$v/bin/postgres "$core" + done + done + fi + + if [ "${run_shell:-}" ]; then + echo "pg_virtualenv: command exited with status $EXIT, dropping you into a shell" + ${SHELL:-/bin/sh} + fi +fi + +exit ${EXIT:-0} diff --git a/pg_virtualenv.pod b/pg_virtualenv.pod new file mode 100644 index 0000000..e1e05f0 --- /dev/null +++ b/pg_virtualenv.pod @@ -0,0 +1,122 @@ +=head1 NAME + +pg_virtualenv - Create a throw-away PostgreSQL environment for running regression tests + +=head1 SYNOPSIS + +B<pg_virtualenv> [I<OPTIONS>] [B<-v> 'I<version ...>'] [I<command>] + +=head1 DESCRIPTION + +B<pg_virtualenv> creates a virtual PostgreSQL server environment, and sets +environment variables such that I<command> can access the PostgreSQL database +server(s). The servers are destroyed when I<command> exits. + +The environment variables B<PGHOST>, B<PGDATABASE>, B<PGUSER>, and +B<PGPASSWORD> will be set. Per default, a single new cluster is created, +using the newest PostgreSQL server version installed. The cluster will use the +first available port number starting from B<5432>, and B<PGPORT> will be set. +B<PGVERSION> is set the the PostgreSQL major version number. + +When clusters for more than one versions are created, they will differ in the +port number used, and B<PGPORT> and B<PGVERSION> are not set. The clusters are +named I<version>/regress. To access a cluster, set +B<PGCLUSTER=>I<version>B</regress>. For ease of access, the clusters are also +registered in F</etc/postgresql-common/pg_service.conf>, with the version +number as cluster name. Clusters can be accessed by passing the connection +string "B<service=>I<version>", e.g. B<psql service=9.2>. + +When invoked as root, the clusters are created in F</etc/postgresql/> as usual; +for other users, B<PG_CLUSTER_CONF_ROOT> and B<PGSYSCONFDIR> are +set to a temporary directory where all files belonging to the clusters are +created. + +If I<command> fails, the tail of the PostgreSQL server log is shown. +Additionally, if B<gdb> is available, the backtrace from any PostgreSQL +coredump is show. + +=head1 OPTIONS + +=over 4 + +=item B<-a> + +Use all PostgreSQL server versions installed. + +=item B<-v> I<version ...> + +Use these versions (space-separated list). + +=item B<-c> I<pg_createcluster options> + +Extra options to pass to B<pg_createcluster>. + +=item B<-i> I<initdb options> + +Extra initdb options to pass to B<pg_createcluster>. + +=item B<-o> I<guc>B<=>I<value> + +Configuration option to set in the C<postgresql.conf> file, passed to +B<pg_createcluster>. + +=item B<-p> I<package> + +Set B<extension_destdir> and B<dynamic_library_path> in cluster to enable +loading and testing extensions at build-time from B<debian/>I<package>B</>. + +This is a Debian-specific PostgreSQL patch. + +=item B<-s> + +Launch a shell inside the virtual environment when I<command> fails. + +=item B<-t> + +Install clusters in a temporary directory, even when running as root. + +=item B<-h> + +Show program help. + +=back + +=head1 EXAMPLE + + # pg_virtualenv make check + +=head1 NOTES + +When run with fakeroot(1), B<pg_virtualenv> will fall back to the non-root mode +of operation. Running "fakeroot pg_virtualenv" as root will fail, though. + +=head1 ENVIRONMENT + +=over 4 + +=item B<PG_VIRTUALENV_NEWPID>=yes + +When non-empty, B<pg_virtualenv> will re-exec itself using newpid(1). + +=item B<PG_VIRTUALENV_UNSHARE>=I<flags> + +When non-empty, B<pg_virtualenv> will re-exec itself using unshare(1) using +these flags. + +=item B<PGPORT>=I<n> + +When set, the value is used for the (single) cluster created. + +=back + +=head1 COMPATIBILITY + +B<PGVERSION> is set in postgresql-common (>= 219~). + +=head1 SEE ALSO + +initdb(1), pg_createcluster(1). + +=head1 AUTHOR + +Christoph Berg L<E<lt>myon@debian.orgE<gt>> diff --git a/pg_wrapper b/pg_wrapper new file mode 100755 index 0000000..5b741c5 --- /dev/null +++ b/pg_wrapper @@ -0,0 +1,329 @@ +#!/usr/bin/perl +# Call a PostgreSQL client program with the version, cluster and default +# database specified in ~/.postgresqlrc or +# /etc/postgresql-common/user_clusters. +# +# (C) 2005-2009 Martin Pitt <mpitt@debian.org> +# (C) 2013-2022 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use warnings; +use strict; +use POSIX; +use PgCommon; + +my ($version, $cluster); +my $explicit_host = exists $ENV{PGHOST}; +my $explicit_port = $ENV{PGPORT}; +my $explicit_service = exists $ENV{PGSERVICE}; + +# Evaluate PGCLUSTER (unless PGHOST is set as well) +if (exists $ENV{'PGCLUSTER'} and not $explicit_host) { + ($version, $cluster) = split ('/', $ENV{'PGCLUSTER'}, 2); + error "Invalid version $version specified in PGCLUSTER" unless version_exists $version; + error 'No cluster specified with $PGCLUSTER' unless $cluster; +} + +# Check for --cluster argument and filter it out +for (my $i = 0; $i <= $#ARGV; ++$i) { + last if $ARGV[$i] eq '--'; + + if ($ARGV[$i] eq '--cluster') { + error '--cluster option needs an argument (<version>/<cluster>)' if ($i >= $#ARGV); + + ($version, $cluster) = split ('/', $ARGV[$i+1], 2); + error "Invalid version $version specified with --cluster" unless version_exists $version; + error 'No cluster specified with --cluster' unless $cluster; + + splice @ARGV, $i, 2; + last; + } elsif ($ARGV[$i] =~ /^--cluster=(\d+\.?\d)\/(.+)/) { + ($version, $cluster) = ($1, $2); + error "Invalid version $version specified with --cluster" unless version_exists $version; + error 'No cluster specified with --cluster' unless $cluster; + + splice @ARGV, $i, 1; + last; + } + # --host or -h on command line, drop info from PGCLUSTER + if ($ARGV[$i] =~ /^--host\b/ || $ARGV[$i] =~ /^-\w*h\w*$/) { + ($version, $cluster) = (undef, undef); + $explicit_host = 1; + delete $ENV{PGCLUSTER}; + } + # --port or -p on command line + if ($ARGV[$i] =~ /^--port\b(?:=(\d+))?/ || $ARGV[$i] =~ /^-\w*p(\d+)?$/) { + if (defined $1) { + $explicit_port = $1; + } elsif ($i < $#ARGV) { + $explicit_port = $ARGV[$i+1]; + } + } + # "service=" in connection string + if ($ARGV[$i] =~ /\bservice=/) { + $explicit_service = 1; + } +} + +# if only a port is specified, look for local cluster on specified port +if ($explicit_port and not $version and not $cluster and not $explicit_host and not $explicit_service) { + LOOP: foreach my $v (reverse get_versions()) { + foreach my $c (get_version_clusters $v) { + my $p = get_cluster_port $v, $c; + if ($p eq $explicit_port) { + $version = $v; + # set PGCLUSTER variable for information + $ENV{PGCLUSTER} = "$version/$c"; + last LOOP; + } + } + } +} + +# if we don't have a cluster, and no specific host or port was given, consult postgresqlrc +# or fall back to default port cluster (on 5432), or undef otherwise +my ($db); +($version, $cluster, $db) = user_cluster_map() unless ($cluster or $explicit_host or $explicit_port); + +my ($host, $port); + +if ($cluster) { + # check if we have a network cluster (N.N/the.host.name:port) + if ($cluster =~ /^(\S+):(\d*)$/) { + $host = $1; + $port = $2 || $PgCommon::defaultport; + } elsif (not cluster_exists($version, $cluster)) { + # a specific cluster was requested, error out because it doesn't exist + error "Cluster $version $cluster does not exist"; + } else { + $host = get_cluster_socketdir ($version, $cluster); + $port = get_cluster_port($version, $cluster); + } + # set PGCLUSTER variable for information + $ENV{PGCLUSTER} = "$version/$cluster"; +} + +# setup environment +$ENV{'PGSYSCONFDIR'} //= '/etc/postgresql-common'; +$ENV{'PGHOST'} = $host if ($host); +$ENV{'PGPORT'} = $port if $port && !$ENV{'PGPORT'}; +$ENV{'PGDATABASE'} = $db if $db && !$ENV{'PGDATABASE'}; + +# check under which name we were called +my $cmdname = (split '/', $0)[-1]; + +unless ($version or $explicit_host or $explicit_port or $explicit_service) { + print STDERR "Warning: No existing cluster is suitable as a default target. Please see man pg_wrapper(1) how to specify one.\n"; +} + +# if we have no version yet, use the latest version. If we were called as psql, +# pg_archivecleanup, or pg_isready, always use latest version +if (not $version or $cmdname =~ /^(psql|pg_archivecleanup|pg_isready)$/) { + my $max_version; + if ($version and $version < 9.2) { # psql 15 only supports PG 9.2+ + $max_version = 14; + } + $version = get_newest_version($cmdname, $max_version); +} +unless ($version) { + error 'You must install at least one postgresql-client-<version> package'; +} +error "PostgreSQL version $version is not installed" unless -d "$PgCommon::binroot$version"; + +my $cmd; +if ($cmdname eq 'pg_wrapper') { + error "pg_wrapper called directly but no program given as argument" + if (@ARGV == 0); + $cmd = shift; # will be unshifted back below +} else { + $cmd = get_program_path ($cmdname, $version); +} + +# libreadline is a lot better than libedit, so prefer that on versions that still use it +if ($cmdname eq 'psql' and $version < 13 and not $PgCommon::rpm) { + my @readlines; + # non-multiarch path + @readlines = sort(</lib/libreadline.so.?>); + + unless (@readlines) { + # get multiarch dir for our architecture + if (open PS, '-|', '/usr/bin/ldd', $cmd) { + my $out; + read PS, $out, 10000; + close PS; + if ($out =~ m!/libreadline.so!) { + # already linked against libreadline + @readlines = (); + } + else + { + my ($lib_path) = $out =~ m!(/lib/.*)/libedit.so!; + + @readlines = sort(<$lib_path/libreadline.so.?>); + } + } + } + + if (@readlines) { + $ENV{'LD_PRELOAD'} = ($ENV{'LD_PRELOAD'} or '') . ':' . $readlines[-1]; + } +} + +error "pg_wrapper: $cmdname was not found in $PgCommon::binroot$version/bin" unless $cmd; +unshift @ARGV, $cmd; +exec @ARGV; + +__END__ + +=head1 NAME + +pg_wrapper - wrapper for PostgreSQL client commands + +=head1 SYNOPSIS + +I<client-program> [B<--cluster> I<version>/I<cluster>] [...] + +(I<client-program>: B<psql>, B<createdb>, B<dropuser>, and all other client +programs installed in C</usr/lib/postgresql/>I<version>C</bin>). + +=head1 DESCRIPTION + +This program is run only as a link to names which correspond to PostgreSQL +programs in C</usr/lib/postgresql/>I<version>C</bin>. It determines the +configured cluster and database for the user and calls the appropriate version +of the desired program to connect to that cluster and database, supplying any +specified options to that command. + +The target cluster is selected by the following means, in descending order of +precedence: + +=over + +=item + +explicit specification with the B<--host> option + +=item + +explicit specification with the B<--cluster> option + +=item + +if the B<PGHOST> environment variable is set, no further cluster selection is +performed. The default PostgreSQL version and port number (from the command +line, the environment variable B<PGPORT>, or default 5432) will be used. + +=item + +explicit specification with the B<PGCLUSTER> environment variable + +=item + +if a port is given (either via B<-p>, B<--port>, or B<PGPORT>), and no host is +given, the local cluster matching that port number is used + +=item + +matching entry in C<~/.postgresqlrc> (see L<postgresqlrc(5)>), if that +file exists + +=item + +matching entry in C</etc/postgresql-common/user_clusters> (see +L<user_clusters(5)>), if that file exists + +=item + +If only one cluster exists on the local system, that one will be selected. + +=item + +If several clusters exist on the local system, the one listening on the default port 5432 +will be selected. + +=back + +If none of these rules match, B<pg_wrapper> does not set any environment +variables and the program called will likely error out with a message like +"could not connect to server: Connection refused". + +For B<psql>, B<pg_archivecleanup>, and B<pg_isready>, B<pg_wrapper> will always use the binary from +the newest PostgreSQL version installed, as these are downwards compatible. +If the cluster version is older than 9.2, the newest considered binary version is 14. + +Note that B<pg_wrapper> needs to be able to read the server config to get the +port number to connect to. If a non-standard port is configured in a place that +pg_wrapper cannot read, connecting will fail. This particularly holds if the +port was configured via B<ALTER SYSTEM> in C<postgresql.auto.conf> and +pg_wrapper is invoked as any user other than B<postgres> and B<root>. + +=head1 OPTIONS + +=over + +=item B<--cluster> I<version>B</>I<cluster> + +=item B<--cluster> I<version>B</>I<host>B<:>[I<port>] + +I<cluster> is either the name of a cluster on the local system, or takes the form +I<host>:I<port> for a remote cluster. If I<port> is left empty (i. e. you just +specify I<host:>), it defaults to 5432. + +=back + +=head1 ENVIRONMENT + +=over + +=item B<PGCLUSTER> + +If C<$PGCLUSTER> is set, its value (of the form I<version>/I<cluster>) +specifies the desired cluster, similar to the B<--cluster> option. However, if +B<--cluster> is specified, it overrides the value of C<$PGCLUSTER>. + +=item B<PG_CLUSTER_CONF_ROOT> + +This specifies an alternative base directory for cluster configurations. This +is usually C</etc/postgresql/>, but for testing/development purposes you can +change this to point to e. g. your home directory, so that you can use the +postgresql-common tools without root privileges. + +=item B<PGSYSCONFDIR> + +This is the location of PostgreSQL's and postgresql-common's global +configuration (e. g. C<pg_service.conf>, L<user_clusters(5)>). The default is +C</etc/postgresql-common/>. + +=back + +=head1 FILES + +=over + +=item C</etc/postgresql-common/user_clusters> + +stores the default cluster and database for users and groups as set by +the administrators. + +=item C<$HOME/.postgresqlrc> + +stores defaults set by the user himself. + +=back + +=head1 SEE ALSO + +L<user_clusters(5)>, L<postgresqlrc(5)> + +=head1 AUTHOR + +Martin Pitt L<E<lt>mpitt@debian.orgE<gt>> diff --git a/pgdg/Makefile b/pgdg/Makefile new file mode 100644 index 0000000..1dc7152 --- /dev/null +++ b/pgdg/Makefile @@ -0,0 +1,2 @@ +all: + if [ -f $(HOME)/apt.postgresql.org/pgapt.conf ]; then ./update; fi diff --git a/pgdg/apt.postgresql.org.asc b/pgdg/apt.postgresql.org.asc new file mode 100644 index 0000000..8480576 --- /dev/null +++ b/pgdg/apt.postgresql.org.asc @@ -0,0 +1,77 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBE6XR8IBEACVdDKT2HEH1IyHzXkb4nIWAY7echjRxo7MTcj4vbXAyBKOfjja +UrBEJWHN6fjKJXOYWXHLIYg0hOGeW9qcSiaa1/rYIbOzjfGfhE4x0Y+NJHS1db0V +G6GUj3qXaeyqIJGS2z7m0Thy4Lgr/LpZlZ78Nf1fliSzBlMo1sV7PpP/7zUO+aA4 +bKa8Rio3weMXQOZgclzgeSdqtwKnyKTQdXY5MkH1QXyFIk1nTfWwyqpJjHlgtwMi +c2cxjqG5nnV9rIYlTTjYG6RBglq0SmzF/raBnF4Lwjxq4qRqvRllBXdFu5+2pMfC +IZ10HPRdqDCTN60DUix+BTzBUT30NzaLhZbOMT5RvQtvTVgWpeIn20i2NrPWNCUh +hj490dKDLpK/v+A5/i8zPvN4c6MkDHi1FZfaoz3863dylUBR3Ip26oM0hHXf4/2U +A/oA4pCl2W0hc4aNtozjKHkVjRx5Q8/hVYu+39csFWxo6YSB/KgIEw+0W8DiTII3 +RQj/OlD68ZDmGLyQPiJvaEtY9fDrcSpI0Esm0i4sjkNbuuh0Cvwwwqo5EF1zfkVj +Tqz2REYQGMJGc5LUbIpk5sMHo1HWV038TWxlDRwtOdzw08zQA6BeWe9FOokRPeR2 +AqhyaJJwOZJodKZ76S+LDwFkTLzEKnYPCzkoRwLrEdNt1M7wQBThnC5z6wARAQAB +tBxQb3N0Z3JlU1FMIERlYmlhbiBSZXBvc2l0b3J5iQJOBBMBCAA4AhsDBQsJCAcD +BRUKCQgLBRYCAwEAAh4BAheAFiEEuXsK/KoaR/BE8kSgf8x9RqzMTPgFAlhtCD8A +CgkQf8x9RqzMTPgECxAAk8uL+dwveTv6eH21tIHcltt8U3Ofajdo+D/ayO53LiYO +xi27kdHD0zvFMUWXLGxQtWyeqqDRvDagfWglHucIcaLxoxNwL8+e+9hVFIEskQAY +kVToBCKMXTQDLarz8/J030Pmcv3ihbwB+jhnykMuyyNmht4kq0CNgnlcMCdVz0d3 +z/09puryIHJrD+A8y3TD4RM74snQuwc9u5bsckvRtRJKbP3GX5JaFZAqUyZNRJRJ +Tn2OQRBhCpxhlZ2afkAPFIq2aVnEt/Ie6tmeRCzsW3lOxEH2K7MQSfSu/kRz7ELf +Cz3NJHj7rMzC+76Rhsas60t9CjmvMuGONEpctijDWONLCuch3Pdj6XpC+MVxpgBy +2VUdkunb48YhXNW0jgFGM/BFRj+dMQOUbY8PjJjsmVV0joDruWATQG/M4C7O8iU0 +B7o6yVv4m8LDEN9CiR6r7H17m4xZseT3f+0QpMe7iQjz6XxTUFRQxXqzmNnloA1T +7VjwPqIIzkj/u0V8nICG/ktLzp1OsCFatWXh7LbU+hwYl6gsFH/mFDqVxJ3+DKQi +vyf1NatzEwl62foVjGUSpvh3ymtmtUQ4JUkNDsXiRBWczaiGSuzD9Qi0ONdkAX3b +ewqmN4TfE+XIpCPxxHXwGq9Rv1IFjOdCX0iG436GHyTLC1tTUIKF5xV4Y0+cXIOI +RgQQEQgABgUCTpdI7gAKCRDFr3dKWFELWqaPAKD1TtT5c3sZz92Fj97KYmqbNQZP ++ACfSC6+hfvlj4GxmUjp1aepoVTo3weJAhwEEAEIAAYFAk6XSQsACgkQTFprqxLS +p64F8Q//cCcutwrH50UoRFejg0EIZav6LUKejC6kpLeubbEtuaIH3r2zMblPGc4i ++eMQKo/PqyQrceRXeNNlqO6/exHozYi2meudxa6IudhwJIOn1MQykJbNMSC2sGUp +1W5M1N5EYgt4hy+qhlfnD66LR4G+9t5FscTJSy84SdiOuqgCOpQmPkVRm1HX5X1+ +dmnzMOCk5LHHQuiacV0qeGO7JcBCVEIDr+uhU1H2u5GPFNHm5u15n25tOxVivb94 +xg6NDjouECBH7cCVuW79YcExH/0X3/9G45rjdHlKPH1OIUJiiX47OTxdG3dAbB4Q +fnViRJhjehFscFvYWSqXo3pgWqUsEvv9qJac2ZEMSz9x2mj0ekWxuM6/hGWxJdB+ ++985rIelPmc7VRAXOjIxWknrXnPCZAMlPlDLu6+vZ5BhFX0Be3y38f7GNCxFkJzl +hWZ4Cj3WojMj+0DaC1eKTj3rJ7OJlt9S9xnO7OOPEUTGyzgNIDAyCiu8F4huLPaT +ape6RupxOMHZeoCVlqx3ouWctelB2oNXcxxiQ/8y+21aHfD4n/CiIFwDvIQjl7dg +mT3u5Lr6yxuosR3QJx1P6rP5ZrDTP9khT30t+HZCbvs5Pq+v/9m6XDmi+NlU7Zuh +Ehy97tL3uBDgoL4b/5BpFL5U9nruPlQzGq1P9jj40dxAaDAX/WKJAj0EEwEIACcC +GwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AFAlB5KywFCQPDFt8ACgkQf8x9RqzM +TPhuCQ//QAjRSAOCQ02qmUAikT+mTB6baOAakkYq6uHbEO7qPZkv4E/M+HPIJ4wd +nBNeSQjfvdNcZBA/x0hr5EMcBneKKPDj4hJ0panOIRQmNSTThQw9OU351gm3YQct +AMPRUu1fTJAL/AuZUQf9ESmhyVtWNlH/56HBfYjE4iVeaRkkNLJyX3vkWdJSMwC/ +LO3Lw/0M3R8itDsm74F8w4xOdSQ52nSRFRh7PunFtREl+QzQ3EA/WB4AIj3VohIG +kWDfPFCzV3cyZQiEnjAe9gG5pHsXHUWQsDFZ12t784JgkGyO5wT26pzTiuApWM3k +/9V+o3HJSgH5hn7wuTi3TelEFwP1fNzI5iUUtZdtxbFOfWMnZAypEhaLmXNkg4zD +kH44r0ss9fR0DAgUav1a25UnbOn4PgIEQy2fgHKHwRpCy20d6oCSlmgyWsR40EPP +YvtGq49A2aK6ibXmdvvFT+Ts8Z+q2SkFpoYFX20mR2nsF0fbt1lfH65P64dukxeR +GteWIeNakDD40bAAOH8+OaoTGVBJ2ACJfLVNM53PEoftavAwUYMrR910qvwYfd/4 +6rh46g1Frr9SFMKYE9uvIJIgDsQB3QBp71houU4H55M5GD8XURYs+bfiQpJG1p7e +B8e5jZx1SagNWc4XwL2FzQ9svrkbg1Y+359buUiP7T6QXX2zY++JAj0EEwEIACcC +GwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AFAlEqbZUFCQg2wEEACgkQf8x9RqzM +TPhFMQ//WxAfKMdpSIA9oIC/yPD/dJpY/+DyouOljpE6MucMy/ArBECjFTBwi/j9 +NYM4ynAk34IkhuNexc1i9/05f5RM6+riLCLgAOsADDbHD4miZzoSxiVr6GQ3YXMb +OGld9kV9Sy6mGNjcUov7iFcf5Hy5w3AjPfKuR9zXswyfzIU1YXObiiZT38l55pp/ +BSgvGVQsvbNjsff5CbEKXS7q3xW+WzN0QWF6YsfNVhFjRGj8hKtHvwKcA02wwjLe +LXVTm6915ZUKhZXUFc0vM4Pj4EgNswH8Ojw9AJaKWJIZmLyW+aP+wpu6YwVCicxB +Y59CzBO2pPJDfKFQzUtrErk9irXeuCCLesDyirxJhv8o0JAvmnMAKOLhNFUrSQ2m ++3EnF7zhfz70gHW+EG8X8mL/EN3/dUM09j6TVrjtw43RLxBzwMDeariFF9yC+5bL +tnGgxjsB9Ik6GV5v34/NEEGf1qBiAzFmDVFRZlrNDkq6gmpvGnA5hUWNr+y0i01L +jGyaLSWHYjgw2UEQOqcUtTFK9MNzbZze4mVaHMEz9/aMfX25R6qbiNqCChveIm8m +Yr5Ds2zdZx+G5bAKdzX7nx2IUAxFQJEE94VLSp3npAaTWv3sHr7dR8tSyUJ9poDw +gw4W9BIcnAM7zvFYbLF5FNggg/26njHCCN70sHt8zGxKQINMc6SJAj0EEwEIACcC +GwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AFAlLpFRkFCQ6EJy0ACgkQf8x9RqzM +TPjOZA//Zp0e25pcvle7cLc0YuFr9pBv2JIkLzPm83nkcwKmxaWayUIG4Sv6pH6h +m8+S/CHQij/yFCX+o3ngMw2J9HBUvafZ4bnbI0RGJ70GsAwraQ0VlkIfg7GUw3Tz +voGYO42rZTru9S0K/6nFP6D1HUu+U+AsJONLeb6oypQgInfXQExPZyliUnHdipei +4WR1YFW6sjSkZT/5C3J1wkAvPl5lvOVthI9Zs6bZlJLZwusKxU0UM4Btgu1Sf3nn +JcHmzisixwS9PMHE+AgPWIGSec/N27a0KmTTvImV6K6nEjXJey0K2+EYJuIBsYUN +orOGBwDFIhfRk9qGlpgt0KRyguV+AP5qvgry95IrYtrOuE7307SidEbSnvO5ezNe +mE7gT9Z1tM7IMPfmoKph4BfpNoH7aXiQh1Wo+ChdP92hZUtQrY2Nm13cmkxYjQ4Z +gMWfYMC+DA/GooSgZM5i6hYqyyfAuUD9kwRN6BqTbuAUAp+hCWYeN4D88sLYpFh3 +paDYNKJ+Gf7Yyi6gThcV956RUFDH3ys5Dk0vDL9NiWwdebWfRFbzoRM3dyGP889a +OyLzS3mh6nHzZrNGhW73kslSQek8tjKrB+56hXOnb4HaElTZGDvD5wmrrhN94kby +Gtz3cydIohvNO9d90+29h0eGEDYti7j7maHkBKUAwlcPvMg5m3Y= +=DA1T +-----END PGP PUBLIC KEY BLOCK----- diff --git a/pgdg/apt.postgresql.org.gpg b/pgdg/apt.postgresql.org.gpg Binary files differnew file mode 100644 index 0000000..afa15cb --- /dev/null +++ b/pgdg/apt.postgresql.org.gpg diff --git a/pgdg/apt.postgresql.org.sh b/pgdg/apt.postgresql.org.sh new file mode 100755 index 0000000..6dd5bbe --- /dev/null +++ b/pgdg/apt.postgresql.org.sh @@ -0,0 +1,291 @@ +#!/bin/sh + +# script to add apt.postgresql.org to sources.list.d + +# Copyright (C) 2013-2023 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# The full text of the GPL is distributed as in +# /usr/share/common-licenses/GPL-2 on Debian systems. + +SOURCESLIST="/etc/apt/sources.list.d/pgdg.sources" +TYPES="deb" +COMPONENTS="main" +PGDG="pgdg" + +# variables imported from https://git.postgresql.org/gitweb/?p=pgapt.git;a=blob;f=pgapt.conf +# checked out in $HOME/apt.postgresql.org/; run "make" to update +PG_BETA_VERSION="16" +PG_DEVEL_VERSION="17" +PG_REPOSITORY_DISTS="sid trixie bookworm bullseye buster mantic lunar kinetic jammy focal bionic" +PG_ARCHIVE_DISTS="sid trixie bookworm bullseye buster stretch jessie wheezy squeeze lenny etch mantic lunar kinetic jammy impish hirsute groovy focal eoan disco cosmic bionic zesty xenial wily utopic saucy precise lucid" + +while getopts "c:f:h:ipstv:y" opt ; do + case $opt in + c) COMPONENTS="main $OPTARG" ;; # make these extra components available + f) SOURCESLIST=$OPTARG ;; # sources.list filename to write to + h) HOST="$OPTARG" ;; # hostname to use in sources.list + i) INSTALL="yes" ;; # install packages for version given with -v + p) PURGE="yes" ;; # purge existing postgresql packages + s) TYPES="deb deb-src" ;; # include source repository as well + t) PGDG="pgdg-testing" ;; # use *-pgdg or *-pgdg-testing + v) PGVERSION="$OPTARG" ;; # set up sources.list to use this version (useful for beta/devel packages) + y) ;; # don't ask for confirmation + *) exit 5 ;; + esac + YES="yes" # don't ask for confirmation if any option is given +done +# shift away args +shift $((OPTIND - 1)) +# check options +if [ "$INSTALL" ] && [ -z "$PGVERSION" ]; then + echo "With -i, a version to install must be provided (-v)" + exit 1 +fi + +# codename from command line +CODENAME="$1" +# parse os-release +if [ -z "$CODENAME" ] && [ -f /etc/os-release ]; then + . /etc/os-release + if [ "$VERSION_CODENAME" ]; then # added in buster/xenial + CODENAME="$VERSION_CODENAME" + else + # Debian: VERSION="7.0 (wheezy)" + # Ubuntu: VERSION="13.04, Raring Ringtail" + # VERSION="18.04.1 LTS (Bionic Beaver)" + CODENAME=$(echo $VERSION | sed -ne 's/.*(\(.*\)).*/\1/') # works on Debian only + fi +fi +# try lsb_release +if [ -z "$CODENAME" ] && command -v lsb_release >/dev/null; then + CODENAME=$(lsb_release -cs 2>/dev/null) +fi +# guess from sources.list +if [ -z "$CODENAME" ] && [ -f /etc/apt/sources.list ]; then + CODENAME=$(grep '^deb ' /etc/apt/sources.list | head -n1 | awk '{ print $3 }') +fi +# complain if no result yet +if [ -z "$CODENAME" ]; then + cat <<EOF +Could not determine the distribution codename. Please report this as a bug to +pgsql-pkg-debian@postgresql.org. As a workaround, you can call this script with +the proper codename as parameter, e.g. "$0 squeeze". +EOF + exit 1 +fi + +# errors are non-fatal above +set -eu + +if [ "${HOST:-}" ]; then + : +elif echo "$PG_REPOSITORY_DISTS" | grep -qw "$CODENAME"; then + # known distribution on apt.postgresql.org + HOST="apt.postgresql.org" +elif echo "$PG_ARCHIVE_DISTS" | grep -qw "$CODENAME"; then + # known distribution on apt.postgresql.org + HOST="apt-archive.postgresql.org" +else # unknown distribution, verify on the web + HOST="apt.postgresql.org" + DISTURL="https://$HOST/pub/repos/apt/dists/" + if [ -x /usr/bin/curl ]; then + DISTHTML=$(curl -s $DISTURL || :) + elif [ -x /usr/bin/wget ]; then + DISTHTML=$(wget --quiet -O - $DISTURL || :) + fi + if [ "${DISTHTML:-}" ]; then + if ! echo "$DISTHTML" | grep -q "$CODENAME-$PGDG"; then + cat <<EOF +Your system is using the distribution codename $CODENAME, but $CODENAME-$PGDG +does not seem to be a valid distribution on +$DISTURL + +We abort the installation here. If you want to use a distribution different +from your system, you can call this script with an explicit codename, e.g. +"$0 precise". + +For more information, refer to https://wiki.postgresql.org/wiki/Apt +or ask on the mailing list for assistance: pgsql-pkg-debian@postgresql.org +EOF + exit 1 + fi + fi +fi + +cat <<EOF +This script will enable the PostgreSQL APT repository on $HOST on +your system. The distribution codename used will be $CODENAME-$PGDG. + +EOF + +if [ -z "${YES:-}" ]; then + echo -n "Press Enter to continue, or Ctrl-C to abort." + read enter + echo +fi + +# keyring needs to be readable for apt +umask 022 +# prefer .gpg keyring from postgresql-common +KEYRING="/usr/share/postgresql-common/pgdg/apt.postgresql.org.gpg" +# otherwise, use the .asc key +test -e $KEYRING || KEYRING="/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc" +echo "Using keyring $KEYRING" +# write .asc key to disk if not yet present +if ! test -e $KEYRING; then + mkdir -p /usr/share/postgresql-common/pgdg + cat > $KEYRING <<EOF +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBE6XR8IBEACVdDKT2HEH1IyHzXkb4nIWAY7echjRxo7MTcj4vbXAyBKOfjja +UrBEJWHN6fjKJXOYWXHLIYg0hOGeW9qcSiaa1/rYIbOzjfGfhE4x0Y+NJHS1db0V +G6GUj3qXaeyqIJGS2z7m0Thy4Lgr/LpZlZ78Nf1fliSzBlMo1sV7PpP/7zUO+aA4 +bKa8Rio3weMXQOZgclzgeSdqtwKnyKTQdXY5MkH1QXyFIk1nTfWwyqpJjHlgtwMi +c2cxjqG5nnV9rIYlTTjYG6RBglq0SmzF/raBnF4Lwjxq4qRqvRllBXdFu5+2pMfC +IZ10HPRdqDCTN60DUix+BTzBUT30NzaLhZbOMT5RvQtvTVgWpeIn20i2NrPWNCUh +hj490dKDLpK/v+A5/i8zPvN4c6MkDHi1FZfaoz3863dylUBR3Ip26oM0hHXf4/2U +A/oA4pCl2W0hc4aNtozjKHkVjRx5Q8/hVYu+39csFWxo6YSB/KgIEw+0W8DiTII3 +RQj/OlD68ZDmGLyQPiJvaEtY9fDrcSpI0Esm0i4sjkNbuuh0Cvwwwqo5EF1zfkVj +Tqz2REYQGMJGc5LUbIpk5sMHo1HWV038TWxlDRwtOdzw08zQA6BeWe9FOokRPeR2 +AqhyaJJwOZJodKZ76S+LDwFkTLzEKnYPCzkoRwLrEdNt1M7wQBThnC5z6wARAQAB +tBxQb3N0Z3JlU1FMIERlYmlhbiBSZXBvc2l0b3J5iQJOBBMBCAA4AhsDBQsJCAcD +BRUKCQgLBRYCAwEAAh4BAheAFiEEuXsK/KoaR/BE8kSgf8x9RqzMTPgFAlhtCD8A +CgkQf8x9RqzMTPgECxAAk8uL+dwveTv6eH21tIHcltt8U3Ofajdo+D/ayO53LiYO +xi27kdHD0zvFMUWXLGxQtWyeqqDRvDagfWglHucIcaLxoxNwL8+e+9hVFIEskQAY +kVToBCKMXTQDLarz8/J030Pmcv3ihbwB+jhnykMuyyNmht4kq0CNgnlcMCdVz0d3 +z/09puryIHJrD+A8y3TD4RM74snQuwc9u5bsckvRtRJKbP3GX5JaFZAqUyZNRJRJ +Tn2OQRBhCpxhlZ2afkAPFIq2aVnEt/Ie6tmeRCzsW3lOxEH2K7MQSfSu/kRz7ELf +Cz3NJHj7rMzC+76Rhsas60t9CjmvMuGONEpctijDWONLCuch3Pdj6XpC+MVxpgBy +2VUdkunb48YhXNW0jgFGM/BFRj+dMQOUbY8PjJjsmVV0joDruWATQG/M4C7O8iU0 +B7o6yVv4m8LDEN9CiR6r7H17m4xZseT3f+0QpMe7iQjz6XxTUFRQxXqzmNnloA1T +7VjwPqIIzkj/u0V8nICG/ktLzp1OsCFatWXh7LbU+hwYl6gsFH/mFDqVxJ3+DKQi +vyf1NatzEwl62foVjGUSpvh3ymtmtUQ4JUkNDsXiRBWczaiGSuzD9Qi0ONdkAX3b +ewqmN4TfE+XIpCPxxHXwGq9Rv1IFjOdCX0iG436GHyTLC1tTUIKF5xV4Y0+cXIOI +RgQQEQgABgUCTpdI7gAKCRDFr3dKWFELWqaPAKD1TtT5c3sZz92Fj97KYmqbNQZP ++ACfSC6+hfvlj4GxmUjp1aepoVTo3weJAhwEEAEIAAYFAk6XSQsACgkQTFprqxLS +p64F8Q//cCcutwrH50UoRFejg0EIZav6LUKejC6kpLeubbEtuaIH3r2zMblPGc4i ++eMQKo/PqyQrceRXeNNlqO6/exHozYi2meudxa6IudhwJIOn1MQykJbNMSC2sGUp +1W5M1N5EYgt4hy+qhlfnD66LR4G+9t5FscTJSy84SdiOuqgCOpQmPkVRm1HX5X1+ +dmnzMOCk5LHHQuiacV0qeGO7JcBCVEIDr+uhU1H2u5GPFNHm5u15n25tOxVivb94 +xg6NDjouECBH7cCVuW79YcExH/0X3/9G45rjdHlKPH1OIUJiiX47OTxdG3dAbB4Q +fnViRJhjehFscFvYWSqXo3pgWqUsEvv9qJac2ZEMSz9x2mj0ekWxuM6/hGWxJdB+ ++985rIelPmc7VRAXOjIxWknrXnPCZAMlPlDLu6+vZ5BhFX0Be3y38f7GNCxFkJzl +hWZ4Cj3WojMj+0DaC1eKTj3rJ7OJlt9S9xnO7OOPEUTGyzgNIDAyCiu8F4huLPaT +ape6RupxOMHZeoCVlqx3ouWctelB2oNXcxxiQ/8y+21aHfD4n/CiIFwDvIQjl7dg +mT3u5Lr6yxuosR3QJx1P6rP5ZrDTP9khT30t+HZCbvs5Pq+v/9m6XDmi+NlU7Zuh +Ehy97tL3uBDgoL4b/5BpFL5U9nruPlQzGq1P9jj40dxAaDAX/WKJAj0EEwEIACcC +GwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AFAlB5KywFCQPDFt8ACgkQf8x9RqzM +TPhuCQ//QAjRSAOCQ02qmUAikT+mTB6baOAakkYq6uHbEO7qPZkv4E/M+HPIJ4wd +nBNeSQjfvdNcZBA/x0hr5EMcBneKKPDj4hJ0panOIRQmNSTThQw9OU351gm3YQct +AMPRUu1fTJAL/AuZUQf9ESmhyVtWNlH/56HBfYjE4iVeaRkkNLJyX3vkWdJSMwC/ +LO3Lw/0M3R8itDsm74F8w4xOdSQ52nSRFRh7PunFtREl+QzQ3EA/WB4AIj3VohIG +kWDfPFCzV3cyZQiEnjAe9gG5pHsXHUWQsDFZ12t784JgkGyO5wT26pzTiuApWM3k +/9V+o3HJSgH5hn7wuTi3TelEFwP1fNzI5iUUtZdtxbFOfWMnZAypEhaLmXNkg4zD +kH44r0ss9fR0DAgUav1a25UnbOn4PgIEQy2fgHKHwRpCy20d6oCSlmgyWsR40EPP +YvtGq49A2aK6ibXmdvvFT+Ts8Z+q2SkFpoYFX20mR2nsF0fbt1lfH65P64dukxeR +GteWIeNakDD40bAAOH8+OaoTGVBJ2ACJfLVNM53PEoftavAwUYMrR910qvwYfd/4 +6rh46g1Frr9SFMKYE9uvIJIgDsQB3QBp71houU4H55M5GD8XURYs+bfiQpJG1p7e +B8e5jZx1SagNWc4XwL2FzQ9svrkbg1Y+359buUiP7T6QXX2zY++JAj0EEwEIACcC +GwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AFAlEqbZUFCQg2wEEACgkQf8x9RqzM +TPhFMQ//WxAfKMdpSIA9oIC/yPD/dJpY/+DyouOljpE6MucMy/ArBECjFTBwi/j9 +NYM4ynAk34IkhuNexc1i9/05f5RM6+riLCLgAOsADDbHD4miZzoSxiVr6GQ3YXMb +OGld9kV9Sy6mGNjcUov7iFcf5Hy5w3AjPfKuR9zXswyfzIU1YXObiiZT38l55pp/ +BSgvGVQsvbNjsff5CbEKXS7q3xW+WzN0QWF6YsfNVhFjRGj8hKtHvwKcA02wwjLe +LXVTm6915ZUKhZXUFc0vM4Pj4EgNswH8Ojw9AJaKWJIZmLyW+aP+wpu6YwVCicxB +Y59CzBO2pPJDfKFQzUtrErk9irXeuCCLesDyirxJhv8o0JAvmnMAKOLhNFUrSQ2m ++3EnF7zhfz70gHW+EG8X8mL/EN3/dUM09j6TVrjtw43RLxBzwMDeariFF9yC+5bL +tnGgxjsB9Ik6GV5v34/NEEGf1qBiAzFmDVFRZlrNDkq6gmpvGnA5hUWNr+y0i01L +jGyaLSWHYjgw2UEQOqcUtTFK9MNzbZze4mVaHMEz9/aMfX25R6qbiNqCChveIm8m +Yr5Ds2zdZx+G5bAKdzX7nx2IUAxFQJEE94VLSp3npAaTWv3sHr7dR8tSyUJ9poDw +gw4W9BIcnAM7zvFYbLF5FNggg/26njHCCN70sHt8zGxKQINMc6SJAj0EEwEIACcC +GwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AFAlLpFRkFCQ6EJy0ACgkQf8x9RqzM +TPjOZA//Zp0e25pcvle7cLc0YuFr9pBv2JIkLzPm83nkcwKmxaWayUIG4Sv6pH6h +m8+S/CHQij/yFCX+o3ngMw2J9HBUvafZ4bnbI0RGJ70GsAwraQ0VlkIfg7GUw3Tz +voGYO42rZTru9S0K/6nFP6D1HUu+U+AsJONLeb6oypQgInfXQExPZyliUnHdipei +4WR1YFW6sjSkZT/5C3J1wkAvPl5lvOVthI9Zs6bZlJLZwusKxU0UM4Btgu1Sf3nn +JcHmzisixwS9PMHE+AgPWIGSec/N27a0KmTTvImV6K6nEjXJey0K2+EYJuIBsYUN +orOGBwDFIhfRk9qGlpgt0KRyguV+AP5qvgry95IrYtrOuE7307SidEbSnvO5ezNe +mE7gT9Z1tM7IMPfmoKph4BfpNoH7aXiQh1Wo+ChdP92hZUtQrY2Nm13cmkxYjQ4Z +gMWfYMC+DA/GooSgZM5i6hYqyyfAuUD9kwRN6BqTbuAUAp+hCWYeN4D88sLYpFh3 +paDYNKJ+Gf7Yyi6gThcV956RUFDH3ys5Dk0vDL9NiWwdebWfRFbzoRM3dyGP889a +OyLzS3mh6nHzZrNGhW73kslSQek8tjKrB+56hXOnb4HaElTZGDvD5wmrrhN94kby +Gtz3cydIohvNO9d90+29h0eGEDYti7j7maHkBKUAwlcPvMg5m3Y= +=DA1T +-----END PGP PUBLIC KEY BLOCK----- +EOF +fi + +for version in ${PGVERSION:-}; do + # devel version comes from *-pgdg-snapshot (with lower default apt pinning priority) + if dpkg --compare-versions $version ge "${PG_DEVEL_VERSION:-999}"; then + COMPONENTS="$COMPONENTS $version" # devel component is likely empty, but add it to be sure + DEVEL_COMPONENT="${DEVEL_COMPONENT:-} $version" + PIN="-t $CODENAME-pgdg-snapshot" + # beta version needs a different component + elif dpkg --compare-versions $version ge "${PG_BETA_VERSION:-999}"; then + COMPONENTS="$COMPONENTS $version" + fi + + # select packages to install + PACKAGES="${PACKAGES:-} postgresql-$version postgresql-server-dev-$version" + case $version in + 8*|9*) PACKAGES="$PACKAGES postgresql-contrib-$version" ;; + esac +done + +echo "Writing $SOURCESLIST ..." +cat > $SOURCESLIST <<EOF +Types: $TYPES +URIs: https://$HOST/pub/repos/apt +Suites: $CODENAME-$PGDG +Components: $COMPONENTS +Signed-By: $KEYRING +EOF + +# write a separate section for devel without main so we don't include all of snapshot +if [ "${DEVEL_COMPONENT:-}" ]; then +cat >> $SOURCESLIST <<EOF + +Types: $TYPES +URIs: https://$HOST/pub/repos/apt +Suites: $CODENAME-pgdg-snapshot +Components: ${DEVEL_COMPONENT# } +Signed-By: $KEYRING +EOF +fi + +if [ "$SOURCESLIST" = "/etc/apt/sources.list.d/pgdg.sources" ]; then + # remove pgdg.list when upgrading to pgdg.sources + rm -vf /etc/apt/sources.list.d/pgdg.list /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg +fi + +echo +echo "Running apt-get update ..." +apt-get update + +cat <<EOF + +You can now start installing packages from $HOST. + +Have a look at https://wiki.postgresql.org/wiki/Apt for more information; +most notably the FAQ at https://wiki.postgresql.org/wiki/Apt/FAQ +EOF + +# remove/install packages +export DEBIAN_FRONTEND=noninteractive +if [ "${PURGE:-}" ]; then + echo + echo "Purging existing PostgreSQL packages ..." + apt-get -y purge postgresql-client-common +fi +if [ "${INSTALL:-}" ]; then + echo + echo "Installing packages for PostgreSQL $PGVERSION ..." + apt-get -y -o DPkg::Options::=--force-confnew \ + install ${PIN:-} $PACKAGES +fi diff --git a/pgdg/update b/pgdg/update new file mode 100755 index 0000000..4a937df --- /dev/null +++ b/pgdg/update @@ -0,0 +1,12 @@ +#!/bin/sh + +. $HOME/apt.postgresql.org/pgapt.conf + +set -eux + +sed -i \ + -e "s/^PG_BETA_VERSION=.*/PG_BETA_VERSION=\"${PG_BETA_VERSION:-}\"/" \ + -e "s/^PG_DEVEL_VERSION=.*/PG_DEVEL_VERSION=\"$PG_DEVEL_VERSION\"/" \ + -e "s/^PG_REPOSITORY_DISTS=.*/PG_REPOSITORY_DISTS=\"$PG_REPOSITORY_DISTS\"/" \ + -e "s/^PG_ARCHIVE_DISTS=.*/PG_ARCHIVE_DISTS=\"$PG_ARCHIVE_DISTS\"/" \ + apt.postgresql.org.sh diff --git a/pgxs_debian_control.mk b/pgxs_debian_control.mk new file mode 100644 index 0000000..2761e1c --- /dev/null +++ b/pgxs_debian_control.mk @@ -0,0 +1,13 @@ +# +# produce a debian/control file from a debian/control.in +# +# In debian/rules, include /usr/share/postgresql-common/pgxs_debian_control.mk +# +# Author: Dimitri Fontaine <dfontaine@hi-media.com> +# +debian/control: debian/control.in debian/pgversions + pg_buildext checkcontrol + +# run check when clean is invoked +clean: debian/control +.PHONY: debian/control diff --git a/postgresqlrc.5 b/postgresqlrc.5 new file mode 100644 index 0000000..b0e52a5 --- /dev/null +++ b/postgresqlrc.5 @@ -0,0 +1,42 @@ +.TH POSTGRESQLRC 5 "Feburary 2005" "Debian" "Debian PostgreSQL infrastructure" + +.SH NAME +~/.postgresqlrc \- Per\-user PostgreSQL cluster configuration + +.SH DESCRIPTION +The file +.B ~/.postgresqlrc +configures the default PostgreSQL version/cluster and the default +database for an user. If it is not present, the system\-wide file +.B /etc/postgresql\-common/user_clusters +is used instead. + +.SH FORMAT +.P +Comments are introduced by the character +.BR # . +Comments may follow data on a line; the first comment character terminates +the data. +Leading whitespace and blank lines are ignored. + +The first uncommented, non\-blank line is used, all following lines are ignored. + +Fields must be given in the following order, separated by white space: + +.TP +.B VERSION +The major PostgreSQL version of the cluster to connect to. +.TP +.B CLUSTER +The name of a cluster to connect to. A remote cluster is specified +with \fIhost\fR:\fIport\fR. If port is empty, it defaults to 5432. +.TP +.B DATABASE +Within the cluster, the database to which the user will connect by default +if he does not specify a database on the command line. If this is +.BR * , +the default database will be the one named by the user's login id. + +.SH SEE ALSO +.BR pg_wrapper (1), +.BR user_clusters (5) diff --git a/rpm/README b/rpm/README new file mode 100644 index 0000000..a6a9067 --- /dev/null +++ b/rpm/README @@ -0,0 +1,52 @@ +postgresql-common for RedHat +============================ + +The postgresql-common framework was written for Debian/Ubuntu, but most parts +of it work as well on other operating systems. The RPM port changes little in +the original code, and even uses many files from the debian/ directory for +building the packages. + +No separate PostgreSQL client/server packages are provided; the port is backed +by the PGDG RPM packages from yum.postgresql.org. + +The filesystem layout is unchanged, /etc/postgresql, /etc/postgresql-common, +and /var/lib/postgresql are used as before. + +Differences between the Debian and RedHat operating modes are: + +* /var/run/postgresql/ is still used for external pid files, but the default + unix socket directory is /tmp, to match the RPM packages' default. + +* The postgres system user home is /var/lib/pgsql. + +* The binroot is changed from /usr/lib/postgresql/ to /usr/pgsql-. (Note the + missing trailing slash, the idea is that the version number can just be + appended to the path, e.g. /usr/lib/postgresql/9.4/bin becomes + /usr/pgsql-9.4/bin.) + +* The various symlinks for frontend programs in /usr/bin like psql are not + direct symlinks to pg_wrapper, but are added as high-priority alternatives to + the alternatives symlinks set up by the PostgreSQL RPM packages. + +* SSL is disabled by default because there is no easily available snakeoil + certificate. Proper certificates can be configured in createcluster.conf. + +* No attempt is made to setup OOM killer protection for the postmaster process. + +* On Debian, the /etc/init.d/postgresql init script skips versions that have + their own /etc/init.d/postgresql-x.y init script, mostly for compatibility + with legacy packages before the advent of the postgresql-common framework. + The RPM packages provide /etc/init.d/postgresql-x.y scripts, which are + ignored by /etc/init.d/postgresql. The postgresql-x.y scripts will not do + anything as long as the user does not use them to create clusters in + /var/lib/pgsql. (In which case they continue to work as if postgresql-common + was not present.) + +* Debian's pre/postinst/rm maintainer scripts are not used. Mostly this means + there is no automatic integration of tsearch with system-provided + dictionaries. + +The postgresql-common testsuite is supported if perl-Test-Simple and +perl-Time-HiRes are installed. + + -- Christoph Berg <christoph.berg@credativ.de> Thu, 26 Jun 2014 16:59:47 +0200 diff --git a/rpm/init-functions-compat b/rpm/init-functions-compat new file mode 100644 index 0000000..da13596 --- /dev/null +++ b/rpm/init-functions-compat @@ -0,0 +1,12 @@ +# Functions missing in older /lib/lsb/init-functions scripts + +function_exists () { + type $1 >/dev/null 2>&1 +} + +function_exists log_daemon_msg || log_daemon_msg () { echo -n "$1:${2:+ $2}"; } +function_exists log_progress_msg || log_progress_msg () { echo -n " $1"; } +function_exists log_end_msg || log_end_msg () { if [ $1 -eq 0 ]; then echo "."; else echo "failed!"; fi; } +# this one exists, but we provide it anyway so we don't need to depend on redhat-lsb-core: +function_exists log_warning_msg || log_warning_msg () { echo "$1"; } +function_exists log_failure_msg || log_failure_msg () { echo "$1"; } diff --git a/rpm/postgresql-common.spec b/rpm/postgresql-common.spec new file mode 100644 index 0000000..cb2e45b --- /dev/null +++ b/rpm/postgresql-common.spec @@ -0,0 +1,139 @@ +Name: postgresql-common +Version: %{version} +Release: 1%{?dist} +BuildArch: noarch +Summary: PostgreSQL database-cluster manager +Packager: Debian PostgreSQL Maintainers <pkg-postgresql-public@lists.alioth.debian.org> + +License: GPLv2+ +URL: https://packages.debian.org/sid/%{name} +Source0: http://ftp.debian.org/debian/pool/main/p/%{name}/%{name}_%{version}.tar.xz +Requires: postgresql-client-common +Requires: perl-JSON + +%description +The postgresql-common package provides a structure under which +multiple versions of PostgreSQL may be installed and/or multiple +clusters maintained at one time. + +%package -n postgresql-client-common +Summary: manager for multiple PostgreSQL client versions +%description -n postgresql-client-common +The postgresql-client-common package provides a structure under which +multiple versions of PostgreSQL client programs may be installed at +the same time. It provides a wrapper which selects the right version +for the particular cluster you want to access (with a command line +option, an environment variable, /etc/postgresql-common/user_clusters, +or ~/.postgresqlrc). + +%prep +# unpack tarball, ignoring the name of the top level directory inside +%setup -c +mv */* . + +%build +make + +%install +rm -rf %{buildroot} +# install in subpackages using the Debian files +for inst in debian/*.install; do + pkg=$(basename $inst .install) + [ "$pkg" = "postgresql-server-dev-all" ] && continue + echo "### Reading $pkg files list from $inst ###" + while read file dir; do + [ "$file" = "supported_versions" ] && continue # only relevant on Debian + mkdir -p %{buildroot}/$dir + cp -r $file %{buildroot}/$dir + echo "/$dir/${file##*/}" >> files-$pkg + done < $inst +done +# install manpages +for manpages in debian/*.manpages; do + pkg=$(basename $manpages .manpages) + [ "$pkg" = "postgresql-server-dev-all" ] && continue + echo "### Reading $pkg manpages list from $manpages ###" + while read file; do + section="${file##*.}" + mandir="%{buildroot}%{_mandir}/man$section" + mkdir -p $mandir + for f in $file; do # expand wildcards + cp $f $mandir + echo "%doc %{_mandir}/man$section/$(basename $f).gz" >> files-$pkg + done + done < $manpages +done +# install pg_wrapper symlinks by augmenting the existing pgdg.rpm alternatives +cat debian/postgresql-*common.links | \ +while read dest link; do + name="pgsql-$(basename $link)" + echo "update-alternatives --install /$link $name /$dest 9999" >> postgresql-client-common.post + echo "update-alternatives --remove $name /$dest" >> postgresql-client-common.preun +done +# activate rpm-specific tweaks +sed -i -e 's/#redhat# //' \ + %{buildroot}/lib/systemd/system-generators/postgresql-generator \ + %{buildroot}/usr/bin/pg_config \ + %{buildroot}/usr/bin/pg_virtualenv \ + %{buildroot}/usr/share/perl5/PgCommon.pm \ + %{buildroot}/usr/share/postgresql-common/init.d-functions \ + %{buildroot}/usr/share/postgresql-common/pg_getwal +# install init script +mkdir -p %{buildroot}/etc/init.d %{buildroot}/etc/logrotate.d +cp debian/postgresql-common.postgresql.init %{buildroot}/etc/init.d/postgresql +#cp debian/postgresql-common.postinst %{buildroot}/usr/share/postgresql-common +cp rpm/init-functions-compat %{buildroot}/usr/share/postgresql-common +# ssl defaults to 'off' here because we don't have pregenerated snakeoil certs +sed -e 's/__SSL__/off/' createcluster.conf > %{buildroot}/etc/postgresql-common/createcluster.conf +cp debian/postgresql-common.logrotate %{buildroot}/etc/logrotate.d/postgresql-common + +%files -n postgresql-common -f files-postgresql-common +%attr(0755, root, root) %config /etc/init.d/postgresql +#%attr(0755, root, root) /usr/share/postgresql-common/postgresql-common.postinst +/usr/share/postgresql-common/init-functions-compat +%config /etc/postgresql-common/createcluster.conf +%config /etc/logrotate.d/postgresql-common + +%if 0%{?rhel} >= 7 +%config /lib/systemd/system/*.service +%config /lib/systemd/system/*.timer +%config /lib/systemd/system-generators/postgresql-generator +%endif + +%files -n postgresql-client-common -f files-postgresql-client-common + +%post +# create postgres user +groupadd -g 26 -o -r postgres >/dev/null 2>&1 || : +useradd -M -n -g postgres -o -r -d /var/lib/pgsql -s /bin/bash \ + -c "PostgreSQL Server" -u 26 postgres >/dev/null 2>&1 || : +# create directories so postgres can create clusters without root +install -d -o postgres -g postgres /etc/postgresql /var/lib/postgresql /var/lib/pgsql /var/log/postgresql /var/run/postgresql +# install logrotate config +version_lt () { + newest=$( ( echo "$1"; echo "$2" ) | sort -V | tail -n1) + [ "$1" != "$newest" ] +} +lrversion=$(rpm --queryformat '%{VERSION}' -q logrotate) +if version_lt $lrversion 3.8; then + echo "Adjusting /etc/logrotate.d/postgresql-common for logrotate version $lrversion" + sed -i -e '/ su /d' /etc/logrotate.d/postgresql-common || : +fi + +%post -n postgresql-client-common -f postgresql-client-common.post +update-alternatives --install /usr/bin/ecpg pgsql-ecpg /usr/share/postgresql-common/pg_wrapper 9999 + +%preun -n postgresql-client-common -f postgresql-client-common.preun +update-alternatives --remove pgsql-ecpg /usr/share/postgresql-common/pg_wrapper + +%changelog +* Tue Sep 29 2020 Christoph Berg <myon@debian.org> 217-1 +- Drop postgresql-server-dev-all package, it's debian-specific only. +* Fri Dec 09 2016 Bernd Helmle <bernd.helmle@credativ.de> 177-1 +- New upstream release 177 +* Fri Jun 03 2016 Bernd Helmle <bernd.helmle@credativ.de> 174-2 +- Fix package dependencies and systemd integration +* Thu Aug 7 2014 Christoph Berg <christoph.berg@credativ.de> 160-1 +- Omit the LD_PRELOAD logic in pg_wrapper +* Thu Jun 5 2014 Christoph Berg <christoph.berg@credativ.de> 158-1 +- Initial specfile version diff --git a/server/README b/server/README new file mode 100644 index 0000000..867757a --- /dev/null +++ b/server/README @@ -0,0 +1,2 @@ +The files in this directory are only used when compiling Debian packages of the +PostgreSQL database server. They are not required at run time. diff --git a/server/catversion b/server/catversion new file mode 100755 index 0000000..fc4af08 --- /dev/null +++ b/server/catversion @@ -0,0 +1,17 @@ +#!/bin/sh + +# Extract server catalog and control file version numbers. +# This information is stored in the packages and used at install time to +# determine if an in-major-version pg_upgradecluster is required. + +set -eu + +CATVERSION=$(awk '/^#define CATALOG_VERSION_NO/ { print $3 }' src/include/catalog/catversion.h) +CONTROLVERSION=$(awk '/^#define PG_CONTROL_VERSION/ { print $3 }' src/include/catalog/pg_control.h) + +case $CONTROLVERSION in + # control file versions used in PG 9.6 .. 15 + # don't append to catversion to avoid spurious warnings for users of existing packages + 960|1002|1100|1201|1300) echo "$CATVERSION" ;; + *) echo "$CATVERSION-$CONTROLVERSION" ;; +esac diff --git a/server/pg_config.pl b/server/pg_config.pl new file mode 100755 index 0000000..2c90236 --- /dev/null +++ b/server/pg_config.pl @@ -0,0 +1,76 @@ +#!/usr/bin/perl + +# Perl reimplementation of PostgreSQL's pg_config binary. +# We provide this as /usr/bin/pg_config to support cross-compilation using +# libpq-dev. Also, this makes the two installed pg_config copies not conflict +# via their debugging symbols. +# +# This code is released under the terms of the PostgreSQL License. +# Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group +# Author: Christoph Berg + +use strict; +use warnings; + +# no arguments, print all items +if (@ARGV == 0) { + while (<DATA>) { + last if /^$/; # begin of help section + print; + } + exit 0; +} + +# --help or -? +if (grep {$_ =~ /^(--help|-\?)$/} @ARGV) { + while (<DATA>) { + last if /^$/; # begin of help section + } + print; # include empty line in output + while (<DATA>) { + next if /^Report bugs/; # Skip bug address in the perl version + print; + } + exit 0; +} + +# specific value(s) requested +my %options; +my $help; +while (<DATA>) { + last if /^$/; # begin of help section + /^(\S+) = (.*)/ or die "malformatted data item"; + $options{'--' . lc $1} = $2; +} + +foreach my $arg (@ARGV) { + unless ($options{$arg}) { + print "pg_config: invalid argument: $arg\n"; + print "Try \"pg_config --help\" for more information.\n"; + exit 1; + } + print "$options{$arg}\n"; +} + +exit 0; + +# The DATA section consists of the `pg_config` output (one KEY = value item per +# line), and the `pg_config --help` text. The first --help line is empty, which +# we use to detect the beginning of the help section. + +__DATA__ +INCLUDEDIR = /usr/include/postgresql + +pg_config provides information about the installed version of PostgreSQL. + +Usage: + pg_config [OPTION]... + +Options: + --includedir show location of C header files of the client + interfaces + -?, --help show this help, then exit + +With no arguments, all known items are shown. + +Report bugs to <pgsql-bugs@postgresql.org>. diff --git a/server/postgresql.mk b/server/postgresql.mk new file mode 100644 index 0000000..8e73f66 --- /dev/null +++ b/server/postgresql.mk @@ -0,0 +1,294 @@ +#!/usr/bin/make -f + +# The PostgreSQL server packages include this in their debian/rules file. + +# MAJOR_VER is used in path names (15 for /usr/lib/postgresql/15) +ifndef MAJOR_VER +$(error MAJOR_VER must be defined before including this file) +endif +# MAJOR_PKG is used in package names (15 for postgresql-15) +MAJOR_PKG = $(MAJOR_VER) + +# path to auxiliary build files +AUX_MK_DIR = /usr/share/postgresql-common/server + +# version comparison +version_ge = $(shell dpkg --compare-versions $(MAJOR_VER) ge $(1) && echo y) + +# include dpkg makefiles +include /usr/share/dpkg/architecture.mk +include /usr/share/dpkg/pkg-info.mk +include /usr/share/dpkg/vendor.mk +export DEB_BUILD_MAINT_OPTIONS = hardening=+all +DPKG_EXPORT_BUILDFLAGS = 1 +include /usr/share/dpkg/buildflags.mk + +# strict symbol checking +export DPKG_GENSYMBOLS_CHECK_LEVEL = 4 + +# server catalog version +CATVERSION = $(shell $(AUX_MK_DIR)/catversion) + +# configure flags + +CONFIGURE_FLAGS = \ + --with-tcl \ + --with-perl \ + --with-python \ + --with-pam \ + --with-openssl \ + --with-libxml \ + --with-libxslt \ + --mandir=/usr/share/postgresql/$(MAJOR_VER)/man \ + --docdir=/usr/share/doc/postgresql-doc-$(MAJOR_VER) \ + --sysconfdir=/etc/postgresql-common \ + --datarootdir=/usr/share/ \ + --datadir=/usr/share/postgresql/$(MAJOR_VER) \ + --bindir=/usr/lib/postgresql/$(MAJOR_VER)/bin \ + --libdir=/usr/lib/$(DEB_HOST_MULTIARCH)/ \ + --libexecdir=/usr/lib/postgresql/ \ + --includedir=/usr/include/postgresql/ \ + --with-extra-version=" ($(DEB_VENDOR) $(DEB_VERSION))" \ + --enable-nls \ + --enable-thread-safety \ + --enable-debug \ + --enable-dtrace \ + --disable-rpath \ + --with-uuid=e2fs \ + --with-gnu-ld \ + --with-gssapi \ + --with-ldap \ + --with-pgport=5432 \ + --with-system-tzdata=/usr/share/zoneinfo \ + AWK=mawk \ + MKDIR_P='/bin/mkdir -p' \ + PROVE='/usr/bin/prove' \ + PYTHON=/usr/bin/python3 \ + TAR='/bin/tar' \ + XSLTPROC='xsltproc --nonet' \ + CFLAGS='$(CFLAGS)' \ + LDFLAGS='$(LDFLAGS)' + +ifeq ($(call version_ge,9.4),y) + CONFIGURE_FLAGS += --enable-tap-tests +endif + +ifeq ($(call version_ge,9.5),y) + ifneq ($(findstring $(DEB_HOST_ARCH), alpha),) + CONFIGURE_FLAGS += --disable-spinlocks + endif +endif + +ifeq ($(call version_ge,10),y) + CONFIGURE_FLAGS += --with-icu +endif + +ifeq ($(call version_ge,11)$(filter pkg.postgresql.nollvm,$(DEB_BUILD_PROFILES)),y) + # if package depends on LLVM, use it + LLVM_VERSIONED_DEP=$(shell grep 'llvm-[0-9]*-dev' debian/control | grep -v "!$(DEB_HOST_ARCH)" | grep -o '[0-9]*' | head -n1) + LLVM_DEP=$(shell grep 'llvm-dev' debian/control | grep -v "!$(DEB_HOST_ARCH)") + ifneq ($(LLVM_VERSIONED_DEP)$(LLVM_DEP),) + ifneq ($(LLVM_VERSIONED_DEP),) + LLVM_VERSION = $(LLVM_VERSIONED_DEP) + LLVM_CONFIG = /usr/bin/llvm-config-$(LLVM_VERSION) + else + LLVM_CONFIG = $(lastword $(shell ls -v /usr/bin/llvm-config-*)) + LLVM_VERSION = $(subst /usr/bin/llvm-config-,,$(LLVM_CONFIG)) + endif + CONFIGURE_FLAGS += --with-llvm LLVM_CONFIG=$(LLVM_CONFIG) CLANG=/usr/bin/clang-$(LLVM_VERSION) + else + LLVM_VERSION = 0.invalid # mute dpkg error on empty version fields in debian/control + endif + TEMP_CONFIG = TEMP_CONFIG=$(AUX_MK_DIR)/test-with-jit.conf +else + LLVM_VERSION = 0.invalid # mute dpkg error on empty version fields in debian/control +endif + +ifeq ($(call version_ge,14),y) + CONFIGURE_FLAGS += --with-lz4 +endif + +ifeq ($(call version_ge,15),y) + ifeq ($(filter pkg.postgresql.nozstd,$(DEB_BUILD_PROFILES)),) + CONFIGURE_FLAGS += --with-zstd + endif +endif + +ifeq ($(call version_ge,17),y) + WITH_PG_BSD_INDENT = y +endif + +# Facilitate hierarchical profile generation on amd64 (#730134) +ifeq ($(DEB_HOST_ARCH),amd64) + CFLAGS += -fno-omit-frame-pointer +endif + +# Work around an ICE bug in GCC 11.2.0, see +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=103395 +ifneq ($(findstring $(DEB_HOST_ARCH), armel armhf),) + CFLAGS+= -DSTAP_SDT_ARG_CONSTRAINT=g +endif + +ifeq ($(DEB_HOST_ARCH_OS),linux) + CONFIGURE_FLAGS += --with-systemd + CONFIGURE_FLAGS += --with-selinux +endif + +ifneq ($(filter pkg.postgresql.cassert,$(DEB_BUILD_PROFILES)),) + CONFIGURE_FLAGS += --enable-cassert + GENCONTROL_FLAGS += -Vcassert='$${Newline}$${Newline}This package has been built with cassert enabled.' +endif + +# alpha/hppa fail stats tests with postgresql-15 +# hurd implemented semaphores only recently and tests still fail a lot +# ia64 fails the infinite_recurse() test with postgresql-16 +# plperl fails on kfreebsd-* (#704802) +# sh4 lets qemu segfaults when building postgresql-16 +# sparc64 fails bin/summarization-and-inprogress-insertion test with postgresql-15 +ifneq ($(filter alpha hppa hurd% ia64 kfreebsd% sh4 sparc64,$(DEB_HOST_ARCH)),) + TEST_FAIL_COMMAND = echo "Ignoring test failures on this architecture" +else + TEST_FAIL_COMMAND = exit 1 +endif + +# recipes + +%: + dh $@ + +override_dh_auto_configure: + dh_auto_configure --builddirectory=build -- $(CONFIGURE_FLAGS) + # remove pre-built documentation + rm -fv doc/src/sgml/*-stamp + +ifeq ($(filter nodoc,$(DEB_BUILD_PROFILES)),) +override_dh_auto_build-indep: + $(MAKE) -C build/doc all # build man + html +endif + +override_dh_auto_build-arch: + # set MAKELEVEL to 0 to force building submake-generated-headers in src/Makefile.global(.in) + MAKELEVEL=0 $(MAKE) -C build/src all + $(MAKE) -C build/doc man # build man only + $(MAKE) -C build/config all + $(MAKE) -C build/contrib all + # build tutorial stuff + $(MAKE) -C build/src/tutorial NO_PGXS=1 +ifeq ($(WITH_PG_BSD_INDENT),y) + $(MAKE) -C build/src/tools/pg_bsd_indent +endif + +override_dh_auto_install-arch: + $(MAKE) -C build/doc/src/sgml install-man DESTDIR=$(CURDIR)/debian/tmp + $(MAKE) -C build/src install DESTDIR=$(CURDIR)/debian/tmp + $(MAKE) -C build/config install DESTDIR=$(CURDIR)/debian/tmp + $(MAKE) -C build/contrib install DESTDIR=$(CURDIR)/debian/tmp + # move SPI examples into server package (they wouldn't be in the doc package in an -A build) + mkdir -p debian/postgresql-$(MAJOR_PKG)/usr/share/doc/postgresql-$(MAJOR_VER) + mv debian/tmp/usr/share/doc/postgresql-doc-$(MAJOR_VER)/extension debian/postgresql-$(MAJOR_PKG)/usr/share/doc/postgresql-$(MAJOR_VER)/examples +ifeq ($(WITH_PG_BSD_INDENT),y) + $(MAKE) -C build/src/tools/pg_bsd_indent install DESTDIR=$(CURDIR)/debian/tmp + install -m755 src/tools/pgindent/pgindent $(CURDIR)/debian/tmp/usr/lib/postgresql/$(MAJOR_VER)/bin + install -m644 src/tools/pgindent/typedefs.list $(CURDIR)/debian/tmp/usr/share/postgresql/$(MAJOR_VER) +endif + +ifeq ($(filter nodoc,$(DEB_BUILD_PROFILES)),) +override_dh_auto_install-indep: + $(MAKE) -C build/doc install DESTDIR=$(CURDIR)/debian/tmp +endif + +override_dh_makeshlibs: + dh_makeshlibs -Xusr/lib/postgresql/$(MAJOR_VER) + +override_dh_auto_clean: + rm -rf build + +override_dh_installchangelogs: + dh_installchangelogs HISTORY + +override_dh_compress: + dh_compress -X.source -X.c + # compress manpages (excluding debian/tmp/) + gzip -9n $(CURDIR)/debian/*-*/usr/share/postgresql/*/man/man*/*.[137] + +override_dh_install-arch: + dh_install -a + + # link README.Debian.gz to postgresql-common + mkdir -p debian/postgresql-$(MAJOR_PKG)/usr/share/doc/postgresql-$(MAJOR_VER) + ln -s ../postgresql-common/README.Debian.gz debian/postgresql-$(MAJOR_PKG)/usr/share/doc/postgresql-$(MAJOR_VER)/README.Debian.gz + + # assemble perl version of pg_config in libpq-dev + mkdir -p debian/libpq-dev/usr/bin + sed -ne '1,/__DATA__/p' $(AUX_MK_DIR)/pg_config.pl > debian/libpq-dev/usr/bin/pg_config + LC_ALL=C debian/postgresql-client-$(MAJOR_PKG)/usr/lib/postgresql/$(MAJOR_VER)/bin/pg_config | sed -e 's![^ ]*/debian/postgresql-client-$(MAJOR_PKG)!!' >> debian/libpq-dev/usr/bin/pg_config + LC_ALL=C debian/postgresql-client-$(MAJOR_PKG)/usr/lib/postgresql/$(MAJOR_VER)/bin/pg_config --help >> debian/libpq-dev/usr/bin/pg_config + chmod 755 debian/libpq-dev/usr/bin/pg_config + [ "$$(debian/libpq-dev/usr/bin/pg_config --bindir)" = "/usr/lib/postgresql/$(MAJOR_VER)/bin" ] + + # remove actual build path from Makefile.global for reproducibility + sed -i -e "s!^abs_top_builddir.*!abs_top_builddir = /build/postgresql-$(MAJOR_PKG)/build!" \ + -e "s!^abs_top_srcdir.*!abs_top_srcdir = /build/postgresql-$(MAJOR_PKG)/build/..!" \ + -e 's!-f\(debug\|file\)-prefix-map=[^ ]* !!g' \ + debian/postgresql-client-$(MAJOR_PKG)/usr/lib/postgresql/$(MAJOR_VER)/lib/pgxs/src/Makefile.global + + # these are shipped in the pl packages + bash -c "rm -v debian/postgresql-$(MAJOR_PKG)/usr/share/postgresql/$(MAJOR_VER)/extension/{plperl,plpython,pltcl,*_pl}*" + bash -c "rm -v debian/postgresql-$(MAJOR_PKG)/usr/lib/postgresql/$(MAJOR_VER)/lib/{plperl,plpython,pltcl,*_pl}*" + bash -c "rm -rfv debian/postgresql-$(MAJOR_PKG)/usr/lib/postgresql/$(MAJOR_VER)/lib/bitcode/*{plperl,plpython,pltcl}*" + + # record catversion in a file + echo $(CATVERSION) > debian/postgresql-$(MAJOR_PKG)/usr/share/postgresql/$(MAJOR_VER)/catalog_version + +override_dh_install-indep: + dh_install -i + + if [ -d debian/postgresql-doc-$(MAJOR_PKG) ]; then set -e; \ + install -d debian/postgresql-doc-$(MAJOR_PKG)/usr/share/doc/postgresql-doc-$(MAJOR_VER)/tutorial; \ + install src/tutorial/*.c src/tutorial/*.source src/tutorial/Makefile src/tutorial/README debian/postgresql-doc-$(MAJOR_PKG)/usr/share/doc/postgresql-doc-$(MAJOR_VER)/tutorial; \ + fi + +override_dh_auto_test-indep: + # nothing to do + +override_dh_auto_test-arch: +ifeq (, $(findstring nocheck, $(DEB_BUILD_OPTIONS))) + # when tests fail, print newest log files + # initdb doesn't like LANG and LC_ALL to contradict, unset LANG and LC_CTYPE here + # temp-install wants to be invoked from a top-level make, unset MAKELEVEL here + # tell pg_upgrade to create its sockets in /tmp to avoid too long paths + unset LANG LC_CTYPE MAKELEVEL; ulimit -c unlimited; \ + if ! make -C build check-world \ + $(TEMP_CONFIG) \ + PGSOCKETDIR="/tmp" \ + PG_TEST_EXTRA='ssl' \ + PROVE_FLAGS="--verbose"; \ + then \ + for l in `find build -name 'regression.*' -o -name '*.log' -o -name '*_log_*' | perl -we 'print map { "$$_\n"; } sort { (stat $$a)[9] <=> (stat $$b)[9] } map { chomp; $$_; } <>' | tail -n 10`; do \ + echo "******** $$l ********"; \ + cat $$l; \ + done; \ + for c in `find build -name 'core*'`; do \ + echo "******** $$c ********"; \ + gdb -batch -ex 'bt full' build/tmp_install/usr/lib/postgresql/$(MAJOR_VER)/bin/postgres $$c || :; \ + done; \ + $(TEST_FAIL_COMMAND); \ + fi +ifeq ($(WITH_PG_BSD_INDENT),y) + $(MAKE) -C build/src/tools/pg_bsd_indent test DESTDIR=$(CURDIR)/debian/tmp +endif +endif + +override_dh_installdeb-arch: + dh_installdeb + # record catversion in preinst + sed -i -e 's/@CATVERSION@/$(CATVERSION)/' debian/postgresql-$(MAJOR_PKG)/DEBIAN/preinst + +override_dh_gencontrol: + # record catversion in .deb control file + dh_gencontrol $(EXCLUDE_PACKAGES) -- -Vpostgresql:Catversion=$(CATVERSION) -Vllvm:Version=$(LLVM_VERSION) $(GENCONTROL_FLAGS) + +ifneq ($(EXCLUDE_PACKAGES),) +override_dh_builddeb: + dh_builddeb $(EXCLUDE_PACKAGES) +endif diff --git a/server/test-with-jit.conf b/server/test-with-jit.conf new file mode 100644 index 0000000..c68a521 --- /dev/null +++ b/server/test-with-jit.conf @@ -0,0 +1,8 @@ +# config used by pg_regress --temp-config + +fsync = off + +# force JITing of all queries in tests +jit = on +jit_above_cost = 0 +jit_optimize_above_cost = 1000 diff --git a/systemd/README.systemd b/systemd/README.systemd new file mode 100644 index 0000000..c818ad3 --- /dev/null +++ b/systemd/README.systemd @@ -0,0 +1,55 @@ +systemd unit files for PostgreSQL on Debian/Ubuntu +-------------------------------------------------- + +Each cluster is run as a separate service, called postgresql@version-cluster. +pg_ctlcluster is invoked with --skip-systemctl-redirect. Logging still goes to +/var/log/postgresql. + +There is a parent service called postgresql.service, that starts/stops/restarts/ +reloads all individual services that are configured as "auto" in +/etc/postgresql/*/*/start.conf. + +The link between start.conf and postgresql.service is established by +postgresql-generator, which creates symlinks in +/run/systemd/generator/postgresql.service.wants/. + +Backup +------ + +Two backup mechanisms are being offered as systemd services: basebackups +capable of point in time recovery (PITR, the preferred method), and SQL-level +dumps. + +pg_basebackup@version-cluster.service +pg_basebackup@version-cluster.timer + + Weekly basebackup in /var/backups/postgresql/version/cluster. + By default, 3 copies are being kept. + + To enable, run + systemctl enable --now pg_basebackup@version-cluster.timer + systemctl start pg_basebackup@version-cluster.service + +pg_receivewal@version-cluster.service + + WAL archival to be used with pg_basebackup@.service for PITR. + + To enable, run + systemctl enable --now pg_basebackup@version-cluster.timer pg_receivewal@version-cluster.service + systemctl start pg_basebackup@version-cluster.service + +pg_dump@version-cluster.service +pg_dump@version-cluster.timer + + Weekly SQL dump in /var/backups/postgresql/version/cluster. + By default, 3 copies are being kept. + + To enable, run + systemctl enable --now pg_dump@version-cluster.timer + systemctl start pg_dump@version-cluster.service + +The mechanisms provided are meant to be used with low to medium size databases. +For larger databases, or databases with high write volume, we advise to use a +full-size backup solution such as pgbackrest or barman. + + -- Christoph Berg <myon@debian.org> Mon, 08 Mar 2021 13:45:26 +0100 diff --git a/systemd/system-generators/postgresql-generator b/systemd/system-generators/postgresql-generator new file mode 100755 index 0000000..12e8102 --- /dev/null +++ b/systemd/system-generators/postgresql-generator @@ -0,0 +1,38 @@ +#!/bin/sh + +# This systemd generator creates dependency symlinks that make all PostgreSQL +# clusters with "auto" in their start.conf file be started/stopped/reloaded +# when postgresql.service is started/stopped/reloaded. + +set -eu + +gendir="$1" +wantdir="$1/postgresql.service.wants" +bindir="/usr/lib/postgresql/" +#redhat# bindir="/usr/pgsql-" +pgservice="/lib/systemd/system/postgresql@.service" + +mkdir -p "$wantdir" + +for conf in /etc/postgresql/*/*/postgresql.conf; do + # abort loop if glob was not expanded (but accept dead symlinks) + if ! test -e "$conf" && ! test -L "$conf"; then continue; fi + + dir="${conf%/*}" + + # evaluate start.conf + if [ -e "$dir/start.conf" ]; then + start=$(sed 's/#.*$//; /^[[:space:]]*$/d; s/^\s*//; s/\s*$//' "$dir/start.conf") + else + start=auto + fi + [ "$start" = "auto" ] || continue + + verdir="${dir%/*}" + version="${verdir##*/}" + test -x "$bindir$version/bin/postgres" || continue # package got removed + cluster="${dir##*/}" + ln -s "$pgservice" "$wantdir/postgresql@$version-$cluster.service" +done + +exit 0 diff --git a/systemd/system/pg_basebackup@.service b/systemd/system/pg_basebackup@.service new file mode 100644 index 0000000..011c11e --- /dev/null +++ b/systemd/system/pg_basebackup@.service @@ -0,0 +1,14 @@ +[Unit] +Description=Basebackup of PostgreSQL Cluster %i +AssertPathExists=/etc/postgresql/%I/postgresql.conf +Wants=postgresql@%i.service +After=postgresql@%i.service +RequiresMountsFor=/var/backups/postgresql + +[Service] +Type=oneshot +User=postgres +Environment="KEEP=3" +ExecStartPre=+/usr/bin/pg_backupcluster %i createdirectory +ExecStart=/usr/bin/pg_backupcluster %i basebackup +ExecStart=/usr/bin/pg_backupcluster %i expirebasebackups $KEEP diff --git a/systemd/system/pg_basebackup@.timer b/systemd/system/pg_basebackup@.timer new file mode 100644 index 0000000..da0bb3f --- /dev/null +++ b/systemd/system/pg_basebackup@.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Weekly Basebackup of PostgreSQL Cluster %i +AssertPathExists=/etc/postgresql/%I/postgresql.conf + +[Timer] +OnCalendar=weekly +RandomizedDelaySec=1h +FixedRandomDelay=true + +[Install] +# when enabled, start along with postgresql@%i +WantedBy=postgresql@%i.service diff --git a/systemd/system/pg_compresswal@.service b/systemd/system/pg_compresswal@.service new file mode 100644 index 0000000..e5eae6b --- /dev/null +++ b/systemd/system/pg_compresswal@.service @@ -0,0 +1,9 @@ +[Unit] +Description=Compress WAL of PostgreSQL Cluster %i +AssertPathExists=/etc/postgresql/%I/postgresql.conf +RequiresMountsFor=/var/backups/postgresql + +[Service] +Type=oneshot +User=postgres +ExecStart=/usr/bin/pg_backupcluster %i compresswal diff --git a/systemd/system/pg_compresswal@.timer b/systemd/system/pg_compresswal@.timer new file mode 100644 index 0000000..6dddbb9 --- /dev/null +++ b/systemd/system/pg_compresswal@.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Daily Compress WAL of PostgreSQL Cluster %i +AssertPathExists=/etc/postgresql/%I/postgresql.conf + +[Timer] +OnCalendar=daily +RandomizedDelaySec=1h +FixedRandomDelay=true + +[Install] +# when enabled, start along with pg_receivewal@%i +WantedBy=pg_receivewal@%i.service diff --git a/systemd/system/pg_dump@.service b/systemd/system/pg_dump@.service new file mode 100644 index 0000000..a7f7f3d --- /dev/null +++ b/systemd/system/pg_dump@.service @@ -0,0 +1,14 @@ +[Unit] +Description=Dump of PostgreSQL Cluster %i +AssertPathExists=/etc/postgresql/%I/postgresql.conf +Wants=postgresql@%i.service +After=postgresql@%i.service +RequiresMountsFor=/var/backups/postgresql + +[Service] +Type=oneshot +User=postgres +Environment="KEEP=3" +ExecStartPre=+/usr/bin/pg_backupcluster %i createdirectory +ExecStart=/usr/bin/pg_backupcluster %i dump +ExecStart=/usr/bin/pg_backupcluster %i expiredumps $KEEP diff --git a/systemd/system/pg_dump@.timer b/systemd/system/pg_dump@.timer new file mode 100644 index 0000000..a1d2799 --- /dev/null +++ b/systemd/system/pg_dump@.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Weekly Dump of PostgreSQL Cluster %i +AssertPathExists=/etc/postgresql/%I/postgresql.conf + +[Timer] +OnCalendar=weekly +RandomizedDelaySec=1h +FixedRandomDelay=true + +[Install] +# when enabled, start along with postgresql@%i +WantedBy=postgresql@%i.service diff --git a/systemd/system/pg_receivewal@.service b/systemd/system/pg_receivewal@.service new file mode 100644 index 0000000..a15b432 --- /dev/null +++ b/systemd/system/pg_receivewal@.service @@ -0,0 +1,21 @@ +[Unit] +Description=WAL archival of PostgreSQL Cluster %i +AssertPathExists=/etc/postgresql/%I/postgresql.conf +Wants=postgresql@%i.service +After=postgresql@%i.service +RequiresMountsFor=/var/backups/postgresql + +[Service] +Type=simple +User=postgres +ExecStartPre=+/usr/bin/pg_backupcluster %i createdirectory +ExecStart=/usr/bin/pg_backupcluster %i receivewal +Restart=on-failure +RestartSec=1min +# pg_receivewal only flushes compressed output on SIGINT +# (https://www.postgresql.org/message-id/flat/Yvo/5No5S0c4EFMj%40msg.df7cb.de) +KillSignal=SIGINT + +[Install] +# when enabled, start along with postgresql@%i +WantedBy=postgresql@%i.service diff --git a/systemd/system/postgresql.service b/systemd/system/postgresql.service new file mode 100644 index 0000000..f53834e --- /dev/null +++ b/systemd/system/postgresql.service @@ -0,0 +1,18 @@ +# postgresql.service is the meta unit for managing all PostgreSQL clusters on +# the system at once. Conceptually, this unit is more like a systemd target, +# but we are using a service since targets cannot be reloaded. +# +# The unit actually managing PostgreSQL clusters is postgresql@.service, +# instantiated as postgresql@15-main.service for individual clusters. + +[Unit] +Description=PostgreSQL RDBMS + +[Service] +Type=oneshot +ExecStart=/bin/true +ExecReload=/bin/true +RemainAfterExit=on + +[Install] +WantedBy=multi-user.target diff --git a/systemd/system/postgresql@.service b/systemd/system/postgresql@.service new file mode 100644 index 0000000..8eed65c --- /dev/null +++ b/systemd/system/postgresql@.service @@ -0,0 +1,40 @@ +# systemd service template for PostgreSQL clusters. The actual instances will +# be called "postgresql@version-cluster", e.g. "postgresql@9.3-main". The +# variable %i expands to "version-cluster", %I expands to "version/cluster". +# (%I breaks for cluster names containing dashes.) + +[Unit] +Description=PostgreSQL Cluster %i +AssertPathExists=/etc/postgresql/%I/postgresql.conf +RequiresMountsFor=/etc/postgresql/%I /var/lib/postgresql/%I +PartOf=postgresql.service +ReloadPropagatedFrom=postgresql.service +Before=postgresql.service +# stop server before networking goes down on shutdown +After=network.target + +[Service] +Type=forking +# -: ignore startup failure (recovery might take arbitrarily long) +# the actual pg_ctl timeout is configured in pg_ctl.conf +ExecStart=-/usr/bin/pg_ctlcluster --skip-systemctl-redirect %i start +# 0 is the same as infinity, but "infinity" needs systemd 229 +TimeoutStartSec=0 +ExecStop=/usr/bin/pg_ctlcluster --skip-systemctl-redirect -m fast %i stop +TimeoutStopSec=1h +ExecReload=/usr/bin/pg_ctlcluster --skip-systemctl-redirect %i reload +PIDFile=/run/postgresql/%i.pid +SyslogIdentifier=postgresql@%i +# prevent OOM killer from choosing the postmaster (individual backends will +# reset the score to 0) +OOMScoreAdjust=-900 +# restarting automatically will prevent "pg_ctlcluster ... stop" from working, +# so we disable it here. Also, the postmaster will restart by itself on most +# problems anyway, so it is questionable if one wants to enable external +# automatic restarts. +#Restart=on-failure +# (This should make pg_ctlcluster stop work, but doesn't:) +#RestartPreventExitStatus=SIGINT SIGTERM + +[Install] +WantedBy=multi-user.target diff --git a/t/001_packages.t b/t/001_packages.t new file mode 100644 index 0000000..697e871 --- /dev/null +++ b/t/001_packages.t @@ -0,0 +1,87 @@ +# Check that the necessary packages are installed + +use warnings; +use strict; + +use lib 't'; +use TestLib; +use POSIX qw/setlocale LC_ALL LC_MESSAGES/; + +use Test::More tests => $PgCommon::rpm ? (3 + 9*@MAJORS) : (15 + 7*@MAJORS); + +ok (-f "/etc/os-release", "/etc/os-release exists"); +my ($os, $osversion) = os_release(); +ok (defined $os, "OS is $os"); +ok (defined $osversion, "OS version is $osversion"); + +note "PostgreSQL versions installed: @MAJORS\n"; +my $f = $ENV{'PG_FLAVOR'} // ''; + +if ($PgCommon::rpm) { + foreach my $v (@MAJORS) { + my $vv = $v; + $vv =~ s/\.//; + + ok ((rpm_installed "postgresql$vv$f"), "postgresql$vv$f installed"); + ok ((rpm_installed "postgresql$vv$f-libs"), "postgresql$vv$f-libs installed"); + ok ((rpm_installed "postgresql$vv$f-server"), "postgresql$vv$f-server installed"); + ok ((rpm_installed "postgresql$vv$f-contrib"), "postgresql$vv$f-contrib installed"); + ok ((rpm_installed "postgresql$vv$f-plperl"), "postgresql$vv$f-plperl installed"); + SKIP: { + skip "No python2 support", 1 unless ($v <= 12); + ok ((rpm_installed "postgresql$vv$f-plpython"), "postgresql$vv$f-plpython installed"); + } + ok ((rpm_installed "postgresql$vv$f-plpython3"), "postgresql$vv$f-plpython3 installed"); + ok ((rpm_installed "postgresql$vv$f-pltcl"), "postgresql$vv$f-pltcl installed"); + ok ((rpm_installed "postgresql$vv$f-devel"), "postgresql$vv$f-devel installed"); + } + exit; +} + +my $docpkgs = 0; +foreach my $v (@MAJORS) { + ok ((deb_installed "postgresql-$v$f"), "postgresql-$v$f installed"); + SKIP: { + skip "No python2 support", 1 unless ($v <= 11 and $PgCommon::have_python2); + ok ((deb_installed "postgresql-plpython-$v$f"), "postgresql-plpython-$v$f installed"); + } + if ($v >= '9.1') { + ok ((deb_installed "postgresql-plpython3-$v$f"), "postgresql-plpython3-$v$f installed"); + } else { + pass "no Python 3 package for version $v"; + } + ok ((deb_installed "postgresql-plperl-$v$f"), "postgresql-plperl-$v$f installed"); + ok ((deb_installed "postgresql-pltcl-$v$f"), "postgresql-pltcl-$v$f installed"); + ok ((deb_installed "postgresql-server-dev-$v$f"), "postgresql-server-dev-$v$f installed"); + SKIP: { + skip "No postgresql-contrib-$v$f package for version $v", 1 if ($v >= 10); + ok ((deb_installed "postgresql-contrib-$v$f"), "postgresql-contrib-$v$f installed"); + } + my $docpkg = "postgresql-doc-$v$f"; + if (deb_installed $docpkg) { + note "$docpkg installed"; + $docpkgs++; + } +} +ok $docpkgs, "At least one doc package installed"; + +ok ((deb_installed 'libecpg-dev'), 'libecpg-dev installed'); +ok ((deb_installed 'procps'), 'procps installed'); +ok ((deb_installed 'netcat-openbsd'), 'netcat-openbsd installed'); + +ok ((deb_installed 'hunspell-en-us'), 'hunspell-en-us installed'); + +# check installed locales to fail tests early if they are missing +ok ((setlocale(LC_MESSAGES, '') =~ /utf8|UTF-8/), 'system has a default UTF-8 locale'); +ok (setlocale (LC_ALL, "ru_RU"), 'locale ru_RU exists'); +ok (setlocale (LC_ALL, "ru_RU.UTF-8"), 'locale ru_RU.UTF-8 exists'); + +my $key_file = '/etc/ssl/private/ssl-cert-snakeoil.key'; +my $pem_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem'; +ok ((getgrnam('ssl-cert'))[3] =~ /postgres/, + 'user postgres in the UNIX group ssl-cert'); +ok (-e $key_file, "$key_file exists"); +is (exec_as ('postgres', "cat $key_file > /dev/null"), 0, "$key_file is readable for postgres"); +ok (-e $pem_file, "$pem_file exists"); + +# vim: filetype=perl diff --git a/t/002_existing_clusters.t b/t/002_existing_clusters.t new file mode 100644 index 0000000..93cc782 --- /dev/null +++ b/t/002_existing_clusters.t @@ -0,0 +1,11 @@ +# Check that no clusters and postgres processes are present for this test. + +use strict; +use Test::More tests => 8; + +use lib 't'; +use TestLib; + +check_clean; + +# vim: filetype=perl diff --git a/t/003_alternatives.t b/t/003_alternatives.t new file mode 100644 index 0000000..dd49d6b --- /dev/null +++ b/t/003_alternatives.t @@ -0,0 +1,37 @@ +# Check that alternatives symlinks point to the correct locations + +use warnings; +use strict; + +use lib 't'; +use TestLib; + +use Test::More tests => $PgCommon::rpm ? 1 : 9; + +if ($PgCommon::rpm) { + ok "No alternatives test on RedHat"; + exit; +} + +# server/client link group +my $newest_version = $ALL_MAJORS[-1]; +note "Newest PG version installed is $newest_version"; +program_ok 0, 'update-alternatives --list psql.1.gz', 0, 'psql.1.gz link group'; +program_ok 0, 'update-alternatives --list postmaster.1.gz', 2, 'postmaster.1.gz link group does not exist'; # removed in pg-common 248 +for my $name (qw(psql pg_dump postgres pg_ctl)) { + is readlink "/etc/alternatives/$name.1.gz", "/usr/share/postgresql/$newest_version/man/man1/$name.1.gz", "$name.1.gz alternative"; +} + +# doc link group +my $newest_doc_version = `dpkg -l 'postgresql-doc-[1-9]*' | sed -ne 's/^ii postgresql-doc-\\([0-9.]*\\).*/\\1/p' | sort -g | tail -n 1`; +note "Newest PG doc version installed is $newest_doc_version"; +SKIP: { + skip "No SPI_connect.3.gz link group on 8.x", 3 if ($newest_doc_version < 9.0); + chomp $newest_doc_version; + program_ok 0, 'update-alternatives --list SPI_connect.3.gz', 0, 'SPI_connect.3.gz link group'; + for my $name (qw(SPI_connect SPI_exec)) { + is readlink "/etc/alternatives/$name.3.gz", "/usr/share/postgresql/$newest_doc_version/man/man3/$name.3.gz", "$name.3.gz alternative"; + } +} + +# vim: filetype=perl diff --git a/t/005_PgCommon.t b/t/005_PgCommon.t new file mode 100644 index 0000000..e1bc337 --- /dev/null +++ b/t/005_PgCommon.t @@ -0,0 +1,311 @@ +# Check PgCommon library functions. + +use strict; + +use File::Temp qw/tempdir/; + +use lib '.'; +use PgCommon; + +use lib 't'; +use TestLib; + +use Test::More tests => 24; + +my $tdir = tempdir (CLEANUP => 1); +$PgCommon::confroot = $tdir; + +# test read_pg_hba with valid file +open P, ">$tdir/pg_hba.conf" or die "Could not create $tdir/pg_hba.conf: $!"; +print P <<EOF; +# comment +local all postgres ident sameuser + +# TYPE DATABASE USER CIDR-ADDRESS METHOD +local foo nobody trust +local foo nobody crypt +local foo nobody,joe krb5 +local foo,bar nobody ident +local all +foogrp password +host \@inc all 127.0.0.1/32 md5 +hostssl all \@inc 192.168.0.0 255.255.0.0 pam +hostnossl all all 192.168.0.0 255.255.0.0 reject +EOF +close P; + +my @expected_records = ( + { 'type' => 'local', 'db' => 'all', 'user' => 'postgres', 'method' => 'ident sameuser' }, + { 'type' => 'local', 'db' => 'foo', 'user' => 'nobody', 'method' => 'trust' }, + { 'type' => 'local', 'db' => 'foo', 'user' => 'nobody', 'method' => 'crypt' }, + { 'type' => 'local', 'db' => 'foo', 'user' => 'nobody,joe', 'method' => 'krb5' }, + { 'type' => 'local', 'db' => 'foo,bar', 'user' => 'nobody', 'method' => 'ident' }, + { 'type' => 'local', 'db' => 'all', 'user' => '+foogrp', 'method' => 'password' }, + { 'type' => 'host', 'db' => '@inc', 'user' => 'all', 'method' => 'md5', 'ip' => '127.0.0.1', 'mask' => '32'}, + { 'type' => 'hostssl', 'db' => 'all', 'user' => '@inc', 'method' => 'pam', 'ip' => '192.168.0.0', 'mask' => '255.255.0.0'}, + { 'type' => 'hostnossl', 'db' => 'all', 'user' => 'all', 'method' => 'reject', 'ip' => '192.168.0.0', 'mask' => '255.255.0.0'}, +); + +my @hba = read_pg_hba "$tdir/pg_hba.conf"; +foreach my $entry (@hba) { + next if $$entry{'type'} eq 'comment'; + if ($#expected_records < 0) { + fail '@expected_records is already empty'; + next; + } + my $expected = shift @expected_records; + my $parsedstr = ''; + my $expectedstr = ''; + foreach my $k (keys %$expected) { + $parsedstr .= $k . ':\'' . $$entry{$k} . '\' '; + $expectedstr .= $k . ':\'' . $$expected{$k} . '\' '; + if ($$expected{$k} ne $$entry{$k}) { + fail "mismatch: $expectedstr ne $parsedstr"; + last; + } + } + pass 'correctly parsed line \'' . $$entry{'line'} . "'"; +} + +ok (($#expected_records == -1), '@expected_records has correct number of entries'); + +# test read_pg_hba with invalid file +my $invalid_hba = <<EOF; +foo all all md5 +local all all foo +host all all foo +host all all 127.0.0.1/32 foo +host all all md5 +host all all 127.0.0.1/32 0.0.0.0 md5 +host all all 127.0.0.1 md5 +EOF +open P, ">$tdir/pg_hba.conf" or die "Could not create $tdir/pg_hba_invalid.conf: $!"; +print P $invalid_hba; +close P; + +@hba = read_pg_hba "$tdir/pg_hba.conf"; +is (scalar (split "\n", $invalid_hba), $#hba+1, 'returned read_pg_hba array has correct number of records'); +foreach my $entry (@hba) { + is $$entry{'type'}, undef, 'line \'' . $$entry{'line'} . '\' parsed as invalid'; +} + +# test read_conf_file() +my %conf = PgCommon::read_conf_file '/nonexisting'; +is_deeply \%conf, {}, 'read_conf_file returns empty dict for nonexisting file'; + +mkdir "$tdir/8.4"; +mkdir "$tdir/8.4/test" or die "mkdir: $!"; +mkdir "$tdir/conf.d" or die "mkdir: $!"; +my $c = "$tdir/8.4/test/foo.conf"; +open F, ">$c" or die "Could not create $c: $!"; +print F <<EOF; +# test configuration file + +# Commented_Int = 12 +# commented_str='foobar' +# commented_bool off +# commented_bool2 off # comment +# commented_bool3 just a comment + +#intval = 1 +Intval = 42 +cintval=1 # blabla +floatval = 1.5e+3 +strval 'hello' +strval2 'world' +cstrval = 'bye' # comment +emptystr = '' +cemptystr = '' # moo! +#testpath = '/bin/bad' +testpath = '/bin/test' +QuoteStr = 'test ! -f \\'/tmp/%f\\' && echo \\'yes\\'' +EOF +close F; +%conf = PgCommon::read_conf_file "$c"; +is_deeply (\%conf, { + 'intval' => 42, + 'cintval' => 1, + 'floatval' => '1.5e+3', + 'strval' => 'hello', + 'strval2' => 'world', + 'cstrval' => 'bye', + 'testpath' => '/bin/test', + 'emptystr' => '', + 'cemptystr' => '', + 'quotestr' => "test ! -f '/tmp/%f' && echo 'yes'" + }, 'read_conf_file() parsing'); + +# test read_conf_file() with include directives +open F, ">$tdir/8.4/test/condinc.conf" or die "Could not create $tdir/condinc.conf: $!"; +print F "condint = 42\n"; +close F; + +open F, ">$tdir/bar.conf" or die "Could not create $tdir/bar.conf: $!"; +print F <<EOF; +# test configuration file + +# Commented_Int = 24 +# commented_str = 'notme' + +intval = -1 +include '8.4/test/foo.conf' # foo +include_dir 'conf.d' # bar +strval = 'howdy' +include_if_exists '/nonexisting.conf' +include_if_exists = '8.4/test/condinc.conf' +EOF +close F; + +open F, ">$tdir/conf.d/sub.conf" or die "Could not create $tdir/conf.d/sub.conf: $!"; +print F <<EOF; +subvalue = 1 +include = '../relative.conf' +EOF +close F; + +open F, ">$tdir/relative.conf" or die "Could not create $tdir/relative.conf: $!"; +print F <<EOF; +relativevalue = 1 +include '$tdir/absolute.conf' +EOF +close F; + +open F, ">$tdir/absolute.conf" or die "Could not create $tdir/absolute.conf: $!"; +print F <<EOF; +absolutevalue = 1 +EOF +close F; + +%conf = PgCommon::read_conf_file "$tdir/bar.conf"; +is_deeply (\%conf, { + 'intval' => 42, + 'cintval' => 1, + 'floatval' => '1.5e+3', + 'strval' => 'howdy', + 'strval2' => 'world', + 'cstrval' => 'bye', + 'testpath' => '/bin/test', + 'emptystr' => '', + 'cemptystr' => '', + 'quotestr' => "test ! -f '/tmp/%f' && echo 'yes'", + 'condint' => 42, + 'subvalue' => 1, + 'relativevalue' => 1, + 'absolutevalue' => 1, + }, 'read_conf_file() parsing with include directives'); + + +# test set_conf_value() +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'commented_int', '24'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'commented_str', 'new foo'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'commented_bool', 'on'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'commented_bool2', 'on'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'commented_bool3', 'on'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'intval', '39'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'cintval', '5'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'strval', 'Howdy'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'newval', 'NEW!'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'testpath', '/bin/new'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'include_dir', 'conf.d'; + +open F, "$c"; +my $conf; +read F, $conf, 1024; +close F; +is ($conf, <<EOF, 'set_conf_value'); +# test configuration file + +Commented_Int = 24 +commented_str='new foo' +commented_bool on +commented_bool2 on # comment +# commented_bool3 just a comment + +#intval = 1 +Intval = 39 +cintval=5 # blabla +floatval = 1.5e+3 +strval Howdy +strval2 'world' +cstrval = 'bye' # comment +emptystr = '' +cemptystr = '' # moo! +#testpath = '/bin/bad' +testpath = '/bin/new' +QuoteStr = 'test ! -f \\'/tmp/%f\\' && echo \\'yes\\'' +commented_bool3 = on +newval = 'NEW!' +include_dir = 'conf.d' +EOF + +# test disable_conf_value() +PgCommon::disable_conf_value '8.4', 'test', 'foo.conf', 'intval', 'ints are out of fashion'; +PgCommon::disable_conf_value '8.4', 'test', 'foo.conf', 'cstrval', 'not used any more'; +PgCommon::disable_conf_value '8.4', 'test', 'foo.conf', 'nonexisting', 'NotMe'; +PgCommon::disable_conf_value '8.4', 'test', 'foo.conf', 'testpath', 'now 2 comments'; + +open F, "$c"; +read F, $conf, 1024; +close F; +is ($conf, <<EOF, 'disable_conf_value'); +# test configuration file + +Commented_Int = 24 +commented_str='new foo' +commented_bool on +commented_bool2 on # comment +# commented_bool3 just a comment + +#intval = 1 +#Intval = 39 #ints are out of fashion +cintval=5 # blabla +floatval = 1.5e+3 +strval Howdy +strval2 'world' +#cstrval = 'bye' # comment #not used any more +emptystr = '' +cemptystr = '' # moo! +#testpath = '/bin/bad' +#testpath = '/bin/new' #now 2 comments +QuoteStr = 'test ! -f \\'/tmp/%f\\' && echo \\'yes\\'' +commented_bool3 = on +newval = 'NEW!' +include_dir = 'conf.d' +EOF + +# test replace_conf_value() +PgCommon::replace_conf_value '8.4', 'test', 'foo.conf', 'strval', + 'renamedstrval', 'newstrval', 'goodbye'; +PgCommon::replace_conf_value '8.4', 'test', 'foo.conf', 'nonexisting', + 'renamednonexisting', 'newnonexisting', 'XXX'; + +open F, "$c"; +read F, $conf, 1024; +close F; +is ($conf, <<EOF, 'replace_conf_value'); +# test configuration file + +Commented_Int = 24 +commented_str='new foo' +commented_bool on +commented_bool2 on # comment +# commented_bool3 just a comment + +#intval = 1 +#Intval = 39 #ints are out of fashion +cintval=5 # blabla +floatval = 1.5e+3 +#strval Howdy #renamedstrval +newstrval = goodbye +strval2 'world' +#cstrval = 'bye' # comment #not used any more +emptystr = '' +cemptystr = '' # moo! +#testpath = '/bin/bad' +#testpath = '/bin/new' #now 2 comments +QuoteStr = 'test ! -f \\'/tmp/%f\\' && echo \\'yes\\'' +commented_bool3 = on +newval = 'NEW!' +include_dir = 'conf.d' +EOF + +# vim: filetype=perl diff --git a/t/006_next_free_port.t b/t/006_next_free_port.t new file mode 100644 index 0000000..4a82bf3 --- /dev/null +++ b/t/006_next_free_port.t @@ -0,0 +1,49 @@ +# Check PgCommon's next_free_port() + +use strict; + +use lib '.'; +use PgCommon; + +use lib 't'; +use TestLib; + +use Test::More tests => 5; + +# test next_free_port(). We are intentionally using nc as an external tool, +# using perl would replicate what next_free_port is doing, and that would +# be a pointless test. +use IPC::Open2; +use Time::HiRes qw(usleep); +my @pids; +# no ports open +is (next_free_port, 5432, 'next_free_port is 5432'); + +# open a localhost ipv4 socket +push @pids, open2(\*CHLD_OUT, \*CHLD_IN, qw(nc -4 -l 127.0.0.1 5432)); +usleep 2*$delay; +is (next_free_port, 5433, 'next_free_port detects localhost ipv4 socket'); +# open a wildcard ipv4 socket +push @pids, open2(\*CHLD_OUT, \*CHLD_IN, qw(nc -4 -l 5433)); +usleep 2*$delay; +is (next_free_port, 5434, 'next_free_port detects wildcard ipv4 socket'); + +SKIP: { + $^V =~ /^v(\d+\.\d+)/; # parse perl version + skip "perl <= 5.10 does not have proper IPv6 support", 2 if ($1 <= 5.10); + skip "skipping IPv6 tests", 2 if ($ENV{SKIP_IPV6}); + + # open a localhost ipv6 socket + push @pids, open2(\*CHLD_OUT, \*CHLD_IN, qw(nc -6 -l ::1 5434)); + usleep 2*$delay; + is (next_free_port, 5435, 'next_free_port detects localhost ipv6 socket'); + # open a wildcard ipv6 socket + push @pids, open2(\*CHLD_OUT, \*CHLD_IN, qw(nc -6 -l 5435)); + usleep 2*$delay; + is (next_free_port, 5436, 'next_free_port detects wildcard ipv6 socket'); +} + +# clean up +kill 15, @pids; + +# vim: filetype=perl diff --git a/t/007_pg_conftool.t b/t/007_pg_conftool.t new file mode 100644 index 0000000..e8465ef --- /dev/null +++ b/t/007_pg_conftool.t @@ -0,0 +1,85 @@ +# Test pg_conftool + +use strict; +use warnings; + +use Test::More tests => 41; +use File::Temp qw/tempdir/; +use lib '.'; +use PgCommon; +use lib 't'; +use TestLib; + +my $tdir = tempdir (CLEANUP => 1); +$ENV{'PG_CLUSTER_CONF_ROOT'} = $tdir; + +open F, "> $tdir/different.conf"; +print F "a = '5'\n"; +print F "#b = '6'\n"; +close F; + +note 'test without cluster'; +is_program_out 0, "pg_conftool show all", 1, "Error: No default cluster found\n"; +is_program_out 0, "pg_conftool foo.conf show all", 1, "Error: No default cluster found\n"; +is_program_out 0, "pg_conftool $tdir/different.conf show all", 0, "a = 5\n"; +is_program_out 0, "pg_conftool 9.7 main show all", 1, "Error: Cluster 9.7 main does not exist\n"; + +my $version = $MAJORS[-1]; +die "Tests past this point need PostgreSQL installed" unless ($version); +mkdir "$tdir/$version"; +mkdir "$tdir/$version/main"; + +open F, "> $tdir/$version/main/postgresql.conf"; +print F "a = '1'\n"; +print F "#b = '2'\n"; +close F; + +open F, "> $tdir/$version/main/other.conf"; +print F "a = '3'\n"; +print F "#b = '4'\n"; +close F; + +sub pgconf { + undef $/; + open F, "$tdir/$version/main/postgresql.conf"; + my $f = <F>; + close F; + return $f; +} + +sub differentconf { + undef $/; + open F, "$tdir/different.conf"; + my $f = <F>; + close F; + return $f; +} + +note 'test show'; +is_program_out 0, "pg_conftool show all", 0, "a = 1\n"; +is_program_out 0, "pg_conftool other.conf show all", 0, "a = 3\n"; +is_program_out 0, "pg_conftool $tdir/different.conf show all", 0, "a = 5\n"; +is_program_out 0, "pg_conftool $version main show all", 0, "a = 1\n"; +is_program_out 0, "pg_conftool $version main other.conf show all", 0, "a = 3\n"; +is_program_out 0, "pg_conftool show a", 0, "a = 1\n"; +is_program_out 0, "pg_conftool -s show a", 0, "1\n"; + +note 'test set'; +is_program_out 0, "pg_conftool set c 7", 0, ""; +undef $/; # slurp mode +is pgconf, "a = '1'\n#b = '2'\nc = 7\n", "file contains new setting"; +is_program_out 0, "pg_conftool set a 8", 0, ""; +is pgconf, "a = 8\n#b = '2'\nc = 7\n", "file contains updated setting"; +is_program_out 0, "pg_conftool $tdir/different.conf set a 9", 0, ""; +is differentconf, "a = 9\n#b = '6'\n", "file with path contains updated setting"; + +note 'test remove'; +is_program_out 0, "pg_conftool remove a", 0, ""; +is pgconf, "#a = 8\n#b = '2'\nc = 7\n", "setting removed from file"; +is_program_out 0, "pg_conftool $tdir/different.conf remove a", 0, ""; +is differentconf, "#a = 9\n#b = '6'\n", "setting removed from file with path"; + +note 'test edit'; +$ENV{EDITOR} = 'cat'; +is_program_out 0, "pg_conftool edit", 0, "#a = 8\n#b = '2'\nc = 7\n"; +is_program_out 0, "pg_conftool $tdir/different.conf edit", 0, "#a = 9\n#b = '6'\n"; diff --git a/t/010_defaultport_cluster.t b/t/010_defaultport_cluster.t new file mode 100644 index 0000000..1ba0daa --- /dev/null +++ b/t/010_defaultport_cluster.t @@ -0,0 +1,33 @@ +# We try to call psql with --version and then on localhost. Since there are no +# clusters, we expect an error message that the connection to port 5432 is +# refused. This checks that pg_wrapper correctly picks the default port and +# uses the highest available version. + +use strict; +use Test::More tests => 14; + +use lib 't'; +use TestLib; + +like_program_out 0, 'psql --version', 0, qr/^psql \(PostgreSQL\) $ALL_MAJORS[-1]/, + 'pg_wrapper selects highest available version number'; + +like_program_out 0, 'env LC_MESSAGES=C psql -h 127.0.0.1 -l', 2, qr/could not connect|connection to server .* failed/, + 'connecting to localhost fails with no clusters'; + +# We check if PGCLUSTER, --cluster, and native psql options are evaluated with +# correct priority. (This is related to the checks in t/090_multicluster.t, but +# easier to do here because no clusters are running.) + +like_program_out 0, "env LC_MESSAGES=C PGCLUSTER=$MAJORS[-1]/127.0.0.2:5431 psql -l", + 2, qr/(could not connect|connection to server).*127.0.0.2.* port 5431/s, 'pg_wrapper uses host and port from PGCLUSTER'; +like_program_out 0, "env LC_MESSAGES=C PGCLUSTER=$MAJORS[-1]/127.0.0.2:5431 psql --cluster $MAJORS[-1]/127.0.0.3:5430 -l", + 2, qr/(could not connect|connection to server).*127.0.0.3.* port 5430/s, 'pg_wrapper uses --cluster from the command line'; +like_program_out 0, "env LC_MESSAGES=C PGCLUSTER=$MAJORS[-1]/127.0.0.2:5431 psql -h 127.0.0.3 -l", + 2, qr/(could not connect|connection to server).*127.0.0.3.* port 5432/s, 'pg_wrapper ignores PGCLUSTER with -h on the command line'; +like_program_out 0, "env LC_MESSAGES=C PGCLUSTER=$MAJORS[-1]/127.0.0.2:5431 psql --host 127.0.0.3 -l", + 2, qr/(could not connect|connection to server).*127.0.0.3.* port 5432/s, 'pg_wrapper ignores PGCLUSTER with --host on the command line'; +like_program_out 0, "env LC_MESSAGES=C PGCLUSTER=$MAJORS[-1]/127.0.0.2:5431 PGHOST=127.0.0.3 psql -l", + 2, qr/(could not connect|connection to server).*127.0.0.3.* port 5432/s, 'pg_wrapper ignores PGCLUSTER if PGHOST is set'; + +# vim: filetype=perl diff --git a/t/015_start_stop.t b/t/015_start_stop.t new file mode 100644 index 0000000..a6c4ebf --- /dev/null +++ b/t/015_start_stop.t @@ -0,0 +1,174 @@ +use strict; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More tests => 71 * @MAJORS; + +my $systemd = (-d "/run/systemd/system" and not $ENV{_SYSTEMCTL_SKIP_REDIRECT}); +note $systemd ? "We are running systemd" : "We are not running systemd"; + +# check cluster status +# arguments: <version> <pg_ctlcluster exit status> <systemctl exit status> <text to print> +sub check_status { + my ($v, $ctlstatus, $scstatus, $text) = @_; + program_ok (0, "pg_ctlcluster $v main status", $ctlstatus, "cluster $v main $text"); + if ($systemd) { + program_ok (0, "systemctl status postgresql\@$v-main", $scstatus, "service postgresql\@$v-main $text"); + } else { + pass ''; + } +} + +sub check_major { + my $v = $_[0]; + my $ctlstopped = $v >= 9.2 ? 3 : 1; # pg_ctl status "not running" changed in 9.2 + note "Running tests for $v"; + + note "Cluster does not exist yet"; ############################### + check_status $v, 1, 3, "does not exist"; + + # try to start postgresql + if ($systemd) { + program_ok (0, "systemctl start postgresql"); + } else { + program_ok (0, "/etc/init.d/postgresql start"); + } + check_status $v, 1, 3, "does not exist"; + + # try to start specific cluster + if ($systemd) { + program_ok (0, "systemctl start postgresql\@$v-main", 1); + } else { + program_ok (0, "/etc/init.d/postgresql start $v"); + } + check_status $v, 1, 3, "does not exist"; + + note "Start/stop postgresql using system tools"; ############################### + + # create cluster + program_ok (0, "pg_createcluster $v main"); + check_status $v, $ctlstopped, 3, "is stopped"; + + # start postgresql + if ($systemd) { + program_ok (0, "systemctl start postgresql"); + } else { + program_ok (0, "/etc/init.d/postgresql start"); + } + check_status $v, 0, 0, "is running"; + + # start postgresql again + if ($systemd) { + program_ok (0, "systemctl start postgresql"); + } else { + program_ok (0, "/etc/init.d/postgresql start"); + } + check_status $v, 0, 0, "is already running"; + + # stop postgresql + if ($systemd) { + program_ok (0, "systemctl stop postgresql"); + sleep 6; # FIXME: systemctl stop postgresql is not yet synchronous (#759725) + } else { + program_ok (0, "/etc/init.d/postgresql stop"); + } + check_status $v, $ctlstopped, 3, "is stopped"; + + # stop postgresql again + if ($systemd) { + program_ok (0, "systemctl stop postgresql"); + } else { + program_ok (0, "/etc/init.d/postgresql stop"); + } + check_status $v, $ctlstopped, 3, "is already stopped"; + + note "Start/stop specific cluster using system tools"; ############################### + + # start cluster using system tools + if ($systemd) { + program_ok (0, "systemctl start postgresql\@$v-main"); + } else { + program_ok (0, "/etc/init.d/postgresql start $v"); + } + check_status $v, 0, 0, "is running"; + + # try start cluster again + if ($systemd) { + program_ok (0, "systemctl start postgresql\@$v-main"); + } else { + program_ok (0, "/etc/init.d/postgresql start $v"); + } + check_status $v, 0, 0, "is running"; + + # restart cluster + if ($systemd) { + program_ok (0, "systemctl restart postgresql\@$v-main"); + } else { + program_ok (0, "/etc/init.d/postgresql restart $v"); + } + check_status $v, 0, 0, "is running"; + + # stop cluster + if ($systemd) { + program_ok (0, "systemctl stop postgresql\@$v-main"); + } else { + program_ok (0, "/etc/init.d/postgresql stop $v"); + } + check_status $v, $ctlstopped, 3, "is stopped"; + + # try to stop cluster again + if ($systemd) { + program_ok (0, "systemctl stop postgresql\@$v-main"); + } else { + program_ok (0, "/etc/init.d/postgresql stop $v"); + } + check_status $v, $ctlstopped, 3, "is already stopped"; + + # drop cluster + program_ok (0, "pg_dropcluster $v main"); + check_status $v, 1, 3, "does not exist"; + + note "Start/stop specific cluster using pg_*cluster"; ############################### + + # try to start cluster + program_ok (0, "pg_ctlcluster start $v main", 1); # syntax variation: action version cluster + check_status $v, 1, 3, "does not exist"; + + # create cluster and start it + program_ok (0, "pg_createcluster $v main --start"); + check_status $v, 0, 0, "is running"; + + # try to start cluster again + my $exitagain = $systemd ? 0 : 2; + program_ok (0, "pg_ctlcluster $v main start", $exitagain); + check_status $v, 0, 0, "is already running"; + + # restart cluster + program_ok (0, "pg_ctlcluster $v-main restart"); # syntax variation: version-cluster action + check_status $v, 0, 0, "is running"; + + # stop cluster + program_ok (0, "pg_ctlcluster $v main stop"); + check_status $v, $ctlstopped, 3, "is stopped"; + + # try to stop cluster again + program_ok (0, "pg_ctlcluster $v main stop", 2); + check_status $v, $ctlstopped, 3, "is already stopped"; + + # start cluster + program_ok (0, "pg_ctlcluster start $v-main"); # syntax variation: action version-cluster + check_status $v, 0, 0, "is running"; + + # stop server, clean up, check for leftovers + program_ok (0, "pg_dropcluster $v main --stop"); + + check_clean; +} + +foreach (@MAJORS) { + check_major $_; +} + +# vim: filetype=perl diff --git a/t/020_create_sql_remove.t b/t/020_create_sql_remove.t new file mode 100644 index 0000000..d208b99 --- /dev/null +++ b/t/020_create_sql_remove.t @@ -0,0 +1,452 @@ +# We create a cluster, execute some basic SQL commands, drop it again, and +# check that we did not leave anything behind. + +use strict; + +use File::Temp qw(tempdir); +use POSIX qw/dup2/; +use Time::HiRes qw/usleep/; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More tests => 147 * @MAJORS; + +$ENV{_SYSTEMCTL_SKIP_REDIRECT} = 1; # FIXME: testsuite is hanging otherwise + +sub check_major { + my $v = $_[0]; + note "Running tests for $v"; + + # create cluster + my $xlogdir = tempdir("/tmp/$v.xlog.XXXXXX", CLEANUP => 1); + rmdir $xlogdir; # recreated by initdb + if ($v > 8.2) { + my $start_command = $v >= 14 ? "$v *main *5432 *online" : # initdb --no-instructions in 14+ + ($v >= 11 and not $PgCommon::rpm) ? "pg_ctlcluster" : "pg_ctl"; # CLUSTER_START_COMMAND supported in initdb 11+ + like_program_out 'root', "pg_createcluster $v main --start -- -X $xlogdir", 0, qr/$start_command/, + "pg_createcluster $v main"; + } else { # 8.2 does not have -X yet + like_program_out 'root', "pg_createcluster $v main --start", 0, qr/pg_ctl/, + "pg_createcluster $v main"; + system "mv /var/lib/postgresql/$v/main/pg_xlog $xlogdir"; + system "ln -s $xlogdir /var/lib/postgresql/$v/main/pg_xlog"; + } + + # check that a /var/run/postgresql/ pid file is created + my @contents = ('.s.PGSQL.5432', '.s.PGSQL.5432.lock', "$v-main.pid", "$v-main.pg_stat_tmp"); + pop @contents if ($v < 8.4 or $v >= 15); # remove pg_stat_tmp + unless ($PgCommon::rpm and $v < 9.4) { + ok_dir '/var/run/postgresql/', [@contents], + 'Socket and pid file are in /var/run/postgresql/'; + } else { + ok_dir '/var/run/postgresql/', [grep {/main/} @contents], 'Pid File is in /var/run/postgresql/'; + } + + # check that the xlog/wal symlink was created + my $first_xlog = $v >= 9.0 ? "000000010000000000000001" : "000000010000000000000000"; + my @expectdir = ($first_xlog, "archive_status"); + push @expectdir, "summaries" if ($v >= 17); + ok_dir $xlogdir, [@expectdir], + "xlog/wal directory $xlogdir was created"; + + # check pg_hba.conf auth methods + my $local_method = $v >= 9.1 ? 'peer' : + ($v >= 8.4 ? 'ident' : + 'sameuser'); # actually "ident sameuser", but the test is lazy + my $host_method = $v >= 14 ? 'scram-sha-256' : 'md5'; + my (%local_methods, %host_methods); + open my $fh, "/etc/postgresql/$v/main/pg_hba.conf"; + while (<$fh>) { + $local_methods{$1} = 1 if (/^local.*\s(\S+)/); + $host_methods{$1} = 1 if (/^host.*\s(\S+)/); + } + close $fh; + is_deeply [keys %local_methods], [$local_method], "local method in pg_hba.conf is $local_method"; + is_deeply [keys %host_methods], [$host_method], "host method in pg_hba.conf is $host_method"; + + # verify that exactly one postgres master is running + my @pm_pids = pidof ('postgres'); + is $#pm_pids, 0, 'Exactly one postgres master process running'; + + # check environment + my %safe_env = qw/LC_ALL 1 LC_CTYPE 1 LANG 1 PWD 1 PGLOCALEDIR 1 PGSYSCONFDIR 1 PG_GRANDPARENT_PID 1 PG_OOM_ADJUST_FILE 1 PG_OOM_ADJUST_VALUE 1 SHLVL 1 PGDATA 1 _ 1/; + my %env = pid_env 'postgres', $pm_pids[0]; + foreach (keys %env) { + fail "postgres has unsafe environment variable $_" unless exists $safe_env{$_}; + } + + # activate external_pid_file + PgCommon::set_conf_value $v, 'main', 'postgresql.conf', 'external_pid_file', ''; + + # add variable to environment file, restart, check if it's there + open E, ">>/etc/postgresql/$v/main/environment" or + die 'could not open environment file for appending'; + print E "PGEXTRAVAR1 = 1 # short one\nPGEXTRAVAR2='foo bar '\n\n# comment"; + close E; + is_program_out 0, "pg_ctlcluster $v main restart", 0, '', + 'cluster restarts with new environment file'; + + @pm_pids = pidof ('postgres'); + is $#pm_pids, 0, 'Exactly one postgres master process running'; + %env = pid_env 'postgres', $pm_pids[0]; + is $env{'PGEXTRAVAR1'}, '1', 'correct value of PGEXTRAVAR1 in environment'; + is $env{'PGEXTRAVAR2'}, 'foo bar ', 'correct value of PGEXTRAVAR2 in environment'; + + # Now there should not be an external PID file any more, since we set it + # explicitly + unless ($PgCommon::rpm and ($v < 9.4 or $v >= 15)) { + ok_dir '/var/run/postgresql', [grep {! /pid/} @contents], + 'Socket and stats dir, but not PID file in /var/run/postgresql/'; + } else { + ok_dir '/var/run/postgresql', ["$v-main.pg_stat_tmp"], 'Only stats dir in /var/run/postgresql/'; + } + + # verify that the correct client version is selected + like_program_out 'postgres', 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $v/, + 'pg_wrapper+createdb selects version number of cluster'; + + # we always want to use the latest version of "psql", though. + my $max_version = $ALL_MAJORS[-1]; + if ($v < 9.2) { + # if version is older than 9.2 pick v14 at most + $max_version = (grep { $_ <= 14 } @ALL_MAJORS)[-1]; + } + like_program_out 'postgres', 'psql --version', 0, qr/^psql \(PostgreSQL\) $max_version/, + "pg_wrapper+psql selects version $max_version"; + + my $default_log = "/var/log/postgresql/postgresql-$v-main.log"; + + # verify that the cluster is displayed + my $ls = `pg_lsclusters -h`; + $ls =~ s/\s+/ /g; + $ls =~ s/\s*$//; + is $ls, "$v main 5432 online postgres /var/lib/postgresql/$v/main $default_log", + 'pg_lscluster reports online cluster on port 5432'; + + # verify that the log file is actually used + ok !-z $default_log, 'log file is actually used'; + + # verify configuration file permissions + my $postgres_uid = (getpwnam 'postgres')[2]; + my @st = stat "/etc/postgresql"; + is $st[4], $postgres_uid, '/etc/postgresql is owned by user "postgres"'; + my @st = stat "/etc/postgresql/$v"; + is $st[4], $postgres_uid, 'version configuration directory file is owned by user "postgres"'; + my @st = stat "/etc/postgresql/$v/main"; + is $st[4], $postgres_uid, 'configuration directory file is owned by user "postgres"'; + + # verify data file permissions + my @st = stat "/var/lib/postgresql/$v"; + is $st[4], $postgres_uid, 'version data directory file is owned by user "postgres"'; + my @st = stat "/var/lib/postgresql/$v/main"; + is $st[4], $postgres_uid, 'data directory file is owned by user "postgres"'; + + # verify log file permissions + my @logstat = stat $default_log; + is $logstat[2], 0100640, 'log file has 0640 permissions'; + is $logstat[4], $postgres_uid, 'log file is owned by user "postgres"'; + is $logstat[5], (getgrnam 'adm')[2], 'log file is owned by group "adm"'; + + # check default log file configuration; when not specifying -l with + # pg_createcluster, we should not have a 'log' symlink + ok !-e "/etc/postgresql/$v/main/log", 'no log symlink by default'; + ok !-z $default_log, "$default_log is the default log if log symlink is missing"; + like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/^$v\s+main.*$default_log\n$/; + + # verify that log symlink works + is ((exec_as 'root', "pg_ctlcluster $v main stop"), 0, 'stopping cluster'); + usleep $delay; + truncate "$default_log", 0; # empty log file + my $p = (PgCommon::cluster_data_directory $v, 'main') . '/mylog'; + symlink $p, "/etc/postgresql/$v/main/log"; + is ((exec_as 'root', "pg_ctlcluster $v main start"), 0, + 'restarting cluster with nondefault log symlink'); + ok !-z $p, "log target is used as log file"; + ok -z $default_log, "default log is not used"; + like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/^$v\s+main.*$p\n$/; + is ((exec_as 'root', "pg_ctlcluster $v main stop"), 0, 'stopping cluster'); + usleep $delay; + truncate "$default_log", 0; # empty log file + + # verify that explicitly configured log file trumps log symlink + PgCommon::set_conf_value ($v, 'main', 'postgresql.conf', + ($v >= '8.3' ? 'logging_collector' : 'redirect_stderr'), 'on'); + PgCommon::set_conf_value $v, 'main', 'postgresql.conf', 'log_filename', "$v#main.log"; + is ((exec_as 'root', "pg_ctlcluster $v main start"), 0, + 'restarting cluster with explicitly configured log file'); + ok -z $default_log, "default log is not used"; + ok !-z $p, "log symlink target is used for startup message"; + my $pg_log = $v >= 10 ? 'log' : 'pg_log'; # log directory in PGDATA changed in PG 10 + my @l = glob ((PgCommon::cluster_data_directory $v, 'main') . "/$pg_log/$v#main.log*"); + is $#l, 0, 'exactly one log file'; + ok (-e $l[0] && ! -z $l[0], 'custom log is actually used'); + SKIP: { skip "no logging_collector in $v", 2 if ($v < 8.3); + like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/^$v\s+main.*$v#main.log\n$/; + } + + # clean up + PgCommon::disable_conf_value ($v, 'main', 'postgresql.conf', + ($v >= '8.3' ? 'logging_collector' : 'redirect_stderr'), ''); + PgCommon::disable_conf_value $v, 'main', 'postgresql.conf', 'log_filename', ''; + unlink "/etc/postgresql/$v/main/log"; + + # check that log creation does not escalate privileges + program_ok 'root', "pg_ctlcluster $v main stop", 0, 'stopping cluster'; + unlink $default_log; + symlink "/etc/postgres-hack", $default_log; + program_ok 'root', "pg_ctlcluster $v main start", 1, 'starting cluster with rouge /var/log/postgresql symlink fails'; + ok !-f "/etc/postgres-hack", "/etc/postgres-hack was not created"; + unlink $default_log; + program_ok 'root', "pg_ctlcluster $v main start", 0, 'restarting cluster'; + + # verify that processes do not have an associated terminal + unlike_program_out 0, 'ps -o tty -U postgres h', 0, qr/tty|pts/, + 'postgres processes do not have an associated terminal'; + + # verify that SSL is enabled (which should work for user postgres in a + # default installation) + my $ssl = config_bool (PgCommon::get_conf_value $v, 'main', 'postgresql.conf', 'ssl'); + my $ssl_linked = `ldd $PgCommon::binroot$v/bin/postgres | grep libssl`; + my ($os, $osversion) = os_release(); + if ($PgCommon::rpm) { + isnt $ssl_linked, '', 'Server is linked with SSL support'; + is $ssl, undef, 'SSL is disabled in postgresql.conf'; + } elsif ($v <= 9.1 and (($os eq 'debian' and ($osversion eq 'unstable' or $osversion > 9)) or # stretch had 1.0 and 1.1 + ($os eq 'ubuntu' and $osversion > 18.04))) { # bionic had 1.0 and 1.1 + is $ssl_linked, '', 'Server is linked without SSL support (old version with only OpenSSL 1.0 support)'; + is $ssl, undef, 'SSL is disabled in postgresql.conf'; + } else { + isnt $ssl_linked, '', 'Server is linked with SSL support'; + is $ssl, 1, 'SSL is enabled in postgresql.conf'; + } + + # Create user nobody, a database 'nobodydb' for him, check the database list + my $outref; + is ((exec_as 'nobody', 'psql -l 2>/dev/null', $outref), 2, 'psql -l fails for nobody'); + is ((exec_as 'postgres', 'createuser nobody -D -R -S'), 0, 'createuser nobody'); + is ((exec_as 'postgres', 'createdb -O nobody nobodydb'), 0, 'createdb nobodydb'); + is ((exec_as 'nobody', 'psql -ltA|grep "|" | cut -f1-3 -d"|"', $outref), 0, 'psql -ltA succeeds for nobody'); + is ($$outref, 'nobodydb|nobody|UTF8 +postgres|postgres|UTF8 +template0|postgres|UTF8 +template1|postgres|UTF8 +', 'psql -ltA output'); + + # Then fill nobodydb with some data. + is ((exec_as 'nobody', 'psql nobodydb -c "create table phone (name varchar(255) PRIMARY KEY, tel int NOT NULL)" 2>/dev/null'), + 0, 'SQL command: create table'); + is ((exec_as 'nobody', 'psql nobodydb -c "insert into phone values (\'Bob\', 1)"'), 0, 'SQL command: insert into table values'); + is ((exec_as 'nobody', 'psql nobodydb -c "insert into phone values (\'Alice\', 2)"'), 0, 'SQL command: insert into table values'); + is ((exec_as 'nobody', 'psql nobodydb -c "insert into phone values (\'Bob\', 3)"'), 1, 'primary key violation'); + + # Check table contents + is_program_out 'nobody', 'psql -tAc "select * from phone order by name" nobodydb', 0, + 'Alice|2 +Bob|1 +', 'SQL command output: select -tA'; + is_program_out 'nobody', 'psql -txc "select * from phone where name = \'Alice\'" nobodydb', 0, + 'name | Alice +tel | 2 + +', 'SQL command output: select -tx'; + is_program_out 'nobody', 'psql -tAxc "select * from phone where name = \'Alice\'" nobodydb', 0, + 'name|Alice +tel|2 +', 'SQL command output: select -tAx'; + + sub create_extension ($$) { + my ($v, $extension) = @_; + return "psql -qc 'CREATE EXTENSION $extension' nobodydb" if ($v >= 9.1); + return "createlang --cluster $v/main $extension nobodydb"; + } + + # Check PL/Perl untrusted + my $fn_cmd = 'CREATE FUNCTION read_file() RETURNS text AS \'open F, \\"/etc/passwd\\"; \\$buf = <F>; close F; return \\$buf;\' LANGUAGE plperl'; + is ((exec_as 'nobody', create_extension($v, 'plperlu')), 1, 'CREATE EXTENSION plperlu fails for user nobody'); + is_program_out 'postgres', create_extension($v, 'plperlu'), 0, '', 'CREATE EXTENSION plperlu succeeds for user postgres'; + is ((exec_as 'nobody', "psql nobodydb -qc \"${fn_cmd}u;\""), 1, 'creating PL/PerlU function as user nobody fails'); + is ((exec_as 'postgres', "psql nobodydb -qc \"${fn_cmd};\""), 1, 'creating unsafe PL/Perl function as user postgres fails'); + is_program_out 'postgres', "psql nobodydb -qc \"${fn_cmd}u;\"", 0, '', 'creating PL/PerlU function as user postgres succeeds'; + like_program_out 'nobody', 'psql nobodydb -Atc "select read_file()"', + 0, qr/^root:/, 'calling PL/PerlU function'; + + # Check PL/Perl trusted + my $pluser = ($v >= '8.3') ? 'nobody' : 'postgres'; # pg_pltemplate allows non-superusers to install trusted languages in 8.3+ + is_program_out $pluser, create_extension($v, 'plperl'), 0, '', "CREATE EXTENSION plperl succeeds for user $pluser"; + is ((exec_as 'nobody', "psql nobodydb -qc \"${fn_cmd};\""), 1, 'creating unsafe PL/Perl function as user nobody fails'); + is_program_out 'nobody', 'psql nobodydb -qc "CREATE FUNCTION remove_vowels(text) RETURNS text AS \'\\$_[0] =~ s/[aeiou]/_/ig; return \\$_[0];\' LANGUAGE plperl;"', + 0, '', 'creating PL/Perl function as user nobody succeeds'; + is_program_out 'nobody', 'psql nobodydb -Atc "select remove_vowels(\'foobArish\')"', + 0, "f__b_r_sh\n", 'calling PL/Perl function'; + + # Check PL/Python (untrusted) + SKIP: { + skip "No python2 support", 6 unless ($v <= 11 and $PgCommon::have_python2); + is_program_out 'postgres', create_extension($v, 'plpythonu'), 0, '', 'CREATE EXTENSION plpythonu succeeds for user postgres'; + is_program_out 'postgres', 'psql nobodydb -qc "CREATE FUNCTION capitalize(text) RETURNS text AS \'import sys; return args[0].capitalize() + sys.version[0]\' LANGUAGE plpythonu;"', + 0, '', 'creating PL/Python function as user postgres succeeds'; + is_program_out 'nobody', 'psql nobodydb -Atc "select capitalize(\'foo\')"', + 0, "Foo2\n", 'calling PL/Python function'; + } + + # Check PL/Python3 (untrusted) + if ($v >= '9.1') { + is_program_out 'postgres', create_extension($v, 'plpython3u'), 0, '', 'CREATE EXTENSION plpython3u succeeds for user postgres'; + is_program_out 'postgres', 'psql nobodydb -qc "CREATE FUNCTION capitalize3(text) RETURNS text AS \'import sys; return args[0].capitalize() + sys.version[0]\' LANGUAGE plpython3u;"', + 0, '', 'creating PL/Python3 function as user postgres succeeds'; + is_program_out 'nobody', 'psql nobodydb -Atc "select capitalize3(\'foo\')"', + 0, "Foo3\n", 'calling PL/Python function'; + } else { + pass "Skipping PL/Python3 test for version $v..."; + pass '...'; + pass '...'; + pass '...'; + pass '...'; + pass '...'; + } + + # Check PL/Tcl (trusted/untrusted) + is_program_out 'postgres', create_extension($v, 'pltcl'), 0, '', 'CREATE EXTENSION pltcl succeeds for user postgres'; + is_program_out 'postgres', create_extension($v, 'pltclu'), 0, '', 'CREATE EXTENSION pltclu succeeds for user postgres'; + is_program_out 'nobody', 'psql nobodydb -qc "CREATE FUNCTION tcl_max(integer, integer) RETURNS integer AS \'if {\\$1 > \\$2} {return \\$1}; return \\$2\' LANGUAGE pltcl STRICT;"', + 0, '', 'creating PL/Tcl function as user nobody succeeds'; + is_program_out 'postgres', 'psql nobodydb -qc "CREATE FUNCTION tcl_max_u(integer, integer) RETURNS integer AS \'if {\\$1 > \\$2} {return \\$1}; return \\$2\' LANGUAGE pltclu STRICT;"', + 0, '', 'creating PL/TclU function as user postgres succeeds'; + is_program_out 'nobody', 'psql nobodydb -Atc "select tcl_max(3,4)"', 0, + "4\n", 'calling PL/Tcl function'; + is_program_out 'nobody', 'psql nobodydb -Atc "select tcl_max_u(5,4)"', 0, + "5\n", 'calling PL/TclU function'; + + # fake rotated logs to check that they are cleaned up properly + open L, ">$default_log.1" or die "could not open fake rotated log file"; + print L "old log .1\n"; + close L; + open L, ">$default_log.2" or die "could not open fake rotated log file"; + print L "old log .2\n"; + close L; + if (system "gzip -9 $default_log.2") { + die "could not gzip fake rotated log"; + } + + # Check that old-style pgdata symbolic link still works (p-common 0.90+ + # does not create them any more, but they still need to work for existing + # installations) + is ((exec_as 'root', "pg_ctlcluster $v main stop"), 0, 'stopping cluster'); + my $datadir = PgCommon::get_conf_value $v, 'main', 'postgresql.conf', 'data_directory'; + symlink $datadir, "/etc/postgresql/$v/main/pgdata"; + + # data_directory should trump the pgdata symlink + PgCommon::set_conf_value $v, 'main', 'postgresql.conf', 'data_directory', '/nonexisting'; + like_program_out 0, "pg_ctlcluster $v main start", 1, + qr/\/nonexisting is not accessible/, + 'cluster fails to start with invalid data_directory and valid pgdata symlink'; + + # if only pgdata symlink is present, it is authoritative + PgCommon::disable_conf_value $v, 'main', 'postgresql.conf', 'data_directory', 'disabled for test'; + is_program_out 0, "pg_ctlcluster $v main start", 0, '', + 'cluster restarts with pgdata symlink'; + + # check properties of backend processes + pipe RH, WH; + my $psql = fork; + if (!$psql) { + close WH; + my @pw = getpwnam 'nobody'; + change_ugid $pw[2], $pw[3]; + open(STDIN, "<& RH"); + dup2(POSIX::open('/dev/null', POSIX::O_WRONLY), 1); + exec 'psql', '-Xq', '-vPROMPT1=', 'nobodydb' or die "could not exec psql process: $!"; + } + close RH; + select WH; $| = 1; # make unbuffered + + open my $pidfile, "/var/lib/postgresql/$v/main/postmaster.pid"; + my $master_pid = <$pidfile>; + chomp $master_pid; + close $pidfile; + + my $client_pid; + while (!$client_pid) { + usleep $delay; + $client_pid = `ps --user postgres hu | grep 'postgres.*: nobody nobodydb' | grep -v grep | awk '{print \$2}'`; + ($client_pid) = ($client_pid =~ /(\d+)/); # untaint + } + + # OOM score adjustment under Linux: postmaster gets bigger shields for >= + # 9.0, but client backends stay at default; this might not work in + # containers with restricted privileges, so skip the check there + my $adj; + my $detect_virt = system 'systemd-detect-virt --container --quiet'; # from systemd + open F, "/proc/$master_pid/oom_score_adj"; + $adj = <F>; + chomp $adj; + close F; + if ($v >= '9.0' and not $PgCommon::rpm) { + SKIP: { + skip 'skipping postmaster OOM killer adjustment in container', 1 if $detect_virt == 0; + cmp_ok $adj, '<=', -500, 'postgres master has OOM killer protection'; + } + } else { + is $adj, 0, 'postgres master has no OOM adjustment'; + } + + open F, "/proc/$client_pid/oom_score_adj"; + $adj = <F>; + chomp $adj; + close F; + is $adj, 0, 'postgres client backend has no OOM adjustment'; + + # test process title update + like_program_out 0, "ps h $client_pid", 0, qr/ idle\s*$/, 'process title is idle'; + print WH "BEGIN;\n"; + usleep $delay; + like_program_out 0, "ps h $client_pid", 0, qr/idle in transaction/, 'process title is idle in transaction'; + + close WH; + kill 15, $psql; + waitpid $psql, 0; + + # log file gets re-created by pg_ctlcluster + is ((exec_as 0, "pg_ctlcluster $v main stop"), 0, 'stopping cluster'); + unlink $default_log; + is ((exec_as 0, "pg_ctlcluster $v main start"), 0, 'starting cluster as postgres works without a log file'); + ok (-e $default_log && ! -z $default_log, 'log file got recreated and used'); + + # create tablespaces + my $spc1 = tempdir("/tmp/$v.spc1.XXXXXX", CLEANUP => 1); + my $spc2 = tempdir("/tmp/$v.spc2.XXXXXX", CLEANUP => 1); + is (mkdir ("$spc2/PG_99_fakedirectory"), 1, 'creating a directory in spc2'); + chown $postgres_uid, 0, $spc1, $spc2, "$spc2/PG_99_fakedirectory"; + is_program_out 'postgres', "psql -qc \"CREATE TABLESPACE spc1 LOCATION '$spc1'\"", 0, '', 'creating tablespace spc1'; + is_program_out 'postgres', "psql -qc 'CREATE TABLE tbl1 (x int) TABLESPACE spc1'", 0, '', 'creating a table in spc1'; + SKIP: { + skip "Non-empty tablespaces not supported before 9.0", 4 if ($v < 9.0); + is_program_out 'postgres', "psql -qc \"CREATE TABLESPACE spc2 LOCATION '$spc2'\"", 0, '', 'creating tablespace spc2'; + is_program_out 'postgres', "psql -qc 'CREATE TABLE tbl2 (x int) TABLESPACE spc2'", 0, '', 'creating a table in spc2'; + } + + # check apt config + is_program_out 0, "grep -Eo 'postgresql.[0-9.*-]+' /etc/apt/apt.conf.d/02autoremove-postgresql", 0, + "postgresql.*-$v\n", "Correct apt NeverAutoRemove config"; + + # stop server, clean up, check for leftovers + ok ((system "pg_dropcluster $v main --stop") == 0, + 'pg_dropcluster removes cluster'); + + is (-e $xlogdir, undef, "xlog/wal directory $xlogdir was deleted"); + ok_dir $spc1, [], "tablespace spc1 was emptied"; + ok_dir $spc2, [qw(PG_99_fakedirectory)], "tablespace spc2 was emptied"; + + is_program_out 0, "grep -Eo 'postgresql.[0-9.*-]+' /etc/apt/apt.conf.d/02autoremove-postgresql", 1, + "", "Correct apt NeverAutoRemove config"; + + check_clean; +} + +foreach (@MAJORS) { + check_major $_; +} + +# vim: filetype=perl diff --git a/t/021_pg_renamecluster.t b/t/021_pg_renamecluster.t new file mode 100644 index 0000000..b3e1cb6 --- /dev/null +++ b/t/021_pg_renamecluster.t @@ -0,0 +1,43 @@ +use strict; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More tests => 20; + +my $v = $MAJORS[-1]; + +# create cluster +ok ((system "pg_createcluster $v main --start >/dev/null") == 0, + "pg_createcluster $v main"); + +# test pg_renamecluster with a running cluster +program_ok (0, "pg_renamecluster $v main donau"); +is_program_out 'postgres', 'psql -tAc "show data_directory"', 0, + "/var/lib/postgresql/$v/donau\n", 'cluster is running and data_directory was moved'; +is ((PgCommon::get_conf_value $v, 'donau', 'postgresql.conf', 'hba_file'), + "/etc/postgresql/$v/donau/pg_hba.conf", 'pg_hba.conf location updated'); +is ((PgCommon::get_conf_value $v, 'donau', 'postgresql.conf', 'ident_file'), + "/etc/postgresql/$v/donau/pg_ident.conf", 'pg_ident.conf location updated'); +is ((PgCommon::get_conf_value $v, 'donau', 'postgresql.conf', 'external_pid_file'), + "/var/run/postgresql/$v-donau.pid", 'external_pid_file location updated'); +ok (-f "/var/run/postgresql/$v-donau.pid", 'external_pid_file exists'); +SKIP: { + skip "no stats_temp_directory in $v", 2 if ($v < 8.4 or $v >= 15); + is ((PgCommon::get_conf_value $v, 'donau', 'postgresql.conf', 'stats_temp_directory'), + "/var/run/postgresql/$v-donau.pg_stat_tmp", 'stats_temp_directory location updated'); + ok (-d "/var/run/postgresql/$v-donau.pg_stat_tmp", 'stats_temp_directory exists'); +} +SKIP: { + skip "cluster name not supported in $v", 1 if ($v < 9.5); + is (PgCommon::get_conf_value ($v, 'donau', 'postgresql.conf', 'cluster_name'), "$v/donau", "cluster_name is updated"); +} + +# stop server, clean up, check for leftovers +ok ((system "pg_dropcluster $v donau --stop") == 0, + 'pg_dropcluster removes cluster'); + +check_clean; + +# vim: filetype=perl diff --git a/t/022_recovery.t b/t/022_recovery.t new file mode 100644 index 0000000..b70a8b5 --- /dev/null +++ b/t/022_recovery.t @@ -0,0 +1,54 @@ +# We create a cluster, stop it ungracefully, and check if recovery works. + +use strict; + +use POSIX qw/dup2/; +use Time::HiRes qw/usleep/; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More tests => 17 * @MAJORS; + +sub check_major { + my $v = $_[0]; + note "Running tests for $v"; + + # create cluster + program_ok (0, "pg_createcluster $v main --start >/dev/null"); + + # try an immediate shutdown and restart + program_ok (0, "pg_ctlcluster $v main stop -m i"); + program_ok (0, "pg_ctlcluster $v main start"); + my $c = 0; # fallback for when pg_isready is missing (PG < 9.3) + while (system ("pg_isready -q 2>&1") >> 8 == 1 and $c++ < 15) { + sleep(1); + } + program_ok ('postgres', "psql -c ''"); + + # try again with an write-protected file + program_ok (0, "pg_ctlcluster $v main stop -m i"); + open F, ">/var/lib/postgresql/$v/main/foo"; + print F "moo\n"; + close F; + ok ((chmod 0444, "/var/lib/postgresql/$v/main/foo"), + "create write-protected file in data directory"); + program_ok (0, "pg_ctlcluster $v main start"); + $c = 0; + while (system ("pg_isready -q 2>&1") >> 8 == 1 and $c++ < 15) { + sleep(1); + } + program_ok ('postgres', "psql -c ''"); + + program_ok (0, "pg_dropcluster $v main --stop", 0, + 'pg_dropcluster removes cluster'); + + check_clean; +} + +foreach (@MAJORS) { + check_major $_; +} + +# vim: filetype=perl diff --git a/t/025_logging.t b/t/025_logging.t new file mode 100644 index 0000000..a7e5c50 --- /dev/null +++ b/t/025_logging.t @@ -0,0 +1,99 @@ +# Test various logging-related things + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Time::HiRes qw/usleep/; + +use Test::More tests => 55 * @MAJORS; + +my $syslog_works = 0; + +sub check_logging ($$) +{ + my ($text, $msg) = @_; + my $ls = `pg_lsclusters -h`; + $ls =~ s/\s+/ /g; + $ls =~ s/\s*$//; + like $ls, $text, $msg; +} + +sub check_major { + my $v = $_[0]; + note "Running tests for $v"; + my $pgdata = "/var/lib/postgresql/$v/main"; + + # create cluster + ok ((system "pg_createcluster $v main --start >/dev/null") == 0, + "pg_createcluster $v main"); + + # default log setup + my $default_log = "/var/log/postgresql/postgresql-$v-main.log"; + check_logging qr($v main 5432 online postgres $pgdata $default_log), "pg_lscluster reports logfile $default_log"; + like_program_out 'postgres', "psql -qc \"'foobar_${v}_$$'\"", 1, qr/syntax error.*foobar_${v}_$$/, 'log an error'; + usleep $delay; + like_program_out 'postgres', "grep --binary-files=text foobar_${v}_$$ $default_log", 0, qr/syntax error.*foobar_${v}_$$/, 'error appears in logfile'; + + # syslog + is_program_out 0, "pg_conftool $v main set log_destination syslog", 0, "", "set log_destination syslog"; + is_program_out 0, "pg_ctlcluster $v main reload", 0, "", "$v main reload"; + is_program_out 'postgres', "psql -Atc \"show log_destination\"", 0, "syslog\n", 'log_destination is syslog'; + check_logging qr($v main 5432 online postgres $pgdata syslog), "pg_lscluster reports syslog"; + SKIP: { + skip "/var/log/syslog not available", 2 unless ($syslog_works); + usleep $delay; + like_program_out 0, "grep --binary-files=text 'postgres.*parameter \"log_destination\" changed to \"syslog\"' /var/log/syslog", 0, qr/log_destination/, 'error appears in /var/log/syslog'; + } + + # turn logging_collector on, csvlog + my $pg_log = $v >= 10 ? 'log' : 'pg_log'; # log directory in PGDATA changed in PG 10 + SKIP: { + skip "No logging collector in 8.2", 30 if ($v <= 8.2); + is_program_out 0, "pg_conftool $v main set logging_collector on", 0, "", "set logging_collector on"; + is_program_out 0, "pg_conftool $v main set log_destination csvlog", 0, "", "set log_destination csvlog"; + is_program_out 0, "pg_ctlcluster $v main restart", 0, "", "$v main restart"; + is_program_out 'postgres', "psql -Atc \"show logging_collector\"", 0, "on\n", 'logging_collector is on'; + is_program_out 'postgres', "psql -Atc \"show log_destination\"", 0, "csvlog\n", 'log_destination is csvlog'; + check_logging qr($v main 5432 online postgres $pgdata $pg_log/.*\.csv), "pg_lscluster reports csvlog"; + like_program_out 'postgres', "psql -qc \"'barbaz_${v}_$$'\"", 1, qr/syntax error.*barbaz_${v}_$$/, 'log an error'; + usleep $delay; + like_program_out 'postgres', "grep --binary-files=text barbaz_${v}_$$ $pgdata/$pg_log/*.csv", 0, qr/syntax error.*barbaz_${v}_$$/, "error appears in $pg_log/*.csv"; + + # stderr,syslog,csvlog + is_program_out 0, "pg_conftool $v main set log_destination stderr,syslog,csvlog", 0, "", "set log_destination stderr,syslog,csvlog"; + is_program_out 0, "pg_ctlcluster $v main reload", 0, "", "$v main reload"; + is_program_out 'postgres', "psql -Atc \"show log_destination\"", 0, "stderr,syslog,csvlog\n", 'log_destination is stderr,syslog,csvlog'; + check_logging qr($v main 5432 online postgres $pgdata $pg_log/.*\.log,syslog,$pg_log/.*\.csv), "pg_lscluster reports stderr,syslog,csvlog"; + like_program_out 'postgres', "psql -qc \"'moo_${v}_$$'\"", 1, qr/syntax error.*moo_${v}_$$/, 'log an error'; + usleep $delay; + like_program_out 'postgres', "grep --binary-files=text moo_${v}_$$ $pgdata/$pg_log/*.log", 0, qr/syntax error.*moo_${v}_$$/, "error appears in $pg_log/*.log"; + SKIP: { + skip "/var/log/syslog not available", 2 unless ($syslog_works); + usleep $delay; + like_program_out 0, "grep --binary-files=text 'postgres.*moo_${v}_$$' /var/log/syslog", 0, qr/moo_${v}_$$/, 'error appears in /var/log/syslog'; + } + like_program_out 'postgres', "grep --binary-files=text moo_${v}_$$ $pgdata/$pg_log/*.csv", 0, qr/syntax error.*moo_${v}_$$/, "error appears in $pg_log/*.csv"; + } + + # stop server, clean up, check for leftovers + is_program_out 0, "pg_dropcluster $v main --stop", 0, "", 'pg_dropcluster removes cluster'; + + check_clean; +} + +system "logger -t '$0' 'test-logging-$$'"; +usleep $delay; +if (system ("grep --binary-files=text -q 'test-logging-$$' /var/log/syslog 2> /dev/null") == 0) { + note 'Logging to /var/log/syslog works'; + $syslog_works = 1; +} else { + note 'Logging to /var/log/syslog does not work, skipping some syslog tests'; +} + +foreach (@MAJORS) { + check_major $_; +} + +# vim: filetype=perl diff --git a/t/030_errors.t b/t/030_errors.t new file mode 100644 index 0000000..88d3700 --- /dev/null +++ b/t/030_errors.t @@ -0,0 +1,336 @@ +# Check all kinds of error conditions. + +use strict; + +require File::Temp; + +use lib 't'; +use TestLib; +use Test::More tests => 156; +use PgCommon; + +my $version = $MAJORS[-1]; + +my $socketdir = '/tmp/postgresql-testsuite/'; +my ($pg_uid, $pg_gid) = (getpwnam 'postgres')[2,3]; + +# create a pid file with content $1 and return its path +sub create_pidfile { + my $fname = "/var/lib/postgresql/$version/main/postmaster.pid"; + open F, ">$fname" or die "open: $!"; + print F $_[0]; + close F; + chown $pg_uid, $pg_gid, $fname or die "chown: $!"; + chmod 0700, $fname or die "chmod: $!"; + return $fname; +} + +sub check_nonexisting_cluster_error { + my $outref; + my $result = exec_as 0, $_[0], $outref; + is $result, 1, "'$_[0]' fails"; + like $$outref, qr/(invalid version|does not exist)/i, "$_[0] gives error message about nonexisting cluster"; + unlike $$outref, qr/invalid symbolic link/i, "$_[0] does not print 'invalid symbolic link' gibberish"; +} + +# check if pg_lsclusters shows a cluster without binaries +mkdir "/etc/postgresql/6.3"; +mkdir "/etc/postgresql/6.3/main"; +open F, ">/etc/postgresql/6.3/main/postgresql.conf"; +close F; +is `pg_lsclusters -h`, "6.3 main <unknown> down,binaries_missing <unknown> <unknown> <unknown>\n", + 'pg_lscluster reports cluster without binaries'; +program_ok 0, "pg_dropcluster 6.3 main"; + +# create cluster +ok ((system "pg_createcluster --socketdir '$socketdir' $version main >/dev/null") == 0, + "pg_createcluster --socketdir"); +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/$version\s*main.*5432.*down/, 'cluster was created'; + +is ((get_cluster_port $version, 'main'), 5432, 'Port of created cluster is 5432'); + +# creating cluster with the same name should fail +like_program_out 'root', "pg_createcluster --socketdir '$socketdir' $version main", 1, qr/already exists/, + "pg_createcluster on existing cluster"; +# and the original one still exists +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/$version\s*main.*5432.*down/, 'original cluster still exists'; + +# attempt to create clusters with an invalid port +like_program_out 0, "pg_createcluster $version test -p foo", 1, + qr/invalid.*number expected/, + 'pg_createcluster -p checks that port option is numeric'; +like_program_out 0, "pg_createcluster $version test -p 42", 1, + qr/must be a positive integer between/, + 'pg_createcluster -p checks valid port range'; + +# chown cluster to an invalid user to test error +(system "chown -R 0 /var/lib/postgresql/$version/main") == 0 or die "chown failed: $!"; +like_program_out 0, "pg_ctlcluster $version main start", 1, qr/must not be owned by root/, + "pg_ctlcluster refuses to start root-owned cluster"; +my $badid = 98; +(system "chown -R $badid /var/lib/postgresql/$version/main") == 0 or die "chown failed: $!"; +like_program_out 0, "pg_ctlcluster $version main start", 1, qr/owned by user id 98 which does not exist/, + 'pg_ctlcluster fails on invalid cluster owner uid'; +(system "chown -R postgres:$badid /var/lib/postgresql/$version/main") == 0 or die "chown failed: $!"; +like_program_out 0, "pg_ctlcluster $version main start", 1, qr/owned by group id 98 which does not exist/, + 'pg_ctlcluster as root fails on invalid cluster owner gid'; +like_program_out 'postgres', "pg_ctlcluster $version main start", 1, qr/owned by group id 98 which does not exist/, + 'pg_ctlcluster as postgres fails on invalid cluster owner gid'; +(system "chown -R postgres:postgres /var/lib/postgresql/$version/main") == 0 or die "chown failed: $!"; +program_ok 0, "pg_ctlcluster $version main start", 0, + 'pg_ctlcluster succeeds on valid cluster owner uid/gid'; + +# check socket +my @contents = ('.s.PGSQL.5432', '.s.PGSQL.5432.lock', "$version-main.pid", "$version-main.pg_stat_tmp"); +pop @contents if ($version < 8.4 or $version >= 15); # remove pg_stat_tmp +ok_dir '/var/run/postgresql', [grep {/main/} @contents], 'No sockets in /var/run/postgresql'; +ok_dir $socketdir, ['.s.PGSQL.5432', '.s.PGSQL.5432.lock'], "Socket is in $socketdir"; + +# stop cluster, check sockets +ok ((system "pg_ctlcluster $version main stop") == 0, + 'cluster stops with custom unix_socket_dir'); +ok_dir $socketdir, [], "No sockets in $socketdir after stopping cluster"; + +# remove default socket dir and check that the socket defaults to +# /var/run/postgresql +open F, "+</etc/postgresql/$version/main/postgresql.conf" or + die "could not open postgresql.conf for r/w: $!"; +my @lines = <F>; +seek F, 0, 0 or die "seek: $!"; +truncate F, 0; +@lines = grep !/^unix_socket_dir/, @lines; # <= 9.2: "_directory", >= 9.3: "_directories" +print F @lines; +close F; + +ok ((system "pg_ctlcluster $version main start") == 0, + 'cluster starts after removing unix_socket_dir'); +if ($PgCommon::rpm) { + ok ((grep { $_ eq '.s.PGSQL.5432' } @{TestLib::dircontent('/tmp')}) == 1, 'Socket is in /tmp'); +} else { + ok_dir '/var/run/postgresql', [@contents], + 'Socket is in default dir /var/run/postgresql'; +} +ok_dir $socketdir, [], "No sockets in $socketdir"; + +# server should not stop with corrupt file +rename "/var/lib/postgresql/$version/main/postmaster.pid", + "/var/lib/postgresql/$version/main/postmaster.pid.orig" or die "rename: $!"; +create_pidfile 'foo'; +is_program_out 'postgres', "pg_ctlcluster $version main stop", 1, + "Error: pid file is invalid, please manually kill the stale server process.\n", + 'pg_ctlcluster fails with corrupted PID file'; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/online/, 'cluster is still online'; + +# restore PID file +(system "cp /var/lib/postgresql/$version/main/postmaster.pid.orig /var/lib/postgresql/$version/main/postmaster.pid") == 0 or die "cp: $!"; +is ((exec_as 'postgres', "pg_ctlcluster $version main stop"), 0, + 'pg_ctlcluster succeeds with restored PID file'); +mkdir $PgCommon::binroot . "foo"; # #940220: infinite recursion in get_program_path +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/down/, 'cluster is down'; +rmdir $PgCommon::binroot . "foo"; + +# stop stopped server +is_program_out 'postgres', "pg_ctlcluster $version main stop", 2, + "Cluster is not running.\n", 'pg_ctlcluster stop fails on stopped cluster'; + +# simulate crashed server +rename "/var/lib/postgresql/$version/main/postmaster.pid.orig", + "/var/lib/postgresql/$version/main/postmaster.pid" or die "rename: $!"; +is_program_out 'postgres', "pg_ctlcluster $version main start", 0, + "Removed stale pid file.\n", 'pg_ctlcluster succeeds with already existing PID file'; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/online/, 'cluster is online'; +is ((exec_as 'postgres', "pg_ctlcluster $version main stop"), 0, + 'pg_ctlcluster stop succeeds'); +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/down/, 'cluster is down'; +ok (! -e "/var/lib/postgresql/$version/main/postmaster.pid", 'no pid file left'); + +# trying to stop a stopped server cleans up corrupt and stale pid files +my $pf = create_pidfile 'foo'; +is_program_out 'postgres', "pg_ctlcluster $version main stop", 2, + "Removed stale pid file.\nCluster is not running.\n", + 'pg_ctlcluster stop succeeds with corrupted PID file'; +ok (! -e $pf, 'pid file was cleaned up'); + +create_pidfile 'foo'; +is_program_out 'postgres', "pg_ctlcluster --force $version main stop", 2, + "Removed stale pid file.\nCluster is not running.\n", + 'pg_ctlcluster --force stop succeeds with corrupted PID file'; +ok (! -e $pf, 'pid file was cleaned up'); + +create_pidfile '99998'; +is_program_out 'postgres', "pg_ctlcluster $version main stop", 2, + "Removed stale pid file.\nCluster is not running.\n", + 'pg_ctlcluster stop succeeds with stale PID file'; +ok (! -e $pf, 'pid file was cleaned up'); + +create_pidfile '99998'; +is_program_out 'postgres', "pg_ctlcluster --force $version main stop", 2, + "Removed stale pid file.\nCluster is not running.\n", + 'pg_ctlcluster --force stop succeeds with stale PID file'; +ok (! -e $pf, 'pid file was cleaned up'); + +create_pidfile ''; +is_program_out 'postgres', "pg_ctlcluster --force $version main stop", 2, + "Removed stale pid file.\nCluster is not running.\n", + 'pg_ctlcluster stop succeeds with empty PID file'; +ok (! -e $pf, 'pid file was cleaned up'); + +# corrupt PID file while server is down +create_pidfile 'foo'; +is_program_out 'postgres', "pg_ctlcluster $version main start", 0, + "Removed stale pid file.\n", 'pg_ctlcluster succeeds with corrupted PID file'; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/online/, 'cluster is online'; + +# start running server +is_program_out 'postgres', "pg_ctlcluster $version main start", 2, + "Cluster is already running.\n", 'pg_ctlcluster start fails on running cluster'; +is ((exec_as 'postgres', "pg_ctlcluster $version main stop"), 0, 'pg_ctlcluster stop'); + +# backup pg_hba.conf +rename "/etc/postgresql/$version/main/pg_hba.conf", + "/etc/postgresql/$version/main/pg_hba.conf.orig" or die "rename: $!"; + +# test check for invalid pg_hba.conf +open F, ">/etc/postgresql/$version/main/pg_hba.conf" or die "could not create pg_hba.conf: $!"; +print F "foo\n"; +close F; +chmod 0644, "/etc/postgresql/$version/main/pg_hba.conf" or die "chmod: $!"; + +if ($version < '8.4') { + like_program_out 'postgres', "pg_ctlcluster $version main start", 0, + qr/WARNING.*connection to the database failed.*pg_hba.conf/is, + 'pg_ctlcluster start warns about invalid pg_hba.conf'; + is_program_out 'postgres', "pg_ctlcluster $version main stop", 0, '', 'stopping cluster'; +} else { + like_program_out 'postgres', "pg_ctlcluster $version main start", 1, + qr/FATAL.*pg_hba.conf/is, + 'pg_ctlcluster start fails on invalid pg_hba.conf'; + is_program_out 'postgres', "pg_ctlcluster $version main stop", 2, + "Cluster is not running.\n", 'stopping cluster'; +} + +# test check for pg_hba.conf with removed passwordless local superuser access +open F, ">/etc/postgresql/$version/main/pg_hba.conf" or die "could not create pg_hba.conf: $!"; +print F "local all all md5\n"; +close F; +chmod 0644, "/etc/postgresql/$version/main/pg_hba.conf" or die "chmod: $!"; + +like_program_out 'postgres', "pg_ctlcluster $version main start", 0, + qr/Warning.*connection to the database failed.*(no password supplied|password authentication failed)/is, + 'pg_ctlcluster start warns about absence of passwordless superuser connection'; +is_program_out 'postgres', "pg_ctlcluster $version main stop", 0, '', 'stopping cluster'; + +# restore pg_hba.conf +unlink "/etc/postgresql/$version/main/pg_hba.conf"; +rename "/etc/postgresql/$version/main/pg_hba.conf.orig", + "/etc/postgresql/$version/main/pg_hba.conf" or die "rename: $!"; + +# leftover files must not create confusion +open F, '>/etc/postgresql/postgresql.conf'; +print F "data_directory = '/nonexisting'\n"; +close F; +my @c = get_version_clusters $version; +is_deeply (\@c, ['main'], + 'leftover /etc/postgresql/postgresql.conf is not regarded as a cluster'); +unlink '/etc/postgresql/postgresql.conf'; + +# fails by default due to access restrictions +# remove cluster and directory; this should work as user "postgres" +is_program_out 'postgres', "pg_dropcluster $version main", 0, '', + , "pg_dropcluster works as user postgres"; + +# graceful handling of absent data dir (might not be mounted) +ok ((system "pg_createcluster $version main >/dev/null") == 0, + "pg_createcluster succeeds"); +rename "/var/lib/postgresql/$version", "/var/lib/postgresql/$version.orig" or die "rename: $!"; +my $outref; +is ((exec_as 0, "pg_ctlcluster $version main start", $outref, 1), 1, + 'pg_ctlcluster fails on nonexisting /var/lib/postgresql'); +like $$outref, qr/^Error:.*\/var\/lib\/postgresql.*not accessible.*$/, 'proper error message for nonexisting /var/lib/postgresql'; + +rename "/var/lib/postgresql/$version.orig", "/var/lib/postgresql/$version" or die "rename: $!"; +is_program_out 'postgres', "pg_ctlcluster $version main start", 0, '', + 'pg_ctlcluster start succeeds again with reappeared /var/lib/postgresql'; +is_program_out 'postgres', "pg_ctlcluster $version main stop", 0, '', 'stopping cluster'; + +# pg_ctlcluster checks colliding ports +ok ((system "pg_createcluster $version other >/dev/null") == 0, + "pg_createcluster other"); +set_cluster_port $version, 'other', '5432'; +is ((exec_as 'postgres', "pg_ctlcluster $version main start"), 0, + 'pg_ctlcluster: main cluster on conflicting port starts'); + +# clusters can run side by side on different socket directories +set_cluster_socketdir $version, 'other', $socketdir; +PgCommon::set_conf_value $version, 'other', 'postgresql.conf', + 'listen_addresses', ''; # otherwise they will conflict on TCP socket +is ((exec_as 'postgres', "pg_ctlcluster $version other start"), 0, + 'pg_ctlcluster: other cluster starts on conflicting port, but different socket dirs'); +is ((exec_as 'postgres', "pg_ctlcluster $version other stop"), 0); + +# ... but will give an error when running on the same port +set_cluster_socketdir $version, 'other', ($PgCommon::rpm and $version < 9.4) ? '/tmp' : '/var/run/postgresql'; +like_program_out 'postgres', "pg_ctlcluster $version other start", 1, + qr/Port conflict:.*port 5432/, + 'pg_ctlcluster other cluster fails on conflicting port and same socket dir'; +is_program_out 'postgres', "pg_ctlcluster $version main stop", 0, '', + 'stopping main cluster'; +is ((exec_as 'postgres', "pg_ctlcluster $version other start"), 0, + 'pg_ctlcluster: other cluster on conflicting port starts after main is down'); +ok ((system "pg_dropcluster $version other --stop") == 0, + 'pg_dropcluster other'); + +# clean up +ok ((system "pg_dropcluster $version main") == 0, + 'pg_dropcluster'); +ok_dir $socketdir, [], 'No sockets any more'; +rmdir $socketdir or die "rmdir: $!"; + +# ensure sane error messages for nonexisting clusters +check_nonexisting_cluster_error 'pg_lsclusters 4.5 foo'; +check_nonexisting_cluster_error 'psql --cluster 4.5/foo'; +check_nonexisting_cluster_error "psql --cluster $MAJORS[0]/foo"; +check_nonexisting_cluster_error "pg_dropcluster 4.5 foo"; +check_nonexisting_cluster_error "pg_dropcluster $MAJORS[0] foo"; +check_nonexisting_cluster_error "pg_upgradecluster 4.5 foo"; +check_nonexisting_cluster_error "pg_upgradecluster $MAJORS[0] foo"; +check_nonexisting_cluster_error "pg_ctlcluster 4.5 foo stop"; +check_nonexisting_cluster_error "pg_ctlcluster $MAJORS[0] foo stop"; + +check_clean; + +# check that pg_dropcluster copes with partially existing cluster +# configurations (which can happen if the disk becomes full) + +mkdir '/etc/postgresql/'; +mkdir "/etc/postgresql/$MAJORS[-1]"; +mkdir "/etc/postgresql/$MAJORS[-1]/broken" or die "mkdir: $!"; +symlink "/var/lib/postgresql/$MAJORS[-1]/broken", "/etc/postgresql/$MAJORS[-1]/broken/pgdata" or die "symlink: $!"; + +unlike_program_out 0, "pg_dropcluster $MAJORS[-1] broken", 0, qr/error/i, + 'pg_dropcluster cleans up broken cluster configuration (only /etc with pgdata)'; + +check_clean; + +mkdir '/etc/postgresql/'; +mkdir '/var/lib/postgresql/'; +mkdir "/etc/postgresql/$MAJORS[-1]" and +mkdir "/etc/postgresql/$MAJORS[-1]/broken"; +mkdir "/var/lib/postgresql/$MAJORS[-1]"; +mkdir "/var/lib/postgresql/$MAJORS[-1]/broken"; +chown $pg_uid, $pg_gid, "/var/lib/postgresql/$MAJORS[-1]/broken"; +mkdir "/var/lib/postgresql/$MAJORS[-1]/broken/base" or die "mkdir: $!"; +open F, ">/etc/postgresql/$MAJORS[-1]/broken/postgresql.conf" or die "open: $!"; +print F "data_directory = '/var/lib/postgresql/$MAJORS[-1]/broken'\n"; +close F; +open F, ">/var/lib/postgresql/$MAJORS[-1]/broken/PG_VERSION" or die "open: $!"; +close F; + +unlike_program_out 0, "pg_dropcluster $MAJORS[-1] broken", 0, qr/error/i, + 'pg_dropcluster cleans up broken cluster configuration (/etc with pgdata and postgresql.conf and partial /var)'; +is -d "/etc/postgresql/$MAJORS[-1]", undef, "/etc/postgresql/$MAJORS[-1] was removed"; +is -d "/var/lib/postgresql/$MAJORS[-1]", undef, "/var/lib/postgresql/$MAJORS[-1] was removed"; + +check_clean; + +# vim: filetype=perl diff --git a/t/031_errors_disk_full.t b/t/031_errors_disk_full.t new file mode 100644 index 0000000..bc0a860 --- /dev/null +++ b/t/031_errors_disk_full.t @@ -0,0 +1,86 @@ +# Check for proper ENOSPC handling + +use strict; + +require File::Temp; + +use lib 't'; +use TestLib; +use Test::More tests => $ENV{NO_TMPFS} ? 1 : 22; + +# skip tests if NO_TMPFS is set +if ($ENV{NO_TMPFS}) { + pass 'Skipping disk full tests, NO_TMPFS is set'; + exit; +} + +# we are using unshare here, won't work with systemd +$ENV{_SYSTEMCTL_SKIP_REDIRECT} = 1; + +my $outref; + +# +note 'check that a failed pg_createcluster leaves no cruft behind: try creating a cluster on a 10 MB tmpfs'; +my $cmd = <<EOF; +exec 2>&1 +set -e +mount --make-rprivate / 2> /dev/null || : +mkdir -p /var/lib/postgresql +trap "umount /var/lib/postgresql" 0 HUP INT QUIT ILL ABRT PIPE TERM +mount -t tmpfs -o size=10000000 none /var/lib/postgresql +# this is supposed to fail +LC_MESSAGES=C pg_createcluster $MAJORS[-1] test && exit 1 || true +echo -n "ls>" +# should not output anything +ls /etc/postgresql +ls /var/lib/postgresql +echo "<ls" +EOF + +my $result; +$result = exec_as 'root', "echo '$cmd' | unshare -m sh", $outref; + +is $result, 0, 'script failed'; +like $$outref, qr/No space left on device/i, + 'pg_createcluster fails due to insufficient disk space'; +like $$outref, qr/\nls><ls\n/, 'does not leave files behind'; + +check_clean; + +# +note 'check disk full conditions on startup'; +my $cmd = <<EOF; +set -e +mount --make-rprivate / 2> /dev/null || : +export LC_MESSAGES=C +dirs="/etc/postgresql /var/lib/postgresql /var/log/postgresql" +mkdir -p \$dirs +trap "umount \$dirs" 0 HUP INT QUIT ILL ABRT PIPE TERM +mount -t tmpfs -o size=1000000 none /etc/postgresql +# an empty cluster needs 69MB on ppc64el, round up to 90 +mount -t tmpfs -o size=90000000 none /var/lib/postgresql +mount -t tmpfs -o size=1000000 none /var/log/postgresql +pg_createcluster $MAJORS[-1] test + +# fill up /var/lib/postgresql +! cat < /dev/zero > /var/lib/postgresql/cruft 2>/dev/null +echo '-- full lib --' +! pg_ctlcluster $MAJORS[-1] test start +echo '-- end full lib --' +echo '-- full lib log --' +cat /var/log/postgresql/postgresql-$MAJORS[-1]-test.log +echo '-- end full lib log --' +rm /var/lib/postgresql/cruft +pg_dropcluster $MAJORS[-1] test --stop +EOF + +$result = exec_as 'root', "echo '$cmd' | unshare -m sh", $outref; +is $result, 0, 'script failed'; +like $$outref, qr/^-- full lib --.*No space left on device.*^-- end full lib --/ims, + 'pg_ctlcluster prints error message'; +like $$outref, qr/^-- full lib log --.*No space left on device.*^-- end full lib log --/ims, + 'log file has error message'; + +check_clean; + +# vim: filetype=perl diff --git a/t/032_ssl_key_permissions.t b/t/032_ssl_key_permissions.t new file mode 100644 index 0000000..929f08a --- /dev/null +++ b/t/032_ssl_key_permissions.t @@ -0,0 +1,60 @@ +use strict; +use warnings; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More tests => $PgCommon::rpm ? 1 : 3 + 19 * @MAJORS; + +if ($PgCommon::rpm) { pass 'No ssl key checks on RedHat'; exit; } + +my ($pg_uid, $pg_gid) = (getpwnam 'postgres')[2,3]; +my $ssl_cert_gid = (getgrnam 'ssl-cert')[2]; # reset permissions +die "Could not determine ssl-cert gid" unless ($ssl_cert_gid); + +my $snakekey = '/etc/ssl/private/ssl-cert-snakeoil.key'; +is ((stat $snakekey)[4], 0, "$snakekey is owned by root"); +is ((stat $snakekey)[5], $ssl_cert_gid, "$snakekey group is ssl-cert"); +is ((stat $snakekey)[2], 0100640, "$snakekey mode is 0640"); + +foreach my $version (@MAJORS) { + my $pkgversion = `dpkg-query -f '\${Version}' -W postgresql-$version`; + note "$version ($pkgversion)"; + if ($version <= 9.1) { + pass "no SSL support on $version" foreach (1..19); + next; + } +SKIP: { + skip "No SSL key check on <= 9.0", 19 if ($version <= 9.0); + program_ok (0, "pg_createcluster $version main"); + + my $nobody_uid = (getpwnam 'nobody')[2]; + chown $nobody_uid, 0, $snakekey; + like_program_out 'postgres', "pg_ctlcluster $version main start", 1, + qr/private key file.*must be owned by the database user or root/s, + 'ssl key owned by nobody refused'; + +SKIP: { + skip "SSL key group check skipped on Debian oldstable packages", 4 if ($version <= 9.4 and $pkgversion !~ /pgdg/); + chown 0, 0, $snakekey; + chmod 0644, $snakekey; + like_program_out 'postgres', "pg_ctlcluster $version main start", 1, + qr/private key file.*has group or world access/, + 'ssl key with permissions root:root 0644 refused'; + + chown $pg_uid, $pg_gid, $snakekey; + chmod 0640, $snakekey; + like_program_out 'postgres', "pg_ctlcluster $version main start", 1, + qr/private key file.*has group or world access/, + 'ssl key with permissions postgres:postgres 0640 refused'; +} + + chown 0, $ssl_cert_gid, $snakekey; + + program_ok (0, "pg_dropcluster $version main --stop"); + is ((stat $snakekey)[4], 0, "$snakekey is owned by root"); + is ((stat $snakekey)[5], $ssl_cert_gid, "$snakekey group is ssl-cert"); + is ((stat $snakekey)[2], 0100640, "$snakekey mode is 0640"); + check_clean; +} +} diff --git a/t/040_upgrade.t b/t/040_upgrade.t new file mode 100644 index 0000000..6818ef8 --- /dev/null +++ b/t/040_upgrade.t @@ -0,0 +1,272 @@ +# Test upgrading from the oldest version to the latest, using the default +# configuration file. + +# Lowest supported "upgrade from" version: 8.4 (lower versions don't have lo_import) +# Lowest supported "upgrade to" version: 9.2 (lower versions don't have pg_upgrade -o) +# Lowest supported "upgrade to" version with pg_dumpall: 9.1 (lower versions don't have pg_dumpall --quote-all-identifiers) + +use strict; + +use File::Temp qw/tempfile tempdir/; +use POSIX qw/dup2/; +use Time::HiRes qw/usleep/; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More tests => (@MAJORS == 1) ? 1 : 123 * 3; + +if (@MAJORS == 1) { + pass 'only one major version installed, skipping upgrade tests'; + exit 0; +} + +foreach my $upgrade_options ('-m dump', '-m upgrade', '-m upgrade --link') { +next if ($ENV{UPGRADE_METHOD} and $upgrade_options !~ /$ENV{UPGRADE_METHOD}$/); # hack to ease debugging individual methods +note ("upgrade method \"$upgrade_options\", $MAJORS[0] -> $MAJORS[-1]"); + +# create cluster +ok ((system "pg_createcluster $MAJORS[0] upgr >/dev/null") == 0, + "pg_createcluster $MAJORS[0] upgr"); +exec_as 'root', "sed -i '/^local.*postgres/ s/\$/\\nlocal all foo trust/' /etc/postgresql/$MAJORS[0]/upgr/pg_hba.conf"; +is ((system "pg_ctlcluster $MAJORS[0] upgr start"), 0, 'Starting upgr cluster'); + +# Create nobody user, test database, and put a table into it +is ((exec_as 'postgres', 'createuser nobody -D -R -s && createdb -O nobody test && createdb -O nobody testnc && createdb -O nobody testro'), + 0, 'Create nobody user and test databases'); +is ((exec_as 'nobody', 'psql test -c "CREATE TABLE phone (name varchar(255) PRIMARY KEY, tel int NOT NULL)"'), + 0, 'create table'); +is ((exec_as 'nobody', 'psql test -c "INSERT INTO phone VALUES (\'Alice\', 2)"'), 0, 'insert Alice into phone table'); +SKIP: { + skip 'datallowconn = f not supported with pg_upgrade', 1 if $upgrade_options =~ /upgrade/; + is ((exec_as 'postgres', 'psql template1 -c "UPDATE pg_database SET datallowconn = \'f\' WHERE datname = \'testnc\'"'), + 0, 'disallow connection to testnc'); +} +is ((exec_as 'nobody', 'psql testro -c "CREATE TABLE nums (num int NOT NULL); INSERT INTO nums VALUES (1)"'), 0, 'create table in testro'); +SKIP: { + skip 'read-only not supported with pg_upgrade', 2 if $upgrade_options =~ /upgrade/; + is ((exec_as 'postgres', 'psql template1 -c "ALTER DATABASE testro SET default_transaction_read_only TO on"'), + 0, 'set testro transaction default to readonly'); + is ((exec_as 'nobody', 'psql testro -c "CREATE TABLE test(num int)"'), + 1, 'creating table in testro fails'); +} + +# create a schema and a table with a name that was un-reserved between 8.4 and 9.1 +is ((exec_as 'nobody', 'psql test -c "CREATE SCHEMA \"old\""'), + 0, 'create schema "old"'); +is ((exec_as 'nobody', 'psql test -c "CREATE TABLE \"old\".\"old\" (\"old\" text)"'), + 0, 'create table "old.old"'); + +# create a sequence +is ((exec_as 'nobody', 'psql test -c "CREATE SEQUENCE odd10 INCREMENT BY 2 MINVALUE 1 MAXVALUE 10 CYCLE"'), + 0, 'create sequence'); +is_program_out 'nobody', 'psql -Atc "SELECT nextval(\'odd10\')" test', 0, "1\n", + 'check next sequence value'; +is_program_out 'nobody', 'psql -Atc "SELECT nextval(\'odd10\')" test', 0, "3\n", + 'check next sequence value'; + +# create a large object +my ($fh, $filename) = tempfile("lo_import.XXXXXX", TMPDIR => 1, UNLINK => 1); +print $fh "Hello world"; +close $fh; +chmod 0644, $filename; +is_program_out 'postgres', "psql -Atc \"SELECT lo_import('$filename', 1234)\"", 0, "1234\n", + 'create large object'; + +# create stored procedures +if ($MAJORS[0] < 9.0) { + is_program_out 'postgres', 'createlang plpgsql test', 0, '', 'createlang plpgsql test'; +} else { + pass '>= 9.0 enables PL/pgsql by default'; + pass '...'; +} +is_program_out 'nobody', 'psql test -c "CREATE FUNCTION inc2(integer) RETURNS integer LANGUAGE plpgsql AS \'BEGIN RETURN \$1 + 2; END;\';"', + 0, "CREATE FUNCTION\n", 'CREATE FUNCTION inc2'; +SKIP: { + skip 'hardcoded library paths not supported by pg_upgrade', 2 if $upgrade_options =~ /upgrade/; + is_program_out 'postgres', "psql -c \"UPDATE pg_proc SET probin = '$PgCommon::binroot$MAJORS[0]/lib/plpgsql.so' where proname = 'plpgsql_call_handler';\" test", + 0, "UPDATE 1\n", 'hardcoding plpgsql lib path'; +} +is_program_out 'nobody', 'psql test -c "CREATE FUNCTION inc3(integer) RETURNS integer LANGUAGE plpgsql AS \'BEGIN RETURN \$1 + 3; END;\';"', + 0, "CREATE FUNCTION\n", 'create function inc3'; +is_program_out 'nobody', 'psql -Atc "SELECT inc2(3)" test', 0, "5\n", + 'call function inc2'; +is_program_out 'nobody', 'psql -Atc "SELECT inc3(3)" test', 0, "6\n", + 'call function inc3'; + +# create user and group +is_program_out 'postgres', "psql -qc 'CREATE USER foo' template1", 0, '', + 'create user foo'; +is_program_out 'postgres', "psql -qc 'CREATE GROUP gfoo' template1", 0, '', + 'create group gfoo'; + +# create per-database and per-table ACL +is_program_out 'postgres', "psql -qc 'GRANT CREATE ON DATABASE test TO foo'", 0, '', + 'GRANT CREATE ON DATABASE'; +is_program_out 'postgres', "psql -qc 'GRANT INSERT ON phone TO foo' test", 0, '', + 'GRANT INSERT'; + +# exercise ACL on old database to ensure they are working +is_program_out 'nobody', 'psql -U foo -qc "CREATE SCHEMA s_foo" test', 0, '', + 'CREATE SCHEMA on old cluster (ACL)'; +is_program_out 'nobody', 'psql -U foo -qc "INSERT INTO phone VALUES (\'Bob\', 1)" test', + 0, '', 'insert Bob into phone table (ACL)'; + +# set config parameters +is_program_out 'postgres', "pg_conftool $MAJORS[0] upgr set log_statement all", + 0, '', 'set postgresql.conf parameter'; +SKIP: { + skip 'postgresql.auto.conf not supported before 9.4', 6 if ($MAJORS[0] < 9.4); + is_program_out 'postgres', "psql -qc \"ALTER SYSTEM SET ident_file = '/etc/postgresql/$MAJORS[0]/upgr/pg_ident.conf'\"", + 0, '', 'set ident_file in postgresql.auto.conf'; + is_program_out 'postgres', 'psql -qc "ALTER SYSTEM SET log_min_duration_statement = \'10s\'"', + 0, '', 'set log_min_duration_statement in postgresql.auto.conf'; + is_program_out 'postgres', "echo \"data_directory = '/var/lib/postgresql/$MAJORS[0]/upgr'\" >> /var/lib/postgresql/$MAJORS[0]/upgr/postgresql.auto.conf", 0, "", "Append bogus data_directory setting to postgresql.auto.conf"; +} +is_program_out 'postgres', 'psql -qc "ALTER DATABASE test SET DateStyle = \'ISO, YMD\'"', + 0, '', 'set database parameter'; + +# create a tablespace +my $tdir = tempdir (CLEANUP => 1); +my ($p_uid, $p_gid) = (getpwnam 'postgres')[2,3]; +chown $p_uid, $p_gid, $tdir; +is_program_out 'postgres', "psql -qc \"CREATE TABLESPACE myts LOCATION '$tdir'\"", + 0, '', "creating tablespace in $tdir"; +is_program_out 'postgres', "psql -qc 'CREATE TABLE tstab (a int) TABLESPACE myts'", + 0, '', "creating table in tablespace"; + +# Check clusters +like_program_out 'nobody', 'pg_lsclusters -h', 0, + qr/^$MAJORS[0]\s+upgr\s+5432 online postgres/; + +# Check SELECT in original cluster +my $select_old; +is ((exec_as 'nobody', 'psql -tAc "SELECT * FROM phone ORDER BY name" test', $select_old), 0, 'SELECT in original cluster succeeds'); +is ($$select_old, 'Alice|2 +Bob|1 +', 'check SELECT output in original cluster'); + +# create inaccessible cwd, to check for confusing error messages +rmdir '/tmp/pgtest'; +mkdir '/tmp/pgtest/' or die "Could not create temporary test directory /tmp/pgtest: $!"; +chmod 0100, '/tmp/pgtest/'; +chdir '/tmp/pgtest'; + +# Upgrade to latest version +my $outref; +is ((exec_as 0, "(env LC_MESSAGES=C pg_upgradecluster -v $MAJORS[-1] $upgrade_options $MAJORS[0] upgr | sed -e 's/^/STDOUT: /')", $outref, 0), 0, 'pg_upgradecluster succeeds'); +like $$outref, qr/Starting upgraded cluster/, 'pg_upgradecluster reported cluster startup'; +like $$outref, qr/Success. Please check/, 'pg_upgradecluster reported successful operation'; +my @err = grep (!/^STDOUT: /, split (/\n/, $$outref)); +if (@err) { + fail 'no error messages during upgrade'; + print (join ("\n", @err)); +} else { + pass "no error messages during upgrade"; +} + +# remove inaccessible test cwd +chdir '/'; +rmdir '/tmp/pgtest/'; + +# Check clusters +like_program_out 'nobody', 'pg_lsclusters -h', 0, + qr"$MAJORS[0] +upgr 5433 down postgres /var/lib/postgresql/$MAJORS[0]/upgr +/var/log/postgresql/postgresql-$MAJORS[0]-upgr.log\n$MAJORS[-1] +upgr 5432 online postgres /var/lib/postgresql/$MAJORS[-1]/upgr +/var/log/postgresql/postgresql-$MAJORS[-1]-upgr.log", 'pg_lsclusters output'; + +# Check that SELECT output is identical +is_program_out 'nobody', 'psql -tAc "SELECT * FROM phone ORDER BY name" test', 0, + $$select_old, 'SELECT output is the same in original and upgraded test'; +is_program_out 'nobody', 'psql -tAc "SELECT * FROM nums" testro', 0, + "1\n", 'SELECT output is the same in original and upgraded testro'; + +# Check that table was analyzed +like_program_out 'nobody', "psql -XAtc \"select analyze_count from pg_stat_user_tables where relname = 'phone'\" test", 0, qr/^[1-3]$/, + 'check analyze count'; # --analyze-in-stages does 3 passes + +# Check sequence value +is_program_out 'nobody', 'psql -Atc "SELECT nextval(\'odd10\')" test', 0, "5\n", + 'check next sequence value'; +is_program_out 'nobody', 'psql -Atc "SELECT nextval(\'odd10\')" test', 0, "7\n", + 'check next sequence value'; +is_program_out 'nobody', 'psql -Atc "SELECT nextval(\'odd10\')" test', 0, "9\n", + 'check next sequence value'; +is_program_out 'nobody', 'psql -Atc "SELECT nextval(\'odd10\')" test', 0, "1\n", + 'check next sequence value (wrap)'; + +# check large objects +is_program_out 'postgres', 'psql -Aqtc "SET bytea_output = \'escape\'; SELECT data FROM pg_largeobject WHERE loid = 1234"', 0, "Hello world\n", + 'check large object'; + +# check stored procedures +is_program_out 'nobody', 'psql -Atc "SELECT inc2(-3)" test', 0, "-1\n", + 'call function inc2'; +is_program_out 'nobody', 'psql -Atc "SELECT inc3(1)" test', 0, "4\n", + 'call function inc3 (formerly hardcoded path)'; + +SKIP: { + skip 'upgrading databases with datallowcon = false not supported by pg_upgrade', 2 if $upgrade_options =~ /upgrade/; + + # Check connection permissions + my $testnc_conn = $upgrade_options =~ /upgrade/ ? 't' : 'f'; + is_program_out 'nobody', 'psql -tAc "SELECT datname, datallowconn FROM pg_database ORDER BY datname" template1', 0, + "postgres|t +template0|f +template1|t +test|t +testnc|$testnc_conn +testro|t +", 'dataallowconn values'; +} + +# check ACLs +is_program_out 'nobody', 'psql -U foo -qc "CREATE SCHEMA s_bar" test', 0, '', + 'CREATE SCHEMA on new cluster (ACL)'; +is_program_out 'nobody', 'psql -U foo -qc "INSERT INTO phone VALUES (\'Chris\', 5)" test', + 0, '', 'insert Chris into phone table (ACL)'; + +# check default transaction r/o +is ((exec_as 'nobody', 'psql test -c "CREATE TABLE test(num int)"'), + 0, 'creating table in test succeeds'); +SKIP: { + skip 'read-only not supported by pg_upgrade', 2 if $upgrade_options =~ /upgrade/; + is ((exec_as 'nobody', 'psql testro -c "CREATE TABLE test(num int)"'), + 1, 'creating table in testro fails'); + is ((exec_as 'postgres', 'psql testro -c "CREATE TABLE test(num int)"'), + 1, 'creating table in testro as superuser fails'); +} +is ((exec_as 'nobody', 'psql testro -c "BEGIN READ WRITE; CREATE TABLE test(num int); COMMIT"'), + 0, 'creating table in testro succeeds with RW transaction'); + +# check config parameters +is_program_out 'postgres', 'psql -Atc "SHOW log_statement" test', 0, "all\n", 'check postgresql.conf parameters'; +SKIP: { + skip 'postgresql.auto.conf not supported before 9.4', 4 if ($MAJORS[0] < 9.4); + is_program_out 'postgres', 'psql -Atc "SHOW log_min_duration_statement" test', 0, "10s\n", 'check postgresql.auto.conf parameter'; + is_program_out 'postgres', "cat /var/lib/postgresql/$MAJORS[-1]/upgr/postgresql.auto.conf", 0, + "# Do not edit this file manually!\n# It will be overwritten by the ALTER SYSTEM command.\nident_file = '/etc/postgresql/$MAJORS[-1]/upgr/pg_ident.conf'\nlog_min_duration_statement = '10s'\n#data_directory = '/var/lib/postgresql/$MAJORS[0]/upgr' #not valid in postgresql.auto.conf\n"; +} +is_program_out 'postgres', 'psql -Atc "SHOW DateStyle" test', 0, "ISO, YMD\n", 'check database parameter'; +SKIP: { + skip "cluster name not supported in $MAJORS[0]", 1 if ($MAJORS[0] < 9.5); + is (PgCommon::get_conf_value ($MAJORS[-1], 'upgr', 'postgresql.conf', 'cluster_name'), "$MAJORS[-1]/upgr", "cluster_name is updated"); +} + +# check tablespace +is_program_out 'postgres', "psql -Atc 'SELECT spcname FROM pg_tablespace ORDER BY spcname'", + 0, "myts\npg_default\npg_global\n", "check tablespace of upgraded table"; +is_program_out 'postgres', "psql -Atc \"SELECT spcname FROM pg_class c LEFT JOIN pg_tablespace t ON (c.reltablespace = t.oid) WHERE c.relname = 'tstab'\"", + 0, "myts\n", "check tablespace of upgraded table"; + +# stop servers, clean up +is ((system "pg_dropcluster $MAJORS[0] upgr --stop"), 0, 'Dropping original cluster'); +is ((system "pg_ctlcluster $MAJORS[-1] upgr restart"), 0, 'Restarting upgraded cluster'); +is_program_out 'nobody', 'psql -Atc "SELECT nextval(\'odd10\')" test', 0, "3\n", + 'upgraded cluster still works after removing old one'; +is ((system "pg_dropcluster $MAJORS[-1] upgr --stop"), 0, 'Dropping upgraded cluster'); +is ((system "rm -rf /var/log/postgresql/pg_upgradecluster-*"), 0, 'Cleaning pg_upgrade log files'); + +check_clean; +} # foreach method + +# vim: filetype=perl diff --git a/t/041_upgrade_custompaths.t b/t/041_upgrade_custompaths.t new file mode 100644 index 0000000..3416b55 --- /dev/null +++ b/t/041_upgrade_custompaths.t @@ -0,0 +1,51 @@ +# Test cluster upgrade with a custom data directory and custom log file. + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More tests => (@MAJORS == 1) ? 1 : 28; + +if (@MAJORS == 1) { + pass 'only one major version installed, skipping upgrade tests'; + exit 0; +} + +ok ((system "pg_createcluster --start --datadir /tmp/postgresql-test -l /tmp/postgresql-test.log $MAJORS[0] upgr >/dev/null") == 0); + +# Upgrade to latest version +my $outref; +is ((exec_as 0, "pg_upgradecluster -v $MAJORS[-1] $MAJORS[0] upgr", $outref, 0), 0, 'pg_upgradecluster succeeds'); +like $$outref, qr/Starting upgraded cluster/, 'pg_upgradecluster reported cluster startup'; +like $$outref, qr/Success. Please check/, 'pg_upgradecluster reported successful operation'; + +# Check clusters +like_program_out 'nobody', 'pg_lsclusters -h', 0, + qr"$MAJORS[0] +upgr 5433 down postgres /tmp/postgresql-test +/tmp/postgresql-test.log\n$MAJORS[-1] +upgr 5432 online postgres /var/lib/postgresql/$MAJORS[-1]/upgr +/var/log/postgresql/postgresql-$MAJORS[-1]-upgr.log", 'pg_lsclusters output'; + +# clean away new cluster and restart the old one +is ((system "pg_dropcluster $MAJORS[-1] upgr --stop"), 0, 'Dropping upgraded cluster'); +is_program_out 0, "pg_ctlcluster $MAJORS[0] upgr start", 0, '', 'Restarting old cluster'; +is_program_out 'nobody', 'pg_lsclusters -h', 0, + "$MAJORS[0] upgr 5433 online postgres /tmp/postgresql-test /tmp/postgresql-test.log +", 'pg_lsclusters output'; + +# Do another upgrade with using a custom defined data directory (and in passing, test --keep-port) +my $outref; +is ((exec_as 0, "pg_upgradecluster --keep-port -v $MAJORS[-1] $MAJORS[0] upgr /tmp/psql-common-testsuite", $outref, 0), 0, 'pg_upgradecluster succeeds'); +unlike $$outref, qr/^pg_restore: /m, 'no pg_restore error messages during upgrade'; +unlike $$outref, qr/^[A-Z]+: /m, 'no server error messages during upgrade'; +like $$outref, qr/Starting upgraded cluster/, 'pg_upgradecluster reported cluster startup'; +like $$outref, qr/Success. Please check/, 'pg_upgradecluster reported successful operation'; + +like_program_out 'nobody', 'pg_lsclusters -h', 0, + qr"$MAJORS[0] +upgr 5433 down postgres /tmp/postgresql-test +/tmp/postgresql-test.log\n$MAJORS[-1] +upgr 5432 online postgres /tmp/psql-common-testsuite +/var/log/postgresql/postgresql-$MAJORS[-1]-upgr.log", 'pg_lsclusters output'; + +# stop servers, clean up +is ((system "pg_dropcluster $MAJORS[0] upgr"), 0, 'Dropping original cluster'); +is ((system "pg_dropcluster $MAJORS[-1] upgr --stop"), 0, 'Dropping upgraded cluster'); + +check_clean; + +# vim: filetype=perl diff --git a/t/042_upgrade_rename.t b/t/042_upgrade_rename.t new file mode 100644 index 0000000..ef44f11 --- /dev/null +++ b/t/042_upgrade_rename.t @@ -0,0 +1,27 @@ +# Test in-version upgrading (usually used after catalog version bumps) + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More tests => 15 * @MAJORS; + +foreach my $v (@MAJORS) { + SKIP: { + skip "pg_upgrade not supported on $v", 15 if ($v < 9.2); + note "PostgreSQL $v"; + + program_ok 0, "pg_createcluster $v main --start", 0; + program_ok 0, "pg_upgradecluster -m upgrade --old-bindir=$PgCommon::binroot$v/bin -v $v --rename upgr $v main", 0; + like_program_out 0, "pg_lsclusters -h", 0, qr/$v main 5433 down.*\n$v upgr 5432 online/; + + program_ok 0, "pg_dropcluster $v main --stop", 0; + program_ok 0, "pg_dropcluster $v upgr --stop", 0; + is ((system "rm -rf /var/log/postgresql/pg_upgradecluster-$v-$v-upgr.*"), 0, 'Cleaning pg_upgrade log files'); + check_clean; + } +} + +# vim: filetype=perl diff --git a/t/043_upgrade_ssl_cert.t b/t/043_upgrade_ssl_cert.t new file mode 100644 index 0000000..d7dc79e --- /dev/null +++ b/t/043_upgrade_ssl_cert.t @@ -0,0 +1,79 @@ +# Test cluster upgrade with a custom ssl certificate + +use strict; +use File::Temp qw/tempdir/; +use lib 't'; +use TestLib; +use PgCommon; +use Test::More tests => (@MAJORS == 1 or $PgCommon::rpm) ? 1 : 22; + +if (@MAJORS == 1) { + pass 'only one major version installed, skipping upgrade tests'; + exit 0; +} +if ($PgCommon::rpm) { + pass 'SSL certificates not handled on RedHat'; + exit 0; +} + + +ok ((system "pg_createcluster $MAJORS[0] upgr >/dev/null") == 0); + +my $tdir = tempdir (CLEANUP => 1); +my ($p_uid, $p_gid) = (getpwnam 'postgres')[2,3]; +chown $p_uid, $p_gid, $tdir; + +my $tempcrt = "$tdir/ssl-cert-snakeoil.pem"; +my $oldcrt = "/var/lib/postgresql/$MAJORS[0]/upgr/server.crt"; +my $newcrt = "/var/lib/postgresql/$MAJORS[-1]/upgr/server.crt"; + +# First upgrade +note "upgrade test: server.crt is a symlink"; +(system "cp -p /etc/ssl/certs/ssl-cert-snakeoil.pem $tempcrt") == 0 or die "cp: $!"; +unlink $oldcrt; # remove file installed by pg_createcluster +symlink $tempcrt, $oldcrt or die "symlink: $!"; + +# Upgrade to latest version +my $outref; +is ((exec_as 0, "pg_upgradecluster --start -v $MAJORS[-1] $MAJORS[0] upgr", $outref, 0), 0, 'pg_upgradecluster succeeds'); +like $$outref, qr/Starting upgraded cluster/, 'pg_upgradecluster reported cluster startup'; +like $$outref, qr/Success. Please check/, 'pg_upgradecluster reported successful operation'; + +if ($MAJORS[-1] >= 9.2) { + is ((-e $newcrt), undef, "new data directory does not contain server.crt"); + is ((PgCommon::get_conf_value $MAJORS[-1], 'upgr', 'postgresql.conf', 'ssl_cert_file'), + $tempcrt, "symlink server.crt target is put into ssl_cert_file"); +} else { + is ((-l $newcrt), 1, "new data directory contains server.crt"); + is ((readlink $newcrt), $tempcrt, "symlink server.crt points to correct location"); +} + +# Clean away new cluster +is ((system "pg_dropcluster $MAJORS[-1] upgr --stop"), 0, 'Dropping upgraded cluster'); +unlink $oldcrt or die "unlink: $!"; + +# Second upgrade +note "upgrade test: server.crt is a plain file"; +(system "cp -p $tempcrt $oldcrt") == 0 or die "cp: $!"; + +# Upgrade to latest version +my $outref; +is ((exec_as 0, "pg_upgradecluster --start -v $MAJORS[-1] $MAJORS[0] upgr", $outref, 0), 0, 'pg_upgradecluster succeeds'); +like $$outref, qr/Starting upgraded cluster/, 'pg_upgradecluster reported cluster startup'; +like $$outref, qr/Success. Please check/, 'pg_upgradecluster reported successful operation'; + +is ((-f $newcrt), 1, "new data directory contains server.crt file"); +if ($MAJORS[-1] >= 9.2) { + is ((PgCommon::get_conf_value $MAJORS[-1], 'upgr', 'postgresql.conf', 'ssl_cert_file'), + $newcrt, "server.crt is put into ssl_cert_file"); +} else { + pass "..."; +} + +# Stop servers, clean up +is ((system "pg_dropcluster $MAJORS[0] upgr"), 0, 'Dropping original cluster'); +is ((system "pg_dropcluster $MAJORS[-1] upgr --stop"), 0, 'Dropping upgraded cluster'); + +check_clean; + +# vim: filetype=perl diff --git a/t/045_backup.t b/t/045_backup.t new file mode 100644 index 0000000..1258ea3 --- /dev/null +++ b/t/045_backup.t @@ -0,0 +1,169 @@ +use strict; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More; +use Time::HiRes qw/usleep/; + +my ($pg_uid, $pg_gid) = (getpwnam 'postgres')[2,3]; +my $systemd = (-d "/run/systemd/system" and not $ENV{_SYSTEMCTL_SKIP_REDIRECT}); +note $systemd ? "We are running systemd" : "We are not running systemd"; + +foreach my $v (@MAJORS) { + if ($v < 9.1) { + ok 1, "pg_backupcluster not supported on $v"; + next; + } + note "PostgreSQL $v"; + + note "create cluster"; + program_ok 0, "pg_createcluster --locale en_US.UTF-8 $v main --start"; + like_program_out 0, "pg_lsclusters -h", 0, qr/$v main 5432 online/; + program_ok 0, "pg_conftool $v main set work_mem 11MB"; + if ($v <= 9.6) { + open my $hba, ">>", "/etc/postgresql/$v/main/pg_hba.conf"; + print $hba "local replication all peer\n"; + close $hba; + program_ok 0, "pg_conftool $v main set max_wal_senders 10"; + program_ok 0, "pg_conftool $v main set wal_level archive"; + program_ok 0, "pg_conftool $v main set max_replication_slots 10" if ($v >= 9.4); + program_ok 0, "pg_conftool $v main set ssl off" if ($v <= 9.1); # cert symlinks not backed up in 9.1 + program_ok 0, "pg_ctlcluster $v main restart"; + } + my $locale_provider = $v >= 15 ? "--locale-provider libc " : ""; + program_ok $pg_uid, "createdb -E SQL_ASCII $locale_provider-T template0 mydb"; + program_ok $pg_uid, "psql -c 'alter database mydb set search_path=public'"; + program_ok $pg_uid, "psql -c 'create table foo (t text)' mydb"; + program_ok $pg_uid, "psql -c \"insert into foo values ('data from backup')\" mydb"; + program_ok $pg_uid, "psql -c 'CREATE USER myuser'"; + program_ok $pg_uid, "psql -c 'alter role myuser set search_path=public, myschema'"; + program_ok $pg_uid, "createdb --locale-provider icu --icu-locale de -T template0 myicudb" if ($v >= 15); + + SKIP: { # in PG 10, ARID is part of globals.sql which we try to restore before databases.sql + skip "alter role in database handling in PG <= 10 not supported", 1 if ($v <= 10); + program_ok $pg_uid, "psql -c 'alter role myuser in database mydb set search_path=public, myotherschema'"; + } + + note "create directory"; + program_ok 0, "pg_backupcluster $v main createdirectory"; + my $dir = "/var/backups/postgresql/$v-main"; + my @stat = stat $dir; + is $stat[4], $pg_uid, "$dir owned by uid postgres"; + is $stat[5], $pg_gid, "$dir owned by gid postgres"; + + my @backups = (); + my $dump = ''; + SKIP: { + skip "dump not supported before 9.3", 1 if ($v < 9.3); + note "dump"; + if ($systemd) { + program_ok 0, "systemctl start pg_dump\@$v-main"; + } else { + program_ok 0, "pg_backupcluster $v main dump"; + } + ($dump) = glob "$dir/*.dump"; + ok -d $dump, "dump created in $dump"; + @stat = stat $dump; + is $stat[4], $pg_uid, "$dump owned by uid postgres"; + is $stat[5], $pg_gid, "$dump owned by gid postgres"; + push @backups, $dump; + } + + note "basebackup"; + my $receivewal_pid; + if ($v >= 9.5) { + if ($systemd) { + program_ok 0, "systemctl start pg_receivewal\@$v-main"; + } else { + $receivewal_pid = fork; + if ($receivewal_pid == 0) { + # suppress "not renaming "000000010000000000000003.gz.partial", segment is not complete" + exec "pg_backupcluster $v main receivewal 2>/dev/null"; + } + } + program_ok $pg_uid, "psql -c 'create table poke_receivewal (t text)' mydb"; + usleep($delay); + my $wal = "000000010000000000000001"; + $wal .= ".gz" if ($v >= 10); + $wal .= ".partial"; + TODO: { + local $TODO = "WAL test is unstable"; + ok_dir "$dir/wal", [$wal], "$dir/wal contains $wal"; + } + } + if ($systemd) { + program_ok 0, "systemctl start pg_basebackup\@$v-main"; + } else { + program_ok 0, "pg_backupcluster --checkpoint=fast $v main basebackup"; + } + my ($basebackup) = glob "$dir/*.backup"; + ok -d $basebackup, "dump created in $basebackup"; + @stat = stat $basebackup; + is $stat[4], $pg_uid, "$basebackup owned by uid postgres"; + is $stat[5], $pg_gid, "$basebackup owned by gid postgres"; + push @backups, $basebackup; + + note "list"; + like_program_out 0, "pg_backupcluster $v main list", 0, qr/$dump.*$basebackup/s; + + note "more database changes"; + program_ok $pg_uid, "psql -c \"insert into foo values ('data later deleted')\" mydb"; + program_ok $pg_uid, "psql -c \"insert into foo values ('data from archive')\" mydb"; + my $timestamp = `su -c "psql -XAtc 'select now()'" postgres`; + ok $timestamp, "retrieve recovery timestamp"; + program_ok $pg_uid, "psql -c \"delete from foo where t = 'data later deleted'\" mydb"; + usleep($delay); + if ($v >= 9.5) { + # since we are stopping pg_receivewal before postgresql, this implicitly tests restoring from .partial WAL files as well + if ($systemd) { + program_ok 0, "systemctl stop pg_receivewal\@$v-main"; + } else { + is kill('INT', $receivewal_pid), 1, "stop receivewal"; + } + } + + for my $backup (@backups) { + note "restore $backup"; + program_ok 0, "pg_dropcluster $v main --stop"; + program_ok 0, "pg_restorecluster $v main $backup --start --datadir /var/lib/postgresql/$v/snowflake"; + like_program_out 0, "pg_lsclusters -h", 0, qr/$v main 5432 online postgres .var.lib.postgresql.$v.snowflake/; + my $outref; + is exec_as($pg_uid, "psql -XAtl", $outref), 0, 'psql -XAtl'; + like $$outref, qr/^mydb\|postgres\|SQL_ASCII\|(libc\|)?en_US.UTF-8\|en_US.UTF-8\|(\|libc\||\|\|)?$/m, "mydb locales"; + like $$outref, qr/^myicudb\|postgres\|UTF8\|(icu\|)?en_US.UTF-8\|en_US.UTF-8\|(de\|icu\||de\|\|)?$/m, "myicudb locales" if ($v >= 15); + is_program_out $pg_uid, "psql -XAtc 'show work_mem'", 0, "11MB\n"; + is_program_out $pg_uid, "psql -XAtc 'select * from foo' mydb", 0, "data from backup\n"; + is_program_out $pg_uid, "psql -XAtc \"select analyze_count between 1 and 3 from pg_stat_user_tables where relname = 'foo'\" mydb", 0, + "t\n"; # --analyze-in-stages does 3 passes + SKIP: { + skip "alter role in database handling in PG <= 10 not supported", 1 if ($v <= 10); + like_program_out $pg_uid, "psql -XAtc '\\drds'", 0, qr/myuser\|mydb\|search_path=public, myotherschema.* +myuser\|\|search_path=public, myschema.* +\|mydb\|search_path=public.*\n/; + } + } + + if ($v >= 9.5) { + note "restore $basebackup with WAL archive"; + program_ok 0, "pg_dropcluster $v main --stop"; + program_ok 0, "pg_restorecluster $v main $basebackup --start --archive --port 5430"; + like_program_out 0, "pg_lsclusters -h", 0, qr/$v main 5430 online postgres .var.lib.postgresql.$v.main/; + is_program_out $pg_uid, "psql -XAtc 'select * from foo order by t' mydb", 0, "data from archive\ndata from backup\n"; + + note "restore $basebackup with PITR"; + program_ok 0, "pg_dropcluster $v main --stop"; + program_ok 0, "pg_restorecluster $v main $basebackup --start --pitr '$timestamp'"; + like_program_out 0, "pg_lsclusters -h", 0, qr/$v main 5432 online postgres .var.lib.postgresql.$v.main/; + is_program_out $pg_uid, "psql -XAtc 'select * from foo order by t' mydb", 0, "data from archive\ndata from backup\ndata later deleted\n"; + } + + program_ok 0, "pg_dropcluster $v main --stop"; + check_clean; + +} # foreach version + +done_testing(); + +# vim: filetype=perl diff --git a/t/050_encodings.t b/t/050_encodings.t new file mode 100644 index 0000000..02dd2f7 --- /dev/null +++ b/t/050_encodings.t @@ -0,0 +1,117 @@ +# Test locale and encoding settings in pg_createcluster. + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More tests => @MAJORS * 52 + 8; + +# create a test cluster with given locale, check the locale/encoding, and +# remove it +# Arguments: <version> <locale> [<encoding>] +sub check_cluster { + my ($v, $locale, $enc) = @_; + note "Checking $v $locale"; + my $cluster_name = $locale; + $cluster_name =~ s/-//g; # strip dashes so postgresql@.service likes it + if (defined $enc) { + $cluster_name .= "_$enc"; + is ((system "LC_ALL='$locale' pg_createcluster --encoding $enc --start $v $cluster_name >/dev/null 2>&1"), 0, + "pg_createcluster version $v for $locale with --encoding succeeded"); + } else { + is ((system "pg_createcluster --start --locale=$locale $v $cluster_name >/dev/null 2>&1"), 0, + "pg_createcluster version $v for $locale without --encoding succeeded"); + } + + # check encoding + sleep 1; + my $outref; + is ((exec_as 'postgres', "psql -Atl --cluster $v/$cluster_name", $outref, 0), 0, + 'psql -l succeeds'); + my $is_unicode = 0; + $is_unicode = 1 if defined $enc && $enc =~ /(UNICODE|UTF-8)/; + $is_unicode = 1 if $locale =~ /UTF-8/; + if ($is_unicode) { + like $$outref, qr/template1.*(UNICODE|UTF8)/, 'template1 is UTF-8 encoded'; + } else { + unlike $$outref, qr/template1.*(UNICODE|UTF8)/, 'template1 is not UTF-8 encoded'; + } + + # create a table and stuff some ISO-8859-5 characters into it (для) + is ((exec_as 'postgres', "createdb test", $outref), 0, 'creating test database'); + is_program_out 'postgres', "printf '\324\333\357' | psql -qc \"set client_encoding='iso-8859-5'; + create table t (x varchar); copy t from stdin\" test", 0, '', + 'creating table with ISO-8859-5 characters'; + is_program_out 'postgres', "echo \"set client_encoding='utf8'; select * from t\" | psql -Atq test", 0, + "\320\264\320\273\321\217\n", 'correct string in UTF-8'; + is_program_out 'postgres', "echo \"set client_encoding='iso-8859-5'; select * from t\" | psql -Atq test", 0, + "\324\333\357\n", 'correct string in ISO-8859-5'; + + # do the same test with using UTF-8 as input + is_program_out 'postgres', "printf '\320\264\320\273\321\217' | psql -qc \"set client_encoding='utf8'; + delete from t; copy t from stdin\" test", 0, '', + 'creating table with UTF-8 characters'; + is_program_out 'postgres', "echo \"set client_encoding='utf8'; select * from t\" | psql -Atq test", 0, + "\320\264\320\273\321\217\n", 'correct string in UTF-8'; + is_program_out 'postgres', "echo \"set client_encoding='iso-8859-5'; select * from t\" | psql -Atq test", 0, + "\324\333\357\n", 'correct string in ISO-8859-1'; + + # check encoding of server error messages (breaks in locale/encoding mismatches, so skip that) + if (!defined $enc) { + # temporarily disable and accept English text, since Russian translations are disabled now + like_program_out 'postgres', 'psql test -c "set client_encoding = \'UTF-8\'; select sqrt(-1)"', 1, + qr/^[^?]*(брать|отрицательного|cannot take square root)[^?]*$/, 'Server error message has correct language and encoding'; + } + + # check that we do not run into 'ignoring unconvertible UTF-8 character' + # breakage on nonmatching lc_messages and client_encoding + PgCommon::set_conf_value $v, $cluster_name, 'postgresql.conf', + 'client_encoding', 'UTF-8'; + PgCommon::set_conf_value $v, $cluster_name, 'postgresql.conf', + 'lc_messages', 'POSIX'; + is_program_out 0, "pg_ctlcluster $v $cluster_name restart", 0, '', + 'cluster starts correctly with nonmatching lc_messages and client_encoding'; + + # check interception of invalidly encoded/escaped strings + if ($is_unicode) { + like_program_out 'postgres', + 'printf "set client_encoding=\'UTF-8\'; select \'\\310\\\\\'a\'" | psql -Atq template1', + 0, qr/(UNICODE|UTF8).*0x(c8.*5c|c8.*27)/, + 'Server rejects incorrect encoding (CVE-2006-2313)'; + like_program_out 'postgres', + 'printf "set client_encoding=\'SJIS\'; select \'\\\\\\\'a\'" | psql -Atq template1', + 0, qr/(\\' is insecure)|(unterminated quoted string)/, + 'Server rejects \\\' escaping in unsafe client encoding (CVE-2006-2314)'; + if ($v >= '9.1') { + like_program_out 'postgres', + "printf \"set client_encoding='UTF-8'; set escape_string_warning='off'; select '\\\\\\'a'\" | psql -Atq template1", + 0, qr/unterminated quoted string/, + 'Server rejects obsolete \\\' escaping in unsafe client encoding (CVE-2006-2314)'; + } else { + is_program_out 'postgres', + "printf \"set client_encoding='UTF-8'; set escape_string_warning='off'; select '\\\\\\'a'\" | psql -Atq template1", + 0, "'a\n", 'Server accepts \\\' escaping in safe client encoding (CVE-2006-2314)'; + } + } + + # drop cluster + is ((system "pg_dropcluster $v $cluster_name --stop"), 0, 'Dropping cluster'); +} + +foreach my $v (@MAJORS) { + check_cluster $v, 'ru_RU'; + check_cluster $v, 'ru_RU.UTF-8'; + + note "Check $v locale environment variables"; + # check LC_* over LANG domination + is ((system "LANGUAGE= LC_ALL=C LANG=bo_GUS.UTF-8 pg_createcluster --start $v main >/dev/null 2>&1"), 0, + "pg_createcluster: LC_ALL dominates LANG"); + like_program_out 'postgres', "psql -Atl --cluster $v/main", 0, + qr/template1.*ASCII/, 'template1 is ASCII encoded'; + is ((system "pg_dropcluster $v main --stop"), 0, 'Dropping cluster'); +} + +check_clean; + +# vim: filetype=perl diff --git a/t/052_upgrade_encodings.t b/t/052_upgrade_encodings.t new file mode 100644 index 0000000..e7a7125 --- /dev/null +++ b/t/052_upgrade_encodings.t @@ -0,0 +1,98 @@ +# Test default and explicit encoding on upgrades + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More tests => (@MAJORS == 1) ? 1 : 46; + +if (@MAJORS == 1) { + pass 'only one major version installed, skipping upgrade tests'; + exit 0; +} + +my $outref; +my $oldv = $MAJORS[0]; +my $newv = $MAJORS[-1]; + +is ((exec_as 0, "pg_createcluster --start --locale=ru_RU $oldv main", $outref), 0, + "creating ru_RU $oldv cluster"); + +is ((exec_as 'postgres', 'psql -c "create database latintest" template1', $outref), 0, + "creating latintest DB with LATIN encoding"); +if ($oldv <= '8.3') { + is ((exec_as 'postgres', 'psql -c "create database asctest encoding = \'SQL_ASCII\'" template1', $outref), 0, + "creating asctest DB with ASCII encoding"); +} else { + is ((exec_as 'postgres', 'psql -c "create database asctest template = template0 lc_collate = \'C\' lc_ctype = \'C\' encoding = \'SQL_ASCII\'" template1', $outref), 0, + "creating asctest DB with C locale"); +} +if ($oldv >= 15) { + program_ok 'postgres', 'psql -c "create database icutest template template0 locale_provider icu icu_locale de"', 0, "creating database with ICU locale"; +} else { + program_ok 'postgres', 'psql -c "create database icutest"', 0, "creating placeholder icutest database"; +} + +is ((exec_as 'postgres', "printf 'A\\324B' | psql -c \"create table t(x varchar); copy t from stdin\" latintest", $outref), + 0, 'write LATIN database content to latintest'); +is ((exec_as 'postgres', "printf 'A\\324B' | psql -c \"create table t(x varchar); copy t from stdin\" asctest", $outref), + 0, 'write LATIN database content to asctest'); + +is_program_out 'postgres', "echo \"select * from t\" | psql -Atq latintest", + 0, "A\324B\n", 'old latintest DB has correctly encoded string'; +is_program_out 'postgres', "echo \"select * from t\" | psql -Atq asctest", + 0, "A\324B\n", 'old asctest DB has correctly encoded string'; + +is ((exec_as 'postgres', 'psql -Atl', $outref), 0, 'psql -Atl on old cluster'); +ok ((index $$outref, 'latintest|postgres|ISO_8859_5') >= 0, 'latintest is LATIN encoded'); +ok ((index $$outref, 'asctest|postgres|SQL_ASCII') >= 0, 'asctest is ASCII encoded'); +if ($oldv >= 15) { + like $$outref, qr/icutest\|postgres\|ISO_8859_5\|icu\|ru_RU\|ru_RU\|de/, 'icutest has proper icu locale'; +} else { + like $$outref, qr/icutest\|postgres\|ISO_8859_5/, 'icutest is LATIN encoded'; +} +ok ((index $$outref, 'template1|postgres|ISO_8859_5') >= 0, 'template1 is LATIN encoded'); + +# upgrade without specifying locales, should be kept +like_program_out 0, "pg_upgradecluster -v $newv $oldv main", 0, qr/^Success. Please check/m; + +is ((exec_as 'postgres', "psql --cluster $newv/main -Atl", $outref), 0, 'psql -Atl on upgraded cluster'); +ok ((index $$outref, 'latintest|postgres|ISO_8859_5') >= 0, 'latintest is LATIN encoded'); +ok ((index $$outref, 'asctest|postgres|SQL_ASCII') >= 0, 'asctest is ASCII encoded'); +if ($oldv >= 15) { + like $$outref, qr/icutest\|postgres\|ISO_8859_5\|icu\|ru_RU\|ru_RU\|de/, 'icutest has proper icu locale'; +} else { + like $$outref, qr/icutest\|postgres\|ISO_8859_5/, 'icutest is LATIN encoded'; +} +ok ((index $$outref, 'template1|postgres|ISO_8859_5') >= 0, 'template1 is LATIN encoded'); +is_program_out 'postgres', "echo \"select * from t\" | psql --cluster $newv/main -Atq latintest", + 0, "A\324B\n", 'new latintest DB has correctly encoded string'; + +is ((system "pg_dropcluster --stop $newv main"), 0, 'dropping upgraded cluster'); +is ((system "pg_ctlcluster $oldv main start"), 0, 'restarting old cluster'); + +# upgrade with explicitly specifying other locale +like_program_out 0, "pg_upgradecluster --locale ru_RU.UTF-8 -v $newv $oldv main", 0, qr/^Success. Please check/m; + +is ((exec_as 'postgres', "psql --cluster $newv/main -Atl", $outref), 0, 'psql -Atl on upgraded cluster'); +if ($newv >= 11) { + ok ((index $$outref, 'latintest|postgres|ISO_8859_5') >= 0, 'latintest is still LATIN encoded'); +} else { + like $$outref, qr/latintest\|postgres\|(UTF8|UNICODE)/, 'latintest is now UTF8 encoded'; +} +ok ((index $$outref, 'asctest|postgres|SQL_ASCII') >= 0, 'asctest is ASCII encoded'); +like $$outref, qr/template1\|postgres\|(UTF8|UNICODE)/, 'template1 is UTF8 encoded'; +is_program_out 'postgres', "echo \"select * from t\" | psql --cluster $newv/main -Atq latintest", + 0, ($newv >= 11 ? "A\324B\n": "AдB\n"), 'new latintest DB has correctly encoded string'; +# ASCII databases don't do automatic encoding conversion, so this remains LATIN +is_program_out 'postgres', "echo \"select * from t\" | psql --cluster $newv/main -Atq asctest", + 0, "A\324B\n", 'new asctest DB has correctly encoded string'; + +is ((system "pg_dropcluster --stop $newv main"), 0, 'dropping upgraded cluster'); + +is ((system "pg_dropcluster $oldv main"), 0, 'dropping old cluster'); + +check_clean; + +# vim: filetype=perl diff --git a/t/060_obsolete_confparams.t b/t/060_obsolete_confparams.t new file mode 100644 index 0000000..c276431 --- /dev/null +++ b/t/060_obsolete_confparams.t @@ -0,0 +1,83 @@ +# Test upgrading from the oldest version to all majors with all possible +# configuration parameters set. This checks that they are correctly +# transitioned. + +use strict; + +use lib 't'; +use TestLib; + +use Test::More; + +if (@MAJORS == 1) { + pass 'only one major version installed, skipping upgrade tests'; + done_testing(); + exit 0; +} + +$ENV{_SYSTEMCTL_SKIP_REDIRECT} = 1; # FIXME: testsuite is hanging otherwise + +# Test one particular upgrade (old version, new version) +sub do_upgrade { + my $cur = $_[0]; + my $new = $_[1]; + note "Testing upgrade $cur -> $new"; + + # Upgrade cluster + like_program_out 0, "env LC_MESSAGES=C pg_upgradecluster -v $new $cur main", 0, qr/^Success. Please check/m; + like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/$new.*online/, + "New $new cluster is online"; +} + +# create cluster for oldest version +is_program_out 0, "pg_createcluster $MAJORS[0] main >/dev/null", 0, ""; + +# generate configuration file with all settings and start cluster +is_program_out 0, "sed -i -e 's/^#\\([a-z]\\)/\\1/' /etc/postgresql/$MAJORS[0]/main/postgresql.conf", + 0, "", "Enabling all settings in /etc/postgresql/$MAJORS[0]/main/postgresql.conf"; +like PgCommon::get_conf_value($MAJORS[0], 'main', 'postgresql.conf', 'work_mem'), qr/MB/, "work_mem is set"; + +# tweak invalid settings +PgCommon::set_conf_value $MAJORS[0], 'main', 'postgresql.conf', 'log_timezone', 'UTC'; +PgCommon::set_conf_value $MAJORS[0], 'main', 'postgresql.conf', 'timezone', 'UTC'; +PgCommon::disable_conf_value $MAJORS[0], 'main', 'postgresql.conf', 'include_dir', "Disable placeholder value"; +PgCommon::disable_conf_value $MAJORS[0], 'main', 'postgresql.conf', 'include_if_exists', "Disable placeholder value"; +PgCommon::disable_conf_value $MAJORS[0], 'main', 'postgresql.conf', 'include', "Disable placeholder value"; +# older versions (<= 9.1 as of 2019-03) do not support ssl anymore +my $postgres = PgCommon::get_program_path('postgres', $MAJORS[0]); +my $ldd = `ldd $postgres 2>/dev/null`; +if ($ldd and $ldd !~ /libssl/) { + is_program_out 0, "sed -i -e 's/^ssl/#ssl/' /etc/postgresql/$MAJORS[0]/main/postgresql.conf", + 0, "", "Disabling ssl settings on server that does not support SSL"; +} + +# start server +is_program_out 0, "pg_ctlcluster $MAJORS[0] main start", 0, ""; + +# Loop over all but the latest major version, testing N->N+1 upgrades +for my $index (0 .. @MAJORS - 2) { + do_upgrade $MAJORS[$index], $MAJORS[$index + 1] +} +# remove all clusters except for the first one +for my $index (1 .. @MAJORS - 1) { + is_program_out 0, "pg_dropcluster $MAJORS[$index] main --stop", 0, "", "Dropping $MAJORS[$index]/main"; +} + +# now test a direct upgrade from oldest to newest, to also catch parameters +# which changed several times, like syslog -> redirect_stderr -> +# logging_collector +if ($#MAJORS > 1) { + is_program_out 0, "pg_ctlcluster $MAJORS[0] main start", 0, ""; + do_upgrade $MAJORS[0], $MAJORS[-1]; + is_program_out 0, "pg_dropcluster $MAJORS[-1] main --stop", 0, "", "Dropping $MAJORS[-1]/main"; +} else { + pass 'only two available versions, skipping tests...'; +} + +# remove first cluster +is_program_out 0, "pg_dropcluster $MAJORS[0] main --stop", 0, "", "Dropping $MAJORS[0]/main"; + +check_clean; +done_testing(); + +# vim: filetype=perl diff --git a/t/070_non_postgres_clusters.t b/t/070_non_postgres_clusters.t new file mode 100644 index 0000000..06d7f13 --- /dev/null +++ b/t/070_non_postgres_clusters.t @@ -0,0 +1,116 @@ +# Test successful operation of clusters which are not owned by +# postgres. Only check the oldest and newest version. + +use strict; + +use lib 't'; +use TestLib; + +use Test::More tests => 40; + +$ENV{_SYSTEMCTL_SKIP_REDIRECT} = 1; # FIXME: testsuite is hanging otherwise + +my $owner = 'nobody'; +my $v = $MAJORS[0]; + +# create cluster +is ((system "pg_createcluster -u $owner $v main >/dev/null"), 0, + "pg_createcluster $v main for owner $owner"); + +# check if start is refused when config and data owner do not match +my $pgconf = "/etc/postgresql/$v/main/postgresql.conf"; +my ($origuid, $origgid) = (stat $pgconf)[4,5]; +chown 1, 1, $pgconf; +like_program_out 0, "pg_ctlcluster $v main start", 1, qr/do not match/, "start refused when config and data owners mismatch"; +chown $origuid, $origgid, $pgconf; +is ((system "pg_ctlcluster $v main start"), 0, "pg_ctlcluster succeeds with owner $owner"); + +# Check cluster +like_program_out $owner, 'pg_lsclusters -h', 0, + qr/^$v\s+main\s+5432\s+online\s+$owner/, + 'pg_lsclusters shows running cluster'; + +like ((ps 'postgres'), qr/^$owner.*bin\/postgres .*\/var\/lib\/postgresql\/$v\/main/m, + "postgres is running as user $owner"); + +is_program_out $owner, 'ls /tmp/.s.PGSQL.*', 0, "/tmp/.s.PGSQL.5432\n/tmp/.s.PGSQL.5432.lock\n", 'socket is in /tmp'; + +ok_dir '/var/run/postgresql', [], '/var/run/postgresql is empty'; + +# verify owner of configuration files +my @st; +my $confdir = "/etc/postgresql/$v/main"; +my ($owneruid, $ownergid) = (getpwnam $owner)[2,3]; +@st = stat $confdir; +is $st[4], $owneruid, 'conf dir is owned by user'; +is $st[5], $ownergid, 'conf dir is owned by user\'s primary group'; +my ($ok_uid, $ok_gid) = (1, 1); +opendir D, $confdir or die "opendir: $!"; +for my $f (readdir D) { + next if $f eq '.' or $f eq '..'; + @st = stat "$confdir/$f" or die "stat: $!"; + if ($st[4] != $owneruid) { + note "$f is not owned by user"; + $ok_uid = 0; + } + if ($st[5] != $ownergid) { + note "$f is not owned by user's primary group"; + $ok_gid = 0; + } +} +closedir D; +is $ok_uid, 1, "files are owned by user"; +is $ok_gid, 1, "files are owned by user's primary group"; + +# verify log file properties +@st = stat "/var/log/postgresql/postgresql-$v-main.log"; +is $st[2], 0100640, 'log file has 0640 permissions'; +is $st[4], $owneruid, 'log file is owned by user'; +# the log file gid setting works on RedHat, but nobody has gid 99 there (and +# there's not good alternative for testing) +my $loggid = $PgCommon::rpm ? (getgrnam 'adm')[2] : $ownergid; +is $st[5], $loggid, 'log file is owned by user\'s primary group'; + +if ($#MAJORS > 0) { + my $newv = $MAJORS[-1]; + + my $outref; + is ((exec_as 0, "(pg_upgradecluster -v $newv $v main | sed -e 's/^/STDOUT: /')", $outref, 0), 0, + 'pg_upgradecluster succeeds'); + like $$outref, qr/Starting upgraded cluster/, 'pg_upgradecluster reported cluster startup'; + like $$outref, qr/Success. Please check/, 'pg_upgradecluster reported successful operation'; + my @err = grep (!/^STDOUT: /, split (/\n/, $$outref)); + if (@err) { + fail 'no error messages during upgrade'; + print (join ("\n", @err)); + } else { + pass "no error messages during upgrade"; + } + + # verify file permissions + @st = stat "/etc/postgresql/$newv/main"; + is $st[4], $owneruid, 'upgraded conf dir is owned by user'; + is $st[5], $ownergid, 'upgraded conf dir is owned by user\'s primary group'; + @st = stat "/etc/postgresql/$newv/main/postgresql.conf"; + is $st[4], $owneruid, 'upgraded postgresql.conf dir is owned by user'; + is $st[5], $ownergid, 'upgraded postgresql.conf dir is owned by user\'s primary group'; + @st = stat "/var/log/postgresql/postgresql-$v-main.log"; + is $st[4], $owneruid, 'upgraded log file is owned by user'; + is $st[5], $loggid, 'upgraded log file is owned by user\'s primary group'; + + is ((system "pg_dropcluster $newv main --stop"), 0, 'pg_dropcluster'); +} else { + pass 'only one major version installed, skipping upgrade test'; + for (my $i = 0; $i < 10; ++$i) { + pass '...'; + } +} + +# Check proper cleanup +is ((system "pg_dropcluster $v main --stop"), 0, 'pg_dropcluster'); +is_program_out $owner, 'pg_lsclusters -h', 0, '', 'No clusters left'; +is ((ps 'postgres'), '', "No postgres processes left"); + +check_clean; + +# vim: filetype=perl diff --git a/t/080_start.conf.t b/t/080_start.conf.t new file mode 100644 index 0000000..2e4d4cf --- /dev/null +++ b/t/080_start.conf.t @@ -0,0 +1,145 @@ +# Check start.conf handling. + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More tests => 73; + +my $systemd = -d '/run/systemd/system'; + +# Do test with oldest version +my $v = $MAJORS[0]; + +# create cluster +is ((system "pg_createcluster $v main >/dev/null"), 0, "pg_createcluster $v main"); + +# Check that we start with 'auto' +note "start.conf auto"; +is ((get_cluster_start_conf $v, 'main'), 'auto', + 'get_cluster_start_conf returns auto'); +is_program_out 'nobody', "grep '^[^\\s#]' /etc/postgresql/$v/main/start.conf", + 0, "auto\n", 'start.conf contains auto'; +SKIP: { + skip 'not running under systemd', 2 unless ($systemd); + ok_dir '/run/systemd/generator/postgresql.service.wants', + ["postgresql\@$v-main.service"], + "systemd generator links cluster"; + is ((readlink "/run/systemd/generator/postgresql.service.wants/postgresql\@$v-main.service"), + "/lib/systemd/system/postgresql@.service", + "systemd generator links correct service file"); +} + +# init script should handle auto cluster +like_program_out 0, "/etc/init.d/postgresql start $v", 0, qr/Start.*($v|systemctl)/; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/online/, 'cluster is online'; +like_program_out 0, "/etc/init.d/postgresql stop $v", 0, qr/Stop.*($v|systemctl)/; +sleep 3 if ($systemd); # FIXME: systemctl stop postgresql is not yet synchronous (#759725) +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/down/, 'cluster is down'; + +# change to manual, verify start.conf contents +note "start.conf manual"; +set_cluster_start_conf $v, 'main', 'manual'; + +is ((get_cluster_start_conf $v, 'main'), 'manual', + 'get_cluster_start_conf returns manual'); +is_program_out 'nobody', "grep '^[^\\s#]' /etc/postgresql/$v/main/start.conf", + 0, "manual\n", 'start.conf contains manual'; +SKIP: { + skip 'not running under systemd', 1 unless ($systemd); + system "systemctl daemon-reload"; + ok_dir '/run/systemd/generator/postgresql.service.wants', + [], "systemd generator doesn't link cluster"; +} + +# init script should not handle manual cluster ... +like_program_out 0, "/etc/init.d/postgresql start $v", 0, qr/Start.*($v|systemctl)/; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/down/, 'cluster is down'; + +# pg_ctlcluster should handle manual cluster +is_program_out 'postgres', "pg_ctlcluster $v main start", 0, ''; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/online/, 'cluster is online'; +is_program_out 'postgres', "pg_ctlcluster $v main stop", 0, ''; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/down/, 'cluster is down'; + +# change to disabled, verify start.conf contents +note "start.conf disabled"; +set_cluster_start_conf $v, 'main', 'disabled'; + +is ((get_cluster_start_conf $v, 'main'), 'disabled', + 'get_cluster_start_conf returns disabled'); +SKIP: { + skip 'not running under systemd', 1 unless ($systemd); + system "systemctl daemon-reload"; + ok_dir '/run/systemd/generator/postgresql.service.wants', + [], "systemd generator doesn't link cluster"; +} + +# init script should not handle disabled cluster +like_program_out 0, "/etc/init.d/postgresql start $v", 0, qr/Start.*($v|systemctl)/; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/down/, 'cluster is down'; + +# pg_ctlcluster should not start disabled cluster +is_program_out 'postgres', "pg_ctlcluster $v main start", 1, + "Error: Cluster is disabled\n"; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/down/, 'cluster is down'; + +# change back to manual, start cluster +set_cluster_start_conf $v, 'main', 'manual'; +is_program_out 'postgres', "pg_ctlcluster $v main start", 0, ''; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/online/, 'cluster is online'; + +# however, we want to stop disabled clusters +set_cluster_start_conf $v, 'main', 'disabled'; +is_program_out 'postgres', "pg_ctlcluster $v main stop", 0, ''; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/down/, 'cluster is down'; + +# set back to manual +set_cluster_start_conf $v, 'main', 'manual'; +is_program_out 'postgres', "pg_ctlcluster $v main start", 0, ''; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/online/, 'cluster is online'; + +# upgrade cluster +note "test upgrade"; +if ($#MAJORS == 0) { + pass 'only one major version installed, skipping upgrade test'; + pass '...'; +} else { + like_program_out 0, "pg_upgradecluster -v $MAJORS[-1] $v main", 0, qr/Success. Please check/; +} + +# check start.conf of old and upgraded cluster +is ((get_cluster_start_conf $v, 'main'), 'manual', + 'get_cluster_start_conf for old cluster returns manual'); +is ((get_cluster_start_conf $MAJORS[-1], 'main'), 'manual', + 'get_cluster_start_conf for new cluster returns manual'); + +# clean up +if ($#MAJORS == 0) { + pass '...'; +} else { + is ((system "pg_dropcluster $v main"), 0, + 'dropping old cluster'); +} + +is ((system "pg_dropcluster $MAJORS[-1] main --stop"), 0, + 'dropping upgraded cluster'); + +is_program_out 'postgres', 'pg_lsclusters -h', 0, '', 'no clusters any more'; + +# create cluster with --start-conf option +is_program_out 0, "pg_createcluster $v main --start-conf foo", 1, + "Error: Invalid --start-conf value: foo\n", + 'pg_createcluster checks --start-conf validity'; +is ((system "pg_createcluster $v main --start-conf manual >/dev/null"), 0, + 'pg_createcluster checks --start-conf manual'); +is ((get_cluster_start_conf $v, 'main'), 'manual', + 'get_cluster_start_conf returns manual'); +is ((system "pg_dropcluster $v main"), 0, + 'dropping cluster'); + +check_clean; + +# vim: filetype=perl diff --git a/t/085_pg_ctl.conf.t b/t/085_pg_ctl.conf.t new file mode 100644 index 0000000..dca2700 --- /dev/null +++ b/t/085_pg_ctl.conf.t @@ -0,0 +1,51 @@ +# Check pg_ctl.conf handling. + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More tests => $MAJORS[-1] >= '8.3' ? 33 : 1; + +# Do test with newest version +my $v = $MAJORS[-1]; +if ($v < '8.3') { + pass 'Skipping core limit tests for versions < 8.3'; + exit 0; +} + +# enable core dumps +# sudo and salsa-ci set the hard limit to 0 by default, undo that +is_program_out 0, "prlimit --core=0:unlimited --pid=$$", 0, '', "set core file size to unlimited"; +is_program_out 'postgres', "sh -c 'ulimit -Hc'", 0, "unlimited\n", "core file size is unlimited"; + +# create cluster +is ((system "pg_createcluster $v main >/dev/null"), 0, "pg_createcluster $v main"); +ok (-f "/etc/postgresql/$v/main/pg_ctl.conf", "/etc/postgresql/$v/main/pg_ctl.conf exists"); + +# Default behaviour, core size=0 +is_program_out 0, "pg_ctlcluster $v main start", 0, '', "starting cluster as root"; +is_program_out 'postgres', "xargs -i awk '/core/ {print \$5}' /proc/{}/limits < /var/run/postgresql/$v-main.pid", 0, "0\n", "soft core size is 0"; +my $hard_limit = `xargs -i awk '/core/ {print \$6}' /proc/{}/limits < /var/run/postgresql/$v-main.pid`; +chomp $hard_limit; +note "hard core file size limit of root-started postgres process is $hard_limit"; + +# -c in pg_ctl.conf, core size=unlimited +ok (set_cluster_pg_ctl_conf($v, 'main', '-c'), "set pg_ctl default option to -c"); +is_program_out 0, "pg_ctlcluster $v main restart", 0, '', "restarting cluster as root"; +is_program_out 'postgres', "xargs -i awk '/core/ {print \$5}' /proc/{}/limits < /var/run/postgresql/$v-main.pid", 0, "$hard_limit\n", "soft core size is $hard_limit"; + +# Back to default behaviour, core size=0 +is_program_out 0, "pg_ctlcluster $v main stop", 0, '', "stopping cluster"; +ok (set_cluster_pg_ctl_conf($v, 'main', ''), "restored pg_ctl default option"); + +# pg_ctl -c, core size=unlimited +is_program_out 'postgres', "pg_ctlcluster $v main start -- -c", 0, '', "starting cluster with -c on the command line as postgres"; +is_program_out 'postgres', "xargs -i awk '/core/ {print \$5}' /proc/{}/limits < /var/run/postgresql/$v-main.pid", 0, "unlimited\n", "soft core size is unlimited"; +is_program_out 'postgres', "pg_ctlcluster $v main stop", 0, '', "stopping cluster"; + +is ((system "pg_dropcluster $v main --stop"), 0, 'dropping cluster'); +check_clean; + +# vim: filetype=perl diff --git a/t/090_multicluster.t b/t/090_multicluster.t new file mode 100644 index 0000000..8322ace --- /dev/null +++ b/t/090_multicluster.t @@ -0,0 +1,286 @@ +# Check operation with multiple clusters + +use strict; + +use lib 't'; +use TestLib; +use Socket; +use PgCommon; + +use Test::More tests => 125; + +# create fake socket at 5433 to verify that this port is skipped +socket (SOCK, PF_INET, SOCK_STREAM, getprotobyname('tcp')) or die "socket: $!"; +setsockopt(SOCK, Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1) or die "setsockopt: $!"; +bind (SOCK, sockaddr_in(5433, INADDR_ANY)) || die "bind: $! "; +listen (SOCK, 0) or die "listen: $!"; + +# create clusters +is ((system "pg_createcluster $MAJORS[0] old -- -A trust >/dev/null"), 0, "pg_createcluster $MAJORS[0] old"); +is ((system "pg_createcluster $MAJORS[-1] new1 -- -A trust >/dev/null"), 0, "pg_createcluster $MAJORS[-1] new1"); +is ((system "pg_createcluster $MAJORS[-1] new2 -p 5440 -- -A trust >/dev/null"), 0, "pg_createcluster $MAJORS[-1] new2"); +close SOCK; + +my $old = "$MAJORS[0]/old"; +my $new1 = "$MAJORS[-1]/new1"; +my $new2 = "$MAJORS[-1]/new2"; + +is ((system "pg_ctlcluster $MAJORS[0] old start >/dev/null"), 0, "starting cluster $old"); +is ((system "pg_ctlcluster $MAJORS[-1] new1 start >/dev/null"), 0, "starting cluster $new1"); +is ((system "pg_ctlcluster $MAJORS[-1] new2 start >/dev/null"), 0, "starting cluster $new2"); + +like_program_out 'postgres', 'pg_lsclusters -h | sort -k3', 0, qr/.*5432.*5434.*5440.*/s, + 'clusters have the correct ports, skipping used 5433'; + +# move user_clusters aside for the test; this will ensure that client programs +# work correctly without any file at all +if (-f '/etc/postgresql-common/user_clusters') { + ok ((rename '/etc/postgresql-common/user_clusters', + '/etc/postgresql-common/user_clusters.psqltestsuite'), + 'Temporarily moving away /etc/postgresql-common/user_clusters'); +} else { + pass '/etc/postgresql-common/user_clusters does not exist'; +} + +# check basic cluster selection +like_program_out 0, 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $MAJORS[0]/, + 'pg_wrapper selects port 5432 as default cluster'; +like_program_out 0, "createdb --cluster $new1 --version", 0, + qr/^createdb \(PostgreSQL\) $MAJORS[-1]/, + 'pg_wrapper --cluster works'; +like_program_out 0, "createdb --cluster $MAJORS[-1]/foo --version", 1, + qr/Cluster .* does not exist/, + 'pg_wrapper --cluster errors out for invalid cluster'; + +# create a database in new1 and check that it doesn't appear in new2 +is_program_out 'postgres', "createdb --cluster $new1 test", 0, ($MAJORS[-1] < 8.3 ? "CREATE DATABASE\n" : ''); +like_program_out 'postgres', "psql -Atl --cluster $new1", 0, + qr/test\|postgres\|/, + 'test db appears in cluster new1'; +unlike_program_out 'postgres', "psql -Atl --cluster $new2", 0, + qr/test\|postgres\|/, + 'test db does not appear in cluster new2'; +unlike_program_out 'postgres', "psql -Atl", 0, qr/test\|postgres\|/, + 'test db does not appear in default cluster'; + +# check network cluster selection +is_program_out 'postgres', "psql --cluster $MAJORS[0]/127.0.0.1: -Atc 'show port' template1", 0, "5432\n", + "psql --cluster $MAJORS[0]/127.0.0.1: defaults to port 5432"; +like_program_out 'postgres', "createdb --cluster $MAJORS[-1]/127.0.0.1:5432 --version", 0, + qr/^createdb \(PostgreSQL\) $MAJORS[-1]/, + "createdb --cluster $MAJORS[-1]/127.0.0.1:5432 uses latest client version"; +like_program_out 'postgres', "psql -Atl --cluster $MAJORS[-1]/localhost:5434", 0, + qr/test\|postgres\|/, "test db appears in cluster $MAJORS[-1]/localhost:5434"; +unlike_program_out 'postgres', "psql -Atl --cluster $MAJORS[-1]/localhost:5440", 0, + qr/test\|postgres\|/, "test db does not appear in cluster $MAJORS[-1]/localhost:5440"; + +# check some erroneous cluster specifications +like_program_out 'postgres', "LC_MESSAGES=C psql -Atl --cluster $MAJORS[-1]/localhost:5435", 2, + qr/could not connect|connection to server .* failed/, "psql --cluster $MAJORS[-1]/localhost:5435 fails due to nonexisting port"; +like_program_out 'postgres', "LC_MESSAGES=C psql -Atl --cluster $MAJORS[-1]/localhost:a", 1, + qr/Cluster .* does not exist/, "psql --cluster $MAJORS[-1]/localhost:a fails due to invalid syntax"; +like_program_out 'postgres', "LC_MESSAGES=C psql -Atl --cluster $MAJORS[-1]/doesnotexi.st", 1, + qr/Cluster .* does not exist/, "psql --cluster $MAJORS[-1]/doesnotexi.st fails due to invalid syntax"; +like_program_out 'postgres', "psql -Atl --cluster 6.4/localhost:", 1, + qr/Invalid version/, "psql --cluster 6.4/localhost: fails due to invalid version"; + +# check that environment variables work +$ENV{'PGCLUSTER'} = $new1; +like_program_out 'postgres', "psql -Atl", 0, qr/test\|postgres\|/, + 'PGCLUSTER selection (1)'; +$ENV{'PGCLUSTER'} = $new2; +unlike_program_out 'postgres', "psql -Atl", 0, qr/test\|postgres\|/, + 'PGCLUSTER selection (2)'; +$ENV{'PGCLUSTER'} = 'foo'; +like_program_out 'postgres', "psql -l", 1, + qr/Invalid version .* specified in PGCLUSTER/, + 'invalid PGCLUSTER value'; +$ENV{'PGCLUSTER'} = "$MAJORS[-1]/127.0.0.1:"; +like_program_out 0, 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $MAJORS[-1]/, + 'PGCLUSTER network cluster selection (1)'; +$ENV{'PGCLUSTER'} = "$MAJORS[-1]/localhost:5434"; +like_program_out 'postgres', 'psql -Atl', 0, + qr/test\|postgres\|/, 'PGCLUSTER network cluster selection (2)'; +$ENV{'PGCLUSTER'} = "$MAJORS[-1]/localhost:5440"; +unlike_program_out 'postgres', 'psql -Atl', 0, + qr/test\|postgres\|/, 'PGCLUSTER network cluster selection (3)'; +$ENV{'PGCLUSTER'} = "$MAJORS[-1]/localhost:5435"; +like_program_out 'postgres', 'LC_MESSAGES=C psql -Atl', 2, + qr/could not connect|connection to server .* failed/, "psql --cluster $MAJORS[-1]/localhost:5435 fails due to nonexisting port"; +delete $ENV{'PGCLUSTER'}; + +# check that PGPORT works +$ENV{'PGPORT'} = '5434'; +is_program_out 'postgres', 'psql -Atc "show port" template1', 0, "5434\n", + 'PGPORT selection (1)'; +$ENV{'PGPORT'} = '5432'; +is_program_out 'postgres', 'psql -Atc "show port" template1', 0, "5432\n", + 'PGPORT selection (2)'; +$ENV{'PGCLUSTER'} = $new2; +delete $ENV{'PGPORT'}; +$ENV{'PGPORT'} = '5432'; +like_program_out 'postgres', 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $MAJORS[-1]/, + 'PGPORT+PGCLUSTER, PGCLUSTER selects version'; +is_program_out 'postgres', 'psql -Atc "show port" template1', 0, "5432\n", + 'PGPORT+PGCLUSTER, PGPORT selects port'; +delete $ENV{'PGPORT'}; +delete $ENV{'PGCLUSTER'}; + +# check that PGDATABASE works +$ENV{'PGDATABASE'} = 'test'; +is_program_out 'postgres', "psql --cluster $new1 -Atc 'select current_database()'", 0, "test\n", + 'PGDATABASE environment variable works'; +delete $ENV{'PGDATABASE'}; + +# check cluster selection with an empty user_clusters +open F, '>/etc/postgresql-common/user_clusters' or die "Could not create user_clusters: $!"; +close F; +chmod 0644, '/etc/postgresql-common/user_clusters'; +like_program_out 0, 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $MAJORS[0]/, + 'pg_wrapper selects port 5432 as default cluster with empty user_clusters'; +like_program_out 0, "createdb --cluster $new1 --version", 0, + qr/^createdb \(PostgreSQL\) $MAJORS[-1]/, + 'pg_wrapper --cluster works with empty user_clusters'; + +# check default cluster selection with user_clusters +open F, '>/etc/postgresql-common/user_clusters' or die "Could not create user_clusters: $!"; +print F "* * $MAJORS[-1] new1 *\n"; +close F; +chmod 0644, '/etc/postgresql-common/user_clusters'; +like_program_out 'postgres', 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $MAJORS[-1]/, + "pg_wrapper selects correct cluster with user_clusters '* * $MAJORS[-1] new1 *'"; + +# check default database selection with user_clusters +open F, '>/etc/postgresql-common/user_clusters' or die "Could not create user_clusters: $!"; +print F "* * $MAJORS[-1] new1 test\n"; +close F; +chmod 0644, '/etc/postgresql-common/user_clusters'; +is_program_out 'postgres', 'psql -Atc "select current_database()"', 0, "test\n", + "pg_wrapper selects correct database with user_clusters '* * $MAJORS[-1] new1 test'"; +$ENV{'PGDATABASE'} = 'template1'; +is_program_out 'postgres', "psql -Atc 'select current_database()'", 0, "template1\n", + 'PGDATABASE environment variable is not overridden by user_clusters'; +delete $ENV{'PGDATABASE'}; + +# check by-user cluster selection with user_clusters +# (also check invalid cluster reporting) +open F, '>/etc/postgresql-common/user_clusters' or die "Could not create user_clusters: $!"; +print F "postgres * $MAJORS[-1] new1 *\nnobody * $MAJORS[0] old *\n* * 5.5 * *"; +close F; +chmod 0644, '/etc/postgresql-common/user_clusters'; +like_program_out 'postgres', 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $MAJORS[-1]/, + 'pg_wrapper selects correct cluster with per-user user_clusters'; +like_program_out 'nobody', 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $MAJORS[0]/, + 'pg_wrapper selects correct cluster with per-user user_clusters'; +like_program_out 0, 'createdb --version', 0, qr/user_clusters.*line 3.*version.*not exist/i, + 'pg_wrapper warning for invalid per-user user_clusters line'; + +# check by-user network cluster selection with user_clusters +# (also check invalid cluster reporting) +open F, '>/etc/postgresql-common/user_clusters' or die "Could not create user_clusters: $!"; +print F "postgres * $MAJORS[0] localhost: *\nnobody * $MAJORS[-1] new1 *\n* * $MAJORS[-1] localhost:a *"; +close F; +chmod 0644, '/etc/postgresql-common/user_clusters'; +like_program_out 'postgres', 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $MAJORS[0]/, + 'pg_wrapper selects correct version with per-user user_clusters'; +like_program_out 'nobody', 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $MAJORS[-1]/, + 'pg_wrapper selects correct version with per-user user_clusters'; +like_program_out 0, 'createdb --version', 0, qr/user_clusters.*line 3.*cluster.*not exist/i, + 'pg_wrapper warning for invalid per-user user_clusters line'; +# check PGHOST environment variable precedence +$ENV{'PGHOST'} = '127.0.0.2'; +like_program_out 'postgres', 'psql -Atl', 2, qr/127.0.0.2/, '$PGHOST overrides user_clusters'; +is_program_out 'postgres', "psql --cluster $MAJORS[-1]/localhost:5434 -Atc 'select current_database()' test", + 0, "test\n", '--cluster overrides $PGHOST'; +delete $ENV{'PGHOST'}; + +# check invalid user_clusters +open F, '>/etc/postgresql-common/user_clusters' or die "Could not create user_clusters: $!"; +print F 'foo'; +close F; +chmod 0644, '/etc/postgresql-common/user_clusters'; +like_program_out 'postgres', 'createdb --version', 0, qr/ignoring invalid line 1/, + 'pg_wrapper ignores invalid lines in user_clusters'; + +# remove test user_clusters +unlink '/etc/postgresql-common/user_clusters' or die + "unlink user_clusters: $!"; + +# check that pg_service.conf works +open F, '>/etc/postgresql-common/pg_service.conf' or die "Could not create pg_service.conf: $!"; +print F "[old_t1] +user=postgres +dbname=template1 +port=5432 + +[new1_test] +user=postgres +dbname=test +port=5434 + +# these do not exist +[new2_test] +user=postgres +dbname=test +port=5440 +"; +close F; +chmod 0644, '/etc/postgresql-common/pg_service.conf'; + +$ENV{'PGSERVICE'} = 'old_t1'; +is_program_out 'postgres', "psql -Atc 'select current_database()'", 0, + "template1\n", 'pg_service conf selection 1'; +$ENV{'PGSERVICE'} = 'new1_test'; +is_program_out 'postgres', "psql -Atc 'select current_database()'", 0, + "test\n", 'pg_service conf selection 2'; +$ENV{'PGSERVICE'} = 'new2_test'; +like_program_out 'postgres', "psql -Atc 'select current_database()'", 2, + qr/FATAL.*test/, 'pg_service conf selection 3'; +delete $ENV{'PGSERVICE'}; +unlink '/etc/postgresql-common/pg_service.conf'; + +# check proper error message if no cluster could be determined as default for +# pg_wrapper +is ((system "pg_ctlcluster $MAJORS[0] old stop >/dev/null"), 0, "stopping cluster $old"); +PgCommon::set_conf_value $MAJORS[0], 'old', 'postgresql.conf', 'port', '5435'; +is ((system "pg_ctlcluster $MAJORS[0] old start >/dev/null"), 0, "restarting cluster $old"); +like_program_out 'postgres', 'pg_lsclusters -h | sort -k3', 0, qr/.*5434.*5435.*5440.*/s, + 'port of first cluster was successfully changed'; +like_program_out 'postgres', "psql -l", 2, + qr/no.*default.*man pg_wrapper.*psql:.*\.s\.PGSQL.5432/is, + 'proper pg_wrapper warning and psql error if no cluster is suitable as default target'; +like_program_out 'postgres', "psql -Atl --cluster $new1", 0, + qr/test\|postgres\|/, + '--cluster selects appropriate cluster'; +like_program_out 'postgres', "psql -Atl -p 5434", 0, + qr/test\|postgres\|/, + '-p selects appropriate cluster'; +like_program_out 'postgres', "psql -Atlp 5434", 0, + qr/test\|postgres\|/, + '-Atlp selects appropriate cluster'; +like_program_out 'postgres', "psql -Atl --port 5434", 0, + qr/test\|postgres\|/, + '--port selects appropriate cluster'; +like_program_out 'postgres', "env PGPORT=5434 psql -Atl", 0, + qr/test\|postgres\|/, + '$PGPORT selects appropriate cluster'; + +# but specifying -p explicitly should work + +# restore original user_clusters +if (-f '/etc/postgresql-common/user_clusters.psqltestsuite') { + ok ((rename '/etc/postgresql-common/user_clusters.psqltestsuite', + '/etc/postgresql-common/user_clusters'), + 'Restoring original /etc/postgresql-common/user_clusters'); +} else { + pass '/etc/postgresql-common/user_clusters did not exist, not restoring'; +} + +# clean up +is ((system "pg_dropcluster $MAJORS[-1] new1 --stop"), 0, "dropping $new1"); +is ((system "pg_dropcluster $MAJORS[-1] new2 --stop"), 0, "dropping $new2"); +is ((system "pg_dropcluster $MAJORS[0] old --stop"), 0, "dropping $old"); + +check_clean; + +# vim: filetype=perl diff --git a/t/110_integrate_cluster.t b/t/110_integrate_cluster.t new file mode 100644 index 0000000..4201a58 --- /dev/null +++ b/t/110_integrate_cluster.t @@ -0,0 +1,44 @@ +# Check integration of an already existing cluster + +use strict; + +use lib 't'; +use TestLib; +use File::Temp qw/tempdir/; +use Time::HiRes qw(usleep); + +my $version = $MAJORS[-1]; + +use Test::More tests => 32; +use PgCommon; + +delete $ENV{'LANG'}; +delete $ENV{'LANGUAGE'}; +$ENV{'LC_ALL'} = 'C'; + +my $wdir = tempdir (CLEANUP => 1); +chmod 0755, $wdir or die "Could not chmod $wdir: $!"; + +# create clusters for different owners and check their integration +for my $o ('postgres', 'nobody') { + my $cdir = "$wdir/c"; + mkdir $cdir; + my $oid = getpwnam $o; + chown $oid, 0, $cdir or die "Could not chown $cdir to $oid: $!"; + like_program_out $o, "$PgCommon::binroot$version/bin/initdb $cdir/$o", + 0, qr/Success/, "creating raw initdb cluster for user $o"; + like_program_out 0, "pg_createcluster $version $o -d $cdir/$o", 0, + qr/Configuring already existing cluster/i, "integrating $o cluster"; + like_program_out 0, "pg_lsclusters", 0, + qr/$version\s+$o\s+5432\s+down\s+$o\s/, 'correct pg_lsclusters output'; + is_program_out $o, "pg_ctlcluster $version $o start", 0, '', "starting cluster $o"; + like_program_out 0, "pg_lsclusters", 0, + qr/$version\s+$o\s+5432\s+online\s+$o\s/, 'correct pg_lsclusters output'; + is ((system "pg_dropcluster $version $o --stop"), 0, "dropping cluster $o"); + ok_dir $cdir, [], 'No files in temporary cluster dir left behind'; + rmdir $cdir; +} + +check_clean; + +# vim: filetype=perl diff --git a/t/120_pg_upgradecluster_scripts.t b/t/120_pg_upgradecluster_scripts.t new file mode 100644 index 0000000..d31e52c --- /dev/null +++ b/t/120_pg_upgradecluster_scripts.t @@ -0,0 +1,114 @@ +# Check /etc/p-c/pg_upgradecluster.d/ scripts and proper handling of already +# existing tables in the target cluster. + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More tests => (@MAJORS == 1) ? 1 : 31; + +if (@MAJORS == 1) { + pass 'only one major version installed, skipping upgrade tests'; + exit 0; +} + + +# create old cluster +is ((system "pg_createcluster $MAJORS[0] main --start >/dev/null"), 0, "pg_createcluster $MAJORS[0] main"); + +# add data table, auxtable with 'old...' values, and an unrelated auxtable in +# another schema +is_program_out 'postgres', + 'psql template1 -qc "create table auxdata (x varchar(10)); insert into auxdata values (\'old1\'); insert into auxdata values (\'old2\')"', + 0, '', 'adding auxdata to template1 and fill in some "old..." values'; +is_program_out 'postgres', "createdb test", 0, ''; +is_program_out 'postgres', 'psql test -qc "create table userdata(x int); insert into userdata values(42); insert into userdata values(256)"', + 0, '', 'creating userdata table'; +is_program_out 'postgres', + 'psql test -qc "create schema s; create table s.auxdata (x varchar(10)); insert into s.auxdata values (\'schema1\')"', + 0, '', 'adding schema s and s.auxdata to test and fill in some values'; + +if (not -d '/etc/postgresql-common/pg_upgradecluster.d') { + mkdir '/etc/postgresql-common/pg_upgradecluster.d' or die "mkdir: $!"; +} + +# move existing files away +for my $f (glob("/etc/postgresql-common/pg_upgradecluster.d/*")) { + next if ($f =~ /\.disabled$/); + rename $f, "$f.disabled"; +} + +# create test scripts +chmod 0755, '/etc/postgresql-common/pg_upgradecluster.d' or die "chmod: $!"; +open F, '>/etc/postgresql-common/pg_upgradecluster.d/auxdata' or die "open: $!"; +print F <<EOS; +#!/bin/sh -e +# Arguments: <old version> <cluster name> <new version> <phase> +oldver=\$1 +cluster=\$2 +newver=\$3 +phase=\$4 + +if [ \$phase = init ]; then + createdb --cluster \$newver/\$cluster idb +fi + +if [ \$phase = finish ]; then + psql --cluster \$newver/\$cluster template1 <<EOF +drop table if exists auxdata; +create table auxdata (x varchar(10)); +insert into auxdata values ('new1'); +insert into auxdata values ('new2'); +EOF +fi + +EOS +close F; +chmod 0755, '/etc/postgresql-common/pg_upgradecluster.d/auxdata' or die "chmod: $!"; + +open F, '>/etc/postgresql-common/pg_upgradecluster.d/badscript' or die "open: $!"; +print F <<EOS; +#!/bin/false +EOS +close F; +chmod 0755, '/etc/postgresql-common/pg_upgradecluster.d/badscript' or die "chmod: $!"; + +# upgrade cluster +my $outref; +is ((exec_as 0, "pg_upgradecluster -v $MAJORS[-1] $MAJORS[0] main", $outref, 1), 1, 'pg_upgradecluster fails with bad script'); +like $$outref, qr/error|fail/i, 'server error messages during upgrade'; +unlink '/etc/postgresql-common/pg_upgradecluster.d/badscript'; + +is ((exec_as 0, "pg_upgradecluster -v $MAJORS[-1] $MAJORS[0] main", $outref, 0), 0, 'pg_upgradecluster succeeds'); +unlike $$outref, qr/error|fail/i, 'no server error messages during upgrade'; +like $$outref, qr/Starting upgraded cluster/, 'pg_upgradecluster reported cluster startup'; +like $$outref, qr/Success. Please check/, 'pg_upgradecluster reported successful operation'; + +is ((system "pg_dropcluster $MAJORS[0] main --stop"), 0, 'Dropping old cluster'); + +# check new version cluster +is_program_out 'postgres', 'psql template1 -Atc "select * from auxdata order by x"', 0, + "new1\nnew2\n", 'new cluster\'s template1/auxdata table is the script\'s version'; + +like_program_out 'postgres', 'psql -Atl', 0, qr/^idb\b.*^test\b/ms, + 'upgraded cluster has idb and test databases'; + +is_program_out 'postgres', 'psql test -Atc "select * from s.auxdata"', 0, + "schema1\n", 'new cluster\'s test/auxdata table in schema s was upgraded normally'; + +# remove test script +unlink '/etc/postgresql-common/pg_upgradecluster.d/auxdata' or die "unlink: $!"; + +# restore original contents +for my $f (glob("/etc/postgresql-common/pg_upgradecluster.d/*.disabled")) { + my $f2 = $f; + $f2 =~ s/\.disabled$//; + rename $f, $f2; +} + +# clean up +is ((system "pg_dropcluster $MAJORS[-1] main --stop"), 0, "pg_dropcluster $MAJORS[-1] main"); +check_clean; + +# vim: filetype=perl diff --git a/t/130_nonroot_admin.t b/t/130_nonroot_admin.t new file mode 100644 index 0000000..5375b73 --- /dev/null +++ b/t/130_nonroot_admin.t @@ -0,0 +1,50 @@ +# Check that cluster administration works as non-root if the invoker has +# sufficient permissions on directories. + +use strict; + +use lib 't'; +use TestLib; + +my $version = $MAJORS[-1]; +my $oldversion = $MAJORS[0]; + +use Test::More tests => 22; +use PgCommon; + +my $testuser = 'postgres'; + +# pg_createcluster and pg_ctlcluster +is ((exec_as $testuser, "pg_createcluster $version main --start"), 0, + "pg_createcluster succeeds as user $testuser with appropriate owner permissions"); + +like_program_out $testuser, 'pg_lsclusters -h', 0, qr/^$version\s+main.*online/m; +like_program_out 'postgres', 'psql -Atl', 0, qr/template1.*UTF8/; + +# pg_dropcluster +is ((exec_as $testuser, "pg_dropcluster $version main --stop"), 0, + "pg_dropcluster succeeds as user $testuser with appropriate directory owner permissions"); + +# pg_upgradecluster +SKIP: { + skip 'Only one major version installed, skipping pg_upgradecluster tests', 8 if ($oldversion eq $version); + + is ((exec_as $testuser, "pg_createcluster $oldversion main --start"), 0, + "pg_createcluster succeeds as user $testuser with appropriate group permissions"); + my $outref; + is ((exec_as $testuser, "pg_upgradecluster -v $version $oldversion main", $outref, 0), 0, + "pg_upgradecluster succeeds as user $testuser"); + like $$outref, qr/Starting upgraded cluster/, 'pg_upgradecluster reported cluster startup'; + like $$outref, qr/Success. Please check/, 'pg_upgradecluster reported successful operation'; + + like_program_out $testuser, 'pg_lsclusters -h', 0, + qr/^$oldversion\s+main.*down.*\n^$version\s+main.*online/m; + + # clean up + is ((exec_as $testuser, "pg_dropcluster $oldversion main"), 0); + is ((exec_as $testuser, "pg_dropcluster $version main --stop"), 0); +} + +check_clean; + +# vim: filetype=perl diff --git a/t/135_pg_virtualenv.t b/t/135_pg_virtualenv.t new file mode 100644 index 0000000..1662e5b --- /dev/null +++ b/t/135_pg_virtualenv.t @@ -0,0 +1,35 @@ +# check if pg_virtualenv runs ok, even under fakeroot + +use strict; +use warnings; + +use lib 't'; +use TestLib; + +use Test::More tests => 12 * @MAJORS + 8; + +foreach my $v (@MAJORS) { + my $args = 'sh -c \'echo "id|$(id -un)"; psql -AtXxc "SELECT current_user"\''; + my $virtualenv = "pg_virtualenv -v $v $args"; + + $ENV{USER} = 'root'; + like_program_out 'root', $virtualenv, 0, qr!id.root\ncurrent_user.postgres!, "running pg_virtualenv as root"; + $ENV{USER} = 'postgres'; + like_program_out 'postgres', $virtualenv, 0, qr!id.postgres\ncurrent_user.postgres!, "running pg_virtualenv as postgres"; + $ENV{USER} = 'nobody'; + like_program_out 'nobody', $virtualenv, 0, qr!id.nobody\ncurrent_user.nobody!, "running pg_virtualenv as nobody"; + + SKIP: { + skip "/usr/bin/fakeroot not available", 6 unless (-x "/usr/bin/fakeroot"); # CentOS doesn't have fakeroot + $ENV{USER} = 'root'; + like_program_out 'root', "fakeroot $virtualenv", 0, qr!id.root\ncurrent_user.postgres!, "running fakeroot pg_virtualenv as root"; + $ENV{USER} = 'postgres'; + like_program_out 'postgres', "fakeroot $virtualenv", 0, qr!id.root\ncurrent_user.postgres!, "running fakeroot pg_virtualenv as postgres"; + $ENV{USER} = 'nobody'; + like_program_out 'nobody', "fakeroot $virtualenv", 0, qr!id.root\ncurrent_user.nobody!, "running fakeroot pg_virtualenv as nobody"; + } +} + +check_clean; + +# vim: filetype=perl diff --git a/t/140_pg_config.t b/t/140_pg_config.t new file mode 100644 index 0000000..770c75c --- /dev/null +++ b/t/140_pg_config.t @@ -0,0 +1,89 @@ +# Check pg_config output + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More tests => 14 * @MAJORS + ($PgCommon::rpm ? 1 : 2) * 12; + +my $multiarch = ''; +unless ($PgCommon::rpm) { + $multiarch = `dpkg-architecture -qDEB_HOST_MULTIARCH 2>/dev/null`; + chomp $multiarch; +} +note "Multiarch is " . ($multiarch ? 'enabled' : 'disabled'); + +my $version; +foreach $version (@MAJORS) { + note "checking version specific output for $version"; + if ($version < '8.2') { + pass "Skipping known-broken pg_config check for version $version"; + for (my $i = 0; $i < 13; ++$i) { pass '...'; } + next; + } + is_program_out 'postgres', "$PgCommon::binroot$version/bin/pg_config --pgxs", 0, + "$PgCommon::binroot$version/lib/pgxs/src/makefiles/pgxs.mk\n"; + my $libdir = "/usr/lib" . ($version >= 9.3 and $multiarch ? "/$multiarch" : "") . "\n"; + $libdir = "$PgCommon::binroot$version/lib\n" if ($PgCommon::rpm); + is_program_out 'postgres', "$PgCommon::binroot$version/bin/pg_config --libdir", 0, + $libdir; + is_program_out 'postgres', "$PgCommon::binroot$version/bin/pg_config --pkglibdir", 0, + "$PgCommon::binroot$version/lib\n"; + is_program_out 'postgres', "$PgCommon::binroot$version/bin/pg_config --bindir", 0, + "$PgCommon::binroot$version/bin\n"; + # mkdir should be in /bin on Debian. If /bin was linked to /usr/bin at build time, usrmerge was installed + SKIP: { + skip 'MKDIR_P not present before 9.0', 2 if ($version < 9.0); + skip 'MKDIR_P not checked on RedHat', 2 if ($PgCommon::rpm); # varies across builds/versions + is_program_out 'postgres', "grep ^MKDIR_P $PgCommon::binroot$version/lib/pgxs/src/Makefile.global", 0, + "MKDIR_P = /bin/mkdir -p\n"; + } + SKIP: { + skip 'build path not canonicalized on RedHat', 4 if ($PgCommon::rpm); + my $pkgversion = `dpkg-query -f '\${Version}' -W postgresql-server-dev-$version`; + # check that we correctly canonicalized the build paths + SKIP: { + skip 'abs_top_builddir introduced in 9.5', 2 if ($version < 9.5); + skip 'abs_top_builddir not patched in Debian (old)stable', 2 if ($version < 10 and $pkgversion !~ /pgdg/); + is_program_out 'postgres', "grep ^abs_top_builddir $PgCommon::binroot$version/lib/pgxs/src/Makefile.global", 0, + "abs_top_builddir = /build/postgresql-$version$ENV{PG_FLAVOR}/build\n"; + } + SKIP: { + skip 'abs_top_srcdir not patched before 9.3', 2 if ($version < 9.3); + skip 'abs_top_srcdir not patched in Debian (old)stable', 2 if ($version < 10 and $pkgversion !~ /pgdg/); + is_program_out 'postgres', "grep ^abs_top_srcdir $PgCommon::binroot$version/lib/pgxs/src/Makefile.global", 0, + "abs_top_srcdir = /build/postgresql-$version$ENV{PG_FLAVOR}/build/..\n"; + } + } +} + +my @pg_configs = $PgCommon::rpm ? qw(pg_config) : qw(pg_config pg_config.libpq-dev); +for my $pg_config (@pg_configs) { + if ($pg_config eq 'pg_config' or $PgCommon::rpm) { # pg_config should point at newest installed postgresql-server-dev-$version + $version = $ALL_MAJORS[-1]; + } else { # pg_config.libpq-dev should point at postgresql-server-dev-$(version of libpq-dev) + my $libpqdev_version = `dpkg-query --showformat '\${Version}' --show libpq-dev`; + $libpqdev_version =~ /^([89].\d|1.)/ or die "could not determine libpq-dev version"; + $version = $1; + } + note "checking $pg_config output (should behave like version $version)"; + + SKIP: { + my $pgc = "$PgCommon::binroot$version/bin/pg_config"; + skip "$pgc not installed, can't check full $pg_config output", 2 unless (-x $pgc); + my $full_output = `$pgc`; + is_program_out 'postgres', "$pg_config", 0, $full_output; + } + like_program_out 'postgres', "$pg_config --help", 0, qr/--includedir-server/; + is_program_out 'postgres', "$pg_config --pgxs", 0, + "$PgCommon::binroot$version/lib/pgxs/src/makefiles/pgxs.mk\n"; + my $libdir = "/usr/lib" . ($version >= 9.3 and $multiarch ? "/$multiarch" : "") . "\n"; + $libdir = "$PgCommon::binroot$version/lib\n" if ($PgCommon::rpm); + is_program_out 'postgres', "$pg_config --libdir", 0, + $libdir; + is_program_out 'postgres', "$pg_config --pkglibdir", 0, + "$PgCommon::binroot$version/lib\n"; + is_program_out 'postgres', "$pg_config --bindir", 0, + "$PgCommon::binroot$version/bin\n"; +} diff --git a/t/150_tsearch_stemming.t b/t/150_tsearch_stemming.t new file mode 100644 index 0000000..a2a6253 --- /dev/null +++ b/t/150_tsearch_stemming.t @@ -0,0 +1,108 @@ +# Check tsearch, and stemming with dynamic creation of .affix/.dict files + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; + +my $version = $MAJORS[-1]; + +use Test::More tests => ($MAJORS[-1] < 8.3 or $PgCommon::rpm) ? 1 : 37; +if ($version < 8.3) { + pass 'tsearch dictionaries not tested before 8.3'; + exit; +} +if ($PgCommon::rpm) { + pass 'tsearch dictionaries not handled by postgresql-common on RedHat'; + exit; +} + +# test pg_updatedicts +unlink '/var/cache/postgresql/dicts/en_us.affix'; +unlink '/var/cache/postgresql/dicts/en_us.dict'; +unlink "/usr/share/postgresql/$version/tsearch_data/en_us.affix"; +unlink "/usr/share/postgresql/$version/tsearch_data/en_us.dict"; +is ((exec_as 0, 'pg_updatedicts'), 0, 'pg_updatedicts succeeded'); +ok -f '/var/cache/postgresql/dicts/en_us.affix', + 'pg_updatedicts created en_us.affix'; +ok -f '/var/cache/postgresql/dicts/en_us.dict', + 'pg_updatedicts created en_us.dict'; +ok -l "/usr/share/postgresql/$version/tsearch_data/en_us.affix", + "pg_updatedicts created $version en_us.affix symlink"; +ok -l "/usr/share/postgresql/$version/tsearch_data/en_us.dict", + "pg_updatedicts created $version en_us.dict symlink"; + +# create cluster +is ((system "pg_createcluster $version main --start >/dev/null"), 0, "pg_createcluster $version main"); + +# create DB with en_US text search configuration +is_program_out 'postgres', 'createdb fts', 0, ''; + +my $outref; + +is ((exec_as 'postgres', 'psql -qd fts -c " + CREATE TEXT SEARCH CONFIGURATION public.sc_english ( COPY = pg_catalog.english ); + CREATE TEXT SEARCH DICTIONARY english_ispell (TEMPLATE = ispell, DictFile = en_US, + AffFile = en_US, StopWords = english); + SET default_text_search_config = \'public.sc_english\'; + ALTER TEXT SEARCH CONFIGURATION public.sc_english + ALTER MAPPING FOR asciiword WITH english_ispell, english_stem;"', $outref), + 0, 'creating en_US full text search configuration ' . $$outref); + +# create test table and index +my $outref; +is ((exec_as 'postgres', 'psql -qd fts -c " + CREATE TABLE stuff (id SERIAL PRIMARY KEY, text TEXT, textsearch tsvector); + UPDATE stuff SET textsearch = to_tsvector(\'public.sc_english\', coalesce(text, \'\')); + CREATE INDEX textsearch_idx ON stuff USING gin(textsearch); + CREATE TRIGGER textsearch_update_trigger BEFORE INSERT OR UPDATE + ON stuff FOR EACH ROW EXECUTE PROCEDURE + tsvector_update_trigger(textsearch, \'public.sc_english\', text); + INSERT INTO stuff (text) VALUES (\'PostgreSQL rocks\'); + INSERT INTO stuff (text) VALUES (\'Linux rocks\'); + INSERT INTO stuff (text) VALUES (\'I am your father\'\'s nephew\'\'s former roommate\'); + INSERT INTO stuff (text) VALUES (\'3 cafés\'); + "'), 0, 'creating data table and search index'); + +# test stemming +is_program_out 'postgres', + 'psql -Atd fts -c "SELECT dictionary, lexemes FROM ts_debug(\'public.sc_english\', \'friendliest\')"', + 0, "english_ispell|{friendly}\n", 'stem search of correct word'; +is_program_out 'postgres', + 'psql -Atd fts -c "SELECT dictionary, lexemes FROM ts_debug(\'public.sc_english\', \'father\'\'s\')"', + 0, "english_ispell|{father}\n|\nenglish_ispell|{}\n", 'stem search of correct word'; +is_program_out 'postgres', + 'psql -Atd fts -c "SELECT dictionary, lexemes FROM ts_debug(\'public.sc_english\', \'duffles\')"', + 0, "english_stem|{duffl}\n", 'stem search of unknown word'; + +# test searching +is_program_out 'postgres', + 'psql -Atd fts -c "SELECT text FROM stuff, to_tsquery(\'rocks\') query WHERE query @@ to_tsvector(text)"', + 0, "PostgreSQL rocks\nLinux rocks\n", 'full text search, exact word'; + +is_program_out 'postgres', + 'psql -Atd fts -c "SELECT text FROM stuff, to_tsquery(\'rock\') query WHERE query @@ to_tsvector(text)"', + 0, "PostgreSQL rocks\nLinux rocks\n", 'full text search for word stem'; + +is_program_out 'postgres', + 'psql -Atd fts -c "SELECT text FROM stuff, to_tsquery(\'roc\') query WHERE query @@ to_tsvector(text)"', + 0, '', 'full text search for word substring fails'; + +is_program_out 'postgres', + 'psql -Atd fts -c "SELECT text FROM stuff, to_tsquery(\'cafés\') query WHERE query @@ to_tsvector(text)"', + 0, "3 cafés\n", 'full text search, exact unicode word'; + +is_program_out 'postgres', + 'psql -Atd fts -c "SELECT text FROM stuff, to_tsquery(\'café\') query WHERE query @@ to_tsvector(text)"', + 0, "3 cafés\n", 'full text search for unicode word stem'; + +is_program_out 'postgres', + 'psql -Atd fts -c "SELECT text FROM stuff, to_tsquery(\'afé\') query WHERE query @@ to_tsvector(text)"', + 0, '', 'full text search for unicode word substring fails'; + +# clean up +is ((system "pg_dropcluster $version main --stop"), 0); +check_clean; + +# vim: filetype=perl diff --git a/t/160_alternate_confroot.t b/t/160_alternate_confroot.t new file mode 100644 index 0000000..e2a9604 --- /dev/null +++ b/t/160_alternate_confroot.t @@ -0,0 +1,57 @@ +# Check that we can do all operations using a per-user $PG_CLUSTER_CONF_ROOT + +use strict; + +use lib 't'; +use TestLib; + +my $version = $MAJORS[0]; + +use Test::More tests => 28; + +# prepare nobody-owned root dir for $PG_CLUSTER_CONF_ROOT +my $rootdir=`su -s /bin/sh -c 'mktemp -d' nobody`; +chomp $rootdir; +($rootdir) = $rootdir =~ m!^([a-zA-Z0-9._/]+)$!; # untaint +$ENV{'PG_CLUSTER_CONF_ROOT'} = "$rootdir/etc"; + +is ((exec_as 'nobody', "pg_createcluster $version test -d $rootdir/data/test -l $rootdir/test.log --start"), 0); + +is_program_out 'nobody', 'env -u PG_CLUSTER_CONF_ROOT pg_lsclusters -h', 0, ''; +like_program_out 'nobody', "pg_lsclusters -h", 0, + qr!^$version\s+test.*online\s+nobody\s+$rootdir/data/test\s+$rootdir/test.log$!; + +like_program_out 'nobody', "psql -Atl", 0, qr/template1.*UTF8/; + +# pg_upgradecluster +if ($MAJORS[0] ne $MAJORS[-1]) { + my $outref; + is ((exec_as 'nobody', "pg_upgradecluster --logfile $rootdir/testupgr.log -v $MAJORS[-1] $version test $rootdir/data/testupgr", $outref, 0), 0); + like $$outref, qr/Starting upgraded cluster/, 'pg_upgradecluster reported cluster startup'; + like $$outref, qr/Success. Please check/, 'pg_upgradecluster reported successful operation'; + + like_program_out 'nobody', 'pg_lsclusters -h', 0, + qr!^$version\s+test.*down.*\n^$MAJORS[-1]\s+test.*online\s+nobody\s+$rootdir/data/testupgr\s+$rootdir/testupgr.log$!m; + + # clean up + is_program_out 'nobody', "pg_dropcluster $version test", 0, ''; + is_program_out 'nobody', "pg_dropcluster $MAJORS[-1] test --stop", 0, ''; +} else { + pass 'Only one major version installed, skipping pg_upgradecluster tests'; + for (my $i = 0; $i < 6; ++$i) { pass '...'; } + + is_program_out 'nobody', "pg_dropcluster $version test --stop", 0, ''; +} + +# pg_dropcluster +is_program_out 'nobody', "pg_lsclusters -h", 0, ''; + +ok_dir "$rootdir/data", [], 'No files in root/data left behind'; +ok_dir "$rootdir", ['etc', 'data'], 'No cruft in root dir left behind'; + +system "rm -rf $rootdir"; + +delete $ENV{'PG_CLUSTER_CONF_ROOT'}; +check_clean; + +# vim: filetype=perl diff --git a/t/170_extensions.t b/t/170_extensions.t new file mode 100644 index 0000000..1abb330 --- /dev/null +++ b/t/170_extensions.t @@ -0,0 +1,87 @@ +# Check that all extensions install successfully. + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More 0.87; # needs libtest-simple-perl backport on lenny + +foreach my $v (@MAJORS) { +note "Running tests for $v"; + +if ($v < '9.1') { + pass 'No extensions for version < 9.1'; + next; +} + +# create cluster +is ((system "pg_createcluster $v main --start >/dev/null"), 0, "pg_createcluster $v main"); + +# plpgsql is installed by default +is_program_out 'postgres', "psql -Atc 'SELECT extname FROM pg_extension'", 0, "plpgsql\n"; + +my %depends = ( + bool_plperl => [qw(plperl)], + bool_plperlu => [qw(plperlu)], + earthdistance => [qw(cube)], + hstore_plperl => [qw(hstore plperl)], + hstore_plperlu => [qw(hstore plperlu)], + hstore_plpython2u => [qw(hstore plpython2u)], + hstore_plpython3u => [qw(hstore plpython3u)], + hstore_plpythonu => [qw(hstore plpythonu)], + jsonb_plperl => [qw(plperl)], # PG 11 + jsonb_plperlu => [qw(plperlu)], # PG 11 + jsonb_plpython2u => [qw(plpython2u)], # PG 11 + jsonb_plpython3u => [qw(plpython3u)], # PG 11 + jsonb_plpythonu => [qw(plpythonu)], # PG 11 + ltree_plpython2u => [qw(ltree plpython2u)], + ltree_plpython3u => [qw(ltree plpython3u)], + ltree_plpythonu => [qw(ltree plpythonu)], +); + +foreach (</usr/share/postgresql/$v/extension/*.control>) { + my ($extname) = $_ =~ /^.*\/(.*)\.control$/; + next if ($extname eq 'plpgsql'); + + if ($depends{$extname}) { + for my $dep (@{$depends{$extname}}) { + is_program_out 'postgres', "psql -qc 'CREATE EXTENSION $dep'", 0, '', + "$extname dependency $dep installs without error"; + } + } + + if ($extname eq 'hstore' && $v eq '9.1') { + # EXFAIL: hstore in 9.1 throws a warning about obsolete => operator + like_program_out 'postgres', "psql -qc 'CREATE EXTENSION \"$extname\"'", 0, + qr/=>/, "extension $extname installs (with warning)"; + } elsif ($extname eq 'chkpass' && $v >= '9.5') { + # chkpass is slightly broken, see + # http://www.postgresql.org/message-id/20141117162116.GA3565@msg.df7cb.de + like_program_out 'postgres', "psql -qc 'CREATE EXTENSION \"$extname\"'", 0, + qr/WARNING: type input function chkpass_in should not be volatile/, + "extension $extname installs (with warning)"; + } else { + is_program_out 'postgres', "psql -qc 'CREATE EXTENSION \"$extname\"'", 0, '', + "extension $extname installs without error"; + } + + is_program_out 'postgres', "psql -qc 'DROP EXTENSION \"$extname\"'", 0, '', + "extension $extname removes without error"; + + if ($depends{$extname}) { + for my $dep (@{$depends{$extname}}) { + is_program_out 'postgres', "psql -qc 'DROP EXTENSION $dep'", 0, '', + "$extname dependency extension $dep removes without error"; + } + } +} + +# clean up +is ((system "pg_dropcluster $v main --stop"), 0, "pg_dropcluster $v main"); +check_clean; +} + +done_testing(); + +# vim: filetype=perl diff --git a/t/180_ecpg.t b/t/180_ecpg.t new file mode 100644 index 0000000..5d3f7a8 --- /dev/null +++ b/t/180_ecpg.t @@ -0,0 +1,56 @@ +# Check that ecpg works + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More tests => 14; + +my $v = $MAJORS[-1]; + +# prepare nobody-owned work dir +my $workdir=`su -s /bin/sh -c 'mktemp -d' nobody`; +chomp $workdir; +chdir $workdir or die "could not chdir to $workdir: $!"; + +# create test code +open F, '>test.pgc' or die "Could not open $workdir/test.pgc: $!"; +print F <<EOF; +#include <stdio.h> +#include <stdlib.h> + +EXEC SQL WHENEVER SQLWARNING SQLPRINT; +EXEC SQL WHENEVER SQLERROR SQLPRINT; + +EXEC SQL BEGIN DECLARE SECTION; + char output[1024]; +EXEC SQL END DECLARE SECTION; + +int main() { + ECPGdebug(1, stderr); + EXEC SQL CONNECT TO template1; + EXEC SQL SELECT 'Database is ' || current_database() INTO :output; + puts(output); + EXEC SQL DISCONNECT ALL; + return 0; +} +EOF +close F; +chmod 0644, 'test.pgc'; + +is_program_out 'nobody', 'ecpg test.pgc', 0, '', 'ecpg processing'; + +is_program_out 'nobody', 'cc -I$(pg_config --includedir) -L$(pg_config --libdir) -o test test.c -lecpg', + 0, '', 'compiling ecpg output'; +chdir '/' or die "could not chdir to /: $!"; + +# run program +like_program_out 'nobody', "pg_virtualenv $workdir/test", 0, qr/Database is template1/, + 'test program runs and gives correct output'; + +# clean up +system "rm -rf $workdir"; +check_clean; + +# vim: filetype=perl diff --git a/t/190_pg_buildext.t b/t/190_pg_buildext.t new file mode 100644 index 0000000..9a50712 --- /dev/null +++ b/t/190_pg_buildext.t @@ -0,0 +1,104 @@ +# Check pg_buildext and that our debhelper integration works + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More; + +if ($PgCommon::rpm) { + pass 'No pg_buildext tests on RedHat'; + done_testing(); + exit; +} + +# when invoked from the postgresql-NN package tests, postgresql-server-dev-all is not installed +if (! -x '/usr/bin/dh_make_pgxs') { + pass "Skipping pg_buildext tests, /usr/bin/dh_make_pgxs is not installed"; + done_testing(); + exit; +} + +my $arch = `dpkg-architecture -qDEB_HOST_ARCH`; +chomp $arch; + +if ($ENV{PG_VERSIONS}) { + note "PG_VERSIONS=$ENV{PG_VERSIONS}"; + $ENV{PG_SUPPORTED_VERSIONS} = join ' ', (grep { $_ >= 9.1 } split /\s+/, $ENV{PG_VERSIONS}); + unless ($ENV{PG_SUPPORTED_VERSIONS}) { + ok 1, 'No versions with extension support to test'; + done_testing(); + exit; + } + note "PG_SUPPORTED_VERSIONS=$ENV{PG_SUPPORTED_VERSIONS}"; +} +my @versions = split /\s+/, `/usr/share/postgresql-common/supported-versions`; + +# prepare build environment +chmod 0777, 't/foo', 't/foo/foo-123', 't/bar/debian'; +umask 0022; +chdir 't/foo'; + +program_ok 0, 'make clean'; +program_ok 'nobody', 'make tar'; +program_ok 'nobody', 'cd foo-123 && echo y | EDITOR=true dh_make_pgxs'; + +note "testing 'dh --with pgxs'"; +program_ok 'nobody', 'cd foo-123 && DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -us -uc'; + +foreach my $ver (@versions) { + my $deb = "postgresql-$ver-foo_123-1_$arch.deb"; + ok (-f $deb, "$deb was built"); + SKIP: { + my $have_extension_destdir = `grep extension_destdir /usr/share/postgresql/$ver/postgresql.conf.sample`; + skip "No in-tree installcheck on PG $ver (missing extension_destdir)", 2 unless ($have_extension_destdir); + like_program_out 'nobody', "cd foo-123 && PG_SUPPORTED_VERSIONS=$ver dh_pgxs_test", + 0, qr/PostgreSQL $ver installcheck.*(test foo * \.\.\. ok|ok 1 * - foo)/s; # old/new PG 16 syntax + } + program_ok 0, "dpkg -i $deb"; + like_program_out 'nobody', "cd foo-123 && pg_buildext installcheck", + 0, qr/PostgreSQL $ver installcheck.*(test foo * \.\.\. ok|ok 1 * - foo)/s; + like_program_out 'nobody', "cd foo-123 && echo 'SELECT 3*41, version()' | pg_buildext psql", 0, qr/123.*PostgreSQL $ver/; + like_program_out 'nobody', "cd foo-123 && echo 'echo --\$PGVERSION--' | pg_buildext virtualenv", 0, qr/--$ver--/; + like_program_out 'nobody', "cd foo-123 && pg_buildext run echo --%v--", 0, qr/--$ver--/; + program_ok 0, "dpkg -r postgresql-$ver-foo"; +} + +note "testing 'dh --with pgxs_loop'"; +system "rm -f postgresql-*.deb"; + +program_ok 'nobody', 'sed -i -e s/pgxs/pgxs_loop/ foo-123/debian/rules'; +program_ok 'nobody', 'cd foo-123 && DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -us -uc'; + +foreach my $ver (@versions) { + my $deb = "postgresql-$ver-foo_123-1_$arch.deb"; + ok (-f $deb, "$deb was built"); +} + +program_ok 'nobody', 'make clean'; + +note "testing pg_buildext updatecontrol"; +chdir '../bar'; +program_ok 'nobody', 'PG_SUPPORTED_VERSIONS="0.9 1.0 1.1 2 3" pg_buildext updatecontrol'; +is `cat debian/control`, "Source: bar +Build-Depends: whatever, postgresql-1.0-moo (>= 1), postgresql-1.1-moo (>= 1), postgresql-2-moo (>= 1), more, + postgresql-2-new, + postgresql-1.0, postgresql-1.1, postgresql-2 + +Package: postgresql-1.0-bar +Architecture: some +Depends: postgresql-1.0-moo + +Package: postgresql-1.1-bar +Architecture: some +Depends: postgresql-1.1-moo + +Package: postgresql-2-bar +Architecture: some +Depends: postgresql-2-moo +", "PGVERSION and PGVERSIONS were correctly replaced"; + +done_testing(); + +# vim: filetype=perl diff --git a/t/200_maintscripts.t b/t/200_maintscripts.t new file mode 100644 index 0000000..3fc5018 --- /dev/null +++ b/t/200_maintscripts.t @@ -0,0 +1,46 @@ +# This test runs last since we are reinstalling postgresql-common, and we want +# to avoid spoiling the other tests with any version skew. + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More tests => $PgCommon::rpm ? 1 : 15; + +if ($PgCommon::rpm) { + pass 'No maintainer script tests on rpm'; + exit; +} + +my $v = $MAJORS[-1]; + +note -d "/run/systemd/system" ? "We are running systemd" : "We are not running systemd"; + +# create cluster +program_ok 0, "pg_createcluster $v main --start"; + +# get postmaster PID +my $postmaster_pid = `head -1 /var/lib/postgresql/$v/main/postmaster.pid`; +chomp $postmaster_pid; +ok $postmaster_pid > 0, "postmaster PID is $postmaster_pid"; + +# "upgrade" postgresql-common to check if postgresql.service is left alone +program_ok 0, 'apt-get update -q', 0, ''; +note `apt-cache policy postgresql-common`; +program_ok 0, 'apt-get install -y --reinstall -o DPkg::Options::=--force-confnew postgresql-common', 0, ''; +note `apt-cache policy postgresql-common`; + +# get postmaster PID again, compare +my $postmaster_pid2 = `head -1 /var/lib/postgresql/$v/main/postmaster.pid`; +chomp $postmaster_pid2; +ok $postmaster_pid2 > 0, "postmaster PID is $postmaster_pid2"; +is $postmaster_pid, $postmaster_pid2, "postmaster was not restarted"; + +# stop server, clean up, check for leftovers +program_ok 0, "pg_dropcluster $v main --stop"; + +check_clean; + +# vim: filetype=perl diff --git a/t/TestLib.pm b/t/TestLib.pm new file mode 100644 index 0000000..408c9f5 --- /dev/null +++ b/t/TestLib.pm @@ -0,0 +1,270 @@ +# Common functionality for postgresql-common self tests +# +# (C) 2005-2009 Martin Pitt <mpitt@debian.org> +# (C) 2013-2022 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +package TestLib; +use strict; +use Exporter; +use Test::More; +use PgCommon qw/get_versions change_ugid next_free_port/; + +our $VERSION = 1.00; +our @ISA = ('Exporter'); +our @EXPORT = qw/os_release ps ok_dir exec_as deb_installed rpm_installed package_version + version_ge program_ok is_program_out like_program_out unlike_program_out + pidof pid_env check_clean + @ALL_MAJORS @MAJORS $delay/; + +our @ALL_MAJORS = get_versions(); # not affected by PG_VERSIONS/-v +our @MAJORS = $ENV{PG_VERSIONS} ? split (/\s+/, $ENV{PG_VERSIONS}) : @ALL_MAJORS; +our $delay = 500_000; # 500ms + +# called if a test fails; spawn a shell if the environment variable +# FAILURE=shell is set +sub fail_debug { + if ($ENV{'FAILURE'} eq 'shell') { + if ((system 'bash') != 0) { + exit 1; + } + } +} + +# parse /etc/os-release and return (os, version number) +sub os_release { + open OS, "/etc/os-release" or return (undef, undef); + my ($os, $osversion); + while (<OS>) { + $os = $1 if /^ID=(.*)/; + $osversion = $1 if /^VERSION_ID="?([^"]*)/; + } + close OS; + $osversion = 'unstable' if ($os eq 'debian' and not defined $osversion); + return ($os, $osversion); +} + +# Return whether a given deb is installed. +# Arguments: <deb name> +sub deb_installed { + open (DPKG, "dpkg -s $_[0] 2>/dev/null|") or die "call dpkg: $!"; + my $result = 0; + while (<DPKG>) { + if (/^Status: install ok installed/) { + $result = 1; + last; + } + } + close DPKG; + + return $result; +} + +# Return whether a given rpm is installed. +# Arguments: <rpm name> +sub rpm_installed { + open (RPM, "rpm -qa $_[0] 2>/dev/null|") or die "call rpm: $!"; + my $out = <RPM>; # returns void or the package name + close RPM; + return ($out =~ /./); +} + +# Return a package version +# Arguments: <package> +sub package_version { + my $package = shift; + if ($PgCommon::rpm) { + return `rpm --queryformat '%{VERSION}' -q $package`; + } else { + my $version = `dpkg-query -f '\${Version}' --show $package`; + chomp $version; + return $version; + } +} + +# Return whether a version is greater or equal to another one +# Arguments: <ver1> <ver2> +sub version_ge { + my ($v1, $v2) = @_; + use IPC::Open2; + open2(\*CHLD_OUT, \*CHLD_IN, 'sort', '-Vr'); + print CHLD_IN "$v1\n"; + print CHLD_IN "$v2\n"; + close CHLD_IN; + my $v_ge = <CHLD_OUT>; + chomp $v_ge; + return $v_ge eq $v1; +} + +# Return the user, group, and command line of running processes for the given +# program. +sub ps { + return `ps h -o user,group,args -C $_[0] | grep '$_[0]' | sort -u`; +} + +# Return array of pids that match the given command name (we require a leading +# slash so the postgres children are filtered out) +sub pidof { + my $prg = shift; + open F, '-|', 'ps', 'h', '-C', $prg, '-o', 'pid,cmd' or die "open: $!"; + my @pids; + while (<F>) { + if ((index $_, "/$prg") >= 0) { + push @pids, (split)[0]; + } + } + close F; + return @pids; +} + +# Return an reference to an array of all entries but . and .. of the given directory. +sub dircontent { + my $dir = $_[0]; + opendir D, $dir or return ["opendir $dir: $!"]; + my @e = grep { $_ ne '.' && $_ ne '..' } readdir (D); + closedir D; + return [sort @e]; +} + +# Return environment of given PID +sub pid_env { + my ($user, $pid) = @_; + my $path = "/proc/$pid/environ"; + my @lines; + open E, "su -c 'cat $path' $user |" or warn "open $path: $!"; + { + local $/; + @lines = split '\0', <E>; + } + close E; + my %env; + foreach (@lines) { + my ($k, $v) = (split '='); + $env{$k} = $v; + } + return %env; +} + +# Check the contents of a directory. +# Arguments: <directory name> <ref to expected dir content> <test description> +sub ok_dir { + my $content = dircontent $_[0]; + if (eq_set $content, $_[1]) { + pass $_[2]; + } else { + diag "Expected directory contents: [@{$_[1]}], actual contents: [@$content]\n"; + fail $_[2]; + } +} + +# Execute a command as a different user and return the output. Prints the +# output of the command if exit code differs from expected one. +# Arguments: <user> <system command> <ref to output> [<expected exit code>] +# Returns: Program exit code +sub exec_as { + my $uid; + if ($_[0] =~ /\d+/) { + $uid = int($_[0]); + } else { + $uid = getpwnam $_[0]; + defined($uid) or die "TestLib::exec_as: target user '$_[0]' does not exist"; + } + change_ugid ($uid, (getpwuid $uid)[3]); + die "changing euid: $!" if $> != $uid; + my $out = `$_[1] 2>&1`; + my $result = $? >> 8; + $< = $> = 0; + $( = $) = 0; + die "changing euid back to root: $!" if $> != 0; + $_[2] = \$out; + + if (defined $_[3] && $_[3] != $result) { + print "command '$_[1]' did not exit with expected code $_[3] but with $result:\n"; + print $out; + fail_debug; + } + return $result; +} + +# Execute a command as a particular user, and check the exit code +# Arguments: <user> <command> [<expected exit code>] [<description>] +sub program_ok { + my ($user, $cmd, $exit, $description) = @_; + $exit ||= 0; + $description ||= $cmd; + my $outref; + ok ((exec_as $user, $cmd, \$outref, $exit) == $exit, $description); +} + +# Execute a command as a particular user, and check the exit code and output +# (merged stdout/stderr). +# Arguments: <user> <command> <expected exit code> <expected output> [<description>] +sub is_program_out { + my $outref; + my $result = exec_as $_[0], $_[1], $outref; + is $result, $_[2], $_[1] or fail_debug; + is ($$outref, $_[3], (defined $_[4] ? $_[4] : "correct output of $_[1]")) or fail_debug; +} + +# Execute a command as a particular user, and check the exit code and output +# against a regular expression (merged stdout/stderr). +# Arguments: <user> <command> <expected exit code> <expected output re> [<description>] +sub like_program_out { + my $outref; + my $result = exec_as $_[0], $_[1], $outref; + is $result, $_[2], $_[1] or fail_debug; + like ($$outref, $_[3], (defined $_[4] ? $_[4] : "correct output of $_[1]")) or fail_debug; +} + +# Execute a command as a particular user, check the exit code, and check that +# the output does not match a regular expression (merged stdout/stderr). +# Arguments: <user> <command> <expected exit code> <expected output re> [<description>] +sub unlike_program_out { + my $outref; + my $result = exec_as $_[0], $_[1], $outref; + is $result, $_[2], $_[1] or fail_debug; + unlike ($$outref, $_[3], (defined $_[4] ? $_[4] : "correct output of $_[1]")) or fail_debug; +} + +# Check that all PostgreSQL related directories are empty and no +# postgres processes are running. Should be called at the end +# of all tests. Does 8 tests. +sub check_clean { + note "Cleanup"; + is (`pg_lsclusters -h`, '', 'Cleanup: No clusters left behind'); + is ((ps 'postgres'), '', 'No postgres processes left behind'); + + my @check_dirs = ('/etc/postgresql', '/var/lib/postgresql', + '/var/run/postgresql'); + foreach (@check_dirs) { + if (-d) { + ok_dir $_, [], "No files in $_ left behind"; + } else { + pass "Directory $_ does not exist"; + } + } + # we always want /var/log/postgresql/ to exist, so that logrotate does not + # complain about missing directories + ok_dir '/var/log/postgresql', [], "No files in /var/log/postgresql left behind"; + + # prefer ss over netstat (until all debian/tests/control files in postgresql-* have been updated) + unless (-x '/bin/netstat' and not -x '/bin/ss') { + is `ss --no-header -tlp 'sport >= 5432 and sport <= 5439'`, '', + 'PostgreSQL TCP ports are closed'; + } else { + is `netstat -avptn 2>/dev/null | grep ":543[2-9]\\b.*LISTEN"`, '', + 'PostgreSQL TCP ports are closed'; + } + is next_free_port(), 5432, "Next free port is 5432"; +} + +1; diff --git a/t/bar/debian/control.in b/t/bar/debian/control.in new file mode 100644 index 0000000..86b8069 --- /dev/null +++ b/t/bar/debian/control.in @@ -0,0 +1,8 @@ +Source: bar +Build-Depends: whatever, postgresql-PGVERSIONS-moo (>= 1), more, + postgresql-PGVERSION-new, + postgresql-PGVERSIONS + +Package: postgresql-PGVERSION-bar +Architecture: some +Depends: postgresql-PGVERSION-moo diff --git a/t/bar/debian/pgversions b/t/bar/debian/pgversions new file mode 100644 index 0000000..41f0ed3 --- /dev/null +++ b/t/bar/debian/pgversions @@ -0,0 +1,3 @@ +1.0 +1.1 +2 diff --git a/t/foo/Makefile b/t/foo/Makefile new file mode 100644 index 0000000..75e95e1 --- /dev/null +++ b/t/foo/Makefile @@ -0,0 +1,12 @@ +TAR = foo_123.orig.tar.gz + +test: $(TAR) + cd foo-123 && echo y | EDITOR=true dh_make_pgxs + cd foo-123 && dpkg-buildpackage -us -uc + +tar $(TAR): + tar cfz $(TAR) foo-123/ + +clean: + rm -f *.* + rm -rf foo-123/build-* foo-123/debian foo-123/*.o* diff --git a/t/foo/foo-123/Makefile b/t/foo/foo-123/Makefile new file mode 100644 index 0000000..9f9a2f8 --- /dev/null +++ b/t/foo/foo-123/Makefile @@ -0,0 +1,10 @@ +MODULE_big = foo +OBJS = foo.o +EXTENSION = foo +DATA = foo--100.sql foo--100--123.sql foo--123.sql +REGRESS = foo upgrade + +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) + diff --git a/t/foo/foo-123/README.md b/t/foo/foo-123/README.md new file mode 100644 index 0000000..14235f9 --- /dev/null +++ b/t/foo/foo-123/README.md @@ -0,0 +1,5 @@ +The PostgreSQL foo extension +============================ + +This extension exists for testing postgresql-common's `dh_make_pgxs` and +`dh --with pgxs` mechanisms. diff --git a/t/foo/foo-123/expected/foo.out b/t/foo/foo-123/expected/foo.out new file mode 100644 index 0000000..62afef7 --- /dev/null +++ b/t/foo/foo-123/expected/foo.out @@ -0,0 +1,8 @@ +CREATE EXTENSION foo; +SELECT foo(); + foo +----- + bar +(1 row) + +DROP EXTENSION foo; diff --git a/t/foo/foo-123/expected/upgrade.out b/t/foo/foo-123/expected/upgrade.out new file mode 100644 index 0000000..98fc8e6 --- /dev/null +++ b/t/foo/foo-123/expected/upgrade.out @@ -0,0 +1,14 @@ +CREATE EXTENSION foo VERSION "100"; +SELECT foo(); + foo +--------- + old bar +(1 row) + +ALTER EXTENSION foo UPDATE TO "123"; +SELECT foo(); + foo +----- + bar +(1 row) + diff --git a/t/foo/foo-123/foo--100--123.sql b/t/foo/foo-123/foo--100--123.sql new file mode 100644 index 0000000..a6e7adc --- /dev/null +++ b/t/foo/foo-123/foo--100--123.sql @@ -0,0 +1,3 @@ +CREATE OR REPLACE FUNCTION foo () +RETURNS text LANGUAGE C +AS '$libdir/foo'; diff --git a/t/foo/foo-123/foo--100.sql b/t/foo/foo-123/foo--100.sql new file mode 100644 index 0000000..5ed8546 --- /dev/null +++ b/t/foo/foo-123/foo--100.sql @@ -0,0 +1,3 @@ +CREATE OR REPLACE FUNCTION foo () +RETURNS text LANGUAGE SQL +AS $$ SELECT 'old bar'::text $$; diff --git a/t/foo/foo-123/foo--123.sql b/t/foo/foo-123/foo--123.sql new file mode 100644 index 0000000..a6e7adc --- /dev/null +++ b/t/foo/foo-123/foo--123.sql @@ -0,0 +1,3 @@ +CREATE OR REPLACE FUNCTION foo () +RETURNS text LANGUAGE C +AS '$libdir/foo'; diff --git a/t/foo/foo-123/foo.c b/t/foo/foo-123/foo.c new file mode 100644 index 0000000..8677acc --- /dev/null +++ b/t/foo/foo-123/foo.c @@ -0,0 +1,13 @@ +#include "postgres.h" +#include "fmgr.h" +#include "utils/builtins.h" + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1 (foo); + +Datum +foo (PG_FUNCTION_ARGS) +{ + PG_RETURN_TEXT_P(cstring_to_text("bar")); +} diff --git a/t/foo/foo-123/foo.control b/t/foo/foo-123/foo.control new file mode 100644 index 0000000..81c95c8 --- /dev/null +++ b/t/foo/foo-123/foo.control @@ -0,0 +1,2 @@ +default_version = '123' +comment = 'The foo extension' diff --git a/t/foo/foo-123/sql/foo.sql b/t/foo/foo-123/sql/foo.sql new file mode 100644 index 0000000..155f63e --- /dev/null +++ b/t/foo/foo-123/sql/foo.sql @@ -0,0 +1,5 @@ +CREATE EXTENSION foo; + +SELECT foo(); + +DROP EXTENSION foo; diff --git a/t/foo/foo-123/sql/upgrade.sql b/t/foo/foo-123/sql/upgrade.sql new file mode 100644 index 0000000..b4427b4 --- /dev/null +++ b/t/foo/foo-123/sql/upgrade.sql @@ -0,0 +1,7 @@ +CREATE EXTENSION foo VERSION "100"; + +SELECT foo(); + +ALTER EXTENSION foo UPDATE TO "123"; + +SELECT foo(); diff --git a/t/template b/t/template new file mode 100644 index 0000000..fa1df59 --- /dev/null +++ b/t/template @@ -0,0 +1,26 @@ +# Check XXX + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More tests => 14; + +my @versions = ($MAJORS[-1]); + +# create clusters +foreach (@versions) { + is ((system "pg_createcluster $_ main --start >/dev/null"), 0, "pg_createcluster $_ main"); + is_program_out 'postgres', "createdb --cluster $_/main XXX", 0, ''; +} + +# XXX + +# clean up +foreach (@versions) { + is ((system "pg_dropcluster $_ main --stop"), 0, "pg_dropcluster $_ main"); +} +check_clean; + +# vim: filetype=perl diff --git a/testsuite b/testsuite new file mode 100755 index 0000000..d3ff77d --- /dev/null +++ b/testsuite @@ -0,0 +1,222 @@ +#!/bin/bash + +# Run integration tests (on the installed packages). +# +# (C) 2005-2012 Martin Pitt <mpitt@debian.org> +# (C) 2012-2023 Christoph Berg <myon@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +set -eu + +# default config +TESTSDIR="$(dirname $0)/t" +: ${PG_UMASKS="077"} # default umask(s) to try + +help () +{ + echo "postgresql-common testsuite" + echo "Syntax: $0 [options] [test ...]" + echo " -D drop all existing clusters (USE WITH CARE)" + echo " -f number run tests starting at this number" + echo " -F suffix flavor suffix on package names, e.g. 'ee'" + echo " -i install packages for versions specified by -v" + echo " -M don't mount tmpfses, run in the host system" + echo " -s start a shell in the testbed on failure" + echo " -u 'umask ...' umasks to run testsuite with [default: 077]" + echo " -v 'version ...' PostgreSQL versions to test [default: client versions installed]" + echo " -V debug output" + exit ${1:-0} +} + +# option parsing +while getopts "Dhf:F:iMsu:v:V" opt ; do + case $opt in + D) DROP_ALL_CLUSTERS=1 ;; + f) FROM="$OPTARG" ;; + F) export PG_FLAVOR="$OPTARG" ;; + i) INSTALL=1 ;; + h) help ;; + M) export NO_TMPFS=1 ;; + s) export FAILURE="shell" ;; + u) PG_UMASKS="$OPTARG" ;; + v) export PG_VERSIONS="$OPTARG" ;; # used in t/TestLib.pm + V) VERBOSE=1 ;; + *) help 1 ;; + esac +done + +if [ "$(id -u)" != 0 ]; then + echo "Error: this test suite needs to be run as root" >&2 + exit 1 +fi + +# install packages for versions specified by -v +# needs network for apt, so run before unshare +if [ "${INSTALL:-}" ] && [ -z "${UNSHARED:-}" ]; then + . /etc/os-release + case $ID in + debian|ubuntu) + for v in $PG_VERSIONS; do + case $v in 8.*|9.*|10|11) + [ "$(perl -I. -le 'use PgCommon; print $PgCommon::have_python2')" = "1" ] && PYTHON2_PACKAGE=postgresql-plpython-$v${PG_FLAVOR:=} + ;; + esac + apt-get install -y \ + postgresql-contrib-$v${PG_FLAVOR:=} \ + postgresql-plperl-$v$PG_FLAVOR \ + ${PYTHON2_PACKAGE:-} \ + $(dpkg --compare-versions $v ge 9.1 && echo postgresql-plpython3-$v$PG_FLAVOR) \ + postgresql-pltcl-$v$PG_FLAVOR \ + postgresql-server-dev-$v$PG_FLAVOR \ + postgresql-doc-$v$PG_FLAVOR + done + apt-get install -y \ + debhelper \ + libecpg-dev \ + locales-all \ + procps systemd \ + netcat-openbsd \ + hunspell-en-us \ + gcc + ;; + redhat|centos) + for v in $PG_VERSIONS; do + vv=$(echo $v | tr -d .) + yum install -y \ + postgresql$vv${PG_FLAVOR:=}-contrib \ + postgresql$vv$PG_FLAVOR-plperl \ + postgresql$vv$PG_FLAVOR-plpython3 \ + postgresql$vv$PG_FLAVOR-pltcl \ + postgresql$vv$PG_FLAVOR-devel \ + postgresql$vv$PG_FLAVOR-doc + done + yum install -y \ + nmap-ncat \ + perl-Test-Simple perl-Time-HiRes \ + gcc + ;; + *) + echo "Unknown distribution ID $ID in /etc/os-release" + exit 1 + ;; + esac +fi + +# shift away args +shift $(($OPTIND - 1)) + +# stop currently running clusters +if [ -d /run/systemd/system ] ; then + # stop postgresql@* explicitly (#759725) + systemctl stop postgresql "postgresql@*" "pg_receivewal@*" +else + service postgresql stop +fi + +# drop all existing clusters if the user requested it +if [ "${DROP_ALL_CLUSTERS:-}" ]; then + pg_lsclusters -h | while read ver cluster rest; do + pg_dropcluster $ver $cluster + done +fi + +# set up test directories +dirs="/etc/postgresql /var/lib/postgresql /var/log/postgresql /var/run/postgresql /var/backups/postgresql" + +if [ -z "${NO_TMPFS:-}" ]; then + # clean up after us + cleanup () { + set +e + umount -l $dirs + sed -i -e '/# by pg-testsuite/d' /etc/postgresql-common/createcluster.conf + ./pg_updateaptconfig + systemctl daemon-reload 2> /dev/null || : # poke generator to handle the system's clusters again + exit + } + trap "cleanup" 0 HUP INT QUIT ILL ABRT PIPE TERM + + for d in $dirs; do + mkdir -p $d + mount -t tmpfs tmpfs $d + done + + chown postgres:postgres /etc/postgresql ; chmod 755 /etc/postgresql + chown postgres:postgres /var/lib/postgresql ; chmod 755 /var/lib/postgresql + chown root:postgres /var/log/postgresql ; chmod 1775 /var/log/postgresql + chown postgres:postgres /var/run/postgresql ; chmod 2775 /var/run/postgresql /var/backups/postgresql +fi + +if [ -d /run/systemd/system ]; then + systemctl daemon-reload # poke generator to forget the system's clusters +fi + +# the RPM packages enable the logging_collector by default, which the testsuite +# doesn't like. We disable it unconditionally here. +if ! grep -q logging_collector /etc/postgresql-common/createcluster.conf; then + echo "logging_collector = off # by pg-testsuite" >> /etc/postgresql-common/createcluster.conf +fi + +# reset core limit for pg_ctl tests +ulimit -S -c 0 + +# set environment +unset TMPDIR +unset LC_ALL +export LANG=en_US.utf8 + +# set variables which cause taint check errors +export IFS +export CDPATH=/usr +export ENV=/nonexisting +export BASH_ENV=/nonexisting + +if [ $# -eq 0 ]; then + set -- $TESTSDIR/*.t +fi + +# dump environment for debugging +if [ "${VERBOSE:-}" ]; then + echo "Environment:" + env | sort + echo "Mounts:" + cat /proc/mounts + echo "Namespaces:" + ls -l /proc/self/ns + echo +fi + +for U in $PG_UMASKS; do + echo "====== Running all tests with umask $U =======" + umask $U + for T; do + TBASE=${T##*/} + [ "${FROM:-}" ] && [ "${TBASE%%_*}" -lt "${FROM:-}" ] && continue + echo "### PostgreSQL test $TBASE ###" + perl -I. $T || { + EXIT=$? + FAILED_TESTS="${FAILED_TESTS:-} $T" + if [ "${FAILURE:-}" ]; then + echo "*** $TBASE failed with status $EXIT, dropping you into a shell in the testbed ***" + ${SHELL:-/bin/sh} + fi + } + echo "### End test $TBASE ###" + echo + done +done + +if [ "${FAILED_TESTS:-}" ]; then + echo "Failed tests: $FAILED_TESTS" + echo +fi + +exit ${EXIT:-0} diff --git a/user_clusters b/user_clusters new file mode 100644 index 0000000..82d2d70 --- /dev/null +++ b/user_clusters @@ -0,0 +1,22 @@ +# This file maps users against the database clusters to which they +# will connect by default. Any user may create ~/.postgresqlrc which +# will supersede the defaults stored here. If a database is +# specified, that will be the one connected to by client tools if none +# is specified on the command line. If the database specified here is +# "*", this is interpreted as the database whose name is the same as +# the user's login. (Setting the database to "*" will provide the +# current default upstream behaviour for command line tools.) +# +# When pg_wrapper scans this file, the first matching line is used. +# It is a good idea to provide a default explicitly, with a final line +# where both user and group are set to "*". If there is no default, +# the implicit default is to connect to the cluster listening on +# port 5432 and to the database matching the user's login name. +# +# In the context of this file, user and group refer to the Unix login +# or group, not to PostgreSQL users and groups. +# +# Please see user_clusters(5) and postgresqlrc(5) for more information. +# +# USER GROUP VERSION CLUSTER DATABASE + diff --git a/user_clusters.5 b/user_clusters.5 new file mode 100644 index 0000000..a173fde --- /dev/null +++ b/user_clusters.5 @@ -0,0 +1,63 @@ +.TH USER_CLUSTERS 5 "Feburary 2005" "Debian" "Debian PostgreSQL infrastructure" + +.SH NAME +user_clusters \- File linking users to PostgreSQL clusters + +.SH DESCRIPTION +The file +.B /etc/postgresql-common/user_clusters +maps users against the database clusters to which they will +connect by default. However, every user can override these settings in +.B ~/.postgresqlrc\fR. + +When scanning this file, the first matching line will be used. It is a +good idea to provide a default explicitly, with a final line where both +user and group are set to +.BR * . + +If there is no default, the implicit default is to connect to the cluster +listening on port 5432 and to the database matching the user's +login name. + +.SH FORMAT +Comments are introduced by the character +.BR # . +Comments may follow data on a line; the first comment character terminates +the data. Leading whitespace and blank lines are ignored. + +Each uncommented, non\-blank line must describe a user, group or the +default (where both user and group are set to \fB*\fR). + +Fields must be given in the following order, separated by white space: + +.TP +.B USER +The login id of the Unix user to whom this line applies. The wildcard character +.B * +means any user. +.TP +.B GROUP +The group name of the Unix group to which this line applies. The wildcard character +.B * +means any group. +.TP +.B VERSION +The major PostgreSQL version of the cluster to connect to. +.TP +.B CLUSTER +The name of a cluster to connect to. A remote cluster is specified +with \fIhost\fR:\fIport\fR. If port is empty, it defaults to 5432. +.TP +.B DATABASE +Within the cluster, the database to which the user will connect by default +if he does not specify a database on the command line. If this is +.BR * , +the default database will be the one named by the user's login id. + +.SH NOTES +.P +Since the first matching line is used, the default line must come last. +.P + +.SH SEE ALSO +.BR pg_wrapper (1), " postgresqlrc" (5) |
