From 53b8d04ebc10d070b7efafd6dfa9de2897916888 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 13 Apr 2024 15:35:12 +0200 Subject: Adding upstream version 258. Signed-off-by: Daniel Baumann --- .vimrc | 3 + Makefile | 64 + PgCommon.pm | 1677 ++++++++++++++++++++ README.md | 114 ++ TODO | 39 + cleanpg | 24 + createcluster.conf | 41 + debhelper/Debian/Debhelper/Buildsystem/pgxs.pm | 58 + .../Debian/Debhelper/Buildsystem/pgxs_loop.pm | 33 + debhelper/Debian/Debhelper/Sequence/pgxs.pm | 22 + debhelper/Debian/Debhelper/Sequence/pgxs_loop.pm | 23 + debhelper/Debian/Debhelper/pgxs.pm | 38 + debhelper/dh_pgxs_test | 11 + debhelper/dh_pgxs_test.pod | 44 + dh_make_pgxs/debian/control.in | 23 + dh_make_pgxs/debian/copyright | 24 + dh_make_pgxs/debian/gitlab-ci.yml | 1 + dh_make_pgxs/debian/pgversions | 1 + dh_make_pgxs/debian/rules | 36 + dh_make_pgxs/debian/source/format | 1 + dh_make_pgxs/debian/tests/control | 5 + dh_make_pgxs/debian/tests/installcheck | 3 + dh_make_pgxs/debian/watch | 2 + dh_make_pgxs/dh_make_pgxs | 136 ++ dh_make_pgxs/dh_make_pgxs.pod | 43 + doc/dependencies.dia | Bin 0 -> 3144 bytes doc/dependencies.svg | 206 +++ doc/postgresql-debian-packaging.md | 741 +++++++++ gitlab/gitlab-ci.yml | 13 + pg_backupcluster | 659 ++++++++ pg_buildext | 503 ++++++ pg_buildext.pod | 358 +++++ pg_checksystem | 59 + pg_config | 33 + pg_conftool | 233 +++ pg_createcluster | 952 +++++++++++ pg_ctlcluster | 649 ++++++++ pg_dropcluster | 226 +++ pg_getwal | 96 ++ pg_hba | 144 ++ pg_lsclusters | 184 +++ pg_renamecluster | 176 ++ pg_restorecluster | 436 +++++ pg_updateaptconfig | 41 + pg_updatedicts | 139 ++ pg_upgradecluster | 966 +++++++++++ pg_upgradecluster.d/analyze | 33 + pg_virtualenv | 280 ++++ pg_virtualenv.pod | 122 ++ pg_wrapper | 329 ++++ pgdg/Makefile | 2 + pgdg/apt.postgresql.org.asc | 77 + pgdg/apt.postgresql.org.gpg | Bin 0 -> 3494 bytes pgdg/apt.postgresql.org.sh | 291 ++++ pgdg/update | 12 + pgxs_debian_control.mk | 13 + postgresqlrc.5 | 42 + rpm/README | 52 + rpm/init-functions-compat | 12 + rpm/postgresql-common.spec | 139 ++ server/README | 2 + server/catversion | 17 + server/pg_config.pl | 76 + server/postgresql.mk | 294 ++++ server/test-with-jit.conf | 8 + systemd/README.systemd | 55 + systemd/system-generators/postgresql-generator | 38 + systemd/system/pg_basebackup@.service | 14 + systemd/system/pg_basebackup@.timer | 12 + systemd/system/pg_compresswal@.service | 9 + systemd/system/pg_compresswal@.timer | 12 + systemd/system/pg_dump@.service | 14 + systemd/system/pg_dump@.timer | 12 + systemd/system/pg_receivewal@.service | 21 + systemd/system/postgresql.service | 18 + systemd/system/postgresql@.service | 40 + t/001_packages.t | 87 + t/002_existing_clusters.t | 11 + t/003_alternatives.t | 37 + t/005_PgCommon.t | 311 ++++ t/006_next_free_port.t | 49 + t/007_pg_conftool.t | 85 + t/010_defaultport_cluster.t | 33 + t/015_start_stop.t | 174 ++ t/020_create_sql_remove.t | 452 ++++++ t/021_pg_renamecluster.t | 43 + t/022_recovery.t | 54 + t/025_logging.t | 99 ++ t/030_errors.t | 336 ++++ t/031_errors_disk_full.t | 86 + t/032_ssl_key_permissions.t | 60 + t/040_upgrade.t | 272 ++++ t/041_upgrade_custompaths.t | 51 + t/042_upgrade_rename.t | 27 + t/043_upgrade_ssl_cert.t | 79 + t/045_backup.t | 169 ++ t/050_encodings.t | 117 ++ t/052_upgrade_encodings.t | 98 ++ t/060_obsolete_confparams.t | 83 + t/070_non_postgres_clusters.t | 116 ++ t/080_start.conf.t | 145 ++ t/085_pg_ctl.conf.t | 51 + t/090_multicluster.t | 286 ++++ t/110_integrate_cluster.t | 44 + t/120_pg_upgradecluster_scripts.t | 114 ++ t/130_nonroot_admin.t | 50 + t/135_pg_virtualenv.t | 35 + t/140_pg_config.t | 89 ++ t/150_tsearch_stemming.t | 108 ++ t/160_alternate_confroot.t | 57 + t/170_extensions.t | 87 + t/180_ecpg.t | 56 + t/190_pg_buildext.t | 104 ++ t/200_maintscripts.t | 46 + t/TestLib.pm | 270 ++++ t/bar/debian/control.in | 8 + t/bar/debian/pgversions | 3 + t/foo/Makefile | 12 + t/foo/foo-123/Makefile | 10 + t/foo/foo-123/README.md | 5 + t/foo/foo-123/expected/foo.out | 8 + t/foo/foo-123/expected/upgrade.out | 14 + t/foo/foo-123/foo--100--123.sql | 3 + t/foo/foo-123/foo--100.sql | 3 + t/foo/foo-123/foo--123.sql | 3 + t/foo/foo-123/foo.c | 13 + t/foo/foo-123/foo.control | 2 + t/foo/foo-123/sql/foo.sql | 5 + t/foo/foo-123/sql/upgrade.sql | 7 + t/template | 26 + testsuite | 222 +++ user_clusters | 22 + user_clusters.5 | 63 + 133 files changed, 16226 insertions(+) create mode 100644 .vimrc create mode 100644 Makefile create mode 100644 PgCommon.pm create mode 100644 README.md create mode 100644 TODO create mode 100755 cleanpg create mode 100644 createcluster.conf create mode 100644 debhelper/Debian/Debhelper/Buildsystem/pgxs.pm create mode 100644 debhelper/Debian/Debhelper/Buildsystem/pgxs_loop.pm create mode 100644 debhelper/Debian/Debhelper/Sequence/pgxs.pm create mode 100644 debhelper/Debian/Debhelper/Sequence/pgxs_loop.pm create mode 100644 debhelper/Debian/Debhelper/pgxs.pm create mode 100755 debhelper/dh_pgxs_test create mode 100644 debhelper/dh_pgxs_test.pod create mode 100644 dh_make_pgxs/debian/control.in create mode 100644 dh_make_pgxs/debian/copyright create mode 100644 dh_make_pgxs/debian/gitlab-ci.yml create mode 100644 dh_make_pgxs/debian/pgversions create mode 100755 dh_make_pgxs/debian/rules create mode 100644 dh_make_pgxs/debian/source/format create mode 100644 dh_make_pgxs/debian/tests/control create mode 100755 dh_make_pgxs/debian/tests/installcheck create mode 100644 dh_make_pgxs/debian/watch create mode 100755 dh_make_pgxs/dh_make_pgxs create mode 100644 dh_make_pgxs/dh_make_pgxs.pod create mode 100644 doc/dependencies.dia create mode 100644 doc/dependencies.svg create mode 100644 doc/postgresql-debian-packaging.md create mode 100644 gitlab/gitlab-ci.yml create mode 100755 pg_backupcluster create mode 100755 pg_buildext create mode 100644 pg_buildext.pod create mode 100755 pg_checksystem create mode 100755 pg_config create mode 100755 pg_conftool create mode 100755 pg_createcluster create mode 100755 pg_ctlcluster create mode 100755 pg_dropcluster create mode 100755 pg_getwal create mode 100755 pg_hba create mode 100755 pg_lsclusters create mode 100755 pg_renamecluster create mode 100755 pg_restorecluster create mode 100755 pg_updateaptconfig create mode 100755 pg_updatedicts create mode 100755 pg_upgradecluster create mode 100755 pg_upgradecluster.d/analyze create mode 100755 pg_virtualenv create mode 100644 pg_virtualenv.pod create mode 100755 pg_wrapper create mode 100644 pgdg/Makefile create mode 100644 pgdg/apt.postgresql.org.asc create mode 100644 pgdg/apt.postgresql.org.gpg create mode 100755 pgdg/apt.postgresql.org.sh create mode 100755 pgdg/update create mode 100644 pgxs_debian_control.mk create mode 100644 postgresqlrc.5 create mode 100644 rpm/README create mode 100644 rpm/init-functions-compat create mode 100644 rpm/postgresql-common.spec create mode 100644 server/README create mode 100755 server/catversion create mode 100755 server/pg_config.pl create mode 100644 server/postgresql.mk create mode 100644 server/test-with-jit.conf create mode 100644 systemd/README.systemd create mode 100755 systemd/system-generators/postgresql-generator create mode 100644 systemd/system/pg_basebackup@.service create mode 100644 systemd/system/pg_basebackup@.timer create mode 100644 systemd/system/pg_compresswal@.service create mode 100644 systemd/system/pg_compresswal@.timer create mode 100644 systemd/system/pg_dump@.service create mode 100644 systemd/system/pg_dump@.timer create mode 100644 systemd/system/pg_receivewal@.service create mode 100644 systemd/system/postgresql.service create mode 100644 systemd/system/postgresql@.service create mode 100644 t/001_packages.t create mode 100644 t/002_existing_clusters.t create mode 100644 t/003_alternatives.t create mode 100644 t/005_PgCommon.t create mode 100644 t/006_next_free_port.t create mode 100644 t/007_pg_conftool.t create mode 100644 t/010_defaultport_cluster.t create mode 100644 t/015_start_stop.t create mode 100644 t/020_create_sql_remove.t create mode 100644 t/021_pg_renamecluster.t create mode 100644 t/022_recovery.t create mode 100644 t/025_logging.t create mode 100644 t/030_errors.t create mode 100644 t/031_errors_disk_full.t create mode 100644 t/032_ssl_key_permissions.t create mode 100644 t/040_upgrade.t create mode 100644 t/041_upgrade_custompaths.t create mode 100644 t/042_upgrade_rename.t create mode 100644 t/043_upgrade_ssl_cert.t create mode 100644 t/045_backup.t create mode 100644 t/050_encodings.t create mode 100644 t/052_upgrade_encodings.t create mode 100644 t/060_obsolete_confparams.t create mode 100644 t/070_non_postgres_clusters.t create mode 100644 t/080_start.conf.t create mode 100644 t/085_pg_ctl.conf.t create mode 100644 t/090_multicluster.t create mode 100644 t/110_integrate_cluster.t create mode 100644 t/120_pg_upgradecluster_scripts.t create mode 100644 t/130_nonroot_admin.t create mode 100644 t/135_pg_virtualenv.t create mode 100644 t/140_pg_config.t create mode 100644 t/150_tsearch_stemming.t create mode 100644 t/160_alternate_confroot.t create mode 100644 t/170_extensions.t create mode 100644 t/180_ecpg.t create mode 100644 t/190_pg_buildext.t create mode 100644 t/200_maintscripts.t create mode 100644 t/TestLib.pm create mode 100644 t/bar/debian/control.in create mode 100644 t/bar/debian/pgversions create mode 100644 t/foo/Makefile create mode 100644 t/foo/foo-123/Makefile create mode 100644 t/foo/foo-123/README.md create mode 100644 t/foo/foo-123/expected/foo.out create mode 100644 t/foo/foo-123/expected/upgrade.out create mode 100644 t/foo/foo-123/foo--100--123.sql create mode 100644 t/foo/foo-123/foo--100.sql create mode 100644 t/foo/foo-123/foo--123.sql create mode 100644 t/foo/foo-123/foo.c create mode 100644 t/foo/foo-123/foo.control create mode 100644 t/foo/foo-123/sql/foo.sql create mode 100644 t/foo/foo-123/sql/upgrade.sql create mode 100644 t/template create mode 100755 testsuite create mode 100644 user_clusters create mode 100644 user_clusters.5 diff --git a/.vimrc b/.vimrc new file mode 100644 index 0000000..1853616 --- /dev/null +++ b/.vimrc @@ -0,0 +1,3 @@ +set expandtab +set shiftwidth=4 +set smarttab diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..07a50bd --- /dev/null +++ b/Makefile @@ -0,0 +1,64 @@ +POD2MAN=pod2man --center "Debian PostgreSQL infrastructure" -r "Debian" +POD1PROGS = pg_backupcluster.1 \ + pg_conftool.1 \ + pg_createcluster.1 \ + pg_ctlcluster.1 \ + pg_dropcluster.1 \ + pg_getwal.1 \ + pg_lsclusters.1 \ + pg_renamecluster.1 \ + pg_restorecluster.1 \ + pg_upgradecluster.1 \ + pg_wrapper.1 +POD1PROGS_POD = pg_buildext.1 \ + pg_virtualenv.1 \ + debhelper/dh_pgxs_test.1 \ + dh_make_pgxs/dh_make_pgxs.1 +POD8PROGS = pg_updatedicts.8 + +all: man sub-pgdg + +man: $(POD1PROGS) $(POD1PROGS_POD) $(POD8PROGS) + +%.1: %.pod + $(POD2MAN) --quotes=none --section 1 $< $@ + +%.1: % + $(POD2MAN) --quotes=none --section 1 $< $@ + +%.8: % + $(POD2MAN) --quotes=none --section 8 $< $@ + +clean: + rm -f *.1 *.8 debhelper/*.1 dh_make_pgxs/*.1 + +sub-pgdg: + $(MAKE) -C pgdg + +# rpm + +DPKG_VERSION=$(shell sed -ne '1s/.*(//; 1s/).*//p' debian/changelog) +RPMDIR=$(CURDIR)/rpm +TARNAME=postgresql-common_$(DPKG_VERSION).tar.xz +TARBALL=$(RPMDIR)/SOURCES/$(TARNAME) + +rpmbuild: $(TARBALL) + rpmbuild -D"%_topdir $(RPMDIR)" --define='version $(DPKG_VERSION)' -ba rpm/postgresql-common.spec + +$(TARBALL): + mkdir -p $(dir $(TARBALL)) + if test -f ../$(TARNAME); then \ + cp -v ../$(TARNAME) $(TARBALL); \ + else \ + git archive --prefix=postgresql-common-$(DPKG_VERSION)/ HEAD | xz > $(TARBALL); \ + fi + +rpminstall: + sudo yum install -y perl-JSON + sudo rpm --upgrade --replacefiles --replacepkgs -v $(RPMDIR)/RPMS/noarch/*-$(DPKG_VERSION)-*.rpm + +rpmremove: + -sudo rpm -e postgresql-common postgresql-client-common postgresql-server-dev-all + +rpmclean: + rm -rf $(RPMDIR)/*/ diff --git a/PgCommon.pm b/PgCommon.pm new file mode 100644 index 0000000..df4d060 --- /dev/null +++ b/PgCommon.pm @@ -0,0 +1,1677 @@ +=head1 NAME + +PgCommon - Common functions for the postgresql-common framework + +=head1 COPYRIGHT AND LICENSE + + (C) 2008-2009 Martin Pitt + (C) 2012-2023 Christoph Berg + +This program is free software; you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the +Free Software Foundation; either +L, +or (at your option) any later version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +=cut + +package PgCommon; +use strict; +use IPC::Open3; +use Socket; +use POSIX; + +use Exporter; +our $VERSION = 1.00; +our @ISA = ('Exporter'); +our @EXPORT = qw/error user_cluster_map get_cluster_port set_cluster_port + get_cluster_socketdir set_cluster_socketdir cluster_port_running + get_cluster_start_conf set_cluster_start_conf set_cluster_pg_ctl_conf + get_program_path cluster_info validate_cluster_owner get_versions get_newest_version version_exists + get_version_clusters next_free_port cluster_exists install_file + change_ugid system_or_error config_bool replace_v_c + get_db_encoding get_db_locales get_cluster_locales get_cluster_controldata + get_cluster_databases cluster_conf_filename read_cluster_conf_file + read_pg_hba read_pidfile valid_hba_method/; +our @EXPORT_OK = qw/$confroot $binroot $rpm $have_python2 + quote_conf_value read_conf_file get_conf_value + set_conf_value set_conffile_value disable_conffile_value disable_conf_value + replace_conf_value cluster_data_directory get_file_device + check_pidfile_running/; + + +=head1 CONTENTS + +=head2 error + + Print an error message to stderr and die with exit status 1 + +=cut + +sub error { + $! = 1; # force exit code 1 + die "Error: $_[0]\n"; +} + +our $confroot = '/etc/postgresql'; +if ($ENV{'PG_CLUSTER_CONF_ROOT'}) { + ($confroot) = $ENV{'PG_CLUSTER_CONF_ROOT'} =~ /(.*)/; # untaint +} +our $common_confdir = "/etc/postgresql-common"; +if ($ENV{'PGSYSCONFDIR'}) { + ($common_confdir) = $ENV{'PGSYSCONFDIR'} =~ /(.*)/; # untaint +} +my $mapfile = "$common_confdir/user_clusters"; +our $binroot = "/usr/lib/postgresql/"; +#redhat# $binroot = "/usr/pgsql-"; +our $rpm = 0; +#redhat# $rpm = 1; +our $defaultport = 5432; +our $have_python2 = 0; # python2 removed in bullseye+ +#py2#$have_python2 = 1; + +=head2 prepare_exec, restore_exec + + Untaint the environment for executing an external program + + Optional arguments: list of additional variables + +=cut + +{ + my %saved_env; + + # untaint the environment for executing an external program + sub prepare_exec { + my @cleanvars = qw/PATH IFS ENV BASH_ENV CDPATH/; + push @cleanvars, @_; + %saved_env = (); + + foreach (@cleanvars) { + $saved_env{$_} = $ENV{$_}; + delete $ENV{$_}; + } + + $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin'; + } + + # restore the environment after prepare_exec() + sub restore_exec { + foreach (keys %saved_env) { + if (defined $saved_env{$_}) { + $ENV{$_} = $saved_env{$_}; + } else { + delete $ENV{$_}; + } + } + } +} + + +=head2 config_bool + + returns '1' if the argument is a configuration file value that stands for + true (ON, TRUE, YES, or 1, case insensitive), '0' if the argument represents + a false value (OFF, FALSE, NO, or 0, case insensitive), or undef otherwise. + +=cut + +sub config_bool { + return undef unless defined($_[0]); + return 1 if ($_[0] =~ /^(on|true|yes|1)$/i); + return 0 if ($_[0] =~ /^(off|false|no|0)$/i); + return undef; +} + + +=head2 quote_conf_value + + Quotes a value with single quotes + + Arguments: + Returns: quoted string + +=cut + +sub quote_conf_value ($) { + my $value = shift; + return $value if ($value =~ /^-?[\d.]+$/); # integer or float + return $value if ($value =~ /^\w+$/); # plain word + $value =~ s/'/''/g; # else quote it + return "'$value'"; +} + + +=head2 replace_v_c + + Replaces %v and %c placeholders + + Arguments: + Returns: string + +=cut + +sub replace_v_c ($$$) { + my ($str, $version, $cluster) = @_; + $str =~ s/%([vc%])/$1 eq 'v' ? $version : + $1 eq 'c' ? $cluster : '%'/eg; + return $str; +} + + +=head2 read_conf_file + + Read a 'var = value' style configuration file and return a hash with the + values. Error out if the file cannot be read. + + If the file name ends with '.conf', the keys will be normalized to + lower case (suitable for e.g. postgresql.conf), otherwise kept intact + (suitable for environment). + + Arguments: + Returns: hash (empty if file does not exist) + +=cut + +sub read_conf_file { + my ($config_path) = @_; + my %conf; + local (*F); + + sub get_absolute_path { + my ($path, $parent_path) = @_; + return $path if ($path =~ m!^/!); # path is absolute + # else strip filename component from parent path + $parent_path =~ s!/[^/]*$!!; + return "$parent_path/$path"; + } + + if (open F, $config_path) { + while () { + if (/^\s*(?:#.*)?$/) { + next; + } elsif(/^\s*include_dir\s*=?\s*'([^']+)'\s*(?:#.*)?$/i) { + # read included configuration directory and merge into %conf + # files in the directory will be read in ascending order + my $path = $1; + my $absolute_path = get_absolute_path($path, $config_path); + next unless -e $absolute_path && -d $absolute_path; + my $dir; + opendir($dir, $absolute_path) or next; + foreach my $filename (sort readdir($dir) ) { + next if ($filename =~ m/^\./ or not $filename =~/\.conf$/ ); + my %include_conf = read_conf_file("$absolute_path/$filename"); + while ( my ($k, $v) = each(%include_conf) ) { + $conf{$k} = $v; + } + } + closedir($dir); + } elsif (/^\s*include(?:_if_exists)?\s*=?\s*'([^']+)'\s*(?:#.*)?$/i) { + # read included file and merge into %conf + my $path = $1; + my $absolute_path = get_absolute_path($path, $config_path); + my %include_conf = read_conf_file($absolute_path); + while ( my ($k, $v) = each(%include_conf) ) { + $conf{$k} = $v; + } + } elsif (/^\s*([a-zA-Z0-9_.-]+)\s*(?:=|\s)\s*'((?:[^']|''|(?:(?<=\\)'))*)'\s*(?:#.*)?$/i) { + # string value + my $v = $2; + my $k = $1; + $k = lc $k if $config_path =~ /\.conf$/; + $v =~ s/\\(.)/$1/g; + $v =~ s/''/'/g; + $conf{$k} = $v; + } elsif (m{^\s*([a-zA-Z0-9_.-]+)\s*(?:=|\s)\s*(-?[[:alnum:]][[:alnum:]._:/+-]*)\s*(?:\#.*)?$}i) { + # simple value (string/float) + my $v = $2; + my $k = $1; + $k = lc $k if $config_path =~ /\.conf$/; + $conf{$k} = $v; + } else { + chomp; + error "invalid line $. in $config_path: $_"; + } + } + close F; + } + + return %conf; +} + +=head2 cluster_conf_filename + + Returns path to cluster config file from a cluster configuration + directory (with /etc/postgresql-common/ as fallback) + and return a hash with the values. Error out if the file cannot be read. + If config file name is postgresql.auto.conf, read from PGDATA + + Arguments: + Returns: hash (empty if the file does not exist) + +=cut + +sub cluster_conf_filename { + my ($version, $cluster, $configfile) = @_; + if ($configfile eq 'postgresql.auto.conf') { + my $data_directory = cluster_data_directory($version, $cluster); + return "$data_directory/$configfile"; + } + my $fname = "$confroot/$version/$cluster/$configfile"; + -e $fname or $fname = "$common_confdir/$configfile"; + return $fname; +} + + +=head2 read_cluster_conf_file + +Read a 'var = value' style configuration file from a cluster configuration + +Arguments: +Returns: hash (empty if the file does not exist) + +=cut + +sub read_cluster_conf_file { + my ($version, $cluster, $configfile) = @_; + my %conf = read_conf_file(cluster_conf_filename($version, $cluster, $configfile)); + + if ($version >= 9.4 and $configfile eq 'postgresql.conf') { # merge settings changed by ALTER SYSTEM + # data_directory cannot be changed by ALTER SYSTEM + my $data_directory = cluster_data_directory($version, $cluster, \%conf); + my %auto_conf = read_conf_file "$data_directory/postgresql.auto.conf"; + foreach my $guc (keys %auto_conf) { + next if ($guc eq 'data_directory'); # defend against pg_upgradecluster bug in 200..202 + $conf{$guc} = $auto_conf{$guc}; + } + } + + return %conf; +} + + +=head2 get_conf_value + + Return parameter from a PostgreSQL configuration file, + or undef if the parameter does not exist. + + Arguments: + +=cut + +sub get_conf_value { + my %conf = (read_cluster_conf_file $_[0], $_[1], $_[2]); + return $conf{$_[3]}; +} + + +=head2 set_conffile_value + + Set parameter of a PostgreSQL configuration file. + + Arguments: + +=cut + +sub set_conffile_value { + my ($fname, $key, $value) = ($_[0], $_[1], quote_conf_value($_[2])); + my @lines; + + # read configuration file lines + open (F, $fname) or die "Error: could not open $fname for reading"; + push @lines, $_ while (); + close F; + + my $found = 0; + # first, search for an uncommented setting + for (my $i=0; $i <= $#lines; ++$i) { + if ($lines[$i] =~ /^\s*($key)(\s*(?:=|\s)\s*)\w+\b((?:\s*#.*)?)/i or + $lines[$i] =~ /^\s*($key)(\s*(?:=|\s)\s*)'[^']*'((?:\s*#.*)?)/i) { + $lines[$i] = "$1$2$value$3\n"; + $found = 1; + last; + } + } + + # now check if the setting exists as a comment; if so, change that instead + # of appending + if (!$found) { + for (my $i=0; $i <= $#lines; ++$i) { + if ($lines[$i] =~ /^\s*#\s*($key)(\s*(?:=|\s)\s*)\w+\b((?:\s*#.*)?)$/i or + $lines[$i] =~ /^\s*#\s*($key)(\s*(?:=|\s)\s*)'[^']*'((?:\s*#.*)?)$/i) { + $lines[$i] = "$1$2$value$3\n"; + $found = 1; + last; + } + } + } + + # not found anywhere, append it + push (@lines, "$key = $value\n") unless $found; + + # write configuration file lines + open (F, ">$fname.new") or die "Error: could not open $fname.new for writing"; + foreach (@lines) { + print F $_ or die "writing $fname.new: $!"; + } + close F; + + # copy permissions + my @st = stat $fname or die "stat: $!"; + chown $st[4], $st[5], "$fname.new"; # might fail as non-root + chmod $st[2], "$fname.new" or die "chmod: $!"; + + rename "$fname.new", "$fname" or die "rename $fname.new $fname: $!"; +} + + +=head2 set_conf_value + + Set parameter of a PostgreSQL cluster configuration file. + + Arguments: + +=cut + +sub set_conf_value { + return set_conffile_value(cluster_conf_filename($_[0], $_[1], $_[2]), $_[3], $_[4]); +} + + +=head2 disable_conffile_value + + Disable a parameter in a PostgreSQL configuration file by prepending it + with a '#'. Appends an optional explanatory comment if given. + + Arguments: + +=cut + +sub disable_conffile_value { + my ($fname, $key, $reason) = @_; + my @lines; + + # read configuration file lines + open (F, $fname) or die "Error: could not open $fname for reading"; + push @lines, $_ while (); + close F; + + my $changed = 0; + for (my $i=0; $i <= $#lines; ++$i) { + if ($lines[$i] =~ /^\s*$key\s*(?:=|\s)/i) { + $lines[$i] =~ s/^/#/; + $lines[$i] =~ s/$/ #$reason/ if $reason; + $changed = 1; + last; + } + } + + # write configuration file lines + if ($changed) { + open (F, ">$fname.new") or die "Error: could not open $fname.new for writing"; + foreach (@lines) { + print F $_ or die "writing $fname.new: $!"; + } + close F; + + # copy permissions + my @st = stat $fname or die "stat: $!"; + chown $st[4], $st[5], "$fname.new"; # might fail as non-root + chmod $st[2], "$fname.new" or die "chmod: $1"; + + rename "$fname.new", "$fname"; + } +} + + +=head2 disable_conf_value + + Disable a parameter in a PostgreSQL cluster configuration file by prepending + it with a '#'. Appends an optional explanatory comment if given. + + Arguments: + +=cut + +sub disable_conf_value { + return disable_conffile_value(cluster_conf_filename($_[0], $_[1], $_[2]), $_[3], $_[4]); +} + + +=head2 replace_conf_value + + Replace a parameter in a PostgreSQL configuration file. The old parameter + is prepended with a '#' and gets an optional explanatory comment + appended, if given. The new parameter is inserted directly after the old one. + + Arguments: + + +=cut + +sub replace_conf_value { + my ($version, $cluster, $configfile, $oldparam, $reason, $newparam, $val) = @_; + my $fname = cluster_conf_filename($version, $cluster, $configfile); + my @lines; + + # quote $val if necessary + unless ($val =~ /^\w+$/) { + $val = "'$val'"; + } + + # read configuration file lines + open (F, $fname) or die "Error: could not open $fname for reading"; + push @lines, $_ while (); + close F; + + my $found = 0; + for (my $i = 0; $i <= $#lines; ++$i) { + if ($lines[$i] =~ /^\s*$oldparam\s*(?:=|\s)/i) { + $lines[$i] = '#'.$lines[$i]; + chomp $lines[$i]; + $lines[$i] .= ' #'.$reason."\n" if $reason; + + # insert the new param + splice @lines, $i+1, 0, "$newparam = $val\n"; + ++$i; + + $found = 1; + last; + } + } + + return if !$found; + + # write configuration file lines + open (F, ">$fname.new") or die "Error: could not open $fname.new for writing"; + foreach (@lines) { + print F $_ or die "writing $fname.new: $!"; + } + close F; + + # copy permissions + my @st = stat $fname or die "stat: $!"; + chown $st[4], $st[5], "$fname.new"; # might fail as non-root + chmod $st[2], "$fname.new" or die "chmod: $1"; + + rename "$fname.new", "$fname"; +} + + +=head2 get_cluster_port + + Return the port of a particular cluster + + Arguments: + +=cut + +sub get_cluster_port { + return get_conf_value($_[0], $_[1], 'postgresql.conf', 'port') || $defaultport; +} + + +=head2 set_cluster_port + + Set the port of a particular cluster. + + Arguments: + +=cut + +sub set_cluster_port { + set_conf_value $_[0], $_[1], 'postgresql.conf', 'port', $_[2]; +} + + +=head2 cluster_data_directory + + Return cluster data directory. + + Arguments: [] + +=cut + +sub cluster_data_directory { + my $d; + if ($_[2]) { + $d = ${$_[2]}{'data_directory'}; + } else { + $d = get_conf_value($_[0], $_[1], 'postgresql.conf', 'data_directory'); + } + my $confdir = "$confroot/$_[0]/$_[1]"; + if (!$d) { + # fall back to /pgdata symlink (supported by earlier p-common releases) + $d = readlink "$confdir/pgdata"; + } + if (!$d and -l $confdir and -f "$confdir/PG_VERSION") { # symlink from /etc/postgresql + $d = readlink $confdir; + } + if (!$d and -f "$confdir/PG_VERSION") { # PGDATA in /etc/postgresql + $d = $confdir; + } + ($d) = $d =~ /(.*)/ if defined $d; #untaint + return $d; +} + + +=head2 get_cluster_socketdir + + Return the socket directory of a particular cluster + or undef if the cluster does not exist. + + Arguments: + +=cut + +sub get_cluster_socketdir { + # if it is explicitly configured, just return it + my $socketdir = get_conf_value($_[0], $_[1], 'postgresql.conf', + $_[0] >= 9.3 ? 'unix_socket_directories' : 'unix_socket_directory'); + $socketdir =~ s/\s*,.*// if ($socketdir); # ignore additional directories for now + return $socketdir if $socketdir; + + #redhat# return '/tmp'; # RedHat PGDG packages default to /tmp + # try to determine whether this is a postgres owned cluster and we default + # to /var/run/postgresql + $socketdir = '/var/run/postgresql'; + my @socketdirstat = stat $socketdir; + + error "Cannot stat $socketdir" unless @socketdirstat; + + if ($_[0] && $_[1]) { + my $datadir = cluster_data_directory $_[0], $_[1]; + error "Invalid data directory for cluster $_[0] $_[1]" unless $datadir; + my @datadirstat = stat $datadir; + unless (@datadirstat) { + my @p = split '/', $datadir; + my $parent = join '/', @p[0..($#p-1)]; + error "$datadir is not accessible; please fix the directory permissions ($parent/ should be world readable)" unless @datadirstat; + } + + $socketdir = '/tmp' if $socketdirstat[4] != $datadirstat[4]; + } + + return $socketdir; +} + + +=head2 set_cluster_socketdir + + Set the socket directory of a particular cluster. + + Arguments: + +=cut + +sub set_cluster_socketdir { + set_conf_value $_[0], $_[1], 'postgresql.conf', + $_[0] >= 9.3 ? 'unix_socket_directories' : 'unix_socket_directory', + $_[2]; +} + + +=head2 get_program_path + + Return the path of a program of a particular version. + + Arguments: [] + +=cut + +sub get_program_path { + my ($program, $version) = @_; + return '' unless defined $program; + $version //= get_newest_version($program); + my $path = "$binroot$version/bin/$program"; + ($path) = $path =~ /(.*)/; #untaint + return $path if -x $path; + return ''; +} + + +=head2 cluster_port_running + + Check whether a postgres server is running at the specified port. + + Arguments: + +=cut + +sub cluster_port_running { + die "port_running: invalid port $_[2]" if $_[2] !~ /\d+/; + my $socketdir = get_cluster_socketdir $_[0], $_[1]; + my $socketpath = "$socketdir/.s.PGSQL.$_[2]"; + return 0 unless -S $socketpath; + + socket(SRV, PF_UNIX, SOCK_STREAM, 0) or die "socket: $!"; + my $running = connect(SRV, sockaddr_un($socketpath)); + close SRV; + return $running ? 1 : 0; +} + + +=head2 get_cluster_start_conf + + Read, verify, and return the current start.conf setting. + + Arguments: + Returns: auto | manual | disabled + +=cut + +sub get_cluster_start_conf { + my $start_conf = "$confroot/$_[0]/$_[1]/start.conf"; + if (-e $start_conf) { + open F, $start_conf or error "Could not open $start_conf: $!"; + while () { + s/#.*$//; + s/^\s*//; + s/\s*$//; + next unless $_; + close F; + return $1 if (/^(auto|manual|disabled)/); + error "Invalid mode in $start_conf, must be one of auto, manual, disabled"; + } + close F; + } + return 'auto'; # default +} + + +=head2 set_cluster_start_conf + + Change start.conf setting. + + Arguments: + = auto | manual | disabled + +=cut + +sub set_cluster_start_conf { + my ($v, $c, $val) = @_; + + error "Invalid mode: '$val'" unless $val eq 'auto' || + $val eq 'manual' || $val eq 'disabled'; + + my $perms = 0644; + + # start.conf setting + my $start_conf = "$confroot/$_[0]/$_[1]/start.conf"; + my $text; + if (-e $start_conf) { + open F, $start_conf or error "Could not open $start_conf: $!"; + while () { + if (/^\s*(?:auto|manual|disabled)\b(.*$)/) { + $text .= $val . $1 . "\n"; + } else { + $text .= $_; + } + } + + # preserve permissions if it already exists + $perms = (stat F)[2]; + error "Could not get permissions of $start_conf: $!" unless $perms; + close F; + } else { + $text = "# Automatic startup configuration +# auto: automatically start the cluster +# manual: manual startup with pg_ctlcluster/postgresql@.service only +# disabled: refuse to start cluster +# See pg_createcluster(1) for details. When running from systemd, +# invoke 'systemctl daemon-reload' after editing this file. + +$val +"; + } + + open F, '>' . $start_conf or error "Could not open $start_conf for writing: $!"; + chmod $perms, $start_conf; + print F $text; + close F; +} + + +=head2 set_cluster_pg_ctl_conf + + Change pg_ctl.conf setting. + + Arguments: + = options passed to pg_ctl(1) + +=cut + +sub set_cluster_pg_ctl_conf { + my ($v, $c, $opts) = @_; + my $perms = 0644; + + # pg_ctl.conf setting + my $pg_ctl_conf = "$confroot/$v/$c/pg_ctl.conf"; + my $text = "# Automatic pg_ctl configuration +# This configuration file contains cluster specific options to be passed to +# pg_ctl(1). + +pg_ctl_options = '$opts' +"; + + open F, '>' . $pg_ctl_conf or error "Could not open $pg_ctl_conf for writing: $!"; + chmod $perms, $pg_ctl_conf; + print F $text; + close F; +} + + +=head2 read_pidfile + + Return the PID from an existing PID file or undef if it does not exist. + + Arguments: + +=cut + +sub read_pidfile { + return undef unless -e $_[0]; + + if (open PIDFILE, $_[0]) { + my $pid = ; + close PIDFILE; + return undef unless ($pid); + chomp $pid; + ($pid) = $pid =~ /^(\d+)\s*$/; # untaint + return $pid; + } else { + return undef; + } +} + + +=head2 check_pidfile_running + + Check whether a pid file is present and belongs to a running postgres. + Returns undef if it cannot be determined + + Arguments: + + postgres does not clean up the PID file when it stops, and it is + not world readable, so only its absence is a definitive result; + if it is present, we need to read it and check the PID, which will + only work as root + +=cut + +sub check_pidfile_running { + return 0 if ! -e $_[0]; + + my $pid = read_pidfile $_[0]; + if (defined $pid and open CL, "/proc/$pid/cmdline") { + my $cmdline = ; + close CL; + if ($cmdline and $cmdline =~ /\bpostgres\b/) { + return 1; + } else { + return 0; + } + } + return undef; +} + + +=head2 cluster_supervisor + + Determine if a cluster is managed by a supervisor (pacemaker, patroni). + Returns undef if it cannot be determined + + Arguments: + + postgres does not clean up the PID file when it stops, and it is + not world readable, so only its absence is a definitive result; if it + is present, we need to read it and check the PID, which will only + work as root + +=cut + +sub cluster_supervisor { + return undef if ! -e $_[0]; + + my $pid = read_pidfile $_[0]; + if (defined $pid and open(CG, "/proc/$pid/cgroup")) { + local $/; # enable localized slurp mode + my $cgroup = ; + close CG; + if ($cgroup and $cgroup =~ /\b(pacemaker|patroni)\b/) { + return $1; + } + } + return undef; +} + + +=head2 cluster_info + + Return a hash with information about a specific cluster (which needs to exist). + + Arguments: + Returns: information hash (keys: pgdata, port, running, logfile [unless it + has a custom one], configdir, owneruid, ownergid, waldir, socketdir, + config->postgresql.conf) + +=cut + +sub cluster_info { + my ($v, $c) = @_; + error 'cluster_info must be called with arguments' unless ($v and $c); + + my %result; + $result{'configdir'} = "$confroot/$v/$c"; + $result{'configuid'} = (stat "$result{configdir}/postgresql.conf")[4]; + + my %postgresql_conf = read_cluster_conf_file $v, $c, 'postgresql.conf'; + $result{'config'} = \%postgresql_conf; + $result{'pgdata'} = cluster_data_directory $v, $c, \%postgresql_conf; + return %result unless (keys %postgresql_conf); + $result{'port'} = $postgresql_conf{'port'} || $defaultport; + $result{'socketdir'} = get_cluster_socketdir $v, $c; + + # if we can determine the running status with the pid file, prefer that + if ($postgresql_conf{'external_pid_file'} && + $postgresql_conf{'external_pid_file'} ne '(none)') { + $result{'running'} = check_pidfile_running $postgresql_conf{'external_pid_file'}; + my $supervisor = cluster_supervisor($postgresql_conf{'external_pid_file'}); + $result{supervisor} = $supervisor if ($supervisor); + } + + # otherwise fall back to probing the port; this is unreliable if the port + # was changed in the configuration file in the meantime + if (!defined ($result{'running'})) { + $result{'running'} = cluster_port_running ($v, $c, $result{'port'}); + } + + if ($result{'pgdata'}) { + ($result{'owneruid'}, $result{'ownergid'}) = + (stat $result{'pgdata'})[4,5]; + if ($v >= 12) { + $result{'recovery'} = 1 if (-e "$result{'pgdata'}/recovery.signal" + or -e "$result{'pgdata'}/standby.signal"); + } else { + $result{'recovery'} = 1 if (-e "$result{'pgdata'}/recovery.conf"); + } + my $waldirname = $v >= 10 ? 'pg_wal' : 'pg_xlog'; + if (-l "$result{pgdata}/$waldirname") { # custom wal directory + ($result{waldir}) = readlink("$result{pgdata}/$waldirname") =~ /(.*)/; # untaint + } + } + $result{'start'} = get_cluster_start_conf $v, $c; + + # default log file (possibly used only for early startup messages) + my $log_symlink = $result{'configdir'} . "/log"; + if (-l $log_symlink) { + ($result{'logfile'}) = readlink ($log_symlink) =~ /(.*)/; # untaint + } else { + $result{'logfile'} = "/var/log/postgresql/postgresql-$v-$c.log"; + } + + return %result; +} + + +=head2 validate_cluster_owner + + Checks if the owner of a cluster is valid, and the owner of the config matches + the owner of the data directory. + + Arguments: cluster_info hash reference + +=cut + +sub validate_cluster_owner($) { + my $info = shift; + + unless ($info->{pgdata}) { + error "Cluster data directory is unknown"; + } + unless (-d $info->{pgdata}) { + error "$info->{pgdata} is not accessible or does not exist"; + } + unless (defined $info->{owneruid}) { + error "Could not determine owner of $info->{pgdata}"; + } + if ($info->{owneruid} == 0) { + error "Data directory $info->{pgdata} must not be owned by root"; + } + unless (getpwuid $info->{owneruid}) { + error "The cluster is owned by user id $info->{owneruid} which does not exist"; + } + unless (getgrgid $info->{ownergid}) { + error "The cluster is owned by group id $info->{ownergid} which does not exist"; + } + # owneruid and configuid need to match, unless configuid is root + if (($< == 0 or $> == 0) and $info->{configuid} != 0 and + $info->{configuid} != $info->{owneruid}) { + my $configowner = (getpwuid $info->{configuid})[0] || "(unknown)"; + my $dataowner = (getpwuid $info->{owneruid})[0]; + error "Config owner ($configowner:$info->{configuid}) and data owner ($dataowner:$info->{owneruid}) do not match, and config owner is not root"; + } +} + + +=head2 get_versions + + Return an array of all available versions (by binaries and postgresql.conf files) + + Arguments: binary to scan for (optional, defaults to postgres), maximum acceptable version (optional) + +=cut + +sub get_versions { + my $program = shift // 'postgres'; + my $max_version = shift; + my %versions = (); + + # enumerate psql versions from /usr/lib/postgresql/* (or /usr/pgsql-*) + my $dir = $binroot; + #redhat# $dir = '/usr'; + if (opendir (D, $dir)) { + my $entry; + while (defined ($entry = readdir D)) { + next if $entry eq '.' || $entry eq '..'; + my $pfx = ''; + #redhat# $pfx = "pgsql-"; + my $version; + ($version) = $entry =~ /^$pfx(\d+\.?\d+)$/; # untaint + next if ($max_version and $version > $max_version); + $versions{$version} = 1 if $version and get_program_path ($program, $version); + } + closedir D; + } + + # enumerate server versions from /etc/postgresql/* + if ($program eq 'postgres' and opendir (D, $confroot)) { + my $v; + while (defined ($v = readdir D)) { + next if $v eq '.' || $v eq '..'; + ($v) = $v =~ /^(\d+\.?\d+)$/; # untaint + next unless ($v); + next if ($max_version and $v > $max_version); + + if (opendir (C, "$confroot/$v")) { + my $c; + while (defined ($c = readdir C)) { + if (-e "$confroot/$v/$c/postgresql.conf") { + $versions{$v} = 1; + last; + } + } + closedir C; + } + } + closedir D; + } + + return sort { $a <=> $b } keys %versions; +} + + +=head2 get_newest_version + + Return the newest available version + + Arguments: binary to scan for (optional), maximum acceptable version (optional) + +=cut + +sub get_newest_version { + my $program = shift; + my $max_version = shift; + my @versions = get_versions($program, $max_version); + return undef unless (@versions); + return $versions[-1]; +} + +=head2 version_exists + + Check whether a version exists + +=cut + +sub version_exists { + my ($version) = @_; + return get_program_path ('psql', $version); +} + + +=head2 get_version_clusters + + Return an array of all available clusters of given version + + Arguments: + +=cut + +sub get_version_clusters { + my $vdir = $confroot.'/'.$_[0].'/'; + my @clusters = (); + if (opendir (D, $vdir)) { + my $entry; + while (defined ($entry = readdir D)) { + next if $entry eq '.' || $entry eq '..'; + ($entry) = $entry =~ /^(.*)$/; # untaint + my $conf = "$vdir$entry/postgresql.conf"; + if (-e $conf or -l $conf) { # existing file, or dead symlink + push @clusters, $entry; + } + } + closedir D; + } + return sort @clusters; +} + + +=head2 cluster_exists + + Check if a cluster exists. + + Arguments: + +=cut + +sub cluster_exists { + for my $c (get_version_clusters $_[0]) { + return 1 if $c eq $_[1]; + } + return 0; +} + + +=head2 next_free_port + + Return the next free PostgreSQL port. + +=cut + +sub next_free_port { + # create list of already used ports + my %ports; + for my $v (get_versions) { + for my $c (get_version_clusters $v) { + $ports{ get_cluster_port ($v, $c) } = 1; + } + } + + my $port; + for ($port = $defaultport; $port < 65536; ++$port) { + # port in use by existing cluster + next if (exists $ports{$port}); + + # IPv4 port in use + my ($have_ip4, $have_ip6); + if (socket (SOCK, PF_INET, SOCK_STREAM, getprotobyname('tcp'))) { + $have_ip4 = 1; + setsockopt(SOCK, Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1) or error "setsockopt: $!"; + my $res4 = bind (SOCK, sockaddr_in($port, INADDR_ANY)) and listen (SOCK, 0); + my $err = $!; + close SOCK; + next unless ($res4); + } + + # IPv6 port in use + if (exists $Socket::{"IN6ADDR_ANY"}) { + if (socket (SOCK, PF_INET6, SOCK_STREAM, getprotobyname('tcp'))) { + $have_ip6 = 1; + setsockopt(SOCK, Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1) or error "setsockopt: $!"; + my $res6 = bind (SOCK, sockaddr_in6($port, Socket::IN6ADDR_ANY)) and listen (SOCK, 0); + my $err = $!; + close SOCK; + next unless ($res6); + } + } + + unless ($have_ip4 or $have_ip6) { + # require at least one protocol to work (PostgreSQL needs it anyway + # for the stats collector) + die "could not create socket: $!"; + } + + close SOCK; + # return port if it is available on all supported protocols + return $port; + } + + die "no free port found"; +} + + +=head2 user_cluster_map + + Return the PostgreSQL version, cluster, and database to connect to. + + Version is always set (defaulting to the version of the default port + if no matching entry is found, or finally to the latest installed version + if there are no clusters at all), cluster and database may be 'undef'. + If only one cluster exists, and no matching entry is found in the map files, + that cluster is returned. + +=cut + +sub user_cluster_map { + my ($user, $pwd, $uid, $gid) = getpwuid $>; + my $group = (getgrgid $gid)[0]; + + # check per-user configuration file + my $home = $ENV{"HOME"} || (getpwuid $>)[7]; + my $homemapfile = $home . '/.postgresqlrc'; + if (open MAP, $homemapfile) { + while () { + s/#.*//; + next if /^\s*$/; + my ($v,$c,$db) = split; + if (!version_exists $v) { + print "Warning: $homemapfile line $.: version $v does not exist\n"; + next; + } + if (!cluster_exists $v, $c and $c !~ /^(\S+):(\d*)$/) { + print "Warning: $homemapfile line $.: cluster $v/$c does not exist\n"; + next; + } + if ($db) { + close MAP; + return ($v, $c, ($db eq "*") ? undef : $db); + } else { + print "Warning: ignoring invalid line $. in $homemapfile\n"; + next; + } + } + close MAP; + } + + # check global map file + if (open MAP, $mapfile) { + while () { + s/#.*//; + next if /^\s*$/; + my ($u,$g,$v,$c,$db) = split; + if (!$db) { + print "Warning: ignoring invalid line $. in $mapfile\n"; + next; + } + if (!version_exists $v) { + print "Warning: $mapfile line $.: version $v does not exist\n"; + next; + } + if (!cluster_exists $v, $c and $c !~ /^(\S+):(\d*)$/) { + print "Warning: $mapfile line $.: cluster $v/$c does not exist\n"; + next; + } + if (($u eq "*" || $u eq $user) && ($g eq "*" || $g eq $group)) { + close MAP; + return ($v,$c, ($db eq "*") ? undef : $db); + } + } + close MAP; + } + + # if only one cluster exists, use that + my $count = 0; + my ($last_version, $last_cluster, $defaultport_version, $defaultport_cluster); + for my $v (get_versions) { + for my $c (get_version_clusters $v) { + my $port = get_cluster_port ($v, $c); + $last_version = $v; + $last_cluster = $c; + if ($port == $defaultport) { + $defaultport_version = $v; + $defaultport_cluster = $c; + } + ++$count; + } + } + return ($last_version, $last_cluster, undef) if $count == 1; + + if ($count == 0) { + # if there are no local clusters, use latest clients for accessing + # network clusters + return (get_newest_version('psql'), undef, undef); + } + + # more than one cluster exists, return cluster at default port + return ($defaultport_version, $defaultport_cluster, undef); +} + + +=head2 install_file + + Copy a file to a destination and setup permissions + + Arguments: + +=cut + +sub install_file { + my ($source, $dest, $uid, $gid, $perm) = @_; + + if (system 'install', '-o', $uid, '-g', $gid, '-m', $perm, $source, $dest) { + error "install_file: could not install $source to $dest"; + } +} + + +=head2 change_ugid + + Change effective and real user and group id. Also activates all auxiliary + groups the user is in. Exits with an error message if user/group ID cannot + be changed. + + Arguments: + +=cut + +sub change_ugid { + my ($uid, $gid) = @_; + + # auxiliary groups + my $uname = (getpwuid $uid)[0]; + prepare_exec; + my $groups = "$gid " . `/usr/bin/id -G $uname`; + restore_exec; + + $) = $groups; + $( = $gid; + $> = $< = $uid; + error 'Could not change user id' if $< != $uid; + error 'Could not change group id' if $( != $gid; +} + + +=head2 system_or_error + + Run a command and error out if it exits with a non-zero status. + + Arguments: + +=cut + +sub system_or_error { + my $ret = system @_; + if ($ret) { + my $message = "@_ failed with exit code $ret"; + $message .= ": $!" if ($!); + error $message; + } +} + + +=head2 get_db_encoding + + Return the encoding of a particular database in a cluster. + + This requires access privileges to that database, so this + function should be called as the cluster owner. + + Arguments: + Returns: Encoding or undef if it cannot be determined. + +=cut + +sub get_db_encoding { + my ($version, $cluster, $db) = @_; + my $port = get_cluster_port $version, $cluster; + my $socketdir = get_cluster_socketdir $version, $cluster; + my $psql = get_program_path 'psql', $version; + return undef unless ($port && $socketdir && $psql); + + # try to swich to cluster owner + prepare_exec 'LC_ALL'; + $ENV{'LC_ALL'} = 'C'; + my $orig_euid = $>; + $> = (stat (cluster_data_directory $version, $cluster))[4]; + open PSQL, '-|', $psql, '-h', $socketdir, '-p', $port, '-AXtc', + 'select getdatabaseencoding()', $db or + die "Internal error: could not call $psql to determine db encoding: $!"; + my $out = ; + close PSQL; + $> = $orig_euid; + restore_exec; + return undef if $?; + chomp $out; + ($out) = $out =~ /^([\w.-]+)$/; # untaint + return $out; +} + + +=head2 get_db_locales + + Return locale of a particular database in a cluster. This requires access + privileges to that database, so this function should be called as the cluster + owner. (For versions >= 8.4; for older versions use get_cluster_locales()). + + Arguments: + Returns: (LC_CTYPE, LC_COLLATE) or (undef,undef) if it cannot be determined. + PG15 adds locale provider and icu locale to the returned values + PG16 adds icu rules + +=cut + +sub get_db_locales { + my ($version, $cluster, $db) = @_; + my $port = get_cluster_port $version, $cluster; + my $socketdir = get_cluster_socketdir $version, $cluster; + my $psql = get_program_path 'psql', $version; + return undef unless ($port && $socketdir && $psql); + my ($ctype, $collate, $locale_provider, $icu_locale, $icu_rules); + + # try to switch to cluster owner + prepare_exec 'LC_ALL'; + $ENV{'LC_ALL'} = 'C'; + my $orig_euid = $>; + $> = (stat (cluster_data_directory $version, $cluster))[4]; + + open PSQL, '-|', $psql, '-h', $socketdir, '-p', $port, '-AXtc', + "SELECT datctype, datcollate FROM pg_database where datname = current_database()", $db or + die "Internal error: could not call $psql to determine datctype and datcollate: $!"; + my $out = // error 'could not determine datctype and datcollate'; + close PSQL; + ($out) = $out =~ /^(.*)$/; # untaint + ($ctype, $collate) = split /\|/, $out; + + if ($version >= 15) { + open PSQL, '-|', $psql, '-h', $socketdir, '-p', $port, '-AXtc', + "SELECT CASE datlocprovider::text WHEN 'c' THEN 'libc' WHEN 'i' THEN 'icu' END, daticulocale" . + ($version >= 16 ? ", daticurules" : "") . + " FROM pg_database where datname = current_database()", $db or + die "Internal error: could not call $psql to determine datlocprovider: $!"; + $out = // error 'could not determine datlocprovider'; + close PSQL; + ($out) = $out =~ /^(.*)$/; # untaint + ($locale_provider, $icu_locale, $icu_rules) = split /\|/, $out; + } + + $> = $orig_euid; + restore_exec; + chomp $ctype; + chomp $collate; + return ($ctype, $collate, $locale_provider, $icu_locale, $icu_rules) unless $?; + return (undef, undef); +} + + +=head2 get_cluster_locales + + Return the CTYPE and COLLATE locales of a cluster. + + This needs to be called as root or as the cluster owner. + (For versions <= 8.3; for >= 8.4, use get_db_locales()). + + Arguments: + Returns: (LC_CTYPE, LC_COLLATE) or (undef,undef) if it cannot be determined. + +=cut + +sub get_cluster_locales { + my ($version, $cluster) = @_; + my ($lc_ctype, $lc_collate) = (undef, undef); + + if ($version >= '8.4') { + print STDERR "Error: get_cluster_locales() does not work for 8.4+\n"; + exit 1; + } + + my $pg_controldata = get_program_path 'pg_controldata', $version; + if (! -e $pg_controldata) { + print STDERR "Error: pg_controldata not found, please install postgresql-$version\n"; + exit 1; + } + prepare_exec ('LC_ALL', 'LANG', 'LANGUAGE'); + $ENV{'LC_ALL'} = 'C'; + my $result = open (CTRL, '-|', $pg_controldata, (cluster_data_directory $version, $cluster)); + restore_exec; + return (undef, undef) unless defined $result; + while () { + if (/^LC_CTYPE\W*(\S+)\s*$/) { + $lc_ctype = $1; + } elsif (/^LC_COLLATE\W*(\S+)\s*$/) { + $lc_collate = $1; + } + } + close CTRL; + return ($lc_ctype, $lc_collate); +} + + +=head2 get_cluster_controldata + + Return the pg_control data for a cluster + + Arguments: + Returns: hashref + +=cut + +sub get_cluster_controldata { + my ($version, $cluster) = @_; + + my $pg_controldata = get_program_path 'pg_controldata', $version; + if (! -e $pg_controldata) { + print STDERR "Error: pg_controldata not found, please install postgresql-$version\n"; + exit 1; + } + prepare_exec ('LC_ALL', 'LANG', 'LANGUAGE'); + $ENV{'LC_ALL'} = 'C'; + my $result = open (CTRL, '-|', $pg_controldata, (cluster_data_directory $version, $cluster)); + restore_exec; + return undef unless defined $result; + my $data = {}; + while () { + if (/^(.+?):\s*(.*)/) { + $data->{$1} = $2; + } else { + error "Invalid pg_controldata output: $_"; + } + } + close CTRL; + return $data; +} + + +=head2 get_cluster_databases + + Return an array with all databases of a cluster. + + This requires connection privileges to template1, so + this function should be called as the cluster owner. + + Arguments: + Returns: array of database names or undef on error. + +=cut + +sub get_cluster_databases { + my ($version, $cluster) = @_; + my $port = get_cluster_port $version, $cluster; + my $socketdir = get_cluster_socketdir $version, $cluster; + my $psql = get_program_path 'psql', $version; + return undef unless ($port && $socketdir && $psql); + + # try to swich to cluster owner + prepare_exec 'LC_ALL'; + $ENV{'LC_ALL'} = 'C'; + my $orig_euid = $>; + $> = (stat (cluster_data_directory $version, $cluster))[4]; + + my @dbs; + my @fields; + if (open PSQL, '-|', $psql, '-h', $socketdir, '-p', $port, '-AXtl') { + while () { + chomp; + @fields = split '\|'; + next if $#fields < 2; # remove access privs which get line broken + push (@dbs, $fields[0]); + } + close PSQL; + } + + $> = $orig_euid; + restore_exec; + + return $? ? undef : @dbs; +} + + +=head2 get_file_device + + Return the device name a file is stored at. + + Arguments: + Returns: device name, or '' if it cannot be determined. + +=cut + +sub get_file_device { + my $dev = ''; + prepare_exec; + my $pid = open3(\*CHLD_IN, \*CHLD_OUT, \*CHLD_ERR, '/bin/df', $_[0]); + waitpid $pid, 0; # we simply ignore exit code and stderr + while () { + if (/^\/dev/) { + $dev = (split)[0]; + } + } + restore_exec; + close CHLD_IN; + close CHLD_OUT; + close CHLD_ERR; + return $dev; +} + + +=head2 parse_hba_line + + Parse a single pg_hba.conf line. + + Arguments: + Returns: Hash reference (or only line and type==undef for invalid lines) + +=over 4 + +=item * + +line -> the verbatim pg_hba line + +=item * + +type -> comment, local, host, hostssl, hostnossl, undef + +=item * + +db -> database name + +=item * + +user -> user name + +=item * + +method -> trust, reject, md5, crypt, password, krb5, ident, pam + +=item * + +ip -> ip address + +=item * + +mask -> network mask (either a single number as number of bits, or bit mask) + +=back + +=cut + +sub parse_hba_line { + my $l = $_[0]; + chomp $l; + + # comment line? + return { 'type' => 'comment', 'line' => $l } if ($l =~ /^\s*($|#)/); + + my $res = { 'line' => $l }; + my @tok = split /\s+/, $l; + goto error if $#tok < 3; + + $$res{'type'} = shift @tok; + $$res{'db'} = shift @tok; + $$res{'user'} = shift @tok; + + # local connection? + if ($$res{'type'} eq 'local') { + goto error if $#tok > 1; + goto error unless valid_hba_method($tok[0]); + $$res{'method'} = join (' ', @tok); + return $res; + } + + # host connection? + if ($$res{'type'} =~ /^host((no)?ssl)?$/) { + my ($i, $c) = split '/', (shift @tok); + goto error unless $i; + $$res{'ip'} = $i; + + # CIDR mask given? + if (defined $c) { + goto error if $c !~ /^(\d+)$/; + $$res{'mask'} = $c; + } else { + $$res{'mask'} = shift @tok; + } + + goto error if $#tok > 1; + goto error unless valid_hba_method($tok[0]); + $$res{'method'} = join (' ', @tok); + return $res; + } + +error: + $$res{'type'} = undef; + return $res; +} + + +=head2 read_pg_hba + + Parse given pg_hba.conf file. + + Arguments: + Returns: Array with hash refs; for hash contents, see parse_hba_line(). + +=cut + +sub read_pg_hba { + open HBA, $_[0] or return undef; + my @hba; + while () { + my $r = parse_hba_line $_; + push @hba, $r; + } + close HBA; + return @hba; +} + + +=head2 valid_hba_method + + Check if hba method is known + + Argument: hba method + Returns: True if method is valid + +=cut + +sub valid_hba_method { + my $method = $_[0]; + + my %valid_methods = qw/trust 1 reject 1 md5 1 crypt 1 password 1 krb5 1 ident 1 pam 1/; + + return exists($valid_methods{$method}); +} + +1; diff --git a/README.md b/README.md new file mode 100644 index 0000000..5e48b28 --- /dev/null +++ b/README.md @@ -0,0 +1,114 @@ +Multi-Version/Multi-Cluster PostgreSQL architecture +=================================================== +2004, Oliver Elphick, Martin Pitt + +Solving a problem +----------------- + +When a new major version of PostgreSQL is released, it is necessary to dump and +reload the database. The old software must be used for the dump, and the new +software for the reload. + +This was a major problem for RedHat and Debian, because a dump and reload was +not required by every upgrade and by the time the need for a dump is realised, +the old software might have been deleted. Debian had certain rather unreliable +procedures to save the old software and use it to do a dump, but these +procedures often went wrong. RedHat's installation environment is so rigid that +it is not practicable for the RedHat packages to attempt an automatic upgrade. +Debian offered a debconf choice for whether to attempt automatic upgrading; if +it failed or was not allowed, a manual upgrade had to be done, either from a +pre-existing dump or by manual invocation of the postgresql-dump script. + +It is possible to run different versions of PostgreSQL simultaneously, and +indeed to run the same version on separate database clusters simultaneously. To +do so, each postgres instance must listen on a different port, so each client +must specify the correct port. By having two separate versions of the +PostgreSQL packages installed simultaneously, it is simple to do database +upgrades by dumping from the old version and uploading to the new. The +PostgreSQL client wrapper is designed to permit this. + +General Architecture idea +------------------------- + +The Debian packaging has been changed to create a new package for each major +version. The criterion for creating a new package is that initdb is required +when upgrading from the previous version. Thus, there are now source packages +`postgresql-8.1` and `postgresql-8.3` (and similarly for all the binary +packages). + +The legacy postgresql and the other existing binary package names have become +dummy packages depending on one of the versioned equivalents. Their only +purpose is now to ensure a smooth upgrade and to register the existing database +cluster to the new architecture. These packages will be removed from the +archive as soon as the next Debian release after Sarge (Etch) is released. + +Each versioned package installs into `/usr/lib/postgresql/version`. In order +to allow users easily to select the right version and cluster when working, the +`postgresql-common` package provides the `pg_wrapper` program, which reads the +per-user and system wide configuration file and forks the correct executable +with the correct library versions according to those preferences. `/usr/bin` +provides executables soft-linked to `pg_wrapper`. + +This architecture also allows separate database clusters to be maintained for +the use of different groups of users; these clusters need not all be of the +same major version. This allows much greater flexibility for those people who +need to make application software changes consequent on a PostgreSQL upgrade. + +Detailed structure +------------------ + +### Configuration hierarchy + +* `/etc/postgresql-common/user_clusters`: maps users against clusters and + default databases + +* `$HOME/.postgresqlrc`: per-user preferences for default version/cluster and + database; overrides `/etc/postgresql-common/user_clusters` + +* `/etc/postgresql/version/clustername`: cluster-specific configuration files: + + * `postgresql.conf`, `pg_hba.conf`, `pg_ident.conf` + * optionally `start.conf`: startup mode of the cluster: `auto` (start/stop in + init script), `manual` (do not start/stop in init script, but manual + control with `pg_ctlcluster` is possible), `disabled` (`pg_ctlcluster` + is not allowed). + * optionally `pg_ctl.conf`: options to be passed to `pg_ctl`. + * optionally a symbolic link `log` which points to the postgres log file. + Defaults to `/var/log/postgresql/postgresql-version-cluster.conf`. + Explicitly setting `log_directory` and/or `log_filename` in + `postgresql.conf` overrides this. + +### Per-version files and programs + +* `/usr/lib/postgresql/version` +* `/usr/share/postgresql/version` +* `/usr/share/doc/postgresql/postgresql-doc-version`: +version specific program and data files + +### Common programs + +* `/usr/share/postgresql-common/pg_wrapper`: environment chooser and program selector +* `/usr/bin/program`: symbolic links to pg_wrapper, for all client programs +* `/usr/bin/pg_lsclusters`: list all available clusters with their status and configuration +* `/usr/bin/pg_createcluster: wrapper for `initdb`, sets up the necessary configuration structure +* `/usr/bin/pg_ctlcluster`: wrapper for `pg_ctl`, control the cluster postgres server +* `/usr/bin/pg_upgradecluster`: upgrade a cluster to a newer major version +* `/usr/bin/pg_dropcluster`: remove a cluster and its configuration + +### /etc/init.d/postgresql + +This script handles the postgres server processes for each version and all +their clusters. However, most of the actual work is done by the new +`pg_ctlcluster` program. + +### pg_upgradecluster + +This program replaces postgresql-dump (a Debian specific program). + +It is used to migrate a cluster from one major version to another. + +Usage: `pg_upgradecluster [-v newversion] version name [data_dir]` + +`-v`: specifies the version to upgrade to; defaults to the newest available version. + + -- The Debian PostgreSQL maintainers diff --git a/TODO b/TODO new file mode 100644 index 0000000..140f6f0 --- /dev/null +++ b/TODO @@ -0,0 +1,39 @@ +postgresql TODO +=============== + +Bugs: +- pg_createcluster with existing cluster: respect symlinks to shared + postgresql.conf files, never remove symlinks in already existing + cluster dirs +- Clean up at purging if /etc/ is there without /var, or vice versa + +Transition bugs: + +Missing selftest: +- --force option for pg_ctlcluster + +Important features: + +Wishlist: +- Add pg_conf to change startup and possibly other things +- add program for web applications, which configure pg_hba.conf: + + pg_hba add|remove|test [options] yourwebappdb yourwebappuser + + Options: + + --cluster: self-explanatory, defaults to default cluster + --ip: IP and netmask for host socket; if not given, defaults to Unix + socket (local) + --method: defaults to "md5" for TCP connections, and "ident sameuser" for + Unix socket connections + --force-ssl: If given, create a "hostssl" entry, otherwise a "host" + entry + + For remove, only --cluster is allowed; it will remove all hba + entries that refer to the given db/user pair. test checks whether the + given connection is allowed; if so, it exits with 0, otherwise it prints the + required pg_hba.conf line to stdout and exits with 1. If pg_hba.conf has a + scrambled format that cannot be parsed by pg_*_hba, the scripts exit with 2. + + Add libnet-cidr-perl dependency! diff --git a/cleanpg b/cleanpg new file mode 100755 index 0000000..22ef371 --- /dev/null +++ b/cleanpg @@ -0,0 +1,24 @@ +#!/bin/sh + +# Clean all traces of PostgreSQL data/configuration from the current system. +# Use with caution, for development/testing only! (This is not installed into +# packages.) +# +# (C) 2008-2009 Martin Pitt +# (C) 2018 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +set -ux + +/etc/init.d/postgresql stop +killall postgres +rm -rf /etc/postgresql/* /var/lib/postgresql/* /var/log/postgresql/* /tmp/postgresql-testsuite/ /var/run/postgresql/* diff --git a/createcluster.conf b/createcluster.conf new file mode 100644 index 0000000..5e66693 --- /dev/null +++ b/createcluster.conf @@ -0,0 +1,41 @@ +# Default values for pg_createcluster(8) +# Occurrences of '%v' are replaced by the major version number, +# and '%c' by the cluster name. Use '%%' for a literal '%'. + +# Create a "main" cluster when a new postgresql-x.y server package is installed +#create_main_cluster = true + +# Default start.conf value, must be one of "auto", "manual", and "disabled". +# See pg_createcluster(8) for more documentation. +#start_conf = 'auto' + +# Default data directory. +#data_directory = '/var/lib/postgresql/%v/%c' + +# Default directory for transaction logs +# Unset by default, i.e. transaction logs remain in the data directory. +#waldir = '/var/lib/postgresql/wal/%v/%c/pg_wal' + +# Options to pass to initdb. +#initdb_options = '' + +# The following options are copied into the new cluster's postgresql.conf: + +# Enable SSL by default (using the "snakeoil" certificates installed by the +# ssl-cert package, unless configured otherwise here) +ssl = on + +# Show cluster name in process title +cluster_name = '%v/%c' + +# Put stats_temp_directory on tmpfs (PG <= 14) +stats_temp_directory = '/var/run/postgresql/%v-%c.pg_stat_tmp' + +# Add prefix to log lines +log_line_prefix = '%%m [%%p] %%q%%u@%%d ' + +# Add "include_dir" in postgresql.conf +add_include_dir = 'conf.d' + +# Directory for additional createcluster config +include_dir '/etc/postgresql-common/createcluster.d' diff --git a/debhelper/Debian/Debhelper/Buildsystem/pgxs.pm b/debhelper/Debian/Debhelper/Buildsystem/pgxs.pm new file mode 100644 index 0000000..f1270ea --- /dev/null +++ b/debhelper/Debian/Debhelper/Buildsystem/pgxs.pm @@ -0,0 +1,58 @@ +# A debhelper build system class for building PostgreSQL extension modules using PGXS +# +# Per PostgreSQL major version, a `build-$version` subdirectory is created. +# +# Copyright: © 2020 Christoph Berg +# License: GPL-2+ + +package Debian::Debhelper::Buildsystem::pgxs; + +use strict; +use warnings; +use parent qw(Debian::Debhelper::Buildsystem); +use Cwd; +use Debian::Debhelper::Dh_Lib; +use Debian::Debhelper::pgxs; + +sub DESCRIPTION { + "PGXS (PostgreSQL extensions), building in subdirectory per PostgreSQL version" +} + +sub check_auto_buildable { + my $this=shift; + unless (-e $this->get_sourcepath("debian/pgversions")) { + error("debian/pgversions is required to build with PGXS"); + } + return (-e $this->get_sourcepath("Makefile")) ? 1 : 0; +} + +sub new { + my $class=shift; + my $this=$class->SUPER::new(@_); + $this->enforce_in_source_building(); + return $this; +} + +sub build { + my $this=shift; + $this->doit_in_sourcedir(qw(pg_buildext build build-%v)); +} + +sub install { + my $this=shift; + my $pattern = package_pattern(); + $this->doit_in_sourcedir(qw(pg_buildext install build-%v), $pattern); +} + +sub test { + my $this=shift; + verbose_print("Postponing tests to install stage"); +} + +sub clean { + my $this=shift; + my $pattern = package_pattern(); + $this->doit_in_sourcedir(qw(pg_buildext clean build-%v), $pattern); +} + +1; diff --git a/debhelper/Debian/Debhelper/Buildsystem/pgxs_loop.pm b/debhelper/Debian/Debhelper/Buildsystem/pgxs_loop.pm new file mode 100644 index 0000000..716bcbf --- /dev/null +++ b/debhelper/Debian/Debhelper/Buildsystem/pgxs_loop.pm @@ -0,0 +1,33 @@ +# A debhelper build system class for building PostgreSQL extension modules using PGXS +# +# For packages not supporting building in subdirectories, the pgxs_loop variant builds +# for each PostgreSQL major version in turn in the top-level directory. +# +# Copyright: © 2020 Christoph Berg +# License: GPL-2+ + +package Debian::Debhelper::Buildsystem::pgxs_loop; + +use strict; +use warnings; +use parent qw(Debian::Debhelper::Buildsystem::pgxs); +use Cwd; +use Debian::Debhelper::Dh_Lib; +use Debian::Debhelper::pgxs; + +sub DESCRIPTION { + "PGXS (PostgreSQL extensions), building for each PostgreSQL version in top level directory" +} + +sub build { + my $this=shift; + verbose_print("Postponing build to install stage; if this package supports out-of-tree builds, replace --buildsystem=pgxs_loop by --buildsystem=pgxs to build in the build stage"); +} + +sub install { + my $this=shift; + my $pattern = package_pattern(); + $this->doit_in_sourcedir(qw(pg_buildext loop), $pattern); +} + +1; diff --git a/debhelper/Debian/Debhelper/Sequence/pgxs.pm b/debhelper/Debian/Debhelper/Sequence/pgxs.pm new file mode 100644 index 0000000..45a1ff1 --- /dev/null +++ b/debhelper/Debian/Debhelper/Sequence/pgxs.pm @@ -0,0 +1,22 @@ +#!/usr/bin/perl + +use warnings; +use strict; +use Debian::Debhelper::Dh_Lib; + +# check if debian/control needs updating from debian/control.in +insert_after("dh_clean", "pg_buildext"); +add_command_options("pg_buildext", "checkcontrol"); + +# use PGXS for clean, build, and install +add_command_options("dh_auto_clean", "--buildsystem=pgxs"); +add_command_options("dh_auto_build", "--buildsystem=pgxs"); +add_command_options("dh_auto_install", "--buildsystem=pgxs"); + +# move tests from dh_auto_test to dh_pgxs_test +remove_command("dh_auto_test"); +if (! get_buildoption("nocheck")) { + insert_after("dh_link", "dh_pgxs_test"); +} + +1; diff --git a/debhelper/Debian/Debhelper/Sequence/pgxs_loop.pm b/debhelper/Debian/Debhelper/Sequence/pgxs_loop.pm new file mode 100644 index 0000000..314085b --- /dev/null +++ b/debhelper/Debian/Debhelper/Sequence/pgxs_loop.pm @@ -0,0 +1,23 @@ +#!/usr/bin/perl + +use warnings; +use strict; +use Debian::Debhelper::Dh_Lib; + +# check if debian/control needs updating from debian/control.in +insert_after("dh_clean", "pg_buildext"); +add_command_options("pg_buildext", "checkcontrol"); + +# use PGXS for clean, build, and install +add_command_options("dh_auto_clean", "--buildsystem=pgxs_loop"); +add_command_options("dh_auto_build", "--buildsystem=pgxs_loop"); +add_command_options("dh_auto_install", "--buildsystem=pgxs_loop"); + +# move tests from dh_auto_test to dh_pgxs_test +remove_command("dh_auto_test"); +if (! get_buildoption("nocheck")) { + insert_after("dh_link", "dh_pgxs_test"); + add_command_options("dh_pgxs_test", "loop"); +} + +1; diff --git a/debhelper/Debian/Debhelper/pgxs.pm b/debhelper/Debian/Debhelper/pgxs.pm new file mode 100644 index 0000000..e3d86b2 --- /dev/null +++ b/debhelper/Debian/Debhelper/pgxs.pm @@ -0,0 +1,38 @@ +# A debhelper build system class for building PostgreSQL extension modules using PGXS +# +# Copyright: © 2020 Christoph Berg +# License: GPL-2+ + +package Debian::Debhelper::pgxs; + +use strict; +use warnings; +use Exporter 'import'; +our @EXPORT = qw(package_pattern); + +=head1 package_pattern() + +From C, look for the package name containing the +B placeholder, and return it in the format suitable for passing to +B, i.e. with B replaced by B<%v>. + +For B it will return B. + +Errors out if more than one package with the B placeholder is found. + +=cut + +sub package_pattern () { + open F, "debian/control.in" or die "debian/control.in: $!"; + my $pattern; + while () { + if (/^Package: (.*)PGVERSION(.*)/) { + die "More than one Package with PGVERSION placeholder found in debian/control.in, cannot build with dh --buildsystem=pgxs. Use pg_buildext manually." if ($pattern); + $pattern = "$1%v$2"; + } + } + close F; + return $pattern; +} + +1; diff --git a/debhelper/dh_pgxs_test b/debhelper/dh_pgxs_test new file mode 100755 index 0000000..c12bacd --- /dev/null +++ b/debhelper/dh_pgxs_test @@ -0,0 +1,11 @@ +#!/usr/bin/perl + +use warnings; +use strict; +use Debian::Debhelper::Dh_Lib; +use Debian::Debhelper::pgxs; + +my $target = (@ARGV and $ARGV[0] eq 'loop') ? "." : "build-%v"; +my $pattern = package_pattern(); + +print_and_doit(qw(pg_buildext installcheck .), $target, $pattern); diff --git a/debhelper/dh_pgxs_test.pod b/debhelper/dh_pgxs_test.pod new file mode 100644 index 0000000..5b24e4b --- /dev/null +++ b/debhelper/dh_pgxs_test.pod @@ -0,0 +1,44 @@ +=head1 NAME + +dh_pgxs_test - Run testsuite during a PGXS PostgreSQL extension build + +=head1 SYNOPSIS + +B [B] + +=head1 DESCRIPTION + +B extensions need to be installed before they can be tested and +hence the usual B way of invoking tests from dh_auto_test(1) does +not work. + +B is a dh(1) sequence point created by the B and +B B extensions that is executed after dh_auto_install(1). +It calls B after a B extension module has +been built and installed into the CI directory. + +Users wishing to change the action called by B should call +B or similar commands. + + override_dh_pgxs_test: + echo "CREATE EXTENSION foo" | pg_buildext psql . . postgresql-%v-foo + +=head1 OPTIONS + +=over 4 + +=item B + +B builds packages in C subdirectories. The B +options corresponds to B and builds in the top-level +directory. + +=back + +=head1 SEE ALSO + +debhelper(7), dh(1), dh_make_pgxs(1), pg_buildext(1). + +=head1 AUTHOR + +Christoph Berg Lmyon@debian.orgE> diff --git a/dh_make_pgxs/debian/control.in b/dh_make_pgxs/debian/control.in new file mode 100644 index 0000000..090ac72 --- /dev/null +++ b/dh_make_pgxs/debian/control.in @@ -0,0 +1,23 @@ +Source: @SOURCE@ +Section: database +Priority: optional +Maintainer: Debian PostgreSQL Maintainers +Uploaders: + @MAINTAINER_NAME@ <@DEBEMAIL@>, +Build-Depends: + debhelper-compat (= @COMPAT@), + postgresql-all (>= 217~), +Standards-Version: @STANDARDS_VERSION@ +Rules-Requires-Root: no +Homepage: @URL@ +Vcs-Browser: https://salsa.debian.org/postgresql/@SOURCE@ +Vcs-Git: https://salsa.debian.org/postgresql/@SOURCE@.git + +Package: postgresql-PGVERSION-@EXTNAME@ +Architecture: any +Depends: + ${misc:Depends}, + ${postgresql:Depends}, + ${shlibs:Depends}, +Description: FIXME PostgreSQL extension @SOURCE@ + FIXME long description here diff --git a/dh_make_pgxs/debian/copyright b/dh_make_pgxs/debian/copyright new file mode 100644 index 0000000..60ca5f0 --- /dev/null +++ b/dh_make_pgxs/debian/copyright @@ -0,0 +1,24 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: @NAME@ +Source: @URL@ + +Files: * +Copyright: Portions Copyright (c) 1996-@YEAR@, PostgreSQL Global Development Group + Portions Copyright (c) 1994, The Regents of the University of California +License: PostgreSQL + Permission to use, copy, modify, and distribute this software and its + documentation for any purpose, without fee, and without a written agreement + is hereby granted, provided that the above copyright notice and this + paragraph and the following two paragraphs appear in all copies. + . + IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR + DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS + DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + . + THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS + ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO + PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. diff --git a/dh_make_pgxs/debian/gitlab-ci.yml b/dh_make_pgxs/debian/gitlab-ci.yml new file mode 100644 index 0000000..67e4816 --- /dev/null +++ b/dh_make_pgxs/debian/gitlab-ci.yml @@ -0,0 +1 @@ +include: https://salsa.debian.org/postgresql/postgresql-common/raw/master/gitlab/gitlab-ci.yml diff --git a/dh_make_pgxs/debian/pgversions b/dh_make_pgxs/debian/pgversions new file mode 100644 index 0000000..0702cb5 --- /dev/null +++ b/dh_make_pgxs/debian/pgversions @@ -0,0 +1 @@ +all diff --git a/dh_make_pgxs/debian/rules b/dh_make_pgxs/debian/rules new file mode 100755 index 0000000..5c0ff40 --- /dev/null +++ b/dh_make_pgxs/debian/rules @@ -0,0 +1,36 @@ +#!/usr/bin/make -f + +%: + dh $@ --with pgxs + +override_dh_installdocs: + dh_installdocs --all README.* + +# if the package does not support building from subdirectories, use +# `--with pgxs_loop` above. +# +# change the way tests are run: +# override_dh_pgxs_test: +# +pg_buildext installcheck . . postgresql-%v-@EXTNAME@ + +# classic `pg_buildext` interface: + +#include /usr/share/postgresql-common/pgxs_debian_control.mk +# +#override_dh_auto_build: +# +pg_buildext build build-%v +# +#override_dh_auto_test: +# # nothing to do here, see debian/tests/* instead +# +#override_dh_auto_install: +# +pg_buildext install build-%v postgresql-%v-@EXTNAME@ +# +#override_dh_installdocs: +# dh_installdocs --all README.* +# +#override_dh_auto_clean: +# +pg_buildext clean build-%v +# +#%: +# dh $@ diff --git a/dh_make_pgxs/debian/source/format b/dh_make_pgxs/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/dh_make_pgxs/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/dh_make_pgxs/debian/tests/control b/dh_make_pgxs/debian/tests/control new file mode 100644 index 0000000..74b0464 --- /dev/null +++ b/dh_make_pgxs/debian/tests/control @@ -0,0 +1,5 @@ +Depends: + make, + @, +Tests: installcheck +Restrictions: allow-stderr diff --git a/dh_make_pgxs/debian/tests/installcheck b/dh_make_pgxs/debian/tests/installcheck new file mode 100755 index 0000000..5a20e78 --- /dev/null +++ b/dh_make_pgxs/debian/tests/installcheck @@ -0,0 +1,3 @@ +#!/bin/sh + +pg_buildext installcheck diff --git a/dh_make_pgxs/debian/watch b/dh_make_pgxs/debian/watch new file mode 100644 index 0000000..acfb2a8 --- /dev/null +++ b/dh_make_pgxs/debian/watch @@ -0,0 +1,2 @@ +version=4 +@URL@/tags .*/(.*).tar.gz diff --git a/dh_make_pgxs/dh_make_pgxs b/dh_make_pgxs/dh_make_pgxs new file mode 100755 index 0000000..6c59b8a --- /dev/null +++ b/dh_make_pgxs/dh_make_pgxs @@ -0,0 +1,136 @@ +#!/bin/bash + +# (C) 2015-2020 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +set -eu + +# basic variables + +template_dir="/usr/share/postgresql-common/dh_make_pgxs/debian" + +DIRECTORY="$(basename $PWD)" +NAME="${DIRECTORY%-*}" # Upstream name +VERSION="${DIRECTORY##*-}" +URL="https://FIXME/$NAME" + +# options + +while getopts "fh:n:v:" opt ; do + case $opt in + f) FORCE=yes ;; + h) URL="$OPTARG" ;; + n) NAME="$OPTARG" ;; + v) VERSION="$OPTARG" ;; + *) exit 5 ;; + esac +done +shift $((OPTIND - 1)) # shift away args + +# more variables + +SOURCE="${NAME//_/-}" # Debian name +EXTNAME="$(echo $SOURCE | sed -e 's/^\(postgresql\|pgsql\|pg\)-//')" # binary package name suffix +DHVERSION="$(dpkg-query --showformat '${Version}' --show debhelper)" +COMPAT="${DHVERSION%%.*}" +STANDARDS_VERSION="$(apt-cache show debian-policy | sed -n 's/Version: \(.*\)\..*/\1/p' | head -n1)" +USERNAME=${LOGNAME:-${USER:-root}} +MAINTAINER_NAME=$(getent passwd $USERNAME | cut -d : -f 5 | sed -e 's/,.*//') +: ${DEBEMAIL:=$USERNAME@localhost} +YEAR=$(date +%Y) + +echo "Upstream (-n): $NAME" +echo "Version (-v): $VERSION" +echo "Source: $SOURCE ($VERSION-1)" +echo "Binaries: postgresql-PGVERSION-$EXTNAME ($VERSION-1)" +echo "Uploader: $MAINTAINER_NAME <$DEBEMAIL>" +echo "Homepage (-h): $URL" +echo +if [ -t 0 ]; then + echo -n "Press Enter to continue, ^C to abort " + read +fi + +# install files + +install_dir () +{ + local directory="debian/$1" + #[ -z "$directory" ] && return + [ -d "$directory" ] && return + echo "Creating $directory/" + mkdir "$directory" +} + +install_template () +{ + local template="$1" + + if [ "${FORCE:-}" ] || ! [ -e debian/$template ]; then + echo "Installing debian/$template" + sed -e "s/@COMPAT@/$COMPAT/g" \ + -e "s/@EXTNAME@/$EXTNAME/g" \ + -e "s/@NAME@/$NAME/g" \ + -e "s/@STANDARDS_VERSION@/$STANDARDS_VERSION/g" \ + -e "s/@SOURCE@/$SOURCE/g" \ + -e "s/@MAINTAINER_NAME@/$MAINTAINER_NAME/g" \ + -e "s/@DEBEMAIL@/$DEBEMAIL/g" \ + -e "s;@URL@;$URL;g" \ + -e "s/@YEAR@/$YEAR/g" \ + "$template_dir/$template" > "debian/$template" + if [ -x $template_dir/$template ]; then + chmod +x "debian/$template" + fi + else + echo "Keeping existing debian/$template" + fi +} + +mkdir -p debian + +for template in $(find $template_dir -mindepth 1 | sort); do + case $template in + *.swp|*~) continue ;; # skip vim stuff + esac + basename=${template##$template_dir/} + if [ -d $template ]; then + install_dir "$basename" + else + install_template "$basename" + fi +done + +if [ "$COMPAT" -lt "12" ]; then + sed -i -e "s/debhelper-compat[^,]*/debhelper (>= $COMPAT)/" debian/control* + echo "$COMPAT" > debian/compat +fi + +echo "Updating debian/control from debian/control.in" +pg_buildext updatecontrol + +if [ "${FORCE:-}" ] || ! [ -e debian/changelog ]; then + rm -f debian/changelog + echo "Creating debian/changelog" + if [ -x /usr/bin/dch ]; then + dch --create --package "$SOURCE" --newversion "$VERSION-1" + else + cat > debian/changelog <<-EOT + $SOURCE ($VERSION-1) UNRELEASED; urgency=medium + + * Initial release. (Closes: #XXXXXX) + + -- $MAINTAINER_NAME <$DEBEMAIL> $(date -R) + EOT + fi +else + echo "Keeping existing debian/changelog" +fi diff --git a/dh_make_pgxs/dh_make_pgxs.pod b/dh_make_pgxs/dh_make_pgxs.pod new file mode 100644 index 0000000..8efd172 --- /dev/null +++ b/dh_make_pgxs/dh_make_pgxs.pod @@ -0,0 +1,43 @@ +=head1 NAME + +dh_make_pgxs - Create a new Debian source package for a PGXS PostgreSQL extension + +=head1 SYNOPSIS + +B [B<-f>] [B<-h> I] [B<-n> I] [B<-v> I] + +=head1 DESCRIPTION + +B creates a F directory tree for PostgreSQL extension +packages using the PGXS build system. The B tool is used for the +build process. + +=head1 OPTIONS + +=over 4 + +=item B<-f> + +Overwrite existing files. + +=item B<-h> I + +Package upstream homepage. + +=item B<-n> I + +Package name to use. Default is to extract it from the current directory's name. + +=item B<-v> I + +Package version to use. Default is to extract it from the current directory's name. + +=back + +=head1 SEE ALSO + +dh_make(1), pg_buildext(1). + +=head1 AUTHOR + +Christoph Berg Lmyon@debian.orgE> diff --git a/doc/dependencies.dia b/doc/dependencies.dia new file mode 100644 index 0000000..ba23097 Binary files /dev/null and b/doc/dependencies.dia differ diff --git a/doc/dependencies.svg b/doc/dependencies.svg new file mode 100644 index 0000000..b4fd8a4 --- /dev/null +++ b/doc/dependencies.svg @@ -0,0 +1,206 @@ + + + + + + + + + + + postgresql:all + M-A:none + + + + + + postgresql-client:all + M-A:foreign + + + + + + + + + + postgresql-NN:any + M-A:none + Provides: postgresql-contrib-NN + + + + + + postgresql-client-NN:any + M-A:foreign + + + + + + postgresql-common:all + M-A:foreign + + + + + + postgresql-client-common:all + M-A:foreign + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + libpq5:amd64 + M-A:same + + + + + + libpq-dev:amd64 + M-A:none + + + + + + + + + + postgresql-doc-NN:all + M-A:foreign + + + + + + postgresql-doc:all + M-A:foreign + + + + + + + + + + postgresql-contrib:all + M-A:none + + + + + + + + + + postgresql-server-dev-NN:any + M-A:none + + + + + + postgresql-server-dev-all:any + M-A:same + + + + + + + + + + + + postgresql-common + + + postgresql-NN + + + + + + + + + + + + + + + + + + + + + postgresql-all:all + M-A:none + + + + + + + + + + + + + + postgresql-plXX:any + M-A:none + Provides: postgresql-plXX + + + + + + + + + + + + diff --git a/doc/postgresql-debian-packaging.md b/doc/postgresql-debian-packaging.md new file mode 100644 index 0000000..4edd9ca --- /dev/null +++ b/doc/postgresql-debian-packaging.md @@ -0,0 +1,741 @@ +Packaging PostgreSQL Extensions for Debian +========================================== +January 2024, Christoph Berg + +Debian ships many PostgreSQL applications and extensions as packages. I often +get asked by developers how they would get their programs packaged, and ended +up writing the same reply over and over. This is a write-up intended as a more +thorough answer. + +# Anatomy of Debian packages + +Debian knows two sorts of packages: "source" packages and "binary" packages. +The latter type is the `.deb` files that get installed using apt or dpkg. The +first type is what this article is mostly about. For both sorts of packages, +there is an unpacked form and a packed form. + +## Binary packages + +Packed binary packages are a single `.deb` file: + +``` +$ ls -al postgresql-16-unit_7.7-1_amd64.deb +-rw-r--r-- 1 myon myon 136740 Jan 12 14:13 postgresql-16-unit_7.7-1_amd64.deb + +$ dpkg-deb -I postgresql-16-unit_7.7-1_amd64.deb + new Debian package, version 2.0. + size 136740 bytes: control archive=1332 bytes. + 643 bytes, 15 lines control + 2046 bytes, 25 lines md5sums + Package: postgresql-16-unit + Source: postgresql-unit + Version: 7.7-1 + Architecture: amd64 + Maintainer: Christoph Berg + Installed-Size: 500 + Depends: postgresql-16, postgresql-16-jit-llvm (>= 16), libc6 (>= 2.14) + Section: database + Priority: optional + Homepage: https://github.com/df7cb/postgresql-unit + Description: SI Units for PostgreSQL + postgresql-unit implements a PostgreSQL datatype for SI units, plus byte. The + base units can be combined to named and unnamed derived units using operators + defined in the PostgreSQL type system. SI prefixes are used for input and + output, and quantities can be converted to arbitrary scale. + +$ dpkg-deb -c postgresql-16-unit_7.7-1_amd64.deb +drwxr-xr-x root/root 0 2023-01-06 16:34 ./ +drwxr-xr-x root/root 0 2023-01-06 16:34 ./usr/ +drwxr-xr-x root/root 0 2023-01-06 16:34 ./usr/share/ +drwxr-xr-x root/root 0 2023-01-06 16:34 ./usr/share/doc/ +drwxr-xr-x root/root 0 2023-01-06 16:34 ./usr/share/doc/postgresql-16-unit/ +-rw-r--r-- root/root 267 2023-01-06 16:34 ./usr/share/doc/postgresql-16-unit/NEWS.Debian.gz +-rw-r--r-- root/root 7202 2023-01-06 16:34 ./usr/share/doc/postgresql-16-unit/README.md.gz +-rw-r--r-- root/root 977 2023-01-06 16:34 ./usr/share/doc/postgresql-16-unit/changelog.Debian.gz +-rw-r--r-- root/root 868 2023-01-06 16:34 ./usr/share/doc/postgresql-16-unit/copyright +drwxr-xr-x root/root 0 2023-01-06 16:34 ./usr/share/postgresql/ +drwxr-xr-x root/root 0 2023-01-06 16:34 ./usr/share/postgresql/16/ +drwxr-xr-x root/root 0 2023-01-06 16:34 ./usr/share/postgresql/16/extension/ +-rw-r--r-- root/root 19259 2023-01-06 16:34 ./usr/share/postgresql/16/extension/unit--7.sql +-rw-r--r-- root/root 244 2023-01-06 16:34 ./usr/share/postgresql/16/extension/unit.control +... +``` + +The unpacked binary package is of course the files being installed on the +system. + +## Source packages + +Unpacked source packages consist of the original upstream source code, with a `debian/` directory added. + +``` +postgresql-unit/ +├── debian/ +│   ├── changelog +│   ├── control +│   ├── control.in +│   ├── copyright +│   ├── gitlab-ci.yml +│   ├── NEWS +│   ├── pgversions +│   ├── rules* +│   ├── source/ +│   │   └── format +│   ├── tests/ +│   │   ├── control +│   │   └── installcheck* +│   ├── upstream/ +│   │   └── metadata +│   └── watch +├── Makefile +├── NEWS.md +├── README.md +├── unit--7.sql.in +├── unit.c +└── unit.control +``` + +Most packages are maintained in Git, which tracks this unpacked source package +form. (Either with or without the original upstream source, more on that +later.) + +Packed source packages are actually a set of files: + +``` +-rw-rw-r-- 1 myon myon 3848 Jan 12 14:31 postgresql-unit_7.7-1.debian.tar.xz +-rw-rw-r-- 1 myon myon 1100 Jan 12 14:31 postgresql-unit_7.7-1.dsc +-rw-rw-r-- 1 myon myon 414114 Jan 12 14:12 postgresql-unit_7.7.orig.tar.gz +``` + +This transport format is used for uploading to the Debian archive and for +retrieving the source code using `apt source`. (It is not stored in Git.) + +The `.orig.tar.gz` is the original source tarball as distributed by upstream, +either as download from the upstream homepage, or for projects hosted on +GitHub, often a tarball automatically generated by GitHub from an upstream Git +tag. + +The `.debiar.tar.xz` file contains the `debian/` directory. + +The `.dsc` file is a descriptor that contains pointers to the other files in +the source package, and more meta information. + +``` +$ cat postgresql-unit_7.7-1.dsc +Format: 3.0 (quilt) +Source: postgresql-unit +Binary: postgresql-16-unit +Architecture: any +Version: 7.7-1 +Maintainer: Christoph Berg +Homepage: https://github.com/df7cb/postgresql-unit +Standards-Version: 4.6.1 +Vcs-Browser: https://github.com/df7cb/postgresql-unit +Vcs-Git: https://github.com/df7cb/postgresql-unit.git +Testsuite: autopkgtest +Testsuite-Triggers: make +Build-Depends: bison, debhelper-compat (= 13), flex, postgresql-server-dev-all (>= 217~) +Package-List: + postgresql-16-unit deb database optional arch=any +Checksums-Sha1: + c2f81968bfbe83fed49b084b737e3aba423bf15a 414114 postgresql-unit_7.7.orig.tar.gz + b8a1917ddecb99b1441218bc74a0b7cb30752235 3848 postgresql-unit_7.7-1.debian.tar.xz +Checksums-Sha256: + 411d05beeb97e5a4abf17572bfcfbb5a68d98d1018918feff995f6ee3bb03e79 414114 postgresql-unit_7.7.orig.tar.gz + 36e89c762e50ddf997b079703200c0df6967b4fe911bde8e9482d8e82dcb6a98 3848 postgresql-unit_7.7-1.debian.tar.xz +Files: + 33a22586c8b81564ba7e9c05f430ad40 414114 postgresql-unit_7.7.orig.tar.gz + a0b31860b86c12c7173a78d6ecd525cb 3848 postgresql-unit_7.7-1.debian.tar.xz +``` + +## Building the source package + +To get started with working with a Debian package, get the unpacked source +package. This could mean invoking `apt source`, but most often checking out the +packaging Git repository is the better option as it might contain changes that +have not been uploaded yet. It also makes contributing changes easier. For +packages, that are already part of Debian, the `debcheckout` tool can automate +that (it uses the `Vcs-Git` field in the metadata). + +To build the packed source packed from the unpacked one, enter the package +directory, and invoke `dpkg-buildpackage -S --no-sign`: + +``` +postgresql-unit $ dpkg-buildpackage -S --no-sign +dpkg-buildpackage: info: source package postgresql-unit +dpkg-buildpackage: info: source version 7.7-1 +dpkg-buildpackage: info: source distribution unstable +dpkg-buildpackage: info: source changed by Christoph Berg + dpkg-source --before-build . + debian/rules clean +dh clean --with pgxs +... + dpkg-source -b . +dpkg-source: info: using source format '3.0 (quilt)' +dpkg-source: info: building postgresql-unit using existing ./postgresql-unit_7.7.orig.tar.gz +dpkg-source: info: building postgresql-unit in postgresql-unit_7.7-1.debian.tar.xz +dpkg-source: info: building postgresql-unit in postgresql-unit_7.7-1.dsc + dpkg-genbuildinfo --build=source -O../postgresql-unit_7.7-1_source.buildinfo + dpkg-genchanges --build=source -O../postgresql-unit_7.7-1_source.changes +dpkg-genchanges: info: including full source code in upload + dpkg-source --after-build . +dpkg-buildpackage: info: source-only upload (original source is included) +``` + +Note that the artifacts produced by `dpkg-buildpackage` are always in the *parent* directory of the working directory. +To make room for that, I always create *two* levels of directory for my working copies, so a typical case looks like this: + +``` +postgresql-unit/ +├── postgresql-unit/ <--- most commands are invoked from here +│   ├── debian/ +│   │   ├── changelog +│   │   ├── control +│   │   ├── control.in +│   │   ├── copyright +│   │   ├── files +│   │   ├── gitlab-ci.yml +│   │   ├── NEWS +│   │   ├── pgversions +│   │   ├── rules* +│   │   ├── source/ +│   │   │   └── format +│   │   ├── tests/ +│   │   │   ├── control +│   │   │   └── installcheck* +│   │   ├── upstream/ +│   │   │   └── metadata +│   │   └── watch +│   ├── Makefile +│   ├── NEWS.md +│   ├── powers.c +│   ├── powers.h +│   ├── README.md +│   ├── unit--7.sql.in +│   ├── unit.c +│   └── unit.control +├── postgresql-16-unit_7.7-1_amd64.deb +├── postgresql-16-unit-dbgsym_7.7-1_amd64.deb +├── postgresql-unit_7.7-1_amd64.changes +├── postgresql-unit_7.7-1.debian.tar.xz +├── postgresql-unit_7.7-1.dsc +└── postgresql-unit_7.7.orig.tar.gz +``` + +## Building binary packages + +Binary packages are built using `dpkg-buildpackage --no-sign`. + +``` + dpkg-buildpackage --no-sign +dpkg-buildpackage: info: source package postgresql-unit +dpkg-buildpackage: info: source version 7.7-1 +dpkg-buildpackage: info: source distribution unstable +dpkg-buildpackage: info: source changed by Christoph Berg +dpkg-buildpackage: info: host architecture amd64 + dpkg-source --before-build . + debian/rules clean +dh clean --with pgxs +... + dpkg-source -b . +dpkg-source: info: using source format '3.0 (quilt)' +dpkg-source: info: building postgresql-unit using existing ./postgresql-unit_7.7.orig.tar.gz +dpkg-source: info: building postgresql-unit in postgresql-unit_7.7-1.debian.tar.xz +dpkg-source: info: building postgresql-unit in postgresql-unit_7.7-1.dsc + debian/rules binary +dh binary --with pgxs + dh_update_autotools_config + dh_autoreconf + dh_auto_configure + dh_auto_build --buildsystem=pgxs + pg_buildext build build-%v +### PostgreSQL 16 build ### +make[1]: Entering directory '/home/myon/projects/postgresql/postgresql-unit/postgresql-unit/build-16' +gcc -Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Werror=vla -Wendif-labels -Wmissing-format-attribute -Wimplicit-fallthrough=3 -Wcast-function-type -Wshadow=compatible-local -Wformat-security -fno-strict-aliasing -fwrapv -fexcess-precision=standard -Wno-format-truncation -Wno-stringop-truncation -g -g -O2 -fstack-protector-strong -fstack-clash-protection -Wformat -Werror=format-security -fcf-protection -fno-omit-frame-pointer -g -O2 -ffile-prefix-map=/home/myon/projects/postgresql/postgresql-unit/postgresql-unit=. -fstack-protector-strong -fstack-clash-protection -Wformat -Werror=format-security -fcf-protection -fPIC -fvisibility=hidden -ffp-contract=off -I. -I/home/myon/postgresql/postgresql-unit/postgresql-unit -I/usr/include/postgresql/16/server -I/usr/include/postgresql/internal -Wdate-time -D_FORTIFY_SOURCE=2 -D_GNU_SOURCE -I/usr/include/libxml2 -c -o unit.o /home/myon/postgresql/postgresql-unit/postgresql-unit/unit.c +... +### End 16 build ### + create-stamp debian/debhelper-build-stamp + dh_prep + dh_auto_install --buildsystem=pgxs --destdir=debian/postgresql-16-unit/ + pg_buildext install build-%v postgresql-%v-unit +### PostgreSQL 16 install ### +make[1]: Entering directory '/home/myon/projects/postgresql/postgresql-unit/postgresql-unit/build-16' +/usr/bin/install -c -m 755 unit.so '/home/myon/postgresql/postgresql-unit/postgresql-unit/debian/postgresql-16-unit/usr/lib/postgresql/16/lib/unit.so' +... +### End 16 install ### + debian/rules override_dh_installdocs +make[1]: Entering directory '/home/myon/projects/postgresql/postgresql-unit/postgresql-unit' +dh_installdocs --all README.* +make[1]: Leaving directory '/home/myon/projects/postgresql/postgresql-unit/postgresql-unit' + dh_installchangelogs + dh_perl + dh_link + debian/rules override_dh_pgxs_test +make[1]: Entering directory '/home/myon/projects/postgresql/postgresql-unit/postgresql-unit' +# defer testing to autopkgtest, the data tables are not in /usr/share/postgresql yet +make[1]: Leaving directory '/home/myon/projects/postgresql/postgresql-unit/postgresql-unit' + dh_strip_nondeterminism + dh_compress + dh_fixperms + dh_missing + dh_dwz -a + dh_strip -a + dh_makeshlibs -a + dh_shlibdeps -a + dh_installdeb + dh_gencontrol + dh_md5sums + dh_builddeb +dpkg-deb: building package 'postgresql-16-unit' in '../postgresql-16-unit_7.7-1_amd64.deb'. +dpkg-deb: building package 'postgresql-16-unit-dbgsym' in '../postgresql-16-unit-dbgsym_7.7-1_amd64.deb'. + dpkg-genbuildinfo -O../postgresql-unit_7.7-1_amd64.buildinfo + dpkg-genchanges -O../postgresql-unit_7.7-1_amd64.changes +dpkg-genchanges: info: including full source code in upload + dpkg-source --after-build . +dpkg-buildpackage: info: full upload (original source is included) +``` + +Again, the resulting `.deb` files are placed in the parent directory. + +By default, this also builds the source. If this fails (often when there are +files that differ from the tarball version, more on patches later), use `-b` to +skip building the source: `dpkg-buildpackage -b --no-sign`. + +If building fails because of missing dependencies, install them using +`apt build-dep .`. + +There are various front-end utilities to automate these building steps better +(git-buildpackage, sbuild, pbuilder, debuild), but dpkg-buildpackage is just +fine. + +## The debian/ directory + +The `debian/` directory contains metadata and build instructions for the +package. "Creating a Debian package" really means editing the files in this +directory to make the package behave as desired. + +### debian/control + +The control file contains one section for the source package, followed by one +or more sections for binary packages. + +``` +$ cat debian/control +Source: postgresql-unit +Section: database +Priority: optional +Maintainer: Christoph Berg +Build-Depends: + bison, + debhelper-compat (= 13), + flex, + postgresql-server-dev-all (>= 217~), +Standards-Version: 4.6.1 +Rules-Requires-Root: no +Vcs-Git: https://github.com/df7cb/postgresql-unit.git +Vcs-Browser: https://github.com/df7cb/postgresql-unit +Homepage: https://github.com/df7cb/postgresql-unit + +Package: postgresql-16-unit +Architecture: any +Depends: ${misc:Depends}, ${shlibs:Depends}, postgresql-16 +Description: SI Units for PostgreSQL + postgresql-unit implements a PostgreSQL datatype for SI units, plus byte. The + base units can be combined to named and unnamed derived units using operators + defined in the PostgreSQL type system. SI prefixes are used for input and + output, and quantities can be converted to arbitrary scale. +``` + +(In the case of PostgreSQL extension packages, the control file is +automatically generated from `debian/control.in`, see below. Normal packages do +not have a control.in file.) + +### debian/rules + +The rules file handles the actual binary package building steps, in Makefile +syntax. + +Almost all packages today use a helper system called `debhelper` which consists +of various building blocks called `dh_*` that handle the build steps. +Historically, rules files would consist of long lists of `dh_*` steps (similar +to the build output above), but there is a "sequencer" command `dh` that knows +how to invoke the basic steps. + +Many packages that don't need any tweaking at that level have a very short +`debian/rules` file. (Make sure the indentation is a tab, not spaces!) + +``` +#!/usr/bin/make -f + +%: + dh $@ +``` + +Or in the case of PostgreSQL extensions: + +``` +#!/usr/bin/make -f + +%: + dh $@ --with pgxs +``` + +If any of the `dh_*` steps need changes, we can override them in the rules file +by adding a `override_dh_*` target: + +``` +#!/usr/bin/make -f + +override_dh_installdocs: + dh_installdocs --all README.* + +override_dh_pgxs_test: + # defer testing to autopkgtest, the data tables are not in /usr/share/postgresql yet + +%: + dh $@ --with pgxs +``` + +### Build steps + +The most interesting build steps to hook into are: + +#### `dh_auto_configure` + +Autodetects the build system and runs `./configure`, `cmake` and the like with +a set of default options. + +To add options, do: + +``` +override_dh_auto_configure: + dh_auto_configure -- -DEXTRA_OPTION=foo --with-bar +``` + +#### `dh_auto_build` + +The main build step. If the automatically run command is wrong, just override it. + +``` +override_dh_auto_build: + $(MAKE) -C some_sub_dir world +``` + +#### `dh_auto_install` + +Runs upstreams' `make install` or equivalent. The files are installed into +`DESTDIR=debian/foo` (single-binary package) or `DESTDIR=debian/tmp` +(multi-binary package). + +#### `dh_install` and `debian/foo.install` + +If there is more than one binary package, the files installed in `debian/tmp` +need to be distributed to the individual binary packages. This is done by +directory/file lists in `debian/*.install`. + +### debian/source/format + +Should be verbatim this: + +``` +3.0 (quilt) +``` + +(More on quilt and patches below.) + +### debian/copyright + +Debian wants copyright information on *all* files in a package. For private +package, this file can be omitted, but for anything official, this file has to +list all the copyright holders, along with any copyright terms and license +texts. + +### debian/watch + +To get informed about new upstream versions, Debian runs a "watch" system that +polls upstream download locations for new package versions. The `debian/watch` +file tells the `uscan` tool where to look and (optionally) how to transform +upstream's version naming scheme into a Debian-compatible one. + +``` +version=4 +https://github.com/df7cb/postgresql-unit/tags .*/([0-9.]*).tar.gz +``` + +This is a simple example where `uscan` looks at some GitHub "tags" URL, parses +the HTML, and recognizes all links pointing to .tar.gz files as new versions. +The regexp part of the URL in parentheses is used as the version number. + +The `uscan` tool is part of the `devscripts` package which holds a bunch of +utilities useful for packaging tasks. + +## Patching the upstream source + +Debian packages are based on upstream tarballs, and `dpkg-buildpackage` does +not like changed (or new) files (except in the `debian/` directory). Changes +need to be done using patch files stored in the `debian/patches/` directory, +and applied in the order listed in `debian/patches/series`. + +At package build time, support is built-in in dpkg-buildpackage. Patches can +edited with whatever tool, with `quilt` being the easiest solution. (It's an +optional package that likely isn't preinstalled.) `quilt` needs some +configuration in `~/.quiltrc`: + +``` +QUILT_DIFF_OPTS="-p" +QUILT_REFRESH_ARGS="-p ab --no-timestamps --no-index" +QUILT_PATCHES=debian/patches +``` + +Some useful commands: + +* Apply all patches: `quilt push -a` +* Unapply all patches: `quilt pop -a` +* Create a new patch: `quilt new foo` +* Add a file to a patch: `quilt add bar` (do this *before* changing the file!) +* Add a file and edit it: `quilt edit bar` +* Show current diff: `quilt diff` +* Save a patch after editing: `quilt refresh` + +Suppose we want to fix something in the `unit.c` file: + +``` +$ quilt new unit-dllexport +Patch unit-dllexport is now on top + +$ quilt add unit.c +File unit.c added to patch unit-dllexport + +$ vi unit.c + +$ quilt diff | cat +Index: postgresql-unit/unit.c +=================================================================== +--- postgresql-unit.orig/unit.c 2023-11-08 20:51:04.343207806 +0100 ++++ postgresql-unit/unit.c 2024-01-12 16:10:23.143509535 +0100 +@@ -136,7 +136,7 @@ unit_get_definitions(void) + + PG_MODULE_MAGIC; + +-void _PG_init(void); ++void PGDLLEXPORT _PG_init(void); + + void + _PG_init(void) + +$ quilt refresh +Refreshed patch unit-dllexport +``` + +This creates the `debian/patches/` directory: + +``` +postgresql-unit/debian/patches/ +├── series +└── unit-dllexport +``` + +At build-time, `dpkg-buildpackage` will then automatically apply and un-apply +patches as needed. + +## Tests + +Package tests are run in two flavors: at build-time and later independently on +packages installed on some system. + +### Build-time package tests + +If an upstream project has a test suite, it is recommended to run it at build +time. In many cases, `dh_auto_test` will guess how to do that and no extra +configuration is needed. If it guesses incorrectly, use `override_dh_auto_test` +to provide a better command. + +### Install-time package tests + +Next to tests at build-time, we can also run tests when the binary packages are +installed on some actual system. This has the advantage that it runs when all +files are in their final location, and can also interface with other packages +and services that might not be available at build time. It can also run +periodically to spot regression, while build-time tests would usually not be +repeated. + +Debian's system to run these tests is `autopkgtest`. The TL;DR version of the +documentation is this: In the `debian/tests/` directory, provide a command +(usually a shell script) that exercises some package smoke test or more complex +scenario. Register that test in `debian/tests/control`. + +``` +postgresql-unit/debian/tests/ +├── control +└── installcheck* + +$ cat debian/tests/control +Depends: @, make +Tests: installcheck +Restrictions: allow-stderr + +$ cat debian/tests/installcheck +#!/bin/sh +pg_buildext -i '--locale=C.UTF-8' installcheck +``` + +In `Depends` the `@` is a shorthand for all binary packages built from this +source. Other packages that the tests needs (and that the binaries don't depend +on) can be listed here. `Restrictions` declare tests properties. Examples are +`allow-stderr` (don't consider stderr output to be a test failure) and +`root-needed` (run test as root instead of an unprivileged user). + +In Debian, these tests are run automatically for QA, see +https://ci.debian.net/. More details are in the autopkgtest package +documentation in /usr/share/doc/autopkgtest/. + +# PostgreSQL extension packages + +Packages for PostgreSQL extensions work as described above, but since +extensions have to be compiled for each PostgreSQL major version separately, +things are a bit more complex. + +For Debian, the process is changed to still have one source package per +upstream project, but to build separate binary packages for each PostgreSQL +major version. The naming scheme is `postgresql-NN-foo`. (In case the upstream +project is called `pg_foo`, make a judgment call if `postgresql-NN-pg-foo` or +`postgresql-NN-foo` is better.) + +In Debian, only one PostgreSQL major version is supported at a time, but in the +https://apt.postgresql.org/ repository, many major versions are supported in +parallel (currently PostgreSQL 10 everything newer, even when 10 is already +EOL). + +## `debian/control.in` + +In order not to have to edit the list of binary packages built when a new +PostgreSQL major version comes out, or when a source package is built both for +Debian and for apt.postgresql.org, the `debian/control` file is generated from +a template in `debian/control.in`. + +``` +$ cat debian/control.in +Source: postgresql-unit +Section: database +Priority: optional +Maintainer: Christoph Berg +Build-Depends: + bison, + debhelper-compat (= 13), + flex, + postgresql-server-dev-all (>= 217~), +Standards-Version: 4.6.2 +Rules-Requires-Root: no +Vcs-Git: https://github.com/df7cb/postgresql-unit.git +Vcs-Browser: https://github.com/df7cb/postgresql-unit +Homepage: https://github.com/df7cb/postgresql-unit + +Package: postgresql-PGVERSION-unit +Architecture: any +Depends: ${misc:Depends}, ${shlibs:Depends}, ${postgresql:Depends} +Description: SI Units for PostgreSQL + postgresql-unit implements a PostgreSQL datatype for SI units, plus byte. The + base units can be combined to named and unnamed derived units using operators + defined in the PostgreSQL type system. SI prefixes are used for input and + output, and quantities can be converted to arbitrary scale. +``` + +The section with the `PGVERSION` token is duplicated for each major version +supported, with the version number filled in. + +## `debian/pgversions` + +Not every package supports all PostgreSQL major versions. The +`debian/pgversions` file is used to mark which versions are actually supported, +so apt.postgresql.org can skip building the other version. + +Unless we know the exact versions supported, we should use `all`: + +``` +$ cat debian/pgversions +all +``` + +If 14 or newer is supported: + +``` +$ cat debian/pgversions +14+ +``` + +### Supported versions + +`debian/pgversions` lists the versions supported *by the package*. The other +half of that system is the set of versions supported *by the system*. This list +is configured in `/etc/postgresql-common/supported_versions` or by setting the +`PG_SUPPORTED_VERSIONS` environment variable. (Set this variable to test +building for other major versions.) The actual build process will use the +intersection of these two lists. + +## `pg_buildext` + +The process of building PostgreSQL extensions for several major versions is +automated by the `pg_buildext` utility. It provides commands for the most +common tasks. + +Package building: + +* `pg_buildext supported-versions` - print list of supported versions. + Use this to loop over versions in `debian/rules`. +* `pg_buildext build build-%v` - build in `build-%v` directory +* `pg_buildext install build-%v postgresql-%v-unit` - invoke `make install` +* `pg_buildext installcheck build-%v postgresql-%v-unit` - invoke + `make installcheck` for build-time testing +* `pg_buildext loop postgresql-%v-unit` - use instead of + `build/install/installcheck` if the package doesn't support out-of-tree + builds in subdirectories +* `pg_buildext updatecontrol` - rebuild `debian/control` from + `debian/control.in`. Run this manually when the set of supported versions has + changed. This is not run automatically because the Debian packaging policy + forbids changing the set of binary packages at build time. (In environments + where this is not an issue, set `PG_UPDATECONTROL=yes`.) + +Package testing: + +* `pg_buildext installed-versions` - print list of installed versions. + Use this to loop over versions in `debian/tests/*`. +* `pg_buildext installcheck` - invoke `make installcheck` on installed packages + +## `dh --with pgxs` + +A debhelper extension `pgxs` is provided that adds builds steps to the `dh` +build sequence. + +``` +$ cat debian/rules +#!/usr/bin/make -f + +%: + dh $@ --with pgxs +``` + +If the package doesn't support out-of-tree builds, use +`dh $@ --with pgxs_loop`. + +``` + dh_auto_build --buildsystem=pgxs + pg_buildext build build-%v + dh_auto_install --buildsystem=pgxs + pg_buildext install build-%v postgresql-%v-unit + dh_pgxs_test + pg_buildext installcheck . build-%v postgresql-%v-unit +``` + +To override any of these steps, use `override_dh_auto_*` in `debian/rules`. + +## Package template + +To get started with a new package, the `dh_make_pgxs` tool can generate a +skeleton `debian/` directory: + +``` +$ dh_make_pgxs +``` + +If the auto-detected values are wrong, hit `^C` and add more command line +parameters. diff --git a/gitlab/gitlab-ci.yml b/gitlab/gitlab-ci.yml new file mode 100644 index 0000000..5c640aa --- /dev/null +++ b/gitlab/gitlab-ci.yml @@ -0,0 +1,13 @@ +# gitlab-ci.yml file for the Debian PostgreSQL packages +# to be used in debian/gitlab-ci.yml: +# include: https://salsa.debian.org/postgresql/postgresql-common/raw/master/gitlab/gitlab-ci.yml + +include: + - https://salsa.debian.org/salsa-ci-team/pipeline/raw/master/salsa-ci.yml + - https://salsa.debian.org/salsa-ci-team/pipeline/raw/master/pipeline-jobs.yml + +variables: + PG_UPDATECONTROL: 'yes' # tell pg_buildext to automatically generate debian/control from debian/control.in (TODO: doesn't work this way yet) + SALSA_CI_DISABLE_REPROTEST: 'yes' # too many packages fail because reprotest runs the build as root, disable for now + SALSA_CI_REPROTEST_ENABLE_DIFFOSCOPE: 'yes' + SALSA_CI_DISABLE_CROSSBUILD_ARM64: 'yes' # clang isn't ready for cross-building yet diff --git a/pg_backupcluster b/pg_backupcluster new file mode 100755 index 0000000..2959056 --- /dev/null +++ b/pg_backupcluster @@ -0,0 +1,659 @@ +#!/usr/bin/perl -wT + +# simple pg_basebackup front-end +# +# Copyright (C) 2021-2023 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use warnings; +use Fcntl qw(LOCK_EX); +use Getopt::Long; +use JSON; +use PgCommon; +use POSIX qw(strftime); + +my ($version, $cluster); + +# untaint environment +$ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin'; +delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; +chdir '/'; +umask 027; + +sub help () { + print "Syntax: $0 [options] +Actions: + createdirectory Create /var/backups/version-cluster + basebackup Backup using pg_basebackup + dump Backup using pg_dump + expiredumps Remove all but last N dumps + expirebasebackups Remove all but last N basebackups + receivewal Launch pg_receivewal + compresswal Compress WAL files in archive + archivecleanup Remove obsolete WAL files from archive + list Show dumps, basebackups, and WAL +Options: + -c --checkpoint Passed to pg_basebackup + -k --keep-on-error Keep faulty backup directory on error + -v --verbose Verbose output +"; +} + +my $checkpoint = 'spread'; +my $keep_on_error; +my $verbose; + +exit 1 unless GetOptions ( + 'c|checkpoint=s' => sub { $checkpoint = $_[1] =~ /^f/ ? "fast" : "spread" }, + 'k|keep-on-error' => \$keep_on_error, + 'v|verbose' => \$verbose, +); + +$verbose = 1 if (-t 1); # verbose output when running on terminal + +# accept both "version cluster action" and "version[-/]cluster action" +if (@ARGV >= 2 and $ARGV[0] =~ m!^(\d+\.?\d)[-/]([^/]+)$!) { + ($version, $cluster) = ($1, $2); + shift @ARGV; +} elsif (@ARGV >= 3 and $ARGV[0] =~ /^(\d+\.?\d)$/) { + $version = $1; + ($cluster) = ($ARGV[1]) =~ m!^([^/]+)$!; + shift @ARGV; + shift @ARGV; +} else { + help(); + exit 1; +} +my $action = $ARGV[0]; + +error "specified cluster $version $cluster does not exist" unless $version && $cluster && cluster_exists $version, $cluster; + +my %info = cluster_info($version, $cluster); +validate_cluster_owner \%info; + +my $rootdir = "/var/backups/postgresql"; +my $clusterdir = "$rootdir/$version-$cluster"; +my $waldir = "$clusterdir/wal"; + +# functions to be run as root + +sub create_directory() { + if (! -d $rootdir) { + my ($pg_uid, $pg_gid) = (getpwnam 'postgres')[2,3]; + ($pg_uid and $pg_gid) or error "getpwnam postgres: $!"; + mkdir $rootdir, 0755 or error "mkdir $rootdir: $!"; + chown $pg_uid, $pg_gid, $rootdir or error "chown $rootdir: $!"; + } + if (! -d $clusterdir) { + mkdir($clusterdir, 0750) or error "mkdir $clusterdir: $!"; + chown $info{owneruid}, $info{ownergid}, $clusterdir or error "chown $clusterdir: $!"; + } +} + +sub switch_to_cluster_owner() { + change_ugid $info{owneruid}, $info{ownergid}; +} + +# backup functions + +sub get_backupdir($$) { + my ($starttime, $suffix) = @_; + my $timestamp = strftime("%FT%H%M%SZ", gmtime($starttime)); + my $backupdir = "$clusterdir/$timestamp.$suffix"; + error "$backupdir already exists" if (-d $backupdir); + print "Creating $suffix in $backupdir\n" if ($verbose); + return $backupdir; +} + +sub remove_backup_on_error($) { + my $backupdir = shift; + if ($keep_on_error) { + print "Not removing $backupdir (--keep-on-error)\n"; + } else { + print "Removing $backupdir ...\n" if ($verbose); + system_or_error "rm", "-rf", $backupdir; + } +} + +sub create_basebackup($) { + my $backupdir = shift; + + system_or_error "pg_basebackup", + "--cluster", "$version/$cluster", + "--verbose", + (-t 2 ? "--progress" : ()), + "--checkpoint=$checkpoint", + "--format", "tar", "--gzip", + ($version < 10 ? "--xlog" : ()), + "-D", $backupdir; +} + +sub create_dumpall($) { + my $backupdir = shift; + mkdir($backupdir, 0750) or error "mkdir $backupdir: $!"; + + my $pg16 = $version >= 16 ? "|| ' --icu-rules ' || daticurules" : ""; + my $pg15 = $version >= 15 ? "CASE datlocprovider::text + WHEN 'c' THEN '--locale-provider libc' + WHEN 'i' THEN '--locale-provider icu --icu-locale ' || daticulocale $pg16 + END," : ""; + my $clusterquery = "SELECT + '--encoding', pg_catalog.pg_encoding_to_char(encoding), + '--lc-collate', datcollate, + '--lc-ctype', datctype, + $pg15 + CASE WHEN current_setting('data_checksums')::boolean THEN '-- --data-checksums' END +FROM pg_database WHERE datname = 'template0'"; + system_or_error "psql", + "--cluster", "$version/$cluster", + "-XAtF", " ", "-c", $clusterquery, + "-o", "$backupdir/createcluster.opts"; + + system_or_error "pg_dumpall", + "--cluster", "$version/$cluster", + "--globals-only", + "--file", "$backupdir/globals.sql"; + + $pg16 = $version >= 16 ? "|| ' ICU_RULES ' || quote_literal(coalesce(daticurules, ''))" : ""; + $pg15 = $version >= 15 ? "|| CASE datlocprovider::text + WHEN 'c' THEN 'LOCALE_PROVIDER libc' + WHEN 'i' THEN 'LOCALE_PROVIDER icu ICU_LOCALE ' || quote_literal(daticulocale) $pg16 + END" : ""; + my $dbquery = "/* database creation */ +SELECT + format('CREATE DATABASE %I WITH OWNER %I TEMPLATE template0 ENCODING %L LC_COLLATE %L LC_CTYPE %L', + datname, rolname, pg_encoding_to_char(encoding), datcollate, datctype) $pg15 || ';' AS command +FROM pg_database +LEFT JOIN pg_roles r ON r.oid = datdba +WHERE datallowconn AND datname NOT IN ('postgres', 'template1') + +UNION ALL + +/* database options */ +SELECT + CASE + WHEN rolname IS NULL THEN format('ALTER DATABASE %I', datname) + ELSE format('ALTER ROLE %I IN DATABASE %I', rolname, datname) + END || + format(' SET %I = ', match[1]) || + CASE + WHEN match[1] IN ('local_preload_libraries', 'search_path', 'session_preload_libraries', 'shared_preload_libraries', 'temp_tablespaces', 'unix_socket_directories') + THEN match[2] + ELSE quote_literal(match[2]) + END || + ';' AS command +FROM pg_db_role_setting s +JOIN pg_database d ON d.oid = setdatabase +LEFT JOIN pg_roles r ON r.oid = setrole, +unnest(setconfig) u(setting), +regexp_matches(setting, '(.+)=(.+)') m(match)"; + system_or_error "psql", + "--cluster", "$version/$cluster", + "-XAtc", $dbquery, + "-o", "$backupdir/databases.sql"; + + my $dblist = 'SELECT datname FROM pg_database WHERE datallowconn ORDER BY datname'; + my $databases = `psql --cluster '$version/$cluster' -XAtc '$dblist'`; + for my $datname ($databases =~ /(.+)/g) { + print "Dumping $datname to $backupdir/$datname.dump ...\n" if ($verbose); + system_or_error "pg_dump", + "--cluster", "$version/$cluster", + "--format=custom", + "--file", "$backupdir/$datname.dump", + $datname; + } +} + +sub create_configbackup($) { + my $backupdir = shift; + $info{configdir} or error "cluster has no configdir"; + system_or_error "tar", + "-C", $info{configdir}, + "-cz", "-f", "$backupdir/config.tar.gz", + "."; +} + +sub create_status($$$$) { + my ($type, $starttime, $backupdir, $status) = @_; + my $statusfile = "$backupdir/status"; + my $endtime = time; + my $statusjson = { + cluster => $cluster, + duration => $endtime - $starttime, + end => strftime("%FT%H%M%SZ", gmtime($endtime)), + start => strftime("%FT%H%M%SZ", gmtime($starttime)), + status => $status, + type => $type, + version => $version, + }; + if (my $hostname = `hostname`) { + chomp $hostname; + $statusjson->{hostname} = $hostname; + } + if (-e '/etc/machine-id') { + open my $fh, '/etc/machine-id'; + my $machine_id = <$fh>; + close $fh; + if ($machine_id) { + chomp $machine_id; + $statusjson->{'machine-id'} = $machine_id; + } + } + if (-e '/etc/machine-info') { + open my $fh, '/etc/machine-info'; + while (<$fh>) { + if (/^DEPLOYMENT=(.*)/) { + $statusjson->{'machine-deployment'} = $1; + } + } + close $fh; + } + + my $json = JSON->new->canonical; + open F, '>', $statusfile or error "$statusfile: $!"; + print F $json->encode($statusjson) . "\n" or error "$statusfile: $!"; + close F or error "$$statusfile: $!"; +} + +sub sync($) { + my $backupdir = shift; + system_or_error "sync $backupdir/*"; +} + +sub expire_backups($$) { + my ($suffix, $number) = @_; + my @backups = glob("$clusterdir/*.$suffix"); + my $found = 0; + for my $backup (reverse @backups) { + # iterate reversely over backups until we have found enough valid ones + if ($found >= $number) { + print "Removing $backup ...\n" if ($verbose); + $backup =~ /(.*)/; # untaint + system_or_error "rm", "-rf", $1; + } else { + if (-f "$backup/status") { + $found++; + } + } + } +} + +sub delete_broken() { + my @backups = (glob("$clusterdir/*.backup"), glob("$clusterdir/*.dump")); + for my $backup (@backups) { + if (! -f "$backup/status") { + print "Removing $backup ...\n" if ($verbose); + $backup =~ /(.*)/; # untaint + system_or_error "rm", "-rf", $1; + } + } +} + +# wal handling + +sub create_wal_directory() { + if (! -d $waldir) { + mkdir($waldir, 0750) or error "mkdir $waldir: $!"; + } +} + +sub receivewal() { + my $pg_receivewal = $version >= 10 ? 'pg_receivewal' : 'pg_receivexlog'; + + # create slot + system_or_error $pg_receivewal, "--cluster=$version/$cluster", "--slot", "pg_receivewal_service", "--create-slot", "--if-not-exists"; + + # launch pg_receivewal + $ENV{PGAPPNAME} = "pg_receivewal\@$version-$cluster.service"; + my @cmd = ($pg_receivewal, "--cluster", "$version/$cluster", + "-D", $waldir, "--slot", "pg_receivewal_service"); + push @cmd, "--compress=5" if ($version >= 10); + exec {$pg_receivewal} @cmd or error "exec $pg_receivewal: $!"; +} + +sub compresswal() { + chdir $waldir or return; # ok if not yet created + open my $lock, $waldir or error "open $waldir: $!"; # protect against concurrent runs + flock $lock, LOCK_EX or error "flock $waldir: $!"; + + for my $wal (glob "0???????????????????????") { + $wal =~ /^([0-9A-F]+)$/ or continue; + $wal = $1; # untaint + if (-f "$wal.gz") { + print STDERR "$waldir/$wal.gz already exists, skipping compression\n"; + next; + } + system_or_error "if ! gzip < $wal > $wal.tmp.gz; then rm -f $wal.tmp.gz; exit 1; fi"; + system_or_error "touch --reference=$wal $wal.tmp.gz"; + system_or_error "mv $wal.tmp.gz $wal.gz"; + system_or_error "sync", "$wal.gz"; + unlink $wal; + } + + close $lock; +} + +sub archivecleanup() { + chdir $waldir or return; # ok if not yet created + my @backups = sort glob "$clusterdir/*.backup"; + for my $backup (@backups) { + next unless (-f "$backup/status"); + + $backup =~ /(.*)/; # untaint + my $basetar = "$1/base.tar.gz"; + my $backup_label = `tar --extract --occurrence=1 --to-stdout --file '$basetar' backup_label` or error "failed to extract backup_label from $basetar"; + + # START WAL LOCATION: 0/2B000028 (file 00000001000000000000002B) + $backup_label =~ /^START WAL LOCATION: .* \(file ([0-9A-F]+)\)/ or error "no start wal location in $basetar"; + my $keep_file = $1; + system_or_error "pg_archivecleanup", "-x", ".gz", $waldir, $keep_file; + + return 0; # process first backup only + } + error "no valid basebackups found in $clusterdir"; +} + +# info functions + +sub dirsize($) { + my $dir = shift; + my $size = 0; + my $files = 0; + for my $f (glob "$dir/*") { + $size += (stat $f)[7]; + $files++; + } + return $size, $files; +} + +sub list() { + print "Cluster $version $cluster backups in $clusterdir:\n"; + my $totalsize = 0; + print "Dumps:\n"; + for my $dir (sort glob "$clusterdir/*.dump") { + my ($size, $files) = dirsize($dir); + my $status = -f "$dir/status" ? '' : ' BROKEN'; + print " $dir: $size Bytes$status\n"; + $totalsize += $size; + } + print "Basebackups:\n"; + for my $dir (sort glob "$clusterdir/*.backup") { + my ($size, $files) = dirsize($dir); + my $status = -f "$dir/status" ? '' : ' BROKEN'; + print " $dir: $size Bytes$status\n"; + $totalsize += $size; + } + if (-d "$clusterdir/wal") { + print "WAL:\n"; + my ($size, $files) = dirsize("$clusterdir/wal"); + print " $clusterdir/wal: $size Bytes, $files Files\n"; + $totalsize += $size; + } + print "Total: $totalsize Bytes\n"; +} + +# main + +if ($action eq 'createdirectory') { + create_directory(); + +} elsif ($action eq 'basebackup') { + error "basebackups of pre-9.1 servers are not supported" if ($version < 9.1); + my $starttime = time; + create_directory(); + switch_to_cluster_owner(); + my $backupdir = get_backupdir($starttime, 'backup'); + $SIG{__DIE__} = sub { remove_backup_on_error($backupdir) }; + create_basebackup($backupdir); + create_configbackup($backupdir); + create_status($action, $starttime, $backupdir, 'ok'); + $SIG{__DIE__} = undef; + sync($backupdir); + compresswal(); + +} elsif ($action eq 'dump') { + error "dumps of pre-9.3 servers are not supported" if ($version < 9.3); + my $starttime = time; + create_directory(); + switch_to_cluster_owner(); + my $backupdir = get_backupdir($starttime, 'dump'); + $SIG{__DIE__} = sub { remove_backup_on_error($backupdir) }; + create_dumpall($backupdir); + create_configbackup($backupdir); + create_status($action, $starttime, $backupdir, 'ok'); + $SIG{__DIE__} = undef; + sync($backupdir); + +} elsif ($action eq 'expiredumps' and @ARGV == 2 and $ARGV[1] =~ /^(\d+)$/) { + switch_to_cluster_owner(); + expire_backups('dump', $1); + +} elsif ($action eq 'expirebasebackups' and @ARGV == 2 and $ARGV[1] =~ /^(\d+)$/) { + switch_to_cluster_owner(); + expire_backups('backup', $1); + archivecleanup(); + compresswal(); + +} elsif ($action eq 'deletebroken') { + switch_to_cluster_owner(); + delete_broken(); + +} elsif ($action eq 'receivewal') { + create_directory(); + switch_to_cluster_owner(); + create_wal_directory(); + compresswal(); + receivewal(); + +} elsif ($action eq 'compresswal') { + switch_to_cluster_owner(); + compresswal(); + +} elsif ($action eq 'archivecleanup') { + switch_to_cluster_owner(); + archivecleanup(); + compresswal(); + +} elsif ($action eq 'list') { + switch_to_cluster_owner(); + list(); + +} else { + help(); + exit(1); +} + +__END__ + +=head1 NAME + +pg_backupcluster - simple pg_basebackup and pg_dump front-end + +=head1 SYNOPSIS + +B [I] I I I + +=head1 DESCRIPTION + +B provides a simple interface to create PostgreSQL cluster +backups using L and L. + +To ease integration with B operation, the alternative syntax +"B IB<->I I" is also supported. + +=head1 ACTIONS + +=over 4 + +=item B + +Create /var/backups and /var/backups/I-I. +This action can be run as root to create the directories required for backups. +All other actions will also attempt to create the directories when missing, but +can of course only do that when running as root. They will switch to the +cluster owner after this step. + +=item B + +Backup using L. The resulting basebackup contains the WAL +files required to run recovery on startup. + +=item B + +Backup using L. Global objects (users, tablespaces) are dumped +using L B<--globals-only>. Individual databases are dumped into +PostgreSQL's custom format. + +=item B I + +Remove all but last the I basebackups. + +=item B I + +Remove all but last the I dumps. + +=item B + +Launch pg_receivewal. WAL files are gzip-compressed in PG 10+. + +=item B + +Compress WAL files in archive. + +=item B + +Remove obsolete WAL files from archive using L. + +=item B + +Show dumps, basebackups, and WAL, with size. + +=back + +=head1 OPTIONS + +=over 4 + +=item B<-c --checkpoint=spread|fast> + +Passed to B. Default is B. + +=item B<-k --keep-on-error> + +Keep faulty backup directory on error. By default backups are delete on error. + +=item B<-v --verbose> + +Verbose output, even when not running on a terminal. + +=back + +=head1 FILES + +=over 4 + +=item /var/backups + +Default root directory for cluster backup directories. + +=item /var/backups/I-I + +Default directory for cluster backups. + +=item /var/backups/I-I/IB<.basebackup> + +Backup from B. + +=over 4 + +=item C + +Tarball of cluster configuration directory (postgresql.conf, pg_hba.conf, ...) +in /etc/postgresql. + +=item IC<.tar.gz>, C, C + +Tablespace and WAL tarballs and backup info written by B. + +=item C + +Completion timestamp of backup run. + +=back + +=item /var/backups/I-I/IB<.dump> + +Backup from B. + +=over 4 + +=item C + +Tarball of cluster configuration directory (postgresql.conf, pg_hba.conf, ...) +in /etc/postgresql. + +=item C + +Options (encoding, locale, data checksums) to be passed to B +for restoring this cluster. + +=item C + +Global objects (roles, tablespaces) from B. + +=item C + +SQL commands to create databases and restore database-level options. + +=item IC<.dump> + +Database dumps from B. + +=item C + +Completion timestamp of backup run. + +=back + +=item /var/backups/I-I/B + +WAL files from B. + +=back + +=head1 CAVEATS + +For dump-style backups, not all properties of the original cluster are preserved: + +=over 2 + +=item * In PostgreSQL 10 and earlier, ALTER ROLE ... IN DATABASE is not supported. + +=item * Not all B options are carried over. Currently supported are B<--encoding>, +B<--lc-collate>, B<--lc-collate>, and B<-k --data-checksums>. + +=back + +The earliest PostgreSQL version supported for dumps is 9.3. +For basebackups, the earliest supported version is 9.1. +B (and hence archive recovery) are supported in 9.5 and later. + +=head1 SEE ALSO + +L, +L, L, +L, L, L. + +=head1 AUTHOR + +Christoph Berg Lmyon@debian.orgE> diff --git a/pg_buildext b/pg_buildext new file mode 100755 index 0000000..304e8a1 --- /dev/null +++ b/pg_buildext @@ -0,0 +1,503 @@ +#!/bin/bash +# +# build a PostgreSQL module based on PGXS for given list of supported major +# versions +# +# (C) 2010 Dimitri Fontaine +# (C) 2011-2023 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +set -eu + +die() { + echo "$(basename $0): error: $*" >&2 + exit 1 +} + +VENVARGS="" +MAKEVARS="" +while getopts "c:i:m:o:s" opt ; do + case $opt in + c|i|o) VENVARGS="$VENVARGS -$opt $OPTARG" ;; + m) MAKEVARS="$OPTARG" ;; + s) VENVARGS="$VENVARGS -$opt" ;; + *) exit 1 ;; + esac +done +# shift away args +shift $(($OPTIND - 1)) + +# positional arguments: action [srcdir] [target [opt]] +[ "${1:-}" ] || die "no action given" +action="$1" +shift + +if [ -d "${1:-}" ] && [ "${2:-}" ]; then # optional: source directory + srcdir="${1:-}" + [ "$srcdir" = "." ] && srcdir="$PWD" + shift +else + srcdir="$PWD" +fi +target="${1:-}" +opt="${2:-}" + +prepare_env() { + local version=$1 + vtarget=`echo $target | sed -e "s:%v:$version:g"` + pgc="/usr/lib/postgresql/$version/bin/pg_config" + [ -e "$pgc" ] || die "$pgc does not exist" + if [ "${CFLAGS:-}" ]; then + export COPT="$CFLAGS" + fi +} + +configure() { + prepare_env $1 + confopts=`echo $opt | sed -e "s:%v:$1:g"` + + mkdir -p $vtarget + ( echo "calling configure in $vtarget" && + cd $vtarget && $srcdir/configure $confopts PG_CONFIG="$pgc" VPATH="$srcdir" ) || return $? +} + +build() { + prepare_env $1 + if [ "$opt" ]; then + cflags="$(echo $opt | sed -e "s:%v:$1:g")" + export COPT="${COPT:+$COPT }$cflags" + fi + + mkdir -p $vtarget + # if a Makefile was created by configure, use it, else the top level Makefile + [ -f $vtarget/Makefile ] || makefile="-f $srcdir/Makefile" + make -C $vtarget ${makefile:-} PG_CONFIG="$pgc" VPATH="$srcdir" USE_PGXS=1 $MAKEVARS || return $? +} + +substvars() { + version="$1" + package="$2" + + depends="postgresql-$version" + if [ -d "debian/$package/usr/lib/postgresql/$version/lib/bitcode" ]; then + pg_provides=$(dpkg-query --show --showformat='${Provides}' postgresql-$version | grep -o 'postgresql-[0-9]*-jit-llvm (= [0-9.]*') + llvm_version=${pg_provides#*= } + if [ "$llvm_version" ]; then # skip if server version doesn't have the Provides yet + depends="$depends, postgresql-$version-jit-llvm (>= $llvm_version)" + fi + fi + echo "postgresql:Depends=$depends" >> "debian/$package.substvars" + + if ! grep -q postgresql:Depends debian/control; then # compat: add to misc:Depends + if grep -q ^misc:Depends "debian/$package.substvars"; then + sed -i -e "s/^misc:Depends=/misc:Depends=$depends, /" "debian/$package.substvars" + else + echo "misc:Depends=$depends" >> "debian/$package.substvars" + fi + fi +} + +install() { + prepare_env $1 + package=`echo $opt | sed -e "s:%v:$1:g"` + + mkdir -p $vtarget + # if a Makefile was created by configure, use it, else the top level Makefile + [ -f $vtarget/Makefile ] || makefile="-f $srcdir/Makefile" + make -C $vtarget ${makefile:-} install DESTDIR="$PWD/debian/$package" PG_CONFIG="$pgc" VPATH="$srcdir" USE_PGXS=1 $MAKEVARS || return $? + substvars "$1" "$package" +} + +clean() { + prepare_env $1 + if [ "$vtarget" ]; then + rm -rf $vtarget + fi +} + +loop() { + prepare_env $1 + package=$(echo $target | sed -e "s:%v:$1:g") + + echo "# $1: make" + make -C "$srcdir" PG_CONFIG="$pgc" USE_PGXS=1 $MAKEVARS || return $? + echo "# $1: make install" + make -C "$srcdir" install DESTDIR="$PWD/debian/$package" PG_CONFIG="$pgc" USE_PGXS=1 $MAKEVARS || return $? + substvars "$1" "$package" + echo "# $1: make clean" + make -C "$srcdir" clean PG_CONFIG="$pgc" USE_PGXS=1 $MAKEVARS # clean errors are fatal +} + +installcheck() { + prepare_env $1 + + # ask pg_regress to output unified diffs + export PG_REGRESS_DIFF_OPTS="-U3" + + # ask pg_virtualenv to create a non-system cluster + if [ "${NONROOT-unset}" = "unset" ]; then + export NONROOT=1 + fi + + # if a package pattern is given, tell pg_virtualenv where the installed files are + if [ "$opt" ]; then + pkg=$(echo "$opt" | sed -e "s:%v:$1:g") + PKGARGS="-p $pkg" + DESTDIR="DESTDIR=$PWD/debian/$pkg" + fi + + if [ "$target" ] && [ "$target" != "." ]; then # if target is given, use it, else stay in the top source dir + # if a Makefile was created by configure, use it, else the top level Makefile + [ -f $vtarget/Makefile ] || makefile="-f $srcdir/Makefile" + if ! pg_virtualenv ${PKGARGS:-} $VENVARGS -v $1 \ + make -C $vtarget ${makefile:-} installcheck ${DESTDIR:-} \ + PG_CONFIG="$pgc" VPATH="$srcdir" USE_PGXS=1 $MAKEVARS; then + if [ -r $vtarget/regression.diffs ]; then + echo "**** $vtarget/regression.diffs ****" + cat $vtarget/regression.diffs + fi + return 1 + fi + else + if ! pg_virtualenv ${PKGARGS:-} $VENVARGS -v $1 \ + make installcheck ${DESTDIR:-} PG_CONFIG="$pgc" USE_PGXS=1 $MAKEVARS; then + if [ -r regression.diffs ]; then + echo "**** regression.diffs ****" + cat regression.diffs + fi + return 1 + fi + # since we are in the top-level directory, clean up behind us + make clean PG_CONFIG="$pgc" USE_PGXS=1 $MAKEVARS + fi +} + +run_run () { + local v=$1 + shift + + ( + echo + "${@//%v/$v}" + "${@//%v/$v}" < $PSQLTMP + ) || return $? +} +run_run_installed () { + run_run "$@" +} + +run_psql () { + prepare_env $1 + + # ask pg_virtualenv to create a non-system cluster + if [ "${NONROOT-unset}" = "unset" ]; then + export NONROOT=1 + fi + + # if a package pattern is given, tell pg_virtualenv where the installed files are + if [ "$opt" ]; then + pkg=$(echo "$opt" | sed -e "s:%v:$1:g") + PKGARGS="-p $pkg" + export DESTDIR="$PWD/debian/$pkg" + fi + + ( + if [ "$target" ] && [ "$target" != "." ]; then # if target is given, use it, else stay in the top source dir + cd $target + fi + pg_virtualenv ${PKGARGS:-} $VENVARGS -v $1 \ + psql -Xe -v ON_ERROR_STOP=1 < $PSQLTMP + ) || return $? +} + +run_virtualenv () { + prepare_env $1 + + # ask pg_virtualenv to create a non-system cluster + if [ "${NONROOT-unset}" = "unset" ]; then + export NONROOT=1 + fi + + # if a package pattern is given, tell pg_virtualenv where the installed files are + if [ "$opt" ]; then + pkg=$(echo "$opt" | sed -e "s:%v:$1:g") + PKGARGS="-p $pkg" + export DESTDIR="$PWD/debian/$pkg" + fi + + ( + if [ "$target" ] && [ "$target" != "." ]; then # if target is given, use it, else stay in the top source dir + cd $target + fi + pg_virtualenv ${PKGARGS:-} $VENVARGS -v $1 ${SHELL:-/bin/sh} -ex < $PSQLTMP + ) || return $? +} + +versions() { + [ -e /usr/share/postgresql-common/supported-versions ] || + die "/usr/share/postgresql-common/supported-versions not found" + [ -e debian/pgversions ] || die "debian/pgversions not found" + supported_versions=$(/usr/share/postgresql-common/supported-versions) + local version + while read version; do + case $version in + all) echo "$supported_versions" ;; + [1-9]*+) + for sup_version in $supported_versions; do + if dpkg --compare-versions "${version%+}" le "$sup_version"; then echo "$sup_version"; fi + done ;; + [1-9]*) + for sup_version in $supported_versions; do + if [ "$version" = "$sup_version" ]; then echo "$sup_version"; fi + done ;; + '#'*) ;; + '') ;; + *) echo "Syntax error in debian/pgversions: $version" >&2 ;; + esac + done < debian/pgversions +} + +# list of PostgreSQL versions for which packages built from this source are installed +# (not necessarily the same as listed in debian/control) +installed_versions() { + [ -e debian/control.in ] || die "debian/control.in not found" + # extract package name template(s) from control.in, split into prefix and suffix + perl -lne 'print "$1 $2" if /^Package: (.*)PGVERSION(.*)/' debian/control.in | \ + while read prefix suffix; do + # translate templates to actually installed packages, and extract version numbers + dpkg-query --showformat '${db:Status-Status} ${Package}\n' --show "$prefix*$suffix" | \ + grep '^installed' | cut -d ' ' -f 2 | sed -e "s/^$prefix//; s/$suffix\$//" | grep -E '^[0-9.]+$' + # if suffix is empty, the grep will filter out noise like '13-dbgsym' + done | sort -uV +} + +gencontrol() { + tmpcontrol=$(mktemp debian/control.XXXXXX) + if [ -f debian/tests/control.in ]; then + tmptestscontrol=$(mktemp debian/tests/control.XXXXXX) + fi + trap "rm -f $tmpcontrol ${tmptestscontrol:-}" EXIT + + export PGVERSIONS="$(versions)" + [ "$PGVERSIONS" ] || die "No current PostgreSQL versions are supported by this package" + + perl -e \ + '$/ = ""; # read paragraphs + my @pgversions = split /\s+/, $ENV{PGVERSIONS}; + my $newest_pgversion = $pgversions[-1]; + + sub replace_all ($) { + $_ = shift; + my @out; + for my $version (@pgversions) { + push @out, s/PGVERSIONS/$version/rg; + } + return join ", ", @out; + } + + while (<>) { + chomp; + if (/^Package: .*PGVERSION/) { + foreach my $version (@pgversions) { + push @out, s/PGVERSION/$version/rg; + } + } else { + s/( + [^[:space:](),]* + PGVERSIONS + [^[:space:](),]* + (?:\s*\([^(),]*\))? # optional part in parentheses + [^[:space:](),]* + )/replace_all($1)/xeg; + s/PGVERSION/$newest_pgversion/g; + push @out, $_; + } + } + print join("\n\n", @out), "\n"; + ' debian/control.in > $tmpcontrol + + if [ -f debian/tests/control.in ]; then + cp debian/tests/control.in $tmptestscontrol + # find words (package names) containing PGVERSION + REGEXP='[[:alnum:]-]*PGVERSION[[:alnum:]-]*' + for pkgpattern in $(grep -Ewo "$REGEXP" debian/tests/control.in | sort -u); do + repl="" + # build an array of replacements separated by , + for v in $(versions); do + repl="${repl:+$repl, }$(echo $pkgpattern | sed -e "s/PGVERSION/$v/g")" + done + # put array into control file + grep -q "$pkgpattern" $tmptestscontrol # assert the pattern didn't get already removed by an earlier replacement + sed -i -e "s/$pkgpattern/$repl/" $tmptestscontrol + done + fi +} + +updatecontrol() { + cat $tmpcontrol > debian/control + + if [ -f debian/tests/control.in ]; then + cat $tmptestscontrol > debian/tests/control + fi + + # re-check build-dependencies (no error raised) + ( dpkg-checkbuilddeps 2>&1 || : ) | sed -e 's/error/notice/' +} + +# when a version is included in the action, just act on this one (this is +# useful if some extra work needs to be done per version, so the loop over +# supported-versions needs to be in the script calling pg_buildext) + +case $action in + configure-*|build-*|install-*|clean-*|installcheck-*) + a=${action%%-*} + v=${action##$a-} + ret=0 + echo "### PostgreSQL $v $a ###" + if $a $v; then + echo "### End $v $a ###" + else + ret=$? + echo "### End $v $a (FAILED with exit code $ret) ###" + fi + exit $ret + ;; + + checkcontrol) + [ -f debian/control.in ] || exit 0 # silently exit if debian/control.in is not there + gencontrol + need_update= + if ! diff -u debian/control $tmpcontrol; then + if [ "${PG_UPDATECONTROL:-no}" != "no" ] || head -1 debian/changelog | grep -Eq -- '-backports|-pgdg|-pgapt'; then + echo "Notice: Updating debian/control from debian/control.in." + need_update=1 + else + echo "Error: debian/control needs updating from debian/control.in. Run 'pg_buildext updatecontrol'." + echo "If you are seeing this message in a buildd log, a sourceful upload is required." + exit 1 + fi + fi + if [ -f debian/tests/control.in ] && ! diff -u debian/tests/control $tmptestscontrol; then + echo "Notice: Updating debian/tests/control from debian/tests/control.in." + need_update=1 + fi + [ "$need_update" ] && updatecontrol + exit 0 + ;; + + updatecontrol) + [ -f debian/control.in ] || die "debian/control.in is missing, cannot update debian/control" + gencontrol + updatecontrol + exit + ;; + + clean) + if [ "$target" ]; then + pattern=$(echo "$target" | sed -e "s:%v:*:g") + echo rm -rf $pattern/ + rm -rf $pattern/ + fi + if [ "$opt" ]; then + pattern=$(echo "$opt" | sed -e "s:%v:*:g") + echo rm -rf debian/$pattern/ debian/$pattern.substvars + rm -rf debian/$pattern/ debian/$pattern.substvars + fi + if [ -f Makefile ]; then + make clean USE_PGXS=1 + fi + exit + ;; + + installed-versions) + installed_versions + exit + ;; + + installcheck) + # prefer testing installed versions over supported versions as the set + # of versions might have changed (unless a package-pattern is provided) + if [ -f debian/control.in ] && [ -z "$opt" ]; then + versions=$(installed_versions) + else + versions=$(versions) + fi + [ "$versions" ] || exit 1 + ret=0 + for v in $versions; do + echo "### PostgreSQL $v $action ###" + if $action $v; then + echo "### End $v $action ###" + else + ret=$? + echo "### End $v $action (FAILED with exit code $ret) ###" + fi + done + exit $ret + ;; + + run|run_installed|psql|virtualenv) + PSQLTMP=$(mktemp --tmpdir psql.XXXXXX) + trap "rm -f $PSQLTMP" EXIT + # if we are fed any input (and hence aren't reading from a terminal), + # store it in a tempfile + [ ! -t 0 ] && cat > $PSQLTMP + + # prefer testing installed versions over supported versions as the set + # of versions might have changed (unless a package-pattern is provided) + if [ "$action" != "run" ] && [ -f debian/control.in ] && [ -z "$opt" ]; then + versions=$(installed_versions) + else + versions=$(versions) + fi + [ "$versions" ] || exit 1 + ret=0 + for v in $versions; do + echo "### PostgreSQL $v $action ###" + if run_$action $v "$@"; then + echo "### End $v $action ###" + else + ret=$? + echo "### End $v $action (FAILED with exit code $ret) ###" + fi + done + exit $ret + ;; +esac + +# loop over versions + +ret=0 +for v in $(versions) +do + case "$action" in + "supported-versions") + echo $v + ;; + + configure|build|install|clean|loop) + [ "$target" ] || die "syntax: pg_buildext $action [] []" + echo "### PostgreSQL $v $action ###" + if $action $v; then + echo "### End $v $action ###" + else + ret=$? + echo "### End $v $action (FAILED with exit code $ret) ###" + fi + ;; + + *) + die "unsupported action '$action'" + ;; + esac +done + +exit $ret diff --git a/pg_buildext.pod b/pg_buildext.pod new file mode 100644 index 0000000..d76c701 --- /dev/null +++ b/pg_buildext.pod @@ -0,0 +1,358 @@ +=head1 NAME + +pg_buildext - Build and install a PostgreSQL extension + +=head1 SYNOPSIS + +B [I] I [I] [I] + +=head1 DESCRIPTION + +B is a script that will build a PostgreSQL extension in a C +way, for potentially several PostgreSQL server versions in parallel. +It builds for the intersection of versions known in +C (versions supported by the package) and in +C (versions supported in this +release). + +Many PostgreSQL extension packages require no special handling at build time +and can use B or B to +automatically execute the steps outlined below. + +=head1 USAGE + +Packages using B should be prepared to build binaries for +PostgreSQL versions that are not present in Debian unstable, e.g. for older +releases when building backports for Debian (old)stable (possibly including +backports of newer PostgreSQL releases), or for all PostgreSQL releases when +the package is built for B. + +As the set of binary packages depends on the target PostgreSQL versions, +C is generated from a template in C when +B is run. +Package sections that contain B in the package name are replaced by +a list of sections, filling in the supported PostgreSQL versions. +Package sections that contain B outside the package name have the +newest supported PostgreSQL version filled in (useful for meta packages); +words containing B be replaced by a list of words with the +supported versions filled in, this is most useful in B. +Include +C in C to +run a check at build time if updating debian/control is required. + +As B invokes B for the B, B, and B +actions, invocations from C (which is a makefile) should be prefixed +with B<+> so the sub-makes can talk with the make jobserver. Additional makefile +variables can be passed to B via the B<-m> option. + +Many extensions support B testing using B. As +this needs the package to be installed, it cannot be run at build time. +Instead, the tests should be run using B from C. + +If C exists, occurrences of package names containing +B are replaced by lists of package names with the target PostgreSQL +versions filled in. (If no replacing is needed in C, it +is fine to provide the tests control file directly.) + +=head1 OPTIONS + +=over 4 + +=item B<-cio> I + +=item B<-s> + +Passed to B when running B. + +=item B<-m> I + +Passed to B. + +=back + +=head1 ACTIONS + +Most actions expect a directory name where to build the sources. It will get +created for you if it does not exist. If the I contains a C<%v> +sign, it will get replaced by the specific version of PostgreSQL being built +against. (Usually this parameter is C.) + +=over 4 + +=item B + +Print effective list of supported versions, i.e. the intersection of the sets +of versions supported by the system +(from C) and the package +(from C). + +Use this when building packages. + +=item B + +In the list of installed packages, look for packages matching the B +package name templates from C, and print the PostgreSQL +major version number part. + +Use this when testing packages. + +=item B + +Check if C needs updating from C. This is +invoked from C. When +building for a B or B suite as determined by +C, this action also updates the control file. Otherwise, +B needs to be run manually. + +=item B + +Update C from C, and C +from C if the latter exists. + +=item B [I] I [I] + +For every supported version, call B<../configure> from the I +directory. (Most PostgreSQL extensions do not have a configure script.) + +=item B [I] I [I] + +Build the extension in the I directory. + +=item B [I] I I + +Invoke B from the I directory. +The third parameter specifies the package name to use. Most packages +use B. Make will be +called with DESTDIR="$(CURDIR)/debian/I". + +The B substitution variable B is set to depend +on the required PostgreSQL server package. For compatibility with previous +packaging standards, the dependency is also added to B if +postgresql:Depends is not used. + +=item B [I] [I] [I] + +Clean the build directories. + +=item B [I] I + +As a variant to calling B and B separately for VPATH builds, +loop over the supported PostgreSQL versions in the top source directory. This +should be used if the package does not support VPATH builds. As it also invokes +B, it should be placed were installation happens in debian/rules, +rather than where build would normally be called. + +=item B [I] [I] [I] + +Use B to run the extension regression tests. +This is meant to be run from C using B. If +I is omitted, the top source directory is used. + +If I is given, options are passed to B to set +up the temporary PostgreSQL instance to find extension files in +C. + +Other than the other actions which run on the "supported" versions, if C exists, this one +runs on the "installed" versions as reported by B (unless +I is provided, which means we are called during a build). + +=item B [I] [I] [I] + +=item B [I] [I] [I] + +Like B, but invokes B, or a shell, both wrapped in +B. Input is read from stdin. + +=item B [I] I + +=item B [I] I + +Runs I, with B<%v> placeholder replaced on all supported versions, or +all installed versions, respectively. + +=back + +Sometimes it is desirable to run extra code per version before invoking the +action, in that case the loop over supported versions needs to be in the +calling script. To facilitate this mode, actions can also be called as +IB<->I. See the installcheck example below. + +=head1 SUPPORTED VERSIONS + +B reads C to decide which PostgreSQL to build +modules/extensions for. This file contains one PostgreSQL version number per +line, in the following formats: + +=over 4 + +=item B + +Support all versions. This is recommended unless there are known incompatibilities. + +=item I + +Support this version. + +=item IB<+> + +Support this and all greater versions. + +=item B<#>I<...> + +Comment. + +=back + +For a version to be used, it must also be listed in the output of +C. See this file for how to +configure the list of supported versions on your system. + +=head1 EXAMPLE + +=over 4 + +=item B + + Source: postgresql-foobar + Rules-Requires-Root: no + Build-Depends: + debhelper, + postgresql-all , + postgresql-server-dev-all (>= 217~), + + Package: postgresql-PGVERSION-foobar + Architecture: any + Depends: + ${misc:Depends}, + ${postgresql:Depends}, + ${shlibs:Depends}, + +=item B + + all + + # alternatives: + #9.6 + #11+ + +=item B using B: + + #!/usr/bin/make -f + + override_dh_installdocs: + dh_installdocs --all README.* + + %: + dh $@ --with pgxs + +=item If the package does no support building from subdirectories, use B: + + #!/usr/bin/make -f + + %: + dh $@ --with pgxs_loop + +=item If the package does not use PGXS's "make installcheck" for testing: + + override_dh_pgxs_test: + +=item B using B directly: + + #!/usr/bin/make -f + + include /usr/share/postgresql-common/pgxs_debian_control.mk + + # omit this if the package does not use autoconf + override_dh_auto_configure: + +pg_buildext configure build-%v "--libdir=/usr/lib/postgresql/%v/lib --datadir=/usr/share/postgresql-%v-foobar" + + override_dh_auto_build: + +pg_buildext build build-%v + + override_dh_auto_test: + # nothing to do here, see debian/tests/* instead + + override_dh_auto_install: + +pg_buildext install build-%v postgresql-%v-foobar + + override_dh_installdocs: + dh_installdocs --all README.* + + override_dh_auto_clean: + +pg_buildext clean build-%v + + %: + dh $@ + +=item B + + Depends: @, postgresql-server-dev-all + Tests: installcheck + Restrictions: allow-stderr + +=item B (optional) + + Depends: @, postgresql-contrib-PGVERSION, postgresql-PGVERSION-bar + Tests: installcheck + Restrictions: allow-stderr + +=item B + + #!/bin/sh + pg_buildext installcheck + # alternatively: pg_buildext installcheck build-%v + + # Running extra code before invoking the actual action: + set -e + for v in $(pg_buildext installed-versions); do + test -L build-$v/sql || ln -s ../sql build-$v/ + test -L build-$v/expected || ln -s ../expected build-$v/ + pg_buildext installcheck-$v build-$v + done + +=back + +=head1 SOURCE DIRECTORY + +If the package source code is not in the top level directory (i.e. the directory +which has C as subdirectory), use the I argument, where +I must be an absolute path. Example: + + override_dh_auto_build: + +pg_buildext build $(CURDIR)/postgresql-module build-%v + +=head1 COMPATIBILITY + +B was introduced in postgresql-server-dev-all (>= 141~). + +The usage of "all" or "NN+" in debian/pgversions was introduced in +postgresql-server-dev-all (>= 148~). + +B was introduced in postgresql-server-dev-all (>= +153~). + +B was introduced in postgresql-common (>= 170~). + +Handling of C with B replacement was +introduced in postgresql-common (>= 171~). + +The action B was introduced in postgresql-common (>= 208~). +B was switched to use it in the same version. + +B and B, the corresponding B<--buildsystem>, and +the B and B actions were introduced in postgresql-server-dev-all (>= 217~). + +The replacement of B (plural) in debian/control.in and +B and B were introduced in +postgresql-common (>= 256~). + + +=head1 SEE ALSO + +C, autopkgtest(1), +pg_virtualenv(1). + +=head1 AUTHORS + +Dimitri Fontaine Ldim@tapoueh.orgE>, with extensions by +Christoph Berg Lmyon@debian.orgE>. diff --git a/pg_checksystem b/pg_checksystem new file mode 100755 index 0000000..bf61fef --- /dev/null +++ b/pg_checksystem @@ -0,0 +1,59 @@ +#!/usr/bin/perl -wT + +# Check various system parameters for PostgreSQL. This needs to be run as root. +# +# (C) 2005-2009 Martin Pitt +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; + +use PgCommon; + +# Check write cache setting for given drive. +# Arguments: +# Returns: 0 = disabled, 1 = enabled, -1: could not determine +sub get_device_write_cache { + open DRV, $_[0] or return -1; + sub HDIO_GET_WCACHE () {0x30e;} + my $pval = pack 'l', 0; + ioctl DRV, &HDIO_GET_WCACHE, $pval or return -1; + my ($val) = unpack 'l', $pval; + close DRV; + return $val; +} + + +error 'This command needs to be executed as root' if $> != 0; + +# collect write cache-enabled drives with databases +my %drive_wcache; +my %seen; +for my $v (get_versions) { + for my $c (get_version_clusters $v) { + my $datadir = PgCommon::cluster_data_directory $v, $c; + my $drv = PgCommon::get_file_device $datadir; + $drv =~ s/[0-9]+$//; + unless (exists $seen{$drv}) { + $drive_wcache{$drv} = 1 if (get_device_write_cache $drv) > 0; + $seen{$drv} = 1; + } + } +} +my @unsave_drives = keys %drive_wcache; + +if (@unsave_drives) { + print 'Warning: The following devices contain databases and have write +caching enabled: ', (join ' ', @unsave_drives), ' +This could destroy the integrity of your databases in the event of power +failure. Consider disabling the write cache with "hdparm -W 0 ". +'; +} diff --git a/pg_config b/pg_config new file mode 100755 index 0000000..efdcf6d --- /dev/null +++ b/pg_config @@ -0,0 +1,33 @@ +#!/bin/sh + +# If postgresql-server-dev-* is installed, call pg_config from the latest +# available one. Otherwise fall back to libpq-dev's version. +# +# (C) 2011 Martin Pitt +# (C) 2014-2018 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +set -e +PGBINROOT="/usr/lib/postgresql/" +#redhat# PGBINROOT="/usr/pgsql-" +LATEST_SERVER_DEV=`ls -v $PGBINROOT*/bin/pg_config 2>/dev/null|tail -n1` + +if [ -n "$LATEST_SERVER_DEV" ]; then + exec "$LATEST_SERVER_DEV" "$@" +else + if [ -x /usr/bin/pg_config.libpq-dev ]; then + exec /usr/bin/pg_config.libpq-dev "$@" + else + echo "You need to install postgresql-server-dev-NN for building a server-side extension or libpq-dev for building a client-side application." >&2 + exit 1 + fi +fi diff --git a/pg_conftool b/pg_conftool new file mode 100755 index 0000000..6772b66 --- /dev/null +++ b/pg_conftool @@ -0,0 +1,233 @@ +#!/usr/bin/perl +# Read and edit PostgreSQL config files +# vim:sw=4:et: +# +# (C) 2014-2017 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use warnings; +use Getopt::Long; +use PgCommon qw(quote_conf_value user_cluster_map read_cluster_conf_file + read_conf_file set_conffile_value set_conf_value disable_conffile_value + disable_conf_value cluster_exists error config_bool); + +## option parsing + +sub help ($;$) +{ + my ($exit, $error) = @_; + print STDERR "Error: $error\n" if ($error); + print "Usage: + $0 [options] [ ] [] + +Options: + -b --boolean Format output as boolean + -s --short Print only value + -v --verbose Verbose output + --help This help + +Commands: + show |all + set + remove + edit +"; + exit $exit; +} + +my $boolean = 0; +my $short = 0; +my $verbose = 0; + +Getopt::Long::Configure ("bundling"); +help(1) unless GetOptions ( + 'help' => sub { help(0); }, + 'b|boolean' => \$boolean, + 's|short' => \$short, + 'v|verbose' => \$verbose, +); + +# find command in argument array +my $cmdidx; +for (my $i = 0; $i < @ARGV; $i++) { + if ($ARGV[$i] =~ /^(show|get|set|remove|delete|disable|edit)$/) { + $cmdidx = $i; + last; + } +} +help(1, 'No command given') unless (defined $cmdidx); + +my ($version, $cluster, $conffile); +if ($cmdidx == 0) { # operate on postgresql.conf in default cluster + ($version, $cluster) = user_cluster_map(); + error "No default cluster found" unless ($cluster); + $conffile = 'postgresql.conf'; +} elsif ($cmdidx == 1) { # operate on given file; default cluster must exist if file is relative + $conffile = $ARGV[0]; + unless ($conffile =~ m!^/!) { + ($version, $cluster) = user_cluster_map(); + error "No default cluster found" unless ($cluster); + } +} elsif ($cmdidx == 2) { # version cluster, postgresql.conf + ($version, $cluster, $conffile) = ($ARGV[0], $ARGV[1], 'postgresql.conf'); +} elsif ($cmdidx == 3) { # version cluster conffile + ($version, $cluster, $conffile) = ($ARGV[0], $ARGV[1], $ARGV[2]); +} else { + help(1, 'Too many arguments before command'); +} + +if ($cluster) { # the cluster needs to exist unless an absolute conffile was given + error "Cluster $version $cluster does not exist" + unless (cluster_exists $version, $cluster); +} + +splice @ARGV, 0, $cmdidx; # remove everything before the command +$ARGV[0] = 'show' if ($ARGV[0] eq 'get'); # accept alternative variants of some commands +$ARGV[0] = 'remove' if ($ARGV[0] =~ /delete|disable/); + +my ($cmd, $key, $value); +if ($ARGV[0] =~ /^(edit)$/) { + help(1, "$ARGV[0] takes no argument") unless (@ARGV == 1); + ($cmd) = @ARGV; +} elsif ($ARGV[0] =~ /^(show|remove)$/) { + help(1, "$ARGV[0] needs exactly one argument") unless (@ARGV == 2); + ($cmd, $key) = @ARGV; +} else { + help(1, "$ARGV[0] needs exactly two arguments") unless (@ARGV == 3); + ($cmd, $key, $value) = @ARGV; +} +#print "$version $cluster $conffile $cmd $key $value\n"; + +## main + +if ($cmd eq 'show') { + my %conf; + if ($conffile =~ m!^/!) { + %conf = read_conf_file ($conffile); + } else { + %conf = read_cluster_conf_file ($version, $cluster, $conffile); + } + + if ($key eq 'all') { + foreach my $k (sort keys %conf) { + printf "%s = %s\n", $k, quote_conf_value($conf{$k}); + } + } elsif (exists $conf{$key}) { + $conf{$key} = config_bool($conf{$key}) ? 'on' : 'off' if ($boolean); # normalize boolean output on request + if ($short) { + printf "%s\n", $conf{$key}; + } else { + printf "%s = %s\n", $key, quote_conf_value($conf{$key}); + } + } else { + print "# $key not found in $conffile\n" if ($verbose); + exit 1; + } + +} elsif ($cmd eq 'set') { + if ($conffile =~ m!^/!) { + set_conffile_value ($conffile, $key, $value); + } else { + set_conf_value ($version, $cluster, $conffile, $key, $value); + } + +} elsif ($cmd eq 'remove') { + if ($conffile =~ m!^/!) { + disable_conffile_value ($conffile, $key); + } else { + disable_conf_value ($version, $cluster, $conffile, $key); + } + +} elsif ($cmd eq 'edit') { + my $editor = 'vi'; + if ($ENV{EDITOR}) { + ($editor) = $ENV{EDITOR} =~ /(.*)/; # untaint + } + if ($conffile =~ m!^/!) { + system $editor, $conffile; + } else { + system $editor, "$PgCommon::confroot/$version/$cluster/$conffile"; + } +} + +__END__ + +=head1 NAME + +pg_conftool - read and edit PostgreSQL cluster configuration files + +=head1 SYNOPSIS + +B [I] [I I] [I] B + +=head1 DESCRIPTION + +B allows showing and setting parameters in PostgreSQL configuration +files. + +If I I is omitted, it defaults to the default cluster (see +user_clusters(5) and postgresqlrc(5)). If I is omitted, it defaults +to B. I can also be a path, in which case +I I is ignored. + +=head1 OPTIONS + +=over 4 + +=item B<-b>, B<--boolean> + +Format boolean value as B or B (not for "show all"). + +=item B<-s>, B<--short> + +Show only the value (instead of key = value pair). + +=item B<-v>, B<--verbose> + +Verbose output. + +=item B<--help> + +Print help. + +=back + +=head1 COMMANDS + +=over 4 + +=item B I|B + +Show a parameter, or all present in this config file. + +=item B I I + +Set or update a parameter. + +=item B I + +Remove (comment out) a parameter from a config file. + +=item B + +Open the config file in an editor. Unless B<$EDITOR> is set, B is used. + +=back + +=head1 SEE ALSO + +user_clusters(5), postgresqlrc(5) + +=head1 AUTHOR + +Christoph Berg Lmyon@debian.orgE> diff --git a/pg_createcluster b/pg_createcluster new file mode 100755 index 0000000..66f0b82 --- /dev/null +++ b/pg_createcluster @@ -0,0 +1,952 @@ +#!/usr/bin/perl -w + +# Create new PostgreSQL cluster or integrate an existing data directory into +# the postgresql-common infrastructure. +# +# (C) 2005-2013 Martin Pitt +# (C) 2012-2021 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use warnings; +use PgCommon; +use Getopt::Long; +use POSIX qw/lchown setlocale LC_ALL LC_CTYPE/; + +$ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin'; # untaint + +my @postgres_user = getpwnam 'postgres'; +my %defaultconf; +my $explicit_auth_config = 0; +my $quiet; + +chdir '/'; + +# call initdb +sub init_db { + my ($version, $cluster, $datadir, $owneruid, $ownergid, $local_method, $host_method, $initdb_opts_from_cli) = @_; + $datadir = readlink $datadir if (-l $datadir); + + if (system 'install', '-d', '-o', $owneruid, '-g', $ownergid, $datadir) { + error 'could not create data directory; you might need to run this program with root privileges'; + } + + # disable copy-on-write semantics for PostgreSQL data on btrfs and similar; + # this fails on file systems which don't support it, so ignore errors + system "chattr +C '$datadir' 2>/dev/null"; + + my @initdb = (get_program_path 'initdb', $version); + die 'Internal error: could not determine initdb path' unless $initdb[0]; + push @initdb, ('-D', $datadir); + if (my $waldir = $defaultconf{'waldir'} || $defaultconf{'xlogdir'}) { + my $wal = $version >= 10 ? 'wal' : 'xlog'; # renamed in PG 10 + push @initdb, ("--${wal}dir", replace_v_c ($waldir, $version, $cluster)); + } + unless ($explicit_auth_config) { + if ($version >= '9.2') { + push @initdb, ('--auth-local', $local_method); + push @initdb, ('--auth-host', $host_method); + } else { + # trust is the default, but set it explicitly to mute a warning from initdb + # the actual method will be filled in by setup_pg_hba() + push @initdb, ('-A', 'trust'); + } + } + + # cluster startup message + if ($version >= 14) { + push @initdb, '--no-instructions'; + } else { + # ask initdb to print a different cluster start command (Debian patch) + $ENV{CLUSTER_START_COMMAND} = "pg_ctlcluster $version $cluster start"; + } + + # options from config and command line + if (my $options = $defaultconf{'initdb_options'}) { + push @initdb, split (/\s+/, replace_v_c ($options, $version, $cluster)); + } + push @initdb, @$initdb_opts_from_cli; + + if (fork) { + wait; + error 'initdb failed' if $?; + } else{ + change_ugid $owneruid, $ownergid; + print "@initdb\n" unless ($quiet); + close STDOUT if ($quiet); # suppress initdb output + exec @initdb; + } +} + +# move a file to a directory with defined permissions; set in +# postgresql.conf. +# Arguments: +sub move_conffile { + my ($file, $target, $version, $cluster, $uid, $gid, $perms, $confopt) = @_; + my $realfile = $file; + while (-l $realfile) { + $realfile = readlink $realfile; + } + if (-e $file) { + install_file $realfile, $target, $uid, $gid, $perms; + unlink $file, $realfile; + + my @pathcomps = split ('/', $file); + $target .= '/' . $pathcomps[-1]; + PgCommon::set_conf_value $version, $cluster, 'postgresql.conf', $confopt, $target if $confopt; + } else { + error "move_conffile: required configuration file $realfile does not exist"; + } +} + +# Set up the default pg_hba.conf file: +# - Add a "local all" entry to pg_hba.conf for the db superuser before all +# other entries. +# - Change default authentication for host entries to md5. +sub setup_pg_hba { + my ($version, $owneruid, $confdir, $local_method, $host_method) = @_; + my $user = (getpwuid $owneruid)[0]; + my $fname = "$confdir/pg_hba.conf"; + my $su_comment = " +# DO NOT DISABLE! +# If you change this first entry you will need to make sure that the +# database superuser can access the database using some other method. +# Noninteractive access to all databases is required during automatic +# maintenance (custom daily cronjobs, replication, and similar tasks). +# +# Database administrative login by Unix domain socket +"; + + open my $hba, '<', $fname or error "could not open $fname for reading"; + my $search = 1; + my @lines; + while (my $line = <$hba>) { + # add superuser entry before column description line + if ($search && $line =~ /#.*TYPE\s+DATABASE/) { + push @lines, $su_comment; + push @lines, sprintf "%-7s %-15s %-39s %s\n", 'local', 'all', $user, $local_method; + push @lines, "\n"; + $search = 0; + } + + if ($version < '9.2' and not $explicit_auth_config) { + # default authentication for Unix socket connections + if ($line =~ /^#?local/) { + $line =~ s/trust/$local_method/; + } + + # default authentication for TCP connections + if ($line =~ /^#?host/) { + $line =~ s/trust/$host_method/; + } + } + + push @lines, $line; + } + close $hba; + + error "setup_pg_hba: did not find insert position" if $search; + + open my $new_hba, '>', $fname or error "could not open $fname for writing"; + foreach (@lines) { + print $new_hba $_; + } + close $new_hba; +} + +# +# Execution starts here +# + +# this flag gets set when we started creating the cluster, and thus we need to +# clean up on errors +my $cleanup_cruft = 0; + +# command line arguments + +my $startconf = ''; +my @pgoptions; +my $createclusterconf = "$PgCommon::common_confdir/createcluster.conf"; +my $environmentfile = "$PgCommon::common_confdir/environment"; +my ($owneruid, $ownergid, $socketdir, $datadir, $custom_logfile, $start, $port); +my ($encoding, $locale, $lc_collate, $lc_ctype, $lc_messages, $lc_monetary, $lc_numeric, $lc_time); +my ($no_status); + +exit 1 unless GetOptions ('u|user=s' => \$owneruid, 'g|group=s' => \$ownergid, + 's|socketdir=s' => \$socketdir, 'd|datadir=s' => \$datadir, + 'start' => \$start, 'e|encoding=s' => \$encoding, + 'l|logfile=s' => \$custom_logfile, 'start-conf=s' => \$startconf, + 'o|pgoption=s' => sub { $_[1] =~ /(.*?)=(.*)/ or error ("No '=' in pgoption '$_[1]'"); + push @pgoptions, [$1, $2]; + }, + 'createclusterconf=s' => \$createclusterconf, + 'environment=s' => \$environmentfile, + 'no-status' => \$no_status, + 'p|port=i' => \$port, + 'q|quiet' => \$quiet, + 'locale=s' => \$locale, + 'lc-collate=s' => \$lc_collate, 'lc-ctype=s' => \$lc_ctype, + 'lc-messages=s' => \$lc_messages, 'lc-monetary=s' => \$lc_monetary, + 'lc-numeric=s' => \$lc_numeric, 'lc-time=s' => \$lc_time); + +# read defaults from /etc/postgresql-common/createcluster.conf +%defaultconf = PgCommon::read_conf_file ($createclusterconf); +# process --pgoption parameters +foreach my $guc (@pgoptions) { + if ($guc->[1] eq '') { + delete $defaultconf{$guc->[0]}; + } else { + $defaultconf{$guc->[0]} = $guc->[1]; + } +} + +$explicit_auth_config = 1 if ($defaultconf{initdb_options} and $defaultconf{initdb_options} =~ /(^| )(-\w*A|--auth\b)/); + +# check validity of locale +unless (setlocale (LC_ALL, "")) { + my $env = join "\n", + map { " $_: $ENV{$_}" } + grep { /^(LC_|LANG)/ } sort keys %ENV; + error ("The locale requested by the environment is invalid:\n$env") +} + +if (@ARGV < 2) { + print "Usage: $0 [options] [-- ] + +Options: + -u cluster owner and superuser (default: 'postgres') + -g group for data files (default: primary group of owner) + -d data directory (default: + /var/lib/postgresql//) + -s socket directory (default: /var/run/postgresql for clusters + owned by 'postgres', /tmp for other clusters) + -l path to desired log file (default: + /var/log/postgresql/postgresql--.log) + --locale + set cluster locale (default: inherit from environment) + --lc-collate/ctype/messages/monetary/numeric/time + like --locale, but only set for a particular category + -e Default encoding (default: derived from locale) + -p port number (default: next free port starting from 5432) + --start start the cluster after creating it + --start-conf auto|manual|disabled + Set automatic startup behaviour in start.conf (default: 'auto') + --createclusterconf=file alternative createcluster.conf to use + --environment=file alternative environment file to use + other options to pass to initdb +"; + exit 1; +} + +$startconf ||= $defaultconf{'start_conf'} || 'auto'; +error "Invalid --start-conf value: $startconf" if $startconf ne 'auto' && + $startconf ne 'manual' && $startconf ne 'disabled'; + +if ($owneruid) { + $owneruid = (getpwnam $owneruid)[2] unless $owneruid =~ /^\d+$/; +} elsif ($> == 0) { + $owneruid = getpwnam 'postgres'; + error 'User postgres does not exist' unless $owneruid; +} else +{ + $owneruid = $>; +} + +if ($ownergid) { + $ownergid = (getgrnam $ownergid)[2] unless $ownergid =~ /^\d+$/; +} else { + $ownergid = (getpwuid $owneruid)[3]; +} + +error 'clusters must not be owned by root' unless $owneruid; + +my ($version) = $ARGV[0] =~ /^(\d+\.?\d+)$/; +error "invalid version '$ARGV[0]'" unless defined $version; +my ($cluster) = $ARGV[1] =~ /^([-.\w]+)$/; +error "invalid cluster name '$ARGV[1]'" unless defined $cluster; +if ($cluster =~ /-/ and -t 1) { + print "Warning: cluster names containing dashes (-) will cause problems when running from systemd. Continuing anyway\n"; +} +splice @ARGV, 0, 2; + +my @initdb_opts_from_cli; +# options passed through to initdb +push @initdb_opts_from_cli, ('--encoding', $encoding) if $encoding; +push @initdb_opts_from_cli, ('--locale', $locale) if $locale; +push @initdb_opts_from_cli, ('--lc-collate', $lc_collate) if $lc_collate; +push @initdb_opts_from_cli, ('--lc-ctype', $lc_ctype) if $lc_ctype; +push @initdb_opts_from_cli, ('--lc-messages', $lc_messages) if $lc_messages; +push @initdb_opts_from_cli, ('--lc-monetary', $lc_monetary) if $lc_monetary; +push @initdb_opts_from_cli, ('--lc-numeric', $lc_numeric) if $lc_numeric; +push @initdb_opts_from_cli, ('--lc-time', $lc_time) if $lc_time; + +# initdb options passed after -- +foreach my $argv (@ARGV) { + # the user passed an authentication method, don't mess with initdb and pg_hba.conf + if ($argv =~ /^(-\w*A|--auth\b)/) { # -A --auth --auth-host --auth-local + $explicit_auth_config = 1; + } + push @initdb_opts_from_cli, $argv =~ /(.*)/; # untaint +} + +# pg_hba.conf authentication settings +my $local_method = $version >= 9.1 ? 'peer' : + ($version >= 8.4 ? 'ident' : + 'ident sameuser'); +my $host_method = $version >= 14 ? 'scram-sha-256' : 'md5'; + +# create parent of data directory if missing +my $datadirp_created; +if (!defined $datadir) { + $datadir = replace_v_c ($defaultconf{'data_directory'} || "/var/lib/postgresql/%v/%c", $version, $cluster); + $datadir =~ s!/+$!!; + my $pd = $datadir; + $pd =~ s!/[^/]*$!!; + + # ensure that the version data dir is owned by postgres as well, so that + # it can be administrated without root permissions + if (!stat $pd) { + my @install = qw(install -d); + push @install, '-o', $postgres_user[2], '-g', $postgres_user[3] if ($> == 0); + system @install, $pd; + $datadirp_created = $pd; # clean up in case of error + } +} +my $confdirp = "$PgCommon::confroot/$version"; +my $confdir = "$confdirp/$cluster"; + +# some sanity checks +error "no initdb program for version $version found" unless get_program_path 'initdb', $version; +error 'cluster configuration already exists' + if -e "$confdir/postgresql.conf" || -e "$confdir/pg_hba.conf"; + +if (defined $port) { + error 'port must be a positive integer between 1024 and 65535' + unless $port =~ /^\d+/ && $port >= 1024 && $port <= 65535; +} else { + $port = next_free_port; +} + +# create configuration directory +if (!stat $confdirp) { + my @install = qw(install -d); + push @install, '-o', $postgres_user[2], '-g', $postgres_user[3] if ($> == 0); + system @install, $confdirp; +} +# now we created the first new directory for this cluster and start to rollback +# on error +$cleanup_cruft = 1; + +error 'could not create configuration directory; you might ' . + 'need to run this program with root privileges' if system ('install', '-d', $confdir); + +# check whether we have an already existing cluster; check version and +# determine owner in this case +my $newcluster = 0; + +if (-f "$datadir/PG_VERSION") { + open my $fh, '<', "$datadir/PG_VERSION" or error "could not open $datadir/PG_VERSION"; + chomp(my $existingver = <$fh>); + close $fh; + + ($owneruid, $ownergid) = (stat "$datadir/PG_VERSION")[4,5]; + if ($existingver == $version) { + print "Configuring already existing cluster (configuration: $confdir, data: $datadir, owner: $owneruid:$ownergid)\n"; + } else { + error "$datadir contains a version $existingver cluster, but $version was requested"; + } +} else { + print "Creating new PostgreSQL cluster $version/$cluster ...\n"; + init_db $version, $cluster, $datadir, $owneruid, $ownergid, + $local_method, $host_method, \@initdb_opts_from_cli; + $newcluster = 1; +} + +# create default "start" file +set_cluster_start_conf $version, $cluster, $startconf; + +# create default pg_ctl.conf file +set_cluster_pg_ctl_conf $version, $cluster, ''; + +# move conffiles, setup permissions +move_conffile "$datadir/postgresql.conf", $confdir, $version, $cluster, $owneruid, $ownergid, '644'; +move_conffile "$datadir/pg_hba.conf", $confdir, $version, $cluster, $owneruid, $ownergid, '640', 'hba_file'; +move_conffile "$datadir/pg_ident.conf", $confdir, $version, $cluster, $owneruid, $ownergid, '640', 'ident_file'; +foreach my $f ($datadir, $confdir, "$confdir/start.conf", "$confdir/pg_ctl.conf") { + lchown $owneruid, $ownergid, $f or error "lchown $f: $!"; +} + +PgCommon::set_conf_value $version, $cluster, 'postgresql.conf', 'data_directory', $datadir; + +# add access for database superuser +setup_pg_hba $version, $owneruid, $confdir, $local_method, $host_method + if $newcluster and not $explicit_auth_config; + +# configure socket directory +if (! $socketdir && ! -e '/var/run/postgresql') { + system 'install', '-d', '-o', $postgres_user[2], '-g', $postgres_user[3], '/var/run/postgresql'; +} +if ($socketdir && ! -e $socketdir) { + if (system 'install', '-d', '-m', '0755', '-o', $owneruid, '-g', $ownergid, $socketdir) { + error "could not create socket directory $socketdir"; + } +} +my $orig_euid = $>; +my $orig_egid = $); +$) = $ownergid; +$> = $owneruid; +unless ($socketdir) { + if ($version < 9.4 and $PgCommon::rpm) { + $socketdir = '/tmp'; # PGDG 9.3 and earlier defaults to /tmp in libpq et al. + } elsif (-w '/var/run/postgresql') { + $socketdir = '/var/run/postgresql'; + } else { + $socketdir='/tmp'; + } +} +set_cluster_socketdir $version, $cluster, $socketdir; +$> = $orig_euid; +$) = $orig_egid; + +set_cluster_port $version, $cluster, $port; + +# create log file +if (! -d '/var/log/postgresql') { + mkdir '/var/log/postgresql' or + error "could not create log directory; you might need to run this program with root privileges"; + chmod 01775, '/var/log/postgresql'; + lchown 0, $postgres_user[3], '/var/log/postgresql'; +} +my $real_logfile = $custom_logfile || "/var/log/postgresql/postgresql-$version-$cluster.log"; +error "logfile $real_logfile is a directory, not a file" if (-d $real_logfile); +if (! -e $real_logfile) { + open my $fh, '>>', $real_logfile or error "could not create log file $real_logfile"; +} +chmod 0640, $real_logfile; +my $g; +if ($owneruid < 500) { + $g = (getgrnam 'adm')[2]; +} else { + $g = $ownergid; +} +lchown $owneruid, $g, $real_logfile; +# if we are using a non-default log file, create a log symlink +if ($custom_logfile) { + symlink $real_logfile, "$confdir/log"; + lchown $owneruid, $ownergid, "$confdir/log"; +} + +# SSL configuration +my $want_ssl = PgCommon::config_bool($defaultconf{ssl} || 'on'); +# older versions (<= 9.1 as of 2019-03) do not support ssl anymore +my $postgres = get_program_path('postgres', $version); +my $ldd = `ldd $postgres 2>/dev/null`; +$want_ssl = 0 if ($ldd and $ldd !~ /libssl/); + +# Check whether we can access the SSL private key as the cluster owner +my $ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key'; +my $ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem'; +my $ssl_ca_file = "$PgCommon::common_confdir/root.crt"; +my $ssl_key_access; +my ($uid, $euid, $gid, $egid) = ($<, $>, $(, $)); +change_ugid $owneruid, $ownergid; +$ssl_key_access = -r $ssl_key_file; + +# check for stats_temp_directory access +delete $defaultconf{stats_temp_directory} if ($version < 8.4 or $version >= 15); +if ($defaultconf{stats_temp_directory}) { + my $stats_temp_directory = replace_v_c ($defaultconf{stats_temp_directory}, $version, $cluster); + $stats_temp_directory =~ s!/$!!; # strip trailing slash + my $stats_temp_parent = $stats_temp_directory; + $stats_temp_parent =~ s!/[^/]+$!!; + if (-d $stats_temp_directory) { + if (! -w $stats_temp_directory) { + print "Warning: The selected stats_temp_directory $stats_temp_directory +is not writable for the cluster owner. Not adding this setting in +postgresql.conf.\n"; + delete $defaultconf{stats_temp_directory}; + } + } elsif (! -d $stats_temp_parent) { + print "Warning: The parent $stats_temp_parent of the selected +stats_temp_directory does not exist. Not adding this setting in +postgresql.conf.\n"; + delete $defaultconf{stats_temp_directory}; + } elsif (! -w $stats_temp_parent) { + print "Warning: The parent $stats_temp_parent of the selected +stats_temp_directory is not writable for the cluster owner. Not adding this +setting in postgresql.conf.\n"; + delete $defaultconf{stats_temp_directory}; + } + # create the stats directory now. pg_ctlcluster would create it anyway, but + # when using pg_upgradecluster -m upgrade, it is not run before the cluster + # is started for the first time + if ($defaultconf{stats_temp_directory}) { + system 'install', '-d', '-m', '750', '-o', $owneruid, '-g', $ownergid, $stats_temp_directory; + } +} + +$> = $euid; +$< = $uid; +$( = $gid; +$) = $egid; +die "changing euid back: $!" if $> != $euid; +die "changing egid back: $!" if $) != $egid; + +# enable SSL if we have the snakeoil default certificate +if ($want_ssl && $newcluster && -e $ssl_cert_file && $ssl_key_access) { + if ($version >= '9.2') { + PgCommon::set_conf_value $version, $cluster, 'postgresql.conf', + 'ssl_cert_file', $ssl_cert_file; + PgCommon::set_conf_value $version, $cluster, 'postgresql.conf', + 'ssl_key_file', $ssl_key_file; + } else { + symlink $ssl_cert_file, "$datadir/server.crt"; + symlink $ssl_key_file, "$datadir/server.key"; + } + + PgCommon::set_conf_value $version, $cluster, 'postgresql.conf', 'ssl', 'on'; +} + +# SSL client certificate CA +if ($want_ssl && $newcluster && -e $ssl_ca_file) { + # check if we have a cert in there or just the boilerplate installed by our postinst + open my $fh, '<', $ssl_ca_file or error "could not open $ssl_ca_file for reading"; + my $val; + read $fh, $val, 4096; + if ($val =~ /^-----BEGIN CERTIFICATE-----/m) { + if ($version >= '9.2') { + PgCommon::set_conf_value $version, $cluster, 'postgresql.conf', + 'ssl_ca_file', $ssl_ca_file; + } else { + symlink $ssl_ca_file, $datadir.'/root.crt'; + } + } +} + +# SSL client certificate revocation list +if ($want_ssl && $newcluster && -e "$PgCommon::common_confdir/root.crl") { + if ($version >= '9.2') { + PgCommon::set_conf_value $version, $cluster, 'postgresql.conf', + 'ssl_crl_file', "$PgCommon::common_confdir/root.crl"; + } else { + symlink "$PgCommon::common_confdir/root.crl", $datadir.'/root.crl'; + } +} + +# create default (empty) environment file +my $defaultenv = "# environment variables for postgres processes +# This file has the same syntax as postgresql.conf: +# VARIABLE = simple_value +# VARIABLE2 = 'any value!' +# I. e. you need to enclose any value which does not only consist of letters, +# numbers, and '-', '_', '.' in single quotes. Shell commands are not +# evaluated. +"; +if (-e $environmentfile) { + open my $env, '<', $environmentfile or error "could not read environment file $environmentfile"; + local $/; # slurp mode + $defaultenv = <$env>; +} +$defaultenv = replace_v_c ($defaultenv, $version, $cluster); +open my $env, '>', "$confdir/environment" or error "could not create environment file $confdir/environment"; +print $env $defaultenv; +close $env; +chmod 0644, "$confdir/environment"; +lchown $owneruid, $ownergid, "$confdir/environment"; + +$cleanup_cruft = 0; + +# configure to create external PID file +if ($socketdir eq '/var/run/postgresql') { + PgCommon::set_conf_value $version, $cluster, 'postgresql.conf', 'external_pid_file', "/var/run/postgresql/$version-$cluster.pid"; +} + +# handle other createcluster.conf parameters, including --pgoption parameters +foreach my $guc (sort keys %defaultconf) { + next if $guc =~ /^(create_main_cluster|start_conf|data_directory|waldir|xlogdir|initdb_options|ssl)$/; + next if $guc eq 'logging_collector' and $version < 8.3; + next if $guc eq 'cluster_name' and $version < 9.5; + my $val = replace_v_c ($defaultconf{$guc}, $version, $cluster); + $guc =~ s/^add_include/include/; # remove harness from include directives in createcluster.conf + if ($guc eq 'include_dir') { + next if ($version < 9.3); + if ($val =~ /^[\w.]+$/ and not -e "$confdir/$val") { # create directory relative to new config directory + mkdir "$confdir/$val", 0755; + lchown $owneruid, $ownergid, "$confdir/$val"; + } + } + PgCommon::set_conf_value $version, $cluster, 'postgresql.conf', $guc, $val; +} + +# notify systemd about the new cluster +if (not exists $ENV{'PG_CLUSTER_CONF_ROOT'} and $startconf eq 'auto' and -d '/run/systemd/system') { + if ($> == 0) { + system 'systemctl daemon-reload'; + } elsif (-t 1) { + print "Warning: systemd does not know about the new cluster yet. Operations like \"service postgresql start\" will not handle it. To fix, run:\n"; + print " sudo systemctl daemon-reload\n"; + } +} + +# notify apt about the new cluster +if (not exists $ENV{'PG_CLUSTER_CONF_ROOT'} and $> == 0) { + system "/usr/share/postgresql-common/pg_updateaptconfig"; +} + +# start it if requested +if ($start) { + system 'pg_ctlcluster', $version, $cluster, 'start'; + die "Could not start cluster\n" if ($?); +} + +# finally, show the cluster we created +system 'pg_lsclusters', $version, $cluster unless ($quiet or $no_status); + +END { + # clean up cruft if something went wrong + if ($cleanup_cruft && defined $version && defined $cluster) { + system "pg_dropcluster $version $cluster 2>/dev/null"; + rmdir $datadirp_created if ($datadirp_created); # clean up after early errors which pg_dropcluster doesn't handle + exit 1; + } +} + +__END__ + +=head1 NAME + +pg_createcluster - create a new PostgreSQL cluster + +=head1 SYNOPSIS + +B [I] I I [B<--> I] + +=head1 DESCRIPTION + +B creates a new PostgreSQL server cluster (i. e. a +collection of databases served by a L instance) and +integrates it into the multi-version/multi-cluster architecture of the +B package. + +Every cluster is uniquely identified by its version and name. The name can be +arbitrary. The default cluster that is created on installation of a server +package is C
. However, you might wish to create other clusters for +testing, with other superusers, a cluster for each user on a shared server, +etc. C will abort with an error if you try to create a +cluster with a name that already exists for that version. + +For compatibility with B service units, the cluster name should not +contain any dashes (B<->). B will warn about the problem, but +succeed with the operation. + +Given a major PostgreSQL I (like "8.2" or "8.3") and a cluster +I, it creates the necessary configuration files in +CICIC; in particular these are +C, C, C, a postgresql-common +specific configuration file C (see B below), +C, and a symbolic link C which points to the log file (by +default, CIC<->IC<.log>). + +C is automatically adapted to use the next available port, i. +e. the first port (starting from 5432) which is not yet used by an already +existing cluster. + +If the data directory does not yet exist, PostgreSQL's L command is +used to generate a new cluster structure. If the data directory already exists, +it is integrated into the B structure by moving the +configuration file and setting the data_directory option. Please note that this +I works for data directories which were created directly with B, i. +e. all the configuration files (C etc.) must be present in the +data directory. + +If a custom socket directory is given and it does not exist, it is created. + +If the log file does not exist, it is created. In any case the permissions are +adjusted to allow write access to the cluster owner. Please note that +C can be customized to specify C and/or +C; if at least one of these options is present, then the symbolic +link C in the cluster configuration directory is ignored. + +If the default snakeoil SSL certificate exists +(C and +C), and the C user is in the +C Unix group, B configures the cluster to use this +certificate, and enables SSL. Therefore all clusters will use the same SSL +certificate by default. For versions up to 9.1, symlinks in the data directory +will be created (C and C); for 9.2 and later, the +appropriate C options will be set (C and +C). Of course you can replace this with a cluster specific +certificate. Similarly for C and +C, these files will be configured as client +certificate CA and revocation list, when present. (C is initially a +placeholder that will only be used if real certificates are added to the file.) + +=head1 OPTIONS + +=over 4 + +=item B<-u> I, B<--user=>I + +Set the user who owns the cluster and becomes the database superuser to the +given name or uid. By default, this is the user B. A cluster must +not be owned by root. + +=item B<-g> I, B<--group=>I + +Change the group of the cluster related data files. By default this will be the +primary group of the database owner. + +=item B<-d> I, B<--datadir=>I + +Explicitly set the data directory path, which is used to store all the actual +databases and tables. This will become quite big (easily in the order of five +times the amount of actual data stored in the cluster). Defaults to +CICI. + +=item B<-s> I, B<--socketdir=>I + +Explicitly set the directory where the L server stores the Unix +socket for local connections. Defaults to C for clusters +owned by the user B, and C for clusters owned by other users. +Please be aware that C is an unsafe directory since everybody can create +a socket there and impersonate the database server. If the given directory does +not exist, it is created with appropriate permissions. + +=item B<-l> I, B<--logfile=>I + +Explicitly set the path for the L server log file. Defaults to +CIC<->IC<.log>. + +=item B<--locale=>I + +Set the default locale for the database cluster. If this option is not +specified, the locale is inherited from the environment that +B runs in. + +=item B<--lc-collate=>I + +=item B<--lc-ctype=>I + +=item B<--lc-messages=>I + +=item B<--lc-monetary=>I + +=item B<--lc-numeric=>I + +=item B<--lc-time=>I + +Like B<--locale>, but only sets the locale in the specified category. + +=item B<-e> I, B<--encoding=>I + +Select the encoding of the template database. This will also be the default +encoding of any database you create later, unless you override it there. The +default is derived from the locale, or SQL_ASCII if that does not work. The +character sets supported by the PostgreSQL server are described in the +documentation. + +B: It is not recommended to set this option directly! Set the locale +instead. + +=item B<-p> I, B<--port=>I + +Select the port the new cluster listens on (for the Unix socket and the TCP +port); this must be a number between 1024 and 65535, since PostgreSQL does not +run as root and thus needs an unprivileged port number. By default the next +free port starting from 5432 is assigned. + +=item B<-q> B<--quiet> B<--no-status> + +Suppress output from B and (or only) the cluster status message at the +end of the output. + +=item B<--start> + +Immediately start a server for the cluster after creating it (i. e. call +C I C on it). By default, the cluster is +not started. + +=item B<--start-conf=>B|B|B + +Set the initial value in the C configuration file. See B below. By default, B is used, which means that the cluster is +handled by C, i. e. starts and stops +automatically on system boot. + +=item B<-o> IB<=>I, B<--pgoption> IB<=>I + +Configuration option to set in the new C file. + +=item B<--createclusterconf=>I + +Alternative B file to use. Default is +C (or +C<$PGSYSCONFDIR/createcluster.conf>). + +=item B<--environment=>I + +Alternative default B file to use. Default is +C (or C<$PGSYSCONFDIR/environment>). +If the file is missing, a placeholder string is used. +%v and %c are replaced; see DEFAULT VALUES below. + +=item B<--> I + +Options passed directly to L. + +Per default, B will update the C file generated +by initdb to use peer authentication on local (unix) connections, and md5 on +TCP (host) connections. If explicit authentication config is included here +(B<-A>, B<--auth>, B<--auth-host>, B<--auth-local>), the C file +will be left untouched. + +I If only one of B<--auth-host> and B<--auth-local> is provided, the +other setting will default to B as per B's defaults, opening a +potential security risk. + +=back + +=head1 STARTUP CONTROL + +The C file in the cluster configuration directory controls the +start/stop behavior of that cluster's postgres process. The file can contain +comment lines (started with '#'), empty lines, and must have exactly one +line with one of the following keywords: + +=over 4 + +=item B + +The postgres process is started/stopped automatically in the init script. + +When running from B, the cluster is started/stopped when +B is started/stopped. +This is also the default if the file is missing. + +=item B + +The postgres process is not handled by the init script, but manually +controlling the cluster with L is permitted. + +When running from B, the cluster is not started automatically when +B is started. However, stopping/restarting +B will stop/restart the cluster. The cluster can be started +using BIB<->I. + +=item B + +Neither the init script, L, nor B are permitted to start/stop the +cluster. Please be aware that this will not stop the cluster owner from calling +lower level tools to control the postgres process; this option is only meant +to prevent accidents during maintenance, not more. + +=back + +When running from B, invoke B after editing +C. + +The C file in the cluster configuration directory can contain +additional options passed to B of that cluster. + +=head1 DEFAULT VALUES + +Some default values used by B can be modified in +C. Occurrences of B<%v> are replaced +by the major version number, and B<%c> by the cluster name. Use B<%%> for a +literal B<%>. + +=over 4 + +=item B (Default: B) + +Create a B
cluster when a new postgresql-NN server package is installed. + +=item B (Default: B) + +Default C value to use. + +=back + +=over 4 + +=item B (Default: B) + +Default data directory. + +=item B (Default: unset) + +Default directory for transaction logs. When used, B will create a +symlink from C (PostgreSQL 9.6 and earlier: C) in the data +directory to this location. Unset by default, i.e. transaction logs remain in +the data directory. Both spellings of this option are accepted, and translated +to the correct initdb invocation depending on the cluster version. + +=item B (Default: unset) + +Other options to pass to B. + +=item Other options + +All other options listed are copied into the new cluster's postgresql.conf, e.g.: + + listen_addresses = '*' + log_line_prefix = '%%t ' + +Some postgresql.conf options are treated specially: + +=over 4 + +=item B + +Only added to postgresql.conf if the default snakeoil certificates exist and +are readable for the cluster owner as detailed above. + +=item B + +Only added to postgresql.conf if existing, and writable for the cluster owner, +or else if the parent directory is writable. Not used on PostgreSQL 15 or later. + +=back + +=item Include files + +=over 4 + +=item B + +=item B + +=item B + +B supports the same include directives as +B. + +=item B + +=item B + +=item B + +To add include directives to the new postgresql.conf file, use the B +directives. The B prefix is removed. + +=back + +=back + +=head1 SEE ALSO + +L, L, L, L + +=head1 AUTHORS + +Martin Pitt Lmpitt@debian.orgE>, Christoph Berg Lmyon@debian.orgE> diff --git a/pg_ctlcluster b/pg_ctlcluster new file mode 100755 index 0000000..1684700 --- /dev/null +++ b/pg_ctlcluster @@ -0,0 +1,649 @@ +#!/usr/bin/perl -wT + +# multiversion/cluster aware pg_ctl wrapper; this also supplies the correct +# configuration parameters to 'start', and makes sure that postgres really +# stops on 'stop'. +# +# (C) 2005-2009 Martin Pitt +# (C) 2009 Cyril Bouthors +# (C) 2013-2021 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use warnings; +use Getopt::Long; +use POSIX qw/setsid dup2 :sys_wait_h/; +use PgCommon; +use Fcntl qw(SEEK_SET O_RDWR O_CREAT O_EXCL); +use POSIX qw(lchown); + +my ($version, $cluster, $pg_ctl, $force); +my (@postgres_auxoptions, @pg_ctl_opts_from_cli); +my (%postgresql_conf, %info); +my $mode = 'fast'; # default shutdown mode +my $foreground = 0; # don't daemonize, use postgres instead of pg_ctl +my $stdlog = 0; # when run in foreground, still log to the default logfile + +# untaint environment +$ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin'; +delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; +chdir '/'; + +# If a pid file is already present, delete it if it is stale/invalid, or exit +# with a notice if it belongs to an already running postgres. +sub start_check_pid_file { + my $pidfile = $info{'pgdata'}.'/postmaster.pid'; + if (PgCommon::check_pidfile_running $pidfile) { + print "Cluster is already running.\n"; + exit 2; + } + + # Remove invalid or stale PID file + if (-e $pidfile) { + unlink $pidfile; + print "Removed stale pid file.\n"; + } +} + +# Check if a pid file is not present or it is invalid. If so, clean up/abort. +sub stop_check_pid_file { + my $pidfile = $info{'pgdata'}.'/postmaster.pid'; + my $pid = read_pidfile $pidfile; + return if (defined $pid && PgCommon::check_pidfile_running $pidfile); + if ($info{'running'}) { + error 'pid file is invalid, please manually kill the stale server process.'; + } + + # Remove invalid or stale PID file + if (-e $pidfile) { + unlink $pidfile; + print "Removed stale pid file.\n"; + } + print "Cluster is not running.\n"; + exit 2; +} + +# check if a cluster reliably connects or fails +# Arguments: +sub cluster_port_ready { + my ($v, $c, $p, $sd) = @_; + my $psql = get_program_path('psql', $v); + error 'cluster_port_ready: could not find psql binary' unless $psql; + my $n = 0; + my $result = 0; + + # probe until we get three successful or failed connections in a row + my $nopwoption = $v >= 8.4 ? '-w' : ''; + if ($v < 8.4) { + $ENV{'PGPASSWORD'} = 'foo'; # prevent hangs if superuser cannot connect without password + } + my $out; + while ($n < ($result ? 10 : 3)) { + select undef, undef, undef, 0.5; + $out = `LC_MESSAGES=C $psql -h '$sd' --port $p $nopwoption -Xc '' template1 2>&1 >/dev/null`; + + if ($? == $result) { + $n++; + } else { + $n = 0; + } + $result = $?; + } + + if ($out =~ /FATAL:|no password supplied/) { + print STDERR "Warning: connection to the database failed, disabling startup checks:\n$out\n"; + return cluster_port_running $v, $c, $p; + } + return !$result; +} + +sub start { + start_check_pid_file; + + # check conflicting port + if (cluster_port_running $version, $cluster, $info{'port'}) { + my $sockdir = get_cluster_socketdir $version, $cluster; + error("Port conflict: another instance is already running on $sockdir with port $info{'port'}"); + } + + # prepare environment (empty except for content of 'environment', and LANG) + %ENV = read_cluster_conf_file $version, $cluster, 'environment'; + # set LANG so non-ascii chars in the server log are not replaced by '?' (affected are non-session contexts) + unless (exists $ENV{LANG}) { + my $lc_messages = PgCommon::get_conf_value $version, $cluster, 'postgresql.conf', 'lc_messages'; + $ENV{LANG} = $lc_messages if $lc_messages; + } + # 9.5 configures OOM killer using env vars + if ($version >= 9.5) { + $ENV{PG_OOM_ADJUST_FILE} = "/proc/self/oom_score_adj" unless (exists $ENV{PG_OOM_ADJUST_FILE}); + # PG_OOM_ADJUST_VALUE defaults to 0, but can be overridden here + } + + my $postgres_opts = ''; + my $usd = $version >= 9.3 ? 'unix_socket_directories' : 'unix_socket_directory'; + if (!(PgCommon::get_conf_value $version, $cluster, 'postgresql.conf', $usd)) { + $postgres_opts .= "-c $usd=\"$info{'socketdir'}\""; + } + + my $cdir = $info{'configdir'}; + $postgres_opts .= " -c config_file=\"$cdir/postgresql.conf\""; + if (!(PgCommon::get_conf_value $version, $cluster, 'postgresql.conf', 'hba_file')) { + $postgres_opts .= " -c hba_file=\"$cdir/pg_hba.conf\""; + } + if (!(PgCommon::get_conf_value $version, $cluster, 'postgresql.conf', 'ident_file')) { + $postgres_opts .= " -c ident_file=\"$cdir/pg_ident.conf\""; + } + + if ((-d '/var/run/postgresql') && !defined (PgCommon::get_conf_value $version, $cluster, 'postgresql.conf', 'external_pid_file')) { + # check whether /var/run/postgresql/ is writable as the cluster owner + my $vrp_writable; + if ($> == 0) { + change_ugid $info{'owneruid'}, $info{'ownergid'}; + $vrp_writable = -w '/var/run/postgresql'; + $< = $> = 0; + $( = $) = 0; + } else { + $vrp_writable = -w '/var/run/postgresql'; + } + if ($vrp_writable) { + $postgres_opts .= " -c external_pid_file=\"/var/run/postgresql/$version-$cluster.pid\""; + } + } + + $postgres_opts .= ' ' . (join ' ', @postgres_auxoptions); + ($postgres_opts) = $postgres_opts =~ /(.*)/; # untaint + + if ($foreground) { + if ($stdlog and $info{'logfile'}) { + my $fd = POSIX::open($info{logfile}, POSIX::O_WRONLY|POSIX::O_APPEND|POSIX::O_CREAT) or error "Could not open logfile $info{logfile}"; + dup2($fd, 1); + dup2($fd, 2); + POSIX::close($fd) or error "Could not close log fd"; + } + my $postgres = get_program_path 'postgres', $version; + error "Could not find postgres executable for version $version" unless ($postgres); + exec '/bin/sh', '-c', "exec $postgres $postgres_opts" or + error "Executing $postgres failed: $!" + } + + # only supply or default logfile path when none is given explicitly in + # postgresql.conf + my @options = ($pg_ctl, 'start', '-D', $info{'pgdata'}); + my $logsize = 0; + if ($info{'logfile'}) { + push @options, ('-l', $info{'logfile'}); + # remember current size of the log + $logsize = (stat $info{'logfile'})[7] || 0; # ignore stat errors + } + + push @options, @pg_ctl_opts_from_cli if @pg_ctl_opts_from_cli; + + my %pg_ctl_opts_from_file = read_cluster_conf_file $version, $cluster, 'pg_ctl.conf'; + push @options, split(' ', $pg_ctl_opts_from_file{'pg_ctl_options'}) + if defined $pg_ctl_opts_from_file{'pg_ctl_options'} and $pg_ctl_opts_from_file{'pg_ctl_options'} ne ''; + + push @options, ('-s', '-o', $postgres_opts); + + if (fork) { + wait; + if ($?) { + my $exit = $? >> 8; + print STDERR "Error: $pg_ctl @options exited with status $exit: $!\n"; + my $currlogsize = (stat $info{'logfile'})[7] if $info{'logfile'} && -r $info{'logfile'}; + if ($currlogsize) { + open LOG, $info{'logfile'} or + error "Could not open log file " . $info{'logfile'}; + seek LOG, $logsize, SEEK_SET; + print STDERR $_ while ; + } + exit $exit; + } + } else { + setsid or error "could not start session: $!"; + if ($info{'logfile'}) { + my $fd = POSIX::open($info{'logfile'}, POSIX::O_WRONLY|POSIX::O_APPEND|POSIX::O_CREAT) or error "Could not open logfile $info{'logfile'}"; + dup2($fd, 1); + dup2($fd, 2); + POSIX::close($fd) or error "Could not close log fd"; + } + exec $pg_ctl @options or error "could not exec $pg_ctl @options: $!"; + } + + # wait a bit until the socket exists + my $success = 0; + my $currlogsize = 0; + my $pidfile = $info{'pgdata'}.'/postmaster.pid'; + for (my $attempt = 0; $attempt < 60; $attempt++) { + select (undef, undef, undef, 0.5); + $currlogsize = (stat $info{'logfile'})[7] if $info{'logfile'} && -r $info{'logfile'}; + if (cluster_port_running $version, $cluster, $info{'port'}) { + $success = 1; + last; + } + + # if postgres wrote something, but the process does not exist any + # more, there must be a problem and we can stop immediately + last if ($currlogsize > $logsize && !PgCommon::check_pidfile_running $pidfile); + } + + # OK, the server runs, now wait until it stabilized + if ($success) { + $success = cluster_port_ready $version, $cluster, $info{'port'}, $info{'socketdir'}; + } + + if (!$success) { + if ($currlogsize) { + print STDERR "The PostgreSQL server failed to start. Please check the log output:\n"; + open LOG, $info{'logfile'} or + error "Could not open log file " . $info{'logfile'}; + seek LOG, $logsize, SEEK_SET; + print STDERR $_ while ; + } else { + print STDERR "The PostgreSQL server failed to start. Please check the log output.\n"; + } + exit 1; + } + + return 0; +} + +sub stop { + stop_check_pid_file; + my $result = 1; + + my @options = ('stop', '-s', '-w', '-D', $info{'pgdata'}); + + push @options, @pg_ctl_opts_from_cli if @pg_ctl_opts_from_cli; + + my %pg_ctl_opts_from_file = read_cluster_conf_file $version, $cluster, 'pg_ctl.conf'; + push @options, split(' ', $pg_ctl_opts_from_file{'pg_ctl_options'}) + if defined $pg_ctl_opts_from_file{'pg_ctl_options'} and $pg_ctl_opts_from_file{'pg_ctl_options'} ne ''; + + if (!fork()) { + close STDOUT; + exec $pg_ctl, @options, '-m', ($force ? 'fast' : $mode); + } else { + wait; + $result = $? >> 8; + } + + # try harder if forced and server hasn't stopped yet + if ($force and -f $info{'pgdata'}.'/postmaster.pid') { + print "(does not shutdown gracefully, now stopping immediately)\n"; + $result = system $pg_ctl, @options, '-m', 'immediate'; + } + + # external_pid_file files are currently not removed by postgres itself + if ($result == 0) { + unlink "/var/run/postgresql/$version-$cluster.pid"; + } + + return $result; +} + +sub restart { + my $result; + + if ($info{'running'}) { + $result = stop; + return $result if $result; + } + return start; +} + +sub reload { + exec $pg_ctl, '-D', $info{'pgdata'}, '-s', @pg_ctl_opts_from_cli, 'reload'; +} + +sub status { + exec $pg_ctl, '-D', $info{'pgdata'}, 'status'; +} + +sub promote { + exec $pg_ctl, '-D', $info{'pgdata'}, '-s', @pg_ctl_opts_from_cli, 'promote'; +} + +# +# main +# + +my ($skip_systemctl_redirect, $bindir); + +exit 1 unless GetOptions ('o|options=s' => \@postgres_auxoptions, + 'f|force' => \$force, + 'm|mode=s' => \$mode, + 'foreground' => \$foreground, + 'skip-systemctl-redirect' => \$skip_systemctl_redirect, + 'stdlog' => \$stdlog, + 'bindir=s' => \$bindir, +); + +if ($mode =~ /^(s(mart)?|f(ast)?|i(mmediate)?)$/) { + $mode = $1; # untaint +} else { + print "Invalid -m mode, valid are: smart fast immediate\n"; + exit 1; +} + +($bindir) = $bindir =~ /^(\/.*)$/ if $bindir; # untaint + +# parse command +my $version_re = qr/(\d+\.?\d)/; +my $cluster_re = qr/([^'"\s]+)/; +my $action_re = qr/(start|stop|restart|reload|status|promote)/; +my $action; + +if (@ARGV >= 3 and "$ARGV[0] $ARGV[1] $ARGV[2]" =~ /^$version_re $cluster_re $action_re$/) { + ($version, $cluster, $action) = ($1, $2, $3); + splice @ARGV, 0, 3; +} elsif (@ARGV >= 3 and "$ARGV[0] $ARGV[1] $ARGV[2]" =~ /^$action_re $version_re $cluster_re$/) { + ($action, $version, $cluster) = ($1, $2, $3); + splice @ARGV, 0, 3; +} elsif (@ARGV >= 2 and "$ARGV[0] $ARGV[1]" =~ m!^$version_re(?:[-/])$cluster_re $action_re$!) { + ($version, $cluster, $action) = ($1, $2, $3); + splice @ARGV, 0, 2; +} elsif (@ARGV >= 2 and "$ARGV[0] $ARGV[1]" =~ m!^$action_re $version_re(?:[-/])$cluster_re$!) { + ($action, $version, $cluster) = ($1, $2, $3); + splice @ARGV, 0, 2; +} else { + error "Usage: $0 [-- ]"; +} + +@pg_ctl_opts_from_cli=(); +foreach my $argv (@ARGV) { + push @pg_ctl_opts_from_cli, $argv =~ /(.*)/; # untaint +} + +error "specified cluster '$version $cluster' does not exist" unless cluster_exists $version, $cluster; +%info = cluster_info ($version, $cluster); +validate_cluster_owner \%info; + +unless ($action eq 'stop') { + # check if cluster is disabled in start.conf + error 'Cluster is disabled' if $info{'start'} eq 'disabled'; +} + +# redirect the request through systemd +if (not $skip_systemctl_redirect and getppid() != 1 and # not run from init + -d '/run/systemd/system' and not exists $ENV{_SYSTEMCTL_SKIP_REDIRECT} and # systemd is running + not $foreground and # foreground operation not requested + not exists $ENV{'PG_CLUSTER_CONF_ROOT'} and # not handling user clusters + $action =~ /^(start|stop|restart)$/ # redirect only these actions +) { + $action = $1; # untaint + system 'systemctl', 'is-active', '-q', "postgresql\@$version-$cluster"; + my $unit_active = $? == 0; + + # if extra options are given, proceed with pg_ctlcluster with a warning + if (@postgres_auxoptions != 0 or @pg_ctl_opts_from_cli != 0 or $bindir) { # extra options given + print "Notice: extra pg_ctl/postgres options given, bypassing systemctl for $action operation\n" if (-t 1); + # if we are root, redirect to systemctl unless stopping a cluster running outside systemd + } elsif ($> == 0) { + if ($action eq 'stop' and not $unit_active) { + # proceed with pg_ctlcluster + } else { + #print "Redirecting $action request to systemctl\n" if (-t 1); + system 'systemctl', $action, "postgresql\@$version-$cluster"; + exit $? >> 8; + # program end + } + # as non-root, raise an error on restarting a cluster running from systemd + } elsif ($action eq 'restart' and $unit_active) { + error "cluster is running from systemd, can only restart it as root. Try instead:\n sudo systemctl $action postgresql\@$version-$cluster"; + # program end + # otherwise just raise a warning on start and restart as non-root + } elsif (-t 1) { + if ($action =~ /start/) { + print "Warning: the cluster will not be running as a systemd service. Consider using systemctl:\n"; + print " sudo systemctl $action postgresql\@$version-$cluster\n"; + } elsif ($unit_active) { + print "Warning: stopping the cluster using pg_ctlcluster will mark the systemd unit as failed. Consider using systemctl:\n"; + print " sudo systemctl $action postgresql\@$version-$cluster\n"; + } + } +} + +# recreate missing standard log dir +if ($> == 0 && ! -e '/var/log/postgresql' && + $info{'logfile'} =~ m!^/var/log/postgresql!) { + system 'install', '-d', '-m', '1775', '-o', 'root', '-g', 'postgres', '/var/log/postgresql'; +} + +# recreate missing log file +if ($action ne 'stop' && $info{'logfile'} && ! -e $info{'logfile'}) { + if ($> == 0) { # drop privileges; this is important if logfile + # was determined via an /etc/postgresql/.../log symlink + change_ugid $info{'owneruid'}, $info{'ownergid'}; + } + sysopen (L, $info{'logfile'}, O_RDWR|O_CREAT|O_EXCL) or + error 'Could not create log file ' . $info{'logfile'}; + close L; + chmod 0640, $info{'logfile'}; + $< = $> = 0; # will silently fail if we were not root before, that's intended + $( = $) = 0; + if ($info{'owneruid'} < 500) { + my $g = (getgrnam 'adm')[2]; + lchown $info{'owneruid'}, $g, $info{'logfile'} if (defined $g); + } +} + +if ($action ne 'stop') { + # recreate /var/run/postgresql while possibly still running as root + if (! -d '/var/run/postgresql') { + system 'install', '-d', '-m', 2775, '-o', 'postgres', '-g', 'postgres', '/var/run/postgresql'; + } + + # allow creating socket directories below /var/run/postgresql for any user + if ($info{socketdir} =~ m!^(/var)/run/postgresql/[\w_.-]+$! and ! -d $info{socketdir}) { + if (mkdir $info{socketdir}, 02775) { # don't use "install" here as it would allow stealing existing directories + chown $info{owneruid}, $info{ownergid}, $info{socketdir}; + } else { + error "Could not create $info{socketdir}: $!"; + } + } + + # allow creating stats_temp_directory below /var/run/postgresql for any user + if ($info{config}->{stats_temp_directory} and $info{config}->{stats_temp_directory}=~ m!^(/var)/run/postgresql/[\w_.-]+$! and ! -d $info{config}->{stats_temp_directory}) { + if (mkdir $info{config}->{stats_temp_directory}, 0750) { # don't use "install" here as it would allow stealing existing directories + chown $info{owneruid}, $info{ownergid}, $info{config}->{stats_temp_directory}; + } else { + error "Could not create $info{config}->{stats_temp_directory}: $!"; + } + } +} + +if ($> == 0) { + # have postgres start with increased OOM killer protection; 9.0 and + # later has builtin support for resetting the adjustment of child processes + if ($action eq 'start' and $version >= '9.0' and not $PgCommon::rpm) { + if (-w '/proc/self/oom_score_adj') { + open F, '>/proc/self/oom_score_adj'; + print F "-900\n"; + close F; + } + } + + change_ugid $info{'owneruid'}, $info{'ownergid'}; +} + +# we are running as the cluster owner now + +if( $> != $info{'owneruid'} ) { + error 'You must run this program as the cluster owner ('. + (getpwuid $info{'owneruid'})[0].') or root'; +} + +# create socket directory (if it wasn't already created in /var/run/postgresql by the code above) +if ($action ne 'stop' && ! -d $info{socketdir}) { + system 'install', '-d', '-m', 2775, $info{socketdir}; +} + +# create stats_temp_directory (if it wasn't already created in /var/run/postgresql by the code above) +if ($action ne 'stop' && $info{config}->{stats_temp_directory} && ! -d $info{config}->{stats_temp_directory}) { + system 'install', '-d', '-m', 750, $info{config}->{stats_temp_directory}; +} + +$pg_ctl = $bindir ? "$bindir/pg_ctl" : get_program_path ('pg_ctl', $version); +error "Could not find pg_ctl executable for version $version" unless ($pg_ctl); + +# do the action +no strict 'refs'; +exit &$action; + +__END__ + +=head1 NAME + +pg_ctlcluster - start/stop/restart/reload a PostgreSQL cluster + +=head1 SYNOPSIS + +B [I] I I I [B<--> I] + +where I = B|B|B|B|B|B + +=head1 DESCRIPTION + +This program controls the B server for a particular cluster. It +essentially wraps the L command. It determines the cluster version +and data path and calls the right version of B with appropriate +configuration parameters and paths. + +You have to start this program as the user who owns the database cluster or as +root. + +To ease integration with B operation, the alternative syntax +"B IB<->I I" is also supported, +as well as putting the action first (matching the ordering used by B). + +=head1 ACTIONS + +=over 4 + +=item B + +A log file for this specific cluster is created if it does not exist yet (by +default, +CIC<->IC<.log>), +and a PostgreSQL server process (L) is started on it. Exits with +0 on success, with 2 if the server is already running, and with 1 on other +failure conditions. + +=item B + +Stops the L server of the given cluster. By default, "fast" +shutdown mode is used. + +=item B + +Stops the server if it is running and starts it (again). + +=item B + +Causes the configuration files to be re-read without a full shutdown of the +server. + +=item B + +Checks whether a server is running. If it is, the PID and the command line +options that were used to invoke it are displayed. + +=item B + +Commands a running standby server to exit recovery and begin read-write +operations. + +=back + +=head1 OPTIONS + +=over 4 + +=item B<-f>|B<--force> + +For B and B, the "fast" mode is used which rolls back all active +transactions, disconnects clients immediately and thus shuts down cleanly. If +that does not work, shutdown is attempted again in "immediate" mode, which can +leave the cluster in an inconsistent state and thus will lead to a recovery run +at the next start. If this still does not help, the B process is +killed. Exits with 0 on success, with 2 if the server is not running, and with +1 on other failure conditions. This mode should only be used when the machine +is about to be shut down. + +=item B<-m>|B<--mode> [B|B|B] + +Shutdown mode to use for B and B actions, default is B. +See pg_ctl(1) for documentation. + +=item B<--foreground> + +Start B in foreground, without daemonizing via B. + +=item B<--stdlog> + +When B<--foreground> is in use, redirect stderr to the standard logfile in +C. (Default when not run in foreground.) + +=item B<--skip-systemctl-redirect> + +When running as root, B redirects actions to B so +running clusters are properly supervised by B. This option skips the +redirect; it is used in the B unit file. The redirect is +also skipped if additional B or B options are provided. + +=item B<--bindir> I + +Path to B. (Default is CIC.) + +=item B<-o>|B<--options> I + +This configuration file controls the start/stop behavior of the cluster. See +section "STARTUP CONTROL" in L for details. + +=back + +=head1 BUGS + +Changing the port number on startup using B<-o -p> will not work as it breaks +the checks for running clusters. + +=head1 SEE ALSO + +L, L, L, L, +L + +=head1 AUTHOR + +Martin Pitt Lmpitt@debian.orgE> + diff --git a/pg_dropcluster b/pg_dropcluster new file mode 100755 index 0000000..def4554 --- /dev/null +++ b/pg_dropcluster @@ -0,0 +1,226 @@ +#!/usr/bin/perl -wT + +# Completely delete a PostgreSQL cluster. Fails if there is still a server +# process attached. +# +# (C) 2005-2009 Martin Pitt +# (C) 2015-2021 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use warnings; +use Getopt::Long; +use PgCommon; + +# untaint environment +$ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin'; +delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; + +my $stopserver = 0; +exit 1 unless GetOptions ('stop-server' => \$stopserver, 'stop' => \$stopserver); + +# command line options +if ($#ARGV != 1) { + print "Usage: $0 [--stop] \n"; + exit 1; +} + +my ($version) = $ARGV[0] =~ /^(\d+\.?\d+)$/; +my ($cluster) = $ARGV[1] =~ /^([-.\w]+)$/; +error 'invalid version' unless defined $version; +error 'invalid cluster name' unless defined $cluster; + +my %info; +my $c; # configuration directory +my $startconf; + +if (cluster_exists $version, $cluster) { + %info = cluster_info ($version, $cluster); + validate_cluster_owner \%info if ($info{pgdata} && -d $info{pgdata}); # ignore missing data directory + + if ($info{'running'}) { + if ($stopserver) { + if ($info{'pgdata'} && -d $info{'pgdata'}) { + if (system ('pg_ctlcluster', $version, $cluster, 'stop')) { + error 'could not stop server, aborting'; + } + } else { + print STDERR "Warning: corrupted cluster: data directory does not exist any more, but server is still running; you have to manually kill the postgres process\n"; + } + } else { + error 'This cluster is still running. Stop it or supply the --stop option'; + } + } + $c = $info{'configdir'}; + $startconf = $info{'start'}; +} else { + $c = "/etc/postgresql/$version/$cluster"; + + # check if we have a broken cluster, clean up if necessary + -d $c or error 'specified cluster does not exist'; +} + +# disable systemd-enabled clusters +if (not exists $ENV{'PG_CLUSTER_CONF_ROOT'} and -d '/run/systemd/system') { + if ($> == 0) { + system "systemctl", "disable", "postgresql\@$version-$cluster"; + system "systemctl", "disable", "--now", "pg_receivewal\@$version-$cluster"; + system "systemctl", "disable", "--now", "pg_basebackup\@$version-$cluster.timer"; + system "systemctl", "disable", "--now", "pg_dump\@$version-$cluster.timer"; + } +} + +if ($info{'pgdata'} && -d $info{'pgdata'}) { + # remove custom wal directory + if ($info{waldir} and -d $info{waldir}) { + my $walowner = (stat($info{waldir}))[4]; + if (defined $walowner and $walowner == $info{owneruid}) { + my $result = system 'rm', '-r', '--', $info{waldir}; + if ($result) { + print STDERR "Warning: could not remove wal directory $info{waldir}"; + } + } else { + print STDERR "Warning: wal directory $info{waldir} is not owned by uid $info{owneruid}, not removing\n"; + } + } + + # remove tablespace subdirectories that belong to our version + for my $link (glob "$info{pgdata}/pg_tblspc/*") { + next unless (-l $link); + my $tblspc = readlink $link; + my $tblspcowner = (stat($tblspc))[4]; + if (defined $tblspcowner and $tblspcowner == $info{owneruid}) { + if ($version >= 9.0) { + for my $dir (glob "$tblspc/PG_${version}_*") { + my $dirowner = (stat($dir))[4]; + if (defined $dirowner and $dirowner == $info{owneruid}) { + my $result = system 'rm', '-r', '--', ($dir =~ /(.*)/); # untaint + if ($result) { + print STDERR "Warning: could not remove tablespace directory $dir"; + } + } else { + print STDERR "Warning: tablespace subdirectory $dir (in tablespace linked from $link) is not owned by uid $info{owneruid}, not removing\n"; + } + } + } else { # before 9.0 + if (open my $fh, '<', "$tblspc/PG_VERSION") { + my $v = <$fh>; + chomp $v; + close $fh; + if ($v eq $version) { + $tblspc =~ /(.*)/; # untaint + my $result = system "rm -r -- $1/*"; + } else { + print STDERR "Warning: tablespace directory $tblspc (linked from $link) is from PostgreSQL version $v, not removing\n"; + } + } else { + print STDERR "Warning: tablespace directory $tblspc (linked from $link) is not a PostgreSQL directory, not removing\n"; + } + } + } else { + print STDERR "Warning: tablespace directory $tblspc (linked from $link) is not owned by uid $info{owneruid}, not considering\n"; + } + } + + # remove pgdata + my $result = system 'rm', '-r', $info{'pgdata'}; + if ($result) { + if (! -w ($info{'pgdata'} . '/..')) { + error 'you might need to run this program with root privileges'; + } + exit $result; + } +} else { + print STDERR "Warning: corrupted cluster: data directory does not exist\n"; +} + +# remove config +unlink $c.'/pg_hba.conf', $c.'/pg_ident.conf', $c.'/postgresql.conf', + $c.'/start.conf', $c.'/log', $c.'/autovacuum_log', $c.'/pgdata', + $c.'/environment', $c.'/pg_ctl.conf'; +rmdir $_ foreach (map { /(.*)/ && $1 } glob "$c/*"); # remove empty conf.d directories + +unlink $info{'logfile'} if defined ($info{'logfile'}); +if ($info{'socketdir'} and $info{'socketdir'} !~ /^(\/tmp|\/var\/run\/postgresql)\/?$/) { + rmdir $info{'socketdir'}; +} +rmdir $c; +rmdir "/etc/postgresql/$version"; +rmdir "/var/lib/postgresql/$version/$cluster"; +rmdir "/var/lib/postgresql/$version"; +unlink "/var/log/postgresql/postgresql-$version-$cluster.log"; +# remove logrotated files +foreach my $f () { + unlink ($f =~ /(.*)/); # untaint +} + +# remove stats_temp_directory +my $statstempdir = $info{config}->{stats_temp_directory}; +if ($statstempdir) { + my $statsowner = (stat($statstempdir))[4]; + if (defined $statsowner and defined $info{owneruid} and $statsowner == $info{owneruid}) { + foreach my $f (<$statstempdir/*.stat>) { + unlink ($f =~ /(.*)/); # untaint + } + rmdir $statstempdir; + } +} + +# notify systemd when an autostarted cluster went away +if (not exists $ENV{'PG_CLUSTER_CONF_ROOT'} and $startconf and $startconf eq 'auto' and -d '/run/systemd/system') { + if ($> == 0) { + system 'systemctl daemon-reload'; + } elsif (-t 1) { + print "Warning: systemd was not informed about the removed cluster yet. Operations like \"service postgresql start\" might fail. To fix, run:\n"; + print " sudo systemctl daemon-reload\n"; + } +} + +# notify apt about the new cluster +if (not exists $ENV{'PG_CLUSTER_CONF_ROOT'} and $> == 0) { + system "/usr/share/postgresql-common/pg_updateaptconfig"; +} + +exit 0; + +__END__ + +=head1 NAME + +pg_dropcluster - completely delete a PostgreSQL cluster + +=head1 SYNOPSIS + +B [B<--stop>] I I + +=head1 DESCRIPTION + +This program removes all files that belong to a given PostgreSQL cluster; that +includes the data, wal, and tablespace directories, the log file, and all configuration files that +were created by L. If the configuration directory +(CICI) is empty after this, it is +removed as well. +An empty socket directory other than B or B is +also removed. + +Usually a cluster which still has a running server attached will not be +deleted. To override this, the B<--stop> option forces a server shutdown +before the files are removed. + +=head1 SEE ALSO + +L, L + +=head1 AUTHOR + +Martin Pitt Lmpitt@debian.orgE> + diff --git a/pg_getwal b/pg_getwal new file mode 100755 index 0000000..8e64e87 --- /dev/null +++ b/pg_getwal @@ -0,0 +1,96 @@ +#!/bin/sh + +# retrieve a WAL file from a pg_receivewal archive +# +# Copyright (C) 2021-2022 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +set -eu + +binroot="/usr/lib/postgresql/" +#redhat# binroot="/usr/pgsql-" + +if [ -z "${2:-}" ]; then + echo "Syntax: $0 /path/to/wal/%f %p" >&2 + exit 1 +fi + +file="$1" +path="$2" + +# sanity-check the first argument +waldir="$(dirname $file)" +if ! [ -d "$waldir" ]; then + echo "$0: $waldir is not a directory" >&2 + exit 129 +fi + +if [ -f "$file.gz" ]; then + gunzip < "$file.gz" > "$path" || exit 129 + +elif [ -f "$file.lz4" ]; then + unlz4 < "$file.lz4" > "$path" || exit 129 + +elif [ -f "$file" ]; then + cp "$file" "$path" || exit 129 + +elif [ -f "$file.gz.partial" ] || [ -f "$file.lz4.partial" ]; then + if [ -s "$file.gz.partial" ]; then + gunzip < "$file.gz.partial" > "$path" || exit 129 + elif [ -s "$file.lz4.partial" ]; then + unlz4 < "$file.lz4.partial" > "$path" || exit 129 + else + # .gz.partial starts completely empty, gunzip doesn't like that + touch "$path" || exit 129 + fi + + # expand file to original size + version=$(cat PG_VERSION) || exit 129 + wal_file_size=$(LC_ALL=C "$binroot$version/bin/pg_controldata" . | awk '/^Bytes per WAL segment:/ { print $5 }') || exit 129 + [ "$wal_file_size" -gt 0 ] || exit 129 + truncate --size="$wal_file_size" "$path" || exit 129 + +elif [ -f "$file.partial" ]; then + cp "$file.partial" "$path" || exit 129 + +else + # file not found, exit silently in order not to spam the server log with errors + exit 1 +fi + +exit 0 + +: <<=cut + +=head1 NAME + +pg_getwal - retrieve a WAL file from a pg_receivewal archive + +=head1 SYNOPSIS + +B I I<%p> + +=head1 DESCRIPTION + +B retrieves and decompresses files from a WAL archive maintained by +B and B. It is put into PostgreSQL's +B by B. + +=head1 SEE ALSO + +L, L. + +=head1 AUTHOR + +Christoph Berg Lmyon@debian.orgE> + +=cut diff --git a/pg_hba b/pg_hba new file mode 100755 index 0000000..350d251 --- /dev/null +++ b/pg_hba @@ -0,0 +1,144 @@ +#!/usr/bin/perl -w + +# Add, remove, or test a pg_hba.conf entry. +# +# (C) 2005-2009 Martin Pitt +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use PgCommon; +use Getopt::Long; +use Net::CIDR; + +# global variables + +my $ip = ''; # default to local unix socket +my $force_ssl = 0; +my ($method, $ver_cluster, $db, $user); +my $mode; +my @hba; + +# Print an error message to stderr and exit with status 2 +sub error2 { + print STDERR 'Error: ', $_[0], "\n"; + exit 2; +} + +# Check if s1 is equal to s2 or s2 is 'all'. +# Arguments: +sub match_all { + return ($_[1] eq 'all' || $_[0] eq $_[1]); +} + +# Check if given IP matches the specification in the HBA record. +# Arguments: +sub match_ip { + my ($ip, $hba) = @_; + + # Don't try to mix IPv4 and IPv6 addresses since that will make cidrlookup + # croak + return 0 if ((index $ip, ':') < 0) ^ ((index $$hba{'ip'}, ':') < 0); + + return Net::CIDR::cidrlookup ($ip, $$hba{'ip'}); +} + +# Check if arguments match any line +# Return: 1 if match was found, 0 otherwise. +sub mode_test { + foreach my $hbarec (@hba) { + if (!defined($$hbarec{'type'})) { + next; + } + next if $$hbarec{'type'} eq 'comment'; + next unless match_all ($user, $$hbarec{'user'}) && + match_all ($db, $$hbarec{'db'}) && + $$hbarec{'method'} eq $method; + + if ($ip) { + return 1 if + (($force_ssl && $$hbarec{'type'} eq 'hostssl') || + $$hbarec{'type'} =~ /^host/) && + match_ip ($ip, $hbarec); + } else { + return 1 if $$hbarec{'type'} eq 'local'; + } + } + + return 0; +} + +# Print hba conf. +sub mode_print { + foreach my $hbarec (@hba) { + print "$$hbarec{'line'}\n"; + } +} + +# Generate a pg_hba.conf line that matches the command line args. +sub create_hba_line { + if ($ip) { + return sprintf "%-7s %-11s %-11s %-35s %s\n", + $force_ssl ? 'hostssl' : 'host', $db, $user, $ip, $method; + } else { + return sprintf "%-7s %-11s %-47s %s\n", 'local', $db, $user, $method; + } +} + +# parse arguments + +my $ip_arg; +exit 3 unless GetOptions ( + 'cluster=s' => \$ver_cluster, + 'ip=s' => \$ip_arg, + 'method=s' => \$method, + 'force-ssl' => \$force_ssl +); + +if ($#ARGV != 2) { + print STDERR "Usage: $0 mode [options] \n"; + exit 2; +} +($mode, $db, $user) = @ARGV; + +error2 '--cluster must be specified' unless $ver_cluster; +my ($version, $cluster) = split ('/', $ver_cluster); +error2 'No version specified with --cluster' unless $version; +error2 'No cluster specified with --cluster' unless $cluster; +error2 'Cluster does not exist' unless cluster_exists $version, $cluster; +if (defined $ip_arg) { + $ip = Net::CIDR::cidrvalidate $ip_arg; + error2 'Invalid --ip argument' unless defined $ip; +} + +unless (defined $method) { + $method = ($ip ? 'md5' : 'ident sameuser'); +} +error2 'Invalid --method argument' unless PgCommon::valid_hba_method($method); + +# parse file + +my $hbafile = "/etc/postgresql/$version/$cluster/pg_hba.conf"; +@hba = read_pg_hba $hbafile; +error2 "Could not read $hbafile" unless $#hba; + +if ($mode eq 'pg_test_hba') { + if (mode_test) { + exit 0; + } else { + print create_hba_line(); + exit 1; + } +} elsif ($mode eq 'pg_print_hba') { + mode_print(); +} else { + error2 "Unknown mode: $mode"; +} diff --git a/pg_lsclusters b/pg_lsclusters new file mode 100755 index 0000000..fec09d7 --- /dev/null +++ b/pg_lsclusters @@ -0,0 +1,184 @@ +#!/usr/bin/perl -wT +# Show all PostgreSQL clusters in a list +# +# (C) 2005-2009 Martin Pitt +# (C) 2013-2018 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use PgCommon; +use Getopt::Long; + +sub help ($) +{ + my $exit = shift; + print "Usage: $0 [-hjs] + +Options: + -h --no-header Omit column headers in output + -j --json JSON output + -s --start-conf Include start.conf information in status column + --help Print help\n"; + + exit $exit; +} + +# option handling +my $no_header; +my $json; +my $start_conf; +help(1) unless GetOptions ( + 'help' => sub { help(0); }, + 'h|no-header' => \$no_header, + 'j|json' => \$json, + 's|start-conf' => \$start_conf, +); + +my (@versions, $ls_cluster); +if (@ARGV == 1 and $ARGV[0] =~ m{^(\d+(?:\.\d+)?)[-/]([-.\w]+)$}) { + push @versions, $1; + $ls_cluster = $2; +} elsif (@ARGV >= 1) { + $ARGV[0] =~ /^(\d+(?:\.\d+)?)$/ or error "Invalid version number \"$ARGV[0]\""; + push @versions, $1; +} else { + @versions = get_versions(); +} +if (@ARGV >= 2 and $ARGV[1] =~ /^([-.\w]+)$/) { + $ls_cluster = $1; +} + +error "Cluster @versions $ls_cluster does not exist" + if ($ls_cluster and not cluster_exists(@versions, $ls_cluster)); + +# data collection +my @lines; +push @lines, ['Ver', 'Cluster', 'Port', 'Status', 'Owner', 'Data directory', 'Log file'] + unless ($no_header); +my $jsoninfo = []; + +foreach my $v (@versions) { + my $pg_log = $v >= 10 ? 'log' : 'pg_log'; # log directory in PGDATA changed in PG 10 + my @clusters = $ls_cluster ? $ls_cluster : get_version_clusters $v; + foreach my $c (@clusters) { + my %info = cluster_info $v, $c; + + my $status = $info{'running'} ? "online" : "down"; + $status .= ",recovery" if ($info{'recovery'}); + $status .= ",$info{supervisor}" if ($info{'supervisor'}); + $status .= ",$info{start}" if ($start_conf); + unless (-e "${PgCommon::binroot}$v/bin/postgres") { + $status .= ",binaries_missing"; + $info{binaries_missing} = 1; + } + + my $logfile = $info{logfile} // ''; # default logfile in /var/log/postgresql + if (config_bool ($info{config}->{logging_collector})) { + my $path = $info{config}->{log_directory} || $pg_log; + my $file = $info{config}->{log_filename} || 'postgresql-%Y-%m-%d_%H%M%S.log'; + $logfile = "$path/$file"; + } + my $destination = $info{config}->{log_destination} || 'stderr'; + $destination =~ s/stderr/$logfile/; + my $csvlog = $logfile; + $csvlog =~ s/(?:\.log)?$/.csv/; + $destination =~ s/csvlog/$csvlog/; + + push @lines, [$v, $c, $info{'port'} // '', $status, + defined $info{'owneruid'} ? (getpwuid $info{'owneruid'})[0] : '', + $info{'pgdata'} || '', $destination]; + $info{version} = $v; + $info{cluster} = $c; + push @$jsoninfo, \%info; + } +} + +# output +if ($json) { + eval { require JSON; }; + error 'Please install JSON.pm for JSON output (Debian: libjson-perl)' + if ($@); + print JSON::encode_json($jsoninfo) . "\n"; + exit 0; +} + +my @colwidth = qw(1 1 1 1 1 1 1); +foreach my $line (@lines) { + for (my $i = 0; $i < @$line - 1; $i++) { # skip adjustment for last column + my $len = length @$line[$i]; + $colwidth[$i] = $len if ($len > $colwidth[$i]); + } +} + +my $color = -t 1; # color output if stdout is a terminal +my $fmtstring = join ' ', map { "%-${_}s" } @colwidth; + +foreach my $line (@lines) { + if ($color and $line->[0] ne 'Ver') { # don't color header + printf "\033[%dm$fmtstring\033[0m\n", + ($line->[3] =~ /^online/ ? 32 : 31), # green, red + @$line; + } else { + printf "$fmtstring\n", @$line; + } +} + +__END__ + +=head1 NAME + +pg_lsclusters - show information about all PostgreSQL clusters + +=head1 SYNOPSIS + +B [I] [I [I]] + +=head1 DESCRIPTION + +This command list the status and some configuration details of all clusters. +If a version and optionally a cluster name are given, only these are shown. + +=head1 OPTIONS + +=over 4 + +=item B<-h>, B<--no-header> + +Do not print the column header line. + +=item B<-j>, B<--json> + +Output information in JSON format. Needs JSON.pm installed. +(Debian package: libjson-perl) + +=item B<-s>, B<--start-conf> + +Include F information in status column. + +=item B<--help> + +Print usage help. + +=back + +=head1 NOTES + +The cluster status is shown as B or B. If a F file +is found in the data directory, B<,recovery> is appended. The latter needs read +access to the data directory, which only root and the cluster owner have. + +The output lines are colored green and red to indicate the cluster status +visually. + +=head1 AUTHOR + +Martin Pitt Lmpitt@debian.orgE> diff --git a/pg_renamecluster b/pg_renamecluster new file mode 100755 index 0000000..e5f96bb --- /dev/null +++ b/pg_renamecluster @@ -0,0 +1,176 @@ +#!/usr/bin/perl -wT + +# Rename a PostgreSQL cluster +# +# (C) 2014-2021 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use warnings; +use PgCommon; +use Getopt::Long; +use POSIX; + +# untaint environment +$ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin'; +delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; + +if (@ARGV != 3) { + print "Usage: $0 [OPTIONS] \n"; + exit 1; +} + +my ($version) = $ARGV[0] =~ /^(\d+\.?\d+)$/; +my ($oldcluster) = $ARGV[1] =~ /^([-.\w]+)$/; +my ($newcluster) = $ARGV[2] =~ /^([-.\w]+)$/; +if ($newcluster =~ /-/ and -t 1) { + print "Warning: cluster names containing dashes (-) will cause problems when running from systemd. Continuing anyway\n"; +} + +error "Old and new name must be different" + if ($oldcluster eq $newcluster); +error "specified cluster $version $oldcluster does not exist" + unless (cluster_exists $version, $oldcluster); +error "target cluster $version $newcluster already exists" + if (cluster_exists $version, $newcluster); +my %info = cluster_info ($version, $oldcluster); +validate_cluster_owner \%info; + +# stopping old cluster, so that we notice early when there are still +# connections +if ($info{'running'}) { + print "Stopping cluster $version $oldcluster ...\n"; + my @argv = ('pg_ctlcluster', $version, $oldcluster, 'stop'); + error "Could not stop cluster" if system @argv; +} + +# Arguments: , , +sub strrepl { + my ($s, $f, $t) = @_; + $s =~ s/\b\Q$f\E\b/$t/g; + return $s; +} + +# rename config directory +my $olddir = "$PgCommon::confroot/$version/$oldcluster"; +my $newdir = "$PgCommon::confroot/$version/$newcluster"; +rename $olddir, $newdir or error "Could not rename config directory $olddir: $!"; + +# adapt paths to configuration files +my %c = read_cluster_conf_file $version, $newcluster, 'postgresql.conf'; +if ($c{hba_file}) { + PgCommon::set_conf_value $version, $newcluster, 'postgresql.conf', 'hba_file', + strrepl($c{hba_file}, $oldcluster, $newcluster); +} +if ($c{ident_file}) { + PgCommon::set_conf_value $version, $newcluster, 'postgresql.conf', 'ident_file', + strrepl($c{ident_file}, $oldcluster, $newcluster); +} +if ($c{external_pid_file}) { + PgCommon::set_conf_value $version, $newcluster, 'postgresql.conf', 'external_pid_file', + strrepl($c{external_pid_file}, $oldcluster, $newcluster); +} + +# update cluster_name +if ($c{cluster_name}) { + PgCommon::set_conf_value $version, $newcluster, 'postgresql.conf', 'cluster_name', + strrepl ($c{cluster_name}, $oldcluster, $newcluster); +} + + +# rename data directory +if ($info{pgdata}) { + my $newpgdata = strrepl ($info{pgdata}, $oldcluster, $newcluster); + if ($info{pgdata} ne $newpgdata) { + rename $info{pgdata}, $newpgdata or + error "Could not rename data directory $info{pgdata}: $!"; + PgCommon::set_conf_value $version, $newcluster, 'postgresql.conf', + 'data_directory', $newpgdata; + } +} + +# rename stats_temp_directory +my $statstempdir = $info{config}->{stats_temp_directory}; +if ($statstempdir) { + my $newstatstempdir = strrepl ($statstempdir, $oldcluster, $newcluster); + if ($statstempdir ne $newstatstempdir) { + PgCommon::set_conf_value $version, $newcluster, 'postgresql.conf', + 'stats_temp_directory', $newstatstempdir; + if (-d $statstempdir) { + rename $statstempdir, $newstatstempdir or + error "Could not rename stats temp directory $statstempdir}: $!"; + } + } +} + +# rename old log files +my $logdir = "/var/log/postgresql"; +if (opendir LOG, $logdir) { + while (my $logfile = readdir LOG) { + next unless $logfile =~ /^(\Qpostgresql-$version-$oldcluster.log\E.*)/; + $logfile = $1; # untaint + my $f = strrepl ($logfile, $oldcluster, $newcluster); + rename "$logdir/$logfile", "$logdir/$f" or error "rename $logdir/$logfile: $!"; + } + closedir LOG; +} + +# notify systemd about the new cluster +if (not exists $ENV{'PG_CLUSTER_CONF_ROOT'} and -d '/run/systemd/system') { + if ($> == 0) { + system 'systemctl daemon-reload'; + } elsif (-t 1) { + print "Warning: systemd does not know about the new cluster yet. Operations like \"service postgresql start\" will not handle it. To fix, run:\n"; + print " sudo systemctl daemon-reload\n"; + } +} + +# start cluster if it was running before +if ($info{'running'}) { + print "Starting cluster $version $newcluster ...\n"; + my @argv = ('pg_ctlcluster', $version, $newcluster, 'start'); + error "Could not start cluster" if system @argv; +} + +__END__ + +=head1 NAME + +pg_renamecluster - rename a PostgreSQL cluster + +=head1 SYNOPSIS + +B I I I + +=head1 DESCRIPTION + +B changes the name of a PostgreSQL cluster, i. e. the name of +the config directory in /etc/postgresql/I/ along with the data +directory in /var/lib/postgresql/I/. Existing log files in +/var/log/postgresql/ are also renamed. The cluster is stopped and started for +the operation. + +The following B config options are updated to refer to the +changed path names: B, B, B, +B, B, B. + +=head1 OPTIONS + +None. + +=head1 SEE ALSO + +L, L, L, L + +=head1 AUTHOR + +Christoph Berg Lmyon@debian.orgE> diff --git a/pg_restorecluster b/pg_restorecluster new file mode 100755 index 0000000..0a4af55 --- /dev/null +++ b/pg_restorecluster @@ -0,0 +1,436 @@ +#!/usr/bin/perl -wT + +# pg_restorecluster: restore from a pg_backupcluster backup +# +# Copyright (C) 2021 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use warnings; +use Cwd 'abs_path'; +use Getopt::Long; +use PgCommon; + +my ($version, $cluster); + +# untaint environment +$ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin'; +delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; +umask 022; + +sub help () { + print "Syntax: $0 [options] +Options: + -d --datadir DIR Data directory for restored cluster (default per createcluster.conf) + -p --port PORT Use port PORT for restored cluster (default is next free port) + -s --start Start cluster after restoring (default for restore from dump) + --archive Configure recovery from WAL archive + --pitr TIMEST Configure point-in-time recovery to TIMESTAMP from WAL archive + --wal-archive DIR Read WAL from archive DIR (default /../wal) +"; +} + +my $createclusterconf = "$PgCommon::common_confdir/createcluster.conf"; +my ($datadir, $port, $start, $recovery_target_time, $archive_recovery, $wal_archive); + +exit 1 unless GetOptions ( + 'd|datadir=s' => \$datadir, + 'p|port=s' => \$port, + 's|start' => \$start, + 'archive' => \$archive_recovery, + 'pitr=s' => \$recovery_target_time, + 'recovery-target-time=s' => \$recovery_target_time, + 'wal-archive=s' => \$wal_archive, + 'createclusterconf=s' => \$createclusterconf, +); +if ($recovery_target_time) { + ($recovery_target_time) = $recovery_target_time =~ /(.*)/; # untaint +} + +# accept both "version cluster" and "version[-/]cluster" +if (@ARGV >= 2 and $ARGV[0] =~ m!^(\d+\.?\d)[-/]([^/]+)$!) { + ($version, $cluster) = ($1, $2); + shift @ARGV; +} elsif (@ARGV >= 3 and $ARGV[0] =~ /^(\d+\.?\d)$/) { + $version = $1; + ($cluster) = ($ARGV[1]) =~ m!^([^/]+)$!; + shift @ARGV; + shift @ARGV; +} else { + help(); + exit 1; +} + +error "cluster $version $cluster already exists" if cluster_exists $version, $cluster; + +my %defaultconf = PgCommon::read_conf_file ($createclusterconf); + +# functions to be run as root + +sub create_cluster_directories($$$$) { + my ($owneruid, $ownergid, $configdir, $datadir) = @_; + my @postgres_user = getpwnam 'postgres'; + my ($pg_uid, $pg_gid) = (getpwnam 'postgres')[2, 3]; + + for my $pgdir ("/etc/postgresql", "/etc/postgresql/$version", "/var/lib/postgresql", "/var/lib/postgresql/$version") { + if (! -e $pgdir) { + mkdir $pgdir or error "mkdir $pgdir: $!"; + chown $pg_uid, $pg_gid, $pgdir or error "chown $pgdir: $!"; + } + } + mkdir $configdir or error "mkdir $configdir: $!"; + chown $owneruid, $ownergid, $configdir or error "chown $configdir: $!"; + mkdir $datadir, 0700 or error "mkdir $datadir: $!"; + chown $owneruid, $ownergid, $datadir or error "chown $datadir: $!"; +} + +sub create_cluster($$$$$) { + my ($backup, $owneruid, $ownergid, $configdir, $datadir) = @_; + + my @createclusteropts = (); + if (-f "$backup/createcluster.opts") { + open my $fh, "$backup/createcluster.opts" or error "$backup/createcluster.opts: $!"; + local $/; # slurp mode + my ($opts) = <$fh> =~ /(.*)/; # untaint + @createclusteropts = split /\s+/, $opts; + close $fh; + } + + system_or_error "pg_createcluster", + "--datadir", $datadir, + "--user", $owneruid, "--group", $ownergid, + $version, $cluster, + "--", + @createclusteropts; +} + +sub start_cluster() { + print "Starting cluster $version $cluster ...\n"; + system_or_error "pg_ctlcluster", $version, $cluster, "start"; +} + +sub switch_to_cluster_owner($$) { + my ($owneruid, $ownergid) = @_; + change_ugid $owneruid, $ownergid; +} + +# restore functions + +sub unpack_tar($$$) { + my ($backup, $tar, $dir) = @_; + + if (-f "$backup/$tar.gz") { + $tar = "$tar.gz"; + } elsif (-f "$backup/$tar.bz2") { + $tar = "$tar.bz2"; + } elsif (-f "$backup/$tar.xz") { + $tar = "$tar.xz"; + } elsif (-f "$backup/$tar") { + # do nothing + } else { + error "$backup/config.tar* is missing"; + } + + print "Restoring $backup/$tar to $dir ...\n"; + system_or_error "tar", "-C", $dir, "-xf", "$backup/$tar"; +} + +sub restore_config($$) { + my ($backup, $configdir) = @_; + unpack_tar($backup, "config.tar", $configdir); +} + +sub update_config($$$) { + my ($configdir, $datadir, $port) = @_; + my %settings = ( + data_directory => $datadir, + hba_file => "$configdir/pg_hba.conf", + ident_file => "$configdir/pg_ident.conf", + external_pid_file => "/var/run/postgresql/$version-$cluster.pid", + port => $port, + ); + $settings{cluster_name} = "$version/$cluster" if ($version >= 9.5); + $settings{stats_temp_directory} = "/var/run/postgresql/$version-$cluster.pg_stat_tmp" if ($version < 15); + my %config = PgCommon::read_conf_file("$configdir/postgresql.conf"); + for my $guc (sort keys %settings) { + if (not exists $config{$guc} or $config{$guc} ne $settings{$guc}) { + print "Setting $guc = $settings{$guc}\n"; + PgCommon::set_conffile_value("$configdir/postgresql.conf", $guc, $settings{$guc}); + } + } +} + +sub restore_basebackup($$) { + my ($backup, $datadir) = @_; + unpack_tar($backup, "base.tar", $datadir); +} + +sub restore_wal($$) { + my ($backup, $datadir) = @_; + return if ($version < 10); # WAL contained in base.tar.gz in PG 9.x + unpack_tar($backup, "pg_wal.tar", "$datadir/pg_wal"); +} + +sub archive_recovery_options($$$) { + my ($backup, $datadir, $wal_archive) = @_; + + $wal_archive = abs_path($wal_archive) or error "$wal_archive: $!"; + -d $wal_archive or error "$wal_archive is not a directory"; + + print "Setting archive recovery options"; + my $recovery_options = "restore_command = '/usr/share/postgresql-common/pg_getwal $wal_archive/%f %p'\n"; + if ($recovery_target_time) { + $recovery_options .= "recovery_target_time = '$recovery_target_time'\n"; + $recovery_options .= "recovery_target_action = 'promote'\n"; + } + + if ($version >= 12) { + my $autoconf = "$datadir/postgresql.auto.conf"; + open my $fh, ">>", $autoconf or error "$autoconf: $!"; + print $fh $recovery_options or error "$autoconf: $!"; + close $fh or error "$autoconf: $!"; + + my $recoverysignal = "$datadir/recovery.signal"; + open my $fh2, ">", $recoverysignal or error "$recoverysignal: $!"; + close $fh2 or error "$recoverysignal: $!"; + + } else { + my $recoveryconf = "$datadir/recovery.conf"; + open my $fh, ">>", $recoveryconf or error "$recoveryconf: $!"; + print $fh $recovery_options or error "$recoveryconf: $!"; + close $fh or error "$recoveryconf: $!"; + } +} + +sub reset_archive_recovery_options() { + if ($version >= 12) { + system_or_error "psql", + "--cluster", "$version/$cluster", + "-XAtqc", "ALTER SYSTEM RESET restore_command"; + system_or_error "psql", + "--cluster", "$version/$cluster", + "-XAtqc", "ALTER SYSTEM RESET recovery_target_time" + if ($recovery_target_time); + system_or_error "psql", + "--cluster", "$version/$cluster", + "-XAtqc", "ALTER SYSTEM RESET recovery_target_action" + if ($recovery_target_time); + } +} + +sub restore_globals($$) { + my ($backup, $owneruid) = @_; + my $owner = (getpwuid $owneruid)[0] or error "UID $owneruid has no name"; + + print "Restoring $backup/globals.sql ...\n"; + open my $globals, "$backup/globals.sql" or error "$backup/globals.sql: $!"; + open my $psql, "|-", "psql", "--cluster", "$version/$cluster", "-vON_ERROR_STOP=1", "-Xq" or error "psql: $!"; + while (my $line = <$globals>) { + next if ($line eq "CREATE ROLE $owner;\n"); + print $psql $line or error "psql: $!"; + } + close $globals; + close $psql; + error "psql failed" if ($?); +} + +sub create_databases($) { + my ($backup) = @_; + + print "Creating databases from $backup/databases.sql ...\n"; + system_or_error "psql", "--cluster", "$version/$cluster", "-vON_ERROR_STOP=1", "-Xqf", "$backup/databases.sql"; +} + +sub restore_dumps($) { + my ($backup) = @_; + + for my $dump (sort glob "$backup/*.dump") { + $dump =~ m!(.*/([^/]*).dump)$!; + $dump = $1; # untaint + my $db = $2; + print "Restoring $dump to database $db ...\n"; + system_or_error "pg_restore", "--cluster", "$version/$cluster", "-d", $db, $dump; + } +} + +sub wait_for_recovery() { + my $sleep = 1; + + print "Waiting for end of recovery ...\n"; + while (1) { + open my $psql, "-|", "psql", "--cluster", "$version/$cluster", "-XAtc", "SELECT pg_is_in_recovery()" or error "psql: $!"; + my $status = <$psql>; + error "psql: $!" unless (defined $status); + close $psql or error "psql: $!"; + last if ($status eq "f\n"); + sleep($sleep++); + } +} + +sub analyze() { + system_or_error "vacuumdb", + "--cluster", "$version/$cluster", + "--analyze-only", + ($version >= 9.4 ? "--analyze-in-stages" : ()), + "--all"; +} + +sub lscluster() { + system_or_error "pg_lsclusters", $version, $cluster; +} + +# main + +my ($backup) = $ARGV[0] =~ /(.*)/; # untaint +error "$backup is not a directory" unless (-d $backup); +$backup =~ s/\/$//; # strip trailing slash +my ($owneruid, $ownergid) = (stat $backup)[4, 5]; +my $configdir = "/etc/postgresql/$version/$cluster"; +$datadir //= replace_v_c($defaultconf{data_directory} // "/var/lib/postgresql/%v/%c", $version, $cluster); +($datadir) = $datadir =~ /(.*)/; # untaint +$wal_archive //= "$backup/../wal"; +$port //= next_free_port(); + +if ($backup =~ /\.backup$/) { + create_cluster_directories($owneruid, $ownergid, $configdir, $datadir); + if (fork == 0) { + switch_to_cluster_owner($owneruid, $ownergid); + restore_config($backup, $configdir); + update_config($configdir, $datadir, $port); + restore_basebackup($backup, $datadir); + restore_wal($backup, $datadir); + archive_recovery_options($backup, $datadir, $wal_archive) + if ($archive_recovery or $recovery_target_time); + exit(0); + } + wait; + exit(1) if ($?); + if ($start) { + print "\n"; + start_cluster(); + switch_to_cluster_owner($owneruid, $ownergid); + wait_for_recovery(); + reset_archive_recovery_options() if ($archive_recovery or $recovery_target_time); + analyze(); + } + print "\n"; + lscluster(); + +} elsif ($backup =~ /\.dump$/) { + create_cluster($backup, $owneruid, $ownergid, $configdir, $datadir); + print "\n"; + if (fork == 0) { + switch_to_cluster_owner($owneruid, $ownergid); + restore_config($backup, $configdir); + update_config($configdir, $datadir, $port); + exit(0); + } + wait; + exit(1) if ($?); + start_cluster(); + switch_to_cluster_owner($owneruid, $ownergid); + restore_globals($backup, $owneruid); + create_databases($backup); + restore_dumps($backup); + analyze(); + print "\n"; + lscluster(); + +} else { + error "$backup must end in either .backup or .dump"; +} + +__END__ + +=head1 NAME + +pg_restorecluster - Restore from a pg_backupcluster backup + +=head1 SYNOPSIS + +B [I] I I I + +=head1 DESCRIPTION + +B restores a PostgreSQL cluster from a backup created by +B. The cluster will be newly created in the system using the +name provided on the command line; this allows renaming a cluster on restore. +The restored cluster configuration will be updated to reflect the new name and +location. + +The I name passed must end in either B<.basebackup> or B<.dump>; +usually this will be the full path to a backup directory in +C as reported by +B. + +Basebackups are restored as-is. For dumps, B is used to +create a new cluster, and schema and data are restored via B. + +=head1 OPTIONS + +=over 4 + +=item B<-d --datadir> I + +Use I as data directory for the restored cluster (default per +createcluster.conf, by default /var/lib/postgresql/I/I). + +=item B<-p --port> I + +Use port I for restored cluster (default is next free port). + +=item B<-s --start> + +Start cluster after restoring (default for restore from dump; off for +basebackup restores). + +After the cluster has been started, B is run on all databases. + +=item B<--archive> + +Configure cluster for recovery from WAL archive. This sets B +to retrieve WAL files from IB. + +=item B<--pitr> I + +=item B<--recovery-target-time> I + +Additionally to setting B, set B to +I for point-in-time recovery. Also sets +B. + +=item B<--wal-archive> I + +For archive recovery, read WAL from archive I (default is +IB). + +=back + +=head1 FILES + +=over 4 + +=item /var/backups + +Default root directory for cluster backup directories. + +=back + +See L for a description of files. + +=head1 SEE ALSO + +L, L, L. + +=head1 AUTHOR + +Christoph Berg Lmyon@debian.orgE> diff --git a/pg_updateaptconfig b/pg_updateaptconfig new file mode 100755 index 0000000..7e041c5 --- /dev/null +++ b/pg_updateaptconfig @@ -0,0 +1,41 @@ +#!/bin/sh + +# Tell apt which PostgreSQL versions have clusters present + +set -eu + +APTCONFDIR="/etc/apt/apt.conf.d" +[ -d "$APTCONFDIR" ] || exit 0 # skip generation on RPM systems +APTCONF="$APTCONFDIR/02autoremove-postgresql" +TMPCONF="$(mktemp --tmpdir pg_updateaptconfig.XXXXXX)" +trap "rm -f $TMPCONF" EXIT + +cat > $TMPCONF <> $TMPCONF +done + +cat >> $TMPCONF < +# (C) 2012-2017 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use warnings; +my @srcdirs = ('/usr/share/hunspell', '/usr/share/myspell/dicts'); +my $cachedir = '/var/cache/postgresql/dicts'; +my $pgsharedir = '/usr/share/postgresql/'; + +use PgCommon; + +# determine encoding of an .aff file +sub get_encoding { + open my $fh, '<', $_[0] or die "cannot open $_[0]: $!"; + while (<$fh>) { + if (/^SET ([\w-]+)\s*$/) { return $1; } + } + return undef; +} + +umask 022; +if ((system 'mkdir', '-p', $cachedir) != 0) { + exit 1; +} + +# keep track of all up to date files, so that we can clean up cruft +my %current; + +print "Building PostgreSQL dictionaries from installed myspell/hunspell packages...\n"; +for my $d (@srcdirs) { + for my $aff (glob "$d/*.aff") { + next if -l $aff; # ignore symlinks + my $dic = substr($aff, 0, -3) . 'dic'; + if (! -f $dic) { + print STDERR "ERROR: $aff does not have corresponding $dic, ignoring\n"; + next; + } + + my $enc = get_encoding $aff; + if (!$enc) { + print STDERR "ERROR: no encoding defined in $aff, ignoring\n"; + next; + } + + my $locale = substr ((split '/', $aff)[-1], 0, -4); + $locale =~ tr /A-Z/a-z/; + + $current{"$cachedir/$locale.affix"} = undef; + $current{"$cachedir/$locale.dict"} = undef; + + # convert to UTF-8 and write to cache dir + print " $locale\n"; + if ((system 'iconv', '-f', $enc, '-t', 'UTF-8', '-o', + "$cachedir/$locale.affix", $aff) != 0) { + unlink "$cachedir/$locale.affix"; + print STDERR "ERROR: Conversion of $aff failed\n"; + next; + } + if ((system 'iconv', '-f', $enc, '-t', 'UTF-8', '-o', + "$cachedir/$locale.dict", $dic) != 0) { + unlink "$cachedir/$locale.affix"; + unlink "$cachedir/$locale.dict"; + print STDERR "ERROR: Conversion of $dic failed\n"; + next; + } + + # install symlinks to all versions >= 8.3 + foreach my $v (get_versions) { + next if $v < '8.3'; + my $dest = "$pgsharedir/$v/tsearch_data"; + next if ! -d $dest; + $current{"$dest/$locale.affix"} = undef; + $current{"$dest/$locale.dict"} = undef; + next if -e "$dest/$locale.affix" && ! -l "$dest/$locale.affix"; + next if -e "$dest/$locale.dict" && ! -l "$dest/$locale.dict"; + unlink "$dest/$locale.affix"; + unlink "$dest/$locale.dict"; + symlink "$cachedir/$locale.affix", "$dest/$locale.affix"; + symlink "$cachedir/$locale.dict", "$dest/$locale.dict"; + } + } +} + +# clean up files for locales which do not exist any more +print "Removing obsolete dictionary files:\n"; +foreach my $f (glob "$cachedir/*") { + next if exists $current{$f}; + print " $f\n"; + unlink $f; +} +foreach my $f ((glob "$pgsharedir/*/tsearch_data/*.affix"), + (glob "$pgsharedir/*/tsearch_data/*.dict")) { + next unless -l $f; + next if exists $current{$f}; + print " $f\n"; + unlink $f; +} + +__END__ + +=head1 NAME + +pg_updatedicts - build PostgreSQL dictionaries from myspell/hunspell ones + +=head1 SYNOPSIS + +B + +=head1 DESCRIPTION + +B makes dictionaries and affix files from installed myspell +and hunspell dictionary packages available to PostgreSQL for usage with tsearch +and word stem support. In particular, it takes all I<*.dic> and I<*.aff> files +from /usr/share/myspell/dicts/, converts them to UTF-8, puts them into +/var/cache/postgresql/dicts/ with I<*.dict> and I<*.affix> suffixes, and +symlinks them into /usr/share/postgresql/I/tsearch_data/, where +PostgreSQL looks for them. + +Through postgresql-common's dpkg trigger, this program is automatically run +whenever a myspell or hunspell dictionary package is installed or upgraded. + +=head1 AUTHOR + +Martin Pitt Lmpitt@debian.orgE> diff --git a/pg_upgradecluster b/pg_upgradecluster new file mode 100755 index 0000000..06f2bf0 --- /dev/null +++ b/pg_upgradecluster @@ -0,0 +1,966 @@ +#!/usr/bin/perl -wT + +# Upgrade a PostgreSQL cluster to a newer major version. +# +# (C) 2005-2009 Martin Pitt +# (C) 2013 Peter Eisentraut +# (C) 2013-2023 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use strict; +use warnings; +use PgCommon; +use File::Temp qw(tempfile); +use Getopt::Long; +use POSIX qw(lchown); + +# untaint environment +$ENV{'PATH'} = '/bin:/usr/bin'; +delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; + +# global variables +my ($version, $newversion, $cluster, $newcluster); +my (%info, %newinfo); +my ($encoding, $old_lc_ctype, $old_lc_collate); # old cluster encoding +my ($old_locale_provider, $old_icu_locale, $old_icu_rules); +my $maintenance_db = 'template1'; +my $keep_on_error = 0; + +# do not trip over cwd not being accessible to postgres superuser +chdir '/'; + +# update the new cluster's conffiles +sub adapt_conffiles { + my ($newversion, $newcluster, $configfile) = @_; + my %c = read_cluster_conf_file $newversion, $newcluster, $configfile; + + # Arguments: + my $deprecate = sub { + my ($conf, $guc, $comment) = @_; + if (defined $conf->{$guc}) { + PgCommon::disable_conf_value $newversion, $newcluster, + $configfile, $guc, $comment; + } + }; + + # Arguments: + my $rename = sub { + my ($conf, $old, $new) = @_; + if (defined ${$conf}{$old}) { + PgCommon::replace_conf_value $newversion, $newcluster, + $configfile, $old, "deprecated in favor of $new", + $new, ${$conf}{$old}; + } + }; + + # Arguments: + my $set = sub { + my ($guc, $val) = @_; + PgCommon::set_conf_value $newversion, $newcluster, $configfile, + $guc, $val; + }; + + # adapt paths to configuration files + if ($configfile eq 'postgresql.conf') { + $set->('data_directory', $newinfo{'pgdata'}); + } else { + # fix bug in pg_upgradecluster 200..202 + $deprecate->(\%c, 'data_directory', 'not valid in postgresql.auto.conf'); + } + for my $guc (qw(hba_file ident_file external_pid_file stats_temp_directory)) { + next unless (defined $c{$guc}); + my $val = $c{$guc}; + $val =~ s/\b\Q$version\E\b/$newversion/g; + $val =~ s/\b\Q$cluster\E\b/$newcluster/g if ($cluster ne $newcluster); + $set->($guc, $val); + } + + if ($newversion >= '8.2') { + # preload_libraries -> shared_preload_libraries transition + $rename->(\%c, 'preload_libraries', 'shared_preload_libraries'); + + # australian_timezones -> timezone_abbreviations transition + my $australian_timezones = config_bool $c{'australian_timezones'}; + if (defined $australian_timezones) { + PgCommon::replace_conf_value $newversion, $newcluster, $configfile, + 'australian_timezones', 'deprecated in favor of timezone_abbreviations', + 'timezone_abbreviations', ($australian_timezones ? 'Australia' : 'Default'); + } + } + + if ($newversion >= '8.3') { + $deprecate->(\%c, 'bgwriter_lru_percent', 'deprecated'); + $deprecate->(\%c, 'bgwriter_all_percent', 'deprecated'); + $deprecate->(\%c, 'bgwriter_all_maxpages', 'deprecated'); + + $rename->(\%c, 'redirect_stderr', 'logging_collector'); + + $rename->(\%c, 'stats_command_string', 'track_activities'); + $deprecate->(\%c, 'stats_start_collector', 'deprecated, always on now'); + $deprecate->(\%c, 'stats_reset_on_server_start', 'deprecated'); + + # stats_block_level and stats_row_level are merged into track_counts + if ($c{'stats_block_level'} || $c{'stats_row_level'}) { + $deprecate->(\%c, 'stats_block_level', 'deprecated in favor of track_counts'); + $deprecate->(\%c, 'stats_row_level', 'deprecated in favor of track_counts'); + $set->('track_counts', (config_bool $c{'stats_block_level'} || config_bool $c{'stats_row_level'}) ? 'on' : 'off'); + } + + # archive_command now has to be enabled explicitly + if ($c{'archive_command'}) { + $set->('archive_mode', 'on'); + } + } + + if ($newversion >= '8.4') { + $deprecate->(\%c, 'max_fsm_pages', 'not needed anymore'); + $deprecate->(\%c, 'max_fsm_relations', 'not needed anymore'); + $deprecate->(\%c, 'krb_server_hostname', 'does not exist anymore'); + $deprecate->(\%c, 'krb_realm', 'does not exist anymore'); + $rename->(\%c, 'explain_pretty_print', 'debug_pretty_print'); + } + + if ($newversion >= '9.0') { + $deprecate->(\%c, 'add_missing_from', 'does not exist anymore'); + $deprecate->(\%c, 'regex_flavor', 'does not exist anymore'); + } + + if ($newversion >= '9.2') { + $deprecate->(\%c, 'wal_sender_delay', 'does not exist anymore'); + $deprecate->(\%c, 'silent_mode', 'does not exist anymore'); + $deprecate->(\%c, 'custom_variable_classes', 'does not exist anymore'); + } + + if ($newversion >= '9.3') { + $rename->(\%c, 'unix_socket_directory', 'unix_socket_directories'); + $rename->(\%c, 'replication_timeout', 'wal_sender_timeout'); + } + + if ($newversion >= '9.4') { + $deprecate->(\%c, 'krb_srvname', 'native krb5 authentication deprecated in favor of GSSAPI'); + # grab dsmt from the new config just written by initdb + if (not $c{dynamic_shared_memory_type} and $configfile eq 'postgresql.conf') { + $set->('dynamic_shared_memory_type', ($newinfo{config}->{dynamic_shared_memory_type} || 'mmap')); + } + } + + if ($newversion >= '9.5') { + if (exists $c{checkpoint_segments}) { + my $max_wal_size = 16*$c{checkpoint_segments} . 'MB'; + $rename->(\%c, 'checkpoint_segments', 'max_wal_size'); + $set->('max_wal_size', $max_wal_size); + } + $deprecate->(\%c, 'ssl_renegotiation_limit', 'does not exist anymore'); + # adapt cluster_name + my $cluster_name = PgCommon::get_conf_value ($newversion, $newcluster, $configfile, 'cluster_name'); + if ($cluster_name) { + $cluster_name =~ s/\Q$version\E/$newversion/g; + $cluster_name =~ s/\Q$cluster\E/$newcluster/g; + $set->('cluster_name', $cluster_name); + } + } + + if ($newversion >= '10') { + $rename->(\%c, 'min_parallel_relation_size', 'min_parallel_table_scan_size'); + $deprecate->(\%c, 'sql_inheritance', 'does not exist anymore'); + } + + if ($newversion >= '11') { + $deprecate->(\%c, 'replacement_sort_tuples', 'does not exist anymore'); + } + + if ($newversion >= '13') { + if (exists $c{wal_keep_segments}) { + my $wal_keep_size = 16*$c{wal_keep_segments} . 'MB'; + $rename->(\%c, 'wal_keep_segments', 'wal_keep_size'); + $set->('wal_keep_size', $wal_keep_size); + } + } + + if ($newversion >= '14') { + $deprecate->(\%c, 'operator_precedence_warning', 'does not exist anymore'); + if ($c{password_encryption} and $c{password_encryption} =~ /^(on|off|true|false)$/) { + $deprecate->(\%c, 'password_encryption', 'password_encryption is not a boolean anymore'); + } + $deprecate->(\%c, 'vacuum_cleanup_index_scale_factor', 'does not exist anymore'); + } + + if ($newversion >= '15') { + $deprecate->(\%c, 'stats_temp_directory', 'does not exist anymore'); + } + + if ($newversion >= '16') { + $deprecate->(\%c, 'promote_trigger_file', 'does not exist anymore, use pg_promote() instead'); + $deprecate->(\%c, 'vacuum_defer_cleanup_age', 'does not exist anymore'); + $deprecate->(\%c, 'force_parallel_mode', 'does not exist anymore'); + } + + if ($newversion >= '17') { + $deprecate->(\%c, 'db_user_namespace', 'does not exist anymore'); + $deprecate->(\%c, 'old_snapshot_threshold', 'does not exist anymore'); + } +} + +sub migrate_config_files() { + # copy configuration files + print "Copying old configuration files...\n"; + install_file $info{'configdir'}.'/postgresql.conf', $newinfo{'configdir'}, + $newinfo{'owneruid'}, $newinfo{'ownergid'}, "644"; + adapt_conffiles $newversion, $newcluster, 'postgresql.conf'; + # copy auto.conf after postgresql.conf has been updated + # (otherwise read_cluster_conf_file would also read auto.conf) + if ($version >= 9.4) { + install_file $info{'pgdata'}.'/postgresql.auto.conf', $newinfo{'pgdata'}, + $newinfo{'owneruid'}, $newinfo{'ownergid'}, "600"; + adapt_conffiles $newversion, $newcluster, 'postgresql.auto.conf'; + } + install_file $info{'configdir'}.'/pg_ident.conf', $newinfo{'configdir'}, + $newinfo{'owneruid'}, $newinfo{'ownergid'}, "640"; + install_file $info{'configdir'}.'/pg_hba.conf', $newinfo{'configdir'}, + $newinfo{'owneruid'}, $newinfo{'ownergid'}, "640"; + if ($version < 8.4 and $newversion >= 8.4) { + print "Removing 'ident sameuser' from pg_hba.conf...\n"; + my $hba = "$PgCommon::confroot/$newversion/$newcluster/pg_hba.conf"; + open O, $hba or error "open $hba: $!"; + open N, ">$hba.new" or error "open $hba.new: $!"; + while () { + s/ident\s+sameuser/ident/; + print N $_; + } + close O; + close N; + lchown $newinfo{'owneruid'}, $newinfo{'ownergid'}, "$hba.new"; + chmod 0640, "$hba.new"; + rename "$hba.new", $hba or error "rename: $!"; + } + if ( -e $info{'configdir'}.'/start.conf') { + print "Copying old start.conf...\n"; + install_file $info{'configdir'}.'/start.conf', $newinfo{'configdir'}, + $newinfo{'owneruid'}, $newinfo{'ownergid'}, "644"; + } + if ( -e $info{'configdir'}.'/pg_ctl.conf') { + print "Copying old pg_ctl.conf...\n"; + install_file $info{'configdir'}.'/pg_ctl.conf', $newinfo{'configdir'}, + $newinfo{'owneruid'}, $newinfo{'ownergid'}, "644"; + } + + # copy SSL files (overwriting any file that pg_createcluster put there) + for my $file (qw/server.crt server.key root.crt root.crl/) { + if ( -e "$info{'pgdata'}/$file") { + print "Copying old $file...\n"; + if (!fork) { # we don't use install_file because that converts symlinks to files + change_ugid $info{'owneruid'}, $info{'ownergid'}; + system "cp -a $info{'pgdata'}/$file $newinfo{'pgdata'}"; + exit 0; + } + wait; + } + } + if ($newversion >= 9.2) { + # SSL certificate paths have an explicit option now, older versions use + # a symlink + for my $f (['server.crt', 'ssl_cert_file'], + ['server.key', 'ssl_key_file'], + ['root.crt', 'ssl_ca_file'], + ['root.crl', 'ssl_crl_file']) { + my $file = "$newinfo{'pgdata'}/$f->[0]"; + if (-l $file) { # migrate symlink to config entry with link target + PgCommon::set_conf_value $newversion, $newcluster, 'postgresql.conf', + $f->[1], (readlink $file); + unlink $file; + } elsif (-e $file) { # plain file in data dir, put in config + PgCommon::set_conf_value $newversion, $newcluster, 'postgresql.conf', + $f->[1], $file; + } + } + } +} + +# Write temporary pg_hba.conf. +# Arguments: +sub temp_hba_conf { + my ($fh, $hba) = tempfile("pg_hba.XXXXXX", TMPDIR => 1, SUFFIX => ".conf"); + + if ($_[0] >= '8.4') { + print $fh "local all $_[2] ident\n"; + } else { + print $fh "local all $_[2] ident sameuser\n"; + } + close $fh; + chmod 0400, $hba; + lchown $_[3], 0, $hba; + + return $hba; +} + +# Get encoding and locales of a running cluster +# Arguments: +sub get_encoding { + my ($version, $cluster) = @_; + $encoding = get_db_encoding $version, $cluster, $maintenance_db; + if ($version <= '8.3') { + ($old_lc_ctype, $old_lc_collate) = get_cluster_locales $version, $cluster; + } else { + ($old_lc_ctype, $old_lc_collate, $old_locale_provider, $old_icu_locale, $old_icu_rules) = get_db_locales $version, $cluster, $maintenance_db; + } + unless ($encoding && $old_lc_ctype && $old_lc_collate) { + error 'could not get cluster locales'; + } +} + +# RedHat's run-parts doesn't support any options, supply a minimalistic implementation here +# BUG: we don't care about validating the filenames yet +# Arguments: +sub run_parts { + my ($dir, @argv) = @_; + for my $script (<$dir/*>) { + my ($s) = $script =~ /(.*)/; # untaint + system ($s, @argv); + error "$s failed: $?" if ($?); + } +} + +sub run_upgrade_scripts($) { + my $phase = shift; + + print "Running $phase phase upgrade hook scripts ...\n"; + if (!fork) { + change_ugid $info{'owneruid'}, $info{'ownergid'}; + + if ($PgCommon::rpm) { + run_parts ("$PgCommon::common_confdir/pg_upgradecluster.d", + $version, $newcluster, $newversion, $phase); + exit; + } + + my @argv = ('run-parts', '--lsbsysinit', '-a', $version, '-a', $newcluster, + '-a', $newversion, '-a', $phase, + "$PgCommon::common_confdir/pg_upgradecluster.d"); + error "$PgCommon::common_confdir/pg_upgradecluster.d script failed" if system @argv; + exit; + } + wait; + if ($? > 0) { + unless ($keep_on_error) { + print STDERR "Error during running upgrade hooks, removing new cluster\n"; + system 'pg_dropcluster', '--stop', $newversion, $newcluster; + } + exit 1; + } +} + +# +# Execution starts here +# + +# command line arguments + +my $newest_version = get_newest_version('postgres'); +$newversion = $newest_version; + +my $method = 'dump'; +my $link = 0; +my $clone = 0; +my $keep_port = 0; +my $start = -1; # -1 = auto + +my ($locale, $lc_collate, $lc_ctype, $lc_messages, $lc_monetary, $lc_numeric, + $lc_time, $logfile, $old_bindir, $jobs); +GetOptions ('v|version=s' => \$newversion, + 'locale=s' => \$locale, + 'lc-collate=s' => \$lc_collate, + 'lc-ctype=s' => \$lc_ctype, + 'lc-messages=s' => \$lc_messages, + 'lc-monetary=s' => \$lc_monetary, + 'lc-numeric=s' => \$lc_numeric, + 'lc-time=s' => \$lc_time, + 'logfile=s' => \$logfile, + 'm|method=s' => \$method, + 'j|jobs=s', => \$jobs, + 'k|link' => \$link, + 'clone' => \$clone, + 'keep-port' => \$keep_port, + 'rename=s' => \$newcluster, + 'old-bindir=s' => \$old_bindir, + 'maintenance-db=s' => \$maintenance_db, + 'start!' => \$start, + 'keep-on-error' => \$keep_on_error, + ) or exit 1; + +if ($method eq 'dump') { + error 'cannot use --link with --method=dump' if ($link); + error 'cannot use --clone with --method=dump' if ($clone); +} elsif ($method eq 'link') { + $method = 'upgrade'; + $link = 1; +} elsif ($method eq 'clone') { + $method = 'upgrade'; + $clone = 1; +} elsif ($method ne 'upgrade') { + error 'method must be "dump", "upgrade", "link", or "clone"'; +} + +# untaint +($newversion) = $newversion =~ /^(\d+\.?\d+)$/; +($locale) = $locale =~ /^([\w@._-]+)$/ if $locale; +($lc_collate) = $lc_collate =~ /^([\w@._-]+)$/ if $lc_collate; +($lc_ctype) = $lc_ctype =~ /^([\w@._-]+)$/ if $lc_ctype; +($lc_messages) = $lc_messages =~ /^([\w@._-]+)$/ if $lc_messages; +($lc_monetary) = $lc_monetary =~ /^([\w@._-]+)$/ if $lc_monetary; +($lc_numeric) = $lc_numeric =~ /^([\w@._-]+)$/ if $lc_numeric; +($lc_time) = $lc_time =~ /^([\w@._-]+)$/ if $lc_time; +($logfile) = $logfile =~ /^([^\n]+)$/ if $logfile; +($old_bindir) = $old_bindir =~ /^(\/.*)$/ if $old_bindir; +($maintenance_db) = $maintenance_db =~ /^([\w-]+)$/ if $maintenance_db; +if ($jobs) { + ($jobs) = $jobs =~ /^(\d+)$/; + $ENV{PGJOBS} = $jobs; # make setting available to upgrade hooks +} + +if ($#ARGV < 1) { + print "Usage: $0 [OPTIONS] []\n"; + exit 1; +} + +($version) = $ARGV[0] =~ /^(\d+\.?\d+)$/; +($cluster) = $ARGV[1] =~ /^([-.\w]+)$/; +$newcluster ||= $cluster; # use old cluster name by default +($newcluster) = $newcluster =~ /^([-.\w]+)$/; +my $datadir; +($datadir) = $ARGV[2] =~ /(.*)/ if defined $ARGV[2]; + +error 'specified cluster does not exist' unless cluster_exists $version, $cluster; +%info = cluster_info ($version, $cluster); +validate_cluster_owner \%info; +error 'cluster is disabled' if $info{'start'} eq 'disabled'; + +error "cluster $version/$cluster is already on version $newversion. " . + "(The newest version installed on this system is $newest_version.)" + if ($version eq $newversion and $cluster eq $newcluster); + +if (cluster_exists $newversion, $newcluster) { + error "target cluster $newversion/$newcluster already exists"; +} + +my $oldcontrol = get_cluster_controldata ($version, $cluster); + +my $oldsocket = get_cluster_socketdir $version, $cluster; +my $owner = getpwuid $info{'owneruid'}; +error 'could not get name of cluster owner' unless $owner; +my $temp_hba_conf = temp_hba_conf $version, $cluster, $owner, $info{'owneruid'}; + +# stop old cluster +if ($info{'running'}) { + get_encoding $version, $cluster; + print "Stopping old cluster...\n"; + my @argv = ('pg_ctlcluster', $version, $cluster, 'stop'); + error "Could not stop old cluster" if system @argv; +} + +if ($method eq 'dump' or ($method eq 'upgrade' and not $info{'running'})) { + print "Restarting old cluster with restricted connections...\n"; + my @argv = ('pg_ctlcluster', + ($old_bindir ? ("--bindir=$old_bindir") : ()), + $version, $cluster, 'start', '-o', "-c hba_file=$temp_hba_conf"); + error "Could not restart old cluster" if system @argv; + + get_encoding $version, $cluster unless ($encoding); # if the cluster was not running before, get encoding now + + if ($method eq 'upgrade') { + print "Stopping old cluster...\n"; + @argv = ('pg_ctlcluster', $version, $cluster, 'stop'); + error "Could not stop old cluster" if system @argv; + } +} + +# in dump mode, old cluster is running now +# in upgrade mode, old cluster is stopped + +my $upgrade_port = next_free_port; + +# create new cluster, preserving encoding and locales +my @argv = ('pg_createcluster', '-u', $info{'owneruid'}, '-g', $info{'ownergid'}, + '--socketdir', $info{'socketdir'}, '--port', $upgrade_port, '--no-status', + $newversion, $newcluster); +push @argv, ('--datadir', $datadir) if $datadir; +push @argv, ('--logfile', $logfile) if $logfile; +push @argv, ('--encoding', $encoding) unless $locale or $lc_ctype; +$lc_ctype ||= $locale || $old_lc_ctype; +$lc_collate ||= $locale || $old_lc_collate; +push @argv, ('--locale', $locale) if $locale; +push @argv, ('--lc-collate', $lc_collate) if $lc_collate; +push @argv, ('--lc-ctype', $lc_ctype) if $lc_ctype; +push @argv, ('--lc-messages', $lc_messages) if $lc_messages; +push @argv, ('--lc-monetary', $lc_monetary) if $lc_monetary; +push @argv, ('--lc-numeric', $lc_numeric) if $lc_numeric; +push @argv, ('--lc-time', $lc_time) if $lc_time; +push @argv, ('--'); +push @argv, ('--locale-provider', $old_locale_provider) if $old_locale_provider; +push @argv, ('--icu-locale', $old_icu_locale) if $old_icu_locale; +push @argv, ('--icu-rules', $old_icu_rules) if $old_icu_rules; +push @argv, ('--data-checksums') if $oldcontrol->{'Data page checksum version'}; # 0 = off +push @argv, ('--encryption-key-command', $info{config}->{encryption_key_command}) if $info{config}->{encryption_key_command}; # PostgreSQL TDE + +# call pg_createcluster +delete $ENV{'LC_ALL'}; +error "Could not create target cluster" if system @argv; +print "\n"; + +# migrate config files to new cluster before running upgrade +%newinfo = cluster_info($newversion, $newcluster); +migrate_config_files(); +set_cluster_port $newversion, $newcluster, $upgrade_port; # use free port during upgrade +%newinfo = cluster_info($newversion, $newcluster); # re-read info after migrate_config_files + +if ($method eq 'dump') { + print "Starting new cluster...\n"; + @argv = ('pg_ctlcluster', $newversion, $newcluster, 'start', '-o', "-c hba_file=$temp_hba_conf"); + error "Could not start target cluster" if system @argv; +} + +my $pg_restore = get_program_path 'pg_restore', $newversion; + +# check whether upgrade scripts exist +my $upgrade_scripts = (-d "$PgCommon::common_confdir/pg_upgradecluster.d" && + ($PgCommon::rpm ? `ls $PgCommon::common_confdir/pg_upgradecluster.d` : + `run-parts --test $PgCommon::common_confdir/pg_upgradecluster.d`)); + +# Run upgrade scripts in init phase +run_upgrade_scripts('init') if ($upgrade_scripts); +print "\n"; + +# dump cluster; drop to cluster owner privileges + +if (!fork) { + change_ugid $info{'owneruid'}, $info{'ownergid'}; + my $pg_dumpall = get_program_path 'pg_dumpall', $newversion; + my $pg_dump = get_program_path 'pg_dump', $newversion; + my $psql = get_program_path 'psql'; + my $newsocket = get_cluster_socketdir $newversion, $newcluster; + + if ($method eq 'dump') { + # get list of databases (value = datallowconn) + my %databases; + open F, '-|', $psql, '-h', $oldsocket, '-p', $info{'port'}, + '-F|', '-d', $maintenance_db, '-AXtc', + 'SELECT datname, datallowconn FROM pg_database' or + error 'Could not get pg_database list'; + while () { + chomp; + my ($n, $a) = split '\|'; + $databases{$n} = ($a eq 't'); + } + close F; + error 'could not get list of databases' if $?; + + # Temporarily enable access to all DBs, so that we can upgrade them + for my $db (keys %databases) { + next if $db eq 'template0'; + + unless ($databases{$db}) { + print "Temporarily enabling access to database $db\n"; + (system $psql, '-h', $oldsocket, '-p', $info{'port'}, '-qX', + '-d', $maintenance_db, '-c', + "BEGIN READ WRITE; UPDATE pg_database SET datallowconn = 't' WHERE datname = '$db'; COMMIT") == 0 or + error 'Could not enable access to database'; + } + } + + # dump schemas + print "Roles, databases, schemas, ACLs...\n"; + open SOURCE, '-|', $pg_dumpall, '-h', $oldsocket, '-p', $info{'port'}, + '-s', '--quote-all-identifiers' or error 'Could not execute pg_dumpall for old cluster'; + my $data = ''; + my $buffer; + while (read SOURCE, $buffer, 1048576) { + $data .= $buffer; + } + close SOURCE; + ($? == 0) or exit 1; + + # remove creation of db superuser role to avoid error message + $data =~ s/^CREATE (ROLE|USER) "\Q$owner\E";\s*$//m; + + # create global objects in target cluster + open SINK, '|-', $psql, '-h', $newsocket, '-p', $newinfo{'port'}, + '-qX', '-d', $maintenance_db or + error 'Could not execute psql for new cluster'; + + # ensure that we can upgrade tables for DBs with default read-only + # transactions + print SINK "BEGIN READ WRITE; ALTER USER $owner SET default_transaction_read_only to off; COMMIT;\n"; + + print SINK $data; + + close SINK; + ($? == 0) or exit 1; + + + # Upgrade databases + for my $db (keys %databases) { + next if $db eq 'template0'; + + print "Fixing hardcoded library paths for stored procedures...\n"; + # starting from 9.0, replace() works on strings; for earlier versions it + # works on bytea + if ($version >= '9.0') { + (system $psql, '-h', $oldsocket, '-p', $info{'port'}, '-qX', '-d', + $db, '-c', "BEGIN READ WRITE; \ + UPDATE pg_proc SET probin = replace(\ + replace(probin, '/usr/lib/postgresql/lib', '\$libdir'), \ + '/usr/lib/postgresql/$version/lib', '\$libdir'); COMMIT") == 0 or + error 'Could not fix library paths'; + } else { + (system $psql, '-h', $oldsocket, '-p', $info{'port'}, '-qX', '-d', + $db, '-c', "BEGIN READ WRITE; \ + UPDATE pg_proc SET probin = decode(replace(\ + replace(encode(probin, 'escape'), '/usr/lib/postgresql/lib', '\$libdir'), \ + '/usr/lib/postgresql/$version/lib', '\$libdir'), 'escape'); COMMIT") == 0 or + error 'Could not fix library paths'; + } + + print 'Upgrading database ', $db, "...\n"; + open SOURCE, '-|', $pg_dump, '-h', $oldsocket, '-p', $info{'port'}, + '-Fc', '--quote-all-identifiers', $db or + error 'Could not execute pg_dump for old cluster'; + + # start pg_restore and copy over everything + my @restore_argv = ($pg_restore, '-h', $newsocket, '-p', + $newinfo{'port'}, '--data-only', '-d', $db, + '--disable-triggers', '--no-data-for-failed-tables'); + open SINK, '|-', @restore_argv or + error 'Could not execute pg_restore for new cluster'; + + my $buffer; + while (read SOURCE, $buffer, 1048576) { + print SINK $buffer; + } + close SOURCE; + ($? == 0) or exit 1; + close SINK; + + # clean up + unless ($databases{$db}) { + print "Disabling access to database $db again\n"; + (system $psql, '-h', $oldsocket, '-p', $info{'port'}, '-qX', + '-d', $maintenance_db, '-c', + "BEGIN READ WRITE; UPDATE pg_database SET datallowconn = 'f' where datname = '$db'; COMMIT") == 0 or + error 'Could not disable access to database in old cluster'; + (system $psql, '-h', $newsocket, '-p', $newinfo{'port'}, '-qX', + '-d', $maintenance_db, '-c', + "BEGIN READ WRITE; UPDATE pg_database SET datallowconn = 'f' where datname = '$db'; COMMIT") == 0 or + error 'Could not disable access to database in new cluster'; + } + } + + # reset owner specific override for default read-only transactions + (system $psql, '-h', $newsocket, '-p', $newinfo{'port'}, '-qX', $maintenance_db, '-c', + "BEGIN READ WRITE; ALTER USER $owner RESET default_transaction_read_only; COMMIT;\n") == 0 or + error 'Could not reset default_transaction_read_only value for superuser'; + } else { + # pg_upgrade + + use File::Temp qw(tempdir); + + my $pg_upgrade = get_program_path 'pg_upgrade', $newversion; + $pg_upgrade or error "pg_upgrade $newversion not found"; + my @argv = ($pg_upgrade, + '-b', ($old_bindir || "$PgCommon::binroot$version/bin"), + '-B', "$PgCommon::binroot$newversion/bin", + '-p', $info{'port'}, + '-P', $newinfo{'port'}, + ); + if ($version <= 9.1) { + push @argv, '-d', $info{pgdata}, '-o', "-D $info{configdir}"; # -o and -D configdir require $newversion >= 9.2 + } else { + push @argv, '-d', $info{configdir}; + } + push @argv, '-D', $newinfo{configdir}; + push @argv, "--link" if $link; + push @argv, "--clone" if $clone; + push @argv, '-j', $jobs if $jobs; + + # Make a directory for pg_upgrade to store its reports and log files. + my $logdir = tempdir("/var/log/postgresql/pg_upgradecluster-$version-$newversion-$newcluster.XXXX"); + chdir $logdir; + + # Run pg_upgrade. + print "@argv\n"; + my $status = system @argv; + + # Remove the PID file of the old cluster (normally removed by + # pg_ctlcluster, but not by pg_upgrade). + unlink "/var/run/postgresql/$version-$cluster.pid"; + + # Move output files to our log directory + if (-d (my $outdir = "$newinfo{pgdata}/pg_upgrade_output.d")) { + system mv => $outdir, $logdir + and error "Could not move $outdir to $logdir"; + } + rmdir $logdir; # remove it if it's empty + print "pg_upgradecluster: pg_upgrade output scripts are in $logdir\n" if (-d $logdir); + + exit 1 if ($status != 0); + } + + exit 0; +} + +wait; + +if ($?) { + print "\n"; + + unless ($keep_on_error) { + print STDERR "Error during cluster dumping, removing new cluster\n"; + system 'pg_dropcluster', '--stop', $newversion, $newcluster; + } + + # Restart old cluster to allow connections again (two steps because we started without systemd) + system 'pg_ctlcluster', $version, $cluster, 'stop'; # ignore errors, it might be down anyway + if ($info{running}) { + print "Starting old cluster again ...\n"; + if (system 'pg_ctlcluster', $version, $cluster, 'start') { + error 'could not start old cluster, please do that manually'; + } + } + exit 1; +} + +if ($method eq 'dump') { + print "Stopping target cluster...\n"; + @argv = ('pg_ctlcluster', $newversion, $newcluster, 'stop'); + error "Could not stop target cluster" if system @argv; + + print "Stopping old cluster...\n"; + @argv = ('pg_ctlcluster', $version, $cluster, 'stop'); + error "Could not stop old cluster" if system @argv; +} + +print "Disabling automatic startup of old cluster...\n"; +my $startconf = $info{'configdir'}.'/start.conf'; +if (open F, ">$startconf") { + print F "# This cluster was upgraded to a newer major version. The old +# cluster has been preserved for backup purposes, but is not started +# automatically. + +manual +"; + close F; +} else { + error "could not create $startconf: $!"; +} + +my $free_port = next_free_port; +unless ($keep_port) { + set_cluster_port $version, $cluster, $upgrade_port; + set_cluster_port $newversion, $newcluster, $info{port}; + $newinfo{port} = $info{port}; +} + +# notify systemd that we modified the old start.conf +if (not exists $ENV{'PG_CLUSTER_CONF_ROOT'} and -d '/run/systemd/system' and $> == 0) { + system 'systemctl daemon-reload'; +} + +# start cluster if it was running before, or upgrade scripts are present +$start = ($info{running} or $upgrade_scripts) if ($start == -1); +if ($start) { + print "Starting upgraded cluster on port $newinfo{port}...\n"; + @argv = ('pg_ctlcluster', $newversion, $newcluster, 'start'); + error "Could not start upgraded cluster; please check configuration and log files" if system @argv; +} + +# Run upgrade scripts in finish phase +if ($upgrade_scripts) { + if ($start) { + run_upgrade_scripts('finish'); + } else { + print "Warning: Skipping upgrade scripts because --no-start was given\n"; + } +} + +print "\nSuccess. Please check that the upgraded cluster works. If it does, +you can remove the old cluster with + pg_dropcluster $version $cluster\n\n"; + +system 'pg_lsclusters', $version, $cluster; +system 'pg_lsclusters', $newversion, $newcluster; + +__END__ + +=head1 NAME + +pg_upgradecluster - upgrade an existing PostgreSQL cluster to a new major version. + +=head1 SYNOPSIS + +B [B<-v> I] I I [I] + +=head1 DESCRIPTION + +B upgrades an existing PostgreSQL server cluster (i. e. a +collection of databases served by a B instance) to a new version +specified by I (default: latest available version). The +configuration files of the old version are copied to the new cluster and +adjusted for the new version. The new cluster is set up to use data page +checksums if the old cluster uses them. + +The cluster of the old version will be configured to use a previously unused +port since the upgraded one will use the original port. The old cluster is not +automatically removed. After upgrading, please verify that the new cluster +indeed works as expected; if so, you should remove the old cluster with +L. Please note that the old cluster is set to "manual" +startup mode, in order to avoid inadvertently changing it; this means that it +will not be started automatically on system boot, and you have to use +L to start/stop it. See section "STARTUP CONTROL" in +L for details. + +The I argument can be used to specify a non-default data directory +of the upgraded cluster. It is passed to B. If not specified, +this defaults to /var/lib/postgresql/I/I. + +=head1 OPTIONS + +=over 4 + +=item B<-v> I + +Set the version to upgrade to (default: latest available). + +=item B<--logfile> I + +Set a custom log file path for the upgraded database cluster. + +=item B<--locale=>I + +Set the default locale for the upgraded database cluster. If this option is not +specified, the locale is inherited from the old cluster. + +When upgrading to PostgreSQL 11 or newer, this option no longer allows +switching the encoding of individual databases. (L was changed to +retain database encodings.) + +=item B<--lc-collate=>I + +=item B<--lc-ctype=>I + +=item B<--lc-messages=>I + +=item B<--lc-monetary=>I + +=item B<--lc-numeric=>I + +=item B<--lc-time=>I + +Like B<--locale>, but only sets the locale in the specified category. + +=item B<-m>, B<--method=>B|B|B|B + +Specify the upgrade method. B uses L and +L, B uses L. The default is +B. + +B and B are shorthands for B<-m upgrade --link> and B<-m upgrade --clone>, +respectively. + +=item B<-k>, B<--link> + +In pg_upgrade mode, use hard links instead of copying files to the new +cluster. This option is merely passed on to pg_upgrade. See +L for details. + +=item B<--clone> + +In pg_upgrade mode, use efficient file cloning (also known as "reflinks" +on some systems) instead of copying files to the new cluster. This option +is merely passed on to pg_upgrade. See L for details. + +=item B<-j>, B<--jobs> + +In pg_upgrade mode, number of simultaneous processes to use. This +option is passed on to pg_upgrade. See L for details. +It is also used by the B upgrade hook (via the B environment +variable). + +=item B<--keep-port> + +By default, the old cluster is moved to a new port, and the new cluster is +moved to the original port so clients will see the upgraded cluster. This +option disables that. + +=item B<--rename=>I + +Use a different name for the upgraded cluster. + +=item B<--old-bindir=>I + +Passed to B. + +=item B<--maintenance-db=>I + +Database to connect to for maintenance queries. The default is B. + +=item B<--[no-]start> + +Start the new database cluster after upgrading. The default is to start the new +cluster if the old cluster was running, or if upgrade hook scripts are present. + +=item B<--keep-on-error> + +If upgrading fails, the newly created cluster is removed. This option disables +that. + +=back + +=head1 HOOK SCRIPTS + +Some PostgreSQL extensions like PostGIS need metadata in auxiliary tables which +must not be upgraded from the old version, but rather initialized for the new +version before copying the table data. For this purpose, extensions (as well as +administrators, of course) can drop upgrade hook scripts into +C. Script file names must consist +entirely of upper and lower case letters, digits, underscores, and hyphens; in +particular, dots (i. e. file extensions) are not allowed. + +Scripts in that directory will be called with the following arguments: + + + +Phases: + +=over + +=item B + +A virgin cluster of version I has been created, i. e. this new +cluster will already have B and B, but no user databases. Please note that +you should not create tables in this phase, since they will be overwritten by +the dump/restore or B operation. + +=item B + +All data from the old version cluster has been dumped/reloaded into the new +one. The old cluster still exists, but is not running. + +=back + +Failing scripts will abort the upgrade. +The scripts are called as the user who owns the database. + +=head1 SEE ALSO + +L, L, L, L + +=head1 AUTHORS + +Martin Pitt Lmpitt@debian.orgE>, Christoph Berg Lmyon@debian.orgE> diff --git a/pg_upgradecluster.d/analyze b/pg_upgradecluster.d/analyze new file mode 100755 index 0000000..f410365 --- /dev/null +++ b/pg_upgradecluster.d/analyze @@ -0,0 +1,33 @@ +#!/bin/sh +# +# Run ANALYZE on all databases in the upgraded cluster + +set -eu + +oldversion="$1" +cluster="$2" +newversion="$3" +phase="$4" + +case $newversion in + 9.2|9.3) + analyze="--analyze-only" + ;; + *) + analyze="--analyze-in-stages" + ;; +esac + +case $newversion in + 9.5|9.6|[1-7]*) + [ "${PGJOBS:-}" ] && jobs="--jobs=$PGJOBS" + ;; +esac + +case $phase in + finish) + vacuumdb --cluster "$newversion/$cluster" --all $analyze ${jobs:-} + ;; +esac + +exit 0 diff --git a/pg_virtualenv b/pg_virtualenv new file mode 100755 index 0000000..f4cfe03 --- /dev/null +++ b/pg_virtualenv @@ -0,0 +1,280 @@ +#!/bin/bash + +# Create a throw-away PostgreSQL environment for running regression tests. +# This does not interfere with existing clusters. +# +# (C) 2005-2012 Martin Pitt +# (C) 2012-2020 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +set -e # no -u here as that breaks PGCONF_OPTS[@] + +# wrap ourselves in newpid if requested +if [ "$PG_VIRTUALENV_NEWPID" ]; then + unset PG_VIRTUALENV_NEWPID + exec newpid $0 "$@" +fi + +# wrap ourselves in unshare if requested +if [ "$PG_VIRTUALENV_UNSHARE" ]; then + export _PG_VIRTUALENV_UNSHARE="$PG_VIRTUALENV_UNSHARE" + unset PG_VIRTUALENV_UNSHARE + exec unshare $_PG_VIRTUALENV_UNSHARE -- $0 "$@" +fi +if [ "$_PG_VIRTUALENV_UNSHARE" ]; then + unset _PG_VIRTUALENV_UNSHARE + # start localhost interface + if [ -x /bin/ip ]; then + ip link set dev lo up || true + else + ifconfig lo up || true + fi +fi + +disable_fakeroot () +{ + case ${LD_PRELOAD:-} in + *fakeroot*) LD_PRELOAD=$(echo "$LD_PRELOAD" | sed -e 's/[^ ]*fakeroot[^ ]*//g') ;; + esac +} + +help () +{ + echo "pg_virtualenv: Create throw-away PostgreSQL environment for regression tests" + echo "Syntax: $0 [options] [command]" + echo " -a use all installed server versions" + echo " -v 'version ...' list of PostgreSQL versions to run [default: latest]" + echo " -c 'options' extra options to pass to pg_createcluster" + echo " -i 'initdb opts' extra initdb options to pass to pg_createcluster" + echo " -o 'guc=value' postgresql.conf options to pass to pg_createcluster" + echo " -p 'package' set options to find extension files in debian/package/" + echo " -s open a shell when command fails" + echo " -t use a temporary cluster directory even as root" + exit ${1:-0} +} + +# option parsing +PGBINROOT="/usr/lib/postgresql/" +#redhat# PGBINROOT="/usr/pgsql-" +PG_VERSIONS="" +PGCONF_OPTS=() +while getopts "ac:i:ho:p:stv:" opt ; do + case $opt in + a) for d in $PGBINROOT*/bin/pg_ctl; do + # prepend version so latest ends up first (i.e. on port 5432) + dir=${d%%/bin/pg_ctl} + PG_VERSIONS="${dir#$PGBINROOT} ${PG_VERSIONS:-}" + done ;; + c) CREATE_OPTS="$OPTARG" ;; + i) INITDB_OPTS="$OPTARG" ;; + h) help ;; + o) PGCONF_OPTS+=("--pgoption" "$OPTARG") ;; + p) PACKAGE="$OPTARG" ;; + s) run_shell=1 ;; + t) NONROOT=1 ;; + v) PG_VERSIONS="$OPTARG" ;; + *) help 1 ;; + esac +done +if [ -z "$PG_VERSIONS" ]; then + # use latest version + d=$(ls -v $PGBINROOT*/bin/pg_ctl 2> /dev/null | tail -1) + if [ -z "$d" ]; then + echo "Could not determine PostgreSQL version, are any PostgreSQL server packages installed?" >&2 + exit 2 + fi + dir=${d%%/bin/pg_ctl} + PG_VERSIONS="${dir#$PGBINROOT}" +fi +# shift away args +shift $(($OPTIND - 1)) +# if no command is given, open a shell +[ "${1:-}" ] || set -- ${SHELL:-/bin/sh} + +# generate a password +if [ -x /usr/bin/pwgen ]; then + export PGPASSWORD=$(pwgen 20 1) +else + export PGPASSWORD=$(dd if=/dev/urandom bs=1k count=1 2>/dev/null | md5sum - | awk '{ print $1 }') +fi + +# we are not root +if [ "$(id -u)" != 0 ]; then + NONROOT=1 +fi +# we aren't really root in fakeroot (but leave it enabled for commands spawned) +case ${LD_PRELOAD:-} in + *fakeroot*) NONROOT=1 ;; +esac + +# non-root operation: create a temp dir where we store everything +if [ "${NONROOT:-}" ]; then + WORKDIR=$(mktemp -d -t pg_virtualenv.XXXXXX) + if [ $(id -u) = 0 ]; then + chown postgres:postgres $WORKDIR + umask 022 + fi + export PG_CLUSTER_CONF_ROOT="$WORKDIR/postgresql" + export PGUSER="${USER:-${LOGNAME:-$(id -un)}}" + [ "$PGUSER" = "root" ] && PGUSER="postgres" + PGSYSCONFDIR="$WORKDIR/postgresql-common" # no export yet so pg_createcluster uses the original createcluster.conf + mkdir "$PGSYSCONFDIR" "$WORKDIR/log" + PWFILE="$PGSYSCONFDIR/pwfile" + LOGDIR="$WORKDIR/log" + echo "$PGPASSWORD" > "$PWFILE" + + cleanup () { + set +e + disable_fakeroot + for v in $PG_VERSIONS; do + # don't drop existing clusters named "regress" + [ -f $PG_CLUSTER_CONF_ROOT/$v/regress/.by_pg_virtualenv ] || continue + echo "Dropping cluster $v/regress ..." + pg_ctlcluster --mode immediate $v regress stop + pg_dropcluster $v regress + done + rm -rf $WORKDIR + } + trap cleanup 0 HUP INT QUIT ILL ABRT PIPE TERM + +# root: keep everything in the standard locations +else + for v in $PG_VERSIONS; do + if [ -d /etc/postgresql/$v/regress ]; then + echo "Cluster $v/regress exists, refusing to overwrite" >&2 + exit 2 + fi + done + + : ${PGSYSCONFDIR:=/etc/postgresql-common} + pg_service="$PGSYSCONFDIR/pg_service.conf" + + export PGUSER="postgres" + PWFILE=$(mktemp -t pgpassword.XXXXXX) + echo "$PGPASSWORD" > "$PWFILE" # write password before chowning the file + chown postgres:postgres "$PWFILE" + + cleanup () { + set +e + rm -f $PWFILE $pg_service + if [ -f $pg_service.pg_virtualenv-save.$$ ]; then + mv -f $pg_service.pg_virtualenv-save.$$ $pg_service + fi + for v in $PG_VERSIONS; do + # don't drop existing clusters named "regress" + [ -f /etc/postgresql/$v/regress/.by_pg_virtualenv ] || continue + echo "Dropping cluster $v/regress ..." + rm -f /etc/postgresql/$v/regress/.by_pg_virtualenv + pg_ctlcluster --mode immediate $v regress stop + pg_dropcluster $v regress + done + } + trap cleanup 0 HUP INT QUIT ILL ABRT PIPE TERM + + if [ -f $pg_service ]; then + mv --no-clobber $pg_service $pg_service.pg_virtualenv-save.$$ + fi +fi + +# create postgres environments +for v in $PG_VERSIONS; do + # create temporary cluster + if [ "${PACKAGE:-}" ]; then + if ! grep -q 'extension_destdir' /usr/share/postgresql/$v/postgresql.conf.sample; then + echo "$0: This PostgreSQL $v installation does not support 'extension_destdir', skipping this version" + [ "$PG_VERSIONS" = "$v" ] && exit 0 # only one version requested + continue + fi + PKGARGS="--pgoption extension_destdir=$PWD/debian/$PACKAGE --pgoption dynamic_library_path=$PWD/debian/$PACKAGE/usr/lib/postgresql/$v/lib:/usr/lib/postgresql/$v/lib" + fi + # we chdir to / so programs don't throw "could not change directory to ..." + ( + cd / + case $v in + 8*|9.0|9.1|9.2) : ;; + *) NOSYNC="--nosync" ;; + esac + if [ "${NONROOT:-}" ]; then + # stats_temp_directory in /var/run/postgresql needs root (or postgres), disable it + case $v in + 8.4|9.*|1[01234]) PGCONF_OPTS+=("--pgoption" "stats_temp_directory=") ;; + esac + fi + # disable fakeroot here because we really can't run as root + # (and still switch to postgres when using fakeroot as root user) + disable_fakeroot + pg_createcluster --quiet \ + ${PGPORT:+-p "$PGPORT"} \ + ${NONROOT:+-d "$WORKDIR/data/$v/regress"} \ + ${NONROOT:+-l "$WORKDIR/log/postgresql-$v-regress.log"} \ + ${CREATE_OPTS:-} --pgoption fsync=off ${PKGARGS:-} "${PGCONF_OPTS[@]}" \ + $v regress -- \ + --username="$PGUSER" --pwfile="$PWFILE" $NOSYNC ${INITDB_OPTS:-} + # in fakeroot, the username will likely default to "postgres" otherwise + echo "This is a temporary throw-away cluster" > ${PG_CLUSTER_CONF_ROOT:-/etc/postgresql}/$v/regress/.by_pg_virtualenv + # start cluster with coredumps enabled + [ $v != 8.2 ] && ENABLE_COREDUMPS="-c" + pg_ctlcluster $v regress start -- ${ENABLE_COREDUMPS:-} + ) + port=$(pg_conftool -s $v regress show port) + + # record cluster information in service file + cat >> $PGSYSCONFDIR/pg_service.conf < /dev/null; then + for v in $PG_VERSIONS; do + PGDATA=$(pg_conftool -s $v regress show data_directory) + for core in $PGDATA/core*; do + [ -f "$core" ] || continue + echo "*** $core ***" + gdb -batch -ex "bt full" /usr/lib/postgresql/$v/bin/postgres "$core" + done + done + fi + + if [ "${run_shell:-}" ]; then + echo "pg_virtualenv: command exited with status $EXIT, dropping you into a shell" + ${SHELL:-/bin/sh} + fi +fi + +exit ${EXIT:-0} diff --git a/pg_virtualenv.pod b/pg_virtualenv.pod new file mode 100644 index 0000000..e1e05f0 --- /dev/null +++ b/pg_virtualenv.pod @@ -0,0 +1,122 @@ +=head1 NAME + +pg_virtualenv - Create a throw-away PostgreSQL environment for running regression tests + +=head1 SYNOPSIS + +B [I] [B<-v> 'I'] [I] + +=head1 DESCRIPTION + +B creates a virtual PostgreSQL server environment, and sets +environment variables such that I can access the PostgreSQL database +server(s). The servers are destroyed when I exits. + +The environment variables B, B, B, and +B will be set. Per default, a single new cluster is created, +using the newest PostgreSQL server version installed. The cluster will use the +first available port number starting from B<5432>, and B will be set. +B is set the the PostgreSQL major version number. + +When clusters for more than one versions are created, they will differ in the +port number used, and B and B are not set. The clusters are +named I/regress. To access a cluster, set +BIB. For ease of access, the clusters are also +registered in F, with the version +number as cluster name. Clusters can be accessed by passing the connection +string "BI", e.g. B. + +When invoked as root, the clusters are created in F as usual; +for other users, B and B are +set to a temporary directory where all files belonging to the clusters are +created. + +If I fails, the tail of the PostgreSQL server log is shown. +Additionally, if B is available, the backtrace from any PostgreSQL +coredump is show. + +=head1 OPTIONS + +=over 4 + +=item B<-a> + +Use all PostgreSQL server versions installed. + +=item B<-v> I + +Use these versions (space-separated list). + +=item B<-c> I + +Extra options to pass to B. + +=item B<-i> I + +Extra initdb options to pass to B. + +=item B<-o> IB<=>I + +Configuration option to set in the C file, passed to +B. + +=item B<-p> I + +Set B and B in cluster to enable +loading and testing extensions at build-time from BIB. + +This is a Debian-specific PostgreSQL patch. + +=item B<-s> + +Launch a shell inside the virtual environment when I fails. + +=item B<-t> + +Install clusters in a temporary directory, even when running as root. + +=item B<-h> + +Show program help. + +=back + +=head1 EXAMPLE + + # pg_virtualenv make check + +=head1 NOTES + +When run with fakeroot(1), B will fall back to the non-root mode +of operation. Running "fakeroot pg_virtualenv" as root will fail, though. + +=head1 ENVIRONMENT + +=over 4 + +=item B=yes + +When non-empty, B will re-exec itself using newpid(1). + +=item B=I + +When non-empty, B will re-exec itself using unshare(1) using +these flags. + +=item B=I + +When set, the value is used for the (single) cluster created. + +=back + +=head1 COMPATIBILITY + +B is set in postgresql-common (>= 219~). + +=head1 SEE ALSO + +initdb(1), pg_createcluster(1). + +=head1 AUTHOR + +Christoph Berg Lmyon@debian.orgE> diff --git a/pg_wrapper b/pg_wrapper new file mode 100755 index 0000000..5b741c5 --- /dev/null +++ b/pg_wrapper @@ -0,0 +1,329 @@ +#!/usr/bin/perl +# Call a PostgreSQL client program with the version, cluster and default +# database specified in ~/.postgresqlrc or +# /etc/postgresql-common/user_clusters. +# +# (C) 2005-2009 Martin Pitt +# (C) 2013-2022 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +use warnings; +use strict; +use POSIX; +use PgCommon; + +my ($version, $cluster); +my $explicit_host = exists $ENV{PGHOST}; +my $explicit_port = $ENV{PGPORT}; +my $explicit_service = exists $ENV{PGSERVICE}; + +# Evaluate PGCLUSTER (unless PGHOST is set as well) +if (exists $ENV{'PGCLUSTER'} and not $explicit_host) { + ($version, $cluster) = split ('/', $ENV{'PGCLUSTER'}, 2); + error "Invalid version $version specified in PGCLUSTER" unless version_exists $version; + error 'No cluster specified with $PGCLUSTER' unless $cluster; +} + +# Check for --cluster argument and filter it out +for (my $i = 0; $i <= $#ARGV; ++$i) { + last if $ARGV[$i] eq '--'; + + if ($ARGV[$i] eq '--cluster') { + error '--cluster option needs an argument (/)' if ($i >= $#ARGV); + + ($version, $cluster) = split ('/', $ARGV[$i+1], 2); + error "Invalid version $version specified with --cluster" unless version_exists $version; + error 'No cluster specified with --cluster' unless $cluster; + + splice @ARGV, $i, 2; + last; + } elsif ($ARGV[$i] =~ /^--cluster=(\d+\.?\d)\/(.+)/) { + ($version, $cluster) = ($1, $2); + error "Invalid version $version specified with --cluster" unless version_exists $version; + error 'No cluster specified with --cluster' unless $cluster; + + splice @ARGV, $i, 1; + last; + } + # --host or -h on command line, drop info from PGCLUSTER + if ($ARGV[$i] =~ /^--host\b/ || $ARGV[$i] =~ /^-\w*h\w*$/) { + ($version, $cluster) = (undef, undef); + $explicit_host = 1; + delete $ENV{PGCLUSTER}; + } + # --port or -p on command line + if ($ARGV[$i] =~ /^--port\b(?:=(\d+))?/ || $ARGV[$i] =~ /^-\w*p(\d+)?$/) { + if (defined $1) { + $explicit_port = $1; + } elsif ($i < $#ARGV) { + $explicit_port = $ARGV[$i+1]; + } + } + # "service=" in connection string + if ($ARGV[$i] =~ /\bservice=/) { + $explicit_service = 1; + } +} + +# if only a port is specified, look for local cluster on specified port +if ($explicit_port and not $version and not $cluster and not $explicit_host and not $explicit_service) { + LOOP: foreach my $v (reverse get_versions()) { + foreach my $c (get_version_clusters $v) { + my $p = get_cluster_port $v, $c; + if ($p eq $explicit_port) { + $version = $v; + # set PGCLUSTER variable for information + $ENV{PGCLUSTER} = "$version/$c"; + last LOOP; + } + } + } +} + +# if we don't have a cluster, and no specific host or port was given, consult postgresqlrc +# or fall back to default port cluster (on 5432), or undef otherwise +my ($db); +($version, $cluster, $db) = user_cluster_map() unless ($cluster or $explicit_host or $explicit_port); + +my ($host, $port); + +if ($cluster) { + # check if we have a network cluster (N.N/the.host.name:port) + if ($cluster =~ /^(\S+):(\d*)$/) { + $host = $1; + $port = $2 || $PgCommon::defaultport; + } elsif (not cluster_exists($version, $cluster)) { + # a specific cluster was requested, error out because it doesn't exist + error "Cluster $version $cluster does not exist"; + } else { + $host = get_cluster_socketdir ($version, $cluster); + $port = get_cluster_port($version, $cluster); + } + # set PGCLUSTER variable for information + $ENV{PGCLUSTER} = "$version/$cluster"; +} + +# setup environment +$ENV{'PGSYSCONFDIR'} //= '/etc/postgresql-common'; +$ENV{'PGHOST'} = $host if ($host); +$ENV{'PGPORT'} = $port if $port && !$ENV{'PGPORT'}; +$ENV{'PGDATABASE'} = $db if $db && !$ENV{'PGDATABASE'}; + +# check under which name we were called +my $cmdname = (split '/', $0)[-1]; + +unless ($version or $explicit_host or $explicit_port or $explicit_service) { + print STDERR "Warning: No existing cluster is suitable as a default target. Please see man pg_wrapper(1) how to specify one.\n"; +} + +# if we have no version yet, use the latest version. If we were called as psql, +# pg_archivecleanup, or pg_isready, always use latest version +if (not $version or $cmdname =~ /^(psql|pg_archivecleanup|pg_isready)$/) { + my $max_version; + if ($version and $version < 9.2) { # psql 15 only supports PG 9.2+ + $max_version = 14; + } + $version = get_newest_version($cmdname, $max_version); +} +unless ($version) { + error 'You must install at least one postgresql-client- package'; +} +error "PostgreSQL version $version is not installed" unless -d "$PgCommon::binroot$version"; + +my $cmd; +if ($cmdname eq 'pg_wrapper') { + error "pg_wrapper called directly but no program given as argument" + if (@ARGV == 0); + $cmd = shift; # will be unshifted back below +} else { + $cmd = get_program_path ($cmdname, $version); +} + +# libreadline is a lot better than libedit, so prefer that on versions that still use it +if ($cmdname eq 'psql' and $version < 13 and not $PgCommon::rpm) { + my @readlines; + # non-multiarch path + @readlines = sort(); + + unless (@readlines) { + # get multiarch dir for our architecture + if (open PS, '-|', '/usr/bin/ldd', $cmd) { + my $out; + read PS, $out, 10000; + close PS; + if ($out =~ m!/libreadline.so!) { + # already linked against libreadline + @readlines = (); + } + else + { + my ($lib_path) = $out =~ m!(/lib/.*)/libedit.so!; + + @readlines = sort(<$lib_path/libreadline.so.?>); + } + } + } + + if (@readlines) { + $ENV{'LD_PRELOAD'} = ($ENV{'LD_PRELOAD'} or '') . ':' . $readlines[-1]; + } +} + +error "pg_wrapper: $cmdname was not found in $PgCommon::binroot$version/bin" unless $cmd; +unshift @ARGV, $cmd; +exec @ARGV; + +__END__ + +=head1 NAME + +pg_wrapper - wrapper for PostgreSQL client commands + +=head1 SYNOPSIS + +I [B<--cluster> I/I] [...] + +(I: B, B, B, and all other client +programs installed in CIC). + +=head1 DESCRIPTION + +This program is run only as a link to names which correspond to PostgreSQL +programs in CIC. It determines the +configured cluster and database for the user and calls the appropriate version +of the desired program to connect to that cluster and database, supplying any +specified options to that command. + +The target cluster is selected by the following means, in descending order of +precedence: + +=over + +=item + +explicit specification with the B<--host> option + +=item + +explicit specification with the B<--cluster> option + +=item + +if the B environment variable is set, no further cluster selection is +performed. The default PostgreSQL version and port number (from the command +line, the environment variable B, or default 5432) will be used. + +=item + +explicit specification with the B environment variable + +=item + +if a port is given (either via B<-p>, B<--port>, or B), and no host is +given, the local cluster matching that port number is used + +=item + +matching entry in C<~/.postgresqlrc> (see L), if that +file exists + +=item + +matching entry in C (see +L), if that file exists + +=item + +If only one cluster exists on the local system, that one will be selected. + +=item + +If several clusters exist on the local system, the one listening on the default port 5432 +will be selected. + +=back + +If none of these rules match, B does not set any environment +variables and the program called will likely error out with a message like +"could not connect to server: Connection refused". + +For B, B, and B, B will always use the binary from +the newest PostgreSQL version installed, as these are downwards compatible. +If the cluster version is older than 9.2, the newest considered binary version is 14. + +Note that B needs to be able to read the server config to get the +port number to connect to. If a non-standard port is configured in a place that +pg_wrapper cannot read, connecting will fail. This particularly holds if the +port was configured via B in C and +pg_wrapper is invoked as any user other than B and B. + +=head1 OPTIONS + +=over + +=item B<--cluster> IBI + +=item B<--cluster> IBIB<:>[I] + +I is either the name of a cluster on the local system, or takes the form +I:I for a remote cluster. If I is left empty (i. e. you just +specify I), it defaults to 5432. + +=back + +=head1 ENVIRONMENT + +=over + +=item B + +If C<$PGCLUSTER> is set, its value (of the form I/I) +specifies the desired cluster, similar to the B<--cluster> option. However, if +B<--cluster> is specified, it overrides the value of C<$PGCLUSTER>. + +=item B + +This specifies an alternative base directory for cluster configurations. This +is usually C, but for testing/development purposes you can +change this to point to e. g. your home directory, so that you can use the +postgresql-common tools without root privileges. + +=item B + +This is the location of PostgreSQL's and postgresql-common's global +configuration (e. g. C, L). The default is +C. + +=back + +=head1 FILES + +=over + +=item C + +stores the default cluster and database for users and groups as set by +the administrators. + +=item C<$HOME/.postgresqlrc> + +stores defaults set by the user himself. + +=back + +=head1 SEE ALSO + +L, L + +=head1 AUTHOR + +Martin Pitt Lmpitt@debian.orgE> diff --git a/pgdg/Makefile b/pgdg/Makefile new file mode 100644 index 0000000..1dc7152 --- /dev/null +++ b/pgdg/Makefile @@ -0,0 +1,2 @@ +all: + if [ -f $(HOME)/apt.postgresql.org/pgapt.conf ]; then ./update; fi diff --git a/pgdg/apt.postgresql.org.asc b/pgdg/apt.postgresql.org.asc new file mode 100644 index 0000000..8480576 --- /dev/null +++ b/pgdg/apt.postgresql.org.asc @@ -0,0 +1,77 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBE6XR8IBEACVdDKT2HEH1IyHzXkb4nIWAY7echjRxo7MTcj4vbXAyBKOfjja +UrBEJWHN6fjKJXOYWXHLIYg0hOGeW9qcSiaa1/rYIbOzjfGfhE4x0Y+NJHS1db0V +G6GUj3qXaeyqIJGS2z7m0Thy4Lgr/LpZlZ78Nf1fliSzBlMo1sV7PpP/7zUO+aA4 +bKa8Rio3weMXQOZgclzgeSdqtwKnyKTQdXY5MkH1QXyFIk1nTfWwyqpJjHlgtwMi +c2cxjqG5nnV9rIYlTTjYG6RBglq0SmzF/raBnF4Lwjxq4qRqvRllBXdFu5+2pMfC +IZ10HPRdqDCTN60DUix+BTzBUT30NzaLhZbOMT5RvQtvTVgWpeIn20i2NrPWNCUh +hj490dKDLpK/v+A5/i8zPvN4c6MkDHi1FZfaoz3863dylUBR3Ip26oM0hHXf4/2U +A/oA4pCl2W0hc4aNtozjKHkVjRx5Q8/hVYu+39csFWxo6YSB/KgIEw+0W8DiTII3 +RQj/OlD68ZDmGLyQPiJvaEtY9fDrcSpI0Esm0i4sjkNbuuh0Cvwwwqo5EF1zfkVj +Tqz2REYQGMJGc5LUbIpk5sMHo1HWV038TWxlDRwtOdzw08zQA6BeWe9FOokRPeR2 +AqhyaJJwOZJodKZ76S+LDwFkTLzEKnYPCzkoRwLrEdNt1M7wQBThnC5z6wARAQAB +tBxQb3N0Z3JlU1FMIERlYmlhbiBSZXBvc2l0b3J5iQJOBBMBCAA4AhsDBQsJCAcD +BRUKCQgLBRYCAwEAAh4BAheAFiEEuXsK/KoaR/BE8kSgf8x9RqzMTPgFAlhtCD8A +CgkQf8x9RqzMTPgECxAAk8uL+dwveTv6eH21tIHcltt8U3Ofajdo+D/ayO53LiYO +xi27kdHD0zvFMUWXLGxQtWyeqqDRvDagfWglHucIcaLxoxNwL8+e+9hVFIEskQAY +kVToBCKMXTQDLarz8/J030Pmcv3ihbwB+jhnykMuyyNmht4kq0CNgnlcMCdVz0d3 +z/09puryIHJrD+A8y3TD4RM74snQuwc9u5bsckvRtRJKbP3GX5JaFZAqUyZNRJRJ +Tn2OQRBhCpxhlZ2afkAPFIq2aVnEt/Ie6tmeRCzsW3lOxEH2K7MQSfSu/kRz7ELf +Cz3NJHj7rMzC+76Rhsas60t9CjmvMuGONEpctijDWONLCuch3Pdj6XpC+MVxpgBy +2VUdkunb48YhXNW0jgFGM/BFRj+dMQOUbY8PjJjsmVV0joDruWATQG/M4C7O8iU0 +B7o6yVv4m8LDEN9CiR6r7H17m4xZseT3f+0QpMe7iQjz6XxTUFRQxXqzmNnloA1T +7VjwPqIIzkj/u0V8nICG/ktLzp1OsCFatWXh7LbU+hwYl6gsFH/mFDqVxJ3+DKQi +vyf1NatzEwl62foVjGUSpvh3ymtmtUQ4JUkNDsXiRBWczaiGSuzD9Qi0ONdkAX3b +ewqmN4TfE+XIpCPxxHXwGq9Rv1IFjOdCX0iG436GHyTLC1tTUIKF5xV4Y0+cXIOI +RgQQEQgABgUCTpdI7gAKCRDFr3dKWFELWqaPAKD1TtT5c3sZz92Fj97KYmqbNQZP ++ACfSC6+hfvlj4GxmUjp1aepoVTo3weJAhwEEAEIAAYFAk6XSQsACgkQTFprqxLS +p64F8Q//cCcutwrH50UoRFejg0EIZav6LUKejC6kpLeubbEtuaIH3r2zMblPGc4i ++eMQKo/PqyQrceRXeNNlqO6/exHozYi2meudxa6IudhwJIOn1MQykJbNMSC2sGUp +1W5M1N5EYgt4hy+qhlfnD66LR4G+9t5FscTJSy84SdiOuqgCOpQmPkVRm1HX5X1+ +dmnzMOCk5LHHQuiacV0qeGO7JcBCVEIDr+uhU1H2u5GPFNHm5u15n25tOxVivb94 +xg6NDjouECBH7cCVuW79YcExH/0X3/9G45rjdHlKPH1OIUJiiX47OTxdG3dAbB4Q +fnViRJhjehFscFvYWSqXo3pgWqUsEvv9qJac2ZEMSz9x2mj0ekWxuM6/hGWxJdB+ ++985rIelPmc7VRAXOjIxWknrXnPCZAMlPlDLu6+vZ5BhFX0Be3y38f7GNCxFkJzl +hWZ4Cj3WojMj+0DaC1eKTj3rJ7OJlt9S9xnO7OOPEUTGyzgNIDAyCiu8F4huLPaT +ape6RupxOMHZeoCVlqx3ouWctelB2oNXcxxiQ/8y+21aHfD4n/CiIFwDvIQjl7dg +mT3u5Lr6yxuosR3QJx1P6rP5ZrDTP9khT30t+HZCbvs5Pq+v/9m6XDmi+NlU7Zuh +Ehy97tL3uBDgoL4b/5BpFL5U9nruPlQzGq1P9jj40dxAaDAX/WKJAj0EEwEIACcC +GwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AFAlB5KywFCQPDFt8ACgkQf8x9RqzM +TPhuCQ//QAjRSAOCQ02qmUAikT+mTB6baOAakkYq6uHbEO7qPZkv4E/M+HPIJ4wd +nBNeSQjfvdNcZBA/x0hr5EMcBneKKPDj4hJ0panOIRQmNSTThQw9OU351gm3YQct +AMPRUu1fTJAL/AuZUQf9ESmhyVtWNlH/56HBfYjE4iVeaRkkNLJyX3vkWdJSMwC/ +LO3Lw/0M3R8itDsm74F8w4xOdSQ52nSRFRh7PunFtREl+QzQ3EA/WB4AIj3VohIG +kWDfPFCzV3cyZQiEnjAe9gG5pHsXHUWQsDFZ12t784JgkGyO5wT26pzTiuApWM3k +/9V+o3HJSgH5hn7wuTi3TelEFwP1fNzI5iUUtZdtxbFOfWMnZAypEhaLmXNkg4zD +kH44r0ss9fR0DAgUav1a25UnbOn4PgIEQy2fgHKHwRpCy20d6oCSlmgyWsR40EPP +YvtGq49A2aK6ibXmdvvFT+Ts8Z+q2SkFpoYFX20mR2nsF0fbt1lfH65P64dukxeR +GteWIeNakDD40bAAOH8+OaoTGVBJ2ACJfLVNM53PEoftavAwUYMrR910qvwYfd/4 +6rh46g1Frr9SFMKYE9uvIJIgDsQB3QBp71houU4H55M5GD8XURYs+bfiQpJG1p7e +B8e5jZx1SagNWc4XwL2FzQ9svrkbg1Y+359buUiP7T6QXX2zY++JAj0EEwEIACcC +GwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AFAlEqbZUFCQg2wEEACgkQf8x9RqzM +TPhFMQ//WxAfKMdpSIA9oIC/yPD/dJpY/+DyouOljpE6MucMy/ArBECjFTBwi/j9 +NYM4ynAk34IkhuNexc1i9/05f5RM6+riLCLgAOsADDbHD4miZzoSxiVr6GQ3YXMb +OGld9kV9Sy6mGNjcUov7iFcf5Hy5w3AjPfKuR9zXswyfzIU1YXObiiZT38l55pp/ +BSgvGVQsvbNjsff5CbEKXS7q3xW+WzN0QWF6YsfNVhFjRGj8hKtHvwKcA02wwjLe +LXVTm6915ZUKhZXUFc0vM4Pj4EgNswH8Ojw9AJaKWJIZmLyW+aP+wpu6YwVCicxB +Y59CzBO2pPJDfKFQzUtrErk9irXeuCCLesDyirxJhv8o0JAvmnMAKOLhNFUrSQ2m ++3EnF7zhfz70gHW+EG8X8mL/EN3/dUM09j6TVrjtw43RLxBzwMDeariFF9yC+5bL +tnGgxjsB9Ik6GV5v34/NEEGf1qBiAzFmDVFRZlrNDkq6gmpvGnA5hUWNr+y0i01L +jGyaLSWHYjgw2UEQOqcUtTFK9MNzbZze4mVaHMEz9/aMfX25R6qbiNqCChveIm8m +Yr5Ds2zdZx+G5bAKdzX7nx2IUAxFQJEE94VLSp3npAaTWv3sHr7dR8tSyUJ9poDw +gw4W9BIcnAM7zvFYbLF5FNggg/26njHCCN70sHt8zGxKQINMc6SJAj0EEwEIACcC +GwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AFAlLpFRkFCQ6EJy0ACgkQf8x9RqzM +TPjOZA//Zp0e25pcvle7cLc0YuFr9pBv2JIkLzPm83nkcwKmxaWayUIG4Sv6pH6h +m8+S/CHQij/yFCX+o3ngMw2J9HBUvafZ4bnbI0RGJ70GsAwraQ0VlkIfg7GUw3Tz +voGYO42rZTru9S0K/6nFP6D1HUu+U+AsJONLeb6oypQgInfXQExPZyliUnHdipei +4WR1YFW6sjSkZT/5C3J1wkAvPl5lvOVthI9Zs6bZlJLZwusKxU0UM4Btgu1Sf3nn +JcHmzisixwS9PMHE+AgPWIGSec/N27a0KmTTvImV6K6nEjXJey0K2+EYJuIBsYUN +orOGBwDFIhfRk9qGlpgt0KRyguV+AP5qvgry95IrYtrOuE7307SidEbSnvO5ezNe +mE7gT9Z1tM7IMPfmoKph4BfpNoH7aXiQh1Wo+ChdP92hZUtQrY2Nm13cmkxYjQ4Z +gMWfYMC+DA/GooSgZM5i6hYqyyfAuUD9kwRN6BqTbuAUAp+hCWYeN4D88sLYpFh3 +paDYNKJ+Gf7Yyi6gThcV956RUFDH3ys5Dk0vDL9NiWwdebWfRFbzoRM3dyGP889a +OyLzS3mh6nHzZrNGhW73kslSQek8tjKrB+56hXOnb4HaElTZGDvD5wmrrhN94kby +Gtz3cydIohvNO9d90+29h0eGEDYti7j7maHkBKUAwlcPvMg5m3Y= +=DA1T +-----END PGP PUBLIC KEY BLOCK----- diff --git a/pgdg/apt.postgresql.org.gpg b/pgdg/apt.postgresql.org.gpg new file mode 100644 index 0000000..afa15cb Binary files /dev/null and b/pgdg/apt.postgresql.org.gpg differ diff --git a/pgdg/apt.postgresql.org.sh b/pgdg/apt.postgresql.org.sh new file mode 100755 index 0000000..6dd5bbe --- /dev/null +++ b/pgdg/apt.postgresql.org.sh @@ -0,0 +1,291 @@ +#!/bin/sh + +# script to add apt.postgresql.org to sources.list.d + +# Copyright (C) 2013-2023 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# The full text of the GPL is distributed as in +# /usr/share/common-licenses/GPL-2 on Debian systems. + +SOURCESLIST="/etc/apt/sources.list.d/pgdg.sources" +TYPES="deb" +COMPONENTS="main" +PGDG="pgdg" + +# variables imported from https://git.postgresql.org/gitweb/?p=pgapt.git;a=blob;f=pgapt.conf +# checked out in $HOME/apt.postgresql.org/; run "make" to update +PG_BETA_VERSION="16" +PG_DEVEL_VERSION="17" +PG_REPOSITORY_DISTS="sid trixie bookworm bullseye buster mantic lunar kinetic jammy focal bionic" +PG_ARCHIVE_DISTS="sid trixie bookworm bullseye buster stretch jessie wheezy squeeze lenny etch mantic lunar kinetic jammy impish hirsute groovy focal eoan disco cosmic bionic zesty xenial wily utopic saucy precise lucid" + +while getopts "c:f:h:ipstv:y" opt ; do + case $opt in + c) COMPONENTS="main $OPTARG" ;; # make these extra components available + f) SOURCESLIST=$OPTARG ;; # sources.list filename to write to + h) HOST="$OPTARG" ;; # hostname to use in sources.list + i) INSTALL="yes" ;; # install packages for version given with -v + p) PURGE="yes" ;; # purge existing postgresql packages + s) TYPES="deb deb-src" ;; # include source repository as well + t) PGDG="pgdg-testing" ;; # use *-pgdg or *-pgdg-testing + v) PGVERSION="$OPTARG" ;; # set up sources.list to use this version (useful for beta/devel packages) + y) ;; # don't ask for confirmation + *) exit 5 ;; + esac + YES="yes" # don't ask for confirmation if any option is given +done +# shift away args +shift $((OPTIND - 1)) +# check options +if [ "$INSTALL" ] && [ -z "$PGVERSION" ]; then + echo "With -i, a version to install must be provided (-v)" + exit 1 +fi + +# codename from command line +CODENAME="$1" +# parse os-release +if [ -z "$CODENAME" ] && [ -f /etc/os-release ]; then + . /etc/os-release + if [ "$VERSION_CODENAME" ]; then # added in buster/xenial + CODENAME="$VERSION_CODENAME" + else + # Debian: VERSION="7.0 (wheezy)" + # Ubuntu: VERSION="13.04, Raring Ringtail" + # VERSION="18.04.1 LTS (Bionic Beaver)" + CODENAME=$(echo $VERSION | sed -ne 's/.*(\(.*\)).*/\1/') # works on Debian only + fi +fi +# try lsb_release +if [ -z "$CODENAME" ] && command -v lsb_release >/dev/null; then + CODENAME=$(lsb_release -cs 2>/dev/null) +fi +# guess from sources.list +if [ -z "$CODENAME" ] && [ -f /etc/apt/sources.list ]; then + CODENAME=$(grep '^deb ' /etc/apt/sources.list | head -n1 | awk '{ print $3 }') +fi +# complain if no result yet +if [ -z "$CODENAME" ]; then + cat < $KEYRING < $SOURCESLIST <> $SOURCESLIST < +# +debian/control: debian/control.in debian/pgversions + pg_buildext checkcontrol + +# run check when clean is invoked +clean: debian/control +.PHONY: debian/control diff --git a/postgresqlrc.5 b/postgresqlrc.5 new file mode 100644 index 0000000..b0e52a5 --- /dev/null +++ b/postgresqlrc.5 @@ -0,0 +1,42 @@ +.TH POSTGRESQLRC 5 "Feburary 2005" "Debian" "Debian PostgreSQL infrastructure" + +.SH NAME +~/.postgresqlrc \- Per\-user PostgreSQL cluster configuration + +.SH DESCRIPTION +The file +.B ~/.postgresqlrc +configures the default PostgreSQL version/cluster and the default +database for an user. If it is not present, the system\-wide file +.B /etc/postgresql\-common/user_clusters +is used instead. + +.SH FORMAT +.P +Comments are introduced by the character +.BR # . +Comments may follow data on a line; the first comment character terminates +the data. +Leading whitespace and blank lines are ignored. + +The first uncommented, non\-blank line is used, all following lines are ignored. + +Fields must be given in the following order, separated by white space: + +.TP +.B VERSION +The major PostgreSQL version of the cluster to connect to. +.TP +.B CLUSTER +The name of a cluster to connect to. A remote cluster is specified +with \fIhost\fR:\fIport\fR. If port is empty, it defaults to 5432. +.TP +.B DATABASE +Within the cluster, the database to which the user will connect by default +if he does not specify a database on the command line. If this is +.BR * , +the default database will be the one named by the user's login id. + +.SH SEE ALSO +.BR pg_wrapper (1), +.BR user_clusters (5) diff --git a/rpm/README b/rpm/README new file mode 100644 index 0000000..a6a9067 --- /dev/null +++ b/rpm/README @@ -0,0 +1,52 @@ +postgresql-common for RedHat +============================ + +The postgresql-common framework was written for Debian/Ubuntu, but most parts +of it work as well on other operating systems. The RPM port changes little in +the original code, and even uses many files from the debian/ directory for +building the packages. + +No separate PostgreSQL client/server packages are provided; the port is backed +by the PGDG RPM packages from yum.postgresql.org. + +The filesystem layout is unchanged, /etc/postgresql, /etc/postgresql-common, +and /var/lib/postgresql are used as before. + +Differences between the Debian and RedHat operating modes are: + +* /var/run/postgresql/ is still used for external pid files, but the default + unix socket directory is /tmp, to match the RPM packages' default. + +* The postgres system user home is /var/lib/pgsql. + +* The binroot is changed from /usr/lib/postgresql/ to /usr/pgsql-. (Note the + missing trailing slash, the idea is that the version number can just be + appended to the path, e.g. /usr/lib/postgresql/9.4/bin becomes + /usr/pgsql-9.4/bin.) + +* The various symlinks for frontend programs in /usr/bin like psql are not + direct symlinks to pg_wrapper, but are added as high-priority alternatives to + the alternatives symlinks set up by the PostgreSQL RPM packages. + +* SSL is disabled by default because there is no easily available snakeoil + certificate. Proper certificates can be configured in createcluster.conf. + +* No attempt is made to setup OOM killer protection for the postmaster process. + +* On Debian, the /etc/init.d/postgresql init script skips versions that have + their own /etc/init.d/postgresql-x.y init script, mostly for compatibility + with legacy packages before the advent of the postgresql-common framework. + The RPM packages provide /etc/init.d/postgresql-x.y scripts, which are + ignored by /etc/init.d/postgresql. The postgresql-x.y scripts will not do + anything as long as the user does not use them to create clusters in + /var/lib/pgsql. (In which case they continue to work as if postgresql-common + was not present.) + +* Debian's pre/postinst/rm maintainer scripts are not used. Mostly this means + there is no automatic integration of tsearch with system-provided + dictionaries. + +The postgresql-common testsuite is supported if perl-Test-Simple and +perl-Time-HiRes are installed. + + -- Christoph Berg Thu, 26 Jun 2014 16:59:47 +0200 diff --git a/rpm/init-functions-compat b/rpm/init-functions-compat new file mode 100644 index 0000000..da13596 --- /dev/null +++ b/rpm/init-functions-compat @@ -0,0 +1,12 @@ +# Functions missing in older /lib/lsb/init-functions scripts + +function_exists () { + type $1 >/dev/null 2>&1 +} + +function_exists log_daemon_msg || log_daemon_msg () { echo -n "$1:${2:+ $2}"; } +function_exists log_progress_msg || log_progress_msg () { echo -n " $1"; } +function_exists log_end_msg || log_end_msg () { if [ $1 -eq 0 ]; then echo "."; else echo "failed!"; fi; } +# this one exists, but we provide it anyway so we don't need to depend on redhat-lsb-core: +function_exists log_warning_msg || log_warning_msg () { echo "$1"; } +function_exists log_failure_msg || log_failure_msg () { echo "$1"; } diff --git a/rpm/postgresql-common.spec b/rpm/postgresql-common.spec new file mode 100644 index 0000000..cb2e45b --- /dev/null +++ b/rpm/postgresql-common.spec @@ -0,0 +1,139 @@ +Name: postgresql-common +Version: %{version} +Release: 1%{?dist} +BuildArch: noarch +Summary: PostgreSQL database-cluster manager +Packager: Debian PostgreSQL Maintainers + +License: GPLv2+ +URL: https://packages.debian.org/sid/%{name} +Source0: http://ftp.debian.org/debian/pool/main/p/%{name}/%{name}_%{version}.tar.xz +Requires: postgresql-client-common +Requires: perl-JSON + +%description +The postgresql-common package provides a structure under which +multiple versions of PostgreSQL may be installed and/or multiple +clusters maintained at one time. + +%package -n postgresql-client-common +Summary: manager for multiple PostgreSQL client versions +%description -n postgresql-client-common +The postgresql-client-common package provides a structure under which +multiple versions of PostgreSQL client programs may be installed at +the same time. It provides a wrapper which selects the right version +for the particular cluster you want to access (with a command line +option, an environment variable, /etc/postgresql-common/user_clusters, +or ~/.postgresqlrc). + +%prep +# unpack tarball, ignoring the name of the top level directory inside +%setup -c +mv */* . + +%build +make + +%install +rm -rf %{buildroot} +# install in subpackages using the Debian files +for inst in debian/*.install; do + pkg=$(basename $inst .install) + [ "$pkg" = "postgresql-server-dev-all" ] && continue + echo "### Reading $pkg files list from $inst ###" + while read file dir; do + [ "$file" = "supported_versions" ] && continue # only relevant on Debian + mkdir -p %{buildroot}/$dir + cp -r $file %{buildroot}/$dir + echo "/$dir/${file##*/}" >> files-$pkg + done < $inst +done +# install manpages +for manpages in debian/*.manpages; do + pkg=$(basename $manpages .manpages) + [ "$pkg" = "postgresql-server-dev-all" ] && continue + echo "### Reading $pkg manpages list from $manpages ###" + while read file; do + section="${file##*.}" + mandir="%{buildroot}%{_mandir}/man$section" + mkdir -p $mandir + for f in $file; do # expand wildcards + cp $f $mandir + echo "%doc %{_mandir}/man$section/$(basename $f).gz" >> files-$pkg + done + done < $manpages +done +# install pg_wrapper symlinks by augmenting the existing pgdg.rpm alternatives +cat debian/postgresql-*common.links | \ +while read dest link; do + name="pgsql-$(basename $link)" + echo "update-alternatives --install /$link $name /$dest 9999" >> postgresql-client-common.post + echo "update-alternatives --remove $name /$dest" >> postgresql-client-common.preun +done +# activate rpm-specific tweaks +sed -i -e 's/#redhat# //' \ + %{buildroot}/lib/systemd/system-generators/postgresql-generator \ + %{buildroot}/usr/bin/pg_config \ + %{buildroot}/usr/bin/pg_virtualenv \ + %{buildroot}/usr/share/perl5/PgCommon.pm \ + %{buildroot}/usr/share/postgresql-common/init.d-functions \ + %{buildroot}/usr/share/postgresql-common/pg_getwal +# install init script +mkdir -p %{buildroot}/etc/init.d %{buildroot}/etc/logrotate.d +cp debian/postgresql-common.postgresql.init %{buildroot}/etc/init.d/postgresql +#cp debian/postgresql-common.postinst %{buildroot}/usr/share/postgresql-common +cp rpm/init-functions-compat %{buildroot}/usr/share/postgresql-common +# ssl defaults to 'off' here because we don't have pregenerated snakeoil certs +sed -e 's/__SSL__/off/' createcluster.conf > %{buildroot}/etc/postgresql-common/createcluster.conf +cp debian/postgresql-common.logrotate %{buildroot}/etc/logrotate.d/postgresql-common + +%files -n postgresql-common -f files-postgresql-common +%attr(0755, root, root) %config /etc/init.d/postgresql +#%attr(0755, root, root) /usr/share/postgresql-common/postgresql-common.postinst +/usr/share/postgresql-common/init-functions-compat +%config /etc/postgresql-common/createcluster.conf +%config /etc/logrotate.d/postgresql-common + +%if 0%{?rhel} >= 7 +%config /lib/systemd/system/*.service +%config /lib/systemd/system/*.timer +%config /lib/systemd/system-generators/postgresql-generator +%endif + +%files -n postgresql-client-common -f files-postgresql-client-common + +%post +# create postgres user +groupadd -g 26 -o -r postgres >/dev/null 2>&1 || : +useradd -M -n -g postgres -o -r -d /var/lib/pgsql -s /bin/bash \ + -c "PostgreSQL Server" -u 26 postgres >/dev/null 2>&1 || : +# create directories so postgres can create clusters without root +install -d -o postgres -g postgres /etc/postgresql /var/lib/postgresql /var/lib/pgsql /var/log/postgresql /var/run/postgresql +# install logrotate config +version_lt () { + newest=$( ( echo "$1"; echo "$2" ) | sort -V | tail -n1) + [ "$1" != "$newest" ] +} +lrversion=$(rpm --queryformat '%{VERSION}' -q logrotate) +if version_lt $lrversion 3.8; then + echo "Adjusting /etc/logrotate.d/postgresql-common for logrotate version $lrversion" + sed -i -e '/ su /d' /etc/logrotate.d/postgresql-common || : +fi + +%post -n postgresql-client-common -f postgresql-client-common.post +update-alternatives --install /usr/bin/ecpg pgsql-ecpg /usr/share/postgresql-common/pg_wrapper 9999 + +%preun -n postgresql-client-common -f postgresql-client-common.preun +update-alternatives --remove pgsql-ecpg /usr/share/postgresql-common/pg_wrapper + +%changelog +* Tue Sep 29 2020 Christoph Berg 217-1 +- Drop postgresql-server-dev-all package, it's debian-specific only. +* Fri Dec 09 2016 Bernd Helmle 177-1 +- New upstream release 177 +* Fri Jun 03 2016 Bernd Helmle 174-2 +- Fix package dependencies and systemd integration +* Thu Aug 7 2014 Christoph Berg 160-1 +- Omit the LD_PRELOAD logic in pg_wrapper +* Thu Jun 5 2014 Christoph Berg 158-1 +- Initial specfile version diff --git a/server/README b/server/README new file mode 100644 index 0000000..867757a --- /dev/null +++ b/server/README @@ -0,0 +1,2 @@ +The files in this directory are only used when compiling Debian packages of the +PostgreSQL database server. They are not required at run time. diff --git a/server/catversion b/server/catversion new file mode 100755 index 0000000..fc4af08 --- /dev/null +++ b/server/catversion @@ -0,0 +1,17 @@ +#!/bin/sh + +# Extract server catalog and control file version numbers. +# This information is stored in the packages and used at install time to +# determine if an in-major-version pg_upgradecluster is required. + +set -eu + +CATVERSION=$(awk '/^#define CATALOG_VERSION_NO/ { print $3 }' src/include/catalog/catversion.h) +CONTROLVERSION=$(awk '/^#define PG_CONTROL_VERSION/ { print $3 }' src/include/catalog/pg_control.h) + +case $CONTROLVERSION in + # control file versions used in PG 9.6 .. 15 + # don't append to catversion to avoid spurious warnings for users of existing packages + 960|1002|1100|1201|1300) echo "$CATVERSION" ;; + *) echo "$CATVERSION-$CONTROLVERSION" ;; +esac diff --git a/server/pg_config.pl b/server/pg_config.pl new file mode 100755 index 0000000..2c90236 --- /dev/null +++ b/server/pg_config.pl @@ -0,0 +1,76 @@ +#!/usr/bin/perl + +# Perl reimplementation of PostgreSQL's pg_config binary. +# We provide this as /usr/bin/pg_config to support cross-compilation using +# libpq-dev. Also, this makes the two installed pg_config copies not conflict +# via their debugging symbols. +# +# This code is released under the terms of the PostgreSQL License. +# Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group +# Author: Christoph Berg + +use strict; +use warnings; + +# no arguments, print all items +if (@ARGV == 0) { + while () { + last if /^$/; # begin of help section + print; + } + exit 0; +} + +# --help or -? +if (grep {$_ =~ /^(--help|-\?)$/} @ARGV) { + while () { + last if /^$/; # begin of help section + } + print; # include empty line in output + while () { + next if /^Report bugs/; # Skip bug address in the perl version + print; + } + exit 0; +} + +# specific value(s) requested +my %options; +my $help; +while () { + last if /^$/; # begin of help section + /^(\S+) = (.*)/ or die "malformatted data item"; + $options{'--' . lc $1} = $2; +} + +foreach my $arg (@ARGV) { + unless ($options{$arg}) { + print "pg_config: invalid argument: $arg\n"; + print "Try \"pg_config --help\" for more information.\n"; + exit 1; + } + print "$options{$arg}\n"; +} + +exit 0; + +# The DATA section consists of the `pg_config` output (one KEY = value item per +# line), and the `pg_config --help` text. The first --help line is empty, which +# we use to detect the beginning of the help section. + +__DATA__ +INCLUDEDIR = /usr/include/postgresql + +pg_config provides information about the installed version of PostgreSQL. + +Usage: + pg_config [OPTION]... + +Options: + --includedir show location of C header files of the client + interfaces + -?, --help show this help, then exit + +With no arguments, all known items are shown. + +Report bugs to . diff --git a/server/postgresql.mk b/server/postgresql.mk new file mode 100644 index 0000000..8e73f66 --- /dev/null +++ b/server/postgresql.mk @@ -0,0 +1,294 @@ +#!/usr/bin/make -f + +# The PostgreSQL server packages include this in their debian/rules file. + +# MAJOR_VER is used in path names (15 for /usr/lib/postgresql/15) +ifndef MAJOR_VER +$(error MAJOR_VER must be defined before including this file) +endif +# MAJOR_PKG is used in package names (15 for postgresql-15) +MAJOR_PKG = $(MAJOR_VER) + +# path to auxiliary build files +AUX_MK_DIR = /usr/share/postgresql-common/server + +# version comparison +version_ge = $(shell dpkg --compare-versions $(MAJOR_VER) ge $(1) && echo y) + +# include dpkg makefiles +include /usr/share/dpkg/architecture.mk +include /usr/share/dpkg/pkg-info.mk +include /usr/share/dpkg/vendor.mk +export DEB_BUILD_MAINT_OPTIONS = hardening=+all +DPKG_EXPORT_BUILDFLAGS = 1 +include /usr/share/dpkg/buildflags.mk + +# strict symbol checking +export DPKG_GENSYMBOLS_CHECK_LEVEL = 4 + +# server catalog version +CATVERSION = $(shell $(AUX_MK_DIR)/catversion) + +# configure flags + +CONFIGURE_FLAGS = \ + --with-tcl \ + --with-perl \ + --with-python \ + --with-pam \ + --with-openssl \ + --with-libxml \ + --with-libxslt \ + --mandir=/usr/share/postgresql/$(MAJOR_VER)/man \ + --docdir=/usr/share/doc/postgresql-doc-$(MAJOR_VER) \ + --sysconfdir=/etc/postgresql-common \ + --datarootdir=/usr/share/ \ + --datadir=/usr/share/postgresql/$(MAJOR_VER) \ + --bindir=/usr/lib/postgresql/$(MAJOR_VER)/bin \ + --libdir=/usr/lib/$(DEB_HOST_MULTIARCH)/ \ + --libexecdir=/usr/lib/postgresql/ \ + --includedir=/usr/include/postgresql/ \ + --with-extra-version=" ($(DEB_VENDOR) $(DEB_VERSION))" \ + --enable-nls \ + --enable-thread-safety \ + --enable-debug \ + --enable-dtrace \ + --disable-rpath \ + --with-uuid=e2fs \ + --with-gnu-ld \ + --with-gssapi \ + --with-ldap \ + --with-pgport=5432 \ + --with-system-tzdata=/usr/share/zoneinfo \ + AWK=mawk \ + MKDIR_P='/bin/mkdir -p' \ + PROVE='/usr/bin/prove' \ + PYTHON=/usr/bin/python3 \ + TAR='/bin/tar' \ + XSLTPROC='xsltproc --nonet' \ + CFLAGS='$(CFLAGS)' \ + LDFLAGS='$(LDFLAGS)' + +ifeq ($(call version_ge,9.4),y) + CONFIGURE_FLAGS += --enable-tap-tests +endif + +ifeq ($(call version_ge,9.5),y) + ifneq ($(findstring $(DEB_HOST_ARCH), alpha),) + CONFIGURE_FLAGS += --disable-spinlocks + endif +endif + +ifeq ($(call version_ge,10),y) + CONFIGURE_FLAGS += --with-icu +endif + +ifeq ($(call version_ge,11)$(filter pkg.postgresql.nollvm,$(DEB_BUILD_PROFILES)),y) + # if package depends on LLVM, use it + LLVM_VERSIONED_DEP=$(shell grep 'llvm-[0-9]*-dev' debian/control | grep -v "!$(DEB_HOST_ARCH)" | grep -o '[0-9]*' | head -n1) + LLVM_DEP=$(shell grep 'llvm-dev' debian/control | grep -v "!$(DEB_HOST_ARCH)") + ifneq ($(LLVM_VERSIONED_DEP)$(LLVM_DEP),) + ifneq ($(LLVM_VERSIONED_DEP),) + LLVM_VERSION = $(LLVM_VERSIONED_DEP) + LLVM_CONFIG = /usr/bin/llvm-config-$(LLVM_VERSION) + else + LLVM_CONFIG = $(lastword $(shell ls -v /usr/bin/llvm-config-*)) + LLVM_VERSION = $(subst /usr/bin/llvm-config-,,$(LLVM_CONFIG)) + endif + CONFIGURE_FLAGS += --with-llvm LLVM_CONFIG=$(LLVM_CONFIG) CLANG=/usr/bin/clang-$(LLVM_VERSION) + else + LLVM_VERSION = 0.invalid # mute dpkg error on empty version fields in debian/control + endif + TEMP_CONFIG = TEMP_CONFIG=$(AUX_MK_DIR)/test-with-jit.conf +else + LLVM_VERSION = 0.invalid # mute dpkg error on empty version fields in debian/control +endif + +ifeq ($(call version_ge,14),y) + CONFIGURE_FLAGS += --with-lz4 +endif + +ifeq ($(call version_ge,15),y) + ifeq ($(filter pkg.postgresql.nozstd,$(DEB_BUILD_PROFILES)),) + CONFIGURE_FLAGS += --with-zstd + endif +endif + +ifeq ($(call version_ge,17),y) + WITH_PG_BSD_INDENT = y +endif + +# Facilitate hierarchical profile generation on amd64 (#730134) +ifeq ($(DEB_HOST_ARCH),amd64) + CFLAGS += -fno-omit-frame-pointer +endif + +# Work around an ICE bug in GCC 11.2.0, see +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=103395 +ifneq ($(findstring $(DEB_HOST_ARCH), armel armhf),) + CFLAGS+= -DSTAP_SDT_ARG_CONSTRAINT=g +endif + +ifeq ($(DEB_HOST_ARCH_OS),linux) + CONFIGURE_FLAGS += --with-systemd + CONFIGURE_FLAGS += --with-selinux +endif + +ifneq ($(filter pkg.postgresql.cassert,$(DEB_BUILD_PROFILES)),) + CONFIGURE_FLAGS += --enable-cassert + GENCONTROL_FLAGS += -Vcassert='$${Newline}$${Newline}This package has been built with cassert enabled.' +endif + +# alpha/hppa fail stats tests with postgresql-15 +# hurd implemented semaphores only recently and tests still fail a lot +# ia64 fails the infinite_recurse() test with postgresql-16 +# plperl fails on kfreebsd-* (#704802) +# sh4 lets qemu segfaults when building postgresql-16 +# sparc64 fails bin/summarization-and-inprogress-insertion test with postgresql-15 +ifneq ($(filter alpha hppa hurd% ia64 kfreebsd% sh4 sparc64,$(DEB_HOST_ARCH)),) + TEST_FAIL_COMMAND = echo "Ignoring test failures on this architecture" +else + TEST_FAIL_COMMAND = exit 1 +endif + +# recipes + +%: + dh $@ + +override_dh_auto_configure: + dh_auto_configure --builddirectory=build -- $(CONFIGURE_FLAGS) + # remove pre-built documentation + rm -fv doc/src/sgml/*-stamp + +ifeq ($(filter nodoc,$(DEB_BUILD_PROFILES)),) +override_dh_auto_build-indep: + $(MAKE) -C build/doc all # build man + html +endif + +override_dh_auto_build-arch: + # set MAKELEVEL to 0 to force building submake-generated-headers in src/Makefile.global(.in) + MAKELEVEL=0 $(MAKE) -C build/src all + $(MAKE) -C build/doc man # build man only + $(MAKE) -C build/config all + $(MAKE) -C build/contrib all + # build tutorial stuff + $(MAKE) -C build/src/tutorial NO_PGXS=1 +ifeq ($(WITH_PG_BSD_INDENT),y) + $(MAKE) -C build/src/tools/pg_bsd_indent +endif + +override_dh_auto_install-arch: + $(MAKE) -C build/doc/src/sgml install-man DESTDIR=$(CURDIR)/debian/tmp + $(MAKE) -C build/src install DESTDIR=$(CURDIR)/debian/tmp + $(MAKE) -C build/config install DESTDIR=$(CURDIR)/debian/tmp + $(MAKE) -C build/contrib install DESTDIR=$(CURDIR)/debian/tmp + # move SPI examples into server package (they wouldn't be in the doc package in an -A build) + mkdir -p debian/postgresql-$(MAJOR_PKG)/usr/share/doc/postgresql-$(MAJOR_VER) + mv debian/tmp/usr/share/doc/postgresql-doc-$(MAJOR_VER)/extension debian/postgresql-$(MAJOR_PKG)/usr/share/doc/postgresql-$(MAJOR_VER)/examples +ifeq ($(WITH_PG_BSD_INDENT),y) + $(MAKE) -C build/src/tools/pg_bsd_indent install DESTDIR=$(CURDIR)/debian/tmp + install -m755 src/tools/pgindent/pgindent $(CURDIR)/debian/tmp/usr/lib/postgresql/$(MAJOR_VER)/bin + install -m644 src/tools/pgindent/typedefs.list $(CURDIR)/debian/tmp/usr/share/postgresql/$(MAJOR_VER) +endif + +ifeq ($(filter nodoc,$(DEB_BUILD_PROFILES)),) +override_dh_auto_install-indep: + $(MAKE) -C build/doc install DESTDIR=$(CURDIR)/debian/tmp +endif + +override_dh_makeshlibs: + dh_makeshlibs -Xusr/lib/postgresql/$(MAJOR_VER) + +override_dh_auto_clean: + rm -rf build + +override_dh_installchangelogs: + dh_installchangelogs HISTORY + +override_dh_compress: + dh_compress -X.source -X.c + # compress manpages (excluding debian/tmp/) + gzip -9n $(CURDIR)/debian/*-*/usr/share/postgresql/*/man/man*/*.[137] + +override_dh_install-arch: + dh_install -a + + # link README.Debian.gz to postgresql-common + mkdir -p debian/postgresql-$(MAJOR_PKG)/usr/share/doc/postgresql-$(MAJOR_VER) + ln -s ../postgresql-common/README.Debian.gz debian/postgresql-$(MAJOR_PKG)/usr/share/doc/postgresql-$(MAJOR_VER)/README.Debian.gz + + # assemble perl version of pg_config in libpq-dev + mkdir -p debian/libpq-dev/usr/bin + sed -ne '1,/__DATA__/p' $(AUX_MK_DIR)/pg_config.pl > debian/libpq-dev/usr/bin/pg_config + LC_ALL=C debian/postgresql-client-$(MAJOR_PKG)/usr/lib/postgresql/$(MAJOR_VER)/bin/pg_config | sed -e 's![^ ]*/debian/postgresql-client-$(MAJOR_PKG)!!' >> debian/libpq-dev/usr/bin/pg_config + LC_ALL=C debian/postgresql-client-$(MAJOR_PKG)/usr/lib/postgresql/$(MAJOR_VER)/bin/pg_config --help >> debian/libpq-dev/usr/bin/pg_config + chmod 755 debian/libpq-dev/usr/bin/pg_config + [ "$$(debian/libpq-dev/usr/bin/pg_config --bindir)" = "/usr/lib/postgresql/$(MAJOR_VER)/bin" ] + + # remove actual build path from Makefile.global for reproducibility + sed -i -e "s!^abs_top_builddir.*!abs_top_builddir = /build/postgresql-$(MAJOR_PKG)/build!" \ + -e "s!^abs_top_srcdir.*!abs_top_srcdir = /build/postgresql-$(MAJOR_PKG)/build/..!" \ + -e 's!-f\(debug\|file\)-prefix-map=[^ ]* !!g' \ + debian/postgresql-client-$(MAJOR_PKG)/usr/lib/postgresql/$(MAJOR_VER)/lib/pgxs/src/Makefile.global + + # these are shipped in the pl packages + bash -c "rm -v debian/postgresql-$(MAJOR_PKG)/usr/share/postgresql/$(MAJOR_VER)/extension/{plperl,plpython,pltcl,*_pl}*" + bash -c "rm -v debian/postgresql-$(MAJOR_PKG)/usr/lib/postgresql/$(MAJOR_VER)/lib/{plperl,plpython,pltcl,*_pl}*" + bash -c "rm -rfv debian/postgresql-$(MAJOR_PKG)/usr/lib/postgresql/$(MAJOR_VER)/lib/bitcode/*{plperl,plpython,pltcl}*" + + # record catversion in a file + echo $(CATVERSION) > debian/postgresql-$(MAJOR_PKG)/usr/share/postgresql/$(MAJOR_VER)/catalog_version + +override_dh_install-indep: + dh_install -i + + if [ -d debian/postgresql-doc-$(MAJOR_PKG) ]; then set -e; \ + install -d debian/postgresql-doc-$(MAJOR_PKG)/usr/share/doc/postgresql-doc-$(MAJOR_VER)/tutorial; \ + install src/tutorial/*.c src/tutorial/*.source src/tutorial/Makefile src/tutorial/README debian/postgresql-doc-$(MAJOR_PKG)/usr/share/doc/postgresql-doc-$(MAJOR_VER)/tutorial; \ + fi + +override_dh_auto_test-indep: + # nothing to do + +override_dh_auto_test-arch: +ifeq (, $(findstring nocheck, $(DEB_BUILD_OPTIONS))) + # when tests fail, print newest log files + # initdb doesn't like LANG and LC_ALL to contradict, unset LANG and LC_CTYPE here + # temp-install wants to be invoked from a top-level make, unset MAKELEVEL here + # tell pg_upgrade to create its sockets in /tmp to avoid too long paths + unset LANG LC_CTYPE MAKELEVEL; ulimit -c unlimited; \ + if ! make -C build check-world \ + $(TEMP_CONFIG) \ + PGSOCKETDIR="/tmp" \ + PG_TEST_EXTRA='ssl' \ + PROVE_FLAGS="--verbose"; \ + then \ + for l in `find build -name 'regression.*' -o -name '*.log' -o -name '*_log_*' | perl -we 'print map { "$$_\n"; } sort { (stat $$a)[9] <=> (stat $$b)[9] } map { chomp; $$_; } <>' | tail -n 10`; do \ + echo "******** $$l ********"; \ + cat $$l; \ + done; \ + for c in `find build -name 'core*'`; do \ + echo "******** $$c ********"; \ + gdb -batch -ex 'bt full' build/tmp_install/usr/lib/postgresql/$(MAJOR_VER)/bin/postgres $$c || :; \ + done; \ + $(TEST_FAIL_COMMAND); \ + fi +ifeq ($(WITH_PG_BSD_INDENT),y) + $(MAKE) -C build/src/tools/pg_bsd_indent test DESTDIR=$(CURDIR)/debian/tmp +endif +endif + +override_dh_installdeb-arch: + dh_installdeb + # record catversion in preinst + sed -i -e 's/@CATVERSION@/$(CATVERSION)/' debian/postgresql-$(MAJOR_PKG)/DEBIAN/preinst + +override_dh_gencontrol: + # record catversion in .deb control file + dh_gencontrol $(EXCLUDE_PACKAGES) -- -Vpostgresql:Catversion=$(CATVERSION) -Vllvm:Version=$(LLVM_VERSION) $(GENCONTROL_FLAGS) + +ifneq ($(EXCLUDE_PACKAGES),) +override_dh_builddeb: + dh_builddeb $(EXCLUDE_PACKAGES) +endif diff --git a/server/test-with-jit.conf b/server/test-with-jit.conf new file mode 100644 index 0000000..c68a521 --- /dev/null +++ b/server/test-with-jit.conf @@ -0,0 +1,8 @@ +# config used by pg_regress --temp-config + +fsync = off + +# force JITing of all queries in tests +jit = on +jit_above_cost = 0 +jit_optimize_above_cost = 1000 diff --git a/systemd/README.systemd b/systemd/README.systemd new file mode 100644 index 0000000..c818ad3 --- /dev/null +++ b/systemd/README.systemd @@ -0,0 +1,55 @@ +systemd unit files for PostgreSQL on Debian/Ubuntu +-------------------------------------------------- + +Each cluster is run as a separate service, called postgresql@version-cluster. +pg_ctlcluster is invoked with --skip-systemctl-redirect. Logging still goes to +/var/log/postgresql. + +There is a parent service called postgresql.service, that starts/stops/restarts/ +reloads all individual services that are configured as "auto" in +/etc/postgresql/*/*/start.conf. + +The link between start.conf and postgresql.service is established by +postgresql-generator, which creates symlinks in +/run/systemd/generator/postgresql.service.wants/. + +Backup +------ + +Two backup mechanisms are being offered as systemd services: basebackups +capable of point in time recovery (PITR, the preferred method), and SQL-level +dumps. + +pg_basebackup@version-cluster.service +pg_basebackup@version-cluster.timer + + Weekly basebackup in /var/backups/postgresql/version/cluster. + By default, 3 copies are being kept. + + To enable, run + systemctl enable --now pg_basebackup@version-cluster.timer + systemctl start pg_basebackup@version-cluster.service + +pg_receivewal@version-cluster.service + + WAL archival to be used with pg_basebackup@.service for PITR. + + To enable, run + systemctl enable --now pg_basebackup@version-cluster.timer pg_receivewal@version-cluster.service + systemctl start pg_basebackup@version-cluster.service + +pg_dump@version-cluster.service +pg_dump@version-cluster.timer + + Weekly SQL dump in /var/backups/postgresql/version/cluster. + By default, 3 copies are being kept. + + To enable, run + systemctl enable --now pg_dump@version-cluster.timer + systemctl start pg_dump@version-cluster.service + +The mechanisms provided are meant to be used with low to medium size databases. +For larger databases, or databases with high write volume, we advise to use a +full-size backup solution such as pgbackrest or barman. + + -- Christoph Berg Mon, 08 Mar 2021 13:45:26 +0100 diff --git a/systemd/system-generators/postgresql-generator b/systemd/system-generators/postgresql-generator new file mode 100755 index 0000000..12e8102 --- /dev/null +++ b/systemd/system-generators/postgresql-generator @@ -0,0 +1,38 @@ +#!/bin/sh + +# This systemd generator creates dependency symlinks that make all PostgreSQL +# clusters with "auto" in their start.conf file be started/stopped/reloaded +# when postgresql.service is started/stopped/reloaded. + +set -eu + +gendir="$1" +wantdir="$1/postgresql.service.wants" +bindir="/usr/lib/postgresql/" +#redhat# bindir="/usr/pgsql-" +pgservice="/lib/systemd/system/postgresql@.service" + +mkdir -p "$wantdir" + +for conf in /etc/postgresql/*/*/postgresql.conf; do + # abort loop if glob was not expanded (but accept dead symlinks) + if ! test -e "$conf" && ! test -L "$conf"; then continue; fi + + dir="${conf%/*}" + + # evaluate start.conf + if [ -e "$dir/start.conf" ]; then + start=$(sed 's/#.*$//; /^[[:space:]]*$/d; s/^\s*//; s/\s*$//' "$dir/start.conf") + else + start=auto + fi + [ "$start" = "auto" ] || continue + + verdir="${dir%/*}" + version="${verdir##*/}" + test -x "$bindir$version/bin/postgres" || continue # package got removed + cluster="${dir##*/}" + ln -s "$pgservice" "$wantdir/postgresql@$version-$cluster.service" +done + +exit 0 diff --git a/systemd/system/pg_basebackup@.service b/systemd/system/pg_basebackup@.service new file mode 100644 index 0000000..011c11e --- /dev/null +++ b/systemd/system/pg_basebackup@.service @@ -0,0 +1,14 @@ +[Unit] +Description=Basebackup of PostgreSQL Cluster %i +AssertPathExists=/etc/postgresql/%I/postgresql.conf +Wants=postgresql@%i.service +After=postgresql@%i.service +RequiresMountsFor=/var/backups/postgresql + +[Service] +Type=oneshot +User=postgres +Environment="KEEP=3" +ExecStartPre=+/usr/bin/pg_backupcluster %i createdirectory +ExecStart=/usr/bin/pg_backupcluster %i basebackup +ExecStart=/usr/bin/pg_backupcluster %i expirebasebackups $KEEP diff --git a/systemd/system/pg_basebackup@.timer b/systemd/system/pg_basebackup@.timer new file mode 100644 index 0000000..da0bb3f --- /dev/null +++ b/systemd/system/pg_basebackup@.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Weekly Basebackup of PostgreSQL Cluster %i +AssertPathExists=/etc/postgresql/%I/postgresql.conf + +[Timer] +OnCalendar=weekly +RandomizedDelaySec=1h +FixedRandomDelay=true + +[Install] +# when enabled, start along with postgresql@%i +WantedBy=postgresql@%i.service diff --git a/systemd/system/pg_compresswal@.service b/systemd/system/pg_compresswal@.service new file mode 100644 index 0000000..e5eae6b --- /dev/null +++ b/systemd/system/pg_compresswal@.service @@ -0,0 +1,9 @@ +[Unit] +Description=Compress WAL of PostgreSQL Cluster %i +AssertPathExists=/etc/postgresql/%I/postgresql.conf +RequiresMountsFor=/var/backups/postgresql + +[Service] +Type=oneshot +User=postgres +ExecStart=/usr/bin/pg_backupcluster %i compresswal diff --git a/systemd/system/pg_compresswal@.timer b/systemd/system/pg_compresswal@.timer new file mode 100644 index 0000000..6dddbb9 --- /dev/null +++ b/systemd/system/pg_compresswal@.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Daily Compress WAL of PostgreSQL Cluster %i +AssertPathExists=/etc/postgresql/%I/postgresql.conf + +[Timer] +OnCalendar=daily +RandomizedDelaySec=1h +FixedRandomDelay=true + +[Install] +# when enabled, start along with pg_receivewal@%i +WantedBy=pg_receivewal@%i.service diff --git a/systemd/system/pg_dump@.service b/systemd/system/pg_dump@.service new file mode 100644 index 0000000..a7f7f3d --- /dev/null +++ b/systemd/system/pg_dump@.service @@ -0,0 +1,14 @@ +[Unit] +Description=Dump of PostgreSQL Cluster %i +AssertPathExists=/etc/postgresql/%I/postgresql.conf +Wants=postgresql@%i.service +After=postgresql@%i.service +RequiresMountsFor=/var/backups/postgresql + +[Service] +Type=oneshot +User=postgres +Environment="KEEP=3" +ExecStartPre=+/usr/bin/pg_backupcluster %i createdirectory +ExecStart=/usr/bin/pg_backupcluster %i dump +ExecStart=/usr/bin/pg_backupcluster %i expiredumps $KEEP diff --git a/systemd/system/pg_dump@.timer b/systemd/system/pg_dump@.timer new file mode 100644 index 0000000..a1d2799 --- /dev/null +++ b/systemd/system/pg_dump@.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Weekly Dump of PostgreSQL Cluster %i +AssertPathExists=/etc/postgresql/%I/postgresql.conf + +[Timer] +OnCalendar=weekly +RandomizedDelaySec=1h +FixedRandomDelay=true + +[Install] +# when enabled, start along with postgresql@%i +WantedBy=postgresql@%i.service diff --git a/systemd/system/pg_receivewal@.service b/systemd/system/pg_receivewal@.service new file mode 100644 index 0000000..a15b432 --- /dev/null +++ b/systemd/system/pg_receivewal@.service @@ -0,0 +1,21 @@ +[Unit] +Description=WAL archival of PostgreSQL Cluster %i +AssertPathExists=/etc/postgresql/%I/postgresql.conf +Wants=postgresql@%i.service +After=postgresql@%i.service +RequiresMountsFor=/var/backups/postgresql + +[Service] +Type=simple +User=postgres +ExecStartPre=+/usr/bin/pg_backupcluster %i createdirectory +ExecStart=/usr/bin/pg_backupcluster %i receivewal +Restart=on-failure +RestartSec=1min +# pg_receivewal only flushes compressed output on SIGINT +# (https://www.postgresql.org/message-id/flat/Yvo/5No5S0c4EFMj%40msg.df7cb.de) +KillSignal=SIGINT + +[Install] +# when enabled, start along with postgresql@%i +WantedBy=postgresql@%i.service diff --git a/systemd/system/postgresql.service b/systemd/system/postgresql.service new file mode 100644 index 0000000..f53834e --- /dev/null +++ b/systemd/system/postgresql.service @@ -0,0 +1,18 @@ +# postgresql.service is the meta unit for managing all PostgreSQL clusters on +# the system at once. Conceptually, this unit is more like a systemd target, +# but we are using a service since targets cannot be reloaded. +# +# The unit actually managing PostgreSQL clusters is postgresql@.service, +# instantiated as postgresql@15-main.service for individual clusters. + +[Unit] +Description=PostgreSQL RDBMS + +[Service] +Type=oneshot +ExecStart=/bin/true +ExecReload=/bin/true +RemainAfterExit=on + +[Install] +WantedBy=multi-user.target diff --git a/systemd/system/postgresql@.service b/systemd/system/postgresql@.service new file mode 100644 index 0000000..8eed65c --- /dev/null +++ b/systemd/system/postgresql@.service @@ -0,0 +1,40 @@ +# systemd service template for PostgreSQL clusters. The actual instances will +# be called "postgresql@version-cluster", e.g. "postgresql@9.3-main". The +# variable %i expands to "version-cluster", %I expands to "version/cluster". +# (%I breaks for cluster names containing dashes.) + +[Unit] +Description=PostgreSQL Cluster %i +AssertPathExists=/etc/postgresql/%I/postgresql.conf +RequiresMountsFor=/etc/postgresql/%I /var/lib/postgresql/%I +PartOf=postgresql.service +ReloadPropagatedFrom=postgresql.service +Before=postgresql.service +# stop server before networking goes down on shutdown +After=network.target + +[Service] +Type=forking +# -: ignore startup failure (recovery might take arbitrarily long) +# the actual pg_ctl timeout is configured in pg_ctl.conf +ExecStart=-/usr/bin/pg_ctlcluster --skip-systemctl-redirect %i start +# 0 is the same as infinity, but "infinity" needs systemd 229 +TimeoutStartSec=0 +ExecStop=/usr/bin/pg_ctlcluster --skip-systemctl-redirect -m fast %i stop +TimeoutStopSec=1h +ExecReload=/usr/bin/pg_ctlcluster --skip-systemctl-redirect %i reload +PIDFile=/run/postgresql/%i.pid +SyslogIdentifier=postgresql@%i +# prevent OOM killer from choosing the postmaster (individual backends will +# reset the score to 0) +OOMScoreAdjust=-900 +# restarting automatically will prevent "pg_ctlcluster ... stop" from working, +# so we disable it here. Also, the postmaster will restart by itself on most +# problems anyway, so it is questionable if one wants to enable external +# automatic restarts. +#Restart=on-failure +# (This should make pg_ctlcluster stop work, but doesn't:) +#RestartPreventExitStatus=SIGINT SIGTERM + +[Install] +WantedBy=multi-user.target diff --git a/t/001_packages.t b/t/001_packages.t new file mode 100644 index 0000000..697e871 --- /dev/null +++ b/t/001_packages.t @@ -0,0 +1,87 @@ +# Check that the necessary packages are installed + +use warnings; +use strict; + +use lib 't'; +use TestLib; +use POSIX qw/setlocale LC_ALL LC_MESSAGES/; + +use Test::More tests => $PgCommon::rpm ? (3 + 9*@MAJORS) : (15 + 7*@MAJORS); + +ok (-f "/etc/os-release", "/etc/os-release exists"); +my ($os, $osversion) = os_release(); +ok (defined $os, "OS is $os"); +ok (defined $osversion, "OS version is $osversion"); + +note "PostgreSQL versions installed: @MAJORS\n"; +my $f = $ENV{'PG_FLAVOR'} // ''; + +if ($PgCommon::rpm) { + foreach my $v (@MAJORS) { + my $vv = $v; + $vv =~ s/\.//; + + ok ((rpm_installed "postgresql$vv$f"), "postgresql$vv$f installed"); + ok ((rpm_installed "postgresql$vv$f-libs"), "postgresql$vv$f-libs installed"); + ok ((rpm_installed "postgresql$vv$f-server"), "postgresql$vv$f-server installed"); + ok ((rpm_installed "postgresql$vv$f-contrib"), "postgresql$vv$f-contrib installed"); + ok ((rpm_installed "postgresql$vv$f-plperl"), "postgresql$vv$f-plperl installed"); + SKIP: { + skip "No python2 support", 1 unless ($v <= 12); + ok ((rpm_installed "postgresql$vv$f-plpython"), "postgresql$vv$f-plpython installed"); + } + ok ((rpm_installed "postgresql$vv$f-plpython3"), "postgresql$vv$f-plpython3 installed"); + ok ((rpm_installed "postgresql$vv$f-pltcl"), "postgresql$vv$f-pltcl installed"); + ok ((rpm_installed "postgresql$vv$f-devel"), "postgresql$vv$f-devel installed"); + } + exit; +} + +my $docpkgs = 0; +foreach my $v (@MAJORS) { + ok ((deb_installed "postgresql-$v$f"), "postgresql-$v$f installed"); + SKIP: { + skip "No python2 support", 1 unless ($v <= 11 and $PgCommon::have_python2); + ok ((deb_installed "postgresql-plpython-$v$f"), "postgresql-plpython-$v$f installed"); + } + if ($v >= '9.1') { + ok ((deb_installed "postgresql-plpython3-$v$f"), "postgresql-plpython3-$v$f installed"); + } else { + pass "no Python 3 package for version $v"; + } + ok ((deb_installed "postgresql-plperl-$v$f"), "postgresql-plperl-$v$f installed"); + ok ((deb_installed "postgresql-pltcl-$v$f"), "postgresql-pltcl-$v$f installed"); + ok ((deb_installed "postgresql-server-dev-$v$f"), "postgresql-server-dev-$v$f installed"); + SKIP: { + skip "No postgresql-contrib-$v$f package for version $v", 1 if ($v >= 10); + ok ((deb_installed "postgresql-contrib-$v$f"), "postgresql-contrib-$v$f installed"); + } + my $docpkg = "postgresql-doc-$v$f"; + if (deb_installed $docpkg) { + note "$docpkg installed"; + $docpkgs++; + } +} +ok $docpkgs, "At least one doc package installed"; + +ok ((deb_installed 'libecpg-dev'), 'libecpg-dev installed'); +ok ((deb_installed 'procps'), 'procps installed'); +ok ((deb_installed 'netcat-openbsd'), 'netcat-openbsd installed'); + +ok ((deb_installed 'hunspell-en-us'), 'hunspell-en-us installed'); + +# check installed locales to fail tests early if they are missing +ok ((setlocale(LC_MESSAGES, '') =~ /utf8|UTF-8/), 'system has a default UTF-8 locale'); +ok (setlocale (LC_ALL, "ru_RU"), 'locale ru_RU exists'); +ok (setlocale (LC_ALL, "ru_RU.UTF-8"), 'locale ru_RU.UTF-8 exists'); + +my $key_file = '/etc/ssl/private/ssl-cert-snakeoil.key'; +my $pem_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem'; +ok ((getgrnam('ssl-cert'))[3] =~ /postgres/, + 'user postgres in the UNIX group ssl-cert'); +ok (-e $key_file, "$key_file exists"); +is (exec_as ('postgres', "cat $key_file > /dev/null"), 0, "$key_file is readable for postgres"); +ok (-e $pem_file, "$pem_file exists"); + +# vim: filetype=perl diff --git a/t/002_existing_clusters.t b/t/002_existing_clusters.t new file mode 100644 index 0000000..93cc782 --- /dev/null +++ b/t/002_existing_clusters.t @@ -0,0 +1,11 @@ +# Check that no clusters and postgres processes are present for this test. + +use strict; +use Test::More tests => 8; + +use lib 't'; +use TestLib; + +check_clean; + +# vim: filetype=perl diff --git a/t/003_alternatives.t b/t/003_alternatives.t new file mode 100644 index 0000000..dd49d6b --- /dev/null +++ b/t/003_alternatives.t @@ -0,0 +1,37 @@ +# Check that alternatives symlinks point to the correct locations + +use warnings; +use strict; + +use lib 't'; +use TestLib; + +use Test::More tests => $PgCommon::rpm ? 1 : 9; + +if ($PgCommon::rpm) { + ok "No alternatives test on RedHat"; + exit; +} + +# server/client link group +my $newest_version = $ALL_MAJORS[-1]; +note "Newest PG version installed is $newest_version"; +program_ok 0, 'update-alternatives --list psql.1.gz', 0, 'psql.1.gz link group'; +program_ok 0, 'update-alternatives --list postmaster.1.gz', 2, 'postmaster.1.gz link group does not exist'; # removed in pg-common 248 +for my $name (qw(psql pg_dump postgres pg_ctl)) { + is readlink "/etc/alternatives/$name.1.gz", "/usr/share/postgresql/$newest_version/man/man1/$name.1.gz", "$name.1.gz alternative"; +} + +# doc link group +my $newest_doc_version = `dpkg -l 'postgresql-doc-[1-9]*' | sed -ne 's/^ii postgresql-doc-\\([0-9.]*\\).*/\\1/p' | sort -g | tail -n 1`; +note "Newest PG doc version installed is $newest_doc_version"; +SKIP: { + skip "No SPI_connect.3.gz link group on 8.x", 3 if ($newest_doc_version < 9.0); + chomp $newest_doc_version; + program_ok 0, 'update-alternatives --list SPI_connect.3.gz', 0, 'SPI_connect.3.gz link group'; + for my $name (qw(SPI_connect SPI_exec)) { + is readlink "/etc/alternatives/$name.3.gz", "/usr/share/postgresql/$newest_doc_version/man/man3/$name.3.gz", "$name.3.gz alternative"; + } +} + +# vim: filetype=perl diff --git a/t/005_PgCommon.t b/t/005_PgCommon.t new file mode 100644 index 0000000..e1bc337 --- /dev/null +++ b/t/005_PgCommon.t @@ -0,0 +1,311 @@ +# Check PgCommon library functions. + +use strict; + +use File::Temp qw/tempdir/; + +use lib '.'; +use PgCommon; + +use lib 't'; +use TestLib; + +use Test::More tests => 24; + +my $tdir = tempdir (CLEANUP => 1); +$PgCommon::confroot = $tdir; + +# test read_pg_hba with valid file +open P, ">$tdir/pg_hba.conf" or die "Could not create $tdir/pg_hba.conf: $!"; +print P < 'local', 'db' => 'all', 'user' => 'postgres', 'method' => 'ident sameuser' }, + { 'type' => 'local', 'db' => 'foo', 'user' => 'nobody', 'method' => 'trust' }, + { 'type' => 'local', 'db' => 'foo', 'user' => 'nobody', 'method' => 'crypt' }, + { 'type' => 'local', 'db' => 'foo', 'user' => 'nobody,joe', 'method' => 'krb5' }, + { 'type' => 'local', 'db' => 'foo,bar', 'user' => 'nobody', 'method' => 'ident' }, + { 'type' => 'local', 'db' => 'all', 'user' => '+foogrp', 'method' => 'password' }, + { 'type' => 'host', 'db' => '@inc', 'user' => 'all', 'method' => 'md5', 'ip' => '127.0.0.1', 'mask' => '32'}, + { 'type' => 'hostssl', 'db' => 'all', 'user' => '@inc', 'method' => 'pam', 'ip' => '192.168.0.0', 'mask' => '255.255.0.0'}, + { 'type' => 'hostnossl', 'db' => 'all', 'user' => 'all', 'method' => 'reject', 'ip' => '192.168.0.0', 'mask' => '255.255.0.0'}, +); + +my @hba = read_pg_hba "$tdir/pg_hba.conf"; +foreach my $entry (@hba) { + next if $$entry{'type'} eq 'comment'; + if ($#expected_records < 0) { + fail '@expected_records is already empty'; + next; + } + my $expected = shift @expected_records; + my $parsedstr = ''; + my $expectedstr = ''; + foreach my $k (keys %$expected) { + $parsedstr .= $k . ':\'' . $$entry{$k} . '\' '; + $expectedstr .= $k . ':\'' . $$expected{$k} . '\' '; + if ($$expected{$k} ne $$entry{$k}) { + fail "mismatch: $expectedstr ne $parsedstr"; + last; + } + } + pass 'correctly parsed line \'' . $$entry{'line'} . "'"; +} + +ok (($#expected_records == -1), '@expected_records has correct number of entries'); + +# test read_pg_hba with invalid file +my $invalid_hba = <$tdir/pg_hba.conf" or die "Could not create $tdir/pg_hba_invalid.conf: $!"; +print P $invalid_hba; +close P; + +@hba = read_pg_hba "$tdir/pg_hba.conf"; +is (scalar (split "\n", $invalid_hba), $#hba+1, 'returned read_pg_hba array has correct number of records'); +foreach my $entry (@hba) { + is $$entry{'type'}, undef, 'line \'' . $$entry{'line'} . '\' parsed as invalid'; +} + +# test read_conf_file() +my %conf = PgCommon::read_conf_file '/nonexisting'; +is_deeply \%conf, {}, 'read_conf_file returns empty dict for nonexisting file'; + +mkdir "$tdir/8.4"; +mkdir "$tdir/8.4/test" or die "mkdir: $!"; +mkdir "$tdir/conf.d" or die "mkdir: $!"; +my $c = "$tdir/8.4/test/foo.conf"; +open F, ">$c" or die "Could not create $c: $!"; +print F < 42, + 'cintval' => 1, + 'floatval' => '1.5e+3', + 'strval' => 'hello', + 'strval2' => 'world', + 'cstrval' => 'bye', + 'testpath' => '/bin/test', + 'emptystr' => '', + 'cemptystr' => '', + 'quotestr' => "test ! -f '/tmp/%f' && echo 'yes'" + }, 'read_conf_file() parsing'); + +# test read_conf_file() with include directives +open F, ">$tdir/8.4/test/condinc.conf" or die "Could not create $tdir/condinc.conf: $!"; +print F "condint = 42\n"; +close F; + +open F, ">$tdir/bar.conf" or die "Could not create $tdir/bar.conf: $!"; +print F <$tdir/conf.d/sub.conf" or die "Could not create $tdir/conf.d/sub.conf: $!"; +print F <$tdir/relative.conf" or die "Could not create $tdir/relative.conf: $!"; +print F <$tdir/absolute.conf" or die "Could not create $tdir/absolute.conf: $!"; +print F < 42, + 'cintval' => 1, + 'floatval' => '1.5e+3', + 'strval' => 'howdy', + 'strval2' => 'world', + 'cstrval' => 'bye', + 'testpath' => '/bin/test', + 'emptystr' => '', + 'cemptystr' => '', + 'quotestr' => "test ! -f '/tmp/%f' && echo 'yes'", + 'condint' => 42, + 'subvalue' => 1, + 'relativevalue' => 1, + 'absolutevalue' => 1, + }, 'read_conf_file() parsing with include directives'); + + +# test set_conf_value() +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'commented_int', '24'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'commented_str', 'new foo'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'commented_bool', 'on'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'commented_bool2', 'on'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'commented_bool3', 'on'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'intval', '39'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'cintval', '5'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'strval', 'Howdy'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'newval', 'NEW!'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'testpath', '/bin/new'; +PgCommon::set_conf_value '8.4', 'test', 'foo.conf', 'include_dir', 'conf.d'; + +open F, "$c"; +my $conf; +read F, $conf, 1024; +close F; +is ($conf, < 5; + +# test next_free_port(). We are intentionally using nc as an external tool, +# using perl would replicate what next_free_port is doing, and that would +# be a pointless test. +use IPC::Open2; +use Time::HiRes qw(usleep); +my @pids; +# no ports open +is (next_free_port, 5432, 'next_free_port is 5432'); + +# open a localhost ipv4 socket +push @pids, open2(\*CHLD_OUT, \*CHLD_IN, qw(nc -4 -l 127.0.0.1 5432)); +usleep 2*$delay; +is (next_free_port, 5433, 'next_free_port detects localhost ipv4 socket'); +# open a wildcard ipv4 socket +push @pids, open2(\*CHLD_OUT, \*CHLD_IN, qw(nc -4 -l 5433)); +usleep 2*$delay; +is (next_free_port, 5434, 'next_free_port detects wildcard ipv4 socket'); + +SKIP: { + $^V =~ /^v(\d+\.\d+)/; # parse perl version + skip "perl <= 5.10 does not have proper IPv6 support", 2 if ($1 <= 5.10); + skip "skipping IPv6 tests", 2 if ($ENV{SKIP_IPV6}); + + # open a localhost ipv6 socket + push @pids, open2(\*CHLD_OUT, \*CHLD_IN, qw(nc -6 -l ::1 5434)); + usleep 2*$delay; + is (next_free_port, 5435, 'next_free_port detects localhost ipv6 socket'); + # open a wildcard ipv6 socket + push @pids, open2(\*CHLD_OUT, \*CHLD_IN, qw(nc -6 -l 5435)); + usleep 2*$delay; + is (next_free_port, 5436, 'next_free_port detects wildcard ipv6 socket'); +} + +# clean up +kill 15, @pids; + +# vim: filetype=perl diff --git a/t/007_pg_conftool.t b/t/007_pg_conftool.t new file mode 100644 index 0000000..e8465ef --- /dev/null +++ b/t/007_pg_conftool.t @@ -0,0 +1,85 @@ +# Test pg_conftool + +use strict; +use warnings; + +use Test::More tests => 41; +use File::Temp qw/tempdir/; +use lib '.'; +use PgCommon; +use lib 't'; +use TestLib; + +my $tdir = tempdir (CLEANUP => 1); +$ENV{'PG_CLUSTER_CONF_ROOT'} = $tdir; + +open F, "> $tdir/different.conf"; +print F "a = '5'\n"; +print F "#b = '6'\n"; +close F; + +note 'test without cluster'; +is_program_out 0, "pg_conftool show all", 1, "Error: No default cluster found\n"; +is_program_out 0, "pg_conftool foo.conf show all", 1, "Error: No default cluster found\n"; +is_program_out 0, "pg_conftool $tdir/different.conf show all", 0, "a = 5\n"; +is_program_out 0, "pg_conftool 9.7 main show all", 1, "Error: Cluster 9.7 main does not exist\n"; + +my $version = $MAJORS[-1]; +die "Tests past this point need PostgreSQL installed" unless ($version); +mkdir "$tdir/$version"; +mkdir "$tdir/$version/main"; + +open F, "> $tdir/$version/main/postgresql.conf"; +print F "a = '1'\n"; +print F "#b = '2'\n"; +close F; + +open F, "> $tdir/$version/main/other.conf"; +print F "a = '3'\n"; +print F "#b = '4'\n"; +close F; + +sub pgconf { + undef $/; + open F, "$tdir/$version/main/postgresql.conf"; + my $f = ; + close F; + return $f; +} + +sub differentconf { + undef $/; + open F, "$tdir/different.conf"; + my $f = ; + close F; + return $f; +} + +note 'test show'; +is_program_out 0, "pg_conftool show all", 0, "a = 1\n"; +is_program_out 0, "pg_conftool other.conf show all", 0, "a = 3\n"; +is_program_out 0, "pg_conftool $tdir/different.conf show all", 0, "a = 5\n"; +is_program_out 0, "pg_conftool $version main show all", 0, "a = 1\n"; +is_program_out 0, "pg_conftool $version main other.conf show all", 0, "a = 3\n"; +is_program_out 0, "pg_conftool show a", 0, "a = 1\n"; +is_program_out 0, "pg_conftool -s show a", 0, "1\n"; + +note 'test set'; +is_program_out 0, "pg_conftool set c 7", 0, ""; +undef $/; # slurp mode +is pgconf, "a = '1'\n#b = '2'\nc = 7\n", "file contains new setting"; +is_program_out 0, "pg_conftool set a 8", 0, ""; +is pgconf, "a = 8\n#b = '2'\nc = 7\n", "file contains updated setting"; +is_program_out 0, "pg_conftool $tdir/different.conf set a 9", 0, ""; +is differentconf, "a = 9\n#b = '6'\n", "file with path contains updated setting"; + +note 'test remove'; +is_program_out 0, "pg_conftool remove a", 0, ""; +is pgconf, "#a = 8\n#b = '2'\nc = 7\n", "setting removed from file"; +is_program_out 0, "pg_conftool $tdir/different.conf remove a", 0, ""; +is differentconf, "#a = 9\n#b = '6'\n", "setting removed from file with path"; + +note 'test edit'; +$ENV{EDITOR} = 'cat'; +is_program_out 0, "pg_conftool edit", 0, "#a = 8\n#b = '2'\nc = 7\n"; +is_program_out 0, "pg_conftool $tdir/different.conf edit", 0, "#a = 9\n#b = '6'\n"; diff --git a/t/010_defaultport_cluster.t b/t/010_defaultport_cluster.t new file mode 100644 index 0000000..1ba0daa --- /dev/null +++ b/t/010_defaultport_cluster.t @@ -0,0 +1,33 @@ +# We try to call psql with --version and then on localhost. Since there are no +# clusters, we expect an error message that the connection to port 5432 is +# refused. This checks that pg_wrapper correctly picks the default port and +# uses the highest available version. + +use strict; +use Test::More tests => 14; + +use lib 't'; +use TestLib; + +like_program_out 0, 'psql --version', 0, qr/^psql \(PostgreSQL\) $ALL_MAJORS[-1]/, + 'pg_wrapper selects highest available version number'; + +like_program_out 0, 'env LC_MESSAGES=C psql -h 127.0.0.1 -l', 2, qr/could not connect|connection to server .* failed/, + 'connecting to localhost fails with no clusters'; + +# We check if PGCLUSTER, --cluster, and native psql options are evaluated with +# correct priority. (This is related to the checks in t/090_multicluster.t, but +# easier to do here because no clusters are running.) + +like_program_out 0, "env LC_MESSAGES=C PGCLUSTER=$MAJORS[-1]/127.0.0.2:5431 psql -l", + 2, qr/(could not connect|connection to server).*127.0.0.2.* port 5431/s, 'pg_wrapper uses host and port from PGCLUSTER'; +like_program_out 0, "env LC_MESSAGES=C PGCLUSTER=$MAJORS[-1]/127.0.0.2:5431 psql --cluster $MAJORS[-1]/127.0.0.3:5430 -l", + 2, qr/(could not connect|connection to server).*127.0.0.3.* port 5430/s, 'pg_wrapper uses --cluster from the command line'; +like_program_out 0, "env LC_MESSAGES=C PGCLUSTER=$MAJORS[-1]/127.0.0.2:5431 psql -h 127.0.0.3 -l", + 2, qr/(could not connect|connection to server).*127.0.0.3.* port 5432/s, 'pg_wrapper ignores PGCLUSTER with -h on the command line'; +like_program_out 0, "env LC_MESSAGES=C PGCLUSTER=$MAJORS[-1]/127.0.0.2:5431 psql --host 127.0.0.3 -l", + 2, qr/(could not connect|connection to server).*127.0.0.3.* port 5432/s, 'pg_wrapper ignores PGCLUSTER with --host on the command line'; +like_program_out 0, "env LC_MESSAGES=C PGCLUSTER=$MAJORS[-1]/127.0.0.2:5431 PGHOST=127.0.0.3 psql -l", + 2, qr/(could not connect|connection to server).*127.0.0.3.* port 5432/s, 'pg_wrapper ignores PGCLUSTER if PGHOST is set'; + +# vim: filetype=perl diff --git a/t/015_start_stop.t b/t/015_start_stop.t new file mode 100644 index 0000000..a6c4ebf --- /dev/null +++ b/t/015_start_stop.t @@ -0,0 +1,174 @@ +use strict; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More tests => 71 * @MAJORS; + +my $systemd = (-d "/run/systemd/system" and not $ENV{_SYSTEMCTL_SKIP_REDIRECT}); +note $systemd ? "We are running systemd" : "We are not running systemd"; + +# check cluster status +# arguments: +sub check_status { + my ($v, $ctlstatus, $scstatus, $text) = @_; + program_ok (0, "pg_ctlcluster $v main status", $ctlstatus, "cluster $v main $text"); + if ($systemd) { + program_ok (0, "systemctl status postgresql\@$v-main", $scstatus, "service postgresql\@$v-main $text"); + } else { + pass ''; + } +} + +sub check_major { + my $v = $_[0]; + my $ctlstopped = $v >= 9.2 ? 3 : 1; # pg_ctl status "not running" changed in 9.2 + note "Running tests for $v"; + + note "Cluster does not exist yet"; ############################### + check_status $v, 1, 3, "does not exist"; + + # try to start postgresql + if ($systemd) { + program_ok (0, "systemctl start postgresql"); + } else { + program_ok (0, "/etc/init.d/postgresql start"); + } + check_status $v, 1, 3, "does not exist"; + + # try to start specific cluster + if ($systemd) { + program_ok (0, "systemctl start postgresql\@$v-main", 1); + } else { + program_ok (0, "/etc/init.d/postgresql start $v"); + } + check_status $v, 1, 3, "does not exist"; + + note "Start/stop postgresql using system tools"; ############################### + + # create cluster + program_ok (0, "pg_createcluster $v main"); + check_status $v, $ctlstopped, 3, "is stopped"; + + # start postgresql + if ($systemd) { + program_ok (0, "systemctl start postgresql"); + } else { + program_ok (0, "/etc/init.d/postgresql start"); + } + check_status $v, 0, 0, "is running"; + + # start postgresql again + if ($systemd) { + program_ok (0, "systemctl start postgresql"); + } else { + program_ok (0, "/etc/init.d/postgresql start"); + } + check_status $v, 0, 0, "is already running"; + + # stop postgresql + if ($systemd) { + program_ok (0, "systemctl stop postgresql"); + sleep 6; # FIXME: systemctl stop postgresql is not yet synchronous (#759725) + } else { + program_ok (0, "/etc/init.d/postgresql stop"); + } + check_status $v, $ctlstopped, 3, "is stopped"; + + # stop postgresql again + if ($systemd) { + program_ok (0, "systemctl stop postgresql"); + } else { + program_ok (0, "/etc/init.d/postgresql stop"); + } + check_status $v, $ctlstopped, 3, "is already stopped"; + + note "Start/stop specific cluster using system tools"; ############################### + + # start cluster using system tools + if ($systemd) { + program_ok (0, "systemctl start postgresql\@$v-main"); + } else { + program_ok (0, "/etc/init.d/postgresql start $v"); + } + check_status $v, 0, 0, "is running"; + + # try start cluster again + if ($systemd) { + program_ok (0, "systemctl start postgresql\@$v-main"); + } else { + program_ok (0, "/etc/init.d/postgresql start $v"); + } + check_status $v, 0, 0, "is running"; + + # restart cluster + if ($systemd) { + program_ok (0, "systemctl restart postgresql\@$v-main"); + } else { + program_ok (0, "/etc/init.d/postgresql restart $v"); + } + check_status $v, 0, 0, "is running"; + + # stop cluster + if ($systemd) { + program_ok (0, "systemctl stop postgresql\@$v-main"); + } else { + program_ok (0, "/etc/init.d/postgresql stop $v"); + } + check_status $v, $ctlstopped, 3, "is stopped"; + + # try to stop cluster again + if ($systemd) { + program_ok (0, "systemctl stop postgresql\@$v-main"); + } else { + program_ok (0, "/etc/init.d/postgresql stop $v"); + } + check_status $v, $ctlstopped, 3, "is already stopped"; + + # drop cluster + program_ok (0, "pg_dropcluster $v main"); + check_status $v, 1, 3, "does not exist"; + + note "Start/stop specific cluster using pg_*cluster"; ############################### + + # try to start cluster + program_ok (0, "pg_ctlcluster start $v main", 1); # syntax variation: action version cluster + check_status $v, 1, 3, "does not exist"; + + # create cluster and start it + program_ok (0, "pg_createcluster $v main --start"); + check_status $v, 0, 0, "is running"; + + # try to start cluster again + my $exitagain = $systemd ? 0 : 2; + program_ok (0, "pg_ctlcluster $v main start", $exitagain); + check_status $v, 0, 0, "is already running"; + + # restart cluster + program_ok (0, "pg_ctlcluster $v-main restart"); # syntax variation: version-cluster action + check_status $v, 0, 0, "is running"; + + # stop cluster + program_ok (0, "pg_ctlcluster $v main stop"); + check_status $v, $ctlstopped, 3, "is stopped"; + + # try to stop cluster again + program_ok (0, "pg_ctlcluster $v main stop", 2); + check_status $v, $ctlstopped, 3, "is already stopped"; + + # start cluster + program_ok (0, "pg_ctlcluster start $v-main"); # syntax variation: action version-cluster + check_status $v, 0, 0, "is running"; + + # stop server, clean up, check for leftovers + program_ok (0, "pg_dropcluster $v main --stop"); + + check_clean; +} + +foreach (@MAJORS) { + check_major $_; +} + +# vim: filetype=perl diff --git a/t/020_create_sql_remove.t b/t/020_create_sql_remove.t new file mode 100644 index 0000000..d208b99 --- /dev/null +++ b/t/020_create_sql_remove.t @@ -0,0 +1,452 @@ +# We create a cluster, execute some basic SQL commands, drop it again, and +# check that we did not leave anything behind. + +use strict; + +use File::Temp qw(tempdir); +use POSIX qw/dup2/; +use Time::HiRes qw/usleep/; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More tests => 147 * @MAJORS; + +$ENV{_SYSTEMCTL_SKIP_REDIRECT} = 1; # FIXME: testsuite is hanging otherwise + +sub check_major { + my $v = $_[0]; + note "Running tests for $v"; + + # create cluster + my $xlogdir = tempdir("/tmp/$v.xlog.XXXXXX", CLEANUP => 1); + rmdir $xlogdir; # recreated by initdb + if ($v > 8.2) { + my $start_command = $v >= 14 ? "$v *main *5432 *online" : # initdb --no-instructions in 14+ + ($v >= 11 and not $PgCommon::rpm) ? "pg_ctlcluster" : "pg_ctl"; # CLUSTER_START_COMMAND supported in initdb 11+ + like_program_out 'root', "pg_createcluster $v main --start -- -X $xlogdir", 0, qr/$start_command/, + "pg_createcluster $v main"; + } else { # 8.2 does not have -X yet + like_program_out 'root', "pg_createcluster $v main --start", 0, qr/pg_ctl/, + "pg_createcluster $v main"; + system "mv /var/lib/postgresql/$v/main/pg_xlog $xlogdir"; + system "ln -s $xlogdir /var/lib/postgresql/$v/main/pg_xlog"; + } + + # check that a /var/run/postgresql/ pid file is created + my @contents = ('.s.PGSQL.5432', '.s.PGSQL.5432.lock', "$v-main.pid", "$v-main.pg_stat_tmp"); + pop @contents if ($v < 8.4 or $v >= 15); # remove pg_stat_tmp + unless ($PgCommon::rpm and $v < 9.4) { + ok_dir '/var/run/postgresql/', [@contents], + 'Socket and pid file are in /var/run/postgresql/'; + } else { + ok_dir '/var/run/postgresql/', [grep {/main/} @contents], 'Pid File is in /var/run/postgresql/'; + } + + # check that the xlog/wal symlink was created + my $first_xlog = $v >= 9.0 ? "000000010000000000000001" : "000000010000000000000000"; + my @expectdir = ($first_xlog, "archive_status"); + push @expectdir, "summaries" if ($v >= 17); + ok_dir $xlogdir, [@expectdir], + "xlog/wal directory $xlogdir was created"; + + # check pg_hba.conf auth methods + my $local_method = $v >= 9.1 ? 'peer' : + ($v >= 8.4 ? 'ident' : + 'sameuser'); # actually "ident sameuser", but the test is lazy + my $host_method = $v >= 14 ? 'scram-sha-256' : 'md5'; + my (%local_methods, %host_methods); + open my $fh, "/etc/postgresql/$v/main/pg_hba.conf"; + while (<$fh>) { + $local_methods{$1} = 1 if (/^local.*\s(\S+)/); + $host_methods{$1} = 1 if (/^host.*\s(\S+)/); + } + close $fh; + is_deeply [keys %local_methods], [$local_method], "local method in pg_hba.conf is $local_method"; + is_deeply [keys %host_methods], [$host_method], "host method in pg_hba.conf is $host_method"; + + # verify that exactly one postgres master is running + my @pm_pids = pidof ('postgres'); + is $#pm_pids, 0, 'Exactly one postgres master process running'; + + # check environment + my %safe_env = qw/LC_ALL 1 LC_CTYPE 1 LANG 1 PWD 1 PGLOCALEDIR 1 PGSYSCONFDIR 1 PG_GRANDPARENT_PID 1 PG_OOM_ADJUST_FILE 1 PG_OOM_ADJUST_VALUE 1 SHLVL 1 PGDATA 1 _ 1/; + my %env = pid_env 'postgres', $pm_pids[0]; + foreach (keys %env) { + fail "postgres has unsafe environment variable $_" unless exists $safe_env{$_}; + } + + # activate external_pid_file + PgCommon::set_conf_value $v, 'main', 'postgresql.conf', 'external_pid_file', ''; + + # add variable to environment file, restart, check if it's there + open E, ">>/etc/postgresql/$v/main/environment" or + die 'could not open environment file for appending'; + print E "PGEXTRAVAR1 = 1 # short one\nPGEXTRAVAR2='foo bar '\n\n# comment"; + close E; + is_program_out 0, "pg_ctlcluster $v main restart", 0, '', + 'cluster restarts with new environment file'; + + @pm_pids = pidof ('postgres'); + is $#pm_pids, 0, 'Exactly one postgres master process running'; + %env = pid_env 'postgres', $pm_pids[0]; + is $env{'PGEXTRAVAR1'}, '1', 'correct value of PGEXTRAVAR1 in environment'; + is $env{'PGEXTRAVAR2'}, 'foo bar ', 'correct value of PGEXTRAVAR2 in environment'; + + # Now there should not be an external PID file any more, since we set it + # explicitly + unless ($PgCommon::rpm and ($v < 9.4 or $v >= 15)) { + ok_dir '/var/run/postgresql', [grep {! /pid/} @contents], + 'Socket and stats dir, but not PID file in /var/run/postgresql/'; + } else { + ok_dir '/var/run/postgresql', ["$v-main.pg_stat_tmp"], 'Only stats dir in /var/run/postgresql/'; + } + + # verify that the correct client version is selected + like_program_out 'postgres', 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $v/, + 'pg_wrapper+createdb selects version number of cluster'; + + # we always want to use the latest version of "psql", though. + my $max_version = $ALL_MAJORS[-1]; + if ($v < 9.2) { + # if version is older than 9.2 pick v14 at most + $max_version = (grep { $_ <= 14 } @ALL_MAJORS)[-1]; + } + like_program_out 'postgres', 'psql --version', 0, qr/^psql \(PostgreSQL\) $max_version/, + "pg_wrapper+psql selects version $max_version"; + + my $default_log = "/var/log/postgresql/postgresql-$v-main.log"; + + # verify that the cluster is displayed + my $ls = `pg_lsclusters -h`; + $ls =~ s/\s+/ /g; + $ls =~ s/\s*$//; + is $ls, "$v main 5432 online postgres /var/lib/postgresql/$v/main $default_log", + 'pg_lscluster reports online cluster on port 5432'; + + # verify that the log file is actually used + ok !-z $default_log, 'log file is actually used'; + + # verify configuration file permissions + my $postgres_uid = (getpwnam 'postgres')[2]; + my @st = stat "/etc/postgresql"; + is $st[4], $postgres_uid, '/etc/postgresql is owned by user "postgres"'; + my @st = stat "/etc/postgresql/$v"; + is $st[4], $postgres_uid, 'version configuration directory file is owned by user "postgres"'; + my @st = stat "/etc/postgresql/$v/main"; + is $st[4], $postgres_uid, 'configuration directory file is owned by user "postgres"'; + + # verify data file permissions + my @st = stat "/var/lib/postgresql/$v"; + is $st[4], $postgres_uid, 'version data directory file is owned by user "postgres"'; + my @st = stat "/var/lib/postgresql/$v/main"; + is $st[4], $postgres_uid, 'data directory file is owned by user "postgres"'; + + # verify log file permissions + my @logstat = stat $default_log; + is $logstat[2], 0100640, 'log file has 0640 permissions'; + is $logstat[4], $postgres_uid, 'log file is owned by user "postgres"'; + is $logstat[5], (getgrnam 'adm')[2], 'log file is owned by group "adm"'; + + # check default log file configuration; when not specifying -l with + # pg_createcluster, we should not have a 'log' symlink + ok !-e "/etc/postgresql/$v/main/log", 'no log symlink by default'; + ok !-z $default_log, "$default_log is the default log if log symlink is missing"; + like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/^$v\s+main.*$default_log\n$/; + + # verify that log symlink works + is ((exec_as 'root', "pg_ctlcluster $v main stop"), 0, 'stopping cluster'); + usleep $delay; + truncate "$default_log", 0; # empty log file + my $p = (PgCommon::cluster_data_directory $v, 'main') . '/mylog'; + symlink $p, "/etc/postgresql/$v/main/log"; + is ((exec_as 'root', "pg_ctlcluster $v main start"), 0, + 'restarting cluster with nondefault log symlink'); + ok !-z $p, "log target is used as log file"; + ok -z $default_log, "default log is not used"; + like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/^$v\s+main.*$p\n$/; + is ((exec_as 'root', "pg_ctlcluster $v main stop"), 0, 'stopping cluster'); + usleep $delay; + truncate "$default_log", 0; # empty log file + + # verify that explicitly configured log file trumps log symlink + PgCommon::set_conf_value ($v, 'main', 'postgresql.conf', + ($v >= '8.3' ? 'logging_collector' : 'redirect_stderr'), 'on'); + PgCommon::set_conf_value $v, 'main', 'postgresql.conf', 'log_filename', "$v#main.log"; + is ((exec_as 'root', "pg_ctlcluster $v main start"), 0, + 'restarting cluster with explicitly configured log file'); + ok -z $default_log, "default log is not used"; + ok !-z $p, "log symlink target is used for startup message"; + my $pg_log = $v >= 10 ? 'log' : 'pg_log'; # log directory in PGDATA changed in PG 10 + my @l = glob ((PgCommon::cluster_data_directory $v, 'main') . "/$pg_log/$v#main.log*"); + is $#l, 0, 'exactly one log file'; + ok (-e $l[0] && ! -z $l[0], 'custom log is actually used'); + SKIP: { skip "no logging_collector in $v", 2 if ($v < 8.3); + like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/^$v\s+main.*$v#main.log\n$/; + } + + # clean up + PgCommon::disable_conf_value ($v, 'main', 'postgresql.conf', + ($v >= '8.3' ? 'logging_collector' : 'redirect_stderr'), ''); + PgCommon::disable_conf_value $v, 'main', 'postgresql.conf', 'log_filename', ''; + unlink "/etc/postgresql/$v/main/log"; + + # check that log creation does not escalate privileges + program_ok 'root', "pg_ctlcluster $v main stop", 0, 'stopping cluster'; + unlink $default_log; + symlink "/etc/postgres-hack", $default_log; + program_ok 'root', "pg_ctlcluster $v main start", 1, 'starting cluster with rouge /var/log/postgresql symlink fails'; + ok !-f "/etc/postgres-hack", "/etc/postgres-hack was not created"; + unlink $default_log; + program_ok 'root', "pg_ctlcluster $v main start", 0, 'restarting cluster'; + + # verify that processes do not have an associated terminal + unlike_program_out 0, 'ps -o tty -U postgres h', 0, qr/tty|pts/, + 'postgres processes do not have an associated terminal'; + + # verify that SSL is enabled (which should work for user postgres in a + # default installation) + my $ssl = config_bool (PgCommon::get_conf_value $v, 'main', 'postgresql.conf', 'ssl'); + my $ssl_linked = `ldd $PgCommon::binroot$v/bin/postgres | grep libssl`; + my ($os, $osversion) = os_release(); + if ($PgCommon::rpm) { + isnt $ssl_linked, '', 'Server is linked with SSL support'; + is $ssl, undef, 'SSL is disabled in postgresql.conf'; + } elsif ($v <= 9.1 and (($os eq 'debian' and ($osversion eq 'unstable' or $osversion > 9)) or # stretch had 1.0 and 1.1 + ($os eq 'ubuntu' and $osversion > 18.04))) { # bionic had 1.0 and 1.1 + is $ssl_linked, '', 'Server is linked without SSL support (old version with only OpenSSL 1.0 support)'; + is $ssl, undef, 'SSL is disabled in postgresql.conf'; + } else { + isnt $ssl_linked, '', 'Server is linked with SSL support'; + is $ssl, 1, 'SSL is enabled in postgresql.conf'; + } + + # Create user nobody, a database 'nobodydb' for him, check the database list + my $outref; + is ((exec_as 'nobody', 'psql -l 2>/dev/null', $outref), 2, 'psql -l fails for nobody'); + is ((exec_as 'postgres', 'createuser nobody -D -R -S'), 0, 'createuser nobody'); + is ((exec_as 'postgres', 'createdb -O nobody nobodydb'), 0, 'createdb nobodydb'); + is ((exec_as 'nobody', 'psql -ltA|grep "|" | cut -f1-3 -d"|"', $outref), 0, 'psql -ltA succeeds for nobody'); + is ($$outref, 'nobodydb|nobody|UTF8 +postgres|postgres|UTF8 +template0|postgres|UTF8 +template1|postgres|UTF8 +', 'psql -ltA output'); + + # Then fill nobodydb with some data. + is ((exec_as 'nobody', 'psql nobodydb -c "create table phone (name varchar(255) PRIMARY KEY, tel int NOT NULL)" 2>/dev/null'), + 0, 'SQL command: create table'); + is ((exec_as 'nobody', 'psql nobodydb -c "insert into phone values (\'Bob\', 1)"'), 0, 'SQL command: insert into table values'); + is ((exec_as 'nobody', 'psql nobodydb -c "insert into phone values (\'Alice\', 2)"'), 0, 'SQL command: insert into table values'); + is ((exec_as 'nobody', 'psql nobodydb -c "insert into phone values (\'Bob\', 3)"'), 1, 'primary key violation'); + + # Check table contents + is_program_out 'nobody', 'psql -tAc "select * from phone order by name" nobodydb', 0, + 'Alice|2 +Bob|1 +', 'SQL command output: select -tA'; + is_program_out 'nobody', 'psql -txc "select * from phone where name = \'Alice\'" nobodydb', 0, + 'name | Alice +tel | 2 + +', 'SQL command output: select -tx'; + is_program_out 'nobody', 'psql -tAxc "select * from phone where name = \'Alice\'" nobodydb', 0, + 'name|Alice +tel|2 +', 'SQL command output: select -tAx'; + + sub create_extension ($$) { + my ($v, $extension) = @_; + return "psql -qc 'CREATE EXTENSION $extension' nobodydb" if ($v >= 9.1); + return "createlang --cluster $v/main $extension nobodydb"; + } + + # Check PL/Perl untrusted + my $fn_cmd = 'CREATE FUNCTION read_file() RETURNS text AS \'open F, \\"/etc/passwd\\"; \\$buf = ; close F; return \\$buf;\' LANGUAGE plperl'; + is ((exec_as 'nobody', create_extension($v, 'plperlu')), 1, 'CREATE EXTENSION plperlu fails for user nobody'); + is_program_out 'postgres', create_extension($v, 'plperlu'), 0, '', 'CREATE EXTENSION plperlu succeeds for user postgres'; + is ((exec_as 'nobody', "psql nobodydb -qc \"${fn_cmd}u;\""), 1, 'creating PL/PerlU function as user nobody fails'); + is ((exec_as 'postgres', "psql nobodydb -qc \"${fn_cmd};\""), 1, 'creating unsafe PL/Perl function as user postgres fails'); + is_program_out 'postgres', "psql nobodydb -qc \"${fn_cmd}u;\"", 0, '', 'creating PL/PerlU function as user postgres succeeds'; + like_program_out 'nobody', 'psql nobodydb -Atc "select read_file()"', + 0, qr/^root:/, 'calling PL/PerlU function'; + + # Check PL/Perl trusted + my $pluser = ($v >= '8.3') ? 'nobody' : 'postgres'; # pg_pltemplate allows non-superusers to install trusted languages in 8.3+ + is_program_out $pluser, create_extension($v, 'plperl'), 0, '', "CREATE EXTENSION plperl succeeds for user $pluser"; + is ((exec_as 'nobody', "psql nobodydb -qc \"${fn_cmd};\""), 1, 'creating unsafe PL/Perl function as user nobody fails'); + is_program_out 'nobody', 'psql nobodydb -qc "CREATE FUNCTION remove_vowels(text) RETURNS text AS \'\\$_[0] =~ s/[aeiou]/_/ig; return \\$_[0];\' LANGUAGE plperl;"', + 0, '', 'creating PL/Perl function as user nobody succeeds'; + is_program_out 'nobody', 'psql nobodydb -Atc "select remove_vowels(\'foobArish\')"', + 0, "f__b_r_sh\n", 'calling PL/Perl function'; + + # Check PL/Python (untrusted) + SKIP: { + skip "No python2 support", 6 unless ($v <= 11 and $PgCommon::have_python2); + is_program_out 'postgres', create_extension($v, 'plpythonu'), 0, '', 'CREATE EXTENSION plpythonu succeeds for user postgres'; + is_program_out 'postgres', 'psql nobodydb -qc "CREATE FUNCTION capitalize(text) RETURNS text AS \'import sys; return args[0].capitalize() + sys.version[0]\' LANGUAGE plpythonu;"', + 0, '', 'creating PL/Python function as user postgres succeeds'; + is_program_out 'nobody', 'psql nobodydb -Atc "select capitalize(\'foo\')"', + 0, "Foo2\n", 'calling PL/Python function'; + } + + # Check PL/Python3 (untrusted) + if ($v >= '9.1') { + is_program_out 'postgres', create_extension($v, 'plpython3u'), 0, '', 'CREATE EXTENSION plpython3u succeeds for user postgres'; + is_program_out 'postgres', 'psql nobodydb -qc "CREATE FUNCTION capitalize3(text) RETURNS text AS \'import sys; return args[0].capitalize() + sys.version[0]\' LANGUAGE plpython3u;"', + 0, '', 'creating PL/Python3 function as user postgres succeeds'; + is_program_out 'nobody', 'psql nobodydb -Atc "select capitalize3(\'foo\')"', + 0, "Foo3\n", 'calling PL/Python function'; + } else { + pass "Skipping PL/Python3 test for version $v..."; + pass '...'; + pass '...'; + pass '...'; + pass '...'; + pass '...'; + } + + # Check PL/Tcl (trusted/untrusted) + is_program_out 'postgres', create_extension($v, 'pltcl'), 0, '', 'CREATE EXTENSION pltcl succeeds for user postgres'; + is_program_out 'postgres', create_extension($v, 'pltclu'), 0, '', 'CREATE EXTENSION pltclu succeeds for user postgres'; + is_program_out 'nobody', 'psql nobodydb -qc "CREATE FUNCTION tcl_max(integer, integer) RETURNS integer AS \'if {\\$1 > \\$2} {return \\$1}; return \\$2\' LANGUAGE pltcl STRICT;"', + 0, '', 'creating PL/Tcl function as user nobody succeeds'; + is_program_out 'postgres', 'psql nobodydb -qc "CREATE FUNCTION tcl_max_u(integer, integer) RETURNS integer AS \'if {\\$1 > \\$2} {return \\$1}; return \\$2\' LANGUAGE pltclu STRICT;"', + 0, '', 'creating PL/TclU function as user postgres succeeds'; + is_program_out 'nobody', 'psql nobodydb -Atc "select tcl_max(3,4)"', 0, + "4\n", 'calling PL/Tcl function'; + is_program_out 'nobody', 'psql nobodydb -Atc "select tcl_max_u(5,4)"', 0, + "5\n", 'calling PL/TclU function'; + + # fake rotated logs to check that they are cleaned up properly + open L, ">$default_log.1" or die "could not open fake rotated log file"; + print L "old log .1\n"; + close L; + open L, ">$default_log.2" or die "could not open fake rotated log file"; + print L "old log .2\n"; + close L; + if (system "gzip -9 $default_log.2") { + die "could not gzip fake rotated log"; + } + + # Check that old-style pgdata symbolic link still works (p-common 0.90+ + # does not create them any more, but they still need to work for existing + # installations) + is ((exec_as 'root', "pg_ctlcluster $v main stop"), 0, 'stopping cluster'); + my $datadir = PgCommon::get_conf_value $v, 'main', 'postgresql.conf', 'data_directory'; + symlink $datadir, "/etc/postgresql/$v/main/pgdata"; + + # data_directory should trump the pgdata symlink + PgCommon::set_conf_value $v, 'main', 'postgresql.conf', 'data_directory', '/nonexisting'; + like_program_out 0, "pg_ctlcluster $v main start", 1, + qr/\/nonexisting is not accessible/, + 'cluster fails to start with invalid data_directory and valid pgdata symlink'; + + # if only pgdata symlink is present, it is authoritative + PgCommon::disable_conf_value $v, 'main', 'postgresql.conf', 'data_directory', 'disabled for test'; + is_program_out 0, "pg_ctlcluster $v main start", 0, '', + 'cluster restarts with pgdata symlink'; + + # check properties of backend processes + pipe RH, WH; + my $psql = fork; + if (!$psql) { + close WH; + my @pw = getpwnam 'nobody'; + change_ugid $pw[2], $pw[3]; + open(STDIN, "<& RH"); + dup2(POSIX::open('/dev/null', POSIX::O_WRONLY), 1); + exec 'psql', '-Xq', '-vPROMPT1=', 'nobodydb' or die "could not exec psql process: $!"; + } + close RH; + select WH; $| = 1; # make unbuffered + + open my $pidfile, "/var/lib/postgresql/$v/main/postmaster.pid"; + my $master_pid = <$pidfile>; + chomp $master_pid; + close $pidfile; + + my $client_pid; + while (!$client_pid) { + usleep $delay; + $client_pid = `ps --user postgres hu | grep 'postgres.*: nobody nobodydb' | grep -v grep | awk '{print \$2}'`; + ($client_pid) = ($client_pid =~ /(\d+)/); # untaint + } + + # OOM score adjustment under Linux: postmaster gets bigger shields for >= + # 9.0, but client backends stay at default; this might not work in + # containers with restricted privileges, so skip the check there + my $adj; + my $detect_virt = system 'systemd-detect-virt --container --quiet'; # from systemd + open F, "/proc/$master_pid/oom_score_adj"; + $adj = ; + chomp $adj; + close F; + if ($v >= '9.0' and not $PgCommon::rpm) { + SKIP: { + skip 'skipping postmaster OOM killer adjustment in container', 1 if $detect_virt == 0; + cmp_ok $adj, '<=', -500, 'postgres master has OOM killer protection'; + } + } else { + is $adj, 0, 'postgres master has no OOM adjustment'; + } + + open F, "/proc/$client_pid/oom_score_adj"; + $adj = ; + chomp $adj; + close F; + is $adj, 0, 'postgres client backend has no OOM adjustment'; + + # test process title update + like_program_out 0, "ps h $client_pid", 0, qr/ idle\s*$/, 'process title is idle'; + print WH "BEGIN;\n"; + usleep $delay; + like_program_out 0, "ps h $client_pid", 0, qr/idle in transaction/, 'process title is idle in transaction'; + + close WH; + kill 15, $psql; + waitpid $psql, 0; + + # log file gets re-created by pg_ctlcluster + is ((exec_as 0, "pg_ctlcluster $v main stop"), 0, 'stopping cluster'); + unlink $default_log; + is ((exec_as 0, "pg_ctlcluster $v main start"), 0, 'starting cluster as postgres works without a log file'); + ok (-e $default_log && ! -z $default_log, 'log file got recreated and used'); + + # create tablespaces + my $spc1 = tempdir("/tmp/$v.spc1.XXXXXX", CLEANUP => 1); + my $spc2 = tempdir("/tmp/$v.spc2.XXXXXX", CLEANUP => 1); + is (mkdir ("$spc2/PG_99_fakedirectory"), 1, 'creating a directory in spc2'); + chown $postgres_uid, 0, $spc1, $spc2, "$spc2/PG_99_fakedirectory"; + is_program_out 'postgres', "psql -qc \"CREATE TABLESPACE spc1 LOCATION '$spc1'\"", 0, '', 'creating tablespace spc1'; + is_program_out 'postgres', "psql -qc 'CREATE TABLE tbl1 (x int) TABLESPACE spc1'", 0, '', 'creating a table in spc1'; + SKIP: { + skip "Non-empty tablespaces not supported before 9.0", 4 if ($v < 9.0); + is_program_out 'postgres', "psql -qc \"CREATE TABLESPACE spc2 LOCATION '$spc2'\"", 0, '', 'creating tablespace spc2'; + is_program_out 'postgres', "psql -qc 'CREATE TABLE tbl2 (x int) TABLESPACE spc2'", 0, '', 'creating a table in spc2'; + } + + # check apt config + is_program_out 0, "grep -Eo 'postgresql.[0-9.*-]+' /etc/apt/apt.conf.d/02autoremove-postgresql", 0, + "postgresql.*-$v\n", "Correct apt NeverAutoRemove config"; + + # stop server, clean up, check for leftovers + ok ((system "pg_dropcluster $v main --stop") == 0, + 'pg_dropcluster removes cluster'); + + is (-e $xlogdir, undef, "xlog/wal directory $xlogdir was deleted"); + ok_dir $spc1, [], "tablespace spc1 was emptied"; + ok_dir $spc2, [qw(PG_99_fakedirectory)], "tablespace spc2 was emptied"; + + is_program_out 0, "grep -Eo 'postgresql.[0-9.*-]+' /etc/apt/apt.conf.d/02autoremove-postgresql", 1, + "", "Correct apt NeverAutoRemove config"; + + check_clean; +} + +foreach (@MAJORS) { + check_major $_; +} + +# vim: filetype=perl diff --git a/t/021_pg_renamecluster.t b/t/021_pg_renamecluster.t new file mode 100644 index 0000000..b3e1cb6 --- /dev/null +++ b/t/021_pg_renamecluster.t @@ -0,0 +1,43 @@ +use strict; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More tests => 20; + +my $v = $MAJORS[-1]; + +# create cluster +ok ((system "pg_createcluster $v main --start >/dev/null") == 0, + "pg_createcluster $v main"); + +# test pg_renamecluster with a running cluster +program_ok (0, "pg_renamecluster $v main donau"); +is_program_out 'postgres', 'psql -tAc "show data_directory"', 0, + "/var/lib/postgresql/$v/donau\n", 'cluster is running and data_directory was moved'; +is ((PgCommon::get_conf_value $v, 'donau', 'postgresql.conf', 'hba_file'), + "/etc/postgresql/$v/donau/pg_hba.conf", 'pg_hba.conf location updated'); +is ((PgCommon::get_conf_value $v, 'donau', 'postgresql.conf', 'ident_file'), + "/etc/postgresql/$v/donau/pg_ident.conf", 'pg_ident.conf location updated'); +is ((PgCommon::get_conf_value $v, 'donau', 'postgresql.conf', 'external_pid_file'), + "/var/run/postgresql/$v-donau.pid", 'external_pid_file location updated'); +ok (-f "/var/run/postgresql/$v-donau.pid", 'external_pid_file exists'); +SKIP: { + skip "no stats_temp_directory in $v", 2 if ($v < 8.4 or $v >= 15); + is ((PgCommon::get_conf_value $v, 'donau', 'postgresql.conf', 'stats_temp_directory'), + "/var/run/postgresql/$v-donau.pg_stat_tmp", 'stats_temp_directory location updated'); + ok (-d "/var/run/postgresql/$v-donau.pg_stat_tmp", 'stats_temp_directory exists'); +} +SKIP: { + skip "cluster name not supported in $v", 1 if ($v < 9.5); + is (PgCommon::get_conf_value ($v, 'donau', 'postgresql.conf', 'cluster_name'), "$v/donau", "cluster_name is updated"); +} + +# stop server, clean up, check for leftovers +ok ((system "pg_dropcluster $v donau --stop") == 0, + 'pg_dropcluster removes cluster'); + +check_clean; + +# vim: filetype=perl diff --git a/t/022_recovery.t b/t/022_recovery.t new file mode 100644 index 0000000..b70a8b5 --- /dev/null +++ b/t/022_recovery.t @@ -0,0 +1,54 @@ +# We create a cluster, stop it ungracefully, and check if recovery works. + +use strict; + +use POSIX qw/dup2/; +use Time::HiRes qw/usleep/; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More tests => 17 * @MAJORS; + +sub check_major { + my $v = $_[0]; + note "Running tests for $v"; + + # create cluster + program_ok (0, "pg_createcluster $v main --start >/dev/null"); + + # try an immediate shutdown and restart + program_ok (0, "pg_ctlcluster $v main stop -m i"); + program_ok (0, "pg_ctlcluster $v main start"); + my $c = 0; # fallback for when pg_isready is missing (PG < 9.3) + while (system ("pg_isready -q 2>&1") >> 8 == 1 and $c++ < 15) { + sleep(1); + } + program_ok ('postgres', "psql -c ''"); + + # try again with an write-protected file + program_ok (0, "pg_ctlcluster $v main stop -m i"); + open F, ">/var/lib/postgresql/$v/main/foo"; + print F "moo\n"; + close F; + ok ((chmod 0444, "/var/lib/postgresql/$v/main/foo"), + "create write-protected file in data directory"); + program_ok (0, "pg_ctlcluster $v main start"); + $c = 0; + while (system ("pg_isready -q 2>&1") >> 8 == 1 and $c++ < 15) { + sleep(1); + } + program_ok ('postgres', "psql -c ''"); + + program_ok (0, "pg_dropcluster $v main --stop", 0, + 'pg_dropcluster removes cluster'); + + check_clean; +} + +foreach (@MAJORS) { + check_major $_; +} + +# vim: filetype=perl diff --git a/t/025_logging.t b/t/025_logging.t new file mode 100644 index 0000000..a7e5c50 --- /dev/null +++ b/t/025_logging.t @@ -0,0 +1,99 @@ +# Test various logging-related things + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Time::HiRes qw/usleep/; + +use Test::More tests => 55 * @MAJORS; + +my $syslog_works = 0; + +sub check_logging ($$) +{ + my ($text, $msg) = @_; + my $ls = `pg_lsclusters -h`; + $ls =~ s/\s+/ /g; + $ls =~ s/\s*$//; + like $ls, $text, $msg; +} + +sub check_major { + my $v = $_[0]; + note "Running tests for $v"; + my $pgdata = "/var/lib/postgresql/$v/main"; + + # create cluster + ok ((system "pg_createcluster $v main --start >/dev/null") == 0, + "pg_createcluster $v main"); + + # default log setup + my $default_log = "/var/log/postgresql/postgresql-$v-main.log"; + check_logging qr($v main 5432 online postgres $pgdata $default_log), "pg_lscluster reports logfile $default_log"; + like_program_out 'postgres', "psql -qc \"'foobar_${v}_$$'\"", 1, qr/syntax error.*foobar_${v}_$$/, 'log an error'; + usleep $delay; + like_program_out 'postgres', "grep --binary-files=text foobar_${v}_$$ $default_log", 0, qr/syntax error.*foobar_${v}_$$/, 'error appears in logfile'; + + # syslog + is_program_out 0, "pg_conftool $v main set log_destination syslog", 0, "", "set log_destination syslog"; + is_program_out 0, "pg_ctlcluster $v main reload", 0, "", "$v main reload"; + is_program_out 'postgres', "psql -Atc \"show log_destination\"", 0, "syslog\n", 'log_destination is syslog'; + check_logging qr($v main 5432 online postgres $pgdata syslog), "pg_lscluster reports syslog"; + SKIP: { + skip "/var/log/syslog not available", 2 unless ($syslog_works); + usleep $delay; + like_program_out 0, "grep --binary-files=text 'postgres.*parameter \"log_destination\" changed to \"syslog\"' /var/log/syslog", 0, qr/log_destination/, 'error appears in /var/log/syslog'; + } + + # turn logging_collector on, csvlog + my $pg_log = $v >= 10 ? 'log' : 'pg_log'; # log directory in PGDATA changed in PG 10 + SKIP: { + skip "No logging collector in 8.2", 30 if ($v <= 8.2); + is_program_out 0, "pg_conftool $v main set logging_collector on", 0, "", "set logging_collector on"; + is_program_out 0, "pg_conftool $v main set log_destination csvlog", 0, "", "set log_destination csvlog"; + is_program_out 0, "pg_ctlcluster $v main restart", 0, "", "$v main restart"; + is_program_out 'postgres', "psql -Atc \"show logging_collector\"", 0, "on\n", 'logging_collector is on'; + is_program_out 'postgres', "psql -Atc \"show log_destination\"", 0, "csvlog\n", 'log_destination is csvlog'; + check_logging qr($v main 5432 online postgres $pgdata $pg_log/.*\.csv), "pg_lscluster reports csvlog"; + like_program_out 'postgres', "psql -qc \"'barbaz_${v}_$$'\"", 1, qr/syntax error.*barbaz_${v}_$$/, 'log an error'; + usleep $delay; + like_program_out 'postgres', "grep --binary-files=text barbaz_${v}_$$ $pgdata/$pg_log/*.csv", 0, qr/syntax error.*barbaz_${v}_$$/, "error appears in $pg_log/*.csv"; + + # stderr,syslog,csvlog + is_program_out 0, "pg_conftool $v main set log_destination stderr,syslog,csvlog", 0, "", "set log_destination stderr,syslog,csvlog"; + is_program_out 0, "pg_ctlcluster $v main reload", 0, "", "$v main reload"; + is_program_out 'postgres', "psql -Atc \"show log_destination\"", 0, "stderr,syslog,csvlog\n", 'log_destination is stderr,syslog,csvlog'; + check_logging qr($v main 5432 online postgres $pgdata $pg_log/.*\.log,syslog,$pg_log/.*\.csv), "pg_lscluster reports stderr,syslog,csvlog"; + like_program_out 'postgres', "psql -qc \"'moo_${v}_$$'\"", 1, qr/syntax error.*moo_${v}_$$/, 'log an error'; + usleep $delay; + like_program_out 'postgres', "grep --binary-files=text moo_${v}_$$ $pgdata/$pg_log/*.log", 0, qr/syntax error.*moo_${v}_$$/, "error appears in $pg_log/*.log"; + SKIP: { + skip "/var/log/syslog not available", 2 unless ($syslog_works); + usleep $delay; + like_program_out 0, "grep --binary-files=text 'postgres.*moo_${v}_$$' /var/log/syslog", 0, qr/moo_${v}_$$/, 'error appears in /var/log/syslog'; + } + like_program_out 'postgres', "grep --binary-files=text moo_${v}_$$ $pgdata/$pg_log/*.csv", 0, qr/syntax error.*moo_${v}_$$/, "error appears in $pg_log/*.csv"; + } + + # stop server, clean up, check for leftovers + is_program_out 0, "pg_dropcluster $v main --stop", 0, "", 'pg_dropcluster removes cluster'; + + check_clean; +} + +system "logger -t '$0' 'test-logging-$$'"; +usleep $delay; +if (system ("grep --binary-files=text -q 'test-logging-$$' /var/log/syslog 2> /dev/null") == 0) { + note 'Logging to /var/log/syslog works'; + $syslog_works = 1; +} else { + note 'Logging to /var/log/syslog does not work, skipping some syslog tests'; +} + +foreach (@MAJORS) { + check_major $_; +} + +# vim: filetype=perl diff --git a/t/030_errors.t b/t/030_errors.t new file mode 100644 index 0000000..88d3700 --- /dev/null +++ b/t/030_errors.t @@ -0,0 +1,336 @@ +# Check all kinds of error conditions. + +use strict; + +require File::Temp; + +use lib 't'; +use TestLib; +use Test::More tests => 156; +use PgCommon; + +my $version = $MAJORS[-1]; + +my $socketdir = '/tmp/postgresql-testsuite/'; +my ($pg_uid, $pg_gid) = (getpwnam 'postgres')[2,3]; + +# create a pid file with content $1 and return its path +sub create_pidfile { + my $fname = "/var/lib/postgresql/$version/main/postmaster.pid"; + open F, ">$fname" or die "open: $!"; + print F $_[0]; + close F; + chown $pg_uid, $pg_gid, $fname or die "chown: $!"; + chmod 0700, $fname or die "chmod: $!"; + return $fname; +} + +sub check_nonexisting_cluster_error { + my $outref; + my $result = exec_as 0, $_[0], $outref; + is $result, 1, "'$_[0]' fails"; + like $$outref, qr/(invalid version|does not exist)/i, "$_[0] gives error message about nonexisting cluster"; + unlike $$outref, qr/invalid symbolic link/i, "$_[0] does not print 'invalid symbolic link' gibberish"; +} + +# check if pg_lsclusters shows a cluster without binaries +mkdir "/etc/postgresql/6.3"; +mkdir "/etc/postgresql/6.3/main"; +open F, ">/etc/postgresql/6.3/main/postgresql.conf"; +close F; +is `pg_lsclusters -h`, "6.3 main down,binaries_missing \n", + 'pg_lscluster reports cluster without binaries'; +program_ok 0, "pg_dropcluster 6.3 main"; + +# create cluster +ok ((system "pg_createcluster --socketdir '$socketdir' $version main >/dev/null") == 0, + "pg_createcluster --socketdir"); +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/$version\s*main.*5432.*down/, 'cluster was created'; + +is ((get_cluster_port $version, 'main'), 5432, 'Port of created cluster is 5432'); + +# creating cluster with the same name should fail +like_program_out 'root', "pg_createcluster --socketdir '$socketdir' $version main", 1, qr/already exists/, + "pg_createcluster on existing cluster"; +# and the original one still exists +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/$version\s*main.*5432.*down/, 'original cluster still exists'; + +# attempt to create clusters with an invalid port +like_program_out 0, "pg_createcluster $version test -p foo", 1, + qr/invalid.*number expected/, + 'pg_createcluster -p checks that port option is numeric'; +like_program_out 0, "pg_createcluster $version test -p 42", 1, + qr/must be a positive integer between/, + 'pg_createcluster -p checks valid port range'; + +# chown cluster to an invalid user to test error +(system "chown -R 0 /var/lib/postgresql/$version/main") == 0 or die "chown failed: $!"; +like_program_out 0, "pg_ctlcluster $version main start", 1, qr/must not be owned by root/, + "pg_ctlcluster refuses to start root-owned cluster"; +my $badid = 98; +(system "chown -R $badid /var/lib/postgresql/$version/main") == 0 or die "chown failed: $!"; +like_program_out 0, "pg_ctlcluster $version main start", 1, qr/owned by user id 98 which does not exist/, + 'pg_ctlcluster fails on invalid cluster owner uid'; +(system "chown -R postgres:$badid /var/lib/postgresql/$version/main") == 0 or die "chown failed: $!"; +like_program_out 0, "pg_ctlcluster $version main start", 1, qr/owned by group id 98 which does not exist/, + 'pg_ctlcluster as root fails on invalid cluster owner gid'; +like_program_out 'postgres', "pg_ctlcluster $version main start", 1, qr/owned by group id 98 which does not exist/, + 'pg_ctlcluster as postgres fails on invalid cluster owner gid'; +(system "chown -R postgres:postgres /var/lib/postgresql/$version/main") == 0 or die "chown failed: $!"; +program_ok 0, "pg_ctlcluster $version main start", 0, + 'pg_ctlcluster succeeds on valid cluster owner uid/gid'; + +# check socket +my @contents = ('.s.PGSQL.5432', '.s.PGSQL.5432.lock', "$version-main.pid", "$version-main.pg_stat_tmp"); +pop @contents if ($version < 8.4 or $version >= 15); # remove pg_stat_tmp +ok_dir '/var/run/postgresql', [grep {/main/} @contents], 'No sockets in /var/run/postgresql'; +ok_dir $socketdir, ['.s.PGSQL.5432', '.s.PGSQL.5432.lock'], "Socket is in $socketdir"; + +# stop cluster, check sockets +ok ((system "pg_ctlcluster $version main stop") == 0, + 'cluster stops with custom unix_socket_dir'); +ok_dir $socketdir, [], "No sockets in $socketdir after stopping cluster"; + +# remove default socket dir and check that the socket defaults to +# /var/run/postgresql +open F, "+; +seek F, 0, 0 or die "seek: $!"; +truncate F, 0; +@lines = grep !/^unix_socket_dir/, @lines; # <= 9.2: "_directory", >= 9.3: "_directories" +print F @lines; +close F; + +ok ((system "pg_ctlcluster $version main start") == 0, + 'cluster starts after removing unix_socket_dir'); +if ($PgCommon::rpm) { + ok ((grep { $_ eq '.s.PGSQL.5432' } @{TestLib::dircontent('/tmp')}) == 1, 'Socket is in /tmp'); +} else { + ok_dir '/var/run/postgresql', [@contents], + 'Socket is in default dir /var/run/postgresql'; +} +ok_dir $socketdir, [], "No sockets in $socketdir"; + +# server should not stop with corrupt file +rename "/var/lib/postgresql/$version/main/postmaster.pid", + "/var/lib/postgresql/$version/main/postmaster.pid.orig" or die "rename: $!"; +create_pidfile 'foo'; +is_program_out 'postgres', "pg_ctlcluster $version main stop", 1, + "Error: pid file is invalid, please manually kill the stale server process.\n", + 'pg_ctlcluster fails with corrupted PID file'; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/online/, 'cluster is still online'; + +# restore PID file +(system "cp /var/lib/postgresql/$version/main/postmaster.pid.orig /var/lib/postgresql/$version/main/postmaster.pid") == 0 or die "cp: $!"; +is ((exec_as 'postgres', "pg_ctlcluster $version main stop"), 0, + 'pg_ctlcluster succeeds with restored PID file'); +mkdir $PgCommon::binroot . "foo"; # #940220: infinite recursion in get_program_path +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/down/, 'cluster is down'; +rmdir $PgCommon::binroot . "foo"; + +# stop stopped server +is_program_out 'postgres', "pg_ctlcluster $version main stop", 2, + "Cluster is not running.\n", 'pg_ctlcluster stop fails on stopped cluster'; + +# simulate crashed server +rename "/var/lib/postgresql/$version/main/postmaster.pid.orig", + "/var/lib/postgresql/$version/main/postmaster.pid" or die "rename: $!"; +is_program_out 'postgres', "pg_ctlcluster $version main start", 0, + "Removed stale pid file.\n", 'pg_ctlcluster succeeds with already existing PID file'; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/online/, 'cluster is online'; +is ((exec_as 'postgres', "pg_ctlcluster $version main stop"), 0, + 'pg_ctlcluster stop succeeds'); +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/down/, 'cluster is down'; +ok (! -e "/var/lib/postgresql/$version/main/postmaster.pid", 'no pid file left'); + +# trying to stop a stopped server cleans up corrupt and stale pid files +my $pf = create_pidfile 'foo'; +is_program_out 'postgres', "pg_ctlcluster $version main stop", 2, + "Removed stale pid file.\nCluster is not running.\n", + 'pg_ctlcluster stop succeeds with corrupted PID file'; +ok (! -e $pf, 'pid file was cleaned up'); + +create_pidfile 'foo'; +is_program_out 'postgres', "pg_ctlcluster --force $version main stop", 2, + "Removed stale pid file.\nCluster is not running.\n", + 'pg_ctlcluster --force stop succeeds with corrupted PID file'; +ok (! -e $pf, 'pid file was cleaned up'); + +create_pidfile '99998'; +is_program_out 'postgres', "pg_ctlcluster $version main stop", 2, + "Removed stale pid file.\nCluster is not running.\n", + 'pg_ctlcluster stop succeeds with stale PID file'; +ok (! -e $pf, 'pid file was cleaned up'); + +create_pidfile '99998'; +is_program_out 'postgres', "pg_ctlcluster --force $version main stop", 2, + "Removed stale pid file.\nCluster is not running.\n", + 'pg_ctlcluster --force stop succeeds with stale PID file'; +ok (! -e $pf, 'pid file was cleaned up'); + +create_pidfile ''; +is_program_out 'postgres', "pg_ctlcluster --force $version main stop", 2, + "Removed stale pid file.\nCluster is not running.\n", + 'pg_ctlcluster stop succeeds with empty PID file'; +ok (! -e $pf, 'pid file was cleaned up'); + +# corrupt PID file while server is down +create_pidfile 'foo'; +is_program_out 'postgres', "pg_ctlcluster $version main start", 0, + "Removed stale pid file.\n", 'pg_ctlcluster succeeds with corrupted PID file'; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/online/, 'cluster is online'; + +# start running server +is_program_out 'postgres', "pg_ctlcluster $version main start", 2, + "Cluster is already running.\n", 'pg_ctlcluster start fails on running cluster'; +is ((exec_as 'postgres', "pg_ctlcluster $version main stop"), 0, 'pg_ctlcluster stop'); + +# backup pg_hba.conf +rename "/etc/postgresql/$version/main/pg_hba.conf", + "/etc/postgresql/$version/main/pg_hba.conf.orig" or die "rename: $!"; + +# test check for invalid pg_hba.conf +open F, ">/etc/postgresql/$version/main/pg_hba.conf" or die "could not create pg_hba.conf: $!"; +print F "foo\n"; +close F; +chmod 0644, "/etc/postgresql/$version/main/pg_hba.conf" or die "chmod: $!"; + +if ($version < '8.4') { + like_program_out 'postgres', "pg_ctlcluster $version main start", 0, + qr/WARNING.*connection to the database failed.*pg_hba.conf/is, + 'pg_ctlcluster start warns about invalid pg_hba.conf'; + is_program_out 'postgres', "pg_ctlcluster $version main stop", 0, '', 'stopping cluster'; +} else { + like_program_out 'postgres', "pg_ctlcluster $version main start", 1, + qr/FATAL.*pg_hba.conf/is, + 'pg_ctlcluster start fails on invalid pg_hba.conf'; + is_program_out 'postgres', "pg_ctlcluster $version main stop", 2, + "Cluster is not running.\n", 'stopping cluster'; +} + +# test check for pg_hba.conf with removed passwordless local superuser access +open F, ">/etc/postgresql/$version/main/pg_hba.conf" or die "could not create pg_hba.conf: $!"; +print F "local all all md5\n"; +close F; +chmod 0644, "/etc/postgresql/$version/main/pg_hba.conf" or die "chmod: $!"; + +like_program_out 'postgres', "pg_ctlcluster $version main start", 0, + qr/Warning.*connection to the database failed.*(no password supplied|password authentication failed)/is, + 'pg_ctlcluster start warns about absence of passwordless superuser connection'; +is_program_out 'postgres', "pg_ctlcluster $version main stop", 0, '', 'stopping cluster'; + +# restore pg_hba.conf +unlink "/etc/postgresql/$version/main/pg_hba.conf"; +rename "/etc/postgresql/$version/main/pg_hba.conf.orig", + "/etc/postgresql/$version/main/pg_hba.conf" or die "rename: $!"; + +# leftover files must not create confusion +open F, '>/etc/postgresql/postgresql.conf'; +print F "data_directory = '/nonexisting'\n"; +close F; +my @c = get_version_clusters $version; +is_deeply (\@c, ['main'], + 'leftover /etc/postgresql/postgresql.conf is not regarded as a cluster'); +unlink '/etc/postgresql/postgresql.conf'; + +# fails by default due to access restrictions +# remove cluster and directory; this should work as user "postgres" +is_program_out 'postgres', "pg_dropcluster $version main", 0, '', + , "pg_dropcluster works as user postgres"; + +# graceful handling of absent data dir (might not be mounted) +ok ((system "pg_createcluster $version main >/dev/null") == 0, + "pg_createcluster succeeds"); +rename "/var/lib/postgresql/$version", "/var/lib/postgresql/$version.orig" or die "rename: $!"; +my $outref; +is ((exec_as 0, "pg_ctlcluster $version main start", $outref, 1), 1, + 'pg_ctlcluster fails on nonexisting /var/lib/postgresql'); +like $$outref, qr/^Error:.*\/var\/lib\/postgresql.*not accessible.*$/, 'proper error message for nonexisting /var/lib/postgresql'; + +rename "/var/lib/postgresql/$version.orig", "/var/lib/postgresql/$version" or die "rename: $!"; +is_program_out 'postgres', "pg_ctlcluster $version main start", 0, '', + 'pg_ctlcluster start succeeds again with reappeared /var/lib/postgresql'; +is_program_out 'postgres', "pg_ctlcluster $version main stop", 0, '', 'stopping cluster'; + +# pg_ctlcluster checks colliding ports +ok ((system "pg_createcluster $version other >/dev/null") == 0, + "pg_createcluster other"); +set_cluster_port $version, 'other', '5432'; +is ((exec_as 'postgres', "pg_ctlcluster $version main start"), 0, + 'pg_ctlcluster: main cluster on conflicting port starts'); + +# clusters can run side by side on different socket directories +set_cluster_socketdir $version, 'other', $socketdir; +PgCommon::set_conf_value $version, 'other', 'postgresql.conf', + 'listen_addresses', ''; # otherwise they will conflict on TCP socket +is ((exec_as 'postgres', "pg_ctlcluster $version other start"), 0, + 'pg_ctlcluster: other cluster starts on conflicting port, but different socket dirs'); +is ((exec_as 'postgres', "pg_ctlcluster $version other stop"), 0); + +# ... but will give an error when running on the same port +set_cluster_socketdir $version, 'other', ($PgCommon::rpm and $version < 9.4) ? '/tmp' : '/var/run/postgresql'; +like_program_out 'postgres', "pg_ctlcluster $version other start", 1, + qr/Port conflict:.*port 5432/, + 'pg_ctlcluster other cluster fails on conflicting port and same socket dir'; +is_program_out 'postgres', "pg_ctlcluster $version main stop", 0, '', + 'stopping main cluster'; +is ((exec_as 'postgres', "pg_ctlcluster $version other start"), 0, + 'pg_ctlcluster: other cluster on conflicting port starts after main is down'); +ok ((system "pg_dropcluster $version other --stop") == 0, + 'pg_dropcluster other'); + +# clean up +ok ((system "pg_dropcluster $version main") == 0, + 'pg_dropcluster'); +ok_dir $socketdir, [], 'No sockets any more'; +rmdir $socketdir or die "rmdir: $!"; + +# ensure sane error messages for nonexisting clusters +check_nonexisting_cluster_error 'pg_lsclusters 4.5 foo'; +check_nonexisting_cluster_error 'psql --cluster 4.5/foo'; +check_nonexisting_cluster_error "psql --cluster $MAJORS[0]/foo"; +check_nonexisting_cluster_error "pg_dropcluster 4.5 foo"; +check_nonexisting_cluster_error "pg_dropcluster $MAJORS[0] foo"; +check_nonexisting_cluster_error "pg_upgradecluster 4.5 foo"; +check_nonexisting_cluster_error "pg_upgradecluster $MAJORS[0] foo"; +check_nonexisting_cluster_error "pg_ctlcluster 4.5 foo stop"; +check_nonexisting_cluster_error "pg_ctlcluster $MAJORS[0] foo stop"; + +check_clean; + +# check that pg_dropcluster copes with partially existing cluster +# configurations (which can happen if the disk becomes full) + +mkdir '/etc/postgresql/'; +mkdir "/etc/postgresql/$MAJORS[-1]"; +mkdir "/etc/postgresql/$MAJORS[-1]/broken" or die "mkdir: $!"; +symlink "/var/lib/postgresql/$MAJORS[-1]/broken", "/etc/postgresql/$MAJORS[-1]/broken/pgdata" or die "symlink: $!"; + +unlike_program_out 0, "pg_dropcluster $MAJORS[-1] broken", 0, qr/error/i, + 'pg_dropcluster cleans up broken cluster configuration (only /etc with pgdata)'; + +check_clean; + +mkdir '/etc/postgresql/'; +mkdir '/var/lib/postgresql/'; +mkdir "/etc/postgresql/$MAJORS[-1]" and +mkdir "/etc/postgresql/$MAJORS[-1]/broken"; +mkdir "/var/lib/postgresql/$MAJORS[-1]"; +mkdir "/var/lib/postgresql/$MAJORS[-1]/broken"; +chown $pg_uid, $pg_gid, "/var/lib/postgresql/$MAJORS[-1]/broken"; +mkdir "/var/lib/postgresql/$MAJORS[-1]/broken/base" or die "mkdir: $!"; +open F, ">/etc/postgresql/$MAJORS[-1]/broken/postgresql.conf" or die "open: $!"; +print F "data_directory = '/var/lib/postgresql/$MAJORS[-1]/broken'\n"; +close F; +open F, ">/var/lib/postgresql/$MAJORS[-1]/broken/PG_VERSION" or die "open: $!"; +close F; + +unlike_program_out 0, "pg_dropcluster $MAJORS[-1] broken", 0, qr/error/i, + 'pg_dropcluster cleans up broken cluster configuration (/etc with pgdata and postgresql.conf and partial /var)'; +is -d "/etc/postgresql/$MAJORS[-1]", undef, "/etc/postgresql/$MAJORS[-1] was removed"; +is -d "/var/lib/postgresql/$MAJORS[-1]", undef, "/var/lib/postgresql/$MAJORS[-1] was removed"; + +check_clean; + +# vim: filetype=perl diff --git a/t/031_errors_disk_full.t b/t/031_errors_disk_full.t new file mode 100644 index 0000000..bc0a860 --- /dev/null +++ b/t/031_errors_disk_full.t @@ -0,0 +1,86 @@ +# Check for proper ENOSPC handling + +use strict; + +require File::Temp; + +use lib 't'; +use TestLib; +use Test::More tests => $ENV{NO_TMPFS} ? 1 : 22; + +# skip tests if NO_TMPFS is set +if ($ENV{NO_TMPFS}) { + pass 'Skipping disk full tests, NO_TMPFS is set'; + exit; +} + +# we are using unshare here, won't work with systemd +$ENV{_SYSTEMCTL_SKIP_REDIRECT} = 1; + +my $outref; + +# +note 'check that a failed pg_createcluster leaves no cruft behind: try creating a cluster on a 10 MB tmpfs'; +my $cmd = <&1 +set -e +mount --make-rprivate / 2> /dev/null || : +mkdir -p /var/lib/postgresql +trap "umount /var/lib/postgresql" 0 HUP INT QUIT ILL ABRT PIPE TERM +mount -t tmpfs -o size=10000000 none /var/lib/postgresql +# this is supposed to fail +LC_MESSAGES=C pg_createcluster $MAJORS[-1] test && exit 1 || true +echo -n "ls>" +# should not output anything +ls /etc/postgresql +ls /var/lib/postgresql +echo " /dev/null || : +export LC_MESSAGES=C +dirs="/etc/postgresql /var/lib/postgresql /var/log/postgresql" +mkdir -p \$dirs +trap "umount \$dirs" 0 HUP INT QUIT ILL ABRT PIPE TERM +mount -t tmpfs -o size=1000000 none /etc/postgresql +# an empty cluster needs 69MB on ppc64el, round up to 90 +mount -t tmpfs -o size=90000000 none /var/lib/postgresql +mount -t tmpfs -o size=1000000 none /var/log/postgresql +pg_createcluster $MAJORS[-1] test + +# fill up /var/lib/postgresql +! cat < /dev/zero > /var/lib/postgresql/cruft 2>/dev/null +echo '-- full lib --' +! pg_ctlcluster $MAJORS[-1] test start +echo '-- end full lib --' +echo '-- full lib log --' +cat /var/log/postgresql/postgresql-$MAJORS[-1]-test.log +echo '-- end full lib log --' +rm /var/lib/postgresql/cruft +pg_dropcluster $MAJORS[-1] test --stop +EOF + +$result = exec_as 'root', "echo '$cmd' | unshare -m sh", $outref; +is $result, 0, 'script failed'; +like $$outref, qr/^-- full lib --.*No space left on device.*^-- end full lib --/ims, + 'pg_ctlcluster prints error message'; +like $$outref, qr/^-- full lib log --.*No space left on device.*^-- end full lib log --/ims, + 'log file has error message'; + +check_clean; + +# vim: filetype=perl diff --git a/t/032_ssl_key_permissions.t b/t/032_ssl_key_permissions.t new file mode 100644 index 0000000..929f08a --- /dev/null +++ b/t/032_ssl_key_permissions.t @@ -0,0 +1,60 @@ +use strict; +use warnings; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More tests => $PgCommon::rpm ? 1 : 3 + 19 * @MAJORS; + +if ($PgCommon::rpm) { pass 'No ssl key checks on RedHat'; exit; } + +my ($pg_uid, $pg_gid) = (getpwnam 'postgres')[2,3]; +my $ssl_cert_gid = (getgrnam 'ssl-cert')[2]; # reset permissions +die "Could not determine ssl-cert gid" unless ($ssl_cert_gid); + +my $snakekey = '/etc/ssl/private/ssl-cert-snakeoil.key'; +is ((stat $snakekey)[4], 0, "$snakekey is owned by root"); +is ((stat $snakekey)[5], $ssl_cert_gid, "$snakekey group is ssl-cert"); +is ((stat $snakekey)[2], 0100640, "$snakekey mode is 0640"); + +foreach my $version (@MAJORS) { + my $pkgversion = `dpkg-query -f '\${Version}' -W postgresql-$version`; + note "$version ($pkgversion)"; + if ($version <= 9.1) { + pass "no SSL support on $version" foreach (1..19); + next; + } +SKIP: { + skip "No SSL key check on <= 9.0", 19 if ($version <= 9.0); + program_ok (0, "pg_createcluster $version main"); + + my $nobody_uid = (getpwnam 'nobody')[2]; + chown $nobody_uid, 0, $snakekey; + like_program_out 'postgres', "pg_ctlcluster $version main start", 1, + qr/private key file.*must be owned by the database user or root/s, + 'ssl key owned by nobody refused'; + +SKIP: { + skip "SSL key group check skipped on Debian oldstable packages", 4 if ($version <= 9.4 and $pkgversion !~ /pgdg/); + chown 0, 0, $snakekey; + chmod 0644, $snakekey; + like_program_out 'postgres', "pg_ctlcluster $version main start", 1, + qr/private key file.*has group or world access/, + 'ssl key with permissions root:root 0644 refused'; + + chown $pg_uid, $pg_gid, $snakekey; + chmod 0640, $snakekey; + like_program_out 'postgres', "pg_ctlcluster $version main start", 1, + qr/private key file.*has group or world access/, + 'ssl key with permissions postgres:postgres 0640 refused'; +} + + chown 0, $ssl_cert_gid, $snakekey; + + program_ok (0, "pg_dropcluster $version main --stop"); + is ((stat $snakekey)[4], 0, "$snakekey is owned by root"); + is ((stat $snakekey)[5], $ssl_cert_gid, "$snakekey group is ssl-cert"); + is ((stat $snakekey)[2], 0100640, "$snakekey mode is 0640"); + check_clean; +} +} diff --git a/t/040_upgrade.t b/t/040_upgrade.t new file mode 100644 index 0000000..6818ef8 --- /dev/null +++ b/t/040_upgrade.t @@ -0,0 +1,272 @@ +# Test upgrading from the oldest version to the latest, using the default +# configuration file. + +# Lowest supported "upgrade from" version: 8.4 (lower versions don't have lo_import) +# Lowest supported "upgrade to" version: 9.2 (lower versions don't have pg_upgrade -o) +# Lowest supported "upgrade to" version with pg_dumpall: 9.1 (lower versions don't have pg_dumpall --quote-all-identifiers) + +use strict; + +use File::Temp qw/tempfile tempdir/; +use POSIX qw/dup2/; +use Time::HiRes qw/usleep/; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More tests => (@MAJORS == 1) ? 1 : 123 * 3; + +if (@MAJORS == 1) { + pass 'only one major version installed, skipping upgrade tests'; + exit 0; +} + +foreach my $upgrade_options ('-m dump', '-m upgrade', '-m upgrade --link') { +next if ($ENV{UPGRADE_METHOD} and $upgrade_options !~ /$ENV{UPGRADE_METHOD}$/); # hack to ease debugging individual methods +note ("upgrade method \"$upgrade_options\", $MAJORS[0] -> $MAJORS[-1]"); + +# create cluster +ok ((system "pg_createcluster $MAJORS[0] upgr >/dev/null") == 0, + "pg_createcluster $MAJORS[0] upgr"); +exec_as 'root', "sed -i '/^local.*postgres/ s/\$/\\nlocal all foo trust/' /etc/postgresql/$MAJORS[0]/upgr/pg_hba.conf"; +is ((system "pg_ctlcluster $MAJORS[0] upgr start"), 0, 'Starting upgr cluster'); + +# Create nobody user, test database, and put a table into it +is ((exec_as 'postgres', 'createuser nobody -D -R -s && createdb -O nobody test && createdb -O nobody testnc && createdb -O nobody testro'), + 0, 'Create nobody user and test databases'); +is ((exec_as 'nobody', 'psql test -c "CREATE TABLE phone (name varchar(255) PRIMARY KEY, tel int NOT NULL)"'), + 0, 'create table'); +is ((exec_as 'nobody', 'psql test -c "INSERT INTO phone VALUES (\'Alice\', 2)"'), 0, 'insert Alice into phone table'); +SKIP: { + skip 'datallowconn = f not supported with pg_upgrade', 1 if $upgrade_options =~ /upgrade/; + is ((exec_as 'postgres', 'psql template1 -c "UPDATE pg_database SET datallowconn = \'f\' WHERE datname = \'testnc\'"'), + 0, 'disallow connection to testnc'); +} +is ((exec_as 'nobody', 'psql testro -c "CREATE TABLE nums (num int NOT NULL); INSERT INTO nums VALUES (1)"'), 0, 'create table in testro'); +SKIP: { + skip 'read-only not supported with pg_upgrade', 2 if $upgrade_options =~ /upgrade/; + is ((exec_as 'postgres', 'psql template1 -c "ALTER DATABASE testro SET default_transaction_read_only TO on"'), + 0, 'set testro transaction default to readonly'); + is ((exec_as 'nobody', 'psql testro -c "CREATE TABLE test(num int)"'), + 1, 'creating table in testro fails'); +} + +# create a schema and a table with a name that was un-reserved between 8.4 and 9.1 +is ((exec_as 'nobody', 'psql test -c "CREATE SCHEMA \"old\""'), + 0, 'create schema "old"'); +is ((exec_as 'nobody', 'psql test -c "CREATE TABLE \"old\".\"old\" (\"old\" text)"'), + 0, 'create table "old.old"'); + +# create a sequence +is ((exec_as 'nobody', 'psql test -c "CREATE SEQUENCE odd10 INCREMENT BY 2 MINVALUE 1 MAXVALUE 10 CYCLE"'), + 0, 'create sequence'); +is_program_out 'nobody', 'psql -Atc "SELECT nextval(\'odd10\')" test', 0, "1\n", + 'check next sequence value'; +is_program_out 'nobody', 'psql -Atc "SELECT nextval(\'odd10\')" test', 0, "3\n", + 'check next sequence value'; + +# create a large object +my ($fh, $filename) = tempfile("lo_import.XXXXXX", TMPDIR => 1, UNLINK => 1); +print $fh "Hello world"; +close $fh; +chmod 0644, $filename; +is_program_out 'postgres', "psql -Atc \"SELECT lo_import('$filename', 1234)\"", 0, "1234\n", + 'create large object'; + +# create stored procedures +if ($MAJORS[0] < 9.0) { + is_program_out 'postgres', 'createlang plpgsql test', 0, '', 'createlang plpgsql test'; +} else { + pass '>= 9.0 enables PL/pgsql by default'; + pass '...'; +} +is_program_out 'nobody', 'psql test -c "CREATE FUNCTION inc2(integer) RETURNS integer LANGUAGE plpgsql AS \'BEGIN RETURN \$1 + 2; END;\';"', + 0, "CREATE FUNCTION\n", 'CREATE FUNCTION inc2'; +SKIP: { + skip 'hardcoded library paths not supported by pg_upgrade', 2 if $upgrade_options =~ /upgrade/; + is_program_out 'postgres', "psql -c \"UPDATE pg_proc SET probin = '$PgCommon::binroot$MAJORS[0]/lib/plpgsql.so' where proname = 'plpgsql_call_handler';\" test", + 0, "UPDATE 1\n", 'hardcoding plpgsql lib path'; +} +is_program_out 'nobody', 'psql test -c "CREATE FUNCTION inc3(integer) RETURNS integer LANGUAGE plpgsql AS \'BEGIN RETURN \$1 + 3; END;\';"', + 0, "CREATE FUNCTION\n", 'create function inc3'; +is_program_out 'nobody', 'psql -Atc "SELECT inc2(3)" test', 0, "5\n", + 'call function inc2'; +is_program_out 'nobody', 'psql -Atc "SELECT inc3(3)" test', 0, "6\n", + 'call function inc3'; + +# create user and group +is_program_out 'postgres', "psql -qc 'CREATE USER foo' template1", 0, '', + 'create user foo'; +is_program_out 'postgres', "psql -qc 'CREATE GROUP gfoo' template1", 0, '', + 'create group gfoo'; + +# create per-database and per-table ACL +is_program_out 'postgres', "psql -qc 'GRANT CREATE ON DATABASE test TO foo'", 0, '', + 'GRANT CREATE ON DATABASE'; +is_program_out 'postgres', "psql -qc 'GRANT INSERT ON phone TO foo' test", 0, '', + 'GRANT INSERT'; + +# exercise ACL on old database to ensure they are working +is_program_out 'nobody', 'psql -U foo -qc "CREATE SCHEMA s_foo" test', 0, '', + 'CREATE SCHEMA on old cluster (ACL)'; +is_program_out 'nobody', 'psql -U foo -qc "INSERT INTO phone VALUES (\'Bob\', 1)" test', + 0, '', 'insert Bob into phone table (ACL)'; + +# set config parameters +is_program_out 'postgres', "pg_conftool $MAJORS[0] upgr set log_statement all", + 0, '', 'set postgresql.conf parameter'; +SKIP: { + skip 'postgresql.auto.conf not supported before 9.4', 6 if ($MAJORS[0] < 9.4); + is_program_out 'postgres', "psql -qc \"ALTER SYSTEM SET ident_file = '/etc/postgresql/$MAJORS[0]/upgr/pg_ident.conf'\"", + 0, '', 'set ident_file in postgresql.auto.conf'; + is_program_out 'postgres', 'psql -qc "ALTER SYSTEM SET log_min_duration_statement = \'10s\'"', + 0, '', 'set log_min_duration_statement in postgresql.auto.conf'; + is_program_out 'postgres', "echo \"data_directory = '/var/lib/postgresql/$MAJORS[0]/upgr'\" >> /var/lib/postgresql/$MAJORS[0]/upgr/postgresql.auto.conf", 0, "", "Append bogus data_directory setting to postgresql.auto.conf"; +} +is_program_out 'postgres', 'psql -qc "ALTER DATABASE test SET DateStyle = \'ISO, YMD\'"', + 0, '', 'set database parameter'; + +# create a tablespace +my $tdir = tempdir (CLEANUP => 1); +my ($p_uid, $p_gid) = (getpwnam 'postgres')[2,3]; +chown $p_uid, $p_gid, $tdir; +is_program_out 'postgres', "psql -qc \"CREATE TABLESPACE myts LOCATION '$tdir'\"", + 0, '', "creating tablespace in $tdir"; +is_program_out 'postgres', "psql -qc 'CREATE TABLE tstab (a int) TABLESPACE myts'", + 0, '', "creating table in tablespace"; + +# Check clusters +like_program_out 'nobody', 'pg_lsclusters -h', 0, + qr/^$MAJORS[0]\s+upgr\s+5432 online postgres/; + +# Check SELECT in original cluster +my $select_old; +is ((exec_as 'nobody', 'psql -tAc "SELECT * FROM phone ORDER BY name" test', $select_old), 0, 'SELECT in original cluster succeeds'); +is ($$select_old, 'Alice|2 +Bob|1 +', 'check SELECT output in original cluster'); + +# create inaccessible cwd, to check for confusing error messages +rmdir '/tmp/pgtest'; +mkdir '/tmp/pgtest/' or die "Could not create temporary test directory /tmp/pgtest: $!"; +chmod 0100, '/tmp/pgtest/'; +chdir '/tmp/pgtest'; + +# Upgrade to latest version +my $outref; +is ((exec_as 0, "(env LC_MESSAGES=C pg_upgradecluster -v $MAJORS[-1] $upgrade_options $MAJORS[0] upgr | sed -e 's/^/STDOUT: /')", $outref, 0), 0, 'pg_upgradecluster succeeds'); +like $$outref, qr/Starting upgraded cluster/, 'pg_upgradecluster reported cluster startup'; +like $$outref, qr/Success. Please check/, 'pg_upgradecluster reported successful operation'; +my @err = grep (!/^STDOUT: /, split (/\n/, $$outref)); +if (@err) { + fail 'no error messages during upgrade'; + print (join ("\n", @err)); +} else { + pass "no error messages during upgrade"; +} + +# remove inaccessible test cwd +chdir '/'; +rmdir '/tmp/pgtest/'; + +# Check clusters +like_program_out 'nobody', 'pg_lsclusters -h', 0, + qr"$MAJORS[0] +upgr 5433 down postgres /var/lib/postgresql/$MAJORS[0]/upgr +/var/log/postgresql/postgresql-$MAJORS[0]-upgr.log\n$MAJORS[-1] +upgr 5432 online postgres /var/lib/postgresql/$MAJORS[-1]/upgr +/var/log/postgresql/postgresql-$MAJORS[-1]-upgr.log", 'pg_lsclusters output'; + +# Check that SELECT output is identical +is_program_out 'nobody', 'psql -tAc "SELECT * FROM phone ORDER BY name" test', 0, + $$select_old, 'SELECT output is the same in original and upgraded test'; +is_program_out 'nobody', 'psql -tAc "SELECT * FROM nums" testro', 0, + "1\n", 'SELECT output is the same in original and upgraded testro'; + +# Check that table was analyzed +like_program_out 'nobody', "psql -XAtc \"select analyze_count from pg_stat_user_tables where relname = 'phone'\" test", 0, qr/^[1-3]$/, + 'check analyze count'; # --analyze-in-stages does 3 passes + +# Check sequence value +is_program_out 'nobody', 'psql -Atc "SELECT nextval(\'odd10\')" test', 0, "5\n", + 'check next sequence value'; +is_program_out 'nobody', 'psql -Atc "SELECT nextval(\'odd10\')" test', 0, "7\n", + 'check next sequence value'; +is_program_out 'nobody', 'psql -Atc "SELECT nextval(\'odd10\')" test', 0, "9\n", + 'check next sequence value'; +is_program_out 'nobody', 'psql -Atc "SELECT nextval(\'odd10\')" test', 0, "1\n", + 'check next sequence value (wrap)'; + +# check large objects +is_program_out 'postgres', 'psql -Aqtc "SET bytea_output = \'escape\'; SELECT data FROM pg_largeobject WHERE loid = 1234"', 0, "Hello world\n", + 'check large object'; + +# check stored procedures +is_program_out 'nobody', 'psql -Atc "SELECT inc2(-3)" test', 0, "-1\n", + 'call function inc2'; +is_program_out 'nobody', 'psql -Atc "SELECT inc3(1)" test', 0, "4\n", + 'call function inc3 (formerly hardcoded path)'; + +SKIP: { + skip 'upgrading databases with datallowcon = false not supported by pg_upgrade', 2 if $upgrade_options =~ /upgrade/; + + # Check connection permissions + my $testnc_conn = $upgrade_options =~ /upgrade/ ? 't' : 'f'; + is_program_out 'nobody', 'psql -tAc "SELECT datname, datallowconn FROM pg_database ORDER BY datname" template1', 0, + "postgres|t +template0|f +template1|t +test|t +testnc|$testnc_conn +testro|t +", 'dataallowconn values'; +} + +# check ACLs +is_program_out 'nobody', 'psql -U foo -qc "CREATE SCHEMA s_bar" test', 0, '', + 'CREATE SCHEMA on new cluster (ACL)'; +is_program_out 'nobody', 'psql -U foo -qc "INSERT INTO phone VALUES (\'Chris\', 5)" test', + 0, '', 'insert Chris into phone table (ACL)'; + +# check default transaction r/o +is ((exec_as 'nobody', 'psql test -c "CREATE TABLE test(num int)"'), + 0, 'creating table in test succeeds'); +SKIP: { + skip 'read-only not supported by pg_upgrade', 2 if $upgrade_options =~ /upgrade/; + is ((exec_as 'nobody', 'psql testro -c "CREATE TABLE test(num int)"'), + 1, 'creating table in testro fails'); + is ((exec_as 'postgres', 'psql testro -c "CREATE TABLE test(num int)"'), + 1, 'creating table in testro as superuser fails'); +} +is ((exec_as 'nobody', 'psql testro -c "BEGIN READ WRITE; CREATE TABLE test(num int); COMMIT"'), + 0, 'creating table in testro succeeds with RW transaction'); + +# check config parameters +is_program_out 'postgres', 'psql -Atc "SHOW log_statement" test', 0, "all\n", 'check postgresql.conf parameters'; +SKIP: { + skip 'postgresql.auto.conf not supported before 9.4', 4 if ($MAJORS[0] < 9.4); + is_program_out 'postgres', 'psql -Atc "SHOW log_min_duration_statement" test', 0, "10s\n", 'check postgresql.auto.conf parameter'; + is_program_out 'postgres', "cat /var/lib/postgresql/$MAJORS[-1]/upgr/postgresql.auto.conf", 0, + "# Do not edit this file manually!\n# It will be overwritten by the ALTER SYSTEM command.\nident_file = '/etc/postgresql/$MAJORS[-1]/upgr/pg_ident.conf'\nlog_min_duration_statement = '10s'\n#data_directory = '/var/lib/postgresql/$MAJORS[0]/upgr' #not valid in postgresql.auto.conf\n"; +} +is_program_out 'postgres', 'psql -Atc "SHOW DateStyle" test', 0, "ISO, YMD\n", 'check database parameter'; +SKIP: { + skip "cluster name not supported in $MAJORS[0]", 1 if ($MAJORS[0] < 9.5); + is (PgCommon::get_conf_value ($MAJORS[-1], 'upgr', 'postgresql.conf', 'cluster_name'), "$MAJORS[-1]/upgr", "cluster_name is updated"); +} + +# check tablespace +is_program_out 'postgres', "psql -Atc 'SELECT spcname FROM pg_tablespace ORDER BY spcname'", + 0, "myts\npg_default\npg_global\n", "check tablespace of upgraded table"; +is_program_out 'postgres', "psql -Atc \"SELECT spcname FROM pg_class c LEFT JOIN pg_tablespace t ON (c.reltablespace = t.oid) WHERE c.relname = 'tstab'\"", + 0, "myts\n", "check tablespace of upgraded table"; + +# stop servers, clean up +is ((system "pg_dropcluster $MAJORS[0] upgr --stop"), 0, 'Dropping original cluster'); +is ((system "pg_ctlcluster $MAJORS[-1] upgr restart"), 0, 'Restarting upgraded cluster'); +is_program_out 'nobody', 'psql -Atc "SELECT nextval(\'odd10\')" test', 0, "3\n", + 'upgraded cluster still works after removing old one'; +is ((system "pg_dropcluster $MAJORS[-1] upgr --stop"), 0, 'Dropping upgraded cluster'); +is ((system "rm -rf /var/log/postgresql/pg_upgradecluster-*"), 0, 'Cleaning pg_upgrade log files'); + +check_clean; +} # foreach method + +# vim: filetype=perl diff --git a/t/041_upgrade_custompaths.t b/t/041_upgrade_custompaths.t new file mode 100644 index 0000000..3416b55 --- /dev/null +++ b/t/041_upgrade_custompaths.t @@ -0,0 +1,51 @@ +# Test cluster upgrade with a custom data directory and custom log file. + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More tests => (@MAJORS == 1) ? 1 : 28; + +if (@MAJORS == 1) { + pass 'only one major version installed, skipping upgrade tests'; + exit 0; +} + +ok ((system "pg_createcluster --start --datadir /tmp/postgresql-test -l /tmp/postgresql-test.log $MAJORS[0] upgr >/dev/null") == 0); + +# Upgrade to latest version +my $outref; +is ((exec_as 0, "pg_upgradecluster -v $MAJORS[-1] $MAJORS[0] upgr", $outref, 0), 0, 'pg_upgradecluster succeeds'); +like $$outref, qr/Starting upgraded cluster/, 'pg_upgradecluster reported cluster startup'; +like $$outref, qr/Success. Please check/, 'pg_upgradecluster reported successful operation'; + +# Check clusters +like_program_out 'nobody', 'pg_lsclusters -h', 0, + qr"$MAJORS[0] +upgr 5433 down postgres /tmp/postgresql-test +/tmp/postgresql-test.log\n$MAJORS[-1] +upgr 5432 online postgres /var/lib/postgresql/$MAJORS[-1]/upgr +/var/log/postgresql/postgresql-$MAJORS[-1]-upgr.log", 'pg_lsclusters output'; + +# clean away new cluster and restart the old one +is ((system "pg_dropcluster $MAJORS[-1] upgr --stop"), 0, 'Dropping upgraded cluster'); +is_program_out 0, "pg_ctlcluster $MAJORS[0] upgr start", 0, '', 'Restarting old cluster'; +is_program_out 'nobody', 'pg_lsclusters -h', 0, + "$MAJORS[0] upgr 5433 online postgres /tmp/postgresql-test /tmp/postgresql-test.log +", 'pg_lsclusters output'; + +# Do another upgrade with using a custom defined data directory (and in passing, test --keep-port) +my $outref; +is ((exec_as 0, "pg_upgradecluster --keep-port -v $MAJORS[-1] $MAJORS[0] upgr /tmp/psql-common-testsuite", $outref, 0), 0, 'pg_upgradecluster succeeds'); +unlike $$outref, qr/^pg_restore: /m, 'no pg_restore error messages during upgrade'; +unlike $$outref, qr/^[A-Z]+: /m, 'no server error messages during upgrade'; +like $$outref, qr/Starting upgraded cluster/, 'pg_upgradecluster reported cluster startup'; +like $$outref, qr/Success. Please check/, 'pg_upgradecluster reported successful operation'; + +like_program_out 'nobody', 'pg_lsclusters -h', 0, + qr"$MAJORS[0] +upgr 5433 down postgres /tmp/postgresql-test +/tmp/postgresql-test.log\n$MAJORS[-1] +upgr 5432 online postgres /tmp/psql-common-testsuite +/var/log/postgresql/postgresql-$MAJORS[-1]-upgr.log", 'pg_lsclusters output'; + +# stop servers, clean up +is ((system "pg_dropcluster $MAJORS[0] upgr"), 0, 'Dropping original cluster'); +is ((system "pg_dropcluster $MAJORS[-1] upgr --stop"), 0, 'Dropping upgraded cluster'); + +check_clean; + +# vim: filetype=perl diff --git a/t/042_upgrade_rename.t b/t/042_upgrade_rename.t new file mode 100644 index 0000000..ef44f11 --- /dev/null +++ b/t/042_upgrade_rename.t @@ -0,0 +1,27 @@ +# Test in-version upgrading (usually used after catalog version bumps) + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More tests => 15 * @MAJORS; + +foreach my $v (@MAJORS) { + SKIP: { + skip "pg_upgrade not supported on $v", 15 if ($v < 9.2); + note "PostgreSQL $v"; + + program_ok 0, "pg_createcluster $v main --start", 0; + program_ok 0, "pg_upgradecluster -m upgrade --old-bindir=$PgCommon::binroot$v/bin -v $v --rename upgr $v main", 0; + like_program_out 0, "pg_lsclusters -h", 0, qr/$v main 5433 down.*\n$v upgr 5432 online/; + + program_ok 0, "pg_dropcluster $v main --stop", 0; + program_ok 0, "pg_dropcluster $v upgr --stop", 0; + is ((system "rm -rf /var/log/postgresql/pg_upgradecluster-$v-$v-upgr.*"), 0, 'Cleaning pg_upgrade log files'); + check_clean; + } +} + +# vim: filetype=perl diff --git a/t/043_upgrade_ssl_cert.t b/t/043_upgrade_ssl_cert.t new file mode 100644 index 0000000..d7dc79e --- /dev/null +++ b/t/043_upgrade_ssl_cert.t @@ -0,0 +1,79 @@ +# Test cluster upgrade with a custom ssl certificate + +use strict; +use File::Temp qw/tempdir/; +use lib 't'; +use TestLib; +use PgCommon; +use Test::More tests => (@MAJORS == 1 or $PgCommon::rpm) ? 1 : 22; + +if (@MAJORS == 1) { + pass 'only one major version installed, skipping upgrade tests'; + exit 0; +} +if ($PgCommon::rpm) { + pass 'SSL certificates not handled on RedHat'; + exit 0; +} + + +ok ((system "pg_createcluster $MAJORS[0] upgr >/dev/null") == 0); + +my $tdir = tempdir (CLEANUP => 1); +my ($p_uid, $p_gid) = (getpwnam 'postgres')[2,3]; +chown $p_uid, $p_gid, $tdir; + +my $tempcrt = "$tdir/ssl-cert-snakeoil.pem"; +my $oldcrt = "/var/lib/postgresql/$MAJORS[0]/upgr/server.crt"; +my $newcrt = "/var/lib/postgresql/$MAJORS[-1]/upgr/server.crt"; + +# First upgrade +note "upgrade test: server.crt is a symlink"; +(system "cp -p /etc/ssl/certs/ssl-cert-snakeoil.pem $tempcrt") == 0 or die "cp: $!"; +unlink $oldcrt; # remove file installed by pg_createcluster +symlink $tempcrt, $oldcrt or die "symlink: $!"; + +# Upgrade to latest version +my $outref; +is ((exec_as 0, "pg_upgradecluster --start -v $MAJORS[-1] $MAJORS[0] upgr", $outref, 0), 0, 'pg_upgradecluster succeeds'); +like $$outref, qr/Starting upgraded cluster/, 'pg_upgradecluster reported cluster startup'; +like $$outref, qr/Success. Please check/, 'pg_upgradecluster reported successful operation'; + +if ($MAJORS[-1] >= 9.2) { + is ((-e $newcrt), undef, "new data directory does not contain server.crt"); + is ((PgCommon::get_conf_value $MAJORS[-1], 'upgr', 'postgresql.conf', 'ssl_cert_file'), + $tempcrt, "symlink server.crt target is put into ssl_cert_file"); +} else { + is ((-l $newcrt), 1, "new data directory contains server.crt"); + is ((readlink $newcrt), $tempcrt, "symlink server.crt points to correct location"); +} + +# Clean away new cluster +is ((system "pg_dropcluster $MAJORS[-1] upgr --stop"), 0, 'Dropping upgraded cluster'); +unlink $oldcrt or die "unlink: $!"; + +# Second upgrade +note "upgrade test: server.crt is a plain file"; +(system "cp -p $tempcrt $oldcrt") == 0 or die "cp: $!"; + +# Upgrade to latest version +my $outref; +is ((exec_as 0, "pg_upgradecluster --start -v $MAJORS[-1] $MAJORS[0] upgr", $outref, 0), 0, 'pg_upgradecluster succeeds'); +like $$outref, qr/Starting upgraded cluster/, 'pg_upgradecluster reported cluster startup'; +like $$outref, qr/Success. Please check/, 'pg_upgradecluster reported successful operation'; + +is ((-f $newcrt), 1, "new data directory contains server.crt file"); +if ($MAJORS[-1] >= 9.2) { + is ((PgCommon::get_conf_value $MAJORS[-1], 'upgr', 'postgresql.conf', 'ssl_cert_file'), + $newcrt, "server.crt is put into ssl_cert_file"); +} else { + pass "..."; +} + +# Stop servers, clean up +is ((system "pg_dropcluster $MAJORS[0] upgr"), 0, 'Dropping original cluster'); +is ((system "pg_dropcluster $MAJORS[-1] upgr --stop"), 0, 'Dropping upgraded cluster'); + +check_clean; + +# vim: filetype=perl diff --git a/t/045_backup.t b/t/045_backup.t new file mode 100644 index 0000000..1258ea3 --- /dev/null +++ b/t/045_backup.t @@ -0,0 +1,169 @@ +use strict; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More; +use Time::HiRes qw/usleep/; + +my ($pg_uid, $pg_gid) = (getpwnam 'postgres')[2,3]; +my $systemd = (-d "/run/systemd/system" and not $ENV{_SYSTEMCTL_SKIP_REDIRECT}); +note $systemd ? "We are running systemd" : "We are not running systemd"; + +foreach my $v (@MAJORS) { + if ($v < 9.1) { + ok 1, "pg_backupcluster not supported on $v"; + next; + } + note "PostgreSQL $v"; + + note "create cluster"; + program_ok 0, "pg_createcluster --locale en_US.UTF-8 $v main --start"; + like_program_out 0, "pg_lsclusters -h", 0, qr/$v main 5432 online/; + program_ok 0, "pg_conftool $v main set work_mem 11MB"; + if ($v <= 9.6) { + open my $hba, ">>", "/etc/postgresql/$v/main/pg_hba.conf"; + print $hba "local replication all peer\n"; + close $hba; + program_ok 0, "pg_conftool $v main set max_wal_senders 10"; + program_ok 0, "pg_conftool $v main set wal_level archive"; + program_ok 0, "pg_conftool $v main set max_replication_slots 10" if ($v >= 9.4); + program_ok 0, "pg_conftool $v main set ssl off" if ($v <= 9.1); # cert symlinks not backed up in 9.1 + program_ok 0, "pg_ctlcluster $v main restart"; + } + my $locale_provider = $v >= 15 ? "--locale-provider libc " : ""; + program_ok $pg_uid, "createdb -E SQL_ASCII $locale_provider-T template0 mydb"; + program_ok $pg_uid, "psql -c 'alter database mydb set search_path=public'"; + program_ok $pg_uid, "psql -c 'create table foo (t text)' mydb"; + program_ok $pg_uid, "psql -c \"insert into foo values ('data from backup')\" mydb"; + program_ok $pg_uid, "psql -c 'CREATE USER myuser'"; + program_ok $pg_uid, "psql -c 'alter role myuser set search_path=public, myschema'"; + program_ok $pg_uid, "createdb --locale-provider icu --icu-locale de -T template0 myicudb" if ($v >= 15); + + SKIP: { # in PG 10, ARID is part of globals.sql which we try to restore before databases.sql + skip "alter role in database handling in PG <= 10 not supported", 1 if ($v <= 10); + program_ok $pg_uid, "psql -c 'alter role myuser in database mydb set search_path=public, myotherschema'"; + } + + note "create directory"; + program_ok 0, "pg_backupcluster $v main createdirectory"; + my $dir = "/var/backups/postgresql/$v-main"; + my @stat = stat $dir; + is $stat[4], $pg_uid, "$dir owned by uid postgres"; + is $stat[5], $pg_gid, "$dir owned by gid postgres"; + + my @backups = (); + my $dump = ''; + SKIP: { + skip "dump not supported before 9.3", 1 if ($v < 9.3); + note "dump"; + if ($systemd) { + program_ok 0, "systemctl start pg_dump\@$v-main"; + } else { + program_ok 0, "pg_backupcluster $v main dump"; + } + ($dump) = glob "$dir/*.dump"; + ok -d $dump, "dump created in $dump"; + @stat = stat $dump; + is $stat[4], $pg_uid, "$dump owned by uid postgres"; + is $stat[5], $pg_gid, "$dump owned by gid postgres"; + push @backups, $dump; + } + + note "basebackup"; + my $receivewal_pid; + if ($v >= 9.5) { + if ($systemd) { + program_ok 0, "systemctl start pg_receivewal\@$v-main"; + } else { + $receivewal_pid = fork; + if ($receivewal_pid == 0) { + # suppress "not renaming "000000010000000000000003.gz.partial", segment is not complete" + exec "pg_backupcluster $v main receivewal 2>/dev/null"; + } + } + program_ok $pg_uid, "psql -c 'create table poke_receivewal (t text)' mydb"; + usleep($delay); + my $wal = "000000010000000000000001"; + $wal .= ".gz" if ($v >= 10); + $wal .= ".partial"; + TODO: { + local $TODO = "WAL test is unstable"; + ok_dir "$dir/wal", [$wal], "$dir/wal contains $wal"; + } + } + if ($systemd) { + program_ok 0, "systemctl start pg_basebackup\@$v-main"; + } else { + program_ok 0, "pg_backupcluster --checkpoint=fast $v main basebackup"; + } + my ($basebackup) = glob "$dir/*.backup"; + ok -d $basebackup, "dump created in $basebackup"; + @stat = stat $basebackup; + is $stat[4], $pg_uid, "$basebackup owned by uid postgres"; + is $stat[5], $pg_gid, "$basebackup owned by gid postgres"; + push @backups, $basebackup; + + note "list"; + like_program_out 0, "pg_backupcluster $v main list", 0, qr/$dump.*$basebackup/s; + + note "more database changes"; + program_ok $pg_uid, "psql -c \"insert into foo values ('data later deleted')\" mydb"; + program_ok $pg_uid, "psql -c \"insert into foo values ('data from archive')\" mydb"; + my $timestamp = `su -c "psql -XAtc 'select now()'" postgres`; + ok $timestamp, "retrieve recovery timestamp"; + program_ok $pg_uid, "psql -c \"delete from foo where t = 'data later deleted'\" mydb"; + usleep($delay); + if ($v >= 9.5) { + # since we are stopping pg_receivewal before postgresql, this implicitly tests restoring from .partial WAL files as well + if ($systemd) { + program_ok 0, "systemctl stop pg_receivewal\@$v-main"; + } else { + is kill('INT', $receivewal_pid), 1, "stop receivewal"; + } + } + + for my $backup (@backups) { + note "restore $backup"; + program_ok 0, "pg_dropcluster $v main --stop"; + program_ok 0, "pg_restorecluster $v main $backup --start --datadir /var/lib/postgresql/$v/snowflake"; + like_program_out 0, "pg_lsclusters -h", 0, qr/$v main 5432 online postgres .var.lib.postgresql.$v.snowflake/; + my $outref; + is exec_as($pg_uid, "psql -XAtl", $outref), 0, 'psql -XAtl'; + like $$outref, qr/^mydb\|postgres\|SQL_ASCII\|(libc\|)?en_US.UTF-8\|en_US.UTF-8\|(\|libc\||\|\|)?$/m, "mydb locales"; + like $$outref, qr/^myicudb\|postgres\|UTF8\|(icu\|)?en_US.UTF-8\|en_US.UTF-8\|(de\|icu\||de\|\|)?$/m, "myicudb locales" if ($v >= 15); + is_program_out $pg_uid, "psql -XAtc 'show work_mem'", 0, "11MB\n"; + is_program_out $pg_uid, "psql -XAtc 'select * from foo' mydb", 0, "data from backup\n"; + is_program_out $pg_uid, "psql -XAtc \"select analyze_count between 1 and 3 from pg_stat_user_tables where relname = 'foo'\" mydb", 0, + "t\n"; # --analyze-in-stages does 3 passes + SKIP: { + skip "alter role in database handling in PG <= 10 not supported", 1 if ($v <= 10); + like_program_out $pg_uid, "psql -XAtc '\\drds'", 0, qr/myuser\|mydb\|search_path=public, myotherschema.* +myuser\|\|search_path=public, myschema.* +\|mydb\|search_path=public.*\n/; + } + } + + if ($v >= 9.5) { + note "restore $basebackup with WAL archive"; + program_ok 0, "pg_dropcluster $v main --stop"; + program_ok 0, "pg_restorecluster $v main $basebackup --start --archive --port 5430"; + like_program_out 0, "pg_lsclusters -h", 0, qr/$v main 5430 online postgres .var.lib.postgresql.$v.main/; + is_program_out $pg_uid, "psql -XAtc 'select * from foo order by t' mydb", 0, "data from archive\ndata from backup\n"; + + note "restore $basebackup with PITR"; + program_ok 0, "pg_dropcluster $v main --stop"; + program_ok 0, "pg_restorecluster $v main $basebackup --start --pitr '$timestamp'"; + like_program_out 0, "pg_lsclusters -h", 0, qr/$v main 5432 online postgres .var.lib.postgresql.$v.main/; + is_program_out $pg_uid, "psql -XAtc 'select * from foo order by t' mydb", 0, "data from archive\ndata from backup\ndata later deleted\n"; + } + + program_ok 0, "pg_dropcluster $v main --stop"; + check_clean; + +} # foreach version + +done_testing(); + +# vim: filetype=perl diff --git a/t/050_encodings.t b/t/050_encodings.t new file mode 100644 index 0000000..02dd2f7 --- /dev/null +++ b/t/050_encodings.t @@ -0,0 +1,117 @@ +# Test locale and encoding settings in pg_createcluster. + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More tests => @MAJORS * 52 + 8; + +# create a test cluster with given locale, check the locale/encoding, and +# remove it +# Arguments: [] +sub check_cluster { + my ($v, $locale, $enc) = @_; + note "Checking $v $locale"; + my $cluster_name = $locale; + $cluster_name =~ s/-//g; # strip dashes so postgresql@.service likes it + if (defined $enc) { + $cluster_name .= "_$enc"; + is ((system "LC_ALL='$locale' pg_createcluster --encoding $enc --start $v $cluster_name >/dev/null 2>&1"), 0, + "pg_createcluster version $v for $locale with --encoding succeeded"); + } else { + is ((system "pg_createcluster --start --locale=$locale $v $cluster_name >/dev/null 2>&1"), 0, + "pg_createcluster version $v for $locale without --encoding succeeded"); + } + + # check encoding + sleep 1; + my $outref; + is ((exec_as 'postgres', "psql -Atl --cluster $v/$cluster_name", $outref, 0), 0, + 'psql -l succeeds'); + my $is_unicode = 0; + $is_unicode = 1 if defined $enc && $enc =~ /(UNICODE|UTF-8)/; + $is_unicode = 1 if $locale =~ /UTF-8/; + if ($is_unicode) { + like $$outref, qr/template1.*(UNICODE|UTF8)/, 'template1 is UTF-8 encoded'; + } else { + unlike $$outref, qr/template1.*(UNICODE|UTF8)/, 'template1 is not UTF-8 encoded'; + } + + # create a table and stuff some ISO-8859-5 characters into it (для) + is ((exec_as 'postgres', "createdb test", $outref), 0, 'creating test database'); + is_program_out 'postgres', "printf '\324\333\357' | psql -qc \"set client_encoding='iso-8859-5'; + create table t (x varchar); copy t from stdin\" test", 0, '', + 'creating table with ISO-8859-5 characters'; + is_program_out 'postgres', "echo \"set client_encoding='utf8'; select * from t\" | psql -Atq test", 0, + "\320\264\320\273\321\217\n", 'correct string in UTF-8'; + is_program_out 'postgres', "echo \"set client_encoding='iso-8859-5'; select * from t\" | psql -Atq test", 0, + "\324\333\357\n", 'correct string in ISO-8859-5'; + + # do the same test with using UTF-8 as input + is_program_out 'postgres', "printf '\320\264\320\273\321\217' | psql -qc \"set client_encoding='utf8'; + delete from t; copy t from stdin\" test", 0, '', + 'creating table with UTF-8 characters'; + is_program_out 'postgres', "echo \"set client_encoding='utf8'; select * from t\" | psql -Atq test", 0, + "\320\264\320\273\321\217\n", 'correct string in UTF-8'; + is_program_out 'postgres', "echo \"set client_encoding='iso-8859-5'; select * from t\" | psql -Atq test", 0, + "\324\333\357\n", 'correct string in ISO-8859-1'; + + # check encoding of server error messages (breaks in locale/encoding mismatches, so skip that) + if (!defined $enc) { + # temporarily disable and accept English text, since Russian translations are disabled now + like_program_out 'postgres', 'psql test -c "set client_encoding = \'UTF-8\'; select sqrt(-1)"', 1, + qr/^[^?]*(брать|отрицательного|cannot take square root)[^?]*$/, 'Server error message has correct language and encoding'; + } + + # check that we do not run into 'ignoring unconvertible UTF-8 character' + # breakage on nonmatching lc_messages and client_encoding + PgCommon::set_conf_value $v, $cluster_name, 'postgresql.conf', + 'client_encoding', 'UTF-8'; + PgCommon::set_conf_value $v, $cluster_name, 'postgresql.conf', + 'lc_messages', 'POSIX'; + is_program_out 0, "pg_ctlcluster $v $cluster_name restart", 0, '', + 'cluster starts correctly with nonmatching lc_messages and client_encoding'; + + # check interception of invalidly encoded/escaped strings + if ($is_unicode) { + like_program_out 'postgres', + 'printf "set client_encoding=\'UTF-8\'; select \'\\310\\\\\'a\'" | psql -Atq template1', + 0, qr/(UNICODE|UTF8).*0x(c8.*5c|c8.*27)/, + 'Server rejects incorrect encoding (CVE-2006-2313)'; + like_program_out 'postgres', + 'printf "set client_encoding=\'SJIS\'; select \'\\\\\\\'a\'" | psql -Atq template1', + 0, qr/(\\' is insecure)|(unterminated quoted string)/, + 'Server rejects \\\' escaping in unsafe client encoding (CVE-2006-2314)'; + if ($v >= '9.1') { + like_program_out 'postgres', + "printf \"set client_encoding='UTF-8'; set escape_string_warning='off'; select '\\\\\\'a'\" | psql -Atq template1", + 0, qr/unterminated quoted string/, + 'Server rejects obsolete \\\' escaping in unsafe client encoding (CVE-2006-2314)'; + } else { + is_program_out 'postgres', + "printf \"set client_encoding='UTF-8'; set escape_string_warning='off'; select '\\\\\\'a'\" | psql -Atq template1", + 0, "'a\n", 'Server accepts \\\' escaping in safe client encoding (CVE-2006-2314)'; + } + } + + # drop cluster + is ((system "pg_dropcluster $v $cluster_name --stop"), 0, 'Dropping cluster'); +} + +foreach my $v (@MAJORS) { + check_cluster $v, 'ru_RU'; + check_cluster $v, 'ru_RU.UTF-8'; + + note "Check $v locale environment variables"; + # check LC_* over LANG domination + is ((system "LANGUAGE= LC_ALL=C LANG=bo_GUS.UTF-8 pg_createcluster --start $v main >/dev/null 2>&1"), 0, + "pg_createcluster: LC_ALL dominates LANG"); + like_program_out 'postgres', "psql -Atl --cluster $v/main", 0, + qr/template1.*ASCII/, 'template1 is ASCII encoded'; + is ((system "pg_dropcluster $v main --stop"), 0, 'Dropping cluster'); +} + +check_clean; + +# vim: filetype=perl diff --git a/t/052_upgrade_encodings.t b/t/052_upgrade_encodings.t new file mode 100644 index 0000000..e7a7125 --- /dev/null +++ b/t/052_upgrade_encodings.t @@ -0,0 +1,98 @@ +# Test default and explicit encoding on upgrades + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More tests => (@MAJORS == 1) ? 1 : 46; + +if (@MAJORS == 1) { + pass 'only one major version installed, skipping upgrade tests'; + exit 0; +} + +my $outref; +my $oldv = $MAJORS[0]; +my $newv = $MAJORS[-1]; + +is ((exec_as 0, "pg_createcluster --start --locale=ru_RU $oldv main", $outref), 0, + "creating ru_RU $oldv cluster"); + +is ((exec_as 'postgres', 'psql -c "create database latintest" template1', $outref), 0, + "creating latintest DB with LATIN encoding"); +if ($oldv <= '8.3') { + is ((exec_as 'postgres', 'psql -c "create database asctest encoding = \'SQL_ASCII\'" template1', $outref), 0, + "creating asctest DB with ASCII encoding"); +} else { + is ((exec_as 'postgres', 'psql -c "create database asctest template = template0 lc_collate = \'C\' lc_ctype = \'C\' encoding = \'SQL_ASCII\'" template1', $outref), 0, + "creating asctest DB with C locale"); +} +if ($oldv >= 15) { + program_ok 'postgres', 'psql -c "create database icutest template template0 locale_provider icu icu_locale de"', 0, "creating database with ICU locale"; +} else { + program_ok 'postgres', 'psql -c "create database icutest"', 0, "creating placeholder icutest database"; +} + +is ((exec_as 'postgres', "printf 'A\\324B' | psql -c \"create table t(x varchar); copy t from stdin\" latintest", $outref), + 0, 'write LATIN database content to latintest'); +is ((exec_as 'postgres', "printf 'A\\324B' | psql -c \"create table t(x varchar); copy t from stdin\" asctest", $outref), + 0, 'write LATIN database content to asctest'); + +is_program_out 'postgres', "echo \"select * from t\" | psql -Atq latintest", + 0, "A\324B\n", 'old latintest DB has correctly encoded string'; +is_program_out 'postgres', "echo \"select * from t\" | psql -Atq asctest", + 0, "A\324B\n", 'old asctest DB has correctly encoded string'; + +is ((exec_as 'postgres', 'psql -Atl', $outref), 0, 'psql -Atl on old cluster'); +ok ((index $$outref, 'latintest|postgres|ISO_8859_5') >= 0, 'latintest is LATIN encoded'); +ok ((index $$outref, 'asctest|postgres|SQL_ASCII') >= 0, 'asctest is ASCII encoded'); +if ($oldv >= 15) { + like $$outref, qr/icutest\|postgres\|ISO_8859_5\|icu\|ru_RU\|ru_RU\|de/, 'icutest has proper icu locale'; +} else { + like $$outref, qr/icutest\|postgres\|ISO_8859_5/, 'icutest is LATIN encoded'; +} +ok ((index $$outref, 'template1|postgres|ISO_8859_5') >= 0, 'template1 is LATIN encoded'); + +# upgrade without specifying locales, should be kept +like_program_out 0, "pg_upgradecluster -v $newv $oldv main", 0, qr/^Success. Please check/m; + +is ((exec_as 'postgres', "psql --cluster $newv/main -Atl", $outref), 0, 'psql -Atl on upgraded cluster'); +ok ((index $$outref, 'latintest|postgres|ISO_8859_5') >= 0, 'latintest is LATIN encoded'); +ok ((index $$outref, 'asctest|postgres|SQL_ASCII') >= 0, 'asctest is ASCII encoded'); +if ($oldv >= 15) { + like $$outref, qr/icutest\|postgres\|ISO_8859_5\|icu\|ru_RU\|ru_RU\|de/, 'icutest has proper icu locale'; +} else { + like $$outref, qr/icutest\|postgres\|ISO_8859_5/, 'icutest is LATIN encoded'; +} +ok ((index $$outref, 'template1|postgres|ISO_8859_5') >= 0, 'template1 is LATIN encoded'); +is_program_out 'postgres', "echo \"select * from t\" | psql --cluster $newv/main -Atq latintest", + 0, "A\324B\n", 'new latintest DB has correctly encoded string'; + +is ((system "pg_dropcluster --stop $newv main"), 0, 'dropping upgraded cluster'); +is ((system "pg_ctlcluster $oldv main start"), 0, 'restarting old cluster'); + +# upgrade with explicitly specifying other locale +like_program_out 0, "pg_upgradecluster --locale ru_RU.UTF-8 -v $newv $oldv main", 0, qr/^Success. Please check/m; + +is ((exec_as 'postgres', "psql --cluster $newv/main -Atl", $outref), 0, 'psql -Atl on upgraded cluster'); +if ($newv >= 11) { + ok ((index $$outref, 'latintest|postgres|ISO_8859_5') >= 0, 'latintest is still LATIN encoded'); +} else { + like $$outref, qr/latintest\|postgres\|(UTF8|UNICODE)/, 'latintest is now UTF8 encoded'; +} +ok ((index $$outref, 'asctest|postgres|SQL_ASCII') >= 0, 'asctest is ASCII encoded'); +like $$outref, qr/template1\|postgres\|(UTF8|UNICODE)/, 'template1 is UTF8 encoded'; +is_program_out 'postgres', "echo \"select * from t\" | psql --cluster $newv/main -Atq latintest", + 0, ($newv >= 11 ? "A\324B\n": "AдB\n"), 'new latintest DB has correctly encoded string'; +# ASCII databases don't do automatic encoding conversion, so this remains LATIN +is_program_out 'postgres', "echo \"select * from t\" | psql --cluster $newv/main -Atq asctest", + 0, "A\324B\n", 'new asctest DB has correctly encoded string'; + +is ((system "pg_dropcluster --stop $newv main"), 0, 'dropping upgraded cluster'); + +is ((system "pg_dropcluster $oldv main"), 0, 'dropping old cluster'); + +check_clean; + +# vim: filetype=perl diff --git a/t/060_obsolete_confparams.t b/t/060_obsolete_confparams.t new file mode 100644 index 0000000..c276431 --- /dev/null +++ b/t/060_obsolete_confparams.t @@ -0,0 +1,83 @@ +# Test upgrading from the oldest version to all majors with all possible +# configuration parameters set. This checks that they are correctly +# transitioned. + +use strict; + +use lib 't'; +use TestLib; + +use Test::More; + +if (@MAJORS == 1) { + pass 'only one major version installed, skipping upgrade tests'; + done_testing(); + exit 0; +} + +$ENV{_SYSTEMCTL_SKIP_REDIRECT} = 1; # FIXME: testsuite is hanging otherwise + +# Test one particular upgrade (old version, new version) +sub do_upgrade { + my $cur = $_[0]; + my $new = $_[1]; + note "Testing upgrade $cur -> $new"; + + # Upgrade cluster + like_program_out 0, "env LC_MESSAGES=C pg_upgradecluster -v $new $cur main", 0, qr/^Success. Please check/m; + like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/$new.*online/, + "New $new cluster is online"; +} + +# create cluster for oldest version +is_program_out 0, "pg_createcluster $MAJORS[0] main >/dev/null", 0, ""; + +# generate configuration file with all settings and start cluster +is_program_out 0, "sed -i -e 's/^#\\([a-z]\\)/\\1/' /etc/postgresql/$MAJORS[0]/main/postgresql.conf", + 0, "", "Enabling all settings in /etc/postgresql/$MAJORS[0]/main/postgresql.conf"; +like PgCommon::get_conf_value($MAJORS[0], 'main', 'postgresql.conf', 'work_mem'), qr/MB/, "work_mem is set"; + +# tweak invalid settings +PgCommon::set_conf_value $MAJORS[0], 'main', 'postgresql.conf', 'log_timezone', 'UTC'; +PgCommon::set_conf_value $MAJORS[0], 'main', 'postgresql.conf', 'timezone', 'UTC'; +PgCommon::disable_conf_value $MAJORS[0], 'main', 'postgresql.conf', 'include_dir', "Disable placeholder value"; +PgCommon::disable_conf_value $MAJORS[0], 'main', 'postgresql.conf', 'include_if_exists', "Disable placeholder value"; +PgCommon::disable_conf_value $MAJORS[0], 'main', 'postgresql.conf', 'include', "Disable placeholder value"; +# older versions (<= 9.1 as of 2019-03) do not support ssl anymore +my $postgres = PgCommon::get_program_path('postgres', $MAJORS[0]); +my $ldd = `ldd $postgres 2>/dev/null`; +if ($ldd and $ldd !~ /libssl/) { + is_program_out 0, "sed -i -e 's/^ssl/#ssl/' /etc/postgresql/$MAJORS[0]/main/postgresql.conf", + 0, "", "Disabling ssl settings on server that does not support SSL"; +} + +# start server +is_program_out 0, "pg_ctlcluster $MAJORS[0] main start", 0, ""; + +# Loop over all but the latest major version, testing N->N+1 upgrades +for my $index (0 .. @MAJORS - 2) { + do_upgrade $MAJORS[$index], $MAJORS[$index + 1] +} +# remove all clusters except for the first one +for my $index (1 .. @MAJORS - 1) { + is_program_out 0, "pg_dropcluster $MAJORS[$index] main --stop", 0, "", "Dropping $MAJORS[$index]/main"; +} + +# now test a direct upgrade from oldest to newest, to also catch parameters +# which changed several times, like syslog -> redirect_stderr -> +# logging_collector +if ($#MAJORS > 1) { + is_program_out 0, "pg_ctlcluster $MAJORS[0] main start", 0, ""; + do_upgrade $MAJORS[0], $MAJORS[-1]; + is_program_out 0, "pg_dropcluster $MAJORS[-1] main --stop", 0, "", "Dropping $MAJORS[-1]/main"; +} else { + pass 'only two available versions, skipping tests...'; +} + +# remove first cluster +is_program_out 0, "pg_dropcluster $MAJORS[0] main --stop", 0, "", "Dropping $MAJORS[0]/main"; + +check_clean; +done_testing(); + +# vim: filetype=perl diff --git a/t/070_non_postgres_clusters.t b/t/070_non_postgres_clusters.t new file mode 100644 index 0000000..06d7f13 --- /dev/null +++ b/t/070_non_postgres_clusters.t @@ -0,0 +1,116 @@ +# Test successful operation of clusters which are not owned by +# postgres. Only check the oldest and newest version. + +use strict; + +use lib 't'; +use TestLib; + +use Test::More tests => 40; + +$ENV{_SYSTEMCTL_SKIP_REDIRECT} = 1; # FIXME: testsuite is hanging otherwise + +my $owner = 'nobody'; +my $v = $MAJORS[0]; + +# create cluster +is ((system "pg_createcluster -u $owner $v main >/dev/null"), 0, + "pg_createcluster $v main for owner $owner"); + +# check if start is refused when config and data owner do not match +my $pgconf = "/etc/postgresql/$v/main/postgresql.conf"; +my ($origuid, $origgid) = (stat $pgconf)[4,5]; +chown 1, 1, $pgconf; +like_program_out 0, "pg_ctlcluster $v main start", 1, qr/do not match/, "start refused when config and data owners mismatch"; +chown $origuid, $origgid, $pgconf; +is ((system "pg_ctlcluster $v main start"), 0, "pg_ctlcluster succeeds with owner $owner"); + +# Check cluster +like_program_out $owner, 'pg_lsclusters -h', 0, + qr/^$v\s+main\s+5432\s+online\s+$owner/, + 'pg_lsclusters shows running cluster'; + +like ((ps 'postgres'), qr/^$owner.*bin\/postgres .*\/var\/lib\/postgresql\/$v\/main/m, + "postgres is running as user $owner"); + +is_program_out $owner, 'ls /tmp/.s.PGSQL.*', 0, "/tmp/.s.PGSQL.5432\n/tmp/.s.PGSQL.5432.lock\n", 'socket is in /tmp'; + +ok_dir '/var/run/postgresql', [], '/var/run/postgresql is empty'; + +# verify owner of configuration files +my @st; +my $confdir = "/etc/postgresql/$v/main"; +my ($owneruid, $ownergid) = (getpwnam $owner)[2,3]; +@st = stat $confdir; +is $st[4], $owneruid, 'conf dir is owned by user'; +is $st[5], $ownergid, 'conf dir is owned by user\'s primary group'; +my ($ok_uid, $ok_gid) = (1, 1); +opendir D, $confdir or die "opendir: $!"; +for my $f (readdir D) { + next if $f eq '.' or $f eq '..'; + @st = stat "$confdir/$f" or die "stat: $!"; + if ($st[4] != $owneruid) { + note "$f is not owned by user"; + $ok_uid = 0; + } + if ($st[5] != $ownergid) { + note "$f is not owned by user's primary group"; + $ok_gid = 0; + } +} +closedir D; +is $ok_uid, 1, "files are owned by user"; +is $ok_gid, 1, "files are owned by user's primary group"; + +# verify log file properties +@st = stat "/var/log/postgresql/postgresql-$v-main.log"; +is $st[2], 0100640, 'log file has 0640 permissions'; +is $st[4], $owneruid, 'log file is owned by user'; +# the log file gid setting works on RedHat, but nobody has gid 99 there (and +# there's not good alternative for testing) +my $loggid = $PgCommon::rpm ? (getgrnam 'adm')[2] : $ownergid; +is $st[5], $loggid, 'log file is owned by user\'s primary group'; + +if ($#MAJORS > 0) { + my $newv = $MAJORS[-1]; + + my $outref; + is ((exec_as 0, "(pg_upgradecluster -v $newv $v main | sed -e 's/^/STDOUT: /')", $outref, 0), 0, + 'pg_upgradecluster succeeds'); + like $$outref, qr/Starting upgraded cluster/, 'pg_upgradecluster reported cluster startup'; + like $$outref, qr/Success. Please check/, 'pg_upgradecluster reported successful operation'; + my @err = grep (!/^STDOUT: /, split (/\n/, $$outref)); + if (@err) { + fail 'no error messages during upgrade'; + print (join ("\n", @err)); + } else { + pass "no error messages during upgrade"; + } + + # verify file permissions + @st = stat "/etc/postgresql/$newv/main"; + is $st[4], $owneruid, 'upgraded conf dir is owned by user'; + is $st[5], $ownergid, 'upgraded conf dir is owned by user\'s primary group'; + @st = stat "/etc/postgresql/$newv/main/postgresql.conf"; + is $st[4], $owneruid, 'upgraded postgresql.conf dir is owned by user'; + is $st[5], $ownergid, 'upgraded postgresql.conf dir is owned by user\'s primary group'; + @st = stat "/var/log/postgresql/postgresql-$v-main.log"; + is $st[4], $owneruid, 'upgraded log file is owned by user'; + is $st[5], $loggid, 'upgraded log file is owned by user\'s primary group'; + + is ((system "pg_dropcluster $newv main --stop"), 0, 'pg_dropcluster'); +} else { + pass 'only one major version installed, skipping upgrade test'; + for (my $i = 0; $i < 10; ++$i) { + pass '...'; + } +} + +# Check proper cleanup +is ((system "pg_dropcluster $v main --stop"), 0, 'pg_dropcluster'); +is_program_out $owner, 'pg_lsclusters -h', 0, '', 'No clusters left'; +is ((ps 'postgres'), '', "No postgres processes left"); + +check_clean; + +# vim: filetype=perl diff --git a/t/080_start.conf.t b/t/080_start.conf.t new file mode 100644 index 0000000..2e4d4cf --- /dev/null +++ b/t/080_start.conf.t @@ -0,0 +1,145 @@ +# Check start.conf handling. + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More tests => 73; + +my $systemd = -d '/run/systemd/system'; + +# Do test with oldest version +my $v = $MAJORS[0]; + +# create cluster +is ((system "pg_createcluster $v main >/dev/null"), 0, "pg_createcluster $v main"); + +# Check that we start with 'auto' +note "start.conf auto"; +is ((get_cluster_start_conf $v, 'main'), 'auto', + 'get_cluster_start_conf returns auto'); +is_program_out 'nobody', "grep '^[^\\s#]' /etc/postgresql/$v/main/start.conf", + 0, "auto\n", 'start.conf contains auto'; +SKIP: { + skip 'not running under systemd', 2 unless ($systemd); + ok_dir '/run/systemd/generator/postgresql.service.wants', + ["postgresql\@$v-main.service"], + "systemd generator links cluster"; + is ((readlink "/run/systemd/generator/postgresql.service.wants/postgresql\@$v-main.service"), + "/lib/systemd/system/postgresql@.service", + "systemd generator links correct service file"); +} + +# init script should handle auto cluster +like_program_out 0, "/etc/init.d/postgresql start $v", 0, qr/Start.*($v|systemctl)/; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/online/, 'cluster is online'; +like_program_out 0, "/etc/init.d/postgresql stop $v", 0, qr/Stop.*($v|systemctl)/; +sleep 3 if ($systemd); # FIXME: systemctl stop postgresql is not yet synchronous (#759725) +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/down/, 'cluster is down'; + +# change to manual, verify start.conf contents +note "start.conf manual"; +set_cluster_start_conf $v, 'main', 'manual'; + +is ((get_cluster_start_conf $v, 'main'), 'manual', + 'get_cluster_start_conf returns manual'); +is_program_out 'nobody', "grep '^[^\\s#]' /etc/postgresql/$v/main/start.conf", + 0, "manual\n", 'start.conf contains manual'; +SKIP: { + skip 'not running under systemd', 1 unless ($systemd); + system "systemctl daemon-reload"; + ok_dir '/run/systemd/generator/postgresql.service.wants', + [], "systemd generator doesn't link cluster"; +} + +# init script should not handle manual cluster ... +like_program_out 0, "/etc/init.d/postgresql start $v", 0, qr/Start.*($v|systemctl)/; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/down/, 'cluster is down'; + +# pg_ctlcluster should handle manual cluster +is_program_out 'postgres', "pg_ctlcluster $v main start", 0, ''; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/online/, 'cluster is online'; +is_program_out 'postgres', "pg_ctlcluster $v main stop", 0, ''; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/down/, 'cluster is down'; + +# change to disabled, verify start.conf contents +note "start.conf disabled"; +set_cluster_start_conf $v, 'main', 'disabled'; + +is ((get_cluster_start_conf $v, 'main'), 'disabled', + 'get_cluster_start_conf returns disabled'); +SKIP: { + skip 'not running under systemd', 1 unless ($systemd); + system "systemctl daemon-reload"; + ok_dir '/run/systemd/generator/postgresql.service.wants', + [], "systemd generator doesn't link cluster"; +} + +# init script should not handle disabled cluster +like_program_out 0, "/etc/init.d/postgresql start $v", 0, qr/Start.*($v|systemctl)/; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/down/, 'cluster is down'; + +# pg_ctlcluster should not start disabled cluster +is_program_out 'postgres', "pg_ctlcluster $v main start", 1, + "Error: Cluster is disabled\n"; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/down/, 'cluster is down'; + +# change back to manual, start cluster +set_cluster_start_conf $v, 'main', 'manual'; +is_program_out 'postgres', "pg_ctlcluster $v main start", 0, ''; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/online/, 'cluster is online'; + +# however, we want to stop disabled clusters +set_cluster_start_conf $v, 'main', 'disabled'; +is_program_out 'postgres', "pg_ctlcluster $v main stop", 0, ''; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/down/, 'cluster is down'; + +# set back to manual +set_cluster_start_conf $v, 'main', 'manual'; +is_program_out 'postgres', "pg_ctlcluster $v main start", 0, ''; +like_program_out 'postgres', 'pg_lsclusters -h', 0, qr/online/, 'cluster is online'; + +# upgrade cluster +note "test upgrade"; +if ($#MAJORS == 0) { + pass 'only one major version installed, skipping upgrade test'; + pass '...'; +} else { + like_program_out 0, "pg_upgradecluster -v $MAJORS[-1] $v main", 0, qr/Success. Please check/; +} + +# check start.conf of old and upgraded cluster +is ((get_cluster_start_conf $v, 'main'), 'manual', + 'get_cluster_start_conf for old cluster returns manual'); +is ((get_cluster_start_conf $MAJORS[-1], 'main'), 'manual', + 'get_cluster_start_conf for new cluster returns manual'); + +# clean up +if ($#MAJORS == 0) { + pass '...'; +} else { + is ((system "pg_dropcluster $v main"), 0, + 'dropping old cluster'); +} + +is ((system "pg_dropcluster $MAJORS[-1] main --stop"), 0, + 'dropping upgraded cluster'); + +is_program_out 'postgres', 'pg_lsclusters -h', 0, '', 'no clusters any more'; + +# create cluster with --start-conf option +is_program_out 0, "pg_createcluster $v main --start-conf foo", 1, + "Error: Invalid --start-conf value: foo\n", + 'pg_createcluster checks --start-conf validity'; +is ((system "pg_createcluster $v main --start-conf manual >/dev/null"), 0, + 'pg_createcluster checks --start-conf manual'); +is ((get_cluster_start_conf $v, 'main'), 'manual', + 'get_cluster_start_conf returns manual'); +is ((system "pg_dropcluster $v main"), 0, + 'dropping cluster'); + +check_clean; + +# vim: filetype=perl diff --git a/t/085_pg_ctl.conf.t b/t/085_pg_ctl.conf.t new file mode 100644 index 0000000..dca2700 --- /dev/null +++ b/t/085_pg_ctl.conf.t @@ -0,0 +1,51 @@ +# Check pg_ctl.conf handling. + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More tests => $MAJORS[-1] >= '8.3' ? 33 : 1; + +# Do test with newest version +my $v = $MAJORS[-1]; +if ($v < '8.3') { + pass 'Skipping core limit tests for versions < 8.3'; + exit 0; +} + +# enable core dumps +# sudo and salsa-ci set the hard limit to 0 by default, undo that +is_program_out 0, "prlimit --core=0:unlimited --pid=$$", 0, '', "set core file size to unlimited"; +is_program_out 'postgres', "sh -c 'ulimit -Hc'", 0, "unlimited\n", "core file size is unlimited"; + +# create cluster +is ((system "pg_createcluster $v main >/dev/null"), 0, "pg_createcluster $v main"); +ok (-f "/etc/postgresql/$v/main/pg_ctl.conf", "/etc/postgresql/$v/main/pg_ctl.conf exists"); + +# Default behaviour, core size=0 +is_program_out 0, "pg_ctlcluster $v main start", 0, '', "starting cluster as root"; +is_program_out 'postgres', "xargs -i awk '/core/ {print \$5}' /proc/{}/limits < /var/run/postgresql/$v-main.pid", 0, "0\n", "soft core size is 0"; +my $hard_limit = `xargs -i awk '/core/ {print \$6}' /proc/{}/limits < /var/run/postgresql/$v-main.pid`; +chomp $hard_limit; +note "hard core file size limit of root-started postgres process is $hard_limit"; + +# -c in pg_ctl.conf, core size=unlimited +ok (set_cluster_pg_ctl_conf($v, 'main', '-c'), "set pg_ctl default option to -c"); +is_program_out 0, "pg_ctlcluster $v main restart", 0, '', "restarting cluster as root"; +is_program_out 'postgres', "xargs -i awk '/core/ {print \$5}' /proc/{}/limits < /var/run/postgresql/$v-main.pid", 0, "$hard_limit\n", "soft core size is $hard_limit"; + +# Back to default behaviour, core size=0 +is_program_out 0, "pg_ctlcluster $v main stop", 0, '', "stopping cluster"; +ok (set_cluster_pg_ctl_conf($v, 'main', ''), "restored pg_ctl default option"); + +# pg_ctl -c, core size=unlimited +is_program_out 'postgres', "pg_ctlcluster $v main start -- -c", 0, '', "starting cluster with -c on the command line as postgres"; +is_program_out 'postgres', "xargs -i awk '/core/ {print \$5}' /proc/{}/limits < /var/run/postgresql/$v-main.pid", 0, "unlimited\n", "soft core size is unlimited"; +is_program_out 'postgres', "pg_ctlcluster $v main stop", 0, '', "stopping cluster"; + +is ((system "pg_dropcluster $v main --stop"), 0, 'dropping cluster'); +check_clean; + +# vim: filetype=perl diff --git a/t/090_multicluster.t b/t/090_multicluster.t new file mode 100644 index 0000000..8322ace --- /dev/null +++ b/t/090_multicluster.t @@ -0,0 +1,286 @@ +# Check operation with multiple clusters + +use strict; + +use lib 't'; +use TestLib; +use Socket; +use PgCommon; + +use Test::More tests => 125; + +# create fake socket at 5433 to verify that this port is skipped +socket (SOCK, PF_INET, SOCK_STREAM, getprotobyname('tcp')) or die "socket: $!"; +setsockopt(SOCK, Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1) or die "setsockopt: $!"; +bind (SOCK, sockaddr_in(5433, INADDR_ANY)) || die "bind: $! "; +listen (SOCK, 0) or die "listen: $!"; + +# create clusters +is ((system "pg_createcluster $MAJORS[0] old -- -A trust >/dev/null"), 0, "pg_createcluster $MAJORS[0] old"); +is ((system "pg_createcluster $MAJORS[-1] new1 -- -A trust >/dev/null"), 0, "pg_createcluster $MAJORS[-1] new1"); +is ((system "pg_createcluster $MAJORS[-1] new2 -p 5440 -- -A trust >/dev/null"), 0, "pg_createcluster $MAJORS[-1] new2"); +close SOCK; + +my $old = "$MAJORS[0]/old"; +my $new1 = "$MAJORS[-1]/new1"; +my $new2 = "$MAJORS[-1]/new2"; + +is ((system "pg_ctlcluster $MAJORS[0] old start >/dev/null"), 0, "starting cluster $old"); +is ((system "pg_ctlcluster $MAJORS[-1] new1 start >/dev/null"), 0, "starting cluster $new1"); +is ((system "pg_ctlcluster $MAJORS[-1] new2 start >/dev/null"), 0, "starting cluster $new2"); + +like_program_out 'postgres', 'pg_lsclusters -h | sort -k3', 0, qr/.*5432.*5434.*5440.*/s, + 'clusters have the correct ports, skipping used 5433'; + +# move user_clusters aside for the test; this will ensure that client programs +# work correctly without any file at all +if (-f '/etc/postgresql-common/user_clusters') { + ok ((rename '/etc/postgresql-common/user_clusters', + '/etc/postgresql-common/user_clusters.psqltestsuite'), + 'Temporarily moving away /etc/postgresql-common/user_clusters'); +} else { + pass '/etc/postgresql-common/user_clusters does not exist'; +} + +# check basic cluster selection +like_program_out 0, 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $MAJORS[0]/, + 'pg_wrapper selects port 5432 as default cluster'; +like_program_out 0, "createdb --cluster $new1 --version", 0, + qr/^createdb \(PostgreSQL\) $MAJORS[-1]/, + 'pg_wrapper --cluster works'; +like_program_out 0, "createdb --cluster $MAJORS[-1]/foo --version", 1, + qr/Cluster .* does not exist/, + 'pg_wrapper --cluster errors out for invalid cluster'; + +# create a database in new1 and check that it doesn't appear in new2 +is_program_out 'postgres', "createdb --cluster $new1 test", 0, ($MAJORS[-1] < 8.3 ? "CREATE DATABASE\n" : ''); +like_program_out 'postgres', "psql -Atl --cluster $new1", 0, + qr/test\|postgres\|/, + 'test db appears in cluster new1'; +unlike_program_out 'postgres', "psql -Atl --cluster $new2", 0, + qr/test\|postgres\|/, + 'test db does not appear in cluster new2'; +unlike_program_out 'postgres', "psql -Atl", 0, qr/test\|postgres\|/, + 'test db does not appear in default cluster'; + +# check network cluster selection +is_program_out 'postgres', "psql --cluster $MAJORS[0]/127.0.0.1: -Atc 'show port' template1", 0, "5432\n", + "psql --cluster $MAJORS[0]/127.0.0.1: defaults to port 5432"; +like_program_out 'postgres', "createdb --cluster $MAJORS[-1]/127.0.0.1:5432 --version", 0, + qr/^createdb \(PostgreSQL\) $MAJORS[-1]/, + "createdb --cluster $MAJORS[-1]/127.0.0.1:5432 uses latest client version"; +like_program_out 'postgres', "psql -Atl --cluster $MAJORS[-1]/localhost:5434", 0, + qr/test\|postgres\|/, "test db appears in cluster $MAJORS[-1]/localhost:5434"; +unlike_program_out 'postgres', "psql -Atl --cluster $MAJORS[-1]/localhost:5440", 0, + qr/test\|postgres\|/, "test db does not appear in cluster $MAJORS[-1]/localhost:5440"; + +# check some erroneous cluster specifications +like_program_out 'postgres', "LC_MESSAGES=C psql -Atl --cluster $MAJORS[-1]/localhost:5435", 2, + qr/could not connect|connection to server .* failed/, "psql --cluster $MAJORS[-1]/localhost:5435 fails due to nonexisting port"; +like_program_out 'postgres', "LC_MESSAGES=C psql -Atl --cluster $MAJORS[-1]/localhost:a", 1, + qr/Cluster .* does not exist/, "psql --cluster $MAJORS[-1]/localhost:a fails due to invalid syntax"; +like_program_out 'postgres', "LC_MESSAGES=C psql -Atl --cluster $MAJORS[-1]/doesnotexi.st", 1, + qr/Cluster .* does not exist/, "psql --cluster $MAJORS[-1]/doesnotexi.st fails due to invalid syntax"; +like_program_out 'postgres', "psql -Atl --cluster 6.4/localhost:", 1, + qr/Invalid version/, "psql --cluster 6.4/localhost: fails due to invalid version"; + +# check that environment variables work +$ENV{'PGCLUSTER'} = $new1; +like_program_out 'postgres', "psql -Atl", 0, qr/test\|postgres\|/, + 'PGCLUSTER selection (1)'; +$ENV{'PGCLUSTER'} = $new2; +unlike_program_out 'postgres', "psql -Atl", 0, qr/test\|postgres\|/, + 'PGCLUSTER selection (2)'; +$ENV{'PGCLUSTER'} = 'foo'; +like_program_out 'postgres', "psql -l", 1, + qr/Invalid version .* specified in PGCLUSTER/, + 'invalid PGCLUSTER value'; +$ENV{'PGCLUSTER'} = "$MAJORS[-1]/127.0.0.1:"; +like_program_out 0, 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $MAJORS[-1]/, + 'PGCLUSTER network cluster selection (1)'; +$ENV{'PGCLUSTER'} = "$MAJORS[-1]/localhost:5434"; +like_program_out 'postgres', 'psql -Atl', 0, + qr/test\|postgres\|/, 'PGCLUSTER network cluster selection (2)'; +$ENV{'PGCLUSTER'} = "$MAJORS[-1]/localhost:5440"; +unlike_program_out 'postgres', 'psql -Atl', 0, + qr/test\|postgres\|/, 'PGCLUSTER network cluster selection (3)'; +$ENV{'PGCLUSTER'} = "$MAJORS[-1]/localhost:5435"; +like_program_out 'postgres', 'LC_MESSAGES=C psql -Atl', 2, + qr/could not connect|connection to server .* failed/, "psql --cluster $MAJORS[-1]/localhost:5435 fails due to nonexisting port"; +delete $ENV{'PGCLUSTER'}; + +# check that PGPORT works +$ENV{'PGPORT'} = '5434'; +is_program_out 'postgres', 'psql -Atc "show port" template1', 0, "5434\n", + 'PGPORT selection (1)'; +$ENV{'PGPORT'} = '5432'; +is_program_out 'postgres', 'psql -Atc "show port" template1', 0, "5432\n", + 'PGPORT selection (2)'; +$ENV{'PGCLUSTER'} = $new2; +delete $ENV{'PGPORT'}; +$ENV{'PGPORT'} = '5432'; +like_program_out 'postgres', 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $MAJORS[-1]/, + 'PGPORT+PGCLUSTER, PGCLUSTER selects version'; +is_program_out 'postgres', 'psql -Atc "show port" template1', 0, "5432\n", + 'PGPORT+PGCLUSTER, PGPORT selects port'; +delete $ENV{'PGPORT'}; +delete $ENV{'PGCLUSTER'}; + +# check that PGDATABASE works +$ENV{'PGDATABASE'} = 'test'; +is_program_out 'postgres', "psql --cluster $new1 -Atc 'select current_database()'", 0, "test\n", + 'PGDATABASE environment variable works'; +delete $ENV{'PGDATABASE'}; + +# check cluster selection with an empty user_clusters +open F, '>/etc/postgresql-common/user_clusters' or die "Could not create user_clusters: $!"; +close F; +chmod 0644, '/etc/postgresql-common/user_clusters'; +like_program_out 0, 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $MAJORS[0]/, + 'pg_wrapper selects port 5432 as default cluster with empty user_clusters'; +like_program_out 0, "createdb --cluster $new1 --version", 0, + qr/^createdb \(PostgreSQL\) $MAJORS[-1]/, + 'pg_wrapper --cluster works with empty user_clusters'; + +# check default cluster selection with user_clusters +open F, '>/etc/postgresql-common/user_clusters' or die "Could not create user_clusters: $!"; +print F "* * $MAJORS[-1] new1 *\n"; +close F; +chmod 0644, '/etc/postgresql-common/user_clusters'; +like_program_out 'postgres', 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $MAJORS[-1]/, + "pg_wrapper selects correct cluster with user_clusters '* * $MAJORS[-1] new1 *'"; + +# check default database selection with user_clusters +open F, '>/etc/postgresql-common/user_clusters' or die "Could not create user_clusters: $!"; +print F "* * $MAJORS[-1] new1 test\n"; +close F; +chmod 0644, '/etc/postgresql-common/user_clusters'; +is_program_out 'postgres', 'psql -Atc "select current_database()"', 0, "test\n", + "pg_wrapper selects correct database with user_clusters '* * $MAJORS[-1] new1 test'"; +$ENV{'PGDATABASE'} = 'template1'; +is_program_out 'postgres', "psql -Atc 'select current_database()'", 0, "template1\n", + 'PGDATABASE environment variable is not overridden by user_clusters'; +delete $ENV{'PGDATABASE'}; + +# check by-user cluster selection with user_clusters +# (also check invalid cluster reporting) +open F, '>/etc/postgresql-common/user_clusters' or die "Could not create user_clusters: $!"; +print F "postgres * $MAJORS[-1] new1 *\nnobody * $MAJORS[0] old *\n* * 5.5 * *"; +close F; +chmod 0644, '/etc/postgresql-common/user_clusters'; +like_program_out 'postgres', 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $MAJORS[-1]/, + 'pg_wrapper selects correct cluster with per-user user_clusters'; +like_program_out 'nobody', 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $MAJORS[0]/, + 'pg_wrapper selects correct cluster with per-user user_clusters'; +like_program_out 0, 'createdb --version', 0, qr/user_clusters.*line 3.*version.*not exist/i, + 'pg_wrapper warning for invalid per-user user_clusters line'; + +# check by-user network cluster selection with user_clusters +# (also check invalid cluster reporting) +open F, '>/etc/postgresql-common/user_clusters' or die "Could not create user_clusters: $!"; +print F "postgres * $MAJORS[0] localhost: *\nnobody * $MAJORS[-1] new1 *\n* * $MAJORS[-1] localhost:a *"; +close F; +chmod 0644, '/etc/postgresql-common/user_clusters'; +like_program_out 'postgres', 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $MAJORS[0]/, + 'pg_wrapper selects correct version with per-user user_clusters'; +like_program_out 'nobody', 'createdb --version', 0, qr/^createdb \(PostgreSQL\) $MAJORS[-1]/, + 'pg_wrapper selects correct version with per-user user_clusters'; +like_program_out 0, 'createdb --version', 0, qr/user_clusters.*line 3.*cluster.*not exist/i, + 'pg_wrapper warning for invalid per-user user_clusters line'; +# check PGHOST environment variable precedence +$ENV{'PGHOST'} = '127.0.0.2'; +like_program_out 'postgres', 'psql -Atl', 2, qr/127.0.0.2/, '$PGHOST overrides user_clusters'; +is_program_out 'postgres', "psql --cluster $MAJORS[-1]/localhost:5434 -Atc 'select current_database()' test", + 0, "test\n", '--cluster overrides $PGHOST'; +delete $ENV{'PGHOST'}; + +# check invalid user_clusters +open F, '>/etc/postgresql-common/user_clusters' or die "Could not create user_clusters: $!"; +print F 'foo'; +close F; +chmod 0644, '/etc/postgresql-common/user_clusters'; +like_program_out 'postgres', 'createdb --version', 0, qr/ignoring invalid line 1/, + 'pg_wrapper ignores invalid lines in user_clusters'; + +# remove test user_clusters +unlink '/etc/postgresql-common/user_clusters' or die + "unlink user_clusters: $!"; + +# check that pg_service.conf works +open F, '>/etc/postgresql-common/pg_service.conf' or die "Could not create pg_service.conf: $!"; +print F "[old_t1] +user=postgres +dbname=template1 +port=5432 + +[new1_test] +user=postgres +dbname=test +port=5434 + +# these do not exist +[new2_test] +user=postgres +dbname=test +port=5440 +"; +close F; +chmod 0644, '/etc/postgresql-common/pg_service.conf'; + +$ENV{'PGSERVICE'} = 'old_t1'; +is_program_out 'postgres', "psql -Atc 'select current_database()'", 0, + "template1\n", 'pg_service conf selection 1'; +$ENV{'PGSERVICE'} = 'new1_test'; +is_program_out 'postgres', "psql -Atc 'select current_database()'", 0, + "test\n", 'pg_service conf selection 2'; +$ENV{'PGSERVICE'} = 'new2_test'; +like_program_out 'postgres', "psql -Atc 'select current_database()'", 2, + qr/FATAL.*test/, 'pg_service conf selection 3'; +delete $ENV{'PGSERVICE'}; +unlink '/etc/postgresql-common/pg_service.conf'; + +# check proper error message if no cluster could be determined as default for +# pg_wrapper +is ((system "pg_ctlcluster $MAJORS[0] old stop >/dev/null"), 0, "stopping cluster $old"); +PgCommon::set_conf_value $MAJORS[0], 'old', 'postgresql.conf', 'port', '5435'; +is ((system "pg_ctlcluster $MAJORS[0] old start >/dev/null"), 0, "restarting cluster $old"); +like_program_out 'postgres', 'pg_lsclusters -h | sort -k3', 0, qr/.*5434.*5435.*5440.*/s, + 'port of first cluster was successfully changed'; +like_program_out 'postgres', "psql -l", 2, + qr/no.*default.*man pg_wrapper.*psql:.*\.s\.PGSQL.5432/is, + 'proper pg_wrapper warning and psql error if no cluster is suitable as default target'; +like_program_out 'postgres', "psql -Atl --cluster $new1", 0, + qr/test\|postgres\|/, + '--cluster selects appropriate cluster'; +like_program_out 'postgres', "psql -Atl -p 5434", 0, + qr/test\|postgres\|/, + '-p selects appropriate cluster'; +like_program_out 'postgres', "psql -Atlp 5434", 0, + qr/test\|postgres\|/, + '-Atlp selects appropriate cluster'; +like_program_out 'postgres', "psql -Atl --port 5434", 0, + qr/test\|postgres\|/, + '--port selects appropriate cluster'; +like_program_out 'postgres', "env PGPORT=5434 psql -Atl", 0, + qr/test\|postgres\|/, + '$PGPORT selects appropriate cluster'; + +# but specifying -p explicitly should work + +# restore original user_clusters +if (-f '/etc/postgresql-common/user_clusters.psqltestsuite') { + ok ((rename '/etc/postgresql-common/user_clusters.psqltestsuite', + '/etc/postgresql-common/user_clusters'), + 'Restoring original /etc/postgresql-common/user_clusters'); +} else { + pass '/etc/postgresql-common/user_clusters did not exist, not restoring'; +} + +# clean up +is ((system "pg_dropcluster $MAJORS[-1] new1 --stop"), 0, "dropping $new1"); +is ((system "pg_dropcluster $MAJORS[-1] new2 --stop"), 0, "dropping $new2"); +is ((system "pg_dropcluster $MAJORS[0] old --stop"), 0, "dropping $old"); + +check_clean; + +# vim: filetype=perl diff --git a/t/110_integrate_cluster.t b/t/110_integrate_cluster.t new file mode 100644 index 0000000..4201a58 --- /dev/null +++ b/t/110_integrate_cluster.t @@ -0,0 +1,44 @@ +# Check integration of an already existing cluster + +use strict; + +use lib 't'; +use TestLib; +use File::Temp qw/tempdir/; +use Time::HiRes qw(usleep); + +my $version = $MAJORS[-1]; + +use Test::More tests => 32; +use PgCommon; + +delete $ENV{'LANG'}; +delete $ENV{'LANGUAGE'}; +$ENV{'LC_ALL'} = 'C'; + +my $wdir = tempdir (CLEANUP => 1); +chmod 0755, $wdir or die "Could not chmod $wdir: $!"; + +# create clusters for different owners and check their integration +for my $o ('postgres', 'nobody') { + my $cdir = "$wdir/c"; + mkdir $cdir; + my $oid = getpwnam $o; + chown $oid, 0, $cdir or die "Could not chown $cdir to $oid: $!"; + like_program_out $o, "$PgCommon::binroot$version/bin/initdb $cdir/$o", + 0, qr/Success/, "creating raw initdb cluster for user $o"; + like_program_out 0, "pg_createcluster $version $o -d $cdir/$o", 0, + qr/Configuring already existing cluster/i, "integrating $o cluster"; + like_program_out 0, "pg_lsclusters", 0, + qr/$version\s+$o\s+5432\s+down\s+$o\s/, 'correct pg_lsclusters output'; + is_program_out $o, "pg_ctlcluster $version $o start", 0, '', "starting cluster $o"; + like_program_out 0, "pg_lsclusters", 0, + qr/$version\s+$o\s+5432\s+online\s+$o\s/, 'correct pg_lsclusters output'; + is ((system "pg_dropcluster $version $o --stop"), 0, "dropping cluster $o"); + ok_dir $cdir, [], 'No files in temporary cluster dir left behind'; + rmdir $cdir; +} + +check_clean; + +# vim: filetype=perl diff --git a/t/120_pg_upgradecluster_scripts.t b/t/120_pg_upgradecluster_scripts.t new file mode 100644 index 0000000..d31e52c --- /dev/null +++ b/t/120_pg_upgradecluster_scripts.t @@ -0,0 +1,114 @@ +# Check /etc/p-c/pg_upgradecluster.d/ scripts and proper handling of already +# existing tables in the target cluster. + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More tests => (@MAJORS == 1) ? 1 : 31; + +if (@MAJORS == 1) { + pass 'only one major version installed, skipping upgrade tests'; + exit 0; +} + + +# create old cluster +is ((system "pg_createcluster $MAJORS[0] main --start >/dev/null"), 0, "pg_createcluster $MAJORS[0] main"); + +# add data table, auxtable with 'old...' values, and an unrelated auxtable in +# another schema +is_program_out 'postgres', + 'psql template1 -qc "create table auxdata (x varchar(10)); insert into auxdata values (\'old1\'); insert into auxdata values (\'old2\')"', + 0, '', 'adding auxdata to template1 and fill in some "old..." values'; +is_program_out 'postgres', "createdb test", 0, ''; +is_program_out 'postgres', 'psql test -qc "create table userdata(x int); insert into userdata values(42); insert into userdata values(256)"', + 0, '', 'creating userdata table'; +is_program_out 'postgres', + 'psql test -qc "create schema s; create table s.auxdata (x varchar(10)); insert into s.auxdata values (\'schema1\')"', + 0, '', 'adding schema s and s.auxdata to test and fill in some values'; + +if (not -d '/etc/postgresql-common/pg_upgradecluster.d') { + mkdir '/etc/postgresql-common/pg_upgradecluster.d' or die "mkdir: $!"; +} + +# move existing files away +for my $f (glob("/etc/postgresql-common/pg_upgradecluster.d/*")) { + next if ($f =~ /\.disabled$/); + rename $f, "$f.disabled"; +} + +# create test scripts +chmod 0755, '/etc/postgresql-common/pg_upgradecluster.d' or die "chmod: $!"; +open F, '>/etc/postgresql-common/pg_upgradecluster.d/auxdata' or die "open: $!"; +print F < +oldver=\$1 +cluster=\$2 +newver=\$3 +phase=\$4 + +if [ \$phase = init ]; then + createdb --cluster \$newver/\$cluster idb +fi + +if [ \$phase = finish ]; then + psql --cluster \$newver/\$cluster template1 </etc/postgresql-common/pg_upgradecluster.d/badscript' or die "open: $!"; +print F < 22; +use PgCommon; + +my $testuser = 'postgres'; + +# pg_createcluster and pg_ctlcluster +is ((exec_as $testuser, "pg_createcluster $version main --start"), 0, + "pg_createcluster succeeds as user $testuser with appropriate owner permissions"); + +like_program_out $testuser, 'pg_lsclusters -h', 0, qr/^$version\s+main.*online/m; +like_program_out 'postgres', 'psql -Atl', 0, qr/template1.*UTF8/; + +# pg_dropcluster +is ((exec_as $testuser, "pg_dropcluster $version main --stop"), 0, + "pg_dropcluster succeeds as user $testuser with appropriate directory owner permissions"); + +# pg_upgradecluster +SKIP: { + skip 'Only one major version installed, skipping pg_upgradecluster tests', 8 if ($oldversion eq $version); + + is ((exec_as $testuser, "pg_createcluster $oldversion main --start"), 0, + "pg_createcluster succeeds as user $testuser with appropriate group permissions"); + my $outref; + is ((exec_as $testuser, "pg_upgradecluster -v $version $oldversion main", $outref, 0), 0, + "pg_upgradecluster succeeds as user $testuser"); + like $$outref, qr/Starting upgraded cluster/, 'pg_upgradecluster reported cluster startup'; + like $$outref, qr/Success. Please check/, 'pg_upgradecluster reported successful operation'; + + like_program_out $testuser, 'pg_lsclusters -h', 0, + qr/^$oldversion\s+main.*down.*\n^$version\s+main.*online/m; + + # clean up + is ((exec_as $testuser, "pg_dropcluster $oldversion main"), 0); + is ((exec_as $testuser, "pg_dropcluster $version main --stop"), 0); +} + +check_clean; + +# vim: filetype=perl diff --git a/t/135_pg_virtualenv.t b/t/135_pg_virtualenv.t new file mode 100644 index 0000000..1662e5b --- /dev/null +++ b/t/135_pg_virtualenv.t @@ -0,0 +1,35 @@ +# check if pg_virtualenv runs ok, even under fakeroot + +use strict; +use warnings; + +use lib 't'; +use TestLib; + +use Test::More tests => 12 * @MAJORS + 8; + +foreach my $v (@MAJORS) { + my $args = 'sh -c \'echo "id|$(id -un)"; psql -AtXxc "SELECT current_user"\''; + my $virtualenv = "pg_virtualenv -v $v $args"; + + $ENV{USER} = 'root'; + like_program_out 'root', $virtualenv, 0, qr!id.root\ncurrent_user.postgres!, "running pg_virtualenv as root"; + $ENV{USER} = 'postgres'; + like_program_out 'postgres', $virtualenv, 0, qr!id.postgres\ncurrent_user.postgres!, "running pg_virtualenv as postgres"; + $ENV{USER} = 'nobody'; + like_program_out 'nobody', $virtualenv, 0, qr!id.nobody\ncurrent_user.nobody!, "running pg_virtualenv as nobody"; + + SKIP: { + skip "/usr/bin/fakeroot not available", 6 unless (-x "/usr/bin/fakeroot"); # CentOS doesn't have fakeroot + $ENV{USER} = 'root'; + like_program_out 'root', "fakeroot $virtualenv", 0, qr!id.root\ncurrent_user.postgres!, "running fakeroot pg_virtualenv as root"; + $ENV{USER} = 'postgres'; + like_program_out 'postgres', "fakeroot $virtualenv", 0, qr!id.root\ncurrent_user.postgres!, "running fakeroot pg_virtualenv as postgres"; + $ENV{USER} = 'nobody'; + like_program_out 'nobody', "fakeroot $virtualenv", 0, qr!id.root\ncurrent_user.nobody!, "running fakeroot pg_virtualenv as nobody"; + } +} + +check_clean; + +# vim: filetype=perl diff --git a/t/140_pg_config.t b/t/140_pg_config.t new file mode 100644 index 0000000..770c75c --- /dev/null +++ b/t/140_pg_config.t @@ -0,0 +1,89 @@ +# Check pg_config output + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More tests => 14 * @MAJORS + ($PgCommon::rpm ? 1 : 2) * 12; + +my $multiarch = ''; +unless ($PgCommon::rpm) { + $multiarch = `dpkg-architecture -qDEB_HOST_MULTIARCH 2>/dev/null`; + chomp $multiarch; +} +note "Multiarch is " . ($multiarch ? 'enabled' : 'disabled'); + +my $version; +foreach $version (@MAJORS) { + note "checking version specific output for $version"; + if ($version < '8.2') { + pass "Skipping known-broken pg_config check for version $version"; + for (my $i = 0; $i < 13; ++$i) { pass '...'; } + next; + } + is_program_out 'postgres', "$PgCommon::binroot$version/bin/pg_config --pgxs", 0, + "$PgCommon::binroot$version/lib/pgxs/src/makefiles/pgxs.mk\n"; + my $libdir = "/usr/lib" . ($version >= 9.3 and $multiarch ? "/$multiarch" : "") . "\n"; + $libdir = "$PgCommon::binroot$version/lib\n" if ($PgCommon::rpm); + is_program_out 'postgres', "$PgCommon::binroot$version/bin/pg_config --libdir", 0, + $libdir; + is_program_out 'postgres', "$PgCommon::binroot$version/bin/pg_config --pkglibdir", 0, + "$PgCommon::binroot$version/lib\n"; + is_program_out 'postgres', "$PgCommon::binroot$version/bin/pg_config --bindir", 0, + "$PgCommon::binroot$version/bin\n"; + # mkdir should be in /bin on Debian. If /bin was linked to /usr/bin at build time, usrmerge was installed + SKIP: { + skip 'MKDIR_P not present before 9.0', 2 if ($version < 9.0); + skip 'MKDIR_P not checked on RedHat', 2 if ($PgCommon::rpm); # varies across builds/versions + is_program_out 'postgres', "grep ^MKDIR_P $PgCommon::binroot$version/lib/pgxs/src/Makefile.global", 0, + "MKDIR_P = /bin/mkdir -p\n"; + } + SKIP: { + skip 'build path not canonicalized on RedHat', 4 if ($PgCommon::rpm); + my $pkgversion = `dpkg-query -f '\${Version}' -W postgresql-server-dev-$version`; + # check that we correctly canonicalized the build paths + SKIP: { + skip 'abs_top_builddir introduced in 9.5', 2 if ($version < 9.5); + skip 'abs_top_builddir not patched in Debian (old)stable', 2 if ($version < 10 and $pkgversion !~ /pgdg/); + is_program_out 'postgres', "grep ^abs_top_builddir $PgCommon::binroot$version/lib/pgxs/src/Makefile.global", 0, + "abs_top_builddir = /build/postgresql-$version$ENV{PG_FLAVOR}/build\n"; + } + SKIP: { + skip 'abs_top_srcdir not patched before 9.3', 2 if ($version < 9.3); + skip 'abs_top_srcdir not patched in Debian (old)stable', 2 if ($version < 10 and $pkgversion !~ /pgdg/); + is_program_out 'postgres', "grep ^abs_top_srcdir $PgCommon::binroot$version/lib/pgxs/src/Makefile.global", 0, + "abs_top_srcdir = /build/postgresql-$version$ENV{PG_FLAVOR}/build/..\n"; + } + } +} + +my @pg_configs = $PgCommon::rpm ? qw(pg_config) : qw(pg_config pg_config.libpq-dev); +for my $pg_config (@pg_configs) { + if ($pg_config eq 'pg_config' or $PgCommon::rpm) { # pg_config should point at newest installed postgresql-server-dev-$version + $version = $ALL_MAJORS[-1]; + } else { # pg_config.libpq-dev should point at postgresql-server-dev-$(version of libpq-dev) + my $libpqdev_version = `dpkg-query --showformat '\${Version}' --show libpq-dev`; + $libpqdev_version =~ /^([89].\d|1.)/ or die "could not determine libpq-dev version"; + $version = $1; + } + note "checking $pg_config output (should behave like version $version)"; + + SKIP: { + my $pgc = "$PgCommon::binroot$version/bin/pg_config"; + skip "$pgc not installed, can't check full $pg_config output", 2 unless (-x $pgc); + my $full_output = `$pgc`; + is_program_out 'postgres', "$pg_config", 0, $full_output; + } + like_program_out 'postgres', "$pg_config --help", 0, qr/--includedir-server/; + is_program_out 'postgres', "$pg_config --pgxs", 0, + "$PgCommon::binroot$version/lib/pgxs/src/makefiles/pgxs.mk\n"; + my $libdir = "/usr/lib" . ($version >= 9.3 and $multiarch ? "/$multiarch" : "") . "\n"; + $libdir = "$PgCommon::binroot$version/lib\n" if ($PgCommon::rpm); + is_program_out 'postgres', "$pg_config --libdir", 0, + $libdir; + is_program_out 'postgres', "$pg_config --pkglibdir", 0, + "$PgCommon::binroot$version/lib\n"; + is_program_out 'postgres', "$pg_config --bindir", 0, + "$PgCommon::binroot$version/bin\n"; +} diff --git a/t/150_tsearch_stemming.t b/t/150_tsearch_stemming.t new file mode 100644 index 0000000..a2a6253 --- /dev/null +++ b/t/150_tsearch_stemming.t @@ -0,0 +1,108 @@ +# Check tsearch, and stemming with dynamic creation of .affix/.dict files + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; + +my $version = $MAJORS[-1]; + +use Test::More tests => ($MAJORS[-1] < 8.3 or $PgCommon::rpm) ? 1 : 37; +if ($version < 8.3) { + pass 'tsearch dictionaries not tested before 8.3'; + exit; +} +if ($PgCommon::rpm) { + pass 'tsearch dictionaries not handled by postgresql-common on RedHat'; + exit; +} + +# test pg_updatedicts +unlink '/var/cache/postgresql/dicts/en_us.affix'; +unlink '/var/cache/postgresql/dicts/en_us.dict'; +unlink "/usr/share/postgresql/$version/tsearch_data/en_us.affix"; +unlink "/usr/share/postgresql/$version/tsearch_data/en_us.dict"; +is ((exec_as 0, 'pg_updatedicts'), 0, 'pg_updatedicts succeeded'); +ok -f '/var/cache/postgresql/dicts/en_us.affix', + 'pg_updatedicts created en_us.affix'; +ok -f '/var/cache/postgresql/dicts/en_us.dict', + 'pg_updatedicts created en_us.dict'; +ok -l "/usr/share/postgresql/$version/tsearch_data/en_us.affix", + "pg_updatedicts created $version en_us.affix symlink"; +ok -l "/usr/share/postgresql/$version/tsearch_data/en_us.dict", + "pg_updatedicts created $version en_us.dict symlink"; + +# create cluster +is ((system "pg_createcluster $version main --start >/dev/null"), 0, "pg_createcluster $version main"); + +# create DB with en_US text search configuration +is_program_out 'postgres', 'createdb fts', 0, ''; + +my $outref; + +is ((exec_as 'postgres', 'psql -qd fts -c " + CREATE TEXT SEARCH CONFIGURATION public.sc_english ( COPY = pg_catalog.english ); + CREATE TEXT SEARCH DICTIONARY english_ispell (TEMPLATE = ispell, DictFile = en_US, + AffFile = en_US, StopWords = english); + SET default_text_search_config = \'public.sc_english\'; + ALTER TEXT SEARCH CONFIGURATION public.sc_english + ALTER MAPPING FOR asciiword WITH english_ispell, english_stem;"', $outref), + 0, 'creating en_US full text search configuration ' . $$outref); + +# create test table and index +my $outref; +is ((exec_as 'postgres', 'psql -qd fts -c " + CREATE TABLE stuff (id SERIAL PRIMARY KEY, text TEXT, textsearch tsvector); + UPDATE stuff SET textsearch = to_tsvector(\'public.sc_english\', coalesce(text, \'\')); + CREATE INDEX textsearch_idx ON stuff USING gin(textsearch); + CREATE TRIGGER textsearch_update_trigger BEFORE INSERT OR UPDATE + ON stuff FOR EACH ROW EXECUTE PROCEDURE + tsvector_update_trigger(textsearch, \'public.sc_english\', text); + INSERT INTO stuff (text) VALUES (\'PostgreSQL rocks\'); + INSERT INTO stuff (text) VALUES (\'Linux rocks\'); + INSERT INTO stuff (text) VALUES (\'I am your father\'\'s nephew\'\'s former roommate\'); + INSERT INTO stuff (text) VALUES (\'3 cafés\'); + "'), 0, 'creating data table and search index'); + +# test stemming +is_program_out 'postgres', + 'psql -Atd fts -c "SELECT dictionary, lexemes FROM ts_debug(\'public.sc_english\', \'friendliest\')"', + 0, "english_ispell|{friendly}\n", 'stem search of correct word'; +is_program_out 'postgres', + 'psql -Atd fts -c "SELECT dictionary, lexemes FROM ts_debug(\'public.sc_english\', \'father\'\'s\')"', + 0, "english_ispell|{father}\n|\nenglish_ispell|{}\n", 'stem search of correct word'; +is_program_out 'postgres', + 'psql -Atd fts -c "SELECT dictionary, lexemes FROM ts_debug(\'public.sc_english\', \'duffles\')"', + 0, "english_stem|{duffl}\n", 'stem search of unknown word'; + +# test searching +is_program_out 'postgres', + 'psql -Atd fts -c "SELECT text FROM stuff, to_tsquery(\'rocks\') query WHERE query @@ to_tsvector(text)"', + 0, "PostgreSQL rocks\nLinux rocks\n", 'full text search, exact word'; + +is_program_out 'postgres', + 'psql -Atd fts -c "SELECT text FROM stuff, to_tsquery(\'rock\') query WHERE query @@ to_tsvector(text)"', + 0, "PostgreSQL rocks\nLinux rocks\n", 'full text search for word stem'; + +is_program_out 'postgres', + 'psql -Atd fts -c "SELECT text FROM stuff, to_tsquery(\'roc\') query WHERE query @@ to_tsvector(text)"', + 0, '', 'full text search for word substring fails'; + +is_program_out 'postgres', + 'psql -Atd fts -c "SELECT text FROM stuff, to_tsquery(\'cafés\') query WHERE query @@ to_tsvector(text)"', + 0, "3 cafés\n", 'full text search, exact unicode word'; + +is_program_out 'postgres', + 'psql -Atd fts -c "SELECT text FROM stuff, to_tsquery(\'café\') query WHERE query @@ to_tsvector(text)"', + 0, "3 cafés\n", 'full text search for unicode word stem'; + +is_program_out 'postgres', + 'psql -Atd fts -c "SELECT text FROM stuff, to_tsquery(\'afé\') query WHERE query @@ to_tsvector(text)"', + 0, '', 'full text search for unicode word substring fails'; + +# clean up +is ((system "pg_dropcluster $version main --stop"), 0); +check_clean; + +# vim: filetype=perl diff --git a/t/160_alternate_confroot.t b/t/160_alternate_confroot.t new file mode 100644 index 0000000..e2a9604 --- /dev/null +++ b/t/160_alternate_confroot.t @@ -0,0 +1,57 @@ +# Check that we can do all operations using a per-user $PG_CLUSTER_CONF_ROOT + +use strict; + +use lib 't'; +use TestLib; + +my $version = $MAJORS[0]; + +use Test::More tests => 28; + +# prepare nobody-owned root dir for $PG_CLUSTER_CONF_ROOT +my $rootdir=`su -s /bin/sh -c 'mktemp -d' nobody`; +chomp $rootdir; +($rootdir) = $rootdir =~ m!^([a-zA-Z0-9._/]+)$!; # untaint +$ENV{'PG_CLUSTER_CONF_ROOT'} = "$rootdir/etc"; + +is ((exec_as 'nobody', "pg_createcluster $version test -d $rootdir/data/test -l $rootdir/test.log --start"), 0); + +is_program_out 'nobody', 'env -u PG_CLUSTER_CONF_ROOT pg_lsclusters -h', 0, ''; +like_program_out 'nobody', "pg_lsclusters -h", 0, + qr!^$version\s+test.*online\s+nobody\s+$rootdir/data/test\s+$rootdir/test.log$!; + +like_program_out 'nobody', "psql -Atl", 0, qr/template1.*UTF8/; + +# pg_upgradecluster +if ($MAJORS[0] ne $MAJORS[-1]) { + my $outref; + is ((exec_as 'nobody', "pg_upgradecluster --logfile $rootdir/testupgr.log -v $MAJORS[-1] $version test $rootdir/data/testupgr", $outref, 0), 0); + like $$outref, qr/Starting upgraded cluster/, 'pg_upgradecluster reported cluster startup'; + like $$outref, qr/Success. Please check/, 'pg_upgradecluster reported successful operation'; + + like_program_out 'nobody', 'pg_lsclusters -h', 0, + qr!^$version\s+test.*down.*\n^$MAJORS[-1]\s+test.*online\s+nobody\s+$rootdir/data/testupgr\s+$rootdir/testupgr.log$!m; + + # clean up + is_program_out 'nobody', "pg_dropcluster $version test", 0, ''; + is_program_out 'nobody', "pg_dropcluster $MAJORS[-1] test --stop", 0, ''; +} else { + pass 'Only one major version installed, skipping pg_upgradecluster tests'; + for (my $i = 0; $i < 6; ++$i) { pass '...'; } + + is_program_out 'nobody', "pg_dropcluster $version test --stop", 0, ''; +} + +# pg_dropcluster +is_program_out 'nobody', "pg_lsclusters -h", 0, ''; + +ok_dir "$rootdir/data", [], 'No files in root/data left behind'; +ok_dir "$rootdir", ['etc', 'data'], 'No cruft in root dir left behind'; + +system "rm -rf $rootdir"; + +delete $ENV{'PG_CLUSTER_CONF_ROOT'}; +check_clean; + +# vim: filetype=perl diff --git a/t/170_extensions.t b/t/170_extensions.t new file mode 100644 index 0000000..1abb330 --- /dev/null +++ b/t/170_extensions.t @@ -0,0 +1,87 @@ +# Check that all extensions install successfully. + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More 0.87; # needs libtest-simple-perl backport on lenny + +foreach my $v (@MAJORS) { +note "Running tests for $v"; + +if ($v < '9.1') { + pass 'No extensions for version < 9.1'; + next; +} + +# create cluster +is ((system "pg_createcluster $v main --start >/dev/null"), 0, "pg_createcluster $v main"); + +# plpgsql is installed by default +is_program_out 'postgres', "psql -Atc 'SELECT extname FROM pg_extension'", 0, "plpgsql\n"; + +my %depends = ( + bool_plperl => [qw(plperl)], + bool_plperlu => [qw(plperlu)], + earthdistance => [qw(cube)], + hstore_plperl => [qw(hstore plperl)], + hstore_plperlu => [qw(hstore plperlu)], + hstore_plpython2u => [qw(hstore plpython2u)], + hstore_plpython3u => [qw(hstore plpython3u)], + hstore_plpythonu => [qw(hstore plpythonu)], + jsonb_plperl => [qw(plperl)], # PG 11 + jsonb_plperlu => [qw(plperlu)], # PG 11 + jsonb_plpython2u => [qw(plpython2u)], # PG 11 + jsonb_plpython3u => [qw(plpython3u)], # PG 11 + jsonb_plpythonu => [qw(plpythonu)], # PG 11 + ltree_plpython2u => [qw(ltree plpython2u)], + ltree_plpython3u => [qw(ltree plpython3u)], + ltree_plpythonu => [qw(ltree plpythonu)], +); + +foreach () { + my ($extname) = $_ =~ /^.*\/(.*)\.control$/; + next if ($extname eq 'plpgsql'); + + if ($depends{$extname}) { + for my $dep (@{$depends{$extname}}) { + is_program_out 'postgres', "psql -qc 'CREATE EXTENSION $dep'", 0, '', + "$extname dependency $dep installs without error"; + } + } + + if ($extname eq 'hstore' && $v eq '9.1') { + # EXFAIL: hstore in 9.1 throws a warning about obsolete => operator + like_program_out 'postgres', "psql -qc 'CREATE EXTENSION \"$extname\"'", 0, + qr/=>/, "extension $extname installs (with warning)"; + } elsif ($extname eq 'chkpass' && $v >= '9.5') { + # chkpass is slightly broken, see + # http://www.postgresql.org/message-id/20141117162116.GA3565@msg.df7cb.de + like_program_out 'postgres', "psql -qc 'CREATE EXTENSION \"$extname\"'", 0, + qr/WARNING: type input function chkpass_in should not be volatile/, + "extension $extname installs (with warning)"; + } else { + is_program_out 'postgres', "psql -qc 'CREATE EXTENSION \"$extname\"'", 0, '', + "extension $extname installs without error"; + } + + is_program_out 'postgres', "psql -qc 'DROP EXTENSION \"$extname\"'", 0, '', + "extension $extname removes without error"; + + if ($depends{$extname}) { + for my $dep (@{$depends{$extname}}) { + is_program_out 'postgres', "psql -qc 'DROP EXTENSION $dep'", 0, '', + "$extname dependency extension $dep removes without error"; + } + } +} + +# clean up +is ((system "pg_dropcluster $v main --stop"), 0, "pg_dropcluster $v main"); +check_clean; +} + +done_testing(); + +# vim: filetype=perl diff --git a/t/180_ecpg.t b/t/180_ecpg.t new file mode 100644 index 0000000..5d3f7a8 --- /dev/null +++ b/t/180_ecpg.t @@ -0,0 +1,56 @@ +# Check that ecpg works + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More tests => 14; + +my $v = $MAJORS[-1]; + +# prepare nobody-owned work dir +my $workdir=`su -s /bin/sh -c 'mktemp -d' nobody`; +chomp $workdir; +chdir $workdir or die "could not chdir to $workdir: $!"; + +# create test code +open F, '>test.pgc' or die "Could not open $workdir/test.pgc: $!"; +print F < +#include + +EXEC SQL WHENEVER SQLWARNING SQLPRINT; +EXEC SQL WHENEVER SQLERROR SQLPRINT; + +EXEC SQL BEGIN DECLARE SECTION; + char output[1024]; +EXEC SQL END DECLARE SECTION; + +int main() { + ECPGdebug(1, stderr); + EXEC SQL CONNECT TO template1; + EXEC SQL SELECT 'Database is ' || current_database() INTO :output; + puts(output); + EXEC SQL DISCONNECT ALL; + return 0; +} +EOF +close F; +chmod 0644, 'test.pgc'; + +is_program_out 'nobody', 'ecpg test.pgc', 0, '', 'ecpg processing'; + +is_program_out 'nobody', 'cc -I$(pg_config --includedir) -L$(pg_config --libdir) -o test test.c -lecpg', + 0, '', 'compiling ecpg output'; +chdir '/' or die "could not chdir to /: $!"; + +# run program +like_program_out 'nobody', "pg_virtualenv $workdir/test", 0, qr/Database is template1/, + 'test program runs and gives correct output'; + +# clean up +system "rm -rf $workdir"; +check_clean; + +# vim: filetype=perl diff --git a/t/190_pg_buildext.t b/t/190_pg_buildext.t new file mode 100644 index 0000000..9a50712 --- /dev/null +++ b/t/190_pg_buildext.t @@ -0,0 +1,104 @@ +# Check pg_buildext and that our debhelper integration works + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More; + +if ($PgCommon::rpm) { + pass 'No pg_buildext tests on RedHat'; + done_testing(); + exit; +} + +# when invoked from the postgresql-NN package tests, postgresql-server-dev-all is not installed +if (! -x '/usr/bin/dh_make_pgxs') { + pass "Skipping pg_buildext tests, /usr/bin/dh_make_pgxs is not installed"; + done_testing(); + exit; +} + +my $arch = `dpkg-architecture -qDEB_HOST_ARCH`; +chomp $arch; + +if ($ENV{PG_VERSIONS}) { + note "PG_VERSIONS=$ENV{PG_VERSIONS}"; + $ENV{PG_SUPPORTED_VERSIONS} = join ' ', (grep { $_ >= 9.1 } split /\s+/, $ENV{PG_VERSIONS}); + unless ($ENV{PG_SUPPORTED_VERSIONS}) { + ok 1, 'No versions with extension support to test'; + done_testing(); + exit; + } + note "PG_SUPPORTED_VERSIONS=$ENV{PG_SUPPORTED_VERSIONS}"; +} +my @versions = split /\s+/, `/usr/share/postgresql-common/supported-versions`; + +# prepare build environment +chmod 0777, 't/foo', 't/foo/foo-123', 't/bar/debian'; +umask 0022; +chdir 't/foo'; + +program_ok 0, 'make clean'; +program_ok 'nobody', 'make tar'; +program_ok 'nobody', 'cd foo-123 && echo y | EDITOR=true dh_make_pgxs'; + +note "testing 'dh --with pgxs'"; +program_ok 'nobody', 'cd foo-123 && DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -us -uc'; + +foreach my $ver (@versions) { + my $deb = "postgresql-$ver-foo_123-1_$arch.deb"; + ok (-f $deb, "$deb was built"); + SKIP: { + my $have_extension_destdir = `grep extension_destdir /usr/share/postgresql/$ver/postgresql.conf.sample`; + skip "No in-tree installcheck on PG $ver (missing extension_destdir)", 2 unless ($have_extension_destdir); + like_program_out 'nobody', "cd foo-123 && PG_SUPPORTED_VERSIONS=$ver dh_pgxs_test", + 0, qr/PostgreSQL $ver installcheck.*(test foo * \.\.\. ok|ok 1 * - foo)/s; # old/new PG 16 syntax + } + program_ok 0, "dpkg -i $deb"; + like_program_out 'nobody', "cd foo-123 && pg_buildext installcheck", + 0, qr/PostgreSQL $ver installcheck.*(test foo * \.\.\. ok|ok 1 * - foo)/s; + like_program_out 'nobody', "cd foo-123 && echo 'SELECT 3*41, version()' | pg_buildext psql", 0, qr/123.*PostgreSQL $ver/; + like_program_out 'nobody', "cd foo-123 && echo 'echo --\$PGVERSION--' | pg_buildext virtualenv", 0, qr/--$ver--/; + like_program_out 'nobody', "cd foo-123 && pg_buildext run echo --%v--", 0, qr/--$ver--/; + program_ok 0, "dpkg -r postgresql-$ver-foo"; +} + +note "testing 'dh --with pgxs_loop'"; +system "rm -f postgresql-*.deb"; + +program_ok 'nobody', 'sed -i -e s/pgxs/pgxs_loop/ foo-123/debian/rules'; +program_ok 'nobody', 'cd foo-123 && DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -us -uc'; + +foreach my $ver (@versions) { + my $deb = "postgresql-$ver-foo_123-1_$arch.deb"; + ok (-f $deb, "$deb was built"); +} + +program_ok 'nobody', 'make clean'; + +note "testing pg_buildext updatecontrol"; +chdir '../bar'; +program_ok 'nobody', 'PG_SUPPORTED_VERSIONS="0.9 1.0 1.1 2 3" pg_buildext updatecontrol'; +is `cat debian/control`, "Source: bar +Build-Depends: whatever, postgresql-1.0-moo (>= 1), postgresql-1.1-moo (>= 1), postgresql-2-moo (>= 1), more, + postgresql-2-new, + postgresql-1.0, postgresql-1.1, postgresql-2 + +Package: postgresql-1.0-bar +Architecture: some +Depends: postgresql-1.0-moo + +Package: postgresql-1.1-bar +Architecture: some +Depends: postgresql-1.1-moo + +Package: postgresql-2-bar +Architecture: some +Depends: postgresql-2-moo +", "PGVERSION and PGVERSIONS were correctly replaced"; + +done_testing(); + +# vim: filetype=perl diff --git a/t/200_maintscripts.t b/t/200_maintscripts.t new file mode 100644 index 0000000..3fc5018 --- /dev/null +++ b/t/200_maintscripts.t @@ -0,0 +1,46 @@ +# This test runs last since we are reinstalling postgresql-common, and we want +# to avoid spoiling the other tests with any version skew. + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; + +use Test::More tests => $PgCommon::rpm ? 1 : 15; + +if ($PgCommon::rpm) { + pass 'No maintainer script tests on rpm'; + exit; +} + +my $v = $MAJORS[-1]; + +note -d "/run/systemd/system" ? "We are running systemd" : "We are not running systemd"; + +# create cluster +program_ok 0, "pg_createcluster $v main --start"; + +# get postmaster PID +my $postmaster_pid = `head -1 /var/lib/postgresql/$v/main/postmaster.pid`; +chomp $postmaster_pid; +ok $postmaster_pid > 0, "postmaster PID is $postmaster_pid"; + +# "upgrade" postgresql-common to check if postgresql.service is left alone +program_ok 0, 'apt-get update -q', 0, ''; +note `apt-cache policy postgresql-common`; +program_ok 0, 'apt-get install -y --reinstall -o DPkg::Options::=--force-confnew postgresql-common', 0, ''; +note `apt-cache policy postgresql-common`; + +# get postmaster PID again, compare +my $postmaster_pid2 = `head -1 /var/lib/postgresql/$v/main/postmaster.pid`; +chomp $postmaster_pid2; +ok $postmaster_pid2 > 0, "postmaster PID is $postmaster_pid2"; +is $postmaster_pid, $postmaster_pid2, "postmaster was not restarted"; + +# stop server, clean up, check for leftovers +program_ok 0, "pg_dropcluster $v main --stop"; + +check_clean; + +# vim: filetype=perl diff --git a/t/TestLib.pm b/t/TestLib.pm new file mode 100644 index 0000000..408c9f5 --- /dev/null +++ b/t/TestLib.pm @@ -0,0 +1,270 @@ +# Common functionality for postgresql-common self tests +# +# (C) 2005-2009 Martin Pitt +# (C) 2013-2022 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +package TestLib; +use strict; +use Exporter; +use Test::More; +use PgCommon qw/get_versions change_ugid next_free_port/; + +our $VERSION = 1.00; +our @ISA = ('Exporter'); +our @EXPORT = qw/os_release ps ok_dir exec_as deb_installed rpm_installed package_version + version_ge program_ok is_program_out like_program_out unlike_program_out + pidof pid_env check_clean + @ALL_MAJORS @MAJORS $delay/; + +our @ALL_MAJORS = get_versions(); # not affected by PG_VERSIONS/-v +our @MAJORS = $ENV{PG_VERSIONS} ? split (/\s+/, $ENV{PG_VERSIONS}) : @ALL_MAJORS; +our $delay = 500_000; # 500ms + +# called if a test fails; spawn a shell if the environment variable +# FAILURE=shell is set +sub fail_debug { + if ($ENV{'FAILURE'} eq 'shell') { + if ((system 'bash') != 0) { + exit 1; + } + } +} + +# parse /etc/os-release and return (os, version number) +sub os_release { + open OS, "/etc/os-release" or return (undef, undef); + my ($os, $osversion); + while () { + $os = $1 if /^ID=(.*)/; + $osversion = $1 if /^VERSION_ID="?([^"]*)/; + } + close OS; + $osversion = 'unstable' if ($os eq 'debian' and not defined $osversion); + return ($os, $osversion); +} + +# Return whether a given deb is installed. +# Arguments: +sub deb_installed { + open (DPKG, "dpkg -s $_[0] 2>/dev/null|") or die "call dpkg: $!"; + my $result = 0; + while () { + if (/^Status: install ok installed/) { + $result = 1; + last; + } + } + close DPKG; + + return $result; +} + +# Return whether a given rpm is installed. +# Arguments: +sub rpm_installed { + open (RPM, "rpm -qa $_[0] 2>/dev/null|") or die "call rpm: $!"; + my $out = ; # returns void or the package name + close RPM; + return ($out =~ /./); +} + +# Return a package version +# Arguments: +sub package_version { + my $package = shift; + if ($PgCommon::rpm) { + return `rpm --queryformat '%{VERSION}' -q $package`; + } else { + my $version = `dpkg-query -f '\${Version}' --show $package`; + chomp $version; + return $version; + } +} + +# Return whether a version is greater or equal to another one +# Arguments: +sub version_ge { + my ($v1, $v2) = @_; + use IPC::Open2; + open2(\*CHLD_OUT, \*CHLD_IN, 'sort', '-Vr'); + print CHLD_IN "$v1\n"; + print CHLD_IN "$v2\n"; + close CHLD_IN; + my $v_ge = ; + chomp $v_ge; + return $v_ge eq $v1; +} + +# Return the user, group, and command line of running processes for the given +# program. +sub ps { + return `ps h -o user,group,args -C $_[0] | grep '$_[0]' | sort -u`; +} + +# Return array of pids that match the given command name (we require a leading +# slash so the postgres children are filtered out) +sub pidof { + my $prg = shift; + open F, '-|', 'ps', 'h', '-C', $prg, '-o', 'pid,cmd' or die "open: $!"; + my @pids; + while () { + if ((index $_, "/$prg") >= 0) { + push @pids, (split)[0]; + } + } + close F; + return @pids; +} + +# Return an reference to an array of all entries but . and .. of the given directory. +sub dircontent { + my $dir = $_[0]; + opendir D, $dir or return ["opendir $dir: $!"]; + my @e = grep { $_ ne '.' && $_ ne '..' } readdir (D); + closedir D; + return [sort @e]; +} + +# Return environment of given PID +sub pid_env { + my ($user, $pid) = @_; + my $path = "/proc/$pid/environ"; + my @lines; + open E, "su -c 'cat $path' $user |" or warn "open $path: $!"; + { + local $/; + @lines = split '\0', ; + } + close E; + my %env; + foreach (@lines) { + my ($k, $v) = (split '='); + $env{$k} = $v; + } + return %env; +} + +# Check the contents of a directory. +# Arguments: +sub ok_dir { + my $content = dircontent $_[0]; + if (eq_set $content, $_[1]) { + pass $_[2]; + } else { + diag "Expected directory contents: [@{$_[1]}], actual contents: [@$content]\n"; + fail $_[2]; + } +} + +# Execute a command as a different user and return the output. Prints the +# output of the command if exit code differs from expected one. +# Arguments: [] +# Returns: Program exit code +sub exec_as { + my $uid; + if ($_[0] =~ /\d+/) { + $uid = int($_[0]); + } else { + $uid = getpwnam $_[0]; + defined($uid) or die "TestLib::exec_as: target user '$_[0]' does not exist"; + } + change_ugid ($uid, (getpwuid $uid)[3]); + die "changing euid: $!" if $> != $uid; + my $out = `$_[1] 2>&1`; + my $result = $? >> 8; + $< = $> = 0; + $( = $) = 0; + die "changing euid back to root: $!" if $> != 0; + $_[2] = \$out; + + if (defined $_[3] && $_[3] != $result) { + print "command '$_[1]' did not exit with expected code $_[3] but with $result:\n"; + print $out; + fail_debug; + } + return $result; +} + +# Execute a command as a particular user, and check the exit code +# Arguments: [] [] +sub program_ok { + my ($user, $cmd, $exit, $description) = @_; + $exit ||= 0; + $description ||= $cmd; + my $outref; + ok ((exec_as $user, $cmd, \$outref, $exit) == $exit, $description); +} + +# Execute a command as a particular user, and check the exit code and output +# (merged stdout/stderr). +# Arguments: [] +sub is_program_out { + my $outref; + my $result = exec_as $_[0], $_[1], $outref; + is $result, $_[2], $_[1] or fail_debug; + is ($$outref, $_[3], (defined $_[4] ? $_[4] : "correct output of $_[1]")) or fail_debug; +} + +# Execute a command as a particular user, and check the exit code and output +# against a regular expression (merged stdout/stderr). +# Arguments: [] +sub like_program_out { + my $outref; + my $result = exec_as $_[0], $_[1], $outref; + is $result, $_[2], $_[1] or fail_debug; + like ($$outref, $_[3], (defined $_[4] ? $_[4] : "correct output of $_[1]")) or fail_debug; +} + +# Execute a command as a particular user, check the exit code, and check that +# the output does not match a regular expression (merged stdout/stderr). +# Arguments: [] +sub unlike_program_out { + my $outref; + my $result = exec_as $_[0], $_[1], $outref; + is $result, $_[2], $_[1] or fail_debug; + unlike ($$outref, $_[3], (defined $_[4] ? $_[4] : "correct output of $_[1]")) or fail_debug; +} + +# Check that all PostgreSQL related directories are empty and no +# postgres processes are running. Should be called at the end +# of all tests. Does 8 tests. +sub check_clean { + note "Cleanup"; + is (`pg_lsclusters -h`, '', 'Cleanup: No clusters left behind'); + is ((ps 'postgres'), '', 'No postgres processes left behind'); + + my @check_dirs = ('/etc/postgresql', '/var/lib/postgresql', + '/var/run/postgresql'); + foreach (@check_dirs) { + if (-d) { + ok_dir $_, [], "No files in $_ left behind"; + } else { + pass "Directory $_ does not exist"; + } + } + # we always want /var/log/postgresql/ to exist, so that logrotate does not + # complain about missing directories + ok_dir '/var/log/postgresql', [], "No files in /var/log/postgresql left behind"; + + # prefer ss over netstat (until all debian/tests/control files in postgresql-* have been updated) + unless (-x '/bin/netstat' and not -x '/bin/ss') { + is `ss --no-header -tlp 'sport >= 5432 and sport <= 5439'`, '', + 'PostgreSQL TCP ports are closed'; + } else { + is `netstat -avptn 2>/dev/null | grep ":543[2-9]\\b.*LISTEN"`, '', + 'PostgreSQL TCP ports are closed'; + } + is next_free_port(), 5432, "Next free port is 5432"; +} + +1; diff --git a/t/bar/debian/control.in b/t/bar/debian/control.in new file mode 100644 index 0000000..86b8069 --- /dev/null +++ b/t/bar/debian/control.in @@ -0,0 +1,8 @@ +Source: bar +Build-Depends: whatever, postgresql-PGVERSIONS-moo (>= 1), more, + postgresql-PGVERSION-new, + postgresql-PGVERSIONS + +Package: postgresql-PGVERSION-bar +Architecture: some +Depends: postgresql-PGVERSION-moo diff --git a/t/bar/debian/pgversions b/t/bar/debian/pgversions new file mode 100644 index 0000000..41f0ed3 --- /dev/null +++ b/t/bar/debian/pgversions @@ -0,0 +1,3 @@ +1.0 +1.1 +2 diff --git a/t/foo/Makefile b/t/foo/Makefile new file mode 100644 index 0000000..75e95e1 --- /dev/null +++ b/t/foo/Makefile @@ -0,0 +1,12 @@ +TAR = foo_123.orig.tar.gz + +test: $(TAR) + cd foo-123 && echo y | EDITOR=true dh_make_pgxs + cd foo-123 && dpkg-buildpackage -us -uc + +tar $(TAR): + tar cfz $(TAR) foo-123/ + +clean: + rm -f *.* + rm -rf foo-123/build-* foo-123/debian foo-123/*.o* diff --git a/t/foo/foo-123/Makefile b/t/foo/foo-123/Makefile new file mode 100644 index 0000000..9f9a2f8 --- /dev/null +++ b/t/foo/foo-123/Makefile @@ -0,0 +1,10 @@ +MODULE_big = foo +OBJS = foo.o +EXTENSION = foo +DATA = foo--100.sql foo--100--123.sql foo--123.sql +REGRESS = foo upgrade + +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) + diff --git a/t/foo/foo-123/README.md b/t/foo/foo-123/README.md new file mode 100644 index 0000000..14235f9 --- /dev/null +++ b/t/foo/foo-123/README.md @@ -0,0 +1,5 @@ +The PostgreSQL foo extension +============================ + +This extension exists for testing postgresql-common's `dh_make_pgxs` and +`dh --with pgxs` mechanisms. diff --git a/t/foo/foo-123/expected/foo.out b/t/foo/foo-123/expected/foo.out new file mode 100644 index 0000000..62afef7 --- /dev/null +++ b/t/foo/foo-123/expected/foo.out @@ -0,0 +1,8 @@ +CREATE EXTENSION foo; +SELECT foo(); + foo +----- + bar +(1 row) + +DROP EXTENSION foo; diff --git a/t/foo/foo-123/expected/upgrade.out b/t/foo/foo-123/expected/upgrade.out new file mode 100644 index 0000000..98fc8e6 --- /dev/null +++ b/t/foo/foo-123/expected/upgrade.out @@ -0,0 +1,14 @@ +CREATE EXTENSION foo VERSION "100"; +SELECT foo(); + foo +--------- + old bar +(1 row) + +ALTER EXTENSION foo UPDATE TO "123"; +SELECT foo(); + foo +----- + bar +(1 row) + diff --git a/t/foo/foo-123/foo--100--123.sql b/t/foo/foo-123/foo--100--123.sql new file mode 100644 index 0000000..a6e7adc --- /dev/null +++ b/t/foo/foo-123/foo--100--123.sql @@ -0,0 +1,3 @@ +CREATE OR REPLACE FUNCTION foo () +RETURNS text LANGUAGE C +AS '$libdir/foo'; diff --git a/t/foo/foo-123/foo--100.sql b/t/foo/foo-123/foo--100.sql new file mode 100644 index 0000000..5ed8546 --- /dev/null +++ b/t/foo/foo-123/foo--100.sql @@ -0,0 +1,3 @@ +CREATE OR REPLACE FUNCTION foo () +RETURNS text LANGUAGE SQL +AS $$ SELECT 'old bar'::text $$; diff --git a/t/foo/foo-123/foo--123.sql b/t/foo/foo-123/foo--123.sql new file mode 100644 index 0000000..a6e7adc --- /dev/null +++ b/t/foo/foo-123/foo--123.sql @@ -0,0 +1,3 @@ +CREATE OR REPLACE FUNCTION foo () +RETURNS text LANGUAGE C +AS '$libdir/foo'; diff --git a/t/foo/foo-123/foo.c b/t/foo/foo-123/foo.c new file mode 100644 index 0000000..8677acc --- /dev/null +++ b/t/foo/foo-123/foo.c @@ -0,0 +1,13 @@ +#include "postgres.h" +#include "fmgr.h" +#include "utils/builtins.h" + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1 (foo); + +Datum +foo (PG_FUNCTION_ARGS) +{ + PG_RETURN_TEXT_P(cstring_to_text("bar")); +} diff --git a/t/foo/foo-123/foo.control b/t/foo/foo-123/foo.control new file mode 100644 index 0000000..81c95c8 --- /dev/null +++ b/t/foo/foo-123/foo.control @@ -0,0 +1,2 @@ +default_version = '123' +comment = 'The foo extension' diff --git a/t/foo/foo-123/sql/foo.sql b/t/foo/foo-123/sql/foo.sql new file mode 100644 index 0000000..155f63e --- /dev/null +++ b/t/foo/foo-123/sql/foo.sql @@ -0,0 +1,5 @@ +CREATE EXTENSION foo; + +SELECT foo(); + +DROP EXTENSION foo; diff --git a/t/foo/foo-123/sql/upgrade.sql b/t/foo/foo-123/sql/upgrade.sql new file mode 100644 index 0000000..b4427b4 --- /dev/null +++ b/t/foo/foo-123/sql/upgrade.sql @@ -0,0 +1,7 @@ +CREATE EXTENSION foo VERSION "100"; + +SELECT foo(); + +ALTER EXTENSION foo UPDATE TO "123"; + +SELECT foo(); diff --git a/t/template b/t/template new file mode 100644 index 0000000..fa1df59 --- /dev/null +++ b/t/template @@ -0,0 +1,26 @@ +# Check XXX + +use strict; + +use lib 't'; +use TestLib; +use PgCommon; +use Test::More tests => 14; + +my @versions = ($MAJORS[-1]); + +# create clusters +foreach (@versions) { + is ((system "pg_createcluster $_ main --start >/dev/null"), 0, "pg_createcluster $_ main"); + is_program_out 'postgres', "createdb --cluster $_/main XXX", 0, ''; +} + +# XXX + +# clean up +foreach (@versions) { + is ((system "pg_dropcluster $_ main --stop"), 0, "pg_dropcluster $_ main"); +} +check_clean; + +# vim: filetype=perl diff --git a/testsuite b/testsuite new file mode 100755 index 0000000..d3ff77d --- /dev/null +++ b/testsuite @@ -0,0 +1,222 @@ +#!/bin/bash + +# Run integration tests (on the installed packages). +# +# (C) 2005-2012 Martin Pitt +# (C) 2012-2023 Christoph Berg +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +set -eu + +# default config +TESTSDIR="$(dirname $0)/t" +: ${PG_UMASKS="077"} # default umask(s) to try + +help () +{ + echo "postgresql-common testsuite" + echo "Syntax: $0 [options] [test ...]" + echo " -D drop all existing clusters (USE WITH CARE)" + echo " -f number run tests starting at this number" + echo " -F suffix flavor suffix on package names, e.g. 'ee'" + echo " -i install packages for versions specified by -v" + echo " -M don't mount tmpfses, run in the host system" + echo " -s start a shell in the testbed on failure" + echo " -u 'umask ...' umasks to run testsuite with [default: 077]" + echo " -v 'version ...' PostgreSQL versions to test [default: client versions installed]" + echo " -V debug output" + exit ${1:-0} +} + +# option parsing +while getopts "Dhf:F:iMsu:v:V" opt ; do + case $opt in + D) DROP_ALL_CLUSTERS=1 ;; + f) FROM="$OPTARG" ;; + F) export PG_FLAVOR="$OPTARG" ;; + i) INSTALL=1 ;; + h) help ;; + M) export NO_TMPFS=1 ;; + s) export FAILURE="shell" ;; + u) PG_UMASKS="$OPTARG" ;; + v) export PG_VERSIONS="$OPTARG" ;; # used in t/TestLib.pm + V) VERBOSE=1 ;; + *) help 1 ;; + esac +done + +if [ "$(id -u)" != 0 ]; then + echo "Error: this test suite needs to be run as root" >&2 + exit 1 +fi + +# install packages for versions specified by -v +# needs network for apt, so run before unshare +if [ "${INSTALL:-}" ] && [ -z "${UNSHARED:-}" ]; then + . /etc/os-release + case $ID in + debian|ubuntu) + for v in $PG_VERSIONS; do + case $v in 8.*|9.*|10|11) + [ "$(perl -I. -le 'use PgCommon; print $PgCommon::have_python2')" = "1" ] && PYTHON2_PACKAGE=postgresql-plpython-$v${PG_FLAVOR:=} + ;; + esac + apt-get install -y \ + postgresql-contrib-$v${PG_FLAVOR:=} \ + postgresql-plperl-$v$PG_FLAVOR \ + ${PYTHON2_PACKAGE:-} \ + $(dpkg --compare-versions $v ge 9.1 && echo postgresql-plpython3-$v$PG_FLAVOR) \ + postgresql-pltcl-$v$PG_FLAVOR \ + postgresql-server-dev-$v$PG_FLAVOR \ + postgresql-doc-$v$PG_FLAVOR + done + apt-get install -y \ + debhelper \ + libecpg-dev \ + locales-all \ + procps systemd \ + netcat-openbsd \ + hunspell-en-us \ + gcc + ;; + redhat|centos) + for v in $PG_VERSIONS; do + vv=$(echo $v | tr -d .) + yum install -y \ + postgresql$vv${PG_FLAVOR:=}-contrib \ + postgresql$vv$PG_FLAVOR-plperl \ + postgresql$vv$PG_FLAVOR-plpython3 \ + postgresql$vv$PG_FLAVOR-pltcl \ + postgresql$vv$PG_FLAVOR-devel \ + postgresql$vv$PG_FLAVOR-doc + done + yum install -y \ + nmap-ncat \ + perl-Test-Simple perl-Time-HiRes \ + gcc + ;; + *) + echo "Unknown distribution ID $ID in /etc/os-release" + exit 1 + ;; + esac +fi + +# shift away args +shift $(($OPTIND - 1)) + +# stop currently running clusters +if [ -d /run/systemd/system ] ; then + # stop postgresql@* explicitly (#759725) + systemctl stop postgresql "postgresql@*" "pg_receivewal@*" +else + service postgresql stop +fi + +# drop all existing clusters if the user requested it +if [ "${DROP_ALL_CLUSTERS:-}" ]; then + pg_lsclusters -h | while read ver cluster rest; do + pg_dropcluster $ver $cluster + done +fi + +# set up test directories +dirs="/etc/postgresql /var/lib/postgresql /var/log/postgresql /var/run/postgresql /var/backups/postgresql" + +if [ -z "${NO_TMPFS:-}" ]; then + # clean up after us + cleanup () { + set +e + umount -l $dirs + sed -i -e '/# by pg-testsuite/d' /etc/postgresql-common/createcluster.conf + ./pg_updateaptconfig + systemctl daemon-reload 2> /dev/null || : # poke generator to handle the system's clusters again + exit + } + trap "cleanup" 0 HUP INT QUIT ILL ABRT PIPE TERM + + for d in $dirs; do + mkdir -p $d + mount -t tmpfs tmpfs $d + done + + chown postgres:postgres /etc/postgresql ; chmod 755 /etc/postgresql + chown postgres:postgres /var/lib/postgresql ; chmod 755 /var/lib/postgresql + chown root:postgres /var/log/postgresql ; chmod 1775 /var/log/postgresql + chown postgres:postgres /var/run/postgresql ; chmod 2775 /var/run/postgresql /var/backups/postgresql +fi + +if [ -d /run/systemd/system ]; then + systemctl daemon-reload # poke generator to forget the system's clusters +fi + +# the RPM packages enable the logging_collector by default, which the testsuite +# doesn't like. We disable it unconditionally here. +if ! grep -q logging_collector /etc/postgresql-common/createcluster.conf; then + echo "logging_collector = off # by pg-testsuite" >> /etc/postgresql-common/createcluster.conf +fi + +# reset core limit for pg_ctl tests +ulimit -S -c 0 + +# set environment +unset TMPDIR +unset LC_ALL +export LANG=en_US.utf8 + +# set variables which cause taint check errors +export IFS +export CDPATH=/usr +export ENV=/nonexisting +export BASH_ENV=/nonexisting + +if [ $# -eq 0 ]; then + set -- $TESTSDIR/*.t +fi + +# dump environment for debugging +if [ "${VERBOSE:-}" ]; then + echo "Environment:" + env | sort + echo "Mounts:" + cat /proc/mounts + echo "Namespaces:" + ls -l /proc/self/ns + echo +fi + +for U in $PG_UMASKS; do + echo "====== Running all tests with umask $U =======" + umask $U + for T; do + TBASE=${T##*/} + [ "${FROM:-}" ] && [ "${TBASE%%_*}" -lt "${FROM:-}" ] && continue + echo "### PostgreSQL test $TBASE ###" + perl -I. $T || { + EXIT=$? + FAILED_TESTS="${FAILED_TESTS:-} $T" + if [ "${FAILURE:-}" ]; then + echo "*** $TBASE failed with status $EXIT, dropping you into a shell in the testbed ***" + ${SHELL:-/bin/sh} + fi + } + echo "### End test $TBASE ###" + echo + done +done + +if [ "${FAILED_TESTS:-}" ]; then + echo "Failed tests: $FAILED_TESTS" + echo +fi + +exit ${EXIT:-0} diff --git a/user_clusters b/user_clusters new file mode 100644 index 0000000..82d2d70 --- /dev/null +++ b/user_clusters @@ -0,0 +1,22 @@ +# This file maps users against the database clusters to which they +# will connect by default. Any user may create ~/.postgresqlrc which +# will supersede the defaults stored here. If a database is +# specified, that will be the one connected to by client tools if none +# is specified on the command line. If the database specified here is +# "*", this is interpreted as the database whose name is the same as +# the user's login. (Setting the database to "*" will provide the +# current default upstream behaviour for command line tools.) +# +# When pg_wrapper scans this file, the first matching line is used. +# It is a good idea to provide a default explicitly, with a final line +# where both user and group are set to "*". If there is no default, +# the implicit default is to connect to the cluster listening on +# port 5432 and to the database matching the user's login name. +# +# In the context of this file, user and group refer to the Unix login +# or group, not to PostgreSQL users and groups. +# +# Please see user_clusters(5) and postgresqlrc(5) for more information. +# +# USER GROUP VERSION CLUSTER DATABASE + diff --git a/user_clusters.5 b/user_clusters.5 new file mode 100644 index 0000000..a173fde --- /dev/null +++ b/user_clusters.5 @@ -0,0 +1,63 @@ +.TH USER_CLUSTERS 5 "Feburary 2005" "Debian" "Debian PostgreSQL infrastructure" + +.SH NAME +user_clusters \- File linking users to PostgreSQL clusters + +.SH DESCRIPTION +The file +.B /etc/postgresql-common/user_clusters +maps users against the database clusters to which they will +connect by default. However, every user can override these settings in +.B ~/.postgresqlrc\fR. + +When scanning this file, the first matching line will be used. It is a +good idea to provide a default explicitly, with a final line where both +user and group are set to +.BR * . + +If there is no default, the implicit default is to connect to the cluster +listening on port 5432 and to the database matching the user's +login name. + +.SH FORMAT +Comments are introduced by the character +.BR # . +Comments may follow data on a line; the first comment character terminates +the data. Leading whitespace and blank lines are ignored. + +Each uncommented, non\-blank line must describe a user, group or the +default (where both user and group are set to \fB*\fR). + +Fields must be given in the following order, separated by white space: + +.TP +.B USER +The login id of the Unix user to whom this line applies. The wildcard character +.B * +means any user. +.TP +.B GROUP +The group name of the Unix group to which this line applies. The wildcard character +.B * +means any group. +.TP +.B VERSION +The major PostgreSQL version of the cluster to connect to. +.TP +.B CLUSTER +The name of a cluster to connect to. A remote cluster is specified +with \fIhost\fR:\fIport\fR. If port is empty, it defaults to 5432. +.TP +.B DATABASE +Within the cluster, the database to which the user will connect by default +if he does not specify a database on the command line. If this is +.BR * , +the default database will be the one named by the user's login id. + +.SH NOTES +.P +Since the first matching line is used, the default line must come last. +.P + +.SH SEE ALSO +.BR pg_wrapper (1), " postgresqlrc" (5) -- cgit v1.2.3