Adding upstream version 16.2.upstream/16.2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

6 files changed, 286 insertions, 0 deletions

diff --git a/src/tools/ci/README b/src/tools/ci/README
new file mode 100644
index 0000000..30ddd20
--- /dev/null
+++ b/src/tools/ci/README
@@ -0,0 +1,84 @@
+Postgres Continuous Integration (CI)
+====================================
+
+Postgres has two forms of CI:
+
+1) All supported branches in the main postgres repository are continuously
+   tested via the buildfarm. As this covers only the main repository, it
+   cannot be used during development of features.
+
+   For details see https://buildfarm.postgresql.org/
+
+2) For not yet merged development work, CI can be enabled for some git hosting
+   providers. This allows developers to test patches on a number of platforms
+   before they are merged (or even submitted).
+
+
+Configuring CI on personal repositories
+=======================================
+
+Currently postgres contains CI support utilizing cirrus-ci. cirrus-ci
+currently is only available for github.
+
+
+Enabling cirrus-ci in a github repository
+=========================================
+
+To enable cirrus-ci on a repository, go to
+https://github.com/marketplace/cirrus-ci and select "Public
+Repositories". Then "Install it for free" and "Complete order". The next page
+allows to configure which repositories cirrus-ci has access to. Choose the
+relevant repository and "Install".
+
+See also https://cirrus-ci.org/guide/quick-start/
+
+Once enabled on a repository, future commits and pull-requests in that
+repository will automatically trigger CI builds. These are visible from the
+commit history / PRs, and can also be viewed in the cirrus-ci UI at
+https://cirrus-ci.com/github/<username>/<reponame>/
+
+Hint: all build log files are uploaded to cirrus-ci and can be downloaded
+from the "Artifacts" section from the cirrus-ci UI after clicking into a
+specific task on a build's summary page.
+
+
+Images used for CI
+==================
+
+To keep CI times tolerable, most platforms use pre-generated images. Some
+platforms use containers, others use full VMs. Images for both are generated
+separately from CI runs, otherwise each git repository that is being tested
+would need to build its own set of containers, which would be wasteful (both
+in space and time.
+
+These images are built, on a daily basis, from the specifications in
+github.com/anarazel/pg-vm-images/
+
+
+Controlling CI via commit messages
+==================================
+
+The behavior of CI can be controlled by special content in commit
+messages. Currently the following controls are available:
+
+- ci-os-only: {(freebsd|linux|macos|windows|mingw)}
+
+  Only runs CI on operating systems specified. This can be useful when
+  addressing portability issues affecting only a subset of platforms.
+
+
+Using custom compute resources for CI
+=====================================
+
+When running a lot of tests in a repository, cirrus-ci's free credits do not
+suffice. In those cases a repository can be configured to use other
+infrastructure for running tests. To do so, the REPO_CI_CONFIG_GIT_URL
+variable can be configured for the repository in the cirrus-ci web interface,
+at https://cirrus-ci.com/github/<user or organization>. The file referenced
+(see https://cirrus-ci.org/guide/programming-tasks/#fs) by the variable can
+overwrite the default execution method for different operating systems,
+defined in .cirrus.yml, by redefining the relevant yaml anchors.
+
+Custom compute resources can be provided using
+- https://cirrus-ci.org/guide/supported-computing-services/
+- https://cirrus-ci.org/guide/persistent-workers/
diff --git a/src/tools/ci/ci_macports_packages.sh b/src/tools/ci/ci_macports_packages.sh
new file mode 100755
index 0000000..4bc594a
--- /dev/null
+++ b/src/tools/ci/ci_macports_packages.sh
@@ -0,0 +1,97 @@
+#!/bin/sh
+
+# Installs the passed in packages via macports. To make it fast enough
+# for CI, cache the installation as a .dmg file.  To avoid
+# unnecessarily updating the cache, the cached image is only modified
+# when packages are installed or removed.  Any package this script is
+# not instructed to install, will be removed again.
+#
+# This currently expects to be run in a macos cirrus-ci environment.
+
+set -e
+# set -x
+
+packages="$@"
+
+macports_url="https://github.com/macports/macports-base/releases/download/v2.8.1/MacPorts-2.8.1-13-Ventura.pkg"
+cache_dmg="macports.hfs.dmg"
+
+if [ "$CIRRUS_CI" != "true" ]; then
+    echo "expect to be called within cirrus-ci" 1>2
+    exit 1
+fi
+
+sudo mkdir -p /opt/local
+mkdir -p ${MACPORTS_CACHE}/
+
+# If we are starting from clean cache, perform a fresh macports
+# install. Otherwise decompress the .dmg we created previously.
+#
+# After this we have a working macports installation, with an unknown set of
+# packages installed.
+new_install=0
+update_cached_image=0
+if [ -e ${MACPORTS_CACHE}/${cache_dmg}.zstd ]; then
+    time zstd -T0 -d ${MACPORTS_CACHE}/${cache_dmg}.zstd -o ${cache_dmg}
+    time sudo hdiutil attach -kernel ${cache_dmg} -owners on -shadow ${cache_dmg}.shadow -mountpoint /opt/local
+else
+    new_install=1
+    curl -fsSL -o macports.pkg "$macports_url"
+    time sudo installer -pkg macports.pkg -target /
+    # this is a throwaway environment, and it'd be a few lines to gin
+    # up a correct user / group when using the cache.
+    echo macportsuser root | sudo tee -a /opt/local/etc/macports/macports.conf
+fi
+export PATH=/opt/local/sbin/:/opt/local/bin/:$PATH
+
+# mark all installed packages unrequested, that allows us to detect
+# packages that aren't needed anymore
+if [ -n "$(port -q installed installed)" ] ; then
+    sudo port unsetrequested installed
+fi
+
+# if setting all the required packages as requested fails, we need
+# to install at least one of them
+if ! sudo port setrequested $packages > /dev/null 2>&1 ; then
+    echo not all required packages installed, doing so now
+    update_cached_image=1
+    # to keep the image small, we deleted the ports tree from the image...
+    sudo port selfupdate
+    # XXX likely we'll need some other way to force an upgrade at some
+    # point...
+    sudo port upgrade outdated
+    sudo port install -N $packages
+    sudo port setrequested $packages
+fi
+
+# check if any ports should be uninstalled
+if [ -n "$(port -q installed rleaves)" ] ; then
+    echo superflous packages installed
+    update_cached_image=1
+    sudo port uninstall --follow-dependencies rleaves
+
+    # remove prior cache contents, don't want to increase size
+    rm -f ${MACPORTS_CACHE}/*
+fi
+
+# Shrink installation if we created / modified it
+if [ "$new_install" -eq 1 -o "$update_cached_image" -eq 1 ]; then
+    sudo /opt/local/bin/port clean --all installed
+    sudo rm -rf /opt/local/var/macports/{software,sources}/*
+fi
+
+# If we're starting from a clean cache, start a new image. If we have
+# an image, but the contents changed, update the image in the cache
+# location.
+if [ "$new_install" -eq 1 ]; then
+    # use a generous size, so additional software can be installed later
+    time sudo hdiutil create -fs HFS+ -format UDRO -size 10g -layout NONE -srcfolder /opt/local/ ${cache_dmg}
+    time zstd -T -10 -z ${cache_dmg} -o ${MACPORTS_CACHE}/${cache_dmg}.zstd
+elif [ "$update_cached_image" -eq 1 ]; then
+    sudo hdiutil detach /opt/local/
+    time hdiutil convert -format UDRO ${cache_dmg} -shadow ${cache_dmg}.shadow -o updated.hfs.dmg
+    rm ${cache_dmg}.shadow
+    mv updated.hfs.dmg ${cache_dmg}
+    time zstd --force -T -10 -z ${cache_dmg} -o ${MACPORTS_CACHE}/${cache_dmg}.zstd
+    time sudo hdiutil attach -kernel ${cache_dmg} -owners on -shadow ${cache_dmg}.shadow -mountpoint /opt/local
+fi
diff --git a/src/tools/ci/cores_backtrace.sh b/src/tools/ci/cores_backtrace.sh
new file mode 100755
index 0000000..28d3cec
--- /dev/null
+++ b/src/tools/ci/cores_backtrace.sh
@@ -0,0 +1,50 @@
+#! /bin/sh
+
+if [ $# -ne 2 ]; then
+    echo "cores_backtrace.sh <os> <directory>"
+    exit 1
+fi
+
+os=$1
+directory=$2
+
+case $os in
+    freebsd|linux|macos)
+    ;;
+    *)
+        echo "unsupported operating system ${os}"
+        exit 1
+    ;;
+esac
+
+first=1
+for corefile in $(find "$directory" -type f) ; do
+    if [ "$first" -eq 1 ]; then
+        first=0
+    else
+        # to make it easier to separate the different crash reports
+        echo -e '\n\n'
+    fi
+
+    if [ "$os" = 'macos' ]; then
+        lldb -c $corefile --batch -o 'thread backtrace all' -o 'quit'
+    else
+        auxv=$(gdb --quiet --core ${corefile} --batch -ex 'info auxv' 2>/dev/null)
+        if [ $? -ne 0 ]; then
+            echo "could not process ${corefile}"
+            continue
+        fi
+
+        if [ "$os" = 'freebsd' ]; then
+            binary=$(echo "$auxv" | grep AT_EXECPATH | perl -pe "s/^.*\"(.*)\"\$/\$1/g")
+        elif [ "$os" = 'linux' ]; then
+            binary=$(echo "$auxv" | grep AT_EXECFN | perl -pe "s/^.*\"(.*)\"\$/\$1/g")
+        else
+            echo 'should not get here'
+            exit 1
+        fi
+
+        echo "dumping ${corefile} for ${binary}"
+        gdb --batch --quiet -ex "thread apply all bt full" -ex "quit" "$binary" "$corefile" 2>/dev/null
+    fi
+done
diff --git a/src/tools/ci/gcp_freebsd_repartition.sh b/src/tools/ci/gcp_freebsd_repartition.sh
new file mode 100755
index 0000000..2d5e173
--- /dev/null
+++ b/src/tools/ci/gcp_freebsd_repartition.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -e
+set -x
+
+# The default filesystem on freebsd gcp images is very slow to run tests on,
+# due to its 32KB block size
+#
+# XXX: It'd probably better to fix this in the image, using something like
+# https://people.freebsd.org/~lidl/blog/re-root.html
+
+# fix backup partition table after resize
+gpart recover da0
+gpart show da0
+# kill swap, so we can delete a partition
+swapoff -a || true
+# (apparently we can only have 4!?)
+gpart delete -i 3 da0
+gpart add -t freebsd-ufs -l data8k -a 4096 da0
+gpart show da0
+newfs -U -b 8192 /dev/da0p3
+
+# Migrate working directory
+du -hs $CIRRUS_WORKING_DIR
+mv $CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR.orig
+mkdir $CIRRUS_WORKING_DIR
+mount -o noatime /dev/da0p3 $CIRRUS_WORKING_DIR
+cp -r $CIRRUS_WORKING_DIR.orig/* $CIRRUS_WORKING_DIR/
diff --git a/src/tools/ci/pg_ci_base.conf b/src/tools/ci/pg_ci_base.conf
new file mode 100644
index 0000000..d8faa9c
--- /dev/null
+++ b/src/tools/ci/pg_ci_base.conf
@@ -0,0 +1,14 @@
+# Tends to produce too many core files, taking a long time
+restart_after_crash = false
+
+# So that tests using the "manually" started postgres on windows can use
+# prepared statements
+max_prepared_transactions = 10
+
+# Settings that make logs more useful
+log_autovacuum_min_duration = 0
+log_checkpoints = true
+log_connections = true
+log_disconnections = true
+log_line_prefix = '%m [%p][%b] %q[%a][%v:%x] '
+log_lock_waits = true
diff --git a/src/tools/ci/windows_build_config.pl b/src/tools/ci/windows_build_config.pl
new file mode 100644
index 0000000..b0d4360
--- /dev/null
+++ b/src/tools/ci/windows_build_config.pl
@@ -0,0 +1,13 @@
+use strict;
+use warnings;
+
+our $config;
+
+$config->{"tap_tests"} = 1;
+$config->{"asserts"} = 1;
+
+$config->{"openssl"} = "c:/openssl/1.1/";
+$config->{"perl"} = "c:/strawberry/$ENV{DEFAULT_PERL_VERSION}/perl/";
+$config->{"python"} = "c:/python/";
+
+1;