10 files changed, 538 insertions, 0 deletions
diff --git a/tools/build/build_libpq.sh b/tools/build/build_libpq.sh
new file mode 100755
index 0000000..4cc79af
--- /dev/null
+++ b/tools/build/build_libpq.sh
@@ -0,0 +1,173 @@
+#!/bin/bash
+
+# Build a modern version of libpq and depending libs from source on Centos 5
+
+set -euo pipefail
+set -x
+
+# Last release: https://www.postgresql.org/ftp/source/
+# IMPORTANT! Change the cache key in packages.yml when upgrading libraries
+postgres_version="${LIBPQ_VERSION:-15.0}"
+
+# last release: https://www.openssl.org/source/
+openssl_version="${OPENSSL_VERSION:-1.1.1r}"
+
+# last release: https://openldap.org/software/download/
+ldap_version="2.6.3"
+
+# last release: https://github.com/cyrusimap/cyrus-sasl/releases
+sasl_version="2.1.28"
+
+export LIBPQ_BUILD_PREFIX=${LIBPQ_BUILD_PREFIX:-/tmp/libpq.build}
+
+if [[ -f "${LIBPQ_BUILD_PREFIX}/lib/libpq.so" ]]; then
+    echo "libpq already available: build skipped" >&2
+    exit 0
+fi
+
+source /etc/os-release
+
+case "$ID" in
+    centos)
+        yum update -y
+        yum install -y zlib-devel krb5-devel pam-devel
+        ;;
+
+    alpine)
+        apk upgrade
+        apk add --no-cache zlib-dev krb5-dev linux-pam-dev openldap-dev
+        ;;
+
+    *)
+        echo "$0: unexpected Linux distribution: '$ID'" >&2
+        exit 1
+        ;;
+esac
+
+if [ "$ID" == "centos" ]; then
+
+    # Build openssl if needed
+    openssl_tag="OpenSSL_${openssl_version//./_}"
+    openssl_dir="openssl-${openssl_tag}"
+    if [ ! -d "${openssl_dir}" ]; then curl -sL \
+            https://github.com/openssl/openssl/archive/${openssl_tag}.tar.gz \
+            | tar xzf -
+
+        cd "${openssl_dir}"
+
+        ./config --prefix=${LIBPQ_BUILD_PREFIX} --openssldir=${LIBPQ_BUILD_PREFIX} \
+            zlib -fPIC shared
+        make depend
+        make
+    else
+        cd "${openssl_dir}"
+    fi
+
+    # Install openssl
+    make install_sw
+    cd ..
+
+fi
+
+
+if [ "$ID" == "centos" ]; then
+
+    # Build libsasl2 if needed
+    # The system package (cyrus-sasl-devel) causes an amazing error on i686:
+    # "unsupported version 0 of Verneed record"
+    # https://github.com/pypa/manylinux/issues/376
+    sasl_tag="cyrus-sasl-${sasl_version}"
+    sasl_dir="cyrus-sasl-${sasl_tag}"
+    if [ ! -d "${sasl_dir}" ]; then
+        curl -sL \
+            https://github.com/cyrusimap/cyrus-sasl/archive/${sasl_tag}.tar.gz \
+            | tar xzf -
+
+        cd "${sasl_dir}"
+
+        autoreconf -i
+        ./configure --prefix=${LIBPQ_BUILD_PREFIX} \
+            CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib
+        make
+    else
+        cd "${sasl_dir}"
+    fi
+
+    # Install libsasl2
+    # requires missing nroff to build
+    touch saslauthd/saslauthd.8
+    make install
+    cd ..
+
+fi
+
+
+if [ "$ID" == "centos" ]; then
+
+    # Build openldap if needed
+    ldap_tag="${ldap_version}"
+    ldap_dir="openldap-${ldap_tag}"
+    if [ ! -d "${ldap_dir}" ]; then
+        curl -sL \
+            https://www.openldap.org/software/download/OpenLDAP/openldap-release/openldap-${ldap_tag}.tgz \
+            | tar xzf -
+
+        cd "${ldap_dir}"
+
+        ./configure --prefix=${LIBPQ_BUILD_PREFIX} --enable-backends=no --enable-null \
+            CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib
+
+        make depend
+        make -C libraries/liblutil/
+        make -C libraries/liblber/
+        make -C libraries/libldap/
+    else
+        cd "${ldap_dir}"
+    fi
+
+    # Install openldap
+    make -C libraries/liblber/ install
+    make -C libraries/libldap/ install
+    make -C include/ install
+    chmod +x ${LIBPQ_BUILD_PREFIX}/lib/{libldap,liblber}*.so*
+    cd ..
+
+fi
+
+
+# Build libpq if needed
+postgres_tag="REL_${postgres_version//./_}"
+postgres_dir="postgres-${postgres_tag}"
+if [ ! -d "${postgres_dir}" ]; then
+    curl -sL \
+        https://github.com/postgres/postgres/archive/${postgres_tag}.tar.gz \
+        | tar xzf -
+
+    cd "${postgres_dir}"
+
+    # Match the default unix socket dir default with what defined on Ubuntu and
+    # Red Hat, which seems the most common location
+    sed -i 's|#define DEFAULT_PGSOCKET_DIR .*'\
+'|#define DEFAULT_PGSOCKET_DIR "/var/run/postgresql"|' \
+        src/include/pg_config_manual.h
+
+    # Often needed, but currently set by the workflow
+    # export LD_LIBRARY_PATH="${LIBPQ_BUILD_PREFIX}/lib"
+
+    ./configure --prefix=${LIBPQ_BUILD_PREFIX} --sysconfdir=/etc/postgresql-common \
+        --without-readline --with-gssapi --with-openssl --with-pam --with-ldap \
+        CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib
+    make -C src/interfaces/libpq
+    make -C src/bin/pg_config
+    make -C src/include
+else
+    cd "${postgres_dir}"
+fi
+
+# Install libpq
+make -C src/interfaces/libpq install
+make -C src/bin/pg_config install
+make -C src/include install
+cd ..
+
+find ${LIBPQ_BUILD_PREFIX} -name \*.so.\* -type f -exec strip --strip-unneeded {} \;
diff --git a/tools/build/build_macos_arm64.sh b/tools/build/build_macos_arm64.sh
new file mode 100755
index 0000000..f8c2fd7
--- /dev/null
+++ b/tools/build/build_macos_arm64.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+
+# Build psycopg-binary wheel packages for Apple M1 (cpNNN-macosx_arm64)
+#
+# This script is designed to run on Scaleway Apple Silicon machines.
+#
+# The script cannot be run as sudo (installing brew fails), but requires sudo,
+# so it can pretty much only be executed by a sudo user as it is.
+
+set -euo pipefail
+set -x
+
+python_versions="3.8.10 3.9.13 3.10.5 3.11.0"
+pg_version=15
+
+# Move to the root of the project
+dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "${dir}/../../"
+
+# Add /usr/local/bin to the path. It seems it's not, in non-interactive sessions
+if ! (echo $PATH | grep -q '/usr/local/bin'); then
+    export PATH=/usr/local/bin:$PATH
+fi
+
+# Install brew, if necessary. Otherwise just make sure it's in the path
+if [[ -x /opt/homebrew/bin/brew ]]; then
+    eval "$(/opt/homebrew/bin/brew shellenv)"
+else
+    command -v brew > /dev/null || (
+        # Not necessary: already installed
+        # xcode-select --install
+        NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL \
+            https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
+    )
+    eval "$(/opt/homebrew/bin/brew shellenv)"
+fi
+
+export PGDATA=/opt/homebrew/var/postgresql@${pg_version}
+
+# Install PostgreSQL, if necessary
+command -v pg_config > /dev/null || (
+    brew install postgresql@${pg_version}
+)
+
+# After PostgreSQL 15, the bin path is not in the path.
+export PATH=$(ls -d1 /opt/homebrew/Cellar/postgresql@${pg_version}/*/bin):$PATH
+
+# Make sure the server is running
+
+# Currently not working
+# brew services start postgresql@${pg_version}
+
+if ! pg_ctl status; then
+    pg_ctl -l /opt/homebrew/var/log/postgresql@${pg_version}.log start
+fi
+
+
+# Install the Python versions we want to build
+for ver3 in $python_versions; do
+    ver2=$(echo $ver3 | sed 's/\([^\.]*\)\(\.[^\.]*\)\(.*\)/\1\2/')
+    command -v python${ver2} > /dev/null || (
+        (cd /tmp &&
+            curl -fsSl -O \
+                https://www.python.org/ftp/python/${ver3}/python-${ver3}-macos11.pkg)
+        sudo installer -pkg /tmp/python-${ver3}-macos11.pkg -target /
+    )
+done
+
+# Create a virtualenv where to work
+if [[ ! -x .venv/bin/python ]]; then
+    python3 -m venv .venv
+fi
+
+source .venv/bin/activate
+pip install cibuildwheel
+
+# Create the psycopg_binary source package
+rm -rf psycopg_binary
+python tools/build/copy_to_binary.py
+
+# Build the binary packages
+export CIBW_PLATFORM=macos
+export CIBW_ARCHS=arm64
+export CIBW_BUILD='cp{38,39,310,311}-*'
+export CIBW_TEST_REQUIRES="./psycopg[test] ./psycopg_pool"
+export CIBW_TEST_COMMAND="pytest {project}/tests -m 'not slow and not flakey' --color yes"
+
+export PSYCOPG_IMPL=binary
+export PSYCOPG_TEST_DSN="dbname=postgres"
+export PSYCOPG_TEST_WANT_LIBPQ_BUILD=">= ${pg_version}"
+export PSYCOPG_TEST_WANT_LIBPQ_IMPORT=">= ${pg_version}"
+
+cibuildwheel psycopg_binary
diff --git a/tools/build/ci_test.sh b/tools/build/ci_test.sh
new file mode 100755
index 0000000..d1d2ee4
--- /dev/null
+++ b/tools/build/ci_test.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Run the tests in Github Action
+#
+# Failed tests run up to three times, to take into account flakey tests.
+# Of course the random generator is not re-seeded between runs, in order to
+# repeat the same result.
+
+set -euo pipefail
+set -x
+
+# Assemble a markers expression from the MARKERS and NOT_MARKERS env vars
+markers=""
+for m in ${MARKERS:-}; do
+    [[ "$markers" != "" ]] && markers="$markers and"
+    markers="$markers $m"
+done
+for m in ${NOT_MARKERS:-}; do
+    [[ "$markers" != "" ]] && markers="$markers and"
+    markers="$markers not $m"
+done
+
+pytest="python -bb -m pytest --color=yes"
+
+$pytest -m "$markers" "$@" && exit 0
+
+$pytest -m "$markers" --lf --randomly-seed=last "$@" && exit 0
+
+$pytest -m "$markers" --lf --randomly-seed=last "$@"
diff --git a/tools/build/copy_to_binary.py b/tools/build/copy_to_binary.py
new file mode 100755
index 0000000..7cab25c
--- /dev/null
+++ b/tools/build/copy_to_binary.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+# Create the psycopg-binary package by renaming and patching psycopg-c
+
+import os
+import re
+import shutil
+from pathlib import Path
+from typing import Union
+
+curdir = Path(__file__).parent
+pdir = curdir / "../.."
+target = pdir / "psycopg_binary"
+
+if target.exists():
+    raise Exception(f"path {target} already exists")
+
+
+def sed_i(pattern: str, repl: str, filename: Union[str, Path]) -> None:
+    with open(filename, "rb") as f:
+        data = f.read()
+    newdata = re.sub(pattern.encode("utf8"), repl.encode("utf8"), data)
+    if newdata != data:
+        with open(filename, "wb") as f:
+            f.write(newdata)
+
+
+shutil.copytree(pdir / "psycopg_c", target)
+shutil.move(str(target / "psycopg_c"), str(target / "psycopg_binary"))
+shutil.move(str(target / "README-binary.rst"), str(target / "README.rst"))
+sed_i("psycopg-c", "psycopg-binary", target / "setup.cfg")
+sed_i(
+    r"__impl__\s*=.*", '__impl__ = "binary"', target / "psycopg_binary/pq.pyx"
+)
+for dirpath, dirnames, filenames in os.walk(target):
+    for filename in filenames:
+        if os.path.splitext(filename)[1] not in (".pyx", ".pxd", ".py"):
+            continue
+        sed_i(r"\bpsycopg_c\b", "psycopg_binary", Path(dirpath) / filename)
diff --git a/tools/build/print_so_versions.sh b/tools/build/print_so_versions.sh
new file mode 100755
index 0000000..a3c4ecd
--- /dev/null
+++ b/tools/build/print_so_versions.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Take a .so file as input and print the Debian packages and versions of the
+# libraries it links.
+
+set -euo pipefail
+# set -x
+
+source /etc/os-release
+
+sofile="$1"
+
+case "$ID" in
+    alpine)
+        depfiles=$( (ldd "$sofile" 2>/dev/null || true) | grep '=>' | sed 's/.*=> \(.*\) (.*)/\1/')
+        (for depfile in $depfiles; do
+             echo "$(basename "$depfile") => $(apk info --who-owns "${depfile}" | awk '{print $(NF)}')"
+        done) | sort | uniq
+        ;;
+
+    debian)
+        depfiles=$(ldd "$sofile" | grep '=>' | sed 's/.*=> \(.*\) (.*)/\1/')
+        (for depfile in $depfiles; do
+            pkgname=$(dpkg -S "${depfile}" | sed 's/\(\): .*/\1/')
+            dpkg -l "${pkgname}" | grep '^ii' | awk '{print $2 " => " $3}'
+        done) | sort | uniq
+        ;;
+
+    centos)
+        echo "TODO!"
+        ;;
+
+    *)
+        echo "$0: unexpected Linux distribution: '$ID'" >&2
+        exit 1
+        ;;
+esac
diff --git a/tools/build/run_build_macos_arm64.sh b/tools/build/run_build_macos_arm64.sh
new file mode 100755
index 0000000..f5ae617
--- /dev/null
+++ b/tools/build/run_build_macos_arm64.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Build psycopg-binary wheel packages for Apple M1 (cpNNN-macosx_arm64)
+#
+# This script is designed to run on a local machine: it will clone the repos
+# remotely and execute the `build_macos_arm64.sh` script remotely, then will
+# download the built packages. A tag to build must be specified.
+#
+# In order to run the script, the `m1` host must be specified in
+# `~/.ssh/config`; for instance:
+#
+#   Host m1
+#     User m1
+#     HostName 1.2.3.4
+
+set -euo pipefail
+# set -x
+
+tag=${1:-}
+
+if [[ ! "${tag}" ]]; then
+    echo "Usage: $0 TAG" >&2
+    exit 2
+fi
+
+rdir=psycobuild
+
+# Clone the repos
+ssh m1 rm -rf "${rdir}"
+ssh m1 git clone https://github.com/psycopg/psycopg.git --branch ${tag} "${rdir}"
+
+# Allow sudoing without password, to allow brew to install
+ssh -t m1 bash -c \
+    'test -f /etc/sudoers.d/m1 || echo "m1 ALL=(ALL) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/m1'
+
+# Build the wheel packages
+ssh m1 "${rdir}/tools/build/build_macos_arm64.sh"
+
+# Transfer the packages locally
+scp -r "m1:${rdir}/wheelhouse" .
diff --git a/tools/build/strip_wheel.sh b/tools/build/strip_wheel.sh
new file mode 100755
index 0000000..bfcd302
--- /dev/null
+++ b/tools/build/strip_wheel.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Strip symbols inplace from the libraries in a zip archive.
+#
+# Stripping symbols is beneficial (reduction of 30% of the final package, >
+# %90% of the installed libraries. However just running `auditwheel repair
+# --strip` breaks some of the libraries included from the system, which fail at
+# import with errors such as "ELF load command address/offset not properly
+# aligned".
+#
+# System libraries are already pretty stripped. Ours go around 24Mb -> 1.5Mb...
+#
+# This script is designed to run on a wheel archive before auditwheel.
+
+set -euo pipefail
+# set -x
+
+source /etc/os-release
+dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+wheel=$(realpath "$1")
+shift
+
+tmpdir=$(mktemp -d)
+trap "rm -r ${tmpdir}" EXIT
+
+cd "${tmpdir}"
+python -m zipfile -e "${wheel}" .
+
+echo "
+Libs before:"
+# Busybox doesn't have "find -ls"
+find . -name \*.so | xargs ls -l
+
+# On Debian, print the package versions libraries come from
+echo "
+Dependencies versions of '_psycopg.so' library:"
+"${dir}/print_so_versions.sh" "$(find . -name \*_psycopg\*.so)"
+
+find . -name \*.so -exec strip "$@" {} \;
+
+echo "
+Libs after:"
+find . -name \*.so | xargs ls -l
+
+python -m zipfile -c ${wheel} *
+
+cd -
diff --git a/tools/build/wheel_linux_before_all.sh b/tools/build/wheel_linux_before_all.sh
new file mode 100755
index 0000000..663e3ef
--- /dev/null
+++ b/tools/build/wheel_linux_before_all.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Configure the libraries needed to build wheel packages on linux.
+# This script is designed to be used by cibuildwheel as CIBW_BEFORE_ALL_LINUX
+
+set -euo pipefail
+set -x
+
+dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+source /etc/os-release
+
+# Install PostgreSQL development files.
+case "$ID" in
+    alpine)
+        # tzdata is required for datetime tests.
+        apk update
+        apk add --no-cache tzdata
+        "${dir}/build_libpq.sh" > /dev/null
+        ;;
+
+    debian)
+        # Note that the pgdg doesn't have an aarch64 repository so wheels are
+        # build with the libpq packaged with Debian 9, which is 9.6.
+        if [ "$AUDITWHEEL_ARCH" != 'aarch64' ]; then
+            echo "deb http://apt.postgresql.org/pub/repos/apt $VERSION_CODENAME-pgdg main" \
+                > /etc/apt/sources.list.d/pgdg.list
+            # TODO: On 2021-11-09 curl fails on 'ppc64le' with:
+            #   curl: (60) SSL certificate problem: certificate has expired
+            # Test again later if -k can be removed.
+            curl -skf https://www.postgresql.org/media/keys/ACCC4CF8.asc \
+                > /etc/apt/trusted.gpg.d/postgresql.asc
+        fi
+
+        apt-get update
+        apt-get -y upgrade
+        apt-get -y install libpq-dev
+        ;;
+
+    centos)
+        "${dir}/build_libpq.sh" > /dev/null
+        ;;
+
+    *)
+        echo "$0: unexpected Linux distribution: '$ID'" >&2
+        exit 1
+        ;;
+esac
diff --git a/tools/build/wheel_macos_before_all.sh b/tools/build/wheel_macos_before_all.sh
new file mode 100755
index 0000000..285a063
--- /dev/null
+++ b/tools/build/wheel_macos_before_all.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Configure the environment needed to build wheel packages on Mac OS.
+# This script is designed to be used by cibuildwheel as CIBW_BEFORE_ALL_MACOS
+
+set -euo pipefail
+set -x
+
+dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+brew update
+brew install gnu-sed postgresql@14
+# Fetch 14.1 if 14.0 is still the default version
+brew reinstall postgresql
+
+# Start the database for testing
+brew services start postgresql
+
+# Wait for postgres to come up
+for i in $(seq 10 -1 0); do
+  eval pg_isready && break
+  if [ $i == 0 ]; then
+      echo "PostgreSQL service not ready, giving up"
+      exit 1
+  fi
+  echo "PostgreSQL service not ready, waiting a bit, attempts left: $i"
+  sleep 5
+done
diff --git a/tools/build/wheel_win32_before_build.bat b/tools/build/wheel_win32_before_build.bat
new file mode 100644
index 0000000..fd35f5d
--- /dev/null
+++ b/tools/build/wheel_win32_before_build.bat
@@ -0,0 +1,3 @@
+@echo on
+pip install delvewheel
+choco upgrade postgresql