diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/announce-release | 279 | ||||
| -rwxr-xr-x | scripts/backport | 146 | ||||
| -rwxr-xr-x | scripts/build-ot.sh | 27 | ||||
| -rwxr-xr-x | scripts/build-ssl.sh | 208 | ||||
| -rwxr-xr-x | scripts/build-vtest.sh | 10 | ||||
| -rwxr-xr-x | scripts/create-release | 237 | ||||
| -rwxr-xr-x | scripts/git-show-backports | 336 | ||||
| -rwxr-xr-x | scripts/make-releases-json | 103 | ||||
| -rwxr-xr-x | scripts/publish-release | 194 | ||||
| -rwxr-xr-x | scripts/run-regtests.sh | 427 |
10 files changed, 1967 insertions, 0 deletions
diff --git a/scripts/announce-release b/scripts/announce-release new file mode 100755 index 0000000..c990821 --- /dev/null +++ b/scripts/announce-release @@ -0,0 +1,279 @@ +#!/usr/bin/env bash +# prepares a template e-mail and HTML file to announce a new release +# Copyright (c) 2006-2016 Willy Tarreau <w@1wt.eu> +# +# In short : +# - requires git +# - wants that last commit is a release/tag +# - no restriction to master, uses last tag +# - creates mail-$version.txt +# - creates web-$version.html +# - indicates how to edit the mail and how to send it + +USAGE="Usage: ${0##*/} [-f] [-p] [-b branch] [-d date] [-o oldver] [-n newver] + -f: force to overwrite existing files and ignore local changes + -p: prepare future release (skip branch and tags existence checks) + -b: force the project branch name to this (def: inherited from the version) + -d: force the release date (e.g. to rework a failed announce) + -o: previous version (def: newver-1) + -n: new version (if not last tag) +" +PREPARE= +FORCE= +OUTPUT= +BRANCH= +HTML= +DATE= +YEAR= +OLD= +LASTCOM= +NEWVER= +NEWTAG= +DIR= + +die() { + [ "$#" -eq 0 ] || echo "$*" >&2 + exit 1 +} + +err() { + echo "$*" >&2 +} + +quit() { + [ "$#" -eq 0 ] || echo "$*" + exit 0 +} + +while [ -n "$1" -a -z "${1##-*}" ]; do + case "$1" in + -d) DATE="$2" ; shift 2 ;; + -b) BRANCH="$2" ; shift 2 ;; + -f) FORCE=1 ; shift ;; + -p) PREPARE=1 ; shift ;; + -o) OLD="$2" ; shift 2 ;; + -n) NEWVER="$2" ; shift 2 ;; + -h|--help) quit "$USAGE" ;; + *) die "$USAGE" ;; + esac +done + +if [ $# -gt 0 ]; then + die "$USAGE" +fi + +if ! git rev-parse --verify -q HEAD >/dev/null; then + die "Failed to check git HEAD." +fi + +# we want to go to the git root dir +DIR="$PWD" +cd $(git rev-parse --show-toplevel) + +if [ -z "$FORCE" -a "$(git diff HEAD|wc -c)" != 0 ]; then + err "You appear to have uncommitted local changes, please commit them first :" + git status -s -uno >&2 + die +fi + +if [ -z "$PREPARE" -a "$(git rev-parse --verify -q HEAD)" != "$(git rev-parse --verify -q master)" ]; then + die "git HEAD doesn't match master branch." +fi + +if [ -n "$NEWVER" ]; then + if git show-ref --tags "v$NEWVER" >/dev/null; then + NEWTAG="v$NEWVER" + else + echo "Note: no matching tag v$NEWVER, using HEAD". + fi +fi + +# version unspecified or no existing tag for it +if [ -z "$NEWTAG" ]; then + NEWTAG="$(git describe --tags HEAD --abbrev=0)" + + if [ -z "$NEWTAG" ]; then + die "Fatal: cannot determine new version, please specify it." + elif [ -n "$PREPARE" ] && ! git show-ref --tags HEAD >/dev/null; then + # HEAD not tagged, hence we have to pretend we're on one version + # after the current tag + echo "Current version not tagged, trying to determine next one." + NEWTAG="${NEWTAG#v}" + if [ -z "$OLD" ]; then + OLD="$NEWTAG" + fi + radix="$NEWTAG" + while [ -n "$radix" -a -z "${radix%%*[0-9]}" ]; do + radix="${radix%[0-9]}" + done + + number=${NEWTAG#$radix} + if [ -z "$number" -o "$radix" = "$NEWTAG" ]; then + die "Fatal: cannot determine new version, please specify it." + fi + NEWTAG="${radix}$((number+1))" + if [ -z "$NEWVER" ]; then + NEWVER="${NEWTAG}" + fi + NEWTAG="v$NEWTAG" + LASTCOM="$(git rev-parse --short HEAD)" + echo "Next version expected to be $NEWVER and next tag $NEWTAG based on commit $LASTCOM" + elif [ "$(git describe --tags HEAD)" != "$NEWTAG" ]; then + die "About to use current HEAD which doesn't seem tagged, it reports '$(git describe --tags HEAD 2>/dev/null)'. Did you release it ?" + fi +elif ! git show-ref --tags "$NEWTAG" >/dev/null 2>&1; then + die "git tag $NEWTAG doesn't exist, did you create the release ?" +fi + +if [ -z "$NEWVER" ]; then + NEWVER="${NEWTAG#v}" +fi + +if [ -z "$LASTCOM" ]; then + LASTCOM="$(git rev-parse --short ${NEWTAG}^)" +fi + +if [ -z "$OLD" ]; then + OLD="$(git describe --tags ${LASTCOM} --abbrev=0)" + OLD="${OLD#v}" +fi + +if ! git rev-parse --verify -q "v$OLD" >/dev/null; then + die "git tag v$OLD doesn't exist." +fi + +# determine the product branch from the new release +if [ -z "$BRANCH" ]; then + subvers=${NEWVER#[0-9]*.[0-9]*[-.]*[0-9].} + [ "${subvers}" = "${NEWVER}" ] && subvers="" + major=${NEWVER%.$subvers} + branch_ext=${major#*[0-9].*[0-9]} + BRANCH=${major%${branch_ext}} +fi + +# determine the release date +if [ -z "$DATE" ]; then + DATE="$(git log -1 --pretty=fuller ${NEWTAG} 2>/dev/null | sed -ne '/^CommitDate:/{s/\(^[^ ]*:\)\|\( [-+].*\)//gp;q}')" + DATE="$(date +%Y/%m/%d -d "$DATE")" +fi +YEAR="${DATE%%/*}" + +OUTPUT="$DIR/mail-haproxy-$NEWVER.txt" +HTML="$DIR/web-haproxy-$NEWVER.html" + +[ -z "$FORCE" ] || rm -f "${OUTPUT}" "${HTML}" + +if [ -e "$OUTPUT" ]; then + die "${OUTPUT##*/} already exists, please remove it or retry with -f." +fi + +if [ -e "$HTML" ]; then + die "$HTML already exists, please remove it or retry with -f." +fi + +( + echo "# Send this using:" + echo "# mutt -H <(tail -n +4 ${OUTPUT##*/}) -s \"[ANNOUNCE] haproxy-$NEWVER\" haproxy@formilux.org" +) >> "$OUTPUT" + +(echo + echo "Hi," + echo + echo -n "HAProxy $NEWVER was released on $DATE. It added " + echo -n $(git log --oneline --reverse --format="%s" "v$OLD".."$LASTCOM" | wc -l) + echo " new commits" + echo "after version $OLD." + echo + echo "- per tag :" + git log --oneline --reverse --format="%s" "v$OLD".."$LASTCOM" | cut -f1 -d':' | sort | uniq -c + echo + echo "major commits :" + git log --oneline --reverse --format=" - %s" "v$OLD".."$LASTCOM" | grep MAJOR + echo + echo "- per file :" + git show "v$OLD".."$LASTCOM" -- src/ | grep ^diff | awk '{ print substr($3,7)}' | sort | uniq -c | sort -nr | head -15 + echo + echo "- per topic :" + git log --oneline --reverse --format="%s" "v$OLD".."$LASTCOM" | cut -f2 -d':' | awk '{sub("s$","",$1); print $1}' | sort | uniq -c + echo + echo "- sorted changelog :" + git log --oneline --reverse --format="%s" "v$OLD".."$LASTCOM" | sort + echo + echo "#############################################################################################" +) >> "$OUTPUT" + +# report the download paths +if [ -z "${NEWVER##*-dev*}" ]; then + gitdir="haproxy.git" +else + gitdir="haproxy-$BRANCH.git" +fi + +(echo "Please find the usual URLs below :" + echo " Site index : https://www.haproxy.org/" + echo " Documentation : https://docs.haproxy.org/" + echo " Wiki : https://github.com/haproxy/wiki/wiki" + echo " Discourse : https://discourse.haproxy.org/" + echo " Slack channel : https://slack.haproxy.org/" + echo " Issue tracker : https://github.com/haproxy/haproxy/issues" + echo " Sources : https://www.haproxy.org/download/${BRANCH}/src/" + echo " Git repository : https://git.haproxy.org/git/${gitdir}/" + echo " Git Web browsing : https://git.haproxy.org/?p=${gitdir}" + echo " Changelog : https://www.haproxy.org/download/${BRANCH}/src/CHANGELOG" + echo " Dataplane API : https://github.com/haproxytech/dataplaneapi/releases/latest" + echo " Pending bugs : https://www.haproxy.org/l/pending-bugs" + echo " Reviewed bugs : https://www.haproxy.org/l/reviewed-bugs" + echo " Code reports : https://www.haproxy.org/l/code-reports" + echo " Latest builds : https://www.haproxy.org/l/dev-packages" +) >> "$OUTPUT" + +# sign +(echo + echo "${GIT_COMMITTER_NAME% *}" +) >> "$OUTPUT" + +(echo "---" + echo "Complete changelog :" + git shortlog "v$OLD".."$LASTCOM" + echo "---" +) >> "$OUTPUT" + + +# prepare the HTML update +set -- $(date +%e -d "$DATE") +case "$1" in + 11|12|13) day="${1}th" ;; + *1) day="${1}st" ;; + *2) day="${2}nd" ;; + *3) day="${1}rd" ;; + *) day="${1}th" ;; +esac + +humandate=$(date "+%B, $day, %Y" -d "$DATE") +(echo "$humandate</b> : <i>$NEWVER</i>" + echo " <p>" + echo " <ul>" + echo "<--------------------------- edit contents below --------------------------->" + echo "- per tag :" + git log --oneline --reverse --format="%s" "v$OLD".."$LASTCOM" | cut -f1 -d':' | sort | uniq -c + echo + echo "- per topic :" + git log --oneline --reverse --format="%s" "v$OLD".."$LASTCOM" | cut -f2 -d':' | awk '{sub("s$","",$1); print $1}' | sort | uniq -c + echo + echo "major commits :" + git log --oneline --reverse --format=" - %s" "v$OLD".."$LASTCOM" | grep MAJOR + echo + echo "<--------------------------------------------------------------------------->" + echo " Code and changelog are available <a href=\"/download/${BRANCH}/src/\">here</a> as usual." + echo " </ul>" + echo " <p>" + echo " <b>" +) >> "$HTML" + +echo "The announce was emitted into file $OUTPUT." +echo "You can edit it and send it this way :" +echo +echo " mutt -H <(tail -n +4 ${OUTPUT##*/}) -s \"[ANNOUNCE] haproxy-$NEWVER\" haproxy@formilux.org" +echo +echo "The HTML block was emitted into $HTML and needs to be finished by hand." +echo diff --git a/scripts/backport b/scripts/backport new file mode 100755 index 0000000..4f60140 --- /dev/null +++ b/scripts/backport @@ -0,0 +1,146 @@ +#!/usr/bin/env bash + +USAGE="Usage: ${0##*/} <last> <commit> [...]" +START="$PWD" +LAST= +UPSTREAM= +COMMIT= +BRANCH= + +die() { + [ "$#" -eq 0 ] || echo "$*" >&2 + exit 1 +} + +err() { + echo "$*" >&2 +} + +quit() { + [ "$#" -eq 0 ] || echo "$*" + exit 0 +} + +short() { + git rev-parse --short "$1" +} + +# returns the latest commit ID in $REPLY. Returns 0 on success, non-zero on +# failure with $REPLY empty. +get_last_commit() { + REPLY=$(git rev-parse HEAD) + test -n "$REPLY" +} + +# returns the name of the current branch (1.8, 1.9, etc) in $REPLY. Returns 0 +# on success, non-zero on failure with $REPLY empty. +get_branch() { + local major subver ext + REPLY=$(git describe --tags HEAD --abbrev=0 2>/dev/null) + REPLY=${REPLY#v} + subver=${REPLY#[0-9]*.[0-9]*[-.]*[0-9].} + [ "${subver}" != "${REPLY}" ] || subver="" + major=${REPLY%.$subver} + ext=${major#*[0-9].*[0-9]} + REPLY=${major%${ext}} + test -n "$REPLY" +} + +# returns the path to the next "up" remote in $REPLY, and zero on success +# or non-zero when the last one was reached. +up() { + REPLY=$(git remote -v | awk '/^up\t.*\(fetch\)$/{print $2}') + test -n "$REPLY" +} + +# returns the path to the next "down" remote in $REPLY, and zero on success +# or non-zero when the last one was reached. +down() { + REPLY=$(git remote -v | awk '/^down\t.*\(fetch\)$/{print $2}') + test -n "$REPLY" +} + +# verifies that the repository is clean of any pending changes +check_clean() { + test -z "$(git status -s -uno)" +} + +# verifies that HEAD is the master +check_master() { + test "$(git rev-parse --verify -q HEAD 2>&1)" = "$(git rev-parse --verify -q master 2>&1)" +} + +# tries to switch to the master branch, only if the current one is clean. Dies on failure. +switch_master() { + check_clean || die "$BRANCH: local changes, stopping on commit $COMMIT (upstream $UPSTREAM)" + git checkout master >/dev/null 2>&1 || die "$BRANCH: failed to checkout master, stopping on commit $COMMIT (upstream $UPSTREAM)" +} + +# walk up to the first repo +walk_up() { + cd "$START" +} + +# updates the "up" remote repository. Returns non-zero on error. +update_up() { + git remote update up >/dev/null 2>&1 +} + +# backports commit "$1" with a signed-off by tag. In case of failure, aborts +# the change and returns non-zero. Unneeded cherry-picks do return an error +# because we don't want to accidentally backport the latest commit instead of +# this one, and we don't know this one's ID. +backport_commit() { + local empty=1 + + if ! git cherry-pick -sx "$1"; then + [ -n "$(git diff)" -o -n "$(git diff HEAD)" ] || empty=0 + git cherry-pick --abort + return 1 + fi +} + +[ "$1" != "-h" -a "$1" != "--help" ] || quit "$USAGE" +[ -n "$1" -a -n "$2" ] || die "$USAGE" + +LAST="$1" +shift + +# go back to the root of the repo +cd $(git rev-parse --show-toplevel) +START="$PWD" + +while [ -n "$1" ]; do + UPSTREAM="$(short $1)" + [ -n "$UPSTREAM" ] || die "branch $BRANCH: unknown commit ID $1, cannot backport." + COMMIT="$UPSTREAM" + BRANCH="-source-" + while :; do + if ! down; then + err "branch $BRANCH: can't go further, is repository 'down' properly set ?" + break + fi + + cd "$REPLY" || die "Failed to 'cd' to '$REPLY' from '$PWD', is repository 'down' properly set ?" + + check_clean || die "Local changes in $PWD, stopping before backporting commit $COMMIT (upstream $UPSTREAM)" + + check_master || switch_master || die "Cannot switch to 'master' branch in $PWD, stopping before backporting commit $COMMIT (upstream $UPSTREAM)" + get_branch || die "Failed to get branch name in $PWD, stopping before backporting commit $COMMIT (upstream $UPSTREAM)" + BRANCH="$REPLY" + + update_up || die "$BRANCH: failed to update repository 'up', stopping before backporting commit $COMMIT (upstream $UPSTREAM)" + + backport_commit "$COMMIT" || die "$BRANCH: failed to backport commit $COMMIT (upstream $UPSTREAM). Leaving repository $PWD intact." + + if [ "$BRANCH" = "$LAST" ]; then + # reached the stop point, don't apply further + break + fi + + get_last_commit || die "$BRANCH: cannot retrieve last commit ID, stopping after backporting commit $COMMIT (upstream $UPSTREAM)" + COMMIT="$(short $REPLY)" + done + walk_up || die "Failed to go back to $PWD, stopping *after* backporting upstream $UPSTREAM" + shift +done diff --git a/scripts/build-ot.sh b/scripts/build-ot.sh new file mode 100755 index 0000000..fb128b3 --- /dev/null +++ b/scripts/build-ot.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# +# OT helper. script built from documentation: https://github.com/haproxytech/opentracing-c-wrapper +# + +set -e + +OT_CPP_VERSION="${OT_CPP_VERSION:-1.6.0}" +OT_PREFIX="${OT_PREFIX:-${HOME}/opt}" + +wget -P download-cache/ "https://github.com/opentracing/opentracing-cpp/archive/v${OT_CPP_VERSION}.tar.gz" + +tar xf download-cache/v${OT_CPP_VERSION}.tar.gz +cd opentracing-cpp-${OT_CPP_VERSION} +mkdir build +cd build +cmake -DCMAKE_INSTALL_PREFIX=${OT_PREFIX} -DBUILD_STATIC_LIBS=OFF -DBUILD_MOCKTRACER=OFF -DBUILD_TESTING=OFF .. +make -j$(nproc) +make install + +git clone https://github.com/haproxytech/opentracing-c-wrapper.git +cd opentracing-c-wrapper + ./scripts/bootstrap + ./configure --prefix=${OT_PREFIX} --with-opentracing=${OT_PREFIX} + make -j$(nproc) + make install diff --git a/scripts/build-ssl.sh b/scripts/build-ssl.sh new file mode 100755 index 0000000..1c17775 --- /dev/null +++ b/scripts/build-ssl.sh @@ -0,0 +1,208 @@ +#!/bin/sh +set -eux + +download_openssl () { + if [ ! -f "download-cache/openssl-${OPENSSL_VERSION}.tar.gz" ]; then + +# +# OpenSSL has different links for latest and previous releases +# since we want to download several versions, let us try to treat +# current version as latest, if it fails, follow with previous +# + + wget -P download-cache/ \ + "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz" || \ + wget -P download-cache/ \ + "https://www.openssl.org/source/old/${OPENSSL_VERSION%[a-z]}/openssl-${OPENSSL_VERSION}.tar.gz" + fi +} + +# recent openssl versions support parallel builds and skipping the docs, +# while older ones require to build everything sequentially. +build_openssl_linux () { + ( + cd "openssl-${OPENSSL_VERSION}/" + ./config shared --prefix="${HOME}/opt" --openssldir="${HOME}/opt" --libdir=lib -DPURIFY + if [ -z "${OPENSSL_VERSION##1.*}" ]; then + make all + else + make -j$(nproc) build_sw + fi + make install_sw + ) +} + +build_openssl_osx () { + ( + cd "openssl-${OPENSSL_VERSION}/" + ./Configure darwin64-x86_64-cc shared \ + --prefix="${HOME}/opt" --openssldir="${HOME}/opt" --libdir=lib -DPURIFY + make depend build_sw install_sw + ) +} + +build_openssl () { + if [ "$(cat ${HOME}/opt/.openssl-version)" != "${OPENSSL_VERSION}" ]; then + tar zxf "download-cache/openssl-${OPENSSL_VERSION}.tar.gz" + case `uname` in + 'Darwin') + build_openssl_osx + ;; + 'Linux') + build_openssl_linux + ;; + esac + echo "${OPENSSL_VERSION}" > "${HOME}/opt/.openssl-version" + fi +} + +download_libressl () { + if [ ! -f "download-cache/libressl-${LIBRESSL_VERSION}.tar.gz" ]; then + wget -P download-cache/ \ + "https://cdn.openbsd.org/pub/OpenBSD/LibreSSL/libressl-${LIBRESSL_VERSION}.tar.gz" + fi +} + +build_libressl () { + if [ "$(cat ${HOME}/opt/.libressl-version)" != "${LIBRESSL_VERSION}" ]; then + tar zxf "download-cache/libressl-${LIBRESSL_VERSION}.tar.gz" + ( + cd "libressl-${LIBRESSL_VERSION}/" + ./configure --prefix="${HOME}/opt" + make all install + ) + echo "${LIBRESSL_VERSION}" > "${HOME}/opt/.libressl-version" + fi +} + +download_boringssl () { + if [ ! -d "download-cache/boringssl" ]; then + git clone --depth=1 https://boringssl.googlesource.com/boringssl download-cache/boringssl + else + ( + cd download-cache/boringssl + git pull + ) + fi +} + +download_aws_lc () { + if [ ! -f "download-cache/aws-lc-${AWS_LC_VERSION}.tar.gz" ]; then + mkdir -p download-cache + wget -q -O "download-cache/aws-lc-${AWS_LC_VERSION}.tar.gz" \ + "https://github.com/aws/aws-lc/archive/refs/tags/v${AWS_LC_VERSION}.tar.gz" + fi +} + +build_aws_lc () { + if [ "$(cat ${HOME}/opt/.aws_lc-version)" != "${AWS_LC_VERSION}" ]; then + tar zxf "download-cache/aws-lc-${AWS_LC_VERSION}.tar.gz" + ( + cd "aws-lc-${AWS_LC_VERSION}/" + mkdir -p build + cd build + cmake -version + cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=1 -DDISABLE_GO=1 -DDISABLE_PERL=1 \ + -DBUILD_TESTING=0 -DCMAKE_INSTALL_PREFIX=${HOME}/opt .. + make -j$(nproc) + make install + ) + echo "${AWS_LC_VERSION}" > "${HOME}/opt/.aws_lc-version" + fi +} + +download_quictls () { + if [ ! -d "download-cache/quictls" ]; then + git clone --depth=1 https://github.com/quictls/openssl download-cache/quictls + else + ( + cd download-cache/quictls + git pull + ) + fi +} + +download_wolfssl () { + if [ ! -f "download-cache/wolfssl-${WOLFSSL_VERSION}.tar.gz" ]; then + mkdir -p download-cache + if [ "${WOLFSSL_VERSION%%-*}" != "git" ]; then + wget -q -O "download-cache/wolfssl-${WOLFSSL_VERSION}.tar.gz" \ + "https://github.com/wolfSSL/wolfssl/archive/refs/tags/v${WOLFSSL_VERSION}-stable.tar.gz" + else + wget -q -O "download-cache/wolfssl-${WOLFSSL_VERSION}.tar.gz" \ + "https://github.com/wolfSSL/wolfssl/archive/${WOLFSSL_VERSION##git-}.tar.gz" + fi + fi +} + +build_wolfssl () { + if [ "$(cat ${HOME}/opt/.wolfssl-version)" != "${WOLFSSL_VERSION}" ]; then + mkdir "wolfssl-${WOLFSSL_VERSION}/" + tar zxf "download-cache/wolfssl-${WOLFSSL_VERSION}.tar.gz" -C "wolfssl-${WOLFSSL_VERSION}/" --strip-components=1 + ( + cd "wolfssl-${WOLFSSL_VERSION}/" + autoreconf -i + ./configure --enable-haproxy --enable-quic --prefix="${HOME}/opt" + make -j$(nproc) + make install + ) + echo "${WOLFSSL_VERSION}" > "${HOME}/opt/.wolfssl-version" + fi +} + +if [ ! -z ${LIBRESSL_VERSION+x} ]; then + download_libressl + build_libressl +fi + +if [ ! -z ${OPENSSL_VERSION+x} ]; then + download_openssl + build_openssl +fi + +if [ ! -z ${BORINGSSL+x} ]; then + ( + + # travis-ci comes with go-1.11, while boringssl requires go-1.13 + eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=1.13 bash)" + + download_boringssl + cd download-cache/boringssl + if [ -d build ]; then rm -rf build; fi + mkdir build + cd build + cmake -GNinja -DCMAKE_BUILD_TYPE=release -DBUILD_SHARED_LIBS=1 .. + ninja + + rm -rf ${HOME}/opt/lib || exit 0 + rm -rf ${HOME}/opt/include || exit 0 + + mkdir -p ${HOME}/opt/lib + cp crypto/libcrypto.so ssl/libssl.so ${HOME}/opt/lib + + mkdir -p ${HOME}/opt/include + cp -r ../include/* ${HOME}/opt/include + ) +fi + +if [ ! -z ${AWS_LC_VERSION+x} ]; then + download_aws_lc + build_aws_lc +fi + +if [ ! -z ${QUICTLS+x} ]; then + ( + download_quictls + cd download-cache/quictls + + ./config shared no-tests ${QUICTLS_EXTRA_ARGS:-} --prefix="${HOME}/opt" --openssldir="${HOME}/opt" --libdir=lib -DPURIFY + make -j$(nproc) build_sw + make install_sw + + ) +fi + +if [ ! -z ${WOLFSSL_VERSION+x} ]; then + download_wolfssl + build_wolfssl +fi diff --git a/scripts/build-vtest.sh b/scripts/build-vtest.sh new file mode 100755 index 0000000..4db35d6 --- /dev/null +++ b/scripts/build-vtest.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -eux + +curl -fsSL https://github.com/vtest/VTest/archive/master.tar.gz -o VTest.tar.gz +mkdir ../vtest +tar xvf VTest.tar.gz -C ../vtest --strip-components=1 +# Special flags due to: https://github.com/vtest/VTest/issues/12 +make -C ../vtest FLAGS="-O2 -s -Wall" + diff --git a/scripts/create-release b/scripts/create-release new file mode 100755 index 0000000..b8a867c --- /dev/null +++ b/scripts/create-release @@ -0,0 +1,237 @@ +#!/usr/bin/env bash +# creates a new haproxy release at the current commit +# Copyright (c) 2006-2016 Willy Tarreau <w@1wt.eu> +# +# In short : +# - requires git +# - works only from master branch +# - finds old and new version by itself +# - builds changelog +# - updates dates and versions in files +# - commits + tags + signs +# - no upload! + +USAGE="Usage: ${0##*/} [-i] [-y] [-t] [-b branch] [-d date] [-o oldver] [-n newver]" +INTERACTIVE= +TAGONLY= +SAYYES= +BRANCH= +DATE= +YEAR= +OLD= +NEW= + +die() { + [ "$#" -eq 0 ] || echo "$*" >&2 + exit 1 +} + +err() { + echo "$*" >&2 +} + +quit() { + [ "$#" -eq 0 ] || echo "$*" + exit 0 +} + +do_commit() { + ( + echo "[RELEASE] Released version $NEW" + echo + echo "Released version $NEW with the following main changes :" + sed -ne '/^[ ]*-/,/^$/{p;b a};d;:a;/^$/q' CHANGELOG + ) | git commit -a -F - +} + +do_tag() { + git tag -u "$GIT_GPG_KEY" -s -m "HAProxy $NEW" v$NEW && echo "Tagged as v$NEW" +} + +if [ -z "$GIT_COMMITTER_NAME" ]; then + GIT_COMMITTER_NAME=$(git config --get user.name) + [ -n "$GIT_COMMITTER_NAME" ] || die "GIT_COMMITTER_NAME not set" +fi + +if [ -z "$GIT_COMMITTER_EMAIL" ]; then + GIT_COMMITTER_EMAIL=$(git config --get user.email) + [ -n "$GIT_COMMITTER_EMAIL" ] || die "GIT_COMMITTER_EMAIL not set" +fi + +while [ -n "$1" -a -z "${1##-*}" ]; do + case "$1" in + -y) SAYYES=1 ; shift ;; + -i) INTERACTIVE=1 ; shift ;; + -t) TAGONLY=1 ; shift ;; + -d) DATE="$2" ; shift 2 ;; + -b) BRANCH="$2" ; shift 2 ;; + -o) OLD="$2" ; shift 2 ;; + -n) NEW="$2" ; shift 2 ;; + -h|--help) quit "$USAGE" ;; + *) die "$USAGE" ;; + esac +done + +if [ $# -gt 0 ]; then + die "$USAGE" +fi + +if [ -z "$GIT_GPG_KEY" ]; then + die "GIT_GPG_KEY is not set, it must contain your GPG key ID." +fi + +if ! git rev-parse --verify -q HEAD >/dev/null; then + die "Failed to check git HEAD." +fi + +# we want to go to the git top dir +cd $(git rev-parse --show-toplevel) + +if [ "$(git rev-parse --verify -q HEAD)" != "$(git rev-parse --verify -q master)" ]; then + die "git HEAD doesn't match master branch." +fi + +if [ "$(git diff HEAD|wc -c)" != 0 ]; then + err "You appear to have uncommitted local changes, please commit them first :" + git status -s -uno >&2 + die +fi + +if [ -z "$OLD" ]; then + OLD="$(git describe --tags HEAD --abbrev=0)" + OLD="${OLD#v}" +fi + +if ! git rev-parse --verify -q "v$OLD" >/dev/null; then + die "git tag v$OLD doesn't exist." +fi + +if [ -z "$NEW" ]; then + radix="$OLD" + while [ -n "$radix" -a -z "${radix%%*[0-9]}" ]; do + radix="${radix%[0-9]}" + done + + number=${OLD#$radix} + if [ -z "$number" -o "$radix" = "$OLD" ]; then + die "Fatal: cannot determine new version, please specify it." + fi + NEW=${radix}$((number+1)) +fi + +if git show-ref --tags "v$NEW" >/dev/null; then + die "git tag v$NEW already exists, please remove it first." +fi + +# determine the product branch from the new release +if [ -z "$BRANCH" ]; then + subvers=${NEW#[0-9]*.[0-9]*[-.]*[0-9].} + [ "${subvers}" = "${NEW}" ] && subvers="" + major=${NEW%.$subvers} + branch_ext=${major#*[0-9].*[0-9]} + BRANCH=${major%${branch_ext}} +fi + + +# determine the release date +if [ -z "$DATE" ]; then + # Uncomment the line below to use the date of the last commit, + # otherwise fall back to current date + DATE="$(git log --pretty=fuller -1 v$NEW 2>/dev/null | sed -ne '/^CommitDate:/{s/\(^[^ ]*:\)\|\( [-+].*\)//gp;q}')" + DATE="$(date +%Y/%m/%d -d "$DATE")" +else + if [ "$DATE" != "$(date +%Y/%m/%d -d "$DATE")" ]; then + die "Date format must exclusively be YYYY/MM/DD ; date was '$DATE'." + fi +fi +YEAR="${DATE%%/*}" + +if [ -n "$TAGONLY" ]; then + do_tag || die "Failed to tag changes" + echo "Done. You may have to push changes." + exit 0 +fi + +echo "About to release version $NEW from $OLD at $DATE (branch $BRANCH)." +if [ -z "$SAYYES" ]; then + echo "Press ENTER to continue or Ctrl-C to abort now!" + read +fi + +echo "Updating CHANGELOG ..." +( echo "ChangeLog :" + echo "===========" + echo + echo "$DATE : $NEW" + #git shortlog v$OLD.. | sed -ne 's/^ / - /p' + if [ $(git log --oneline v$OLD.. | wc -l) = 0 ]; then + echo " - exact copy of $OLD" + else + git log --oneline --reverse --format=" - %s" v$OLD.. + fi + echo + tail -n +4 CHANGELOG +) >.chglog.tmp && mv .chglog.tmp CHANGELOG + +echo "Updating VERSION ..." +rm -f VERSION VERDATE +echo "$NEW" > VERSION + +echo "Updating VERDATE ..." +echo '$Format:%ci$' > VERDATE +echo "$DATE" >> VERDATE + +# updating branch and date in all modified doc files except the outdated architecture.txt +for file in doc/intro.txt doc/configuration.txt doc/management.txt $(git diff --name-only v${OLD}.. -- doc); do + if [ ! -e "$file" ]; then continue; fi + if [ "$file" = doc/architecture.txt ]; then continue; fi + echo "Updating $file ..." + sed -e "1,10s:\(\sversion\s\).*:\1$BRANCH:" \ + -e "1,10s:\(\s\)\(20[0-9]\{2\}/[0-9]\{1,2\}/[0-9]\{1,2\}\):\1$DATE:" \ + -i "$file" +done + +echo "Updating haproxy.c ..." +sed -e "s:Copyright 2000-[0-9]*\s*Willy Tarreau.*>:Copyright 2000-$YEAR Willy Tarreau <willy@haproxy.org>:" \ + -i src/haproxy.c + +echo "Updating version.h ..." +sed -e "s:^\(#define\s*PRODUCT_BRANCH\s*\)\"[^\"]*\":\1\"$BRANCH\":" \ + -i include/haproxy/version.h + +if [ -n "$INTERACTIVE" ]; then + vi CHANGELOG VERSION VERDATE \ + src/haproxy.c doc/configuration.txt \ + $(git diff --name-only v${OLD}.. -- doc) +fi + +if [ "$(git diff -- CHANGELOG | wc -c)" = 0 ]; then + die "CHANGELOG must be updated." +fi + +if [ -z "$SAYYES" ]; then + echo "Press ENTER to review the changes..." + read +fi + +git diff + +echo +echo "About to commit and tag version $NEW with the following message:" +echo +echo "[RELEASE] Released version $NEW with the following main changes :" +sed -ne '/^[ ]*-/,/^$/{p;b a};d;:a;/^$/q' CHANGELOG + +echo +echo "LAST chance to cancel! Press ENTER to proceed now or Ctrl-C to abort." +read + +do_commit || die "Failed to commit changes" +do_tag || die "Failed to tag changes" + +remote=$(git config --get branch.master.remote) +echo "Do not forget to push updates, publish and announce this version :" +echo +echo "git push ${remote:-origin} master v$NEW" +echo "${0%/*}/publish-release" +echo "${0%/*}/announce-release" diff --git a/scripts/git-show-backports b/scripts/git-show-backports new file mode 100755 index 0000000..f2c40fe --- /dev/null +++ b/scripts/git-show-backports @@ -0,0 +1,336 @@ +#!/usr/bin/env bash +# +# Compares multiple branches against a reference and shows which ones contain +# each commit, and the level of backports since the origin or its own ancestors. +# +# Copyright (c) 2016 Willy Tarreau <w@1wt.eu> +# +# The purpose is to make it easy to visualize what backports might be missing +# in a maintenance branch, and to easily spot the ones that are needed and the +# ones that are not. It solely relies on the "cherry-picked from" tags in the +# commit messages to find what commit is available where, and can even find a +# reference commit's ancestor in another branch's commit ancestors as well to +# detect that the patch is present. When done with the proper references and +# a correct ordering of the branches, it can be used to quickly apply a set of +# fixes to a branch since it dumps suggested commands at the end. When doing +# so it is a good idea to use "HEAD" as the last branch to avoid doing mistakes. +# +# Examples : +# - find what's in master and not in current branch : +# show-backports -q -m -r master HEAD +# - find what's in 1.6/master and in hapee-maint-1.5r2 but not in current branch : +# show-backports -q -m -r 1.6/master hapee-maint-1.5r2 HEAD | grep ' [a-f0-9]\{8\}[-+][0-9] ' +# - check that no recent fix from master is missing in any maintenance branch : +# show-backports -r master hapee-maint-1.5r2 aloha-7.5 hapee-maint-1.5r1 aloha-7.0 +# - see what was recently merged into 1.6 and has no equivalent in local master : +# show-backports -q -m -r 1.6/master -b "1.6/master@{1 week ago}" master +# - check what extra backports are present in hapee-r2 compared to hapee-r1 : +# show-backports -q -m -r hapee-r2 hapee-r1 + + +USAGE="Usage: ${0##*/} [-q] [-H] [-m] [-u] [-r reference] [-l logexpr] [-s subject] [-b base] {branch|range} [...] [-- file*]" +BASES=( ) +BRANCHES=( ) +REF= +BASE= +QUIET= +LOGEXPR= +SUBJECT= +MISSING= +UPSTREAM= +BODYHASH= + +die() { + [ "$#" -eq 0 ] || echo "$*" >&2 + exit 1 +} + +err() { + echo "$*" >&2 +} + +quit() { + [ "$#" -eq 0 ] || echo "$*" + exit 0 +} + +short() { + # git rev-parse --short $1 + echo "${1::8}" +} + +dump_commit_matrix() { + title=":$REF:" + for branch in "${BRANCHES[@]}"; do + #echo -n " $branch" + title="$title :${branch}:" + done + title="$title |" + + count=0 + # now look up commits + while read ref subject; do + if [ -n "$MISSING" -a "${subject:0:9}" = "[RELEASE]" ]; then + continue + fi + + upstream="none" + missing=0 + refbhash="" + line="" + for branch in "${BRANCHES[@]}"; do + set -- $(grep -m 1 $ref "$WORK/${branch//\//_}") + newhash=$1 ; shift + bhash="" + # count the number of cherry-picks after this one. Since we shift, + # the result is in "$#" + while [ -n "$1" -a "$1" != "$ref" ]; do + shift + done + if [ -n "$newhash" ]; then + line="${line} $(short $newhash)-$#" + else + # before giving up we can check if our current commit was + # itself cherry-picked and check this again. In order not + # to have to do it all the time, we can cache the result + # for the current line. If a match is found we report it + # with the '+' delimiter instead of '-'. + if [ "$upstream" = "none" ]; then + upstream=( $(git log -1 --pretty --format=%B "$ref" | \ + sed -n 's/^commit \([^)]*\) upstream\.$/\1/p;s/^(cherry picked from commit \([^)]*\))/\1/p') ) + fi + newhash="" + for h in ${upstream[@]}; do + set -- $(grep -m 1 $h "$WORK/${branch//\//_}") + newhash=$1 ; shift + while [ -n "$1" -a "$1" != "$h" ]; do + shift + done + if [ -n "$newhash" ]; then + line="${line} $(short $newhash)+$#" + break + fi + done + if [ -z "$newhash" -a -n "$BODYHASH" ]; then + if [ -z "$refbhash" ]; then + refbhash=$(git log -1 --pretty="%an|%ae|%at|%B" "$ref" | sed -n '/^\(Signed-off-by\|(cherry picked\)/q;p' | md5sum) + fi + + + set -- $(grep -m 1 "H$refbhash\$" "$WORK/${branch//\//_}") + newhash=$1 ; shift + if [ -n "$newhash" ]; then + line="${line} $(short $newhash)+?" + break + fi + fi + if [ -z "$newhash" ]; then + line="${line} -" + missing=1 + fi + fi + done + line="${line} |" + if [ -z "$MISSING" -o $missing -gt 0 ]; then + [ $((count++)) -gt 0 ] || echo "$title" + [ "$QUIET" != "" -o $count -lt 20 ] || count=0 + if [ -z "$UPSTREAM" -o "$upstream" = "none" -o -z "$upstream" ]; then + echo "$(short $ref) $line" + else + echo "$(short $upstream) $line" + fi + fi + done < "$WORK/${REF//\//_}" +} + +while [ -n "$1" -a -z "${1##-*}" ]; do + case "$1" in + -b) BASE="$2" ; shift 2 ;; + -r) REF="$2" ; shift 2 ;; + -l) LOGEXPR="$2" ; shift 2 ;; + -s) SUBJECT="$2" ; shift 2 ;; + -q) QUIET=1 ; shift ;; + -m) MISSING=1 ; shift ;; + -u) UPSTREAM=1 ; shift ;; + -H) BODYHASH=1 ; shift ;; + -h|--help) quit "$USAGE" ;; + *) die "$USAGE" ;; + esac +done + +# if no ref, either we're checking missing backports and we'll guess +# the upstream reference branch based on which one contains most of +# the latest commits, or we'll use master. +if [ -z "$REF" ]; then + if [ -n "$MISSING" ]; then + # check the last 10 commits in the base branch, and see where + # the seem to be coming from. + TAG="$(git describe --tags ${BASE:-HEAD} --abbrev=0)" + LAST_COMMITS=( $(git rev-list --abbrev-commit --reverse "$TAG^^.." | tail -n10) ) + REF=$(for i in "${LAST_COMMITS[@]}"; do + upstream=$(git log -1 --pretty --format=%B $i | + sed -n 's/^commit \([^)]*\) upstream\.$/\1/p;s/^(cherry picked from commit \([^)]*\))/\1/p' | + tail -n1) + if [ -n "$upstream" ]; then + # use local first then remote branch + ( git branch --sort=refname --contains $upstream | head -n1 ; + git branch -r --sort=refname --contains $upstream | head -n1) 2>&1 | + grep 'master\|maint' | head -n1 + fi + done | sort | uniq -c | sort -nr | awk '{ print $NF; exit;}') + # here we have a name, e.g. "2.6/master" in REF + REF="${REF:-master}" + err "Warning! No ref specified, using $REF." + else + REF=master + fi +fi + +# branches may also appear as id1..id2 to limit the history instead of looking +# back to the common base. The field is left empty if not set. +BRANCHES=( ) +BASES=( ) +while [ $# -gt 0 ]; do + if [ "$1" = "--" ]; then + shift + break + fi + branch="${1##*..}" + if [ "$branch" == "$1" ]; then + base="" + else + base="${1%%..*}" + fi + BASES[${#BRANCHES[@]}]="$base" + BRANCHES[${#BRANCHES[@]}]="$branch" + shift +done + +# args left for git-log +ARGS=( "$@" ) + +if [ ${#BRANCHES[@]} = 0 ]; then + if [ -n "$MISSING" ]; then + BRANCHES=( HEAD ) + else + die "$USAGE" + fi +fi + +for branch in "$REF" "${BRANCHES[@]}"; do + if ! git rev-parse --verify -q "$branch" >/dev/null; then + die "Failed to check git branch $branch." + fi +done + +if [ -z "$BASE" -a -n "$MISSING" ]; then + err "Warning! No base specified, checking latest backports from current branch since last tag." + + TAG="$(git describe --tags HEAD --abbrev=0)" + COMMITS=( $(git rev-list --abbrev-commit --reverse "$TAG^^..") ) + tip="" + for commit in "${COMMITS[@]}"; do + parent=$(git log -1 --pretty --format=%B $commit | + sed -n 's/^commit \([^)]*\) upstream\.$/\1/p;s/^(cherry picked from commit \([^)]*\))/\1/p' | + tail -n1) + if [ -z "$tip" ]; then + tip=$parent + elif [ -n "$parent" ]; then + base=$(git merge-base "$tip" "$parent") + if [ "$base" = "$tip" ]; then + # tip is older than parent, switch tip to it if it + # belongs to the upstream branch + if [ "$(git merge-base $parent $REF)" = "$parent" ]; then + tip=$parent + fi + fi + fi + done + BASE="$tip" + if [ -n "$BASE" ]; then + echo "Restarting from $(git log -1 --no-decorate --oneline $BASE)" + else + echo "Could not figure the base." + fi +fi + +if [ -z "$BASE" ]; then + err "Warning! No base specified, looking for common ancestor." + BASE=$(git merge-base --all "$REF" "${BRANCHES[@]}") + if [ -z "$BASE" ]; then + die "Couldn't find a common ancestor between these branches" + fi +fi + +# we want to go to the git root dir +DIR="$PWD" +cd $(git rev-parse --show-toplevel) + +mkdir -p .git/.show-backports #|| die "Can't create .git/.show-backports" +WORK=.git/.show-backports + +rm -f "$WORK/${REF//\//_}" +git log --reverse ${LOGEXPR:+--grep $LOGEXPR} --pretty="%H %s" "$BASE".."$REF" -- "${ARGS[@]}" | grep "${SUBJECT}" > "$WORK/${REF//\//_}" + +# for each branch, enumerate all commits and their ancestry + +branch_num=0; +while [ $branch_num -lt "${#BRANCHES[@]}" ]; do + branch="${BRANCHES[$branch_num]}" + base="${BASES[$branch_num]}" + base="${base:-$BASE}" + rm -f "$WORK/${branch//\//_}" + git log --reverse --pretty="%H %s" "$base".."$branch" -- "${ARGS[@]}" | grep "${SUBJECT}" | while read h subject; do + echo -n "$h" $(git log -1 --pretty --format=%B "$h" | \ + sed -n 's/^commit \([^)]*\) upstream\.$/\1/p;s/^(cherry picked from commit \([^)]*\))/\1/p') + if [ -n "$BODYHASH" ]; then + echo " H$(git log -1 --pretty="%an|%ae|%at|%B" "$h" | sed -n '/^\(Signed-off-by\|(cherry picked\)/q;p' | md5sum)" + else + echo + fi + done > "$WORK/${branch//\//_}" + (( branch_num++ )) +done + +count=0 +dump_commit_matrix | column -t | \ +( + left_commits=( ) + right_commits=( ) + while read line; do + # append the subject at the end of the line + set -- $line + echo -n "$line " + if [ "${line::1}" = ":" ]; then + echo "---- Subject ----" + else + # doing it this way prevents git from abusing the terminal + echo "$(git log -1 --pretty="%s" "$1")" + left_commits[${#left_commits[@]}]="$1" + comm="" + while [ -n "$1" -a "$1" != "-" -a "$1" != "|" ]; do + comm="${1%-*}" + shift + done + right_commits[${#right_commits[@]}]="$comm" + fi + done + if [ -n "$MISSING" -a ${#left_commits[@]} -eq 0 ]; then + echo "No missing commit to apply." + elif [ -n "$MISSING" ]; then + echo + echo + echo "In order to show and/or apply all leftmost commits to current branch :" + echo " git show --pretty=format:'%C(yellow)commit %H%C(normal)%nAuthor: %an <%ae>%nDate: %aD%n%n%C(green)%C(bold)git cherry-pick -sx %h%n%n%w(72,4,4)%B%N' ${left_commits[@]}" + echo + echo " git cherry-pick -sx ${left_commits[@]}" + echo + if [ "${left_commits[*]}" != "${right_commits[*]}" ]; then + echo "In order to show and/or apply all rightmost commits to current branch :" + echo " git show --pretty=format:'%C(yellow)commit %H%C(normal)%nAuthor: %an <%ae>%nDate: %aD%n%n%C(green)%C(bold)git cherry-pick -sx %h%n%n%w(72,4,4)%B%N' ${right_commits[@]}" + echo + echo " git cherry-pick -sx ${right_commits[@]}" + echo + fi + fi +) diff --git a/scripts/make-releases-json b/scripts/make-releases-json new file mode 100755 index 0000000..ba05665 --- /dev/null +++ b/scripts/make-releases-json @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# +# Scan a branch directory for source tarballs and rebuild the releases.json +# file for that branch. md5 and sha256 are added if present. The highest +# numbered version is referenced as the latest release. +# +# Usage: $0 [-b branch] [-o outfile] /path/to/download/branch +# + +USAGE="Usage: ${0##*/} [-b branch] [-o outfile] DIR" +OUTPUT= +BRANCH= +DIR= + +die() { + [ "$#" -eq 0 ] || echo "$*" >&2 + exit 1 +} + +err() { + echo "$*" >&2 +} + +quit() { + [ "$#" -eq 0 -o -n "$QUIET" ] || echo "$*" + exit 0 +} + +emit_json() { + printf '{\n "branch": "%s",\n' ${BRANCH} + latest="" + for file in $(find "$DIR/src" -name 'haproxy-[0-9]*.gz' -printf "%P\n" |grep -v '[0-9]-patches*' | sort -rV ); do + rel="${file##*haproxy-}" + rel="${rel%%.tar.*}" + if [ -z "$latest" ]; then + latest="$rel"; + printf ' "latest_release": "%s",\n' ${latest} + printf ' "releases": {\n' + else + printf ",\n" + fi + printf ' "%s": {\n' ${rel} + printf ' "file": "%s"' ${file} + if [ -s "$DIR/src/$file.md5" ]; then + printf ',\n "md5": "%s"' $(awk '{print $1}' "$DIR/src/$file.md5") + fi + if [ -s "$DIR/src/$file.sha256" ]; then + printf ',\n "sha256": "%s"' $(awk '{print $1}' "$DIR/src/$file.sha256") + fi + printf '\n }' + done + + if [ -n "$latest" ]; then + printf "\n }" ## "releases" + fi + + printf '\n}\n' +} + + +### main + +while [ -n "$1" -a -z "${1##-*}" ]; do + case "$1" in + -b) BRANCH="$2" ; shift 2 ;; + -o) OUTPUT="$2" ; shift 2 ;; + -h|--help) quit "$USAGE" ;; + *) die "$USAGE" ;; + esac +done + +if [ $# -ne 1 ]; then + die "$USAGE" +fi + +DIR="$1" ; shift +if [ -z "$DIR" ]; then + die "Missing download directory name." +fi + +if [ ! -d "$DIR/." ]; then + die "Download directory doesn't exist : $DIR" +fi + +if [ ! -d "$DIR/src" ]; then + die "Download directory must contain 'src' : $DIR" +fi + +if [ -z "$BRANCH" ]; then + BRANCH=${DIR##*/} + if [ -n "${BRANCH//[0-9.]}" ]; then + die "Couldn't determine branch number from dir name: $BRANCH" + fi +fi + +# echo "debug: DIR=$DIR BRANCH=$BRANCH" +if [ -n "$OUTPUT" ]; then + emit_json > "$OUTPUT.tmp" + mv -f "$OUTPUT.tmp" "$OUTPUT" + rm -f "$OUTPUT.tmp" +else + emit_json +fi diff --git a/scripts/publish-release b/scripts/publish-release new file mode 100755 index 0000000..9066d4a --- /dev/null +++ b/scripts/publish-release @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +# puts the public files online after a release +# Copyright (c) 2006-2016 Willy Tarreau <w@1wt.eu> +# +# In short : +# - requires git +# - no restriction to master, uses last tag +# - copies & compresses files, changelog & docs to the final destination +# - shows a listing of the final file + +USAGE="Usage: ${0##*/} [-a] [-q] [-y] [-b branch] [-n newver] DIR" +CMD_GZIP="${CMD_GZIP:-gzip -nc9}" +TARGET_DIR= +OUTPUT= +SAYYES= +BRANCH= +DEVEL= +QUIET= +AUTO= +ARG0="$0" +NEW= +DIR= +DOC=( ) + +# need to have group write on emitted files for others to update +umask 002 + +die() { + [ "$#" -eq 0 ] || echo "$*" >&2 + exit 1 +} + +err() { + echo "$*" >&2 +} + +quit() { + [ "$#" -eq 0 -o -n "$QUIET" ] || echo "$*" + exit 0 +} + +while [ -n "$1" -a -z "${1##-*}" ]; do + case "$1" in + -a) AUTO=1 ; shift ;; + -q) QUIET=1 ; shift ;; + -y) SAYYES=1 ; shift ;; + -b) BRANCH="$2" ; shift 2 ;; + -n) NEW="$2" ; shift 2 ;; + -h|--help) quit "$USAGE" ;; + *) die "$USAGE" ;; + esac +done + +if [ $# -ne 1 ]; then + die "$USAGE" +fi + +DIR="$1" ; shift +if [ -z "$DIR" ]; then + die "Missing target directory name." +fi + +if [ -n "${DIR##/*}" ]; then + DIR="$PWD/$DIR" +fi + +if [ ! -d "$DIR/." ]; then + die "Target directory doesn't exist : $DIR" +fi + +if ! git rev-parse --verify -q HEAD >/dev/null; then + die "Failed to check git HEAD." +fi + +# we want to go to the git top dir +toplvl=$(git rev-parse --show-toplevel) +if [ -n "$toplvl" ]; then + cd "$toplvl" +fi + +# ensure that a master branch exists here +if [ -z "$(git rev-parse --verify -q master 2>/dev/null)" ]; then + die "Current directory doesn't seem to be a valid git directory (no master branch)." +fi + +if [ "$(git rev-parse --verify -q HEAD)" != "$(git rev-parse --verify -q master)" ]; then + die "git HEAD doesn't match master branch." +fi + +if [ "$(git diff HEAD 2>/dev/null |wc -c)" != 0 ]; then + err "You appear to have uncommitted local changes, please commit them first :" + git status -s -uno >&2 + die +fi + +if [ -z "$NEW" -o -n "$AUTO" ]; then + if [ -z "$NEW" ]; then + NEW="$(git describe --tags HEAD --abbrev=0)" + NEW="${NEW#v}" + if [ -z "$NEW" ]; then + die "Fatal: cannot determine new version, please specify it." + fi + fi + + if [ "$(git describe --tags HEAD)" != "v$NEW" ]; then + if [ -n "$AUTO" ]; then + quit "Not tagged, nothing to do." + fi + die "Current version doesn't seem tagged, it reports $(git describe --tags "v$NEW"). Did you release it ?" + fi +fi + +if ! git show-ref --tags "v$NEW" >/dev/null; then + die "git tag v$NEW doesn't exist, did you create the release ?" +fi + +# determine the product branch from the new release +if [ -z "$BRANCH" ]; then + subvers=${NEW#[0-9]*.[0-9]*[-.]*[0-9].} + [ "${subvers}" = "${NEW}" ] && subvers="" + major=${NEW%.$subvers} + branch_ext=${major#*[0-9].*[0-9]} + BRANCH=${major%${branch_ext}} +fi + +TARGET_DIR="$DIR/$BRANCH" +if [ ! -d "$TARGET_DIR/." ]; then + die "Target directory doesn't contain branch $BRANCH. You may have to create it in $DIR." +fi + +if [ -z "${NEW##*-dev*}" ]; then + DEVEL="/devel" +fi + +if [ -n "$AUTO" -a -e "$TARGET_DIR/src${DEVEL}/haproxy-$NEW.tar.gz.md5" ]; then + quit "Version $NEW Already released." +fi + +if ! mkdir -p "$TARGET_DIR/src$DEVEL" "$TARGET_DIR/doc"; then + die "failed to create target directories." +fi + +case "$BRANCH" in + 1.3) DOC=( doc/{haproxy-en,haproxy-fr,configuration,architecture}.txt ) ;; + 1.4) DOC=( doc/{haproxy-en,haproxy-fr,configuration}.txt ) ;; + 1.5) DOC=( doc/{coding-style,configuration,proxy-protocol}.txt ) ;; + 1.6) DOC=( doc/{coding-style,intro,management,configuration,proxy-protocol,lua}.txt ) ;; + *) DOC=( doc/{coding-style,intro,management,configuration,proxy-protocol,lua,SPOE}.txt ) ;; +esac + +if [ -z "$AUTO" ]; then + echo "Ready to produce the following files in $TARGET_DIR/ :" + echo " haproxy-$NEW.tar.gz -> src${DEVEL}/" + echo " CHANGELOG -> src/CHANGELOG" + echo " ${DOC[@]} -> doc/*{,.gz}" + echo + + git ls-tree -l --abbrev=12 "v$NEW" -- CHANGELOG "${DOC[@]}" + + if [ -z "$SAYYES" ]; then + echo "Press ENTER to continue or Ctrl-C to abort now!" + read + fi +fi + +echo "Archiving sources for version $NEW ..." +rm -f "${TARGET_DIR}/src${DEVEL}/haproxy-${NEW}.tar.gz"{,.md5,.sha256} +if ! git archive --format=tar --prefix="haproxy-${NEW}/" "v$NEW" | \ + $CMD_GZIP > "${TARGET_DIR}/src${DEVEL}/haproxy-${NEW}.tar.gz"; then + die "Failed to produce the tar.gz archive" +fi + +( cd "$TARGET_DIR/src${DEVEL}" ; \ + md5sum haproxy-$NEW.tar.gz > haproxy-$NEW.tar.gz.md5 ; \ + sha256sum haproxy-$NEW.tar.gz > haproxy-$NEW.tar.gz.sha256 ) + +echo "Extracting doc ..." +git show "v$NEW:CHANGELOG" > "$TARGET_DIR/src/CHANGELOG" + +for i in "${DOC[@]}"; do + git show "v$NEW:$i" > "$TARGET_DIR/doc/${i#doc/}" + $CMD_GZIP < "$TARGET_DIR/doc/${i#doc/}" > "$TARGET_DIR/doc/${i#doc/}.gz" +done + +if [ -x "${ARG0%/*}/make-releases-json" ]; then + # regenerate versions + "${ARG0%/*}/make-releases-json" -o "$TARGET_DIR/src/releases.json" "$TARGET_DIR" +fi + +echo "Done : ls -l ${TARGET_DIR}" +( cd "$TARGET_DIR" ; + ls -l src/CHANGELOG "src${DEVEL}/haproxy-${NEW}".tar.gz{,.md5,.sha256} $(for i in "${DOC[@]}"; do echo "doc/${i#doc/}"{,.gz}; done) +) +echo diff --git a/scripts/run-regtests.sh b/scripts/run-regtests.sh new file mode 100755 index 0000000..85f1341 --- /dev/null +++ b/scripts/run-regtests.sh @@ -0,0 +1,427 @@ +#!/bin/sh + +_help() +{ + cat << EOF +### run-regtests.sh ### + Running run-regtests.sh --help shows this information about how to use it + + Run without parameters to run all tests in the current folder (including subfolders) + run-regtests.sh + + Provide paths to run tests from (including subfolders): + run-regtests.sh ./tests1 ./tests2 + + Parameters: + --j <NUM>, To run vtest with multiple jobs / threads for a faster overall result + run-regtests.sh ./fasttest --j 16 + + --v, to run verbose + run-regtests.sh --v, disables the default vtest 'quiet' parameter + + --debug to show test logs on standard output (implies --v) + run-regtests.sh --debug + + --keep-logs to keep all log directories (by default kept if test fails) + run-regtests.sh --keep-logs + + --vtestparams <ARGS>, passes custom ARGS to vtest + run-regtests.sh --vtestparams "-n 10" + + --type <reg tests types> filter the types of the tests to be run, depending on + the commented REGTESTS_TYPE variable value in each VTC file. + The value of REGTESTS_TYPE supported are: default, slow, bug, broken, devel + and experimental. When not specified, it is set to 'default' as default value. + + run-regtest.sh --type slow,default + + --clean to cleanup previous reg-tests log directories and exit + run-regtests.sh --clean + + Including text below into a .vtc file will check for its requirements + related to haproxy's target and compilation options + # Below targets are not capable of completing this test successfully + #EXCLUDE_TARGET=freebsd, abns sockets are not available on freebsd + + #EXCLUDE_TARGETS=dos,freebsd,windows + + # Below option is required to complete this test successfully + #REQUIRE_OPTION=OPENSSL, this test needs OPENSSL compiled in. + #REQUIRE_OPTIONS=ZLIB|SLZ,OPENSSL,LUA + + #REQUIRE_SERVICE=prometheus-exporter + #REQUIRE_SERVICES=prometheus-exporter,foo + + # To define a range of versions that a test can run with: + #REQUIRE_VERSION=0.0 + #REQUIRE_VERSION_BELOW=99.9 + + Configure environment variables to set the haproxy and vtest binaries to use + setenv HAPROXY_PROGRAM /usr/local/sbin/haproxy + setenv VTEST_PROGRAM /usr/local/bin/vtest + setenv HAPROXY_ARGS "-dM -de -m 50" + or + export HAPROXY_PROGRAM=/usr/local/sbin/haproxy + export VTEST_PROGRAM=/usr/local/bin/vtest + export HAPROXY_ARGS="-dM -de -m 50" +EOF + exit 0 +} + +add_range_to_test_list() +{ + level0="*.vtc" + level1="h*.vtc" + level2="s*.vtc" + level3="l*.vtc" + level4="b*.vtc" + level5="k*.vtc" + level6="e*.vtc" + + new_range=$(echo $1 | tr '-' ' ') + non_digit=$(echo $new_range | grep '[^0-9 ]') + if [ -n "$non_digit" ] ; then + return + fi + if [ "$new_range" = "$1" ] ; then + if [ $1 -gt 6 ] ; then + return + fi + eval echo '$'level$1 + return + fi + if [ -z "$new_range" ] ; then + return + fi + list= + for l in $(seq $new_range) ; do + if [ -n "l" ] ; then + if [ -z "$list" ] ; then + list="$(eval echo '$'level${l})" + else + list="$list $(eval echo '$'level${l})" + fi + fi + done + + echo $list +} + +_startswith() { + _str="$1" + _sub="$2" + echo "$_str" | grep "^$_sub" >/dev/null 2>&1 +} + +_findtests() { + set -f + + REGTESTS_TYPES="${REGTESTS_TYPES:-default,bug,devel,slow}" + any_test=$(echo $REGTESTS_TYPES | grep -cw "any") + for i in $( find "$1" -name *.vtc ); do + skiptest= + OLDIFS="$IFS"; IFS="$LINEFEED" + set -- $(grep '^#[0-9A-Z_]*=' "$i") + IFS="$OLDIFS" + + require_version=""; require_version_below=""; require_options=""; + require_services=""; exclude_targets=""; regtest_type="" + requiredoption=""; requiredservice=""; excludedtarget=""; + + while [ $# -gt 0 ]; do + v="$1"; v="${v#*=}" + case "$1" in + "#REQUIRE_VERSION="*) require_version="$v" ;; + "#REQUIRE_VERSION_BELOW="*) require_version_below="$v" ;; + "#REQUIRE_OPTIONS="*) require_options="$v" ;; + "#REQUIRE_SERVICES="*) require_services="$v" ;; + "#EXCLUDE_TARGETS="*) exclude_targets="$v" ;; + "#REGTEST_TYPE="*) regtest_type="$v" ;; + "#REQUIRE_OPTION="*) requiredoption="${v%,*}" ;; + "#REQUIRE_SERVICE="*) required_service="${v%,*}" ;; + "#EXCLUDE_TARGET="*) excludedtarget="${v%,*}" ;; + # Note: any new variable declared here must be initialized above. + esac + shift + done + + if [ $any_test -ne 1 ] ; then + if [ -z $regtest_type ] ; then + regtest_type=default + fi + if ! $(echo $REGTESTS_TYPES | grep -wq $regtest_type) ; then + echo " Skip $i because its type '"$regtest_type"' is excluded" + skiptest=1 + fi + fi + + if [ -n "$requiredoption" ]; then + require_options="$require_options,$requiredoption" + fi + + if [ -n "$requiredservice" ]; then + require_services="$require_services,$requiredservice" + fi + + if [ -n "$excludedtarget" ]; then + exclude_targets="$exclude_targets,$excludedtarget" + fi + + IFS=","; set -- $require_options; IFS=$OLDIFS; require_options="$*" + IFS=","; set -- $require_services; IFS=$OLDIFS; require_services="$*" + IFS=","; set -- $exclude_targets; IFS=$OLDIFS; exclude_targets="$*" + + if [ -n "$require_version" ]; then + if [ $(_version "$HAPROXY_VERSION") -lt $(_version "$require_version") ]; then + echo " Skip $i because option haproxy is version: $HAPROXY_VERSION" + echo " REASON: this test requires at least version: $require_version" + skiptest=1 + fi + fi + if [ -n "$require_version_below" ]; then + if [ $(_version "$HAPROXY_VERSION") -ge $(_version "$require_version_below") ]; then + echo " Skip $i because option haproxy is version: $HAPROXY_VERSION" + echo " REASON: this test requires a version below: $require_version_below" + skiptest=1 + fi + fi + + for excludedtarget in $exclude_targets; do + if [ "$excludedtarget" = "$TARGET" ]; then + echo " Skip $i because haproxy is compiled for the excluded target $TARGET" + skiptest=1 + fi + done + + for requiredoption in $require_options; do + IFS="|"; set -- $requiredoption; IFS=$OLDIFS; alternatives="$*" + found= + for alt in $alternatives; do + if [ -z "${FEATURES_PATTERN##* +$alt *}" ]; then + found=1; + fi + done + if [ -z $found ]; then + echo " Skip $i because haproxy is not compiled with the required option $requiredoption" + skiptest=1 + fi + done + + for requiredservice in $require_services; do + IFS="|"; set -- $requiredservice; IFS=$OLDIFS; alternatives="$*" + found= + for alt in $alternatives; do + if [ -z "${SERVICES_PATTERN##* $alt *}" ]; then + found=1; + fi + done + if [ -z $found ]; then + echo " Skip $i because haproxy is not compiled with the required service $requiredservice" + skiptest=1 + fi + done + + if [ -z $skiptest ]; then + echo " Add test: $i" + testlist="$testlist $i" + fi + done +} + +_cleanup() +{ + DIRS=$(find "${TESTDIR}" -maxdepth 1 -type d -name "haregtests-*" -exec basename {} \; 2>/dev/null) + if [ -z "${DIRS}" ]; then + echo "No reg-tests log directory found" + else + echo "Cleanup following reg-tests log directories:" + for d in ${DIRS}; do + echo " o ${TESTDIR}/$d" + done + read -p "Continue (y/n)?" reply + case "$reply" in + y|Y) + for d in ${DIRS}; do + rm -r "${TESTDIR}/$d" + done + echo "done" + exit 0 + ;; + *) + echo "aborted" + exit 1 + ;; + esac + fi +} + + +_process() { + while [ ${#} -gt 0 ]; do + if _startswith "$1" "-"; then + case "${1}" in + --j) + jobcount="$2" + shift + ;; + --vtestparams) + vtestparams="$2" + shift + ;; + --v) + verbose="" + ;; + --debug) + verbose="" + debug="-v" + ;; + --keep-logs) + keep_logs="-L" + ;; + --type) + REGTESTS_TYPES="$2" + shift + ;; + --clean) + _cleanup + exit 0 + ;; + --help) + _help + ;; + *) + echo "Unknown parameter : $1" + exit 1 + ;; + esac + else + REGTESTS="${REGTESTS} $1" + fi + shift 1 + done +} + +# compute a version from up to 4 sub-version components, each multiplied +# by a power of 1000, and padded left with 0, 1 or 2 zeroes. +_version() { + OLDIFS="$IFS"; IFS="."; set -- $*; IFS="$OLDIFS" + set -- ${1%%[!0-9]*} 000${2%%[!0-9]*} 000${3%%[!0-9]*} 000${4%%[!0-9]*} + prf2=${2%???}; prf3=${3%???}; prf4=${4%???} + echo ${1}${2#$prf2}${3#$prf3}${4#$prf4} +} + + +HAPROXY_PROGRAM="${HAPROXY_PROGRAM:-${PWD}/haproxy}" +HAPROXY_ARGS="${HAPROXY_ARGS--dM}" +VTEST_PROGRAM="${VTEST_PROGRAM:-vtest}" +TESTDIR="${TMPDIR:-/tmp}" +REGTESTS="" +LINEFEED=" +" + +jobcount="" +verbose="-q" +debug="" +keep_logs="-l" +testlist="" + +_process "$@"; + +echo "" +echo "########################## Preparing to run tests ##########################" + +preparefailed= +if ! [ -x "$(command -v $HAPROXY_PROGRAM)" ]; then + echo "haproxy not found in path, please specify HAPROXY_PROGRAM environment variable" + preparefailed=1 +fi +if ! [ -x "$(command -v $VTEST_PROGRAM)" ]; then + echo "vtest not found in path, please specify VTEST_PROGRAM environment variable" + preparefailed=1 +fi +if [ $preparefailed ]; then + exit 1 +fi + +{ read HAPROXY_VERSION; read TARGET; read FEATURES; read SERVICES; } << EOF +$($HAPROXY_PROGRAM $HAPROXY_ARGS -vv | grep 'HA-\?Proxy version\|TARGET.*=\|^Feature\|^Available services' | sed 's/.* [:=] //') +EOF + +HAPROXY_VERSION=$(echo $HAPROXY_VERSION | cut -d " " -f 3) +echo "Testing with haproxy version: $HAPROXY_VERSION" + +PROJECT_VERSION=$(${MAKE:-make} version 2>&1 | grep '^VERSION:\|^SUBVERS:'|cut -f2 -d' '|tr -d '\012') +if [ -z "${PROJECT_VERSION}${MAKE}" ]; then + # try again with gmake, just in case + PROJECT_VERSION=$(gmake version 2>&1 | grep '^VERSION:\|^SUBVERS:'|cut -f2 -d' '|tr -d '\012') +fi + +FEATURES_PATTERN=" $FEATURES " +SERVICES_PATTERN=" $SERVICES " + +TESTRUNDATETIME="$(date '+%Y-%m-%d_%H-%M-%S')" + +mkdir -p "$TESTDIR" || exit 1 +TESTDIR=$(mktemp -d "$TESTDIR/haregtests-$TESTRUNDATETIME.XXXXXX") || exit 1 + +export TMPDIR="$TESTDIR" +export HAPROXY_PROGRAM="$HAPROXY_PROGRAM" +if [ -n "$HAPROXY_ARGS" ]; then + export HAPROXY_ARGS +fi + +echo "Target : $TARGET" +echo "Options : $FEATURES" +echo "Services : $SERVICES" + +echo "########################## Gathering tests to run ##########################" + +if [ -z "$REGTESTS" ]; then + _findtests reg-tests/ +else + for t in $REGTESTS; do + _findtests $t + done +fi + +echo "########################## Starting vtest ##########################" +echo "Testing with haproxy version: $HAPROXY_VERSION" + +if [ -n "$PROJECT_VERSION" -a "$PROJECT_VERSION" != "$HAPROXY_VERSION" ]; then + echo "Warning: version does not match the current tree ($PROJECT_VERSION)" +fi + +_vtresult=0 +if [ -n "$testlist" ]; then + if [ -n "$jobcount" ]; then + jobcount="-j $jobcount" + fi + cmd="$VTEST_PROGRAM -b $((2<<20)) -k -t 10 $keep_logs $verbose $debug $jobcount $vtestparams $testlist" + eval $cmd + _vtresult=$? +else + echo "No tests found that meet the required criteria" +fi + + +if [ $_vtresult -eq 0 ]; then + # all tests were successful, removing tempdir (the last part.) + # ignore errors is the directory is not empty or if it does not exist + rmdir "$TESTDIR" 2>/dev/null +fi + +if [ -d "${TESTDIR}" ]; then + echo "########################## Gathering results ##########################" + export TESTDIR + find "$TESTDIR" -type d -name "vtc.*" -exec sh -c 'for i; do + if [ ! -e "$i/LOG" ] ; then continue; fi + + cat <<- EOF | tee -a "$TESTDIR/failedtests.log" +$(echo "###### $(cat "$i/INFO") ######") +$(echo "## test results in: \"$i\"") +$(grep -E -- "^(----|\* diag)" "$i/LOG") +EOF + done' sh {} + +fi + +exit $_vtresult |