diff options
Diffstat (limited to 'scripts/cowpoke.sh')
| -rwxr-xr-x | scripts/cowpoke.sh | 542 |
1 files changed, 542 insertions, 0 deletions
diff --git a/scripts/cowpoke.sh b/scripts/cowpoke.sh new file mode 100755 index 0000000..54dd9fc --- /dev/null +++ b/scripts/cowpoke.sh @@ -0,0 +1,542 @@ +#!/bin/bash +# Simple shell script for driving a remote cowbuilder via ssh +# +# Copyright(C) 2007, 2008, 2009, 2011, 2012, 2014, Ron <ron@debian.org> +# This script is distributed according to the terms of the GNU GPL. + +set -e + +#BUILDD_HOST= +#BUILDD_USER= +BUILDD_ARCH="$(dpkg-architecture -qDEB_BUILD_ARCH 2>/dev/null)" + +# The 'default' dist is whatever cowbuilder is locally configured for +BUILDD_DIST="default" + +INCOMING_DIR="cowbuilder-incoming" +PBUILDER_BASE="/var/cache/pbuilder" + +#SIGN_KEYID= +#UPLOAD_QUEUE="ftp-master" +BUILDD_ROOTCMD="sudo" + +REMOTE_SCRIPT="cowssh_it" +DEBOOTSTRAP="cdebootstrap" + +for f in /etc/cowpoke.conf ~/.cowpoke .cowpoke "$COWPOKE_CONF"; do [ -r "$f" ] && . "$f"; done + + +get_archdist_vars() { + _ARCHDIST_OPTIONS="RESULT_DIR BASE_PATH BASE_DIST CREATE_OPTS UPDATE_OPTS BUILD_OPTS SIGN_KEYID UPLOAD_QUEUE" + _RESULT_DIR="result" + _BASE_PATH="base.cow" + + for arch in $BUILDD_ARCH; do + for dist in $BUILDD_DIST; do + for var in $_ARCHDIST_OPTIONS; do + eval "val=( \"\${${arch}_${dist}_${var}[@]}\" )" + + if [ "$1" = "display" ]; then + case $var in + RESULT_DIR | BASE_PATH ) + [ ${#val[@]} -gt 0 ] || eval "val=\"$PBUILDER_BASE/$arch/$dist/\$_$var\"" + echo " ${arch}_${dist}_${var} = $val" + ;; + + *_OPTS ) + # Don't display these if they are overridden on the command line. + eval "override=( \"\${OVERRIDE_${var}[@]}\" )" + [ ${#override[@]} -gt 0 ] || [ ${#val[@]} -eq 0 ] || + echo " ${arch}_${dist}_${var} =$(printf " '%s'" "${val[@]}")" + ;; + + * ) + [ ${#val[@]} -eq 0 ] || echo " ${arch}_${dist}_${var} = $val" + ;; + esac + else + case $var in + RESULT_DIR | BASE_PATH ) + # These are always a single value, and must always be set, + # either by the user or to their default value. + [ ${#val[@]} -gt 0 ] || eval "val=\"$PBUILDER_BASE/$arch/$dist/\$_$var\"" + echo "${arch}_${dist}_${var}='$val'" + ;; + + *_OPTS ) + # These may have zero, one, or many values which we must not word-split. + # They can safely remain unset if there are no values. + # + # We don't need to worry about the command line overrides here, + # they will be taken care of in the remote script. + [ ${#val[@]} -eq 0 ] || + echo "${arch}_${dist}_${var}=($(printf " %q" "${val[@]}") )" + ;; + + SIGN_KEYID | UPLOAD_QUEUE ) + # We don't need these in the remote script + ;; + + * ) + # These may have zero or one value. + # They can safely remain unset if there are no values. + [ ${#val[@]} -eq 0 ] || echo "${arch}_${dist}_${var}='$val'" + ;; + esac + fi + done + done + done +} + +display_override_vars() { + _OVERRIDE_OPTIONS="CREATE_OPTS UPDATE_OPTS BUILD_OPTS SIGN_KEYID UPLOAD_QUEUE" + + for var in $_OVERRIDE_OPTIONS; do + eval "override=( \"\${OVERRIDE_${var}[@]}\" )" + [ ${#override[@]} -eq 0 ] || echo " override: $var =$(printf " '%s'" "${override[@]}")" + done +} + + +PROGNAME=${0##*/} +version() { + echo \ +"This is $PROGNAME, from the Debian devscripts package, version ###VERSION### +This code is Copyright 2007-2014, Ron <ron@debian.org>. +This program comes with ABSOLUTELY NO WARRANTY. +You are free to redistribute this code under the terms of the +GNU General Public License." + exit 0 +} + +usage() { + cat 1>&2 <<EOF + +cowpoke [options] package.dsc + + Uploads a Debian source package to a cowbuilder host and builds it, + optionally also signing and uploading the result to an incoming queue. + The following options are supported: + + --arch="arch" Specify the Debian architecture(s) to build for. + --dist="dist" Specify the Debian distribution(s) to build for. + --buildd="host" Specify the remote host to build on. + --buildd-user="name" Specify the remote user to build as. + --create Create the remote cowbuilder root if necessary. + --return[="path"] Copy results of the build to 'path'. If path is + not specified, return them to the current directory. + --no-return Do not copy results of the build to RETURN_DIR + (overriding a path set for it in the config files). + --sign="keyid" Specify the key to sign packages with. + --upload="queue" Specify the dput queue to upload signed packages to. + + The current default configuration is: + + BUILDD_HOST = $BUILDD_HOST + BUILDD_USER = $BUILDD_USER + BUILDD_ARCH = $BUILDD_ARCH + BUILDD_DIST = $BUILDD_DIST + RETURN_DIR = $RETURN_DIR + SIGN_KEYID = $SIGN_KEYID + UPLOAD_QUEUE = $UPLOAD_QUEUE + + The expected remote paths are: + + INCOMING_DIR = $INCOMING_DIR + PBUILDER_BASE = ${PBUILDER_BASE:-/} + +$(get_archdist_vars display) +$(display_override_vars) + + The cowbuilder image must have already been created on the build host + and the expected remote paths must already exist if the --create option + is not passed. You must have ssh access to the build host as BUILDD_USER + if that is set, else as the user executing cowpoke or a user specified + in your ssh config for '$BUILDD_HOST'. + That user must be able to execute cowbuilder as root using '$BUILDD_ROOTCMD'. + +EOF + + exit $1 +} + + +for arg; do + case "$arg" in + --arch=*) + BUILDD_ARCH="${arg#*=}" + ;; + + --dist=*) + BUILDD_DIST="${arg#*=}" + ;; + + --buildd=*) + BUILDD_HOST="${arg#*=}" + ;; + + --buildd-user=*) + BUILDD_USER="${arg#*=}" + ;; + + --create) + CREATE_COW="yes" + ;; + + --return=*) + RETURN_DIR="${arg#*=}" + ;; + + --return) + RETURN_DIR=. + ;; + + --no-return) + RETURN_DIR= + ;; + + --dpkg-opts=*) + # This one is a bit tricky, given the combination of the calling convention here, + # the calling convention for cowbuilder, and the behaviour of things that might + # pass this option to us. Some things, like when we are called from the gitpkg + # hook using options from git-config, will preserve any quoting that was used in + # the .gitconfig file, which is natural for anyone to want to use in a construct + # like: options = --dpkg-opts='-uc -us -j6'. People are going to cringe if we + # tell them they must not use quotes there no matter how much it may 'make sense' + # if you know too much about the internals. And it will only get worse when we + # then tell them they must quote it like that if they type it directly in their + # shell ... + # + # So we do the only thing that seems sensible, and try to Deal With It here. + # If the outermost characters are paired quotes, we manually strip them off. + # We don't want to let the shell do quote removal, since that might change a + # part of this which we don't want modified. + # We collect however many sets of those we are passed in an array, which we'll + # then combine back into a single argument at the final point of use. + # + # Which _should_ DTRT for anyone who isn't trying to blow this up deliberately + # and maybe will still do it for them too in spite of their efforts. But unless + # someone finds a sensible case this fails on, I'm not going to cry over people + # who want to stuff up their own system with input they created themselves. + val=${arg#*=} + [[ $val == \'*\' || $val == \"*\" ]] && val=${val:1:-1} + DEBBUILDOPTS+=( "$val" ) + ;; + + --create-opts=*) + OVERRIDE_CREATE_OPTS+=( "${arg#*=}" ) + ;; + + --update-opts=*) + OVERRIDE_UPDATE_OPTS+=( "${arg#*=}" ) + ;; + + --build-opts=*) + OVERRIDE_BUILD_OPTS+=( "${arg#*=}" ) + ;; + + --sign=*) + OVERRIDE_SIGN_KEYID=${arg#*=} + ;; + + --upload=*) + OVERRIDE_UPLOAD_QUEUE=${arg#*=} + ;; + + *.dsc) + DSC="$arg" + ;; + + --help) + usage 0 + ;; + + --version) + version + ;; + + *) + echo "ERROR: unrecognised option '$arg'" + usage 1 + ;; + esac +done + +if [ -z "$REMOTE_SCRIPT" ]; then + echo "No remote script name set. Aborted." + exit 1 +fi +if [ -z "$DSC" ]; then + echo "ERROR: No package .dsc specified" + usage 1 +fi +if ! [ -r "$DSC" ]; then + echo "ERROR: '$DSC' not found." + exit 1 +fi +if [ -z "$BUILDD_ARCH" ]; then + echo "No BUILDD_ARCH set. Aborted." + exit 1 +fi +if [ -z "$BUILDD_HOST" ]; then + echo "No BUILDD_HOST set. Aborted." + exit 1 +fi +if [ -z "$BUILDD_ROOTCMD" ]; then + echo "No BUILDD_ROOTCMD set. Aborted." + exit 1 +fi +if [ -e "$REMOTE_SCRIPT" ]; then + echo "$REMOTE_SCRIPT file already exists and will be overwritten." + echo -n "Do you wish to continue (Y/n)? " + read -e yesno + case "$yesno" in + N* | n*) + echo "Ok, bailing out." + echo "You should set the REMOTE_SCRIPT variable to some other value" + echo "if this name conflicts with something you already expect to use" + exit 1 + ;; + *) ;; + esac +fi + +[ -z "$BUILDD_USER" ] || BUILDD_USER="$BUILDD_USER@" + +PACKAGE="$(basename $DSC .dsc)" +DATE="$(date +%Y%m%d 2>/dev/null)" + + +cat > "$REMOTE_SCRIPT" <<-EOF + #!/bin/bash + # cowpoke generated remote worker script. + # Normally this should have been deleted already, you can safely remove it now. + + compare_changes() { + p1="\${1%_*.changes}" + p2="\${2%_*.changes}" + p1="\${p1##*_}" + p2="\${p2##*_}" + + dpkg --compare-versions "\$p1" gt "\$p2" + } + + $(get_archdist_vars) + + for arch in $BUILDD_ARCH; do + for dist in $BUILDD_DIST; do + + echo " ------- Begin build for \$arch \$dist -------" + + CHANGES="\$arch.changes" + LOGFILE="$INCOMING_DIR/build.${PACKAGE}_\$arch.\$dist.log" + UPDATELOG="$INCOMING_DIR/cowbuilder-\${arch}-\${dist}-update-log-$DATE" + eval "RESULT_DIR=\"\\\$\${arch}_\${dist}_RESULT_DIR\"" + eval "BASE_PATH=\"\\\$\${arch}_\${dist}_BASE_PATH\"" + eval "BASE_DIST=\"\\\$\${arch}_\${dist}_BASE_DIST\"" + eval "CREATE_OPTS=( \"\\\${\${arch}_\${dist}_CREATE_OPTS[@]}\" )" + eval "UPDATE_OPTS=( \"\\\${\${arch}_\${dist}_UPDATE_OPTS[@]}\" )" + eval "BUILD_OPTS=( \"\\\${\${arch}_\${dist}_BUILD_OPTS[@]}\" )" + + [ -n "\$BASE_DIST" ] || BASE_DIST=\$dist + [ ${#OVERRIDE_CREATE_OPTS[@]} -eq 0 ] || CREATE_OPTS=("${OVERRIDE_CREATE_OPTS[@]}") + [ ${#OVERRIDE_UPDATE_OPTS[@]} -eq 0 ] || UPDATE_OPTS=("${OVERRIDE_UPDATE_OPTS[@]}") + [ ${#OVERRIDE_BUILD_OPTS[@]} -eq 0 ] || BUILD_OPTS=("${OVERRIDE_BUILD_OPTS[@]}") + [ ${#DEBBUILDOPTS[*]} -eq 0 ] || DEBBUILDOPTS=("--debbuildopts" "${DEBBUILDOPTS[*]}") + + + # Sort the list of old changes files for this package to try and + # determine the most recent one preceding this version. We will + # debdiff to this revision in the final sanity checks if one exists. + # This is adapted from the insertion sort trickery in git-debimport. + + OLD_CHANGES="\$(find "\$RESULT_DIR/" -maxdepth 1 -type f \\ + -name "${PACKAGE%%_*}_*_\$CHANGES" 2>/dev/null \\ + | sort 2>/dev/null)" + P=( \$OLD_CHANGES ) + count=\${#P[*]} + + for(( i=1; i < count; ++i )) do + j=i + #echo "was \$i: \${P[i]}" + while ((\$j)) && compare_changes "\${P[j-1]}" "\${P[i]}"; do ((--j)); done + ((i==j)) || P=( \${P[@]:0:j} \${P[i]} \${P[j]} \${P[@]:j+1:i-(j+1)} \${P[@]:i+1} ) + done + #for(( i=1; i < count; ++i )) do echo "now \$i: \${P[i]}"; done + + OLD_CHANGES= + for(( i=count-1; i >= 0; --i )) do + if [ "\${P[i]}" != "\$RESULT_DIR/${PACKAGE}_\$CHANGES" ]; then + OLD_CHANGES="\${P[i]}" + break + fi + done + + + set -eo pipefail + + if ! [ -e "\$BASE_PATH" ]; then + if [ "$CREATE_COW" = "yes" ]; then + mkdir -p "\$RESULT_DIR" + mkdir -p "\$(dirname \$BASE_PATH)" + mkdir -p "$PBUILDER_BASE/aptcache" + $BUILDD_ROOTCMD cowbuilder --create --distribution \$BASE_DIST \\ + --basepath "\$BASE_PATH" \\ + --aptcache "$PBUILDER_BASE/aptcache" \\ + --debootstrap "$DEBOOTSTRAP" \\ + --debootstrapopts --arch="\$arch" \\ + "\${CREATE_OPTS[@]}" \\ + 2>&1 | tee "\$UPDATELOG" + else + echo "SKIPPING \$dist/\$arch build, '\$BASE_PATH' does not exist" | tee "\$LOGFILE" + echo " use the cowpoke --create option to bootstrap a new build root" | tee -a "\$LOGFILE" + continue + fi + elif ! [ -e "\$UPDATELOG" ]; then + $BUILDD_ROOTCMD cowbuilder --update --distribution \$BASE_DIST \\ + --basepath "\$BASE_PATH" \\ + --aptcache "$PBUILDER_BASE/aptcache" \\ + --autocleanaptcache \\ + "\${UPDATE_OPTS[@]}" \\ + 2>&1 | tee "\$UPDATELOG" + fi + $BUILDD_ROOTCMD cowbuilder --build --basepath "\$BASE_PATH" \\ + --aptcache "$PBUILDER_BASE/aptcache" \\ + --buildplace "$PBUILDER_BASE/build" \\ + --buildresult "\$RESULT_DIR" \\ + "\${DEBBUILDOPTS[@]}" \\ + "\${BUILD_OPTS[@]}" \\ + "$INCOMING_DIR/$(basename $DSC)" 2>&1 \\ + | tee "\$LOGFILE" + + set +eo pipefail + + + echo >> "\$LOGFILE" + echo "lintian \$RESULT_DIR/${PACKAGE}_\$CHANGES" >> "\$LOGFILE" + lintian "\$RESULT_DIR/${PACKAGE}_\$CHANGES" 2>&1 | tee -a "\$LOGFILE" + + if [ -n "\$OLD_CHANGES" ]; then + echo >> "\$LOGFILE" + echo "debdiff \$OLD_CHANGES ${PACKAGE}_\$CHANGES" >> "\$LOGFILE" + debdiff "\$OLD_CHANGES" "\$RESULT_DIR/${PACKAGE}_\$CHANGES" 2>&1 \\ + | tee -a "\$LOGFILE" + else + echo >> "\$LOGFILE" + echo "No previous packages for \$dist/\$arch to compare" >> "\$LOGFILE" + fi + + done + done + +EOF +chmod 755 "$REMOTE_SCRIPT" + + +if ! dcmd rsync -vP $DSC "$REMOTE_SCRIPT" "$BUILDD_USER$BUILDD_HOST:$INCOMING_DIR"; +then + dcmd scp $DSC "$REMOTE_SCRIPT" "$BUILDD_USER$BUILDD_HOST:$INCOMING_DIR" +fi + +ssh -t "$BUILDD_USER$BUILDD_HOST" "\"$INCOMING_DIR/$REMOTE_SCRIPT\" && rm -f \"$INCOMING_DIR/$REMOTE_SCRIPT\"" + +echo +echo "Build completed." + +for arch in $BUILDD_ARCH; do + CHANGES="$arch.changes" + for dist in $BUILDD_DIST; do + + sign_keyid=$OVERRIDE_SIGN_KEYID + [ -n "$sign_keyid" ] || eval "sign_keyid=\"\$${arch}_${dist}_SIGN_KEYID\"" + [ -n "$sign_keyid" ] || sign_keyid="$SIGN_KEYID" + [ -n "$sign_keyid" ] || continue + + eval "RESULT_DIR=\"\$${arch}_${dist}_RESULT_DIR\"" + [ -n "$RESULT_DIR" ] || RESULT_DIR="$PBUILDER_BASE/$arch/$dist/result" + + _desc="$dist/$arch" + [ "$dist" != "default" ] || _desc="$arch" + + while true; do + echo -n "Sign $_desc $PACKAGE with key '$sign_keyid' (yes/no)? " + read -e yesno + case "$yesno" in + YES | yes) + debsign "-k$sign_keyid" -r "$BUILDD_USER$BUILDD_HOST" "$RESULT_DIR/${PACKAGE}_$CHANGES" + + upload_queue=$OVERRIDE_UPLOAD_QUEUE + [ -n "$upload_queue" ] || eval "upload_queue=\"\$${arch}_${dist}_UPLOAD_QUEUE\"" + [ -n "$upload_queue" ] || upload_queue="$UPLOAD_QUEUE" + + if [ -n "$upload_queue" ]; then + while true; do + echo -n "Upload $_desc $PACKAGE to '$upload_queue' (yes/no)? " + read -e upload + case "$upload" in + YES | yes) + ssh "$BUILDD_USER$BUILDD_HOST" \ + "cd \"$RESULT_DIR/\" && dput \"$upload_queue\" \"${PACKAGE}_$CHANGES\"" + break 2 + ;; + + NO | no) + echo "Package upload skipped." + break 2 + ;; + *) + echo "Please answer 'yes' or 'no'" + ;; + esac + done + fi + break + ;; + + NO | no) + echo "Package signing skipped." + break + ;; + *) + echo "Please answer 'yes' or 'no'" + ;; + esac + done + done +done + +if [ -n "$RETURN_DIR" ]; then + for arch in $BUILDD_ARCH; do + CHANGES="$arch.changes" + for dist in $BUILDD_DIST; do + + eval "RESULT_DIR=\"\$${arch}_${dist}_RESULT_DIR\"" + [ -n "$RESULT_DIR" ] || RESULT_DIR="$PBUILDER_BASE/$arch/$dist/result" + + + cache_dir="./cowpoke-return-cache" + mkdir -p $cache_dir + + scp "$BUILDD_USER$BUILDD_HOST:$RESULT_DIR/${PACKAGE}_$CHANGES" $cache_dir + + for f in $(cd $cache_dir && dcmd ${PACKAGE}_$CHANGES); do + RESULTS="$RESULTS $RESULT_DIR/$f" + done + + rm -f $cache_dir/${PACKAGE}_$CHANGES + rmdir $cache_dir + + + if ! rsync -vP "$BUILDD_USER$BUILDD_HOST:$RESULTS" "$RETURN_DIR" ; + then + scp "$BUILDD_USER$BUILDD_HOST:$RESULTS" "$RETURN_DIR" + fi + + done + done +fi + +rm -f "$REMOTE_SCRIPT" + +# vi:sts=4:sw=4:noet:foldmethod=marker |