diff options
Diffstat (limited to '')
5 files changed, 282 insertions, 0 deletions
diff --git a/taskcluster/docker/python-dependency-update/Dockerfile b/taskcluster/docker/python-dependency-update/Dockerfile new file mode 100644 index 0000000000..cb4ac2a381 --- /dev/null +++ b/taskcluster/docker/python-dependency-update/Dockerfile @@ -0,0 +1,33 @@ +FROM ubuntu:bionic +MAINTAINER Ben Hearsum <bhearsum@mozilla.com> + +# Required software +ENV DEBIAN_FRONTEND noninteractive + +# %include python/mozbuild/mozbuild/action/tooltool.py +ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py + +# %include testing/mozharness/external_tools/robustcheckout.py +ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py + +# %include taskcluster/docker/recipes/hgrc +COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc + +# %include taskcluster/docker/recipes/install-mercurial.sh +ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh + +ADD setup.sh /setup/setup.sh + +RUN cd /setup && ./setup.sh + +COPY runme.sh / +COPY scripts/* /home/worker/scripts/ + +ENV HOME /home/worker +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME worker +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 + +CMD ["/runme.sh"] diff --git a/taskcluster/docker/python-dependency-update/README.md b/taskcluster/docker/python-dependency-update/README.md new file mode 100644 index 0000000000..ab560623ee --- /dev/null +++ b/taskcluster/docker/python-dependency-update/README.md @@ -0,0 +1,15 @@ + +==Python Dependency Updates== + +This docker image contains the necessary dependencies and scripts to update +in-tree requirement.txt produced by `pip-compile --generate-hashes`, produce a +diff, and submit it to Phabricator. + + +==Quick Start== + +```sh +docker build -t python-dependency-update --no-cache --rm . + +docker run -e PYTHON3="1" -e BRANCH="mozilla-central" -e REQUIREMENTS_FILE="taskcluster/docker/funsize-update-generator/requirements.in" python-dependency-update +``` diff --git a/taskcluster/docker/python-dependency-update/runme.sh b/taskcluster/docker/python-dependency-update/runme.sh new file mode 100755 index 0000000000..395ec0cd64 --- /dev/null +++ b/taskcluster/docker/python-dependency-update/runme.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -xe + +# Things to be set by task definition. +# -b branch +# -f requirements_file +# -3 use python3 + + +test "${BRANCH}" +test "${REQUIREMENTS_FILE}" + +PIP_ARG="-2" +if [ -n "${PYTHON3}" ]; then + PIP_ARG="-3" +fi + +export ARTIFACTS_DIR="/home/worker/artifacts" +mkdir -p "$ARTIFACTS_DIR" + +# duplicate the functionality of taskcluster-lib-urls, but in bash.. +queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" + +# Get Arcanist API token + +if [ -n "${TASK_ID}" ] +then + curl --location --retry 10 --retry-delay 10 -o /home/worker/task.json "$queue_base/task/$TASK_ID" + ARC_SECRET=$(jq -r '.scopes[] | select(contains ("arc-phabricator-token"))' /home/worker/task.json | awk -F: '{print $3}') +fi +if [ -n "${ARC_SECRET}" ] && getent hosts taskcluster +then + set +x # Don't echo these + secrets_url="${TASKCLUSTER_PROXY_URL}/api/secrets/v1/secret/${ARC_SECRET}" + SECRET=$(curl "${secrets_url}") + TOKEN=$(echo "${SECRET}" | jq -r '.secret.token') +elif [ -n "${ARC_TOKEN}" ] # Allow for local testing. +then + TOKEN="${ARC_TOKEN}" +fi + +if [ -n "${TOKEN}" ] +then + cat >"${HOME}/.arcrc" <<END +{ + "hosts": { + "https://phabricator.services.mozilla.com/api/": { + "token": "${TOKEN}" + } + } +} +END + set -x + chmod 600 "${HOME}/.arcrc" +fi + +export HGPLAIN=1 + +# shellcheck disable=SC2086 +/home/worker/scripts/update_pipfiles.sh -b "${BRANCH}" -f "${REQUIREMENTS_FILE}" ${PIP_ARG} diff --git a/taskcluster/docker/python-dependency-update/scripts/update_pipfiles.sh b/taskcluster/docker/python-dependency-update/scripts/update_pipfiles.sh new file mode 100755 index 0000000000..c6f9b10701 --- /dev/null +++ b/taskcluster/docker/python-dependency-update/scripts/update_pipfiles.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +set -e + +function usage { + cat <<EOF + +Usage: $(basename "$0") -h # Displays this usage/help text +Usage: $(basename "$0") -x # lists exit codes +Usage: $(basename "$0") -b branch -r REQUIREMENTS_FILE [-2] [-3] + +EOF +} + +BRANCH="" +PIP="" +COMMIT_AUTHOR='ffxbld <ffxbld@mozilla.com>' +REPODIR='' +HGHOST="hg.mozilla.org" +BASEDIR="${HOME}" +REQUIREMENTS_FILE="" + +HG="$(command -v hg)" + +# Clones an hg repo +function clone_repo { + cd "${BASEDIR}" + if [ ! -d "${REPODIR}" ]; then + CLONE_CMD="${HG} clone ${HGREPO} ${REPODIR}" + ${CLONE_CMD} + fi + + ${HG} -R "${REPODIR}" pull + ${HG} -R "${REPODIR}" update -C default +} + +# Push all pending commits to Phabricator +function push_repo { + cd "${REPODIR}" + if [ ! -r "${HOME}/.arcrc" ] + then + return 1 + fi + if ! ARC=$(command -v arc) + then + return 1 + fi + if [ -z "${REVIEWERS}" ] + then + return 1 + fi + # Clean up older review requests + # Turn Needs Review D624: No bug, Automated HSTS ... + # into D624 + for diff in $($ARC list | grep "Needs Review" | grep "${REQUIREMENTS_FILE} pip-update" | awk 'match($0, /D[0-9]+[^: ]/) { print substr($0, RSTART, RLENGTH) }') + do + echo "Removing old request $diff" + # There is no 'arc abandon', see bug 1452082 + echo '{"transactions": [{"type":"abandon"}], "objectIdentifier": "'"${diff}"'"}' | arc call-conduit differential.revision.edit + done + + $ARC diff --verbatim --reviewers "${REVIEWERS}" +} + +function update_requirements { + pushd "${REPODIR}/${1}" + pip-compile --generate-hashes "${2}" + popd +} + +# Main + +# Parse our command-line options. +while [ $# -gt 0 ]; do + case "$1" in + -h) usage; exit 0 ;; + -b) BRANCH="$2"; shift ;; + -r) REPODIR="$2"; shift ;; + -2) PIP="pip" ;; + -3) PIP="pip3" ;; + -f) REQUIREMENTS_FILE="$2"; shift ;; + -*) usage + exit 11 ;; + *) break ;; # terminate while loop + esac + shift +done + +# Must supply a code branch to work with. +if [ "${PIP}" == "" ]; then + echo "Error: You must specify a python version with -2 or -3" >&2 + usage + exit 12 +fi + +# Must supply a code branch to work with. +if [ "${BRANCH}" == "" ]; then + echo "Error: You must specify a branch with -b branchname." >&2 + usage + exit 13 +fi + +if [ "${REPODIR}" == "" ]; then + REPODIR="${BASEDIR}/$(basename "${BRANCH}")" +fi + +if [ "${BRANCH}" == "mozilla-central" ]; then + HGREPO="https://${HGHOST}/${BRANCH}" +elif [[ "${BRANCH}" == mozilla-* ]]; then + HGREPO="https://${HGHOST}/releases/${BRANCH}" +else + HGREPO="https://${HGHOST}/projects/${BRANCH}" +fi + +clone_repo + +${PIP} install pip-tools + +requirements_basefile="$(basename "${REQUIREMENTS_FILE}")" +requirements_dir="$(dirname "${REQUIREMENTS_FILE}")" +update_requirements "${requirements_dir}" "${requirements_basefile}" +requirements_newfile="${requirements_basefile%%.in}.txt" +DIFF_ARTIFACT="${ARTIFACTS_DIR}/${requirements_newfile}.diff" + +echo "INFO: diffing old/new ${requirements_newfile} into ${DIFF_ARTIFACT}" +${HG} -R "${REPODIR}" diff "${BASEDIR}/${BRANCH}/${requirements_dir}/${requirements_newfile}" | tee "${DIFF_ARTIFACT}" + +COMMIT_MESSAGE="No Bug, ${requirements_dir}/${requirements_newfile} pip-update." + +if ${HG} -R "${REPODIR}" commit -u "${COMMIT_AUTHOR}" -m "${COMMIT_MESSAGE}" +then + ${HG} -R "${REPODIR}" out + push_repo +fi + +echo "All done" diff --git a/taskcluster/docker/python-dependency-update/setup.sh b/taskcluster/docker/python-dependency-update/setup.sh new file mode 100755 index 0000000000..99d59ed7a9 --- /dev/null +++ b/taskcluster/docker/python-dependency-update/setup.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set -ve + +tooltool_fetch() { + cat >manifest.tt + python2.7 /setup/tooltool.py fetch + rm manifest.tt +} + +useradd -d /home/worker -s /bin/bash -m worker + +apt-get update -q +apt-get install -y --no-install-recommends \ + arcanist \ + curl \ + gcc \ + jq \ + libdpkg-perl \ + liblzma-dev \ + python \ + python-dev \ + python-pip \ + python3.8 \ + python3.8-dev \ + python3-setuptools \ + python3-wheel \ + python3-pip + +apt-get clean + +. install-mercurial.sh + +rm -rf /setup |