diff options
Diffstat (limited to 'taskcluster/docker/diffoscope')
| -rw-r--r-- | taskcluster/docker/diffoscope/Dockerfile | 34 | ||||
| -rw-r--r-- | taskcluster/docker/diffoscope/get_and_diffoscope | 109 | ||||
| -rw-r--r-- | taskcluster/docker/diffoscope/readelf | 13 | ||||
| -rwxr-xr-x | taskcluster/docker/diffoscope/report_error | 36 | ||||
| -rw-r--r-- | taskcluster/docker/diffoscope/test_diffoscope | 47 |
5 files changed, 239 insertions, 0 deletions
diff --git a/taskcluster/docker/diffoscope/Dockerfile b/taskcluster/docker/diffoscope/Dockerfile new file mode 100644 index 0000000000..1ac4addbd8 --- /dev/null +++ b/taskcluster/docker/diffoscope/Dockerfile @@ -0,0 +1,34 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Mike Hommey <mhommey@mozilla.com> + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +ENV LANG=en_US.UTF-8 + +RUN apt-get install \ + binutils-multiarch \ + bzip2 \ + curl \ + enjarify \ + diffoscope \ + jsbeautifier \ + libc++abi1 \ + locales \ + default-jdk-headless \ + python3-progressbar \ + unzip \ + xxd \ + zip \ + && \ + sed -i '/en_US.UTF-8/s/^# *//' /etc/locale.gen && \ + locale-gen + +COPY get_and_diffoscope /builds/worker/bin/get_and_diffoscope +COPY readelf /builds/worker/bin/readelf +COPY report_error /builds/worker/bin/report_error +COPY test_diffoscope /builds/worker/bin/test_diffoscope + +RUN chown -R worker:worker /builds/worker/bin && chmod 755 /builds/worker/bin/* && \ + /builds/worker/bin/test_diffoscope diff --git a/taskcluster/docker/diffoscope/get_and_diffoscope b/taskcluster/docker/diffoscope/get_and_diffoscope new file mode 100644 index 0000000000..8f72970f77 --- /dev/null +++ b/taskcluster/docker/diffoscope/get_and_diffoscope @@ -0,0 +1,109 @@ +#!/bin/bash + +set -e +set -x + +cd /builds/worker + +mkdir a b + +# /builds/worker/bin contains wrapper binaries to divert what diffoscope +# needs to use, so it needs to appear first. +export PATH=/builds/worker/bin:$PATH + +# Until https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=879010 is +# implemented, it's better to first manually extract the data. +# Plus dmg files are not supported yet. + +RESOURCE_DIR=firefox +case "$ORIG_URL" in +*.zip|*.apk) + curl -L "$ORIG_URL" > a.zip + curl -L "$NEW_URL" > b.zip + unzip -d a a.zip + unzip -d b b.zip + ;; +*.tar.zst) + curl -L "$ORIG_URL" | tar -C a --zstd -xf - + curl -L "$NEW_URL" | tar -C b --zstd -xf - + ;; +*.tar.bz2) + curl -L "$ORIG_URL" | tar -C a -jxf - + curl -L "$NEW_URL" | tar -C b -jxf - + ;; +*.tar.gz) + curl -L "$ORIG_URL" | tar -C a -zxf - + curl -L "$NEW_URL" | tar -C b -zxf - + ;; +*.dmg) + for tool in lipo otool; do + ln -s $MOZ_FETCHES_DIR/cctools/bin/x86_64-apple-darwin*-$tool bin/$tool + done + curl -L "$ORIG_URL" > a.dmg + curl -L "$NEW_URL" > b.dmg + for i in a b; do + $MOZ_FETCHES_DIR/dmg/dmg extract $i.dmg $i.hfs + $MOZ_FETCHES_DIR/dmg/hfsplus $i.hfs extractall / $i + done + RESOURCE_DIR=$(cd b; echo */Contents/Resources) + ;; +*) + ARTIFACT=$(basename "${ORIG_URL}") + curl -L "$ORIG_URL" > "a/${ARTIFACT}" + curl -L "$NEW_URL" > "b/${ARTIFACT}" +esac + +case "$ORIG_URL" in +*/target.apk) + OMNIJAR=assets/omni.ja + ;; +*) + OMNIJAR=omni.ja + ;; +esac + +# Builds are 99% of the time differing in some small ways, so it's not +# really useful to report a failure (at least not until we actually +# care about the builds being 100% identical). +POST=true + +fail() { + exit 1 +} + +for option; do + case "$option" in + --unpack) + CURDIR=$PWD + for dir in a b; do + # Need to run mach python from inside the gecko source. + # See bug #1533642. + (cd $GECKO_PATH && ./mach python toolkit/mozapps/installer/unpack.py --omnijar $OMNIJAR "$CURDIR/$dir/$RESOURCE_DIR") + done + ;; + --fail) + POST="fail" + ;; + *) + echo "Unsupported option: $option" >&2 + exit 1 + esac +done + +if [ -n "$PRE_DIFF" ]; then + eval $PRE_DIFF +fi + +if eval diffoscope \ + --html diff.html \ + --text diff.txt \ + --progress \ + $DIFFOSCOPE_ARGS \ + a b +then + # Ok + : +else + $(dirname $0)/report_error diff + $POST +fi diff --git a/taskcluster/docker/diffoscope/readelf b/taskcluster/docker/diffoscope/readelf new file mode 100644 index 0000000000..6b864171d7 --- /dev/null +++ b/taskcluster/docker/diffoscope/readelf @@ -0,0 +1,13 @@ +#!/bin/sh + +case "$1 $2" in +"--wide --symbols") + # When called with --wide --symbols, we remove the first column (which + # is essentially a line number that is not very useful), and then sort, + # which will order symbols by address, making a diff more useful. + /usr/bin/readelf "$@" | awk -F: '{print $2}' | sort + ;; +*) + exec /usr/bin/readelf "$@" + ;; +esac diff --git a/taskcluster/docker/diffoscope/report_error b/taskcluster/docker/diffoscope/report_error new file mode 100755 index 0000000000..9074e33cae --- /dev/null +++ b/taskcluster/docker/diffoscope/report_error @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +import sys +stem = sys.argv[1] + +# We "parse" the diff output, so we look at the lines that contain a "tee", like: +# ├── +++ b/firefox +# │ ├── +++ b/firefox/libxul.so +# │ │ ├── readelf --wide --notes {} +# We ignore lines like the last one, to only report file names. And we ignore +# lines for directories such as the first one. +TEE = "├── " +VERTICAL_LINE = "│" +paths = set() +with open(f"{stem}.txt") as fh: + for l in fh: + l = l.rstrip() + before, tee, after = l.partition(TEE) + if not tee: + continue + before = before.split() + assert all(x == VERTICAL_LINE for x in before) + depth = len(before) + _, plus, after = after.partition("+++ ") + if not plus: + continue + _, b, full_path = after.partition("b/") + assert b == "b/" + parent_path = "/".join(full_path.split("/")[:-1]) + if parent_path in paths: + paths.remove(parent_path) + if full_path: + paths.add(full_path) + +for p in sorted(paths): + print(f"TEST-UNEXPECTED-FAIL | {p} differs. See the {stem}.html or {stem}.txt artifact") diff --git a/taskcluster/docker/diffoscope/test_diffoscope b/taskcluster/docker/diffoscope/test_diffoscope new file mode 100644 index 0000000000..ab1c6c1eee --- /dev/null +++ b/taskcluster/docker/diffoscope/test_diffoscope @@ -0,0 +1,47 @@ +#!/bin/bash + +set -e + +WORKDIR=$(mktemp -d) +cd $WORKDIR + +mkdir -p a/foo/bar/bar +mkdir -p a/foo/bar/baz +mkdir -p b/foo/bar/bar +mkdir -p b/foo/bar/baz + +# A file that is modified +echo qux > a/foo/bar/qux +echo quz > b/foo/bar/qux + +# A binary file that is modified +cp $(which ls) a/foo/bin +cp $(which cat) b/foo/bin + +# A file that is removed +echo hoge > a/foo/bar/bar/hoge + +# A file that is created +echo fuga > b/foo/bar/baz/fuga + +# Also add a zip file with the same contents +(cd a/foo; zip -r bar.zip bar) +(cd b/foo; zip -r bar.zip bar) + +if TERM=linux diffoscope --no-progress --text diff.txt a b; then + echo "diffoscope didn't find differences?" + exit 1 +fi + +cat > expected.txt <<EOF +TEST-UNEXPECTED-FAIL | foo/bar.zip differs. See the diff.html or diff.txt artifact +TEST-UNEXPECTED-FAIL | foo/bar/bar differs. See the diff.html or diff.txt artifact +TEST-UNEXPECTED-FAIL | foo/bar/baz differs. See the diff.html or diff.txt artifact +TEST-UNEXPECTED-FAIL | foo/bar/qux differs. See the diff.html or diff.txt artifact +TEST-UNEXPECTED-FAIL | foo/bin differs. See the diff.html or diff.txt artifact +EOF + +$(dirname $0)/report_error diff | diff -u - expected.txt || exit 1 + +cd $OLDPWD +rm -rf $WORKDIR |