summaryrefslogtreecommitdiffstats
path: root/taskcluster/docker/diffoscope
diff options
context:
space:
mode:
Diffstat (limited to 'taskcluster/docker/diffoscope')
-rw-r--r--taskcluster/docker/diffoscope/Dockerfile34
-rw-r--r--taskcluster/docker/diffoscope/get_and_diffoscope109
-rw-r--r--taskcluster/docker/diffoscope/readelf13
-rwxr-xr-xtaskcluster/docker/diffoscope/report_error36
-rw-r--r--taskcluster/docker/diffoscope/test_diffoscope47
5 files changed, 239 insertions, 0 deletions
diff --git a/taskcluster/docker/diffoscope/Dockerfile b/taskcluster/docker/diffoscope/Dockerfile
new file mode 100644
index 0000000000..1ac4addbd8
--- /dev/null
+++ b/taskcluster/docker/diffoscope/Dockerfile
@@ -0,0 +1,34 @@
+FROM $DOCKER_IMAGE_PARENT
+MAINTAINER Mike Hommey <mhommey@mozilla.com>
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/workspace
+VOLUME /builds/worker/tooltool-cache
+
+ENV LANG=en_US.UTF-8
+
+RUN apt-get install \
+ binutils-multiarch \
+ bzip2 \
+ curl \
+ enjarify \
+ diffoscope \
+ jsbeautifier \
+ libc++abi1 \
+ locales \
+ default-jdk-headless \
+ python3-progressbar \
+ unzip \
+ xxd \
+ zip \
+ && \
+ sed -i '/en_US.UTF-8/s/^# *//' /etc/locale.gen && \
+ locale-gen
+
+COPY get_and_diffoscope /builds/worker/bin/get_and_diffoscope
+COPY readelf /builds/worker/bin/readelf
+COPY report_error /builds/worker/bin/report_error
+COPY test_diffoscope /builds/worker/bin/test_diffoscope
+
+RUN chown -R worker:worker /builds/worker/bin && chmod 755 /builds/worker/bin/* && \
+ /builds/worker/bin/test_diffoscope
diff --git a/taskcluster/docker/diffoscope/get_and_diffoscope b/taskcluster/docker/diffoscope/get_and_diffoscope
new file mode 100644
index 0000000000..8f72970f77
--- /dev/null
+++ b/taskcluster/docker/diffoscope/get_and_diffoscope
@@ -0,0 +1,109 @@
+#!/bin/bash
+
+set -e
+set -x
+
+cd /builds/worker
+
+mkdir a b
+
+# /builds/worker/bin contains wrapper binaries to divert what diffoscope
+# needs to use, so it needs to appear first.
+export PATH=/builds/worker/bin:$PATH
+
+# Until https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=879010 is
+# implemented, it's better to first manually extract the data.
+# Plus dmg files are not supported yet.
+
+RESOURCE_DIR=firefox
+case "$ORIG_URL" in
+*.zip|*.apk)
+ curl -L "$ORIG_URL" > a.zip
+ curl -L "$NEW_URL" > b.zip
+ unzip -d a a.zip
+ unzip -d b b.zip
+ ;;
+*.tar.zst)
+ curl -L "$ORIG_URL" | tar -C a --zstd -xf -
+ curl -L "$NEW_URL" | tar -C b --zstd -xf -
+ ;;
+*.tar.bz2)
+ curl -L "$ORIG_URL" | tar -C a -jxf -
+ curl -L "$NEW_URL" | tar -C b -jxf -
+ ;;
+*.tar.gz)
+ curl -L "$ORIG_URL" | tar -C a -zxf -
+ curl -L "$NEW_URL" | tar -C b -zxf -
+ ;;
+*.dmg)
+ for tool in lipo otool; do
+ ln -s $MOZ_FETCHES_DIR/cctools/bin/x86_64-apple-darwin*-$tool bin/$tool
+ done
+ curl -L "$ORIG_URL" > a.dmg
+ curl -L "$NEW_URL" > b.dmg
+ for i in a b; do
+ $MOZ_FETCHES_DIR/dmg/dmg extract $i.dmg $i.hfs
+ $MOZ_FETCHES_DIR/dmg/hfsplus $i.hfs extractall / $i
+ done
+ RESOURCE_DIR=$(cd b; echo */Contents/Resources)
+ ;;
+*)
+ ARTIFACT=$(basename "${ORIG_URL}")
+ curl -L "$ORIG_URL" > "a/${ARTIFACT}"
+ curl -L "$NEW_URL" > "b/${ARTIFACT}"
+esac
+
+case "$ORIG_URL" in
+*/target.apk)
+ OMNIJAR=assets/omni.ja
+ ;;
+*)
+ OMNIJAR=omni.ja
+ ;;
+esac
+
+# Builds are 99% of the time differing in some small ways, so it's not
+# really useful to report a failure (at least not until we actually
+# care about the builds being 100% identical).
+POST=true
+
+fail() {
+ exit 1
+}
+
+for option; do
+ case "$option" in
+ --unpack)
+ CURDIR=$PWD
+ for dir in a b; do
+ # Need to run mach python from inside the gecko source.
+ # See bug #1533642.
+ (cd $GECKO_PATH && ./mach python toolkit/mozapps/installer/unpack.py --omnijar $OMNIJAR "$CURDIR/$dir/$RESOURCE_DIR")
+ done
+ ;;
+ --fail)
+ POST="fail"
+ ;;
+ *)
+ echo "Unsupported option: $option" >&2
+ exit 1
+ esac
+done
+
+if [ -n "$PRE_DIFF" ]; then
+ eval $PRE_DIFF
+fi
+
+if eval diffoscope \
+ --html diff.html \
+ --text diff.txt \
+ --progress \
+ $DIFFOSCOPE_ARGS \
+ a b
+then
+ # Ok
+ :
+else
+ $(dirname $0)/report_error diff
+ $POST
+fi
diff --git a/taskcluster/docker/diffoscope/readelf b/taskcluster/docker/diffoscope/readelf
new file mode 100644
index 0000000000..6b864171d7
--- /dev/null
+++ b/taskcluster/docker/diffoscope/readelf
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+case "$1 $2" in
+"--wide --symbols")
+ # When called with --wide --symbols, we remove the first column (which
+ # is essentially a line number that is not very useful), and then sort,
+ # which will order symbols by address, making a diff more useful.
+ /usr/bin/readelf "$@" | awk -F: '{print $2}' | sort
+ ;;
+*)
+ exec /usr/bin/readelf "$@"
+ ;;
+esac
diff --git a/taskcluster/docker/diffoscope/report_error b/taskcluster/docker/diffoscope/report_error
new file mode 100755
index 0000000000..9074e33cae
--- /dev/null
+++ b/taskcluster/docker/diffoscope/report_error
@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+import sys
+stem = sys.argv[1]
+
+# We "parse" the diff output, so we look at the lines that contain a "tee", like:
+# ├── +++ b/firefox
+# │ ├── +++ b/firefox/libxul.so
+# │ │ ├── readelf --wide --notes {}
+# We ignore lines like the last one, to only report file names. And we ignore
+# lines for directories such as the first one.
+TEE = "├── "
+VERTICAL_LINE = "│"
+paths = set()
+with open(f"{stem}.txt") as fh:
+ for l in fh:
+ l = l.rstrip()
+ before, tee, after = l.partition(TEE)
+ if not tee:
+ continue
+ before = before.split()
+ assert all(x == VERTICAL_LINE for x in before)
+ depth = len(before)
+ _, plus, after = after.partition("+++ ")
+ if not plus:
+ continue
+ _, b, full_path = after.partition("b/")
+ assert b == "b/"
+ parent_path = "/".join(full_path.split("/")[:-1])
+ if parent_path in paths:
+ paths.remove(parent_path)
+ if full_path:
+ paths.add(full_path)
+
+for p in sorted(paths):
+ print(f"TEST-UNEXPECTED-FAIL | {p} differs. See the {stem}.html or {stem}.txt artifact")
diff --git a/taskcluster/docker/diffoscope/test_diffoscope b/taskcluster/docker/diffoscope/test_diffoscope
new file mode 100644
index 0000000000..ab1c6c1eee
--- /dev/null
+++ b/taskcluster/docker/diffoscope/test_diffoscope
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -e
+
+WORKDIR=$(mktemp -d)
+cd $WORKDIR
+
+mkdir -p a/foo/bar/bar
+mkdir -p a/foo/bar/baz
+mkdir -p b/foo/bar/bar
+mkdir -p b/foo/bar/baz
+
+# A file that is modified
+echo qux > a/foo/bar/qux
+echo quz > b/foo/bar/qux
+
+# A binary file that is modified
+cp $(which ls) a/foo/bin
+cp $(which cat) b/foo/bin
+
+# A file that is removed
+echo hoge > a/foo/bar/bar/hoge
+
+# A file that is created
+echo fuga > b/foo/bar/baz/fuga
+
+# Also add a zip file with the same contents
+(cd a/foo; zip -r bar.zip bar)
+(cd b/foo; zip -r bar.zip bar)
+
+if TERM=linux diffoscope --no-progress --text diff.txt a b; then
+ echo "diffoscope didn't find differences?"
+ exit 1
+fi
+
+cat > expected.txt <<EOF
+TEST-UNEXPECTED-FAIL | foo/bar.zip differs. See the diff.html or diff.txt artifact
+TEST-UNEXPECTED-FAIL | foo/bar/bar differs. See the diff.html or diff.txt artifact
+TEST-UNEXPECTED-FAIL | foo/bar/baz differs. See the diff.html or diff.txt artifact
+TEST-UNEXPECTED-FAIL | foo/bar/qux differs. See the diff.html or diff.txt artifact
+TEST-UNEXPECTED-FAIL | foo/bin differs. See the diff.html or diff.txt artifact
+EOF
+
+$(dirname $0)/report_error diff | diff -u - expected.txt || exit 1
+
+cd $OLDPWD
+rm -rf $WORKDIR