summaryrefslogtreecommitdiffstats
path: root/taskcluster/docker
diff options
context:
space:
mode:
Diffstat (limited to 'taskcluster/docker')
-rw-r--r--taskcluster/docker/debian-repackage/Dockerfile36
-rw-r--r--taskcluster/docker/github-sync/Dockerfile14
-rwxr-xr-xtaskcluster/docker/github-sync/prepare.sh31
-rw-r--r--taskcluster/docker/github-sync/requirements.txt3
-rw-r--r--taskcluster/docker/sentry/Dockerfile2
-rw-r--r--taskcluster/docker/snap-coreXX-build/snap-tests/README.md4
-rw-r--r--taskcluster/docker/snap-coreXX-build/snap-tests/record.py24
-rwxr-xr-xtaskcluster/docker/snap-coreXX-build/snap-tests/tests.sh10
-rw-r--r--taskcluster/docker/system-symbols-linux-scraper/Dockerfile2
-rw-r--r--taskcluster/docker/system-symbols-linux-scraper/README64
-rw-r--r--taskcluster/docker/updatebot/updatebot-version.sh2
-rw-r--r--taskcluster/docker/updatebot/windows-setup.sh4
12 files changed, 129 insertions, 67 deletions
diff --git a/taskcluster/docker/debian-repackage/Dockerfile b/taskcluster/docker/debian-repackage/Dockerfile
index ae257931d4..f068731b58 100644
--- a/taskcluster/docker/debian-repackage/Dockerfile
+++ b/taskcluster/docker/debian-repackage/Dockerfile
@@ -3,22 +3,16 @@ MAINTAINER Mozilla Releng <release@mozilla.com>
VOLUME /builds/worker/workspace
-# At the moment the Firefox build baseline is jessie.
-# Things in the build/CI system that run in the task's container
-# (mach, run-task, etc.) do not support jessie.
-# There's also no jessie `base` images or packages generated in the CI.
-# To generate Firefox Debian packages compatible with the baseline
-# we bootstrap basic i386 and amd64 jessie systems on bullseye.
-# We use these to generate shared library dependencies in jessie using chroot.
-# python/mozbuild/mozbuild/repackaging/deb.py checks for jessie systems
-# bootstrapped under /srv/jessie-i386 and /srv/jessie-amd64
-# If they aren't there, deb.py runs the repackage in the working directory without using chroot.
-# To keep the build and repackage enviroments consistent the Debian baseline used
-# here (jessie) should be synchronized with the baseline used in
-# taskcluster/scripts/misc/build-sysroot.sh
+# At the moment the Firefox build sysroot for i386 and amd64 use jessie and arm64 uses buster.
+# To generate Firefox Debian packages we bootstrap jessie and buster sysroots.
+# We use these sysroots to generate shared library dependencies for
+# i386 (jessie), amd64 (jessie), and arm64 (buster) packages.
+# To keep the build and repackage enviroments consistent the Debian distributions used by the sysroots
+# created here should match the distributions used by the sysroots created in taskcluster/scripts/misc/build-sysroot.sh
ARG TASKCLUSTER_ROOT_URL
ARG DOCKER_IMAGE_PACKAGES
RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES && \
+ dpkg --add-architecture arm64 && \
apt-get update && \
apt-get install debootstrap && set -xe && \
for arch in i386 amd64; do \
@@ -30,6 +24,20 @@ RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAG
--verbose \
jessie \
/srv/jessie-$arch \
- https://archive.debian.org/debian; \
+ https://archive.debian.org/debian || (echo "Debootstrap failed, outputting log:" && cat /srv/jessie-$arch/debootstrap/debootstrap.log && exit 1); \
done && \
+ # bootstrap a buster/amd64 root to generate an arm64 .deb package.
+ debootstrap \
+ --arch=amd64 \
+ --variant=buildd \
+ --include=debhelper,binutils-multiarch,binutils-aarch64-linux-gnu \
+ --keyring=/usr/share/keyrings/debian-archive-buster-stable.gpg \
+ --verbose \
+ buster \
+ /srv/buster-amd64 \
+ http://deb.debian.org/debian || (echo "Debootstrap failed, outputting log:" && cat /srv/buster-amd64/debootstrap/debootstrap.log && exit 1) && \
+ chroot /srv/buster-amd64 /bin/bash -c "\
+ dpkg --add-architecture arm64 && \
+ apt-get update && \
+ apt-get install -y libasound2:arm64 libgtk-3-0:arm64 libx11-xcb1:arm64 libxtst6:arm64" && \
/usr/local/sbin/clean_packages.sh $DOCKER_IMAGE_PACKAGES
diff --git a/taskcluster/docker/github-sync/Dockerfile b/taskcluster/docker/github-sync/Dockerfile
deleted file mode 100644
index bff131ad7b..0000000000
--- a/taskcluster/docker/github-sync/Dockerfile
+++ /dev/null
@@ -1,14 +0,0 @@
-FROM $DOCKER_IMAGE_PARENT
-MAINTAINER Kartikaya Gupta <kgupta@mozilla.com>
-
-VOLUME /builds/worker/checkouts
-VOLUME /builds/worker/workspace
-VOLUME /builds/worker/tooltool-cache
-VOLUME /builds/worker/.ghsync
-
-ADD prepare.sh /setup/prepare-docker.sh
-ADD requirements.txt /setup/requirements.txt
-RUN /bin/bash /setup/prepare-docker.sh /setup/requirements.txt && rm -R /setup
-
-# Set a default command useful for debugging
-CMD ["/bin/bash", "--login"]
diff --git a/taskcluster/docker/github-sync/prepare.sh b/taskcluster/docker/github-sync/prepare.sh
deleted file mode 100755
index 6c0be27168..0000000000
--- a/taskcluster/docker/github-sync/prepare.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-set -o errexit
-set -o nounset
-set -o pipefail
-set -o xtrace
-
-test "$(whoami)" == 'root'
-
-# Install stuff we need
-apt-get -y update
-apt-get install -y \
- cmake \
- curl \
- gcc \
- git \
- g++ \
- libffi-dev \
- libgit2-dev \
- libssl-dev \
- python3 \
- python3-dev \
- python3-pip \
- python3-setuptools
-
-# Python packages
-pip3 install --break-system-packages -r "$1"
diff --git a/taskcluster/docker/github-sync/requirements.txt b/taskcluster/docker/github-sync/requirements.txt
deleted file mode 100644
index 13fd42681d..0000000000
--- a/taskcluster/docker/github-sync/requirements.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-requests == 2.21.0
-pygit2 == 1.11.1
-python-hglib == 2.6.1
diff --git a/taskcluster/docker/sentry/Dockerfile b/taskcluster/docker/sentry/Dockerfile
index 2f606c5529..c07fdd3961 100644
--- a/taskcluster/docker/sentry/Dockerfile
+++ b/taskcluster/docker/sentry/Dockerfile
@@ -1,5 +1,5 @@
FROM $DOCKER_IMAGE_PARENT
-LABEL maintainer="Mitchell Hentges mhentges@mozilla.com"
+LABEL maintainer="Alex Hochheiden ahochheiden@mozilla.com"
VOLUME /builds/worker/checkouts
diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/README.md b/taskcluster/docker/snap-coreXX-build/snap-tests/README.md
index 73f78c72bc..bdc79dbe4b 100644
--- a/taskcluster/docker/snap-coreXX-build/snap-tests/README.md
+++ b/taskcluster/docker/snap-coreXX-build/snap-tests/README.md
@@ -16,6 +16,10 @@ inspecting the browser at the end of a test.
Data URL containing the diff screenshot will be dumped to stdout/stderr when
`TEST_DUMP_DIFF` is set in the environment.
+A recording of the desktop can be captured via GNOME Screencast feature by
+passing `TEST_RECORD_SCREEN=true` in the enviromnent. Upon completion of the
+task it should upload a WebM file.
+
Updating reference screenshots
==============================
- `./mach try fuzzy --push-to-lando --full --env TEST_COLLECT_REFERENCE=1 -q "'snap-upstream-test"`
diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/record.py b/taskcluster/docker/snap-coreXX-build/snap-tests/record.py
new file mode 100644
index 0000000000..51fe47f6d8
--- /dev/null
+++ b/taskcluster/docker/snap-coreXX-build/snap-tests/record.py
@@ -0,0 +1,24 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+import os
+import time
+
+import dbus
+
+session_bus = dbus.SessionBus()
+session_bus.call_blocking(
+ "org.gnome.Shell.Screencast",
+ "/org/gnome/Shell/Screencast",
+ "org.gnome.Shell.Screencast",
+ "Screencast",
+ signature="sa{sv}",
+ args=[
+ os.path.join(os.environ.get("ARTIFACT_DIR", ""), "video_%d_%t.webm"),
+ {"draw-cursor": True, "framerate": 35},
+ ],
+)
+
+while True:
+ time.sleep(30)
diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/tests.sh b/taskcluster/docker/snap-coreXX-build/snap-tests/tests.sh
index 854ef2c9af..3412d04727 100755
--- a/taskcluster/docker/snap-coreXX-build/snap-tests/tests.sh
+++ b/taskcluster/docker/snap-coreXX-build/snap-tests/tests.sh
@@ -60,6 +60,16 @@ if [ -n "${MOZ_LOG}" ]; then
export MOZ_LOG_FILE="${ARTIFACT_DIR}/gecko-log"
fi
+RECORD_SCREEN_PID=0
+if [ "${TEST_RECORD_SCREEN}" = "true" ]; then
+ python3 record.py &
+ RECORD_SCREEN_PID=$!
+ echo "Recording with PID ${RECORD_SCREEN_PID}"
+
+ trap 'echo [EXIT] Stopping screen recording from PID ${RECORD_SCREEN_PID} && kill ${RECORD_SCREEN_PID}' EXIT
+ trap 'echo [ERR] Stopping screen recording from PID ${RECORD_SCREEN_PID} && kill ${RECORD_SCREEN_PID}' ERR
+fi;
+
if [ "${SUITE}" = "basic" ]; then
sed -e "s/#RUNTIME_VERSION#/${RUNTIME_VERSION}/#" < basic_tests/expectations.json.in > basic_tests/expectations.json
python3 basic_tests.py basic_tests/expectations.json
diff --git a/taskcluster/docker/system-symbols-linux-scraper/Dockerfile b/taskcluster/docker/system-symbols-linux-scraper/Dockerfile
index edafc97c83..e9785c93c1 100644
--- a/taskcluster/docker/system-symbols-linux-scraper/Dockerfile
+++ b/taskcluster/docker/system-symbols-linux-scraper/Dockerfile
@@ -12,7 +12,7 @@ VOLUME /builds/worker/checkouts
RUN apt-get update && \
apt-get install --no-install-recommends -y \
- 7zip binutils build-essential cpio curl debuginfod elfutils flatpak jq \
+ file 7zip binutils build-essential cpio curl debuginfod elfutils flatpak jq \
libxml2-utils python3-pip rpm2cpio squashfs-tools unzip wget zip && \
apt-get autoremove -y && rm -rf /var/lib/apt/lists/*
diff --git a/taskcluster/docker/system-symbols-linux-scraper/README b/taskcluster/docker/system-symbols-linux-scraper/README
new file mode 100644
index 0000000000..ac37a2d3f3
--- /dev/null
+++ b/taskcluster/docker/system-symbols-linux-scraper/README
@@ -0,0 +1,64 @@
+Performing a (re)bootstrapping of symbols scraping process
+==========================================================
+
+Whenever for any reason the symbol scraping process might have been faulty long
+enough, we can end up (currently) in a situation where the recorded status of
+`SHA256SUMS.zip` on the TaskCluster index is inconsistent with what we really
+processed.
+
+This document aims at explaining what needs to be done and where to recover
+from that state (this is based on the experience from bug 1893156).
+
+First, you need to identify since how long the problem has been present. As of
+now there is no really better tooling than processing manually the cron tasks
+logs and see when it started to fail.
+
+Once you have identified a date, the next step is to work on the bootstrapping
+content. As visible in
+https://searchfox.org/mozilla-central/rev/f6e3b81aac49e602f06c204f9278da30993cdc8a/taskcluster/docker/system-symbols-linux-scraper/run.sh#62,
+the first source of truth is the gh-pages branch of the symbol-scrapers github
+repository: https://github.com/mozilla/symbol-scrapers/tree/gh-pages. This
+source of truth is evaluated ONLY if the TaskCluster index is NOT present. The
+route is being computed from the running task's definition:
+https://searchfox.org/mozilla-central/rev/f6e3b81aac49e602f06c204f9278da30993cdc8a/taskcluster/docker/system-symbols-linux-scraper/run.sh#14
+from which we ONLY consider the `latest` alias.
+
+As of today the index is for example for debian:
+index.gecko.v2.mozilla-central.latest.system-symbols.debian and thus one can
+explore the content at
+https://firefox-ci-tc.services.mozilla.com/tasks/index/gecko.v2.mozilla-central.latest.system-symbols.debian,
+other means of browsing including pushdate allows to find e.g.,
+https://firefox-ci-tc.services.mozilla.com/tasks/index/gecko.v2.mozilla-central.pushdate.2024.04.20.20240420094034.system-symbols/debian
+from which we can get a link to the sums file:
+https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2024.04.20.20240420094034.system-symbols.debian/artifacts/public%2Fbuild%2FSHA256SUMS.zip
+
+Once you have identified WHEN the problem arose, you can take the above URL
+(adapting with the correct date) and adapting to the various distributions.
+
+Make sure you have an uptodate git clone of the mozilla/symbol-scrapers
+repository, checkout a new branch out of the gh-pages tree, and you can proceed
+to the data extraction following (example with a different date):
+ for distro in alpine archlinux debian fedora firefox-flatpak firefox-snap gnome-sdk-snap mint opensuse ubuntu; do
+ wget https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2024.02.07.latest.system-symbols.$distro/artifacts/public%2Fbuild%2FSHA256SUMS.zip -O $distro/SHA256SUMS.zip;
+ done;
+ mv archlinux/SHA256SUMS.zip arch/
+
+Please note that there's a slight difference in naming, archlinux vs arch.
+Please note other distros might have been added since so you need to adapt.
+
+Send a pull request once this is OK, make it reviewed or merge it.
+
+As of now, the content of the boostrapping process is in the state we want, but
+if you run a symbols scraping task, it will still pull data from the
+TaskCluster index. This time, you need to use the index that refers to latest
+and NOT the pushdate or another one, so the index used in the example SHOULD be
+good. You just have to run deleteTask (with the appropriate credentials if you
+have them, or ask releng for help in #firefox-ci):
+ for distro in alpine archlinux debian fedora firefox-flatpak firefox-snap gnome-sdk-snap mint opensuse ubuntu; do
+ taskcluster api index deleteTask gecko.v2.mozilla-central.latest.system-symbols.$distro
+ done;
+
+From there, HTTP queries to (for the debian example)
+https://firefox-ci-tc.services.mozilla.com/tasks/index/gecko.v2.mozilla-central.latest.system-symbols.debian
+would return 404, which will make the symbol scraping tasks search its data on
+GitHub.
diff --git a/taskcluster/docker/updatebot/updatebot-version.sh b/taskcluster/docker/updatebot/updatebot-version.sh
index ea788dd130..0491fdf447 100644
--- a/taskcluster/docker/updatebot/updatebot-version.sh
+++ b/taskcluster/docker/updatebot/updatebot-version.sh
@@ -1,2 +1,2 @@
-export UPDATEBOT_REVISION=5cf21801808c95a3948916fd03f1001d964d796a
+export UPDATEBOT_REVISION=d7689dd2e47c6df15f5fece49a85a5297d4c4ff3
diff --git a/taskcluster/docker/updatebot/windows-setup.sh b/taskcluster/docker/updatebot/windows-setup.sh
index c04d0a6fbf..975aaea249 100644
--- a/taskcluster/docker/updatebot/windows-setup.sh
+++ b/taskcluster/docker/updatebot/windows-setup.sh
@@ -27,7 +27,7 @@ mv depot_tools.git depot_tools
# Generating a new version of the preloaded depot_tools download can be done by:
# 1) Running the task, uncommenting the variable assignment below, uncommenting the
-# _GENERATE_DEPOT_TOOLS_BINARIES_ section in taskcluster/ci/updatebot/kind.yml,
+# _GENERATE_DEPOT_TOOLS_BINARIES_ section in taskcluster/kinds/updatebot/kind.yml,
# and ensuring that an angle update will actually take place (so it downloads the depot_tools)
# 2) Downloading and sanity-checking the depot_tools-preloaded-binaries-GIT_HASH-DATE.zip artifact
# 3) Adding it to tooltool
@@ -121,7 +121,7 @@ if test -n "$GENERATE_DEPOT_TOOLS_BINARIES"; then
# Convoluted way to get the git hash, because we don't have a .git directory
# Adding extra print statements just in case we need to debug it
- GIT_HASH=$(grep depot_tools -A 1 "$GECKO_PATH/taskcluster/ci/fetch/updatebot.yml" | tee /dev/tty | grep revision | tee /dev/tty | awk -F': *' '{print $2}' | tee /dev/tty)
+ GIT_HASH=$(grep depot_tools -A 1 "$GECKO_PATH/taskcluster/kinds/fetch/updatebot.yml" | tee /dev/tty | grep revision | tee /dev/tty | awk -F': *' '{print $2}' | tee /dev/tty)
DATE=$(date -I)
mv depot_tools-preloaded-binaries.zip "depot_tools-preloaded-binaries-$GIT_HASH-$DATE.zip"