From 8dd16259287f58f9273002717ec4d27e97127719 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 12 Jun 2024 07:43:14 +0200 Subject: Merging upstream version 127.0. Signed-off-by: Daniel Baumann --- taskcluster/docker/debian-repackage/Dockerfile | 36 +++++++----- taskcluster/docker/github-sync/Dockerfile | 14 ----- taskcluster/docker/github-sync/prepare.sh | 31 ----------- taskcluster/docker/github-sync/requirements.txt | 3 - taskcluster/docker/sentry/Dockerfile | 2 +- .../docker/snap-coreXX-build/snap-tests/README.md | 4 ++ .../docker/snap-coreXX-build/snap-tests/record.py | 24 ++++++++ .../docker/snap-coreXX-build/snap-tests/tests.sh | 10 ++++ .../docker/system-symbols-linux-scraper/Dockerfile | 2 +- .../docker/system-symbols-linux-scraper/README | 64 ++++++++++++++++++++++ taskcluster/docker/updatebot/updatebot-version.sh | 2 +- taskcluster/docker/updatebot/windows-setup.sh | 4 +- 12 files changed, 129 insertions(+), 67 deletions(-) delete mode 100644 taskcluster/docker/github-sync/Dockerfile delete mode 100755 taskcluster/docker/github-sync/prepare.sh delete mode 100644 taskcluster/docker/github-sync/requirements.txt create mode 100644 taskcluster/docker/snap-coreXX-build/snap-tests/record.py create mode 100644 taskcluster/docker/system-symbols-linux-scraper/README (limited to 'taskcluster/docker') diff --git a/taskcluster/docker/debian-repackage/Dockerfile b/taskcluster/docker/debian-repackage/Dockerfile index ae257931d4..f068731b58 100644 --- a/taskcluster/docker/debian-repackage/Dockerfile +++ b/taskcluster/docker/debian-repackage/Dockerfile @@ -3,22 +3,16 @@ MAINTAINER Mozilla Releng VOLUME /builds/worker/workspace -# At the moment the Firefox build baseline is jessie. -# Things in the build/CI system that run in the task's container -# (mach, run-task, etc.) do not support jessie. -# There's also no jessie `base` images or packages generated in the CI. -# To generate Firefox Debian packages compatible with the baseline -# we bootstrap basic i386 and amd64 jessie systems on bullseye. -# We use these to generate shared library dependencies in jessie using chroot. -# python/mozbuild/mozbuild/repackaging/deb.py checks for jessie systems -# bootstrapped under /srv/jessie-i386 and /srv/jessie-amd64 -# If they aren't there, deb.py runs the repackage in the working directory without using chroot. -# To keep the build and repackage enviroments consistent the Debian baseline used -# here (jessie) should be synchronized with the baseline used in -# taskcluster/scripts/misc/build-sysroot.sh +# At the moment the Firefox build sysroot for i386 and amd64 use jessie and arm64 uses buster. +# To generate Firefox Debian packages we bootstrap jessie and buster sysroots. +# We use these sysroots to generate shared library dependencies for +# i386 (jessie), amd64 (jessie), and arm64 (buster) packages. +# To keep the build and repackage enviroments consistent the Debian distributions used by the sysroots +# created here should match the distributions used by the sysroots created in taskcluster/scripts/misc/build-sysroot.sh ARG TASKCLUSTER_ROOT_URL ARG DOCKER_IMAGE_PACKAGES RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES && \ + dpkg --add-architecture arm64 && \ apt-get update && \ apt-get install debootstrap && set -xe && \ for arch in i386 amd64; do \ @@ -30,6 +24,20 @@ RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAG --verbose \ jessie \ /srv/jessie-$arch \ - https://archive.debian.org/debian; \ + https://archive.debian.org/debian || (echo "Debootstrap failed, outputting log:" && cat /srv/jessie-$arch/debootstrap/debootstrap.log && exit 1); \ done && \ + # bootstrap a buster/amd64 root to generate an arm64 .deb package. + debootstrap \ + --arch=amd64 \ + --variant=buildd \ + --include=debhelper,binutils-multiarch,binutils-aarch64-linux-gnu \ + --keyring=/usr/share/keyrings/debian-archive-buster-stable.gpg \ + --verbose \ + buster \ + /srv/buster-amd64 \ + http://deb.debian.org/debian || (echo "Debootstrap failed, outputting log:" && cat /srv/buster-amd64/debootstrap/debootstrap.log && exit 1) && \ + chroot /srv/buster-amd64 /bin/bash -c "\ + dpkg --add-architecture arm64 && \ + apt-get update && \ + apt-get install -y libasound2:arm64 libgtk-3-0:arm64 libx11-xcb1:arm64 libxtst6:arm64" && \ /usr/local/sbin/clean_packages.sh $DOCKER_IMAGE_PACKAGES diff --git a/taskcluster/docker/github-sync/Dockerfile b/taskcluster/docker/github-sync/Dockerfile deleted file mode 100644 index bff131ad7b..0000000000 --- a/taskcluster/docker/github-sync/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM $DOCKER_IMAGE_PARENT -MAINTAINER Kartikaya Gupta - -VOLUME /builds/worker/checkouts -VOLUME /builds/worker/workspace -VOLUME /builds/worker/tooltool-cache -VOLUME /builds/worker/.ghsync - -ADD prepare.sh /setup/prepare-docker.sh -ADD requirements.txt /setup/requirements.txt -RUN /bin/bash /setup/prepare-docker.sh /setup/requirements.txt && rm -R /setup - -# Set a default command useful for debugging -CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/github-sync/prepare.sh b/taskcluster/docker/github-sync/prepare.sh deleted file mode 100755 index 6c0be27168..0000000000 --- a/taskcluster/docker/github-sync/prepare.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -set -o errexit -set -o nounset -set -o pipefail -set -o xtrace - -test "$(whoami)" == 'root' - -# Install stuff we need -apt-get -y update -apt-get install -y \ - cmake \ - curl \ - gcc \ - git \ - g++ \ - libffi-dev \ - libgit2-dev \ - libssl-dev \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools - -# Python packages -pip3 install --break-system-packages -r "$1" diff --git a/taskcluster/docker/github-sync/requirements.txt b/taskcluster/docker/github-sync/requirements.txt deleted file mode 100644 index 13fd42681d..0000000000 --- a/taskcluster/docker/github-sync/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -requests == 2.21.0 -pygit2 == 1.11.1 -python-hglib == 2.6.1 diff --git a/taskcluster/docker/sentry/Dockerfile b/taskcluster/docker/sentry/Dockerfile index 2f606c5529..c07fdd3961 100644 --- a/taskcluster/docker/sentry/Dockerfile +++ b/taskcluster/docker/sentry/Dockerfile @@ -1,5 +1,5 @@ FROM $DOCKER_IMAGE_PARENT -LABEL maintainer="Mitchell Hentges mhentges@mozilla.com" +LABEL maintainer="Alex Hochheiden ahochheiden@mozilla.com" VOLUME /builds/worker/checkouts diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/README.md b/taskcluster/docker/snap-coreXX-build/snap-tests/README.md index 73f78c72bc..bdc79dbe4b 100644 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/README.md +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/README.md @@ -16,6 +16,10 @@ inspecting the browser at the end of a test. Data URL containing the diff screenshot will be dumped to stdout/stderr when `TEST_DUMP_DIFF` is set in the environment. +A recording of the desktop can be captured via GNOME Screencast feature by +passing `TEST_RECORD_SCREEN=true` in the enviromnent. Upon completion of the +task it should upload a WebM file. + Updating reference screenshots ============================== - `./mach try fuzzy --push-to-lando --full --env TEST_COLLECT_REFERENCE=1 -q "'snap-upstream-test"` diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/record.py b/taskcluster/docker/snap-coreXX-build/snap-tests/record.py new file mode 100644 index 0000000000..51fe47f6d8 --- /dev/null +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/record.py @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import os +import time + +import dbus + +session_bus = dbus.SessionBus() +session_bus.call_blocking( + "org.gnome.Shell.Screencast", + "/org/gnome/Shell/Screencast", + "org.gnome.Shell.Screencast", + "Screencast", + signature="sa{sv}", + args=[ + os.path.join(os.environ.get("ARTIFACT_DIR", ""), "video_%d_%t.webm"), + {"draw-cursor": True, "framerate": 35}, + ], +) + +while True: + time.sleep(30) diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/tests.sh b/taskcluster/docker/snap-coreXX-build/snap-tests/tests.sh index 854ef2c9af..3412d04727 100755 --- a/taskcluster/docker/snap-coreXX-build/snap-tests/tests.sh +++ b/taskcluster/docker/snap-coreXX-build/snap-tests/tests.sh @@ -60,6 +60,16 @@ if [ -n "${MOZ_LOG}" ]; then export MOZ_LOG_FILE="${ARTIFACT_DIR}/gecko-log" fi +RECORD_SCREEN_PID=0 +if [ "${TEST_RECORD_SCREEN}" = "true" ]; then + python3 record.py & + RECORD_SCREEN_PID=$! + echo "Recording with PID ${RECORD_SCREEN_PID}" + + trap 'echo [EXIT] Stopping screen recording from PID ${RECORD_SCREEN_PID} && kill ${RECORD_SCREEN_PID}' EXIT + trap 'echo [ERR] Stopping screen recording from PID ${RECORD_SCREEN_PID} && kill ${RECORD_SCREEN_PID}' ERR +fi; + if [ "${SUITE}" = "basic" ]; then sed -e "s/#RUNTIME_VERSION#/${RUNTIME_VERSION}/#" < basic_tests/expectations.json.in > basic_tests/expectations.json python3 basic_tests.py basic_tests/expectations.json diff --git a/taskcluster/docker/system-symbols-linux-scraper/Dockerfile b/taskcluster/docker/system-symbols-linux-scraper/Dockerfile index edafc97c83..e9785c93c1 100644 --- a/taskcluster/docker/system-symbols-linux-scraper/Dockerfile +++ b/taskcluster/docker/system-symbols-linux-scraper/Dockerfile @@ -12,7 +12,7 @@ VOLUME /builds/worker/checkouts RUN apt-get update && \ apt-get install --no-install-recommends -y \ - 7zip binutils build-essential cpio curl debuginfod elfutils flatpak jq \ + file 7zip binutils build-essential cpio curl debuginfod elfutils flatpak jq \ libxml2-utils python3-pip rpm2cpio squashfs-tools unzip wget zip && \ apt-get autoremove -y && rm -rf /var/lib/apt/lists/* diff --git a/taskcluster/docker/system-symbols-linux-scraper/README b/taskcluster/docker/system-symbols-linux-scraper/README new file mode 100644 index 0000000000..ac37a2d3f3 --- /dev/null +++ b/taskcluster/docker/system-symbols-linux-scraper/README @@ -0,0 +1,64 @@ +Performing a (re)bootstrapping of symbols scraping process +========================================================== + +Whenever for any reason the symbol scraping process might have been faulty long +enough, we can end up (currently) in a situation where the recorded status of +`SHA256SUMS.zip` on the TaskCluster index is inconsistent with what we really +processed. + +This document aims at explaining what needs to be done and where to recover +from that state (this is based on the experience from bug 1893156). + +First, you need to identify since how long the problem has been present. As of +now there is no really better tooling than processing manually the cron tasks +logs and see when it started to fail. + +Once you have identified a date, the next step is to work on the bootstrapping +content. As visible in +https://searchfox.org/mozilla-central/rev/f6e3b81aac49e602f06c204f9278da30993cdc8a/taskcluster/docker/system-symbols-linux-scraper/run.sh#62, +the first source of truth is the gh-pages branch of the symbol-scrapers github +repository: https://github.com/mozilla/symbol-scrapers/tree/gh-pages. This +source of truth is evaluated ONLY if the TaskCluster index is NOT present. The +route is being computed from the running task's definition: +https://searchfox.org/mozilla-central/rev/f6e3b81aac49e602f06c204f9278da30993cdc8a/taskcluster/docker/system-symbols-linux-scraper/run.sh#14 +from which we ONLY consider the `latest` alias. + +As of today the index is for example for debian: +index.gecko.v2.mozilla-central.latest.system-symbols.debian and thus one can +explore the content at +https://firefox-ci-tc.services.mozilla.com/tasks/index/gecko.v2.mozilla-central.latest.system-symbols.debian, +other means of browsing including pushdate allows to find e.g., +https://firefox-ci-tc.services.mozilla.com/tasks/index/gecko.v2.mozilla-central.pushdate.2024.04.20.20240420094034.system-symbols/debian +from which we can get a link to the sums file: +https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2024.04.20.20240420094034.system-symbols.debian/artifacts/public%2Fbuild%2FSHA256SUMS.zip + +Once you have identified WHEN the problem arose, you can take the above URL +(adapting with the correct date) and adapting to the various distributions. + +Make sure you have an uptodate git clone of the mozilla/symbol-scrapers +repository, checkout a new branch out of the gh-pages tree, and you can proceed +to the data extraction following (example with a different date): + for distro in alpine archlinux debian fedora firefox-flatpak firefox-snap gnome-sdk-snap mint opensuse ubuntu; do + wget https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2024.02.07.latest.system-symbols.$distro/artifacts/public%2Fbuild%2FSHA256SUMS.zip -O $distro/SHA256SUMS.zip; + done; + mv archlinux/SHA256SUMS.zip arch/ + +Please note that there's a slight difference in naming, archlinux vs arch. +Please note other distros might have been added since so you need to adapt. + +Send a pull request once this is OK, make it reviewed or merge it. + +As of now, the content of the boostrapping process is in the state we want, but +if you run a symbols scraping task, it will still pull data from the +TaskCluster index. This time, you need to use the index that refers to latest +and NOT the pushdate or another one, so the index used in the example SHOULD be +good. You just have to run deleteTask (with the appropriate credentials if you +have them, or ask releng for help in #firefox-ci): + for distro in alpine archlinux debian fedora firefox-flatpak firefox-snap gnome-sdk-snap mint opensuse ubuntu; do + taskcluster api index deleteTask gecko.v2.mozilla-central.latest.system-symbols.$distro + done; + +From there, HTTP queries to (for the debian example) +https://firefox-ci-tc.services.mozilla.com/tasks/index/gecko.v2.mozilla-central.latest.system-symbols.debian +would return 404, which will make the symbol scraping tasks search its data on +GitHub. diff --git a/taskcluster/docker/updatebot/updatebot-version.sh b/taskcluster/docker/updatebot/updatebot-version.sh index ea788dd130..0491fdf447 100644 --- a/taskcluster/docker/updatebot/updatebot-version.sh +++ b/taskcluster/docker/updatebot/updatebot-version.sh @@ -1,2 +1,2 @@ -export UPDATEBOT_REVISION=5cf21801808c95a3948916fd03f1001d964d796a +export UPDATEBOT_REVISION=d7689dd2e47c6df15f5fece49a85a5297d4c4ff3 diff --git a/taskcluster/docker/updatebot/windows-setup.sh b/taskcluster/docker/updatebot/windows-setup.sh index c04d0a6fbf..975aaea249 100644 --- a/taskcluster/docker/updatebot/windows-setup.sh +++ b/taskcluster/docker/updatebot/windows-setup.sh @@ -27,7 +27,7 @@ mv depot_tools.git depot_tools # Generating a new version of the preloaded depot_tools download can be done by: # 1) Running the task, uncommenting the variable assignment below, uncommenting the -# _GENERATE_DEPOT_TOOLS_BINARIES_ section in taskcluster/ci/updatebot/kind.yml, +# _GENERATE_DEPOT_TOOLS_BINARIES_ section in taskcluster/kinds/updatebot/kind.yml, # and ensuring that an angle update will actually take place (so it downloads the depot_tools) # 2) Downloading and sanity-checking the depot_tools-preloaded-binaries-GIT_HASH-DATE.zip artifact # 3) Adding it to tooltool @@ -121,7 +121,7 @@ if test -n "$GENERATE_DEPOT_TOOLS_BINARIES"; then # Convoluted way to get the git hash, because we don't have a .git directory # Adding extra print statements just in case we need to debug it - GIT_HASH=$(grep depot_tools -A 1 "$GECKO_PATH/taskcluster/ci/fetch/updatebot.yml" | tee /dev/tty | grep revision | tee /dev/tty | awk -F': *' '{print $2}' | tee /dev/tty) + GIT_HASH=$(grep depot_tools -A 1 "$GECKO_PATH/taskcluster/kinds/fetch/updatebot.yml" | tee /dev/tty | grep revision | tee /dev/tty | awk -F': *' '{print $2}' | tee /dev/tty) DATE=$(date -I) mv depot_tools-preloaded-binaries.zip "depot_tools-preloaded-binaries-$GIT_HASH-$DATE.zip" -- cgit v1.2.3