diff options
Diffstat (limited to 'taskcluster/docker')
138 files changed, 9527 insertions, 0 deletions
diff --git a/taskcluster/docker/REGISTRY b/taskcluster/docker/REGISTRY new file mode 100644 index 0000000000..cb1e1bb482 --- /dev/null +++ b/taskcluster/docker/REGISTRY @@ -0,0 +1 @@ +taskcluster diff --git a/taskcluster/docker/android-build/Dockerfile b/taskcluster/docker/android-build/Dockerfile new file mode 100644 index 0000000000..22eb3c51c8 --- /dev/null +++ b/taskcluster/docker/android-build/Dockerfile @@ -0,0 +1,45 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Nick Alexander <nalexander@mozilla.com> + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +# rsync is required for l10n single locale repacks. less, screen, and +# vim, help debugging interactive tasks in Task Cluster. +# git and openssh-client are used to upload GeckoView javadoc to Github. +RUN apt-get update && \ + apt-get install \ + autoconf2.13 \ + build-essential \ + base-files \ + ccache \ + cmake \ + curl \ + file \ + gnupg \ + jq \ + less \ + openssh-client \ + procps \ + python3-dev \ + rsync \ + screen \ + sudo \ + tar \ + unzip \ + uuid \ + valgrind \ + vim \ + x11-utils \ + xvfb \ + wget \ + zip \ + zstd + +# Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb. +# %include taskcluster/docker/recipes/xvfb.sh +COPY topsrcdir/taskcluster/docker/recipes/xvfb.sh /builds/worker/scripts/xvfb.sh + +# Back to the workdir, matching desktop-build. +WORKDIR /builds/worker diff --git a/taskcluster/docker/android-build/README.md b/taskcluster/docker/android-build/README.md new file mode 100644 index 0000000000..6096b08368 --- /dev/null +++ b/taskcluster/docker/android-build/README.md @@ -0,0 +1,2 @@ +This is a docker script for fetching Android Gradle dependenices for +use in Mozilla's build clusters. diff --git a/taskcluster/docker/android-build/VERSION b/taskcluster/docker/android-build/VERSION new file mode 100644 index 0000000000..4e379d2bfe --- /dev/null +++ b/taskcluster/docker/android-build/VERSION @@ -0,0 +1 @@ +0.0.2 diff --git a/taskcluster/docker/condprof/Dockerfile b/taskcluster/docker/condprof/Dockerfile new file mode 100644 index 0000000000..76817e7927 --- /dev/null +++ b/taskcluster/docker/condprof/Dockerfile @@ -0,0 +1,15 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Gregory Mierzwinski <gmierzwinski@mozilla.com> + +VOLUME /builds/worker/.cache +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/tooltool-cache +VOLUME /builds/worker/workspace + +RUN apt-get -y update && \ + apt-get install -y python3.7 \ + python3.7-venv \ + python3.7-dev + +CMD ["/bin/bash", "--login"] + diff --git a/taskcluster/docker/custom-v8/Dockerfile b/taskcluster/docker/custom-v8/Dockerfile new file mode 100644 index 0000000000..bb3e8340bc --- /dev/null +++ b/taskcluster/docker/custom-v8/Dockerfile @@ -0,0 +1,21 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Joel Maher <jmaher@mozilla.com> + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +ENV XZ_OPT=-T0 + +RUN apt-get update && \ + apt-get install \ + bzip2 \ + curl \ + libbz2-dev \ + libglib2.0-dev \ + libtinfo5 \ + pkg-config \ + tar \ + unzip \ + wget \ + zip diff --git a/taskcluster/docker/debian-base/Dockerfile b/taskcluster/docker/debian-base/Dockerfile new file mode 100644 index 0000000000..ad48749a38 --- /dev/null +++ b/taskcluster/docker/debian-base/Dockerfile @@ -0,0 +1,66 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Mike Hommey <mhommey@mozilla.com> + +### Add worker user and setup its workspace. +RUN mkdir /builds && \ + groupadd -g 1000 worker && \ + useradd -u 1000 -g 1000 -d /builds/worker -s /bin/bash -m worker && \ + mkdir -p /builds/worker/workspace && \ + chown -R worker:worker /builds + +# Declare default working folder +WORKDIR /builds/worker + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +# Set variable normally configured at login, by the shells parent process, these +# are taken from GNU su manual +ENV HOME=/builds/worker \ + SHELL=/bin/bash \ + USER=worker \ + LOGNAME=worker \ + HOSTNAME=taskcluster-worker + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] + +ARG TASKCLUSTER_ROOT_URL +ARG DOCKER_IMAGE_PACKAGES +RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES && \ + apt-get update && \ + apt-get dist-upgrade && \ + apt-get install \ + git \ + less \ + make \ + mercurial \ + patch \ + python3 \ + python3-distutils-extra \ + python3-minimal \ + python3-zstandard \ + python3-psutil \ + python3-venv \ + vim-tiny \ + xz-utils \ + zstd + +# %include testing/mozharness/external_tools/robustcheckout.py +COPY topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py + +# %include taskcluster/docker/recipes/hgrc +COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc + +# Add pip configuration, among other things. +# %include taskcluster/docker/recipes/dot-config +COPY topsrcdir/taskcluster/docker/recipes/dot-config /builds/worker/.config + +# %include taskcluster/scripts/run-task +COPY topsrcdir/taskcluster/scripts/run-task /builds/worker/bin/run-task + +# %include taskcluster/scripts/misc/fetch-content +ADD topsrcdir/taskcluster/scripts/misc/fetch-content /builds/worker/bin/fetch-content + +RUN chown -R worker:worker /builds/worker/bin && chmod 755 /builds/worker/bin/* diff --git a/taskcluster/docker/debian-build/Dockerfile b/taskcluster/docker/debian-build/Dockerfile new file mode 100644 index 0000000000..4e0e04c150 --- /dev/null +++ b/taskcluster/docker/debian-build/Dockerfile @@ -0,0 +1,51 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Mike Hommey <mhommey@mozilla.com> + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +ENV XZ_OPT=-T0 + +ARG TASKCLUSTER_ROOT_URL +ARG DOCKER_IMAGE_PACKAGES +RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES + +# binutils is needed for base toolchain builds. Ideally, it wouldn't be needed. +# libc6-i386 and lib32gcc1 are needed for wine. +# libdbus-glib-1-2 and libgtk-3-0 are needed to run xpcshell during the build. +# lib32atomic1, lib32stdc++6 and lib32z1 are needed to run some 32-bits +# cpio is needed by xar (mac repackaging on linux) +# spidermonkey tests. libasound2 is needed to run xpcshell after we introduced +# the dependencies on alsa via Web MIDI. +RUN apt-get update && \ + apt-get dist-upgrade && \ + apt-get install \ + binutils \ + bzip2 \ + curl \ + cpio \ + file \ + gawk \ + gnupg \ + jq \ + lib32atomic1 \ + 'lib32gcc(1|-s1)$' \ + lib32stdc++6 \ + lib32z1 \ + libasound2 \ + libc6-i386 \ + libdbus-glib-1-2 \ + libgtk-3-0 \ + libucl1 \ + p7zip-full \ + procps \ + python3-dev \ + rsync \ + screen \ + tar \ + unzip \ + uuid \ + wget \ + x11-utils \ + zip diff --git a/taskcluster/docker/debian-packages/Dockerfile b/taskcluster/docker/debian-packages/Dockerfile new file mode 100644 index 0000000000..1e651a48b7 --- /dev/null +++ b/taskcluster/docker/debian-packages/Dockerfile @@ -0,0 +1,11 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Mike Hommey <mhommey@mozilla.com> + +RUN apt-get install --install-recommends \ + apt-utils \ + aptitude \ + build-essential \ + devscripts \ + equivs \ + fakeroot \ + git diff --git a/taskcluster/docker/debian-raw/Dockerfile b/taskcluster/docker/debian-raw/Dockerfile new file mode 100644 index 0000000000..d9e30c5d8a --- /dev/null +++ b/taskcluster/docker/debian-raw/Dockerfile @@ -0,0 +1,59 @@ +ARG BASE_IMAGE +FROM $BASE_IMAGE +MAINTAINER Mike Hommey <mhommey@mozilla.com> + +ENV DEBIAN_FRONTEND=noninteractive + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] + +# %include taskcluster/docker/recipes/setup_packages.sh +COPY topsrcdir/taskcluster/docker/recipes/setup_packages.sh /usr/local/sbin/ + +COPY taskcluster-hack.sh /usr/local/sbin +COPY gpgvnoexpkeysig /usr/local/sbin + +ARG DIST +ARG SNAPSHOT +ARG TASKCLUSTER_ROOT_URL +# Set apt sources list to a snapshot. +# Note: the use of gpgvnoexpkeysig is because the Debian Jessie GPG key expired. +RUN if [ -n "$SNAPSHOT" -a -n "$DIST" ]; then for s in debian_$DIST debian_$DIST-updates debian_$DIST-backports debian-security_$DIST-security debian-debug_$DIST-debug debian-debug_$DIST-proposed-updates-debug debian-debug_$DIST-backports-debug; do \ + case "$s" in \ + debian_jessie-backports) \ + echo "deb http://snapshot.debian.org/archive/debian-archive/20190328T105444Z/debian/ ${s#*_} main"; \ + ;; \ + debian-debug_jessie*) \ + : No debian-debug archive for Jessie; \ + ;; \ + debian-security_jessie-security|debian-security_buster-security) \ + d=${s#*_}; \ + echo "deb http://snapshot.debian.org/archive/${s%_*}/$SNAPSHOT/ ${d%-security}/updates main"; \ + ;; \ + *) \ + echo "deb http://snapshot.debian.org/archive/${s%_*}/$SNAPSHOT/ ${s#*_} main"; \ + ;; \ + esac; \ + done > /etc/apt/sources.list ; fi && \ + ( echo 'quiet "true";'; \ + echo 'APT::Get::Assume-Yes "true";'; \ + echo 'APT::Install-Recommends "false";'; \ + echo 'Acquire::Check-Valid-Until "false";'; \ + echo 'Acquire::Retries "5";'; \ + if dpkg --compare-versions $(apt --version | awk '{print $2}') ge 2.1.15; then \ + echo 'dir::bin::methods::https "/usr/local/sbin/taskcluster-hack.sh";'; \ + fi; \ + if [ "$DIST" = "jessie" ]; then \ + echo 'Dir::Bin::gpg "/usr/local/sbin/gpgvnoexpkeysig";'; \ + fi; \ + ) > /etc/apt/apt.conf.d/99taskcluster && \ + ( echo 'Package: *'; \ + echo 'Pin: origin "'$TASKCLUSTER_ROOT_URL'"' | sed 's,https://,,'; \ + echo 'Pin-Priority: 1001'; \ + ) > /etc/apt/preferences.d/99taskcluster + +RUN apt-get update && \ + apt-get dist-upgrade && \ + apt-get install \ + apt-transport-https \ + ca-certificates diff --git a/taskcluster/docker/debian-raw/gpgvnoexpkeysig b/taskcluster/docker/debian-raw/gpgvnoexpkeysig new file mode 100755 index 0000000000..fbbfd7a658 --- /dev/null +++ b/taskcluster/docker/debian-raw/gpgvnoexpkeysig @@ -0,0 +1,52 @@ +#!/bin/sh +# +# Downloaded from https://gitlab.mister-muffin.de/josch/mmdebstrap/raw/branch/main/gpgvnoexpkeysig +# +# This script is in the public domain +# +# Author: Johannes Schauer Marin Rodrigues <josch@mister-muffin.de> +# +# This is a wrapper around gpgv as invoked by apt. It turns EXPKEYSIG results +# from gpgv into GOODSIG results. This is necessary for apt to access very old +# timestamps from snapshot.debian.org for which the GPG key is already expired: +# +# Get:1 http://snapshot.debian.org/archive/debian/20150106T000000Z unstable InRelease [242 kB] +# Err:1 http://snapshot.debian.org/archive/debian/20150106T000000Z unstable InRelease +# The following signatures were invalid: EXPKEYSIG 8B48AD6246925553 Debian Archive Automatic Signing Key (7.0/wheezy) <ftpmaster@debian.org> +# Reading package lists... +# W: GPG error: http://snapshot.debian.org/archive/debian/20150106T000000Z unstable InRelease: The following signatures were invalid: EXPKEYSIG 8B48AD6246925553 Debian Archive Automatic Signing Key (7.0/wheezy) <ftpmaster@debian.org> +# E: The repository 'http://snapshot.debian.org/archive/debian/20150106T000000Z unstable InRelease' is not signed. +# +# To use this script, call apt with +# +# -o Apt::Key::gpgvcommand=/usr/libexec/mmdebstrap/gpgvnoexpkeysig +# +# Scripts doing similar things can be found here: +# +# * debuerreotype as /usr/share/debuerreotype/scripts/.gpgv-ignore-expiration.sh +# * derivative census: salsa.d.o/deriv-team/census/-/blob/master/bin/fakegpgv + +set -eu + +find_gpgv_status_fd() { + while [ "$#" -gt 0 ]; do + if [ "$1" = '--status-fd' ]; then + echo "$2" + return 0 + fi + shift + done + # default fd is stdout + echo 1 +} +GPGSTATUSFD="$(find_gpgv_status_fd "$@")" + +case $GPGSTATUSFD in + ''|*[!0-9]*) + echo "invalid --status-fd argument" >&2 + exit 1 + ;; +esac + +# we need eval because we cannot redirect a variable fd +eval 'exec gpgv "$@" '"$GPGSTATUSFD"'>&1 | sed "s/^\[GNUPG:\] EXPKEYSIG /[GNUPG:] GOODSIG /" >&'"$GPGSTATUSFD" diff --git a/taskcluster/docker/debian-raw/taskcluster-hack.sh b/taskcluster/docker/debian-raw/taskcluster-hack.sh new file mode 100755 index 0000000000..eecac021ec --- /dev/null +++ b/taskcluster/docker/debian-raw/taskcluster-hack.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# APT version 2.1.15 and newer changed how they handle quoting in redirections +# in a way that breaks the setup for APT repos in taskcluster artifacts +# (unfortunately, there's also no setup on the taskcluster end that would work +# with both old and newer versions of APT, short of removing redirections +# entirely). +/usr/lib/apt/methods/https | sed -u '/^New-URI:/s/+/%2b/g' diff --git a/taskcluster/docker/debian-repackage/Dockerfile b/taskcluster/docker/debian-repackage/Dockerfile new file mode 100644 index 0000000000..494d2b437a --- /dev/null +++ b/taskcluster/docker/debian-repackage/Dockerfile @@ -0,0 +1,34 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Mozilla Releng <release@mozilla.com> + +VOLUME /builds/worker/workspace + +# At the moment the Firefox build baseline is jessie. +# Things in the build/CI system that run in the task's container +# (mach, run-task, etc.) do not support jessie. +# There's also no jessie `base` images or packages generated in the CI. +# To generate Firefox Debian packages compatible with the baseline +# we bootstrap basic i386 and amd64 jessie systems on bullseye. +# We use these to generate shared library dependencies in jessie using chroot. +# python/mozbuild/mozbuild/repackaging/deb.py checks for jessie systems +# bootstrapped under /srv/jessie-i386 and /srv/jessie-amd64 +# If they aren't there, deb.py runs the repackage in the working directory without using chroot. +# To keep the build and repackage enviroments consistent the Debian baseline used +# here (jessie) should be synchronized with the baseline used in +# taskcluster/scripts/misc/build-sysroot.sh +ARG TASKCLUSTER_ROOT_URL +ARG DOCKER_IMAGE_PACKAGES +RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES && \ + apt-get update && \ + apt-get install debootstrap python3-distutils && set -x && \ + for arch in i386 amd64; do \ + debootstrap \ + --arch=$arch \ + --variant=buildd \ + --include=debhelper,libasound2,libdbus-glib-1-2,libgtk-3-0,libx11-xcb1,libxtst6 \ + --keyring=/usr/share/keyrings/debian-archive-removed-keys.gpg \ + --verbose \ + jessie \ + /srv/jessie-$arch \ + http://snapshot.debian.org/archive/debian/20210208T213147Z; \ + done diff --git a/taskcluster/docker/decision/Dockerfile b/taskcluster/docker/decision/Dockerfile new file mode 100644 index 0000000000..f2aa83903e --- /dev/null +++ b/taskcluster/docker/decision/Dockerfile @@ -0,0 +1,15 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Andrew Halberstadt <ahal@mozilla.com> + +RUN mkdir /builds/worker/artifacts && \ + chown worker:worker /builds/worker/artifacts + +ADD system-setup.sh /tmp/system-setup.sh +RUN bash /tmp/system-setup.sh + +ADD comm-task-env /builds/worker/bin/comm-task-env + +ENV PATH=/builds/worker/bin:$PATH + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/decision/HASH b/taskcluster/docker/decision/HASH new file mode 100644 index 0000000000..b854e10d0f --- /dev/null +++ b/taskcluster/docker/decision/HASH @@ -0,0 +1 @@ +sha256:9f69fe08c28e3cb3cc296451f0a2735df6e25d0e3c877ea735ef1b7f0b345b06 diff --git a/taskcluster/docker/decision/README.md b/taskcluster/docker/decision/README.md new file mode 100644 index 0000000000..4490880be8 --- /dev/null +++ b/taskcluster/docker/decision/README.md @@ -0,0 +1,5 @@ +# Decision Tasks + +The decision image is a "boostrapping" image for the in tree logic it +deals with cloning gecko and the related utilities for providing an +environment where we can run gecko. diff --git a/taskcluster/docker/decision/VERSION b/taskcluster/docker/decision/VERSION new file mode 100644 index 0000000000..fcdb2e109f --- /dev/null +++ b/taskcluster/docker/decision/VERSION @@ -0,0 +1 @@ +4.0.0 diff --git a/taskcluster/docker/decision/comm-task-env b/taskcluster/docker/decision/comm-task-env new file mode 100755 index 0000000000..65481497ae --- /dev/null +++ b/taskcluster/docker/decision/comm-task-env @@ -0,0 +1,199 @@ +#!/usr/bin/python3 -u +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Thunderbird build environment prep for run-task, +for use with comm-central derived repositories. + +This script is meant to run prior to run-task on repositories like +comm-central that need to check out a copy of a mozilla repository +in order to build. +See bug 1491371 for background on why this is necessary. + +A project will have a file named ".gecko_rev.yml" in it's root. See the +constant "GECKO_REV_CONF" if you want to change that. To download it, the +script uses the project repository URL and the revision number. +Those are defined in the environment variables: +COMM_HEAD_REPOSITORY +COMM_HEAD_REV + +.gecko_rev.yml has a structure like (for comm-central): +``` +GECKO_BASE_REPOSITORY: https://hg.mozilla.org/mozilla-unified +GECKO_HEAD_REPOSITORY: https://hg.mozilla.org/mozilla-central +GECKO_HEAD_REF: default +``` +or for branches: +``` +GECKO_BASE_REPOSITORY: https://hg.mozilla.org/mozilla-unified +GECKO_HEAD_REPOSITORY: https://hg.mozilla.org/releases/mozilla-beta +GECKO_HEAD_REF: THUNDERBIRD_60_VERBRANCH +GECKO_HEAD_REV: 6a830d12f15493a70b1192022c9985eba2139910 + +Note about GECKO_HEAD_REV and GECKO_HEAD_REF: +GECKO_HEAD_REF is a branch name or "default". +GECKO_HEAD_REV is a revision hash. +``` +""" + +import sys + +import os +import socket +import time +from datetime import datetime +from pprint import pformat + +import urllib.error +import urllib.request + +import yaml + +if sys.version_info[0:2] < (3, 5): + print('run-task-wrapper requires Python 3.5+') + sys.exit(1) + +GECKO_REV_CONF = ".gecko_rev.yml" +DEBUG = bool(os.environ.get("RTW_DEBUG", False)) + + +def print_message(msg, prefix=__file__, level=""): + """ + Print messages. + :param object msg: message to print, usually a string, but not always + :param str prefix: message prefix + :param str level: message level (DEBUG, ERROR, INFO) + """ + if not isinstance(msg, str): + msg = pformat(msg) + now = datetime.utcnow().isoformat() + # slice microseconds to 3 decimals. + now = now[:-3] if now[-7:-6] == '.' else now + if level: + sys.stdout.write('[{prefix} {now}Z] {level}: {msg}\n'.format( + prefix=prefix, now=now, level=level, msg=msg)) + else: + sys.stdout.write('[{prefix} {now}Z] {msg}\n'.format( + prefix=prefix, now=now, msg=msg)) + sys.stdout.flush() + + +def error_exit(msg): + """Print the error message and exit with error.""" + print_message(msg, level="ERROR") + if DEBUG: + raise Exception(msg) + + sys.exit(1) + + +def print_debug(msg): + """Prints a message with DEBUG prefix if DEBUG is enabled + with the environment variable "RTW_DEBUG". + """ + if DEBUG: + print_message(msg, level="DEBUG") + + +def check_environ(): + """Check that the necessary environment variables to find the + comm- repository are defined. (Set in .taskcluster.yml) + :return: tuple(str, str) + """ + print_debug("Checking environment variables...") + project_head_repo = os.environ.get("COMM_HEAD_REPOSITORY", None) + project_head_rev = os.environ.get("COMM_HEAD_REV", None) + + if project_head_repo is None or project_head_rev is None: + error_exit("Environment NOT Ok:\n\tHead: {}\n\tRev: {}\n").format( + project_head_repo, project_head_rev) + + print_debug("Environment Ok:\n\tHead: {}\n\tRev: {}\n".format( + project_head_repo, project_head_rev)) + return project_head_repo, project_head_rev + + +def download_url(url, retry=1): + """Downloads the given URL. Naively retries (when asked) upon failure + :param url: str + :param retry: int + :return: str + """ + # Use 1-based counting for display and calculation purposes. + for i in range(1, retry+1): + try: + print_message('Fetching {}. Attempt {} of {}.'.format( + url, i, retry)) + with urllib.request.urlopen(url, timeout=10) as response: + data = response.read().decode("utf-8") + return data + except (urllib.error.URLError, socket.timeout) as exc: + print_message('Unable to retrieve {}'.format(url)) + if isinstance(exc, urllib.error.URLError): + print_message(exc.reason) + else: # socket.timeout + print_message('Connection timed out.') + + if i < retry: # No more retries + wait_time = i * 5 # fail #1: sleep 5s. #2, sleep 10s + print_message('Retrying in {} seconds.'.format(wait_time)) + time.sleep(wait_time) + + error_exit('No more retry attempts! Aborting.') + + +def fetch_gecko_conf(project_head_repo, project_revision): + """Downloads .gecko_rev.yml from the project repository + :param project_head_repo: str + :param project_revision: str + :return: dict + """ + gecko_conf_url = '/'.join( + [project_head_repo, 'raw-file', project_revision, GECKO_REV_CONF]) + + gecko_conf_yml = download_url(gecko_conf_url, retry=5) + + try: + gecko_conf = yaml.safe_load(gecko_conf_yml) + return gecko_conf + except yaml.YAMLError as exc: + err_txt = ["Error processing Gecko YAML configuration."] + if hasattr(exc, "problem_mark"): + mark = exc.problem_mark # pylint: disable=no-member + err_txt.append("Error position: line {}, column {}".format( + mark.line + 1, mark.column + 1)) + error_exit('\n'.join(err_txt)) + + +def update_environment(gecko_conf): + """Adds the new variables defined in gecko_conf to the + running environment. + :param gecko_conf: dict + """ + print_message("Updating environment with:") + print_message(gecko_conf) + os.environ.update(gecko_conf) + + print_debug("New environment:") + print_debug(os.environ) + + +def exec_run_task(args): + """Executes run-task with a modified environment.""" + print_message("Executing: {}".format(pformat(args))) + os.execv(args[0], args[0:]) + + +def main(): + """Main function.""" + args = sys.argv[1:] # Remaining args starting with run-task + + project_head_repo, project_revision = check_environ() + gecko_conf = fetch_gecko_conf(project_head_repo, project_revision) + update_environment(gecko_conf) + exec_run_task(args) + + +if __name__ == "__main__": + main() diff --git a/taskcluster/docker/decision/system-setup.sh b/taskcluster/docker/decision/system-setup.sh new file mode 100644 index 0000000000..0f9d04fa50 --- /dev/null +++ b/taskcluster/docker/decision/system-setup.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -v -e + +test "$(whoami)" == 'root' + +apt-get update +apt-get install \ + python \ + sudo \ + python3-yaml + +rm -rf /var/lib/apt/lists/ +rm "$0" diff --git a/taskcluster/docker/diffoscope/Dockerfile b/taskcluster/docker/diffoscope/Dockerfile new file mode 100644 index 0000000000..22e68fd784 --- /dev/null +++ b/taskcluster/docker/diffoscope/Dockerfile @@ -0,0 +1,33 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Mike Hommey <mhommey@mozilla.com> + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +ENV LANG=en_US.UTF-8 + +RUN apt-get install \ + binutils-multiarch \ + bzip2 \ + curl \ + enjarify \ + diffoscope \ + jsbeautifier \ + libc++abi1 \ + locales \ + default-jdk-headless \ + python3-progressbar \ + unzip \ + zip \ + && \ + sed -i '/en_US.UTF-8/s/^# *//' /etc/locale.gen && \ + locale-gen + +COPY get_and_diffoscope /builds/worker/bin/get_and_diffoscope +COPY readelf /builds/worker/bin/readelf +COPY report_error /builds/worker/bin/report_error +COPY test_diffoscope /builds/worker/bin/test_diffoscope + +RUN chown -R worker:worker /builds/worker/bin && chmod 755 /builds/worker/bin/* && \ + /builds/worker/bin/test_diffoscope diff --git a/taskcluster/docker/diffoscope/get_and_diffoscope b/taskcluster/docker/diffoscope/get_and_diffoscope new file mode 100644 index 0000000000..9c51af2508 --- /dev/null +++ b/taskcluster/docker/diffoscope/get_and_diffoscope @@ -0,0 +1,103 @@ +#!/bin/bash + +set -e +set -x + +cd /builds/worker + +mkdir a b + +# /builds/worker/bin contains wrapper binaries to divert what diffoscope +# needs to use, so it needs to appear first. +export PATH=/builds/worker/bin:$PATH + +# Until https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=879010 is +# implemented, it's better to first manually extract the data. +# Plus dmg files are not supported yet. + +case "$ORIG_URL" in +*.zip|*.apk) + curl -L "$ORIG_URL" > a.zip + curl -L "$NEW_URL" > b.zip + unzip -d a a.zip + unzip -d b b.zip + ;; +*.tar.bz2) + curl -L "$ORIG_URL" | tar -C a -jxf - + curl -L "$NEW_URL" | tar -C b -jxf - + ;; +*.tar.gz) + curl -L "$ORIG_URL" | tar -C a -zxf - + curl -L "$NEW_URL" | tar -C b -zxf - + ;; +*.dmg) + for tool in lipo otool; do + ln -s $MOZ_FETCHES_DIR/cctools/bin/x86_64-apple-darwin*-$tool bin/$tool + done + curl -L "$ORIG_URL" > a.dmg + curl -L "$NEW_URL" > b.dmg + for i in a b; do + $MOZ_FETCHES_DIR/dmg/dmg extract $i.dmg $i.hfs + $MOZ_FETCHES_DIR/dmg/hfsplus $i.hfs extractall / $i + done + ;; +*) + ARTIFACT=$(basename "${ORIG_URL}") + curl -L "$ORIG_URL" > "a/${ARTIFACT}" + curl -L "$NEW_URL" > "b/${ARTIFACT}" +esac + +case "$ORIG_URL" in +*/target.apk) + OMNIJAR=assets/omni.ja + ;; +*) + OMNIJAR=omni.ja + ;; +esac + +# Builds are 99% of the time differing in some small ways, so it's not +# really useful to report a failure (at least not until we actually +# care about the builds being 100% identical). +POST=true + +fail() { + exit 1 +} + +for option; do + case "$option" in + --unpack) + CURDIR=$PWD + for dir in a b; do + # Need to run mach python from inside the gecko source. + # See bug #1533642. + (cd $GECKO_PATH && ./mach python toolkit/mozapps/installer/unpack.py --omnijar $OMNIJAR $CURDIR/$dir) + done + ;; + --fail) + POST="fail" + ;; + *) + echo "Unsupported option: $option" >&2 + exit 1 + esac +done + +if [ -n "$PRE_DIFF" ]; then + eval $PRE_DIFF +fi + +if diffoscope \ + --html diff.html \ + --text diff.txt \ + --progress \ + $DIFFOSCOPE_ARGS \ + a b +then + # Ok + : +else + $(dirname $0)/report_error diff + $POST +fi diff --git a/taskcluster/docker/diffoscope/readelf b/taskcluster/docker/diffoscope/readelf new file mode 100644 index 0000000000..6b864171d7 --- /dev/null +++ b/taskcluster/docker/diffoscope/readelf @@ -0,0 +1,13 @@ +#!/bin/sh + +case "$1 $2" in +"--wide --symbols") + # When called with --wide --symbols, we remove the first column (which + # is essentially a line number that is not very useful), and then sort, + # which will order symbols by address, making a diff more useful. + /usr/bin/readelf "$@" | awk -F: '{print $2}' | sort + ;; +*) + exec /usr/bin/readelf "$@" + ;; +esac diff --git a/taskcluster/docker/diffoscope/report_error b/taskcluster/docker/diffoscope/report_error new file mode 100755 index 0000000000..9074e33cae --- /dev/null +++ b/taskcluster/docker/diffoscope/report_error @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +import sys +stem = sys.argv[1] + +# We "parse" the diff output, so we look at the lines that contain a "tee", like: +# ├── +++ b/firefox +# │ ├── +++ b/firefox/libxul.so +# │ │ ├── readelf --wide --notes {} +# We ignore lines like the last one, to only report file names. And we ignore +# lines for directories such as the first one. +TEE = "├── " +VERTICAL_LINE = "│" +paths = set() +with open(f"{stem}.txt") as fh: + for l in fh: + l = l.rstrip() + before, tee, after = l.partition(TEE) + if not tee: + continue + before = before.split() + assert all(x == VERTICAL_LINE for x in before) + depth = len(before) + _, plus, after = after.partition("+++ ") + if not plus: + continue + _, b, full_path = after.partition("b/") + assert b == "b/" + parent_path = "/".join(full_path.split("/")[:-1]) + if parent_path in paths: + paths.remove(parent_path) + if full_path: + paths.add(full_path) + +for p in sorted(paths): + print(f"TEST-UNEXPECTED-FAIL | {p} differs. See the {stem}.html or {stem}.txt artifact") diff --git a/taskcluster/docker/diffoscope/test_diffoscope b/taskcluster/docker/diffoscope/test_diffoscope new file mode 100644 index 0000000000..ab1c6c1eee --- /dev/null +++ b/taskcluster/docker/diffoscope/test_diffoscope @@ -0,0 +1,47 @@ +#!/bin/bash + +set -e + +WORKDIR=$(mktemp -d) +cd $WORKDIR + +mkdir -p a/foo/bar/bar +mkdir -p a/foo/bar/baz +mkdir -p b/foo/bar/bar +mkdir -p b/foo/bar/baz + +# A file that is modified +echo qux > a/foo/bar/qux +echo quz > b/foo/bar/qux + +# A binary file that is modified +cp $(which ls) a/foo/bin +cp $(which cat) b/foo/bin + +# A file that is removed +echo hoge > a/foo/bar/bar/hoge + +# A file that is created +echo fuga > b/foo/bar/baz/fuga + +# Also add a zip file with the same contents +(cd a/foo; zip -r bar.zip bar) +(cd b/foo; zip -r bar.zip bar) + +if TERM=linux diffoscope --no-progress --text diff.txt a b; then + echo "diffoscope didn't find differences?" + exit 1 +fi + +cat > expected.txt <<EOF +TEST-UNEXPECTED-FAIL | foo/bar.zip differs. See the diff.html or diff.txt artifact +TEST-UNEXPECTED-FAIL | foo/bar/bar differs. See the diff.html or diff.txt artifact +TEST-UNEXPECTED-FAIL | foo/bar/baz differs. See the diff.html or diff.txt artifact +TEST-UNEXPECTED-FAIL | foo/bar/qux differs. See the diff.html or diff.txt artifact +TEST-UNEXPECTED-FAIL | foo/bin differs. See the diff.html or diff.txt artifact +EOF + +$(dirname $0)/report_error diff | diff -u - expected.txt || exit 1 + +cd $OLDPWD +rm -rf $WORKDIR diff --git a/taskcluster/docker/fetch/Dockerfile b/taskcluster/docker/fetch/Dockerfile new file mode 100644 index 0000000000..792b3f43a6 --- /dev/null +++ b/taskcluster/docker/fetch/Dockerfile @@ -0,0 +1,36 @@ +FROM $DOCKER_IMAGE_PARENT + +### Add worker user and setup its workspace. +RUN mkdir /builds && \ + groupadd -g 1000 worker && \ + useradd -u 1000 -g 1000 -d /builds/worker -s /bin/bash -m worker && \ + mkdir -p /builds/worker/workspace && \ + chown -R worker:worker /builds + +# Declare default working folder +WORKDIR /builds/worker + +ARG TASKCLUSTER_ROOT_URL +ARG DOCKER_IMAGE_PACKAGES +RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES && \ + apt-get update && \ + apt-get install \ + gnupg \ + bzip2 \ + git \ + openssh-client \ + python3-pip \ + python3-requests \ + python3-zstandard \ + unzip + +# %include taskcluster/scripts/run-task +ADD topsrcdir/taskcluster/scripts/run-task /builds/worker/bin/run-task + +# %include taskcluster/scripts/misc/fetch-content +ADD topsrcdir/taskcluster/scripts/misc/fetch-content /builds/worker/bin/fetch-content + +# %include taskcluster/scripts/misc/fetch-chromium.py +ADD topsrcdir/taskcluster/scripts/misc/fetch-chromium.py /builds/worker/bin/fetch-chromium.py + +RUN pip3 install redo==2.0.4 diff --git a/taskcluster/docker/firefox-flatpak/Dockerfile b/taskcluster/docker/firefox-flatpak/Dockerfile new file mode 100644 index 0000000000..b59c697f13 --- /dev/null +++ b/taskcluster/docker/firefox-flatpak/Dockerfile @@ -0,0 +1,15 @@ +FROM freedesktopsdk/flatpak:21.08-x86_64 +MAINTAINER release@mozilla.com + +RUN mkdir /scripts/ +WORKDIR /scripts/ +# Copy everything in the docker/firefox-flatpak folder but the Dockerfile +# +# XXX The following pattern is neither a regex nor a glob one. It's +# documented at https://golang.org/pkg/path/filepath/#Match. There's no +# way of explicitly filtering out "Dockerfile". If one day, someone needs +# to add a file starting with "D", then we must revisit the pattern below. +COPY [^D]* /scripts/ + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/firefox-flatpak/default-preferences.js b/taskcluster/docker/firefox-flatpak/default-preferences.js new file mode 100644 index 0000000000..95663d03db --- /dev/null +++ b/taskcluster/docker/firefox-flatpak/default-preferences.js @@ -0,0 +1,11 @@ +/*global pref*/ +/*eslint no-undef: "error"*/ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +pref("intl.locale.requested", ""); +pref("app.update.auto", false); +pref("app.update.enabled", false); +pref("app.update.autoInstallEnabled", false); +pref("browser.shell.checkDefaultBrowser", false); +pref("spellchecker.dictionary_path", "/usr/share/hunspell"); diff --git a/taskcluster/docker/firefox-flatpak/extract_locales_from_l10n_json.py b/taskcluster/docker/firefox-flatpak/extract_locales_from_l10n_json.py new file mode 100644 index 0000000000..b1eb745d7f --- /dev/null +++ b/taskcluster/docker/firefox-flatpak/extract_locales_from_l10n_json.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +""" +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +""" + + +import json +import sys + +l10n_changesets_json_path = sys.argv[1] +with open(l10n_changesets_json_path) as f: + locales = json.load(f).keys() +linux_locales = [l for l in locales if l != "ja-JP-mac"] + +print("\n".join(sorted(linux_locales))) diff --git a/taskcluster/docker/firefox-flatpak/launch-script.sh b/taskcluster/docker/firefox-flatpak/launch-script.sh new file mode 100644 index 0000000000..98279e71eb --- /dev/null +++ b/taskcluster/docker/firefox-flatpak/launch-script.sh @@ -0,0 +1,3 @@ +#!/bin/bash +export TMPDIR=$XDG_CACHE_HOME/tmp +exec /app/lib/firefox/firefox "$@" diff --git a/taskcluster/docker/firefox-flatpak/org.mozilla.firefox.appdata.xml.in b/taskcluster/docker/firefox-flatpak/org.mozilla.firefox.appdata.xml.in new file mode 100644 index 0000000000..543b2015b0 --- /dev/null +++ b/taskcluster/docker/firefox-flatpak/org.mozilla.firefox.appdata.xml.in @@ -0,0 +1,33 @@ +<?xml version="1.0" encoding="UTF-8"?> +<application> + <id type="desktop">org.mozilla.firefox</id> + <launchable type="desktop-id">org.mozilla.firefox.desktop</launchable> + <name>Firefox</name> + <developer_name>Mozilla</developer_name> + <summary>Fast, Private & Safe Web Browser</summary> + <metadata_license>CC0-1.0</metadata_license> + <project_license>MPL-2.0</project_license> + <description> + <p>When it comes to your life online, you have a choice: accept the factory settings or put your privacy first. When you choose Firefox as your default browser, you’re choosing to protect your data while supporting an independent tech company. Firefox is also the only major browser backed by a non-profit fighting to give you more openness, transparency and control of your life online. Join hundreds of millions of people who choose to protect what's important by choosing Firefox - a web browser designed to be fast, easy to use, customizable and private.</p> + </description> + <releases> + <release version="$VERSION" date="$DATE"/> + </releases> + <keywords> + <keyword>mozilla</keyword> + <keyword>internet</keyword> + <keyword>web</keyword> + </keywords> + <content_rating type="oars-1.1" /> + <url type="homepage">https://www.mozilla.org/firefox/</url> + <url type="donation">https://donate.mozilla.org/</url> + <url type="bugtracker">https://bugzilla.mozilla.org/</url> + <url type="help">https://support.mozilla.org/</url> + <url type="translate">https://wiki.mozilla.org/L10n:Starting_a_localization</url> + + <screenshots> + <screenshot type="default">https://raw.githubusercontent.com/mozilla-releng/scriptworker-scripts/master/pushflatpakscript/media/screenshots/image1.png</screenshot> + <screenshot>https://raw.githubusercontent.com/mozilla-releng/scriptworker-scripts/master/pushflatpakscript/media/screenshots/image2.png</screenshot> + <screenshot>https://raw.githubusercontent.com/mozilla-releng/scriptworker-scripts/master/pushflatpakscript/media/screenshots/image3.png</screenshot> + </screenshots> +</application> diff --git a/taskcluster/docker/firefox-flatpak/org.mozilla.firefox.desktop b/taskcluster/docker/firefox-flatpak/org.mozilla.firefox.desktop new file mode 100644 index 0000000000..0c8b68a938 --- /dev/null +++ b/taskcluster/docker/firefox-flatpak/org.mozilla.firefox.desktop @@ -0,0 +1,386 @@ +[Desktop Entry] +Version=1.0 +Name=Firefox Web Browser +Name[ar]=متصفح الويب فَيَرفُكْس +Name[ast]=Restolador web Firefox +Name[bn]=ফায়ারফক্স ওয়েব ব্রাউজার +Name[ca]=Navegador web Firefox +Name[cs]=Firefox Webový prohlížeč +Name[da]=Firefox - internetbrowser +Name[el]=Περιηγητής Firefox +Name[es]=Navegador web Firefox +Name[et]=Firefoxi veebibrauser +Name[fa]=مرورگر اینترنتی Firefox +Name[fi]=Firefox-selain +Name[fr]=Navigateur Web Firefox +Name[gl]=Navegador web Firefox +Name[he]=דפדפן האינטרנט Firefox +Name[hr]=Firefox web preglednik +Name[hu]=Firefox webböngésző +Name[it]=Firefox Browser Web +Name[ja]=Firefox ウェブ・ブラウザ +Name[ko]=Firefox 웹 브라우저 +Name[ku]=Geroka torê Firefox +Name[lt]=Firefox interneto naršyklė +Name[nb]=Firefox Nettleser +Name[nl]=Firefox webbrowser +Name[nn]=Firefox Nettlesar +Name[no]=Firefox Nettleser +Name[pl]=Przeglądarka WWW Firefox +Name[pt]=Firefox Navegador Web +Name[pt_BR]=Navegador Web Firefox +Name[ro]=Firefox – Navigator Internet +Name[ru]=Веб-браузер Firefox +Name[sk]=Firefox - internetový prehliadač +Name[sl]=Firefox spletni brskalnik +Name[sv]=Firefox webbläsare +Name[tr]=Firefox Web Tarayıcısı +Name[ug]=Firefox توركۆرگۈ +Name[uk]=Веб-браузер Firefox +Name[vi]=Trình duyệt web Firefox +Name[zh_CN]=Firefox 网络浏览器 +Name[zh_TW]=Firefox 網路瀏覽器 +Comment=Browse the World Wide Web +Comment[ar]=تصفح الشبكة العنكبوتية العالمية +Comment[ast]=Restola pela Rede +Comment[bn]=ইন্টারনেট ব্রাউজ করুন +Comment[ca]=Navegueu per la web +Comment[cs]=Prohlížení stránek World Wide Webu +Comment[da]=Surf på internettet +Comment[de]=Im Internet surfen +Comment[el]=Μπορείτε να περιηγηθείτε στο διαδίκτυο (Web) +Comment[es]=Navegue por la web +Comment[et]=Lehitse veebi +Comment[fa]=صفحات شبکه جهانی اینترنت را مرور نمایید +Comment[fi]=Selaa Internetin WWW-sivuja +Comment[fr]=Naviguer sur le Web +Comment[gl]=Navegar pola rede +Comment[he]=גלישה ברחבי האינטרנט +Comment[hr]=Pretražite web +Comment[hu]=A világháló böngészése +Comment[it]=Esplora il web +Comment[ja]=ウェブを閲覧します +Comment[ko]=웹을 돌아 다닙니다 +Comment[ku]=Li torê bigere +Comment[lt]=Naršykite internete +Comment[nb]=Surf på nettet +Comment[nl]=Verken het internet +Comment[nn]=Surf på nettet +Comment[no]=Surf på nettet +Comment[pl]=Przeglądanie stron WWW +Comment[pt]=Navegue na Internet +Comment[pt_BR]=Navegue na Internet +Comment[ro]=Navigați pe Internet +Comment[ru]=Доступ в Интернет +Comment[sk]=Prehliadanie internetu +Comment[sl]=Brskajte po spletu +Comment[sv]=Surfa på webben +Comment[tr]=İnternet'te Gezinin +Comment[ug]=دۇنيادىكى توربەتلەرنى كۆرگىلى بولىدۇ +Comment[uk]=Перегляд сторінок Інтернету +Comment[vi]=Để duyệt các trang web +Comment[zh_CN]=浏览互联网 +Comment[zh_TW]=瀏覽網際網路 +GenericName=Web Browser +GenericName[ar]=متصفح ويب +GenericName[ast]=Restolador Web +GenericName[bn]=ওয়েব ব্রাউজার +GenericName[ca]=Navegador web +GenericName[cs]=Webový prohlížeč +GenericName[da]=Webbrowser +GenericName[el]=Περιηγητής διαδικτύου +GenericName[es]=Navegador web +GenericName[et]=Veebibrauser +GenericName[fa]=مرورگر اینترنتی +GenericName[fi]=WWW-selain +GenericName[fr]=Navigateur Web +GenericName[gl]=Navegador Web +GenericName[he]=דפדפן אינטרנט +GenericName[hr]=Web preglednik +GenericName[hu]=Webböngésző +GenericName[it]=Browser web +GenericName[ja]=ウェブ・ブラウザ +GenericName[ko]=웹 브라우저 +GenericName[ku]=Geroka torê +GenericName[lt]=Interneto naršyklė +GenericName[nb]=Nettleser +GenericName[nl]=Webbrowser +GenericName[nn]=Nettlesar +GenericName[no]=Nettleser +GenericName[pl]=Przeglądarka WWW +GenericName[pt]=Navegador Web +GenericName[pt_BR]=Navegador Web +GenericName[ro]=Navigator Internet +GenericName[ru]=Веб-браузер +GenericName[sk]=Internetový prehliadač +GenericName[sl]=Spletni brskalnik +GenericName[sv]=Webbläsare +GenericName[tr]=Web Tarayıcı +GenericName[ug]=توركۆرگۈ +GenericName[uk]=Веб-браузер +GenericName[vi]=Trình duyệt Web +GenericName[zh_CN]=网络浏览器 +GenericName[zh_TW]=網路瀏覽器 +Keywords=Internet;WWW;Browser;Web;Explorer +Keywords[ar]=انترنت;إنترنت;متصفح;ويب;وب +Keywords[ast]=Internet;WWW;Restolador;Web;Esplorador +Keywords[ca]=Internet;WWW;Navegador;Web;Explorador;Explorer +Keywords[cs]=Internet;WWW;Prohlížeč;Web;Explorer +Keywords[da]=Internet;Internettet;WWW;Browser;Browse;Web;Surf;Nettet +Keywords[de]=Internet;WWW;Browser;Web;Explorer;Webseite;Site;surfen;online;browsen +Keywords[el]=Internet;WWW;Browser;Web;Explorer;Διαδίκτυο;Περιηγητής;Firefox;Φιρεφοχ;Ιντερνετ +Keywords[es]=Explorador;Internet;WWW +Keywords[fi]=Internet;WWW;Browser;Web;Explorer;selain;Internet-selain;internetselain;verkkoselain;netti;surffaa +Keywords[fr]=Internet;WWW;Browser;Web;Explorer;Fureteur;Surfer;Navigateur +Keywords[he]=דפדפן;אינטרנט;רשת;אתרים;אתר;פיירפוקס;מוזילה; +Keywords[hr]=Internet;WWW;preglednik;Web +Keywords[hu]=Internet;WWW;Böngésző;Web;Háló;Net;Explorer +Keywords[it]=Internet;WWW;Browser;Web;Navigatore +Keywords[is]=Internet;WWW;Vafri;Vefur;Netvafri;Flakk +Keywords[ja]=Internet;WWW;Web;インターネット;ブラウザ;ウェブ;エクスプローラ +Keywords[nb]=Internett;WWW;Nettleser;Explorer;Web;Browser;Nettside +Keywords[nl]=Internet;WWW;Browser;Web;Explorer;Verkenner;Website;Surfen;Online +Keywords[pt]=Internet;WWW;Browser;Web;Explorador;Navegador +Keywords[pt_BR]=Internet;WWW;Browser;Web;Explorador;Navegador +Keywords[ru]=Internet;WWW;Browser;Web;Explorer;интернет;браузер;веб;файрфокс;огнелис +Keywords[sk]=Internet;WWW;Prehliadač;Web;Explorer +Keywords[sl]=Internet;WWW;Browser;Web;Explorer;Brskalnik;Splet +Keywords[tr]=İnternet;WWW;Tarayıcı;Web;Gezgin;Web sitesi;Site;sörf;çevrimiçi;tara +Keywords[uk]=Internet;WWW;Browser;Web;Explorer;Інтернет;мережа;переглядач;оглядач;браузер;веб;файрфокс;вогнелис;перегляд +Keywords[vi]=Internet;WWW;Browser;Web;Explorer;Trình duyệt;Trang web +Keywords[zh_CN]=Internet;WWW;Browser;Web;Explorer;网页;浏览;上网;火狐;Firefox;ff;互联网;网站; +Keywords[zh_TW]=Internet;WWW;Browser;Web;Explorer;網際網路;網路;瀏覽器;上網;網頁;火狐 +Exec=firefox %u +Icon=org.mozilla.firefox +Terminal=false +Type=Application +MimeType=text/html;text/xml;application/xhtml+xml;application/vnd.mozilla.xul+xml;text/mml;x-scheme-handler/http;x-scheme-handler/https; +StartupNotify=true +Categories=Network;WebBrowser; +Keywords=web;browser;internet; +Actions=new-window;new-private-window;profile-manager-window; +StartupWMClass=firefox + + +[Desktop Action new-window] +Name=Open a New Window +Name[ach]=Dirica manyen +Name[af]=Nuwe venster +Name[an]=Nueva finestra +Name[ar]=نافذة جديدة +Name[as]=নতুন উইন্ডো +Name[ast]=Ventana nueva +Name[az]=Yeni Pəncərə +Name[be]=Новае акно +Name[bg]=Нов прозорец +Name[bn-BD]=নতুন উইন্ডো (N) +Name[bn-IN]=নতুন উইন্ডো +Name[br]=Prenestr nevez +Name[brx]=गोदान उइन्ड'(N) +Name[bs]=Novi prozor +Name[ca]=Finestra nova +Name[cak]=K'ak'a' tzuwäch +Name[cs]=Nové okno +Name[cy]=Ffenestr Newydd +Name[da]=Nyt vindue +Name[de]=Neues Fenster +Name[dsb]=Nowe wokno +Name[el]=Νέο παράθυρο +Name[en-GB]=New Window +Name[en-US]=New Window +Name[en-ZA]=New Window +Name[eo]=Nova fenestro +Name[es-AR]=Nueva ventana +Name[es-CL]=Nueva ventana +Name[es-ES]=Nueva ventana +Name[es-MX]=Nueva ventana +Name[et]=Uus aken +Name[eu]=Leiho berria +Name[fa]=پنجره جدید +Name[ff]=Henorde Hesere +Name[fi]=Uusi ikkuna +Name[fr]=Nouvelle fenêtre +Name[fy-NL]=Nij finster +Name[ga-IE]=Fuinneog Nua +Name[gd]=Uinneag ùr +Name[gl]=Nova xanela +Name[gn]=Ovetã pyahu +Name[gu-IN]=નવી વિન્ડો +Name[he]=חלון חדש +Name[hi-IN]=नया विंडो +Name[hr]=Novi prozor +Name[hsb]=Nowe wokno +Name[hu]=Új ablak +Name[hy-AM]=Նոր Պատուհան +Name[id]=Jendela Baru +Name[is]=Nýr gluggi +Name[it]=Nuova finestra +Name[ja]=新しいウィンドウ +Name[ja-JP-mac]=新規ウインドウ +Name[ka]=ახალი ფანჯარა +Name[kk]=Жаңа терезе +Name[km]=បង្អួចថ្មី +Name[kn]=ಹೊಸ ಕಿಟಕಿ +Name[ko]=새 창 +Name[kok]=नवें जनेल +Name[ks]=نئئ وِنڈو +Name[lij]=Neuvo barcon +Name[lo]=ຫນ້າຕ່າງໃຫມ່ +Name[lt]=Naujas langas +Name[ltg]=Jauns lūgs +Name[lv]=Jauns logs +Name[mai]=नव विंडो +Name[mk]=Нов прозорец +Name[ml]=പുതിയ ജാലകം +Name[mr]=नवीन पटल +Name[ms]=Tetingkap Baru +Name[my]=ဝင်းဒိုးအသစ် +Name[nb-NO]=Nytt vindu +Name[ne-NP]=नयाँ सञ्झ्याल +Name[nl]=Nieuw venster +Name[nn-NO]=Nytt vindauge +Name[or]=ନୂତନ ୱିଣ୍ଡୋ +Name[pa-IN]=ਨਵੀਂ ਵਿੰਡੋ +Name[pl]=Nowe okno +Name[pt-BR]=Nova janela +Name[pt-PT]=Nova janela +Name[rm]=Nova fanestra +Name[ro]=Fereastră nouă +Name[ru]=Новое окно +Name[sat]=नावा विंडो (N) +Name[si]=නව කවුළුවක් +Name[sk]=Nové okno +Name[sl]=Novo okno +Name[son]=Zanfun taaga +Name[sq]=Dritare e Re +Name[sr]=Нови прозор +Name[sv-SE]=Nytt fönster +Name[ta]=புதிய சாளரம் +Name[te]=కొత్త విండో +Name[th]=หน้าต่างใหม่ +Name[tr]=Yeni pencere +Name[tsz]=Eraatarakua jimpani +Name[uk]=Нове вікно +Name[ur]=نیا دریچہ +Name[uz]=Yangi oyna +Name[vi]=Cửa sổ mới +Name[wo]=Palanteer bu bees +Name[xh]=Ifestile entsha +Name[zh-CN]=新建窗口 +Name[zh-TW]=開新視窗 +Exec=firefox --new-window %u + +[Desktop Action new-private-window] +Name=Open a New Private Window +Name[ach]=Dirica manyen me mung +Name[af]=Nuwe privaatvenster +Name[an]=Nueva finestra privada +Name[ar]=نافذة خاصة جديدة +Name[as]=নতুন ব্যক্তিগত উইন্ডো +Name[ast]=Ventana privada nueva +Name[az]=Yeni Məxfi Pəncərə +Name[be]=Новае акно адасаблення +Name[bg]=Нов прозорец за поверително сърфиране +Name[bn-BD]=নতুন ব্যক্তিগত উইন্ডো +Name[bn-IN]=নতুন ব্যক্তিগত উইন্ডো +Name[br]=Prenestr merdeiñ prevez nevez +Name[brx]=गोदान प्राइभेट उइन्ड' +Name[bs]=Novi privatni prozor +Name[ca]=Finestra privada nova +Name[cak]=K'ak'a' ichinan tzuwäch +Name[cs]=Nové anonymní okno +Name[cy]=Ffenestr Breifat Newydd +Name[da]=Nyt privat vindue +Name[de]=Neues privates Fenster +Name[dsb]=Nowe priwatne wokno +Name[el]=Νέο παράθυρο ιδιωτικής περιήγησης +Name[en-GB]=New Private Window +Name[en-US]=New Private Window +Name[en-ZA]=New Private Window +Name[eo]=Nova privata fenestro +Name[es-AR]=Nueva ventana privada +Name[es-CL]=Nueva ventana privada +Name[es-ES]=Nueva ventana privada +Name[es-MX]=Nueva ventana privada +Name[et]=Uus privaatne aken +Name[eu]=Leiho pribatu berria +Name[fa]=پنجره ناشناس جدید +Name[ff]=Henorde Suturo Hesere +Name[fi]=Uusi yksityinen ikkuna +Name[fr]=Nouvelle fenêtre de navigation privée +Name[fy-NL]=Nij priveefinster +Name[ga-IE]=Fuinneog Nua Phríobháideach +Name[gd]=Uinneag phrìobhaideach ùr +Name[gl]=Nova xanela privada +Name[gn]=Ovetã ñemi pyahu +Name[gu-IN]=નવી ખાનગી વિન્ડો +Name[he]=חלון פרטי חדש +Name[hi-IN]=नयी निजी विंडो +Name[hr]=Novi privatni prozor +Name[hsb]=Nowe priwatne wokno +Name[hu]=Új privát ablak +Name[hy-AM]=Սկսել Գաղտնի դիտարկում +Name[id]=Jendela Mode Pribadi Baru +Name[is]=Nýr huliðsgluggi +Name[it]=Nuova finestra anonima +Name[ja]=新しいプライベートウィンドウ +Name[ja-JP-mac]=新規プライベートウインドウ +Name[ka]=ახალი პირადი ფანჯარა +Name[kk]=Жаңа жекелік терезе +Name[km]=បង្អួចឯកជនថ្មី +Name[kn]=ಹೊಸ ಖಾಸಗಿ ಕಿಟಕಿ +Name[ko]=새 사생활 보호 모드 +Name[kok]=नवो खाजगी विंडो +Name[ks]=نْو پرایوٹ وینڈو& +Name[lij]=Neuvo barcon privou +Name[lo]=ເປີດຫນ້າຕ່າງສວນຕົວຂື້ນມາໃຫມ່ +Name[lt]=Naujas privataus naršymo langas +Name[ltg]=Jauns privatais lūgs +Name[lv]=Jauns privātais logs +Name[mai]=नया निज विंडो (W) +Name[mk]=Нов приватен прозорец +Name[ml]=പുതിയ സ്വകാര്യ ജാലകം +Name[mr]=नवीन वैयक्तिक पटल +Name[ms]=Tetingkap Persendirian Baharu +Name[my]=New Private Window +Name[nb-NO]=Nytt privat vindu +Name[ne-NP]=नयाँ निजी सञ्झ्याल +Name[nl]=Nieuw privévenster +Name[nn-NO]=Nytt privat vindauge +Name[or]=ନୂତନ ବ୍ୟକ୍ତିଗତ ୱିଣ୍ଡୋ +Name[pa-IN]=ਨਵੀਂ ਪ੍ਰਾਈਵੇਟ ਵਿੰਡੋ +Name[pl]=Nowe okno prywatne +Name[pt-BR]=Nova janela privativa +Name[pt-PT]=Nova janela privada +Name[rm]=Nova fanestra privata +Name[ro]=Fereastră privată nouă +Name[ru]=Новое приватное окно +Name[sat]=नावा निजेराक् विंडो (W ) +Name[si]=නව පුද්ගලික කවුළුව (W) +Name[sk]=Nové okno v režime Súkromné prehliadanie +Name[sl]=Novo zasebno okno +Name[son]=Sutura zanfun taaga +Name[sq]=Dritare e Re Private +Name[sr]=Нови приватан прозор +Name[sv-SE]=Nytt privat fönster +Name[ta]=புதிய தனிப்பட்ட சாளரம் +Name[te]=కొత్త ఆంతరంగిక విండో +Name[th]=หน้าต่างส่วนตัวใหม่ +Name[tr]=Yeni gizli pencere +Name[tsz]=Juchiiti eraatarakua jimpani +Name[uk]=Приватне вікно +Name[ur]=نیا نجی دریچہ +Name[uz]=Yangi maxfiy oyna +Name[vi]=Cửa sổ riêng tư mới +Name[wo]=Panlanteeru biir bu bees +Name[xh]=Ifestile yangasese entsha +Name[zh-CN]=新建隐私浏览窗口 +Name[zh-TW]=新增隱私視窗 +Exec=firefox --private-window %u + +[Desktop Action profile-manager-window] +Name=Open the Profile Manager +Name[cs]=Správa profilů +Name[de]=Profilverwaltung öffnen +Name[fr]=Ouvrir le gestionnaire de profils +Exec=firefox --ProfileManager diff --git a/taskcluster/docker/firefox-flatpak/policies.json b/taskcluster/docker/firefox-flatpak/policies.json new file mode 100644 index 0000000000..2645f20c0c --- /dev/null +++ b/taskcluster/docker/firefox-flatpak/policies.json @@ -0,0 +1,6 @@ +{ + "policies": { + "DisableAppUpdate": true, + "DontCheckDefaultBrowser": true + } +} diff --git a/taskcluster/docker/firefox-flatpak/runme.sh b/taskcluster/docker/firefox-flatpak/runme.sh new file mode 100755 index 0000000000..6766af3362 --- /dev/null +++ b/taskcluster/docker/firefox-flatpak/runme.sh @@ -0,0 +1,178 @@ +#!/bin/bash +set -xe + +# Future products supporting Flatpaks will set this accordingly +: PRODUCT "${PRODUCT:=firefox}" + +# Required env variables + +test "$VERSION" +test "$BUILD_NUMBER" +test "$CANDIDATES_DIR" +test "$L10N_CHANGESETS" +test "$FLATPAK_BRANCH" + +# Optional env variables +: WORKSPACE "${WORKSPACE:=/home/worker/workspace}" +: ARTIFACTS_DIR "${ARTIFACTS_DIR:=/home/worker/artifacts}" + +pwd + +# XXX: this is used to populate the datetime in org.mozilla.firefox.appdata.xml +DATE=$(date +%Y-%m-%d) +export DATE + +SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +TARGET_TAR_XZ_FULL_PATH="$ARTIFACTS_DIR/target.flatpak.tar.xz" +SOURCE_DEST="${WORKSPACE}/source" +FREEDESKTOP_VERSION="22.08" +FIREFOX_BASEAPP_CHANNEL="22.08" + + +# XXX: these commands are temporarily, there's an upcoming fix in the upstream Docker image +# that we work on top of, from `freedesktopsdk`, that will make these two lines go away eventually +mkdir -p /root /tmp /var/tmp +mkdir -p "$ARTIFACTS_DIR" +rm -rf "$SOURCE_DEST" && mkdir -p "$SOURCE_DEST" + +# XXX ensure we have a clean slate in the local flatpak repo +rm -rf ~/.local/share/flatpak/ + + +CURL="curl --location --retry 10 --retry-delay 10" + +# Download en-US linux64 binary +$CURL -o "${WORKSPACE}/firefox.tar.bz2" \ + "${CANDIDATES_DIR}/${VERSION}-candidates/build${BUILD_NUMBER}/linux-x86_64/en-US/firefox-${VERSION}.tar.bz2" + +# Use list of locales to fetch L10N XPIs +$CURL -o "${WORKSPACE}/l10n_changesets.json" "$L10N_CHANGESETS" +locales=$(python3 "$SCRIPT_DIRECTORY/extract_locales_from_l10n_json.py" "${WORKSPACE}/l10n_changesets.json") + +DISTRIBUTION_DIR="$SOURCE_DEST/distribution" +if [[ "$PRODUCT" == "firefox" ]]; then + # Get Flatpak configuration + PARTNER_CONFIG_DIR="$WORKSPACE/partner_config" + git clone https://github.com/mozilla-partners/flatpak.git "$PARTNER_CONFIG_DIR" + mv "$PARTNER_CONFIG_DIR/desktop/flatpak/distribution" "$DISTRIBUTION_DIR" +else + mkdir -p "$DISTRIBUTION_DIR" +fi + +mkdir -p "$DISTRIBUTION_DIR/extensions" +for locale in $locales; do + $CURL -o "$DISTRIBUTION_DIR/extensions/langpack-${locale}@firefox.mozilla.org.xpi" \ + "$CANDIDATES_DIR/${VERSION}-candidates/build${BUILD_NUMBER}/linux-x86_64/xpi/${locale}.xpi" +done + +envsubst < "$SCRIPT_DIRECTORY/org.mozilla.firefox.appdata.xml.in" > "${WORKSPACE}/org.mozilla.firefox.appdata.xml" +cp -v "$SCRIPT_DIRECTORY/org.mozilla.firefox.desktop" "$WORKSPACE" +# Add a group policy file to disable app updates, as those are handled by Flathub +cp -v "$SCRIPT_DIRECTORY/policies.json" "$WORKSPACE" +cp -v "$SCRIPT_DIRECTORY/default-preferences.js" "$WORKSPACE" +cp -v "$SCRIPT_DIRECTORY/launch-script.sh" "$WORKSPACE" +cd "${WORKSPACE}" + +flatpak remote-add --user --if-not-exists --from flathub https://dl.flathub.org/repo/flathub.flatpakrepo +# XXX: added --user to `flatpak install` to avoid ambiguity +flatpak install --user -y flathub org.mozilla.firefox.BaseApp//${FIREFOX_BASEAPP_CHANNEL} --no-deps + +# XXX: this command is temporarily, there's an upcoming fix in the upstream Docker image +# that we work on top of, from `freedesktopsdk`, that will make these two lines go away eventually +mkdir -p build +cp -r ~/.local/share/flatpak/app/org.mozilla.firefox.BaseApp/current/active/files build/files + +ARCH=$(flatpak --default-arch) +cat <<EOF > build/metadata +[Application] +name=org.mozilla.firefox +runtime=org.freedesktop.Platform/${ARCH}/${FREEDESKTOP_VERSION} +sdk=org.freedesktop.Sdk/${ARCH}/${FREEDESKTOP_VERSION} +base=app/org.mozilla.firefox.BaseApp/${ARCH}/${FIREFOX_BASEAPP_CHANNEL} +[Extension org.mozilla.firefox.Locale] +directory=share/runtime/langpack +autodelete=true +locale-subset=true + +[Extension org.freedesktop.Platform.ffmpeg-full] +directory=lib/ffmpeg +add-ld-path=. +no-autodownload=true +version=${FREEDESKTOP_VERSION} +EOF + +cat <<EOF > build/metadata.locale +[Runtime] +name=org.mozilla.firefox.Locale + +[ExtensionOf] +ref=app/org.mozilla.firefox/${ARCH}/${FLATPAK_BRANCH} +EOF + +appdir=build/files +install -d "${appdir}/lib/" +(cd "${appdir}/lib/" && tar jxf "${WORKSPACE}/firefox.tar.bz2") +install -D -m644 -t "${appdir}/share/appdata" org.mozilla.firefox.appdata.xml +install -D -m644 -t "${appdir}/share/applications" org.mozilla.firefox.desktop +for size in 16 32 48 64 128; do + install -D -m644 "${appdir}/lib/firefox/browser/chrome/icons/default/default${size}.png" "${appdir}/share/icons/hicolor/${size}x${size}/apps/org.mozilla.firefox.png" +done +mkdir -p "${appdir}/lib/ffmpeg" + +appstream-compose --prefix="${appdir}" --origin=flatpak --basename=org.mozilla.firefox org.mozilla.firefox +appstream-util mirror-screenshots "${appdir}"/share/app-info/xmls/org.mozilla.firefox.xml.gz "https://dl.flathub.org/repo/screenshots/org.mozilla.firefox-${FLATPAK_BRANCH}" build/screenshots "build/screenshots/org.mozilla.firefox-${FLATPAK_BRANCH}" + +# XXX: we used to `install -D` before which automatically created the components +# of target, now we need to manually do this since we're symlinking +mkdir -p "${appdir}/lib/firefox/distribution/extensions" +# XXX: we put the langpacks in /app/share/locale/$LANG_CODE and symlink that +# directory to where Firefox looks them up; this way only subset configured +# on user system is downloaded vs all locales +for locale in $locales; do + install -D -m644 -t "${appdir}/share/runtime/langpack/${locale%%-*}/" "${DISTRIBUTION_DIR}/extensions/langpack-${locale}@firefox.mozilla.org.xpi" + ln -sf "/app/share/runtime/langpack/${locale%%-*}/langpack-${locale}@firefox.mozilla.org.xpi" "${appdir}/lib/firefox/distribution/extensions/langpack-${locale}@firefox.mozilla.org.xpi" +done +install -D -m644 -t "${appdir}/lib/firefox/distribution" "$DISTRIBUTION_DIR/distribution.ini" +install -D -m644 -t "${appdir}/lib/firefox/distribution" policies.json +install -D -m644 -t "${appdir}/lib/firefox/browser/defaults/preferences" default-preferences.js +install -D -m755 launch-script.sh "${appdir}/bin/firefox" + +# We need to set GTK_PATH to load cups printing backend which is missing in +# freedesktop sdk. +flatpak build-finish build \ + --share=ipc \ + --share=network \ + --env=GTK_PATH=/app/lib/gtkmodules \ + --socket=pulseaudio \ + --socket=wayland \ + --socket=x11 \ + --socket=pcsc \ + --socket=cups \ + --require-version=0.11.1 \ + --persist=.mozilla \ + --filesystem=xdg-download:rw \ + --device=all \ + --talk-name=org.freedesktop.FileManager1 \ + --system-talk-name=org.freedesktop.NetworkManager \ + --talk-name=org.a11y.Bus \ + --talk-name=org.gnome.SessionManager \ + --talk-name=org.freedesktop.ScreenSaver \ + --talk-name="org.gtk.vfs.*" \ + --talk-name=org.freedesktop.Notifications \ + --own-name="org.mpris.MediaPlayer2.firefox.*" \ + --own-name="org.mozilla.firefox.*" \ + --own-name="org.mozilla.firefox_beta.*" \ + --command=firefox + +flatpak build-export --disable-sandbox --no-update-summary --exclude='/share/runtime/langpack/*/*' repo build "$FLATPAK_BRANCH" +flatpak build-export --disable-sandbox --no-update-summary --metadata=metadata.locale --files=files/share/runtime/langpack repo build "$FLATPAK_BRANCH" +ostree commit --repo=repo --canonical-permissions --branch=screenshots/x86_64 build/screenshots +flatpak build-update-repo --generate-static-deltas repo +tar cvfJ flatpak.tar.xz repo + +mv -- flatpak.tar.xz "$TARGET_TAR_XZ_FULL_PATH" + +# XXX: if we ever wanted to go back to building flatpak bundles, we can revert this command; useful for testing individual artifacts, not publishable +# flatpak build-bundle "$WORKSPACE"/repo org.mozilla.firefox.flatpak org.mozilla.firefox +# TARGET_FULL_PATH="$ARTIFACTS_DIR/target.flatpak" +# mv -- *.flatpak "$TARGET_FULL_PATH" diff --git a/taskcluster/docker/firefox-snap/Dockerfile b/taskcluster/docker/firefox-snap/Dockerfile new file mode 100644 index 0000000000..dc002c0133 --- /dev/null +++ b/taskcluster/docker/firefox-snap/Dockerfile @@ -0,0 +1,71 @@ +## The below code is from snapcraft/docker/stable.Dockerfile +## The modifications done are part of the documentation for enabling core18 snaps. +## https://snapcraft.io/docs/t/creating-docker-images-for-snapcraft/11739 + +# /!\ The base image must follow the version of the `coreXX` package +# https://forum.snapcraft.io/t/issues-dockerizing-a-snapcraft-build-process/30294/5 +FROM ubuntu:focal + +ENV LANG='en_US.UTF-8' \ + LANGUAGE='en_US:en' \ + LC_ALL='en_US.UTF-8' \ + PATH="/snap/bin:$PATH" \ + SNAP='/snap/snapcraft/current' \ + SNAP_NAME='snapcraft' \ + SNAP_ARCH='amd64' \ + TERM='dumb' \ + TZ='Etc/UTC' + +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Grab dependencies. snapd is now required per https://github.com/snapcore/snapcraft/pull/3210 +RUN apt-get update && \ + apt-get dist-upgrade --yes && \ + apt-get install --yes \ + curl \ + jq \ + squashfs-tools \ + locales \ + bzip2 \ + curl \ + gcc \ + git \ + python3 \ + locales \ + snapd \ + sudo \ + && \ +locale-gen "$LANG" + +COPY download_and_install_snap.sh . + +RUN bash download_and_install_snap.sh 'core20' +RUN bash download_and_install_snap.sh 'gnome-3-38-2004' +RUN bash download_and_install_snap.sh 'gnome-3-38-2004-sdk' +RUN bash download_and_install_snap.sh 'snapcraft' + +# Fix Python3 installation: Make sure we use the interpreter from +# the snapcraft snap: +RUN unlink /snap/snapcraft/current/usr/bin/python3 && \ + ln -s /snap/snapcraft/current/usr/bin/python3.* /snap/snapcraft/current/usr/bin/python3 && \ + echo /snap/snapcraft/current/lib/python3.*/site-packages >> /snap/snapcraft/current/usr/lib/python3/dist-packages/site-packages.pth + +# Create a snapcraft runner (TODO: move version detection to the core of snapcraft). +RUN mkdir -p /snap/bin +RUN echo "#!/bin/sh" > /snap/bin/snapcraft +RUN snap_version="$(awk '/^version:/{print $2}' /snap/snapcraft/current/meta/snap.yaml)" && echo "export SNAP_VERSION=\"$snap_version\"" >> /snap/bin/snapcraft +RUN echo 'exec "$SNAP/usr/bin/python3" "$SNAP/bin/snapcraft" "$@"' >> /snap/bin/snapcraft +RUN chmod +x /snap/bin/snapcraft + +RUN mkdir /scripts/ +WORKDIR /scripts/ +# Copy everything in the docker/firefox-snap folder but the Dockerfile +# +# XXX The following pattern is neither a regex nor a glob one. It's +# documented at https://golang.org/pkg/path/filepath/#Match. There's no +# way of explicitly filtering out "Dockerfile". If one day, someone needs +# to add a file starting with "D", then we must revisit the pattern below. +COPY [^D]* /scripts/ + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/firefox-snap/Makefile b/taskcluster/docker/firefox-snap/Makefile new file mode 100644 index 0000000000..d71dc70881 --- /dev/null +++ b/taskcluster/docker/firefox-snap/Makefile @@ -0,0 +1,12 @@ +DOCKERIO_USERNAME =$(error DOCKERIO_USERNAME should be set) +IMAGE_NAME = firefox-snapcraft +FULL_IMAGE_NAME = $(DOCKERIO_USERNAME)/$(IMAGE_NAME) + +build: + docker build -t $(FULL_IMAGE_NAME) --no-cache --rm . + +push: + docker push $(FULL_IMAGE_NAME):latest + +pull: + docker pull $(FULL_IMAGE_NAME):latest diff --git a/taskcluster/docker/firefox-snap/download_and_install_snap.sh b/taskcluster/docker/firefox-snap/download_and_install_snap.sh new file mode 100644 index 0000000000..7def88b2aa --- /dev/null +++ b/taskcluster/docker/firefox-snap/download_and_install_snap.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -ex + +SNAP_NAME="$1" +SNAP_CHANNEL="${2:-stable}" +SNAP_INSTALL_LOCATION="${3:-/snap}" + +SNAP_METADATA="$(curl --header 'X-Ubuntu-Series: 16' "https://api.snapcraft.io/api/v1/snaps/details/$SNAP_NAME?channel=$SNAP_CHANNEL")" + +set +x +SNAP_SHA512="$(echo "$SNAP_METADATA" | jq '.download_sha512' -r)" +SNAP_DOWNLOAD_URL="$(echo "$SNAP_METADATA" | jq '.download_url' -r)" +SNAP_LAST_UPDATED="$(echo "$SNAP_METADATA" | jq '.last_updated' -r)" +SNAP_REVISION="$(echo "$SNAP_METADATA" | jq '.revision' -r)" +SNAP_VERSION="$(echo "$SNAP_METADATA" | jq '.version' -r)" +set -x + +echo "Downloading $SNAP_NAME, version $SNAP_VERSION, revision $SNAP_REVISION (last updated: $SNAP_LAST_UPDATED)..." +curl --location "$SNAP_DOWNLOAD_URL" --output "$SNAP_NAME.snap" +sha512sum -c <(echo "$SNAP_SHA512 $SNAP_NAME.snap") + +mkdir -p "$SNAP_INSTALL_LOCATION/$SNAP_NAME" +unsquashfs -d "$SNAP_INSTALL_LOCATION/$SNAP_NAME/current" "$SNAP_NAME.snap" +rm "$SNAP_NAME.snap" + +echo "$SNAP_NAME version $SNAP_VERSION has correctly been uploaded and installed."
\ No newline at end of file diff --git a/taskcluster/docker/firefox-snap/extract_locales_from_l10n_json.py b/taskcluster/docker/firefox-snap/extract_locales_from_l10n_json.py new file mode 100644 index 0000000000..07bc007a31 --- /dev/null +++ b/taskcluster/docker/firefox-snap/extract_locales_from_l10n_json.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import sys + +l10n_changesets_json_path = sys.argv[1] +with open(l10n_changesets_json_path) as f: + locales = json.load(f).keys() +linux_locales = [l for l in locales if l != "ja-JP-mac"] + +print("\n".join(sorted(linux_locales))) diff --git a/taskcluster/docker/firefox-snap/firefox.desktop b/taskcluster/docker/firefox-snap/firefox.desktop new file mode 100644 index 0000000000..73e36e8d83 --- /dev/null +++ b/taskcluster/docker/firefox-snap/firefox.desktop @@ -0,0 +1,219 @@ +[Desktop Entry] +Version=1.0 +Name=Firefox Web Browser +Name[ar]=متصفح الويب فَيَرفُكْس +Name[ast]=Restolador web Firefox +Name[bn]=ফায়ারফক্স ওয়েব ব্রাউজার +Name[ca]=Navegador web Firefox +Name[cs]=Firefox Webový prohlížeč +Name[da]=Firefox - internetbrowser +Name[el]=Περιηγητής Firefox +Name[es]=Navegador web Firefox +Name[et]=Firefoxi veebibrauser +Name[fa]=مرورگر اینترنتی Firefox +Name[fi]=Firefox-selain +Name[fr]=Navigateur Web Firefox +Name[gl]=Navegador web Firefox +Name[he]=דפדפן האינטרנט Firefox +Name[hr]=Firefox web preglednik +Name[hu]=Firefox webböngésző +Name[it]=Firefox Browser Web +Name[ja]=Firefox ウェブ・ブラウザ +Name[ko]=Firefox 웹 브라우저 +Name[ku]=Geroka torê Firefox +Name[lt]=Firefox interneto naršyklė +Name[nb]=Firefox Nettleser +Name[nl]=Firefox webbrowser +Name[nn]=Firefox Nettlesar +Name[no]=Firefox Nettleser +Name[pl]=Przeglądarka WWW Firefox +Name[pt]=Firefox Navegador Web +Name[pt_BR]=Navegador Web Firefox +Name[ro]=Firefox – Navigator Internet +Name[ru]=Веб-браузер Firefox +Name[sk]=Firefox - internetový prehliadač +Name[sl]=Firefox spletni brskalnik +Name[sv]=Firefox webbläsare +Name[tr]=Firefox Web Tarayıcısı +Name[ug]=Firefox توركۆرگۈ +Name[uk]=Веб-браузер Firefox +Name[vi]=Trình duyệt web Firefox +Name[zh_CN]=Firefox 网络浏览器 +Name[zh_TW]=Firefox 網路瀏覽器 +Comment=Browse the World Wide Web +Comment[ar]=تصفح الشبكة العنكبوتية العالمية +Comment[ast]=Restola pela Rede +Comment[bn]=ইন্টারনেট ব্রাউজ করুন +Comment[ca]=Navegueu per la web +Comment[cs]=Prohlížení stránek World Wide Webu +Comment[da]=Surf på internettet +Comment[de]=Im Internet surfen +Comment[el]=Μπορείτε να περιηγηθείτε στο διαδίκτυο (Web) +Comment[es]=Navegue por la web +Comment[et]=Lehitse veebi +Comment[fa]=صفحات شبکه جهانی اینترنت را مرور نمایید +Comment[fi]=Selaa Internetin WWW-sivuja +Comment[fr]=Naviguer sur le Web +Comment[gl]=Navegar pola rede +Comment[he]=גלישה ברחבי האינטרנט +Comment[hr]=Pretražite web +Comment[hu]=A világháló böngészése +Comment[it]=Esplora il web +Comment[ja]=ウェブを閲覧します +Comment[ko]=웹을 돌아 다닙니다 +Comment[ku]=Li torê bigere +Comment[lt]=Naršykite internete +Comment[nb]=Surf på nettet +Comment[nl]=Verken het internet +Comment[nn]=Surf på nettet +Comment[no]=Surf på nettet +Comment[pl]=Przeglądanie stron WWW +Comment[pt]=Navegue na Internet +Comment[pt_BR]=Navegue na Internet +Comment[ro]=Navigați pe Internet +Comment[ru]=Доступ в Интернет +Comment[sk]=Prehliadanie internetu +Comment[sl]=Brskajte po spletu +Comment[sv]=Surfa på webben +Comment[tr]=İnternet'te Gezinin +Comment[ug]=دۇنيادىكى توربەتلەرنى كۆرگىلى بولىدۇ +Comment[uk]=Перегляд сторінок Інтернету +Comment[vi]=Để duyệt các trang web +Comment[zh_CN]=浏览互联网 +Comment[zh_TW]=瀏覽網際網路 +GenericName=Web Browser +GenericName[ar]=متصفح ويب +GenericName[ast]=Restolador Web +GenericName[bn]=ওয়েব ব্রাউজার +GenericName[ca]=Navegador web +GenericName[cs]=Webový prohlížeč +GenericName[da]=Webbrowser +GenericName[el]=Περιηγητής διαδικτύου +GenericName[es]=Navegador web +GenericName[et]=Veebibrauser +GenericName[fa]=مرورگر اینترنتی +GenericName[fi]=WWW-selain +GenericName[fr]=Navigateur Web +GenericName[gl]=Navegador Web +GenericName[he]=דפדפן אינטרנט +GenericName[hr]=Web preglednik +GenericName[hu]=Webböngésző +GenericName[it]=Browser web +GenericName[ja]=ウェブ・ブラウザ +GenericName[ko]=웹 브라우저 +GenericName[ku]=Geroka torê +GenericName[lt]=Interneto naršyklė +GenericName[nb]=Nettleser +GenericName[nl]=Webbrowser +GenericName[nn]=Nettlesar +GenericName[no]=Nettleser +GenericName[pl]=Przeglądarka WWW +GenericName[pt]=Navegador Web +GenericName[pt_BR]=Navegador Web +GenericName[ro]=Navigator Internet +GenericName[ru]=Веб-браузер +GenericName[sk]=Internetový prehliadač +GenericName[sl]=Spletni brskalnik +GenericName[sv]=Webbläsare +GenericName[tr]=Web Tarayıcı +GenericName[ug]=توركۆرگۈ +GenericName[uk]=Веб-браузер +GenericName[vi]=Trình duyệt Web +GenericName[zh_CN]=网络浏览器 +GenericName[zh_TW]=網路瀏覽器 +Keywords=Internet;WWW;Browser;Web;Explorer +Keywords[ar]=انترنت;إنترنت;متصفح;ويب;وب +Keywords[ast]=Internet;WWW;Restolador;Web;Esplorador +Keywords[ca]=Internet;WWW;Navegador;Web;Explorador;Explorer +Keywords[cs]=Internet;WWW;Prohlížeč;Web;Explorer +Keywords[da]=Internet;Internettet;WWW;Browser;Browse;Web;Surf;Nettet +Keywords[de]=Internet;WWW;Browser;Web;Explorer;Webseite;Site;surfen;online;browsen +Keywords[el]=Internet;WWW;Browser;Web;Explorer;Διαδίκτυο;Περιηγητής;Firefox;Φιρεφοχ;Ιντερνετ +Keywords[es]=Explorador;Internet;WWW +Keywords[fi]=Internet;WWW;Browser;Web;Explorer;selain;Internet-selain;internetselain;verkkoselain;netti;surffaa +Keywords[fr]=Internet;WWW;Browser;Web;Explorer;Fureteur;Surfer;Navigateur +Keywords[he]=דפדפן;אינטרנט;רשת;אתרים;אתר;פיירפוקס;מוזילה; +Keywords[hr]=Internet;WWW;preglednik;Web +Keywords[hu]=Internet;WWW;Böngésző;Web;Háló;Net;Explorer +Keywords[it]=Internet;WWW;Browser;Web;Navigatore +Keywords[is]=Internet;WWW;Vafri;Vefur;Netvafri;Flakk +Keywords[ja]=Internet;WWW;Web;インターネット;ブラウザ;ウェブ;エクスプローラ +Keywords[nb]=Internett;WWW;Nettleser;Explorer;Web;Browser;Nettside +Keywords[nl]=Internet;WWW;Browser;Web;Explorer;Verkenner;Website;Surfen;Online +Keywords[pt]=Internet;WWW;Browser;Web;Explorador;Navegador +Keywords[pt_BR]=Internet;WWW;Browser;Web;Explorador;Navegador +Keywords[ru]=Internet;WWW;Browser;Web;Explorer;интернет;браузер;веб;файрфокс;огнелис +Keywords[sk]=Internet;WWW;Prehliadač;Web;Explorer +Keywords[sl]=Internet;WWW;Browser;Web;Explorer;Brskalnik;Splet +Keywords[tr]=İnternet;WWW;Tarayıcı;Web;Gezgin;Web sitesi;Site;sörf;çevrimiçi;tara +Keywords[uk]=Internet;WWW;Browser;Web;Explorer;Інтернет;мережа;переглядач;оглядач;браузер;веб;файрфокс;вогнелис;перегляд +Keywords[vi]=Internet;WWW;Browser;Web;Explorer;Trình duyệt;Trang web +Keywords[zh_CN]=Internet;WWW;Browser;Web;Explorer;网页;浏览;上网;火狐;Firefox;ff;互联网;网站; +Keywords[zh_TW]=Internet;WWW;Browser;Web;Explorer;網際網路;網路;瀏覽器;上網;網頁;火狐 +Exec=firefox %u +Terminal=false +X-MultipleArgs=false +Type=Application +Icon=/browser/chrome/icons/default/default128.png +Categories=GNOME;GTK;Network;WebBrowser; +MimeType=text/html;text/xml;application/xhtml+xml;application/xml;application/rss+xml;application/rdf+xml;image/gif;image/jpeg;image/png;x-scheme-handler/http;x-scheme-handler/https;x-scheme-handler/ftp;x-scheme-handler/chrome;video/webm;application/x-xpinstall; +StartupNotify=true +Actions=NewWindow;NewPrivateWindow; + +[Desktop Action NewWindow] +Name=Open a New Window +Name[ar]=افتح نافذة جديدة +Name[ast]=Abrir una ventana nueva +Name[bn]=Abrir una ventana nueva +Name[ca]=Obre una finestra nova +Name[cs]=Otevřít nové okno +Name[da]=Åbn et nyt vindue +Name[de]=Ein neues Fenster öffnen +Name[el]=Άνοιγμα νέου παραθύρου +Name[es]=Abrir una ventana nueva +Name[fi]=Avaa uusi ikkuna +Name[fr]=Ouvrir une nouvelle fenêtre +Name[gl]=Abrir unha nova xanela +Name[he]=פתיחת חלון חדש +Name[hr]=Otvori novi prozor +Name[hu]=Új ablak nyitása +Name[it]=Apri una nuova finestra +Name[ja]=新しいウィンドウを開く +Name[ko]=새 창 열기 +Name[ku]=Paceyeke nû veke +Name[lt]=Atverti naują langą +Name[nb]=Åpne et nytt vindu +Name[nl]=Nieuw venster openen +Name[pt]=Abrir nova janela +Name[pt_BR]=Abrir nova janela +Name[ro]=Deschide o fereastră nouă +Name[ru]=Новое окно +Name[sk]=Otvoriť nové okno +Name[sl]=Odpri novo okno +Name[sv]=Öppna ett nytt fönster +Name[tr]=Yeni pencere aç +Name[ug]=يېڭى كۆزنەك ئېچىش +Name[uk]=Відкрити нове вікно +Name[vi]=Mở cửa sổ mới +Name[zh_CN]=新建窗口 +Name[zh_TW]=開啟新視窗 +Exec=firefox -new-window + +[Desktop Action NewPrivateWindow] +Name=Open a New Private Window +Name[ar]=افتح نافذة جديدة للتصفح الخاص +Name[ca]=Obre una finestra nova en mode d'incògnit +Name[de]=Ein neues privates Fenster öffnen +Name[es]=Abrir una ventana privada nueva +Name[fi]=Avaa uusi yksityinen ikkuna +Name[fr]=Ouvrir une nouvelle fenêtre de navigation privée +Name[he]=פתיחת חלון גלישה פרטית חדש +Name[hu]=Új privát ablak nyitása +Name[it]=Apri una nuova finestra anonima +Name[nb]=Åpne et nytt privat vindu +Name[ru]=Новое приватное окно +Name[sl]=Odpri novo okno zasebnega brskanja +Name[tr]=Yeni bir pencere aç +Name[uk]=Відкрити нове вікно у потайливому режимі +Name[zh_TW]=開啟新隱私瀏覽視窗 +Exec=firefox -private-window diff --git a/taskcluster/docker/firefox-snap/firefox.snapcraft.yaml.in b/taskcluster/docker/firefox-snap/firefox.snapcraft.yaml.in new file mode 100644 index 0000000000..18dfc64089 --- /dev/null +++ b/taskcluster/docker/firefox-snap/firefox.snapcraft.yaml.in @@ -0,0 +1,92 @@ +name: firefox +version: @VERSION@-@BUILD_NUMBER@ +summary: Mozilla Firefox web browser +description: Firefox is a powerful, extensible web browser with support for modern web application technologies. +confinement: strict +grade: stable +base: core20 +compression: lzo + +apps: + firefox: + command: firefox + command-chain: [tmpdir] + desktop: distribution/firefox.desktop + extensions: [gnome-3-38] + environment: + HOME: "$SNAP_USER_COMMON" + GTK_USE_PORTAL: 1 + slots: + - dbus-daemon + - mpris + plugs: + - avahi-observe + - browser-sandbox + - camera + - cups-control + - gsettings + - hardware-observe + - home + - joystick + - network + - network-observe + - opengl + - pulseaudio + - removable-media + - screen-inhibit-control + - system-packages-doc + - u2f-devices + - unity7 + - upower-observe + +plugs: + browser-sandbox: + interface: browser-support + allow-sandbox: true + etc-firefox-policies: + interface: system-files + read: [/etc/firefox/policies] + +layout: + /usr/share/libdrm: + bind: $SNAP/gnome-platform/usr/share/libdrm + +parts: + firefox: + plugin: dump + source: source + stage-packages: + - libxt6 + - libdbus-glib-1-2 + - libasound2 + - libpci3 + - libpulse0 + - libgl1-mesa-dri + - libgl1-mesa-glx + - libmirclient9 + - desktop-file-utils + - ffmpeg + - libc-bin + - locales-all + - libcurl3-gnutls + prime: + - -usr/lib/*/libharfbuzz* + - -usr/lib/*/*pango* + + # Find files provided by the base and platform snap and ensure they aren't + # duplicated in this snap + cleanup: + after: [firefox] + plugin: nil + build-snaps: [core20, gnome-3-38-2004] + override-prime: | + set -eux + for snap in "core20" "gnome-3-38-2004"; do + cd "/snap/$snap/current" && find . -type f,l -exec rm -f "$SNAPCRAFT_PRIME/{}" \; + done + +slots: + dbus-daemon: + interface: dbus + bus: session + name: org.mozilla.firefox diff --git a/taskcluster/docker/firefox-snap/policies.json b/taskcluster/docker/firefox-snap/policies.json new file mode 100644 index 0000000000..f36622021f --- /dev/null +++ b/taskcluster/docker/firefox-snap/policies.json @@ -0,0 +1,5 @@ +{ + "policies": { + "DisableAppUpdate": true + } +} diff --git a/taskcluster/docker/firefox-snap/runme.sh b/taskcluster/docker/firefox-snap/runme.sh new file mode 100755 index 0000000000..308640a8c8 --- /dev/null +++ b/taskcluster/docker/firefox-snap/runme.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +set -xe + +# Thunderbird Snap builds will set this to "thunderbird" +: PRODUCT "${PRODUCT:=firefox}" + +# Required env variables +test "$VERSION" +test "$BUILD_NUMBER" +test "$CANDIDATES_DIR" +test "$L10N_CHANGESETS" + +# Optional env variables +: WORKSPACE "${WORKSPACE:=/home/worker/workspace}" +: ARTIFACTS_DIR "${ARTIFACTS_DIR:=/home/worker/artifacts}" +: PUSH_TO_CHANNEL "" + +SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +TARGET="target.snap" +TARGET_FULL_PATH="$ARTIFACTS_DIR/$TARGET" +SOURCE_DEST="${WORKSPACE}/source" + +mkdir -p "$ARTIFACTS_DIR" +rm -rf "$SOURCE_DEST" && mkdir -p "$SOURCE_DEST" + +CURL="curl --location --retry 10 --retry-delay 10" + +# Download and extract en-US linux64 binary +$CURL -o "${WORKSPACE}/${PRODUCT}.tar.bz2" \ + "${CANDIDATES_DIR}/${VERSION}-candidates/build${BUILD_NUMBER}/linux-x86_64/en-US/${PRODUCT}-${VERSION}.tar.bz2" +tar -C "$SOURCE_DEST" -xf "${WORKSPACE}/${PRODUCT}.tar.bz2" --strip-components=1 + +DISTRIBUTION_DIR="$SOURCE_DEST/distribution" +if [[ "$PRODUCT" == "firefox" ]]; then + # Get Ubuntu configuration + PARTNER_CONFIG_DIR="$WORKSPACE/partner_config" + git clone https://github.com/mozilla-partners/canonical.git "$PARTNER_CONFIG_DIR" + mv "$PARTNER_CONFIG_DIR/desktop/ubuntu/distribution" "$DISTRIBUTION_DIR" +else + mkdir -p "$DISTRIBUTION_DIR" +fi + +cp -v "$SCRIPT_DIRECTORY/${PRODUCT}.desktop" "$DISTRIBUTION_DIR" + +# Add a group policy file to disable app updates, as those are handled by snapd +cp -v "$SCRIPT_DIRECTORY/policies.json" "$DISTRIBUTION_DIR" + +# Use list of locales to fetch L10N XPIs +$CURL -o "${WORKSPACE}/l10n_changesets.json" "$L10N_CHANGESETS" +locales=$(python3 "$SCRIPT_DIRECTORY/extract_locales_from_l10n_json.py" "${WORKSPACE}/l10n_changesets.json") + +mkdir -p "$DISTRIBUTION_DIR/extensions" +for locale in $locales; do + $CURL -o "$SOURCE_DEST/distribution/extensions/langpack-${locale}@${PRODUCT}.mozilla.org.xpi" \ + "$CANDIDATES_DIR/${VERSION}-candidates/build${BUILD_NUMBER}/linux-x86_64/xpi/${locale}.xpi" +done + +# In addition to the packages downloaded below, snapcraft fetches deb packages from ubuntu.com, +# when a snap is built,. They may bump packages there and remove the old ones. Updating the +# database allows snapcraft to find the latest packages. +# For more context, see 1448239 +apt-get update + +# Extract gtk30.mo from Ubuntu language packs +apt download language-pack-gnome-*-base +for i in *.deb; do + # shellcheck disable=SC2086 + dpkg-deb --fsys-tarfile $i | tar xv -C "$SOURCE_DEST" --wildcards "./usr/share/locale-langpack/*/LC_MESSAGES/gtk30.mo" || true +done + +# Add wrapper script to set TMPDIR appropriate for the snap +cp -v "$SCRIPT_DIRECTORY/tmpdir" "$SOURCE_DEST" + +# Generate snapcraft manifest +sed -e "s/@VERSION@/${VERSION}/g" -e "s/@BUILD_NUMBER@/${BUILD_NUMBER}/g" ${PRODUCT}.snapcraft.yaml.in > "${WORKSPACE}/snapcraft.yaml" +cd "${WORKSPACE}" + +# Make sure snapcraft knows we're building amd64, even though we may not be on this arch. +export SNAP_ARCH='amd64' + +snapcraft + +mv -- *.snap "$TARGET_FULL_PATH" + +cd "$ARTIFACTS_DIR" + +# Generate checksums file +size=$(stat --printf="%s" "$TARGET_FULL_PATH") +sha=$(sha512sum "$TARGET_FULL_PATH" | awk '{print $1}') +echo "$sha sha512 $size $TARGET" > "$TARGET.checksums" + +echo "Generating signing manifest" +hash=$(sha512sum "$TARGET.checksums" | awk '{print $1}') + +cat << EOF > signing_manifest.json +[{"file_to_sign": "$TARGET.checksums", "hash": "$hash"}] +EOF + +# For posterity +find . -ls +cat "$TARGET.checksums" +cat signing_manifest.json diff --git a/taskcluster/docker/firefox-snap/tmpdir b/taskcluster/docker/firefox-snap/tmpdir new file mode 100755 index 0000000000..e7e60801fa --- /dev/null +++ b/taskcluster/docker/firefox-snap/tmpdir @@ -0,0 +1,6 @@ +#!/bin/bash + +# Set TMPDIR to be under the user's default Downloads dir +export TMPDIR=$(xdg-user-dir DOWNLOAD)/firefox.tmp + +exec "$@" diff --git a/taskcluster/docker/funsize-update-generator/Dockerfile b/taskcluster/docker/funsize-update-generator/Dockerfile new file mode 100644 index 0000000000..c032e2231b --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/Dockerfile @@ -0,0 +1,52 @@ +FROM ubuntu:18.04 +MAINTAINER Simon Fraser <sfraser@mozilla.com> + +# Required software +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update -q && \ + apt-get install -yyq --no-install-recommends \ + bzip2 \ + ca-certificates \ + curl \ + gcc \ + jq \ + libdpkg-perl \ + libgetopt-simple-perl \ + liblzma-dev \ + locales \ + python3.8 \ + libpython3.8-dev \ + python3-dev \ + xz-utils +RUN useradd -d /home/worker -s /bin/bash -m worker +COPY requirements.txt / + +RUN locale-gen en_CA.UTF-8 +ENV LANG en_CA.UTF-8 +ENV LANGUAGE en_CA.UTF-8 +ENV LANG_ALL en_CA.UTF-8 +ENV LC_ALL en_CA.UTF-8 + +# python-pip installs a lot of dependencies increasing the size of an image +# drastically. Install it like this saves us almost 200M. +RUN bash -c "curl -L https://bootstrap.pypa.io/get-pip.py | python3.8" + +RUN ["pip", "install", "-r", "/requirements.txt"] + +# scripts +RUN mkdir /home/worker/bin +COPY scripts/* /home/worker/bin/ + +COPY runme.sh /runme.sh +RUN chmod 755 /home/worker/bin/* /*.sh +RUN mkdir /home/worker/keys +COPY *.pubkey /home/worker/keys/ + +ENV HOME /home/worker +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME worker + +USER worker + +CMD ["/runme.sh"] diff --git a/taskcluster/docker/funsize-update-generator/Makefile b/taskcluster/docker/funsize-update-generator/Makefile new file mode 100644 index 0000000000..6b67f0ed90 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/Makefile @@ -0,0 +1,9 @@ +IMAGE_NAME = funsize-update-generator + +build: + docker build -t $(IMAGE_NAME) --no-cache --rm . + +update_pubkeys: + curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/nightly_aurora_level3_primary.der | openssl x509 -inform DER -pubkey -noout > nightly.pubkey + curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/dep1.der | openssl x509 -inform DER -pubkey -noout > dep.pubkey + curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/release_primary.der | openssl x509 -inform DER -pubkey -noout > release.pubkey diff --git a/taskcluster/docker/funsize-update-generator/README b/taskcluster/docker/funsize-update-generator/README new file mode 100644 index 0000000000..5e9507be71 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/README @@ -0,0 +1,7 @@ + +To run this locally for testing/development purposes: + +1. Find a funsize generating task ID +2. docker run -t -e TASKCLUSTER_ROOT_URL="https://firefox-ci-tc.services.mozilla.com" -e SIGNING_CERT='nightly' -e MAR_CHANNEL_ID='firefox-mozilla-central' -e TASK_ID="${TASK_ID}" -e EXTRA_PARAMS="--arch=x86_64" funsize-update-generator /runme.sh + +The TASK_ID should be a recent "partials" Task. diff --git a/taskcluster/docker/funsize-update-generator/dep1.pubkey b/taskcluster/docker/funsize-update-generator/dep1.pubkey new file mode 100644 index 0000000000..927b2cc947 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/dep1.pubkey @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA8Y6AS+xwKoXZl0X5qOKr +0I00xC4UN+IMjA1LIQoZ2GBkiqQF3q8v2nWTFE0+47+3NtP0l8tvsQY+LSYR4Fek +v2Vx4m/CAMKmWzW6Vtlj80y6rQ04V19l41bZXvCIBW5fm9sAvPgc7CngkcLySNqk +8vf57cUEpOmbsjSOCmK0j8hh03I1eWogpbAVEchSm1xN2sUJaVTvz5j8BfE6Vm0i +nN7V0zF+AOxzvntZIpfUqMZbHRiMkGn4l9rjia1Rz0qUc9RNCJkNocyKtQ2N2wnN +FjHpmK9x2V71cS1JQGhgLegrswPCAWY1lTmiLk9LweqGoVL0rqR4LCkb0VCaeSRe +6bUEYcU1ZQedE80zGKB3AfoC5br1shYY0xjmyRSCQ8m8WE60HzXhL8wczKrn5yoJ +iF6BxFwcYsvrWBPgIYVZLcqjODfR/M62o8yIfTC7yBcIdycJ0sWhB47dHAFxv1kc +wv8Ik9ftvDyupE8kwcl58fNOXz93j7IxMry/ey27NyYpESPOUNcjT8TP26FdGebg +4iJx0/LaYmaNUdchfBBlaYqGdH6ZGK0OeVxzHstGuG0gebm/igYcpaFxiQzvWijX +MIAU56s4g+yj7pSzT5/s9r8Gv+YhsNHKm4hnwLZaITV0lLMT5h/OZGseQTPMBnAR +hK3CIfcqG0I23hdwI29ZuUMCAwEAAQ== +-----END PUBLIC KEY----- diff --git a/taskcluster/docker/funsize-update-generator/nightly.pubkey b/taskcluster/docker/funsize-update-generator/nightly.pubkey new file mode 100644 index 0000000000..e51049844c --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/nightly.pubkey @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAth151NGY8PBzn0bii9Yc +AjYHZDwP9Lj1c3owG0zLqW2kPcdp86QTAcoYunHGYFFakNG3tooZhzwkMjZ1OrXc +ERjD6AuVSGIBdsKtKP4vLtMjDUteFN4K2+rveozcnYFZuTWEajGu8uoYsv4QgdEA +nTBC39j0J33xlfUR+XKuxzhxNrFX+fRFWuLDJrPziMcVA/mzf0gXlhtEsfV0HYyg +yWpHdIWww+llysD1QOQAHk94Ss8c/4BFXFxlwlLeNlB1ZqLm1LsNy0jUy9EHeO3C +H6eqmiFEbpdjlrkJdgR1NcTzeY/Qf/nhWH6BAZrSapQycF7OSLU+rFWMQUElSPLc +NVl7oNAAfSYLTvRjPGi+mJK3wGFQw1EpwQl+elE1oj4+sHvIVpDrLb6btpxfr1cZ +pR4Di/hkOIymxEDWvtUhOxUXnYbDKQSDcAHKM/xR3sdIAiVtVuL4hyBwlAqkQc2j +H+SmnCbazgnq5+dN4y5DjoOgbZQ/koE3s3bUzzMeIxaul9v4gMtGROw3PQ3OZcP0 +lgjPRhY+NeTnWMo2nGb4/eS6Cn2qFLfbEQjsj6pJJBNKfvK/gm1jXb3PgXXdf8+d +2xTPOX8QNpSK7C0w4vYlvSpYZlsx2cznEOV6LDqP0QHUnmd/k1xWRRGiQ7gtT+BV +Fn0h7JyTGmEdFu6l4OhS8hMCAwEAAQ== +-----END PUBLIC KEY----- diff --git a/taskcluster/docker/funsize-update-generator/release.pubkey b/taskcluster/docker/funsize-update-generator/release.pubkey new file mode 100644 index 0000000000..ec1103d828 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/release.pubkey @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxCHbY+fP3dvaP9XVbmK6 +i4rbqo72INEWgDSYbr/DIYfCSzHC9H8pU8dyjt+Nd8OtoUZtBD1N9fP7SlrvPZSI +ZSW4k0e9Ky5aV3Uy+ivamSvYszkhqdeP2y7MBu73XHKYONR9PnKa+ovmREwSEI+h +1e0ebm8zvF7Ndwx0mOeZkDu9SDkDGg4aj2xrJyBBOuGVjuctMZ6l1davANI5xiJ0 +GBEU3tR1gJs1T4vLBis5mEFn9y4kgyw/HrxmRYGnZL4fLb2fTI+pNW0Twu3KWwwi +LgLkkVrNWiHSk7YWqxjcg5IA3pQETQ17paTHoB5Mnkvuh6MkDXvRG5VgAHZAigr6 +fJMsasOUaBeos/cD1LDQEIObpetlxc0Fiu/lvUts0755otkhI+yv35+wUa6GJrsE +CsT7c/LaFtQXg06aGXbMLDn0bE/e+nw9KWT/rE1iYXMFkzrqoTeYJ+v7/fD/ywU8 +m8l4CZmXxzd/RogMrM3xl+j4ucAAltDQyL4yLySaIT05w5U8z2zJDEXFvpFDSRfF +K3kjLwGub7wNwaQDuh/msIUdavu4g+GNikCXAJ8AssLuYatyHoltd2tf+EIIDW3U +zzLpymnLo3cAz3IPfXyqVB+mcLcpqbHjl3hWms6l1wGtz6S4WqdrWs/KfzS5EyDK +r63xn1Rg/XFmR57EsFEXAZ8CAwEAAQ== +-----END PUBLIC KEY----- diff --git a/taskcluster/docker/funsize-update-generator/requirements.in b/taskcluster/docker/funsize-update-generator/requirements.in new file mode 100644 index 0000000000..3360c71690 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/requirements.in @@ -0,0 +1,7 @@ +aiohttp +awscli +mar +redo +requests +scriptworker +sh diff --git a/taskcluster/docker/funsize-update-generator/requirements.txt b/taskcluster/docker/funsize-update-generator/requirements.txt new file mode 100644 index 0000000000..ecdafe40dc --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/requirements.txt @@ -0,0 +1,407 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --generate-hashes requirements.in +# +aiohttp==3.7.4.post0 \ + --hash=sha256:02f46fc0e3c5ac58b80d4d56eb0a7c7d97fcef69ace9326289fb9f1955e65cfe \ + --hash=sha256:0563c1b3826945eecd62186f3f5c7d31abb7391fedc893b7e2b26303b5a9f3fe \ + --hash=sha256:114b281e4d68302a324dd33abb04778e8557d88947875cbf4e842c2c01a030c5 \ + --hash=sha256:14762875b22d0055f05d12abc7f7d61d5fd4fe4642ce1a249abdf8c700bf1fd8 \ + --hash=sha256:15492a6368d985b76a2a5fdd2166cddfea5d24e69eefed4630cbaae5c81d89bd \ + --hash=sha256:17c073de315745a1510393a96e680d20af8e67e324f70b42accbd4cb3315c9fb \ + --hash=sha256:209b4a8ee987eccc91e2bd3ac36adee0e53a5970b8ac52c273f7f8fd4872c94c \ + --hash=sha256:230a8f7e24298dea47659251abc0fd8b3c4e38a664c59d4b89cca7f6c09c9e87 \ + --hash=sha256:2e19413bf84934d651344783c9f5e22dee452e251cfd220ebadbed2d9931dbf0 \ + --hash=sha256:393f389841e8f2dfc86f774ad22f00923fdee66d238af89b70ea314c4aefd290 \ + --hash=sha256:3cf75f7cdc2397ed4442594b935a11ed5569961333d49b7539ea741be2cc79d5 \ + --hash=sha256:3d78619672183be860b96ed96f533046ec97ca067fd46ac1f6a09cd9b7484287 \ + --hash=sha256:40eced07f07a9e60e825554a31f923e8d3997cfc7fb31dbc1328c70826e04cde \ + --hash=sha256:493d3299ebe5f5a7c66b9819eacdcfbbaaf1a8e84911ddffcdc48888497afecf \ + --hash=sha256:4b302b45040890cea949ad092479e01ba25911a15e648429c7c5aae9650c67a8 \ + --hash=sha256:515dfef7f869a0feb2afee66b957cc7bbe9ad0cdee45aec7fdc623f4ecd4fb16 \ + --hash=sha256:547da6cacac20666422d4882cfcd51298d45f7ccb60a04ec27424d2f36ba3eaf \ + --hash=sha256:5df68496d19f849921f05f14f31bd6ef53ad4b00245da3195048c69934521809 \ + --hash=sha256:64322071e046020e8797117b3658b9c2f80e3267daec409b350b6a7a05041213 \ + --hash=sha256:7615dab56bb07bff74bc865307aeb89a8bfd9941d2ef9d817b9436da3a0ea54f \ + --hash=sha256:79ebfc238612123a713a457d92afb4096e2148be17df6c50fb9bf7a81c2f8013 \ + --hash=sha256:7b18b97cf8ee5452fa5f4e3af95d01d84d86d32c5e2bfa260cf041749d66360b \ + --hash=sha256:932bb1ea39a54e9ea27fc9232163059a0b8855256f4052e776357ad9add6f1c9 \ + --hash=sha256:a00bb73540af068ca7390e636c01cbc4f644961896fa9363154ff43fd37af2f5 \ + --hash=sha256:a5ca29ee66f8343ed336816c553e82d6cade48a3ad702b9ffa6125d187e2dedb \ + --hash=sha256:af9aa9ef5ba1fd5b8c948bb11f44891968ab30356d65fd0cc6707d989cd521df \ + --hash=sha256:bb437315738aa441251214dad17428cafda9cdc9729499f1d6001748e1d432f4 \ + --hash=sha256:bdb230b4943891321e06fc7def63c7aace16095be7d9cf3b1e01be2f10fba439 \ + --hash=sha256:c6e9dcb4cb338d91a73f178d866d051efe7c62a7166653a91e7d9fb18274058f \ + --hash=sha256:cffe3ab27871bc3ea47df5d8f7013945712c46a3cc5a95b6bee15887f1675c22 \ + --hash=sha256:d012ad7911653a906425d8473a1465caa9f8dea7fcf07b6d870397b774ea7c0f \ + --hash=sha256:d9e13b33afd39ddeb377eff2c1c4f00544e191e1d1dee5b6c51ddee8ea6f0cf5 \ + --hash=sha256:e4b2b334e68b18ac9817d828ba44d8fcb391f6acb398bcc5062b14b2cbeac970 \ + --hash=sha256:e54962802d4b8b18b6207d4a927032826af39395a3bd9196a5af43fc4e60b009 \ + --hash=sha256:f705e12750171c0ab4ef2a3c76b9a4024a62c4103e3a55dd6f99265b9bc6fcfc \ + --hash=sha256:f881853d2643a29e643609da57b96d5f9c9b93f62429dcc1cbb413c7d07f0e1a \ + --hash=sha256:fe60131d21b31fd1a14bd43e6bb88256f69dfc3188b3a89d736d6c71ed43ec95 + # via + # -r requirements.in + # scriptworker + # taskcluster +aiomemoizettl==0.0.3 \ + --hash=sha256:07a6becac60f6cd2604b9f2b73bcd9a50079a0b7b55e2a4e45b1eec5a3ea9659 \ + --hash=sha256:0a80d2dc765e545263f515363b6700ec8cf86fa3968b529f56390b28e34f743d + # via scriptworker +arrow==1.0.3 \ + --hash=sha256:3515630f11a15c61dcb4cdd245883270dd334c83f3e639824e65a4b79cc48543 \ + --hash=sha256:399c9c8ae732270e1aa58ead835a79a40d7be8aa109c579898eb41029b5a231d + # via scriptworker +asn1crypto==1.4.0 \ + --hash=sha256:4bcdf33c861c7d40bdcd74d8e4dd7661aac320fcdf40b9a3f95b4ee12fde2fa8 \ + --hash=sha256:f4f6e119474e58e04a2b1af817eb585b4fd72bdd89b998624712b5c99be7641c + # via mar +async-timeout==3.0.1 \ + --hash=sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f \ + --hash=sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3 + # via + # aiohttp + # taskcluster +attrs==20.3.0 \ + --hash=sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6 \ + --hash=sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700 + # via + # aiohttp + # jsonschema +awscli==1.19.33 \ + --hash=sha256:473d1a653b05577440747d387778efbea8b5c05b88231dd215d74e36133c73c1 \ + --hash=sha256:d97316d781f3fb52702779a3e5ca8d5780eb253a957ee5c35542607b6014f737 + # via -r requirements.in +backports.lzma==0.0.14 \ + --hash=sha256:16d8b68e4d3cd4e6c9ddb059850452946da3914c8a8e197a7f2b0954559f2df4 + # via mar +botocore==1.20.33 \ + --hash=sha256:a33e862685259fe22d9790d9c9f3567feda8b824d44d3c62a3617af1133543a4 \ + --hash=sha256:e355305309699d3aca1e0050fc21d48595b40db046cb0d2491cd57ff5b26920b + # via + # awscli + # s3transfer +certifi==2020.12.5 \ + --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \ + --hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830 + # via requests +cffi==1.14.5 \ + --hash=sha256:005a36f41773e148deac64b08f233873a4d0c18b053d37da83f6af4d9087b813 \ + --hash=sha256:0857f0ae312d855239a55c81ef453ee8fd24136eaba8e87a2eceba644c0d4c06 \ + --hash=sha256:1071534bbbf8cbb31b498d5d9db0f274f2f7a865adca4ae429e147ba40f73dea \ + --hash=sha256:158d0d15119b4b7ff6b926536763dc0714313aa59e320ddf787502c70c4d4bee \ + --hash=sha256:1f436816fc868b098b0d63b8920de7d208c90a67212546d02f84fe78a9c26396 \ + --hash=sha256:2894f2df484ff56d717bead0a5c2abb6b9d2bf26d6960c4604d5c48bbc30ee73 \ + --hash=sha256:29314480e958fd8aab22e4a58b355b629c59bf5f2ac2492b61e3dc06d8c7a315 \ + --hash=sha256:34eff4b97f3d982fb93e2831e6750127d1355a923ebaeeb565407b3d2f8d41a1 \ + --hash=sha256:35f27e6eb43380fa080dccf676dece30bef72e4a67617ffda586641cd4508d49 \ + --hash=sha256:3d3dd4c9e559eb172ecf00a2a7517e97d1e96de2a5e610bd9b68cea3925b4892 \ + --hash=sha256:43e0b9d9e2c9e5d152946b9c5fe062c151614b262fda2e7b201204de0b99e482 \ + --hash=sha256:48e1c69bbacfc3d932221851b39d49e81567a4d4aac3b21258d9c24578280058 \ + --hash=sha256:51182f8927c5af975fece87b1b369f722c570fe169f9880764b1ee3bca8347b5 \ + --hash=sha256:58e3f59d583d413809d60779492342801d6e82fefb89c86a38e040c16883be53 \ + --hash=sha256:5de7970188bb46b7bf9858eb6890aad302577a5f6f75091fd7cdd3ef13ef3045 \ + --hash=sha256:65fa59693c62cf06e45ddbb822165394a288edce9e276647f0046e1ec26920f3 \ + --hash=sha256:69e395c24fc60aad6bb4fa7e583698ea6cc684648e1ffb7fe85e3c1ca131a7d5 \ + --hash=sha256:6c97d7350133666fbb5cf4abdc1178c812cb205dc6f41d174a7b0f18fb93337e \ + --hash=sha256:6e4714cc64f474e4d6e37cfff31a814b509a35cb17de4fb1999907575684479c \ + --hash=sha256:72d8d3ef52c208ee1c7b2e341f7d71c6fd3157138abf1a95166e6165dd5d4369 \ + --hash=sha256:8ae6299f6c68de06f136f1f9e69458eae58f1dacf10af5c17353eae03aa0d827 \ + --hash=sha256:8b198cec6c72df5289c05b05b8b0969819783f9418e0409865dac47288d2a053 \ + --hash=sha256:99cd03ae7988a93dd00bcd9d0b75e1f6c426063d6f03d2f90b89e29b25b82dfa \ + --hash=sha256:9cf8022fb8d07a97c178b02327b284521c7708d7c71a9c9c355c178ac4bbd3d4 \ + --hash=sha256:9de2e279153a443c656f2defd67769e6d1e4163952b3c622dcea5b08a6405322 \ + --hash=sha256:9e93e79c2551ff263400e1e4be085a1210e12073a31c2011dbbda14bda0c6132 \ + --hash=sha256:9ff227395193126d82e60319a673a037d5de84633f11279e336f9c0f189ecc62 \ + --hash=sha256:a465da611f6fa124963b91bf432d960a555563efe4ed1cc403ba5077b15370aa \ + --hash=sha256:ad17025d226ee5beec591b52800c11680fca3df50b8b29fe51d882576e039ee0 \ + --hash=sha256:afb29c1ba2e5a3736f1c301d9d0abe3ec8b86957d04ddfa9d7a6a42b9367e396 \ + --hash=sha256:b85eb46a81787c50650f2392b9b4ef23e1f126313b9e0e9013b35c15e4288e2e \ + --hash=sha256:bb89f306e5da99f4d922728ddcd6f7fcebb3241fc40edebcb7284d7514741991 \ + --hash=sha256:cbde590d4faaa07c72bf979734738f328d239913ba3e043b1e98fe9a39f8b2b6 \ + --hash=sha256:cd2868886d547469123fadc46eac7ea5253ea7fcb139f12e1dfc2bbd406427d1 \ + --hash=sha256:d42b11d692e11b6634f7613ad8df5d6d5f8875f5d48939520d351007b3c13406 \ + --hash=sha256:f2d45f97ab6bb54753eab54fffe75aaf3de4ff2341c9daee1987ee1837636f1d \ + --hash=sha256:fd78e5fee591709f32ef6edb9a015b4aa1a5022598e36227500c8f4e02328d9c + # via cryptography +chardet==4.0.0 \ + --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \ + --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 + # via + # aiohttp + # requests +click==7.1.2 \ + --hash=sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a \ + --hash=sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc + # via mar +colorama==0.4.3 \ + --hash=sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff \ + --hash=sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1 + # via awscli +construct==2.10.61 \ + --hash=sha256:d75384a04cb36ae5417dd34ce230c392e6085d872ab8a99ebd756c34bac0dff5 + # via mar +cryptography==3.4.6 \ + --hash=sha256:066bc53f052dfeda2f2d7c195cf16fb3e5ff13e1b6b7415b468514b40b381a5b \ + --hash=sha256:0923ba600d00718d63a3976f23cab19aef10c1765038945628cd9be047ad0336 \ + --hash=sha256:2d32223e5b0ee02943f32b19245b61a62db83a882f0e76cc564e1cec60d48f87 \ + --hash=sha256:4169a27b818de4a1860720108b55a2801f32b6ae79e7f99c00d79f2a2822eeb7 \ + --hash=sha256:57ad77d32917bc55299b16d3b996ffa42a1c73c6cfa829b14043c561288d2799 \ + --hash=sha256:5ecf2bcb34d17415e89b546dbb44e73080f747e504273e4d4987630493cded1b \ + --hash=sha256:600cf9bfe75e96d965509a4c0b2b183f74a4fa6f5331dcb40fb7b77b7c2484df \ + --hash=sha256:66b57a9ca4b3221d51b237094b0303843b914b7d5afd4349970bb26518e350b0 \ + --hash=sha256:93cfe5b7ff006de13e1e89830810ecbd014791b042cbe5eec253be11ac2b28f3 \ + --hash=sha256:9e98b452132963678e3ac6c73f7010fe53adf72209a32854d55690acac3f6724 \ + --hash=sha256:df186fcbf86dc1ce56305becb8434e4b6b7504bc724b71ad7a3239e0c9d14ef2 \ + --hash=sha256:fec7fb46b10da10d9e1d078d1ff8ed9e05ae14f431fdbd11145edd0550b9a964 + # via + # jwcrypto + # mar + # scriptworker +dictdiffer==0.8.1 \ + --hash=sha256:1adec0d67cdf6166bda96ae2934ddb5e54433998ceab63c984574d187cc563d2 \ + --hash=sha256:d79d9a39e459fe33497c858470ca0d2e93cb96621751de06d631856adfd9c390 + # via scriptworker +docutils==0.15.2 \ + --hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \ + --hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \ + --hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99 + # via awscli +github3.py==2.0.0 \ + --hash=sha256:7e153e5dc559f27e4d15c91571a2d6e1522c9faa4406e188459d9731f0b98875 \ + --hash=sha256:8dd4ac612fd60cb277eaf6e2ce02f68dda54aba06870ca6fa2b28369bf39aa14 + # via scriptworker +idna==2.10 \ + --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \ + --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 + # via + # requests + # yarl +immutabledict==1.2.0 \ + --hash=sha256:1e6bb628eb158d32caf961bed5bb283f9e57201b50f48cd5063175612f0f3efd \ + --hash=sha256:9ba015dc189af8aac5921248af18a06d76a1d026e362f917a8e9ee29e8de5133 + # via scriptworker +jmespath==0.10.0 \ + --hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9 \ + --hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f + # via botocore +json-e==4.4.1 \ + --hash=sha256:7d9f6235f855ce70418b9d6158c043c588c3c3d7d0902972f3fb7c5399997ce9 + # via scriptworker +jsonschema==3.2.0 \ + --hash=sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163 \ + --hash=sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a + # via scriptworker +jwcrypto==0.8 \ + --hash=sha256:16e17faa4dce36551ade3a3ccb06236a61e5924ea1db163c9be9827acf935a82 \ + --hash=sha256:b7fee2635bbefdf145399392f5be26ad54161c8271c66b5fe107b4b452f06c24 + # via github3.py +mar==3.1.0 \ + --hash=sha256:1939df482f2d3f6221405da00d6286d77d2dd60d372a0fd37532a8f00544f64f \ + --hash=sha256:5d2904a063f0da625e37515fa9eb340082e69fa1c00fdbeb82d28b7ff3c51e28 + # via -r requirements.in +mohawk==1.1.0 \ + --hash=sha256:3ed296a30453d0b724679e0fd41e4e940497f8e461a9a9c3b7f36e43bab0fa09 \ + --hash=sha256:d2a0e3ab10a209cc79e95e28f2dd54bd4a73fd1998ffe27b7ba0f962b6be9723 + # via taskcluster +multidict==5.1.0 \ + --hash=sha256:018132dbd8688c7a69ad89c4a3f39ea2f9f33302ebe567a879da8f4ca73f0d0a \ + --hash=sha256:051012ccee979b2b06be928a6150d237aec75dd6bf2d1eeeb190baf2b05abc93 \ + --hash=sha256:05c20b68e512166fddba59a918773ba002fdd77800cad9f55b59790030bab632 \ + --hash=sha256:07b42215124aedecc6083f1ce6b7e5ec5b50047afa701f3442054373a6deb656 \ + --hash=sha256:0e3c84e6c67eba89c2dbcee08504ba8644ab4284863452450520dad8f1e89b79 \ + --hash=sha256:0e929169f9c090dae0646a011c8b058e5e5fb391466016b39d21745b48817fd7 \ + --hash=sha256:1ab820665e67373de5802acae069a6a05567ae234ddb129f31d290fc3d1aa56d \ + --hash=sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5 \ + --hash=sha256:2e68965192c4ea61fff1b81c14ff712fc7dc15d2bd120602e4a3494ea6584224 \ + --hash=sha256:2f1a132f1c88724674271d636e6b7351477c27722f2ed789f719f9e3545a3d26 \ + --hash=sha256:37e5438e1c78931df5d3c0c78ae049092877e5e9c02dd1ff5abb9cf27a5914ea \ + --hash=sha256:3a041b76d13706b7fff23b9fc83117c7b8fe8d5fe9e6be45eee72b9baa75f348 \ + --hash=sha256:3a4f32116f8f72ecf2a29dabfb27b23ab7cdc0ba807e8459e59a93a9be9506f6 \ + --hash=sha256:46c73e09ad374a6d876c599f2328161bcd95e280f84d2060cf57991dec5cfe76 \ + --hash=sha256:46dd362c2f045095c920162e9307de5ffd0a1bfbba0a6e990b344366f55a30c1 \ + --hash=sha256:4b186eb7d6ae7c06eb4392411189469e6a820da81447f46c0072a41c748ab73f \ + --hash=sha256:54fd1e83a184e19c598d5e70ba508196fd0bbdd676ce159feb412a4a6664f952 \ + --hash=sha256:585fd452dd7782130d112f7ddf3473ffdd521414674c33876187e101b588738a \ + --hash=sha256:5cf3443199b83ed9e955f511b5b241fd3ae004e3cb81c58ec10f4fe47c7dce37 \ + --hash=sha256:6a4d5ce640e37b0efcc8441caeea8f43a06addace2335bd11151bc02d2ee31f9 \ + --hash=sha256:7df80d07818b385f3129180369079bd6934cf70469f99daaebfac89dca288359 \ + --hash=sha256:806068d4f86cb06af37cd65821554f98240a19ce646d3cd24e1c33587f313eb8 \ + --hash=sha256:830f57206cc96ed0ccf68304141fec9481a096c4d2e2831f311bde1c404401da \ + --hash=sha256:929006d3c2d923788ba153ad0de8ed2e5ed39fdbe8e7be21e2f22ed06c6783d3 \ + --hash=sha256:9436dc58c123f07b230383083855593550c4d301d2532045a17ccf6eca505f6d \ + --hash=sha256:9dd6e9b1a913d096ac95d0399bd737e00f2af1e1594a787e00f7975778c8b2bf \ + --hash=sha256:ace010325c787c378afd7f7c1ac66b26313b3344628652eacd149bdd23c68841 \ + --hash=sha256:b47a43177a5e65b771b80db71e7be76c0ba23cc8aa73eeeb089ed5219cdbe27d \ + --hash=sha256:b797515be8743b771aa868f83563f789bbd4b236659ba52243b735d80b29ed93 \ + --hash=sha256:b7993704f1a4b204e71debe6095150d43b2ee6150fa4f44d6d966ec356a8d61f \ + --hash=sha256:d5c65bdf4484872c4af3150aeebe101ba560dcfb34488d9a8ff8dbcd21079647 \ + --hash=sha256:d81eddcb12d608cc08081fa88d046c78afb1bf8107e6feab5d43503fea74a635 \ + --hash=sha256:dc862056f76443a0db4509116c5cd480fe1b6a2d45512a653f9a855cc0517456 \ + --hash=sha256:ecc771ab628ea281517e24fd2c52e8f31c41e66652d07599ad8818abaad38cda \ + --hash=sha256:f200755768dc19c6f4e2b672421e0ebb3dd54c38d5a4f262b872d8cfcc9e93b5 \ + --hash=sha256:f21756997ad8ef815d8ef3d34edd98804ab5ea337feedcd62fb52d22bf531281 \ + --hash=sha256:fc13a9524bc18b6fb6e0dbec3533ba0496bbed167c56d0aabefd965584557d80 + # via + # aiohttp + # yarl +pyasn1==0.4.8 \ + --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \ + --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba + # via rsa +pycparser==2.20 \ + --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \ + --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 + # via cffi +pyrsistent==0.17.3 \ + --hash=sha256:2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e + # via jsonschema +python-dateutil==2.8.1 \ + --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \ + --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a + # via + # arrow + # botocore + # github3.py +pyyaml==5.4.1 \ + --hash=sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf \ + --hash=sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696 \ + --hash=sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393 \ + --hash=sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77 \ + --hash=sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922 \ + --hash=sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5 \ + --hash=sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8 \ + --hash=sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10 \ + --hash=sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc \ + --hash=sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018 \ + --hash=sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e \ + --hash=sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253 \ + --hash=sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347 \ + --hash=sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183 \ + --hash=sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541 \ + --hash=sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb \ + --hash=sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185 \ + --hash=sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc \ + --hash=sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db \ + --hash=sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa \ + --hash=sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46 \ + --hash=sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122 \ + --hash=sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b \ + --hash=sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63 \ + --hash=sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df \ + --hash=sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc \ + --hash=sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247 \ + --hash=sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6 \ + --hash=sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0 + # via + # awscli + # scriptworker +redo==2.0.4 \ + --hash=sha256:81066955041c853b0e6491eb65a0877dce45131c4cfa3d42d923fc2aa8f7a043 \ + --hash=sha256:c76e4c23ab2f8840261736a851323cd98493710e7a9d36a1058535dca501f293 + # via -r requirements.in +requests==2.25.1 \ + --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \ + --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e + # via + # -r requirements.in + # github3.py + # taskcluster +rsa==4.5 \ + --hash=sha256:35c5b5f6675ac02120036d97cf96f1fde4d49670543db2822ba5015e21a18032 \ + --hash=sha256:4d409f5a7d78530a4a2062574c7bd80311bc3af29b364e293aa9b03eea77714f + # via awscli +s3transfer==0.3.6 \ + --hash=sha256:5d48b1fd2232141a9d5fb279709117aaba506cacea7f86f11bc392f06bfa8fc2 \ + --hash=sha256:c5dadf598762899d8cfaecf68eba649cd25b0ce93b6c954b156aaa3eed160547 + # via awscli +scriptworker==37.0.2 \ + --hash=sha256:306e5864d95649ab89056274402f5d933ed525e3b98794beb10eb8994edfd947 \ + --hash=sha256:bb264565a97e84e1f09ce4edc1b453ab9e10a385f2fea10add99ba9b51cb92df + # via -r requirements.in +sh==1.14.1 \ + --hash=sha256:39aa9af22f6558a0c5d132881cf43e34828ca03e4ae11114852ca6a55c7c1d8e \ + --hash=sha256:75e86a836f47de095d4531718fe8489e6f7446c75ddfa5596f632727b919ffae + # via -r requirements.in +six==1.15.0 \ + --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ + --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced + # via + # jsonschema + # mohawk + # python-dateutil + # taskcluster +slugid==2.0.0 \ + --hash=sha256:a950d98b72691178bdd4d6c52743c4a2aa039207cf7a97d71060a111ff9ba297 \ + --hash=sha256:aec8b0e01c4ad32e38e12d609eab3ec912fd129aaf6b2ded0199b56a5f8fd67c + # via taskcluster +taskcluster-urls==13.0.1 \ + --hash=sha256:5e25e7e6818e8877178b175ff43d2e6548afad72694aa125f404a7329ece0973 \ + --hash=sha256:b25e122ecec249c4299ac7b20b08db76e3e2025bdaeb699a9d444556de5fd367 \ + --hash=sha256:f66dcbd6572a6216ab65949f0fa0b91f2df647918028436c384e6af5cd12ae2b + # via taskcluster +taskcluster==39.2.0 \ + --hash=sha256:07dec0b12c80122498d772a242f92373d65731f133fb0d9aba5c2adf78475b13 \ + --hash=sha256:844d58207f7ccba73cce2413ff79ad83ef39e4095e0269fb82ff14fa9b1c3e2f \ + --hash=sha256:e5fe89c1f85b3caf141106053c1e98bcb930a629889e67acb2bd9aa92bd95182 + # via scriptworker +typing-extensions==3.7.4.3 \ + --hash=sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918 \ + --hash=sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c \ + --hash=sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f + # via aiohttp +uritemplate==3.0.1 \ + --hash=sha256:07620c3f3f8eed1f12600845892b0e036a2420acf513c53f7de0abd911a5894f \ + --hash=sha256:5af8ad10cec94f215e3f48112de2022e1d5a37ed427fbd88652fa908f2ab7cae + # via github3.py +urllib3==1.26.4 \ + --hash=sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df \ + --hash=sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937 + # via + # botocore + # requests +yarl==1.6.3 \ + --hash=sha256:00d7ad91b6583602eb9c1d085a2cf281ada267e9a197e8b7cae487dadbfa293e \ + --hash=sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434 \ + --hash=sha256:15263c3b0b47968c1d90daa89f21fcc889bb4b1aac5555580d74565de6836366 \ + --hash=sha256:2ce4c621d21326a4a5500c25031e102af589edb50c09b321049e388b3934eec3 \ + --hash=sha256:31ede6e8c4329fb81c86706ba8f6bf661a924b53ba191b27aa5fcee5714d18ec \ + --hash=sha256:324ba3d3c6fee56e2e0b0d09bf5c73824b9f08234339d2b788af65e60040c959 \ + --hash=sha256:329412812ecfc94a57cd37c9d547579510a9e83c516bc069470db5f75684629e \ + --hash=sha256:4736eaee5626db8d9cda9eb5282028cc834e2aeb194e0d8b50217d707e98bb5c \ + --hash=sha256:4953fb0b4fdb7e08b2f3b3be80a00d28c5c8a2056bb066169de00e6501b986b6 \ + --hash=sha256:4c5bcfc3ed226bf6419f7a33982fb4b8ec2e45785a0561eb99274ebbf09fdd6a \ + --hash=sha256:547f7665ad50fa8563150ed079f8e805e63dd85def6674c97efd78eed6c224a6 \ + --hash=sha256:5b883e458058f8d6099e4420f0cc2567989032b5f34b271c0827de9f1079a424 \ + --hash=sha256:63f90b20ca654b3ecc7a8d62c03ffa46999595f0167d6450fa8383bab252987e \ + --hash=sha256:68dc568889b1c13f1e4745c96b931cc94fdd0defe92a72c2b8ce01091b22e35f \ + --hash=sha256:69ee97c71fee1f63d04c945f56d5d726483c4762845400a6795a3b75d56b6c50 \ + --hash=sha256:6d6283d8e0631b617edf0fd726353cb76630b83a089a40933043894e7f6721e2 \ + --hash=sha256:72a660bdd24497e3e84f5519e57a9ee9220b6f3ac4d45056961bf22838ce20cc \ + --hash=sha256:73494d5b71099ae8cb8754f1df131c11d433b387efab7b51849e7e1e851f07a4 \ + --hash=sha256:7356644cbed76119d0b6bd32ffba704d30d747e0c217109d7979a7bc36c4d970 \ + --hash=sha256:8a9066529240171b68893d60dca86a763eae2139dd42f42106b03cf4b426bf10 \ + --hash=sha256:8aa3decd5e0e852dc68335abf5478a518b41bf2ab2f330fe44916399efedfae0 \ + --hash=sha256:97b5bdc450d63c3ba30a127d018b866ea94e65655efaf889ebeabc20f7d12406 \ + --hash=sha256:9ede61b0854e267fd565e7527e2f2eb3ef8858b301319be0604177690e1a3896 \ + --hash=sha256:b2e9a456c121e26d13c29251f8267541bd75e6a1ccf9e859179701c36a078643 \ + --hash=sha256:b5dfc9a40c198334f4f3f55880ecf910adebdcb2a0b9a9c23c9345faa9185721 \ + --hash=sha256:bafb450deef6861815ed579c7a6113a879a6ef58aed4c3a4be54400ae8871478 \ + --hash=sha256:c49ff66d479d38ab863c50f7bb27dee97c6627c5fe60697de15529da9c3de724 \ + --hash=sha256:ce3beb46a72d9f2190f9e1027886bfc513702d748047b548b05dab7dfb584d2e \ + --hash=sha256:d26608cf178efb8faa5ff0f2d2e77c208f471c5a3709e577a7b3fd0445703ac8 \ + --hash=sha256:d597767fcd2c3dc49d6eea360c458b65643d1e4dbed91361cf5e36e53c1f8c96 \ + --hash=sha256:d5c32c82990e4ac4d8150fd7652b972216b204de4e83a122546dce571c1bdf25 \ + --hash=sha256:d8d07d102f17b68966e2de0e07bfd6e139c7c02ef06d3a0f8d2f0f055e13bb76 \ + --hash=sha256:e46fba844f4895b36f4c398c5af062a9808d1f26b2999c58909517384d5deda2 \ + --hash=sha256:e6b5460dc5ad42ad2b36cca524491dfcaffbfd9c8df50508bddc354e787b8dc2 \ + --hash=sha256:f040bcc6725c821a4c0665f3aa96a4d0805a7aaf2caf266d256b8ed71b9f041c \ + --hash=sha256:f0b059678fd549c66b89bed03efcabb009075bd131c248ecdf087bdb6faba24a \ + --hash=sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71 + # via aiohttp + +# WARNING: The following packages were not pinned, but pip requires them to be +# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag. +# setuptools diff --git a/taskcluster/docker/funsize-update-generator/runme.sh b/taskcluster/docker/funsize-update-generator/runme.sh new file mode 100644 index 0000000000..99d80c09e2 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/runme.sh @@ -0,0 +1,61 @@ +#!/bin/sh + +set -xe + +test "$TASK_ID" +test "$SIGNING_CERT" + +ARTIFACTS_DIR="/home/worker/artifacts" +mkdir -p "$ARTIFACTS_DIR" + +# Strip trailing / if present +TASKCLUSTER_ROOT_URL="${TASKCLUSTER_ROOT_URL%/}" +export TASKCLUSTER_ROOT_URL + +# duplicate the functionality of taskcluster-lib-urls, but in bash.. +queue_base="${TASKCLUSTER_ROOT_URL%/}/api/queue/v1" + +curl --location --retry 10 --retry-delay 10 -o /home/worker/task.json "$queue_base/task/$TASK_ID" + +# auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/ +# -> bucket of tc-gp-private-1d-us-east-1, path of releng/mbsdiff-cache/ +# Trailing slash is important, due to prefix permissions in S3. +S3_BUCKET_AND_PATH=$(jq -r '.scopes[] | select(contains ("auth:aws-s3"))' /home/worker/task.json | awk -F: '{print $4}') + +# Will be empty if there's no scope for AWS S3. +if [ -n "${S3_BUCKET_AND_PATH}" ] && getent hosts taskcluster +then + # Does this parse as we expect? + S3_PATH=${S3_BUCKET_AND_PATH#*/} + AWS_BUCKET_NAME=${S3_BUCKET_AND_PATH%/${S3_PATH}*} + test "${S3_PATH}" + test "${AWS_BUCKET_NAME}" + + set +x # Don't echo these. + secret_url="${TASKCLUSTER_PROXY_URL}/api/auth/v1/aws/s3/read-write/${AWS_BUCKET_NAME}/${S3_PATH}" + AUTH=$(curl "${secret_url}") + AWS_ACCESS_KEY_ID=$(echo "${AUTH}" | jq -r '.credentials.accessKeyId') + AWS_SECRET_ACCESS_KEY=$(echo "${AUTH}" | jq -r '.credentials.secretAccessKey') + AWS_SESSION_TOKEN=$(echo "${AUTH}" | jq -r '.credentials.sessionToken') + export AWS_ACCESS_KEY_ID + export AWS_SECRET_ACCESS_KEY + export AWS_SESSION_TOKEN + AUTH= + + if [ -n "$AWS_ACCESS_KEY_ID" ] && [ -n "$AWS_SECRET_ACCESS_KEY" ]; then + # Pass the full bucket/path prefix, as the script just appends local files. + export MBSDIFF_HOOK="/home/worker/bin/mbsdiff_hook.sh -S ${S3_BUCKET_AND_PATH}" + fi + set -x +else + # disable caching + export MBSDIFF_HOOK= +fi + +# EXTRA_PARAMS is optional +# shellcheck disable=SC2086 +python3.8 /home/worker/bin/funsize.py \ + --artifacts-dir "$ARTIFACTS_DIR" \ + --task-definition /home/worker/task.json \ + --signing-cert "/home/worker/keys/${SIGNING_CERT}.pubkey" \ + $EXTRA_PARAMS diff --git a/taskcluster/docker/funsize-update-generator/scripts/funsize.py b/taskcluster/docker/funsize-update-generator/scripts/funsize.py new file mode 100644 index 0000000000..84fd2fbd0b --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/scripts/funsize.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import asyncio +import configparser +import json +import logging +import os +import shutil +import tempfile +import time +from contextlib import AsyncExitStack +from distutils.util import strtobool +from pathlib import Path + +import aiohttp +from mardor.reader import MarReader +from mardor.signing import get_keysize +from scriptworker.utils import get_hash, retry_async + +log = logging.getLogger(__name__) + + +ROOT_URL = os.environ.get( + "TASKCLUSTER_ROOT_URL", "https://firefox-ci-tc.services.mozilla.com" +) +QUEUE_PREFIX = f"{ROOT_URL}/api/queue/" +ALLOWED_URL_PREFIXES = ( + "http://download.cdn.mozilla.net/pub/mozilla.org/firefox/nightly/", + "http://download.cdn.mozilla.net/pub/firefox/nightly/", + "http://ftp.mozilla.org/", + "http://download.mozilla.org/", + "https://archive.mozilla.org/", + "http://archive.mozilla.org/", + QUEUE_PREFIX, +) +STAGING_URL_PREFIXES = ( + "http://ftp.stage.mozaws.net/", + "https://ftp.stage.mozaws.net/", +) + +BCJ_OPTIONS = { + "x86": ["--x86"], + "x86_64": ["--x86"], + "aarch64": [], + # macOS Universal Builds + "macos-x86_64-aarch64": [], +} + + +def verify_signature(mar, cert): + log.info("Checking %s signature", mar) + with open(mar, "rb") as mar_fh: + m = MarReader(mar_fh) + if not m.verify(verify_key=cert): + raise ValueError( + "MAR Signature invalid: %s (%s) against %s", mar, m.signature_type, cert + ) + + +def process_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument("--artifacts-dir", required=True) + parser.add_argument("--signing-cert", type=argparse.FileType("rb"), required=True) + parser.add_argument("--task-definition", required=True, type=argparse.FileType("r")) + parser.add_argument( + "--allow-staging-prefixes", + action="store_true", + default=strtobool(os.environ.get("FUNSIZE_ALLOW_STAGING_PREFIXES", "false")), + help="Allow files from staging buckets.", + ) + parser.add_argument( + "-q", + "--quiet", + dest="log_level", + action="store_const", + const=logging.INFO, + default=logging.DEBUG, + ) + parser.add_argument( + "--arch", + type=str, + required=True, + choices=BCJ_OPTIONS.keys(), + help="The archtecture you are building.", + ) + return parser.parse_args() + + +def validate_mar_channel_id(mar, channel_ids): + log.info("Checking %s for MAR_CHANNEL_ID %s", mar, channel_ids) + # We may get a string with a list representation, or a single entry string. + channel_ids = set(channel_ids.split(",")) + + product_info = MarReader(open(mar, "rb")).productinfo + if not isinstance(product_info, tuple): + raise ValueError( + "Malformed product information in mar: {}".format(product_info) + ) + + found_channel_ids = set(product_info[1].split(",")) + + if not found_channel_ids.issubset(channel_ids): + raise ValueError( + "MAR_CHANNEL_ID mismatch, {} not in {}".format(product_info[1], channel_ids) + ) + + log.info("%s channel %s in %s", mar, product_info[1], channel_ids) + + +async def retry_download(*args, semaphore=None, **kwargs): # noqa: E999 + """Retry download() calls.""" + async with AsyncExitStack() as stack: + if semaphore: + await stack.enter_async_context(semaphore) + await retry_async( + download, + retry_exceptions=(aiohttp.ClientError, asyncio.TimeoutError), + args=args, + kwargs=kwargs, + ) + + +def verify_allowed_url(mar, allowed_url_prefixes): + if not any(mar.startswith(prefix) for prefix in allowed_url_prefixes): + raise ValueError( + "{mar} is not in allowed URL prefixes: {p}".format( + mar=mar, p=allowed_url_prefixes + ) + ) + + +async def download(url, dest, mode=None): # noqa: E999 + log.info("Downloading %s to %s", url, dest) + chunk_size = 4096 + bytes_downloaded = 0 + async with aiohttp.ClientSession(raise_for_status=True) as session: + start = time.time() + async with session.get(url, timeout=120) as resp: + # Additional early logging for download timeouts. + log.debug("Fetching from url %s", resp.url) + for history in resp.history: + log.debug("Redirection history: %s", history.url) + log.debug("Headers for %s: %s", resp.url, resp.headers) + if "Content-Length" in resp.headers: + log.debug( + "Content-Length expected for %s: %s", + url, + resp.headers["Content-Length"], + ) + log_interval = chunk_size * 1024 + with open(dest, "wb") as fd: + while True: + chunk = await resp.content.read(chunk_size) + if not chunk: + break + fd.write(chunk) + bytes_downloaded += len(chunk) + log_interval -= len(chunk) + if log_interval <= 0: + log.debug("Bytes downloaded for %s: %d", url, bytes_downloaded) + log_interval = chunk_size * 1024 + end = time.time() + log.info( + "Downloaded %s, %s bytes in %s seconds: sha256:%s", + url, + bytes_downloaded, + int(end - start), + get_hash(dest, hash_alg="sha256"), + ) + if mode: + log.info("chmod %o %s", mode, dest) + os.chmod(dest, mode) + + +async def download_buildsystem_bits(partials_config, downloads, tools_dir): + """Download external tools needed to make partials.""" + + # We're making the assumption that the "to" mar is the same for all, + # as that's the way this task is currently used. + to_url = extract_download_urls(partials_config, mar_type="to").pop() + + repo = get_option( + downloads[to_url]["extracted_path"], + filename="platform.ini", + section="Build", + option="SourceRepository", + ) + revision = get_option( + downloads[to_url]["extracted_path"], + filename="platform.ini", + section="Build", + option="SourceStamp", + ) + + urls = { + "make_incremental_update.sh": f"{repo}/raw-file/{revision}/tools/" + "update-packaging/make_incremental_update.sh", + "common.sh": f"{repo}/raw-file/{revision}/tools/update-packaging/common.sh", + "mar": "https://archive.mozilla.org/pub/mozilla.org/firefox/nightly/" + "latest-mozilla-central/mar-tools/linux64/mar", + "mbsdiff": "https://archive.mozilla.org/pub/mozilla.org/firefox/nightly/" + "latest-mozilla-central/mar-tools/linux64/mbsdiff", + } + for filename, url in urls.items(): + filename = tools_dir / filename + await retry_download(url, dest=filename, mode=0o755) + + +def find_file(directory, filename): + log.debug("Searching for %s in %s", filename, directory) + return next(Path(directory).rglob(filename)) + + +def get_option(directory, filename, section, option): + log.info("Extracting [%s]: %s from %s/**/%s", section, option, directory, filename) + f = find_file(directory, filename) + config = configparser.ConfigParser() + config.read(f) + rv = config.get(section, option) + log.info("Found %s", rv) + return rv + + +def extract_download_urls(partials_config, mar_type): + """Extract a set of urls to download from the task configuration. + + mar_type should be one of "from", "to" + """ + return {definition[f"{mar_type}_mar"] for definition in partials_config} + + +async def download_and_verify_mars(partials_config, allowed_url_prefixes, signing_cert): + """Download, check signature, channel ID and unpack MAR files.""" + # Separate these categories so we can opt to perform checks on only 'to' downloads. + from_urls = extract_download_urls(partials_config, mar_type="from") + to_urls = extract_download_urls(partials_config, mar_type="to") + tasks = list() + downloads = dict() + + semaphore = asyncio.Semaphore(2) # Magic 2 to reduce network timeout errors. + for url in from_urls.union(to_urls): + verify_allowed_url(url, allowed_url_prefixes) + downloads[url] = { + "download_path": Path(tempfile.mkdtemp()) / Path(url).name, + } + tasks.append( + retry_download(url, downloads[url]["download_path"], semaphore=semaphore) + ) + + await asyncio.gather(*tasks) + + for url in downloads: + # Verify signature, but not from an artifact as we don't + # depend on the signing task + if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION") and not url.startswith( + QUEUE_PREFIX + ): + verify_signature(downloads[url]["download_path"], signing_cert) + + # Only validate the target channel ID, as we update from beta->release + if url in to_urls: + validate_mar_channel_id( + downloads[url]["download_path"], os.environ["MAR_CHANNEL_ID"] + ) + + downloads[url]["extracted_path"] = tempfile.mkdtemp() + with open(downloads[url]["download_path"], "rb") as mar_fh: + log.info( + "Unpacking %s into %s", + downloads[url]["download_path"], + downloads[url]["extracted_path"], + ) + m = MarReader(mar_fh) + m.extract(downloads[url]["extracted_path"]) + + return downloads + + +async def run_command(cmd, cwd="/", env=None, label=None, silent=False): + log.info("Running: %s", cmd) + if not env: + env = dict() + process = await asyncio.create_subprocess_shell( + cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=cwd, + env=env, + ) + if label: + label = "{}: ".format(label) + else: + label = "" + + async def read_output(stream, label, printcmd): + while True: + line = await stream.readline() + if line == b"": + break + printcmd("%s%s", label, line.decode("utf-8").rstrip()) + + if silent: + await process.wait() + else: + await asyncio.gather( + read_output(process.stdout, label, log.info), + read_output(process.stderr, label, log.warning), + ) + await process.wait() + + +async def generate_partial(from_dir, to_dir, dest_mar, mar_data, tools_dir, arch): + log.info("Generating partial %s", dest_mar) + env = os.environ.copy() + env["LC_ALL"] = "C" + env["MAR"] = tools_dir / "mar" + env["MBSDIFF"] = tools_dir / "mbsdiff" + if arch: + env["BCJ_OPTIONS"] = " ".join(BCJ_OPTIONS[arch]) + env["MOZ_PRODUCT_VERSION"] = mar_data["version"] + env["MAR_CHANNEL_ID"] = mar_data["MAR_CHANNEL_ID"] + env["BRANCH"] = mar_data["branch"] + + make_incremental_update = tools_dir / "make_incremental_update.sh" + cmd = f"{make_incremental_update} {dest_mar} {from_dir} {to_dir}" + + await run_command(cmd, cwd=dest_mar.parent, env=env, label=dest_mar.name) + validate_mar_channel_id(dest_mar, mar_data["MAR_CHANNEL_ID"]) + + +async def manage_partial( + partial_def, artifacts_dir, tools_dir, downloads, semaphore, arch=None +): + from_url = partial_def["from_mar"] + to_url = partial_def["to_mar"] + from_path = downloads[from_url]["extracted_path"] + to_path = downloads[to_url]["extracted_path"] + + mar_data = { + "MAR_CHANNEL_ID": os.environ["MAR_CHANNEL_ID"], + "version": get_option( + to_path, filename="application.ini", section="App", option="Version" + ), + "appName": get_option( + from_path, filename="application.ini", section="App", option="Name" + ), + # Use Gecko repo and rev from platform.ini, not application.ini + "repo": get_option( + to_path, filename="platform.ini", section="Build", option="SourceRepository" + ), + "revision": get_option( + to_path, filename="platform.ini", section="Build", option="SourceStamp" + ), + "locale": partial_def["locale"], + "from_mar": partial_def["from_mar"], + "from_size": os.path.getsize(downloads[from_url]["download_path"]), + "from_hash": get_hash(downloads[from_url]["download_path"], hash_alg="sha512"), + "from_buildid": get_option( + from_path, filename="application.ini", section="App", option="BuildID" + ), + "to_mar": partial_def["to_mar"], + "to_size": os.path.getsize(downloads[to_url]["download_path"]), + "to_hash": get_hash(downloads[to_url]["download_path"], hash_alg="sha512"), + "to_buildid": get_option( + to_path, filename="application.ini", section="App", option="BuildID" + ), + "mar": partial_def["dest_mar"], + } + # if branch not set explicitly use repo-name + mar_data["branch"] = partial_def.get("branch", Path(mar_data["repo"]).name) + + for field in ( + "update_number", + "previousVersion", + "previousBuildNumber", + "toVersion", + "toBuildNumber", + ): + if field in partial_def: + mar_data[field] = partial_def[field] + + dest_mar = Path(artifacts_dir) / mar_data["mar"] + + async with semaphore: + await generate_partial(from_path, to_path, dest_mar, mar_data, tools_dir, arch) + + mar_data["size"] = os.path.getsize(dest_mar) + mar_data["hash"] = get_hash(dest_mar, hash_alg="sha512") + return mar_data + + +async def async_main(args, signing_cert): + tasks = [] + + allowed_url_prefixes = list(ALLOWED_URL_PREFIXES) + if args.allow_staging_prefixes: + allowed_url_prefixes += STAGING_URL_PREFIXES + + task = json.load(args.task_definition) + + downloads = await download_and_verify_mars( + task["extra"]["funsize"]["partials"], allowed_url_prefixes, signing_cert + ) + + tools_dir = Path(tempfile.mkdtemp()) + await download_buildsystem_bits( + partials_config=task["extra"]["funsize"]["partials"], + downloads=downloads, + tools_dir=tools_dir, + ) + + # May want to consider os.cpu_count() if we ever run on osx/win. + # sched_getaffinity is the list of cores we can run on, not the total. + semaphore = asyncio.Semaphore(len(os.sched_getaffinity(0))) + for definition in task["extra"]["funsize"]["partials"]: + tasks.append( + asyncio.ensure_future( + retry_async( + manage_partial, + retry_exceptions=(aiohttp.ClientError, asyncio.TimeoutError), + kwargs=dict( + partial_def=definition, + artifacts_dir=args.artifacts_dir, + tools_dir=tools_dir, + arch=args.arch, + downloads=downloads, + semaphore=semaphore, + ), + ) + ) + ) + manifest = await asyncio.gather(*tasks) + + for url in downloads: + downloads[url]["download_path"].unlink() + shutil.rmtree(downloads[url]["extracted_path"]) + shutil.rmtree(tools_dir) + + return manifest + + +def main(): + args = process_arguments() + + logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") + log.setLevel(args.log_level) + + signing_cert = args.signing_cert.read() + assert get_keysize(signing_cert) == 4096 + + artifacts_dir = Path(args.artifacts_dir) + if not artifacts_dir.exists(): + artifacts_dir.mkdir() + + loop = asyncio.get_event_loop() + manifest = loop.run_until_complete(async_main(args, signing_cert)) + loop.close() + + manifest_file = artifacts_dir / "manifest.json" + with open(manifest_file, "w") as fp: + json.dump(manifest, fp, indent=2, sort_keys=True) + + log.debug("{}".format(json.dumps(manifest, indent=2, sort_keys=True))) + + +if __name__ == "__main__": + main() diff --git a/taskcluster/docker/funsize-update-generator/scripts/mbsdiff_hook.sh b/taskcluster/docker/funsize-update-generator/scripts/mbsdiff_hook.sh new file mode 100755 index 0000000000..965d938247 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/scripts/mbsdiff_hook.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# This tool contains functions that are to be used to handle/enable funsize +# Author: Mihai Tabara +# + +HOOK= +AWS_BUCKET_NAME= +LOCAL_CACHE_DIR= + +# Don't cache files smaller than this, as it's slower with S3 +# Bug 1437473 +CACHE_THRESHOLD=500000 + +S3_CACHE_HITS=0 +S3_CACHE_MISSES=0 + +getsha512(){ + openssl sha512 "${1}" | awk '{print $2}' +} + +print_usage(){ + echo "$(basename "$0") [-S S3-BUCKET-NAME] [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] PATH-FROM-URL PATH-TO-URL PATH-PATCH" + echo "Script that saves/retrieves from cache presumptive patches as args" + echo "" + echo "-A SERVER-URL - host where to send the files" + echo "-c LOCAL-CACHE-DIR-PATH local path to which patches are cached" + echo "-g pre hook - tests whether patch already in cache" + echo "-u post hook - upload patch to cache for future use" + echo "" + echo "PATH-FROM-URL : path on disk for source file" + echo "PATH-TO-URL : path on disk for destination file" + echo "PATH-PATCH : path on disk for patch between source and destination" +} + +upload_patch(){ + if [ "$(stat -c "%s" "$2")" -lt ${CACHE_THRESHOLD} ] + then + return 0 + fi + sha_from=$(getsha512 "$1") + sha_to=$(getsha512 "$2") + patch_path="$3" + patch_filename="$(basename "$3")" + + # save to local cache first + if [ -n "$LOCAL_CACHE_DIR" ]; then + local_cmd="mkdir -p "$LOCAL_CACHE_DIR/$sha_from"" + if $local_cmd >&2; then + cp -avf "${patch_path}" "$LOCAL_CACHE_DIR/$sha_from/$sha_to" + echo "${patch_path} saved on local cache." + fi + fi + + if [ -n "${AWS_BUCKET_NAME}" ]; then + BUCKET_PATH="s3://${AWS_BUCKET_NAME}${sha_from}/${sha_to}/${patch_filename}" + if aws s3 cp "${patch_path}" "${BUCKET_PATH}"; then + echo "${patch_path} saved on s://${AWS_BUCKET_NAME}" + return 0 + fi + echo "${patch_path} failed to be uploaded to s3://${AWS_BUCKET_NAME}" + return 1 + fi + return 0 +} + +get_patch(){ + # $1 and $2 are the /path/to/filename + if [ "$(stat -c "%s" "$2")" -lt ${CACHE_THRESHOLD} ] + then + return 1 + fi + sha_from=$(getsha512 "$1") + sha_to=$(getsha512 "$2") + destination_file="$3" + s3_filename="$(basename "$3")" + + # Try to retrieve from local cache first. + if [ -n "$LOCAL_CACHE_DIR" ]; then + if [ -r "$LOCAL_CACHE_DIR/$sha_from/$sha_to" ]; then + cp -avf "$LOCAL_CACHE_DIR/$sha_from/$sha_to" "$destination_file" + echo "Successful retrieved ${destination_file} from local cache." + return 0 + fi + fi + # If not in the local cache, we might find it remotely. + + if [ -n "${AWS_BUCKET_NAME}" ]; then + BUCKET_PATH="s3://${AWS_BUCKET_NAME}${sha_from}/${sha_to}/${s3_filename}" + if aws s3 ls "${BUCKET_PATH}"; then + ((S3_CACHE_HITS++)) + echo "s3 cache hit for ${s3_filename} (${S3_CACHE_HITS} total hits)" + if aws s3 cp "${BUCKET_PATH}" "${destination_file}"; then + echo "Successful retrieved ${destination_file} from s3://${AWS_BUCKET_NAME}" + return 0 + else + echo "Failed to retrieve ${destination_file} from s3://${AWS_BUCKET_NAME}" + return 1 + fi + # Not found, fall through to default error + else + ((S3_CACHE_MISSES++)) + echo "s3 cache miss for ${s3_filename} (${S3_CACHE_MISSES} total misses)" + fi + fi + return 1 +} + +OPTIND=1 + +while getopts ":S:c:gu" option; do + case $option in + S) + # This will probably be bucketname/path/prefix but we can use it either way + AWS_BUCKET_NAME="$OPTARG" + # Ensure trailing slash is there. + if [[ ! $AWS_BUCKET_NAME =~ .*/$ ]]; then + AWS_BUCKET_NAME="${AWS_BUCKET_NAME}/" + fi + ;; + c) + LOCAL_CACHE_DIR="$OPTARG" + ;; + g) + HOOK="PRE" + ;; + u) + HOOK="POST" + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + print_usage + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + print_usage + exit 1 + ;; + *) + echo "Unimplemented option: -$OPTARG" >&2 + print_usage + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ "$HOOK" == "PRE" ]; then + get_patch "$1" "$2" "$3" +elif [ "$HOOK" == "POST" ]; then + upload_patch "$1" "$2" "$3" +fi diff --git a/taskcluster/docker/gdb-test/Dockerfile b/taskcluster/docker/gdb-test/Dockerfile new file mode 100644 index 0000000000..d00504cdd1 --- /dev/null +++ b/taskcluster/docker/gdb-test/Dockerfile @@ -0,0 +1,15 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Mike Hommey <mhommey@mozilla.com> + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +ENV XZ_OPT=-T0 + +ARG TASKCLUSTER_ROOT_URL +ARG DOCKER_IMAGE_PACKAGES +RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES + +RUN apt-get update && \ + apt-get install gdb diff --git a/taskcluster/docker/github-sync/Dockerfile b/taskcluster/docker/github-sync/Dockerfile new file mode 100644 index 0000000000..bff131ad7b --- /dev/null +++ b/taskcluster/docker/github-sync/Dockerfile @@ -0,0 +1,14 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Kartikaya Gupta <kgupta@mozilla.com> + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache +VOLUME /builds/worker/.ghsync + +ADD prepare.sh /setup/prepare-docker.sh +ADD requirements.txt /setup/requirements.txt +RUN /bin/bash /setup/prepare-docker.sh /setup/requirements.txt && rm -R /setup + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/github-sync/prepare.sh b/taskcluster/docker/github-sync/prepare.sh new file mode 100755 index 0000000000..8efdec2e27 --- /dev/null +++ b/taskcluster/docker/github-sync/prepare.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +set -o errexit +set -o nounset +set -o pipefail +set -o xtrace + +test "$(whoami)" == 'root' + +# Install stuff we need +apt-get -y update +apt-get install -y \ + cmake \ + curl \ + gcc \ + git \ + g++ \ + libffi-dev \ + libgit2-dev \ + libssl-dev \ + python3 \ + python3-dev \ + python3-pip \ + python3-setuptools + +# Python packages +pip3 install -r "$1" diff --git a/taskcluster/docker/github-sync/requirements.txt b/taskcluster/docker/github-sync/requirements.txt new file mode 100644 index 0000000000..13fd42681d --- /dev/null +++ b/taskcluster/docker/github-sync/requirements.txt @@ -0,0 +1,3 @@ +requests == 2.21.0 +pygit2 == 1.11.1 +python-hglib == 2.6.1 diff --git a/taskcluster/docker/image_builder/Dockerfile b/taskcluster/docker/image_builder/Dockerfile new file mode 100644 index 0000000000..ad38a48c49 --- /dev/null +++ b/taskcluster/docker/image_builder/Dockerfile @@ -0,0 +1,99 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +FROM golang:1.14 as skopeo + +WORKDIR /go/src/ +RUN ["git", "clone", "--no-checkout", "--depth=1", "--branch=v1.1.1", "https://github.com/containers/skopeo", "."] +RUN ["git", "checkout", "67abbb3cefbdc876447583d5ea45e76bf441eba7"] +ENV GO111MODULE=on CGO_ENABLED=0 +RUN ["go", "build", \ + "-mod=vendor", "-o", "out/skopeo", \ + "-tags", "exclude_graphdriver_devicemapper exclude_graphdriver_btrfs containers_image_openpgp", \ + # Set unixTempDirForBigFiles so skopeo will extract in a directory hidden by kaniko + # We create the directory below. + "-ldflags", " -X github.com/containers/image/v5/internal/tmpdir.unixTempDirForBigFiles=/workspace/tmp -X github.com/containers/image/v5/signature.systemDefaultPolicyPath=/kaniko/containers/policy.json -extldflags \"-static\" -w -s", \ + "./cmd/skopeo"] + +FROM golang:1.14 as kaniko +WORKDIR /go/src/ +RUN ["git", "clone", "--no-checkout", "--depth=1", "--branch=v1.0.0", "https://github.com/GoogleContainerTools/kaniko", "."] +RUN ["git", "checkout", "146ec6a9cd6f87b4a12e8119ded575d5edca35ac"] +RUN ["make"] + +# Build the `build-image` command as a static binary using musl +# The setup is loosely based on a stripped down version of +# https://github.com/emk/rust-musl-builder/blob/master/Dockerfile +FROM debian:buster as build-image + +COPY apt.conf /etc/apt/apt.conf.d/99taskcluster + +RUN apt-get update && \ + apt-get install \ + build-essential \ + ca-certificates \ + curl \ + musl-dev \ + musl-tools \ + && \ + useradd rust --user-group --create-home --shell /bin/bash + +# Run all further code as user `rust`, and create our working directories +# as the appropriate user. +USER rust + +# Set up our path with all our binary directories, including those for the +# musl-gcc toolchain and for our Rust toolchain. +ENV PATH=/home/rust/.cargo/bin:$PATH + +# The Rust toolchain to use when building our image. Set by `hooks/build`. +ENV TOOLCHAIN=1.42.0 \ + TARGET=x86_64-unknown-linux-musl + +# Install our Rust toolchain and the `musl` target. We patch the +# command-line we pass to the installer so that it won't attempt to +# interact with the user or fool around with TTYs. We also set the default +# `--target` to musl so that our users don't need to keep overriding it +# manually. +RUN curl https://sh.rustup.rs -sSf | \ + sh -s -- -y \ + --profile minimal \ + --default-toolchain $TOOLCHAIN \ + --target $TARGET + +# Expect our source code to live in /home/rust/src. We'll run the build as +# user `rust`, which will be uid 1000, gid 1000 outside the container. +RUN mkdir -p /home/rust/src +WORKDIR /home/rust/src +# Add our source code. +ADD --chown=rust:rust build-image/ ./ + +# --out-dir is not yet stable +ENV RUSTC_BOOTSTRAP=1 +# Build our application. +RUN ["cargo", "build", "--target", "x86_64-unknown-linux-musl", "--out-dir=bin", "--release", "-Zunstable-options"] + +FROM scratch as empty + +FROM scratch + +COPY --from=skopeo /go/src/out/skopeo /kaniko-bootstrap/skopeo +COPY --from=kaniko /go/src/out/executor /kaniko-bootstrap/executor +COPY --from=build-image \ + /home/rust/src/bin/build-image \ + /kaniko-bootstrap/build-image + +ADD https://mkcert.org/generate/ /kaniko-bootstrap/ssl/certs/ca-certificats.crt +ENV SSL_CERT_DIR=/kaniko/ssl/certs + +ADD policy.json /kaniko-bootstrap/containers/policy.json + +ENV HOME /root +ENV USER /root +WORKDIR /workspace + +ENV PATH /usr/local/bin:/kaniko + +VOLUME /workspace +ENTRYPOINT ["/kaniko-bootstrap/build-image"] diff --git a/taskcluster/docker/image_builder/VERSION b/taskcluster/docker/image_builder/VERSION new file mode 100644 index 0000000000..fcdb2e109f --- /dev/null +++ b/taskcluster/docker/image_builder/VERSION @@ -0,0 +1 @@ +4.0.0 diff --git a/taskcluster/docker/image_builder/apt.conf b/taskcluster/docker/image_builder/apt.conf new file mode 100644 index 0000000000..84c0cf10ef --- /dev/null +++ b/taskcluster/docker/image_builder/apt.conf @@ -0,0 +1,5 @@ +quiet "true"; +APT::Get::Assume-Yes "true"; +APT::Install-Recommends "false"; +Acquire::Check-Valid-Until "false"; +Acquire::Retries "5"; diff --git a/taskcluster/docker/image_builder/build-image/Cargo.lock b/taskcluster/docker/image_builder/build-image/Cargo.lock new file mode 100644 index 0000000000..7e07920591 --- /dev/null +++ b/taskcluster/docker/image_builder/build-image/Cargo.lock @@ -0,0 +1,1085 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "anyhow" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "autocfg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "base64" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "build-image" +version = "0.1.0" +dependencies = [ + "anyhow 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)", + "envy 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "reqwest 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)", + "url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "zstd 0.5.3+zstd.1.4.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bumpalo" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bytes" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "loom 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cc" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "jobserver 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "dtoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "either" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "encoding_rs" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "envy" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fs_extra" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-channel" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-core" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-io" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-macro" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro-hack 0.5.16 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-sink" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-task" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "once_cell 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-util" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-io 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-macro 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-task 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-project 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-utils 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.5.16 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-nested 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "generator" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "h2" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "indexmap 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hermit-abi" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "http" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "http-body" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "httparse" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "hyper" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-channel 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "h2 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "http-body 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-project 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)", + "socket2 0.3.12 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "want 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hyper-rustls" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.13.6 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-rustls 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki 0.21.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "idna" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-normalization 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "indexmap" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "itoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "jobserver" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "js-sys" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "wasm-bindgen 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.71" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "log" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "loom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "generator 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", + "scoped-tls 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "memchr" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "mime 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "mio" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.34 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "miow" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.34 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "net2" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "hermit-abi 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "once_cell" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "pin-project" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "pin-project-internal 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro-hack" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro-nested" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro2" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "redox_syscall" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "reqwest" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "base64 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "encoding_rs 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "http-body 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.13.6 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper-rustls 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)", + "js-sys 0.3.41 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "mime 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", + "mime_guess 2.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-project-lite 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_urlencoded 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-rustls 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)", + "url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-futures 0.4.14 (registry+https://github.com/rust-lang/crates.io-index)", + "web-sys 0.3.41 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki-roots 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winreg 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ring" +version = "0.16.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "once_cell 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "untrusted 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "web-sys 0.3.41 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustls" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "ring 0.16.15 (registry+https://github.com/rust-lang/crates.io-index)", + "sct 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki 0.21.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "scoped-tls" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "sct" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ring 0.16.15 (registry+https://github.com/rust-lang/crates.io-index)", + "untrusted 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde" +version = "1.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde_derive 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_derive" +version = "1.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_json" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "itoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "ryu 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_urlencoded" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "dtoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", + "url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "socket2" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "syn" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "time" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tinyvec" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "tokio" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.22 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-project-lite 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-rustls" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki 0.21.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-project-lite 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tower-service" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "try-lock" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "version_check 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "tinyvec 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "url" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "version_check" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-macro 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bumpalo 3.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-shared 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "js-sys 0.3.41 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", + "web-sys 0.3.41 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-macro-support 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-backend 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-shared 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "web-sys" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "js-sys 0.3.41 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "webpki" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ring 0.16.15 (registry+https://github.com/rust-lang/crates.io-index)", + "untrusted 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "webpki-roots" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "webpki 0.21.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "zstd" +version = "0.5.3+zstd.1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "zstd-safe 2.0.5+zstd.1.4.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "zstd-safe" +version = "2.0.5+zstd.1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "zstd-sys 1.4.17+zstd.1.4.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "zstd-sys" +version = "1.4.17+zstd.1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)", + "glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[metadata] +"checksum anyhow 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)" = "85bb70cc08ec97ca5450e6eba421deeea5f172c0fc61f78b5357b2a8e8be195f" +"checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" +"checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" +"checksum base64 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" +"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +"checksum bumpalo 3.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" +"checksum bytes 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "118cf036fbb97d0816e3c34b2d7a1e8cfc60f68fcf63d550ddbe9bd5f59c213b" +"checksum cc 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)" = "77c1f1d60091c1b73e2b1f4560ab419204b178e625fa945ded7b660becd2bd46" +"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +"checksum dtoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" +"checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" +"checksum encoding_rs 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)" = "e8ac63f94732332f44fe654443c46f6375d1939684c17b0afb6cb56b0456e171" +"checksum envy 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f938a4abd5b75fe3737902dbc2e79ca142cc1526827a9e40b829a086758531a9" +"checksum fnv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +"checksum fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5f2a4a2034423744d2cc7ca2068453168dcdb82c438419e639a26bd87839c674" +"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +"checksum futures-channel 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f366ad74c28cca6ba456d95e6422883cfb4b252a83bed929c83abfdbbf2967d5" +"checksum futures-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "59f5fff90fd5d971f936ad674802482ba441b6f09ba5e15fd8b39145582ca399" +"checksum futures-io 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "de27142b013a8e869c14957e6d2edeef89e97c289e69d042ee3a49acd8b51789" +"checksum futures-macro 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d0b5a30a4328ab5473878237c447333c093297bded83a4983d10f4deea240d39" +"checksum futures-sink 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3f2032893cb734c7a05d85ce0cc8b8c4075278e93b24b66f9de99d6eb0fa8acc" +"checksum futures-task 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "bdb66b5f09e22019b1ab0830f7785bcea8e7a42148683f99214f73f8ec21a626" +"checksum futures-util 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "8764574ff08b701a084482c3c7031349104b07ac897393010494beaa18ce32c6" +"checksum generator 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "add72f17bb81521258fcc8a7a3245b1e184e916bfbe34f0ea89558f440df5c68" +"checksum glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +"checksum h2 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "79b7246d7e4b979c03fa093da39cfb3617a96bbeee6310af63991668d7e843ff" +"checksum hermit-abi 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "b9586eedd4ce6b3c498bc3b4dd92fc9f11166aa908a914071953768066c67909" +"checksum http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" +"checksum http-body 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +"checksum httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" +"checksum hyper 0.13.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a6e7655b9594024ad0ee439f3b5a7299369dc2a3f459b47c696f9ff676f9aa1f" +"checksum hyper-rustls 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac965ea399ec3a25ac7d13b8affd4b8f39325cca00858ddf5eb29b79e6b14b08" +"checksum idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +"checksum indexmap 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c398b2b113b55809ceb9ee3e753fcbac793f1956663f3c36549c1346015c2afe" +"checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +"checksum itertools 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +"checksum itoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" +"checksum jobserver 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2" +"checksum js-sys 0.3.41 (registry+https://github.com/rust-lang/crates.io-index)" = "c4b9172132a62451e56142bff9afc91c8e4a4500aa5b847da36815b63bfda916" +"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +"checksum libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)" = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" +"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +"checksum loom 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4ecc775857611e1df29abba5c41355cdf540e7e9d4acfdf0f355eefee82330b7" +"checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +"checksum memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" +"checksum mime 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)" = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +"checksum mime_guess 2.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +"checksum mio 0.6.22 (registry+https://github.com/rust-lang/crates.io-index)" = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" +"checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +"checksum net2 0.2.34 (registry+https://github.com/rust-lang/crates.io-index)" = "2ba7c918ac76704fb42afcbbb43891e72731f3dcca3bef2a19786297baf14af7" +"checksum num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +"checksum once_cell 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d" +"checksum percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +"checksum pin-project 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)" = "12e3a6cdbfe94a5e4572812a0201f8c0ed98c1c452c7b8563ce2276988ef9c17" +"checksum pin-project-internal 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)" = "6a0ffd45cf79d88737d7cc85bfd5d2894bee1139b356e616fe85dc389c61aaf7" +"checksum pin-project-lite 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "282adbf10f2698a7a77f8e983a74b2d18176c19a7fd32a45446139ae7b02b715" +"checksum pin-utils 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +"checksum proc-macro-hack 0.5.16 (registry+https://github.com/rust-lang/crates.io-index)" = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4" +"checksum proc-macro-nested 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" +"checksum proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)" = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" +"checksum quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +"checksum reqwest 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)" = "3b82c9238b305f26f53443e3a4bc8528d64b8d0bee408ec949eb7bf5635ec680" +"checksum ring 0.16.15 (registry+https://github.com/rust-lang/crates.io-index)" = "952cd6b98c85bbc30efa1ba5783b8abf12fec8b3287ffa52605b9432313e34e4" +"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +"checksum rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1" +"checksum ryu 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +"checksum scoped-tls 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "332ffa32bf586782a3efaeb58f127980944bbc8c4d6913a86107ac2a5ab24b28" +"checksum sct 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c" +"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +"checksum serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)" = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3" +"checksum serde_derive 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)" = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e" +"checksum serde_json 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)" = "3433e879a558dde8b5e8feb2a04899cf34fdde1fafb894687e52105fc1162ac3" +"checksum serde_urlencoded 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97" +"checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +"checksum socket2 0.3.12 (registry+https://github.com/rust-lang/crates.io-index)" = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918" +"checksum spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +"checksum syn 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)" = "e8d5d96e8cbb005d6959f119f773bfaebb5684296108fb32600c00cde305b2cd" +"checksum time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +"checksum tinyvec 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "53953d2d3a5ad81d9f844a32f14ebb121f50b650cd59d0ee2a07cf13c617efed" +"checksum tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)" = "d099fa27b9702bed751524694adbe393e18b36b204da91eb1cbbbbb4a5ee2d58" +"checksum tokio-rustls 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)" = "15cb62a0d2770787abc96e99c1cd98fcf17f94959f3af63ca85bdfb203f051b4" +"checksum tokio-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +"checksum tower-service 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" +"checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" +"checksum unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +"checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +"checksum unicode-normalization 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6fb19cf769fa8c6a80a162df694621ebeb4dafb606470b2b2fce0be40a98a977" +"checksum unicode-xid 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +"checksum untrusted 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +"checksum url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" +"checksum version_check 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +"checksum want 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +"checksum wasm-bindgen 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)" = "6a634620115e4a229108b71bde263bb4220c483b3f07f5ba514ee8d15064c4c2" +"checksum wasm-bindgen-backend 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)" = "3e53963b583d18a5aa3aaae4b4c1cb535218246131ba22a71f05b518098571df" +"checksum wasm-bindgen-futures 0.4.14 (registry+https://github.com/rust-lang/crates.io-index)" = "dba48d66049d2a6cc8488702e7259ab7afc9043ad0dc5448444f46f2a453b362" +"checksum wasm-bindgen-macro 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)" = "3fcfd5ef6eec85623b4c6e844293d4516470d8f19cd72d0d12246017eb9060b8" +"checksum wasm-bindgen-macro-support 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)" = "9adff9ee0e94b926ca81b57f57f86d5545cdcb1d259e21ec9bdd95b901754c75" +"checksum wasm-bindgen-shared 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)" = "7f7b90ea6c632dd06fd765d44542e234d5e63d9bb917ecd64d79778a13bd79ae" +"checksum web-sys 0.3.41 (registry+https://github.com/rust-lang/crates.io-index)" = "863539788676619aac1a23e2df3655e96b32b0e05eb72ca34ba045ad573c625d" +"checksum webpki 0.21.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ab146130f5f790d45f82aeeb09e55a256573373ec64409fc19a6fb82fb1032ae" +"checksum webpki-roots 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8eff4b7516a57307f9349c64bf34caa34b940b66fed4b2fb3136cb7386e5739" +"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" +"checksum winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +"checksum winreg 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +"checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +"checksum zstd 0.5.3+zstd.1.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "01b32eaf771efa709e8308605bbf9319bf485dc1503179ec0469b611937c0cd8" +"checksum zstd-safe 2.0.5+zstd.1.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "1cfb642e0d27f64729a639c52db457e0ae906e7bc6f5fe8f5c453230400f1055" +"checksum zstd-sys 1.4.17+zstd.1.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "b89249644df056b522696b1bb9e7c18c87e8ffa3e2f0dc3b0155875d6498f01b" diff --git a/taskcluster/docker/image_builder/build-image/Cargo.toml b/taskcluster/docker/image_builder/build-image/Cargo.toml new file mode 100644 index 0000000000..be42fe1e34 --- /dev/null +++ b/taskcluster/docker/image_builder/build-image/Cargo.toml @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +[package] +name = "build-image" +version = "0.1.0" +authors = ["Tom Prince <tom.prince@twistedmatrix.com>"] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +reqwest = { version= "0.10.4", features = ["rustls-tls", "blocking"], default-features = false} +zstd = "0.5.1" +url = "2.1.1" +anyhow = "1.0.27" +serde = { version = "1.0.105", features = ["derive"]} +serde_json = "1.0.50" +envy = "0.4.1" +fs_extra = "1.1" + +[workspace] diff --git a/taskcluster/docker/image_builder/build-image/src/config.rs b/taskcluster/docker/image_builder/build-image/src/config.rs new file mode 100644 index 0000000000..94c1d55a10 --- /dev/null +++ b/taskcluster/docker/image_builder/build-image/src/config.rs @@ -0,0 +1,112 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use anyhow::Result; +use serde::de::Error; +use serde::Deserialize; +use std::collections::HashMap; + +fn default_image_name() -> String { + "mozilla.org/taskgraph/default-image:latest".into() +} +fn default_zstd_level() -> i32 { + 3 +} + +fn from_json<'de, D, T>(deserializer: D) -> Result<T, D::Error> +where + D: serde::de::Deserializer<'de>, + T: serde::de::DeserializeOwned, +{ + let value: String = serde::Deserialize::deserialize(deserializer)?; + serde_json::from_str(&value).map_err(|err| { + D::Error::invalid_value(serde::de::Unexpected::Str(&value), &&*err.to_string()) + }) +} + +#[derive(Deserialize, Debug, PartialEq, Eq)] +pub struct Config { + pub context_task_id: String, + pub context_path: String, + pub parent_task_id: Option<String>, + #[serde(default = "default_image_name")] + pub image_name: String, + #[serde(default = "default_zstd_level")] + pub docker_image_zstd_level: i32, + #[serde(default)] + pub debug: bool, + #[serde(default, deserialize_with = "from_json")] + pub docker_build_args: HashMap<String, String>, +} + +impl Config { + pub fn from_env() -> Result<Config> { + Ok(envy::from_env()?) + } +} + +#[cfg(test)] +mod test { + use anyhow::Result; + + #[test] + fn test() -> Result<()> { + let env: Vec<(String, String)> = vec![ + ("CONTEXT_TASK_ID".into(), "xGRRgzG6QlCCwsFsyuqm0Q".into()), + ( + "CONTEXT_PATH".into(), + "public/docker-contexts/image.tar.gz".into(), + ), + ]; + let config: super::Config = envy::from_iter(env.into_iter())?; + assert_eq!( + config, + super::Config { + context_task_id: "xGRRgzG6QlCCwsFsyuqm0Q".into(), + context_path: "public/docker-contexts/image.tar.gz".into(), + parent_task_id: None, + image_name: "mozilla.org/taskgraph/default-image:latest".into(), + docker_image_zstd_level: 3, + debug: false, + docker_build_args: Default::default() + } + ); + Ok(()) + } + + #[test] + fn test_docker_build_args() -> Result<()> { + let env: Vec<(String, String)> = vec![ + ("CONTEXT_TASK_ID".into(), "xGRRgzG6QlCCwsFsyuqm0Q".into()), + ( + "CONTEXT_PATH".into(), + "public/docker-contexts/image.tar.gz".into(), + ), + ( + "DOCKER_BUILD_ARGS".into(), + serde_json::json! ({ + "test": "Value", + }) + .to_string(), + ), + ]; + let config: super::Config = envy::from_iter(env.into_iter())?; + assert_eq!( + config, + super::Config { + context_task_id: "xGRRgzG6QlCCwsFsyuqm0Q".into(), + context_path: "public/docker-contexts/image.tar.gz".into(), + parent_task_id: None, + image_name: "mozilla.org/taskgraph/default-image:latest".into(), + docker_image_zstd_level: 3, + debug: false, + docker_build_args: [("test".to_string(), "Value".to_string())] + .iter() + .cloned() + .collect(), + } + ); + Ok(()) + } +} diff --git a/taskcluster/docker/image_builder/build-image/src/main.rs b/taskcluster/docker/image_builder/build-image/src/main.rs new file mode 100644 index 0000000000..997617c84e --- /dev/null +++ b/taskcluster/docker/image_builder/build-image/src/main.rs @@ -0,0 +1,182 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#![forbid(unsafe_code)] + +use std::collections::HashMap; +use std::path::Path; +use std::process::Command; + +use anyhow::{ensure, Context, Result}; +use fs_extra::dir::{move_dir, CopyOptions}; +use serde::Deserialize; + +mod config; +mod taskcluster; + +use config::Config; + +fn log_step(msg: &str) { + println!("[build-image] {}", msg); +} + +fn read_image_digest(path: &str) -> Result<String> { + let output = Command::new("/kaniko/skopeo") + .arg("inspect") + .arg(format!("docker-archive:{}", path)) + .stdout(std::process::Stdio::piped()) + .spawn()? + .wait_with_output()?; + ensure!(output.status.success(), "Could not inspect parent image."); + + #[derive(Deserialize, Debug)] + #[serde(rename_all = "PascalCase")] + struct ImageInfo { + #[serde(skip_serializing_if = "Option::is_none")] + name: Option<String>, + #[serde(skip_serializing_if = "Option::is_none")] + tag: Option<String>, + digest: String, + // ... + } + + let image_info: ImageInfo = serde_json::from_slice(&output.stdout) + .with_context(|| format!("Could parse image info from {:?}", path))?; + Ok(image_info.digest) +} + +fn download_parent_image( + cluster: &taskcluster::TaskCluster, + task_id: &str, + dest: &str, +) -> Result<String> { + zstd::stream::copy_decode( + cluster.stream_artifact(&task_id, "public/image.tar.zst")?, + std::fs::File::create(dest)?, + ) + .context("Could not download parent image.")?; + + read_image_digest(dest) +} + +fn build_image( + context_path: &str, + dest: &str, + debug: bool, + build_args: HashMap<String, String>, +) -> Result<()> { + let mut command = Command::new("/kaniko/executor"); + command + .stderr(std::process::Stdio::inherit()) + .args(&["--context", &format!("tar://{}", context_path)]) + .args(&["--destination", "image"]) + .args(&["--dockerfile", "Dockerfile"]) + .arg("--no-push") + .args(&["--cache-dir", "/workspace/cache"]) + .arg("--single-snapshot") + // FIXME: Generating reproducible layers currently causes OOM. + // .arg("--reproducible") + .arg("--whitelist-var-run=false") + .args(&["--tarPath", dest]); + if debug { + command.args(&["-v", "debug"]); + } + for (key, value) in build_args { + command.args(&["--build-arg", &format!("{}={}", key, value)]); + } + let status = command.status()?; + ensure!(status.success(), "Could not build image."); + Ok(()) +} + +fn repack_image(source: &str, dest: &str, image_name: &str) -> Result<()> { + let status = Command::new("/kaniko/skopeo") + .arg("copy") + .arg(format!("docker-archive:{}", source)) + .arg(format!("docker-archive:{}:{}", dest, image_name)) + .stderr(std::process::Stdio::inherit()) + .status()?; + ensure!(status.success(), "Could repack image."); + Ok(()) +} + +fn main() -> Result<()> { + // Kaniko expects everything to be in /kaniko, so if not running from there, move + // everything there. + if let Some(path) = std::env::current_exe()?.parent() { + if path != Path::new("/kaniko") { + let mut options = CopyOptions::new(); + options.copy_inside = true; + move_dir(path, "/kaniko", &options)?; + } + } + + let config = Config::from_env().context("Could not parse environment variables.")?; + + let cluster = taskcluster::TaskCluster::from_env()?; + + let mut build_args = config.docker_build_args; + + build_args.insert("TASKCLUSTER_ROOT_URL".into(), cluster.root_url()); + + log_step("Downloading context."); + + std::io::copy( + &mut cluster.stream_artifact(&config.context_task_id, &config.context_path)?, + &mut std::fs::File::create("/workspace/context.tar.gz")?, + ) + .context("Could not download image context.")?; + + if let Some(parent_task_id) = config.parent_task_id { + log_step("Downloading image."); + let digest = download_parent_image(&cluster, &parent_task_id, "/workspace/parent.tar")?; + + log_step(&format!("Parent image digest {}", &digest)); + std::fs::create_dir_all("/workspace/cache")?; + std::fs::rename( + "/workspace/parent.tar", + format!("/workspace/cache/{}", digest), + )?; + + build_args.insert( + "DOCKER_IMAGE_PARENT".into(), + format!("parent:latest@{}", digest), + ); + } + + log_step("Building image."); + build_image( + "/workspace/context.tar.gz", + "/workspace/image-pre.tar", + config.debug, + build_args, + )?; + log_step("Repacking image."); + repack_image( + "/workspace/image-pre.tar", + "/workspace/image.tar", + &config.image_name, + )?; + + log_step("Compressing image."); + compress_file( + "/workspace/image.tar", + "/workspace/image.tar.zst", + config.docker_image_zstd_level, + )?; + + Ok(()) +} + +fn compress_file( + source: impl AsRef<std::path::Path>, + dest: impl AsRef<std::path::Path>, + zstd_level: i32, +) -> Result<()> { + Ok(zstd::stream::copy_encode( + std::fs::File::open(source)?, + std::fs::File::create(dest)?, + zstd_level, + )?) +} diff --git a/taskcluster/docker/image_builder/build-image/src/taskcluster.rs b/taskcluster/docker/image_builder/build-image/src/taskcluster.rs new file mode 100644 index 0000000000..3b39d669f0 --- /dev/null +++ b/taskcluster/docker/image_builder/build-image/src/taskcluster.rs @@ -0,0 +1,55 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use anyhow::{Context, Result}; + +pub struct TaskCluster { + root_url: url::Url, + client: reqwest::blocking::Client, +} + +impl TaskCluster { + pub fn from_env() -> Result<Self> { + std::env::var("TASKCLUSTER_ROOT_URL") + .context("TASKCLUSTER_ROOT_URL not set.") + .and_then(|var| var.parse().context("Couldn't parse TASKCLUSTER_ROOT_URL.")) + .map(|root_url| TaskCluster { + root_url, + client: reqwest::blocking::Client::new(), + }) + } + + /// Return the root URL as suitable for passing to other processes. + /// + /// In particular, any trailing slashes are removed. + pub fn root_url(&self) -> String { + self.root_url.as_str().trim_end_matches("/").to_string() + } + + pub fn task_artifact_url(&self, task_id: &str, path: &str) -> url::Url { + let mut url = self.root_url.clone(); + url.set_path(&format!("api/queue/v1/task/{}/artifacts/{}", task_id, path)); + url + } + + pub fn stream_artifact(&self, task_id: &str, path: &str) -> Result<impl std::io::Read> { + let url = self.task_artifact_url(task_id, path); + Ok(self.client.get(url).send()?.error_for_status()?) + } +} + +#[cfg(test)] +mod test { + #[test] + fn test_url() { + let cluster = super::TaskCluster { + root_url: url::Url::parse("http://taskcluster.example").unwrap(), + client: reqwest::blocking::Client::new(), + }; + assert_eq!( + cluster.task_artifact_url("QzDLgP4YRwanIvgPt6ClfA","public/docker-contexts/decision.tar.gz"), + url::Url::parse("http://taskcluster.example/api/queue/v1/task/QzDLgP4YRwanIvgPt6ClfA/artifacts/public/docker-contexts/decision.tar.gz").unwrap(), + ); + } +} diff --git a/taskcluster/docker/image_builder/policy.json b/taskcluster/docker/image_builder/policy.json new file mode 100644 index 0000000000..c709def1ec --- /dev/null +++ b/taskcluster/docker/image_builder/policy.json @@ -0,0 +1,11 @@ +{ + "default": [{"type": "reject"}], + "transports": { + "docker-archive": { + "": [{"type": "insecureAcceptAnything"}] + }, + "dir": { + "": [{"type": "insecureAcceptAnything"}] + } + } +} diff --git a/taskcluster/docker/index-task/.eslintrc.js b/taskcluster/docker/index-task/.eslintrc.js new file mode 100644 index 0000000000..ff84d9cb69 --- /dev/null +++ b/taskcluster/docker/index-task/.eslintrc.js @@ -0,0 +1,11 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +module.exports = { + env: { + node: true, + }, +}; diff --git a/taskcluster/docker/index-task/Dockerfile b/taskcluster/docker/index-task/Dockerfile new file mode 100644 index 0000000000..493eda4e20 --- /dev/null +++ b/taskcluster/docker/index-task/Dockerfile @@ -0,0 +1,11 @@ +FROM node:10-alpine + +ENV NODE_ENV production +RUN mkdir /app +ADD insert-indexes.js /app/ +ADD package.json /app/ +ADD yarn.lock /app/ +WORKDIR /app +RUN yarn --frozen-lockfile && yarn cache clean + +ENTRYPOINT ["node"] diff --git a/taskcluster/docker/index-task/README b/taskcluster/docker/index-task/README new file mode 100644 index 0000000000..9ec00e7897 --- /dev/null +++ b/taskcluster/docker/index-task/README @@ -0,0 +1,36 @@ +Index-Image +=========== + +This image is designed to be used for indexing other tasks. It takes a task +definition as follows: +```js +{ + ..., + scopes: [ + 'index:insert-task:my-index.namespace', + 'index:insert-task:...', + ], + payload: { + image: '...', + env: { + TARGET_TASKID: '<taskId-to-be-indexed>', + }, + command: [ + 'insert-indexes.js', + 'my-index.namespace.one', + 'my-index.namespace.two', + '....', + ], + features: { + taskclusterProxy: true, + }, + maxRunTime: 600, + }, +} +``` + +As can be seen the `taskId` to be indexed is given by the environment variable +`TARGET_TASKID` and the `command` arguments specifies namespaces that it must +be index under. It is **important** to also include scopes on the form +`index:insert-task:<...>` for all namespaces `<...>` given as `command` +arguments. diff --git a/taskcluster/docker/index-task/insert-indexes.js b/taskcluster/docker/index-task/insert-indexes.js new file mode 100644 index 0000000000..39ff45fd9c --- /dev/null +++ b/taskcluster/docker/index-task/insert-indexes.js @@ -0,0 +1,73 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +let taskcluster = require("taskcluster-client"); + +// Create instance of index client +let index = new taskcluster.Index({ + delayFactor: 750, // Good solid delay for background process + retries: 8, // A few extra retries for robustness + rootUrl: + process.env.TASKCLUSTER_PROXY_URL || process.env.TASKCLUSTER_ROOT_URL, +}); + +// Create queue instance for fetching taskId +let queue = new taskcluster.Queue({ + delayFactor: 750, // Good solid delay for background process + retries: 8, // A few extra retries for robustness + rootUrl: + process.env.TASKCLUSTER_PROXY_URL || process.env.TASKCLUSTER_ROOT_URL, +}); + +// Load input +let taskId = process.env.TARGET_TASKID; +let rank = parseInt(process.env.INDEX_RANK, 10); +let namespaces = process.argv.slice(2); + +// Validate input +if (!taskId) { + console.log("Expected target task as environment variable: TARGET_TASKID"); + process.exit(1); +} + +if (isNaN(rank)) { + console.log("Expected index rank as environment variable: INDEX_RANK"); + process.exit(1); +} + +// Fetch task definition to get expiration and then insert into index +queue + .task(taskId) + .then(task => task.expires) + .then(expires => { + return Promise.all( + namespaces.map(namespace => { + console.log( + "Inserting %s into index (rank %d) under: %s", + taskId, + rank, + namespace + ); + return index.insertTask(namespace, { + taskId, + rank, + data: {}, + expires, + }); + }) + ); + }) + .then(() => { + console.log("indexing successfully completed."); + process.exit(0); + }) + .catch(err => { + console.log("Error:\n%s", err); + if (err.stack) { + console.log("Stack:\n%s", err.stack); + } + console.log("Properties:\n%j", err); + throw err; + }) + .catch(() => process.exit(1)); diff --git a/taskcluster/docker/index-task/package.json b/taskcluster/docker/index-task/package.json new file mode 100644 index 0000000000..14c035123c --- /dev/null +++ b/taskcluster/docker/index-task/package.json @@ -0,0 +1,12 @@ +{ + "private": true, + "scripts": { + "start": "node index.js" + }, + "dependencies": { + "taskcluster-client": "^12.2.0" + }, + "engines": { + "node": "10" + } +} diff --git a/taskcluster/docker/index-task/yarn.lock b/taskcluster/docker/index-task/yarn.lock new file mode 100644 index 0000000000..326936d8a6 --- /dev/null +++ b/taskcluster/docker/index-task/yarn.lock @@ -0,0 +1,326 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +amqplib@^0.5.1: + version "0.5.3" + resolved "https://registry.yarnpkg.com/amqplib/-/amqplib-0.5.3.tgz#7ccfc85d12ee7cd3c6dc861bb07f0648ec3d7193" + integrity sha512-ZOdUhMxcF+u62rPI+hMtU1NBXSDFQ3eCJJrenamtdQ7YYwh7RZJHOIM1gonVbZ5PyVdYH4xqBPje9OYqk7fnqw== + dependencies: + bitsyntax "~0.1.0" + bluebird "^3.5.2" + buffer-more-ints "~1.0.0" + readable-stream "1.x >=1.1.9" + safe-buffer "~5.1.2" + url-parse "~1.4.3" + +asap@~2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46" + integrity sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY= + +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" + integrity sha1-x57Zf380y48robyXkLzDZkdLS3k= + +bitsyntax@~0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/bitsyntax/-/bitsyntax-0.1.0.tgz#b0c59acef03505de5a2ed62a2f763c56ae1d6205" + integrity sha512-ikAdCnrloKmFOugAfxWws89/fPc+nw0OOG1IzIE72uSOg/A3cYptKCjSUhDTuj7fhsJtzkzlv7l3b8PzRHLN0Q== + dependencies: + buffer-more-ints "~1.0.0" + debug "~2.6.9" + safe-buffer "~5.1.2" + +bluebird@^3.5.2: + version "3.5.3" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.3.tgz#7d01c6f9616c9a51ab0f8c549a79dfe6ec33efa7" + integrity sha512-/qKPUQlaW1OyR51WeCPBvRnAlnZFUJkCSG5HzGnuIqhgyJtF+T94lFnn33eiazjRm2LAHVy2guNnaq48X9SJuw== + +boom@4.x.x: + version "4.3.1" + resolved "https://registry.yarnpkg.com/boom/-/boom-4.3.1.tgz#4f8a3005cb4a7e3889f749030fd25b96e01d2e31" + integrity sha1-T4owBctKfjiJ90kDD9JbluAdLjE= + dependencies: + hoek "4.x.x" + +boom@5.x.x: + version "5.2.0" + resolved "https://registry.yarnpkg.com/boom/-/boom-5.2.0.tgz#5dd9da6ee3a5f302077436290cb717d3f4a54e02" + integrity sha512-Z5BTk6ZRe4tXXQlkqftmsAUANpXmuwlsF5Oov8ThoMbQRzdGTA1ngYRW160GexgOgjsFOKJz0LYhoNi+2AMBUw== + dependencies: + hoek "4.x.x" + +buffer-more-ints@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz#ef4f8e2dddbad429ed3828a9c55d44f05c611422" + integrity sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg== + +combined-stream@^1.0.6: + version "1.0.7" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.7.tgz#2d1d24317afb8abe95d6d2c0b07b57813539d828" + integrity sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w== + dependencies: + delayed-stream "~1.0.0" + +component-emitter@^1.2.0: + version "1.2.1" + resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.2.1.tgz#137918d6d78283f7df7a6b7c5a63e140e69425e6" + integrity sha1-E3kY1teCg/ffemt8WmPhQOaUJeY= + +cookiejar@^2.1.0: + version "2.1.2" + resolved "https://registry.yarnpkg.com/cookiejar/-/cookiejar-2.1.2.tgz#dd8a235530752f988f9a0844f3fc589e3111125c" + integrity sha512-Mw+adcfzPxcPeI+0WlvRrr/3lGVO0bD75SxX6811cxSh1Wbxx7xZBGK1eVtDf6si8rg2lhnUjsVLMFMfbRIuwA== + +core-util-is@~1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" + integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= + +cryptiles@3.x.x: + version "3.1.4" + resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-3.1.4.tgz#769a68c95612b56faadfcebf57ac86479cbe8322" + integrity sha512-8I1sgZHfVwcSOY6mSGpVU3lw/GSIZvusg8dD2+OGehCJpOhQRLNcH0qb9upQnOH4XhgxxFJSg6E2kx95deb1Tw== + dependencies: + boom "5.x.x" + +debug@^3.1.0: + version "3.2.6" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b" + integrity sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ== + dependencies: + ms "^2.1.1" + +debug@~2.6.9: + version "2.6.9" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" + integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== + dependencies: + ms "2.0.0" + +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= + +extend@^3.0.0: + version "3.0.2" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" + integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== + +form-data@^2.3.1: + version "2.3.3" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" + integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.6" + mime-types "^2.1.12" + +formidable@^1.2.0: + version "1.2.1" + resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.1.tgz#70fb7ca0290ee6ff961090415f4b3df3d2082659" + integrity sha512-Fs9VRguL0gqGHkXS5GQiMCr1VhZBxz0JnJs4JmMp/2jL18Fmbzvv7vOFRU+U8TBkHEE/CX1qDXzJplVULgsLeg== + +hawk@^6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/hawk/-/hawk-6.0.2.tgz#af4d914eb065f9b5ce4d9d11c1cb2126eecc3038" + integrity sha512-miowhl2+U7Qle4vdLqDdPt9m09K6yZhkLDTWGoUiUzrQCn+mHHSmfJgAyGaLRZbPmTqfFFjRV1QWCW0VWUJBbQ== + dependencies: + boom "4.x.x" + cryptiles "3.x.x" + hoek "4.x.x" + sntp "2.x.x" + +hoek@4.x.x: + version "4.2.1" + resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.1.tgz#9634502aa12c445dd5a7c5734b572bb8738aacbb" + integrity sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA== + +inherits@~2.0.1, inherits@~2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" + integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= + +isarray@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" + integrity sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8= + +isarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= + +lodash@^4.17.4: + version "4.17.11" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d" + integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg== + +methods@^1.1.1: + version "1.1.2" + resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" + integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4= + +mime-db@~1.37.0: + version "1.37.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.37.0.tgz#0b6a0ce6fdbe9576e25f1f2d2fde8830dc0ad0d8" + integrity sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg== + +mime-types@^2.1.12: + version "2.1.21" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.21.tgz#28995aa1ecb770742fe6ae7e58f9181c744b3f96" + integrity sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg== + dependencies: + mime-db "~1.37.0" + +mime@^1.4.1: + version "1.6.0" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" + integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== + +ms@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= + +ms@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a" + integrity sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg== + +process-nextick-args@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.0.tgz#a37d732f4271b4ab1ad070d35508e8290788ffaa" + integrity sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw== + +promise@^8.0.1: + version "8.0.2" + resolved "https://registry.yarnpkg.com/promise/-/promise-8.0.2.tgz#9dcd0672192c589477d56891271bdc27547ae9f0" + integrity sha512-EIyzM39FpVOMbqgzEHhxdrEhtOSDOtjMZQ0M6iVfCE+kWNgCkAyOdnuCWqfmflylftfadU6FkiMgHZA2kUzwRw== + dependencies: + asap "~2.0.6" + +qs@^6.5.1: + version "6.6.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.6.0.tgz#a99c0f69a8d26bf7ef012f871cdabb0aee4424c2" + integrity sha512-KIJqT9jQJDQx5h5uAVPimw6yVg2SekOKu959OCtktD3FjzbpvaPr8i4zzg07DOMz+igA4W/aNM7OV8H37pFYfA== + +querystringify@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.1.0.tgz#7ded8dfbf7879dcc60d0a644ac6754b283ad17ef" + integrity sha512-sluvZZ1YiTLD5jsqZcDmFyV2EwToyXZBfpoVOmktMmW+VEnhgakFHnasVph65fOjGPTWN0Nw3+XQaSeMayr0kg== + +"readable-stream@1.x >=1.1.9": + version "1.1.14" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" + integrity sha1-fPTFTvZI44EwhMY23SB54WbAgdk= + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + +readable-stream@^2.3.5: + version "2.3.6" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" + integrity sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw== + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + +requires-port@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" + integrity sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8= + +safe-buffer@~5.1.0, safe-buffer@~5.1.1, safe-buffer@~5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== + +slugid@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/slugid/-/slugid-1.1.0.tgz#e09f00899c09f5a7058edc36dd49f046fd50a82a" + integrity sha1-4J8AiZwJ9acFjtw23UnwRv1QqCo= + dependencies: + uuid "^2.0.1" + +sntp@2.x.x: + version "2.1.0" + resolved "https://registry.yarnpkg.com/sntp/-/sntp-2.1.0.tgz#2c6cec14fedc2222739caf9b5c3d85d1cc5a2cc8" + integrity sha512-FL1b58BDrqS3A11lJ0zEdnJ3UOKqVxawAkF3k7F0CVN7VQ34aZrV+G8BZ1WC9ZL7NyrwsW0oviwsWDgRuVYtJg== + dependencies: + hoek "4.x.x" + +string_decoder@~0.10.x: + version "0.10.31" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" + integrity sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ= + +string_decoder@~1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" + integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== + dependencies: + safe-buffer "~5.1.0" + +superagent@~3.8.1: + version "3.8.3" + resolved "https://registry.yarnpkg.com/superagent/-/superagent-3.8.3.tgz#460ea0dbdb7d5b11bc4f78deba565f86a178e128" + integrity sha512-GLQtLMCoEIK4eDv6OGtkOoSMt3D+oq0y3dsxMuYuDvaNUvuT8eFBuLmfR0iYYzHC1e8hpzC6ZsxbuP6DIalMFA== + dependencies: + component-emitter "^1.2.0" + cookiejar "^2.1.0" + debug "^3.1.0" + extend "^3.0.0" + form-data "^2.3.1" + formidable "^1.2.0" + methods "^1.1.1" + mime "^1.4.1" + qs "^6.5.1" + readable-stream "^2.3.5" + +taskcluster-client@^12.2.0: + version "12.2.0" + resolved "https://registry.yarnpkg.com/taskcluster-client/-/taskcluster-client-12.2.0.tgz#423aee3b17566d14f8ad23e4e47532265a74fb89" + integrity sha512-2Fu5ICS2663kC2t8ymJYzRDnipj3DsCK//b+H/83RjJvC6cWZ0akKzq0ySvPlNA6ic2UcL4I03bJTCJYBX1dqg== + dependencies: + amqplib "^0.5.1" + debug "^3.1.0" + hawk "^6.0.2" + lodash "^4.17.4" + promise "^8.0.1" + slugid "^1.1.0" + superagent "~3.8.1" + taskcluster-lib-urls "^10.0.0" + +taskcluster-lib-urls@^10.0.0: + version "10.1.1" + resolved "https://registry.yarnpkg.com/taskcluster-lib-urls/-/taskcluster-lib-urls-10.1.1.tgz#67d5b9449b947e5234eafdd15c46267dde29bf74" + integrity sha512-tdrK++rCX73FMXk/cXwS6RLTjA3pX8hJlxg1ECLs3L3llCOPMNhQ4wi6lb6yMgHc/s5on/Edj6AlAH7gkxzgPg== + +url-parse@~1.4.3: + version "1.4.4" + resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.4.4.tgz#cac1556e95faa0303691fec5cf9d5a1bc34648f8" + integrity sha512-/92DTTorg4JjktLNLe6GPS2/RvAd/RGr6LuktmWSMLEOa6rjnlrFXNgSbSmkNvCoL2T028A0a1JaJLzRMlFoHg== + dependencies: + querystringify "^2.0.0" + requires-port "^1.0.0" + +util-deprecate@~1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" + integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= + +uuid@^2.0.1: + version "2.0.3" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-2.0.3.tgz#67e2e863797215530dff318e5bf9dcebfd47b21a" + integrity sha1-Z+LoY3lyFVMN/zGOW/nc6/1Hsho= diff --git a/taskcluster/docker/lint/Dockerfile b/taskcluster/docker/lint/Dockerfile new file mode 100644 index 0000000000..e34d9730d7 --- /dev/null +++ b/taskcluster/docker/lint/Dockerfile @@ -0,0 +1,36 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Andrew Halberstadt <ahalberstadt@mozilla.com> + +VOLUME /builds/worker/.cache +VOLUME /builds/worker/checkouts + +# We do want to install recommended packages. +RUN sed -i /APT::Install-Recommends/d /etc/apt/apt.conf.d/99taskcluster + +RUN mkdir /build +# %include python/mozbuild/mozbuild/action/tooltool.py +ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /build/tooltool.py + +# %include taskcluster/docker/recipes/install-node.sh +ADD topsrcdir/taskcluster/docker/recipes/install-node.sh /build/install-node.sh + +ADD system-setup.sh /tmp/system-setup.sh +# %include tools/lint/eslint/manifest.tt +ADD topsrcdir/tools/lint/eslint/manifest.tt /tmp/eslint.tt +# %include tools/lint/eslint/eslint-plugin-mozilla/manifest.tt +ADD topsrcdir/tools/lint/eslint/eslint-plugin-mozilla/manifest.tt /tmp/eslint-plugin-mozilla.tt +# %include tools/lint/spell/codespell_requirements.txt +ADD topsrcdir/tools/lint/spell/codespell_requirements.txt /tmp/codespell_requirements.txt +# %include tools/lint/tox/tox_requirements.txt +ADD topsrcdir/tools/lint/tox/tox_requirements.txt /tmp/tox_requirements.txt +RUN bash /tmp/system-setup.sh + +RUN chown -R worker:worker /builds/worker/bin && chmod 755 /builds/worker/bin/* + +# Set variable normally configured at login, by the shells parent process, these +# are taken from GNU su manual +ENV LANG en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/lint/system-setup.sh b/taskcluster/docker/lint/system-setup.sh new file mode 100644 index 0000000000..18da4b7ade --- /dev/null +++ b/taskcluster/docker/lint/system-setup.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +set -ve + +test "$(whoami)" == 'root' + +mkdir -p /setup +cd /setup + +apt_packages=() +apt_packages+=('curl') +apt_packages+=('iproute2') +apt_packages+=('locales') +apt_packages+=('graphviz') +apt_packages+=('python3-pip') +apt_packages+=('python-is-python3') +apt_packages+=('shellcheck') +apt_packages+=('sudo') +apt_packages+=('wget') +apt_packages+=('unzip') +apt_packages+=('tar') +apt_packages+=('zstd') + +apt-get update +apt-get install "${apt_packages[@]}" + +# Without this we get spurious "LC_ALL: cannot change locale (en_US.UTF-8)" errors, +# and python scripts raise UnicodeEncodeError when trying to print unicode characters. +locale-gen en_US.UTF-8 +dpkg-reconfigure locales + +su -c 'git config --global user.email "worker@mozilla.test"' worker +su -c 'git config --global user.name "worker"' worker + +tooltool_fetch() { + cat >manifest.tt + /build/tooltool.py fetch + rm manifest.tt +} + +cd /build + +### +# ESLint Setup +### + +# install node +# shellcheck disable=SC1091 +. install-node.sh + +npm install -g yarn@1.22.18 + +/build/tooltool.py fetch -m /tmp/eslint.tt +mv /build/node_modules /build/node_modules_eslint +/build/tooltool.py fetch -m /tmp/eslint-plugin-mozilla.tt +mv /build/node_modules /build/node_modules_eslint-plugin-mozilla + +### +# fzf setup +### + +tooltool_fetch <<EOF +[ + { + "size": 1161860, + "digest": "3246470715e1ddf4c7e5136fdddd2ca269928c2de3074a98233faef189efd88fc9b28ddbe68642a31cf647a97f630941d764187006c5115e6f357d49322ef58d", + "algorithm": "sha512", + "filename": "fzf-0.20.0-linux_amd64.tgz", + "unpack": true + } +] +EOF +mv fzf /usr/local/bin + +### +# codespell Setup +### + +cd /setup + +pip3 install --require-hashes -r /tmp/codespell_requirements.txt + +### +# tox Setup +### + +cd /setup + +pip3 install --require-hashes -r /tmp/tox_requirements.txt + +cd / +rm -rf /setup diff --git a/taskcluster/docker/partner-repack/Dockerfile b/taskcluster/docker/partner-repack/Dockerfile new file mode 100644 index 0000000000..aae9d893da --- /dev/null +++ b/taskcluster/docker/partner-repack/Dockerfile @@ -0,0 +1,20 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Ben Hearsum <bhearsum@mozilla.com> + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace + +RUN dpkg --add-architecture amd64 + +RUN apt-get update && \ + apt-get install \ + bzip2 \ + curl \ + git \ + gzip \ + openssh-client \ + python2 \ + unzip \ + zip + +COPY known_hosts /etc/ssh/ssh_known_hosts diff --git a/taskcluster/docker/partner-repack/known_hosts b/taskcluster/docker/partner-repack/known_hosts new file mode 100644 index 0000000000..73b210c5aa --- /dev/null +++ b/taskcluster/docker/partner-repack/known_hosts @@ -0,0 +1,2 @@ +|1|l1+RDluaBh2vgAVjzSQP5cMsazU=|zA0AvSBB4c3SJ6OM1H0pjrTY/BQ= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== +|1|A4q4hHD/XV6BPf9bom/6780dCwk=|gm+K1qkEPDUMm1axYNTOkbI1BQ4= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== diff --git a/taskcluster/docker/periodic-updates/.eslintrc.js b/taskcluster/docker/periodic-updates/.eslintrc.js new file mode 100644 index 0000000000..15d7603afd --- /dev/null +++ b/taskcluster/docker/periodic-updates/.eslintrc.js @@ -0,0 +1,72 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +module.exports = { + extends: ["plugin:mozilla/valid-jsdoc"], + + globals: { + // JS files in this folder are commonly xpcshell scripts where |arguments| + // is defined in the global scope. + arguments: false, + }, + rules: { + // Enforce return statements in callbacks of array methods. + "array-callback-return": "error", + + // Verify calls of super() in constructors. + "constructor-super": "error", + + // Require default case in switch statements. + "default-case": "error", + + // Disallow use of alert(), confirm(), and prompt(). + "no-alert": "error", + + // Disallow likely erroneous `switch` scoped lexical declarations in + // case/default clauses. + "no-case-declarations": "error", + + // Disallow use of the console API. + "no-console": "error", + + // Disallow constant expressions in conditions (except for loops). + "no-constant-condition": ["error", { checkLoops: false }], + + // Disallow extending of native objects. + "no-extend-native": "error", + + // Disallow case statement fallthrough without explicit `// falls through` + // annotation. + "no-fallthrough": "error", + + // No reassigning native JS objects or read only globals. + "no-global-assign": "error", + + // Disallow use of assignment in return statement. + "no-return-assign": ["error", "always"], + + // Disallow template literal placeholder syntax in regular strings. + "no-template-curly-in-string": "error", + + // Disallow use of this/super before calling super() in constructors. + "no-this-before-super": "error", + + // Disallow unmodified loop conditions. + "no-unmodified-loop-condition": "error", + + // No expressions where a statement is expected + "no-unused-expressions": "error", + + // Disallow unnecessary escape usage in strings and regular expressions. + "no-useless-escape": "error", + + // Require "use strict" to be defined globally in the script. + strict: ["error", "global"], + + // Disallow Yoda conditions. + yoda: ["error", "never"], + }, +}; diff --git a/taskcluster/docker/periodic-updates/Dockerfile b/taskcluster/docker/periodic-updates/Dockerfile new file mode 100644 index 0000000000..24cabe02b5 --- /dev/null +++ b/taskcluster/docker/periodic-updates/Dockerfile @@ -0,0 +1,11 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Julien Cristau <jcristau@mozilla.com> + +ADD setup.sh /setup/setup.sh + +RUN cd /setup && ./setup.sh + +COPY runme.sh / +COPY scripts/* /home/worker/scripts/ + +CMD ["/runme.sh"] diff --git a/taskcluster/docker/periodic-updates/README.md b/taskcluster/docker/periodic-updates/README.md new file mode 100644 index 0000000000..d21c0c3656 --- /dev/null +++ b/taskcluster/docker/periodic-updates/README.md @@ -0,0 +1,96 @@ + +==Periodic File Updates== + +This docker image examines the in-tree files for HSTS preload data, HPKP pinning and blocklisting, and +will produce a diff for each necessary to update the in-tree files. + +If given a conduit API token, it will also use the arcanist client to submit the commits for review. + + +==Quick Start== + +```sh +docker build -t hsts-local --no-cache --rm . + +docker run -e DO_HSTS=1 -e DO_HPKP=1 -e PRODUCT="firefox" -e BRANCH="mozilla-central" -e USE_MOZILLA_CENTRAL=1 hsts-local +``` + +HSTS checks will only be run if the `DO_HSTS` environment variable is set. +Likewise for `DO_HPKP` and the HPKP checks. Environment variables are used +rather than command line arguments to make constructing taskcluster tasks +easier. + +To prevent a full build when landing with Phabricator, set the `DONTBUILD` +environment variable. + +==Background== + +These scripts have been moved from +`https://hg.mozilla.org/build/tools/scripts/periodic_file_updates/` and +`security/manager/tools/` in the main repos. + +==HSTS Checks== + +`scripts/getHSTSPreloadList.js` will examine the current contents of +nsSTSPreloadList.inc from whichever `BRANCH` is specified, add in the mandatory +hosts, and those from the Chromium source, and check them all to see if their +SSL configuration is valid, and whether or not they have the +Strict-Transport-Security header set with an appropriate `max-age`. + +This javascript has been modified to use async calls to improve performance. + +==HPKP Checks== + +`scripts/genHPKPStaticPins.js` will ensure the list of pinned public keys are +up to date. + +==Example Taskcluster Task== + +https://firefox-ci-tc.services.mozilla.com/tasks/create/ + +```yaml +provisionerId: aws-provisioner-v1 +workerType: gecko-1-b-linux +retries: 0 +created: '2018-02-07T14:45:57.347Z' +deadline: '2018-02-07T17:45:57.348Z' +expires: '2019-02-07T17:45:57.348Z' +scopes: [] +payload: + image: srfraser/hsts1 + maxRunTime: 1800 + artifacts: + public/build/nsSTSPreloadList.diff: + path: /home/worker/artifacts/nsSTSPreloadList.diff + expires: '2019-02-07T13:57:35.448Z' + type: file + public/build/StaticHPKPins.h.diff: + path: /home/worker/artifacts/StaticHPKPins.h.diff + expires: '2019-02-07T13:57:35.448Z' + type: file + public/build/blocklist.diff: + path: /home/worker/artifacts/blocklist.diff + expires: '2019-02-07T13:57:35.448Z' + type: file + env: + DO_HSTS: 1 + DO_HPKP: 1 + PRODUCT: firefox + BRANCH: mozilla-central + USE_MOZILLA_CENTRAL: 1 + REVIEWERS: catlee +metadata: + name: Periodic updates testing + description: Produce diffs for HSTS and HPKP in-tree files. + owner: sfraser@mozilla.com + source: 'https://firefox-ci-tc.services.mozilla.com/tasks/create' +tags: {} +extra: + treeherder: + jobKind: test + machine: + platform: linux64 + tier: 1 + symbol: 'hsts' + +``` diff --git a/taskcluster/docker/periodic-updates/runme.sh b/taskcluster/docker/periodic-updates/runme.sh new file mode 100755 index 0000000000..368963aff6 --- /dev/null +++ b/taskcluster/docker/periodic-updates/runme.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +set -xe + +# Things to be set by task definition. +# --pinset --hsts --hpkp +# -b branch +# --use-mozilla-central +# -p firefox +# Artifact directory +# Artifact names. + + +test "${BRANCH}" +test "${PRODUCT}" + +PARAMS="" + +if [ -n "${USE_MOZILLA_CENTRAL}" ] +then + PARAMS="${PARAMS} --use-mozilla-central" +fi + +# TODO change these, so that they're run if the artifact location is specified? +if [ -n "${DO_HSTS}" ] +then + PARAMS="${PARAMS} --hsts" +fi + +if [ -n "${DO_HPKP}" ] +then + PARAMS="${PARAMS} --hpkp" +fi + +if [ -n "${DO_REMOTE_SETTINGS}" ] +then + PARAMS="${PARAMS} --remote-settings" +fi + +if [ -n "${DO_SUFFIX_LIST}" ] +then + PARAMS="${PARAMS} --suffix-list" +fi + +if [ -n "${DONTBUILD}" ] +then + PARAMS="${PARAMS} -d" +fi + + +export ARTIFACTS_DIR="/home/worker/artifacts" +mkdir -p "$ARTIFACTS_DIR" + +# duplicate the functionality of taskcluster-lib-urls, but in bash.. +queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" + +# Get Arcanist API token + +if [ -n "${TASK_ID}" ] +then + curl --location --retry 10 --retry-delay 10 -o /home/worker/task.json "$queue_base/task/$TASK_ID" + ARC_SECRET=$(jq -r '.scopes[] | select(contains ("arc-phabricator-token"))' /home/worker/task.json | awk -F: '{print $3}') +fi +if [ -n "${ARC_SECRET}" ] && getent hosts taskcluster +then + set +x # Don't echo these + secrets_url="${TASKCLUSTER_PROXY_URL}/api/secrets/v1/secret/${ARC_SECRET}" + SECRET=$(curl "${secrets_url}") + TOKEN=$(echo "${SECRET}" | jq -r '.secret.token') +elif [ -n "${ARC_TOKEN}" ] # Allow for local testing. +then + TOKEN="${ARC_TOKEN}" +fi + +if [ -n "${TOKEN}" ] +then + cat >"${HOME}/.arcrc" <<END +{ + "hosts": { + "https://phabricator.services.mozilla.com/api/": { + "token": "${TOKEN}" + } + } +} +END + set -x + chmod 600 "${HOME}/.arcrc" +fi + +export HGPLAIN=1 + +# shellcheck disable=SC2086 +/home/worker/scripts/periodic_file_updates.sh -p "${PRODUCT}" -b "${BRANCH}" -a ${PARAMS} diff --git a/taskcluster/docker/periodic-updates/scripts/genHPKPStaticPins.js b/taskcluster/docker/periodic-updates/scripts/genHPKPStaticPins.js new file mode 100644 index 0000000000..6984f54d26 --- /dev/null +++ b/taskcluster/docker/periodic-updates/scripts/genHPKPStaticPins.js @@ -0,0 +1,674 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// How to run this file: +// 1. [obtain firefox source code] +// 2. [build/obtain firefox binaries] +// 3. run `[path to]/run-mozilla.sh [path to]/xpcshell \ +// [path to]/genHPKPStaticpins.js \ +// [absolute path to]/PreloadedHPKPins.json \ +// [absolute path to]/StaticHPKPins.h +"use strict"; + +if (arguments.length != 2) { + throw new Error( + "Usage: genHPKPStaticPins.js " + + "<absolute path to PreloadedHPKPins.json> " + + "<absolute path to StaticHPKPins.h>" + ); +} + +var { NetUtil } = ChromeUtils.import("resource://gre/modules/NetUtil.jsm"); +var { FileUtils } = ChromeUtils.importESModule( + "resource://gre/modules/FileUtils.sys.mjs" +); + +var gCertDB = Cc["@mozilla.org/security/x509certdb;1"].getService( + Ci.nsIX509CertDB +); + +const SHA256_PREFIX = "sha256/"; +const GOOGLE_PIN_PREFIX = "GOOGLE_PIN_"; + +// Pins expire in 14 weeks (6 weeks on Beta + 8 weeks on stable) +const PINNING_MINIMUM_REQUIRED_MAX_AGE = 60 * 60 * 24 * 7 * 14; + +const FILE_HEADER = + "/* This Source Code Form is subject to the terms of the Mozilla Public\n" + + " * License, v. 2.0. If a copy of the MPL was not distributed with this\n" + + " * file, You can obtain one at http://mozilla.org/MPL/2.0/. */\n" + + "\n" + + "/*****************************************************************************/\n" + + "/* This is an automatically generated file. If you're not */\n" + + "/* PublicKeyPinningService.cpp, you shouldn't be #including it. */\n" + + "/*****************************************************************************/\n" + + "#include <stdint.h>" + + "\n"; + +const DOMAINHEADER = + "/* Domainlist */\n" + + "struct TransportSecurityPreload {\n" + + " // See bug 1338873 about making these fields const.\n" + + " const char* mHost;\n" + + " bool mIncludeSubdomains;\n" + + " bool mTestMode;\n" + + " bool mIsMoz;\n" + + " int32_t mId;\n" + + " const StaticFingerprints* pinset;\n" + + "};\n\n"; + +const PINSETDEF = + "/* Pinsets are each an ordered list by the actual value of the fingerprint */\n" + + "struct StaticFingerprints {\n" + + " // See bug 1338873 about making these fields const.\n" + + " size_t size;\n" + + " const char* const* data;\n" + + "};\n\n"; + +// Command-line arguments +var gStaticPins = parseJson(arguments[0]); + +// Open the output file. +var file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile); +file.initWithPath(arguments[1]); +var gFileOutputStream = FileUtils.openSafeFileOutputStream(file); + +function writeString(string) { + gFileOutputStream.write(string, string.length); +} + +function readFileToString(filename) { + let file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile); + file.initWithPath(filename); + let stream = Cc["@mozilla.org/network/file-input-stream;1"].createInstance( + Ci.nsIFileInputStream + ); + stream.init(file, -1, 0, 0); + let buf = NetUtil.readInputStreamToString(stream, stream.available()); + return buf; +} + +function stripComments(buf) { + let lines = buf.split("\n"); + let entryRegex = /^\s*\/\//; + let data = ""; + for (let i = 0; i < lines.length; ++i) { + let match = entryRegex.exec(lines[i]); + if (!match) { + data = data + lines[i]; + } + } + return data; +} + +function download(filename) { + let req = new XMLHttpRequest(); + req.open("GET", filename, false); // doing the request synchronously + try { + req.send(); + } catch (e) { + throw new Error(`ERROR: problem downloading '${filename}': ${e}`); + } + + if (req.status != 200) { + throw new Error( + "ERROR: problem downloading '" + filename + "': status " + req.status + ); + } + + let resultDecoded; + try { + resultDecoded = atob(req.responseText); + } catch (e) { + throw new Error( + "ERROR: could not decode data as base64 from '" + filename + "': " + e + ); + } + return resultDecoded; +} + +function downloadAsJson(filename) { + // we have to filter out '//' comments, while not mangling the json + let result = download(filename).replace(/^(\s*)?\/\/[^\n]*\n/gm, ""); + let data = null; + try { + data = JSON.parse(result); + } catch (e) { + throw new Error( + "ERROR: could not parse data from '" + filename + "': " + e + ); + } + return data; +} + +// Returns a Subject Public Key Digest from the given pem, if it exists. +function getSKDFromPem(pem) { + let cert = gCertDB.constructX509FromBase64(pem, pem.length); + return cert.sha256SubjectPublicKeyInfoDigest; +} + +/** + * Hashes |input| using the SHA-256 algorithm in the following manner: + * btoa(sha256(atob(input))) + * + * @param {string} input Base64 string to decode and return the hash of. + * @returns {string} Base64 encoded SHA-256 hash. + */ +function sha256Base64(input) { + let decodedValue; + try { + decodedValue = atob(input); + } catch (e) { + throw new Error(`ERROR: could not decode as base64: '${input}': ${e}`); + } + + // Convert |decodedValue| to an array so that it can be hashed by the + // nsICryptoHash instance below. + // In most cases across the code base, convertToByteArray() of + // nsIScriptableUnicodeConverter is used to do this, but the method doesn't + // seem to work here. + let data = []; + for (let i = 0; i < decodedValue.length; i++) { + data[i] = decodedValue.charCodeAt(i); + } + + let hasher = Cc["@mozilla.org/security/hash;1"].createInstance( + Ci.nsICryptoHash + ); + hasher.init(hasher.SHA256); + hasher.update(data, data.length); + + // true is passed so that the hasher returns a Base64 encoded string. + return hasher.finish(true); +} + +// Downloads the static certs file and tries to map Google Chrome nicknames +// to Mozilla nicknames, as well as storing any hashes for pins for which we +// don't have root PEMs. Each entry consists of a line containing the name of +// the pin followed either by a hash in the format "sha256/" + base64(hash), +// a PEM encoded public key, or a PEM encoded certificate. +// For certificates that we have in our database, +// return a map of Google's nickname to ours. For ones that aren't return a +// map of Google's nickname to SHA-256 values. This code is modeled after agl's +// https://github.com/agl/transport-security-state-generate, which doesn't +// live in the Chromium repo because go is not an official language in +// Chromium. +// For all of the entries in this file: +// - If the entry has a hash format, find the Mozilla pin name (cert nickname) +// and stick the hash into certSKDToName +// - If the entry has a PEM format, parse the PEM, find the Mozilla pin name +// and stick the hash in certSKDToName +// We MUST be able to find a corresponding cert nickname for the Chrome names, +// otherwise we skip all pinsets referring to that Chrome name. +function downloadAndParseChromeCerts(filename, certNameToSKD, certSKDToName) { + // Prefixes that we care about. + const BEGIN_CERT = "-----BEGIN CERTIFICATE-----"; + const END_CERT = "-----END CERTIFICATE-----"; + const BEGIN_PUB_KEY = "-----BEGIN PUBLIC KEY-----"; + const END_PUB_KEY = "-----END PUBLIC KEY-----"; + + // Parsing states. + const PRE_NAME = 0; + const POST_NAME = 1; + const IN_CERT = 2; + const IN_PUB_KEY = 3; + let state = PRE_NAME; + + let lines = download(filename).split("\n"); + let pemCert = ""; + let pemPubKey = ""; + let hash = ""; + let chromeNameToHash = {}; + let chromeNameToMozName = {}; + let chromeName; + for (let line of lines) { + // Skip comments and newlines. + if (!line.length || line[0] == "#") { + continue; + } + switch (state) { + case PRE_NAME: + chromeName = line; + state = POST_NAME; + break; + case POST_NAME: + if (line.startsWith(SHA256_PREFIX)) { + hash = line.substring(SHA256_PREFIX.length); + chromeNameToHash[chromeName] = hash; + certNameToSKD[chromeName] = hash; + certSKDToName[hash] = chromeName; + state = PRE_NAME; + } else if (line.startsWith(BEGIN_CERT)) { + state = IN_CERT; + } else if (line.startsWith(BEGIN_PUB_KEY)) { + state = IN_PUB_KEY; + } else if ( + chromeName == "PinsListTimestamp" && + line.match(/^[0-9]+$/) + ) { + // If the name of this entry is "PinsListTimestamp", this line should + // be the pins list timestamp. It should consist solely of digits. + // Ignore it and expect other entries to come. + state = PRE_NAME; + } else { + throw new Error( + "ERROR: couldn't parse Chrome certificate file line: " + line + ); + } + break; + case IN_CERT: + if (line.startsWith(END_CERT)) { + state = PRE_NAME; + hash = getSKDFromPem(pemCert); + pemCert = ""; + let mozName; + if (hash in certSKDToName) { + mozName = certSKDToName[hash]; + } else { + // Not one of our built-in certs. Prefix the name with + // GOOGLE_PIN_. + mozName = GOOGLE_PIN_PREFIX + chromeName; + dump( + "Can't find hash in builtin certs for Chrome nickname " + + chromeName + + ", inserting " + + mozName + + "\n" + ); + certSKDToName[hash] = mozName; + certNameToSKD[mozName] = hash; + } + chromeNameToMozName[chromeName] = mozName; + } else { + pemCert += line; + } + break; + case IN_PUB_KEY: + if (line.startsWith(END_PUB_KEY)) { + state = PRE_NAME; + hash = sha256Base64(pemPubKey); + pemPubKey = ""; + chromeNameToHash[chromeName] = hash; + certNameToSKD[chromeName] = hash; + certSKDToName[hash] = chromeName; + } else { + pemPubKey += line; + } + break; + default: + throw new Error( + "ERROR: couldn't parse Chrome certificate file " + line + ); + } + } + return [chromeNameToHash, chromeNameToMozName]; +} + +// We can only import pinsets from chrome if for every name in the pinset: +// - We have a hash from Chrome's static certificate file +// - We have a builtin cert +// If the pinset meets these requirements, we store a map array of pinset +// objects: +// { +// pinset_name : { +// // Array of names with entries in certNameToSKD +// sha256_hashes: [] +// } +// } +// and an array of imported pinset entries: +// { name: string, include_subdomains: boolean, test_mode: boolean, +// pins: pinset_name } +function downloadAndParseChromePins( + filename, + chromeNameToHash, + chromeNameToMozName, + certNameToSKD, + certSKDToName +) { + let chromePreloads = downloadAsJson(filename); + let chromePins = chromePreloads.pinsets; + let chromeImportedPinsets = {}; + let chromeImportedEntries = []; + + chromePins.forEach(function(pin) { + let valid = true; + let pinset = { name: pin.name, sha256_hashes: [] }; + // Translate the Chrome pinset format to ours + pin.static_spki_hashes.forEach(function(name) { + if (name in chromeNameToHash) { + let hash = chromeNameToHash[name]; + pinset.sha256_hashes.push(certSKDToName[hash]); + + // We should have already added hashes for all of these when we + // imported the certificate file. + if (!certNameToSKD[name]) { + throw new Error("ERROR: No hash for name: " + name); + } + } else if (name in chromeNameToMozName) { + pinset.sha256_hashes.push(chromeNameToMozName[name]); + } else { + dump( + "Skipping Chrome pinset " + + pinset.name + + ", couldn't find " + + "builtin " + + name + + " from cert file\n" + ); + valid = false; + } + }); + if (valid) { + chromeImportedPinsets[pinset.name] = pinset; + } + }); + + // Grab the domain entry lists. Chrome's entry format is similar to + // ours, except theirs includes a HSTS mode. + const cData = gStaticPins.chromium_data; + let entries = chromePreloads.entries; + entries.forEach(function(entry) { + // HSTS entry only + if (!entry.pins) { + return; + } + let pinsetName = cData.substitute_pinsets[entry.pins]; + if (!pinsetName) { + pinsetName = entry.pins; + } + + // We trim the entry name here to avoid breaking hostname comparisons in the + // HPKP implementation. + entry.name = entry.name.trim(); + + let isProductionDomain = cData.production_domains.includes(entry.name); + let isProductionPinset = cData.production_pinsets.includes(pinsetName); + let excludeDomain = cData.exclude_domains.includes(entry.name); + let isTestMode = !isProductionPinset && !isProductionDomain; + if (entry.pins && !excludeDomain && chromeImportedPinsets[entry.pins]) { + chromeImportedEntries.push({ + name: entry.name, + include_subdomains: entry.include_subdomains, + test_mode: isTestMode, + is_moz: false, + pins: pinsetName, + }); + } + }); + return [chromeImportedPinsets, chromeImportedEntries]; +} + +// Returns a pair of maps [certNameToSKD, certSKDToName] between cert +// nicknames and digests of the SPKInfo for the mozilla trust store +function loadNSSCertinfo(extraCertificates) { + let allCerts = gCertDB.getCerts(); + let certNameToSKD = {}; + let certSKDToName = {}; + for (let cert of allCerts) { + if (!cert.isBuiltInRoot) { + continue; + } + let name = cert.displayName; + let SKD = cert.sha256SubjectPublicKeyInfoDigest; + certNameToSKD[name] = SKD; + certSKDToName[SKD] = name; + } + + for (let cert of extraCertificates) { + let name = cert.commonName; + let SKD = cert.sha256SubjectPublicKeyInfoDigest; + certNameToSKD[name] = SKD; + certSKDToName[SKD] = name; + } + + { + // This is the pinning test certificate. The key hash identifies the + // default RSA key from pykey. + let name = "End Entity Test Cert"; + let SKD = "VCIlmPM9NkgFQtrs4Oa5TeFcDu6MWRTKSNdePEhOgD8="; + certNameToSKD[name] = SKD; + certSKDToName[SKD] = name; + } + return [certNameToSKD, certSKDToName]; +} + +function parseJson(filename) { + let json = stripComments(readFileToString(filename)); + return JSON.parse(json); +} + +function nameToAlias(certName) { + // change the name to a string valid as a c identifier + // remove non-ascii characters + certName = certName.replace(/[^[:ascii:]]/g, "_"); + // replace non word characters + certName = certName.replace(/[^A-Za-z0-9]/g, "_"); + + return "k" + certName + "Fingerprint"; +} + +function compareByName(a, b) { + return a.name.localeCompare(b.name); +} + +function genExpirationTime() { + let now = new Date(); + let nowMillis = now.getTime(); + let expirationMillis = nowMillis + PINNING_MINIMUM_REQUIRED_MAX_AGE * 1000; + let expirationMicros = expirationMillis * 1000; + return ( + "static const PRTime kPreloadPKPinsExpirationTime = INT64_C(" + + expirationMicros + + ");\n" + ); +} + +function writeFullPinset(certNameToSKD, certSKDToName, pinset) { + if (!pinset.sha256_hashes || !pinset.sha256_hashes.length) { + throw new Error(`ERROR: Pinset ${pinset.name} does not contain any hashes`); + } + writeFingerprints( + certNameToSKD, + certSKDToName, + pinset.name, + pinset.sha256_hashes + ); +} + +function writeFingerprints(certNameToSKD, certSKDToName, name, hashes) { + let varPrefix = "kPinset_" + name; + writeString("static const char* const " + varPrefix + "_Data[] = {\n"); + let SKDList = []; + for (let certName of hashes) { + if (!(certName in certNameToSKD)) { + throw new Error(`ERROR: Can't find '${certName}' in certNameToSKD`); + } + SKDList.push(certNameToSKD[certName]); + } + for (let skd of SKDList.sort()) { + writeString(" " + nameToAlias(certSKDToName[skd]) + ",\n"); + } + if (!hashes.length) { + // ANSI C requires that an initialiser list be non-empty. + writeString(" 0\n"); + } + writeString("};\n"); + writeString( + "static const StaticFingerprints " + + varPrefix + + " = {\n " + + "sizeof(" + + varPrefix + + "_Data) / sizeof(const char*),\n " + + varPrefix + + "_Data\n};\n\n" + ); +} + +function writeEntry(entry) { + let printVal = ` { "${entry.name}", `; + if (entry.include_subdomains) { + printVal += "true, "; + } else { + printVal += "false, "; + } + // Default to test mode if not specified. + let testMode = true; + if (entry.hasOwnProperty("test_mode")) { + testMode = entry.test_mode; + } + if (testMode) { + printVal += "true, "; + } else { + printVal += "false, "; + } + if ( + entry.is_moz || + (entry.pins.includes("mozilla") && entry.pins != "mozilla_test") + ) { + printVal += "true, "; + } else { + printVal += "false, "; + } + if ("id" in entry) { + if (entry.id >= 256) { + throw new Error("ERROR: Not enough buckets in histogram"); + } + if (entry.id >= 0) { + printVal += entry.id + ", "; + } + } else { + printVal += "-1, "; + } + printVal += "&kPinset_" + entry.pins; + printVal += " },\n"; + writeString(printVal); +} + +function writeDomainList(chromeImportedEntries) { + writeString("/* Sort hostnames for binary search. */\n"); + writeString( + "static const TransportSecurityPreload " + + "kPublicKeyPinningPreloadList[] = {\n" + ); + let count = 0; + let mozillaDomains = {}; + gStaticPins.entries.forEach(function(entry) { + mozillaDomains[entry.name] = true; + }); + // For any domain for which we have set pins, exclude them from + // chromeImportedEntries. + for (let i = chromeImportedEntries.length - 1; i >= 0; i--) { + if (mozillaDomains[chromeImportedEntries[i].name]) { + dump( + "Skipping duplicate pinset for domain " + + JSON.stringify(chromeImportedEntries[i], undefined, 2) + + "\n" + ); + chromeImportedEntries.splice(i, 1); + } + } + let sortedEntries = gStaticPins.entries; + sortedEntries.push.apply(sortedEntries, chromeImportedEntries); + for (let entry of sortedEntries.sort(compareByName)) { + count++; + writeEntry(entry); + } + writeString("};\n"); + + writeString("\n// Pinning Preload List Length = " + count + ";\n"); + writeString("\nstatic const int32_t kUnknownId = -1;\n"); +} + +function writeFile( + certNameToSKD, + certSKDToName, + chromeImportedPinsets, + chromeImportedEntries +) { + // Compute used pins from both Chrome's and our pinsets, so we can output + // them later. + let usedFingerprints = {}; + let mozillaPins = {}; + gStaticPins.pinsets.forEach(function(pinset) { + mozillaPins[pinset.name] = true; + pinset.sha256_hashes.forEach(function(name) { + usedFingerprints[name] = true; + }); + }); + for (let key in chromeImportedPinsets) { + let pinset = chromeImportedPinsets[key]; + pinset.sha256_hashes.forEach(function(name) { + usedFingerprints[name] = true; + }); + } + + writeString(FILE_HEADER); + + // Write actual fingerprints. + Object.keys(usedFingerprints) + .sort() + .forEach(function(certName) { + if (certName) { + writeString("/* " + certName + " */\n"); + writeString("static const char " + nameToAlias(certName) + "[] =\n"); + writeString(' "' + certNameToSKD[certName] + '";\n'); + writeString("\n"); + } + }); + + // Write the pinsets + writeString(PINSETDEF); + writeString("/* PreloadedHPKPins.json pinsets */\n"); + gStaticPins.pinsets.sort(compareByName).forEach(function(pinset) { + writeFullPinset(certNameToSKD, certSKDToName, pinset); + }); + writeString("/* Chrome static pinsets */\n"); + for (let key in chromeImportedPinsets) { + if (mozillaPins[key]) { + dump("Skipping duplicate pinset " + key + "\n"); + } else { + dump("Writing pinset " + key + "\n"); + writeFullPinset(certNameToSKD, certSKDToName, chromeImportedPinsets[key]); + } + } + + // Write the domainlist entries. + writeString(DOMAINHEADER); + writeDomainList(chromeImportedEntries); + writeString("\n"); + writeString(genExpirationTime()); +} + +function loadExtraCertificates(certStringList) { + let constructedCerts = []; + for (let certString of certStringList) { + constructedCerts.push(gCertDB.constructX509FromBase64(certString)); + } + return constructedCerts; +} + +var extraCertificates = loadExtraCertificates(gStaticPins.extra_certificates); +var [certNameToSKD, certSKDToName] = loadNSSCertinfo(extraCertificates); +var [chromeNameToHash, chromeNameToMozName] = downloadAndParseChromeCerts( + gStaticPins.chromium_data.cert_file_url, + certNameToSKD, + certSKDToName +); +var [chromeImportedPinsets, chromeImportedEntries] = downloadAndParseChromePins( + gStaticPins.chromium_data.json_file_url, + chromeNameToHash, + chromeNameToMozName, + certNameToSKD, + certSKDToName +); + +writeFile( + certNameToSKD, + certSKDToName, + chromeImportedPinsets, + chromeImportedEntries +); + +FileUtils.closeSafeFileOutputStream(gFileOutputStream); diff --git a/taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js b/taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js new file mode 100644 index 0000000000..37f5ab9221 --- /dev/null +++ b/taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js @@ -0,0 +1,557 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +// How to run this file: +// 1. [obtain firefox source code] +// 2. [build/obtain firefox binaries] +// 3. run `[path to]/run-mozilla.sh [path to]/xpcshell [path to]/getHSTSPreloadlist.js [absolute path to]/nsSTSPreloadlist.inc' +// Note: Running this file outputs a new nsSTSPreloadlist.inc in the current +// working directory. + +var gSSService = Cc["@mozilla.org/ssservice;1"].getService( + Ci.nsISiteSecurityService +); + +const { FileUtils } = ChromeUtils.importESModule( + "resource://gre/modules/FileUtils.sys.mjs" +); + +const SOURCE = + "https://chromium.googlesource.com/chromium/src/+/refs/heads/main/net/http/transport_security_state_static.json?format=TEXT"; +const TOOL_SOURCE = + "https://hg.mozilla.org/mozilla-central/file/default/taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js"; +const OUTPUT = "nsSTSPreloadList.inc"; +const MINIMUM_REQUIRED_MAX_AGE = 60 * 60 * 24 * 7 * 18; +const MAX_CONCURRENT_REQUESTS = 500; +const MAX_RETRIES = 1; +const REQUEST_TIMEOUT = 30 * 1000; +const ERROR_NONE = "no error"; +const ERROR_CONNECTING_TO_HOST = "could not connect to host"; +const ERROR_NO_HSTS_HEADER = "did not receive HSTS header"; +const ERROR_MAX_AGE_TOO_LOW = "max-age too low: "; +const HEADER = `/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/*****************************************************************************/ +/* This is an automatically generated file. If you're not */ +/* nsSiteSecurityService.cpp, you shouldn't be #including it. */ +/*****************************************************************************/ + +#include <stdint.h> +`; + +const GPERF_DELIM = "%%\n"; + +function download() { + let req = new XMLHttpRequest(); + req.open("GET", SOURCE, false); // doing the request synchronously + try { + req.send(); + } catch (e) { + throw new Error(`ERROR: problem downloading '${SOURCE}': ${e}`); + } + + if (req.status != 200) { + throw new Error( + "ERROR: problem downloading '" + SOURCE + "': status " + req.status + ); + } + + let resultDecoded; + try { + resultDecoded = atob(req.responseText); + } catch (e) { + throw new Error( + "ERROR: could not decode data as base64 from '" + SOURCE + "': " + e + ); + } + + // we have to filter out '//' comments, while not mangling the json + let result = resultDecoded.replace(/^(\s*)?\/\/[^\n]*\n/gm, ""); + let data = null; + try { + data = JSON.parse(result); + } catch (e) { + throw new Error(`ERROR: could not parse data from '${SOURCE}': ${e}`); + } + return data; +} + +function getHosts(rawdata) { + let hosts = []; + + if (!rawdata || !rawdata.entries) { + throw new Error( + "ERROR: source data not formatted correctly: 'entries' not found" + ); + } + + for (let entry of rawdata.entries) { + if (entry.mode && entry.mode == "force-https") { + if (entry.name) { + // We trim the entry name here to avoid malformed URI exceptions when we + // later try to connect to the domain. + entry.name = entry.name.trim(); + entry.retries = MAX_RETRIES; + // We prefer the camelCase variable to the JSON's snake case version + entry.includeSubdomains = entry.include_subdomains; + hosts.push(entry); + } else { + throw new Error("ERROR: entry not formatted correctly: no name found"); + } + } + } + + return hosts; +} + +function processStsHeader(host, header, status, securityInfo) { + let maxAge = { + value: 0, + }; + let includeSubdomains = { + value: false, + }; + let error = ERROR_NONE; + if ( + header != null && + securityInfo != null && + securityInfo.overridableErrorCategory == + Ci.nsITransportSecurityInfo.ERROR_UNSET + ) { + try { + let uri = Services.io.newURI("https://" + host.name); + gSSService.processHeader(uri, header, {}, maxAge, includeSubdomains); + } catch (e) { + dump( + "ERROR: could not process header '" + + header + + "' from " + + host.name + + ": " + + e + + "\n" + ); + error = e; + } + } else if (status == 0) { + error = ERROR_CONNECTING_TO_HOST; + } else { + error = ERROR_NO_HSTS_HEADER; + } + + if (error == ERROR_NONE && maxAge.value < MINIMUM_REQUIRED_MAX_AGE) { + error = ERROR_MAX_AGE_TOO_LOW; + } + + return { + name: host.name, + maxAge: maxAge.value, + includeSubdomains: includeSubdomains.value, + error, + retries: host.retries - 1, + forceInclude: host.forceInclude, + }; +} + +// RedirectAndAuthStopper prevents redirects and HTTP authentication +function RedirectAndAuthStopper() {} + +RedirectAndAuthStopper.prototype = { + // nsIChannelEventSink + asyncOnChannelRedirect(oldChannel, newChannel, flags, callback) { + throw Components.Exception("", Cr.NS_ERROR_ENTITY_CHANGED); + }, + + // nsIAuthPrompt2 + promptAuth(channel, level, authInfo) { + return false; + }, + + asyncPromptAuth(channel, callback, context, level, authInfo) { + throw Components.Exception("", Cr.NS_ERROR_NOT_IMPLEMENTED); + }, + + getInterface(iid) { + return this.QueryInterface(iid); + }, + + QueryInterface: ChromeUtils.generateQI([ + "nsIChannelEventSink", + "nsIAuthPrompt2", + ]), +}; + +function fetchstatus(host) { + return new Promise((resolve, reject) => { + let xhr = new XMLHttpRequest(); + let uri = "https://" + host.name + "/"; + + xhr.open("head", uri, true); + xhr.setRequestHeader("X-Automated-Tool", TOOL_SOURCE); + xhr.timeout = REQUEST_TIMEOUT; + + let errorHandler = () => { + dump("ERROR: exception making request to " + host.name + "\n"); + resolve( + processStsHeader( + host, + null, + xhr.status, + xhr.channel && xhr.channel.securityInfo + ) + ); + }; + + xhr.onerror = errorHandler; + xhr.ontimeout = errorHandler; + xhr.onabort = errorHandler; + + xhr.onload = () => { + let header = xhr.getResponseHeader("strict-transport-security"); + resolve( + processStsHeader(host, header, xhr.status, xhr.channel.securityInfo) + ); + }; + + xhr.channel.notificationCallbacks = new RedirectAndAuthStopper(); + xhr.send(); + }); +} + +async function getHSTSStatus(host) { + do { + host = await fetchstatus(host); + } while (shouldRetry(host)); + return host; +} + +function compareHSTSStatus(a, b) { + if (a.name > b.name) { + return 1; + } + if (a.name < b.name) { + return -1; + } + return 0; +} + +function writeTo(string, fos) { + fos.write(string, string.length); +} + +// Determines and returns a string representing a declaration of when this +// preload list should no longer be used. +// This is the current time plus MINIMUM_REQUIRED_MAX_AGE. +function getExpirationTimeString() { + let now = new Date(); + let nowMillis = now.getTime(); + // MINIMUM_REQUIRED_MAX_AGE is in seconds, so convert to milliseconds + let expirationMillis = nowMillis + MINIMUM_REQUIRED_MAX_AGE * 1000; + let expirationMicros = expirationMillis * 1000; + return ( + "const PRTime gPreloadListExpirationTime = INT64_C(" + + expirationMicros + + ");\n" + ); +} + +function shouldRetry(response) { + return ( + response.error != ERROR_NO_HSTS_HEADER && + response.error != ERROR_MAX_AGE_TOO_LOW && + response.error != ERROR_NONE && + response.retries > 0 + ); +} + +// Copied from browser/components/migration/MigrationUtils.sys.mjs +function spinResolve(promise) { + if (!(promise instanceof Promise)) { + return promise; + } + let done = false; + let result = null; + let error = null; + promise + .catch(e => { + error = e; + }) + .then(r => { + result = r; + done = true; + }); + + Services.tm.spinEventLoopUntil( + "getHSTSPreloadList.js:spinResolve", + () => done + ); + if (error) { + throw error; + } else { + return result; + } +} + +async function probeHSTSStatuses(inHosts) { + let totalLength = inHosts.length; + dump("Examining " + totalLength + " hosts.\n"); + + // Make requests in batches of MAX_CONCURRENT_REQUESTS. Otherwise, we have + // too many in-flight requests and the time it takes to process them causes + // them all to time out. + let allResults = []; + while (inHosts.length) { + let promises = []; + for (let i = 0; i < MAX_CONCURRENT_REQUESTS && inHosts.length; i++) { + let host = inHosts.shift(); + promises.push(getHSTSStatus(host)); + } + let results = await Promise.all(promises); + let progress = ( + (100 * (totalLength - inHosts.length)) / + totalLength + ).toFixed(2); + dump(progress + "% done\n"); + allResults = allResults.concat(results); + } + + dump("HSTS Probe received " + allResults.length + " statuses.\n"); + return allResults; +} + +function readCurrentList(filename) { + var currentHosts = {}; + var file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile); + file.initWithPath(filename); + var fis = Cc["@mozilla.org/network/file-input-stream;1"].createInstance( + Ci.nsILineInputStream + ); + fis.init(file, -1, -1, Ci.nsIFileInputStream.CLOSE_ON_EOF); + var line = {}; + + // While we generate entries matching the latest version format, + // we still need to be able to read entries in the previous version formats + // for bootstrapping a latest version preload list from a previous version + // preload list. Hence these regexes. + const entryRegexes = [ + /([^,]+), (0|1)/, // v3 + / {2}\/\* "([^"]*)", (true|false) \*\//, // v2 + / {2}{ "([^"]*)", (true|false) },/, // v1 + ]; + + while (fis.readLine(line)) { + let match; + entryRegexes.find(r => { + match = r.exec(line.value); + return match; + }); + if (match) { + currentHosts[match[1]] = match[2] == "1" || match[2] == "true"; + } + } + return currentHosts; +} + +function combineLists(newHosts, currentHosts) { + let newHostsSet = new Set(); + + for (let newHost of newHosts) { + newHostsSet.add(newHost.name); + } + + for (let currentHost in currentHosts) { + if (!newHostsSet.has(currentHost)) { + newHosts.push({ name: currentHost, retries: MAX_RETRIES }); + } + } +} + +const TEST_ENTRIES = [ + { + name: "includesubdomains.preloaded.test", + includeSubdomains: true, + }, + { + name: "includesubdomains2.preloaded.test", + includeSubdomains: true, + }, + { + name: "noincludesubdomains.preloaded.test", + includeSubdomains: false, + }, +]; + +function deleteTestHosts(currentHosts) { + for (let testEntry of TEST_ENTRIES) { + delete currentHosts[testEntry.name]; + } +} + +function getTestHosts() { + let hosts = []; + for (let testEntry of TEST_ENTRIES) { + hosts.push({ + name: testEntry.name, + maxAge: MINIMUM_REQUIRED_MAX_AGE, + includeSubdomains: testEntry.includeSubdomains, + error: ERROR_NONE, + // This deliberately doesn't have a value for `retries` (because we should + // never attempt to connect to this host). + forceInclude: true, + }); + } + return hosts; +} + +async function insertHosts(inoutHostList, inAddedHosts) { + for (let host of inAddedHosts) { + inoutHostList.push(host); + } +} + +function filterForcedInclusions(inHosts, outNotForced, outForced) { + // Apply our filters (based on policy today) to determine which entries + // will be included without being checked (forced); the others will be + // checked using active probing. + for (let host of inHosts) { + if ( + host.policy == "google" || + host.policy == "public-suffix" || + host.policy == "public-suffix-requested" + ) { + host.forceInclude = true; + host.error = ERROR_NONE; + outForced.push(host); + } else { + outNotForced.push(host); + } + } +} + +function output(statuses) { + dump("INFO: Writing output to " + OUTPUT + "\n"); + try { + let file = new FileUtils.File( + PathUtils.join(Services.dirsvc.get("CurWorkD", Ci.nsIFile).path, OUTPUT) + ); + let fos = FileUtils.openSafeFileOutputStream(file); + writeTo(HEADER, fos); + writeTo(getExpirationTimeString(), fos); + + writeTo(GPERF_DELIM, fos); + + for (let status of statuses) { + let includeSubdomains = status.includeSubdomains ? 1 : 0; + writeTo(status.name + ", " + includeSubdomains + "\n", fos); + } + + writeTo(GPERF_DELIM, fos); + FileUtils.closeSafeFileOutputStream(fos); + dump("finished writing output file\n"); + } catch (e) { + dump("ERROR: problem writing output to '" + OUTPUT + "': " + e + "\n"); + throw e; + } +} + +function errorToString(status) { + return status.error == ERROR_MAX_AGE_TOO_LOW + ? status.error + status.maxAge + : status.error; +} + +async function main(args) { + if (args.length != 1) { + throw new Error( + "Usage: getHSTSPreloadList.js <absolute path to current nsSTSPreloadList.inc>" + ); + } + + // get the current preload list + let currentHosts = readCurrentList(args[0]); + // delete any hosts we use in tests so we don't actually connect to them + deleteTestHosts(currentHosts); + // disable the current preload list so it won't interfere with requests we make + Services.prefs.setBoolPref( + "network.stricttransportsecurity.preloadlist", + false + ); + // download and parse the raw json file from the Chromium source + let rawdata = download(); + // get just the hosts with mode: "force-https" + let hosts = getHosts(rawdata); + // add hosts in the current list to the new list (avoiding duplicates) + combineLists(hosts, currentHosts); + + // Don't contact hosts that are forced to be included anyway + let hostsToContact = []; + let forcedHosts = []; + filterForcedInclusions(hosts, hostsToContact, forcedHosts); + + // Initialize the final status list + let hstsStatuses = []; + // Add the hosts we use in tests + dump("Adding test hosts\n"); + insertHosts(hstsStatuses, getTestHosts()); + // Add in the hosts that are forced + dump("Adding forced hosts\n"); + insertHosts(hstsStatuses, forcedHosts); + + let total = await probeHSTSStatuses(hostsToContact) + .then(function(probedStatuses) { + return hstsStatuses.concat(probedStatuses); + }) + .then(function(statuses) { + return statuses.sort(compareHSTSStatus); + }) + .then(function(statuses) { + for (let status of statuses) { + // If we've encountered an error for this entry (other than the site not + // sending an HSTS header), be safe and don't remove it from the list + // (given that it was already on the list). + if ( + !status.forceInclude && + status.error != ERROR_NONE && + status.error != ERROR_NO_HSTS_HEADER && + status.error != ERROR_MAX_AGE_TOO_LOW && + status.name in currentHosts + ) { + // dump("INFO: error connecting to or processing " + status.name + " - using previous status on list\n"); + status.maxAge = MINIMUM_REQUIRED_MAX_AGE; + status.includeSubdomains = currentHosts[status.name]; + } + } + return statuses; + }) + .then(function(statuses) { + // Filter out entries we aren't including. + var includedStatuses = statuses.filter(function(status) { + if (status.maxAge < MINIMUM_REQUIRED_MAX_AGE && !status.forceInclude) { + // dump("INFO: " + status.name + " NOT ON the preload list\n"); + return false; + } + + // dump("INFO: " + status.name + " ON the preload list (includeSubdomains: " + status.includeSubdomains + ")\n"); + if (status.forceInclude && status.error != ERROR_NONE) { + dump( + status.name + + ": " + + errorToString(status) + + " (error ignored - included regardless)\n" + ); + } + return true; + }); + return includedStatuses; + }); + + // Write the output file + output(total); + + dump("HSTS probing all done\n"); +} + +// arguments is a global within xpcshell +spinResolve(main(arguments)); diff --git a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh new file mode 100755 index 0000000000..b88ee476da --- /dev/null +++ b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh @@ -0,0 +1,618 @@ +#!/bin/bash + +set -ex + +function usage { + cat <<EOF + +Usage: $(basename "$0") -h # Displays this usage/help text +Usage: $(basename "$0") -x # lists exit codes +Usage: $(basename "$0") [-p product] + [-r existing_repo_dir] + # Use mozilla-central builds to check HSTS & HPKP + [--use-mozilla-central] + # Use archive.m.o instead of the taskcluster index to get xpcshell + [--use-ftp-builds] + # One (or more) of the following actions must be specified. + --hsts | --hpkp | --remote-settings | --suffix-list + -b branch + +EOF +} + +PRODUCT="firefox" +BRANCH="" +PLATFORM_EXT="tar.bz2" +UNPACK_CMD="tar jxf" +CLOSED_TREE=false +DONTBUILD=false +APPROVAL=false +COMMIT_AUTHOR='ffxbld <ffxbld@mozilla.com>' +REPODIR='' +HGHOST="hg.mozilla.org" +STAGEHOST="archive.mozilla.org" +WGET="wget -nv" +UNTAR="tar -zxf" +DIFF="$(command -v diff) -u" +BASEDIR="${HOME}" + +SCRIPTDIR="$(realpath "$(dirname "$0")")" +HG="$(command -v hg)" +DATADIR="${BASEDIR}/data" +mkdir -p "${DATADIR}" + +USE_MC=false +USE_TC=true +JQ="$(command -v jq)" + +DO_HSTS=false +HSTS_PRELOAD_SCRIPT="${SCRIPTDIR}/getHSTSPreloadList.js" +HSTS_PRELOAD_ERRORS="nsSTSPreloadList.errors" +HSTS_PRELOAD_INC_OLD="${DATADIR}/nsSTSPreloadList.inc" +HSTS_PRELOAD_INC_NEW="${BASEDIR}/${PRODUCT}/nsSTSPreloadList.inc" +HSTS_UPDATED=false + +DO_HPKP=false +HPKP_PRELOAD_SCRIPT="${SCRIPTDIR}/genHPKPStaticPins.js" +HPKP_PRELOAD_ERRORS="StaticHPKPins.errors" +HPKP_PRELOAD_JSON="${DATADIR}/PreloadedHPKPins.json" +HPKP_PRELOAD_INC="StaticHPKPins.h" +HPKP_PRELOAD_INPUT="${DATADIR}/${HPKP_PRELOAD_INC}" +HPKP_PRELOAD_OUTPUT="${DATADIR}/${HPKP_PRELOAD_INC}.out" +HPKP_UPDATED=false + +DO_REMOTE_SETTINGS=false +REMOTE_SETTINGS_SERVER='' +REMOTE_SETTINGS_INPUT="${DATADIR}/remote-settings.in" +REMOTE_SETTINGS_OUTPUT="${DATADIR}/remote-settings.out" +REMOTE_SETTINGS_DIR="/services/settings/dumps" +REMOTE_SETTINGS_UPDATED=false + +DO_SUFFIX_LIST=false +GITHUB_SUFFIX_URL="https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat" +GITHUB_SUFFIX_LOCAL="public_suffix_list.dat" +HG_SUFFIX_LOCAL="effective_tld_names.dat" +HG_SUFFIX_PATH="/netwerk/dns/${HG_SUFFIX_LOCAL}" +SUFFIX_LIST_UPDATED=false + +ARTIFACTS_DIR="${ARTIFACTS_DIR:-.}" +# Defaults +HSTS_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${HSTS_DIFF_ARTIFACT:-"nsSTSPreloadList.diff"}" +HPKP_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${HPKP_DIFF_ARTIFACT:-"StaticHPKPins.h.diff"}" +REMOTE_SETTINGS_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${REMOTE_SETTINGS_DIFF_ARTIFACT:-"remote-settings.diff"}" +SUFFIX_LIST_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${SUFFIX_LIST_DIFF_ARTIFACT:-"effective_tld_names.diff"}" + +# duplicate the functionality of taskcluster-lib-urls, but in bash.. +queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" +index_base="$TASKCLUSTER_ROOT_URL/api/index/v1" + +# Cleanup common artifacts. +function preflight_cleanup { + cd "${BASEDIR}" + rm -rf "${PRODUCT}" tests "${BROWSER_ARCHIVE}" "${TESTS_ARCHIVE}" +} + +function download_shared_artifacts_from_ftp { + cd "${BASEDIR}" + + # Download everything we need to run js with xpcshell + echo "INFO: Downloading all the necessary pieces from ${STAGEHOST}..." + ARTIFACT_DIR="nightly/latest-${REPODIR}" + if [ "${USE_MC}" == "true" ]; then + ARTIFACT_DIR="nightly/latest-mozilla-central" + fi + + BROWSER_ARCHIVE_URL="https://${STAGEHOST}/pub/mozilla.org/${PRODUCT}/${ARTIFACT_DIR}/${BROWSER_ARCHIVE}" + TESTS_ARCHIVE_URL="https://${STAGEHOST}/pub/mozilla.org/${PRODUCT}/${ARTIFACT_DIR}/${TESTS_ARCHIVE}" + + echo "INFO: ${WGET} ${BROWSER_ARCHIVE_URL}" + ${WGET} "${BROWSER_ARCHIVE_URL}" + echo "INFO: ${WGET} ${TESTS_ARCHIVE_URL}" + ${WGET} "${TESTS_ARCHIVE_URL}" +} + +function download_shared_artifacts_from_tc { + cd "${BASEDIR}" + TASKID_FILE="taskId.json" + + # Download everything we need to run js with xpcshell + echo "INFO: Downloading all the necessary pieces from the taskcluster index..." + TASKID_URL="$index_base/task/gecko.v2.${REPODIR}.shippable.latest.${PRODUCT}.linux64-opt" + if [ "${USE_MC}" == "true" ]; then + TASKID_URL="$index_base/task/gecko.v2.mozilla-central.shippable.latest.${PRODUCT}.linux64-opt" + fi + ${WGET} -O ${TASKID_FILE} "${TASKID_URL}" + INDEX_TASK_ID="$($JQ -r '.taskId' ${TASKID_FILE})" + if [ -z "${INDEX_TASK_ID}" ]; then + echo "Failed to look up taskId at ${TASKID_URL}" + exit 22 + else + echo "INFO: Got taskId of $INDEX_TASK_ID" + fi + + TASKSTATUS_FILE="taskstatus.json" + STATUS_URL="$queue_base/task/${INDEX_TASK_ID}/status" + ${WGET} -O "${TASKSTATUS_FILE}" "${STATUS_URL}" + LAST_RUN_INDEX=$(($(jq '.status.runs | length' ${TASKSTATUS_FILE}) - 1)) + echo "INFO: Examining run number ${LAST_RUN_INDEX}" + + BROWSER_ARCHIVE_URL="$queue_base/task/${INDEX_TASK_ID}/runs/${LAST_RUN_INDEX}/artifacts/public/build/${BROWSER_ARCHIVE}" + echo "INFO: ${WGET} ${BROWSER_ARCHIVE_URL}" + ${WGET} "${BROWSER_ARCHIVE_URL}" + + TESTS_ARCHIVE_URL="$queue_base/task/${INDEX_TASK_ID}/runs/${LAST_RUN_INDEX}/artifacts/public/build/${TESTS_ARCHIVE}" + echo "INFO: ${WGET} ${TESTS_ARCHIVE_URL}" + ${WGET} "${TESTS_ARCHIVE_URL}" +} + +function unpack_artifacts { + cd "${BASEDIR}" + if [ ! -f "${BROWSER_ARCHIVE}" ]; then + echo "Downloaded file '${BROWSER_ARCHIVE}' not found in directory '$(pwd)'." >&2 + exit 31 + fi + if [ ! -f "${TESTS_ARCHIVE}" ]; then + echo "Downloaded file '${TESTS_ARCHIVE}' not found in directory '$(pwd)'." >&2 + exit 32 + fi + # Unpack the browser and move xpcshell in place for updating the preload list. + echo "INFO: Unpacking resources..." + ${UNPACK_CMD} "${BROWSER_ARCHIVE}" + mkdir -p tests + cd tests + ${UNTAR} "../${TESTS_ARCHIVE}" + cd "${BASEDIR}" + cp tests/bin/xpcshell "${PRODUCT}" +} + +# Downloads the current in-tree HSTS (HTTP Strict Transport Security) files. +# Runs a simple xpcshell script to generate up-to-date HSTS information. +# Compares the new HSTS output with the old to determine whether we need to update. +function compare_hsts_files { + cd "${BASEDIR}" + + HSTS_PRELOAD_INC_HG="${HGREPO}/raw-file/default/security/manager/ssl/$(basename "${HSTS_PRELOAD_INC_OLD}")" + + echo "INFO: Downloading existing include file..." + rm -rf "${HSTS_PRELOAD_ERRORS}" "${HSTS_PRELOAD_INC_OLD}" + echo "INFO: ${WGET} ${HSTS_PRELOAD_INC_HG}" + ${WGET} -O "${HSTS_PRELOAD_INC_OLD}" "${HSTS_PRELOAD_INC_HG}" + + if [ ! -f "${HSTS_PRELOAD_INC_OLD}" ]; then + echo "Downloaded file '${HSTS_PRELOAD_INC_OLD}' not found in directory '$(pwd)' - this should have been downloaded above from ${HSTS_PRELOAD_INC_HG}." >&2 + exit 41 + fi + + # Run the script to get an updated preload list. + echo "INFO: Generating new HSTS preload list..." + cd "${BASEDIR}/${PRODUCT}" + if ! LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:. ./xpcshell "${HSTS_PRELOAD_SCRIPT}" "${HSTS_PRELOAD_INC_OLD}"; then + echo "HSTS preload list generation failed" >&2 + exit 43 + fi + + # The created files should be non-empty. + echo "INFO: Checking whether new HSTS preload list is valid..." + if [ ! -s "${HSTS_PRELOAD_INC_NEW}" ]; then + echo "New HSTS preload list ${HSTS_PRELOAD_INC_NEW} is empty. That's less good." >&2 + exit 42 + fi + cd "${BASEDIR}" + + # Check for differences + echo "INFO: diffing old/new HSTS preload lists into ${HSTS_DIFF_ARTIFACT}" + ${DIFF} "${HSTS_PRELOAD_INC_OLD}" "${HSTS_PRELOAD_INC_NEW}" | tee "${HSTS_DIFF_ARTIFACT}" + if [ -s "${HSTS_DIFF_ARTIFACT}" ] + then + return 0 + fi + return 1 +} + +# Downloads the current in-tree HPKP (HTTP public key pinning) files. +# Runs a simple xpcshell script to generate up-to-date HPKP information. +# Compares the new HPKP output with the old to determine whether we need to update. +function compare_hpkp_files { + cd "${BASEDIR}" + HPKP_PRELOAD_JSON_HG="${HGREPO}/raw-file/default/security/manager/tools/$(basename "${HPKP_PRELOAD_JSON}")" + + HPKP_PRELOAD_OUTPUT_HG="${HGREPO}/raw-file/default/security/manager/ssl/${HPKP_PRELOAD_INC}" + + rm -f "${HPKP_PRELOAD_OUTPUT}" + ${WGET} -O "${HPKP_PRELOAD_INPUT}" "${HPKP_PRELOAD_OUTPUT_HG}" + ${WGET} -O "${HPKP_PRELOAD_JSON}" "${HPKP_PRELOAD_JSON_HG}" + + # Run the script to get an updated preload list. + echo "INFO: Generating new HPKP preload list..." + cd "${BASEDIR}/${PRODUCT}" + if ! LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:. ./xpcshell "${HPKP_PRELOAD_SCRIPT}" "${HPKP_PRELOAD_JSON}" "${HPKP_PRELOAD_OUTPUT}" > "${HPKP_PRELOAD_ERRORS}"; then + echo "HPKP preload list generation failed" >&2 + exit 54 + fi + + # The created files should be non-empty. + echo "INFO: Checking whether new HPKP preload list is valid..." + + if [ ! -s "${HPKP_PRELOAD_OUTPUT}" ]; then + echo "${HPKP_PRELOAD_OUTPUT} is empty. That's less good." >&2 + exit 52 + fi + if ! grep kPreloadPKPinsExpirationTime "${HPKP_PRELOAD_OUTPUT}"; then + echo "${HPKP_PRELOAD_OUTPUT} is missing an expiration time. Truncated?" >&2 + exit 53 + fi + cd "${BASEDIR}" + + echo "INFO: diffing old/new HPKP preload lists..." + ${DIFF} "${HPKP_PRELOAD_INPUT}" "${HPKP_PRELOAD_OUTPUT}" | tee "${HPKP_DIFF_ARTIFACT}" + if [ -s "${HPKP_DIFF_ARTIFACT}" ] + then + return 0 + fi + return 1 +} + +function is_valid_xml { + xmlfile=$1 + XMLLINT=$(command -v xmllint 2>/dev/null | head -n1) + + if [ ! -x "${XMLLINT}" ]; then + echo "ERROR: xmllint not found in PATH" + exit 60 + fi + ${XMLLINT} --nonet --noout "${xmlfile}" +} + +# Downloads the public suffix list +function compare_suffix_lists { + HG_SUFFIX_URL="${HGREPO}/raw-file/default/${HG_SUFFIX_PATH}" + cd "${BASEDIR}" + + echo "INFO: ${WGET} -O ${GITHUB_SUFFIX_LOCAL} ${GITHUB_SUFFIX_URL}" + rm -f "${GITHUB_SUFFIX_LOCAL}" + ${WGET} -O "${GITHUB_SUFFIX_LOCAL}" "${GITHUB_SUFFIX_URL}" + + echo "INFO: ${WGET} -O ${HG_SUFFIX_LOCAL} ${HG_SUFFIX_URL}" + rm -f "${HG_SUFFIX_LOCAL}" + ${WGET} -O "${HG_SUFFIX_LOCAL}" "${HG_SUFFIX_URL}" + + echo "INFO: diffing in-tree suffix list against the suffix list from AMO..." + ${DIFF} ${GITHUB_SUFFIX_LOCAL} ${HG_SUFFIX_LOCAL} | tee "${SUFFIX_LIST_DIFF_ARTIFACT}" + if [ -s "${SUFFIX_LIST_DIFF_ARTIFACT}" ] + then + return 0 + fi + return 1 +} + +function compare_remote_settings_files { + REMOTE_SETTINGS_SERVER="https://firefox.settings.services.mozilla.com/v1" + + # 1. List remote settings collections from server. + echo "INFO: fetch remote settings list from server" + ${WGET} -qO- "${REMOTE_SETTINGS_SERVER}/buckets/monitor/collections/changes/records" |\ + ${JQ} -r '.data[] | .bucket+"/"+.collection+"/"+(.last_modified|tostring)' |\ + # 2. For each entry ${bucket, collection, last_modified} + while IFS="/" read -r bucket collection last_modified; do + + # 3. Download the dump from HG into REMOTE_SETTINGS_INPUT folder + hg_dump_url="${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${bucket}/${collection}.json" + local_location_input="$REMOTE_SETTINGS_INPUT/${bucket}/${collection}.json" + mkdir -p "$REMOTE_SETTINGS_INPUT/${bucket}" + ${WGET} -qO "$local_location_input" "$hg_dump_url" + if [ $? -eq 8 ]; then + # We don't keep any dump for this collection, skip it. + # Try to clean up in case no collection in this bucket has dump. + rmdir "$REMOTE_SETTINGS_INPUT/${bucket}" --ignore-fail-on-non-empty + continue + fi + + # 4. Download server version into REMOTE_SETTINGS_OUTPUT folder + remote_records_url="$REMOTE_SETTINGS_SERVER/buckets/${bucket}/collections/${collection}/changeset?_expected=${last_modified}" + local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json" + mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}" + ${WGET} -qO- "$remote_records_url" | ${JQ} '{"data": .changes, "timestamp": .timestamp}' > "${local_location_output}" + + # 5. Download attachments if needed. + if [ "${bucket}" = "blocklists" ] && [ "${collection}" = "addons-bloomfilters" ]; then + # Find the attachment with the most recent generation_time, like _updateMLBF in Blocklist.jsm. + # The server should return one "bloomfilter-base" record, but in case it returns multiple, + # return the most recent one. The server may send multiple entries if we ever decide to use + # the "filter_expression" feature of Remote Settings to send different records to specific + # channels. In that case this code should be updated to recognize the filter expression, + # but until we do, simply select the most recent record - can't go wrong with that. + # Note that "attachment_type" and "generation_time" are specific to addons-bloomfilters. + update_remote_settings_attachment "${bucket}" "${collection}" addons-mlbf.bin \ + 'map(select(.attachment_type == "bloomfilter-base")) | sort_by(.generation_time) | last' + fi + # Here is an example to download an attachment with record identifier "ID": + # update_remote_settings_attachment "${bucket}" "${collection}" ID '.[] | select(.id == "ID")' + # NOTE: The downloaded data is not validated. xpcshell should be used for that. + done + + echo "INFO: diffing old/new remote settings dumps..." + ${DIFF} -r "${REMOTE_SETTINGS_INPUT}" "${REMOTE_SETTINGS_OUTPUT}" > "${REMOTE_SETTINGS_DIFF_ARTIFACT}" + if [ -s "${REMOTE_SETTINGS_DIFF_ARTIFACT}" ] + then + return 0 + fi + return 1 +} + +# Helper for compare_remote_settings_files to download attachments from remote settings. +# The format and location is documented at: +# https://firefox-source-docs.mozilla.org/services/common/services/RemoteSettings.html#packaging-attachments +function update_remote_settings_attachment() { + local bucket=$1 + local collection=$2 + local attachment_id=$3 + # $4 is a jq filter on the arrays that should return one record with the attachment + local jq_attachment_selector=".data | map(select(.attachment)) | $4" + + # These paths match _readAttachmentDump in services/settings/Attachments.jsm. + local path_to_attachment="${bucket}/${collection}/${attachment_id}" + local path_to_meta="${bucket}/${collection}/${attachment_id}.meta.json" + local old_meta="$REMOTE_SETTINGS_INPUT/${path_to_meta}" + local new_meta="$REMOTE_SETTINGS_OUTPUT/${path_to_meta}" + + # Those files should have been created by compare_remote_settings_files before the function call. + local local_location_input="$REMOTE_SETTINGS_INPUT/${bucket}/${collection}.json" + local local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json" + + # Compute the metadata based on already-downloaded records. + mkdir -p "$REMOTE_SETTINGS_INPUT/${bucket}/${collection}" + ${JQ} -cj <"$local_location_input" "${jq_attachment_selector}" > "${old_meta}" + mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}" + ${JQ} -cj <"$local_location_output" "${jq_attachment_selector}" > "${new_meta}" + + if cmp --silent "${old_meta}" "${new_meta}" ; then + # Metadata not changed, don't bother downloading the attachments themselves. + return + fi + # Metadata changed. Download attachments. + + echo "INFO: Downloading updated remote settings dump: ${bucket}/${collection}/${attachment_id}" + + # Overwrited old_meta with the actual file from the repo. The content should be equivalent, + # but can have minor differences (e.g. different line endings) if the checked in file was not + # generated by this script (e.g. manually checked in). + ${WGET} -qO "${old_meta}" "${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${path_to_meta}" + + ${WGET} -qO "${REMOTE_SETTINGS_INPUT}/${path_to_attachment}" "${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${path_to_attachment}" + + if [ -z "${ATTACHMENT_BASE_URL}" ] ; then + ATTACHMENT_BASE_URL=$(${WGET} -qO- "${REMOTE_SETTINGS_SERVER}" | ${JQ} -r .capabilities.attachments.base_url) + fi + attachment_path_from_meta=$(${JQ} -r < "${new_meta}" .attachment.location) + ${WGET} -qO "${REMOTE_SETTINGS_OUTPUT}/${path_to_attachment}" "${ATTACHMENT_BASE_URL}${attachment_path_from_meta}" +} + +# Clones an hg repo +function clone_repo { + cd "${BASEDIR}" + if [ ! -d "${REPODIR}" ]; then + ${HG} robustcheckout --sharebase /tmp/hg-store -b default "${HGREPO}" "${REPODIR}" + fi + + ${HG} -R "${REPODIR}" pull + ${HG} -R "${REPODIR}" update -C default +} + +# Copies new HSTS files in place, and commits them. +function stage_hsts_files { + cd "${BASEDIR}" + cp -f "${HSTS_PRELOAD_INC_NEW}" "${REPODIR}/security/manager/ssl/" +} + +function stage_hpkp_files { + cd "${BASEDIR}" + cp -f "${HPKP_PRELOAD_OUTPUT}" "${REPODIR}/security/manager/ssl/${HPKP_PRELOAD_INC}" +} + +function stage_remote_settings_files { + cd "${BASEDIR}" + cp -a "${REMOTE_SETTINGS_OUTPUT}"/* "${REPODIR}${REMOTE_SETTINGS_DIR}" +} + +function stage_tld_suffix_files { + cd "${BASEDIR}" + cp -a "${GITHUB_SUFFIX_LOCAL}" "${REPODIR}/${HG_SUFFIX_PATH}" +} + +# Push all pending commits to Phabricator +function push_repo { + cd "${REPODIR}" + if [ ! -r "${HOME}/.arcrc" ] + then + return 1 + fi + if ! ARC=$(command -v arc) + then + return 1 + fi + if [ -z "${REVIEWERS}" ] + then + return 1 + fi + # Clean up older review requests + # Turn Needs Review D624: No bug, Automated HSTS ... + # into D624 + for diff in $($ARC list | grep "Needs Review" | grep -E "${BRANCH} repo-update" | awk 'match($0, /D[0-9]+[^: ]/) { print substr($0, RSTART, RLENGTH) }') + do + echo "Removing old request $diff" + # There is no 'arc abandon', see bug 1452082 + echo '{"transactions": [{"type":"abandon", "value": true}], "objectIdentifier": "'"${diff}"'"}' | $ARC call-conduit -- differential.revision.edit + done + + $ARC diff --verbatim --reviewers "${REVIEWERS}" +} + + + +# Main + +# Parse our command-line options. +while [ $# -gt 0 ]; do + case "$1" in + -h) usage; exit 0 ;; + -p) PRODUCT="$2"; shift ;; + -b) BRANCH="$2"; shift ;; + -n) DRY_RUN=true ;; + -c) CLOSED_TREE=true ;; + -d) DONTBUILD=true ;; + -a) APPROVAL=true ;; + --pinset) DO_PRELOAD_PINSET=true ;; + --hsts) DO_HSTS=true ;; + --hpkp) DO_HPKP=true ;; + --remote-settings) DO_REMOTE_SETTINGS=true ;; + --suffix-list) DO_SUFFIX_LIST=true ;; + -r) REPODIR="$2"; shift ;; + --use-mozilla-central) USE_MC=true ;; + --use-ftp-builds) USE_TC=false ;; + -*) usage + exit 11 ;; + *) break ;; # terminate while loop + esac + shift +done + +# Must supply a code branch to work with. +if [ "${BRANCH}" == "" ]; then + echo "Error: You must specify a branch with -b branchname." >&2 + usage + exit 12 +fi + +# Must choose at least one update action. +if [ "$DO_HSTS" == "false" ] && [ "$DO_HPKP" == "false" ] && [ "$DO_REMOTE_SETTINGS" == "false" ] && [ "$DO_SUFFIX_LIST" == "false" ] +then + echo "Error: you must specify at least one action from: --hsts, --hpkp, --remote-settings, or --suffix-list" >&2 + usage + exit 13 +fi + +# per-product constants +case "${PRODUCT}" in + thunderbird) + COMMIT_AUTHOR="tbirdbld <tbirdbld@thunderbird.net>" + ;; + firefox) + ;; + *) + echo "Error: Invalid product specified" + usage + exit 14 + ;; +esac + +if [ "${REPODIR}" == "" ]; then + REPODIR="$(basename "${BRANCH}")" +fi + +case "${BRANCH}" in + mozilla-central|comm-central|try ) + HGREPO="https://${HGHOST}/${BRANCH}" + ;; + mozilla-*|comm-* ) + HGREPO="https://${HGHOST}/releases/${BRANCH}" + ;; + * ) + HGREPO="https://${HGHOST}/projects/${BRANCH}" + ;; +esac + +BROWSER_ARCHIVE="target.${PLATFORM_EXT}" +TESTS_ARCHIVE="target.common.tests.tar.gz" + +preflight_cleanup +if [ "${DO_HSTS}" == "true" ] || [ "${DO_HPKP}" == "true" ] || [ "${DO_PRELOAD_PINSET}" == "true" ] +then + if [ "${USE_TC}" == "true" ]; then + download_shared_artifacts_from_tc + else + download_shared_artifacts_from_ftp + fi + unpack_artifacts +fi + +if [ "${DO_HSTS}" == "true" ]; then + if compare_hsts_files + then + HSTS_UPDATED=true + fi +fi +if [ "${DO_HPKP}" == "true" ]; then + if compare_hpkp_files + then + HPKP_UPDATED=true + fi +fi +if [ "${DO_REMOTE_SETTINGS}" == "true" ]; then + if compare_remote_settings_files + then + REMOTE_SETTINGS_UPDATED=true + fi +fi +if [ "${DO_SUFFIX_LIST}" == "true" ]; then + if compare_suffix_lists + then + SUFFIX_LIST_UPDATED=true + fi +fi + + +if [ "${HSTS_UPDATED}" == "false" ] && [ "${HPKP_UPDATED}" == "false" ] && [ "${REMOTE_SETTINGS_UPDATED}" == "false" ] && [ "${SUFFIX_LIST_UPDATED}" == "false" ]; then + echo "INFO: no updates required. Exiting." + exit 0 +else + if [ "${DRY_RUN}" == "true" ]; then + echo "INFO: Updates are available, not updating hg in dry-run mode." + exit 2 + fi +fi + +clone_repo + +COMMIT_MESSAGE="No Bug, ${BRANCH} repo-update" +if [ "${HSTS_UPDATED}" == "true" ] +then + stage_hsts_files + COMMIT_MESSAGE="${COMMIT_MESSAGE} HSTS" +fi + +if [ "${HPKP_UPDATED}" == "true" ] +then + stage_hpkp_files + COMMIT_MESSAGE="${COMMIT_MESSAGE} HPKP" +fi + +if [ "${REMOTE_SETTINGS_UPDATED}" == "true" ] +then + stage_remote_settings_files + COMMIT_MESSAGE="${COMMIT_MESSAGE} remote-settings" +fi + +if [ "${SUFFIX_LIST_UPDATED}" == "true" ] +then + stage_tld_suffix_files + COMMIT_MESSAGE="${COMMIT_MESSAGE} tld-suffixes" +fi + + +if [ ${DONTBUILD} == true ]; then + COMMIT_MESSAGE="${COMMIT_MESSAGE} - (DONTBUILD)" +fi +if [ ${CLOSED_TREE} == true ]; then + COMMIT_MESSAGE="${COMMIT_MESSAGE} - CLOSED TREE" +fi +if [ ${APPROVAL} == true ]; then + COMMIT_MESSAGE="${COMMIT_MESSAGE} - a=repo-update" +fi + + +if ${HG} -R "${REPODIR}" commit -u "${COMMIT_AUTHOR}" -m "${COMMIT_MESSAGE}" +then + push_repo +fi + +echo "All done" diff --git a/taskcluster/docker/periodic-updates/setup.sh b/taskcluster/docker/periodic-updates/setup.sh new file mode 100755 index 0000000000..d05c60883c --- /dev/null +++ b/taskcluster/docker/periodic-updates/setup.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set -ve + +apt-get update -q +apt-get install \ + arcanist \ + curl \ + jq \ + libasound2 \ + libdbus-glib-1-2 \ + libgtk-3-0 \ + libx11-xcb1 \ + libxml2-utils \ + libxt6 \ + libxtst6 \ + shellcheck \ + unzip \ + bzip2 \ + wget + +rm -rf /setup diff --git a/taskcluster/docker/push-to-try/Dockerfile b/taskcluster/docker/push-to-try/Dockerfile new file mode 100644 index 0000000000..4725c7e48b --- /dev/null +++ b/taskcluster/docker/push-to-try/Dockerfile @@ -0,0 +1,22 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +FROM $DOCKER_IMAGE_PARENT + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +RUN apt-get update && \ + apt-get install \ + openssh-client + +COPY known_hosts /etc/ssh/ssh_known_hosts +COPY hgrc /etc/mercurial/hgrc.d/push-to-try.rc + +RUN hg clone -r 19a77a5d7b2a90ba225cf5015afd1ac44787abec https://hg.mozilla.org/hgcustom/version-control-tools /builds/worker/.mozbuild/version-control-tools + +# Create directory for scripts to put an hg config in, for configuring ssh +RUN mkdir /builds/worker/.config/hg +RUN chown -R worker:worker /builds/worker/.config/hg diff --git a/taskcluster/docker/push-to-try/hgrc b/taskcluster/docker/push-to-try/hgrc new file mode 100644 index 0000000000..d1a0318499 --- /dev/null +++ b/taskcluster/docker/push-to-try/hgrc @@ -0,0 +1,2 @@ +[extensions] +push-to-try = ~/.mozbuild/version-control-tools/hgext/push-to-try diff --git a/taskcluster/docker/push-to-try/known_hosts b/taskcluster/docker/push-to-try/known_hosts new file mode 100644 index 0000000000..1dc2e39e70 --- /dev/null +++ b/taskcluster/docker/push-to-try/known_hosts @@ -0,0 +1,2 @@ +hg.mozilla.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDEsS2fK+TVkHl4QvvOHB6R5xxngsSYJR+pA4+xDhw4mZT9tgCRU9BBG3LazSLp6PUxnpfok78475/tx6Z8QwbTyUTmLElZ9Z9eJzjaGz/olHzQSWv0VB3kT+VZt0LK7pEuaG+Ph/qwxbtUZZOApYLEvu8uctDlS66doofxZylbsgl1kpRQ5HNu+/DgVo9K9dyMOm9OLoy4tXHSE5pofn4tKYdFRa2lt6OVtIP5/hKNb2i0+JmgM8C3bJTPvzJ4C8p2h83ro29XPUkNAfWrgD5CmAPPqHFXyefDCfdefcvI8B8Za9v4j4LynBDZHsGfII+wIfzyLIxy9K6Op6nqDZgCciBRdgxh4uZQINEhB/JJP03Pxo42ExdG28oU3aL8kRRTORT5ehFtImFfr9QESHaUnbVzBbU5DmOB5voYDMle3RgyY+RXJ7+4OxjLRnJvGks9QCn8QrIvabs/PTCnenI8+yDhMlLUkWTiR4JK8vDBYB2Rm++EmVsN9WjllfDNg3Aj1aYe8XiBD4tS+lg7Ur4rJL8X20H4yMvq56sQ0qfH8PCIQGyGL725E7Yuwj/MHvou5xrPM/Lqo/MtX5T2njrzkeaBmI/zFJaLwbphdrwmrzepbcim7OYJFF2pz8u56KDPD1pUQ7C1gEIAx/4mHiDOGCYooSvyfD+JRdjkZUZMiQ== +hg.mozilla.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGuRsL+/OrvIekv3iBST1zdVBLBuh/DMIj+ZN72N9a0g diff --git a/taskcluster/docker/recipes/common.sh b/taskcluster/docker/recipes/common.sh new file mode 100644 index 0000000000..0182154495 --- /dev/null +++ b/taskcluster/docker/recipes/common.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +tooltool_fetch() { + cat >manifest.tt + python3 /setup/tooltool.py fetch + rm manifest.tt +} diff --git a/taskcluster/docker/recipes/debian-test-system-setup.sh b/taskcluster/docker/recipes/debian-test-system-setup.sh new file mode 100644 index 0000000000..a0023220a9 --- /dev/null +++ b/taskcluster/docker/recipes/debian-test-system-setup.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +set -ve + +test "$(whoami)" == 'root' + +mkdir -p /setup +cd /setup + +apt_packages=() + +apt_packages+=('autoconf2.13') +apt_packages+=('bluez-cups') +apt_packages+=('build-essential') +apt_packages+=('ccache') +apt_packages+=('curl') +apt_packages+=('fonts-kacst') +apt_packages+=('fonts-kacst-one') +apt_packages+=('fonts-liberation') +apt_packages+=('fonts-stix') +apt_packages+=('fonts-unfonts-core') +apt_packages+=('fonts-unfonts-extra') +apt_packages+=('fonts-vlgothic') +apt_packages+=('g++-multilib') +apt_packages+=('gcc-multilib') +apt_packages+=('gir1.2-gnomebluetooth-1.0') +apt_packages+=('gnome-keyring') +apt_packages+=('libasound2-dev') +apt_packages+=('libcanberra-pulse') +apt_packages+=('libcurl4-openssl-dev') +apt_packages+=('libdbus-1-dev') +apt_packages+=('libdbus-glib-1-dev') +apt_packages+=('libgconf2-dev') +apt_packages+=('libiw-dev') +apt_packages+=('libnotify-dev') +apt_packages+=('libpulse-dev') +apt_packages+=('libsox-fmt-alsa') +apt_packages+=('libxt-dev') +apt_packages+=('libxxf86vm1') +apt_packages+=('llvm') +apt_packages+=('llvm-dev') +apt_packages+=('llvm-runtime') +apt_packages+=('locales') +apt_packages+=('locales-all') +apt_packages+=('net-tools') +apt_packages+=('openssh-client') +apt_packages+=('qemu-kvm') +apt_packages+=('rlwrap') +apt_packages+=('screen') +apt_packages+=('software-properties-common') +apt_packages+=('sudo') +apt_packages+=('tar') +apt_packages+=('ttf-dejavu') +apt_packages+=('unzip') +apt_packages+=('uuid') +apt_packages+=('wget') +apt_packages+=('xdg-user-dirs') +apt_packages+=('xvfb') +apt_packages+=('zip') + +# use Ubuntu's Python-2.7 (2.7.3 on Precise) +apt_packages+=('python-dev') +apt_packages+=('python-pip') + +apt-get update +apt-get install "${apt_packages[@]}" + +dpkg-reconfigure locales + +# pip 19.3 is causing errors building the docker image, pin to 19.2.3 for now. +# See https://github.com/pypa/pip/issues/7206 +pip install --upgrade pip==19.2.3 +hash -r +pip install virtualenv==15.2.0 + +# clean up +apt-get autoremove + +# We don't need no docs! +rm -rf /usr/share/help /usr/share/doc /usr/share/man + +cd / +rm -rf /setup ~/.ccache ~/.cache ~/.npm +rm -f "$0" diff --git a/taskcluster/docker/recipes/dot-config/pip/pip.conf b/taskcluster/docker/recipes/dot-config/pip/pip.conf new file mode 100644 index 0000000000..73c2b2a52c --- /dev/null +++ b/taskcluster/docker/recipes/dot-config/pip/pip.conf @@ -0,0 +1,2 @@ +[global] +disable-pip-version-check = true diff --git a/taskcluster/docker/recipes/hgrc b/taskcluster/docker/recipes/hgrc new file mode 100755 index 0000000000..f6a2f6643c --- /dev/null +++ b/taskcluster/docker/recipes/hgrc @@ -0,0 +1,33 @@ +# By default the progress bar starts after 3s and updates every 0.1s. We +# change this so it shows and updates every 1.0s. +# We also tell progress to assume a TTY is present so updates are printed +# even if there is no known TTY. +[progress] +delay = 1.0 +refresh = 1.0 +assume-tty = true + +[extensions] +share = +sparse = +robustcheckout = /usr/local/mercurial/robustcheckout.py + +[hostsecurity] +# When running a modern Python, Mercurial will default to TLS 1.1+. +# When running on a legacy Python, Mercurial will default to TLS 1.0+. +# There is no good reason we shouldn't be running a modern Python +# capable of speaking TLS 1.2. And the only Mercurial servers we care +# about should be running TLS 1.2. So make TLS 1.2 the minimum. +minimumprotocol = tls1.2 + +# Settings to make 1-click loaners more useful. +[extensions] +histedit = +rebase = + +[diff] +git = 1 +showfunc = 1 + +[pager] +pager = LESS=FRSXQ less diff --git a/taskcluster/docker/recipes/install-node.sh b/taskcluster/docker/recipes/install-node.sh new file mode 100644 index 0000000000..daeba2edc9 --- /dev/null +++ b/taskcluster/docker/recipes/install-node.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script installs Node v16. +# XXX For now, this should match the version installed in +# taskcluster/scripts/misc/repack-node.sh. Later we'll get the ESLint builder +# to use the linux64-node toolchain directly. + +wget -O node.xz --progress=dot:mega https://nodejs.org/dist/v16.19.0/node-v16.19.0-linux-x64.tar.xz +echo 'c88b52497ab38a3ddf526e5b46a41270320409109c3f74171b241132984fd08f' node.xz | sha256sum -c +tar -C /usr/local -xJ --strip-components 1 < node.xz +node -v # verify +npm -v diff --git a/taskcluster/docker/recipes/setup_packages.sh b/taskcluster/docker/recipes/setup_packages.sh new file mode 100755 index 0000000000..48dbf493f3 --- /dev/null +++ b/taskcluster/docker/recipes/setup_packages.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +TASKCLUSTER_ROOT_URL=$1 +shift + +# duplicate the functionality of taskcluster-lib-urls, but in bash.. +queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" + + +for task in "$@"; do + echo "adding package source $queue_base/task/$task/artifacts/public/build/" + echo "deb [trusted=yes] $queue_base/task/$task/artifacts/public/build/ apt/" > "/etc/apt/sources.list.d/99$task.list" +done diff --git a/taskcluster/docker/recipes/ubuntu1804-test-system-setup-base.sh b/taskcluster/docker/recipes/ubuntu1804-test-system-setup-base.sh new file mode 100644 index 0000000000..cb934529bf --- /dev/null +++ b/taskcluster/docker/recipes/ubuntu1804-test-system-setup-base.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash + +set -ve + +test "$(whoami)" == 'root' + +# We do want to install recommended packages. +sed -i /APT::Install-Recommends/d /etc/apt/apt.conf.d/99taskcluster + +# To speed up docker image build times as well as number of network/disk I/O +# build a list of packages to be installed nad call it in one go. +apt_packages=() + +apt_packages+=('autoconf2.13') +apt_packages+=('bluez-cups') +apt_packages+=('build-essential') +apt_packages+=('ca-certificates') +apt_packages+=('ccache') +apt_packages+=('compiz') +apt_packages+=('curl') +apt_packages+=('fontconfig') +apt_packages+=('fonts-kacst') +apt_packages+=('fonts-kacst-one') +apt_packages+=('fonts-liberation') +apt_packages+=('fonts-stix') +apt_packages+=('fonts-unfonts-core') +apt_packages+=('fonts-unfonts-extra') +apt_packages+=('fonts-vlgothic') +apt_packages+=('g++-multilib') +apt_packages+=('gcc-multilib') +apt_packages+=('gir1.2-gnomebluetooth-1.0') +apt_packages+=('git') +apt_packages+=('gnome-icon-theme') +apt_packages+=('gstreamer1.0-gtk3') +apt_packages+=('gstreamer1.0-plugins-base') +apt_packages+=('gstreamer1.0-plugins-good') +apt_packages+=('gstreamer1.0-tools') +apt_packages+=('gstreamer1.0-pulseaudio') +apt_packages+=('language-pack-en-base') +apt_packages+=('libc6-dbg') +apt_packages+=('libasound2-dev') +apt_packages+=('libavcodec-extra57') +apt_packages+=('libcanberra-gtk3-module') +apt_packages+=('libcanberra-pulse') +apt_packages+=('libcurl4-openssl-dev') +apt_packages+=('libdbus-1-dev') +apt_packages+=('libdbus-glib-1-dev') +apt_packages+=('libfreetype6') +apt_packages+=('libgconf2-dev') +apt_packages+=('libgl1-mesa-dri') +apt_packages+=('libgl1-mesa-glx') +apt_packages+=('libgstreamer-plugins-base1.0-dev') +apt_packages+=('libgstreamer1.0-dev') +apt_packages+=('libgtk-3-0') +apt_packages+=('libiw-dev') +apt_packages+=('libx11-xcb1') +apt_packages+=('libxcb1') +apt_packages+=('libxcb-render0') +apt_packages+=('libxcb-shm0') +apt_packages+=('libxcb-glx0') +apt_packages+=('libxcb-shape0') +apt_packages+=('libnotify-dev') +apt_packages+=('libpulse0') +apt_packages+=('libpulse-dev') +apt_packages+=('libxt-dev') +apt_packages+=('libxxf86vm1') +apt_packages+=('llvm') +apt_packages+=('llvm-dev') +apt_packages+=('llvm-runtime') +apt_packages+=('mesa-common-dev') +apt_packages+=('net-tools') +apt_packages+=('pulseaudio') +apt_packages+=('pulseaudio-module-bluetooth') +apt_packages+=('pulseaudio-module-gconf') +apt_packages+=('python-dev') +apt_packages+=('python-pip') +apt_packages+=('python3-pip') +apt_packages+=('qemu-kvm') +apt_packages+=('rlwrap') +apt_packages+=('screen') +apt_packages+=('software-properties-common') +apt_packages+=('sudo') +apt_packages+=('ttf-dejavu') +apt_packages+=('ubuntu-desktop') +apt_packages+=('unzip') +apt_packages+=('uuid') +apt_packages+=('wget') +apt_packages+=('xvfb') +apt_packages+=('xwit') +apt_packages+=('yasm') +apt_packages+=('zip') + +# xvinfo for test-linux.sh to monitor Xvfb startup +apt_packages+=('x11-utils') + +# Bug 1232407 - this allows the user to start vnc +apt_packages+=('x11vnc') + +# Bug 1176031 - need `xset` to disable screensavers +apt_packages+=('x11-xserver-utils') + +# Build a list of packages to install from the multiverse repo. +apt_packages+=('ubuntu-restricted-extras') + +# APT update takes very long on Ubuntu. Run it at the last possible minute. +apt-get update + +# Also force the cleanup after installation of packages to reduce image size. +apt-get install --allow-downgrades "${apt_packages[@]}" + +# Enable i386 packages +dpkg --add-architecture i386 +apt-get update + +# Make sure we have libraries for 32-bit tests +apt_packages=() +apt_packages+=('libavcodec-extra57:i386') +apt_packages+=('libdbus-glib-1-2:i386') +apt_packages+=('libpulse0:i386') +apt_packages+=('libxt6:i386') +apt_packages+=('libxtst6:i386') +apt_packages+=('libsecret-1-0:i386') +apt_packages+=('libgtk-3-0:i386') +apt_packages+=('libx11-xcb1:i386') +apt_packages+=('libxcb1:i386') +apt_packages+=('libasound2:i386') + +apt-get install --allow-downgrades "${apt_packages[@]}" +rm -rf /var/lib/apt/lists/* + +# Build a list of packages to purge from the image. +apt_packages=() +apt_packages+=('*cheese*') +apt_packages+=('example-content') +apt_packages+=('git') +apt_packages+=('gnome-calendar') +apt_packages+=('gnome-initial-setup') +apt_packages+=('gnome-mahjongg') +apt_packages+=('gnome-mines') +apt_packages+=('gnome-sudoku') +apt_packages+=('libx11-doc') +apt_packages+=('manpages-dev') +apt_packages+=('orca') +apt_packages+=('rhythmbox') +apt_packages+=('thunderbird') +apt_packages+=('ubuntu-release-upgrader*') +apt_packages+=('update-manager-core') +apt_packages+=('update-manager') +apt_packages+=('yelp') + +# Purge unnecessary packages +apt-get purge "${apt_packages[@]}" + +# We don't need no docs! +rm -rf /usr/share/help /usr/share/doc /usr/share/man + +# Remove all locale files other than en_US.UTF-8 +rm -rf /usr/share/locale/ /usr/share/locale-langpack/ /usr/share/locales/ + +# Further cleanup +apt-get autoremove + +rm -f "$0" diff --git a/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh b/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh new file mode 100644 index 0000000000..e22b21f51d --- /dev/null +++ b/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -ve + +test "$(whoami)" == 'root' + +cd /setup + +# Install tooltool and node now that dependencies are in place. +. /setup/common.sh +. /setup/install-node.sh + +# Upgrade pip and install virtualenv to specified versions. +pip install --upgrade pip==19.2.3 +hash -r +pip install virtualenv==15.2.0 + +pip3 install -r /setup/psutil_requirements.txt +pip install -r /setup/psutil_requirements.txt + +# Cleanup +cd / +rm -rf /setup ~/.ccache ~/.cache ~/.npm +rm -f "$0" diff --git a/taskcluster/docker/recipes/xvfb.sh b/taskcluster/docker/recipes/xvfb.sh new file mode 100644 index 0000000000..6e0e79f7d4 --- /dev/null +++ b/taskcluster/docker/recipes/xvfb.sh @@ -0,0 +1,75 @@ +#! /bin/bash -x + +set -x + +fail() { + echo # make sure error message is on a new line + echo "[xvfb.sh:error]" "${@}" + exit 1 +} + +cleanup_xvfb() { + # When you call this script with START_VNC or TASKCLUSTER_INTERACTIVE + # we make sure we do not kill xvfb so you do not lose your connection + local xvfb_pid=`pidof Xvfb` + local vnc=${START_VNC:-false} + local interactive=${TASKCLUSTER_INTERACTIVE:-false} + if [ -n "$xvfb_pid" ] && [[ $vnc == false ]] && [[ $interactive == false ]] ; then + kill $xvfb_pid || true + screen -XS xvfb quit || true + fi +} + +# Attempt to start xvfb in a screen session with the given resolution and display +# number. Up to 5 attempts will be made to start xvfb with a short delay +# between retries +try_xvfb() { + screen -dmS xvfb Xvfb :$2 -nolisten tcp -screen 0 $1 \ + > ~/artifacts/xvfb/xvfb.log 2>&1 + export DISPLAY=:$2 + + # Only error code 255 matters, because it signifies that no + # display could be opened. As long as we can open the display + # tests should work. We'll retry a few times with a sleep before + # failing. + local retry_count=0 + local max_retries=5 + xvfb_test=0 + until [ $retry_count -gt $max_retries ]; do + xvinfo || xvfb_test=$? + if [ $xvfb_test != 255 ]; then + retry_count=$(($max_retries + 1)) + else + retry_count=$(($retry_count + 1)) + echo "Failed to start Xvfb, retry: $retry_count" + sleep 2 + fi + done + if [ $xvfb_test == 255 ]; then + return 1 + else + return 0 + fi +} + +start_xvfb() { + set +e + mkdir -p ~/artifacts/xvfb + local retry_count=0 + local max_retries=2 + local success=1 + until [ $retry_count -gt $max_retries ]; do + try_xvfb $1 $2 + success=$? + if [ $success -eq 0 ]; then + retry_count=$(($max_retries + 1)) + else + retry_count=$(($retry_count + 1)) + sleep 10 + fi + done + set -e + if [ $success -eq 1 ]; then + fail "Could not start xvfb after ${max_retries} attempts" + fi +} diff --git a/taskcluster/docker/sentry/Dockerfile b/taskcluster/docker/sentry/Dockerfile new file mode 100644 index 0000000000..2f606c5529 --- /dev/null +++ b/taskcluster/docker/sentry/Dockerfile @@ -0,0 +1,11 @@ +FROM $DOCKER_IMAGE_PARENT +LABEL maintainer="Mitchell Hentges mhentges@mozilla.com" + +VOLUME /builds/worker/checkouts + +ADD prepare.sh /setup/prepare-docker.sh +ADD submit_sentry_release.sh /usr/bin/submit_sentry_release.sh +RUN /bin/bash /setup/prepare-docker.sh && rm -R /setup + +# Set a default command useful for debugging +CMD ["/bin/sh", "--login"] diff --git a/taskcluster/docker/sentry/prepare.sh b/taskcluster/docker/sentry/prepare.sh new file mode 100755 index 0000000000..7cce3e1955 --- /dev/null +++ b/taskcluster/docker/sentry/prepare.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +set -o errexit +set -o nounset +set -o xtrace + +apt-get -y update + +# Install: +# * curl to fetch sentry-cli +# * jq to parse hgmo pushlog +apt-get install -y \ + curl \ + jq + +# Install sentry-cli to publish releases +curl -L https://github.com/getsentry/sentry-cli/releases/download/1.63.1/sentry-cli-Linux-x86_64 -o /usr/bin/sentry-cli +chmod +x /usr/bin/sentry-cli diff --git a/taskcluster/docker/sentry/submit_sentry_release.sh b/taskcluster/docker/sentry/submit_sentry_release.sh new file mode 100644 index 0000000000..136e06a58a --- /dev/null +++ b/taskcluster/docker/sentry/submit_sentry_release.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -o nounset +set -o pipefail + +run() { + revisions=$(curl "$HG_PUSHLOG_URL" | jq -c -r ".pushes[].changesets | @sh" | tr -d \') || return 1 + sentry_api_key=$(curl "http://taskcluster/secrets/v1/secret/$SENTRY_SECRET" | jq -r ".secret.sentryToken") || return 1 + for revision in $revisions; do + SENTRY_AUTH_TOKEN=$sentry_api_key SENTRY_ORG=mozilla sentry-cli --url https://sentry.io/ releases --project mach new "hg-rev-$revision" || return 1 + done +} + +with_backoff() { + failures=0 + while ! "$@"; do + failures=$(( failures + 1 )) + if (( failures >= 5 )); then + echo "[with_backoff] Unable to succeed after 5 tries, failing the job." + return 1 + else + seconds=$(( 2 ** (failures - 1) )) + echo "[with_backoff] Retrying in $seconds second(s)" + sleep $seconds + fi + done +} + +with_backoff run diff --git a/taskcluster/docker/static-analysis-build/Dockerfile b/taskcluster/docker/static-analysis-build/Dockerfile new file mode 100644 index 0000000000..e53ff4fa2e --- /dev/null +++ b/taskcluster/docker/static-analysis-build/Dockerfile @@ -0,0 +1,61 @@ +FROM $DOCKER_IMAGE_PARENT +LABEL maintainer="Andi-Bogdan Postelnicu <andi@mozilla.com>" + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +ENV XZ_OPT=-T0 + +ARG TASKCLUSTER_ROOT_URL +ARG DOCKER_IMAGE_PACKAGES +RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES + +RUN apt-get update && \ + apt-get install \ + autoconf2.13 \ + automake \ + bison \ + bzip2 \ + cmake \ + flex \ + curl \ + libsqlite3-dev \ + file \ + gawk \ + gcc-multilib \ + gnupg \ + jq \ + libc6-dev \ + libstdc++-*-dev \ + libmpfr-dev \ + nasm \ + opam \ + patch \ + p7zip-full \ + procps \ + python3-dev \ + rsync \ + screen \ + tar \ + unzip \ + uuid \ + valgrind \ + wget \ + zip \ + zlib1g-dev \ + x11-utils \ + xvfb \ + linux-libc-dev \ + libdbus-glib-1-dev \ + libfontconfig1-dev \ + libfreetype6-dev \ + libgconf2-dev \ + libgmp-dev \ + libgtk-3-dev \ + libpango1.0-dev \ + libpulse-dev \ + libx11-xcb-dev \ + libxt-dev \ + lib32z1 \ + patchelf diff --git a/taskcluster/docker/system-symbols-linux-scraper/Dockerfile b/taskcluster/docker/system-symbols-linux-scraper/Dockerfile new file mode 100644 index 0000000000..1b2c340ff0 --- /dev/null +++ b/taskcluster/docker/system-symbols-linux-scraper/Dockerfile @@ -0,0 +1,28 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Gabriele Svelto <gsvelto@mozilla.com> + +RUN mkdir -p /builds +RUN id worker || useradd -d /builds/worker -s /bin/bash -m worker +WORKDIR /builds/worker + +# We need to declare all potentially cache volumes as caches. Also, +# making high I/O paths volumes increase I/O throughput because of +# AUFS slowness. +VOLUME /builds/worker/checkouts + +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + binutils build-essential cpio curl elfutils jq libxml2-utils \ + python3-pip rpm2cpio squashfs-tools unzip wget zip && \ + apt-get autoremove -y && rm -rf /var/lib/apt/lists/* + +COPY run.sh /builds/worker/run.sh + +COPY SHA256SUMS.txt /builds/worker/ + +RUN chown -R worker:worker /builds/worker + +RUN pip3 install crashstats-tools==1.3.0 + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/system-symbols-linux-scraper/SHA256SUMS.txt b/taskcluster/docker/system-symbols-linux-scraper/SHA256SUMS.txt new file mode 100644 index 0000000000..46a456d21a --- /dev/null +++ b/taskcluster/docker/system-symbols-linux-scraper/SHA256SUMS.txt @@ -0,0 +1,6 @@ +b4ba66e218aa6f27e19bfbc48142f86f3fddb64b6540fd5beb30bad11f73a409 arch/SHA256SUMS.zip +6083ab8cb46a5be2a4d9bc51dce1443375f71d9536c325da39a14ba68c1de81a debian/SHA256SUMS.zip +64003a50133900e5d01b29e37022b3029652788a0e094b67330e5e32f1bf5e12 fedora/SHA256SUMS.zip +00e3e54788b2d7867535eb25bc9ce41541dc6299dca0a8ce8d0d4e5b49839ac3 mint/SHA256SUMS.zip +c4d02cc24f2a50b0caea20dd5fd0cd6bc960d0a619e8a355f7fec14a766e7966 opensuse/SHA256SUMS.zip +aed77c596f5652f4ecdc48b37aaa1c2b0b516c7e74ccc0cf183e7d7de101b6d7 ubuntu/SHA256SUMS.zip diff --git a/taskcluster/docker/system-symbols-linux-scraper/run.sh b/taskcluster/docker/system-symbols-linux-scraper/run.sh new file mode 100755 index 0000000000..d7837d112e --- /dev/null +++ b/taskcluster/docker/system-symbols-linux-scraper/run.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +set -ex + +SHA256SUMS=SHA256SUMS.zip + +function get_route() +{ + local task_url=${TASKCLUSTER_ROOT_URL}/api/queue/v1/task/${TASK_ID} + local payload + payload=$(curl -sSL "${task_url}") + + local route + route=$(echo "${payload}" | jq -r '.routes[] | select(contains("latest")) | select(contains("pushdate") | not) ' | sed -e 's/^index\.//') + echo "${route}" +} + +function get_sha256sum_url() +{ + local route + route=$(get_route) + local sha256sums_url=${TASKCLUSTER_ROOT_URL}/api/index/v1/task/${route}/artifacts/public/build/${SHA256SUMS} + echo "${sha256sums_url}" +} + +function has_sha256sums_on_index() +{ + local url + url=$(get_sha256sum_url) + curl -sSL --head --fail -o /dev/null "${url}" +} + +function download_verify_extract_sha256sums() +{ + local url=$1 + + if [ ! -f "/builds/worker/SHA256SUMS.txt" ]; then + echo "Missing checksums, aborting." + exit 1 + fi + + curl -sSL "${url}" -o ${SHA256SUMS} + + # We verify SHA256SUMS when we bootstrap and thus download from GitHub + # The one downloaded from TaskCluster will have been updated by previous tasks + if [ -n "${BOOTSTRAP_SHA256}" ]; then + echo "BOOTSTRAP_SHA256 was set, ensuring consistent set of SHA256SUMS" + local sha_check + sha256sum --quiet --status -c <(grep "${DISTRO}/" /builds/worker/SHA256SUMS.txt | sed -e "s/${DISTRO}\///") + sha_check=$? + if [ ${sha_check} -ne 0 ]; then + echo "Checksum verification failed, aborting." + exit 1 + fi + fi + + unzip ${SHA256SUMS} && rm ${SHA256SUMS} +} + +DISTRO=$1 + +sha256=https://mozilla.github.io/symbol-scrapers/${DISTRO}/${SHA256SUMS} +if [ -z "${BOOTSTRAP_SHA256}" ]; then + if has_sha256sums_on_index; then + sha256=$(get_sha256sum_url) + fi +fi + +mkdir -p /builds/worker/artifacts/ + +pushd "${MOZ_FETCHES_DIR}/symbol-scrapers/${DISTRO}" + download_verify_extract_sha256sums "${sha256}" + DUMP_SYMS=${MOZ_FETCHES_DIR}/dump_syms/dump_syms /bin/bash script.sh + zip -r9 /builds/worker/artifacts/${SHA256SUMS} SHA256SUMS + cp wget*.log /builds/worker/artifacts/ +popd + +if [ ! -f "/builds/worker/artifacts/target.crashreporter-symbols.zip" ]; then + echo "No symbols zip produced, upload task will fail" +fi diff --git a/taskcluster/docker/system-symbols-mac/Dockerfile b/taskcluster/docker/system-symbols-mac/Dockerfile new file mode 100644 index 0000000000..5c35d40824 --- /dev/null +++ b/taskcluster/docker/system-symbols-mac/Dockerfile @@ -0,0 +1,35 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Gabriele Svelto <gsvelto@mozilla.com> + +RUN mkdir -p /builds +RUN id worker || useradd -d /builds/worker -s /bin/bash -m worker +WORKDIR /builds/worker + +# We need to declare all potentially cache volumes as caches. Also, +# making high I/O paths volumes increase I/O throughput because of +# AUFS slowness. +VOLUME /builds/worker/checkouts + +RUN apt-get update && \ + apt-get install --no-install-recommends -y python3-dev python3-pip python3-requests-futures python3-setuptools curl pax build-essential libxml2-dev libssl-dev zlib1g-dev libusb-dev cmake libbz2-dev libpng-dev wget zip liblzma-dev && \ + rm -rf /var/lib/apt/lists/* + +RUN mkdir /opt/data-reposado/ +RUN chown -R worker.worker /opt/data-reposado/ + +COPY setup.sh /setup/ +COPY requirements.txt /setup/ +# %include tools/crashreporter/system-symbols/mac + +RUN /bin/sh /setup/setup.sh + +ADD https://github.com/marco-c/breakpad-mac-update-symbols/raw/21221733edfbcac49d40e50fc219fab7d17437a0/lipo /usr/local/bin/ +RUN chmod +x /usr/local/bin/lipo + +COPY topsrcdir/tools/crashreporter/system-symbols/mac /builds/worker/mac-update-symbols +WORKDIR /builds/worker/mac-update-symbols + +RUN chown -R worker:worker /builds/worker + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/system-symbols-mac/requirements.txt b/taskcluster/docker/system-symbols-mac/requirements.txt new file mode 100644 index 0000000000..775756115c --- /dev/null +++ b/taskcluster/docker/system-symbols-mac/requirements.txt @@ -0,0 +1,2 @@ +futures +requests diff --git a/taskcluster/docker/system-symbols-mac/setup.sh b/taskcluster/docker/system-symbols-mac/setup.sh new file mode 100644 index 0000000000..c0c4e766ac --- /dev/null +++ b/taskcluster/docker/system-symbols-mac/setup.sh @@ -0,0 +1,27 @@ +#!/bin/sh +set -v -e -x + +ncpu=-j$(grep -c ^processor /proc/cpuinfo) + +WORK=/setup/ +cd $WORK +git clone --depth=1 --single-branch -b system-symbols-mac https://github.com/gabrielesvelto/libdmg-hfsplus.git +cd libdmg-hfsplus +cmake . +make "$ncpu" dmg-bin hfsplus +# `make install` installs way too much stuff +cp dmg/dmg hfs/hfsplus /builds/worker/bin +strip /builds/worker/bin/dmg /builds/worker/bin/hfsplus + +pip3 install --no-cache-dir git+https://github.com/mozilla/reposado + +python3 /usr/local/bin/repoutil --configure <<EOF +/opt/data-reposado/html/ +/opt/data-reposado/metadata/ +http://example.com/ +EOF + +pip3 install --no-cache-dir -r /setup/requirements.txt + +cd / +rm -rf /setup diff --git a/taskcluster/docker/system-symbols-win/Dockerfile b/taskcluster/docker/system-symbols-win/Dockerfile new file mode 100644 index 0000000000..93a91bf199 --- /dev/null +++ b/taskcluster/docker/system-symbols-win/Dockerfile @@ -0,0 +1,26 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Gabriele Svelto <gsvelto@mozilla.com> + +RUN mkdir -p /builds +RUN id worker || useradd -d /builds/worker -s /bin/bash -m worker +WORKDIR /builds/worker + +# We need to declare all potentially cache volumes as caches. Also, +# making high I/O paths volumes increase I/O throughput because of +# AUFS slowness. +VOLUME /builds/worker/checkouts + +COPY requirements.txt /builds/worker/requirements.txt +RUN apt-get update && \ + apt-get install --no-install-recommends -y gcc python3-dev python3-pip python3-setuptools libffi-dev && \ + rm -rf /var/lib/apt/lists/* +RUN pip3 install pip==20.3.4 --upgrade +RUN pip3 install --no-cache-dir --require-hashes -r /builds/worker/requirements.txt + +# %include tools/crashreporter/system-symbols/win +COPY topsrcdir/tools/crashreporter/system-symbols/win /builds/worker + +RUN chown -R worker:worker /builds/worker + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/system-symbols-win/requirements.txt b/taskcluster/docker/system-symbols-win/requirements.txt new file mode 100644 index 0000000000..d717b6efda --- /dev/null +++ b/taskcluster/docker/system-symbols-win/requirements.txt @@ -0,0 +1,19 @@ +aiodns==3.0.0 --hash=sha256:2b19bc5f97e5c936638d28e665923c093d8af2bf3aa88d35c43417fa25d136a2 +aiofile==3.7.4 --hash=sha256:0e2a524e4714efda47ce8964b13d4da94cf553411f9f6da813df615a4cd73d95 +aiohttp==3.8.1 --hash=sha256:d15367ce87c8e9e09b0f989bfd72dc641bcd04ba091c68cd305312d00962addd +aiosignal==1.2.0 --hash=sha256:26e62109036cd181df6e6ad646f91f0dcfd05fe16d0cb924138ff2ab75d64e3a +asyncio==3.4.3 --hash=sha256:c4d18b22701821de07bd6aea8b53d21449ec0ec5680645e5317062ea21817d2d +asynctest==0.13.0 --hash=sha256:5da6118a7e6d6b54d83a8f7197769d046922a44d2a99c21382f0a6e4fadae676 +async-timeout==4.0.2 --hash=sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c +attrs==21.4.0 --hash=sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4 +caio==0.9.0 --hash=sha256:399de8f266ede4e87ffa1c647c8bb6859df14039354b6fcf74a47d3d756fdc41 +cffi==1.15.0 --hash=sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954 +chardet==4.0.0 --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 +charset-normalizer==2.0.12 --hash=sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df +frozenlist==1.3.0 --hash=sha256:acb267b09a509c1df5a4ca04140da96016f40d2ed183cdc356d237286c971b51 +idna==3.3 --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff +multidict==6.0.2 --hash=sha256:5ff3bd75f38e4c43f1f470f2df7a4d430b821c4ce22be384e1459cb57d6bb013 +pycares==4.1.2 --hash=sha256:dc942692fca0e27081b7bb414bb971d34609c80df5e953f6d0c62ecc8019acd9 +pycparser==2.21 --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 +typing_extensions==4.2.0 --hash=sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708 +yarl==1.7.2 --hash=sha256:f64394bd7ceef1237cc604b5a89bf748c95982a84bcd3c4bbeb40f685c810794 diff --git a/taskcluster/docker/toolchain-build/Dockerfile b/taskcluster/docker/toolchain-build/Dockerfile new file mode 100644 index 0000000000..d8af78319e --- /dev/null +++ b/taskcluster/docker/toolchain-build/Dockerfile @@ -0,0 +1,57 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Mike Hommey <mhommey@mozilla.com> + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +ENV XZ_OPT=-T0 + +ARG DOCKER_IMAGE_PACKAGES +ARG TASKCLUSTER_ROOT_URL +RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES + +RUN apt-get update && \ + apt-get install \ + autoconf \ + autoconf2.13 \ + automake \ + bison \ + build-essential \ + curl \ + clang-13 \ + cmake \ + flex \ + gawk \ + gcc-multilib \ + git \ + gnupg \ + jq \ + libbz2-dev \ + libssl-dev \ + libtool \ + libucl-dev \ + libxml2-dev \ + lld-13 \ + msitools \ + ninja-build \ + openssh-client \ + p7zip-full \ + pkg-config \ + procps \ + python3-distutils-extra \ + python3-requests \ + python3-toml \ + scons \ + subversion \ + tar \ + unzip \ + uuid \ + uuid-dev \ + wget \ + zip \ + zlib1g-dev + +# mmdebstrap is not available in jessie, but we only need it in the buster-based image. +# We also want zstd when available. +RUN apt-get install fakechroot fakeroot mmdebstrap zstd || true diff --git a/taskcluster/docker/ubuntu1804-test-base/Dockerfile b/taskcluster/docker/ubuntu1804-test-base/Dockerfile new file mode 100644 index 0000000000..a8556340e5 --- /dev/null +++ b/taskcluster/docker/ubuntu1804-test-base/Dockerfile @@ -0,0 +1,24 @@ +# This docker image only sets up packages on top of the original Ubuntu image. +# Everything else is setup is a child docker image derived from this one, such +# that changes to our scripts don't trigger the rebuild of this base image, +# possibly leading to breakage. (See bug 1503756 and bug 1678451 for examples +# of such recent breakages) +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Edwin Takahashi <egao@mozilla.com> + +ARG TASKCLUSTER_ROOT_URL +ARG DOCKER_IMAGE_PACKAGES +RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES + +# %include taskcluster/docker/recipes/ubuntu1804-test-system-setup-base.sh +ADD topsrcdir/taskcluster/docker/recipes/ubuntu1804-test-system-setup-base.sh /setup/system-setup.sh +RUN bash /setup/system-setup.sh + +# gnome-keyring-daemon is configured to have the IPC_LOCK capability (to lock pages with secrets in +# memory), but docker isn't run with that capability granted. So, if we were to try running +# gnome-keyring-daemon without first clearing the capability, it would just exit with the message +# "Operation not permitted". Luckily it doesn't actually require the capability. +RUN setcap -r /usr/bin/gnome-keyring-daemon + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/ubuntu1804-test/Dockerfile b/taskcluster/docker/ubuntu1804-test/Dockerfile new file mode 100644 index 0000000000..69385ca632 --- /dev/null +++ b/taskcluster/docker/ubuntu1804-test/Dockerfile @@ -0,0 +1,86 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Edwin Takahashi <egao@mozilla.com> + +# Create necessary directories +RUN mkdir -p /artifacts + +# We need to declare all potentially cache volumes as caches. Also, +# making high I/O paths volumes increase I/O throughput because of +# AUFS slowness. +VOLUME /builds/worker/.cache +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/tooltool-cache +VOLUME /builds/worker/workspace + +# In test.sh we accept START_VNC to start a vnc daemon. +# Exposing this port allows it to work. +EXPOSE 5900 + +# %include python/mozbuild/mozbuild/action/tooltool.py +ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py + +# %include taskcluster/docker/recipes/common.sh +ADD topsrcdir/taskcluster/docker/recipes/common.sh /setup/common.sh + +# %include taskcluster/docker/recipes/install-node.sh +ADD topsrcdir/taskcluster/docker/recipes/install-node.sh /setup/install-node.sh + +# Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb +# %include taskcluster/docker/recipes/xvfb.sh +ADD topsrcdir/taskcluster/docker/recipes/xvfb.sh /builds/worker/scripts/xvfb.sh + +ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /builds/worker/scripts/tooltool.py + +# %include build/psutil_requirements.txt +ADD topsrcdir/build/psutil_requirements.txt /setup/psutil_requirements.txt +# %include taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh +ADD topsrcdir/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh /setup/system-setup.sh +RUN bash /setup/system-setup.sh + +# %include taskcluster/scripts/tester/test-linux.sh +ADD topsrcdir/taskcluster/scripts/tester/test-linux.sh /builds/worker/bin/test-linux.sh + +# Set variable normally configured at login, by the shells parent process, these +# are taken from GNU su manual +ENV LANG en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 +ENV PATH $PATH:/builds/worker/bin + +# This helps not forgetting setting DISPLAY=:0 when running +# tests outside of test.sh +ENV DISPLAY :0 + +# Add utilities and configuration +COPY dot-files/config /builds/worker/.config + +# Disable apport (app crash reporter) to avoid stealing focus from test runs +ADD apport /etc/default/apport + +# Disable font antialiasing for now to match releng's setup +ADD fonts.conf /builds/worker/.fonts.conf + +# Bug 1345105 - Do not run periodical update checks and downloads +ADD autostart/gnome-software-service.desktop /etc/xdg/autostart/ + +# allow the worker user to access video devices +RUN usermod -a -G video worker + +# Set execution and ownership privileges +RUN chmod +x bin/*; chown -R worker:worker /builds/worker + +# Set up first-run experience for interactive mode +ADD motd /etc/taskcluster-motd +ADD taskcluster-interactive-shell /bin/taskcluster-interactive-shell +RUN chmod +x /bin/taskcluster-interactive-shell + +# Bug 1638183 - increase xserver maximum client count +COPY dot-files/config/xorg/99-serverflags.conf /usr/share/X11/xorg.conf.d/ + +# Ubuntu 18.04 (or any other GNOME3 based systems) needs dbus to have various +# test function as expected. Use entrypoint to initialize dbus as root. +COPY dbus.sh /usr/local/bin/dbus.sh +RUN chmod +x /usr/local/bin/dbus.sh +ENTRYPOINT ["/usr/local/bin/dbus.sh"] + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/ubuntu1804-test/apport b/taskcluster/docker/ubuntu1804-test/apport new file mode 100644 index 0000000000..42e5f8d3a1 --- /dev/null +++ b/taskcluster/docker/ubuntu1804-test/apport @@ -0,0 +1 @@ +enabled=0 diff --git a/taskcluster/docker/ubuntu1804-test/autostart/deja-dup-monitor.desktop b/taskcluster/docker/ubuntu1804-test/autostart/deja-dup-monitor.desktop new file mode 100644 index 0000000000..c3b8a4c676 --- /dev/null +++ b/taskcluster/docker/ubuntu1804-test/autostart/deja-dup-monitor.desktop @@ -0,0 +1,19 @@ +[Desktop Entry] +Version=1.0 +X-Ubuntu-Gettext-Domain=deja-dup + +Name=Backup Monitor +Comment=Schedules backups at regular intervals + +Icon=deja-dup +TryExec=/usr/lib/deja-dup/deja-dup/deja-dup-monitor +Exec=/usr/lib/deja-dup/deja-dup/deja-dup-monitor + +# Bug 984944/1240084 - It prevents taking screenshots +X-GNOME-Autostart-Delay=false + +StartupNotify=false +NoDisplay=true + +Type=Application +Categories=System;Utility;Archiving; diff --git a/taskcluster/docker/ubuntu1804-test/autostart/gnome-software-service.desktop b/taskcluster/docker/ubuntu1804-test/autostart/gnome-software-service.desktop new file mode 100644 index 0000000000..b563cc306b --- /dev/null +++ b/taskcluster/docker/ubuntu1804-test/autostart/gnome-software-service.desktop @@ -0,0 +1,9 @@ +[Desktop Entry] +Type=Application +Name=GNOME Software +Exec=/usr/bin/gnome-software --gapplication-service +OnlyShowIn=GNOME;Unity; +X-Ubuntu-Gettext-Domain=gnome-software + +# Bug 1345105 - Do not run periodical update checks and downloads +X-GNOME-Autostart-enabled=false diff --git a/taskcluster/docker/ubuntu1804-test/dbus.sh b/taskcluster/docker/ubuntu1804-test/dbus.sh new file mode 100644 index 0000000000..f27ec664f1 --- /dev/null +++ b/taskcluster/docker/ubuntu1804-test/dbus.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +/etc/init.d/dbus start 2>&1 + +exec "${@}" diff --git a/taskcluster/docker/ubuntu1804-test/dot-files/config/user-dirs.dirs b/taskcluster/docker/ubuntu1804-test/dot-files/config/user-dirs.dirs new file mode 100644 index 0000000000..2db2718d24 --- /dev/null +++ b/taskcluster/docker/ubuntu1804-test/dot-files/config/user-dirs.dirs @@ -0,0 +1,15 @@ +# This file is written by xdg-user-dirs-update +# If you want to change or add directories, just edit the line you're +# interested in. All local changes will be retained on the next run +# Format is XDG_xxx_DIR="$HOME/yyy", where yyy is a shell-escaped +# homedir-relative path, or XDG_xxx_DIR="/yyy", where /yyy is an +# absolute path. No other format is supported. + +XDG_DESKTOP_DIR="$HOME/Desktop" +XDG_DOWNLOAD_DIR="$HOME/Downloads" +XDG_TEMPLATES_DIR="$HOME/Templates" +XDG_PUBLICSHARE_DIR="$HOME/Public" +XDG_DOCUMENTS_DIR="$HOME/Documents" +XDG_MUSIC_DIR="$HOME/Music" +XDG_PICTURES_DIR="$HOME/Pictures" +XDG_VIDEOS_DIR="$HOME/Videos" diff --git a/taskcluster/docker/ubuntu1804-test/dot-files/config/user-dirs.locale b/taskcluster/docker/ubuntu1804-test/dot-files/config/user-dirs.locale new file mode 100644 index 0000000000..7741b83a3e --- /dev/null +++ b/taskcluster/docker/ubuntu1804-test/dot-files/config/user-dirs.locale @@ -0,0 +1 @@ +en_US diff --git a/taskcluster/docker/ubuntu1804-test/dot-files/config/xorg/99-serverflags.conf b/taskcluster/docker/ubuntu1804-test/dot-files/config/xorg/99-serverflags.conf new file mode 100644 index 0000000000..4c335f44e3 --- /dev/null +++ b/taskcluster/docker/ubuntu1804-test/dot-files/config/xorg/99-serverflags.conf @@ -0,0 +1,3 @@ +Section "ServerFlags" + Option "MaxClients" "2048" +EndSection
\ No newline at end of file diff --git a/taskcluster/docker/ubuntu1804-test/dot-files/pulse/client.conf b/taskcluster/docker/ubuntu1804-test/dot-files/pulse/client.conf new file mode 100644 index 0000000000..4b2dab2a66 --- /dev/null +++ b/taskcluster/docker/ubuntu1804-test/dot-files/pulse/client.conf @@ -0,0 +1 @@ +autospawn = no
\ No newline at end of file diff --git a/taskcluster/docker/ubuntu1804-test/fonts.conf b/taskcluster/docker/ubuntu1804-test/fonts.conf new file mode 100644 index 0000000000..9784fcc981 --- /dev/null +++ b/taskcluster/docker/ubuntu1804-test/fonts.conf @@ -0,0 +1,5 @@ +<match target="font"> + <edit name="antialias" mode="assign"> + <bool>false</bool> + </edit> +</match> diff --git a/taskcluster/docker/ubuntu1804-test/motd b/taskcluster/docker/ubuntu1804-test/motd new file mode 100644 index 0000000000..f958393cd8 --- /dev/null +++ b/taskcluster/docker/ubuntu1804-test/motd @@ -0,0 +1,6 @@ +Welcome to your taskcluster interactive shell! The regularly scheduled task +has been paused to give you a chance to set up your debugging environment. + +For your convenience, the exact mozharness command needed for this task can +be invoked using the 'run-mozharness' command. + diff --git a/taskcluster/docker/ubuntu1804-test/taskcluster-interactive-shell b/taskcluster/docker/ubuntu1804-test/taskcluster-interactive-shell new file mode 100644 index 0000000000..c782c0ea96 --- /dev/null +++ b/taskcluster/docker/ubuntu1804-test/taskcluster-interactive-shell @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +download() { + name=`basename $1` + url=${GECKO_HEAD_REPOSITORY}/raw-file/${GECKO_HEAD_REV}/$1 + if ! curl --fail --silent -o ./$name --retry 10 $url; then + fail "failed downloading $1 from ${GECKO_HEAD_REPOSITORY}" + fi +} + +cd $HOME/bin; +download taskcluster/scripts/tester/run-wizard; +chmod +x run-wizard; +./run-wizard; + +SPAWN="$SHELL"; +if [ "$SHELL" = "bash" ]; then + SPAWN="bash -li"; +fi; + +cd $HOME; +exec $SPAWN; diff --git a/taskcluster/docker/update-verify/Dockerfile b/taskcluster/docker/update-verify/Dockerfile new file mode 100644 index 0000000000..5cddeac1f0 --- /dev/null +++ b/taskcluster/docker/update-verify/Dockerfile @@ -0,0 +1,32 @@ +# This is an LTS! We should upgrade after the next LTS is released, unless +# we've switched to the in-tree debian base images by then. +FROM $DOCKER_IMAGE_PARENT + +MAINTAINER release@mozilla.com + +# We do want to install recommended packages. +RUN sed -i /APT::Install-Recommends/d /etc/apt/apt.conf.d/99taskcluster + +RUN dpkg --add-architecture i386 && apt-get -q update \ + # p7zip-full is for extracting Windows and OS X packages + # wget is for downloading update.xml, installers, and MARs + # libgtk-3-0 is required to run the Firefox updater + && apt-get -q --yes install p7zip-full wget libgtk-3-0 libgtk-3.0:i386 python3-cairo + +VOLUME /builds/worker/.cache +VOLUME /builds/worker/checkouts + +RUN mkdir /build +ADD system-setup.sh /tmp/system-setup.sh +RUN bash /tmp/system-setup.sh + +RUN chown -R worker:worker /builds/worker/bin && chmod 755 /builds/worker/bin/* +ENV PATH $PATH:/builds/worker/bin + +# Set variable normally configured at login, by the shells parent process, these +# are taken from GNU su manual +ENV LANG en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/update-verify/system-setup.sh b/taskcluster/docker/update-verify/system-setup.sh new file mode 100644 index 0000000000..8a55f255f8 --- /dev/null +++ b/taskcluster/docker/update-verify/system-setup.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -ve + +test "$(whoami)" == 'root' + +mkdir -p /setup +cd /setup + +apt_packages=() +apt_packages+=('curl') +apt_packages+=('locales') +apt_packages+=('python3-pip') +apt_packages+=('python3-aiohttp') +apt_packages+=('shellcheck') +apt_packages+=('sudo') + +apt-get update +apt-get install "${apt_packages[@]}" + +# Without this we get spurious "LC_ALL: cannot change locale (en_US.UTF-8)" errors, +# and python scripts raise UnicodeEncodeError when trying to print unicode characters. +locale-gen en_US.UTF-8 +dpkg-reconfigure locales + +su -c 'git config --global user.email "worker@mozilla.test"' worker +su -c 'git config --global user.name "worker"' worker + +rm -rf /setup diff --git a/taskcluster/docker/updatebot/Dockerfile b/taskcluster/docker/updatebot/Dockerfile new file mode 100644 index 0000000000..a58c9c07a8 --- /dev/null +++ b/taskcluster/docker/updatebot/Dockerfile @@ -0,0 +1,24 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Tom Ritter <tom@mozilla.com> + +VOLUME /builds/worker/checkouts + +# %include taskcluster/docker/recipes/install-node.sh +ADD topsrcdir/taskcluster/docker/recipes/install-node.sh /setup/install-node.sh + +COPY privileged-setup.sh /setup/privileged-setup.sh +COPY updatebot-version.sh /setup/updatebot-version.sh +COPY setup.sh /builds/worker/setup.sh +COPY run.py /builds/worker/run.py +COPY hgrc /etc/mercurial/hgrc.d/updatebot.rc + +RUN cd /setup && ./privileged-setup.sh + +ENV HOME /builds/worker +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME worker +ENV PYTHONUNBUFFERED 1 +ENV PATH "/builds/worker/go/bin:/builds/worker/fetches/rustc/bin:/builds/worker/fetches/cargo-vet:${PATH}" + +RUN cd /builds/worker && ./setup.sh diff --git a/taskcluster/docker/updatebot/VERSION b/taskcluster/docker/updatebot/VERSION new file mode 100644 index 0000000000..56a6051ca2 --- /dev/null +++ b/taskcluster/docker/updatebot/VERSION @@ -0,0 +1 @@ +1
\ No newline at end of file diff --git a/taskcluster/docker/updatebot/arcanist_patch_size.patch b/taskcluster/docker/updatebot/arcanist_patch_size.patch new file mode 100644 index 0000000000..7540c76fbd --- /dev/null +++ b/taskcluster/docker/updatebot/arcanist_patch_size.patch @@ -0,0 +1,13 @@ +diff --git a/src/workflow/ArcanistDiffWorkflow.php b/src/workflow/ArcanistDiffWorkflow.php +index 38aa4b62..c9b7e215 100644 +--- a/src/workflow/ArcanistDiffWorkflow.php ++++ b/src/workflow/ArcanistDiffWorkflow.php +@@ -899,7 +899,7 @@ EOTEXT + throw new Exception(pht('Repository API is not supported.')); + } + +- $limit = 1024 * 1024 * 4; ++ $limit = 1024 * 1024 * 12; + foreach ($changes as $change) { + $size = 0; + foreach ($change->getHunks() as $hunk) { diff --git a/taskcluster/docker/updatebot/arcanist_windows_stream.patch b/taskcluster/docker/updatebot/arcanist_windows_stream.patch new file mode 100644 index 0000000000..953fbdce85 --- /dev/null +++ b/taskcluster/docker/updatebot/arcanist_windows_stream.patch @@ -0,0 +1,19 @@ +diff --git a/src/channel/PhutilSocketChannel.php b/src/channel/PhutilSocketChannel.php +index 4bd2a47a..2ad3fd85 100644 +--- a/src/channel/PhutilSocketChannel.php ++++ b/src/channel/PhutilSocketChannel.php +@@ -46,9 +46,11 @@ final class PhutilSocketChannel extends PhutilChannel { + if (!$socket) { + continue; + } +- $ok = stream_set_blocking($socket, false); +- if (!$ok) { +- throw new Exception(pht('Failed to set socket nonblocking!')); ++ if (strtoupper(substr(PHP_OS, 0, 3)) !== 'WIN') { ++ $ok = stream_set_blocking($socket, false); ++ if (!$ok) { ++ throw new Exception(pht('Failed to set socket nonblocking!')); ++ } + } + } + diff --git a/taskcluster/docker/updatebot/hgrc b/taskcluster/docker/updatebot/hgrc new file mode 100644 index 0000000000..7d1db24ae0 --- /dev/null +++ b/taskcluster/docker/updatebot/hgrc @@ -0,0 +1,7 @@ +[ui] +ssh = ssh -i $HOME/id_rsa -l updatebot@mozilla.com -o UserKnownHostsFile=$HOME/ssh_known_hosts +username = Updatebot <updatebot@mozilla.com> + +[extensions] +strip = +push-to-try = $HOME/.mozbuild/version-control-tools/hgext/push-to-try
\ No newline at end of file diff --git a/taskcluster/docker/updatebot/moz.build b/taskcluster/docker/updatebot/moz.build new file mode 100644 index 0000000000..315dc32600 --- /dev/null +++ b/taskcluster/docker/updatebot/moz.build @@ -0,0 +1,8 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +with Files("**"): + BUG_COMPONENT = ("Developer Infrastructure", "Mach Vendor & Updatebot") diff --git a/taskcluster/docker/updatebot/privileged-setup.sh b/taskcluster/docker/updatebot/privileged-setup.sh new file mode 100755 index 0000000000..640b0826e2 --- /dev/null +++ b/taskcluster/docker/updatebot/privileged-setup.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set -vex + +. ./updatebot-version.sh # Get UPDATEBOT_REVISION + +# If you edit this, be sure to edit fetch/updatebot.yml +export SQLPROXY_REVISION=fb1939ab92846761595833361c6b0b0ecd543861 + +export DEBIAN_FRONTEND=noninteractive + +# Update apt-get lists +apt-get update -y + +# Install dependencies +apt-get install -y --no-install-recommends \ + arcanist \ + ca-certificates \ + curl \ + ed \ + golang-go \ + gcc \ + libc6-dev \ + python3-minimal \ + python3-wheel \ + python3-pip \ + python3-venv \ + python3-requests \ + python3-requests-unixsocket \ + python3-setuptools \ + openssh-client \ + rsync \ + wget + +mkdir -p /builds/worker/.mozbuild +chown -R worker:worker /builds/worker/ +export GOPATH=/builds/worker/go + +# nodejs 16 for pdfjs +. install-node.sh + +# pdf.js setup +# We want to aviod downloading a ton of packages all the time, so +# we will preload the pdf.js repo (and packages) in the Docker image +# and only update it at runtime. This means that the `./mach vendor` +# behavior for pdf.js will also be kind of custom +npm install -g gulp-cli +cd /builds/worker/ +git clone https://github.com/mozilla/pdf.js.git +cd /builds/worker/pdf.js +npm ci --legacy-peer-deps + +# seed a v8 repository because it's large, and doing an update will +# be much faster than a new clone each time. +cd /builds/worker/ +git clone https://github.com/v8/v8.git + +# Build Google's Cloud SQL Proxy from source +cd /builds/worker/ +mkdir cloud_sql_proxy +cd cloud_sql_proxy +go mod init cloud_sql_proxy +go get github.com/GoogleCloudPlatform/cloudsql-proxy/cmd/cloud_sql_proxy@$SQLPROXY_REVISION + +# Check out source code +cd /builds/worker/ +git clone https://github.com/mozilla-services/updatebot.git +cd updatebot +git checkout "$UPDATEBOT_REVISION" + +# Set up dependencies +cd /builds/worker/ +chown -R worker:worker . +chown -R worker:worker .* + +python3 -m pip install -U pip +python3 -m pip install poetry==1.2.2 + +rm -rf /setup diff --git a/taskcluster/docker/updatebot/run.py b/taskcluster/docker/updatebot/run.py new file mode 100755 index 0000000000..ad9bcb8788 --- /dev/null +++ b/taskcluster/docker/updatebot/run.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys + +sys.path.append("/builds/worker/checkouts/gecko/third_party/python") +sys.path.append(".") + +import base64 +import os +import platform +import signal +import stat +import subprocess + +import requests + +import taskcluster + +# Bump this number when you need to cause a commit for the job to re-run: 21 + +if len(sys.argv) < 3: + print("Usage:", sys.argv[0], "gecko-dev-path updatebot-path [moz-fetches-dir]") + sys.exit(1) + +GECKO_DEV_PATH = sys.argv[1].replace("/", os.path.sep) +UPDATEBOT_PATH = sys.argv[2].replace("/", os.path.sep) + +# Only needed on Windows +if len(sys.argv) > 3: + FETCHES_PATH = sys.argv[3].replace("/", os.path.sep) +else: + FETCHES_PATH = None + +HOME_PATH = os.path.expanduser("~") + +OPERATING_MODE = ( + "prod" + if os.environ.get("GECKO_HEAD_REPOSITORY", "") + == "https://hg.mozilla.org/mozilla-central" + else "dev" +) + +DEV_PHAB_URL = "https://phabricator-dev.allizom.org/" +PROD_PHAB_URL = "https://phabricator.services.mozilla.com/" + +phabricator_url = DEV_PHAB_URL if OPERATING_MODE == "dev" else PROD_PHAB_URL + + +def log(*args): + print(*args) + + +def get_secret(name): + secret = None + if "TASK_ID" in os.environ: + secrets_url = ( + "http://taskcluster/secrets/v1/secret/project/updatebot/" + + ("3" if OPERATING_MODE == "prod" else "2") + + "/" + + name + ) + res = requests.get(secrets_url) + res.raise_for_status() + secret = res.json() + else: + secrets = taskcluster.Secrets(taskcluster.optionsFromEnvironment()) + secret = secrets.get("project/updatebot/" + OPERATING_MODE + "/" + name) + secret = secret["secret"] if "secret" in secret else None + secret = secret["value"] if "value" in secret else None + return secret + + +# Get TC Secrets ======================================= +log("Operating mode is ", OPERATING_MODE) +log("Getting secrets...") +bugzilla_api_key = get_secret("bugzilla-api-key") +phabricator_token = get_secret("phabricator-token") +try_sshkey = get_secret("try-sshkey") +database_config = get_secret("database-password") +sentry_url = get_secret("sentry-url") +sql_proxy_config = get_secret("sql-proxy-config") + +# Update Updatebot ======================================= +if OPERATING_MODE == "dev": + """ + If we are in development mode, we will update from github. + (This command will probably only work if we checkout a branch FWIW.) + + This allows us to iterate faster by committing to github and + re-running the cron job on Taskcluster, without rebuilding the + Docker image. + + However, this mechanism is bypassing the security feature we + have in-tree, where upstream out-of-tree code is fixed at a known + revision and cannot be changed without a commit to m-c. + + Therefore, we only do this in dev mode when running on try. + """ + + os.chdir(UPDATEBOT_PATH) + log("Performing git repo update...") + command = ["git", "symbolic-ref", "-q", "HEAD"] + + r = subprocess.run(command) + if r.returncode == 0: + # This indicates we are on a branch, and not a specific revision + subprocess.check_call(["git", "pull", "origin"]) + +# Set Up SSH & Phabricator ============================== +os.chdir(HOME_PATH) +log("Setting up ssh and phab keys...") +with open("id_rsa", "w") as sshkey: + sshkey.write(try_sshkey) +os.chmod("id_rsa", stat.S_IRUSR | stat.S_IWUSR) + +arc_filename = ".arcrc" +if platform.system() == "Windows": + arc_path = os.path.join(FETCHES_PATH, "..", "AppData", "Roaming") + os.makedirs(arc_path, exist_ok=True) + os.chdir(arc_path) + log("Writing %s to %s" % (arc_filename, arc_path)) +else: + os.chdir(HOME_PATH) + +arcrc = open(arc_filename, "w") +towrite = """ +{ + "hosts": { + "PHAB_URL_HERE": { + "token": "TOKENHERE" + } + } +} +""".replace( + "TOKENHERE", phabricator_token +).replace( + "PHAB_URL_HERE", phabricator_url + "api/" +) +arcrc.write(towrite) +arcrc.close() +os.chmod(arc_filename, stat.S_IRUSR | stat.S_IWUSR) + +# Set up the Cloud SQL Proxy ============================= +os.chdir(HOME_PATH) +log("Setting up cloud_sql_proxy...") +with open("sql-proxy-key", "w") as proxy_key_file: + proxy_key_file.write( + base64.b64decode(sql_proxy_config["key-value"]).decode("utf-8") + ) + +instance_name = sql_proxy_config["instance-name"] +if platform.system() == "Linux": + sql_proxy_command = "/builds/worker/go/bin/cloud_sql_proxy" +else: + sql_proxy_command = os.path.join(UPDATEBOT_PATH, "..", "cloud_sql_proxy.exe") + +sql_proxy_command += ( + " -instances=" + instance_name + "=tcp:3306 -credential_file=sql-proxy-key" +) +sql_proxy_args = { + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "shell": True, + "start_new_session": True, +} + +if platform.system() == "Windows": + si = subprocess.STARTUPINFO() + si.dwFlags = subprocess.CREATE_NEW_PROCESS_GROUP + + sql_proxy_args["startupinfo"] = si + +sql_proxy = subprocess.Popen((sql_proxy_command), **sql_proxy_args) + +try: + (stdout, stderr) = sql_proxy.communicate(input=None, timeout=2) + log("sql proxy stdout:", stdout.decode("utf-8")) + log("sql proxy stderr:", stderr.decode("utf-8")) +except subprocess.TimeoutExpired: + log("no sqlproxy output in 2 seconds, this means it probably didn't error.") + log("sqlproxy pid:", sql_proxy.pid) + +database_config["host"] = "127.0.0.1" + +# Vendor ================================================= +log("Getting Updatebot ready...") +os.chdir(UPDATEBOT_PATH) +localconfig = { + "General": { + "env": OPERATING_MODE, + "gecko-path": GECKO_DEV_PATH, + }, + "Logging": { + "local": True, + "sentry": True, + "sentry_config": {"url": sentry_url, "debug": False}, + }, + "Database": database_config, + "Bugzilla": { + "apikey": bugzilla_api_key, + }, + "Taskcluster": { + "url_treeherder": "https://treeherder.mozilla.org/", + "url_taskcluster": "http://taskcluster/", + }, +} + +log("Writing local config file") +config = open("localconfig.py", "w") +config.write("localconfig = " + str(localconfig)) +config.close() + +log("Running updatebot") +# On Windows, Updatebot is run by windows-setup.sh +if platform.system() == "Linux": + subprocess.check_call(["python3", "-m", "poetry", "run", "./automation.py"]) + + # Clean up =============================================== + log("Killing cloud_sql_proxy") + os.kill(sql_proxy.pid, signal.SIGTERM) diff --git a/taskcluster/docker/updatebot/setup.sh b/taskcluster/docker/updatebot/setup.sh new file mode 100755 index 0000000000..e7c6fe03e0 --- /dev/null +++ b/taskcluster/docker/updatebot/setup.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set -vex + +# Copy the system known_hosts to the home directory so we have uniformity with Windows +# and the ssh command will find them in the same place. +cp /etc/ssh/ssh_known_hosts "$HOME/ssh_known_hosts" + +# If poetry is not run as worker, then it won't work when run as user later. +cd /builds/worker/updatebot +/usr/local/bin/poetry install --no-ansi diff --git a/taskcluster/docker/updatebot/updatebot-version.sh b/taskcluster/docker/updatebot/updatebot-version.sh new file mode 100644 index 0000000000..5f0719c64b --- /dev/null +++ b/taskcluster/docker/updatebot/updatebot-version.sh @@ -0,0 +1,2 @@ +export UPDATEBOT_REVISION=d646ac13bd300241fdf03e529fcf37d231f10bfc + diff --git a/taskcluster/docker/updatebot/windows-php.ini b/taskcluster/docker/updatebot/windows-php.ini new file mode 100644 index 0000000000..8ccaa05ce4 --- /dev/null +++ b/taskcluster/docker/updatebot/windows-php.ini @@ -0,0 +1,130 @@ +[PHP]
+
+engine = On
+short_open_tag = Off
+precision = 14
+output_buffering = 4096
+zlib.output_compression = Off
+implicit_flush = Off
+unserialize_callback_func =
+serialize_precision = -1
+disable_functions =
+disable_classes =
+zend.enable_gc = On
+zend.exception_ignore_args = Off
+expose_php = On
+max_execution_time = 30
+max_input_time = 60
+memory_limit = 128M
+error_reporting = E_ALL
+display_errors = On
+display_startup_errors = On
+log_errors = On
+log_errors_max_len = 1024
+ignore_repeated_errors = Off
+ignore_repeated_source = Off
+report_memleaks = On
+variables_order = "GPCS"
+request_order = "GP"
+register_argc_argv = Off
+auto_globals_jit = On
+post_max_size = 8M
+auto_prepend_file =
+auto_append_file =
+default_mimetype = "text/html"
+default_charset = "UTF-8"
+doc_root =
+user_dir =
+extension_dir = "ext"
+enable_dl = Off
+file_uploads = On
+upload_max_filesize = 2M
+max_file_uploads = 20
+allow_url_fopen = On
+allow_url_include = Off
+default_socket_timeout = 60
+extension=curl
+
+[CLI Server]
+cli_server.color = On
+
+[Pdo_mysql]
+pdo_mysql.default_socket=
+
+[mail function]
+SMTP = localhost
+smtp_port = 25
+mail.add_x_header = Off
+
+[ODBC]
+odbc.allow_persistent = On
+odbc.check_persistent = On
+odbc.max_persistent = -1
+odbc.max_links = -1
+odbc.defaultlrl = 4096
+odbc.defaultbinmode = 1
+
+[MySQLi]
+mysqli.max_persistent = -1
+mysqli.allow_persistent = On
+mysqli.max_links = -1
+mysqli.default_port = 3306
+mysqli.default_socket =
+mysqli.default_host =
+mysqli.default_user =
+mysqli.default_pw =
+mysqli.reconnect = Off
+
+[mysqlnd]
+mysqlnd.collect_statistics = On
+mysqlnd.collect_memory_statistics = On
+
+[PostgreSQL]
+pgsql.allow_persistent = On
+pgsql.auto_reset_persistent = Off
+pgsql.max_persistent = -1
+pgsql.max_links = -1
+pgsql.ignore_notice = 0
+pgsql.log_notice = 0
+
+[bcmath]
+bcmath.scale = 0
+
+[Session]
+session.save_handler = files
+session.use_strict_mode = 0
+session.use_cookies = 1
+session.use_only_cookies = 1
+session.name = PHPSESSID
+session.auto_start = 0
+session.cookie_lifetime = 0
+session.cookie_path = /
+session.cookie_domain =
+session.cookie_httponly =
+session.cookie_samesite =
+session.serialize_handler = php
+session.gc_probability = 1
+session.gc_divisor = 1000
+session.gc_maxlifetime = 1440
+session.referer_check =
+session.cache_limiter = nocache
+session.cache_expire = 180
+session.use_trans_sid = 0
+session.sid_length = 26
+session.trans_sid_tags = "a=href,area=href,frame=src,form="
+session.sid_bits_per_character = 5
+
+[Assertion]
+zend.assertions = 1
+
+[Tidy]
+tidy.clean_output = Off
+
+[soap]
+soap.wsdl_cache_enabled=1
+soap.wsdl_cache_dir="/tmp"
+soap.wsdl_cache_ttl=86400
+soap.wsdl_cache_limit = 5
+
+[ldap]
+ldap.max_links = -1
diff --git a/taskcluster/docker/updatebot/windows-setup.sh b/taskcluster/docker/updatebot/windows-setup.sh new file mode 100644 index 0000000000..c04d0a6fbf --- /dev/null +++ b/taskcluster/docker/updatebot/windows-setup.sh @@ -0,0 +1,135 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set -vex + +. ./taskcluster/docker/updatebot/updatebot-version.sh # Get UPDATEBOT_REVISION + +HOME=$(python3 -c "import os;print(os.path.expanduser('~'))") +export HOME +GECKO_PATH="$PWD" +UPDATEBOT_PATH="$MOZ_FETCHES_DIR/updatebot" + +# MOZ_FETCHES_DIR is in Z:/ format. When we update the PATH we need to use +# /z/ format. Fortunately, we can translate them like so: +cd "$MOZ_FETCHES_DIR" +MOZ_FETCHES_PATH="$PWD" + +######################################################### +# Install dependencies + +# Move depot_tools +cd "$MOZ_FETCHES_DIR" +mv depot_tools.git depot_tools + + +# Generating a new version of the preloaded depot_tools download can be done by: +# 1) Running the task, uncommenting the variable assignment below, uncommenting the +# _GENERATE_DEPOT_TOOLS_BINARIES_ section in taskcluster/ci/updatebot/kind.yml, +# and ensuring that an angle update will actually take place (so it downloads the depot_tools) +# 2) Downloading and sanity-checking the depot_tools-preloaded-binaries-GIT_HASH-DATE.zip artifact +# 3) Adding it to tooltool +# 4) Updating the updatebot manifest +# Note that even for the same git revision the downloaded tools can change, so they are tagged +# with both the git hash and the date it was generated + +# export GENERATE_DEPOT_TOOLS_BINARIES=1 + +if test -n "$GENERATE_DEPOT_TOOLS_BINARIES"; then + cp -r depot_tools depot_tools-from-git +fi + +# Git is at /c/Program Files/Git/cmd/git.exe +# It's in PATH for this script (confusingly) but not in PATH so we need to add it +export PATH="/c/Program Files/Git/cmd:$PATH" + +# php & arcanist +if [ -n "$TOOLTOOL_MANIFEST" ]; then + . "$GECKO_PATH/taskcluster/scripts/misc/tooltool-download.sh" +fi + +cp "$MOZ_FETCHES_DIR/vcruntime140.dll" "$MOZ_FETCHES_DIR/php-win" +cp "$GECKO_PATH/taskcluster/docker/updatebot/windows-php.ini" "$MOZ_FETCHES_DIR/php-win/php.ini" + +cd "$MOZ_FETCHES_DIR/arcanist" +patch -p1 < "$GECKO_PATH/taskcluster/docker/updatebot/arcanist_windows_stream.patch" +patch -p1 < "$GECKO_PATH/taskcluster/docker/updatebot/arcanist_patch_size.patch" +cd "$MOZ_FETCHES_DIR" + +export PATH="$MOZ_FETCHES_PATH/php-win:$PATH" +export PATH="$MOZ_FETCHES_PATH/arcanist/bin:$PATH" + +# get Updatebot +cd "$MOZ_FETCHES_DIR" +git clone https://github.com/mozilla-services/updatebot.git +cd updatebot +git checkout "$UPDATEBOT_REVISION" + +# base python needs +python3 -m pip install --no-warn-script-location --user -U pip +python3 -m pip install --no-warn-script-location --user poetry wheel requests setuptools + +# updatebot dependencies +cd "$UPDATEBOT_PATH" +python3 -m poetry install + +# taskcluster secrets and writing out localconfig +cd "$GECKO_PATH" +python3 ./taskcluster/docker/updatebot/run.py "$GECKO_PATH" "$UPDATEBOT_PATH" "$MOZ_FETCHES_PATH" + +# mercurial configuration +cp "$GECKO_PATH/taskcluster/docker/updatebot/hgrc" "$HOME/.hgrc" +# Windows is not happy with $HOME in the hgrc so we need to do a hack to replace it +# with the actual value +( echo "cat <<EOF" ; cat "$HOME/.hgrc" ) | sh > tmp +mv tmp "$HOME/.hgrc" + +# ssh known hosts +cp "$GECKO_PATH/taskcluster/docker/push-to-try/known_hosts" "$HOME/ssh_known_hosts" + +######################################################### +# Run it +export PYTHONIOENCODING=utf8 +export PYTHONUNBUFFERED=1 + +cd "$UPDATEBOT_PATH" +python3 -m poetry run python3 ./automation.py + +######################################################### +if test -n "$GENERATE_DEPOT_TOOLS_BINARIES"; then + # Artifacts + + cd "$MOZ_FETCHES_PATH" + mv depot_tools depot_tools-from-tc + + # Clean out unneeded files + # Need to use cmd because for some reason rm from bash throws 'Access Denied' + cmd '/c for /d /r %i in (*__pycache__) do rmdir /s /q %i' + rm -rf depot_tools-from-git/.git || true + + # Delete the files that are already in git + find depot_tools-from-git -mindepth 1 -maxdepth 1 | sed s/depot_tools-from-git/depot_tools-from-tc/ | while read -r d; do rm -rf "$d"; done + + # Make the artifact + rm -rf depot_tools-preloaded-binaries #remove it if it existed (i.e. we probably have one from tooltool already) + mv depot_tools-from-tc depot_tools-preloaded-binaries + + # zip can't add symbolic links, and exits with an error code. || true avoids a script crash + zip -r depot_tools-preloaded-binaries.zip depot_tools-preloaded-binaries/ || true + + # Convoluted way to get the git hash, because we don't have a .git directory + # Adding extra print statements just in case we need to debug it + GIT_HASH=$(grep depot_tools -A 1 "$GECKO_PATH/taskcluster/ci/fetch/updatebot.yml" | tee /dev/tty | grep revision | tee /dev/tty | awk -F': *' '{print $2}' | tee /dev/tty) + DATE=$(date -I) + mv depot_tools-preloaded-binaries.zip "depot_tools-preloaded-binaries-$GIT_HASH-$DATE.zip" + + # Put the artifact into the directory we will look for it + mkdir -p "$GECKO_PATH/obj-build/depot_tools" || true + mv "depot_tools-preloaded-binaries-$GIT_HASH-$DATE.zip" "$GECKO_PATH/obj-build/depot_tools" +fi + +######################################################### +echo "Killing SQL Proxy" +taskkill -f -im cloud_sql_proxy.exe || true diff --git a/taskcluster/docker/valgrind-build/Dockerfile b/taskcluster/docker/valgrind-build/Dockerfile new file mode 100644 index 0000000000..2bc35e1171 --- /dev/null +++ b/taskcluster/docker/valgrind-build/Dockerfile @@ -0,0 +1,66 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Mike Hommey <mhommey@mozilla.com> + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +ARG TASKCLUSTER_ROOT_URL +ARG DOCKER_IMAGE_PACKAGES +RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES + +# We could try to be smart and install all the -dbg packages corresponding to +# the installed packages, but, not all of them are actually for libraries used +# by Firefox, leading to a larger docker image. Moreover, some of the -dbg +# packages for unnecessary libraries pull other packages through dependencies, +# that make for even larger docker images. +RUN apt-get update && apt-get install \ + dbus-x11 \ + libgtk-3-bin \ + libdbus-1-3-dbgsym \ + libatk-bridge2.0-0-dbgsym \ + libatk1.0-0-dbgsym \ + libcairo2-dbgsym \ + libdbus-glib-1-2-dbgsym \ + libdrm2-dbgsym \ + libffi7-dbgsym \ + libfontconfig1-dbgsym \ + libglx-mesa0-dbgsym \ + libglapi-mesa-dbgsym \ + libglib2.0-0-dbgsym \ + libgdk-pixbuf-2.0-0-dbgsym \ + libgtk-3-0-dbgsym \ + libice6-dbgsym \ + libicu67-dbgsym \ + libpango-1.0-0-dbgsym \ + libpcre3-dbg \ + libpixman-1-0-dbgsym \ + libsm6-dbgsym \ + libvorbis0a-dbgsym \ + libwayland-client0-dbgsym \ + libwayland-cursor0-dbgsym \ + libx11-6-dbgsym \ + libx11-xcb1-dbgsym \ + libxau6-dbgsym \ + libxcb-glx0-dbgsym \ + libxcb-render0-dbgsym \ + libxcb-shm0-dbgsym \ + libxcb1-dbgsym \ + libxcomposite1-dbgsym \ + libxcursor1-dbgsym \ + libxdamage1-dbgsym \ + libxdmcp6-dbg \ + libxext6-dbg \ + libxfixes3-dbgsym \ + libxi6-dbgsym \ + libxinerama1-dbgsym \ + libxrandr2-dbgsym \ + libxrender1-dbgsym \ + libxt6-dbgsym \ + libxxf86vm1-dbg \ + valgrind-dbg \ + xvfb \ + zlib1g-dbgsym + +# %include taskcluster/docker/recipes/xvfb.sh +COPY topsrcdir/taskcluster/docker/recipes/xvfb.sh /builds/worker/scripts/xvfb.sh diff --git a/taskcluster/docker/webrender/Dockerfile b/taskcluster/docker/webrender/Dockerfile new file mode 100644 index 0000000000..73b876cf78 --- /dev/null +++ b/taskcluster/docker/webrender/Dockerfile @@ -0,0 +1,16 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Kartikaya Gupta <kgupta@mozilla.com> + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +# %include gfx/wr/ci-scripts/docker-image/setup.sh +ADD topsrcdir/gfx/wr/ci-scripts/docker-image/setup.sh /tmp/wr-setup.sh +RUN /bin/bash /tmp/wr-setup.sh && rm /tmp/wr-setup.sh + +# We need this to install cargo-vendor as part of the wrench-deps task +RUN apt-get install libssl-dev + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] |