diff options
Diffstat (limited to 'taskcluster/docker/recipes')
-rw-r--r-- | taskcluster/docker/recipes/common.sh | 10 | ||||
-rw-r--r-- | taskcluster/docker/recipes/debian-build-system-setup.sh | 11 | ||||
-rw-r--r-- | taskcluster/docker/recipes/debian-test-system-setup.sh | 93 | ||||
-rw-r--r-- | taskcluster/docker/recipes/dot-config/pip/pip.conf | 2 | ||||
-rwxr-xr-x | taskcluster/docker/recipes/hgrc | 33 | ||||
-rwxr-xr-x | taskcluster/docker/recipes/install-mercurial.sh | 134 | ||||
-rw-r--r-- | taskcluster/docker/recipes/install-node.sh | 16 | ||||
-rwxr-xr-x | taskcluster/docker/recipes/prepare_openjdk.sh | 9 | ||||
-rw-r--r-- | taskcluster/docker/recipes/ubuntu1804-test-system-setup-base.sh | 167 | ||||
-rw-r--r-- | taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh | 28 | ||||
-rw-r--r-- | taskcluster/docker/recipes/xvfb.sh | 75 |
11 files changed, 578 insertions, 0 deletions
diff --git a/taskcluster/docker/recipes/common.sh b/taskcluster/docker/recipes/common.sh new file mode 100644 index 0000000000..dbf0503887 --- /dev/null +++ b/taskcluster/docker/recipes/common.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +tooltool_fetch() { + cat >manifest.tt + python2.7 /setup/tooltool.py fetch + rm manifest.tt +} diff --git a/taskcluster/docker/recipes/debian-build-system-setup.sh b/taskcluster/docker/recipes/debian-build-system-setup.sh new file mode 100644 index 0000000000..de4e04cee7 --- /dev/null +++ b/taskcluster/docker/recipes/debian-build-system-setup.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +cd /setup || exit + +. /setup/common.sh +. /setup/install-mercurial.sh + +rm -rf /setup diff --git a/taskcluster/docker/recipes/debian-test-system-setup.sh b/taskcluster/docker/recipes/debian-test-system-setup.sh new file mode 100644 index 0000000000..410c0e874d --- /dev/null +++ b/taskcluster/docker/recipes/debian-test-system-setup.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash + +set -ve + +test "$(whoami)" == 'root' + +mkdir -p /setup +cd /setup + +apt_packages=() + +apt_packages+=('autoconf2.13') +apt_packages+=('bluez-cups') +apt_packages+=('build-essential') +apt_packages+=('ca-certificates') +apt_packages+=('ccache') +apt_packages+=('curl') +apt_packages+=('fonts-kacst') +apt_packages+=('fonts-kacst-one') +apt_packages+=('fonts-liberation') +apt_packages+=('fonts-stix') +apt_packages+=('fonts-unfonts-core') +apt_packages+=('fonts-unfonts-extra') +apt_packages+=('fonts-vlgothic') +apt_packages+=('g++-multilib') +apt_packages+=('gcc-multilib') +apt_packages+=('gir1.2-gnomebluetooth-1.0') +apt_packages+=('git') +apt_packages+=('gnome-keyring') +apt_packages+=('libasound2-dev') +apt_packages+=('libcanberra-pulse') +apt_packages+=('libcurl4-openssl-dev') +apt_packages+=('libdbus-1-dev') +apt_packages+=('libdbus-glib-1-dev') +apt_packages+=('libgconf2-dev') +apt_packages+=('libgtk2.0-dev') +apt_packages+=('libiw-dev') +apt_packages+=('libnotify-dev') +apt_packages+=('libpulse-dev') +apt_packages+=('libsox-fmt-alsa') +apt_packages+=('libxt-dev') +apt_packages+=('libxxf86vm1') +apt_packages+=('llvm') +apt_packages+=('llvm-dev') +apt_packages+=('llvm-runtime') +apt_packages+=('locales') +apt_packages+=('locales-all') +apt_packages+=('net-tools') +apt_packages+=('qemu-kvm') +apt_packages+=('rlwrap') +apt_packages+=('screen') +apt_packages+=('software-properties-common') +apt_packages+=('sudo') +apt_packages+=('tar') +apt_packages+=('ttf-dejavu') +apt_packages+=('unzip') +apt_packages+=('uuid') +apt_packages+=('wget') +apt_packages+=('xvfb') +apt_packages+=('yasm') +apt_packages+=('zip') + +# use Ubuntu's Python-2.7 (2.7.3 on Precise) +apt_packages+=('python-dev') +apt_packages+=('python-pip') + +apt-get update +# This allows packages to be installed without human interaction +export DEBIAN_FRONTEND=noninteractive +apt-get install -y -f "${apt_packages[@]}" + +dpkg-reconfigure locales + +. /setup/common.sh +. /setup/install-mercurial.sh + +# pip 19.3 is causing errors building the docker image, pin to 19.2.3 for now. +# See https://github.com/pypa/pip/issues/7206 +pip install --upgrade pip==19.2.3 +hash -r +pip install virtualenv==15.2.0 + +# clean up +apt-get -y autoremove + +# We don't need no docs! +rm -rf /usr/share/help /usr/share/doc /usr/share/man + +cd / +rm -rf /setup ~/.ccache ~/.cache ~/.npm +apt-get clean +apt-get autoclean +rm -f "$0" diff --git a/taskcluster/docker/recipes/dot-config/pip/pip.conf b/taskcluster/docker/recipes/dot-config/pip/pip.conf new file mode 100644 index 0000000000..73c2b2a52c --- /dev/null +++ b/taskcluster/docker/recipes/dot-config/pip/pip.conf @@ -0,0 +1,2 @@ +[global] +disable-pip-version-check = true diff --git a/taskcluster/docker/recipes/hgrc b/taskcluster/docker/recipes/hgrc new file mode 100755 index 0000000000..f6a2f6643c --- /dev/null +++ b/taskcluster/docker/recipes/hgrc @@ -0,0 +1,33 @@ +# By default the progress bar starts after 3s and updates every 0.1s. We +# change this so it shows and updates every 1.0s. +# We also tell progress to assume a TTY is present so updates are printed +# even if there is no known TTY. +[progress] +delay = 1.0 +refresh = 1.0 +assume-tty = true + +[extensions] +share = +sparse = +robustcheckout = /usr/local/mercurial/robustcheckout.py + +[hostsecurity] +# When running a modern Python, Mercurial will default to TLS 1.1+. +# When running on a legacy Python, Mercurial will default to TLS 1.0+. +# There is no good reason we shouldn't be running a modern Python +# capable of speaking TLS 1.2. And the only Mercurial servers we care +# about should be running TLS 1.2. So make TLS 1.2 the minimum. +minimumprotocol = tls1.2 + +# Settings to make 1-click loaners more useful. +[extensions] +histedit = +rebase = + +[diff] +git = 1 +showfunc = 1 + +[pager] +pager = LESS=FRSXQ less diff --git a/taskcluster/docker/recipes/install-mercurial.sh b/taskcluster/docker/recipes/install-mercurial.sh new file mode 100755 index 0000000000..2e394c6775 --- /dev/null +++ b/taskcluster/docker/recipes/install-mercurial.sh @@ -0,0 +1,134 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script installs and configures Mercurial. + +set -e + +# Detect OS. +if [ -f /etc/lsb-release ]; then + # Disabled so linting works on Mac + # shellcheck disable=SC1091 + . /etc/lsb-release + + if [ "${DISTRIB_ID}" = "Ubuntu" ] && [[ "${DISTRIB_RELEASE}" = "20.04" ]]; then + HG_APT=1 + elif [ "${DISTRIB_ID}" = "Ubuntu" ] && [[ "${DISTRIB_RELEASE}" = "16.04" || "${DISTRIB_RELEASE}" = "17.10" || "${DISTRIB_RELEASE}" = "18.04" ]] + then + HG_DEB=1 + HG_DIGEST=54a215232a340139707a968b58943c2903a8297f0da32f96622d1acab47de6013a5b96d2ca4ba241b1fee142b4098a6cdd236b308a1657c31f42807d7385d327 + HG_SIZE=278440 + HG_FILENAME=mercurial_4.8.1_amd64.deb + + HG_COMMON_DIGEST=5577fec8d0f6643d17751b3f6be76b0c2bb888ae1920a8b085245e05110e3d5cfe1c4e9d51e334ab0dd0865fe553c63c704e72852e00b71eb668980cb6b33fa4 + HG_COMMON_SIZE=2439436 + HG_COMMON_FILENAME=mercurial-common_4.8.1_all.deb + elif [ "${DISTRIB_ID}" = "Ubuntu" ] && [ "${DISTRIB_RELEASE}" = "12.04" ] + then + echo "Ubuntu 12.04 not supported" + exit 1 + fi + + CERT_PATH=/etc/ssl/certs/ca-certificates.crt + +elif [ -f /etc/os-release ]; then + # Disabled so linting works on Mac + # shellcheck disable=SC1091 + . /etc/os-release + + if [ "${ID}" = "debian" ]; then + if [ -f /usr/bin/pip2 ]; then + PIP_PATH=/usr/bin/pip2 + elif [ -f /usr/bin/pip ]; then + # Versions of debian that don't have pip2 have pip pointing to the python2 version. + PIP_PATH=/usr/bin/pip + else + echo "We currently require Python 2.7 and pip to run Mercurial" + exit 1 + fi + else + echo "Unsupported debian-like system with ID '${ID}' and VERSION_ID '${VERSION_ID}'" + exit 1 + fi + + CERT_PATH=/etc/ssl/certs/ca-certificates.crt + +elif [ -f /etc/centos-release ]; then + CENTOS_VERSION="$(rpm -q --queryformat '%{VERSION}' centos-release)" + if [ "${CENTOS_VERSION}" = "6" ]; then + if [ -f /usr/bin/pip2.7 ]; then + PIP_PATH=/usr/bin/pip2.7 + else + echo "We currently require Python 2.7 and /usr/bin/pip2.7 to run Mercurial" + exit 1 + fi + else + echo "Unsupported CentOS version: ${CENTOS_VERSION}" + exit 1 + fi + + CERT_PATH=/etc/ssl/certs/ca-bundle.crt +fi + +if [ -n "${HG_APT}" ]; then +apt-get install -y mercurial +elif [ -n "${HG_DEB}" ]; then +tooltool_fetch <<EOF +[ + { + "size": ${HG_SIZE}, + "digest": "${HG_DIGEST}", + "algorithm": "sha512", + "filename": "${HG_FILENAME}" + }, + { + "size": ${HG_COMMON_SIZE}, + "digest": "${HG_COMMON_DIGEST}", + "algorithm": "sha512", + "filename": "${HG_COMMON_FILENAME}" + } +] +EOF + + dpkg -i --auto-deconfigure ${HG_COMMON_FILENAME} ${HG_FILENAME} +elif [ -n "${HG_RPM}" ]; then +tooltool_fetch <<EOF +[ + { + "size": ${HG_SIZE}, + "digest": "${HG_DIGEST}", + "algorithm": "sha512", + "filename": "${HG_FILENAME}" + } +] +EOF + + rpm -i ${HG_FILENAME} +elif [ -n "${PIP_PATH}" ]; then +tooltool_fetch <<EOF +[ + { + "size": 6869733, + "digest": "a4485c22f9bb0bb752bf42941f613cb3542c66cbec5d7d49be2090ac544f5dca0f476e4535a56e3f4f4f5fc02fb12739e6d1c7b407264fc2ba4b19b0230b9f93", + "algorithm": "sha512", + "filename": "mercurial-4.8.1.tar.gz" + } +] +EOF + + ${PIP_PATH} install mercurial-4.8.1.tar.gz +else + echo "Do not know how to install Mercurial on this OS" + exit 1 +fi + +chmod 644 /usr/local/mercurial/robustcheckout.py + +cat >/etc/mercurial/hgrc.d/cacerts.rc <<EOF +[web] +cacerts = ${CERT_PATH} +EOF + +chmod 644 /etc/mercurial/hgrc.d/cacerts.rc diff --git a/taskcluster/docker/recipes/install-node.sh b/taskcluster/docker/recipes/install-node.sh new file mode 100644 index 0000000000..5b480a603f --- /dev/null +++ b/taskcluster/docker/recipes/install-node.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script installs Node v10. +# XXX For now, this should match the version installed in +# taskcluster/scripts/misc/repack-node.sh. Later we'll get the ESLint builder +# to use the linux64-node toolchain directly. + +wget -O node.xz --progress=dot:mega https://nodejs.org/dist/v10.23.1/node-v10.23.1-linux-x64.tar.xz + +echo '207e5ec77ca655ba6fcde922d6b329acbb09898b0bd793ccfcce6c27a36fdff0 node.xz' | sha256sum -c +tar -C /usr/local -xJ --strip-components 1 < node.xz +node -v # verify +npm -v diff --git a/taskcluster/docker/recipes/prepare_openjdk.sh b/taskcluster/docker/recipes/prepare_openjdk.sh new file mode 100755 index 0000000000..828c8dc697 --- /dev/null +++ b/taskcluster/docker/recipes/prepare_openjdk.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -x + +# Debian 10 doesn't have openjdk-8, so add the Debian 9 repository, which contains it. +if grep -q ^10\\. /etc/debian_version; then + sed s/buster/stretch/ /etc/apt/sources.list | tee /etc/apt/sources.list.d/stretch.list + apt-get update +fi diff --git a/taskcluster/docker/recipes/ubuntu1804-test-system-setup-base.sh b/taskcluster/docker/recipes/ubuntu1804-test-system-setup-base.sh new file mode 100644 index 0000000000..03ed14be83 --- /dev/null +++ b/taskcluster/docker/recipes/ubuntu1804-test-system-setup-base.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash + +set -ve + +test "$(whoami)" == 'root' + +# To speed up docker image build times as well as number of network/disk I/O +# build a list of packages to be installed nad call it in one go. +apt_packages=() + +apt_packages+=('autoconf2.13') +apt_packages+=('bluez-cups') +apt_packages+=('build-essential') +apt_packages+=('ca-certificates') +apt_packages+=('ccache') +apt_packages+=('compiz') +apt_packages+=('curl') +apt_packages+=('fontconfig') +apt_packages+=('fonts-kacst') +apt_packages+=('fonts-kacst-one') +apt_packages+=('fonts-liberation') +apt_packages+=('fonts-stix') +apt_packages+=('fonts-unfonts-core') +apt_packages+=('fonts-unfonts-extra') +apt_packages+=('fonts-vlgothic') +apt_packages+=('g++-multilib') +apt_packages+=('gcc-multilib') +apt_packages+=('gir1.2-gnomebluetooth-1.0') +apt_packages+=('git') +apt_packages+=('gnome-icon-theme') +apt_packages+=('gstreamer1.0-gtk3') +apt_packages+=('gstreamer1.0-plugins-base') +apt_packages+=('gstreamer1.0-plugins-good') +apt_packages+=('gstreamer1.0-tools') +apt_packages+=('gstreamer1.0-pulseaudio') +apt_packages+=('language-pack-en-base') +apt_packages+=('libc6-dbg') +apt_packages+=('libasound2-dev') +apt_packages+=('libavcodec-extra57') +apt_packages+=('libcanberra-gtk3-module') +apt_packages+=('libcanberra-pulse') +apt_packages+=('libcurl4-openssl-dev') +apt_packages+=('libdbus-1-dev') +apt_packages+=('libdbus-glib-1-dev') +apt_packages+=('libfreetype6') +apt_packages+=('libgconf2-dev') +apt_packages+=('libgl1-mesa-dri') +apt_packages+=('libgl1-mesa-glx') +apt_packages+=('libgstreamer-plugins-base1.0-dev') +apt_packages+=('libgstreamer1.0-dev') +apt_packages+=('libgtk2.0-dev') +apt_packages+=('libgtk-3-0') +apt_packages+=('libiw-dev') +apt_packages+=('libx11-xcb1') +apt_packages+=('libxcb1') +apt_packages+=('libxcb-render0') +apt_packages+=('libxcb-shm0') +apt_packages+=('libxcb-glx0') +apt_packages+=('libxcb-shape0') +apt_packages+=('libnotify-dev') +apt_packages+=('libpulse0') +apt_packages+=('libpulse-dev') +apt_packages+=('libxt-dev') +apt_packages+=('libxxf86vm1') +apt_packages+=('llvm') +apt_packages+=('llvm-dev') +apt_packages+=('llvm-runtime') +apt_packages+=('mesa-common-dev') +apt_packages+=('net-tools') +apt_packages+=('pulseaudio') +apt_packages+=('pulseaudio-module-bluetooth') +apt_packages+=('pulseaudio-module-gconf') +apt_packages+=('python-dev') +apt_packages+=('python-pip') +apt_packages+=('python3-pip') +apt_packages+=('qemu-kvm') +apt_packages+=('rlwrap') +apt_packages+=('screen') +apt_packages+=('software-properties-common') +apt_packages+=('sudo') +apt_packages+=('ttf-dejavu') +apt_packages+=('ubuntu-desktop') +apt_packages+=('unzip') +apt_packages+=('uuid') +apt_packages+=('wget') +apt_packages+=('xvfb') +apt_packages+=('xwit') +apt_packages+=('yasm') +apt_packages+=('zip') + +# xvinfo for test-linux.sh to monitor Xvfb startup +apt_packages+=('x11-utils') + +# Bug 1232407 - this allows the user to start vnc +apt_packages+=('x11vnc') + +# Bug 1176031 - need `xset` to disable screensavers +apt_packages+=('x11-xserver-utils') + +# Build a list of packages to install from the multiverse repo. +apt_packages+=('ubuntu-restricted-extras') + +# APT update takes very long on Ubuntu. Run it at the last possible minute. +apt-get update + +# This allows ubuntu-desktop to be installed without human interaction. +# Also force the cleanup after installation of packages to reduce image size. +export DEBIAN_FRONTEND=noninteractive +apt-get install -y --allow-downgrades -f "${apt_packages[@]}" + +# Enable i386 packages +dpkg --add-architecture i386 +apt-get update + +# Make sure we have libraries for 32-bit tests +apt_packages=() +apt_packages+=('libavcodec-extra57:i386') +apt_packages+=('libdbus-glib-1-2:i386') +apt_packages+=('libpulse0:i386') +apt_packages+=('libxt6:i386') +apt_packages+=('libxtst6:i386') +apt_packages+=('libsecret-1-0:i386') +apt_packages+=('libgtk2.0-0:i386') +apt_packages+=('libgtk-3-0:i386') +apt_packages+=('libx11-xcb1:i386') +apt_packages+=('libxcb1:i386') + +apt-get install -y --allow-downgrades -f "${apt_packages[@]}" +rm -rf /var/lib/apt/lists/* + +# Build a list of packages to purge from the image. +apt_packages=() +apt_packages+=('*cheese*') +apt_packages+=('example-content') +apt_packages+=('git') +apt_packages+=('gnome-calendar') +apt_packages+=('gnome-initial-setup') +apt_packages+=('gnome-mahjongg') +apt_packages+=('gnome-mines') +apt_packages+=('gnome-sudoku') +apt_packages+=('libx11-doc') +apt_packages+=('manpages-dev') +apt_packages+=('orca') +apt_packages+=('rhythmbox') +apt_packages+=('thunderbird') +apt_packages+=('ubuntu-release-upgrader*') +apt_packages+=('update-manager-core') +apt_packages+=('update-manager') +apt_packages+=('yelp') + +# Purge unnecessary packages +apt-get purge -y -f "${apt_packages[@]}" + +# Clear apt cache one last time +rm -rf /var/cache/apt/archives + +# We don't need no docs! +rm -rf /usr/share/help /usr/share/doc /usr/share/man + +# Remove all locale files other than en_US.UTF-8 +rm -rf /usr/share/locale/ /usr/share/locale-langpack/ /usr/share/locales/ + +# Further cleanup +apt-get -y autoremove +apt-get clean +apt-get autoclean +rm -f "$0" diff --git a/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh b/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh new file mode 100644 index 0000000000..d5b78985c2 --- /dev/null +++ b/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -ve + +test "$(whoami)" == 'root' + +cd /setup + +# Install tooltool, mercurial and node now that dependencies are in place. +. /setup/common.sh +. /setup/install-mercurial.sh +. /setup/install-node.sh + +# Upgrade pip and install virtualenv to specified versions. +pip install --upgrade pip==19.2.3 +hash -r +pip install virtualenv==15.2.0 + +pip install zstandard==0.13.0 +pip3 install zstandard==0.13.0 + +pip install psutil==5.7.0 +pip3 install psutil==5.7.0 + +# Cleanup +cd / +rm -rf /setup ~/.ccache ~/.cache ~/.npm +rm -f "$0" diff --git a/taskcluster/docker/recipes/xvfb.sh b/taskcluster/docker/recipes/xvfb.sh new file mode 100644 index 0000000000..6e0e79f7d4 --- /dev/null +++ b/taskcluster/docker/recipes/xvfb.sh @@ -0,0 +1,75 @@ +#! /bin/bash -x + +set -x + +fail() { + echo # make sure error message is on a new line + echo "[xvfb.sh:error]" "${@}" + exit 1 +} + +cleanup_xvfb() { + # When you call this script with START_VNC or TASKCLUSTER_INTERACTIVE + # we make sure we do not kill xvfb so you do not lose your connection + local xvfb_pid=`pidof Xvfb` + local vnc=${START_VNC:-false} + local interactive=${TASKCLUSTER_INTERACTIVE:-false} + if [ -n "$xvfb_pid" ] && [[ $vnc == false ]] && [[ $interactive == false ]] ; then + kill $xvfb_pid || true + screen -XS xvfb quit || true + fi +} + +# Attempt to start xvfb in a screen session with the given resolution and display +# number. Up to 5 attempts will be made to start xvfb with a short delay +# between retries +try_xvfb() { + screen -dmS xvfb Xvfb :$2 -nolisten tcp -screen 0 $1 \ + > ~/artifacts/xvfb/xvfb.log 2>&1 + export DISPLAY=:$2 + + # Only error code 255 matters, because it signifies that no + # display could be opened. As long as we can open the display + # tests should work. We'll retry a few times with a sleep before + # failing. + local retry_count=0 + local max_retries=5 + xvfb_test=0 + until [ $retry_count -gt $max_retries ]; do + xvinfo || xvfb_test=$? + if [ $xvfb_test != 255 ]; then + retry_count=$(($max_retries + 1)) + else + retry_count=$(($retry_count + 1)) + echo "Failed to start Xvfb, retry: $retry_count" + sleep 2 + fi + done + if [ $xvfb_test == 255 ]; then + return 1 + else + return 0 + fi +} + +start_xvfb() { + set +e + mkdir -p ~/artifacts/xvfb + local retry_count=0 + local max_retries=2 + local success=1 + until [ $retry_count -gt $max_retries ]; do + try_xvfb $1 $2 + success=$? + if [ $success -eq 0 ]; then + retry_count=$(($max_retries + 1)) + else + retry_count=$(($retry_count + 1)) + sleep 10 + fi + done + set -e + if [ $success -eq 1 ]; then + fail "Could not start xvfb after ${max_retries} attempts" + fi +} |