diff options
Diffstat (limited to 'taskcluster/docker/recipes')
| -rw-r--r-- | taskcluster/docker/recipes/common.sh | 10 | ||||
| -rw-r--r-- | taskcluster/docker/recipes/debian-test-system-setup.sh | 84 | ||||
| -rw-r--r-- | taskcluster/docker/recipes/dot-config/pip/pip.conf | 2 | ||||
| -rwxr-xr-x | taskcluster/docker/recipes/hgrc | 33 | ||||
| -rw-r--r-- | taskcluster/docker/recipes/install-node.sh | 15 | ||||
| -rwxr-xr-x | taskcluster/docker/recipes/setup_packages.sh | 13 | ||||
| -rw-r--r-- | taskcluster/docker/recipes/ubuntu1804-test-system-setup-base.sh | 163 | ||||
| -rw-r--r-- | taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh | 24 | ||||
| -rw-r--r-- | taskcluster/docker/recipes/xvfb.sh | 75 |
9 files changed, 419 insertions, 0 deletions
diff --git a/taskcluster/docker/recipes/common.sh b/taskcluster/docker/recipes/common.sh new file mode 100644 index 0000000000..0182154495 --- /dev/null +++ b/taskcluster/docker/recipes/common.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +tooltool_fetch() { + cat >manifest.tt + python3 /setup/tooltool.py fetch + rm manifest.tt +} diff --git a/taskcluster/docker/recipes/debian-test-system-setup.sh b/taskcluster/docker/recipes/debian-test-system-setup.sh new file mode 100644 index 0000000000..a0023220a9 --- /dev/null +++ b/taskcluster/docker/recipes/debian-test-system-setup.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +set -ve + +test "$(whoami)" == 'root' + +mkdir -p /setup +cd /setup + +apt_packages=() + +apt_packages+=('autoconf2.13') +apt_packages+=('bluez-cups') +apt_packages+=('build-essential') +apt_packages+=('ccache') +apt_packages+=('curl') +apt_packages+=('fonts-kacst') +apt_packages+=('fonts-kacst-one') +apt_packages+=('fonts-liberation') +apt_packages+=('fonts-stix') +apt_packages+=('fonts-unfonts-core') +apt_packages+=('fonts-unfonts-extra') +apt_packages+=('fonts-vlgothic') +apt_packages+=('g++-multilib') +apt_packages+=('gcc-multilib') +apt_packages+=('gir1.2-gnomebluetooth-1.0') +apt_packages+=('gnome-keyring') +apt_packages+=('libasound2-dev') +apt_packages+=('libcanberra-pulse') +apt_packages+=('libcurl4-openssl-dev') +apt_packages+=('libdbus-1-dev') +apt_packages+=('libdbus-glib-1-dev') +apt_packages+=('libgconf2-dev') +apt_packages+=('libiw-dev') +apt_packages+=('libnotify-dev') +apt_packages+=('libpulse-dev') +apt_packages+=('libsox-fmt-alsa') +apt_packages+=('libxt-dev') +apt_packages+=('libxxf86vm1') +apt_packages+=('llvm') +apt_packages+=('llvm-dev') +apt_packages+=('llvm-runtime') +apt_packages+=('locales') +apt_packages+=('locales-all') +apt_packages+=('net-tools') +apt_packages+=('openssh-client') +apt_packages+=('qemu-kvm') +apt_packages+=('rlwrap') +apt_packages+=('screen') +apt_packages+=('software-properties-common') +apt_packages+=('sudo') +apt_packages+=('tar') +apt_packages+=('ttf-dejavu') +apt_packages+=('unzip') +apt_packages+=('uuid') +apt_packages+=('wget') +apt_packages+=('xdg-user-dirs') +apt_packages+=('xvfb') +apt_packages+=('zip') + +# use Ubuntu's Python-2.7 (2.7.3 on Precise) +apt_packages+=('python-dev') +apt_packages+=('python-pip') + +apt-get update +apt-get install "${apt_packages[@]}" + +dpkg-reconfigure locales + +# pip 19.3 is causing errors building the docker image, pin to 19.2.3 for now. +# See https://github.com/pypa/pip/issues/7206 +pip install --upgrade pip==19.2.3 +hash -r +pip install virtualenv==15.2.0 + +# clean up +apt-get autoremove + +# We don't need no docs! +rm -rf /usr/share/help /usr/share/doc /usr/share/man + +cd / +rm -rf /setup ~/.ccache ~/.cache ~/.npm +rm -f "$0" diff --git a/taskcluster/docker/recipes/dot-config/pip/pip.conf b/taskcluster/docker/recipes/dot-config/pip/pip.conf new file mode 100644 index 0000000000..73c2b2a52c --- /dev/null +++ b/taskcluster/docker/recipes/dot-config/pip/pip.conf @@ -0,0 +1,2 @@ +[global] +disable-pip-version-check = true diff --git a/taskcluster/docker/recipes/hgrc b/taskcluster/docker/recipes/hgrc new file mode 100755 index 0000000000..f6a2f6643c --- /dev/null +++ b/taskcluster/docker/recipes/hgrc @@ -0,0 +1,33 @@ +# By default the progress bar starts after 3s and updates every 0.1s. We +# change this so it shows and updates every 1.0s. +# We also tell progress to assume a TTY is present so updates are printed +# even if there is no known TTY. +[progress] +delay = 1.0 +refresh = 1.0 +assume-tty = true + +[extensions] +share = +sparse = +robustcheckout = /usr/local/mercurial/robustcheckout.py + +[hostsecurity] +# When running a modern Python, Mercurial will default to TLS 1.1+. +# When running on a legacy Python, Mercurial will default to TLS 1.0+. +# There is no good reason we shouldn't be running a modern Python +# capable of speaking TLS 1.2. And the only Mercurial servers we care +# about should be running TLS 1.2. So make TLS 1.2 the minimum. +minimumprotocol = tls1.2 + +# Settings to make 1-click loaners more useful. +[extensions] +histedit = +rebase = + +[diff] +git = 1 +showfunc = 1 + +[pager] +pager = LESS=FRSXQ less diff --git a/taskcluster/docker/recipes/install-node.sh b/taskcluster/docker/recipes/install-node.sh new file mode 100644 index 0000000000..daeba2edc9 --- /dev/null +++ b/taskcluster/docker/recipes/install-node.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script installs Node v16. +# XXX For now, this should match the version installed in +# taskcluster/scripts/misc/repack-node.sh. Later we'll get the ESLint builder +# to use the linux64-node toolchain directly. + +wget -O node.xz --progress=dot:mega https://nodejs.org/dist/v16.19.0/node-v16.19.0-linux-x64.tar.xz +echo 'c88b52497ab38a3ddf526e5b46a41270320409109c3f74171b241132984fd08f' node.xz | sha256sum -c +tar -C /usr/local -xJ --strip-components 1 < node.xz +node -v # verify +npm -v diff --git a/taskcluster/docker/recipes/setup_packages.sh b/taskcluster/docker/recipes/setup_packages.sh new file mode 100755 index 0000000000..48dbf493f3 --- /dev/null +++ b/taskcluster/docker/recipes/setup_packages.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +TASKCLUSTER_ROOT_URL=$1 +shift + +# duplicate the functionality of taskcluster-lib-urls, but in bash.. +queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" + + +for task in "$@"; do + echo "adding package source $queue_base/task/$task/artifacts/public/build/" + echo "deb [trusted=yes] $queue_base/task/$task/artifacts/public/build/ apt/" > "/etc/apt/sources.list.d/99$task.list" +done diff --git a/taskcluster/docker/recipes/ubuntu1804-test-system-setup-base.sh b/taskcluster/docker/recipes/ubuntu1804-test-system-setup-base.sh new file mode 100644 index 0000000000..cb934529bf --- /dev/null +++ b/taskcluster/docker/recipes/ubuntu1804-test-system-setup-base.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash + +set -ve + +test "$(whoami)" == 'root' + +# We do want to install recommended packages. +sed -i /APT::Install-Recommends/d /etc/apt/apt.conf.d/99taskcluster + +# To speed up docker image build times as well as number of network/disk I/O +# build a list of packages to be installed nad call it in one go. +apt_packages=() + +apt_packages+=('autoconf2.13') +apt_packages+=('bluez-cups') +apt_packages+=('build-essential') +apt_packages+=('ca-certificates') +apt_packages+=('ccache') +apt_packages+=('compiz') +apt_packages+=('curl') +apt_packages+=('fontconfig') +apt_packages+=('fonts-kacst') +apt_packages+=('fonts-kacst-one') +apt_packages+=('fonts-liberation') +apt_packages+=('fonts-stix') +apt_packages+=('fonts-unfonts-core') +apt_packages+=('fonts-unfonts-extra') +apt_packages+=('fonts-vlgothic') +apt_packages+=('g++-multilib') +apt_packages+=('gcc-multilib') +apt_packages+=('gir1.2-gnomebluetooth-1.0') +apt_packages+=('git') +apt_packages+=('gnome-icon-theme') +apt_packages+=('gstreamer1.0-gtk3') +apt_packages+=('gstreamer1.0-plugins-base') +apt_packages+=('gstreamer1.0-plugins-good') +apt_packages+=('gstreamer1.0-tools') +apt_packages+=('gstreamer1.0-pulseaudio') +apt_packages+=('language-pack-en-base') +apt_packages+=('libc6-dbg') +apt_packages+=('libasound2-dev') +apt_packages+=('libavcodec-extra57') +apt_packages+=('libcanberra-gtk3-module') +apt_packages+=('libcanberra-pulse') +apt_packages+=('libcurl4-openssl-dev') +apt_packages+=('libdbus-1-dev') +apt_packages+=('libdbus-glib-1-dev') +apt_packages+=('libfreetype6') +apt_packages+=('libgconf2-dev') +apt_packages+=('libgl1-mesa-dri') +apt_packages+=('libgl1-mesa-glx') +apt_packages+=('libgstreamer-plugins-base1.0-dev') +apt_packages+=('libgstreamer1.0-dev') +apt_packages+=('libgtk-3-0') +apt_packages+=('libiw-dev') +apt_packages+=('libx11-xcb1') +apt_packages+=('libxcb1') +apt_packages+=('libxcb-render0') +apt_packages+=('libxcb-shm0') +apt_packages+=('libxcb-glx0') +apt_packages+=('libxcb-shape0') +apt_packages+=('libnotify-dev') +apt_packages+=('libpulse0') +apt_packages+=('libpulse-dev') +apt_packages+=('libxt-dev') +apt_packages+=('libxxf86vm1') +apt_packages+=('llvm') +apt_packages+=('llvm-dev') +apt_packages+=('llvm-runtime') +apt_packages+=('mesa-common-dev') +apt_packages+=('net-tools') +apt_packages+=('pulseaudio') +apt_packages+=('pulseaudio-module-bluetooth') +apt_packages+=('pulseaudio-module-gconf') +apt_packages+=('python-dev') +apt_packages+=('python-pip') +apt_packages+=('python3-pip') +apt_packages+=('qemu-kvm') +apt_packages+=('rlwrap') +apt_packages+=('screen') +apt_packages+=('software-properties-common') +apt_packages+=('sudo') +apt_packages+=('ttf-dejavu') +apt_packages+=('ubuntu-desktop') +apt_packages+=('unzip') +apt_packages+=('uuid') +apt_packages+=('wget') +apt_packages+=('xvfb') +apt_packages+=('xwit') +apt_packages+=('yasm') +apt_packages+=('zip') + +# xvinfo for test-linux.sh to monitor Xvfb startup +apt_packages+=('x11-utils') + +# Bug 1232407 - this allows the user to start vnc +apt_packages+=('x11vnc') + +# Bug 1176031 - need `xset` to disable screensavers +apt_packages+=('x11-xserver-utils') + +# Build a list of packages to install from the multiverse repo. +apt_packages+=('ubuntu-restricted-extras') + +# APT update takes very long on Ubuntu. Run it at the last possible minute. +apt-get update + +# Also force the cleanup after installation of packages to reduce image size. +apt-get install --allow-downgrades "${apt_packages[@]}" + +# Enable i386 packages +dpkg --add-architecture i386 +apt-get update + +# Make sure we have libraries for 32-bit tests +apt_packages=() +apt_packages+=('libavcodec-extra57:i386') +apt_packages+=('libdbus-glib-1-2:i386') +apt_packages+=('libpulse0:i386') +apt_packages+=('libxt6:i386') +apt_packages+=('libxtst6:i386') +apt_packages+=('libsecret-1-0:i386') +apt_packages+=('libgtk-3-0:i386') +apt_packages+=('libx11-xcb1:i386') +apt_packages+=('libxcb1:i386') +apt_packages+=('libasound2:i386') + +apt-get install --allow-downgrades "${apt_packages[@]}" +rm -rf /var/lib/apt/lists/* + +# Build a list of packages to purge from the image. +apt_packages=() +apt_packages+=('*cheese*') +apt_packages+=('example-content') +apt_packages+=('git') +apt_packages+=('gnome-calendar') +apt_packages+=('gnome-initial-setup') +apt_packages+=('gnome-mahjongg') +apt_packages+=('gnome-mines') +apt_packages+=('gnome-sudoku') +apt_packages+=('libx11-doc') +apt_packages+=('manpages-dev') +apt_packages+=('orca') +apt_packages+=('rhythmbox') +apt_packages+=('thunderbird') +apt_packages+=('ubuntu-release-upgrader*') +apt_packages+=('update-manager-core') +apt_packages+=('update-manager') +apt_packages+=('yelp') + +# Purge unnecessary packages +apt-get purge "${apt_packages[@]}" + +# We don't need no docs! +rm -rf /usr/share/help /usr/share/doc /usr/share/man + +# Remove all locale files other than en_US.UTF-8 +rm -rf /usr/share/locale/ /usr/share/locale-langpack/ /usr/share/locales/ + +# Further cleanup +apt-get autoremove + +rm -f "$0" diff --git a/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh b/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh new file mode 100644 index 0000000000..e22b21f51d --- /dev/null +++ b/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -ve + +test "$(whoami)" == 'root' + +cd /setup + +# Install tooltool and node now that dependencies are in place. +. /setup/common.sh +. /setup/install-node.sh + +# Upgrade pip and install virtualenv to specified versions. +pip install --upgrade pip==19.2.3 +hash -r +pip install virtualenv==15.2.0 + +pip3 install -r /setup/psutil_requirements.txt +pip install -r /setup/psutil_requirements.txt + +# Cleanup +cd / +rm -rf /setup ~/.ccache ~/.cache ~/.npm +rm -f "$0" diff --git a/taskcluster/docker/recipes/xvfb.sh b/taskcluster/docker/recipes/xvfb.sh new file mode 100644 index 0000000000..6e0e79f7d4 --- /dev/null +++ b/taskcluster/docker/recipes/xvfb.sh @@ -0,0 +1,75 @@ +#! /bin/bash -x + +set -x + +fail() { + echo # make sure error message is on a new line + echo "[xvfb.sh:error]" "${@}" + exit 1 +} + +cleanup_xvfb() { + # When you call this script with START_VNC or TASKCLUSTER_INTERACTIVE + # we make sure we do not kill xvfb so you do not lose your connection + local xvfb_pid=`pidof Xvfb` + local vnc=${START_VNC:-false} + local interactive=${TASKCLUSTER_INTERACTIVE:-false} + if [ -n "$xvfb_pid" ] && [[ $vnc == false ]] && [[ $interactive == false ]] ; then + kill $xvfb_pid || true + screen -XS xvfb quit || true + fi +} + +# Attempt to start xvfb in a screen session with the given resolution and display +# number. Up to 5 attempts will be made to start xvfb with a short delay +# between retries +try_xvfb() { + screen -dmS xvfb Xvfb :$2 -nolisten tcp -screen 0 $1 \ + > ~/artifacts/xvfb/xvfb.log 2>&1 + export DISPLAY=:$2 + + # Only error code 255 matters, because it signifies that no + # display could be opened. As long as we can open the display + # tests should work. We'll retry a few times with a sleep before + # failing. + local retry_count=0 + local max_retries=5 + xvfb_test=0 + until [ $retry_count -gt $max_retries ]; do + xvinfo || xvfb_test=$? + if [ $xvfb_test != 255 ]; then + retry_count=$(($max_retries + 1)) + else + retry_count=$(($retry_count + 1)) + echo "Failed to start Xvfb, retry: $retry_count" + sleep 2 + fi + done + if [ $xvfb_test == 255 ]; then + return 1 + else + return 0 + fi +} + +start_xvfb() { + set +e + mkdir -p ~/artifacts/xvfb + local retry_count=0 + local max_retries=2 + local success=1 + until [ $retry_count -gt $max_retries ]; do + try_xvfb $1 $2 + success=$? + if [ $success -eq 0 ]; then + retry_count=$(($max_retries + 1)) + else + retry_count=$(($retry_count + 1)) + sleep 10 + fi + done + set -e + if [ $success -eq 1 ]; then + fail "Could not start xvfb after ${max_retries} attempts" + fi +} |