diff options
Diffstat (limited to 'taskcluster/docker/decision')
| -rw-r--r-- | taskcluster/docker/decision/Dockerfile | 15 | ||||
| -rw-r--r-- | taskcluster/docker/decision/HASH | 1 | ||||
| -rw-r--r-- | taskcluster/docker/decision/README.md | 5 | ||||
| -rw-r--r-- | taskcluster/docker/decision/VERSION | 1 | ||||
| -rwxr-xr-x | taskcluster/docker/decision/comm-task-env | 199 | ||||
| -rw-r--r-- | taskcluster/docker/decision/system-setup.sh | 14 |
6 files changed, 235 insertions, 0 deletions
diff --git a/taskcluster/docker/decision/Dockerfile b/taskcluster/docker/decision/Dockerfile new file mode 100644 index 0000000000..f2aa83903e --- /dev/null +++ b/taskcluster/docker/decision/Dockerfile @@ -0,0 +1,15 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Andrew Halberstadt <ahal@mozilla.com> + +RUN mkdir /builds/worker/artifacts && \ + chown worker:worker /builds/worker/artifacts + +ADD system-setup.sh /tmp/system-setup.sh +RUN bash /tmp/system-setup.sh + +ADD comm-task-env /builds/worker/bin/comm-task-env + +ENV PATH=/builds/worker/bin:$PATH + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/decision/HASH b/taskcluster/docker/decision/HASH new file mode 100644 index 0000000000..b854e10d0f --- /dev/null +++ b/taskcluster/docker/decision/HASH @@ -0,0 +1 @@ +sha256:9f69fe08c28e3cb3cc296451f0a2735df6e25d0e3c877ea735ef1b7f0b345b06 diff --git a/taskcluster/docker/decision/README.md b/taskcluster/docker/decision/README.md new file mode 100644 index 0000000000..4490880be8 --- /dev/null +++ b/taskcluster/docker/decision/README.md @@ -0,0 +1,5 @@ +# Decision Tasks + +The decision image is a "boostrapping" image for the in tree logic it +deals with cloning gecko and the related utilities for providing an +environment where we can run gecko. diff --git a/taskcluster/docker/decision/VERSION b/taskcluster/docker/decision/VERSION new file mode 100644 index 0000000000..fcdb2e109f --- /dev/null +++ b/taskcluster/docker/decision/VERSION @@ -0,0 +1 @@ +4.0.0 diff --git a/taskcluster/docker/decision/comm-task-env b/taskcluster/docker/decision/comm-task-env new file mode 100755 index 0000000000..65481497ae --- /dev/null +++ b/taskcluster/docker/decision/comm-task-env @@ -0,0 +1,199 @@ +#!/usr/bin/python3 -u +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Thunderbird build environment prep for run-task, +for use with comm-central derived repositories. + +This script is meant to run prior to run-task on repositories like +comm-central that need to check out a copy of a mozilla repository +in order to build. +See bug 1491371 for background on why this is necessary. + +A project will have a file named ".gecko_rev.yml" in it's root. See the +constant "GECKO_REV_CONF" if you want to change that. To download it, the +script uses the project repository URL and the revision number. +Those are defined in the environment variables: +COMM_HEAD_REPOSITORY +COMM_HEAD_REV + +.gecko_rev.yml has a structure like (for comm-central): +``` +GECKO_BASE_REPOSITORY: https://hg.mozilla.org/mozilla-unified +GECKO_HEAD_REPOSITORY: https://hg.mozilla.org/mozilla-central +GECKO_HEAD_REF: default +``` +or for branches: +``` +GECKO_BASE_REPOSITORY: https://hg.mozilla.org/mozilla-unified +GECKO_HEAD_REPOSITORY: https://hg.mozilla.org/releases/mozilla-beta +GECKO_HEAD_REF: THUNDERBIRD_60_VERBRANCH +GECKO_HEAD_REV: 6a830d12f15493a70b1192022c9985eba2139910 + +Note about GECKO_HEAD_REV and GECKO_HEAD_REF: +GECKO_HEAD_REF is a branch name or "default". +GECKO_HEAD_REV is a revision hash. +``` +""" + +import sys + +import os +import socket +import time +from datetime import datetime +from pprint import pformat + +import urllib.error +import urllib.request + +import yaml + +if sys.version_info[0:2] < (3, 5): + print('run-task-wrapper requires Python 3.5+') + sys.exit(1) + +GECKO_REV_CONF = ".gecko_rev.yml" +DEBUG = bool(os.environ.get("RTW_DEBUG", False)) + + +def print_message(msg, prefix=__file__, level=""): + """ + Print messages. + :param object msg: message to print, usually a string, but not always + :param str prefix: message prefix + :param str level: message level (DEBUG, ERROR, INFO) + """ + if not isinstance(msg, str): + msg = pformat(msg) + now = datetime.utcnow().isoformat() + # slice microseconds to 3 decimals. + now = now[:-3] if now[-7:-6] == '.' else now + if level: + sys.stdout.write('[{prefix} {now}Z] {level}: {msg}\n'.format( + prefix=prefix, now=now, level=level, msg=msg)) + else: + sys.stdout.write('[{prefix} {now}Z] {msg}\n'.format( + prefix=prefix, now=now, msg=msg)) + sys.stdout.flush() + + +def error_exit(msg): + """Print the error message and exit with error.""" + print_message(msg, level="ERROR") + if DEBUG: + raise Exception(msg) + + sys.exit(1) + + +def print_debug(msg): + """Prints a message with DEBUG prefix if DEBUG is enabled + with the environment variable "RTW_DEBUG". + """ + if DEBUG: + print_message(msg, level="DEBUG") + + +def check_environ(): + """Check that the necessary environment variables to find the + comm- repository are defined. (Set in .taskcluster.yml) + :return: tuple(str, str) + """ + print_debug("Checking environment variables...") + project_head_repo = os.environ.get("COMM_HEAD_REPOSITORY", None) + project_head_rev = os.environ.get("COMM_HEAD_REV", None) + + if project_head_repo is None or project_head_rev is None: + error_exit("Environment NOT Ok:\n\tHead: {}\n\tRev: {}\n").format( + project_head_repo, project_head_rev) + + print_debug("Environment Ok:\n\tHead: {}\n\tRev: {}\n".format( + project_head_repo, project_head_rev)) + return project_head_repo, project_head_rev + + +def download_url(url, retry=1): + """Downloads the given URL. Naively retries (when asked) upon failure + :param url: str + :param retry: int + :return: str + """ + # Use 1-based counting for display and calculation purposes. + for i in range(1, retry+1): + try: + print_message('Fetching {}. Attempt {} of {}.'.format( + url, i, retry)) + with urllib.request.urlopen(url, timeout=10) as response: + data = response.read().decode("utf-8") + return data + except (urllib.error.URLError, socket.timeout) as exc: + print_message('Unable to retrieve {}'.format(url)) + if isinstance(exc, urllib.error.URLError): + print_message(exc.reason) + else: # socket.timeout + print_message('Connection timed out.') + + if i < retry: # No more retries + wait_time = i * 5 # fail #1: sleep 5s. #2, sleep 10s + print_message('Retrying in {} seconds.'.format(wait_time)) + time.sleep(wait_time) + + error_exit('No more retry attempts! Aborting.') + + +def fetch_gecko_conf(project_head_repo, project_revision): + """Downloads .gecko_rev.yml from the project repository + :param project_head_repo: str + :param project_revision: str + :return: dict + """ + gecko_conf_url = '/'.join( + [project_head_repo, 'raw-file', project_revision, GECKO_REV_CONF]) + + gecko_conf_yml = download_url(gecko_conf_url, retry=5) + + try: + gecko_conf = yaml.safe_load(gecko_conf_yml) + return gecko_conf + except yaml.YAMLError as exc: + err_txt = ["Error processing Gecko YAML configuration."] + if hasattr(exc, "problem_mark"): + mark = exc.problem_mark # pylint: disable=no-member + err_txt.append("Error position: line {}, column {}".format( + mark.line + 1, mark.column + 1)) + error_exit('\n'.join(err_txt)) + + +def update_environment(gecko_conf): + """Adds the new variables defined in gecko_conf to the + running environment. + :param gecko_conf: dict + """ + print_message("Updating environment with:") + print_message(gecko_conf) + os.environ.update(gecko_conf) + + print_debug("New environment:") + print_debug(os.environ) + + +def exec_run_task(args): + """Executes run-task with a modified environment.""" + print_message("Executing: {}".format(pformat(args))) + os.execv(args[0], args[0:]) + + +def main(): + """Main function.""" + args = sys.argv[1:] # Remaining args starting with run-task + + project_head_repo, project_revision = check_environ() + gecko_conf = fetch_gecko_conf(project_head_repo, project_revision) + update_environment(gecko_conf) + exec_run_task(args) + + +if __name__ == "__main__": + main() diff --git a/taskcluster/docker/decision/system-setup.sh b/taskcluster/docker/decision/system-setup.sh new file mode 100644 index 0000000000..0f9d04fa50 --- /dev/null +++ b/taskcluster/docker/decision/system-setup.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -v -e + +test "$(whoami)" == 'root' + +apt-get update +apt-get install \ + python \ + sudo \ + python3-yaml + +rm -rf /var/lib/apt/lists/ +rm "$0" |