diff options
Diffstat (limited to 'taskcluster/taskgraph/parameters.py')
| -rw-r--r-- | taskcluster/taskgraph/parameters.py | 327 |
1 files changed, 327 insertions, 0 deletions
diff --git a/taskcluster/taskgraph/parameters.py b/taskcluster/taskgraph/parameters.py new file mode 100644 index 0000000000..76ed47c950 --- /dev/null +++ b/taskcluster/taskgraph/parameters.py @@ -0,0 +1,327 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import io +import logging +import os.path +import json +from datetime import datetime + +from mozbuild.util import ReadOnlyDict, memoize +from mozversioncontrol import get_repository_object +from taskgraph.util.schema import validate_schema +from voluptuous import ( + ALLOW_EXTRA, + Any, + Inclusive, + Required, + Schema, +) + +import six +from six import text_type + +from . import GECKO +from .util.attributes import release_level + +logger = logging.getLogger(__name__) + + +class ParameterMismatch(Exception): + """Raised when a parameters.yml has extra or missing parameters.""" + + +@memoize +def get_head_ref(): + return six.ensure_text(get_repository_object(GECKO).head_ref) + + +def get_contents(path): + with io.open(path, "r") as fh: + contents = fh.readline().rstrip() + return contents + + +def get_version(product_dir="browser"): + version_path = os.path.join(GECKO, product_dir, "config", "version_display.txt") + return get_contents(version_path) + + +def get_app_version(product_dir="browser"): + app_version_path = os.path.join(GECKO, product_dir, "config", "version.txt") + return get_contents(app_version_path) + + +base_schema = Schema( + { + Required("app_version"): text_type, + Required("backstop"): bool, + Required("base_repository"): text_type, + Required("build_date"): int, + Required("build_number"): int, + Inclusive("comm_base_repository", "comm"): text_type, + Inclusive("comm_head_ref", "comm"): text_type, + Inclusive("comm_head_repository", "comm"): text_type, + Inclusive("comm_head_rev", "comm"): text_type, + Required("do_not_optimize"): [text_type], + Required("existing_tasks"): {text_type: text_type}, + Required("filters"): [text_type], + Required("head_ref"): text_type, + Required("head_repository"): text_type, + Required("head_rev"): text_type, + Required("hg_branch"): text_type, + Required("level"): text_type, + Required("message"): text_type, + Required("moz_build_date"): text_type, + Required("next_version"): Any(None, text_type), + Required("optimize_strategies"): Any(None, text_type), + Required("optimize_target_tasks"): bool, + Required("owner"): text_type, + Required("phabricator_diff"): Any(None, text_type), + Required("project"): text_type, + Required("pushdate"): int, + Required("pushlog_id"): text_type, + Required("release_enable_emefree"): bool, + Required("release_enable_partner_repack"): bool, + Required("release_enable_partner_attribution"): bool, + Required("release_eta"): Any(None, text_type), + Required("release_history"): {text_type: dict}, + Required("release_partners"): Any(None, [text_type]), + Required("release_partner_config"): Any(None, dict), + Required("release_partner_build_number"): int, + Required("release_type"): text_type, + Required("release_product"): Any(None, text_type), + Required("required_signoffs"): [text_type], + Required("signoff_urls"): dict, + # target-kind is not included, since it should never be + # used at run-time + Required("target_tasks_method"): text_type, + Required("tasks_for"): text_type, + Required("test_manifest_loader"): text_type, + Required("try_mode"): Any(None, text_type), + Required("try_options"): Any(None, dict), + Required("try_task_config"): dict, + Required("version"): text_type, + } +) + + +COMM_PARAMETERS = [ + "comm_base_repository", + "comm_head_ref", + "comm_head_repository", + "comm_head_rev", +] + + +def extend_parameters_schema(schema): + """ + Extend the schema for parameters to include per-project configuration. + + This should be called by the `taskgraph.register` function in the + graph-configuration. + """ + global base_schema + base_schema = base_schema.extend(schema) + + +class Parameters(ReadOnlyDict): + """An immutable dictionary with nicer KeyError messages on failure""" + + def __init__(self, strict=True, **kwargs): + self.strict = strict + + if not self.strict: + # apply defaults to missing parameters + kwargs = Parameters._fill_defaults(**kwargs) + + ReadOnlyDict.__init__(self, **kwargs) + + @staticmethod + def _fill_defaults(**kwargs): + now = datetime.utcnow() + epoch = datetime.utcfromtimestamp(0) + seconds_from_epoch = int((now - epoch).total_seconds()) + + defaults = { + "app_version": get_app_version(), + "backstop": False, + "base_repository": "https://hg.mozilla.org/mozilla-unified", + "build_date": seconds_from_epoch, + "build_number": 1, + "do_not_optimize": [], + "existing_tasks": {}, + "filters": ["target_tasks_method"], + "head_ref": get_head_ref(), + "head_repository": "https://hg.mozilla.org/mozilla-central", + "head_rev": get_head_ref(), + "hg_branch": "default", + "level": "3", + "message": "", + "moz_build_date": six.ensure_text(now.strftime("%Y%m%d%H%M%S")), + "next_version": None, + "optimize_strategies": None, + "optimize_target_tasks": True, + "owner": "nobody@mozilla.com", + "phabricator_diff": None, + "project": "mozilla-central", + "pushdate": seconds_from_epoch, + "pushlog_id": "0", + "release_enable_emefree": False, + "release_enable_partner_repack": False, + "release_enable_partner_attribution": False, + "release_eta": "", + "release_history": {}, + "release_partners": [], + "release_partner_config": None, + "release_partner_build_number": 1, + "release_product": None, + "release_type": "nightly", + "required_signoffs": [], + "signoff_urls": {}, + "target_tasks_method": "default", + "tasks_for": "hg-push", + "test_manifest_loader": "default", + "try_mode": None, + "try_options": None, + "try_task_config": {}, + "version": get_version(), + } + + if set(COMM_PARAMETERS) & set(kwargs): + defaults.update( + { + "comm_base_repository": "https://hg.mozilla.org/comm-central", + "comm_head_repository": "https://hg.mozilla.org/comm-central", + } + ) + + for name, default in defaults.items(): + if name not in kwargs: + kwargs[name] = default + + return kwargs + + def check(self): + schema = ( + base_schema if self.strict else base_schema.extend({}, extra=ALLOW_EXTRA) + ) + validate_schema(schema, self.copy(), "Invalid parameters:") + + def __getitem__(self, k): + try: + return super(Parameters, self).__getitem__(k) + except KeyError: + raise KeyError("taskgraph parameter {!r} not found".format(k)) + + def is_try(self): + """ + Determine whether this graph is being built on a try project or for + `mach try fuzzy`. + """ + return "try" in self["project"] or self["try_mode"] == "try_select" + + def file_url(self, path, pretty=False): + """ + Determine the VCS URL for viewing a file in the tree, suitable for + viewing by a human. + + :param text_type path: The path, relative to the root of the repository. + :param bool pretty: Whether to return a link to a formatted version of the + file, or the raw file version. + :return text_type: The URL displaying the given path. + """ + if path.startswith("comm/"): + path = path[len("comm/") :] + repo = self["comm_head_repository"] + rev = self["comm_head_rev"] + else: + repo = self["head_repository"] + rev = self["head_rev"] + + endpoint = "file" if pretty else "raw-file" + return "{}/{}/{}/{}".format(repo, endpoint, rev, path) + + def release_level(self): + """ + Whether this is a staging release or not. + + :return six.text_type: One of "production" or "staging". + """ + return release_level(self["project"]) + + +def load_parameters_file(filename, strict=True, overrides=None, trust_domain=None): + """ + Load parameters from a path, url, decision task-id or project. + + Examples: + task-id=fdtgsD5DQUmAQZEaGMvQ4Q + project=mozilla-central + """ + import requests + from taskgraph.util.taskcluster import get_artifact_url, find_task_id + from taskgraph.util import yaml + + if overrides is None: + overrides = {} + + if not filename: + return Parameters(strict=strict, **overrides) + + try: + # reading parameters from a local parameters.yml file + f = open(filename) + except IOError: + # fetching parameters.yml using task task-id, project or supplied url + task_id = None + if filename.startswith("task-id="): + task_id = filename.split("=")[1] + elif filename.startswith("project="): + if trust_domain is None: + raise ValueError( + "Can't specify parameters by project " + "if trust domain isn't supplied.", + ) + index = "{trust_domain}.v2.{project}.latest.taskgraph.decision".format( + trust_domain=trust_domain, + project=filename.split("=")[1], + ) + task_id = find_task_id(index) + + if task_id: + filename = get_artifact_url(task_id, "public/parameters.yml") + logger.info("Loading parameters from {}".format(filename)) + resp = requests.get(filename, stream=True) + resp.raise_for_status() + f = resp.raw + + if filename.endswith(".yml"): + kwargs = yaml.load_stream(f) + elif filename.endswith(".json"): + kwargs = json.load(f) + else: + raise TypeError("Parameters file `{}` is not JSON or YAML".format(filename)) + + kwargs.update(overrides) + + return Parameters(strict=strict, **kwargs) + + +def parameters_loader(filename, strict=True, overrides=None): + def get_parameters(graph_config): + parameters = load_parameters_file( + filename, + strict=strict, + overrides=overrides, + trust_domain=graph_config["trust-domain"], + ) + parameters.check() + return parameters + + return get_parameters |