# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """Make scriptworker.cot.verify more user friendly by making scopes dynamic. Scriptworker uses certain scopes to determine which sets of credentials to use. Certain scopes are restricted by branch in chain of trust verification, and are checked again at the script level. This file provides functions to adjust these scopes automatically by project; this makes pushing to try, forking a project branch, and merge day uplifts more user friendly. In the future, we may adjust scopes by other settings as well, e.g. different scopes for `push-to-candidates` rather than `push-to-releases`, even if both happen on mozilla-beta and mozilla-release. Additional configuration is found in the :ref:`graph config `. """ import functools import itertools import json import os from datetime import datetime import jsone from mozbuild.util import memoize from taskgraph.util.schema import resolve_keyed_by from taskgraph.util.taskcluster import get_artifact_prefix from taskgraph.util.yaml import load_yaml from gecko_taskgraph.util.copy_task import copy_task # constants {{{1 """Map signing scope aliases to sets of projects. Currently m-c and DevEdition on m-b use nightly signing; Beta on m-b and m-r use release signing. These data structures aren't set-up to handle different scopes on the same repo, so we use a different set of them for DevEdition, and callers are responsible for using the correct one (by calling the appropriate helper below). More context on this in https://bugzilla.mozilla.org/show_bug.cgi?id=1358601. We will need to add esr support at some point. Eventually we want to add nuance so certain m-b and m-r tasks use dep or nightly signing, and we only release sign when we have a signed-off set of candidate builds. This current approach works for now, though. This is a list of list-pairs, for ordering. """ SIGNING_SCOPE_ALIAS_TO_PROJECT = [ [ "all-nightly-branches", { "mozilla-central", "comm-central", # bug 1845368: pine is a permanent project branch used for testing # nightly updates "pine", # bug 1877483: larch has similar needs for nightlies "larch", }, ], [ "all-release-branches", { "mozilla-beta", "mozilla-release", "mozilla-esr115", "comm-beta", "comm-release", "comm-esr115", }, ], ] """Map the signing scope aliases to the actual scopes. """ SIGNING_CERT_SCOPES = { "all-release-branches": "signing:cert:release-signing", "all-nightly-branches": "signing:cert:nightly-signing", "default": "signing:cert:dep-signing", } DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT = [ [ "beta", { "mozilla-beta", }, ] ] DEVEDITION_SIGNING_CERT_SCOPES = { "beta": "signing:cert:nightly-signing", "default": "signing:cert:dep-signing", } """Map beetmover scope aliases to sets of projects. """ BEETMOVER_SCOPE_ALIAS_TO_PROJECT = [ [ "all-nightly-branches", { "mozilla-central", "comm-central", # bug 1845368: pine is a permanent project branch used for testing # nightly updates "pine", # bug 1877483: larch has similar needs for nightlies "larch", }, ], [ "all-release-branches", { "mozilla-beta", "mozilla-release", "mozilla-esr115", "comm-beta", "comm-release", "comm-esr115", }, ], ] """Map the beetmover scope aliases to the actual scopes. """ BEETMOVER_BUCKET_SCOPES = { "all-release-branches": "beetmover:bucket:release", "all-nightly-branches": "beetmover:bucket:nightly", "default": "beetmover:bucket:dep", } """Map the beetmover scope aliases to the actual scopes. These are the scopes needed to import artifacts into the product delivery APT repos. """ BEETMOVER_APT_REPO_SCOPES = { "all-release-branches": "beetmover:apt-repo:release", "all-nightly-branches": "beetmover:apt-repo:nightly", "default": "beetmover:apt-repo:dep", } """Map the beetmover tasks aliases to the actual action scopes. """ BEETMOVER_ACTION_SCOPES = { "nightly": "beetmover:action:push-to-nightly", # bug 1845368: pine is a permanent project branch used for testing # nightly updates "nightly-pine": "beetmover:action:push-to-nightly", # bug 1877483: larch has similar needs for nightlies "nightly-larch": "beetmover:action:push-to-nightly", "default": "beetmover:action:push-to-candidates", } """Map the beetmover tasks aliases to the actual action scopes. The action scopes are generic across different repo types. """ BEETMOVER_REPO_ACTION_SCOPES = { "default": "beetmover:action:import-from-gcs-to-artifact-registry", } """Known balrog actions.""" BALROG_ACTIONS = ( "submit-locale", "submit-toplevel", "schedule", "v2-submit-locale", "v2-submit-toplevel", ) """Map balrog scope aliases to sets of projects. This is a list of list-pairs, for ordering. """ BALROG_SCOPE_ALIAS_TO_PROJECT = [ [ "nightly", { "mozilla-central", "comm-central", # bug 1845368: pine is a permanent project branch used for testing # nightly updates "pine", # bug 1877483: larch has similar needs for nightlies "larch", }, ], [ "beta", { "mozilla-beta", "comm-beta", }, ], [ "release", { "mozilla-release", "comm-release", }, ], [ "esr115", { "mozilla-esr115", "comm-esr115", }, ], ] """Map the balrog scope aliases to the actual scopes. """ BALROG_SERVER_SCOPES = { "nightly": "balrog:server:nightly", "aurora": "balrog:server:aurora", "beta": "balrog:server:beta", "release": "balrog:server:release", "esr115": "balrog:server:esr", "default": "balrog:server:dep", } """ The list of the release promotion phases which we send notifications for """ RELEASE_NOTIFICATION_PHASES = ("promote", "push", "ship") def add_scope_prefix(config, scope): """ Prepends the scriptworker scope prefix from the :ref:`graph config `. Args: config (TransformConfig): The configuration for the kind being transformed. scope (string): The suffix of the scope Returns: string: the scope to use. """ return "{prefix}:{scope}".format( prefix=config.graph_config["scriptworker"]["scope-prefix"], scope=scope, ) def with_scope_prefix(f): """ Wraps a function, calling :py:func:`add_scope_prefix` on the result of calling the wrapped function. Args: f (callable): A function that takes a ``config`` and some keyword arguments, and returns a scope suffix. Returns: callable: the wrapped function """ @functools.wraps(f) def wrapper(config, **kwargs): scope_or_scopes = f(config, **kwargs) if isinstance(scope_or_scopes, list): return map(functools.partial(add_scope_prefix, config), scope_or_scopes) return add_scope_prefix(config, scope_or_scopes) return wrapper # scope functions {{{1 @with_scope_prefix def get_scope_from_project(config, alias_to_project_map, alias_to_scope_map): """Determine the restricted scope from `config.params['project']`. Args: config (TransformConfig): The configuration for the kind being transformed. alias_to_project_map (list of lists): each list pair contains the alias and the set of projects that match. This is ordered. alias_to_scope_map (dict): the alias alias to scope Returns: string: the scope to use. """ for alias, projects in alias_to_project_map: if config.params["project"] in projects and alias in alias_to_scope_map: return alias_to_scope_map[alias] return alias_to_scope_map["default"] @with_scope_prefix def get_scope_from_release_type(config, release_type_to_scope_map): """Determine the restricted scope from `config.params['target_tasks_method']`. Args: config (TransformConfig): The configuration for the kind being transformed. release_type_to_scope_map (dict): the maps release types to scopes Returns: string: the scope to use. """ return release_type_to_scope_map.get( config.params["release_type"], release_type_to_scope_map["default"] ) def get_phase_from_target_method(config, alias_to_tasks_map, alias_to_phase_map): """Determine the phase from `config.params['target_tasks_method']`. Args: config (TransformConfig): The configuration for the kind being transformed. alias_to_tasks_map (list of lists): each list pair contains the alias and the set of target methods that match. This is ordered. alias_to_phase_map (dict): the alias to phase map Returns: string: the phase to use. """ for alias, tasks in alias_to_tasks_map: if ( config.params["target_tasks_method"] in tasks and alias in alias_to_phase_map ): return alias_to_phase_map[alias] return alias_to_phase_map["default"] get_signing_cert_scope = functools.partial( get_scope_from_project, alias_to_project_map=SIGNING_SCOPE_ALIAS_TO_PROJECT, alias_to_scope_map=SIGNING_CERT_SCOPES, ) get_devedition_signing_cert_scope = functools.partial( get_scope_from_project, alias_to_project_map=DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT, alias_to_scope_map=DEVEDITION_SIGNING_CERT_SCOPES, ) get_beetmover_bucket_scope = functools.partial( get_scope_from_project, alias_to_project_map=BEETMOVER_SCOPE_ALIAS_TO_PROJECT, alias_to_scope_map=BEETMOVER_BUCKET_SCOPES, ) get_beetmover_apt_repo_scope = functools.partial( get_scope_from_project, alias_to_project_map=BEETMOVER_SCOPE_ALIAS_TO_PROJECT, alias_to_scope_map=BEETMOVER_APT_REPO_SCOPES, ) get_beetmover_repo_action_scope = functools.partial( get_scope_from_release_type, release_type_to_scope_map=BEETMOVER_REPO_ACTION_SCOPES, ) get_beetmover_action_scope = functools.partial( get_scope_from_release_type, release_type_to_scope_map=BEETMOVER_ACTION_SCOPES, ) get_balrog_server_scope = functools.partial( get_scope_from_project, alias_to_project_map=BALROG_SCOPE_ALIAS_TO_PROJECT, alias_to_scope_map=BALROG_SERVER_SCOPES, ) cached_load_yaml = memoize(load_yaml) # release_config {{{1 def get_release_config(config): """Get the build number and version for a release task. Currently only applies to beetmover tasks. Args: config (TransformConfig): The configuration for the kind being transformed. Returns: dict: containing both `build_number` and `version`. This can be used to update `task.payload`. """ release_config = {} partial_updates = os.environ.get("PARTIAL_UPDATES", "") if partial_updates != "" and config.kind in ( "release-bouncer-sub", "release-bouncer-check", "release-update-verify-config", "release-secondary-update-verify-config", "release-balrog-submit-toplevel", "release-secondary-balrog-submit-toplevel", ): partial_updates = json.loads(partial_updates) release_config["partial_versions"] = ", ".join( [ "{}build{}".format(v, info["buildNumber"]) for v, info in partial_updates.items() ] ) if release_config["partial_versions"] == "{}": del release_config["partial_versions"] release_config["version"] = config.params["version"] release_config["appVersion"] = config.params["app_version"] release_config["next_version"] = config.params["next_version"] release_config["build_number"] = config.params["build_number"] return release_config def get_signing_cert_scope_per_platform(build_platform, is_shippable, config): if "devedition" in build_platform: return get_devedition_signing_cert_scope(config) if is_shippable: return get_signing_cert_scope(config) return add_scope_prefix(config, "signing:cert:dep-signing") # generate_beetmover_upstream_artifacts {{{1 def generate_beetmover_upstream_artifacts( config, job, platform, locale=None, dependencies=None, **kwargs ): """Generate the upstream artifacts for beetmover, using the artifact map. Currently only applies to beetmover tasks. Args: job (dict): The current job being generated dependencies (list): A list of the job's dependency labels. platform (str): The current build platform locale (str): The current locale being beetmoved. Returns: list: A list of dictionaries conforming to the upstream_artifacts spec. """ base_artifact_prefix = get_artifact_prefix(job) resolve_keyed_by( job, "attributes.artifact_map", "artifact map", **{ "release-type": config.params["release_type"], "platform": platform, }, ) map_config = copy_task(cached_load_yaml(job["attributes"]["artifact_map"])) upstream_artifacts = list() if not locale: locales = map_config["default_locales"] elif isinstance(locale, list): locales = locale else: locales = [locale] if not dependencies: if job.get("dependencies"): dependencies = job["dependencies"].keys() else: raise Exception(f"Unsupported type of dependency. Got job: {job}") for locale, dep in itertools.product(locales, dependencies): paths = list() for filename in map_config["mapping"]: resolve_keyed_by( map_config["mapping"][filename], "from", f"beetmover filename {filename}", platform=platform, ) if dep not in map_config["mapping"][filename]["from"]: continue if locale != "en-US" and not map_config["mapping"][filename]["all_locales"]: continue if ( "only_for_platforms" in map_config["mapping"][filename] and platform not in map_config["mapping"][filename]["only_for_platforms"] ): continue if ( "not_for_platforms" in map_config["mapping"][filename] and platform in map_config["mapping"][filename]["not_for_platforms"] ): continue if "partials_only" in map_config["mapping"][filename]: continue # The next time we look at this file it might be a different locale. file_config = copy_task(map_config["mapping"][filename]) resolve_keyed_by( file_config, "source_path_modifier", "source path modifier", locale=locale, ) kwargs["locale"] = locale paths.append( os.path.join( base_artifact_prefix, jsone.render(file_config["source_path_modifier"], kwargs), jsone.render(filename, kwargs), ) ) if ( job.get("dependencies") and getattr(job["dependencies"][dep], "attributes", None) and job["dependencies"][dep].attributes.get("release_artifacts") ): paths = [ path for path in paths if path in job["dependencies"][dep].attributes["release_artifacts"] ] if not paths: continue upstream_artifacts.append( { "taskId": {"task-reference": f"<{dep}>"}, "taskType": map_config["tasktype_map"].get(dep), "paths": sorted(paths), "locale": locale, } ) upstream_artifacts.sort(key=lambda u: u["paths"]) return upstream_artifacts def generate_artifact_registry_gcs_sources(dep): gcs_sources = [] locale = dep.attributes.get("locale") if not locale: repackage_deb_reference = "" repackage_deb_artifact = "public/build/target.deb" else: repackage_deb_reference = "" repackage_deb_artifact = f"public/build/{locale}/target.langpack.deb" for config in dep.task["payload"]["artifactMap"]: if ( config["taskId"]["task-reference"] == repackage_deb_reference and repackage_deb_artifact in config["paths"] ): gcs_sources.append( config["paths"][repackage_deb_artifact]["destinations"][0] ) return gcs_sources # generate_beetmover_artifact_map {{{1 def generate_beetmover_artifact_map(config, job, **kwargs): """Generate the beetmover artifact map. Currently only applies to beetmover tasks. Args: config (): Current taskgraph configuration. job (dict): The current job being generated Common kwargs: platform (str): The current build platform locale (str): The current locale being beetmoved. Returns: list: A list of dictionaries containing source->destination maps for beetmover. """ platform = kwargs.get("platform", "") resolve_keyed_by( job, "attributes.artifact_map", job["label"], **{ "release-type": config.params["release_type"], "platform": platform, }, ) map_config = copy_task(cached_load_yaml(job["attributes"]["artifact_map"])) base_artifact_prefix = map_config.get( "base_artifact_prefix", get_artifact_prefix(job) ) artifacts = list() dependencies = job["dependencies"].keys() if kwargs.get("locale"): if isinstance(kwargs["locale"], list): locales = kwargs["locale"] else: locales = [kwargs["locale"]] else: locales = map_config["default_locales"] resolve_keyed_by(map_config, "s3_bucket_paths", job["label"], platform=platform) for locale, dep in sorted(itertools.product(locales, dependencies)): paths = dict() for filename in map_config["mapping"]: # Relevancy checks resolve_keyed_by( map_config["mapping"][filename], "from", "blah", platform=platform ) if dep not in map_config["mapping"][filename]["from"]: # We don't get this file from this dependency. continue if locale != "en-US" and not map_config["mapping"][filename]["all_locales"]: # This locale either doesn't produce or shouldn't upload this file. continue if ( "only_for_platforms" in map_config["mapping"][filename] and platform not in map_config["mapping"][filename]["only_for_platforms"] ): # This platform either doesn't produce or shouldn't upload this file. continue if ( "not_for_platforms" in map_config["mapping"][filename] and platform in map_config["mapping"][filename]["not_for_platforms"] ): # This platform either doesn't produce or shouldn't upload this file. continue if "partials_only" in map_config["mapping"][filename]: continue # copy_task because the next time we look at this file the locale will differ. file_config = copy_task(map_config["mapping"][filename]) for field in [ "destinations", "locale_prefix", "source_path_modifier", "update_balrog_manifest", "pretty_name", "checksums_path", ]: resolve_keyed_by( file_config, field, job["label"], locale=locale, platform=platform ) # This format string should ideally be in the configuration file, # but this would mean keeping variable names in sync between code + config. destinations = [ "{s3_bucket_path}/{dest_path}/{locale_prefix}{filename}".format( s3_bucket_path=bucket_path, dest_path=dest_path, locale_prefix=file_config["locale_prefix"], filename=file_config.get("pretty_name", filename), ) for dest_path, bucket_path in itertools.product( file_config["destinations"], map_config["s3_bucket_paths"] ) ] # Creating map entries # Key must be artifact path, to avoid trampling duplicates, such # as public/build/target.apk and public/build/en-US/target.apk key = os.path.join( base_artifact_prefix, file_config["source_path_modifier"], filename, ) paths[key] = { "destinations": destinations, } if file_config.get("checksums_path"): paths[key]["checksums_path"] = file_config["checksums_path"] # optional flag: balrog manifest if file_config.get("update_balrog_manifest"): paths[key]["update_balrog_manifest"] = True if file_config.get("balrog_format"): paths[key]["balrog_format"] = file_config["balrog_format"] if not paths: # No files for this dependency/locale combination. continue # Render all variables for the artifact map platforms = copy_task(map_config.get("platform_names", {})) if platform: for key in platforms.keys(): resolve_keyed_by(platforms, key, job["label"], platform=platform) upload_date = datetime.fromtimestamp(config.params["build_date"]) kwargs.update( { "locale": locale, "version": config.params["version"], "branch": config.params["project"], "build_number": config.params["build_number"], "year": upload_date.year, "month": upload_date.strftime("%m"), # zero-pad the month "upload_date": upload_date.strftime("%Y-%m-%d-%H-%M-%S"), } ) kwargs.update(**platforms) paths = jsone.render(paths, kwargs) artifacts.append( { "taskId": {"task-reference": f"<{dep}>"}, "locale": locale, "paths": paths, } ) return artifacts # generate_beetmover_partials_artifact_map {{{1 def generate_beetmover_partials_artifact_map(config, job, partials_info, **kwargs): """Generate the beetmover partials artifact map. Currently only applies to beetmover tasks. Args: config (): Current taskgraph configuration. job (dict): The current job being generated partials_info (dict): Current partials and information about them in a dict Common kwargs: platform (str): The current build platform locale (str): The current locale being beetmoved. Returns: list: A list of dictionaries containing source->destination maps for beetmover. """ platform = kwargs.get("platform", "") resolve_keyed_by( job, "attributes.artifact_map", "artifact map", **{ "release-type": config.params["release_type"], "platform": platform, }, ) map_config = copy_task(cached_load_yaml(job["attributes"]["artifact_map"])) base_artifact_prefix = map_config.get( "base_artifact_prefix", get_artifact_prefix(job) ) artifacts = list() dependencies = job["dependencies"].keys() if kwargs.get("locale"): locales = [kwargs["locale"]] else: locales = map_config["default_locales"] resolve_keyed_by( map_config, "s3_bucket_paths", "s3_bucket_paths", platform=platform ) platforms = copy_task(map_config.get("platform_names", {})) if platform: for key in platforms.keys(): resolve_keyed_by(platforms, key, key, platform=platform) upload_date = datetime.fromtimestamp(config.params["build_date"]) for locale, dep in itertools.product(locales, dependencies): paths = dict() for filename in map_config["mapping"]: # Relevancy checks if dep not in map_config["mapping"][filename]["from"]: # We don't get this file from this dependency. continue if locale != "en-US" and not map_config["mapping"][filename]["all_locales"]: # This locale either doesn't produce or shouldn't upload this file. continue if "partials_only" not in map_config["mapping"][filename]: continue # copy_task because the next time we look at this file the locale will differ. file_config = copy_task(map_config["mapping"][filename]) for field in [ "destinations", "locale_prefix", "source_path_modifier", "update_balrog_manifest", "from_buildid", "pretty_name", "checksums_path", ]: resolve_keyed_by( file_config, field, field, locale=locale, platform=platform ) # This format string should ideally be in the configuration file, # but this would mean keeping variable names in sync between code + config. destinations = [ "{s3_bucket_path}/{dest_path}/{locale_prefix}{filename}".format( s3_bucket_path=bucket_path, dest_path=dest_path, locale_prefix=file_config["locale_prefix"], filename=file_config.get("pretty_name", filename), ) for dest_path, bucket_path in itertools.product( file_config["destinations"], map_config["s3_bucket_paths"] ) ] # Creating map entries # Key must be artifact path, to avoid trampling duplicates, such # as public/build/target.apk and public/build/en-US/target.apk key = os.path.join( base_artifact_prefix, file_config["source_path_modifier"], filename, ) partials_paths = {} for pname, info in partials_info.items(): partials_paths[key] = { "destinations": destinations, } if file_config.get("checksums_path"): partials_paths[key]["checksums_path"] = file_config[ "checksums_path" ] # optional flag: balrog manifest if file_config.get("update_balrog_manifest"): partials_paths[key]["update_balrog_manifest"] = True if file_config.get("balrog_format"): partials_paths[key]["balrog_format"] = file_config[ "balrog_format" ] # optional flag: from_buildid if file_config.get("from_buildid"): partials_paths[key]["from_buildid"] = file_config["from_buildid"] # render buildid kwargs.update( { "partial": pname, "from_buildid": info["buildid"], "previous_version": info.get("previousVersion"), "buildid": str(config.params["moz_build_date"]), "locale": locale, "version": config.params["version"], "branch": config.params["project"], "build_number": config.params["build_number"], "year": upload_date.year, "month": upload_date.strftime("%m"), # zero-pad the month "upload_date": upload_date.strftime("%Y-%m-%d-%H-%M-%S"), } ) kwargs.update(**platforms) paths.update(jsone.render(partials_paths, kwargs)) if not paths: continue artifacts.append( { "taskId": {"task-reference": f"<{dep}>"}, "locale": locale, "paths": paths, } ) artifacts.sort(key=lambda a: sorted(a["paths"].items())) return artifacts