Adding upstream version 86.0.1.upstream/86.0.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 424 insertions, 0 deletions

diff --git a/taskcluster/taskgraph/transforms/l10n.py b/taskcluster/taskgraph/transforms/l10n.py
new file mode 100644
index 0000000000..ffa8319b4d
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -0,0 +1,424 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Do transforms specific to l10n kind
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import copy
+import io
+import json
+import six
+
+from mozbuild.chunkify import chunkify
+from six import text_type
+from taskgraph.loader.multi_dep import schema
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import (
+    optionally_keyed_by,
+    resolve_keyed_by,
+    taskref_or_string,
+)
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
+from taskgraph.util.taskcluster import get_artifact_prefix
+from taskgraph.util.treeherder import add_suffix
+from taskgraph.transforms.job import job_description_schema
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import (
+    Any,
+    Optional,
+    Required,
+)
+
+
+def _by_platform(arg):
+    return optionally_keyed_by("build-platform", arg)
+
+
+l10n_description_schema = schema.extend(
+    {
+        # Name for this job, inferred from the dependent job before validation
+        Required("name"): text_type,
+        # build-platform, inferred from dependent job before validation
+        Required("build-platform"): text_type,
+        # max run time of the task
+        Required("run-time"): _by_platform(int),
+        # Locales not to repack for
+        Required("ignore-locales"): _by_platform([text_type]),
+        # All l10n jobs use mozharness
+        Required("mozharness"): {
+            # Script to invoke for mozharness
+            Required("script"): _by_platform(text_type),
+            # Config files passed to the mozharness script
+            Required("config"): _by_platform([text_type]),
+            # Additional paths to look for mozharness configs in. These should be
+            # relative to the base of the source checkout
+            Optional("config-paths"): [text_type],
+            # Options to pass to the mozharness script
+            Optional("options"): _by_platform([text_type]),
+            # Action commands to provide to mozharness script
+            Required("actions"): _by_platform([text_type]),
+            # if true, perform a checkout of a comm-central based branch inside the
+            # gecko checkout
+            Optional("comm-checkout"): bool,
+        },
+        # Items for the taskcluster index
+        Optional("index"): {
+            # Product to identify as in the taskcluster index
+            Required("product"): _by_platform(text_type),
+            # Job name to identify as in the taskcluster index
+            Required("job-name"): _by_platform(text_type),
+            # Type of index
+            Optional("type"): _by_platform(text_type),
+        },
+        # Description of the localized task
+        Required("description"): _by_platform(text_type),
+        Optional("run-on-projects"): job_description_schema["run-on-projects"],
+        # worker-type to utilize
+        Required("worker-type"): _by_platform(text_type),
+        # File which contains the used locales
+        Required("locales-file"): _by_platform(text_type),
+        # Tooltool visibility required for task.
+        Required("tooltool"): _by_platform(Any("internal", "public")),
+        # Docker image required for task.  We accept only in-tree images
+        # -- generally desktop-build or android-build -- for now.
+        Optional("docker-image"): _by_platform(
+            # an in-tree generated docker image (from `taskcluster/docker/<name>`)
+            {"in-tree": text_type},
+        ),
+        Optional("fetches"): {
+            text_type: _by_platform([text_type]),
+        },
+        # The set of secret names to which the task has access; these are prefixed
+        # with `project/releng/gecko/{treeherder.kind}/level-{level}/`.  Setting
+        # this will enable any worker features required and set the task's scopes
+        # appropriately.  `true` here means ['*'], all secrets.  Not supported on
+        # Windows
+        Optional("secrets"): _by_platform(Any(bool, [text_type])),
+        # Information for treeherder
+        Required("treeherder"): {
+            # Platform to display the task on in treeherder
+            Required("platform"): _by_platform(text_type),
+            # Symbol to use
+            Required("symbol"): text_type,
+            # Tier this task is
+            Required("tier"): _by_platform(int),
+        },
+        # Extra environment values to pass to the worker
+        Optional("env"): _by_platform({text_type: taskref_or_string}),
+        # Max number locales per chunk
+        Optional("locales-per-chunk"): _by_platform(int),
+        # Task deps to chain this task with, added in transforms from primary-dependency
+        # if this is a shippable-style build
+        Optional("dependencies"): {text_type: text_type},
+        # Run the task when the listed files change (if present).
+        Optional("when"): {"files-changed": [text_type]},
+        # passed through directly to the job description
+        Optional("attributes"): job_description_schema["attributes"],
+        Optional("extra"): job_description_schema["extra"],
+        # Shipping product and phase
+        Optional("shipping-product"): task_description_schema["shipping-product"],
+        Optional("shipping-phase"): task_description_schema["shipping-phase"],
+    }
+)
+
+transforms = TransformSequence()
+
+
+def parse_locales_file(locales_file, platform=None):
+    """Parse the passed locales file for a list of locales."""
+    locales = []
+
+    with io.open(locales_file, mode="r") as f:
+        if locales_file.endswith("json"):
+            all_locales = json.load(f)
+            # XXX Only single locales are fetched
+            locales = {
+                locale: data["revision"]
+                for locale, data in all_locales.items()
+                if platform is None or platform in data["platforms"]
+            }
+        else:
+            all_locales = f.read().split()
+            # 'default' is the hg revision at the top of hg repo, in this context
+            locales = {locale: "default" for locale in all_locales}
+    return locales
+
+
+def _remove_locales(locales, to_remove=None):
+    # ja-JP-mac is a mac-only locale, but there are no mac builds being repacked,
+    # so just omit it unconditionally
+    return {
+        locale: revision
+        for locale, revision in locales.items()
+        if locale not in to_remove
+    }
+
+
+@transforms.add
+def setup_name(config, jobs):
+    for job in jobs:
+        dep = job["primary-dependency"]
+        # Set the name to the same as the dep task, without kind name.
+        # Label will get set automatically with this kinds name.
+        job["name"] = job.get("name", dep.name)
+        yield job
+
+
+@transforms.add
+def copy_in_useful_magic(config, jobs):
+    for job in jobs:
+        dep = job["primary-dependency"]
+        attributes = copy_attributes_from_dependent_job(dep)
+        attributes.update(job.get("attributes", {}))
+        # build-platform is needed on `job` for by-build-platform
+        job["build-platform"] = attributes.get("build_platform")
+        job["attributes"] = attributes
+        yield job
+
+
+transforms.add_validate(l10n_description_schema)
+
+
+@transforms.add
+def setup_shippable_dependency(config, jobs):
+    """ Sets up a task dependency to the signing job this relates to """
+    for job in jobs:
+        job["dependencies"] = {"build": job["dependent-tasks"]["build"].label}
+        if job["attributes"]["build_platform"].startswith("win") or job["attributes"][
+            "build_platform"
+        ].startswith("linux"):
+            job["dependencies"].update(
+                {
+                    "build-signing": job["dependent-tasks"]["build-signing"].label,
+                }
+            )
+        if job["attributes"]["build_platform"].startswith("macosx"):
+            job["dependencies"].update(
+                {"repackage": job["dependent-tasks"]["repackage"].label}
+            )
+        yield job
+
+
+@transforms.add
+def handle_keyed_by(config, jobs):
+    """Resolve fields that can be keyed by platform, etc."""
+    fields = [
+        "locales-file",
+        "locales-per-chunk",
+        "worker-type",
+        "description",
+        "run-time",
+        "docker-image",
+        "secrets",
+        "fetches.toolchain",
+        "fetches.fetch",
+        "tooltool",
+        "env",
+        "ignore-locales",
+        "mozharness.config",
+        "mozharness.options",
+        "mozharness.actions",
+        "mozharness.script",
+        "treeherder.tier",
+        "treeherder.platform",
+        "index.type",
+        "index.product",
+        "index.job-name",
+        "when.files-changed",
+    ]
+    for job in jobs:
+        job = copy.deepcopy(job)  # don't overwrite dict values here
+        for field in fields:
+            resolve_keyed_by(item=job, field=field, item_name=job["name"])
+        yield job
+
+
+@transforms.add
+def handle_artifact_prefix(config, jobs):
+    """Resolve ``artifact_prefix`` in env vars"""
+    for job in jobs:
+        artifact_prefix = get_artifact_prefix(job)
+        for k1, v1 in six.iteritems(job.get("env", {})):
+            if isinstance(v1, text_type):
+                job["env"][k1] = v1.format(artifact_prefix=artifact_prefix)
+            elif isinstance(v1, dict):
+                for k2, v2 in six.iteritems(v1):
+                    job["env"][k1][k2] = v2.format(artifact_prefix=artifact_prefix)
+        yield job
+
+
+@transforms.add
+def all_locales_attribute(config, jobs):
+    for job in jobs:
+        locales_platform = job["attributes"]["build_platform"].replace("-shippable", "")
+        locales_platform = locales_platform.replace("-pgo", "")
+        locales_with_changesets = parse_locales_file(
+            job["locales-file"], platform=locales_platform
+        )
+        locales_with_changesets = _remove_locales(
+            locales_with_changesets, to_remove=job["ignore-locales"]
+        )
+
+        locales = sorted(locales_with_changesets.keys())
+        attributes = job.setdefault("attributes", {})
+        attributes["all_locales"] = locales
+        attributes["all_locales_with_changesets"] = locales_with_changesets
+        if job.get("shipping-product"):
+            attributes["shipping_product"] = job["shipping-product"]
+        yield job
+
+
+@transforms.add
+def chunk_locales(config, jobs):
+    """ Utilizes chunking for l10n stuff """
+    for job in jobs:
+        locales_per_chunk = job.get("locales-per-chunk")
+        locales_with_changesets = job["attributes"]["all_locales_with_changesets"]
+        if locales_per_chunk:
+            chunks, remainder = divmod(len(locales_with_changesets), locales_per_chunk)
+            if remainder:
+                chunks = int(chunks + 1)
+            for this_chunk in range(1, chunks + 1):
+                chunked = copy.deepcopy(job)
+                chunked["name"] = chunked["name"].replace(
+                    "/", "-{}/".format(this_chunk), 1
+                )
+                chunked["mozharness"]["options"] = chunked["mozharness"].get(
+                    "options", []
+                )
+                # chunkify doesn't work with dicts
+                locales_with_changesets_as_list = sorted(
+                    locales_with_changesets.items()
+                )
+                chunked_locales = chunkify(
+                    locales_with_changesets_as_list, this_chunk, chunks
+                )
+                chunked["mozharness"]["options"].extend(
+                    [
+                        "locale={}:{}".format(locale, changeset)
+                        for locale, changeset in chunked_locales
+                    ]
+                )
+                chunked["attributes"]["l10n_chunk"] = str(this_chunk)
+                # strip revision
+                chunked["attributes"]["chunk_locales"] = [
+                    locale for locale, _ in chunked_locales
+                ]
+
+                # add the chunk number to the TH symbol
+                chunked["treeherder"]["symbol"] = add_suffix(
+                    chunked["treeherder"]["symbol"], this_chunk
+                )
+                yield chunked
+        else:
+            job["mozharness"]["options"] = job["mozharness"].get("options", [])
+            job["mozharness"]["options"].extend(
+                [
+                    "locale={}:{}".format(locale, changeset)
+                    for locale, changeset in sorted(locales_with_changesets.items())
+                ]
+            )
+            yield job
+
+
+transforms.add_validate(l10n_description_schema)
+
+
+@transforms.add
+def stub_installer(config, jobs):
+    for job in jobs:
+        job.setdefault("attributes", {})
+        job.setdefault("env", {})
+        if job["attributes"].get("stub-installer"):
+            job["env"].update({"USE_STUB_INSTALLER": "1"})
+        yield job
+
+
+@transforms.add
+def set_extra_config(config, jobs):
+    for job in jobs:
+        job["mozharness"].setdefault("extra-config", {})["branch"] = config.params[
+            "project"
+        ]
+        if "update-channel" in job["attributes"]:
+            job["mozharness"]["extra-config"]["update_channel"] = job["attributes"][
+                "update-channel"
+            ]
+        yield job
+
+
+@transforms.add
+def make_job_description(config, jobs):
+    for job in jobs:
+        job["mozharness"].update(
+            {
+                "using": "mozharness",
+                "job-script": "taskcluster/scripts/builder/build-l10n.sh",
+                "secrets": job.get("secrets", False),
+            }
+        )
+        job_description = {
+            "name": job["name"],
+            "worker-type": job["worker-type"],
+            "description": job["description"],
+            "run": job["mozharness"],
+            "attributes": job["attributes"],
+            "treeherder": {
+                "kind": "build",
+                "tier": job["treeherder"]["tier"],
+                "symbol": job["treeherder"]["symbol"],
+                "platform": job["treeherder"]["platform"],
+            },
+            "run-on-projects": job.get("run-on-projects")
+            if job.get("run-on-projects")
+            else [],
+        }
+        if job.get("extra"):
+            job_description["extra"] = job["extra"]
+
+        job_description["run"]["tooltool-downloads"] = job["tooltool"]
+
+        job_description["worker"] = {
+            "max-run-time": job["run-time"],
+            "chain-of-trust": True,
+        }
+        if job["worker-type"] == "b-win2012":
+            job_description["worker"]["os"] = "windows"
+            job_description["run"]["use-simple-package"] = False
+            job_description["run"]["use-magic-mh-args"] = False
+        else:
+            job_description["run"]["need-xvfb"] = True
+
+        if job.get("docker-image"):
+            job_description["worker"]["docker-image"] = job["docker-image"]
+
+        if job.get("fetches"):
+            job_description["fetches"] = job["fetches"]
+
+        if job.get("index"):
+            job_description["index"] = {
+                "product": job["index"]["product"],
+                "job-name": job["index"]["job-name"],
+                "type": job["index"].get("type", "generic"),
+            }
+
+        if job.get("dependencies"):
+            job_description["dependencies"] = job["dependencies"]
+        if job.get("env"):
+            job_description["worker"]["env"] = job["env"]
+        if job.get("when", {}).get("files-changed"):
+            job_description.setdefault("when", {})
+            job_description["when"]["files-changed"] = [job["locales-file"]] + job[
+                "when"
+            ]["files-changed"]
+
+        if "shipping-phase" in job:
+            job_description["shipping-phase"] = job["shipping-phase"]
+
+        if "shipping-product" in job:
+            job_description["shipping-product"] = job["shipping-product"]
+
+        yield job_description