Adding upstream version 115.8.0esr.upstream/115.8.0esr

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 615 insertions, 0 deletions

diff --git a/python/mozbuild/mozbuild/artifact_commands.py b/python/mozbuild/mozbuild/artifact_commands.py
new file mode 100644
index 0000000000..12184ce0d9
--- /dev/null
+++ b/python/mozbuild/mozbuild/artifact_commands.py
@@ -0,0 +1,615 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import argparse
+import hashlib
+import json
+import logging
+import os
+import shutil
+from collections import OrderedDict
+
+import mozversioncontrol
+import six
+from mach.decorators import Command, CommandArgument, SubCommand
+
+from mozbuild.artifact_builds import JOB_CHOICES
+from mozbuild.base import MachCommandConditions as conditions
+from mozbuild.util import ensureParentDir
+
+_COULD_NOT_FIND_ARTIFACTS_TEMPLATE = (
+    "ERROR!!!!!! Could not find artifacts for a toolchain build named "
+    "`{build}`. Local commits, dirty/stale files, and other changes in your "
+    "checkout may cause this error. Make sure you are on a fresh, current "
+    "checkout of mozilla-central. Beware that commands like `mach bootstrap` "
+    "and `mach artifact` are unlikely to work on any versions of the code "
+    "besides recent revisions of mozilla-central."
+)
+
+
+class SymbolsAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        # If this function is called, it means the --symbols option was given,
+        # so we want to store the value `True` if no explicit value was given
+        # to the option.
+        setattr(namespace, self.dest, values or True)
+
+
+class ArtifactSubCommand(SubCommand):
+    def __call__(self, func):
+        after = SubCommand.__call__(self, func)
+        args = [
+            CommandArgument("--tree", metavar="TREE", type=str, help="Firefox tree."),
+            CommandArgument(
+                "--job", metavar="JOB", choices=JOB_CHOICES, help="Build job."
+            ),
+            CommandArgument(
+                "--verbose", "-v", action="store_true", help="Print verbose output."
+            ),
+        ]
+        for arg in args:
+            after = arg(after)
+        return after
+
+
+# Fetch and install binary artifacts from Mozilla automation.
+
+
+@Command(
+    "artifact",
+    category="post-build",
+    description="Use pre-built artifacts to build Firefox.",
+)
+def artifact(command_context):
+    """Download, cache, and install pre-built binary artifacts to build Firefox.
+
+    Use ``mach build`` as normal to freshen your installed binary libraries:
+    artifact builds automatically download, cache, and install binary
+    artifacts from Mozilla automation, replacing whatever may be in your
+    object directory.  Use ``mach artifact last`` to see what binary artifacts
+    were last used.
+
+    Never build libxul again!
+
+    """
+    pass
+
+
+def _make_artifacts(
+    command_context,
+    tree=None,
+    job=None,
+    skip_cache=False,
+    download_tests=True,
+    download_symbols=False,
+    download_maven_zip=False,
+    no_process=False,
+):
+    state_dir = command_context._mach_context.state_dir
+    cache_dir = os.path.join(state_dir, "package-frontend")
+
+    hg = None
+    if conditions.is_hg(command_context):
+        hg = command_context.substs["HG"]
+
+    git = None
+    if conditions.is_git(command_context):
+        git = command_context.substs["GIT"]
+
+    # If we're building Thunderbird, we should be checking for comm-central artifacts.
+    topsrcdir = command_context.substs.get("commtopsrcdir", command_context.topsrcdir)
+
+    if download_maven_zip:
+        if download_tests:
+            raise ValueError("--maven-zip requires --no-tests")
+        if download_symbols:
+            raise ValueError("--maven-zip requires no --symbols")
+        if not no_process:
+            raise ValueError("--maven-zip requires --no-process")
+
+    from mozbuild.artifacts import Artifacts
+
+    artifacts = Artifacts(
+        tree,
+        command_context.substs,
+        command_context.defines,
+        job,
+        log=command_context.log,
+        cache_dir=cache_dir,
+        skip_cache=skip_cache,
+        hg=hg,
+        git=git,
+        topsrcdir=topsrcdir,
+        download_tests=download_tests,
+        download_symbols=download_symbols,
+        download_maven_zip=download_maven_zip,
+        no_process=no_process,
+        mozbuild=command_context,
+    )
+    return artifacts
+
+
+@ArtifactSubCommand("artifact", "install", "Install a good pre-built artifact.")
+@CommandArgument(
+    "source",
+    metavar="SRC",
+    nargs="?",
+    type=str,
+    help="Where to fetch and install artifacts from.  Can be omitted, in "
+    "which case the current hg repository is inspected; an hg revision; "
+    "a remote URL; or a local file.",
+    default=None,
+)
+@CommandArgument(
+    "--skip-cache",
+    action="store_true",
+    help="Skip all local caches to force re-fetching remote artifacts.",
+    default=False,
+)
+@CommandArgument("--no-tests", action="store_true", help="Don't install tests.")
+@CommandArgument("--symbols", nargs="?", action=SymbolsAction, help="Download symbols.")
+@CommandArgument("--distdir", help="Where to install artifacts to.")
+@CommandArgument(
+    "--no-process",
+    action="store_true",
+    help="Don't process (unpack) artifact packages, just download them.",
+)
+@CommandArgument(
+    "--maven-zip", action="store_true", help="Download Maven zip (Android-only)."
+)
+def artifact_install(
+    command_context,
+    source=None,
+    skip_cache=False,
+    tree=None,
+    job=None,
+    verbose=False,
+    no_tests=False,
+    symbols=False,
+    distdir=None,
+    no_process=False,
+    maven_zip=False,
+):
+    command_context._set_log_level(verbose)
+    artifacts = _make_artifacts(
+        command_context,
+        tree=tree,
+        job=job,
+        skip_cache=skip_cache,
+        download_tests=not no_tests,
+        download_symbols=symbols,
+        download_maven_zip=maven_zip,
+        no_process=no_process,
+    )
+
+    return artifacts.install_from(source, distdir or command_context.distdir)
+
+
+@ArtifactSubCommand(
+    "artifact",
+    "clear-cache",
+    "Delete local artifacts and reset local artifact cache.",
+)
+def artifact_clear_cache(command_context, tree=None, job=None, verbose=False):
+    command_context._set_log_level(verbose)
+    artifacts = _make_artifacts(command_context, tree=tree, job=job)
+    artifacts.clear_cache()
+    return 0
+
+
+@SubCommand("artifact", "toolchain")
+@CommandArgument("--verbose", "-v", action="store_true", help="Print verbose output.")
+@CommandArgument(
+    "--cache-dir",
+    metavar="DIR",
+    help="Directory where to store the artifacts cache",
+)
+@CommandArgument(
+    "--skip-cache",
+    action="store_true",
+    help="Skip all local caches to force re-fetching remote artifacts.",
+    default=False,
+)
+@CommandArgument(
+    "--from-build",
+    metavar="BUILD",
+    nargs="+",
+    help="Download toolchains resulting from the given build(s); "
+    "BUILD is a name of a toolchain task, e.g. linux64-clang",
+)
+@CommandArgument(
+    "--from-task",
+    metavar="TASK_ID:ARTIFACT",
+    nargs="+",
+    help="Download toolchain artifact from a given task.",
+)
+@CommandArgument(
+    "--tooltool-manifest",
+    metavar="MANIFEST",
+    help="Explicit tooltool manifest to process",
+)
+@CommandArgument(
+    "--no-unpack", action="store_true", help="Do not unpack any downloaded file"
+)
+@CommandArgument(
+    "--retry", type=int, default=4, help="Number of times to retry failed downloads"
+)
+@CommandArgument(
+    "--bootstrap",
+    action="store_true",
+    help="Whether this is being called from bootstrap. "
+    "This verifies the toolchain is annotated as a toolchain used for local development.",
+)
+@CommandArgument(
+    "--artifact-manifest",
+    metavar="FILE",
+    help="Store a manifest about the downloaded taskcluster artifacts",
+)
+def artifact_toolchain(
+    command_context,
+    verbose=False,
+    cache_dir=None,
+    skip_cache=False,
+    from_build=(),
+    from_task=(),
+    tooltool_manifest=None,
+    no_unpack=False,
+    retry=0,
+    bootstrap=False,
+    artifact_manifest=None,
+):
+    """Download, cache and install pre-built toolchains."""
+    import time
+
+    import redo
+    import requests
+    from taskgraph.util.taskcluster import get_artifact_url
+
+    from mozbuild.action.tooltool import FileRecord, open_manifest, unpack_file
+    from mozbuild.artifacts import ArtifactCache
+
+    start = time.monotonic()
+    command_context._set_log_level(verbose)
+    # Normally, we'd use command_context.log_manager.enable_unstructured(),
+    # but that enables all logging, while we only really want tooltool's
+    # and it also makes structured log output twice.
+    # So we manually do what it does, and limit that to the tooltool
+    # logger.
+    if command_context.log_manager.terminal_handler:
+        logging.getLogger("mozbuild.action.tooltool").addHandler(
+            command_context.log_manager.terminal_handler
+        )
+        logging.getLogger("redo").addHandler(
+            command_context.log_manager.terminal_handler
+        )
+        command_context.log_manager.terminal_handler.addFilter(
+            command_context.log_manager.structured_filter
+        )
+    if not cache_dir:
+        cache_dir = os.path.join(command_context._mach_context.state_dir, "toolchains")
+
+    tooltool_host = os.environ.get("TOOLTOOL_HOST", "tooltool.mozilla-releng.net")
+    taskcluster_proxy_url = os.environ.get("TASKCLUSTER_PROXY_URL")
+    if taskcluster_proxy_url:
+        tooltool_url = "{}/{}".format(taskcluster_proxy_url, tooltool_host)
+    else:
+        tooltool_url = "https://{}".format(tooltool_host)
+
+    cache = ArtifactCache(
+        cache_dir=cache_dir, log=command_context.log, skip_cache=skip_cache
+    )
+
+    class DownloadRecord(FileRecord):
+        def __init__(self, url, *args, **kwargs):
+            super(DownloadRecord, self).__init__(*args, **kwargs)
+            self.url = url
+            self.basename = self.filename
+
+        def fetch_with(self, cache):
+            self.filename = cache.fetch(self.url)
+            return self.filename
+
+        def validate(self):
+            if self.size is None and self.digest is None:
+                return True
+            return super(DownloadRecord, self).validate()
+
+    class ArtifactRecord(DownloadRecord):
+        def __init__(self, task_id, artifact_name):
+            for _ in redo.retrier(attempts=retry + 1, sleeptime=60):
+                cot = cache._download_manager.session.get(
+                    get_artifact_url(task_id, "public/chain-of-trust.json")
+                )
+                if cot.status_code >= 500:
+                    continue
+                cot.raise_for_status()
+                break
+            else:
+                cot.raise_for_status()
+
+            digest = algorithm = None
+            data = json.loads(cot.text)
+            for algorithm, digest in (
+                data.get("artifacts", {}).get(artifact_name, {}).items()
+            ):
+                pass
+
+            name = os.path.basename(artifact_name)
+            artifact_url = get_artifact_url(
+                task_id,
+                artifact_name,
+                use_proxy=not artifact_name.startswith("public/"),
+            )
+            super(ArtifactRecord, self).__init__(
+                artifact_url, name, None, digest, algorithm, unpack=True
+            )
+
+    records = OrderedDict()
+    downloaded = []
+
+    if tooltool_manifest:
+        manifest = open_manifest(tooltool_manifest)
+        for record in manifest.file_records:
+            url = "{}/{}/{}".format(tooltool_url, record.algorithm, record.digest)
+            records[record.filename] = DownloadRecord(
+                url,
+                record.filename,
+                record.size,
+                record.digest,
+                record.algorithm,
+                unpack=record.unpack,
+                version=record.version,
+                visibility=record.visibility,
+            )
+
+    if from_build:
+        if "MOZ_AUTOMATION" in os.environ:
+            command_context.log(
+                logging.ERROR,
+                "artifact",
+                {},
+                "Do not use --from-build in automation; all dependencies "
+                "should be determined in the decision task.",
+            )
+            return 1
+        from gecko_taskgraph.optimize.strategies import IndexSearch
+
+        from mozbuild.toolchains import toolchain_task_definitions
+
+        tasks = toolchain_task_definitions()
+
+        for b in from_build:
+            user_value = b
+
+            if not b.startswith("toolchain-"):
+                b = "toolchain-{}".format(b)
+
+            task = tasks.get(b)
+            if not task:
+                command_context.log(
+                    logging.ERROR,
+                    "artifact",
+                    {"build": user_value},
+                    "Could not find a toolchain build named `{build}`",
+                )
+                return 1
+
+            # Ensure that toolchains installed by `mach bootstrap` have the
+            # `local-toolchain attribute set. Taskgraph ensures that these
+            # are built on trunk projects, so the task will be available to
+            # install here.
+            if bootstrap and not task.attributes.get("local-toolchain"):
+                command_context.log(
+                    logging.ERROR,
+                    "artifact",
+                    {"build": user_value},
+                    "Toolchain `{build}` is not annotated as used for local development.",
+                )
+                return 1
+
+            artifact_name = task.attributes.get("toolchain-artifact")
+            command_context.log(
+                logging.DEBUG,
+                "artifact",
+                {
+                    "name": artifact_name,
+                    "index": task.optimization.get("index-search"),
+                },
+                "Searching for {name} in {index}",
+            )
+            deadline = None
+            task_id = IndexSearch().should_replace_task(
+                task, {}, deadline, task.optimization.get("index-search", [])
+            )
+            if task_id in (True, False) or not artifact_name:
+                command_context.log(
+                    logging.ERROR,
+                    "artifact",
+                    {"build": user_value},
+                    _COULD_NOT_FIND_ARTIFACTS_TEMPLATE,
+                )
+                # Get and print some helpful info for diagnosis.
+                repo = mozversioncontrol.get_repository_object(
+                    command_context.topsrcdir
+                )
+                if not isinstance(repo, mozversioncontrol.SrcRepository):
+                    changed_files = set(repo.get_outgoing_files()) | set(
+                        repo.get_changed_files()
+                    )
+                    if changed_files:
+                        command_context.log(
+                            logging.ERROR,
+                            "artifact",
+                            {},
+                            "Hint: consider reverting your local changes "
+                            "to the following files: %s" % sorted(changed_files),
+                        )
+                if "TASKCLUSTER_ROOT_URL" in os.environ:
+                    command_context.log(
+                        logging.ERROR,
+                        "artifact",
+                        {"build": user_value},
+                        "Due to the environment variable TASKCLUSTER_ROOT_URL "
+                        "being set, the artifacts were expected to be found "
+                        "on {}. If this was unintended, unset "
+                        "TASKCLUSTER_ROOT_URL and try again.".format(
+                            os.environ["TASKCLUSTER_ROOT_URL"]
+                        ),
+                    )
+                return 1
+
+            command_context.log(
+                logging.DEBUG,
+                "artifact",
+                {"name": artifact_name, "task_id": task_id},
+                "Found {name} in {task_id}",
+            )
+
+            record = ArtifactRecord(task_id, artifact_name)
+            records[record.filename] = record
+
+    # Handle the list of files of the form task_id:path from --from-task.
+    for f in from_task or ():
+        task_id, colon, name = f.partition(":")
+        if not colon:
+            command_context.log(
+                logging.ERROR,
+                "artifact",
+                {},
+                "Expected an argument of the form task_id:path",
+            )
+            return 1
+        record = ArtifactRecord(task_id, name)
+        records[record.filename] = record
+
+    for record in six.itervalues(records):
+        command_context.log(
+            logging.INFO,
+            "artifact",
+            {"name": record.basename},
+            "Setting up artifact {name}",
+        )
+        valid = False
+        # sleeptime is 60 per retry.py, used by tooltool_wrapper.sh
+        for attempt, _ in enumerate(redo.retrier(attempts=retry + 1, sleeptime=60)):
+            try:
+                record.fetch_with(cache)
+            except (
+                requests.exceptions.HTTPError,
+                requests.exceptions.ChunkedEncodingError,
+                requests.exceptions.ConnectionError,
+            ) as e:
+
+                if isinstance(e, requests.exceptions.HTTPError):
+                    # The relengapi proxy likes to return error 400 bad request
+                    # which seems improbably to be due to our (simple) GET
+                    # being borked.
+                    status = e.response.status_code
+                    should_retry = status >= 500 or status == 400
+                else:
+                    should_retry = True
+
+                if should_retry or attempt < retry:
+                    level = logging.WARN
+                else:
+                    level = logging.ERROR
+                command_context.log(level, "artifact", {}, str(e))
+                if not should_retry:
+                    break
+                if attempt < retry:
+                    command_context.log(
+                        logging.INFO, "artifact", {}, "Will retry in a moment..."
+                    )
+                continue
+            try:
+                valid = record.validate()
+            except Exception:
+                pass
+            if not valid:
+                os.unlink(record.filename)
+                if attempt < retry:
+                    command_context.log(
+                        logging.INFO,
+                        "artifact",
+                        {},
+                        "Corrupt download. Will retry in a moment...",
+                    )
+                continue
+
+            downloaded.append(record)
+            break
+
+        if not valid:
+            command_context.log(
+                logging.ERROR,
+                "artifact",
+                {"name": record.basename},
+                "Failed to download {name}",
+            )
+            return 1
+
+    artifacts = {} if artifact_manifest else None
+
+    for record in downloaded:
+        local = os.path.join(os.getcwd(), record.basename)
+        if os.path.exists(local):
+            os.unlink(local)
+        # unpack_file needs the file with its final name to work
+        # (https://github.com/mozilla/build-tooltool/issues/38), so we
+        # need to copy it, even though we remove it later. Use hard links
+        # when possible.
+        try:
+            os.link(record.filename, local)
+        except Exception:
+            shutil.copy(record.filename, local)
+        # Keep a sha256 of each downloaded file, for the chain-of-trust
+        # validation.
+        if artifact_manifest is not None:
+            with open(local, "rb") as fh:
+                h = hashlib.sha256()
+                while True:
+                    data = fh.read(1024 * 1024)
+                    if not data:
+                        break
+                    h.update(data)
+            artifacts[record.url] = {"sha256": h.hexdigest()}
+        if record.unpack and not no_unpack:
+            unpack_file(local)
+            os.unlink(local)
+
+    if not downloaded:
+        command_context.log(logging.ERROR, "artifact", {}, "Nothing to download")
+        if from_task:
+            return 1
+
+    if artifacts:
+        ensureParentDir(artifact_manifest)
+        with open(artifact_manifest, "w") as fh:
+            json.dump(artifacts, fh, indent=4, sort_keys=True)
+
+    if "MOZ_AUTOMATION" in os.environ:
+        end = time.monotonic()
+
+        perfherder_data = {
+            "framework": {"name": "build_metrics"},
+            "suites": [
+                {
+                    "name": "mach_artifact_toolchain",
+                    "value": end - start,
+                    "lowerIsBetter": True,
+                    "shouldAlert": False,
+                    "subtests": [],
+                }
+            ],
+        }
+        command_context.log(
+            logging.INFO,
+            "perfherder",
+            {"data": json.dumps(perfherder_data)},
+            "PERFHERDER_DATA: {data}",
+        )
+
+    return 0