diff options
Diffstat (limited to 'python/mozbuild/mozbuild/artifact_commands.py')
| -rw-r--r-- | python/mozbuild/mozbuild/artifact_commands.py | 615 |
1 files changed, 615 insertions, 0 deletions
diff --git a/python/mozbuild/mozbuild/artifact_commands.py b/python/mozbuild/mozbuild/artifact_commands.py new file mode 100644 index 0000000000..12184ce0d9 --- /dev/null +++ b/python/mozbuild/mozbuild/artifact_commands.py @@ -0,0 +1,615 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import argparse +import hashlib +import json +import logging +import os +import shutil +from collections import OrderedDict + +import mozversioncontrol +import six +from mach.decorators import Command, CommandArgument, SubCommand + +from mozbuild.artifact_builds import JOB_CHOICES +from mozbuild.base import MachCommandConditions as conditions +from mozbuild.util import ensureParentDir + +_COULD_NOT_FIND_ARTIFACTS_TEMPLATE = ( + "ERROR!!!!!! Could not find artifacts for a toolchain build named " + "`{build}`. Local commits, dirty/stale files, and other changes in your " + "checkout may cause this error. Make sure you are on a fresh, current " + "checkout of mozilla-central. Beware that commands like `mach bootstrap` " + "and `mach artifact` are unlikely to work on any versions of the code " + "besides recent revisions of mozilla-central." +) + + +class SymbolsAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + # If this function is called, it means the --symbols option was given, + # so we want to store the value `True` if no explicit value was given + # to the option. + setattr(namespace, self.dest, values or True) + + +class ArtifactSubCommand(SubCommand): + def __call__(self, func): + after = SubCommand.__call__(self, func) + args = [ + CommandArgument("--tree", metavar="TREE", type=str, help="Firefox tree."), + CommandArgument( + "--job", metavar="JOB", choices=JOB_CHOICES, help="Build job." + ), + CommandArgument( + "--verbose", "-v", action="store_true", help="Print verbose output." + ), + ] + for arg in args: + after = arg(after) + return after + + +# Fetch and install binary artifacts from Mozilla automation. + + +@Command( + "artifact", + category="post-build", + description="Use pre-built artifacts to build Firefox.", +) +def artifact(command_context): + """Download, cache, and install pre-built binary artifacts to build Firefox. + + Use ``mach build`` as normal to freshen your installed binary libraries: + artifact builds automatically download, cache, and install binary + artifacts from Mozilla automation, replacing whatever may be in your + object directory. Use ``mach artifact last`` to see what binary artifacts + were last used. + + Never build libxul again! + + """ + pass + + +def _make_artifacts( + command_context, + tree=None, + job=None, + skip_cache=False, + download_tests=True, + download_symbols=False, + download_maven_zip=False, + no_process=False, +): + state_dir = command_context._mach_context.state_dir + cache_dir = os.path.join(state_dir, "package-frontend") + + hg = None + if conditions.is_hg(command_context): + hg = command_context.substs["HG"] + + git = None + if conditions.is_git(command_context): + git = command_context.substs["GIT"] + + # If we're building Thunderbird, we should be checking for comm-central artifacts. + topsrcdir = command_context.substs.get("commtopsrcdir", command_context.topsrcdir) + + if download_maven_zip: + if download_tests: + raise ValueError("--maven-zip requires --no-tests") + if download_symbols: + raise ValueError("--maven-zip requires no --symbols") + if not no_process: + raise ValueError("--maven-zip requires --no-process") + + from mozbuild.artifacts import Artifacts + + artifacts = Artifacts( + tree, + command_context.substs, + command_context.defines, + job, + log=command_context.log, + cache_dir=cache_dir, + skip_cache=skip_cache, + hg=hg, + git=git, + topsrcdir=topsrcdir, + download_tests=download_tests, + download_symbols=download_symbols, + download_maven_zip=download_maven_zip, + no_process=no_process, + mozbuild=command_context, + ) + return artifacts + + +@ArtifactSubCommand("artifact", "install", "Install a good pre-built artifact.") +@CommandArgument( + "source", + metavar="SRC", + nargs="?", + type=str, + help="Where to fetch and install artifacts from. Can be omitted, in " + "which case the current hg repository is inspected; an hg revision; " + "a remote URL; or a local file.", + default=None, +) +@CommandArgument( + "--skip-cache", + action="store_true", + help="Skip all local caches to force re-fetching remote artifacts.", + default=False, +) +@CommandArgument("--no-tests", action="store_true", help="Don't install tests.") +@CommandArgument("--symbols", nargs="?", action=SymbolsAction, help="Download symbols.") +@CommandArgument("--distdir", help="Where to install artifacts to.") +@CommandArgument( + "--no-process", + action="store_true", + help="Don't process (unpack) artifact packages, just download them.", +) +@CommandArgument( + "--maven-zip", action="store_true", help="Download Maven zip (Android-only)." +) +def artifact_install( + command_context, + source=None, + skip_cache=False, + tree=None, + job=None, + verbose=False, + no_tests=False, + symbols=False, + distdir=None, + no_process=False, + maven_zip=False, +): + command_context._set_log_level(verbose) + artifacts = _make_artifacts( + command_context, + tree=tree, + job=job, + skip_cache=skip_cache, + download_tests=not no_tests, + download_symbols=symbols, + download_maven_zip=maven_zip, + no_process=no_process, + ) + + return artifacts.install_from(source, distdir or command_context.distdir) + + +@ArtifactSubCommand( + "artifact", + "clear-cache", + "Delete local artifacts and reset local artifact cache.", +) +def artifact_clear_cache(command_context, tree=None, job=None, verbose=False): + command_context._set_log_level(verbose) + artifacts = _make_artifacts(command_context, tree=tree, job=job) + artifacts.clear_cache() + return 0 + + +@SubCommand("artifact", "toolchain") +@CommandArgument("--verbose", "-v", action="store_true", help="Print verbose output.") +@CommandArgument( + "--cache-dir", + metavar="DIR", + help="Directory where to store the artifacts cache", +) +@CommandArgument( + "--skip-cache", + action="store_true", + help="Skip all local caches to force re-fetching remote artifacts.", + default=False, +) +@CommandArgument( + "--from-build", + metavar="BUILD", + nargs="+", + help="Download toolchains resulting from the given build(s); " + "BUILD is a name of a toolchain task, e.g. linux64-clang", +) +@CommandArgument( + "--from-task", + metavar="TASK_ID:ARTIFACT", + nargs="+", + help="Download toolchain artifact from a given task.", +) +@CommandArgument( + "--tooltool-manifest", + metavar="MANIFEST", + help="Explicit tooltool manifest to process", +) +@CommandArgument( + "--no-unpack", action="store_true", help="Do not unpack any downloaded file" +) +@CommandArgument( + "--retry", type=int, default=4, help="Number of times to retry failed downloads" +) +@CommandArgument( + "--bootstrap", + action="store_true", + help="Whether this is being called from bootstrap. " + "This verifies the toolchain is annotated as a toolchain used for local development.", +) +@CommandArgument( + "--artifact-manifest", + metavar="FILE", + help="Store a manifest about the downloaded taskcluster artifacts", +) +def artifact_toolchain( + command_context, + verbose=False, + cache_dir=None, + skip_cache=False, + from_build=(), + from_task=(), + tooltool_manifest=None, + no_unpack=False, + retry=0, + bootstrap=False, + artifact_manifest=None, +): + """Download, cache and install pre-built toolchains.""" + import time + + import redo + import requests + from taskgraph.util.taskcluster import get_artifact_url + + from mozbuild.action.tooltool import FileRecord, open_manifest, unpack_file + from mozbuild.artifacts import ArtifactCache + + start = time.monotonic() + command_context._set_log_level(verbose) + # Normally, we'd use command_context.log_manager.enable_unstructured(), + # but that enables all logging, while we only really want tooltool's + # and it also makes structured log output twice. + # So we manually do what it does, and limit that to the tooltool + # logger. + if command_context.log_manager.terminal_handler: + logging.getLogger("mozbuild.action.tooltool").addHandler( + command_context.log_manager.terminal_handler + ) + logging.getLogger("redo").addHandler( + command_context.log_manager.terminal_handler + ) + command_context.log_manager.terminal_handler.addFilter( + command_context.log_manager.structured_filter + ) + if not cache_dir: + cache_dir = os.path.join(command_context._mach_context.state_dir, "toolchains") + + tooltool_host = os.environ.get("TOOLTOOL_HOST", "tooltool.mozilla-releng.net") + taskcluster_proxy_url = os.environ.get("TASKCLUSTER_PROXY_URL") + if taskcluster_proxy_url: + tooltool_url = "{}/{}".format(taskcluster_proxy_url, tooltool_host) + else: + tooltool_url = "https://{}".format(tooltool_host) + + cache = ArtifactCache( + cache_dir=cache_dir, log=command_context.log, skip_cache=skip_cache + ) + + class DownloadRecord(FileRecord): + def __init__(self, url, *args, **kwargs): + super(DownloadRecord, self).__init__(*args, **kwargs) + self.url = url + self.basename = self.filename + + def fetch_with(self, cache): + self.filename = cache.fetch(self.url) + return self.filename + + def validate(self): + if self.size is None and self.digest is None: + return True + return super(DownloadRecord, self).validate() + + class ArtifactRecord(DownloadRecord): + def __init__(self, task_id, artifact_name): + for _ in redo.retrier(attempts=retry + 1, sleeptime=60): + cot = cache._download_manager.session.get( + get_artifact_url(task_id, "public/chain-of-trust.json") + ) + if cot.status_code >= 500: + continue + cot.raise_for_status() + break + else: + cot.raise_for_status() + + digest = algorithm = None + data = json.loads(cot.text) + for algorithm, digest in ( + data.get("artifacts", {}).get(artifact_name, {}).items() + ): + pass + + name = os.path.basename(artifact_name) + artifact_url = get_artifact_url( + task_id, + artifact_name, + use_proxy=not artifact_name.startswith("public/"), + ) + super(ArtifactRecord, self).__init__( + artifact_url, name, None, digest, algorithm, unpack=True + ) + + records = OrderedDict() + downloaded = [] + + if tooltool_manifest: + manifest = open_manifest(tooltool_manifest) + for record in manifest.file_records: + url = "{}/{}/{}".format(tooltool_url, record.algorithm, record.digest) + records[record.filename] = DownloadRecord( + url, + record.filename, + record.size, + record.digest, + record.algorithm, + unpack=record.unpack, + version=record.version, + visibility=record.visibility, + ) + + if from_build: + if "MOZ_AUTOMATION" in os.environ: + command_context.log( + logging.ERROR, + "artifact", + {}, + "Do not use --from-build in automation; all dependencies " + "should be determined in the decision task.", + ) + return 1 + from gecko_taskgraph.optimize.strategies import IndexSearch + + from mozbuild.toolchains import toolchain_task_definitions + + tasks = toolchain_task_definitions() + + for b in from_build: + user_value = b + + if not b.startswith("toolchain-"): + b = "toolchain-{}".format(b) + + task = tasks.get(b) + if not task: + command_context.log( + logging.ERROR, + "artifact", + {"build": user_value}, + "Could not find a toolchain build named `{build}`", + ) + return 1 + + # Ensure that toolchains installed by `mach bootstrap` have the + # `local-toolchain attribute set. Taskgraph ensures that these + # are built on trunk projects, so the task will be available to + # install here. + if bootstrap and not task.attributes.get("local-toolchain"): + command_context.log( + logging.ERROR, + "artifact", + {"build": user_value}, + "Toolchain `{build}` is not annotated as used for local development.", + ) + return 1 + + artifact_name = task.attributes.get("toolchain-artifact") + command_context.log( + logging.DEBUG, + "artifact", + { + "name": artifact_name, + "index": task.optimization.get("index-search"), + }, + "Searching for {name} in {index}", + ) + deadline = None + task_id = IndexSearch().should_replace_task( + task, {}, deadline, task.optimization.get("index-search", []) + ) + if task_id in (True, False) or not artifact_name: + command_context.log( + logging.ERROR, + "artifact", + {"build": user_value}, + _COULD_NOT_FIND_ARTIFACTS_TEMPLATE, + ) + # Get and print some helpful info for diagnosis. + repo = mozversioncontrol.get_repository_object( + command_context.topsrcdir + ) + if not isinstance(repo, mozversioncontrol.SrcRepository): + changed_files = set(repo.get_outgoing_files()) | set( + repo.get_changed_files() + ) + if changed_files: + command_context.log( + logging.ERROR, + "artifact", + {}, + "Hint: consider reverting your local changes " + "to the following files: %s" % sorted(changed_files), + ) + if "TASKCLUSTER_ROOT_URL" in os.environ: + command_context.log( + logging.ERROR, + "artifact", + {"build": user_value}, + "Due to the environment variable TASKCLUSTER_ROOT_URL " + "being set, the artifacts were expected to be found " + "on {}. If this was unintended, unset " + "TASKCLUSTER_ROOT_URL and try again.".format( + os.environ["TASKCLUSTER_ROOT_URL"] + ), + ) + return 1 + + command_context.log( + logging.DEBUG, + "artifact", + {"name": artifact_name, "task_id": task_id}, + "Found {name} in {task_id}", + ) + + record = ArtifactRecord(task_id, artifact_name) + records[record.filename] = record + + # Handle the list of files of the form task_id:path from --from-task. + for f in from_task or (): + task_id, colon, name = f.partition(":") + if not colon: + command_context.log( + logging.ERROR, + "artifact", + {}, + "Expected an argument of the form task_id:path", + ) + return 1 + record = ArtifactRecord(task_id, name) + records[record.filename] = record + + for record in six.itervalues(records): + command_context.log( + logging.INFO, + "artifact", + {"name": record.basename}, + "Setting up artifact {name}", + ) + valid = False + # sleeptime is 60 per retry.py, used by tooltool_wrapper.sh + for attempt, _ in enumerate(redo.retrier(attempts=retry + 1, sleeptime=60)): + try: + record.fetch_with(cache) + except ( + requests.exceptions.HTTPError, + requests.exceptions.ChunkedEncodingError, + requests.exceptions.ConnectionError, + ) as e: + + if isinstance(e, requests.exceptions.HTTPError): + # The relengapi proxy likes to return error 400 bad request + # which seems improbably to be due to our (simple) GET + # being borked. + status = e.response.status_code + should_retry = status >= 500 or status == 400 + else: + should_retry = True + + if should_retry or attempt < retry: + level = logging.WARN + else: + level = logging.ERROR + command_context.log(level, "artifact", {}, str(e)) + if not should_retry: + break + if attempt < retry: + command_context.log( + logging.INFO, "artifact", {}, "Will retry in a moment..." + ) + continue + try: + valid = record.validate() + except Exception: + pass + if not valid: + os.unlink(record.filename) + if attempt < retry: + command_context.log( + logging.INFO, + "artifact", + {}, + "Corrupt download. Will retry in a moment...", + ) + continue + + downloaded.append(record) + break + + if not valid: + command_context.log( + logging.ERROR, + "artifact", + {"name": record.basename}, + "Failed to download {name}", + ) + return 1 + + artifacts = {} if artifact_manifest else None + + for record in downloaded: + local = os.path.join(os.getcwd(), record.basename) + if os.path.exists(local): + os.unlink(local) + # unpack_file needs the file with its final name to work + # (https://github.com/mozilla/build-tooltool/issues/38), so we + # need to copy it, even though we remove it later. Use hard links + # when possible. + try: + os.link(record.filename, local) + except Exception: + shutil.copy(record.filename, local) + # Keep a sha256 of each downloaded file, for the chain-of-trust + # validation. + if artifact_manifest is not None: + with open(local, "rb") as fh: + h = hashlib.sha256() + while True: + data = fh.read(1024 * 1024) + if not data: + break + h.update(data) + artifacts[record.url] = {"sha256": h.hexdigest()} + if record.unpack and not no_unpack: + unpack_file(local) + os.unlink(local) + + if not downloaded: + command_context.log(logging.ERROR, "artifact", {}, "Nothing to download") + if from_task: + return 1 + + if artifacts: + ensureParentDir(artifact_manifest) + with open(artifact_manifest, "w") as fh: + json.dump(artifacts, fh, indent=4, sort_keys=True) + + if "MOZ_AUTOMATION" in os.environ: + end = time.monotonic() + + perfherder_data = { + "framework": {"name": "build_metrics"}, + "suites": [ + { + "name": "mach_artifact_toolchain", + "value": end - start, + "lowerIsBetter": True, + "shouldAlert": False, + "subtests": [], + } + ], + } + command_context.log( + logging.INFO, + "perfherder", + {"data": json.dumps(perfherder_data)}, + "PERFHERDER_DATA: {data}", + ) + + return 0 |