diff options
Diffstat (limited to 'build/moz.configure/bootstrap.configure')
| -rw-r--r-- | build/moz.configure/bootstrap.configure | 252 |
1 files changed, 252 insertions, 0 deletions
diff --git a/build/moz.configure/bootstrap.configure b/build/moz.configure/bootstrap.configure new file mode 100644 index 0000000000..ec17c3e34c --- /dev/null +++ b/build/moz.configure/bootstrap.configure @@ -0,0 +1,252 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +option( + env="MOZ_FETCHES_DIR", + nargs=1, + when="MOZ_AUTOMATION", + help="Directory containing fetched artifacts", +) + + +@depends("MOZ_FETCHES_DIR", when="MOZ_AUTOMATION") +def moz_fetches_dir(value): + if value: + return value[0] + + +@depends(vcs_checkout_type, milestone.is_nightly, "MOZ_AUTOMATION") +def bootstrap_default(vcs_checkout_type, is_nightly, automation): + if automation: + return False + # We only enable if building off a VCS checkout of central. + if is_nightly and vcs_checkout_type: + return True + + +option( + "--enable-bootstrap", + default=bootstrap_default, + help="{Automatically bootstrap or update some toolchains|Disable bootstrap or update of toolchains}", +) + + +@depends(developer_options, "--enable-bootstrap", moz_fetches_dir) +def bootstrap_search_path_order(developer_options, bootstrap, moz_fetches_dir): + if moz_fetches_dir: + log.debug("Prioritizing MOZ_FETCHES_DIR in toolchain path.") + return "prepend" + + if bootstrap: + log.debug( + "Prioritizing mozbuild state dir in toolchain paths because " + "bootstrap mode is enabled." + ) + return "prepend" + + if developer_options: + log.debug( + "Prioritizing mozbuild state dir in toolchain paths because " + "you are not building in release mode." + ) + return "prepend" + + log.debug( + "Prioritizing system over mozbuild state dir in " + "toolchain paths because you are building in " + "release mode." + ) + return "append" + + +toolchains_base_dir = moz_fetches_dir | mozbuild_state_path + + +@dependable +@imports("os") +@imports(_from="os", _import="environ") +def original_path(): + return environ["PATH"].split(os.pathsep) + + +@depends(host, when="--enable-bootstrap") +@imports("os") +@imports("traceback") +@imports(_from="mozbuild.toolchains", _import="toolchain_task_definitions") +@imports(_from="__builtin__", _import="Exception") +def bootstrap_toolchain_tasks(host): + prefix = { + ("x86_64", "GNU", "Linux"): "linux64", + ("x86_64", "OSX", "Darwin"): "macosx64", + ("aarch64", "OSX", "Darwin"): "macosx64-aarch64", + ("x86_64", "WINNT", "WINNT"): "win64", + ("aarch64", "WINNT", "WINNT"): "win64-aarch64", + }.get((host.cpu, host.os, host.kernel)) + try: + tasks = toolchain_task_definitions() + except Exception as e: + message = traceback.format_exc() + log.warning(str(e)) + log.debug(message) + return None + # We only want to use toolchains annotated with "local-toolchain". We also limit the + # amount of data to what we use, so that trace logs can be more useful. + tasks = { + k: { + "index": t.optimization["index-search"], + "artifact": t.attributes["toolchain-artifact"], + } + for k, t in tasks.items() + if t.attributes.get("local-toolchain") and "index-search" in t.optimization + } + + return namespace(prefix=prefix, tasks=tasks) + + +@template +def bootstrap_path(path, **kwargs): + when = kwargs.pop("when", None) + if kwargs: + configure_error("bootstrap_path only takes `when` as a keyword argument") + + @depends( + "--enable-bootstrap", + toolchains_base_dir, + moz_fetches_dir, + bootstrap_toolchain_tasks, + build_environment, + dependable(path), + when=when, + ) + @imports("os") + @imports("subprocess") + @imports("sys") + @imports(_from="mozbuild.util", _import="ensureParentDir") + @imports(_from="importlib", _import="import_module") + @imports(_from="__builtin__", _import="open") + @imports(_from="__builtin__", _import="Exception") + def bootstrap_path( + bootstrap, toolchains_base_dir, moz_fetches_dir, tasks, build_env, path + ): + if not path: + return + path_parts = path.split("/") + + def try_bootstrap(exists): + if not tasks: + return False + prefixes = [""] + if tasks.prefix: + prefixes.insert(0, "{}-".format(tasks.prefix)) + for prefix in prefixes: + label = "toolchain-{}{}".format(prefix, path_parts[0]) + task = tasks.tasks.get(label) + if task: + break + log.debug("Trying to bootstrap %s", label) + if not task: + return False + task_index = task["index"] + log.debug("Resolved %s to %s", label, task_index[0]) + task_index = task_index[0].split(".")[-1] + artifact = task["artifact"] + # `mach artifact toolchain` doesn't support authentication for + # private artifacts. + if not artifact.startswith("public/"): + log.debug("Cannot bootstrap %s: not a public artifact", label) + return False + index_file = os.path.join(toolchains_base_dir, "indices", path_parts[0]) + try: + with open(index_file) as fh: + index = fh.read().strip() + except Exception: + # On automation, if there's an artifact in MOZ_FETCHES_DIR, we assume it's + # up-to-date. + index = task_index if moz_fetches_dir else None + if index == task_index and exists: + log.debug("%s is up-to-date", label) + return True + # Manually import with import_module so that we can gracefully disable bootstrap + # when e.g. building from a js standalone tarball, that doesn't contain the + # taskgraph code. In those cases, `mach artifact toolchain --from-build` would + # also fail. + try: + IndexSearch = import_module( + "gecko_taskgraph.optimize.strategies" + ).IndexSearch + except Exception: + log.debug("Cannot bootstrap %s: missing taskgraph module", label) + return False + task_id = IndexSearch().should_replace_task(task, {}, None, task["index"]) + if task_id: + # If we found the task in the index, use the `mach artifact toolchain` + # fast path. + flags = ["--from-task", f"{task_id}:{artifact}"] + else: + # Otherwise, use the slower path, which will print a better error than + # we would be able to. + flags = ["--from-build", label] + + log.info( + "%s bootstrapped toolchain in %s", + "Updating" if exists else "Installing", + os.path.join(toolchains_base_dir, path_parts[0]), + ) + os.makedirs(toolchains_base_dir, exist_ok=True) + subprocess.run( + [ + sys.executable, + os.path.join(build_env.topsrcdir, "mach"), + "--log-no-times", + "artifact", + "toolchain", + ] + + flags, + cwd=toolchains_base_dir, + check=True, + ) + ensureParentDir(index_file) + with open(index_file, "w") as fh: + fh.write(task_index) + return True + + path = os.path.join(toolchains_base_dir, *path_parts) + if bootstrap: + try: + if not try_bootstrap(os.path.exists(path)): + # If there aren't toolchain artifacts to use for this build, + # don't return a path. + return None + except Exception as e: + log.error("%s", e) + die("If you can't fix the above, retry with --disable-bootstrap.") + # We re-test whether the path exists because it may have been created by + # try_bootstrap. Automation will not have gone through the bootstrap + # process, but we want to return the path if it exists. + if os.path.exists(path): + return path + + return bootstrap_path + + +@template +def bootstrap_search_path(path, paths=original_path, **kwargs): + @depends( + bootstrap_path(path, **kwargs), + bootstrap_search_path_order, + paths, + original_path, + ) + def bootstrap_search_path(path, order, paths, original_path): + if paths is None: + paths = original_path + if not path: + return paths + if order == "prepend": + return [path] + paths + return paths + [path] + + return bootstrap_search_path |