summaryrefslogtreecommitdiffstats
path: root/python/mozboot
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 09:22:09 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 09:22:09 +0000
commit43a97878ce14b72f0981164f87f2e35e14151312 (patch)
tree620249daf56c0258faa40cbdcf9cfba06de2a846 /python/mozboot
parentInitial commit. (diff)
downloadfirefox-upstream.tar.xz
firefox-upstream.zip
Adding upstream version 110.0.1.upstream/110.0.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'python/mozboot')
-rw-r--r--python/mozboot/.isort.cfg3
-rw-r--r--python/mozboot/README.rst20
-rwxr-xr-xpython/mozboot/bin/bootstrap.py439
-rw-r--r--python/mozboot/mozboot/__init__.py0
-rw-r--r--python/mozboot/mozboot/android-avds/arm.json26
-rw-r--r--python/mozboot/mozboot/android-avds/arm64.json27
-rw-r--r--python/mozboot/mozboot/android-avds/x86_64.json26
-rw-r--r--python/mozboot/mozboot/android-emulator-packages.txt2
-rw-r--r--python/mozboot/mozboot/android-packages.txt4
-rw-r--r--python/mozboot/mozboot/android-system-images-packages.txt1
-rw-r--r--python/mozboot/mozboot/android.py891
-rw-r--r--python/mozboot/mozboot/archlinux.py33
-rw-r--r--python/mozboot/mozboot/base.py741
-rw-r--r--python/mozboot/mozboot/bootstrap.py765
-rw-r--r--python/mozboot/mozboot/centosfedora.py80
-rw-r--r--python/mozboot/mozboot/debian.py83
-rw-r--r--python/mozboot/mozboot/freebsd.py74
-rw-r--r--python/mozboot/mozboot/gentoo.py29
-rw-r--r--python/mozboot/mozboot/linux_common.py113
-rw-r--r--python/mozboot/mozboot/mach_commands.py111
-rw-r--r--python/mozboot/mozboot/mozconfig.py156
-rw-r--r--python/mozboot/mozboot/mozillabuild.py240
-rw-r--r--python/mozboot/mozboot/openbsd.py38
-rw-r--r--python/mozboot/mozboot/opensuse.py63
-rw-r--r--python/mozboot/mozboot/osx.py324
-rw-r--r--python/mozboot/mozboot/rust.py185
-rw-r--r--python/mozboot/mozboot/sccache.py9
-rw-r--r--python/mozboot/mozboot/solus.py32
-rw-r--r--python/mozboot/mozboot/static_analysis.py8
-rw-r--r--python/mozboot/mozboot/test/python.ini4
-rw-r--r--python/mozboot/mozboot/test/test_mozconfig.py229
-rw-r--r--python/mozboot/mozboot/util.py48
-rw-r--r--python/mozboot/mozboot/void.py41
-rw-r--r--python/mozboot/mozboot/windows.py132
-rw-r--r--python/mozboot/setup.py16
35 files changed, 4993 insertions, 0 deletions
diff --git a/python/mozboot/.isort.cfg b/python/mozboot/.isort.cfg
new file mode 100644
index 0000000000..875efb0395
--- /dev/null
+++ b/python/mozboot/.isort.cfg
@@ -0,0 +1,3 @@
+[settings]
+profile=black
+known_first_party=mozboot \ No newline at end of file
diff --git a/python/mozboot/README.rst b/python/mozboot/README.rst
new file mode 100644
index 0000000000..97dc3c97b2
--- /dev/null
+++ b/python/mozboot/README.rst
@@ -0,0 +1,20 @@
+mozboot - Bootstrap your system to build Mozilla projects
+=========================================================
+
+This package contains code used for bootstrapping a system to build
+mozilla-central.
+
+This code is not part of the build system per se. Instead, it is related
+to everything up to invoking the actual build system.
+
+If you have a copy of the source tree, you run:
+
+ python bin/bootstrap.py
+
+If you don't have a copy of the source tree, you can run:
+
+ curl https://hg.mozilla.org/mozilla-central/raw-file/default/python/mozboot/bin/bootstrap.py -o bootstrap.py
+ python bootstrap.py
+
+The bootstrap script will download everything it needs from hg.mozilla.org
+automatically!
diff --git a/python/mozboot/bin/bootstrap.py b/python/mozboot/bin/bootstrap.py
new file mode 100755
index 0000000000..733042bbc5
--- /dev/null
+++ b/python/mozboot/bin/bootstrap.py
@@ -0,0 +1,439 @@
+#!/usr/bin/env python3
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This script provides one-line bootstrap support to configure systems to build
+# the tree. It does so by cloning the repo before calling directly into `mach
+# bootstrap`.
+
+# Note that this script can't assume anything in particular about the host
+# Python environment (except that it's run with a sufficiently recent version of
+# Python 3), so we are restricted to stdlib modules.
+
+import sys
+
+major, minor = sys.version_info[:2]
+if (major < 3) or (major == 3 and minor < 6):
+ print(
+ "Bootstrap currently only runs on Python 3.6+."
+ "Please try re-running with python3.6+."
+ )
+ sys.exit(1)
+
+import ctypes
+import os
+import shutil
+import subprocess
+import tempfile
+from optparse import OptionParser
+from pathlib import Path
+
+CLONE_MERCURIAL_PULL_FAIL = """
+Failed to pull from hg.mozilla.org.
+
+This is most likely because of unstable network connection.
+Try running `cd %s && hg pull https://hg.mozilla.org/mozilla-unified` manually,
+or download a mercurial bundle and use it:
+https://firefox-source-docs.mozilla.org/contributing/vcs/mercurial_bundles.html"""
+
+WINDOWS = sys.platform.startswith("win32") or sys.platform.startswith("msys")
+VCS_HUMAN_READABLE = {
+ "hg": "Mercurial",
+ "git": "Git",
+}
+
+
+def which(name):
+ """Python implementation of which.
+
+ It returns the path of an executable or None if it couldn't be found.
+ """
+ search_dirs = os.environ["PATH"].split(os.pathsep)
+ potential_names = [name]
+ if WINDOWS:
+ potential_names.insert(0, name + ".exe")
+
+ for path in search_dirs:
+ for executable_name in potential_names:
+ test = Path(path) / executable_name
+ if test.is_file() and os.access(test, os.X_OK):
+ return test
+
+ return None
+
+
+def validate_clone_dest(dest: Path):
+ dest = dest.resolve()
+
+ if not dest.exists():
+ return dest
+
+ if not dest.is_dir():
+ print(f"ERROR! Destination {dest} exists but is not a directory.")
+ return None
+
+ if not any(dest.iterdir()):
+ return dest
+ else:
+ print(f"ERROR! Destination directory {dest} exists but is nonempty.")
+ print(
+ f"To re-bootstrap the existing checkout, go into '{dest}' and run './mach bootstrap'."
+ )
+ return None
+
+
+def input_clone_dest(vcs, no_interactive):
+ repo_name = "mozilla-unified"
+ print(f"Cloning into {repo_name} using {VCS_HUMAN_READABLE[vcs]}...")
+ while True:
+ dest = None
+ if not no_interactive:
+ dest = input(
+ f"Destination directory for clone (leave empty to use "
+ f"default destination of {repo_name}): "
+ ).strip()
+ if not dest:
+ dest = repo_name
+ dest = validate_clone_dest(Path(dest).expanduser())
+ if dest:
+ return dest
+ if no_interactive:
+ return None
+
+
+def hg_clone_firefox(hg: Path, dest: Path, head_repo, head_rev):
+ # We create an empty repo then modify the config before adding data.
+ # This is necessary to ensure storage settings are optimally
+ # configured.
+ args = [
+ str(hg),
+ # The unified repo is generaldelta, so ensure the client is as
+ # well.
+ "--config",
+ "format.generaldelta=true",
+ "init",
+ str(dest),
+ ]
+ res = subprocess.call(args)
+ if res:
+ print("unable to create destination repo; please try cloning manually")
+ return None
+
+ # Strictly speaking, this could overwrite a config based on a template
+ # the user has installed. Let's pretend this problem doesn't exist
+ # unless someone complains about it.
+ with open(dest / ".hg" / "hgrc", "a") as fh:
+ fh.write("[paths]\n")
+ fh.write("default = https://hg.mozilla.org/mozilla-unified\n")
+ fh.write("\n")
+
+ # The server uses aggressivemergedeltas which can blow up delta chain
+ # length. This can cause performance to tank due to delta chains being
+ # too long. Limit the delta chain length to something reasonable
+ # to bound revlog read time.
+ fh.write("[format]\n")
+ fh.write("# This is necessary to keep performance in check\n")
+ fh.write("maxchainlen = 10000\n")
+
+ # Pulling a specific revision into an empty repository induces a lot of
+ # load on the Mercurial server, so we always pull from mozilla-unified (which,
+ # when done from an empty repository, is equivalent to a clone), and then pull
+ # the specific revision we want (if we want a specific one, otherwise we just
+ # use the "central" bookmark), at which point it will be an incremental pull,
+ # that the server can process more easily.
+ # This is the same thing that robustcheckout does on automation.
+ res = subprocess.call(
+ [str(hg), "pull", "https://hg.mozilla.org/mozilla-unified"], cwd=str(dest)
+ )
+ if not res and head_repo:
+ res = subprocess.call(
+ [str(hg), "pull", head_repo, "-r", head_rev], cwd=str(dest)
+ )
+ print("")
+ if res:
+ print(CLONE_MERCURIAL_PULL_FAIL % dest)
+ return None
+
+ head_rev = head_rev or "central"
+ print(f'updating to "{head_rev}" - the development head of Gecko and Firefox')
+ res = subprocess.call([str(hg), "update", "-r", head_rev], cwd=str(dest))
+ if res:
+ print(
+ f"error updating; you will need to `cd {dest} && hg update -r central` "
+ "manually"
+ )
+ return dest
+
+
+def git_clone_firefox(git: Path, dest: Path, watchman: Path, head_repo, head_rev):
+ tempdir = None
+ cinnabar = None
+ env = dict(os.environ)
+ try:
+ cinnabar = which("git-cinnabar")
+ if not cinnabar:
+ from urllib.request import urlopen
+
+ cinnabar_url = "https://github.com/glandium/git-cinnabar/"
+ # If git-cinnabar isn't installed already, that's fine; we can
+ # download a temporary copy. `mach bootstrap` will install a copy
+ # in the state dir; we don't want to copy all that logic to this
+ # tiny bootstrapping script.
+ tempdir = Path(tempfile.mkdtemp())
+ with open(tempdir / "download.py", "wb") as fh:
+ shutil.copyfileobj(
+ urlopen(f"{cinnabar_url}/raw/master/download.py"), fh
+ )
+
+ subprocess.check_call(
+ [sys.executable, str(tempdir / "download.py")],
+ cwd=str(tempdir),
+ )
+ env["PATH"] = str(tempdir) + os.pathsep + env["PATH"]
+ print(
+ "WARNING! git-cinnabar is required for Firefox development "
+ "with git. After the clone is complete, the bootstrapper "
+ "will ask if you would like to configure git; answer yes, "
+ "and be sure to add git-cinnabar to your PATH according to "
+ "the bootstrapper output."
+ )
+
+ # We're guaranteed to have `git-cinnabar` installed now.
+ # Configure git per the git-cinnabar requirements.
+ subprocess.check_call(
+ [
+ str(git),
+ "clone",
+ "--no-checkout",
+ "hg::https://hg.mozilla.org/mozilla-unified",
+ str(dest),
+ ],
+ env=env,
+ )
+ subprocess.check_call(
+ [str(git), "config", "fetch.prune", "true"], cwd=str(dest), env=env
+ )
+ subprocess.check_call(
+ [str(git), "config", "pull.ff", "only"], cwd=str(dest), env=env
+ )
+
+ if head_repo:
+ subprocess.check_call(
+ [str(git), "cinnabar", "fetch", f"hg::{head_repo}", head_rev],
+ cwd=str(dest),
+ env=env,
+ )
+
+ subprocess.check_call(
+ [str(git), "checkout", "FETCH_HEAD" if head_rev else "bookmarks/central"],
+ cwd=str(dest),
+ env=env,
+ )
+
+ watchman_sample = dest / ".git/hooks/fsmonitor-watchman.sample"
+ # Older versions of git didn't include fsmonitor-watchman.sample.
+ if watchman and watchman_sample.exists():
+ print("Configuring watchman")
+ watchman_config = dest / ".git/hooks/query-watchman"
+ if not watchman_config.exists():
+ print(f"Copying {watchman_sample} to {watchman_config}")
+ copy_args = [
+ "cp",
+ ".git/hooks/fsmonitor-watchman.sample",
+ ".git/hooks/query-watchman",
+ ]
+ subprocess.check_call(copy_args, cwd=str(dest))
+
+ config_args = [
+ str(git),
+ "config",
+ "core.fsmonitor",
+ ".git/hooks/query-watchman",
+ ]
+ subprocess.check_call(config_args, cwd=str(dest), env=env)
+ return dest
+ finally:
+ if tempdir:
+ shutil.rmtree(str(tempdir))
+
+
+def add_microsoft_defender_antivirus_exclusions(dest, no_system_changes):
+ if no_system_changes:
+ return
+
+ if not WINDOWS:
+ return
+
+ powershell_exe = which("powershell")
+
+ if not powershell_exe:
+ return
+
+ def print_attempt_exclusion(path):
+ print(
+ f"Attempting to add exclusion path to Microsoft Defender Antivirus for: {path}"
+ )
+
+ powershell_exe = str(powershell_exe)
+ paths = []
+
+ # mozilla-unified / clone dest
+ repo_dir = Path.cwd() / dest
+ paths.append(repo_dir)
+ print_attempt_exclusion(repo_dir)
+
+ # MOZILLABUILD
+ mozillabuild_dir = os.getenv("MOZILLABUILD")
+ if mozillabuild_dir:
+ paths.append(mozillabuild_dir)
+ print_attempt_exclusion(mozillabuild_dir)
+
+ # .mozbuild
+ mozbuild_dir = Path.home() / ".mozbuild"
+ paths.append(mozbuild_dir)
+ print_attempt_exclusion(mozbuild_dir)
+
+ args = ";".join(f"Add-MpPreference -ExclusionPath '{path}'" for path in paths)
+ command = f'-Command "{args}"'
+
+ # This will attempt to run as administrator by triggering a UAC prompt
+ # for admin credentials. If "No" is selected, no exclusions are added.
+ ctypes.windll.shell32.ShellExecuteW(None, "runas", powershell_exe, command, None, 0)
+
+
+def clone(options):
+ vcs = options.vcs
+ no_interactive = options.no_interactive
+ no_system_changes = options.no_system_changes
+
+ if vcs == "hg":
+ hg = which("hg")
+ if not hg:
+ print("Mercurial is not installed. Mercurial is required to clone Firefox.")
+ try:
+ # We're going to recommend people install the Mercurial package with
+ # pip3. That will work if `pip3` installs binaries to a location
+ # that's in the PATH, but it might not be. To help out, if we CAN
+ # import "mercurial" (in which case it's already been installed),
+ # offer that as a solution.
+ import mercurial # noqa: F401
+
+ print(
+ "Hint: have you made sure that Mercurial is installed to a "
+ "location in your PATH?"
+ )
+ except ImportError:
+ print("Try installing hg with `pip3 install Mercurial`.")
+ return None
+ binary = hg
+ else:
+ binary = which(vcs)
+ if not binary:
+ print("Git is not installed.")
+ print("Try installing git using your system package manager.")
+ return None
+
+ dest = input_clone_dest(vcs, no_interactive)
+ if not dest:
+ return None
+
+ add_microsoft_defender_antivirus_exclusions(dest, no_system_changes)
+
+ print(f"Cloning Firefox {VCS_HUMAN_READABLE[vcs]} repository to {dest}")
+
+ head_repo = os.environ.get("GECKO_HEAD_REPOSITORY")
+ head_rev = os.environ.get("GECKO_HEAD_REV")
+
+ if vcs == "hg":
+ return hg_clone_firefox(binary, dest, head_repo, head_rev)
+ else:
+ watchman = which("watchman")
+ return git_clone_firefox(binary, dest, watchman, head_repo, head_rev)
+
+
+def bootstrap(srcdir: Path, application_choice, no_interactive, no_system_changes):
+ args = [sys.executable, "mach"]
+
+ if no_interactive:
+ # --no-interactive is a global argument, not a command argument,
+ # so it needs to be specified before "bootstrap" is appended.
+ args += ["--no-interactive"]
+
+ args += ["bootstrap"]
+
+ if application_choice:
+ args += ["--application-choice", application_choice]
+ if no_system_changes:
+ args += ["--no-system-changes"]
+
+ print("Running `%s`" % " ".join(args))
+ return subprocess.call(args, cwd=str(srcdir))
+
+
+def main(args):
+ parser = OptionParser()
+ parser.add_option(
+ "--application-choice",
+ dest="application_choice",
+ help='Pass in an application choice (see "APPLICATIONS" in '
+ "python/mozboot/mozboot/bootstrap.py) instead of using the "
+ "default interactive prompt.",
+ )
+ parser.add_option(
+ "--vcs",
+ dest="vcs",
+ default="hg",
+ choices=["git", "hg"],
+ help="VCS (hg or git) to use for downloading the source code, "
+ "instead of using the default interactive prompt.",
+ )
+ parser.add_option(
+ "--no-interactive",
+ dest="no_interactive",
+ action="store_true",
+ help="Answer yes to any (Y/n) interactive prompts.",
+ )
+ parser.add_option(
+ "--no-system-changes",
+ dest="no_system_changes",
+ action="store_true",
+ help="Only executes actions that leave the system " "configuration alone.",
+ )
+
+ options, leftover = parser.parse_args(args)
+ try:
+ srcdir = clone(options)
+ if not srcdir:
+ return 1
+ print("Clone complete.")
+ print(
+ "If you need to run the tooling bootstrapping again, "
+ "then consider running './mach bootstrap' instead."
+ )
+ if not options.no_interactive:
+ remove_bootstrap_file = input(
+ "Unless you are going to have more local copies of Firefox source code, "
+ "this 'bootstrap.py' file is no longer needed and can be deleted. "
+ "Clean up the bootstrap.py file? (Y/n)"
+ )
+ if not remove_bootstrap_file:
+ remove_bootstrap_file = "y"
+ if options.no_interactive or remove_bootstrap_file == "y":
+ try:
+ Path(sys.argv[0]).unlink()
+ except FileNotFoundError:
+ print("File could not be found !")
+ return bootstrap(
+ srcdir,
+ options.application_choice,
+ options.no_interactive,
+ options.no_system_changes,
+ )
+ except Exception:
+ print("Could not bootstrap Firefox! Consider filing a bug.")
+ raise
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv))
diff --git a/python/mozboot/mozboot/__init__.py b/python/mozboot/mozboot/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/python/mozboot/mozboot/__init__.py
diff --git a/python/mozboot/mozboot/android-avds/arm.json b/python/mozboot/mozboot/android-avds/arm.json
new file mode 100644
index 0000000000..bcabc737fd
--- /dev/null
+++ b/python/mozboot/mozboot/android-avds/arm.json
@@ -0,0 +1,26 @@
+{
+ "emulator_package": "system-images;android-24;default;armeabi-v7a",
+ "emulator_avd_name": "mozemulator-armeabi-v7a",
+ "emulator_extra_args": [
+ "-skip-adb-auth",
+ "-verbose",
+ "-show-kernel",
+ "-ranchu",
+ "-selinux", "permissive",
+ "-memory", "3072",
+ "-cores", "4",
+ "-skin", "800x1280",
+ "-gpu", "on",
+ "-no-snapstorage",
+ "-no-snapshot",
+ "-no-window",
+ "-no-accel",
+ "-prop", "ro.test_harness=true"
+ ],
+ "emulator_extra_config": {
+ "hw.keyboard": "yes",
+ "hw.lcd.density": "320",
+ "disk.dataPartition.size": "4000MB",
+ "sdcard.size": "600M"
+ }
+}
diff --git a/python/mozboot/mozboot/android-avds/arm64.json b/python/mozboot/mozboot/android-avds/arm64.json
new file mode 100644
index 0000000000..767f9299d4
--- /dev/null
+++ b/python/mozboot/mozboot/android-avds/arm64.json
@@ -0,0 +1,27 @@
+{
+ "emulator_package": "system-images;android-30;default;arm64-v8a",
+ "emulator_avd_name": "mozemulator-arm64",
+ "emulator_extra_args": [
+ "-skip-adb-auth",
+ "-verbose",
+ "-show-kernel",
+ "-ranchu",
+ "-selinux", "permissive",
+ "-memory", "3072",
+ "-cores", "4",
+ "-skin", "800x1280",
+ "-gpu", "on",
+ "-no-snapstorage",
+ "-no-snapshot",
+ "-no-window",
+ "-no-accel",
+ "-prop", "ro.test_harness=true"
+ ],
+ "emulator_extra_config": {
+ "hw.keyboard": "yes",
+ "hw.lcd.density": "320",
+ "disk.dataPartition.size": "4000MB",
+ "sdcard.size": "600M"
+ },
+ "emulator_prewarm": false
+}
diff --git a/python/mozboot/mozboot/android-avds/x86_64.json b/python/mozboot/mozboot/android-avds/x86_64.json
new file mode 100644
index 0000000000..68f99a9937
--- /dev/null
+++ b/python/mozboot/mozboot/android-avds/x86_64.json
@@ -0,0 +1,26 @@
+{
+ "emulator_package": "system-images;android-24;default;x86_64",
+ "emulator_avd_name": "mozemulator-x86_64",
+ "emulator_extra_args": [
+ "-skip-adb-auth",
+ "-verbose",
+ "-show-kernel",
+ "-ranchu",
+ "-selinux", "permissive",
+ "-memory", "3072",
+ "-cores", "4",
+ "-skin", "800x1280",
+ "-gpu", "on",
+ "-no-snapstorage",
+ "-no-snapshot",
+ "-no-window",
+ "-no-accel",
+ "-prop", "ro.test_harness=true"
+ ],
+ "emulator_extra_config": {
+ "hw.keyboard": "yes",
+ "hw.lcd.density": "320",
+ "disk.dataPartition.size": "4000MB",
+ "sdcard.size": "600M"
+ }
+}
diff --git a/python/mozboot/mozboot/android-emulator-packages.txt b/python/mozboot/mozboot/android-emulator-packages.txt
new file mode 100644
index 0000000000..3e782df670
--- /dev/null
+++ b/python/mozboot/mozboot/android-emulator-packages.txt
@@ -0,0 +1,2 @@
+platform-tools
+emulator
diff --git a/python/mozboot/mozboot/android-packages.txt b/python/mozboot/mozboot/android-packages.txt
new file mode 100644
index 0000000000..e36a300d83
--- /dev/null
+++ b/python/mozboot/mozboot/android-packages.txt
@@ -0,0 +1,4 @@
+emulator
+platform-tools
+build-tools;33.0.1
+platforms;android-33
diff --git a/python/mozboot/mozboot/android-system-images-packages.txt b/python/mozboot/mozboot/android-system-images-packages.txt
new file mode 100644
index 0000000000..86069f7680
--- /dev/null
+++ b/python/mozboot/mozboot/android-system-images-packages.txt
@@ -0,0 +1 @@
+emulator
diff --git a/python/mozboot/mozboot/android.py b/python/mozboot/mozboot/android.py
new file mode 100644
index 0000000000..0297c37f0e
--- /dev/null
+++ b/python/mozboot/mozboot/android.py
@@ -0,0 +1,891 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this,
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import errno
+import json
+import os
+import stat
+import subprocess
+import sys
+import time
+from pathlib import Path
+from typing import Optional, Union
+
+import requests
+from tqdm import tqdm
+
+# We need the NDK version in multiple different places, and it's inconvenient
+# to pass down the NDK version to all relevant places, so we have this global
+# variable.
+from mozboot.bootstrap import MOZCONFIG_SUGGESTION_TEMPLATE
+
+NDK_VERSION = "r21d"
+CMDLINE_TOOLS_VERSION_STRING = "7.0"
+CMDLINE_TOOLS_VERSION = "8512546"
+
+BUNDLETOOL_VERSION = "1.11.0"
+
+# We expect the emulator AVD definitions to be platform agnostic
+LINUX_X86_64_ANDROID_AVD = "linux64-android-avd-x86_64-repack"
+LINUX_ARM_ANDROID_AVD = "linux64-android-avd-arm-repack"
+
+MACOS_X86_64_ANDROID_AVD = "linux64-android-avd-x86_64-repack"
+MACOS_ARM_ANDROID_AVD = "linux64-android-avd-arm-repack"
+MACOS_ARM64_ANDROID_AVD = "linux64-android-avd-arm64-repack"
+
+WINDOWS_X86_64_ANDROID_AVD = "linux64-android-avd-x86_64-repack"
+WINDOWS_ARM_ANDROID_AVD = "linux64-android-avd-arm-repack"
+
+AVD_MANIFEST_X86_64 = Path(__file__).resolve().parent / "android-avds/x86_64.json"
+AVD_MANIFEST_ARM = Path(__file__).resolve().parent / "android-avds/arm.json"
+AVD_MANIFEST_ARM64 = Path(__file__).resolve().parent / "android-avds/arm64.json"
+
+JAVA_VERSION_MAJOR = "17"
+JAVA_VERSION_MINOR = "0.5"
+JAVA_VERSION_PATCH = "8"
+
+ANDROID_NDK_EXISTS = """
+Looks like you have the correct version of the Android NDK installed at:
+%s
+"""
+
+ANDROID_SDK_EXISTS = """
+Looks like you have the Android SDK installed at:
+%s
+We will install all required Android packages.
+"""
+
+ANDROID_SDK_TOO_OLD = """
+Looks like you have an outdated Android SDK installed at:
+%s
+I can't update outdated Android SDKs to have the required 'sdkmanager'
+tool. Move it out of the way (or remove it entirely) and then run
+bootstrap again.
+"""
+
+INSTALLING_ANDROID_PACKAGES = """
+We are now installing the following Android packages:
+%s
+You may be prompted to agree to the Android license. You may see some of
+output as packages are downloaded and installed.
+"""
+
+MOBILE_ANDROID_MOZCONFIG_TEMPLATE = """
+# Build GeckoView/Firefox for Android:
+ac_add_options --enable-project=mobile/android
+
+# Targeting the following architecture.
+# For regular phones, no --target is needed.
+# For x86 emulators (and x86 devices, which are uncommon):
+# ac_add_options --target=i686
+# For newer phones or Apple silicon
+# ac_add_options --target=aarch64
+# For x86_64 emulators (and x86_64 devices, which are even less common):
+# ac_add_options --target=x86_64
+
+{extra_lines}
+"""
+
+MOBILE_ANDROID_ARTIFACT_MODE_MOZCONFIG_TEMPLATE = """
+# Build GeckoView/Firefox for Android Artifact Mode:
+ac_add_options --enable-project=mobile/android
+ac_add_options --enable-artifact-builds
+
+{extra_lines}
+# Write build artifacts to:
+mk_add_options MOZ_OBJDIR=./objdir-frontend
+"""
+
+
+class GetNdkVersionError(Exception):
+ pass
+
+
+def install_mobile_android_sdk_or_ndk(url, path: Path):
+ """
+ Fetch an Android SDK or NDK from |url| and unpack it into the given |path|.
+
+ We use, and 'requests' respects, https. We could also include SHAs for a
+ small improvement in the integrity guarantee we give. But this script is
+ bootstrapped over https anyway, so it's a really minor improvement.
+
+ We keep a cache of the downloaded artifacts, writing into |path|/mozboot.
+ We don't yet clean the cache; it's better to waste some disk space and
+ not require a long re-download than to wipe the cache prematurely.
+ """
+
+ download_path = path / "mozboot"
+ try:
+ download_path.mkdir(parents=True)
+ except OSError as e:
+ if e.errno == errno.EEXIST and download_path.is_dir():
+ pass
+ else:
+ raise
+
+ file_name = url.split("/")[-1]
+ download_file_path = download_path / file_name
+ download(url, download_file_path)
+
+ if file_name.endswith(".tar.gz") or file_name.endswith(".tgz"):
+ cmd = ["tar", "zxf", str(download_file_path)]
+ elif file_name.endswith(".tar.bz2"):
+ cmd = ["tar", "jxf", str(download_file_path)]
+ elif file_name.endswith(".zip"):
+ cmd = ["unzip", "-q", str(download_file_path)]
+ elif file_name.endswith(".bin"):
+ # Execute the .bin file, which unpacks the content.
+ mode = os.stat(path).st_mode
+ download_file_path.chmod(mode | stat.S_IXUSR)
+ cmd = [str(download_file_path)]
+ else:
+ raise NotImplementedError(f"Don't know how to unpack file: {file_name}")
+
+ print(f"Unpacking {download_file_path}...")
+
+ with open(os.devnull, "w") as stdout:
+ # These unpack commands produce a ton of output; ignore it. The
+ # .bin files are 7z archives; there's no command line flag to quiet
+ # output, so we use this hammer.
+ subprocess.check_call(cmd, stdout=stdout, cwd=str(path))
+
+ print(f"Unpacking {download_file_path}... DONE")
+ # Now delete the archive
+ download_file_path.unlink()
+
+
+def download(
+ url,
+ download_file_path: Path,
+):
+ with requests.Session() as session:
+ request = session.head(url, allow_redirects=True)
+ request.raise_for_status()
+ remote_file_size = int(request.headers["content-length"])
+
+ if download_file_path.is_file():
+ local_file_size = download_file_path.stat().st_size
+
+ if local_file_size == remote_file_size:
+ print(
+ f"{download_file_path.name} already downloaded. Skipping download..."
+ )
+ else:
+ print(f"Partial download detected. Resuming download of {url}...")
+ download_internal(
+ download_file_path,
+ session,
+ url,
+ remote_file_size,
+ local_file_size,
+ )
+ else:
+ print(f"Downloading {url}...")
+ download_internal(download_file_path, session, url, remote_file_size)
+
+
+def download_internal(
+ download_file_path: Path,
+ session,
+ url,
+ remote_file_size,
+ resume_from_byte_pos: int = None,
+):
+ """
+ Handles both a fresh SDK/NDK download, as well as resuming a partial one
+ """
+ # "ab" will behave same as "wb" if file does not exist
+ with open(download_file_path, "ab") as file:
+ # 64 KB/s should be fine on even the slowest internet connections
+ chunk_size = 1024 * 64
+ # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Range#directives
+ resume_header = (
+ {"Range": f"bytes={resume_from_byte_pos}-"}
+ if resume_from_byte_pos
+ else None
+ )
+
+ request = session.get(
+ url, stream=True, allow_redirects=True, headers=resume_header
+ )
+
+ with tqdm(
+ total=int(remote_file_size),
+ unit="B",
+ unit_scale=True,
+ unit_divisor=1024,
+ desc=download_file_path.name,
+ initial=resume_from_byte_pos if resume_from_byte_pos else 0,
+ ) as progress_bar:
+ for chunk in request.iter_content(chunk_size):
+ file.write(chunk)
+ progress_bar.update(len(chunk))
+
+
+def get_ndk_version(ndk_path: Union[str, Path]):
+ """Given the path to the NDK, return the version as a 3-tuple of (major,
+ minor, human).
+ """
+ ndk_path = Path(ndk_path)
+ with open(ndk_path / "source.properties", "r") as f:
+ revision = [line for line in f if line.startswith("Pkg.Revision")]
+ if not revision:
+ raise GetNdkVersionError(
+ "Cannot determine NDK version from source.properties"
+ )
+ if len(revision) != 1:
+ raise GetNdkVersionError("Too many Pkg.Revision lines in source.properties")
+
+ (_, version) = revision[0].split("=")
+ if not version:
+ raise GetNdkVersionError(
+ "Unexpected Pkg.Revision line in source.properties"
+ )
+
+ (major, minor, revision) = version.strip().split(".")
+ if not major or not minor:
+ raise GetNdkVersionError("Unexpected NDK version string: " + version)
+
+ # source.properties contains a $MAJOR.$MINOR.$PATCH revision number,
+ # but the more common nomenclature that Google uses is alphanumeric
+ # version strings like "r20" or "r19c". Convert the source.properties
+ # notation into an alphanumeric string.
+ int_minor = int(minor)
+ alphas = "abcdefghijklmnop"
+ ascii_minor = alphas[int_minor] if int_minor > 0 else ""
+ human = "r%s%s" % (major, ascii_minor)
+ return (major, minor, human)
+
+
+def get_paths(os_name):
+ mozbuild_path = Path(
+ os.environ.get("MOZBUILD_STATE_PATH", Path("~/.mozbuild").expanduser())
+ )
+ sdk_path = Path(
+ os.environ.get("ANDROID_SDK_HOME", mozbuild_path / f"android-sdk-{os_name}"),
+ )
+ ndk_path = Path(
+ os.environ.get(
+ "ANDROID_NDK_HOME", mozbuild_path / f"android-ndk-{NDK_VERSION}"
+ ),
+ )
+ avd_home_path = Path(
+ os.environ.get("ANDROID_AVD_HOME", mozbuild_path / "android-device" / "avd")
+ )
+ return mozbuild_path, sdk_path, ndk_path, avd_home_path
+
+
+def sdkmanager_tool(sdk_path: Path):
+ # sys.platform is win32 even if Python/Win64.
+ sdkmanager = "sdkmanager.bat" if sys.platform.startswith("win") else "sdkmanager"
+ return (
+ sdk_path / "cmdline-tools" / CMDLINE_TOOLS_VERSION_STRING / "bin" / sdkmanager
+ )
+
+
+def avdmanager_tool(sdk_path: Path):
+ # sys.platform is win32 even if Python/Win64.
+ sdkmanager = "avdmanager.bat" if sys.platform.startswith("win") else "avdmanager"
+ return (
+ sdk_path / "cmdline-tools" / CMDLINE_TOOLS_VERSION_STRING / "bin" / sdkmanager
+ )
+
+
+def adb_tool(sdk_path: Path):
+ adb = "adb.bat" if sys.platform.startswith("win") else "adb"
+ return sdk_path / "platform-tools" / adb
+
+
+def emulator_tool(sdk_path: Path):
+ emulator = "emulator.bat" if sys.platform.startswith("win") else "emulator"
+ return sdk_path / "emulator" / emulator
+
+
+def ensure_android(
+ os_name,
+ os_arch,
+ artifact_mode=False,
+ ndk_only=False,
+ system_images_only=False,
+ emulator_only=False,
+ avd_manifest_path: Optional[Path] = None,
+ prewarm_avd=False,
+ no_interactive=False,
+ list_packages=False,
+):
+ """
+ Ensure the Android SDK (and NDK, if `artifact_mode` is falsy) are
+ installed. If not, fetch and unpack the SDK and/or NDK from the
+ given URLs. Ensure the required Android SDK packages are
+ installed.
+
+ `os_name` can be 'linux', 'macosx' or 'windows'.
+ """
+ # The user may have an external Android SDK (in which case we
+ # save them a lengthy download), or they may have already
+ # completed the download. We unpack to
+ # ~/.mozbuild/{android-sdk-$OS_NAME, android-ndk-$VER}.
+ mozbuild_path, sdk_path, ndk_path, avd_home_path = get_paths(os_name)
+
+ if os_name == "macosx":
+ os_tag = "mac"
+ elif os_name == "windows":
+ os_tag = "win"
+ else:
+ os_tag = os_name
+
+ sdk_url = "https://dl.google.com/android/repository/commandlinetools-{0}-{1}_latest.zip".format( # NOQA: E501
+ os_tag, CMDLINE_TOOLS_VERSION
+ )
+ ndk_url = android_ndk_url(os_name)
+ bundletool_url = "https://github.com/google/bundletool/releases/download/{v}/bundletool-all-{v}.jar".format( # NOQA: E501
+ v=BUNDLETOOL_VERSION
+ )
+
+ ensure_android_sdk_and_ndk(
+ mozbuild_path,
+ os_name,
+ sdk_path=sdk_path,
+ sdk_url=sdk_url,
+ ndk_path=ndk_path,
+ ndk_url=ndk_url,
+ bundletool_url=bundletool_url,
+ artifact_mode=artifact_mode,
+ ndk_only=ndk_only,
+ emulator_only=emulator_only,
+ )
+
+ if ndk_only:
+ return
+
+ avd_manifest = None
+ if avd_manifest_path is not None:
+ with open(avd_manifest_path) as f:
+ avd_manifest = json.load(f)
+ # Some AVDs cannot be prewarmed in CI because they cannot run on linux64
+ # (like the arm64 AVD).
+ if "emulator_prewarm" in avd_manifest:
+ prewarm_avd = prewarm_avd and avd_manifest["emulator_prewarm"]
+
+ # We expect the |sdkmanager| tool to be at
+ # ~/.mozbuild/android-sdk-$OS_NAME/tools/cmdline-tools/$CMDLINE_TOOLS_VERSION_STRING/bin/sdkmanager. # NOQA: E501
+ ensure_android_packages(
+ os_name,
+ os_arch,
+ sdkmanager_tool=sdkmanager_tool(sdk_path),
+ emulator_only=emulator_only,
+ system_images_only=system_images_only,
+ avd_manifest=avd_manifest,
+ no_interactive=no_interactive,
+ list_packages=list_packages,
+ )
+
+ if emulator_only or system_images_only:
+ return
+
+ ensure_android_avd(
+ avdmanager_tool=avdmanager_tool(sdk_path),
+ adb_tool=adb_tool(sdk_path),
+ emulator_tool=emulator_tool(sdk_path),
+ avd_home_path=avd_home_path,
+ sdk_path=sdk_path,
+ no_interactive=no_interactive,
+ avd_manifest=avd_manifest,
+ prewarm_avd=prewarm_avd,
+ )
+
+
+def ensure_android_sdk_and_ndk(
+ mozbuild_path: Path,
+ os_name,
+ sdk_path: Path,
+ sdk_url,
+ ndk_path: Path,
+ ndk_url,
+ bundletool_url,
+ artifact_mode,
+ ndk_only,
+ emulator_only,
+):
+ """
+ Ensure the Android SDK and NDK are found at the given paths. If not, fetch
+ and unpack the SDK and/or NDK from the given URLs into
+ |mozbuild_path/{android-sdk-$OS_NAME,android-ndk-$VER}|.
+ """
+
+ # It's not particularly bad to overwrite the NDK toolchain, but it does take
+ # a while to unpack, so let's avoid the disk activity if possible. The SDK
+ # may prompt about licensing, so we do this first.
+ # Check for Android NDK only if we are not in artifact mode.
+ if not artifact_mode and not emulator_only:
+ install_ndk = True
+ if ndk_path.is_dir():
+ try:
+ _, _, human = get_ndk_version(ndk_path)
+ if human == NDK_VERSION:
+ print(ANDROID_NDK_EXISTS % ndk_path)
+ install_ndk = False
+ except GetNdkVersionError:
+ pass # Just do the install.
+ if install_ndk:
+ # The NDK archive unpacks into a top-level android-ndk-$VER directory.
+ install_mobile_android_sdk_or_ndk(ndk_url, mozbuild_path)
+
+ if ndk_only:
+ return
+
+ # We don't want to blindly overwrite, since we use the
+ # |sdkmanager| tool to install additional parts of the Android
+ # toolchain. If we overwrite, we lose whatever Android packages
+ # the user may have already installed.
+ if sdkmanager_tool(sdk_path).is_file():
+ print(ANDROID_SDK_EXISTS % sdk_path)
+ elif sdk_path.is_dir():
+ raise NotImplementedError(ANDROID_SDK_TOO_OLD % sdk_path)
+ else:
+ # The SDK archive used to include a top-level
+ # android-sdk-$OS_NAME directory; it no longer does so. We
+ # preserve the old convention to smooth detecting existing SDK
+ # installations.
+ cmdline_tools_path = mozbuild_path / f"android-sdk-{os_name}" / "cmdline-tools"
+ install_mobile_android_sdk_or_ndk(sdk_url, cmdline_tools_path)
+ # The tools package *really* wants to be in
+ # <sdk>/cmdline-tools/$CMDLINE_TOOLS_VERSION_STRING
+ (cmdline_tools_path / "cmdline-tools").rename(
+ cmdline_tools_path / CMDLINE_TOOLS_VERSION_STRING
+ )
+ download(bundletool_url, mozbuild_path / "bundletool.jar")
+
+
+def get_packages_to_install(packages_file_content, avd_manifest):
+ packages = []
+ packages += map(lambda package: package.strip(), packages_file_content)
+ if avd_manifest is not None:
+ packages += [avd_manifest["emulator_package"]]
+ return packages
+
+
+def ensure_android_avd(
+ avdmanager_tool: Path,
+ adb_tool: Path,
+ emulator_tool: Path,
+ avd_home_path: Path,
+ sdk_path: Path,
+ no_interactive=False,
+ avd_manifest=None,
+ prewarm_avd=False,
+):
+ """
+ Use the given sdkmanager tool (like 'sdkmanager') to install required
+ Android packages.
+ """
+ if avd_manifest is None:
+ return
+
+ avd_home_path.mkdir(parents=True, exist_ok=True)
+ # The AVD needs this folder to boot, so make sure it exists here.
+ (sdk_path / "platforms").mkdir(parents=True, exist_ok=True)
+
+ avd_name = avd_manifest["emulator_avd_name"]
+ args = [
+ str(avdmanager_tool),
+ "--verbose",
+ "create",
+ "avd",
+ "--force",
+ "--name",
+ avd_name,
+ "--package",
+ avd_manifest["emulator_package"],
+ ]
+
+ if not no_interactive:
+ subprocess.check_call(args)
+ return
+
+ # Flush outputs before running sdkmanager.
+ sys.stdout.flush()
+ env = os.environ.copy()
+ env["ANDROID_AVD_HOME"] = str(avd_home_path)
+ proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env)
+ proc.communicate("no\n".encode("UTF-8"))
+
+ retcode = proc.poll()
+ if retcode:
+ cmd = args[0]
+ e = subprocess.CalledProcessError(retcode, cmd)
+ raise e
+
+ avd_path = avd_home_path / (str(avd_name) + ".avd")
+ config_file_name = avd_path / "config.ini"
+
+ print(f"Writing config at {config_file_name}")
+
+ if config_file_name.is_file():
+ with open(config_file_name, "a") as config:
+ for key, value in avd_manifest["emulator_extra_config"].items():
+ config.write("%s=%s\n" % (key, value))
+ else:
+ raise NotImplementedError(
+ f"Could not find config file at {config_file_name}, something went wrong"
+ )
+ if prewarm_avd:
+ run_prewarm_avd(adb_tool, emulator_tool, env, avd_name, avd_manifest)
+ # When running in headless mode, the emulator does not run the cleanup
+ # step, and thus doesn't delete lock files. On some platforms, left-over
+ # lock files can cause the emulator to not start, so we remove them here.
+ for lock_file in ["hardware-qemu.ini.lock", "multiinstance.lock"]:
+ lock_file_path = avd_path / lock_file
+ try:
+ lock_file_path.unlink()
+ print(f"Removed lock file {lock_file_path}")
+ except OSError:
+ # The lock file is not there, nothing to do.
+ pass
+
+
+def run_prewarm_avd(
+ adb_tool: Path,
+ emulator_tool: Path,
+ env,
+ avd_name,
+ avd_manifest,
+):
+ """
+ Ensures the emulator is fully booted to save time on future iterations.
+ """
+ args = [str(emulator_tool), "-avd", avd_name] + avd_manifest["emulator_extra_args"]
+
+ # Flush outputs before running emulator.
+ sys.stdout.flush()
+ proc = subprocess.Popen(args, env=env)
+
+ booted = False
+ for i in range(100):
+ boot_completed_cmd = [str(adb_tool), "shell", "getprop", "sys.boot_completed"]
+ completed_proc = subprocess.Popen(
+ boot_completed_cmd, env=env, stdout=subprocess.PIPE
+ )
+ try:
+ out, err = completed_proc.communicate(timeout=30)
+ boot_completed = out.decode("UTF-8").strip()
+ print("sys.boot_completed = %s" % boot_completed)
+ time.sleep(30)
+ if boot_completed == "1":
+ booted = True
+ break
+ except subprocess.TimeoutExpired:
+ # Sometimes the adb command hangs, that's ok
+ print("sys.boot_completed = Timeout")
+
+ if not booted:
+ raise NotImplementedError("Could not prewarm emulator")
+
+ # Wait until the emulator completely shuts down
+ subprocess.Popen([str(adb_tool), "emu", "kill"], env=env).wait()
+ proc.wait()
+
+
+def ensure_android_packages(
+ os_name,
+ os_arch,
+ sdkmanager_tool: Path,
+ emulator_only=False,
+ system_images_only=False,
+ avd_manifest=None,
+ no_interactive=False,
+ list_packages=False,
+):
+ """
+ Use the given sdkmanager tool (like 'sdkmanager') to install required
+ Android packages.
+ """
+
+ # This tries to install all the required Android packages. The user
+ # may be prompted to agree to the Android license.
+ if system_images_only:
+ packages_file_name = "android-system-images-packages.txt"
+ elif emulator_only:
+ packages_file_name = "android-emulator-packages.txt"
+ else:
+ packages_file_name = "android-packages.txt"
+
+ packages_file_path = (Path(__file__).parent / packages_file_name).resolve()
+
+ with open(packages_file_path) as packages_file:
+ packages_file_content = packages_file.readlines()
+
+ packages = get_packages_to_install(packages_file_content, avd_manifest)
+ print(INSTALLING_ANDROID_PACKAGES % "\n".join(packages))
+
+ args = [str(sdkmanager_tool)]
+ if os_name == "macosx" and os_arch == "arm64":
+ # Support for Apple Silicon is still in nightly
+ args.append("--channel=3")
+ args.extend(packages)
+
+ # sdkmanager needs JAVA_HOME
+ java_bin_path = ensure_java(os_name, os_arch)
+ env = os.environ.copy()
+ env["JAVA_HOME"] = str(java_bin_path.parent)
+
+ if not no_interactive:
+ subprocess.check_call(args, env=env)
+ return
+
+ # Flush outputs before running sdkmanager.
+ sys.stdout.flush()
+ sys.stderr.flush()
+ # Emulate yes. For a discussion of passing input to check_output,
+ # see https://stackoverflow.com/q/10103551.
+ yes = "\n".join(["y"] * 100).encode("UTF-8")
+ proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env)
+ proc.communicate(yes)
+
+ retcode = proc.poll()
+ if retcode:
+ cmd = args[0]
+ e = subprocess.CalledProcessError(retcode, cmd)
+ raise e
+ if list_packages:
+ subprocess.check_call([str(sdkmanager_tool), "--list"])
+
+
+def generate_mozconfig(os_name, artifact_mode=False):
+ moz_state_dir, sdk_path, ndk_path, avd_home_path = get_paths(os_name)
+
+ extra_lines = []
+ if extra_lines:
+ extra_lines.append("")
+
+ if artifact_mode:
+ template = MOBILE_ANDROID_ARTIFACT_MODE_MOZCONFIG_TEMPLATE
+ else:
+ template = MOBILE_ANDROID_MOZCONFIG_TEMPLATE
+
+ kwargs = dict(
+ sdk_path=sdk_path,
+ ndk_path=ndk_path,
+ avd_home_path=avd_home_path,
+ moz_state_dir=moz_state_dir,
+ extra_lines="\n".join(extra_lines),
+ )
+ return template.format(**kwargs).strip()
+
+
+def android_ndk_url(os_name, ver=NDK_VERSION):
+ # Produce a URL like
+ # 'https://dl.google.com/android/repository/android-ndk-$VER-linux-x86_64.zip
+ base_url = "https://dl.google.com/android/repository/android-ndk"
+
+ if os_name == "macosx":
+ # |mach bootstrap| uses 'macosx', but Google uses 'darwin'.
+ os_name = "darwin"
+
+ if sys.maxsize > 2 ** 32:
+ arch = "x86_64"
+ else:
+ arch = "x86"
+
+ return "%s-%s-%s-%s.zip" % (base_url, ver, os_name, arch)
+
+
+def main(argv):
+ import optparse # No argparse, which is new in Python 2.7.
+ import platform
+
+ parser = optparse.OptionParser()
+ parser.add_option(
+ "-a",
+ "--artifact-mode",
+ dest="artifact_mode",
+ action="store_true",
+ help="If true, install only the Android SDK (and not the Android NDK).",
+ )
+ parser.add_option(
+ "--jdk-only",
+ dest="jdk_only",
+ action="store_true",
+ help="If true, install only the Java JDK.",
+ )
+ parser.add_option(
+ "--ndk-only",
+ dest="ndk_only",
+ action="store_true",
+ help="If true, install only the Android NDK (and not the Android SDK).",
+ )
+ parser.add_option(
+ "--system-images-only",
+ dest="system_images_only",
+ action="store_true",
+ help="If true, install only the system images for the AVDs.",
+ )
+ parser.add_option(
+ "--no-interactive",
+ dest="no_interactive",
+ action="store_true",
+ help="Accept the Android SDK licenses without user interaction.",
+ )
+ parser.add_option(
+ "--emulator-only",
+ dest="emulator_only",
+ action="store_true",
+ help="If true, install only the Android emulator (and not the SDK or NDK).",
+ )
+ parser.add_option(
+ "--avd-manifest",
+ dest="avd_manifest_path",
+ help="If present, generate AVD from the manifest pointed by this argument.",
+ )
+ parser.add_option(
+ "--prewarm-avd",
+ dest="prewarm_avd",
+ action="store_true",
+ help="If true, boot the AVD and wait until completed to speed up subsequent boots.",
+ )
+ parser.add_option(
+ "--list-packages",
+ dest="list_packages",
+ action="store_true",
+ help="If true, list installed packages.",
+ )
+
+ options, _ = parser.parse_args(argv)
+
+ if options.artifact_mode and options.ndk_only:
+ raise NotImplementedError("Use no options to install the NDK and the SDK.")
+
+ if options.artifact_mode and options.emulator_only:
+ raise NotImplementedError("Use no options to install the SDK and emulators.")
+
+ os_name = None
+ if platform.system() == "Darwin":
+ os_name = "macosx"
+ elif platform.system() == "Linux":
+ os_name = "linux"
+ elif platform.system() == "Windows":
+ os_name = "windows"
+ else:
+ raise NotImplementedError(
+ "We don't support bootstrapping the Android SDK (or Android "
+ "NDK) on {0} yet!".format(platform.system())
+ )
+
+ os_arch = platform.machine()
+
+ if options.jdk_only:
+ ensure_java(os_name, os_arch)
+ return 0
+
+ avd_manifest_path = (
+ Path(options.avd_manifest_path) if options.avd_manifest_path else None
+ )
+
+ ensure_android(
+ os_name,
+ os_arch,
+ artifact_mode=options.artifact_mode,
+ ndk_only=options.ndk_only,
+ system_images_only=options.system_images_only,
+ emulator_only=options.emulator_only,
+ avd_manifest_path=avd_manifest_path,
+ prewarm_avd=options.prewarm_avd,
+ no_interactive=options.no_interactive,
+ list_packages=options.list_packages,
+ )
+ mozconfig = generate_mozconfig(os_name, options.artifact_mode)
+
+ # |./mach bootstrap| automatically creates a mozconfig file for you if it doesn't
+ # exist. However, here, we don't know where the "topsrcdir" is, and it's not worth
+ # pulling in CommandContext (and its dependencies) to find out.
+ # So, instead, we'll politely ask users to create (or update) the file themselves.
+ suggestion = MOZCONFIG_SUGGESTION_TEMPLATE % ("$topsrcdir/mozconfig", mozconfig)
+ print("\n" + suggestion)
+
+ return 0
+
+
+def ensure_java(os_name, os_arch):
+ mozbuild_path, _, _, _ = get_paths(os_name)
+
+ if os_name == "macosx":
+ os_tag = "mac"
+ else:
+ os_tag = os_name
+
+ if os_arch == "x86_64":
+ arch = "x64"
+ elif os_arch == "arm64":
+ arch = "aarch64"
+ else:
+ arch = os_arch
+
+ ext = "zip" if os_name == "windows" else "tar.gz"
+
+ java_path = java_bin_path(os_name, mozbuild_path)
+ if not java_path:
+ raise NotImplementedError(f"Could not bootstrap java for {os_name}.")
+
+ if not java_path.exists():
+ # e.g. https://github.com/adoptium/temurin17-binaries/releases/
+ # download/jdk-17.0.5%2B8/OpenJDK17U-jdk_x64_linux_hotspot_17.0.5_8.tar.gz
+ java_url = (
+ "https://github.com/adoptium/temurin{major}-binaries/releases/"
+ "download/jdk-{major}.{minor}%2B{patch}/"
+ "OpenJDK{major}U-jdk_{arch}_{os}_hotspot_{major}.{minor}_{patch}.{ext}"
+ ).format(
+ major=JAVA_VERSION_MAJOR,
+ minor=JAVA_VERSION_MINOR,
+ patch=JAVA_VERSION_PATCH,
+ os=os_tag,
+ arch=arch,
+ ext=ext,
+ )
+ install_mobile_android_sdk_or_ndk(java_url, mozbuild_path / "jdk")
+ return java_path
+
+
+def java_bin_path(os_name, toolchain_path: Path):
+ # Like jdk-17.0.5+8
+ jdk_folder = "jdk-{major}.{minor}+{patch}".format(
+ major=JAVA_VERSION_MAJOR, minor=JAVA_VERSION_MINOR, patch=JAVA_VERSION_PATCH
+ )
+
+ java_path = toolchain_path / "jdk" / jdk_folder
+
+ if os_name == "macosx":
+ return java_path / "Contents" / "Home" / "bin"
+ elif os_name == "linux":
+ return java_path / "bin"
+ elif os_name == "windows":
+ return java_path / "bin"
+ else:
+ return None
+
+
+def locate_java_bin_path(host_kernel, toolchain_path: Union[str, Path]):
+ if host_kernel == "WINNT":
+ os_name = "windows"
+ elif host_kernel == "Darwin":
+ os_name = "macosx"
+ elif host_kernel == "Linux":
+ os_name = "linux"
+ else:
+ # Default to Linux
+ os_name = "linux"
+ path = java_bin_path(os_name, Path(toolchain_path))
+ if not path.is_dir():
+ raise JavaLocationFailedException(
+ f"Could not locate Java at {path}, please run "
+ "./mach bootstrap --no-system-changes"
+ )
+ return str(path)
+
+
+class JavaLocationFailedException(Exception):
+ pass
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv))
diff --git a/python/mozboot/mozboot/archlinux.py b/python/mozboot/mozboot/archlinux.py
new file mode 100644
index 0000000000..391bd3e3ff
--- /dev/null
+++ b/python/mozboot/mozboot/archlinux.py
@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+
+from mozboot.base import BaseBootstrapper
+from mozboot.linux_common import LinuxBootstrapper
+
+
+class ArchlinuxBootstrapper(LinuxBootstrapper, BaseBootstrapper):
+ """Archlinux experimental bootstrapper."""
+
+ def __init__(self, version, dist_id, **kwargs):
+ print("Using an experimental bootstrapper for Archlinux.", file=sys.stderr)
+ BaseBootstrapper.__init__(self, **kwargs)
+
+ def install_packages(self, packages):
+ # watchman is not available via pacman
+ packages = [p for p in packages if p != "watchman"]
+ self.pacman_install(*packages)
+
+ def upgrade_mercurial(self, current):
+ self.pacman_install("mercurial")
+
+ def pacman_install(self, *packages):
+ command = ["pacman", "-S", "--needed"]
+ if self.no_interactive:
+ command.append("--noconfirm")
+
+ command.extend(packages)
+
+ self.run_as_root(command)
diff --git a/python/mozboot/mozboot/base.py b/python/mozboot/mozboot/base.py
new file mode 100644
index 0000000000..e506bf04e0
--- /dev/null
+++ b/python/mozboot/mozboot/base.py
@@ -0,0 +1,741 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+from mach.util import to_optional_path, win_to_msys_path
+from mozbuild.bootstrap import bootstrap_all_toolchains_for, bootstrap_toolchain
+from mozfile import which
+from packaging.version import Version
+
+from mozboot import rust
+from mozboot.util import (
+ MINIMUM_RUST_VERSION,
+ get_mach_virtualenv_binary,
+ http_download_and_save,
+)
+
+NO_MERCURIAL = """
+Could not find Mercurial (hg) in the current shell's path. Try starting a new
+shell and running the bootstrapper again.
+"""
+
+MERCURIAL_UNABLE_UPGRADE = """
+You are currently running Mercurial %s. Running %s or newer is
+recommended for performance and stability reasons.
+
+Unfortunately, this bootstrapper currently does not know how to automatically
+upgrade Mercurial on your machine.
+
+You can usually install Mercurial through your package manager or by
+downloading a package from http://mercurial.selenic.com/.
+"""
+
+MERCURIAL_UPGRADE_FAILED = """
+We attempted to upgrade Mercurial to a modern version (%s or newer).
+However, you appear to have version %s still.
+
+It's possible your package manager doesn't support a modern version of
+Mercurial. It's also possible Mercurial is not being installed in the search
+path for this shell. Try creating a new shell and run this bootstrapper again.
+
+If it continues to fail, consider installing Mercurial by following the
+instructions at http://mercurial.selenic.com/.
+"""
+
+MERCURIAL_INSTALL_PROMPT = """
+Mercurial releases a new version every 3 months and your distro's package
+may become out of date. This may cause incompatibility with some
+Mercurial extensions that rely on new Mercurial features. As a result,
+you may not have an optimal version control experience.
+
+To have the best Mercurial experience possible, we recommend installing
+Mercurial via the "pip" Python packaging utility. This will likely result
+in files being placed in /usr/local/bin and /usr/local/lib.
+
+How would you like to continue?
+ 1. Install a modern Mercurial via pip [default]
+ 2. Install a legacy Mercurial via the distro package manager
+ 3. Do not install Mercurial
+Your choice: """
+
+PYTHON_UNABLE_UPGRADE = """
+You are currently running Python %s. Running %s or newer (but
+not 3.x) is required.
+
+Unfortunately, this bootstrapper does not currently know how to automatically
+upgrade Python on your machine.
+
+Please search the Internet for how to upgrade your Python and try running this
+bootstrapper again to ensure your machine is up to date.
+"""
+
+RUST_INSTALL_COMPLETE = """
+Rust installation complete. You should now have rustc and cargo
+in %(cargo_bin)s
+
+The installer tries to add these to your default shell PATH, so
+restarting your shell and running this script again may work.
+If it doesn't, you'll need to add the new command location
+manually.
+
+If restarting doesn't work, edit your shell initialization
+script, which may be called ~/.bashrc or ~/.bash_profile or
+~/.profile, and add the following line:
+
+ %(cmd)s
+
+Then restart your shell and run the bootstrap script again.
+"""
+
+RUST_NOT_IN_PATH = """
+You have some rust files in %(cargo_bin)s
+but they're not part of this shell's PATH.
+
+To add these to the PATH, edit your shell initialization
+script, which may be called ~/.bashrc or ~/.bash_profile or
+~/.profile, and add the following line:
+
+ %(cmd)s
+
+Then restart your shell and run the bootstrap script again.
+"""
+
+RUSTUP_OLD = """
+We found an executable called `rustup` which we normally use to install
+and upgrade Rust programming language support, but we didn't understand
+its output. It may be an old version, or not be the installer from
+https://rustup.rs/
+
+Please move it out of the way and run the bootstrap script again.
+Or if you prefer and know how, use the current rustup to install
+a compatible version of the Rust programming language yourself.
+"""
+
+RUST_UPGRADE_FAILED = """
+We attempted to upgrade Rust to a modern version (%s or newer).
+However, you appear to still have version %s.
+
+It's possible rustup failed. It's also possible the new Rust is not being
+installed in the search path for this shell. Try creating a new shell and
+run this bootstrapper again.
+
+If this continues to fail and you are sure you have a modern Rust on your
+system, ensure it is on the $PATH and try again. If that fails, you'll need to
+install Rust manually.
+
+We recommend the installer from https://rustup.rs/ for installing Rust,
+but you may be able to get a recent enough version from a software install
+tool or package manager on your system, or directly from https://rust-lang.org/
+"""
+
+BROWSER_ARTIFACT_MODE_MOZCONFIG = """
+# Automatically download and use compiled C++ components:
+ac_add_options --enable-artifact-builds
+""".strip()
+
+JS_MOZCONFIG_TEMPLATE = """\
+# Build only the SpiderMonkey JS test shell
+ac_add_options --enable-project=js
+"""
+
+# Upgrade Mercurial older than this.
+# This should match the OLDEST_NON_LEGACY_VERSION in
+# version-control-tools/hgext/configwizard/__init__.py.
+MODERN_MERCURIAL_VERSION = Version("4.9")
+
+# Upgrade rust older than this.
+MODERN_RUST_VERSION = Version(MINIMUM_RUST_VERSION)
+
+
+class BaseBootstrapper(object):
+ """Base class for system bootstrappers."""
+
+ def __init__(self, no_interactive=False, no_system_changes=False):
+ self.package_manager_updated = False
+ self.no_interactive = no_interactive
+ self.no_system_changes = no_system_changes
+ self.state_dir = None
+ self.srcdir = None
+
+ def validate_environment(self):
+ """
+ Called once the current firefox checkout has been detected.
+ Platform-specific implementations should check the environment and offer advice/warnings
+ to the user, if necessary.
+ """
+
+ def suggest_install_distutils(self):
+ """Called if distutils.{sysconfig,spawn} can't be imported."""
+ print(
+ "Does your distro require installing another package for distutils?",
+ file=sys.stderr,
+ )
+
+ def suggest_install_pip3(self):
+ """Called if pip3 can't be found."""
+ print(
+ "Try installing pip3 with your system's package manager.", file=sys.stderr
+ )
+
+ def install_system_packages(self):
+ """
+ Install packages shared by all applications. These are usually
+ packages required by the development (like mercurial) or the
+ build system (like autoconf).
+ """
+ raise NotImplementedError(
+ "%s must implement install_system_packages()" % __name__
+ )
+
+ def install_browser_packages(self, mozconfig_builder):
+ """
+ Install packages required to build Firefox for Desktop (application
+ 'browser').
+ """
+ raise NotImplementedError(
+ "Cannot bootstrap Firefox for Desktop: "
+ "%s does not yet implement install_browser_packages()" % __name__
+ )
+
+ def ensure_browser_packages(self):
+ """
+ Install pre-built packages needed to build Firefox for Desktop (application 'browser')
+
+ Currently this is not needed and kept for compatibility with Firefox for Android.
+ """
+ pass
+
+ def ensure_js_packages(self):
+ """
+ Install pre-built packages needed to build SpiderMonkey JavaScript Engine
+
+ Currently this is not needed and kept for compatibility with Firefox for Android.
+ """
+ pass
+
+ def ensure_browser_artifact_mode_packages(self):
+ """
+ Install pre-built packages needed to build Firefox for Desktop (application 'browser')
+
+ Currently this is not needed and kept for compatibility with Firefox for Android.
+ """
+ pass
+
+ def generate_browser_mozconfig(self):
+ """
+ Print a message to the console detailing what the user's mozconfig
+ should contain.
+
+ Firefox for Desktop can in simple cases determine its build environment
+ entirely from configure.
+ """
+ pass
+
+ def install_js_packages(self, mozconfig_builder):
+ """
+ Install packages required to build SpiderMonkey JavaScript Engine
+ (application 'js').
+ """
+ return self.install_browser_packages(mozconfig_builder)
+
+ def generate_js_mozconfig(self):
+ """
+ Create a reasonable starting point for a JS shell build.
+ """
+ return JS_MOZCONFIG_TEMPLATE
+
+ def install_browser_artifact_mode_packages(self, mozconfig_builder):
+ """
+ Install packages required to build Firefox for Desktop (application
+ 'browser') in Artifact Mode.
+ """
+ raise NotImplementedError(
+ "Cannot bootstrap Firefox for Desktop Artifact Mode: "
+ "%s does not yet implement install_browser_artifact_mode_packages()"
+ % __name__
+ )
+
+ def generate_browser_artifact_mode_mozconfig(self):
+ """
+ Print a message to the console detailing what the user's mozconfig
+ should contain.
+
+ Firefox for Desktop Artifact Mode needs to enable artifact builds and
+ a path where the build artifacts will be written to.
+ """
+ return BROWSER_ARTIFACT_MODE_MOZCONFIG
+
+ def install_mobile_android_packages(self, mozconfig_builder):
+ """
+ Install packages required to build GeckoView (application
+ 'mobile/android').
+ """
+ raise NotImplementedError(
+ "Cannot bootstrap GeckoView/Firefox for Android: "
+ "%s does not yet implement install_mobile_android_packages()" % __name__
+ )
+
+ def ensure_mobile_android_packages(self):
+ """
+ Install pre-built packages required to run GeckoView (application 'mobile/android')
+ """
+ raise NotImplementedError(
+ "Cannot bootstrap GeckoView/Firefox for Android: "
+ "%s does not yet implement ensure_mobile_android_packages()" % __name__
+ )
+
+ def ensure_mobile_android_artifact_mode_packages(self):
+ """
+ Install pre-built packages required to run GeckoView Artifact Build
+ (application 'mobile/android')
+ """
+ self.ensure_mobile_android_packages()
+
+ def generate_mobile_android_mozconfig(self):
+ """
+ Print a message to the console detailing what the user's mozconfig
+ should contain.
+
+ GeckoView/Firefox for Android needs an application and an ABI set, and it needs
+ paths to the Android SDK and NDK.
+ """
+ raise NotImplementedError(
+ "%s does not yet implement generate_mobile_android_mozconfig()" % __name__
+ )
+
+ def install_mobile_android_artifact_mode_packages(self, mozconfig_builder):
+ """
+ Install packages required to build GeckoView/Firefox for Android (application
+ 'mobile/android', also known as Fennec) in Artifact Mode.
+ """
+ raise NotImplementedError(
+ "Cannot bootstrap GeckoView/Firefox for Android Artifact Mode: "
+ "%s does not yet implement install_mobile_android_artifact_mode_packages()"
+ % __name__
+ )
+
+ def generate_mobile_android_artifact_mode_mozconfig(self):
+ """
+ Print a message to the console detailing what the user's mozconfig
+ should contain.
+
+ GeckoView/Firefox for Android Artifact Mode needs an application and an ABI set,
+ and it needs paths to the Android SDK.
+ """
+ raise NotImplementedError(
+ "%s does not yet implement generate_mobile_android_artifact_mode_mozconfig()"
+ % __name__
+ )
+
+ def ensure_clang_static_analysis_package(self):
+ """
+ Install the clang static analysis package
+ """
+ raise NotImplementedError(
+ "%s does not yet implement ensure_clang_static_analysis_package()"
+ % __name__
+ )
+
+ def ensure_sccache_packages(self):
+ """
+ Install sccache.
+ """
+ pass
+
+ def install_toolchain_static_analysis(self, toolchain_job):
+ clang_tools_path = self.state_dir / "clang-tools"
+ if not clang_tools_path.exists():
+ clang_tools_path.mkdir()
+ self.install_toolchain_artifact_impl(clang_tools_path, toolchain_job)
+
+ def install_toolchain_artifact(self, toolchain_job, no_unpack=False):
+ if no_unpack:
+ return self.install_toolchain_artifact_impl(
+ self.state_dir, toolchain_job, no_unpack
+ )
+ bootstrap_toolchain(toolchain_job)
+
+ def install_toolchain_artifact_impl(
+ self, install_dir: Path, toolchain_job, no_unpack=False
+ ):
+ mach_binary = (self.srcdir / "mach").resolve()
+ if not mach_binary.exists():
+ raise ValueError(f"mach not found at {mach_binary}")
+
+ if not self.state_dir:
+ raise ValueError(
+ "Need a state directory (e.g. ~/.mozbuild) to download " "artifacts"
+ )
+ python_location = get_mach_virtualenv_binary()
+ if not python_location.exists():
+ raise ValueError(f"python not found at {python_location}")
+
+ cmd = [
+ str(python_location),
+ str(mach_binary),
+ "artifact",
+ "toolchain",
+ "--bootstrap",
+ "--from-build",
+ toolchain_job,
+ ]
+
+ if no_unpack:
+ cmd += ["--no-unpack"]
+
+ subprocess.check_call(cmd, cwd=str(install_dir))
+
+ def auto_bootstrap(self, application):
+ args = ["--with-ccache=sccache"]
+ if application.endswith("_artifact_mode"):
+ args.append("--enable-artifact-builds")
+ application = application[: -len("_artifact_mode")]
+ args.append("--enable-project={}".format(application.replace("_", "/")))
+ bootstrap_all_toolchains_for(args)
+
+ def run_as_root(self, command, may_use_sudo=True):
+ if os.geteuid() != 0:
+ if may_use_sudo and which("sudo"):
+ command.insert(0, "sudo")
+ else:
+ command = ["su", "root", "-c", " ".join(command)]
+
+ print("Executing as root:", subprocess.list2cmdline(command))
+
+ subprocess.check_call(command, stdin=sys.stdin)
+
+ def prompt_int(self, prompt, low, high, default=None):
+ """Prompts the user with prompt and requires an integer between low and high.
+
+ If the user doesn't select an option and a default isn't provided, then
+ the lowest option is used. This is because some option must be implicitly
+ selected if mach is invoked with "--no-interactive"
+ """
+ if default is not None:
+ assert isinstance(default, int)
+ assert low <= default <= high
+ else:
+ default = low
+
+ if self.no_interactive:
+ print(prompt)
+ print('Selecting "{}" because context is not interactive.'.format(default))
+ return default
+
+ while True:
+ choice = input(prompt)
+ if choice == "" and default is not None:
+ return default
+ try:
+ choice = int(choice)
+ if low <= choice <= high:
+ return choice
+ except ValueError:
+ pass
+ print("ERROR! Please enter a valid option!")
+
+ def prompt_yesno(self, prompt):
+ """Prompts the user with prompt and requires a yes/no answer."""
+ if self.no_interactive:
+ print(prompt)
+ print('Selecting "Y" because context is not interactive.')
+ return True
+
+ while True:
+ choice = input(prompt + " (Yn): ").strip().lower()[:1]
+ if choice == "":
+ return True
+ elif choice in ("y", "n"):
+ return choice == "y"
+
+ print("ERROR! Please enter y or n!")
+
+ def _ensure_package_manager_updated(self):
+ if self.package_manager_updated:
+ return
+
+ self._update_package_manager()
+ self.package_manager_updated = True
+
+ def _update_package_manager(self):
+ """Updates the package manager's manifests/package list.
+
+ This should be defined in child classes.
+ """
+
+ def _parse_version_impl(self, path: Path, name, env, version_param):
+ """Execute the given path, returning the version.
+
+ Invokes the path argument with the --version switch
+ and returns a Version representing the output
+ if successful. If not, returns None.
+
+ An optional name argument gives the expected program
+ name returned as part of the version string, if it's
+ different from the basename of the executable.
+
+ An optional env argument allows modifying environment
+ variable during the invocation to set options, PATH,
+ etc.
+ """
+ if not name:
+ name = path.name
+ if name.lower().endswith(".exe"):
+ name = name[:-4]
+
+ process = subprocess.run(
+ [str(path), version_param],
+ env=env,
+ universal_newlines=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ )
+ if process.returncode != 0:
+ # This can happen e.g. if the user has an inactive pyenv shim in
+ # their path. Just silently treat this as a failure to parse the
+ # path and move on.
+ return None
+
+ match = re.search(name + " ([a-z0-9\.]+)", process.stdout)
+ if not match:
+ print("ERROR! Unable to identify %s version." % name)
+ return None
+
+ return Version(match.group(1))
+
+ def _parse_version(self, path: Path, name=None, env=None):
+ return self._parse_version_impl(path, name, env, "--version")
+
+ def _hg_cleanenv(self, load_hgrc=False):
+ """Returns a copy of the current environment updated with the HGPLAIN
+ and HGRCPATH environment variables.
+
+ HGPLAIN prevents Mercurial from applying locale variations to the output
+ making it suitable for use in scripts.
+
+ HGRCPATH controls the loading of hgrc files. Setting it to the empty
+ string forces that no user or system hgrc file is used.
+ """
+ env = os.environ.copy()
+ env["HGPLAIN"] = "1"
+ if not load_hgrc:
+ env["HGRCPATH"] = ""
+
+ return env
+
+ def is_mercurial_modern(self):
+ hg = to_optional_path(which("hg"))
+ if not hg:
+ print(NO_MERCURIAL)
+ return False, False, None
+
+ our = self._parse_version(hg, "version", self._hg_cleanenv())
+ if not our:
+ return True, False, None
+
+ return True, our >= MODERN_MERCURIAL_VERSION, our
+
+ def ensure_mercurial_modern(self):
+ installed, modern, version = self.is_mercurial_modern()
+
+ if modern:
+ print("Your version of Mercurial (%s) is sufficiently modern." % version)
+ return installed, modern
+
+ self._ensure_package_manager_updated()
+
+ if installed:
+ print("Your version of Mercurial (%s) is not modern enough." % version)
+ print(
+ "(Older versions of Mercurial have known security vulnerabilities. "
+ "Unless you are running a patched Mercurial version, you may be "
+ "vulnerable."
+ )
+ else:
+ print("You do not have Mercurial installed")
+
+ if self.upgrade_mercurial(version) is False:
+ return installed, modern
+
+ installed, modern, after = self.is_mercurial_modern()
+
+ if installed and not modern:
+ print(MERCURIAL_UPGRADE_FAILED % (MODERN_MERCURIAL_VERSION, after))
+
+ return installed, modern
+
+ def upgrade_mercurial(self, current):
+ """Upgrade Mercurial.
+
+ Child classes should reimplement this.
+
+ Return False to not perform a version check after the upgrade is
+ performed.
+ """
+ print(MERCURIAL_UNABLE_UPGRADE % (current, MODERN_MERCURIAL_VERSION))
+
+ def warn_if_pythonpath_is_set(self):
+ if "PYTHONPATH" in os.environ:
+ print(
+ "WARNING: Your PYTHONPATH environment variable is set. This can "
+ "cause flaky installations of the requirements, and other unexpected "
+ "issues with mach. It is recommended to unset this variable."
+ )
+
+ def is_rust_modern(self, cargo_bin: Path):
+ rustc = to_optional_path(which("rustc", extra_search_dirs=[str(cargo_bin)]))
+ if not rustc:
+ print("Could not find a Rust compiler.")
+ return False, None
+
+ our = self._parse_version(rustc)
+ if not our:
+ return False, None
+
+ return our >= MODERN_RUST_VERSION, our
+
+ def cargo_home(self):
+ cargo_home = Path(os.environ.get("CARGO_HOME", Path("~/.cargo").expanduser()))
+ cargo_bin = cargo_home / "bin"
+ return cargo_home, cargo_bin
+
+ def print_rust_path_advice(self, template, cargo_home: Path, cargo_bin: Path):
+ # Suggest ~/.cargo/env if it exists.
+ if (cargo_home / "env").exists():
+ cmd = f"source {cargo_home}/env"
+ else:
+ # On Windows rustup doesn't write out ~/.cargo/env
+ # so fall back to a manual PATH update. Bootstrap
+ # only runs under msys, so a unix-style shell command
+ # is appropriate there.
+ cargo_bin = win_to_msys_path(cargo_bin)
+ cmd = f"export PATH={cargo_bin}:$PATH"
+ print(template % {"cargo_bin": cargo_bin, "cmd": cmd})
+
+ def ensure_rust_modern(self):
+ cargo_home, cargo_bin = self.cargo_home()
+ modern, version = self.is_rust_modern(cargo_bin)
+
+ rustup = to_optional_path(which("rustup", extra_search_dirs=[str(cargo_bin)]))
+
+ if modern:
+ print("Your version of Rust (%s) is new enough." % version)
+
+ elif version:
+ print("Your version of Rust (%s) is too old." % version)
+
+ if rustup and not modern:
+ rustup_version = self._parse_version(rustup)
+ if not rustup_version:
+ print(RUSTUP_OLD)
+ sys.exit(1)
+ print("Found rustup. Will try to upgrade.")
+ self.upgrade_rust(rustup)
+
+ modern, after = self.is_rust_modern(cargo_bin)
+ if not modern:
+ print(RUST_UPGRADE_FAILED % (MODERN_RUST_VERSION, after))
+ sys.exit(1)
+ elif not rustup:
+ # No rustup. Download and run the installer.
+ print("Will try to install Rust.")
+ self.install_rust()
+ modern, version = self.is_rust_modern(cargo_bin)
+ rustup = to_optional_path(
+ which("rustup", extra_search_dirs=[str(cargo_bin)])
+ )
+
+ self.ensure_rust_targets(rustup, version)
+
+ def ensure_rust_targets(self, rustup: Path, rust_version):
+ """Make sure appropriate cross target libraries are installed."""
+ target_list = subprocess.check_output(
+ [str(rustup), "target", "list"], universal_newlines=True
+ )
+ targets = [
+ line.split()[0]
+ for line in target_list.splitlines()
+ if "installed" in line or "default" in line
+ ]
+ print("Rust supports %s targets." % ", ".join(targets))
+
+ # Support 32-bit Windows on 64-bit Windows.
+ win32 = "i686-pc-windows-msvc"
+ win64 = "x86_64-pc-windows-msvc"
+ if rust.platform() == win64 and win32 not in targets:
+ subprocess.check_call([str(rustup), "target", "add", win32])
+
+ if "mobile_android" in self.application:
+ # Let's add the most common targets.
+ if rust_version < Version("1.33"):
+ arm_target = "armv7-linux-androideabi"
+ else:
+ arm_target = "thumbv7neon-linux-androideabi"
+ android_targets = (
+ arm_target,
+ "aarch64-linux-android",
+ "i686-linux-android",
+ "x86_64-linux-android",
+ )
+ for target in android_targets:
+ if target not in targets:
+ subprocess.check_call([str(rustup), "target", "add", target])
+
+ def upgrade_rust(self, rustup: Path):
+ """Upgrade Rust.
+
+ Invoke rustup from the given path to update the rust install."""
+ subprocess.check_call([str(rustup), "update"])
+ # This installs rustfmt when not already installed, or nothing
+ # otherwise, while the update above would have taken care of upgrading
+ # it.
+ subprocess.check_call([str(rustup), "component", "add", "rustfmt"])
+
+ def install_rust(self):
+ """Download and run the rustup installer."""
+ import errno
+ import stat
+ import tempfile
+
+ platform = rust.platform()
+ url = rust.rustup_url(platform)
+ checksum = rust.rustup_hash(platform)
+ if not url or not checksum:
+ print("ERROR: Could not download installer.")
+ sys.exit(1)
+ print("Downloading rustup-init... ", end="")
+ fd, rustup_init = tempfile.mkstemp(prefix=Path(url).name)
+ rustup_init = Path(rustup_init)
+ os.close(fd)
+ try:
+ http_download_and_save(url, rustup_init, checksum)
+ mode = rustup_init.stat().st_mode
+ rustup_init.chmod(mode | stat.S_IRWXU)
+ print("Ok")
+ print("Running rustup-init...")
+ subprocess.check_call(
+ [
+ str(rustup_init),
+ "-y",
+ "--default-toolchain",
+ "stable",
+ "--default-host",
+ platform,
+ "--component",
+ "rustfmt",
+ ]
+ )
+ cargo_home, cargo_bin = self.cargo_home()
+ self.print_rust_path_advice(RUST_INSTALL_COMPLETE, cargo_home, cargo_bin)
+ finally:
+ try:
+ rustup_init.unlink()
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootstrap.py
new file mode 100644
index 0000000000..36fb9128b7
--- /dev/null
+++ b/python/mozboot/mozboot/bootstrap.py
@@ -0,0 +1,765 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import platform
+import re
+import shutil
+import stat
+import subprocess
+import sys
+import time
+from collections import OrderedDict
+from pathlib import Path
+from typing import Optional
+
+# Use distro package to retrieve linux platform information
+import distro
+from mach.site import MachSiteManager
+from mach.telemetry import initialize_telemetry_setting
+from mach.util import (
+ UserError,
+ get_state_dir,
+ to_optional_path,
+ to_optional_str,
+ win_to_msys_path,
+)
+from mozbuild.base import MozbuildObject
+from mozfile import which
+from packaging.version import Version
+
+from mozboot.archlinux import ArchlinuxBootstrapper
+from mozboot.base import MODERN_RUST_VERSION
+from mozboot.centosfedora import CentOSFedoraBootstrapper
+from mozboot.debian import DebianBootstrapper
+from mozboot.freebsd import FreeBSDBootstrapper
+from mozboot.gentoo import GentooBootstrapper
+from mozboot.mozconfig import MozconfigBuilder
+from mozboot.mozillabuild import MozillaBuildBootstrapper
+from mozboot.openbsd import OpenBSDBootstrapper
+from mozboot.opensuse import OpenSUSEBootstrapper
+from mozboot.osx import OSXBootstrapper, OSXBootstrapperLight
+from mozboot.solus import SolusBootstrapper
+from mozboot.void import VoidBootstrapper
+from mozboot.windows import WindowsBootstrapper
+
+APPLICATION_CHOICE = """
+Note on Artifact Mode:
+
+Artifact builds download prebuilt C++ components rather than building
+them locally. Artifact builds are faster!
+
+Artifact builds are recommended for people working on Firefox or
+Firefox for Android frontends, or the GeckoView Java API. They are unsuitable
+for those working on C++ code. For more information see:
+https://firefox-source-docs.mozilla.org/contributing/build/artifact_builds.html.
+
+Please choose the version of Firefox you want to build (see note above):
+%s
+Your choice: """
+
+APPLICATIONS = OrderedDict(
+ [
+ ("Firefox for Desktop Artifact Mode", "browser_artifact_mode"),
+ ("Firefox for Desktop", "browser"),
+ ("GeckoView/Firefox for Android Artifact Mode", "mobile_android_artifact_mode"),
+ ("GeckoView/Firefox for Android", "mobile_android"),
+ ("SpiderMonkey JavaScript engine", "js"),
+ ]
+)
+
+FINISHED = """
+Your system should be ready to build %s!
+"""
+
+MOZCONFIG_SUGGESTION_TEMPLATE = """
+Paste the lines between the chevrons (>>> and <<<) into
+%s:
+
+>>>
+%s
+<<<
+"""
+
+MOZCONFIG_MISMATCH_WARNING_TEMPLATE = """
+WARNING! Mismatch detected between the selected build target and the
+mozconfig file %s:
+
+Current config
+>>>
+%s
+<<<
+
+Expected config
+>>>
+%s
+<<<
+"""
+
+CONFIGURE_MERCURIAL = """
+Mozilla recommends a number of changes to Mercurial to enhance your
+experience with it.
+
+Would you like to run a configuration wizard to ensure Mercurial is
+optimally configured?"""
+
+CONFIGURE_GIT = """
+Mozilla recommends using git-cinnabar to work with mozilla-central (or
+mozilla-unified).
+
+Would you like to run a few configuration steps to ensure Git is
+optimally configured?"""
+
+DEBIAN_DISTROS = (
+ "debian",
+ "ubuntu",
+ "linuxmint",
+ "elementary",
+ "neon",
+ "pop",
+ "kali",
+ "devuan",
+ "pureos",
+ "deepin",
+ "tuxedo",
+)
+
+FEDORA_DISTROS = (
+ "centos",
+ "fedora",
+ "rocky",
+ "oracle",
+)
+
+ADD_GIT_CINNABAR_PATH = """
+To add git-cinnabar to the PATH, edit your shell initialization script, which
+may be called {prefix}/.bash_profile or {prefix}/.profile, and add the following
+lines:
+
+ export PATH="{cinnabar_dir}:$PATH"
+
+Then restart your shell.
+"""
+
+
+OLD_REVISION_WARNING = """
+WARNING! You appear to be running `mach bootstrap` from an old revision.
+bootstrap is meant primarily for getting developer environments up-to-date to
+build the latest version of tree. Running bootstrap on old revisions may fail
+and is not guaranteed to bring your machine to any working state in particular.
+Proceed at your own peril.
+"""
+
+
+# Version 2.24 changes the "core.commitGraph" setting to be "True" by default.
+MINIMUM_RECOMMENDED_GIT_VERSION = Version("2.24")
+OLD_GIT_WARNING = """
+You are running an older version of git ("{old_version}").
+We recommend upgrading to at least version "{minimum_recommended_version}" to improve
+performance.
+""".strip()
+
+
+class Bootstrapper(object):
+ """Main class that performs system bootstrap."""
+
+ def __init__(
+ self,
+ choice=None,
+ no_interactive=False,
+ hg_configure=False,
+ no_system_changes=False,
+ mach_context=None,
+ ):
+ self.instance = None
+ self.choice = choice
+ self.hg_configure = hg_configure
+ self.no_system_changes = no_system_changes
+ self.mach_context = mach_context
+ cls = None
+ args = {
+ "no_interactive": no_interactive,
+ "no_system_changes": no_system_changes,
+ }
+
+ if sys.platform.startswith("linux"):
+ # distro package provides reliable ids for popular distributions so
+ # we use those instead of the full distribution name
+ dist_id, version, codename = distro.linux_distribution(
+ full_distribution_name=False
+ )
+
+ if dist_id in FEDORA_DISTROS:
+ cls = CentOSFedoraBootstrapper
+ args["distro"] = dist_id
+ elif dist_id in DEBIAN_DISTROS:
+ cls = DebianBootstrapper
+ args["distro"] = dist_id
+ args["codename"] = codename
+ elif dist_id in ("gentoo", "funtoo"):
+ cls = GentooBootstrapper
+ elif dist_id in ("solus"):
+ cls = SolusBootstrapper
+ elif dist_id in ("arch") or Path("/etc/arch-release").exists():
+ cls = ArchlinuxBootstrapper
+ elif dist_id in ("void"):
+ cls = VoidBootstrapper
+ elif dist_id in (
+ "opensuse",
+ "opensuse-leap",
+ "opensuse-tumbleweed",
+ "suse",
+ ):
+ cls = OpenSUSEBootstrapper
+ else:
+ raise NotImplementedError(
+ "Bootstrap support for this Linux "
+ "distro not yet available: " + dist_id
+ )
+
+ args["version"] = version
+ args["dist_id"] = dist_id
+
+ elif sys.platform.startswith("darwin"):
+ # TODO Support Darwin platforms that aren't OS X.
+ osx_version = platform.mac_ver()[0]
+ if platform.machine() == "arm64" or _macos_is_running_under_rosetta():
+ cls = OSXBootstrapperLight
+ else:
+ cls = OSXBootstrapper
+ args["version"] = osx_version
+
+ elif sys.platform.startswith("openbsd"):
+ cls = OpenBSDBootstrapper
+ args["version"] = platform.uname()[2]
+
+ elif sys.platform.startswith(("dragonfly", "freebsd", "netbsd")):
+ cls = FreeBSDBootstrapper
+ args["version"] = platform.release()
+ args["flavor"] = platform.system()
+
+ elif sys.platform.startswith("win32") or sys.platform.startswith("msys"):
+ if "MOZILLABUILD" in os.environ:
+ cls = MozillaBuildBootstrapper
+ else:
+ cls = WindowsBootstrapper
+ if cls is None:
+ raise NotImplementedError(
+ "Bootstrap support is not yet available " "for your OS."
+ )
+
+ self.instance = cls(**args)
+
+ def maybe_install_private_packages_or_exit(self, application):
+ # Install the clang packages needed for building the style system, as
+ # well as the version of NodeJS that we currently support.
+ # Also install the clang static-analysis package by default
+ # The best place to install our packages is in the state directory
+ # we have. We should have created one above in non-interactive mode.
+ self.instance.auto_bootstrap(application)
+ self.instance.install_toolchain_artifact("fix-stacks")
+ self.instance.install_toolchain_artifact("minidump-stackwalk")
+ if not self.instance.artifact_mode:
+ self.instance.ensure_clang_static_analysis_package()
+ self.instance.ensure_sccache_packages()
+ # Like 'ensure_browser_packages' or 'ensure_mobile_android_packages'
+ getattr(self.instance, "ensure_%s_packages" % application)()
+
+ def check_code_submission(self, checkout_root: Path):
+ if self.instance.no_interactive or which("moz-phab"):
+ return
+
+ # Skip moz-phab install until bug 1696357 is fixed and makes it to a moz-phab
+ # release.
+ if sys.platform.startswith("darwin") and platform.machine() == "arm64":
+ return
+
+ if not self.instance.prompt_yesno("Will you be submitting commits to Mozilla?"):
+ return
+
+ mach_binary = checkout_root / "mach"
+ subprocess.check_call((sys.executable, str(mach_binary), "install-moz-phab"))
+
+ def bootstrap(self, settings):
+ if self.choice is None:
+ applications = APPLICATIONS
+ # Like ['1. Firefox for Desktop', '2. Firefox for Android Artifact Mode', ...].
+ labels = [
+ "%s. %s" % (i, name) for i, name in enumerate(applications.keys(), 1)
+ ]
+ choices = [" {} [default]".format(labels[0])]
+ choices += [" {}".format(label) for label in labels[1:]]
+ prompt = APPLICATION_CHOICE % "\n".join(choices)
+ prompt_choice = self.instance.prompt_int(
+ prompt=prompt, low=1, high=len(applications)
+ )
+ name, application = list(applications.items())[prompt_choice - 1]
+ elif self.choice in APPLICATIONS.keys():
+ name, application = self.choice, APPLICATIONS[self.choice]
+ elif self.choice in APPLICATIONS.values():
+ name, application = next(
+ (k, v) for k, v in APPLICATIONS.items() if v == self.choice
+ )
+ else:
+ raise Exception(
+ "Please pick a valid application choice: (%s)"
+ % "/".join(APPLICATIONS.keys())
+ )
+
+ mozconfig_builder = MozconfigBuilder()
+ self.instance.application = application
+ self.instance.artifact_mode = "artifact_mode" in application
+
+ self.instance.warn_if_pythonpath_is_set()
+
+ if sys.platform.startswith("darwin") and not os.environ.get(
+ "MACH_I_DO_WANT_TO_USE_ROSETTA"
+ ):
+ # If running on arm64 mac, check whether we're running under
+ # Rosetta and advise against it.
+ if _macos_is_running_under_rosetta():
+ print(
+ "Python is being emulated under Rosetta. Please use a native "
+ "Python instead. If you still really want to go ahead, set "
+ "the MACH_I_DO_WANT_TO_USE_ROSETTA environment variable.",
+ file=sys.stderr,
+ )
+ return 1
+
+ state_dir = Path(get_state_dir())
+ self.instance.state_dir = state_dir
+
+ hg = to_optional_path(which("hg"))
+
+ # We need to enable the loading of hgrc in case extensions are
+ # required to open the repo.
+ (checkout_type, checkout_root) = current_firefox_checkout(
+ env=self.instance._hg_cleanenv(load_hgrc=True),
+ hg=hg,
+ )
+ self.instance.srcdir = checkout_root
+ self.instance.validate_environment()
+ self._validate_python_environment(checkout_root)
+
+ if self.instance.no_system_changes:
+ self.maybe_install_private_packages_or_exit(application)
+ self._output_mozconfig(application, mozconfig_builder)
+ sys.exit(0)
+
+ self.instance.install_system_packages()
+
+ # Like 'install_browser_packages' or 'install_mobile_android_packages'.
+ getattr(self.instance, "install_%s_packages" % application)(mozconfig_builder)
+
+ if not self.instance.artifact_mode:
+ self.instance.ensure_rust_modern()
+
+ git = to_optional_path(which("git"))
+
+ # Possibly configure Mercurial, but not if the current checkout or repo
+ # type is Git.
+ hg_installed = bool(hg)
+ if checkout_type == "hg":
+ hg_installed, hg_modern = self.instance.ensure_mercurial_modern()
+
+ if hg_installed and checkout_type == "hg":
+ if not self.instance.no_interactive:
+ configure_hg = self.instance.prompt_yesno(prompt=CONFIGURE_MERCURIAL)
+ else:
+ configure_hg = self.hg_configure
+
+ if configure_hg:
+ configure_mercurial(hg, state_dir)
+
+ # Offer to configure Git, if the current checkout or repo type is Git.
+ elif git and checkout_type == "git":
+ should_configure_git = False
+ if not self.instance.no_interactive:
+ should_configure_git = self.instance.prompt_yesno(prompt=CONFIGURE_GIT)
+ else:
+ # Assuming default configuration setting applies to all VCS.
+ should_configure_git = self.hg_configure
+
+ if should_configure_git:
+ configure_git(
+ git,
+ to_optional_path(which("git-cinnabar")),
+ state_dir,
+ checkout_root,
+ )
+
+ self.maybe_install_private_packages_or_exit(application)
+ self.check_code_submission(checkout_root)
+ # Wait until after moz-phab setup to check telemetry so that employees
+ # will be automatically opted-in.
+ if not self.instance.no_interactive and not settings.mach_telemetry.is_set_up:
+ initialize_telemetry_setting(settings, str(checkout_root), str(state_dir))
+
+ self._output_mozconfig(application, mozconfig_builder)
+
+ print(FINISHED % name)
+ if not (
+ which("rustc")
+ and self.instance._parse_version(Path("rustc")) >= MODERN_RUST_VERSION
+ ):
+ print(
+ "To build %s, please restart the shell (Start a new terminal window)"
+ % name
+ )
+
+ def _default_mozconfig_path(self):
+ return Path(self.mach_context.topdir) / "mozconfig"
+
+ def _read_default_mozconfig(self):
+ path = self._default_mozconfig_path()
+ with open(path, "r") as mozconfig_file:
+ return mozconfig_file.read()
+
+ def _write_default_mozconfig(self, raw_mozconfig):
+ path = self._default_mozconfig_path()
+ with open(path, "w") as mozconfig_file:
+ mozconfig_file.write(raw_mozconfig)
+ print(f'Your requested configuration has been written to "{path}".')
+
+ def _show_mozconfig_suggestion(self, raw_mozconfig):
+ suggestion = MOZCONFIG_SUGGESTION_TEMPLATE % (
+ self._default_mozconfig_path(),
+ raw_mozconfig,
+ )
+ print(suggestion, end="")
+
+ def _check_default_mozconfig_mismatch(
+ self, current_mozconfig_info, expected_application, expected_raw_mozconfig
+ ):
+ current_raw_mozconfig = self._read_default_mozconfig()
+ current_application = current_mozconfig_info["project"][0].replace("/", "_")
+ if current_mozconfig_info["artifact-builds"]:
+ current_application += "_artifact_mode"
+
+ if expected_application == current_application:
+ if expected_raw_mozconfig == current_raw_mozconfig:
+ return
+
+ # There's minor difference, show the suggestion.
+ self._show_mozconfig_suggestion(expected_raw_mozconfig)
+ return
+
+ warning = MOZCONFIG_MISMATCH_WARNING_TEMPLATE % (
+ self._default_mozconfig_path(),
+ current_raw_mozconfig,
+ expected_raw_mozconfig,
+ )
+ print(warning)
+
+ if not self.instance.prompt_yesno("Do you want to overwrite the config?"):
+ return
+
+ self._write_default_mozconfig(expected_raw_mozconfig)
+
+ def _output_mozconfig(self, application, mozconfig_builder):
+ # Like 'generate_browser_mozconfig' or 'generate_mobile_android_mozconfig'.
+ additional_mozconfig = getattr(
+ self.instance, "generate_%s_mozconfig" % application
+ )()
+ if additional_mozconfig:
+ mozconfig_builder.append(additional_mozconfig)
+ raw_mozconfig = mozconfig_builder.generate()
+
+ current_mozconfig_info = MozbuildObject.get_base_mozconfig_info(
+ self.mach_context.topdir, None, ""
+ )
+ current_mozconfig_path = current_mozconfig_info["mozconfig"]["path"]
+
+ if current_mozconfig_path:
+ # mozconfig file exists
+ if self._default_mozconfig_path().exists() and Path.samefile(
+ Path(current_mozconfig_path), self._default_mozconfig_path()
+ ):
+ # This mozconfig file may be created by bootstrap.
+ self._check_default_mozconfig_mismatch(
+ current_mozconfig_info, application, raw_mozconfig
+ )
+ elif raw_mozconfig:
+ # The mozconfig file is created by user.
+ self._show_mozconfig_suggestion(raw_mozconfig)
+ elif raw_mozconfig:
+ # No mozconfig file exists yet
+ self._write_default_mozconfig(raw_mozconfig)
+
+ def _validate_python_environment(self, topsrcdir):
+ valid = True
+ try:
+ # distutils is singled out here because some distros (namely Ubuntu)
+ # include it in a separate package outside of the main Python
+ # installation.
+ import distutils.spawn
+ import distutils.sysconfig
+
+ assert distutils.sysconfig is not None and distutils.spawn is not None
+ except ImportError as e:
+ print("ERROR: Could not import package %s" % e.name, file=sys.stderr)
+ self.instance.suggest_install_distutils()
+ valid = False
+ except AssertionError:
+ print("ERROR: distutils is not behaving as expected.", file=sys.stderr)
+ self.instance.suggest_install_distutils()
+ valid = False
+ pip3 = to_optional_path(which("pip3"))
+ if not pip3:
+ print("ERROR: Could not find pip3.", file=sys.stderr)
+ self.instance.suggest_install_pip3()
+ valid = False
+ if not valid:
+ print(
+ "ERROR: Your Python installation will not be able to run "
+ "`mach bootstrap`. `mach bootstrap` cannot maintain your "
+ "Python environment for you; fix the errors shown here, and "
+ "then re-run `mach bootstrap`.",
+ file=sys.stderr,
+ )
+ sys.exit(1)
+
+ mach_site = MachSiteManager.from_environment(
+ topsrcdir,
+ lambda: os.path.normpath(get_state_dir(True, topsrcdir=topsrcdir)),
+ )
+ mach_site.attempt_populate_optional_packages()
+
+
+def update_vct(hg: Path, root_state_dir: Path):
+ """Ensure version-control-tools in the state directory is up to date."""
+ vct_dir = root_state_dir / "version-control-tools"
+
+ # Ensure the latest revision of version-control-tools is present.
+ update_mercurial_repo(
+ hg, "https://hg.mozilla.org/hgcustom/version-control-tools", vct_dir, "@"
+ )
+
+ return vct_dir
+
+
+def configure_mercurial(hg: Optional[Path], root_state_dir: Path):
+ """Run the Mercurial configuration wizard."""
+ vct_dir = update_vct(hg, root_state_dir)
+
+ hg = to_optional_str(hg)
+
+ # Run the config wizard from v-c-t.
+ args = [
+ hg,
+ "--config",
+ f"extensions.configwizard={vct_dir}/hgext/configwizard",
+ "configwizard",
+ ]
+ subprocess.call(args)
+
+
+def update_mercurial_repo(hg: Path, url, dest: Path, revision):
+ """Perform a clone/pull + update of a Mercurial repository."""
+ # Disable common extensions whose older versions may cause `hg`
+ # invocations to abort.
+ pull_args = [str(hg)]
+ if dest.exists():
+ pull_args.extend(["pull", url])
+ cwd = dest
+ else:
+ pull_args.extend(["clone", "--noupdate", url, str(dest)])
+ cwd = "/"
+
+ update_args = [str(hg), "update", "-r", revision]
+
+ print("=" * 80)
+ print(f"Ensuring {url} is up to date at {dest}")
+
+ env = os.environ.copy()
+ env.update({"HGPLAIN": "1"})
+
+ try:
+ subprocess.check_call(pull_args, cwd=str(cwd), env=env)
+ subprocess.check_call(update_args, cwd=str(dest), env=env)
+ finally:
+ print("=" * 80)
+
+
+def current_firefox_checkout(env, hg: Optional[Path] = None):
+ """Determine whether we're in a Firefox checkout.
+
+ Returns one of None, ``git``, or ``hg``.
+ """
+ HG_ROOT_REVISIONS = set(
+ [
+ # From mozilla-unified.
+ "8ba995b74e18334ab3707f27e9eb8f4e37ba3d29"
+ ]
+ )
+
+ path = Path.cwd()
+ while path:
+ hg_dir = path / ".hg"
+ git_dir = path / ".git"
+ if hg and hg_dir.exists():
+ # Verify the hg repo is a Firefox repo by looking at rev 0.
+ try:
+ node = subprocess.check_output(
+ [str(hg), "log", "-r", "0", "--template", "{node}"],
+ cwd=str(path),
+ env=env,
+ universal_newlines=True,
+ )
+ if node in HG_ROOT_REVISIONS:
+ _warn_if_risky_revision(path)
+ return "hg", path
+ # Else the root revision is different. There could be nested
+ # repos. So keep traversing the parents.
+ except subprocess.CalledProcessError:
+ pass
+
+ # Just check for known-good files in the checkout, to prevent attempted
+ # foot-shootings. Determining a canonical git checkout of mozilla-unified
+ # is...complicated
+ elif git_dir.exists() or hg_dir.exists():
+ moz_configure = path / "moz.configure"
+ if moz_configure.exists():
+ _warn_if_risky_revision(path)
+ return ("git" if git_dir.exists() else "hg"), path
+
+ if not len(path.parents):
+ break
+ path = path.parent
+
+ raise UserError(
+ "Could not identify the root directory of your checkout! "
+ "Are you running `mach bootstrap` in an hg or git clone?"
+ )
+
+
+def update_git_tools(git: Optional[Path], root_state_dir: Path):
+ """Update git tools, hooks and extensions"""
+ # Ensure git-cinnabar is up to date.
+ cinnabar_dir = root_state_dir / "git-cinnabar"
+ cinnabar_exe = cinnabar_dir / "git-cinnabar"
+
+ if sys.platform.startswith(("win32", "msys")):
+ cinnabar_exe = cinnabar_exe.with_suffix(".exe")
+
+ # Previously, this script would do a full clone of the git-cinnabar
+ # repository. It now only downloads prebuilt binaries, so if we are
+ # updating from an old setup, remove the repository and start over.
+ if (cinnabar_dir / ".git").exists():
+ # git sets pack files read-only, which causes problems removing
+ # them on Windows. To work around that, we use an error handler
+ # on rmtree that retries to remove the file after chmod'ing it.
+ def onerror(func, path, exc):
+ if func == os.unlink:
+ os.chmod(path, stat.S_IRWXU)
+ func(path)
+ else:
+ raise
+
+ shutil.rmtree(str(cinnabar_dir), onerror=onerror)
+
+ # If we already have an executable, ask it to update itself.
+ exists = cinnabar_exe.exists()
+ if exists:
+ try:
+ subprocess.check_call([str(cinnabar_exe), "self-update"])
+ except subprocess.CalledProcessError as e:
+ print(e)
+
+ # git-cinnabar 0.6.0rc1 self-update had a bug that could leave an empty
+ # file. If that happens, install from scratch.
+ if not exists or cinnabar_exe.stat().st_size == 0:
+ from urllib.request import urlopen
+
+ if not cinnabar_dir.exists():
+ cinnabar_dir.mkdir()
+
+ cinnabar_url = "https://github.com/glandium/git-cinnabar/"
+ download_py = cinnabar_dir / "download.py"
+ with open(download_py, "wb") as fh:
+ shutil.copyfileobj(urlopen(f"{cinnabar_url}/raw/master/download.py"), fh)
+
+ try:
+ subprocess.check_call(
+ [sys.executable, str(download_py)], cwd=str(cinnabar_dir)
+ )
+ except subprocess.CalledProcessError as e:
+ print(e)
+ finally:
+ download_py.unlink()
+
+ return cinnabar_dir
+
+
+def configure_git(
+ git: Optional[Path],
+ cinnabar: Optional[Path],
+ root_state_dir: Path,
+ top_src_dir: Path,
+):
+ """Run the Git configuration steps."""
+
+ git_str = to_optional_str(git)
+
+ match = re.search(
+ r"(\d+\.\d+\.\d+)",
+ subprocess.check_output([git_str, "--version"], universal_newlines=True),
+ )
+ if not match:
+ raise Exception("Could not find git version")
+ git_version = Version(match.group(1))
+
+ if git_version < MINIMUM_RECOMMENDED_GIT_VERSION:
+ print(
+ OLD_GIT_WARNING.format(
+ old_version=git_version,
+ minimum_recommended_version=MINIMUM_RECOMMENDED_GIT_VERSION,
+ )
+ )
+
+ if git_version >= Version("2.17"):
+ # "core.untrackedCache" has a bug before 2.17
+ subprocess.check_call(
+ [git_str, "config", "core.untrackedCache", "true"], cwd=str(top_src_dir)
+ )
+
+ cinnabar_dir = str(update_git_tools(git, root_state_dir))
+
+ if not cinnabar:
+ if "MOZILLABUILD" in os.environ:
+ # Slightly modify the path on Windows to be correct
+ # for the copy/paste into the .bash_profile
+ cinnabar_dir = win_to_msys_path(cinnabar_dir)
+
+ print(
+ ADD_GIT_CINNABAR_PATH.format(
+ prefix="%USERPROFILE%", cinnabar_dir=cinnabar_dir
+ )
+ )
+ else:
+ print(ADD_GIT_CINNABAR_PATH.format(prefix="~", cinnabar_dir=cinnabar_dir))
+
+
+def _warn_if_risky_revision(path: Path):
+ # Warn the user if they're trying to bootstrap from an obviously old
+ # version of tree as reported by the version control system (a month in
+ # this case). This is an approximate calculation but is probably good
+ # enough for our purposes.
+ NUM_SECONDS_IN_MONTH = 60 * 60 * 24 * 30
+ from mozversioncontrol import get_repository_object
+
+ repo = get_repository_object(path)
+ if (time.time() - repo.get_commit_time()) >= NUM_SECONDS_IN_MONTH:
+ print(OLD_REVISION_WARNING)
+
+
+def _macos_is_running_under_rosetta():
+ proc = subprocess.run(
+ ["sysctl", "-n", "sysctl.proc_translated"],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL,
+ )
+ return (
+ proc.returncode == 0 and proc.stdout.decode("ascii", "replace").strip() == "1"
+ )
diff --git a/python/mozboot/mozboot/centosfedora.py b/python/mozboot/mozboot/centosfedora.py
new file mode 100644
index 0000000000..37aa0e8eaa
--- /dev/null
+++ b/python/mozboot/mozboot/centosfedora.py
@@ -0,0 +1,80 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import subprocess
+
+from mozfile import which
+
+from mozboot.base import BaseBootstrapper
+from mozboot.linux_common import LinuxBootstrapper
+
+
+class CentOSFedoraBootstrapper(LinuxBootstrapper, BaseBootstrapper):
+ def __init__(self, distro, version, dist_id, **kwargs):
+ BaseBootstrapper.__init__(self, **kwargs)
+
+ self.distro = distro
+ self.version = int(version.split(".")[0])
+ self.dist_id = dist_id
+
+ def install_packages(self, packages):
+ if self.version >= 33 and "perl" in packages:
+ packages.append("perl-FindBin")
+ # watchman is not available on centos/rocky
+ if self.distro in ("centos", "rocky", "oracle"):
+ packages = [p for p in packages if p != "watchman"]
+ self.dnf_install(*packages)
+
+ def upgrade_mercurial(self, current):
+ if current is None:
+ self.dnf_install("mercurial")
+ else:
+ self.dnf_update("mercurial")
+
+ def dnf_install(self, *packages):
+ if which("dnf"):
+
+ def not_installed(package):
+ # We could check for "Error: No matching Packages to list", but
+ # checking `dnf`s exit code is sufficent.
+ # Ideally we'd invoke dnf with '--cacheonly', but there's:
+ # https://bugzilla.redhat.com/show_bug.cgi?id=2030255
+ is_installed = subprocess.run(
+ ["dnf", "list", "--installed", package],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ )
+ if is_installed.returncode not in [0, 1]:
+ stdout = is_installed.stdout
+ raise Exception(
+ f'Failed to determine whether package "{package}" is installed: "{stdout}"'
+ )
+ return is_installed.returncode != 0
+
+ packages = list(filter(not_installed, packages))
+ if len(packages) == 0:
+ # avoid sudo prompt (support unattended re-bootstrapping)
+ return
+
+ command = ["dnf", "install"]
+ else:
+ command = ["yum", "install"]
+
+ if self.no_interactive:
+ command.append("-y")
+ command.extend(packages)
+
+ self.run_as_root(command)
+
+ def dnf_update(self, *packages):
+ if which("dnf"):
+ command = ["dnf", "update"]
+ else:
+ command = ["yum", "update"]
+
+ if self.no_interactive:
+ command.append("-y")
+ command.extend(packages)
+
+ self.run_as_root(command)
diff --git a/python/mozboot/mozboot/debian.py b/python/mozboot/mozboot/debian.py
new file mode 100644
index 0000000000..34e328586e
--- /dev/null
+++ b/python/mozboot/mozboot/debian.py
@@ -0,0 +1,83 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+
+from mozboot.base import MERCURIAL_INSTALL_PROMPT, BaseBootstrapper
+from mozboot.linux_common import LinuxBootstrapper
+
+
+class DebianBootstrapper(LinuxBootstrapper, BaseBootstrapper):
+ def __init__(self, distro, version, dist_id, codename, **kwargs):
+ BaseBootstrapper.__init__(self, **kwargs)
+
+ self.distro = distro
+ self.version = version
+ self.dist_id = dist_id
+ self.codename = codename
+
+ def suggest_install_distutils(self):
+ print(
+ "HINT: Try installing distutils with "
+ "`apt-get install python3-distutils`.",
+ file=sys.stderr,
+ )
+
+ def suggest_install_pip3(self):
+ print(
+ "HINT: Try installing pip3 with `apt-get install python3-pip`.",
+ file=sys.stderr,
+ )
+
+ def install_packages(self, packages):
+ try:
+ if int(self.version) < 11:
+ # watchman is only available starting from Debian 11.
+ packages = [p for p in packages if p != "watchman"]
+ except ValueError:
+ pass
+
+ self.apt_install(*packages)
+
+ def _update_package_manager(self):
+ self.apt_update()
+
+ def upgrade_mercurial(self, current):
+ """Install Mercurial from pip because Debian packages typically lag."""
+ if self.no_interactive:
+ # Install via Apt in non-interactive mode because it is the more
+ # conservative option and less likely to make people upset.
+ self.apt_install("mercurial")
+ return
+
+ res = self.prompt_int(MERCURIAL_INSTALL_PROMPT, 1, 3)
+
+ # Apt.
+ if res == 2:
+ self.apt_install("mercurial")
+ return False
+
+ # No Mercurial.
+ if res == 3:
+ print("Not installing Mercurial.")
+ return False
+
+ # pip.
+ assert res == 1
+ self.run_as_root(["pip3", "install", "--upgrade", "Mercurial"])
+
+ def apt_install(self, *packages):
+ command = ["apt-get", "install"]
+ if self.no_interactive:
+ command.append("-y")
+ command.extend(packages)
+
+ self.run_as_root(command)
+
+ def apt_update(self):
+ command = ["apt-get", "update"]
+ if self.no_interactive:
+ command.append("-y")
+
+ self.run_as_root(command)
diff --git a/python/mozboot/mozboot/freebsd.py b/python/mozboot/mozboot/freebsd.py
new file mode 100644
index 0000000000..aac689b8a3
--- /dev/null
+++ b/python/mozboot/mozboot/freebsd.py
@@ -0,0 +1,74 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+
+from mozfile import which
+
+from mozboot.base import BaseBootstrapper
+
+
+class FreeBSDBootstrapper(BaseBootstrapper):
+ def __init__(self, version, flavor, **kwargs):
+ BaseBootstrapper.__init__(self, **kwargs)
+ self.version = int(version.split(".")[0])
+ self.flavor = flavor.lower()
+
+ self.packages = [
+ "gmake",
+ "gtar",
+ "m4",
+ "npm",
+ "pkgconf",
+ "py%d%d-sqlite3" % sys.version_info[0:2],
+ "rust",
+ "watchman",
+ ]
+
+ self.browser_packages = [
+ "dbus-glib",
+ "libXt",
+ "nasm",
+ "pulseaudio",
+ ]
+
+ if not which("as"):
+ self.packages.append("binutils")
+
+ if not which("unzip"):
+ self.packages.append("unzip")
+
+ def pkg_install(self, *packages):
+ if sys.platform.startswith("netbsd"):
+ command = ["pkgin", "install"]
+ else:
+ command = ["pkg", "install"]
+ if self.no_interactive:
+ command.append("-y")
+
+ command.extend(packages)
+ self.run_as_root(command)
+
+ def install_system_packages(self):
+ self.pkg_install(*self.packages)
+
+ def install_browser_packages(self, mozconfig_builder, artifact_mode=False):
+ # TODO: Figure out what not to install for artifact mode
+ packages = self.browser_packages.copy()
+ if not artifact_mode:
+ if sys.platform.startswith("netbsd"):
+ packages.extend(["brotli", "gtk3+", "libv4l", "cbindgen"])
+ else:
+ packages.extend(["gtk3", "mesa-dri", "v4l_compat", "rust-cbindgen"])
+ self.pkg_install(*packages)
+
+ def install_browser_artifact_mode_packages(self, mozconfig_builder):
+ self.install_browser_packages(mozconfig_builder, artifact_mode=True)
+
+ def ensure_clang_static_analysis_package(self):
+ # TODO: we don't ship clang base static analysis for this platform
+ pass
+
+ def upgrade_mercurial(self, current):
+ self.pkg_install("mercurial")
diff --git a/python/mozboot/mozboot/gentoo.py b/python/mozboot/mozboot/gentoo.py
new file mode 100644
index 0000000000..4ddf86696f
--- /dev/null
+++ b/python/mozboot/mozboot/gentoo.py
@@ -0,0 +1,29 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mozboot.base import BaseBootstrapper
+from mozboot.linux_common import LinuxBootstrapper
+
+
+class GentooBootstrapper(LinuxBootstrapper, BaseBootstrapper):
+ def __init__(self, version, dist_id, **kwargs):
+ BaseBootstrapper.__init__(self, **kwargs)
+
+ self.version = version
+ self.dist_id = dist_id
+
+ def install_packages(self, packages):
+ DISAMBIGUATE = {
+ "gzip": "app-arch/gzip",
+ "tar": "app-arch/tar",
+ }
+ # watchman is available but requires messing with USEs.
+ packages = [DISAMBIGUATE.get(p, p) for p in packages if p != "watchman"]
+ self.run_as_root(["emerge", "--noreplace"] + packages)
+
+ def _update_package_manager(self):
+ self.run_as_root(["emerge", "--sync"])
+
+ def upgrade_mercurial(self, current):
+ self.run_as_root(["emerge", "--update", "mercurial"])
diff --git a/python/mozboot/mozboot/linux_common.py b/python/mozboot/mozboot/linux_common.py
new file mode 100644
index 0000000000..456745b56f
--- /dev/null
+++ b/python/mozboot/mozboot/linux_common.py
@@ -0,0 +1,113 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# An easy way for distribution-specific bootstrappers to share the code
+# needed to install Stylo and Node dependencies. This class must come before
+# BaseBootstrapper in the inheritance list.
+
+import platform
+
+
+def is_non_x86_64():
+ return platform.machine() != "x86_64"
+
+
+class ClangStaticAnalysisInstall(object):
+ def __init__(self, **kwargs):
+ pass
+
+ def ensure_clang_static_analysis_package(self):
+ if is_non_x86_64():
+ print(
+ "Cannot install static analysis tools from taskcluster.\n"
+ "Please install these tools manually."
+ )
+ return
+
+ from mozboot import static_analysis
+
+ self.install_toolchain_static_analysis(static_analysis.LINUX_CLANG_TIDY)
+
+
+class MobileAndroidBootstrapper(object):
+ def __init__(self, **kwargs):
+ pass
+
+ def install_mobile_android_packages(self, mozconfig_builder, artifact_mode=False):
+ from mozboot import android
+
+ os_arch = platform.machine()
+ android.ensure_android(
+ "linux",
+ os_arch,
+ artifact_mode=artifact_mode,
+ no_interactive=self.no_interactive,
+ )
+ android.ensure_android(
+ "linux",
+ os_arch,
+ artifact_mode=artifact_mode,
+ no_interactive=self.no_interactive,
+ system_images_only=True,
+ avd_manifest_path=android.AVD_MANIFEST_X86_64,
+ )
+ android.ensure_android(
+ "linux",
+ os_arch,
+ artifact_mode=artifact_mode,
+ no_interactive=self.no_interactive,
+ system_images_only=True,
+ avd_manifest_path=android.AVD_MANIFEST_ARM,
+ )
+
+ def install_mobile_android_artifact_mode_packages(self, mozconfig_builder):
+ self.install_mobile_android_packages(mozconfig_builder, artifact_mode=True)
+
+ def ensure_mobile_android_packages(self):
+ from mozboot import android
+
+ android.ensure_java("linux", platform.machine())
+ self.install_toolchain_artifact(android.LINUX_X86_64_ANDROID_AVD)
+ self.install_toolchain_artifact(android.LINUX_ARM_ANDROID_AVD)
+
+ def generate_mobile_android_mozconfig(self, artifact_mode=False):
+ from mozboot import android
+
+ return android.generate_mozconfig("linux", artifact_mode=artifact_mode)
+
+ def generate_mobile_android_artifact_mode_mozconfig(self):
+ return self.generate_mobile_android_mozconfig(artifact_mode=True)
+
+
+class LinuxBootstrapper(
+ ClangStaticAnalysisInstall,
+ MobileAndroidBootstrapper,
+):
+ def __init__(self, **kwargs):
+ pass
+
+ def ensure_sccache_packages(self):
+ pass
+
+ def install_system_packages(self):
+ self.install_packages(
+ [
+ "bash",
+ "findutils", # contains xargs
+ "gzip",
+ "libxml2", # used by bootstrapped clang
+ "m4",
+ "make",
+ "perl",
+ "tar",
+ "unzip",
+ "watchman",
+ ]
+ )
+
+ def install_browser_packages(self, mozconfig_builder, artifact_mode=False):
+ pass
+
+ def install_browser_artifact_mode_packages(self, mozconfig_builder):
+ pass
diff --git a/python/mozboot/mozboot/mach_commands.py b/python/mozboot/mozboot/mach_commands.py
new file mode 100644
index 0000000000..26aa6123d2
--- /dev/null
+++ b/python/mozboot/mozboot/mach_commands.py
@@ -0,0 +1,111 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this,
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import errno
+import sys
+from pathlib import Path
+
+from mach.decorators import Command, CommandArgument
+
+from mozboot.bootstrap import APPLICATIONS
+
+
+@Command(
+ "bootstrap",
+ category="devenv",
+ description="Install required system packages for building.",
+)
+@CommandArgument(
+ "--application-choice",
+ choices=list(APPLICATIONS.keys()) + list(APPLICATIONS.values()),
+ default=None,
+ help="Pass in an application choice instead of using the default "
+ "interactive prompt.",
+)
+@CommandArgument(
+ "--no-system-changes",
+ dest="no_system_changes",
+ action="store_true",
+ help="Only execute actions that leave the system configuration alone.",
+)
+def bootstrap(command_context, application_choice=None, no_system_changes=False):
+ """Bootstrap system and mach for optimal development experience."""
+ from mozboot.bootstrap import Bootstrapper
+
+ bootstrapper = Bootstrapper(
+ choice=application_choice,
+ no_interactive=not command_context._mach_context.is_interactive,
+ no_system_changes=no_system_changes,
+ mach_context=command_context._mach_context,
+ )
+ bootstrapper.bootstrap(command_context.settings)
+
+
+@Command(
+ "vcs-setup",
+ category="devenv",
+ description="Help configure a VCS for optimal development.",
+)
+@CommandArgument(
+ "-u",
+ "--update-only",
+ action="store_true",
+ help="Only update recommended extensions, don't run the wizard.",
+)
+def vcs_setup(command_context, update_only=False):
+ """Ensure a Version Control System (Mercurial or Git) is optimally
+ configured.
+
+ This command will inspect your VCS configuration and
+ guide you through an interactive wizard helping you configure the
+ VCS for optimal use on Mozilla projects.
+
+ User choice is respected: no changes are made without explicit
+ confirmation from you.
+
+ If "--update-only" is used, the interactive wizard is disabled
+ and this command only ensures that remote repositories providing
+ VCS extensions are up to date.
+ """
+ import mozversioncontrol
+ from mach.util import to_optional_path
+ from mozfile import which
+
+ import mozboot.bootstrap as bootstrap
+
+ repo = mozversioncontrol.get_repository_object(command_context._mach_context.topdir)
+ tool = "hg"
+ if repo.name == "git":
+ tool = "git"
+
+ # "hg" is an executable script with a shebang, which will be found by
+ # which. We need to pass a win32 executable to the function because we
+ # spawn a process from it.
+ if sys.platform in ("win32", "msys"):
+ tool += ".exe"
+
+ vcs = to_optional_path(which(tool))
+ if not vcs:
+ raise OSError(errno.ENOENT, "Could not find {} on $PATH".format(tool))
+
+ if update_only:
+ if repo.name == "git":
+ bootstrap.update_git_tools(
+ vcs,
+ Path(command_context._mach_context.state_dir),
+ )
+ else:
+ bootstrap.update_vct(vcs, Path(command_context._mach_context.state_dir))
+ else:
+ if repo.name == "git":
+ bootstrap.configure_git(
+ vcs,
+ to_optional_path(which("git-cinnabar")),
+ Path(command_context._mach_context.state_dir),
+ Path(command_context._mach_context.topdir),
+ )
+ else:
+ bootstrap.configure_mercurial(
+ vcs, Path(command_context._mach_context.state_dir)
+ )
diff --git a/python/mozboot/mozboot/mozconfig.py b/python/mozboot/mozboot/mozconfig.py
new file mode 100644
index 0000000000..a1ae4c8523
--- /dev/null
+++ b/python/mozboot/mozboot/mozconfig.py
@@ -0,0 +1,156 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import filecmp
+import os
+from pathlib import Path
+from typing import Union
+
+MOZ_MYCONFIG_ERROR = """
+The MOZ_MYCONFIG environment variable to define the location of mozconfigs
+is deprecated. If you wish to define the mozconfig path via an environment
+variable, use MOZCONFIG instead.
+""".strip()
+
+MOZCONFIG_LEGACY_PATH_ERROR = """
+You currently have a mozconfig at %s. This implicit location is no longer
+supported. Please move it to %s/.mozconfig or set an explicit path
+via the $MOZCONFIG environment variable.
+""".strip()
+
+DEFAULT_TOPSRCDIR_PATHS = (".mozconfig", "mozconfig")
+DEPRECATED_TOPSRCDIR_PATHS = ("mozconfig.sh", "myconfig.sh")
+DEPRECATED_HOME_PATHS = (".mozconfig", ".mozconfig.sh", ".mozmyconfig.sh")
+
+
+class MozconfigFindException(Exception):
+ """Raised when a mozconfig location is not defined properly."""
+
+
+class MozconfigBuilder(object):
+ def __init__(self):
+ self._lines = []
+
+ def append(self, block):
+ self._lines.extend([line.strip() for line in block.split("\n") if line.strip()])
+
+ def generate(self):
+ return "".join(line + "\n" for line in self._lines)
+
+
+def find_mozconfig(topsrcdir: Union[str, Path], env=os.environ):
+ """Find the active mozconfig file for the current environment.
+
+ This emulates the logic in mozconfig-find.
+
+ 1) If ENV[MOZCONFIG] is set, use that
+ 2) If $TOPSRCDIR/mozconfig or $TOPSRCDIR/.mozconfig exists, use it.
+ 3) If both exist or if there are legacy locations detected, error out.
+
+ The absolute path to the found mozconfig will be returned on success.
+ None will be returned if no mozconfig could be found. A
+ MozconfigFindException will be raised if there is a bad state,
+ including conditions from #3 above.
+ """
+ topsrcdir = Path(topsrcdir)
+
+ # Check for legacy methods first.
+ if "MOZ_MYCONFIG" in env:
+ raise MozconfigFindException(MOZ_MYCONFIG_ERROR)
+
+ env_path = env.get("MOZCONFIG", None) or None
+
+ if env_path is not None:
+ env_path = Path(env_path)
+
+ if env_path is not None:
+ if not env_path.is_absolute():
+ potential_roots = [topsrcdir, Path.cwd()]
+ # Attempt to eliminate duplicates for e.g.
+ # self.topsrcdir == Path.cwd().
+ potential_roots_strings = set(str(p.resolve()) for p in potential_roots)
+ existing = [
+ root
+ for root in potential_roots_strings
+ if (Path(root) / env_path).exists()
+ ]
+ if len(existing) > 1:
+ # There are multiple files, but we might have a setup like:
+ #
+ # somedirectory/
+ # srcdir/
+ # objdir/
+ #
+ # MOZCONFIG=../srcdir/some/path/to/mozconfig
+ #
+ # and be configuring from the objdir. So even though we
+ # have multiple existing files, they are actually the same
+ # file.
+ mozconfigs = [root / env_path for root in existing]
+ if not all(
+ map(
+ lambda p1, p2: filecmp.cmp(p1, p2, shallow=False),
+ mozconfigs[:-1],
+ mozconfigs[1:],
+ )
+ ):
+ raise MozconfigFindException(
+ "MOZCONFIG environment variable refers to a path that "
+ + "exists in more than one of "
+ + ", ".join(potential_roots_strings)
+ + ". Remove all but one."
+ )
+ elif not existing:
+ raise MozconfigFindException(
+ "MOZCONFIG environment variable refers to a path that "
+ + "does not exist in any of "
+ + ", ".join(potential_roots_strings)
+ )
+
+ env_path = existing[0] / env_path
+ elif not env_path.exists(): # non-relative path
+ raise MozconfigFindException(
+ "MOZCONFIG environment variable refers to a path that "
+ f"does not exist: {env_path}"
+ )
+
+ if not env_path.is_file():
+ raise MozconfigFindException(
+ "MOZCONFIG environment variable refers to a " f"non-file: {env_path}"
+ )
+
+ srcdir_paths = [topsrcdir / p for p in DEFAULT_TOPSRCDIR_PATHS]
+ existing = [p for p in srcdir_paths if p.is_file()]
+
+ if env_path is None and len(existing) > 1:
+ raise MozconfigFindException(
+ "Multiple default mozconfig files "
+ "present. Remove all but one. " + ", ".join(str(p) for p in existing)
+ )
+
+ path = None
+
+ if env_path is not None:
+ path = env_path
+ elif len(existing):
+ assert len(existing) == 1
+ path = existing[0]
+
+ if path is not None:
+ return Path.cwd() / path
+
+ deprecated_paths = [topsrcdir / s for s in DEPRECATED_TOPSRCDIR_PATHS]
+
+ home = env.get("HOME", None)
+ if home is not None:
+ home = Path(home)
+ deprecated_paths.extend([home / s for s in DEPRECATED_HOME_PATHS])
+
+ for path in deprecated_paths:
+ if path.exists():
+ raise MozconfigFindException(
+ MOZCONFIG_LEGACY_PATH_ERROR % (path, topsrcdir)
+ )
+
+ return None
diff --git a/python/mozboot/mozboot/mozillabuild.py b/python/mozboot/mozboot/mozillabuild.py
new file mode 100644
index 0000000000..a7379d9c53
--- /dev/null
+++ b/python/mozboot/mozboot/mozillabuild.py
@@ -0,0 +1,240 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import ctypes
+import os
+import platform
+import subprocess
+import sys
+from pathlib import Path
+
+from mozbuild.util import mozilla_build_version
+from packaging.version import Version
+
+from mozboot.base import BaseBootstrapper
+
+
+def is_aarch64_host():
+ from ctypes import wintypes
+
+ kernel32 = ctypes.windll.kernel32
+ IMAGE_FILE_MACHINE_UNKNOWN = 0
+ IMAGE_FILE_MACHINE_ARM64 = 0xAA64
+
+ try:
+ iswow64process2 = kernel32.IsWow64Process2
+ except Exception:
+ # If we can't access the symbol, we know we're not on aarch64.
+ return False
+
+ currentProcess = kernel32.GetCurrentProcess()
+ processMachine = wintypes.USHORT(IMAGE_FILE_MACHINE_UNKNOWN)
+ nativeMachine = wintypes.USHORT(IMAGE_FILE_MACHINE_UNKNOWN)
+
+ gotValue = iswow64process2(
+ currentProcess, ctypes.byref(processMachine), ctypes.byref(nativeMachine)
+ )
+ # If this call fails, we have no idea.
+ if not gotValue:
+ return False
+
+ return nativeMachine.value == IMAGE_FILE_MACHINE_ARM64
+
+
+def get_is_windefender_disabled():
+ import winreg
+
+ try:
+ with winreg.OpenKeyEx(
+ winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Microsoft\Windows Defender"
+ ) as windefender_key:
+ is_antivirus_disabled, _ = winreg.QueryValueEx(
+ windefender_key, "DisableAntiSpyware"
+ )
+ # is_antivirus_disabled is either 0 (False) or 1 (True)
+ return bool(is_antivirus_disabled)
+ except FileNotFoundError:
+ return True
+
+
+def get_windefender_exclusion_paths():
+ import winreg
+
+ paths = []
+ try:
+ with winreg.OpenKeyEx(
+ winreg.HKEY_LOCAL_MACHINE,
+ r"SOFTWARE\Microsoft\Windows Defender\Exclusions\Paths",
+ ) as exclusions_key:
+ _, values_count, __ = winreg.QueryInfoKey(exclusions_key)
+ for i in range(0, values_count):
+ path, _, __ = winreg.EnumValue(exclusions_key, i)
+ paths.append(Path(path))
+ except FileNotFoundError:
+ pass
+
+ return paths
+
+
+def is_windefender_affecting_srcdir(src_dir: Path):
+ if get_is_windefender_disabled():
+ return False
+
+ # When there's a match, but path cases aren't the same between srcdir and exclusion_path,
+ # commonpath will use the casing of the first path provided.
+ # To avoid surprises here, we normcase(...) so we don't get unexpected breakage if we change
+ # the path order.
+ src_dir = src_dir.resolve()
+
+ try:
+ exclusion_paths = get_windefender_exclusion_paths()
+ except OSError as e:
+ if e.winerror == 5:
+ # A version of Windows 10 released in 2021 raises an "Access is denied"
+ # error (ERROR_ACCESS_DENIED == 5) to un-elevated processes when they
+ # query Windows Defender's exclusions. Skip the exclusion path checking.
+ return
+ raise
+
+ for exclusion_path in exclusion_paths:
+ exclusion_path = exclusion_path.resolve()
+ try:
+ if Path(os.path.commonpath((exclusion_path, src_dir))) == exclusion_path:
+ # exclusion_path is an ancestor of srcdir
+ return False
+ except ValueError:
+ # ValueError: Paths don't have the same drive - can't be ours
+ pass
+ return True
+
+
+class MozillaBuildBootstrapper(BaseBootstrapper):
+ """Bootstrapper for MozillaBuild to install rustup."""
+
+ def __init__(self, no_interactive=False, no_system_changes=False):
+ BaseBootstrapper.__init__(
+ self, no_interactive=no_interactive, no_system_changes=no_system_changes
+ )
+
+ def validate_environment(self):
+ if self.application.startswith("mobile_android"):
+ print(
+ "WARNING!!! Building Firefox for Android on Windows is not "
+ "fully supported. See https://bugzilla.mozilla.org/show_bug."
+ "cgi?id=1169873 for details.",
+ file=sys.stderr,
+ )
+
+ if is_windefender_affecting_srcdir(self.srcdir):
+ print(
+ "Warning: the Firefox checkout directory is currently not in the "
+ "Windows Defender exclusion list. This can cause the build process "
+ "to be dramatically slowed or broken. To resolve this, follow the "
+ "directions here: "
+ "https://firefox-source-docs.mozilla.org/setup/windows_build.html"
+ "#antivirus-performance",
+ file=sys.stderr,
+ )
+
+ def install_system_packages(self):
+ pass
+
+ def upgrade_mercurial(self, current):
+ # Mercurial upstream sometimes doesn't upload wheels, and building
+ # from source requires MS Visual C++ 9.0. So we force pip to install
+ # the last version that comes with wheels.
+ if mozilla_build_version() >= Version("4.0"):
+ pip_dir = (
+ Path(os.environ["MOZILLABUILD"]) / "python3" / "Scripts" / "pip.exe"
+ )
+ else:
+ pip_dir = (
+ Path(os.environ["MOZILLABUILD"]) / "python" / "Scripts" / "pip.exe"
+ )
+
+ command = [
+ str(pip_dir),
+ "install",
+ "--upgrade",
+ "mercurial",
+ "--only-binary",
+ "mercurial",
+ ]
+ self.run(command)
+
+ def install_browser_packages(self, mozconfig_builder):
+ pass
+
+ def install_browser_artifact_mode_packages(self, mozconfig_builder):
+ pass
+
+ def _os_arch(self):
+ os_arch = platform.machine()
+ if os_arch == "AMD64":
+ # On Windows, x86_64 is reported as AMD64 but we use x86_64
+ # everywhere else, so let's normalized it here.
+ return "x86_64"
+ return os_arch
+
+ def install_mobile_android_packages(self, mozconfig_builder, artifact_mode=False):
+ from mozboot import android
+
+ os_arch = self._os_arch()
+ android.ensure_android(
+ "windows",
+ os_arch,
+ artifact_mode=artifact_mode,
+ no_interactive=self.no_interactive,
+ )
+ android.ensure_android(
+ "windows",
+ os_arch,
+ system_images_only=True,
+ artifact_mode=artifact_mode,
+ no_interactive=self.no_interactive,
+ avd_manifest_path=android.AVD_MANIFEST_X86_64,
+ )
+ android.ensure_android(
+ "windows",
+ os_arch,
+ system_images_only=True,
+ artifact_mode=artifact_mode,
+ no_interactive=self.no_interactive,
+ avd_manifest_path=android.AVD_MANIFEST_ARM,
+ )
+
+ def ensure_mobile_android_packages(self):
+ from mozboot import android
+
+ android.ensure_java("windows", self._os_arch())
+ self.install_toolchain_artifact(android.WINDOWS_X86_64_ANDROID_AVD)
+ self.install_toolchain_artifact(android.WINDOWS_ARM_ANDROID_AVD)
+
+ def install_mobile_android_artifact_mode_packages(self, mozconfig_builder):
+ self.install_mobile_android_packages(mozconfig_builder, artifact_mode=True)
+
+ def generate_mobile_android_mozconfig(self, artifact_mode=False):
+ from mozboot import android
+
+ return android.generate_mozconfig("windows", artifact_mode=artifact_mode)
+
+ def generate_mobile_android_artifact_mode_mozconfig(self):
+ return self.generate_mobile_android_mozconfig(artifact_mode=True)
+
+ def ensure_clang_static_analysis_package(self):
+ from mozboot import static_analysis
+
+ self.install_toolchain_static_analysis(static_analysis.WINDOWS_CLANG_TIDY)
+
+ def ensure_sccache_packages(self):
+ from mozboot import sccache
+
+ self.install_toolchain_artifact(sccache.RUSTC_DIST_TOOLCHAIN, no_unpack=True)
+ self.install_toolchain_artifact(sccache.CLANG_DIST_TOOLCHAIN, no_unpack=True)
+
+ def _update_package_manager(self):
+ pass
+
+ def run(self, command):
+ subprocess.check_call(command, stdin=sys.stdin)
diff --git a/python/mozboot/mozboot/openbsd.py b/python/mozboot/mozboot/openbsd.py
new file mode 100644
index 0000000000..614091c388
--- /dev/null
+++ b/python/mozboot/mozboot/openbsd.py
@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mozboot.base import BaseBootstrapper
+
+
+class OpenBSDBootstrapper(BaseBootstrapper):
+ def __init__(self, version, **kwargs):
+ BaseBootstrapper.__init__(self, **kwargs)
+
+ self.packages = ["gmake", "gtar", "rust", "unzip"]
+
+ self.browser_packages = [
+ "llvm",
+ "cbindgen",
+ "nasm",
+ "node",
+ "gtk+3",
+ "dbus-glib",
+ "pulseaudio",
+ ]
+
+ def install_system_packages(self):
+ # we use -z because there's no other way to say "any autoconf-2.13"
+ self.run_as_root(["pkg_add", "-z"] + self.packages)
+
+ def install_browser_packages(self, mozconfig_builder, artifact_mode=False):
+ # TODO: Figure out what not to install for artifact mode
+ # we use -z because there's no other way to say "any autoconf-2.13"
+ self.run_as_root(["pkg_add", "-z"] + self.browser_packages)
+
+ def install_browser_artifact_mode_packages(self, mozconfig_builder):
+ self.install_browser_packages(mozconfig_builder, artifact_mode=True)
+
+ def ensure_clang_static_analysis_package(self):
+ # TODO: we don't ship clang base static analysis for this platform
+ pass
diff --git a/python/mozboot/mozboot/opensuse.py b/python/mozboot/mozboot/opensuse.py
new file mode 100644
index 0000000000..051ee97f4b
--- /dev/null
+++ b/python/mozboot/mozboot/opensuse.py
@@ -0,0 +1,63 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mozboot.base import MERCURIAL_INSTALL_PROMPT, BaseBootstrapper
+from mozboot.linux_common import LinuxBootstrapper
+
+
+class OpenSUSEBootstrapper(LinuxBootstrapper, BaseBootstrapper):
+ """openSUSE experimental bootstrapper."""
+
+ def __init__(self, version, dist_id, **kwargs):
+ print("Using an experimental bootstrapper for openSUSE.")
+ BaseBootstrapper.__init__(self, **kwargs)
+
+ def install_packages(self, packages):
+ ALTERNATIVE_NAMES = {
+ "libxml2": "libxml2-2",
+ }
+ # watchman is not available
+ packages = [ALTERNATIVE_NAMES.get(p, p) for p in packages if p != "watchman"]
+ self.zypper_install(*packages)
+
+ def _update_package_manager(self):
+ self.zypper_update()
+
+ def upgrade_mercurial(self, current):
+ """Install Mercurial from pip because system packages could lag."""
+ if self.no_interactive:
+ # Install via zypper in non-interactive mode because it is the more
+ # conservative option and less likely to make people upset.
+ self.zypper_install("mercurial")
+ return
+
+ res = self.prompt_int(MERCURIAL_INSTALL_PROMPT, 1, 3)
+
+ # zypper.
+ if res == 2:
+ self.zypper_install("mercurial")
+ return False
+
+ # No Mercurial.
+ if res == 3:
+ print("Not installing Mercurial.")
+ return False
+
+ # pip.
+ assert res == 1
+ self.run_as_root(["pip3", "install", "--upgrade", "Mercurial"])
+
+ def zypper(self, *args):
+ if self.no_interactive:
+ command = ["zypper", "-n", *args]
+ else:
+ command = ["zypper", *args]
+
+ self.run_as_root(command)
+
+ def zypper_install(self, *packages):
+ self.zypper("install", *packages)
+
+ def zypper_update(self, *packages):
+ self.zypper("update", *packages)
diff --git a/python/mozboot/mozboot/osx.py b/python/mozboot/mozboot/osx.py
new file mode 100644
index 0000000000..bcab9d6d28
--- /dev/null
+++ b/python/mozboot/mozboot/osx.py
@@ -0,0 +1,324 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import platform
+import subprocess
+import sys
+import tempfile
+
+try:
+ from urllib2 import urlopen
+except ImportError:
+ from urllib.request import urlopen
+
+from mach.util import to_optional_path, to_optional_str
+from mozfile import which
+from packaging.version import Version
+
+from mozboot.base import BaseBootstrapper
+
+HOMEBREW_BOOTSTRAP = (
+ "https://raw.githubusercontent.com/Homebrew/install/master/install.sh"
+)
+
+BREW_INSTALL = """
+We will install the Homebrew package manager to install required packages.
+
+You will be prompted to install Homebrew with its default settings. If you
+would prefer to do this manually, hit CTRL+c, install Homebrew yourself, ensure
+"brew" is in your $PATH, and relaunch bootstrap.
+"""
+
+BREW_PACKAGES = """
+We are now installing all required packages via Homebrew. You will see a lot of
+output as packages are built.
+"""
+
+NO_BREW_INSTALLED = "It seems you don't have Homebrew installed."
+
+
+class OSXAndroidBootstrapper(object):
+ def install_mobile_android_packages(self, mozconfig_builder, artifact_mode=False):
+ os_arch = platform.machine()
+ if os_arch != "x86_64" and os_arch != "arm64":
+ raise Exception(
+ "You need a 64-bit version of Mac OS X to build "
+ "GeckoView/Firefox for Android."
+ )
+
+ from mozboot import android
+
+ android.ensure_android(
+ "macosx",
+ os_arch,
+ artifact_mode=artifact_mode,
+ no_interactive=self.no_interactive,
+ )
+
+ if os_arch == "x86_64" or os_arch == "x86":
+ android.ensure_android(
+ "macosx",
+ os_arch,
+ system_images_only=True,
+ artifact_mode=artifact_mode,
+ no_interactive=self.no_interactive,
+ avd_manifest_path=android.AVD_MANIFEST_X86_64,
+ )
+ android.ensure_android(
+ "macosx",
+ os_arch,
+ system_images_only=True,
+ artifact_mode=artifact_mode,
+ no_interactive=self.no_interactive,
+ avd_manifest_path=android.AVD_MANIFEST_ARM,
+ )
+ else:
+ android.ensure_android(
+ "macosx",
+ os_arch,
+ system_images_only=True,
+ artifact_mode=artifact_mode,
+ no_interactive=self.no_interactive,
+ avd_manifest_path=android.AVD_MANIFEST_ARM64,
+ )
+
+ def ensure_mobile_android_packages(self):
+ from mozboot import android
+
+ arch = platform.machine()
+ android.ensure_java("macosx", arch)
+
+ if arch == "x86_64" or arch == "x86":
+ self.install_toolchain_artifact(android.MACOS_X86_64_ANDROID_AVD)
+ self.install_toolchain_artifact(android.MACOS_ARM_ANDROID_AVD)
+ elif arch == "arm64":
+ # The only emulator supported on Apple Silicon is the Arm64 one.
+ self.install_toolchain_artifact(android.MACOS_ARM64_ANDROID_AVD)
+
+ def install_mobile_android_artifact_mode_packages(self, mozconfig_builder):
+ self.install_mobile_android_packages(mozconfig_builder, artifact_mode=True)
+
+ def generate_mobile_android_mozconfig(self):
+ return self._generate_mobile_android_mozconfig()
+
+ def generate_mobile_android_artifact_mode_mozconfig(self):
+ return self._generate_mobile_android_mozconfig(artifact_mode=True)
+
+ def _generate_mobile_android_mozconfig(self, artifact_mode=False):
+ from mozboot import android
+
+ return android.generate_mozconfig("macosx", artifact_mode=artifact_mode)
+
+
+def ensure_command_line_tools():
+ # We need either the command line tools or Xcode (one is sufficient).
+ # Python 3, required to run this code, is not installed by default on macos
+ # as of writing (macos <= 11.x).
+ # There are at least 5 different ways to obtain it:
+ # - macports
+ # - homebrew
+ # - command line tools
+ # - Xcode
+ # - python.org
+ # The first two require to install the command line tools.
+ # So only in the last case we may not have command line tools or xcode
+ # available.
+ # When the command line tools are installed, `xcode-select --print-path`
+ # prints their path.
+ # When Xcode is installed, `xcode-select --print-path` prints its path.
+ # When neither is installed, `xcode-select --print-path` prints an error
+ # to stderr and nothing to stdout.
+ # So in the rare case where we detect neither the command line tools or
+ # Xcode is installed, we trigger an intall of the command line tools
+ # (via `xcode-select --install`).
+ proc = subprocess.run(
+ ["xcode-select", "--print-path"],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL,
+ )
+ if not proc.stdout:
+ subprocess.run(["xcode-select", "--install"], check=True)
+ # xcode-select --install triggers a separate process to be started by
+ # launchd, and tracking its successful outcome would require something
+ # like figuring its pid and using kqueue to get a notification when it
+ # finishes. Considering how unlikely it is that someone would end up
+ # here in the first place, we just bail out.
+ print("Please follow the command line tools installer instructions")
+ print("and rerun `./mach bootstrap` when it's finished.")
+ sys.exit(1)
+
+
+class OSXBootstrapperLight(OSXAndroidBootstrapper, BaseBootstrapper):
+ def __init__(self, version, **kwargs):
+ BaseBootstrapper.__init__(self, **kwargs)
+
+ def install_system_packages(self):
+ ensure_command_line_tools()
+
+ # All the installs below are assumed to be handled by mach configure/build by
+ # default, which is true for arm64.
+ def install_browser_packages(self, mozconfig_builder):
+ pass
+
+ def install_browser_artifact_mode_packages(self, mozconfig_builder):
+ pass
+
+ def ensure_clang_static_analysis_package(self):
+ pass
+
+
+class OSXBootstrapper(OSXAndroidBootstrapper, BaseBootstrapper):
+ def __init__(self, version, **kwargs):
+ BaseBootstrapper.__init__(self, **kwargs)
+
+ self.os_version = Version(version)
+
+ if self.os_version < Version("10.6"):
+ raise Exception("OS X 10.6 or above is required.")
+
+ self.minor_version = version.split(".")[1]
+
+ def install_system_packages(self):
+ ensure_command_line_tools()
+
+ self.ensure_homebrew_installed()
+ _, hg_modern, _ = self.is_mercurial_modern()
+ if not hg_modern:
+ print(
+ "Mercurial wasn't found or is not sufficiently modern. "
+ "It will be installed with brew"
+ )
+
+ packages = ["git", "gnu-tar", "terminal-notifier", "watchman"]
+ if not hg_modern:
+ packages.append("mercurial")
+ self._ensure_homebrew_packages(packages)
+
+ def install_browser_packages(self, mozconfig_builder):
+ pass
+
+ def install_browser_artifact_mode_packages(self, mozconfig_builder):
+ pass
+
+ def _ensure_homebrew_found(self):
+ self.brew = to_optional_path(which("brew"))
+
+ return self.brew is not None
+
+ def _ensure_homebrew_packages(self, packages, is_for_cask=False):
+ package_type_flag = "--cask" if is_for_cask else "--formula"
+ self.ensure_homebrew_installed()
+
+ def create_homebrew_cmd(*parameters):
+ base_cmd = [to_optional_str(self.brew)]
+ base_cmd.extend(parameters)
+ return base_cmd + [package_type_flag]
+
+ installed = set(
+ subprocess.check_output(
+ create_homebrew_cmd("list"), universal_newlines=True
+ ).split()
+ )
+ outdated = set(
+ subprocess.check_output(
+ create_homebrew_cmd("outdated", "--quiet"), universal_newlines=True
+ ).split()
+ )
+
+ to_install = set(package for package in packages if package not in installed)
+ to_upgrade = set(package for package in packages if package in outdated)
+
+ if to_install or to_upgrade:
+ print(BREW_PACKAGES)
+ if to_install:
+ subprocess.check_call(create_homebrew_cmd("install") + list(to_install))
+ if to_upgrade:
+ subprocess.check_call(create_homebrew_cmd("upgrade") + list(to_upgrade))
+
+ def _ensure_homebrew_casks(self, casks):
+ self._ensure_homebrew_found()
+
+ known_taps = subprocess.check_output([to_optional_str(self.brew), "tap"])
+
+ # Ensure that we can access old versions of packages.
+ if b"homebrew/cask-versions" not in known_taps:
+ subprocess.check_output(
+ [to_optional_str(self.brew), "tap", "homebrew/cask-versions"]
+ )
+
+ # "caskroom/versions" has been renamed to "homebrew/cask-versions", so
+ # it is safe to remove the old tap. Removing the old tap is necessary
+ # to avoid the error "Cask [name of cask] exists in multiple taps".
+ # See https://bugzilla.mozilla.org/show_bug.cgi?id=1544981
+ if b"caskroom/versions" in known_taps:
+ subprocess.check_output(
+ [to_optional_str(self.brew), "untap", "caskroom/versions"]
+ )
+
+ self._ensure_homebrew_packages(casks, is_for_cask=True)
+
+ def ensure_homebrew_browser_packages(self):
+ # TODO: Figure out what not to install for artifact mode
+ packages = ["yasm"]
+ self._ensure_homebrew_packages(packages)
+
+ def ensure_homebrew_installed(self):
+ """
+ Search for Homebrew in sys.path, if not found, prompt the user to install it.
+ Then assert our PATH ordering is correct.
+ """
+ homebrew_found = self._ensure_homebrew_found()
+ if not homebrew_found:
+ self.install_homebrew()
+
+ def ensure_clang_static_analysis_package(self):
+ from mozboot import static_analysis
+
+ if platform.machine() == "arm64":
+ self.install_toolchain_static_analysis(
+ static_analysis.MACOS_AARCH64_CLANG_TIDY
+ )
+ else:
+ self.install_toolchain_static_analysis(static_analysis.MACOS_CLANG_TIDY)
+
+ def ensure_sccache_packages(self):
+ from mozboot import sccache
+
+ self.install_toolchain_artifact(sccache.RUSTC_DIST_TOOLCHAIN, no_unpack=True)
+ self.install_toolchain_artifact(sccache.CLANG_DIST_TOOLCHAIN, no_unpack=True)
+
+ def install_homebrew(self):
+ print(BREW_INSTALL)
+ bootstrap = urlopen(url=HOMEBREW_BOOTSTRAP, timeout=20).read()
+ with tempfile.NamedTemporaryFile() as tf:
+ tf.write(bootstrap)
+ tf.flush()
+
+ subprocess.check_call(["bash", tf.name])
+
+ homebrew_found = self._ensure_homebrew_found()
+ if not homebrew_found:
+ print(
+ "Homebrew was just installed but can't be found on PATH. "
+ "Please file a bug."
+ )
+ sys.exit(1)
+
+ def _update_package_manager(self):
+ subprocess.check_call([to_optional_str(self.brew), "-v", "update"])
+
+ def _upgrade_package(self, package):
+ self._ensure_homebrew_installed()
+
+ try:
+ subprocess.check_output(
+ [to_optional_str(self.brew), "-v", "upgrade", package],
+ stderr=subprocess.STDOUT,
+ )
+ except subprocess.CalledProcessError as e:
+ if b"already installed" not in e.output:
+ raise
+
+ def upgrade_mercurial(self, current):
+ self._upgrade_package("mercurial")
diff --git a/python/mozboot/mozboot/rust.py b/python/mozboot/mozboot/rust.py
new file mode 100644
index 0000000000..90607fccff
--- /dev/null
+++ b/python/mozboot/mozboot/rust.py
@@ -0,0 +1,185 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this,
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import platform as platform_mod
+import sys
+
+# Base url for pulling the rustup installer.
+RUSTUP_URL_BASE = "https://static.rust-lang.org/rustup"
+
+# Pull this to get the lastest stable version number.
+RUSTUP_MANIFEST = RUSTUP_URL_BASE + "/release-stable.toml"
+
+# We bake in a known version number so we can verify a checksum.
+RUSTUP_VERSION = "1.23.1"
+
+# SHA-256 checksums of the installers, per platform.
+RUSTUP_HASHES = {
+ "x86_64-unknown-freebsd": "3fb56018ec6009c5a3e345f07d7ea2fbc67d4c6768e528c6d990c7ebe2388d09",
+ "aarch64-apple-darwin": "6d56735284181b2eb804ed7f57f76cf5ff924251e8ab69d9b5822c3be1ca1dc7",
+ "x86_64-apple-darwin": "39101feb178a7e3e4443b09b36338e794a9e00385e5f44a2f7789aefb91354a9",
+ "x86_64-unknown-linux-gnu": "ed7773edaf1d289656bdec2aacad12413b38ad0193fff54b2231f5140a4b07c5",
+ "x86_64-pc-windows-msvc": "a586cf9de3e4aa791fd5796b6a5f99ca05591ccef8bb94e53af5b69f0261fb03",
+ "x86_64-unknown-netbsd": "8b29918e765f2cec3b81a911652b164471c42f8f31241f7401bb89582d6a3ed5",
+}
+
+NO_PLATFORM = """
+Sorry, we have no installer configured for your platform.
+
+Please try installing rust for your system from https://rustup.rs/
+or from https://rust-lang.org/ or from your package manager.
+"""
+
+
+def rustup_url(host, version=RUSTUP_VERSION):
+ """Download url for a particular version of the installer."""
+ return "%(base)s/archive/%(version)s/%(host)s/rustup-init%(ext)s" % {
+ "base": RUSTUP_URL_BASE,
+ "version": version,
+ "host": host,
+ "ext": exe_suffix(host),
+ }
+
+
+def rustup_hash(host):
+ """Look up the checksum for the given installer."""
+ return RUSTUP_HASHES.get(host, None)
+
+
+def platform():
+ """Determine the appropriate rust platform string for the current host"""
+ if sys.platform.startswith("darwin"):
+ if platform_mod.machine() == "arm64":
+ return "aarch64-apple-darwin"
+ return "x86_64-apple-darwin"
+ elif sys.platform.startswith(("win32", "msys")):
+ # Bravely assume we'll be building 64-bit Firefox.
+ return "x86_64-pc-windows-msvc"
+ elif sys.platform.startswith("linux"):
+ return "x86_64-unknown-linux-gnu"
+ elif sys.platform.startswith("freebsd"):
+ return "x86_64-unknown-freebsd"
+ elif sys.platform.startswith("netbsd"):
+ return "x86_64-unknown-netbsd"
+
+ return None
+
+
+def exe_suffix(host=None):
+ if not host:
+ host = platform()
+ if "windows" in host:
+ return ".exe"
+ return ""
+
+
+USAGE = """
+python rust.py [--update]
+
+Pass the --update option print info for the latest release of rustup-init.
+
+When invoked without the --update option, it queries the latest version
+and verifies the current stored checksums against the distribution server,
+but doesn't update the version installed by `mach bootstrap`.
+"""
+
+
+def unquote(s):
+ """Strip outer quotation marks from a string."""
+ return s.strip("'").strip('"')
+
+
+def rustup_latest_version():
+ """Query the latest version of the rustup installer."""
+ import requests
+
+ r = requests.get(RUSTUP_MANIFEST)
+ # The manifest is toml, but we might not have the toml4 python module
+ # available, so use ad-hoc parsing to obtain the current release version.
+ #
+ # The manifest looks like:
+ #
+ # schema-version = '1'
+ # version = '0.6.5'
+ #
+ for line in r.iter_lines():
+ line = line.decode("utf-8")
+ key, value = map(str.strip, line.split("=", 2))
+ if key == "schema-version":
+ schema = int(unquote(value))
+ if schema != 1:
+ print("ERROR: Unknown manifest schema %s" % value)
+ sys.exit(1)
+ elif key == "version":
+ return unquote(value)
+ return None
+
+
+def http_download_and_hash(url):
+ import hashlib
+
+ import requests
+
+ h = hashlib.sha256()
+ r = requests.get(url, stream=True)
+ for data in r.iter_content(4096):
+ h.update(data)
+ return h.hexdigest()
+
+
+def make_checksums(version, validate=False):
+ hashes = []
+ for platform in RUSTUP_HASHES.keys():
+ if validate:
+ print("Checking %s... " % platform, end="", flush=True)
+ else:
+ print("Fetching %s... " % platform, end="", flush=True)
+ checksum = http_download_and_hash(rustup_url(platform, version))
+ if validate and checksum != rustup_hash(platform):
+ print(
+ "mismatch:\n script: %s\n server: %s"
+ % (RUSTUP_HASHES[platform], checksum)
+ )
+ else:
+ print("OK")
+ hashes.append((platform, checksum))
+ return hashes
+
+
+if __name__ == "__main__":
+ """Allow invoking the module as a utility to update checksums."""
+
+ update = False
+ if len(sys.argv) > 1:
+ if sys.argv[1] == "--update":
+ update = True
+ else:
+ print(USAGE)
+ sys.exit(1)
+
+ print("Checking latest installer version... ", end="", flush=True)
+ version = rustup_latest_version()
+ if not version:
+ print("ERROR: Could not query current rustup installer version.")
+ sys.exit(1)
+ print(version)
+
+ if version == RUSTUP_VERSION:
+ print("We're up to date. Validating checksums.")
+ make_checksums(version, validate=True)
+ exit()
+
+ if not update:
+ print("Out of date. We use %s. Validating checksums." % RUSTUP_VERSION)
+ make_checksums(RUSTUP_VERSION, validate=True)
+ exit()
+
+ print("Out of date. We use %s. Calculating checksums." % RUSTUP_VERSION)
+ hashes = make_checksums(version)
+ print("")
+ print("RUSTUP_VERSION = '%s'" % version)
+ print("RUSTUP_HASHES = {")
+ for item in hashes:
+ print(" '%s':\n '%s'," % item)
+ print("}")
diff --git a/python/mozboot/mozboot/sccache.py b/python/mozboot/mozboot/sccache.py
new file mode 100644
index 0000000000..b3cc9fbae9
--- /dev/null
+++ b/python/mozboot/mozboot/sccache.py
@@ -0,0 +1,9 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# sccache-dist currently expects clients to provide toolchains when
+# distributing from macOS or Windows, so we download linux binaries capable
+# of cross-compiling for these cases.
+RUSTC_DIST_TOOLCHAIN = "rustc-dist-toolchain"
+CLANG_DIST_TOOLCHAIN = "clang-dist-toolchain"
diff --git a/python/mozboot/mozboot/solus.py b/python/mozboot/mozboot/solus.py
new file mode 100644
index 0000000000..664b5285aa
--- /dev/null
+++ b/python/mozboot/mozboot/solus.py
@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mozboot.base import BaseBootstrapper
+from mozboot.linux_common import LinuxBootstrapper
+
+
+class SolusBootstrapper(LinuxBootstrapper, BaseBootstrapper):
+ """Solus experimental bootstrapper."""
+
+ def __init__(self, version, dist_id, **kwargs):
+ print("Using an experimental bootstrapper for Solus.")
+ BaseBootstrapper.__init__(self, **kwargs)
+
+ def install_packages(self, packages):
+ self.package_install(*packages)
+
+ def _update_package_manager(self):
+ pass
+
+ def upgrade_mercurial(self, current):
+ self.package_install("mercurial")
+
+ def package_install(self, *packages):
+ command = ["eopkg", "install"]
+ if self.no_interactive:
+ command.append("--yes-all")
+
+ command.extend(packages)
+
+ self.run_as_root(command)
diff --git a/python/mozboot/mozboot/static_analysis.py b/python/mozboot/mozboot/static_analysis.py
new file mode 100644
index 0000000000..cd79f7c230
--- /dev/null
+++ b/python/mozboot/mozboot/static_analysis.py
@@ -0,0 +1,8 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+WINDOWS_CLANG_TIDY = "win64-clang-tidy"
+LINUX_CLANG_TIDY = "linux64-clang-tidy"
+MACOS_CLANG_TIDY = "macosx64-clang-tidy"
+MACOS_AARCH64_CLANG_TIDY = "macosx64-aarch64-clang-tidy"
diff --git a/python/mozboot/mozboot/test/python.ini b/python/mozboot/mozboot/test/python.ini
new file mode 100644
index 0000000000..4947f160f5
--- /dev/null
+++ b/python/mozboot/mozboot/test/python.ini
@@ -0,0 +1,4 @@
+[DEFAULT]
+subsuite = mozbuild
+
+[test_mozconfig.py]
diff --git a/python/mozboot/mozboot/test/test_mozconfig.py b/python/mozboot/mozboot/test/test_mozconfig.py
new file mode 100644
index 0000000000..b7375e8529
--- /dev/null
+++ b/python/mozboot/mozboot/test/test_mozconfig.py
@@ -0,0 +1,229 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+import unittest
+from pathlib import Path
+from shutil import rmtree
+from tempfile import gettempdir, mkdtemp
+
+import pytest
+from mozunit import main
+
+from mozboot.mozconfig import (
+ DEFAULT_TOPSRCDIR_PATHS,
+ DEPRECATED_HOME_PATHS,
+ DEPRECATED_TOPSRCDIR_PATHS,
+ MozconfigFindException,
+ find_mozconfig,
+)
+
+
+class TestFindMozconfig(unittest.TestCase):
+ def setUp(self):
+ self._old_env = dict(os.environ)
+ os.environ.pop("MOZCONFIG", None)
+ os.environ.pop("MOZ_OBJDIR", None)
+ os.environ.pop("CC", None)
+ os.environ.pop("CXX", None)
+ self._temp_dirs = set()
+
+ def tearDown(self):
+ os.environ.clear()
+ os.environ.update(self._old_env)
+
+ for temp_dir in self._temp_dirs:
+ rmtree(str(temp_dir))
+
+ def get_temp_dir(self):
+ new_temp_dir = Path(mkdtemp())
+ self._temp_dirs.add(new_temp_dir)
+
+ return new_temp_dir
+
+ def test_find_legacy_env(self):
+ """Ensure legacy mozconfig path definitions result in error."""
+
+ os.environ["MOZ_MYCONFIG"] = "/foo"
+
+ with self.assertRaises(MozconfigFindException) as e:
+ find_mozconfig(self.get_temp_dir())
+
+ self.assertTrue(str(e.exception).startswith("The MOZ_MYCONFIG"))
+
+ def test_find_multiple_configs(self):
+ """Ensure multiple relative-path MOZCONFIGs result in error."""
+ relative_mozconfig = ".mconfig"
+ os.environ["MOZCONFIG"] = relative_mozconfig
+
+ src_dir = self.get_temp_dir()
+ cur_dir = self.get_temp_dir()
+ dirs = [src_dir, cur_dir]
+ for iter_dir in dirs:
+ path = iter_dir / relative_mozconfig
+ with open(path, "w") as file:
+ file.write(str(path))
+
+ orig_dir = Path.cwd()
+ try:
+ os.chdir(cur_dir)
+ with self.assertRaises(MozconfigFindException) as e:
+ find_mozconfig(src_dir)
+ finally:
+ os.chdir(orig_dir)
+
+ self.assertIn("exists in more than one of", str(e.exception))
+ for iter_dir in dirs:
+ self.assertIn(str(iter_dir.resolve()), str(e.exception))
+
+ def test_find_multiple_but_identical_configs(self):
+ """Ensure multiple relative-path MOZCONFIGs pointing at the same file are OK."""
+ relative_mozconfig = "../src/.mconfig"
+ os.environ["MOZCONFIG"] = relative_mozconfig
+
+ top_dir = self.get_temp_dir()
+ src_dir = top_dir / "src"
+ src_dir.mkdir()
+ cur_dir = top_dir / "obj"
+ cur_dir.mkdir()
+
+ path = src_dir / relative_mozconfig
+ with open(path, "w"):
+ pass
+
+ orig_dir = Path.cwd()
+ try:
+ os.chdir(cur_dir)
+ self.assertEqual(Path(find_mozconfig(src_dir)).resolve(), path.resolve())
+ finally:
+ os.chdir(orig_dir)
+
+ def test_find_no_relative_configs(self):
+ """Ensure a missing relative-path MOZCONFIG is detected."""
+ relative_mozconfig = ".mconfig"
+ os.environ["MOZCONFIG"] = relative_mozconfig
+
+ src_dir = self.get_temp_dir()
+ cur_dir = self.get_temp_dir()
+ dirs = [src_dir, cur_dir]
+
+ orig_dir = Path.cwd()
+ try:
+ os.chdir(cur_dir)
+ with self.assertRaises(MozconfigFindException) as e:
+ find_mozconfig(src_dir)
+ finally:
+ os.chdir(orig_dir)
+
+ self.assertIn("does not exist in any of", str(e.exception))
+ for iter_dir in dirs:
+ self.assertIn(str(iter_dir.resolve()), str(e.exception))
+
+ def test_find_relative_mozconfig(self):
+ """Ensure a relative MOZCONFIG can be found in the srcdir."""
+ relative_mozconfig = ".mconfig"
+ os.environ["MOZCONFIG"] = relative_mozconfig
+
+ src_dir = Path(self.get_temp_dir())
+ cur_dir = Path(self.get_temp_dir())
+
+ path = src_dir / relative_mozconfig
+ with open(path, "w"):
+ pass
+
+ orig_dir = Path.cwd()
+ try:
+ os.chdir(cur_dir)
+ self.assertEqual(
+ str(Path(find_mozconfig(src_dir)).resolve()), str(path.resolve())
+ )
+ finally:
+ os.chdir(orig_dir)
+
+ @pytest.mark.skipif(
+ sys.platform.startswith("win"),
+ reason="This test uses unix-style absolute paths, since we now use Pathlib, and "
+ "`is_absolute()` always returns `False` on Windows if there isn't a drive"
+ " letter, this test is invalid for Windows.",
+ )
+ def test_find_abs_path_not_exist(self):
+ """Ensure a missing absolute path is detected."""
+ os.environ["MOZCONFIG"] = "/foo/bar/does/not/exist"
+
+ with self.assertRaises(MozconfigFindException) as e:
+ find_mozconfig(self.get_temp_dir())
+
+ self.assertIn("path that does not exist", str(e.exception))
+ self.assertIn("/foo/bar/does/not/exist", str(e.exception))
+
+ def test_find_path_not_file(self):
+ """Ensure non-file paths are detected."""
+
+ os.environ["MOZCONFIG"] = gettempdir()
+
+ with self.assertRaises(MozconfigFindException) as e:
+ find_mozconfig(self.get_temp_dir())
+
+ self.assertIn("refers to a non-file", str(e.exception))
+ self.assertTrue(str(e.exception).endswith(gettempdir()))
+
+ def test_find_default_files(self):
+ """Ensure default paths are used when present."""
+ for default_dir in DEFAULT_TOPSRCDIR_PATHS:
+ temp_dir = self.get_temp_dir()
+ path = temp_dir / default_dir
+
+ with open(path, "w"):
+ pass
+
+ self.assertEqual(Path(find_mozconfig(temp_dir)), path)
+
+ def test_find_multiple_defaults(self):
+ """Ensure we error when multiple default files are present."""
+ self.assertGreater(len(DEFAULT_TOPSRCDIR_PATHS), 1)
+
+ temp_dir = self.get_temp_dir()
+ for default_dir in DEFAULT_TOPSRCDIR_PATHS:
+ with open(temp_dir / default_dir, "w"):
+ pass
+
+ with self.assertRaises(MozconfigFindException) as e:
+ find_mozconfig(temp_dir)
+
+ self.assertIn("Multiple default mozconfig files present", str(e.exception))
+
+ def test_find_deprecated_path_srcdir(self):
+ """Ensure we error when deprecated path locations are present."""
+ for deprecated_dir in DEPRECATED_TOPSRCDIR_PATHS:
+ temp_dir = self.get_temp_dir()
+ with open(temp_dir / deprecated_dir, "w"):
+ pass
+
+ with self.assertRaises(MozconfigFindException) as e:
+ find_mozconfig(temp_dir)
+
+ self.assertIn("This implicit location is no longer", str(e.exception))
+ self.assertIn(str(temp_dir), str(e.exception))
+
+ def test_find_deprecated_home_paths(self):
+ """Ensure we error when deprecated home directory paths are present."""
+
+ for deprecated_path in DEPRECATED_HOME_PATHS:
+ home = self.get_temp_dir()
+ os.environ["HOME"] = str(home)
+ path = home / deprecated_path
+
+ with open(path, "w"):
+ pass
+
+ with self.assertRaises(MozconfigFindException) as e:
+ find_mozconfig(self.get_temp_dir())
+
+ self.assertIn("This implicit location is no longer", str(e.exception))
+ self.assertIn(str(path), str(e.exception))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/python/mozboot/mozboot/util.py b/python/mozboot/mozboot/util.py
new file mode 100644
index 0000000000..667df3bf8f
--- /dev/null
+++ b/python/mozboot/mozboot/util.py
@@ -0,0 +1,48 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import hashlib
+import os
+from pathlib import Path
+from urllib.request import urlopen
+
+from mach.site import PythonVirtualenv
+from mach.util import get_state_dir
+
+MINIMUM_RUST_VERSION = "1.65.0"
+
+
+def get_tools_dir(srcdir=False):
+ if os.environ.get("MOZ_AUTOMATION") and "MOZ_FETCHES_DIR" in os.environ:
+ return os.environ["MOZ_FETCHES_DIR"]
+ return get_state_dir(srcdir)
+
+
+def get_mach_virtualenv_root():
+ return Path(get_state_dir(specific_to_topsrcdir=True)) / "_virtualenvs" / "mach"
+
+
+def get_mach_virtualenv_binary():
+ root = get_mach_virtualenv_root()
+ return Path(PythonVirtualenv(str(root)).python_path)
+
+
+def http_download_and_save(url, dest: Path, hexhash, digest="sha256"):
+ """Download the given url and save it to dest. hexhash is a checksum
+ that will be used to validate the downloaded file using the given
+ digest algorithm. The value of digest can be any value accepted by
+ hashlib.new. The default digest used is 'sha256'."""
+ f = urlopen(url)
+ h = hashlib.new(digest)
+ with open(dest, "wb") as out:
+ while True:
+ data = f.read(4096)
+ if data:
+ out.write(data)
+ h.update(data)
+ else:
+ break
+ if h.hexdigest() != hexhash:
+ dest.unlink()
+ raise ValueError("Hash of downloaded file does not match expected hash")
diff --git a/python/mozboot/mozboot/void.py b/python/mozboot/mozboot/void.py
new file mode 100644
index 0000000000..8ae0198ace
--- /dev/null
+++ b/python/mozboot/mozboot/void.py
@@ -0,0 +1,41 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mozboot.base import BaseBootstrapper
+from mozboot.linux_common import LinuxBootstrapper
+
+
+class VoidBootstrapper(LinuxBootstrapper, BaseBootstrapper):
+ def __init__(self, version, dist_id, **kwargs):
+ BaseBootstrapper.__init__(self, **kwargs)
+
+ self.distro = "void"
+ self.version = version
+ self.dist_id = dist_id
+
+ def run_as_root(self, command):
+ # VoidLinux doesn't support users sudo'ing most commands by default because of the group
+ # configuration.
+ super().run_as_root(command, may_use_sudo=False)
+
+ def xbps_install(self, *packages):
+ command = ["xbps-install"]
+ if self.no_interactive:
+ command.append("-y")
+ command.extend(packages)
+
+ self.run_as_root(command)
+
+ def xbps_update(self):
+ command = ["xbps-install", "-Su"]
+ if self.no_interactive:
+ command.append("-y")
+
+ self.run_as_root(command)
+
+ def install_packages(self, packages):
+ self.xbps_install(*packages)
+
+ def _update_package_manager(self):
+ self.xbps_update()
diff --git a/python/mozboot/mozboot/windows.py b/python/mozboot/mozboot/windows.py
new file mode 100644
index 0000000000..5c8ddf7be4
--- /dev/null
+++ b/python/mozboot/mozboot/windows.py
@@ -0,0 +1,132 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import ctypes
+import os
+import subprocess
+import sys
+
+from mozfile import which
+
+from mozboot.base import BaseBootstrapper
+
+
+def is_aarch64_host():
+ from ctypes import wintypes
+
+ kernel32 = ctypes.windll.kernel32
+ IMAGE_FILE_MACHINE_UNKNOWN = 0
+ IMAGE_FILE_MACHINE_ARM64 = 0xAA64
+
+ try:
+ iswow64process2 = kernel32.IsWow64Process2
+ except Exception:
+ # If we can't access the symbol, we know we're not on aarch64.
+ return False
+
+ currentProcess = kernel32.GetCurrentProcess()
+ processMachine = wintypes.USHORT(IMAGE_FILE_MACHINE_UNKNOWN)
+ nativeMachine = wintypes.USHORT(IMAGE_FILE_MACHINE_UNKNOWN)
+
+ gotValue = iswow64process2(
+ currentProcess, ctypes.byref(processMachine), ctypes.byref(nativeMachine)
+ )
+ # If this call fails, we have no idea.
+ if not gotValue:
+ return False
+
+ return nativeMachine.value == IMAGE_FILE_MACHINE_ARM64
+
+
+class WindowsBootstrapper(BaseBootstrapper):
+ """Bootstrapper for msys2 based environments for building in Windows."""
+
+ SYSTEM_PACKAGES = [
+ "mingw-w64-x86_64-make",
+ "mingw-w64-x86_64-perl",
+ "patch",
+ "patchutils",
+ "diffutils",
+ "tar",
+ "unzip",
+ "mingw-w64-x86_64-toolchain", # TODO: Remove when Mercurial is installable from a wheel.
+ "mingw-w64-i686-toolchain",
+ ]
+
+ BROWSER_PACKAGES = ["mingw-w64-x86_64-nasm", "mingw-w64-i686-nsis"]
+
+ def __init__(self, **kwargs):
+ if (
+ "MOZ_WINDOWS_BOOTSTRAP" not in os.environ
+ or os.environ["MOZ_WINDOWS_BOOTSTRAP"] != "1"
+ ):
+ raise NotImplementedError(
+ "Bootstrap support for Windows is under development. For "
+ "now use MozillaBuild to set up a build environment on "
+ "Windows. If you are testing Windows Bootstrap support, "
+ "try `export MOZ_WINDOWS_BOOTSTRAP=1`"
+ )
+ BaseBootstrapper.__init__(self, **kwargs)
+ if not which("pacman"):
+ raise NotImplementedError(
+ "The Windows bootstrapper only works with msys2 with "
+ "pacman. Get msys2 at http://msys2.github.io/"
+ )
+ print("Using an experimental bootstrapper for Windows.")
+
+ def install_system_packages(self):
+ self.pacman_install(*self.SYSTEM_PACKAGES)
+
+ def upgrade_mercurial(self, current):
+ self.pip_install("mercurial")
+
+ def install_browser_packages(self, mozconfig_builder):
+ self.pacman_install(*self.BROWSER_PACKAGES)
+
+ def install_mobile_android_packages(self, mozconfig_builder):
+ raise NotImplementedError(
+ "We do not support building Android on Windows. Sorry!"
+ )
+
+ def ensure_mobile_android_packages(self):
+ raise NotImplementedError(
+ "We do not support building Android on Windows. Sorry!"
+ )
+
+ def install_mobile_android_artifact_mode_packages(self, mozconfig_builder):
+ raise NotImplementedError(
+ "We do not support building Android on Windows. Sorry!"
+ )
+
+ def ensure_clang_static_analysis_package(self):
+ from mozboot import static_analysis
+
+ self.install_toolchain_static_analysis(static_analysis.WINDOWS_CLANG_TIDY)
+
+ def _update_package_manager(self):
+ self.pacman_update()
+
+ def run(self, command):
+ subprocess.check_call(command, stdin=sys.stdin)
+
+ def pacman_update(self):
+ command = ["pacman", "--sync", "--refresh"]
+ self.run(command)
+
+ def pacman_upgrade(self):
+ command = ["pacman", "--sync", "--refresh", "--sysupgrade"]
+ self.run(command)
+
+ def pacman_install(self, *packages):
+ command = ["pacman", "--sync", "--needed"]
+ if self.no_interactive:
+ command.append("--noconfirm")
+
+ command.extend(packages)
+ self.run(command)
+
+ def pip_install(self, *packages):
+ command = ["pip", "install", "--upgrade"]
+ command.extend(packages)
+ self.run(command)
diff --git a/python/mozboot/setup.py b/python/mozboot/setup.py
new file mode 100644
index 0000000000..234650dc8a
--- /dev/null
+++ b/python/mozboot/setup.py
@@ -0,0 +1,16 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from distutils.core import setup
+
+VERSION = "0.1"
+
+setup(
+ name="mozboot",
+ description="System bootstrap for building Mozilla projects.",
+ license="MPL 2.0",
+ packages=["mozboot"],
+ version=VERSION,
+ scripts=["bin/bootstrap.py"],
+)