diff options
Diffstat (limited to 'python/mozboot/mozboot')
30 files changed, 4463 insertions, 0 deletions
diff --git a/python/mozboot/mozboot/__init__.py b/python/mozboot/mozboot/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/python/mozboot/mozboot/__init__.py diff --git a/python/mozboot/mozboot/android-avds/arm.json b/python/mozboot/mozboot/android-avds/arm.json new file mode 100644 index 0000000000..756018631c --- /dev/null +++ b/python/mozboot/mozboot/android-avds/arm.json @@ -0,0 +1,27 @@ +{ + "emulator_package": "system-images;android-24;default;armeabi-v7a", + "emulator_avd_name": "mozemulator-armeabi-v7a", + "emulator_extra_args": [ + "-skip-adb-auth", + "-verbose", + "-show-kernel", + "-ranchu", + "-selinux", "permissive", + "-memory", "3072", + "-cores", "4", + "-skin", "800x1280", + "-gpu", "on", + "-no-snapstorage", + "-no-snapshot", + "-no-window", + "-no-accel", + "-prop", "ro.test_harness=true" + ], + "emulator_extra_config": { + "hw.keyboard": "yes", + "hw.lcd.density": "320", + "disk.dataPartition.size": "4000MB", + "sdcard.size": "600M" + }, + "emulator_prewarm": false +} diff --git a/python/mozboot/mozboot/android-avds/arm64.json b/python/mozboot/mozboot/android-avds/arm64.json new file mode 100644 index 0000000000..767f9299d4 --- /dev/null +++ b/python/mozboot/mozboot/android-avds/arm64.json @@ -0,0 +1,27 @@ +{ + "emulator_package": "system-images;android-30;default;arm64-v8a", + "emulator_avd_name": "mozemulator-arm64", + "emulator_extra_args": [ + "-skip-adb-auth", + "-verbose", + "-show-kernel", + "-ranchu", + "-selinux", "permissive", + "-memory", "3072", + "-cores", "4", + "-skin", "800x1280", + "-gpu", "on", + "-no-snapstorage", + "-no-snapshot", + "-no-window", + "-no-accel", + "-prop", "ro.test_harness=true" + ], + "emulator_extra_config": { + "hw.keyboard": "yes", + "hw.lcd.density": "320", + "disk.dataPartition.size": "4000MB", + "sdcard.size": "600M" + }, + "emulator_prewarm": false +} diff --git a/python/mozboot/mozboot/android-avds/x86_64.json b/python/mozboot/mozboot/android-avds/x86_64.json new file mode 100644 index 0000000000..68f99a9937 --- /dev/null +++ b/python/mozboot/mozboot/android-avds/x86_64.json @@ -0,0 +1,26 @@ +{ + "emulator_package": "system-images;android-24;default;x86_64", + "emulator_avd_name": "mozemulator-x86_64", + "emulator_extra_args": [ + "-skip-adb-auth", + "-verbose", + "-show-kernel", + "-ranchu", + "-selinux", "permissive", + "-memory", "3072", + "-cores", "4", + "-skin", "800x1280", + "-gpu", "on", + "-no-snapstorage", + "-no-snapshot", + "-no-window", + "-no-accel", + "-prop", "ro.test_harness=true" + ], + "emulator_extra_config": { + "hw.keyboard": "yes", + "hw.lcd.density": "320", + "disk.dataPartition.size": "4000MB", + "sdcard.size": "600M" + } +} diff --git a/python/mozboot/mozboot/android-emulator-packages.txt b/python/mozboot/mozboot/android-emulator-packages.txt new file mode 100644 index 0000000000..3e782df670 --- /dev/null +++ b/python/mozboot/mozboot/android-emulator-packages.txt @@ -0,0 +1,2 @@ +platform-tools +emulator diff --git a/python/mozboot/mozboot/android-packages.txt b/python/mozboot/mozboot/android-packages.txt new file mode 100644 index 0000000000..e36a300d83 --- /dev/null +++ b/python/mozboot/mozboot/android-packages.txt @@ -0,0 +1,4 @@ +emulator +platform-tools +build-tools;33.0.1 +platforms;android-33 diff --git a/python/mozboot/mozboot/android-system-images-packages.txt b/python/mozboot/mozboot/android-system-images-packages.txt new file mode 100644 index 0000000000..86069f7680 --- /dev/null +++ b/python/mozboot/mozboot/android-system-images-packages.txt @@ -0,0 +1 @@ +emulator diff --git a/python/mozboot/mozboot/android.py b/python/mozboot/mozboot/android.py new file mode 100644 index 0000000000..26929da696 --- /dev/null +++ b/python/mozboot/mozboot/android.py @@ -0,0 +1,886 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this, +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import errno +import json +import os +import stat +import subprocess +import sys +import time +from pathlib import Path +from typing import Optional, Union + +import requests +from tqdm import tqdm + +# We need the NDK version in multiple different places, and it's inconvenient +# to pass down the NDK version to all relevant places, so we have this global +# variable. +from mozboot.bootstrap import MOZCONFIG_SUGGESTION_TEMPLATE + +NDK_VERSION = "r23c" +CMDLINE_TOOLS_VERSION_STRING = "9.0" +CMDLINE_TOOLS_VERSION = "9477386" + +BUNDLETOOL_VERSION = "1.14.1" + +# We expect the emulator AVD definitions to be platform agnostic +LINUX_X86_64_ANDROID_AVD = "linux64-android-avd-x86_64-repack" +LINUX_ARM_ANDROID_AVD = "linux64-android-avd-arm-repack" + +MACOS_X86_64_ANDROID_AVD = "linux64-android-avd-x86_64-repack" +MACOS_ARM_ANDROID_AVD = "linux64-android-avd-arm-repack" +MACOS_ARM64_ANDROID_AVD = "linux64-android-avd-arm64-repack" + +WINDOWS_X86_64_ANDROID_AVD = "linux64-android-avd-x86_64-repack" +WINDOWS_ARM_ANDROID_AVD = "linux64-android-avd-arm-repack" + +AVD_MANIFEST_X86_64 = Path(__file__).resolve().parent / "android-avds/x86_64.json" +AVD_MANIFEST_ARM = Path(__file__).resolve().parent / "android-avds/arm.json" +AVD_MANIFEST_ARM64 = Path(__file__).resolve().parent / "android-avds/arm64.json" + +JAVA_VERSION_MAJOR = "17" +JAVA_VERSION_MINOR = "0.7" +JAVA_VERSION_PATCH = "7" + +ANDROID_NDK_EXISTS = """ +Looks like you have the correct version of the Android NDK installed at: +%s +""" + +ANDROID_SDK_EXISTS = """ +Looks like you have the Android SDK installed at: +%s +We will install all required Android packages. +""" + +ANDROID_SDK_TOO_OLD = """ +Looks like you have an outdated Android SDK installed at: +%s +I can't update outdated Android SDKs to have the required 'sdkmanager' +tool. Move it out of the way (or remove it entirely) and then run +bootstrap again. +""" + +INSTALLING_ANDROID_PACKAGES = """ +We are now installing the following Android packages: +%s +You may be prompted to agree to the Android license. You may see some of +output as packages are downloaded and installed. +""" + +MOBILE_ANDROID_MOZCONFIG_TEMPLATE = """ +# Build GeckoView/Firefox for Android: +ac_add_options --enable-project=mobile/android + +# Targeting the following architecture. +# For regular phones, no --target is needed. +# For x86 emulators (and x86 devices, which are uncommon): +# ac_add_options --target=i686 +# For newer phones or Apple silicon +# ac_add_options --target=aarch64 +# For x86_64 emulators (and x86_64 devices, which are even less common): +# ac_add_options --target=x86_64 + +{extra_lines} +""" + +MOBILE_ANDROID_ARTIFACT_MODE_MOZCONFIG_TEMPLATE = """ +# Build GeckoView/Firefox for Android Artifact Mode: +ac_add_options --enable-project=mobile/android +ac_add_options --enable-artifact-builds + +{extra_lines} +# Write build artifacts to: +mk_add_options MOZ_OBJDIR=./objdir-frontend +""" + + +class GetNdkVersionError(Exception): + pass + + +def install_mobile_android_sdk_or_ndk(url, path: Path): + """ + Fetch an Android SDK or NDK from |url| and unpack it into the given |path|. + + We use, and 'requests' respects, https. We could also include SHAs for a + small improvement in the integrity guarantee we give. But this script is + bootstrapped over https anyway, so it's a really minor improvement. + + We keep a cache of the downloaded artifacts, writing into |path|/mozboot. + We don't yet clean the cache; it's better to waste some disk space and + not require a long re-download than to wipe the cache prematurely. + """ + + download_path = path / "mozboot" + try: + download_path.mkdir(parents=True) + except OSError as e: + if e.errno == errno.EEXIST and download_path.is_dir(): + pass + else: + raise + + file_name = url.split("/")[-1] + download_file_path = download_path / file_name + download(url, download_file_path) + + if file_name.endswith(".tar.gz") or file_name.endswith(".tgz"): + cmd = ["tar", "zxf", str(download_file_path)] + elif file_name.endswith(".tar.bz2"): + cmd = ["tar", "jxf", str(download_file_path)] + elif file_name.endswith(".zip"): + cmd = ["unzip", "-q", str(download_file_path)] + elif file_name.endswith(".bin"): + # Execute the .bin file, which unpacks the content. + mode = os.stat(path).st_mode + download_file_path.chmod(mode | stat.S_IXUSR) + cmd = [str(download_file_path)] + else: + raise NotImplementedError(f"Don't know how to unpack file: {file_name}") + + print(f"Unpacking {download_file_path}...") + + with open(os.devnull, "w") as stdout: + # These unpack commands produce a ton of output; ignore it. The + # .bin files are 7z archives; there's no command line flag to quiet + # output, so we use this hammer. + subprocess.check_call(cmd, stdout=stdout, cwd=str(path)) + + print(f"Unpacking {download_file_path}... DONE") + # Now delete the archive + download_file_path.unlink() + + +def download( + url, + download_file_path: Path, +): + with requests.Session() as session: + request = session.head(url, allow_redirects=True) + request.raise_for_status() + remote_file_size = int(request.headers["content-length"]) + + if download_file_path.is_file(): + local_file_size = download_file_path.stat().st_size + + if local_file_size == remote_file_size: + print( + f"{download_file_path.name} already downloaded. Skipping download..." + ) + else: + print(f"Partial download detected. Resuming download of {url}...") + download_internal( + download_file_path, + session, + url, + remote_file_size, + local_file_size, + ) + else: + print(f"Downloading {url}...") + download_internal(download_file_path, session, url, remote_file_size) + + +def download_internal( + download_file_path: Path, + session, + url, + remote_file_size, + resume_from_byte_pos: int = None, +): + """ + Handles both a fresh SDK/NDK download, as well as resuming a partial one + """ + # "ab" will behave same as "wb" if file does not exist + with open(download_file_path, "ab") as file: + # 64 KB/s should be fine on even the slowest internet connections + chunk_size = 1024 * 64 + # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Range#directives + resume_header = ( + {"Range": f"bytes={resume_from_byte_pos}-"} + if resume_from_byte_pos + else None + ) + + request = session.get( + url, stream=True, allow_redirects=True, headers=resume_header + ) + + with tqdm( + total=int(remote_file_size), + unit="B", + unit_scale=True, + unit_divisor=1024, + desc=download_file_path.name, + initial=resume_from_byte_pos if resume_from_byte_pos else 0, + ) as progress_bar: + for chunk in request.iter_content(chunk_size): + file.write(chunk) + progress_bar.update(len(chunk)) + + +def get_ndk_version(ndk_path: Union[str, Path]): + """Given the path to the NDK, return the version as a 3-tuple of (major, + minor, human). + """ + ndk_path = Path(ndk_path) + with open(ndk_path / "source.properties", "r") as f: + revision = [line for line in f if line.startswith("Pkg.Revision")] + if not revision: + raise GetNdkVersionError( + "Cannot determine NDK version from source.properties" + ) + if len(revision) != 1: + raise GetNdkVersionError("Too many Pkg.Revision lines in source.properties") + + (_, version) = revision[0].split("=") + if not version: + raise GetNdkVersionError( + "Unexpected Pkg.Revision line in source.properties" + ) + + (major, minor, revision) = version.strip().split(".") + if not major or not minor: + raise GetNdkVersionError("Unexpected NDK version string: " + version) + + # source.properties contains a $MAJOR.$MINOR.$PATCH revision number, + # but the more common nomenclature that Google uses is alphanumeric + # version strings like "r20" or "r19c". Convert the source.properties + # notation into an alphanumeric string. + int_minor = int(minor) + alphas = "abcdefghijklmnop" + ascii_minor = alphas[int_minor] if int_minor > 0 else "" + human = "r%s%s" % (major, ascii_minor) + return (major, minor, human) + + +def get_paths(os_name): + mozbuild_path = Path( + os.environ.get("MOZBUILD_STATE_PATH", Path("~/.mozbuild").expanduser()) + ) + sdk_path = Path( + os.environ.get("ANDROID_SDK_HOME", mozbuild_path / f"android-sdk-{os_name}"), + ) + ndk_path = Path( + os.environ.get( + "ANDROID_NDK_HOME", mozbuild_path / f"android-ndk-{NDK_VERSION}" + ), + ) + avd_home_path = Path( + os.environ.get("ANDROID_AVD_HOME", mozbuild_path / "android-device" / "avd") + ) + return mozbuild_path, sdk_path, ndk_path, avd_home_path + + +def sdkmanager_tool(sdk_path: Path): + # sys.platform is win32 even if Python/Win64. + sdkmanager = "sdkmanager.bat" if sys.platform.startswith("win") else "sdkmanager" + return ( + sdk_path / "cmdline-tools" / CMDLINE_TOOLS_VERSION_STRING / "bin" / sdkmanager + ) + + +def avdmanager_tool(sdk_path: Path): + # sys.platform is win32 even if Python/Win64. + sdkmanager = "avdmanager.bat" if sys.platform.startswith("win") else "avdmanager" + return ( + sdk_path / "cmdline-tools" / CMDLINE_TOOLS_VERSION_STRING / "bin" / sdkmanager + ) + + +def adb_tool(sdk_path: Path): + adb = "adb.bat" if sys.platform.startswith("win") else "adb" + return sdk_path / "platform-tools" / adb + + +def emulator_tool(sdk_path: Path): + emulator = "emulator.bat" if sys.platform.startswith("win") else "emulator" + return sdk_path / "emulator" / emulator + + +def ensure_android( + os_name, + os_arch, + artifact_mode=False, + ndk_only=False, + system_images_only=False, + emulator_only=False, + avd_manifest_path: Optional[Path] = None, + prewarm_avd=False, + no_interactive=False, + list_packages=False, +): + """ + Ensure the Android SDK (and NDK, if `artifact_mode` is falsy) are + installed. If not, fetch and unpack the SDK and/or NDK from the + given URLs. Ensure the required Android SDK packages are + installed. + + `os_name` can be 'linux', 'macosx' or 'windows'. + """ + # The user may have an external Android SDK (in which case we + # save them a lengthy download), or they may have already + # completed the download. We unpack to + # ~/.mozbuild/{android-sdk-$OS_NAME, android-ndk-$VER}. + mozbuild_path, sdk_path, ndk_path, avd_home_path = get_paths(os_name) + + if os_name == "macosx": + os_tag = "mac" + elif os_name == "windows": + os_tag = "win" + else: + os_tag = os_name + + sdk_url = "https://dl.google.com/android/repository/commandlinetools-{0}-{1}_latest.zip".format( # NOQA: E501 + os_tag, CMDLINE_TOOLS_VERSION + ) + ndk_url = android_ndk_url(os_name) + bundletool_url = "https://github.com/google/bundletool/releases/download/{v}/bundletool-all-{v}.jar".format( # NOQA: E501 + v=BUNDLETOOL_VERSION + ) + + ensure_android_sdk_and_ndk( + mozbuild_path, + os_name, + sdk_path=sdk_path, + sdk_url=sdk_url, + ndk_path=ndk_path, + ndk_url=ndk_url, + bundletool_url=bundletool_url, + artifact_mode=artifact_mode, + ndk_only=ndk_only, + emulator_only=emulator_only, + ) + + if ndk_only: + return + + avd_manifest = None + if avd_manifest_path is not None: + with open(avd_manifest_path) as f: + avd_manifest = json.load(f) + # Some AVDs cannot be prewarmed in CI because they cannot run on linux64 + # (like the arm64 AVD). + if "emulator_prewarm" in avd_manifest: + prewarm_avd = prewarm_avd and avd_manifest["emulator_prewarm"] + + # We expect the |sdkmanager| tool to be at + # ~/.mozbuild/android-sdk-$OS_NAME/tools/cmdline-tools/$CMDLINE_TOOLS_VERSION_STRING/bin/sdkmanager. # NOQA: E501 + ensure_android_packages( + os_name, + os_arch, + sdkmanager_tool=sdkmanager_tool(sdk_path), + emulator_only=emulator_only, + system_images_only=system_images_only, + avd_manifest=avd_manifest, + no_interactive=no_interactive, + list_packages=list_packages, + ) + + if emulator_only or system_images_only: + return + + ensure_android_avd( + avdmanager_tool=avdmanager_tool(sdk_path), + adb_tool=adb_tool(sdk_path), + emulator_tool=emulator_tool(sdk_path), + avd_home_path=avd_home_path, + sdk_path=sdk_path, + no_interactive=no_interactive, + avd_manifest=avd_manifest, + prewarm_avd=prewarm_avd, + ) + + +def ensure_android_sdk_and_ndk( + mozbuild_path: Path, + os_name, + sdk_path: Path, + sdk_url, + ndk_path: Path, + ndk_url, + bundletool_url, + artifact_mode, + ndk_only, + emulator_only, +): + """ + Ensure the Android SDK and NDK are found at the given paths. If not, fetch + and unpack the SDK and/or NDK from the given URLs into + |mozbuild_path/{android-sdk-$OS_NAME,android-ndk-$VER}|. + """ + + # It's not particularly bad to overwrite the NDK toolchain, but it does take + # a while to unpack, so let's avoid the disk activity if possible. The SDK + # may prompt about licensing, so we do this first. + # Check for Android NDK only if we are not in artifact mode. + if not artifact_mode and not emulator_only: + install_ndk = True + if ndk_path.is_dir(): + try: + _, _, human = get_ndk_version(ndk_path) + if human == NDK_VERSION: + print(ANDROID_NDK_EXISTS % ndk_path) + install_ndk = False + except GetNdkVersionError: + pass # Just do the install. + if install_ndk: + # The NDK archive unpacks into a top-level android-ndk-$VER directory. + install_mobile_android_sdk_or_ndk(ndk_url, mozbuild_path) + + if ndk_only: + return + + # We don't want to blindly overwrite, since we use the + # |sdkmanager| tool to install additional parts of the Android + # toolchain. If we overwrite, we lose whatever Android packages + # the user may have already installed. + if sdkmanager_tool(sdk_path).is_file(): + print(ANDROID_SDK_EXISTS % sdk_path) + elif sdk_path.is_dir(): + raise NotImplementedError(ANDROID_SDK_TOO_OLD % sdk_path) + else: + # The SDK archive used to include a top-level + # android-sdk-$OS_NAME directory; it no longer does so. We + # preserve the old convention to smooth detecting existing SDK + # installations. + cmdline_tools_path = mozbuild_path / f"android-sdk-{os_name}" / "cmdline-tools" + install_mobile_android_sdk_or_ndk(sdk_url, cmdline_tools_path) + # The tools package *really* wants to be in + # <sdk>/cmdline-tools/$CMDLINE_TOOLS_VERSION_STRING + (cmdline_tools_path / "cmdline-tools").rename( + cmdline_tools_path / CMDLINE_TOOLS_VERSION_STRING + ) + download(bundletool_url, mozbuild_path / "bundletool.jar") + + +def get_packages_to_install(packages_file_content, avd_manifest): + packages = [] + packages += map(lambda package: package.strip(), packages_file_content) + if avd_manifest is not None: + packages += [avd_manifest["emulator_package"]] + return packages + + +def ensure_android_avd( + avdmanager_tool: Path, + adb_tool: Path, + emulator_tool: Path, + avd_home_path: Path, + sdk_path: Path, + no_interactive=False, + avd_manifest=None, + prewarm_avd=False, +): + """ + Use the given sdkmanager tool (like 'sdkmanager') to install required + Android packages. + """ + if avd_manifest is None: + return + + avd_home_path.mkdir(parents=True, exist_ok=True) + # The AVD needs this folder to boot, so make sure it exists here. + (sdk_path / "platforms").mkdir(parents=True, exist_ok=True) + + avd_name = avd_manifest["emulator_avd_name"] + args = [ + str(avdmanager_tool), + "--verbose", + "create", + "avd", + "--force", + "--name", + avd_name, + "--package", + avd_manifest["emulator_package"], + ] + + if not no_interactive: + subprocess.check_call(args) + return + + # Flush outputs before running sdkmanager. + sys.stdout.flush() + env = os.environ.copy() + env["ANDROID_AVD_HOME"] = str(avd_home_path) + proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env) + proc.communicate("no\n".encode("UTF-8")) + + retcode = proc.poll() + if retcode: + cmd = args[0] + e = subprocess.CalledProcessError(retcode, cmd) + raise e + + avd_path = avd_home_path / (str(avd_name) + ".avd") + config_file_name = avd_path / "config.ini" + + print(f"Writing config at {config_file_name}") + + if config_file_name.is_file(): + with open(config_file_name, "a") as config: + for key, value in avd_manifest["emulator_extra_config"].items(): + config.write("%s=%s\n" % (key, value)) + else: + raise NotImplementedError( + f"Could not find config file at {config_file_name}, something went wrong" + ) + if prewarm_avd: + run_prewarm_avd(adb_tool, emulator_tool, env, avd_name, avd_manifest) + # When running in headless mode, the emulator does not run the cleanup + # step, and thus doesn't delete lock files. On some platforms, left-over + # lock files can cause the emulator to not start, so we remove them here. + for lock_file in ["hardware-qemu.ini.lock", "multiinstance.lock"]: + lock_file_path = avd_path / lock_file + try: + lock_file_path.unlink() + print(f"Removed lock file {lock_file_path}") + except OSError: + # The lock file is not there, nothing to do. + pass + + +def run_prewarm_avd( + adb_tool: Path, + emulator_tool: Path, + env, + avd_name, + avd_manifest, +): + """ + Ensures the emulator is fully booted to save time on future iterations. + """ + args = [str(emulator_tool), "-avd", avd_name] + avd_manifest["emulator_extra_args"] + + # Flush outputs before running emulator. + sys.stdout.flush() + proc = subprocess.Popen(args, env=env) + + booted = False + for i in range(100): + boot_completed_cmd = [str(adb_tool), "shell", "getprop", "sys.boot_completed"] + completed_proc = subprocess.Popen( + boot_completed_cmd, env=env, stdout=subprocess.PIPE + ) + try: + out, err = completed_proc.communicate(timeout=30) + boot_completed = out.decode("UTF-8").strip() + print("sys.boot_completed = %s" % boot_completed) + time.sleep(30) + if boot_completed == "1": + booted = True + break + except subprocess.TimeoutExpired: + # Sometimes the adb command hangs, that's ok + print("sys.boot_completed = Timeout") + + if not booted: + raise NotImplementedError("Could not prewarm emulator") + + # Wait until the emulator completely shuts down + subprocess.Popen([str(adb_tool), "emu", "kill"], env=env).wait() + proc.wait() + + +def ensure_android_packages( + os_name, + os_arch, + sdkmanager_tool: Path, + emulator_only=False, + system_images_only=False, + avd_manifest=None, + no_interactive=False, + list_packages=False, +): + """ + Use the given sdkmanager tool (like 'sdkmanager') to install required + Android packages. + """ + + # This tries to install all the required Android packages. The user + # may be prompted to agree to the Android license. + if system_images_only: + packages_file_name = "android-system-images-packages.txt" + elif emulator_only: + packages_file_name = "android-emulator-packages.txt" + else: + packages_file_name = "android-packages.txt" + + packages_file_path = (Path(__file__).parent / packages_file_name).resolve() + + with open(packages_file_path) as packages_file: + packages_file_content = packages_file.readlines() + + packages = get_packages_to_install(packages_file_content, avd_manifest) + print(INSTALLING_ANDROID_PACKAGES % "\n".join(packages)) + + args = [str(sdkmanager_tool)] + if os_name == "macosx" and os_arch == "arm64": + # Support for Apple Silicon is still in nightly + args.append("--channel=3") + args.extend(packages) + + # sdkmanager needs JAVA_HOME + java_bin_path = ensure_java(os_name, os_arch) + env = os.environ.copy() + env["JAVA_HOME"] = str(java_bin_path.parent) + + if not no_interactive: + subprocess.check_call(args, env=env) + return + + # Flush outputs before running sdkmanager. + sys.stdout.flush() + sys.stderr.flush() + # Emulate yes. For a discussion of passing input to check_output, + # see https://stackoverflow.com/q/10103551. + yes = "\n".join(["y"] * 100).encode("UTF-8") + proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env) + proc.communicate(yes) + + retcode = proc.poll() + if retcode: + cmd = args[0] + e = subprocess.CalledProcessError(retcode, cmd) + raise e + if list_packages: + subprocess.check_call([str(sdkmanager_tool), "--list"]) + + +def generate_mozconfig(os_name, artifact_mode=False): + moz_state_dir, sdk_path, ndk_path, avd_home_path = get_paths(os_name) + + extra_lines = [] + if extra_lines: + extra_lines.append("") + + if artifact_mode: + template = MOBILE_ANDROID_ARTIFACT_MODE_MOZCONFIG_TEMPLATE + else: + template = MOBILE_ANDROID_MOZCONFIG_TEMPLATE + + kwargs = dict( + sdk_path=sdk_path, + ndk_path=ndk_path, + avd_home_path=avd_home_path, + moz_state_dir=moz_state_dir, + extra_lines="\n".join(extra_lines), + ) + return template.format(**kwargs).strip() + + +def android_ndk_url(os_name, ver=NDK_VERSION): + # Produce a URL like + # 'https://dl.google.com/android/repository/android-ndk-$VER-linux.zip + base_url = "https://dl.google.com/android/repository/android-ndk" + + if os_name == "macosx": + # |mach bootstrap| uses 'macosx', but Google uses 'darwin'. + os_name = "darwin" + + return "%s-%s-%s.zip" % (base_url, ver, os_name) + + +def main(argv): + import optparse # No argparse, which is new in Python 2.7. + import platform + + parser = optparse.OptionParser() + parser.add_option( + "-a", + "--artifact-mode", + dest="artifact_mode", + action="store_true", + help="If true, install only the Android SDK (and not the Android NDK).", + ) + parser.add_option( + "--jdk-only", + dest="jdk_only", + action="store_true", + help="If true, install only the Java JDK.", + ) + parser.add_option( + "--ndk-only", + dest="ndk_only", + action="store_true", + help="If true, install only the Android NDK (and not the Android SDK).", + ) + parser.add_option( + "--system-images-only", + dest="system_images_only", + action="store_true", + help="If true, install only the system images for the AVDs.", + ) + parser.add_option( + "--no-interactive", + dest="no_interactive", + action="store_true", + help="Accept the Android SDK licenses without user interaction.", + ) + parser.add_option( + "--emulator-only", + dest="emulator_only", + action="store_true", + help="If true, install only the Android emulator (and not the SDK or NDK).", + ) + parser.add_option( + "--avd-manifest", + dest="avd_manifest_path", + help="If present, generate AVD from the manifest pointed by this argument.", + ) + parser.add_option( + "--prewarm-avd", + dest="prewarm_avd", + action="store_true", + help="If true, boot the AVD and wait until completed to speed up subsequent boots.", + ) + parser.add_option( + "--list-packages", + dest="list_packages", + action="store_true", + help="If true, list installed packages.", + ) + + options, _ = parser.parse_args(argv) + + if options.artifact_mode and options.ndk_only: + raise NotImplementedError("Use no options to install the NDK and the SDK.") + + if options.artifact_mode and options.emulator_only: + raise NotImplementedError("Use no options to install the SDK and emulators.") + + os_name = None + if platform.system() == "Darwin": + os_name = "macosx" + elif platform.system() == "Linux": + os_name = "linux" + elif platform.system() == "Windows": + os_name = "windows" + else: + raise NotImplementedError( + "We don't support bootstrapping the Android SDK (or Android " + "NDK) on {0} yet!".format(platform.system()) + ) + + os_arch = platform.machine() + + if options.jdk_only: + ensure_java(os_name, os_arch) + return 0 + + avd_manifest_path = ( + Path(options.avd_manifest_path) if options.avd_manifest_path else None + ) + + ensure_android( + os_name, + os_arch, + artifact_mode=options.artifact_mode, + ndk_only=options.ndk_only, + system_images_only=options.system_images_only, + emulator_only=options.emulator_only, + avd_manifest_path=avd_manifest_path, + prewarm_avd=options.prewarm_avd, + no_interactive=options.no_interactive, + list_packages=options.list_packages, + ) + mozconfig = generate_mozconfig(os_name, options.artifact_mode) + + # |./mach bootstrap| automatically creates a mozconfig file for you if it doesn't + # exist. However, here, we don't know where the "topsrcdir" is, and it's not worth + # pulling in CommandContext (and its dependencies) to find out. + # So, instead, we'll politely ask users to create (or update) the file themselves. + suggestion = MOZCONFIG_SUGGESTION_TEMPLATE % ("$topsrcdir/mozconfig", mozconfig) + print("\n" + suggestion) + + return 0 + + +def ensure_java(os_name, os_arch): + mozbuild_path, _, _, _ = get_paths(os_name) + + if os_name == "macosx": + os_tag = "mac" + else: + os_tag = os_name + + if os_arch == "x86_64": + arch = "x64" + elif os_arch == "arm64": + arch = "aarch64" + else: + arch = os_arch + + ext = "zip" if os_name == "windows" else "tar.gz" + + java_path = java_bin_path(os_name, mozbuild_path) + if not java_path: + raise NotImplementedError(f"Could not bootstrap java for {os_name}.") + + if not java_path.exists(): + # e.g. https://github.com/adoptium/temurin17-binaries/releases/ + # download/jdk-17.0.7%2B7/OpenJDK17U-jre_x64_linux_hotspot_17.0.7_7.tar.gz + java_url = ( + "https://github.com/adoptium/temurin{major}-binaries/releases/" + "download/jdk-{major}.{minor}%2B{patch}/" + "OpenJDK{major}U-jdk_{arch}_{os}_hotspot_{major}.{minor}_{patch}.{ext}" + ).format( + major=JAVA_VERSION_MAJOR, + minor=JAVA_VERSION_MINOR, + patch=JAVA_VERSION_PATCH, + os=os_tag, + arch=arch, + ext=ext, + ) + install_mobile_android_sdk_or_ndk(java_url, mozbuild_path / "jdk") + return java_path + + +def java_bin_path(os_name, toolchain_path: Path): + # Like jdk-17.0.7+7 + jdk_folder = "jdk-{major}.{minor}+{patch}".format( + major=JAVA_VERSION_MAJOR, minor=JAVA_VERSION_MINOR, patch=JAVA_VERSION_PATCH + ) + + java_path = toolchain_path / "jdk" / jdk_folder + + if os_name == "macosx": + return java_path / "Contents" / "Home" / "bin" + elif os_name == "linux": + return java_path / "bin" + elif os_name == "windows": + return java_path / "bin" + else: + return None + + +def locate_java_bin_path(host_kernel, toolchain_path: Union[str, Path]): + if host_kernel == "WINNT": + os_name = "windows" + elif host_kernel == "Darwin": + os_name = "macosx" + elif host_kernel == "Linux": + os_name = "linux" + else: + # Default to Linux + os_name = "linux" + path = java_bin_path(os_name, Path(toolchain_path)) + if not path.is_dir(): + raise JavaLocationFailedException( + f"Could not locate Java at {path}, please run " + "./mach bootstrap --no-system-changes" + ) + return str(path) + + +class JavaLocationFailedException(Exception): + pass + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/python/mozboot/mozboot/archlinux.py b/python/mozboot/mozboot/archlinux.py new file mode 100644 index 0000000000..391bd3e3ff --- /dev/null +++ b/python/mozboot/mozboot/archlinux.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys + +from mozboot.base import BaseBootstrapper +from mozboot.linux_common import LinuxBootstrapper + + +class ArchlinuxBootstrapper(LinuxBootstrapper, BaseBootstrapper): + """Archlinux experimental bootstrapper.""" + + def __init__(self, version, dist_id, **kwargs): + print("Using an experimental bootstrapper for Archlinux.", file=sys.stderr) + BaseBootstrapper.__init__(self, **kwargs) + + def install_packages(self, packages): + # watchman is not available via pacman + packages = [p for p in packages if p != "watchman"] + self.pacman_install(*packages) + + def upgrade_mercurial(self, current): + self.pacman_install("mercurial") + + def pacman_install(self, *packages): + command = ["pacman", "-S", "--needed"] + if self.no_interactive: + command.append("--noconfirm") + + command.extend(packages) + + self.run_as_root(command) diff --git a/python/mozboot/mozboot/base.py b/python/mozboot/mozboot/base.py new file mode 100644 index 0000000000..c32946c4eb --- /dev/null +++ b/python/mozboot/mozboot/base.py @@ -0,0 +1,733 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import re +import subprocess +import sys +from pathlib import Path + +from mach.util import to_optional_path, win_to_msys_path +from mozbuild.bootstrap import bootstrap_all_toolchains_for, bootstrap_toolchain +from mozfile import which +from packaging.version import Version + +from mozboot import rust +from mozboot.util import ( + MINIMUM_RUST_VERSION, + get_mach_virtualenv_binary, + http_download_and_save, +) + +NO_MERCURIAL = """ +Could not find Mercurial (hg) in the current shell's path. Try starting a new +shell and running the bootstrapper again. +""" + +MERCURIAL_UNABLE_UPGRADE = """ +You are currently running Mercurial %s. Running %s or newer is +recommended for performance and stability reasons. + +Unfortunately, this bootstrapper currently does not know how to automatically +upgrade Mercurial on your machine. + +You can usually install Mercurial through your package manager or by +downloading a package from http://mercurial.selenic.com/. +""" + +MERCURIAL_UPGRADE_FAILED = """ +We attempted to upgrade Mercurial to a modern version (%s or newer). +However, you appear to have version %s still. + +It's possible your package manager doesn't support a modern version of +Mercurial. It's also possible Mercurial is not being installed in the search +path for this shell. Try creating a new shell and run this bootstrapper again. + +If it continues to fail, consider installing Mercurial by following the +instructions at http://mercurial.selenic.com/. +""" + +MERCURIAL_INSTALL_PROMPT = """ +Mercurial releases a new version every 3 months and your distro's package +may become out of date. This may cause incompatibility with some +Mercurial extensions that rely on new Mercurial features. As a result, +you may not have an optimal version control experience. + +To have the best Mercurial experience possible, we recommend installing +Mercurial via the "pip" Python packaging utility. This will likely result +in files being placed in /usr/local/bin and /usr/local/lib. + +How would you like to continue? + 1. Install a modern Mercurial via pip [default] + 2. Install a legacy Mercurial via the distro package manager + 3. Do not install Mercurial +Your choice: """ + +PYTHON_UNABLE_UPGRADE = """ +You are currently running Python %s. Running %s or newer (but +not 3.x) is required. + +Unfortunately, this bootstrapper does not currently know how to automatically +upgrade Python on your machine. + +Please search the Internet for how to upgrade your Python and try running this +bootstrapper again to ensure your machine is up to date. +""" + +RUST_INSTALL_COMPLETE = """ +Rust installation complete. You should now have rustc and cargo +in %(cargo_bin)s + +The installer tries to add these to your default shell PATH, so +restarting your shell and running this script again may work. +If it doesn't, you'll need to add the new command location +manually. + +If restarting doesn't work, edit your shell initialization +script, which may be called ~/.bashrc or ~/.bash_profile or +~/.profile, and add the following line: + + %(cmd)s + +Then restart your shell and run the bootstrap script again. +""" + +RUST_NOT_IN_PATH = """ +You have some rust files in %(cargo_bin)s +but they're not part of this shell's PATH. + +To add these to the PATH, edit your shell initialization +script, which may be called ~/.bashrc or ~/.bash_profile or +~/.profile, and add the following line: + + %(cmd)s + +Then restart your shell and run the bootstrap script again. +""" + +RUSTUP_OLD = """ +We found an executable called `rustup` which we normally use to install +and upgrade Rust programming language support, but we didn't understand +its output. It may be an old version, or not be the installer from +https://rustup.rs/ + +Please move it out of the way and run the bootstrap script again. +Or if you prefer and know how, use the current rustup to install +a compatible version of the Rust programming language yourself. +""" + +RUST_UPGRADE_FAILED = """ +We attempted to upgrade Rust to a modern version (%s or newer). +However, you appear to still have version %s. + +It's possible rustup failed. It's also possible the new Rust is not being +installed in the search path for this shell. Try creating a new shell and +run this bootstrapper again. + +If this continues to fail and you are sure you have a modern Rust on your +system, ensure it is on the $PATH and try again. If that fails, you'll need to +install Rust manually. + +We recommend the installer from https://rustup.rs/ for installing Rust, +but you may be able to get a recent enough version from a software install +tool or package manager on your system, or directly from https://rust-lang.org/ +""" + +BROWSER_ARTIFACT_MODE_MOZCONFIG = """ +# Automatically download and use compiled C++ components: +ac_add_options --enable-artifact-builds +""".strip() + +JS_MOZCONFIG_TEMPLATE = """\ +# Build only the SpiderMonkey JS test shell +ac_add_options --enable-project=js +""" + +# Upgrade Mercurial older than this. +# This should match the OLDEST_NON_LEGACY_VERSION in +# version-control-tools/hgext/configwizard/__init__.py. +MODERN_MERCURIAL_VERSION = Version("4.9") + +# Upgrade rust older than this. +MODERN_RUST_VERSION = Version(MINIMUM_RUST_VERSION) + + +class BaseBootstrapper(object): + """Base class for system bootstrappers.""" + + def __init__(self, no_interactive=False, no_system_changes=False): + self.package_manager_updated = False + self.no_interactive = no_interactive + self.no_system_changes = no_system_changes + self.state_dir = None + self.srcdir = None + + def validate_environment(self): + """ + Called once the current firefox checkout has been detected. + Platform-specific implementations should check the environment and offer advice/warnings + to the user, if necessary. + """ + + def suggest_install_distutils(self): + """Called if distutils.{sysconfig,spawn} can't be imported.""" + print( + "Does your distro require installing another package for distutils?", + file=sys.stderr, + ) + + def suggest_install_pip3(self): + """Called if pip3 can't be found.""" + print( + "Try installing pip3 with your system's package manager.", file=sys.stderr + ) + + def install_system_packages(self): + """ + Install packages shared by all applications. These are usually + packages required by the development (like mercurial) or the + build system (like autoconf). + """ + raise NotImplementedError( + "%s must implement install_system_packages()" % __name__ + ) + + def install_browser_packages(self, mozconfig_builder): + """ + Install packages required to build Firefox for Desktop (application + 'browser'). + """ + raise NotImplementedError( + "Cannot bootstrap Firefox for Desktop: " + "%s does not yet implement install_browser_packages()" % __name__ + ) + + def ensure_browser_packages(self): + """ + Install pre-built packages needed to build Firefox for Desktop (application 'browser') + + Currently this is not needed and kept for compatibility with Firefox for Android. + """ + pass + + def ensure_js_packages(self): + """ + Install pre-built packages needed to build SpiderMonkey JavaScript Engine + + Currently this is not needed and kept for compatibility with Firefox for Android. + """ + pass + + def ensure_browser_artifact_mode_packages(self): + """ + Install pre-built packages needed to build Firefox for Desktop (application 'browser') + + Currently this is not needed and kept for compatibility with Firefox for Android. + """ + pass + + def generate_browser_mozconfig(self): + """ + Print a message to the console detailing what the user's mozconfig + should contain. + + Firefox for Desktop can in simple cases determine its build environment + entirely from configure. + """ + pass + + def install_js_packages(self, mozconfig_builder): + """ + Install packages required to build SpiderMonkey JavaScript Engine + (application 'js'). + """ + return self.install_browser_packages(mozconfig_builder) + + def generate_js_mozconfig(self): + """ + Create a reasonable starting point for a JS shell build. + """ + return JS_MOZCONFIG_TEMPLATE + + def install_browser_artifact_mode_packages(self, mozconfig_builder): + """ + Install packages required to build Firefox for Desktop (application + 'browser') in Artifact Mode. + """ + raise NotImplementedError( + "Cannot bootstrap Firefox for Desktop Artifact Mode: " + "%s does not yet implement install_browser_artifact_mode_packages()" + % __name__ + ) + + def generate_browser_artifact_mode_mozconfig(self): + """ + Print a message to the console detailing what the user's mozconfig + should contain. + + Firefox for Desktop Artifact Mode needs to enable artifact builds and + a path where the build artifacts will be written to. + """ + return BROWSER_ARTIFACT_MODE_MOZCONFIG + + def install_mobile_android_packages(self, mozconfig_builder): + """ + Install packages required to build GeckoView (application + 'mobile/android'). + """ + raise NotImplementedError( + "Cannot bootstrap GeckoView/Firefox for Android: " + "%s does not yet implement install_mobile_android_packages()" % __name__ + ) + + def ensure_mobile_android_packages(self): + """ + Install pre-built packages required to run GeckoView (application 'mobile/android') + """ + raise NotImplementedError( + "Cannot bootstrap GeckoView/Firefox for Android: " + "%s does not yet implement ensure_mobile_android_packages()" % __name__ + ) + + def ensure_mobile_android_artifact_mode_packages(self): + """ + Install pre-built packages required to run GeckoView Artifact Build + (application 'mobile/android') + """ + self.ensure_mobile_android_packages() + + def generate_mobile_android_mozconfig(self): + """ + Print a message to the console detailing what the user's mozconfig + should contain. + + GeckoView/Firefox for Android needs an application and an ABI set, and it needs + paths to the Android SDK and NDK. + """ + raise NotImplementedError( + "%s does not yet implement generate_mobile_android_mozconfig()" % __name__ + ) + + def install_mobile_android_artifact_mode_packages(self, mozconfig_builder): + """ + Install packages required to build GeckoView/Firefox for Android (application + 'mobile/android', also known as Fennec) in Artifact Mode. + """ + raise NotImplementedError( + "Cannot bootstrap GeckoView/Firefox for Android Artifact Mode: " + "%s does not yet implement install_mobile_android_artifact_mode_packages()" + % __name__ + ) + + def generate_mobile_android_artifact_mode_mozconfig(self): + """ + Print a message to the console detailing what the user's mozconfig + should contain. + + GeckoView/Firefox for Android Artifact Mode needs an application and an ABI set, + and it needs paths to the Android SDK. + """ + raise NotImplementedError( + "%s does not yet implement generate_mobile_android_artifact_mode_mozconfig()" + % __name__ + ) + + def ensure_sccache_packages(self): + """ + Install sccache. + """ + pass + + def install_toolchain_artifact(self, toolchain_job, no_unpack=False): + if no_unpack: + return self.install_toolchain_artifact_impl( + self.state_dir, toolchain_job, no_unpack + ) + bootstrap_toolchain(toolchain_job) + + def install_toolchain_artifact_impl( + self, install_dir: Path, toolchain_job, no_unpack=False + ): + if type(self.srcdir) is str: + mach_binary = Path(self.srcdir) / "mach" + else: + mach_binary = (self.srcdir / "mach").resolve() + if not mach_binary.exists(): + raise ValueError(f"mach not found at {mach_binary}") + + if not self.state_dir: + raise ValueError( + "Need a state directory (e.g. ~/.mozbuild) to download " "artifacts" + ) + python_location = get_mach_virtualenv_binary() + if not python_location.exists(): + raise ValueError(f"python not found at {python_location}") + + cmd = [ + str(python_location), + str(mach_binary), + "artifact", + "toolchain", + "--bootstrap", + "--from-build", + toolchain_job, + ] + + if no_unpack: + cmd += ["--no-unpack"] + + subprocess.check_call(cmd, cwd=str(install_dir)) + + def auto_bootstrap(self, application, exclude=[]): + args = ["--with-ccache=sccache"] + if application.endswith("_artifact_mode"): + args.append("--enable-artifact-builds") + application = application[: -len("_artifact_mode")] + args.append("--enable-project={}".format(application.replace("_", "/"))) + if exclude: + args.append( + "--enable-bootstrap={}".format(",".join(f"-{x}" for x in exclude)) + ) + bootstrap_all_toolchains_for(args) + + def run_as_root(self, command, may_use_sudo=True): + if os.geteuid() != 0: + if may_use_sudo and which("sudo"): + command.insert(0, "sudo") + else: + command = ["su", "root", "-c", " ".join(command)] + + print("Executing as root:", subprocess.list2cmdline(command)) + + subprocess.check_call(command, stdin=sys.stdin) + + def prompt_int(self, prompt, low, high, default=None): + """Prompts the user with prompt and requires an integer between low and high. + + If the user doesn't select an option and a default isn't provided, then + the lowest option is used. This is because some option must be implicitly + selected if mach is invoked with "--no-interactive" + """ + if default is not None: + assert isinstance(default, int) + assert low <= default <= high + else: + default = low + + if self.no_interactive: + print(prompt) + print('Selecting "{}" because context is not interactive.'.format(default)) + return default + + while True: + choice = input(prompt) + if choice == "" and default is not None: + return default + try: + choice = int(choice) + if low <= choice <= high: + return choice + except ValueError: + pass + print("ERROR! Please enter a valid option!") + + def prompt_yesno(self, prompt): + """Prompts the user with prompt and requires a yes/no answer.""" + if self.no_interactive: + print(prompt) + print('Selecting "Y" because context is not interactive.') + return True + + while True: + choice = input(prompt + " (Yn): ").strip().lower()[:1] + if choice == "": + return True + elif choice in ("y", "n"): + return choice == "y" + + print("ERROR! Please enter y or n!") + + def _ensure_package_manager_updated(self): + if self.package_manager_updated: + return + + self._update_package_manager() + self.package_manager_updated = True + + def _update_package_manager(self): + """Updates the package manager's manifests/package list. + + This should be defined in child classes. + """ + + def _parse_version_impl(self, path: Path, name, env, version_param): + """Execute the given path, returning the version. + + Invokes the path argument with the --version switch + and returns a Version representing the output + if successful. If not, returns None. + + An optional name argument gives the expected program + name returned as part of the version string, if it's + different from the basename of the executable. + + An optional env argument allows modifying environment + variable during the invocation to set options, PATH, + etc. + """ + if not name: + name = path.name + if name.lower().endswith(".exe"): + name = name[:-4] + + process = subprocess.run( + [str(path), version_param], + env=env, + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + if process.returncode != 0: + # This can happen e.g. if the user has an inactive pyenv shim in + # their path. Just silently treat this as a failure to parse the + # path and move on. + return None + + match = re.search(name + " ([a-z0-9\.]+)", process.stdout) + if not match: + print("ERROR! Unable to identify %s version." % name) + return None + + return Version(match.group(1)) + + def _parse_version(self, path: Path, name=None, env=None): + return self._parse_version_impl(path, name, env, "--version") + + def _hg_cleanenv(self, load_hgrc=False): + """Returns a copy of the current environment updated with the HGPLAIN + and HGRCPATH environment variables. + + HGPLAIN prevents Mercurial from applying locale variations to the output + making it suitable for use in scripts. + + HGRCPATH controls the loading of hgrc files. Setting it to the empty + string forces that no user or system hgrc file is used. + """ + env = os.environ.copy() + env["HGPLAIN"] = "1" + if not load_hgrc: + env["HGRCPATH"] = "" + + return env + + def is_mercurial_modern(self): + hg = to_optional_path(which("hg")) + if not hg: + print(NO_MERCURIAL) + return False, False, None + + our = self._parse_version(hg, "version", self._hg_cleanenv()) + if not our: + return True, False, None + + return True, our >= MODERN_MERCURIAL_VERSION, our + + def ensure_mercurial_modern(self): + installed, modern, version = self.is_mercurial_modern() + + if modern: + print("Your version of Mercurial (%s) is sufficiently modern." % version) + return installed, modern + + self._ensure_package_manager_updated() + + if installed: + print("Your version of Mercurial (%s) is not modern enough." % version) + print( + "(Older versions of Mercurial have known security vulnerabilities. " + "Unless you are running a patched Mercurial version, you may be " + "vulnerable." + ) + else: + print("You do not have Mercurial installed") + + if self.upgrade_mercurial(version) is False: + return installed, modern + + installed, modern, after = self.is_mercurial_modern() + + if installed and not modern: + print(MERCURIAL_UPGRADE_FAILED % (MODERN_MERCURIAL_VERSION, after)) + + return installed, modern + + def upgrade_mercurial(self, current): + """Upgrade Mercurial. + + Child classes should reimplement this. + + Return False to not perform a version check after the upgrade is + performed. + """ + print(MERCURIAL_UNABLE_UPGRADE % (current, MODERN_MERCURIAL_VERSION)) + + def warn_if_pythonpath_is_set(self): + if "PYTHONPATH" in os.environ: + print( + "WARNING: Your PYTHONPATH environment variable is set. This can " + "cause flaky installations of the requirements, and other unexpected " + "issues with mach. It is recommended to unset this variable." + ) + + def is_rust_modern(self, cargo_bin: Path): + rustc = to_optional_path(which("rustc", extra_search_dirs=[str(cargo_bin)])) + if not rustc: + print("Could not find a Rust compiler.") + return False, None + + our = self._parse_version(rustc) + if not our: + return False, None + + return our >= MODERN_RUST_VERSION, our + + def cargo_home(self): + cargo_home = Path(os.environ.get("CARGO_HOME", Path("~/.cargo").expanduser())) + cargo_bin = cargo_home / "bin" + return cargo_home, cargo_bin + + def print_rust_path_advice(self, template, cargo_home: Path, cargo_bin: Path): + # Suggest ~/.cargo/env if it exists. + if (cargo_home / "env").exists(): + cmd = f"source {cargo_home}/env" + else: + # On Windows rustup doesn't write out ~/.cargo/env + # so fall back to a manual PATH update. Bootstrap + # only runs under msys, so a unix-style shell command + # is appropriate there. + cargo_bin = win_to_msys_path(cargo_bin) + cmd = f"export PATH={cargo_bin}:$PATH" + print(template % {"cargo_bin": cargo_bin, "cmd": cmd}) + + def ensure_rust_modern(self): + cargo_home, cargo_bin = self.cargo_home() + modern, version = self.is_rust_modern(cargo_bin) + + rustup = to_optional_path(which("rustup", extra_search_dirs=[str(cargo_bin)])) + + if modern: + print("Your version of Rust (%s) is new enough." % version) + + elif version: + print("Your version of Rust (%s) is too old." % version) + + if rustup and not modern: + rustup_version = self._parse_version(rustup) + if not rustup_version: + print(RUSTUP_OLD) + sys.exit(1) + print("Found rustup. Will try to upgrade.") + self.upgrade_rust(rustup) + + modern, after = self.is_rust_modern(cargo_bin) + if not modern: + print(RUST_UPGRADE_FAILED % (MODERN_RUST_VERSION, after)) + sys.exit(1) + elif not rustup: + # No rustup. Download and run the installer. + print("Will try to install Rust.") + self.install_rust() + modern, version = self.is_rust_modern(cargo_bin) + rustup = to_optional_path( + which("rustup", extra_search_dirs=[str(cargo_bin)]) + ) + + self.ensure_rust_targets(rustup, version) + + def ensure_rust_targets(self, rustup: Path, rust_version): + """Make sure appropriate cross target libraries are installed.""" + target_list = subprocess.check_output( + [str(rustup), "target", "list"], universal_newlines=True + ) + targets = [ + line.split()[0] + for line in target_list.splitlines() + if "installed" in line or "default" in line + ] + print("Rust supports %s targets." % ", ".join(targets)) + + # Support 32-bit Windows on 64-bit Windows. + win32 = "i686-pc-windows-msvc" + win64 = "x86_64-pc-windows-msvc" + if rust.platform() == win64 and win32 not in targets: + subprocess.check_call([str(rustup), "target", "add", win32]) + + if "mobile_android" in self.application: + # Let's add the most common targets. + if rust_version < Version("1.33"): + arm_target = "armv7-linux-androideabi" + else: + arm_target = "thumbv7neon-linux-androideabi" + android_targets = ( + arm_target, + "aarch64-linux-android", + "i686-linux-android", + "x86_64-linux-android", + ) + for target in android_targets: + if target not in targets: + subprocess.check_call([str(rustup), "target", "add", target]) + + def upgrade_rust(self, rustup: Path): + """Upgrade Rust. + + Invoke rustup from the given path to update the rust install.""" + subprocess.check_call([str(rustup), "update"]) + # This installs rustfmt when not already installed, or nothing + # otherwise, while the update above would have taken care of upgrading + # it. + subprocess.check_call([str(rustup), "component", "add", "rustfmt"]) + + def install_rust(self): + """Download and run the rustup installer.""" + import errno + import stat + import tempfile + + platform = rust.platform() + url = rust.rustup_url(platform) + checksum = rust.rustup_hash(platform) + if not url or not checksum: + print("ERROR: Could not download installer.") + sys.exit(1) + print("Downloading rustup-init... ", end="") + fd, rustup_init = tempfile.mkstemp(prefix=Path(url).name) + rustup_init = Path(rustup_init) + os.close(fd) + try: + http_download_and_save(url, rustup_init, checksum) + mode = rustup_init.stat().st_mode + rustup_init.chmod(mode | stat.S_IRWXU) + print("Ok") + print("Running rustup-init...") + subprocess.check_call( + [ + str(rustup_init), + "-y", + "--default-toolchain", + "stable", + "--default-host", + platform, + "--component", + "rustfmt", + ] + ) + cargo_home, cargo_bin = self.cargo_home() + self.print_rust_path_advice(RUST_INSTALL_COMPLETE, cargo_home, cargo_bin) + finally: + try: + rustup_init.unlink() + except OSError as e: + if e.errno != errno.ENOENT: + raise diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootstrap.py new file mode 100644 index 0000000000..e57f496f29 --- /dev/null +++ b/python/mozboot/mozboot/bootstrap.py @@ -0,0 +1,776 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import platform +import re +import shutil +import stat +import subprocess +import sys +import time +from collections import OrderedDict +from pathlib import Path +from typing import Optional + +# Use distro package to retrieve linux platform information +import distro +from mach.site import MachSiteManager +from mach.telemetry import initialize_telemetry_setting +from mach.util import ( + UserError, + get_state_dir, + to_optional_path, + to_optional_str, + win_to_msys_path, +) +from mozbuild.base import MozbuildObject +from mozfile import which +from packaging.version import Version + +from mozboot.archlinux import ArchlinuxBootstrapper +from mozboot.base import MODERN_RUST_VERSION +from mozboot.centosfedora import CentOSFedoraBootstrapper +from mozboot.debian import DebianBootstrapper +from mozboot.freebsd import FreeBSDBootstrapper +from mozboot.gentoo import GentooBootstrapper +from mozboot.mozconfig import MozconfigBuilder +from mozboot.mozillabuild import MozillaBuildBootstrapper +from mozboot.openbsd import OpenBSDBootstrapper +from mozboot.opensuse import OpenSUSEBootstrapper +from mozboot.osx import OSXBootstrapper, OSXBootstrapperLight +from mozboot.solus import SolusBootstrapper +from mozboot.void import VoidBootstrapper +from mozboot.windows import WindowsBootstrapper + +APPLICATION_CHOICE = """ +Note on Artifact Mode: + +Artifact builds download prebuilt C++ components rather than building +them locally. Artifact builds are faster! + +Artifact builds are recommended for people working on Firefox or +Firefox for Android frontends, or the GeckoView Java API. They are unsuitable +for those working on C++ code. For more information see: +https://firefox-source-docs.mozilla.org/contributing/build/artifact_builds.html. + +Please choose the version of Firefox you want to build (see note above): +%s +Your choice: """ + +APPLICATIONS = OrderedDict( + [ + ("Firefox for Desktop Artifact Mode", "browser_artifact_mode"), + ("Firefox for Desktop", "browser"), + ("GeckoView/Firefox for Android Artifact Mode", "mobile_android_artifact_mode"), + ("GeckoView/Firefox for Android", "mobile_android"), + ("SpiderMonkey JavaScript engine", "js"), + ] +) + +FINISHED = """ +Your system should be ready to build %s! +""" + +MOZCONFIG_SUGGESTION_TEMPLATE = """ +Paste the lines between the chevrons (>>> and <<<) into +%s: + +>>> +%s +<<< +""" + +MOZCONFIG_MISMATCH_WARNING_TEMPLATE = """ +WARNING! Mismatch detected between the selected build target and the +mozconfig file %s: + +Current config +>>> +%s +<<< + +Expected config +>>> +%s +<<< +""" + +CONFIGURE_MERCURIAL = """ +Mozilla recommends a number of changes to Mercurial to enhance your +experience with it. + +Would you like to run a configuration wizard to ensure Mercurial is +optimally configured? (This will also ensure 'version-control-tools' is up-to-date)""" + +CONFIGURE_GIT = """ +Mozilla recommends using git-cinnabar to work with mozilla-central (or +mozilla-unified). + +Would you like to run a few configuration steps to ensure Git is +optimally configured?""" + +DEBIAN_DISTROS = ( + "debian", + "ubuntu", + "linuxmint", + "elementary", + "neon", + "pop", + "kali", + "devuan", + "pureos", + "deepin", + "tuxedo", +) + +FEDORA_DISTROS = ( + "centos", + "fedora", + "rocky", + "oracle", +) + +ADD_GIT_CINNABAR_PATH = """ +To add git-cinnabar to the PATH, edit your shell initialization script, which +may be called {prefix}/.bash_profile or {prefix}/.profile, and add the following +lines: + + export PATH="{cinnabar_dir}:$PATH" + +Then restart your shell. +""" + + +OLD_REVISION_WARNING = """ +WARNING! You appear to be running `mach bootstrap` from an old revision. +bootstrap is meant primarily for getting developer environments up-to-date to +build the latest version of tree. Running bootstrap on old revisions may fail +and is not guaranteed to bring your machine to any working state in particular. +Proceed at your own peril. +""" + + +# Version 2.24 changes the "core.commitGraph" setting to be "True" by default. +MINIMUM_RECOMMENDED_GIT_VERSION = Version("2.24") +OLD_GIT_WARNING = """ +You are running an older version of git ("{old_version}"). +We recommend upgrading to at least version "{minimum_recommended_version}" to improve +performance. +""".strip() + + +class Bootstrapper(object): + """Main class that performs system bootstrap.""" + + def __init__( + self, + choice=None, + no_interactive=False, + hg_configure=False, + no_system_changes=False, + exclude=[], + mach_context=None, + ): + self.instance = None + self.choice = choice + self.hg_configure = hg_configure + self.no_system_changes = no_system_changes + self.exclude = exclude + self.mach_context = mach_context + cls = None + args = { + "no_interactive": no_interactive, + "no_system_changes": no_system_changes, + } + + if sys.platform.startswith("linux"): + # distro package provides reliable ids for popular distributions so + # we use those instead of the full distribution name + dist_id, version, codename = distro.linux_distribution( + full_distribution_name=False + ) + + if dist_id in FEDORA_DISTROS: + cls = CentOSFedoraBootstrapper + args["distro"] = dist_id + elif dist_id in DEBIAN_DISTROS: + cls = DebianBootstrapper + args["distro"] = dist_id + args["codename"] = codename + elif dist_id in ("gentoo", "funtoo"): + cls = GentooBootstrapper + elif dist_id in ("solus"): + cls = SolusBootstrapper + elif dist_id in ("arch") or Path("/etc/arch-release").exists(): + cls = ArchlinuxBootstrapper + elif dist_id in ("void"): + cls = VoidBootstrapper + elif dist_id in ( + "opensuse", + "opensuse-leap", + "opensuse-tumbleweed", + "suse", + ): + cls = OpenSUSEBootstrapper + else: + raise NotImplementedError( + "Bootstrap support for this Linux " + "distro not yet available: " + dist_id + ) + + args["version"] = version + args["dist_id"] = dist_id + + elif sys.platform.startswith("darwin"): + # TODO Support Darwin platforms that aren't OS X. + osx_version = platform.mac_ver()[0] + if platform.machine() == "arm64" or _macos_is_running_under_rosetta(): + cls = OSXBootstrapperLight + else: + cls = OSXBootstrapper + args["version"] = osx_version + + elif sys.platform.startswith("openbsd"): + cls = OpenBSDBootstrapper + args["version"] = platform.uname()[2] + + elif sys.platform.startswith(("dragonfly", "freebsd", "netbsd")): + cls = FreeBSDBootstrapper + args["version"] = platform.release() + args["flavor"] = platform.system() + + elif sys.platform.startswith("win32") or sys.platform.startswith("msys"): + if "MOZILLABUILD" in os.environ: + cls = MozillaBuildBootstrapper + else: + cls = WindowsBootstrapper + if cls is None: + raise NotImplementedError( + "Bootstrap support is not yet available " "for your OS." + ) + + self.instance = cls(**args) + + def maybe_install_private_packages_or_exit(self, application, checkout_type): + # Install the clang packages needed for building the style system, as + # well as the version of NodeJS that we currently support. + # Also install the clang static-analysis package by default + # The best place to install our packages is in the state directory + # we have. We should have created one above in non-interactive mode. + self.instance.auto_bootstrap(application, self.exclude) + self.instance.install_toolchain_artifact("fix-stacks") + self.instance.install_toolchain_artifact("minidump-stackwalk") + if not self.instance.artifact_mode: + self.instance.install_toolchain_artifact("clang-tools/clang-tidy") + self.instance.ensure_sccache_packages() + # Like 'ensure_browser_packages' or 'ensure_mobile_android_packages' + getattr(self.instance, "ensure_%s_packages" % application)() + + def check_code_submission(self, checkout_root: Path): + if self.instance.no_interactive or which("moz-phab"): + return + + # Skip moz-phab install until bug 1696357 is fixed and makes it to a moz-phab + # release. + if sys.platform.startswith("darwin") and platform.machine() == "arm64": + return + + if not self.instance.prompt_yesno("Will you be submitting commits to Mozilla?"): + return + + mach_binary = checkout_root / "mach" + subprocess.check_call((sys.executable, str(mach_binary), "install-moz-phab")) + + def bootstrap(self, settings): + if self.choice is None: + applications = APPLICATIONS + # Like ['1. Firefox for Desktop', '2. Firefox for Android Artifact Mode', ...]. + labels = [ + "%s. %s" % (i, name) for i, name in enumerate(applications.keys(), 1) + ] + choices = [" {} [default]".format(labels[0])] + choices += [" {}".format(label) for label in labels[1:]] + prompt = APPLICATION_CHOICE % "\n".join(choices) + prompt_choice = self.instance.prompt_int( + prompt=prompt, low=1, high=len(applications) + ) + name, application = list(applications.items())[prompt_choice - 1] + elif self.choice in APPLICATIONS.keys(): + name, application = self.choice, APPLICATIONS[self.choice] + elif self.choice in APPLICATIONS.values(): + name, application = next( + (k, v) for k, v in APPLICATIONS.items() if v == self.choice + ) + else: + raise Exception( + "Please pick a valid application choice: (%s)" + % "/".join(APPLICATIONS.keys()) + ) + + mozconfig_builder = MozconfigBuilder() + self.instance.application = application + self.instance.artifact_mode = "artifact_mode" in application + + self.instance.warn_if_pythonpath_is_set() + + if sys.platform.startswith("darwin") and not os.environ.get( + "MACH_I_DO_WANT_TO_USE_ROSETTA" + ): + # If running on arm64 mac, check whether we're running under + # Rosetta and advise against it. + if _macos_is_running_under_rosetta(): + print( + "Python is being emulated under Rosetta. Please use a native " + "Python instead. If you still really want to go ahead, set " + "the MACH_I_DO_WANT_TO_USE_ROSETTA environment variable.", + file=sys.stderr, + ) + return 1 + + state_dir = Path(get_state_dir()) + self.instance.state_dir = state_dir + + hg = to_optional_path(which("hg")) + + # We need to enable the loading of hgrc in case extensions are + # required to open the repo. + (checkout_type, checkout_root) = current_firefox_checkout( + env=self.instance._hg_cleanenv(load_hgrc=True), + hg=hg, + ) + self.instance.srcdir = checkout_root + self.instance.validate_environment() + self._validate_python_environment(checkout_root) + + if self.instance.no_system_changes: + self.maybe_install_private_packages_or_exit(application, checkout_type) + self._output_mozconfig(application, mozconfig_builder) + sys.exit(0) + + self.instance.install_system_packages() + + # Like 'install_browser_packages' or 'install_mobile_android_packages'. + getattr(self.instance, "install_%s_packages" % application)(mozconfig_builder) + + if not self.instance.artifact_mode: + self.instance.ensure_rust_modern() + + git = to_optional_path(which("git")) + + # Possibly configure Mercurial, but not if the current checkout or repo + # type is Git. + hg_installed = bool(hg) + if checkout_type == "hg": + hg_installed, hg_modern = self.instance.ensure_mercurial_modern() + + if hg_installed and checkout_type == "hg": + if not self.instance.no_interactive: + configure_hg = self.instance.prompt_yesno(prompt=CONFIGURE_MERCURIAL) + else: + configure_hg = self.hg_configure + + if configure_hg: + configure_mercurial(hg, state_dir) + + # Offer to configure Git, if the current checkout or repo type is Git. + elif git and checkout_type == "git": + should_configure_git = False + if not self.instance.no_interactive: + should_configure_git = self.instance.prompt_yesno(prompt=CONFIGURE_GIT) + else: + # Assuming default configuration setting applies to all VCS. + should_configure_git = self.hg_configure + + if should_configure_git: + configure_git( + git, + to_optional_path(which("git-cinnabar")), + state_dir, + checkout_root, + ) + + self.maybe_install_private_packages_or_exit(application, checkout_type) + self.check_code_submission(checkout_root) + # Wait until after moz-phab setup to check telemetry so that employees + # will be automatically opted-in. + if not self.instance.no_interactive and not settings.mach_telemetry.is_set_up: + initialize_telemetry_setting(settings, str(checkout_root), str(state_dir)) + + self._output_mozconfig(application, mozconfig_builder) + + print(FINISHED % name) + if not ( + which("rustc") + and self.instance._parse_version(Path("rustc")) >= MODERN_RUST_VERSION + ): + print( + "To build %s, please restart the shell (Start a new terminal window)" + % name + ) + + def _default_mozconfig_path(self): + return Path(self.mach_context.topdir) / "mozconfig" + + def _read_default_mozconfig(self): + path = self._default_mozconfig_path() + with open(path, "r") as mozconfig_file: + return mozconfig_file.read() + + def _write_default_mozconfig(self, raw_mozconfig): + path = self._default_mozconfig_path() + with open(path, "w") as mozconfig_file: + mozconfig_file.write(raw_mozconfig) + print(f'Your requested configuration has been written to "{path}".') + + def _show_mozconfig_suggestion(self, raw_mozconfig): + suggestion = MOZCONFIG_SUGGESTION_TEMPLATE % ( + self._default_mozconfig_path(), + raw_mozconfig, + ) + print(suggestion, end="") + + def _check_default_mozconfig_mismatch( + self, current_mozconfig_info, expected_application, expected_raw_mozconfig + ): + current_raw_mozconfig = self._read_default_mozconfig() + current_application = current_mozconfig_info["project"][0].replace("/", "_") + if current_mozconfig_info["artifact-builds"]: + current_application += "_artifact_mode" + + if expected_application == current_application: + if expected_raw_mozconfig == current_raw_mozconfig: + return + + # There's minor difference, show the suggestion. + self._show_mozconfig_suggestion(expected_raw_mozconfig) + return + + warning = MOZCONFIG_MISMATCH_WARNING_TEMPLATE % ( + self._default_mozconfig_path(), + current_raw_mozconfig, + expected_raw_mozconfig, + ) + print(warning) + + if not self.instance.prompt_yesno("Do you want to overwrite the config?"): + return + + self._write_default_mozconfig(expected_raw_mozconfig) + + def _output_mozconfig(self, application, mozconfig_builder): + # Like 'generate_browser_mozconfig' or 'generate_mobile_android_mozconfig'. + additional_mozconfig = getattr( + self.instance, "generate_%s_mozconfig" % application + )() + if additional_mozconfig: + mozconfig_builder.append(additional_mozconfig) + raw_mozconfig = mozconfig_builder.generate() + + current_mozconfig_info = MozbuildObject.get_base_mozconfig_info( + self.mach_context.topdir, None, "" + ) + current_mozconfig_path = current_mozconfig_info["mozconfig"]["path"] + + if current_mozconfig_path: + # mozconfig file exists + if self._default_mozconfig_path().exists() and Path.samefile( + Path(current_mozconfig_path), self._default_mozconfig_path() + ): + # This mozconfig file may be created by bootstrap. + self._check_default_mozconfig_mismatch( + current_mozconfig_info, application, raw_mozconfig + ) + elif raw_mozconfig: + # The mozconfig file is created by user. + self._show_mozconfig_suggestion(raw_mozconfig) + elif raw_mozconfig: + # No mozconfig file exists yet + self._write_default_mozconfig(raw_mozconfig) + + def _validate_python_environment(self, topsrcdir): + valid = True + try: + # distutils is singled out here because some distros (namely Ubuntu) + # include it in a separate package outside of the main Python + # installation. + import distutils.spawn + import distutils.sysconfig + + assert distutils.sysconfig is not None and distutils.spawn is not None + except ImportError as e: + print("ERROR: Could not import package %s" % e.name, file=sys.stderr) + self.instance.suggest_install_distutils() + valid = False + except AssertionError: + print("ERROR: distutils is not behaving as expected.", file=sys.stderr) + self.instance.suggest_install_distutils() + valid = False + pip3 = to_optional_path(which("pip3")) + if not pip3: + print("ERROR: Could not find pip3.", file=sys.stderr) + self.instance.suggest_install_pip3() + valid = False + if not valid: + print( + "ERROR: Your Python installation will not be able to run " + "`mach bootstrap`. `mach bootstrap` cannot maintain your " + "Python environment for you; fix the errors shown here, and " + "then re-run `mach bootstrap`.", + file=sys.stderr, + ) + sys.exit(1) + + mach_site = MachSiteManager.from_environment( + topsrcdir, + lambda: os.path.normpath(get_state_dir(True, topsrcdir=topsrcdir)), + ) + mach_site.attempt_populate_optional_packages() + + +def update_vct(hg: Path, root_state_dir: Path): + """Ensure version-control-tools in the state directory is up to date.""" + vct_dir = root_state_dir / "version-control-tools" + + # Ensure the latest revision of version-control-tools is present. + update_mercurial_repo( + hg, "https://hg.mozilla.org/hgcustom/version-control-tools", vct_dir, "@" + ) + + return vct_dir + + +def configure_mercurial(hg: Optional[Path], root_state_dir: Path): + """Run the Mercurial configuration wizard.""" + vct_dir = update_vct(hg, root_state_dir) + + hg = to_optional_str(hg) + + # Run the config wizard from v-c-t. + args = [ + hg, + "--config", + f"extensions.configwizard={vct_dir}/hgext/configwizard", + "configwizard", + ] + subprocess.call(args) + + +def update_mercurial_repo(hg: Path, url, dest: Path, revision): + """Perform a clone/pull + update of a Mercurial repository.""" + # Disable common extensions whose older versions may cause `hg` + # invocations to abort. + pull_args = [str(hg)] + if dest.exists(): + pull_args.extend(["pull", url]) + cwd = dest + else: + pull_args.extend(["clone", "--noupdate", url, str(dest)]) + cwd = "/" + + update_args = [str(hg), "update", "-r", revision] + + print("=" * 80) + print(f"Ensuring {url} is up to date at {dest}") + + env = os.environ.copy() + env.update({"HGPLAIN": "1"}) + + try: + subprocess.check_call(pull_args, cwd=str(cwd), env=env) + subprocess.check_call(update_args, cwd=str(dest), env=env) + finally: + print("=" * 80) + + +def current_firefox_checkout(env, hg: Optional[Path] = None): + """Determine whether we're in a Firefox checkout. + + Returns one of None, ``git``, or ``hg``. + """ + HG_ROOT_REVISIONS = set( + [ + # From mozilla-unified. + "8ba995b74e18334ab3707f27e9eb8f4e37ba3d29" + ] + ) + + path = Path.cwd() + while path: + hg_dir = path / ".hg" + git_dir = path / ".git" + known_file = path / "config" / "milestone.txt" + if hg and hg_dir.exists(): + # Verify the hg repo is a Firefox repo by looking at rev 0. + try: + node = subprocess.check_output( + [str(hg), "log", "-r", "0", "--template", "{node}"], + cwd=str(path), + env=env, + universal_newlines=True, + ) + if node in HG_ROOT_REVISIONS: + _warn_if_risky_revision(path) + return "hg", path + # Else the root revision is different. There could be nested + # repos. So keep traversing the parents. + except subprocess.CalledProcessError: + pass + + # Just check for known-good files in the checkout, to prevent attempted + # foot-shootings. Determining a canonical git checkout of mozilla-unified + # is...complicated + elif git_dir.exists() or hg_dir.exists(): + if known_file.exists(): + _warn_if_risky_revision(path) + return ("git" if git_dir.exists() else "hg"), path + elif known_file.exists(): + return "SOURCE", path + + if not len(path.parents): + break + path = path.parent + + raise UserError( + "Could not identify the root directory of your checkout! " + "Are you running `mach bootstrap` in an hg or git clone?" + ) + + +def update_git_tools(git: Optional[Path], root_state_dir: Path): + """Update git tools, hooks and extensions""" + # Ensure git-cinnabar is up to date. + cinnabar_dir = root_state_dir / "git-cinnabar" + cinnabar_exe = cinnabar_dir / "git-cinnabar" + + if sys.platform.startswith(("win32", "msys")): + cinnabar_exe = cinnabar_exe.with_suffix(".exe") + + # Previously, this script would do a full clone of the git-cinnabar + # repository. It now only downloads prebuilt binaries, so if we are + # updating from an old setup, remove the repository and start over. + if (cinnabar_dir / ".git").exists(): + # git sets pack files read-only, which causes problems removing + # them on Windows. To work around that, we use an error handler + # on rmtree that retries to remove the file after chmod'ing it. + def onerror(func, path, exc): + if func == os.unlink: + os.chmod(path, stat.S_IRWXU) + func(path) + else: + raise + + shutil.rmtree(str(cinnabar_dir), onerror=onerror) + + # If we already have an executable, ask it to update itself. + exists = cinnabar_exe.exists() + if exists: + try: + subprocess.check_call([str(cinnabar_exe), "self-update"]) + except subprocess.CalledProcessError as e: + print(e) + + # git-cinnabar 0.6.0rc1 self-update had a bug that could leave an empty + # file. If that happens, install from scratch. + if not exists or cinnabar_exe.stat().st_size == 0: + from urllib.request import urlopen + + import certifi + + if not cinnabar_dir.exists(): + cinnabar_dir.mkdir() + + cinnabar_url = "https://github.com/glandium/git-cinnabar/" + download_py = cinnabar_dir / "download.py" + with open(download_py, "wb") as fh: + shutil.copyfileobj( + urlopen( + f"{cinnabar_url}/raw/master/download.py", cafile=certifi.where() + ), + fh, + ) + + try: + subprocess.check_call( + [sys.executable, str(download_py)], cwd=str(cinnabar_dir) + ) + except subprocess.CalledProcessError as e: + print(e) + finally: + download_py.unlink() + + return cinnabar_dir + + +def configure_git( + git: Optional[Path], + cinnabar: Optional[Path], + root_state_dir: Path, + top_src_dir: Path, +): + """Run the Git configuration steps.""" + + git_str = to_optional_str(git) + + match = re.search( + r"(\d+\.\d+\.\d+)", + subprocess.check_output([git_str, "--version"], universal_newlines=True), + ) + if not match: + raise Exception("Could not find git version") + git_version = Version(match.group(1)) + + if git_version < MINIMUM_RECOMMENDED_GIT_VERSION: + print( + OLD_GIT_WARNING.format( + old_version=git_version, + minimum_recommended_version=MINIMUM_RECOMMENDED_GIT_VERSION, + ) + ) + + if git_version >= Version("2.17"): + # "core.untrackedCache" has a bug before 2.17 + subprocess.check_call( + [git_str, "config", "core.untrackedCache", "true"], cwd=str(top_src_dir) + ) + + cinnabar_dir = str(update_git_tools(git, root_state_dir)) + + if not cinnabar: + if "MOZILLABUILD" in os.environ: + # Slightly modify the path on Windows to be correct + # for the copy/paste into the .bash_profile + cinnabar_dir = win_to_msys_path(cinnabar_dir) + + print( + ADD_GIT_CINNABAR_PATH.format( + prefix="%USERPROFILE%", cinnabar_dir=cinnabar_dir + ) + ) + else: + print(ADD_GIT_CINNABAR_PATH.format(prefix="~", cinnabar_dir=cinnabar_dir)) + + +def _warn_if_risky_revision(path: Path): + # Warn the user if they're trying to bootstrap from an obviously old + # version of tree as reported by the version control system (a month in + # this case). This is an approximate calculation but is probably good + # enough for our purposes. + NUM_SECONDS_IN_MONTH = 60 * 60 * 24 * 30 + from mozversioncontrol import get_repository_object + + repo = get_repository_object(path) + if (time.time() - repo.get_commit_time()) >= NUM_SECONDS_IN_MONTH: + print(OLD_REVISION_WARNING) + + +def _macos_is_running_under_rosetta(): + proc = subprocess.run( + ["sysctl", "-n", "sysctl.proc_translated"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + return ( + proc.returncode == 0 and proc.stdout.decode("ascii", "replace").strip() == "1" + ) diff --git a/python/mozboot/mozboot/centosfedora.py b/python/mozboot/mozboot/centosfedora.py new file mode 100644 index 0000000000..37aa0e8eaa --- /dev/null +++ b/python/mozboot/mozboot/centosfedora.py @@ -0,0 +1,80 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import subprocess + +from mozfile import which + +from mozboot.base import BaseBootstrapper +from mozboot.linux_common import LinuxBootstrapper + + +class CentOSFedoraBootstrapper(LinuxBootstrapper, BaseBootstrapper): + def __init__(self, distro, version, dist_id, **kwargs): + BaseBootstrapper.__init__(self, **kwargs) + + self.distro = distro + self.version = int(version.split(".")[0]) + self.dist_id = dist_id + + def install_packages(self, packages): + if self.version >= 33 and "perl" in packages: + packages.append("perl-FindBin") + # watchman is not available on centos/rocky + if self.distro in ("centos", "rocky", "oracle"): + packages = [p for p in packages if p != "watchman"] + self.dnf_install(*packages) + + def upgrade_mercurial(self, current): + if current is None: + self.dnf_install("mercurial") + else: + self.dnf_update("mercurial") + + def dnf_install(self, *packages): + if which("dnf"): + + def not_installed(package): + # We could check for "Error: No matching Packages to list", but + # checking `dnf`s exit code is sufficent. + # Ideally we'd invoke dnf with '--cacheonly', but there's: + # https://bugzilla.redhat.com/show_bug.cgi?id=2030255 + is_installed = subprocess.run( + ["dnf", "list", "--installed", package], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + if is_installed.returncode not in [0, 1]: + stdout = is_installed.stdout + raise Exception( + f'Failed to determine whether package "{package}" is installed: "{stdout}"' + ) + return is_installed.returncode != 0 + + packages = list(filter(not_installed, packages)) + if len(packages) == 0: + # avoid sudo prompt (support unattended re-bootstrapping) + return + + command = ["dnf", "install"] + else: + command = ["yum", "install"] + + if self.no_interactive: + command.append("-y") + command.extend(packages) + + self.run_as_root(command) + + def dnf_update(self, *packages): + if which("dnf"): + command = ["dnf", "update"] + else: + command = ["yum", "update"] + + if self.no_interactive: + command.append("-y") + command.extend(packages) + + self.run_as_root(command) diff --git a/python/mozboot/mozboot/debian.py b/python/mozboot/mozboot/debian.py new file mode 100644 index 0000000000..34e328586e --- /dev/null +++ b/python/mozboot/mozboot/debian.py @@ -0,0 +1,83 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys + +from mozboot.base import MERCURIAL_INSTALL_PROMPT, BaseBootstrapper +from mozboot.linux_common import LinuxBootstrapper + + +class DebianBootstrapper(LinuxBootstrapper, BaseBootstrapper): + def __init__(self, distro, version, dist_id, codename, **kwargs): + BaseBootstrapper.__init__(self, **kwargs) + + self.distro = distro + self.version = version + self.dist_id = dist_id + self.codename = codename + + def suggest_install_distutils(self): + print( + "HINT: Try installing distutils with " + "`apt-get install python3-distutils`.", + file=sys.stderr, + ) + + def suggest_install_pip3(self): + print( + "HINT: Try installing pip3 with `apt-get install python3-pip`.", + file=sys.stderr, + ) + + def install_packages(self, packages): + try: + if int(self.version) < 11: + # watchman is only available starting from Debian 11. + packages = [p for p in packages if p != "watchman"] + except ValueError: + pass + + self.apt_install(*packages) + + def _update_package_manager(self): + self.apt_update() + + def upgrade_mercurial(self, current): + """Install Mercurial from pip because Debian packages typically lag.""" + if self.no_interactive: + # Install via Apt in non-interactive mode because it is the more + # conservative option and less likely to make people upset. + self.apt_install("mercurial") + return + + res = self.prompt_int(MERCURIAL_INSTALL_PROMPT, 1, 3) + + # Apt. + if res == 2: + self.apt_install("mercurial") + return False + + # No Mercurial. + if res == 3: + print("Not installing Mercurial.") + return False + + # pip. + assert res == 1 + self.run_as_root(["pip3", "install", "--upgrade", "Mercurial"]) + + def apt_install(self, *packages): + command = ["apt-get", "install"] + if self.no_interactive: + command.append("-y") + command.extend(packages) + + self.run_as_root(command) + + def apt_update(self): + command = ["apt-get", "update"] + if self.no_interactive: + command.append("-y") + + self.run_as_root(command) diff --git a/python/mozboot/mozboot/freebsd.py b/python/mozboot/mozboot/freebsd.py new file mode 100644 index 0000000000..f4d6d1847b --- /dev/null +++ b/python/mozboot/mozboot/freebsd.py @@ -0,0 +1,70 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys + +from mozfile import which + +from mozboot.base import BaseBootstrapper + + +class FreeBSDBootstrapper(BaseBootstrapper): + def __init__(self, version, flavor, **kwargs): + BaseBootstrapper.__init__(self, **kwargs) + self.version = int(version.split(".")[0]) + self.flavor = flavor.lower() + + self.packages = [ + "gmake", + "gtar", + "m4", + "npm", + "pkgconf", + "py%d%d-sqlite3" % sys.version_info[0:2], + "rust", + "watchman", + ] + + self.browser_packages = [ + "dbus-glib", + "libXt", + "nasm", + "pulseaudio", + ] + + if not which("as"): + self.packages.append("binutils") + + if not which("unzip"): + self.packages.append("unzip") + + def pkg_install(self, *packages): + if sys.platform.startswith("netbsd"): + command = ["pkgin", "install"] + else: + command = ["pkg", "install"] + if self.no_interactive: + command.append("-y") + + command.extend(packages) + self.run_as_root(command) + + def install_system_packages(self): + self.pkg_install(*self.packages) + + def install_browser_packages(self, mozconfig_builder, artifact_mode=False): + # TODO: Figure out what not to install for artifact mode + packages = self.browser_packages.copy() + if not artifact_mode: + if sys.platform.startswith("netbsd"): + packages.extend(["brotli", "gtk3+", "libv4l", "cbindgen"]) + else: + packages.extend(["gtk3", "mesa-dri", "v4l_compat", "rust-cbindgen"]) + self.pkg_install(*packages) + + def install_browser_artifact_mode_packages(self, mozconfig_builder): + self.install_browser_packages(mozconfig_builder, artifact_mode=True) + + def upgrade_mercurial(self, current): + self.pkg_install("mercurial") diff --git a/python/mozboot/mozboot/gentoo.py b/python/mozboot/mozboot/gentoo.py new file mode 100644 index 0000000000..4ddf86696f --- /dev/null +++ b/python/mozboot/mozboot/gentoo.py @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozboot.base import BaseBootstrapper +from mozboot.linux_common import LinuxBootstrapper + + +class GentooBootstrapper(LinuxBootstrapper, BaseBootstrapper): + def __init__(self, version, dist_id, **kwargs): + BaseBootstrapper.__init__(self, **kwargs) + + self.version = version + self.dist_id = dist_id + + def install_packages(self, packages): + DISAMBIGUATE = { + "gzip": "app-arch/gzip", + "tar": "app-arch/tar", + } + # watchman is available but requires messing with USEs. + packages = [DISAMBIGUATE.get(p, p) for p in packages if p != "watchman"] + self.run_as_root(["emerge", "--noreplace"] + packages) + + def _update_package_manager(self): + self.run_as_root(["emerge", "--sync"]) + + def upgrade_mercurial(self, current): + self.run_as_root(["emerge", "--update", "mercurial"]) diff --git a/python/mozboot/mozboot/linux_common.py b/python/mozboot/mozboot/linux_common.py new file mode 100644 index 0000000000..c6751d333b --- /dev/null +++ b/python/mozboot/mozboot/linux_common.py @@ -0,0 +1,93 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# An easy way for distribution-specific bootstrappers to share the code +# needed to install Stylo and Node dependencies. This class must come before +# BaseBootstrapper in the inheritance list. + +import platform + + +def is_non_x86_64(): + return platform.machine() != "x86_64" + + +class MobileAndroidBootstrapper(object): + def __init__(self, **kwargs): + pass + + def install_mobile_android_packages(self, mozconfig_builder, artifact_mode=False): + from mozboot import android + + os_arch = platform.machine() + android.ensure_android( + "linux", + os_arch, + artifact_mode=artifact_mode, + no_interactive=self.no_interactive, + ) + android.ensure_android( + "linux", + os_arch, + artifact_mode=artifact_mode, + no_interactive=self.no_interactive, + system_images_only=True, + avd_manifest_path=android.AVD_MANIFEST_X86_64, + ) + android.ensure_android( + "linux", + os_arch, + artifact_mode=artifact_mode, + no_interactive=self.no_interactive, + system_images_only=True, + avd_manifest_path=android.AVD_MANIFEST_ARM, + ) + + def install_mobile_android_artifact_mode_packages(self, mozconfig_builder): + self.install_mobile_android_packages(mozconfig_builder, artifact_mode=True) + + def ensure_mobile_android_packages(self): + from mozboot import android + + android.ensure_java("linux", platform.machine()) + self.install_toolchain_artifact(android.LINUX_X86_64_ANDROID_AVD) + self.install_toolchain_artifact(android.LINUX_ARM_ANDROID_AVD) + + def generate_mobile_android_mozconfig(self, artifact_mode=False): + from mozboot import android + + return android.generate_mozconfig("linux", artifact_mode=artifact_mode) + + def generate_mobile_android_artifact_mode_mozconfig(self): + return self.generate_mobile_android_mozconfig(artifact_mode=True) + + +class LinuxBootstrapper(MobileAndroidBootstrapper): + def __init__(self, **kwargs): + pass + + def ensure_sccache_packages(self): + pass + + def install_system_packages(self): + self.install_packages( + [ + "bash", + "findutils", # contains xargs + "gzip", + "libxml2", # used by bootstrapped clang + "m4", + "make", + "perl", + "tar", + "unzip", + "watchman", + ] + ) + + def install_browser_packages(self, mozconfig_builder, artifact_mode=False): + pass + + def install_browser_artifact_mode_packages(self, mozconfig_builder): + pass diff --git a/python/mozboot/mozboot/mach_commands.py b/python/mozboot/mozboot/mach_commands.py new file mode 100644 index 0000000000..02cc69f54b --- /dev/null +++ b/python/mozboot/mozboot/mach_commands.py @@ -0,0 +1,119 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this, +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import errno +import sys +from pathlib import Path + +from mach.decorators import Command, CommandArgument + +from mozboot.bootstrap import APPLICATIONS + + +@Command( + "bootstrap", + category="devenv", + description="Install required system packages for building.", +) +@CommandArgument( + "--application-choice", + choices=list(APPLICATIONS.keys()) + list(APPLICATIONS.values()), + default=None, + help="Pass in an application choice instead of using the default " + "interactive prompt.", +) +@CommandArgument( + "--no-system-changes", + dest="no_system_changes", + action="store_true", + help="Only execute actions that leave the system configuration alone.", +) +@CommandArgument( + "--exclude", + nargs="+", + help="A list of bootstrappable elements not to bootstrap.", +) +def bootstrap( + command_context, application_choice=None, no_system_changes=False, exclude=[] +): + """Bootstrap system and mach for optimal development experience.""" + from mozboot.bootstrap import Bootstrapper + + bootstrapper = Bootstrapper( + choice=application_choice, + no_interactive=not command_context._mach_context.is_interactive, + no_system_changes=no_system_changes, + exclude=exclude, + mach_context=command_context._mach_context, + ) + bootstrapper.bootstrap(command_context.settings) + + +@Command( + "vcs-setup", + category="devenv", + description="Help configure a VCS for optimal development.", +) +@CommandArgument( + "-u", + "--update-only", + action="store_true", + help="Only update recommended extensions, don't run the wizard.", +) +def vcs_setup(command_context, update_only=False): + """Ensure a Version Control System (Mercurial or Git) is optimally + configured. + + This command will inspect your VCS configuration and + guide you through an interactive wizard helping you configure the + VCS for optimal use on Mozilla projects. + + User choice is respected: no changes are made without explicit + confirmation from you. + + If "--update-only" is used, the interactive wizard is disabled + and this command only ensures that remote repositories providing + VCS extensions are up to date. + """ + import mozversioncontrol + from mach.util import to_optional_path + from mozfile import which + + import mozboot.bootstrap as bootstrap + + repo = mozversioncontrol.get_repository_object(command_context._mach_context.topdir) + tool = "hg" + if repo.name == "git": + tool = "git" + + # "hg" is an executable script with a shebang, which will be found by + # which. We need to pass a win32 executable to the function because we + # spawn a process from it. + if sys.platform in ("win32", "msys"): + tool += ".exe" + + vcs = to_optional_path(which(tool)) + if not vcs: + raise OSError(errno.ENOENT, "Could not find {} on $PATH".format(tool)) + + if update_only: + if repo.name == "git": + bootstrap.update_git_tools( + vcs, + Path(command_context._mach_context.state_dir), + ) + else: + bootstrap.update_vct(vcs, Path(command_context._mach_context.state_dir)) + else: + if repo.name == "git": + bootstrap.configure_git( + vcs, + to_optional_path(which("git-cinnabar")), + Path(command_context._mach_context.state_dir), + Path(command_context._mach_context.topdir), + ) + else: + bootstrap.configure_mercurial( + vcs, Path(command_context._mach_context.state_dir) + ) diff --git a/python/mozboot/mozboot/mozconfig.py b/python/mozboot/mozboot/mozconfig.py new file mode 100644 index 0000000000..a1ae4c8523 --- /dev/null +++ b/python/mozboot/mozboot/mozconfig.py @@ -0,0 +1,156 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import filecmp +import os +from pathlib import Path +from typing import Union + +MOZ_MYCONFIG_ERROR = """ +The MOZ_MYCONFIG environment variable to define the location of mozconfigs +is deprecated. If you wish to define the mozconfig path via an environment +variable, use MOZCONFIG instead. +""".strip() + +MOZCONFIG_LEGACY_PATH_ERROR = """ +You currently have a mozconfig at %s. This implicit location is no longer +supported. Please move it to %s/.mozconfig or set an explicit path +via the $MOZCONFIG environment variable. +""".strip() + +DEFAULT_TOPSRCDIR_PATHS = (".mozconfig", "mozconfig") +DEPRECATED_TOPSRCDIR_PATHS = ("mozconfig.sh", "myconfig.sh") +DEPRECATED_HOME_PATHS = (".mozconfig", ".mozconfig.sh", ".mozmyconfig.sh") + + +class MozconfigFindException(Exception): + """Raised when a mozconfig location is not defined properly.""" + + +class MozconfigBuilder(object): + def __init__(self): + self._lines = [] + + def append(self, block): + self._lines.extend([line.strip() for line in block.split("\n") if line.strip()]) + + def generate(self): + return "".join(line + "\n" for line in self._lines) + + +def find_mozconfig(topsrcdir: Union[str, Path], env=os.environ): + """Find the active mozconfig file for the current environment. + + This emulates the logic in mozconfig-find. + + 1) If ENV[MOZCONFIG] is set, use that + 2) If $TOPSRCDIR/mozconfig or $TOPSRCDIR/.mozconfig exists, use it. + 3) If both exist or if there are legacy locations detected, error out. + + The absolute path to the found mozconfig will be returned on success. + None will be returned if no mozconfig could be found. A + MozconfigFindException will be raised if there is a bad state, + including conditions from #3 above. + """ + topsrcdir = Path(topsrcdir) + + # Check for legacy methods first. + if "MOZ_MYCONFIG" in env: + raise MozconfigFindException(MOZ_MYCONFIG_ERROR) + + env_path = env.get("MOZCONFIG", None) or None + + if env_path is not None: + env_path = Path(env_path) + + if env_path is not None: + if not env_path.is_absolute(): + potential_roots = [topsrcdir, Path.cwd()] + # Attempt to eliminate duplicates for e.g. + # self.topsrcdir == Path.cwd(). + potential_roots_strings = set(str(p.resolve()) for p in potential_roots) + existing = [ + root + for root in potential_roots_strings + if (Path(root) / env_path).exists() + ] + if len(existing) > 1: + # There are multiple files, but we might have a setup like: + # + # somedirectory/ + # srcdir/ + # objdir/ + # + # MOZCONFIG=../srcdir/some/path/to/mozconfig + # + # and be configuring from the objdir. So even though we + # have multiple existing files, they are actually the same + # file. + mozconfigs = [root / env_path for root in existing] + if not all( + map( + lambda p1, p2: filecmp.cmp(p1, p2, shallow=False), + mozconfigs[:-1], + mozconfigs[1:], + ) + ): + raise MozconfigFindException( + "MOZCONFIG environment variable refers to a path that " + + "exists in more than one of " + + ", ".join(potential_roots_strings) + + ". Remove all but one." + ) + elif not existing: + raise MozconfigFindException( + "MOZCONFIG environment variable refers to a path that " + + "does not exist in any of " + + ", ".join(potential_roots_strings) + ) + + env_path = existing[0] / env_path + elif not env_path.exists(): # non-relative path + raise MozconfigFindException( + "MOZCONFIG environment variable refers to a path that " + f"does not exist: {env_path}" + ) + + if not env_path.is_file(): + raise MozconfigFindException( + "MOZCONFIG environment variable refers to a " f"non-file: {env_path}" + ) + + srcdir_paths = [topsrcdir / p for p in DEFAULT_TOPSRCDIR_PATHS] + existing = [p for p in srcdir_paths if p.is_file()] + + if env_path is None and len(existing) > 1: + raise MozconfigFindException( + "Multiple default mozconfig files " + "present. Remove all but one. " + ", ".join(str(p) for p in existing) + ) + + path = None + + if env_path is not None: + path = env_path + elif len(existing): + assert len(existing) == 1 + path = existing[0] + + if path is not None: + return Path.cwd() / path + + deprecated_paths = [topsrcdir / s for s in DEPRECATED_TOPSRCDIR_PATHS] + + home = env.get("HOME", None) + if home is not None: + home = Path(home) + deprecated_paths.extend([home / s for s in DEPRECATED_HOME_PATHS]) + + for path in deprecated_paths: + if path.exists(): + raise MozconfigFindException( + MOZCONFIG_LEGACY_PATH_ERROR % (path, topsrcdir) + ) + + return None diff --git a/python/mozboot/mozboot/mozillabuild.py b/python/mozboot/mozboot/mozillabuild.py new file mode 100644 index 0000000000..c783809656 --- /dev/null +++ b/python/mozboot/mozboot/mozillabuild.py @@ -0,0 +1,235 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import ctypes +import os +import platform +import subprocess +import sys +from pathlib import Path + +from mozbuild.util import mozilla_build_version +from packaging.version import Version + +from mozboot.base import BaseBootstrapper + + +def is_aarch64_host(): + from ctypes import wintypes + + kernel32 = ctypes.windll.kernel32 + IMAGE_FILE_MACHINE_UNKNOWN = 0 + IMAGE_FILE_MACHINE_ARM64 = 0xAA64 + + try: + iswow64process2 = kernel32.IsWow64Process2 + except Exception: + # If we can't access the symbol, we know we're not on aarch64. + return False + + currentProcess = kernel32.GetCurrentProcess() + processMachine = wintypes.USHORT(IMAGE_FILE_MACHINE_UNKNOWN) + nativeMachine = wintypes.USHORT(IMAGE_FILE_MACHINE_UNKNOWN) + + gotValue = iswow64process2( + currentProcess, ctypes.byref(processMachine), ctypes.byref(nativeMachine) + ) + # If this call fails, we have no idea. + if not gotValue: + return False + + return nativeMachine.value == IMAGE_FILE_MACHINE_ARM64 + + +def get_is_windefender_disabled(): + import winreg + + try: + with winreg.OpenKeyEx( + winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Microsoft\Windows Defender" + ) as windefender_key: + is_antivirus_disabled, _ = winreg.QueryValueEx( + windefender_key, "DisableAntiSpyware" + ) + # is_antivirus_disabled is either 0 (False) or 1 (True) + return bool(is_antivirus_disabled) + except FileNotFoundError: + return True + + +def get_windefender_exclusion_paths(): + import winreg + + paths = [] + try: + with winreg.OpenKeyEx( + winreg.HKEY_LOCAL_MACHINE, + r"SOFTWARE\Microsoft\Windows Defender\Exclusions\Paths", + ) as exclusions_key: + _, values_count, __ = winreg.QueryInfoKey(exclusions_key) + for i in range(0, values_count): + path, _, __ = winreg.EnumValue(exclusions_key, i) + paths.append(Path(path)) + except FileNotFoundError: + pass + + return paths + + +def is_windefender_affecting_srcdir(src_dir: Path): + if get_is_windefender_disabled(): + return False + + # When there's a match, but path cases aren't the same between srcdir and exclusion_path, + # commonpath will use the casing of the first path provided. + # To avoid surprises here, we normcase(...) so we don't get unexpected breakage if we change + # the path order. + src_dir = src_dir.resolve() + + try: + exclusion_paths = get_windefender_exclusion_paths() + except OSError as e: + if e.winerror == 5: + # A version of Windows 10 released in 2021 raises an "Access is denied" + # error (ERROR_ACCESS_DENIED == 5) to un-elevated processes when they + # query Windows Defender's exclusions. Skip the exclusion path checking. + return + raise + + for exclusion_path in exclusion_paths: + exclusion_path = exclusion_path.resolve() + try: + if Path(os.path.commonpath((exclusion_path, src_dir))) == exclusion_path: + # exclusion_path is an ancestor of srcdir + return False + except ValueError: + # ValueError: Paths don't have the same drive - can't be ours + pass + return True + + +class MozillaBuildBootstrapper(BaseBootstrapper): + """Bootstrapper for MozillaBuild to install rustup.""" + + def __init__(self, no_interactive=False, no_system_changes=False): + BaseBootstrapper.__init__( + self, no_interactive=no_interactive, no_system_changes=no_system_changes + ) + + def validate_environment(self): + if self.application.startswith("mobile_android"): + print( + "WARNING!!! Building Firefox for Android on Windows is not " + "fully supported. See https://bugzilla.mozilla.org/show_bug." + "cgi?id=1169873 for details.", + file=sys.stderr, + ) + + if is_windefender_affecting_srcdir(self.srcdir): + print( + "Warning: the Firefox checkout directory is currently not in the " + "Windows Defender exclusion list. This can cause the build process " + "to be dramatically slowed or broken. To resolve this, follow the " + "directions here: " + "https://firefox-source-docs.mozilla.org/setup/windows_build.html" + "#antivirus-performance", + file=sys.stderr, + ) + + def install_system_packages(self): + pass + + def upgrade_mercurial(self, current): + # Mercurial upstream sometimes doesn't upload wheels, and building + # from source requires MS Visual C++ 9.0. So we force pip to install + # the last version that comes with wheels. + if mozilla_build_version() >= Version("4.0"): + pip_dir = ( + Path(os.environ["MOZILLABUILD"]) / "python3" / "Scripts" / "pip.exe" + ) + else: + pip_dir = ( + Path(os.environ["MOZILLABUILD"]) / "python" / "Scripts" / "pip.exe" + ) + + command = [ + str(pip_dir), + "install", + "--upgrade", + "mercurial", + "--only-binary", + "mercurial", + ] + self.run(command) + + def install_browser_packages(self, mozconfig_builder): + pass + + def install_browser_artifact_mode_packages(self, mozconfig_builder): + pass + + def _os_arch(self): + os_arch = platform.machine() + if os_arch == "AMD64": + # On Windows, x86_64 is reported as AMD64 but we use x86_64 + # everywhere else, so let's normalized it here. + return "x86_64" + return os_arch + + def install_mobile_android_packages(self, mozconfig_builder, artifact_mode=False): + from mozboot import android + + os_arch = self._os_arch() + android.ensure_android( + "windows", + os_arch, + artifact_mode=artifact_mode, + no_interactive=self.no_interactive, + ) + android.ensure_android( + "windows", + os_arch, + system_images_only=True, + artifact_mode=artifact_mode, + no_interactive=self.no_interactive, + avd_manifest_path=android.AVD_MANIFEST_X86_64, + ) + android.ensure_android( + "windows", + os_arch, + system_images_only=True, + artifact_mode=artifact_mode, + no_interactive=self.no_interactive, + avd_manifest_path=android.AVD_MANIFEST_ARM, + ) + + def ensure_mobile_android_packages(self): + from mozboot import android + + android.ensure_java("windows", self._os_arch()) + self.install_toolchain_artifact(android.WINDOWS_X86_64_ANDROID_AVD) + self.install_toolchain_artifact(android.WINDOWS_ARM_ANDROID_AVD) + + def install_mobile_android_artifact_mode_packages(self, mozconfig_builder): + self.install_mobile_android_packages(mozconfig_builder, artifact_mode=True) + + def generate_mobile_android_mozconfig(self, artifact_mode=False): + from mozboot import android + + return android.generate_mozconfig("windows", artifact_mode=artifact_mode) + + def generate_mobile_android_artifact_mode_mozconfig(self): + return self.generate_mobile_android_mozconfig(artifact_mode=True) + + def ensure_sccache_packages(self): + from mozboot import sccache + + self.install_toolchain_artifact(sccache.RUSTC_DIST_TOOLCHAIN, no_unpack=True) + self.install_toolchain_artifact(sccache.CLANG_DIST_TOOLCHAIN, no_unpack=True) + + def _update_package_manager(self): + pass + + def run(self, command): + subprocess.check_call(command, stdin=sys.stdin) diff --git a/python/mozboot/mozboot/openbsd.py b/python/mozboot/mozboot/openbsd.py new file mode 100644 index 0000000000..a862525ece --- /dev/null +++ b/python/mozboot/mozboot/openbsd.py @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozboot.base import BaseBootstrapper + + +class OpenBSDBootstrapper(BaseBootstrapper): + def __init__(self, version, **kwargs): + BaseBootstrapper.__init__(self, **kwargs) + + self.packages = ["gmake", "gtar", "rust", "unzip"] + + self.browser_packages = [ + "llvm", + "cbindgen", + "nasm", + "node", + "gtk+3", + "dbus-glib", + "pulseaudio", + ] + + def install_system_packages(self): + # we use -z because there's no other way to say "any autoconf-2.13" + self.run_as_root(["pkg_add", "-z"] + self.packages) + + def install_browser_packages(self, mozconfig_builder, artifact_mode=False): + # TODO: Figure out what not to install for artifact mode + # we use -z because there's no other way to say "any autoconf-2.13" + self.run_as_root(["pkg_add", "-z"] + self.browser_packages) + + def install_browser_artifact_mode_packages(self, mozconfig_builder): + self.install_browser_packages(mozconfig_builder, artifact_mode=True) diff --git a/python/mozboot/mozboot/opensuse.py b/python/mozboot/mozboot/opensuse.py new file mode 100644 index 0000000000..051ee97f4b --- /dev/null +++ b/python/mozboot/mozboot/opensuse.py @@ -0,0 +1,63 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozboot.base import MERCURIAL_INSTALL_PROMPT, BaseBootstrapper +from mozboot.linux_common import LinuxBootstrapper + + +class OpenSUSEBootstrapper(LinuxBootstrapper, BaseBootstrapper): + """openSUSE experimental bootstrapper.""" + + def __init__(self, version, dist_id, **kwargs): + print("Using an experimental bootstrapper for openSUSE.") + BaseBootstrapper.__init__(self, **kwargs) + + def install_packages(self, packages): + ALTERNATIVE_NAMES = { + "libxml2": "libxml2-2", + } + # watchman is not available + packages = [ALTERNATIVE_NAMES.get(p, p) for p in packages if p != "watchman"] + self.zypper_install(*packages) + + def _update_package_manager(self): + self.zypper_update() + + def upgrade_mercurial(self, current): + """Install Mercurial from pip because system packages could lag.""" + if self.no_interactive: + # Install via zypper in non-interactive mode because it is the more + # conservative option and less likely to make people upset. + self.zypper_install("mercurial") + return + + res = self.prompt_int(MERCURIAL_INSTALL_PROMPT, 1, 3) + + # zypper. + if res == 2: + self.zypper_install("mercurial") + return False + + # No Mercurial. + if res == 3: + print("Not installing Mercurial.") + return False + + # pip. + assert res == 1 + self.run_as_root(["pip3", "install", "--upgrade", "Mercurial"]) + + def zypper(self, *args): + if self.no_interactive: + command = ["zypper", "-n", *args] + else: + command = ["zypper", *args] + + self.run_as_root(command) + + def zypper_install(self, *packages): + self.zypper("install", *packages) + + def zypper_update(self, *packages): + self.zypper("update", *packages) diff --git a/python/mozboot/mozboot/osx.py b/python/mozboot/mozboot/osx.py new file mode 100644 index 0000000000..8cd180f4ab --- /dev/null +++ b/python/mozboot/mozboot/osx.py @@ -0,0 +1,310 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import platform +import subprocess +import sys +import tempfile +from urllib.request import urlopen + +import certifi +from mach.util import to_optional_path, to_optional_str +from mozfile import which +from packaging.version import Version + +from mozboot.base import BaseBootstrapper + +HOMEBREW_BOOTSTRAP = ( + "https://raw.githubusercontent.com/Homebrew/install/master/install.sh" +) + +BREW_INSTALL = """ +We will install the Homebrew package manager to install required packages. + +You will be prompted to install Homebrew with its default settings. If you +would prefer to do this manually, hit CTRL+c, install Homebrew yourself, ensure +"brew" is in your $PATH, and relaunch bootstrap. +""" + +BREW_PACKAGES = """ +We are now installing all required packages via Homebrew. You will see a lot of +output as packages are built. +""" + +NO_BREW_INSTALLED = "It seems you don't have Homebrew installed." + + +class OSXAndroidBootstrapper(object): + def install_mobile_android_packages(self, mozconfig_builder, artifact_mode=False): + os_arch = platform.machine() + if os_arch != "x86_64" and os_arch != "arm64": + raise Exception( + "You need a 64-bit version of Mac OS X to build " + "GeckoView/Firefox for Android." + ) + + from mozboot import android + + android.ensure_android( + "macosx", + os_arch, + artifact_mode=artifact_mode, + no_interactive=self.no_interactive, + ) + + if os_arch == "x86_64" or os_arch == "x86": + android.ensure_android( + "macosx", + os_arch, + system_images_only=True, + artifact_mode=artifact_mode, + no_interactive=self.no_interactive, + avd_manifest_path=android.AVD_MANIFEST_X86_64, + ) + android.ensure_android( + "macosx", + os_arch, + system_images_only=True, + artifact_mode=artifact_mode, + no_interactive=self.no_interactive, + avd_manifest_path=android.AVD_MANIFEST_ARM, + ) + else: + android.ensure_android( + "macosx", + os_arch, + system_images_only=True, + artifact_mode=artifact_mode, + no_interactive=self.no_interactive, + avd_manifest_path=android.AVD_MANIFEST_ARM64, + ) + + def ensure_mobile_android_packages(self): + from mozboot import android + + arch = platform.machine() + android.ensure_java("macosx", arch) + + if arch == "x86_64" or arch == "x86": + self.install_toolchain_artifact(android.MACOS_X86_64_ANDROID_AVD) + self.install_toolchain_artifact(android.MACOS_ARM_ANDROID_AVD) + elif arch == "arm64": + # The only emulator supported on Apple Silicon is the Arm64 one. + self.install_toolchain_artifact(android.MACOS_ARM64_ANDROID_AVD) + + def install_mobile_android_artifact_mode_packages(self, mozconfig_builder): + self.install_mobile_android_packages(mozconfig_builder, artifact_mode=True) + + def generate_mobile_android_mozconfig(self): + return self._generate_mobile_android_mozconfig() + + def generate_mobile_android_artifact_mode_mozconfig(self): + return self._generate_mobile_android_mozconfig(artifact_mode=True) + + def _generate_mobile_android_mozconfig(self, artifact_mode=False): + from mozboot import android + + return android.generate_mozconfig("macosx", artifact_mode=artifact_mode) + + +def ensure_command_line_tools(): + # We need either the command line tools or Xcode (one is sufficient). + # Python 3, required to run this code, is not installed by default on macos + # as of writing (macos <= 11.x). + # There are at least 5 different ways to obtain it: + # - macports + # - homebrew + # - command line tools + # - Xcode + # - python.org + # The first two require to install the command line tools. + # So only in the last case we may not have command line tools or xcode + # available. + # When the command line tools are installed, `xcode-select --print-path` + # prints their path. + # When Xcode is installed, `xcode-select --print-path` prints its path. + # When neither is installed, `xcode-select --print-path` prints an error + # to stderr and nothing to stdout. + # So in the rare case where we detect neither the command line tools or + # Xcode is installed, we trigger an intall of the command line tools + # (via `xcode-select --install`). + proc = subprocess.run( + ["xcode-select", "--print-path"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + if not proc.stdout: + subprocess.run(["xcode-select", "--install"], check=True) + # xcode-select --install triggers a separate process to be started by + # launchd, and tracking its successful outcome would require something + # like figuring its pid and using kqueue to get a notification when it + # finishes. Considering how unlikely it is that someone would end up + # here in the first place, we just bail out. + print("Please follow the command line tools installer instructions") + print("and rerun `./mach bootstrap` when it's finished.") + sys.exit(1) + + +class OSXBootstrapperLight(OSXAndroidBootstrapper, BaseBootstrapper): + def __init__(self, version, **kwargs): + BaseBootstrapper.__init__(self, **kwargs) + + def install_system_packages(self): + ensure_command_line_tools() + + # All the installs below are assumed to be handled by mach configure/build by + # default, which is true for arm64. + def install_browser_packages(self, mozconfig_builder): + pass + + def install_browser_artifact_mode_packages(self, mozconfig_builder): + pass + + +class OSXBootstrapper(OSXAndroidBootstrapper, BaseBootstrapper): + def __init__(self, version, **kwargs): + BaseBootstrapper.__init__(self, **kwargs) + + self.os_version = Version(version) + + if self.os_version < Version("10.6"): + raise Exception("OS X 10.6 or above is required.") + + self.minor_version = version.split(".")[1] + + def install_system_packages(self): + ensure_command_line_tools() + + self.ensure_homebrew_installed() + _, hg_modern, _ = self.is_mercurial_modern() + if not hg_modern: + print( + "Mercurial wasn't found or is not sufficiently modern. " + "It will be installed with brew" + ) + + packages = ["git", "gnu-tar", "terminal-notifier", "watchman"] + if not hg_modern: + packages.append("mercurial") + self._ensure_homebrew_packages(packages) + + def install_browser_packages(self, mozconfig_builder): + pass + + def install_browser_artifact_mode_packages(self, mozconfig_builder): + pass + + def _ensure_homebrew_found(self): + self.brew = to_optional_path(which("brew")) + + return self.brew is not None + + def _ensure_homebrew_packages(self, packages, is_for_cask=False): + package_type_flag = "--cask" if is_for_cask else "--formula" + self.ensure_homebrew_installed() + + def create_homebrew_cmd(*parameters): + base_cmd = [to_optional_str(self.brew)] + base_cmd.extend(parameters) + return base_cmd + [package_type_flag] + + installed = set( + subprocess.check_output( + create_homebrew_cmd("list"), universal_newlines=True + ).split() + ) + outdated = set( + subprocess.check_output( + create_homebrew_cmd("outdated", "--quiet"), universal_newlines=True + ).split() + ) + + to_install = set(package for package in packages if package not in installed) + to_upgrade = set(package for package in packages if package in outdated) + + if to_install or to_upgrade: + print(BREW_PACKAGES) + if to_install: + subprocess.check_call(create_homebrew_cmd("install") + list(to_install)) + if to_upgrade: + subprocess.check_call(create_homebrew_cmd("upgrade") + list(to_upgrade)) + + def _ensure_homebrew_casks(self, casks): + self._ensure_homebrew_found() + + known_taps = subprocess.check_output([to_optional_str(self.brew), "tap"]) + + # Ensure that we can access old versions of packages. + if b"homebrew/cask-versions" not in known_taps: + subprocess.check_output( + [to_optional_str(self.brew), "tap", "homebrew/cask-versions"] + ) + + # "caskroom/versions" has been renamed to "homebrew/cask-versions", so + # it is safe to remove the old tap. Removing the old tap is necessary + # to avoid the error "Cask [name of cask] exists in multiple taps". + # See https://bugzilla.mozilla.org/show_bug.cgi?id=1544981 + if b"caskroom/versions" in known_taps: + subprocess.check_output( + [to_optional_str(self.brew), "untap", "caskroom/versions"] + ) + + self._ensure_homebrew_packages(casks, is_for_cask=True) + + def ensure_homebrew_browser_packages(self): + # TODO: Figure out what not to install for artifact mode + packages = ["yasm"] + self._ensure_homebrew_packages(packages) + + def ensure_homebrew_installed(self): + """ + Search for Homebrew in sys.path, if not found, prompt the user to install it. + Then assert our PATH ordering is correct. + """ + homebrew_found = self._ensure_homebrew_found() + if not homebrew_found: + self.install_homebrew() + + def ensure_sccache_packages(self): + from mozboot import sccache + + self.install_toolchain_artifact(sccache.RUSTC_DIST_TOOLCHAIN, no_unpack=True) + self.install_toolchain_artifact(sccache.CLANG_DIST_TOOLCHAIN, no_unpack=True) + + def install_homebrew(self): + print(BREW_INSTALL) + bootstrap = urlopen( + url=HOMEBREW_BOOTSTRAP, cafile=certifi.where(), timeout=20 + ).read() + with tempfile.NamedTemporaryFile() as tf: + tf.write(bootstrap) + tf.flush() + + subprocess.check_call(["bash", tf.name]) + + homebrew_found = self._ensure_homebrew_found() + if not homebrew_found: + print( + "Homebrew was just installed but can't be found on PATH. " + "Please file a bug." + ) + sys.exit(1) + + def _update_package_manager(self): + subprocess.check_call([to_optional_str(self.brew), "-v", "update"]) + + def _upgrade_package(self, package): + self._ensure_homebrew_installed() + + try: + subprocess.check_output( + [to_optional_str(self.brew), "-v", "upgrade", package], + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as e: + if b"already installed" not in e.output: + raise + + def upgrade_mercurial(self, current): + self._upgrade_package("mercurial") diff --git a/python/mozboot/mozboot/rust.py b/python/mozboot/mozboot/rust.py new file mode 100644 index 0000000000..90607fccff --- /dev/null +++ b/python/mozboot/mozboot/rust.py @@ -0,0 +1,185 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this, +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import platform as platform_mod +import sys + +# Base url for pulling the rustup installer. +RUSTUP_URL_BASE = "https://static.rust-lang.org/rustup" + +# Pull this to get the lastest stable version number. +RUSTUP_MANIFEST = RUSTUP_URL_BASE + "/release-stable.toml" + +# We bake in a known version number so we can verify a checksum. +RUSTUP_VERSION = "1.23.1" + +# SHA-256 checksums of the installers, per platform. +RUSTUP_HASHES = { + "x86_64-unknown-freebsd": "3fb56018ec6009c5a3e345f07d7ea2fbc67d4c6768e528c6d990c7ebe2388d09", + "aarch64-apple-darwin": "6d56735284181b2eb804ed7f57f76cf5ff924251e8ab69d9b5822c3be1ca1dc7", + "x86_64-apple-darwin": "39101feb178a7e3e4443b09b36338e794a9e00385e5f44a2f7789aefb91354a9", + "x86_64-unknown-linux-gnu": "ed7773edaf1d289656bdec2aacad12413b38ad0193fff54b2231f5140a4b07c5", + "x86_64-pc-windows-msvc": "a586cf9de3e4aa791fd5796b6a5f99ca05591ccef8bb94e53af5b69f0261fb03", + "x86_64-unknown-netbsd": "8b29918e765f2cec3b81a911652b164471c42f8f31241f7401bb89582d6a3ed5", +} + +NO_PLATFORM = """ +Sorry, we have no installer configured for your platform. + +Please try installing rust for your system from https://rustup.rs/ +or from https://rust-lang.org/ or from your package manager. +""" + + +def rustup_url(host, version=RUSTUP_VERSION): + """Download url for a particular version of the installer.""" + return "%(base)s/archive/%(version)s/%(host)s/rustup-init%(ext)s" % { + "base": RUSTUP_URL_BASE, + "version": version, + "host": host, + "ext": exe_suffix(host), + } + + +def rustup_hash(host): + """Look up the checksum for the given installer.""" + return RUSTUP_HASHES.get(host, None) + + +def platform(): + """Determine the appropriate rust platform string for the current host""" + if sys.platform.startswith("darwin"): + if platform_mod.machine() == "arm64": + return "aarch64-apple-darwin" + return "x86_64-apple-darwin" + elif sys.platform.startswith(("win32", "msys")): + # Bravely assume we'll be building 64-bit Firefox. + return "x86_64-pc-windows-msvc" + elif sys.platform.startswith("linux"): + return "x86_64-unknown-linux-gnu" + elif sys.platform.startswith("freebsd"): + return "x86_64-unknown-freebsd" + elif sys.platform.startswith("netbsd"): + return "x86_64-unknown-netbsd" + + return None + + +def exe_suffix(host=None): + if not host: + host = platform() + if "windows" in host: + return ".exe" + return "" + + +USAGE = """ +python rust.py [--update] + +Pass the --update option print info for the latest release of rustup-init. + +When invoked without the --update option, it queries the latest version +and verifies the current stored checksums against the distribution server, +but doesn't update the version installed by `mach bootstrap`. +""" + + +def unquote(s): + """Strip outer quotation marks from a string.""" + return s.strip("'").strip('"') + + +def rustup_latest_version(): + """Query the latest version of the rustup installer.""" + import requests + + r = requests.get(RUSTUP_MANIFEST) + # The manifest is toml, but we might not have the toml4 python module + # available, so use ad-hoc parsing to obtain the current release version. + # + # The manifest looks like: + # + # schema-version = '1' + # version = '0.6.5' + # + for line in r.iter_lines(): + line = line.decode("utf-8") + key, value = map(str.strip, line.split("=", 2)) + if key == "schema-version": + schema = int(unquote(value)) + if schema != 1: + print("ERROR: Unknown manifest schema %s" % value) + sys.exit(1) + elif key == "version": + return unquote(value) + return None + + +def http_download_and_hash(url): + import hashlib + + import requests + + h = hashlib.sha256() + r = requests.get(url, stream=True) + for data in r.iter_content(4096): + h.update(data) + return h.hexdigest() + + +def make_checksums(version, validate=False): + hashes = [] + for platform in RUSTUP_HASHES.keys(): + if validate: + print("Checking %s... " % platform, end="", flush=True) + else: + print("Fetching %s... " % platform, end="", flush=True) + checksum = http_download_and_hash(rustup_url(platform, version)) + if validate and checksum != rustup_hash(platform): + print( + "mismatch:\n script: %s\n server: %s" + % (RUSTUP_HASHES[platform], checksum) + ) + else: + print("OK") + hashes.append((platform, checksum)) + return hashes + + +if __name__ == "__main__": + """Allow invoking the module as a utility to update checksums.""" + + update = False + if len(sys.argv) > 1: + if sys.argv[1] == "--update": + update = True + else: + print(USAGE) + sys.exit(1) + + print("Checking latest installer version... ", end="", flush=True) + version = rustup_latest_version() + if not version: + print("ERROR: Could not query current rustup installer version.") + sys.exit(1) + print(version) + + if version == RUSTUP_VERSION: + print("We're up to date. Validating checksums.") + make_checksums(version, validate=True) + exit() + + if not update: + print("Out of date. We use %s. Validating checksums." % RUSTUP_VERSION) + make_checksums(RUSTUP_VERSION, validate=True) + exit() + + print("Out of date. We use %s. Calculating checksums." % RUSTUP_VERSION) + hashes = make_checksums(version) + print("") + print("RUSTUP_VERSION = '%s'" % version) + print("RUSTUP_HASHES = {") + for item in hashes: + print(" '%s':\n '%s'," % item) + print("}") diff --git a/python/mozboot/mozboot/sccache.py b/python/mozboot/mozboot/sccache.py new file mode 100644 index 0000000000..b3cc9fbae9 --- /dev/null +++ b/python/mozboot/mozboot/sccache.py @@ -0,0 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# sccache-dist currently expects clients to provide toolchains when +# distributing from macOS or Windows, so we download linux binaries capable +# of cross-compiling for these cases. +RUSTC_DIST_TOOLCHAIN = "rustc-dist-toolchain" +CLANG_DIST_TOOLCHAIN = "clang-dist-toolchain" diff --git a/python/mozboot/mozboot/solus.py b/python/mozboot/mozboot/solus.py new file mode 100644 index 0000000000..664b5285aa --- /dev/null +++ b/python/mozboot/mozboot/solus.py @@ -0,0 +1,32 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozboot.base import BaseBootstrapper +from mozboot.linux_common import LinuxBootstrapper + + +class SolusBootstrapper(LinuxBootstrapper, BaseBootstrapper): + """Solus experimental bootstrapper.""" + + def __init__(self, version, dist_id, **kwargs): + print("Using an experimental bootstrapper for Solus.") + BaseBootstrapper.__init__(self, **kwargs) + + def install_packages(self, packages): + self.package_install(*packages) + + def _update_package_manager(self): + pass + + def upgrade_mercurial(self, current): + self.package_install("mercurial") + + def package_install(self, *packages): + command = ["eopkg", "install"] + if self.no_interactive: + command.append("--yes-all") + + command.extend(packages) + + self.run_as_root(command) diff --git a/python/mozboot/mozboot/test/python.ini b/python/mozboot/mozboot/test/python.ini new file mode 100644 index 0000000000..4947f160f5 --- /dev/null +++ b/python/mozboot/mozboot/test/python.ini @@ -0,0 +1,4 @@ +[DEFAULT] +subsuite = mozbuild + +[test_mozconfig.py] diff --git a/python/mozboot/mozboot/test/test_mozconfig.py b/python/mozboot/mozboot/test/test_mozconfig.py new file mode 100644 index 0000000000..b7375e8529 --- /dev/null +++ b/python/mozboot/mozboot/test/test_mozconfig.py @@ -0,0 +1,229 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import sys +import unittest +from pathlib import Path +from shutil import rmtree +from tempfile import gettempdir, mkdtemp + +import pytest +from mozunit import main + +from mozboot.mozconfig import ( + DEFAULT_TOPSRCDIR_PATHS, + DEPRECATED_HOME_PATHS, + DEPRECATED_TOPSRCDIR_PATHS, + MozconfigFindException, + find_mozconfig, +) + + +class TestFindMozconfig(unittest.TestCase): + def setUp(self): + self._old_env = dict(os.environ) + os.environ.pop("MOZCONFIG", None) + os.environ.pop("MOZ_OBJDIR", None) + os.environ.pop("CC", None) + os.environ.pop("CXX", None) + self._temp_dirs = set() + + def tearDown(self): + os.environ.clear() + os.environ.update(self._old_env) + + for temp_dir in self._temp_dirs: + rmtree(str(temp_dir)) + + def get_temp_dir(self): + new_temp_dir = Path(mkdtemp()) + self._temp_dirs.add(new_temp_dir) + + return new_temp_dir + + def test_find_legacy_env(self): + """Ensure legacy mozconfig path definitions result in error.""" + + os.environ["MOZ_MYCONFIG"] = "/foo" + + with self.assertRaises(MozconfigFindException) as e: + find_mozconfig(self.get_temp_dir()) + + self.assertTrue(str(e.exception).startswith("The MOZ_MYCONFIG")) + + def test_find_multiple_configs(self): + """Ensure multiple relative-path MOZCONFIGs result in error.""" + relative_mozconfig = ".mconfig" + os.environ["MOZCONFIG"] = relative_mozconfig + + src_dir = self.get_temp_dir() + cur_dir = self.get_temp_dir() + dirs = [src_dir, cur_dir] + for iter_dir in dirs: + path = iter_dir / relative_mozconfig + with open(path, "w") as file: + file.write(str(path)) + + orig_dir = Path.cwd() + try: + os.chdir(cur_dir) + with self.assertRaises(MozconfigFindException) as e: + find_mozconfig(src_dir) + finally: + os.chdir(orig_dir) + + self.assertIn("exists in more than one of", str(e.exception)) + for iter_dir in dirs: + self.assertIn(str(iter_dir.resolve()), str(e.exception)) + + def test_find_multiple_but_identical_configs(self): + """Ensure multiple relative-path MOZCONFIGs pointing at the same file are OK.""" + relative_mozconfig = "../src/.mconfig" + os.environ["MOZCONFIG"] = relative_mozconfig + + top_dir = self.get_temp_dir() + src_dir = top_dir / "src" + src_dir.mkdir() + cur_dir = top_dir / "obj" + cur_dir.mkdir() + + path = src_dir / relative_mozconfig + with open(path, "w"): + pass + + orig_dir = Path.cwd() + try: + os.chdir(cur_dir) + self.assertEqual(Path(find_mozconfig(src_dir)).resolve(), path.resolve()) + finally: + os.chdir(orig_dir) + + def test_find_no_relative_configs(self): + """Ensure a missing relative-path MOZCONFIG is detected.""" + relative_mozconfig = ".mconfig" + os.environ["MOZCONFIG"] = relative_mozconfig + + src_dir = self.get_temp_dir() + cur_dir = self.get_temp_dir() + dirs = [src_dir, cur_dir] + + orig_dir = Path.cwd() + try: + os.chdir(cur_dir) + with self.assertRaises(MozconfigFindException) as e: + find_mozconfig(src_dir) + finally: + os.chdir(orig_dir) + + self.assertIn("does not exist in any of", str(e.exception)) + for iter_dir in dirs: + self.assertIn(str(iter_dir.resolve()), str(e.exception)) + + def test_find_relative_mozconfig(self): + """Ensure a relative MOZCONFIG can be found in the srcdir.""" + relative_mozconfig = ".mconfig" + os.environ["MOZCONFIG"] = relative_mozconfig + + src_dir = Path(self.get_temp_dir()) + cur_dir = Path(self.get_temp_dir()) + + path = src_dir / relative_mozconfig + with open(path, "w"): + pass + + orig_dir = Path.cwd() + try: + os.chdir(cur_dir) + self.assertEqual( + str(Path(find_mozconfig(src_dir)).resolve()), str(path.resolve()) + ) + finally: + os.chdir(orig_dir) + + @pytest.mark.skipif( + sys.platform.startswith("win"), + reason="This test uses unix-style absolute paths, since we now use Pathlib, and " + "`is_absolute()` always returns `False` on Windows if there isn't a drive" + " letter, this test is invalid for Windows.", + ) + def test_find_abs_path_not_exist(self): + """Ensure a missing absolute path is detected.""" + os.environ["MOZCONFIG"] = "/foo/bar/does/not/exist" + + with self.assertRaises(MozconfigFindException) as e: + find_mozconfig(self.get_temp_dir()) + + self.assertIn("path that does not exist", str(e.exception)) + self.assertIn("/foo/bar/does/not/exist", str(e.exception)) + + def test_find_path_not_file(self): + """Ensure non-file paths are detected.""" + + os.environ["MOZCONFIG"] = gettempdir() + + with self.assertRaises(MozconfigFindException) as e: + find_mozconfig(self.get_temp_dir()) + + self.assertIn("refers to a non-file", str(e.exception)) + self.assertTrue(str(e.exception).endswith(gettempdir())) + + def test_find_default_files(self): + """Ensure default paths are used when present.""" + for default_dir in DEFAULT_TOPSRCDIR_PATHS: + temp_dir = self.get_temp_dir() + path = temp_dir / default_dir + + with open(path, "w"): + pass + + self.assertEqual(Path(find_mozconfig(temp_dir)), path) + + def test_find_multiple_defaults(self): + """Ensure we error when multiple default files are present.""" + self.assertGreater(len(DEFAULT_TOPSRCDIR_PATHS), 1) + + temp_dir = self.get_temp_dir() + for default_dir in DEFAULT_TOPSRCDIR_PATHS: + with open(temp_dir / default_dir, "w"): + pass + + with self.assertRaises(MozconfigFindException) as e: + find_mozconfig(temp_dir) + + self.assertIn("Multiple default mozconfig files present", str(e.exception)) + + def test_find_deprecated_path_srcdir(self): + """Ensure we error when deprecated path locations are present.""" + for deprecated_dir in DEPRECATED_TOPSRCDIR_PATHS: + temp_dir = self.get_temp_dir() + with open(temp_dir / deprecated_dir, "w"): + pass + + with self.assertRaises(MozconfigFindException) as e: + find_mozconfig(temp_dir) + + self.assertIn("This implicit location is no longer", str(e.exception)) + self.assertIn(str(temp_dir), str(e.exception)) + + def test_find_deprecated_home_paths(self): + """Ensure we error when deprecated home directory paths are present.""" + + for deprecated_path in DEPRECATED_HOME_PATHS: + home = self.get_temp_dir() + os.environ["HOME"] = str(home) + path = home / deprecated_path + + with open(path, "w"): + pass + + with self.assertRaises(MozconfigFindException) as e: + find_mozconfig(self.get_temp_dir()) + + self.assertIn("This implicit location is no longer", str(e.exception)) + self.assertIn(str(path), str(e.exception)) + + +if __name__ == "__main__": + main() diff --git a/python/mozboot/mozboot/util.py b/python/mozboot/mozboot/util.py new file mode 100644 index 0000000000..583c08bf76 --- /dev/null +++ b/python/mozboot/mozboot/util.py @@ -0,0 +1,49 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import hashlib +import os +from pathlib import Path +from urllib.request import urlopen + +import certifi +from mach.site import PythonVirtualenv +from mach.util import get_state_dir + +MINIMUM_RUST_VERSION = "1.66.0" + + +def get_tools_dir(srcdir=False): + if os.environ.get("MOZ_AUTOMATION") and "MOZ_FETCHES_DIR" in os.environ: + return os.environ["MOZ_FETCHES_DIR"] + return get_state_dir(srcdir) + + +def get_mach_virtualenv_root(): + return Path(get_state_dir(specific_to_topsrcdir=True)) / "_virtualenvs" / "mach" + + +def get_mach_virtualenv_binary(): + root = get_mach_virtualenv_root() + return Path(PythonVirtualenv(str(root)).python_path) + + +def http_download_and_save(url, dest: Path, hexhash, digest="sha256"): + """Download the given url and save it to dest. hexhash is a checksum + that will be used to validate the downloaded file using the given + digest algorithm. The value of digest can be any value accepted by + hashlib.new. The default digest used is 'sha256'.""" + f = urlopen(url, cafile=certifi.where()) + h = hashlib.new(digest) + with open(dest, "wb") as out: + while True: + data = f.read(4096) + if data: + out.write(data) + h.update(data) + else: + break + if h.hexdigest() != hexhash: + dest.unlink() + raise ValueError("Hash of downloaded file does not match expected hash") diff --git a/python/mozboot/mozboot/void.py b/python/mozboot/mozboot/void.py new file mode 100644 index 0000000000..8ae0198ace --- /dev/null +++ b/python/mozboot/mozboot/void.py @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozboot.base import BaseBootstrapper +from mozboot.linux_common import LinuxBootstrapper + + +class VoidBootstrapper(LinuxBootstrapper, BaseBootstrapper): + def __init__(self, version, dist_id, **kwargs): + BaseBootstrapper.__init__(self, **kwargs) + + self.distro = "void" + self.version = version + self.dist_id = dist_id + + def run_as_root(self, command): + # VoidLinux doesn't support users sudo'ing most commands by default because of the group + # configuration. + super().run_as_root(command, may_use_sudo=False) + + def xbps_install(self, *packages): + command = ["xbps-install"] + if self.no_interactive: + command.append("-y") + command.extend(packages) + + self.run_as_root(command) + + def xbps_update(self): + command = ["xbps-install", "-Su"] + if self.no_interactive: + command.append("-y") + + self.run_as_root(command) + + def install_packages(self, packages): + self.xbps_install(*packages) + + def _update_package_manager(self): + self.xbps_update() diff --git a/python/mozboot/mozboot/windows.py b/python/mozboot/mozboot/windows.py new file mode 100644 index 0000000000..8f70a70b2f --- /dev/null +++ b/python/mozboot/mozboot/windows.py @@ -0,0 +1,127 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import ctypes +import os +import subprocess +import sys + +from mozfile import which + +from mozboot.base import BaseBootstrapper + + +def is_aarch64_host(): + from ctypes import wintypes + + kernel32 = ctypes.windll.kernel32 + IMAGE_FILE_MACHINE_UNKNOWN = 0 + IMAGE_FILE_MACHINE_ARM64 = 0xAA64 + + try: + iswow64process2 = kernel32.IsWow64Process2 + except Exception: + # If we can't access the symbol, we know we're not on aarch64. + return False + + currentProcess = kernel32.GetCurrentProcess() + processMachine = wintypes.USHORT(IMAGE_FILE_MACHINE_UNKNOWN) + nativeMachine = wintypes.USHORT(IMAGE_FILE_MACHINE_UNKNOWN) + + gotValue = iswow64process2( + currentProcess, ctypes.byref(processMachine), ctypes.byref(nativeMachine) + ) + # If this call fails, we have no idea. + if not gotValue: + return False + + return nativeMachine.value == IMAGE_FILE_MACHINE_ARM64 + + +class WindowsBootstrapper(BaseBootstrapper): + """Bootstrapper for msys2 based environments for building in Windows.""" + + SYSTEM_PACKAGES = [ + "mingw-w64-x86_64-make", + "mingw-w64-x86_64-perl", + "patch", + "patchutils", + "diffutils", + "tar", + "unzip", + "mingw-w64-x86_64-toolchain", # TODO: Remove when Mercurial is installable from a wheel. + "mingw-w64-i686-toolchain", + ] + + BROWSER_PACKAGES = ["mingw-w64-x86_64-nasm", "mingw-w64-i686-nsis"] + + def __init__(self, **kwargs): + if ( + "MOZ_WINDOWS_BOOTSTRAP" not in os.environ + or os.environ["MOZ_WINDOWS_BOOTSTRAP"] != "1" + ): + raise NotImplementedError( + "Bootstrap support for Windows is under development. For " + "now use MozillaBuild to set up a build environment on " + "Windows. If you are testing Windows Bootstrap support, " + "try `export MOZ_WINDOWS_BOOTSTRAP=1`" + ) + BaseBootstrapper.__init__(self, **kwargs) + if not which("pacman"): + raise NotImplementedError( + "The Windows bootstrapper only works with msys2 with " + "pacman. Get msys2 at http://msys2.github.io/" + ) + print("Using an experimental bootstrapper for Windows.") + + def install_system_packages(self): + self.pacman_install(*self.SYSTEM_PACKAGES) + + def upgrade_mercurial(self, current): + self.pip_install("mercurial") + + def install_browser_packages(self, mozconfig_builder): + self.pacman_install(*self.BROWSER_PACKAGES) + + def install_mobile_android_packages(self, mozconfig_builder): + raise NotImplementedError( + "We do not support building Android on Windows. Sorry!" + ) + + def ensure_mobile_android_packages(self): + raise NotImplementedError( + "We do not support building Android on Windows. Sorry!" + ) + + def install_mobile_android_artifact_mode_packages(self, mozconfig_builder): + raise NotImplementedError( + "We do not support building Android on Windows. Sorry!" + ) + + def _update_package_manager(self): + self.pacman_update() + + def run(self, command): + subprocess.check_call(command, stdin=sys.stdin) + + def pacman_update(self): + command = ["pacman", "--sync", "--refresh"] + self.run(command) + + def pacman_upgrade(self): + command = ["pacman", "--sync", "--refresh", "--sysupgrade"] + self.run(command) + + def pacman_install(self, *packages): + command = ["pacman", "--sync", "--needed"] + if self.no_interactive: + command.append("--noconfirm") + + command.extend(packages) + self.run(command) + + def pip_install(self, *packages): + command = ["pip", "install", "--upgrade"] + command.extend(packages) + self.run(command) |