diff options
Diffstat (limited to 'python/mozboot')
39 files changed, 6098 insertions, 0 deletions
diff --git a/python/mozboot/README.rst b/python/mozboot/README.rst new file mode 100644 index 0000000000..97dc3c97b2 --- /dev/null +++ b/python/mozboot/README.rst @@ -0,0 +1,20 @@ +mozboot - Bootstrap your system to build Mozilla projects +========================================================= + +This package contains code used for bootstrapping a system to build +mozilla-central. + +This code is not part of the build system per se. Instead, it is related +to everything up to invoking the actual build system. + +If you have a copy of the source tree, you run: + + python bin/bootstrap.py + +If you don't have a copy of the source tree, you can run: + + curl https://hg.mozilla.org/mozilla-central/raw-file/default/python/mozboot/bin/bootstrap.py -o bootstrap.py + python bootstrap.py + +The bootstrap script will download everything it needs from hg.mozilla.org +automatically! diff --git a/python/mozboot/bin/bootstrap.py b/python/mozboot/bin/bootstrap.py new file mode 100755 index 0000000000..2722438330 --- /dev/null +++ b/python/mozboot/bin/bootstrap.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script provides one-line bootstrap support to configure systems to build +# the tree. It does so by cloning the repo before calling directly into `mach +# bootstrap`. + +# Note that this script can't assume anything in particular about the host +# Python environment (except that it's run with a sufficiently recent version of +# Python 3), so we are restricted to stdlib modules. + +from __future__ import absolute_import, print_function, unicode_literals + +import sys + +major, minor = sys.version_info[:2] +if (major < 3) or (major == 3 and minor < 5): + print( + "Bootstrap currently only runs on Python 3.5+." + "Please try re-running with python3.5+." + ) + sys.exit(1) + +import os +import shutil +import stat +import subprocess +import tempfile +import zipfile + +from optparse import OptionParser +from urllib.request import urlopen + +CLONE_MERCURIAL_PULL_FAIL = """ +Failed to pull from hg.mozilla.org. + +This is most likely because of unstable network connection. +Try running `cd %s && hg pull https://hg.mozilla.org/mozilla-unified` manually, +or download a mercurial bundle and use it: +https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code/Mercurial/Bundles""" + +WINDOWS = sys.platform.startswith("win32") or sys.platform.startswith("msys") +VCS_HUMAN_READABLE = { + "hg": "Mercurial", + "git": "Git", +} + + +def which(name): + """Python implementation of which. + + It returns the path of an executable or None if it couldn't be found. + """ + # git-cinnabar.exe doesn't exist, but .exe versions of the other executables + # do. + if WINDOWS and name != "git-cinnabar": + name += ".exe" + search_dirs = os.environ["PATH"].split(os.pathsep) + + for path in search_dirs: + test = os.path.join(path, name) + if os.path.isfile(test) and os.access(test, os.X_OK): + return test + + return None + + +def validate_clone_dest(dest): + dest = os.path.abspath(dest) + + if not os.path.exists(dest): + return dest + + if not os.path.isdir(dest): + print("ERROR! Destination %s exists but is not a directory." % dest) + return None + + if not os.listdir(dest): + return dest + else: + print("ERROR! Destination directory %s exists but is nonempty." % dest) + return None + + +def input_clone_dest(vcs, no_interactive): + repo_name = "mozilla-unified" + print("Cloning into %s using %s..." % (repo_name, VCS_HUMAN_READABLE[vcs])) + while True: + dest = None + if not no_interactive: + dest = input( + "Destination directory for clone (leave empty to use " + "default destination of %s): " % repo_name + ).strip() + if not dest: + dest = repo_name + dest = validate_clone_dest(os.path.expanduser(dest)) + if dest: + return dest + if no_interactive: + return None + + +def hg_clone_firefox(hg, dest): + # We create an empty repo then modify the config before adding data. + # This is necessary to ensure storage settings are optimally + # configured. + args = [ + hg, + # The unified repo is generaldelta, so ensure the client is as + # well. + "--config", + "format.generaldelta=true", + "init", + dest, + ] + res = subprocess.call(args) + if res: + print("unable to create destination repo; please try cloning manually") + return None + + # Strictly speaking, this could overwrite a config based on a template + # the user has installed. Let's pretend this problem doesn't exist + # unless someone complains about it. + with open(os.path.join(dest, ".hg", "hgrc"), "a") as fh: + fh.write("[paths]\n") + fh.write("default = https://hg.mozilla.org/mozilla-unified\n") + fh.write("\n") + + # The server uses aggressivemergedeltas which can blow up delta chain + # length. This can cause performance to tank due to delta chains being + # too long. Limit the delta chain length to something reasonable + # to bound revlog read time. + fh.write("[format]\n") + fh.write("# This is necessary to keep performance in check\n") + fh.write("maxchainlen = 10000\n") + + res = subprocess.call( + [hg, "pull", "https://hg.mozilla.org/mozilla-unified"], cwd=dest + ) + print("") + if res: + print(CLONE_MERCURIAL_PULL_FAIL % dest) + return None + + print('updating to "central" - the development head of Gecko and Firefox') + res = subprocess.call([hg, "update", "-r", "central"], cwd=dest) + if res: + print( + "error updating; you will need to `cd %s && hg update -r central` " + "manually" % dest + ) + return dest + + +def git_clone_firefox(git, dest, watchman): + tempdir = None + cinnabar = None + env = dict(os.environ) + try: + cinnabar = which("git-cinnabar") + if not cinnabar: + cinnabar_url = ( + "https://github.com/glandium/git-cinnabar/archive/" "master.zip" + ) + # If git-cinnabar isn't installed already, that's fine; we can + # download a temporary copy. `mach bootstrap` will clone a full copy + # of the repo in the state dir; we don't want to copy all that logic + # to this tiny bootstrapping script. + tempdir = tempfile.mkdtemp() + with open(os.path.join(tempdir, "git-cinnabar.zip"), mode="w+b") as archive: + with urlopen(cinnabar_url) as repo: + shutil.copyfileobj(repo, archive) + archive.seek(0) + with zipfile.ZipFile(archive) as zipf: + zipf.extractall(path=tempdir) + cinnabar_dir = os.path.join(tempdir, "git-cinnabar-master") + cinnabar = os.path.join(cinnabar_dir, "git-cinnabar") + # Make git-cinnabar and git-remote-hg executable. + st = os.stat(cinnabar) + os.chmod(cinnabar, st.st_mode | stat.S_IEXEC) + st = os.stat(os.path.join(cinnabar_dir, "git-remote-hg")) + os.chmod( + os.path.join(cinnabar_dir, "git-remote-hg"), st.st_mode | stat.S_IEXEC + ) + env["PATH"] = cinnabar_dir + os.pathsep + env["PATH"] + subprocess.check_call( + ["git", "cinnabar", "download"], cwd=cinnabar_dir, env=env + ) + print( + "WARNING! git-cinnabar is required for Firefox development " + "with git. After the clone is complete, the bootstrapper " + "will ask if you would like to configure git; answer yes, " + "and be sure to add git-cinnabar to your PATH according to " + "the bootstrapper output." + ) + + # We're guaranteed to have `git-cinnabar` installed now. + # Configure git per the git-cinnabar requirements. + subprocess.check_call( + [ + git, + "clone", + "-b", + "bookmarks/central", + "hg::https://hg.mozilla.org/mozilla-unified", + dest, + ], + env=env, + ) + subprocess.check_call([git, "config", "fetch.prune", "true"], cwd=dest, env=env) + subprocess.check_call([git, "config", "pull.ff", "only"], cwd=dest, env=env) + + watchman_sample = os.path.join(dest, ".git/hooks/fsmonitor-watchman.sample") + # Older versions of git didn't include fsmonitor-watchman.sample. + if watchman and os.path.exists(watchman_sample): + print("Configuring watchman") + watchman_config = os.path.join(dest, ".git/hooks/query-watchman") + if not os.path.exists(watchman_config): + print("Copying %s to %s" % (watchman_sample, watchman_config)) + copy_args = [ + "cp", + ".git/hooks/fsmonitor-watchman.sample", + ".git/hooks/query-watchman", + ] + subprocess.check_call(copy_args, cwd=dest) + + config_args = [git, "config", "core.fsmonitor", ".git/hooks/query-watchman"] + subprocess.check_call(config_args, cwd=dest, env=env) + return dest + finally: + if not cinnabar: + print( + "Failed to install git-cinnabar. Try performing a manual " + "installation: https://github.com/glandium/git-cinnabar/wiki/" + "Mozilla:-A-git-workflow-for-Gecko-development" + ) + if tempdir: + shutil.rmtree(tempdir) + + +def clone(vcs, no_interactive): + hg = which("hg") + if not hg: + print( + "Mercurial is not installed. Mercurial is required to clone " + "Firefox%s." % (", even when cloning with Git" if vcs == "git" else "") + ) + try: + # We're going to recommend people install the Mercurial package with + # pip3. That will work if `pip3` installs binaries to a location + # that's in the PATH, but it might not be. To help out, if we CAN + # import "mercurial" (in which case it's already been installed), + # offer that as a solution. + import mercurial # noqa: F401 + + print( + "Hint: have you made sure that Mercurial is installed to a " + "location in your PATH?" + ) + except ImportError: + print("Try installing hg with `pip3 install Mercurial`.") + return None + if vcs == "hg": + binary = hg + else: + binary = which(vcs) + if not binary: + print("Git is not installed.") + print("Try installing git using your system package manager.") + return None + + dest = input_clone_dest(vcs, no_interactive) + if not dest: + return None + + print("Cloning Firefox %s repository to %s" % (VCS_HUMAN_READABLE[vcs], dest)) + if vcs == "hg": + return hg_clone_firefox(binary, dest) + else: + watchman = which("watchman") + return git_clone_firefox(binary, dest, watchman) + + +def bootstrap(srcdir, application_choice, no_interactive, no_system_changes): + args = [sys.executable, os.path.join(srcdir, "mach"), "bootstrap"] + if application_choice: + args += ["--application-choice", application_choice] + if no_interactive: + args += ["--no-interactive"] + if no_system_changes: + args += ["--no-system-changes"] + print("Running `%s`" % " ".join(args)) + return subprocess.call(args, cwd=srcdir) + + +def main(args): + parser = OptionParser() + parser.add_option( + "--application-choice", + dest="application_choice", + help='Pass in an application choice (see "APPLICATIONS" in ' + "python/mozboot/mozboot/bootstrap.py) instead of using the " + "default interactive prompt.", + ) + parser.add_option( + "--vcs", + dest="vcs", + default="hg", + choices=["git", "hg"], + help="VCS (hg or git) to use for downloading the source code, " + "instead of using the default interactive prompt.", + ) + parser.add_option( + "--no-interactive", + dest="no_interactive", + action="store_true", + help="Answer yes to any (Y/n) interactive prompts.", + ) + parser.add_option( + "--no-system-changes", + dest="no_system_changes", + action="store_true", + help="Only executes actions that leave the system " "configuration alone.", + ) + + options, leftover = parser.parse_args(args) + + try: + srcdir = clone(options.vcs, options.no_interactive) + if not srcdir: + return 1 + print("Clone complete.") + return bootstrap( + srcdir, + options.application_choice, + options.no_interactive, + options.no_system_changes, + ) + except Exception: + print("Could not bootstrap Firefox! Consider filing a bug.") + raise + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/python/mozboot/mozboot/__init__.py b/python/mozboot/mozboot/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/python/mozboot/mozboot/__init__.py diff --git a/python/mozboot/mozboot/android-emulator-packages.txt b/python/mozboot/mozboot/android-emulator-packages.txt new file mode 100644 index 0000000000..3e782df670 --- /dev/null +++ b/python/mozboot/mozboot/android-emulator-packages.txt @@ -0,0 +1,2 @@ +platform-tools +emulator diff --git a/python/mozboot/mozboot/android-packages.txt b/python/mozboot/mozboot/android-packages.txt new file mode 100644 index 0000000000..ebb4d72591 --- /dev/null +++ b/python/mozboot/mozboot/android-packages.txt @@ -0,0 +1,4 @@ +platform-tools +build-tools;29.0.3 +platforms;android-29 +emulator diff --git a/python/mozboot/mozboot/android.py b/python/mozboot/mozboot/android.py new file mode 100644 index 0000000000..c4f29aec6b --- /dev/null +++ b/python/mozboot/mozboot/android.py @@ -0,0 +1,485 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this, +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import errno +import os +import stat +import subprocess +import sys + +# We need the NDK version in multiple different places, and it's inconvenient +# to pass down the NDK version to all relevant places, so we have this global +# variable. +from mozboot.bootstrap import MOZCONFIG_SUGGESTION_TEMPLATE + +NDK_VERSION = "r20" + +ANDROID_NDK_EXISTS = """ +Looks like you have the correct version of the Android NDK installed at: +%s +""" + +ANDROID_SDK_EXISTS = """ +Looks like you have the Android SDK installed at: +%s +We will install all required Android packages. +""" + +ANDROID_SDK_TOO_OLD = """ +Looks like you have an outdated Android SDK installed at: +%s +I can't update outdated Android SDKs to have the required 'sdkmanager' +tool. Move it out of the way (or remove it entirely) and then run +bootstrap again. +""" + +INSTALLING_ANDROID_PACKAGES = """ +We are now installing the following Android packages: +%s +You may be prompted to agree to the Android license. You may see some of +output as packages are downloaded and installed. +""" + +MOBILE_ANDROID_MOZCONFIG_TEMPLATE = """ +# Build GeckoView/Firefox for Android: +ac_add_options --enable-application=mobile/android + +# Targeting the following architecture. +# For regular phones, no --target is needed. +# For x86 emulators (and x86 devices, which are uncommon): +# ac_add_options --target=i686 +# For newer phones. +# ac_add_options --target=aarch64 +# For x86_64 emulators (and x86_64 devices, which are even less common): +# ac_add_options --target=x86_64 + +{extra_lines} +""" + +MOBILE_ANDROID_ARTIFACT_MODE_MOZCONFIG_TEMPLATE = """ +# Build GeckoView/Firefox for Android Artifact Mode: +ac_add_options --enable-application=mobile/android +ac_add_options --target=arm-linux-androideabi +ac_add_options --enable-artifact-builds + +{extra_lines} +# Write build artifacts to: +mk_add_options MOZ_OBJDIR=./objdir-frontend +""" + + +class GetNdkVersionError(Exception): + pass + + +def install_mobile_android_sdk_or_ndk(url, path): + """ + Fetch an Android SDK or NDK from |url| and unpack it into + the given |path|. + + We expect wget to be installed and found on the system path. + + We use, and wget respects, https. We could also include SHAs for a + small improvement in the integrity guarantee we give. But this script is + bootstrapped over https anyway, so it's a really minor improvement. + + We use |wget --continue| as a cheap cache of the downloaded artifacts, + writing into |path|/mozboot. We don't yet clean the cache; it's better + to waste disk and not require a long re-download than to wipe the cache + prematurely. + """ + + old_path = os.getcwd() + try: + download_path = os.path.join(path, "mozboot") + try: + os.makedirs(download_path) + except OSError as e: + if e.errno == errno.EEXIST and os.path.isdir(download_path): + pass + else: + raise + + os.chdir(download_path) + subprocess.check_call(["wget", "--continue", url]) + file = url.split("/")[-1] + + os.chdir(path) + abspath = os.path.join(download_path, file) + if file.endswith(".tar.gz") or file.endswith(".tgz"): + cmd = ["tar", "zxf", abspath] + elif file.endswith(".tar.bz2"): + cmd = ["tar", "jxf", abspath] + elif file.endswith(".zip"): + cmd = ["unzip", "-q", abspath] + elif file.endswith(".bin"): + # Execute the .bin file, which unpacks the content. + mode = os.stat(path).st_mode + os.chmod(abspath, mode | stat.S_IXUSR) + cmd = [abspath] + else: + raise NotImplementedError("Don't know how to unpack file: %s" % file) + + print("Unpacking %s..." % abspath) + + with open(os.devnull, "w") as stdout: + # These unpack commands produce a ton of output; ignore it. The + # .bin files are 7z archives; there's no command line flag to quiet + # output, so we use this hammer. + subprocess.check_call(cmd, stdout=stdout) + + print("Unpacking %s... DONE" % abspath) + # Now delete the archive + os.unlink(abspath) + finally: + os.chdir(old_path) + + +def get_ndk_version(ndk_path): + """Given the path to the NDK, return the version as a 3-tuple of (major, + minor, human). + """ + with open(os.path.join(ndk_path, "source.properties"), "r") as f: + revision = [line for line in f if line.startswith("Pkg.Revision")] + if not revision: + raise GetNdkVersionError( + "Cannot determine NDK version from source.properties" + ) + if len(revision) != 1: + raise GetNdkVersionError("Too many Pkg.Revision lines in source.properties") + + (_, version) = revision[0].split("=") + if not version: + raise GetNdkVersionError( + "Unexpected Pkg.Revision line in source.properties" + ) + + (major, minor, revision) = version.strip().split(".") + if not major or not minor: + raise GetNdkVersionError("Unexpected NDK version string: " + version) + + # source.properties contains a $MAJOR.$MINOR.$PATCH revision number, + # but the more common nomenclature that Google uses is alphanumeric + # version strings like "r20" or "r19c". Convert the source.properties + # notation into an alphanumeric string. + int_minor = int(minor) + alphas = "abcdefghijklmnop" + ascii_minor = alphas[int_minor] if int_minor > 0 else "" + human = "r%s%s" % (major, ascii_minor) + return (major, minor, human) + + +def get_paths(os_name): + mozbuild_path = os.environ.get( + "MOZBUILD_STATE_PATH", os.path.expanduser(os.path.join("~", ".mozbuild")) + ) + sdk_path = os.environ.get( + "ANDROID_SDK_HOME", + os.path.join(mozbuild_path, "android-sdk-{0}".format(os_name)), + ) + ndk_path = os.environ.get( + "ANDROID_NDK_HOME", + os.path.join(mozbuild_path, "android-ndk-{0}".format(NDK_VERSION)), + ) + return (mozbuild_path, sdk_path, ndk_path) + + +def sdkmanager_tool(sdk_path): + # sys.platform is win32 even if Python/Win64. + sdkmanager = "sdkmanager.bat" if sys.platform.startswith("win") else "sdkmanager" + return os.path.join(sdk_path, "tools", "bin", sdkmanager) + + +def ensure_dir(dir): + """Ensures the given directory exists""" + if dir and not os.path.exists(dir): + try: + os.makedirs(dir) + except OSError as error: + if error.errno != errno.EEXIST: + raise + + +def ensure_android( + os_name, + artifact_mode=False, + ndk_only=False, + emulator_only=False, + no_interactive=False, +): + """ + Ensure the Android SDK (and NDK, if `artifact_mode` is falsy) are + installed. If not, fetch and unpack the SDK and/or NDK from the + given URLs. Ensure the required Android SDK packages are + installed. + + `os_name` can be 'linux' or 'macosx'. + """ + # The user may have an external Android SDK (in which case we + # save them a lengthy download), or they may have already + # completed the download. We unpack to + # ~/.mozbuild/{android-sdk-$OS_NAME, android-ndk-$VER}. + mozbuild_path, sdk_path, ndk_path = get_paths(os_name) + os_tag = "darwin" if os_name == "macosx" else os_name + sdk_url = ( + "https://dl.google.com/android/repository/sdk-tools-{0}-4333796.zip".format( + os_tag + ) + ) + ndk_url = android_ndk_url(os_name) + + ensure_android_sdk_and_ndk( + mozbuild_path, + os_name, + sdk_path=sdk_path, + sdk_url=sdk_url, + ndk_path=ndk_path, + ndk_url=ndk_url, + artifact_mode=artifact_mode, + ndk_only=ndk_only, + emulator_only=emulator_only, + ) + + if ndk_only: + return + + # We expect the |sdkmanager| tool to be at + # ~/.mozbuild/android-sdk-$OS_NAME/tools/bin/sdkmanager. + ensure_android_packages( + sdkmanager_tool=sdkmanager_tool(sdk_path), + emulator_only=emulator_only, + no_interactive=no_interactive, + ) + + +def ensure_android_sdk_and_ndk( + mozbuild_path, + os_name, + sdk_path, + sdk_url, + ndk_path, + ndk_url, + artifact_mode, + ndk_only, + emulator_only, +): + """ + Ensure the Android SDK and NDK are found at the given paths. If not, fetch + and unpack the SDK and/or NDK from the given URLs into + |mozbuild_path/{android-sdk-$OS_NAME,android-ndk-$VER}|. + """ + + # It's not particularly bad to overwrite the NDK toolchain, but it does take + # a while to unpack, so let's avoid the disk activity if possible. The SDK + # may prompt about licensing, so we do this first. + # Check for Android NDK only if we are not in artifact mode. + if not artifact_mode and not emulator_only: + install_ndk = True + if os.path.isdir(ndk_path): + try: + _, _, human = get_ndk_version(ndk_path) + if human == NDK_VERSION: + print(ANDROID_NDK_EXISTS % ndk_path) + install_ndk = False + except GetNdkVersionError: + pass # Just do the install. + if install_ndk: + # The NDK archive unpacks into a top-level android-ndk-$VER directory. + install_mobile_android_sdk_or_ndk(ndk_url, mozbuild_path) + + if ndk_only: + return + + # We don't want to blindly overwrite, since we use the + # |sdkmanager| tool to install additional parts of the Android + # toolchain. If we overwrite, we lose whatever Android packages + # the user may have already installed. + if os.path.isfile(sdkmanager_tool(sdk_path)): + print(ANDROID_SDK_EXISTS % sdk_path) + elif os.path.isdir(sdk_path): + raise NotImplementedError(ANDROID_SDK_TOO_OLD % sdk_path) + else: + # The SDK archive used to include a top-level + # android-sdk-$OS_NAME directory; it no longer does so. We + # preserve the old convention to smooth detecting existing SDK + # installations. + install_mobile_android_sdk_or_ndk( + sdk_url, os.path.join(mozbuild_path, "android-sdk-{0}".format(os_name)) + ) + + +def get_packages_to_install(packages_file_name): + """ + sdkmanager version 26.1.1 (current) and some versions below have a bug that makes + the following command fail: + args = [sdkmanager_tool, '--package_file={0}'.format(package_file_name)] + subprocess.check_call(args) + The error is in the sdkmanager, where the --package_file param isn't recognized. + The error is being tracked here https://issuetracker.google.com/issues/66465833 + Meanwhile, this workaround achives installing all required Android packages by reading + them out of the same file that --package_file would have used, and passing them as strings. + So from here: https://developer.android.com/studio/command-line/sdkmanager + Instead of: + sdkmanager --package_file=package_file [options] + We're doing: + sdkmanager "platform-tools" "platforms;android-26" + """ + with open(packages_file_name) as package_file: + return map(lambda package: package.strip(), package_file.readlines()) + + +def ensure_android_packages(sdkmanager_tool, emulator_only=False, no_interactive=False): + """ + Use the given sdkmanager tool (like 'sdkmanager') to install required + Android packages. + """ + + # This tries to install all the required Android packages. The user + # may be prompted to agree to the Android license. + if emulator_only: + package_file_name = os.path.abspath( + os.path.join(os.path.dirname(__file__), "android-emulator-packages.txt") + ) + else: + package_file_name = os.path.abspath( + os.path.join(os.path.dirname(__file__), "android-packages.txt") + ) + print(INSTALLING_ANDROID_PACKAGES % open(package_file_name, "rt").read()) + + args = [sdkmanager_tool] + args.extend(get_packages_to_install(package_file_name)) + + if not no_interactive: + subprocess.check_call(args) + return + + # Flush outputs before running sdkmanager. + sys.stdout.flush() + sys.stderr.flush() + # Emulate yes. For a discussion of passing input to check_output, + # see https://stackoverflow.com/q/10103551. + yes = "\n".join(["y"] * 100).encode("UTF-8") + proc = subprocess.Popen(args, stdin=subprocess.PIPE) + proc.communicate(yes) + + retcode = proc.poll() + if retcode: + cmd = args[0] + e = subprocess.CalledProcessError(retcode, cmd) + raise e + + +def generate_mozconfig(os_name, artifact_mode=False): + moz_state_dir, sdk_path, ndk_path = get_paths(os_name) + + extra_lines = [] + if extra_lines: + extra_lines.append("") + + if artifact_mode: + template = MOBILE_ANDROID_ARTIFACT_MODE_MOZCONFIG_TEMPLATE + else: + template = MOBILE_ANDROID_MOZCONFIG_TEMPLATE + + kwargs = dict( + sdk_path=sdk_path, + ndk_path=ndk_path, + moz_state_dir=moz_state_dir, + extra_lines="\n".join(extra_lines), + ) + return template.format(**kwargs).strip() + + +def android_ndk_url(os_name, ver=NDK_VERSION): + # Produce a URL like + # 'https://dl.google.com/android/repository/android-ndk-$VER-linux-x86_64.zip + base_url = "https://dl.google.com/android/repository/android-ndk" + + if os_name == "macosx": + # |mach bootstrap| uses 'macosx', but Google uses 'darwin'. + os_name = "darwin" + + if sys.maxsize > 2 ** 32: + arch = "x86_64" + else: + arch = "x86" + + return "%s-%s-%s-%s.zip" % (base_url, ver, os_name, arch) + + +def main(argv): + import optparse # No argparse, which is new in Python 2.7. + import platform + + parser = optparse.OptionParser() + parser.add_option( + "-a", + "--artifact-mode", + dest="artifact_mode", + action="store_true", + help="If true, install only the Android SDK (and not the Android NDK).", + ) + parser.add_option( + "--ndk-only", + dest="ndk_only", + action="store_true", + help="If true, install only the Android NDK (and not the Android SDK).", + ) + parser.add_option( + "--no-interactive", + dest="no_interactive", + action="store_true", + help="Accept the Android SDK licenses without user interaction.", + ) + parser.add_option( + "--emulator-only", + dest="emulator_only", + action="store_true", + help="If true, install only the Android emulator (and not the SDK or NDK).", + ) + + options, _ = parser.parse_args(argv) + + if options.artifact_mode and options.ndk_only: + raise NotImplementedError("Use no options to install the NDK and the SDK.") + + if options.artifact_mode and options.emulator_only: + raise NotImplementedError("Use no options to install the SDK and emulators.") + + os_name = None + if platform.system() == "Darwin": + os_name = "macosx" + elif platform.system() == "Linux": + os_name = "linux" + elif platform.system() == "Windows": + os_name = "windows" + else: + raise NotImplementedError( + "We don't support bootstrapping the Android SDK (or Android " + "NDK) on {0} yet!".format(platform.system()) + ) + + ensure_android( + os_name, + artifact_mode=options.artifact_mode, + ndk_only=options.ndk_only, + emulator_only=options.emulator_only, + no_interactive=options.no_interactive, + ) + mozconfig = generate_mozconfig(os_name, options.artifact_mode) + + # |./mach bootstrap| automatically creates a mozconfig file for you if it doesn't + # exist. However, here, we don't know where the "topsrcdir" is, and it's not worth + # pulling in CommandContext (and its dependencies) to find out. + # So, instead, we'll politely ask users to create (or update) the file themselves. + suggestion = MOZCONFIG_SUGGESTION_TEMPLATE % ("$topsrcdir/mozconfig", mozconfig) + print("\n" + suggestion) + + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/python/mozboot/mozboot/archlinux.py b/python/mozboot/mozboot/archlinux.py new file mode 100644 index 0000000000..43409c4c28 --- /dev/null +++ b/python/mozboot/mozboot/archlinux.py @@ -0,0 +1,196 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import sys +import tempfile +import subprocess +import glob + +from mozboot.base import BaseBootstrapper +from mozboot.linux_common import LinuxBootstrapper + +# NOTE: This script is intended to be run with a vanilla Python install. We +# have to rely on the standard library instead of Python 2+3 helpers like +# the six module. +if sys.version_info < (3,): + input = raw_input # noqa + + +class ArchlinuxBootstrapper(LinuxBootstrapper, BaseBootstrapper): + """Archlinux experimental bootstrapper.""" + + SYSTEM_PACKAGES = [ + "base-devel", + "nodejs", + "unzip", + "zip", + ] + + BROWSER_PACKAGES = [ + "alsa-lib", + "dbus-glib", + "gtk2", + "gtk3", + "libevent", + "libvpx", + "libxt", + "mime-types", + "nasm", + "startup-notification", + "gst-plugins-base-libs", + "libpulse", + "xorg-server-xvfb", + "yasm", + "gst-libav", + "gst-plugins-good", + ] + + BROWSER_AUR_PACKAGES = [ + "https://aur.archlinux.org/cgit/aur.git/snapshot/uuid.tar.gz", + ] + + MOBILE_ANDROID_COMMON_PACKAGES = [ + # It would be nice to handle alternative JDKs. See + # https://wiki.archlinux.org/index.php/Java. + "jdk8-openjdk", + # For downloading the Android SDK and NDK. + "wget", + # See comment about 32 bit binaries and multilib below. + "multilib/lib32-ncurses", + "multilib/lib32-readline", + "multilib/lib32-zlib", + ] + + def __init__(self, version, dist_id, **kwargs): + print("Using an experimental bootstrapper for Archlinux.") + BaseBootstrapper.__init__(self, **kwargs) + + def install_system_packages(self): + self.pacman_install(*self.SYSTEM_PACKAGES) + + def install_browser_packages(self, mozconfig_builder): + self.ensure_browser_packages() + + def install_browser_artifact_mode_packages(self, mozconfig_builder): + self.ensure_browser_packages(artifact_mode=True) + + def install_mobile_android_packages(self, mozconfig_builder): + self.ensure_mobile_android_packages(mozconfig_builder) + + def install_mobile_android_artifact_mode_packages(self, mozconfig_builder): + self.ensure_mobile_android_packages(mozconfig_builder, artifact_mode=True) + + def ensure_browser_packages(self, artifact_mode=False): + # TODO: Figure out what not to install for artifact mode + self.aur_install(*self.BROWSER_AUR_PACKAGES) + self.pacman_install(*self.BROWSER_PACKAGES) + + def ensure_nasm_packages(self, state_dir, checkout_root): + # installed via ensure_browser_packages + pass + + def ensure_mobile_android_packages(self, mozconfig_builder, artifact_mode=False): + # Multi-part process: + # 1. System packages. + # 2. Android SDK. Android NDK only if we are not in artifact mode. Android packages. + + # 1. This is hard to believe, but the Android SDK binaries are 32-bit + # and that conflicts with 64-bit Arch installations out of the box. The + # solution is to add the multilibs repository; unfortunately, this + # requires manual intervention. + try: + self.pacman_install(*self.MOBILE_ANDROID_COMMON_PACKAGES) + except Exception as e: + print( + "Failed to install all packages. The Android developer " + "toolchain requires 32 bit binaries be enabled (see " + "https://wiki.archlinux.org/index.php/Android). You may need to " + "manually enable the multilib repository following the instructions " + "at https://wiki.archlinux.org/index.php/Multilib." + ) + raise e + + # 2. Android pieces. + self.ensure_java(mozconfig_builder) + super().ensure_mobile_android_packages(artifact_mode=artifact_mode) + + def _update_package_manager(self): + self.pacman_update() + + def upgrade_mercurial(self, current): + self.pacman_install("mercurial") + + def pacman_install(self, *packages): + command = ["pacman", "-S", "--needed"] + if self.no_interactive: + command.append("--noconfirm") + + command.extend(packages) + + self.run_as_root(command) + + def pacman_update(self): + command = ["pacman", "-S", "--refresh"] + + self.run_as_root(command) + + def run(self, command, env=None): + subprocess.check_call(command, stdin=sys.stdin, env=env) + + def download(self, uri): + command = ["curl", "-L", "-O", uri] + self.run(command) + + def unpack(self, path, name, ext): + if ext == "gz": + compression = "-z" + elif ext == "bz": + compression == "-j" + elif exit == "xz": + compression == "x" + + name = os.path.join(path, name) + ".tar." + ext + command = ["tar", "-x", compression, "-f", name, "-C", path] + self.run(command) + + def makepkg(self, name): + command = ["makepkg", "-s"] + makepkg_env = os.environ.copy() + makepkg_env["PKGDEST"] = "." + makepkg_env["PKGEXT"] = ".pkg.tar.xz" + self.run(command, env=makepkg_env) + pack = glob.glob(name + "*.pkg.tar.xz")[0] + command = ["pacman", "-U"] + if self.no_interactive: + command.append("--noconfirm") + command.append(pack) + self.run_as_root(command) + + def aur_install(self, *packages): + path = tempfile.mkdtemp() + if not self.no_interactive: + print( + "WARNING! This script requires to install packages from the AUR " + "This is potentially unsecure so I recommend that you carefully " + "read each package description and check the sources." + "These packages will be built in " + path + "." + ) + choice = input("Do you want to continue? (yes/no) [no]") + if choice != "yes": + sys.exit(1) + + base_dir = os.getcwd() + os.chdir(path) + for package in packages: + name, _, ext = package.split("/")[-1].split(".") + directory = os.path.join(path, name) + self.download(package) + self.unpack(path, name, ext) + os.chdir(directory) + self.makepkg(name) + + os.chdir(base_dir) diff --git a/python/mozboot/mozboot/base.py b/python/mozboot/mozboot/base.py new file mode 100644 index 0000000000..5756d32628 --- /dev/null +++ b/python/mozboot/mozboot/base.py @@ -0,0 +1,904 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import hashlib +import os +import platform +import re +import subprocess +import sys + +from distutils.version import LooseVersion +from mozboot import rust +from mozboot.util import ( + get_mach_virtualenv_binary, + locate_java_bin_path, + MINIMUM_RUST_VERSION, +) +from mozfile import which + +# NOTE: This script is intended to be run with a vanilla Python install. We +# have to rely on the standard library instead of Python 2+3 helpers like +# the six module. +if sys.version_info < (3,): + from urllib2 import urlopen + + input = raw_input # noqa +else: + from urllib.request import urlopen + + +NO_MERCURIAL = """ +Could not find Mercurial (hg) in the current shell's path. Try starting a new +shell and running the bootstrapper again. +""" + +MERCURIAL_UNABLE_UPGRADE = """ +You are currently running Mercurial %s. Running %s or newer is +recommended for performance and stability reasons. + +Unfortunately, this bootstrapper currently does not know how to automatically +upgrade Mercurial on your machine. + +You can usually install Mercurial through your package manager or by +downloading a package from http://mercurial.selenic.com/. +""" + +MERCURIAL_UPGRADE_FAILED = """ +We attempted to upgrade Mercurial to a modern version (%s or newer). +However, you appear to have version %s still. + +It's possible your package manager doesn't support a modern version of +Mercurial. It's also possible Mercurial is not being installed in the search +path for this shell. Try creating a new shell and run this bootstrapper again. + +If it continues to fail, consider installing Mercurial by following the +instructions at http://mercurial.selenic.com/. +""" + +PYTHON_UNABLE_UPGRADE = """ +You are currently running Python %s. Running %s or newer (but +not 3.x) is required. + +Unfortunately, this bootstrapper does not currently know how to automatically +upgrade Python on your machine. + +Please search the Internet for how to upgrade your Python and try running this +bootstrapper again to ensure your machine is up to date. +""" + +RUST_INSTALL_COMPLETE = """ +Rust installation complete. You should now have rustc and cargo +in %(cargo_bin)s + +The installer tries to add these to your default shell PATH, so +restarting your shell and running this script again may work. +If it doesn't, you'll need to add the new command location +manually. + +If restarting doesn't work, edit your shell initialization +script, which may be called ~/.bashrc or ~/.bash_profile or +~/.profile, and add the following line: + + %(cmd)s + +Then restart your shell and run the bootstrap script again. +""" + +RUST_NOT_IN_PATH = """ +You have some rust files in %(cargo_bin)s +but they're not part of this shell's PATH. + +To add these to the PATH, edit your shell initialization +script, which may be called ~/.bashrc or ~/.bash_profile or +~/.profile, and add the following line: + + %(cmd)s + +Then restart your shell and run the bootstrap script again. +""" + +RUSTUP_OLD = """ +We found an executable called `rustup` which we normally use to install +and upgrade Rust programming language support, but we didn't understand +its output. It may be an old version, or not be the installer from +https://rustup.rs/ + +Please move it out of the way and run the bootstrap script again. +Or if you prefer and know how, use the current rustup to install +a compatible version of the Rust programming language yourself. +""" + +RUST_UPGRADE_FAILED = """ +We attempted to upgrade Rust to a modern version (%s or newer). +However, you appear to still have version %s. + +It's possible rustup failed. It's also possible the new Rust is not being +installed in the search path for this shell. Try creating a new shell and +run this bootstrapper again. + +If this continues to fail and you are sure you have a modern Rust on your +system, ensure it is on the $PATH and try again. If that fails, you'll need to +install Rust manually. + +We recommend the installer from https://rustup.rs/ for installing Rust, +but you may be able to get a recent enough version from a software install +tool or package manager on your system, or directly from https://rust-lang.org/ +""" + +BROWSER_ARTIFACT_MODE_MOZCONFIG = """ +# Automatically download and use compiled C++ components: +ac_add_options --enable-artifact-builds +""".strip() + +# Upgrade Mercurial older than this. +# This should match the OLDEST_NON_LEGACY_VERSION in +# version-control-tools/hgext/configwizard/__init__.py. +MODERN_MERCURIAL_VERSION = LooseVersion("4.9") + +MODERN_PYTHON2_VERSION = LooseVersion("2.7.3") +MODERN_PYTHON3_VERSION = LooseVersion("3.6.0") + +# Upgrade rust older than this. +MODERN_RUST_VERSION = LooseVersion(MINIMUM_RUST_VERSION) + +# Upgrade nasm older than this. +MODERN_NASM_VERSION = LooseVersion("2.14") + + +class BaseBootstrapper(object): + """Base class for system bootstrappers.""" + + INSTALL_PYTHON_GUIDANCE = ( + "We do not have specific instructions for your platform on how to " + "install Python. You may find Pyenv (https://github.com/pyenv/pyenv) " + "helpful, if your system package manager does not provide a way to " + "install a recent enough Python 3 and 2." + ) + + def __init__(self, no_interactive=False, no_system_changes=False): + self.package_manager_updated = False + self.no_interactive = no_interactive + self.no_system_changes = no_system_changes + self.state_dir = None + + def validate_environment(self, srcdir): + """ + Called once the current firefox checkout has been detected. + Platform-specific implementations should check the environment and offer advice/warnings + to the user, if necessary. + """ + + def suggest_install_distutils(self): + """Called if distutils.{sysconfig,spawn} can't be imported.""" + print( + "Does your distro require installing another package for distutils?", + file=sys.stderr, + ) + + def suggest_install_pip3(self): + """Called if pip3 can't be found.""" + print( + "Try installing pip3 with your system's package manager.", file=sys.stderr + ) + + def install_system_packages(self): + """ + Install packages shared by all applications. These are usually + packages required by the development (like mercurial) or the + build system (like autoconf). + """ + raise NotImplementedError( + "%s must implement install_system_packages()" % __name__ + ) + + def install_browser_packages(self, mozconfig_builder): + """ + Install packages required to build Firefox for Desktop (application + 'browser'). + """ + raise NotImplementedError( + "Cannot bootstrap Firefox for Desktop: " + "%s does not yet implement install_browser_packages()" % __name__ + ) + + def generate_browser_mozconfig(self): + """ + Print a message to the console detailing what the user's mozconfig + should contain. + + Firefox for Desktop can in simple cases determine its build environment + entirely from configure. + """ + pass + + def install_browser_artifact_mode_packages(self, mozconfig_builder): + """ + Install packages required to build Firefox for Desktop (application + 'browser') in Artifact Mode. + """ + raise NotImplementedError( + "Cannot bootstrap Firefox for Desktop Artifact Mode: " + "%s does not yet implement install_browser_artifact_mode_packages()" + % __name__ + ) + + def generate_browser_artifact_mode_mozconfig(self): + """ + Print a message to the console detailing what the user's mozconfig + should contain. + + Firefox for Desktop Artifact Mode needs to enable artifact builds and + a path where the build artifacts will be written to. + """ + return BROWSER_ARTIFACT_MODE_MOZCONFIG + + def install_mobile_android_packages(self, mozconfig_builder): + """ + Install packages required to build Firefox for Android (application + 'mobile/android', also known as Fennec). + """ + raise NotImplementedError( + "Cannot bootstrap GeckoView/Firefox for Android: " + "%s does not yet implement install_mobile_android_packages()" % __name__ + ) + + def generate_mobile_android_mozconfig(self): + """ + Print a message to the console detailing what the user's mozconfig + should contain. + + GeckoView/Firefox for Android needs an application and an ABI set, and it needs + paths to the Android SDK and NDK. + """ + raise NotImplementedError( + "%s does not yet implement generate_mobile_android_mozconfig()" % __name__ + ) + + def install_mobile_android_artifact_mode_packages(self, mozconfig_builder): + """ + Install packages required to build GeckoView/Firefox for Android (application + 'mobile/android', also known as Fennec) in Artifact Mode. + """ + raise NotImplementedError( + "Cannot bootstrap GeckoView/Firefox for Android Artifact Mode: " + "%s does not yet implement install_mobile_android_artifact_mode_packages()" + % __name__ + ) + + def generate_mobile_android_artifact_mode_mozconfig(self): + """ + Print a message to the console detailing what the user's mozconfig + should contain. + + GeckoView/Firefox for Android Artifact Mode needs an application and an ABI set, + and it needs paths to the Android SDK. + """ + raise NotImplementedError( + "%s does not yet implement generate_mobile_android_artifact_mode_mozconfig()" + % __name__ + ) + + def ensure_mach_environment(self, checkout_root): + mach_binary = os.path.abspath(os.path.join(checkout_root, "mach")) + if not os.path.exists(mach_binary): + raise ValueError("mach not found at %s" % mach_binary) + cmd = [sys.executable, mach_binary, "create-mach-environment"] + subprocess.check_call(cmd, cwd=checkout_root) + + def ensure_clang_static_analysis_package(self, state_dir, checkout_root): + """ + Install the clang static analysis package + """ + raise NotImplementedError( + "%s does not yet implement ensure_clang_static_analysis_package()" + % __name__ + ) + + def ensure_stylo_packages(self, state_dir, checkout_root): + """ + Install any necessary packages needed for Stylo development. + """ + raise NotImplementedError( + "%s does not yet implement ensure_stylo_packages()" % __name__ + ) + + def ensure_nasm_packages(self, state_dir, checkout_root): + """ + Install nasm. + """ + raise NotImplementedError( + "%s does not yet implement ensure_nasm_packages()" % __name__ + ) + + def ensure_sccache_packages(self, state_dir, checkout_root): + """ + Install sccache. + """ + pass + + def ensure_lucetc_packages(self, state_dir, checkout_root): + """ + Install lucetc. + """ + pass + + def ensure_wasi_sysroot_packages(self, state_dir, checkout_root): + """ + Install the wasi sysroot. + """ + pass + + def ensure_node_packages(self, state_dir, checkout_root): + """ + Install any necessary packages needed to supply NodeJS""" + raise NotImplementedError( + "%s does not yet implement ensure_node_packages()" % __name__ + ) + + def ensure_dump_syms_packages(self, state_dir, checkout_root): + """ + Install dump_syms. + """ + pass + + def ensure_fix_stacks_packages(self, state_dir, checkout_root): + """ + Install fix-stacks. + """ + pass + + def ensure_minidump_stackwalk_packages(self, state_dir, checkout_root): + """ + Install minidump_stackwalk. + """ + pass + + def install_toolchain_static_analysis( + self, state_dir, checkout_root, toolchain_job + ): + clang_tools_path = os.path.join(state_dir, "clang-tools") + if not os.path.exists(clang_tools_path): + os.mkdir(clang_tools_path) + self.install_toolchain_artifact(clang_tools_path, checkout_root, toolchain_job) + + def install_toolchain_artifact( + self, state_dir, checkout_root, toolchain_job, no_unpack=False + ): + mach_binary = os.path.join(checkout_root, "mach") + mach_binary = os.path.abspath(mach_binary) + if not os.path.exists(mach_binary): + raise ValueError("mach not found at %s" % mach_binary) + + # NOTE: Use self.state_dir over the passed-in state_dir, which might be + # a subdirectory of the actual state directory. + if not self.state_dir: + raise ValueError( + "Need a state directory (e.g. ~/.mozbuild) to download " "artifacts" + ) + python_location = get_mach_virtualenv_binary(state_dir=self.state_dir) + if not os.path.exists(python_location): + raise ValueError("python not found at %s" % python_location) + + cmd = [ + python_location, + mach_binary, + "artifact", + "toolchain", + "--bootstrap", + "--from-build", + toolchain_job, + ] + + if no_unpack: + cmd += ["--no-unpack"] + + subprocess.check_call(cmd, cwd=state_dir) + + def run_as_root(self, command): + if os.geteuid() != 0: + if which("sudo"): + command.insert(0, "sudo") + else: + command = ["su", "root", "-c", " ".join(command)] + + print("Executing as root:", subprocess.list2cmdline(command)) + + subprocess.check_call(command, stdin=sys.stdin) + + def dnf_install(self, *packages): + if which("dnf"): + command = ["dnf", "install"] + else: + command = ["yum", "install"] + + if self.no_interactive: + command.append("-y") + command.extend(packages) + + self.run_as_root(command) + + def dnf_groupinstall(self, *packages): + if which("dnf"): + command = ["dnf", "groupinstall"] + else: + command = ["yum", "groupinstall"] + + if self.no_interactive: + command.append("-y") + command.extend(packages) + + self.run_as_root(command) + + def dnf_update(self, *packages): + if which("dnf"): + command = ["dnf", "update"] + else: + command = ["yum", "update"] + + if self.no_interactive: + command.append("-y") + command.extend(packages) + + self.run_as_root(command) + + def apt_install(self, *packages): + command = ["apt-get", "install"] + if self.no_interactive: + command.append("-y") + command.extend(packages) + + self.run_as_root(command) + + def apt_update(self): + command = ["apt-get", "update"] + if self.no_interactive: + command.append("-y") + + self.run_as_root(command) + + def apt_add_architecture(self, arch): + command = ["dpkg", "--add-architecture"] + command.extend(arch) + + self.run_as_root(command) + + def prompt_int(self, prompt, low, high, limit=5): + """ Prompts the user with prompt and requires an integer between low and high. """ + valid = False + while not valid and limit > 0: + try: + choice = int(input(prompt)) + if not low <= choice <= high: + print("ERROR! Please enter a valid option!") + limit -= 1 + else: + valid = True + except ValueError: + print("ERROR! Please enter a valid option!") + limit -= 1 + + if limit > 0: + return choice + else: + raise Exception("Error! Reached max attempts of entering option.") + + def prompt_yesno(self, prompt): + """ Prompts the user with prompt and requires a yes/no answer.""" + valid = False + while not valid: + choice = input(prompt + " (Yn): ").strip().lower()[:1] + if choice == "": + choice = "y" + if choice not in ("y", "n"): + print("ERROR! Please enter y or n!") + else: + valid = True + + return choice == "y" + + def _ensure_package_manager_updated(self): + if self.package_manager_updated: + return + + self._update_package_manager() + self.package_manager_updated = True + + def _update_package_manager(self): + """Updates the package manager's manifests/package list. + + This should be defined in child classes. + """ + + def _parse_version_impl(self, path, name, env, version_param): + """Execute the given path, returning the version. + + Invokes the path argument with the --version switch + and returns a LooseVersion representing the output + if successful. If not, returns None. + + An optional name argument gives the expected program + name returned as part of the version string, if it's + different from the basename of the executable. + + An optional env argument allows modifying environment + variable during the invocation to set options, PATH, + etc. + """ + if not name: + name = os.path.basename(path) + if name.lower().endswith(".exe"): + name = name[:-4] + + process = subprocess.run( + [path, version_param], + env=env, + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + if process.returncode != 0: + # This can happen e.g. if the user has an inactive pyenv shim in + # their path. Just silently treat this as a failure to parse the + # path and move on. + return None + + match = re.search(name + " ([a-z0-9\.]+)", process.stdout) + if not match: + print("ERROR! Unable to identify %s version." % name) + return None + + return LooseVersion(match.group(1)) + + def _parse_version(self, path, name=None, env=None): + return self._parse_version_impl(path, name, env, "--version") + + def _parse_version_short(self, path, name=None, env=None): + return self._parse_version_impl(path, name, env, "-v") + + def _hg_cleanenv(self, load_hgrc=False): + """Returns a copy of the current environment updated with the HGPLAIN + and HGRCPATH environment variables. + + HGPLAIN prevents Mercurial from applying locale variations to the output + making it suitable for use in scripts. + + HGRCPATH controls the loading of hgrc files. Setting it to the empty + string forces that no user or system hgrc file is used. + """ + env = os.environ.copy() + env["HGPLAIN"] = "1" + if not load_hgrc: + env["HGRCPATH"] = "" + + return env + + def is_mercurial_modern(self): + hg = which("hg") + if not hg: + print(NO_MERCURIAL) + return False, False, None + + our = self._parse_version(hg, "version", self._hg_cleanenv()) + if not our: + return True, False, None + + return True, our >= MODERN_MERCURIAL_VERSION, our + + def ensure_mercurial_modern(self): + installed, modern, version = self.is_mercurial_modern() + + if modern: + print("Your version of Mercurial (%s) is sufficiently modern." % version) + return installed, modern + + self._ensure_package_manager_updated() + + if installed: + print("Your version of Mercurial (%s) is not modern enough." % version) + print( + "(Older versions of Mercurial have known security vulnerabilities. " + "Unless you are running a patched Mercurial version, you may be " + "vulnerable." + ) + else: + print("You do not have Mercurial installed") + + if self.upgrade_mercurial(version) is False: + return installed, modern + + installed, modern, after = self.is_mercurial_modern() + + if installed and not modern: + print(MERCURIAL_UPGRADE_FAILED % (MODERN_MERCURIAL_VERSION, after)) + + return installed, modern + + def upgrade_mercurial(self, current): + """Upgrade Mercurial. + + Child classes should reimplement this. + + Return False to not perform a version check after the upgrade is + performed. + """ + print(MERCURIAL_UNABLE_UPGRADE % (current, MODERN_MERCURIAL_VERSION)) + + def is_python_modern(self, major): + assert major in (2, 3) + + our = None + + if major == 3: + our = LooseVersion(platform.python_version()) + else: + for test in ("python2.7", "python"): + python = which(test) + if python: + candidate_version = self._parse_version(python, "Python") + if candidate_version and candidate_version.version[0] == major: + our = candidate_version + break + + if our is None: + return False, None + + modern = { + 2: MODERN_PYTHON2_VERSION, + 3: MODERN_PYTHON3_VERSION, + } + return our >= modern[major], our + + def ensure_python_modern(self): + modern, version = self.is_python_modern(3) + if modern: + print("Your version of Python 3 (%s) is new enough." % version) + else: + print( + "ERROR: Your version of Python 3 (%s) is not new enough. You " + "must have Python >= %s to build Firefox." + % (version, MODERN_PYTHON3_VERSION) + ) + print(self.INSTALL_PYTHON_GUIDANCE) + sys.exit(1) + modern, version = self.is_python_modern(2) + if modern: + print("Your version of Python 2 (%s) is new enough." % version) + else: + print( + "WARNING: Your version of Python 2 (%s) is not new enough. " + "You must have Python >= %s to build Firefox. Python 2 is " + "not required to build, so we will proceed. However, Python " + "2 is required for other development tasks, like running " + "tests; you may like to have Python 2 installed for that " + "reason." % (version, MODERN_PYTHON2_VERSION) + ) + print(self.INSTALL_PYTHON_GUIDANCE) + + def warn_if_pythonpath_is_set(self): + if "PYTHONPATH" in os.environ: + print( + "WARNING: Your PYTHONPATH environment variable is set. This can " + "cause flaky installations of the requirements, and other unexpected " + "issues with mach. It is recommended to unset this variable." + ) + + def is_nasm_modern(self): + nasm = which("nasm") + if not nasm: + return False + + our = self._parse_version_short(nasm, "version") + if not our: + return False + + return our >= MODERN_NASM_VERSION + + def is_rust_modern(self, cargo_bin): + rustc = which("rustc", extra_search_dirs=[cargo_bin]) + if not rustc: + print("Could not find a Rust compiler.") + return False, None + + our = self._parse_version(rustc) + if not our: + return False, None + + return our >= MODERN_RUST_VERSION, our + + def cargo_home(self): + cargo_home = os.environ.get( + "CARGO_HOME", os.path.expanduser(os.path.join("~", ".cargo")) + ) + cargo_bin = os.path.join(cargo_home, "bin") + return cargo_home, cargo_bin + + def win_to_msys_path(self, path): + """Convert a windows-style path to msys style.""" + drive, path = os.path.splitdrive(path) + path = "/".join(path.split("\\")) + if drive: + if path[0] == "/": + path = path[1:] + path = "/%s/%s" % (drive[:-1], path) + return path + + def print_rust_path_advice(self, template, cargo_home, cargo_bin): + # Suggest ~/.cargo/env if it exists. + if os.path.exists(os.path.join(cargo_home, "env")): + cmd = "source %s/env" % cargo_home + else: + # On Windows rustup doesn't write out ~/.cargo/env + # so fall back to a manual PATH update. Bootstrap + # only runs under msys, so a unix-style shell command + # is appropriate there. + cargo_bin = self.win_to_msys_path(cargo_bin) + cmd = "export PATH=%s:$PATH" % cargo_bin + print( + template + % { + "cargo_bin": cargo_bin, + "cmd": cmd, + } + ) + + def ensure_rust_modern(self): + cargo_home, cargo_bin = self.cargo_home() + modern, version = self.is_rust_modern(cargo_bin) + + if modern: + print("Your version of Rust (%s) is new enough." % version) + rustup = which("rustup", extra_search_dirs=[cargo_bin]) + if rustup: + self.ensure_rust_targets(rustup, version) + return + + if version: + print("Your version of Rust (%s) is too old." % version) + + rustup = which("rustup", extra_search_dirs=[cargo_bin]) + if rustup: + rustup_version = self._parse_version(rustup) + if not rustup_version: + print(RUSTUP_OLD) + sys.exit(1) + print("Found rustup. Will try to upgrade.") + self.upgrade_rust(rustup) + + modern, after = self.is_rust_modern(cargo_bin) + if not modern: + print(RUST_UPGRADE_FAILED % (MODERN_RUST_VERSION, after)) + sys.exit(1) + else: + # No rustup. Download and run the installer. + print("Will try to install Rust.") + self.install_rust() + + def ensure_rust_targets(self, rustup, rust_version): + """Make sure appropriate cross target libraries are installed.""" + target_list = subprocess.check_output( + [rustup, "target", "list"], universal_newlines=True + ) + targets = [ + line.split()[0] + for line in target_list.splitlines() + if "installed" in line or "default" in line + ] + print("Rust supports %s targets." % ", ".join(targets)) + + # Support 32-bit Windows on 64-bit Windows. + win32 = "i686-pc-windows-msvc" + win64 = "x86_64-pc-windows-msvc" + if rust.platform() == win64 and win32 not in targets: + subprocess.check_call([rustup, "target", "add", win32]) + + if "mobile_android" in self.application: + # Let's add the most common targets. + if rust_version < LooseVersion("1.33"): + arm_target = "armv7-linux-androideabi" + else: + arm_target = "thumbv7neon-linux-androideabi" + android_targets = ( + arm_target, + "aarch64-linux-android", + "i686-linux-android", + "x86_64-linux-android", + ) + for target in android_targets: + if target not in targets: + subprocess.check_call([rustup, "target", "add", target]) + + def upgrade_rust(self, rustup): + """Upgrade Rust. + + Invoke rustup from the given path to update the rust install.""" + subprocess.check_call([rustup, "update"]) + # This installs rustfmt when not already installed, or nothing + # otherwise, while the update above would have taken care of upgrading + # it. + subprocess.check_call([rustup, "component", "add", "rustfmt"]) + + def install_rust(self): + """Download and run the rustup installer.""" + import errno + import stat + import tempfile + + platform = rust.platform() + url = rust.rustup_url(platform) + checksum = rust.rustup_hash(platform) + if not url or not checksum: + print("ERROR: Could not download installer.") + sys.exit(1) + print("Downloading rustup-init... ", end="") + fd, rustup_init = tempfile.mkstemp(prefix=os.path.basename(url)) + os.close(fd) + try: + self.http_download_and_save(url, rustup_init, checksum) + mode = os.stat(rustup_init).st_mode + os.chmod(rustup_init, mode | stat.S_IRWXU) + print("Ok") + print("Running rustup-init...") + subprocess.check_call( + [ + rustup_init, + "-y", + "--default-toolchain", + "stable", + "--default-host", + platform, + "--component", + "rustfmt", + ] + ) + cargo_home, cargo_bin = self.cargo_home() + self.print_rust_path_advice(RUST_INSTALL_COMPLETE, cargo_home, cargo_bin) + finally: + try: + os.remove(rustup_init) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + def http_download_and_save(self, url, dest, hexhash, digest="sha256"): + """Download the given url and save it to dest. hexhash is a checksum + that will be used to validate the downloaded file using the given + digest algorithm. The value of digest can be any value accepted by + hashlib.new. The default digest used is 'sha256'.""" + f = urlopen(url) + h = hashlib.new(digest) + with open(dest, "wb") as out: + while True: + data = f.read(4096) + if data: + out.write(data) + h.update(data) + else: + break + if h.hexdigest() != hexhash: + os.remove(dest) + raise ValueError("Hash of downloaded file does not match expected hash") + + def ensure_java(self, mozconfig_builder): + """Verify the presence of java. + + Finds a valid Java (throwing an error if not possible) and encodes it to the mozconfig. + """ + + bin_dir = locate_java_bin_path() + mozconfig_builder.append( + """ + # Use the same Java binary that was specified in bootstrap. This way, if the default system + # Java is different than what Firefox needs, users should just need to override it (with + # $JAVA_HOME) when running bootstrap, rather than when interacting with the build. + ac_add_options --with-java-bin-path={} + """.format( + bin_dir + ) + ) + + print("Your version of Java ({}) is 1.8.".format(bin_dir)) + return bin_dir diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootstrap.py new file mode 100644 index 0000000000..529e9332e5 --- /dev/null +++ b/python/mozboot/mozboot/bootstrap.py @@ -0,0 +1,735 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from collections import OrderedDict + +import os +import platform +import re +import sys +import subprocess +import time +from distutils.version import LooseVersion +from mozfile import which + +# NOTE: This script is intended to be run with a vanilla Python install. We +# have to rely on the standard library instead of Python 2+3 helpers like +# the six module. +if sys.version_info < (3,): + from ConfigParser import ( + Error as ConfigParserError, + RawConfigParser, + ) + + input = raw_input # noqa +else: + from configparser import ( + Error as ConfigParserError, + RawConfigParser, + ) + +from mach.util import UserError + +from mozboot.base import MODERN_RUST_VERSION +from mozboot.centosfedora import CentOSFedoraBootstrapper +from mozboot.opensuse import OpenSUSEBootstrapper +from mozboot.debian import DebianBootstrapper +from mozboot.freebsd import FreeBSDBootstrapper +from mozboot.gentoo import GentooBootstrapper +from mozboot.osx import OSXBootstrapper +from mozboot.openbsd import OpenBSDBootstrapper +from mozboot.archlinux import ArchlinuxBootstrapper +from mozboot.solus import SolusBootstrapper +from mozboot.void import VoidBootstrapper +from mozboot.windows import WindowsBootstrapper +from mozboot.mozillabuild import MozillaBuildBootstrapper +from mozboot.mozconfig import find_mozconfig, MozconfigBuilder +from mozboot.util import get_state_dir + +# Use distro package to retrieve linux platform information +import distro + +APPLICATION_CHOICE = """ +Note on Artifact Mode: + +Artifact builds download prebuilt C++ components rather than building +them locally. Artifact builds are faster! + +Artifact builds are recommended for people working on Firefox or +Firefox for Android frontends, or the GeckoView Java API. They are unsuitable +for those working on C++ code. For more information see: +https://firefox-source-docs.mozilla.org/contributing/build/artifact_builds.html. + +Please choose the version of Firefox you want to build: +%s +Your choice: """ + +APPLICATIONS = OrderedDict( + [ + ("Firefox for Desktop Artifact Mode", "browser_artifact_mode"), + ("Firefox for Desktop", "browser"), + ("GeckoView/Firefox for Android Artifact Mode", "mobile_android_artifact_mode"), + ("GeckoView/Firefox for Android", "mobile_android"), + ] +) + +STATE_DIR_INFO = """ +The Firefox build system and related tools store shared, persistent state +in a common directory on the filesystem. On this machine, that directory +is: + + {statedir} + +If you would like to use a different directory, hit CTRL+c and set the +MOZBUILD_STATE_PATH environment variable to the directory you'd like to +use and re-run the bootstrapper. + +Would you like to create this directory?""" + +FINISHED = """ +Your system should be ready to build %s! +""" + +MOZCONFIG_SUGGESTION_TEMPLATE = """ +Paste the lines between the chevrons (>>> and <<<) into +%s: + +>>> +%s +<<< +""".strip() + +CONFIGURE_MERCURIAL = """ +Mozilla recommends a number of changes to Mercurial to enhance your +experience with it. + +Would you like to run a configuration wizard to ensure Mercurial is +optimally configured?""" + +CONFIGURE_GIT = """ +Mozilla recommends using git-cinnabar to work with mozilla-central (or +mozilla-unified). + +Would you like to run a few configuration steps to ensure Git is +optimally configured?""" + +DEBIAN_DISTROS = ( + "debian", + "ubuntu", + "linuxmint", + "elementary", + "neon", + "pop", +) + +ADD_GIT_CINNABAR_PATH = """ +To add git-cinnabar to the PATH, edit your shell initialization script, which +may be called ~/.bashrc or ~/.bash_profile or ~/.profile, and add the following +lines: + + export PATH="{}:$PATH" + +Then restart your shell. +""" + +TELEMETRY_OPT_IN_PROMPT = """ +Build system telemetry + +Mozilla collects data about local builds in order to make builds faster and +improve developer tooling. To learn more about the data we intend to collect +read here: + + https://firefox-source-docs.mozilla.org/build/buildsystem/telemetry.html + +If you have questions, please ask in #build on Matrix: + + https://chat.mozilla.org/#/room/#build:mozilla.org + +If you would like to opt out of data collection, select (N) at the prompt. + +Would you like to enable build system telemetry?""" + + +OLD_REVISION_WARNING = """ +WARNING! You appear to be running `mach bootstrap` from an old revision. +bootstrap is meant primarily for getting developer environments up-to-date to +build the latest version of tree. Running bootstrap on old revisions may fail +and is not guaranteed to bring your machine to any working state in particular. +Proceed at your own peril. +""" + + +# Version 2.24 changes the "core.commitGraph" setting to be "True" by default. +MINIMUM_RECOMMENDED_GIT_VERSION = LooseVersion("2.24") +OLD_GIT_WARNING = """ +You are running an older version of git ("{old_version}"). +We recommend upgrading to at least version "{minimum_recommended_version}" to improve +performance. +""".strip() + + +def update_or_create_build_telemetry_config(path): + """Write a mach config file enabling build telemetry to `path`. If the file does not exist, + create it. If it exists, add the new setting to the existing data. + + This is standalone from mach's `ConfigSettings` so we can use it during bootstrap + without a source checkout. + """ + config = RawConfigParser() + if os.path.exists(path): + try: + config.read([path]) + except ConfigParserError as e: + print( + "Your mach configuration file at `{path}` is not parseable:\n{error}".format( + path=path, error=e + ) + ) + return False + if not config.has_section("build"): + config.add_section("build") + config.set("build", "telemetry", "true") + with open(path, "w") as f: + config.write(f) + return True + + +class Bootstrapper(object): + """Main class that performs system bootstrap.""" + + def __init__( + self, + choice=None, + no_interactive=False, + hg_configure=False, + no_system_changes=False, + mach_context=None, + ): + self.instance = None + self.choice = choice + self.hg_configure = hg_configure + self.no_system_changes = no_system_changes + self.mach_context = mach_context + cls = None + args = { + "no_interactive": no_interactive, + "no_system_changes": no_system_changes, + } + + if sys.platform.startswith("linux"): + # distro package provides reliable ids for popular distributions so + # we use those instead of the full distribution name + dist_id, version, codename = distro.linux_distribution( + full_distribution_name=False + ) + + if dist_id in ("centos", "fedora"): + cls = CentOSFedoraBootstrapper + args["distro"] = dist_id + elif dist_id in DEBIAN_DISTROS: + cls = DebianBootstrapper + args["distro"] = dist_id + args["codename"] = codename + elif dist_id in ("gentoo", "funtoo"): + cls = GentooBootstrapper + elif dist_id in ("solus"): + cls = SolusBootstrapper + elif dist_id in ("arch") or os.path.exists("/etc/arch-release"): + cls = ArchlinuxBootstrapper + elif dist_id in ("void"): + cls = VoidBootstrapper + elif os.path.exists("/etc/SUSE-brand"): + cls = OpenSUSEBootstrapper + else: + raise NotImplementedError( + "Bootstrap support for this Linux " + "distro not yet available: " + dist_id + ) + + args["version"] = version + args["dist_id"] = dist_id + + elif sys.platform.startswith("darwin"): + # TODO Support Darwin platforms that aren't OS X. + osx_version = platform.mac_ver()[0] + + cls = OSXBootstrapper + args["version"] = osx_version + + elif sys.platform.startswith("openbsd"): + cls = OpenBSDBootstrapper + args["version"] = platform.uname()[2] + + elif sys.platform.startswith("dragonfly") or sys.platform.startswith("freebsd"): + cls = FreeBSDBootstrapper + args["version"] = platform.release() + args["flavor"] = platform.system() + + elif sys.platform.startswith("win32") or sys.platform.startswith("msys"): + if "MOZILLABUILD" in os.environ: + cls = MozillaBuildBootstrapper + else: + cls = WindowsBootstrapper + + if cls is None: + raise NotImplementedError( + "Bootstrap support is not yet available " "for your OS." + ) + + self.instance = cls(**args) + + def create_state_dir(self): + state_dir = get_state_dir() + + if not os.path.exists(state_dir): + should_create_state_dir = True + if not self.instance.no_interactive: + should_create_state_dir = self.instance.prompt_yesno( + prompt=STATE_DIR_INFO.format(statedir=state_dir) + ) + + # This directory is by default in $HOME, or overridden via an env + # var, so we probably shouldn't gate it on --no-system-changes. + if should_create_state_dir: + print("Creating global state directory: %s" % state_dir) + os.makedirs(state_dir, mode=0o770) + else: + raise UserError( + "Need permission to create global state " + "directory at %s" % state_dir + ) + + return state_dir + + def maybe_install_private_packages_or_exit(self, state_dir, checkout_root): + # Install the clang packages needed for building the style system, as + # well as the version of NodeJS that we currently support. + # Also install the clang static-analysis package by default + # The best place to install our packages is in the state directory + # we have. We should have created one above in non-interactive mode. + self.instance.ensure_node_packages(state_dir, checkout_root) + self.instance.ensure_fix_stacks_packages(state_dir, checkout_root) + self.instance.ensure_minidump_stackwalk_packages(state_dir, checkout_root) + if not self.instance.artifact_mode: + self.instance.ensure_stylo_packages(state_dir, checkout_root) + self.instance.ensure_clang_static_analysis_package(state_dir, checkout_root) + self.instance.ensure_nasm_packages(state_dir, checkout_root) + self.instance.ensure_sccache_packages(state_dir, checkout_root) + self.instance.ensure_lucetc_packages(state_dir, checkout_root) + self.instance.ensure_wasi_sysroot_packages(state_dir, checkout_root) + self.instance.ensure_dump_syms_packages(state_dir, checkout_root) + + def check_telemetry_opt_in(self, state_dir): + # Don't prompt if the user already has a setting for this value. + if ( + self.mach_context is not None + and "telemetry" in self.mach_context.settings.build + ): + return self.mach_context.settings.build.telemetry + # We can't prompt the user. + if self.instance.no_interactive: + return False + choice = self.instance.prompt_yesno(prompt=TELEMETRY_OPT_IN_PROMPT) + if choice: + cfg_file = os.path.join(state_dir, "machrc") + if update_or_create_build_telemetry_config(cfg_file): + print( + "\nThanks for enabling build telemetry! You can change this setting at " + + "any time by editing the config file `{}`\n".format(cfg_file) + ) + return choice + + def check_code_submission(self, checkout_root): + if self.instance.no_interactive or which("moz-phab"): + return + + if not self.instance.prompt_yesno("Will you be submitting commits to Mozilla?"): + return + + mach_binary = os.path.join(checkout_root, "mach") + subprocess.check_call((sys.executable, mach_binary, "install-moz-phab")) + + def bootstrap(self): + if sys.version_info[0] < 3: + print( + "This script must be run with Python 3. \n" + 'Try "python3 bootstrap.py".' + ) + sys.exit(1) + + if self.choice is None: + # Like ['1. Firefox for Desktop', '2. Firefox for Android Artifact Mode', ...]. + labels = [ + "%s. %s" % (i, name) for i, name in enumerate(APPLICATIONS.keys(), 1) + ] + prompt = APPLICATION_CHOICE % "\n".join( + " {}".format(label) for label in labels + ) + prompt_choice = self.instance.prompt_int( + prompt=prompt, low=1, high=len(APPLICATIONS) + ) + name, application = list(APPLICATIONS.items())[prompt_choice - 1] + elif self.choice in APPLICATIONS.keys(): + name, application = self.choice, APPLICATIONS[self.choice] + elif self.choice in APPLICATIONS.values(): + name, application = next( + (k, v) for k, v in APPLICATIONS.items() if v == self.choice + ) + else: + raise Exception( + "Please pick a valid application choice: (%s)" + % "/".join(APPLICATIONS.keys()) + ) + + mozconfig_builder = MozconfigBuilder() + self.instance.application = application + self.instance.artifact_mode = "artifact_mode" in application + + self.instance.warn_if_pythonpath_is_set() + + # This doesn't affect any system state and we'd like to bail out as soon + # as possible if this check fails. + self.instance.ensure_python_modern() + + state_dir = self.create_state_dir() + self.instance.state_dir = state_dir + + # We need to enable the loading of hgrc in case extensions are + # required to open the repo. + (checkout_type, checkout_root) = current_firefox_checkout( + env=self.instance._hg_cleanenv(load_hgrc=True), hg=which("hg") + ) + self.instance.validate_environment(checkout_root) + self._validate_python_environment() + + if self.instance.no_system_changes: + self.instance.ensure_mach_environment(checkout_root) + self.check_telemetry_opt_in(state_dir) + self.maybe_install_private_packages_or_exit(state_dir, checkout_root) + self._output_mozconfig(application, mozconfig_builder) + sys.exit(0) + + self.instance.install_system_packages() + # Install mach environment python packages after system packages. + # Some mach packages require building native modules, which require + # tools which are installed to the system. + self.instance.ensure_mach_environment(checkout_root) + + # Like 'install_browser_packages' or 'install_mobile_android_packages'. + getattr(self.instance, "install_%s_packages" % application)(mozconfig_builder) + + hg_installed, hg_modern = self.instance.ensure_mercurial_modern() + if not self.instance.artifact_mode: + self.instance.ensure_rust_modern() + + # Possibly configure Mercurial, but not if the current checkout or repo + # type is Git. + if hg_installed and checkout_type == "hg": + configure_hg = False + if not self.instance.no_interactive: + configure_hg = self.instance.prompt_yesno(prompt=CONFIGURE_MERCURIAL) + else: + configure_hg = self.hg_configure + + if configure_hg: + configure_mercurial(which("hg"), state_dir) + + # Offer to configure Git, if the current checkout or repo type is Git. + elif which("git") and checkout_type == "git": + should_configure_git = False + if not self.instance.no_interactive: + should_configure_git = self.instance.prompt_yesno(prompt=CONFIGURE_GIT) + else: + # Assuming default configuration setting applies to all VCS. + should_configure_git = self.hg_configure + + if should_configure_git: + configure_git( + which("git"), which("git-cinnabar"), state_dir, checkout_root + ) + + self.check_telemetry_opt_in(state_dir) + self.maybe_install_private_packages_or_exit(state_dir, checkout_root) + self.check_code_submission(checkout_root) + + print(FINISHED % name) + if not ( + which("rustc") + and self.instance._parse_version("rustc") >= MODERN_RUST_VERSION + ): + print( + "To build %s, please restart the shell (Start a new terminal window)" + % name + ) + + self._output_mozconfig(application, mozconfig_builder) + + def _output_mozconfig(self, application, mozconfig_builder): + # Like 'generate_browser_mozconfig' or 'generate_mobile_android_mozconfig'. + additional_mozconfig = getattr( + self.instance, "generate_%s_mozconfig" % application + )() + if additional_mozconfig: + mozconfig_builder.append(additional_mozconfig) + raw_mozconfig = mozconfig_builder.generate() + + if raw_mozconfig: + mozconfig_path = find_mozconfig(self.mach_context.topdir) + if not mozconfig_path: + # No mozconfig file exists yet + mozconfig_path = os.path.join(self.mach_context.topdir, "mozconfig") + with open(mozconfig_path, "w") as mozconfig_file: + mozconfig_file.write(raw_mozconfig) + print( + 'Your requested configuration has been written to "%s".' + % mozconfig_path + ) + else: + suggestion = MOZCONFIG_SUGGESTION_TEMPLATE % ( + mozconfig_path, + raw_mozconfig, + ) + print(suggestion) + + def _validate_python_environment(self): + valid = True + try: + # distutils is singled out here because some distros (namely Ubuntu) + # include it in a separate package outside of the main Python + # installation. + import distutils.sysconfig + import distutils.spawn + + assert distutils.sysconfig is not None and distutils.spawn is not None + except ImportError as e: + print("ERROR: Could not import package %s" % e.name, file=sys.stderr) + self.instance.suggest_install_distutils() + valid = False + except AssertionError: + print("ERROR: distutils is not behaving as expected.", file=sys.stderr) + self.instance.suggest_install_distutils() + valid = False + pip3 = which("pip3") + if not pip3: + print("ERROR: Could not find pip3.", file=sys.stderr) + self.instance.suggest_install_pip3() + valid = False + if not valid: + print( + "ERROR: Your Python installation will not be able to run " + "`mach bootstrap`. `mach bootstrap` cannot maintain your " + "Python environment for you; fix the errors shown here, and " + "then re-run `mach bootstrap`.", + file=sys.stderr, + ) + sys.exit(1) + + +def update_vct(hg, root_state_dir): + """Ensure version-control-tools in the state directory is up to date.""" + vct_dir = os.path.join(root_state_dir, "version-control-tools") + + # Ensure the latest revision of version-control-tools is present. + update_mercurial_repo( + hg, "https://hg.mozilla.org/hgcustom/version-control-tools", vct_dir, "@" + ) + + return vct_dir + + +def configure_mercurial(hg, root_state_dir): + """Run the Mercurial configuration wizard.""" + vct_dir = update_vct(hg, root_state_dir) + + # Run the config wizard from v-c-t. + args = [ + hg, + "--config", + "extensions.configwizard=%s/hgext/configwizard" % vct_dir, + "configwizard", + ] + subprocess.call(args) + + +def update_mercurial_repo(hg, url, dest, revision): + """Perform a clone/pull + update of a Mercurial repository.""" + # Disable common extensions whose older versions may cause `hg` + # invocations to abort. + disable_exts = [ + "bzexport", + "bzpost", + "firefoxtree", + "hgwatchman", + "mozext", + "mqext", + "qimportbz", + "push-to-try", + "reviewboard", + ] + + def disable_extensions(args): + for ext in disable_exts: + args.extend(["--config", "extensions.%s=!" % ext]) + + pull_args = [hg] + disable_extensions(pull_args) + + if os.path.exists(dest): + pull_args.extend(["pull", url]) + cwd = dest + else: + pull_args.extend(["clone", "--noupdate", url, dest]) + cwd = "/" + + update_args = [hg] + disable_extensions(update_args) + update_args.extend(["update", "-r", revision]) + + print("=" * 80) + print("Ensuring %s is up to date at %s" % (url, dest)) + + try: + subprocess.check_call(pull_args, cwd=cwd) + subprocess.check_call(update_args, cwd=dest) + finally: + print("=" * 80) + + +def current_firefox_checkout(env, hg=None): + """Determine whether we're in a Firefox checkout. + + Returns one of None, ``git``, or ``hg``. + """ + HG_ROOT_REVISIONS = set( + [ + # From mozilla-unified. + "8ba995b74e18334ab3707f27e9eb8f4e37ba3d29", + ] + ) + + path = os.getcwd() + while path: + hg_dir = os.path.join(path, ".hg") + git_dir = os.path.join(path, ".git") + if hg and os.path.exists(hg_dir): + # Verify the hg repo is a Firefox repo by looking at rev 0. + try: + node = subprocess.check_output( + [hg, "log", "-r", "0", "--template", "{node}"], + cwd=path, + env=env, + universal_newlines=True, + ) + if node in HG_ROOT_REVISIONS: + _warn_if_risky_revision(path) + return ("hg", path) + # Else the root revision is different. There could be nested + # repos. So keep traversing the parents. + except subprocess.CalledProcessError: + pass + + # Just check for known-good files in the checkout, to prevent attempted + # foot-shootings. Determining a canonical git checkout of mozilla-unified + # is...complicated + elif os.path.exists(git_dir): + moz_configure = os.path.join(path, "moz.configure") + if os.path.exists(moz_configure): + _warn_if_risky_revision(path) + return ("git", path) + + path, child = os.path.split(path) + if child == "": + break + + raise UserError( + "Could not identify the root directory of your checkout! " + "Are you running `mach bootstrap` in an hg or git clone?" + ) + + +def update_git_tools(git, root_state_dir, top_src_dir): + """Update git tools, hooks and extensions""" + # Ensure git-cinnabar is up to date. + cinnabar_dir = os.path.join(root_state_dir, "git-cinnabar") + + # Ensure the latest revision of git-cinnabar is present. + update_git_repo(git, "https://github.com/glandium/git-cinnabar.git", cinnabar_dir) + + # Perform a download of cinnabar. + download_args = [git, "cinnabar", "download"] + + try: + subprocess.check_call(download_args, cwd=cinnabar_dir) + except subprocess.CalledProcessError as e: + print(e) + return cinnabar_dir + + +def update_git_repo(git, url, dest): + """Perform a clone/pull + update of a Git repository.""" + pull_args = [git] + + if os.path.exists(dest): + pull_args.extend(["pull"]) + cwd = dest + else: + pull_args.extend(["clone", "--no-checkout", url, dest]) + cwd = "/" + + update_args = [git, "checkout"] + + print("=" * 80) + print("Ensuring %s is up to date at %s" % (url, dest)) + + try: + subprocess.check_call(pull_args, cwd=cwd) + subprocess.check_call(update_args, cwd=dest) + finally: + print("=" * 80) + + +def configure_git(git, cinnabar, root_state_dir, top_src_dir): + """Run the Git configuration steps.""" + + match = re.search( + r"(\d+\.\d+\.\d+)", + subprocess.check_output([git, "--version"], universal_newlines=True), + ) + if not match: + raise Exception("Could not find git version") + git_version = LooseVersion(match.group(1)) + + if git_version < MINIMUM_RECOMMENDED_GIT_VERSION: + print( + OLD_GIT_WARNING.format( + old_version=git_version, + minimum_recommended_version=MINIMUM_RECOMMENDED_GIT_VERSION, + ) + ) + + if git_version >= LooseVersion("2.17"): + # "core.untrackedCache" has a bug before 2.17 + subprocess.check_call( + [git, "config", "core.untrackedCache", "true"], cwd=top_src_dir + ) + + cinnabar_dir = update_git_tools(git, root_state_dir, top_src_dir) + + if not cinnabar: + print(ADD_GIT_CINNABAR_PATH.format(cinnabar_dir)) + + +def _warn_if_risky_revision(path): + # Warn the user if they're trying to bootstrap from an obviously old + # version of tree as reported by the version control system (a month in + # this case). This is an approximate calculation but is probably good + # enough for our purposes. + NUM_SECONDS_IN_MONTH = 60 * 60 * 24 * 30 + from mozversioncontrol import get_repository_object + + repo = get_repository_object(path) + if (time.time() - repo.get_commit_time()) >= NUM_SECONDS_IN_MONTH: + print(OLD_REVISION_WARNING) diff --git a/python/mozboot/mozboot/centosfedora.py b/python/mozboot/mozboot/centosfedora.py new file mode 100644 index 0000000000..c4a7e550f9 --- /dev/null +++ b/python/mozboot/mozboot/centosfedora.py @@ -0,0 +1,147 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import platform + +from mozboot.base import BaseBootstrapper +from mozboot.linux_common import LinuxBootstrapper + + +class CentOSFedoraBootstrapper(LinuxBootstrapper, BaseBootstrapper): + def __init__(self, distro, version, dist_id, **kwargs): + BaseBootstrapper.__init__(self, **kwargs) + + self.distro = distro + self.version = int(version.split(".")[0]) + self.dist_id = dist_id + + self.group_packages = [] + + # For CentOS 7, later versions of nodejs come from nodesource + # and include the npm package. + self.packages = [ + "nodejs", + "python-devel", + "which", + ] + + self.browser_group_packages = [ + "GNOME Software Development", + ] + + self.browser_packages = [ + "alsa-lib-devel", + "dbus-glib-devel", + "glibc-static", + "gtk2-devel", # It is optional in Fedora 20's GNOME Software + # Development group. + "libstdc++-static", + "libXt-devel", + "nasm", + "pulseaudio-libs-devel", + "wireless-tools-devel", + "yasm", + "gcc-c++", + ] + + self.mobile_android_packages = [ + "java-1.8.0-openjdk-devel", + # For downloading the Android SDK and NDK. + "wget", + ] + + if self.distro in ("centos"): + self.group_packages += [ + "Development Tools", + ] + + self.packages += [ + "curl-devel", + ] + + self.browser_packages += [ + "gtk3-devel", + ] + + if self.version == 6: + self.group_packages += [ + "Development Libraries", + "GNOME Software Development", + ] + + self.packages += [ + "npm", + ] + + else: + self.packages += [ + "redhat-rpm-config", + ] + + self.browser_group_packages = [ + "Development Tools", + ] + + elif self.distro == "fedora": + self.group_packages += [ + "C Development Tools and Libraries", + ] + + self.packages += [ + "npm", + "redhat-rpm-config", + ] + + self.mobile_android_packages += [ + "ncurses-compat-libs", + ] + + def install_system_packages(self): + self.dnf_groupinstall(*self.group_packages) + self.dnf_install(*self.packages) + + def install_browser_packages(self, mozconfig_builder): + self.ensure_browser_packages() + + def install_browser_artifact_mode_packages(self, mozconfig_builder): + self.ensure_browser_packages(artifact_mode=True) + + def install_mobile_android_packages(self, mozconfig_builder): + self.ensure_mobile_android_packages(mozconfig_builder, artifact_mode=False) + + def install_mobile_android_artifact_mode_packages(self, mozconfig_builder): + self.ensure_mobile_android_packages(mozconfig_builder, artifact_mode=True) + + def ensure_browser_packages(self, artifact_mode=False): + # TODO: Figure out what not to install for artifact mode + self.dnf_groupinstall(*self.browser_group_packages) + self.dnf_install(*self.browser_packages) + + if self.distro in ("centos") and self.version == 6: + yasm = ( + "http://dl.fedoraproject.org/pub/epel/6/i386/" + "Packages/y/yasm-1.2.0-1.el6.i686.rpm" + ) + if platform.architecture()[0] == "64bit": + yasm = ( + "http://dl.fedoraproject.org/pub/epel/6/x86_64/" + "Packages/y/yasm-1.2.0-1.el6.x86_64.rpm" + ) + + self.run_as_root(["rpm", "-ivh", yasm]) + + def ensure_mobile_android_packages(self, mozconfig_builder, artifact_mode=False): + # Install Android specific packages. + self.dnf_install(*self.mobile_android_packages) + + self.ensure_java(mozconfig_builder) + super().ensure_mobile_android_packages(artifact_mode=artifact_mode) + + def upgrade_mercurial(self, current): + if current is None: + self.dnf_install("mercurial") + else: + self.dnf_update("mercurial") diff --git a/python/mozboot/mozboot/debian.py b/python/mozboot/mozboot/debian.py new file mode 100644 index 0000000000..334acee6a3 --- /dev/null +++ b/python/mozboot/mozboot/debian.py @@ -0,0 +1,154 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from mozboot.base import BaseBootstrapper +from mozboot.linux_common import LinuxBootstrapper + +import sys + +MERCURIAL_INSTALL_PROMPT = """ +Mercurial releases a new version every 3 months and your distro's package +may become out of date. This may cause incompatibility with some +Mercurial extensions that rely on new Mercurial features. As a result, +you may not have an optimal version control experience. + +To have the best Mercurial experience possible, we recommend installing +Mercurial via the "pip" Python packaging utility. This will likely result +in files being placed in /usr/local/bin and /usr/local/lib. + +How would you like to continue? + 1. Install a modern Mercurial via pip (recommended) + 2. Install a legacy Mercurial via apt + 3. Do not install Mercurial +Your choice: """ + + +class DebianBootstrapper(LinuxBootstrapper, BaseBootstrapper): + + # These are common packages for all Debian-derived distros (such as + # Ubuntu). + COMMON_PACKAGES = [ + "build-essential", + "libpython3-dev", + "nodejs", + "unzip", + "uuid", + "zip", + ] + + # Ubuntu and Debian don't often differ, but they do for npm. + DEBIAN_PACKAGES = [ + # Comment the npm package until Debian bring it back + # 'npm' + ] + + # These are common packages for building Firefox for Desktop + # (browser) for all Debian-derived distros (such as Ubuntu). + BROWSER_COMMON_PACKAGES = [ + "libasound2-dev", + "libcurl4-openssl-dev", + "libdbus-1-dev", + "libdbus-glib-1-dev", + "libdrm-dev", + "libgtk-3-dev", + "libgtk2.0-dev", + "libpulse-dev", + "libx11-xcb-dev", + "libxt-dev", + "xvfb", + "yasm", + ] + + # These are common packages for building Firefox for Android + # (mobile/android) for all Debian-derived distros (such as Ubuntu). + MOBILE_ANDROID_COMMON_PACKAGES = [ + "openjdk-8-jdk-headless", # Android's `sdkmanager` requires Java 1.8 exactly. + "wget", # For downloading the Android SDK and NDK. + ] + + def __init__(self, distro, version, dist_id, codename, **kwargs): + BaseBootstrapper.__init__(self, **kwargs) + + self.distro = distro + self.version = version + self.dist_id = dist_id + self.codename = codename + + self.packages = list(self.COMMON_PACKAGES) + if self.distro == "debian": + self.packages += self.DEBIAN_PACKAGES + + def suggest_install_distutils(self): + print( + "HINT: Try installing distutils with " + "`apt-get install python3-distutils`.", + file=sys.stderr, + ) + + def suggest_install_pip3(self): + print( + "HINT: Try installing pip3 with `apt-get install python3-pip`.", + file=sys.stderr, + ) + + def install_system_packages(self): + self.apt_install(*self.packages) + + def install_browser_packages(self, mozconfig_builder): + self.ensure_browser_packages() + + def install_browser_artifact_mode_packages(self, mozconfig_builder): + self.ensure_browser_packages(artifact_mode=True) + + def install_mobile_android_packages(self, mozconfig_builder): + self.ensure_mobile_android_packages(mozconfig_builder) + + def install_mobile_android_artifact_mode_packages(self, mozconfig_builder): + self.ensure_mobile_android_packages(mozconfig_builder, artifact_mode=True) + + def ensure_browser_packages(self, artifact_mode=False): + # TODO: Figure out what not to install for artifact mode + self.apt_install(*self.BROWSER_COMMON_PACKAGES) + modern = self.is_nasm_modern() + if not modern: + self.apt_install("nasm") + + def ensure_mobile_android_packages(self, mozconfig_builder, artifact_mode=False): + # Multi-part process: + # 1. System packages. + # 2. Android SDK. Android NDK only if we are not in artifact mode. Android packages. + self.apt_install(*self.MOBILE_ANDROID_COMMON_PACKAGES) + + # 2. Android pieces. + self.ensure_java(mozconfig_builder) + super().ensure_mobile_android_packages(artifact_mode=artifact_mode) + + def _update_package_manager(self): + self.apt_update() + + def upgrade_mercurial(self, current): + """Install Mercurial from pip because Debian packages typically lag.""" + if self.no_interactive: + # Install via Apt in non-interactive mode because it is the more + # conservative option and less likely to make people upset. + self.apt_install("mercurial") + return + + res = self.prompt_int(MERCURIAL_INSTALL_PROMPT, 1, 3) + + # Apt. + if res == 2: + self.apt_install("mercurial") + return False + + # No Mercurial. + if res == 3: + print("Not installing Mercurial.") + return False + + # pip. + assert res == 1 + self.run_as_root(["pip3", "install", "--upgrade", "Mercurial"]) diff --git a/python/mozboot/mozboot/dump_syms.py b/python/mozboot/mozboot/dump_syms.py new file mode 100644 index 0000000000..4987d55bbf --- /dev/null +++ b/python/mozboot/mozboot/dump_syms.py @@ -0,0 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +LINUX_DUMP_SYMS = "linux64-dump-syms" +MACOS_DUMP_SYMS = "macosx64-dump-syms" +WIN64_DUMP_SYMS = "win64-dump-syms" diff --git a/python/mozboot/mozboot/fix_stacks.py b/python/mozboot/mozboot/fix_stacks.py new file mode 100644 index 0000000000..924af3fae3 --- /dev/null +++ b/python/mozboot/mozboot/fix_stacks.py @@ -0,0 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +LINUX_FIX_STACKS = "linux64-fix-stacks" +MACOS_FIX_STACKS = "macosx64-fix-stacks" +WINDOWS_FIX_STACKS = "win32-fix-stacks" diff --git a/python/mozboot/mozboot/freebsd.py b/python/mozboot/mozboot/freebsd.py new file mode 100644 index 0000000000..abce6b4854 --- /dev/null +++ b/python/mozboot/mozboot/freebsd.py @@ -0,0 +1,83 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals +import sys + +from mozboot.base import BaseBootstrapper +from mozfile import which + + +class FreeBSDBootstrapper(BaseBootstrapper): + def __init__(self, version, flavor, **kwargs): + BaseBootstrapper.__init__(self, **kwargs) + self.version = int(version.split(".")[0]) + self.flavor = flavor.lower() + + self.packages = [ + "gmake", + "gtar", + "pkgconf", + "py%s%s-sqlite3" % sys.version_info[0:2], + "rust", + "watchman", + "zip", + ] + + self.browser_packages = [ + "dbus-glib", + "gtk2", + "gtk3", + "libXt", + "mesa-dri", # depends on llvm* + "nasm", + "pulseaudio", + "v4l_compat", + "yasm", + ] + + if not which("as"): + self.packages.append("binutils") + + if not which("unzip"): + self.packages.append("unzip") + + def pkg_install(self, *packages): + command = ["pkg", "install"] + if self.no_interactive: + command.append("-y") + + command.extend(packages) + self.run_as_root(command) + + def install_system_packages(self): + self.pkg_install(*self.packages) + + def install_browser_packages(self, mozconfig_builder): + self.ensure_browser_packages() + + def install_browser_artifact_mode_packages(self, mozconfig_builder): + self.ensure_browser_packages(artifact_mode=True) + + def ensure_browser_packages(self, artifact_mode=False): + # TODO: Figure out what not to install for artifact mode + self.pkg_install(*self.browser_packages) + + def ensure_clang_static_analysis_package(self, state_dir, checkout_root): + # TODO: we don't ship clang base static analysis for this platform + pass + + def ensure_stylo_packages(self, state_dir, checkout_root): + # Clang / llvm already installed as browser package + self.pkg_install("rust-cbindgen") + + def ensure_nasm_packages(self, state_dir, checkout_root): + # installed via ensure_browser_packages + pass + + def ensure_node_packages(self, state_dir, checkout_root): + self.pkg_install("npm") + + def upgrade_mercurial(self, current): + self.pkg_install("mercurial") diff --git a/python/mozboot/mozboot/gentoo.py b/python/mozboot/mozboot/gentoo.py new file mode 100644 index 0000000000..dd521feb1a --- /dev/null +++ b/python/mozboot/mozboot/gentoo.py @@ -0,0 +1,71 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from mozboot.base import BaseBootstrapper +from mozboot.linux_common import LinuxBootstrapper + + +class GentooBootstrapper(LinuxBootstrapper, BaseBootstrapper): + def __init__(self, version, dist_id, **kwargs): + BaseBootstrapper.__init__(self, **kwargs) + + self.version = version + self.dist_id = dist_id + + def install_system_packages(self): + self.ensure_system_packages() + + def install_browser_packages(self, mozconfig_builder): + self.ensure_browser_packages() + + def install_browser_artifact_mode_packages(self, mozconfig_builder): + self.ensure_browser_packages(artifact_mode=True) + + def install_mobile_android_packages(self, mozconfig_builder): + self.ensure_mobile_android_packages(mozconfig_builder, artifact_mode=False) + + def install_mobile_android_artifact_mode_packages(self, mozconfig_builder): + self.ensure_mobile_android_packages(mozconfig_builder, artifact_mode=True) + + def ensure_system_packages(self): + self.run_as_root( + [ + "emerge", + "--noreplace", + "--quiet", + "app-arch/zip", + ] + ) + + def ensure_browser_packages(self, artifact_mode=False): + # TODO: Figure out what not to install for artifact mode + self.run_as_root( + [ + "emerge", + "--oneshot", + "--noreplace", + "--quiet", + "--newuse", + "dev-lang/yasm", + "dev-libs/dbus-glib", + "media-sound/pulseaudio", + "x11-libs/gtk+:2", + "x11-libs/gtk+:3", + "x11-libs/libXt", + ] + ) + + def ensure_mobile_android_packages(self, mozconfig_builder, artifact_mode=False): + self.run_as_root(["emerge", "--noreplace", "--quiet", "dev-java/openjdk-bin"]) + + self.ensure_java(mozconfig_builder) + super().ensure_mobile_android_packages(artifact_mode=artifact_mode) + + def _update_package_manager(self): + self.run_as_root(["emerge", "--sync"]) + + def upgrade_mercurial(self, current): + self.run_as_root(["emerge", "--update", "mercurial"]) diff --git a/python/mozboot/mozboot/linux_common.py b/python/mozboot/mozboot/linux_common.py new file mode 100644 index 0000000000..546c7fc506 --- /dev/null +++ b/python/mozboot/mozboot/linux_common.py @@ -0,0 +1,198 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# An easy way for distribution-specific bootstrappers to share the code +# needed to install Stylo and Node dependencies. This class must come before +# BaseBootstrapper in the inheritance list. + +from __future__ import absolute_import, print_function, unicode_literals + +import os + + +def is_non_x86_64(): + return os.uname()[4] != "x86_64" + + +class SccacheInstall(object): + def __init__(self, **kwargs): + pass + + def ensure_sccache_packages(self, state_dir, checkout_root): + from mozboot import sccache + + self.install_toolchain_artifact(state_dir, checkout_root, sccache.LINUX_SCCACHE) + + +class FixStacksInstall(object): + def __init__(self, **kwargs): + pass + + def ensure_fix_stacks_packages(self, state_dir, checkout_root): + from mozboot import fix_stacks + + self.install_toolchain_artifact( + state_dir, checkout_root, fix_stacks.LINUX_FIX_STACKS + ) + + +class LucetcInstall(object): + def __init__(self, **kwargs): + pass + + def ensure_lucetc_packages(self, state_dir, checkout_root): + from mozboot import lucetc + + self.install_toolchain_artifact(state_dir, checkout_root, lucetc.LINUX_LUCETC) + + +class WasiSysrootInstall(object): + def __init__(self, **kwargs): + pass + + def ensure_wasi_sysroot_packages(self, state_dir, checkout_root): + from mozboot import wasi_sysroot + + self.install_toolchain_artifact( + state_dir, checkout_root, wasi_sysroot.LINUX_WASI_SYSROOT + ) + + +class StyloInstall(object): + def __init__(self, **kwargs): + pass + + def ensure_stylo_packages(self, state_dir, checkout_root): + from mozboot import stylo + + if is_non_x86_64(): + print( + "Cannot install bindgen clang and cbindgen packages from taskcluster.\n" + "Please install these packages manually." + ) + return + + self.install_toolchain_artifact(state_dir, checkout_root, stylo.LINUX_CLANG) + self.install_toolchain_artifact(state_dir, checkout_root, stylo.LINUX_CBINDGEN) + + +class NasmInstall(object): + def __init__(self, **kwargs): + pass + + def ensure_nasm_packages(self, state_dir, checkout_root): + if is_non_x86_64(): + print( + "Cannot install nasm from taskcluster.\n" + "Please install this package manually." + ) + return + + from mozboot import nasm + + self.install_toolchain_artifact(state_dir, checkout_root, nasm.LINUX_NASM) + + +class NodeInstall(object): + def __init__(self, **kwargs): + pass + + def ensure_node_packages(self, state_dir, checkout_root): + if is_non_x86_64(): + print( + "Cannot install node package from taskcluster.\n" + "Please install this package manually." + ) + return + + from mozboot import node + + self.install_toolchain_artifact(state_dir, checkout_root, node.LINUX) + + +class ClangStaticAnalysisInstall(object): + def __init__(self, **kwargs): + pass + + def ensure_clang_static_analysis_package(self, state_dir, checkout_root): + if is_non_x86_64(): + print( + "Cannot install static analysis tools from taskcluster.\n" + "Please install these tools manually." + ) + return + + from mozboot import static_analysis + + self.install_toolchain_static_analysis( + state_dir, checkout_root, static_analysis.LINUX_CLANG_TIDY + ) + + +class MinidumpStackwalkInstall(object): + def __init__(self, **kwargs): + pass + + def ensure_minidump_stackwalk_packages(self, state_dir, checkout_root): + from mozboot import minidump_stackwalk + + self.install_toolchain_artifact( + state_dir, checkout_root, minidump_stackwalk.LINUX_MINIDUMP_STACKWALK + ) + + +class DumpSymsInstall(object): + def __init__(self, **kwargs): + pass + + def ensure_dump_syms_packages(self, state_dir, checkout_root): + from mozboot import dump_syms + + self.install_toolchain_artifact( + state_dir, checkout_root, dump_syms.LINUX_DUMP_SYMS + ) + + +class MobileAndroidBootstrapper(object): + def __init__(self, **kwargs): + pass + + def ensure_mobile_android_packages(self, artifact_mode=False): + from mozboot import android + + android.ensure_android( + "linux", artifact_mode=artifact_mode, no_interactive=self.no_interactive + ) + + def generate_mobile_android_mozconfig(self, artifact_mode=False): + from mozboot import android + + return android.generate_mozconfig("linux", artifact_mode=artifact_mode) + + def generate_mobile_android_artifact_mode_mozconfig(self): + return self.generate_mobile_android_mozconfig(artifact_mode=True) + + +class LinuxBootstrapper( + ClangStaticAnalysisInstall, + FixStacksInstall, + DumpSymsInstall, + LucetcInstall, + MinidumpStackwalkInstall, + MobileAndroidBootstrapper, + NasmInstall, + NodeInstall, + SccacheInstall, + StyloInstall, + WasiSysrootInstall, +): + + INSTALL_PYTHON_GUIDANCE = ( + "See https://firefox-source-docs.mozilla.org/setup/linux_build.html" + "#installingpython for guidance on how to install Python on your " + "system." + ) + + def __init__(self, **kwargs): + pass diff --git a/python/mozboot/mozboot/lucetc.py b/python/mozboot/mozboot/lucetc.py new file mode 100644 index 0000000000..823d536116 --- /dev/null +++ b/python/mozboot/mozboot/lucetc.py @@ -0,0 +1,7 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +LINUX_LUCETC = "linux64-lucetc" diff --git a/python/mozboot/mozboot/mach_commands.py b/python/mozboot/mozboot/mach_commands.py new file mode 100644 index 0000000000..583194a36d --- /dev/null +++ b/python/mozboot/mozboot/mach_commands.py @@ -0,0 +1,124 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this, +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import errno +import sys + +from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, +) +from mozbuild.base import MachCommandBase +from mozboot.bootstrap import APPLICATIONS + + +@CommandProvider +class Bootstrap(MachCommandBase): + """Bootstrap system and mach for optimal development experience.""" + + @Command( + "bootstrap", + category="devenv", + description="Install required system packages for building.", + ) + @CommandArgument( + "--application-choice", + choices=list(APPLICATIONS.keys()) + list(APPLICATIONS.values()), + default=None, + help="Pass in an application choice instead of using the default " + "interactive prompt.", + ) + @CommandArgument( + "--no-interactive", + dest="no_interactive", + action="store_true", + help="Answer yes to any (Y/n) interactive prompts.", + ) + @CommandArgument( + "--no-system-changes", + dest="no_system_changes", + action="store_true", + help="Only execute actions that leave the system " "configuration alone.", + ) + def bootstrap( + self, application_choice=None, no_interactive=False, no_system_changes=False + ): + from mozboot.bootstrap import Bootstrapper + + bootstrapper = Bootstrapper( + choice=application_choice, + no_interactive=no_interactive, + no_system_changes=no_system_changes, + mach_context=self._mach_context, + ) + bootstrapper.bootstrap() + + +@CommandProvider +class VersionControlCommands(MachCommandBase): + @Command( + "vcs-setup", + category="devenv", + description="Help configure a VCS for optimal development.", + ) + @CommandArgument( + "-u", + "--update-only", + action="store_true", + help="Only update recommended extensions, don't run the wizard.", + ) + def vcs_setup(self, update_only=False): + """Ensure a Version Control System (Mercurial or Git) is optimally + configured. + + This command will inspect your VCS configuration and + guide you through an interactive wizard helping you configure the + VCS for optimal use on Mozilla projects. + + User choice is respected: no changes are made without explicit + confirmation from you. + + If "--update-only" is used, the interactive wizard is disabled + and this command only ensures that remote repositories providing + VCS extensions are up to date. + """ + import mozboot.bootstrap as bootstrap + import mozversioncontrol + from mozfile import which + + repo = mozversioncontrol.get_repository_object(self._mach_context.topdir) + tool = "hg" + if repo.name == "git": + tool = "git" + + # "hg" is an executable script with a shebang, which will be found by + # which. We need to pass a win32 executable to the function because we + # spawn a process from it. + if sys.platform in ("win32", "msys"): + tool += ".exe" + + vcs = which(tool) + if not vcs: + raise OSError(errno.ENOENT, "Could not find {} on $PATH".format(tool)) + + if update_only: + if repo.name == "git": + bootstrap.update_git_tools( + vcs, self._mach_context.state_dir, self._mach_context.topdir + ) + else: + bootstrap.update_vct(vcs, self._mach_context.state_dir) + else: + if repo.name == "git": + bootstrap.configure_git( + vcs, + which("git-cinnabar"), + self._mach_context.state_dir, + self._mach_context.topdir, + ) + else: + bootstrap.configure_mercurial(vcs, self._mach_context.state_dir) diff --git a/python/mozboot/mozboot/minidump_stackwalk.py b/python/mozboot/mozboot/minidump_stackwalk.py new file mode 100644 index 0000000000..ad8fc0aaa6 --- /dev/null +++ b/python/mozboot/mozboot/minidump_stackwalk.py @@ -0,0 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +LINUX_MINIDUMP_STACKWALK = "linux64-minidump-stackwalk" +MACOS_MINIDUMP_STACKWALK = "macosx64-minidump-stackwalk" +WINDOWS_MINIDUMP_STACKWALK = "win32-minidump-stackwalk" diff --git a/python/mozboot/mozboot/mozconfig.py b/python/mozboot/mozboot/mozconfig.py new file mode 100644 index 0000000000..81f8c53791 --- /dev/null +++ b/python/mozboot/mozboot/mozconfig.py @@ -0,0 +1,150 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import filecmp +import os + + +MOZ_MYCONFIG_ERROR = """ +The MOZ_MYCONFIG environment variable to define the location of mozconfigs +is deprecated. If you wish to define the mozconfig path via an environment +variable, use MOZCONFIG instead. +""".strip() + +MOZCONFIG_LEGACY_PATH_ERROR = """ +You currently have a mozconfig at %s. This implicit location is no longer +supported. Please move it to %s/.mozconfig or set an explicit path +via the $MOZCONFIG environment variable. +""".strip() + +DEFAULT_TOPSRCDIR_PATHS = (".mozconfig", "mozconfig") +DEPRECATED_TOPSRCDIR_PATHS = ("mozconfig.sh", "myconfig.sh") +DEPRECATED_HOME_PATHS = (".mozconfig", ".mozconfig.sh", ".mozmyconfig.sh") + + +class MozconfigFindException(Exception): + """Raised when a mozconfig location is not defined properly.""" + + +class MozconfigBuilder(object): + def __init__(self): + self._lines = [] + + def append(self, block): + self._lines.extend([line.strip() for line in block.split("\n") if line.strip()]) + + def generate(self): + return "\n".join(self._lines) + + +def find_mozconfig(topsrcdir, env=os.environ): + """Find the active mozconfig file for the current environment. + + This emulates the logic in mozconfig-find. + + 1) If ENV[MOZCONFIG] is set, use that + 2) If $TOPSRCDIR/mozconfig or $TOPSRCDIR/.mozconfig exists, use it. + 3) If both exist or if there are legacy locations detected, error out. + + The absolute path to the found mozconfig will be returned on success. + None will be returned if no mozconfig could be found. A + MozconfigFindException will be raised if there is a bad state, + including conditions from #3 above. + """ + # Check for legacy methods first. + if "MOZ_MYCONFIG" in env: + raise MozconfigFindException(MOZ_MYCONFIG_ERROR) + + env_path = env.get("MOZCONFIG", None) or None + if env_path is not None: + if not os.path.isabs(env_path): + potential_roots = [topsrcdir, os.getcwd()] + # Attempt to eliminate duplicates for e.g. + # self.topsrcdir == os.curdir. + potential_roots = set(os.path.abspath(p) for p in potential_roots) + existing = [ + root + for root in potential_roots + if os.path.exists(os.path.join(root, env_path)) + ] + if len(existing) > 1: + # There are multiple files, but we might have a setup like: + # + # somedirectory/ + # srcdir/ + # objdir/ + # + # MOZCONFIG=../srcdir/some/path/to/mozconfig + # + # and be configuring from the objdir. So even though we + # have multiple existing files, they are actually the same + # file. + mozconfigs = [os.path.join(root, env_path) for root in existing] + if not all( + map( + lambda p1, p2: filecmp.cmp(p1, p2, shallow=False), + mozconfigs[:-1], + mozconfigs[1:], + ) + ): + raise MozconfigFindException( + "MOZCONFIG environment variable refers to a path that " + + "exists in more than one of " + + ", ".join(potential_roots) + + ". Remove all but one." + ) + elif not existing: + raise MozconfigFindException( + "MOZCONFIG environment variable refers to a path that " + + "does not exist in any of " + + ", ".join(potential_roots) + ) + + env_path = os.path.join(existing[0], env_path) + elif not os.path.exists(env_path): # non-relative path + raise MozconfigFindException( + "MOZCONFIG environment variable refers to a path that " + "does not exist: " + env_path + ) + + if not os.path.isfile(env_path): + raise MozconfigFindException( + "MOZCONFIG environment variable refers to a " "non-file: " + env_path + ) + + srcdir_paths = [os.path.join(topsrcdir, p) for p in DEFAULT_TOPSRCDIR_PATHS] + existing = [p for p in srcdir_paths if os.path.isfile(p)] + + if env_path is None and len(existing) > 1: + raise MozconfigFindException( + "Multiple default mozconfig files " + "present. Remove all but one. " + ", ".join(existing) + ) + + path = None + + if env_path is not None: + path = env_path + elif len(existing): + assert len(existing) == 1 + path = existing[0] + + if path is not None: + return os.path.abspath(path) + + deprecated_paths = [os.path.join(topsrcdir, s) for s in DEPRECATED_TOPSRCDIR_PATHS] + + home = env.get("HOME", None) + if home is not None: + deprecated_paths.extend([os.path.join(home, s) for s in DEPRECATED_HOME_PATHS]) + + for path in deprecated_paths: + if os.path.exists(path): + raise MozconfigFindException( + MOZCONFIG_LEGACY_PATH_ERROR % (path, topsrcdir) + ) + + return None diff --git a/python/mozboot/mozboot/mozillabuild.py b/python/mozboot/mozboot/mozillabuild.py new file mode 100644 index 0000000000..fdd023da54 --- /dev/null +++ b/python/mozboot/mozboot/mozillabuild.py @@ -0,0 +1,253 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import ctypes +import os +import sys +import subprocess + +from mozboot.base import BaseBootstrapper + + +def is_aarch64_host(): + from ctypes import wintypes + + kernel32 = ctypes.windll.kernel32 + IMAGE_FILE_MACHINE_UNKNOWN = 0 + IMAGE_FILE_MACHINE_ARM64 = 0xAA64 + + try: + iswow64process2 = kernel32.IsWow64Process2 + except Exception: + # If we can't access the symbol, we know we're not on aarch64. + return False + + currentProcess = kernel32.GetCurrentProcess() + processMachine = wintypes.USHORT(IMAGE_FILE_MACHINE_UNKNOWN) + nativeMachine = wintypes.USHORT(IMAGE_FILE_MACHINE_UNKNOWN) + + gotValue = iswow64process2( + currentProcess, ctypes.byref(processMachine), ctypes.byref(nativeMachine) + ) + # If this call fails, we have no idea. + if not gotValue: + return False + + return nativeMachine.value == IMAGE_FILE_MACHINE_ARM64 + + +def get_is_windefender_disabled(): + import winreg + + try: + with winreg.OpenKeyEx( + winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Microsoft\Windows Defender" + ) as windefender_key: + is_antivirus_disabled, _ = winreg.QueryValueEx( + windefender_key, "DisableAntiSpyware" + ) + # is_antivirus_disabled is either 0 (False) or 1 (True) + return bool(is_antivirus_disabled) + except (FileNotFoundError, OSError): + return True + + +def get_windefender_exclusion_paths(): + import winreg + + paths = [] + try: + with winreg.OpenKeyEx( + winreg.HKEY_LOCAL_MACHINE, + r"SOFTWARE\Microsoft\Windows Defender\Exclusions\Paths", + ) as exclusions_key: + _, values_count, __ = winreg.QueryInfoKey(exclusions_key) + for i in range(0, values_count): + path, _, __ = winreg.EnumValue(exclusions_key, i) + paths.append(path) + except (FileNotFoundError, OSError): + pass + + return paths + + +def is_windefender_affecting_srcdir(srcdir): + if get_is_windefender_disabled(): + return False + + # When there's a match, but path cases aren't the same between srcdir and exclusion_path, + # commonpath will use the casing of the first path provided. + # To avoid surprises here, we normcase(...) so we don't get unexpected breakage if we change + # the path order. + srcdir = os.path.normcase(os.path.abspath(srcdir)) + for exclusion_path in get_windefender_exclusion_paths(): + exclusion_path = os.path.normcase(os.path.abspath(exclusion_path)) + try: + if os.path.commonpath([exclusion_path, srcdir]) == exclusion_path: + # exclusion_path is an ancestor of srcdir + return False + except ValueError: + # ValueError: Paths don't have the same drive - can't be ours + pass + return True + + +class MozillaBuildBootstrapper(BaseBootstrapper): + """Bootstrapper for MozillaBuild to install rustup.""" + + INSTALL_PYTHON_GUIDANCE = ( + "Python is provided by MozillaBuild; ensure your MozillaBuild " + "installation is up to date." + ) + + def __init__(self, no_interactive=False, no_system_changes=False): + BaseBootstrapper.__init__( + self, no_interactive=no_interactive, no_system_changes=no_system_changes + ) + + def validate_environment(self, srcdir): + if self.application.startswith("mobile_android"): + print( + "WARNING!!! Building Firefox for Android on Windows is not " + "fully supported. See https://bugzilla.mozilla.org/show_bug." + "cgi?id=1169873 for details.", + file=sys.stderr, + ) + + if is_windefender_affecting_srcdir(srcdir): + print( + "Warning: the Firefox checkout directory is currently not in the " + "Windows Defender exclusion list. This can cause the build process " + "to be dramatically slowed or broken. To resolve this, follow the " + "directions here: " + "https://firefox-source-docs.mozilla.org/setup/windows_build.html" + "#antivirus-performance", + file=sys.stderr, + ) + + def install_system_packages(self): + pass + + def upgrade_mercurial(self, current): + # Mercurial upstream sometimes doesn't upload wheels, and building + # from source requires MS Visual C++ 9.0. So we force pip to install + # the last version that comes with wheels. + self.pip_install("mercurial", "--only-binary", "mercurial") + + def install_browser_packages(self, mozconfig_builder): + pass + + def install_browser_artifact_mode_packages(self, mozconfig_builder): + pass + + def install_mobile_android_packages(self, mozconfig_builder): + self.ensure_mobile_android_packages(mozconfig_builder) + + def install_mobile_android_artifact_mode_packages(self, mozconfig_builder): + self.ensure_mobile_android_packages(mozconfig_builder, artifact_mode=True) + + def ensure_mobile_android_packages(self, mozconfig_builder, artifact_mode=False): + java_bin_dir = self.ensure_java(mozconfig_builder) + from mach.util import setenv + + setenv("PATH", "{}{}{}".format(java_bin_dir, os.pathsep, os.environ["PATH"])) + + from mozboot import android + + android.ensure_android( + "windows", artifact_mode=artifact_mode, no_interactive=self.no_interactive + ) + + def generate_mobile_android_mozconfig(self, artifact_mode=False): + from mozboot import android + + return android.generate_mozconfig("windows", artifact_mode=artifact_mode) + + def generate_mobile_android_artifact_mode_mozconfig(self): + return self.generate_mobile_android_mozconfig(artifact_mode=True) + + def ensure_clang_static_analysis_package(self, state_dir, checkout_root): + from mozboot import static_analysis + + self.install_toolchain_static_analysis( + state_dir, checkout_root, static_analysis.WINDOWS_CLANG_TIDY + ) + + def ensure_sccache_packages(self, state_dir, checkout_root): + from mozboot import sccache + + self.install_toolchain_artifact(state_dir, checkout_root, sccache.WIN64_SCCACHE) + self.install_toolchain_artifact( + state_dir, checkout_root, sccache.RUSTC_DIST_TOOLCHAIN, no_unpack=True + ) + self.install_toolchain_artifact( + state_dir, checkout_root, sccache.CLANG_DIST_TOOLCHAIN, no_unpack=True + ) + + def ensure_stylo_packages(self, state_dir, checkout_root): + # On-device artifact builds are supported; on-device desktop builds are not. + if is_aarch64_host(): + raise Exception( + "You should not be performing desktop builds on an " + "AArch64 device. If you want to do artifact builds " + "instead, please choose the appropriate artifact build " + "option when beginning bootstrap." + ) + + from mozboot import stylo + + self.install_toolchain_artifact(state_dir, checkout_root, stylo.WINDOWS_CLANG) + self.install_toolchain_artifact( + state_dir, checkout_root, stylo.WINDOWS_CBINDGEN + ) + + def ensure_nasm_packages(self, state_dir, checkout_root): + from mozboot import nasm + + self.install_toolchain_artifact(state_dir, checkout_root, nasm.WINDOWS_NASM) + + def ensure_node_packages(self, state_dir, checkout_root): + from mozboot import node + + # We don't have native aarch64 node available, but aarch64 windows + # runs x86 binaries, so just use the x86 packages for such hosts. + node_artifact = node.WIN32 if is_aarch64_host() else node.WIN64 + self.install_toolchain_artifact(state_dir, checkout_root, node_artifact) + + def ensure_dump_syms_packages(self, state_dir, checkout_root): + from mozboot import dump_syms + + self.install_toolchain_artifact( + state_dir, checkout_root, dump_syms.WIN64_DUMP_SYMS + ) + + def ensure_fix_stacks_packages(self, state_dir, checkout_root): + from mozboot import fix_stacks + + self.install_toolchain_artifact( + state_dir, checkout_root, fix_stacks.WINDOWS_FIX_STACKS + ) + + def ensure_minidump_stackwalk_packages(self, state_dir, checkout_root): + from mozboot import minidump_stackwalk + + self.install_toolchain_artifact( + state_dir, checkout_root, minidump_stackwalk.WINDOWS_MINIDUMP_STACKWALK + ) + + def _update_package_manager(self): + pass + + def run(self, command): + subprocess.check_call(command, stdin=sys.stdin) + + def pip_install(self, *packages): + pip_dir = os.path.join( + os.environ["MOZILLABUILD"], "python", "Scripts", "pip.exe" + ) + command = [pip_dir, "install", "--upgrade"] + command.extend(packages) + self.run(command) diff --git a/python/mozboot/mozboot/nasm.py b/python/mozboot/mozboot/nasm.py new file mode 100644 index 0000000000..476e1bfcc8 --- /dev/null +++ b/python/mozboot/mozboot/nasm.py @@ -0,0 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +WINDOWS_NASM = "win64-nasm" +LINUX_NASM = "linux64-nasm" +MACOS_NASM = "macosx64-nasm" diff --git a/python/mozboot/mozboot/node.py b/python/mozboot/mozboot/node.py new file mode 100644 index 0000000000..992dd7c493 --- /dev/null +++ b/python/mozboot/mozboot/node.py @@ -0,0 +1,10 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +WIN32 = "win32-node" +WIN64 = "win64-node" +LINUX = "linux64-node" +OSX = "macosx64-node" diff --git a/python/mozboot/mozboot/openbsd.py b/python/mozboot/mozboot/openbsd.py new file mode 100644 index 0000000000..b7143cf542 --- /dev/null +++ b/python/mozboot/mozboot/openbsd.py @@ -0,0 +1,61 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from mozboot.base import BaseBootstrapper + + +class OpenBSDBootstrapper(BaseBootstrapper): + def __init__(self, version, **kwargs): + BaseBootstrapper.__init__(self, **kwargs) + + self.packages = [ + "gmake", + "gtar", + "rust", + "wget", + "unzip", + "zip", + ] + + self.browser_packages = [ + "llvm", + "nasm", + "yasm", + "gtk+2", + "gtk+3", + "dbus-glib", + "pulseaudio", + ] + + def install_system_packages(self): + # we use -z because there's no other way to say "any autoconf-2.13" + self.run_as_root(["pkg_add", "-z"] + self.packages) + + def install_browser_packages(self, mozconfig_builder): + self.ensure_browser_packages() + + def install_browser_artifact_mode_packages(self, mozconfig_builder): + self.ensure_browser_packages(artifact_mode=True) + + def ensure_browser_packages(self, artifact_mode=False): + # TODO: Figure out what not to install for artifact mode + # we use -z because there's no other way to say "any autoconf-2.13" + self.run_as_root(["pkg_add", "-z"] + self.browser_packages) + + def ensure_clang_static_analysis_package(self, state_dir, checkout_root): + # TODO: we don't ship clang base static analysis for this platform + pass + + def ensure_stylo_packages(self, state_dir, checkout_root): + # Clang / llvm already installed as browser package + self.run_as_root(["pkg_add", "cbindgen"]) + + def ensure_nasm_packages(self, state_dir, checkout_root): + # installed via ensure_browser_packages + pass + + def ensure_node_packages(self, state_dir, checkout_root): + self.run_as_root(["pkg_add", "node"]) diff --git a/python/mozboot/mozboot/opensuse.py b/python/mozboot/mozboot/opensuse.py new file mode 100644 index 0000000000..899bcbe42b --- /dev/null +++ b/python/mozboot/mozboot/opensuse.py @@ -0,0 +1,147 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from mozboot.base import BaseBootstrapper +from mozboot.linux_common import LinuxBootstrapper + + +class OpenSUSEBootstrapper(LinuxBootstrapper, BaseBootstrapper): + """openSUSE experimental bootstrapper.""" + + SYSTEM_PACKAGES = [ + "nodejs", + "npm", + "which", + "rpmconf", + "libcurl-devel", + "libpulse-devel", + ] + + BROWSER_PACKAGES = [ + "alsa-devel", + "gcc-c++", + "gtk3-devel", + "dbus-1-glib-devel", + "gconf2-devel", + "glibc-devel-static", + "libstdc++-devel", + "libXt-devel", + "libproxy-devel", + "libuuid-devel", + "yasm", + "gtk2-devel", + "clang-devel", + "patterns-gnome-devel_gnome", + ] + + BROWSER_GROUP_PACKAGES = [ + "devel_C_C++", + "devel_gnome", + ] + + MOBILE_ANDROID_COMMON_PACKAGES = [ + "java-1_8_0-openjdk", + "wget", + ] + + def __init__(self, version, dist_id, **kwargs): + print("Using an experimental bootstrapper for openSUSE.") + BaseBootstrapper.__init__(self, **kwargs) + + def install_system_packages(self): + self.zypper_install(*self.SYSTEM_PACKAGES) + + def install_browser_packages(self, mozconfig_builder): + self.ensure_browser_packages() + + def install_browser_group_packages(self): + self.ensure_browser_group_packages() + + def install_browser_artifact_mode_packages(self, mozconfig_builder): + self.ensure_browser_packages(artifact_mode=True) + + def install_mobile_android_packages(self, mozconfig_builder): + self.ensure_mobile_android_packages() + + def install_mobile_android_artifact_mode_packages(self, mozconfig_builder): + self.ensure_mobile_android_packages(artifact_mode=True) + + def install_mercurial(self): + self.run_as_root(["pip", "install", "--upgrade", "pip"]) + self.run_as_root(["pip", "install", "--upgrade", "Mercurial"]) + + def ensure_clang_static_analysis_package(self, state_dir, checkout_root): + from mozboot import static_analysis + + self.install_toolchain_static_analysis( + state_dir, checkout_root, static_analysis.LINUX_CLANG_TIDY + ) + + def ensure_browser_packages(self, artifact_mode=False): + # TODO: Figure out what not to install for artifact mode + self.zypper_install(*self.BROWSER_PACKAGES) + + def ensure_browser_group_packages(self, artifact_mode=False): + # TODO: Figure out what not to install for artifact mode + self.zypper_patterninstall(*self.BROWSER_GROUP_PACKAGES) + + def ensure_mobile_android_packages(self, artifact_mode=False): + # Multi-part process: + # 1. System packages. + # 2. Android SDK. Android NDK only if we are not in artifact mode. Android packages. + + # 1. This is hard to believe, but the Android SDK binaries are 32-bit + # and that conflicts with 64-bit Arch installations out of the box. The + # solution is to add the multilibs repository; unfortunately, this + # requires manual intervention. + try: + self.zypper_install(*self.MOBILE_ANDROID_COMMON_PACKAGES) + except Exception as e: + print( + "Failed to install all packages. The Android developer " + "toolchain requires 32 bit binaries be enabled" + ) + raise e + + # 2. Android pieces. + super().ensure_mobile_android_packages(artifact_mode=artifact_mode) + + def _update_package_manager(self): + self.zypper_update + + def upgrade_mercurial(self, current): + self.run_as_root(["pip3", "install", "--upgrade", "pip"]) + self.run_as_root(["pip3", "install", "--upgrade", "Mercurial"]) + + def ensure_nasm_packages(self, state_dir, checkout_root): + self.zypper_install("nasm") + + def zypper_install(self, *packages): + command = ["zypper", "install"] + if self.no_interactive: + command.append("-n") + + command.extend(packages) + + self.run_as_root(command) + + def zypper_update(self, *packages): + command = ["zypper", "update"] + if self.no_interactive: + command.append("-n") + + command.extend(packages) + + self.run_as_root(command) + + def zypper_patterninstall(self, *packages): + command = ["zypper", "install", "-t", "pattern"] + if self.no_interactive: + command.append("-y") + + command.extend(packages) + + self.run_as_root(command) diff --git a/python/mozboot/mozboot/osx.py b/python/mozboot/mozboot/osx.py new file mode 100644 index 0000000000..de4412e99a --- /dev/null +++ b/python/mozboot/mozboot/osx.py @@ -0,0 +1,662 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import platform +import re +import subprocess +import sys +import tempfile + +try: + from urllib2 import urlopen +except ImportError: + from urllib.request import urlopen + +from distutils.version import StrictVersion + +from mozboot.base import BaseBootstrapper +from mozfile import which + +HOMEBREW_BOOTSTRAP = "https://raw.githubusercontent.com/Homebrew/install/master/install" +XCODE_APP_STORE = "macappstore://itunes.apple.com/app/id497799835?mt=12" +XCODE_LEGACY = ( + "https://developer.apple.com/downloads/download.action?path=Developer_Tools/" + "xcode_3.2.6_and_ios_sdk_4.3__final/xcode_3.2.6_and_ios_sdk_4.3.dmg" +) + +MACPORTS_URL = { + "14": "https://distfiles.macports.org/MacPorts/MacPorts-2.5.4-10.14-Mojave.pkg", + "13": "https://distfiles.macports.org/MacPorts/MacPorts-2.5.4-10.13-HighSierra.pkg", + "12": "https://distfiles.macports.org/MacPorts/MacPorts-2.5.4-10.12-Sierra.pkg", + "11": "https://distfiles.macports.org/MacPorts/MacPorts-2.5.4-10.11-ElCapitan.pkg", + "10": "https://distfiles.macports.org/MacPorts/MacPorts-2.5.4-10.10-Yosemite.pkg", + "9": "https://distfiles.macports.org/MacPorts/MacPorts-2.5.4-10.9-Mavericks.pkg", + "8": "https://distfiles.macports.org/MacPorts/MacPorts-2.5.4-10.8-MountainLion.pkg", + "7": "https://distfiles.macports.org/MacPorts/MacPorts-2.5.4-10.7-Lion.pkg", + "6": "https://distfiles.macports.org/MacPorts/MacPorts-2.5.4-10.6-SnowLeopard.pkg", +} + +RE_CLANG_VERSION = re.compile("Apple (?:clang|LLVM) version (\d+\.\d+)") + +APPLE_CLANG_MINIMUM_VERSION = StrictVersion("4.2") + +XCODE_REQUIRED = """ +Xcode is required to build Firefox. Please complete the install of Xcode +through the App Store. + +It's possible Xcode is already installed on this machine but it isn't being +detected. This is possible with developer preview releases of Xcode, for +example. To correct this problem, run: + + `xcode-select --switch /path/to/Xcode.app`. + +e.g. `sudo xcode-select --switch /Applications/Xcode.app`. +""" + +XCODE_REQUIRED_LEGACY = """ +You will need to download and install Xcode to build Firefox. + +Please complete the Xcode download and then relaunch this script. +""" + +XCODE_NO_DEVELOPER_DIRECTORY = """ +xcode-select says you don't have a developer directory configured. We think +this is due to you not having Xcode installed (properly). We're going to +attempt to install Xcode through the App Store. If the App Store thinks you +have Xcode installed, please run xcode-select by hand until it stops +complaining and then re-run this script. +""" + +XCODE_COMMAND_LINE_TOOLS_MISSING = """ +The Xcode command line tools are required to build Firefox. +""" + +INSTALL_XCODE_COMMAND_LINE_TOOLS_STEPS = """ +Perform the following steps to install the Xcode command line tools: + + 1) Open Xcode.app + 2) Click through any first-run prompts + 3) From the main Xcode menu, select Preferences (Command ,) + 4) Go to the Download tab (near the right) + 5) Install the "Command Line Tools" + +When that has finished installing, please relaunch this script. +""" + +UPGRADE_XCODE_COMMAND_LINE_TOOLS = """ +An old version of the Xcode command line tools is installed. You will need to +install a newer version in order to compile Firefox. If Xcode itself is old, +its command line tools may be too old even if it claims there are no updates +available, so if you are seeing this message multiple times, please update +Xcode first. +""" + +PACKAGE_MANAGER_INSTALL = """ +We will install the %s package manager to install required packages. + +You will be prompted to install %s with its default settings. If you +would prefer to do this manually, hit CTRL+c, install %s yourself, ensure +"%s" is in your $PATH, and relaunch bootstrap. +""" + +PACKAGE_MANAGER_PACKAGES = """ +We are now installing all required packages via %s. You will see a lot of +output as packages are built. +""" + +PACKAGE_MANAGER_OLD_CLANG = """ +We require a newer compiler than what is provided by your version of Xcode. + +We will install a modern version of Clang through %s. +""" + +PACKAGE_MANAGER_CHOICE = """ +Please choose a package manager you'd like: + 1. Homebrew + 2. MacPorts (Does not yet support bootstrapping GeckoView/Firefox for Android.) +Your choice: """ + +NO_PACKAGE_MANAGER_WARNING = """ +It seems you don't have any supported package manager installed. +""" + +PACKAGE_MANAGER_EXISTS = """ +Looks like you have %s installed. We will install all required packages via %s. +""" + +MULTI_PACKAGE_MANAGER_EXISTS = """ +It looks like you have multiple package managers installed. +""" + +# May add support for other package manager on os x. +PACKAGE_MANAGER = {"Homebrew": "brew", "MacPorts": "port"} + +PACKAGE_MANAGER_CHOICES = ["Homebrew", "MacPorts"] + +PACKAGE_MANAGER_BIN_MISSING = """ +A package manager is installed. However, your current shell does +not know where to find '%s' yet. You'll need to start a new shell +to pick up the environment changes so it can be found. + +Please start a new shell or terminal window and run this +bootstrapper again. + +If this problem persists, you will likely want to adjust your +shell's init script (e.g. ~/.bash_profile) to export a PATH +environment variable containing the location of your package +manager binary. e.g. + +Homebrew: + export PATH=/usr/local/bin:$PATH + +MacPorts: + export PATH=/opt/local/bin:$PATH +""" + +BAD_PATH_ORDER = """ +Your environment's PATH variable lists a system path directory (%s) +before the path to your package manager's binaries (%s). +This means that the package manager's binaries likely won't be +detected properly. + +Modify your shell's configuration (e.g. ~/.profile or +~/.bash_profile) to have %s appear in $PATH before %s. e.g. + + export PATH=%s:$PATH + +Once this is done, start a new shell (likely Command+T) and run +this bootstrap again. +""" + + +class OSXBootstrapper(BaseBootstrapper): + + INSTALL_PYTHON_GUIDANCE = ( + "See https://firefox-source-docs.mozilla.org/setup/macos_build.html" + "#install-via-homebrew for guidance on how to install Python on your " + "system." + ) + + def __init__(self, version, **kwargs): + BaseBootstrapper.__init__(self, **kwargs) + + self.os_version = StrictVersion(version) + + if self.os_version < StrictVersion("10.6"): + raise Exception("OS X 10.6 or above is required.") + + if platform.machine() == "arm64": + print( + "Bootstrap is not supported on Apple Silicon yet.\n" + "Please see instructions at https://bit.ly/36bUmEx in the meanwhile" + ) + sys.exit(1) + + self.minor_version = version.split(".")[1] + + def install_system_packages(self): + self.ensure_xcode() + + choice = self.ensure_package_manager() + self.package_manager = choice + _, hg_modern, _ = self.is_mercurial_modern() + if not hg_modern: + print( + "Mercurial wasn't found or is not sufficiently modern. " + "It will be installed with %s" % self.package_manager + ) + getattr(self, "ensure_%s_system_packages" % self.package_manager)(not hg_modern) + + def install_browser_packages(self, mozconfig_builder): + getattr(self, "ensure_%s_browser_packages" % self.package_manager)() + + def install_browser_artifact_mode_packages(self, mozconfig_builder): + getattr(self, "ensure_%s_browser_packages" % self.package_manager)( + artifact_mode=True + ) + + def install_mobile_android_packages(self, mozconfig_builder): + getattr(self, "ensure_%s_mobile_android_packages" % self.package_manager)( + mozconfig_builder + ) + + def install_mobile_android_artifact_mode_packages(self, mozconfig_builder): + getattr(self, "ensure_%s_mobile_android_packages" % self.package_manager)( + mozconfig_builder, artifact_mode=True + ) + + def generate_mobile_android_mozconfig(self): + return self._generate_mobile_android_mozconfig() + + def generate_mobile_android_artifact_mode_mozconfig(self): + return self._generate_mobile_android_mozconfig(artifact_mode=True) + + def _generate_mobile_android_mozconfig(self, artifact_mode=False): + from mozboot import android + + return android.generate_mozconfig("macosx", artifact_mode=artifact_mode) + + def ensure_xcode(self): + if self.os_version < StrictVersion("10.7"): + if not os.path.exists("/Developer/Applications/Xcode.app"): + print(XCODE_REQUIRED_LEGACY) + + subprocess.check_call(["open", XCODE_LEGACY]) + sys.exit(1) + + # OS X 10.7 have Xcode come from the app store. However, users can + # still install Xcode into any arbitrary location. We honor the + # location of Xcode as set by xcode-select. This should also pick up + # developer preview releases of Xcode, which can be installed into + # paths like /Applications/Xcode5-DP6.app. + elif self.os_version >= StrictVersion("10.7"): + select = which("xcode-select") + try: + output = subprocess.check_output( + [select, "--print-path"], stderr=subprocess.STDOUT + ) + except subprocess.CalledProcessError as e: + # This seems to appear on fresh OS X machines before any Xcode + # has been installed. It may only occur on OS X 10.9 and later. + if b"unable to get active developer directory" in e.output: + print(XCODE_NO_DEVELOPER_DIRECTORY) + self._install_xcode_app_store() + assert False # Above should exit. + + output = e.output + + # This isn't the most robust check in the world. It relies on the + # default value not being in an application bundle, which seems to + # hold on at least Mavericks. + if b".app/" not in output: + print(XCODE_REQUIRED) + self._install_xcode_app_store() + assert False # Above should exit. + + # Once Xcode is installed, you need to agree to the license before you can + # use it. + try: + output = subprocess.check_output( + ["/usr/bin/xcrun", "clang"], stderr=subprocess.STDOUT + ) + except subprocess.CalledProcessError as e: + if b"license" in e.output: + xcodebuild = which("xcodebuild") + try: + subprocess.check_output( + [xcodebuild, "-license"], stderr=subprocess.STDOUT + ) + except subprocess.CalledProcessError as e: + if b"requires admin privileges" in e.output: + self.run_as_root([xcodebuild, "-license"]) + + # Even then we're not done! We need to install the Xcode command line tools. + # As of Mountain Lion, apparently the only way to do this is to go through a + # menu dialog inside Xcode itself. We're not making this up. + if self.os_version >= StrictVersion("10.7"): + if not os.path.exists("/usr/bin/clang"): + print(XCODE_COMMAND_LINE_TOOLS_MISSING) + print(INSTALL_XCODE_COMMAND_LINE_TOOLS_STEPS) + sys.exit(1) + + output = subprocess.check_output( + ["/usr/bin/clang", "--version"], universal_newlines=True + ) + match = RE_CLANG_VERSION.search(output) + if match is None: + raise Exception("Could not determine Clang version.") + + version = StrictVersion(match.group(1)) + + if version < APPLE_CLANG_MINIMUM_VERSION: + print(UPGRADE_XCODE_COMMAND_LINE_TOOLS) + print(INSTALL_XCODE_COMMAND_LINE_TOOLS_STEPS) + sys.exit(1) + + def _install_xcode_app_store(self): + subprocess.check_call(["open", XCODE_APP_STORE]) + print("Once the install has finished, please relaunch this script.") + sys.exit(1) + + def _ensure_homebrew_found(self): + if not hasattr(self, "brew"): + self.brew = which("brew") + # Earlier code that checks for valid package managers ensures + # which('brew') is found. + assert self.brew is not None + + def _ensure_homebrew_packages(self, packages, is_for_cask=False): + package_type_flag = "--cask" if is_for_cask else "--formula" + self._ensure_homebrew_found() + self._ensure_package_manager_updated() + + def create_homebrew_cmd(*parameters): + base_cmd = [self.brew] + base_cmd.extend(parameters) + return base_cmd + [package_type_flag] + + installed = set( + subprocess.check_output( + create_homebrew_cmd("list"), universal_newlines=True + ).split() + ) + outdated = set( + subprocess.check_output( + create_homebrew_cmd("outdated", "--quiet"), universal_newlines=True + ).split() + ) + + to_install = set(package for package in packages if package not in installed) + to_upgrade = set(package for package in packages if package in outdated) + + if to_install or to_upgrade: + print(PACKAGE_MANAGER_PACKAGES % ("Homebrew",)) + if to_install: + subprocess.check_call(create_homebrew_cmd("install") + list(to_install)) + if to_upgrade: + subprocess.check_call(create_homebrew_cmd("upgrade") + list(to_upgrade)) + + def _ensure_homebrew_casks(self, casks): + self._ensure_homebrew_found() + + known_taps = subprocess.check_output([self.brew, "tap"]) + + # Ensure that we can access old versions of packages. + if b"homebrew/cask-versions" not in known_taps: + subprocess.check_output([self.brew, "tap", "homebrew/cask-versions"]) + + # "caskroom/versions" has been renamed to "homebrew/cask-versions", so + # it is safe to remove the old tap. Removing the old tap is necessary + # to avoid the error "Cask [name of cask] exists in multiple taps". + # See https://bugzilla.mozilla.org/show_bug.cgi?id=1544981 + if b"caskroom/versions" in known_taps: + subprocess.check_output([self.brew, "untap", "caskroom/versions"]) + + self._ensure_homebrew_packages(casks, is_for_cask=True) + + def ensure_homebrew_system_packages(self, install_mercurial): + packages = [ + "git", + "gnu-tar", + "terminal-notifier", + "watchman", + ] + if install_mercurial: + packages.append("mercurial") + self._ensure_homebrew_packages(packages) + + def ensure_homebrew_browser_packages(self, artifact_mode=False): + # TODO: Figure out what not to install for artifact mode + packages = [ + "yasm", + ] + self._ensure_homebrew_packages(packages) + + def ensure_homebrew_mobile_android_packages( + self, mozconfig_builder, artifact_mode=False + ): + # Multi-part process: + # 1. System packages. + # 2. Android SDK. Android NDK only if we are not in artifact mode. Android packages. + + # 1. System packages. + packages = [ + "wget", + ] + self._ensure_homebrew_packages(packages) + + casks = [ + "adoptopenjdk8", + ] + self._ensure_homebrew_casks(casks) + + is_64bits = sys.maxsize > 2 ** 32 + if not is_64bits: + raise Exception( + "You need a 64-bit version of Mac OS X to build " + "GeckoView/Firefox for Android." + ) + + # 2. Android pieces. + java_path = self.ensure_java(mozconfig_builder) + # Prefer our validated java binary by putting it on the path first. + os.environ["PATH"] = "{}{}{}".format(java_path, os.pathsep, os.environ["PATH"]) + from mozboot import android + + android.ensure_android( + "macosx", artifact_mode=artifact_mode, no_interactive=self.no_interactive + ) + + def _ensure_macports_packages(self, packages): + self.port = which("port") + assert self.port is not None + + installed = set( + subprocess.check_output( + [self.port, "installed"], universal_newlines=True + ).split() + ) + + missing = [package for package in packages if package not in installed] + if missing: + print(PACKAGE_MANAGER_PACKAGES % ("MacPorts",)) + self.run_as_root([self.port, "-v", "install"] + missing) + + def ensure_macports_system_packages(self, install_mercurial): + packages = ["gnutar", "watchman"] + if install_mercurial: + packages.append("mercurial") + + self._ensure_macports_packages(packages) + + pythons = set( + subprocess.check_output( + [self.port, "select", "--list", "python"], universal_newlines=True + ).split("\n") + ) + active = "" + for python in pythons: + if "active" in python: + active = python + if "python27" not in active: + self.run_as_root([self.port, "select", "--set", "python", "python27"]) + else: + print("The right python version is already active.") + + def ensure_macports_browser_packages(self, artifact_mode=False): + # TODO: Figure out what not to install for artifact mode + packages = [ + "yasm", + ] + + self._ensure_macports_packages(packages) + + def ensure_macports_mobile_android_packages( + self, mozconfig_builder, artifact_mode=False + ): + # Multi-part process: + # 1. System packages. + # 2. Android SDK. Android NDK only if we are not in artifact mode. Android packages. + + # 1. System packages. + packages = [ + "wget", + ] + self._ensure_macports_packages(packages) + + is_64bits = sys.maxsize > 2 ** 32 + if not is_64bits: + raise Exception( + "You need a 64-bit version of Mac OS X to build " + "GeckoView/Firefox for Android." + ) + + # 2. Android pieces. + self.ensure_java(mozconfig_builder) + from mozboot import android + + android.ensure_android( + "macosx", artifact_mode=artifact_mode, no_interactive=self.no_interactive + ) + + def ensure_package_manager(self): + """ + Search package mgr in sys.path, if none is found, prompt the user to install one. + If only one is found, use that one. If both are found, prompt the user to choose + one. + """ + installed = [] + for name, cmd in PACKAGE_MANAGER.items(): + if which(cmd) is not None: + installed.append(name) + + active_name, active_cmd = None, None + + if not installed: + print(NO_PACKAGE_MANAGER_WARNING) + choice = self.prompt_int(prompt=PACKAGE_MANAGER_CHOICE, low=1, high=2) + active_name = PACKAGE_MANAGER_CHOICES[choice - 1] + active_cmd = PACKAGE_MANAGER[active_name] + getattr(self, "install_%s" % active_name.lower())() + elif len(installed) == 1: + print(PACKAGE_MANAGER_EXISTS % (installed[0], installed[0])) + active_name = installed[0] + active_cmd = PACKAGE_MANAGER[active_name] + else: + print(MULTI_PACKAGE_MANAGER_EXISTS) + choice = self.prompt_int(prompt=PACKAGE_MANAGER_CHOICE, low=1, high=2) + + active_name = PACKAGE_MANAGER_CHOICES[choice - 1] + active_cmd = PACKAGE_MANAGER[active_name] + + # Ensure the active package manager is in $PATH and it comes before + # /usr/bin. If it doesn't come before /usr/bin, we'll pick up system + # packages before package manager installed packages and the build may + # break. + p = which(active_cmd) + if not p: + print(PACKAGE_MANAGER_BIN_MISSING % active_cmd) + sys.exit(1) + + p_dir = os.path.dirname(p) + for path in os.environ["PATH"].split(os.pathsep): + if path == p_dir: + break + + for check in ("/bin", "/usr/bin"): + if path == check: + print(BAD_PATH_ORDER % (check, p_dir, p_dir, check, p_dir)) + sys.exit(1) + + return active_name.lower() + + def ensure_clang_static_analysis_package(self, state_dir, checkout_root): + from mozboot import static_analysis + + self.install_toolchain_static_analysis( + state_dir, checkout_root, static_analysis.MACOS_CLANG_TIDY + ) + + def ensure_sccache_packages(self, state_dir, checkout_root): + from mozboot import sccache + + self.install_toolchain_artifact(state_dir, checkout_root, sccache.MACOS_SCCACHE) + self.install_toolchain_artifact( + state_dir, checkout_root, sccache.RUSTC_DIST_TOOLCHAIN, no_unpack=True + ) + self.install_toolchain_artifact( + state_dir, checkout_root, sccache.CLANG_DIST_TOOLCHAIN, no_unpack=True + ) + + def ensure_fix_stacks_packages(self, state_dir, checkout_root): + from mozboot import fix_stacks + + self.install_toolchain_artifact( + state_dir, checkout_root, fix_stacks.MACOS_FIX_STACKS + ) + + def ensure_stylo_packages(self, state_dir, checkout_root): + from mozboot import stylo + + self.install_toolchain_artifact(state_dir, checkout_root, stylo.MACOS_CLANG) + self.install_toolchain_artifact(state_dir, checkout_root, stylo.MACOS_CBINDGEN) + + def ensure_nasm_packages(self, state_dir, checkout_root): + from mozboot import nasm + + self.install_toolchain_artifact(state_dir, checkout_root, nasm.MACOS_NASM) + + def ensure_node_packages(self, state_dir, checkout_root): + # XXX from necessary? + from mozboot import node + + self.install_toolchain_artifact(state_dir, checkout_root, node.OSX) + + def ensure_minidump_stackwalk_packages(self, state_dir, checkout_root): + from mozboot import minidump_stackwalk + + self.install_toolchain_artifact( + state_dir, checkout_root, minidump_stackwalk.MACOS_MINIDUMP_STACKWALK + ) + + def ensure_dump_syms_packages(self, state_dir, checkout_root): + from mozboot import dump_syms + + self.install_toolchain_artifact( + state_dir, checkout_root, dump_syms.MACOS_DUMP_SYMS + ) + + def install_homebrew(self): + print(PACKAGE_MANAGER_INSTALL % ("Homebrew", "Homebrew", "Homebrew", "brew")) + bootstrap = urlopen(url=HOMEBREW_BOOTSTRAP, timeout=20).read() + with tempfile.NamedTemporaryFile() as tf: + tf.write(bootstrap) + tf.flush() + + subprocess.check_call(["ruby", tf.name]) + + def install_macports(self): + url = MACPORTS_URL.get(self.minor_version, None) + if not url: + raise Exception( + "We do not have a MacPorts install URL for your " + "OS X version. You will need to install MacPorts manually." + ) + + print(PACKAGE_MANAGER_INSTALL % ("MacPorts", "MacPorts", "MacPorts", "port")) + pkg = urlopen(url=url, timeout=300).read() + with tempfile.NamedTemporaryFile(suffix=".pkg") as tf: + tf.write(pkg) + tf.flush() + + self.run_as_root(["installer", "-pkg", tf.name, "-target", "/"]) + + def _update_package_manager(self): + if self.package_manager == "homebrew": + subprocess.check_call([self.brew, "-v", "update"]) + else: + assert self.package_manager == "macports" + self.run_as_root([self.port, "selfupdate"]) + + def _upgrade_package(self, package): + self._ensure_package_manager_updated() + + if self.package_manager == "homebrew": + try: + subprocess.check_output( + [self.brew, "-v", "upgrade", package], stderr=subprocess.STDOUT + ) + except subprocess.CalledProcessError as e: + if b"already installed" not in e.output: + raise + else: + assert self.package_manager == "macports" + + self.run_as_root([self.port, "upgrade", package]) + + def upgrade_mercurial(self, current): + self._upgrade_package("mercurial") diff --git a/python/mozboot/mozboot/rust.py b/python/mozboot/mozboot/rust.py new file mode 100644 index 0000000000..595a873f00 --- /dev/null +++ b/python/mozboot/mozboot/rust.py @@ -0,0 +1,193 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this, +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import sys + +# Base url for pulling the rustup installer. +# Use the no-CNAME host for compatibilty with Python 2.7 +# which doesn't support SNI. +RUSTUP_URL_BASE = "https://static-rust-lang-org.s3.amazonaws.com/rustup" + +# Pull this to get the lastest stable version number. +RUSTUP_MANIFEST = RUSTUP_URL_BASE + "/release-stable.toml" + +# We bake in a known version number so we can verify a checksum. +RUSTUP_VERSION = "1.21.1" + +# SHA-256 checksums of the installers, per platform. +RUSTUP_HASHES = { + "x86_64-unknown-freebsd": "a6bfc71c58b7ac3dad0d6ea0937990ca72f3b636096244c0c9ba814a627cbcc1", + "x86_64-apple-darwin": "fd76f7093bd810f9ee9050786678c74155d6f5fcc3aac958d24c0783e435a994", + "x86_64-unknown-linux-gnu": "ad1f8b5199b3b9e231472ed7aa08d2e5d1d539198a15c5b1e53c746aad81d27b", + "x86_64-pc-windows-msvc": "9f9e33fa4759075ec60e4da13798d1d66a4c2f43c5500e08714399313409dcf5", +} + +NO_PLATFORM = """ +Sorry, we have no installer configured for your platform. + +Please try installing rust for your system from https://rustup.rs/ +or from https://rust-lang.org/ or from your package manager. +""" + + +def rustup_url(host, version=RUSTUP_VERSION): + """Download url for a particular version of the installer.""" + return "%(base)s/archive/%(version)s/%(host)s/rustup-init%(ext)s" % { + "base": RUSTUP_URL_BASE, + "version": version, + "host": host, + "ext": exe_suffix(host), + } + + +def rustup_hash(host): + """Look up the checksum for the given installer.""" + return RUSTUP_HASHES.get(host, None) + + +def platform(): + """Determine the appropriate rust platform string for the current host""" + if sys.platform.startswith("darwin"): + return "x86_64-apple-darwin" + elif sys.platform.startswith(("win32", "msys")): + # Bravely assume we'll be building 64-bit Firefox. + return "x86_64-pc-windows-msvc" + elif sys.platform.startswith("linux"): + return "x86_64-unknown-linux-gnu" + elif sys.platform.startswith("freebsd"): + return "x86_64-unknown-freebsd" + + return None + + +def exe_suffix(host=None): + if not host: + host = platform() + if "windows" in host: + return ".exe" + return "" + + +USAGE = """ +python rust.py [--update] + +Pass the --update option print info for the latest release of rustup-init. + +When invoked without the --update option, it queries the latest version +and verifies the current stored checksums against the distribution server, +but doesn't update the version installed by `mach bootstrap`. +""" + + +def unquote(s): + """Strip outer quotation marks from a string.""" + return s.strip("'").strip('"') + + +def rustup_latest_version(): + """Query the latest version of the rustup installer.""" + import urllib2 + + f = urllib2.urlopen(RUSTUP_MANIFEST) + # The manifest is toml, but we might not have the toml4 python module + # available, so use ad-hoc parsing to obtain the current release version. + # + # The manifest looks like: + # + # schema-version = '1' + # version = '0.6.5' + # + for line in f: + key, value = map(str.strip, line.split(b"=", 2)) + if key == "schema-version": + schema = int(unquote(value)) + if schema != 1: + print("ERROR: Unknown manifest schema %s" % value) + sys.exit(1) + elif key == "version": + return unquote(value) + return None + + +def http_download_and_hash(url): + import hashlib + import requests + + h = hashlib.sha256() + r = requests.get(url, stream=True) + for data in r.iter_content(4096): + h.update(data) + return h.hexdigest() + + +def make_checksums(version, validate=False): + hashes = [] + for platform in RUSTUP_HASHES.keys(): + if validate: + print("Checking %s... " % platform, end="") + else: + print("Fetching %s... " % platform, end="") + checksum = http_download_and_hash(rustup_url(platform, version)) + if validate and checksum != rustup_hash(platform): + print( + "mismatch:\n script: %s\n server: %s" + % (RUSTUP_HASHES[platform], checksum) + ) + else: + print("OK") + hashes.append((platform, checksum)) + return hashes + + +if __name__ == "__main__": + """Allow invoking the module as a utility to update checksums.""" + + # Unbuffer stdout so our two-part 'Checking...' messages print correctly + # even if there's network delay. + sys.stdout = os.fdopen(sys.stdout.fileno(), "w", 0) + + # Hook the requests module from the greater source tree. We can't import + # this at the module level since we might be imported into the bootstrap + # script in standalone mode. + # + # This module is necessary for correct https certificate verification. + mod_path = os.path.dirname(__file__) + sys.path.insert(0, os.path.join(mod_path, "..", "..", "requests")) + + update = False + if len(sys.argv) > 1: + if sys.argv[1] == "--update": + update = True + else: + print(USAGE) + sys.exit(1) + + print("Checking latest installer version... ", end="") + version = rustup_latest_version() + if not version: + print("ERROR: Could not query current rustup installer version.") + sys.exit(1) + print(version) + + if version == RUSTUP_VERSION: + print("We're up to date. Validating checksums.") + make_checksums(version, validate=True) + exit() + + if not update: + print("Out of date. We use %s. Validating checksums." % RUSTUP_VERSION) + make_checksums(RUSTUP_VERSION, validate=True) + exit() + + print("Out of date. We use %s. Calculating checksums." % RUSTUP_VERSION) + hashes = make_checksums(version) + print("") + print("RUSTUP_VERSION = '%s'" % version) + print("RUSTUP_HASHES = {") + for item in hashes: + print(" '%s':\n '%s'," % item) + print("}") diff --git a/python/mozboot/mozboot/sccache.py b/python/mozboot/mozboot/sccache.py new file mode 100644 index 0000000000..476375be3f --- /dev/null +++ b/python/mozboot/mozboot/sccache.py @@ -0,0 +1,15 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +LINUX_SCCACHE = "linux64-sccache" +MACOS_SCCACHE = "macosx64-sccache" +WIN64_SCCACHE = "win64-sccache" + +# sccache-dist currently expects clients to provide toolchains when +# distributing from macOS or Windows, so we download linux binaries capable +# of cross-compiling for these cases. +RUSTC_DIST_TOOLCHAIN = "rustc-dist-toolchain" +CLANG_DIST_TOOLCHAIN = "clang-dist-toolchain" diff --git a/python/mozboot/mozboot/solus.py b/python/mozboot/mozboot/solus.py new file mode 100644 index 0000000000..c220b1bd50 --- /dev/null +++ b/python/mozboot/mozboot/solus.py @@ -0,0 +1,122 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import sys +import subprocess + +from mozboot.base import BaseBootstrapper +from mozboot.linux_common import LinuxBootstrapper + +# NOTE: This script is intended to be run with a vanilla Python install. We +# have to rely on the standard library instead of Python 2+3 helpers like +# the six module. +if sys.version_info < (3,): + input = raw_input # noqa + + +class SolusBootstrapper(LinuxBootstrapper, BaseBootstrapper): + """Solus experimental bootstrapper.""" + + SYSTEM_PACKAGES = [ + "nodejs", + "unzip", + "zip", + ] + SYSTEM_COMPONENTS = [ + "system.devel", + ] + + BROWSER_PACKAGES = [ + "alsa-lib", + "dbus", + "libgtk-2", + "libgtk-3", + "libevent", + "libvpx", + "libxt", + "nasm", + "libstartup-notification", + "gst-plugins-base", + "gst-plugins-good", + "pulseaudio", + "xorg-server-xvfb", + "yasm", + ] + + MOBILE_ANDROID_COMMON_PACKAGES = [ + "openjdk-8", + # For downloading the Android SDK and NDK. + "wget", + # See comment about 32 bit binaries and multilib below. + "ncurses-32bit", + "readline-32bit", + "zlib-32bit", + ] + + def __init__(self, version, dist_id, **kwargs): + print("Using an experimental bootstrapper for Solus.") + BaseBootstrapper.__init__(self, **kwargs) + + def install_system_packages(self): + self.package_install(*self.SYSTEM_PACKAGES) + self.component_install(*self.SYSTEM_COMPONENTS) + + def install_browser_packages(self, mozconfig_builder): + self.ensure_browser_packages() + + def install_browser_artifact_mode_packages(self, mozconfig_builder): + self.ensure_browser_packages(artifact_mode=True) + + def install_mobile_android_packages(self, mozconfig_builder): + self.ensure_mobile_android_packages(mozconfig_builder) + + def install_mobile_android_artifact_mode_packages(self, mozconfig_builder): + self.ensure_mobile_android_packages(mozconfig_builder, artifact_mode=True) + + def ensure_browser_packages(self, artifact_mode=False): + self.package_install(*self.BROWSER_PACKAGES) + + def ensure_nasm_packages(self, state_dir, checkout_root): + # installed via ensure_browser_packages + pass + + def ensure_mobile_android_packages(self, mozconfig_builder, artifact_mode=False): + try: + self.package_install(*self.MOBILE_ANDROID_COMMON_PACKAGES) + except Exception as e: + print("Failed to install all packages!") + raise e + + # 2. Android pieces. + self.ensure_java(mozconfig_builder) + super().ensure_mobile_android_packages(artifact_mode=artifact_mode) + + def _update_package_manager(self): + pass + + def upgrade_mercurial(self, current): + self.package_install("mercurial") + + def package_install(self, *packages): + command = ["eopkg", "install"] + if self.no_interactive: + command.append("--yes-all") + + command.extend(packages) + + self.run_as_root(command) + + def component_install(self, *components): + command = ["eopkg", "install", "-c"] + if self.no_interactive: + command.append("--yes-all") + + command.extend(components) + + self.run_as_root(command) + + def run(self, command, env=None): + subprocess.check_call(command, stdin=sys.stdin, env=env) diff --git a/python/mozboot/mozboot/static_analysis.py b/python/mozboot/mozboot/static_analysis.py new file mode 100644 index 0000000000..8ecb7dfaad --- /dev/null +++ b/python/mozboot/mozboot/static_analysis.py @@ -0,0 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +WINDOWS_CLANG_TIDY = "win64-clang-tidy" +LINUX_CLANG_TIDY = "linux64-clang-tidy" +MACOS_CLANG_TIDY = "macosx64-clang-tidy" diff --git a/python/mozboot/mozboot/stylo.py b/python/mozboot/mozboot/stylo.py new file mode 100644 index 0000000000..fb97756110 --- /dev/null +++ b/python/mozboot/mozboot/stylo.py @@ -0,0 +1,12 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +WINDOWS_CLANG = "win64-clang-cl" +WINDOWS_CBINDGEN = "win64-cbindgen" +LINUX_CLANG = "linux64-clang" +LINUX_CBINDGEN = "linux64-cbindgen" +MACOS_CLANG = "macosx64-clang" +MACOS_CBINDGEN = "macosx64-cbindgen" diff --git a/python/mozboot/mozboot/test/python.ini b/python/mozboot/mozboot/test/python.ini new file mode 100644 index 0000000000..fe588a7e59 --- /dev/null +++ b/python/mozboot/mozboot/test/python.ini @@ -0,0 +1,5 @@ +[DEFAULT] +subsuite = mozbuild + +[test_mozconfig.py] +[test_write_config.py] diff --git a/python/mozboot/mozboot/test/test_mozconfig.py b/python/mozboot/mozboot/test/test_mozconfig.py new file mode 100644 index 0000000000..df05c0b95f --- /dev/null +++ b/python/mozboot/mozboot/test/test_mozconfig.py @@ -0,0 +1,228 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import unittest + +from shutil import rmtree + +from tempfile import ( + gettempdir, + mkdtemp, +) + +from mozboot.mozconfig import ( + MozconfigFindException, + find_mozconfig, + DEFAULT_TOPSRCDIR_PATHS, + DEPRECATED_TOPSRCDIR_PATHS, + DEPRECATED_HOME_PATHS, +) +from mozunit import main + + +class TestFindMozconfig(unittest.TestCase): + def setUp(self): + self._old_env = dict(os.environ) + os.environ.pop("MOZCONFIG", None) + os.environ.pop("MOZ_OBJDIR", None) + os.environ.pop("CC", None) + os.environ.pop("CXX", None) + self._temp_dirs = set() + + def tearDown(self): + os.environ.clear() + os.environ.update(self._old_env) + + for d in self._temp_dirs: + rmtree(d) + + def get_temp_dir(self): + d = mkdtemp() + self._temp_dirs.add(d) + + return d + + def test_find_legacy_env(self): + """Ensure legacy mozconfig path definitions result in error.""" + + os.environ["MOZ_MYCONFIG"] = "/foo" + + with self.assertRaises(MozconfigFindException) as e: + find_mozconfig(self.get_temp_dir()) + + self.assertTrue(str(e.exception).startswith("The MOZ_MYCONFIG")) + + def test_find_multiple_configs(self): + """Ensure multiple relative-path MOZCONFIGs result in error.""" + relative_mozconfig = ".mconfig" + os.environ["MOZCONFIG"] = relative_mozconfig + + srcdir = self.get_temp_dir() + curdir = self.get_temp_dir() + dirs = [srcdir, curdir] + for d in dirs: + path = os.path.join(d, relative_mozconfig) + with open(path, "w") as f: + f.write(path) + + orig_dir = os.getcwd() + try: + os.chdir(curdir) + with self.assertRaises(MozconfigFindException) as e: + find_mozconfig(srcdir) + finally: + os.chdir(orig_dir) + + self.assertIn("exists in more than one of", str(e.exception)) + for d in dirs: + self.assertIn(d, str(e.exception)) + + def test_find_multiple_but_identical_configs(self): + """Ensure multiple relative-path MOZCONFIGs pointing at the same file are OK.""" + relative_mozconfig = "../src/.mconfig" + os.environ["MOZCONFIG"] = relative_mozconfig + + topdir = self.get_temp_dir() + srcdir = os.path.join(topdir, "src") + os.mkdir(srcdir) + curdir = os.path.join(topdir, "obj") + os.mkdir(curdir) + + path = os.path.join(srcdir, relative_mozconfig) + with open(path, "w"): + pass + + orig_dir = os.getcwd() + try: + os.chdir(curdir) + self.assertEqual( + os.path.realpath(find_mozconfig(srcdir)), os.path.realpath(path) + ) + finally: + os.chdir(orig_dir) + + def test_find_no_relative_configs(self): + """Ensure a missing relative-path MOZCONFIG is detected.""" + relative_mozconfig = ".mconfig" + os.environ["MOZCONFIG"] = relative_mozconfig + + srcdir = self.get_temp_dir() + curdir = self.get_temp_dir() + dirs = [srcdir, curdir] + + orig_dir = os.getcwd() + try: + os.chdir(curdir) + with self.assertRaises(MozconfigFindException) as e: + find_mozconfig(srcdir) + finally: + os.chdir(orig_dir) + + self.assertIn("does not exist in any of", str(e.exception)) + for d in dirs: + self.assertIn(d, str(e.exception)) + + def test_find_relative_mozconfig(self): + """Ensure a relative MOZCONFIG can be found in the srcdir.""" + relative_mozconfig = ".mconfig" + os.environ["MOZCONFIG"] = relative_mozconfig + + srcdir = self.get_temp_dir() + curdir = self.get_temp_dir() + + path = os.path.join(srcdir, relative_mozconfig) + with open(path, "w"): + pass + + orig_dir = os.getcwd() + try: + os.chdir(curdir) + self.assertEqual( + os.path.normpath(find_mozconfig(srcdir)), os.path.normpath(path) + ) + finally: + os.chdir(orig_dir) + + def test_find_abs_path_not_exist(self): + """Ensure a missing absolute path is detected.""" + os.environ["MOZCONFIG"] = "/foo/bar/does/not/exist" + + with self.assertRaises(MozconfigFindException) as e: + find_mozconfig(self.get_temp_dir()) + + self.assertIn("path that does not exist", str(e.exception)) + self.assertTrue(str(e.exception).endswith("/foo/bar/does/not/exist")) + + def test_find_path_not_file(self): + """Ensure non-file paths are detected.""" + + os.environ["MOZCONFIG"] = gettempdir() + + with self.assertRaises(MozconfigFindException) as e: + find_mozconfig(self.get_temp_dir()) + + self.assertIn("refers to a non-file", str(e.exception)) + self.assertTrue(str(e.exception).endswith(gettempdir())) + + def test_find_default_files(self): + """Ensure default paths are used when present.""" + for p in DEFAULT_TOPSRCDIR_PATHS: + d = self.get_temp_dir() + path = os.path.join(d, p) + + with open(path, "w"): + pass + + self.assertEqual(find_mozconfig(d), path) + + def test_find_multiple_defaults(self): + """Ensure we error when multiple default files are present.""" + self.assertGreater(len(DEFAULT_TOPSRCDIR_PATHS), 1) + + d = self.get_temp_dir() + for p in DEFAULT_TOPSRCDIR_PATHS: + with open(os.path.join(d, p), "w"): + pass + + with self.assertRaises(MozconfigFindException) as e: + find_mozconfig(d) + + self.assertIn("Multiple default mozconfig files present", str(e.exception)) + + def test_find_deprecated_path_srcdir(self): + """Ensure we error when deprecated path locations are present.""" + for p in DEPRECATED_TOPSRCDIR_PATHS: + d = self.get_temp_dir() + with open(os.path.join(d, p), "w"): + pass + + with self.assertRaises(MozconfigFindException) as e: + find_mozconfig(d) + + self.assertIn("This implicit location is no longer", str(e.exception)) + self.assertIn(d, str(e.exception)) + + def test_find_deprecated_home_paths(self): + """Ensure we error when deprecated home directory paths are present.""" + + for p in DEPRECATED_HOME_PATHS: + home = self.get_temp_dir() + os.environ["HOME"] = home + path = os.path.join(home, p) + + with open(path, "w"): + pass + + with self.assertRaises(MozconfigFindException) as e: + find_mozconfig(self.get_temp_dir()) + + self.assertIn("This implicit location is no longer", str(e.exception)) + self.assertIn(path, str(e.exception)) + + +if __name__ == "__main__": + main() diff --git a/python/mozboot/mozboot/test/test_write_config.py b/python/mozboot/mozboot/test/test_write_config.py new file mode 100644 index 0000000000..1869997806 --- /dev/null +++ b/python/mozboot/mozboot/test/test_write_config.py @@ -0,0 +1,107 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import mozunit +import pytest + +from mach.config import ConfigSettings +from mach.decorators import SettingsProvider +from mozboot.bootstrap import update_or_create_build_telemetry_config + + +# Duplicated from python/mozbuild/mozbuild/mach_commands.py because we can't +# actually import that module here. +@SettingsProvider +class TelemetrySettings: + config_settings = [ + ( + "build.telemetry", + "boolean", + """ +Enable submission of build system telemetry. + """.strip(), + False, + ), + ] + + +@SettingsProvider +class OtherSettings: + config_settings = [ + ("foo.bar", "int", "", 1), + ("build.abc", "string", "", ""), + ] + + +def read(path): + s = ConfigSettings() + s.register_provider(TelemetrySettings) + s.register_provider(OtherSettings) + s.load_file(path) + return s + + +@pytest.fixture +def config_path(tmpdir): + return str(tmpdir.join("machrc")) + + +@pytest.fixture +def write_config(config_path): + def _config(contents): + with open(config_path, "w") as f: + f.write(contents) + + return _config + + +def test_nonexistent(config_path): + update_or_create_build_telemetry_config(config_path) + s = read(config_path) + assert s.build.telemetry + + +def test_file_exists_no_build_section(config_path, write_config): + write_config( + """[foo] +bar = 2 +""" + ) + update_or_create_build_telemetry_config(config_path) + s = read(config_path) + assert s.build.telemetry + assert s.foo.bar == 2 + + +def test_existing_build_section(config_path, write_config): + write_config( + """[foo] +bar = 2 + +[build] +abc = xyz +""" + ) + update_or_create_build_telemetry_config(config_path) + s = read(config_path) + assert s.build.telemetry + assert s.build.abc == "xyz" + assert s.foo.bar == 2 + + +def test_malformed_file(config_path, write_config): + """Ensure that a malformed config file doesn't cause breakage.""" + write_config( + """[foo +bar = 1 +""" + ) + assert not update_or_create_build_telemetry_config(config_path) + # Can't read config, it will not have been written! + + +if __name__ == "__main__": + mozunit.main() diff --git a/python/mozboot/mozboot/util.py b/python/mozboot/mozboot/util.py new file mode 100644 index 0000000000..596d2dc84c --- /dev/null +++ b/python/mozboot/mozboot/util.py @@ -0,0 +1,310 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import hashlib +import os +import platform +import subprocess +from subprocess import CalledProcessError + +from mozbuild.virtualenv import VirtualenvHelper +from mozfile import which + + +here = os.path.join(os.path.dirname(__file__)) + + +MINIMUM_RUST_VERSION = "1.47.0" + + +def get_state_dir(srcdir=False): + """Obtain path to a directory to hold state. + + Args: + srcdir (bool): If True, return a state dir specific to the current + srcdir instead of the global state dir (default: False) + + Returns: + A path to the state dir (str) + """ + state_dir = os.environ.get("MOZBUILD_STATE_PATH", os.path.expanduser("~/.mozbuild")) + if not srcdir: + return state_dir + + # This function can be called without the build virutualenv, and in that + # case srcdir is supposed to be False. Import mozbuild here to avoid + # breaking that usage. + from mozbuild.base import MozbuildObject + + srcdir = os.path.abspath(MozbuildObject.from_environment(cwd=here).topsrcdir) + # Shortening to 12 characters makes these directories a bit more manageable + # in a terminal and is more than good enough for this purpose. + srcdir_hash = hashlib.sha256(srcdir.encode("utf-8")).hexdigest()[:12] + + state_dir = os.path.join( + state_dir, "srcdirs", "{}-{}".format(os.path.basename(srcdir), srcdir_hash) + ) + + if not os.path.isdir(state_dir): + # We create the srcdir here rather than 'mach_bootstrap.py' so direct + # consumers of this function don't create the directory inconsistently. + print("Creating local state directory: %s" % state_dir) + os.makedirs(state_dir, mode=0o770) + # Save the topsrcdir that this state dir corresponds to so we can clean + # it up in the event its srcdir was deleted. + with open(os.path.join(state_dir, "topsrcdir.txt"), "w") as fh: + fh.write(srcdir) + + return state_dir + + +def get_mach_virtualenv_root(state_dir=None, py2=False): + return os.path.join( + state_dir or get_state_dir(), "_virtualenvs", "mach_py2" if py2 else "mach" + ) + + +def get_mach_virtualenv_binary(state_dir=None, py2=False): + root = get_mach_virtualenv_root(state_dir=state_dir, py2=py2) + return VirtualenvHelper(root).python_path + + +class JavaLocationFailedException(Exception): + pass + + +def locate_java_bin_path(): + """Locate an expected version of Java. + + We require an installation of Java that is: + * a JDK (not just a JRE) because we compile Java. + * version 1.8 because the Android `sdkmanager` tool still needs it. + """ + + if "JAVA_HOME" in os.environ: + java_home = os.environ["JAVA_HOME"] + bin_path = os.path.join(java_home, "bin") + java_path = which("java", path=bin_path) + javac_path = which("javac", path=bin_path) + + if not java_path: + raise JavaLocationFailedException( + 'The $JAVA_HOME environment variable ("{}") is not ' + "pointing to a valid Java installation. Please " + "change $JAVA_HOME.".format(java_home) + ) + + version = _resolve_java_version(java_path) + + if not _is_java_version_correct(version): + raise JavaLocationFailedException( + 'The $JAVA_HOME environment variable ("{}") is ' + 'pointing to a Java installation with version "{}". ' + "Howevever, Firefox depends on version 1.8. Please " + "change $JAVA_HOME.".format(java_home, version) + ) + + if not javac_path: + raise JavaLocationFailedException( + 'The $JAVA_HOME environment variable ("{}") is ' + 'pointing to a "JRE". Since Firefox depends on Java ' + 'tools that are bundled in a "JDK", you will need ' + "to update $JAVA_HOME to point to a Java 1.8 JDK " + "instead (You may need to install a JDK first).".format(java_home) + ) + + return bin_path + + system = platform.system() + if system == "Windows": + jdk_bin_path = ( + _windows_registry_get_adopt_open_jdk_8_path() + or _windows_registry_get_oracle_jdk_8_path() + ) + + if not jdk_bin_path: + raise JavaLocationFailedException( + "Could not find the Java 1.8 JDK on your machine. " + "Please install it: " + "https://adoptopenjdk.net/?variant=openjdk8" + ) + + # No need for version/type validation: the registry keys we're looking up correspond to + # the specific Java installations we want. + return jdk_bin_path + elif system == "Darwin": + no_matching_java_version_exception = JavaLocationFailedException( + 'Could not find Java on your machine. Please install it, either via "mach bootstrap" ' + "(if you have brew) or directly from https://adoptopenjdk.net/?variant=openjdk8." + ) + try: + java_home_path = subprocess.check_output( + ["/usr/libexec/java_home", "-v", "1.8"], universal_newlines=True + ).strip() + except CalledProcessError: + raise no_matching_java_version_exception + + if not java_home_path: + raise no_matching_java_version_exception + + bin_path = os.path.join(java_home_path, "bin") + javac_path = which("javac", path=bin_path) + + if not javac_path: + raise JavaLocationFailedException( + "The Java 1.8 installation found by " + '"/usr/libexec/java_home" ("{}") is a JRE, not a ' + "JDK. Since Firefox depends on the JDK to compile " + "Java, you should install the Java 1.8 JDK, either " + 'via "mach bootstrap" (if you have brew) or ' + "directly from " + "https://adoptopenjdk.net/?variant=openjdk8.\n" + # In some cases, such as reported in bug 1670264, the + # "java_home" binary might return a JRE instead of a + # JDK. There's two workarounds: + # 1. Have the user uninstall the 1.8 JRE, so the JDK + # takes priority + # 2. Have the user explicitly specify $JAVA_HOME so + # that the JDK is selected. + "Note: if you have already installed a 1.8 JDK and " + "are still seeing this message, then you may need " + "to set $JAVA_HOME.".format(java_home_path) + ) + + return bin_path + else: # Handle Linux and other OSes by finding Java from the $PATH + java_path = which("java") + javac_path = which("javac") + if not java_path: + raise JavaLocationFailedException( + 'Could not find "java" on the $PATH. Please install ' + "the Java 1.8 JDK and/or set $JAVA_HOME." + ) + + java_version = _resolve_java_version(java_path) + if not _is_java_version_correct(java_version): + raise JavaLocationFailedException( + 'The "java" located on the $PATH has version "{}", ' + "but Firefox depends on version 1.8. Please install " + "the Java 1.8 JDK and/or set $JAVA_HOME.".format(java_version) + ) + + if not javac_path: + raise JavaLocationFailedException( + 'Could not find "javac" on the $PATH, even though ' + '"java" was located. This probably means that you ' + "have a JRE installed, not a JDK. Since Firefox " + "needs to compile Java, you should install the " + "Java 1.8 JDK and/or set $JAVA_HOME." + ) + + javac_bin_path = os.path.dirname(os.path.realpath(javac_path)) + javac_version = _resolve_java_version(which("java", path=javac_bin_path)) + + # On Ubuntu, there's an "update-alternatives" command that can be used to change the + # default version of specific binaries. To ensure that "java" and "javac" are + # pointing at the same "alternative", we compare their versions. + if java_version != javac_version: + raise JavaLocationFailedException( + 'The "java" ("{}") and "javac" ("{}") binaries on ' + "the $PATH are currently coming from two different " + "Java installations with different versions. Please " + "resolve this, or explicitly set $JAVA_HOME.".format( + java_version, javac_version + ) + ) + + # We use the "bin/" detected as a parent of "javac" instead of "java" because the + # structure of some JDKs places "java" in a different directory: + # + # $JDK/ + # bin/ + # javac + # java -> ../jre/bin/java + # ... + # jre/ + # bin/ + # java + # ... + # ... + # + # Realpath-ing "javac" should consistently gives us a JDK bin dir + # containing both "java" and JDK tools. + return javac_bin_path + + +def _resolve_java_version(java_bin): + output = subprocess.check_output( + [java_bin, "-XshowSettings:properties", "-version"], + stderr=subprocess.STDOUT, + universal_newlines=True, + ).rstrip() + + # -version strings are pretty free-form, like: 'java version + # "1.8.0_192"' or 'openjdk version "11.0.1" 2018-10-16', but the + # -XshowSettings:properties gives the information (to stderr, sigh) + # like 'java.specification.version = 8'. That flag is non-standard + # but has been around since at least 2011. + version = [ + line for line in output.splitlines() if "java.specification.version" in line + ] + + if len(version) != 1: + return None + + return version[0].split(" = ")[-1] + + +def _is_java_version_correct(version): + return version in ["1.8", "8"] + + +def _windows_registry_get_oracle_jdk_8_path(): + try: + import _winreg + except ImportError: + import winreg as _winreg + + try: + with _winreg.OpenKeyEx( + _winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\JavaSoft\Java Development Kit\1.8" + ) as key: + path, _ = _winreg.QueryValueEx(key, "JavaHome") + return os.path.join(path, "bin") + except FileNotFoundError: + return None + + +def _windows_registry_get_adopt_open_jdk_8_path(): + try: + import _winreg + except ImportError: + import winreg as _winreg + + try: + # The registry key name looks like: + # HKLM\SOFTWARE\AdoptOpenJDK\JDK\8.0.252.09\hotspot\MSI:Path + # ^^^^^^^^^^ + # Due to the very precise version in the path, we can't just OpenKey("<static path>"). + # Instead, we need to enumerate the list of JDKs, find the one that seems to be what + # we're looking for (JDK 1.8), then get the path from there. + with _winreg.OpenKeyEx( + _winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\AdoptOpenJDK\JDK" + ) as jdk_key: + index = 0 + while True: + version_key_name = _winreg.EnumKey(jdk_key, index) + if not version_key_name.startswith("8."): + index += 1 + continue + + with _winreg.OpenKeyEx( + jdk_key, r"{}\hotspot\MSI".format(version_key_name) + ) as msi_key: + path, _ = _winreg.QueryValueEx(msi_key, "Path") + return os.path.join(path, "bin") + except (FileNotFoundError, OSError): + return None diff --git a/python/mozboot/mozboot/void.py b/python/mozboot/mozboot/void.py new file mode 100644 index 0000000000..572fcdd92d --- /dev/null +++ b/python/mozboot/mozboot/void.py @@ -0,0 +1,108 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import subprocess +import sys + +from mozboot.base import BaseBootstrapper +from mozboot.linux_common import LinuxBootstrapper + + +class VoidBootstrapper(LinuxBootstrapper, BaseBootstrapper): + + PACKAGES = [ + "clang", + "make", + "mercurial", + "nodejs", + "unzip", + "zip", + ] + + BROWSER_PACKAGES = [ + "dbus-devel", + "dbus-glib-devel", + "gtk+3-devel", + "pulseaudio", + "pulseaudio-devel", + "libcurl-devel", + "libxcb-devel", + "libXt-devel", + "yasm", + ] + + MOBILE_ANDROID_PACKAGES = [ + "openjdk8", # Android's `sdkmanager` requires Java 1.8 exactly. + "wget", # For downloading the Android SDK and NDK. + ] + + def __init__(self, version, dist_id, **kwargs): + BaseBootstrapper.__init__(self, **kwargs) + + self.distro = "void" + self.version = version + self.dist_id = dist_id + + self.packages = self.PACKAGES + self.browser_packages = self.BROWSER_PACKAGES + self.mobile_android_packages = self.MOBILE_ANDROID_PACKAGES + + def run_as_root(self, command): + # VoidLinux doesn't support users sudo'ing most commands by default because of the group + # configuration. + if os.geteuid() != 0: + command = ["su", "root", "-c", " ".join(command)] + + print("Executing as root:", subprocess.list2cmdline(command)) + + subprocess.check_call(command, stdin=sys.stdin) + + def xbps_install(self, *packages): + command = ["xbps-install"] + if self.no_interactive: + command.append("-y") + command.extend(packages) + + self.run_as_root(command) + + def xbps_update(self): + command = ["xbps-install", "-Su"] + if self.no_interactive: + command.append("-y") + + self.run_as_root(command) + + def install_system_packages(self): + self.xbps_install(*self.packages) + + def install_browser_packages(self, mozconfig_builder): + self.ensure_browser_packages() + + def install_browser_artifact_mode_packages(self, mozconfig_builder): + self.ensure_browser_packages(artifact_mode=True) + + def install_mobile_android_packages(self, mozconfig_builder): + self.ensure_mobile_android_packages(mozconfig_builder) + + def install_mobile_android_artifact_mode_packages(self, mozconfig_builder): + self.ensure_mobile_android_packages(mozconfig_builder, artifact_mode=True) + + def ensure_browser_packages(self, artifact_mode=False): + self.xbps_install(*self.browser_packages) + + def ensure_mobile_android_packages(self, mozconfig_builder, artifact_mode=False): + # Multi-part process: + # 1. System packages. + # 2. Android SDK. Android NDK only if we are not in artifact mode. Android packages. + self.xbps_install(*self.mobile_android_packages) + + # 2. Android pieces. + self.ensure_java(mozconfig_builder) + super().ensure_mobile_android_packages(artifact_mode=artifact_mode) + + def _update_package_manager(self): + self.xbps_update() diff --git a/python/mozboot/mozboot/wasi_sysroot.py b/python/mozboot/mozboot/wasi_sysroot.py new file mode 100644 index 0000000000..6b2905afbf --- /dev/null +++ b/python/mozboot/mozboot/wasi_sysroot.py @@ -0,0 +1,7 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +LINUX_WASI_SYSROOT = "wasi-sysroot" diff --git a/python/mozboot/mozboot/windows.py b/python/mozboot/mozboot/windows.py new file mode 100644 index 0000000000..784449fd08 --- /dev/null +++ b/python/mozboot/mozboot/windows.py @@ -0,0 +1,167 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import ctypes +import os +import sys +import subprocess + +from mozboot.base import BaseBootstrapper +from mozfile import which + + +def is_aarch64_host(): + from ctypes import wintypes + + kernel32 = ctypes.windll.kernel32 + IMAGE_FILE_MACHINE_UNKNOWN = 0 + IMAGE_FILE_MACHINE_ARM64 = 0xAA64 + + try: + iswow64process2 = kernel32.IsWow64Process2 + except Exception: + # If we can't access the symbol, we know we're not on aarch64. + return False + + currentProcess = kernel32.GetCurrentProcess() + processMachine = wintypes.USHORT(IMAGE_FILE_MACHINE_UNKNOWN) + nativeMachine = wintypes.USHORT(IMAGE_FILE_MACHINE_UNKNOWN) + + gotValue = iswow64process2( + currentProcess, ctypes.byref(processMachine), ctypes.byref(nativeMachine) + ) + # If this call fails, we have no idea. + if not gotValue: + return False + + return nativeMachine.value == IMAGE_FILE_MACHINE_ARM64 + + +class WindowsBootstrapper(BaseBootstrapper): + """Bootstrapper for msys2 based environments for building in Windows.""" + + SYSTEM_PACKAGES = [ + "mingw-w64-x86_64-make", + "mingw-w64-x86_64-perl", + "patch", + "patchutils", + "diffutils", + "tar", + "zip", + "unzip", + "mingw-w64-x86_64-toolchain", # TODO: Remove when Mercurial is installable from a wheel. + "mingw-w64-i686-toolchain", + ] + + BROWSER_PACKAGES = [ + "mingw-w64-x86_64-nasm", + "mingw-w64-x86_64-yasm", + "mingw-w64-i686-nsis", + ] + + MOBILE_ANDROID_COMMON_PACKAGES = ["wget"] + + def __init__(self, **kwargs): + if ( + "MOZ_WINDOWS_BOOTSTRAP" not in os.environ + or os.environ["MOZ_WINDOWS_BOOTSTRAP"] != "1" + ): + raise NotImplementedError( + "Bootstrap support for Windows is under development. For " + "now use MozillaBuild to set up a build environment on " + "Windows. If you are testing Windows Bootstrap support, " + "try `export MOZ_WINDOWS_BOOTSTRAP=1`" + ) + BaseBootstrapper.__init__(self, **kwargs) + if not which("pacman"): + raise NotImplementedError( + "The Windows bootstrapper only works with msys2 with " + "pacman. Get msys2 at http://msys2.github.io/" + ) + print("Using an experimental bootstrapper for Windows.") + + def install_system_packages(self): + self.pacman_install(*self.SYSTEM_PACKAGES) + + def upgrade_mercurial(self, current): + self.pip_install("mercurial") + + def install_browser_packages(self, mozconfig_builder): + self.pacman_install(*self.BROWSER_PACKAGES) + + def install_mobile_android_packages(self, mozconfig_builder): + raise NotImplementedError( + "We do not support building Android on Windows. Sorry!" + ) + + def install_mobile_android_artifact_mode_packages(self, mozconfig_builder): + raise NotImplementedError( + "We do not support building Android on Windows. Sorry!" + ) + + def ensure_clang_static_analysis_package(self, state_dir, checkout_root): + from mozboot import static_analysis + + self.install_toolchain_static_analysis( + state_dir, checkout_root, static_analysis.WINDOWS_CLANG_TIDY + ) + + def ensure_stylo_packages(self, state_dir, checkout_root): + # On-device artifact builds are supported; on-device desktop builds are not. + if is_aarch64_host(): + raise Exception( + "You should not be performing desktop builds on an " + "AArch64 device. If you want to do artifact builds " + "instead, please choose the appropriate artifact build " + "option when beginning bootstrap." + ) + + from mozboot import stylo + + self.install_toolchain_artifact(state_dir, checkout_root, stylo.WINDOWS_CLANG) + self.install_toolchain_artifact( + state_dir, checkout_root, stylo.WINDOWS_CBINDGEN + ) + + def ensure_nasm_packages(self, state_dir, checkout_root): + from mozboot import nasm + + self.install_toolchain_artifact(state_dir, checkout_root, nasm.WINDOWS_NASM) + + def ensure_node_packages(self, state_dir, checkout_root): + from mozboot import node + + # We don't have native aarch64 node available, but aarch64 windows + # runs x86 binaries, so just use the x86 packages for such hosts. + node_artifact = node.WIN32 if is_aarch64_host() else node.WIN64 + self.install_toolchain_artifact(state_dir, checkout_root, node_artifact) + + def _update_package_manager(self): + self.pacman_update() + + def run(self, command): + subprocess.check_call(command, stdin=sys.stdin) + + def pacman_update(self): + command = ["pacman", "--sync", "--refresh"] + self.run(command) + + def pacman_upgrade(self): + command = ["pacman", "--sync", "--refresh", "--sysupgrade"] + self.run(command) + + def pacman_install(self, *packages): + command = ["pacman", "--sync", "--needed"] + if self.no_interactive: + command.append("--noconfirm") + + command.extend(packages) + self.run(command) + + def pip_install(self, *packages): + command = ["pip", "install", "--upgrade"] + command.extend(packages) + self.run(command) diff --git a/python/mozboot/setup.py b/python/mozboot/setup.py new file mode 100644 index 0000000000..df7000d85a --- /dev/null +++ b/python/mozboot/setup.py @@ -0,0 +1,18 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from distutils.core import setup + +VERSION = "0.1" + +setup( + name="mozboot", + description="System bootstrap for building Mozilla projects.", + license="MPL 2.0", + packages=["mozboot"], + version=VERSION, + scripts=["bin/bootstrap.py"], +) |