diff options
Diffstat (limited to 'python/mozbuild/mozbuild/build_commands.py')
-rw-r--r-- | python/mozbuild/mozbuild/build_commands.py | 366 |
1 files changed, 366 insertions, 0 deletions
diff --git a/python/mozbuild/mozbuild/build_commands.py b/python/mozbuild/mozbuild/build_commands.py new file mode 100644 index 0000000000..47398dc3a0 --- /dev/null +++ b/python/mozbuild/mozbuild/build_commands.py @@ -0,0 +1,366 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import os +import subprocess + +import mozpack.path as mozpath +from mach.decorators import Command, CommandArgument + +from mozbuild.backend import backends +from mozbuild.mozconfig import MozconfigLoader +from mozbuild.util import MOZBUILD_METRICS_PATH + +BUILD_WHAT_HELP = """ +What to build. Can be a top-level make target or a relative directory. If +multiple options are provided, they will be built serially. BUILDING ONLY PARTS +OF THE TREE CAN RESULT IN BAD TREE STATE. USE AT YOUR OWN RISK. +""".strip() + + +def _set_priority(priority, verbose): + # Choose the Windows API structure to standardize on. + PRIO_CLASS_BY_KEY = { + "idle": "IDLE_PRIORITY_CLASS", + "less": "BELOW_NORMAL_PRIORITY_CLASS", + "normal": "NORMAL_PRIORITY_CLASS", + "more": "ABOVE_NORMAL_PRIORITY_CLASS", + "high": "HIGH_PRIORITY_CLASS", + } + try: + prio_class = PRIO_CLASS_BY_KEY[priority] + except KeyError: + raise KeyError(f"priority '{priority}' not in {list(PRIO_CLASS_BY_KEY)}") + + if "nice" in dir(os): + # Translate the Windows priority classes into niceness values. + NICENESS_BY_PRIO_CLASS = { + "IDLE_PRIORITY_CLASS": 19, + "BELOW_NORMAL_PRIORITY_CLASS": 10, + "NORMAL_PRIORITY_CLASS": 0, + "ABOVE_NORMAL_PRIORITY_CLASS": -10, + "HIGH_PRIORITY_CLASS": -20, + } + niceness = NICENESS_BY_PRIO_CLASS[prio_class] + + os.nice(niceness) + if verbose: + print(f"os.nice({niceness})") + return True + + try: + import psutil + + prio_class_val = getattr(psutil, prio_class) + except ModuleNotFoundError: + return False + except AttributeError: + return False + + psutil.Process().nice(prio_class_val) + if verbose: + print(f"psutil.Process().nice(psutil.{prio_class})") + return True + + +# Interface to build the tree. + + +@Command( + "build", + category="build", + description="Build the tree.", + metrics_path=MOZBUILD_METRICS_PATH, + virtualenv_name="build", +) +@CommandArgument( + "--jobs", + "-j", + default="0", + metavar="jobs", + type=int, + help="Number of concurrent jobs to run. Default is based on the number of " + "CPUs and the estimated size of the jobs (see --job-size).", +) +@CommandArgument( + "--job-size", + default="0", + metavar="size", + type=float, + help="Estimated RAM required, in GiB, for each parallel job. Used to " + "compute a default number of concurrent jobs.", +) +@CommandArgument( + "-C", + "--directory", + default=None, + help="Change to a subdirectory of the build directory first.", +) +@CommandArgument("what", default=None, nargs="*", help=BUILD_WHAT_HELP) +@CommandArgument( + "-v", + "--verbose", + action="store_true", + help="Verbose output for what commands the build is running.", +) +@CommandArgument( + "--keep-going", + action="store_true", + help="Keep building after an error has occurred", +) +@CommandArgument( + "--priority", + default="less", + metavar="priority", + type=str, + help="idle/less/normal/more/high. (Default less)", +) +def build( + command_context, + what=None, + jobs=0, + job_size=0, + directory=None, + verbose=False, + keep_going=False, + priority="less", +): + """Build the source tree. + + With no arguments, this will perform a full build. + + Positional arguments define targets to build. These can be make targets + or patterns like "<dir>/<target>" to indicate a make target within a + directory. + + There are a few special targets that can be used to perform a partial + build faster than what `mach build` would perform: + + * binaries - compiles and links all C/C++ sources and produces shared + libraries and executables (binaries). + + * faster - builds JavaScript, XUL, CSS, etc files. + + "binaries" and "faster" almost fully complement each other. However, + there are build actions not captured by either. If things don't appear to + be rebuilding, perform a vanilla `mach build` to rebuild the world. + """ + from mozbuild.controller.building import BuildDriver + + command_context.log_manager.enable_all_structured_loggers() + + loader = MozconfigLoader(command_context.topsrcdir) + mozconfig = loader.read_mozconfig(loader.AUTODETECT) + configure_args = mozconfig["configure_args"] + doing_pgo = configure_args and "MOZ_PGO=1" in configure_args + # Force verbosity on automation. + verbose = verbose or bool(os.environ.get("MOZ_AUTOMATION", False)) + # Keep going by default on automation so that we exhaust as many errors as + # possible. + keep_going = keep_going or bool(os.environ.get("MOZ_AUTOMATION", False)) + append_env = None + + # By setting the current process's priority, by default our child processes + # will also inherit this same priority. + if not _set_priority(priority, verbose): + print("--priority not supported on this platform.") + + if doing_pgo: + if what: + raise Exception("Cannot specify targets (%s) in MOZ_PGO=1 builds" % what) + instr = command_context._spawn(BuildDriver) + orig_topobjdir = instr._topobjdir + instr._topobjdir = mozpath.join(instr._topobjdir, "instrumented") + + append_env = {"MOZ_PROFILE_GENERATE": "1"} + status = instr.build( + command_context.metrics, + what=what, + jobs=jobs, + job_size=job_size, + directory=directory, + verbose=verbose, + keep_going=keep_going, + mach_context=command_context._mach_context, + append_env=append_env, + virtualenv_topobjdir=orig_topobjdir, + ) + if status != 0: + return status + + # Packaging the instrumented build is required to get the jarlog + # data. + status = instr._run_make( + directory=".", + target="package", + silent=not verbose, + ensure_exit_code=False, + append_env=append_env, + ) + if status != 0: + return status + + pgo_env = os.environ.copy() + if instr.config_environment.substs.get("CC_TYPE") in ("clang", "clang-cl"): + pgo_env["LLVM_PROFDATA"] = instr.config_environment.substs.get( + "LLVM_PROFDATA" + ) + pgo_env["JARLOG_FILE"] = mozpath.join(orig_topobjdir, "jarlog/en-US.log") + pgo_cmd = [ + command_context.virtualenv_manager.python_path, + mozpath.join(command_context.topsrcdir, "build/pgo/profileserver.py"), + ] + subprocess.check_call(pgo_cmd, cwd=instr.topobjdir, env=pgo_env) + + # Set the default build to MOZ_PROFILE_USE + append_env = {"MOZ_PROFILE_USE": "1"} + + driver = command_context._spawn(BuildDriver) + return driver.build( + command_context.metrics, + what=what, + jobs=jobs, + job_size=job_size, + directory=directory, + verbose=verbose, + keep_going=keep_going, + mach_context=command_context._mach_context, + append_env=append_env, + ) + + +@Command( + "configure", + category="build", + description="Configure the tree (run configure and config.status).", + metrics_path=MOZBUILD_METRICS_PATH, + virtualenv_name="build", +) +@CommandArgument( + "options", default=None, nargs=argparse.REMAINDER, help="Configure options" +) +def configure( + command_context, + options=None, + buildstatus_messages=False, + line_handler=None, +): + from mozbuild.controller.building import BuildDriver + + command_context.log_manager.enable_all_structured_loggers() + driver = command_context._spawn(BuildDriver) + + return driver.configure( + command_context.metrics, + options=options, + buildstatus_messages=buildstatus_messages, + line_handler=line_handler, + ) + + +@Command( + "resource-usage", + category="post-build", + description="Show information about system resource usage for a build.", + virtualenv_name="build", +) +@CommandArgument( + "--address", + default="localhost", + help="Address the HTTP server should listen on.", +) +@CommandArgument( + "--port", + type=int, + default=0, + help="Port number the HTTP server should listen on.", +) +@CommandArgument( + "--browser", + default="firefox", + help="Web browser to automatically open. See webbrowser Python module.", +) +@CommandArgument("--url", help="URL of JSON document to display") +def resource_usage(command_context, address=None, port=None, browser=None, url=None): + import webbrowser + + from mozbuild.html_build_viewer import BuildViewerServer + + server = BuildViewerServer(address, port) + + if url: + server.add_resource_json_url("url", url) + else: + last = command_context._get_state_filename("build_resources.json") + if not os.path.exists(last): + print( + "Build resources not available. If you have performed a " + "build and receive this message, the psutil Python package " + "likely failed to initialize properly." + ) + return 1 + + server.add_resource_json_file("last", last) + try: + webbrowser.get(browser).open_new_tab(server.url) + except Exception: + print("Cannot get browser specified, trying the default instead.") + try: + browser = webbrowser.get().open_new_tab(server.url) + except Exception: + print("Please open %s in a browser." % server.url) + + print("Hit CTRL+c to stop server.") + server.run() + + +@Command( + "build-backend", + category="build", + description="Generate a backend used to build the tree.", + virtualenv_name="build", +) +@CommandArgument("-d", "--diff", action="store_true", help="Show a diff of changes.") +# It would be nice to filter the choices below based on +# conditions, but that is for another day. +@CommandArgument( + "-b", + "--backend", + nargs="+", + choices=sorted(backends), + help="Which backend to build.", +) +@CommandArgument("-v", "--verbose", action="store_true", help="Verbose output.") +@CommandArgument( + "-n", + "--dry-run", + action="store_true", + help="Do everything except writing files out.", +) +def build_backend(command_context, backend, diff=False, verbose=False, dry_run=False): + python = command_context.virtualenv_manager.python_path + config_status = os.path.join(command_context.topobjdir, "config.status") + + if not os.path.exists(config_status): + print( + "config.status not found. Please run |mach configure| " + "or |mach build| prior to building the %s build backend." % backend + ) + return 1 + + args = [python, config_status] + if backend: + args.append("--backend") + args.extend(backend) + if diff: + args.append("--diff") + if verbose: + args.append("--verbose") + if dry_run: + args.append("--dry-run") + + return command_context._run_command_in_objdir( + args=args, pass_thru=True, ensure_exit_code=False + ) |