diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
commit | 26a029d407be480d791972afb5975cf62c9360a6 (patch) | |
tree | f435a8308119effd964b339f76abb83a57c29483 /testing/raptor/mach_commands.py | |
parent | Initial commit. (diff) | |
download | firefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz firefox-26a029d407be480d791972afb5975cf62c9360a6.zip |
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'testing/raptor/mach_commands.py')
-rw-r--r-- | testing/raptor/mach_commands.py | 458 |
1 files changed, 458 insertions, 0 deletions
diff --git a/testing/raptor/mach_commands.py b/testing/raptor/mach_commands.py new file mode 100644 index 0000000000..b1c5002efc --- /dev/null +++ b/testing/raptor/mach_commands.py @@ -0,0 +1,458 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Originally taken from /talos/mach_commands.py + +# Integrates raptor mozharness with mach + +import json +import logging +import os +import socket +import subprocess +import sys + +from mach.decorators import Command +from mach.util import get_state_dir +from mozbuild.base import BinaryNotFoundException, MozbuildObject +from mozbuild.base import MachCommandConditions as Conditions + +HERE = os.path.dirname(os.path.realpath(__file__)) + +ANDROID_BROWSERS = ["geckoview", "refbrow", "fenix", "chrome-m", "cstm-car-m"] + + +class RaptorRunner(MozbuildObject): + def run_test(self, raptor_args, kwargs): + """Setup and run mozharness. + + We want to do a few things before running Raptor: + + 1. Clone mozharness + 2. Make the config for Raptor mozharness + 3. Run mozharness + """ + # Validate that the user is using a supported python version before doing anything else + max_py_major, max_py_minor = 3, 11 + sys_maj, sys_min = sys.version_info.major, sys.version_info.minor + if sys_min > max_py_minor: + raise PythonVersionException( + print( + f"\tPlease downgrade your Python version as Raptor does not yet support Python " + f"versions greater than {max_py_major}.{max_py_minor}." + f"\n\tYou seem to currently be using Python {sys_maj}.{sys_min}." + f"\n\tSee here for a possible solution in debugging your python environment: " + f"https://firefox-source-docs.mozilla.org/testing/perfdocs/" + f"debugging.html#debugging-local-python-environment" + ) + ) + self.init_variables(raptor_args, kwargs) + self.make_config() + self.write_config() + self.make_args() + return self.run_mozharness() + + def init_variables(self, raptor_args, kwargs): + self.raptor_args = raptor_args + + if kwargs.get("host") == "HOST_IP": + kwargs["host"] = os.environ["HOST_IP"] + self.host = kwargs["host"] + self.is_release_build = kwargs["is_release_build"] + self.live_sites = kwargs["live_sites"] + self.disable_perf_tuning = kwargs["disable_perf_tuning"] + self.conditioned_profile = kwargs["conditioned_profile"] + self.device_name = kwargs["device_name"] + self.enable_marionette_trace = kwargs["enable_marionette_trace"] + self.browsertime_visualmetrics = kwargs["browsertime_visualmetrics"] + self.browsertime_node = kwargs["browsertime_node"] + self.clean = kwargs["clean"] + + if Conditions.is_android(self) or kwargs["app"] in ANDROID_BROWSERS: + self.binary_path = None + else: + self.binary_path = kwargs.get("binary") or self.get_binary_path() + + self.python = sys.executable + + self.raptor_dir = os.path.join(self.topsrcdir, "testing", "raptor") + self.mozharness_dir = os.path.join(self.topsrcdir, "testing", "mozharness") + self.config_file_path = os.path.join( + self._topobjdir, "testing", "raptor-in_tree_conf.json" + ) + + self.virtualenv_script = os.path.join( + self.topsrcdir, "third_party", "python", "virtualenv", "virtualenv.py" + ) + self.virtualenv_path = os.path.join(self._topobjdir, "testing", "raptor-venv") + + def make_config(self): + default_actions = [ + "populate-webroot", + "create-virtualenv", + "install-chromium-distribution", + "run-tests", + ] + self.config = { + "run_local": True, + "binary_path": self.binary_path, + "repo_path": self.topsrcdir, + "raptor_path": self.raptor_dir, + "obj_path": self.topobjdir, + "log_name": "raptor", + "virtualenv_path": self.virtualenv_path, + "pypi_url": "http://pypi.org/simple", + "base_work_dir": self.mozharness_dir, + "exes": { + "python": self.python, + "virtualenv": [self.python, self.virtualenv_script], + }, + "title": socket.gethostname(), + "default_actions": default_actions, + "raptor_cmd_line_args": self.raptor_args, + "host": self.host, + "live_sites": self.live_sites, + "disable_perf_tuning": self.disable_perf_tuning, + "conditioned_profile": self.conditioned_profile, + "is_release_build": self.is_release_build, + "device_name": self.device_name, + "enable_marionette_trace": self.enable_marionette_trace, + "browsertime_visualmetrics": self.browsertime_visualmetrics, + "browsertime_node": self.browsertime_node, + "mozbuild_path": get_state_dir(), + "clean": self.clean, + } + + sys.path.insert(0, os.path.join(self.topsrcdir, "tools", "browsertime")) + try: + import platform + + import mach_commands as browsertime + + # We don't set `browsertime_{chromedriver,geckodriver} -- those will be found by + # browsertime in its `node_modules` directory, which is appropriate for local builds. + # We don't set `browsertime_ffmpeg` yet: it will need to be on the path. There is code + # to configure the environment including the path in + # `tools/browsertime/mach_commands.py` but integrating it here will take more effort. + self.config.update( + { + "browsertime_browsertimejs": browsertime.browsertime_path(), + "browsertime_vismet_script": browsertime.visualmetrics_path(), + } + ) + + def _get_browsertime_package(): + with open( + os.path.join( + self.topsrcdir, + "tools", + "browsertime", + "node_modules", + "browsertime", + "package.json", + ) + ) as package: + return json.load(package) + + def _get_browsertime_resolved(): + try: + with open( + os.path.join( + self.topsrcdir, + "tools", + "browsertime", + "node_modules", + ".package-lock.json", + ) + ) as package_lock: + return json.load(package_lock)["packages"][ + "node_modules/browsertime" + ]["resolved"] + except FileNotFoundError: + # Older versions of node/npm add this metadata to package.json + return _get_browsertime_package()["_from"] + + def _should_install(): + # If ffmpeg doesn't exist in the .mozbuild directory, + # then we should install + btime_cache = os.path.join(self.config["mozbuild_path"], "browsertime") + if not os.path.exists(btime_cache) or not any( + ["ffmpeg" in cache_dir for cache_dir in os.listdir(btime_cache)] + ): + return True + + # If browsertime doesn't exist, install it + if not os.path.exists( + self.config["browsertime_browsertimejs"] + ) or not os.path.exists(self.config["browsertime_vismet_script"]): + return True + + # Browsertime exists, check if it's outdated + with open( + os.path.join(self.topsrcdir, "tools", "browsertime", "package.json") + ) as new: + new_pkg = json.load(new) + + return not _get_browsertime_resolved().endswith( + new_pkg["devDependencies"]["browsertime"] + ) + + def _get_browsertime_version(): + # Returns the (version number, current commit) used + return ( + _get_browsertime_package()["version"], + _get_browsertime_resolved(), + ) + + # Check if browsertime scripts exist and try to install them if + # they aren't + if _should_install(): + # TODO: Make this "integration" nicer in the near future + print("Missing browsertime files...attempting to install") + subprocess.check_output( + [ + os.path.join(self.topsrcdir, "mach"), + "browsertime", + "--setup", + "--clobber", + ], + shell="windows" in platform.system().lower(), + ) + if _should_install(): + raise Exception( + "Failed installation attempt. Cannot find browsertime scripts. " + "Run `./mach browsertime --setup --clobber` to set it up." + ) + + # Bug 1766112 - For the time being, we need to trigger a + # clean build to upgrade browsertime. This should be disabled + # after some time. + print( + "Setting --clean to True to rebuild Python " + "environment for Browsertime upgrade..." + ) + self.config["clean"] = True + + print("Using browsertime version %s from %s" % _get_browsertime_version()) + + finally: + sys.path = sys.path[1:] + + def make_args(self): + self.args = { + "config": {}, + "initial_config_file": self.config_file_path, + } + + def write_config(self): + try: + config_file = open(self.config_file_path, "w") + config_file.write(json.dumps(self.config)) + config_file.close() + except IOError as e: + err_str = "Error writing to Raptor Mozharness config file {0}:{1}" + print(err_str.format(self.config_file_path, str(e))) + raise e + + def run_mozharness(self): + sys.path.insert(0, self.mozharness_dir) + from mozharness.mozilla.testing.raptor import Raptor + + raptor_mh = Raptor( + config=self.args["config"], + initial_config_file=self.args["initial_config_file"], + ) + return raptor_mh.run() + + +def setup_node(command_context): + """Fetch the latest node-16 binary and install it into the .mozbuild directory.""" + import platform + + from mozbuild.artifact_commands import artifact_toolchain + from mozbuild.nodeutil import find_node_executable + from packaging.version import Version + + print("Setting up node for browsertime...") + state_dir = get_state_dir() + cache_path = os.path.join(state_dir, "browsertime", "node-16") + + def __check_for_node(): + # Check standard locations first + node_exe = find_node_executable(min_version=Version("16.0.0")) + if node_exe and (node_exe[0] is not None): + return node_exe[0] + if not os.path.exists(cache_path): + return None + + # Check the browsertime-specific node location next + node_name = "node" + if platform.system() == "Windows": + node_name = "node.exe" + node_exe_path = os.path.join( + state_dir, + "browsertime", + "node-16", + "node", + ) + else: + node_exe_path = os.path.join( + state_dir, + "browsertime", + "node-16", + "node", + "bin", + ) + + node_exe = os.path.join(node_exe_path, node_name) + if not os.path.exists(node_exe): + return None + + return node_exe + + node_exe = __check_for_node() + if node_exe is None: + toolchain_job = "{}-node-16" + plat = platform.system() + if plat == "Windows": + toolchain_job = toolchain_job.format("win64") + elif plat == "Darwin": + if platform.processor() == "arm": + toolchain_job = toolchain_job.format("macosx64-aarch64") + else: + toolchain_job = toolchain_job.format("macosx64") + else: + toolchain_job = toolchain_job.format("linux64") + + print( + "Downloading Node v16 from Taskcluster toolchain {}...".format( + toolchain_job + ) + ) + + if not os.path.exists(cache_path): + os.makedirs(cache_path, exist_ok=True) + + # Change directories to where node should be installed + # before installing. Otherwise, it gets installed in the + # top level of the repo (or the current working directory). + cur_dir = os.getcwd() + os.chdir(cache_path) + artifact_toolchain( + command_context, + verbose=False, + from_build=[toolchain_job], + no_unpack=False, + retry=0, + cache_dir=cache_path, + ) + os.chdir(cur_dir) + + node_exe = __check_for_node() + if node_exe is None: + raise Exception("Could not find Node v16 binary for Raptor-Browsertime") + + print("Finished downloading Node v16 from Taskcluster") + + print("Node v16+ found at: %s" % node_exe) + return node_exe + + +def create_parser(): + sys.path.insert(0, HERE) # allow to import the raptor package + from raptor.cmdline import create_parser + + return create_parser(mach_interface=True) + + +@Command( + "raptor", + category="testing", + description="Run Raptor performance tests.", + parser=create_parser, +) +def run_raptor(command_context, **kwargs): + build_obj = command_context + + # Setup node for browsertime + kwargs["browsertime_node"] = setup_node(command_context) + + is_android = Conditions.is_android(build_obj) or kwargs["app"] in ANDROID_BROWSERS + + if is_android: + from mozrunner.devices.android_device import ( + InstallIntent, + verify_android_device, + ) + + install = ( + InstallIntent.NO if kwargs.pop("noinstall", False) else InstallIntent.YES + ) + verbose = False + if ( + kwargs.get("log_mach_verbose") + or kwargs.get("log_tbpl_level") == "debug" + or kwargs.get("log_mach_level") == "debug" + or kwargs.get("log_raw_level") == "debug" + ): + verbose = True + if not verify_android_device( + build_obj, + install=install, + app=kwargs["binary"], + verbose=verbose, + xre=True, + ): # Equivalent to 'run_local' = True. + print( + "****************************************************************************" + ) + print( + "Unable to verify device, please check your attached/connected android device" + ) + print( + "****************************************************************************" + ) + return 1 + # Disable fission until geckoview supports fission by default. + # Need fission on Android? Use '--setpref fission.autostart=true' + kwargs["fission"] = False + + # Remove mach global arguments from sys.argv to prevent them + # from being consumed by raptor. Treat any item in sys.argv + # occuring before "raptor" as a mach global argument. + argv = [] + in_mach = True + for arg in sys.argv: + if not in_mach: + argv.append(arg) + if arg.startswith("raptor"): + in_mach = False + + raptor = command_context._spawn(RaptorRunner) + + try: + return raptor.run_test(argv, kwargs) + except BinaryNotFoundException as e: + command_context.log( + logging.ERROR, "raptor", {"error": str(e)}, "ERROR: {error}" + ) + command_context.log(logging.INFO, "raptor", {"help": e.help()}, "{help}") + return 1 + except Exception as e: + print(repr(e)) + return 1 + + +@Command( + "raptor-test", + category="testing", + description="Run Raptor performance tests.", + parser=create_parser, +) +def run_raptor_test(command_context, **kwargs): + return run_raptor(command_context, **kwargs) + + +class PythonVersionException(Exception): + pass |