diff options
Diffstat (limited to 'testing/mozharness/mozharness/mozilla/testing/raptor.py')
| -rw-r--r-- | testing/mozharness/mozharness/mozilla/testing/raptor.py | 1478 |
1 files changed, 1478 insertions, 0 deletions
diff --git a/testing/mozharness/mozharness/mozilla/testing/raptor.py b/testing/mozharness/mozharness/mozilla/testing/raptor.py new file mode 100644 index 0000000000..ceb97da963 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py @@ -0,0 +1,1478 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import copy +import glob +import multiprocessing +import os +import pathlib +import re +import subprocess +import sys +import tempfile +from shutil import copyfile, rmtree + +from six import string_types + +import mozharness +from mozharness.base.errors import PythonErrorList +from mozharness.base.log import CRITICAL, DEBUG, ERROR, INFO, OutputParser +from mozharness.base.python import Python3Virtualenv +from mozharness.base.vcs.vcsbase import MercurialScript +from mozharness.mozilla.automation import ( + EXIT_STATUS_DICT, + TBPL_RETRY, + TBPL_SUCCESS, + TBPL_WORST_LEVEL_TUPLE, +) +from mozharness.mozilla.testing.android import AndroidMixin +from mozharness.mozilla.testing.codecoverage import ( + CodeCoverageMixin, + code_coverage_config_options, +) +from mozharness.mozilla.testing.errors import HarnessErrorList, TinderBoxPrintRe +from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options + +scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))) +external_tools_path = os.path.join(scripts_path, "external_tools") +here = os.path.abspath(os.path.dirname(__file__)) + +RaptorErrorList = ( + PythonErrorList + + HarnessErrorList + + [ + {"regex": re.compile(r"""run-as: Package '.*' is unknown"""), "level": DEBUG}, + {"substr": r"""raptorDebug""", "level": DEBUG}, + { + "regex": re.compile(r"""^raptor[a-zA-Z-]*( - )?( )?(?i)error(:)?"""), + "level": ERROR, + }, + { + "regex": re.compile(r"""^raptor[a-zA-Z-]*( - )?( )?(?i)critical(:)?"""), + "level": CRITICAL, + }, + { + "regex": re.compile(r"""No machine_name called '.*' can be found"""), + "level": CRITICAL, + }, + { + "substr": r"""No such file or directory: 'browser_output.txt'""", + "level": CRITICAL, + "explanation": "Most likely the browser failed to launch, or the test otherwise " + "failed to start.", + }, + ] +) + +# When running raptor locally, we can attempt to make use of +# the users locally cached ffmpeg binary from from when the user +# ran `./mach browsertime --setup` +FFMPEG_LOCAL_CACHE = { + "mac": "ffmpeg-macos", + "linux": "ffmpeg-4.4.1-i686-static", + "win": "ffmpeg-4.4.1-full_build", +} + + +class Raptor( + TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin, Python3Virtualenv +): + """ + Install and run Raptor tests + """ + + # Options to Browsertime. Paths are expected to be absolute. + browsertime_options = [ + [ + ["--browsertime-node"], + {"dest": "browsertime_node", "default": None, "help": argparse.SUPPRESS}, + ], + [ + ["--browsertime-browsertimejs"], + { + "dest": "browsertime_browsertimejs", + "default": None, + "help": argparse.SUPPRESS, + }, + ], + [ + ["--browsertime-vismet-script"], + { + "dest": "browsertime_vismet_script", + "default": None, + "help": argparse.SUPPRESS, + }, + ], + [ + ["--browsertime-chromedriver"], + { + "dest": "browsertime_chromedriver", + "default": None, + "help": argparse.SUPPRESS, + }, + ], + [ + ["--browsertime-ffmpeg"], + {"dest": "browsertime_ffmpeg", "default": None, "help": argparse.SUPPRESS}, + ], + [ + ["--browsertime-geckodriver"], + { + "dest": "browsertime_geckodriver", + "default": None, + "help": argparse.SUPPRESS, + }, + ], + [ + ["--browsertime-video"], + { + "dest": "browsertime_video", + "action": "store_true", + "default": False, + "help": argparse.SUPPRESS, + }, + ], + [ + ["--browsertime-visualmetrics"], + { + "dest": "browsertime_visualmetrics", + "action": "store_true", + "default": False, + "help": argparse.SUPPRESS, + }, + ], + [ + ["--browsertime-no-ffwindowrecorder"], + { + "dest": "browsertime_no_ffwindowrecorder", + "action": "store_true", + "default": False, + "help": argparse.SUPPRESS, + }, + ], + [ + ["--browsertime-arg"], + { + "action": "append", + "metavar": "PREF=VALUE", + "dest": "browsertime_user_args", + "default": [], + "help": argparse.SUPPRESS, + }, + ], + [ + ["--browsertime"], + { + "dest": "browsertime", + "action": "store_true", + "default": True, + "help": argparse.SUPPRESS, + }, + ], + ] + + config_options = ( + [ + [ + ["--test"], + {"action": "store", "dest": "test", "help": "Raptor test to run"}, + ], + [ + ["--app"], + { + "default": "firefox", + "choices": [ + "firefox", + "chrome", + "chrome-m", + "chromium", + "fennec", + "geckoview", + "refbrow", + "fenix", + "safari", + "custom-car", + ], + "dest": "app", + "help": "Name of the application we are testing (default: firefox).", + }, + ], + [ + ["--activity"], + { + "dest": "activity", + "help": "The Android activity used to launch the Android app. " + "e.g.: org.mozilla.fenix.browser.BrowserPerformanceTestActivity", + }, + ], + [ + ["--intent"], + { + "dest": "intent", + "help": "Name of the Android intent action used to launch the Android app", + }, + ], + [ + ["--is-release-build"], + { + "action": "store_true", + "dest": "is_release_build", + "help": "Whether the build is a release build which requires work arounds " + "using MOZ_DISABLE_NONLOCAL_CONNECTIONS to support installing unsigned " + "webextensions. Defaults to False.", + }, + ], + [ + ["--add-option"], + { + "action": "extend", + "dest": "raptor_cmd_line_args", + "default": None, + "help": "Extra options to Raptor.", + }, + ], + [ + ["--device-name"], + { + "dest": "device_name", + "default": None, + "help": "Device name of mobile device.", + }, + ], + [ + ["--geckoProfile"], + { + "dest": "gecko_profile", + "action": "store_true", + "default": False, + "help": argparse.SUPPRESS, + }, + ], + [ + ["--geckoProfileInterval"], + { + "dest": "gecko_profile_interval", + "type": "int", + "help": argparse.SUPPRESS, + }, + ], + [ + ["--geckoProfileEntries"], + { + "dest": "gecko_profile_entries", + "type": "int", + "help": argparse.SUPPRESS, + }, + ], + [ + ["--geckoProfileFeatures"], + { + "dest": "gecko_profile_features", + "type": "str", + "help": argparse.SUPPRESS, + }, + ], + [ + ["--gecko-profile"], + { + "dest": "gecko_profile", + "action": "store_true", + "default": False, + "help": "Whether to profile the test run and save the profile results.", + }, + ], + [ + ["--gecko-profile-interval"], + { + "dest": "gecko_profile_interval", + "type": "int", + "help": "The interval between samples taken by the profiler (ms).", + }, + ], + [ + ["--gecko-profile-entries"], + { + "dest": "gecko_profile_entries", + "type": "int", + "help": "How many samples to take with the profiler.", + }, + ], + [ + ["--gecko-profile-threads"], + { + "dest": "gecko_profile_threads", + "type": "str", + "help": "Comma-separated list of threads to sample.", + }, + ], + [ + ["--gecko-profile-features"], + { + "dest": "gecko_profile_features", + "type": "str", + "help": "Features to enable in the profiler.", + }, + ], + [ + ["--extra-profiler-run"], + { + "dest": "extra_profiler_run", + "action": "store_true", + "default": False, + "help": "Run the tests again with profiler enabled after the main run.", + }, + ], + [ + ["--page-cycles"], + { + "dest": "page_cycles", + "type": "int", + "help": ( + "How many times to repeat loading the test page (for page load " + "tests); for benchmark tests this is how many times the benchmark test " + "will be run." + ), + }, + ], + [ + ["--page-timeout"], + { + "dest": "page_timeout", + "type": "int", + "help": "How long to wait (ms) for one page_cycle to complete, before timing out.", # NOQA: E501 + }, + ], + [ + ["--browser-cycles"], + { + "dest": "browser_cycles", + "type": "int", + "help": ( + "The number of times a cold load test is repeated (for cold load tests " + "only, where the browser is shutdown and restarted between test " + "iterations)." + ), + }, + ], + [ + ["--project"], + { + "action": "store", + "dest": "project", + "default": "mozilla-central", + "type": "str", + "help": "Name of the project (try, mozilla-central, etc.)", + }, + ], + [ + ["--test-url-params"], + { + "action": "store", + "dest": "test_url_params", + "help": "Parameters to add to the test_url query string.", + }, + ], + [ + ["--host"], + { + "dest": "host", + "type": "str", + "default": "127.0.0.1", + "help": "Hostname from which to serve urls (default: 127.0.0.1). " + "The value HOST_IP will cause the value of host to be " + "to be loaded from the environment variable HOST_IP.", + }, + ], + [ + ["--power-test"], + { + "dest": "power_test", + "action": "store_true", + "default": False, + "help": ( + "Use Raptor to measure power usage on Android browsers (Geckoview " + "Example, Fenix, Refbrow, and Fennec) as well as on Intel-based MacOS " + "machines that have Intel Power Gadget installed." + ), + }, + ], + [ + ["--memory-test"], + { + "dest": "memory_test", + "action": "store_true", + "default": False, + "help": "Use Raptor to measure memory usage.", + }, + ], + [ + ["--cpu-test"], + { + "dest": "cpu_test", + "action": "store_true", + "default": False, + "help": "Use Raptor to measure CPU usage.", + }, + ], + [ + ["--disable-perf-tuning"], + { + "action": "store_true", + "dest": "disable_perf_tuning", + "default": False, + "help": "Disable performance tuning on android.", + }, + ], + [ + ["--conditioned-profile"], + { + "dest": "conditioned_profile", + "type": "str", + "default": None, + "help": ( + "Name of conditioned profile to use. Prefix with `artifact:` " + "if we should obtain the profile from CI.", + ), + }, + ], + [ + ["--live-sites"], + { + "dest": "live_sites", + "action": "store_true", + "default": False, + "help": "Run tests using live sites instead of recorded sites.", + }, + ], + [ + ["--test-bytecode-cache"], + { + "dest": "test_bytecode_cache", + "action": "store_true", + "default": False, + "help": ( + "If set, the pageload test will set the preference " + "`dom.script_loader.bytecode_cache.strategy=-1` and wait 20 seconds " + "after the first cold pageload to populate the bytecode cache before " + "running a warm pageload test. Only available if `--chimera` " + "is also provided." + ), + }, + ], + [ + ["--chimera"], + { + "dest": "chimera", + "action": "store_true", + "default": False, + "help": "Run tests in chimera mode. Each browser cycle will run a cold and warm test.", # NOQA: E501 + }, + ], + [ + ["--debug-mode"], + { + "dest": "debug_mode", + "action": "store_true", + "default": False, + "help": "Run Raptor in debug mode (open browser console, limited page-cycles, etc.)", # NOQA: E501 + }, + ], + [ + ["--noinstall"], + { + "dest": "noinstall", + "action": "store_true", + "default": False, + "help": "Do not offer to install Android APK.", + }, + ], + [ + ["--disable-e10s"], + { + "dest": "e10s", + "action": "store_false", + "default": True, + "help": "Run without multiple processes (e10s).", + }, + ], + [ + ["--disable-fission"], + { + "action": "store_false", + "dest": "fission", + "default": True, + "help": "Disable Fission (site isolation) in Gecko.", + }, + ], + [ + ["--setpref"], + { + "action": "append", + "metavar": "PREF=VALUE", + "dest": "extra_prefs", + "default": [], + "help": "Set a browser preference. May be used multiple times.", + }, + ], + [ + ["--setenv"], + { + "action": "append", + "metavar": "NAME=VALUE", + "dest": "environment", + "default": [], + "help": "Set a variable in the test environment. May be used multiple times.", + }, + ], + [ + ["--skip-preflight"], + { + "action": "store_true", + "dest": "skip_preflight", + "default": False, + "help": "skip preflight commands to prepare machine.", + }, + ], + [ + ["--cold"], + { + "action": "store_true", + "dest": "cold", + "default": False, + "help": "Enable cold page-load for browsertime tp6", + }, + ], + [ + ["--verbose"], + { + "action": "store_true", + "dest": "verbose", + "default": False, + "help": "Verbose output", + }, + ], + [ + ["--enable-marionette-trace"], + { + "action": "store_true", + "dest": "enable_marionette_trace", + "default": False, + "help": "Enable marionette tracing", + }, + ], + [ + ["--clean"], + { + "action": "store_true", + "dest": "clean", + "default": False, + "help": ( + "Clean the python virtualenv (remove, and rebuild) for " + "Raptor before running tests." + ), + }, + ], + [ + ["--webext"], + { + "action": "store_true", + "dest": "webext", + "default": False, + "help": ( + "Whether to use webextension to execute pageload tests " + "(WebExtension is being deprecated).", + ), + }, + ], + [ + ["--collect-perfstats"], + { + "action": "store_true", + "dest": "collect_perfstats", + "default": False, + "help": ( + "If set, the test will collect perfstats in addition to " + "the regular metrics it gathers." + ), + }, + ], + [ + ["--extra-summary-methods"], + { + "action": "append", + "metavar": "OPTION", + "dest": "extra_summary_methods", + "default": [], + "help": ( + "Alternative methods for summarizing technical and visual" + "pageload metrics." + "Options: geomean, mean." + ), + }, + ], + [ + ["--benchmark-repository"], + { + "dest": "benchmark_repository", + "type": "str", + "default": None, + "help": ( + "Repository that should be used for a particular benchmark test. " + "e.g. https://github.com/mozilla-mobile/firefox-android" + ), + }, + ], + [ + ["--benchmark-revision"], + { + "dest": "benchmark_revision", + "type": "str", + "default": None, + "help": ( + "Repository revision that should be used for a particular " + "benchmark test." + ), + }, + ], + [ + ["--benchmark-branch"], + { + "dest": "benchmark_branch", + "type": "str", + "default": None, + "help": ( + "Repository branch that should be used for a particular benchmark test." + ), + }, + ], + ] + + testing_config_options + + copy.deepcopy(code_coverage_config_options) + + browsertime_options + ) + + def __init__(self, **kwargs): + kwargs.setdefault("config_options", self.config_options) + kwargs.setdefault( + "all_actions", + [ + "clobber", + "download-and-extract", + "populate-webroot", + "create-virtualenv", + "install-chrome-android", + "install-chromium-distribution", + "install", + "run-tests", + ], + ) + kwargs.setdefault( + "default_actions", + [ + "clobber", + "download-and-extract", + "populate-webroot", + "create-virtualenv", + "install-chromium-distribution", + "install", + "run-tests", + ], + ) + kwargs.setdefault("config", {}) + super(Raptor, self).__init__(**kwargs) + + # Convenience + self.workdir = self.query_abs_dirs()["abs_work_dir"] + + self.run_local = self.config.get("run_local") + + # App (browser testing on) defaults to firefox + self.app = "firefox" + + if self.run_local: + # Get app from command-line args, passed in from mach, inside 'raptor_cmd_line_args' + # Command-line args can be in two formats depending on how the user entered them + # i.e. "--app=geckoview" or separate as "--app", "geckoview" so we have to + # parse carefully. It's simplest to use `argparse` to parse partially. + self.app = "firefox" + if "raptor_cmd_line_args" in self.config: + sub_parser = argparse.ArgumentParser() + # It's not necessary to limit the allowed values: each value + # will be parsed and verifed by raptor/raptor.py. + sub_parser.add_argument("--app", default=None, dest="app") + sub_parser.add_argument("-i", "--intent", default=None, dest="intent") + sub_parser.add_argument( + "-a", "--activity", default=None, dest="activity" + ) + + # We'd prefer to use `parse_known_intermixed_args`, but that's + # new in Python 3.7. + known, unknown = sub_parser.parse_known_args( + self.config["raptor_cmd_line_args"] + ) + + if known.app: + self.app = known.app + if known.intent: + self.intent = known.intent + if known.activity: + self.activity = known.activity + else: + # Raptor initiated in production via mozharness + self.test = self.config["test"] + self.app = self.config.get("app", "firefox") + self.binary_path = self.config.get("binary_path", None) + + if self.app in ("refbrow", "fenix"): + self.app_name = self.binary_path + + self.installer_url = self.config.get("installer_url") + self.raptor_json_url = self.config.get("raptor_json_url") + self.raptor_json = self.config.get("raptor_json") + self.raptor_json_config = self.config.get("raptor_json_config") + self.repo_path = self.config.get("repo_path") + self.obj_path = self.config.get("obj_path") + self.mozbuild_path = self.config.get("mozbuild_path") + self.test = None + self.gecko_profile = self.config.get( + "gecko_profile" + ) or "--geckoProfile" in self.config.get("raptor_cmd_line_args", []) + self.gecko_profile_interval = self.config.get("gecko_profile_interval") + self.gecko_profile_entries = self.config.get("gecko_profile_entries") + self.gecko_profile_threads = self.config.get("gecko_profile_threads") + self.gecko_profile_features = self.config.get("gecko_profile_features") + self.extra_profiler_run = self.config.get("extra_profiler_run") + self.test_packages_url = self.config.get("test_packages_url") + self.test_url_params = self.config.get("test_url_params") + self.host = self.config.get("host") + if self.host == "HOST_IP": + self.host = os.environ["HOST_IP"] + self.power_test = self.config.get("power_test") + self.memory_test = self.config.get("memory_test") + self.cpu_test = self.config.get("cpu_test") + self.live_sites = self.config.get("live_sites") + self.chimera = self.config.get("chimera") + self.disable_perf_tuning = self.config.get("disable_perf_tuning") + self.conditioned_profile = self.config.get("conditioned_profile") + self.extra_prefs = self.config.get("extra_prefs") + self.environment = self.config.get("environment") + self.is_release_build = self.config.get("is_release_build") + self.debug_mode = self.config.get("debug_mode", False) + self.chromium_dist_path = None + self.firefox_android_browsers = ["fennec", "geckoview", "refbrow", "fenix"] + self.android_browsers = self.firefox_android_browsers + ["chrome-m"] + self.browsertime_visualmetrics = self.config.get("browsertime_visualmetrics") + self.browsertime_node = self.config.get("browsertime_node") + self.browsertime_user_args = self.config.get("browsertime_user_args") + self.browsertime_video = False + self.enable_marionette_trace = self.config.get("enable_marionette_trace") + self.browser_cycles = self.config.get("browser_cycles") + self.clean = self.config.get("clean") + + for (arg,), details in Raptor.browsertime_options: + # Allow overriding defaults on the `./mach raptor-test ...` command-line. + value = self.config.get(details["dest"]) + if value and arg not in self.config.get("raptor_cmd_line_args", []): + setattr(self, details["dest"], value) + + # We accept some configuration options from the try commit message in the + # format mozharness: <options>. Example try commit message: mozharness: + # --geckoProfile try: <stuff> + def query_gecko_profile_options(self): + gecko_results = [] + # If gecko_profile is set, we add that to Raptor's options + if self.gecko_profile: + gecko_results.append("--gecko-profile") + if self.gecko_profile_interval: + gecko_results.extend( + ["--gecko-profile-interval", str(self.gecko_profile_interval)] + ) + if self.gecko_profile_entries: + gecko_results.extend( + ["--gecko-profile-entries", str(self.gecko_profile_entries)] + ) + if self.gecko_profile_features: + gecko_results.extend( + ["--gecko-profile-features", self.gecko_profile_features] + ) + if self.gecko_profile_threads: + gecko_results.extend( + ["--gecko-profile-threads", self.gecko_profile_threads] + ) + else: + if self.extra_profiler_run: + gecko_results.append("--extra-profiler-run") + return gecko_results + + def query_abs_dirs(self): + if self.abs_dirs: + return self.abs_dirs + abs_dirs = super(Raptor, self).query_abs_dirs() + abs_dirs["abs_blob_upload_dir"] = os.path.join( + abs_dirs["abs_work_dir"], "blobber_upload_dir" + ) + abs_dirs["abs_test_install_dir"] = os.path.join( + abs_dirs["abs_work_dir"], "tests" + ) + + self.abs_dirs = abs_dirs + return self.abs_dirs + + def install_chrome_android(self): + """Install Google Chrome for Android in production from tooltool""" + if self.app != "chrome-m": + self.info("Google Chrome for Android not required") + return + if self.config.get("run_local"): + self.info( + "Google Chrome for Android will not be installed " + "from tooltool when running locally" + ) + return + self.info("Fetching and installing Google Chrome for Android") + self.device.shell_output("cmd package install-existing com.android.chrome") + self.info("Google Chrome for Android successfully installed") + + def download_chrome_android(self): + # Fetch the APK + tmpdir = tempfile.mkdtemp() + self.tooltool_fetch( + os.path.join( + self.raptor_path, + "raptor", + "tooltool-manifests", + "chrome-android", + "chrome87.manifest", + ), + output_dir=tmpdir, + ) + files = os.listdir(tmpdir) + if len(files) > 1: + raise Exception( + "Found more than one chrome APK file after tooltool download" + ) + chromeapk = os.path.join(tmpdir, files[0]) + + # Disable verification and install the APK + self.device.shell_output("settings put global verifier_verify_adb_installs 0") + self.install_android_app(chromeapk, replace=True) + + # Re-enable verification and delete the temporary directory + self.device.shell_output("settings put global verifier_verify_adb_installs 1") + rmtree(tmpdir) + + def install_chromium_distribution(self): + """Install Google Chromium distribution in production""" + linux, mac, win = "linux", "mac", "win" + chrome, chromium, chromium_release = "chrome", "chromium", "custom-car" + + available_chromium_dists = [chrome, chromium, chromium_release] + binary_location = { + chromium: { + linux: ["chrome-linux", "chrome"], + mac: ["chrome-mac", "Chromium.app", "Contents", "MacOS", "Chromium"], + win: ["chrome-win", "Chrome.exe"], + }, + chromium_release: { + linux: ["chromium", "Default", "chrome"], + win: ["chromium", "Default", "chrome.exe"], + }, + } + + if self.app not in available_chromium_dists: + self.info("Google Chrome or Chromium distributions are not required.") + return + + if self.app == "chrome": + self.info("Chrome should be preinstalled.") + if win in self.platform_name(): + base_path = "C:\\%s\\Google\\Chrome\\Application\\chrome.exe" + self.chromium_dist_path = base_path % "Progra~1" + if not os.path.exists(self.chromium_dist_path): + self.chromium_dist_path = base_path % "Progra~2" + elif linux in self.platform_name(): + self.chromium_dist_path = "/usr/bin/google-chrome" + elif mac in self.platform_name(): + self.chromium_dist_path = ( + "/Applications/Google Chrome.app/" "Contents/MacOS/Google Chrome" + ) + else: + self.error( + "Chrome is not installed on the platform %s yet." + % self.platform_name() + ) + + if os.path.exists(self.chromium_dist_path): + self.info( + "Google Chrome found in expected location %s" + % self.chromium_dist_path + ) + else: + self.error("Cannot find Google Chrome at %s" % self.chromium_dist_path) + + return + + chromium_dist = self.app + + if self.config.get("run_local"): + self.info("Expecting %s to be pre-installed locally" % chromium_dist) + return + + self.info("Getting fetched %s build" % chromium_dist) + self.chromium_dist_dest = os.path.normpath( + os.path.abspath(os.environ["MOZ_FETCHES_DIR"]) + ) + + if mac in self.platform_name(): + self.chromium_dist_path = os.path.join( + self.chromium_dist_dest, *binary_location[chromium_dist][mac] + ) + + elif linux in self.platform_name(): + self.chromium_dist_path = os.path.join( + self.chromium_dist_dest, *binary_location[chromium_dist][linux] + ) + + else: + self.chromium_dist_path = os.path.join( + self.chromium_dist_dest, *binary_location[chromium_dist][win] + ) + + self.info("%s dest is: %s" % (chromium_dist, self.chromium_dist_dest)) + self.info("%s path is: %s" % (chromium_dist, self.chromium_dist_path)) + + # Now ensure Chromium binary exists + if os.path.exists(self.chromium_dist_path): + self.info( + "Successfully installed %s to: %s" + % (chromium_dist, self.chromium_dist_path) + ) + else: + self.info("Abort: failed to install %s" % chromium_dist) + + def raptor_options(self, args=None, **kw): + """Return options to Raptor""" + options = [] + kw_options = {} + + # Get the APK location to be able to get the browser version + # through mozversion + if self.app in self.firefox_android_browsers and not self.run_local: + kw_options["installerpath"] = self.installer_path + + # If testing on Firefox, the binary path already came from mozharness/pro; + # otherwise the binary path is forwarded from command-line arg (raptor_cmd_line_args). + kw_options["app"] = self.app + if self.app == "firefox" or ( + self.app in self.firefox_android_browsers and not self.run_local + ): + binary_path = self.binary_path or self.config.get("binary_path") + if not binary_path: + self.fatal("Raptor requires a path to the binary.") + kw_options["binary"] = binary_path + if self.app in self.firefox_android_browsers: + # In production ensure we have correct app name, + # i.e. fennec_aurora or fennec_release etc. + kw_options["binary"] = self.query_package_name() + self.info( + "Set binary to %s instead of %s" + % (kw_options["binary"], binary_path) + ) + elif self.app == "safari" and not self.run_local: + binary_path = "/Applications/Safari.app/Contents/MacOS/Safari" + kw_options["binary"] = binary_path + else: # Running on Chromium + if not self.run_local: + # When running locally we already set the Chromium binary above, in init. + # In production, we already installed Chromium, so set the binary path + # to our install. + kw_options["binary"] = self.chromium_dist_path or "" + + # Options overwritten from **kw + if "test" in self.config: + kw_options["test"] = self.config["test"] + if "binary" in self.config: + kw_options["binary"] = self.config["binary"] + if self.symbols_path: + kw_options["symbolsPath"] = self.symbols_path + if self.config.get("obj_path", None) is not None: + kw_options["obj-path"] = self.config["obj_path"] + if self.config.get("mozbuild_path", None) is not None: + kw_options["mozbuild-path"] = self.config["mozbuild_path"] + if self.test_url_params: + kw_options["test-url-params"] = self.test_url_params + if self.config.get("device_name") is not None: + kw_options["device-name"] = self.config["device_name"] + if self.config.get("activity") is not None: + kw_options["activity"] = self.config["activity"] + if self.config.get("conditioned_profile") is not None: + kw_options["conditioned-profile"] = self.config["conditioned_profile"] + if self.config.get("benchmark_repository"): + kw_options["benchmark_repository"] = self.config["benchmark_repository"] + if self.config.get("benchmark_revision"): + kw_options["benchmark_revision"] = self.config["benchmark_revision"] + if self.config.get("benchmark_repository"): + kw_options["benchmark_branch"] = self.config["benchmark_branch"] + + kw_options.update(kw) + if self.host: + kw_options["host"] = self.host + # Configure profiling options + options.extend(self.query_gecko_profile_options()) + # Extra arguments + if args is not None: + options += args + if os.getenv("PERF_FLAGS"): + for option in os.getenv("PERF_FLAGS").split(): + if "=" in option: + kw_option, value = option.split("=") + kw_options[kw_option] = value + else: + options.extend(["--" + option]) + + if self.config.get("run_local", False): + options.extend(["--run-local"]) + if "raptor_cmd_line_args" in self.config: + options += self.config["raptor_cmd_line_args"] + if self.config.get("code_coverage", False): + options.extend(["--code-coverage"]) + if self.config.get("is_release_build", False): + options.extend(["--is-release-build"]) + if self.config.get("power_test", False): + options.extend(["--power-test"]) + if self.config.get("memory_test", False): + options.extend(["--memory-test"]) + if self.config.get("cpu_test", False): + options.extend(["--cpu-test"]) + if self.config.get("live_sites", False): + options.extend(["--live-sites"]) + if self.config.get("chimera", False): + options.extend(["--chimera"]) + if self.config.get("disable_perf_tuning", False): + options.extend(["--disable-perf-tuning"]) + if self.config.get("cold", False): + options.extend(["--cold"]) + if not self.config.get("fission", True): + options.extend(["--disable-fission"]) + if self.config.get("verbose", False): + options.extend(["--verbose"]) + if self.config.get("extra_prefs"): + options.extend( + ["--setpref={}".format(i) for i in self.config.get("extra_prefs")] + ) + if self.config.get("environment"): + options.extend( + ["--setenv={}".format(i) for i in self.config.get("environment")] + ) + if self.config.get("enable_marionette_trace", False): + options.extend(["--enable-marionette-trace"]) + if self.config.get("browser_cycles"): + options.extend( + ["--browser-cycles={}".format(self.config.get("browser_cycles"))] + ) + if self.config.get("test_bytecode_cache", False): + options.extend(["--test-bytecode-cache"]) + if self.config.get("collect_perfstats", False): + options.extend(["--collect-perfstats"]) + if self.config.get("extra_summary_methods"): + options.extend( + [ + "--extra-summary-methods={}".format(method) + for method in self.config.get("extra_summary_methods") + ] + ) + if self.config.get("webext", False): + options.extend(["--webext"]) + else: + for (arg,), details in Raptor.browsertime_options: + # Allow overriding defaults on the `./mach raptor-test ...` command-line + value = self.config.get(details["dest"]) + if value is None or value != getattr(self, details["dest"], None): + # Check for modifications done to the instance variables + value = getattr(self, details["dest"], None) + if value and arg not in self.config.get("raptor_cmd_line_args", []): + if isinstance(value, string_types): + options.extend([arg, os.path.expandvars(value)]) + elif isinstance(value, (tuple, list)): + for val in value: + options.extend([arg, val]) + else: + options.extend([arg]) + + for key, value in kw_options.items(): + options.extend(["--%s" % key, value]) + + return options + + def populate_webroot(self): + """Populate the production test machines' webroots""" + self.raptor_path = os.path.join( + self.query_abs_dirs()["abs_test_install_dir"], "raptor" + ) + if self.config.get("run_local"): + self.raptor_path = os.path.join(self.repo_path, "testing", "raptor") + + def clobber(self): + # Recreate the upload directory for storing the logcat collected + # during APK installation. + super(Raptor, self).clobber() + upload_dir = self.query_abs_dirs()["abs_blob_upload_dir"] + if not os.path.isdir(upload_dir): + self.mkdir_p(upload_dir) + + def install_android_app(self, apk, replace=False): + # Override AndroidMixin's install_android_app in order to capture + # logcat during the installation. If the installation fails, + # the logcat file will be left in the upload directory. + self.logcat_start() + try: + super(Raptor, self).install_android_app(apk, replace=replace) + finally: + self.logcat_stop() + + def download_and_extract(self, extract_dirs=None, suite_categories=None): + # Use in-tree wptserve for Python 3.10 compatibility + extract_dirs = [ + "tools/wptserve/*", + "tools/wpt_third_party/pywebsocket3/*", + ] + return super(Raptor, self).download_and_extract( + extract_dirs=extract_dirs, suite_categories=["common", "condprof", "raptor"] + ) + + def create_virtualenv(self, **kwargs): + """VirtualenvMixin.create_virtualenv() assumes we're using + self.config['virtualenv_modules']. Since we're installing + raptor from its source, we have to wrap that method here.""" + # If virtualenv already exists, just add to path and don't re-install. + # We need it in-path to import jsonschema later when validating output for perfherder. + _virtualenv_path = self.config.get("virtualenv_path") + + if self.clean: + rmtree(_virtualenv_path, ignore_errors=True) + + _python_interp = self.query_exe("python") + if "win" in self.platform_name() and os.path.exists(_python_interp): + multiprocessing.set_executable(_python_interp) + + if self.run_local and os.path.exists(_virtualenv_path): + self.info("Virtualenv already exists, skipping creation") + # ffmpeg exists outside of this virtual environment so + # we re-add it to the platform environment on repeated + # local runs of browsertime visual metric tests + self.setup_local_ffmpeg() + + if "win" in self.platform_name(): + _path = os.path.join(_virtualenv_path, "Lib", "site-packages") + else: + _path = os.path.join( + _virtualenv_path, + "lib", + os.path.basename(_python_interp), + "site-packages", + ) + + sys.path.append(_path) + return + + # virtualenv doesn't already exist so create it + # Install mozbase first, so we use in-tree versions + # Additionally, decide where to pull raptor requirements from. + if not self.run_local: + mozbase_requirements = os.path.join( + self.query_abs_dirs()["abs_test_install_dir"], + "config", + "mozbase_requirements.txt", + ) + raptor_requirements = os.path.join(self.raptor_path, "requirements.txt") + else: + mozbase_requirements = os.path.join( + os.path.dirname(self.raptor_path), + "config", + "mozbase_source_requirements.txt", + ) + raptor_requirements = os.path.join( + self.raptor_path, "source_requirements.txt" + ) + self.register_virtualenv_module( + requirements=[mozbase_requirements], + two_pass=True, + editable=True, + ) + + modules = ["pip>=1.5"] + + # Add modules required for visual metrics + py3_minor = sys.version_info.minor + if py3_minor <= 7: + modules.extend( + [ + "numpy==1.16.1", + "Pillow==6.1.0", + "scipy==1.2.3", + "pyssim==0.4", + "opencv-python==4.5.4.60", + ] + ) + else: # python version >= 3.8 + modules.extend( + [ + "numpy==1.22.0", + "Pillow==9.0.0", + "scipy==1.7.3", + "pyssim==0.4", + "opencv-python==4.5.4.60", + ] + ) + + if self.run_local: + self.setup_local_ffmpeg() + + # Require pip >= 1.5 so pip will prefer .whl files to install + super(Raptor, self).create_virtualenv(modules=modules) + + # Install Raptor dependencies + self.install_module(requirements=[raptor_requirements]) + + def setup_local_ffmpeg(self): + """Make use of the users local ffmpeg when running browsertime visual + metrics tests. + """ + + if "ffmpeg" in os.environ["PATH"]: + return + + platform = self.platform_name() + btime_cache = os.path.join(self.config["mozbuild_path"], "browsertime") + if "mac" in platform: + path_to_ffmpeg = os.path.join( + btime_cache, + FFMPEG_LOCAL_CACHE["mac"], + ) + elif "linux" in platform: + path_to_ffmpeg = os.path.join( + btime_cache, + FFMPEG_LOCAL_CACHE["linux"], + ) + elif "win" in platform: + path_to_ffmpeg = os.path.join( + btime_cache, + FFMPEG_LOCAL_CACHE["win"], + "bin", + ) + + if os.path.exists(path_to_ffmpeg): + os.environ["PATH"] += os.pathsep + path_to_ffmpeg + self.browsertime_ffmpeg = path_to_ffmpeg + self.info( + "Added local ffmpeg found at: %s to environment." % path_to_ffmpeg + ) + else: + raise Exception( + "No local ffmpeg binary found. Expected it to be here: %s" + % path_to_ffmpeg + ) + + def install(self): + if not self.config.get("noinstall", False): + if self.app in self.firefox_android_browsers: + self.device.uninstall_app(self.binary_path) + + # Check if the user supplied their own APK, and install + # that instead + installer_path = pathlib.Path( + self.raptor_path, "raptor", "user_upload.apk" + ) + if not installer_path.exists(): + installer_path = self.installer_path + + self.info(f"Installing APK from: {installer_path}") + self.install_android_app(str(installer_path)) + else: + super(Raptor, self).install() + + def _artifact_perf_data(self, src, dest): + if not os.path.isdir(os.path.dirname(dest)): + # create upload dir if it doesn't already exist + self.info("Creating dir: %s" % os.path.dirname(dest)) + os.makedirs(os.path.dirname(dest)) + self.info("Copying raptor results from %s to %s" % (src, dest)) + try: + copyfile(src, dest) + except Exception as e: + self.critical("Error copying results %s to upload dir %s" % (src, dest)) + self.info(str(e)) + + def run_tests(self, args=None, **kw): + """Run raptor tests""" + + # Get Raptor options + options = self.raptor_options(args=args, **kw) + + # Python version check + python = self.query_python_path() + self.run_command([python, "--version"]) + parser = RaptorOutputParser( + config=self.config, log_obj=self.log_obj, error_list=RaptorErrorList + ) + env = {} + env["MOZ_UPLOAD_DIR"] = self.query_abs_dirs()["abs_blob_upload_dir"] + if not self.run_local: + env["MINIDUMP_STACKWALK"] = self.query_minidump_stackwalk() + env["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"] + env["RUST_BACKTRACE"] = "full" + if not os.path.isdir(env["MOZ_UPLOAD_DIR"]): + self.mkdir_p(env["MOZ_UPLOAD_DIR"]) + env = self.query_env(partial_env=env, log_level=INFO) + # adjust PYTHONPATH to be able to use raptor as a python package + if "PYTHONPATH" in env: + env["PYTHONPATH"] = self.raptor_path + os.pathsep + env["PYTHONPATH"] + else: + env["PYTHONPATH"] = self.raptor_path + + # mitmproxy needs path to mozharness when installing the cert, and tooltool + env["SCRIPTSPATH"] = scripts_path + env["EXTERNALTOOLSPATH"] = external_tools_path + + # Needed to load unsigned Raptor WebExt on release builds + if self.is_release_build: + env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" + + if self.repo_path is not None: + env["MOZ_DEVELOPER_REPO_DIR"] = self.repo_path + if self.obj_path is not None: + env["MOZ_DEVELOPER_OBJ_DIR"] = self.obj_path + if self.mozbuild_path is not None: + env["MOZ_MOZBUILD_DIR"] = self.mozbuild_path + + # Sets a timeout for how long Raptor should run without output + output_timeout = self.config.get("raptor_output_timeout", 3600) + # Run Raptor tests + run_tests = os.path.join(self.raptor_path, "raptor", "raptor.py") + + # Dynamically set the log level based on the raptor config for consistency + # throughout the test + mozlog_opts = [f"--log-tbpl-level={self.config['log_level']}"] + + if not self.run_local and "suite" in self.config: + fname_pattern = "%s_%%s.log" % self.config["test"] + mozlog_opts.append( + "--log-errorsummary=%s" + % os.path.join(env["MOZ_UPLOAD_DIR"], fname_pattern % "errorsummary") + ) + + def launch_in_debug_mode(cmdline): + cmdline = set(cmdline) + debug_opts = {"--debug", "--debugger", "--debugger_args"} + + return bool(debug_opts.intersection(cmdline)) + + if self.app in self.android_browsers: + self.logcat_start() + + command = [python, run_tests] + options + mozlog_opts + if launch_in_debug_mode(command): + raptor_process = subprocess.Popen(command, cwd=self.workdir, env=env) + raptor_process.wait() + else: + self.return_code = self.run_command( + command, + cwd=self.workdir, + output_timeout=output_timeout, + output_parser=parser, + env=env, + ) + + if self.app in self.android_browsers: + self.logcat_stop() + + if parser.minidump_output: + self.info("Looking at the minidump files for debugging purposes...") + for item in parser.minidump_output: + self.run_command(["ls", "-l", item]) + + elif not self.run_local: + # Copy results to upload dir so they are included as an artifact + self.info("Copying Raptor results to upload dir:") + + src = os.path.join(self.query_abs_dirs()["abs_work_dir"], "raptor.json") + dest = os.path.join(env["MOZ_UPLOAD_DIR"], "perfherder-data.json") + self.info(str(dest)) + self._artifact_perf_data(src, dest) + + # Make individual perfherder data JSON's for each supporting data type + for file in glob.glob( + os.path.join(self.query_abs_dirs()["abs_work_dir"], "*") + ): + path, filename = os.path.split(file) + + if not filename.startswith("raptor-"): + continue + + # filename is expected to contain a unique data name + # i.e. raptor-os-baseline-power.json would result in + # the data name os-baseline-power + data_name = "-".join(filename.split("-")[1:]) + data_name = ".".join(data_name.split(".")[:-1]) + + src = file + dest = os.path.join( + env["MOZ_UPLOAD_DIR"], "perfherder-data-%s.json" % data_name + ) + self._artifact_perf_data(src, dest) + + src = os.path.join( + self.query_abs_dirs()["abs_work_dir"], "screenshots.html" + ) + if os.path.exists(src): + dest = os.path.join(env["MOZ_UPLOAD_DIR"], "screenshots.html") + self.info(str(dest)) + self._artifact_perf_data(src, dest) + + # Allow log failures to over-ride successful runs of the test harness and + # give log failures priority, so that, for instance, log failures resulting + # in TBPL_RETRY cause a retry rather than simply reporting an error. + if parser.tbpl_status != TBPL_SUCCESS: + parser_status = EXIT_STATUS_DICT[parser.tbpl_status] + self.info( + "return code %s changed to %s due to log output" + % (str(self.return_code), str(parser_status)) + ) + self.return_code = parser_status + + +class RaptorOutputParser(OutputParser): + minidump_regex = re.compile( + r'''raptorError: "error executing: '(\S+) (\S+) (\S+)'"''' + ) + RE_PERF_DATA = re.compile(r".*PERFHERDER_DATA:\s+(\{.*\})") + + def __init__(self, **kwargs): + super(RaptorOutputParser, self).__init__(**kwargs) + self.minidump_output = None + self.found_perf_data = [] + self.tbpl_status = TBPL_SUCCESS + self.worst_log_level = INFO + self.harness_retry_re = TinderBoxPrintRe["harness_error"]["retry_regex"] + + def parse_single_line(self, line): + m = self.minidump_regex.search(line) + if m: + self.minidump_output = (m.group(1), m.group(2), m.group(3)) + + m = self.RE_PERF_DATA.match(line) + if m: + self.found_perf_data.append(m.group(1)) + + if self.harness_retry_re.search(line): + self.critical(" %s" % line) + self.worst_log_level = self.worst_level(CRITICAL, self.worst_log_level) + self.tbpl_status = self.worst_level( + TBPL_RETRY, self.tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE + ) + return # skip base parse_single_line + super(RaptorOutputParser, self).parse_single_line(line) |