path: root/testing/mozharness/mozharness/mozilla/testing/raptor.py

#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import argparse
import copy
import glob
import multiprocessing
import os
import pathlib
import re
import subprocess
import sys
import tempfile
from shutil import copyfile, rmtree

from six import string_types

import mozharness
from mozharness.base.errors import PythonErrorList
from mozharness.base.log import CRITICAL, DEBUG, ERROR, INFO, OutputParser
from mozharness.base.python import Python3Virtualenv
from mozharness.base.vcs.vcsbase import MercurialScript
from mozharness.mozilla.automation import (
    EXIT_STATUS_DICT,
    TBPL_RETRY,
    TBPL_SUCCESS,
    TBPL_WORST_LEVEL_TUPLE,
)
from mozharness.mozilla.testing.android import AndroidMixin
from mozharness.mozilla.testing.codecoverage import (
    CodeCoverageMixin,
    code_coverage_config_options,
)
from mozharness.mozilla.testing.errors import HarnessErrorList, TinderBoxPrintRe
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options

scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__)))
external_tools_path = os.path.join(scripts_path, "external_tools")
here = os.path.abspath(os.path.dirname(__file__))

RaptorErrorList = (
    PythonErrorList
    + HarnessErrorList
    + [
        {"regex": re.compile(r"""run-as: Package '.*' is unknown"""), "level": DEBUG},
        {"substr": r"""raptorDebug""", "level": DEBUG},
        {
            "regex": re.compile(r"""^raptor[a-zA-Z-]*( - )?( )?(?i)error(:)?"""),
            "level": ERROR,
        },
        {
            "regex": re.compile(r"""^raptor[a-zA-Z-]*( - )?( )?(?i)critical(:)?"""),
            "level": CRITICAL,
        },
        {
            "regex": re.compile(r"""No machine_name called '.*' can be found"""),
            "level": CRITICAL,
        },
        {
            "substr": r"""No such file or directory: 'browser_output.txt'""",
            "level": CRITICAL,
            "explanation": "Most likely the browser failed to launch, or the test otherwise "
            "failed to start.",
        },
    ]
)

# When running raptor locally, we can attempt to make use of
# the users locally cached ffmpeg binary from from when the user
# ran `./mach browsertime --setup`
FFMPEG_LOCAL_CACHE = {
    "mac": "ffmpeg-macos",
    "linux": "ffmpeg-4.4.1-i686-static",
    "win": "ffmpeg-4.4.1-full_build",
}


class Raptor(
    TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin, Python3Virtualenv
):
    """
    Install and run Raptor tests
    """

    # Options to Browsertime.  Paths are expected to be absolute.
    browsertime_options = [
        [
            ["--browsertime-node"],
            {"dest": "browsertime_node", "default": None, "help": argparse.SUPPRESS},
        ],
        [
            ["--browsertime-browsertimejs"],
            {
                "dest": "browsertime_browsertimejs",
                "default": None,
                "help": argparse.SUPPRESS,
            },
        ],
        [
            ["--browsertime-vismet-script"],
            {
                "dest": "browsertime_vismet_script",
                "default": None,
                "help": argparse.SUPPRESS,
            },
        ],
        [
            ["--browsertime-chromedriver"],
            {
                "dest": "browsertime_chromedriver",
                "default": None,
                "help": argparse.SUPPRESS,
            },
        ],
        [
            ["--browsertime-ffmpeg"],
            {"dest": "browsertime_ffmpeg", "default": None, "help": argparse.SUPPRESS},
        ],
        [
            ["--browsertime-geckodriver"],
            {
                "dest": "browsertime_geckodriver",
                "default": None,
                "help": argparse.SUPPRESS,
            },
        ],
        [
            ["--browsertime-video"],
            {
                "dest": "browsertime_video",
                "action": "store_true",
                "default": False,
                "help": argparse.SUPPRESS,
            },
        ],
        [
            ["--browsertime-visualmetrics"],
            {
                "dest": "browsertime_visualmetrics",
                "action": "store_true",
                "default": False,
                "help": argparse.SUPPRESS,
            },
        ],
        [
            ["--browsertime-no-ffwindowrecorder"],
            {
                "dest": "browsertime_no_ffwindowrecorder",
                "action": "store_true",
                "default": False,
                "help": argparse.SUPPRESS,
            },
        ],
        [
            ["--browsertime-arg"],
            {
                "action": "append",
                "metavar": "PREF=VALUE",
                "dest": "browsertime_user_args",
                "default": [],
                "help": argparse.SUPPRESS,
            },
        ],
        [
            ["--browsertime"],
            {
                "dest": "browsertime",
                "action": "store_true",
                "default": True,
                "help": argparse.SUPPRESS,
            },
        ],
    ]

    config_options = (
        [
            [
                ["--test"],
                {"action": "store", "dest": "test", "help": "Raptor test to run"},
            ],
            [
                ["--app"],
                {
                    "default": "firefox",
                    "choices": [
                        "firefox",
                        "chrome",
                        "chrome-m",
                        "chromium",
                        "fennec",
                        "geckoview",
                        "refbrow",
                        "fenix",
                        "safari",
                        "custom-car",
                    ],
                    "dest": "app",
                    "help": "Name of the application we are testing (default: firefox).",
                },
            ],
            [
                ["--activity"],
                {
                    "dest": "activity",
                    "help": "The Android activity used to launch the Android app. "
                    "e.g.: org.mozilla.fenix.browser.BrowserPerformanceTestActivity",
                },
            ],
            [
                ["--intent"],
                {
                    "dest": "intent",
                    "help": "Name of the Android intent action used to launch the Android app",
                },
            ],
            [
                ["--is-release-build"],
                {
                    "action": "store_true",
                    "dest": "is_release_build",
                    "help": "Whether the build is a release build which requires work arounds "
                    "using MOZ_DISABLE_NONLOCAL_CONNECTIONS to support installing unsigned "
                    "webextensions. Defaults to False.",
                },
            ],
            [
                ["--add-option"],
                {
                    "action": "extend",
                    "dest": "raptor_cmd_line_args",
                    "default": None,
                    "help": "Extra options to Raptor.",
                },
            ],
            [
                ["--device-name"],
                {
                    "dest": "device_name",
                    "default": None,
                    "help": "Device name of mobile device.",
                },
            ],
            [
                ["--geckoProfile"],
                {
                    "dest": "gecko_profile",
                    "action": "store_true",
                    "default": False,
                    "help": argparse.SUPPRESS,
                },
            ],
            [
                ["--geckoProfileInterval"],
                {
                    "dest": "gecko_profile_interval",
                    "type": "int",
                    "help": argparse.SUPPRESS,
                },
            ],
            [
                ["--geckoProfileEntries"],
                {
                    "dest": "gecko_profile_entries",
                    "type": "int",
                    "help": argparse.SUPPRESS,
                },
            ],
            [
                ["--geckoProfileFeatures"],
                {
                    "dest": "gecko_profile_features",
                    "type": "str",
                    "help": argparse.SUPPRESS,
                },
            ],
            [
                ["--gecko-profile"],
                {
                    "dest": "gecko_profile",
                    "action": "store_true",
                    "default": False,
                    "help": "Whether to profile the test run and save the profile results.",
                },
            ],
            [
                ["--gecko-profile-interval"],
                {
                    "dest": "gecko_profile_interval",
                    "type": "int",
                    "help": "The interval between samples taken by the profiler (ms).",
                },
            ],
            [
                ["--gecko-profile-entries"],
                {
                    "dest": "gecko_profile_entries",
                    "type": "int",
                    "help": "How many samples to take with the profiler.",
                },
            ],
            [
                ["--gecko-profile-threads"],
                {
                    "dest": "gecko_profile_threads",
                    "type": "str",
                    "help": "Comma-separated list of threads to sample.",
                },
            ],
            [
                ["--gecko-profile-features"],
                {
                    "dest": "gecko_profile_features",
                    "type": "str",
                    "help": "Features to enable in the profiler.",
                },
            ],
            [
                ["--extra-profiler-run"],
                {
                    "dest": "extra_profiler_run",
                    "action": "store_true",
                    "default": False,
                    "help": "Run the tests again with profiler enabled after the main run.",
                },
            ],
            [
                ["--page-cycles"],
                {
                    "dest": "page_cycles",
                    "type": "int",
                    "help": (
                        "How many times to repeat loading the test page (for page load "
                        "tests); for benchmark tests this is how many times the benchmark test "
                        "will be run."
                    ),
                },
            ],
            [
                ["--page-timeout"],
                {
                    "dest": "page_timeout",
                    "type": "int",
                    "help": "How long to wait (ms) for one page_cycle to complete, before timing out.",  # NOQA: E501
                },
            ],
            [
                ["--browser-cycles"],
                {
                    "dest": "browser_cycles",
                    "type": "int",
                    "help": (
                        "The number of times a cold load test is repeated (for cold load tests "
                        "only, where the browser is shutdown and restarted between test "
                        "iterations)."
                    ),
                },
            ],
            [
                ["--project"],
                {
                    "action": "store",
                    "dest": "project",
                    "default": "mozilla-central",
                    "type": "str",
                    "help": "Name of the project (try, mozilla-central, etc.)",
                },
            ],
            [
                ["--test-url-params"],
                {
                    "action": "store",
                    "dest": "test_url_params",
                    "help": "Parameters to add to the test_url query string.",
                },
            ],
            [
                ["--host"],
                {
                    "dest": "host",
                    "type": "str",
                    "default": "127.0.0.1",
                    "help": "Hostname from which to serve urls (default: 127.0.0.1). "
                    "The value HOST_IP will cause the value of host to be "
                    "to be loaded from the environment variable HOST_IP.",
                },
            ],
            [
                ["--power-test"],
                {
                    "dest": "power_test",
                    "action": "store_true",
                    "default": False,
                    "help": (
                        "Use Raptor to measure power usage on Android browsers (Geckoview "
                        "Example, Fenix, Refbrow, and Fennec) as well as on Intel-based MacOS "
                        "machines that have Intel Power Gadget installed."
                    ),
                },
            ],
            [
                ["--memory-test"],
                {
                    "dest": "memory_test",
                    "action": "store_true",
                    "default": False,
                    "help": "Use Raptor to measure memory usage.",
                },
            ],
            [
                ["--cpu-test"],
                {
                    "dest": "cpu_test",
                    "action": "store_true",
                    "default": False,
                    "help": "Use Raptor to measure CPU usage.",
                },
            ],
            [
                ["--disable-perf-tuning"],
                {
                    "action": "store_true",
                    "dest": "disable_perf_tuning",
                    "default": False,
                    "help": "Disable performance tuning on android.",
                },
            ],
            [
                ["--conditioned-profile"],
                {
                    "dest": "conditioned_profile",
                    "type": "str",
                    "default": None,
                    "help": (
                        "Name of conditioned profile to use. Prefix with `artifact:` "
                        "if we should obtain the profile from CI.",
                    ),
                },
            ],
            [
                ["--live-sites"],
                {
                    "dest": "live_sites",
                    "action": "store_true",
                    "default": False,
                    "help": "Run tests using live sites instead of recorded sites.",
                },
            ],
            [
                ["--test-bytecode-cache"],
                {
                    "dest": "test_bytecode_cache",
                    "action": "store_true",
                    "default": False,
                    "help": (
                        "If set, the pageload test will set the preference "
                        "`dom.script_loader.bytecode_cache.strategy=-1` and wait 20 seconds "
                        "after the first cold pageload to populate the bytecode cache before "
                        "running a warm pageload test. Only available if `--chimera` "
                        "is also provided."
                    ),
                },
            ],
            [
                ["--chimera"],
                {
                    "dest": "chimera",
                    "action": "store_true",
                    "default": False,
                    "help": "Run tests in chimera mode. Each browser cycle will run a cold and warm test.",  # NOQA: E501
                },
            ],
            [
                ["--debug-mode"],
                {
                    "dest": "debug_mode",
                    "action": "store_true",
                    "default": False,
                    "help": "Run Raptor in debug mode (open browser console, limited page-cycles, etc.)",  # NOQA: E501
                },
            ],
            [
                ["--noinstall"],
                {
                    "dest": "noinstall",
                    "action": "store_true",
                    "default": False,
                    "help": "Do not offer to install Android APK.",
                },
            ],
            [
                ["--disable-e10s"],
                {
                    "dest": "e10s",
                    "action": "store_false",
                    "default": True,
                    "help": "Run without multiple processes (e10s).",
                },
            ],
            [
                ["--disable-fission"],
                {
                    "action": "store_false",
                    "dest": "fission",
                    "default": True,
                    "help": "Disable Fission (site isolation) in Gecko.",
                },
            ],
            [
                ["--setpref"],
                {
                    "action": "append",
                    "metavar": "PREF=VALUE",
                    "dest": "extra_prefs",
                    "default": [],
                    "help": "Set a browser preference. May be used multiple times.",
                },
            ],
            [
                ["--setenv"],
                {
                    "action": "append",
                    "metavar": "NAME=VALUE",
                    "dest": "environment",
                    "default": [],
                    "help": "Set a variable in the test environment. May be used multiple times.",
                },
            ],
            [
                ["--skip-preflight"],
                {
                    "action": "store_true",
                    "dest": "skip_preflight",
                    "default": False,
                    "help": "skip preflight commands to prepare machine.",
                },
            ],
            [
                ["--cold"],
                {
                    "action": "store_true",
                    "dest": "cold",
                    "default": False,
                    "help": "Enable cold page-load for browsertime tp6",
                },
            ],
            [
                ["--verbose"],
                {
                    "action": "store_true",
                    "dest": "verbose",
                    "default": False,
                    "help": "Verbose output",
                },
            ],
            [
                ["--enable-marionette-trace"],
                {
                    "action": "store_true",
                    "dest": "enable_marionette_trace",
                    "default": False,
                    "help": "Enable marionette tracing",
                },
            ],
            [
                ["--clean"],
                {
                    "action": "store_true",
                    "dest": "clean",
                    "default": False,
                    "help": (
                        "Clean the python virtualenv (remove, and rebuild) for "
                        "Raptor before running tests."
                    ),
                },
            ],
            [
                ["--webext"],
                {
                    "action": "store_true",
                    "dest": "webext",
                    "default": False,
                    "help": (
                        "Whether to use webextension to execute pageload tests "
                        "(WebExtension is being deprecated).",
                    ),
                },
            ],
            [
                ["--collect-perfstats"],
                {
                    "action": "store_true",
                    "dest": "collect_perfstats",
                    "default": False,
                    "help": (
                        "If set, the test will collect perfstats in addition to "
                        "the regular metrics it gathers."
                    ),
                },
            ],
            [
                ["--extra-summary-methods"],
                {
                    "action": "append",
                    "metavar": "OPTION",
                    "dest": "extra_summary_methods",
                    "default": [],
                    "help": (
                        "Alternative methods for summarizing technical and visual"
                        "pageload metrics."
                        "Options: geomean, mean."
                    ),
                },
            ],
            [
                ["--benchmark-repository"],
                {
                    "dest": "benchmark_repository",
                    "type": "str",
                    "default": None,
                    "help": (
                        "Repository that should be used for a particular benchmark test. "
                        "e.g. https://github.com/mozilla-mobile/firefox-android"
                    ),
                },
            ],
            [
                ["--benchmark-revision"],
                {
                    "dest": "benchmark_revision",
                    "type": "str",
                    "default": None,
                    "help": (
                        "Repository revision that should be used for a particular "
                        "benchmark test."
                    ),
                },
            ],
            [
                ["--benchmark-branch"],
                {
                    "dest": "benchmark_branch",
                    "type": "str",
                    "default": None,
                    "help": (
                        "Repository branch that should be used for a particular benchmark test."
                    ),
                },
            ],
        ]
        + testing_config_options
        + copy.deepcopy(code_coverage_config_options)
        + browsertime_options
    )

    def __init__(self, **kwargs):
        kwargs.setdefault("config_options", self.config_options)
        kwargs.setdefault(
            "all_actions",
            [
                "clobber",
                "download-and-extract",
                "populate-webroot",
                "create-virtualenv",
                "install-chrome-android",
                "install-chromium-distribution",
                "install",
                "run-tests",
            ],
        )
        kwargs.setdefault(
            "default_actions",
            [
                "clobber",
                "download-and-extract",
                "populate-webroot",
                "create-virtualenv",
                "install-chromium-distribution",
                "install",
                "run-tests",
            ],
        )
        kwargs.setdefault("config", {})
        super(Raptor, self).__init__(**kwargs)

        # Convenience
        self.workdir = self.query_abs_dirs()["abs_work_dir"]

        self.run_local = self.config.get("run_local")

        # App (browser testing on) defaults to firefox
        self.app = "firefox"

        if self.run_local:
            # Get app from command-line args, passed in from mach, inside 'raptor_cmd_line_args'
            # Command-line args can be in two formats depending on how the user entered them
            # i.e. "--app=geckoview" or separate as "--app", "geckoview" so we have to
            # parse carefully.  It's simplest to use `argparse` to parse partially.
            self.app = "firefox"
            if "raptor_cmd_line_args" in self.config:
                sub_parser = argparse.ArgumentParser()
                # It's not necessary to limit the allowed values: each value
                # will be parsed and verifed by raptor/raptor.py.
                sub_parser.add_argument("--app", default=None, dest="app")
                sub_parser.add_argument("-i", "--intent", default=None, dest="intent")
                sub_parser.add_argument(
                    "-a", "--activity", default=None, dest="activity"
                )

                # We'd prefer to use `parse_known_intermixed_args`, but that's
                # new in Python 3.7.
                known, unknown = sub_parser.parse_known_args(
                    self.config["raptor_cmd_line_args"]
                )

                if known.app:
                    self.app = known.app
                if known.intent:
                    self.intent = known.intent
                if known.activity:
                    self.activity = known.activity
        else:
            # Raptor initiated in production via mozharness
            self.test = self.config["test"]
            self.app = self.config.get("app", "firefox")
            self.binary_path = self.config.get("binary_path", None)

            if self.app in ("refbrow", "fenix"):
                self.app_name = self.binary_path

        self.installer_url = self.config.get("installer_url")
        self.raptor_json_url = self.config.get("raptor_json_url")
        self.raptor_json = self.config.get("raptor_json")
        self.raptor_json_config = self.config.get("raptor_json_config")
        self.repo_path = self.config.get("repo_path")
        self.obj_path = self.config.get("obj_path")
        self.mozbuild_path = self.config.get("mozbuild_path")
        self.test = None
        self.gecko_profile = self.config.get(
            "gecko_profile"
        ) or "--geckoProfile" in self.config.get("raptor_cmd_line_args", [])
        self.gecko_profile_interval = self.config.get("gecko_profile_interval")
        self.gecko_profile_entries = self.config.get("gecko_profile_entries")
        self.gecko_profile_threads = self.config.get("gecko_profile_threads")
        self.gecko_profile_features = self.config.get("gecko_profile_features")
        self.extra_profiler_run = self.config.get("extra_profiler_run")
        self.test_packages_url = self.config.get("test_packages_url")
        self.test_url_params = self.config.get("test_url_params")
        self.host = self.config.get("host")
        if self.host == "HOST_IP":
            self.host = os.environ["HOST_IP"]
        self.power_test = self.config.get("power_test")
        self.memory_test = self.config.get("memory_test")
        self.cpu_test = self.config.get("cpu_test")
        self.live_sites = self.config.get("live_sites")
        self.chimera = self.config.get("chimera")
        self.disable_perf_tuning = self.config.get("disable_perf_tuning")
        self.conditioned_profile = self.config.get("conditioned_profile")
        self.extra_prefs = self.config.get("extra_prefs")
        self.environment = self.config.get("environment")
        self.is_release_build = self.config.get("is_release_build")
        self.debug_mode = self.config.get("debug_mode", False)
        self.chromium_dist_path = None
        self.firefox_android_browsers = ["fennec", "geckoview", "refbrow", "fenix"]
        self.android_browsers = self.firefox_android_browsers + ["chrome-m"]
        self.browsertime_visualmetrics = self.config.get("browsertime_visualmetrics")
        self.browsertime_node = self.config.get("browsertime_node")
        self.browsertime_user_args = self.config.get("browsertime_user_args")
        self.browsertime_video = False
        self.enable_marionette_trace = self.config.get("enable_marionette_trace")
        self.browser_cycles = self.config.get("browser_cycles")
        self.clean = self.config.get("clean")

        for (arg,), details in Raptor.browsertime_options:
            # Allow overriding defaults on the `./mach raptor-test ...` command-line.
            value = self.config.get(details["dest"])
            if value and arg not in self.config.get("raptor_cmd_line_args", []):
                setattr(self, details["dest"], value)

    # We accept some configuration options from the try commit message in the
    # format mozharness: <options>. Example try commit message: mozharness:
    # --geckoProfile try: <stuff>
    def query_gecko_profile_options(self):
        gecko_results = []
        # If gecko_profile is set, we add that to Raptor's options
        if self.gecko_profile:
            gecko_results.append("--gecko-profile")
            if self.gecko_profile_interval:
                gecko_results.extend(
                    ["--gecko-profile-interval", str(self.gecko_profile_interval)]
                )
            if self.gecko_profile_entries:
                gecko_results.extend(
                    ["--gecko-profile-entries", str(self.gecko_profile_entries)]
                )
            if self.gecko_profile_features:
                gecko_results.extend(
                    ["--gecko-profile-features", self.gecko_profile_features]
                )
            if self.gecko_profile_threads:
                gecko_results.extend(
                    ["--gecko-profile-threads", self.gecko_profile_threads]
                )
        else:
            if self.extra_profiler_run:
                gecko_results.append("--extra-profiler-run")
        return gecko_results

    def query_abs_dirs(self):
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = super(Raptor, self).query_abs_dirs()
        abs_dirs["abs_blob_upload_dir"] = os.path.join(
            abs_dirs["abs_work_dir"], "blobber_upload_dir"
        )
        abs_dirs["abs_test_install_dir"] = os.path.join(
            abs_dirs["abs_work_dir"], "tests"
        )

        self.abs_dirs = abs_dirs
        return self.abs_dirs

    def install_chrome_android(self):
        """Install Google Chrome for Android in production from tooltool"""
        if self.app != "chrome-m":
            self.info("Google Chrome for Android not required")
            return
        if self.config.get("run_local"):
            self.info(
                "Google Chrome for Android will not be installed "
                "from tooltool when running locally"
            )
            return
        self.info("Fetching and installing Google Chrome for Android")
        self.device.shell_output("cmd package install-existing com.android.chrome")
        self.info("Google Chrome for Android successfully installed")

    def download_chrome_android(self):
        # Fetch the APK
        tmpdir = tempfile.mkdtemp()
        self.tooltool_fetch(
            os.path.join(
                self.raptor_path,
                "raptor",
                "tooltool-manifests",
                "chrome-android",
                "chrome87.manifest",
            ),
            output_dir=tmpdir,
        )
        files = os.listdir(tmpdir)
        if len(files) > 1:
            raise Exception(
                "Found more than one chrome APK file after tooltool download"
            )
        chromeapk = os.path.join(tmpdir, files[0])

        # Disable verification and install the APK
        self.device.shell_output("settings put global verifier_verify_adb_installs 0")
        self.install_android_app(chromeapk, replace=True)

        # Re-enable verification and delete the temporary directory
        self.device.shell_output("settings put global verifier_verify_adb_installs 1")
        rmtree(tmpdir)

    def install_chromium_distribution(self):
        """Install Google Chromium distribution in production"""
        linux, mac, win = "linux", "mac", "win"
        chrome, chromium, chromium_release = "chrome", "chromium", "custom-car"

        available_chromium_dists = [chrome, chromium, chromium_release]
        binary_location = {
            chromium: {
                linux: ["chrome-linux", "chrome"],
                mac: ["chrome-mac", "Chromium.app", "Contents", "MacOS", "Chromium"],
                win: ["chrome-win", "Chrome.exe"],
            },
            chromium_release: {
                linux: ["chromium", "Default", "chrome"],
                win: ["chromium", "Default", "chrome.exe"],
            },
        }

        if self.app not in available_chromium_dists:
            self.info("Google Chrome or Chromium distributions are not required.")
            return

        if self.app == "chrome":
            self.info("Chrome should be preinstalled.")
            if win in self.platform_name():
                base_path = "C:\\%s\\Google\\Chrome\\Application\\chrome.exe"
                self.chromium_dist_path = base_path % "Progra~1"
                if not os.path.exists(self.chromium_dist_path):
                    self.chromium_dist_path = base_path % "Progra~2"
            elif linux in self.platform_name():
                self.chromium_dist_path = "/usr/bin/google-chrome"
            elif mac in self.platform_name():
                self.chromium_dist_path = (
                    "/Applications/Google Chrome.app/" "Contents/MacOS/Google Chrome"
                )
            else:
                self.error(
                    "Chrome is not installed on the platform %s yet."
                    % self.platform_name()
                )

            if os.path.exists(self.chromium_dist_path):
                self.info(
                    "Google Chrome found in expected location %s"
                    % self.chromium_dist_path
                )
            else:
                self.error("Cannot find Google Chrome at %s" % self.chromium_dist_path)

            return

        chromium_dist = self.app

        if self.config.get("run_local"):
            self.info("Expecting %s to be pre-installed locally" % chromium_dist)
            return

        self.info("Getting fetched %s build" % chromium_dist)
        self.chromium_dist_dest = os.path.normpath(
            os.path.abspath(os.environ["MOZ_FETCHES_DIR"])
        )

        if mac in self.platform_name():
            self.chromium_dist_path = os.path.join(
                self.chromium_dist_dest, *binary_location[chromium_dist][mac]
            )

        elif linux in self.platform_name():
            self.chromium_dist_path = os.path.join(
                self.chromium_dist_dest, *binary_location[chromium_dist][linux]
            )

        else:
            self.chromium_dist_path = os.path.join(
                self.chromium_dist_dest, *binary_location[chromium_dist][win]
            )

        self.info("%s dest is: %s" % (chromium_dist, self.chromium_dist_dest))
        self.info("%s path is: %s" % (chromium_dist, self.chromium_dist_path))

        # Now ensure Chromium binary exists
        if os.path.exists(self.chromium_dist_path):
            self.info(
                "Successfully installed %s to: %s"
                % (chromium_dist, self.chromium_dist_path)
            )
        else:
            self.info("Abort: failed to install %s" % chromium_dist)

    def raptor_options(self, args=None, **kw):
        """Return options to Raptor"""
        options = []
        kw_options = {}

        # Get the APK location to be able to get the browser version
        # through mozversion
        if self.app in self.firefox_android_browsers and not self.run_local:
            kw_options["installerpath"] = self.installer_path

        # If testing on Firefox, the binary path already came from mozharness/pro;
        # otherwise the binary path is forwarded from command-line arg (raptor_cmd_line_args).
        kw_options["app"] = self.app
        if self.app == "firefox" or (
            self.app in self.firefox_android_browsers and not self.run_local
        ):
            binary_path = self.binary_path or self.config.get("binary_path")
            if not binary_path:
                self.fatal("Raptor requires a path to the binary.")
            kw_options["binary"] = binary_path
            if self.app in self.firefox_android_browsers:
                # In production ensure we have correct app name,
                # i.e. fennec_aurora or fennec_release etc.
                kw_options["binary"] = self.query_package_name()
                self.info(
                    "Set binary to %s instead of %s"
                    % (kw_options["binary"], binary_path)
                )
        elif self.app == "safari" and not self.run_local:
            binary_path = "/Applications/Safari.app/Contents/MacOS/Safari"
            kw_options["binary"] = binary_path
        else:  # Running on Chromium
            if not self.run_local:
                # When running locally we already set the Chromium binary above, in init.
                # In production, we already installed Chromium, so set the binary path
                # to our install.
                kw_options["binary"] = self.chromium_dist_path or ""

        # Options overwritten from **kw
        if "test" in self.config:
            kw_options["test"] = self.config["test"]
        if "binary" in self.config:
            kw_options["binary"] = self.config["binary"]
        if self.symbols_path:
            kw_options["symbolsPath"] = self.symbols_path
        if self.config.get("obj_path", None) is not None:
            kw_options["obj-path"] = self.config["obj_path"]
        if self.config.get("mozbuild_path", None) is not None:
            kw_options["mozbuild-path"] = self.config["mozbuild_path"]
        if self.test_url_params:
            kw_options["test-url-params"] = self.test_url_params
        if self.config.get("device_name") is not None:
            kw_options["device-name"] = self.config["device_name"]
        if self.config.get("activity") is not None:
            kw_options["activity"] = self.config["activity"]
        if self.config.get("conditioned_profile") is not None:
            kw_options["conditioned-profile"] = self.config["conditioned_profile"]
        if self.config.get("benchmark_repository"):
            kw_options["benchmark_repository"] = self.config["benchmark_repository"]
        if self.config.get("benchmark_revision"):
            kw_options["benchmark_revision"] = self.config["benchmark_revision"]
        if self.config.get("benchmark_repository"):
            kw_options["benchmark_branch"] = self.config["benchmark_branch"]

        kw_options.update(kw)
        if self.host:
            kw_options["host"] = self.host
        # Configure profiling options
        options.extend(self.query_gecko_profile_options())
        # Extra arguments
        if args is not None:
            options += args
        if os.getenv("PERF_FLAGS"):
            for option in os.getenv("PERF_FLAGS").split():
                if "=" in option:
                    kw_option, value = option.split("=")
                    kw_options[kw_option] = value
                else:
                    options.extend(["--" + option])

        if self.config.get("run_local", False):
            options.extend(["--run-local"])
        if "raptor_cmd_line_args" in self.config:
            options += self.config["raptor_cmd_line_args"]
        if self.config.get("code_coverage", False):
            options.extend(["--code-coverage"])
        if self.config.get("is_release_build", False):
            options.extend(["--is-release-build"])
        if self.config.get("power_test", False):
            options.extend(["--power-test"])
        if self.config.get("memory_test", False):
            options.extend(["--memory-test"])
        if self.config.get("cpu_test", False):
            options.extend(["--cpu-test"])
        if self.config.get("live_sites", False):
            options.extend(["--live-sites"])
        if self.config.get("chimera", False):
            options.extend(["--chimera"])
        if self.config.get("disable_perf_tuning", False):
            options.extend(["--disable-perf-tuning"])
        if self.config.get("cold", False):
            options.extend(["--cold"])
        if not self.config.get("fission", True):
            options.extend(["--disable-fission"])
        if self.config.get("verbose", False):
            options.extend(["--verbose"])
        if self.config.get("extra_prefs"):
            options.extend(
                ["--setpref={}".format(i) for i in self.config.get("extra_prefs")]
            )
        if self.config.get("environment"):
            options.extend(
                ["--setenv={}".format(i) for i in self.config.get("environment")]
            )
        if self.config.get("enable_marionette_trace", False):
            options.extend(["--enable-marionette-trace"])
        if self.config.get("browser_cycles"):
            options.extend(
                ["--browser-cycles={}".format(self.config.get("browser_cycles"))]
            )
        if self.config.get("test_bytecode_cache", False):
            options.extend(["--test-bytecode-cache"])
        if self.config.get("collect_perfstats", False):
            options.extend(["--collect-perfstats"])
        if self.config.get("extra_summary_methods"):
            options.extend(
                [
                    "--extra-summary-methods={}".format(method)
                    for method in self.config.get("extra_summary_methods")
                ]
            )
        if self.config.get("webext", False):
            options.extend(["--webext"])
        else:
            for (arg,), details in Raptor.browsertime_options:
                # Allow overriding defaults on the `./mach raptor-test ...` command-line
                value = self.config.get(details["dest"])
                if value is None or value != getattr(self, details["dest"], None):
                    # Check for modifications done to the instance variables
                    value = getattr(self, details["dest"], None)
                if value and arg not in self.config.get("raptor_cmd_line_args", []):
                    if isinstance(value, string_types):
                        options.extend([arg, os.path.expandvars(value)])
                    elif isinstance(value, (tuple, list)):
                        for val in value:
                            options.extend([arg, val])
                    else:
                        options.extend([arg])

        for key, value in kw_options.items():
            options.extend(["--%s" % key, value])

        return options

    def populate_webroot(self):
        """Populate the production test machines' webroots"""
        self.raptor_path = os.path.join(
            self.query_abs_dirs()["abs_test_install_dir"], "raptor"
        )
        if self.config.get("run_local"):
            self.raptor_path = os.path.join(self.repo_path, "testing", "raptor")

    def clobber(self):
        # Recreate the upload directory for storing the logcat collected
        # during APK installation.
        super(Raptor, self).clobber()
        upload_dir = self.query_abs_dirs()["abs_blob_upload_dir"]
        if not os.path.isdir(upload_dir):
            self.mkdir_p(upload_dir)

    def install_android_app(self, apk, replace=False):
        # Override AndroidMixin's install_android_app in order to capture
        # logcat during the installation. If the installation fails,
        # the logcat file will be left in the upload directory.
        self.logcat_start()
        try:
            super(Raptor, self).install_android_app(apk, replace=replace)
        finally:
            self.logcat_stop()

    def download_and_extract(self, extract_dirs=None, suite_categories=None):
        # Use in-tree wptserve for Python 3.10 compatibility
        extract_dirs = [
            "tools/wptserve/*",
            "tools/wpt_third_party/pywebsocket3/*",
        ]
        return super(Raptor, self).download_and_extract(
            extract_dirs=extract_dirs, suite_categories=["common", "condprof", "raptor"]
        )

    def create_virtualenv(self, **kwargs):
        """VirtualenvMixin.create_virtualenv() assumes we're using
        self.config['virtualenv_modules']. Since we're installing
        raptor from its source, we have to wrap that method here."""
        # If virtualenv already exists, just add to path and don't re-install.
        # We need it in-path to import jsonschema later when validating output for perfherder.
        _virtualenv_path = self.config.get("virtualenv_path")

        if self.clean:
            rmtree(_virtualenv_path, ignore_errors=True)

        _python_interp = self.query_exe("python")
        if "win" in self.platform_name() and os.path.exists(_python_interp):
            multiprocessing.set_executable(_python_interp)

        if self.run_local and os.path.exists(_virtualenv_path):
            self.info("Virtualenv already exists, skipping creation")
            # ffmpeg exists outside of this virtual environment so
            # we re-add it to the platform environment on repeated
            # local runs of browsertime visual metric tests
            self.setup_local_ffmpeg()

            if "win" in self.platform_name():
                _path = os.path.join(_virtualenv_path, "Lib", "site-packages")
            else:
                _path = os.path.join(
                    _virtualenv_path,
                    "lib",
                    os.path.basename(_python_interp),
                    "site-packages",
                )

            sys.path.append(_path)
            return

        # virtualenv doesn't already exist so create it
        # Install mozbase first, so we use in-tree versions
        # Additionally, decide where to pull raptor requirements from.
        if not self.run_local:
            mozbase_requirements = os.path.join(
                self.query_abs_dirs()["abs_test_install_dir"],
                "config",
                "mozbase_requirements.txt",
            )
            raptor_requirements = os.path.join(self.raptor_path, "requirements.txt")
        else:
            mozbase_requirements = os.path.join(
                os.path.dirname(self.raptor_path),
                "config",
                "mozbase_source_requirements.txt",
            )
            raptor_requirements = os.path.join(
                self.raptor_path, "source_requirements.txt"
            )
        self.register_virtualenv_module(
            requirements=[mozbase_requirements],
            two_pass=True,
            editable=True,
        )

        modules = ["pip>=1.5"]

        # Add modules required for visual metrics
        py3_minor = sys.version_info.minor
        if py3_minor <= 7:
            modules.extend(
                [
                    "numpy==1.16.1",
                    "Pillow==6.1.0",
                    "scipy==1.2.3",
                    "pyssim==0.4",
                    "opencv-python==4.5.4.60",
                ]
            )
        else:  # python version >= 3.8
            modules.extend(
                [
                    "numpy==1.22.0",
                    "Pillow==9.0.0",
                    "scipy==1.7.3",
                    "pyssim==0.4",
                    "opencv-python==4.5.4.60",
                ]
            )

        if self.run_local:
            self.setup_local_ffmpeg()

        # Require pip >= 1.5 so pip will prefer .whl files to install
        super(Raptor, self).create_virtualenv(modules=modules)

        # Install Raptor dependencies
        self.install_module(requirements=[raptor_requirements])

    def setup_local_ffmpeg(self):
        """Make use of the users local ffmpeg when running browsertime visual
        metrics tests.
        """

        if "ffmpeg" in os.environ["PATH"]:
            return

        platform = self.platform_name()
        btime_cache = os.path.join(self.config["mozbuild_path"], "browsertime")
        if "mac" in platform:
            path_to_ffmpeg = os.path.join(
                btime_cache,
                FFMPEG_LOCAL_CACHE["mac"],
            )
        elif "linux" in platform:
            path_to_ffmpeg = os.path.join(
                btime_cache,
                FFMPEG_LOCAL_CACHE["linux"],
            )
        elif "win" in platform:
            path_to_ffmpeg = os.path.join(
                btime_cache,
                FFMPEG_LOCAL_CACHE["win"],
                "bin",
            )

        if os.path.exists(path_to_ffmpeg):
            os.environ["PATH"] += os.pathsep + path_to_ffmpeg
            self.browsertime_ffmpeg = path_to_ffmpeg
            self.info(
                "Added local ffmpeg found at: %s to environment." % path_to_ffmpeg
            )
        else:
            raise Exception(
                "No local ffmpeg binary found. Expected it to be here: %s"
                % path_to_ffmpeg
            )

    def install(self):
        if not self.config.get("noinstall", False):
            if self.app in self.firefox_android_browsers:
                self.device.uninstall_app(self.binary_path)

                # Check if the user supplied their own APK, and install
                # that instead
                installer_path = pathlib.Path(
                    self.raptor_path, "raptor", "user_upload.apk"
                )
                if not installer_path.exists():
                    installer_path = self.installer_path

                self.info(f"Installing APK from: {installer_path}")
                self.install_android_app(str(installer_path))
            else:
                super(Raptor, self).install()

    def _artifact_perf_data(self, src, dest):
        if not os.path.isdir(os.path.dirname(dest)):
            # create upload dir if it doesn't already exist
            self.info("Creating dir: %s" % os.path.dirname(dest))
            os.makedirs(os.path.dirname(dest))
        self.info("Copying raptor results from %s to %s" % (src, dest))
        try:
            copyfile(src, dest)
        except Exception as e:
            self.critical("Error copying results %s to upload dir %s" % (src, dest))
            self.info(str(e))

    def run_tests(self, args=None, **kw):
        """Run raptor tests"""

        # Get Raptor options
        options = self.raptor_options(args=args, **kw)

        # Python version check
        python = self.query_python_path()
        self.run_command([python, "--version"])
        parser = RaptorOutputParser(
            config=self.config, log_obj=self.log_obj, error_list=RaptorErrorList
        )
        env = {}
        env["MOZ_UPLOAD_DIR"] = self.query_abs_dirs()["abs_blob_upload_dir"]
        if not self.run_local:
            env["MINIDUMP_STACKWALK"] = self.query_minidump_stackwalk()
        env["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"]
        env["RUST_BACKTRACE"] = "full"
        if not os.path.isdir(env["MOZ_UPLOAD_DIR"]):
            self.mkdir_p(env["MOZ_UPLOAD_DIR"])
        env = self.query_env(partial_env=env, log_level=INFO)
        # adjust PYTHONPATH to be able to use raptor as a python package
        if "PYTHONPATH" in env:
            env["PYTHONPATH"] = self.raptor_path + os.pathsep + env["PYTHONPATH"]
        else:
            env["PYTHONPATH"] = self.raptor_path

        # mitmproxy needs path to mozharness when installing the cert, and tooltool
        env["SCRIPTSPATH"] = scripts_path
        env["EXTERNALTOOLSPATH"] = external_tools_path

        # Needed to load unsigned Raptor WebExt on release builds
        if self.is_release_build:
            env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"

        if self.repo_path is not None:
            env["MOZ_DEVELOPER_REPO_DIR"] = self.repo_path
        if self.obj_path is not None:
            env["MOZ_DEVELOPER_OBJ_DIR"] = self.obj_path
        if self.mozbuild_path is not None:
            env["MOZ_MOZBUILD_DIR"] = self.mozbuild_path

        # Sets a timeout for how long Raptor should run without output
        output_timeout = self.config.get("raptor_output_timeout", 3600)
        # Run Raptor tests
        run_tests = os.path.join(self.raptor_path, "raptor", "raptor.py")

        # Dynamically set the log level based on the raptor config for consistency
        # throughout the test
        mozlog_opts = [f"--log-tbpl-level={self.config['log_level']}"]

        if not self.run_local and "suite" in self.config:
            fname_pattern = "%s_%%s.log" % self.config["test"]
            mozlog_opts.append(
                "--log-errorsummary=%s"
                % os.path.join(env["MOZ_UPLOAD_DIR"], fname_pattern % "errorsummary")
            )

        def launch_in_debug_mode(cmdline):
            cmdline = set(cmdline)
            debug_opts = {"--debug", "--debugger", "--debugger_args"}

            return bool(debug_opts.intersection(cmdline))

        if self.app in self.android_browsers:
            self.logcat_start()

        command = [python, run_tests] + options + mozlog_opts
        if launch_in_debug_mode(command):
            raptor_process = subprocess.Popen(command, cwd=self.workdir, env=env)
            raptor_process.wait()
        else:
            self.return_code = self.run_command(
                command,
                cwd=self.workdir,
                output_timeout=output_timeout,
                output_parser=parser,
                env=env,
            )

        if self.app in self.android_browsers:
            self.logcat_stop()

        if parser.minidump_output:
            self.info("Looking at the minidump files for debugging purposes...")
            for item in parser.minidump_output:
                self.run_command(["ls", "-l", item])

        elif not self.run_local:
            # Copy results to upload dir so they are included as an artifact
            self.info("Copying Raptor results to upload dir:")

            src = os.path.join(self.query_abs_dirs()["abs_work_dir"], "raptor.json")
            dest = os.path.join(env["MOZ_UPLOAD_DIR"], "perfherder-data.json")
            self.info(str(dest))
            self._artifact_perf_data(src, dest)

            # Make individual perfherder data JSON's for each supporting data type
            for file in glob.glob(
                os.path.join(self.query_abs_dirs()["abs_work_dir"], "*")
            ):
                path, filename = os.path.split(file)

                if not filename.startswith("raptor-"):
                    continue

                # filename is expected to contain a unique data name
                # i.e. raptor-os-baseline-power.json would result in
                # the data name os-baseline-power
                data_name = "-".join(filename.split("-")[1:])
                data_name = ".".join(data_name.split(".")[:-1])

                src = file
                dest = os.path.join(
                    env["MOZ_UPLOAD_DIR"], "perfherder-data-%s.json" % data_name
                )
                self._artifact_perf_data(src, dest)

            src = os.path.join(
                self.query_abs_dirs()["abs_work_dir"], "screenshots.html"
            )
            if os.path.exists(src):
                dest = os.path.join(env["MOZ_UPLOAD_DIR"], "screenshots.html")
                self.info(str(dest))
                self._artifact_perf_data(src, dest)

        # Allow log failures to over-ride successful runs of the test harness and
        # give log failures priority, so that, for instance, log failures resulting
        # in TBPL_RETRY cause a retry rather than simply reporting an error.
        if parser.tbpl_status != TBPL_SUCCESS:
            parser_status = EXIT_STATUS_DICT[parser.tbpl_status]
            self.info(
                "return code %s changed to %s due to log output"
                % (str(self.return_code), str(parser_status))
            )
            self.return_code = parser_status


class RaptorOutputParser(OutputParser):
    minidump_regex = re.compile(
        r'''raptorError: "error executing: '(\S+) (\S+) (\S+)'"'''
    )
    RE_PERF_DATA = re.compile(r".*PERFHERDER_DATA:\s+(\{.*\})")

    def __init__(self, **kwargs):
        super(RaptorOutputParser, self).__init__(**kwargs)
        self.minidump_output = None
        self.found_perf_data = []
        self.tbpl_status = TBPL_SUCCESS
        self.worst_log_level = INFO
        self.harness_retry_re = TinderBoxPrintRe["harness_error"]["retry_regex"]

    def parse_single_line(self, line):
        m = self.minidump_regex.search(line)
        if m:
            self.minidump_output = (m.group(1), m.group(2), m.group(3))

        m = self.RE_PERF_DATA.match(line)
        if m:
            self.found_perf_data.append(m.group(1))

        if self.harness_retry_re.search(line):
            self.critical(" %s" % line)
            self.worst_log_level = self.worst_level(CRITICAL, self.worst_log_level)
            self.tbpl_status = self.worst_level(
                TBPL_RETRY, self.tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE
            )
            return  # skip base parse_single_line
        super(RaptorOutputParser, self).parse_single_line(line)