path: root/testing/mozharness/mozharness/mozilla/testing/talos.py

#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""
run talos tests in a virtualenv
"""

import copy
import io
import json
import multiprocessing
import os
import pprint
import re
import shutil
import subprocess
import sys

import six

import mozharness
from mozharness.base.config import parse_config_file
from mozharness.base.errors import PythonErrorList
from mozharness.base.log import CRITICAL, DEBUG, ERROR, INFO, WARNING, OutputParser
from mozharness.base.python import Python3Virtualenv
from mozharness.base.vcs.vcsbase import MercurialScript
from mozharness.mozilla.automation import (
    TBPL_FAILURE,
    TBPL_RETRY,
    TBPL_SUCCESS,
    TBPL_WARNING,
    TBPL_WORST_LEVEL_TUPLE,
)
from mozharness.mozilla.testing.codecoverage import (
    CodeCoverageMixin,
    code_coverage_config_options,
)
from mozharness.mozilla.testing.errors import TinderBoxPrintRe
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
from mozharness.mozilla.tooltool import TooltoolMixin

scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__)))
external_tools_path = os.path.join(scripts_path, "external_tools")

TalosErrorList = PythonErrorList + [
    {"regex": re.compile(r"""run-as: Package '.*' is unknown"""), "level": DEBUG},
    {"substr": r"""FAIL: Graph server unreachable""", "level": CRITICAL},
    {"substr": r"""FAIL: Busted:""", "level": CRITICAL},
    {"substr": r"""FAIL: failed to cleanup""", "level": ERROR},
    {"substr": r"""erfConfigurator.py: Unknown error""", "level": CRITICAL},
    {"substr": r"""talosError""", "level": CRITICAL},
    {
        "regex": re.compile(r"""No machine_name called '.*' can be found"""),
        "level": CRITICAL,
    },
    {
        "substr": r"""No such file or directory: 'browser_output.txt'""",
        "level": CRITICAL,
        "explanation": "Most likely the browser failed to launch, or the test was otherwise "
        "unsuccessful in even starting.",
    },
]

GeckoProfilerSettings = (
    "gecko_profile_interval",
    "gecko_profile_entries",
    "gecko_profile_features",
    "gecko_profile_threads",
)

# TODO: check for running processes on script invocation


class TalosOutputParser(OutputParser):
    minidump_regex = re.compile(
        r'''talosError: "error executing: '(\S+) (\S+) (\S+)'"'''
    )
    RE_PERF_DATA = re.compile(r".*PERFHERDER_DATA:\s+(\{.*\})")
    worst_tbpl_status = TBPL_SUCCESS

    def __init__(self, **kwargs):
        super(TalosOutputParser, self).__init__(**kwargs)
        self.minidump_output = None
        self.found_perf_data = []

    def update_worst_log_and_tbpl_levels(self, log_level, tbpl_level):
        self.worst_log_level = self.worst_level(log_level, self.worst_log_level)
        self.worst_tbpl_status = self.worst_level(
            tbpl_level, self.worst_tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE
        )

    def parse_single_line(self, line):
        """In Talos land, every line that starts with RETURN: needs to be
        printed with a TinderboxPrint:"""
        if line.startswith("RETURN:"):
            line.replace("RETURN:", "TinderboxPrint:")
        m = self.minidump_regex.search(line)
        if m:
            self.minidump_output = (m.group(1), m.group(2), m.group(3))

        m = self.RE_PERF_DATA.match(line)
        if m:
            self.found_perf_data.append(m.group(1))

        # now let's check if we should retry
        harness_retry_re = TinderBoxPrintRe["harness_error"]["retry_regex"]
        if harness_retry_re.search(line):
            self.critical(" %s" % line)
            self.update_worst_log_and_tbpl_levels(CRITICAL, TBPL_RETRY)
            return  # skip base parse_single_line
        super(TalosOutputParser, self).parse_single_line(line)


class Talos(
    TestingMixin, MercurialScript, TooltoolMixin, Python3Virtualenv, CodeCoverageMixin
):
    """
    install and run Talos tests
    """

    config_options = (
        [
            [
                ["--use-talos-json"],
                {
                    "action": "store_true",
                    "dest": "use_talos_json",
                    "default": False,
                    "help": "Use talos config from talos.json",
                },
            ],
            [
                ["--suite"],
                {
                    "action": "store",
                    "dest": "suite",
                    "help": "Talos suite to run (from talos json)",
                },
            ],
            [
                ["--system-bits"],
                {
                    "action": "store",
                    "dest": "system_bits",
                    "type": "choice",
                    "default": "32",
                    "choices": ["32", "64"],
                    "help": "Testing 32 or 64 (for talos json plugins)",
                },
            ],
            [
                ["--add-option"],
                {
                    "action": "extend",
                    "dest": "talos_extra_options",
                    "default": None,
                    "help": "extra options to talos",
                },
            ],
            [
                ["--gecko-profile"],
                {
                    "dest": "gecko_profile",
                    "action": "store_true",
                    "default": False,
                    "help": "Whether or not to profile the test run and save the profile results",
                },
            ],
            [
                ["--gecko-profile-interval"],
                {
                    "dest": "gecko_profile_interval",
                    "type": "int",
                    "help": "The interval between samples taken by the profiler (milliseconds)",
                },
            ],
            [
                ["--gecko-profile-entries"],
                {
                    "dest": "gecko_profile_entries",
                    "type": "int",
                    "help": "How many samples to take with the profiler",
                },
            ],
            [
                ["--gecko-profile-features"],
                {
                    "dest": "gecko_profile_features",
                    "type": "str",
                    "default": None,
                    "help": "The features to enable in the profiler (comma-separated)",
                },
            ],
            [
                ["--gecko-profile-threads"],
                {
                    "dest": "gecko_profile_threads",
                    "type": "str",
                    "help": "Comma-separated list of threads to sample.",
                },
            ],
            [
                ["--disable-e10s"],
                {
                    "dest": "e10s",
                    "action": "store_false",
                    "default": True,
                    "help": "Run without multiple processes (e10s).",
                },
            ],
            [
                ["--disable-fission"],
                {
                    "action": "store_false",
                    "dest": "fission",
                    "default": True,
                    "help": "Disable Fission (site isolation) in Gecko.",
                },
            ],
            [
                ["--project"],
                {
                    "dest": "project",
                    "type": "str",
                    "help": "The project branch we're running tests on. Used for "
                    "disabling/skipping tests.",
                },
            ],
            [
                ["--setpref"],
                {
                    "action": "append",
                    "metavar": "PREF=VALUE",
                    "dest": "extra_prefs",
                    "default": [],
                    "help": "Set a browser preference. May be used multiple times.",
                },
            ],
            [
                ["--skip-preflight"],
                {
                    "action": "store_true",
                    "dest": "skip_preflight",
                    "default": False,
                    "help": "skip preflight commands to prepare machine.",
                },
            ],
        ]
        + testing_config_options
        + copy.deepcopy(code_coverage_config_options)
    )

    def __init__(self, **kwargs):
        kwargs.setdefault("config_options", self.config_options)
        kwargs.setdefault(
            "all_actions",
            [
                "clobber",
                "download-and-extract",
                "populate-webroot",
                "create-virtualenv",
                "install",
                "run-tests",
            ],
        )
        kwargs.setdefault(
            "default_actions",
            [
                "clobber",
                "download-and-extract",
                "populate-webroot",
                "create-virtualenv",
                "install",
                "run-tests",
            ],
        )
        kwargs.setdefault("config", {})
        super(Talos, self).__init__(**kwargs)

        self.workdir = self.query_abs_dirs()["abs_work_dir"]  # convenience

        self.run_local = self.config.get("run_local")
        self.installer_url = self.config.get("installer_url")
        self.test_packages_url = self.config.get("test_packages_url")
        self.talos_json_url = self.config.get("talos_json_url")
        self.talos_json = self.config.get("talos_json")
        self.talos_json_config = self.config.get("talos_json_config")
        self.repo_path = self.config.get("repo_path")
        self.obj_path = self.config.get("obj_path")
        self.tests = None
        extra_opts = self.config.get("talos_extra_options", [])
        self.gecko_profile = (
            self.config.get("gecko_profile") or "--gecko-profile" in extra_opts
        )
        for setting in GeckoProfilerSettings:
            value = self.config.get(setting)
            arg = "--" + setting.replace("_", "-")
            if value is None:
                try:
                    value = extra_opts[extra_opts.index(arg) + 1]
                except ValueError:
                    pass  # Not found
            if value is not None:
                setattr(self, setting, value)
                if not self.gecko_profile:
                    self.warning("enabling Gecko profiler for %s setting!" % setting)
                    self.gecko_profile = True
        self.pagesets_name = None
        self.benchmark_zip = None
        self.webextensions_zip = None

    # We accept some configuration options from the try commit message in the format
    # mozharness: <options>
    # Example try commit message:
    #   mozharness: --gecko-profile try: <stuff>
    def query_gecko_profile_options(self):
        gecko_results = []
        # finally, if gecko_profile is set, we add that to the talos options
        if self.gecko_profile:
            gecko_results.append("--gecko-profile")
            for setting in GeckoProfilerSettings:
                value = getattr(self, setting, None)
                if value:
                    arg = "--" + setting.replace("_", "-")
                    gecko_results.extend([arg, str(value)])
        return gecko_results

    def query_abs_dirs(self):
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = super(Talos, self).query_abs_dirs()
        abs_dirs["abs_blob_upload_dir"] = os.path.join(
            abs_dirs["abs_work_dir"], "blobber_upload_dir"
        )
        abs_dirs["abs_test_install_dir"] = os.path.join(
            abs_dirs["abs_work_dir"], "tests"
        )
        self.abs_dirs = abs_dirs
        return self.abs_dirs

    def query_talos_json_config(self):
        """Return the talos json config."""
        if self.talos_json_config:
            return self.talos_json_config
        if not self.talos_json:
            self.talos_json = os.path.join(self.talos_path, "talos.json")
        self.talos_json_config = parse_config_file(self.talos_json)
        self.info(pprint.pformat(self.talos_json_config))
        return self.talos_json_config

    def make_talos_domain(self, host):
        return host + "-talos"

    def split_path(self, path):
        result = []
        while True:
            path, folder = os.path.split(path)
            if folder:
                result.append(folder)
                continue
            elif path:
                result.append(path)
            break

        result.reverse()
        return result

    def merge_paths(self, lhs, rhs):
        backtracks = 0
        for subdir in rhs:
            if subdir == "..":
                backtracks += 1
            else:
                break
        return lhs[:-backtracks] + rhs[backtracks:]

    def replace_relative_iframe_paths(self, directory, filename):
        """This will find iframes with relative paths and replace them with
        absolute paths containing domains derived from the original source's
        domain. This helps us better simulate real-world cases for fission
        """
        if not filename.endswith(".html"):
            return

        directory_pieces = self.split_path(directory)
        while directory_pieces and directory_pieces[0] != "fis":
            directory_pieces = directory_pieces[1:]
        path = os.path.join(directory, filename)

        # XXX: ugh, is there a better way to account for multiple encodings than just
        # trying each of them?
        encodings = ["utf-8", "latin-1"]
        iframe_pattern = re.compile(r'(iframe.*")(\.\./.*\.html)"')
        for encoding in encodings:
            try:
                with io.open(path, "r", encoding=encoding) as f:
                    content = f.read()

                def replace_iframe_src(match):
                    src = match.group(2)
                    split = self.split_path(src)
                    merged = self.merge_paths(directory_pieces, split)
                    host = merged[3]
                    site_origin_hash = self.make_talos_domain(host)
                    new_url = 'http://%s/%s"' % (
                        site_origin_hash,
                        "/".join(merged),  # pylint --py3k: W1649
                    )
                    self.info(
                        "Replacing %s with %s in iframe inside %s"
                        % (match.group(2), new_url, path)
                    )
                    return match.group(1) + new_url

                content = re.sub(iframe_pattern, replace_iframe_src, content)
                with io.open(path, "w", encoding=encoding) as f:
                    f.write(content)
                break
            except UnicodeDecodeError:
                pass

    def query_pagesets_name(self):
        """Certain suites require external pagesets to be downloaded and
        extracted.
        """
        if self.pagesets_name:
            return self.pagesets_name
        if self.query_talos_json_config() and self.suite is not None:
            self.pagesets_name = self.talos_json_config["suites"][self.suite].get(
                "pagesets_name"
            )
            self.pagesets_name_manifest = "tp5n-pageset.manifest"
            return self.pagesets_name

    def query_benchmark_zip(self):
        """Certain suites require external benchmarks to be downloaded and
        extracted.
        """
        if self.benchmark_zip:
            return self.benchmark_zip
        if self.query_talos_json_config() and self.suite is not None:
            self.benchmark_zip = self.talos_json_config["suites"][self.suite].get(
                "benchmark_zip"
            )
            self.benchmark_zip_manifest = "jetstream-benchmark.manifest"
            return self.benchmark_zip

    def query_webextensions_zip(self):
        """Certain suites require external WebExtension sets to be downloaded and
        extracted.
        """
        if self.webextensions_zip:
            return self.webextensions_zip
        if self.query_talos_json_config() and self.suite is not None:
            self.webextensions_zip = self.talos_json_config["suites"][self.suite].get(
                "webextensions_zip"
            )
            self.webextensions_zip_manifest = "webextensions.manifest"
            return self.webextensions_zip

    def get_suite_from_test(self):
        """Retrieve the talos suite name from a given talos test name."""
        # running locally, single test name provided instead of suite; go through tests and
        # find suite name
        suite_name = None
        if self.query_talos_json_config():
            if "-a" in self.config["talos_extra_options"]:
                test_name_index = self.config["talos_extra_options"].index("-a") + 1
            if "--activeTests" in self.config["talos_extra_options"]:
                test_name_index = (
                    self.config["talos_extra_options"].index("--activeTests") + 1
                )
            if test_name_index < len(self.config["talos_extra_options"]):
                test_name = self.config["talos_extra_options"][test_name_index]
                for talos_suite in self.talos_json_config["suites"]:
                    if test_name in self.talos_json_config["suites"][talos_suite].get(
                        "tests"
                    ):
                        suite_name = talos_suite
            if not suite_name:
                # no suite found to contain the specified test, error out
                self.fatal("Test name is missing or invalid")
        else:
            self.fatal("Talos json config not found, cannot verify suite")
        return suite_name

    def query_suite_extra_prefs(self):
        if self.query_talos_json_config() and self.suite is not None:
            return self.talos_json_config["suites"][self.suite].get("extra_prefs", [])

        return []

    def validate_suite(self):
        """Ensure suite name is a valid talos suite."""
        if self.query_talos_json_config() and self.suite is not None:
            if self.suite not in self.talos_json_config.get("suites"):
                self.fatal(
                    "Suite '%s' is not valid (not found in talos json config)"
                    % self.suite
                )

    def talos_options(self, args=None, **kw):
        """return options to talos"""
        # binary path
        binary_path = self.binary_path or self.config.get("binary_path")
        if not binary_path:
            msg = """Talos requires a path to the binary.  You can specify binary_path or add
            download-and-extract to your action list."""
            self.fatal(msg)

        # talos options
        options = []
        # talos can't gather data if the process name ends with '.exe'
        if binary_path.endswith(".exe"):
            binary_path = binary_path[:-4]
        # options overwritten from **kw
        kw_options = {"executablePath": binary_path}
        if "suite" in self.config:
            kw_options["suite"] = self.config["suite"]
        if self.config.get("title"):
            kw_options["title"] = self.config["title"]
        if self.symbols_path:
            kw_options["symbolsPath"] = self.symbols_path
        if self.config.get("project", None):
            kw_options["project"] = self.config["project"]

        kw_options.update(kw)
        # talos expects tests to be in the format (e.g.) 'ts:tp5:tsvg'
        tests = kw_options.get("activeTests")
        if tests and not isinstance(tests, six.string_types):
            tests = ":".join(tests)  # Talos expects this format
            kw_options["activeTests"] = tests
        for key, value in kw_options.items():
            options.extend(["--%s" % key, value])
        # configure profiling options
        options.extend(self.query_gecko_profile_options())
        # extra arguments
        if args is not None:
            options += args
        if "talos_extra_options" in self.config:
            options += self.config["talos_extra_options"]
        if self.config.get("code_coverage", False):
            options.extend(["--code-coverage"])

        # Add extra_prefs defined by individual test suites in talos.json
        extra_prefs = self.query_suite_extra_prefs()
        # Add extra_prefs from the configuration
        if self.config["extra_prefs"]:
            extra_prefs.extend(self.config["extra_prefs"])

        options.extend(["--setpref={}".format(p) for p in extra_prefs])

        # disabling fission can come from the --disable-fission cmd line argument; or in CI
        # it comes from a taskcluster transform which adds a --setpref for fission.autostart
        if (not self.config["fission"]) or "fission.autostart=false" in self.config[
            "extra_prefs"
        ]:
            options.extend(["--disable-fission"])

        return options

    def populate_webroot(self):
        """Populate the production test machines' webroots"""
        self.talos_path = os.path.join(
            self.query_abs_dirs()["abs_test_install_dir"], "talos"
        )

        # need to determine if talos pageset is required to be downloaded
        if self.config.get("run_local") and "talos_extra_options" in self.config:
            # talos initiated locally, get and verify test/suite from cmd line
            self.talos_path = os.path.dirname(self.talos_json)
            if (
                "-a" in self.config["talos_extra_options"]
                or "--activeTests" in self.config["talos_extra_options"]
            ):
                # test name (-a or --activeTests) specified, find out what suite it is a part of
                self.suite = self.get_suite_from_test()
            elif "--suite" in self.config["talos_extra_options"]:
                # --suite specified, get suite from cmd line and ensure is valid
                suite_name_index = (
                    self.config["talos_extra_options"].index("--suite") + 1
                )
                if suite_name_index < len(self.config["talos_extra_options"]):
                    self.suite = self.config["talos_extra_options"][suite_name_index]
                    self.validate_suite()
                else:
                    self.fatal("Suite name not provided")
        else:
            # talos initiated in production via mozharness
            self.suite = self.config["suite"]

        tooltool_artifacts = []
        src_talos_pageset_dest = os.path.join(self.talos_path, "talos", "tests")
        # unfortunately this path has to be short and can't be descriptive, because
        # on Windows we tend to already push the boundaries of the max path length
        # constraint. This will contain the tp5 pageset, but adjusted to have
        # absolute URLs on iframes for the purposes of better modeling things for
        # fission.
        src_talos_pageset_multidomain_dest = os.path.join(
            self.talos_path, "talos", "fis"
        )
        webextension_dest = os.path.join(self.talos_path, "talos", "webextensions")

        if self.query_pagesets_name():
            tooltool_artifacts.append(
                {
                    "name": self.pagesets_name,
                    "manifest": self.pagesets_name_manifest,
                    "dest": src_talos_pageset_dest,
                }
            )
            tooltool_artifacts.append(
                {
                    "name": self.pagesets_name,
                    "manifest": self.pagesets_name_manifest,
                    "dest": src_talos_pageset_multidomain_dest,
                    "postprocess": self.replace_relative_iframe_paths,
                }
            )

        if self.query_benchmark_zip():
            tooltool_artifacts.append(
                {
                    "name": self.benchmark_zip,
                    "manifest": self.benchmark_zip_manifest,
                    "dest": src_talos_pageset_dest,
                }
            )

        if self.query_webextensions_zip():
            tooltool_artifacts.append(
                {
                    "name": self.webextensions_zip,
                    "manifest": self.webextensions_zip_manifest,
                    "dest": webextension_dest,
                }
            )

        # now that have the suite name, check if artifact is required, if so download it
        # the --no-download option will override this
        for artifact in tooltool_artifacts:
            if "--no-download" not in self.config.get("talos_extra_options", []):
                self.info("Downloading %s with tooltool..." % artifact)

                archive = os.path.join(artifact["dest"], artifact["name"])
                output_dir_path = re.sub(r"\.zip$", "", archive)
                if not os.path.exists(archive):
                    manifest_file = os.path.join(self.talos_path, artifact["manifest"])
                    self.tooltool_fetch(
                        manifest_file,
                        output_dir=artifact["dest"],
                        cache=self.config.get("tooltool_cache"),
                    )
                    unzip = self.query_exe("unzip")
                    unzip_cmd = [unzip, "-q", "-o", archive, "-d", artifact["dest"]]
                    self.run_command(unzip_cmd, halt_on_failure=True)

                    if "postprocess" in artifact:
                        for subdir, dirs, files in os.walk(output_dir_path):
                            for file in files:
                                artifact["postprocess"](subdir, file)
                else:
                    self.info("%s already available" % artifact)

            else:
                self.info(
                    "Not downloading %s because the no-download option was specified"
                    % artifact
                )

        # if running webkit tests locally, need to copy webkit source into talos/tests
        if self.config.get("run_local") and (
            "stylebench" in self.suite or "motionmark" in self.suite
        ):
            self.get_webkit_source()

    def get_webkit_source(self):
        # in production the build system auto copies webkit source into place;
        # but when run locally we need to do this manually, so that talos can find it
        src = os.path.join(self.repo_path, "third_party", "webkit", "PerformanceTests")
        dest = os.path.join(
            self.talos_path, "talos", "tests", "webkit", "PerformanceTests"
        )

        if os.path.exists(dest):
            shutil.rmtree(dest)

        self.info("Copying webkit benchmarks from %s to %s" % (src, dest))
        try:
            shutil.copytree(src, dest)
        except Exception:
            self.critical("Error copying webkit benchmarks from %s to %s" % (src, dest))

    # Action methods. {{{1
    # clobber defined in BaseScript

    def download_and_extract(self, extract_dirs=None, suite_categories=None):
        # Use in-tree wptserve for Python 3.10 compatibility
        extract_dirs = [
            "tools/wptserve/*",
            "tools/wpt_third_party/pywebsocket3/*",
        ]
        return super(Talos, self).download_and_extract(
            extract_dirs=extract_dirs, suite_categories=["common", "talos"]
        )

    def create_virtualenv(self, **kwargs):
        """VirtualenvMixin.create_virtualenv() assuemes we're using
        self.config['virtualenv_modules']. Since we are installing
        talos from its source, we have to wrap that method here."""
        # if virtualenv already exists, just add to path and don't re-install, need it
        # in path so can import jsonschema later when validating output for perfherder
        _virtualenv_path = self.config.get("virtualenv_path")

        _python_interp = self.query_exe("python")
        if "win" in self.platform_name() and os.path.exists(_python_interp):
            multiprocessing.set_executable(_python_interp)

        if self.run_local and os.path.exists(_virtualenv_path):
            self.info("Virtualenv already exists, skipping creation")

            if "win" in self.platform_name():
                _path = os.path.join(_virtualenv_path, "Lib", "site-packages")
            else:
                _path = os.path.join(
                    _virtualenv_path,
                    "lib",
                    os.path.basename(_python_interp),
                    "site-packages",
                )

            sys.path.append(_path)
            return

        # virtualenv doesn't already exist so create it
        # install mozbase first, so we use in-tree versions
        # Additionally, decide where to pull talos requirements from.
        if not self.run_local:
            mozbase_requirements = os.path.join(
                self.query_abs_dirs()["abs_test_install_dir"],
                "config",
                "mozbase_requirements.txt",
            )
            talos_requirements = os.path.join(self.talos_path, "requirements.txt")
        else:
            mozbase_requirements = os.path.join(
                os.path.dirname(self.talos_path),
                "config",
                "mozbase_source_requirements.txt",
            )
            talos_requirements = os.path.join(
                self.talos_path, "source_requirements.txt"
            )
        self.register_virtualenv_module(
            requirements=[mozbase_requirements],
            two_pass=True,
            editable=True,
        )
        super(Talos, self).create_virtualenv()
        # talos in harness requires what else is
        # listed in talos requirements.txt file.
        self.install_module(requirements=[talos_requirements])

    def _validate_treeherder_data(self, parser):
        # late import is required, because install is done in create_virtualenv
        import jsonschema

        if len(parser.found_perf_data) != 1:
            self.critical(
                "PERFHERDER_DATA was seen %d times, expected 1."
                % len(parser.found_perf_data)
            )
            parser.update_worst_log_and_tbpl_levels(WARNING, TBPL_WARNING)
            return

        schema_path = os.path.join(
            external_tools_path, "performance-artifact-schema.json"
        )
        self.info("Validating PERFHERDER_DATA against %s" % schema_path)
        try:
            with open(schema_path) as f:
                schema = json.load(f)
            data = json.loads(parser.found_perf_data[0])
            jsonschema.validate(data, schema)
        except Exception:
            self.exception("Error while validating PERFHERDER_DATA")
            parser.update_worst_log_and_tbpl_levels(WARNING, TBPL_WARNING)

    def _artifact_perf_data(self, parser, dest):
        src = os.path.join(self.query_abs_dirs()["abs_work_dir"], "local.json")
        try:
            shutil.copyfile(src, dest)
        except Exception:
            self.critical("Error copying results %s to upload dir %s" % (src, dest))
            parser.update_worst_log_and_tbpl_levels(CRITICAL, TBPL_FAILURE)

    def run_tests(self, args=None, **kw):
        """run Talos tests"""

        # get talos options
        options = self.talos_options(args=args, **kw)

        # XXX temporary python version check
        python = self.query_python_path()
        self.run_command([python, "--version"])
        parser = TalosOutputParser(
            config=self.config, log_obj=self.log_obj, error_list=TalosErrorList
        )
        env = {}
        env["MOZ_UPLOAD_DIR"] = self.query_abs_dirs()["abs_blob_upload_dir"]
        if not self.run_local:
            env["MINIDUMP_STACKWALK"] = self.query_minidump_stackwalk()
        env["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"]
        env["RUST_BACKTRACE"] = "full"
        if not os.path.isdir(env["MOZ_UPLOAD_DIR"]):
            self.mkdir_p(env["MOZ_UPLOAD_DIR"])
        env = self.query_env(partial_env=env, log_level=INFO)
        # adjust PYTHONPATH to be able to use talos as a python package
        if "PYTHONPATH" in env:
            env["PYTHONPATH"] = self.talos_path + os.pathsep + env["PYTHONPATH"]
        else:
            env["PYTHONPATH"] = self.talos_path

        if self.repo_path is not None:
            env["MOZ_DEVELOPER_REPO_DIR"] = self.repo_path
        if self.obj_path is not None:
            env["MOZ_DEVELOPER_OBJ_DIR"] = self.obj_path

        # sets a timeout for how long talos should run without output
        output_timeout = self.config.get("talos_output_timeout", 3600)
        # run talos tests
        run_tests = os.path.join(self.talos_path, "talos", "run_tests.py")

        # Dynamically set the log level based on the talos config for consistency
        # throughout the test
        mozlog_opts = [f"--log-tbpl-level={self.config['log_level']}"]

        if not self.run_local and "suite" in self.config:
            fname_pattern = "%s_%%s.log" % self.config["suite"]
            mozlog_opts.append(
                "--log-errorsummary=%s"
                % os.path.join(env["MOZ_UPLOAD_DIR"], fname_pattern % "errorsummary")
            )

        def launch_in_debug_mode(cmdline):
            cmdline = set(cmdline)
            debug_opts = {"--debug", "--debugger", "--debugger_args"}

            return bool(debug_opts.intersection(cmdline))

        command = [python, run_tests] + options + mozlog_opts
        if launch_in_debug_mode(command):
            talos_process = subprocess.Popen(
                command, cwd=self.workdir, env=env, bufsize=0
            )
            talos_process.wait()
        else:
            self.return_code = self.run_command(
                command,
                cwd=self.workdir,
                output_timeout=output_timeout,
                output_parser=parser,
                env=env,
            )
        if parser.minidump_output:
            self.info("Looking at the minidump files for debugging purposes...")
            for item in parser.minidump_output:
                self.run_command(["ls", "-l", item])

        if self.return_code not in [0]:
            # update the worst log level and tbpl status
            log_level = ERROR
            tbpl_level = TBPL_FAILURE
            if self.return_code == 1:
                log_level = WARNING
                tbpl_level = TBPL_WARNING
            if self.return_code == 4:
                log_level = WARNING
                tbpl_level = TBPL_RETRY

            parser.update_worst_log_and_tbpl_levels(log_level, tbpl_level)
        elif "--no-upload-results" not in options:
            if not self.gecko_profile:
                self._validate_treeherder_data(parser)
                if not self.run_local:
                    # copy results to upload dir so they are included as an artifact
                    dest = os.path.join(env["MOZ_UPLOAD_DIR"], "perfherder-data.json")
                    self._artifact_perf_data(parser, dest)

        self.record_status(parser.worst_tbpl_status, level=parser.worst_log_level)