path: root/testing/mozharness/mozharness/mozilla/testing/per_test_base.py

#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****

import itertools
import json
import math
import os
import posixpath
import sys

import mozinfo


class SingleTestMixin(object):
    """Utility functions for per-test testing like test verification and per-test coverage."""

    def __init__(self, **kwargs):
        super(SingleTestMixin, self).__init__(**kwargs)

        self.suites = {}
        self.tests_downloaded = False
        self.reftest_test_dir = None
        self.jsreftest_test_dir = None
        # Map from full test path on the test machine to a relative path in the source checkout.
        # Use self._map_test_path_to_source(test_machine_path, source_path) to add a mapping.
        self.test_src_path = {}
        self.per_test_log_index = 1

    def _map_test_path_to_source(self, test_machine_path, source_path):
        test_machine_path = test_machine_path.replace(os.sep, posixpath.sep)
        source_path = source_path.replace(os.sep, posixpath.sep)
        self.test_src_path[test_machine_path] = source_path

    def _is_gpu_suite(self, suite):
        if suite and (suite == "gpu" or suite.startswith("webgl")):
            return True
        return False

    def _find_misc_tests(self, dirs, changed_files, gpu=False):
        manifests = [
            (
                os.path.join(dirs["abs_mochitest_dir"], "tests", "mochitest.toml"),
                "mochitest-plain",
            ),
            (
                os.path.join(dirs["abs_mochitest_dir"], "chrome", "chrome.toml"),
                "mochitest-chrome",
            ),
            (
                os.path.join(
                    dirs["abs_mochitest_dir"], "browser", "browser-chrome.toml"
                ),
                "mochitest-browser-chrome",
            ),
            (
                os.path.join(dirs["abs_mochitest_dir"], "a11y", "a11y.toml"),
                "mochitest-a11y",
            ),
            (
                os.path.join(dirs["abs_xpcshell_dir"], "tests", "xpcshell.toml"),
                "xpcshell",
            ),
        ]
        is_fission = "fission.autostart=true" in self.config.get("extra_prefs", [])
        tests_by_path = {}
        all_disabled = []
        #  HACK: import here so we don't need import for rest of class
        from manifestparser import TestManifest

        for path, suite in manifests:
            if os.path.exists(path):
                man = TestManifest([path], strict=False)
                active = man.active_tests(
                    exists=False, disabled=True, filters=[], **mozinfo.info
                )
                # Remove disabled tests. Also, remove tests with the same path as
                # disabled tests, even if they are not disabled, since per-test mode
                # specifies tests by path (it cannot distinguish between two or more
                # tests with the same path specified in multiple manifests).
                disabled = [t["relpath"] for t in active if "disabled" in t]
                all_disabled += disabled
                new_by_path = {
                    t["relpath"]: (suite, t.get("subsuite"), None)
                    for t in active
                    if "disabled" not in t and t["relpath"] not in disabled
                }
                tests_by_path.update(new_by_path)
                self.info(
                    "Per-test run updated with manifest %s (%d active, %d skipped)"
                    % (path, len(new_by_path), len(disabled))
                )

        ref_manifests = [
            (
                os.path.join(
                    dirs["abs_reftest_dir"],
                    "tests",
                    "layout",
                    "reftests",
                    "reftest.list",
                ),
                "reftest",
                "gpu",
            ),  # gpu
            (
                os.path.join(
                    dirs["abs_reftest_dir"],
                    "tests",
                    "testing",
                    "crashtest",
                    "crashtests.list",
                ),
                "crashtest",
                None,
            ),
        ]
        sys.path.append(dirs["abs_reftest_dir"])
        import manifest

        self.reftest_test_dir = os.path.join(dirs["abs_reftest_dir"], "tests")
        for path, suite, subsuite in ref_manifests:
            if os.path.exists(path):
                man = manifest.ReftestManifest()
                man.load(path)
                for t in man.tests:
                    relpath = os.path.relpath(t["path"], self.reftest_test_dir)
                    referenced = (
                        t["referenced-test"] if "referenced-test" in t else None
                    )
                    tests_by_path[relpath] = (suite, subsuite, referenced)
                    self._map_test_path_to_source(t["path"], relpath)
                self.info(
                    "Per-test run updated with manifest %s (%d tests)"
                    % (path, len(man.tests))
                )

        suite = "jsreftest"
        self.jsreftest_test_dir = os.path.join(
            dirs["abs_test_install_dir"], "jsreftest", "tests"
        )
        path = os.path.join(self.jsreftest_test_dir, "jstests.list")
        if os.path.exists(path):
            man = manifest.ReftestManifest()
            man.load(path)
            for t in man.files:
                # expect manifest test to look like:
                #    ".../tests/jsreftest/tests/jsreftest.html?test=test262/.../some_test.js"
                # while the test is in mercurial at:
                #    js/src/tests/test262/.../some_test.js
                epos = t.find("=")
                if epos > 0:
                    relpath = t[epos + 1 :]
                    test_path = os.path.join(self.jsreftest_test_dir, relpath)
                    relpath = os.path.join("js", "src", "tests", relpath)
                    self._map_test_path_to_source(test_path, relpath)
                    tests_by_path.update({relpath: (suite, None, None)})
                else:
                    self.warning("unexpected jsreftest test format: %s" % str(t))
            self.info(
                "Per-test run updated with manifest %s (%d tests)"
                % (path, len(man.files))
            )

        # for each changed file, determine if it is a test file, and what suite it is in
        for file in changed_files:
            # manifest paths use os.sep (like backslash on Windows) but
            # automation-relevance uses posixpath.sep
            file = file.replace(posixpath.sep, os.sep)
            entry = tests_by_path.get(file)
            if not entry:
                if file in all_disabled:
                    self.info("'%s' has been skipped on this platform." % file)
                if os.environ.get("MOZHARNESS_TEST_PATHS", None) is not None:
                    self.info("Per-test run could not find requested test '%s'" % file)
                continue

            if gpu and not self._is_gpu_suite(entry[1]):
                self.info(
                    "Per-test run (gpu) discarded non-gpu test %s (%s)"
                    % (file, entry[1])
                )
                continue
            elif not gpu and self._is_gpu_suite(entry[1]):
                self.info(
                    "Per-test run (non-gpu) discarded gpu test %s (%s)"
                    % (file, entry[1])
                )
                continue

            if is_fission and (
                (entry[0] == "mochitest-a11y") or (entry[0] == "mochitest-chrome")
            ):
                self.info(
                    "Per-test run (fission) discarded non-e10s test %s (%s)"
                    % (file, entry[0])
                )
                continue

            if entry[2] is not None and "about:" not in entry[2]:
                # Test name substitution, for reftest reference file handling:
                #  - if both test and reference modified, run the test file
                #  - if only reference modified, run the test file
                test_file = os.path.join(
                    os.path.dirname(file), os.path.basename(entry[2])
                )
                self.info("Per-test run substituting %s for %s" % (test_file, file))
                file = test_file

            self.info("Per-test run found test %s (%s/%s)" % (file, entry[0], entry[1]))
            subsuite_mapping = {
                # Map (<suite>, <subsuite>): <full-suite>
                #   <suite> is associated with a manifest, explicitly in code above
                #   <subsuite> comes from "subsuite" tags in some manifest entries
                #   <full-suite> is a unique id for the suite, matching desktop mozharness configs
                (
                    "mochitest-browser-chrome",
                    "a11y",
                    None,
                ): "mochitest-browser-a11y",
                (
                    "mochitest-browser-chrome",
                    "media-bc",
                    None,
                ): "mochitest-browser-media",
                (
                    "mochitest-browser-chrome",
                    "devtools",
                    None,
                ): "mochitest-devtools-chrome",
                ("mochitest-browser-chrome", "remote", None): "mochitest-remote",
                (
                    "mochitest-browser-chrome",
                    "screenshots",
                    None,
                ): "mochitest-browser-screenshots",  # noqa
                ("mochitest-plain", "media", None): "mochitest-media",
                # below should be on test-verify-gpu job
                ("mochitest-chrome", "gpu", None): "mochitest-chrome-gpu",
                ("mochitest-plain", "gpu", None): "mochitest-plain-gpu",
                ("mochitest-plain", "webgl1-core", None): "mochitest-webgl1-core",
                ("mochitest-plain", "webgl1-ext", None): "mochitest-webgl1-ext",
                ("mochitest-plain", "webgl2-core", None): "mochitest-webgl2-core",
                ("mochitest-plain", "webgl2-ext", None): "mochitest-webgl2-ext",
                ("mochitest-plain", "webgl2-deqp", None): "mochitest-webgl2-deqp",
                ("mochitest-plain", "webgpu", None): "mochitest-webgpu",
            }
            if entry in subsuite_mapping:
                suite = subsuite_mapping[entry]
            else:
                suite = entry[0]
            suite_files = self.suites.get(suite)
            if not suite_files:
                suite_files = []
            if file not in suite_files:
                suite_files.append(file)
            self.suites[suite] = suite_files

    def _find_wpt_tests(self, dirs, changed_files):
        # Setup sys.path to include all the dependencies required to import
        # the web-platform-tests manifest parser. web-platform-tests provides
        # the localpaths.py to do the path manipulation, which we load,
        # providing the __file__ variable so it can resolve the relative
        # paths correctly.
        paths_file = os.path.join(
            dirs["abs_wpttest_dir"], "tests", "tools", "localpaths.py"
        )
        with open(paths_file, "r") as f:
            exec(f.read(), {"__file__": paths_file})
        import manifest as wptmanifest

        tests_root = os.path.join(dirs["abs_wpttest_dir"], "tests")

        for extra in ("", "mozilla"):
            base_path = os.path.join(dirs["abs_wpttest_dir"], extra)
            man_path = os.path.join(base_path, "meta", "MANIFEST.json")
            man = wptmanifest.manifest.load(tests_root, man_path)
            self.info("Per-test run updated with manifest %s" % man_path)

            repo_tests_path = os.path.join("testing", "web-platform", extra, "tests")
            tests_path = os.path.join("tests", "web-platform", extra, "tests")
            for type, path, test in man:
                if type not in ["testharness", "reftest", "wdspec"]:
                    continue
                repo_path = os.path.join(repo_tests_path, path)
                # manifest paths use os.sep (like backslash on Windows) but
                # automation-relevance uses posixpath.sep
                repo_path = repo_path.replace(os.sep, posixpath.sep)
                if repo_path in changed_files:
                    self.info(
                        "Per-test run found web-platform test '%s', type %s"
                        % (path, type)
                    )
                    suite_files = self.suites.get(type)
                    if not suite_files:
                        suite_files = []
                    test_path = os.path.join(tests_path, path)
                    suite_files.append(test_path)
                    self.suites[type] = suite_files
                    self._map_test_path_to_source(test_path, repo_path)
                    changed_files.remove(repo_path)

        if os.environ.get("MOZHARNESS_TEST_PATHS", None) is not None:
            for file in changed_files:
                self.info(
                    "Per-test run could not find requested web-platform test '%s'"
                    % file
                )

    def find_modified_tests(self):
        """
        For each file modified on this push, determine if the modified file
        is a test, by searching test manifests. Populate self.suites
        with test files, organized by suite.

        This depends on test manifests, so can only run after test zips have
        been downloaded and extracted.
        """
        repository = os.environ.get("GECKO_HEAD_REPOSITORY")
        revision = os.environ.get("GECKO_HEAD_REV")
        if not repository or not revision:
            self.warning("unable to run tests in per-test mode: no repo or revision!")
            self.suites = {}
            self.tests_downloaded = True
            return

        def get_automationrelevance():
            response = self.load_json_url(url)
            return response

        dirs = self.query_abs_dirs()
        mozinfo.find_and_update_from_json(dirs["abs_test_install_dir"])
        e10s = self.config.get("e10s", False)
        mozinfo.update({"e10s": e10s})
        is_fission = "fission.autostart=true" in self.config.get("extra_prefs", [])
        mozinfo.update({"fission": is_fission})
        headless = self.config.get("headless", False)
        mozinfo.update({"headless": headless})
        if mozinfo.info["buildapp"] == "mobile/android":
            # extra android mozinfo normally comes from device queries, but this
            # code may run before the device is ready, so rely on configuration
            mozinfo.update(
                {"android_version": str(self.config.get("android_version", 24))}
            )
            mozinfo.update({"is_emulator": self.config.get("is_emulator", True)})
        mozinfo.update({"verify": True})
        self.info("Per-test run using mozinfo: %s" % str(mozinfo.info))

        # determine which files were changed on this push
        changed_files = set()
        url = "%s/json-automationrelevance/%s" % (repository.rstrip("/"), revision)
        contents = self.retry(get_automationrelevance, attempts=2, sleeptime=10)
        for c in contents["changesets"]:
            self.info(
                " {cset} {desc}".format(
                    cset=c["node"][0:12],
                    desc=c["desc"].splitlines()[0].encode("ascii", "ignore"),
                )
            )
            changed_files |= set(c["files"])
        changed_files = list(changed_files)

        # check specified test paths, as from 'mach try ... <path>'
        if os.environ.get("MOZHARNESS_TEST_PATHS", None) is not None:
            suite_to_paths = json.loads(os.environ["MOZHARNESS_TEST_PATHS"])
            specified_paths = itertools.chain.from_iterable(suite_to_paths.values())
            specified_paths = list(specified_paths)
            # filter the list of changed files to those found under the
            # specified path(s)
            changed_and_specified = set()
            for changed in changed_files:
                for specified in specified_paths:
                    if changed.startswith(specified):
                        changed_and_specified.add(changed)
                        break
            if changed_and_specified:
                changed_files = changed_and_specified
            else:
                # if specified paths do not match changed files, assume the
                # specified paths are explicitly requested tests
                changed_files = set()
                changed_files.update(specified_paths)
            self.info("Per-test run found explicit request in MOZHARNESS_TEST_PATHS:")
            self.info(str(changed_files))

        if self.config.get("per_test_category") == "web-platform":
            self._find_wpt_tests(dirs, changed_files)
        elif self.config.get("gpu_required", False) is not False:
            self._find_misc_tests(dirs, changed_files, gpu=True)
        else:
            self._find_misc_tests(dirs, changed_files)

        # per test mode run specific tests from any given test suite
        # _find_*_tests organizes tests to run into suites so we can
        # run each suite at a time

        # chunk files
        total_tests = sum([len(self.suites[x]) for x in self.suites])

        if total_tests == 0:
            self.warning("No tests to verify.")
            self.suites = {}
            self.tests_downloaded = True
            return

        files_per_chunk = total_tests / float(self.config.get("total_chunks", 1))
        files_per_chunk = int(math.ceil(files_per_chunk))

        chunk_number = int(self.config.get("this_chunk", 1))
        suites = {}
        start = (chunk_number - 1) * files_per_chunk
        end = chunk_number * files_per_chunk
        current = -1
        for suite in self.suites:
            for test in self.suites[suite]:
                current += 1
                if current >= start and current < end:
                    if suite not in suites:
                        suites[suite] = []
                    suites[suite].append(test)
            if current >= end:
                break

        self.suites = suites
        self.tests_downloaded = True

    def query_args(self, suite):
        """
        For the specified suite, return an array of command line arguments to
        be passed to test harnesses when running in per-test mode.

        Each array element is an array of command line arguments for a modified
        test in the suite.
        """
        # not in verify or per-test coverage mode: run once, with no additional args
        if not self.per_test_coverage and not self.verify_enabled:
            return [[]]

        files = []
        jsreftest_extra_dir = os.path.join("js", "src", "tests")
        # For some suites, the test path needs to be updated before passing to
        # the test harness.
        for file in self.suites.get(suite):
            if self.config.get("per_test_category") != "web-platform" and suite in [
                "reftest",
                "crashtest",
            ]:
                file = os.path.join(self.reftest_test_dir, file)
            elif (
                self.config.get("per_test_category") != "web-platform"
                and suite == "jsreftest"
            ):
                file = os.path.relpath(file, jsreftest_extra_dir)
                file = os.path.join(self.jsreftest_test_dir, file)

            if file is None:
                continue

            file = file.replace(os.sep, posixpath.sep)
            files.append(file)

        self.info("Per-test file(s) for '%s': %s" % (suite, files))

        args = []
        for file in files:
            cur = []

            cur.extend(self.coverage_args)
            cur.extend(self.verify_args)

            cur.append(file)
            args.append(cur)

        return args

    def query_per_test_category_suites(self, category, all_suites):
        """
        In per-test mode, determine which suites are active, for the given
        suite category.
        """
        suites = None
        if self.verify_enabled or self.per_test_coverage:
            if self.config.get("per_test_category") == "web-platform":
                suites = list(self.suites)
                self.info("Per-test suites: %s" % suites)
            elif all_suites and self.tests_downloaded:
                suites = dict(
                    (key, all_suites.get(key))
                    for key in self.suites
                    if key in all_suites.keys()
                )
                self.info("Per-test suites: %s" % suites)
            else:
                # Until test zips are downloaded, manifests are not available,
                # so it is not possible to determine which suites are active/
                # required for per-test mode; assume all suites from supported
                # suite categories are required.
                if category in ["mochitest", "xpcshell", "reftest"]:
                    suites = all_suites
        return suites

    def log_per_test_status(self, test_name, tbpl_status, log_level):
        """
        Log status of a single test. This will display in the
        Job Details pane in treeherder - a convenient summary of per-test mode.
        Special test name formatting is needed because treeherder truncates
        lines that are too long, and may remove duplicates after truncation.
        """
        max_test_name_len = 40
        if len(test_name) > max_test_name_len:
            head = test_name
            new = ""
            previous = None
            max_test_name_len = max_test_name_len - len(".../")
            while len(new) < max_test_name_len:
                head, tail = os.path.split(head)
                previous = new
                new = os.path.join(tail, new)
            test_name = os.path.join("...", previous or new)
            test_name = test_name.rstrip(os.path.sep)
        self.log(
            "TinderboxPrint: Per-test run of %s<br/>: %s" % (test_name, tbpl_status),
            level=log_level,
        )

    def get_indexed_logs(self, dir, test_suite):
        """
        Per-test tasks need distinct file names for the raw and errorsummary logs
        on each run.
        """
        index = ""
        if self.verify_enabled or self.per_test_coverage:
            index = "-test%d" % self.per_test_log_index
            self.per_test_log_index += 1
        raw_log_file = os.path.join(dir, "%s%s_raw.log" % (test_suite, index))
        error_summary_file = os.path.join(
            dir, "%s%s_errorsummary.log" % (test_suite, index)
        )
        return raw_log_file, error_summary_file