Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 477 insertions, 0 deletions

diff --git a/testing/mozharness/scripts/android_hardware_unittest.py b/testing/mozharness/scripts/android_hardware_unittest.py
new file mode 100644
index 0000000000..065d74557b
--- /dev/null
+++ b/testing/mozharness/scripts/android_hardware_unittest.py
@@ -0,0 +1,477 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+
+import copy
+import datetime
+import json
+import os
+import subprocess
+import sys
+
+# load modules from parent dir
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.log import WARNING
+from mozharness.base.script import BaseScript, PreScriptAction
+from mozharness.mozilla.automation import TBPL_RETRY
+from mozharness.mozilla.mozbase import MozbaseMixin
+from mozharness.mozilla.testing.android import AndroidMixin
+from mozharness.mozilla.testing.codecoverage import CodeCoverageMixin
+from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
+
+SUITE_DEFAULT_E10S = ["geckoview-junit", "mochitest", "reftest"]
+SUITE_NO_E10S = ["cppunittest", "gtest", "jittest"]
+SUITE_REPEATABLE = ["mochitest", "reftest", "xpcshell"]
+
+
+class AndroidHardwareTest(
+    TestingMixin, BaseScript, MozbaseMixin, CodeCoverageMixin, AndroidMixin
+):
+    config_options = [
+        [["--test-suite"], {"action": "store", "dest": "test_suite", "default": None}],
+        [
+            ["--adb-path"],
+            {
+                "action": "store",
+                "dest": "adb_path",
+                "default": None,
+                "help": "Path to adb",
+            },
+        ],
+        [
+            ["--total-chunk"],
+            {
+                "action": "store",
+                "dest": "total_chunks",
+                "default": None,
+                "help": "Number of total chunks",
+            },
+        ],
+        [
+            ["--this-chunk"],
+            {
+                "action": "store",
+                "dest": "this_chunk",
+                "default": None,
+                "help": "Number of this chunk",
+            },
+        ],
+        [
+            ["--log-raw-level"],
+            {
+                "action": "store",
+                "dest": "log_raw_level",
+                "default": "info",
+                "help": "Set log level (debug|info|warning|error|critical|fatal)",
+            },
+        ],
+        [
+            ["--log-tbpl-level"],
+            {
+                "action": "store",
+                "dest": "log_tbpl_level",
+                "default": "info",
+                "help": "Set log level (debug|info|warning|error|critical|fatal)",
+            },
+        ],
+        [
+            ["--disable-e10s"],
+            {
+                "action": "store_false",
+                "dest": "e10s",
+                "default": True,
+                "help": "Run tests without multiple processes (e10s).",
+            },
+        ],
+        [
+            ["--disable-fission"],
+            {
+                "action": "store_true",
+                "dest": "disable_fission",
+                "default": False,
+                "help": "Run with Fission disabled.",
+            },
+        ],
+        [
+            ["--repeat"],
+            {
+                "action": "store",
+                "type": "int",
+                "dest": "repeat",
+                "default": 0,
+                "help": "Repeat the tests the given number of times. Supported "
+                "by mochitest, reftest, crashtest, ignored otherwise.",
+            },
+        ],
+        [
+            [
+                "--setpref",
+            ],
+            {
+                "action": "append",
+                "dest": "extra_prefs",
+                "default": [],
+                "help": "Extra user prefs.",
+            },
+        ],
+        [
+            ["--jittest-flags"],
+            {
+                "action": "store",
+                "dest": "jittest_flags",
+                "default": "debug",
+                "help": "Flags to run with jittest (all, debug, etc.).",
+            },
+        ],
+    ] + copy.deepcopy(testing_config_options)
+
+    def __init__(self, require_config_file=False):
+        super(AndroidHardwareTest, self).__init__(
+            config_options=self.config_options,
+            all_actions=[
+                "clobber",
+                "download-and-extract",
+                "create-virtualenv",
+                "verify-device",
+                "install",
+                "run-tests",
+            ],
+            require_config_file=require_config_file,
+            config={
+                "virtualenv_modules": [],
+                "virtualenv_requirements": [],
+                "require_test_zip": True,
+                # IP address of the host as seen from the device.
+                "remote_webserver": os.environ["HOST_IP"],
+            },
+        )
+
+        # these are necessary since self.config is read only
+        c = self.config
+        self.installer_url = c.get("installer_url")
+        self.installer_path = c.get("installer_path")
+        self.test_url = c.get("test_url")
+        self.test_packages_url = c.get("test_packages_url")
+        self.test_manifest = c.get("test_manifest")
+        suite = c.get("test_suite")
+        self.test_suite = suite
+        self.this_chunk = c.get("this_chunk")
+        self.total_chunks = c.get("total_chunks")
+        self.xre_path = None
+        self.log_raw_level = c.get("log_raw_level")
+        self.log_tbpl_level = c.get("log_tbpl_level")
+        self.disable_e10s = c.get("disable_e10s")
+        self.disable_fission = c.get("disable_fission")
+        self.extra_prefs = c.get("extra_prefs")
+        self.jittest_flags = c.get("jittest_flags")
+
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+        abs_dirs = super(AndroidHardwareTest, self).query_abs_dirs()
+        dirs = {}
+        dirs["abs_test_install_dir"] = os.path.join(abs_dirs["abs_work_dir"], "tests")
+        dirs["abs_test_bin_dir"] = os.path.join(
+            abs_dirs["abs_work_dir"], "tests", "bin"
+        )
+        dirs["abs_xre_dir"] = os.path.join(abs_dirs["abs_work_dir"], "hostutils")
+        dirs["abs_modules_dir"] = os.path.join(dirs["abs_test_install_dir"], "modules")
+        dirs["abs_blob_upload_dir"] = os.path.join(
+            abs_dirs["abs_work_dir"], "blobber_upload_dir"
+        )
+        dirs["abs_mochitest_dir"] = os.path.join(
+            dirs["abs_test_install_dir"], "mochitest"
+        )
+        dirs["abs_reftest_dir"] = os.path.join(dirs["abs_test_install_dir"], "reftest")
+        dirs["abs_xpcshell_dir"] = os.path.join(
+            dirs["abs_test_install_dir"], "xpcshell"
+        )
+
+        for key in dirs.keys():
+            if key not in abs_dirs:
+                abs_dirs[key] = dirs[key]
+        self.abs_dirs = abs_dirs
+        return self.abs_dirs
+
+    def _query_tests_dir(self):
+        dirs = self.query_abs_dirs()
+        try:
+            test_dir = self.config["suite_definitions"][self.test_suite]["testsdir"]
+        except Exception:
+            test_dir = self.test_suite
+        return os.path.join(dirs["abs_test_install_dir"], test_dir)
+
+    def _build_command(self):
+        c = self.config
+        dirs = self.query_abs_dirs()
+
+        if self.test_suite not in self.config["suite_definitions"]:
+            self.fatal("Key '%s' not defined in the config!" % self.test_suite)
+
+        cmd = [
+            self.query_python_path("python"),
+            "-u",
+            os.path.join(
+                self._query_tests_dir(),
+                self.config["suite_definitions"][self.test_suite]["run_filename"],
+            ),
+        ]
+
+        raw_log_file, error_summary_file = self.get_indexed_logs(
+            dirs["abs_blob_upload_dir"], self.test_suite
+        )
+
+        str_format_values = {
+            "device_serial": self.device_serial,
+            "remote_webserver": c["remote_webserver"],
+            "xre_path": self.xre_path,
+            "utility_path": self.xre_path,
+            "http_port": "8854",  # starting http port  to use for the mochitest server
+            "ssl_port": "4454",  # starting ssl port to use for the server
+            "certs_path": os.path.join(dirs["abs_work_dir"], "tests/certs"),
+            # TestingMixin._download_and_extract_symbols() will set
+            # self.symbols_path when downloading/extracting.
+            "symbols_path": self.symbols_path,
+            "modules_dir": dirs["abs_modules_dir"],
+            "installer_path": self.installer_path,
+            "raw_log_file": raw_log_file,
+            "log_tbpl_level": self.log_tbpl_level,
+            "log_raw_level": self.log_raw_level,
+            "error_summary_file": error_summary_file,
+            "xpcshell_extra": c.get("xpcshell_extra", ""),
+            "jittest_flags": self.jittest_flags,
+        }
+
+        user_paths = json.loads(os.environ.get("MOZHARNESS_TEST_PATHS", '""'))
+
+        for option in self.config["suite_definitions"][self.test_suite]["options"]:
+            opt = option.split("=")[0]
+            # override configured chunk options with script args, if specified
+            if opt in ("--this-chunk", "--total-chunks"):
+                if (
+                    user_paths
+                    or getattr(self, opt.replace("-", "_").strip("_"), None) is not None
+                ):
+                    continue
+
+            if "%(app)" in option:
+                # only query package name if requested
+                cmd.extend([option % {"app": self.query_package_name()}])
+            else:
+                option = option % str_format_values
+                if option:
+                    cmd.extend([option])
+
+        if user_paths:
+            if self.test_suite in user_paths:
+                cmd.extend(user_paths[self.test_suite])
+        elif not self.verify_enabled:
+            if self.this_chunk is not None:
+                cmd.extend(["--this-chunk", self.this_chunk])
+            if self.total_chunks is not None:
+                cmd.extend(["--total-chunks", self.total_chunks])
+
+        if "mochitest" in self.test_suite:
+            category = "mochitest"
+        elif "reftest" in self.test_suite or "crashtest" in self.test_suite:
+            category = "reftest"
+        else:
+            category = self.test_suite
+        if c.get("repeat"):
+            if category in SUITE_REPEATABLE:
+                cmd.extend(["--repeat=%s" % c.get("repeat")])
+            else:
+                self.log("--repeat not supported in {}".format(category), level=WARNING)
+
+        if category not in SUITE_NO_E10S:
+            if category in SUITE_DEFAULT_E10S and not c["e10s"]:
+                cmd.append("--disable-e10s")
+            elif category not in SUITE_DEFAULT_E10S and c["e10s"]:
+                cmd.append("--e10s")
+
+        if self.disable_fission and category not in SUITE_NO_E10S:
+            cmd.append("--disable-fission")
+
+        cmd.extend(["--setpref={}".format(p) for p in self.extra_prefs])
+
+        try_options, try_tests = self.try_args(self.test_suite)
+        cmd.extend(try_options)
+        if not self.verify_enabled and not self.per_test_coverage:
+            cmd.extend(
+                self.query_tests_args(
+                    self.config["suite_definitions"][self.test_suite].get("tests"),
+                    None,
+                    try_tests,
+                )
+            )
+
+        return cmd
+
+    def _query_suites(self):
+        if self.test_suite:
+            return [(self.test_suite, self.test_suite)]
+        # per-test mode: determine test suites to run
+        all = [
+            (
+                "mochitest",
+                {
+                    "mochitest-plain": "mochitest-plain",
+                    "mochitest-plain-gpu": "mochitest-plain-gpu",
+                },
+            ),
+            ("reftest", {"reftest": "reftest", "crashtest": "crashtest"}),
+            ("xpcshell", {"xpcshell": "xpcshell"}),
+        ]
+        suites = []
+        for (category, all_suites) in all:
+            cat_suites = self.query_per_test_category_suites(category, all_suites)
+            for k in cat_suites.keys():
+                suites.append((k, cat_suites[k]))
+        return suites
+
+    def _query_suite_categories(self):
+        if self.test_suite:
+            categories = [self.test_suite]
+        else:
+            # per-test mode
+            categories = ["mochitest", "reftest", "xpcshell"]
+        return categories
+
+    ##########################################
+    # Actions for AndroidHardwareTest        #
+    ##########################################
+
+    def preflight_install(self):
+        # in the base class, this checks for mozinstall, but we don't use it
+        pass
+
+    @PreScriptAction("create-virtualenv")
+    def pre_create_virtualenv(self, action):
+        dirs = self.query_abs_dirs()
+        requirements = None
+        suites = self._query_suites()
+        if ("mochitest-media", "mochitest-media") in suites:
+            # mochitest-media is the only thing that needs this
+            requirements = os.path.join(
+                dirs["abs_mochitest_dir"],
+                "websocketprocessbridge",
+                "websocketprocessbridge_requirements_3.txt",
+            )
+        if requirements:
+            self.register_virtualenv_module(requirements=[requirements], two_pass=True)
+
+    def download_and_extract(self):
+        """
+        Download and extract product APK, tests.zip, and host utils.
+        """
+        super(AndroidHardwareTest, self).download_and_extract(
+            suite_categories=self._query_suite_categories()
+        )
+        dirs = self.query_abs_dirs()
+        self.xre_path = self.download_hostutils(dirs["abs_xre_dir"])
+
+    def install(self):
+        """
+        Install APKs on the device.
+        """
+        install_needed = (not self.test_suite) or self.config["suite_definitions"][
+            self.test_suite
+        ].get("install")
+        if install_needed is False:
+            self.info("Skipping apk installation for %s" % self.test_suite)
+            return
+        assert (
+            self.installer_path is not None
+        ), "Either add installer_path to the config or use --installer-path."
+        self.uninstall_android_app()
+        self.install_android_app(self.installer_path)
+        self.info("Finished installing apps for %s" % self.device_name)
+
+    def run_tests(self):
+        """
+        Run the tests
+        """
+        self.start_time = datetime.datetime.now()
+        max_per_test_time = datetime.timedelta(minutes=60)
+
+        per_test_args = []
+        suites = self._query_suites()
+        minidump = self.query_minidump_stackwalk()
+        for (per_test_suite, suite) in suites:
+            self.test_suite = suite
+
+            try:
+                cwd = self._query_tests_dir()
+            except Exception:
+                self.fatal("Don't know how to run --test-suite '%s'!" % self.test_suite)
+            env = self.query_env()
+            if minidump:
+                env["MINIDUMP_STACKWALK"] = minidump
+            env["MOZ_UPLOAD_DIR"] = self.query_abs_dirs()["abs_blob_upload_dir"]
+            env["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"]
+            env["RUST_BACKTRACE"] = "full"
+
+            summary = None
+            for per_test_args in self.query_args(per_test_suite):
+                if (datetime.datetime.now() - self.start_time) > max_per_test_time:
+                    # Running tests has run out of time. That is okay! Stop running
+                    # them so that a task timeout is not triggered, and so that
+                    # (partial) results are made available in a timely manner.
+                    self.info(
+                        "TinderboxPrint: Running tests took too long: "
+                        "Not all tests were executed.<br/>"
+                    )
+                    # Signal per-test time exceeded, to break out of suites and
+                    # suite categories loops also.
+                    return
+
+                cmd = self._build_command()
+                final_cmd = copy.copy(cmd)
+                if len(per_test_args) > 0:
+                    # in per-test mode, remove any chunk arguments from command
+                    for arg in final_cmd:
+                        if "total-chunk" in arg or "this-chunk" in arg:
+                            final_cmd.remove(arg)
+                final_cmd.extend(per_test_args)
+
+                self.info(
+                    "Running on %s the command %s"
+                    % (self.device_name, subprocess.list2cmdline(final_cmd))
+                )
+                self.info("##### %s log begins" % self.test_suite)
+
+                suite_category = self.test_suite
+                parser = self.get_test_output_parser(
+                    suite_category,
+                    config=self.config,
+                    log_obj=self.log_obj,
+                    error_list=[],
+                )
+                self.run_command(final_cmd, cwd=cwd, env=env, output_parser=parser)
+                tbpl_status, log_level, summary = parser.evaluate_parser(0, summary)
+                parser.append_tinderboxprint_line(self.test_suite)
+
+                self.info("##### %s log ends" % self.test_suite)
+
+                if len(per_test_args) > 0:
+                    self.record_status(tbpl_status, level=log_level)
+                    self.log_per_test_status(per_test_args[-1], tbpl_status, log_level)
+                    if tbpl_status == TBPL_RETRY:
+                        self.info("Per-test run abandoned due to RETRY status")
+                        return
+                else:
+                    self.record_status(tbpl_status, level=log_level)
+                    # report as INFO instead of log_level to avoid extra Treeherder lines
+                    self.info(
+                        "The %s suite: %s ran with return status: %s"
+                        % (suite_category, suite, tbpl_status),
+                    )
+
+
+if __name__ == "__main__":
+    test = AndroidHardwareTest()
+    test.run_and_exit()