summaryrefslogtreecommitdiffstats
path: root/testing/mozharness/scripts/android_emulator_unittest.py
diff options
context:
space:
mode:
Diffstat (limited to 'testing/mozharness/scripts/android_emulator_unittest.py')
-rw-r--r--testing/mozharness/scripts/android_emulator_unittest.py550
1 files changed, 550 insertions, 0 deletions
diff --git a/testing/mozharness/scripts/android_emulator_unittest.py b/testing/mozharness/scripts/android_emulator_unittest.py
new file mode 100644
index 0000000000..47cf13dde3
--- /dev/null
+++ b/testing/mozharness/scripts/android_emulator_unittest.py
@@ -0,0 +1,550 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+
+import copy
+import datetime
+import json
+import os
+import subprocess
+import sys
+
+# load modules from parent dir
+here = os.path.abspath(os.path.dirname(__file__))
+sys.path.insert(1, os.path.dirname(here))
+
+from mozharness.base.log import WARNING
+from mozharness.base.script import BaseScript, PreScriptAction
+from mozharness.mozilla.automation import TBPL_RETRY
+from mozharness.mozilla.mozbase import MozbaseMixin
+from mozharness.mozilla.testing.android import AndroidMixin
+from mozharness.mozilla.testing.codecoverage import (
+ CodeCoverageMixin,
+ code_coverage_config_options,
+)
+from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
+
+SUITE_DEFAULT_E10S = ["geckoview-junit", "mochitest", "reftest"]
+SUITE_NO_E10S = ["cppunittest", "gtest", "jittest", "xpcshell"]
+SUITE_REPEATABLE = ["mochitest", "reftest", "xpcshell"]
+
+
+class AndroidEmulatorTest(
+ TestingMixin, BaseScript, MozbaseMixin, CodeCoverageMixin, AndroidMixin
+):
+ """
+ A mozharness script for Android functional tests (like mochitests and reftests)
+ run on an Android emulator. This script starts and manages an Android emulator
+ for the duration of the required tests. This is like desktop_unittest.py, but
+ for Android emulator test platforms.
+ """
+
+ config_options = (
+ [
+ [
+ ["--test-suite"],
+ {"action": "store", "dest": "test_suite", "default": None},
+ ],
+ [
+ ["--total-chunk"],
+ {
+ "action": "store",
+ "dest": "total_chunks",
+ "default": None,
+ "help": "Number of total chunks",
+ },
+ ],
+ [
+ ["--this-chunk"],
+ {
+ "action": "store",
+ "dest": "this_chunk",
+ "default": None,
+ "help": "Number of this chunk",
+ },
+ ],
+ [
+ ["--enable-xorigin-tests"],
+ {
+ "action": "store_true",
+ "dest": "enable_xorigin_tests",
+ "default": False,
+ "help": "Run tests in a cross origin iframe.",
+ },
+ ],
+ [
+ ["--gpu-required"],
+ {
+ "action": "store_true",
+ "dest": "gpu_required",
+ "default": False,
+ "help": "Run additional verification on modified tests using gpu instances.",
+ },
+ ],
+ [
+ ["--log-raw-level"],
+ {
+ "action": "store",
+ "dest": "log_raw_level",
+ "default": "info",
+ "help": "Set log level (debug|info|warning|error|critical|fatal)",
+ },
+ ],
+ [
+ ["--log-tbpl-level"],
+ {
+ "action": "store",
+ "dest": "log_tbpl_level",
+ "default": "info",
+ "help": "Set log level (debug|info|warning|error|critical|fatal)",
+ },
+ ],
+ [
+ ["--disable-e10s"],
+ {
+ "action": "store_false",
+ "dest": "e10s",
+ "default": True,
+ "help": "Run tests without multiple processes (e10s).",
+ },
+ ],
+ [
+ ["--disable-fission"],
+ {
+ "action": "store_true",
+ "dest": "disable_fission",
+ "default": False,
+ "help": "Run without Fission enabled.",
+ },
+ ],
+ [
+ ["--web-content-isolation-strategy"],
+ {
+ "action": "store",
+ "type": "int",
+ "dest": "web_content_isolation_strategy",
+ "help": "Strategy used to determine whether or not a particular site should"
+ "load into a webIsolated content process, see "
+ "fission.webContentIsolationStrategy.",
+ },
+ ],
+ [
+ ["--repeat"],
+ {
+ "action": "store",
+ "type": "int",
+ "dest": "repeat",
+ "default": 0,
+ "help": "Repeat the tests the given number of times. Supported "
+ "by mochitest, reftest, crashtest, ignored otherwise.",
+ },
+ ],
+ [
+ ["--setpref"],
+ {
+ "action": "append",
+ "metavar": "PREF=VALUE",
+ "dest": "extra_prefs",
+ "default": [],
+ "help": "Extra user prefs.",
+ },
+ ],
+ ]
+ + copy.deepcopy(testing_config_options)
+ + copy.deepcopy(code_coverage_config_options)
+ )
+
+ def __init__(self, require_config_file=False):
+ super(AndroidEmulatorTest, self).__init__(
+ config_options=self.config_options,
+ all_actions=[
+ "clobber",
+ "download-and-extract",
+ "create-virtualenv",
+ "start-emulator",
+ "verify-device",
+ "install",
+ "run-tests",
+ ],
+ require_config_file=require_config_file,
+ config={
+ "virtualenv_modules": [],
+ "virtualenv_requirements": [],
+ "require_test_zip": True,
+ },
+ )
+
+ # these are necessary since self.config is read only
+ c = self.config
+ self.installer_url = c.get("installer_url")
+ self.installer_path = c.get("installer_path")
+ self.test_url = c.get("test_url")
+ self.test_packages_url = c.get("test_packages_url")
+ self.test_manifest = c.get("test_manifest")
+ suite = c.get("test_suite")
+ self.test_suite = suite
+ self.this_chunk = c.get("this_chunk")
+ self.total_chunks = c.get("total_chunks")
+ self.xre_path = None
+ self.device_serial = "emulator-5554"
+ self.log_raw_level = c.get("log_raw_level")
+ self.log_tbpl_level = c.get("log_tbpl_level")
+ # AndroidMixin uses this when launching the emulator. We only want
+ # GLES3 if we're running WebRender (default)
+ self.use_gles3 = True
+ self.disable_e10s = c.get("disable_e10s")
+ self.disable_fission = c.get("disable_fission")
+ self.web_content_isolation_strategy = c.get("web_content_isolation_strategy")
+ self.extra_prefs = c.get("extra_prefs")
+
+ def query_abs_dirs(self):
+ if self.abs_dirs:
+ return self.abs_dirs
+ abs_dirs = super(AndroidEmulatorTest, self).query_abs_dirs()
+ dirs = {}
+ dirs["abs_test_install_dir"] = os.path.join(abs_dirs["abs_work_dir"], "tests")
+ dirs["abs_test_bin_dir"] = os.path.join(
+ abs_dirs["abs_work_dir"], "tests", "bin"
+ )
+ dirs["abs_xre_dir"] = os.path.join(abs_dirs["abs_work_dir"], "hostutils")
+ dirs["abs_modules_dir"] = os.path.join(dirs["abs_test_install_dir"], "modules")
+ dirs["abs_blob_upload_dir"] = os.path.join(
+ abs_dirs["abs_work_dir"], "blobber_upload_dir"
+ )
+ dirs["abs_mochitest_dir"] = os.path.join(
+ dirs["abs_test_install_dir"], "mochitest"
+ )
+ dirs["abs_reftest_dir"] = os.path.join(dirs["abs_test_install_dir"], "reftest")
+ dirs["abs_xpcshell_dir"] = os.path.join(
+ dirs["abs_test_install_dir"], "xpcshell"
+ )
+ work_dir = os.environ.get("MOZ_FETCHES_DIR") or abs_dirs["abs_work_dir"]
+ dirs["abs_sdk_dir"] = os.path.join(work_dir, "android-sdk-linux")
+ dirs["abs_avds_dir"] = os.path.join(work_dir, "android-device")
+ dirs["abs_bundletool_path"] = os.path.join(work_dir, "bundletool.jar")
+
+ for key in dirs.keys():
+ if key not in abs_dirs:
+ abs_dirs[key] = dirs[key]
+ self.abs_dirs = abs_dirs
+ return self.abs_dirs
+
+ def _query_tests_dir(self, test_suite):
+ dirs = self.query_abs_dirs()
+ try:
+ test_dir = self.config["suite_definitions"][test_suite]["testsdir"]
+ except Exception:
+ test_dir = test_suite
+ return os.path.join(dirs["abs_test_install_dir"], test_dir)
+
+ def _get_mozharness_test_paths(self, suite):
+ test_paths = os.environ.get("MOZHARNESS_TEST_PATHS")
+ if not test_paths:
+ return
+
+ return json.loads(test_paths).get(suite)
+
+ def _build_command(self):
+ c = self.config
+ dirs = self.query_abs_dirs()
+
+ if self.test_suite not in self.config["suite_definitions"]:
+ self.fatal("Key '%s' not defined in the config!" % self.test_suite)
+
+ cmd = [
+ self.query_python_path("python"),
+ "-u",
+ os.path.join(
+ self._query_tests_dir(self.test_suite),
+ self.config["suite_definitions"][self.test_suite]["run_filename"],
+ ),
+ ]
+
+ raw_log_file, error_summary_file = self.get_indexed_logs(
+ dirs["abs_blob_upload_dir"], self.test_suite
+ )
+
+ str_format_values = {
+ "device_serial": self.device_serial,
+ # IP address of the host as seen from the emulator
+ "remote_webserver": "10.0.2.2",
+ "xre_path": self.xre_path,
+ "utility_path": self.xre_path,
+ "http_port": "8854", # starting http port to use for the mochitest server
+ "ssl_port": "4454", # starting ssl port to use for the server
+ "certs_path": os.path.join(dirs["abs_work_dir"], "tests/certs"),
+ # TestingMixin._download_and_extract_symbols() will set
+ # self.symbols_path when downloading/extracting.
+ "symbols_path": self.symbols_path,
+ "modules_dir": dirs["abs_modules_dir"],
+ "installer_path": self.installer_path,
+ "raw_log_file": raw_log_file,
+ "log_tbpl_level": self.log_tbpl_level,
+ "log_raw_level": self.log_raw_level,
+ "error_summary_file": error_summary_file,
+ "xpcshell_extra": c.get("xpcshell_extra", ""),
+ "gtest_dir": os.path.join(dirs["abs_test_install_dir"], "gtest"),
+ }
+
+ user_paths = self._get_mozharness_test_paths(self.test_suite)
+
+ for option in self.config["suite_definitions"][self.test_suite]["options"]:
+ opt = option.split("=")[0]
+ # override configured chunk options with script args, if specified
+ if opt in ("--this-chunk", "--total-chunks"):
+ if (
+ user_paths
+ or getattr(self, opt.replace("-", "_").strip("_"), None) is not None
+ ):
+ continue
+
+ if "%(app)" in option:
+ # only query package name if requested
+ cmd.extend([option % {"app": self.query_package_name()}])
+ else:
+ option = option % str_format_values
+ if option:
+ cmd.extend([option])
+
+ if "mochitest" in self.test_suite:
+ category = "mochitest"
+ elif "reftest" in self.test_suite or "crashtest" in self.test_suite:
+ category = "reftest"
+ else:
+ category = self.test_suite
+ if c.get("repeat"):
+ if category in SUITE_REPEATABLE:
+ cmd.extend(["--repeat=%s" % c.get("repeat")])
+ else:
+ self.log("--repeat not supported in {}".format(category), level=WARNING)
+
+ # do not add --disable fission if we don't have --disable-e10s
+ if c["disable_fission"] and category not in ["gtest", "cppunittest"]:
+ cmd.append("--disable-fission")
+
+ if "web_content_isolation_strategy" in c:
+ cmd.append(
+ "--web-content-isolation-strategy=%s"
+ % c["web_content_isolation_strategy"]
+ )
+ cmd.extend(["--setpref={}".format(p) for p in self.extra_prefs])
+
+ if not (self.verify_enabled or self.per_test_coverage):
+ if user_paths:
+ cmd.extend(user_paths)
+ elif not (self.verify_enabled or self.per_test_coverage):
+ if self.this_chunk is not None:
+ cmd.extend(["--this-chunk", self.this_chunk])
+ if self.total_chunks is not None:
+ cmd.extend(["--total-chunks", self.total_chunks])
+
+ if category not in SUITE_NO_E10S:
+ if category in SUITE_DEFAULT_E10S and not c["e10s"]:
+ cmd.append("--disable-e10s")
+ elif category not in SUITE_DEFAULT_E10S and c["e10s"]:
+ cmd.append("--e10s")
+
+ if c.get("enable_xorigin_tests"):
+ cmd.extend(["--enable-xorigin-tests"])
+
+ try_options, try_tests = self.try_args(self.test_suite)
+ cmd.extend(try_options)
+ if not self.verify_enabled and not self.per_test_coverage:
+ cmd.extend(
+ self.query_tests_args(
+ self.config["suite_definitions"][self.test_suite].get("tests"),
+ None,
+ try_tests,
+ )
+ )
+
+ if self.java_code_coverage_enabled:
+ cmd.extend(
+ [
+ "--enable-coverage",
+ "--coverage-output-dir",
+ self.java_coverage_output_dir,
+ ]
+ )
+
+ return cmd
+
+ def _query_suites(self):
+ if self.test_suite:
+ return [(self.test_suite, self.test_suite)]
+ # per-test mode: determine test suites to run
+
+ # For each test category, provide a list of supported sub-suites and a mapping
+ # between the per_test_base suite name and the android suite name.
+ all = [
+ (
+ "mochitest",
+ {
+ "mochitest-plain": "mochitest-plain",
+ "mochitest-media": "mochitest-media",
+ "mochitest-plain-gpu": "mochitest-plain-gpu",
+ },
+ ),
+ (
+ "reftest",
+ {
+ "reftest": "reftest",
+ "crashtest": "crashtest",
+ "jsreftest": "jsreftest",
+ },
+ ),
+ ("xpcshell", {"xpcshell": "xpcshell"}),
+ ]
+ suites = []
+ for (category, all_suites) in all:
+ cat_suites = self.query_per_test_category_suites(category, all_suites)
+ for k in cat_suites.keys():
+ suites.append((k, cat_suites[k]))
+ return suites
+
+ def _query_suite_categories(self):
+ if self.test_suite:
+ categories = [self.test_suite]
+ else:
+ # per-test mode
+ categories = ["mochitest", "reftest", "xpcshell"]
+ return categories
+
+ ##########################################
+ # Actions for AndroidEmulatorTest #
+ ##########################################
+
+ def preflight_install(self):
+ # in the base class, this checks for mozinstall, but we don't use it
+ pass
+
+ @PreScriptAction("create-virtualenv")
+ def pre_create_virtualenv(self, action):
+ dirs = self.query_abs_dirs()
+ requirements = None
+ suites = self._query_suites()
+ if ("mochitest-media", "mochitest-media") in suites:
+ # mochitest-media is the only thing that needs this
+ requirements = os.path.join(
+ dirs["abs_mochitest_dir"],
+ "websocketprocessbridge",
+ "websocketprocessbridge_requirements_3.txt",
+ )
+ if requirements:
+ self.register_virtualenv_module(requirements=[requirements], two_pass=True)
+
+ def download_and_extract(self):
+ """
+ Download and extract product APK, tests.zip, and host utils.
+ """
+ super(AndroidEmulatorTest, self).download_and_extract(
+ suite_categories=self._query_suite_categories()
+ )
+ dirs = self.query_abs_dirs()
+ self.xre_path = self.download_hostutils(dirs["abs_xre_dir"])
+
+ def install(self):
+ """
+ Install APKs on the device.
+ """
+ install_needed = (not self.test_suite) or self.config["suite_definitions"][
+ self.test_suite
+ ].get("install")
+ if install_needed is False:
+ self.info("Skipping apk installation for %s" % self.test_suite)
+ return
+ assert (
+ self.installer_path is not None
+ ), "Either add installer_path to the config or use --installer-path."
+ self.install_android_app(self.installer_path)
+ self.info("Finished installing apps for %s" % self.device_serial)
+
+ def run_tests(self):
+ """
+ Run the tests
+ """
+ self.start_time = datetime.datetime.now()
+ max_per_test_time = datetime.timedelta(minutes=60)
+
+ per_test_args = []
+ suites = self._query_suites()
+ minidump = self.query_minidump_stackwalk()
+ for (per_test_suite, suite) in suites:
+ self.test_suite = suite
+
+ try:
+ cwd = self._query_tests_dir(self.test_suite)
+ except Exception:
+ self.fatal("Don't know how to run --test-suite '%s'!" % self.test_suite)
+
+ env = self.query_env()
+ if minidump:
+ env["MINIDUMP_STACKWALK"] = minidump
+ env["MOZ_UPLOAD_DIR"] = self.query_abs_dirs()["abs_blob_upload_dir"]
+ env["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"]
+ env["RUST_BACKTRACE"] = "full"
+ if self.config["nodejs_path"]:
+ env["MOZ_NODE_PATH"] = self.config["nodejs_path"]
+
+ summary = {}
+ for per_test_args in self.query_args(per_test_suite):
+ if (datetime.datetime.now() - self.start_time) > max_per_test_time:
+ # Running tests has run out of time. That is okay! Stop running
+ # them so that a task timeout is not triggered, and so that
+ # (partial) results are made available in a timely manner.
+ self.info(
+ "TinderboxPrint: Running tests took too long: "
+ "Not all tests were executed.<br/>"
+ )
+ # Signal per-test time exceeded, to break out of suites and
+ # suite categories loops also.
+ return
+
+ cmd = self._build_command()
+ final_cmd = copy.copy(cmd)
+ if len(per_test_args) > 0:
+ # in per-test mode, remove any chunk arguments from command
+ for arg in final_cmd:
+ if "total-chunk" in arg or "this-chunk" in arg:
+ final_cmd.remove(arg)
+ final_cmd.extend(per_test_args)
+
+ self.info("Running the command %s" % subprocess.list2cmdline(final_cmd))
+ self.info("##### %s log begins" % self.test_suite)
+
+ suite_category = self.test_suite
+ parser = self.get_test_output_parser(
+ suite_category,
+ config=self.config,
+ log_obj=self.log_obj,
+ error_list=[],
+ )
+ self.run_command(final_cmd, cwd=cwd, env=env, output_parser=parser)
+ tbpl_status, log_level, summary = parser.evaluate_parser(
+ 0, previous_summary=summary
+ )
+ parser.append_tinderboxprint_line(self.test_suite)
+
+ self.info("##### %s log ends" % self.test_suite)
+
+ if len(per_test_args) > 0:
+ self.record_status(tbpl_status, level=log_level)
+ self.log_per_test_status(per_test_args[-1], tbpl_status, log_level)
+ if tbpl_status == TBPL_RETRY:
+ self.info("Per-test run abandoned due to RETRY status")
+ return
+ else:
+ self.record_status(tbpl_status, level=log_level)
+ # report as INFO instead of log_level to avoid extra Treeherder lines
+ self.info(
+ "The %s suite: %s ran with return status: %s"
+ % (suite_category, suite, tbpl_status),
+ )
+
+
+if __name__ == "__main__":
+ test = AndroidEmulatorTest()
+ test.run_and_exit()