#!/usr/bin/env python # ***** BEGIN LICENSE BLOCK ***** # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. # ***** END LICENSE BLOCK ***** import copy import datetime import json import os import subprocess import sys # load modules from parent dir sys.path.insert(1, os.path.dirname(sys.path[0])) from mozharness.base.log import WARNING from mozharness.base.script import BaseScript, PreScriptAction from mozharness.mozilla.automation import TBPL_RETRY from mozharness.mozilla.mozbase import MozbaseMixin from mozharness.mozilla.testing.android import AndroidMixin from mozharness.mozilla.testing.codecoverage import CodeCoverageMixin from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options SUITE_DEFAULT_E10S = ["geckoview-junit", "mochitest", "reftest"] SUITE_NO_E10S = ["cppunittest", "gtest", "jittest"] SUITE_REPEATABLE = ["mochitest", "reftest", "xpcshell"] class AndroidHardwareTest( TestingMixin, BaseScript, MozbaseMixin, CodeCoverageMixin, AndroidMixin ): config_options = [ [["--test-suite"], {"action": "store", "dest": "test_suite", "default": None}], [ ["--adb-path"], { "action": "store", "dest": "adb_path", "default": None, "help": "Path to adb", }, ], [ ["--total-chunk"], { "action": "store", "dest": "total_chunks", "default": None, "help": "Number of total chunks", }, ], [ ["--this-chunk"], { "action": "store", "dest": "this_chunk", "default": None, "help": "Number of this chunk", }, ], [ ["--log-raw-level"], { "action": "store", "dest": "log_raw_level", "default": "info", "help": "Set log level (debug|info|warning|error|critical|fatal)", }, ], [ ["--log-tbpl-level"], { "action": "store", "dest": "log_tbpl_level", "default": "info", "help": "Set log level (debug|info|warning|error|critical|fatal)", }, ], [ ["--disable-e10s"], { "action": "store_false", "dest": "e10s", "default": True, "help": "Run tests without multiple processes (e10s).", }, ], [ ["--disable-fission"], { "action": "store_true", "dest": "disable_fission", "default": False, "help": "Run with Fission disabled.", }, ], [ ["--repeat"], { "action": "store", "type": "int", "dest": "repeat", "default": 0, "help": "Repeat the tests the given number of times. Supported " "by mochitest, reftest, crashtest, ignored otherwise.", }, ], [ [ "--setpref", ], { "action": "append", "dest": "extra_prefs", "default": [], "help": "Extra user prefs.", }, ], [ ["--jittest-flags"], { "action": "store", "dest": "jittest_flags", "default": "debug", "help": "Flags to run with jittest (all, debug, etc.).", }, ], ] + copy.deepcopy(testing_config_options) def __init__(self, require_config_file=False): super(AndroidHardwareTest, self).__init__( config_options=self.config_options, all_actions=[ "clobber", "download-and-extract", "create-virtualenv", "verify-device", "install", "run-tests", ], require_config_file=require_config_file, config={ "virtualenv_modules": [], "virtualenv_requirements": [], "require_test_zip": True, # IP address of the host as seen from the device. "remote_webserver": os.environ["HOST_IP"], }, ) # these are necessary since self.config is read only c = self.config self.installer_url = c.get("installer_url") self.installer_path = c.get("installer_path") self.test_url = c.get("test_url") self.test_packages_url = c.get("test_packages_url") self.test_manifest = c.get("test_manifest") suite = c.get("test_suite") self.test_suite = suite self.this_chunk = c.get("this_chunk") self.total_chunks = c.get("total_chunks") self.xre_path = None self.log_raw_level = c.get("log_raw_level") self.log_tbpl_level = c.get("log_tbpl_level") self.disable_e10s = c.get("disable_e10s") self.disable_fission = c.get("disable_fission") self.extra_prefs = c.get("extra_prefs") self.jittest_flags = c.get("jittest_flags") def query_abs_dirs(self): if self.abs_dirs: return self.abs_dirs abs_dirs = super(AndroidHardwareTest, self).query_abs_dirs() dirs = {} dirs["abs_test_install_dir"] = os.path.join(abs_dirs["abs_work_dir"], "tests") dirs["abs_test_bin_dir"] = os.path.join( abs_dirs["abs_work_dir"], "tests", "bin" ) dirs["abs_modules_dir"] = os.path.join(dirs["abs_test_install_dir"], "modules") dirs["abs_blob_upload_dir"] = os.path.join( abs_dirs["abs_work_dir"], "blobber_upload_dir" ) dirs["abs_mochitest_dir"] = os.path.join( dirs["abs_test_install_dir"], "mochitest" ) dirs["abs_reftest_dir"] = os.path.join(dirs["abs_test_install_dir"], "reftest") dirs["abs_xpcshell_dir"] = os.path.join( dirs["abs_test_install_dir"], "xpcshell" ) work_dir = os.environ.get("MOZ_FETCHES_DIR") or abs_dirs["abs_work_dir"] dirs["abs_xre_dir"] = os.path.join(work_dir, "hostutils") for key in dirs.keys(): if key not in abs_dirs: abs_dirs[key] = dirs[key] self.abs_dirs = abs_dirs return self.abs_dirs def _query_tests_dir(self): dirs = self.query_abs_dirs() try: test_dir = self.config["suite_definitions"][self.test_suite]["testsdir"] except Exception: test_dir = self.test_suite return os.path.join(dirs["abs_test_install_dir"], test_dir) def _build_command(self): c = self.config dirs = self.query_abs_dirs() if self.test_suite not in self.config["suite_definitions"]: self.fatal("Key '%s' not defined in the config!" % self.test_suite) cmd = [ self.query_python_path("python"), "-u", os.path.join( self._query_tests_dir(), self.config["suite_definitions"][self.test_suite]["run_filename"], ), ] raw_log_file, error_summary_file = self.get_indexed_logs( dirs["abs_blob_upload_dir"], self.test_suite ) str_format_values = { "device_serial": self.device_serial, "remote_webserver": c["remote_webserver"], "xre_path": self.xre_path, "utility_path": self.xre_path, "http_port": "8854", # starting http port to use for the mochitest server "ssl_port": "4454", # starting ssl port to use for the server "certs_path": os.path.join(dirs["abs_work_dir"], "tests/certs"), # TestingMixin._download_and_extract_symbols() will set # self.symbols_path when downloading/extracting. "symbols_path": self.symbols_path, "modules_dir": dirs["abs_modules_dir"], "installer_path": self.installer_path, "raw_log_file": raw_log_file, "log_tbpl_level": self.log_tbpl_level, "log_raw_level": self.log_raw_level, "error_summary_file": error_summary_file, "xpcshell_extra": c.get("xpcshell_extra", ""), "jittest_flags": self.jittest_flags, } user_paths = json.loads(os.environ.get("MOZHARNESS_TEST_PATHS", '""')) confirm_paths = json.loads(os.environ.get("MOZHARNESS_CONFIRM_PATHS", '""')) for option in self.config["suite_definitions"][self.test_suite]["options"]: opt = option.split("=")[0] # override configured chunk options with script args, if specified if opt in ("--this-chunk", "--total-chunks"): if ( user_paths or getattr(self, opt.replace("-", "_").strip("_"), None) is not None ): continue if "%(app)" in option: # only query package name if requested cmd.extend([option % {"app": self.query_package_name()}]) else: option = option % str_format_values if option: cmd.extend([option]) if not self.verify_enabled and not user_paths: if self.this_chunk is not None: cmd.extend(["--this-chunk", self.this_chunk]) if self.total_chunks is not None: cmd.extend(["--total-chunks", self.total_chunks]) if "mochitest" in self.test_suite: category = "mochitest" elif "reftest" in self.test_suite or "crashtest" in self.test_suite: category = "reftest" else: category = self.test_suite if c.get("repeat"): if category in SUITE_REPEATABLE: cmd.extend(["--repeat=%s" % c.get("repeat")]) else: self.log("--repeat not supported in {}".format(category), level=WARNING) if category not in SUITE_NO_E10S: if category in SUITE_DEFAULT_E10S and not c["e10s"]: cmd.append("--disable-e10s") elif category not in SUITE_DEFAULT_E10S and c["e10s"]: cmd.append("--e10s") if self.disable_fission and category not in SUITE_NO_E10S: cmd.append("--disable-fission") cmd.extend(["--setpref={}".format(p) for p in self.extra_prefs]) try_options, try_tests = self.try_args(self.test_suite) if try_options: cmd.extend(try_options) if user_paths: # reftest on android-hw uses a subset (reftest-qr) of tests, # but scheduling only knows about 'reftest' suite = self.test_suite if suite == "reftest-qr": suite = "reftest" if user_paths.get(suite, []): suite_test_paths = user_paths.get(suite, []) # NOTE: we do not want to prepend 'tests' if a single path if confirm_paths and confirm_paths.get(suite, []): suite_test_paths = confirm_paths.get(suite, []) suite_test_paths = [os.path.join("tests", p) for p in suite_test_paths] cmd.extend(suite_test_paths) elif not self.verify_enabled and not self.per_test_coverage: cmd.extend( self.query_tests_args( self.config["suite_definitions"][self.test_suite].get("tests"), None, try_tests, ) ) if self.config.get("restartAfterFailure", False): cmd.append("--restartAfterFailure") return cmd def _query_suites(self): if self.test_suite: return [(self.test_suite, self.test_suite)] # per-test mode: determine test suites to run all = [ ( "mochitest", { "mochitest-plain": "mochitest-plain", "mochitest-plain-gpu": "mochitest-plain-gpu", }, ), ("reftest", {"reftest": "reftest", "crashtest": "crashtest"}), ("xpcshell", {"xpcshell": "xpcshell"}), ] suites = [] for category, all_suites in all: cat_suites = self.query_per_test_category_suites(category, all_suites) for k in cat_suites.keys(): suites.append((k, cat_suites[k])) return suites def _query_suite_categories(self): if self.test_suite: categories = [self.test_suite] else: # per-test mode categories = ["mochitest", "reftest", "xpcshell"] return categories ########################################## # Actions for AndroidHardwareTest # ########################################## def preflight_install(self): # in the base class, this checks for mozinstall, but we don't use it pass @PreScriptAction("create-virtualenv") def pre_create_virtualenv(self, action): dirs = self.query_abs_dirs() requirements = None suites = self._query_suites() if ("mochitest-media", "mochitest-media") in suites: # mochitest-media is the only thing that needs this requirements = os.path.join( dirs["abs_mochitest_dir"], "websocketprocessbridge", "websocketprocessbridge_requirements_3.txt", ) if requirements: self.register_virtualenv_module(requirements=[requirements]) def download_and_extract(self): """ Download and extract product APK, tests.zip, and host utils. """ super(AndroidHardwareTest, self).download_and_extract( suite_categories=self._query_suite_categories() ) dirs = self.query_abs_dirs() self.xre_path = dirs["abs_xre_dir"] def install(self): """ Install APKs on the device. """ install_needed = (not self.test_suite) or self.config["suite_definitions"][ self.test_suite ].get("install") if install_needed is False: self.info("Skipping apk installation for %s" % self.test_suite) return assert ( self.installer_path is not None ), "Either add installer_path to the config or use --installer-path." self.uninstall_android_app() self.install_android_app(self.installer_path) self.info("Finished installing apps for %s" % self.device_name) def run_tests(self): """ Run the tests """ self.start_time = datetime.datetime.now() max_per_test_time = datetime.timedelta(minutes=60) per_test_args = [] suites = self._query_suites() minidump = self.query_minidump_stackwalk() for per_test_suite, suite in suites: self.test_suite = suite try: cwd = self._query_tests_dir() except Exception: self.fatal("Don't know how to run --test-suite '%s'!" % self.test_suite) env = self.query_env() if minidump: env["MINIDUMP_STACKWALK"] = minidump env["MOZ_UPLOAD_DIR"] = self.query_abs_dirs()["abs_blob_upload_dir"] env["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"] env["RUST_BACKTRACE"] = "full" summary = None for per_test_args in self.query_args(per_test_suite): if (datetime.datetime.now() - self.start_time) > max_per_test_time: # Running tests has run out of time. That is okay! Stop running # them so that a task timeout is not triggered, and so that # (partial) results are made available in a timely manner. self.info( "TinderboxPrint: Running tests took too long: " "Not all tests were executed.
" ) # Signal per-test time exceeded, to break out of suites and # suite categories loops also. return cmd = self._build_command() final_cmd = copy.copy(cmd) if len(per_test_args) > 0: # in per-test mode, remove any chunk arguments from command for arg in final_cmd: if "total-chunk" in arg or "this-chunk" in arg: final_cmd.remove(arg) final_cmd.extend(per_test_args) self.info( "Running on %s the command %s" % (self.device_name, subprocess.list2cmdline(final_cmd)) ) self.info("##### %s log begins" % self.test_suite) suite_category = self.test_suite parser = self.get_test_output_parser( suite_category, config=self.config, log_obj=self.log_obj, error_list=[], ) self.run_command(final_cmd, cwd=cwd, env=env, output_parser=parser) tbpl_status, log_level, summary = parser.evaluate_parser(0, summary) parser.append_tinderboxprint_line(self.test_suite) self.info("##### %s log ends" % self.test_suite) if len(per_test_args) > 0: self.record_status(tbpl_status, level=log_level) self.log_per_test_status(per_test_args[-1], tbpl_status, log_level) if tbpl_status == TBPL_RETRY: self.info("Per-test run abandoned due to RETRY status") return else: self.record_status(tbpl_status, level=log_level) # report as INFO instead of log_level to avoid extra Treeherder lines self.info( "The %s suite: %s ran with return status: %s" % (suite_category, suite, tbpl_status), ) if __name__ == "__main__": test = AndroidHardwareTest() test.run_and_exit()