#!/usr/bin/env python # ***** BEGIN LICENSE BLOCK ***** # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. # ***** END LICENSE BLOCK ***** from __future__ import absolute_import import copy import json import os import sys # load modules from parent dir sys.path.insert(1, os.path.dirname(sys.path[0])) from mozharness.base.errors import BaseErrorList, TarErrorList from mozharness.base.log import INFO from mozharness.base.script import PreScriptAction from mozharness.base.transfer import TransferMixin from mozharness.base.vcs.vcsbase import MercurialScript from mozharness.mozilla.testing.errors import LogcatErrorList from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper from mozharness.mozilla.testing.codecoverage import ( CodeCoverageMixin, code_coverage_config_options, ) from mozharness.mozilla.testing.errors import HarnessErrorList from mozharness.mozilla.structuredlog import StructuredOutputParser class MarionetteTest(TestingMixin, MercurialScript, TransferMixin, CodeCoverageMixin): config_options = ( [ [ ["--application"], { "action": "store", "dest": "application", "default": None, "help": "application name of binary", }, ], [ ["--app-arg"], { "action": "store", "dest": "app_arg", "default": None, "help": "Optional command-line argument to pass to the browser", }, ], [ ["--marionette-address"], { "action": "store", "dest": "marionette_address", "default": None, "help": "The host:port of the Marionette server running inside Gecko. " "Unused for emulator testing", }, ], [ ["--emulator"], { "action": "store", "type": "choice", "choices": ["arm", "x86"], "dest": "emulator", "default": None, "help": "Use an emulator for testing", }, ], [ ["--test-manifest"], { "action": "store", "dest": "test_manifest", "default": "unit-tests.ini", "help": "Path to test manifest to run relative to the Marionette " "tests directory", }, ], [ ["--total-chunks"], { "action": "store", "dest": "total_chunks", "help": "Number of total chunks", }, ], [ ["--this-chunk"], { "action": "store", "dest": "this_chunk", "help": "Number of this chunk", }, ], [ ["--setpref"], { "action": "append", "metavar": "PREF=VALUE", "dest": "extra_prefs", "default": [], "help": "Extra user prefs.", }, ], [ ["--headless"], { "action": "store_true", "dest": "headless", "default": False, "help": "Run tests in headless mode.", }, ], [ ["--headless-width"], { "action": "store", "dest": "headless_width", "default": "1600", "help": "Specify headless virtual screen width (default: 1600).", }, ], [ ["--headless-height"], { "action": "store", "dest": "headless_height", "default": "1200", "help": "Specify headless virtual screen height (default: 1200).", }, ], [ ["--allow-software-gl-layers"], { "action": "store_true", "dest": "allow_software_gl_layers", "default": False, "help": "Permits a software GL implementation (such as LLVMPipe) to use the GL compositor.", # NOQA: E501 }, ], [ ["--disable-actors"], { "action": "store_true", "dest": "disable_actors", "default": False, "help": "Disable the usage of JSWindowActors in Marionette.", }, ], [ ["--enable-webrender"], { "action": "store_true", "dest": "enable_webrender", "default": False, "help": "Enable the WebRender compositor in Gecko.", }, ], ] + copy.deepcopy(testing_config_options) + copy.deepcopy(code_coverage_config_options) ) repos = [] def __init__(self, require_config_file=False): super(MarionetteTest, self).__init__( config_options=self.config_options, all_actions=[ "clobber", "pull", "download-and-extract", "create-virtualenv", "install", "run-tests", ], default_actions=[ "clobber", "pull", "download-and-extract", "create-virtualenv", "install", "run-tests", ], require_config_file=require_config_file, config={"require_test_zip": True}, ) # these are necessary since self.config is read only c = self.config self.installer_url = c.get("installer_url") self.installer_path = c.get("installer_path") self.binary_path = c.get("binary_path") self.test_url = c.get("test_url") self.test_packages_url = c.get("test_packages_url") self.test_suite = self._get_test_suite(c.get("emulator")) if self.test_suite not in self.config["suite_definitions"]: self.fatal("{} is not defined in the config!".format(self.test_suite)) if c.get("structured_output"): self.parser_class = StructuredOutputParser else: self.parser_class = TestSummaryOutputParserHelper def _pre_config_lock(self, rw_config): super(MarionetteTest, self)._pre_config_lock(rw_config) if not self.config.get("emulator") and not self.config.get( "marionette_address" ): self.fatal( "You need to specify a --marionette-address for non-emulator tests! " "(Try --marionette-address localhost:2828 )" ) def _query_tests_dir(self): dirs = self.query_abs_dirs() test_dir = self.config["suite_definitions"][self.test_suite]["testsdir"] return os.path.join(dirs["abs_test_install_dir"], test_dir) def query_abs_dirs(self): if self.abs_dirs: return self.abs_dirs abs_dirs = super(MarionetteTest, self).query_abs_dirs() dirs = {} dirs["abs_test_install_dir"] = os.path.join(abs_dirs["abs_work_dir"], "tests") dirs["abs_marionette_dir"] = os.path.join( dirs["abs_test_install_dir"], "marionette", "harness", "marionette_harness" ) dirs["abs_marionette_tests_dir"] = os.path.join( dirs["abs_test_install_dir"], "marionette", "tests", "testing", "marionette", "harness", "marionette_harness", "tests", ) dirs["abs_gecko_dir"] = os.path.join(abs_dirs["abs_work_dir"], "gecko") dirs["abs_emulator_dir"] = os.path.join(abs_dirs["abs_work_dir"], "emulator") dirs["abs_blob_upload_dir"] = os.path.join( abs_dirs["abs_work_dir"], "blobber_upload_dir" ) for key in dirs.keys(): if key not in abs_dirs: abs_dirs[key] = dirs[key] self.abs_dirs = abs_dirs return self.abs_dirs @PreScriptAction("create-virtualenv") def _configure_marionette_virtualenv(self, action): dirs = self.query_abs_dirs() requirements = os.path.join( dirs["abs_test_install_dir"], "config", "marionette_requirements.txt" ) if not os.path.isfile(requirements): self.fatal( "Could not find marionette requirements file: {}".format(requirements) ) # marionette_requirements.txt must use the legacy resolver until bug 1684969 is resolved. self.register_virtualenv_module( requirements=[requirements], two_pass=True, legacy_resolver=True ) def _get_test_suite(self, is_emulator): """ Determine which in tree options group to use and return the appropriate key. """ platform = "emulator" if is_emulator else "desktop" # Currently running marionette on an emulator means webapi # tests. This method will need to change if this does. testsuite = "webapi" if is_emulator else "marionette" return "{}_{}".format(testsuite, platform) def download_and_extract(self): super(MarionetteTest, self).download_and_extract() if self.config.get("emulator"): dirs = self.query_abs_dirs() self.mkdir_p(dirs["abs_emulator_dir"]) tar = self.query_exe("tar", return_type="list") self.run_command( tar + ["zxf", self.installer_path], cwd=dirs["abs_emulator_dir"], error_list=TarErrorList, halt_on_failure=True, fatal_exit_code=3, ) def install(self): if self.config.get("emulator"): self.info("Emulator tests; skipping.") else: super(MarionetteTest, self).install() def run_tests(self): """ Run the Marionette tests """ dirs = self.query_abs_dirs() raw_log_file = os.path.join(dirs["abs_blob_upload_dir"], "marionette_raw.log") error_summary_file = os.path.join( dirs["abs_blob_upload_dir"], "marionette_errorsummary.log" ) html_report_file = os.path.join(dirs["abs_blob_upload_dir"], "report.html") config_fmt_args = { # emulator builds require a longer timeout "timeout": 60000 if self.config.get("emulator") else 10000, "profile": os.path.join(dirs["abs_work_dir"], "profile"), "xml_output": os.path.join(dirs["abs_work_dir"], "output.xml"), "html_output": os.path.join(dirs["abs_blob_upload_dir"], "output.html"), "logcat_dir": dirs["abs_work_dir"], "emulator": "arm", "symbols_path": self.symbols_path, "binary": self.binary_path, "address": self.config.get("marionette_address"), "raw_log_file": raw_log_file, "error_summary_file": error_summary_file, "html_report_file": html_report_file, "gecko_log": dirs["abs_blob_upload_dir"], "this_chunk": self.config.get("this_chunk", 1), "total_chunks": self.config.get("total_chunks", 1), } self.info("The emulator type: %s" % config_fmt_args["emulator"]) # build the marionette command arguments python = self.query_python_path("python") cmd = [python, "-u", os.path.join(dirs["abs_marionette_dir"], "runtests.py")] manifest = os.path.join( dirs["abs_marionette_tests_dir"], self.config["test_manifest"] ) if self.config.get("app_arg"): config_fmt_args["app_arg"] = self.config["app_arg"] if self.config["disable_actors"]: cmd.append("--disable-actors") if self.config["enable_webrender"]: cmd.append("--enable-webrender") cmd.extend(["--setpref={}".format(p) for p in self.config["extra_prefs"]]) cmd.append("--gecko-log=-") if self.config.get("structured_output"): cmd.append("--log-raw=-") for arg in self.config["suite_definitions"][self.test_suite]["options"]: cmd.append(arg % config_fmt_args) if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1: # Make sure that the logging directory exists self.fatal("Could not create blobber upload directory") test_paths = json.loads(os.environ.get("MOZHARNESS_TEST_PATHS", '""')) if test_paths and "marionette" in test_paths: paths = [ os.path.join(dirs["abs_test_install_dir"], "marionette", "tests", p) for p in test_paths["marionette"] ] cmd.extend(paths) else: cmd.append(manifest) try_options, try_tests = self.try_args("marionette") cmd.extend(self.query_tests_args(try_tests, str_format_values=config_fmt_args)) env = {} if self.query_minidump_stackwalk(): env["MINIDUMP_STACKWALK"] = self.minidump_stackwalk_path env["MOZ_UPLOAD_DIR"] = self.query_abs_dirs()["abs_blob_upload_dir"] env["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"] env["RUST_BACKTRACE"] = "full" if self.config["allow_software_gl_layers"]: env["MOZ_LAYERS_ALLOW_SOFTWARE_GL"] = "1" if self.config["headless"]: env["MOZ_HEADLESS"] = "1" env["MOZ_HEADLESS_WIDTH"] = self.config["headless_width"] env["MOZ_HEADLESS_HEIGHT"] = self.config["headless_height"] if not os.path.isdir(env["MOZ_UPLOAD_DIR"]): self.mkdir_p(env["MOZ_UPLOAD_DIR"]) env = self.query_env(partial_env=env) try: cwd = self._query_tests_dir() except Exception as e: self.fatal( "Don't know how to run --test-suite '{0}': {1}!".format( self.test_suite, e ) ) marionette_parser = self.parser_class( config=self.config, log_obj=self.log_obj, error_list=BaseErrorList + HarnessErrorList, strict=False, ) return_code = self.run_command( cmd, cwd=cwd, output_timeout=1000, output_parser=marionette_parser, env=env ) level = INFO tbpl_status, log_level, summary = marionette_parser.evaluate_parser( return_code=return_code ) marionette_parser.append_tinderboxprint_line("marionette") qemu = os.path.join(dirs["abs_work_dir"], "qemu.log") if os.path.isfile(qemu): self.copyfile(qemu, os.path.join(dirs["abs_blob_upload_dir"], "qemu.log")) # dump logcat output if there were failures if self.config.get("emulator"): if ( marionette_parser.failed != "0" or "T-FAIL" in marionette_parser.tsummary ): logcat = os.path.join(dirs["abs_work_dir"], "emulator-5554.log") if os.access(logcat, os.F_OK): self.info("dumping logcat") self.run_command(["cat", logcat], error_list=LogcatErrorList) else: self.info("no logcat file found") else: # .. or gecko.log if it exists gecko_log = os.path.join(self.config["base_work_dir"], "gecko.log") if os.access(gecko_log, os.F_OK): self.info("dumping gecko.log") self.run_command(["cat", gecko_log]) self.rmtree(gecko_log) else: self.info("gecko.log not found") marionette_parser.print_summary("marionette") self.log( "Marionette exited with return code %s: %s" % (return_code, tbpl_status), level=level, ) self.record_status(tbpl_status) if __name__ == "__main__": marionetteTest = MarionetteTest() marionetteTest.run_and_exit()