summaryrefslogtreecommitdiffstats
path: root/testing/mochitest/runjunit.py
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--testing/mochitest/runjunit.py717
1 files changed, 717 insertions, 0 deletions
diff --git a/testing/mochitest/runjunit.py b/testing/mochitest/runjunit.py
new file mode 100644
index 0000000000..8bccd14de3
--- /dev/null
+++ b/testing/mochitest/runjunit.py
@@ -0,0 +1,717 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import os
+import posixpath
+import re
+import shutil
+import sys
+import tempfile
+import traceback
+
+import mozcrash
+import mozinfo
+import mozlog
+import moznetwork
+import six
+from mozdevice import ADBDeviceFactory, ADBError, ADBTimeoutError
+from mozprofile import DEFAULT_PORTS, Profile
+from mozprofile.cli import parse_preferences
+from mozprofile.permissions import ServerLocations
+from runtests import MochitestDesktop, update_mozinfo
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+try:
+ from mach.util import UserError
+ from mozbuild.base import MachCommandConditions as conditions
+ from mozbuild.base import MozbuildObject
+
+ build_obj = MozbuildObject.from_environment(cwd=here)
+except ImportError:
+ build_obj = None
+ conditions = None
+ UserError = Exception
+
+
+class JavaTestHarnessException(Exception):
+ pass
+
+
+class JUnitTestRunner(MochitestDesktop):
+ """
+ A test harness to run geckoview junit tests on a remote device.
+ """
+
+ def __init__(self, log, options):
+ self.log = log
+ self.verbose = False
+ self.http3Server = None
+ self.dohServer = None
+ if (
+ options.log_tbpl_level == "debug"
+ or options.log_mach_level == "debug"
+ or options.verbose
+ ):
+ self.verbose = True
+ self.device = ADBDeviceFactory(
+ adb=options.adbPath or "adb",
+ device=options.deviceSerial,
+ test_root=options.remoteTestRoot,
+ verbose=self.verbose,
+ run_as_package=options.app,
+ )
+ self.options = options
+ self.log.debug("options=%s" % vars(options))
+ update_mozinfo()
+ self.remote_profile = posixpath.join(self.device.test_root, "junit-profile")
+ self.remote_filter_list = posixpath.join(
+ self.device.test_root, "junit-filters.list"
+ )
+
+ if self.options.coverage and not self.options.coverage_output_dir:
+ raise UserError(
+ "--coverage-output-dir is required when using --enable-coverage"
+ )
+ if self.options.coverage:
+ self.remote_coverage_output_file = posixpath.join(
+ self.device.test_root, "junit-coverage.ec"
+ )
+ self.coverage_output_file = os.path.join(
+ self.options.coverage_output_dir, "junit-coverage.ec"
+ )
+
+ self.server_init()
+
+ self.cleanup()
+ self.device.clear_logcat()
+ self.build_profile()
+ self.startServers(self.options, debuggerInfo=None, public=True)
+ self.log.debug("Servers started")
+
+ def collectLogcatForCurrentTest(self):
+ # These are unique start and end markers logged by GeckoSessionTestRule.java
+ START_MARKER = "1f0befec-3ff2-40ff-89cf-b127eb38b1ec"
+ END_MARKER = "c5ee677f-bc83-49bd-9e28-2d35f3d0f059"
+ logcat = self.device.get_logcat()
+ test_logcat = ""
+ started = False
+ for l in logcat:
+ if START_MARKER in l and self.test_name in l:
+ started = True
+ if started:
+ test_logcat += l + "\n"
+ if started and END_MARKER in l:
+ return test_logcat
+
+ def needsWebsocketProcessBridge(self, options):
+ """
+ Overrides MochitestDesktop.needsWebsocketProcessBridge and always
+ returns False as the junit tests do not use the websocket process
+ bridge. This is needed to satisfy MochitestDesktop.startServers.
+ """
+ return False
+
+ def server_init(self):
+ """
+ Additional initialization required to satisfy MochitestDesktop.startServers
+ """
+ self._locations = None
+ self.server = None
+ self.wsserver = None
+ self.websocketProcessBridge = None
+ self.SERVER_STARTUP_TIMEOUT = 180 if mozinfo.info.get("debug") else 90
+ if self.options.remoteWebServer is None:
+ self.options.remoteWebServer = moznetwork.get_ip()
+ self.options.webServer = self.options.remoteWebServer
+ self.options.webSocketPort = "9988"
+ self.options.httpdPath = None
+ self.options.http3ServerPath = None
+ self.options.keep_open = False
+ self.options.pidFile = ""
+ self.options.subsuite = None
+ self.options.xrePath = None
+ self.options.useHttp3Server = False
+ if build_obj and "MOZ_HOST_BIN" in os.environ:
+ self.options.xrePath = os.environ["MOZ_HOST_BIN"]
+ if not self.options.utilityPath:
+ self.options.utilityPath = self.options.xrePath
+ if not self.options.xrePath:
+ self.options.xrePath = self.options.utilityPath
+ if build_obj:
+ self.options.certPath = os.path.join(
+ build_obj.topsrcdir, "build", "pgo", "certs"
+ )
+
+ def build_profile(self):
+ """
+ Create a local profile with test prefs and proxy definitions and
+ push it to the remote device.
+ """
+
+ self.profile = Profile(locations=self.locations, proxy=self.proxy(self.options))
+ self.options.profilePath = self.profile.profile
+
+ # Set preferences
+ self.merge_base_profiles(self.options, "geckoview-junit")
+
+ if self.options.web_content_isolation_strategy is not None:
+ self.options.extra_prefs.append(
+ "fission.webContentIsolationStrategy=%s"
+ % self.options.web_content_isolation_strategy
+ )
+ self.options.extra_prefs.append("fission.autostart=true")
+ if self.options.disable_fission:
+ self.options.extra_prefs.pop()
+ self.options.extra_prefs.append("fission.autostart=false")
+ prefs = parse_preferences(self.options.extra_prefs)
+ self.profile.set_preferences(prefs)
+
+ if self.fillCertificateDB(self.options):
+ self.log.error("Certificate integration failed")
+
+ self.device.push(self.profile.profile, self.remote_profile)
+ self.log.debug(
+ "profile %s -> %s" % (str(self.profile.profile), str(self.remote_profile))
+ )
+
+ def cleanup(self):
+ try:
+ self.stopServers()
+ self.log.debug("Servers stopped")
+ self.device.stop_application(self.options.app)
+ self.device.rm(self.remote_profile, force=True, recursive=True)
+ if hasattr(self, "profile"):
+ del self.profile
+ self.device.rm(self.remote_filter_list, force=True)
+ except Exception:
+ traceback.print_exc()
+ self.log.info("Caught and ignored an exception during cleanup")
+
+ def build_command_line(self, test_filters_file, test_filters):
+ """
+ Construct and return the 'am instrument' command line.
+ """
+ cmd = "am instrument -w -r"
+ # profile location
+ cmd = cmd + " -e args '-profile %s'" % self.remote_profile
+ # chunks (shards)
+ shards = self.options.totalChunks
+ shard = self.options.thisChunk
+ if shards is not None and shard is not None:
+ shard -= 1 # shard index is 0 based
+ cmd = cmd + " -e numShards %d -e shardIndex %d" % (shards, shard)
+
+ # test filters: limit run to specific test(s)
+ # filter can be class-name or 'class-name#method-name' (single test)
+ # Multiple filters must be specified as a line-separated text file
+ # and then pushed to the device.
+ filter_list_name = None
+
+ if test_filters_file:
+ # We specified a pre-existing file, so use that
+ filter_list_name = test_filters_file
+ elif test_filters:
+ if len(test_filters) > 1:
+ # Generate the list file from test_filters
+ with tempfile.NamedTemporaryFile(delete=False, mode="w") as filter_list:
+ for f in test_filters:
+ print(f, file=filter_list)
+ filter_list_name = filter_list.name
+ else:
+ # A single filter may be directly appended to the command line
+ cmd = cmd + " -e class %s" % test_filters[0]
+
+ if filter_list_name:
+ self.device.push(filter_list_name, self.remote_filter_list)
+
+ if test_filters:
+ # We only remove the filter list if we generated it as a
+ # temporary file.
+ os.remove(filter_list_name)
+
+ cmd = cmd + " -e testFile %s" % self.remote_filter_list
+
+ # enable code coverage reports
+ if self.options.coverage:
+ cmd = cmd + " -e coverage true"
+ cmd = cmd + " -e coverageFile %s" % self.remote_coverage_output_file
+ # environment
+ env = {}
+ env["MOZ_CRASHREPORTER"] = "1"
+ env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
+ env["XPCOM_DEBUG_BREAK"] = "stack"
+ env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
+ env["MOZ_IN_AUTOMATION"] = "1"
+ env["R_LOG_VERBOSE"] = "1"
+ env["R_LOG_LEVEL"] = "6"
+ env["R_LOG_DESTINATION"] = "stderr"
+ # webrender needs gfx.webrender.all=true, gtest doesn't use prefs
+ env["MOZ_WEBRENDER"] = "1"
+ # FIXME: When android switches to using Fission by default,
+ # MOZ_FORCE_DISABLE_FISSION will need to be configured correctly.
+ if self.options.disable_fission:
+ env["MOZ_FORCE_DISABLE_FISSION"] = "1"
+ else:
+ env["MOZ_FORCE_ENABLE_FISSION"] = "1"
+
+ # Add additional env variables
+ for [key, value] in [p.split("=", 1) for p in self.options.add_env]:
+ env[key] = value
+
+ for (env_count, (env_key, env_val)) in enumerate(six.iteritems(env)):
+ cmd = cmd + " -e env%d %s=%s" % (env_count, env_key, env_val)
+ # runner
+ cmd = cmd + " %s/%s" % (self.options.app, self.options.runner)
+ return cmd
+
+ @property
+ def locations(self):
+ if self._locations is not None:
+ return self._locations
+ locations_file = os.path.join(here, "server-locations.txt")
+ self._locations = ServerLocations(locations_file)
+ return self._locations
+
+ def need_more_runs(self):
+ if self.options.run_until_failure and (self.fail_count == 0):
+ return True
+ if self.runs <= self.options.repeat:
+ return True
+ return False
+
+ def run_tests(self, test_filters_file=None, test_filters=None):
+ """
+ Run the tests.
+ """
+ if not self.device.is_app_installed(self.options.app):
+ raise UserError("%s is not installed" % self.options.app)
+ if self.device.process_exist(self.options.app):
+ raise UserError(
+ "%s already running before starting tests" % self.options.app
+ )
+ # test_filters_file and test_filters must be mutually-exclusive
+ if test_filters_file and test_filters:
+ raise UserError(
+ "Test filters may not be specified when test-filters-file is provided"
+ )
+
+ self.test_started = False
+ self.pass_count = 0
+ self.fail_count = 0
+ self.todo_count = 0
+ self.total_count = 0
+ self.runs = 0
+ self.seen_last_test = False
+
+ def callback(line):
+ # Output callback: Parse the raw junit log messages, translating into
+ # treeherder-friendly test start/pass/fail messages.
+
+ line = six.ensure_str(line)
+ self.log.process_output(self.options.app, str(line))
+ # Expect per-test info like: "INSTRUMENTATION_STATUS: class=something"
+ match = re.match(r"INSTRUMENTATION_STATUS:\s*class=(.*)", line)
+ if match:
+ self.class_name = match.group(1)
+ # Expect per-test info like: "INSTRUMENTATION_STATUS: test=something"
+ match = re.match(r"INSTRUMENTATION_STATUS:\s*test=(.*)", line)
+ if match:
+ self.test_name = match.group(1)
+ match = re.match(r"INSTRUMENTATION_STATUS:\s*numtests=(.*)", line)
+ if match:
+ self.total_count = int(match.group(1))
+ match = re.match(r"INSTRUMENTATION_STATUS:\s*current=(.*)", line)
+ if match:
+ self.current_test_id = int(match.group(1))
+ match = re.match(r"INSTRUMENTATION_STATUS:\s*stack=(.*)", line)
+ if match:
+ self.exception_message = match.group(1)
+ if (
+ "org.mozilla.geckoview.test.rule.TestHarnessException"
+ in self.exception_message
+ ):
+ # This is actually a problem in the test harness itself
+ raise JavaTestHarnessException(self.exception_message)
+
+ # Expect per-test info like: "INSTRUMENTATION_STATUS_CODE: 0|1|..."
+ match = re.match(r"INSTRUMENTATION_STATUS_CODE:\s*([+-]?\d+)", line)
+ if match:
+ status = match.group(1)
+ full_name = "%s#%s" % (self.class_name, self.test_name)
+ if full_name == self.current_full_name:
+ # A crash in the test harness might cause us to ignore tests,
+ # so we double check that we've actually ran all the tests
+ if self.total_count == self.current_test_id:
+ self.seen_last_test = True
+
+ if status == "0":
+ message = ""
+ status = "PASS"
+ expected = "PASS"
+ self.pass_count += 1
+ if self.verbose:
+ self.log.info("Printing logcat for test:")
+ print(self.collectLogcatForCurrentTest())
+ elif status == "-3": # ignored (skipped)
+ message = ""
+ status = "SKIP"
+ expected = "SKIP"
+ self.todo_count += 1
+ elif status == "-4": # known fail
+ message = ""
+ status = "FAIL"
+ expected = "FAIL"
+ self.todo_count += 1
+ else:
+ if self.exception_message:
+ message = self.exception_message
+ else:
+ message = "status %s" % status
+ status = "FAIL"
+ expected = "PASS"
+ self.fail_count += 1
+ self.log.info("Printing logcat for test:")
+ print(self.collectLogcatForCurrentTest())
+ self.log.test_end(full_name, status, expected, message)
+ self.test_started = False
+ else:
+ if self.test_started:
+ # next test started without reporting previous status
+ self.fail_count += 1
+ status = "FAIL"
+ expected = "PASS"
+ self.log.test_end(
+ self.current_full_name,
+ status,
+ expected,
+ "missing test completion status",
+ )
+ self.log.test_start(full_name)
+ self.test_started = True
+ self.current_full_name = full_name
+
+ # Ideally all test names should be reported to suite_start, but these test
+ # names are not known in advance.
+ self.log.suite_start(["geckoview-junit"])
+ try:
+ self.device.grant_runtime_permissions(self.options.app)
+ self.device.add_change_device_settings(self.options.app)
+ self.device.add_mock_location(self.options.app)
+ cmd = self.build_command_line(
+ test_filters_file=test_filters_file, test_filters=test_filters
+ )
+ while self.need_more_runs():
+ self.class_name = ""
+ self.exception_message = ""
+ self.test_name = ""
+ self.current_full_name = ""
+ self.current_test_id = 0
+ self.runs += 1
+ self.log.info("launching %s" % cmd)
+ p = self.device.shell(
+ cmd, timeout=self.options.max_time, stdout_callback=callback
+ )
+ if p.timedout:
+ self.log.error(
+ "TEST-UNEXPECTED-TIMEOUT | runjunit.py | "
+ "Timed out after %d seconds" % self.options.max_time
+ )
+ self.log.info("Passed: %d" % self.pass_count)
+ self.log.info("Failed: %d" % self.fail_count)
+ self.log.info("Todo: %d" % self.todo_count)
+ if not self.seen_last_test:
+ self.log.error(
+ "TEST-UNEXPECTED-FAIL | runjunit.py | "
+ "Some tests did not run (probably due to a crash in the harness)"
+ )
+ finally:
+ self.log.suite_end()
+
+ if self.check_for_crashes():
+ self.fail_count = 1
+
+ if self.options.coverage:
+ try:
+ self.device.pull(
+ self.remote_coverage_output_file, self.coverage_output_file
+ )
+ except ADBError:
+ # Avoid a task retry in case the code coverage file is not found.
+ self.log.error(
+ "No code coverage file (%s) found on remote device"
+ % self.remote_coverage_output_file
+ )
+ return -1
+
+ return 1 if self.fail_count else 0
+
+ def check_for_crashes(self):
+ symbols_path = self.options.symbolsPath
+ try:
+ dump_dir = tempfile.mkdtemp()
+ remote_dir = posixpath.join(self.remote_profile, "minidumps")
+ if not self.device.is_dir(remote_dir):
+ return False
+ self.device.pull(remote_dir, dump_dir)
+ crashed = mozcrash.log_crashes(
+ self.log, dump_dir, symbols_path, test=self.current_full_name
+ )
+ finally:
+ try:
+ shutil.rmtree(dump_dir)
+ except Exception:
+ self.log.warning("unable to remove directory: %s" % dump_dir)
+ return crashed
+
+
+class JunitArgumentParser(argparse.ArgumentParser):
+ """
+ An argument parser for geckoview-junit.
+ """
+
+ def __init__(self, **kwargs):
+ super(JunitArgumentParser, self).__init__(**kwargs)
+
+ self.add_argument(
+ "--appname",
+ action="store",
+ type=str,
+ dest="app",
+ default="org.mozilla.geckoview.test",
+ help="Test package name.",
+ )
+ self.add_argument(
+ "--adbpath",
+ action="store",
+ type=str,
+ dest="adbPath",
+ default=None,
+ help="Path to adb binary.",
+ )
+ self.add_argument(
+ "--deviceSerial",
+ action="store",
+ type=str,
+ dest="deviceSerial",
+ help="adb serial number of remote device. This is required "
+ "when more than one device is connected to the host. "
+ "Use 'adb devices' to see connected devices. ",
+ )
+ self.add_argument(
+ "--setenv",
+ dest="add_env",
+ action="append",
+ default=[],
+ help="Set target environment variable, like FOO=BAR",
+ )
+ self.add_argument(
+ "--remoteTestRoot",
+ action="store",
+ type=str,
+ dest="remoteTestRoot",
+ help="Remote directory to use as test root "
+ "(eg. /data/local/tmp/test_root).",
+ )
+ self.add_argument(
+ "--max-time",
+ action="store",
+ type=int,
+ dest="max_time",
+ default="3000",
+ help="Max time in seconds to wait for tests (default 3000s).",
+ )
+ self.add_argument(
+ "--runner",
+ action="store",
+ type=str,
+ dest="runner",
+ default="androidx.test.runner.AndroidJUnitRunner",
+ help="Test runner name.",
+ )
+ self.add_argument(
+ "--symbols-path",
+ action="store",
+ type=str,
+ dest="symbolsPath",
+ default=None,
+ help="Path to directory containing breakpad symbols, "
+ "or the URL of a zip file containing symbols.",
+ )
+ self.add_argument(
+ "--utility-path",
+ action="store",
+ type=str,
+ dest="utilityPath",
+ default=None,
+ help="Path to directory containing host utility programs.",
+ )
+ self.add_argument(
+ "--total-chunks",
+ action="store",
+ type=int,
+ dest="totalChunks",
+ default=None,
+ help="Total number of chunks to split tests into.",
+ )
+ self.add_argument(
+ "--this-chunk",
+ action="store",
+ type=int,
+ dest="thisChunk",
+ default=None,
+ help="If running tests by chunks, the chunk number to run.",
+ )
+ self.add_argument(
+ "--verbose",
+ "-v",
+ action="store_true",
+ dest="verbose",
+ default=False,
+ help="Verbose output - enable debug log messages",
+ )
+ self.add_argument(
+ "--enable-coverage",
+ action="store_true",
+ dest="coverage",
+ default=False,
+ help="Enable code coverage collection.",
+ )
+ self.add_argument(
+ "--coverage-output-dir",
+ action="store",
+ type=str,
+ dest="coverage_output_dir",
+ default=None,
+ help="If collecting code coverage, save the report file in this dir.",
+ )
+ self.add_argument(
+ "--disable-fission",
+ action="store_true",
+ dest="disable_fission",
+ default=False,
+ help="Run the tests without Fission (site isolation) enabled.",
+ )
+ self.add_argument(
+ "--web-content-isolation-strategy",
+ type=int,
+ dest="web_content_isolation_strategy",
+ help="Strategy used to determine whether or not a particular site should load into "
+ "a webIsolated content process, see fission.webContentIsolationStrategy.",
+ )
+ self.add_argument(
+ "--repeat",
+ type=int,
+ default=0,
+ help="Repeat the tests the given number of times.",
+ )
+ self.add_argument(
+ "--run-until-failure",
+ action="store_true",
+ dest="run_until_failure",
+ default=False,
+ help="Run tests repeatedly but stop the first time a test fails.",
+ )
+ self.add_argument(
+ "--setpref",
+ action="append",
+ dest="extra_prefs",
+ default=[],
+ metavar="PREF=VALUE",
+ help="Defines an extra user preference.",
+ )
+ # Additional options for server.
+ self.add_argument(
+ "--certificate-path",
+ action="store",
+ type=str,
+ dest="certPath",
+ default=None,
+ help="Path to directory containing certificate store.",
+ ),
+ self.add_argument(
+ "--http-port",
+ action="store",
+ type=str,
+ dest="httpPort",
+ default=DEFAULT_PORTS["http"],
+ help="http port of the remote web server.",
+ )
+ self.add_argument(
+ "--remote-webserver",
+ action="store",
+ type=str,
+ dest="remoteWebServer",
+ help="IP address of the remote web server.",
+ )
+ self.add_argument(
+ "--ssl-port",
+ action="store",
+ type=str,
+ dest="sslPort",
+ default=DEFAULT_PORTS["https"],
+ help="ssl port of the remote web server.",
+ )
+ self.add_argument(
+ "--test-filters-file",
+ action="store",
+ type=str,
+ dest="test_filters_file",
+ default=None,
+ help="Line-delimited file containing test filter(s)",
+ )
+ # Remaining arguments are test filters.
+ self.add_argument(
+ "test_filters",
+ nargs="*",
+ help="Test filter(s): class and/or method names of test(s) to run.",
+ )
+
+ mozlog.commandline.add_logging_group(self)
+
+
+def run_test_harness(parser, options):
+ if hasattr(options, "log"):
+ log = options.log
+ else:
+ log = mozlog.commandline.setup_logging(
+ "runjunit", options, {"tbpl": sys.stdout}
+ )
+ runner = JUnitTestRunner(log, options)
+ result = -1
+ try:
+ device_exception = False
+ result = runner.run_tests(
+ test_filters_file=options.test_filters_file,
+ test_filters=options.test_filters,
+ )
+ except KeyboardInterrupt:
+ log.info("runjunit.py | Received keyboard interrupt")
+ result = -1
+ except JavaTestHarnessException as e:
+ log.error(
+ "TEST-UNEXPECTED-FAIL | runjunit.py | The previous test failed because "
+ "of an error in the test harness | %s" % (str(e))
+ )
+ except Exception as e:
+ traceback.print_exc()
+ log.error("runjunit.py | Received unexpected exception while running tests")
+ result = 1
+ if isinstance(e, ADBTimeoutError):
+ device_exception = True
+ finally:
+ if not device_exception:
+ runner.cleanup()
+ return result
+
+
+def main(args=sys.argv[1:]):
+ parser = JunitArgumentParser()
+ options = parser.parse_args()
+ return run_test_harness(parser, options)
+
+
+if __name__ == "__main__":
+ sys.exit(main())