diff options
Diffstat (limited to 'testing/gtest/remotegtests.py')
| -rw-r--r-- | testing/gtest/remotegtests.py | 478 |
1 files changed, 478 insertions, 0 deletions
diff --git a/testing/gtest/remotegtests.py b/testing/gtest/remotegtests.py new file mode 100644 index 0000000000..e2073b6719 --- /dev/null +++ b/testing/gtest/remotegtests.py @@ -0,0 +1,478 @@ +#!/usr/bin/env python +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import datetime +import glob +import os +import posixpath +import shutil +import sys +import tempfile +import time +import traceback + +import mozcrash +import mozdevice +import mozinfo +import mozlog +import six + +LOGGER_NAME = "gtest" +log = mozlog.unstructured.getLogger(LOGGER_NAME) + + +class RemoteGTests(object): + """ + A test harness to run gtest on Android. + """ + + def __init__(self): + self.device = None + + def build_environment(self, shuffle, test_filter): + """ + Create and return a dictionary of all the appropriate env variables + and values. + """ + env = {} + env["XPCOM_DEBUG_BREAK"] = "stack-and-abort" + env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" + env["MOZ_CRASHREPORTER"] = "1" + env["MOZ_RUN_GTEST"] = "1" + # custom output parser is mandatory on Android + env["MOZ_TBPL_PARSER"] = "1" + env["MOZ_GTEST_LOG_PATH"] = self.remote_log + env["MOZ_GTEST_CWD"] = self.remote_profile + env["MOZ_GTEST_MINIDUMPS_PATH"] = self.remote_minidumps + env["MOZ_IN_AUTOMATION"] = "1" + env["MOZ_ANDROID_LIBDIR_OVERRIDE"] = posixpath.join( + self.remote_libdir, "libxul.so" + ) + if shuffle: + env["GTEST_SHUFFLE"] = "True" + if test_filter: + env["GTEST_FILTER"] = test_filter + + # webrender needs gfx.webrender.all=true, gtest doesn't use prefs + env["MOZ_WEBRENDER"] = "1" + + return env + + def run_gtest( + self, + test_dir, + shuffle, + test_filter, + package, + adb_path, + device_serial, + remote_test_root, + libxul_path, + symbols_path, + ): + """ + Launch the test app, run gtest, collect test results and wait for completion. + Return False if a crash or other failure is detected, else True. + """ + update_mozinfo() + self.device = mozdevice.ADBDeviceFactory( + adb=adb_path, + device=device_serial, + test_root=remote_test_root, + logger_name=LOGGER_NAME, + verbose=False, + run_as_package=package, + ) + root = self.device.test_root + self.remote_profile = posixpath.join(root, "gtest-profile") + self.remote_minidumps = posixpath.join(root, "gtest-minidumps") + self.remote_log = posixpath.join(root, "gtest.log") + self.remote_libdir = posixpath.join(root, "gtest") + + self.package = package + self.cleanup() + self.device.mkdir(self.remote_profile) + self.device.mkdir(self.remote_minidumps) + self.device.mkdir(self.remote_libdir) + + log.info("Running Android gtest") + if not self.device.is_app_installed(self.package): + raise Exception("%s is not installed on this device" % self.package) + + # Push the gtest libxul.so to the device. The harness assumes an architecture- + # appropriate library is specified and pushes it to the arch-agnostic remote + # directory. + # TODO -- consider packaging the gtest libxul.so in an apk + self.device.push(libxul_path, self.remote_libdir) + + # Push support files to device. Avoid sub-directories so that libxul.so + # is not included. + for f in glob.glob(os.path.join(test_dir, "*")): + if not os.path.isdir(f): + self.device.push(f, self.remote_profile) + + if test_filter is not None: + test_filter = six.ensure_text(test_filter) + env = self.build_environment(shuffle, test_filter) + args = [ + "-unittest", + "--gtest_death_test_style=threadsafe", + "-profile %s" % self.remote_profile, + ] + if "geckoview" in self.package: + activity = "TestRunnerActivity" + self.device.launch_activity( + self.package, + activity_name=activity, + e10s=False, # gtest is non-e10s on desktop + moz_env=env, + extra_args=args, + wait=False, + ) + else: + self.device.launch_fennec(self.package, moz_env=env, extra_args=args) + waiter = AppWaiter(self.device, self.remote_log) + timed_out = waiter.wait(self.package) + self.shutdown(use_kill=True if timed_out else False) + if self.check_for_crashes(symbols_path): + return False + return True + + def shutdown(self, use_kill): + """ + Stop the remote application. + If use_kill is specified, a multi-stage kill procedure is used, + attempting to trigger ANR and minidump reports before ending + the process. + """ + if not use_kill: + self.device.stop_application(self.package) + else: + # Trigger an ANR report with "kill -3" (SIGQUIT) + try: + self.device.pkill(self.package, sig=3, attempts=1) + except mozdevice.ADBTimeoutError: + raise + except: # NOQA: E722 + pass + time.sleep(3) + # Trigger a breakpad dump with "kill -6" (SIGABRT) + try: + self.device.pkill(self.package, sig=6, attempts=1) + except mozdevice.ADBTimeoutError: + raise + except: # NOQA: E722 + pass + # Wait for process to end + retries = 0 + while retries < 3: + if self.device.process_exist(self.package): + log.info("%s still alive after SIGABRT: waiting..." % self.package) + time.sleep(5) + else: + break + retries += 1 + if self.device.process_exist(self.package): + try: + self.device.pkill(self.package, sig=9, attempts=1) + except mozdevice.ADBTimeoutError: + raise + except: # NOQA: E722 + log.warning("%s still alive after SIGKILL!" % self.package) + if self.device.process_exist(self.package): + self.device.stop_application(self.package) + # Test harnesses use the MOZ_CRASHREPORTER environment variables to suppress + # the interactive crash reporter, but that may not always be effective; + # check for and cleanup errant crashreporters. + crashreporter = "%s.CrashReporter" % self.package + if self.device.process_exist(crashreporter): + log.warning("%s unexpectedly found running. Killing..." % crashreporter) + try: + self.device.pkill(crashreporter) + except mozdevice.ADBTimeoutError: + raise + except: # NOQA: E722 + pass + if self.device.process_exist(crashreporter): + log.error("%s still running!!" % crashreporter) + + def check_for_crashes(self, symbols_path): + """ + Pull minidumps from the remote device and generate crash reports. + Returns True if a crash was detected, or suspected. + """ + try: + dump_dir = tempfile.mkdtemp() + remote_dir = self.remote_minidumps + if not self.device.is_dir(remote_dir): + return False + self.device.pull(remote_dir, dump_dir) + crashed = mozcrash.check_for_crashes( + dump_dir, symbols_path, test_name="gtest" + ) + except Exception as e: + log.error("unable to check for crashes: %s" % str(e)) + crashed = True + finally: + try: + shutil.rmtree(dump_dir) + except Exception: + log.warning("unable to remove directory: %s" % dump_dir) + return crashed + + def cleanup(self): + if self.device: + self.device.stop_application(self.package) + self.device.rm(self.remote_log, force=True) + self.device.rm(self.remote_profile, recursive=True, force=True) + self.device.rm(self.remote_minidumps, recursive=True, force=True) + self.device.rm(self.remote_libdir, recursive=True, force=True) + + +class AppWaiter(object): + def __init__( + self, + device, + remote_log, + test_proc_timeout=1200, + test_proc_no_output_timeout=300, + test_proc_start_timeout=60, + output_poll_interval=10, + ): + self.device = device + self.remote_log = remote_log + self.start_time = datetime.datetime.now() + self.timeout_delta = datetime.timedelta(seconds=test_proc_timeout) + self.output_timeout_delta = datetime.timedelta( + seconds=test_proc_no_output_timeout + ) + self.start_timeout_delta = datetime.timedelta(seconds=test_proc_start_timeout) + self.output_poll_interval = output_poll_interval + self.last_output_time = datetime.datetime.now() + self.remote_log_len = 0 + + def start_timed_out(self): + if datetime.datetime.now() - self.start_time > self.start_timeout_delta: + return True + return False + + def timed_out(self): + if datetime.datetime.now() - self.start_time > self.timeout_delta: + return True + return False + + def output_timed_out(self): + if datetime.datetime.now() - self.last_output_time > self.output_timeout_delta: + return True + return False + + def get_top(self): + top = self.device.get_top_activity(timeout=60) + if top is None: + log.info("Failed to get top activity, retrying, once...") + top = self.device.get_top_activity(timeout=60) + return top + + def wait_for_start(self, package): + top = None + while top != package and not self.start_timed_out(): + if self.update_log(): + # if log content is available, assume the app started; otherwise, + # a short run (few tests) might complete without ever being detected + # in the foreground + return package + time.sleep(1) + top = self.get_top() + return top + + def wait(self, package): + """ + Wait until: + - the app loses foreground, or + - no new output is observed for the output timeout, or + - the timeout is exceeded. + While waiting, update the log every periodically: pull the gtest log from + device and log any new content. + """ + top = self.wait_for_start(package) + if top != package: + log.testFail("gtest | %s failed to start" % package) + return + while not self.timed_out(): + if not self.update_log(): + top = self.get_top() + if top != package or self.output_timed_out(): + time.sleep(self.output_poll_interval) + break + time.sleep(self.output_poll_interval) + self.update_log() + if self.timed_out(): + log.testFail( + "gtest | timed out after %d seconds", self.timeout_delta.seconds + ) + elif self.output_timed_out(): + log.testFail( + "gtest | timed out after %d seconds without output", + self.output_timeout_delta.seconds, + ) + else: + log.info("gtest | wait for %s complete; top activity=%s" % (package, top)) + return True if top == package else False + + def update_log(self): + """ + Pull the test log from the remote device and display new content. + """ + if not self.device.is_file(self.remote_log): + log.info("gtest | update_log %s is not a file." % self.remote_log) + return False + try: + new_content = self.device.get_file( + self.remote_log, offset=self.remote_log_len + ) + except mozdevice.ADBTimeoutError: + raise + except Exception as e: + log.info("gtest | update_log : exception reading log: %s" % str(e)) + return False + if not new_content: + log.info("gtest | update_log : no new content") + return False + new_content = six.ensure_text(new_content) + last_full_line_pos = new_content.rfind("\n") + if last_full_line_pos <= 0: + # wait for a full line + return False + # trim partial line + new_content = new_content[:last_full_line_pos] + self.remote_log_len += len(new_content) + for line in new_content.lstrip("\n").split("\n"): + print(line) + self.last_output_time = datetime.datetime.now() + return True + + +class remoteGtestOptions(argparse.ArgumentParser): + def __init__(self): + super(remoteGtestOptions, self).__init__( + usage="usage: %prog [options] test_filter" + ) + self.add_argument( + "--package", + dest="package", + default="org.mozilla.geckoview.test_runner", + help="Package name of test app.", + ) + self.add_argument( + "--adbpath", + action="store", + type=str, + dest="adb_path", + default="adb", + help="Path to adb binary.", + ) + self.add_argument( + "--deviceSerial", + action="store", + type=str, + dest="device_serial", + help="adb serial number of remote device. This is required " + "when more than one device is connected to the host. " + "Use 'adb devices' to see connected devices. ", + ) + self.add_argument( + "--remoteTestRoot", + action="store", + type=str, + dest="remote_test_root", + help="Remote directory to use as test root " + "(eg. /data/local/tmp/test_root).", + ) + self.add_argument( + "--libxul", + action="store", + type=str, + dest="libxul_path", + default=None, + help="Path to gtest libxul.so.", + ) + self.add_argument( + "--symbols-path", + dest="symbols_path", + default=None, + help="absolute path to directory containing breakpad " + "symbols, or the URL of a zip file containing symbols", + ) + self.add_argument( + "--shuffle", + action="store_true", + default=False, + help="Randomize the execution order of tests.", + ) + self.add_argument( + "--tests-path", + default=None, + help="Path to gtest directory containing test support files.", + ) + self.add_argument("args", nargs=argparse.REMAINDER) + + +def update_mozinfo(): + """ + Walk up directories to find mozinfo.json and update the info. + """ + path = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) + dirs = set() + while path != os.path.expanduser("~"): + if path in dirs: + break + dirs.add(path) + path = os.path.split(path)[0] + mozinfo.find_and_update_from_json(*dirs) + + +def main(): + parser = remoteGtestOptions() + options = parser.parse_args() + args = options.args + if not options.libxul_path: + parser.error("--libxul is required") + sys.exit(1) + if len(args) > 1: + parser.error("only one test_filter is allowed") + sys.exit(1) + test_filter = args[0] if args else None + tester = RemoteGTests() + result = False + try: + device_exception = False + result = tester.run_gtest( + options.tests_path, + options.shuffle, + test_filter, + options.package, + options.adb_path, + options.device_serial, + options.remote_test_root, + options.libxul_path, + options.symbols_path, + ) + except KeyboardInterrupt: + log.info("gtest | Received keyboard interrupt") + except Exception as e: + log.error(str(e)) + traceback.print_exc() + if isinstance(e, mozdevice.ADBTimeoutError): + device_exception = True + finally: + if not device_exception: + tester.cleanup() + sys.exit(0 if result else 1) + + +if __name__ == "__main__": + main() |