diff options
Diffstat (limited to 'testing/mochitest/runtestsremote.py')
| -rw-r--r-- | testing/mochitest/runtestsremote.py | 489 |
1 files changed, 489 insertions, 0 deletions
diff --git a/testing/mochitest/runtestsremote.py b/testing/mochitest/runtestsremote.py new file mode 100644 index 0000000000..4bec8b92f1 --- /dev/null +++ b/testing/mochitest/runtestsremote.py @@ -0,0 +1,489 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import +import datetime +import os +import posixpath +import shutil +import sys +import tempfile +import traceback +import uuid + +sys.path.insert(0, os.path.abspath(os.path.realpath(os.path.dirname(__file__)))) + +from runtests import MochitestDesktop, MessageLogger +from mochitest_options import MochitestArgumentParser, build_obj +from mozdevice import ADBDeviceFactory, ADBTimeoutError, RemoteProcessMonitor +from mozscreenshot import dump_screen, dump_device_screen +import mozcrash +import mozinfo + +SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) + + +class MochiRemote(MochitestDesktop): + localProfile = None + logMessages = [] + + def __init__(self, options): + MochitestDesktop.__init__(self, options.flavor, vars(options)) + + verbose = False + if ( + options.log_mach_verbose + or options.log_tbpl_level == "debug" + or options.log_mach_level == "debug" + or options.log_raw_level == "debug" + ): + verbose = True + if hasattr(options, "log"): + delattr(options, "log") + + self.certdbNew = True + self.chromePushed = False + + expected = options.app.split("/")[-1] + self.device = ADBDeviceFactory( + adb=options.adbPath or "adb", + device=options.deviceSerial, + test_root=options.remoteTestRoot, + verbose=verbose, + run_as_package=expected, + ) + + if options.remoteTestRoot is None: + options.remoteTestRoot = self.device.test_root + options.dumpOutputDirectory = options.remoteTestRoot + self.remoteLogFile = posixpath.join( + options.remoteTestRoot, "logs", "mochitest.log" + ) + logParent = posixpath.dirname(self.remoteLogFile) + self.device.rm(logParent, force=True, recursive=True) + self.device.mkdir(logParent, parents=True) + + self.remoteProfile = posixpath.join(options.remoteTestRoot, "profile") + self.device.rm(self.remoteProfile, force=True, recursive=True) + + self.message_logger = MessageLogger(logger=None) + self.message_logger.logger = self.log + + # Check that Firefox is installed + expected = options.app.split("/")[-1] + if not self.device.is_app_installed(expected): + raise Exception("%s is not installed on this device" % expected) + + self.device.clear_logcat() + + self.remoteModulesDir = posixpath.join(options.remoteTestRoot, "modules/") + + self.remoteCache = posixpath.join(options.remoteTestRoot, "cache/") + self.device.rm(self.remoteCache, force=True, recursive=True) + + # move necko cache to a location that can be cleaned up + options.extraPrefs += [ + "browser.cache.disk.parent_directory=%s" % self.remoteCache + ] + + self.remoteMozLog = posixpath.join(options.remoteTestRoot, "mozlog") + self.device.rm(self.remoteMozLog, force=True, recursive=True) + self.device.mkdir(self.remoteMozLog, parents=True) + + self.remoteChromeTestDir = posixpath.join(options.remoteTestRoot, "chrome") + self.device.rm(self.remoteChromeTestDir, force=True, recursive=True) + self.device.mkdir(self.remoteChromeTestDir, parents=True) + + self.appName = options.remoteappname + self.device.stop_application(self.appName) + if self.device.process_exist(self.appName): + self.log.warning("unable to kill %s before running tests!" % self.appName) + + # Add Android version (SDK level) to mozinfo so that manifest entries + # can be conditional on android_version. + self.log.info( + "Android sdk version '%s'; will use this to filter manifests" + % str(self.device.version) + ) + mozinfo.info["android_version"] = str(self.device.version) + mozinfo.info["is_fennec"] = not ("geckoview" in options.app) + mozinfo.info["is_emulator"] = self.device._device_serial.startswith("emulator-") + + def cleanup(self, options, final=False): + if final: + self.device.rm(self.remoteChromeTestDir, force=True, recursive=True) + self.chromePushed = False + uploadDir = os.environ.get("MOZ_UPLOAD_DIR", None) + if uploadDir and self.device.is_dir(self.remoteMozLog): + self.device.pull(self.remoteMozLog, uploadDir) + self.device.rm(self.remoteLogFile, force=True) + self.device.rm(self.remoteProfile, force=True, recursive=True) + self.device.rm(self.remoteCache, force=True, recursive=True) + MochitestDesktop.cleanup(self, options, final) + self.localProfile = None + + def dumpScreen(self, utilityPath): + if self.haveDumpedScreen: + self.log.info( + "Not taking screenshot here: see the one that was previously logged" + ) + return + self.haveDumpedScreen = True + if self.device._device_serial.startswith("emulator-"): + dump_screen(utilityPath, self.log) + else: + dump_device_screen(self.device, self.log) + + def findPath(self, paths, filename=None): + for path in paths: + p = path + if filename: + p = os.path.join(p, filename) + if os.path.exists(self.getFullPath(p)): + return path + return None + + # This seems kludgy, but this class uses paths from the remote host in the + # options, except when calling up to the base class, which doesn't + # understand the distinction. This switches out the remote values for local + # ones that the base class understands. This is necessary for the web + # server, SSL tunnel and profile building functions. + def switchToLocalPaths(self, options): + """ Set local paths in the options, return a function that will restore remote values """ + remoteXrePath = options.xrePath + remoteProfilePath = options.profilePath + remoteUtilityPath = options.utilityPath + + paths = [ + options.xrePath, + ] + if build_obj: + paths.append(os.path.join(build_obj.topobjdir, "dist", "bin")) + options.xrePath = self.findPath(paths) + if options.xrePath is None: + self.log.error( + "unable to find xulrunner path for %s, please specify with --xre-path" + % os.name + ) + sys.exit(1) + + xpcshell = "xpcshell" + if os.name == "nt": + xpcshell += ".exe" + + if options.utilityPath: + paths = [options.utilityPath, options.xrePath] + else: + paths = [options.xrePath] + options.utilityPath = self.findPath(paths, xpcshell) + + if options.utilityPath is None: + self.log.error( + "unable to find utility path for %s, please specify with --utility-path" + % os.name + ) + sys.exit(1) + + xpcshell_path = os.path.join(options.utilityPath, xpcshell) + if RemoteProcessMonitor.elf_arm(xpcshell_path): + self.log.error( + "xpcshell at %s is an ARM binary; please use " + "the --utility-path argument to specify the path " + "to a desktop version." % xpcshell_path + ) + sys.exit(1) + + if self.localProfile: + options.profilePath = self.localProfile + else: + options.profilePath = None + + def fixup(): + options.xrePath = remoteXrePath + options.utilityPath = remoteUtilityPath + options.profilePath = remoteProfilePath + + return fixup + + def startServers(self, options, debuggerInfo, public=None): + """ Create the servers on the host and start them up """ + restoreRemotePaths = self.switchToLocalPaths(options) + MochitestDesktop.startServers(self, options, debuggerInfo, public=True) + restoreRemotePaths() + + def buildProfile(self, options): + restoreRemotePaths = self.switchToLocalPaths(options) + if options.testingModulesDir: + try: + self.device.push(options.testingModulesDir, self.remoteModulesDir) + self.device.chmod(self.remoteModulesDir, recursive=True) + except Exception: + self.log.error( + "Automation Error: Unable to copy test modules to device." + ) + raise + savedTestingModulesDir = options.testingModulesDir + options.testingModulesDir = self.remoteModulesDir + else: + savedTestingModulesDir = None + manifest = MochitestDesktop.buildProfile(self, options) + if savedTestingModulesDir: + options.testingModulesDir = savedTestingModulesDir + self.localProfile = options.profilePath + + restoreRemotePaths() + options.profilePath = self.remoteProfile + return manifest + + def buildURLOptions(self, options, env): + saveLogFile = options.logFile + options.logFile = self.remoteLogFile + options.profilePath = self.localProfile + env["MOZ_HIDE_RESULTS_TABLE"] = "1" + retVal = MochitestDesktop.buildURLOptions(self, options, env) + + # we really need testConfig.js (for browser chrome) + try: + self.device.push(options.profilePath, self.remoteProfile) + self.device.chmod(self.remoteProfile, recursive=True) + except Exception: + self.log.error("Automation Error: Unable to copy profile to device.") + raise + + options.profilePath = self.remoteProfile + options.logFile = saveLogFile + return retVal + + def getChromeTestDir(self, options): + local = super(MochiRemote, self).getChromeTestDir(options) + remote = self.remoteChromeTestDir + if options.flavor == "chrome" and not self.chromePushed: + self.log.info("pushing %s to %s on device..." % (local, remote)) + local = os.path.join(local, "chrome") + self.device.push(local, remote) + self.chromePushed = True + return remote + + def getLogFilePath(self, logFile): + return logFile + + def printDeviceInfo(self, printLogcat=False): + try: + if printLogcat: + logcat = self.device.get_logcat() + for l in logcat: + ul = l.decode("utf-8", errors="replace") + sl = ul.encode("iso8859-1", errors="replace") + self.log.info(sl) + self.log.info("Device info:") + devinfo = self.device.get_info() + for category in devinfo: + if type(devinfo[category]) is list: + self.log.info(" %s:" % category) + for item in devinfo[category]: + self.log.info(" %s" % item) + else: + self.log.info(" %s: %s" % (category, devinfo[category])) + self.log.info("Test root: %s" % self.device.test_root) + except ADBTimeoutError: + raise + except Exception as e: + self.log.warning("Error getting device information: %s" % str(e)) + + def getGMPPluginPath(self, options): + # TODO: bug 1149374 + return None + + def environment(self, env=None, crashreporter=True, **kwargs): + # Since running remote, do not mimic the local env: do not copy os.environ + if env is None: + env = {} + + if crashreporter: + env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" + env["MOZ_CRASHREPORTER"] = "1" + env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" + else: + env["MOZ_CRASHREPORTER_DISABLE"] = "1" + + # Crash on non-local network connections by default. + # MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily + # enable non-local connections for the purposes of local testing. + # Don't override the user's choice here. See bug 1049688. + env.setdefault("MOZ_DISABLE_NONLOCAL_CONNECTIONS", "1") + + # Send an env var noting that we are in automation. Passing any + # value except the empty string will declare the value to exist. + # + # This may be used to disabled network connections during testing, e.g. + # Switchboard & telemetry uploads. + env.setdefault("MOZ_IN_AUTOMATION", "1") + + # Set WebRTC logging in case it is not set yet. + env.setdefault("R_LOG_LEVEL", "6") + env.setdefault("R_LOG_DESTINATION", "stderr") + env.setdefault("R_LOG_VERBOSE", "1") + + return env + + def buildBrowserEnv(self, options, debugger=False): + browserEnv = MochitestDesktop.buildBrowserEnv(self, options, debugger=debugger) + # remove desktop environment not used on device + if "XPCOM_MEM_BLOAT_LOG" in browserEnv: + del browserEnv["XPCOM_MEM_BLOAT_LOG"] + if self.mozLogs: + browserEnv["MOZ_LOG_FILE"] = os.path.join( + self.remoteMozLog, "moz-pid=%PID-uid={}.log".format(str(uuid.uuid4())) + ) + if options.dmd: + browserEnv["DMD"] = "1" + # Contents of remoteMozLog will be pulled from device and copied to the + # host MOZ_UPLOAD_DIR, to be made available as test artifacts. Make + # MOZ_UPLOAD_DIR available to the browser environment so that tests + # can use it as though they were running on the host. + browserEnv["MOZ_UPLOAD_DIR"] = self.remoteMozLog + return browserEnv + + def runApp( + self, + testUrl, + env, + app, + profile, + extraArgs, + utilityPath, + debuggerInfo=None, + valgrindPath=None, + valgrindArgs=None, + valgrindSuppFiles=None, + symbolsPath=None, + timeout=-1, + detectShutdownLeaks=False, + screenshotOnFail=False, + bisectChunk=None, + marionette_args=None, + e10s=True, + ): + """ + Run the app, log the duration it took to execute, return the status code. + Kill the app if it outputs nothing for |timeout| seconds. + """ + + if timeout == -1: + timeout = self.DEFAULT_TIMEOUT + + rpm = RemoteProcessMonitor( + self.appName, + self.device, + self.log, + self.message_logger, + self.remoteLogFile, + self.remoteProfile, + ) + startTime = datetime.datetime.now() + status = 0 + profileDirectory = self.remoteProfile + "/" + extraArgs.extend(("-no-remote", "-profile", profileDirectory)) + + pid = rpm.launch( + app, + debuggerInfo, + testUrl, + extraArgs, + env=self.environment(env=env, crashreporter=not debuggerInfo), + e10s=e10s, + ) + self.log.info("runtestsremote.py | Application pid: %d" % pid) + if not rpm.wait(timeout): + status = 1 + self.log.info( + "runtestsremote.py | Application ran for: %s" + % str(datetime.datetime.now() - startTime) + ) + crashed = self.check_for_crashes(symbolsPath, rpm.last_test_seen) + if crashed: + status = 1 + + self.countpass += rpm.counts["pass"] + self.countfail += rpm.counts["fail"] + self.counttodo += rpm.counts["todo"] + + return status, rpm.last_test_seen + + def check_for_crashes(self, symbols_path, last_test_seen): + """ + Pull any minidumps from remote profile and log any associated crashes. + """ + try: + dump_dir = tempfile.mkdtemp() + remote_crash_dir = posixpath.join(self.remoteProfile, "minidumps") + if not self.device.is_dir(remote_crash_dir): + return False + self.device.pull(remote_crash_dir, dump_dir) + crashed = mozcrash.log_crashes( + self.log, dump_dir, symbols_path, test=last_test_seen + ) + finally: + try: + shutil.rmtree(dump_dir) + except Exception as e: + self.log.warning( + "unable to remove directory %s: %s" % (dump_dir, str(e)) + ) + return crashed + + +def run_test_harness(parser, options): + parser.validate(options) + + if options is None: + raise ValueError( + "Invalid options specified, use --help for a list of valid options" + ) + + options.runByManifest = True + + mochitest = MochiRemote(options) + + if options.log_mach is None and not options.verify: + mochitest.printDeviceInfo() + + try: + device_exception = False + if options.verify: + retVal = mochitest.verifyTests(options) + else: + retVal = mochitest.runTests(options) + except Exception as e: + mochitest.log.error("Automation Error: Exception caught while running tests") + traceback.print_exc() + if isinstance(e, ADBTimeoutError): + mochitest.log.info("Device disconnected. Will not run mochitest.cleanup().") + device_exception = True + else: + try: + mochitest.cleanup(options) + except Exception: + # device error cleaning up... oh well! + traceback.print_exc() + retVal = 1 + + if not device_exception and options.log_mach is None and not options.verify: + mochitest.printDeviceInfo(printLogcat=(retVal != 0)) + + mochitest.archiveMozLogs() + mochitest.message_logger.finish() + + return retVal + + +def main(args=sys.argv[1:]): + parser = MochitestArgumentParser(app="android") + options = parser.parse_args(args) + + return run_test_harness(parser, options) + + +if __name__ == "__main__": + sys.exit(main()) |