diff options
Diffstat (limited to 'layout/tools/reftest/remotereftest.py')
| -rw-r--r-- | layout/tools/reftest/remotereftest.py | 581 |
1 files changed, 581 insertions, 0 deletions
diff --git a/layout/tools/reftest/remotereftest.py b/layout/tools/reftest/remotereftest.py new file mode 100644 index 0000000000..ac2f0bbc2c --- /dev/null +++ b/layout/tools/reftest/remotereftest.py @@ -0,0 +1,581 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function + +import datetime +import os +import posixpath +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import traceback +from contextlib import closing + +from six.moves.urllib_request import urlopen + +from mozdevice import ADBDeviceFactory, ADBTimeoutError, RemoteProcessMonitor +import mozcrash + +from output import OutputHandler +from runreftest import RefTest, ReftestResolver, build_obj +import reftestcommandline + +# We need to know our current directory so that we can serve our test files from it. +SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) + + +class RemoteReftestResolver(ReftestResolver): + def absManifestPath(self, path): + script_abs_path = os.path.join(SCRIPT_DIRECTORY, path) + if os.path.exists(script_abs_path): + rv = script_abs_path + elif os.path.exists(os.path.abspath(path)): + rv = os.path.abspath(path) + else: + print("Could not find manifest %s" % script_abs_path, file=sys.stderr) + sys.exit(1) + return os.path.normpath(rv) + + def manifestURL(self, options, path): + # Dynamically build the reftest URL if possible, beware that + # args[0] should exist 'inside' webroot. It's possible for + # this url to have a leading "..", but reftest.js will fix + # that. Use the httpdPath to determine if we are running in + # production or locally. If we are running the jsreftests + # locally, strip text up to jsreftest. We want the docroot of + # the server to include a link jsreftest that points to the + # test-stage location of the test files. The desktop oriented + # setup has already created a link for tests which points + # directly into the source tree. For the remote tests we need + # a separate symbolic link to point to the staged test files. + if "jsreftest" not in path or os.environ.get("MOZ_AUTOMATION"): + relPath = os.path.relpath(path, SCRIPT_DIRECTORY) + else: + relPath = "jsreftest/" + path.split("jsreftest/")[-1] + return "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, relPath) + + +class ReftestServer: + """Web server used to serve Reftests, for closer fidelity to the real web. + It is virtually identical to the server used in mochitest and will only + be used for running reftests remotely. + Bug 581257 has been filed to refactor this wrapper around httpd.js into + it's own class and use it in both remote and non-remote testing.""" + + def __init__(self, options, scriptDir, log): + self.log = log + self.utilityPath = options.utilityPath + self.xrePath = options.xrePath + self.profileDir = options.serverProfilePath + self.webServer = options.remoteWebServer + self.httpPort = options.httpPort + self.scriptDir = scriptDir + self.httpdPath = os.path.abspath(options.httpdPath) + if options.remoteWebServer == "10.0.2.2": + # probably running an Android emulator and 10.0.2.2 will + # not be visible from host + shutdownServer = "127.0.0.1" + else: + shutdownServer = self.webServer + self.shutdownURL = "http://%(server)s:%(port)s/server/shutdown" % { + "server": shutdownServer, + "port": self.httpPort, + } + + def start(self): + "Run the Refest server, returning the process ID of the server." + + env = dict(os.environ) + env["XPCOM_DEBUG_BREAK"] = "warn" + bin_suffix = "" + if sys.platform in ("win32", "msys", "cygwin"): + env["PATH"] = env["PATH"] + ";" + self.xrePath + bin_suffix = ".exe" + else: + if "LD_LIBRARY_PATH" not in env or env["LD_LIBRARY_PATH"] is None: + env["LD_LIBRARY_PATH"] = self.xrePath + else: + env["LD_LIBRARY_PATH"] = ":".join( + [self.xrePath, env["LD_LIBRARY_PATH"]] + ) + + args = [ + "-g", + self.xrePath, + "-f", + os.path.join(self.httpdPath, "httpd.js"), + "-e", + "const _PROFILE_PATH = '%(profile)s';const _SERVER_PORT = " + "'%(port)s'; const _SERVER_ADDR ='%(server)s';" + % { + "profile": self.profileDir.replace("\\", "\\\\"), + "port": self.httpPort, + "server": self.webServer, + }, + "-f", + os.path.join(self.scriptDir, "server.js"), + ] + + xpcshell = os.path.join(self.utilityPath, "xpcshell" + bin_suffix) + + if not os.access(xpcshell, os.F_OK): + raise Exception("xpcshell not found at %s" % xpcshell) + if RemoteProcessMonitor.elf_arm(xpcshell): + raise Exception( + "xpcshell at %s is an ARM binary; please use " + "the --utility-path argument to specify the path " + "to a desktop version." % xpcshell + ) + + self._process = subprocess.Popen([xpcshell] + args, env=env) + pid = self._process.pid + if pid < 0: + self.log.error( + "TEST-UNEXPECTED-FAIL | remotereftests.py | Error starting server." + ) + return 2 + self.log.info("INFO | remotereftests.py | Server pid: %d" % pid) + + def ensureReady(self, timeout): + assert timeout >= 0 + + aliveFile = os.path.join(self.profileDir, "server_alive.txt") + i = 0 + while i < timeout: + if os.path.exists(aliveFile): + break + time.sleep(1) + i += 1 + else: + self.log.error( + "TEST-UNEXPECTED-FAIL | remotereftests.py | " + "Timed out while waiting for server startup." + ) + self.stop() + return 1 + + def stop(self): + if hasattr(self, "_process"): + try: + with closing(urlopen(self.shutdownURL)) as c: + c.read() + + rtncode = self._process.poll() + if rtncode is None: + self._process.terminate() + except Exception: + self.log.info("Failed to shutdown server at %s" % self.shutdownURL) + traceback.print_exc() + self._process.kill() + + +class RemoteReftest(RefTest): + use_marionette = False + resolver_cls = RemoteReftestResolver + + def __init__(self, options, scriptDir): + RefTest.__init__(self, options.suite) + self.run_by_manifest = False + self.scriptDir = scriptDir + self.localLogName = options.localLogName + + verbose = False + if ( + options.log_mach_verbose + or options.log_tbpl_level == "debug" + or options.log_mach_level == "debug" + or options.log_raw_level == "debug" + ): + verbose = True + print("set verbose!") + expected = options.app.split("/")[-1] + self.device = ADBDeviceFactory( + adb=options.adb_path or "adb", + device=options.deviceSerial, + test_root=options.remoteTestRoot, + verbose=verbose, + run_as_package=expected, + ) + if options.remoteTestRoot is None: + options.remoteTestRoot = posixpath.join(self.device.test_root, "reftest") + options.remoteProfile = posixpath.join(options.remoteTestRoot, "profile") + options.remoteLogFile = posixpath.join(options.remoteTestRoot, "reftest.log") + options.logFile = options.remoteLogFile + self.remoteProfile = options.remoteProfile + self.remoteTestRoot = options.remoteTestRoot + + if not options.ignoreWindowSize: + parts = self.device.get_info("screen")["screen"][0].split() + width = int(parts[0].split(":")[1]) + height = int(parts[1].split(":")[1]) + if width < 1366 or height < 1050: + self.error( + "ERROR: Invalid screen resolution %sx%s, " + "please adjust to 1366x1050 or higher" % (width, height) + ) + + self._populate_logger(options) + self.outputHandler = OutputHandler( + self.log, options.utilityPath, options.symbolsPath + ) + + self.SERVER_STARTUP_TIMEOUT = 90 + + self.remoteCache = os.path.join(options.remoteTestRoot, "cache/") + + # Check that Firefox is installed + expected = options.app.split("/")[-1] + if not self.device.is_app_installed(expected): + raise Exception("%s is not installed on this device" % expected) + self.device.run_as_package = expected + self.device.clear_logcat() + + self.device.rm(self.remoteCache, force=True, recursive=True) + + procName = options.app.split("/")[-1] + self.device.stop_application(procName) + if self.device.process_exist(procName): + self.log.error("unable to kill %s before starting tests!" % procName) + + def findPath(self, paths, filename=None): + for path in paths: + p = path + if filename: + p = os.path.join(p, filename) + if os.path.exists(self.getFullPath(p)): + return path + return None + + def startWebServer(self, options): + """ Create the webserver on the host and start it up """ + remoteXrePath = options.xrePath + remoteUtilityPath = options.utilityPath + + paths = [options.xrePath] + if build_obj: + paths.append(os.path.join(build_obj.topobjdir, "dist", "bin")) + options.xrePath = self.findPath(paths) + if options.xrePath is None: + print( + "ERROR: unable to find xulrunner path for %s, " + "please specify with --xre-path" % (os.name) + ) + return 1 + paths.append("bin") + paths.append(os.path.join("..", "bin")) + + xpcshell = "xpcshell" + if os.name == "nt": + xpcshell += ".exe" + + if options.utilityPath: + paths.insert(0, options.utilityPath) + options.utilityPath = self.findPath(paths, xpcshell) + if options.utilityPath is None: + print( + "ERROR: unable to find utility path for %s, " + "please specify with --utility-path" % (os.name) + ) + return 1 + + options.serverProfilePath = tempfile.mkdtemp() + self.server = ReftestServer(options, self.scriptDir, self.log) + retVal = self.server.start() + if retVal: + return retVal + retVal = self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT) + if retVal: + return retVal + + options.xrePath = remoteXrePath + options.utilityPath = remoteUtilityPath + return 0 + + def stopWebServer(self, options): + self.server.stop() + + def killNamedProc(self, pname, orphans=True): + """ Kill processes matching the given command name """ + try: + import psutil + except ImportError as e: + self.log.warning("Unable to import psutil: %s" % str(e)) + self.log.warning("Unable to verify that %s is not already running." % pname) + return + + self.log.info("Checking for %s processes..." % pname) + + for proc in psutil.process_iter(): + try: + if proc.name() == pname: + procd = proc.as_dict(attrs=["pid", "ppid", "name", "username"]) + if proc.ppid() == 1 or not orphans: + self.log.info("killing %s" % procd) + try: + os.kill( + proc.pid, getattr(signal, "SIGKILL", signal.SIGTERM) + ) + except Exception as e: + self.log.info( + "Failed to kill process %d: %s" % (proc.pid, str(e)) + ) + else: + self.log.info("NOT killing %s (not an orphan?)" % procd) + except Exception: + # may not be able to access process info for all processes + continue + + def createReftestProfile(self, options, **kwargs): + profile = RefTest.createReftestProfile( + self, + options, + server=options.remoteWebServer, + port=options.httpPort, + **kwargs + ) + profileDir = profile.profile + prefs = {} + prefs["app.update.url.android"] = "" + prefs["reftest.remote"] = True + prefs["datareporting.policy.dataSubmissionPolicyBypassAcceptance"] = True + # move necko cache to a location that can be cleaned up + prefs["browser.cache.disk.parent_directory"] = self.remoteCache + + prefs["layout.css.devPixelsPerPx"] = "1.0" + # Because Fennec is a little wacky (see bug 1156817) we need to load the + # reftest pages at 1.0 zoom, rather than zooming to fit the CSS viewport. + prefs["apz.allow_zooming"] = False + + # Set the extra prefs. + profile.set_preferences(prefs) + + try: + self.device.push(profileDir, options.remoteProfile) + # make sure the parent directories of the profile which + # may have been created by the push, also have their + # permissions set to allow access. + self.device.chmod(options.remoteTestRoot, recursive=True) + except Exception: + print("Automation Error: Failed to copy profiledir to device") + raise + + return profile + + def printDeviceInfo(self, printLogcat=False): + try: + if printLogcat: + logcat = self.device.get_logcat() + for l in logcat: + ul = l.decode("utf-8", errors="replace") + sl = ul.encode("iso8859-1", errors="replace") + print("%s\n" % sl) + print("Device info:") + devinfo = self.device.get_info() + for category in devinfo: + if type(devinfo[category]) is list: + print(" %s:" % category) + for item in devinfo[category]: + print(" %s" % item) + else: + print(" %s: %s" % (category, devinfo[category])) + print("Test root: %s" % self.device.test_root) + except ADBTimeoutError: + raise + except Exception as e: + print("WARNING: Error getting device information: %s" % str(e)) + + def environment(self, env=None, crashreporter=True, **kwargs): + # Since running remote, do not mimic the local env: do not copy os.environ + if env is None: + env = {} + + if crashreporter: + env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" + env["MOZ_CRASHREPORTER"] = "1" + env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" + else: + env["MOZ_CRASHREPORTER_DISABLE"] = "1" + + # Crash on non-local network connections by default. + # MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily + # enable non-local connections for the purposes of local testing. + # Don't override the user's choice here. See bug 1049688. + env.setdefault("MOZ_DISABLE_NONLOCAL_CONNECTIONS", "1") + + # Send an env var noting that we are in automation. Passing any + # value except the empty string will declare the value to exist. + # + # This may be used to disabled network connections during testing, e.g. + # Switchboard & telemetry uploads. + env.setdefault("MOZ_IN_AUTOMATION", "1") + + # Set WebRTC logging in case it is not set yet. + env.setdefault("R_LOG_LEVEL", "6") + env.setdefault("R_LOG_DESTINATION", "stderr") + env.setdefault("R_LOG_VERBOSE", "1") + + return env + + def buildBrowserEnv(self, options, profileDir): + browserEnv = RefTest.buildBrowserEnv(self, options, profileDir) + # remove desktop environment not used on device + if "XPCOM_MEM_BLOAT_LOG" in browserEnv: + del browserEnv["XPCOM_MEM_BLOAT_LOG"] + return browserEnv + + def runApp( + self, + options, + cmdargs=None, + timeout=None, + debuggerInfo=None, + symbolsPath=None, + valgrindPath=None, + valgrindArgs=None, + valgrindSuppFiles=None, + **profileArgs + ): + if cmdargs is None: + cmdargs = [] + + if self.use_marionette: + cmdargs.append("-marionette") + + binary = options.app + profile = self.createReftestProfile(options, **profileArgs) + + # browser environment + env = self.buildBrowserEnv(options, profile.profile) + + self.log.info("Running with e10s: {}".format(options.e10s)) + self.log.info("Running with fission: {}".format(options.fission)) + + rpm = RemoteProcessMonitor( + binary, + self.device, + self.log, + self.outputHandler, + options.remoteLogFile, + self.remoteProfile, + ) + startTime = datetime.datetime.now() + status = 0 + profileDirectory = self.remoteProfile + "/" + cmdargs.extend(("-no-remote", "-profile", profileDirectory)) + + pid = rpm.launch( + binary, + debuggerInfo, + None, + cmdargs, + env=env, + e10s=options.e10s, + ) + self.log.info("remotereftest.py | Application pid: %d" % pid) + if not rpm.wait(timeout): + status = 1 + self.log.info( + "remotereftest.py | Application ran for: %s" + % str(datetime.datetime.now() - startTime) + ) + crashed = self.check_for_crashes(symbolsPath, rpm.last_test_seen) + if crashed: + status = 1 + + self.cleanup(profile.profile) + return status + + def check_for_crashes(self, symbols_path, last_test_seen): + """ + Pull any minidumps from remote profile and log any associated crashes. + """ + try: + dump_dir = tempfile.mkdtemp() + remote_crash_dir = posixpath.join(self.remoteProfile, "minidumps") + if not self.device.is_dir(remote_crash_dir): + return False + self.device.pull(remote_crash_dir, dump_dir) + crashed = mozcrash.log_crashes( + self.log, dump_dir, symbols_path, test=last_test_seen + ) + finally: + try: + shutil.rmtree(dump_dir) + except Exception as e: + self.log.warning( + "unable to remove directory %s: %s" % (dump_dir, str(e)) + ) + return crashed + + def cleanup(self, profileDir): + self.device.rm(self.remoteTestRoot, force=True, recursive=True) + self.device.rm(self.remoteProfile, force=True, recursive=True) + self.device.rm(self.remoteCache, force=True, recursive=True) + RefTest.cleanup(self, profileDir) + + +def run_test_harness(parser, options): + reftest = RemoteReftest(options, SCRIPT_DIRECTORY) + parser.validate_remote(options) + parser.validate(options, reftest) + + # Hack in a symbolic link for jsreftest in the SCRIPT_DIRECTORY + # which is the document root for the reftest web server. This + # allows a separate redirection for the jsreftests which must + # run through the web server using the staged tests files and + # the desktop which will use the tests symbolic link to find + # the JavaScript tests. + jsreftest_target = str(os.path.join(SCRIPT_DIRECTORY, "jsreftest")) + if os.environ.get("MOZ_AUTOMATION"): + os.system("ln -s ../jsreftest " + jsreftest_target) + else: + jsreftest_source = os.path.join( + build_obj.topobjdir, "dist", "test-stage", "jsreftest" + ) + if not os.path.islink(jsreftest_target): + os.symlink(jsreftest_source, jsreftest_target) + + # Despite our efforts to clean up servers started by this script, in practice + # we still see infrequent cases where a process is orphaned and interferes + # with future tests, typically because the old server is keeping the port in use. + # Try to avoid those failures by checking for and killing servers before + # trying to start new ones. + reftest.killNamedProc("ssltunnel") + reftest.killNamedProc("xpcshell") + + # Start the webserver + retVal = reftest.startWebServer(options) + if retVal: + return retVal + + if options.printDeviceInfo and not options.verify: + reftest.printDeviceInfo() + + retVal = 0 + try: + if options.verify: + retVal = reftest.verifyTests(options.tests, options) + else: + retVal = reftest.runTests(options.tests, options) + except Exception: + print("Automation Error: Exception caught while running tests") + traceback.print_exc() + retVal = 1 + + reftest.stopWebServer(options) + + if options.printDeviceInfo and not options.verify: + reftest.printDeviceInfo(printLogcat=(retVal != 0)) + + return retVal + + +if __name__ == "__main__": + parser = reftestcommandline.RemoteArgumentsParser() + options = parser.parse_args() + sys.exit(run_test_harness(parser, options)) |