diff options
Diffstat (limited to 'layout/tools/reftest/runreftest.py')
| -rw-r--r-- | layout/tools/reftest/runreftest.py | 1198 |
1 files changed, 1198 insertions, 0 deletions
diff --git a/layout/tools/reftest/runreftest.py b/layout/tools/reftest/runreftest.py new file mode 100644 index 0000000000..763acc105d --- /dev/null +++ b/layout/tools/reftest/runreftest.py @@ -0,0 +1,1198 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Runs the reftest test harness. +""" +import json +import multiprocessing +import os +import platform +import posixpath +import re +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +from collections import defaultdict +from datetime import datetime, timedelta + +SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) +if SCRIPT_DIRECTORY not in sys.path: + sys.path.insert(0, SCRIPT_DIRECTORY) + +import mozcrash +import mozdebug +import mozfile +import mozinfo +import mozleak +import mozlog +import mozprocess +import mozprofile +import mozrunner +from manifestparser import TestManifest +from manifestparser import filters as mpf +from mozrunner.utils import get_stack_fixer_function, test_environment +from mozscreenshot import dump_screen, printstatus +from six import reraise, string_types +from six.moves import range + +try: + from marionette_driver.addons import Addons + from marionette_driver.marionette import Marionette +except ImportError as e: # noqa + # Defer ImportError until attempt to use Marionette. + # Python 3 deletes the exception once the except block + # is exited. Save a version to raise later. + e_save = ImportError(str(e)) + + def reraise_(*args, **kwargs): + raise (e_save) # noqa + + Marionette = reraise_ + +import reftestcommandline +from output import OutputHandler, ReftestFormatter + +here = os.path.abspath(os.path.dirname(__file__)) + +try: + from mozbuild.base import MozbuildObject + + build_obj = MozbuildObject.from_environment(cwd=here) +except ImportError: + build_obj = None + + +def categoriesToRegex(categoryList): + return "\\(" + ", ".join(["(?P<%s>\\d+) %s" % c for c in categoryList]) + "\\)" + + +summaryLines = [ + ("Successful", [("pass", "pass"), ("loadOnly", "load only")]), + ( + "Unexpected", + [ + ("fail", "unexpected fail"), + ("pass", "unexpected pass"), + ("asserts", "unexpected asserts"), + ("fixedAsserts", "unexpected fixed asserts"), + ("failedLoad", "failed load"), + ("exception", "exception"), + ], + ), + ( + "Known problems", + [ + ("knownFail", "known fail"), + ("knownAsserts", "known asserts"), + ("random", "random"), + ("skipped", "skipped"), + ("slow", "slow"), + ], + ), +] + + +if sys.version_info[0] == 3: + + def reraise_(tp_, value_, tb_=None): + if value_ is None: + value_ = tp_() + if value_.__traceback__ is not tb_: + raise value_.with_traceback(tb_) + raise value_ + + +else: + exec("def reraise_(tp_, value_, tb_=None):\n raise tp_, value_, tb_\n") + + +def update_mozinfo(): + """walk up directories to find mozinfo.json update the info""" + # TODO: This should go in a more generic place, e.g. mozinfo + + path = SCRIPT_DIRECTORY + dirs = set() + while path != os.path.expanduser("~"): + if path in dirs: + break + dirs.add(path) + path = os.path.split(path)[0] + mozinfo.find_and_update_from_json(*dirs) + + +# Python's print is not threadsafe. +printLock = threading.Lock() + + +class ReftestThread(threading.Thread): + def __init__(self, cmdargs): + threading.Thread.__init__(self) + self.cmdargs = cmdargs + self.summaryMatches = {} + self.retcode = -1 + for text, _ in summaryLines: + self.summaryMatches[text] = None + + def run(self): + with printLock: + print("Starting thread with", self.cmdargs) + sys.stdout.flush() + process = subprocess.Popen(self.cmdargs, stdout=subprocess.PIPE) + for chunk in self.chunkForMergedOutput(process.stdout): + with printLock: + print(chunk, end=" ") + sys.stdout.flush() + self.retcode = process.wait() + + def chunkForMergedOutput(self, logsource): + """Gather lines together that should be printed as one atomic unit. + Individual test results--anything between 'REFTEST TEST-START' and + 'REFTEST TEST-END' lines--are an atomic unit. Lines with data from + summaries are parsed and the data stored for later aggregation. + Other lines are considered their own atomic units and are permitted + to intermix freely.""" + testStartRegex = re.compile("^REFTEST TEST-START") + testEndRegex = re.compile("^REFTEST TEST-END") + summaryHeadRegex = re.compile("^REFTEST INFO \\| Result summary:") + summaryRegexFormatString = ( + "^REFTEST INFO \\| (?P<message>{text}): (?P<total>\\d+) {regex}" + ) + summaryRegexStrings = [ + summaryRegexFormatString.format( + text=text, regex=categoriesToRegex(categories) + ) + for (text, categories) in summaryLines + ] + summaryRegexes = [re.compile(regex) for regex in summaryRegexStrings] + + for line in logsource: + if testStartRegex.search(line) is not None: + chunkedLines = [line] + for lineToBeChunked in logsource: + chunkedLines.append(lineToBeChunked) + if testEndRegex.search(lineToBeChunked) is not None: + break + yield "".join(chunkedLines) + continue + + haveSuppressedSummaryLine = False + for regex in summaryRegexes: + match = regex.search(line) + if match is not None: + self.summaryMatches[match.group("message")] = match + haveSuppressedSummaryLine = True + break + if haveSuppressedSummaryLine: + continue + + if summaryHeadRegex.search(line) is None: + yield line + + +class ReftestResolver(object): + def defaultManifest(self, suite): + return { + "reftest": "reftest.list", + "crashtest": "crashtests.list", + "jstestbrowser": "jstests.list", + }[suite] + + def directoryManifest(self, suite, path): + return os.path.join(path, self.defaultManifest(suite)) + + def findManifest(self, suite, test_file, subdirs=True): + """Return a tuple of (manifest-path, filter-string) for running test_file. + + test_file is a path to a test or a manifest file + """ + rv = [] + default_manifest = self.defaultManifest(suite) + relative_path = None + if not os.path.isabs(test_file): + relative_path = test_file + test_file = self.absManifestPath(test_file) + + if os.path.isdir(test_file): + for dirpath, dirnames, filenames in os.walk(test_file): + if default_manifest in filenames: + rv.append((os.path.join(dirpath, default_manifest), None)) + # We keep recursing into subdirectories which means that in the case + # of include directives we get the same manifest multiple times. + # However reftest.js will only read each manifest once + + if ( + len(rv) == 0 + and relative_path + and suite == "jstestbrowser" + and build_obj + ): + # The relative path can be from staging area. + staged_js_dir = os.path.join( + build_obj.topobjdir, "dist", "test-stage", "jsreftest" + ) + staged_file = os.path.join(staged_js_dir, "tests", relative_path) + return self.findManifest(suite, staged_file, subdirs) + elif test_file.endswith(".list"): + if os.path.exists(test_file): + rv = [(test_file, None)] + else: + dirname, pathname = os.path.split(test_file) + found = True + while not os.path.exists(os.path.join(dirname, default_manifest)): + dirname, suffix = os.path.split(dirname) + pathname = posixpath.join(suffix, pathname) + if os.path.dirname(dirname) == dirname: + found = False + break + if found: + rv = [ + ( + os.path.join(dirname, default_manifest), + r".*%s(?:[#?].*)?$" % pathname.replace("?", "\?"), + ) + ] + + return rv + + def absManifestPath(self, path): + return os.path.normpath(os.path.abspath(path)) + + def manifestURL(self, options, path): + return "file://%s" % path + + def resolveManifests(self, options, tests): + suite = options.suite + manifests = {} + for testPath in tests: + for manifest, filter_str in self.findManifest(suite, testPath): + if manifest not in manifests: + manifests[manifest] = set() + manifests[manifest].add(filter_str) + manifests_by_url = {} + for key in manifests.keys(): + id = os.path.relpath( + os.path.abspath(os.path.dirname(key)), options.topsrcdir + ) + id = id.replace(os.sep, posixpath.sep) + if None in manifests[key]: + manifests[key] = (None, id) + else: + manifests[key] = ("|".join(list(manifests[key])), id) + url = self.manifestURL(options, key) + manifests_by_url[url] = manifests[key] + return manifests_by_url + + +class RefTest(object): + oldcwd = os.getcwd() + resolver_cls = ReftestResolver + use_marionette = True + + def __init__(self, suite): + update_mozinfo() + self.lastTestSeen = None + self.lastTest = None + self.haveDumpedScreen = False + self.resolver = self.resolver_cls() + self.log = None + self.outputHandler = None + self.testDumpFile = os.path.join(tempfile.gettempdir(), "reftests.json") + self.currentManifest = "No test started" + + self.run_by_manifest = True + if suite in ("crashtest", "jstestbrowser"): + self.run_by_manifest = False + + def _populate_logger(self, options): + if self.log: + return + + self.log = getattr(options, "log", None) + if self.log: + return + + mozlog.commandline.log_formatters["tbpl"] = ( + ReftestFormatter, + "Reftest specific formatter for the" + "benefit of legacy log parsers and" + "tools such as the reftest analyzer", + ) + fmt_options = {} + if not options.log_tbpl_level and os.environ.get("MOZ_REFTEST_VERBOSE"): + options.log_tbpl_level = fmt_options["level"] = "debug" + self.log = mozlog.commandline.setup_logging( + "reftest harness", options, {"tbpl": sys.stdout}, fmt_options + ) + + def getFullPath(self, path): + "Get an absolute path relative to self.oldcwd." + return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path))) + + def createReftestProfile( + self, + options, + tests=None, + manifests=None, + server="localhost", + port=0, + profile_to_clone=None, + prefs=None, + ): + """Sets up a profile for reftest. + + :param options: Object containing command line options + :param tests: List of test objects to run + :param manifests: List of manifest files to parse (only takes effect + if tests were not passed in) + :param server: Server name to use for http tests + :param profile_to_clone: Path to a profile to use as the basis for the + test profile + :param prefs: Extra preferences to set in the profile + """ + locations = mozprofile.permissions.ServerLocations() + locations.add_host(server, scheme="http", port=port) + locations.add_host(server, scheme="https", port=port) + + sandbox_whitelist_paths = options.sandboxReadWhitelist + if platform.system() == "Linux" or platform.system() in ( + "Windows", + "Microsoft", + ): + # Trailing slashes are needed to indicate directories on Linux and Windows + sandbox_whitelist_paths = map( + lambda p: os.path.join(p, ""), sandbox_whitelist_paths + ) + + addons = [] + if not self.use_marionette: + addons.append(options.reftestExtensionPath) + + if options.specialPowersExtensionPath is not None: + if not self.use_marionette: + addons.append(options.specialPowersExtensionPath) + + # Install distributed extensions, if application has any. + distExtDir = os.path.join( + options.app[: options.app.rfind(os.sep)], "distribution", "extensions" + ) + if os.path.isdir(distExtDir): + for f in os.listdir(distExtDir): + addons.append(os.path.join(distExtDir, f)) + + # Install custom extensions. + for f in options.extensionsToInstall: + addons.append(self.getFullPath(f)) + + kwargs = { + "addons": addons, + "locations": locations, + "whitelistpaths": sandbox_whitelist_paths, + } + if profile_to_clone: + profile = mozprofile.Profile.clone(profile_to_clone, **kwargs) + else: + profile = mozprofile.Profile(**kwargs) + + # First set prefs from the base profiles under testing/profiles. + + # In test packages used in CI, the profile_data directory is installed + # in the SCRIPT_DIRECTORY. + profile_data_dir = os.path.join(SCRIPT_DIRECTORY, "profile_data") + # If possible, read profile data from topsrcdir. This prevents us from + # requiring a re-build to pick up newly added extensions in the + # <profile>/extensions directory. + if build_obj: + path = os.path.join(build_obj.topsrcdir, "testing", "profiles") + if os.path.isdir(path): + profile_data_dir = path + # Still not found? Look for testing/profiles relative to layout/tools/reftest. + if not os.path.isdir(profile_data_dir): + path = os.path.abspath( + os.path.join(SCRIPT_DIRECTORY, "..", "..", "..", "testing", "profiles") + ) + if os.path.isdir(path): + profile_data_dir = path + + with open(os.path.join(profile_data_dir, "profiles.json"), "r") as fh: + base_profiles = json.load(fh)["reftest"] + + for name in base_profiles: + path = os.path.join(profile_data_dir, name) + profile.merge(path) + + # Second set preferences for communication between our command line + # arguments and the reftest harness. Preferences that are required for + # reftest to work should instead be set under srcdir/testing/profiles. + prefs = prefs or {} + prefs["reftest.timeout"] = options.timeout * 1000 + if options.logFile: + prefs["reftest.logFile"] = options.logFile + if options.ignoreWindowSize: + prefs["reftest.ignoreWindowSize"] = True + if options.shuffle: + prefs["reftest.shuffle"] = True + if options.repeat: + prefs["reftest.repeat"] = options.repeat + if options.runUntilFailure: + prefs["reftest.runUntilFailure"] = True + if not options.repeat: + prefs["reftest.repeat"] = 30 + if options.verify: + prefs["reftest.verify"] = True + if options.cleanupCrashes: + prefs["reftest.cleanupPendingCrashes"] = True + prefs["reftest.focusFilterMode"] = options.focusFilterMode + prefs["reftest.logLevel"] = options.log_tbpl_level or "info" + prefs["reftest.suite"] = options.suite + prefs["gfx.font_rendering.ahem_antialias_none"] = True + # Run the "deferred" font-loader immediately, because if it finishes + # mid-test, the extra reflow that is triggered can disrupt the test. + prefs["gfx.font_loader.delay"] = 0 + # Ensure bundled fonts are activated, even if not enabled by default + # on the platform, so that tests can rely on them. + prefs["gfx.bundled-fonts.activate"] = 1 + # Disable dark scrollbars because it's semi-transparent. + prefs["widget.disable-dark-scrollbar"] = True + prefs["reftest.isCoverageBuild"] = mozinfo.info.get("ccov", False) + + # config specific flags + prefs["sandbox.apple_silicon"] = mozinfo.info.get("apple_silicon", False) + + # Set tests to run or manifests to parse. + if tests: + testlist = os.path.join(profile.profile, "reftests.json") + with open(testlist, "w") as fh: + json.dump(tests, fh) + prefs["reftest.tests"] = testlist + elif manifests: + prefs["reftest.manifests"] = json.dumps(manifests) + + # Unconditionally update the e10s pref, default True + prefs["browser.tabs.remote.autostart"] = True + if not options.e10s: + prefs["browser.tabs.remote.autostart"] = False + + # default fission to True + prefs["fission.autostart"] = True + if options.disableFission: + prefs["fission.autostart"] = False + + if not self.run_by_manifest: + if options.totalChunks: + prefs["reftest.totalChunks"] = options.totalChunks + if options.thisChunk: + prefs["reftest.thisChunk"] = options.thisChunk + + # Bug 1262954: For winXP + e10s disable acceleration + if ( + platform.system() in ("Windows", "Microsoft") + and "5.1" in platform.version() + and options.e10s + ): + prefs["layers.acceleration.disabled"] = True + + # Bug 1300355: Disable canvas cache for win7 as it uses + # too much memory and causes OOMs. + if ( + platform.system() in ("Windows", "Microsoft") + and "6.1" in platform.version() + ): + prefs["reftest.nocache"] = True + + if options.marionette: + # options.marionette can specify host:port + port = options.marionette.split(":")[1] + prefs["marionette.port"] = int(port) + + # Enable tracing output for detailed failures in case of + # failing connection attempts, and hangs (bug 1397201) + prefs["remote.log.level"] = "Trace" + + # Third, set preferences passed in via the command line. + for v in options.extraPrefs: + thispref = v.split("=") + if len(thispref) < 2: + print("Error: syntax error in --setpref=" + v) + sys.exit(1) + prefs[thispref[0]] = thispref[1].strip() + + for pref in prefs: + prefs[pref] = mozprofile.Preferences.cast(prefs[pref]) + profile.set_preferences(prefs) + + if os.path.join(here, "chrome") not in options.extraProfileFiles: + options.extraProfileFiles.append(os.path.join(here, "chrome")) + + self.copyExtraFilesToProfile(options, profile) + + self.log.info( + "Running with e10s: {}".format(prefs["browser.tabs.remote.autostart"]) + ) + self.log.info("Running with fission: {}".format(prefs["fission.autostart"])) + + return profile + + def environment(self, **kwargs): + kwargs["log"] = self.log + return test_environment(**kwargs) + + def buildBrowserEnv(self, options, profileDir): + browserEnv = self.environment( + xrePath=options.xrePath, debugger=options.debugger + ) + browserEnv["XPCOM_DEBUG_BREAK"] = "stack" + if options.topsrcdir: + browserEnv["MOZ_DEVELOPER_REPO_DIR"] = options.topsrcdir + if hasattr(options, "topobjdir"): + browserEnv["MOZ_DEVELOPER_OBJ_DIR"] = options.topobjdir + + if mozinfo.info["asan"]: + # Disable leak checking for reftests for now + if "ASAN_OPTIONS" in browserEnv: + browserEnv["ASAN_OPTIONS"] += ":detect_leaks=0" + else: + browserEnv["ASAN_OPTIONS"] = "detect_leaks=0" + + # Set environment defaults for jstestbrowser. Keep in sync with the + # defaults used in js/src/tests/lib/tests.py. + if options.suite == "jstestbrowser": + browserEnv["TZ"] = "PST8PDT" + browserEnv["LC_ALL"] = "en_US.UTF-8" + + for v in options.environment: + ix = v.find("=") + if ix <= 0: + print("Error: syntax error in --setenv=" + v) + return None + browserEnv[v[:ix]] = v[ix + 1 :] + + # Enable leaks detection to its own log file. + self.leakLogFile = os.path.join(profileDir, "runreftest_leaks.log") + browserEnv["XPCOM_MEM_BLOAT_LOG"] = self.leakLogFile + + # TODO: this is always defined (as part of --enable-webrender which is default) + # can we make this default in the browser? + browserEnv["MOZ_ACCELERATED"] = "1" + + if options.headless: + browserEnv["MOZ_HEADLESS"] = "1" + + return browserEnv + + def cleanup(self, profileDir): + if profileDir: + shutil.rmtree(profileDir, True) + + def verifyTests(self, tests, options): + """ + Support --verify mode: Run test(s) many times in a variety of + configurations/environments in an effort to find intermittent + failures. + """ + + self._populate_logger(options) + + # Number of times to repeat test(s) when running with --repeat + VERIFY_REPEAT = 10 + # Number of times to repeat test(s) when running test in separate browser + VERIFY_REPEAT_SINGLE_BROWSER = 5 + + def step1(): + options.repeat = VERIFY_REPEAT + options.runUntilFailure = True + result = self.runTests(tests, options) + return result + + def step2(): + options.repeat = 0 + options.runUntilFailure = False + for i in range(VERIFY_REPEAT_SINGLE_BROWSER): + result = self.runTests(tests, options) + if result != 0: + break + return result + + def step3(): + options.repeat = VERIFY_REPEAT + options.runUntilFailure = True + options.environment.append("MOZ_CHAOSMODE=0xfb") + result = self.runTests(tests, options) + options.environment.remove("MOZ_CHAOSMODE=0xfb") + return result + + def step4(): + options.repeat = 0 + options.runUntilFailure = False + options.environment.append("MOZ_CHAOSMODE=0xfb") + for i in range(VERIFY_REPEAT_SINGLE_BROWSER): + result = self.runTests(tests, options) + if result != 0: + break + options.environment.remove("MOZ_CHAOSMODE=0xfb") + return result + + steps = [ + ("1. Run each test %d times in one browser." % VERIFY_REPEAT, step1), + ( + "2. Run each test %d times in a new browser each time." + % VERIFY_REPEAT_SINGLE_BROWSER, + step2, + ), + ( + "3. Run each test %d times in one browser, in chaos mode." + % VERIFY_REPEAT, + step3, + ), + ( + "4. Run each test %d times in a new browser each time, in chaos mode." + % VERIFY_REPEAT_SINGLE_BROWSER, + step4, + ), + ] + + stepResults = {} + for (descr, step) in steps: + stepResults[descr] = "not run / incomplete" + + startTime = datetime.now() + maxTime = timedelta(seconds=options.verify_max_time) + finalResult = "PASSED" + for (descr, step) in steps: + if (datetime.now() - startTime) > maxTime: + self.log.info("::: Test verification is taking too long: Giving up!") + self.log.info( + "::: So far, all checks passed, but not all checks were run." + ) + break + self.log.info(":::") + self.log.info('::: Running test verification step "%s"...' % descr) + self.log.info(":::") + result = step() + if result != 0: + stepResults[descr] = "FAIL" + finalResult = "FAILED!" + break + stepResults[descr] = "Pass" + + self.log.info(":::") + self.log.info("::: Test verification summary for:") + self.log.info(":::") + for test in tests: + self.log.info("::: " + test) + self.log.info(":::") + for descr in sorted(stepResults.keys()): + self.log.info("::: %s : %s" % (descr, stepResults[descr])) + self.log.info(":::") + self.log.info("::: Test verification %s" % finalResult) + self.log.info(":::") + + return result + + def runTests(self, tests, options, cmdargs=None): + cmdargs = cmdargs or [] + self._populate_logger(options) + self.outputHandler = OutputHandler( + self.log, options.utilityPath, options.symbolsPath + ) + + if options.cleanupCrashes: + mozcrash.cleanup_pending_crash_reports() + + manifests = self.resolver.resolveManifests(options, tests) + if options.filter: + manifests[""] = (options.filter, None) + + if not getattr(options, "runTestsInParallel", False): + return self.runSerialTests(manifests, options, cmdargs) + + cpuCount = multiprocessing.cpu_count() + + # We have the directive, technology, and machine to run multiple test instances. + # Experimentation says that reftests are not overly CPU-intensive, so we can run + # multiple jobs per CPU core. + # + # Our Windows machines in automation seem to get upset when we run a lot of + # simultaneous tests on them, so tone things down there. + if sys.platform == "win32": + jobsWithoutFocus = cpuCount + else: + jobsWithoutFocus = 2 * cpuCount + + totalJobs = jobsWithoutFocus + 1 + perProcessArgs = [sys.argv[:] for i in range(0, totalJobs)] + + host = "localhost" + port = 2828 + if options.marionette: + host, port = options.marionette.split(":") + + # First job is only needs-focus tests. Remaining jobs are + # non-needs-focus and chunked. + perProcessArgs[0].insert(-1, "--focus-filter-mode=needs-focus") + for (chunkNumber, jobArgs) in enumerate(perProcessArgs[1:], start=1): + jobArgs[-1:-1] = [ + "--focus-filter-mode=non-needs-focus", + "--total-chunks=%d" % jobsWithoutFocus, + "--this-chunk=%d" % chunkNumber, + "--marionette=%s:%d" % (host, port), + ] + port += 1 + + for jobArgs in perProcessArgs: + try: + jobArgs.remove("--run-tests-in-parallel") + except Exception: + pass + jobArgs[0:0] = [sys.executable, "-u"] + + threads = [ReftestThread(args) for args in perProcessArgs[1:]] + for t in threads: + t.start() + + while True: + # The test harness in each individual thread will be doing timeout + # handling on its own, so we shouldn't need to worry about any of + # the threads hanging for arbitrarily long. + for t in threads: + t.join(10) + if not any(t.is_alive() for t in threads): + break + + # Run the needs-focus tests serially after the other ones, so we don't + # have to worry about races between the needs-focus tests *actually* + # needing focus and the dummy windows in the non-needs-focus tests + # trying to focus themselves. + focusThread = ReftestThread(perProcessArgs[0]) + focusThread.start() + focusThread.join() + + # Output the summaries that the ReftestThread filters suppressed. + summaryObjects = [defaultdict(int) for s in summaryLines] + for t in threads: + for (summaryObj, (text, categories)) in zip(summaryObjects, summaryLines): + threadMatches = t.summaryMatches[text] + for (attribute, description) in categories: + amount = int(threadMatches.group(attribute) if threadMatches else 0) + summaryObj[attribute] += amount + amount = int(threadMatches.group("total") if threadMatches else 0) + summaryObj["total"] += amount + + print("REFTEST INFO | Result summary:") + for (summaryObj, (text, categories)) in zip(summaryObjects, summaryLines): + details = ", ".join( + [ + "%d %s" % (summaryObj[attribute], description) + for (attribute, description) in categories + ] + ) + print( + "REFTEST INFO | " + + text + + ": " + + str(summaryObj["total"]) + + " (" + + details + + ")" + ) + + return int(any(t.retcode != 0 for t in threads)) + + def handleTimeout(self, timeout, proc, utilityPath, debuggerInfo): + """handle process output timeout""" + # TODO: bug 913975 : _processOutput should call self.processOutputLine + # one more time one timeout (I think) + self.log.error( + "%s | application timed out after %d seconds with no output" + % (self.lastTestSeen, int(timeout)) + ) + self.log.warning("Force-terminating active process(es).") + self.killAndGetStack( + proc, utilityPath, debuggerInfo, dump_screen=not debuggerInfo + ) + + def dumpScreen(self, utilityPath): + if self.haveDumpedScreen: + self.log.info( + "Not taking screenshot here: see the one that was previously logged" + ) + return + self.haveDumpedScreen = True + dump_screen(utilityPath, self.log) + + def killAndGetStack(self, process, utilityPath, debuggerInfo, dump_screen=False): + """ + Kill the process, preferrably in a way that gets us a stack trace. + Also attempts to obtain a screenshot before killing the process + if specified. + """ + + if dump_screen: + self.dumpScreen(utilityPath) + + if mozinfo.info.get("crashreporter", True) and not debuggerInfo: + if mozinfo.isWin: + # We should have a "crashinject" program in our utility path + crashinject = os.path.normpath( + os.path.join(utilityPath, "crashinject.exe") + ) + if os.path.exists(crashinject): + status = subprocess.Popen([crashinject, str(process.pid)]).wait() + printstatus("crashinject", status) + if status == 0: + return + else: + try: + process.kill(sig=signal.SIGABRT) + except OSError: + # https://bugzilla.mozilla.org/show_bug.cgi?id=921509 + self.log.info("Can't trigger Breakpad, process no longer exists") + return + self.log.info("Can't trigger Breakpad, just killing process") + process.kill() + + def runApp( + self, + options, + cmdargs=None, + timeout=None, + debuggerInfo=None, + symbolsPath=None, + valgrindPath=None, + valgrindArgs=None, + valgrindSuppFiles=None, + **profileArgs + ): + + if cmdargs is None: + cmdargs = [] + cmdargs = cmdargs[:] + + if self.use_marionette: + cmdargs.append("-marionette") + + binary = options.app + profile = self.createReftestProfile(options, **profileArgs) + + # browser environment + env = self.buildBrowserEnv(options, profile.profile) + + def timeoutHandler(): + self.handleTimeout(timeout, proc, options.utilityPath, debuggerInfo) + + interactive = False + debug_args = None + if debuggerInfo: + interactive = debuggerInfo.interactive + debug_args = [debuggerInfo.path] + debuggerInfo.args + + def record_last_test(message): + """Records the last test seen by this harness for the benefit of crash logging.""" + + def testid(test): + if " " in test: + return test.split(" ")[0] + return test + + if message["action"] == "test_start": + self.lastTestSeen = testid(message["test"]) + elif message["action"] == "test_end": + if self.lastTest and message["test"] == self.lastTest: + self.lastTestSeen = self.currentManifest + else: + self.lastTestSeen = "{} (finished)".format(testid(message["test"])) + + self.log.add_handler(record_last_test) + + kp_kwargs = { + "kill_on_timeout": False, + "cwd": SCRIPT_DIRECTORY, + "onTimeout": [timeoutHandler], + "processOutputLine": [self.outputHandler], + } + + if mozinfo.isWin or mozinfo.isMac: + # Prevents log interleaving on Windows at the expense of losing + # true log order. See bug 798300 and bug 1324961 for more details. + kp_kwargs["processStderrLine"] = [self.outputHandler] + + if interactive: + # If an interactive debugger is attached, + # don't use timeouts, and don't capture ctrl-c. + timeout = None + signal.signal(signal.SIGINT, lambda sigid, frame: None) + + runner_cls = mozrunner.runners.get( + mozinfo.info.get("appname", "firefox"), mozrunner.Runner + ) + runner = runner_cls( + profile=profile, + binary=binary, + process_class=mozprocess.ProcessHandlerMixin, + cmdargs=cmdargs, + env=env, + process_args=kp_kwargs, + ) + runner.start( + debug_args=debug_args, interactive=interactive, outputTimeout=timeout + ) + proc = runner.process_handler + self.outputHandler.proc_name = "GECKO({})".format(proc.pid) + + # Used to defer a possible IOError exception from Marionette + marionette_exception = None + + if self.use_marionette: + marionette_args = { + "socket_timeout": options.marionette_socket_timeout, + "startup_timeout": options.marionette_startup_timeout, + "symbols_path": options.symbolsPath, + } + if options.marionette: + host, port = options.marionette.split(":") + marionette_args["host"] = host + marionette_args["port"] = int(port) + + try: + marionette = Marionette(**marionette_args) + marionette.start_session() + + addons = Addons(marionette) + if options.specialPowersExtensionPath: + addons.install(options.specialPowersExtensionPath, temp=True) + + addons.install(options.reftestExtensionPath, temp=True) + + marionette.delete_session() + except IOError: + # Any IOError as thrown by Marionette means that something is + # wrong with the process, like a crash or the socket is no + # longer open. We defer raising this specific error so that + # post-test checks for leaks and crashes are performed and + # reported first. + marionette_exception = sys.exc_info() + + status = runner.wait() + runner.process_handler = None + self.outputHandler.proc_name = None + + crashed = mozcrash.log_crashes( + self.log, + os.path.join(profile.profile, "minidumps"), + options.symbolsPath, + test=self.lastTestSeen, + ) + + if crashed: + # log suite_end to wrap up, this is usually done with in in-browser harness + if not self.outputHandler.results: + # TODO: while .results is a defaultdict(int), it is proxied via log_actions as data, not type + self.outputHandler.results = { + "Pass": 0, + "LoadOnly": 0, + "Exception": 0, + "FailedLoad": 0, + "UnexpectedFail": 1, + "UnexpectedPass": 0, + "AssertionUnexpected": 0, + "AssertionUnexpectedFixed": 0, + "KnownFail": 0, + "AssertionKnown": 0, + "Random": 0, + "Skip": 0, + "Slow": 0, + } + self.log.suite_end(extra={"results": self.outputHandler.results}) + + if not status and crashed: + status = 1 + + if status and not crashed: + msg = ( + "TEST-UNEXPECTED-FAIL | %s | application terminated with exit code %s" + % (self.lastTestSeen, status) + ) + # use process_output so message is logged verbatim + self.log.process_output(None, msg) + + runner.cleanup() + self.cleanup(profile.profile) + + if marionette_exception is not None: + exc, value, tb = marionette_exception + raise reraise(exc, value, tb) + + self.log.info("Process mode: {}".format("e10s" if options.e10s else "non-e10s")) + return status + + def getActiveTests(self, manifests, options, testDumpFile=None): + # These prefs will cause reftest.jsm to parse the manifests, + # dump the resulting tests to a file, and exit. + prefs = { + "reftest.manifests": json.dumps(manifests), + "reftest.manifests.dumpTests": testDumpFile or self.testDumpFile, + } + cmdargs = [] + self.runApp(options, cmdargs=cmdargs, prefs=prefs) + + if not os.path.isfile(self.testDumpFile): + print("Error: parsing manifests failed!") + sys.exit(1) + + with open(self.testDumpFile, "r") as fh: + tests = json.load(fh) + + if os.path.isfile(self.testDumpFile): + mozfile.remove(self.testDumpFile) + + for test in tests: + # Name and path are expected by manifestparser, but not used in reftest. + test["name"] = test["path"] = test["url1"] + + mp = TestManifest(strict=False) + mp.tests = tests + + filters = [] + if options.totalChunks: + filters.append( + mpf.chunk_by_manifest(options.thisChunk, options.totalChunks) + ) + + tests = mp.active_tests(exists=False, filters=filters) + return tests + + def runSerialTests(self, manifests, options, cmdargs=None): + debuggerInfo = None + if options.debugger: + debuggerInfo = mozdebug.get_debugger_info( + options.debugger, options.debuggerArgs, options.debuggerInteractive + ) + + def run(**kwargs): + if kwargs.get("tests"): + self.lastTest = kwargs["tests"][-1]["identifier"] + if not isinstance(self.lastTest, string_types): + self.lastTest = " ".join(self.lastTest) + + status = self.runApp( + options, + manifests=manifests, + cmdargs=cmdargs, + # We generally want the JS harness or marionette + # to handle timeouts if they can. + # The default JS harness timeout is currently + # 300 seconds (default options.timeout). + # The default Marionette socket timeout is + # currently 360 seconds. + # Give the JS harness extra time to deal with + # its own timeouts and try to usually exceed + # the 360 second marionette socket timeout. + # See bug 479518 and bug 1414063. + timeout=options.timeout + 70.0, + debuggerInfo=debuggerInfo, + symbolsPath=options.symbolsPath, + **kwargs + ) + + # do not process leak log when we crash/assert + if status == 0: + mozleak.process_leak_log( + self.leakLogFile, + leak_thresholds=options.leakThresholds, + stack_fixer=get_stack_fixer_function( + options.utilityPath, options.symbolsPath + ), + ) + return status + + if not self.run_by_manifest: + return run() + + tests = self.getActiveTests(manifests, options) + tests_by_manifest = defaultdict(list) + ids_by_manifest = defaultdict(list) + for t in tests: + tests_by_manifest[t["manifest"]].append(t) + test_id = t["identifier"] + if not isinstance(test_id, string_types): + test_id = " ".join(test_id) + ids_by_manifest[t["manifestID"]].append(test_id) + + self.log.suite_start(ids_by_manifest, name=options.suite) + + overall = 0 + status = -1 + for manifest, tests in tests_by_manifest.items(): + self.log.info("Running tests in {}".format(manifest)) + self.currentManifest = manifest + status = run(tests=tests) + overall = overall or status + if status == -1: + # we didn't run anything + overall = 1 + + self.log.suite_end(extra={"results": self.outputHandler.results}) + return overall + + def copyExtraFilesToProfile(self, options, profile): + "Copy extra files or dirs specified on the command line to the testing profile." + profileDir = profile.profile + for f in options.extraProfileFiles: + abspath = self.getFullPath(f) + if os.path.isfile(abspath): + if os.path.basename(abspath) == "user.js": + extra_prefs = mozprofile.Preferences.read_prefs(abspath) + profile.set_preferences(extra_prefs) + elif os.path.basename(abspath).endswith(".dic"): + hyphDir = os.path.join(profileDir, "hyphenation") + if not os.path.exists(hyphDir): + os.makedirs(hyphDir) + shutil.copy2(abspath, hyphDir) + else: + shutil.copy2(abspath, profileDir) + elif os.path.isdir(abspath): + dest = os.path.join(profileDir, os.path.basename(abspath)) + shutil.copytree(abspath, dest) + else: + self.log.warning( + "runreftest.py | Failed to copy %s to profile" % abspath + ) + continue + + +def run_test_harness(parser, options): + reftest = RefTest(options.suite) + parser.validate(options, reftest) + + # We have to validate options.app here for the case when the mach + # command is able to find it after argument parsing. This can happen + # when running from a tests archive. + if not options.app: + parser.error("could not find the application path, --appname must be specified") + + options.app = reftest.getFullPath(options.app) + if not os.path.exists(options.app): + parser.error( + "Error: Path %(app)s doesn't exist. Are you executing " + "$objdir/_tests/reftest/runreftest.py?" % {"app": options.app} + ) + + if options.xrePath is None: + options.xrePath = os.path.dirname(options.app) + + if options.verify: + result = reftest.verifyTests(options.tests, options) + else: + result = reftest.runTests(options.tests, options) + + return result + + +if __name__ == "__main__": + parser = reftestcommandline.DesktopArgumentsParser() + options = parser.parse_args() + sys.exit(run_test_harness(parser, options)) |