diff options
Diffstat (limited to 'layout/tools/reftest/reftestcommandline.py')
| -rw-r--r-- | layout/tools/reftest/reftestcommandline.py | 645 |
1 files changed, 645 insertions, 0 deletions
diff --git a/layout/tools/reftest/reftestcommandline.py b/layout/tools/reftest/reftestcommandline.py new file mode 100644 index 0000000000..e2f0baff8c --- /dev/null +++ b/layout/tools/reftest/reftestcommandline.py @@ -0,0 +1,645 @@ +import argparse +import os +import sys +from collections import OrderedDict + +import mozinfo +import mozlog +from six.moves.urllib.parse import urlparse + +here = os.path.abspath(os.path.dirname(__file__)) + + +class ReftestArgumentsParser(argparse.ArgumentParser): + def __init__(self, **kwargs): + super(ReftestArgumentsParser, self).__init__(**kwargs) + + # Try to import a MozbuildObject. Success indicates that we are + # running from a source tree. This allows some defaults to be set + # from the source tree. + try: + from mozbuild.base import MozbuildObject + + self.build_obj = MozbuildObject.from_environment(cwd=here) + except ImportError: + self.build_obj = None + + self.add_argument( + "--xre-path", + action="store", + type=str, + dest="xrePath", + # individual scripts will set a sane default + default=None, + help="absolute path to directory containing XRE (probably xulrunner)", + ) + + self.add_argument( + "--symbols-path", + action="store", + type=str, + dest="symbolsPath", + default=None, + help="absolute path to directory containing breakpad symbols, " + "or the URL of a zip file containing symbols", + ) + + self.add_argument( + "--debugger", + action="store", + dest="debugger", + help="use the given debugger to launch the application", + ) + + self.add_argument( + "--debugger-args", + action="store", + dest="debuggerArgs", + help="pass the given args to the debugger _before_ " + "the application on the command line", + ) + + self.add_argument( + "--debugger-interactive", + action="store_true", + dest="debuggerInteractive", + help="prevents the test harness from redirecting " + "stdout and stderr for interactive debuggers", + ) + + self.add_argument( + "--appname", + action="store", + type=str, + dest="app", + default=None, + help="absolute path to application, overriding default", + ) + + self.add_argument( + "--extra-profile-file", + action="append", + dest="extraProfileFiles", + default=[], + help="copy specified files/dirs to testing profile", + ) + + self.add_argument( + "--timeout", + action="store", + dest="timeout", + type=int, + default=300, # 5 minutes per bug 479518 + help="reftest will timeout in specified number of seconds. " + "[default %(default)s].", + ) + + self.add_argument( + "--leak-threshold", + action="store", + type=int, + dest="defaultLeakThreshold", + default=0, + help="fail if the number of bytes leaked in default " + "processes through refcounted objects (or bytes " + "in classes with MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) " + "is greater than the given number", + ) + + self.add_argument( + "--utility-path", + action="store", + type=str, + dest="utilityPath", + default=self.build_obj.bindir if self.build_obj else None, + help="absolute path to directory containing utility " + "programs (xpcshell, ssltunnel, certutil)", + ) + + self.add_argument( + "--total-chunks", + type=int, + dest="totalChunks", + help="how many chunks to split the tests up into", + ) + + self.add_argument( + "--this-chunk", + type=int, + dest="thisChunk", + help="which chunk to run between 1 and --total-chunks", + ) + + self.add_argument( + "--log-file", + action="store", + type=str, + dest="logFile", + default=None, + help="file to log output to in addition to stdout", + ) + + self.add_argument( + "--skip-slow-tests", + dest="skipSlowTests", + action="store_true", + default=False, + help="skip tests marked as slow when running", + ) + + self.add_argument( + "--ignore-window-size", + dest="ignoreWindowSize", + action="store_true", + default=False, + help="ignore the window size, which may cause spurious " + "failures and passes", + ) + + self.add_argument( + "--install-extension", + action="append", + dest="extensionsToInstall", + default=[], + help="install the specified extension in the testing profile. " + "The extension file's name should be <id>.xpi where <id> is " + "the extension's id as indicated in its install.rdf. " + "An optional path can be specified too.", + ) + + self.add_argument( + "--marionette", + default=None, + help="host:port to use when connecting to Marionette", + ) + + self.add_argument( + "--marionette-socket-timeout", default=None, help=argparse.SUPPRESS + ) + + self.add_argument( + "--marionette-startup-timeout", default=None, help=argparse.SUPPRESS + ) + + self.add_argument( + "--setenv", + action="append", + type=str, + default=[], + dest="environment", + metavar="NAME=VALUE", + help="sets the given variable in the application's " "environment", + ) + + self.add_argument( + "--filter", + action="store", + type=str, + dest="filter", + help="specifies a regular expression (as could be passed to the JS " + "RegExp constructor) to test against URLs in the reftest manifest; " + "only test items that have a matching test URL will be run.", + ) + + self.add_argument( + "--shuffle", + action="store_true", + default=False, + dest="shuffle", + help="run reftests in random order", + ) + + self.add_argument( + "--run-until-failure", + action="store_true", + default=False, + dest="runUntilFailure", + help="stop running on the first failure. Useful for RR recordings.", + ) + + self.add_argument( + "--repeat", + action="store", + type=int, + default=0, + dest="repeat", + help="number of times the select test(s) will be executed. Useful for " + "finding intermittent failures.", + ) + + self.add_argument( + "--focus-filter-mode", + action="store", + type=str, + dest="focusFilterMode", + default="all", + help="filters tests to run by whether they require focus. " + "Valid values are `all', `needs-focus', or `non-needs-focus'. " + "Defaults to `all'.", + ) + + self.add_argument( + "--disable-e10s", + action="store_false", + default=True, + dest="e10s", + help="disables content processes", + ) + + self.add_argument( + "--disable-fission", + action="store_true", + default=False, + dest="disableFission", + help="Run tests with fission (site isolation) disabled.", + ) + + self.add_argument( + "--setpref", + action="append", + type=str, + default=[], + dest="extraPrefs", + metavar="PREF=VALUE", + help="defines an extra user preference", + ) + + self.add_argument( + "--reftest-extension-path", + action="store", + dest="reftestExtensionPath", + help="Path to the reftest extension", + ) + + self.add_argument( + "--special-powers-extension-path", + action="store", + dest="specialPowersExtensionPath", + help="Path to the special powers extension", + ) + + self.add_argument( + "--suite", + choices=["reftest", "crashtest", "jstestbrowser"], + default=None, + help=argparse.SUPPRESS, + ) + + self.add_argument( + "--cleanup-crashes", + action="store_true", + dest="cleanupCrashes", + default=False, + help="Delete pending crash reports before running tests.", + ) + + self.add_argument( + "--max-retries", + type=int, + dest="maxRetries", + default=4, + help="The maximum number of attempts to try and recover from a " + "crash before aborting the test run [default 4].", + ) + + self.add_argument( + "tests", + metavar="TEST_PATH", + nargs="*", + help="Path to test file, manifest file, or directory containing " + "tests. For jstestbrowser, the relative path can be either from " + "topsrcdir or the staged area " + "($OBJDIR/dist/test-stage/jsreftest/tests)", + ) + + self.add_argument( + "--sandbox-read-whitelist", + action="append", + dest="sandboxReadWhitelist", + default=[], + help="Path to add to the sandbox whitelist.", + ) + + self.add_argument( + "--verify", + action="store_true", + default=False, + help="Run tests in verification mode: Run many times in different " + "ways, to see if there are intermittent failures.", + ) + + self.add_argument( + "--verify-max-time", + type=int, + default=3600, + help="Maximum time, in seconds, to run in --verify mode..", + ) + + self.add_argument( + "--enable-webrender", + action="store_true", + dest="enable_webrender", + default=False, + help="Enable the WebRender compositor in Gecko.", + ) + + self.add_argument( + "--headless", + action="store_true", + dest="headless", + default=False, + help="Run tests in headless mode.", + ) + + self.add_argument( + "--topsrcdir", + action="store", + type=str, + dest="topsrcdir", + default=None, + help="Path to source directory", + ) + + mozlog.commandline.add_logging_group(self) + + def get_ip(self): + import moznetwork + + if os.name != "nt": + return moznetwork.get_ip() + else: + self.error("ERROR: you must specify a --remote-webserver=<ip address>\n") + + def set_default_suite(self, options): + manifests = OrderedDict( + [ + ("reftest.list", "reftest"), + ("crashtests.list", "crashtest"), + ("jstests.list", "jstestbrowser"), + ] + ) + + for test_path in options.tests: + file_name = os.path.basename(test_path) + if file_name in manifests: + options.suite = manifests[file_name] + return + + for test_path in options.tests: + for manifest_file, suite in manifests.iteritems(): + if os.path.exists(os.path.join(test_path, manifest_file)): + options.suite = suite + return + + self.error( + "Failed to determine test suite; supply --suite to set this explicitly" + ) + + def validate(self, options, reftest): + if not options.tests: + # Can't just set this in the argument parser because mach will set a default + self.error( + "Must supply at least one path to a manifest file, " + "test directory, or test file to run." + ) + + if options.suite is None: + self.set_default_suite(options) + + if options.totalChunks is not None and options.thisChunk is None: + self.error("thisChunk must be specified when totalChunks is specified") + + if options.totalChunks: + if not 1 <= options.thisChunk <= options.totalChunks: + self.error("thisChunk must be between 1 and totalChunks") + + if not options.disableFission and not options.e10s: + self.error("Fission is not supported without e10s.") + + if options.logFile: + options.logFile = reftest.getFullPath(options.logFile) + + if options.xrePath is not None: + if not os.access(options.xrePath, os.F_OK): + self.error("--xre-path '%s' not found" % options.xrePath) + if not os.path.isdir(options.xrePath): + self.error("--xre-path '%s' is not a directory" % options.xrePath) + options.xrePath = reftest.getFullPath(options.xrePath) + + if options.reftestExtensionPath is None: + if self.build_obj is not None: + reftestExtensionPath = os.path.join( + self.build_obj.distdir, "xpi-stage", "reftest" + ) + else: + reftestExtensionPath = os.path.join(here, "reftest") + options.reftestExtensionPath = os.path.normpath(reftestExtensionPath) + + if options.specialPowersExtensionPath is None: + if self.build_obj is not None: + specialPowersExtensionPath = os.path.join( + self.build_obj.distdir, "xpi-stage", "specialpowers" + ) + else: + specialPowersExtensionPath = os.path.join(here, "specialpowers") + options.specialPowersExtensionPath = os.path.normpath( + specialPowersExtensionPath + ) + + options.leakThresholds = { + "default": options.defaultLeakThreshold, + "tab": options.defaultLeakThreshold, + } + + if mozinfo.isWin: + if mozinfo.info["bits"] == 32: + # See bug 1408554. + options.leakThresholds["tab"] = 3000 + else: + # See bug 1404482. + options.leakThresholds["tab"] = 100 + + if options.topsrcdir is None: + if self.build_obj: + options.topsrcdir = self.build_obj.topsrcdir + else: + options.topsrcdir = os.getcwd() + + +class DesktopArgumentsParser(ReftestArgumentsParser): + def __init__(self, **kwargs): + super(DesktopArgumentsParser, self).__init__(**kwargs) + + self.add_argument( + "--run-tests-in-parallel", + action="store_true", + default=False, + dest="runTestsInParallel", + help="run tests in parallel if possible", + ) + + def _prefs_gpu(self): + if mozinfo.os != "win": + return ["layers.acceleration.force-enabled=true"] + return [] + + def validate(self, options, reftest): + super(DesktopArgumentsParser, self).validate(options, reftest) + + if options.runTestsInParallel: + if options.logFile is not None: + self.error("cannot specify logfile with parallel tests") + if options.totalChunks is not None or options.thisChunk is not None: + self.error( + "cannot specify thisChunk or totalChunks with parallel tests" + ) + if options.focusFilterMode != "all": + self.error("cannot specify focusFilterMode with parallel tests") + if options.debugger is not None: + self.error("cannot specify a debugger with parallel tests") + + if options.debugger: + # valgrind and some debuggers may cause Gecko to start slowly. Make sure + # marionette waits long enough to connect. + options.marionette_startup_timeout = 900 + options.marionette_socket_timeout = 540 + + if not options.tests: + self.error("No test files specified.") + + if options.app is None: + if ( + self.build_obj + and self.build_obj.substs["MOZ_BUILD_APP"] != "mobile/android" + ): + from mozbuild.base import BinaryNotFoundException + + try: + bin_dir = self.build_obj.get_binary_path() + except BinaryNotFoundException as e: + print("{}\n\n{}\n".format(e, e.help()), file=sys.stderr) + sys.exit(1) + else: + bin_dir = None + + if bin_dir: + options.app = bin_dir + + if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2: + options.symbolsPath = reftest.getFullPath(options.symbolsPath) + + options.utilityPath = reftest.getFullPath(options.utilityPath) + + +class RemoteArgumentsParser(ReftestArgumentsParser): + def __init__(self, **kwargs): + super(RemoteArgumentsParser, self).__init__() + + # app, xrePath and utilityPath variables are set in main function + self.set_defaults( + logFile="reftest.log", app="", xrePath="", utilityPath="", localLogName=None + ) + + self.add_argument( + "--adbpath", + action="store", + type=str, + dest="adb_path", + default=None, + help="Path to adb binary.", + ) + + self.add_argument( + "--deviceSerial", + action="store", + type=str, + dest="deviceSerial", + help="adb serial number of remote device. This is required " + "when more than one device is connected to the host. " + "Use 'adb devices' to see connected devices.", + ) + + self.add_argument( + "--remote-webserver", + action="store", + type=str, + dest="remoteWebServer", + help="IP address of the remote web server.", + ) + + self.add_argument( + "--http-port", + action="store", + type=str, + dest="httpPort", + help="http port of the remote web server.", + ) + + self.add_argument( + "--ssl-port", + action="store", + type=str, + dest="sslPort", + help="ssl port of the remote web server.", + ) + + self.add_argument( + "--remoteTestRoot", + action="store", + type=str, + dest="remoteTestRoot", + help="Remote directory to use as test root " + "(eg. /data/local/tmp/test_root).", + ) + + self.add_argument( + "--httpd-path", + action="store", + type=str, + dest="httpdPath", + help="Path to the httpd.js file.", + ) + + self.add_argument( + "--no-install", + action="store_true", + default=False, + help="Skip the installation of the APK.", + ) + + def validate_remote(self, options): + DEFAULT_HTTP_PORT = 8888 + DEFAULT_SSL_PORT = 4443 + + if options.remoteWebServer is None: + options.remoteWebServer = self.get_ip() + + if options.remoteWebServer == "127.0.0.1": + self.error( + "ERROR: Either you specified the loopback for the remote webserver or ", + "your local IP cannot be detected. " + "Please provide the local ip in --remote-webserver", + ) + + if not options.httpPort: + options.httpPort = DEFAULT_HTTP_PORT + + if not options.sslPort: + options.sslPort = DEFAULT_SSL_PORT + + if options.xrePath is None: + self.error( + "ERROR: You must specify the path to the controller xre directory" + ) + else: + # Ensure xrepath is a full path + options.xrePath = os.path.abspath(options.xrePath) + + # httpd-path is specified by standard makefile targets and may be specified + # on the command line to select a particular version of httpd.js. If not + # specified, try to select the one from hostutils.zip, as required in + # bug 882932. + if not options.httpdPath: + options.httpdPath = os.path.join(options.utilityPath, "components") + + return options |