summaryrefslogtreecommitdiffstats
path: root/testing/webcompat/runner.py
diff options
context:
space:
mode:
Diffstat (limited to 'testing/webcompat/runner.py')
-rw-r--r--testing/webcompat/runner.py263
1 files changed, 263 insertions, 0 deletions
diff --git a/testing/webcompat/runner.py b/testing/webcompat/runner.py
new file mode 100644
index 0000000000..fefef7bf70
--- /dev/null
+++ b/testing/webcompat/runner.py
@@ -0,0 +1,263 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import errno
+import os
+import shutil
+import tempfile
+
+import pytest
+
+GVE = "org.mozilla.geckoview_example"
+
+
+def run(
+ logger,
+ path,
+ webdriver_binary,
+ webdriver_port,
+ webdriver_ws_port,
+ browser_binary=None,
+ device_serial=None,
+ package_name=None,
+ environ=None,
+ bug=None,
+ debug=False,
+ interventions=None,
+ shims=None,
+ config=None,
+ headless=False,
+ addon=None,
+ do2fa=False,
+):
+ """"""
+ old_environ = os.environ.copy()
+ try:
+ with TemporaryDirectory() as cache:
+ if environ:
+ os.environ.update(environ)
+
+ config_plugin = WDConfig()
+ result_recorder = ResultRecorder(logger)
+
+ args = [
+ "--strict", # turn warnings into errors
+ "-vv", # show each individual subtest and full failure logs
+ "--capture",
+ "no", # enable stdout/stderr from tests
+ "--basetemp",
+ cache, # temporary directory
+ "--showlocals", # display contents of variables in local scope
+ "-p",
+ "no:mozlog", # use the WPT result recorder
+ "--disable-warnings",
+ "-rfEs",
+ "-p",
+ "no:cacheprovider", # disable state preservation across invocations
+ "-o=console_output_style=classic", # disable test progress bar
+ "--browser",
+ "firefox",
+ "--webdriver-binary",
+ webdriver_binary,
+ "--webdriver-port",
+ webdriver_port,
+ "--webdriver-ws-port",
+ webdriver_ws_port,
+ ]
+
+ if debug:
+ args.append("--pdb")
+
+ if headless:
+ args.append("--headless")
+
+ if browser_binary:
+ args.append("--browser-binary")
+ args.append(browser_binary)
+
+ if device_serial:
+ args.append("--device-serial")
+ args.append(device_serial)
+
+ if package_name:
+ args.append("--package-name")
+ args.append(package_name)
+
+ if addon:
+ args.append("--addon")
+ args.append(addon)
+
+ if bug:
+ args.append("--bug")
+ args.append(bug)
+
+ if do2fa:
+ args.append("--do2fa")
+
+ if config:
+ args.append("--config")
+ args.append(config)
+
+ if interventions is not None and shims is not None:
+ raise ValueError(
+ "Must provide only one of interventions or shims argument"
+ )
+ elif interventions is None and shims is None:
+ raise ValueError(
+ "Must provide either an interventions or shims argument"
+ )
+
+ name = "webcompat-interventions"
+ if interventions == "enabled":
+ args.extend(["-m", "with_interventions"])
+ elif interventions == "disabled":
+ args.extend(["-m", "without_interventions"])
+ elif interventions is not None:
+ raise ValueError(f"Invalid value for interventions {interventions}")
+ if shims == "enabled":
+ args.extend(["-m", "with_shims"])
+ name = "smartblock-shims"
+ elif shims == "disabled":
+ args.extend(["-m", "without_shims"])
+ name = "smartblock-shims"
+ elif shims is not None:
+ raise ValueError(f"Invalid value for shims {shims}")
+ else:
+ name = "smartblock-shims"
+
+ if bug is not None:
+ args.extend(["-k", bug])
+
+ args.append(path)
+ try:
+ logger.suite_start([], name=name)
+ pytest.main(args, plugins=[config_plugin, result_recorder])
+ except Exception as e:
+ logger.critical(str(e))
+ finally:
+ logger.suite_end()
+
+ finally:
+ os.environ = old_environ
+
+
+class WDConfig:
+ def pytest_addoption(self, parser):
+ parser.addoption(
+ "--browser-binary", action="store", help="Path to browser binary"
+ )
+ parser.addoption(
+ "--webdriver-binary", action="store", help="Path to webdriver binary"
+ )
+ parser.addoption(
+ "--webdriver-port",
+ action="store",
+ default="4444",
+ help="Port on which to run WebDriver",
+ )
+ parser.addoption(
+ "--webdriver-ws-port",
+ action="store",
+ default="9222",
+ help="Port on which to run WebDriver BiDi websocket",
+ )
+ parser.addoption(
+ "--browser", action="store", choices=["firefox"], help="Name of the browser"
+ )
+ parser.addoption("--bug", action="store", help="Bug number to run tests for")
+ parser.addoption(
+ "--do2fa",
+ action="store_true",
+ default=False,
+ help="Do two-factor auth live in supporting tests",
+ )
+ parser.addoption(
+ "--config",
+ action="store",
+ help="Path to JSON file containing logins and other settings",
+ )
+ parser.addoption(
+ "--addon",
+ action="store",
+ help="Path to the webcompat addon XPI to use",
+ )
+ parser.addoption(
+ "--device-serial",
+ action="store",
+ help="Emulator device serial number",
+ )
+ parser.addoption(
+ "--package-name",
+ action="store",
+ default=GVE,
+ help="Android package to run/connect to",
+ )
+ parser.addoption(
+ "--headless", action="store_true", help="Run browser in headless mode"
+ )
+
+
+class ResultRecorder(object):
+ def __init__(self, logger):
+ self.logger = logger
+
+ def pytest_runtest_logreport(self, report):
+ if report.passed and report.when == "call":
+ self.record_pass(report)
+ elif report.failed:
+ if report.when != "call":
+ self.record_error(report)
+ else:
+ self.record_fail(report)
+ elif report.skipped:
+ self.record_skip(report)
+
+ def record_pass(self, report):
+ self.record(report.nodeid, "PASS")
+
+ def record_fail(self, report):
+ # pytest outputs the stacktrace followed by an error message prefixed
+ # with "E ", e.g.
+ #
+ # def test_example():
+ # > assert "fuu" in "foobar"
+ # > E AssertionError: assert 'fuu' in 'foobar'
+ message = ""
+ for line in report.longreprtext.splitlines():
+ if line.startswith("E "):
+ message = line[1:].strip()
+ break
+
+ self.record(report.nodeid, "FAIL", message=message, stack=report.longrepr)
+
+ def record_error(self, report):
+ # error in setup/teardown
+ if report.when != "call":
+ message = "%s error" % report.when
+ self.record(report.nodeid, "ERROR", message, report.longrepr)
+
+ def record_skip(self, report):
+ self.record(report.nodeid, "SKIP")
+
+ def record(self, test, status, message=None, stack=None):
+ if stack is not None:
+ stack = str(stack)
+ self.logger.test_start(test)
+ self.logger.test_end(
+ test=test, status=status, expected="PASS", message=message, stack=stack
+ )
+
+
+class TemporaryDirectory(object):
+ def __enter__(self):
+ self.path = tempfile.mkdtemp(prefix="pytest-")
+ return self.path
+
+ def __exit__(self, *args):
+ try:
+ shutil.rmtree(self.path)
+ except OSError as e:
+ # no such file or directory
+ if e.errno != errno.ENOENT:
+ raise