diff options
Diffstat (limited to 'testing/web-platform/tests/tools')
37 files changed, 381 insertions, 645 deletions
diff --git a/testing/web-platform/tests/tools/ci/requirements_tc.txt b/testing/web-platform/tests/tools/ci/requirements_tc.txt index 8151646605..e1ae4dbf70 100644 --- a/testing/web-platform/tests/tools/ci/requirements_tc.txt +++ b/testing/web-platform/tests/tools/ci/requirements_tc.txt @@ -1,4 +1,4 @@ pygithub==2.2.0 pyyaml==6.0.1 requests==2.31.0 -taskcluster==60.3.2 +taskcluster==60.4.1 diff --git a/testing/web-platform/tests/tools/ci/tc/tasks/test.yml b/testing/web-platform/tests/tools/ci/tc/tasks/test.yml index ea9c7f9dae..c172e6b731 100644 --- a/testing/web-platform/tests/tools/ci/tc/tasks/test.yml +++ b/testing/web-platform/tests/tools/ci/tc/tasks/test.yml @@ -4,7 +4,7 @@ components: workerType: ci schedulerId: taskcluster-github deadline: "24 hours" - image: webplatformtests/wpt:0.56 + image: webplatformtests/wpt:0.57 maxRunTime: 7200 artifacts: public/results: @@ -232,7 +232,7 @@ tasks: browser: servo channel: nightly use: - - trigger-weekly + - trigger-daily - trigger-push - vars: browser: firefox_android diff --git a/testing/web-platform/tests/tools/ci/tc/tests/test_valid.py b/testing/web-platform/tests/tools/ci/tc/tests/test_valid.py index 6960a2cc47..62bb09a1c3 100644 --- a/testing/web-platform/tests/tools/ci/tc/tests/test_valid.py +++ b/testing/web-platform/tests/tools/ci/tc/tests/test_valid.py @@ -295,6 +295,22 @@ def test_verify_payload(): 'wpt-webkitgtk_minibrowser-nightly-testharness-14', 'wpt-webkitgtk_minibrowser-nightly-testharness-15', 'wpt-webkitgtk_minibrowser-nightly-testharness-16', + 'wpt-servo-nightly-testharness-1', + 'wpt-servo-nightly-testharness-2', + 'wpt-servo-nightly-testharness-3', + 'wpt-servo-nightly-testharness-4', + 'wpt-servo-nightly-testharness-5', + 'wpt-servo-nightly-testharness-6', + 'wpt-servo-nightly-testharness-7', + 'wpt-servo-nightly-testharness-8', + 'wpt-servo-nightly-testharness-9', + 'wpt-servo-nightly-testharness-10', + 'wpt-servo-nightly-testharness-11', + 'wpt-servo-nightly-testharness-12', + 'wpt-servo-nightly-testharness-13', + 'wpt-servo-nightly-testharness-14', + 'wpt-servo-nightly-testharness-15', + 'wpt-servo-nightly-testharness-16', 'wpt-firefox_android-nightly-testharness-1', 'wpt-firefox_android-nightly-testharness-2', 'wpt-firefox_android-nightly-testharness-3', @@ -343,6 +359,12 @@ def test_verify_payload(): 'wpt-webkitgtk_minibrowser-nightly-reftest-4', 'wpt-webkitgtk_minibrowser-nightly-reftest-5', 'wpt-webkitgtk_minibrowser-nightly-reftest-6', + 'wpt-servo-nightly-reftest-1', + 'wpt-servo-nightly-reftest-2', + 'wpt-servo-nightly-reftest-3', + 'wpt-servo-nightly-reftest-4', + 'wpt-servo-nightly-reftest-5', + 'wpt-servo-nightly-reftest-6', 'wpt-firefox_android-nightly-reftest-1', 'wpt-firefox_android-nightly-reftest-2', 'wpt-firefox_android-nightly-reftest-3', @@ -357,12 +379,15 @@ def test_verify_payload(): 'wpt-chrome-stable-wdspec-2', 'wpt-webkitgtk_minibrowser-nightly-wdspec-1', 'wpt-webkitgtk_minibrowser-nightly-wdspec-2', + 'wpt-servo-nightly-wdspec-1', + 'wpt-servo-nightly-wdspec-2', 'wpt-firefox_android-nightly-wdspec-1', 'wpt-firefox_android-nightly-wdspec-2', 'wpt-firefox-stable-crashtest-1', 'wpt-chromium-nightly-crashtest-1', 'wpt-chrome-stable-crashtest-1', 'wpt-webkitgtk_minibrowser-nightly-crashtest-1', + 'wpt-servo-nightly-crashtest-1', 'wpt-firefox_android-nightly-crashtest-1', 'wpt-firefox-stable-print-reftest-1', 'wpt-chromium-nightly-print-reftest-1', diff --git a/testing/web-platform/tests/tools/manifest/sourcefile.py b/testing/web-platform/tests/tools/manifest/sourcefile.py index 23aa7f491f..71eab54bea 100644 --- a/testing/web-platform/tests/tools/manifest/sourcefile.py +++ b/testing/web-platform/tests/tools/manifest/sourcefile.py @@ -70,6 +70,7 @@ def read_script_metadata(f: BinaryIO, regexp: Pattern[bytes]) -> Iterable[Tuple[ _any_variants: Dict[Text, Dict[Text, Any]] = { "window": {"suffix": ".any.html"}, + "window-module": {}, "serviceworker": {"force_https": True}, "serviceworker-module": {"force_https": True}, "sharedworker": {}, diff --git a/testing/web-platform/tests/tools/requirements_tests.txt b/testing/web-platform/tests/tools/requirements_tests.txt index 0604dceb0a..6455286736 100644 --- a/testing/web-platform/tests/tools/requirements_tests.txt +++ b/testing/web-platform/tests/tools/requirements_tests.txt @@ -2,5 +2,5 @@ httpx[http2]==0.24.1 json-e==4.5.3 jsonschema==4.17.3 pyyaml==6.0.1 -taskcluster==60.3.2 +taskcluster==60.4.1 mozterm==1.0.0 diff --git a/testing/web-platform/tests/tools/serve/serve.py b/testing/web-platform/tests/tools/serve/serve.py index 116a98c0fc..300f8270a6 100644 --- a/testing/web-platform/tests/tools/serve/serve.py +++ b/testing/web-platform/tests/tools/serve/serve.py @@ -315,6 +315,20 @@ class WindowHandler(HtmlWrapperHandler): """ +class WindowModulesHandler(HtmlWrapperHandler): + global_type = "window-module" + path_replace = [(".any.window-module.html", ".any.js")] + wrapper = """<!doctype html> +<meta charset=utf-8> +%(meta)s +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +%(script)s +<div id=log></div> +<script type=module src="%(path)s"></script> +""" + + class AnyHtmlHandler(HtmlWrapperHandler): global_type = "window" path_replace = [(".any.html", ".any.js")] @@ -577,6 +591,7 @@ class RoutesBuilder: ("GET", "*.any.serviceworker.html", ServiceWorkersHandler), ("GET", "*.any.serviceworker-module.html", ServiceWorkerModulesHandler), ("GET", "*.any.shadowrealm.html", ShadowRealmHandler), + ("GET", "*.any.window-module.html", WindowModulesHandler), ("GET", "*.any.worker.js", ClassicWorkerHandler), ("GET", "*.any.worker-module.js", ModuleWorkerHandler), ("GET", "*.asis", handlers.AsIsHandler), @@ -585,6 +600,7 @@ class RoutesBuilder: ("*", "/.well-known/attribution-reporting/report-aggregate-attribution", handlers.PythonScriptHandler), ("*", "/.well-known/attribution-reporting/debug/report-aggregate-attribution", handlers.PythonScriptHandler), ("*", "/.well-known/attribution-reporting/debug/verbose", handlers.PythonScriptHandler), + ("GET", "/.well-known/interest-group/permissions/", handlers.PythonScriptHandler), ("*", "/.well-known/private-aggregation/*", handlers.PythonScriptHandler), ("*", "/.well-known/web-identity", handlers.PythonScriptHandler), ("*", "*.py", handlers.PythonScriptHandler), diff --git a/testing/web-platform/tests/tools/webdriver/webdriver/bidi/client.py b/testing/web-platform/tests/tools/webdriver/webdriver/bidi/client.py index 73bba55791..045bc048c4 100644 --- a/testing/web-platform/tests/tools/webdriver/webdriver/bidi/client.py +++ b/testing/web-platform/tests/tools/webdriver/webdriver/bidi/client.py @@ -205,7 +205,7 @@ class BidiSession: if not listeners: listeners = self.event_listeners.get(None, []) for listener in listeners: - await listener(data["method"], data["params"]) + asyncio.create_task(listener(data["method"], data["params"])) else: raise ValueError(f"Unexpected message: {data!r}") diff --git a/testing/web-platform/tests/tools/webdriver/webdriver/bidi/error.py b/testing/web-platform/tests/tools/webdriver/webdriver/bidi/error.py index 42361ca90b..bb3bd4d7fc 100644 --- a/testing/web-platform/tests/tools/webdriver/webdriver/bidi/error.py +++ b/testing/web-platform/tests/tools/webdriver/webdriver/bidi/error.py @@ -95,6 +95,10 @@ class UnableToSetCookieException(BidiException): error_code = "unable to set cookie" +class UnableToSetFileInputException(BidiException): + error_code = "unable to set file input" + + class UnderspecifiedStoragePartitionException(BidiException): error_code = "underspecified storage partition" diff --git a/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/browsing_context.py b/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/browsing_context.py index cdb5e11816..f2d2fad858 100644 --- a/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/browsing_context.py +++ b/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/browsing_context.py @@ -3,7 +3,7 @@ from enum import Enum from typing import Any, Dict, List, Mapping, MutableMapping, Optional, Union from ._module import BidiModule, command -from .script import OwnershipModel, SerializationOptions +from .script import SerializationOptions from ..undefined import UNDEFINED, Undefined @@ -140,17 +140,11 @@ class BrowsingContext(BidiModule): context: str, locator: Mapping[str, Any], max_node_count: Optional[int] = None, - ownership: Optional[OwnershipModel] = None, - sandbox: Optional[str] = None, serialization_options: Optional[SerializationOptions] = None, start_nodes: Optional[List[Mapping[str, Any]]] = None) -> Mapping[str, Any]: params: MutableMapping[str, Any] = {"context": context, "locator": locator} if max_node_count is not None: params["maxNodeCount"] = max_node_count - if ownership is not None: - params["ownership"] = ownership - if sandbox is not None: - params["sandbox"] = sandbox if serialization_options is not None: params["serializationOptions"] = serialization_options if start_nodes is not None: diff --git a/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/input.py b/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/input.py index b2703843b1..ee4f8136e9 100644 --- a/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/input.py +++ b/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/input.py @@ -410,6 +410,17 @@ class Input(BidiModule): params: MutableMapping[str, Any] = {"context": context} return params + @command + def set_files( + self, context: str, element: Any, files: List[str] + ) -> Mapping[str, Any]: + params: MutableMapping[str, Any] = { + "context": context, + "element": element, + "files": files, + } + return params + def get_element_origin(element: Any) -> Mapping[str, Any]: return {"type": "element", "element": {"sharedId": element["sharedId"]}} diff --git a/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/network.py b/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/network.py index 8bc51334d2..4523f67e9c 100644 --- a/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/network.py +++ b/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/network.py @@ -106,7 +106,7 @@ URLPattern = Union[URLPatternPattern, URLPatternString] class Network(BidiModule): @command def add_intercept( - self, phases: List[str], url_patterns: Optional[List[URLPattern]] = None + self, phases: List[str], url_patterns: Optional[List[URLPattern]] = None, contexts: Optional[List[str]] = None ) -> Mapping[str, Any]: params: MutableMapping[str, Any] = { "phases": phases, @@ -115,6 +115,9 @@ class Network(BidiModule): if url_patterns is not None: params["urlPatterns"] = url_patterns + if contexts is not None: + params["contexts"] = contexts + return params @add_intercept.result diff --git a/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/script.py b/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/script.py index 737426a5d5..01855766d8 100644 --- a/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/script.py +++ b/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/script.py @@ -3,6 +3,7 @@ from typing import Any, Dict, List, Mapping, MutableMapping, Optional, Union from ..error import UnknownErrorException from ._module import BidiModule, command +from ..undefined import UNDEFINED, Undefined class ScriptEvaluateResultException(Exception): @@ -78,15 +79,15 @@ Target = Union[RealmTarget, ContextTarget] class SerializationOptions(Dict[str, Any]): def __init__( self, - max_dom_depth: Optional[int] = None, - max_object_depth: Optional[int] = None, - include_shadow_tree: Optional[str] = None + max_dom_depth: Union[Optional[int], Undefined] = UNDEFINED, + max_object_depth: Union[Optional[int], Undefined] = UNDEFINED, + include_shadow_tree: Union[Optional[str], Undefined] = UNDEFINED ): - if max_dom_depth is not None: + if max_dom_depth is not UNDEFINED: self["maxDomDepth"] = max_dom_depth - if max_object_depth is not None: + if max_object_depth is not UNDEFINED: self["maxObjectDepth"] = max_object_depth - if include_shadow_tree is not None: + if include_shadow_tree is not UNDEFINED and include_shadow_tree is not None: self["includeShadowTree"] = include_shadow_tree diff --git a/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/session.py b/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/session.py index fe1c038510..725aab1bec 100644 --- a/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/session.py +++ b/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/session.py @@ -40,9 +40,9 @@ class Session(BidiModule): @command def unsubscribe(self, - events: Optional[List[str]] = None, + events: List[str], contexts: Optional[List[str]] = None) -> Mapping[str, Any]: - params: MutableMapping[str, Any] = {"events": events if events is not None else []} + params: MutableMapping[str, Any] = {"events": events} if contexts is not None: params["contexts"] = contexts return params diff --git a/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/storage.py b/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/storage.py index 882306ea72..14e8fa9434 100644 --- a/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/storage.py +++ b/testing/web-platform/tests/tools/webdriver/webdriver/bidi/modules/storage.py @@ -95,6 +95,20 @@ class Storage(BidiModule): return params @command + def delete_cookies( + self, + filter: Optional[CookieFilter] = None, + partition: Optional[PartitionDescriptor] = None, + ) -> Mapping[str, Any]: + params: MutableMapping[str, Any] = {} + + if filter is not None: + params["filter"] = filter + if partition is not None: + params["partition"] = partition + return params + + @command def set_cookie( self, cookie: PartialCookie, diff --git a/testing/web-platform/tests/tools/wpt/android.py b/testing/web-platform/tests/tools/wpt/android.py index 89dc9fad25..f25350db07 100644 --- a/testing/web-platform/tests/tools/wpt/android.py +++ b/testing/web-platform/tests/tools/wpt/android.py @@ -17,9 +17,9 @@ here = os.path.abspath(os.path.dirname(__file__)) wpt_root = os.path.abspath(os.path.join(here, os.pardir, os.pardir)) -NDK_VERSION = "r25c" -CMDLINE_TOOLS_VERSION_STRING = "11.0" -CMDLINE_TOOLS_VERSION = "9644228" +NDK_VERSION = "r26c" +CMDLINE_TOOLS_VERSION_STRING = "12.0" +CMDLINE_TOOLS_VERSION = "11076708" AVD_MANIFEST_X86_64 = { "emulator_package": "system-images;android-24;default;x86_64", @@ -100,6 +100,8 @@ def install_fixed_emulator_version(logger, paths): emulator_path = os.path.join(paths["sdk"], "emulator") latest_emulator_path = os.path.join(paths["sdk"], "emulator_latest") + if os.path.exists(latest_emulator_path): + shutil.rmtree(latest_emulator_path) os.rename(emulator_path, latest_emulator_path) download_and_extract(url, paths["sdk"]) @@ -323,7 +325,16 @@ def start(logger, dest=None, reinstall=False, prompt=True, device_serial=None): emulator.start() timer = threading.Timer(300, cancel_start(threading.get_ident())) timer.start() - emulator.wait_for_start() + for i in range(10): + logger.info(f"Wait for emulator to start attempt {i + 1}/10") + try: + emulator.wait_for_start() + except Exception: + import traceback + logger.warning(f"""emulator.wait_for_start() failed: +{traceback.format_exc()}""") + else: + break timer.cancel() return emulator diff --git a/testing/web-platform/tests/tools/wpt/browser.py b/testing/web-platform/tests/tools/wpt/browser.py index c7f67d334e..2f9c453131 100644 --- a/testing/web-platform/tests/tools/wpt/browser.py +++ b/testing/web-platform/tests/tools/wpt/browser.py @@ -1282,6 +1282,11 @@ class Chrome(ChromeChromiumBase): version = self.version(browser_binary) if version is None: + # Check if the user has given a Chromium binary. + chromium = Chromium(self.logger) + if chromium.version(browser_binary): + raise ValueError("Provided binary is a Chromium binary and should be run using " + "\"./wpt run chromium\" or similar.") raise ValueError(f"Unable to detect browser version from binary at {browser_binary}. " " Cannot install ChromeDriver without a valid version to match.") diff --git a/testing/web-platform/tests/tools/wpt/update.py b/testing/web-platform/tests/tools/wpt/update.py index 4dba7e69df..06fe9bfbad 100644 --- a/testing/web-platform/tests/tools/wpt/update.py +++ b/testing/web-platform/tests/tools/wpt/update.py @@ -43,7 +43,6 @@ def update_expectations(_, **kwargs): update_properties = metadata.get_properties(properties_file=kwargs["properties_file"], extra_properties=kwargs["extra_property"], - config=kwargs["config"], product=kwargs["product"]) manifest_update(kwargs["test_paths"]) diff --git a/testing/web-platform/tests/tools/wptrunner/requirements_opera.txt b/testing/web-platform/tests/tools/wptrunner/requirements_opera.txt index 4ff0fedd32..db0c5dd992 100644 --- a/testing/web-platform/tests/tools/wptrunner/requirements_opera.txt +++ b/testing/web-platform/tests/tools/wptrunner/requirements_opera.txt @@ -1,2 +1,2 @@ mozprocess==1.3.1 -selenium==4.14.0 +selenium==4.18.1 diff --git a/testing/web-platform/tests/tools/wptrunner/requirements_sauce.txt b/testing/web-platform/tests/tools/wptrunner/requirements_sauce.txt index 5538fb0672..c9e42346ce 100644 --- a/testing/web-platform/tests/tools/wptrunner/requirements_sauce.txt +++ b/testing/web-platform/tests/tools/wptrunner/requirements_sauce.txt @@ -1,2 +1,2 @@ -selenium==4.14.0 +selenium==4.18.1 requests==2.31.0 diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/chrome.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/chrome.py index 05f81461e2..7cb46783fc 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/chrome.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/chrome.py @@ -97,6 +97,8 @@ def executor_kwargs(logger, test_type, test_environment, run_info_data, chrome_options["args"].append("--use-fake-ui-for-media-stream") # Use a fake UI for FedCM to allow testing it. chrome_options["args"].append("--use-fake-ui-for-fedcm") + # Use a fake UI for digital identity to allow testing it. + chrome_options["args"].append("--use-fake-ui-for-digital-identity") # Shorten delay for Reporting <https://w3c.github.io/reporting/>. chrome_options["args"].append("--short-reporting-delay") # Point all .test domains to localhost for Chrome @@ -148,6 +150,7 @@ def executor_kwargs(logger, test_type, test_environment, run_info_data, # set. '--headless' should always mean the new headless mode, as the old # headless mode is not used anyway. if kwargs["headless"] and ("--headless=new" not in chrome_options["args"] and + "--headless=old" not in chrome_options["args"] and "--headless" not in chrome_options["args"]): chrome_options["args"].append("--headless=new") diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/content_shell.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/content_shell.py index 23f4e99da6..6df8671e0a 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/content_shell.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/content_shell.py @@ -1,29 +1,17 @@ # mypy: allow-untyped-defs -import contextlib -import os -import subprocess -from multiprocessing import Queue, Event -from threading import Thread -from urllib.parse import urljoin - from . import chrome_spki_certs -from .base import ( - Browser, - ExecutorBrowser, - OutputHandler, - browser_command, -) +from .base import cmd_arg, require_arg from .base import get_timeout_multiplier # noqa: F401 -from .chrome import debug_args +from .chrome import ChromeBrowser, debug_args from ..executors import executor_kwargs as base_executor_kwargs -from ..executors.base import server_url -from ..executors.executorcontentshell import ( # noqa: F401 - ContentShellCrashtestExecutor, - ContentShellPrintRefTestExecutor, - ContentShellRefTestExecutor, - ContentShellTestharnessExecutor, +from ..executors.base import WdspecExecutor # noqa: F401 +from ..executors.executorchrome import ( # noqa: F401 + ChromeDriverPrintRefTestExecutor, + ChromeDriverRefTestExecutor, + ChromeDriverTestharnessExecutor, ) +from ..executors.executorwebdriver import WebDriverCrashtestExecutor # noqa: F401 ENABLE_THREADED_COMPOSITING_FLAG = '--enable-threaded-compositing' DISABLE_THREADED_COMPOSITING_FLAG = '--disable-threaded-compositing' @@ -34,10 +22,11 @@ __wptrunner__ = {"product": "content_shell", "check_args": "check_args", "browser": "ContentShellBrowser", "executor": { - "crashtest": "ContentShellCrashtestExecutor", - "print-reftest": "ContentShellPrintRefTestExecutor", - "reftest": "ContentShellRefTestExecutor", - "testharness": "ContentShellTestharnessExecutor", + "crashtest": "WebDriverCrashtestExecutor", + "print-reftest": "ChromeDriverPrintRefTestExecutor", + "reftest": "ChromeDriverRefTestExecutor", + "testharness": "ChromeDriverTestharnessExecutor", + "wdspec": "WdspecExecutor", }, "browser_kwargs": "browser_kwargs", "executor_kwargs": "executor_kwargs", @@ -48,54 +37,79 @@ __wptrunner__ = {"product": "content_shell", def check_args(**kwargs): - pass + require_arg(kwargs, "webdriver_binary") + + +def browser_kwargs(logger, test_type, run_info_data, config, **kwargs): + return {"binary": kwargs["binary"], + "webdriver_binary": kwargs["webdriver_binary"], + "webdriver_args": kwargs.get("webdriver_args"), + "debug_info": kwargs["debug_info"]} -def browser_kwargs(logger, test_type, run_info_data, config, subsuite, **kwargs): - args = [] - args.append("--ignore-certificate-errors-spki-list=%s" % +def executor_kwargs(logger, test_type, test_environment, run_info_data, subsuite, + **kwargs): + sanitizer_enabled = kwargs.get("sanitizer_enabled") + if sanitizer_enabled: + test_type = "crashtest" + executor_kwargs = base_executor_kwargs(test_type, test_environment, run_info_data, + subsuite, **kwargs) + executor_kwargs["sanitizer_enabled"] = sanitizer_enabled + executor_kwargs["close_after_done"] = True + executor_kwargs["reuse_window"] = kwargs.get("reuse_window", False) + + capabilities = { + "goog:chromeOptions": { + "prefs": { + "profile": { + "default_content_setting_values": { + "popups": 1 + } + } + }, + "excludeSwitches": ["enable-automation"], + "w3c": True, + } + } + + chrome_options = capabilities["goog:chromeOptions"] + if kwargs["binary"] is not None: + chrome_options["binary"] = kwargs["binary"] + + chrome_options["args"] = [] + chrome_options["args"].append("--ignore-certificate-errors-spki-list=%s" % ','.join(chrome_spki_certs.IGNORE_CERTIFICATE_ERRORS_SPKI_LIST)) # For WebTransport tests. - args.append("--webtransport-developer-mode") + chrome_options["args"].append("--webtransport-developer-mode") + chrome_options["args"].append("--enable-blink-test-features") - if not kwargs["headless"]: - args.append("--disable-headless-mode") + # always run in headful mode for content_shell if kwargs["debug_info"]: - args.extend(debug_args(kwargs["debug_info"])) + chrome_options["args"].extend(debug_args(kwargs["debug_info"])) - # `--run-web-tests -` are specific to content_shell - they activate web - # test protocol mode. - args.append("--run-web-tests") for arg in kwargs.get("binary_args", []): - if arg not in args: - args.append(arg) + # skip empty --user-data-dir args, and allow chromedriver to pick one. + # Do not pass in --run-web-tests, otherwise content_shell will hang. + if arg in ['--user-data-dir', '--run-web-tests']: + continue + if arg not in chrome_options["args"]: + chrome_options["args"].append(arg) # Temporary workaround to align with RWT behavior. Unless a vts explicitly # enables threaded compositing, we should use single threaded compositing - if ENABLE_THREADED_COMPOSITING_FLAG not in subsuite.config.get("binary_args", []): - args.extend([DISABLE_THREADED_COMPOSITING_FLAG, - DISABLE_THREADED_ANIMATION_FLAG]) + # Do not pass in DISABLE_THREADED_COMPOSITING_FLAG or + # DISABLE_THREADED_ANIMATION_FLAG. Content shell will hang due to that. + #if ENABLE_THREADED_COMPOSITING_FLAG not in subsuite.config.get("binary_args", []): + # chrome_options["args"].extend([DISABLE_THREADED_COMPOSITING_FLAG, + # DISABLE_THREADED_ANIMATION_FLAG]) for arg in subsuite.config.get("binary_args", []): - if arg not in args: - args.append(arg) - args.append("-") - - return {"binary": kwargs["binary"], - "binary_args": args, - "debug_info": kwargs["debug_info"], - "pac_origin": server_url(config, "http")} + if arg not in chrome_options["args"]: + chrome_options["args"].append(arg) + executor_kwargs["capabilities"] = capabilities -def executor_kwargs(logger, test_type, test_environment, run_info_data, - **kwargs): - sanitizer_enabled = kwargs.get("sanitizer_enabled") - if sanitizer_enabled: - test_type = "crashtest" - executor_kwargs = base_executor_kwargs(test_type, test_environment, run_info_data, - **kwargs) - executor_kwargs["sanitizer_enabled"] = sanitizer_enabled return executor_kwargs @@ -105,7 +119,6 @@ def env_extras(**kwargs): def env_options(): return {"server_host": "127.0.0.1", - "testharnessreport": "testharnessreport-content-shell.js", "supports_debugger": True} @@ -113,187 +126,9 @@ def update_properties(): return (["debug", "os", "processor"], {"os": ["version"], "processor": ["bits"]}) -class ContentShellBrowser(Browser): - """Class that represents an instance of content_shell. - - Upon startup, the stdout, stderr, and stdin pipes of the underlying content_shell - process are connected to multiprocessing Queues so that the runner process can - interact with content_shell through its protocol mode. - - See Also: - Protocol Mode: https://chromium.googlesource.com/chromium/src.git/+/HEAD/content/web_test/browser/test_info_extractor.h - """ - # Seconds to wait for the process to stop after it was sent a `QUIT` - # command, after which `SIGTERM` or `TerminateProcess()` forces termination. - # The timeout is ported from: - # https://chromium.googlesource.com/chromium/src/+/b175d48d3ea4ea66eea35c88c11aa80d233f3bee/third_party/blink/tools/blinkpy/web_tests/port/base.py#476 - termination_timeout: float = 3 - - def __init__(self, logger, binary="content_shell", binary_args=None, - debug_info=None, pac_origin=None, **kwargs): - super().__init__(logger) - self._debug_cmd_prefix, self._browser_cmd = browser_command( - binary, binary_args or [], debug_info) - self._output_handler = None - self._proc = None - self._pac_origin = pac_origin - self._pac = None - - def start(self, group_metadata, **settings): - browser_cmd, pac = list(self._browser_cmd), settings.get("pac") - if pac: - browser_cmd.insert(1, f"--proxy-pac-url={pac}") - self.logger.debug(f"Starting content shell: {browser_cmd[0]}...") - args = [*self._debug_cmd_prefix, *browser_cmd] - self._output_handler = OutputHandler(self.logger, args) - if os.name == "posix": - close_fds, preexec_fn = True, lambda: os.setpgid(0, 0) - else: - close_fds, preexec_fn = False, None - self._proc = subprocess.Popen(args, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - close_fds=close_fds, - preexec_fn=preexec_fn) - self._output_handler.after_process_start(self._proc.pid) - - self._stdout_queue = Queue() - self._stderr_queue = Queue() - self._stdin_queue = Queue() - self._io_stopped = Event() - - self._stdout_reader = self._create_reader_thread("stdout-reader", - self._proc.stdout, - self._stdout_queue, - prefix=b"OUT: ") - self._stderr_reader = self._create_reader_thread("stderr-reader", - self._proc.stderr, - self._stderr_queue, - prefix=b"ERR: ") - self._stdin_writer = self._create_writer_thread("stdin-writer", - self._proc.stdin, - self._stdin_queue) - - # Content shell is likely still in the process of initializing. The actual waiting - # for the startup to finish is done in the ContentShellProtocol. - self.logger.debug("Content shell has been started.") - self._output_handler.start(group_metadata=group_metadata, **settings) - - def stop(self, force=False): - self.logger.debug("Stopping content shell...") - - clean_shutdown = stopped = True - if self.is_alive(): - clean_shutdown = self._terminate_process(force=force) - - # Close these queues cleanly to avoid broken pipe error spam in the logs. - self._stdin_queue.put(None) - for thread in [self._stdout_reader, self._stderr_reader, self._stdin_writer]: - thread.join(2) - if thread.is_alive(): - self.logger.warning(f"Content shell IO thread {thread.name} did not shut down gracefully.") - stopped = False - - if not self.is_alive(): - self.logger.debug( - "Content shell has been stopped " - f"(PID: {self._proc.pid}, exit code: {self._proc.returncode})") - else: - stopped = False - self.logger.warning(f"Content shell failed to stop (PID: {self._proc.pid})") - if stopped and self._output_handler is not None: - self._output_handler.after_process_stop(clean_shutdown) - self._output_handler = None - return stopped - - def _terminate_process(self, force: bool = False) -> bool: - self._stdin_queue.put(b"QUIT\n") - with contextlib.suppress(subprocess.TimeoutExpired): - self._proc.wait(timeout=self.termination_timeout) - return True - self.logger.warning( - "Content shell failed to respond to QUIT command " - f"(PID: {self._proc.pid}, timeout: {self.termination_timeout}s)") - # Skip `terminate()` on Windows, which is an alias for `kill()`, and - # only `kill()` for `force=True`. - # - # [1]: https://docs.python.org/3/library/subprocess.html#subprocess.Popen.kill - if os.name == "posix": - self._proc.terminate() - with contextlib.suppress(subprocess.TimeoutExpired): - self._proc.wait(timeout=1) - return False - if force: - self._proc.kill() - return False - - def is_alive(self): - return self._proc is not None and self._proc.poll() is None - - def pid(self): - return self._proc.pid if self._proc else None - - def executor_browser(self): - """This function returns the `ExecutorBrowser` object that is used by other - processes to interact with content_shell. In our case, this consists of the three - multiprocessing Queues as well as an `io_stopped` event to signal when the - underlying pipes have reached EOF. - """ - return ExecutorBrowser, {"stdout_queue": self._stdout_queue, - "stderr_queue": self._stderr_queue, - "stdin_queue": self._stdin_queue, - "io_stopped": self._io_stopped} - - def check_crash(self, process, test): - return not self.is_alive() - - def settings(self, test): - pac_path = test.environment.get("pac") - if self._pac_origin and pac_path: - self._pac = urljoin(self._pac_origin, pac_path) - return {"pac": self._pac} - return {} - - def _create_reader_thread(self, name, stream, queue, prefix=b""): - """This creates (and starts) a background thread which reads lines from `stream` and - puts them into `queue` until `stream` reports EOF. - """ - def reader_thread(stream, queue, stop_event): - while True: - line = stream.readline() - if not line: - break - self._output_handler(prefix + line.rstrip()) - queue.put(line) - - stop_event.set() - queue.close() - queue.join_thread() - - result = Thread(name=name, - target=reader_thread, - args=(stream, queue, self._io_stopped), - daemon=True) - result.start() - return result - - def _create_writer_thread(self, name, stream, queue): - """This creates (and starts) a background thread which gets items from `queue` and - writes them into `stream` until it encounters a None item in the queue. - """ - def writer_thread(stream, queue): - while True: - line = queue.get() - if not line: - break - - stream.write(line) - stream.flush() - - result = Thread(name=name, - target=writer_thread, - args=(stream, queue), - daemon=True) - result.start() - return result +class ContentShellBrowser(ChromeBrowser): + def make_command(self): + return [self.webdriver_binary, + cmd_arg("port", str(self.port)), + cmd_arg("url-base", self.base_path), + cmd_arg("enable-chrome-logs")] + self.webdriver_args diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox_android.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox_android.py index 3ce3b11d1f..0e90c8a6e4 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox_android.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox_android.py @@ -1,8 +1,9 @@ # mypy: allow-untyped-defs import os -import subprocess import re +import subprocess +import traceback from mozrunner import FennecEmulatorRunner, get_app_context @@ -349,7 +350,13 @@ class FirefoxAndroidBrowser(Browser): def check_crash(self, process, test): if not os.environ.get("MINIDUMP_STACKWALK", "") and self.stackwalk_binary: os.environ["MINIDUMP_STACKWALK"] = self.stackwalk_binary - return bool(self.runner.check_for_crashes(test_name=test)) + try: + return bool(self.runner.check_for_crashes(test_name=test)) + except Exception: + # We sometimes see failures trying to copy the minidump files + self.logger.warning(f"""Failed to complete crash check, assuming no crash: +{traceback.format_exc()}""") + return False class FirefoxAndroidWdSpecBrowser(FirefoxWdSpecBrowser): diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executorcontentshell.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executorcontentshell.py deleted file mode 100644 index 82a6aebcdb..0000000000 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executorcontentshell.py +++ /dev/null @@ -1,328 +0,0 @@ -# mypy: allow-untyped-defs - -from .base import RefTestExecutor, RefTestImplementation, CrashtestExecutor, TestharnessExecutor -from .executorchrome import make_sanitizer_mixin -from .protocol import Protocol, ProtocolPart -from time import time -from queue import Empty -from base64 import b64encode -import json - - -class CrashError(BaseException): - pass - -class LeakError(BaseException): - pass - -def _read_line(io_queue, deadline=None, encoding=None, errors="strict", raise_crash_leak=True): - """Reads a single line from the io queue. The read must succeed before `deadline` or - a TimeoutError is raised. The line is returned as a bytestring or optionally with the - specified `encoding`. If `raise_crash_leak` is set, a CrashError is raised if the line - happens to be a crash message, or a LeakError is raised if the line happens to be a - leak message. - """ - current_time = time() - - if deadline and current_time > deadline: - raise TimeoutError() - - try: - line = io_queue.get(True, deadline - current_time if deadline else None) - if raise_crash_leak and line.startswith(b"#CRASHED"): - raise CrashError() - if raise_crash_leak and line.startswith(b"#LEAK"): - raise LeakError() - except Empty as e: - raise TimeoutError() from e - - return line.decode(encoding, errors) if encoding else line - - -class ContentShellTestPart(ProtocolPart): - """This protocol part is responsible for running tests via content_shell's protocol mode. - - For more details, see: - https://chromium.googlesource.com/chromium/src.git/+/HEAD/content/web_test/browser/test_info_extractor.h - """ - name = "content_shell_test" - eof_marker = '#EOF\n' # Marker sent by content_shell after blocks. - - def __init__(self, parent): - super().__init__(parent) - self.stdout_queue = parent.browser.stdout_queue - self.stdin_queue = parent.browser.stdin_queue - - def do_test(self, command, timeout=None): - """Send a command to content_shell and return the resulting outputs. - - A command consists of a URL to navigate to, followed by an optional - expected image hash and 'print' mode specifier. The syntax looks like: - http://web-platform.test:8000/test.html['<hash>['print]] - """ - self._send_command(command) - - deadline = time() + timeout if timeout else None - # The first block can also contain audio data but not in WPT. - text = self._read_block(deadline) - image = self._read_block(deadline) - - return text, image - - def _send_command(self, command): - """Sends a single `command`, i.e. a URL to open, to content_shell. - """ - self.stdin_queue.put((command + "\n").encode("utf-8")) - - def _read_block(self, deadline=None): - """Tries to read a single block of content from stdout before the `deadline`. - """ - while True: - line = _read_line(self.stdout_queue, deadline, "latin-1").rstrip() - - if line == "Content-Type: text/plain": - return self._read_text_block(deadline) - - if line == "Content-Type: image/png": - return self._read_image_block(deadline) - - if line == "#EOF": - return None - - def _read_text_block(self, deadline=None): - """Tries to read a plain text block in utf-8 encoding before the `deadline`. - """ - result = "" - - while True: - line = _read_line(self.stdout_queue, deadline, "utf-8", "replace", False) - - if line.endswith(self.eof_marker): - result += line[:-len(self.eof_marker)] - break - elif line.endswith('#EOF\r\n'): - result += line[:-len('#EOF\r\n')] - self.logger.warning('Got a CRLF-terminated #EOF - this is a driver bug.') - break - - result += line - - return result - - def _read_image_block(self, deadline=None): - """Tries to read an image block (as a binary png) before the `deadline`. - """ - content_length_line = _read_line(self.stdout_queue, deadline, "utf-8") - assert content_length_line.startswith("Content-Length:") - content_length = int(content_length_line[15:]) - - result = bytearray() - - while True: - line = _read_line(self.stdout_queue, deadline, raise_crash_leak=False) - excess = len(line) + len(result) - content_length - - if excess > 0: - # This is the line that contains the EOF marker. - assert excess == len(self.eof_marker) - result += line[:-excess] - break - - result += line - - return result - - -class ContentShellErrorsPart(ProtocolPart): - """This protocol part is responsible for collecting the errors reported by content_shell. - """ - name = "content_shell_errors" - - def __init__(self, parent): - super().__init__(parent) - self.stderr_queue = parent.browser.stderr_queue - - def read_errors(self): - """Reads the entire content of the stderr queue as is available right now (no blocking). - """ - result = "" - - while not self.stderr_queue.empty(): - # There is no potential for race conditions here because this is the only place - # where we read from the stderr queue. - result += _read_line(self.stderr_queue, None, "utf-8", "replace", False) - - return result - - -class ContentShellBasePart(ProtocolPart): - """This protocol part provides functionality common to all executors. - - In particular, this protocol part implements `wait()`, which, when - `--pause-after-test` is enabled, test runners block on until the next test - should run. - """ - name = "base" - - def __init__(self, parent): - super().__init__(parent) - self.io_stopped = parent.browser.io_stopped - - def wait(self): - # This worker is unpaused when the browser window is closed, which this - # `multiprocessing.Event` signals. - self.io_stopped.wait() - # Never rerun the test. - return False - - -class ContentShellProtocol(Protocol): - implements = [ - ContentShellBasePart, - ContentShellTestPart, - ContentShellErrorsPart, - ] - init_timeout = 10 # Timeout (seconds) to wait for #READY message. - - def connect(self): - """Waits for content_shell to emit its "#READY" message which signals that it is fully - initialized. We wait for a maximum of self.init_timeout seconds. - """ - deadline = time() + self.init_timeout - - while True: - if _read_line(self.browser.stdout_queue, deadline).rstrip() == b"#READY": - break - - def after_connect(self): - pass - - def teardown(self): - # Close the queue properly to avoid broken pipe spam in the log. - self.browser.stdin_queue.close() - self.browser.stdin_queue.join_thread() - - def is_alive(self): - """Checks if content_shell is alive by determining if the IO pipes are still - open. This does not guarantee that the process is responsive. - """ - return self.browser.io_stopped.is_set() - - -def _convert_exception(test, exception, errors): - """Converts our TimeoutError and CrashError exceptions into test results. - """ - if isinstance(exception, TimeoutError): - return (test.make_result("EXTERNAL-TIMEOUT", errors), []) - if isinstance(exception, CrashError): - return (test.make_result("CRASH", errors), []) - if isinstance(exception, LeakError): - # TODO: the internal error is to force a restart, but it doesn't correctly - # describe what the issue is. Need to find a way to return a "FAIL", - # and restart the content_shell after the test run. - return (test.make_result("INTERNAL-ERROR", errors), []) - raise exception - - -def timeout_for_test(executor, test): - if executor.debug_info and executor.debug_info.interactive: - return None - return test.timeout * executor.timeout_multiplier - - -class ContentShellCrashtestExecutor(CrashtestExecutor): - def __init__(self, logger, browser, server_config, timeout_multiplier=1, debug_info=None, - **kwargs): - super().__init__(logger, browser, server_config, timeout_multiplier, debug_info, **kwargs) - self.protocol = ContentShellProtocol(self, browser) - - def do_test(self, test): - try: - _ = self.protocol.content_shell_test.do_test(self.test_url(test), - timeout_for_test(self, test)) - self.protocol.content_shell_errors.read_errors() - return self.convert_result(test, {"status": "PASS", "message": None}) - except BaseException as exception: - return _convert_exception(test, exception, self.protocol.content_shell_errors.read_errors()) - - -_SanitizerMixin = make_sanitizer_mixin(ContentShellCrashtestExecutor) - - -class ContentShellRefTestExecutor(RefTestExecutor, _SanitizerMixin): # type: ignore - def __init__(self, logger, browser, server_config, timeout_multiplier=1, screenshot_cache=None, - debug_info=None, reftest_screenshot="unexpected", **kwargs): - super().__init__(logger, browser, server_config, timeout_multiplier, screenshot_cache, - debug_info, reftest_screenshot, **kwargs) - self.implementation = RefTestImplementation(self) - self.protocol = ContentShellProtocol(self, browser) - - def reset(self): - self.implementation.reset() - - def do_test(self, test): - try: - result = self.implementation.run_test(test) - self.protocol.content_shell_errors.read_errors() - return self.convert_result(test, result) - except BaseException as exception: - return _convert_exception(test, exception, self.protocol.content_shell_errors.read_errors()) - - def screenshot(self, test, viewport_size, dpi, page_ranges): - # Currently, the page size and DPI are hardcoded for print-reftests: - # https://chromium.googlesource.com/chromium/src/+/4e1b7bc33d42b401d7d9ad1dcba72883add3e2af/content/web_test/renderer/test_runner.cc#100 - # Content shell has an internal `window.testRunner.setPrintingSize(...)` - # API, but it's not callable with protocol mode. - assert dpi is None - command = self.test_url(test) - if self.is_print: - # Currently, `content_shell` uses the expected image hash to avoid - # dumping a matching image as an optimization. In Chromium, the - # hash can be computed from an expected screenshot checked into the - # source tree (i.e., without looking at a reference). This is not - # possible in `wpt`, so pass an empty hash here to force a dump. - command += "''print" - - _, image = self.protocol.content_shell_test.do_test(command, - timeout_for_test(self, test)) - if not image: - return False, ("ERROR", self.protocol.content_shell_errors.read_errors()) - return True, b64encode(image).decode() - - -class ContentShellPrintRefTestExecutor(ContentShellRefTestExecutor): - is_print = True - - -class ContentShellTestharnessExecutor(TestharnessExecutor, _SanitizerMixin): # type: ignore - # Chromium's `testdriver-vendor.js` partially implements testdriver support - # with internal APIs [1]. - # - # [1]: https://chromium.googlesource.com/chromium/src/+/HEAD/docs/testing/writing_web_tests.md#Relying-on-Blink_Specific-Testing-APIs - supports_testdriver = True - - def __init__(self, logger, browser, server_config, timeout_multiplier=1, debug_info=None, - **kwargs): - super().__init__(logger, browser, server_config, timeout_multiplier, debug_info, **kwargs) - self.protocol = ContentShellProtocol(self, browser) - - def do_test(self, test): - try: - text, _ = self.protocol.content_shell_test.do_test(self.test_url(test), - timeout_for_test(self, test)) - errors = self.protocol.content_shell_errors.read_errors() - if not text: - return (test.make_result("ERROR", errors), []) - - result_url, status, message, stack, subtest_results = json.loads(text) - if result_url != test.url: - # Suppress `convert_result`'s URL validation. - # See `testharnessreport-content-shell.js` for details. - self.logger.warning('Got results from %s, expected %s' % (result_url, test.url)) - self.logger.warning('URL mismatch may be a false positive ' - 'if the test navigates') - result_url = test.url - raw_result = result_url, status, message, stack, subtest_results - return self.convert_result(test, raw_result) - except BaseException as exception: - return _convert_exception(test, exception, self.protocol.content_shell_errors.read_errors()) diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py index a5bf61d405..0f640d7741 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py @@ -548,7 +548,9 @@ class MarionetteCoverageProtocolPart(CoverageProtocolPart): return script = """ - const {PerTestCoverageUtils} = ChromeUtils.import("chrome://remote/content/marionette/PerTestCoverageUtils.jsm"); + const {PerTestCoverageUtils} = ChromeUtils.importESModule( + "chrome://remote/content/marionette/PerTestCoverageUtils.sys.mjs" + ); return PerTestCoverageUtils.enabled; """ with self.marionette.using_context(self.marionette.CONTEXT_CHROME): @@ -558,7 +560,9 @@ class MarionetteCoverageProtocolPart(CoverageProtocolPart): script = """ var callback = arguments[arguments.length - 1]; - const {PerTestCoverageUtils} = ChromeUtils.import("chrome://remote/content/marionette/PerTestCoverageUtils.jsm"); + const {PerTestCoverageUtils} = ChromeUtils.importESModule( + "chrome://remote/content/marionette/PerTestCoverageUtils.sys.mjs" + ); PerTestCoverageUtils.beforeTest().then(callback, callback); """ with self.marionette.using_context(self.marionette.CONTEXT_CHROME): @@ -578,7 +582,9 @@ class MarionetteCoverageProtocolPart(CoverageProtocolPart): script = """ var callback = arguments[arguments.length - 1]; - const {PerTestCoverageUtils} = ChromeUtils.import("chrome://remote/content/marionette/PerTestCoverageUtils.jsm"); + const {PerTestCoverageUtils} = ChromeUtils.importESModule( + "chrome://remote/content/marionette/PerTestCoverageUtils.sys.mjs" + ); PerTestCoverageUtils.afterTest().then(callback, callback); """ with self.marionette.using_context(self.marionette.CONTEXT_CHROME): @@ -695,11 +701,9 @@ class MarionetteDebugProtocolPart(DebugProtocolPart): def load_devtools(self): with self.marionette.using_context(self.marionette.CONTEXT_CHROME): - # Once ESR is 107 is released, we can replace the ChromeUtils.import(DevToolsShim.jsm) - # with ChromeUtils.importESModule(DevToolsShim.sys.mjs) in this snippet: self.parent.base.execute_script(""" -const { DevToolsShim } = ChromeUtils.import( - "chrome://devtools-startup/content/DevToolsShim.jsm" +const { DevToolsShim } = ChromeUtils.importESModule( + "chrome://devtools-startup/content/DevToolsShim.sys.mjs" ); const callback = arguments[arguments.length - 1]; diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executorwebdriver.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executorwebdriver.py index b49b9e2b57..6df2d96461 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executorwebdriver.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executorwebdriver.py @@ -614,7 +614,7 @@ class WebDriverTestharnessExecutor(TestharnessExecutor): # cases where the browser crashes during script execution: # # https://github.com/w3c/webdriver/issues/1308 - if not isinstance(result, list) or len(result) != 2: + if not isinstance(result, list) or len(result) != 3: try: is_alive = self.is_alive() except error.WebDriverException: @@ -623,6 +623,16 @@ class WebDriverTestharnessExecutor(TestharnessExecutor): if not is_alive: raise Exception("Browser crashed during script execution.") + # A user prompt created after starting execution of the resume + # script will resolve the script with `null` [1, 2]. In that case, + # cycle this event loop and handle the prompt the next time the + # resume script executes. + # + # [1]: Step 5.3 of https://www.w3.org/TR/webdriver/#execute-async-script + # [2]: https://www.w3.org/TR/webdriver/#dfn-execute-a-function-body + if result is None: + continue + done, rv = handler(result) if done: break diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py index b9cb61eb07..5d7ea2b011 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py @@ -10,7 +10,6 @@ from six import ensure_str, ensure_text from sys import intern from . import manifestupdate -from . import products from . import testloader from . import wptmanifest from . import wpttest @@ -57,12 +56,15 @@ def get_properties(properties_file=None, extra_properties=None, config=None, pro :param properties_file: Path to a JSON file containing properties. :param extra_properties: List of extra properties to use - :param config: (deprecated) wptrunner config - :param Product: (deprecated) product name (requires a config argument to be used) + :param config: (deprecated, unused) wptrunner config + :param Product: (deprecated) product name """ properties = [] dependents = {} + if config is not None: + logger.warning("Got `config` in metadata.get_properties; this is ignored") + if properties_file is not None: logger.debug(f"Reading update properties from {properties_file}") try: @@ -99,12 +101,8 @@ def get_properties(properties_file=None, extra_properties=None, config=None, pro elif product is not None: logger.warning("Falling back to getting metadata update properties from wptrunner browser " "product file, this will be removed") - if config is None: - msg = "Must provide a config together with a product" - logger.critical(msg) - raise ValueError(msg) - properties, dependents = products.load_product_update(config, product) + properties, dependents = product.update_properties if extra_properties is not None: properties.extend(extra_properties) diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/products.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/products.py index c81396f3dd..9657bde5f5 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/products.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/products.py @@ -39,22 +39,11 @@ class Product: cls = getattr(module, cls_name) self.executor_classes[test_type] = cls + self.update_properties = (getattr(module, data["update_properties"])() + if "update_properties" in data else (["product"], {})) + + def get_browser_cls(self, test_type): if test_type in self._browser_cls: return self._browser_cls[test_type] return self._browser_cls[None] - - -def load_product_update(config, product): - """Return tuple of (property_order, boolean_properties) indicating the - run_info properties to use when constructing the expectation data for - this product. None for either key indicates that the default keys - appropriate for distinguishing based on platform will be used.""" - - module = product_module(config, product) - data = module.__wptrunner__ - - update_properties = (getattr(module, data["update_properties"])() - if "update_properties" in data else (["product"], {})) - - return update_properties diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/update/metadata.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/update/metadata.py index 48519900e7..f1ab74accf 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/update/metadata.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/update/metadata.py @@ -2,25 +2,18 @@ import os -from .. import metadata, products +from .. import metadata from .base import Step, StepRunner -class GetUpdatePropertyList(Step): - provides = ["update_properties"] - - def create(self, state): - state.update_properties = products.load_product_update(state.config, state.product.name) - - class UpdateExpected(Step): """Do the metadata update on the local checkout""" def create(self, state): metadata.update_expected(state.paths, state.run_log, - update_properties=state.update_properties, + update_properties=state.product.update_properties, full_update=state.full_update, disable_intermittent=state.disable_intermittent, update_intermittent=state.update_intermittent, @@ -57,6 +50,5 @@ class CreateMetadataPatch(Step): class MetadataUpdateRunner(StepRunner): """(Sub)Runner for updating metadata""" - steps = [GetUpdatePropertyList, - UpdateExpected, + steps = [UpdateExpected, CreateMetadataPatch] diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/defaultpy/default.py b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/defaultpy/default.py new file mode 100644 index 0000000000..c235aeb58a --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/defaultpy/default.py @@ -0,0 +1,3 @@ +def main(request, response): + response.headers.set("Content-Type", "text/plain") + return "default" diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/defaultpy/default.sub.py b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/defaultpy/default.sub.py new file mode 100644 index 0000000000..32a10d8535 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/defaultpy/default.sub.py @@ -0,0 +1,3 @@ +def main(request, response): + response.headers.set("Content-Type", "text/plain") + return "default.sub" diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/defaultsubpy/default.sub.py b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/defaultsubpy/default.sub.py new file mode 100644 index 0000000000..c549100066 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/defaultsubpy/default.sub.py @@ -0,0 +1,3 @@ +def main(request, response): + response.headers.set("Content-Type", "text/plain") + return "{{host}}" diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/foo.any.window-module.html b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/foo.any.window-module.html new file mode 100644 index 0000000000..59646e6abd --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/foo.any.window-module.html @@ -0,0 +1,8 @@ +<!doctype html> +<meta charset=utf-8> + +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> + +<div id=log></div> +<script type=module src="/foo.any.js"></script> diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/test_handlers.py b/testing/web-platform/tests/tools/wptserve/tests/functional/test_handlers.py index 623a0e5b6a..91b05f4fe4 100644 --- a/testing/web-platform/tests/tools/wptserve/tests/functional/test_handlers.py +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/test_handlers.py @@ -275,6 +275,17 @@ class TestPythonHandler(TestUsingServer): self.assertEqual("text/plain", resp.info()["Content-Type"]) self.assertEqual(b"PASS", resp.read()) + def test_directory(self): + route = ("GET", "/defaultpy", wptserve.handlers.python_script_handler) + self.server.router.register(*route) + resp = self.request("/defaultpy") + self.assertEqual(200, resp.getcode()) + self.assertEqual("text/plain", resp.info()["Content-Type"]) + # default.py returns "default", default.sub.py returns "default.sub". + # Because this should find the first matching default*.py file + # lexicographically sorted, this should have gotten "default". + self.assertEqual(b"default", resp.read()) + def test_no_main(self): with pytest.raises(HTTPError) as cm: self.request("/no_main.py") @@ -325,6 +336,15 @@ class TestAsIsHandler(TestUsingServer): self.assertEqual(b"Content", resp.read()) #Add a check that the response is actually sane + def test_directory_fails(self): + route = ("GET", "/subdir", wptserve.handlers.as_is_handler) + self.server.router.register(*route) + with pytest.raises(HTTPError) as cm: + self.request("/subdir") + + assert cm.value.code == 500 + del cm + class TestH2Handler(TestUsingH2Server): def test_handle_headers(self): @@ -410,6 +430,12 @@ class TestWindowHandler(TestWrapperHandlerUsingServer): self.run_wrapper_test('foo.window.html', 'text/html', serve.WindowHandler) +class TestWindowModulesHandler(TestWrapperHandlerUsingServer): + dummy_files = {'foo.any.js': b'// META: global=window-module\n'} + + def test_any_window_module_html(self): + self.run_wrapper_test('foo.any.window-module.html', + 'text/html', serve.WindowModulesHandler) class TestAnyHtmlHandler(TestWrapperHandlerUsingServer): dummy_files = {'foo.any.js': b'', diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/test_pipes.py b/testing/web-platform/tests/tools/wptserve/tests/functional/test_pipes.py index beb124d1db..c11577acb5 100644 --- a/testing/web-platform/tests/tools/wptserve/tests/functional/test_pipes.py +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/test_pipes.py @@ -232,6 +232,12 @@ class TestPipesWithVariousHandlers(TestUsingServer): resp = self.request("/document.txt", query="pipe=gzip") self.assertEqual(resp.getcode(), 200) + def test_sub_default_py(self): + route = ("GET", "/defaultsubpy", wptserve.handlers.python_script_handler) + self.server.router.register(*route) + resp = self.request("/defaultsubpy") + self.assertEqual(b"localhost", resp.read()) + if __name__ == '__main__': unittest.main() diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/test_server.py b/testing/web-platform/tests/tools/wptserve/tests/functional/test_server.py index 939396ddee..39ef5889be 100644 --- a/testing/web-platform/tests/tools/wptserve/tests/functional/test_server.py +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/test_server.py @@ -1,10 +1,15 @@ +import os +import socket +import ssl import unittest +from urllib.error import HTTPError import pytest -from urllib.error import HTTPError + +from localpaths import repo_root wptserve = pytest.importorskip("wptserve") -from .base import TestUsingServer, TestUsingH2Server +from .base import TestUsingH2Server, TestUsingServer, doc_root class TestFileHandler(TestUsingServer): @@ -60,6 +65,65 @@ class TestRequestHandler(TestUsingServer): self.assertEqual(200, resp.getcode()) +class TestH1TLSHandshake(TestUsingServer): + def setUp(self): + self.server = wptserve.server.WebTestHttpd( + host="localhost", + port=0, + use_ssl=True, + key_file=os.path.join(repo_root, "tools", "certs", "web-platform.test.key"), + certificate=os.path.join( + repo_root, "tools", "certs", "web-platform.test.pem" + ), + doc_root=doc_root, + ) + self.server.start() + + def test_no_handshake(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.load_verify_locations( + os.path.join(repo_root, "tools", "certs", "cacert.pem") + ) + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s_no_handshake: + s_no_handshake.connect(("localhost", self.server.port)) + # Note: this socket is left open, notably not sending the TLS handshake. + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as sock: + sock.settimeout(10) + with context.wrap_socket( + sock, + do_handshake_on_connect=False, + server_hostname="web-platform.test", + ) as ssock: + ssock.connect(("localhost", self.server.port)) + ssock.do_handshake() + # The pass condition here is essentially "don't raise TimeoutError". + + +class TestH2TLSHandshake(TestUsingH2Server): + def test_no_handshake(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.load_verify_locations( + os.path.join(repo_root, "tools", "certs", "cacert.pem") + ) + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s_no_handshake: + s_no_handshake.connect(("localhost", self.server.port)) + # Note: this socket is left open, notably not sending the TLS handshake. + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as sock: + sock.settimeout(10) + with context.wrap_socket( + sock, + do_handshake_on_connect=False, + server_hostname="web-platform.test", + ) as ssock: + ssock.connect(("localhost", self.server.port)) + ssock.do_handshake() + # The pass condition here is essentially "don't raise TimeoutError". + + class TestH2Version(TestUsingH2Server): # The purpose of this test is to ensure that all TestUsingH2Server tests # actually end up using HTTP/2, in case there's any protocol negotiation. diff --git a/testing/web-platform/tests/tools/wptserve/wptserve/handlers.py b/testing/web-platform/tests/tools/wptserve/wptserve/handlers.py index 6d79230a32..62faf47d64 100644 --- a/testing/web-platform/tests/tools/wptserve/wptserve/handlers.py +++ b/testing/web-platform/tests/tools/wptserve/wptserve/handlers.py @@ -2,6 +2,7 @@ import json import os +import pathlib from collections import defaultdict from urllib.parse import quote, unquote, urljoin @@ -289,6 +290,10 @@ class PythonScriptHandler: """ This loads the requested python file as an environ variable. + If the requested file is a directory, this instead loads the first + lexicographically sorted file found in that directory that matches + "default*.py". + Once the environ is loaded, the passed `func` is run with this loaded environ. :param request: The request object @@ -298,6 +303,14 @@ class PythonScriptHandler: """ path = filesystem_path(self.base_path, request, self.url_base) + # Find a default Python file if the specified path is a directory + if os.path.isdir(path): + default_py_files = sorted(list(filter( + pathlib.Path.is_file, + pathlib.Path(path).glob("default*.py")))) + if default_py_files: + path = str(default_py_files[0]) + try: environ = {"__file__": path} with open(path, 'rb') as f: @@ -416,6 +429,9 @@ class AsIsHandler: def __call__(self, request, response): path = filesystem_path(self.base_path, request, self.url_base) + if os.path.isdir(path): + raise HTTPException( + 500, "AsIsHandler cannot process directory, %s" % path) try: with open(path, 'rb') as f: diff --git a/testing/web-platform/tests/tools/wptserve/wptserve/server.py b/testing/web-platform/tests/tools/wptserve/wptserve/server.py index e1772d00b1..8ce36201ee 100644 --- a/testing/web-platform/tests/tools/wptserve/wptserve/server.py +++ b/testing/web-platform/tests/tools/wptserve/wptserve/server.py @@ -130,7 +130,9 @@ class RequestRewriter: class WebTestServer(http.server.ThreadingHTTPServer): allow_reuse_address = True - acceptable_errors = (errno.EPIPE, errno.ECONNABORTED) + # Older versions of Python might throw `OSError: [Errno 0] Error` + # instead of `SSLEOFError`. + acceptable_errors = (errno.EPIPE, errno.ECONNABORTED, 0) request_queue_size = 2000 # Ensure that we don't hang on shutdown waiting for requests @@ -214,14 +216,21 @@ class WebTestServer(http.server.ThreadingHTTPServer): ssl_context.load_cert_chain(keyfile=self.key_file, certfile=self.certificate) ssl_context.set_alpn_protocols(['h2']) self.socket = ssl_context.wrap_socket(self.socket, + do_handshake_on_connect=False, server_side=True) else: ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) ssl_context.load_cert_chain(keyfile=self.key_file, certfile=self.certificate) self.socket = ssl_context.wrap_socket(self.socket, + do_handshake_on_connect=False, server_side=True) + def finish_request(self, request, client_address): + if isinstance(self.socket, ssl.SSLSocket): + request.do_handshake() + super().finish_request(request, client_address) + def handle_error(self, request, client_address): error = sys.exc_info()[1] @@ -229,7 +238,11 @@ class WebTestServer(http.server.ThreadingHTTPServer): isinstance(error.args, tuple) and error.args[0] in self.acceptable_errors) or (isinstance(error, IOError) and - error.errno in self.acceptable_errors)): + error.errno in self.acceptable_errors) or + # `SSLEOFError` and `SSLError` may occur when a client + # (e.g., wptrunner's `TestEnvironment`) tests for connectivity + # but doesn't perform the handshake. + isinstance(error, ssl.SSLEOFError) or isinstance(error, ssl.SSLError)): pass # remote hang up before the result is sent else: msg = traceback.format_exc() |